I run the producer, it generates N messages, i see them on the dashboard.
When I run a receiver it receive all messages from the queue and the queue is an empty.
static void Receive(string QueName)
{
ConnectionFactory connectionFactory = new ConnectionFactory
{
HostName = HostName,
UserName = UserName,
Password = Password,
};
var connection = connectionFactory.CreateConnection();
var channel = connection.CreateModel();
channel.BasicQos(0, 1, false);
MessageReceiver messageReceiver = new MessageReceiver(channel);
channel.BasicConsume(QueName, false, messageReceiver);
Console.WriteLine(" Press [enter] to exit.");
Console.ReadLine();
}
// Receiver
public class MessageReceiver : DefaultBasicConsumer
{
private readonly IModel _channel;
public MessageReceiver(IModel channel)
{
_channel = channel;
}
public override void HandleBasicDeliver(string consumerTag, ulong deliveryTag, bool redelivered, string exchange, string routingKey, IBasicProperties properties, ReadOnlyMemory<byte> body)
{
Console.WriteLine($"------------------------------");
Console.WriteLine($"Consuming Message");
Console.WriteLine(string.Concat("Message received from the exchange ", exchange));
Console.WriteLine(string.Concat("Consumer tag: ", consumerTag));
Console.WriteLine(string.Concat("Delivery tag: ", deliveryTag));
Console.WriteLine(string.Concat("Routing tag: ", routingKey));
//Console.WriteLine(string.Concat("Message: ", Encoding.UTF8.GetString(body)));
var message = Encoding.UTF8.GetString(body.ToArray());
Console.WriteLine(string.Concat("Message: ", message));
Console.WriteLine($"------------------------------");
_channel.BasicAck(deliveryTag, false);
}
}
I need to have multiple producers which generate messages to the same queue.
And multiple customers receive messages from the queue. And messages will be deleted by queue TTL.
But now the 1st receiver gets all messages from the queue.
How can I do this?
The best solution is : every client should have its own queue, may be with TTL, may be with expiration parameter.
We use “exchange” here, just to show the exchange mechanics in same sample, it’s not really needed for the task (check Worker2 project, it works with another queue, which is binded to the same exchange):
channel.ExchangeDeclare(exchange: “logs”, type: ExchangeType.Fanout);
Full sample of consumption
using RabbitMQ.Client;
using RabbitMQ.Client.Events;
using System.Text;
using System.Timers;
class Worker
{
public static void Main()
{
// Test of timer handler
System.Timers.TimeraTimer = new System.Timers.Timer();
aTimer.Elapsed += new ElapsedEventHandler((source, e)
=> Console.Write("Timer Test"));
aTimer.Interval=3000;
// Test timer
// aTimer.Enabled = true;
var factory = new ConnectionFactory()
{
HostName = "localhost", UserName="user", Password="password",
// DispatchConsumersAsync = true
};
var connection = factory.CreateConnection();
// Add multiple consumers, so that queue can be processed "in
// parallel"
for (int i=1; i<10; i++)
{
var j=i;
var channel = connection.CreateModel();
channel.ExchangeDeclare(exchange: "logs", type:
ExchangeType.Fanout);
var queueName=channel.QueueDeclare("test1", durable: true,
autoDelete: false, exclusive: false);
// take 1 message per consumer
channel.BasicQos(0, 1, false);
channel.QueueBind(queue: queueName,
exchange: "logs",
routingKey: "");
Console.WriteLine($" [*] Waiting for messages in {j}");
var consumer = new EventingBasicConsumer(channel);
consumer. Received+= (model, ea) =>
{
byte[] body = ea.Body.ToArray();
var message = Encoding.UTF8.GetString(body);
Console.WriteLine($" [x] Received in {j} -> {message} at
{DateTime.Now}");
// Thread.Sleep(dots * 1000);
// await Task.Delay(3000);
Thread.Sleep(10000);
// async works too
if (j==5)
{
// Test special case of returning item to queue: in
// this case we received the message, but did not process
// it because of some reason.
// QOS is 1, so our consumer is already full. We need
// to return the message to the queue, so that another
// consumer can work with it
Console.WriteLine($"[-] CANT PROCESS {j} consumer!
Error with -> {message}");
channel.BasicNack(deliveryTag: ea.DeliveryTag,
multiple: false, true);
}
else
{
Console.WriteLine($" [x] Done {j} -> {message} at
{DateTime.Now}");
// here channel could also be accessed as
((EventingBasicConsumer)sender).Model
channel.BasicAck(deliveryTag: ea.DeliveryTag,
multiple: false);
}
};
channel.BasicConsume(queue: queueName, autoAck: false,
consumer: consumer);
}
Console.WriteLine(" Press [enter] to exit.");
Console.ReadLine();
}
}
full example in link
Related
I have one queue-publisher and one hundred queue-subscribers. Subscribers bind to one queue. The Rabbit`s guide said that to each queue Rabbit creates single thread.
I want to send all messages to subscribers through one queue and save all unsended messages in the same queue if destination subscriber is offline.
I have two solutions:
I can send all messages through one queue to all subscribers by declaring exchange type "Direct" and bind to one QueueName. But if routingKey in publishing message not equals queue name, it doesnt save in queue, if direct consumer is offline.
suscriber`s code
static void Main(string[] args)
{
List<string> serevities1 = new List<string>() { "qwerty.red" };
List<string> serevities2 = new List<string>() { "asdfgh.green" };
string exchange = "topic_logs";
//string direction = ExchangeType.Topic;
string direction = ExchangeType.Direct;
var consumer1 = new MqDll.MqConsumer(exchange, direction, serevities1);
MqDll.MqConsumer consumer2;
Task.Run(() => {
Thread.Sleep(10000);
consumer2 = new MqDll.MqConsumer(exchange, direction, serevities2);
});
Console.ReadLine();
}
public class MqConsumer
{
private IConnection connection;
private IModel channel;
//ExchangeType.Fanout
public MqConsumer(string exchange, string direction, List<string> severities = null)
{
var factory = new ConnectionFactory() { HostName = "localhost" };
connection = factory.CreateConnection();
channel = connection.CreateModel();
channel.ExchangeDeclare(exchange, direction);
string queueName = "task_queue";
// queueName= channel.QueueDeclare().QueueName;
channel.QueueDeclare(queue: queueName,
durable: true,
exclusive: false,
autoDelete: false,
arguments: null);
Bind(queueName, exchange, severities);
var consumer = new EventingBasicConsumer(channel);
consumer.Received += MsgReceived;
channel.BasicConsume(queue: queueName, autoAck: true, consumer: consumer);
Console.WriteLine("Consumer created");
}
~MqConsumer()
{
channel.Close();
connection.Close();
}
private void Bind(string queuename, string exchange, List<string> severities)
{
if (severities != null)
{
severities.ForEach(x =>
{
channel.QueueBind(queue: queuename,
exchange: exchange,
routingKey: x);
});
}
else
{
channel.QueueBind(queue: queuename,
exchange: exchange,
routingKey: "");
}
}
private void MsgReceived(object model, BasicDeliverEventArgs ea)
{
var body = ea.Body.ToArray();
var message = Encoding.UTF8.GetString(body);
var rkey = ea.RoutingKey;
Console.WriteLine($" [x] Received {message} {rkey}");
Console.WriteLine($" [x] Done {DateTime.Now} | {this.GetHashCode()}");
}
}
publisher`s code
static void Main(string[] args)
{
List<string> serevities = new List<string>() { "qwerty.red", "asdfgh.green" };
string exchange = "topic_logs";
//string direction = ExchangeType.Topic;
string direction = ExchangeType.Direct;
var publisher = new MqDll.MqPublisher(exchange, direction);
Console.WriteLine("Publisher created");
var msg = Console.ReadLine();
while (msg != "q")
{
serevities.ForEach(x =>
{
publisher.Publish("SomeMsg..", "topic_logs", x);
});
msg = Console.ReadLine();
}
}
public class MqPublisher
{
private IConnection connection;
private IModel channel;
public MqPublisher(string exchange, string type)
{
var factory = new ConnectionFactory() { HostName = "localhost" };
connection = factory.CreateConnection();
channel = connection.CreateModel();
channel.QueueDeclare(queue: "task_queue",
durable: true,
exclusive: false,
autoDelete: false,
arguments: null);
channel.ExchangeDeclare(exchange, type);
}
~MqPublisher()
{
channel.Close();
connection.Close();
}
public void Publish(string msg, string exchange = "logs", string severity = "", bool isPersistent = true)
{
var properties = channel.CreateBasicProperties();
properties.Persistent = isPersistent;
channel.BasicPublish(exchange: exchange,
routingKey: severity,
basicProperties: properties,
body: Encoding.UTF8.GetBytes(msg));
Console.WriteLine(" [x] Sent {0}", msg);
}
}
Or I can create queues to each subscriber and remove binding. Source code is simple and equals code in giude.
Is there a way to combine these two solutions and make one queue to all subscribers, bind subscribers to unique routing key and save messages if direct subscriber (bound to direct routing key) is offline?
Have you considered maybe a DB for users that are not online..
Queue => consumer is offline => send to DB
Consumer is online => Publisher checks db for any messages that might have been missed => then directs them to Consumber
I have different message types on the queue and I'm trying to create multiple C# consumers that each just take a certain type off the queue.
Apparently, I'm supposed to be able to specify the routing key, like shown here, to get just those messages:
channel.QueueBind(queue: queueName, exchange: "SandboxEventBus", routingKey: "MessageEvent");
No matter why I try, I get all of the messages from the queue and not just the ones with that routing key. What am I missing?
This is the class:
public static class RabbitReceiver
{
public static void PullFromQueue(string caller, int sleepTimeInMilliseconds)
{
string hostName = "localhost";
string queueName = "Sandbox.v1";
var factory = new ConnectionFactory { HostName = hostName, UserName = "guest", Password = "guest", Port = 6003 };
var connection = factory.CreateConnection();
var channel = connection.CreateModel();
var arguments = new Dictionary<string, object>();
arguments.Add("x-message-ttl", 864000000); // Not sure why I have to specify this. Got an exception if I didn't.
//arguments.Add("topic", "MessageEvent");
channel.BasicQos(0, 1, false);
// Enabling these lines doesn't change anything. It was part of my effort of trying different things.
//channel.ExchangeDeclare("SandboxEventBus", "topic", durable: true); // topic is case-sensitive
//channel.QueueDeclare(queue: queueName, durable: true, exclusive: false, autoDelete: false, arguments: arguments);
channel.QueueBind(queue: queueName, exchange: "SandboxEventBus", routingKey: "MessageEvent");
var consumer = new EventingBasicConsumer(channel);
consumer.Received += (model, eventArgs) =>
{
var body = eventArgs.Body.ToArray();
var message = Encoding.UTF8.GetString(body);
Console.WriteLine($"Message: {message}");
Console.WriteLine($"{caller} - Pausing {sleepTimeInMilliseconds} ms before getting next message (so we can watch the Rabbit dashboard update).");
Thread.Sleep(sleepTimeInMilliseconds);
// This should probably go in a finally block.
channel.BasicAck(eventArgs.DeliveryTag, false);
};
channel.BasicConsume(queue: queueName, autoAck: false, consumer: consumer);
}
}
Here is the publishing code. I'm simply publishing two different message types.
public class MessagePublisherService : BackgroundService
{
private readonly ICapPublisher _capBus;
public MessagePublisherService(ICapPublisher capPublisher)
{
_capBus = capPublisher;
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
// This publishes messages. To get the messages off the queue, run the RabbitMQ project in this solution.
try
{
// To have a different event, add this.
await _capBus.PublishAsync(nameof(SnuhEvent), new SnuhEvent());
Console.WriteLine($"Published message {nameof(SnuhEvent)}.");
for (int i = 1; i <= 10; i++)
{
//_capBus.Publish(nameof(MessageEvent), new MessageEvent(i));
await _capBus.PublishAsync(nameof(MessageEvent), new MessageEvent(i));
Console.WriteLine($"Published message {i}.");
}
}
catch (Exception ex)
{
Console.WriteLine("In catch block of publisher.");
Console.WriteLine(ex);
}
}
}
And here is the queue. Notice each message type shows as a different routing key.
Here is the RMQ code in Startup.cs:
public void ConfigureServices(IServiceCollection services)
{
services.AddCap(x =>
{
x.UseSqlServer("Server=localhost; Database=Sandbox; Integrated Security=true;");
x.DefaultGroup = "Sandbox";
x.FailedRetryCount = 3;
x.FailedRetryInterval = 2; // CAP will invoke the subscriber again if not finished in N seconds.
x.UseRabbitMQ(conf =>
{
conf.ExchangeName = "SandboxEventBus";
conf.ConnectionFactoryOptions = x =>
{
x.Uri = BuildRabbitUri();
};
});
});
services.AddHostedService<MessagePublisherService>();
}
public static Uri BuildRabbitUri()
{
string protocol = "amqp";
string userName = "guest";
string password = "guest";
string host = "localhost";
string port = "6003";
return new Uri(protocol + "://" + userName + ":" + password + "#" + host + ":" + port);
}
Using a topic exchange, I would like to have a publish/subscribe messaging pattern with the following features:
Have "publisher confirms" implemented.
Have the consumer acknowledge each message as well once it has processed it.
Use routing keys to route messages to one or more consumers.
Have persistent consumer queues so if the consumer application is down temporarily, it can pick up messages from its queue when it comes back up.
So I have created 2 console applications (Send and Receive) to test the above.
Send
static void Main(string[] args)
{
Console.WriteLine(" Type [exit] to exit.");
Publisher publisher = new Publisher();
do
{
var userInput = Console.ReadLine();
if (userInput == "exit")
{
break;
}
publisher.SendMessageToBroker("localhost", "main", "user.update", userInput);
} while (true);
}
Publisher
public class Publisher
{
const string ExchangeType = "topic";
Dictionary<ulong, string> unConfirmedMessageTags = new Dictionary<ulong, string>();
public void SendMessageToBroker(string host, string exchangeName, string routingKey, string message)
{
var factory = new ConnectionFactory() { HostName = host };
using (var connection = factory.CreateConnection())
using (var channel = connection.CreateModel())
{
channel.BasicAcks += (sender, ea) => OnBasicAcks(ea.Multiple, ea.DeliveryTag);
channel.BasicNacks += (sender, ea) => OnBasicNacks(ea.Multiple, ea.DeliveryTag);
channel.ConfirmSelect();
channel.ExchangeDeclare(exchangeName, ExchangeType);
var body = Encoding.UTF8.GetBytes(message);
var properties = channel.CreateBasicProperties();
properties.Persistent = true;
unConfirmedMessageTags.TryAdd(channel.NextPublishSeqNo, message);
channel.BasicPublish(exchange: exchangeName,
routingKey: routingKey,
basicProperties: properties,
body: body);
Console.WriteLine(" [x] Sent {0}", message);
}
}
private void OnBasicNacks(bool multiple, ulong deliveryTag)
{
if (multiple)
{
Console.WriteLine("Messages with delivery tag LESS THAN {0} have been LOST and must be resent.", deliveryTag);
}
else
{
Console.WriteLine("Message with delivery tag {0} has been LOST and must be resent.", deliveryTag);
}
}
private void OnBasicAcks(bool multiple, ulong deliveryTag)
{
if (multiple)
{
var confirmed = unConfirmedMessageTags.Where(k => k.Key <= deliveryTag);
foreach (var entry in confirmed)
{
unConfirmedMessageTags.Remove(entry.Key);
Console.WriteLine("Message with delivery tag {0} has been confirmed and deleted.", entry.Key);
}
}
else
{
unConfirmedMessageTags.Remove(deliveryTag);
Console.WriteLine("Message with delivery tag {0} has been confirmed and deleted.", deliveryTag);
}
}
}
}
Receive
static void Main(string[] args)
{
const string ExchangeName = "main";
const string QueueName = "q1";
const string ExchangeType = "topic";
const string RoutingKey = "user.update";
var factory = new ConnectionFactory() { HostName = "localhost" };
using(var connection = factory.CreateConnection())
using(var channel = connection.CreateModel())
{
channel.ExchangeDeclare(ExchangeName, ExchangeType);
channel.QueueDeclare(queue: QueueName,
durable: true,
autoDelete: false,
exclusive: false,
arguments: null);
channel.QueueBind(QueueName, ExchangeName, RoutingKey);
//channel.BasicQos(prefetchSize: 0, prefetchCount: 1, global: false);
var consumer = new EventingBasicConsumer(channel);
consumer.Received += (model, ea) => Basic_Ack(channel, ea.DeliveryTag, ea.Body);
channel.BasicConsume(queue: QueueName, autoAck: false, consumer: consumer);
Console.WriteLine(" Press [enter] to exit.");
Console.ReadLine();
}
}
private static void Basic_Ack(IModel channel, ulong deliveryTag, ReadOnlyMemory<byte> body)
{
var message = Encoding.UTF8.GetString(body.ToArray());
Console.WriteLine(" [x] Received {0}", message);
Thread.Sleep(2000);
channel.BasicAck(deliveryTag: deliveryTag, multiple: false);
Console.WriteLine(" [x] Processed {0}", message);
}
}
The problem is the OnBasicAcks in my Send program only gets called once for the first message.
For anyone else out there who may hit this issue, I was opening a connection and channel (virtual connection) for every publish which is discouraged:
"Connections are meant to be long-lived. Opening a connection for every operation (e.g. publishing a message) would be very inefficient and is highly discouraged."
Also see here:
"Publisher confirms are enabled at the channel level with the ConfirmSelect method... This method must be called on every channel that you expect to use publisher confirms. Confirms should be enabled just once, not for every message published."
Switching to using a long-lived connection solved the issue for me.
I'm trying to convert my RabbitMQ micro-service to a windows service. I have used TopShelf for the conversion. My RabbitMQ micro-service works perfectly fine on its own but when I run it as a service it no longer receives messages. In my public static void Main(string[] args) I have:
HostFactory.Run(host =>
{
host.Service<PersonService>(s =>
{
s.ConstructUsing(name => new PersonService());
s.WhenStarted(tc => tc.Start());
s.WhenStopped(tc => tc.Stop());
});
host.SetDescription("Windows service that provides database access totables.");
host.SetDisplayName("Service");
host.SetServiceName("Service");
});
}
Then in my PersonService class I have
public void Start() {
ConsumeMessage();
}
And finally my ConsumeMessage function:
private static void ConsumeMessage() {
MessagingConfig.SetInstance(new MessagingConstants());
IMessageFactory pmfInst = MessageFactory.Instance;
//message worker
var factory = new ConnectionFactory() {
HostName = MessagingConfig.Instance.GetBrokerHostName(),
UserName = MessagingConfig.Instance.GetBrokerUserName(),
Password = MessagingConfig.Instance.GetBrokerPassword()
};
var connection = factory.CreateConnection();
using (var channel = connection.CreateModel()) {
channel.QueueDeclare(queue: MessagingConfig.Instance.GetServiceQueueName(),
durable: true,
exclusive: false,
autoDelete: false,
arguments: null);
channel.BasicQos(0, 1, false);
var consumer = new EventingBasicConsumer(channel);
channel.BasicConsume(queue: MessagingConfig.Instance.GetServiceQueueName(),
noAck: false,
consumer: consumer);
Console.WriteLine("Service.");
Console.WriteLine(" [x] Awaiting RPC requests");
// Code Below Is Not Executed In Service
consumer.Received += (model, ea) => {
string response = null;
var body = ea.Body;
var props = ea.BasicProperties;
var replyProps = channel.CreateBasicProperties();
replyProps.CorrelationId = props.CorrelationId;
string receivedMessage = null;
try {
receivedMessage = Encoding.UTF8.GetString(body);
response = ProcessMessage(receivedMessage);
}
catch (Exception e) {
// Received message is not valid.
WinLogger.Log.Error(
"Errror Processing Message: " + receivedMessage + " :" + e.Message);
response = "";
}
finally {
var responseBytes = Encoding.UTF8.GetBytes(response);
channel.BasicPublish(exchange: "", routingKey: props.ReplyTo,
basicProperties: replyProps, body: responseBytes);
channel.BasicAck(deliveryTag: ea.DeliveryTag,
multiple: false);
}
};
Console.ReadLine();
}
Looking at A similar SO question it looks like it has something to do with the return the Windows Service is wanting, but I'm not sure of how to call ConsumeMessage so consumer.Received += (model, ea) => {...}; is executed.
EDIT: It looks like my blocking mechanism Console.ReadLine(); is ignored by the service so it just continues on and disposes of the message consumer. So how do I block there for messages to be received?
You code uses using construct, which means when your OnStart method returns, your channel will actually be disposed. The docs suggest to do your initialization on OnStart, so create your channel and consumer there, but don't use using:
this.connection = factory.CreateConnection();
this.channel = connection.CreateModel();
this.consumer = new EventingBasicConsumer(this.channel);
Then those objects will continue to exist after OnStart method is finished. You should dispose of them in the OnStop method.
I am learning RabbitMq with .NET. According to the tutorial, simplest implementation of consumer looks like :
public class Receive
{
public static void Main()
{
var factory = new ConnectionFactory() { HostName = "localhost" };
using(var connection = factory.CreateConnection())
using(var channel = connection.CreateModel())
{
channel.QueueDeclare(queue: "hello",
durable: false,
exclusive: false,
autoDelete: false,
arguments: null);
var consumer = new EventingBasicConsumer(channel);
consumer.Received += (model, ea) =>
{
var body = ea.Body;
var message = Encoding.UTF8.GetString(body);
Console.WriteLine(" [x] Received {0}", message);
};
channel.BasicConsume(queue: "hello",
noAck: true,
consumer: consumer);
Console.WriteLine(" Press [enter] to exit.");
Console.ReadLine();
}
}
}
And it works correctly. However, I wanted to refactor it: let define functionality of receiver in separate method. It looks like :
public class Recieve
{
private ConnectionFactory factory;
public void ConsumeSimpleMessage(string queueName = "default")
{
using(var connection = factory.CreateConnection())
{
using(var channel = connection.CreateModel())
{
channel.QueueDeclare(queue: queueName, durable: false,
exclusive: false, autoDelete: false, arguments: null);
var consumer = new EventingBasicConsumer(channel);
consumer.Received += (model, ea) =>
{
var body = ea.Body;
var message = Encoding.UTF8.GetString(body);
Console.WriteLine(message);
};
channel.BasicConsume(queue: queueName,
noAck: true,
consumer: consumer);
}
}
}
public Recieve(string hostName = "localhost")
{
factory = new ConnectionFactory() { HostName = hostName };
}
}
And when I try to call this method in Main():
class Program
{
static void Main(string[] args)
{
Recieve reciever = new Recieve();
reciever.ConsumeSimpleMessage();
Console.ReadLine();
}
}
here it doesn't work. It show nothing. However, messages will be deleted, meaning that they were recieved. Why does it happen? Is there anything I didn't know about Event Handing?
Try to see if it works without doing using, Or if you want to use the using statement keep the Console.Read() inside the using statement and see if that works. You can keep the connection and channel open and don't have to close it manually.
If you really want to debug then you can put the breakpoint on the consumer.Receive and see if you can see the message staying unacked. That way you know when the message will be deleted.
Also I usually recommend using tracer for rabbitmq as it logs all the messages coming in the server which makes it easy to trace.