I have shared library as Bus, and I am trying to recieve messages from rabbitmq but ConsumerOnReceived never get triggered.
namespace Bus
{
public class MessageListener
{
private static IConnection _connection;
private static IModel _channel;
public void Start(string hostName, int port, string queueName)
{
var factory = new ConnectionFactory() { HostName = hostName, Port = port };
using (var connection = factory.CreateConnection())
using (var channel = connection.CreateModel())
{
channel.QueueDeclare(queue: queueName,
durable: false,
exclusive: false,
autoDelete: false,
arguments: null);
var consumer = new EventingBasicConsumer(channel);
consumer.Received += ConsumerOnReceived;
channel.BasicConsume(queue: queueName,
noAck: true,
consumer: consumer);
}
}
public static void Stop()
{
_channel.Close(200, "Goodbye");
_connection.Close();
}
public virtual void ConsumerOnReceived(object sender, BasicDeliverEventArgs ea)
{
var body = ea.Body;
var message = Encoding.UTF8.GetString(body);
}
}
public static class MessageSender
{
public static void Send(string hostName, int port, string queueName, string message)
{
var factory = new ConnectionFactory() { HostName = hostName, Port = port };
using (var connection = factory.CreateConnection())
using (var channel = connection.CreateModel())
{
channel.QueueDeclare(queue: queueName, durable: false, exclusive: false, autoDelete: false, arguments: null);
var body = Encoding.UTF8.GetBytes(message.ToString());
channel.BasicPublish(exchange: "", routingKey: queueName, basicProperties: null, body: body);
}
}
}
}
Core
namespace Core
{
class Program
{
static void Main(string[] args)
{
new MessageListener().Start("localhost", 5672, "MakePayment");
Console.WriteLine("Core Service");
string line = Console.ReadLine();
}
}
}
namespace Core
{
public class MessageListener : Bus.MessageListener
{
public override void ConsumerOnReceived(object sender, BasicDeliverEventArgs ea)
{
var body = ea.Body;
var message = Encoding.UTF8.GetString(body);
}
}
}
The problem is here
channel.BasicConsume(queue: queueName, noAck: true, consumer: consumer);
However, BasicConsume is not a blocking method therefore when you call Start you create a connection and a channel but then it gets disposed immediately.
The following is NOT a solution but you can confirm by doing the following:
channel.BasicConsume(queue: queueName, noAck: true, consumer: consumer);
Console.ReadKey();//←Added Line
Your program will work this way.
This is my proposed solution. Please take notice that _channel.BasicConsume(queue: queueName, noAck: true, consumer: consumer); will start on another thread so you do not need to use while(...)
using System;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using RabbitMQ.Client;
using RabbitMQ.Client.Events;
namespace Bus {
public abstract class BaseMessageListener {
private static IModel _channel;
private static IConnection _connection;
public abstract void ConsumerOnReceived(object sender, BasicDeliverEventArgs ea);
public void Start(string hostName, int port, string queueName) {
var factory = new ConnectionFactory() { HostName = hostName, Port = port };
_connection = factory.CreateConnection();
_channel = _connection.CreateModel();
_channel.QueueDeclare(queue: queueName, durable: false, exclusive: false, autoDelete: false);
var consumer = new EventingBasicConsumer(_channel);
consumer.Received += ConsumerOnReceived;
_channel.BasicConsume(queue: queueName, noAck: true, consumer: consumer);//This will start another thread!
}
public void Stop() {
_channel.Close(200, "Goodbye");
_connection.Close();
}
}
}
namespace StackOverfFLow.RabbitMQSolution {
using Bus;
public class MessageListener : BaseMessageListener {
public override void ConsumerOnReceived(object sender, BasicDeliverEventArgs ea) {
var body = ea.Body;
var message = Encoding.UTF8.GetString(body);
Console.WriteLine(message);
}
}
internal class Program {
private static void Main(string[] args) {
var listener = new MessageListener();
listener.Start("localhost", 5672, "MakePayment");
Console.WriteLine("Core Service Started!");
Console.ReadKey();
listener.Stop();
}
}
}
Related
I need to receive messages from a queue broker and, after some processing, store them in a database. In order to save records in blocks, i use a BatchBlock.
All this works through the BackgroundService
Code:
public class Worker : BackgroundService
{
private readonly IServiceProvider _serviceProvider;
private readonly BatchBlock<Comment> _batchBlock;
private readonly ActionBlock<Comment[]> _importer;
private readonly ConnectionFactory _factory;
private readonly IModel _channel;
private const string _queueName = "Comment";
private static int count;
public Worker(IConfiguration configuration, IServiceProvider serviceProvider)
{
_serviceProvider = serviceProvider;
_factory = new ConnectionFactory
{ Uri = new Uri(configuration.GetSection("RabbitMqConnection").Value), DispatchConsumersAsync = true };
var connection = _factory.CreateConnection();
_channel = connection.CreateModel();
_channel.QueueDeclare(queue: _queueName, durable: true, exclusive: false, autoDelete: false, arguments: null);
_batchBlock = new BatchBlock<Comment>(50);
_importer = new ActionBlock<Comment[]>(SaveCommentsToDb);
_batchBlock.LinkTo(_importer, new DataflowLinkOptions { PropagateCompletion = true });
}
protected override Task ExecuteAsync(CancellationToken stoppingToken)
{
stoppingToken.ThrowIfCancellationRequested();
var consumer = new AsyncEventingBasicConsumer(_channel);
consumer.Received += async (_, ea) =>
await StartCommentHandlerGroup(ea);
_channel.BasicConsume(_queueName, false, consumer);
return Task.CompletedTask;
}
private async Task StartCommentHandlerGroup(BasicDeliverEventArgs message)
{
try
{
var content = Encoding.UTF8.GetString(message.Body.ToArray());
var comment = JsonConvert.DeserializeObject<Comment>(content);
await _batchBlock.SendAsync(comment);
}
catch (Exception e)
{
Log.Error($"!!!Handler error!!! {e.Message}{Environment.NewLine} !!!Message!!!{message.Body}");
throw;
}
}
private async Task SaveCommentsToDb(Comment[] comments)
{
using var scope = _serviceProvider.CreateScope();
var dbContext = scope.ServiceProvider.GetRequiredService<Context>();
await dbContext.AddMessagesToDb(comments);
}
}
It works very weird.....
If i set the BatchSize parameter to 10, about 200 records get into the database, if i set it to 2, about 300 hits. I set the BatchSize to 50 and higher and nothing gets into the database.
In this case, the queue is consumed correctly. SendAsync works correctly in the BatchBlock, the Task returns true.
After a certain part of the processing is done, i get the InputCount parameters in the _importer - 0 and the status is IsCompleted - true.
No matter how many new messages from the broker i receive in the future, this does not change the situation in any way and the ActionBlock is not called.
await dbContext.AddMessagesToDb(comments) contains:
foreach (var model in models)
await DbContext.AddAsync(model);
await DbContext.SaveChangesAsync();
I tried adding the BoundedCapacity parameter, but it didn't work.
batchBlock = new BatchBlock<Comment>(10, new GroupingDataflowBlockOptions(){BoundedCapacity = 50});
I used the tutorial from rabbitMQ on RCP in C#
server program.cs
using RabbitMQ.Client;
using RabbitMQ.Client.Events;
using System;
using System.Text;
namespace RabbitMQServerTest
{
internal class Program
{
static void Main(string[] args)
{
var factory = new ConnectionFactory() { HostName = "localhost" };
using (var connection = factory.CreateConnection())
{
using (var channel = connection.CreateModel())
{
channel.QueueDeclare(queue:"RPC_TEST",true,false,false,null);
channel.BasicQos(0,1,false);
var consumer = new EventingBasicConsumer(channel);
channel.BasicConsume(queue: "RPC_TEST",
autoAck: true, consumer: consumer);
Console.WriteLine(" [x] Awaiting RPC requests");
consumer.Received += (model, ea) =>
{
string response = null;
var body = ea.Body.ToArray();
var props = ea.BasicProperties;
var replyProps = channel.CreateBasicProperties();
replyProps.CorrelationId = props.CorrelationId;
try
{
var message = Encoding.UTF8.GetString(body);
int n = int.Parse(message);
Console.WriteLine($" [.] fib({message}) from {replyProps.CorrelationId }");
response = fib(n).ToString();
}
catch (Exception e)
{
Console.WriteLine(" [.] " + e.Message);
response = "";
}
finally
{
var responseBytes = Encoding.UTF8.GetBytes(response);
channel.BasicPublish(exchange: "", routingKey: props.ReplyTo,
basicProperties: replyProps, body: responseBytes);
channel.BasicAck(deliveryTag: ea.DeliveryTag,
multiple: false);
}
};
Console.WriteLine(" Press [enter] to exit.");
Console.ReadLine();
}
}
}
private static int fib(int n)
{
if (n == 0 || n == 1)
{
return n;
}
return fib(n - 1) + fib(n - 2);
}
}
}
client program.cs
internal class Program
{
private static RpcClient m_rpcClient;
static void Main(string[] args)
{
m_rpcClient = new RpcClient();
callFib();
m_rpcClient.Close();
}
static void callFib()
{
Console.WriteLine("Write number:");
var num = Console.ReadLine();
if (num != "q")
{
var response = m_rpcClient.Call("30").Result;
Console.WriteLine(" [.] Got '{0}'", response);
callFib();
}
}
}
client RpcClient.cs
public class RpcClient :IDisposable
{
private readonly IConnection connection;
private readonly IModel channel;
private readonly string replyQueueName;
private readonly EventingBasicConsumer consumer;
private ConcurrentDictionary<string, TaskCompletionSource<string>> m_ActiveQueue = new ConcurrentDictionary<string, TaskCompletionSource<string>>();
private readonly IBasicProperties props;
public RpcClient()
{
var factory = new ConnectionFactory() { HostName = "localhost" };
connection = factory.CreateConnection();
channel = connection.CreateModel();
replyQueueName = channel.QueueDeclare().QueueName;
consumer = new EventingBasicConsumer(channel);
consumer.Received += (model, ea) =>
{
var body = ea.Body.ToArray();
var response = Encoding.UTF8.GetString(body);
if (m_ActiveQueue.TryRemove(ea.BasicProperties.CorrelationId, out
var taskCompletionSource))
{
taskCompletionSource.SetResult(response);
}
};
channel.BasicConsume(
consumer: consumer,
queue: replyQueueName,
autoAck: true);
}
public Task<string> Call(string message)
{
var props = channel.CreateBasicProperties();
var correlationId = Guid.NewGuid().ToString();
props.CorrelationId = correlationId;
props.ReplyTo = replyQueueName;
var taskCompletionSource = new TaskCompletionSource<string>();
var messageBytes = Encoding.UTF8.GetBytes(message);
channel.BasicPublish(
exchange: "",
routingKey: "RPC_TEST",
basicProperties: props,
body: messageBytes);
m_ActiveQueue.TryAdd(correlationId, taskCompletionSource);
return taskCompletionSource.Task;
}
public void Close()
{
try
{
channel?.Close();
connection?.Close();
}
catch (Exception ex)
{
}
}
public void Dispose()
{
Close();
}
}
It's working only with single message from single client
if I'm trying to run few clients , only the first that sent the message get response
the seconds message sends the server doesn't even get to the consumer.Received event in server program.cs
I have two services communicating through RabbitMQ, I want to apply a scoped lifetime per received message.
My consumer:
public class Consumer
{
private IModel channel;
public Consumer(IModel channel)
{
this.channel = channel;
}
public void Consume()
{
var queue = channel.QueueDeclare(queue: MessagingConfigs.QueueName,
durable: false,
exclusive: false,
autoDelete: false,
arguments: null);
var consumer = new EventingBasicConsumer(channel);
consumer.Received += (model, args) => HandleMessageReceived(model, args);
channel.BasicConsume(queue: MessagingConfigs.QueueName, autoAck: false, consumer: consumer);
}
private async Task HandleMessageReceived(object model, BasicDeliverEventArgs args)
{
//do logic
}
}
HandleMessageReceived will be called each time an event Received is fired, how can I open a new scope each time HandleMessageReceived is called?
Found a solution online:
public void ConfigureServices(IServiceCollection services)
{
services.AddScoped<IMyScopedService, MyScopedService>();
var serviceProvider = services.BuildServiceProvider();
var serviceScopeFactory = serviceProvider.GetRequiredService<IServiceScopeFactory>();
IMyScopedService scopedOne;
using (var scope = serviceScopeFactory.CreateScope())
{
scopedOne = scope.ServiceProvider.GetService<Consumer>();
Consumer.Consume();
}
}
I am thinking that if I use the above in startup.cs Consumer will be called once and scope will be opened once in application lifetime, the following might work:
private async Task HandleMessageReceived(object model, BasicDeliverEventArgs args)
{ using (var scope = serviceScopeFactory.CreateScope())
{
//do logic
}
}
but I don't want to include lifetime registration in my consumer code, is there a way in .Net core to open a scope automatically each time a method is called, something that I can use in startup.cs or program.cs?
My application is .net core console application not ASP.NET
Why not wrap the method during attachment?
First, you create wrapper for a method :
public static class ScopeFactoryExtension
{
public static EventHandler<BasicDeliverEventArgs> WrapInScope(this IServiceScopeFactory scopeFactory, Func<IMyScopedService, object, BasicDeliverEventArgs, Task> func)
{
async void InScope(object arg1, BasicDeliverEventArgs arg2)
{
using (var scope = scopeFactory.CreateScope())
{
await func(scope, arg1, arg2);
}
}
return InScope;
}
}
Then wrap whole method :
consumer.Received += scopeFactory.WrapInScope(HandleMessageReceived);
Also need to extend the method :
private async Task HandleMessageReceived(IMyScopedService service, object model, BasicDeliverEventArgs args)
{
//do logic with service
}
I have a microservice (Web API) within an eventdriven architecture receiving messages from RabbitMQ and it is supposed to save them into a PostgreSQL DB using ADO.NET.
Unfortunately, my connection pool (currently set to 50) gets exhausted quite fast, giving me this error message:
The connection pool has been exhausted, either raise MaxPoolSize
My RabbitMQ Consumer is set up like this (Singleton):
public class Listener : RabbitMqConnection
{
public AsyncEventingBasicConsumer _asyncConsumer;
private static readonly SemaphoreSlim AsyncLock = new SemaphoreSlim(1, 1);
public Listener()
{
_asyncConsumer = new AsyncEventingBasicConsumer(_channel);
_asyncConsumer.Received += ConsumerReceived;
}
public async Task ConsumerReceived(object sender, BasicDeliverEventArgs message)
{
await AsyncLock.WaitAsync();
try
{
//Performing logic and saving into database
//....
using (var ctx = ContextFactory.GetContext<PostgreSqlDatabaseContext>(_connectionString))
{
//Creating query with StringBuilder...
await ctx.Database.ExecuteSqlCommandAsync(query.ToString(), parameters);
}
_channel.BasicAck(message.DeliveryTag, false);
}
catch (DecoderFallbackException decoderFallbackException)
{
_logger.LogError($"...");
_channel.BasicNack(message.DeliveryTag, false, false);
}
finally {
AsyncLock.Release();
}
}
}
ContextFactory
internal class ContextFactory
{
public static T GetContext<T>(string sqlConnection) where T : DbContext
{
var optionsBuilder = new DbContextOptionsBuilder<PostgreSqlDatabaseContext>();
optionsBuilder.UseNpgsql(sqlConnection);
return new PostgreSqlDatabaseContext(optionsBuilder.Options) as T;
}
}
RabbitMqConnection:
public abstract class RabbitMQConnection
{
public IModel _channel;
public IBasicProperties _properties;
public AsyncEventingBasicConsumer _asyncConsumer;
public ConnectionFactory _factory;
public ConnectConfiguration _connectConfiguration;
bool isConnected = false;
public void Connect(ConnectConfiguration connectConfiguration)
{
if (!isConnected)
{
_connectConfiguration = connectConfiguration;
CreateFactory(_connectConfiguration);
SetupConfiguration(_connectConfiguration.Exchange);
}
}
private void CreateFactory(ConnectConfiguration config)
{
_factory = new ConnectionFactory
{
AutomaticRecoveryEnabled = true,
DispatchConsumersAsync = true,
UseBackgroundThreadsForIO = true,
RequestedHeartbeat = 15,
HostName = config.Server,
UserName = config.UserName,
Password = config.Password
};
if (!string.IsNullOrWhiteSpace(config.Vhost))
_factory.VirtualHost = config.Vhost;
}
private void SetupConfiguration(string exchange)
{
var connection = _factory.CreateConnection();
_channel = connection.CreateModel();
_properties = _channel.CreateBasicProperties();
_properties.Persistent = true;
_channel.BasicQos(0, 10, false);
_channel.ExchangeDeclare(exchange, "topic", true);
isConnected = true;
}
}
I can´t not understand why I keep getting this error. Isn´t the SemaphoreSlim with WaitAsync() and Release() suppose to prevent the ConsumerReceived method from running the logic?
I have an ASP.NET Core application where I would like to consume RabbitMQ messages.
I have successfully set up the publishers and consumers in command line applications, but I'm not sure how to set it up properly in a web application.
I was thinking of initializing it in Startup.cs, but of course it dies once startup is complete.
How to initialize the consumer in a the right way from a web app?
Use the Singleton pattern for a consumer/listener to preserve it while the application is running. Use the IApplicationLifetime interface to start/stop the consumer on the application start/stop.
public class Startup
{
public void ConfigureServices(IServiceCollection services)
{
services.AddSingleton<RabbitListener>();
}
public void Configure(IApplicationBuilder app)
{
app.UseRabbitListener();
}
}
public static class ApplicationBuilderExtentions
{
//the simplest way to store a single long-living object, just for example.
private static RabbitListener _listener { get; set; }
public static IApplicationBuilder UseRabbitListener(this IApplicationBuilder app)
{
_listener = app.ApplicationServices.GetService<RabbitListener>();
var lifetime = app.ApplicationServices.GetService<IApplicationLifetime>();
lifetime.ApplicationStarted.Register(OnStarted);
//press Ctrl+C to reproduce if your app runs in Kestrel as a console app
lifetime.ApplicationStopping.Register(OnStopping);
return app;
}
private static void OnStarted()
{
_listener.Register();
}
private static void OnStopping()
{
_listener.Deregister();
}
}
You should take care of where your app is hosted. For example, IIS can recycle and stop your code from running.
This pattern can be extended to a pool of listeners.
This is My Listener:
public class RabbitListener
{
ConnectionFactory factory { get; set; }
IConnection connection { get; set; }
IModel channel { get; set; }
public void Register()
{
channel.QueueDeclare(queue: "hello", durable: false, exclusive: false, autoDelete: false, arguments: null);
var consumer = new EventingBasicConsumer(channel);
consumer.Received += (model, ea) =>
{
var body = ea.Body;
var message = Encoding.UTF8.GetString(body);
int m = 0;
};
channel.BasicConsume(queue: "hello", autoAck: true, consumer: consumer);
}
public void Deregister()
{
this.connection.Close();
}
public RabbitListener()
{
this.factory = new ConnectionFactory() { HostName = "localhost" };
this.connection = factory.CreateConnection();
this.channel = connection.CreateModel();
}
}
Another option is Hosted Services.
You can create a HostedService and call a method to register RabbitMq listener.
public interface IConsumerService
{
Task ReadMessgaes();
}
public class ConsumerService : IConsumerService, IDisposable
{
private readonly IModel _model;
private readonly IConnection _connection;
public ConsumerService(IRabbitMqService rabbitMqService)
{
_connection = rabbitMqService.CreateChannel();
_model = _connection.CreateModel();
_model.QueueDeclare(_queueName, durable: true, exclusive: false, autoDelete: false);
_model.ExchangeDeclare("your.exchange.name", ExchangeType.Fanout, durable: true, autoDelete: false);
_model.QueueBind(_queueName, "your.exchange.name", string.Empty);
}
const string _queueName = "your.queue.name";
public async Task ReadMessgaes()
{
var consumer = new AsyncEventingBasicConsumer(_model);
consumer.Received += async (ch, ea) =>
{
var body = ea.Body.ToArray();
var text = System.Text.Encoding.UTF8.GetString(body);
Console.WriteLine(text);
await Task.CompletedTask;
_model.BasicAck(ea.DeliveryTag, false);
};
_model.BasicConsume(_queueName, false, consumer);
await Task.CompletedTask;
}
public void Dispose()
{
if (_model.IsOpen)
_model.Close();
if (_connection.IsOpen)
_connection.Close();
}
}
RabbitMqService:
public interface IRabbitMqService
{
IConnection CreateChannel();
}
public class RabbitMqService : IRabbitMqService
{
private readonly RabbitMqConfiguration _configuration;
public RabbitMqService(IOptions<RabbitMqConfiguration> options)
{
_configuration = options.Value;
}
public IConnection CreateChannel()
{
ConnectionFactory connection = new ConnectionFactory()
{
UserName = _configuration.Username,
Password = _configuration.Password,
HostName = _configuration.HostName
};
connection.DispatchConsumersAsync = true;
var channel = connection.CreateConnection();
return channel;
}
}
And finally create a HostedService and call ReadMessages method to register:
public class ConsumerHostedService : BackgroundService
{
private readonly IConsumerService _consumerService;
public ConsumerHostedService(IConsumerService consumerService)
{
_consumerService = consumerService;
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
await _consumerService.ReadMessgaes();
}
}
Register services:
services.AddSingleton<IRabbitMqService, RabbitMqService>();
services.AddSingleton<IConsumerService, ConsumerService>();
services.AddHostedService<ConsumerHostedService>();
In this case when application stopped, your consumer automatically will stop.
Additional information:
appsettings.json:
{
"RabbitMqConfiguration": {
"HostName": "localhost",
"Username": "guest",
"Password": "guest"
}
}
RabbitMqConfiguration
public class RabbitMqConfiguration
{
public string HostName { get; set; }
public string Username { get; set; }
public string Password { get; set; }
}
One of the best ways I found is to use a BackgroundService
public class TempConsumer : BackgroundService
{
private readonly ConnectionFactory _factory;
private IConnection _connection;
private IModel _channel;
public TempConsumer()
{
_factory = new ConnectionFactory()
{
HostName = "localhost",
UserName = "guest",
Password = "password",
VirtualHost = "/",
};
_connection = _factory.CreateConnection() ;
_channel = _connection.CreateModel();
_channel.QueueDeclare(queue: "queue",
durable: false,
exclusive: false,
autoDelete: false,
arguments: null);
}
protected override Task ExecuteAsync(CancellationToken stoppingToken)
{
stoppingToken.ThrowIfCancellationRequested();
var consumer = new EventingBasicConsumer(_channel);
consumer.Shutdown += OnConsumerShutdown;
consumer.Registered += OnConsumerRegistered;
consumer.Unregistered += OnConsumerUnregistered;
consumer.ConsumerCancelled += OnConsumerConsumerCancelled;
consumer.Received += (model, ea) =>
{
Console.WriteLine("Recieved");
var body = ea.Body;
var message = Encoding.UTF8.GetString(body.ToArray());
_channel.BasicAck(deliveryTag: ea.DeliveryTag, multiple: false);
Console.WriteLine(message);
};
_channel.BasicConsume(queue: "queue",
autoAck: false,
consumer: consumer);
return Task.CompletedTask;
}
private void OnConsumerConsumerCancelled(object sender, ConsumerEventArgs e) { }
private void OnConsumerUnregistered(object sender, ConsumerEventArgs e) { }
private void OnConsumerRegistered(object sender, ConsumerEventArgs e) { }
private void OnConsumerShutdown(object sender, ShutdownEventArgs e) { }
private void RabbitMQ_ConnectionShutdown(object sender, ShutdownEventArgs e) { }
Then, Register the consumer as a hosted service
services.AddHostedService<EmailConsumer>();