I need to create software that can read events from Azure Event Hub in an infinite way and save them on the db following some logic. The first question I wanted to ask is if such thing is feasible using:
EventProcessorClient (or rather EventProcessor<TPartition> as I would always like to use the SqlServer for checkpoints instead of an Azure Blob Storage) in a BackgroundService as Windows Service and then run it in infinitely and every so many events to checkpoint?
I tried this code reading from a single partition but the application crashes after an exception and the loop ends without continuing to process the events.
public class LogConsumerBackgroundService : BackgroundService
{
private const string EventHubConnectionString =
"ConnString";
private const string ConsumerGroup = "ConsGroup";
private static readonly JsonSerializerOptions SerializerOptions = new()
{
PropertyNameCaseInsensitive = true
};
private readonly ILogger<LogConsumerBackgroundService> _logger;
private readonly IServiceProvider _serviceProvider;
private int _count;
public LogConsumerBackgroundService(ILogger<LogConsumerBackgroundService> logger, IServiceProvider serviceProvider)
{
Guard.Against.Null(logger);
Guard.Against.Null(serviceProvider);
_logger = logger;
_serviceProvider = serviceProvider;
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
try
{
var eventHubConsumerClientOptions = new EventHubConsumerClientOptions
{
ConnectionOptions =
{
TransportType = EventHubsTransportType.AmqpWebSockets
}
};
await using var consumer = new EventHubConsumerClient(ConsumerGroup, EventHubConnectionString, eventHubConsumerClientOptions);
var startingPosition = EventPosition.Earliest;
var partitionIds = await consumer.GetPartitionIdsAsync(CancellationToken.None);
var partitionId = partitionIds.First();
var scope = _serviceProvider.CreateScope();
var repository = scope.ServiceProvider.GetRequiredService<IRepository>();
while (!stoppingToken.IsCancellationRequested)
{
await foreach (var receivedEvent in consumer.ReadEventsFromPartitionAsync(partitionId, startingPosition,
CancellationToken.None))
{
try
{
// Some logic to save data on bd
_logger.LogInformation("Processed row: {Count}", countInfo += _count);
await repository.SaveChangesAsync();
}
catch (Exception e)
{
_logger.LogError(e.Demystify(), "{EventBody}", receivedEvent.Data?.EventBody);
}
}
}
}
catch (TaskCanceledException e)
{
_logger.LogError(e.Demystify(), "Task was canceled");
}
catch (Exception e)
{
_logger.LogError(e.Demystify(), "An unhandled exception has occurred");
}
}
}
Related
I need to receive messages from a queue broker and, after some processing, store them in a database. In order to save records in blocks, i use a BatchBlock.
All this works through the BackgroundService
Code:
public class Worker : BackgroundService
{
private readonly IServiceProvider _serviceProvider;
private readonly BatchBlock<Comment> _batchBlock;
private readonly ActionBlock<Comment[]> _importer;
private readonly ConnectionFactory _factory;
private readonly IModel _channel;
private const string _queueName = "Comment";
private static int count;
public Worker(IConfiguration configuration, IServiceProvider serviceProvider)
{
_serviceProvider = serviceProvider;
_factory = new ConnectionFactory
{ Uri = new Uri(configuration.GetSection("RabbitMqConnection").Value), DispatchConsumersAsync = true };
var connection = _factory.CreateConnection();
_channel = connection.CreateModel();
_channel.QueueDeclare(queue: _queueName, durable: true, exclusive: false, autoDelete: false, arguments: null);
_batchBlock = new BatchBlock<Comment>(50);
_importer = new ActionBlock<Comment[]>(SaveCommentsToDb);
_batchBlock.LinkTo(_importer, new DataflowLinkOptions { PropagateCompletion = true });
}
protected override Task ExecuteAsync(CancellationToken stoppingToken)
{
stoppingToken.ThrowIfCancellationRequested();
var consumer = new AsyncEventingBasicConsumer(_channel);
consumer.Received += async (_, ea) =>
await StartCommentHandlerGroup(ea);
_channel.BasicConsume(_queueName, false, consumer);
return Task.CompletedTask;
}
private async Task StartCommentHandlerGroup(BasicDeliverEventArgs message)
{
try
{
var content = Encoding.UTF8.GetString(message.Body.ToArray());
var comment = JsonConvert.DeserializeObject<Comment>(content);
await _batchBlock.SendAsync(comment);
}
catch (Exception e)
{
Log.Error($"!!!Handler error!!! {e.Message}{Environment.NewLine} !!!Message!!!{message.Body}");
throw;
}
}
private async Task SaveCommentsToDb(Comment[] comments)
{
using var scope = _serviceProvider.CreateScope();
var dbContext = scope.ServiceProvider.GetRequiredService<Context>();
await dbContext.AddMessagesToDb(comments);
}
}
It works very weird.....
If i set the BatchSize parameter to 10, about 200 records get into the database, if i set it to 2, about 300 hits. I set the BatchSize to 50 and higher and nothing gets into the database.
In this case, the queue is consumed correctly. SendAsync works correctly in the BatchBlock, the Task returns true.
After a certain part of the processing is done, i get the InputCount parameters in the _importer - 0 and the status is IsCompleted - true.
No matter how many new messages from the broker i receive in the future, this does not change the situation in any way and the ActionBlock is not called.
await dbContext.AddMessagesToDb(comments) contains:
foreach (var model in models)
await DbContext.AddAsync(model);
await DbContext.SaveChangesAsync();
I tried adding the BoundedCapacity parameter, but it didn't work.
batchBlock = new BatchBlock<Comment>(10, new GroupingDataflowBlockOptions(){BoundedCapacity = 50});
I'm fairly new to bot dev but know a thing or too about C# however I've been looking for quite some time and still cannot seem to find a solution to my problem. I'm using Discord.NET and my bot was working great until I introduced Discord.Addons.Interactive addon and I don't know how to fix it.
Whenever I run my program I get the error: System.InvalidOperationException: 'No type reader found for type CommandContext, one must be specified'.
Here is my Program.cs:
class Program
{
public static void Main(string[] args)
{
new Program().MainAsync().GetAwaiter().GetResult();
}
private DiscordSocketClient _client;
private CommandService _commands;
private IServiceProvider _services;
public Program()
{
}
public async Task MainAsync()
{
_client = new DiscordSocketClient();
_commands = new CommandService();
_services = new ServiceCollection()
.AddSingleton(_client)
.AddSingleton(_commands)
.AddSingleton<CommandHandler>()
.AddSingleton<InteractiveService>()
.BuildServiceProvider();
_client.Log += LogAsync;
_client.Ready += ReadyAsync;
var tocken = "my_tocken";
await _client.LoginAsync(TokenType.Bot, tocken);
await _client.StartAsync();
await _services.GetRequiredService<CommandHandler>().InstallCommandAsync();
await Task.Delay(-1);
}
private Task LogAsync(LogMessage msg)
{
Console.WriteLine(msg.ToString());
return Task.CompletedTask;
}
private Task ReadyAsync()
{
Console.WriteLine($"Connected as -> [{_client.CurrentUser}]");
return Task.CompletedTask;
}
}
Here is my CommandHandler.cs:
public class CommandHandler
{
private readonly DiscordSocketClient _client;
private readonly CommandService _commands;
private readonly IServiceProvider _services;
public CommandHandler(DiscordSocketClient client, CommandService commands, IServiceProvider services, InteractiveService interactions)
{
_commands = commands;
_client = client;
_services = services;
_commands.CommandExecuted += CommandExecutedAsync;
_client.MessageReceived += HandleCommandAsync;
}
public async Task InstallCommandAsync()
{
await _commands.AddModulesAsync(Assembly.GetEntryAssembly(), _services);
}
private async Task HandleCommandAsync(SocketMessage messageParam)
{
var message = messageParam as SocketUserMessage;
var context = new SocketCommandContext(_client, message);
if (message.Author.IsBot) return;
int argPos = 0;
if (message.HasCharPrefix('-', ref argPos))
{
await _commands.ExecuteAsync(context, argPos, _services);
}
}
public async Task CommandExecutedAsync(Optional<CommandInfo> command, ICommandContext context, IResult result)
{
if (!command.IsSpecified)
{
Console.WriteLine($"Command failed to execute for [{context.User.Username}] <-> [{result.ErrorReason}]!");
return;
}
if (result.IsSuccess)
{
Console.WriteLine($"Command [{command.Value.Name}] execute for [{context.User.Username}]!");
return;
}
await context.Channel.SendMessageAsync($"{context.User.Username} Something went wrong -> [{result}]!");
}
}
If you need anymore information then just let me know.
EDIT:
I found the issue, I had put a CommandContext parameter in one of my commands.
Hope this helps anyone in the future :)
I have created a simple windows services to consume messages from Azure service bus queue. I used the Topshelf to create windows service. Code snipped below following example from here: https://learn.microsoft.com/en-us/azure/service-bus-messaging/service-bus-dotnet-get-started-with-queues
var hf = HostFactory.New(x =>
{
x.Service<ServiceBusHelper>(s =>
{
s.ConstructUsing(serviceProvider.GetService<ServiceBusHelper>);
s.WhenStarted(async service => await service.ReceiveMessagesAsync());
s.WhenStopped(async service => await service.Stop());
});
x.RunAsNetworkService()
.StartAutomatically()
.EnableServiceRecovery(rc => rc.RestartService(1));
x.SetServiceName("MyWindowsService");
x.SetDisplayName("MyWindowsService");
x.SetDescription("MyWindowsService");
});
hf.Run();
ServiceBusHelper class:
public async Task ReceiveMessagesAsync()
{
var connectionString = _configuration.GetValue<string>("ServiceBusConnectionString");
var queueName = _configuration.GetValue<string>("ServiceBusQueueName");
await using (ServiceBusClient client = new ServiceBusClient(connectionString))
{
ServiceBusProcessor processor = client.CreateProcessor(queueName, new ServiceBusProcessorOptions());
processor.ProcessMessageAsync += MessageHandler;
processor.ProcessErrorAsync += ErrorHandler;
await processor.StartProcessingAsync();
System.Threading.Thread.Sleep(1000);//Wait for a minute before stop processing
await processor.StopProcessingAsync();
}
}
public async Task MessageHandler(ProcessMessageEventArgs args)
{
string body = args.Message.Body.ToString();
var messageBytes = Encoding.ASCII.GetBytes(body);
ProcessMessage(messageBytes);
await args.CompleteMessageAsync(args.Message);
}
public Task ErrorHandler(ProcessErrorEventArgs args)
{
return Task.CompletedTask;
}
public Task Stop()
{
return Task.CompletedTask;
}
Window service gets installed successfully and the status show running. However, it would not automatically consume the message from the service bus. If I manually stop and start the service it will pick up the message from the queue. Not sure what am I missing with this implementation. Any suggestions appreciated.
.NetCore 3.1 introduced a new extension to work along side Microsoft.AspNetCore.Hosting
Adding NuGet package Microsoft.Extensions.Hosting.WindowsServices
you can add
.UseWindowsService(). this will allow you run this as a windows service or Console app.
public static IHostBuilder CreateHostBuilder(string[] args) =>
Host.CreateDefaultBuilder(args)
.UseWindowsService()
.ConfigureAppConfiguration((context, config) =>
{
// configure the app here.
})
.ConfigureServices((hostContext, services) =>
{
services.AddHostedService<QueueWorker>();
}).UseSerilog();
}
you can then create a background worker to start and stop processing the servicebus queue. Here is my implementaion:
public class QueueWorker : BackgroundService, IDisposable
{
protected ILogger<QueueWorker> _logger;
protected IQueueMessageReceiver _queueProcessor;
public QueueWorker()
{
}
public QueueWorker(ILogger<QueueWorker> logger, IQueueMessageReceiver queueMessageReceiver)
{
_logger = logger;
_queueProcessor = queueMessageReceiver;
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
await Task.CompletedTask.ConfigureAwait(false);
}
public override Task StartAsync(CancellationToken cancellationToken)
{
_logger.LogInformation("Service Starting");
var task = _queueProcessor.StartProcessor(cancellationToken);
task.Wait();
if (task.IsFaulted)
{
throw new Exception("Unable to start Processor");
}
return base.StartAsync(cancellationToken);
}
public override async Task StopAsync(CancellationToken cancellationToken)
{
_logger.LogInformation("Stopping Service");
await _queueProcessor.StopProcessor().ConfigureAwait(false);
await base.StopAsync(cancellationToken).ConfigureAwait(false);
}
public override void Dispose()
{
_logger.LogInformation("Disposing Service");
var loopCount = 0;
while (_queueProcessor != null && !_queueProcessor.IsClosedOrClosing() && loopCount < 5)
{
var task = Task.Delay(600);
task.Wait();
loopCount++;
}
base.Dispose();
GC.SuppressFinalize(this);
}
And The actual processor:
public class QueueMessageReceiver : IQueueMessageReceiver
{
private readonly ServiceBusClient _queueClient;
private ServiceBusProcessor _processor;
private readonly ReceiverConfiguration _configuration;
private readonly ILogger _logger;
private readonly ILoggerFactory _loggerFactory;
private Dictionary<string, string> _executionMatrix;
private readonly IServiceProvider _provider;
private CancellationToken _cancellationToken;
public QueueMessageReceiver(ReceiverConfiguration configuration, ILogger<QueueMessageReceiver> logger, IExecutionMatrix executionMatrix, ILoggerFactory loggerFactory, IServiceProvider serviceProvider)
{
if (configuration == null) throw new ArgumentException($"Configuration is missing from the expected ");
_configuration = configuration;
_logger = logger;
_loggerFactory = loggerFactory;
_executionMatrix = executionMatrix.GetExecutionMatrix();
_provider = serviceProvider;
_queueClient = new ServiceBusClient(_configuration.ConnectionString);
if (string.IsNullOrWhiteSpace(configuration.ConnectionString)) throw new ArgumentException($"ServiceBusConnectionString Object missing from the expected configuration under ConnectionStrings ");
if (configuration.QueueName == null) throw new ArgumentException($"Queue Name value missing from the expected configuration");
}
public async Task StartProcessor(CancellationToken cancellationToken)
{
if (!IsClosedOrClosing())
{
throw new FatalSystemException("ServiceBusProcessor is already running. ");
}
_cancellationToken = cancellationToken;
var options = new ServiceBusProcessorOptions
{
AutoCompleteMessages = _configuration.AutoComplete,
MaxConcurrentCalls = _configuration.MaxConcurrentCalls,
MaxAutoLockRenewalDuration = _configuration.MaxAutoRenewDuration
};
_processor = _queueClient.CreateProcessor(_configuration.QueueName, options);
_processor.ProcessMessageAsync += ProcessMessagesAsync;
_processor.ProcessErrorAsync += ProcessErrorAsync;
await _processor.StartProcessingAsync().ConfigureAwait(false);
}
public async Task StopProcessor()
{
await _processor.StopProcessingAsync();
await _processor.CloseAsync();
}
private Task ProcessErrorAsync(ProcessErrorEventArgs args)
{
_logger.LogError(args.Exception, "Uncaught handled exception", args.ErrorSource, args.FullyQualifiedNamespace, args.EntityPath);
return Task.CompletedTask;
}
private async Task ProcessMessagesAsync(ProcessMessageEventArgs args)
{
var message = args.Message;
// Process the message.
var sbMessage = $"Received message: SequenceNumber:{message.SequenceNumber} Body:{Encoding.UTF8.GetString(message.Body)}";
_logger.LogInformation(sbMessage);
//Handle your message
}
public bool IsClosedOrClosing()
{
return ((_processor == null) || _processor.IsClosed || !_processor.IsProcessing);
}
}
I am trying to implement a background service, QueuedBackground service with Mediator.
So far I am able to implement the queues but I am unable to execute Mediator.
Interface
public interface IBackgroundTaskQueueService
{
void QueueBackgroundWorkItem(object workItem, CancellationToken token);
Task<object> DequeueAsync(
CancellationToken cancellationToken);
}
Implementation
public class BackgroundTaskQueueService : IBackgroundTaskQueueService
{
private readonly ConcurrentQueue<(object,CancellationToken)> _workItems =
new ConcurrentQueue<(object,CancellationToken)>();
private SemaphoreSlim _signal = new SemaphoreSlim(0);
public void QueueBackgroundWorkItem(object workItem, CancellationToken token)
{
if (workItem == null)
{
throw new ArgumentNullException(nameof(workItem));
}
_workItems.Enqueue((workItem,token));
_signal.Release();
}
public async Task<object> DequeueAsync( CancellationToken cancellationToken)
{
await _signal.WaitAsync(cancellationToken);
_workItems.TryDequeue(out var workItem);
return workItem.Item1;
}
}
Background Service
public class QueuedHostedService : BackgroundService
{
private readonly ILogger _logger;
private readonly IMediator _mediator;
public QueuedHostedService(IBackgroundTaskQueueService taskQueue, ILoggerFactory loggerFactory, IMediator mediator)
{
TaskQueue = taskQueue;
_mediator = mediator;
_logger = loggerFactory.CreateLogger<QueuedHostedService>();
}
public IBackgroundTaskQueueService TaskQueue { get; }
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
while (false == stoppingToken.IsCancellationRequested)
{
try
{
var workItem = await TaskQueue.DequeueAsync(stoppingToken);
await _mediator.Send(workItem, stoppingToken);
// await _mediator.Send(new UpdateProductCostByMaterialRequestModel()
// {
// Id = 1
// }, stoppingToken);
}
catch (Exception ex)
{
_logger.LogError(ex, $"Error occurred executing Work item.");
}
}
}
}
Usage
_queueService.QueueBackgroundWorkItem(new UpdateProductCostByMaterialRequestModel()
{
Id = request.ProductId
}, CancellationToken.None);
Now with the above code I am able to receive the class object but when I pass it in the Mediator I get InvalidOperation Handler not registered.
I am confused.
Okay I found the issue
Instead of passing it from the Constructor I had to use the ServiceFactory Interface
My Solution
BackgroundService is a Singleton. You cannot inject a Scoped into a Singleton.
public class QueuedHostedService : BackgroundService
{
private readonly ILogger _logger;
private readonly IServiceScopeFactory _serviceScopeFactory;
public QueuedHostedService(IBackgroundTaskQueueService taskQueue, ILoggerFactory loggerFactory, IServiceScopeFactory serviceScopeFactory)
{
TaskQueue = taskQueue;
_serviceScopeFactory = serviceScopeFactory;
_logger = loggerFactory.CreateLogger<QueuedHostedService>();
}
public IBackgroundTaskQueueService TaskQueue { get; }
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
using var scope = _serviceScopeFactory.CreateScope();
var mediator = scope.ServiceProvider.GetRequiredService<IMediator>();
while (false == stoppingToken.IsCancellationRequested)
{
try
{
var workItem = await TaskQueue.DequeueAsync(stoppingToken);
if (workItem is IRequest<object> item)
{
await mediator.Send(workItem, stoppingToken);
}
// await _mediator.Send(new UpdateProductCostByMaterialRequestModel()
// {
// Id = 1
// }, stoppingToken);
}
catch (Exception ex)
{
_logger.LogError(ex, $"Error occurred executing Work item.");
}
}
}
}
I have a microservice (Web API) within an eventdriven architecture receiving messages from RabbitMQ and it is supposed to save them into a PostgreSQL DB using ADO.NET.
Unfortunately, my connection pool (currently set to 50) gets exhausted quite fast, giving me this error message:
The connection pool has been exhausted, either raise MaxPoolSize
My RabbitMQ Consumer is set up like this (Singleton):
public class Listener : RabbitMqConnection
{
public AsyncEventingBasicConsumer _asyncConsumer;
private static readonly SemaphoreSlim AsyncLock = new SemaphoreSlim(1, 1);
public Listener()
{
_asyncConsumer = new AsyncEventingBasicConsumer(_channel);
_asyncConsumer.Received += ConsumerReceived;
}
public async Task ConsumerReceived(object sender, BasicDeliverEventArgs message)
{
await AsyncLock.WaitAsync();
try
{
//Performing logic and saving into database
//....
using (var ctx = ContextFactory.GetContext<PostgreSqlDatabaseContext>(_connectionString))
{
//Creating query with StringBuilder...
await ctx.Database.ExecuteSqlCommandAsync(query.ToString(), parameters);
}
_channel.BasicAck(message.DeliveryTag, false);
}
catch (DecoderFallbackException decoderFallbackException)
{
_logger.LogError($"...");
_channel.BasicNack(message.DeliveryTag, false, false);
}
finally {
AsyncLock.Release();
}
}
}
ContextFactory
internal class ContextFactory
{
public static T GetContext<T>(string sqlConnection) where T : DbContext
{
var optionsBuilder = new DbContextOptionsBuilder<PostgreSqlDatabaseContext>();
optionsBuilder.UseNpgsql(sqlConnection);
return new PostgreSqlDatabaseContext(optionsBuilder.Options) as T;
}
}
RabbitMqConnection:
public abstract class RabbitMQConnection
{
public IModel _channel;
public IBasicProperties _properties;
public AsyncEventingBasicConsumer _asyncConsumer;
public ConnectionFactory _factory;
public ConnectConfiguration _connectConfiguration;
bool isConnected = false;
public void Connect(ConnectConfiguration connectConfiguration)
{
if (!isConnected)
{
_connectConfiguration = connectConfiguration;
CreateFactory(_connectConfiguration);
SetupConfiguration(_connectConfiguration.Exchange);
}
}
private void CreateFactory(ConnectConfiguration config)
{
_factory = new ConnectionFactory
{
AutomaticRecoveryEnabled = true,
DispatchConsumersAsync = true,
UseBackgroundThreadsForIO = true,
RequestedHeartbeat = 15,
HostName = config.Server,
UserName = config.UserName,
Password = config.Password
};
if (!string.IsNullOrWhiteSpace(config.Vhost))
_factory.VirtualHost = config.Vhost;
}
private void SetupConfiguration(string exchange)
{
var connection = _factory.CreateConnection();
_channel = connection.CreateModel();
_properties = _channel.CreateBasicProperties();
_properties.Persistent = true;
_channel.BasicQos(0, 10, false);
_channel.ExchangeDeclare(exchange, "topic", true);
isConnected = true;
}
}
I can´t not understand why I keep getting this error. Isn´t the SemaphoreSlim with WaitAsync() and Release() suppose to prevent the ConsumerReceived method from running the logic?