I have a background task that runs every few seconds in my ASP.NET Core application. It returns the following error:
An exception occurred while iterating over the results of a query for context type 'ThaiLiveApi.Data.DataContext'.
System.InvalidOperationException: A second operation started on this context before a previous operation completed.
This is usually caused by different threads using the same instance of DbContext.
I would think because im using the dbcontext in scoped that this would not be possible and only use one dbcontext for that scope. Am I missing something?
Here is my worker code:
public class FacebookWorker : IHostedService
{
private readonly IServiceScopeFactory _serviceScopeFactory;
private readonly ILogger<FacebookWorker> _logger;
public FacebookWorker(IServiceScopeFactory serviceScopeFactory, ILogger<FacebookWorker> logger)
{
_serviceScopeFactory = serviceScopeFactory;
_logger = logger;
}
private readonly int JobIntervalInSecs = 5;
private Timer _timer;
public Task StartAsync(CancellationToken cancellationToken)
{
if (cancellationToken.IsCancellationRequested)
{
cancellationToken.ThrowIfCancellationRequested();
}
// Invoke the DoWork method every 5 seconds.
_timer = new Timer(callback: async o => await DoWork(o),
state: null, dueTime: TimeSpan.FromSeconds(0),
period: TimeSpan.FromSeconds(JobIntervalInSecs));
return Task.CompletedTask;
}
private async Task DoWork(object state) {
// allow only a certain number of concurrent work. In this case,
// only allow one job to run at a time.
if (State.numberOfActiveJobs < State.maxNumberOfActiveJobs) {
// Update number of running jobs in one atomic operation.
try {
Interlocked.Increment(ref State.numberOfActiveJobs);
_logger.LogInformation("Fetching comments " + DateTime.Now);
using (var scope = _serviceScopeFactory.CreateScope()) {
var facebookService = scope.ServiceProvider.GetRequiredService<IFacebookService>();
await facebookService.FetchAndHandleComments();
}
}
finally {
Interlocked.Decrement(ref State.numberOfActiveJobs);
}
}
}
}
}
I suppose I'm getting this error because of my Worker? Is it possible this is caused in another part of my code?
It happens when a process started but last process is not complete. try to use using block when call DbContext. One more thing Check your DbContext instance is it static or not.
Related
I have a need in my asp.net webapi (framework .Net 4.7.2) to call Redis (using StackExchange.Redis) in order to delete a key in a fire and forget way and I am making some stress test.
As I am comparing the various way to have the max speed :
I have already test executing the command with the FireAndForget flag,
I have also measured a simple command to Redis by await it.
And I am now searching a way to collect a list of commands received in a window of 15ms and execute them all in one go by pipeling them.
I have first try to use a Task.Run Action to call Redis but the problem that I am observing is that under stress, the memory of my webapi keep climbing.
The memory is full of System.Threading.IThreadPoolWorkItem[] objects with the folowing code :
[HttpPost]
[Route("api/values/testpostfireforget")]
public ApiResult<int> DeleteFromBasketId([FromBody] int basketId)
{
var response = new DeleteFromBasketResponse<int>();
var cpt = Interlocked.Increment(ref counter);
Task.Run(async () => {
await db.StringSetAsync($"BASKET_TO_DELETE_{cpt}",cpt.ToString())
.ConfigureAwait(false);
});
return response;
}
So I think that under stress my api keep enqueing background task in memory and execute them one after the other as fast as it can but less than the request coming in...
So I am searching for a way to have only one long lived background thread running with the asp.net webapi, that could capture the commands to send to Redis and execute them by pipeling them.
I was thinking in runnning a background task by implementing IHostedService interface, but it seems that in this case the background task would not share any state with my current http request. So implementing a IhostedService would be handy for a scheduled background task but not in my case, or I do not know how...
Based on StackExchange.Redis documentation you can use CommandFlags.FireAndForget flag:
[HttpPost]
[Route("api/values/testpostfireforget")]
public ApiResult<int> DeleteFromBasketId([FromBody] int basketId)
{
var response = new DeleteFromBasketResponse<int>();
var cpt = Interlocked.Increment(ref counter);
db.StringSet($"BASKET_TO_DELETE_{cpt}", cpt.ToString(), flags: CommandFlags.FireAndForget);
return response;
}
Edit 1: another solution based on comment
You can use pub/sub approach. Something like this should work:
public class MessageBatcher
{
private readonly IDatabase target;
private readonly BlockingCollection<Action<IDatabaseAsync>> tasks = new();
private Task worker;
public MessageBatcher(IDatabase target) => this.target = target;
public void AddMessage(Action<IDatabaseAsync> task) => tasks.Add(task);
public IDisposable Start(int batchSize)
{
var cancellationTokenSource = new CancellationTokenSource();
worker = Task.Factory.StartNew(state =>
{
var count = 0;
var tokenSource = (CancellationTokenSource) state;
var box = new StrongBox<IBatch>(target.CreateBatch());
tokenSource.Token.Register(b => ((StrongBox<IBatch>)b).Value.Execute(), box);
foreach (var task in tasks.GetConsumingEnumerable(tokenSource.Token))
{
var batch = box.Value;
task(batch);
if (++count == batchSize)
{
batch.Execute();
box.Value = target.CreateBatch();
count = 0;
}
}
}, cancellationTokenSource, cancellationTokenSource.Token, TaskCreationOptions.LongRunning, TaskScheduler.Current);
return new Disposer(worker, cancellationTokenSource);
}
private class Disposer : IDisposable
{
private readonly Task worker;
private readonly CancellationTokenSource tokenSource;
public Disposer(Task worker, CancellationTokenSource tokenSource) => (this.worker, this.tokenSource) = (worker, tokenSource);
public void Dispose()
{
tokenSource.Cancel();
worker.Wait();
tokenSource.Dispose();
}
}
}
Usage:
private readonly MessageBatcher batcher;
ctor(MessageBatcher batcher) // ensure that passed `handler` is singleton and already already started
{
this.batcher= batcher;
}
[HttpPost]
[Route("api/values/testpostfireforget")]
public ApiResult<int> DeleteFromBasketId([FromBody] int basketId)
{
var response = new DeleteFromBasketResponse<int>();
var cpt = Interlocked.Increment(ref counter);
batcher.AddMessage(db => db.StringSetAsync($"BASKET_TO_DELETE_{cpt}", cpt.ToString(), flags: CommandFlags.FireAndForget));
return response;
}
I have a Background Worker implementing the BackgroundService (provided by MS).
See this simple implementation:
public class MyService : BackgroundService {
private readonly MyDbContext _context;
public MyService(MyDbContext context) {
//...
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
try {
while (true)
{
stoppingToken.ThrowIfCancellationRequested();
// Do some work
}
} catch(OperationCancelledException) {
_context.Add(new MyLogMessage(){ Error = "MyService cancelled!" });
_context.SaveChanges();
}
// ...
}
}
When the graceful shutdown (in console: CTRL+C) is requested the catch block is triggered, and also the SaveChanges() seems to be executed. But, sometimes the error is stored into the database and the most of the time it is not. Also the EntityFramework is printing an insert statement on the console, but the log is not in the db.
I assume that the shutdown is happening faster then writting the data to the DB?
Can anyone give me a hint how to handle this situation and store the error into the database?
It seems like the stoppingToken isn't cancelled as expected when the application shuts down. I managed to get around this using IHostApplicationLifetime and a new field where I can store if a shutdown is in progress.
public class TestService : BackgroundService {
private readonly IHostApplicationLifetime _lifetime;
private readonly ILogger<TestService> _logger;
private bool _shutownRequested;
public TestService(IHostApplicationLifetime lifetime, ILogger<TestService> logger) {
_lifetime = lifetime;
_logger = logger;
}
public override Task StartAsync(CancellationToken cancellationToken) {
_lifetime.ApplicationStopping.Register(OnShutdown);
return Task.CompletedTask;
}
private void OnShutdown() {
_shutdownRequested = true;
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken) {
try {
while(true) {
stoppingToken.ThrowIfCancellationRequested();
if(_shutdownRequested) {
throw new OperationCanceledException();
}
await Task.Delay(100, CancellationToken.None);
}
} catch(OperationCanceledException) {
_logger.LogWarning("TestService canceled");
}
}
}
Now it might be better to now throw a new exception there, but as an example it will do.
The reason why the log entry doesn't appear in the database is that the host shutdown period is lower than what it takes to process a task in a while loop and send a log to the database. The default timeout is 5 seconds.
What you could do, is to increase the timeout to a larger value, for example a minute a two:
services.Configure<HostOptions>(
opts => opts.ShutdownTimeout = TimeSpan.FromMinutes(2));
Make sure to let enough time for a service to finish the iteration inside a while loop and log the message.
Please check Extending the shutdown timeout setting to ensure graceful IHostedService shutdown for more details.
Recently I asked a question on StackOverflow about long running background Tasks in asp.net core. Since then I have tried everything from here https://learn.microsoft.com/cs-cz/aspnet/core/fundamentals/host/hosted-services?view=aspnetcore-5.0&tabs=visual-studio and if i do what I do it will just stop at some time. And it will stop even if I wrap it in IServiceProvider.CreateScope and await it. The only thing I still didn't try and I'm trying to avoid it, is creating dedicated .net application that would just read queue and do what it's supposed to do. And also I thing that it's overkill to create queue for it, I just want to run it in background asynchronously but it just stops. Sorry if it's some stupid bug but this is my first asp.net project and I'm fixing this problem for week now.
This is Queue version
public class QueuedHostedService : BackgroundService
{
private readonly IServiceProvider _serviceProvider;
private readonly IBackgroundTaskQueue _queue;
private readonly ILogger<QueuedHostedService> _logger;
public QueuedHostedService(IServiceProvider serviceProvider, IBackgroundTaskQueue queue, ILogger<QueuedHostedService> logger)
{
_serviceProvider = serviceProvider;
_queue = queue;
_logger = logger;
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
await BackgroundProcessing(stoppingToken);
}
private async Task BackgroundProcessing(CancellationToken stoppingToken)
{
while (!stoppingToken.IsCancellationRequested)
{
var workItem = await _queue.DequeueAsync(stoppingToken);
try
{
var scope = _serviceProvider.CreateScope();
var scrapeUrl = scope.ServiceProvider.GetRequiredService<IScopedScrapeUrl>();
// The scrape Sound Cloud Task is taking hours
await scrapeUrl.ScrapeSoundCloud(workItem);
}catch(Exception ex)
{
_logger.LogError($"Error occurred executing {nameof(workItem)},\n{ex}");
}
}
}
public override async Task StopAsync(CancellationToken stoppingToken)
{
_logger.LogInformation("Consume Scoped Scrape Url Hosted Service is stopping.");
await base.StopAsync(stoppingToken);
}
public class BackgroundScrapeQueue : IBackgroundTaskQueue
{
private readonly Channel<Scrape> _queue;
private readonly ILogger<BackgroundScrapeQueue> _logger;
public BackgroundScrapeQueue(ILogger<BackgroundScrapeQueue> logger)
{
var options = new BoundedChannelOptions(100)
{
FullMode = BoundedChannelFullMode.Wait
};
_queue = Channel.CreateBounded<Scrape>(options);
_logger = logger;
}
public async ValueTask<Scrape> DequeueAsync(CancellationToken stoppingToken)
{
var workItem = await _queue.Reader.ReadAsync(stoppingToken);
return workItem;
}
public async ValueTask QueueBackgroundWorkItemAsync(Scrape scrape)
{
if(scrape == null)
{
_logger.LogError("Invalid Scrape for queue");
return;
}
await _queue.Writer.WriteAsync(scrape);
}
}
public interface IBackgroundTaskQueue
{
ValueTask QueueBackgroundWorkItemAsync(Scrape scrape);
ValueTask<Scrape> DequeueAsync(CancellationToken stoppingToken);
}
it will just stop at some time. And it will stop even if I wrap it in IServiceProvider.CreateScope and await it.
Yes. That's the problem with in-memory background services. They can be stopped at any time, because they're hosted in an ASP.NET process that determines it's safe to shut down when requests are complete. ASP.NET will actually request a shutdown and then wait for a while for the services to complete, but there's also a timer where they'll be forced out if they don't complete within 10 minutes or so.
The bottom line is that shutdowns are normal. Any code that assumes it can run indefinitely in ASP.NET is inherently buggy.
The only thing I still didn't try and I'm trying to avoid it, is creating dedicated .net application that would just read queue and do what it's supposed to do.
That is the only reliable solution.
I work with some WIFI devices such as cameras.
The basic fellow that I implemented:
Someone presses a button.
The button calls my Web API endpoint.
My Web API end point calls one of the API's of camera (by HttpRequest).
Processing each request takes 5 second. And between each request should be 1 second delay. For instance, If you press the button 2 times with one second delay after each: First we expect 5 second for processing the first press, then one second delay and in the end we expect 5 second for the last process (second press).
To do that I am using Queued background tasks based on Fire and Forgot manner in .NetCore 3.1 project and it works fine when I am dealing with just one camera.
But the new requirement of the project is, The background task should handle multiple cameras. It means one queue per camera, and queues should work parallel based on the fellow that I described above.
For example if we have 2 devices camera-001 and camera-002 and 2 connected buttons btn-cam-001 and btn-cam-002, And the order of pressing(0.5sec delay after each press) : 2X btn-cam-001 and 1X btn-cam-002.
What really happens is FIFO. First the requests of btn-cam-001 will be processed and then btn-cam-002.
What I expect and need: Camera-002 should not wait to receive the request and the first requests towards both cameras 001/002 should be processed in a same time(Based on the exmaple). Like each camera has own queue and own process.
The question is how can I achieve that in .NetCore 3.1?
Appreciate any help.
My current background service:
public class QueuedHostedService : BackgroundService
{
public IBackgroundTaskQueue TaskQueue { get; }
private readonly ILogger _logger;
public QueuedHostedService(IBackgroundTaskQueue taskQueue, ILoggerFactory loggerFactory)
{
TaskQueue = taskQueue;
_logger = loggerFactory.CreateLogger<QueuedHostedService>();
}
protected override async Task ExecuteAsync(CancellationToken cancellationToken)
{
_logger.LogInformation("Queued Hosted Service is starting.");
while (!cancellationToken.IsCancellationRequested)
{
var workItem = await TaskQueue.DequeueAsync(cancellationToken);
try
{
await workItem(cancellationToken);
}
catch (Exception exception)
{
_logger.LogError(exception, $"Error occurred executing {nameof(workItem)}.");
}
}
_logger.LogInformation("Queued Hosted Service is stopping.");
}
}
And the current BackgroundTaskQueue:
public class BackgroundTaskQueue : IBackgroundTaskQueue
{
private readonly SemaphoreSlim _signal = new SemaphoreSlim(0);
private readonly ConcurrentQueue<Func<CancellationToken, Task>> _workItems =
new ConcurrentQueue<Func<CancellationToken, Task>>();
public void QueueBackgroundWorkItem(Func<CancellationToken, Task> workItem)
{
if (workItem is null)
{
throw new ArgumentNullException(nameof(workItem));
}
_workItems.Enqueue(workItem);
_signal.Release();
}
public async Task<Func<CancellationToken, Task>> DequeueAsync(CancellationToken cancellationToken)
{
await _signal.WaitAsync(cancellationToken);
_workItems.TryDequeue(out var workItem);
return workItem;
}
}
My current endpoint:
[HttpPost("hit")]
public ActionResult TurnOnAsync([FromBody] HitRequest request, CancellationToken cancellationToken = default)
{
try
{
var camera = ConfigurationHelper.GetAndValidateCamera(request.Device, _configuration);
_taskQueue.QueueBackgroundWorkItem(async x =>
{
await _cameraRelayService.TurnOnAsync(request.Device, cancellationToken);
Thread.Sleep(TimeSpan.FromSeconds(1));
});
return Ok();
}
catch (Exception exception)
{
_logger.LogError(exception, "Error when truning on the lamp {DeviceName}.", request.Device);
return StatusCode(StatusCodes.Status500InternalServerError, exception.Message);
}
}
Instead of a single BackgroundTaskQueue you could have one per camera. You could store the queues in a dictionary, having the camera as the key:
public IDictionary<IDevice, IBackgroundTaskQueue> TaskQueues { get; }
Then in your end-point use the queue that is associated with the requested camera:
_taskQueues[camera].QueueBackgroundWorkItem(async x =>
I'm using mvc with .net core, I need to run a task on start, and stop it when application stops. In Startup.cs I registered events for application start and stop. The problem is, I don't know how to run a task that has to be run in a specific class from startup. The task looks like this:
public void PreventStatusChange()
{
while (forceStatusChange)
{
foreach (var ext in GetExtensions())
{
ext.Status = StatusType.Available;
}
Thread.Sleep(1000);
}
}
Variable forceStatusChange is declared in the same class, so I don't see it from my Startup.cs. What is the best way to do it?
You need to create a class that implements IHostedService. This interface defines just two methods, StartAsync which is called when the application starts and StopAsync which is called when it terminates.
You need to register it as a hosted service with :
services.AddHostedService<TimedHostedService>();
Be careful to use AddHostedService, NOT AddSingleton. If you use AddSingleton the runtime won't know to call StartAsync and StopAsync when appropriate.
The article Background tasks with hosted services in ASP.NET Core shows how to implement a service using a timer :
internal class TimedHostedService : IHostedService, IDisposable
{
private readonly ILogger _logger;
private Timer _timer;
public TimedHostedService(ILogger<TimedHostedService> logger)
{
_logger = logger;
}
public Task StartAsync(CancellationToken cancellationToken)
{
_logger.LogInformation("Timed Background Service is starting.");
_timer = new Timer(DoWork, null, TimeSpan.Zero,
TimeSpan.FromSeconds(10));
return Task.CompletedTask;
}
private void DoWork(object state)
{
_logger.LogInformation("Timed Background Service is working.");
}
public Task StopAsync(CancellationToken cancellationToken)
{
_logger.LogInformation("Timed Background Service is stopping.");
_timer?.Change(Timeout.Infinite, 0);
return Task.CompletedTask;
}
public void Dispose()
{
_timer?.Dispose();
}
}
There's nothing particularly interesting in this code - just start a timer when StartAsync is called and stop it on StopAsync
Cancelling long-running tasks
When the runtime needs to recycle or stop, it will call the StopAsync method on all hosted services, wait a while for them to finish gracefully and then warn them to cancel immediatelly. After a short while it will go ahead and terminate the app or recycle it.
The cancellationToken parameter is used to signal that the service should stop immediatelly. Typically, that means that you'd have to write your own code to check it, warn your own task to terminate, wait for all of them to finish etc, similar to the code shown in this article
This is pretty much boilerplate though, which is why the BackgroundService class can be used to create a class that only needs to implement ExecuteAsync(CancellationToken). Starting and stopping that task is provided by BackgroundService, eg :
public class PollingService : BackgroundService
{
private readonly ILogger _logger;
public PollingService(ILogger<PollingService> logger)
{
_logger = logger;
}
protected async override Task ExecuteAsync(
CancellationToken cancellationToken)
{
while (!cancellationToken.IsCancellationRequested)
{
try
{
await DoSomething(cancellationToken);
await Task.Delay(1000,cancellationToken);
}
catch (Exception ex)
{
_logger.LogError(ex,
$"Error occurred executing {nameof(workItem)}.");
}
}
_logger.LogInformation("Queued Hosted Service is stopping.");
}
}
In this case Task.Delay() itself will be cancelled as soon as the runtime raises the cancellation token. DoSomething() itself should be implemented in a way that checks the cancellation token, eg pass it to any asynchronous method that accepts it as a parameter, test the IsCancellationRequested property on every loop and exit it it's raised.
For example :
protected async override Task ExecuteAsync(
CancellationToken cancellationToken)
{
while (!cancellationToken.IsCancellationRequested)
{
try
{
foreach (var ext in GetExtensions())
{
//Oops, time to cancel
if(cancellationToken.IsCancellationRequested)
{
break;
}
//Otherwise, keep working
ext.Status = StatusType.Available;
}
await Task.Delay(1000,cancellationToken);
}
catch (Exception ex)
{
...
}
}
_logger.LogInformation("Hosted Service is stopping.");
}
You can use BackgroundService
public class LongRunningService : BackgroundService
{
public LongRunningService()
{
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
while (!stoppingToken.IsCancellationRequested && forceStatusChange)
{
foreach (var ext in GetExtensions())
{
ext.Status = StatusType.Available;
}
await Task.Delay(1000, stoppingToken);
}
}
protected override async Task StopAsync (CancellationToken stoppingToken)
{
// Run your graceful clean-up actions
}
}
And register it:
public void ConfigureServices(IServiceCollection services)
{
...
services.AddSingleton<IHostedService, LongRunningService>();
...
}