I am running hangfire in a single web application, my application is being run on 2 physical servers but hangfire is in 1 database.
At the moment, i am generating a server for each queue, because each queue i need to run 1 worker at a time and they must be in order. I set them up like this
// core
services.AddHangfire(options =>
{
options.SetDataCompatibilityLevel(CompatibilityLevel.Version_170);
options.UseSimpleAssemblyNameTypeSerializer();
options.UseRecommendedSerializerSettings();
options.UseSqlServerStorage(appSettings.Data.DefaultConnection.ConnectionString, storageOptions);
});
// add multiple servers, this way we get to control how many workers are in each queue
services.AddHangfireServer(options =>
{
options.ServerName = "workflow-queue";
options.WorkerCount = 1;
options.Queues = new string[] { "workflow-queue" };
options.SchedulePollingInterval = TimeSpan.FromSeconds(10);
});
services.AddHangfireServer(options =>
{
options.ServerName = "alert-schedule";
options.WorkerCount = 1;
options.Queues = new string[] { "alert-schedule" };
options.SchedulePollingInterval = TimeSpan.FromMinutes(1);
});
services.AddHangfireServer(options =>
{
options.ServerName = string.Format("trigger-schedule");
options.WorkerCount = 1;
options.Queues = new string[] { "trigger-schedule" };
options.SchedulePollingInterval = TimeSpan.FromMinutes(1);
});
services.AddHangfireServer(options =>
{
options.ServerName = "report-schedule";
options.WorkerCount = 1;
options.Queues = new string[] { "report-schedule" };
options.SchedulePollingInterval = TimeSpan.FromMinutes(1);
});
services.AddHangfireServer(options =>
{
options.ServerName = "maintenance";
options.WorkerCount = 5;
options.Queues = new string[] { "maintenance" };
options.SchedulePollingInterval = TimeSpan.FromMinutes(10);
});
My problem is that it is generating multiple queues on the servers, with different ports.
In my code i am then trying to stop jobs from running if they are queued/retrying, but if the job is being run on a different physical server, it is not found and queued again.
Here is the code to check if its running already
public async Task<bool> IsAlreadyQueuedAsync(PerformContext context)
{
var disableJob = false;
var monitoringApi = JobStorage.Current.GetMonitoringApi();
// get the jobId, method and queue using performContext
var jobId = context.BackgroundJob.Id;
var methodInfo = context.BackgroundJob.Job.Method;
var queueAttribute = (QueueAttribute)Attribute.GetCustomAttribute(context.BackgroundJob.Job.Method, typeof(QueueAttribute));
// enqueuedJobs
var enqueuedjobStatesToCheck = new[] { "Processing" };
var enqueuedJobs = monitoringApi.EnqueuedJobs(queueAttribute.Queue, 0, 1000);
var enqueuedJobsAlready = enqueuedJobs.Count(e => e.Key != jobId && e.Value != null && e.Value.Job != null && e.Value.Job.Method.Equals(methodInfo) && enqueuedjobStatesToCheck.Contains(e.Value.State));
if (enqueuedJobsAlready > 0)
disableJob = true;
// scheduledJobs
if (!disableJob)
{
// check if there are any scheduledJobs that are processing
var scheduledJobs = monitoringApi.ScheduledJobs(0, 1000);
var scheduledJobsAlready = scheduledJobs.Count(e => e.Key != jobId && e.Value != null && e.Value.Job != null && e.Value.Job.Method.Equals(methodInfo));
if (scheduledJobsAlready > 0)
disableJob = true;
}
// failedJobs
if (!disableJob)
{
var failedJobs = monitoringApi.FailedJobs(0, 1000);
var failedJobsAlready = failedJobs.Count(e => e.Key != jobId && e.Value != null && e.Value.Job != null && e.Value.Job.Method.Equals(methodInfo));
if (failedJobsAlready > 0)
disableJob = true;
}
// if runBefore is true, then lets remove the current job running, else it will write a "successful" message in the logs
if (disableJob)
{
// use hangfire delete, for cleanup
BackgroundJob.Delete(jobId);
// create our sqlBuilder to remove the entries altogether including the count
var sqlBuilder = new SqlBuilder()
.DELETE_FROM("Hangfire.[Job]")
.WHERE("[Id] = {0};", jobId);
sqlBuilder.Append("DELETE TOP(1) FROM Hangfire.[Counter] WHERE [Key] = 'stats:deleted' AND [Value] = 1;");
using (var cmd = _context.CreateCommand(sqlBuilder))
await cmd.ExecuteNonQueryAsync();
return true;
}
return false;
}
Each method has something like the following attributes as well
public interface IAlertScheduleService
{
[Hangfire.Queue("alert-schedule")]
[Hangfire.DisableConcurrentExecution(60 * 60 * 5)]
Task RunAllAsync(PerformContext context);
}
Simple implementation of the interface
public class AlertScheduleService : IAlertScheduleService
{
public Task RunAllAsync(PerformContext context)
{
if (IsAlreadyQueuedAsync(context))
return;
// guess it isnt queued, so run it here....
}
}
Here is how i am adding my scheduled jobs
//// our recurring jobs
//// set these to run hourly, so they can play "catch-up" if needed
RecurringJob.AddOrUpdate<IAlertScheduleService>(e => e.RunAllAsync(null), Cron.Hourly(0), queue: "alert-schedule");
Why does this happen? How can i stop it happening?
Somewhat of a blind shot, preventing a job to be queued if a job is already queued in the same queue.
The try-catch logic is quite ugly but I have no better idea right now...
Also, really not sure the lock logic always prevents from having two jobs in EnqueudState, but it should help anyway. Maybe mixing with an IApplyStateFilter.
public class DoNotQueueIfAlreadyQueued : IElectStateFilter
{
public void OnStateElection(ElectStateContext context)
{
if (context.CandidateState is EnqueuedState)
{
EnqueuedState es = context.CandidateState as EnqueuedState;
IDisposable distributedLock = null;
try
{
while (distributedLock == null)
{
try
{
distributedLock = context.Connection.AcquireDistributedLock($"{nameof(DoNotQueueIfAlreadyQueued)}-{es.Queue}", TimeSpan.FromSeconds(1));
}
catch { }
}
var m = context.Storage.GetMonitoringApi();
if (m.EnqueuedCount(es.Queue) > 0)
{
context.CandidateState = new DeletedState();
}
}
finally
{
distributedLock.Dispose();
}
}
}
}
The filter can be declared as in this answer
There seems to be a bug with your currently used hangfire storage implementation:
https://github.com/HangfireIO/Hangfire/issues/1025
The current options are:
Switching to HangFire.LiteDB as commented here: https://github.com/HangfireIO/Hangfire/issues/1025#issuecomment-686433594
Implementing your own logic to enqueue a job, but this would take more effort.
Making your job execution idempotent to avoid side effects in case it's executed multiple times.
In either option, you should still apply DisableConcurrentExecution and make your job execution idempotent as explained below, so i think you can just go with below option:
Applying DisableConcurrentExecution is necessary, but it's not enough as there are no reliable automatic failure detectors in distributed systems. That's the nature of distributed systems, we usually have to rely on timeouts to detect failures, but it's not reliable.
Hangfire is designed to run with at-least-once execution semantics. Explained below:
One of your servers may be executing the job, but it's detected as being failed due to various reasons. For example: your current processing server does not send heartbeats in time due to a temporary network issue or due to temporary high load.
When the current processing server is assumed to be failed (but it's not), the job will be scheduled to another server which causes it to be executed more than once.
The solution should be still applying DisableConcurrentExecution attribute as a best effort to prevent multiple executions of the same job, but the main thing is that you need to make the execution of the job idempotent which does not cause side effects in case it's executed multiple times.
Please refer to some quotes from https://docs.hangfire.io/en/latest/background-processing/throttling.html:
Throttlers apply only to different background jobs, and there’s no
reliable way to prevent multiple executions of the same background job
other than by using transactions in background job method itself.
DisableConcurrentExecution may help a bit by narrowing the safety
violation surface, but it heavily relies on an active connection,
which may be broken (and lock is released) without any notification
for our background job.
As there are no reliable automatic failure detectors in distributed
systems, it is possible that the same job is being processed on
different workers in some corner cases. Unlike OS-based mutexes,
mutexes in this package don’t protect from this behavior so develop
accordingly.
DisableConcurrentExecution filter may reduce the probability of
violation of this safety property, but the only way to guarantee it is
to use transactions or CAS-based operations in our background jobs to
make them idempotent.
You can also refer to this as Hangfire timeouts behavior seems to be dependent on storage as well: https://github.com/HangfireIO/Hangfire/issues/1960#issuecomment-962884011
Related
Closed. This question needs debugging details. It is not currently accepting answers.
Edit the question to include desired behavior, a specific problem or error, and the shortest code necessary to reproduce the problem. This will help others answer the question.
Closed 1 year ago.
Improve this question
I have a web application that makes industrial scheduling calculations, I'm trying to log events in Azure cosmos db table after every update that happens to the schedule without affecting the application performance (screen Loading time).
That means, I want to fire log method in the BACKGROUND and the end user will not feel it (no freeze or extra loading time) and without making the UI wait for this operation to be done.
I added the next C# lines of code just after finishing the whole calculations:
private List<JPIEventEntity> batch = new List<JPIEventEntity>();
private List<List<JPIEventEntity>> batchesList = new List<List<JPIEventEntity>>();
Thread newThread = new Thread(() => myJPIEventHandler.checkForJPIEventsToSend(customer, author, model));
newThread.Start();
/*
* check if there are any changes or updates in the calculations and log their events.
*/
internal void checkForJPIEventsToSend(JPIBaseCustomer customer, JPIBaseUser author, SchedulingModel afterModel)
{
myCustomer = customer;
myUser = author;
// Looking for deleted Jobs
foreach (Job beforeJob in myBeforeModel.Jobs)
{
if (!afterModel.Jobs.Any(x => x.Guid == beforeJob.Guid))
{
//Looking for deleted Tasks and Log the deletion
foreach (Operation beforeOp in beforeJob.Operations)
{
//if (!afterJob.Operations.Any(x => x.Guid == beforeOp.Guid))
logTaskEvent(EventType.Delete, beforeOp, "", "");
}
//Log Job Deletion
logJobEvent(EventType.Delete, beforeJob, "", "");
}
}
//Comparison
foreach (Job afterJob in afterModel.Jobs)
{
if (myBeforeModel.Jobs.Any(x => x.Guid == afterJob.Guid))
{
Job beforeJob = myBeforeModel.Jobs.First(x => x.Guid == afterJob.Guid);
if (beforeJob.Name != afterJob.Name)
logJobEvent(EventType.NameChanged, afterJob, beforeJob.Name, afterJob.Name);
if (beforeJob.ReleaseDate != afterJob.ReleaseDate)
logJobEvent(EventType.ReleaseDateChanged, afterJob, beforeJob.ReleaseDate, afterJob.ReleaseDate);
if (beforeJob.DueDate != afterJob.DueDate)
logJobEvent(EventType.DueDateChanged, afterJob, beforeJob.DueDate, afterJob.DueDate);
if (beforeJob.IsDueDateExceeded != afterJob.IsDueDateExceeded)
logJobEvent(EventType.DueDateExceededChanged, afterJob, beforeJob.IsDueDateExceeded.ToString(), afterJob.IsDueDateExceeded.ToString());
if (beforeJob.ProcessingState != afterJob.ProcessingState)
{
logJobEvent(EventType.StatusChanged, afterJob,
convertProcessingStateToStatus(beforeJob.ProcessingState.ToString()), convertProcessingStateToStatus(afterJob.ProcessingState.ToString()));
}
if (beforeJob.SequenceNumber != afterJob.SequenceNumber && afterJob.ProcessingState != JobProcessingState.Finished)
logJobEvent(EventType.SequenceNumberChanged, afterJob, beforeJob.SequenceNumber, afterJob.SequenceNumber);
if (beforeJob.CustomQuantity != afterJob.CustomQuantity)
logJobEvent(EventType.QuantityChanged, afterJob, beforeJob.CustomQuantity, afterJob.CustomQuantity);
DateTime? beforeStart = beforeJob.ProcessingStart != null ? beforeJob.ProcessingStart : beforeJob.PlannedStart;
DateTime? afterStart = afterJob.ProcessingStart != null ? afterJob.ProcessingStart : afterJob.PlannedStart;
if (beforeStart != afterStart)
logJobEvent(EventType.StartDateChanged, afterJob, beforeStart, afterStart);
DateTime? beforeEnd = beforeJob.ProcessingEnd != null ? beforeJob.ProcessingEnd : beforeJob.PlannedEnd;
DateTime? afterEnd = afterJob.ProcessingEnd != null ? afterJob.ProcessingEnd : afterJob.PlannedEnd;
if (beforeEnd != afterEnd)
logJobEvent(EventType.EndDateChanged, afterJob, beforeEnd, afterEnd);
TimeSpan? beforeBuffer = beforeJob.DueDate != null ? (beforeJob.DueDate - beforeEnd) : new TimeSpan(0L);
TimeSpan? afterBuffer = afterJob.DueDate != null ? (afterJob.DueDate - afterEnd) : new TimeSpan(0L);
if (beforeBuffer != afterBuffer)
logJobEvent(EventType.BufferChanged, afterJob, beforeBuffer, afterBuffer);
}
//Collect the Batches in one list of batches
CollectBatches();
//Log all the Batches
LogBatches(batchesList);
}
/*
* Collectes the events in one batch
*/
private void logJobEvent(EventType evtType, Job afterJob, string oldValue, string newValue)
{
var eventGuid = Guid.NewGuid();
JPIEventEntity evt = new JPIEventEntity();
evt.Value = newValue;
evt.PrevValue = oldValue;
evt.ObjectGuid = afterJob.Guid.ToString();
evt.PartitionKey = myCustomer.ID; //customer guid
evt.RowKey = eventGuid.ToString();
evt.EventType = evtType;
evt.CustomerName = myCustomer.Name;
evt.User = myUser.Email;
evt.ParentName = null;
evt.ObjectType = JOB;
evt.ObjectName = afterJob.Name;
evt.CreatedAt = DateTime.Now;
batch.Add(evt);
}
/*
* Collectes the Events lists in an enumerable of Batches (max capacity of a single batch insertion is 100).
*/
private void CollectBatches()
{
//batchesList = new List<List<JPIEventEntity>>();
if (batch.Count > 0)
{
int rest = batch.Count;
var nextBatch = new List<JPIEventEntity>();
if (batch.Count > MaxBatchSize) //MaxBatchSize = 100
{
foreach (var item in batch)
{
nextBatch.Add(item);
rest = rest - 1; //rest = rest - (MaxBatchSize * hundreds);
if (rest < MaxBatchSize && nextBatch.Count == (batch.Count % MaxBatchSize))
{
batchesList.Add(nextBatch);
}
else if (nextBatch.Count == MaxBatchSize)
{
batchesList.Add(nextBatch);
nextBatch = new List<JPIEventEntity>();
}
}
}
else
{
batchesList.Add(batch);
}
}
}
private void LogBatches(List<List<JPIEventEntity>> batchesList)
{
if (batchesList.Count > 0)
{
JPIEventHandler.LogBatches(batchesList);
}
}
/*
* Insert Events into database
*/
public static void LogBatches(List<List<JPIEventEntity>> batchesList)
{
foreach (var batch in batchesList)
{
var batchOperationObj = new TableBatchOperation();
//Iterating through each batch entities
foreach (var Event in batch)
{
batchOperationObj.InsertOrReplace(Event);
}
var res = table.ExecuteBatch(batchOperationObj);
}
}
Inside the 'checkForJPIEventsToSend' method, I'm checking if there's any changes or updates in the calculations and insert events (hundreds or even thousands of lines) into the cosmos db table as batches.
After putting the method in a separate thread (as shown above) I still have an EXTRA LOADING duration of 2 to 4 seconds after every operation, which is something critical and bad for us.
Am I using the multi-threading correctly?
Thank you in advance.
As I understand your situation you have a front end application such as a desktop app or a website that creates requests. For each request you
Perform some calculations
Write some data to storage (Cosmos DB)
It is unclear whether you must display some result to the front end after these steps are complete. Your options depend on this question.
Scenario 1: The front end is waiting for the results of the calculations or database changes
The front end requires some result from the calculations or database changes, so the user is forced to wait for this to complete. However you want to avoid freezing your front end whilst you perform the long running tasks.
The solution most developers reach for here is to perform the work in a background thread. Your main thread waits for this background thread to complete and return a result, at which point the main/UI thread will update the front end with the result. This is because the main thread is often the only thread allowed to update the front end.
How you offload the work to a background thread depends on the type of work load you have. If the work is mostly I/O such as File I/O, Network I/O (writing to Azure CosmosDB) then you want to use the Async methods of the Cosmos API and async/await.
See https://stackoverflow.com/a/18033198/6662878
If the work you are doing is mostly CPU based, then threads will only speed up the processing if the problem can be broken into parts and run in parallel across multiple background threads. If the problem cannot be broken down and parallelised then running the work on a single background thread has a small cost associated with thread switching, but in turn this frees up the main/UI thread whilst the CPU based work is in progress in the background.
See https://learn.microsoft.com/en-us/dotnet/standard/asynchronous-programming-patterns/consuming-the-task-based-asynchronous-pattern
You will need to think about how you handle exceptions that occur on background threads, and how the code you run in your background thread will respond to a request to stop processing.
See https://learn.microsoft.com/en-us/dotnet/standard/threading/canceling-threads-cooperatively
Caveat: if any thread is carrying out very CPU intensive work (such as compressing or encrypting large amounts of data, encoding audio or video etc) this can often cause the front end to freeze, stop responding, drop network requests etc. If you have some processor intensive work to complete you need to think about how the work is spread over CPU cores, or CPUs.
Scenario 2: The front end does not need to display any specific result for the request
In this scenario you have more flexibilty about how and when you perform your background work because you can simply respond to the front end request with an acknowledgement that the request is received and will be processed in the (near) future. For example a Web API may respond with a 201 ACCEPTED HTTP response code to signal this.
You now want to queue the requests and process them somewhere other than your main/UI thread. There are a number of options, background threads being one of them, though not the simplest. You may also consider using a framework like https://www.hangfire.io/.
Another popular approach is to create a completely separate service or microservice that is responsible for your picking up requests from a queue and performing the work.
See https://learn.microsoft.com/en-us/dotnet/architecture/microservices/architect-microservice-container-applications/asynchronous-message-based-communication
Multithreading should come with a big warning message. Sometimes it is unavoidable, but it is always difficult and troublesome to get right. The C# APIs have evolved over time and so there's a lot to learn and a lot of ground to cover. It is often seen as a quick option to convert an application to be multithreaded, though you should be wary of this. Although more complex architectures as discussed in the link above seem overly burdensome, they invariably force you to think through a number of issues that come up when you begin to split up your application into threads, or processes, or services.
I've checked into the documentation regarding Scheduling with Azure Service Bus, but I am not clear on how exactly to send a message from a "disconnected" Bus.
Here's how I've configured my Service that is processing messages on the server:
builder.AddMassTransit(mt =>
{
mt.AddConsumers(cqrsAssembly);
mt.AddBus(context => Bus.Factory.CreateUsingAzureServiceBus(x =>
{
x.RequiresSession = true;
x.MaxConcurrentCalls = 500;
x.MessageWaitTimeout = TimeSpan.FromMinutes(5);
x.UseRenewLock(TimeSpan.FromMinutes(4));
x.UseServiceBusMessageScheduler();
var host = x.Host(serviceUri, h =>
{
h.SharedAccessSignature(s =>
{
s.KeyName = "key-name";
s.SharedAccessKey = "access-key";
s.TokenTimeToLive = TimeSpan.FromDays(1);
s.TokenScope = TokenScope.Namespace;
});
h.OperationTimeout = TimeSpan.FromMinutes(2);
});
x.ReceiveEndpoint(host, $"mt.myqueue", ep =>
{
ep.RequiresSession = true;
ep.MaxConcurrentCalls = 500;
ep.RemoveSubscriptions = true;
ep.UseMessageRetry(r =>
{
r.Interval(4, TimeSpan.FromSeconds(30));
r.Handle<TransientCommandException>();
});
ep.ConfigureConsumers(context);
});
});
});
I've explicitly called UseServiceBusMessageScheduler().
In the project that is creating and sending messages to the queue (runs in a different context, so is done to "send only"), we have this:
var bus = Bus.Factory.CreateUsingAzureServiceBus(x =>
{
x.RequiresSession = true;
x.MessageWaitTimeout = TimeSpan.FromMinutes(5);
x.UseRenewLock(TimeSpan.FromMinutes(4));
x.Send<ICommand>(s => s.UseSessionIdFormatter(ctx => ctx.Message.SessionId ?? Guid.NewGuid().ToString()));
var host = x.Host(serviceUri, h =>
{
h.SharedAccessSignature(s =>
{
s.KeyName = "key-name";
s.SharedAccessKey = "key";
s.TokenTimeToLive = TimeSpan.FromDays(1);
s.TokenScope = TokenScope.Namespace;
});
h.OperationTimeout = TimeSpan.FromMinutes(2);
});
EndpointConvention.Map<ICommand>(new Uri($"{serviceUri.ToString()}mt.myqueue"));
EndpointConvention.Map<Command>(new Uri($"{serviceUri.ToString()}mt.myqueue"));
});
Now, to send a scheduled message, we do this:
var dest = "what?";
await bus.ScheduleSend(dest, scheduledEnqueueTimeUtc.Value, message);
I am unsure of what needs to be passed into the destinationAddress.
I've tried:
- serviceUri
- `{serviceUri}mt.myqueue"
But checking the queues, I don't see my message in either the base queue, the skipped queue or the error queue.
Am I missing some other configuration, and if not, how does one determine the destination queue?
Am using version 5.5.4 of Mass Transit, and every overload to ScheduleSend() requires it.
First of all, yes your Uri format is correct. In the end after formatting you need something like this :
new Uri(#"sb://yourdomain.servicebus.windows.net/yourapp/your_message_queue")
Also make sure you added when you configured your endPoint. (See the link below)
configurator.UseServiceBusMessageScheduler();
If you follow the Mass-Transit documentation, scheduling is done from a ConsumeContext. See Mass-Transit Azure Scheduling
public class ScheduleNotificationConsumer :
IConsumer<AssignSeat>
{
Uri _schedulerAddress;
Uri _notificationService;
public async Task Consume(ConsumeContext<AssignSeat> context)
{
if(context.Message.ReservationTime - DateTime.Now < TimeSpan.FromHours(8))
{
// assign the seat for the reservation
}
else
{
// seats can only be assigned eight hours before the reservation
context.ScheduleMessage(context.Message.ReservationTime - TimeSpan.FromHours(8), context.Message);
}
}
}
However in a use case we faced this week we needed to schedule from outside a consumeContext, or simply didn't want to forward the context down to where we scheduled. When using IBusControl.ScheduleSend we get no error feedback, but we also don't really get any scheduling done.
After looking at what Mass-Transit does it turns out that from an IBusControl, it creates a new scheduling provider. Whereas from the Context it uses the ServiceBusScheduleMessageProvider.
So what we're doing now until we clean up this bit, is calling the ServiceBusScheduleMessageProvider outright.
await new ServiceBusScheduleMessageProvider(_busControl).ScheduleSend(destinationUri
, scheduleDateTime.UtcDateTime
, Task.FromResult<T>(message)
, Pipe.Empty<SendContext>()
, default);
Hope it makes sense and helps a bit.
I have a an aync method that is looped:
private Task<HttpResponseMessage> GetResponseMessage(Region region, DateTime startDate, DateTime endDate)
{
var longLatString = $"q={region.LongLat.Lat},{region.LongLat.Long}";
var startDateString = $"{startDateQueryParam}={ConvertDateTimeToApixuQueryString(startDate)}";
var endDateString = $"{endDateQueryParam}={ConvertDateTimeToApixuQueryString(endDate)}";
var url = $"http://api?key={Config.Key}&{longLatString}&{startDateString}&{endDateString}";
return Client.GetAsync(url);
}
I then take the response and save it to my ef core database, however in some instances I get this Exception message: The Operaiton was canceled
I really dont understand that. This is a TCP handshake issue?
Edit:
For context I am making many of these calls, passing response to the method that writes to db (which is also so slow Its unbelievable):
private async Task<int> WriteResult(Response apiResponse, Region region)
{
// since context is not thread safe we ensure we have a new one for each insert
// since a .net core app can insert data at the same time from different users different instances of context
// must be thread safe
using (var context = new DalContext(ContextOptions))
{
var batch = new List<HistoricalWeather>();
foreach (var forecast in apiResponse.Forecast.Forecastday)
{
// avoid inserting duplicates
var existingRecord = context.HistoricalWeather
.FirstOrDefault(x => x.RegionId == region.Id &&
IsOnSameDate(x.Date.UtcDateTime, forecast.Date));
if (existingRecord != null)
{
continue;
}
var newHistoricalWeather = new HistoricalWeather
{
RegionId = region.Id,
CelsiusMin = forecast.Day.Mintemp_c,
CelsiusMax = forecast.Day.Maxtemp_c,
CelsiusAverage = forecast.Day.Avgtemp_c,
MaxWindMph = forecast.Day.Maxwind_mph,
PrecipitationMillimeters = forecast.Day.Totalprecip_mm,
AverageHumidity = forecast.Day.Avghumidity,
AverageVisibilityMph = forecast.Day.Avgvis_miles,
UvIndex = forecast.Day.Uv,
Date = new DateTimeOffset(forecast.Date),
Condition = forecast.Day.Condition.Text
};
batch.Add(newHistoricalWeather);
}
context.HistoricalWeather.AddRange(batch);
var inserts = await context.SaveChangesAsync();
return inserts;
}
Edit: I am making 150,000 calls. I know this is questionable since It all goes in memory I guess before even doing a save but this is where I got to in trying to make this run faster... only I guess my actual writing code is blocking :/
var dbInserts = await Task.WhenAll(
getTasks // the list of all api get requests
.Select(async x => {
// parsed can be null if get failed
var parsed = await ParseApixuResponse(x.Item1); // readcontentasync and just return the deserialized json
return new Tuple<ApiResult, Region>(parsed, x.Item2);
})
.Select(async x => {
var finishedGet = await x;
if(finishedGet.Item1 == null)
{
return 0;
}
return await writeResult(finishedGet.Item1, finishedGet.Item2);
})
);
.net core has a DefaultConnectionLimit setting as answered in comments.
this limits outgoing connections to specific domains to ensure all ports are not taken etc.
i did my parallel work incorrectly causing it to go over the limit - which everything i read says should not be 2 on .net core but it was - and that caused connections to close before receiving responses.
I made it greater, did parallel work correctly, lowered it again.
Is there a simple way of stopping a hangfire.io job from enqueuing if one is already enqueued?
Looking at the jobfilterattribute, nothing stands out as how to get the state of anything on the server. Can I use the connection objects and query the store?
Thanks
Have a look at the following gist by the library owner https://gist.github.com/odinserj/a8332a3f486773baa009
This should prevent the same job from being en-queued more than once by querying the fingerprint.
You can activate it per background job by decorating the method with the attribute [DisableMultipleQueuedItemsFilter].
Or you can enable it globally GlobalJobFilters.Filters.Add(new DisableMultipleQueuedItemsFilter());
I am using following code block to check to add new job or not, depending on its current state: (I know that i am currently looking at only first 1000 jobs. You can implement your type of logic))
private static bool IsOKToAddJob(string JobName, string QueueName, out string NotOKKey)
{
try
{
var monapi = JobStorage.Current.GetMonitoringApi();
var processingJobs = monapi.ProcessingJobs(0, 1000);
NotOKKey = processingJobs.Where(j => j.Value.Job.ToString() == JobName).FirstOrDefault().Key;
if (!string.IsNullOrEmpty(NotOKKey)) return false;
var scheduledJobs = monapi.ScheduledJobs(0, 1000);
NotOKKey = scheduledJobs.Where(j => j.Value.Job.ToString() == JobName).FirstOrDefault().Key;
if (!string.IsNullOrEmpty(NotOKKey)) return false;
var enqueuedJobs = monapi.EnqueuedJobs(QueueName, 0, 1000);
NotOKKey = enqueuedJobs.Where(j => j.Value.Job.ToString() == JobName).FirstOrDefault().Key;
if (!string.IsNullOrEmpty(NotOKKey)) return false;
NotOKKey = null;
return true;
}
catch (Exception ex)
{
//LOG your Exception;
}
}
And the usage is simple:
if (IsOKToAddJob(YOURJOBNAME, QueueName, out NOTOKKey))
var id = BackgroundJob.Enqueue(() =>YOURMETHOD());
//rest
I have a fairly simple WPF application that uses Entity Framework. The main page of the application has a list of records that I am getting from a database on startup.
Each record has a picture, so the operation can be a little slow when the wireless signal is poor. I'd like this (and many of my SQL operations) to perform in the background if possible. I have async/await setup and at first it seemed to be working exactly as I wanted but now I'm seeing that my application is becoming unresponsive when accessing the DB.
Eventually I'm thinking I'm going to load up the text in one query and the images in another background operation and load them as they come in. This way I get the important stuff right away and the pictures can come in in the background, but the way things are going it's still looking like it will lock up if I do this.
On top of that, I'm trying to implement something to handle connectivity issues (in case the wifi cuts out momentarily) so that the application notifies the user of a connection issue, automatically retries a few times, etc. I put a try catch for SQL exception which seems to be working for me, but the whole application locks up for about a minute while it is trying to connect to the DB.
I tried testing my async/await using await Task.Delay() and everything is very responsive as expected while awaiting the delay, but everything locks up when awaiting the .ToListAsync(). Is this normal and expected? My understanding of async/await is pretty limited.
My code is kind of messy (I'm new) but it does what I need it to do for the most part. I understand there's probably plenty of improvements I can make and better ways to do things, but one step at a time here. My main goal right now is to keep the application from crashing during database accessing exceptions and to keep the user notified of what the application is doing (searching, trying to access db, unable to reach DB and retrying, etc) as opposed to being frozen, which is what they're going to think when they see it being unresponsive for over a minute.
Some of my code:
In my main view model
DataHelper data = new DataHelper();
private async void GetQualityRegisterQueueAsync()
{
try
{
var task = data.GetQualityRegisterAsync();
IsSearching = true;
await task;
IsSearching = false;
QualityRegisterItems = new ObservableCollection<QualityRegisterQueue>(task.Result);
OrderQualityRegisterItems();
}
catch (M1Exception ex)
{
Debug.WriteLine(ex.Message);
Debug.WriteLine("QualityRegisterLogViewModel.GetQualityRegisterQueue() Operation Failed");
}
}
My Data Helper Class
public class DataHelper
{
private bool debugging = false;
private const int MAX_RETRY = 2;
private const double LONG_WAIT_SECONDS = 5;
private const double SHORT_WAIT_SECONDS = 0.5;
private static readonly TimeSpan longWait = TimeSpan.FromSeconds(LONG_WAIT_SECONDS);
private static readonly TimeSpan shortWait = TimeSpan.FromSeconds(SHORT_WAIT_SECONDS);
private enum RetryableSqlErrors
{
ServerNotFound = -1,
Timeout = -2,
NoLock = 1204,
Deadlock = 1205,
}
public async Task<List<QualityRegisterQueue>> GetQualityRegisterAsync()
{
if(debugging) await Task.Delay(5000);
var retryCount = 0;
using (M1Context m1 = new M1Context())
{
for (; ; )
{
try
{
return await (from a in m1.QualityRegisters
where (a.qanClosed == 0)
//orderby a.qanAssignedDate descending, a.qanOpenedDate
orderby a.qanAssignedDate.HasValue descending, a.qanAssignedDate, a.qanOpenedDate
select new QualityRegisterQueue
{
QualityRegisterID = a.qanQualityRegisterID,
JobID = a.qanJobID.Trim(),
JobAssemblyID = a.qanJobAssemblyID,
JobOperationID = a.qanJobOperationID,
PartID = a.qanPartID.Trim(),
PartRevisionID = a.qanPartRevisionID.Trim(),
PartShortDescription = a.qanPartShortDescription.Trim(),
OpenedByEmployeeID = a.qanOpenedByEmployeeID.Trim(),
OpenedByEmployeeName = a.OpenedEmployee.lmeEmployeeName.Trim(),
OpenedDate = a.qanOpenedDate,
PartImage = a.JobAssembly.ujmaPartImage,
AssignedDate = a.qanAssignedDate,
AssignedToEmployeeID = a.qanAssignedToEmployeeID.Trim(),
AssignedToEmployeeName = a.AssignedEmployee.lmeEmployeeName.Trim()
}).ToListAsync();
}
catch (SqlException ex)
{
Debug.WriteLine("SQL Exception number = " + ex.Number);
if (!Enum.IsDefined(typeof(RetryableSqlErrors), ex.Number))
throw new M1Exception(ex.Message, ex);
retryCount++;
if (retryCount > MAX_RETRY) throw new M1Exception(ex.Message, ex); ;
Debug.WriteLine("Retrying. Count = " + retryCount);
Thread.Sleep(ex.Number == (int)RetryableSqlErrors.Timeout ?
longWait : shortWait);
}
}
}
}
}
Edit: Mostly looking for general guidance here, though a specific example of what to do would be great. For these types of operations where I am downloading data, is it just a given that if I need the application to be responsive I need to be making multiple threads? Is that a common solution to this type of problem? Is this not something I should be expecting async/await to solve?
If you call this method from your UI thread, you will overload the capture of UI thread context and back on itself. Also, your service will not be necessarily "Performant" because it must wait until the UI thread is free before it can continue.
The solution is simple: just call the method passing the ConfigureAwait "false" parameter when you made the call.
.ToListAsync().ConfigureAwaiter(false);
I hope it helps