Develop Reference

Hangfire in single application on multiple physical servers

I am running hangfire in a single web application, my application is being run on 2 physical servers but hangfire is in 1 database.
At the moment, i am generating a server for each queue, because each queue i need to run 1 worker at a time and they must be in order. I set them up like this
// core
services.AddHangfire(options =>
{
options.SetDataCompatibilityLevel(CompatibilityLevel.Version_170);
options.UseSimpleAssemblyNameTypeSerializer();
options.UseRecommendedSerializerSettings();
options.UseSqlServerStorage(appSettings.Data.DefaultConnection.ConnectionString, storageOptions);
});
// add multiple servers, this way we get to control how many workers are in each queue
services.AddHangfireServer(options =>
{
options.ServerName = "workflow-queue";
options.WorkerCount = 1;
options.Queues = new string[] { "workflow-queue" };
options.SchedulePollingInterval = TimeSpan.FromSeconds(10);
});
services.AddHangfireServer(options =>
{
options.ServerName = "alert-schedule";
options.WorkerCount = 1;
options.Queues = new string[] { "alert-schedule" };
options.SchedulePollingInterval = TimeSpan.FromMinutes(1);
});
services.AddHangfireServer(options =>
{
options.ServerName = string.Format("trigger-schedule");
options.WorkerCount = 1;
options.Queues = new string[] { "trigger-schedule" };
options.SchedulePollingInterval = TimeSpan.FromMinutes(1);
});
services.AddHangfireServer(options =>
{
options.ServerName = "report-schedule";
options.WorkerCount = 1;
options.Queues = new string[] { "report-schedule" };
options.SchedulePollingInterval = TimeSpan.FromMinutes(1);
});
services.AddHangfireServer(options =>
{
options.ServerName = "maintenance";
options.WorkerCount = 5;
options.Queues = new string[] { "maintenance" };
options.SchedulePollingInterval = TimeSpan.FromMinutes(10);
});
My problem is that it is generating multiple queues on the servers, with different ports.
In my code i am then trying to stop jobs from running if they are queued/retrying, but if the job is being run on a different physical server, it is not found and queued again.
Here is the code to check if its running already
public async Task<bool> IsAlreadyQueuedAsync(PerformContext context)
{
var disableJob = false;
var monitoringApi = JobStorage.Current.GetMonitoringApi();
// get the jobId, method and queue using performContext
var jobId = context.BackgroundJob.Id;
var methodInfo = context.BackgroundJob.Job.Method;
var queueAttribute = (QueueAttribute)Attribute.GetCustomAttribute(context.BackgroundJob.Job.Method, typeof(QueueAttribute));
// enqueuedJobs
var enqueuedjobStatesToCheck = new[] { "Processing" };
var enqueuedJobs = monitoringApi.EnqueuedJobs(queueAttribute.Queue, 0, 1000);
var enqueuedJobsAlready = enqueuedJobs.Count(e => e.Key != jobId && e.Value != null && e.Value.Job != null && e.Value.Job.Method.Equals(methodInfo) && enqueuedjobStatesToCheck.Contains(e.Value.State));
if (enqueuedJobsAlready > 0)
disableJob = true;
// scheduledJobs
if (!disableJob)
{
// check if there are any scheduledJobs that are processing
var scheduledJobs = monitoringApi.ScheduledJobs(0, 1000);
var scheduledJobsAlready = scheduledJobs.Count(e => e.Key != jobId && e.Value != null && e.Value.Job != null && e.Value.Job.Method.Equals(methodInfo));
if (scheduledJobsAlready > 0)
disableJob = true;
}
// failedJobs
if (!disableJob)
{
var failedJobs = monitoringApi.FailedJobs(0, 1000);
var failedJobsAlready = failedJobs.Count(e => e.Key != jobId && e.Value != null && e.Value.Job != null && e.Value.Job.Method.Equals(methodInfo));
if (failedJobsAlready > 0)
disableJob = true;
}
// if runBefore is true, then lets remove the current job running, else it will write a "successful" message in the logs
if (disableJob)
{
// use hangfire delete, for cleanup
BackgroundJob.Delete(jobId);
// create our sqlBuilder to remove the entries altogether including the count
var sqlBuilder = new SqlBuilder()
.DELETE_FROM("Hangfire.[Job]")
.WHERE("[Id] = {0};", jobId);
sqlBuilder.Append("DELETE TOP(1) FROM Hangfire.[Counter] WHERE [Key] = 'stats:deleted' AND [Value] = 1;");
using (var cmd = _context.CreateCommand(sqlBuilder))
await cmd.ExecuteNonQueryAsync();
return true;
}
return false;
}
Each method has something like the following attributes as well
public interface IAlertScheduleService
{
[Hangfire.Queue("alert-schedule")]
[Hangfire.DisableConcurrentExecution(60 * 60 * 5)]
Task RunAllAsync(PerformContext context);
}
Simple implementation of the interface
public class AlertScheduleService : IAlertScheduleService
{
public Task RunAllAsync(PerformContext context)
{
if (IsAlreadyQueuedAsync(context))
return;
// guess it isnt queued, so run it here....
}
}
Here is how i am adding my scheduled jobs
//// our recurring jobs
//// set these to run hourly, so they can play "catch-up" if needed
RecurringJob.AddOrUpdate<IAlertScheduleService>(e => e.RunAllAsync(null), Cron.Hourly(0), queue: "alert-schedule");
Why does this happen? How can i stop it happening?

Somewhat of a blind shot, preventing a job to be queued if a job is already queued in the same queue.
The try-catch logic is quite ugly but I have no better idea right now...
Also, really not sure the lock logic always prevents from having two jobs in EnqueudState, but it should help anyway. Maybe mixing with an IApplyStateFilter.
public class DoNotQueueIfAlreadyQueued : IElectStateFilter
{
public void OnStateElection(ElectStateContext context)
{
if (context.CandidateState is EnqueuedState)
{
EnqueuedState es = context.CandidateState as EnqueuedState;
IDisposable distributedLock = null;
try
{
while (distributedLock == null)
{
try
{
distributedLock = context.Connection.AcquireDistributedLock($"{nameof(DoNotQueueIfAlreadyQueued)}-{es.Queue}", TimeSpan.FromSeconds(1));
}
catch { }
}
var m = context.Storage.GetMonitoringApi();
if (m.EnqueuedCount(es.Queue) > 0)
{
context.CandidateState = new DeletedState();
}
}
finally
{
distributedLock.Dispose();
}
}
}
}
The filter can be declared as in this answer

There seems to be a bug with your currently used hangfire storage implementation:
https://github.com/HangfireIO/Hangfire/issues/1025
The current options are:
Switching to HangFire.LiteDB as commented here: https://github.com/HangfireIO/Hangfire/issues/1025#issuecomment-686433594
Implementing your own logic to enqueue a job, but this would take more effort.
Making your job execution idempotent to avoid side effects in case it's executed multiple times.
In either option, you should still apply DisableConcurrentExecution and make your job execution idempotent as explained below, so i think you can just go with below option:
Applying DisableConcurrentExecution is necessary, but it's not enough as there are no reliable automatic failure detectors in distributed systems. That's the nature of distributed systems, we usually have to rely on timeouts to detect failures, but it's not reliable.
Hangfire is designed to run with at-least-once execution semantics. Explained below:
One of your servers may be executing the job, but it's detected as being failed due to various reasons. For example: your current processing server does not send heartbeats in time due to a temporary network issue or due to temporary high load.
When the current processing server is assumed to be failed (but it's not), the job will be scheduled to another server which causes it to be executed more than once.
The solution should be still applying DisableConcurrentExecution attribute as a best effort to prevent multiple executions of the same job, but the main thing is that you need to make the execution of the job idempotent which does not cause side effects in case it's executed multiple times.
Please refer to some quotes from https://docs.hangfire.io/en/latest/background-processing/throttling.html:
Throttlers apply only to different background jobs, and there’s no
reliable way to prevent multiple executions of the same background job
other than by using transactions in background job method itself.
DisableConcurrentExecution may help a bit by narrowing the safety
violation surface, but it heavily relies on an active connection,
which may be broken (and lock is released) without any notification
for our background job.
As there are no reliable automatic failure detectors in distributed
systems, it is possible that the same job is being processed on
different workers in some corner cases. Unlike OS-based mutexes,
mutexes in this package don’t protect from this behavior so develop
accordingly.
DisableConcurrentExecution filter may reduce the probability of
violation of this safety property, but the only way to guarantee it is
to use transactions or CAS-based operations in our background jobs to
make them idempotent.
You can also refer to this as Hangfire timeouts behavior seems to be dependent on storage as well: https://github.com/HangfireIO/Hangfire/issues/1960#issuecomment-962884011

`WaitAll()` or `WhenAll` when expecting data

I've never attempted to use WaitAll() or WhenAll() when running async functionality. After looking at many documentations, SO posts, and tutorials, I haven't found enough information for this, so here I am.
I'm trying to figure out the best/proper way(s) to do the following:
Using EF6, get data as List<Entity>.
Iterate through each Entity and call an external API to perform some action.
External API returns data per Entity which I need to store on the same Entity.
Currently I have built (not tested) the following (without the error handling code):
public IEnumerable<Entity> Process() {
bool hasChanged = false;
var data = _db.Entity.Where(x => !x.IsRegistered);
foreach (var entity in data) {
var result = await CallExternalApi(entity.Id, entity.Name);
entity.RegistrationId = result.RegistrationId;
entity.IsRegistered = true;
_db.Entry(entity).State = EntityState.Modified;
hasChanges = true;
}
if (hasChanges) {
uow.Commit();
}
return data;
}
I feel like I may be able to take advantage of some other functionality/feature in async, but if I can I'm not sure how to implement it here.
Any guidance is really appreciated.
Update
The API I'm calling is the Zoom Api to add Registrants. While they do have an route to batch add Registrants, it does not return the RegistrantId and the Join Url I need.

First, figure out if your external API might have a way to get all the items you want in a batch. If it does, use that instead of sending a whole bunch of requests.
If you need to send a separate request for each item, but want to do it concurrently, you could do this:
public async Task<IReadOnlyCollection<Entity>> Process() {
var data = _db.Entity.Where(x => !x.IsRegistered).ToList();
if(!data.Any()) { return data; }
var entityResultTasks = data
.Select(async entity => new { entity, result = await CallExternalApi(entity.Id, entity.Name) })
.ToList();
var entityResults = await Task.WhenAll(entityResultTasks);
foreach (var entityResult in entityResults) {
var entity = entityResult.entity;
var result = entityResult.result;
entity.RegistrationId = result.RegistrationId;
entity.IsRegistered = true;
_db.Entry(entity).State = EntityState.Modified;
}
uow.Commit();
return data;
}
You will want to watch out for possible concurrency limits on the target source. Consider using Chunk to break your work into batches of acceptable sizes, or leveraging a semaphore or something to throttle the number of calls you're making.

Why Does Await Not Appear to Prevent Second Operation on EF Context

Within an ASP.NET MVC Application I'm recieving the following error message for one of my controller methods that uses my Entity Framework context.
A second operation started on this context before a previous asynchronous operation completed. Use 'await' to ensure that any asynchronous operations have completed before calling another method on this context. Any instance members are not guaranteed to be thread safe.
I'm aware that you cannot run queries in parallel, and everything appears to be awaited properly. If I debug the program and step and inspect some of the data returned from EF then it works, probably because this forces the queries to complete.
EDIT If I place a breakpoint at the null check in the controller method and inspect the data of shipmentDetail the exception is NOT thrown.
Here's a snippit of the code:
Controller Method:
[Route("{id:int}/Deliveries")]
public async Task<ActionResult> DeliveryInfo(int id)
{
var shipmentDetail = await db.ShipmentDetails.SingleOrDefaultAsync(s => s.Id == id);
if (shipmentDetail == null)
return HttpNotFound(string.Format("No shipment detail found with id {0}", id));
var model = await DeliveryInfoModel.CreateModel(db, shipmentDetail);
return View("DeliveryInfo", model);
}
CreateModel Method:
public static async Task<DeliveryInfoModel> CreateModel(Context db, ShipmentDetail shipment)
{
DeliveryInfoModel model = new DeliveryInfoModel()
{
ShipmentInfo = shipment
};
//initialize processing dictionary
Dictionary<int, bool> boxesProcessed = new Dictionary<int, bool>();
List<DeliveryBoxStatus> statuses = new List<DeliveryBoxStatus>();
for (int i = 1; i <= shipment.BoxCount; i++ )
{
boxesProcessed.Add(i, false);
}
//work backwards through process
//check for dispositions from this shipment
if(shipment.Dispositions.Count > 0)
{
foreach (var d in shipment.Dispositions)
{
DeliveryBoxStatus status = new DeliveryBoxStatus()
{
BoxNumber = d.BoxNumber,
LastUpdated = d.Date,
Status = d.Type.GetDescription().ToUpper()
};
statuses.Add(status);
boxesProcessed[d.BoxNumber] = true;
}
}
//return if all boxes have been accounted for
if (boxesProcessed.Count(kv => kv.Value) == shipment.BoxCount)
{
model.BoxStatuses = statuses;
return model;
}
//check for deliveries
if(shipment.Job_Detail.Count > 0)
{
foreach (var j in shipment.Job_Detail.SelectMany(d => d.DeliveryInfos))
{
DeliveryBoxStatus status = new DeliveryBoxStatus()
{
BoxNumber = j.BoxNumber,
LastUpdated = j.Job_Detail.To_Client.GetValueOrDefault(),
Status = "DELIVERED"
};
statuses.Add(status);
boxesProcessed[j.BoxNumber] = true;
}
}
//check for items still in processing & where
foreach (int boxNum in boxesProcessed.Where(kv => !kv.Value).Select(kv => kv.Key))
{
//THIS LINE THROWS THE EXCEPTION
var processInfo = await db.Processes.Where(p => p.Jobs__.Equals(shipment.Job.Job__, StringComparison.InvariantCultureIgnoreCase) && p.Shipment == shipment.ShipmentNum && p.Box == boxNum)
.OrderByDescending(p => p.date)
.FirstOrDefaultAsync();
//process returned data
//...
}
model.BoxStatuses = statuses;
return model;
}
I'm not completely sure if it's because of the query made in the controller, or because of the queries made in the loop that aren't completing causing this behavior. Is there something I'm not understanding about when the queries are actually made/returned due to EF's laziness, or how async/await works in this situation? I have a lot of other methods & controllers that make async EF calls and haven't run into this previously.
EDIT
My context is injected into my controller using Ninject as my IoC container. Here's its config inside of NinjectWebCommon's RegisterServices method:
kernel.Bind<Context>().ToSelf().InRequestScope();

Avoid lazy loading when using async with Entity Framework. Instead, either load the data you need first, or use Include()'s to ensure the data you need is loaded with the query.
https://msdn.microsoft.com/en-gb/magazine/dn802603.aspx
Current State of Async Support
... Async
support was added to Entity Framework (in the EntityFramework NuGet
package) in version 6. You do have to be careful to avoid lazy
loading when working asynchronously, though, because lazy loading is
always performed synchronously. ...
(Emphasis mine)
Also:
https://entityframework.codeplex.com/wikipage?title=Task-based%20Asynchronous%20Pattern%20support%20in%20EF.#ThreadSafety

How properly to throttle access to DocumentDb from WebJobs

I have an Azure WebKob with blob and queue triggers to save data to Azure DocumentDb.
From time to time I'm getting an error:
Microsoft.Azure.Documents.RequestRateTooLargeException: Message: {"Errors":["Request rate is large"]}
Currently I throttle requests using this code. A WebJob function:
public async Task ParseCategoriesFromCsv(...)
{
double find = 2.23, add = 5.9, replace = 10.67;
double requestCharge = Math.Round(find + Math.Max(add, replace));
await categoryProvider.SaveCategories(requestCharge , categories);
}
Category provider to manipulate document db client:
public async Task<ResourceResponse<Document>[]> SaveCategories(double requestCharge, Category[] categories)
{
var requestDelay = TimeSpan.FromSeconds(60.0 / (collectionOptions.RequestUnits / requestCharge));
var scheduler = new IntervalTaskScheduler(requestDelay, Scheduler.Default); // Rx
var client = new DocumentClient(endpoint, authorizationKey,
new ConnectionPolicy
{
ConnectionMode = documentDbOptions.ConnectionMode,
ConnectionProtocol = documentDbOptions.ConnectionProtocol
});
return await Task.WhenAll(documents.Select(async d =>
await scheduler.ScheduleTask(
() => client.PutDocumentToDb(collectionOptions.CollectionLink, d.SearchIndex, d))));
}
Task scheduler to throttle/measure/synchronize requests:
private readonly Subject<Action> _requests = new Subject<Action>();
private readonly IDisposable _observable;
public IntervalTaskScheduler(TimeSpan requestDelay, IScheduler scheduler)
{
_observable = _requests.Select(i => Observable.Empty<Action>()
.Delay(requestDelay)
.StartWith(i))
.Concat()
.ObserveOn(scheduler)
.Subscribe(action => action());
}
public Task<T> ScheduleTask<T>(Func<Task<T>> request)
{
var tcs = new TaskCompletionSource<T>();
_requests.OnNext(async () =>
{
try
{
T result = await request();
tcs.SetResult(result);
}
catch (Exception ex)
{
tcs.SetException(ex);
}
});
return tcs.Task;
}
So it's basically a number of constants from ResourceResponse<Document>.RequestCharge but:
When I have 1 queue triggered it works fine but when 8 queue it throws an error.
If increase request charge in 8 times then 8 queues work fine but just 1 works 8 times slower than it could.
What a throttling/measuring/synchronization mechanism could work here well?

Starting .NET SDK 1.8.0, we automatically handle the Request Rate too large exceptions to a reasonable extent(will retry 9 times by default and honor the retry after returned from server for next retry).
In case you need greater control, you can configure the RetryOptions on the ConnectionPolicy instance that you pass in to the DocumentClient objectand we would override the default retry policy with it.
So you no longer need to add any custom logic for handling 429 exceptions in your application code like above.

When getting a 429 (Request rate too large) the response tells you how long to wait. There is a header x-ms-retry-after. This has a value. Wait for that time period in ms.
catch (AggregateException ex) when (ex.InnerException is DocumentClientException)
{
DocumentClientException dce = (DocumentClientException)ex.InnerException;
switch ((int)dce.StatusCode)
{
case 429:
Thread.Sleep(dce.RetryAfter);
break;
default:
Console.WriteLine(" Failed: {0}", ex.InnerException.Message);
throw;
}
}

It seems to me that you should be able to do this with your SaveCategories method to make it work nicely with Rx:
public IObservable<ResourceResponse<Document>[]> SaveCategories(double requestCharge, Category[] categories)
{
var requestDelay = TimeSpan.FromSeconds(60.0 / (collectionOptions.RequestUnits / requestCharge));
var client = new DocumentClient(endpoint, authorizationKey,
new ConnectionPolicy
{
ConnectionMode = documentDbOptions.ConnectionMode,
ConnectionProtocol = documentDbOptions.ConnectionProtocol
});
return
Observable.Interval(requestDelay)
.Zip(documents, (delay, doc) => doc)
.SelectMany(doc => Observable.FromAsync(() => client.PutDocumentToDb(collectionOptions.CollectionLink, doc.SearchIndex, doc)))
.ToArray();
}
This totally gets rid of your IntervalTaskScheduler class and ensures that you limit the request rate to one request per the requestDelay time span, but allows the response to take as long as needed. To .ToArray() call turns the IObservable<ResourceResponse<Document>> that returns many values into an IObservable<ResourceResponse<Document>[]> that returns a single array of values when the observable completes.
I couldn't test your code, so I tested a sample which I think simulates your code:
var r = new Random();
var a = Enumerable.Range(0, 1000);
var i = Observable.Interval(TimeSpan.FromSeconds(2.0));
var sw = Stopwatch.StartNew();
var query =
i.Zip(a, (ii, aa) => aa)
.SelectMany(aa => Observable.Start(() =>
{
var x = sw.Elapsed.TotalMilliseconds;
Thread.Sleep(r.Next(0, 5000));
return x;
}))
.Select(x => new
{
started = x,
ended = sw.Elapsed.TotalMilliseconds
});
I got this kind of result which shows that the requests were throttled:
4026.2983 5259.7043
2030.1287 6940.2326
6027.0439 9664.1045
8027.9993 10207.0579
10028.1762 12301.4746
12028.3190 12711.4440
14040.7972 17433.1964
16040.9267 17574.5924
18041.0529 19077.5545

What Should I be using here? Threading? Async?

I am not sure what to use in this scenario.
I have an asp.net web api method that basically does this
Finds points of interests from foursquare near user.
Uses the foursquare locations to do queries in my database to find unique data about point of interest near user.
However since I need to store some of the foursquare information to link to my unique data to that location I decided to store all the information in my database and have my database act as my caching system.
This means anything new point of interest that comes in I have to insert into my database, check if it exists and if so then skip it or if it exists check the last refresh date(foursquare policy states all data must be refreshed after 30 day) and if it out past the refresh date I have to update the data.
I want to slow the user down and have to wait for the above to happen. I want my code to do step 1 and then do what I just mentioned while at the same time doing step 2.
Once step 2 finishes I want to return the data and let the user get on their way. If my caching system is not finished then it should keep going but not bog down the user.
I won't use any of these new results in step 2 as if I am inserting it then I won't have any data on that location at this time.
Not sure if I need to make a thread or use the async/await to achieve this.
Edit
Here is what I am trying to do
public HttpResponseMessage Get()
{
// this will do a foursquare lookup to find all stores near the user
// I want to insert them into my database and link it to my unquie data.
// stores pulled from foursquare will
// a) Be new and not in my database
// b) exist in my database but have been refreshed lately
// c) have not been refreshed in timeframe of foursquare policy
// THIS SHOULD WORK IN THE BACKGROUND
storeService.PointsOfInterestNearUser(80, -130); //As you can see it is
//void. Not sure where to put the async/await stuff
// find this product. Should be happening at the same time as above line.
var product = productService.FindProduct("Noodles");
//This will get returned to the user.
// the new stores taht are being added in StoreNearUser
//won't effect this search as I will have not data on this new store
// if existing store is being refreshed it is possible old
//address might be picked up...
//I can live with that as I doubt the address will change much.
// this should happen after product
var allStores = storeService.FindStoresThatHaveItem(product);
// this should be returned as soon as above line is finished.
//If StoreNearUser is not done, it should keep going but not hold up user.
return allStores;
}
public void StoresNearUser(double latitude, double longitude)
{
// get all categories I can about in foursquare.
//First time from db otherwise cached.
List<StoreCategory> storeCategories = GetStoreCategories();
// do a request and get everything in near the user
//(provided it is also in a category I care about)
var request = CreateFoursquareStoreRequest
(latitude, longitude, storeCategories);
// do the actual call.
var response = client.Execute<VenueSearch>(request);
if (response.StatusCode == System.Net.HttpStatusCode.OK)
{
// start going through the results, add or update or skip of entry will happen
AddUpdateStores(storeCategories, response);
}
else
{
ErrorSignal.FromCurrentContext().Raise(response.ErrorException);
}
}
Edit 2
public async Task StoresNearUser(double latitude, double longitude)
{
// get all categories I can about in foursquare. First time from db otherwise cached.
List<StoreCategory> storeCategories = GetStoreCategories();
// do a request and get everything in near the user(provided it is also in a category I care about)
var request = CreateFoursquareStoreRequest(latitude, longitude, storeCategories);
await client.ExecuteAsync<VenueSearch>
( request
, response =>
{
if (response.StatusCode == System.Net.HttpStatusCode.OK)
{
AddUpdateStores(storeCategories, response);
}
else
{
ErrorSignal.FromCurrentContext()
.Raise(response.ErrorException);
}
}
);
}
gives me this error
Cannot await 'RestSharp.RestRequestAsyncHandle'
I also don't get the difference between Task and void. From what I read if you just use Task it means you are sending nothing back of meaning, then why not just use void?
Edit 2
I found this post to show me how to make the wrapper for Restsharp. It is not 100% what I want but that is a separate issue.
public async Task StoresNearUser(double latitude, double longitude)
{
List<StoreCategory> storeCategories = GetStoreCategories();
var request = CreateFoursquareStoreRequest
(latitude, longitude, maxRadius, returnLimit, storeCategories);
var response = await client.GetResponseAsync(request);
if (response.StatusCode == HttpStatusCode.OK)
{
// had to use json.net right now as the wrapper does not expose restsharps deserilizer
var venue = JsonConvert
.DeserializeObject<VenueSearch>(response.Content);
AddUpdateStores(storeCategories, venue);
}
else
{
ErrorSignal.FromCurrentContext()
.Raise(response.ErrorException);
}
}
public async Task<HttpResponseMessage>Get()
{
await storeService.PointsOfInterestNearUser(80, -130);
var product = productService.FindProduct("Noodles");
var allStores = storeService.FindStoresThatHaveItem(product);
return allStores;
}
When I watch from the debugger it looks like it is still all going in order. I think product and allStores need to be since I need the product before I can find the stores but PointsOfInterestNearUser should be going at the same time as FindProduct.
Edit 3
Here is my FindProduct Method. Not sure what to make async to me it looks like everything needs to wait.
public ResponseResult<Product> FindProduct(string barcode)
{
ResponseResult<Product> responseResult = new ResponseResult<Product>();
Product product = null;
try
{
var findBarCode = context.Barcodes.Where(x => x.Code == barcode).Select(x => x.Product).FirstOrDefault();
responseResult.Response = product;
if (product == null)
{
responseResult.Status.Code = HttpStatusCode.NotFound;
}
else
{
responseResult.Status.Code = HttpStatusCode.OK;
}
}
catch (SqlException ex)
{
ErrorSignal.FromCurrentContext().Raise(ex);
responseResult.Status.Code = HttpStatusCode.InternalServerError;
responseResult.Status.Message = GenericErrors.InternalError;
}
return responseResult;
}
Edit 4
Still not sure how to do the Task.WhenAll()
public async Task<HttpResponseMessage>Get()
{
Task[] tasks = new Task[2];
tasks[0] = storeService.PointsOfInterestNearUser(80, -130);
tasks[1] = productService.FindProduct("Noodles");
await Task.WhenAll(tasks);
// not sure how to get product back out. I looked in the debugger and saw a "Result" that has it but when I do tasks[1].Result inetllisene cannot find .Result
var allStores = storeService.FindStoresThatHaveItem(product);
return allStores;
}

I would recommend using async/await for this. Updating a cache is one of the rare situations where returning early from an ASP.NET request is acceptable. You can see my blog post on the subject for some helpful code.
So, something like this (simplified to just look up one "interesting place" per location):
public async Task<PlaceWithData> FindPlaceAsync(Location myLocation)
{
Place place = await GetPlaceFromFoursquareAsync(myLocation);
PlaceWithData ret = await GetExtraDataFromDatabaseAsync(place);
if (ret.NeedsRefresh)
BackgroundTaskManager.Run(() => UpdateDatabaseAsync(place, ret));
return ret;
}
You may also want to consider extending the ASP.NET caching system rather than doing a "roll your own" cache.