Develop Reference

Hangfire in single application on multiple physical servers

I am running hangfire in a single web application, my application is being run on 2 physical servers but hangfire is in 1 database.
At the moment, i am generating a server for each queue, because each queue i need to run 1 worker at a time and they must be in order. I set them up like this
// core
services.AddHangfire(options =>
{
options.SetDataCompatibilityLevel(CompatibilityLevel.Version_170);
options.UseSimpleAssemblyNameTypeSerializer();
options.UseRecommendedSerializerSettings();
options.UseSqlServerStorage(appSettings.Data.DefaultConnection.ConnectionString, storageOptions);
});
// add multiple servers, this way we get to control how many workers are in each queue
services.AddHangfireServer(options =>
{
options.ServerName = "workflow-queue";
options.WorkerCount = 1;
options.Queues = new string[] { "workflow-queue" };
options.SchedulePollingInterval = TimeSpan.FromSeconds(10);
});
services.AddHangfireServer(options =>
{
options.ServerName = "alert-schedule";
options.WorkerCount = 1;
options.Queues = new string[] { "alert-schedule" };
options.SchedulePollingInterval = TimeSpan.FromMinutes(1);
});
services.AddHangfireServer(options =>
{
options.ServerName = string.Format("trigger-schedule");
options.WorkerCount = 1;
options.Queues = new string[] { "trigger-schedule" };
options.SchedulePollingInterval = TimeSpan.FromMinutes(1);
});
services.AddHangfireServer(options =>
{
options.ServerName = "report-schedule";
options.WorkerCount = 1;
options.Queues = new string[] { "report-schedule" };
options.SchedulePollingInterval = TimeSpan.FromMinutes(1);
});
services.AddHangfireServer(options =>
{
options.ServerName = "maintenance";
options.WorkerCount = 5;
options.Queues = new string[] { "maintenance" };
options.SchedulePollingInterval = TimeSpan.FromMinutes(10);
});
My problem is that it is generating multiple queues on the servers, with different ports.
In my code i am then trying to stop jobs from running if they are queued/retrying, but if the job is being run on a different physical server, it is not found and queued again.
Here is the code to check if its running already
public async Task<bool> IsAlreadyQueuedAsync(PerformContext context)
{
var disableJob = false;
var monitoringApi = JobStorage.Current.GetMonitoringApi();
// get the jobId, method and queue using performContext
var jobId = context.BackgroundJob.Id;
var methodInfo = context.BackgroundJob.Job.Method;
var queueAttribute = (QueueAttribute)Attribute.GetCustomAttribute(context.BackgroundJob.Job.Method, typeof(QueueAttribute));
// enqueuedJobs
var enqueuedjobStatesToCheck = new[] { "Processing" };
var enqueuedJobs = monitoringApi.EnqueuedJobs(queueAttribute.Queue, 0, 1000);
var enqueuedJobsAlready = enqueuedJobs.Count(e => e.Key != jobId && e.Value != null && e.Value.Job != null && e.Value.Job.Method.Equals(methodInfo) && enqueuedjobStatesToCheck.Contains(e.Value.State));
if (enqueuedJobsAlready > 0)
disableJob = true;
// scheduledJobs
if (!disableJob)
{
// check if there are any scheduledJobs that are processing
var scheduledJobs = monitoringApi.ScheduledJobs(0, 1000);
var scheduledJobsAlready = scheduledJobs.Count(e => e.Key != jobId && e.Value != null && e.Value.Job != null && e.Value.Job.Method.Equals(methodInfo));
if (scheduledJobsAlready > 0)
disableJob = true;
}
// failedJobs
if (!disableJob)
{
var failedJobs = monitoringApi.FailedJobs(0, 1000);
var failedJobsAlready = failedJobs.Count(e => e.Key != jobId && e.Value != null && e.Value.Job != null && e.Value.Job.Method.Equals(methodInfo));
if (failedJobsAlready > 0)
disableJob = true;
}
// if runBefore is true, then lets remove the current job running, else it will write a "successful" message in the logs
if (disableJob)
{
// use hangfire delete, for cleanup
BackgroundJob.Delete(jobId);
// create our sqlBuilder to remove the entries altogether including the count
var sqlBuilder = new SqlBuilder()
.DELETE_FROM("Hangfire.[Job]")
.WHERE("[Id] = {0};", jobId);
sqlBuilder.Append("DELETE TOP(1) FROM Hangfire.[Counter] WHERE [Key] = 'stats:deleted' AND [Value] = 1;");
using (var cmd = _context.CreateCommand(sqlBuilder))
await cmd.ExecuteNonQueryAsync();
return true;
}
return false;
}
Each method has something like the following attributes as well
public interface IAlertScheduleService
{
[Hangfire.Queue("alert-schedule")]
[Hangfire.DisableConcurrentExecution(60 * 60 * 5)]
Task RunAllAsync(PerformContext context);
}
Simple implementation of the interface
public class AlertScheduleService : IAlertScheduleService
{
public Task RunAllAsync(PerformContext context)
{
if (IsAlreadyQueuedAsync(context))
return;
// guess it isnt queued, so run it here....
}
}
Here is how i am adding my scheduled jobs
//// our recurring jobs
//// set these to run hourly, so they can play "catch-up" if needed
RecurringJob.AddOrUpdate<IAlertScheduleService>(e => e.RunAllAsync(null), Cron.Hourly(0), queue: "alert-schedule");
Why does this happen? How can i stop it happening?

Somewhat of a blind shot, preventing a job to be queued if a job is already queued in the same queue.
The try-catch logic is quite ugly but I have no better idea right now...
Also, really not sure the lock logic always prevents from having two jobs in EnqueudState, but it should help anyway. Maybe mixing with an IApplyStateFilter.
public class DoNotQueueIfAlreadyQueued : IElectStateFilter
{
public void OnStateElection(ElectStateContext context)
{
if (context.CandidateState is EnqueuedState)
{
EnqueuedState es = context.CandidateState as EnqueuedState;
IDisposable distributedLock = null;
try
{
while (distributedLock == null)
{
try
{
distributedLock = context.Connection.AcquireDistributedLock($"{nameof(DoNotQueueIfAlreadyQueued)}-{es.Queue}", TimeSpan.FromSeconds(1));
}
catch { }
}
var m = context.Storage.GetMonitoringApi();
if (m.EnqueuedCount(es.Queue) > 0)
{
context.CandidateState = new DeletedState();
}
}
finally
{
distributedLock.Dispose();
}
}
}
}
The filter can be declared as in this answer

There seems to be a bug with your currently used hangfire storage implementation:
https://github.com/HangfireIO/Hangfire/issues/1025
The current options are:
Switching to HangFire.LiteDB as commented here: https://github.com/HangfireIO/Hangfire/issues/1025#issuecomment-686433594
Implementing your own logic to enqueue a job, but this would take more effort.
Making your job execution idempotent to avoid side effects in case it's executed multiple times.
In either option, you should still apply DisableConcurrentExecution and make your job execution idempotent as explained below, so i think you can just go with below option:
Applying DisableConcurrentExecution is necessary, but it's not enough as there are no reliable automatic failure detectors in distributed systems. That's the nature of distributed systems, we usually have to rely on timeouts to detect failures, but it's not reliable.
Hangfire is designed to run with at-least-once execution semantics. Explained below:
One of your servers may be executing the job, but it's detected as being failed due to various reasons. For example: your current processing server does not send heartbeats in time due to a temporary network issue or due to temporary high load.
When the current processing server is assumed to be failed (but it's not), the job will be scheduled to another server which causes it to be executed more than once.
The solution should be still applying DisableConcurrentExecution attribute as a best effort to prevent multiple executions of the same job, but the main thing is that you need to make the execution of the job idempotent which does not cause side effects in case it's executed multiple times.
Please refer to some quotes from https://docs.hangfire.io/en/latest/background-processing/throttling.html:
Throttlers apply only to different background jobs, and there’s no
reliable way to prevent multiple executions of the same background job
other than by using transactions in background job method itself.
DisableConcurrentExecution may help a bit by narrowing the safety
violation surface, but it heavily relies on an active connection,
which may be broken (and lock is released) without any notification
for our background job.
As there are no reliable automatic failure detectors in distributed
systems, it is possible that the same job is being processed on
different workers in some corner cases. Unlike OS-based mutexes,
mutexes in this package don’t protect from this behavior so develop
accordingly.
DisableConcurrentExecution filter may reduce the probability of
violation of this safety property, but the only way to guarantee it is
to use transactions or CAS-based operations in our background jobs to
make them idempotent.
You can also refer to this as Hangfire timeouts behavior seems to be dependent on storage as well: https://github.com/HangfireIO/Hangfire/issues/1960#issuecomment-962884011

How to Update score in Firebase Unity instead of replacing/re-writing it?

Whenever I reopen the game and start scoring again it replaces the previous data. Please help me with this and tell me where I'm doin it wrong.
I'm using Firebase Realtime Database.
This is my score script down below:
public static int cashValue;
Text cash;
void Start()
{
cash = GetComponent();
}
void Update()
{
cash.text = "" + cashValue;
}
This is where I save the score in Firebase Realtime Database:
private IEnumerator UpdateKills(int _kills)
{
//Set the currently logged in user kills
var DBTask = DBreference.Child("users").Child(User.UserId).Child("kills").SetValueAsync(_kills);
yield return new WaitUntil(predicate: () => DBTask.IsCompleted);
if (DBTask.Exception != null)
{
Debug.LogWarning(message: $"Failed to register task with {DBTask.Exception}");
}
else
{
//Kills are now updated
}
}
And This is Where I Retrieve the data from Firebase Realtime Database:
private IEnumerator LoadUserData()
{
//Get the currently logged in user data
var DBTask = DBreference.Child("users").Child(User.UserId).GetValueAsync();
yield return new WaitUntil(predicate: () => DBTask.IsCompleted);
if (DBTask.Exception != null)
{
Debug.LogWarning(message: $"Failed to register task with {DBTask.Exception}");
}
else if (DBTask.Result.Value == null)
{
//No data exists yet
xpField.text = "0";
killsField.text = "0";
deathsField.text = "0";
}
else
{
//Data has been retrieved
DataSnapshot snapshot = DBTask.Result;
xpField.text = snapshot.Child("xp").Value.ToString();
killsField.text = snapshot.Child("kills").Value.ToString();
deathsField.text = snapshot.Child("deaths").Value.ToString();
}
}
It works perfectly, but the only problem is that it replacing the Score instead of updating it or start from where I left.

The flow should go like this:
int myData;
bool isMyDataLoaded = false;
void Start() {
LoadMyData();
}
void LoadMyData() {
StartCoroutine(IELoadMyData());
}
void SetMyData() {
if(isMyDataLoaded) {
// Set only if the data from database was retreived
StartCoroutine(IESetMyData());
}
}
IEnumerator IELoadMyData() {
// Load your data from database here
if(<fetch was successful>) {
myData = <the result>;
isMyDataLoaded = true; // Set data loaded as true
}
}
IEnumerator IESetMyData() {
// Update your data here
}
Now whenever you want to update, Call SetMyData. It will only set the new data in the DB if the data was first fetched in the local game successfully.

Something to watch out for is that you might be falling victim to caching. Basically, GetValueAsync typically returns whatever's cached locally (unless you disable persistence -- which you probably shouldn't) and asks the server for an update. SetValueAsync writes to the local cache and eventually syncs it up in the background.
Therefore, to update a value, you should always use a Transaction. From https://firebase.google.com/docs/database/unity/save-data#save_data_as_transactions
private void UpdateKills(int _kills) {
DBreference.Child("users").Child(User.UserId).Child("kills").RunTransaction(mutableData => {
// if the data isn't an int or is null, just make it 0
// then add the new number of kills
var kills = ((mutableData.value as? int) ?? 0) + _kills;
return TransactionResult.Success(mutableData);
});
}
This would require reworking your logic so that rather than saving the total kills, you record the delta. Then this transaction would run repeatedly until it succeeds (usually once, unless your data is out of sync in which case usually twice).
For getting data, you're best off registering ValueChanged listeners and updating your UI directly from those. This way you're always in sync with your server, and it's smart enough to use a local cache so you don't have to worry about always blocking on a network access (you'll get a null if there's nothing cached). Also, as the local cache updates in the Transaction, you will get ValueChanged events with the latest data.
Obviously, not every game can be stable around randomly receiving value updates from a server. But if you can move your data to be more reactive around Realtime Database, you'll make the best use of its inbuilt local caching and sync logic.

Problem trying to clone a Project Server database using OData and Entity Framework [duplicate]

This question already has answers here:
How can I use Fast Member to Bulk Copy data into a table with inconsistent column names?
(2 answers)
Closed 2 years ago.
I am having trouble updating my entities with Parallel.Foreach. The program I have, works fine by using foreach to update the entities, but if I use Parallel.Foreach it gives me the error like : "Argument Exception: An item with the same key has already been added". I have no idea why it happens, shouldn't it be thread safe? Or why giving me this error? How to resolve this issue?
The program itself get some data from a database and copy it to another one. If the datarow exists with the same guid (see below), and the status unchanged the matching datarow in the second must be updated. If theres a match, and status changed, modifications must be ignored. Finally if no match in the second database, then insert the datarow into the second database. (Synchronize the two databases). I just want to speed up the process somehow, that is why I first think of parallel processing.
(I am using Autofac as an IoC container and dependency injection if that matters)
Here is the code snippet which tries to update:
/* #param reports: data from the first database */
public string SynchronizeData(List<Reports> reports, int statusid)
{
// reportdataindatabase - the second database data, List() actually selects all, see next code snippet
List<Reports> reportdataindatabase = unitOfWorkTAFeedBack.ReportsRepository.List().ToList();
int allcount = reports.Count;
int insertedcount = 0;
int updatedcount = 0;
int ignoredcount = 0;
// DOES NOT WORK, GIVES THE ERROR
Parallel.ForEach(reports, r =>
{
var guid = reportdataindatabase.FirstOrDefault(x => x.AssignmentGUID == r.AssignmentGUID);
if (guid == null)
{
unitOfWorkTAFeedBack.ReportsRepository.Add(r); // an insert on the repository
insertedcount++;
}
else
{
if (guid.StatusId == statusid)
{
r.ReportsID = guid.ReportsID;
unitOfWorkTAFeedBack.ReportsRepository.Update(r); // update on the repo
updatedcount++;
}
else
{
ignoredcount++;
}
}
});
/* WORKS PERFECTLY BUT RELATIVELY SLOW - takes 80 seconds to update 1287 records
foreach (Reports r in reports)
{
var guid = reportdataindatabase.FirstOrDefault(x => x.AssignmentGUID == r.AssignmentGUID); // find match between the two databases
if (guid == null)
{
unitOfWorkTAFeedBack.ReportsRepository.Add(r); // no match, insert
insertedcount++;
}
else
{
if (guid.StatusId == statusid)
{
r.ReportsID = guid.ReportsID;
unitOfWorkTAFeedBack.ReportsRepository.Update(r);
updatedcount++;
}
else
{
ignoredcount++;
}
}
} */
unitOfWorkTAFeedBack.Commit(); // this only calls SaveChanges() on DbContext object
int allprocessed = insertedcount + updatedcount + ignoredcount;
string result = "Synchronization finished. " + allprocessed + " reports processed out of " + allcount + ", "
+ insertedcount + " has been inserted, " + updatedcount + " has been updated and "
+ ignoredcount + " has been ignored. \n Press a button to dismiss this window." ;
return result;
}
The program breaks on this Repository class in the Update method (with Parallel.Foreach, no problem with the standard foreach):
public class EntityFrameworkReportsRepository : IReportsRepository
{
private readonly TAFeedBackContext tAFeedBackContext;
public EntityFrameworkReportsRepository(TAFeedBackContext tAFeedBackContext)
{
this.tAFeedBackContext = tAFeedBackContext;
}
public void Add(Reports r)
{
tAFeedBackContext.Reports.Add(r);
}
public void Delete(int Id)
{
var obj = tAFeedBackContext.Reports.Find(Id);
tAFeedBackContext.Reports.Remove(obj);
}
public Reports Get(int Id)
{
var obj = tAFeedBackContext.Reports.Find(Id);
return obj;
}
public IQueryable<Reports> List()
{
return tAFeedBackContext.Reports.AsNoTracking();
}
public void Update(Reports r)
{
var entry = tAFeedBackContext.Entry(r); // The Program Breaks At This Point!
if (entry.State == EntityState.Detached)
{
tAFeedBackContext.Reports.Attach(r);
tAFeedBackContext.Entry(r).State = EntityState.Modified;
}
else
{
tAFeedBackContext.Entry(r).CurrentValues.SetValues(r);
}
}
}

Please bear in mind it hard to give a complete answer as there are thing I need clarity on … but comments should help with building a picture.
Parallel.ForEach(reports, r => //Parallel.ForEach is not the answer..
{
//reportdataindatabase is done..before so ok here
// do you really want FirstOrDefault vs SingleOrDefault
var guid = reportdataindatabase.FirstOrDefault(x => x.AssignmentGUID == r.AssignmentGUID);
if (guid == null)
{
// this is done on the context not the DB, unresolved..(excuted)
unitOfWorkTAFeedBack.ReportsRepository.Add(r); // an insert on the repository
//insertedcount++; u would need a lock
}
else
{
if (guid.StatusId == statusid)
{
r.ReportsID = guid.ReportsID;
// this is done on the context not the DB, unresolved..(excuted)
unitOfWorkTAFeedBack.ReportsRepository.Update(r); // update on the repo
//updatedcount++; u would need a lock
}
else
{
//ignoredcount++; u would need a lock
}
}
});
the issue here... as reportdataindatabase can contain the same key twice..
and the context is only updated after the fact aka when it get here..
unitOfWorkTAFeedBack.Commit();
it may have been called twice for the same entity
as above (commit) is where the work is... doing the add/update above in Parallel wont save you any real time, as that part is quick..
//takes 80 seconds to update 1287 records... does seem long...
//List reportdataindatabase = unitOfWorkTAFeedBack.ReportsRepository.List().ToList();
//PS Add how reports are retrieved.. you want something like
TAFeedBackContext db = new TAFeedBackContext();
var remoteReports = DatafromAnotherPLace //include how this was retrieved;
var localReports = TAFeedBackContext.Reports.ToList(); //these are tracked.. (by default)
foreach (var item in remoteReports)
{
//i assume more than one is invalid.
var localEntity = localReports.SingleOrDefault(x => x.AssignmentGUID == item.AssignmentGUID);
if (localEntity == null)
{
//add as it doenst exist
TAFeedBackContext.Reports.Add(new Report() { *set fields* });
}
else
{
if (localEntity.StatusId == statusid) //only update if status is the passed in status.
{
//why are you modifying the remote entity
item.ReportsID = localEntity.ReportsID;
//update remove entity?, i get the impression its from a different context,
//if not then cool, but you need to show how reports is retrieved
}
else
{
}
}
}
TAFeedBackContext.SaveChanges();

Entity Framework not Saving Changes into Database

I'm puzzled as to why this code is not working, it should save changes to database after the loops but when I place the SaveChanges method inside the loop, it saves the record into the database but outside it doesn't save anything? it's about only 300 ~ 1000 records
static bool lisReady = false;
static bool sacclReady = false;
static void Main(string[] args)
{
Logger("Starting services");
ConnectDBLis().Wait();
ConnectDBSaccl().Wait();
Thread.Sleep(1000);
if (lisReady & sacclReady){
//start
Logger("Services ready");
StartExport().Wait();
}
}
static async Task<bool> StartExport()
{
lis lisdb = new lis();
nrlsaccl saccldb = new nrlsaccl();
var getTestOrders = await lisdb.test_orders.ToListAsync();
Logger("Services starting");
foreach (var tO in getTestOrders.Where(x => x.entry_datetime.Value.Year == 2016))
{
foreach (var tr in tO.test_results)
{
foreach (var tL in tr.test_result_logs)
{
results_availability postResults = new results_availability
{
first_name = tO.patient_orders.patient.first_name,
middle_name = tO.patient_orders.patient.middle_name,
last_name = tO.patient_orders.patient.last_name,
birthdate = tO.patient_orders.patient.birthdate,
};
if (postResults.id == 0)
{
saccldb.results_availability.Add(postResults);
}
else
{
saccldb.Entry(postResults).State = EntityState.Modified;
}
}
}
}
await saccldb.SaveChangesAsync();
return true;
}
Edit:
So i limit the records to 100 and the save changes works, 3000 records at instant does not work, any solutions?

This code doesn't completely resolve your issue, this are some consideration for your problem.
Note: This works for me when adding 1200 records and 300 modifications
static async Task<bool> StartExport()
{
using (var db = new Entities())
{
var appraisals = await db.Appraisals.ToListAsync();
db.Database.CommandTimeout = 300;
//Disabling auto detect changes enabled will bring some performance tweaks
db.Configuration.AutoDetectChangesEnabled = false;
foreach (var appraisal in appraisals.Where(g => g.Id > 1))
{
if (appraisal.Id == 10)
{
appraisal.AppraisalName = "New name";
db.Entry(appraisal).State = EntityState.Added;
}
else
{
appraisal.AppraisalName = "Modified name";
db.Entry(appraisal).State = EntityState.Modified;
}
}
db.Configuration.AutoDetectChangesEnabled = true;
if (await db.SaveChangesAsync() > 1)
return true;
else
return false;
}
}
You could use db.Database.CommandTimeout = 300; to increase the timeout of you connection.
Entity framework 6 provides AddRange() This will insert the items in one shot, it will disable AutoDetectChangesEnabled and insert the entities
In your case you don't want to mark the entites as Modified, the EF already tracks it well. Entity Framework - Why explicitly set entity state to modified?
The purpose of change tracking to find that you have changed a value on attached entity and put it to modified state. Setting state manually is important in case of detached entities (entities loaded without change tracking or created outside of the current context).
Here we have all entities attached to the context itself

Use
saccldb.SaveChanges()
Simply because the async nature of await saccldb.SaveChangesAsync() cause your thread to continue and exit the function before the saving to the db completes. In your case it returns true.
I would suggest not using any async operations on a console application unless it has a user interface that you would want to keep going.

Hangfire get last execution time

I'm using hangfire 1.5.3. In my recurring job I want to call a service that uses the time since the last run. Unfortunately the LastExecution is set to the current time, because the job data was updated before executing the job.
Job
public abstract class RecurringJobBase
{
protected RecurringJobDto GetJob(string jobId)
{
using (var connection = JobStorage.Current.GetConnection())
{
return connection.GetRecurringJobs().FirstOrDefault(p => p.Id == jobId);
}
}
protected DateTime GetLastRun(string jobId)
{
var job = GetJob(jobId);
if (job != null && job.LastExecution.HasValue)
{
return job.LastExecution.Value.ToLocalTime();
}
return DateTime.Today;
}
}
public class NotifyQueryFilterSubscribersJob : RecurringJobBase
{
public const string JobId = "NotifyQueryFilterSubscribersJob";
private readonly IEntityFilterChangeNotificationService _notificationService;
public NotifyQueryFilterSubscribersJob(IEntityFilterChangeNotificationService notificationService)
{
_notificationService = notificationService;
}
public void Run()
{
var lastRun = GetLastRun(JobId);
_notificationService.CheckChangesAndSendNotifications(DateTime.Now - lastRun);
}
}
Register
RecurringJob.AddOrUpdate<NotifyQueryFilterSubscribersJob>(NotifyQueryFilterSubscribersJob.JobId, job => job.Run(), Cron.Minutely, TimeZoneInfo.Local);
I know, that it is configured as minutely, so I could calculate the time roughly. But I'd like to have a configuration independent implementation. So my Question is: How can I implement RecurringJobBase.GetLastRun to return the time of the previous run?

To address my comment above, where you might have more than one type of recurring job running but want to check previous states, you can check that the previous job info actually relates to this type of job by the following (although this feels a bit hacky/convoluted).
If you're passing the PerformContext into the job method than you can use this:
var jobName = performContext.BackgroundJob.Job.ToString();
var currentJobId = int.Parse(performContext.BackgroundJob.Id);
JobData jobFoundInfo = null;
using (var connection = JobStorage.Current.GetConnection()) {
var decrementId = currentJobId;
while (decrementId > currentJobId - 50 && decrementId > 1) { // try up to 50 jobs previously
decrementId--;
var jobInfo = connection.GetJobData(decrementId.ToString());
if (jobInfo.Job.ToString().Equals(jobName)) { // **THIS IS THE CHECK**
jobFoundInfo = jobInfo;
break;
}
}
if (jobFoundInfo == null) {
throw new Exception($"Could not find the previous run for job with name {jobName}");
}
return jobFoundInfo;
}

You could take advantage of the fact you already stated - "Unfortunately the LastExecution is set to the current time, because the job data was updated before executing the job".
The job includes the "LastJobId" property which seems to be an incremented Id. Hence, you should be able to get the "real" previous job by decrement LastJobId and querying the job data for that Id.
var currentJob = connection.GetRecurringJobs().FirstOrDefault(p => p.Id == CheckForExpiredPasswordsId);
if (currentJob == null)
{
return null; // Or whatever suits you
}
var previousJob = connection.GetJobData((Convert.ToInt32(currentJob.LastJobId) - 1).ToString());
return previousJob.CreatedAt;
Note that this is the time of creation, not execution. But it might be accurate enough for you. Bear in mind the edge case when it is your first run, hence there will be no previous job.

After digging around, I came up with the following solution.
var lastSucceded = JobStorage.Current.GetMonitoringApi().SucceededJobs(0, 1000).OrderByDescending(j => j.Value.SucceededAt).FirstOrDefault(j => j.Value.Job.Method.Name == "MethodName" && j.Value.Job.Type.FullName == "NameSpace.To.Class.Containing.The.Method").Value;
var lastExec = lastSucceded.SucceededAt?.AddMilliseconds(Convert.ToDouble(-lastSucceded.TotalDuration));
It's not perfect but i think a little cleaner than the other solutions.
Hopefully they will implement an official way soon.

The answer by #Marius Steinbach is often good enough but if you have thousands of job executions (my case) loading all of them from DB doesn't seem that great. So finally I decided to write a simple SQL query and use it directly (this is for PostgreSQL storage though changing it to SqlServer should be straightforward):
private async Task<DateTime?> GetLastSuccessfulExecutionTime(string jobType)
{
await using var conn = new NpgsqlConnection(_connectionString);
if (conn.State == ConnectionState.Closed)
conn.Open();
await using var cmd = new NpgsqlCommand(#"
SELECT s.data FROM hangfire.job j
LEFT JOIN hangfire.state s ON j.stateid = s.id
WHERE j.invocationdata LIKE $1 AND j.statename = $2
ORDER BY s.createdat DESC
LIMIT 1", conn)
{
Parameters =
{
new() { Value = $"%{jobType}%" } ,
new() { Value = SucceededState.StateName }
}
};
var result = await cmd.ExecuteScalarAsync();
if (result is not string data)
return null;
var stateData = JsonSerializer.Deserialize<Dictionary<string, string>>(data);
return JobHelper.DeserializeNullableDateTime(stateData?.GetValueOrDefault("SucceededAt"));
}

Use this method that return Last exucution time and Next execution time of one job. this method return last and next execution time of one job.
public static (DateTime?, DateTime?) GetExecutionDateTimes(string jobName)
{
DateTime? lastExecutionDateTime = null;
DateTime? nextExecutionDateTime = null;
using (var connection = JobStorage.Current.GetConnection())
{
var job = connection.GetRecurringJobs().FirstOrDefault(p => p.Id == jobName);
if (job != null && job.LastExecution.HasValue)
lastExecutionDateTime = job.LastExecution;
if (job != null && job.NextExecution.HasValue)
nextExecutionDateTime = job.NextExecution;
}
return (lastExecutionDateTime, nextExecutionDateTime);
}