Develop Reference

Entity Framework - fast insert/ update of millions of records

I need to insert 1.9 million new records into a MySQL database. To use this I'm using C# Entity Framework, but the process seems incredibly slow. At the current rate, it would take several days to process these records.
What am I doing wrong and how do I speed this up?
In the database I have 2 tables: Hashes and Categories. Each hash should be unique and can have multiple categories, with only 1 category being active per hash.
The process that I need to follow is to first check if the hash exist. If it does, then I need to find the current category, deactivate it and add the new one.
The problem is that my try{ } statement is taking about 150ms and the block that does SaveChanges() takes about 15-30 seconds. So, doing 1.9M records this way will take several days.
using (var reader = new StreamReader(File.OpenRead(filepath)))
using (MySQLContext db = new MySQLContext(options))
{
// Disable auto detect changes
db.ChangeTracker.AutoDetectChangesEnabled = false;
int loopCounter = 0;
string line;
// Load up the db tables in memory
var hashes = db.Hashes.Select(x => x).ToList();
var category = db.Categories.Select(a => a).ToList();
while ((line = reader.ReadLine()) != null)
{
var matches = Regex.Matches(line, "(?<MD5>[a-zA-Z0-9]+)(?<Category>[0-9])");
InputHashModel inputHash = new InputHashModel()
{
MD5 = matches[0].Groups["MD5"].Value,
Category = matches[0].Groups["Category"].Value
};
try
{
// Check if hash already exists
Hash hash = hashes.Where(h => h.MD5 == inputHash.MD5).FirstOrDefault();
// If hash doesn't exist - add it
if (hash == null)
hash = new Hash(inputHash.MD5);
else
{
// Check if category already exists
Category category = categories.Where(a => a.Active == true && a.HashId == hash.Id).FirstOrDefault();
// If it exists - deactivate it
if (category != null)
{
// If the same category already exists - proceed to next hash
if (category.Source == "ThisInput" && category.Category == inputHash.Category)
{
loopCounter++
continue;
}
category.Active = false;
category.DeactivatedTimestamp = DateTime.Now;
}
}
// Add new category
Category new_category = new Category() { Hash = hash, Source = "ThisInput", Category = inputHash.Category, Active = true);
db.Categories.Add(new_category);
// Save changes every 1000
if (loopCounter % 1000 == 0)
{
db.ChangeTracker.DetectChanges();
db.SaveChanges();
}
}
catch (Exception e)
{
Console.WriteLine("Exception: " + e);
}
loopCounter++;
}
db.ChangeTracker.AutoDetectChangesEnabled = true;
db.SaveChanges();
Console.WriteLine("Finished");
}

This is never going to be the fastest method, but at a minimum you need to aviod accumulating all the entities in the change tracker. EG after each SaveChanges() run
foreach (var e in db.ChangeTracker.Entries())
{
e.State = EntityState.Detached;
}

I managed to achieve this through the use of the BulkInsert() and BulkUpdate() functions from EntityFramework Extensions.
I went about it a slightly different way though ...
Ingest the new entries into the EF models with the new relationship.
Download all the entries from the database to memory.
Compare the new entries to the ones from the database with the .Intersects(myCustomComparer). Do that twice to separate unique and duplicate entries into two lists.
Download the relationship entries for the duplicate entries only (lookup by ID). Run through the relationship entries and compare - if needs update, add it to a list. If it needs a new entry, add that to a separate list.
Update the relationship entries and add the new ones.
Bulk insert the unique entries and their relationship entries.
Here's the code for the functions:
public class HashMD5Comparer : IEqualityComparer<Hash>
{
//Products are equal if their names and product numbers are equal.
public bool Equals(Hash x, Hash y)
{
//Check whether the compared objects reference the same data.
if (Object.ReferenceEquals(x, y)) return true;
//Check whether any of the compared objects is null.
if (Object.ReferenceEquals(x, null) || Object.ReferenceEquals(y, null))
return false;
//Check whether the hash' properties are equal.
return x.MD5 == y.MD5;
}
// If Equals() returns true for a pair of objects
// then GetHashCode() must return the same value for these objects.
public int GetHashCode(Hash hash)
{
//Check whether the object is null
if (Object.ReferenceEquals(hash, null)) return 0;
//Get hash code for the Name field if it is not null.
int hashMD5 = hash.MD5 == null ? 0 : hash.MD5.GetHashCode();
//Calculate the hash code for the hash.
return hashMD5;
}
}
public class HashComparer
{
private static Hash[] uniqueHashes;
private static Hash[] duplicateHashes;
private static Hash[] duplicateHashesInDb;
private static void SortHashes(Hash[] hashes)
{
Hash[] hashesInDatabase;
// Download hashes from database
using (MySQLContext db = new MySQLContext())
{
hashesInDatabase = db.Hashes.Where(h => h.MD5 != null).ToArray();
}
// Find duplicates in database
duplicateHashes = hashes.Intersect(hashesInDatabase, new HashMD5Comparer()).ToArray();
duplicatehashesInDatabase = hashesInDatabase.Intersect(hashes, new HashMD5Comparer()).ToArray();
// Find uniques in database
uniqueHashes = hashes.Except(duplicateHashes, new HashMD5Comparer()).ToArray();
}
private static void ActionDuplicateHashes()
{
Assessment[] assessmentsInDatabase;
List<Assessment> assessmentsToDeactivate = new List<Assessment>();
List<Assessment> assessmentsToAdd = new List<Assessment>();
// Download assessments from database
using (MySQLContext db = new MySQLContext(GenerateMySQLOptions()))
{
var duplicateHashIds = duplicateHashesInDb.Select(h => h.Id).ToArray();
assessmentsInDatabase = db.Assessments.Where(a => duplicateHashIds.Contains(a.HashId)).ToArray();
}
foreach (var inputHash in duplicateHashes)
{
// Lookup the hash in the database to get the ID
var liveHashId = Array.Find(duplicateHashesInDb, h => h.MD5 == inputHash.MD5).Id;
// Find the assessment in the database to compare (and deactive if needed)
var liveAsssessment = Array.Find(assessmentsInDatabase, a => a.HashId == liveHashId);
// Get the new assessment of the hash
var newAssessment = inputHash.Assessments.FirstOrDefault();
if (newAssessment == null)
{
Console.WriteLine($"Failed lookup for new assessment {inputHash.MD5}");
return;
}
// Set the hashId (relationship) for the new assessment
newAssessment.HashId = liveHashId;
if (liveAsssessment != null)
{
if (liveAsssessment.Origin == newAssessment.Origin &&
liveAsssessment.Category == newAssessment.Category)
{
// Exact duplicate - leave as is
}
else
{
// Deactivate the current assessment in the database
liveAsssessment.Active = false;
// Add the assessment to a list to deactive (update)
assessmentsToDeactivate.Add(liveAsssessment);
// Add the new assessment that will be added once the old one gets deactivated
assessmentsToAdd.Add(newAssessment);
}
}
else
{
// No assessment for the hash in the database - just add a new one
assessmentsToAdd.Add(newAssessment);
}
}
// Bulk update the assessments in the database that are to be deactivated
using (MySQLContext db = new MySQLContext(GenerateMySQLOptions()))
{
db.Assessments.BulkUpdate(assessmentsToDeactivate);
}
// Bulk insert the new assessments
using (MySQLContext db = new MySQLContext(GenerateMySQLOptions()))
{
db.Assessments.BulkInsert(assessmentsToAdd);
}
}
private static void ActionUniqueHashes()
{
// Bulk insert all unique hashes and their assessments
using (MySQLContext db = new MySQLContext())
{
// options.IncludeGraph adds any relationships to the database as well
db.Hashes.BulkInsert(uniqueHashes, options => options.IncludeGraph = true);
}
}
}
There is more optimisation to be done because this uses A LOT of RAM. A specially when doing the unique hashes bulk insert (not sure why). But all in all it works.

Error while insert/update records in asp.net core

When I am trying to insert/update the records I am getting the below error.
The instance of entity type cannot be tracked because another instance
with the same key value for {'Id'} is already being tracked.
Below is my code for the same. Here I am creating/generating the ID(Primary Key) by increment with 1. I am getting error at both Save & Update
public bool SaveDataCapDetails(List<TDataCapDetails> lstDataCapDetails)
{
bool IsSuccess = false;
using (var dbContextTransaction = _objContext.Database.BeginTransaction())
{
try
{
List<TDataCapDetails> lstDataCapDetailsRecords = null;
if (lstDataCapDetails.Where(x => x.Id == 0).Count() > 0)
{
lstDataCapDetailsRecords = new List<TDataCapDetails>();
lstDataCapDetailsRecords.InsertRange(0, lstDataCapDetails);
int? id = _objContext.TDataCapDetails.Max(x => (int?)x.Id);
id = id == null ? 0 : id;
foreach (var item in lstDataCapDetailsRecords.Where(x => x.Id == 0))
{
id = id + 1;
item.Id = (int)id;
}
_objContext.Entry(lstDataCapDetailsRecords).State = EntityState.Detached;
_objContext.AddRange(lstDataCapDetailsRecords);
_objContext.SaveChanges();
}
if (lstDataCapDetails.Where(x => x.Id > 0).Count() > 0)
{
lstDataCapDetailsRecords = new List<TDataCapDetails>();
lstDataCapDetailsRecords = lstDataCapDetails.Where(x => x.Id > 0).ToList();
_objContext.UpdateRange(lstDataCapDetailsRecords);
_objContext.SaveChanges();
}
dbContextTransaction.Commit();
}
catch (Exception ex)
{
dbContextTransaction.Rollback();
throw ex;
}
}
return IsSuccess;
}
The above method I am calling from business layer like below
bool success = dal.SaveDataCapDetails(lstDataCapDetails)
I have tried with AsNoTracking and other options available, but still I am not able to resolve this issue.
Any help on this appreciated.

If you want to have table with Primary-Key and also with Identity-Incremental you have to create your Table after set ForeignKey you have to set Identity-Incremental for that ForeignKey. like:
With this issue, your code change to this:
public bool SaveDataCapDetails(List<TDataCapDetails> lstDataCapDetails)
{
bool IsSuccess = false;
using (var dbContextTransaction = _objContext.Database.BeginTransaction())
{
try
{
List<TDataCapDetails> lstDataCapDetailsRecords = null;
if (lstDataCapDetails.Where(x => x.Id == 0).Count() > 0)
{
lstDataCapDetailsRecords = new List<TDataCapDetails>();
_objContext.AddRange(lstDataCapDetailsRecords);
_objContext.SaveChanges();
}
if (lstDataCapDetails.Where(x => x.Id > 0).Count() > 0)
{
lstDataCapDetailsRecords = new List<TDataCapDetails>();
lstDataCapDetailsRecords = lstDataCapDetails.Where(x => x.Id > 0).ToList();
_objContext.UpdateRange(lstDataCapDetailsRecords);
_objContext.SaveChanges();
}
dbContextTransaction.Commit();
}
catch (Exception ex)
{
dbContextTransaction.Rollback();
throw ex;
}
}
return IsSuccess;
}

First of all, alot of people are missing those checks, thus result in runtime exceptions.
So you always should check the state of your entity:
context.YourEntities.Local.Any(e => e.Id == id);
Or
context.ChangeTracker.Entries<YourEntity>().Any(e => e.Entity.Id == id);
To make sure your entity is 'safe to use', if you want a convenient method to use, you can use this:
/// <summary>
/// Determines whether the specified entity key is attached is attached.
/// </summary>
/// <param name="context">The context.</param>
/// <param name="key">The key.</param>
/// <returns>
/// <c>true</c> if the specified context is attached; otherwise, <c>false</c>.
/// </returns>
internal static bool IsAttached(this ObjectContext context, EntityKey key)
{
if (key == null)
{
throw new ArgumentNullException("key");
}
ObjectStateEntry entry;
if (context.ObjectStateManager.TryGetObjectStateEntry(key, out entry))
{
return (entry.State != EntityState.Detached);
}
return false;
}
and then:
if (!_objectContext.IsAttached(entity.EntityKey))
{
_objectContext.Attach(entity);
}
This will save you the trouble of overriding the SaveChanges() method, which is not recommended at any case, unless you really have to.

I see that you are using same method for Add and Update entities, my first suggestion is separate concerns and make different methods one for for Add, and another one for Update if it is possible.
On the other hand, it is not clear if the list of records "lstDataCapDetails" may contain all new records or a mix of new and existing records for update, in the second case your code will give you error because you may try to assign Id to an existing record, or you may try to update a totally new record.
Re the error you can overcome by checking if the entity is being tracked and detach it, then attach the modified entity and update it.
here you can see a modified version of your method:
public bool SaveDataCapDetails(List<TDataCapDetails> lstDataCapDetails)
{
bool IsSuccess = false;
using (var dbContextTransaction = _objContext.Database.BeginTransaction())
{
try
{
int? id = _objContext.TDataCapDetails.Max(x => (int?)x.Id);
id = id == null ? 0 : id;
// entities with Id == 0 --> new entities
// you may need to check if Id == null as well (depends on your data model)
var entitiesToAdd = lstDataCapDetails.Where(x => x.Id == 0);
foreach(var entity in entitiesToAdd)
{
entity.Id = id++;
// new entities is not tracked, its state can be changed to Added
_objContext.Entry(entity).State = EntityState.Added;
}
// entities with Id > 0 is already exists in db and needs to be updated
var entitiesToUpdate = lstDataCapDetails.Where(x => x.Id > 0);
foreach (var entity in entitiesToUpdate)
{
// check if entity is being tracked
var local = _objContext.Set<TDataCapDetails>().Local.FirstOrDefault(x => x.Id.Equals(entity.Id));
// if entity is tracked detach it from context
if (local != null)
_objContext.Entry<TDataCapDetails>(local).State = EntityState.Detached;
// attach modified entity and change its state to modified
_objContext.Attach(entity).State = EntityState.Modified;
}
// optional: assign value for IsSuccess
IsSuccess = _objContext.SaveChanges() > 0;
dbContextTransaction.Commit();
}
catch (Exception ex)
{
dbContextTransaction.Rollback();
throw ex;
}
}
return IsSuccess;
}

This error happens when you want to track the changes made to the model, not to actually keep an untracked model in memory. I have a little suggestion or actually an alternative suggestion approach which will fix your problem.
EntityFramework will automatically track changes. But you can Ovverride SaveChanges() in your DbContext.
public override int SaveChanges()
{
foreach (var ent in ChangeTracker.Entries<Client>())
{
if (ent.State == EntityState.Modified)
{
// Get the changed values
var modifiedProps = ObjectStateManager.GetObjectStateEntry(ent.EntityKey).GetModifiedProperties();
var currentValues = ObjectStateManager.GetObjectStateEntry(ent.EntityKey).CurrentValues;
foreach (var propName in modifiedProps)
{
var newValue = currentValues[propName];
//log your changes
}
}
}
return base.SaveChanges();
}

Entity Framework 6 Upsert

I'm trying to get a upsert working with Entity Framework. I have the following code that is throwing an error:
using (var db = new Entities.DB.DConn())
{
//...
foreach (Account account in accounts)
{
Entities.DB.Account dlAccount = new Entities.DB.Account();
dlAccount.GId = dlG.Id;
dlAccount.AccountName = account.NameAtFI;
dlAccount.AccountNumber = account.AcctNumber;
dlAccount.AcctType = account.AcctType;
dlAccount.AsOfDate = account.DateCreated;
dlAccount.IsDeleted = false;
dlAccount.DateModified = DateTime.UtcNow.ToUniversalTime();
db.Entry(dlAccount).State = ((dlAccount.GId == dlG.Id) ? EntityState.Modified : EntityState.Added);
db.SaveChanges();
}
}
Exception:
Store update, insert, or delete statement affected an unexpected number of rows (0). Entities may have been modified or deleted since entities were loaded.
Essentially all I want to do is update the record if dlAccount.GId == dlG.Id or insert it if it does not exist. The following code achieves what i want without using EntityState:
using (var db = new Entities.DB.DConn())
{
//...
foreach (Account account in accounts)
{
bool isNewRecord = false;
Entities.DB.Account dlAccount = new Entities.DB.Account();
Entities.DB.Account exisitngAcct = db.Accounts.Where(x => x.GId == dlG.Id).FirstOrDefault(); //x.GId is NOT ad primary key
if (exisitngAcct != null)
{
dlAccount = exisitngAcct;
isNewRecord = true;
}
dlAccount.GId = dlG.Id;
dlAccount.AccountName = account.NameAtFI;
dlAccount.AccountNumber = account.AcctNumber;
dlAccount.AcctType = account.AcctType;
dlAccount.AsOfDate = account.DateCreated;
dlAccount.IsDeleted = false;
dlAccount.DateModified = DateTime.UtcNow.ToUniversalTime();
if (isNewRecord)
{
dldb.Accounts.Add(dlAccount);
}
db.SaveChanges();
}
}
Can anyone see anything that I'm perhaps doing wrong here? I would really like to get this working and avoid having to use over bloated code like above.
TIA

First I should point out that the logic in the (non-EntityState) example you posted doesnt look like I would expect it to - at least based on my understanding, which may be wrong :)
*Disclaimer - I've hacked this out in a text editor, please excuse any bugs.
If we take this as your requirement:
Essentially all I want to do is update the record if dlAccount.GId == dlG.Id or insert it if it does not exist.
Then I would expect the non-EntityState version to look like this:
using (var db = new Entities.DB.DConn())
{
//...
foreach (Account account in accounts)
{
Entities.DB.Account dlAccount = null;
Entities.DB.Account exisitngAcct = db.Accounts.Where(x => x.GId == dlG.Id).FirstOrDefault(); //x.GId is NOT ad primary key
if (exisitngAcct != null)
{
//If there is an EXISTING account, it will already be tracked by EF so no need to attach it.
dlAccount = exisitngAcct;
}
else
{
//No account exists, so we need to create one, and ADD it to our EF context as a new Entity
dlAccount = new Entities.DB.Account();
db.Accounts.Add(dlAccount);
}
dlAccount.GId = dlG.Id;
dlAccount.AccountName = account.NameAtFI;
dlAccount.AccountNumber = account.AcctNumber;
dlAccount.AcctType = account.AcctType;
dlAccount.AsOfDate = account.DateCreated;
dlAccount.IsDeleted = false;
dlAccount.DateModified = DateTime.UtcNow.ToUniversalTime();
db.SaveChanges();
}
}
Assuming the above is what you need, and also assuming that we have a good reason not to just use EF tracking as above, then manually handling EF state would look something like this:
using (var db = new Entities.DB.DConn())
{
//...
foreach (Account account in accounts)
{
Entities.DB.Account exisitngAcct = db.Accounts.FirstOrDefault(x => x.GId == dlG.Id).FirstOrDefault(); //x.GId is NOT ad primary key
//NB. Since we're already pulling up the record with EF, there is *probably* no measurable advantage in not just using EF tracking at this point (unless this is a HUUUGE list of objects)
// in which case we should use the .AsNoTracking() modifier when we load the records (and they should be loaded in batches/all at once, to reduce DB hits)
Entities.DB.Account dlAccount = new Entities.DB.Account();
if(exisitngAcct == null)
{
db.Entry(dlAccount).State = EntityState.Added;
}
else
{
dlAccount.Id = exisitngAcct.Id; //We have to set the PK, so that EF knows which object to update
db.Entry(dlAccount).State = EntityState.Modified;
}
dlAccount.GId = dlG.Id;
dlAccount.AccountName = account.NameAtFI;
dlAccount.AccountNumber = account.AcctNumber;
dlAccount.AcctType = account.AcctType;
dlAccount.AsOfDate = account.DateCreated;
dlAccount.IsDeleted = false;
dlAccount.DateModified = DateTime.UtcNow.ToUniversalTime();
db.SaveChanges();
}
}

How to prevent to change other column values into table while updating single column using Entity Framework?

Here i have a method in ASP.NET MVC. What i am doing is to update single column checking every column of table is not null. If null then IsModified property changing to false. I have to write statement for every column.
My Sample Method -
public int ServicesEdit(helplineservice _helplineservice)
{
int result = 0;
try
{
db.Entry(_helplineservice).State = EntityState.Modified;
if (string.IsNullOrEmpty(_helplineservice.description))
{
db.Entry(_helplineservice).Property(p => p.description).IsModified = false;
}
else
{
db.Entry(_helplineservice).Property(p => p.description).IsModified = true;
}
if (string.IsNullOrEmpty(_helplineservice.title))
{
db.Entry(_helplineservice).Property(p => p.title).IsModified = false;
}
else
{
db.Entry(_helplineservice).Property(p => p.title).IsModified = true;
}
if (string.IsNullOrEmpty(_helplineservice.contactnumber))
{
db.Entry(_helplineservice).Property(p => p.contactnumber).IsModified = false;
}
else
{
db.Entry(_helplineservice).Property(p => p.contactnumber).IsModified = true;
}
//if (string.IsNullOrEmpty(_helplineservice.active.ToString()))
//{
// db.Entry(_helplineservice).Property(p => p.active).IsModified = false;
//}
//else
//{
// db.Entry(_helplineservice).Property(p => p.active).IsModified = true;
//}
db.SaveChanges();
result = 1;
}
catch (Exception ex)
{
result = 0;
}
return result;
}
Calling Above Method -
helplineservice _helplineservice = new helplineservice();
_helplineservice.helplineid =sectionid;
_helplineservice.allowedtoapp = allow;
result = _ftwCommonMethods.ServicesEdit(_helplineservice);
This code not look logical i think. Tell me better way to do this. How Can i Update Single Column of Table by not writing this much code? Thanks in Advance.

You can avoid all of the checking by loading the entity you want to update first.
var context = new DbContext();
// Load entity via whatever Id parameter you have.
var entityToUpdate = context.Set<Type>().FirstOrDefault(x => x.Id == idToUpdate);
if(entityToUpdate != null)
{
entityToUpdate.Value1 = newValue1;
entityToUpdate.Value2 = newValue2;
context.SaveChanges();
}
Only Value1 and Value2 will be updated. All other existing values will remain unchanged.
In your case, what you are doing is creating a new empty entity, then setting its Key to something that already exists in the database. When you attach that entity to the context, EF has no choice but to assume that all the values in that entity are the new updated values. This is standard behavior for EF.

Upsert with composite key?

Using EF6.1, I'm working with a table with a composite key as below:
How can I run an upsert method using Entity Framework?
I've written the following method but I've read that it should not be used for upserts, only migrations (please disregard the design pattern for now):
public void UpsertNumberOfMarkets(List<Entities.NumberOfMarkets> marketsList)
{
using (MyDbContext db = new MyDbContext())
{
foreach (var market in marketsList)
{
db.NumberOfMarkets.AddOrUpdate(market);
}
}
}
I'm also not certain that it works correctly. Any thoughts? I'd like to avoid deleting and inserting as we have an audit logging table for updates.
EDIT: I've written the following method which may handle this - is this the preferred approach?
public void UpsertNumberOfMarkets(List<Entities.NumberOfMarkets> marketsList)
{
using (MyDbContext db = new MyDbContext())
{
foreach (var market in marketsList)
{
var predicate = PredicateBuilder.True<Entities.NumberOfMarkets>();
predicate = predicate.And(n => n.ProjectId == market.ProjectId);
predicate = predicate.And(n => n.Year == market.Year);
var existingMarketEntry = db.NumberOfMarkets.AsExpandable().Where(predicate).FirstOrDefault();
if (existingMarketEntry != null)
existingMarketEntry.Markets = market.Markets;
else
{
db.NumberOfMarkets.Add(market);
}
}
db.SaveChanges();
}
}

Yes.AddOrUpdate should only be used for migrations for below reason
It Updates all the values which are provided but mark all other values as NULL (which are not provided) which may not be the behaviour we want in real world application.
In your case you can follow below steps
using (MyDbContext db = new MyDbContext())
{
foreach (var market in marketsList)
{
var existingMarket =
db.Markets.FirstOrDefault(x => x.ProjectID == market.ProjectID && x.Year == market.Year);
if (existingMarket != null)
{
//Set properties for existing market
existingMarket.Year == market.Year
//etc
}
else
{
db.Markets.Add(market);
}
db.SaveChanges();
}
}