Develop Reference

Performance issue when performing operations on entity objects

Im facing performance issue in below code in multiple foreach loops. First im getting a list of ReturnDetails and then based on detail id get the HandlingInfo object. Then based on value of action, update the ReturnsDetail Object again.
It take more than a minute for loading 3000 records of ReturnsDetail. While debugging locally, it runs for infinite amount of time.
Please let me know in anyway i can refactor this code .
Thanks for your help.
lstReturnsDetail = dcReturnsService.GetReturnDetailsInfo(header_id);
List<HandlingInfo> lstHandlingInfo = null;
foreach (ReturnsDetail oReturnsDetail in lstReturnsDetail)
{
using (DCReturns_Entities entities = new DCReturns_Entities())
{
lstHandlingInfo = entities.HandlingInfoes.Where(f => f.detail_id == oReturnsDetail.id).ToList();
if(lstHandlingInfo != null)
{
foreach (HandlingInfo oHandlingInfo in lstHandlingInfo)
{
if (oHandlingInfo.action == "DST")
{
oReturnsDetail.destroy += Convert.ToInt32(oHandlingInfo.qty);
}
else if (oHandlingInfo.action == "SHP")
{
oReturnsDetail.to_shop += Convert.ToInt32(oHandlingInfo.qty);
}
else if (oHandlingInfo.action == "RBX")
{
oReturnsDetail.in_stock += Convert.ToInt32(oHandlingInfo.qty);
}
}
}
}
oReturnsDetail.received_qty = oReturnsDetail.destroy + oReturnsDetail.to_shop + oReturnsDetail.in_stock;
}
dgReturnsDetail.DataSource = lstReturnsDetail.OrderByDescending(g => g.id).ToList();
Session[DCReturnsConstants.Returns_Detail_Entity] = lstReturnsDetail;
dgReturnsDetail.DataBind();

this is su-do code! but you should get the jist.
//modify this to return all of them into mem, and then filter on this...
//if it can not be done here then do below..
var lstReturnsDetail = dcReturnsService.GetReturnDetailsInfo(header_id);
//then create a list here which fetches all,
List<[type]> somelist
List<int> listId = lstReturnsDetail.select(x=>x.id).tolist();
using (var db = new DCReturns_Entities())
{
somelist = db.HandlingInfoes.Where(f => listId.Contains( f.detail_id)).ToList();
}
foreach (ReturnsDetail oReturnsDetail in lstReturnsDetail)
{
//performance issue is here
//using (DCReturns_Entities entities = new DCReturns_Entities())
//{
// lstHandlingInfo = entities.HandlingInfoes.Where(f => f.detail_id == oReturnsDetail.id).ToList();
//}
//insead fetach all before, into mem and filter from that list.
var lstHandlingInfo = somelist.Where(f => f.detail_id == oReturnsDetail.id).ToList();
//code ommited for reaablity
}
//code ommited for reaablity

JSON Array to Entity Framework Core VERY Slow?

I'm working on a utility to read through a JSON file I've been given and to transform it into SQL Server. My weapon of choice is a .NET Core Console App (I'm trying to do all of my new work with .NET Core unless there is a compelling reason not to). I have the whole thing "working" but there is clearly a problem somewhere because the performance is truly horrifying almost to the point of being unusable.
The JSON file is approximately 27MB and contains a main array of 214 elements and each of those contains a couple of fields along with an array of from 150-350 records (that array has several fields and potentially a small <5 record array or two). Total records are approximately 35,000.
In the code below I've changed some names and stripped out a few of the fields to keep it more readable but all of the logic and code that does actual work is unchanged.
Keep in mind, I've done a lot of testing with the placement and number of calls to SaveChanges() think initially that number of trips to the Db was the problem. Although the version below is calling SaveChanges() once for each iteration of the 214-record loop, I've tried moving it outside of the entire looping structure and there is no discernible change in performance. In other words, with zero trips to the Db, this is still SLOW. How slow you ask, how does > 24 hours to run hit you? I'm willing to try anything at this point and am even considering moving the whole process into SQL Server but would much reather work in C# than TSQL.
static void Main(string[] args)
{
string statusMsg = String.Empty;
JArray sets = JArray.Parse(File.ReadAllText(#"C:\Users\Public\Downloads\ImportFile.json"));
try
{
using (var _db = new WidgetDb())
{
for (int s = 0; s < sets.Count; s++)
{
Console.WriteLine($"{s.ToString()}: {sets[s]["name"]}");
// First we create the Set
Set eSet = new Set()
{
SetCode = (string)sets[s]["code"],
SetName = (string)sets[s]["name"],
Type = (string)sets[s]["type"],
Block = (string)sets[s]["block"] ?? ""
};
_db.Entry(eSet).State = Microsoft.EntityFrameworkCore.EntityState.Added;
JArray widgets = sets[s]["widgets"].ToObject<JArray>();
for (int c = 0; c < widgets.Count; c++)
{
Widget eWidget = new Widget()
{
WidgetId = (string)widgets[c]["id"],
Layout = (string)widgets[c]["layout"] ?? "",
WidgetName = (string)widgets[c]["name"],
WidgetNames = "",
ReleaseDate = releaseDate,
SetCode = (string)sets[s]["code"]
};
// WidgetColors
if (widgets[c]["colors"] != null)
{
JArray widgetColors = widgets[c]["colors"].ToObject<JArray>();
for (int cc = 0; cc < widgetColors.Count; cc++)
{
WidgetColor eWidgetColor = new WidgetColor()
{
WidgetId = eWidget.WidgetId,
Color = (string)widgets[c]["colors"][cc]
};
_db.Entry(eWidgetColor).State = Microsoft.EntityFrameworkCore.EntityState.Added;
}
}
// WidgetTypes
if (widgets[c]["types"] != null)
{
JArray widgetTypes = widgets[c]["types"].ToObject<JArray>();
for (int ct = 0; ct < widgetTypes.Count; ct++)
{
WidgetType eWidgetType = new WidgetType()
{
WidgetId = eWidget.WidgetId,
Type = (string)widgets[c]["types"][ct]
};
_db.Entry(eWidgetType).State = Microsoft.EntityFrameworkCore.EntityState.Added;
}
}
// WidgetVariations
if (widgets[c]["variations"] != null)
{
JArray widgetVariations = widgets[c]["variations"].ToObject<JArray>();
for (int cv = 0; cv < widgetVariations.Count; cv++)
{
WidgetVariation eWidgetVariation = new WidgetVariation()
{
WidgetId = eWidget.WidgetId,
Variation = (string)widgets[c]["variations"][cv]
};
_db.Entry(eWidgetVariation).State = Microsoft.EntityFrameworkCore.EntityState.Added;
}
}
}
_db.SaveChanges();
}
}
statusMsg = "Import Complete";
}
catch (Exception ex)
{
statusMsg = ex.Message + " (" + ex.InnerException + ")";
}
Console.WriteLine(statusMsg);
Console.ReadKey();
}

I had an issue with that kind of code, lots of loops and tons of changing state.
Any change / manipulation you make in _db context, will generate a "trace" of it. And it making your context slower each time. Read more here.
The fix for me was to create new EF context(_db) at some key points. It saved me a few hours per run!
You could try to create a new instance of _db each iteration in this loop
contains a main array of 214 elements
If it make no change, try to add some stopwatch to get a best idea of what/where is taking so long.

If you're making thousands of updates then EF is not really the way to go. Something like SQLBulkCopy will do the trick.
You could try the bulkwriter library.
IEnumerable<string> ReadFile(string path)
{
using (var stream = File.OpenRead(path))
using (var reader = new StreamReader(stream))
{
while (reader.Peek() >= 0)
{
yield return reader.ReadLine();
}
}
}
var items =
from line in ReadFile(#"C:\products.csv")
let values = line.Split(',')
select new Product {Sku = values[0], Name = values[1]};
then
using (var bulkWriter = new BulkWriter<Product>(connectionString)) {
bulkWriter.WriteToDatabase(items);
}

Getting The Wait Operation Timeout Exception for a query from Skip Value 100

I am writing a small data migration tools from one big database to another small database. All of the others data migration method worked satisfactorily, but the following method has given an exception from the SKIP VALUE IS 100. I run this console script remotely as well as inside of the source server also. I tried in many different was to find the actual problem what it is. After then I found that only from the SKIP VALUE IS 100 it is not working for any TAKE 1,2,3,4,5 or ....
Dear expertise, I don't have any prior knowledge on that type of problem. Any kind of suggestions or comments is appreciatable to resolve this problem. Thanks for you time.
I know this code is not clean and the method is too long. I just tried solve this by adding some line of extra code. Because the problem solving is my main concern. I just copy past the last edited method.
In shot the problem I can illustrate with this following two line
var temp = queryable.Skip(90).Take(10).ToList(); //no exception
var temp = queryable.Skip(100).Take(10).ToList(); getting exception
private static void ImporterDataMigrateToRmgDb(SourceDBEntities sourceDb, RmgDbContext rmgDb)
{
int skip = 0;
int take = 10;
int count = sourceDb.FormAs.Where(x=> x.FormAStateId == 8).GroupBy(x=> x.ImporterName).Count();
Console.WriteLine("Total Possible Importer: " + count);
for (int i = 0; i < count/take; i++)
{
IOrderedQueryable<FormA> queryable = sourceDb.FormAs.Where(x => x.FormAStateId == 8).OrderBy(x => x.ImporterName);
List<IGrouping<string, FormA>> list;
try
{
list = queryable.Skip(skip).Take(take).GroupBy(x => x.ImporterName).ToList();
//this line is getting timeout exception from the skip value of 100.
}
catch (Exception exception)
{
Console.WriteLine(exception.Message);
sourceDb.Dispose();
rmgDb.Dispose();
sourceDb = new SourceDBEntities();
rmgDb = new RmgDbContext();
skip += take;
continue;
}
if (list.Count > 0)
{
foreach (var l in list)
{
List<FormA> formAs = l.ToList();
FormA formA = formAs.FirstOrDefault();
if (formA == null) continue;
Importer importer = formA.ConvertToRmgImporterFromFormA();
Console.WriteLine(formA.FormANo + " " + importer.Name);
var importers = rmgDb.Importers.Where(x => x.Name.ToLower() == importer.Name.ToLower()).ToList();
//bool any = rmgDb.Importers.Any(x => x.Name.ToLower() == formA.ImporterName.ToLower());
if (importers.Count() == 1)
{
foreach (var imp in importers)
{
Importer entity = rmgDb.Importers.Find(imp.Id);
entity.Country = importer.Country;
entity.TotalImportedAmountInUsd = importer.TotalImportedAmountInUsd;
rmgDb.Entry(entity).State = EntityState.Modified;
}
}
else
{
rmgDb.Importers.Add(importer);
}
rmgDb.SaveChanges();
Console.WriteLine(importer.Name);
}
}
skip += take;
}
Console.WriteLine("Importer Data Migration Completed");
}

I have fixed my problem by modifying following code
var queryable =
sourceDb.FormAs.Where(x => x.FormAStateId == 8)
.Select(x => new Adapters.ImporterBindingModel()
{
Id = Guid.NewGuid().ToString(),
Active = true,
Created = DateTime.Now,
CreatedBy = "System",
Modified = DateTime.Now,
ModifiedBy = "System",
Name = x.ImporterName,
Address = x.ImporterAddress,
City = x.City,
ZipCode = x.ZipCode,
CountryId = x.CountryId
})
.OrderBy(x => x.Name);

Fetch the objectId from Parse.ParseObject

I am trying to figure out how to obtain the objectId from an query. What i am trying to accomplish is that I want to know if a object with a specific username and password exists in my class and then if yes, get the objectId and save it in my Unity game!
Here is how I am trying to do so with no luck:
void OnClick(){
PrimeAI.ShowSpinner();
LoginUsername = GameObject.Find("UsernameLogin").GetComponent<UIInput>().value;
LoginPassword = GameObject.Find("PasswordLogin").GetComponent<UIInput>().value;
var query = ParseObject.GetQuery("Players")
.WhereEqualTo("playername", LoginUsername)
.WhereEqualTo("password", Constant.Md5Sum(LoginPassword));
query.CountAsync().ContinueWith(t => {
int count = t.Result;
if(count == 1){
GetUserData(LoginUsername);
} else {
LoginError = true;
}
});
}
public void GetUserData(string UserName){
var query = ParseObject.GetQuery("Players")
.WhereEqualTo("playername", UserName);
query.FindAsync().ContinueWith(t => {
IEnumerable<ParseObject> results = t.Result;
OnlyOnce = true;
foreach (var result in results) {
print("Here: "+result["objectId"]);
}
});
}
It works fine until the foreach loop... How can I accomplish this and what am i doing wrong?
Any help is appreciated :-)

Do a Find rather than a CountAsync. Then just access the objectID from the returned object. If you are concerned about moving the whole object when you don't need it, use selectKeys to select only the attributes you want.
-Bob

foreach (var result in results) {
print("Here: "+ result.objectId);
}

Updating multiple records that share a many to many relationship fails

Details
I have 2 tables (Procedures, Surgeons) with a lookup table (ProcSurg) to create a many to many relationship.
scar_Requests scar_Procedures scar_ProcSurg scar_Surgeons
------------- --------------- ------------- -------------
RequestID <> ProcedureID <> ProcedureID(fk) <> SurgeonID
... RequestID SurgeonID(fk) ...
...
A single request can have multiple procedures and each procedure can have multiple surgeons.
Everything saves correctly until I have 2 procedures each that share the same Surgeon.
Error: InvalidOperationException was unhandled
The relationship between the two objects cannot be defined because they are attached to different ObjectContext objects.
I separated out the code for saving this part of the record to try to isolate my problem..
Addprocedures is a class that contains 1 Procedure and a list of Surgeons
class Procedure
{
public scar_Procedures Procedure { get; set; }
public List<scar_Surgeons> Surgeons { get; set; }
public void RemoveSurgeon(int SurgeonID)
{
Surgeons.Remove(Surgeons.Where(x => x.SurgeonID == SurgeonID).FirstOrDefault());
}
public Procedure()
{
Surgeons = new List<scar_Surgeons>();
}
}
Saving code: using DBContext
private void SaveProcSurg()
{
using (MCASURGContext db2 = new MCASURGContext())
{
foreach (Procedure p in AddProcedures)
{
if (p.Procedure.RequestID == 0)
{
p.Procedure.RequestID = ReqID;
}
p.Procedure.scar_Surgeons.Clear();
foreach (scar_Surgeons s in p.Surgeons)
{
if (db2.ChangeTracker.Entries<scar_Surgeons>().Where(x => x.Entity.SurgeonID == s.SurgeonID).FirstOrDefault() == null)
{
db2.scar_Surgeons.Attach(s);
}
p.Procedure.scar_Surgeons.Add(s);
}
if (p.Procedure.ProcedureID == 0)
{
db2.scar_Procedures.Add(p.Procedure);
db2.Entry(p.Procedure).State = System.Data.Entity.EntityState.Added;
}
else
{
db2.scar_Procedures.Attach(p.Procedure);
db2.Entry(p.Procedure).State = System.Data.Entity.EntityState.Modified;
}
}
db2.SaveChanges();
}
}
I've tried several different ways of saving the record and this is the closest I've come to doing it correctly.
I feel like it has something to do with the way I'm attaching the surgeons to the entity and then to the procedure. Any help, idea's or suggestions on where I can find an answer would be great!
I've been searching google endlessly for over a week and I've been trying to wrap my mind around what exactly Entity Framework is doing but I'm still pretty new to this.
Edited 9/24/2013
Sorry this is the complete code snippet from the comments section with the req variable included
//Internal variable
private scar_Requests req;
private List<Procedure> AddProcedures = new List<Procedure>();
//Gets a scar_Request from the DB
private void GetRequest()
{
using (MCASURGContext db = new MCASURGContext())
{
req = db.scar_Requests.Include("scar_Procedures.scar_Surgeons").Include("scar_Status").Include("scar_Users.scar_Service").Where(x => x.RequestID == ReqID).FirstOrDefault();
foreach (scar_Procedures p in req.scar_Procedures) { AddProcedures.Add(new Procedure() { Proc = p, Surgeons = p.scar_Surgeons.ToList() }); }
}
}

Keeping with good form I'll post my answer since I think I figured it out. Maybe it will help someone in the future.
I completely re-wrote the saving and cut out a lot of useless code that I was using before and less calls to the DB. There was other methods that I didn't post above that saved other parts of the record that I condensed into a single method.
Basically I get the record and its joined tables from the DB and iterate through all the fields/joined tables that need to be updated and save it back to the DB. (Seems super obvious now but I tried this way before and I must have had something wrong because it didn't work the first few times I tried it this way.)
I don't know if its 100% correct or written up to normal coding standards and I still have some final tweaking to do before its completely done.
private void SaveProcSurg()
{
using (MCASURGContext db2 = new MCASURGContext())
{
//Get Record from DB
scar_Requests sReq = db2.scar_Requests.Include("scar_Users").Include("scar_Status").Include("scar_Procedures.scar_Surgeons").Where(x => x.RequestID == ReqID).FirstOrDefault();
//Update Record fields
sReq.CreationDate = req.CreationDate == null ? DateTime.Now : req.CreationDate = req.CreationDate;
sReq.DateOfSurgery = dtpDateOfSurgery.Value;
sReq.IsDeleted = false;
sReq.IsScheduled = false;
sReq.LatexAllergy = cbLatexAllergy.Checked;
sReq.ModifiedDate = DateTime.Now;
sReq.MRN = txtMRN.Text;
sReq.PatientName = txtPatientName.Text;
foreach (RadioButton rb in gbPatientType.Controls) if (rb.Checked == true) sReq.PatientType = rb.Text;
sReq.PreOpDiagnosis = txtPreOpDiag.Text;
sReq.PrimarySurgeon = txtPrimarySurgeon.Text;
sReq.PrivateComment = txtPrivateComment.Text;
sReq.PublicComment = txtPublicComment.Text;
sReq.RequestID = ReqID;
sReq.StatusID = req.StatusID;
sReq.UserID = req.UserID;
//Update Users/Status
sReq.scar_Users = db2.scar_Users.Where(x => x.UserID == sReq.UserID).FirstOrDefault();
sReq.scar_Status = db2.scar_Status.Where(x => x.StatusID == req.StatusID).FirstOrDefault();
//Attach to DBContext
db2.scar_Requests.Attach(sReq);
//Update Procedures
foreach (Procedure p in AddProcedures)
{
scar_Procedures pro = sReq.scar_Procedures.Where(x => x.ProcedureID == p.Proc.ProcedureID && p.Proc.ProcedureID != 0).FirstOrDefault();
if (pro != null)
{
pro.EnRecovery = p.Proc.EnRecovery;
pro.IsPrimary = p.Proc.IsPrimary;
pro.Laterality = p.Proc.Laterality;
pro.OrthoFastTrack = p.Proc.OrthoFastTrack;
pro.ProcedureID = p.Proc.ProcedureID;
pro.ProcedureText = p.Proc.ProcedureText;
pro.RequestID = ReqID;
pro.Site = p.Proc.Site;
}
else
{
pro = new scar_Procedures();
pro.EnRecovery = p.Proc.EnRecovery;
pro.IsPrimary = p.Proc.IsPrimary;
pro.Laterality = p.Proc.Laterality;
pro.OrthoFastTrack = p.Proc.OrthoFastTrack;
pro.ProcedureID = p.Proc.ProcedureID;
pro.ProcedureText = p.Proc.ProcedureText;
pro.RequestID = ReqID;
pro.Site = p.Proc.Site; ;
pro.scar_Requests = sReq;
}
//Update Surgeons
pro.scar_Surgeons.Clear();
foreach (scar_Surgeons s in p.Surgeons)
{
pro.scar_Surgeons.Add(db2.scar_Surgeons.Where(x=> x.SurgeonID == s.SurgeonID).FirstOrDefault());
}
}
//Set State and Save
db2.Entry(sReq).State = System.Data.Entity.EntityState.Modified;
db2.SaveChanges();
}
}