Develop Reference

Why does my parallel.ForEach method return the wrong values?

I have a large List of objects (over 100k rows in a txt file). I have to do some data work with each item in the list I save the object to the database if it doesnt exist and if it does I increase a number column in the for the item and then update the database while saving the updated item.
If I use a regular foreach loop everything works out fine but it takes over 24 hours to finish.
The issue is that when I use a parallel loop the records do not match and sometimes I will get an error on the .Save() method that the item ID was changed.
THIS CODE WORKS
public List<SreckaIsplacena> UpisiUTabeleSerijski (List<SreckaTemp>srt)
{
_srecke = _glavniRepository.UcitajSamoaktivneSrecke().OrderByDescending(item => item.ID).ToList<Srecka>();
List<SreckaIsplacena> pomList = new List<SreckaIsplacena>();
SreckaIsplacena _isplacena;
foreach (SreckaTemp lt in srt)
{
string beznula = lt.sreIspBroj.TrimStart('0');
SreckeDobici srd = new SreckeDobici();
Srecka sr = (from s in _srecke
where s.Sifra == lt.sreSif && s.Serija == lt.sreSerija
select s).First();
List<SreckeDobici> srDob = _glavniRepository.DohvatiSreckeDobiciZaIsplatniBroj(sr, beznula);
if (srDob.Count == 1)
{
srd = srDob.ElementAt(0);
}
List<SreckaNagrade> sreckaNagrade = new List<SreckaNagrade>(_glavniRepository.DohvatiNagradeZaSrecku(sr.ID).OrderBy(item => item.SifraFox));
double iznos = lt.sreIznDob / lt.sreBrDob;
SreckaNagrade nag = (from sn in sreckaNagrade
where sn.Iznos == lt.sreIznDob
select sn).FirstOrDefault();
Odobrenje odo = new Odobrenje();
odo = odo.DohvatiOdobrenje(valutaGlavna.ID, lt.sreIsplatio).FirstOrDefault();
List<PorezSrecka> listaPoreza = _glavniRepository.UcitajPorezSrecka(valutaGlavna, odo, sr, nag.NagradaId);
List<SreckaIsplacena> sveIsplacene = _glavniRepository.DohvatiIsplaceneSreckeZaValutuIProdavacaSreckuNagraduNovo(valutaGlavna.ID, sr.ID, nag.NagradaId, lt.sreIsplatio);
if (sveIsplacene.Count > 1)
{
System.Windows.MessageBox.Show("Greska, ista srecka s istim dobitkom kod istog prodavaca u istoj valuti 2 put nadjena u bazi");
}
else if (sveIsplacene.Count == 1)
{
_isplacena = sveIsplacene.ElementAt(0);
_isplacena.BrojDobitaka = _isplacena.BrojDobitaka + lt.sreBrDob;
_isplacena.Update();
var index = pomList.FindIndex(r => r.ID == _isplacena.ID);
if (index != -1)
{
pomList[index] = _isplacena;
}
}
else if (sveIsplacene.Count==0)
{
_isplacena = new SreckaIsplacena();
decimal iz = Convert.ToDecimal(lt.sreIznDob);
_isplacena.BrojDobitaka = lt.sreBrDob;
_isplacena.Iznos = iz;
_isplacena.Nagrada = nag;
_isplacena.Prodavac = lt.sreIsplatio;
_isplacena.Valuta = valutaGlavna;
_isplacena.Srecka = sr;
_isplacena.Cijena = Convert.ToDecimal(sr.Cijena);
if (listaPoreza.Count == 1)
{
PorezSrecka ps = listaPoreza.ElementAt(0);
_isplacena.SreckaPorez = ps;
}
_isplacena.Save();
int ispID = _isplacena.ID;
if (ispID != 0)
{
srd.Sre_isplatio = lt.sreIsplatio;
srd.Sre_valuta = valutaGlavna;
srd.Update();
}
pomList.Add(_isplacena);
}
}
return pomList;
}
THIS CODE DOES NOT WORK
public List<SreckaIsplacena> UpisiSamoUTabele(ConcurrentBag<SreckaTemp> srt)
{
_srecke = _glavniRepository.UcitajSamoaktivneSrecke().OrderByDescending(item => item.ID).ToList<Srecka>();
List<SreckaIsplacena> pomList = new List<SreckaIsplacena>();
SreckaIsplacena _isplacena;
Parallel.ForEach(srt, (lt) =>
{
string beznula = lt.sreIspBroj.TrimStart('0');
SreckeDobici srd = new SreckeDobici();
Srecka sr = (from s in _srecke
where s.Sifra == lt.sreSif && s.Serija == lt.sreSerija
select s).First();
List<SreckeDobici> srDob = _glavniRepository.DohvatiSreckeDobiciZaIsplatniBroj(sr, beznula);
if (srDob.Count == 1)
{
srd = srDob.ElementAt(0);
}
List<SreckaNagrade> sreckaNagrade = new List<SreckaNagrade>(_glavniRepository.DohvatiNagradeZaSrecku(sr.ID).OrderBy(item => item.SifraFox));
SreckaNagrade nag = (from sn in sreckaNagrade
where sn.Iznos == lt.sreIznDob
select sn).FirstOrDefault();
Odobrenje odo = new Odobrenje();
odo = odo.DohvatiOdobrenje(valutaGlavna.ID, lt.sreIsplatio).FirstOrDefault();
List<PorezSrecka> listaPoreza = _glavniRepository.UcitajPorezSrecka(valutaGlavna, odo, sr, nag.NagradaId);
List<SreckaIsplacena> sveIsplacene = _glavniRepository.DohvatiIsplaceneSreckeZaValutuIProdavacaSreckuNagraduNovo(valutaGlavna.ID, sr.ID, nag.NagradaId, lt.sreIsplatio);
if (sveIsplacene.Count == 1)
{
_isplacena = sveIsplacene.ElementAt(0);
lock (_isplacena)
{
_isplacena.BrojDobitaka = _isplacena.BrojDobitaka + lt.sreBrDob;
_isplacena.Update();
var index = pomList.FindIndex(r => r.ID == _isplacena.ID);
if (index != -1)
{
pomList[index] = _isplacena;
}
}
}
else if (sveIsplacene.Count==0)
{
_isplacena = new SreckaIsplacena();
decimal iz = Convert.ToDecimal(lt.sreIznDob);
_isplacena.BrojDobitaka = lt.sreBrDob;
_isplacena.Iznos = iz;
_isplacena.Nagrada = nag;
_isplacena.Prodavac = lt.sreIsplatio;
_isplacena.Valuta = valutaGlavna;
_isplacena.Srecka = sr;
_isplacena.Cijena = Convert.ToDecimal(sr.Cijena);
lock(_isplacena)
{
if (listaPoreza.Count == 1)
{
PorezSrecka ps = listaPoreza.ElementAt(0);
_isplacena.SreckaPorez = ps;
}
_isplacena.Save();
int ispID = _isplacena.ID;
if (ispID != 0)
{
srd.Sre_isplatio = lt.sreIsplatio;
srd.Sre_valuta = valutaGlavna;
srd.Update();
}
pomList.Add(_isplacena);
}
}
});
return pomList;
}
I tried using ConcurrentBag instead of List but ran into an issue because ConcurrentBag does not have the FindIndex method (var index = pomList.FindIndex(r => r.ID == _isplacena.ID);). I know there is some way to take out the object and put back in but not sure how to do that and from what I read Lists should be ok if you use locking which I am.
Please help.

Insert multiple records in database using parallel.foreach

I am trying to insert around 5000 records in database using foreach loop. It is taking around 10 min which is not acceptable as per the requirement. I also thought about the approach in which first insert the records in a datatable and then converting it into XML pass it to stored procedure which do the insertion. But unfortunately it is not getting fit in my situation. Now i am doing the same thing using parallel.foreach but after inserting 10 records i am getting "Unique constraint violation for the primary key" error msg. As i am new to parallel programming so not getting the solution for it. Below is my code that i have done so for.
public ActionResult ChannelBulkUpload(HttpPostedFileBase excelFile)
{
bool flag = true;
string path = Server.MapPath("~/Content/UploadFolder/" + excelFile.FileName);
if (System.IO.File.Exists(path))
{
System.IO.File.Delete(path);
}
excelFile.SaveAs(path);
DataTable dt = GetDataTableFromExcel(excelFile, true);
ParallelOptions options = new ParallelOptions
{
MaxDegreeOfParallelism = 4
};
Parallel.ForEach(dt.AsEnumerable(), row =>
{
flag = true;
decimal Key = 0;
string value = "";
decimal channelMstKey = 0;
decimal channelGrpMstKey = 0;
decimal srcFuncKey = 0;
string ExcelMasterChDisplayName = row["MASTER_CHANNEL_DISPLAY_NAME"].ToString();
string ExcelGenreValue = row["GENRE"].ToString();
string ExcelAdsharpValue = row["ADSHARP"].ToString();
string ExcelClusterValue = row["CLUSTER"].ToString();
string ExcelNetworkValue = row["NETWORK"].ToString();
string ExcelBroadCastValue = row["BROADCAST"].ToString();
string ExcelFunctionalAreaname = row["FUNCTIONAL_AREA"].ToString();
string[] Ch_Grp_types = { "GENRE", "ADSHARP", "CLUSTER", "NETWORK", "PLATFORM" };
BarcDataContext bc = new BarcDataContext();
srcFuncKey = bc.REF_SRC_FUNC_AREA.Where(m => m.SRC_FUNC_AREA == ExcelFunctionalAreaname).FirstOrDefault().SRC_FUNC_KEY;
for (int j = 0; j < Ch_Grp_types.Length && flag; j++)
{
if (Ch_Grp_types[j] == "GENRE")
{
Key = 1;
value = ExcelGenreValue;
}
else if (Ch_Grp_types[j] == "NETWORK")
{
Key = 2;
value = ExcelNetworkValue;
}
else if (Ch_Grp_types[j] == "ADSHARP")
{
Key = 3;
value = ExcelAdsharpValue;
}
else if (Ch_Grp_types[j] == "CLUSTER")
{
Key = 4;
value = ExcelClusterValue;
}
else if (Ch_Grp_types[j] == "PLATFORM")
{
Key = 5;
value = ExcelBroadCastValue;
}
DIM_CHANNEL_MST objChMst = bc.DIM_CHANNEL_MST.Where(m => m.CHANNEL_MST_NAME_UPPER == ExcelMasterChDisplayName.ToUpper().Trim()).FirstOrDefault();
if (objChMst == null)
{
flag = false;
}
else
{
if (!string.IsNullOrEmpty(value))
{
var query =
(from A in bc.XREF_CH_GRP_DET_TAG
join B in bc.XREF_CH_GRP_MST_TAG on A.CH_GRP_MST_KEY equals B.CH_GRP_MST_KEY
where A.IS_ACTIVE == "Y" && B.IS_ACTIVE == "Y" && B.CH_GRP_TYPE_KEY == Key && B.CH_GRP_MST_NAME_UPPER == value.ToUpper()
select new XrefChannelGrpDetailTagVM
{
channelGrpDetKey = A.CH_GRP_DET_KEY,
channelGrpMasterNameUpper = B.CH_GRP_MST_NAME_UPPER,
}).Distinct().ToList();
var query2 =
(from A in bc.XREF_CH_GRP_DET_TAG
join B in bc.XREF_CH_GRP_MST_TAG on A.CH_GRP_MST_KEY equals B.CH_GRP_MST_KEY
where A.CHANNEL_MST_KEY == objChMst.CHANNEL_MST_KEY && B.CH_GRP_TYPE_KEY == Key && B.SRC_FUNC_KEY == srcFuncKey
select new XrefChannelGrpDetailTagVM
{
sr_no = A.SR_NO,
channelMstKey = A.CHANNEL_MST_KEY,
channelGrpDetKey = A.CH_GRP_DET_KEY,
channelGrpMstKey = A.CH_GRP_MST_KEY,
srcFuncKey = B.SRC_FUNC_KEY,
channelGrpTypeKey = B.CH_GRP_TYPE_KEY
}).Distinct().ToList();
XREF_CH_GRP_MST_TAG objXrefChGrpMst = bc.XREF_CH_GRP_MST_TAG.Where(m => m.CH_GRP_TYPE_KEY == Key && m.SRC_FUNC_KEY == srcFuncKey && m.CH_GRP_MST_NAME_UPPER == value.ToUpper()).FirstOrDefault();
if (objXrefChGrpMst != null)
{
channelMstKey = objChMst.CHANNEL_MST_KEY;
channelGrpMstKey = objXrefChGrpMst.CH_GRP_MST_KEY;
XREF_CH_GRP_DET_TAG objGrpDetail = new XREF_CH_GRP_DET_TAG();
if (query.Count == 0)
{
objGrpDetail.CH_GRP_DET_KEY = Get_Max_Of_Ch_Grp_Det_Key();
}
else
{
foreach (var detKey in query)
{
if (detKey.channelGrpMasterNameUpper == value.ToUpper())
{
objGrpDetail.CH_GRP_DET_KEY = detKey.channelGrpDetKey;
}
else
{
objGrpDetail.CH_GRP_DET_KEY = Get_Max_Of_Ch_Grp_Det_Key();
}
}
}
if (query2.Count > 0)
{
foreach (var abc in query2)
{
if (abc.channelMstKey == objChMst.CHANNEL_MST_KEY && abc.srcFuncKey == srcFuncKey && abc.channelGrpTypeKey == Key)
{
if (abc.channelGrpDetKey == objGrpDetail.CH_GRP_DET_KEY && abc.channelGrpMstKey == objXrefChGrpMst.CH_GRP_MST_KEY)
{
//Reject
}
else
{
//Update
XREF_CH_GRP_DET_TAG obj = bc.XREF_CH_GRP_DET_TAG.Where(m => m.SR_NO == abc.sr_no).FirstOrDefault();
obj.CH_GRP_DET_KEY = objGrpDetail.CH_GRP_DET_KEY;
obj.CH_GRP_MST_KEY = objXrefChGrpMst.CH_GRP_MST_KEY;
objGrpDetail.CREATE_DATE = DateTime.Now;
objGrpDetail.LAST_UPD_DATE = DateTime.Now;
objGrpDetail.IS_ACTIVE = "Y";
bc.SaveChanges();
}
}
else
{
//Insert
objGrpDetail.CH_GRP_MST_KEY = channelGrpMstKey;
objGrpDetail.CHANNEL_MST_KEY = channelMstKey;
objGrpDetail.SR_NO = Get_Max_Of_XREF_CH_GRP_DET();
objGrpDetail.CREATE_DATE = DateTime.Now;
objGrpDetail.LAST_UPD_DATE = DateTime.Now;
objGrpDetail.IS_ACTIVE = "Y";
bc.XREF_CH_GRP_DET_TAG.Add(objGrpDetail);
bc.SaveChanges();
}
}
}
else
{
objGrpDetail.CH_GRP_MST_KEY = channelGrpMstKey;
objGrpDetail.CHANNEL_MST_KEY = channelMstKey;
objGrpDetail.SR_NO = Get_Max_Of_XREF_CH_GRP_DET();
objGrpDetail.CH_GRP_DET_KEY = objGrpDetail.CH_GRP_DET_KEY;
objGrpDetail.CREATE_DATE = DateTime.Now;
objGrpDetail.LAST_UPD_DATE = DateTime.Now;
objGrpDetail.IS_ACTIVE = "Y";
bc.XREF_CH_GRP_DET_TAG.Add(objGrpDetail);
bc.SaveChanges();
}
}
}
}
}
});
TempData["SuccessMsg"] = "Records uploaded Successfully";
return RedirectToAction("CreateChannel");
}
Getting error when generating the primary key value using the below function :
public static decimal Get_Max_Of_XREF_CH_GRP_DET()
{
try
{
BarcDataContext bc = new BarcDataContext();
return bc.XREF_CH_GRP_DET_TAG.Max(m => m.SR_NO) + 1;
}
catch (Exception e)
{
return 1;
}
}
Where SR_NO is the primary key in that table.
Any help will be very much appreciated. Thanks in advance.

The fastest way to do such inserts is using ADO.NET, specifically, SQL Bulk Insert. If you're using SQL Server as your database, the relevant code will be something like this:
DataTable dt = GetDataTableFromExcel(excelFile, true);
using (var copy = new SqlBulkCopy(yourConnectionString) //There are other overloads too
{
BulkCopyTimeout = 10000,
DestinationTableName = dt.TableName,
})
{
foreach (DataColumn column in dt.Columns)
{
copy.ColumnMappings.Add(column.ColumnName, column.ColumnName);
}
copy.WriteToServer(dt);
}

Please look at my comment to other question
You could use guid as primary key into your table. It would help you to avoid problem with ##IDENTITY. At first you should generate new guid-identity, thank insert the generated value into a row

Finding the date of an allday event in google calendar

I have a small C# program to read my Google Calender and return events.
public static IEnumerable<EventEntry> GetAllEvents(CalendarService service, DateTime? startData, DateTime? endData, int nbrToGet)
{
EventQuery query = new EventQuery(); // Create the query object:
if (startData != null)
{
query.StartTime = startData.Value;
}
if (endData != null)
{
query.EndTime = endData.Value;
}
query.SortOrder = CalendarSortOrder.ascending;
query.Uri = new Uri("https://www.google.com/calendar/feeds/" + service.Credentials.Username + "/private/full" + "?max-results=" + nbrToGet.ToString());
EventFeed calFeed = service.Query(query); // Tell the service to query:
return calFeed.Entries.Cast<EventEntry>();
}
I do a foreach on the returned events to display the event details:
foreach (var q in myevents)
{
var one = q.Title.Text;
var two = q.Times.ToList();
if (two.Count > 0)
{
a1 = two[0].StartTime.ToString();
a2 = two[0].EndTime.ToString();
}
else
{
a1 = string.Empty;
a2 = string.Empty;
}
var three = q.Content.Content;
test.Add(new MyCalendarData { Title = one, StartTime = a1, EndTime = a2, Content = three });
}
On all day events, there is no Start time or End time. Is there a way to find the date associated with the event?

Parallel.Foreach giving error " Index was outside the bounds of the array "

I am facing some problem in parallel.foreach which is "Index was outside the bounds of the array". I am attaching some code for parallel.foreach and where it is crashing.
var lstFRItems = session.CreateCriteria<TFRItem>().Add(Restrictions.Eq("TSCEnterprise.FEnterpriseID", EnterpriseId)).AddOrder(Order.Asc("FName")).List<TFRItem>();
List<FRItemAccount> lstItemAccount = new List<FRItemAccount>();
var ListAccounts = session.CreateCriteria<TFRItemAccount>().List<TFRItemAccount>(); //lstFRItems.Select(i => new { i.TFRItemAccounts }).ToList();
//foreach (var item in lstFRItems)
Parallel.ForEach(lstFRItems, item =>
{
var lstItemAcc = ListAccounts.Where(i => i.TFRItem == item); //item.TFRItemAccounts.ToList();
FRItemAccount account = new FRItemAccount();
account.ItemID = item.FItemID;
account.ItemAccount = new List<ItemAccount>();
// foreach (var itemAcct in lstItemAcc)
Parallel.ForEach(lstItemAcc, itemAcct =>
{
ItemAccount oItemAccount = new ItemAccount();
if (itemAcct != null)
{
oItemAccount.ItemAccountID = itemAcct.FItemAccountID;
if (itemAcct.TSCProperty == null)
{
oItemAccount.ForID = itemAcct.TSCCompany.FCompanyID;
oItemAccount.ForCompanyName = "Co# " + "- " + itemAcct.TSCCompany.FID + " " + itemAcct.TSCCompany.FName;
oItemAccount.FID = itemAcct.TSCCompany.FID;
oItemAccount.ForType = 1;
}
else
{
oItemAccount.ForID = itemAcct.TSCProperty.FPropertyID;
oItemAccount.ForCompanyName = "Prop# " + "- " + itemAcct.TSCProperty.FID + " " + itemAcct.TSCProperty.FName;
oItemAccount.FID = itemAcct.TSCProperty.FID;
oItemAccount.ForType = 2;
}
oItemAccount.Account = itemAcct.FAccount;
account.GLAccount = itemAcct.FAccount.ToString("#0.000"); //Formatted by Lhore Bansal
// account.Account = itemAcct.FAccount;
oItemAccount.isExisting = true;
//Original TFRItemAccount
oItemAccount.orgItemAccount = itemAcct;
}
if (lstItemAcc == null)
account.ItemID = item.FItemID;
account.ItemAccount.Add(oItemAccount);
});
//Original tFRItem
account.Item = item;
//account.BaseAccount = Convert.ToDouble(item.FBaseAccount.ToString("F0")); // commented by jeet
account.BaseAccount = Convert.ToDouble((int)item.FBaseAccount); // added by jeet
account.Name = item.FName;
account.Type = item.FType;
lstItemAccount.Add(account);
});
// tx.Commit();
return Item = lstItemAccount;
It is crashing at third last line "lstItemAccount.Add(account)". When I saw in lstItemAccount, It has some counts and in base section it has a error "base {System.SystemException} = {"Source array was not long enough. Check srcIndex and length, and the array's lower bounds."}".
What is the solution of this error?

I would use a ConcurrentBag<T> instead of List<T>. List<T> is designed for only one thread's access.

It has to do with
ListAccounts.Where(i => i.TFRItem == item);
account.ItemAccount.Add(oItemAccount);
lstItemAccount.Add(account);
Lists and Arrays are not Thread safe. Use ConcurrentBag instead

Collection property not properly added to parent on save

Look at the following code example.
What it does:
Iterates a bunch of customers. If it already knows the customer, it retrieves the existing database object for that customer (this is the problem-ridden part). Otherwise, it creates a new object (this works fine).
All loans where the social security number matches (CPR) will be added to the new or existing customer.
The problem: it works for new customer objects, but when I retrieve an existing customer object, the loans lose their relation to the customer when saved (CustomerID = null). They are still saved to the database.
Any ideas?
protected void BuildCustomerData()
{
Console.WriteLine(" Starting the customer build.");
var counter = 0;
var recycleCount = 100;
var reportingCount = 100;
var sTime = DateTime.Now;
var q = from c in db.IntermediaryRkos
select c.CPR;
var distincts = q.Distinct().ToArray();
var numbersToProcess = distincts.Count();
Console.WriteLine(" Identified " + numbersToProcess + " customers. " + (DateTime.Now - sTime).TotalSeconds);
foreach (var item in distincts)
{
var loans = from c in db.IntermediaryRkos
where c.CPR == item
select c;
var existing = db.Customers.Where(x => x.CPR == item).FirstOrDefault();
if (existing != null)
{
this.GenerateLoanListFor(existing, loans);
db.Entry(existing).State = System.Data.EntityState.Modified;
}
else
{
var customer = new Customer
{
CPR = item,
};
this.GenerateLoanListFor(customer, loans);
db.Customers.Add(customer);
db.Entry(customer).State = System.Data.EntityState.Added;
}
counter++;
if (counter % recycleCount == 0)
{
this.SaveAndRecycleContext();
}
if (counter % reportingCount == 0)
{
Console.WriteLine(" Processed " + counter + " customers of " + numbersToProcess + ".");
}
}
db.SaveChanges();
}
protected void GenerateLoanListFor(Customer customer, IQueryable<IntermediaryRko> loans)
{
customer.Loans = new List<Loan>();
foreach (var item in loans.Where(x => x.DebtPrefix == "SomeCategory").ToList())
{
var transformed = StudentLoanMap.CreateFrom(item);
customer.Loans.Add(transformed);
db.Entry(transformed).State = System.Data.EntityState.Added;
}
}
EDIT 1:
As pointed out, I am manually setting the state. This is due to the RecycleContext call, which is implemented for maximum db transaction performance:
protected void SaveAndRecycleContext()
{
db.SaveChanges();
db.Dispose();
db = new SolutionDatabase();
db.Configuration.AutoDetectChangesEnabled = false;
db.Configuration.ValidateOnSaveEnabled = false;
}

Existing loan or not, you wipe out the customer.Loans Property when you call
customer.Loans = new List<Loan>();