I am accessing a database with c#. Now I have to make some calculations with data that I get from some tables and write it into other existing tables. This works quite good in most cases, but for complex operations it takes a huge amount of time. Now I want to know what would be good practice to speed up my querys and get to my results. Here is what I do:
I get a data table that contains 3 values (lom(unique id), laktanfang, laktende) that contains about 700 rows.
For each row in this table I do a query from another table. This results in another data table containing 2 values (lom(unique id), behanddatum)
Now I check if the the value of behanddatum is in between laktanfang and laktende --> yes: Add the row to the data table that gets returned by my function --> no: go on
In the end I have to get the number of positive results from my data table
Here is the code I currently use. I hope it's not too confusing.
public DataTable HoleAbgeschlosseneLaktationenMitDiagnosen(DateTime daAnfang, DateTime daEnde, string[] stDiagnosen = null)
{
DataTable dtRet = new DataTable();
dtRet.Columns.Add("lom", typeof(string));
dtRet.Columns.Add("laktanfang", typeof(DateTime));
dtRet.Columns.Add("laktende", typeof(DateTime));
DataTable dtAbgänge = HoleAbgängeVonEinzeltierZugang(daEnde, daAnfang);
//Abgeschlossene Laktationen für abgegegangene Tiere
foreach (DataRow dr in dtAbgänge.Rows)
{
if (dr != null)
{
DateTime daAbgangsdatum = dr.Field<DateTime>("abgangsdatum");
string stLom = dr.Field<string>("lom");
DataTable dtKalbungVorAbgang = HoleLetzteKalbungFuerTier(stLom, daAbgangsdatum);
if (dtKalbungVorAbgang.Rows.Count > 0 && !dtKalbungVorAbgang.Rows[0].IsNull("kalbedatum"))
{
DateTime daKalbedatum = (DateTime)dtKalbungVorAbgang.Rows[0]["kalbedatum"];
int inLaktation = (int)dtKalbungVorAbgang.Rows[0]["laktation"];
if (PrüfeObDiagnoseInZeitraumAufgetreten(stLom, stDiagnosen, daKalbedatum, daAbgangsdatum) || stDiagnosen == null)
{
DataRow drLaktAbgang = dtRet.NewRow();
drLaktAbgang["lom"] = stLom;
drLaktAbgang["laktanfang"] = daKalbedatum;
drLaktAbgang["laktende"] = daAbgangsdatum;
dtRet.Rows.Add(drLaktAbgang);
}
if (daKalbedatum >= daAnfang && inLaktation > 1)
{
DataTable dtVorherigeKalbung = HoleLetzteKalbungFuerTier(stLom, daKalbedatum.AddDays(-1));
DateTime daVorhKalbung = (DateTime)dtVorherigeKalbung.Rows[0]["kalbedatum"];
if (dtVorherigeKalbung.Rows.Count > 0 && !dtVorherigeKalbung.Rows[0].IsNull("kalbedatum"))
{
if (PrüfeObDiagnoseInZeitraumAufgetreten(stLom, stDiagnosen, daKalbedatum, daAbgangsdatum) || stDiagnosen == null)
{
DataRow drLaktVorhKalbung = dtRet.NewRow();
drLaktVorhKalbung["lom"] = stLom;
drLaktVorhKalbung["laktanfang"] = daVorhKalbung;
drLaktVorhKalbung["laktende"] = daKalbedatum;
dtRet.Rows.Add(drLaktVorhKalbung);
}
}
}
}
}
}
DataTable dtKalbungen = HoleKalbungenFürLebendTiere(daEnde, daAnfang);
//Abgeschlossene Laktationen für lebende Tiere
foreach (DataRow dr in dtKalbungen.Rows)
{
DateTime daKalbedatumLetzte = dr.Field<DateTime>("kalbedatum");
string stLom = dr.Field<string>("lom");
int inLaktation = dr.Field<int>("laktation");
if (inLaktation > 1)
{
DataTable dtKalbungVorErster = HoleLetzteKalbungFuerTier(stLom, daKalbedatumLetzte.AddDays(-1));
if (!dtKalbungVorErster.Rows[0].IsNull("kalbedatum"))
{
DateTime daKalbedatum = (DateTime)dtKalbungVorErster.Rows[0]["kalbedatum"];
if (PrüfeObDiagnoseInZeitraumAufgetreten(stLom, stDiagnosen, daKalbedatum, daKalbedatumLetzte) || stDiagnosen == null)
{
DataRow drLaktKalbung = dtRet.NewRow();
drLaktKalbung["lom"] = stLom;
drLaktKalbung["laktanfang"] = daKalbedatum;
drLaktKalbung["laktende"] = daKalbedatumLetzte;
dtRet.Rows.Add(drLaktKalbung);
}
inLaktation = (int)dtKalbungVorErster.Rows[0]["laktation"];
if (daKalbedatum >= daAnfang && inLaktation > 1)
{
DataTable dtVorherigeKalbung = HoleLetzteKalbungFuerTier(stLom, daKalbedatum.AddDays(-1));
if (dtVorherigeKalbung.Rows.Count > 0 && !dtVorherigeKalbung.Rows[0].IsNull("kalbedatum"))
{
DateTime daVorhKalbung = (DateTime)dtVorherigeKalbung.Rows[0]["kalbedatum"];
if (PrüfeObDiagnoseInZeitraumAufgetreten(stLom, stDiagnosen, daVorhKalbung, daKalbedatum) || stDiagnosen == null)
{
DataRow drLaktVorhKalbung = dtRet.NewRow();
drLaktVorhKalbung["lom"] = stLom;
drLaktVorhKalbung["laktanfang"] = daVorhKalbung;
drLaktVorhKalbung["laktende"] = daKalbedatum;
dtRet.Rows.Add(drLaktVorhKalbung);
}
}
}
}
}
}
return dtRet;
}
private bool PrüfeObDiagnoseInZeitraumAufgetreten(string stLom, string[] stDiagnosen, DateTime daAnfang, DateTime daEnde)
{
SqlCommand cmd = new SqlCommand();
DataTable dtDiagnosenGefunden = new DataTable();
cmd.CommandText = "SELECT diagnose " +
"FROM b_milch_hms_diagnose " +
"WHERE lom=#lom AND behanddatum >= #datumanfang AND behanddatum <= #datumende";
if (stDiagnosen != null)
{
int i = 0;
foreach (string st in stDiagnosen)
{
if (st != "")
{
if (i == 0)
cmd.CommandText += " AND diagnose LIKE #gesuchte_diagnose" + i;
else
cmd.CommandText += " OR diagnose LIKE #gesuchte_diagnose" + i;
cmd.Parameters.AddWithValue("#gesuchte_diagnose" + i, st + "%");
}
i++;
}
}
cmd.Parameters.AddWithValue("#lom", stLom);
cmd.Parameters.AddWithValue("#datumanfang", daAnfang);
cmd.Parameters.AddWithValue("#datumende", daEnde);
dtDiagnosenGefunden = w_milch.FühreSqlAus(cmd);
if (dtDiagnosenGefunden.Rows.Count > 0 && !dtDiagnosenGefunden.Rows[0].IsNull("diagnose"))
return true;
return false;
}
I hope you can help me to improve this function to work more efficient or at least give me some hints.
Thanks in advance
You have created a N+1 problem. One way to solve this would be to change HoleAbgängeVonEinzeltierZugang so that it joins in the data you need from the b_milch_hms_diagnose table.
If you are using .NET 4.0, you can also try to use parallel foreach and see the impact it has on the loop execution time. (This is a more general advice you could apply to many examples)
_dtAbgänge.Rows.AsParallel().ForEach(dr=>
{
//do work
});
Related
I have a Stock Configuration Application and every stock I have amount of items. Every each item, to show in my GridView I have to add some column to my database View. Here is my code:
EuroPOS_BusinessLayer.Item b = new EuroPOS_BusinessLayer.Item();
DataTable dt = b.ListViewByID(Convert.ToInt32(grdSearchForItems.DataKeys[grdSearchForItems.SelectedIndex].Values["item_id"]));
dt.Columns.Add("countedStorage", typeof(Int32));
dt.Columns.Add("countedStorageText", typeof(String));
dt.Columns.Add("countedQuantity", typeof(Decimal));
dt.Columns.Add("countedDescription");
if (!dt.Columns.Contains("PROC"))
dt.Columns.Add("PROC");
dt.Rows[0]["countedStorage"] = Convert.ToInt32(ddlItemSelectedStorage.SelectedValue);
dt.Rows[0]["countedStorageText"] =(ddlItemSelectedStorage.SelectedItem.Text);
dt.Rows[0]["countedQuantity"] = Convert.ToDecimal(txtItemCountedQuantity.Text);
dt.Rows[0]["countedDescription"] = txtItemDescription.Text;
dt.Rows[0]["PROC"] = "INS";
DataTable _tblItems = (DataTable)ViewState["TblItems"];
if (_tblItems == null)
{
_tblItems = dt;
}
else
{
DataRow dr = _tblItems.Select("item_id = " + grdSearchForItems.DataKeys[grdSearchForItems.SelectedIndex].Values[0] + " AND countedStorage = " + ddlItemSelectedStorage.SelectedValue).FirstOrDefault();
if (dr != null)
{
dr["countedQuantity"] = txtItemCountedQuantity.Text;
dr["countedDescription"] = txtItemDescription.Text;
if (dr.Table.Columns.Contains("IdNumber") && dr["IdNumber"] != DBNull.Value)
dr["PROC"] = "UPD";
}
else
{
_tblItems.Merge(dt);
}
}
After through tough research, I can not find a solution for that. Can you please help me?
Edit: I tried table1.Merge(TempTable, True, MissingSchemaAction.Ignore) but does not work!
You have a datatype missmatch:
In one case decimal:
dt.Rows[0]["countedQuantity"] = Convert.ToDecimal(txtItemCountedQuantity.Text);
And the other string:
dr["countedQuantity"] = txtItemCountedQuantity.Text;
The view expects one of the two, so stick to the same in both cases.
I am using below query
DataRow[] oFilteredRows =
oTrgFTPTrigList.Select($"ftpPath='{oSrcRow["ftpPath"]}'
and ftpUserName='{oSrcRow["ftpUserName"]}'");
But it fails since ftpPath is url ftp://testURL/
I need to filter and get records with same data as of source (Comparing target and source DataBases)
Here is the answer.
I have converted data like below.
string ftpPath = Uri.EscapeUriString(oSrcRow["ftpPath"].ToString());
Here is the full code sample.
public DataTable compareFTPTriggers(string targetConnectionString)
{
DataTable oFTPTriggerTableDiffs = new DataTable("oFTPTriggerDiffs");
oFTPTriggerTableDiffs.Columns.Add("oFTPTriggerID");
oFTPTriggerTableDiffs.Columns.Add("oFTPTriggerName");
oFTPTriggerTableDiffs.Columns.Add("Comments");
try
{
deFTPTrigger oSrcFTPTrigger = new deFTPTrigger(m_connectString, m_externalUserID);
DataTable oSrcFTPTrigList = oSrcFTPTrigger.getAllFTPTriggers();
string systemUserID = clsSettings.systemUserID(targetConnectionString);
deFTPTrigger oTrgFTPTrigger = new deFTPTrigger(targetConnectionString, systemUserID);
DataTable oTrgFTPTrigList = oTrgFTPTrigger.getAllFTPTriggers();
foreach (DataRow oSrcRow in oSrcFTPTrigList.Rows)
{
string ftpPath = Uri.EscapeUriString(oSrcRow["ftpPath"].ToString());
DataRow[] oFilteredRows = oTrgFTPTrigList.Select($"ftpPath= '{ftpPath}' and ftpUserName='{oSrcRow["ftpUserName"]}'");
string sKey = $"{oSrcRow["ftpPath"]} - {oSrcRow["ftpUserName"]}";
if (oFilteredRows.Length == 0)
{
oFTPTriggerTableDiffs.Rows.Add(oSrcRow["ftpTriggerID"], sKey, "Addition");
}
else
{
if (
oSrcRow["ftpPassword"].ToString() != oFilteredRows[0]["ftpPassword"].ToString() ||
oSrcRow["waitTime"].ToString() != oFilteredRows[0]["waitTime"].ToString() || oSrcRow["waitType"].ToString() != oFilteredRows[0]["waitType"].ToString() ||
oSrcRow["definitionID"].ToString() != oFilteredRows[0]["definitionID"].ToString() || oSrcRow["variableName"].ToString() != oFilteredRows[0]["variableName"].ToString() ||
oSrcRow["enabled"].ToString() != oFilteredRows[0]["enabled"].ToString() || oSrcRow["globalName"].ToString() != oFilteredRows[0]["globalName"].ToString()
)
{
oFTPTriggerTableDiffs.Rows.Add(oSrcRow["ftpTriggerID"], sKey, "Properties are different");
}
}
}
}
catch (Exception ex)
{
// m_oErrorProvider.writeError(ex.Message);
}
DataView dvSorted = oFTPTriggerTableDiffs.DefaultView;
dvSorted.Sort = "oFTPTriggerName ASC";
oFTPTriggerTableDiffs = dvSorted.ToTable();
return (oFTPTriggerTableDiffs);
}
I have over 200,000 records in c# Winforms gridview, it takes around an hour to get inserted into my database. I'm trying to improve the performance of this insert. I'm looking to insert all of the records within 5 to 10 minutes.
I am using For loop to populate each and every row to get insert into DB with a SQL transactions and I don't think that SqlBulkCopy will work out because all 200,000 records needs to be validated with the DB before insertion into DB.
Save-Function:
if (chkretailprice.Checked)
{
DataTable dt_grid = (DataTable)gcPromotion.DataSource;
dt_grid.AcceptChanges();
for (int tt = 0; tt < gvPromotion.RowCount; tt++)
{
gvPromotion.FocusedRowHandle = tt;
double dRGridMinus = Convert.ToDouble(gvPromotion.GetRowCellValue(tt, gvPromotion.Columns["PromotionalRetailPrice"]));
string sItem = Convert.ToString(gvPromotion.GetRowCellValue(tt, gvPromotion.Columns["ItemName"]).ToString());
string sPack = Convert.ToString(gvPromotion.GetRowCellValue(tt, gvPromotion.Columns["Package"]).ToString());
if (dRGridMinus < 0)
{
gvPromotion.FocusedRowHandle = tt;
MessageBoxInfo("Promotional RetailPrice contains Negative Values for this ItemName-'" + sItem + "' & Package-'" + sPack + "'");
gvPromotion.Focus();
return;
}
}
int iReCount = dt_grid.Select("PromotionalRetailPrice='0.00'").Length;
if (iReCount != 0)
{
MessageBoxInfo("Promotional RetailPrice Must not be 0");
gvPromotion.Focus();
return;
}
}
if (rgPromotion.Checked)
{
for (int p = 0; p < gvPromotion.RowCount; p++)
{
string[] sbranchArr = sBranchIDs.Split(',');
for (int pp = 0; pp < sbranchArr.Length; pp++)
{
objProEntity.PromotionMasterId = objProEntity.PromotionMasterId;
objProEntity.BranchId = Convert.ToInt32(sbranchArr[pp]);//gvPromotion.GetRowCellValue(p, gvPromotion.Columns["BranchID"]));
objProEntity.ItemId = Convert.ToInt64(gvPromotion.GetRowCellValue(p, gvPromotion.Columns["ItemID"]));
objProEntity.PackId = Convert.ToInt32(gvPromotion.GetRowCellValue(p, gvPromotion.Columns["PackTypeID"]));
objProEntity.PromotionValueType = Convert.ToString(gvPromotion.GetRowCellValue(p, gvPromotion.Columns["PromotionValueType"]));
objProEntity.PromotionValue = Convert.ToString(gvPromotion.GetRowCellValue(p, gvPromotion.Columns["PromotionValue"]));
if (chkretailprice.Checked && chkwholesaleprice.Checked)// when both retailprice & wholesaleprice checkbox is checked
{
objProEntity.ActualRetailPrice = Convert.ToDecimal(gvPromotion.GetRowCellValue(p, gvPromotion.Columns["ActualRetailPrice"]));
objProEntity.PromoRetailPrice = Convert.ToDecimal(gvPromotion.GetRowCellValue(p, gvPromotion.Columns["PromotionalRetailPrice"]));
objProEntity.ActualWholeSalePrice = Convert.ToDecimal(gvPromotion.GetRowCellValue(p, gvPromotion.Columns["ActualWholeSalePrice"]));
objProEntity.PromoWholesalePrice = Convert.ToDecimal(gvPromotion.GetRowCellValue(p, gvPromotion.Columns["PromotionalWholeSalePrice"]));
}
else if (chkretailprice.Checked)// when retailprice checkbox is checked
{
objProEntity.ActualRetailPrice = Convert.ToDecimal(gvPromotion.GetRowCellValue(p, gvPromotion.Columns["ActualRetailPrice"]));
objProEntity.PromoRetailPrice = Convert.ToDecimal(gvPromotion.GetRowCellValue(p, gvPromotion.Columns["PromotionalRetailPrice"]));
objProEntity.ActualWholeSalePrice = Convert.ToDecimal(gvPromotion.GetRowCellValue(p, gvPromotion.Columns["ActualWholeSalePrice"]));
objProEntity.PromoWholesalePrice = Convert.ToDecimal(0);
}
else if (chkwholesaleprice.Checked)// when wholesaleprice checkbox is checked
{
objProEntity.ActualRetailPrice = Convert.ToDecimal(gvPromotion.GetRowCellValue(p, gvPromotion.Columns["ActualRetailPrice"]));
objProEntity.PromoRetailPrice = Convert.ToDecimal(0);
objProEntity.ActualWholeSalePrice = Convert.ToDecimal(gvPromotion.GetRowCellValue(p, gvPromotion.Columns["ActualWholeSalePrice"]));
objProEntity.PromoWholesalePrice = Convert.ToDecimal(gvPromotion.GetRowCellValue(p, gvPromotion.Columns["PromotionalWholeSalePrice"]));
}
objProEntity.DiscountAllowed = Convert.ToBoolean(gvPromotion.GetRowCellValue(p, gvPromotion.Columns["DiscountAllowed"]));
DataTable dt_Check = new DataTable();
dt_Check = SalesPromotionData.IsCheckItemExists(objProEntity, SQLTrans);
if (dt_Check.Rows.Count == 0)
{
if (!IsEdit)
{
DataTable dt_child = SalesPromotionData.InsertChildData(objProEntity, SQLTrans); // Insert Child Details when isEdit=false
}
else
{
if (gvPromotion.Columns.Contains(gvPromotion.Columns["PromotionChildId"]))
if ((DBNull.Value.Equals(gvPromotion.GetRowCellValue(p, gvPromotion.Columns["PromotionChildId"]))) || (gvPromotion.GetRowCellValue(p, gvPromotion.Columns["PromotionChildId"]) == "") || Convert.ToString(gvPromotion.GetRowCellValue(p, gvPromotion.Columns["PromotionChildId"]).ToString()) == "0")
{
objProEntity.PromotionMasterId = masterid;
SalesPromotionData.InsertChildData(objProEntity, SQLTrans);// insert child details when isEdit=true
}
else
{
objProEntity.PromotionChildId = Convert.ToInt64(gvPromotion.GetRowCellValue(p, gvPromotion.Columns["PromotionChildId"]).ToString());
SalesPromotionData.UpdateChildDetails(objProEntity, SQLTrans); // update child details when isEdit=true
}
else
{
objProEntity.PromotionMasterId = masterid;
SalesPromotionData.InsertChildData(objProEntity, SQLTrans);// insert child details when isEdit=true
}
}
}
}
}
}
Normally, you'd stage your data into the database by bulk inserting it into [a] work table(s), with no referential integrity or anything -- just the raw data plus any [non-unique] indices you might need. Once you've got it staged, you can then:
Validate the data in the work table(s) against your database and
apply it to the "real" tables in question.
I'm trying to update a DataTable that is part of a DataSet. I search through the table using a DataRow then pass the value into a DataRow for updating to the DataSet, however this is not working, can anyone advise where I am going wrong?
foreach(DataRow drHierarchyTrueValueList in ds.Tables["HierarchyTypeValueList"].Rows)
{
foreach(DataRow drHierarchyListToUpdate in ds.Tables["Hierarchy"].Rows)
{
if (drHierarchyListToUpdate["HierarchyParent"] ==
drHierarchyTrueValueList["HierarchyValueDescription"] &&
Convert.ToInt32(drHierarchyListToUpdate["HierarchyParentType"]) != 0)
{
drHierarchyListToUpdate["HierarchyParentValue"] =
Convert.ToInt32(drHierarchyTrueValueList["HierarchyTrueValue"]);
}
}
ds.Tables["Hierarchy"].AcceptChanges();
}
I changed it to this, and it worked:
foreach(DataRow drHierarchyListToUpdate in ds.Tables["Hierarchy"].Rows)
{
string HierarchyParent = drHierarchyListToUpdate["HierarchyParent"].ToString();
string HierarchyValueDescription = drHierarchyTrueValueList["HierarchyValueDescription"].ToString();
int HierarchyParentType = Convert.ToInt32(drHierarchyListToUpdate["HierarchyParentType"]);
if (HierarchyParent == HierarchyValueDescription && HierarchyParentType != 0)
{
drHierarchyListToUpdate["HierarchyParentValue"] = Convert.ToInt32(drHierarchyTrueValueList["HierarchyTrueValue"]);
}
else if (HierarchyParent != HierarchyValueDescription && HierarchyParentType == 0)
{
drHierarchyListToUpdate["HierarchyParentValue"] = 0;
}
}
ds.Tables["Hierarchy"].AcceptChanges();
I get a datatable (ora_dt) from an Oracle DB, now i need to add an column(colu_sql) in ora_dt,however i have to get colu_sql's value from some Sqlserver DBs.
Here is my code:
public void ProcessDataTable(DataSet _ds)
{
_ds.Tables[0].Columns.Add(new DataColumn("Descpition", typeof(string)));
int countryId = -1;
string des_ID = string.Empty;
string geo = string.Empty;
foreach (DataRow row in _ds.Tables[0].Rows)
{
if (row["des_ID"] != DBNull.Value)
des_ID = row["des_ID"].ToString();
if (!string.IsNullOrEmpty(des_ID))
{
if (countryId == 12 || countryId == 13)
geo = "NA";
else if ((countryId == 10 || countryId == 11))
geo = "LA";
else
geo = "EMEA";
row["Descpition"] = GetDes(geo, des_ID);
}
else { row["ExemptionDes"] = string.Empty; }
}
}
For every DataRow, inorder to get row["Descpition"] value, i have to check its geo and des_id, and select them from another SqlserverDB.
If the row counts in DataTable is very large, then when i foreach the DataTable, i have to visit the sqlserver db many times, it makes performance bad,
Actually i cannot add new column description in Oracle.how can get high performance when foreach DataRow in a DataTable in my code?
private string GetDes(string geo, string des_ID)
{
string description = string.Empty;
string query = "select description from geo_exemption where des_ID= " + des_ID;
Database DbSQL = DbSQLFactory.CreateDatabase(geo);
using (DataReader dr = DbSQL.ExecuteReader(sqlCmd))
{
while (dr.Read())
{
if (dr["description"] != null)
description = dr["description"].ToString();
}
dr.Close();
}
return description;
}
My suggestion would be getting all the records for description and des_ID from geo_exemption in a single datatable at one go and then using LINQ to filter out the records based on des_ID. In this way you would need to hit the database only once. Rest all the operations will happen on asp.net side.
EDIT:
public void ProcessDataTable(DataSet _ds)
{
if (Session["GeoExpAllRec"] == null)
{
//Fetch all records here and add it to a datatable i.e. "select des_ID, description from geo_exemption"
//Then Add the datatable to the session variable Session["GeoExpAllRec"]
}
_ds.Tables[0].Columns.Add(new DataColumn("Descpition", typeof(string)));
int countryId = -1;
string des_ID = string.Empty;
string geo = string.Empty;
foreach (DataRow row in _ds.Tables[0].Rows)
{
if (row["des_ID"] != DBNull.Value)
des_ID = row["des_ID"].ToString();
if (!string.IsNullOrEmpty(des_ID))
{
if (countryId == 12 || countryId == 13)
geo = "NA";
else if ((countryId == 10 || countryId == 11))
geo = "LA";
else
geo = "EMEA";
//Instead of calling "GetDes" function which will hit the database
//Type-cast the session variable Session["GeoExpAllRec"] to datatable i.e. (Session["GeoExpAllRec"] as DataTable)
//Fire a LINQ query on the datatable to get the desired Description like below
//row["Descpition"] = GetDes(geo, des_ID);
DataTable dt = (Session["GeoExpAllRec"] as DataTable);
row["Descpition"] = dt.AsEnumerable().Where(r => r.Field<string>("des_ID") == des_ID).First()["description"];
}
else { row["ExemptionDes"] = string.Empty; }
}
}
Hope this helps.