Develop Reference

How can I improve the performance of the following code?

This code is working but taking too much time. Every data table contains 1000nds of rows and each time I need to filter data from another data tables with respect to a column.
for (int i = 0; i < dsResult.Tables[0].Rows.Count; i++)
{
DataTable dtFiltered = dtWorkExp.Clone();
foreach (DataRow drr in dtWorkExp.Rows)
{
if (drr["UserId"].ToString() == dsResult.Tables[0].Rows[i]["Registration NO."].ToString())
{
dtFiltered.ImportRow(drr);
}
}
DataTable dtFilteredAward= dtAwards.Clone();
foreach (DataRow drr in dtAwards.Rows)
{
if (drr["UserId"].ToString() == dsResult.Tables[0].Rows[i]["Registration NO."].ToString())
{
dtFilteredAward.ImportRow(drr);
}
}
DataTable dtFilteredOtherQual = dtOtherQual.Clone();
foreach (DataRow drr in dtOtherQual.Rows)
{
if (drr["UserId"].ToString() == dsResult.Tables[0].Rows[i]["Registration NO."].ToString())
{
dtFilteredOtherQual.ImportRow(drr);
}
}
//Do some operation with filtered Data Tables
}

You can declare these lines outside the for loop.
DataTable dtFiltered = dtWorkExp.Clone();
And instead of doing accessing dsResult.Table[0] each time, you can assign this to one variable and use it.
You can also replace the foreach loop with LINQ.

What I would do:
All rows of the main datatable as enumerable:
var rows = dsResult.Tables[0].AsEnumerable();
Get the column you're going to filter with:
var filter = rows.Select(r => r.Field<string>("Registration NO."));
Create a method that accepts that filter, a table to filter and a field to compare.
public static DataTable Filter<T>(EnumerableRowCollection<T> filter, DataTable table, string fieldName)
{
return table.AsEnumerable().Where(r => filter.Contains(r.Field<T>(fieldName))).CopyToDataTable();
}
Finally use the method to filter all tables:
var dtFiltered = Filter<string>(filter, dtWorkExp, "UserId");
var dtFilteredAward = Filter<string>(filter, dtAwards, "UserId");
var dtFilteredOtherQual = Filter<string>(filter, dtOtherQual, "UserId");
All together woul be something like this
public void YourMethod()
{
var rows = dsResult.Tables[0].AsEnumerable();
var filter = rows.Select(r => r.Field<string>("Registration NO."));
var dtFiltered = Filter<string>(filter, dtWorkExp, "UserId");
var dtFilteredAward = Filter<string>(filter, dtAwards, "UserId");
var dtFilteredOtherQual = Filter<string>(filter, dtOtherQual, "UserId");
}
public static DataTable Filter<T>(EnumerableRowCollection<T> filter, DataTable table, string fieldName)
{
return table.AsEnumerable().Where(r => filter.Contains(r.Field<T>(fieldName))).CopyToDataTable();
}

Put the value of the expression in a variable.
var regNo = dsResult.Tables[0].Rows[i]["Registration NO."].ToString();
Put the index of column to the variable. Access by index more faster then by column name.
int index = dtWorkExp.Columns["UserId"].Ordinal;
Result code:
int dtWorkIndex = dtWorkExp.Columns["UserId"].Ordinal;
int dtAwardsIndex = dtAwards.Columns["UserId"].Ordinal;
int dtOtherQualIdex = dtOtherQual.Columns["UserId"].Ordinal;
for (int i = 0; i < dsResult.Tables[0].Rows.Count; i++)
{
var regNo = dsResult.Tables[0].Rows[i]["Registration NO."].ToString();
DataTable dtFiltered = dtWorkExp.Clone();
foreach (DataRow drr in dtWorkExp.Rows)
{
if (drr[dtWorkIndex].ToString() == regNo)
{
dtFiltered.ImportRow(drr);
}
}
...
Of course, the column index can be set as a constant if you know it exactly in advance. Also, if the UserId indexes match in all tables, a single variable is sufficient.
You can also try using the BeginLoadData and EndLoadData methods.
DataTable dtFiltered = dtWorkExp.Clone();
dtFiltered.BeginLoadData();
foreach (DataRow drr in dtWorkExp.Rows)
{
if (drr[dtWorkIndex].ToString() == regNo)
{
dtFiltered.ImportRow(drr);
}
}
dtFiltered.EndLoadData();
But I'm not sure if they make sense together with ImportRow.
Finally, parallelization comes to help.
for (int i = 0; i < dsResult.Tables[0].Rows.Count; i++)
{
var regNo = ...;
var workTask = Task.Run(() =>
{
DataTable dtFiltered = dtWorkExp.Clone();
foreach (DataRow drr in dtWorkExp.Rows)
{
if (drr[dtWorkIndex].ToString() == regNo)
{
dtFiltered.ImportRow(drr);
}
}
return dtFiltered;
});
var awardTask = Task.Run(() =>
...
var otherQualTask = Task.Run(() =>
...
//Task.WaitAll(workTask, awardTask, otherQualTask);
await Task.WhenAll(workTask, awardTask, otherQualTask);
//Do some operation with filtered Data Tables
}

Create DataTable using LINQ to select multiple columns

I need to create a DataTable from another DataTable that I have but with only some columns (exactly same rows).
Is there an easy way to do this using LINQ?
I tried:
DataTable dataCopy = dt.AsEnumerable()
.Select(r => new { A = r.Field<string>("Column A"),
B = r.Field<string>("Column B") });
But it doesn't have a "CopyToDataTable" method.
I'm looking for the best performance way to do this because my "source" DataTable is huge!

You can simply copy the dataTable using dataTable.Copy() and remove the unwanted columns from the copied object.
var dt1 = dt.Copy();
dt.Columns.Remove("<columnsToRemove>")

Stopwatch time = new Stopwatch();
time.Start();
//COMPARE YOUR CODE (.Copy, Clone or Develop yourself)
DataTable dtTarget = dtSource.CopyDataTable(new List<string>() { "A", "B" });
foreach (DataColumn column in dtTarget.Columns)
{
Console.WriteLine("ColumnName : {0}", column.ColumnName);
foreach (DataRow row in dtTarget.Rows)
{
Console.WriteLine("Rows : {0}", row[column.ColumnName]);
}
}
time.Stop();
Console.WriteLine("{0}", time.Elapsed);
public static class DataTableHelper
{
public static DataTable CopyDataTable(
this DataTable dtSource,
List<string> columnsName)
{
DataTable dtTarget = new DataTable();
if (dtSource.Columns.Count > 0)
{
foreach (DataColumn columnSource in dtSource.Columns)
{
var columnTargetMapped = columnsName.FirstOrDefault(c => c == columnSource.ColumnName);
if(columnTargetMapped == null)
{
continue;
}
dtTarget.Columns.Add(columnTargetMapped);
foreach (DataRow drSource in dtSource.Rows)
{
var valueColumn = drSource[columnSource];
DataRow drTarget = dtTarget.NewRow();
drTarget[columnTargetMapped] = valueColumn;
dtTarget.Rows.Add(drTarget);
}
}
}
return dtTarget;
}

public static DataTable GetDataTablePart(this DataTable dt, params string[] ColumnNames)
{
var dtPart = new DataTable("TablePart");
var Names = new List<DataColumn>();
foreach (DataColumn Column in dt.Columns)
{
if(ColumnNames.Contains(Column.ColumnName))
{
Names.Add(Column);
}
}
dtPart.Columns.AddRange(Names.ToArray());
foreach(DataRow row in dt.Rows)
{
var NewRow = new object[Names.Count()];
var i = 0;
foreach (var Name in Names)
{
NewRow[i] = row[Name];
i = i + 1;
}
dtPart.LoadDataRow(NewRow, false);
}
return dtPart;
}
linq version....
public static DataTable GetDataTablePart(this DataTable dt, params string[] ColumnNames)
{
var RowCount = 0;
var dtPart = new DataTable("TablePart");
dtPart.Columns.AddRange((from column in dt.Columns.Cast<DataColumn>()
where ColumnNames.Contains(column.ColumnName)
select column).ToArray());
return (from row in dt.AsEnumerable()
let rowCount = RowCount = dt.Rows.Count
let RowValues = (from column in dtPart.Columns.Cast<DataColumn>()
select row[column]).ToArray()
let decCount = RowCount = RowCount - 1
where dtPart.LoadDataRow(RowValues,LoadOption.OverwriteChanges) != default && decCount > 0
select dtPart).FirstOrDefault();
}

Fill Data To a particular Cell In DataTable -C#

I have data like
ConcurrentDictionary<string, ConcurrentDictionary<string,string> OneTwoThree =
new ConcurrentDictionary<string, ConcurrentDictionary<string, string>();
I want a result like this Final Image
What I tried:
DataTable dt = new DataTable();
dt.Columns.Add("TwoDetails");
ISet<string> two = new HashSet<string>();
Parallel.ForEach(OneTwoThree , One=>
{
dt.Columns.Add(One.Key);
foreach(var Two in One.Value)
{
two.Add(Two.Key); // To get Distinct Values
}
});
foreach(var item in two)
{
var row = dt.NewRow();
row["TwoDetails"] = row;
}
Now I don't have the idea to append "Three-Values" to a particular cell, as shown in the image.
Any Suggestions.

Another pivot table question. Done a 1000. See code below :
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Data;
namespace ConsoleApplication31
{
class Program
{
static void Main(string[] args)
{
Dictionary<string, Dictionary<string, string>> OneTwoThree = new Dictionary<string, Dictionary<string, string>>() {
{"A", new Dictionary<string,string>(){{"U","s"}, {"Z","a"}}},
{"B", new Dictionary<string,string>(){{"W","e"},{"X","d"},{"Y","d"}}},
{"C", new Dictionary<string,string>(){{"V","f"}, {"W","a"},{"Z","w"}}},
};
string[] columns = OneTwoThree.Select(x => x.Key).OrderBy(x => x).ToArray();
DataTable dt = new DataTable();
dt.Columns.Add("TwoDetails", typeof(string));
foreach(string column in columns)
{
dt.Columns.Add(column, typeof(string));
}
string[] rows = OneTwoThree.Select(x => x.Value.Select(y => y.Key)).SelectMany(x => x).Distinct().OrderBy(x => x).ToArray();
var flip = rows.Select(x => new { row = x, columns = OneTwoThree.Where(y => y.Value.ContainsKey(x)).Select(y => new { col = y.Key, value = y.Value[x] }).ToList() }).ToList();
//create pivot table
foreach (var row in flip)
{
DataRow newRow = dt.Rows.Add();
newRow["TwoDetails"] = row.row;
foreach (var column in row.columns)
{
newRow[column.col] = column.value;
}
}
}
}
}

You can not access the data table rows with their names but using row numbers you can find a particular row in the data table.So for that, You need to create one class like
public class DataTableDetails
{
public string RowName { get; set; }
public int RowNumber { get; set; }
}
And this is not effecient solution, but can help you
ISet<string> two = new HashSet<string>();
List<DataTableDetails> dtdetaills=new List<DataTableDetails>();
Parallel.ForEach(OneTwoThree , One=>
{
dt.Columns.Add(One.Key);
foreach(var Two in One.Value)
{
two.Add(Two.Key); // To get Distinct Values
}
});
int count=0;
foreach(var item in two)
{
var row = dt.NewRow();
row["TwoDetails"] = row;
DataTableDetails details = new DataTableDetails();
details.RowName = item;
details.RowNumber = count++; // we can easily get row number
dtdetails.Add(details);
}
And Finally
foreach(var One in OnrTwoThree)
{
foreach(var Two in One.Value)
{
foreach(var rowdetails in dtdetails)
{
if(Two.Key==rowdetails.RowName)
{
dt.Rows[rowdetails.RowNumber][One.Key] = Two.Value;
}
}
}
}

Coverting List of Dictionary to DataTable

Currently we are doing this by looping through each value of list and dictionary:
private DataTable ChangeToDictionary(List<Dictionary<string,int>> list)
{
DataTable datatTableReturn = new DataTable();
if (list.Count() > 0)
{
Dictionary<string, int> haeders = list.ElementAt(0);
foreach (var colHead in haeders)
{
datatTableReturn.Columns.Add(colHead.Key);
}
}
foreach (var row in list)
{
DataRow dataRow = datatTableReturn.NewRow();
foreach (var col in row)
{
dataRow[col.Key] = col.Value;
}
datatTableReturn.Rows.Add(dataRow);
}
return datatTableReturn;
}
But is there a better way? Looping through so many times doesn't feel good

The answers above don't address the issue of the dictionary having more than 1 row. This solution addresses the issue.
static DataTable ToDataTable(List<Dictionary<string, int>> list)
{
DataTable result = new DataTable();
if (list.Count == 0)
return result;
var columnNames = list.SelectMany(dict=>dict.Keys).Distinct();
result.Columns.AddRange(columnNames.Select(c=>new DataColumn(c)).ToArray());
foreach (Dictionary<string,int> item in list)
{
var row = result.NewRow();
foreach (var key in item.Keys)
{
row[key] = item[key];
}
result.Rows.Add(row);
}
return result;
}
static void Main(string[] args)
{
List<Dictionary<string, int>> it = new List<Dictionary<string, int>>();
Dictionary<string, int> dict = new Dictionary<string, int>();
dict.Add("a", 1);
dict.Add("b", 2);
dict.Add("c", 3);
it.Add(dict);
dict = new Dictionary<string, int>();
dict.Add("bob", 34);
dict.Add("tom", 37);
it.Add(dict);
dict = new Dictionary<string, int>();
dict.Add("Yip Yip", 8);
dict.Add("Yap Yap", 9);
it.Add(dict);
DataTable table = ToDictionary(it);
foreach (DataColumn col in table.Columns)
Console.Write("{0}\t", col.ColumnName);
Console.WriteLine();
foreach (DataRow row in table.Rows)
{
foreach (DataColumn column in table.Columns)
Console.Write("{0}\t", row[column].ToString());
Console.WriteLine();
}
Console.ReadLine();
}
And the output looks like...
a b c bob tom Yip Yip Yap Yap
1 2 3
34 37
8 9

Speed, elegance and reusability don't go together. You always choose more important one, and try to balance other two.
Faster the code, uglier it is.
Prettier it is, less reusable it is.
Here's an example of "elegant" solution, but that goes with it not being very readable.
private static DataTable ToDictionary(List<Dictionary<string, int>> list)
{
DataTable result = new DataTable();
if (list.Count == 0)
return result;
result.Columns.AddRange(
list.First().Select(r => new DataColumn(r.Key)).ToArray()
);
list.ForEach(r => result.Rows.Add(r.Select(c => c.Value).Cast<object>().ToArray()));
return result;
}

Try this
List<Dictionary<string, object>> ListDic;
var stringListDic = JsonConvert.SerializeObject(ListDic);
var dataTable = JsonConvert.DeserializeObject<DataTable>(stringListDic);

Try this:
private DataTable GetDataTableFromDictionaries<T>(List<Dictionary<string, T>> list)
{
DataTable dataTable = new DataTable();
if (list == null || !list.Any()) return dataTable;
foreach (var column in list.First().Select(c => new DataColumn(c.Key, typeof(T))))
{
dataTable.Columns.Add(column);
}
foreach (var row in list.Select(
r =>
{
var dataRow = dataTable.NewRow();
r.ToList().ForEach(c => dataRow.SetField(c.Key, c.Value));
return dataRow;
}))
{
dataTable.Rows.Add(row);
}
return dataTable;
}

How about something like the code below?
Good, because it iterates each row exactly once. It should be pretty quick, I've included obvious exceptions to make the code safer.
private static DataTable DictionariesToDataTable<T>(
IEnumerable<IDictionary<string, T>> source)
{
if (source == null)
{
return null;
}
var result = new DataTable();
using (var e = source.GetEnumerator())
{
if (!e.MoveNext())
{
return result;
}
if (e.Current.Keys.Length == 0)
{
throw new InvalidOperationException();
}
var length = e.Current.Keys.Length;
result.Columns.AddRange(
e.Current.Keys.Select(k => new DataColumn(k, typeof(T))).ToArray());
do
{
if (e.Current.Values.Length != length)
{
throw new InvalidOperationException();
}
result.Rows.Add(e.Current.Values);
}
while (e.MoveNext());
return result;
}
}

try my solution, seems very clean to me:
private DataTable DictonarysToDataTable(List<Dictionary<string, int>> list)
{
DataTable table = new DataTable();
foreach (Dictionary<string,string> dict in list) //for every dictonary in the list ..
{
foreach (KeyValuePair<string,int> entry in dict) //for every entry in every dict
{
if (!myTable.Columns.Contains(entry.Key.ToString()))//if it doesn't exist yet
{
myTable.Columns.Add(entry.Key); //add all it's keys as columns to the table
}
}
table.Rows.Add(dict.Values.ToArray()); //add the the Values of every dict in the list as a new row
}
return table;
}
Edit:
Oh Snap, this works only for one Dictionary .. i didn't think it through.
But maybie you can modify it to work for a List of Dictionarys ..

Give this a try please
DataTable table = new DataTable();
foreach (IDictionary<string, object> row in DeviceTypeReport)
{
foreach (KeyValuePair<string, object> entry in row)
{
if (!table.Columns.Contains(entry.Key.ToString()))
{
table.Columns.Add(entry.Key);
}
}
table.Rows.Add(row.Values.ToArray());
}

private DataTable toDataTable(List<RetirementDiskModelDto> retirementDiskModelDtos)
{
DataTable result = new DataTable();
foreach (var col in retirementDiskModelDtos.FirstOrDefault().Items)
result.Columns.Add(col.Key);
foreach (var row in retirementDiskModelDtos)
{
DataRow newrow = result.NewRow();
foreach (var col in retirementDiskModelDtos.FirstOrDefault().Items)
newrow[col.Key] = col.Value;
result.Rows.Add(newrow);
}
return result;
}

Distinct Enumerable DataTable based on column?

This is what i'm trying to accomplish. I want to select only distinct values from all Rows in Column[0].
Then I want to get all the distinct values from column[2] and group them on column[0].
so basically i got a DataTable like so:
Fruit|Apples
Fruit|Pears
Vegetables|Peas
Vegetables|Carrots
so I want to do a foreach on the distinct values, so I would Enumerate Fruit once and then pick up Apples and Pears, and VegeTables once and pick up Peas and Carrots.
I'm doing this to create Accordion Panes, where I want to group my results under one header, the below code does such, however, it creates two panes of Fruit because it does not realize it already went though fruit.
foreach (DataRow dtrow in dtTable.Rows)
{
string idRow = dtrow[0].ToString();
AccordionPane currentPane = new AccordionPane();
currentPane.ID = "AccordionPane" + Guid.NewGuid().ToString();
currentPane.HeaderContainer.Controls.Add(new LiteralControl(dtrow[0].ToString()));
foreach(DataRow dtRow2 in dtTable.Rows)
{
if(dtRow2[0].ToString() == idRow)
{
currentPane.ContentContainer.Controls.Add(new LiteralControl(dtRow2[1].ToString()));
}
}
NavigateAccordion.Panes.Add(currentPane);
}

You can accomplish this easily when using linq, see for yourself:
var groupedRows = from row in dtTable.Rows.AsEnumerable()
group row by row[0] into grouped
select grouped;
foreach (var group in groupedRows)
{
currentPane = new AccordionPane();
currentPane.HeaderContainer.Controls.Add(group.Key.ToString());
foreach (var row in group)
{
currentPane.ContentContainer.Controls.Add(row[1].ToString());
}
}
Or if you want to stick with your current non-linq approach:
foreach (DataRow dtrow in dtTable.Rows)
{
bool skip = false;
foreach (var pane in NavigateAccordion.Panes)
{
if (pane.HeaderContainer.Controls[0].Text == dtRow[0].ToString())
{
skip = true;
break;
}
}
if (!skip)
{
string idRow = dtrow[0].ToString();
AccordionPane currentPane = new AccordionPane();
currentPane.ID = "AccordionPane" + Guid.NewGuid().ToString();
currentPane.HeaderContainer.Controls.Add(new LiteralControl(dtrow[0].ToString()));
foreach(DataRow dtRow2 in dtTable.Rows)
{
if(dtRow2[0].ToString() == idRow)
{
currentPane.ContentContainer.Controls.Add(new LiteralControl(dtRow2[1].ToString()));
}
}
NavigateAccordion.Panes.Add(currentPane);
}
}