Develop Reference

How to melt a DataTable in C# .NET (wide to long format)?

How to melt a DataTable in C# (wide to long format) as Python Pandas.melt does? https://pandas.pydata.org/docs/reference/api/pandas.melt.html
Is there any method already implemented? If not, how the code for melting a DataTable would
For example:
I have one DataTable which is in wide format, that is that has one row per id and has as many columns as variables. I would like to transform this DataTable to long format that has as many rows as combinations of id with each variable column. You can see this example in top image.
Please, if there is not clear enough visit Pandas documentation, there is more clear. (https://pandas.pydata.org/docs/reference/api/pandas.melt.html)
Note: I would like a solution that is DataTable independent, that is, that the solution is able to take parameters as id_vars, value_vars, etc... like Pandas.melt does
Any help is appreciated.

I don't know that Melt method but according to docs it seems to be an unpivot method:
public static DataTable MeltTable(DataTable inputTable, string outputColumn, params string[] unpivotColumns)
{
DataTable resultTable = new DataTable();
DataColumn col = new DataColumn(outputColumn, inputTable.Columns[outputColumn].DataType);
resultTable.Columns.Add(col);
resultTable.Columns.Add("Variable");
resultTable.Columns.Add("Value");
foreach(string unpivotColumn in unpivotColumns)
{
foreach (DataRow row in inputTable.Rows)
{
resultTable.Rows.Add(row[outputColumn], unpivotColumn, row[unpivotColumn]);
}
}
return resultTable;
}
You use it in this way:
DataTable table = new DataTable();
table.Columns.Add("Name");
table.Columns.Add("Course");
table.Columns.Add("Age", typeof(int));
table.Rows.Add("Tim", "Masters", 47);
table.Rows.Add("Bob", "Graduate", 19);
table.Rows.Add("Sheila", "Graduate", 20);
DataTable resultTable = MeltTable(table, "Name", "Course", "Age");
Result:
Name Variable Value
Tim Course Masters
Bob Course Graduate
Sheila Course Graduate
Tim Age 47
Bob Age 19
Sheila Age 20

#TimSchmelter gave me the answer but I modified a little bit to be a more general solution. Here's the code:
public static List<string> GetDifferenceColumns(DataTable dt, List<string> diffCols)
{
string[] columns = GetColumnsList(dt).ToArray();
IEnumerable<string> differenceColumns = from column in columns.Except(diffCols.ToArray()) select column;
return differenceColumns.ToList();
}
public static DataTable Melt(DataTable dt, List<string> idCols = null, List<string> varCols = null)
{
string errorPrefixString = "Error in DataProcessing Melt Method:\n";
bool varsColsIsNull = (varCols == null || varCols.Count == 0);
bool idColsIsNull = (idCols == null || idCols.Count == 0);
string varsName = "Variable";
string valueName = "Value";
if (dt.Rows.Count == 0)
{
throw new Exception(errorPrefixString + "DataTable is empty");
}
if (varsColsIsNull && varsColsIsNull)
{
throw new Exception(errorPrefixString+"You should past at least varCols or idCols");
}
if (varsColsIsNull)
{
varCols = GetDifferenceColumns(dt, idCols);
}
if (idColsIsNull)
{
idCols = GetDifferenceColumns(dt, varCols);
}
DataTable resultTable = new DataTable();
// Creating final columns of resultTable
foreach (string id in idCols)
{
resultTable.Columns.Add(id);
}
resultTable.Columns.Add(varsName);
resultTable.Columns.Add(valueName);
// Populating resultTable with the new rows
// generated by unpivoting varCols
foreach (string varCol in varCols)
{
foreach (DataRow row in dt.Rows)
{
DataRow resultRow = resultTable.NewRow();
foreach(string id in idCols)
{
resultRow[id] = row[id]; // create id cols
}
resultRow[varsName] = varCol;
resultRow[valueName] = row[varCol];
resultTable.Rows.Add(resultRow);
}
}
return resultTable;
}
How to use it:
DataTable dt = new DataTable();
dt.Columns.Add("Name");
dt.Columns.Add("Course");
dt.Columns.Add("Age");
dt.Rows.Add("Tim", "Masters", 47);
dt.Rows.Add("Bob", "Graduate", 19);
dt.Rows.Add("Sheila", "Graduate", 20);
List<string> varCols = new List<string> { "Course", "Age" };
DataTable finalDataTable = Melt(dt, varCols: varCols);

Create DataTable using LINQ to select multiple columns

I need to create a DataTable from another DataTable that I have but with only some columns (exactly same rows).
Is there an easy way to do this using LINQ?
I tried:
DataTable dataCopy = dt.AsEnumerable()
.Select(r => new { A = r.Field<string>("Column A"),
B = r.Field<string>("Column B") });
But it doesn't have a "CopyToDataTable" method.
I'm looking for the best performance way to do this because my "source" DataTable is huge!

You can simply copy the dataTable using dataTable.Copy() and remove the unwanted columns from the copied object.
var dt1 = dt.Copy();
dt.Columns.Remove("<columnsToRemove>")

Stopwatch time = new Stopwatch();
time.Start();
//COMPARE YOUR CODE (.Copy, Clone or Develop yourself)
DataTable dtTarget = dtSource.CopyDataTable(new List<string>() { "A", "B" });
foreach (DataColumn column in dtTarget.Columns)
{
Console.WriteLine("ColumnName : {0}", column.ColumnName);
foreach (DataRow row in dtTarget.Rows)
{
Console.WriteLine("Rows : {0}", row[column.ColumnName]);
}
}
time.Stop();
Console.WriteLine("{0}", time.Elapsed);
public static class DataTableHelper
{
public static DataTable CopyDataTable(
this DataTable dtSource,
List<string> columnsName)
{
DataTable dtTarget = new DataTable();
if (dtSource.Columns.Count > 0)
{
foreach (DataColumn columnSource in dtSource.Columns)
{
var columnTargetMapped = columnsName.FirstOrDefault(c => c == columnSource.ColumnName);
if(columnTargetMapped == null)
{
continue;
}
dtTarget.Columns.Add(columnTargetMapped);
foreach (DataRow drSource in dtSource.Rows)
{
var valueColumn = drSource[columnSource];
DataRow drTarget = dtTarget.NewRow();
drTarget[columnTargetMapped] = valueColumn;
dtTarget.Rows.Add(drTarget);
}
}
}
return dtTarget;
}

public static DataTable GetDataTablePart(this DataTable dt, params string[] ColumnNames)
{
var dtPart = new DataTable("TablePart");
var Names = new List<DataColumn>();
foreach (DataColumn Column in dt.Columns)
{
if(ColumnNames.Contains(Column.ColumnName))
{
Names.Add(Column);
}
}
dtPart.Columns.AddRange(Names.ToArray());
foreach(DataRow row in dt.Rows)
{
var NewRow = new object[Names.Count()];
var i = 0;
foreach (var Name in Names)
{
NewRow[i] = row[Name];
i = i + 1;
}
dtPart.LoadDataRow(NewRow, false);
}
return dtPart;
}
linq version....
public static DataTable GetDataTablePart(this DataTable dt, params string[] ColumnNames)
{
var RowCount = 0;
var dtPart = new DataTable("TablePart");
dtPart.Columns.AddRange((from column in dt.Columns.Cast<DataColumn>()
where ColumnNames.Contains(column.ColumnName)
select column).ToArray());
return (from row in dt.AsEnumerable()
let rowCount = RowCount = dt.Rows.Count
let RowValues = (from column in dtPart.Columns.Cast<DataColumn>()
select row[column]).ToArray()
let decCount = RowCount = RowCount - 1
where dtPart.LoadDataRow(RowValues,LoadOption.OverwriteChanges) != default && decCount > 0
select dtPart).FirstOrDefault();
}

I need helping adding two different rows(with same columns) to the same datatable

Working on a windows form application that reads in data from csv files and adds the data to a Datagridview. I ran into an issue with all of the rows being added to the datable and being displayed on the datagridview. The datagridview displays the datarows from the first two if conditions and OneRow if condition only. It will not add the rows from the twoRow if condition if the datable and datagridview rows are populated with the OneRow if condition rows. But i want the rows from both OneRow and TwoRow to be displyed. Also the rows from TwoRow do populate the datatable and datagridview when I comment(/**/) out the OneRow if condition. But I need both to populate the table. Thanks in advance!
Construct.MainDataTable.Columns.Add("Date", typeof(DateTime));
Construct.MainDataTable.Columns.Add("Time");
Construct.MainDataTable.Columns.Add("Serial");
Construct.MainDataTable.Columns.Add("Type");
Construct.MainDataTable.Columns.Add("level");
Construct.MainDataTable.Columns.Add("price");
Construct.MainDataTable.Columns.Add(" Limit");
Construct.MainDataTable.Columns.Add("last Limit");
Construct.MainDataTable.Columns.Add("Data");
..........................
...............................................
DataRow oneRow = Construct.MainDataTable.NewRow();
DataRow twoRow = Construct.MainDataTable.NewRow();
dataGridView2.AllowUserToAddRows = false;
if (line.Split(',')[2].Equals("Time"))
{
time = line.Split(',')[3];
date = line.Split(',')[1];
}
if (line.Split(',')[2].Equals("Level"))
{
level = line.Split(',')[3];
}
//OneROw(IF condition)
if ((Convert.ToDecimal(line.Split(',')[8])) < (Convert.ToDecimal (line.Split(',')[12])))
{
type = line.Split(',')[1];
serial = line.Split(',')[7];
price = line.Split(',')[3];
Limit = line.Split(',')[8];
lastLimit = line.Split(',')[10];
Data = line.Split(',')[12];
oneRow["Date"] = date;
oneRow["Time"] = time;
oneRow["Serial"] = serial;
oneRow["Type"] = type;
oneRow["level"] = level;
oneRow["price"] = price;
oneRow[" Limit"] = Limit;
oneRow["last Limit"] = lastlimit;
oneRow["Data"] = Data;
Construct.MainDataTable.Rows.Add(oneRow);
}
//TwoROw(IF condition)
if ((line.Contains('"')) && ((line.Contains("NG"))))
{
price = line.Split(',')[3];
type = line.Split(',')[1];
serial = line.Split(',')[7];
Limit = line.Split('"')[7];
var valLimit = Limit.Split(',').Select(a => Convert.ToInt32(a, 16));
var limitJoin = String.Join(",", valLimit);
lastlimit = line.Split('"')[1];
var vallastLimit = lastlimit.Split(',').Select(d => Convert.ToInt32(d, 16));
var lastJoin = String.Join(",", vallastLimit);
Data = line.Split('"')[5];
var valDatas = Data.Split(',').Select(s => Convert.ToInt32(s, 16));
var dataJoin = String.Join(",", valDatas);
twoRow["Date"] = date;
twoRow["Time"] = time;
twoRow["Serial"] = serial;
twoRow["Type"] = type;
twoRow["level"] = level;
twoRow["price"] = price;
twoRow["Limit"] = limitJoin;
twoRow["last Limit"] = lastJoin;
twoRow["Data"] = dataJoin;
Construct.MainDataTable.Rows.Add(twoRow);
}
dataGridView2.DataSource = Construct.MainDataTable;

Can't add a comment because I don't have enough karma so I ask my questions here: So, if I understood your problem you can't add data from one .csv file if it have more then one row? Why are you using 2 different if conditions for row in .csv file?
If you have empty data in row never mind you can still place them to your DataTable column, so you can use loop to add data from .csv to your DataTable. Try some thing like this:
public static DataTable CsvToDataTable(string csv)
{
DataTable dt = new DataTable();
string[] lines = csv.Split(new[] { "\r\n", "\r", "\n" }, StringSplitOptions.RemoveEmptyEntries);
Regex onlyDeimiterComma = new Regex(",(?=(?:[^\"]*\"[^\"]*\")*(?![^\"]*\"))");
for (int i = 0; i < lines.Length; i++)
{
DataRow row = dt.NewRow();
string[] cells = onlyDeimiterComma.Split(lines[i]);
for (int j = 0; j < cells.Length; j++)
{
if (i == 0)
{
if (j == 0)
{
dt.Columns.Add(cells[j], typeof(DateTime));
}
else
{
dt.Columns.Add(cells[j]);
}
}
else
{
row[j] = cells[j];
}
}
dt.Rows.Add(row);
}
return dt;
}
Just call this method anywhere in your code and give it string read from your .csv file.
You can try to compile this code here and see how it works on .csv data with different data (empty columns, quoted text, quoted commas)
UPD: If you need to fill DataTable from two different .csv files you can still use code above. Just call it twice for both files and then Merge two DataTable, like this:
DataTable dt = CsvToDataTable(csvFileOne);
DataTable dtTwo = CsvToDataTable(csvFileTwo);
dt.Merge(dtTwo);

How i can update my second combobox when "SelectedItem" is changed

I've two comboboxes in a form, the first combobox display each "mark" of truck and second may display the models of each "mark". For example if mark is volvo the second combobox may display "FH, FH6 ...", otherwise if mark is "MAN" the second combobox may display "TGX, TGX 38" ...
I've try to do a cycle to refesh my second combobox, each time, that the first combobox is changed.
while (comboBox1.SelectedIndex == -1)
{
comboBox2.Items.Clear();
if (comboBox1.SelectedIndex == 1)
{
comboBox2.DisplayMember = "Modelo";
comboBox2.DataSource = d.Tables["Modelos-MAN"].DefaultView;
}
if (comboBox1.SelectedIndex == 2)
{
comboBox2.DisplayMember = "ModeloV";
comboBox2.DataSource = d.Tables["Modelos-VOLVO"].DefaultView;
comboBox2.Refresh();
}
}
if (comboBox1.SelectedIndex==1)
{
comboBox2.DisplayMember = "Modelo";
comboBox2.DataSource = d.Tables["Modelos-MAN"].DefaultView;
}
if(comboBox1.SelectedIndex == 2)
{
comboBox2.DisplayMember = "ModeloV";
comboBox2.DataSource = d.Tables["Modelos-VOLVO"].DefaultView;
comboBox2.Refresh();
}
Happens that if i select "MAN", it shows "MAN" models and then if i change, and select "Volvo", the second combobox show me "System.Data.DataRowView".
What is happening?How i can solve this?
Note: I've also, a method to generate truck "mark" and "models":
public DataSet LoadMarkAndModel()
{
var marcas = new string[6];
marcas[0] = "Marca ...";
marcas[1] = "MAN";
marcas[2] = "Volvo";//Linha 2 coluna 2
marcas[3] = "DAF";
marcas[4] = "Renault";//Linha 4 coluna 2
marcas[5] = "Mercedes-Benz";
var modelosMAN = new string[7];
var modelosVolvo = new string[5];
modelosMAN[0] = "TGX";
modelosMAN[1] = "TGX D38";
modelosMAN[2] = "TGX EfficientLine 2";
modelosMAN[3] = "TGS";
modelosMAN[4] = "TGA";
modelosMAN[5] = "TGA-WW";
modelosMAN[6] = "TGM";
modelosVolvo[0] = "FH16";
modelosVolvo[1] = "FH";
modelosVolvo[2] = "FM";
modelosVolvo[3] = "FE";
modelosVolvo[4] = "FL";
DataSet data = new DataSet("DataSetOfTrucks");
DataTable table = new DataTable("Marcas");
DataColumn colunaID = new DataColumn("ID", typeof(int));
DataColumn colunaMarca = new DataColumn("Marca", typeof(string));
//table.Columns.Add("Description", typeof(string));
table.Columns.Add(colunaID);
table.Columns.Add(colunaMarca);
for (int i = 0; i < marcas.Length; i++)
{
DataRow manyMark = table.NewRow();
manyMark["ID"] = i + 1;
manyMark["Marca"] = marcas[i];
table.Rows.Add(manyMark);
}
data.Tables.Add(table);
DataTable modelsTable = new DataTable("Modelos-MAN");
DataColumn modelsIDParent = new DataColumn("IDP", typeof(int));
DataColumn modelo = new DataColumn("Modelo", typeof(string));
modelsTable.Columns.Add(modelsIDParent);
modelsTable.Columns.Add(modelo);
int m = 0;
while (m<modelosMAN.Length)
{
DataRow manyModelsMAN = modelsTable.NewRow();
manyModelsMAN["IDP"] = 1;
manyModelsMAN["Modelo"] = modelosMAN[m];
modelsTable.Rows.Add(manyModelsMAN);
m++;
//if (m > modelosMAN.Length)
// break;
}
data.Tables.Add(modelsTable);
////vOLVO
DataTable modelsTableVOLVO = new DataTable("Modelos-VOLVO");
DataColumn modelsIDParentVOLVO = new DataColumn("IDPVolvo", typeof(int));
DataColumn modeloVolvo = new DataColumn("ModeloV", typeof(string));
modelsTableVOLVO.Columns.Add(modelsIDParentVOLVO);
modelsTableVOLVO.Columns.Add(modeloVolvo);
int count = 0;
while (count < modelosVolvo.Length)
{
DataRow manyModelsVolvo = modelsTableVOLVO.NewRow();
manyModelsVolvo["IDPVolvo"] = 2;
manyModelsVolvo["ModeloV"] = modelosVolvo[count];
modelsTableVOLVO.Rows.Add(manyModelsVolvo);
count++;
}
data.Tables.Add(modelsTableVOLVO);
return data;
}

comboBox1_SelectedIndexChanged()
{
comboBox2.DataSource =null;
if (comboBox1.SelectedIndex==1)
{
comboBox2.DataSource = d.Tables["Modelos-MAN"];
comboBox2.ValueMember="Modelo";
comboBox2.DisplayMember = "Modelo";
}
if(comboBox1.SelectedIndex == 2)
{
comboBox2.DataSource = d.Tables["Modelos-VOLVO"];
comboBox2.DisplayMember = "ModeloV";
comboBox2.ValueMember = "ModeloV";
}
}

Error: Datatable named "Items" already belongs to this dataset

Why am I receiving the error Datatable named "Items" already belongs to this dataset in the code below? The error is only occurring when I am attempting to add values where if ((x > 1) || (x == 1)). I can successfully create multiple new "Item" datatables and then add them to the ds dataset in the while loop, however trying to more than one "AItem" datatable to AllItems dataset causes the error. The only difference between these datasets is that the ds dataset has many other datatables added to it which have data adapters filling them from a SQL db.
DataSet AllItems = new DataSet("Items");
DataSet ds = new DataSet("Header");
foreach (DataRow fieldRow in myDataset.Tables["tempTable"].AsEnumerable())
{
while (x < 1)
{
x++;
DataTable Item = new DataTable("Item");
Item.Columns.Add("ID");
Item.Columns.Add("LineNumber");
Item.Columns.Add("ItemID");
Item.Columns.Add("UnitPrice");
Item.Columns.Add("Description");
Item.Columns.Add("OrderUOM");
Item.Columns.Add("OrderQty");
DataRow aItem = Item.NewRow();
Item["ID"] = 1.ToString();
Item["LineNumber"] = x;
Item["ItemID"] = fieldRow[0].ToString();
Item["UnitPrice"] = fieldRow[1].ToString();
Item["Description"] = fieldRow[2].ToString();
Item["OrderUOM"] = fieldRow[3].ToString();
Item["OrderQty"] = fieldRow[4].ToString();
Item.Rows.Add(aItem);
ds.Tables.Add(Item);
}
if ((x > 1) || (x == 1))
{
DataTable AItem = new DataTable("AItem");
Item.Columns.Add("ID");
Item.Columns.Add("LineNumber");
Item.Columns.Add("ItemID");
Item.Columns.Add("UnitPrice");
Item.Columns.Add("Description");
Item.Columns.Add("OrderUOM");
Item.Columns.Add("OrderQty");
DataRow aItem = Item.NewRow();
Item["ID"] = 1.ToString();
Item["LineNumber"] = x;
Item["ItemID"] = fieldRow[0].ToString();
Item["UnitPrice"] = fieldRow[1].ToString();
Item["Description"] = fieldRow[2].ToString();
Item["OrderUOM"] = fieldRow[3].ToString();
Item["OrderQty"] = fieldRow[4].ToString();
Item.Rows.Add(aItem);
AllItems.Tables.Add(AItem);
}
}

I'm not sure if this is your only problem but you are using the DataTable as if it was a DataRow. Item is the table, it has no string-indexer, so replace
DataRow aItem = Item.NewRow();
Item["ID"] = 1.ToString();
Item["LineNumber"] = x;
Item["ItemID"] = fieldRow[0].ToString();
Item["UnitPrice"] = fieldRow[1].ToString();
Item["Description"] = fieldRow[2].ToString();
Item["OrderUOM"] = fieldRow[3].ToString();
Item["OrderQty"] = fieldRow[4].ToString();
Item.Rows.Add(aItem);
with
DataRow aItem = Item.NewRow();
aItem["ID"] = 1.ToString();
aItem["LineNumber"] = x;
aItem["ItemID"] = fieldRow[0].ToString();
aItem["UnitPrice"] = fieldRow[1].ToString();
aItem["Description"] = fieldRow[2].ToString();
aItem["OrderUOM"] = fieldRow[3].ToString();
aItem["OrderQty"] = fieldRow[4].ToString();
Item.Rows.Add(aItem);
But i assume that this is just the next error, "Datatable named “Items” already belongs to this dataset" is thrown because you are adding a table to a DataSet with the same name of another table. You are adding tables with the static name "AItem" in the loop here:
foreach (DataRow fieldRow in myDataset.Tables["tempTable"].AsEnumerable())
{
// ...
DataTable AItem = new DataTable("AItem");
// ...
AllItems.Tables.Add(AItem);
// ...
}
I don't understand the logic, maybe it's sufficient to use the default constructor without a name.