Develop Reference

how to access records in csv helper

The code below reads a column of a csv file.
It does that properly.
I want to copy var records to a decimal array.
I am using csv helper.
How best to do it?
using (var reader = new StreamReader(filename))
using (var csv = new CsvReader(reader, CultureInfo.InvariantCulture))
{
var records = new List<Filevalues>();
csv.Read();
csv.ReadHeader();
while (csv.Read())
{
var record = new Filevalues
{
File_vals = csv.GetField<decimal>("File_vals"),
};
records.Add(record);
}
}
public class Filevalues
{
public decimal File_vals{ get; set; }
}

At its most simple:
using var csv = new CsvReader(new StreamReader(filename), CultureInfo.InvariantCulture));
csv.GetRecords<Filevalues>().Select(f => f.File_vals).ToArray();
(I think I'd ToList() it instead of array, and work with the List)
If the CSV is so simple that it's just a list of decimals, I might skip using a CSV library all together:
File.ReadLines(path).Skip(1).Select(decimal.Parse).ToArray();

Either:
using (var reader = new StreamReader(filename))
using (var csv = new CsvReader(reader, CultureInfo.InvariantCulture))
{
var listOfDecimals = new List<decimal>();
csv.Read();
csv.ReadHeader();
while (csv.Read())
{
listOfDecimals.Add(csv.GetField<decimal>("File_vals"));
}
var arrayOfDecimals = listOfDecimals.ToArray();
}
ToArray: https://learn.microsoft.com/en-us/dotnet/api/system.linq.enumerable.toarray
Or:
using (var reader = new StreamReader(filename))
using (var csv = new CsvReader(reader, CultureInfo.InvariantCulture))
{
var records = new List<Filevalues>();
csv.Read();
csv.ReadHeader();
while (csv.Read())
{
var record = new Filevalues
{
File_vals = csv.GetField<decimal>("File_vals"),
};
records.Add(record);
}
var arrayOfDecimals = records.Select(x => x.File_vals).ToArray();
}
public class Filevalues
{
public decimal File_vals{ get; set; }
}
LINQ Projection: https://learn.microsoft.com/en-us/dotnet/csharp/programming-guide/concepts/linq/projection-operations

CSV Helper, Write List of Expando Objects with Properties Keys Sorted

I am writing a csv file using CSV helper
var entries = new List<ExpandoObject>();
ExpandoObject dynamic = GetVersionInfo(list.Id.ToString(), $"{file.MajorVersion}.0");
entries.Add(dynamic);
using (var writer = new StreamWriter("file.csv"))
using (var csv = new CsvWriter(writer, CultureInfo.InvariantCulture))
{
var writeList = new List<dynamic>();
writeList.AddRange(entries);
csv.WriteRecords(writeList);
}
this code works fine and I can see a csv file has been created but columns data is not in correct order. for example
Version Date is being written in Document Owner column. Reason for this mismatch is that I am using expando objects and order of the properties in the expando object is not in a proper sequence.
In first object Version Date is at number 3 in expando object while it is at number 4 in 2nd object.
I tried to sort the properties with this method but I get error in generating the csv, for example
writeList.AddRange(entries.Select(x => x.OrderByDescending(y => y.Key).ToList()));
csv.WriteRecords(writeList); //exception here
Exception I get is
Types that inherit IEnumerable cannot be auto mapped. Did you accidentally call GetRecord or WriteRecord which acts on a single record instead of calling GetRecords or WriteRecords which acts on a list of records?
how can I sort and also write a collection of Expando objects in a CSV file?

The problem with entries.Select(x => x.OrderByDescending(y => y.Key).ToList() is you no longer have a List<ExpandoObject>, but IEnumerable<List<KeyValuePair<String,Object>>> There might be a more concise way to do this, but you need to get it back to List<ExpandoObject>.
void Main()
{
var entries = new List<ExpandoObject>();
dynamic obj1 = new ExpandoObject();
obj1.Version = 11;
obj1.Modified = new DateTime(2017,9,18);
obj1.VersionDate = new DateTime(2016,6,29);
obj1.DocumentOwner = "Unknown";
dynamic obj2 = new ExpandoObject();
obj2.Version = 1;
obj2.Modified = new DateTime(2021, 3, 17);
obj2.DocumentOwner = "Emergency Management Plan";
obj2.VersionDate = new DateTime(2019, 1, 4);
entries.Add(obj1);
entries.Add(obj2);
entries = SortEntries(entries);
//using (var writer = new StreamWriter("path\\to\\file.csv"))
using (var csv = new CsvWriter(Console.Out, CultureInfo.InvariantCulture))
{
var writeList = new List<dynamic>();
writeList.AddRange(entries);
csv.WriteRecords(writeList);
}
}
public List<ExpandoObject> SortEntries(IList<ExpandoObject> entries)
{
var newList = new List<ExpandoObject>();
foreach (var entry in entries)
{
var x = new ExpandoObject() as IDictionary<string, Object>;
var sorted = entry.OrderByDescending(y => y.Key);
foreach (var item in sorted)
{
x.Add(item.Key, item.Value);
}
newList.Add((ExpandoObject)x);
}
return newList;
}

How do I serialize multiple items with foreach using JSON in c#?

I am trying to serialize objects from 2 ListViews, ListView#2 is used to display objects that are grouped while ListView#2 displays the said groups. (Selecting a group in #1 displays different set of objects in #2)
JsonSerializer serializer = new JsonSerializer();
using (StreamWriter sw = new StreamWriter(path + "\\data.txt"))
{
foreach (ListViewItem group in lV_groups.Items)
{
foreach (ListViewItem item in lV_items.Items)
{
List<ItemSerObj> itemsObj = new List<ItemSerObj>()
{
new ItemSerObj
{
ItemName = item.SubItems[0].Text,
Value = item.SubItems[1].Text,
Quality = item.SubItems[2].Text,
TimeStamp = item.SubItems[3].Text
}
};
GroupSerObj serializeGroup = new GroupSerObj
{
GroupName = group.SubItems[0].Text,
UpdateRate = group.SubItems[1].Text,
Active = group.SubItems[2].Text,
Items = itemsObj
};
using (JsonWriter writer = new JsonTextWriter(sw))
{
serializer.Serialize(writer, serializeGroup); //Where exception occurs.
}
}
}
}
I am getting "System.ObjectDisposedException: 'Cannot write to a closed TextWriter'" exception.

Simply change it to something like this:
JsonSerializer serializer = new JsonSerializer();
using (StreamWriter sw = new StreamWriter(path + "\\data.txt"))
{
using (JsonWriter writer = new JsonTextWriter(sw))
{
foreach (ListViewItem group in lV_groups.Items)
{
List<ItemSerObj> itemsObj = new List<ItemSerObj>();
foreach (ListViewItem item in lV_items.Items)
{
itemsObj.Add(
new ItemSerObj
{
ItemName = item.SubItems[0].Text,
Value = item.SubItems[1].Text,
Quality = item.SubItems[2].Text,
TimeStamp = item.SubItems[3].Text
});
}
GroupSerObj serializeGroup = new GroupSerObj
{
GroupName = group.SubItems[0].Text,
UpdateRate = group.SubItems[1].Text,
Active = group.SubItems[2].Text,
Items = itemsObj
};
serializer.Serialize(writer, serializeGroup);
}
}
}
Each group iteration creates new List and this list is filled inside inner foreach loop. Later, it is added to GroupSerObj and serialized

Modify CSV file headers/column names using Cinchoo ETL

I have a .Net Core application where I want to change the column names of a csv file. I'm using the Cinchoo ETL library. I have tried the following:
string csv = "../../../../data.csv";
using (var w = new ChoCSVWriter(csv).WithFirstLineHeader().Setup(s => s.FileHeaderWrite += (o, e) =>
{
e.HeaderText = "Test,Test2";
}))
{
w.Write(csv);
}
This is what my data.csv file looks like:
ID,Name
1, David
2, Bob
This is what my csv looks like after running my code:
Test,Test2
../../../../data.csv
The csv header names have changed but my issue is that it deleted all my data and added the path to the file for some odd reason. Any ideas on why that is?

Couple of ways you can rename the columns with new names and produce the CSV output
Option1:
StringBuilder csvIn = new StringBuilder(#"ID,Name
1, David
2, Bob");
StringBuilder csvOut = new StringBuilder();
using (var r = new ChoCSVReader(csvIn)
.WithFirstLineHeader()
)
{
using (var w = new ChoCSVWriter(csvOut)
.WithFirstLineHeader()
)
w.Write(r.Select(r1 => new { Test1 = r1.ID, Test2 = r1.Name }));
}
Console.WriteLine(csvOut.ToString());
Option2:
StringBuilder csvIn = new StringBuilder(#"ID,Name
1, David
2, Bob");
StringBuilder csvOut = new StringBuilder();
using (var r = new ChoCSVReader(csvIn)
.WithFirstLineHeader()
)
{
using (var w = new ChoCSVWriter(csvOut)
.WithFirstLineHeader()
.Setup(s => s.FileHeaderWrite += (o, e) =>
{
e.HeaderText = "Test,Test2";
})
)
w.Write(r);
}
Console.WriteLine(csvOut.ToString());
UPDATE:
Using CSV files instead of text input
string csvInFilePath = #"C:\CSVIn.csv"
string csvOutFilePath = #"C:\CSVOut.csv"
using (var r = new ChoCSVReader(csvInFilePath)
.WithFirstLineHeader()
)
{
using (var w = new ChoCSVWriter(csvOutFilePath)
.WithFirstLineHeader()
)
w.Write(r.Select(r1 => new { Test1 = r1.ID, Test2 = r1.Name }));
}
UPDATE:
To get the headers, cast record to IDictionary and use Keys property on it to get the keys
string csvInFilePath = #"C:\CSVIn.csv"
string csvOutFilePath = #"C:\CSVOut.csv"
using (var r = new ChoCSVReader(csvInFilePath)
.WithFirstLineHeader()
)
{
foreach (IDictionary<string, object> rec in r)
{
var keys = rec.Keys.ToArray();
}
}
In order to auto discover the datatypes of CSV columns, you must set the MaxScanRows on parser. Otherwise all columns will be treated as string type.
StringBuilder csvIn = new StringBuilder(#"ID,Name,Date
1, David, 1/1/2018
2, Bob, 2/12/2019");
using (var r = new ChoCSVReader(csvIn)
.WithFirstLineHeader()
.WithMaxScanRows(2)
)
{
foreach (IDictionary<string, object> rec in r.Take(1))
{
foreach (var kvp in rec)
Console.WriteLine($"{kvp.Key} - {r.Configuration[kvp.Key].FieldType}");
}
}
Hope it helps.

How do I convert a BigQuery row to JSON using the C# API?

I am pulling some data from a BigQuery table using the code below in C#
BigQueryClient client = BigQueryClient.Create("<Project Name>");
BigQueryTable table = client.GetTable("<Database>", "Students");
string sql = $"select * FROM {table} where Marks='50'";
BigQueryResults results = client.ExecuteQuery(sql);
foreach (BigQueryRow row in results.GetRows())
{
}
I want to be able to either read the entire results variable into JSON or be able to get the JSON out of each row.
Of course, I could create a class that models the table. And inside the foreach loop, I could just read each row into the class object. The class object I can try to serialize into JSON using a third party like "newton soft".
Something like :
class Student{
int id; // assume these are columns in the db
string name;
}
My foreach would now look like:
foreach (BigQueryRow row in results.GetRows())
{
Student s=new Student();
s.id = Convert.ToString(row["id"]);
s.name= Convert.ToString(row["name"]);
// something like string x=x+ s.toJSON(); //using newton soft
}
This way string x will have the JSON generated and appended for each row.
Or is there a way I can just add each student to a collection or List and then get the JSON from the whole list?
This whole reading row by row and field by field seems tedious to me and there must be a simpler way I feel. Did not see any support from Google BigQuery for C# to directly convert to JSON. They did have something in Python.
If not then the list to JSON would be better but I am not sure if it supported.
Update :
https://github.com/GoogleCloudPlatform/google-cloud-dotnet/blob/master/apis/Google.Cloud.BigQuery.V2/Google.Cloud.BigQuery.V2/BigQueryRow.cs
Looks like the Big Query Row class has a RawRow field which is of Type TableRow. And the class uses JSON references so , I am sure they have the data of the row in JSON format . How can I expose it to me ?

This might be a little late but you can use:
var latestResult = _bigQueryClient.ExecuteQuery($"SELECT TO_JSON_STRING(t) FROM `{ProjectId}.{DatasetId}.{TableName}` as t", null
All columns will be serialized as json and placed in the first column on each row. You can then use something like Newtonsoft to parse each row easily.

I ran into the same issue.
I am posting this solution which is not optimized for performance but very simple for multiple data types.
This allows you to deserialize anything (almost)
public class BQ
{
private string projectId = "YOUR_PROJECT_ID";
public BQ()
{
}
public List<T> Execute<T>(string sql)
{
var client = BigQueryClient.Create(projectId);
List<T> result = new List<T>();
try
{
string query = sql;
BigQueryResults results = client.ExecuteQuery(query, parameters: null);
List<string> fields = new List<string>();
foreach (var col in results.Schema.Fields)
{
fields.Add(col.Name);
}
Dictionary<string, object> rowoDict;
foreach (var row in results)
{
rowoDict = new Dictionary<string, object>();
foreach (var col in fields)
{
rowoDict.Add(col, row[col]);
}
string json = Newtonsoft.Json.JsonConvert.SerializeObject(rowoDict);
T o = Newtonsoft.Json.JsonConvert.DeserializeObject<T>(json);
result.Add(o);
}
}
catch (Exception ex)
{
Console.WriteLine(ex.ToString());
}
finally
{
client.Dispose();
Console.WriteLine("Done.");
}
return result;
}
}

You can use Newtonsoft.Json. First download by PackageManager Console the Nuget Package, here you can get the command to do that.
After download you can use it as the following code:
List<Student> list = new List<Student>();
foreach (BigQueryRow row in results.GetRows())
{
Student s=new Student();
s.id = Convert.ToString(row["id"]);
s.name= Convert.ToString(row["name"]);
list.Add(s);
}
var jsonResult = Newtonsoft.Json.JsonConvert.SerializeObject(list);
I hope this can help you.

Here is the complete solution for casting BigQueryResults or GetQueryResultsResponse or QueryResponse data to Model/JSON format using C# reflection:
public List<T> GetBQAsModel<T>(string query) where T : class, new()
{
var bqClient = GetBigqueryClient();
var res = bqClient.ExecuteQuery(query, parameters: null);
return GetModels<T>(res);
}
private List<T> GetModels<T>(BigQueryResults tableRows) where T : class, new()
{
var lst = new List<T>();
foreach (var item in tableRows)
{
var lstColumns = new T().GetType().GetProperties(BindingFlags.DeclaredOnly | BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic).ToList();
var newObject = new T();
for (var i = 0; i < item.RawRow.F.Count; i++)
{
var name = item.Schema.Fields[i].Name;
PropertyInfo prop = lstColumns.FirstOrDefault(a => a.Name.ToLower().Equals(name.ToLower()));
if (prop == null)
{
continue;
}
var val = item.RawRow.F[i].V;
prop.SetValue(newObject, Convert.ChangeType(val, prop.PropertyType), null);
}
lst.Add(newObject);
}
return lst;
}
private List<T> GetModels<T>(GetQueryResultsResponse getQueryResultsResponse) where T : class, new()
{
var lst = new List<T>();
foreach (var item in getQueryResultsResponse.Rows)
{
var lstColumns = new T().GetType().GetProperties(BindingFlags.DeclaredOnly | BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic).ToList();
var newObject = new T();
for (var i = 0; i < item.F.Count; i++)
{
var name = getQueryResultsResponse.Schema.Fields[i].Name;
PropertyInfo prop = lstColumns.FirstOrDefault(a => a.Name.ToLower().Equals(name.ToLower()));
if (prop == null)
{
continue;
}
var val = item.F[i].V;
prop.SetValue(newObject, Convert.ChangeType(val, prop.PropertyType), null);
}
lst.Add(newObject);
}
return lst;
}
private List<T> GetModels<T>(QueryResponse queryResponse) where T : class, new()
{
var lst = new List<T>();
foreach (var item in queryResponse.Rows)
{
var lstColumns = new T().GetType().GetProperties(BindingFlags.DeclaredOnly | BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic).ToList();
var newObject = new T();
for (var i = 0; i < item.F.Count; i++)
{
var name = queryResponse.Schema.Fields[i].Name;
PropertyInfo prop = lstColumns.FirstOrDefault(a => a.Name.ToLower().Equals(name.ToLower()));
if (prop == null)
{
continue;
}
var val = item.F[i].V;
prop.SetValue(newObject, Convert.ChangeType(val, prop.PropertyType), null);
}
lst.Add(newObject);
}
return lst;
}

I would do something like this:
var res = Result. Getrows. Select(x=> new student(){id=x[`ID']}).
And then:
var js = json. Conver(res);
This way is much faster and clearer.