Develop Reference

How do I convert a BigQuery row to JSON using the C# API?

I am pulling some data from a BigQuery table using the code below in C#
BigQueryClient client = BigQueryClient.Create("<Project Name>");
BigQueryTable table = client.GetTable("<Database>", "Students");
string sql = $"select * FROM {table} where Marks='50'";
BigQueryResults results = client.ExecuteQuery(sql);
foreach (BigQueryRow row in results.GetRows())
{
}
I want to be able to either read the entire results variable into JSON or be able to get the JSON out of each row.
Of course, I could create a class that models the table. And inside the foreach loop, I could just read each row into the class object. The class object I can try to serialize into JSON using a third party like "newton soft".
Something like :
class Student{
int id; // assume these are columns in the db
string name;
}
My foreach would now look like:
foreach (BigQueryRow row in results.GetRows())
{
Student s=new Student();
s.id = Convert.ToString(row["id"]);
s.name= Convert.ToString(row["name"]);
// something like string x=x+ s.toJSON(); //using newton soft
}
This way string x will have the JSON generated and appended for each row.
Or is there a way I can just add each student to a collection or List and then get the JSON from the whole list?
This whole reading row by row and field by field seems tedious to me and there must be a simpler way I feel. Did not see any support from Google BigQuery for C# to directly convert to JSON. They did have something in Python.
If not then the list to JSON would be better but I am not sure if it supported.
Update :
https://github.com/GoogleCloudPlatform/google-cloud-dotnet/blob/master/apis/Google.Cloud.BigQuery.V2/Google.Cloud.BigQuery.V2/BigQueryRow.cs
Looks like the Big Query Row class has a RawRow field which is of Type TableRow. And the class uses JSON references so , I am sure they have the data of the row in JSON format . How can I expose it to me ?

This might be a little late but you can use:
var latestResult = _bigQueryClient.ExecuteQuery($"SELECT TO_JSON_STRING(t) FROM `{ProjectId}.{DatasetId}.{TableName}` as t", null
All columns will be serialized as json and placed in the first column on each row. You can then use something like Newtonsoft to parse each row easily.

I ran into the same issue.
I am posting this solution which is not optimized for performance but very simple for multiple data types.
This allows you to deserialize anything (almost)
public class BQ
{
private string projectId = "YOUR_PROJECT_ID";
public BQ()
{
}
public List<T> Execute<T>(string sql)
{
var client = BigQueryClient.Create(projectId);
List<T> result = new List<T>();
try
{
string query = sql;
BigQueryResults results = client.ExecuteQuery(query, parameters: null);
List<string> fields = new List<string>();
foreach (var col in results.Schema.Fields)
{
fields.Add(col.Name);
}
Dictionary<string, object> rowoDict;
foreach (var row in results)
{
rowoDict = new Dictionary<string, object>();
foreach (var col in fields)
{
rowoDict.Add(col, row[col]);
}
string json = Newtonsoft.Json.JsonConvert.SerializeObject(rowoDict);
T o = Newtonsoft.Json.JsonConvert.DeserializeObject<T>(json);
result.Add(o);
}
}
catch (Exception ex)
{
Console.WriteLine(ex.ToString());
}
finally
{
client.Dispose();
Console.WriteLine("Done.");
}
return result;
}
}

You can use Newtonsoft.Json. First download by PackageManager Console the Nuget Package, here you can get the command to do that.
After download you can use it as the following code:
List<Student> list = new List<Student>();
foreach (BigQueryRow row in results.GetRows())
{
Student s=new Student();
s.id = Convert.ToString(row["id"]);
s.name= Convert.ToString(row["name"]);
list.Add(s);
}
var jsonResult = Newtonsoft.Json.JsonConvert.SerializeObject(list);
I hope this can help you.

Here is the complete solution for casting BigQueryResults or GetQueryResultsResponse or QueryResponse data to Model/JSON format using C# reflection:
public List<T> GetBQAsModel<T>(string query) where T : class, new()
{
var bqClient = GetBigqueryClient();
var res = bqClient.ExecuteQuery(query, parameters: null);
return GetModels<T>(res);
}
private List<T> GetModels<T>(BigQueryResults tableRows) where T : class, new()
{
var lst = new List<T>();
foreach (var item in tableRows)
{
var lstColumns = new T().GetType().GetProperties(BindingFlags.DeclaredOnly | BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic).ToList();
var newObject = new T();
for (var i = 0; i < item.RawRow.F.Count; i++)
{
var name = item.Schema.Fields[i].Name;
PropertyInfo prop = lstColumns.FirstOrDefault(a => a.Name.ToLower().Equals(name.ToLower()));
if (prop == null)
{
continue;
}
var val = item.RawRow.F[i].V;
prop.SetValue(newObject, Convert.ChangeType(val, prop.PropertyType), null);
}
lst.Add(newObject);
}
return lst;
}
private List<T> GetModels<T>(GetQueryResultsResponse getQueryResultsResponse) where T : class, new()
{
var lst = new List<T>();
foreach (var item in getQueryResultsResponse.Rows)
{
var lstColumns = new T().GetType().GetProperties(BindingFlags.DeclaredOnly | BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic).ToList();
var newObject = new T();
for (var i = 0; i < item.F.Count; i++)
{
var name = getQueryResultsResponse.Schema.Fields[i].Name;
PropertyInfo prop = lstColumns.FirstOrDefault(a => a.Name.ToLower().Equals(name.ToLower()));
if (prop == null)
{
continue;
}
var val = item.F[i].V;
prop.SetValue(newObject, Convert.ChangeType(val, prop.PropertyType), null);
}
lst.Add(newObject);
}
return lst;
}
private List<T> GetModels<T>(QueryResponse queryResponse) where T : class, new()
{
var lst = new List<T>();
foreach (var item in queryResponse.Rows)
{
var lstColumns = new T().GetType().GetProperties(BindingFlags.DeclaredOnly | BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic).ToList();
var newObject = new T();
for (var i = 0; i < item.F.Count; i++)
{
var name = queryResponse.Schema.Fields[i].Name;
PropertyInfo prop = lstColumns.FirstOrDefault(a => a.Name.ToLower().Equals(name.ToLower()));
if (prop == null)
{
continue;
}
var val = item.F[i].V;
prop.SetValue(newObject, Convert.ChangeType(val, prop.PropertyType), null);
}
lst.Add(newObject);
}
return lst;
}

I would do something like this:
var res = Result. Getrows. Select(x=> new student(){id=x[`ID']}).
And then:
var js = json. Conver(res);
This way is much faster and clearer.

How to convert nested List to Dataset in C#

I have a nested list and I need to convert it to DataSet in C#. I found many example about this but they dont do what I need. I have a list in a list, I need the nested list in another DataTable in the DataSet.
Here is an example of list
public class InvoiceResponse(){
public string BillToName { get; set; }
public string BillToAddr1 { get; set; }
...
...
public List<InvoiceItemResponse> Items { get; set; }
}
I have used the code below to convert the List to DataSet but It didnt convert Items to DataTable in DataSet
public DataSet CreateDataSet<T>(List<T> list)
{
//list is nothing or has nothing, return nothing (or add exception handling)
if (list == null || list.Count == 0) { return null; }
//get the type of the first obj in the list
var obj = list[0].GetType();
//now grab all properties
var properties = obj.GetProperties();
//make sure the obj has properties, return nothing (or add exception handling)
if (properties.Length == 0) { return null; }
//it does so create the dataset and table
var dataSet = new DataSet();
var dataTable = new DataTable();
//now build the columns from the properties
var columns = new DataColumn[properties.Length];
for (int i = 0; i < properties.Length; i++)
{
columns[i] = new DataColumn(properties[i].Name, properties[i].PropertyType);
}
//add columns to table
dataTable.Columns.AddRange(columns);
//now add the list values to the table
foreach (var item in list)
{
//create a new row from table
var dataRow = dataTable.NewRow();
//now we have to iterate thru each property of the item and retrieve it's value for the corresponding row's cell
var itemProperties = item.GetType().GetProperties();
for (int i = 0; i < itemProperties.Length; i++)
{
dataRow[i] = itemProperties[i].GetValue(item, null);
}
//now add the populated row to the table
dataTable.Rows.Add(dataRow);
}
//add table to dataset
dataSet.Tables.Add(dataTable);
//return dataset
return dataSet;
}
How can I convert the Items list to another DataTable into the DataSet?

You can just add a bit of recursivity to your code, something like this:
var set = new DataSet();
AddToDataSet(set, resp);
...
public static void AddToDataSet(DataSet set, object value)
{
if (set == null)
throw new ArgumentNullException(nameof(set));
if (value == null)
return;
var type = value.GetType();
var table = set.Tables[type.FullName];
if (table == null)
{
// create one table per type (name)
table = new DataTable(type.FullName);
set.Tables.Add(table);
foreach (var prop in type.GetProperties().Where(p => p.CanRead))
{
if (IsEnumerable(prop))
continue;
var col = new DataColumn(prop.Name, prop.PropertyType);
table.Columns.Add(col);
}
}
var row = table.NewRow();
foreach (var prop in type.GetProperties().Where(p => p.CanRead))
{
object propValue = prop.GetValue(value);
if (IsEnumerable(prop))
{
if (propValue != null)
{
foreach (var child in (ICollection)propValue)
{
AddToDataSet(set, child);
}
}
continue;
}
row[prop.Name] = propValue;
}
table.Rows.Add(row);
}
private static bool IsEnumerable(PropertyInfo pi)
{
// note: we could also use IEnumerable (but string, arrays are IEnumerable...)
return typeof(ICollection).IsAssignableFrom(pi.PropertyType);
}

Convert DataTable to LINQ Anonymous Type

I want a function which takes in a datatable & returns a List (object is not DataRow)
Eg. :
I know I can do this (but this requires column names to be known) :
// Datatable dt = Filled from a Database query & has 3 columns Code,Description & ShortCode
List<object> rtn = new List<object>();
var x = from vals in dt.Select()
select new
{
Code = vals["Code"],
Description = vals["Description"],
ShortCode = vals["ShortCode"],
};
rtn.AddRange(x)
return rtn;
What i want is a generic version so that i can pass in any datatable & it will generate based on column names in the datatable.

Since the property names are not known at compile time and you want to use the data for JSON serialization, you can use the following to create a list of dictionary. If you use Newtonsoft JSON, then the serialization takes care of converting the key value pairs in a JSON object format.
IEnumerable<Dictionary<string,object>> result = dt.Select().Select(x => x.ItemArray.Select((a, i) => new { Name = dt.Columns[i].ColumnName, Value = a })
.ToDictionary(a => a.Name, a => a.Value));

In order to dynamically create properties so as to treat different dataTables with different set of Columns, we can use the System.Dynamic.ExpandoObject. It basically implements, IDictionary <string,object>. The format, which can easily be converted to JSON.
int colCount = dt.Columns.Count;
foreach (DataRow dr in dt.Rows)
{
dynamic objExpando = new System.Dynamic.ExpandoObject();
var obj = objExpando as IDictionary<string, object>;
for (int i = 0; i < colCount; i++)
{
string key = dr.Table.Columns[i].ColumnName.ToString();
string val = dr[key].ToString();
obj[key] = val;
}
rtn.Add(obj);
}
String json = new System.Web.Script.Serialization.JavaScriptSerializer().Serialize(rtn);

You can use the following generic function:-
private static List<T> ConvertDataTable<T>(DataTable dt)
{
List<T> data = newList<T>();
foreach (DataRowrow in dt.Rows)
{
Titem = GetItem<T>(row);
data.Add(item);
}
return data;
}
private static TGetItem<T>(DataRow dr)
{
Type temp = typeof(T);
T obj =Activator.CreateInstance<T>();
foreach (DataColumncolumn in dr.Table.Columns)
{
foreach (PropertyInfopro in temp.GetProperties())
{
if (pro.Name == column.ColumnName)
pro.SetValue(obj,dr[column.ColumnName], null);
else
continue;
}
}
return obj;
}
Please check my article, which has complete demonstration on how to use this generic method.

Here is the original question:
// Datatable dt = Filled from a Database query & has 3 columns Code,Description & ShortCode
List<object> rtn = new List<object>();
var x = from vals in dt.Select()
select new
{
Code = vals["Code"],
Description = vals["Description"],
ShortCode = vals["ShortCode"],
};
rtn.AddRange(x)
return rtn;
Just replace with
List<object> rtn = JsonConvert.DeserializeObject<List<object>>(JsonConvert.SerializeObject(dt));

You will have the provide the anonymous object as a parameter and use json/xml serialization:
protected static List<T> ToAnonymousCollection<T>(DataTable dt, T anonymousObject)
{
List<DataColumn> dataColumns = dt.Columns.OfType<DataColumn>().ToList();
return dt.Rows.OfType<DataRow>().Select(dr =>
{
Dictionary<string, object> dict = new Dictionary<string, object>();
dataColumns.Each(dc => dict.Add(dc.ColumnName, dr[dc]));
return JsonConvert.DeserializeAnonymousType(JsonConvert.SerializeObject(dict), anonymousObject);
}).ToList();
}
Usage:
var anonymousCollection = ToAnonymousCollection(dt, new { Code = [ColumnTypeValue, eg. 0], Description = [ColumnTypeValue, eg. string.Empty], ShortCode = Code=[ColumnTypeValue, eg. 0] })

Convert dynamic list to datatable c#

am searching and cracking my brain on how to convert a dynamic list to a databale,
c#, please advise, thanks
List<dynamic>dlist=new List<dynamic>
to
DataTable

I think you looking something like this. Hope it's working for you.
From dynamic list to DataTable:
List<dynamic> dlist=new List<dynamic>
var json = JsonConvert.SerializeObject(dlist);
DataTable dataTable = (DataTable)JsonConvert.DeserializeObject(json, (typeof(DataTable)));
Also to get JsonString from DataTable:
string JSONresult = JsonConvert.SerializeObject(dataTable);

The following is the method through which you can convert any list object to datatable..
public DataTable ConvertToDataTable<T>(IList<T> data)
{
PropertyDescriptorCollection properties =
TypeDescriptor.GetProperties(typeof(T));
DataTable table = new DataTable();
foreach (PropertyDescriptor prop in properties)
table.Columns.Add(prop.Name, Nullable.GetUnderlyingType(prop.PropertyType) ?? prop.PropertyType);
foreach (T item in data)
{
DataRow row = table.NewRow();
foreach (PropertyDescriptor prop in properties)
row[prop.Name] = prop.GetValue(item) ?? DBNull.Value;
table.Rows.Add(row);
}
return table;
}
http://social.msdn.microsoft.com/Forums/vstudio/en-US/6ffcb247-77fb-40b4-bcba-08ba377ab9db/converting-a-list-to-datatable?forum=csharpgeneral

public DataTable ToDataTable<T>(dynamic items)
{
DataTable dtDataTable = new DataTable();
if (items.Count == 0) return dtDataTable;
((IEnumerable)items[0]).Cast<dynamic>().Select(p => p.Name).ToList().ForEach(col => { dtDataTable.Columns.Add(col); });
((IEnumerable)items).Cast<dynamic>().ToList().
ForEach(data =>
{
DataRow dr = dtDataTable.NewRow();
((IEnumerable)data).Cast<dynamic>().ToList().ForEach(Col => { dr[Col.Name] = Col.Value; });
dtDataTable.Rows.Add(dr);
});
return dtDataTable;
}

I don't know why you need this, however, you can use this ObjectShredder using reflection which can convert anything to DataTable, so even dynamic or anonymous types:
Implement CopyToDataTable<T> Where the generic Type T Is Not a DataRow
However, my suggestion is to not name that extension method CopyToDataTable but for example CopyAnyToDataTable to avoid conflicts with the existing extension method CopyToDataTable.

Use this function ,
public static DataTable ConvertToDatatable<T>(this IList<T> data)
{
PropertyDescriptorCollection props =
TypeDescriptor.GetProperties(typeof(T));
DataTable table = new DataTable();
for(int i = 0 ; i < props.Count ; i++)
{
PropertyDescriptor prop = props[i];
table.Columns.Add(prop.Name, prop.PropertyType);
}
object[] values = new object[props.Count];
foreach (T item in data)
{
for (int i = 0; i < values.Length; i++)
{
values[i] = props[i].GetValue(item);
}
table.Rows.Add(values);
}
return table;
}

If the underlying type is ExpandoObject, then you need to check for IDictionary instead of going via reflection. Hopefully this helps someone else in the future:
public static DataTable ConvertToDataTable<T>(IEnumerable<T> data)
{
DataTable table = new DataTable();
foreach (T item in data)
{
if (item is IDictionary<string, object> dict)
{
foreach (var key in dict)
{
table.Columns.Add(key.Key, key.Value?.GetType() ?? typeof(object));
}
break;
}
foreach (var prop in typeof(T).GetProperties())
{
table.Columns.Add(prop.Name, prop.PropertyType);
}
break;
}
DataRow row = null;
foreach (T item in data)
{
if (item is IDictionary<string, object> dict)
{
row = table.NewRow();
foreach (var key in dict)
{
row[key.Key] = key.Value;
}
table.Rows.Add(row);
continue;
}
row = table.NewRow();
foreach (var prop in typeof(T).GetProperties())
{
row[prop.Name] = prop.GetValue(item);
}
table.Rows.Add(row);
}
return table;
}

To Convert the Dynamic List object into DataTable using C#
public DataTable DynamicToDT(List<object> objects)
{
DataTable dt = new DataTable("StudyRecords"); // Runtime Datatable
string[] arr = { "Name", "Department", "CollegeName", "Address" };// Column Name for DataTable
if (objects != null && objects.Count > 0)
{
for (int i = 0; i < objects.Count; i++)
{
dt.Columns.Add(arr[i]);
if (i == 0)
{
var items = objects[0] as IEnumerable<string>;
foreach (var itm in items)
{
DataRow dr1 = dt.NewRow(); // Adding values to Datatable
dr1[arr[i]] = itm;
dt.Rows.Add(dr1);
}
}
else
{
var items = objects[i] as IEnumerable<string>;
int count = 0;
foreach (var itm in items)
{
dt.Rows[count][arr[i]] = itm;
count++;
}
}
}
return dt; // Converted Dynamic list to Datatable
}
return null;
}

public static DataTable DynamicToDT(List objects)
{
var data = objects.ToArray();
if (data.Count() == 0) return null;
var dt = new DataTable();
foreach (var key in ((IDictionary<string, object>)data[0]).Keys)
{
dt.Columns.Add(key);
}
foreach (var d in data)
{
dt.Rows.Add(((IDictionary<string, object>)d).Values.ToArray());
}
return dt;
}

// Obtem a lista dinamica via chamada API
List<dynamic> resultado = ExecutaQuery(sql);
// converte a lista dinamica com o resultado em JSON
string json = JsonConvert.SerializeObject(resultado);
// converte o json em datatable
DataTable dataTable = (DataTable)JsonConvert.DeserializeObject(json, (typeof(DataTable)));

Convert datatable to list of generic type

How to convert datatable into list of generic type. Below is the scenario.
I have datatable with name table1 and contains columns col1,col2. how could we convert this table into a list of type name table1bj(which can be different per the datatable name) with properties col1 and col2 with compatible datatype as of datatable column data types.
There are many post on SO but these are with the converting datatable into predefined object list. Here in my case I have to generate object and list dynamically from the datatable. Thanks.

Assuming that you've already created the class table1bj (consider to make it uppercase due to .NET naming conventions) with two properties col1,col2 (the same). You just have to use Enumerable.Select to create instances of this class and ToList to create a generic List<table1bj>:
List<table1bj> result = table1.AsEnumerable()
.Select(row => new table1bj
{
col1 = row.Field<string>("col1"),
col1 = row.Field<string>("col1")
}
).ToList();
I have also presumed that these properties are strings, otherwise use the correct type with the Field extension method. If you don't know the type you should stay with your DataTable since it's already an in-memory collection with dynamic types.

You can do like this...
Create Class with properties :
public class table1bj
{
public string col1{ get; set; }
public string col2{ get; set; }
}
Convert DataTable to Generic Type :
List<table1bj> Objtable1bj = table1.ToCollection<table1bj>();

I know this question asked many times ago, but also I need a solutions for convert data table to dynamic or generic types in one method and I can't find answer for this, so post my answer.
You can use a extension method to convert data table to any type like below:
public static class Extension
{
public static IList<T> ToList<T>(this DataTable dt, bool isFirstRowColumnsHeader = false) where T : new()
{
var results = new List<T>();
if (dt != null && dt.Rows.Count > 0)
{
var columns = dt.Columns.Cast<DataColumn>().ToList();
var rows = dt.Rows.Cast<DataRow>().ToList();
var headerNames = columns.Select(col => col.ColumnName).ToList();
//
// Find properties name or columns name
if (isFirstRowColumnsHeader)
{
for (var i = 0; i < headerNames.Count; i++)
{
if (rows[0][i] != DBNull.Value && !string.IsNullOrEmpty(rows[0][i].ToString()))
headerNames[i] = rows[0][i].ToString();
}
//
// remove first row because that is header
rows.RemoveAt(0);
}
// Create dynamic or anonymous object for `T type
if (typeof(T) == typeof(System.Dynamic.ExpandoObject) ||
typeof(T) == typeof(System.Dynamic.DynamicObject) ||
typeof(T) == typeof(System.Object))
{
var dynamicDt = new List<dynamic>();
foreach (var row in rows)
{
dynamic dyn = new ExpandoObject();
dynamicDt.Add(dyn);
for (var i = 0; i < columns.Count; i++)
{
var dic = (IDictionary<string, object>)dyn;
dic[headerNames[i]] = row[columns[i]];
}
}
return (dynamic)dynamicDt;
}
else // other types of `T
{
var properties = typeof(T).GetProperties();
if (columns.Any() && properties.Any())
{
foreach (var row in rows)
{
var entity = new T();
for (var i = 0; i < columns.Count; i++)
{
if (!row.IsNull(columns[i]))
{
typeof(T).GetProperty(headerNames[i])? // ? -> maybe the property by name `headerNames[i]` is not exist in entity then get null!
.SetValue(entity, row[columns[i]] == DBNull.Value ? null : row[columns[i]]);
}
}
results.Add(entity);
}
}
}
}
return results;
}
}

We can do it by Reflection also, this method helps to set ClassObject properties by DataTable:
using System.Reflection;
public void SetObjectProperties(object objClass, DataTable dataTable)
{
DataRow _dataRow = dataTable.Rows[0];
Type objType = objClass.GetType();
List<PropertyInfo> propertyList = new List<PropertyInfo>(objType.GetProperties());
foreach (DataColumn dc in _dataRow.Table.Columns)
{
var _prop = propertyList.Where(a => a.Name == dc.ColumnName).Select(a => a).FirstOrDefault();
if (_prop == null) continue;
_prop.SetValue(objClass, Convert.ChangeType(_dataRow[dc], Nullable.GetUnderlyingType(_prop.PropertyType) ?? _prop.PropertyType), null);
}
}