Develop Reference

How to read and separate segments of a txt file?

I have a txt file, that has headers and then 3 columns of values (i.e)
Description=null
area = 100
1,2,3
1,2,4
2,1,5 ...
... 1,2,1//(these are the values that I need in one list)
Then another segment
Description=null
area = 10
1,2,3
1,2,4
2,1,5 ...
... 1,2,1//(these are the values that I need in one list).
In fact I just need one list per "Table" of values, the values always are in 3 columns but, there are n segments, any idea?
Thanks!
List<double> VMM40xyz = new List<double>();
foreach (var item in VMM40blocklines)
{
if (item.Contains(','))
{
VMM40xyz.AddRange(item.Split(',').Select(double.Parse).ToList());
}
}
I tried this, but it just work with the values in just one big list.

It looks like you want your data to end up in a format like this:
public class SetOfData //Feel free to name these parts better.
{
public string Description = "";
public string Area = "";
public List<double> Data = new List<double>();
}
...stored somewhere in...
List<SetOfData> finalData = new List<SetOfData>();
So, here's how I'd read that in:
public static List<SetOfData> ReadCustomFile(string Filename)
{
if (!File.Exists(Filename))
{
throw new FileNotFoundException($"{Filename} does not exist.");
}
List<SetOfData> returnData = new List<SetOfData>();
SetOfData currentDataSet = null;
using (FileStream fs = new FileStream(Filename, FileMode.Open))
{
using (StreamReader reader = new StreamReader(fs))
{
while (!reader.EndOfStream)
{
string line = reader.ReadLine();
//This will start a new object on every 'Description' line.
if (line.Contains("Description="))
{
//Save off the old data set if there is one.
if (currentDataSet != null)
returnData.Add(currentDataSet);
currentDataSet = new SetOfData();
//Now, to make sure there is something after "Description=" and to set the Description if there is.
//Your example data used "null" here, which this will take literally to be a string containing the letters "null". You can check the contents of parts[1] inside the if block to change this.
string[] parts = line.Split('=');
if (parts.Length > 1)
currentDataSet.Description = parts[1].Trim();
}
else if (line.Contains("area = "))
{
//Just in case your file didn't start with a "Description" line for some reason.
if (currentDataSet == null)
currentDataSet = new SetOfData();
//And then we do some string splitting like we did for Description.
string[] parts = line.Split('=');
if (parts.Length > 1)
currentDataSet.Area = parts[1].Trim();
}
else
{
//Just in case your file didn't start with a "Description" line for some reason.
if (currentDataSet == null)
currentDataSet = new SetOfData();
string[] parts = line.Split(',');
foreach (string part in parts)
{
if (double.TryParse(part, out double number))
{
currentDataSet.Data.Add(number);
}
}
}
}
//Make sure to add the last set.
returnData.Add(currentDataSet);
}
}
return returnData;
}

Handling DateTime fields from CsvDataReader that contains an empty string

I'm trying load in data from a CSV file using CsvHelper to create a datatable with datacolumns that have a specified type.
var textReader = new StreamReader(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, $"{tableName}.csv"));
var csvReader = new CsvReader(textReader);
var csvDataReader = new CsvDataReader(csvReader);
var dataTable = new DataTable();
foreach(var column in metaColumns)
{
var dataColumn = new DataColumn(column.columnName, GetPropertyType(column.dataType));
dataColumn.AllowDBNull = column.isNull;
dataTable.Columns.Add(dataColumn);
}
dataTable.Load(csvDataReader);
On the load method I'm getting the following error:
String '' was not recognized as a valid DateTime.Couldn't store <> in
derived_mdd_date Column. Expected type is DateTime.
Apparently CsvHelper is loading the column from the CSV file as an empty string and then when given the DateTime type it's not converting the empty string to a null value.
After some research and just trying things I've added
csvReader.Configuration.TypeConverterOptionsCache.GetOptions<DateTime>().NullValues.Add("null");
csvReader.Configuration.TypeConverterOptionsCache.GetOptions<DateTime?>().NullValues.Add("null");
csvReader.Configuration.TypeConverterOptionsCache.GetOptions<string>().NullValues.Add("null");
csvReader.Configuration.TypeConverterCache.AddConverter<DateTime>(new DateFieldConverter());
csvReader.Configuration.TypeConverterCache.AddConverter<DateTime?>(new DateFieldConverter());
...
public class DateFieldConverter : DateTimeConverter
{
public override object ConvertFromString(string text, IReaderRow row, MemberMapData memberMapData)
{
bool result = DateTime.TryParse(text, out DateTime ret);
if (result) return ret;
return null;
}
}
Still getting the same error. I placed a breakpoint on the DateFieldConverter and it's never getting hit so something isn't syncing up correctly. I would think that the default behavior for a DateTime column would be either DateTime.MinValue or null but it's just throwing the error instead.

Unfortunately, it looks like CsvDataReader treats all values as strings and ignores the TypeConverters for other types. There appears to be a feature request to add that capability.
I can offer a workaround that might work for you. You might also check my answer here for another option.
public static void Main(string[] args)
{
using (MemoryStream stream = new MemoryStream())
using (StreamWriter writer = new StreamWriter(stream))
using (StreamReader reader = new StreamReader(stream))
using (CsvReader csv = new CsvReader(reader))
{
writer.WriteLine("DateTime,DateTimeNullable");
writer.WriteLine("5/4/2019,");
writer.WriteLine(",5/5/2019");
writer.Flush();
stream.Position = 0;
csv.Configuration.TypeConverterCache.AddConverter<DateTime>(new DateFieldConverter());
csv.Configuration.TypeConverterCache.AddConverter<DateTime?>(new DateFieldNullableConverter());
var dataTable = new DataTable();
dataTable.Columns.Add("DateTime", typeof(DateTime)).AllowDBNull = false;
dataTable.Columns.Add("DateTimeNullable", typeof(DateTime)).AllowDBNull = true;
csv.Read();
csv.ReadHeader();
while (csv.Read())
{
var row = dataTable.NewRow();
foreach (DataColumn column in dataTable.Columns)
{
if (column.DataType == typeof(DateTime) && column.AllowDBNull)
{
row[column.ColumnName] = csv.GetField(typeof(DateTime?), column.ColumnName);
}
else
{
row[column.ColumnName] = csv.GetField(column.DataType, column.ColumnName);
}
}
dataTable.Rows.Add(row);
}
}
}
public class DateFieldConverter : DateTimeConverter
{
public override object ConvertFromString(string text, IReaderRow row, MemberMapData memberMapData)
{
if (text == string.Empty)
{
return DateTime.MinValue;
}
return base.ConvertFromString(text, row, memberMapData);
}
}
public class DateFieldNullableConverter : DateTimeConverter
{
public override object ConvertFromString(string text, IReaderRow row, MemberMapData memberMapData)
{
if (text == string.Empty)
{
return DBNull.Value;
}
return base.ConvertFromString(text, row, memberMapData);
}
}

Instantiate object depending on type

I have a function that reads a CSV file and returns a list of objects whose parameters depends on the content of the CSV. Right now it works if I hardcode one object. I would like to return different object types.
public static List<CSVObject> ImportCsvIntoObject(string csvFile, string delimiter)
{
List<CSVObject> list = new List<CSVObject>();
using (TextFieldParser csvReader = new TextFieldParser(csvFile))
{
csvReader.SetDelimiters(new String[] { delimiter });
csvReader.HasFieldsEnclosedInQuotes = true;
//Parse the file and creates a list of CSVObject
//example with a csv file with 3 columns
while (!csvReader.EndOfData)
{
string[] fieldData = csvReader.ReadFields();
string parameter1 = fieldData[0];
string parameter2 = fieldData[1];
string parameter3 = fieldData[2];
CSVObject example = new CSVObject(parameter1, parameter2, parameter3);
list.Add(example);
}
}
return list;
}
The following solution works but I'm not sure if there are not better ways to do this.
public static List<Object> ImportCsvIntoList(string csvFile, string delimiter, Type type)
{
List<Object> list = new List<Object>();
using (TextFieldParser csvReader = new TextFieldParser(csvFile))
{
csvReader.SetDelimiters(new String[] { delimiter });
csvReader.HasFieldsEnclosedInQuotes = true;
while (!csvReader.EndOfData)
{
string[] fieldData = csvReader.ReadFields();
string parameter1 = fieldData[0];
string parameter2 = fieldData[1];
string parameter3 = fieldData[2];
var example = Activator.CreateInstance(type, parameter1, parameter2, parameter3);
list.Add(example);
}
}
return list;
}
Furthermore, right now it only works with a hardcoded amount of parameters. Unfortunately, my objects all have a different amount of parameters. How can I call Activator.CreateInstance with different amount of parameters ?
It is my first question so sorry if it isn't written properly, suggestion to improve are more than welcome.

The following might work for you using generics and delegates
public static List<T> ImportCsvIntoObject<T>(string csvFile, string delimiter, Func<List<string>, T> createObject)
{
List<T> list = new List<T>();
using (TextFieldParser csvReader = new TextFieldParser(csvFile))
{
csvReader.SetDelimiters(new String[] { delimiter });
csvReader.HasFieldsEnclosedInQuotes = true;
//Parse the file and creates a list of CSVObject
//example with a csv file with 3 columns
while (!csvReader.EndOfData)
{
string[] fieldData = csvReader.ReadFields();
CSVObject example = createObject(fieldData.ToList())
list.Add(example);
}
}
return list;
}
And you would call the following using:
List<CSVObject> objectList = ImportCsvIntoObject("csvData", ",", (list) => { new CSVObject(list[0], list[1], list[2]); });

The Activator.CreateInstance() function can take an array of parameters, so that you might not know how many you need before runtime, but as you read your CSV, you create arrays corresponding to the number of parameters needed for this particular object (fortunately, your field data object seems to already do this).
So it could be like this :
string[] fieldData = csvReader.ReadFields();
var example = Activator.CreateInstance(type, fieldData);
list.Add(example);
This is because the Activator.CreateInstance function uses the params keyword

How can I add elements to a JSON array without adding "?

I'm developing a library with C#, .NET Framework 4.0 and Newtonsoft.Json 6.0.8.
I'm trying to create my own serializer:
public static string Serialize(List<Models.Codes> codes)
{
if (codes == null)
throw new ArgumentNullException("codes");
if (codes.Count == 0)
throw new ArgumentOutOfRangeException("codes");
StringWriter sw = new StringWriter();
JsonTextWriter writer = new JsonTextWriter(sw);
writer.WriteStartArray();
foreach (Models.Codes code in codes)
writer.WriteValue(CodesSerializer.Serialize(code));
writer.WriteEndArray();
return sw.ToString();
}
public static string Serialize(Models.Codes code)
{
if (code == null)
throw new ArgumentNullException("code");
StringWriter sw = new StringWriter();
JsonTextWriter writer = new JsonTextWriter(sw);
writer.WriteStartObject();
writer.WritePropertyName("Code");
writer.WriteValue(code.Code);
writer.WritePropertyName("BatchId");
writer.WriteValue(code.BatchId);
writer.WritePropertyName("ProductId");
writer.WriteValue(code.ProductId);
writer.WritePropertyName("CodeLevel");
writer.WriteValue(code.CodeLevel);
writer.WritePropertyName("CommisioningFlag");
writer.WriteValue(code.CommisioningFlag);
writer.WritePropertyName("Timespan");
writer.WriteValue(code.Timespan);
if (!string.IsNullOrWhiteSpace(code.Username))
{
writer.WritePropertyName("Username");
writer.WriteValue(code.Username);
}
if (!string.IsNullOrWhiteSpace(code.Source))
{
writer.WritePropertyName("Source");
writer.WriteValue(code.Source);
}
if (!string.IsNullOrWhiteSpace(code.Reason))
{
writer.WritePropertyName("Reason");
writer.WriteValue(code.Reason);
}
writer.WriteEndObject();
string text = sw.ToString();
return text;
}
But it generates a string like this:
[\"{\\\"Code\\\":\\\"81861400008000002386\\\",\\\"BatchId\\\":5,\\\"ProductId\\\":7,\\\"CodeLevel\\\":1,\\\"CommisioningFlag\\\":1,\\\"Timespan\\\":null}\",
Have you notices the extra " before {?
How can I don't put that "?
I think the problem is that I am adding a value to an array inside the loop:
writer.WriteStartArray();
foreach (Models.Codes code in codes)
writer.WriteValue(CodesSerializer.Serialize(code));
writer.WriteEndArray();
I have changed writer.WriteValue(CodesSerializer.Serialize(code)); with writer.WriteRaw(CodesSerializer.Serialize(code)); and now it doesn't write ", but now I need to write JSON value delimiter and I can't use writer.WriteValueDelimiter();.
How can I add elements to a JSON array without adding "?

I have found the solution. I have to use writer.WriteRawValue():
writer.WriteStartArray();
foreach (Models.Codes code in codes)
writer.WriteRawValue(CodesSerializer.Serialize(code));
writer.WriteEndArray();

Export iQueryable of complex objects to Excel

We have some code that exports data from a database to Excel and it works well.
The main problem is we use a DB view to gather all our data. This creates the issue of having a sizable view as we have multiple types of objects of which we export.
class Event
int id
string title
List<EventDate> Dates
string desc
class EventDate
DateTime start
DateTime end
List<EventLocation> Locations
class EventLocation
string address
string city
string zip
class Birthday : Event
int balloonsOrdered
string cakeText
class Meeting : Event
string Organizer
string Topic
So, above is the model. Birthday and Meeting inherit from Event and all Event objects have a list of EventDate objects. Each EventDate object has a start date, end date and a list of Location objects.
Our goal is to find a dynamic way to get data from the DB to an Excel doc. We'd rather not maintain a massive view in the database (as we'll add more event types eventually).
I'm not all that familiar with .NET's XML capabilities, but the solution we are using now uses OpenXML and the code makes sense.

You could create a CSV file using List<T> and and following code:
using System;
using System.Collections.Generic;
using System.IO;
using System.Reflection;
using System.Web;
public static void CreateCSV<T>(List<T> list, string csvNameWithExt)
{
if (list == null || list.Count == 0) return;
HttpContext.Current.Response.Clear();
HttpContext.Current.Response.AddHeader(
"content-disposition", string.Format("attachment; filename={0}", csvNameWithExt));
HttpContext.Current.Response.ContentType = "application/vnd.ms-excel";
//get type from 0th member
Type t = list[0].GetType();
string newLine = Environment.NewLine;
using (StringWriter sw = new StringWriter())
{
//gets all properties
PropertyInfo[] props = t.GetProperties();
//this is the header row
//foreach of the properties in class above, write out properties
foreach (PropertyInfo pi in props)
{
sw.Write(pi.Name.ToUpper() + ",");
}
sw.Write(newLine);
//this acts as datarow
foreach (T item in list)
{
//this acts as datacolumn
foreach (PropertyInfo Column in props)
{
//this is the row+col intersection (the value)
string value = item.GetType().GetProperty(Column.Name).GetValue(item, null).ToString();
if (value.Contains(","))
{
value = "\"" + value + "\"";
}
sw.Write(value + ",");
}
sw.Write(newLine);
}
// render the htmlwriter into the response
HttpContext.Current.Response.Write(sw.ToString());
HttpContext.Current.Response.End();
}
}

I improved the solution from Brad to better work with Entity Framework Data Annotations :
it gets the display name from annotations instead of the column name
it does not export columns that are marked "scaffold = false" via annotations
it offers you a way to deal with complex types (not primitive types)
public static class ExportListToCSV
{
public static void CreateCSV<T>(List<T> list, string csvNameWithExt)
{
if (list == null || list.Count == 0) return;
HttpContext.Current.Response.Clear();
HttpContext.Current.Response.AddHeader(
"content-disposition", string.Format("attachment; filename={0}", csvNameWithExt));
HttpContext.Current.Response.ContentType = "application/vnd.ms-excel";
//get type from 0th member
Type t = list[0].GetType();
string newLine = Environment.NewLine;
//gets all columns
PropertyInfo[] columns = t.GetProperties();
// skip columns where ScaffoldColumn = false
// useful to skip column containing IDs for Foreign Keys
var props = t.GetProperties(BindingFlags.Instance | BindingFlags.Public)
.Select(p => new
{
Property = p,
Attribute = p.GetCustomAttribute<ScaffoldColumnAttribute>()
})
.Where(p => p.Attribute == null || (p.Attribute != null && p.Attribute.Scaffold != false))
.ToList();
using (StringWriter sw = new StringWriter())
{
//this is the header row
foreach (var prop in props)
{
var pi = prop.Property;
string columnName = "";
// outputs raw column name, but this is not really meaningful for the end user
//sw.Write(pi.Name + ",");
columnName = pi.GetDisplayName();
sw.Write(columnName + ",");
}
sw.Write(newLine);
//this acts as datarow
foreach (T item in list)
{
//this acts as datacolumn
foreach (var prop in props)
{
var column = prop.Property;
//this is the row+col intersection (the value)
PropertyInfo info = item.GetType().GetProperty(column.Name);
//string value = info.GetValue(item, null);
string value = GetDescriptionForComplexObjects(info.GetValue(item, null));
if (value.Contains(","))
{
value = "\"" + value + "\"";
}
sw.Write(value + ",");
}
sw.Write(newLine);
}
// render the htmlwriter into the response
HttpContext.Current.Response.Write(sw.ToString());
HttpContext.Current.Response.End();
}
}
private static string GetDescriptionForComplexObjects(object value)
{
string desc = "";
if (
(value != null && !IsSimpleType(value.GetType()))
)
{
dynamic dynObject = value;
if (dynObject != null)
{
//using Model Extensions,
//I made sure all my objects have a DESCRIPTION property
//this property must return a string
//for an employee object, you would return the employee's name for example
//desc = dynObject.DESCRIPTION;
}
}
else
{
desc = "" + value;
}
return desc;
}
public static string GetDisplayName(this PropertyInfo pi)
{
if (pi == null)
{
throw new ArgumentNullException(nameof(pi));
}
return pi.IsDefined(typeof(DisplayAttribute)) ? pi.GetCustomAttribute<DisplayAttribute>().GetName() : pi.Name;
}
public static bool IsSimpleType(Type type)
{
return
type.IsPrimitive ||
new Type[] {
typeof(Enum),
typeof(String),
typeof(Decimal),
typeof(DateTime),
typeof(DateTimeOffset),
typeof(TimeSpan),
typeof(Guid)
}.Contains(type) ||
Convert.GetTypeCode(type) != TypeCode.Object ||
(type.IsGenericType && type.GetGenericTypeDefinition() == typeof(Nullable<>) && IsSimpleType(type.GetGenericArguments()[0]))
;
}
}
}