Develop Reference

How Do I Deserialize Into Local Collections in .NET?

I am trying to build a data pipeline in .NET. I have been given an xsd and have used the XML Schema Definition Tool to generate C# classes that represent the object model. In order to load this data into my data store I need to transform the data coming out of the XML into structure that matches my application schema and collect/dedupe elements. To do this I have a parser class that will read a file and load the contents into local collections which will then be loaded into my database. I see two options to do this -
Manually loop through the XML with an XmlReader and pull out the data I need, loading it into the local collections in stream. This is not desirable because it does not take advantage of the strongly typed/strict xsd that I was given and requires a lot of hard coding things like while (reader.Read()), check for specific XML nodes, and then `reader.GetAttribute("HardCodedString").
Use XmlSerializer to deserialize the whole file at once and then loop through the deserialized collections and insert into my local collections. This is not desirable because the files could be very large and this method forces me to loop through all of the data twice (once to deserialize and once to extract the data to my local collections).
Ideally I would like some way to register a delegate to be executed as each object is deserialized to insert into my local collections. Is there something in the framework that allows me to do this? Requirements are as follows:
Performant - Only loop through the data once.
Functional - Data is inserted into the local collections during deserialization.
Maintainable - Utilize strongly typed classes that were generated via the xsd.
I have created a minimal example to illustrate my point.
Example XML File:
<Hierarchy xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://www.example.com/example">
<Children>
<Child ChildId="1" ChildName="First">
<Parents>
<Parent ParentId="1" ParentName="First" RelationshipStart="1900-01-01T00:00:00"/>
<Parent ParentId="2" ParentName="Second" RelationshipStart="2000-01-01T00:00:00"/>
</Parents>
</Child>
<Child ChildId="2" ChildName="Second">
<Parents>
<Parent ParentId="2" ParentName="Second" RelationshipStart="1900-01-01T00:00:00"/>
<Parent ParentId="3" ParentName="Third" RelationshipStart="2000-01-01T00:00:00"/>
</Parents>
</Child>
</Children>
</Hierarchy>
Local collections I am trying to load:
public Dictionary<int, string> Parents { get; }
public Dictionary<int, string> Children { get; }
public List<Relationship> Relationships { get; }
Manual version (not maintainable and doesn't use xsd):
public void ParseFileManually(string fileName)
{
using (var reader = XmlReader.Create(fileName))
{
while (reader.Read())
{
if (reader.NodeType == XmlNodeType.Element && reader.Name == "Hierarchy")
{
while (reader.Read())
{
if (reader.NodeType == XmlNodeType.Element && reader.Name == "Child")
{
int childId = int.Parse(reader.GetAttribute("ChildId"));
string childName = reader.GetAttribute("ChildName");
Children[childId] = childName;
while (reader.Read())
{
if (reader.NodeType == XmlNodeType.Element && reader.Name == "Parent")
{
int parentId = int.Parse(reader.GetAttribute("ParentId"));
string parentName = reader.GetAttribute("ParentName");
DateTime relationshipStart = DateTime.Parse(reader.GetAttribute("RelationshipStart"));
Parents[parentId] = parentName;
Relationships.Add(
new Relationship{
ParentId = parentId,
ChildId = childId,
Start = relationshipStart
});
}
else if (reader.NodeType == XmlNodeType.EndElement && reader.Name == "Child")
{
break;
}
}
}
}
}
}
}
}
Deserialize version (loops through the data twice):
public void ParseFileWithDeserialize(string fileName)
{
var serializer = new XmlSerializer(typeof(Hierarchy));
using (var fileStream = new FileStream(fileName, FileMode.Open))
{
var fileData = (Hierarchy) serializer.Deserialize(fileStream);
foreach (var child in fileData.Children)
{
Children[child.ChildId] = child.ChildName;
foreach (var parent in child.Parents)
{
Parents[parent.ParentId] = parent.ParentName;
Relationships.Add(
new Relationship
{
ParentId = parent.ParentId,
ChildId = child.ChildId,
Start = parent.RelationshipStart
});
}
}
}
}

You should use some annotations to get the data from the correct field in the XML, if you use these definitions;
public class Hierarchy
{
public Hierarchy()
{
Children = new List<Child>();
}
public List<Child> Children { get; set; }
}
public class Child
{
public Child()
{
Parents = new List<Parent>();
}
[XmlAttribute("ChildId")]
public int ChildId { get; set; }
[XmlAttribute("ChildName")]
public string ChildName { get; set; }
public List<Parent> Parents { get; set; }
}
public class Parent
{
[XmlAttribute("ParentId")]
public int ParentId { get; set; }
[XmlAttribute("ParentName")]
public string ParentName { get; set; }
[XmlAttribute("RelationshipStart")]
public DateTime RelationshipStart { get; set; }
}
Then you should be able to simplify your code to;
public static Hierarchy Deserialize(string fileName)
{
using (var fileStream = new StreamReader(fileName, Encoding.UTF8))
{
XmlSerializer ser = new XmlSerializer(typeof(Hierarchy));
return (Hierarchy)ser.Deserialize(fileStream);
}
}
To test it out you can create a sample data set and serialize it to a file, then use the above code to read it back
public static void Serialize(Hierarchy h, string fileName)
{
System.Xml.Serialization.XmlSerializer ser = new System.Xml.Serialization.XmlSerializer(typeof(Hierarchy));
StreamWriter sw = new StreamWriter(fileName, false, Encoding.UTF8);
ser.Serialize(sw, h);
}
Test Code
static void Test()
{
Hierarchy h = new Hierarchy();
Parent p1 = new Parent() { ParentId = 1, ParentName = "First", RelationshipStart = DateTime.Now };
Parent p2 = new Parent() { ParentId = 2, ParentName = "Second", RelationshipStart = DateTime.Now };
Parent p3 = new Parent() { ParentId = 3, ParentName = "Third", RelationshipStart = DateTime.Now };
Child c1 = new Child() { ChildId = 1, ChildName = "First" };
c1.Parents.Add(p1);
c1.Parents.Add(p2);
Child c2 = new Child() { ChildId = 2, ChildName = "Second" };
c2.Parents.Add(p2);
c2.Parents.Add(p3);
h.Children.Add(c1);
h.Children.Add(c2);
Serialize(h, AppContext.BaseDirectory + "Text.xml");
Hierarchy hReadBack = Deserialize(AppContext.BaseDirectory + "Text.xml");
}
Edit : To answer your question
Use these classes
public class Hierarchy
{
public Hierarchy()
{
Children = new List<Child>();
}
public List<Child> Children { get; set; }
private Dictionary<int, string> _parents;
private Dictionary<int, string> _childrenList;
private List<Relationship> _relationships;
private void CalcuateLists()
{
_parents = new Dictionary<int, string>();
_childrenList = new Dictionary<int, string>();
_relationships = new List<Relationship>();
foreach (Child c in this.Children)
{
if (!_childrenList.ContainsKey(c.ChildId))
{
_childrenList.Add(c.ChildId, c.ChildName);
}
foreach (Parent p in c.Parents)
{
if (!_parents.ContainsKey(p.ParentId))
{
_parents.Add(p.ParentId, p.ParentName);
}
if (_relationships.FirstOrDefault(dat => dat.ParentId == p.ParentId && dat.ChildId == c.ChildId) == null)
{
_relationships.Add(new Relationship() { ChildId = c.ChildId, ParentId = p.ParentId, Start = p.RelationshipStart });
}
}
}
}
public Dictionary<int, string> Parents {
get
{
if (_parents == null)
CalcuateLists();
return _parents;
}
}
public Dictionary<int, string> ChildrenList {
get
{
if (_childrenList == null)
CalcuateLists();
return _childrenList;
}
}
public List<Relationship> Relationships {
get
{
if (_relationships == null)
CalcuateLists();
return _relationships;
}
}
}
public class Child
{
public Child()
{
Parents = new List<Parent>();
}
[XmlAttribute("ChildId")]
public int ChildId { get; set; }
[XmlAttribute("ChildName")]
public string ChildName { get; set; }
public List<Parent> Parents { get; set; }
}
public class Parent
{
[XmlAttribute("ParentId")]
public int ParentId { get; set; }
[XmlAttribute("ParentName")]
public string ParentName { get; set; }
[XmlAttribute("RelationshipStart")]
public DateTime RelationshipStart { get; set; }
}
Then your test code becomes
public static void Test()
{
Hierarchy h = new Hierarchy();
Parent p1 = new Parent() { ParentId = 1, ParentName = "First", RelationshipStart = DateTime.Now };
Parent p2 = new Parent() { ParentId = 2, ParentName = "Second", RelationshipStart = DateTime.Now };
Parent p3 = new Parent() { ParentId = 3, ParentName = "Third", RelationshipStart = DateTime.Now };
Child c1 = new Child() { ChildId = 1, ChildName = "First" };
c1.Parents.Add(p1);
c1.Parents.Add(p2);
Child c2 = new Child() { ChildId = 2, ChildName = "Second" };
c2.Parents.Add(p2);
c2.Parents.Add(p3);
h.Children.Add(c1);
h.Children.Add(c2);
Serialize(h, AppContext.BaseDirectory + "Text.xml");
Hierarchy hReadBack = Deserialize(AppContext.BaseDirectory + "Text.xml");
Dictionary<int, string> Parents = hReadBack.Parents;
Dictionary<int, string> Children = hReadBack.ChildrenList;
List<Relationship> Relationships = hReadBack.Relationships;
}
EDIT
To get the results directly without looping
You will need this class
public class Relationship
{
public int ParentId { get; set; }
public int ChildId { get; set; }
public DateTime Start { get; set; }
}
And this selection
// Get a list of child ids and names
Dictionary<int, string> Children = (from c in hReadBack.Children select new { ChildId = c.ChildId, Name = c.ChildName}).ToDictionary(dat => dat.ChildId, dat => dat.Name);
// Get a parent ids and names
Dictionary<int, string> Parents = (from p in hReadBack.Children.SelectMany(i => i.Parents) select new { ParentId = p.ParentId, Name = p.ParentName }).Distinct().ToDictionary(dat => dat.ParentId, dat => dat.Name);
// Get the relationships
List<Relationship> Relationship = (from Child c in hReadBack.Children from Parent p in c.Parents select new Relationship() { ChildId = c.ChildId, ParentId = p.ParentId, Start = p.RelationshipStart }).ToList();

How do i export child objects with EPPlus as Excel

I am using EPPlus to help me export data as excel. I am still learning to export data properly but somehow am stuck at a point where i am not able to export an object with child objects all flatted out.
ParentObject
public string A;
public string B;
public ChildObject ChildObject;
ChildObject
public string C;
public string D;
so i want my exported excel to look like
A B C D
aa1 bb1 cc1 dd1
aa2 bb2 cc2 dd2
aa3 bb3 cc3 dd3
This is how my current implementation looks like
public void CreateExcel(IEnumerable<T> dataCollection, string fullyQualifiedFileName, string worksheetName)
{
using (var package = new ExcelPackage(new FileInfo(fullyQualifiedFileName)))
{
var worksheet =
package.Workbook.Worksheets.FirstOrDefault(excelWorksheet => excelWorksheet.Name == worksheetName) ??
package.Workbook.Worksheets.Add(worksheetName);
var membersToInclude = typeof(T)
.GetMembers(BindingFlags.Instance | BindingFlags.Public)
.Where(p => Attribute.IsDefined(p, typeof(ExcelUtilityIgnoreAttribute)) == false
|| p.GetCustomAttribute<ExcelUtilityIgnoreAttribute>().IsIgnored == false)
.ToArray();
worksheet.Cells["A1"].LoadFromCollection(dataCollection, true, OfficeOpenXml.Table.TableStyles.None,
BindingFlags.Public, membersToInclude);
package.Save();
}
}
I tried using Microsoft generics using expando object but EPPlus wont work with generics, is there a way where in i can export objects with child objects ?
also: is there any other library that i could use ?

There is no native function that could do that. Hard to come up with something generic as it would require a great deal of assumption. What property type should be automatically exported vs what should be treated a child node and have ITS properties exported or expanded. But if you come up with that it is a basic tree traversal from there.
Below is something I adapted from a similar task. Here, I assume that anything that is a either a string or a data type without properties is considered an value type for exporting (int, double, etc.). But it is very easy to tweak as needed. I threw this together so it may not be fully optimized:
public static void ExportFlatExcel<T>(IEnumerable<T> dataCollection, FileInfo file, string worksheetName)
{
using (var package = new ExcelPackage(file))
{
var worksheet =
package.Workbook.Worksheets.FirstOrDefault(excelWorksheet => excelWorksheet.Name == worksheetName) ??
package.Workbook.Worksheets.Add(worksheetName);
const BindingFlags flags = BindingFlags.Instance | BindingFlags.Public;
var props = typeof (T).GetProperties(flags);
//Map the properties to types
var rootTree = new Branch<PropertyInfo>(null);
var stack = new Stack<KeyValuePair<PropertyInfo, IBranch<PropertyInfo>>>(
props
.Reverse()
.Select(pi =>
new KeyValuePair<PropertyInfo, IBranch<PropertyInfo>>(
pi
, rootTree
)
)
);
//Do a non-recursive traversal of the properties
while (stack.Any())
{
var node = stack.Pop();
var prop = node.Key;
var branch = node.Value;
//Print strings
if (prop.PropertyType == typeof (string))
{
branch.AddNode(new Leaf<PropertyInfo>(prop));
continue;
}
//Values type do not have properties
var childProps = prop.PropertyType.GetProperties(flags);
if (!childProps.Any())
{
branch.AddNode(new Leaf<PropertyInfo>(prop));
continue;
}
//Add children to stack
var child = new Branch<PropertyInfo>(prop);
branch.AddNode(child);
childProps
.Reverse()
.ToList()
.ForEach(pi => stack
.Push(new KeyValuePair<PropertyInfo, IBranch<PropertyInfo>>(
pi
, child
)
)
);
}
//Go through the data
var rows = dataCollection.ToList();
for (var r = 0; r < rows.Count; r++)
{
var currRow = rows[r];
var col = 0;
foreach (var child in rootTree.Children)
{
var nodestack = new Stack<Tuple<INode, object>>();
nodestack.Push(new Tuple<INode, object>(child, currRow));
while (nodestack.Any())
{
var tuple = nodestack.Pop();
var node = tuple.Item1;
var currobj = tuple.Item2;
var branch = node as IBranch<PropertyInfo>;
if (branch != null)
{
currobj = branch.Data.GetValue(currobj, null);
branch
.Children
.Reverse()
.ToList()
.ForEach(cnode => nodestack.Push(
new Tuple<INode, object>(cnode, currobj)
));
continue;
}
var leaf = node as ILeaf<PropertyInfo>;
if (leaf == null)
continue;
worksheet.Cells[r + 2, ++col].Value = leaf.Data.GetValue(currobj, null);
if (r == 0)
worksheet.Cells[r + 1, col].Value = leaf.Data.Name;
}
}
}
package.Save();
package.Dispose();
}
}
So say you have these as a structure:
#region Classes
public class Parent
{
public string A { get; set; }
public Child1 Child1 { get; set; }
public string D { get; set; }
public int E { get; set; }
public Child2 Child2 { get; set; }
}
public class Child1
{
public string B { get; set; }
public string C { get; set; }
}
public class Child2
{
public Child1 Child1 { get; set; }
public string F { get; set; }
public string G { get; set; }
}
#endregion
#region Tree Nodes
public interface INode { }
public interface ILeaf<T> : INode
{
T Data { get; set; }
}
public interface IBranch<T> : ILeaf<T>
{
IList<INode> Children { get; }
void AddNode(INode node);
}
public class Leaf<T> : ILeaf<T>
{
public Leaf() { }
public Leaf(T data) { Data = data; }
public T Data { get; set; }
}
public class Branch<T> : IBranch<T>
{
public Branch(T data) { Data = data; }
public T Data { get; set; }
public IList<INode> Children { get; } = new List<INode>();
public void AddNode(INode node)
{
Children.Add(node);
}
}
#endregion
And this as a test:
[TestMethod]
public void ExportFlatTest()
{
var list = new List<Parent>();
for (var i = 0; i < 20; i++)
list.Add(new Parent
{
A = $"A-{i}",
D = $"D-{i}",
E = i*10,
Child1 = new Child1
{
B = $"Child1-B-{i}",
C = $"Child1-C-{i}",
},
Child2 = new Child2
{
F = $"F-{i}",
G = $"G-{i}",
Child1 = new Child1
{
B = $"Child2-Child1-B-{i}",
C = $"Child2-Child1-C-{i}",
}
}
});
var file = new FileInfo(#"c:\temp\flat.xlsx");
if (file.Exists)
file.Delete();
TestExtensions.ExportFlatExcel(
list
, file
, "Test1"
);
}
Will give you this:

using linq and c# to create a nested list object from a flat list object

I have a question. It's about linq in combination with c#.
I want to create a tree structure from a flatten structure in a pre defined object structure.
The following code which I've got work, but both are not exactly what i want.
In linq:
var result = listAgenderingen.GroupBy(records => records.Agnnummer)
.Select(group => new { AgnNummer = group.Key, Items = group.ToList()}).ToList();
the issue is that this does not result in the object I want.
So I've rewritten this to the following code
List<string> test = listAgenderingen.Select(x => x.Agnnummer).Distinct().ToList();
foreach (var item in test)
{
List<Agendering> listAgendering = listAgenderingen.Where(agend => agend.Agnnummer == item).OrderBy(ord => ord.Agnnummer).ToList();
AgnAgendering AgnAgendering = new AgnAgendering() {AgnNummer =item, Agenderingen = listAgendering };
}
this code actually works correct. but for 200000 records, it's taking a lot of time while the original linq takes a few seconds.
my question is can the linq be rewritten so it will create or convert to the richt object?
the structure of the classes:
public class Agendering
{
public int AgnID { get; set; }
public string Agnnummer { get; set; }
}
public class AgnAgendering
{
public string AgnNummer { get; set; }
public List<Agendering> Agenderingen { get; set; }
}
I hope someone has a sollution.

If I understand correctly, you want:
var result = listAgenderingen.GroupBy(records => records.Agnnummer)
.Select(group => new AgnAgendering { AgnNummer = group.Key, Agenderingen = group.ToList()}).ToList();

Your properties naming makes it absolutely unreadable and unclear.
Assuming that you have a flat structure like:
public class Item
{
public int ID { get; set; }
public int? ParentID { get; set; }
}
and you want a tree-like structure:
public class TreeItem
{
public int ID { get; set; }
public TreeItem Parent { get; set; }
public List<TreeItem> Children { get; set; }
public TreeItem(int id)
{
ID = id;
Children = new List<TreeItem>();
}
public TreeItem(int id, TreeItem parent) : this(id)
{
Parent = parent;
}
}
You can do most optimally in O(n) using Dictionary:
Item[] items = ...;
Dictionary<int, TreeItem> result = new Dictionary<int, TreeItem>();
foreach (var item in items.OrderBy(x => x.ParentID ?? -1))
{
TreeItem current;
if (item.ParentID.HasValue)
{
TreeItem parent = result[item.ParentID]; // guaranteed to exist due to order
current = new TreeItem(item.ID, parent);
parent.Children.Add(current);
} else {
current = new TreeItem(item.ID);
}
}
TreeItem[] treeItems = result.Values.ToArray();

Convert list of items to list of tree nodes that have children

I have the following class
public class Item
{
public int Id { get; set; }
public int? ParentId { get; set; }
public string Text { get; set; }
}
And the following class for tree view
public class TreeViewModel
{
public TreeViewModel()
{
this.Children = new List<TreeViewModel>();
}
public int Id { get; set; }
public int NodeId { get; set; }
public string Text { get; set; }
public bool Expanded { get; set; }
public bool Checked { get; set; }
public bool HasChildren
{
get { return Children.Any(); }
}
public IList<TreeViewModel> Children { get; private set; }
}
I will receive list of items and I would to convert it to tree.
the item that not have parent id it will the main node.
example: if i have the following items
item[0] = Id:0 Text:User ParentId:3
item[1] = Id:1 Text:Role ParentId:3
item[2] = Id:2 Text:SubUser ParentId:0
item[3] = Id:3 Text:Admin ParentId:null
item[4] = Id:4 Text:SuperAdmin ParentId:null
item[5] = Id:5 Text:Doha ParentId:4
the following item it will list of tree
I tried to make recursive function to do that , but i have no result

You don't need a recursive function to do this:
var models = items.Select(i => new TreeViewModel
{
Id = i.Id,
...
}).ToList();
foreach (var model in models){
model.Children.AddRange(models.Where(m => m.ParentId == model.Id));
}
If you then want to get the roots of your tree, you can use:
var roots = models.Where(m => !m.ParentId.HasValue);

Here is a fast O(N) time complexity method of doing that:
List<Item> list = ...;
// Pre create all nodes and build map by Id for fast lookup
var nodeById = list
.Select(item => new TreeViewModel { Id = item.Id, Text = item.Text })
.ToDictionary(item => item.Id);
// Build hierarchy
var tree = new List<TreeViewModel>();
foreach (var item in list)
{
var nodeList = item.ParentId == null ? tree, nodeById[item.ParentId.Value].Children;
nodeList.Add(nodeById[item.Id]);
}

Linq At least one object must implement IComparable

I am trying to order a List of Entities that contains another list of Entities. I have implemented IComparable for all entities and still get the exception. All of the examples I have seen address the issue where you have one list and you order by a given field in that list but not where you have a list of lists. This issue is happening for Linq to Objects per below and also for Linq to Entities. What am I missing?
[TestClass]
public class OrderBy
{
[TestMethod]
public void OrderByTest()
{
var hobbies = new Collection<Hobby> { new Hobby { HobbyId = 1, Name = "Eating" }, new Hobby() { HobbyId = 2, Name = "Breathing" } };
var p1 = new Person
{
PersonId = 1,
Name = "A",
PersonHobbies = new Collection<PersonHobby> { new PersonHobby() { PersonHobbyId = 1}}
};
var p2 = new Person
{
PersonId = 2,
Name = "Z",
PersonHobbies = new Collection<PersonHobby> { new PersonHobby() { PersonHobbyId = 2 }}
};
var people = new List<Person> { p1, p2 };
var pplEnumerable = people.AsEnumerable();
pplEnumerable = pplEnumerable.OrderByDescending(r => r.PersonHobbies.OrderByDescending(p => p.Hobby.Name));
foreach (var person in pplEnumerable)
{
Console.WriteLine(person.Name);
}
}
public class Person : IComparable
{
public int PersonId { get; set; }
public string Name { get; set; }
public virtual ICollection<PersonHobby> PersonHobbies { get; set; }
public int CompareTo(object obj)
{
if (obj == null) return 1;
var otherPerson = obj as Person;
return PersonId.CompareTo(otherPerson.PersonId);
}
}
public class PersonHobby : IComparable
{
public int PersonHobbyId { get; set; }
public int HobbyId { get; set; }
public virtual Person Person{ get; set; }
public int PersonId { get; set; }
public virtual Hobby Hobby { get; set; }
public int CompareTo(object obj)
{
if (obj == null) return 1;
var otherPersonHobby = obj as PersonHobby;
return PersonHobbyId.CompareTo(otherPersonHobby.PersonHobbyId);
}
}
public class Hobby : IComparable
{
public int HobbyId { get; set; }
public string Name { get; set; }
public int CompareTo(object obj)
{
if (obj == null) return 1;
var otherHobby = obj as Hobby;
return HobbyId.CompareTo(otherHobby.HobbyId);
}
}
}

You cannot apply ordering to lists by default. You need to write up a custom class (sort of EquatableList etc.) or use LINQ Except & Intersect operators to compare lists.
But based on your comment, if you're looking for the LINQ equivalent of:
select * from Person p join PersonHobby ph
on ph.PersonId = p.PersonId join Hobby h
on h.HobbyId = ph.HobbyId order by h.Name
then that can be achieved as:
var query = people.SelectMany(p => p.PersonHobbies)
.Join(hobbies, ph => ph.HobbyId, h => h.HobbyId,
(ph, h) => new
{
Person = ph.Person, PersonHobby = ph, Hobby = h
})
.OrderBy(r => r.Hobby.Name);
basically we join person, person hobbies and hobby on the keys, and project all columns and sort it by the hobby.name field, as mentioned in your SQL.