Develop Reference

Compare CSV Header to Map Class

I have a process whereby we have written a class to import a large (ish) CSV into our app using CsvHelper (https://joshclose.github.io/CsvHelper).
I would like to compare the header to the Map to ensure the header's integrity. We get the CSV file from a 3rd party and I want to ensure it doesn't change over time and thought the best way to do this would be to compare it against the map.
We have a class set up as so (trimmed):
public class VisitExport
{
public int? Count { get; set; }
public string CustomerName { get; set; }
public string CustomerAddress { get; set; }
}
And its corresponding map (also trimmed):
public class VisitMap : ClassMap<VisitExport>
{
public VisitMap()
{
Map(m => m.Count).Name("Count");
Map(m => m.CustomerName).Name("Customer Name");
Map(m => m.CustomerAddress).Name("Customer Address");
}
}
This is the code I have for reading the CSV file and it works great. I have a try catch in place for the error but ideally, if it fails specifically for a header miss match, I'd like to handle that specifically.
private void fileLoadedLink_LinkClicked(object sender, LinkLabelLinkClickedEventArgs e)
{
try
{
var filePath = string.Empty;
data = new List<VisitExport>();
using (OpenFileDialog openFileDialog = new OpenFileDialog())
{
openFileDialog.InitialDirectory = new KnownFolder(KnownFolderType.Downloads).Path;
openFileDialog.Filter = "csv files (*.csv)|*.csv";
openFileDialog.FilterIndex = 2;
openFileDialog.RestoreDirectory = true;
if (openFileDialog.ShowDialog() == DialogResult.OK)
{
filePath = openFileDialog.FileName;
var fileStream = openFileDialog.OpenFile();
var culture = CultureInfo.GetCultureInfo("en-GB");
using (StreamReader reader = new StreamReader(fileStream))
using (var readCsv = new CsvReader(reader, culture))
{
var map = new VisitMap();
readCsv.Context.RegisterClassMap(map);
var fileContent = readCsv.GetRecords<VisitExport>();
data = fileContent.ToList();
fileLoadedLink.Text = filePath;
viewModel.IsFileLoaded = true;
}
}
}
}
catch (CsvHelperException ex)
{
Console.WriteLine(ex.InnerException != null ? ex.InnerException.Message : ex.Message);
fileLoadedLink.Text = "Error loading file.";
viewModel.IsFileLoaded = false;
}
}
Is there a way of comparing the Csv header vs my map?

There are two basic cases for CSV files with headers: missing CSV columns, and extra CSV columns. The first is already detected by CsvHelper while the detection of the second is not implemented out of the box and requires subclassing of CsvReader.
(As CsvHelper maps CSV columns to model properties by name, permuting the order of the columns in the CSV file would not be considered a breaking change.)
Note that this only applies to CSV files that actually contain headers. Since you are not setting CsvConfiguration.HasHeaderRecord = false I assume that this applies to your use case.
Details about each of the two cases follow.
Missing CSV columns.
Currently CsvHelper already throws an exception by default in such situations. When unmapped data model properties are found, CsvConfiguration.HeaderValidated is invoked. By default this is set to ConfigurationFunctions.HeaderValidated whose current behavior is to throw a HeaderValidationException if there are any unmapped model properties. You can replace or extend HeaderValidated with logic of your own if you prefer:
var culture = CultureInfo.GetCultureInfo("en-GB");
var config = new CsvConfiguration (culture)
{
HeaderValidated = (args) =>
{
// Add additional logic as required here
ConfigurationFunctions.HeaderValidated(args);
},
};
using (var readCsv = new CsvReader(reader, config))
{
// Remainder unchanged
Demo fiddle #1 here.
Extra CSV columns.
Currently CsvHelper does not inform the application when this happens. See Throw if csv contains unexpected columns #1032 which confirms that this is not implemented out of the box.
In a GitHub comment, user leopignataro suggests a workaround, which is to subclass CsvReader and add the necessary validation logic oneself. However the version shown in the comment doesn't seem to handle duplicated column names or embedded references. The following subclass of CsvHelper should do this correctly. It is based on the logic in CsvReader.ValidateHeader(ClassMap map, List<InvalidHeader> invalidHeaders). It recursively walks the incoming ClassMap, attempts to find a CSV header corresponding to each member or constructor parameter, and flags the index of each one that is mapped. Afterwards, if there are any unmapped headers, the supplied Action<CsvContext, List<string>> OnUnmappedCsvHeaders is invoked to notify the application of the problem and throw some exception if desired:
public class ValidatingCsvReader : CsvReader
{
public ValidatingCsvReader(TextReader reader, CultureInfo culture, bool leaveOpen = false) : this(new CsvParser(reader, culture, leaveOpen)) { }
public ValidatingCsvReader(TextReader reader, CsvConfiguration configuration) : this(new CsvParser(reader, configuration)) { }
public ValidatingCsvReader(IParser parser) : base(parser) { }
public Action<CsvContext, List<string>> OnUnmappedCsvHeaders { get; set; }
public override void ValidateHeader(Type type)
{
base.ValidateHeader(type);
var headerRecord = HeaderRecord;
var mapped = new BitArray(headerRecord.Length);
var map = Context.Maps[type];
FlagMappedHeaders(map, mapped);
var unmappedHeaders = Enumerable.Range(0, headerRecord.Length).Where(i => !mapped[i]).Select(i => headerRecord[i]).ToList();
if (unmappedHeaders.Count > 0)
{
OnUnmappedCsvHeaders?.Invoke(Context, unmappedHeaders);
}
}
protected virtual void FlagMappedHeaders(ClassMap map, BitArray mapped)
{
// Logic adapted from https://github.com/JoshClose/CsvHelper/blob/0d753ff09294b425e4bc5ab346145702eeeb1b6f/src/CsvHelper/CsvReader.cs#L157
// By https://github.com/JoshClose
foreach (var parameter in map.ParameterMaps)
{
if (parameter.Data.Ignore)
continue;
if (parameter.Data.IsConstantSet)
// If ConvertUsing and Constant don't require a header.
continue;
if (parameter.Data.IsIndexSet && !parameter.Data.IsNameSet)
// If there is only an index set, we don't want to validate the header name.
continue;
if (parameter.ConstructorTypeMap != null)
{
FlagMappedHeaders(parameter.ConstructorTypeMap, mapped);
}
else if (parameter.ReferenceMap != null)
{
FlagMappedHeaders(parameter.ReferenceMap.Data.Mapping, mapped);
}
else
{
var index = GetFieldIndex(parameter.Data.Names.ToArray(), parameter.Data.NameIndex, true);
if (index >= 0)
mapped.Set(index, true);
}
}
foreach (var memberMap in map.MemberMaps)
{
if (memberMap.Data.Ignore || !CanRead(memberMap))
continue;
if (memberMap.Data.ReadingConvertExpression != null || memberMap.Data.IsConstantSet)
// If ConvertUsing and Constant don't require a header.
continue;
if (memberMap.Data.IsIndexSet && !memberMap.Data.IsNameSet)
// If there is only an index set, we don't want to validate the header name.
continue;
var index = GetFieldIndex(memberMap.Data.Names.ToArray(), memberMap.Data.NameIndex, true);
if (index >= 0)
mapped.Set(index, true);
}
foreach (var referenceMap in map.ReferenceMaps)
{
if (!CanRead(referenceMap))
continue;
FlagMappedHeaders(referenceMap.Data.Mapping, mapped);
}
}
}
And then in your code, handle the OnUnmappedCsvHeaders callback however you would like, such as by throwing a CsvHelperException or some other custom exception:
using (var readCsv = new ValidatingCsvReader(reader, culture)
{
OnUnmappedCsvHeaders = (context, headers) => throw new CsvHelperException(context, string.Format("Unmapped CSV headers: \"{0}\"", string.Join(",", headers))),
})
Demo fiddles:
#2 (your model).
#3 (with external references).
#4 (duplicate names).
#5 (using the auto-generated map).
This could use additional testing, e.g. for data models with parameterized constructors and additional, mutable properties.

How about catching HeaderValidationException before catching CsvHelperException
catch (HeaderValidationException ex)
{
var message = ex.Message.Split('\n')[0];
var currentHeader = ex.Context.Reader.HeaderRecord;
message += $"{Environment.NewLine}Header: \"{string.Join(",", currentHeader)}\"";
Console.WriteLine(message);
fileLoadedLink.Text = "Error loading file.";
viewModel.IsFileLoaded = false;
}
catch (CsvHelperException ex)
{
Console.WriteLine(ex.InnerException != null ? ex.InnerException.Message : ex.Message);
fileLoadedLink.Text = "Error loading file.";
viewModel.IsFileLoaded = false;
}

Openxml inserts italics at bookmarks.(several characters in a string are italics)

Here is the code for my work.
public void InsertValue(WordprocessingDocument doc, string bookMark, string txt)
{
try
{
RemoveBookMarkContent(doc, bookMark);
var bmStart = FindBookMarkStart(doc, bookMark);
if (bmStart == null)
return;
var run = new Run();
run.Append(GetRunProperties());
run.Append(new Text(txt));
bmStart.Parent.InsertAfter(run, bmStart);
}
catch (Exception c)
{
//not Exception
}
}
private void RemoveBookMarkContent(WordprocessingDocument doc, string bmName)
{
BookmarkStart bmStart = FindBookMarkStart(doc, bmName);
if (bmStart == null)
return;
BookmarkEnd bmEnd = FindBookMarkEnd(doc, bmStart.Id);
while (true)
{
var run = bmStart.NextSibling();
if (run == null)
{
break;
}
if (run is BookmarkEnd && (BookmarkEnd)run == bmEnd)
{
break;
}
run.Remove();
}
}
There are still several auxiliary classes not written.Work process, first find the bookmark location, delete the content of the bookmark location, and then add it.I've also tried to add one Paragraph to the bookmark location.But that doesn't work.
Document to insert in bookmark eg:露点：U=0.15℃（k=2）；相对湿度：U=1.0%RH(k=2).Both u and K must be italics.Any help will be appreciated.Thanks.

I tried a new component.[Spire.Office.][1]
At the beginning, I didn't think of a solution, but I used the global search and replacement to determine whether the search location has bookmarks, which perfectly solved the problem.
Here is the code for my work.
var selection = document.FindAllString("U", false, true);
foreach (var sec in selection)
{
var t = sec.GetAsOneRange();
if (sec.GetAsOneRange()?.Owner?.LastChild?.DocumentObjectType == DocumentObjectType.BookmarkEnd)
{
sec.GetAsOneRange().CharacterFormat.Italic = true;
}
}
I didn't try to do this with openxml, but I think the principle should be consistent.
[1]: https://www.e-iceblue.cn/Buy/Spire-PDF-NET.html

Printing listbox Items in order of occurence

I have an application that I have built and is just sort of printing app.
First I populate ListBox and format it then goto c:\somefolder
in some folder are pdf files it looks on listbox and see if there is any file name that matches an item on the listbox and sends it to the default printer.Everything is working great ,but say I have JohnDoe.pdf,ShahRukh.pdf,Vijay.pdf
how can I make it to send the files in that order, for now it works great ,but I want to be able to print ==> JohnDoe.pdf first, then ShahRukh.pdf and so on and so forth.Please if you have any idea to spare is much welcome.
Thanks in advance. This what I have right now it works great but print all items that match but randomly .I want it to respond or print matches in order of occurrence from top to bottom.
public class Pdf
{
public static Boolean PrintPDFs(string pdfFileName)
{
try
{
ProcessStartInfo stackOverflowHelp = new ProcessStartInfo();
stackOverflowHelp.Verb = "print";
stackOverflowHelp.FileName = pdfFileName;
stackOverflowHelp.CreateNoWindow = true;
stackOverflowHelp.WindowStyle = ProcessWindowStyle.Hidden;
Process gamingBoy = new Process();
gamingBoy.StartInfo = stackOverflowHelp;
gamingBoy.Start();
gamingBoy.WaitForInputIdle();
if (gamingBoy.HasExited == false)
{
gamingBoy.WaitForExit(20000);
//return true;
}
// System.Threading.Thread.Sleep(1000);
gamingBoy.EnableRaisingEvents = true;
gamingBoy.Close();
// return true;
//proc.Close();
//KillAdobe("AcroRd32");
return true;
}
catch
{
return false;
}
}
private void button2_Click(object sender, EventArgs e)
{
string dir = #"C:\slim\slimyyyy";//
if (Directory.Exists(dir))//If a directory defined above exists then do the followings
{
string[] pdf_Files = Directory.GetFiles(dir);
if (pdf_Files.Length > 0)
{
foreach (string file in pdf_Files)
{
string fileName = Path.GetFileName(file);
foreach (object item in listBox1.Items)
{
if (fileName == line.ToString())
{
Pdf.PrintPDFs((file));
}
}
}
}
}
}

Just reverse the order of your foreach loops:
foreach (object item in listBox1.Items)
{
foreach (string file in pdf_Files)
{
string fileName = Path.GetFileName(f);
if (fileName == line.ToString())
{
Pdf.PrintPDFs((file));
}
}
}

The easy answer is to change the order of your loops. That is, change your loop to:
foreach (var item in listBox1.Items)
{
var fname = item.ToString();
if (pdf_files.Contains(fname))
{
Pdf.PrintPDFs(fname);
}
}

Refresing the C# DataGrid from a background thread

I'm using a DataGrid bound to an ObservableCollection source to display two columns, a file name and a number that I get from analysing the file.
ObservableCollection<SearchFile> fileso;
//...
private void worker_DoWork(object sender, DoWorkEventArgs e)
{
/* Get all the files to check. */
int dirCount = searchFoldersListView.Items.Count;
List<string> allFiles = new List<string>();
for(int i = 0; i < dirCount; i++)
{
try
{
allFiles.AddRange(Directory.GetFiles(searchFoldersListView.Items[i].ToString(), "*.txt").ToList());
allFiles.AddRange(Directory.GetFiles(searchFoldersListView.Items[i].ToString(), "*.pdf").ToList());
}
catch
{ /* stuff */ }
}
/* Clear the collection and populate it with unchecked files again, refreshing the grid. */
this.Dispatcher.Invoke(new Action(delegate
{
fileso.Clear();
foreach(var file in allFiles)
{
SearchFile sf = new SearchFile() { path=file, occurrences=0 };
fileso.Add(sf);
}
}));
/* Check the files. */
foreach(var file in allFiles)
{
this.Dispatcher.Invoke(new Action(delegate
{
int occurences;
bool result = FileSearcher.searchFile(file, searchTermTextBox.Text, out occurences);
fileso.AddOccurrences(file, occurences); // This is an extension method that alters the collection by finding the relevant item and changing it.
}));
}
}
//...
public static void AddOccurrences(this ObservableCollection<SearchFile> collection, string path, int occurrences)
{
for(int i = 0; i < collection.Count; i++)
{
if(collection[i].path == path)
{
collection[i].occurrences = occurrences;
break;
}
}
}
//...
public static bool searchTxtFile(string path, string term, out int occurences)
{
string contents = File.ReadAllText(path);
occurences = Regex.Matches(contents, term, RegexOptions.IgnoreCase).Count;
if(occurences>0)
return true;
return false;
}
public static bool searchDocxFile(string path, string term, out int occurences)
{
occurences = 0;
string tempPath = Path.GetTempPath();
string rawName = Path.GetFileNameWithoutExtension(path);
string destFile = System.IO.Path.Combine(tempPath, rawName + ".zip");
System.IO.File.Copy(path, destFile, true);
using(ZipFile zf = new ZipFile(destFile))
{
ZipEntry ze = zf.GetEntry("word/document.xml");
if(ze != null)
{
using(Stream zipstream = zf.GetInputStream(ze))
{
using(StreamReader sr = new StreamReader(zipstream))
{
string docContents = sr.ReadToEnd();
string rawText = Extensions.StripTagsRegexCompiled(docContents);
occurences = Regex.Matches(rawText, term, RegexOptions.IgnoreCase).Count;
if(occurences>0)
return true;
return false;
}
}
}
}
return false;
}
public static bool searchFile(string path, string term, out int occurences)
{
occurences = 0;
string ext = System.IO.Path.GetExtension(path);
switch(ext)
{
case ".txt":
return searchTxtFile(path, term, out occurences);
//case ".doc":
// return searchDocFile(path, term, out occurences);
case ".docx":
return searchDocxFile(path, term, out occurences);
}
return false;
}
But the problem is that sometimes, when I hit the update button (which starts the worker with the the do_work method above), some of the time, I'm getting random zeroes in the number column instead of the correct number. Why is that? I'm assuming it's because there's some problem with updating the number column twice, with sometimes the first zeroing getting applied after the actual update, but I'm not sure about the details.

I think this is a case of an access to a modified closure
/* Check the files. */
foreach(var file in allFiles)
{
var fileTmp = file; // avoid access to modified closure
this.Dispatcher.Invoke(new Action(delegate
{
int occurences;
bool result = FileSearcher.searchFile(fileTmp, searchTermTextBox.Text, out occurences);
fileso.AddOccurrences(fileTmp, occurences); // This is an extension method that alters the collection by finding the relevant item and changing it.
}));
}
Basically what's happening is that you are passing the file variable to a lambda expression, but file will be modified by the foreach loop before the action is actually invoked, using a temp variable to hold file should solve this.

Reading a line from a streamreader without consuming?

Is there a way to read ahead one line to test if the next line contains specific tag data?
I'm dealing with a format that has a start tag but no end tag.
I would like to read a line add it to a structure then test the line below to make sure it not a new "node" and if it isn't keep adding if it is close off that struct and make a new one
the only solution i can think of is to have two stream readers going at the same time kinda suffling there way along lock step but that seems wastefull (if it will even work)
i need something like peek but peekline

The problem is the underlying stream may not even be seekable. If you take a look at the stream reader implementation it uses a buffer so it can implement TextReader.Peek() even if the stream is not seekable.
You could write a simple adapter that reads the next line and buffers it internally, something like this:
public class PeekableStreamReaderAdapter
{
private StreamReader Underlying;
private Queue<string> BufferedLines;
public PeekableStreamReaderAdapter(StreamReader underlying)
{
Underlying = underlying;
BufferedLines = new Queue<string>();
}
public string PeekLine()
{
string line = Underlying.ReadLine();
if (line == null)
return null;
BufferedLines.Enqueue(line);
return line;
}
public string ReadLine()
{
if (BufferedLines.Count > 0)
return BufferedLines.Dequeue();
return Underlying.ReadLine();
}
}

You could store the position accessing StreamReader.BaseStream.Position, then read the line next line, do your test, then seek to the position before you read the line:
// Peek at the next line
long peekPos = reader.BaseStream.Position;
string line = reader.ReadLine();
if (line.StartsWith("<tag start>"))
{
// This is a new tag, so we reset the position
reader.BaseStream.Seek(pos);
}
else
{
// This is part of the same node.
}
This is a lot of seeking and re-reading the same lines. Using some logic, you may be able to avoid this altogether - for instance, when you see a new tag start, close out the existing structure and start a new one - here's a basic algorithm:
SomeStructure myStructure = null;
while (!reader.EndOfStream)
{
string currentLine = reader.ReadLine();
if (currentLine.StartsWith("<tag start>"))
{
// Close out existing structure.
if (myStructure != null)
{
// Close out the existing structure.
}
// Create a new structure and add this line.
myStructure = new Structure();
// Append to myStructure.
}
else
{
// Add to the existing structure.
if (myStructure != null)
{
// Append to existing myStructure
}
else
{
// This means the first line was not part of a structure.
// Either handle this case, or throw an exception.
}
}
}

Why the difficulty? Return the next line, regardless. Check if it is a new node, if not, add it to the struct. If it is, create a new struct.
// Not exactly C# but close enough
Collection structs = new Collection();
Struct struct;
while ((line = readline()) != null)) {
if (IsNode(line)) {
if (struct != null) structs.add(struct);
struct = new Struct();
continue;
}
// Whatever processing you need to do
struct.addLine(line);
}
structs.add(struct); // Add the last one to the collection
// Use your structures here
foreach s in structs {
}

Here is what i go so far. I went more of the split route than the streamreader line by line route.
I'm sure there are a few places that are dieing to be more elegant but for right now it seems to be working.
Please let me know what you think
struct INDI
{
public string ID;
public string Name;
public string Sex;
public string BirthDay;
public bool Dead;
}
struct FAM
{
public string FamID;
public string type;
public string IndiID;
}
List<INDI> Individuals = new List<INDI>();
List<FAM> Family = new List<FAM>();
private void button1_Click(object sender, EventArgs e)
{
string path = #"C:\mostrecent.ged";
ParseGedcom(path);
}
private void ParseGedcom(string path)
{
//Open path to GED file
StreamReader SR = new StreamReader(path);
//Read entire block and then plit on 0 # for individuals and familys (no other info is needed for this instance)
string[] Holder = SR.ReadToEnd().Replace("0 #", "\u0646").Split('\u0646');
//For each new cell in the holder array look for Individuals and familys
foreach (string Node in Holder)
{
//Sub Split the string on the returns to get a true block of info
string[] SubNode = Node.Replace("\r\n", "\r").Split('\r');
//If a individual is found
if (SubNode[0].Contains("INDI"))
{
//Create new Structure
INDI I = new INDI();
//Add the ID number and remove extra formating
I.ID = SubNode[0].Replace("#", "").Replace(" INDI", "").Trim();
//Find the name remove extra formating for last name
I.Name = SubNode[FindIndexinArray(SubNode, "NAME")].Replace("1 NAME", "").Replace("/", "").Trim();
//Find Sex and remove extra formating
I.Sex = SubNode[FindIndexinArray(SubNode, "SEX")].Replace("1 SEX ", "").Trim();
//Deterine if there is a brithday -1 means no
if (FindIndexinArray(SubNode, "1 BIRT ") != -1)
{
// add birthday to Struct
I.BirthDay = SubNode[FindIndexinArray(SubNode, "1 BIRT ") + 1].Replace("2 DATE ", "").Trim();
}
// deterimin if there is a death tag will return -1 if not found
if (FindIndexinArray(SubNode, "1 DEAT ") != -1)
{
//convert Y or N to true or false ( defaults to False so no need to change unless Y is found.
if (SubNode[FindIndexinArray(SubNode, "1 DEAT ")].Replace("1 DEAT ", "").Trim() == "Y")
{
//set death
I.Dead = true;
}
}
//add the Struct to the list for later use
Individuals.Add(I);
}
// Start Family section
else if (SubNode[0].Contains("FAM"))
{
//grab Fam id from node early on to keep from doing it over and over
string FamID = SubNode[0].Replace("# FAM", "");
// Multiple children can exist for each family so this section had to be a bit more dynaimic
// Look at each line of node
foreach (string Line in SubNode)
{
// If node is HUSB
if (Line.Contains("1 HUSB "))
{
FAM F = new FAM();
F.FamID = FamID;
F.type = "PAR";
F.IndiID = Line.Replace("1 HUSB ", "").Replace("#","").Trim();
Family.Add(F);
}
//If node for Wife
else if (Line.Contains("1 WIFE "))
{
FAM F = new FAM();
F.FamID = FamID;
F.type = "PAR";
F.IndiID = Line.Replace("1 WIFE ", "").Replace("#", "").Trim();
Family.Add(F);
}
//if node for multi children
else if (Line.Contains("1 CHIL "))
{
FAM F = new FAM();
F.FamID = FamID;
F.type = "CHIL";
F.IndiID = Line.Replace("1 CHIL ", "").Replace("#", "");
Family.Add(F);
}
}
}
}
}
private int FindIndexinArray(string[] Arr, string search)
{
int Val = -1;
for (int i = 0; i < Arr.Length; i++)
{
if (Arr[i].Contains(search))
{
Val = i;
}
}
return Val;
}