Develop Reference

Fastest way to fuzzy match two csv files

I have written a very simple program using a nuget package in c# to read in 2 csv files and fuzzy match them and output a new csv file with all the matches. The problem is i need the program to be able to read and compare files up to 700k and comparw it to 100k. I havent been able to find a way to speed up the process. Is there any way i can do this? I will even use another language if need be.
you can ignore all the commented code its just there for when i was using it for testing purposes. sorry im a newer programmer.
the read csv funciton is for reading in the csv. the rest is code inside another function where i pass in the string arrays to pass them through fuzzymatch
static string[] ReadCSV(string path)
{
List<string> name = new List<string>();
List<string> address = new List<string>();
List<string> city = new List<string>();
List<string> state = new List<string>();
List<string> zip = new List<string>();
using (var reader = new StreamReader(path))
{
reader.ReadLine();
while (!reader.EndOfStream)
{
var line = reader.ReadLine();
var values = line.Split(',');
name.Add(values[0] +", "+ values[1]);
//address.Add(values[1]);
//city.Add(values[2]);
//state.Add(values[3]);
//zip.Add(values[4]);
}
}
string[] name1 = name.ToArray();
return name1;
//foreach (var item in name)
//{
// Console.WriteLine(item.ToString());
//}
}
StringBuilder csvcontent = new StringBuilder();
string csvpath = #"C:\Users\bigel\Documents\outputtest.csv";
csvcontent.AppendLine("Name,Address,Match");
//Console.WriteLine("Levenshtein Edit Distance:");
int x = 1;
foreach (var name in string1)
{
for (int i = 0; i < length; i++)
{
int leven = match[i].LevenshteinDistance(name);
//Console.WriteLine(match[i] + "\t{0} against {1}", leven, name);
if (leven <= 7)
{
output[i] = input[i] + ",match";
csvcontent.AppendLine(output[i]);
//Console.WriteLine(match[i] + " " + leven + " against " + name + " is a Match");
//Console.WriteLine(output[i]);
}
else
{
if (i == 500)
{
Console.WriteLine(x);
x++;
}
}
}
}
File.AppendAllText(csvpath, csvcontent.ToString());

Comparing rows in a csv file

I am currently using the below code to compare two csv files with each other. This code gives an output with all the rows that are not the same. But when a row is missing everything after that row is not the same. How can I fix this? Thanks in advance.
List<string> lines = new List<string>();
List<string> lines2 = new List<string>();
try
{
StreamReader reader = new StreamReader(System.IO.File.OpenRead(file1));
StreamReader read = new StreamReader(System.IO.File.OpenRead(file2));
List<string> differences = new List<string>();
string line;
string line2;
int i = 0;
while ((line = reader.ReadLine()) != null && (line2 = read.ReadLine()) != null)
{
string[] split = line.Split(Convert.ToChar("\t"));
string[] split2 = line2.Split(Convert.ToChar("\t"));
if (split[i] != split2[i])
{
differences.Add("this row is not the same:, " + line);
}
else
{
}
i++;
}
System.IO.File.WriteAllLines(differencesFile, differences);
reader.Dispose();
read.Dispose();
}
catch
{
}

After help from a friend I made it work with this code:
List<string> file1 = new List<string>();
List<string> output = new List<string>();
string differencesFile = path;
File.WriteAllText(differencesFile, "");
try
{
StreamReader readFile1 = new StreamReader(System.IO.File.OpenRead(pathfile1));
string lineFile1;
while ((lineFile1 = readFile1.ReadLine()) != null)
{
bool match = false;
string[] colums = lineFile1.Split('\t');
StreamReader readFile2 = new StreamReader(System.IO.File.OpenRead(pathfile2));
string line2;
while ((line2 = readFile2.ReadLine()) != null)
{
string[] columsFile2 = line2.Split('\t');
if (colums[0] == columsFile2[0])
{
match = true;
}
}
if (!match)
{
output.Add(colums[0] + "; doesnt exist in pathfile2");
}
}
System.IO.File.WriteAllLines(differencesFile, output);
}
catch { }

StreamWriter: Starting and ending on a specific line number

I would like to ask some tips and help on a reading/writing part of my C#.
Situation:
I have to read a CSV file; - OK
If the CSV file name starts with "Load_", I want to write on another CSV the data from line 2 to the last one;
If the CSV file name starts with "RO_", I want to write on 2 different CSVs, 1 with the line 1 to 4 and the other 4 to the last one;
What I have so far is:
public static void ProcessFile(string[] ProcessFile)
{
// Keeps track of your current position within a record
int wCurrLine = 0;
// Number of rows in the file that constitute a record
const int LINES_PER_ROW = 1;
int ctr = 0;
foreach (string filename in ProcessFile)
{
var sbText = new System.Text.StringBuilder(100000);
int stop_line = 0;
int start_line = 0;
// Used for the output name of the file
var dir = Path.GetDirectoryName(filename);
var fileName = Path.GetFileNameWithoutExtension(filename);
var ext = Path.GetExtension(filename);
var folderbefore = Path.GetFullPath(Path.Combine(dir, #"..\"));
var lineCount = File.ReadAllLines(#filename).Length;
string outputname = folderbefore + "output\\" + fileName;
using (StreamReader Reader = new StreamReader(#filename))
{
if (filename.Contains("RO_"))
{
start_line = 1;
stop_line = 5;
}
else
{
start_line = 2;
stop_line = lineCount;
}
ctr = 0;
while (!Reader.EndOfStream && ctr < stop_line)
{
// Add the text
sbText.Append(Reader.ReadLine());
// Increment our current record row counter
wCurrLine++;
// If we have read all of the rows for this record
if (wCurrLine == LINES_PER_ROW)
{
// Add a line to our buffer
sbText.AppendLine();
// And reset our record row count
wCurrLine = 0;
}
ctr++;
} // end of the while
}
int total_lenght = sbText.Length
// When all of the data has been loaded, write it to the text box in one fell swoop
using (StreamWriter Writer = new StreamWriter(dir + "\\" + "output\\" + fileName + "_out" + ext))
{
Writer.Write.(sbText.);
}
} // end of the foreach
} // end of ProcessFile
I was thinking about using the IF/ELSE: "using (StreamWriter Writer = new StreamWriter(dir + "\" + "output\" + fileName + "_out" + ext))" part. However, I am not sure how to pass, to StreamWriter, to only write from/to a specific line number.
Any Help is welcome! If I am missing some information, please, let me know (I am pretty new on stackoverflow).
Thank you.

Code is way too complicated
using System.Collections.ObjectModel;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.IO;
namespace ConsoleApplication57
{
class Program
{
static void Main(string[] args)
{
}
public static void ProcessFile(string[] ProcessFile)
{
foreach (string filename in ProcessFile)
{
// Used for the output name of the file
var dir = Path.GetDirectoryName(filename);
var fileName = Path.GetFileNameWithoutExtension(filename);
var ext = Path.GetExtension(filename);
var folderbefore = Path.GetFullPath(Path.Combine(dir, #"..\"));
var lineCount = File.ReadAllLines(#filename).Length;
string outputname = folderbefore + "output\\" + fileName;
using (StreamWriter Writer = new StreamWriter(dir + "\\" + "output\\" + fileName + "_out" + ext))
{
int rowCount = 0;
using (StreamReader Reader = new StreamReader(#filename))
{
rowCount++;
string inputLine = "";
while ((inputLine = Reader.ReadLine()) != null)
{
if (filename.Contains("RO_"))
{
if (rowCount <= 4)
{
Writer.WriteLine(inputLine);
}
if (rowCount == 4) break;
}
else
{
if (rowCount >= 2)
{
Writer.WriteLine(inputLine);
}
}
} // end of the while
Writer.Flush();
}
}
} // end of the foreach
} // end of ProcessFile
}
}

You can use LINQ to Take and Skip lines.
public abstract class CsvProcessor
{
private readonly IEnumerable<string> processFiles;
public CsvProcessor(IEnumerable<string> processFiles)
{
this.processFiles = processFiles;
}
protected virtual IEnumerable<string> GetAllLinesFromFile(string fileName)
{
using(var stream = new FileStream(fileName, FileMode.Open, FileAccess.Read, FileShare.Read))
using(var reader = new StreamReader(stream))
{
var line = String.Empty;
while((line = reader.ReadLine()) != null)
{
yield return line;
}
}
}
protected virtual void ProcessFiles()
{
var sb1 = new StringBuilder();
var sb2 = new StringBuilder();
foreach(var file in this.processFiles)
{
var fileName = Path.GetFileNameWithoutExtension(file);
var lines = GetAllLinesFromFile(file);
if(fileName.StartsWith("RO_", StringComparison.InvariantCultureIgnoreCase))
{
sb1.AppendLine(lines.Take(4)); //take only the first four lines
sb2.AppendLine(lines.Skip(4).TakeWhile(s => !String.IsNullOrEmpty(s))); //skip the first four lines, take everything else
}
else if(fileName.StartsWith("Load_", StringComparison.InvariantCultureIgnoreCase)
{
sb2.AppendLine(lines.Skip(1).TakeWhile(s => !String.IsNullOrEmpty(s)));
}
}
// now write your StringBuilder objects to file...
}
protected virtual void WriteFile(StringBuilder sb1, StringBuilder sb2)
{
// ... etc..
}
}

File name from StreamReader C# - asp.net MVC3 to array

my application is MVC3 C#; I am populating two dropdownlists using json using the following:
public ActionResult CheckWord(string cword)
{
try
{
List<string[]> arrayList = new List<string[]>();
List<string[]> stateList = new List<string[]>();
//
List<string[]> fileList = new List<string[]>();
//
string[] filePaths = Directory.GetFiles(System.Web.HttpContext.Current.Server.MapPath("/Video"), "*.srt");
string[] fnList = new string[filePaths.Length];
for (int i = 0; i < fnList.Length; ++i)
{
FileInfo fi = new FileInfo(filePaths[i]);
fnList[i] = fi.Name.Substring(0, fi.Name.LastIndexOf(".srt"));
}
int nFiles = filePaths.Length;
string cacheline = "";
string line;
for (int i = 0; i < nFiles; ++i)
{
StreamReader file = new StreamReader(System.Web.HttpContext.Current.Server.MapPath("/Video/" + fnList[i] + ".srt"));
List<string> lines = new List<string>();
List<string> statments = new List<string>();
//
List<string> fnames = new List<string>();
//
while ((line = file.ReadLine()) != null)
{
if (line.Contains(cword))
{
statments.Add(line);
// fnames.Add(file);
lines.Add(cacheline);
}
cacheline = line;
}
file.Close();
var array = lines.ToArray();
arrayList.Add(array);
stateList.Add(statments.ToArray());
}
return Json(new { success = true, fnList = fnList, arrayList = arrayList.ToArray(), stateList = stateList.ToArray() });
}
catch { }
return Json(new { success = false });
}
I am checking if a word exists in a group of files; then display the names of files in one dropdownlist and the lines from each file in the other dropdownlist. It works fine, however it gives me a list of all files becasue I am sending back fnlist. However I am trying to display only the files that contain that word; I could not get the file name from the StreamReader and add it to an array fileList. I would appreciate your suggestions, thanks in advance.

Already so many lists! Why not another? You already open the file with fnList[i] within the context of the loop, so...
List<string[]> results = new List<string[]>();
....
while ((line = file.ReadLine()) != null) {
if (line.Contains(cword)) {
results.Add(fnList[i]);
break; // optional, if possible, but if you need to continue check for dupes
}
}
....
return Json(new {
success = true,
fnList = results.ToArray(),
arrayList = arrayList.ToArray(),
stateList = stateList.ToArray()
});

System.IO.StreamReader file = new System.IO.StreamReader("setup.txt");
Later on, we would like to print the name of the file being used by stream reader.
eg, if there is an error, I would like a message box that displays "error reading file: 'filename'"
MessageBox.Show("Error loading " + ((FileStream)file.BaseStream).Name);

Not sure what exactly you are looking for but since you are creating StreamReader from a file name why not have file name in a separate variable and use it later:
var fileName = System.Web.HttpContext.Current.Server.MapPath(
"/Video/" + fnList[i] + ".srt");
StreamReader file = new StreamReader(fileName);

Add a line in a txt file

I have a txt file with data such as the following:
:10FF800000040B4E00040B4E00047D1400047D148D
:10FF900000040B4E0004CF6200040B4E00040B4E15
:10FFA00000040B4E00040B4E00040B4E00040B4EDD
:10FFB00000047D1400047D1400047D1400047D14ED
:10FFC00000040B4E000000000000000000000000D4
:10FFD0000000000000040B4E0000000000000000C4
:10FFE0000000000000000000000000000000000011
:10FFF0000000000000000000060000000000BFF844
:020000020000FC
:020000040014E6
:043FF0005AC8A58C7A
:00000001FF
what I want to do with my C# program is to add a line after or before a specific line, lets say add the line:
:020000098723060
before this line:
:020000020000FC
I have tried using the File.ReadLines("file.txt").Last(); but that just gives me the last one, what if i want the third or fourth? also, is there any way to identify the ":" in the file?

The simplest way - if you're happy to read the whole file into memory - would be just:
public void InsertLineBefore(string file, string lineToFind, string lineToInsert)
{
List<string> lines = File.ReadLines(file).ToList();
int index = lines.IndexOf(lineToFind);
// TODO: Validation (if index is -1, we couldn't find it)
lines.Insert(index, lineToInsert);
File.WriteAllLines(file, lines);
}
public void InsertLineAfter(string file, string lineToFind, string lineToInsert)
{
List<string> lines = File.ReadLines(file).ToList();
int index = lines.IndexOf(lineToFind);
// TODO: Validation (if index is -1, we couldn't find it)
lines.Insert(index + 1, lineToInsert);
File.WriteAllLines(file, lines);
}
There are significantly more efficient ways of doing this, but this approach is really simple.

A brute force approach
string[] lines = File.ReadAllLines("file.txt");
using(StreamWrite sw = new StreamWriter("file.txt"))
{
foreach(string line in lines)
{
if(line == ":020000020000FC")
sw.WriteLine(":020000098723060");
sw.WriteLine(line);
}
}

I would say it is better to read and write line by line, especially if the target file tend to be of large size:
using (StreamReader r = new StreamReader("Test.txt"))
{
using (StreamWriter w = new StreamWriter("TestOut.txt"))
{
while (!r.EndOfStream)
{
string line = r.ReadLine();
w.WriteLine(line);
if (line == ":020000020000FC")
w.WriteLine(":020000098723060");
}
w.Close();
r.Close();
}
}

Not sure if you're trying to avoid reading the entire file in due to size, etc...but can't you just read the file and then replace...e.g.
var text = readFile(somePath);
writeFile( text.replace(":020000020000FC\n",":020000098723060\n:020000020000FC\n") , somePath);

Here is a solution, though it may not be the best, it does work:
public void AddTextToFile(string filePath, int lineNumber, string txt) //zero based lineNumber
{
Collection<string> newLines = new Collection<string>(File.ReadAllLines(filePath).ToList());
if (lineNumber < newLines.Count)
newLines.Insert(lineNumber, txt);
else
newLines.Add(txt);
using (StreamWriter writer = new StreamWriter(filePath, false))
{
foreach (string s in newLines)
writer.WriteLine(s);
}
}
And to answer your question about determining if ":" exists in a string, the answer is yes, in the example above, you could check if the line contains it by...
if(newLines[idx].Contains(':'))
//do something

The ":" character doesn't really help the implementation, the lines are all newline-delimited already.
Here's an attempt at a method that doesn't load it all to memory or output to a different file.
Never cross the streams.
static Int32 GetCharPos(StreamReader s)
{
var ia = BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.GetField;
Int32 charpos = (Int32)s.GetType().InvokeMember("charPos", ia, null, s, null);
Int32 charlen = (Int32)s.GetType().InvokeMember("charLen", ia, null, s, null);
return (Int32)s.BaseStream.Position - charlen + charpos;
}
static void Appsert(string data, string precedingEntry = null)
{
if (precedingEntry == null)
{
using (var filestream = new FileStream(dataPath, FileMode.Append))
using (var tw = new StreamWriter(filestream))
{
tw.WriteLine(data);
return;
}
}
int seekPos = -1;
using (var readstream = new FileStream(dataPath,
FileMode.Open, FileAccess.Read, FileShare.Write))
using (var writestream = new FileStream(dataPath,
FileMode.Open, FileAccess.Write, FileShare.Read))
using (var tr = new StreamReader(readstream))
{
while (seekPos == -1)
{
var line = tr.ReadLine();
if (line == precedingEntry)
seekPos = GetCharPos(tr);
else if (tr.EndOfStream)
seekPos = (int)readstream.Length;
}
writestream.Seek(seekPos, SeekOrigin.Begin);
readstream.Seek(seekPos, SeekOrigin.Begin);
int readLength = 0;
var readBuffer = new byte[4096];
var writeBuffer = new byte[4096];
var writeData = tr.CurrentEncoding.GetBytes(data + Environment.NewLine);
int writeLength = writeData.Length;
writeData.CopyTo(writeBuffer, 0);
while (true & writeLength > 0)
{
readLength = readstream.Read(readBuffer, 0, readBuffer.Length);
writestream.Write(writeBuffer, 0, writeLength);
var tmp = writeBuffer;
writeBuffer = readBuffer;
writeLength = readLength;
readBuffer = tmp;
}
}
}