This question already has an answer here:
When using yield within a "using" statement, when does Dispose occur?
(1 answer)
Closed 1 year ago.
I wrote below code, which works:
//VERSION 1;
static IEnumerable<string> ReadAsLines(string filename)
{
using (StreamReader reader = new StreamReader(filename))
{
while (!reader.EndOfStream)
yield return reader.ReadLine();
}
}
Using above method:
const string fileData = #"path\to\somePipeDelimitedData.txt";
var reader = ReadAsLines(fileData);
var headerArr = reader.First().Split('|');
foreach (var column in headerArr)
{
var dummy = column;
}
var recordsEnumerable = reader.Skip(1); //skip first header Line
//Read other lines...
foreach (var record in recordsEnumerable)
{
//read each line
var rowArray = record.Split('|');
//etc...
}
Now suppose I start off with a Stream instead of a file;
I tried re-writing the above code, but am struggling with the stream getting closed.
How can I fix the version below?
//VERSION 2;
static IEnumerable<string> ReadAsLines(Stream stream)
{
using (StreamReader reader = new StreamReader(stream))
{
while (!reader.EndOfStream)
yield return reader.ReadLine();
}
}
Calling version 2:
byte[] dataByteArr = File.ReadAllBytes(fileData);
MemoryStream memStr = new MemoryStream(dataByteArr);
var reader2 = ReadAsLines(memStr);
var headerArr2 = reader2.First().Split('|'); //*** STREAM gets closed after this line
foreach (var column in headerArr2)
{
var dummy = column;
}
var recordsEnumerable2 = reader2.Skip(1); //skip first header Line
//Read other lines... *** ERROR OCCURS HERE, as the Stream is closed.
foreach (var record in recordsEnumerable2)
{
//read each line
var rowArray = record.Split('|');
//etc...
}
I re-organized my initial attempt by pulling the StreamReader out of the Enumerable method and disposing it outside when I'm really done.
byte[] dataByteArr = File.ReadAllBytes(fileData); //decoded bytes
var memStr = new MemoryStream(dataByteArr);
using (StreamReader sr = new StreamReader(memStr))
{
var dataAsEnumerable = ReadAsLines(sr, memStr);
var headerArr2 = dataAsEnumerable.First().Split('|');
//*** HA! stream is still open !
foreach (var column in headerArr2)
{
var dummy = column;
}
var dataMinusHeader = dataAsEnumerable.Skip(1);
//Read other lines...
foreach (var record in dataMinusHeader)
{
//read each line
var rowArray = record.Split('|');
//etc...
}
}
Related
I was trying to read CSV file in C#.
I have tried File.ReadAllLines(path).Select(a => a.Split(';')) way but the issue is when there is \n multiple line in a cell it is not working.
So I have tried below
using LumenWorks.Framework.IO.Csv;
var csvTable = new DataTable();
using (TextReader fileReader = File.OpenText(path))
using (var csvReader = new CsvReader(fileReader, false))
{
csvTable.Load(csvReader);
}
for (int i = 0; i < csvTable.Rows.Count; i++)
{
if (!(csvTable.Rows[i][0] is DBNull))
{
var row1= csvTable.Rows[i][0];
}
if (!(csvTable.Rows[i][1] is DBNull))
{
var row2= csvTable.Rows[i][1];
}
}
The issue is the above code throwing exception as
The CSV appears to be corrupt near record '0' field '5 at position '63'
This is because the header of CSV's having two double quote as below
"Header1",""Header2""
Is there a way that I can ignore double quotes and process the CSV's.
update
I have tried with TextFieldParser as below
public static void GetCSVData()
{
using (var parser = new TextFieldParser(path))
{
parser.HasFieldsEnclosedInQuotes = false;
parser.Delimiters = new[] { "," };
while (parser.PeekChars(1) != null)
{
string[] fields = parser.ReadFields();
foreach (var field in fields)
{
Console.Write(field + " ");
}
Console.WriteLine(Environment.NewLine);
}
}
}
The output:
Sample CSV data I have used:
Any help is appreciated.
Hope this works!
Please replace two double quotes as below from csv:
using (FileStream fs = new FileStream(Path, FileMode.Open, FileAccess.ReadWrite, FileShare.None))
{
StreamReader sr = new StreamReader(fs);
string contents = sr.ReadToEnd();
// replace "" with "
contents = contents.Replace("\"\"", "\"");
// go back to the beginning of the stream
fs.Seek(0, SeekOrigin.Begin);
// adjust the length to make sure all original
// contents is overritten
fs.SetLength(contents.Length);
StreamWriter sw = new StreamWriter(fs);
sw.Write(contents);
sw.Close();
}
Then use the same CSV helper
using LumenWorks.Framework.IO.Csv;
var csvTable = new DataTable();
using (TextReader fileReader = File.OpenText(path))
using (var csvReader = new CsvReader(fileReader, false))
{
csvTable.Load(csvReader);
}
Thanks.
I have the following code that I'm trying to use to parse a CSV file that is being uploaded:
private Dictionary<string, string[]> LoadData(IFormFile file)
{
// Verify that the user selected a file
if (file != null && file.Length > 0)
{
string wwwPath = this.environment.WebRootPath;
// string contentPath = this.environment.ContentRootPath;
string path = Path.Combine(wwwPath, "WeeklySchedules");
if (!Directory.Exists(path))
{
Directory.CreateDirectory(path);
}
string fileName = Path.GetFileName(file.FileName);
using (FileStream stream = new FileStream(Path.Combine(path, fileName), FileMode.Create))
{
file.CopyTo(stream);
// System.Threading.Thread.Sleep(1000);
using (TextFieldParser parser = new TextFieldParser(Path.Combine(path, fileName)))
{
parser.TextFieldType = FieldType.Delimited;
parser.SetDelimiters(",");
Dictionary<string, string[]> parsedData = new Dictionary<string, string[]>();
while (!parser.EndOfData)
{
// Process row
string[] fields = parser.ReadFields();
int count = 0;
if (count++ == 0)
{
continue;
}
var pickup = fields[0];
var pickupDate = fields[1];
var dropoff = fields[2];
var dropoffDate = fields[3];
var driver = fields[7];
var pickupTime = DateTime.Parse(pickupDate).ToLongTimeString();
// string[] data =
}
}
}
}
return null;
}
You will note that I am passing the path to the uploaded stream to the parser, rather than the stream itself. I tried passing in the stream, but that doesn't work either. When I check in wwwroot/WeeklySchedules, the file is there. But when the parser gets to it, it comes back as empty. I even threw in a Sleep() to see if I was just hitting the file too soon. But that didn't make any difference.
I am getting some weird errors with the original stream, but the file is written, which is puzzling to me.
The errors are:
stream.ReadTimeout = 'stream.ReadTimeout' threw an exception of type 'System.InvalidOperationException'
stream.WriteTimeout = 'stream.WriteTimeout' threw an exception of type 'System.InvalidOperationException'
I've read through a bunch of blog posts and SO questions on the technique for loading/parsing a CSV file, but none of them indicate this as an issue.
Does anyone have any ideas?
Your first file stream is still open in your first using and you try to read it again with TextFieldParser
private Dictionary<string, string[]> LoadData(IFormFile file)
{
// Verify that the user selected a file
if (file != null && file.Length > 0)
{
string wwwPath = this.environment.WebRootPath;
// string contentPath = this.environment.ContentRootPath;
string path = Path.Combine(wwwPath, "WeeklySchedules");
if (!Directory.Exists(path))
{
Directory.CreateDirectory(path);
}
string fileName = Path.GetFileName(file.FileName);
using (FileStream stream = new FileStream(Path.Combine(path, fileName), FileMode.Create))
{
file.CopyTo(stream);
}
// System.Threading.Thread.Sleep(1000);
using (TextFieldParser parser = new TextFieldParser(Path.Combine(path, fileName)))
{
parser.TextFieldType = FieldType.Delimited;
parser.SetDelimiters(",");
Dictionary<string, string[]> parsedData = new Dictionary<string, string[]>();
while (!parser.EndOfData)
{
// Process row
string[] fields = parser.ReadFields();
int count = 0;
if (count++ == 0)
{
continue;
}
var pickup = fields[0];
var pickupDate = fields[1];
var dropoff = fields[2];
var dropoffDate = fields[3];
var driver = fields[7];
var pickupTime = DateTime.Parse(pickupDate).ToLongTimeString();
// string[] data =
}
}
}
return null;
}
Preserving your code going via a file; untangle the 2 using statements, to ensure the file has been written completely and has been closed properly, before the parser starts reading it.
using (FileStream stream = new FileStream(Path.Combine(path, fileName), FileMode.Create))
{
file.CopyTo(stream);
}
using (TextFieldParser parser = new TextFieldParser(Path.Combine(path, fileName)))
{
// ..
}
When writing to stream (Maybe other destinations too) CsvHelper does not return anything if my DataTable contains less than 12 rows. I tested adding rows one by one until I get a result in the string myCsvAsString variable.
Anyone ran into this problem? Here is the code I am using to reproduce it:
var stream = new MemoryStream();
using (var writer = new StreamWriter(stream))
using (var csvWriter = new CsvWriter(writer, CultureInfo.InvariantCulture))
{
if (includeHeaders)
{
foreach (DataColumn column in dataTable.Columns)
{
csvWriter.WriteField(column.ColumnName);
}
csvWriter.NextRecord();
}
foreach (DataRow row in dataTable.Rows)
{
for (var i = 0; i < dataTable.Columns.Count; i++)
{
csvWriter.WriteField(row[i]);
}
csvWriter.NextRecord();
}
csvWriter.Flush();
stream.Position = 0;
StreamReader reader = new StreamReader(stream);
string myCsvAsString = reader.ReadToEnd();
}
Ok I found the issue, I was flushing the csvWriter but I did not flush the StreamWriter.
I added writer.Flush() just after csvWriter.Flush() and the stream is complete.
How can I create a clone object of streamreader, when i do Serializing on streamreader object, program give me exception :
Unhandled Exception: System.Runtime.Serialization.SerializationException: Type '
System.IO.FileStream' in Assembly 'mscorlib, Version=4.0.0.0, Culture=neutral,
is not marked as serializable.
how can i do this?
suppose i have text file like with the text:
1
2
3
4
5
my program:
[Serializable()]
class Program
{
static void Main(string[] args)
{
Program obj = new Program();
obj.read();
}
void read()
{
StreamReader reader1 = new StreamReader(#"d:\test.txt");
string s = reader1.ReadLine();
Console.WriteLine(s);
SerializeObject("text.txt", reader1);
StreamReader reader2;
for (int i = 0; i < 3; i++)
{
reader1.ReadLine();
}
s = reader1.ReadLine();
Console.WriteLine(s);
reader2 = DeSerializeObject("text.txt");
s = reader2.ReadLine();
Console.WriteLine(s);
}
public void SerializeObject(string filename, StreamReader objectToSerialize)
{
Stream stream = File.Open(filename, FileMode.Create);
BinaryFormatter bFormatter = new BinaryFormatter();
bFormatter.Serialize(stream, objectToSerialize);
stream.Close();
}
public StreamReader DeSerializeObject(string filename)
{
StreamReader objectToSerialize;
Stream stream = File.Open(filename, FileMode.Open);
BinaryFormatter bFormatter = new BinaryFormatter();
objectToSerialize = (StreamReader)bFormatter.Deserialize(stream);
stream.Close();
return objectToSerialize;
}
}
I want to output must be:
1
5
2
OK, so you are simply trying to read a file and write to a different file. There is no serialization involved in what you are trying to do. Serialization generally involves persisting objects.
I believe this is what you are after:
static void Main()
{
using(StreamReader reader = new StreamReader(#"d:\input.txt"))
using(StreamWriter writer = new StreamWriter(#"d:\output.txt"))
{
string line;
// Write 1st line
line = reader.ReadLine();
writer.WriteLine(line);
// Skip 3 lines
for (int i = 0; i < 3; i++)
{
reader.ReadLine();
}
// Write 5th & 6th line
for (int i = 0; i < 2; i++)
{
line = reader.ReadLine();
writer.WriteLine(line);
}
}
}
UPDATE
Write the first line, then the fifth line, then the second line:
static void Main()
{
using(StreamReader reader = new StreamReader(#"d:\input.txt"))
using(StreamWriter writer = new StreamWriter(#"d:\output.txt"))
{
string line;
// Write first line
line = reader.ReadLine();
writer.WriteLine(line);
// Read the second line
string second = reader.ReadLine(); ;
// Skip 3rd & 4th lines
for (int i = 0; i < 2; i++)
{
reader.ReadLine();
}
// Write 5th line
line = reader.ReadLine();
writer.WriteLine(line);
// Write the 2nd line
writer.WriteLine(second);
}
}
I have a process that loads data into a sql table from a flat file then needs to immediately move the file to an archive folder.
However when running the code it imports the data but throws and IOException
{"The process cannot access the file because it is being used by another process."}
There appears to be some contention in the process. Where and how should I avoid this?
internal class Program
{
private static void Main(string[] args)
{
string sourceFolder = #"c:\ImportFiles\";
string destinationFolder = #"c:\ImportFiles\Archive\";
foreach (string fileName in Directory.GetFiles(sourceFolder, "*.*"))
{
string sourceFileName = Path.GetFileName(fileName);
string destinationFileName = Path.GetFileName(fileName) + ".arc";
ProcessFile(fileName);
string source = String.Concat(sourceFolder,sourceFileName);
string destination = String.Concat(destinationFolder,destinationFileName);
File.Move(source, destination);
}
}
static void ProcessFile(string fileName)
{
Encoding enc = new UTF8Encoding(true, true);
DataTable dt = LoadRecordsFromFile(fileName, enc, ',');
SqlBulkCopy bulkCopy = new SqlBulkCopy("Server=(local);Database=test;Trusted_Connection=True;",
SqlBulkCopyOptions.TableLock);
bulkCopy.DestinationTableName = "dbo.tblManualDataLoad";
bulkCopy.WriteToServer(dt);
bulkCopy.Close();
}
public static DataTable LoadRecordsFromFile(string fileName, Encoding encoding, char delimeter)
{
DataTable table = null;
if (fileName != null &&
!fileName.Equals(string.Empty) &&
File.Exists(fileName))
{
try
{
string tableName = "DataImport";
FileStream fs = new FileStream(fileName, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
List<string> rows = new List<string>();
StreamReader reader = new StreamReader(fs, encoding);
string record = reader.ReadLine();
while (record != null)
{
rows.Add(record);
record = reader.ReadLine();
}
List<string[]> rowObjects = new List<string[]>();
int maxColsCount = 0;
foreach (string s in rows)
{
string[] convertedRow = s.Split(new char[] { delimeter });
if (convertedRow.Length > maxColsCount)
maxColsCount = convertedRow.Length;
rowObjects.Add(convertedRow);
}
table = new DataTable(tableName);
for (int i = 0; i < maxColsCount; i++)
{
table.Columns.Add(new DataColumn());
}
foreach (string[] rowArray in rowObjects)
{
table.Rows.Add(rowArray);
}
//Remove Header Row From Import file
DataRow row = table.Rows[0];
row.Delete();
table.AcceptChanges();
}
catch
{
//TODO SEND EMAIL ALERT ON ERROR
throw new Exception("Error in ReadFromFile: IO error.");
}
}
else
{
//TODO SEND EMAIL ALERT ON ERROR
throw new FileNotFoundException("Error in ReadFromFile: the file path could not be found.");
}
return table;
}
}
Your program is likely holding the file open. You should wrap FileStream and StreamReader objects in using statements. This closes those objects when the using block finishes.
The part of your LoadRecordsFromFile function that reads the file should look something like:
...
string tableName = "DataImport";
List<string> rows = new List<string>();
using (FileStream fs = new FileStream(fileName, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
{
using (StreamReader reader = new StreamReader(fs, encoding))
{
string record = reader.ReadLine();
while (record != null)
{
rows.Add(record);
record = reader.ReadLine();
}
}
}
...