Edit: See my Solution below...
I had the following Problem to solve:
We receive Files (mostly adress-Information) from different sources, these can be in Windows Standard with CR/LF ('\r''\n') as Line Break or UNIX with LF ('\n').
When reading text in using the StreamReader.ReadLine() method, this is no Problem because it handles both cases equally.
The Problem occurs when you have a CR or a LF somewhere in the File that is not supposed to be there.
This happens for example if you Export a EXCEL-File with Cells that contain LineBreaks within the Cell to .CSV or other Flat-Files.
Now you have a File that for example has the following structure:
FirstName;LastName;Street;HouseNumber;PostalCode;City;Country'\r''\n'
Jane;Doe;co James Doe'\n'TestStreet;5;TestCity;TestCountry'\r''\n'
John;Hancock;Teststreet;1;4586;TestCity;TestCounty'\r''\n'
Now the StreamReader.ReadLine() Method reads the First Line as:
FirstName;LastName;Street;HouseNumber;PostalCode;City;Country
Which is fine but the seccond Line will be:
Jane;Doe;co James Doe
This will either break your Code or you will have false Results, as the following Line will be:
TestStreet;5;TestCity;TestCountry
So we usualy ran the File trough a tool that checks if there are loose '\n' or '\r' arround and delete them.
But this step is easy to Forget and so I tried to implement a ReadLine() method of my own. The requirement was that it would be able to use one or two LineBreak characters and those characters could be defined freely by the consuming logic.
This is the Class that I came up with:
public class ReadFile
{
private FileStream file;
private StreamReader reader;
private string fileLocation;
private Encoding fileEncoding;
private char lineBreak1;
private char lineBreak2;
private bool useSeccondLineBreak;
private bool streamCreated = false;
private bool endOfStream;
public bool EndOfStream
{
get { return endOfStream; }
set { endOfStream = value; }
}
public ReadFile(string FileLocation, Encoding FileEncoding, char LineBreak1, char LineBreak2, bool UseSeccondLineBreak)
{
fileLocation = FileLocation;
fileEncoding = FileEncoding;
lineBreak1 = LineBreak1;
lineBreak2 = LineBreak2;
useSeccondLineBreak = UseSeccondLineBreak;
}
public string ReadLine()
{
if (streamCreated == false)
{
file = new FileStream(fileLocation, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
reader = new StreamReader(file, fileEncoding);
streamCreated = true;
}
StringBuilder builder = new StringBuilder();
char[] buffer = new char[1];
char lastChar = new char();
char currentChar = new char();
bool first = true;
while (reader.EndOfStream != true)
{
if (useSeccondLineBreak == true)
{
reader.Read(buffer, 0, 1);
lastChar = currentChar;
if (currentChar == lineBreak1 && buffer[0] == lineBreak2)
{
break;
}
else
{
currentChar = buffer[0];
}
if (first == false)
{
builder.Append(lastChar);
}
else
{
first = false;
}
}
else
{
reader.Read(buffer, 0, 1);
if (buffer[0] == lineBreak1)
{
break;
}
else
{
currentChar = buffer[0];
}
builder.Append(currentChar);
}
}
if (reader.EndOfStream == true)
{
EndOfStream = true;
}
return builder.ToString();
}
public void Close()
{
if (streamCreated == true)
{
reader.Close();
file.Close();
}
}
}
This code works fine, it does what it is supposed to do but compared to the original StreamReader.ReadLine() method, it is ~3 Times slower. As we work with large row-Counts the difference is not only messured but also reflected in real world Performance.
(for 700'000 Rows it takes ~ 5 Seconds to read all Lines, extract a Chunk and write it to a new File, with my method it takes ~15 Seconds on my system)
I tried different aproaches with bigger buffers but so far I wasn't able to increase Performance.
What I would be interessted in:
Any suggestions how I could improve the performance of this code to get closer to the original Performance of StreamReader.ReadLine()?
Solution:
This now takes ~6 Seconds (compared to ~5 Sec using the Default 'StreamReader.ReadLine()' ) for 700'000 Rows to do the same things as the code above does.
Thanks Jim Mischel for pointing me in the right direction!
public class ReadFile
{
private FileStream file;
private StreamReader reader;
private string fileLocation;
private Encoding fileEncoding;
private char lineBreak1;
private char lineBreak2;
private bool useSeccondLineBreak;
const int BufferSize = 8192;
int bufferedCount;
char[] rest = new char[BufferSize];
int position = 0;
char lastChar;
bool useLastChar;
private bool streamCreated = false;
private bool endOfStream;
public bool EndOfStream
{
get { return endOfStream; }
set { endOfStream = value; }
}
public ReadFile(string FileLocation, Encoding FileEncoding, char LineBreak1, char LineBreak2, bool UseSeccondLineBreak)
{
fileLocation = FileLocation;
fileEncoding = FileEncoding;
lineBreak1 = LineBreak1;
lineBreak2 = LineBreak2;
useSeccondLineBreak = UseSeccondLineBreak;
}
private int readInBuffer()
{
return reader.Read(rest, 0, BufferSize);
}
public string ReadLine()
{
StringBuilder builder = new StringBuilder();
bool lineFound = false;
if (streamCreated == false)
{
file = new FileStream(fileLocation, FileMode.Open, FileAccess.Read, FileShare.ReadWrite, 8192);
reader = new StreamReader(file, fileEncoding);
streamCreated = true;
bufferedCount = readInBuffer();
}
while (lineFound == false && EndOfStream != true)
{
if (position < bufferedCount)
{
for (int i = position; i < BufferSize; i++)
{
if (useLastChar == true)
{
useLastChar = false;
if (rest[i] == lineBreak2)
{
count++;
position = i + 1;
lineFound = true;
break;
}
else
{
builder.Append(lastChar);
}
}
if (rest[i] == lineBreak1)
{
if (useSeccondLineBreak == true)
{
if (i + 1 <= BufferSize - 1)
{
if (rest[i + 1] == lineBreak2)
{
position = i + 2;
lineFound = true;
break;
}
else
{
builder.Append(rest[i]);
}
}
else
{
useLastChar = true;
lastChar = rest[i];
}
}
else
{
position = i + 1;
lineFound = true;
break;
}
}
else
{
builder.Append(rest[i]);
}
position = i + 1;
}
}
else
{
bufferedCount = readInBuffer();
position = 0;
}
}
if (reader.EndOfStream == true && position == bufferedCount)
{
EndOfStream = true;
}
return builder.ToString();
}
public void Close()
{
if (streamCreated == true)
{
reader.Close();
file.Close();
}
}
}
The way to speed this up would be to have it read more than one character at a time. For example, create a 4 kilobyte buffer, read data into that buffer, and then go character-by-character. If you copy character-by-character to a StringBuilder, it's pretty easy.
The code below shows how to parse out lines in a loop. You'd have to split this up so that it can maintain state between calls, but it should give you the idea.
const int BufferSize = 4096;
const string newline = "\r\n";
using (var strm = new StreamReader(....))
{
int newlineIndex = 0;
var buffer = new char[BufferSize];
StringBuilder sb = new StringBuilder();
int charsInBuffer = 0;
int bufferIndex = 0;
char lastChar = (char)-1;
while (!(strm.EndOfStream && bufferIndex >= charsInBuffer))
{
if (bufferIndex > charsInBuffer)
{
charsInBuffer = strm.Read(buffer, 0, buffer.Length);
if (charsInBuffer == 0)
{
// nothing read. Must be at end of stream.
break;
}
bufferIndex = 0;
}
if (buffer[bufferIndex] == newline[newlineIndex])
{
++newlineIndex;
if (newlineIndex == newline.Length)
{
// found a line
Console.WriteLine(sb.ToString());
newlineIndex = 0;
sb = new StringBuilder();
}
}
else
{
if (newlineIndex > 0)
{
// copy matched newline characters
sb.Append(newline.Substring(0, newlineIndex));
newlineIndex = 0;
}
sb.Append(buffer[bufferIndex]);
}
++bufferIndex;
}
// Might be a line left, without a newline
if (newlineIndex > 0)
{
sb.Append(newline.Substring(0, newlineIndex));
}
if (sb.Length > 0)
{
Console.WriteLine(sb.ToString());
}
}
You could optimize this a bit by keeping track of the starting position so that when you find a line you create a string from buffer[start] to buffer[current], without creating a StringBuilder. Instead you call the String(char[], int32, int32) constructor. That's a little tricky to handle when you cross a buffer boundary. Probably would want to handle crossing the buffer boundary as a special case and use a StringBuilder for temporary storage in that case.
I wouldn't bother with that optimization, though, until after I got this first version working.
Related
So I'm building my own custom-drawn large text edit control, which I want to be able to open a large text file, and only a small chunk of it. I am able to successfully do that, but the problem is large lines. I have it so that every time I read a char (which shouldn't be too many as it's limited to the size of the screen), it will measure the length of that line. If it's greater than the width of the screen, it will stop rendering that line. However, I still have to find the position of the next line which means having to read that line (which could be hundreds of GB large). Here is my code so far:
if (encoding == Encoding.ASCII || encoding == Encoding.Default)
{
fileStream.Seek(visibleLinesRanges[0].Start, SeekOrigin.Begin);
int maxNumberLines = MaxLinesOnScreen;
int linesRead = 0;
int read;
byte[] byteBuffer = new byte[bufferSize];
StringBuilder text = new StringBuilder();
string lineText = string.Empty;
bool dontAddLine = false;
void AddLine()
{
if (dontAddLine)
{
lineText = string.Empty;
return;
}
linesRead++;
bool visible = linesRead >= maxNumberLines;
if (vScrollBar.Visible != visible)
{
vScrollBar.Visible = visible;
}
text.Append(lineText);
lineText = string.Empty;
}
while ((read = fileStream.Read(byteBuffer, 0, bufferSize)) > 0)
{
char[] charBuffer = encoding.GetChars(byteBuffer, 0, read);
for (int i = 0; i < read; i++)
{
char c = charBuffer[i];
if (!dontAddLine)
{
string newLineText = lineText + c;
if (TextRenderer.MeasureText(newLineText, Font).Width > Width)
{
lineText += "\n";
AddLine();
if (linesRead > maxNumberLines)
{
goto Drawing;
}
dontAddLine = true;
}
else
{
lineText = newLineText;
}
}
if (c == '\r')
{
char c1 = ReadCharSingle(charBuffer, i, read);
if (c1 != '\n')
{
AddLine();
if (linesRead > maxNumberLines)
{
goto Drawing;
}
dontAddLine = false;
}
}
else if (c == '\n')
{
AddLine();
if (linesRead > maxNumberLines)
{
goto Drawing;
}
dontAddLine = false;
}
}
}
Drawing:
TextRenderer.DrawText(g, text.ToString(), Font, Point.Empty, Color.Gainsboro);
}
This works, and barely uses any memory, but the problem is that it's very slow when opening a file with a single line that is about ~120MB in size. Also, whenever I resize the window, I have to call this code (in the Paint event), so that the text is updated.
This is very slow. Is there any way to speed this up? Thanks.
I can easily receive the data's response from device and show this in Textbox with Serial.ReadExisting(). When showing the data, the problem is some of control character (ACK, SOH, ETX, ...) unprintable.
I try to detect the control character from Serial response with the code below but something wrong in the comparison.
The code is:
public bool read_port(ref string rs232data, int timeout)
{
try
{
int index_read = 0;
int total_read = 0;
int i = 0;
int delay_ms = 1;
byte[] buffer = new byte[serialPort1.ReadBufferSize];
rs232data = "";
string buffer_str = "";
if ((serialPort1.IsOpen == false))
{
#if DEBUG
MessageBox.Show("COM port is not opening");
#endif
return false;
}
do
{
Application.DoEvents();
PauseForMilliSeconds(delay_ms); //System.Threading.Thread.Sleep(delay_ms);
i++;
//serial port receive data
if (serialPort1.BytesToRead <= 0) continue;
if (i > 1) i--;
index_read = serialPort1.Read(buffer, total_read, serialPort1.BytesToRead);
total_read += index_read;
buffer_str = ToStringLsb(buffer);
//// Find a Start byte
int ack_index = buffer_str.LastIndexOf(ToStringLsb(ACK));
int nak_index = buffer_str.LastIndexOf(ToStringLsb(NAK));
//Find a Stop byte
int stop_index_etx = buffer_str.LastIndexOf(ToStringLsb(ETX));
int stop_index_lf = buffer_str.LastIndexOf(ToStringLsb(LF));
if ((stop_index_etx < 0) && (stop_index_lf < 0) && (ack_index < 0)) continue; // Can't find a byte Stop
if (stop_index_etx > 0)
{
if ((stop_index_etx + 1) < total_read)
{
stop_index_etx++;
}
else
{
if (buffer[total_read - 2] == ETX)
{
stop_index_etx--;
}
else continue;
}
}
rs232data = buffer_str.Substring(0, total_read);
return true;
}
while (i < timeout);
return false;
}
catch (Exception ex)
{
MessageBox.Show(ex.Message);
return false;
}
}
public static string ToStringLsb(byte bytevalue)
{
try
{
byte[] bytearray = { bytevalue };
return System.Text.Encoding.GetEncoding("iso-8859-1").GetString(bytearray);
}
catch (System.Exception ex)
{
MessageBox.Show(ex.Message);
return "";
}
}
public static string ToStringLsb(byte[] bytearray)
{
return System.Text.Encoding.GetEncoding("iso-8859-1").GetString(bytearray);
}
You need to look at a ASCII map and determine which values are the control values you are looking for.
So for example I would define the following control characters in my code, then in your DataReceived event you can look for those characters.
char ACK = (char)6;
char NAK = (char)21;
char SOH = (char)1;
char LF = (char)10;
StringBuilder sb = new StringBuilder();
private void serialPort1_DataReceived(object sender, System.IO.Ports.SerialDataReceivedEventArgs e)
{
string Data = serialPort1.ReadExisting();
foreach (char c in Data)
{
if (c == LF)
{
sb.Append(c);
CurrentLine = sb.ToString();
sb.Clear();
//parse CurrentLine here or print it to textbox
}
if (c == ACK)
{
sb.Append("<ACK>"); //or whatever you want to print
}
else
{
sb.Append(c);
}
}
}
edit:
To answer some of your questions in the comments below. The DataReceived event fires when it gets characters in it's buffer, but it could fire while only getting half of your message. So when using this event, you have to build a string until you know you have the whole message. In my example above, I'm assuming a LF (Line Feed) indicates I have the whole message. For you, you would use whatever character you are searching for that marks the end of your message (maybe ACK in your case?). You can choose to append that character to your string or not, it depends on your requirements. In my example you can see I append the LF but you could easily take that line out.
I am trying to Design a program that reads an ASCII text file one byte at a time and outputs the frequency count of how many times each character appears.
The output does not need to be sorted. The character frequency objects need to be processed and stored using a Linked List.
I know that my character frequency class is okay, however I cannot figure out how to count the characters. This is the code I have so far, but my counts end up as 0.
LinkedList<CharacterFrequency> list = new LinkedList<CharacterFrequency>();
CharacterFrequency cf = new CharacterFrequency();
StreamReader file = new StreamReader(#"input.txt");
var ch = -1;
while ((ch = file.Read ()) !=-1)
{
LinkedListNode<CharacterFrequency> node;
node = list.Find(cf);
if (list.Any())
{
cf.Increment();
}
if (node != null)
{
list.AddLast(cf);
}
}
Console.WriteLine(cf.ToString());
file.Close();
//and here is my character frequency class;
class CharacterFrequency
{
private Char m_char;
private int m_count;
public CharacterFrequency()
{
m_char = '\0';
m_count = 0;
}
public CharacterFrequency(Char ch)
{
m_char = ch;
m_count = 0;
}
public CharacterFrequency(Char ch, int count)
{
m_char = ch;
m_count = count;
}
public CharacterFrequency(CharacterFrequency cf)
{
m_char = cf.Character;
m_count = cf.Count;
}
public Char Character
{
get
{
return m_char;
}
set
{
m_char = value;
}
}
public int Count
{
get
{
return m_count;
}
set
{
if (m_count > value)
m_count = value;
}
}
public void Increment()
{
m_count++;
}
public override string ToString()
{
return String.Format("{0}({1}) {2}", m_char, (int)m_char, m_count);
}
public override int GetHashCode()
{
return (int)m_char;
}
public override bool Equals(object obj)
{
if (obj == null)
return false;
if (obj == this)
return true;
if (!(obj.GetType() == GetType()))
return false;
CharacterFrequency rhs = (CharacterFrequency)obj;
return this.Character == rhs.Character;
}
}
Updated code (I have not been able to work through the issue with the increment method yet):
class Program
{
static void Main(string[] args)
{
int ch;
const int Input_File = 0;
const int Output_File = 1;
StreamReader reader = null;
StreamWriter writer = null;
//linked list declaration
LinkedList<CharacterFrequency> list =
new LinkedList<CharacterFrequency>();
//read the characters from the input file
reader = new StreamReader(File.OpenRead(args[Input_File]));
while ((ch = reader.Read ()) !=-1)//casts to integer
{
CharacterFrequency cf = LinkedList<CharacterFrequency>list(ch); //attempt to create linked list from character frequency BIG ISSUES HERE!
LinkedListNode<CharacterFrequency> node; //attempt to create node
node = list.Find(cf); // find method
if (node != null) //increment if found
{
node.Value.Increment();
}
else //add if not found
{
list.AddLast(cf);
}
}
reader.Close();
//write outputfile
writer = new StreamWriter(File.OpenWrite(args[Output_File]));
foreach (CharacterFrequency cf in list)
writer.WriteLine(cf.ToString());
writer.Close();
}
}
}
I suspect the problem is here :
if (list.Any())
{
cf.Increment();
}
I think it should be (You may need to adjust it more):
if(node != null) {
node.getData().increment();
}
else {
list.AddLast(cf);
}
There's also a problem with the fact that you don't use ch . At the beggining of the loop, you should recreate a LinkedListNode<CharacterFrequency> cf from the char you've read, or the code won't work .
EDIT: You can't declare a variable more than once, so simply take it outside the loop
CharacterFrequency cf;
while ((ch = reader.Read ()) !=-1)//casts to integer
{
cf = LinkedList<CharacterFrequency>list(ch);
....
I get this error. Later I searched and found out the reason of illegal characters in my XML and its solution. But I don't have the access to edit any of these files. My job is to read and fetch the tag value, attribute value and similar stuff. SO I can't replace the binary characters with escapes like '\x01' with . Also I tried to include CheckCharacters =false in XMLreader settings. It doesn't take this. Still it is throwing the same error.
Is it not possible to fix in XMLreader? I read about XMLtextReader. It can skip the exception. But already I have coded for all my features using XMLreader. It would be good if I can find a solution for this. Otherwise I would have to change all my code.
My code:
private void button1_Click(object sender, EventArgs e)
{
int i = 0;
var filenames = System.IO.Directory
.EnumerateFiles(textBox1.Text, "*.xml", System.IO.SearchOption.AllDirectories)
.Select(System.IO.Path.GetFullPath);
foreach (var f in filenames)
{
var resolver = new XmlUrlOverrideResolver();
resolver.DtdFileMap[#"X1.DTD"] = #"\\location\X1.DTD";
resolver.DtdFileMap[#"R2.DTD"] = #"\\location\X2.DTD";
resolver.DtdFileMap[#"R5.DTD"] = #"\\location\R5.DTD";
XmlReaderSettings settings = new XmlReaderSettings();
settings.DtdProcessing = DtdProcessing.Parse;
settings.XmlResolver = resolver;
XmlReader doc = XmlReader.Create(f, settings);
while (doc.Read())
{
if ((doc.NodeType == XmlNodeType.Element) && (doc.Name == "ap"))
{
if (doc.HasAttributes)
{
String fin = doc.GetAttribute("ap");
if (fin == "no")
{
String[] array = new String[10000];
array[i] = (f);
File.AppendAllText(#"\\location\NAPP.txt", array[i] + Environment.NewLine);
i++;
}
else
{
String[] abs = new String[10000];
abs[i] = (f);
File.AppendAllText(#"\\location\APP.txt", abs[i] + Environment.NewLine);
i++;
}
}
}
}
}
MessageBox.Show("Done");
}
This is a very simple example of character "filter" that will replae the 0x06 character with a space:
public class MyStreamReader : StreamReader {
public MyStreamReader(string path)
: base(path) {
}
public override int Read(char[] buffer, int index, int count) {
int res = base.Read(buffer, index, count);
for (int i = 0; i < res; i++) {
if (buffer[i] == 0x06) {
buffer[i] = ' ';
}
}
return res;
}
}
You use it this way:
using (var sr = new MyStreamReader(f)) {
var doc = XmlReader.Create(sr, settings);
Note that it's very simple because it's replacing a character (the 0x06) with another character of the same "length" (the space). If you wanted to replace a character with a "sequence" of characters (to escape it), it would get more complex (not impossible, 30 minutes of work difficult)
(I have checked and it seems the XmlTextReader only uses that method and not the Read() method)
As always, when a programmer tells you 30 minutes, it means 0 minutes or 2 hours :-)
This is the "more complex" ReplacingStreamReader:
/// <summary>
/// Only the Read methods are supported!
/// </summary>
public class ReplacingStreamReader : StreamReader
{
public ReplacingStreamReader(string path)
: base(path)
{
}
public Func<char, string> ReplaceWith { get; set; }
protected char[] RemainingChars { get; set; }
protected int RemainingCharsIndex { get; set; }
public override int Read()
{
int ch;
if (RemainingChars != null)
{
ch = RemainingChars[RemainingCharsIndex];
RemainingCharsIndex++;
if (RemainingCharsIndex == RemainingChars.Length)
{
RemainingCharsIndex = 0;
RemainingChars = null;
}
}
else
{
ch = base.Read();
if (ch != -1)
{
string replace = ReplaceWith((char)ch);
if (replace == null)
{
// Do nothing
}
else if (replace.Length == 1)
{
ch = replace[0];
}
else
{
ch = replace[0];
RemainingChars = replace.ToCharArray(1, replace.Length - 1);
RemainingCharsIndex = 0;
}
}
}
return ch;
}
public override int Read(char[] buffer, int index, int count)
{
int res = 0;
// We leave error handling to the StreamReader :-)
// We handle only "working" parameters
if (RemainingChars != null && buffer != null && index >= 0 && count > 0 && index + count <= buffer.Length)
{
int remainingCharsCount = RemainingChars.Length - RemainingCharsIndex;
res = Math.Min(remainingCharsCount, count);
Array.Copy(RemainingChars, RemainingCharsIndex, buffer, index, res);
RemainingCharsIndex += res;
if (RemainingCharsIndex == RemainingChars.Length)
{
RemainingCharsIndex = 0;
RemainingChars = null;
}
if (res == count)
{
return res;
}
index += res;
count -= res;
}
while (true)
{
List<char> sb = null;
int res2 = base.Read(buffer, index, count);
if (res2 == 0 || ReplaceWith == null)
{
return res;
}
int j = 0;
for (int i = 0; i < res2; i++)
{
char ch = buffer[index + i];
string replace = ReplaceWith(ch);
if (sb != null)
{
if (replace == null)
{
sb.Add(ch);
}
else
{
sb.AddRange(replace);
}
}
else if (replace == null)
{
buffer[j] = ch;
j++;
}
else if (replace.Length == 1)
{
buffer[j] = replace[0];
j++;
}
else if (replace.Length == 0)
{
// We do not advance
}
else
{
sb = new List<char>();
sb.AddRange(replace);
}
}
res2 = j;
if (sb != null)
{
int res3 = Math.Min(sb.Count, count - res2);
sb.CopyTo(0, buffer, index + res2, res3);
if (res3 < sb.Count)
{
RemainingChars = new char[sb.Count - res3];
RemainingCharsIndex = 0;
sb.CopyTo(res3, RemainingChars, 0, RemainingChars.Length);
}
res += res3;
}
else
{
res2 = j;
// Can't happen if sb != null (at least a character must
// have been added)
if (res2 == 0)
{
continue;
}
}
res += res2;
return res;
}
}
}
Use it like:
using (var sr = new ReplacingStreamReader(f))
{
sr.ReplaceWith = x =>
{
return x == 0x6 ? " " : null;
// return x == '.' ? " " : null; // Replace all . with
};
var doc = XmlReader.Create(sr, settings);
Be aware that the ReplacingStreamReader doesn't "know" which part of the xml it is modifying, so rarely a "blind" replace is ok :-) Other than this limitation, you can replace any character with any string (null in the ReplaceWith means "keep the current character", equivalent to x.ToString() in the example given. Returning string.Empty is valid, means remove the current character).
The class is quite interesting: it keeps a char[] RemainingChars with the chars that have been read (and filtered by ReplaceWith) but that haven't been returned by a Read() method because the passed buffer was too much small (the ReplaceWith method could "enlarge" the read string, making it too much big for the buffer!). Note that sb is a List<char> instead of a StringBuilder. Probably using one or the other would be nearly equivalent, code-wise.
You could first read the content into a string replace (escape) the content, and then load it into a XmlReader:
foreach (var f in filenames) {
string text;
using (StreamReader s = new StreamReader(f,Encoding.UTF8)) {
text = s.ReadToEnd();
}
text = text.Replace("\x01",#""); //replace the content
//load some settings
var resolver = new XmlUrlOverrideResolver();
resolver.DtdFileMap[#"X1.DTD"] = #"\\location\X1.DTD";
resolver.DtdFileMap[#"R2.DTD"] = #"\\location\X2.DTD";
resolver.DtdFileMap[#"R5.DTD"] = #"\\location\R5.DTD";
XmlReaderSettings settings = new XmlReaderSettings();
settings.DtdProcessing = DtdProcessing.Parse;
settings.XmlResolver = resolver;
XmlReader doc = XmlReader.Create(text, settings);
//perform processing task
//...
}
I am trying to create a program which can sort the number of results associated with any specified google search. I need a big table very fast so I thought about using a loop. Each time I try it though, the debugger crashes due to a "System.Windows.Markup.XamlParseException".
public long resultStat(string a)
{
var req = (HttpWebRequest)WebRequest.Create("https://www.google.ca/search?hl=fr&output=search&sclient=psy-ab&q=a" + a + "&btnK=");
using (req as IDisposable)
{
WebResponse rep = req.GetResponse();
Stream str = rep.GetResponseStream();
StreamReader rdr = new StreamReader(str);
string res = rdr.ReadToEnd();
rdr.Close();
//This is my code to get the number results (it works perfectly)
int index = res.IndexOf(">Environ");
int cond = 0;
string final = "";
try
{
while (res[++index] != '<')
{
if (cond-- == 0 && res[index] != '&')
{ final += res[index]; cond = 0; }
else if (res[index] == '&') cond = 5;
}
}
catch { return 0; }
string temp = "";
foreach (char i in final) if (i < 48 && i > 58) temp += i;
return Int64.Parse(temp);
}
}
This whole method is simply used in the main in a for loop such as :
public void main()
{
//Other code
for (int i = 0; i < 3; i++) resultStat(i.ToString()); // For example
//Other code
}
I know it's the problem because as soon as I comment the loop, or lower it to one rep, nothing goes wrong. I've tried:
HttpWebRequest().Abort();
HttpWebRequest().KeepAlive = false;
It didn't work
I don't think the away you are doing is the correct way to do this. The simple one i can tell you is use Lib curl c#. You can send in an array of urls and get response as an array. That would be perfect for what you require here. Here is a sample class code below that does the multitasking itself. You just send in the urls.
public class MultiHttp
{
public static string UserAgent = "Mozilla 5.0";
public static string Header = "Content-Type: application/x-www-form-urlencoded; charset=UTF-8";
private static string[] Result;
public static string[] MultiPost(string[] Url, string post, int timeOut)
{
Result = new string[post.Length];
try
{
Curl.GlobalInit((int)CURLinitFlag.CURL_GLOBAL_ALL);
Easy.WriteFunction wf = new Easy.WriteFunction(OnWriteData);
//Easy.HeaderFunction hf = new Easy.HeaderFunction(OnHeaderData);
Easy[] easy = new Easy[Url.Length];
Multi multi = new Multi();
for (int i = 0; i < Url.Length; i++)
{
if (Url[i] != null)
{
easy[i] = new Easy();
easy[i].SetOpt(CURLoption.CURLOPT_URL, Url[i]);
easy[i].SetOpt(CURLoption.CURLOPT_WRITEFUNCTION, wf);
easy[i].SetOpt(CURLoption.CURLOPT_WRITEDATA, i);
//easy[i].SetOpt(CURLoption.CURLOPT_HEADERFUNCTION, hf);
//easy[i].SetOpt(CURLoption.CURLOPT_HEADERDATA, i);
easy[i].SetOpt(CURLoption.CURLOPT_TIMEOUT, timeOut);
easy[i].SetOpt(CURLoption.CURLOPT_USERAGENT, UserAgent);
Slist sl = new Slist();
sl.Append(Header);
easy[i].SetOpt(CURLoption.CURLOPT_HTTPHEADER, sl);
easy[i].SetOpt(CURLoption.CURLOPT_POSTFIELDS, post);
easy[i].SetOpt(CURLoption.CURLOPT_FOLLOWLOCATION, true);
easy[i].SetOpt(CURLoption.CURLOPT_POST, true);
//easy[i].SetOpt(CURLoption.CURLOPT_NOBODY, true);
if (Url[i].Contains("https"))
{
easy[i].SetOpt(CURLoption.CURLOPT_SSL_VERIFYHOST, 1);
easy[i].SetOpt(CURLoption.CURLOPT_SSL_VERIFYPEER, 0);
}
multi.AddHandle(easy[i]);
}
}
int stillRunning = 1;
while (multi.Perform(ref stillRunning) == CURLMcode.CURLM_CALL_MULTI_PERFORM) ;
while (stillRunning != 0)
{
multi.FDSet();
int rc = multi.Select(1000); // one second
switch (rc)
{
case -1:
stillRunning = 0;
break;
case 0:
default:
{
while (multi.Perform(ref stillRunning) == CURLMcode.CURLM_CALL_MULTI_PERFORM) ;
break;
}
}
}
// various cleanups
multi.Cleanup();
for (int i = 0; i < easy.Length; i++)
{
easy[i].Cleanup();
}
Curl.GlobalCleanup();
}
catch (Exception)
{
//r = ex+"";
}
return Result;
}
public static Int32 OnWriteData(Byte[] buf, Int32 size, Int32 nmemb,
Object extraData)
{
int tmp = Convert.ToInt32(extraData.ToString()); ;
Result[tmp] += System.Text.Encoding.UTF8.GetString(buf);
return size * nmemb;
}
}
Call it like :
String[] url= new String[2];
url[1]="https://www.google.ca/search?hl=fr&output=search&sclient=psy-ab&q=a1&btnK=";
url[2]="https://www.google.ca/search?hl=fr&output=search&sclient=psy-ab&q=a2&btnK=";
string postString=""; // IF YOU DO NOT WANT TO POST ANYTHING YOU CAN DO THE SAME THING KEEP URL THE SAME SEND POST ARRAY AND CHANGE THE CLASS IT WORKS BOTH WAYS
String[] result = MultiHttp.MultiPost(url, postString, timeOut);
Its just a sample but will get you the working idea to sort out your problem.