Develop Reference

Download websites web pages in parallel

I have the following code and it works fine when I loop through one by one using foreach, but when I change it to use Parallel.ForEach I'm getting errors. Trying to figure out how I can correct this. FYI the dParms list contain unique id's.
The error I'm getting is
Error writing to path..... ErrorMessage:Item has already been added.
Key in dictionary: 'IIS_WasUrlRewritten' Key being added:
'IIS_WasUrlRewritten',
private void GeneratePages(RequestStatus response, string localDir, List<Parameters> dParms, int total, DateTime generatedTime)
{
int current = 0;
Parallel.ForEach(dParms, curJob =>
{
try
{
DownloadPage(localDir, curJob, generatedTime);
}
catch (Exception ex)
{
response.Status = false;
response.Message = ex.Message;
}
finally
{
Interlocked.Increment(ref current);
if (current % 10 == 0)
//to do: Send Progress to UI
}
//});
}
private string DownloadPage(string localDir, Parameters p, DateTime generatedTime)
{
string strExtension = "html";
string url = string.Empty;
url = this.Url.Action("MyAction", "Home", new { area = "", #id = p.MyId, #generatedTime = generatedTime }, this.Request.Url.Scheme);
var document = new HtmlWeb().Load(url);
string strFileName = url.Substring(url.LastIndexOf("/") + 1);
strFileName = strFileName.Substring(0, strFileName.IndexOf("generatedTime") - 1);
string strDiskFileName = strFileName.Replace(".aspx?", "");
strDiskFileName = strDiskFileName.Replace("?", "");
strDiskFileName = strDiskFileName.Replace(".aspx", "");
strDiskFileName = strDiskFileName.Replace("&", "");
strDiskFileName = strDiskFileName.Replace("=", "");
strDiskFileName = strDiskFileName.Replace("%20", "");
strDiskFileName += "." + strExtension;
document.Save(localDir + strDiskFileName);
return url;
}

Opening a file and using the string within

When I open a file (I made myself) I need to use somethings out of the string of text that comes trough. I want to use some parts of the text as coordinates to draw a graph with.
private void openToolStripMenuItem_Click(object sender, EventArgs e)
{
char XgetalEen;
char XgetalTwee;
char YgetalEen;
char Ygetaltwee;
string XgetalSamen = "";
string YgetalSamen = "";
int coordinaatX;
int coordinaatY;
DialogResult lel = MessageBox.Show("Do you want to close this file?", "OPEN", MessageBoxButtons.YesNo, MessageBoxIcon.Question);
if (lel == DialogResult.Yes)
{
Open();
foreach(string s in Gcowde)
{
XgetalEen = s[5];
XgetalTwee = s[6];
YgetalEen = s[8];
Ygetaltwee = s[9];
XgetalSamen += XgetalEen + XgetalTwee;
YgetalSamen += YgetalEen + Ygetaltwee;
if(XgetalTwee==' ')
{
XgetalSamen = "";
XgetalTwee = '0';
XgetalSamen += XgetalTwee + XgetalEen;
YgetalEen = s[7];
Ygetaltwee = s[8];
YgetalSamen = "";
YgetalSamen += YgetalEen + Ygetaltwee;
}
if(Ygetaltwee==' ')
{
Ygetaltwee = '0';
YgetalSamen = "";
YgetalSamen += Ygetaltwee + YgetalEen;
}
MessageBox.Show(XgetalSamen + " " + YgetalSamen);
Int32.TryParse(XgetalSamen, out coordinaatX);
Int32.TryParse(YgetalSamen, out coordinaatY);
currentLocation.X += coordinaatX;
currentLocation.Y += coordinaatY;
Coord.Add(new Point(currentLocation.X, currentLocation.Y));
}
drawerryting();
}
}
public void Open()
{
Gcowde.Clear();
listBox1.Items.Clear();
Coord.Clear();
werkVlak.Clear(Color.Black);
Coord.Add(new Point(pictureBox1.Width / 2, pictureBox1.Height / 2));
drawerryting();
OpenFileDialog openFileDialog1 = new OpenFileDialog();
if (openFileDialog1.ShowDialog() == DialogResult.OK)
{
StreamReader sr = new StreamReader(openFileDialog1.FileName);
string errything = sr.ReadToEnd();
string charAdded = "";
foreach (char s in errything)
{
if (s == '\n')
{
Gcowde.Add(charAdded);
charAdded = "";
}
else
{
charAdded += s;
}
}
foreach (string s in Gcowde)
{
listBox1.Items.Add(s);
}
sr.Close();
}
This is the code how I open the file and get the coordinates out of the string. The string is of this kind L1 G2 X50 Y50. i need to get the 2 50s out of the string.
ps.: the variables are in dutch.
XgetalEen = XnumberOne, XgetalTwee=XnumberTwo,
same goes for Y.
XgetalSamen=XnumberTogether, YgetalSamen=YnumberTogether.

This is a simple example how you could parse the file:
// Read your file using File.ReadAllLines
String[] lines = new[] { "L1 G2 X50 Y50", "L1 G2 X50 Y50" };
foreach (var line in lines)
{
String[] values = line.Split(' ');
string x = values.Where(s => s.StartsWith("X")).First().Replace("X", String.Empty);
int xCoordinate = Convert.ToInt32(x);
}
Don't forget to add all necessary checks and reading of other variables.

alchemy api in c# for text extractor

I want to extract text from web page at a run time then, use alchemy api in asp.net with c# but I don't know how to use this api in c#. I am trying to find out what the parameter of text extractor is. If needed I can also try to regular expression for extracting the web page but this was not clean html tags.
private void Form3_Load(object sender, EventArgs e)
{
}
void GetPosition(Uri url, string searchTerm)
{
string raw = "http://www.google.co.in/search?num=39&q={0}&btnG=Search"; string search = string.Format(raw,
HttpUtility.UrlEncode(searchTerm)); HttpWebRequest request = (HttpWebRequest)WebRequest.Create(search); using (HttpWebResponse
response = (HttpWebResponse)request.GetResponse())
{
using (StreamReader
reader = new StreamReader(response.GetResponseStream(),
Encoding.ASCII))
{
string html = reader.ReadToEnd();
//return FindPosition(html, url);
fillgoogle(html);
}
}
}
//New Fill Google
void fillgoogle(string html)
{
listBox1.Items.Clear();
// string pattern = #"h3 class=";
string pattern = "<h3 class=\"r\"><a href=";
/*for (int i = 0; i < 10; i++)
{
int start;
int end;
int pos;
pos = html.IndexOf(pattern);
start = html.IndexOf("href=", pos);
end = html.IndexOf("/", start + 15);
ListBox1.Items.Add(prepare(html.Substring(start + 6, end - start)));
html = html.Substring(end);
}*/
// int start;
int end;
int pos;
// string[] strUrl;
pos = html.IndexOf(pattern);
string[] Arr = Regex.Split(html, pattern);
for (int x = 1; x <= Arr.Length - 1; x++)
{
//string find = Arr[x].ToString();
//string RealData=find.Substring
// listBox1.Items.Add(Arr[x].ToString());
end = Arr[x].IndexOf("/", 38);
str1 = Arr[x].Substring(0, end);
// strUrl = Regex.Split(Arr[1], "&");
//string n = string.Join("/url?q=", Arr);
str1 = str1.Replace('"', ' ');
str1 = str1.Trim();
str1 = str1.Remove(0, 7).ToString();
listBox1.Items.Add(str1);
// ListBox1.Items.Add(Arr[x].Substring(0, end));
if (x == 10)
{
break;
}
}
}
void finalList()
{
listBox2.Items.Clear();
for (int i = 0; i < listBox1.Items.Count; i++)
{
string Link = listBox1.Items[i].ToString();
if (Link.IndexOf("&") != -1)
{
int end = Link.IndexOf("&");
string real = Link.Substring(0, end);
listBox2.Items.Add(real);
//MessageBox.Show(real);
}
}
}
string prepare(string url)
{
string temp;
int i;
i = url.IndexOf("//");
int j;
j = url.IndexOf("/", i + 3);
temp = url.Substring(0, j);
return (temp);
}
private static int FindPosition(
string html, Uri url)
{// h3 class=\"r\"><a href=\"http://www.godaddy.com/\
string lookup = "(<h3 class=r><a href=\")(\\*)";
MatchCollection matches = Regex.Matches(html, lookup);
for (int i = 0; i < matches.Count; i++)
{
string match = matches[i].Groups[2].Value;
if (match.Contains(url.Host))
return i + 1;
} return 0;
}
private void button1_Click(object sender, EventArgs e)
{
richTextBox1.Text = "";
Uri url = new Uri("http://www.godaddy.com");
GetPosition(url, textBox1.Text);
finalList();
webPage page = new webPage();
page.URL = listBox2.Items[0].ToString();
page.Load(); //Load the text from the specified URL
label3.Visible = true;
linkLabel1.Visible = true;
label3.Text = listBox2.Items[0].ToString();
//Display the page TITLE on the screen
//richTextBox1.Text = "Title: " + page.Title + Environment.NewLine + Environment.NewLine;
//Display a list of INTERNAL links on the screen (to include external links, see below)
//richTextBox1.Text += "LINKS" + Environment.NewLine + "=====" + Environment.NewLine;
//foreach (String link in page.LinksArray)
//{
// richTextBox1.Text += link + Environment.NewLine;
//}
//Display the BODY TEXT on the screen
richTextBox1.Text += Environment.NewLine + page.Body;
//richTextBox1.Text += Environment.NewLine + page.Paragraph;
}
public class webPage
{
public String URL;
private String sTitle;
private String sBody;
private String sParagraph;
private ArrayList aList;
public String Title
{
get
{
return sTitle;
}
}
public ArrayList LinksArray
{
get
{
return aList;
}
}
public String Body
{
get
{
return sBody;
}
}
public String Paragraph
{
get
{
return sParagraph;
}
}
public void Load()
{
try
{
WebRequest objRequest = WebRequest.Create(this.URL);
WebResponse objResponse = objRequest.GetResponse();
StreamReader oSR = new StreamReader(objResponse.GetResponseStream());
string strContent = oSR.ReadToEnd();
this.sTitle = getTitle(strContent);
this.aList = fetchLinks(strContent, URL);
this.sBody = fetchText(strContent);
this.sParagraph = GetFirstParagraph(strContent);
}
catch (Exception e)
{
MessageBox.Show(e.ToString());
}
}
private String getTitle(String sHTMLContent)
{
//Retrieve the title from the HTML code
return Regex.Match(sHTMLContent, "<title>(?<title>[^<]+)</title>", RegexOptions.IgnoreCase).Groups["title"].ToString();
}
private ArrayList fetchLinks(String sHTMLContent, String sURL)
{
//Find all the links in the HTML code and put them
//into an array
Match mMatch;
ArrayList aMatch = new ArrayList();
mMatch = Regex.Match(sHTMLContent, "href\\s*=\\s*(?:\"(?<1>[^\"]*)\"|(?<1>\\S+))", RegexOptions.IgnoreCase);
while (mMatch.Success)
{
String sMatch = processURL(mMatch.Groups[1].ToString(), sURL);
//Currently, this code only lists INTERNAL URLs. If you would
//like to include EXTERNAL URLs as well, comment out the fol-
//lowing IF statement EXCEPT the "aMatch.Add(sMatch);" line
if (sMatch.IndexOf(sURL) >= 0 && checkFormat(sMatch))
{
aMatch.Add(sMatch);
}
mMatch = mMatch.NextMatch();
}
return aMatch;
}
static string GetFirstParagraph(string s)
{
Match m = Regex.Match(s, #"<p>\s*(.+?)\s*</p>");
if (m.Success)
{
return m.Groups[1].Value;
}
else
{
return "";
}
}
private String fetchText(String s)
{
//Filter out HTML and JavaScript from the page, leaving only body text
s = Convert.ToString(Regex.Match(s, #"<body.+?</body>", RegexOptions.Singleline | RegexOptions.IgnoreCase)); //strip everything but <BODY>
s = Regex.Replace(s, "<script[^>]*?>.*?</script>", "", RegexOptions.Singleline | RegexOptions.IgnoreCase); //strip JavaScript
s = Regex.Replace(s, "<[^>]*>", ""); //strip HTML tags
s = Regex.Replace(s, "&(copy|#169);|&(quot|#34);|&(amp|#38);|&(lt|#60);&(gt|#62);|&(nbsp|#160);|&(iexcl|#161);|&(cent|#162);|&(pound|#163);|·", " "); //strip symbols
s = s.Replace("\t", " "); //strip tabs
s = Regex.Replace(s, "([\r\n])+", " "); //strip carriage returns
s = Regex.Replace(s, "\\s\\s+", " "); //strip white space (must be last)
return s.Trim();
}
private String processURL(String sInput, String sURL)
{
sURL = "http://" + Convert.ToString(sURL.Split('/').GetValue(2));
if (sInput.IndexOf("http://") < 0)
{
if (!sInput.StartsWith("/") && !sURL.EndsWith("/"))
{
return sURL + "/" + sInput;
}
else
{
if (sInput.StartsWith("/") && sURL.EndsWith("/"))
{
return sURL.Substring(0, sURL.Length - 1) + sInput;
}
else
{
return sURL + sInput;
}
}
}
else
{
return sInput;
}
}
private bool checkFormat(String sURL)
{
//List only pages ending with valid extensions
String[] validExt = { ".html", ".php", ".asp", ".htm", ".jsp", ".shtml", ".php3", ".aspx", ".pl", ".cfm" };
sURL = Convert.ToString(sURL.Split('?').GetValue(0));
foreach (String ext in validExt)
{
if (sURL.Substring(sURL.Length - ext.Length, ext.Length).ToLower() == ext) { return true; }
}
return false;
}
}
private void linkLabel1_LinkClicked(object sender, LinkLabelLinkClickedEventArgs e)
{
System.Diagnostics.Process.Start(label3.Text);
}
}
}

I do not see any attempts to call Alchemy API in your example but here's what you need to know:
Alchemy API uses standard Web Service calls with XML responses as default. However you can specify the response you want (JSON / RDP).
Here's the start-up documentation, Text Extraction and Requirements and URLGetText Endpoint (but check on the documentation website for the endpoint you need).

Union of million line urls in 2 files

File A B contains million urls.
1, go through the url in file A one by one.
2, extract subdomain.com (http://subdomain.com/path/file)
3, if subdomain.com exist file B, save it to file C.
Any quickest way to get file C with c#?
Thanks.
when i use readline, it have no much different.
// stat
DateTime start = DateTime.Now;
int totalcount = 0;
int n1;
if (!int.TryParse(num1.Text, out n1))
n1 = 0;
// memory
dZLinklist = new Dictionary<string, string>();
// read file
string fileName = openFileDialog1.FileName; // get file name
textBox1.Text = fileName;
StreamReader sr = new StreamReader(textBox1.Text);
string fullfile = File.ReadAllText(#textBox1.Text);
string[] sArray = fullfile.Split( '\n');
//IEnumerable<string> sArray = tool.GetSplit(fullfile, '\n');
//string sLine = "";
//while (sLine != null)
foreach ( string sLine in sArray)
{
totalcount++;
//sLine = sr.ReadLine();
if (sLine != null)
{
//string reg = "http[s]*://.*?/";
//Regex R = new Regex(reg, RegexOptions.Compiled);
//Match m = R.Match(sLine);
//if(m.Success)
int length = sLine.IndexOf(' ', n1); // default http://
if(length > 0)
{
//string urls = sLine.Substring(0, length);
dZLinklist[sLine.Substring(0,length)] = sLine;
}
}
}
TimeSpan time = DateTime.Now - start;
int count = dZLinklist.Count;
double sec = Math.Round(time.TotalSeconds,2);
label1.Text = "(" + totalcount + ")" + count.ToString() + " / " + sec + " = " + (Math.Round(count / sec,2)).ToString();
sr.Close();

I would go for using Microsoft LogParser for processing big files: MS LogParser. Are you limited to implement it in described way only?

Alternative to ReadLine?

I'm trying to read some files with ReadLine, but my file have some break lines that I need to catch (not all of them), and I don't know how to get them in the same array, neither in any other array with these separators... because... ReadLine reads lines, and break these lines, huh?
I can't replace these because I need to check it after the process, so I need to get the breaklines AND the content after that. That's the problem. How can I do that?
Here's my code:
public class ReadFile
{
string extension;
string filename;
System.IO.StreamReader sr;
public ReadFile(string arquivo, System.IO.StreamReader sr)
{
string ext = Path.GetExtension(arquivo);
sr = new StreamReader(arquivo, System.Text.Encoding.Default);
this.sr = sr;
this.extension = ext;
this.filename = Path.GetFileNameWithoutExtension(arquivo);
if (ext.Equals(".EXP", StringComparison.OrdinalIgnoreCase))
{
ReadEXP(arquivo);
}
else MessageBox.Show("Extensão de arquivo não suportada: "+ext);
}
public void ReadEXP(string arquivo)
{
string line = sr.ReadLine();
string[] words;
string[] Separators = new string[] { "<Segment>", "</Segment>", "<Source>", "</Source>", "<Target>", "</Target>" };
string ID = null;
string Source = null;
string Target = null;
DataBase db = new DataBase();
//db.CreateTable_EXP(filename);
db.CreateTable_EXP();
while ((line = sr.ReadLine()) != null)
{
try
{
if (line.Contains("<Segment>"))
{
ID = "";
words = line.Split(Separators, StringSplitOptions.None);
ID = words[0];
for (int i = 1; i < words.Length; i++ )
ID += words[i];
MessageBox.Show("Segment[" + words.Length + "]: " + ID);
}
if (line.Contains("<Source>"))
{
Source = "";
words = line.Split(Separators, StringSplitOptions.None);
Source = words[0];
for (int i = 1; i < words.Length; i++)
Source += words[i];
MessageBox.Show("Source[" + words.Length + "]: " + Source);
}
if (line.Contains("<Target>"))
{
Target = "";
words = line.Split(Separators, StringSplitOptions.None);
Target = words[0];
for (int i = 1; i < words.Length; i++)
Target += words[i];
MessageBox.Show("Target[" + words.Length + "]: " + Target);
db.PopulateTable_EXP(ID, Source, Target);
MessageBox.Show("ID: " + ID + "\nSource: " + Source + "\nTarget: " + Target);
}
}
catch (IndexOutOfRangeException e)
{
MessageBox.Show(e.Message.ToString());
MessageBox.Show("ID: " + ID + "\nSource: " + Source + "\nTarget: " + Target);
}
}
return;
}

If you are trying to read XML, try using the built in libaries, here is a simple example of loading a section of XML with <TopLevelTag> in it.
var xmlData = XDocument.Load(#"C:\folder\file.xml").Element("TopLevelTag");
if (xmlData == null) throw new Exception("Failed To Load XML");
Here is a tidy way to get content without it throwing an exception if missing from the XML.
var xmlBit = (string)xmlData.Element("SomeSubTag") ?? "";
If you really have to roll your own, then look at examples for CSV parsers,
where ReadBlock can be used to get the raw data including line breaks.
private char[] chunkBuffer = new char[4096];
var fileStream = new System.IO.StreamReader(new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite));
var chunkLength = fileStream.ReadBlock(chunkBuffer, 0, chunkBuffer.Length);