Develop Reference

C# Webscraper to grab amount of Google Results given a specific search term

I've been working on a webscraper as a Windows Forms application in C#. The user enter a search term and the term and the program will then split the search string for each individual words and look up the amount of search results through Yahoo and Google.
My issue lies with the orientation of the huge HTML document. I've tried multiple approaches such as
iterating recursively and comparing ids aswell as with lamba and the Where statements. Both results in null. I also manually looked into the html document to make sure the id of the div I want exist in the document.
The id I'm looking for is "resultStats" but it is suuuuuper nested. My code looks like this:
using HtmlAgilityPack;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace WebScraper2._0
{
public class Webscraper
{
private string Google = "http://google.com/#q=";
private string Yahoo = "http://search.yahoo.com/search?p=";
private HtmlWeb web = new HtmlWeb();
private HtmlDocument GoogleDoc = new HtmlDocument();
private HtmlDocument YahooDoc = new HtmlDocument();
public Webscraper()
{
Console.WriteLine("Init");
}
public int WebScrape(string searchterms)
{
//Console.WriteLine(searchterms);
string[] ssize = searchterms.Split(new char[0]);
int YahooMatches = 0;
int GoogleMatches = 0;
foreach (var term in ssize)
{
//Console.WriteLine(term);
var y = web.Load(Yahoo + term);
var g = web.Load(Google + term + "&cad=h");
YahooMatches += YahooFilter(y);
GoogleMatches += GoogleFilter(g);
}
Console.WriteLine("Yahoo found " + YahooMatches.ToString() + " matches");
Console.WriteLine("Google found " + GoogleMatches.ToString() + " matches");
return YahooMatches + GoogleMatches;
}
//Parse to get correct info
public int YahooFilter(HtmlDocument doc)
{
//Look for node with correct ID
IEnumerable<HtmlNode> nodes = doc.DocumentNode.Descendants().Where(n => n.HasClass("mw-jump-link"));
foreach (var item in nodes)
{
// displaying final output
Console.WriteLine(item.InnerText);
}
//TODO: Return search resultamount.
return 0;
}
int testCounter = 0;
string toReturn = "";
bool foundMatch = false;
//Parse to get correct info
public int GoogleFilter(HtmlDocument doc)
{
if (doc == null)
{
Console.WriteLine("Null");
}
foreach (var node in doc.DocumentNode.ChildNodes)
{
toReturn += Looper(node, testCounter, toReturn, foundMatch);
}
Console.WriteLine(toReturn);
/*
var stuff = doc.DocumentNode.Descendants("div")
.Where(node => node.GetAttributeValue("id", "")
.Equals("extabar")).ToList();
IEnumerable<HtmlNode> nodes = doc.DocumentNode.Descendants().Where(n => n.HasClass("appbar"));
*/
return 0;
}
public string Looper(HtmlNode node, int counter, string returnstring, bool foundMatch)
{
Console.WriteLine("Loop started" + counter.ToString());
counter++;
Console.WriteLine(node.Id);
if (node.Id == "resultStats")
{
returnstring += node.InnerText;
}
foreach (HtmlNode n in node.Descendants())
{
Looper(n, counter, returnstring, foundMatch);
}
return returnstring;
}
}
}

I made an google HTML Scraper a few weeks ago, a few things to consider
First: Google don't like when you try to Scrape their Search HTML, while i was running a list of companies trying to get their addresses and phone number, Google block my IP from accessing their website for a little bit (Which cause a hilarious panic in the office)
Second: Google will change the HTML (Id names and etc) of the page so using ID's won't work, on my case i used the combination of HTML Tags and specific information to parse the response and extract the information that i wanted.
Third: It's better to just use their API to grab the information you need, just make sure you respect their free tier query limit and you should be golden.
Here is the Code i used.
public static string getBetween(string strSource, string strStart, string strEnd)
{
int Start, End;
if (strSource.Contains(strStart) && strSource.Contains(strEnd))
{
Start = strSource.IndexOf(strStart, 0) + strStart.Length;
End = strSource.IndexOf(strEnd, Start);
return strSource.Substring(Start, End - Start);
}
else
{
return "";
}
}
public void SearchResult()
{
//Run a Google Search
string uriString = "http://www.google.com/search";
string keywordString = "Search String";
WebClient webClient = new WebClient();
NameValueCollection nameValueCollection = new NameValueCollection();
nameValueCollection.Add("q", keywordString);
webClient.QueryString.Add(nameValueCollection);
string result = webClient.DownloadString(uriString);
string search = getBetween(result, "Address", "Hours");
rtbHtml.Text = getBetween(search, "\">", "<");
}
On my case i used the String Address and Hours to limit what information i wanted to extract.
Edit: Fixed the Logic and added the Code i used.
Edit2: forgot to add the GetBetween Class. (sorry it's my first Answer)

How to take a CSV field and write to columns in SQL

I have the following code which takes a CSV and writes to a console:
using (CsvReader csv = new CsvReader(
new StreamReader("data.csv"), true))
{
// missing fields will not throw an exception,
// but will instead be treated as if there was a null value
csv.MissingFieldAction = MissingFieldAction.ReplaceByNull;
// to replace by "" instead, then use the following action:
//csv.MissingFieldAction = MissingFieldAction.ReplaceByEmpty;
int fieldCount = csv.FieldCount;
string[] headers = csv.GetFieldHeaders();
while (csv.ReadNextRecord())
{
for (int i = 0; i < fieldCount; i++)
Console.Write(string.Format("{0} = {1};",
headers[i],
csv[i] == null ? "MISSING" : csv[i]));
Console.WriteLine();
}
}
The CSV file has 7 headers for which I have 7 columns in my SQL table.
What is the best way to take each csv[i] and write to a row for each column and then move to the next row?
I tried to add the ccsv[i] to a string array but that didn't work.
I also tried the following:
SqlCommand sql = new SqlCommand("INSERT INTO table1 [" + csv[i] + "]", mysqlconnectionstring);
sql.ExecuteNonQuery();
My table (table1) is like this:
name address city zipcode phone fax device

your problem is simple but I will take it one step further and let you know a better way to approach the issue.
when you have a problem to sold, always break it down into parts and apply each part in each own method. For example, in your case:
1 - read from the file
2 - create a sql query
3 - run the query
and you can even add validation to the file (imagine your file does not even have 7 fields in one or more lines...) and the example below it to be taken, only if your file never passes around 500 lines, as if it does normally you should consider to use a SQL statement that takes your file directly in to the database, it's called bulk insert
1 - read from file:
I would use a List<string> to hold the line entries and I always use StreamReader to read from text files.
using (StreamReader sr = File.OpenText(this.CsvPath))
{
while ((line = sr.ReadLine()) != null)
{
splittedLine = line.Split(new string[] { this.Separator }, StringSplitOptions.None);
if (iLine == 0 && this.HasHeader)
// header line
this.Header = splittedLine;
else
this.Lines.Add(splittedLine);
iLine++;
}
}
2 - generate the sql
foreach (var line in this.Lines)
{
string entries = string.Concat("'", string.Join("','", line))
.TrimEnd('\'').TrimEnd(','); // remove last ",'"
this.Query.Add(string.Format(this.LineTemplate, entries));
}
3 - run the query
SqlCommand sql = new SqlCommand(string.Join("", query), mysqlconnectionstring);
sql.ExecuteNonQuery();
having some fun I end up doing the solution and you can download it here, the output is:
The code can be found here. It needs more tweaks but I will left that for others. Solution written in C#, VS 2013.
The ExtractCsvIntoSql class is as follows:
public class ExtractCsvIntoSql
{
private string CsvPath, Separator;
private bool HasHeader;
private List<string[]> Lines;
private List<string> Query;
/// <summary>
/// Header content of the CSV File
/// </summary>
public string[] Header { get; private set; }
/// <summary>
/// Template to be used in each INSERT Query statement
/// </summary>
public string LineTemplate { get; set; }
public ExtractCsvIntoSql(string csvPath, string separator, bool hasHeader = false)
{
this.CsvPath = csvPath;
this.Separator = separator;
this.HasHeader = hasHeader;
this.Lines = new List<string[]>();
// you can also set this
this.LineTemplate = "INSERT INTO [table1] SELECT ({0});";
}
/// <summary>
/// Generates the SQL Query
/// </summary>
/// <returns></returns>
public List<string> Generate()
{
if(this.CsvPath == null)
throw new ArgumentException("CSV Path can't be empty");
// extract csv into object
Extract();
// generate sql query
GenerateQuery();
return this.Query;
}
private void Extract()
{
string line;
string[] splittedLine;
int iLine = 0;
try
{
using (StreamReader sr = File.OpenText(this.CsvPath))
{
while ((line = sr.ReadLine()) != null)
{
splittedLine = line.Split(new string[] { this.Separator }, StringSplitOptions.None);
if (iLine == 0 && this.HasHeader)
// header line
this.Header = splittedLine;
else
this.Lines.Add(splittedLine);
iLine++;
}
}
}
catch (Exception ex)
{
if(ex.InnerException != null)
while (ex.InnerException != null)
ex = ex.InnerException;
throw ex;
}
// Lines will have all rows and each row, the column entry
}
private void GenerateQuery()
{
foreach (var line in this.Lines)
{
string entries = string.Concat("'", string.Join("','", line))
.TrimEnd('\'').TrimEnd(','); // remove last ",'"
this.Query.Add(string.Format(this.LineTemplate, entries));
}
}
}
and you can run it as:
class Program
{
static void Main(string[] args)
{
string file = Ask("What is the CSV file path? (full path)");
string separator = Ask("What is the current separator? (; or ,)");
var extract = new ExtractCsvIntoSql(file, separator);
var sql = extract.Generate();
Output(sql);
}
private static void Output(IEnumerable<string> sql)
{
foreach(var query in sql)
Console.WriteLine(query);
Console.WriteLine("*******************************************");
Console.Write("END ");
Console.ReadLine();
}
private static string Ask(string question)
{
Console.WriteLine("*******************************************");
Console.WriteLine(question);
Console.Write("= ");
return Console.ReadLine();
}
}

Usually i like to be a bit more generic so i'll try to explain a very basic flow i use from time to time:
I don't like the hard coded attitude so even if your code will work it will be dedicated specifically to one type. I prefer i simple reflection, first to understand what DTO is it and then to understand what repository should i use to manipulate it:
For example:
public class ImportProvider
{
private readonly string _path;
private readonly ObjectResolver _objectResolver;
public ImportProvider(string path)
{
_path = path;
_objectResolver = new ObjectResolver();
}
public void Import()
{
var filePaths = Directory.GetFiles(_path, "*.csv");
foreach (var filePath in filePaths)
{
var fileName = Path.GetFileName(filePath);
var className = fileName.Remove(fileName.Length-4);
using (var reader = new CsvFileReader(filePath))
{
var row = new CsvRow();
var repository = (DaoBase)_objectResolver.Resolve("DAL.Repository", className + "Dao");
while (reader.ReadRow(row))
{
var dtoInstance = (DtoBase)_objectResolver.Resolve("DAL.DTO", className + "Dto");
dtoInstance.FillInstance(row.ToArray());
repository.Save(dtoInstance);
}
}
}
}
}
Above is a very basic class responsible importing the data. Nevertheless of how this piece of code parsing CSV files (CsvFileReader), the important part is thata "CsvRow" is a simple List.
Below is the implementation of the ObjectResolver:
public class ObjectResolver
{
private readonly Assembly _myDal;
public ObjectResolver()
{
_myDal = Assembly.Load("DAL");
}
public object Resolve(string nameSpace, string name)
{
var myLoadClass = _myDal.GetType(nameSpace + "." + name);
return Activator.CreateInstance(myLoadClass);
}
}
The idea is to simple follow a naming convetion, in my case is using a "Dto" suffix for reflecting the instances, and "Dao" suffix for reflecting the responsible dao. The full name of the Dto or the Dao can be taken from the csv name or from the header (as you wish)
Next step is filling the Dto, each dto or implements the following simple abstract:
public abstract class DtoBase
{
public abstract void FillInstance(params string[] parameters);
}
Since each Dto "knows" his structure (just like you knew to create an appropriate table in the database), it can easily implement the FillInstanceMethod, here is a simple Dto example:
public class ProductDto : DtoBase
{
public int ProductId { get; set; }
public double Weight { get; set; }
public int FamilyId { get; set; }
public override void FillInstance(params string[] parameters)
{
ProductId = int.Parse(parameters[0]);
Weight = double.Parse(parameters[1]);
FamilyId = int.Parse(parameters[2]);
}
}
After you have your Dto filled with data you should find the appropriate Dao to handle it
which is basically happens in reflection in this line of the Import() method:
var repository = (DaoBase)_objectResolver.Resolve("DAL.Repository", className + "Dao");
In my case the Dao implements an abstract base class - but it's not that relevant to your problem, your DaoBase can be a simple abstract with a single Save() method.
This way you have a dedicated Dao to CRUD your Dto's - each Dao simply knows how to save for its relevant Dto. Below is the corresponding ProductDao to the ProductDto:
public class ProductDao : DaoBase
{
private const string InsertProductQuery = #"SET foreign_key_checks = 0;
Insert into product (productID, weight, familyID)
VALUES (#productId, #weight, #familyId);
SET foreign_key_checks = 1;";
public override void Save(DtoBase dto)
{
var productToSave = dto as ProductDto;
var saveproductCommand = GetDbCommand(InsertProductQuery);
if (productToSave != null)
{
saveproductCommand.Parameters.Add(CreateParameter("#productId", productToSave.ProductId));
saveproductCommand.Parameters.Add(CreateParameter("#weight", productToSave.Weight));
saveproductCommand.Parameters.Add(CreateParameter("#familyId", productToSave.FamilyId));
ExecuteNonQuery(ref saveproductCommand);
}
}
}
Please ignore the CreateParameter() method, since it's an abstraction from the base classs. you can just use a CreateSqlParameter or CreateDataParameter etc.
Just notice, it's a real naive implementation - you can easily remodel it better, depends on your needs.

From the first impression of your questionc I guess you would be having hugely number of records (more than lacs). If yes I would consider the SQL bulk copies an option. If the record would be less go ahead single record. Insert. The reason for you insert not working is u not providing all the columns of the table and also there's some syntax error.

Edit video metadata tags of MP4 files

I want to edit following video tags of MP4 files like Title, Subtitle, Rating, Coment, Author. I have search alot and find solution
Taglib sharp. But taglib sharp only edit title and comment. I also explore Directshow and UltraD3lib but my problem is
still there. If anybody have example or any opensource library then please share with me.

Here is a couple short methods I use to update my Video library files
Movie is a POCO object with the name, IMDB number, file name, year.. properties
ADOHelper is a class that I use to simplify ADO DB/SQL calls
With the code below I get the following icons on my videos
And when I look at the properties of the video I get the following
private static void UpdateActors(Movie found, FileInfo movie)
{
var sql = $"SELECT n.primaryName FROM Principals p Inner join Names n on p.nconst = n.nconst where tconst = '{found.IMDB}'";
var table = ADOHelper.ReturnDataTable(Conn, sql, CommandType.Text);
var value = (from DataRow row in table.Rows select row[0].ToString()).ToList();
var f = TagLib.File.Create(movie.FullName);
f.Tag.Artists = value.ToArray();
f.Tag.AlbumArtists = value.ToArray();
f.Save();
}
private static string UpdateGenre(Movie found, FileInfo movie)
{
var returned = string.Empty;
try
{
var sql = $"SELECT Genre FROM TitleGenre where TitleId = '{found.IMDB}' order by sort";
var table = ADOHelper.ReturnDataTable(Conn, sql, CommandType.Text);
var f = TagLib.File.Create(movie.FullName);
f.Tag.Genres = (from DataRow row in table.Rows select row[0].ToString()).ToArray();
returned = table.Rows[0][0].ToString();
f.Save();
}
catch (Exception e)
{
Console.WriteLine(found.FilePath + " : " + e.Message);
}
return returned;
}
private static int UpdateYear(Movie found, FileInfo movie)
{
var returned = 0;
try
{
var f = TagLib.File.Create(movie.FullName);
var sql = $"SELECT startYear FROM Titles where tconst = '{found.IMDB}'";
var table = ADOHelper.ReturnDataTable(Conn, sql, CommandType.Text);
f.Tag.Year = uint.Parse(table.Rows[0][0].ToString());
returned = (int)f.Tag.Year;
f.Tag.Title = found.Name;
f.Save();
}
catch (Exception e)
{
Console.WriteLine(found.FilePath + " : " + e.Message);
}
return returned;
}
private static string UpdateName(Movie found, FileInfo movie, string description)
{
var returned = string.Empty;
try
{
var f = TagLib.File.Create(movie.FullName);
f.Tag.Title = description;
returned = description;
f.Save();
}
catch (Exception e)
{
Console.WriteLine(found.FilePath + " : " + e.Message);
}
return returned;
}
private static void UpdatePicture(Movie found, FileInfo movie)
{
try
{
var poster = $"{PosterPath}\\{found.IMDB}.jpg";
if (File.Exists(poster))
{
var f = TagLib.File.Create(movie.FullName);
IPicture picture = new Picture(poster);
f.Tag.Pictures = new[] { picture };
f.Save();
}
else
{
using (var writer = new StreamWriter("G:\\NeedPicture.txt", true))
writer.WriteLine($"{found.IMDB} : {found.Name} No Picture");
}
}
catch (Exception e)
{
Console.WriteLine(found.FilePath + " : " + e.Message);
}
}
Hope this helps

WMI is being too slow

Is there a way to limit the number of entries WMI retrieves with a WQL statement?
I say this because running a query to retrieve all Win32_NTLogEvent instances is taking forever! All I really need are the most recent events (for about a week, or 2000 entries)
Here's a snippet of the code I'm using to get the log data. Other queries such as Win32_Processor are nice and quick.
if (Configuration.OnlyErrorLogs)
{
// If Information logs should be suppressed, only get events where event type is not 3
WMIDataTemp1 = DataRetriever.GetWMIData("Win32_NTLogEvent", "EventType<>3");
}
else
{
WMIDataTemp1 = DataRetriever.GetWMIData("Win32_NTLogEvent");
}
foreach (ManagementObject Object in WMIDataTemp1)
{
this.Log.Add(new Log(Object));
}
And the functions to get WMI data are as follows:
public static ManagementObject[] GetWMIData(string wmiClass) { return GetWMIData(wmiClass, "", "CIMV2"); }
public static ManagementObject[] GetWMIData(string wmiClass, string whereClause) { return GetWMIData(wmiClass, whereClause, "CIMV2"); }
public static ManagementObject[] GetWMIData(string wmiClass, string whereClause, string nameSpace)
{
try
{
// If a where clause has been set, prepare the clause to add to the query string
if (whereClause != "")
{
whereClause = " WHERE " + whereClause;
}
// Create a search query
string query = "SELECT * FROM " + wmiClass + whereClause;
ManagementObjectSearcher wmiSearcher = new ManagementObjectSearcher("root\\" + nameSpace, query);
ManagementObjectCollection matches = wmiSearcher.Get();
// Create an array to hold the matches
ManagementObject[] matchArray = new ManagementObject[matches.Count];
// If matches found, copy to output
if(matches.Count > 0)
{
// Copy the search matches into this array
matches.CopyTo(matchArray, 0);
}
// Return array
return matchArray;
}
catch (Exception e)
{
ErrorDialogue errorReporter = new ErrorDialogue(e);
return null;
}
}
Where each Log gets stored:
public class Log
{
public string Category = "N/A";
public string DateTime = "N/A";
public UInt16 ID = 0;
public string Level = "N/A";
public string Message = "N/A";
public string Source = "N/A";
public Log() { }
public Log(ManagementObject wmiLogEvent)
{
this.GetInfo(wmiLogEvent);
}
public void GetInfo(ManagementObject wmiLogEvent)
{
try
{
this.Category = DataRetriever.GetValue(wmiLogEvent, "CategoryString");
this.DateTime = DataRetriever.GetValue(wmiLogEvent, "TimeGenerated");
this.ID = DataRetriever.GetValueUInt16(wmiLogEvent, "EventIdentifier");
this.Level = DataRetriever.ConvertEventType(DataRetriever.GetValueUInt16(wmiLogEvent, "CategoryString"));
this.Message = DataRetriever.GetValue(wmiLogEvent, "Message");
this.Source = DataRetriever.GetValue(wmiLogEvent, "SourceName");
}
catch (Exception e)
{
ErrorDialogue errorReporter = new ErrorDialogue(e);
}
}
}

One option is to use a WHERE clause to specify the range of the entries you want...
For example you could use TimeGenerated in the WHERE clause to specify a time-based range...
Another option is to set BlockSize accordingly when creating ManagementObjectSearcher.
You could use that to specify that you want 2000 entries per call for example - together with an ORDER BY TimeGenerated DESC this should give a nice result.

Speed is not a strong suit for WMI. It tends to be quite memory intensive. However, the question has been addressed and there are a few things you can do. Check out Why are my queries taking such a long time to complete? from Microsoft TechNet.

Now using the System.Diagnostics.EventLog class as a faster alternative. Much more beneficial to the program compared to WMI.
http://msdn.microsoft.com/en-us/library/system.diagnostics.eventlog.aspx

How to track application usage? (2)

I asked this question yesterday and got a great response/code example. The only problem is that I forgot to mention that I am forced to work with the .Net Framework 2.0 and can't use the List.Select ( I assume the linq namespace). Does anyone have a good work around for List.Select seen below:
class Program
{
struct ProcessStartTimePair
{
public Process Process { get; set; }
public DateTime StartTime { get; set; }
public DateTime ExitTime
{
get
{
return DateTime.Now; // approximate value
}
}
public ProcessStartTimePair(Process p) : this()
{
Process = p;
try
{
StartTime = p.StartTime;
}
catch (System.ComponentModel.Win32Exception)
{
StartTime = DateTime.Now; // approximate value
}
}
}
static void Main(string[] args)
{
SqlConnection cnn = new SqlConnection(#"Data Source=XXXXXX;Initial Catalog=XXXXXX;User ID=XXXX;Password=XXXX");
List<ProcessStartTimePair> knownProcesses = new List<ProcessStartTimePair>();
while (true)
{
foreach (Process p in Process.GetProcesses())
{
if (!knownProcesses.Select(x => x.Process.Id).Contains(p.Id))
{
knownProcesses.Add(new ProcessStartTimePair(p));
//Console.WriteLine("Detected new process: " + p.ProcessName);
}
}
for (int i = 0; i < knownProcesses.Count; i++)
{
ProcessStartTimePair pair = knownProcesses[i];
try
{
if (pair.Process.HasExited)
{
Console.WriteLine(pair.Process.ProcessName + " has exited (alive from {0} to {1}).", pair.StartTime.ToString(), pair.ExitTime.ToString());
knownProcesses.Remove(pair);
i--; // List was modified, 1 item less
// TODO: Store in the info in the database
String sql = "insert into procs (machine,login,process,start_time,end_time) ";
sql += "values ('" + Environment.MachineName + "','" + System.Security.Principal.WindowsIdentity.GetCurrent().Name.ToString().Split('\\')[1] + "','" + pair.Process.ProcessName + "','" + pair.StartTime.ToString() + "','" + pair.ExitTime.ToString() + "');";
SqlCommand cmd = new SqlCommand(sql, cnn);
try
{
cnn.Open();
cmd.ExecuteNonQuery();
}
catch (Exception ex)
{
//Console.WriteLine(ex.Message);
}
finally
{
cnn.Close();
}
}
}
catch (System.ComponentModel.Win32Exception)
{
// Would have to check whether the process still exists in Process.GetProcesses().
// The process probably is a system process.
}
}
//Console.WriteLine();
System.Threading.Thread.Sleep(5000);
}
}
}

I'm not sure the datatype of Id. I'll assume an int, you get the idea:
List<int> idList = new List<int>();
foreach(ProcessStartTimePair proc in knownProcesses)
{
idList.Add(proc.Process.Id);
}
if(idList.Contains(p.Id))
{
// ...
}
You just have to do the work of getting the list of IDs yourself.
Also, it's generally a better idea to edit your original question, and leave comments on the answers of others.

Try this:
if(!knownProcesses.Exists(x => x.Process.Id == p.Id))
Or, if you are using Visual Studio 2005 (not 2008),
if(!knownProcesses.Exists(delegate(ProcessStartTimePair x) { return x.Process.Id == p.Id; }))