Develop Reference

Multithreaded c# console app to scrape data from sites

I have written an app that goes through our own properties and scraps the data. To make sure I don't run through the same URLs, I am using a MySQL database to store the URL, flag it once its processed. All this was being done in a single thread and it's fine if I had only few thousand entries. But I have few hundred thousand entries that I need to parse so I need to make changes in the code (I am newbie in multithreading in general). I found an example and was trying to copy the style but doesn't seem to work. Anyone know what the issue is with the following code?
EDIT: Sorry didn't mean to make people guess the issue but was stupid of me to include the exception. Here is the exception
"System.InValidCastException: 'Specified cast is not valid.'"
When I start the process it collects the URLs from the database and then never hits DoWork method
//This will get the entries from the database
List<Mappings> items = bot.GetUrlsToProcess(100);
if (items != null)
{
var tokenSource = new CancellationTokenSource();
var token = tokenSource.Token;
Worker.Done = new Worker.DoneDelegate(WorkerDone);
foreach (var item in items)
{
urls.Add(item.Url);
WaitingTasks.Enqueue(new Task(id => new Worker().DoWork((int)id, item.Url, token), item.Url, token));
}
LaunchTasks();
}
static async void LaunchTasks()
{
// keep checking until we're done
while ((WaitingTasks.Count > 0) || (RunningTasks.Count > 0))
{
// launch tasks when there's room
while ((WaitingTasks.Count > 0) && (RunningTasks.Count < MaxRunningTasks))
{
Task task = WaitingTasks.Dequeue();
lock (RunningTasks) RunningTasks.Add((int)task.AsyncState, task);
task.Start();
}
UpdateConsole();
await Task.Delay(300); // wait before checking again
}
UpdateConsole(); // all done
}
static void UpdateConsole()
{
Console.Write(string.Format("\rwaiting: {0,3:##0} running: {1,3:##0} ", WaitingTasks.Count, RunningTasks.Count));
}
static void WorkerDone(int id)
{
lock (RunningTasks) RunningTasks.Remove(id);
}
public class Worker
{
public delegate void DoneDelegate(int taskId);
public static DoneDelegate Done { private get; set; }
public async void DoWork(object id, string url, CancellationToken token)
{
if (token.IsCancellationRequested) return;
Content obj;
try
{
int tries = 0;
bool IsUrlProcessed = true;
DateTime dtStart = DateTime.Now;
string articleDate = string.Empty;
try
{
ScrapeWeb bot = new ScrapeWeb();
SearchApi searchApi = new SearchApi();
SearchHits searchHits = searchApi.Url(url, 5, 0);
if (searchHits.Hits.Count() == 0)
{
obj = await bot.ReturnArticleObject(url);
if (obj.Code != HttpStatusCode.OK)
{
Console.WriteLine(string.Format("\r Status is {0}", obj.Code));
tries = itemfound.UrlMaxTries + 1;
IsUrlProcessed = false;
itemfound.HttpCode = obj.Code;
}
else
{
string title = obj.Title;
string content = obj.Contents;
string description = obj.Description;
Articles article = new Articles();
article.Site = url.GetSite();
article.Content = content;
article.Title = title;
article.Url = url.ToLower();
article.Description = description;
string strThumbNail = HtmlHelper.GetImageUrl(url, obj.RawResponse);
article.Author = HtmlHelper.GetAuthor(url, obj.RawResponse);
if (!string.IsNullOrEmpty(strThumbNail))
{
//This condition needs to be added to remove ?n=<number> from EP thumbnails
if (strThumbNail.Contains("?"))
{
article.ImageUrl = strThumbNail.Substring(0, strThumbNail.IndexOf("?")).Replace("http:", "https:");
}
else
article.ImageUrl = strThumbNail.Replace("http:", "https:");
}
else
{
article.ImageUrl = string.IsNullOrEmpty(strThumbNail) ? article.Url.GetDefaultImageUrls() : strThumbNail.Replace("http:", "https:");
}
articleDate = HtmlHelper.GetPublishDate(url, obj.RawResponse);
if (string.IsNullOrEmpty(articleDate))
article.Pubdate = DateTime.Now;
else
article.Pubdate = DateTime.Parse(articleDate);
var client = new Index(searchApi);
var result = client.Upsert(article);
itemfound.HttpCode = obj.Code;
if (result)
{
itemfound.DateCreated = DateTime.Parse(articleDate);
itemfound.DateModified = DateTime.Parse(articleDate);
UpdateItem(itemfound);
}
else
{
tries = itemfound.UrlMaxTries + 1;
IsUrlProcessed = false;
itemfound.DateCreated = DateTime.Parse(articleDate);
itemfound.DateModified = DateTime.Parse(articleDate) == null ? DateTime.Now : DateTime.Parse(articleDate);
UpdateItem(itemfound, tries, IsUrlProcessed);
}
}
}
else
{
tries = itemfound.UrlMaxTries + 1;
IsUrlProcessed = true;
itemfound.HttpCode = HttpStatusCode.OK;
itemfound.DateCreated = DateTime.Parse(articleDate);
itemfound.DateModified = DateTime.Parse(articleDate) == null ? DateTime.Now : DateTime.Parse(articleDate);
}
}
catch (Exception e)
{
tries = itemfound.UrlMaxTries + 1;
IsUrlProcessed = false;
itemfound.DateCreated = DateTime.Parse(articleDate);
itemfound.DateModified = DateTime.Parse(articleDate) == null ? DateTime.Now : DateTime.Parse(articleDate);
}
finally
{
DateTime dtEnd = DateTime.Now;
Console.WriteLine(string.Format("\r Total time taken to process items is {0}", (dtEnd - dtStart).TotalSeconds));
}
}
catch (Exception e)
{
Console.WriteLine(e);
}
Done((int)id);
}
}
All this code is based from Best multi-thread approach for multiple web requests this link. Can someone tell me how to get this approach running?

I think the problem is in the way you're creating your tasks:
new Task(id => new Worker().DoWork((int)id, item.Url, token), item.Url, token)
This Task constructor overload expected Action<object> delegate. That means id will be typed as object and you need to cast it back to something useful first.
Parameters
action
Type: System.Action<Object>
The delegate that represents the code to execute in the task.
state
Type: System.Object
An object representing data to be used by the action.
cancellationToken
Type: System.Threading.CancellationToken
-The CancellationToken that that the new task will observe.
You decided to cast it to int by calling (int)id, but you're passing item.Url as the object itself. I can't tell you 100% what the type of Url is but I don't expect Url-named property to be of type int.

Based on what #MarcinJuraszek said I just went back to my code and added an int as I couldn't find another way to resolve it. Here is the change I made
int i=0
foreach (var item in items)
{
urls.Add(item.Url);
WaitingTasks.Enqueue(new Task(id => new Worker().DoWork((string)id, item.Url, token), item.Url, token));
i++;
}

Without blocking UI using async and await

My requirement is, need to render all framework elements which are bound with data using async and await.
Tried below possibilities:
If i am using asyncmethod().result it will block the UI and it
will be waiting for a long time to completion.
If I am using await keyword the controlstoconfigure foreach(2nd
one) will be hit for before completion of parallel.foreach()
So can you please suggest me for rendering elements, call the 2nd for each after completion of parallel foreach or without blocking UI?
Code snippet:
1)Renderingwidget - this method is used to get framework elements in the collection which is bound with data.
2)FetchData - this method is used to get the data from the server for particular framework element.
3)GetTablefromserver - this method is used to get the data based on queries.
public async void RenderingWidget()
{
ConcurrentDictionary<string, EngineChangedEventArgs> controlsToConfigure = new ConcurrentDictionary<string, EngineChangedEventArgs>();
foreach (var engines in MainWindow.ViewModel.RelationalDataManagerList.Values)
{
Dictionary<string, FrameworkElement> controlcollection = CurrentDashboardReport.WidgetCollection;
Parallel.ForEach(controlcollection, async item =>
{
try
{
try
{
controlsToConfigure.TryAdd(item.Key, await FetchDataInParallel(MainWindow.ViewModel.RelationalDashboardReportList[engines.DataSourceName].Reports, item.Key,));
}
catch (Exception ex)
{
ExceptionLog.WriteExceptionLog(ex, null);
throw new ParallelException(ex, item.Key);
}
}
catch (ParallelException ex)
{
exceptions.Enqueue(ex);
ExceptionLog.WriteExceptionLog(ex, GeneratedQueryText);
}
});
if (exceptions.Count > 0)
{
foreach (var nonRenderedControls in exceptions)
{
controlsToConfigure.TryAdd(nonRenderedControls.ReportName, await FetchDataInParallel(CurrentDashboardReport.Reports, nonRenderedControls.ReportName));
}
}
}
foreach (var control in controlsToConfigure)
{
(CurrentDashboardReport.WidgetCollection[control.Key] as DashboardDataControl).CurrentDashboardReport.OnActiveColumnsChanged(control.Value);
}
}
public async Task<EngineChangedEventArgs> FetchDataInParallel(RelationalReportCollection reports, string myReport)
{
var dataTable = await GetTableFromServer(reports.Query);
eventArgs = new EngineChangedEventArgs
{
ItemsSource = dataTable
};
return eventArgs;
}
public async Task<DataTable> GetTableFromServer(string query)
{
var resultTable = new DataTable
{
Locale = CultureInfo.InvariantCulture
};
SqlConnection sqlConnection = new SqlConnection(connectionString);
SqlCommand command = new SqlCommand(query, sqlConnection)
SqlDataReader dataReader = null;
try
{
if (sqlConnection.State != ConnectionState.Open)
{
sqlConnection.Open();
}
dataReader =await command.ExecuteReaderAsync();
resultTable.Load(dataReader);
return resultTable;
}
finally
{
if (dataReader != null)
{
dataReader.Dispose();
}
command.Dispose();
return resultTable;
}
}

Parallel.ForEach doesn't play well with async/await as you release the thread for the duration of the async call. Please refer to the following question for more information about this: Nesting await in Parallel.ForEach
You may just call Parallel.ForEach on a thread pool thread and then use the Task.WhenAll method to wait until all tasks have completed Before you continue:
public async void RenderingWidget()
{
ConcurrentDictionary<string, EngineChangedEventArgs> controlsToConfigure = new ConcurrentDictionary<string, EngineChangedEventArgs>();
List<Task> tasks = new List<Task>();
foreach (var engines in MainWindow.ViewModel.RelationalDataManagerList.Values)
{
Dictionary<string, FrameworkElement> controlcollection = CurrentDashboardReport.WidgetCollection;
tasks.Add(Task.Run(() =>
{
Parallel.ForEach(controlcollection, item =>
{
try
{
try
{
controlsToConfigure.TryAdd(item.Key, FetchDataInParallel(MainWindow.ViewModel.RelationalDashboardReportList[engines.DataSourceName].Reports, item.Key).Result);
}
catch (Exception ex)
{
ExceptionLog.WriteExceptionLog(ex, null);
throw new ParallelException(ex, item.Key);
}
}
catch (ParallelException ex)
{
exceptions.Enqueue(ex);
ExceptionLog.WriteExceptionLog(ex, GeneratedQueryText);
}
});
if (exceptions.Count > 0)
{
foreach (var nonRenderedControls in exceptions)
{
controlsToConfigure.TryAdd(nonRenderedControls.ReportName, FetchDataInParallel(CurrentDashboardReport.Reports, nonRenderedControls.ReportName).Result);
}
}
}));
}
await Task.WhenAll(tasks);
foreach (var control in controlsToConfigure)
{
(CurrentDashboardReport.WidgetCollection[control.Key] as DashboardDataControl).CurrentDashboardReport.OnActiveColumnsChanged(control.Value);
}
}
public async Task<EngineChangedEventArgs> FetchDataInParallel(RelationalReportCollection reports, string myReport)
{
var dataTable = await GetTableFromServer(reports.Query).ConfigureAwait(false);
return new EngineChangedEventArgs
{
ItemsSource = dataTable
};
}

Database async query and processing

Let me rephrase.
I have a method that generates strings(paths) after given a start string(path)
IF those paths are for a directory I want to enqueue that in the input of the method.
After processing the path synchronously, I want to get the Data and clone it async into multiple paths of a pipeline, were each path needs to get the datablock. So the Broadcastblock is out of the question (it cant send a blocking signal to the blocks before itself),
The joinblock, joining the results is relatively straight forward.
So to sum up
Is there a Block in Dataflow block, where i can access the inputqueue from the delegate, if when, how?
Is there a construct that acts like the broadcastblock but can block the blocks that came before it?
I tried doing it via almighty google:
class subversion
{
private static string repo;
private static string user;
private static string pw;
private static DateTime start;
private static DateTime end;
private static List<parserObject> output;
public static List<parserObject> svnOutputList
{
get {return output; }
}
private static List<string> extension_whitelist;
public async void run(string link, string i_user, string i_pw, DateTime i_start, DateTime i_end)
{
repo = link;
user = i_user;
pw = i_pw;
start = i_start;
end = i_end;
output = new List<parserObject>();
BufferBlock<string> crawler_que = new BufferBlock<string>();
BufferBlock<svnFile> parser_que = new BufferBlock<svnFile>();
var svn = crawl(crawler_que, parser_que);
var broadcaster = new ActionBlock<svnFile>(async file =>
{//tried to addapt the code from this ensure always send broadcastblock -> see link below
List<Task> todo = new List<Task>();
todo.Add(mLoc);//error cannot convert methodgroup to task
foreach (var task in todo)//error: Only assignment, call, increment, decrement, await, and new object expressions can be used as a statement?
{
task.SendAsync(file);//error cannot convert task to targetblock
}
await Task.WhenAll(todo.ToArray());
});
parser_que.LinkTo(broadcaster);
await Task.WhenAll(broadcaster, svn);//error cannot convert actionblock to task
}
private static async Task crawl(BufferBlock<string> in_queue, BufferBlock<svnFile> out_queue)
{
SvnClient client = new SvnClient();
client.Authentication.ForceCredentials(user, pw);
SvnListArgs arg = new SvnListArgs
{
Depth = SvnDepth.Children,
RetrieveEntries = SvnDirEntryItems.AllFieldsV15
};
while (await in_queue.OutputAvailableAsync())
{
string buffer_author = null;
string prev_author = null;
System.Collections.ObjectModel.Collection<SvnListEventArgs> contents;
string link = await in_queue.ReceiveAsync();
if (client.GetList(new Uri(link), arg, out contents))
{
foreach (SvnListEventArgs item in contents)
{
if (item.Entry.NodeKind == SvnNodeKind.Directory)
{
in_queue.Post(item.Path);
}
else if (item.Entry.NodeKind == SvnNodeKind.File)
{
try
{
int length = item.Name.LastIndexOf(".");
if (length <= 0)
{
continue;
}
string ext = item.Name.Substring(length);
if (extension_whitelist.Contains(ext))
{
Uri target = new Uri((repo + link));
SvnRevisionRange range;
SvnBlameArgs args = new SvnBlameArgs
{
Start = start.AddDays(-1),
End = end
};
try
{
svnFile file_instance = new svnFile();
client.Blame(target, args, delegate(object sender3, SvnBlameEventArgs e)
{
if (e.Author != null)
{
buffer_author = e.Author;
prev_author = e.Author;
}
else
{
buffer_author = prev_author;
}
file_instance.lines.Add(new svnLine(buffer_author, e.Line));
});
out_queue.Post(file_instance);
}
catch (Exception a) { Console.WriteLine("exception:" + a.Message);}
}
}
catch (Exception a)
{
}
}
}
}
}
}
private static async Task mLoc(svnFile file)
{
List<parserPart> parts = new List<parserPart>();
int find;
foreach (svnLine line in file.lines)
{
if ((find = parts.FindIndex(x => x.uploader_id == line.author)) > 0)
{
parts[find].count += 1;
}
else
{
parts.Add(new parserPart(line.author));
}
find = 0;
}
parserObject ret = new parserObject(parts, "mLoc");
await output.Add(ret);
return;
}
}
broadcastblock answer: Alternate to Dataflow BroadcastBlock with guaranteed delivery

Not reach the code as expected

I have a telephony application, in which I want to invoke simultaneous calls,. Each call will occupy a channel or port. So I added all channels to a BlockingCollection. The application is a windows service.
Let's see the code.
public static BlockingCollection<Tuple<ChannelResource, string>> bc = new BlockingCollection<Tuple<ChannelResource, string>>();
public static List<string> list = new List<string>();// then add 100 test items to it.
The main application has the code:
while (true)
{
ThreadEvent.WaitOne(waitingTime, false);
lock (SyncVar)
{
Console.WriteLine("Block begin");
for (int i = 0; i < ports; i++)
{
var firstItem = list.FirstOrDefault();
if (bc.Count >= ports)
bc.CompleteAdding();
else
{
ChannelResource cr = OvrTelephonyServer.GetChannel();
bc.TryAdd(Tuple.Create(cr, firstItem));
list.Remove(firstItem);
}
}
pc.SimultaneousCall();
Console.WriteLine("Blocking end");
if (ThreadState != State.Running) break;
}
Now for the simultaneous call code:
public void SimultaneousCall()
{
Console.WriteLine("There are {0} channels to be processed.", bc.Count);
var workItemBlock = new ActionBlock<Tuple<ChannelResource, string>>(
workItem =>
{
ProcessEachChannel(workItem);
});
foreach (var workItem in bc.GetConsumingEnumerable())
{
bool result = workItemBlock.SendAsync(workItem).Result;
}
workItemBlock.Complete();
}
private void ProcessEachChannel(Tuple<ChannelResource, string> workItem)
{
ChannelResource cr = workItem.Item1;
string sipuri = workItem.Item2;
VoiceResource vr = workItem.Item1.VoiceResource;
workItem.Item1.Disconnected += new Disconnected(workItemItem1_Disconnected);
bool success = false;
try
{
Console.WriteLine("Working on {0}", sipuri);
DialResult dr = new DialResult();
// blah blah for calling....
}
catch (Exception ex)
{
Console.WriteLine("Exception: {0}", ex.Message);
}
finally
{
if (cr != null && cr.VoiceResource != null)
{
cr.Disconnect();
cr.Dispose();
cr = null;
Console.WriteLine("Release channel for item {0}.", sipuri);
}
}
}
The question was when I tested the application with 4 ports, I thought the code should reach at
Console.WriteLine("Blocking end");
However it was not. Please see the snapshot.
The application is just hanging on after releasing the last channel. I guess that I may use the blockingcollection incorrectly. Thanks for help.
UPDATE:
Even I changed the code by using POST action as below, the situation is still unchanged.
private bool ProcessEachChannel(Tuple<ChannelResource, string> workItem)
{
// blah blah to return true or false respectively.
public void SimultaneousCall()
{
Console.WriteLine("There are {0} channels to be processed.", bc.Count);
var workItemBlock = new ActionBlock<Tuple<ChannelResource, string>>(
workItem =>
{
bool success = ProcessEachChannel(workItem);
});
foreach (var workItem in bc.GetConsumingEnumerable())
{
workItemBlock.Post(workItem);
}
workItemBlock.Complete();
}

I believe the problem is that you never call bc.CompleteAdding(): the if means it would be called in ports + 1-th iteration of the loop, but the loop iterates only ports-times. Because of this, GetConsumingEnumerable() returns a sequence that never ends, which means the foreach inside SimultaneousCall() blocks forever.
I think the right solution is to call bc.CompleteAdding() after the for loop, not in an impossible condition inside it.

How to create async http requests in a loop?

Yesterday I've found out how to create several async http requests without async/await. But today I need to do it in a loop: if some of responses don't satisfy some condition - I need to change a request for them and send these requests again. It may be repeated several times.
I've tried this code:
do
{
var loadingCoordinatesTasks = new List<Task<Terminal>>();
var totalCountOfTerminals = terminalPresetNode.ChildNodes.Count;
var uiTaskScheduler = TaskScheduler.FromCurrentSynchronizationContext();
foreach (var terminal in terminals.Except(_terminalsWithCoordinates))
{
var address = terminal.GetNextAddress();
var webRequest = (HttpWebRequest)WebRequest.Create(GeoCoder.GeoCodeUrl + address);
var webRequestTask = Task.Factory.FromAsync<WebResponse>(webRequest.BeginGetResponse,
webRequest.EndGetResponse,
terminal);
var parsingTask = webRequestTask.ContinueWith(antecedent =>
{
// Parse the response
});
loadingCoordinatesTasks.Add(parsingTask);
}
Task.Factory.ContinueWhenAll(loadingCoordinatesTasks.ToArray(), antecedents =>
{
foreach (var antecedent in antecedents)
{
var terminalWithCoordinates = antecedent.Result;
if (antecedent.Status == TaskStatus.RanToCompletion &&
!terminalWithCoordinates.Coordinates.AreUnknown)
{
_terminalsWithCoordinates.Add(terminalWithCoordinates);
_countOfProcessedTerminals++;
}
}
});
} while (_countOfProcessedTerminals < totalCountOfTerminals);
but is it possible to check the condition in while just after every single set of requests executed?

You can perform the check after increasing the count:
_countOfProcessedTerminals++;
if (_countOfProcessedTerminals >= totalCountOfTerminals)
{
break;
}
Is _countOfProcessedTerminals thread-safe though?

I manage to do it using recursion:
public void RunAgainFailedTasks(IEnumerable<Task<Terminal>> tasks)
{
Task.Factory.ContinueWhenAll(tasks.ToArray(), antecedents =>
{
var failedTasks = new List<Task<Terminal>>();
foreach (var antecedent in antecedents)
{
var terminal = antecedent.Result;
// Previous request was failed
if (terminal.Coordinates.AreUnknown)
{
string address;
try
{
address = terminal.GetNextAddress();
}
catch (FormatException) // No versions more
{
continue;
}
var getCoordinatesTask = CreateGetCoordinatesTask(terminal, address);
failedTasks.Add(getCoordinatesTask);
}
else
{
_terminalsWithCoordinates.Add(terminal);
}
}
if (failedTasks.Any())
{
RunAgainFailedTasks(failedTasks);
}
else
{
// Display a map
}
}, CancellationToken.None,
TaskContinuationOptions.None,
TaskScheduler.FromCurrentSynchronizationContext());
}
private Task<Terminal> CreateGetCoordinatesTask(Terminal terminal, string address)
{
var webRequest = (HttpWebRequest)WebRequest.Create(GeoCoder.GeoCodeUrl + address);
webRequest.KeepAlive = false;
webRequest.ProtocolVersion = HttpVersion.Version10;
var webRequestTask = Task.Factory.FromAsync<WebResponse>(webRequest.BeginGetResponse,
webRequest.EndGetResponse,
terminal);
var parsingTask = webRequestTask.ContinueWith(webReqTask =>
{
// Parse the response
});
return parsingTask;
}