I have an async operation that takes in an List of Airline as parameter and returns some data, and I have a list of Airlines for which I want to get data for.
However, if I can't get the data for all those Airlines after some predefined amount of time, I want to stop waiting and return something else to the user.
public async Task Run()
{
var watch = System.Diagnostics.Stopwatch.StartNew();
await RunAirline();
watch.Stop();
Console.WriteLine($"Total Execution Time: {watch.ElapsedMilliseconds + Environment.NewLine}");
//return $"Total Execution Time: {watch.ElapsedMilliseconds + Environment.NewLine}";
//Console.ReadLine();
}
private static async Task RunAirline()
{
try
{
List<string> AirlineList = GetAirLineCodes();
List<Task<WebsiteDataModel.WebsiteDataModel>> taskList = new List<Task<WebsiteDataModel.WebsiteDataModel>>();
foreach (string AirlineCode in AirlineList)
{
taskList.Add(Task.Run(() => CallindividualAirline(AirlineCode)));
}
var result = await Task.WhenAll(taskList);
foreach (WebsiteDataModel.WebsiteDataModel model in result)
{
Display(model);
}
}
catch (Exception Ex)
{
Console.WriteLine(Ex.Message.ToString());
}
}
private static List<string> GetAirLineCodes()
{
return new List<string>()
{
"A",
"B",
"C"
};
}
private static void Display(WebsiteDataModel.WebsiteDataModel result)
{
Console.WriteLine($"Website Content as {result.DataContent} , Website Name as : {result.WebsiteName} Status as : {result.Status} , Content length as : {result.WebsiteData.Length} ----- Error as : {result.error.FaultException.ToString()}." + Environment.NewLine);
}
private static WebsiteDataModel.WebsiteDataModel CallindividualAirline(string AirlineCode)
{
WebsiteDataModel.WebsiteDataModel LobjWebsiteDataModel = new WebsiteDataModel.WebsiteDataModel();
WebsiteDataModel.ErrorData LobjErrorData = new WebsiteDataModel.ErrorData();
try
{
switch (AirlineCode)
{
// calling Airline API...........
case "A":
ClsAirOne LobjAirOne = new ClsAirOne();
LobjWebsiteDataModel = LobjAirOne.GetAirDataData("https://book.xxxxx.com");
return LobjWebsiteDataModel;
case "B":
ClsAirTwo LobjAirTwo = new ClsAirTwo();
LobjWebsiteDataModel = LobjAirTwo.GetAirData("https://book.xxxxx.in");
return LobjWebsiteDataModel;
case "C":
ClsAirThree LobjAirThree = new ClsAirThree();
LobjWebsiteDataModel = LobjAirThree.GetAirData("https://xxxxx.in/");
return LobjWebsiteDataModel;
default:
return LobjWebsiteDataModel;
}
}
catch (Exception Ex)
{
LobjWebsiteDataModel.Status = "0";
LobjWebsiteDataModel.WebsiteData = "";
LobjErrorData.FaultException = "ERR-01" + Ex.Message.ToString();
LobjWebsiteDataModel.error = LobjErrorData;
return LobjWebsiteDataModel;
}
}
The best way to do this is to cancel each operation passed to the Task.WhenAll. You can create a cancellation token source with a timeout, and then pass its CancellationToken down to the methods that actually do the I/O.
E.g.:
public async Task Run()
{
...
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(10));
await RunAirline(cts.Token);
...
}
private static async Task RunAirline(CancellationToken cancellationToken)
{
...
foreach (string AirlineCode in AirlineList)
taskList.Add(Task.Run(() => CallindividualAirline(AirlineCode, cancellationToken)));
...
}
private static WebsiteDataModel.WebsiteDataModel CallindividualAirline(string AirlineCode, CancellationToken cancellationToken)
{
...
ClsAirOne LobjAirOne = new ClsAirOne();
LobjWebsiteDataModel = LobjAirOne.GetAirDataData("https://book.xxxxx.com", cancellationToken);
...
ClsAirTwo LobjAirTwo = new ClsAirTwo();
LobjWebsiteDataModel = LobjAirTwo.GetAirData("https://book.xxxxx.in", cancellationToken);
...
}
Related
I'd like to spawn some threads and in each thread sequentially make calls to an API and aggregate the results (some sort of stress testing). Here is my attempt:
private async Task DoWork()
{
var allResponses = new List<int>();
for (int i = 0; i < 10; i++)
{
await Task.Run(() =>
{
var responses = Enumerable.Range(0, 50).Select(i => CallApiAndGetStatusCode());
allResponses.AddRange(responses);
});
}
// do more work
}
private int CallApiAndGetStatusCode()
{
try
{
var request = new HttpRequestMessage(httpMethod.Get, "some url");
var responseResult = httpClient.SendAsync(request).Result;
return (int)responseResult.StatusCode;
}
catch (Exception e)
{
logger.LogError(e, "Calling API failed");
}
}
However, this code always ends up the catch with the inner exception being {"A task was canceled."}. What am I doing wrong here?
There is no benefit to using either Enumerable.Range or .AddRange in your example, since you do not need the seeded number. Your code must be converted to async/await to avoid deadlocks and in doing so, you can simply loop inside of each task and avoid any odd interactions between Enumerable.Select and await:
private async Task DoWork()
{
var allTasks = new List<Task>(10);
var allResponses = new List<int>();
for (int i = 0; i < 10; i++)
{
allTasks.Add(Task.Run(async () =>
{
var tempResults = new List<int>();
for (int i = 0; i < 50; i++)
{
var result = await CallApiAndGetStatusCode();
if (result > 0) tempResults.Add(result);
}
if (tempResults.Count > 0)
{
lock (allResponses)
{
allResponses.AddRange(tempResults);
}
}
}));
}
await Task.WhenAll(allTasks);
// do more work
}
private async Task<int> CallApiAndGetStatusCode()
{
try
{
var request = new HttpRequestMessage(HttpMethod.Get, "some url");
var responseResult = await httpClient.SendAsync(request);
return (int)responseResult.StatusCode;
}
catch (Exception e)
{
logger.LogError(e, "Calling API failed");
}
return -1;
}
Note that this code is overly protective, locking the overall batch before adding the temp results.
I changed your code to this and work
async Task DoWork()
{
var allResponses = new List<int>();
for (int i = 0; i < 10; i++)
{
await Task.Run(() =>
{
var responses = Enumerable.Range(0, 3).Select(i => CallApiAndGetStatusCodeAsync());
allResponses.AddRange(responses.Select(x => x.Result));
});
}
// do more work
}
async Task<int> CallApiAndGetStatusCodeAsync()
{
try
{
var request = new HttpRequestMessage(HttpMethod.Get, "http://www.google.com");
var responseResult = await httpClient.SendAsync(request);
return (int)responseResult.StatusCode;
}
catch (Exception e)
{
logger.LogError(e, "Calling API failed");
return -1;
}
}
I have created a task which creates an XML string. The task can last multiple seconds. When the task isn't finished after 5 seconds, I want to cancel the Task 'smootly' and continue with writing the rest of the XML. So I built in cancellation inside my task. But although I see the following message in the Log:
ProcessInformationTask timed out
I also see this line in my log
Adding the process information took 10001 ms
I wonder why this can happen because I want to cancel the Task after 5 seconds (if not finished). So I expect the task to last 5 seconds max. How can I solve this? Probably the cancelation isn't setup properly?
Code where I call my task
string additionalInformation = null;
var contextInfo = new StringBuilder();
var xmlWriterSettings = new XmlWriterSettings()
{
OmitXmlDeclaration = true,
ConformanceLevel = ConformanceLevel.Fragment
};
var tokenSource = new CancellationTokenSource();
var token = tokenSource.Token;
using (XmlWriter xmlWriter = XmlWriter.Create(contextInfo, xmlWriterSettings))
{
try
{
xmlWriter.WriteStartElement("AdditionalInformation");
//Write xml (not long running)
var watch = System.Diagnostics.Stopwatch.StartNew();
string processInformation = AddProcessesInformation(xmlWriterSettings);
watch.Stop();
var elapsedMs = watch.ElapsedMilliseconds;
Log.Info("Adding the process information took : " + elapsedMs + " ms");
if (!string.IsNullOrEmpty(processInformation))
{
xmlWriter.WriteRaw(processInformation);
}
//Write xml (not long running)
xmlWriter.WriteEndElement();
additionalInformation = contextInfo.ToString();
}
catch (Exception e)
{
Log.Info("An exception occured during writing the additional information: " + e.Message);
return false;
}
return true;
}
Task method
private static string AddProcessesInformation(XmlWriterSettings xmlWriterSettings)
{
var tokenSource = new CancellationTokenSource();
var token = tokenSource.Token;
var contextInfo = new StringBuilder();
var processInformationTask = Task<string>.Factory.StartNew(() =>
{
if (token.IsCancellationRequested)
{
Log.Info("Cancellation request for the ProcessInformationTask");
}
Thread.Sleep(10000);
return "Ran without problems'";
}, token);
if (!processInformationTask.Wait(5000, token))
{
Log.Info("ProcessInformationTask timed out");
tokenSource.Cancel();
}
return processInformationTask?.Result;
}
I think you should check if cancellation is requested multiple Times, after each step in your method. Here is example using for loop:
private static string AddProcessesInformation(XmlWriterSettings xmlWriterSettings)
{
var tokenSource = new CancellationTokenSource();
var token = tokenSource.Token;
var contextInfo = new StringBuilder();
var processInformationTask = Task<string>.Factory.StartNew(() =>
{
for(int i = 0; i < 10; i++)
{
if (token.IsCancellationRequested)
{
Console.WriteLine("Cancellation request for the ProcessInformationTask");
return string.Empty;
}
Thread.Sleep(1000);
}
return "Ran without problems'";
}, token);
if (!processInformationTask.Wait(5000, token))
{
Console.WriteLine("ProcessInformationTask timed out");
tokenSource.Cancel();
}
return processInformationTask?.Result;
}
If inside task method you call other methods, you can pass cancellation token to them and use method token.ThrowIfCancellationRequested() inside.
var processInformationTask = Task<string>.Factory.StartNew(() =>
{
try
{
Method1(token);
Thread.Sleep(1000);
Method2(token);
Thread.Sleep(1000);
}
catch(OperationCanceledException ex)
{
return string.Empty;
}
return "Ran without problems'";
}, token);
private static void Method1(CancellationToken token)
{
token.ThrowIfCancellationRequested();
// do more work
}
I have a ImageProcessor class that creates a list of tasks for each of the image providers, inside of these tasks I then run a parallel.foreach loop for all the images for each provider, I want to be able to cancel all the tasks and nested parallel loops from the console, I've found example of how to cancel tasks and how to cancel parallel loops, but I am unsure as how to do it for nested processes.
Below is the code I have at the moment
From my console app:
using (IUnityContainer container = Bootstrapper.Initialise())
{
IImageProcessor processor = container.Resolve<IImageProcessor>();
processor.Container = container;
try
{
InputArguments arguments = new InputArguments(args);
if (arguments.Contains("fs"))
{
processor.Initialise(arguments["fs"]);
}
else
{
processor.Initialise();
}
processor.Run();
Console.WriteLine("\r\n\n\nPress any key to Exit");
Console.ReadLine();
return (int)ExitCode.Success;
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
Console.WriteLine("\r\n\n\nPress any key to Exit");
Console.ReadLine();
return (int)ExitCode.UnknownError;
}
}
The Run method
public void Run()
{
List<Task> tasks = new List<Task>();
foreach (IFileService fileservice in this.fileServices)
{
Task task = Task.Factory.StartNew((arg) =>
{
IFileService fs = (IFileService)arg;
string msg = $"Processing {fs.ToString()}...";
FileLogger.Write(msg, fs.ToString());
ConsoleLogger.WriteLine(msg);
fs.ProcessFiles();
//fileservice.ReprocessUnMatchedData();
}, fileservice);
tasks.Add(task);
}
Task.WaitAll(tasks.ToArray());
}
and inside each file service I have call this method:
protected bool ProcessFileRecord<T>() where T : IDataRecord
{
int matched = 0;
int notMatched = 0;
int skipped = 0;
bool result;
object lockObject = new object();
try
{
processTracker = GetTracker();
if (databaseHelper.TrackerFullyProcessed(processTracker))
{
LoggingService.Write("File already processed... Skipping.", LoggingTarget.All, serviceName);
result = true;
}
LoggingService.Write($"\r\nProcessing index file {fileRecord.IndexFileName}", LoggingTarget.File, serviceName);
Parallel.ForEach(
fileRecord.DataRecords,
new ParallelOptions() { MaxDegreeOfParallelism = Thread.CurrentThread.ManagedThreadId },
(item) =>
{
switch ((RecordProcessResult)ProcessRecord(item))
{
case RecordProcessResult.Invalid:
break;
case RecordProcessResult.Matched:
Increment(ref matched);
break;
case RecordProcessResult.NotMatched:
Increment(ref notMatched);
break;
case RecordProcessResult.Skipped:
Increment(ref skipped);
break;
default:
break;
}
lock (lockObject)
{
if ((matched + notMatched + skipped) % 100 == 0)
{
LoggingService.Write($"\tMatched: {matched}\tNot Matched: {notMatched}\tSkipped: {skipped}\t total: {matched + notMatched + skipped}", LoggingTarget.Trace & LoggingTarget.Console, serviceName);
}
}
});
LoggingService.Write($"Total Lines: {matched + notMatched + skipped} \r\nMatched: {matched} \r\nNot Matched: {notMatched} \r\nSkipped: {skipped}", LoggingTarget.All, serviceName);
this.databaseHelper.UpdateTracker(this.processTracker, matched, notMatched);
result = true;
}
catch (Exception ex)
{
LoggingService.Write($"Error processing data file:{fileRecord.IndexFileName}", LoggingTarget.All, serviceName);
LoggingService.Write($"{ex.ExceptionTreeAsString()}", LoggingTarget.All, serviceName);
LoggingService.Write($"Total Lines: {(matched + notMatched + skipped)} \r\nMatched: {matched} \r\nNot Matched: {notMatched} \r\nSkipped: {skipped}", LoggingTarget.All, serviceName);
this.databaseHelper.UpdateTracker(this.processTracker, matched, notMatched);
result = false;
}
return result;
}
Thank.
Please, look an example on this page
CancellationTokenSource
Let me rephrase.
I have a method that generates strings(paths) after given a start string(path)
IF those paths are for a directory I want to enqueue that in the input of the method.
After processing the path synchronously, I want to get the Data and clone it async into multiple paths of a pipeline, were each path needs to get the datablock. So the Broadcastblock is out of the question (it cant send a blocking signal to the blocks before itself),
The joinblock, joining the results is relatively straight forward.
So to sum up
Is there a Block in Dataflow block, where i can access the inputqueue from the delegate, if when, how?
Is there a construct that acts like the broadcastblock but can block the blocks that came before it?
I tried doing it via almighty google:
class subversion
{
private static string repo;
private static string user;
private static string pw;
private static DateTime start;
private static DateTime end;
private static List<parserObject> output;
public static List<parserObject> svnOutputList
{
get {return output; }
}
private static List<string> extension_whitelist;
public async void run(string link, string i_user, string i_pw, DateTime i_start, DateTime i_end)
{
repo = link;
user = i_user;
pw = i_pw;
start = i_start;
end = i_end;
output = new List<parserObject>();
BufferBlock<string> crawler_que = new BufferBlock<string>();
BufferBlock<svnFile> parser_que = new BufferBlock<svnFile>();
var svn = crawl(crawler_que, parser_que);
var broadcaster = new ActionBlock<svnFile>(async file =>
{//tried to addapt the code from this ensure always send broadcastblock -> see link below
List<Task> todo = new List<Task>();
todo.Add(mLoc);//error cannot convert methodgroup to task
foreach (var task in todo)//error: Only assignment, call, increment, decrement, await, and new object expressions can be used as a statement?
{
task.SendAsync(file);//error cannot convert task to targetblock
}
await Task.WhenAll(todo.ToArray());
});
parser_que.LinkTo(broadcaster);
await Task.WhenAll(broadcaster, svn);//error cannot convert actionblock to task
}
private static async Task crawl(BufferBlock<string> in_queue, BufferBlock<svnFile> out_queue)
{
SvnClient client = new SvnClient();
client.Authentication.ForceCredentials(user, pw);
SvnListArgs arg = new SvnListArgs
{
Depth = SvnDepth.Children,
RetrieveEntries = SvnDirEntryItems.AllFieldsV15
};
while (await in_queue.OutputAvailableAsync())
{
string buffer_author = null;
string prev_author = null;
System.Collections.ObjectModel.Collection<SvnListEventArgs> contents;
string link = await in_queue.ReceiveAsync();
if (client.GetList(new Uri(link), arg, out contents))
{
foreach (SvnListEventArgs item in contents)
{
if (item.Entry.NodeKind == SvnNodeKind.Directory)
{
in_queue.Post(item.Path);
}
else if (item.Entry.NodeKind == SvnNodeKind.File)
{
try
{
int length = item.Name.LastIndexOf(".");
if (length <= 0)
{
continue;
}
string ext = item.Name.Substring(length);
if (extension_whitelist.Contains(ext))
{
Uri target = new Uri((repo + link));
SvnRevisionRange range;
SvnBlameArgs args = new SvnBlameArgs
{
Start = start.AddDays(-1),
End = end
};
try
{
svnFile file_instance = new svnFile();
client.Blame(target, args, delegate(object sender3, SvnBlameEventArgs e)
{
if (e.Author != null)
{
buffer_author = e.Author;
prev_author = e.Author;
}
else
{
buffer_author = prev_author;
}
file_instance.lines.Add(new svnLine(buffer_author, e.Line));
});
out_queue.Post(file_instance);
}
catch (Exception a) { Console.WriteLine("exception:" + a.Message);}
}
}
catch (Exception a)
{
}
}
}
}
}
}
private static async Task mLoc(svnFile file)
{
List<parserPart> parts = new List<parserPart>();
int find;
foreach (svnLine line in file.lines)
{
if ((find = parts.FindIndex(x => x.uploader_id == line.author)) > 0)
{
parts[find].count += 1;
}
else
{
parts.Add(new parserPart(line.author));
}
find = 0;
}
parserObject ret = new parserObject(parts, "mLoc");
await output.Add(ret);
return;
}
}
broadcastblock answer: Alternate to Dataflow BroadcastBlock with guaranteed delivery
I'm trying to download multiple files from an FTP server using WebClient.DownloadFileTaskAsync and repeatedly have the issue that several files end up being 0KB.
I've tried different suggested solutions but I just don't manage to get all files. What am I doing wrong?
class Program
{
static void Main()
{
Setup(); // This only sets up working folders etc
Task t = ProcessAsync();
t.ContinueWith(bl =>
{
if (bl.Status == TaskStatus.RanToCompletion)
Logger.Info("All done.");
else
Logger.Warn("Something went wrong.");
});
t.Wait();
}
private static void Setup() {...}
static async Task<bool> ProcessAsync()
{
var c = new Catalog();
var maxItems = Settings.GetInt("maxItems");
Logger.Info((maxItems == 0 ? "Processing all items" : "Processing first {0} items"), maxItems);
await c.ProcessCatalogAsync(maxItems);
return true; // This is not really used atm
}
}
public class Catalog
{
public async Task ProcessCatalogAsync(int maxItems)
{
var client = new Client();
var d = await client.GetFoldersAsync(_remoteFolder, maxItems);
var list = d as IList<string> ?? d.ToList();
Logger.Info("Found {0} folders", list.Count());
await ProcessFoldersAsync(list);
}
private async Task ProcessFoldersAsync(IEnumerable<string> list)
{
var client = new Client();
foreach (var mFolder in list.Select(folder => _folder + "/" + folder))
{
var items = await client.GetItemsAsync(mFolder);
var file = items.FirstOrDefault(n => n.ToLower().EndsWith(".xml"));
if (string.IsNullOrEmpty(file))
{
Logger.Warn("No metadata file found in {0}", mFolder);
continue;
}
await client.DownloadFileAsync(mFolder, file);
// Continue processing the received file...
}
}
}
public class Client
{
public async Task<IEnumerable<string>> GetItemsAsync(string subfolder)
{
return await GetFolderItemsAsync(subfolder, false);
}
public async Task<IEnumerable<string>> GetFoldersAsync(string subfolder, int maxItems)
{
var folders = await GetFolderItemsAsync(subfolder, true);
return maxItems == 0 ? folders : folders.Take(maxItems);
}
private async Task<IEnumerable<string>> GetFolderItemsAsync(string subfolder, bool onlyFolders)
{
// Downloads folder contents using WebRequest
}
public async Task DownloadFileAsync(string path, string file)
{
var remote = new Uri("ftp://" + _hostname + path + "/" + file);
var local = _workingFolder + #"\" + file;
using (var ftpClient = new WebClient { Credentials = new NetworkCredential(_username, _password) })
{
ftpClient.DownloadFileCompleted += (sender, e) => DownloadFileCompleted(sender, e, file);
await ftpClient.DownloadFileTaskAsync(remote, local);
}
}
public void DownloadFileCompleted(object sender, AsyncCompletedEventArgs e, Uri remote, string local)
{
if (e.Error != null)
{
Logger.Warn("Failed downloading\n\t{0}\n\t{1}", file, e.Error.Message);
return;
}
Logger.Info("Downloaded \n\t{1}", file);
}
}
Seems like some of your tasks aren't completed. Try do like this (I do the same when putting bunch of files to ftp)
Create array of tasks for every file being downloaded. Run them in cycle like this:
Task[] tArray = new Task[DictToUpload.Count];
foreach (var pair in DictToUpload)
{
tArray[i] = Task.Factory.StartNew(()=>{/* some stuff */});
}
await Task.WhenAll(tArray);
Use await Task.WhenAll(taskArray) instead of await each task. This guarantees all your tasks are completed.
Learn some peace of TPL ;)