I have tried many different ways to get this to work, and I am sure that is not the proper way to wire up async/await for multi threading. Here is what I have so far. It is a directory walker that I attempted to make async. I know that you don't see any async or await keywords and that is because I was unsuccessful, but that is what I am trying to do. Right now it runs in a console application but I will abstract and refactor later once I get a working POC. Any guidance is appreciated.
static void RunProgram(CancellationToken ct)
{
try
{
foreach (var dir in _directoriesToProcess)
{
var newTask = CreateNewTask(dir, ct);
_tasks.Add(newTask);
}
while (_tasks.Count > 0)
{
lock (_collectionLock)
{
var t = _tasks.Where(x => x.IsCompleted == true).ToList();
if (t != null)
foreach (var task in t)
{
_tasks.Remove(task);
}
}
}
OutputFiles();
StopAndCleanup();
}
catch (Exception ex)
{
Log(LogColor.Red, "Error: " + ex.Message, false);
_cts.Cancel();
}
}
static Task CreateNewTask(string Path, CancellationToken ct)
{
return Task.Factory.StartNew(() => GetDirectoryFiles(Path, ct), ct);
}
static void GetDirectoryFiles(string Path, CancellationToken ct)
{
if (!ct.IsCancellationRequested)
{
List<string> subDirs = new List<string>();
int currentFileCount = 0;
try
{
currentFileCount = Directory.GetFiles(Path, _fileExtension).Count();
subDirs = Directory.GetDirectories(Path).ToList();
lock (_objLock)
{
_overallFileCount += currentFileCount;
Log(LogColor.White, "- Current path: " + Path);
Log(LogColor.Yellow, "-- Sub directory count: " + subDirs.Count);
Log(LogColor.Yellow, "-- File extension: " + _fileExtension);
Log(LogColor.Yellow, "-- Current count: " + currentFileCount);
Log(LogColor.Red, "-- Running total: " + _overallFileCount);
_csvBuilder.Add(string.Format("{0},{1},{2},{3}", Path, subDirs.Count, _fileExtension, currentFileCount));
Console.Clear();
Log(LogColor.White, "Running file count: " + _overallFileCount, false, true);
}
foreach (var dir in subDirs)
{
lock (_collectionLock)
{
var newTask = CreateNewTask(dir, ct);
_tasks.Add(newTask);
}
}
}
catch (Exception ex)
{
Log(LogColor.Red, "Error: " + ex.Message, false);
_cts.Cancel();
}
}
}
I don't think there's any issue with what you're trying to do, just be cautious about uncontrolled concurrency e.g. reading too many directories at once on different threads. Context switching could end up making it slower.
Instead of doing things as side effects in your methods, try returning the collected values. e.g.
static async Task<IEnumerable<DirectoryStat>> GetDirectoryFiles(string path, string fileExtension, CancellationToken ct)
{
var thisDirectory = await Task.Run(() => /* Get directory file count and return a DirectoryStat object */);
var subDirectoriesResults = await Task.WhenAll(Directory.GetDirectories(path).Select(dir => GetDirectoryFiles(dir, fileExtension, ct)));
return (new[] { thisDirectory }).Concat(subDirectoryResults);
}
You can then iterate them later and pull the data you need from DirectoryStat (and sum your file counts as per _overallFileCount etc)
NOTE: Untested :)
You can run Synchronous Code Async with Task.Run(() => { //code });
Also change your Return Type to Taskso you can await it
I would rewrite you code as follows:
static void RunProgram(CancellationToken ct)
{
try
{
foreach (var dir in _directoriesToProcess)
{
var newTask = CreateNewTask(dir, ct);
_tasks.Add(newTask);
}
//change your while so it does not execute all the time
while (_tasks.Count > 0)
{
lock (_collectionLock)
{
var tsk = _tasks.FirstOrDefault();
if (tsk != null)
{
if (tsk.Status <= TaskStatus.Running)
await tsk;
_tasks.Remove(tsk);
}
}
}
OutputFiles();
StopAndCleanup();
}
catch (Exception ex)
{
Log(LogColor.Red, "Error: " + ex.Message, false);
_cts.Cancel();
}
}
static Task CreateNewTask(string Path, CancellationToken ct)
{
return Task.Factory.StartNew(() => GetDirectoryFiles(Path, ct), ct);
}
//always use Task (or Task<T>) as return so you can await the process
static async Task GetDirectoryFiles(string Path, CancellationToken ct)
{
if (!ct.IsCancellationRequested)
{
//Insert Magic
await Task.Run(() => {
List<string> subDirs = new List<string>();
int currentFileCount = 0;
try
{
currentFileCount = Directory.GetFiles(Path, _fileExtension).Count();
subDirs = Directory.GetDirectories(Path).ToList();
lock (_objLock)
{
_overallFileCount += currentFileCount;
Log(LogColor.White, "- Current path: " + Path);
Log(LogColor.Yellow, "-- Sub directory count: " + subDirs.Count);
Log(LogColor.Yellow, "-- File extension: " + _fileExtension);
Log(LogColor.Yellow, "-- Current count: " + currentFileCount);
Log(LogColor.Red, "-- Running total: " + _overallFileCount);
_csvBuilder.Add(string.Format("{0},{1},{2},{3}", Path, subDirs.Count, _fileExtension, currentFileCount));
Console.Clear();
Log(LogColor.White, "Running file count: " + _overallFileCount, false, true);
}
foreach (var dir in subDirs)
{
lock (_collectionLock)
{
var newTask = CreateNewTask(dir, ct);
_tasks.Add(newTask);
}
}
});
}
catch (Exception ex)
{
Log(LogColor.Red, "Error: " + ex.Message, false);
_cts.Cancel();
}
}
}
Related
I'm trying to copy large set of files from one S3 to another S3, using asynchronous method. To achieve the same, the large set of files is broken into batches and each batch is handed over to a list of async method. The issue is, each async method is not processing more than 1 file in the batch, whereas each batch contains more than 1k files, not sure why async doesn't go back to process the remaining files.
Here is the code:
public void CreateAndExecuteSpawn(string srcBucket, List<List<string>> pdfFileList, IAmazonS3 s3client)
{
int i = 0;
List<Action> actions = new List<Action>();
LambdaLogger.Log("PDF Set count: " + pdfFileList.Count.ToString());
foreach (var list in pdfFileList)
actions.Add(() => RenameFilesAsync(srcBucket, list, s3client));
foreach (var method in actions)
{
method.Invoke();
LambdaLogger.Log("Mehtod invoked: "+ i++.ToString());
}
}
public async void RenameFilesAsync(string srcBucket, List<string> pdfFiles, IAmazonS3 s3client)
{
LambdaLogger.Log("In RenameFileAsync method");
CopyObjectRequest copyRequest = new CopyObjectRequest
{
SourceBucket = srcBucket,
DestinationBucket = srcBucket
};
try
{
foreach (var file in pdfFiles)
{
if (!file.Contains("index.xml"))
{
string[] newFilename = file.Split('{');
string[] destKey = file.Split('/');
copyRequest.SourceKey = file;
copyRequest.DestinationKey = destKey[0] + "/" + destKey[1] + "/Renamed/" + newFilename[1];
LambdaLogger.Log("About to rename File: " + file);
//Here after copying one file, function doesn't return to foreach loop
CopyObjectResponse response = await s3client.CopyObjectAsync(copyRequest);
//await s3client.CopyObjectAsync(copyRequest);
LambdaLogger.Log("Rename done: ");
}
}
}
catch(Exception ex)
{
LambdaLogger.Log(ex.Message);
LambdaLogger.Log(copyRequest.DestinationKey);
}
}
public void FunctionHandler(S3Event evnt, ILambdaContext context)
{
//Some code here
CreateAndExecuteSpawn(bucket, pdfFileSet, s3client);
}
First you need to fix the batch so that it will process the batches one at a time. Avoid async void; use async Task instead:
public async Task CreateAndExecuteSpawnAsync(string srcBucket, List<List<string>> pdfFileList, IAmazonS3 s3client)
{
int i = 0;
List<Func<Task>> actions = new();
LambdaLogger.Log("PDF Set count: " + pdfFileList.Count.ToString());
foreach (var list in pdfFileList)
actions.Add(() => RenameFilesAsync(srcBucket, list, s3client));
foreach (var method in actions)
{
await method();
LambdaLogger.Log("Mehtod invoked: "+ i++.ToString());
}
}
public async Task RenameFilesAsync(string srcBucket, List<string> pdfFiles, IAmazonS3 s3client)
Then you can add asynchronous concurrency within each batch. The current code is just a foreach loop, so of course it only processes one at a time. You can change this to be asynchronously concurrent by Selecting the tasks to run and then doing a Task.WhenAll at the end:
LambdaLogger.Log("In RenameFileAsync method");
CopyObjectRequest copyRequest = new CopyObjectRequest
{
SourceBucket = srcBucket,
DestinationBucket = srcBucket
};
try
{
var tasks = pdfFiles
.Where(file => !file.Contains("index.xml"))
.Select(async file =>
{
string[] newFilename = file.Split('{');
string[] destKey = file.Split('/');
copyRequest.SourceKey = file;
copyRequest.DestinationKey = destKey[0] + "/" + destKey[1] + "/Renamed/" + newFilename[1];
LambdaLogger.Log("About to rename File: " + file);
CopyObjectResponse response = await s3client.CopyObjectAsync(copyRequest);
LambdaLogger.Log("Rename done: ");
})
.ToList();
await Task.WhenAll(tasks);
}
I am getting the following error:
Exception Info: System.OutOfMemoryException
Stack:
at System.Threading.ExecutionContext.CreateCopy()
at System.Threading.Tasks.Task.CopyExecutionContext(System.Threading.ExecutionContext)
at System.Threading.Tasks.Task.ExecuteWithThreadLocal(System.Threading.Tasks.Task ByRef)
at System.Threading.Tasks.Task.ExecuteEntry(Boolean)
at System.Threading.Tasks.Task.System.Threading.IThreadPoolWorkItem.ExecuteWorkItem()
at System.Threading.ThreadPoolWorkQueue.Dispatch()
at System.Threading._ThreadPoolWaitCallback.PerformWaitCallback()
The application is a c# windows service (using TopShelf). The app is using HanFire intitiate RunScan() method.
I do not see where in the code, but I suspect it is add to the Blocking Collection.
Code:
public void RunScan(IJobCancellationToken cancellationToken, string path, int pathId)
{
SmartScanDAO scDAO = new SmartScanDAO();
PathInfo RootPathInfo = scDAO.ScanStarted(pathId);
try
{
if (RootPathInfo == null)
{
ErrorLogger.LogEvent(RootPathInfo.Id, string.Format("Path ({1}): {0} is null", path, pathId), EventLogEntryType.Error);
return;
}
if (RootPathInfo.Status == ScanStatus.Processing)
{
ErrorLogger.LogEvent(RootPathInfo.Id, string.Format("Path {0} is currently being scanned", path), EventLogEntryType.Information);
return;
}
RootPathInfo.Status = ScanStatus.Processing;
scDAO.ScanStatus(RootPathInfo);
ErrorLogger.LogEvent(string.Format("Scanning {0}", path), EventLogEntryType.Information);
if (!Directory.Exists(path))
{
scDAO.DisableIsilonScanPathById(RootPathInfo.Id);
ErrorLogger.LogEvent(RootPathInfo.Id, "The Path does not exists: " + path, EventLogEntryType.Error);
return;
}
// Get Directories to Skip
skipPaths = scDAO.GetDuplicateIsilonScanPaths(RootPathInfo.Path);
DirectoryInfo di = new DirectoryInfo(path);
SplunkExport.DeleteFiles(path, new List<string>() { "acl", "path" }, RootPathInfo.Id);
DirectoryType = (DirectoryType)Enum.Parse(typeof(DirectoryType), RootPathInfo.DirectoryType);
RootPathInfo.Path = di.FullName.ToLower();
RootPathInfo.Owner = GetAcl(RootPathInfo.Id, RootPathInfo.Path, DirectoryType, true, false, true);// SecurityUtils.GetOwner(di.FullName, (DirectoryType)Enum.Parse(typeof(DirectoryType), RootPathInfo.DirectoryType));
RootPathInfo.Files = 0;
RootPathInfo.Size = 0;
Interlocked.Add(ref FileCount, di.GetFiles().Length);
Interlocked.Add(ref DirectorySize, (int)di.GetFiles().Sum(f => f.Length));
Task<List<string>> outputMetaDataTask = Task.Factory.StartNew(() => WriteOutput(RootPathInfo.Path, SplunkFileType.MetaData, MetaDataQueue), TaskCreationOptions.LongRunning);
Task<List<string>> outputACLTask = Task.Factory.StartNew(() => WriteOutput(RootPathInfo.Path, SplunkFileType.ACL, ACLQueue), TaskCreationOptions.LongRunning);
Action action = (() => UpdateStats(RootPathInfo.Id, MetaDataQueue, ACLQueue));
CancellationTokenSource UpdateStatsToken = new CancellationTokenSource();
IObservable<long> observable = Observable.Interval(TimeSpan.FromMinutes(10));
// Subscribe the obserable to the task on execution.
observable.Subscribe(x =>
{
Task task = new Task(action); task.Start();
// task.ContinueWith(c => resumeAction());
}, UpdateStatsToken.Token);
MetaDataQueue.Add(string.Format("\"{0}\",\"{1}\",\"{2}\",\"{3}\",\"{4}\",\"{5}\",\"{6}\",\"{7}\",\"{8}\"", DateTime.UtcNow + " UTC", di.FullName.ToLower(), 1, ((DirectoryInfo)di).GetFiles().Length, string.Format("{0:0.0}", ((DirectoryInfo)di).GetFiles().Sum(f => f.Length) / 1024 / 1024), RootPathInfo.Owner, di.LastAccessTimeUtc, di.CreationTimeUtc, di.LastWriteTimeUtc, ""));
//
// Traverse the path
GetSystemObjects(cancellationToken, di, RootPathInfo.Id, DirectoryType);
// Complete adding
MetaDataQueue.CompleteAdding();
ACLQueue.CompleteAdding();
// wait for
outputMetaDataTask.Wait();
outputACLTask.Wait();
//Send Files to Splunk
SplunkExport.CopyFilesToSplunk(outputMetaDataTask.Result, outputACLTask.Result, RootPathInfo.Id);
SmartScanDAO dao = new SmartScanDAO();
RootPathInfo.Size = DirectorySize;
}
catch (OperationCanceledException cex)
{
RootPathInfo.Status = ScanStatus.Cancelled;
if (scDAO == null)
scDAO = new SmartScanDAO();
scDAO.ScanStatus(RootPathInfo);
ErrorLogger.LogEvent(cex, RootPathInfo.Id);
}
catch (Exception ex)
{
if (RootPathInfo == null)
{
RootPathInfo = new PathInfo();
RootPathInfo.Id = pathId;
}
ErrorLogger.LogEvent(ex, RootPathInfo.Id);
RootPathInfo.Status = ScanStatus.Error;
if (scDAO == null)
scDAO = new SmartScanDAO();
scDAO.ScanStatus(RootPathInfo);
}
}
List<string> WriteOutput(string path, SplunkFileType fileType, BlockingCollection<string> queue)
{
var fileList = new List<string>();
int filecount = 1;
int linecount = 0;
int maxlinecount = 200000;
string header = (fileType.ToString() == SplunkFileType.ACL.ToString()?aclHeader:metaDataHeader);
var filepattern = SplunkExport.GetPaths(path, fileType.ToString());
string filename = string.Format(filepattern, filecount);
fileList.Add(filename);
while (true)
{
using (var strm = File.AppendText(filename))
{
foreach (var s in queue.GetConsumingEnumerable())
{
if (linecount == 0)
strm.WriteLine(header);
strm.WriteLine(s);
// if you want to make sure it's written to disk immediately,
// call Flush. This will slow performance, however.
strm.Flush();
linecount++;
if (linecount > maxlinecount)
{
linecount = 0;
filecount++;
break;
}
}
}
if (queue.IsCompleted)
break;
filename = string.Format(filepattern, filecount);
fileList.Add(filename);
}
return fileList;
}
private void GetSystemObjects(IJobCancellationToken cancellationToken, DirectoryInfo di, int pathid, DirectoryType directorytype = DirectoryType.Share)
{
long files = 0;
long size = 0;
int mb = 1024 * 1024;
try
{
Parallel.ForEach<FileSystemInfo>(di.EnumerateFileSystemInfos("*", System.IO.SearchOption.TopDirectoryOnly).Where(r => !r.FullName.Contains(#"\~snapshot") ), (FileSystemInfo fso) =>
{
if (skipPaths.Contains(fso.FullName))
return;
if (cancellationToken != null)
cancellationToken.ThrowIfCancellationRequested();
bool isDirectory = fso.EntryInfo.IsDirectory;
string owner = "";
owner = GetAcl(pathid, fso.FullName, directorytype, isDirectory, false);
try
{
if (isDirectory)
{
DirectoryInfo dis = new DirectoryInfo(fso.FullName);
lock (lckObject)
{
files = ((DirectoryInfo)fso).GetFiles().Length;
size = ((DirectoryInfo)fso).GetFiles().Sum(f => f.Length);
}
Interlocked.Add(ref FileCount, files);
Interlocked.Add(ref DirectorySize, size);
ErrorLogger.LogEvent(pathid, string.Format("Scan Directory\t{0}\t{1}", fso.FullName, files));
}
else
{
size = ((FileInfo)fso).Length;
files = 0;
}
MetaDataQueue.Add(string.Format("\"{0}\",\"{1}\",\"{2}\",\"{3}\",\"{4}\",\"{5}\",\"{6}\",\"{7}\",\"{8}\"", DateTime.UtcNow + " UTC", fso.FullName.ToLower(), (fso.EntryInfo.IsDirectory ? 1 : 0), files, string.Format("{0:0.0}", size / mb), owner, fso.LastAccessTimeUtc, fso.CreationTimeUtc, fso.LastWriteTimeUtc));
}
catch (Exception ex)
{
ErrorLogger.LogEvent(ex, pathid);
}
if (isDirectory)
GetSystemObjects(cancellationToken, (DirectoryInfo)fso, pathid, directorytype);
fso = null;
}); // end of ForEach
}
catch (Exception ex)
{
ErrorLogger.LogEvent(ex, pathid);
}
}
Any suggestions where the error may be occurring or how to get closer to the root cause. In the exception, I don't see any indication where it is.
I have a ImageProcessor class that creates a list of tasks for each of the image providers, inside of these tasks I then run a parallel.foreach loop for all the images for each provider, I want to be able to cancel all the tasks and nested parallel loops from the console, I've found example of how to cancel tasks and how to cancel parallel loops, but I am unsure as how to do it for nested processes.
Below is the code I have at the moment
From my console app:
using (IUnityContainer container = Bootstrapper.Initialise())
{
IImageProcessor processor = container.Resolve<IImageProcessor>();
processor.Container = container;
try
{
InputArguments arguments = new InputArguments(args);
if (arguments.Contains("fs"))
{
processor.Initialise(arguments["fs"]);
}
else
{
processor.Initialise();
}
processor.Run();
Console.WriteLine("\r\n\n\nPress any key to Exit");
Console.ReadLine();
return (int)ExitCode.Success;
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
Console.WriteLine("\r\n\n\nPress any key to Exit");
Console.ReadLine();
return (int)ExitCode.UnknownError;
}
}
The Run method
public void Run()
{
List<Task> tasks = new List<Task>();
foreach (IFileService fileservice in this.fileServices)
{
Task task = Task.Factory.StartNew((arg) =>
{
IFileService fs = (IFileService)arg;
string msg = $"Processing {fs.ToString()}...";
FileLogger.Write(msg, fs.ToString());
ConsoleLogger.WriteLine(msg);
fs.ProcessFiles();
//fileservice.ReprocessUnMatchedData();
}, fileservice);
tasks.Add(task);
}
Task.WaitAll(tasks.ToArray());
}
and inside each file service I have call this method:
protected bool ProcessFileRecord<T>() where T : IDataRecord
{
int matched = 0;
int notMatched = 0;
int skipped = 0;
bool result;
object lockObject = new object();
try
{
processTracker = GetTracker();
if (databaseHelper.TrackerFullyProcessed(processTracker))
{
LoggingService.Write("File already processed... Skipping.", LoggingTarget.All, serviceName);
result = true;
}
LoggingService.Write($"\r\nProcessing index file {fileRecord.IndexFileName}", LoggingTarget.File, serviceName);
Parallel.ForEach(
fileRecord.DataRecords,
new ParallelOptions() { MaxDegreeOfParallelism = Thread.CurrentThread.ManagedThreadId },
(item) =>
{
switch ((RecordProcessResult)ProcessRecord(item))
{
case RecordProcessResult.Invalid:
break;
case RecordProcessResult.Matched:
Increment(ref matched);
break;
case RecordProcessResult.NotMatched:
Increment(ref notMatched);
break;
case RecordProcessResult.Skipped:
Increment(ref skipped);
break;
default:
break;
}
lock (lockObject)
{
if ((matched + notMatched + skipped) % 100 == 0)
{
LoggingService.Write($"\tMatched: {matched}\tNot Matched: {notMatched}\tSkipped: {skipped}\t total: {matched + notMatched + skipped}", LoggingTarget.Trace & LoggingTarget.Console, serviceName);
}
}
});
LoggingService.Write($"Total Lines: {matched + notMatched + skipped} \r\nMatched: {matched} \r\nNot Matched: {notMatched} \r\nSkipped: {skipped}", LoggingTarget.All, serviceName);
this.databaseHelper.UpdateTracker(this.processTracker, matched, notMatched);
result = true;
}
catch (Exception ex)
{
LoggingService.Write($"Error processing data file:{fileRecord.IndexFileName}", LoggingTarget.All, serviceName);
LoggingService.Write($"{ex.ExceptionTreeAsString()}", LoggingTarget.All, serviceName);
LoggingService.Write($"Total Lines: {(matched + notMatched + skipped)} \r\nMatched: {matched} \r\nNot Matched: {notMatched} \r\nSkipped: {skipped}", LoggingTarget.All, serviceName);
this.databaseHelper.UpdateTracker(this.processTracker, matched, notMatched);
result = false;
}
return result;
}
Thank.
Please, look an example on this page
CancellationTokenSource
I'm trying to download multiple files from an FTP server using WebClient.DownloadFileTaskAsync and repeatedly have the issue that several files end up being 0KB.
I've tried different suggested solutions but I just don't manage to get all files. What am I doing wrong?
class Program
{
static void Main()
{
Setup(); // This only sets up working folders etc
Task t = ProcessAsync();
t.ContinueWith(bl =>
{
if (bl.Status == TaskStatus.RanToCompletion)
Logger.Info("All done.");
else
Logger.Warn("Something went wrong.");
});
t.Wait();
}
private static void Setup() {...}
static async Task<bool> ProcessAsync()
{
var c = new Catalog();
var maxItems = Settings.GetInt("maxItems");
Logger.Info((maxItems == 0 ? "Processing all items" : "Processing first {0} items"), maxItems);
await c.ProcessCatalogAsync(maxItems);
return true; // This is not really used atm
}
}
public class Catalog
{
public async Task ProcessCatalogAsync(int maxItems)
{
var client = new Client();
var d = await client.GetFoldersAsync(_remoteFolder, maxItems);
var list = d as IList<string> ?? d.ToList();
Logger.Info("Found {0} folders", list.Count());
await ProcessFoldersAsync(list);
}
private async Task ProcessFoldersAsync(IEnumerable<string> list)
{
var client = new Client();
foreach (var mFolder in list.Select(folder => _folder + "/" + folder))
{
var items = await client.GetItemsAsync(mFolder);
var file = items.FirstOrDefault(n => n.ToLower().EndsWith(".xml"));
if (string.IsNullOrEmpty(file))
{
Logger.Warn("No metadata file found in {0}", mFolder);
continue;
}
await client.DownloadFileAsync(mFolder, file);
// Continue processing the received file...
}
}
}
public class Client
{
public async Task<IEnumerable<string>> GetItemsAsync(string subfolder)
{
return await GetFolderItemsAsync(subfolder, false);
}
public async Task<IEnumerable<string>> GetFoldersAsync(string subfolder, int maxItems)
{
var folders = await GetFolderItemsAsync(subfolder, true);
return maxItems == 0 ? folders : folders.Take(maxItems);
}
private async Task<IEnumerable<string>> GetFolderItemsAsync(string subfolder, bool onlyFolders)
{
// Downloads folder contents using WebRequest
}
public async Task DownloadFileAsync(string path, string file)
{
var remote = new Uri("ftp://" + _hostname + path + "/" + file);
var local = _workingFolder + #"\" + file;
using (var ftpClient = new WebClient { Credentials = new NetworkCredential(_username, _password) })
{
ftpClient.DownloadFileCompleted += (sender, e) => DownloadFileCompleted(sender, e, file);
await ftpClient.DownloadFileTaskAsync(remote, local);
}
}
public void DownloadFileCompleted(object sender, AsyncCompletedEventArgs e, Uri remote, string local)
{
if (e.Error != null)
{
Logger.Warn("Failed downloading\n\t{0}\n\t{1}", file, e.Error.Message);
return;
}
Logger.Info("Downloaded \n\t{1}", file);
}
}
Seems like some of your tasks aren't completed. Try do like this (I do the same when putting bunch of files to ftp)
Create array of tasks for every file being downloaded. Run them in cycle like this:
Task[] tArray = new Task[DictToUpload.Count];
foreach (var pair in DictToUpload)
{
tArray[i] = Task.Factory.StartNew(()=>{/* some stuff */});
}
await Task.WhenAll(tArray);
Use await Task.WhenAll(taskArray) instead of await each task. This guarantees all your tasks are completed.
Learn some peace of TPL ;)
I have a list of task and few task are dependent on others, I would like to run the tasks now and if any of the task fails during execution, need to stop all the running task and close the application.
How can do this with TPL?
How to stop the running task?
I need to optimize the below code.
Detailed Requirement
- Start the Logon screen as a task.
o Run all other task in parallel only if the Logon is successful.
o Exit application in case of Logon failure or Cancel
- Exit application if any of the task fails
var done = new List<TaskDetails>();
var executing = new List<TaskDetails>();
var unblocked = new List<TaskDetails>();
var blocked = new List<TaskDetails>();
foreach (var startupTest in startupTests) {
if (startupTest.DependsOn == null) {
unblocked.Add(startupTest);
} else {
blocked.Add(startupTest);
}
}
IDictionary<int, TaskDetails> tasksByID = new Dictionary<int, TaskDetails>();
var tasksTPL = new Task<object>[startupTests.Count];
var taskCount = 0;
var cancellationTokenSource = new CancellationTokenSource();
var cancellationToken = cancellationTokenSource.Token;
while (done.Count < startupTests.Count) {
while (executing.Count < config.MaximumConcurrency && unblocked.Count > 0) {
TaskDetails nextTask = unblocked[0];
lock (syncLock) {
unblocked.Remove(nextTask);
executing.Add(nextTask);
}
// Execute
try {
var method = GetMethod(
nextTask.AssemblyName, nextTask.ClassName, nextTask.MethodName
);
if (method == null) {
throw new Exception("Method" + nextTask.MethodName + " not available.");
}
tasksTPL[taskCount] =
Task<object>.Factory.StartNew(() => method.Invoke(null, null),
cancellationToken);
tasksByID.Add(tasksTPL[taskCount].Id, nextTask);
tasksTPL[taskCount].ContinueWith(tsk => {
lock (syncLock) {
done.Add(tasksByID[tsk.Id]);
executing.Remove(tasksByID[tsk.Id]);
}
if (tsk.Exception != null) {
TraceAlways(
"Caught Exception while running startuptest: " +
tsk.Exception
);
}
});
taskCount++;
} catch (TargetInvocationException e) {
TraceAlways(
"Failed running " + nextTask.MethodName + " method." + e.Message);
}
}
Task.WaitAny(tasksTPL.Where(task => task != null).ToArray());
var toRemove = new List<TaskDetails>();
lock (syncLock) {
List<string> doneTaskName =
done.Select(TaskDetails => TaskDetails.Name).ToList();
foreach (var task in blocked) {
bool isBlocked = task.DependsOn.Any(dep => !doneTaskName.Contains(dep));
if (!isBlocked) {
toRemove.Add(task);
unblocked.Add(task);
}
}
foreach (var TaskDetails in toRemove) {
blocked.Remove(TaskDetails);
}
}
if (executing.Count == 0 && unblocked.Count == 0 && blocked.Count > 0) {
throw new Exception("Cyclic Dependency");
}
}
taskCount = 0;
foreach (var task in tasksTPL) {
if (
(task.Status != TaskStatus.Faulted) &&
(task.Result is bool) &&
(!(bool)task.Result)
) {
TraceAlways("Startup Test" + startupTests[taskCount].MethodName + " failed.");
if (startupTests[taskCount].ShowNotification) {
cancellationTokenSource.Cancel();
MessageBox.Show(
"An error has accoured. See log for more details.", "Startup Error"
);
}
Environment.Exit(0);
break;
}
taskCount++;
}
Below is how I would implement it conceptually (if I understood the question correctly), although I did not attempt to meet all of your detailed requirements.
The code doesn't use ContinueWith.
DoTaskAsync is an individual independent task.
DoTaskSequenceAsync is a sequence of tasks, where some tasks depend on the result of the others.
BadTaskAsync is an example of the throwing tasks, its failure should cancel all other pending tasks.
WrapAsync wraps a task with try/catch to catch the task's exceptions and raise the global cancellation from inside.
All tasks also support the global cancellation from outside.
You can compile and try it as a console app.
using System;
using System.Collections.Generic;
using System.Runtime.ExceptionServices;
using System.Threading;
using System.Threading.Tasks;
namespace MultipleTasks
{
class Program
{
class Worker
{
// a single async Task
async Task<object> DoTaskAsync(string id, CancellationToken token, int delay)
{
Console.WriteLine("Task: " + id);
await Task.Delay(delay, token); // do some work
return id;
}
// DoTaskSequenceAsync depends on Task1, Task2, Task3
async Task<object> DoTaskSequenceAsync(string id, CancellationToken token)
{
Console.WriteLine("Task: " + id);
await DoTaskAsync(id + "." + "Task1", token, 1000);
await DoTaskAsync(id + "." + "Task2", token, 2000);
await DoTaskAsync(id + "." + "Task3", token, 3000);
// do more
return id;
}
// a bad task which throws
async Task<object> BadTaskAsync(string id, CancellationToken token, int delay)
{
Console.WriteLine("Task: " + id);
await Task.Delay(delay, token);
throw new ApplicationException(id);
}
// wraps a task and requests the cancellation if the task has failed
async Task<T> WrapAsync<T>(CancellationTokenSource cts,
Func<CancellationToken, Task<T>> taskFactory)
{
try
{
return await taskFactory(cts.Token);
}
catch
{
if (!cts.IsCancellationRequested)
{
cts.Cancel(); // cancel the others
}
throw; // rethrow
}
}
// run all tasks
public async Task DoWorkAsync(CancellationToken outsideCt)
{
var tasks = new List<Task<object>>();
var cts = new CancellationTokenSource();
ExceptionDispatchInfo capturedException = null;
try
{
using (outsideCt.Register(() => cts.Cancel()))
{
// these tasks run in parallel
tasks.Add(WrapAsync(cts, (token) => DoTaskAsync("Task1", token, 500)));
tasks.Add(WrapAsync(cts, (token) => DoTaskSequenceAsync("Sequence1", token)));
tasks.Add(WrapAsync(cts, (token) => DoTaskAsync("Task2", token, 1000)));
tasks.Add(WrapAsync(cts, (token) => BadTaskAsync("BadTask", token, 1200)));
tasks.Add(WrapAsync(cts, (token) => DoTaskSequenceAsync("Sequence2", token)));
tasks.Add(WrapAsync(cts, (token) => DoTaskAsync("Task3", token, 1500)));
await Task.WhenAll(tasks.ToArray());
}
}
catch (Exception e)
{
capturedException = ExceptionDispatchInfo.Capture(e);
}
if (outsideCt.IsCancellationRequested)
{
Console.WriteLine("Cancelled from outside.");
return;
}
if (cts.IsCancellationRequested || capturedException != null)
{
if (cts.IsCancellationRequested)
{
Console.WriteLine("Cancelled by a failed task.");
// find the failed task in tasks or via capturedException
}
if (capturedException != null && capturedException.SourceException != null)
{
Console.WriteLine("Source exception: " + capturedException.SourceException.ToString());
// could rethrow the original exception:
// capturedException.Throw();
}
}
Console.WriteLine("Results:");
tasks.ForEach((task) =>
Console.WriteLine(String.Format("Status: {0}, result: {1}",
task.Status.ToString(),
task.Status == TaskStatus.RanToCompletion? task.Result.ToString(): String.Empty)));
}
}
static void Main(string[] args)
{
var cts = new CancellationTokenSource(10000);
new Worker().DoWorkAsync(cts.Token).Wait();
Console.WriteLine("Done.");
Console.ReadLine();
}
}
}