Why do some files result in 0KB when using WebClient.DownloadFileTaskAsync

Why do some files result in 0KB when using WebClient.DownloadFileTaskAsync - c#

I'm trying to download multiple files from an FTP server using WebClient.DownloadFileTaskAsync and repeatedly have the issue that several files end up being 0KB.
I've tried different suggested solutions but I just don't manage to get all files. What am I doing wrong?
class Program
{
static void Main()
{
Setup(); // This only sets up working folders etc
Task t = ProcessAsync();
t.ContinueWith(bl =>
{
if (bl.Status == TaskStatus.RanToCompletion)
Logger.Info("All done.");
else
Logger.Warn("Something went wrong.");
});
t.Wait();
}
private static void Setup() {...}
static async Task<bool> ProcessAsync()
{
var c = new Catalog();
var maxItems = Settings.GetInt("maxItems");
Logger.Info((maxItems == 0 ? "Processing all items" : "Processing first {0} items"), maxItems);
await c.ProcessCatalogAsync(maxItems);
return true; // This is not really used atm
}
}
public class Catalog
{
public async Task ProcessCatalogAsync(int maxItems)
{
var client = new Client();
var d = await client.GetFoldersAsync(_remoteFolder, maxItems);
var list = d as IList<string> ?? d.ToList();
Logger.Info("Found {0} folders", list.Count());
await ProcessFoldersAsync(list);
}
private async Task ProcessFoldersAsync(IEnumerable<string> list)
{
var client = new Client();
foreach (var mFolder in list.Select(folder => _folder + "/" + folder))
{
var items = await client.GetItemsAsync(mFolder);
var file = items.FirstOrDefault(n => n.ToLower().EndsWith(".xml"));
if (string.IsNullOrEmpty(file))
{
Logger.Warn("No metadata file found in {0}", mFolder);
continue;
}
await client.DownloadFileAsync(mFolder, file);
// Continue processing the received file...
}
}
}
public class Client
{
public async Task<IEnumerable<string>> GetItemsAsync(string subfolder)
{
return await GetFolderItemsAsync(subfolder, false);
}
public async Task<IEnumerable<string>> GetFoldersAsync(string subfolder, int maxItems)
{
var folders = await GetFolderItemsAsync(subfolder, true);
return maxItems == 0 ? folders : folders.Take(maxItems);
}
private async Task<IEnumerable<string>> GetFolderItemsAsync(string subfolder, bool onlyFolders)
{
// Downloads folder contents using WebRequest
}
public async Task DownloadFileAsync(string path, string file)
{
var remote = new Uri("ftp://" + _hostname + path + "/" + file);
var local = _workingFolder + #"\" + file;
using (var ftpClient = new WebClient { Credentials = new NetworkCredential(_username, _password) })
{
ftpClient.DownloadFileCompleted += (sender, e) => DownloadFileCompleted(sender, e, file);
await ftpClient.DownloadFileTaskAsync(remote, local);
}
}
public void DownloadFileCompleted(object sender, AsyncCompletedEventArgs e, Uri remote, string local)
{
if (e.Error != null)
{
Logger.Warn("Failed downloading\n\t{0}\n\t{1}", file, e.Error.Message);
return;
}
Logger.Info("Downloaded \n\t{1}", file);
}
}

Seems like some of your tasks aren't completed. Try do like this (I do the same when putting bunch of files to ftp)
Create array of tasks for every file being downloaded. Run them in cycle like this:
Task[] tArray = new Task[DictToUpload.Count];
foreach (var pair in DictToUpload)
{
tArray[i] = Task.Factory.StartNew(()=>{/* some stuff */});
}
await Task.WhenAll(tArray);
Use await Task.WhenAll(taskArray) instead of await each task. This guarantees all your tasks are completed.
Learn some peace of TPL ;)

Related

Unresponsive UI using background worker, processing files are very slow

Unresponsive UI using background worker, processing files are very slow.
Two different issues encountered here. GDpicture SDK is used for image processing. CPU Utilization is bare minimum, how can I maximize performance, ultimately have responsive and fast wpf application.
namespace OCR
{
public partial class MainWindow : Window
{
BackgroundWorker bw;/*= new BackgroundWorker();*/
private SynchronizationContext threadSyn = null;
string log_cap = string.Empty;
List<string> log_list = new List<string>();
string value = "Merged";
public MainWindow()
{
try
{
InitializeComponent();
InitializeBackgroundWorker();
File_process();
string configpath = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, #"path.txt");
string[] configfile = File.ReadAllLines(configpath);
if (configfile.Length > 1)
{
ip.Text = configfile[0];
op.Text = configfile[1];
ex_tb.Text = configfile[2];
Protb.Text = configfile[3];
}
cbPDFConform.Items.Clear();
for (int i = 0; i < Enum.GetNames(typeof(PdfConformance)).Length - 1; i++)
{
ComboBoxItem cbi = new ComboBoxItem();
cbi.Content = Enum.GetName(typeof(PdfConformance), (PdfConformance)i);
PdfConformance test = (PdfConformance)i;
cbi.Tag = (PdfConformance)i;
cbPDFConform.Items.Add(cbi);
}
cbPDFConform.SelectedIndex = 0;
cbProcessorCount.Items.Clear();
for (int i = 1; i <= Environment.ProcessorCount; i++)
{
cbProcessorCount.Items.Add(i.ToString());
if (Environment.ProcessorCount / 2 == i) { cbProcessorCount.SelectedIndex = i - 1; }
}
LicenseManager oLicenseManager = new LicenseManager();
oLicenseManager.RegisterKEY("");
configpath = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, #"path.txt");
configfile = File.ReadAllLines(configpath);
if (configfile.Length > 1)
{
ip.Text = configfile[0];
op.Text = configfile[1];
ex_tb.Text = configfile[2];
Protb.Text = configfile[3];
}
GrantAccess(ip.Text);
GrantAccess(op.Text);
GrantAccess(ex_tb.Text);
GrantAccess(Protb.Text);
threadSyn = SynchronizationContext.Current;
}
catch (Exception e1)
{ MessageBox.Show("e1" + e1.Message); }
}
private void InitializeBackgroundWorker()
{
bw = new BackgroundWorker();
bw.DoWork += Bw_DoWork;
bw.WorkerSupportsCancellation = true;
}
public async void File_process()
{
await Task.Run(() => converttiffpdfreducer());
}
private void Bw_DoWork(object sender, DoWorkEventArgs e)
{
this.Dispatcher.Invoke(() =>
{
try
{
using (StreamWriter sw = new StreamWriter(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, #"path.txt"), false))
{
sw.WriteLine(ip.Text);
sw.WriteLine(op.Text);
sw.WriteLine(ex_tb.Text);
sw.WriteLine(Protb.Text);
sw.Close();
}
ParallelOptions parallelOptions = new ParallelOptions();
parallelOptions.MaxDegreeOfParallelism = int.Parse(cbProcessorCount.SelectedItem.ToString());
var watch1 = new System.Diagnostics.Stopwatch();
watch1.Start();
converttiffpdfreducer();
//deletenew();
watch1.Stop();
TimeSpan ts1 = watch1.Elapsed;
ts1.ToString("mm\\:ss");
if (MergeChk.IsChecked == false)
{
value = "OCRed";
}
WriteLn("All documents have been successfully " + value + " " + ts1 + " " + DateTime.Now +" "+Environment.UserName);
}
catch (Exception DOwork)
{ MessageBox.Show("e2 " + DOwork.Message); }
});
}
private void GrantAccess(string fullPath)
{
DirectoryInfo dInfo = new DirectoryInfo(fullPath);
DirectorySecurity dSecurity = dInfo.GetAccessControl();
dSecurity.AddAccessRule(new FileSystemAccessRule(new SecurityIdentifier(WellKnownSidType.WorldSid, null), FileSystemRights.FullControl, InheritanceFlags.ObjectInherit | InheritanceFlags.ContainerInherit, PropagationFlags.NoPropagateInherit, AccessControlType.Allow));
dInfo.SetAccessControl(dSecurity);
}
private string[] mutliocr(string[] arr)
{
string box = string.Empty;
string box1 = string.Empty;
try
{
string filepath = string.Empty;
string outpath = ex_tb.Text;
if (MergeChk.IsChecked == true)
{ filepath = op.Text; }
else if (MergeChk.IsChecked == false)
{ filepath = Protb.Text; }
System.Windows.Threading.Dispatcher.CurrentDispatcher.Invoke((Action)(() =>
{
Thread.CurrentThread.IsBackground = true;
var watch2 = new System.Diagnostics.Stopwatch();
watch2.Start();
string[] getfilearray = arr;
for (int f = 0; f < getfilearray.Length; f++)
{
string dirName = Directory.GetParent(getfilearray[f]).FullName;
string folder = Directory.GetParent(getfilearray[f]).FullName;
box = Path.GetDirectoryName(getfilearray[f]);
box1 = Path.GetDirectoryName(box);
string getextension = Path.GetExtension(getfilearray[f]);
string[] newF = Directory.EnumerateFiles(dirName, "*.*", SearchOption.AllDirectories).ToArray();
string FN = Directory.GetParent(getfilearray[f]).Name;
string ocrfolder = (new FileInfo(getfilearray[f]).Directory.FullName);
string filen = Path.Combine(ocrfolder, folder, FN + "-ocr" + getextension);
string dict = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "Redist", "OCR");
if (!Directory.Exists(ocrfolder))
{
Directory.CreateDirectory(ocrfolder);
}
GrantAccess(ocrfolder);
GdPicturePDF oGdPicturePDF = new GdPicturePDF();
oGdPicturePDF.OcrPagesDone += OcrPagesDone;
void OcrPagesDone(GdPictureStatus status1)
{
if (oGdPicturePDF.SaveToFile(filen) == GdPictureStatus.OK)
{ }
else
MessageBox.Show("PDF: The OCR-ed file has failed to save. Status: " + oGdPicturePDF.GetStat().ToString());
}
GdPictureStatus status = GdPictureStatus.OK;
if (oGdPicturePDF.LoadFromFile(getfilearray[f], false) == GdPictureStatus.OK)
if (status == GdPictureStatus.OK)
{
if (oGdPicturePDF.OcrPages_4("*", 0, "eng", dict, "", 300, OCRMode.FavorSpeed, 1, true) == GdPictureStatus.OK)
if (status == GdPictureStatus.OK)
{ }
else
{ MessageBox.Show("PDF: The OCR process has failed. Status: " + status.ToString()); }
}
else
{ MessageBox.Show("PDF: The PDF file has failed to load. Status: " + status.ToString()); }
oGdPicturePDF.Dispose();
GrantAccess(getfilearray[f]);
File.Delete(getfilearray[f]);
watch2.Stop();
TimeSpan ts2 = watch2.Elapsed;
ts2.ToString("mm\\:ss");
WriteLn(" OCR pages " + filen.Replace(op.Text, "") + " " + ts2 + " " + DateTime.Now);
}
if (MergeChk.IsChecked == true)
{
foreach (string str in Directory.EnumerateFiles(op.Text, "*.pdf", SearchOption.AllDirectories).ToArray())
{
if (Path.GetFileNameWithoutExtension(str).EndsWith("-ocr"))
File.Move(str, Path.Combine(Path.GetDirectoryName(str), Path.GetFileNameWithoutExtension(str).Substring(0, Path.GetFileNameWithoutExtension(str).Length - 4) + ".pdf"));
}
}
if (MergeChk.IsChecked == false)
{
FileSystem.MoveDirectory(Protb.Text, op.Text, UIOption.AllDialogs);
Directory.CreateDirectory(Protb.Text);
string FF = string.Empty;
foreach (string str in Directory.EnumerateFiles(op.Text, "*.pdf", SearchOption.AllDirectories))
{
if (Path.GetFileNameWithoutExtension(str).EndsWith("-ocr"))
File.Move(str, Path.Combine(Path.GetDirectoryName(str), Path.GetFileNameWithoutExtension(str).Substring(0, Path.GetFileNameWithoutExtension(str).Length - 4) + ".pdf"));
}
}
}));
}
catch (Exception mul)
{
}
return arr;
}
public static string browseFolder()
{
Microsoft.Win32.OpenFileDialog dlg = new Microsoft.Win32.OpenFileDialog();
System.Windows.Forms.FolderBrowserDialog fbd = new System.Windows.Forms.FolderBrowserDialog();
System.Windows.Forms.DialogResult result = fbd.ShowDialog();
string path = string.Empty;
if (result == (System.Windows.Forms.DialogResult)MessageBoxResult.OK)
{
path = fbd.SelectedPath;
if (path[path.Length - 1] != '\\')
{
path = path + "\\";
}
}
return path;
}
private string[] converttiffpdfreducer()
{
string[] dir = null;
string box = string.Empty;
string box1 = string.Empty;
string[] gg = null;
try
{
string filepath = ip.Text;
string outpath = Protb.Text;
System.Windows.Threading.Dispatcher.CurrentDispatcher.Invoke((Action)(() =>
{
PdfConformance optPDFConform = PdfConformance.Unknown;
dir = Directory.EnumerateDirectories(filepath, "*.*", SearchOption.AllDirectories).Where(l => l.Length != 0).OrderBy(f => f).ToArray();
for (int ad = 0; ad < dir.Length; ad++)
{ string[] getfilearray = Directory.EnumerateFiles(dir[ad], "*.*", SearchOption.AllDirectories).ToArray();
if (getfilearray.Length == 0)
break;
if (getfilearray.Length != 0)
for (int f = 0; f < getfilearray.Length; f++)
{
string getext = Path.GetExtension(getfilearray[f]);
string fd = Path.GetDirectoryName(getfilearray[f]);
string op_path = fd.Replace(filepath, Protb.Text);
string getextension = Path.GetExtension(getfilearray[f]);
string dict = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "Redist", "OCR");
string outputPath = fd.Replace(filepath, outpath);
string FNAME = Path.GetFileNameWithoutExtension(getfilearray[f]);
string fn = Path.GetDirectoryName(getfilearray[f]).Replace(filepath, outpath);
string filen = Path.Combine(outputPath, fn, FNAME + ".pdf");
string savefile = Path.Combine(op_path, filen);
string pathString = getfilearray[f];
box = Path.GetDirectoryName(getfilearray[f]);
box1 = Path.GetDirectoryName(box);
using (GdPictureDocumentConverter oConverter = new GdPictureDocumentConverter()) {
GdPictureStatus status = new GdPictureStatus();
if (Path.GetExtension(getfilearray[f]).ToUpper() == ".PDF")
{
status = oConverter.LoadFromFile(getfilearray[f], GdPicture14.DocumentFormat.DocumentFormatPDF);
}
else if (Path.GetExtension(getfilearray[f]).ToUpper() == ".TIF" || Path.GetExtension(getfilearray[f]).ToUpper() == ".TIFF")
{
status = oConverter.LoadFromFile(getfilearray[f], GdPicture14.DocumentFormat.DocumentFormatTIFF);
}
else if (Path.GetExtension(getfilearray[f]).ToUpper() == ".JPG")
{
status = oConverter.LoadFromFile(getfilearray[f], GdPicture14.DocumentFormat.DocumentFormatJPEG);
}
if (status == GdPictureStatus.OK)
{
if (!Directory.Exists(op_path))
{
Directory.CreateDirectory(op_path);
}
GrantAccess(op_path);
optPDFConform = (PdfConformance)((ComboBoxItem)cbPDFConform.SelectedItem).Tag;
status = oConverter.SaveAsPDF(savefile, optPDFConform);
if (status == GdPictureStatus.OK)
{ }
else
{ }
}
else
{ }
}
}
string BOXX = box.Replace(ip.Text, Protb.Text);
string[] Arr = Directory.EnumerateFiles(BOXX, "*.pdf", SearchOption.AllDirectories).ToArray();
if (MergeChk.IsChecked == true)
{ merge(Arr); }
else if (MergeChk.IsChecked == false)
{
mutliocr(Arr);
}
}
}));
}
catch (Exception ee)
{ }
return dir;
}
private string[] merge(string[] arr)
{
string box = string.Empty;
string box1 = string.Empty; string[] gg = null;
System.Windows.Threading.Dispatcher.CurrentDispatcher.Invoke((Action)(() =>
{
box = Path.GetDirectoryName(arr[0]);
box1 = Path.GetDirectoryName(box);
string dirName = Directory.GetParent(arr[0]).FullName;
string BOXFILES = Path.GetDirectoryName(dirName);
string folder = Directory.GetParent(arr[0]).FullName.Replace(Protb.Text, op.Text);
string ocrfolder = (new FileInfo(arr[0]).Directory.FullName).Replace(Protb.Text, op.Text);
string fn = Directory.GetParent(arr[0]).Name;
string filen = Path.Combine(ocrfolder, folder, fn + ".pdf");
if (!Directory.Exists(ocrfolder))
{
Directory.CreateDirectory(ocrfolder);
}
GrantAccess(ocrfolder);
using (GdPicturePDF oGdPicturePDF = new GdPicturePDF())
{
GdPictureStatus status = oGdPicturePDF.MergeDocuments(ref arr, filen);
if (status == GdPictureStatus.OK)
{ }
else
{ }
oGdPicturePDF.Dispose();
}
Directory.Delete(box, true);
string BOXX = box.Replace(Protb.Text, op.Text);//op
string[] files = Directory.EnumerateFiles(BOXX, "*.pdf", SearchOption.AllDirectories).ToArray();
if (MergeChk.IsChecked == true)
{ mutliocr(files); }
}));
return gg;
}
private void inbtn_Click(object sender, RoutedEventArgs e)
{
try
{ ip.Text = browseFolder(); }
catch (Exception e7)
{ MessageBox.Show("e7" + e7.Message); }
}
private void obtn_Click(object sender, RoutedEventArgs e)
{
try
{ op.Text = browseFolder(); }
catch (Exception e8)
{ MessageBox.Show("e8" + e8.Message); }
}
private void start_btn_Click(object sender, RoutedEventArgs e)
{
if (!bw.IsBusy)
{
// Cancel the asynchronous operation.
this.bw.CancelAsync();
// Disable the Cancel button.
bw.RunWorkerAsync();
start_btn.Content = "Stop";
//this.Status.Content = "Running....";
}
else
{
bw.CancelAsync();
start_btn.Content = "Start";
//this.Status.Content = "Stopped....";
}
}
private void pro_btn_Click(object sender, RoutedEventArgs e)
{
try
{ Protb.Text = browseFolder(); }
catch (Exception e10)
{ MessageBox.Show("e10" + e10.Message); }
}
private void excep_Click(object sender, RoutedEventArgs e)
{
try
{ ex_tb.Text = browseFolder(); }
catch (Exception e11)
{ MessageBox.Show("e111" + e11.Message); }
}
private void WriteLn(string text)
{
logtb.Dispatcher.BeginInvoke(new Action(() =>
{
logtb.Text += text + Environment.NewLine;
}));
log_list.Add(text);
log_cap = text + Environment.NewLine + log_cap;
using (StreamWriter sw = new StreamWriter(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, #"FileProcessing_log.txt"), false))
{
foreach (string l in log_list)
sw.WriteLine(l);
sw.Close();
}
}
}
}

The point of a BackgroundWorker is to push things off the UI thread. You do that, but then in Bw_DoWork you immediately push things back the UI thread, with this.Dispatcher.Invoke, which blocks the worker thread until the work now running on the UI thread completes.
Basically: remove that this.Dispatcher.Invoke call, and just run the code on the worker. If you need to touch the UI, then you'll need to be on the UI thread for those bits, but only those buts.
Likewise, I doubt that mutliocr should be using dispatch invoke, and it certainly shouldn't change the UI thread to being a background thread (Thread.CurrentThread.IsBackground = true;).

Your code is very very smelly and ugly. You seem to follow absolutely no naming convention at all. Locals and class members are camelCase and PascalCase and some use no casing at all and others use underscores. You should really review you code carefully with many aspects in mind and clean it up. There are some redundancies too.
A really bad habbit of yours is the excessive use of the Dispatcher. For example, you create a background thread and post the complete! work of this thread back to the Dispatcher/UI thread. A huge waste of resources and elimination of any multithreading benefit.
You don't want to put all your work on the Dispatcher. You want to offload CPU intensive eork to a background thread. You want to make use of asynchronous APIs where possible. Because you want to keep the UI responsive. Dispatcher means UI thread.
Some highly important points of interest
GrantAccess implementation is a severe security violation of user rights. Don't modify access rules. Rather filter and ignore resources where the current user is not authorized for access. What is especially critical, is that you never revert the access rights to the original state.
As a general rule: don't execute IO related code (e.g., database, HTTP streams, file IO) on a background thread. Use async APIs instead. Use threading only for CPU bound work (e.g. computations, conversions).
File has an async API you should always use. For example File.OpenRead returns a FileStream which exposes a FileStream.ReadAsnyc member. if you need more convenience when handling files like read line by line, then wrap the FileStream into a StreamReader/StreamWriter and use members like StreamReader.ReadLineAsync
To avoid Dispatcher calls, pass the required UI values to the concurent methods as argument. Better use data binding (which won't eliminate the cross threading issues writing, but would make your code more readable and eliminates Dispatcher invocation to read the values). Take alook at the refactored File_process method below. It shows how to pass UI values as argument to the converttiffpdfreducer method which is then executed on a background thread.
Consider to add cancellation support to to the longrunning converttiffpdfreducer()
Avoid calling ToArray or ToList on an IEnumerable. Those methods are finalizers that immediately execute the originally deferred LINQ queries.
Don't call ToArray or ToList on the result of EnumerateFiles and its likes. Those methods are used to improve performance as they return the filesystem objects item by item. This is especially important if you recursively iterate over the complete filesystem structure. If the filesystem tree is very deep and broad, calling ToArray will force the itereation to complete and then to return all results at once. ToArray on EnumerateFiles is like using GetFiles. You should review your complete code and refactor it properly. You always use EnumerateXYZ wrong!
"The EnumerateFiles and GetFiles methods differ as follows: When you
use EnumerateFiles, you can start enumerating the collection of names
before the whole collection is returned. When you use GetFiles, you
must wait for the whole array of names to be returned before you can
access the array. Therefore, when you are working with many files and
directories, EnumerateFiles can be more efficient."
Use data binding instead of directly accessing UI elements. This allows you to read the properties on the background thread without using the Dispatcher.
Never execute long running operation from the constructor
Never call async code from the constructor
Always keep object instantiation cheep and fast and without hidden performance/resource costs
Never catch Exception. Always catch a specialized exception type.
Don't use empty catch blocks. Either handle the exception if you can or laet it crash your application to give you a chance to fix bugs. When you swallow exceptions bugs will silently creep into your application. You will have a really hard time to discover them. Logging exceptions is not considered handling - rethrow in such case.
You don't have to close a resource explicitly if you declare the resource using the using statement. The implicit call to Dispose once the instruction pointer leaves the using scope will close the resource automatically.
Implementing all the suggestion will significantly speed up your application.
I have refactored only some of your code to show how to properly use async APIs and Task.Run instead of the BackgroundWorker. I have removed every Dispatcher invocation. Instead of direct access to UI elements in order to read their values from the background thread, I have extracted those values before invoking the concurent method and passed those prefetched values as method arguments. If you would use data binding you could read the property values directly and therefore ommit the method parameters.
The MainWindow should be shown manually from App.xaml.cs to allow asynchronous and longrunning initialization of the instance. For this pattern let the class that requires such initialization implement a public InitializeAsync method that can be awaited from the caller's context. Alternatively use Lazy<T> to defer initialization when required e.g., when initialization is depending on explicit access to members.
Although the refactored code will significantly improve the applicatoin's performance, you will have do to do some important refactoring yourself (following the pattern of the already refactored code sections).
Take a look at
InitializeAsync and WriteLnAsync to learn how to use the async file IO API.
converttiffpdfreducer to learn how to use the EnumerateFiles and EnumerateDirectories methods properly in order to significantly improve the performance.
mutliocr, merge and converttiffpdfreducer to learn how pass UI element values as argument in order to avoid Dispatcher invocations.
start_btn_Click and converttiffpdfreducer to learn how to implemnent cancellation and to guard your API against calls during an uninitialized state
App.xaml
<Application Startup="Application_Startup">
</Application>
App.xaml.cs
class App : Application
{
private async void Application_Startup(object sender, StartupEventArgs e)
{
var mainWindow = new MainWindow();
// Because InitializeAsync depends on UI elements,
// we have to wait until the Ui is loaded.
mainWindow.Loaded += OnMainWindowLoaded;
// Either call Show() before initialization or after.
// If before, ensure access to uninitialized members and resources is denied
// e.g. by querying the MainWindow.IsInitialized property in public members and event handlers.
mainWindow.Show();
}
private async void OnMainWindowLoaded(object sender, EventArgs args)
=> await mainWindow.InitializeAsync();
}
MainWindow.xaml.cs
public partial class MainWindow : Window
{
public bool IsInitialized { get; private set; }
private bool IsBusy { get; set; }
private CancellationTokenSource CancellationTokenSource { get; set; }
public MainWindow()
{
InitializeComponent();
CancellationTokenSource = new CancellationTokenSource();
}
// Execute blocking initialization routines asynchronously
public async Task InitializeAsync()
{
if (IsInitialized)
{
return;
}
// Will execute the intesive CPU bound work on a background thread.
await File_process(Cancellationtoken.None);
string configpath = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, #"path.txt");
// Use async API to read/write from/to files and other IO resources
using (FileStream configfile = File.OpenRead(configpath))
{
using (var fileReader = new StreamReader(configfile))
{
var configFileContent = new List<string>();
while (!fileReader.EndOfStream)
{
string lineOfFile = await fileReader.ReadLineAsync();
configFileContent.Add(lineOfFile);
}
if (configFileContent.Any())
{
ip.Text = configFileContent[0];
GrantAccess(configFileContent[0]);
op.Text = configFileContent[1];
GrantAccess(configFileContent[1]);
ex_tb.Text = configFileContent[2];
GrantAccess(configFileContent[2]);
Protb.Text = configFileContent[3];
GrantAccess(configFileContent[3]);
}
}
}
cbPDFConform.Items.Clear();
for (int i = 0; i < Enum.GetNames(typeof(PdfConformance)).Length - 1; i++)
{
ComboBoxItem cbi = new ComboBoxItem();
cbi.Content = Enum.GetName(typeof(PdfConformance), (PdfConformance)i);
PdfConformance test = (PdfConformance)i;
cbi.Tag = (PdfConformance)i;
cbPDFConform.Items.Add(cbi);
}
cbPDFConform.SelectedIndex = 0;
cbProcessorCount.Items.Clear();
for (int i = 1; i <= Environment.ProcessorCount; i++)
{
cbProcessorCount.Items.Add(i.ToString());
if (Environment.ProcessorCount / 2 == i) { cbProcessorCount.SelectedIndex = i - 1; }
}
LicenseManager oLicenseManager = new LicenseManager();
oLicenseManager.RegisterKEY("");
threadSyn = SynchronizationContext.Current;
IsInitialiezd = true;
}
public async Task File_process(CancellationToken cancellationToken)
{
// Read UI values to avoid Dispatcher calls from the background thread
string ipText = ip.Text;
string protbText = Protb.Text;
string opText = op.Text;
// Execute the intesive CPU bound work on a background thread.
await Task.Run(() => converttiffpdfreducer(ipText, protbText, opText, cancellationToken), cancellationToken);
}
private async Task DoWorkAsync(CancellationToken cancellationToken)
{
IsBusy = true;
using (var sw = new StreamWriter(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, #"path.txt"), false))
{
await sw.WriteLineAsync(ip.Text);
await sw.WriteLineAsync(op.Text);
await sw.WriteLineAsync(ex_tb.Text);
await sw.WriteLineAsync(Protb.Text);
}
try
{
cancellationToken.ThrowIfCancellationRequested();
var watch1 = new System.Diagnostics.Stopwatch();
watch1.Start();
// Consider to add cancellation support to File_process
await File_process(cancellationToken);
watch1.Stop();
TimeSpan ts1 = watch1.Elapsed;
ts1.ToString("mm\\:ss");
if (MergeChk.IsChecked == false)
{
value = "OCRed";
}
await WriteLnAsync("All documents have been successfully " + value + " " + ts1 + " " + DateTime.Now + " " + Environment.UserName, cancellationToken);
IsBusy = false;
}
catch (OperationCanceledException)
{
IsBusy = false;
throw;
}
}
private async Task WriteLnAsync(string text, CancellationToken cancellationToken)
{
logtb.Text += text + Environment.NewLine;
log_list.Add(text);
log_cap = text + Environment.NewLine + log_cap;
using (var sw = new StreamWriter(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, #"FileProcessing_log.txt"), false))
{
foreach (string l in log_list)
{
cancellationToken.ThrowIfCancellationRequested();
await sw.WriteLineAsync(l);
}
}
}
private async void start_btn_Click(object sender, RoutedEventArgs e)
{
if (!IsInitialized)
{
return;
}
if (IsBusy)
{
// Cancel the longrunning operation.
this.CancellationTokenSource.Cancel();
}
start_btn.Content = "Start";
try
{
await DoWorkAsync(CancellationTokenSource.Token);
}
catch (OperationCanceledException)
{
CancellationTokenSource?.Dispose();
CancellationTokenSource = new CancellationTokenSource();
}
}
private void converttiffpdfreducer(
string ipText,
string protbText,
string opText,
CancellationToken cancellationToken)
{
string[] dir = null;
string box = string.Empty;
string box1 = string.Empty;
string[] gg = null;
PdfConformance optPDFConform = PdfConformance.Unknown;
foreach (var directoryPath in Directory.EnumerateDirectories(ipText, "*.*", SearchOption.AllDirectories).Where(l => l.Length != 0))
{
cancellationToken.ThrowIfCancellationRequested();
foreach (var filePath in Directory.EnumerateFiles(directoryPath, "*.*", SearchOption.AllDirectories))
{
cancellationToken.ThrowIfCancellationRequested();
string getext = Path.GetExtension(filePath);
string fd = Path.GetDirectoryName(filePath);
string op_path = fd.Replace(ipText, protbText);
string getextension = Path.GetExtension(filePath);
string dict = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "Redist", "OCR");
string outputPath = fd.Replace(ipText, protbText);
string FNAME = Path.GetFileNameWithoutExtension(filePath);
string fn = Path.GetDirectoryName(filePath).Replace(ipText, protbText);
string filen = Path.Combine(outputPath, fn, FNAME + ".pdf");
string savefile = Path.Combine(op_path, filen);
box = Path.GetDirectoryName(filePath);
box1 = Path.GetDirectoryName(box);
using (GdPictureDocumentConverter oConverter = new GdPictureDocumentConverter())
{
GdPictureStatus status = new GdPictureStatus();
if (Path.GetExtension(filePath).ToUpper() == ".PDF")
{
status = oConverter.LoadFromFile(filePath, GdPicture14.DocumentFormat.DocumentFormatPDF);
}
else if (Path.GetExtension(filePath).ToUpper() == ".TIF" || Path.GetExtension(filePath).ToUpper() == ".TIFF")
{
status = oConverter.LoadFromFile(filePath, GdPicture14.DocumentFormat.DocumentFormatTIFF);
}
else if (Path.GetExtension(filePath).ToUpper() == ".JPG")
{
status = oConverter.LoadFromFile(filePath, GdPicture14.DocumentFormat.DocumentFormatJPEG);
}
if (status == GdPictureStatus.OK)
{
if (!Directory.Exists(op_path))
{
Directory.CreateDirectory(op_path);
}
GrantAccess(op_path);
optPDFConform = (PdfConformance)((ComboBoxItem)cbPDFConform.SelectedItem).Tag;
status = oConverter.SaveAsPDF(savefile, optPDFConform);
if (status == GdPictureStatus.OK)
{ }
else
{ }
}
else
{ }
}
}
}
string BOXX = box.Replace(ipText, protbText);
// TODO::Refactor 'merge' and replace 'ToArray' with 'foreach'
string[] Arr = Directory.EnumerateFiles(BOXX, "*.pdf", SearchOption.AllDirectories).ToArray();
if (MergeChk.IsChecked == true)
{ merge(Arr, protbText); }
else if (MergeChk.IsChecked == false)
{
mutliocr(Arr);
}
}
private string[] merge(string[] arr, string protbText, string opText)
{
string box = string.Empty;
string box1 = string.Empty; string[] gg = null;
System.Windows.Threading.Dispatcher.CurrentDispatcher.Invoke((Action)(() =>
{
box = Path.GetDirectoryName(arr[0]);
box1 = Path.GetDirectoryName(box);
string dirName = Directory.GetParent(arr[0]).FullName;
string BOXFILES = Path.GetDirectoryName(dirName);
string folder = Directory.GetParent(arr[0]).FullName.Replace(protbText, opText);
string ocrfolder = (new FileInfo(arr[0]).Directory.FullName).Replace(protbText, opText);
string fn = Directory.GetParent(arr[0]).Name;
string filen = Path.Combine(ocrfolder, folder, fn + ".pdf");
if (!Directory.Exists(ocrfolder))
{
Directory.CreateDirectory(ocrfolder);
}
GrantAccess(ocrfolder);
using (GdPicturePDF oGdPicturePDF = new GdPicturePDF())
{
GdPictureStatus status = oGdPicturePDF.MergeDocuments(ref arr, filen);
if (status == GdPictureStatus.OK)
{ }
else
{ }
oGdPicturePDF.Dispose();
}
Directory.Delete(box, true);
string BOXX = box.Replace(protbText, opText);//op
string[] files = Directory.EnumerateFiles(BOXX, "*.pdf", SearchOption.AllDirectories).ToArray();
if (MergeChk.IsChecked == true)
{ mutliocr(files, protbText, opText); }
}));
return gg;
}
private string[] mutliocr(string[] arr, string protbText, string opText)
{
string box = string.Empty;
string box1 = string.Empty;
try
{
string filepath = string.Empty;
if (MergeChk.IsChecked == true)
{ filepath = opText; }
else if (MergeChk.IsChecked == false)
{ filepath = protbText; }
System.Windows.Threading.Dispatcher.CurrentDispatcher.Invoke((Action)(() =>
{
Thread.CurrentThread.IsBackground = true;
var watch2 = new System.Diagnostics.Stopwatch();
watch2.Start();
string[] getfilearray = arr;
for (int f = 0; f < getfilearray.Length; f++)
{
string dirName = Directory.GetParent(getfilearray[f]).FullName;
string folder = Directory.GetParent(getfilearray[f]).FullName;
box = Path.GetDirectoryName(getfilearray[f]);
box1 = Path.GetDirectoryName(box);
string getextension = Path.GetExtension(getfilearray[f]);
string[] newF = Directory.EnumerateFiles(dirName, "*.*", SearchOption.AllDirectories).ToArray();
string FN = Directory.GetParent(getfilearray[f]).Name;
string ocrfolder = (new FileInfo(getfilearray[f]).Directory.FullName);
string filen = Path.Combine(ocrfolder, folder, FN + "-ocr" + getextension);
string dict = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "Redist", "OCR");
if (!Directory.Exists(ocrfolder))
{
Directory.CreateDirectory(ocrfolder);
}
GrantAccess(ocrfolder);
GdPicturePDF oGdPicturePDF = new GdPicturePDF();
oGdPicturePDF.OcrPagesDone += OcrPagesDone;
void OcrPagesDone(GdPictureStatus status1)
{
if (oGdPicturePDF.SaveToFile(filen) == GdPictureStatus.OK)
{ }
else
MessageBox.Show("PDF: The OCR-ed file has failed to save. Status: " + oGdPicturePDF.GetStat().ToString());
}
GdPictureStatus status = GdPictureStatus.OK;
if (oGdPicturePDF.LoadFromFile(getfilearray[f], false) == GdPictureStatus.OK)
if (status == GdPictureStatus.OK)
{
if (oGdPicturePDF.OcrPages_4("*", 0, "eng", dict, "", 300, OCRMode.FavorSpeed, 1, true) == GdPictureStatus.OK)
if (status == GdPictureStatus.OK)
{ }
else
{ MessageBox.Show("PDF: The OCR process has failed. Status: " + status.ToString()); }
}
else
{ MessageBox.Show("PDF: The PDF file has failed to load. Status: " + status.ToString()); }
oGdPicturePDF.Dispose();
GrantAccess(getfilearray[f]);
File.Delete(getfilearray[f]);
watch2.Stop();
TimeSpan ts2 = watch2.Elapsed;
ts2.ToString("mm\\:ss");
WriteLn(" OCR pages " + filen.Replace(opText, "") + " " + ts2 + " " + DateTime.Now);
}
if (MergeChk.IsChecked == true)
{
foreach (string str in Directory.EnumerateFiles(opText, "*.pdf", SearchOption.AllDirectories).ToArray())
{
if (Path.GetFileNameWithoutExtension(str).EndsWith("-ocr"))
File.Move(str, Path.Combine(Path.GetDirectoryName(str), Path.GetFileNameWithoutExtension(str).Substring(0, Path.GetFileNameWithoutExtension(str).Length - 4) + ".pdf"));
}
}
if (MergeChk.IsChecked == false)
{
FileSystem.MoveDirectory(protbText, opText, UIOption.AllDialogs);
Directory.CreateDirectory(protbText);
string FF = string.Empty;
foreach (string str in Directory.EnumerateFiles(opText, "*.pdf", SearchOption.AllDirectories))
{
if (Path.GetFileNameWithoutExtension(str).EndsWith("-ocr"))
File.Move(str, Path.Combine(Path.GetDirectoryName(str), Path.GetFileNameWithoutExtension(str).Substring(0, Path.GetFileNameWithoutExtension(str).Length - 4) + ".pdf"));
}
}
}));
}
catch (Exception mul)
{
}
return arr;
}
}

Async function calls not executing

I'm trying to copy large set of files from one S3 to another S3, using asynchronous method. To achieve the same, the large set of files is broken into batches and each batch is handed over to a list of async method. The issue is, each async method is not processing more than 1 file in the batch, whereas each batch contains more than 1k files, not sure why async doesn't go back to process the remaining files.
Here is the code:
public void CreateAndExecuteSpawn(string srcBucket, List<List<string>> pdfFileList, IAmazonS3 s3client)
{
int i = 0;
List<Action> actions = new List<Action>();
LambdaLogger.Log("PDF Set count: " + pdfFileList.Count.ToString());
foreach (var list in pdfFileList)
actions.Add(() => RenameFilesAsync(srcBucket, list, s3client));
foreach (var method in actions)
{
method.Invoke();
LambdaLogger.Log("Mehtod invoked: "+ i++.ToString());
}
}
public async void RenameFilesAsync(string srcBucket, List<string> pdfFiles, IAmazonS3 s3client)
{
LambdaLogger.Log("In RenameFileAsync method");
CopyObjectRequest copyRequest = new CopyObjectRequest
{
SourceBucket = srcBucket,
DestinationBucket = srcBucket
};
try
{
foreach (var file in pdfFiles)
{
if (!file.Contains("index.xml"))
{
string[] newFilename = file.Split('{');
string[] destKey = file.Split('/');
copyRequest.SourceKey = file;
copyRequest.DestinationKey = destKey[0] + "/" + destKey[1] + "/Renamed/" + newFilename[1];
LambdaLogger.Log("About to rename File: " + file);
//Here after copying one file, function doesn't return to foreach loop
CopyObjectResponse response = await s3client.CopyObjectAsync(copyRequest);
//await s3client.CopyObjectAsync(copyRequest);
LambdaLogger.Log("Rename done: ");
}
}
}
catch(Exception ex)
{
LambdaLogger.Log(ex.Message);
LambdaLogger.Log(copyRequest.DestinationKey);
}
}
public void FunctionHandler(S3Event evnt, ILambdaContext context)
{
//Some code here
CreateAndExecuteSpawn(bucket, pdfFileSet, s3client);
}

First you need to fix the batch so that it will process the batches one at a time. Avoid async void; use async Task instead:
public async Task CreateAndExecuteSpawnAsync(string srcBucket, List<List<string>> pdfFileList, IAmazonS3 s3client)
{
int i = 0;
List<Func<Task>> actions = new();
LambdaLogger.Log("PDF Set count: " + pdfFileList.Count.ToString());
foreach (var list in pdfFileList)
actions.Add(() => RenameFilesAsync(srcBucket, list, s3client));
foreach (var method in actions)
{
await method();
LambdaLogger.Log("Mehtod invoked: "+ i++.ToString());
}
}
public async Task RenameFilesAsync(string srcBucket, List<string> pdfFiles, IAmazonS3 s3client)
Then you can add asynchronous concurrency within each batch. The current code is just a foreach loop, so of course it only processes one at a time. You can change this to be asynchronously concurrent by Selecting the tasks to run and then doing a Task.WhenAll at the end:
LambdaLogger.Log("In RenameFileAsync method");
CopyObjectRequest copyRequest = new CopyObjectRequest
{
SourceBucket = srcBucket,
DestinationBucket = srcBucket
};
try
{
var tasks = pdfFiles
.Where(file => !file.Contains("index.xml"))
.Select(async file =>
{
string[] newFilename = file.Split('{');
string[] destKey = file.Split('/');
copyRequest.SourceKey = file;
copyRequest.DestinationKey = destKey[0] + "/" + destKey[1] + "/Renamed/" + newFilename[1];
LambdaLogger.Log("About to rename File: " + file);
CopyObjectResponse response = await s3client.CopyObjectAsync(copyRequest);
LambdaLogger.Log("Rename done: ");
})
.ToList();
await Task.WhenAll(tasks);
}

Database async query and processing

Let me rephrase.
I have a method that generates strings(paths) after given a start string(path)
IF those paths are for a directory I want to enqueue that in the input of the method.
After processing the path synchronously, I want to get the Data and clone it async into multiple paths of a pipeline, were each path needs to get the datablock. So the Broadcastblock is out of the question (it cant send a blocking signal to the blocks before itself),
The joinblock, joining the results is relatively straight forward.
So to sum up
Is there a Block in Dataflow block, where i can access the inputqueue from the delegate, if when, how?
Is there a construct that acts like the broadcastblock but can block the blocks that came before it?
I tried doing it via almighty google:
class subversion
{
private static string repo;
private static string user;
private static string pw;
private static DateTime start;
private static DateTime end;
private static List<parserObject> output;
public static List<parserObject> svnOutputList
{
get {return output; }
}
private static List<string> extension_whitelist;
public async void run(string link, string i_user, string i_pw, DateTime i_start, DateTime i_end)
{
repo = link;
user = i_user;
pw = i_pw;
start = i_start;
end = i_end;
output = new List<parserObject>();
BufferBlock<string> crawler_que = new BufferBlock<string>();
BufferBlock<svnFile> parser_que = new BufferBlock<svnFile>();
var svn = crawl(crawler_que, parser_que);
var broadcaster = new ActionBlock<svnFile>(async file =>
{//tried to addapt the code from this ensure always send broadcastblock -> see link below
List<Task> todo = new List<Task>();
todo.Add(mLoc);//error cannot convert methodgroup to task
foreach (var task in todo)//error: Only assignment, call, increment, decrement, await, and new object expressions can be used as a statement?
{
task.SendAsync(file);//error cannot convert task to targetblock
}
await Task.WhenAll(todo.ToArray());
});
parser_que.LinkTo(broadcaster);
await Task.WhenAll(broadcaster, svn);//error cannot convert actionblock to task
}
private static async Task crawl(BufferBlock<string> in_queue, BufferBlock<svnFile> out_queue)
{
SvnClient client = new SvnClient();
client.Authentication.ForceCredentials(user, pw);
SvnListArgs arg = new SvnListArgs
{
Depth = SvnDepth.Children,
RetrieveEntries = SvnDirEntryItems.AllFieldsV15
};
while (await in_queue.OutputAvailableAsync())
{
string buffer_author = null;
string prev_author = null;
System.Collections.ObjectModel.Collection<SvnListEventArgs> contents;
string link = await in_queue.ReceiveAsync();
if (client.GetList(new Uri(link), arg, out contents))
{
foreach (SvnListEventArgs item in contents)
{
if (item.Entry.NodeKind == SvnNodeKind.Directory)
{
in_queue.Post(item.Path);
}
else if (item.Entry.NodeKind == SvnNodeKind.File)
{
try
{
int length = item.Name.LastIndexOf(".");
if (length <= 0)
{
continue;
}
string ext = item.Name.Substring(length);
if (extension_whitelist.Contains(ext))
{
Uri target = new Uri((repo + link));
SvnRevisionRange range;
SvnBlameArgs args = new SvnBlameArgs
{
Start = start.AddDays(-1),
End = end
};
try
{
svnFile file_instance = new svnFile();
client.Blame(target, args, delegate(object sender3, SvnBlameEventArgs e)
{
if (e.Author != null)
{
buffer_author = e.Author;
prev_author = e.Author;
}
else
{
buffer_author = prev_author;
}
file_instance.lines.Add(new svnLine(buffer_author, e.Line));
});
out_queue.Post(file_instance);
}
catch (Exception a) { Console.WriteLine("exception:" + a.Message);}
}
}
catch (Exception a)
{
}
}
}
}
}
}
private static async Task mLoc(svnFile file)
{
List<parserPart> parts = new List<parserPart>();
int find;
foreach (svnLine line in file.lines)
{
if ((find = parts.FindIndex(x => x.uploader_id == line.author)) > 0)
{
parts[find].count += 1;
}
else
{
parts.Add(new parserPart(line.author));
}
find = 0;
}
parserObject ret = new parserObject(parts, "mLoc");
await output.Add(ret);
return;
}
}
broadcastblock answer: Alternate to Dataflow BroadcastBlock with guaranteed delivery

Async Await - Need some guidance

I have tried many different ways to get this to work, and I am sure that is not the proper way to wire up async/await for multi threading. Here is what I have so far. It is a directory walker that I attempted to make async. I know that you don't see any async or await keywords and that is because I was unsuccessful, but that is what I am trying to do. Right now it runs in a console application but I will abstract and refactor later once I get a working POC. Any guidance is appreciated.
static void RunProgram(CancellationToken ct)
{
try
{
foreach (var dir in _directoriesToProcess)
{
var newTask = CreateNewTask(dir, ct);
_tasks.Add(newTask);
}
while (_tasks.Count > 0)
{
lock (_collectionLock)
{
var t = _tasks.Where(x => x.IsCompleted == true).ToList();
if (t != null)
foreach (var task in t)
{
_tasks.Remove(task);
}
}
}
OutputFiles();
StopAndCleanup();
}
catch (Exception ex)
{
Log(LogColor.Red, "Error: " + ex.Message, false);
_cts.Cancel();
}
}
static Task CreateNewTask(string Path, CancellationToken ct)
{
return Task.Factory.StartNew(() => GetDirectoryFiles(Path, ct), ct);
}
static void GetDirectoryFiles(string Path, CancellationToken ct)
{
if (!ct.IsCancellationRequested)
{
List<string> subDirs = new List<string>();
int currentFileCount = 0;
try
{
currentFileCount = Directory.GetFiles(Path, _fileExtension).Count();
subDirs = Directory.GetDirectories(Path).ToList();
lock (_objLock)
{
_overallFileCount += currentFileCount;
Log(LogColor.White, "- Current path: " + Path);
Log(LogColor.Yellow, "-- Sub directory count: " + subDirs.Count);
Log(LogColor.Yellow, "-- File extension: " + _fileExtension);
Log(LogColor.Yellow, "-- Current count: " + currentFileCount);
Log(LogColor.Red, "-- Running total: " + _overallFileCount);
_csvBuilder.Add(string.Format("{0},{1},{2},{3}", Path, subDirs.Count, _fileExtension, currentFileCount));
Console.Clear();
Log(LogColor.White, "Running file count: " + _overallFileCount, false, true);
}
foreach (var dir in subDirs)
{
lock (_collectionLock)
{
var newTask = CreateNewTask(dir, ct);
_tasks.Add(newTask);
}
}
}
catch (Exception ex)
{
Log(LogColor.Red, "Error: " + ex.Message, false);
_cts.Cancel();
}
}
}

I don't think there's any issue with what you're trying to do, just be cautious about uncontrolled concurrency e.g. reading too many directories at once on different threads. Context switching could end up making it slower.
Instead of doing things as side effects in your methods, try returning the collected values. e.g.
static async Task<IEnumerable<DirectoryStat>> GetDirectoryFiles(string path, string fileExtension, CancellationToken ct)
{
var thisDirectory = await Task.Run(() => /* Get directory file count and return a DirectoryStat object */);
var subDirectoriesResults = await Task.WhenAll(Directory.GetDirectories(path).Select(dir => GetDirectoryFiles(dir, fileExtension, ct)));
return (new[] { thisDirectory }).Concat(subDirectoryResults);
}
You can then iterate them later and pull the data you need from DirectoryStat (and sum your file counts as per _overallFileCount etc)
NOTE: Untested :)

You can run Synchronous Code Async with Task.Run(() => { //code });
Also change your Return Type to Taskso you can await it
I would rewrite you code as follows:
static void RunProgram(CancellationToken ct)
{
try
{
foreach (var dir in _directoriesToProcess)
{
var newTask = CreateNewTask(dir, ct);
_tasks.Add(newTask);
}
//change your while so it does not execute all the time
while (_tasks.Count > 0)
{
lock (_collectionLock)
{
var tsk = _tasks.FirstOrDefault();
if (tsk != null)
{
if (tsk.Status <= TaskStatus.Running)
await tsk;
_tasks.Remove(tsk);
}
}
}
OutputFiles();
StopAndCleanup();
}
catch (Exception ex)
{
Log(LogColor.Red, "Error: " + ex.Message, false);
_cts.Cancel();
}
}
static Task CreateNewTask(string Path, CancellationToken ct)
{
return Task.Factory.StartNew(() => GetDirectoryFiles(Path, ct), ct);
}
//always use Task (or Task<T>) as return so you can await the process
static async Task GetDirectoryFiles(string Path, CancellationToken ct)
{
if (!ct.IsCancellationRequested)
{
//Insert Magic
await Task.Run(() => {
List<string> subDirs = new List<string>();
int currentFileCount = 0;
try
{
currentFileCount = Directory.GetFiles(Path, _fileExtension).Count();
subDirs = Directory.GetDirectories(Path).ToList();
lock (_objLock)
{
_overallFileCount += currentFileCount;
Log(LogColor.White, "- Current path: " + Path);
Log(LogColor.Yellow, "-- Sub directory count: " + subDirs.Count);
Log(LogColor.Yellow, "-- File extension: " + _fileExtension);
Log(LogColor.Yellow, "-- Current count: " + currentFileCount);
Log(LogColor.Red, "-- Running total: " + _overallFileCount);
_csvBuilder.Add(string.Format("{0},{1},{2},{3}", Path, subDirs.Count, _fileExtension, currentFileCount));
Console.Clear();
Log(LogColor.White, "Running file count: " + _overallFileCount, false, true);
}
foreach (var dir in subDirs)
{
lock (_collectionLock)
{
var newTask = CreateNewTask(dir, ct);
_tasks.Add(newTask);
}
}
});
}
catch (Exception ex)
{
Log(LogColor.Red, "Error: " + ex.Message, false);
_cts.Cancel();
}
}
}

Make a C# Windows Store app wait for an unzip to finish

I'm trying to make an epub parsing app in a Windows Store with C#, and it won't wait for the archive (epubs are actually zip files) to finish extracting before it tries to parse the not-yet-existing table of contents. How do I make my app be a bit more patient?
I've tried making my UnZip() function return a task and having the epub constructor (epub is a class) use UnZip().Wait(), but that just freezes the app. What do I do?
Edit: Here's my relevant code:
public class epub
{
public string filename;
private StorageFolder unzipFolder;
private IList<epubChapter> _contents;
private bool _parsed = false;
public bool parsed { get { return _parsed; } } //Epub and contents are fully parsed
public epub(string newFilename)
{
_contents = new List<epubChapter>();
filename = newFilename;
UnZipFile().Wait();
getTableOfContents();
}
private async Task UnZipFile()
{
var sourceFolder = Windows.ApplicationModel.Package.Current.InstalledLocation;
StorageFolder localFolder = ApplicationData.Current.LocalFolder;
unzipFolder = await localFolder.CreateFolderAsync(filename, CreationCollisionOption.OpenIfExists);
using (var zipStream = await sourceFolder.OpenStreamForReadAsync(filename))
{
using (MemoryStream zipMemoryStream = new MemoryStream((int)zipStream.Length))
{
await zipStream.CopyToAsync(zipMemoryStream);
using (var archive = new ZipArchive(zipMemoryStream, ZipArchiveMode.Read))
{
foreach (ZipArchiveEntry entry in archive.Entries)
{
if (entry.Name != "")
{
using (Stream fileData = entry.Open())
{
try
{
await unzipFolder.GetFileAsync(entry.Name);
Debug.WriteLine("File at {0} already exists", entry.Name);
continue;
}
catch (FileNotFoundException)
{
Debug.WriteLine("Creating file {0}", entry.Name);
}
StorageFile outputFile = await unzipFolder.CreateFileAsync(entry.Name, CreationCollisionOption.OpenIfExists);
//Debug.WriteLine("Output file created at {0}", outputFile.Path);
using (Stream outputFileStream = await outputFile.OpenStreamForWriteAsync())
{
await fileData.CopyToAsync(outputFileStream);
await outputFileStream.FlushAsync();
}
}
if (entry.Name == "toc.ncx")
{
Debug.WriteLine("toc.ncx found in epub file; parsing it");
getTableOfContents();
}
}
}
}
}
}
}
public void getTableOfContents()
{
string contentsPath = unzipFolder.Path + #"\toc.ncx"; //The file is always called this in valid epubs
try
{
XDocument toc = XDocument.Load(contentsPath);
string nameSpace = getNameSpace(toc);
XElement navMap = firstElementNamed(toc.Root, "navMap");
parseNavPoints(navMap, nameSpace, 0);
_parsed = true;
}
catch(FileNotFoundException)
{
Debug.WriteLine("File toc.ncx was not found!");
}
}

Basically, your question seems to be: How do I call an async method from a constructor?
The short answer is that you don't, instead create an async factory method for your class.
Longer answer: As you noticed, if you call Wait(), your code will block. You can't use await, because constructors can't be async. And if you don't do anything, the constructor is going to return too early.
The solution here is to use an async factory method instead of a constructor. Something like:
private epub(string newFilename)
{
_contents = new List<epubChapter>();
filename = newFilename;
}
public static async Task<epub> CreateAsync(string newFilename)
{
var result = new epub(newFilename);
await result.UnZipFile();
result.getTableOfContents();
return result;
}
For some more information and alternative solutions, see Stephen Cleary's article about async and contructors.

Develop Reference

C# (C-Sharp) is a programming language developed by Microsoft that runs on the .NET Framework.

Why do some files result in 0KB when using WebClient.DownloadFileTaskAsync - c#

Related

Unresponsive UI using background worker, processing files are very slow

Async function calls not executing

Database async query and processing

Async Await - Need some guidance

Make a C# Windows Store app wait for an unzip to finish

Categories

Resources