using System;
using System.ComponentModel;
using System.IO;
using System.Net.Http;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using System.Windows.Forms;
using System.Collections.Generic;
using System.Collections.Concurrent;
using System.Net;
namespace Seranking_Scraper
{
public partial class Form1 : Form
{
public Form1()
{
InitializeComponent();
btnStop.Enabled = false;
}
CancellationTokenSource cts = null;
List<string> proxyList = new List<string>();
int _proxyIndex = 0;
int retries = 0;
int linesCount = 1;
int totalLinesCount;
List<Task> tasks = null;
string regex = "XXXXXX";
private static HttpClient client = null;
private async void BtnStart_Click(object sender, EventArgs e)
{
dataGridView1.Rows.Clear();
cts = new CancellationTokenSource();
btnStart.Enabled = false;
btnStop.Enabled = true;
btnExport.Enabled = false;
btnOpen.Enabled = false;
btnClear.Enabled = false;
totalLinesCount = listBox_domains.Items.Count;
List<string> urls = new List<string>();
for (int i = 0; i < listBox_domains.Items.Count; i++)
{
urls.Add(listBox_domains.Items[i].ToString());
}
if (textBox_Proxies.Text != null)
{
for (int i = 0; i < textBox_Proxies.Lines.Length; i++)
{
proxyList.Add(textBox_Proxies.Lines[i]);
}
}
var maxThreads = (int)numericUpDown1.Value;
var q = new ConcurrentQueue<string>(urls);
tasks = new List<Task>();
for (int n = 0; n < maxThreads; n++)
{
tasks.Add(Task.Run(async () =>
{
while (q.TryDequeue(out string url))
{
await SendHttpRequestAsync(url, cts.Token);
Thread.Sleep(1);
if (cts.IsCancellationRequested)
{
break;
}
foreach (Task eTask in tasks)
{
if (eTask.IsCompleted)
eTask.Dispose();
}
}
}, cts.Token));
}
await Task.WhenAll(tasks).ContinueWith((FinalWork) =>
{
Invoke(new Action(() =>
{
btnStart.Enabled = true;
btnExport.Enabled = true;
btnOpen.Enabled = true;
btnClear.Enabled = true;
timer1.Enabled = false;
timer1.Stop();
progressBar1.Style = ProgressBarStyle.Blocks;
progressBar1.Invoke((Action)(() => progressBar1.Value = 100));
if(!cts.IsCancellationRequested)
MessageBox.Show(new Form { TopMost = true }, "Completed!", "Status", MessageBoxButtons.OK, MessageBoxIcon.Exclamation);
}));
}, TaskContinuationOptions.OnlyOnRanToCompletion);
//var options = new ParallelOptions()
//{
// MaxDegreeOfParallelism = (int)numericUpDown1.Value
//};
//Parallel.For(0, listBox_domains.Items.Count, async j =>
//{
// await SendHttpRequestAsync(listBox_domains.Items[j].ToString());
// Thread.Sleep(10);
//});
}
private string GetProxy()
{
if (proxyList.Count <=0) return null;
if (_proxyIndex >= proxyList.Count - 1) _proxyIndex = 0;
var proxy = proxyList[_proxyIndex];
_proxyIndex++;
return proxy;
}
private async Task ToCsV(DataGridView dGV, string filename)
{
await Task.Yield();
string stOutput = "";
// Export titles:
string sHeaders = "";
for (int j = 0; j < dGV.Columns.Count; j++)
sHeaders = sHeaders.ToString() + Convert.ToString(dGV.Columns[j].HeaderText) + "\t";
stOutput += sHeaders + "\r\n";
// Export data.
for (int i = 0; i < dGV.RowCount - 1; i++)
{
string stLine = "";
for (int j = 0; j < dGV.Rows[i].Cells.Count; j++)
stLine = stLine.ToString() + Convert.ToString(dGV.Rows[i].Cells[j].Value) + "\t";
stOutput += stLine + "\r\n";
//progressBar1.Style = ProgressBarStyle.Blocks;
//progressBar1.Value = (i / 100) * 100;
}
Encoding utf16 = Encoding.GetEncoding(1254);
byte[] output = utf16.GetBytes(stOutput);
FileStream fs = new FileStream(filename, FileMode.Create);
BinaryWriter bw = new BinaryWriter(fs);
bw.Write(output, 0, output.Length); //write the encoded file
bw.Flush();
bw.Close();
fs.Close();
}
private async Task SendHttpRequestAsync(string url, CancellationToken ct)
{
var httpClientHandler = new HttpClientHandler
{
Proxy = new WebProxy(GetProxy(), false),
UseProxy = true
};
//httpClientHandler.MaxConnectionsPerServer = 1;
httpClientHandler.AllowAutoRedirect = true;
httpClientHandler.MaxAutomaticRedirections = 3;
try
{
using (client = new HttpClient(httpClientHandler))
{
client.Timeout = TimeSpan.FromMilliseconds(1000 * (int)numericUpDown_timeout.Value); //adjust based on your network
client.DefaultRequestHeaders.ConnectionClose = true;
ServicePointManager.DefaultConnectionLimit = 100;
//var byteArray = Encoding.ASCII.GetBytes("username:password1234");
//client.DefaultRequestHeaders.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Basic", Convert.ToBase64String(byteArray));
try
{
using (HttpResponseMessage response = await client.GetAsync("xxxx))
{
if (response.IsSuccessStatusCode)
{
using (HttpContent content = response.Content)
{
//response.Dispose();
string result = await content.ReadAsStringAsync();
Regex match = new Regex(regex, RegexOptions.Singleline);
MatchCollection collection = Regex.Matches(result, regex);
try
{
if (collection.Count > 0)
{
await AddDataToDgv(url, collection[0].ToString(), collection[1].ToString(), collection[2].ToString());
}
else if (result.Contains("No data for your search query"))
{
await AddDataToDgv(url, "nodata", "nodata", "nodata");
}
}
catch (Exception ex)
{
//MessageBox.Show(ex.ToString());
await AddDataToDgv(url, "errorCount", "errorCount", "errorCount");
}
}
}
else
{
await RetriesProxyFail(url, ct);
}
}
}catch(Exception ex)
{
await RetriesProxyFail(url, ct, ex);
client.Dispose();
}
}
}
catch (Exception ex)
{
MessageBox.Show(ex.ToString());
}
}
public async Task RetriesProxyFail(string url, CancellationToken ct, Exception ex = null)
{
client.DefaultRequestHeaders.ConnectionClose = true;
if (!cts.IsCancellationRequested)
{
retries++;
if (retries > (int)numericUpDown_Retries.Value)
{
retries = 0;
Invoke(new Action(async () =>
{
lbl_RemainingLines.Text = "Remaining Urls: " + (totalLinesCount - (dataGridView1.Rows.Count)).ToString();
await AddDataToDgv(url, "timeout", "timeout", "timeout");
}));
}
else
{
await SendHttpRequestAsync(url, ct);
}
}
}
public async Task AddDataToDgv(string url, string tcost, string tTraffic, string tValue)
{
try
{
await Task.Yield();
Invoke(new Action(() =>
{
dataGridView1.Rows.Add(url, tcost, tTraffic, tValue);
lbl_RemainingLines.Text = "Remaining Urls: " + (totalLinesCount - (dataGridView1.Rows.Count)).ToString();
if (Application.RenderWithVisualStyles)
progressBar1.Style = ProgressBarStyle.Marquee;
else
{
progressBar1.Style = ProgressBarStyle.Continuous;
progressBar1.Maximum = 100;
progressBar1.Value = 0;
timer1.Enabled = true;
}
}));
}
catch (Exception ex)
{
Invoke(new Action(async () =>
{
lbl_RemainingLines.Text = "Remaining Urls: " + (totalLinesCount - (dataGridView1.Rows.Count)).ToString();
await AddDataToDgv(url, "error", "error", "error");
}));
}
}
private void BtnOpen_Click(object sender, EventArgs e)
{
linesCount = 1;
try
{
openFileDialog1.ShowDialog();
openFileDialog1.Title = "Please select text file that contains root domains.";
openFileDialog1.DefaultExt = "txt";
openFileDialog1.Filter = "txt files (*.txt)|*.txt|All files (*.*)|*.*";
openFileDialog1.FilterIndex = 2;
openFileDialog1.CheckFileExists = true;
openFileDialog1.CheckPathExists = true;
this.openFileDialog1.Multiselect = true;
myWorker_ReadTxtFile = new BackgroundWorker();
myWorker_ReadTxtFile.DoWork += new DoWorkEventHandler(MyWorker_ReadTxtFile_DoWork);
myWorker_ReadTxtFile.RunWorkerCompleted += new RunWorkerCompletedEventHandler(MyWorker_ReadTxtFile_RunWorkerCompleted);
myWorker_ReadTxtFile.ProgressChanged += new ProgressChangedEventHandler(MyWorker_ReadTxtFile_ProgressChanged);
myWorker_ReadTxtFile.WorkerReportsProgress = true;
myWorker_ReadTxtFile.WorkerSupportsCancellation = true;
listBox_domains.Items.Clear();
foreach (String fileName_Domains in openFileDialog1.FileNames)
{
myWorker_ReadTxtFile.RunWorkerAsync(fileName_Domains);
}
}
catch (Exception ex)
{
}
}
private void OpenFileDialog1_FileOk(object sender, CancelEventArgs e)
{
}
private void MyWorker_ReadTxtFile_ProgressChanged(object sender, ProgressChangedEventArgs e)
{
listBox_domains.Items.Add(e.UserState.ToString());
lbl_totallines.Text = "TLines: " + linesCount++.ToString();
}
private void MyWorker_ReadTxtFile_RunWorkerCompleted(object sender, RunWorkerCompletedEventArgs e)
{
}
public void ReadLinesToListBox(string fileName_Domains)
{
using (StreamReader sr = File.OpenText(fileName_Domains))
{
string s = String.Empty;
while ((s = sr.ReadLine()) != null)
{
myWorker_ReadTxtFile.ReportProgress(0, s);
Thread.Sleep(1);
}
}
}
private void MyWorker_ReadTxtFile_DoWork(object sender, DoWorkEventArgs e)
{
BackgroundWorker sendingWorker = (BackgroundWorker)sender;//Capture the BackgroundWorker that fired the event
object fileName_Domains = (object)e.Argument;//Collect the array of objects the we received from the main thread
string s = fileName_Domains.ToString();//Get the string value
ReadLinesToListBox(s);
}
private void Label2_Click(object sender, EventArgs e)
{
}
private void BtnStop_Click(object sender, EventArgs e)
{
if (cts != null)
{
cts.Cancel();
cts.Dispose();
btnStart.Enabled = true;
btnStop.Enabled = false;
btnExport.Enabled = true;
btnOpen.Enabled = true;
btnClear.Enabled = true;
progressBar1.Style = ProgressBarStyle.Blocks;
progressBar1.Value = 100;
MessageBox.Show(new Form { TopMost = true }, "Cancelled!", "Status", MessageBoxButtons.OK, MessageBoxIcon.Exclamation);
}
}
private async void BtnExport_Click(object sender, EventArgs e)
{
SaveFileDialog sfd = new SaveFileDialog();
sfd.Filter = "Excel Documents (*.xls)|*.xls";
sfd.FileName = "Site Metrics";
if (sfd.ShowDialog() == DialogResult.OK)
{
await ToCsV(dataGridView1, sfd.FileName); // Here dataGridview1 is your grid view name
}
MessageBox.Show(new Form { TopMost = true }, "Exported!", "Status", MessageBoxButtons.OK, MessageBoxIcon.Exclamation);
}
private void BtnClear_Click(object sender, EventArgs e)
{
listBox_domains.Items.Clear();
dataGridView1.Rows.Clear();
}
private void Form1_Load(object sender, EventArgs e)
{
this.FormBorderStyle = System.Windows.Forms.FormBorderStyle.FixedSingle;
}
private void BtnPasteProxies_Click(object sender, EventArgs e)
{
textBox_Proxies.Text = Clipboard.GetText();
}
private void Timer1_Tick(object sender, EventArgs e)
{
progressBar1.Value += 5;
if (progressBar1.Value > 100)
progressBar1.Value = 0;
}
}
}
The above code is working fine but the problem is it slowly increasing the memory usage. When I start it use some around 40 mb slowly its increasing I tried memory tests, the increase in memory is slow but its increasing slowly.
Any one help me whats wrong in my code?
I am testing with some 4k urls, when it reaches 2k urls my memory usage is 60 MB. Still increasing slowly.
A slow memory increase is totally normal. .NET uses Garbage Collection Memory Management approach. Now while the collection runs, every other thread has to stop. This can cause human noticeable stalls. And is one big reason GC and realtime programming do not mix that well.
To avoid those stalls, the GC is lazy with running. It is aiming for only running once - at application closure. Delays will not be that noticeable then. And it might even be able to save work, as the memory will be handed back to the OS afterwards and not be reused. Short of running finalizers, there might not be much work to be done.
There are only a few things that can force it to run earlier:
there is a danger of a OutOfMemory exception. The GC will have collected and defragmented everything it possibly could before you ever get that Exception.
you call GC.Collect(); It allows you to force a collection now. Note that this only adviseable for debugging and testing. Production code should not have this
you pick a GC strategy that differs from Desktop applciation default. There are ones that do the reachabiltiy checks in anotehr thread, to have that work not pause all threads. Ones that run regulary but avoid defragmentation. Even ones that run regulary with a upper bound on runtime (to limit the stalls).
Related
I have a simple work to do, On button click it will get all Network IP addresses from a range, loop through them and put the active in a list. While performing the process a panel will be shown on which ip address will be displayed that is being checked. Code runs fine but form hangs up, application go to not responding and ip address not displayed even panel is not shown. how to do that?
my code is:
private void btnAutoSearch_Click(object sender, EventArgs e)
{
panel_Search.Visible = true; // Not Working
Cursor.Current = Cursors.WaitCursor;
string ipBase = getIPAddress();
string[] ipParts = ipBase.Split('.');
ipBase = ipParts[0] + "." + ipParts[1] + "." + ipParts[2] + ".";
for (int i = 1; i < 255; i++)
{
string ip = ipBase + i.ToString();
Ping p = new Ping();
PingReply pingresult = p.Send(ip, 100);
if (pingresult.Status == IPStatus.Success)
{
lstIPs.Add(ip);
lblConnecting.Text = ip; // Not Working
}
}
GridConn.Rows.Clear();
foreach (string s in lstIPs)
{
Device obj = new Device();
obj.IPAddress = s;
lblConnecting.Text = s;
int vSMSLimit = 0;
int.TryParse(txtSMSLimit.Text, out vSMSLimit);
obj.SMSLimit = 0;
if (obj.ConnectToHostServer())
{
obj.SendConnectionMessage();
obj.ReceiveConnectionMessage();
MyDevices.lst.Add(obj);
GridConn.Rows.Add(true, obj.IPAddress, obj.PhoneModel, obj.PhoneStatus, obj.SoftwareVersion);
}
}
Cursor.Current = Cursors.Default;
panel_Search.Visible = false;
}
The issue is that the UI must be updated from the 'Main' thread (always called UI thread), but if you do other processing on this thread then the UI itself will lock up. So you should put long running process onto another thread.
This will cause the problem however that this thread cannot update the UI, as it is not the Main/UI thread. So then you have to invoke the UI thread yourself to update it.
Luckily there is simple way of doing this using the BackgroundWorker class in C# which can help you a lot. But you need to separate out your UI and background tasks.
//Define worker
BackgroundWorker myBGWorker;
//Initalise worker and start it from your button
private void btnAutoSearch_Click(object sender, EventArgs e)
{
panel_Search.Visible = true; // Not Working
Cursor.Current = Cursors.WaitCursor;
myBGWorker = new BackgroundWorker()
//This method will execute your processing on a background thread
myBGWorker.DoWork += new DoWorkEventHandler(bgw_DoWork);
//This method will execute when your background worker finishes
//It runs on the Main/UI thread
myBGWorker.RunWorkerCompleted += new RunWorkerCompletedEventHandler(bgw_RunWorkerCompleted);
//This method will execute when the background thread wants to update the progress
//It runs on the Main/UI Thread
myBGWorker.ProgressChanged += new ProgressChangedEventHandler(bgw_ProgressChanged);
//Tell it we will be reporting progress
myBGWorker.WorkerReportsProgress = true;
//Start!
myBGWorker.RunWorkerAsync()
}
private void bgw_DoWork(object sender, DoWorkEventArgs e)
{
string ipBase = getIPAddress();
string[] ipParts = ipBase.Split('.');
ipBase = ipParts[0] + "." + ipParts[1] + "." + ipParts[2] + ".";
for (int i = 1; i < 255; i++)
{
string ip = ipBase + i.ToString();
Ping p = new Ping();
PingReply pingresult = p.Send(ip, 100);
if (pingresult.Status == IPStatus.Success)
{
lstIPs.Add(ip);
//Below reports the progress. The number shouuld represent the percentage of process, the object can be anything you want
double percentDone = (100.0 / 255.0) * i;
e.ReportProgress(Convert.ToInt32(percentDone), ip);
}
}
}
private void bgw_ProgressChanged(object sender, ProgressChangedEventArgs e)
{
lblConnecting.Text = e.UserState as string;
}
private void bgw_RunWorkerCompleted(object sender, RunWorkerCompletedEventArgs e)
{
if (e.Cancelled)
MessageBox.Show("Operation was canceled");
else if (e.Error != null)
MessageBox.Show(e.Error.Message);
else
{
GridConn.Rows.Clear();
foreach (string s in lstIPs)
{
Device obj = new Device();
obj.IPAddress = s;
lblConnecting.Text = s;
int vSMSLimit = 0;
int.TryParse(txtSMSLimit.Text, out vSMSLimit);
obj.SMSLimit = 0;
if (obj.ConnectToHostServer())
{
obj.SendConnectionMessage();
obj.ReceiveConnectionMessage();
MyDevices.lst.Add(obj);
GridConn.Rows.Add(true, obj.IPAddress, obj.PhoneModel, obj.PhoneStatus, obj.SoftwareVersion);
}
}
Cursor.Current = Cursors.Default;
panel_Search.Visible = false;
}
}
Desktop applications use one single thread to handle UI, and event handlers run synchronously on the UI thread, that is why form hangs, because it cannot handle other UI interactions while the event handler is running.
Best practice is that event handlers take UI thread for very short time.
When you need to perform long running tasks because of user interaction, you should perform the task in another thread, you should not use the UI thread for long running tasks.
You can use BackgroundWorker to move the task to another thread, but it is better to use asynchronous event handlers.
For example:
private async void myButton_Click(object sender, EventArgs e)
{
await PerformLongRunningTaskAsync();
//TODO: update UI after completing task
await Task.Run(() => PerformLongRunningTaskSynchronously());
//TODO: update UI after completing task;
}
private async Task PerformLongRunnigTaskAsync() {
//TODO: implement this async method
}
private void PerformLongRunningTaskSynchronously() {
//TODO: implement this synchronus method
}
Your code should be something like this:
private async void btnAutoSearch_Click(object sender, EventArgs e)
{
panel_Search.Visible = true; // Not Working
Cursor.Current = Cursors.WaitCursor;
string ipBase = getIPAddress();
string[] ipParts = ipBase.Split('.');
ipBase = ipParts[0] + "." + ipParts[1] + "." + ipParts[2] + ".";
for (int i = 1; i < 255; i++)
{
string ip = ipBase + i.ToString();
Ping p = new Ping();
PingReply pingresult = await p.SendPingAsync(ip, 100);
if (pingresult.Status == IPStatus.Success)
{
lstIPs.Add(ip);
lblConnecting.Text = ip; // Not Working
}
}
GridConn.Rows.Clear();
foreach (string s in lstIPs)
{
Device obj = new Device();
obj.IPAddress = s;
lblConnecting.Text = s;
int vSMSLimit = 0;
int.TryParse(txtSMSLimit.Text, out vSMSLimit);
obj.SMSLimit = 0;
if (await Task.Run(() => obj.ConnectToHostServer())
{
await Task.Run(() => obj.SendConnectionMessage());
await Task.Run(() => obj.ReceiveConnectionMessage());
MyDevices.lst.Add(obj);
GridConn.Rows.Add(true, obj.IPAddress, obj.PhoneModel, obj.PhoneStatus, obj.SoftwareVersion);
}
}
Cursor.Current = Cursors.Default;
panel_Search.Visible = false;
}
Here is how i did the work thanks to Brian Rogers,
private void btnAutoSearch_Click(object sender, EventArgs e)
{
backgroundWorker1.WorkerSupportsCancellation = true;
if (backgroundWorker1.IsBusy != true)
{
// Start the asynchronous operation.
backgroundWorker1.RunWorkerAsync();
}
}
private void backgroundWorker1_DoWork(object sender, DoWorkEventArgs e)
{
BackgroundWorker worker = sender as BackgroundWorker;
for (int j = 1; j <= 10; j++)
{
if (worker.CancellationPending == true)
{
e.Cancel = true;
break;
}
else
{
Cursor.Current = Cursors.WaitCursor;
panel_Search.Location = new Point(380, 72);
// Perform a time consuming operation and report progress.
string ipBase = getIPAddress();
string[] ipParts = ipBase.Split('.');
ipBase = ipParts[0] + "." + ipParts[1] + "." + ipParts[2] + ".";
for (int i = 1; i < 255; i++)
{
string ip = ipBase + i.ToString();
Ping p = new Ping();
PingReply pingresult = p.Send(ip, 100);
if (pingresult.Status == IPStatus.Success)
{
lstIPs.Add(ip);
lblConnecting.Text = ip;
//listBox1.Items.Add(ip);
}
}
GridConn.Rows.Clear();
foreach (string s in lstIPs)
{
Device obj = new Device();
obj.IPAddress = s;
lblConnecting.Text = s;
int vSMSLimit = 0;
int.TryParse(txtSMSLimit.Text, out vSMSLimit);
obj.SMSLimit = 0;
if (obj.ConnectToHostServer())
{
obj.SendConnectionMessage();
obj.ReceiveConnectionMessage();
MyDevices.lst.Add(obj);
GridConn.Rows.Add(true, obj.IPAddress, obj.PhoneModel, obj.PhoneStatus, obj.SoftwareVersion);
}
}
Cursor.Current = Cursors.Default;
panel_Search.Location = new Point(333, 252);
}
}
}
I am trying to cancel my tasks but it creating high cpu usage. My cpu goes to 100%
I tried different codes in internet all codes giving same output.
Start code
private async void BtnStart_Click(object sender, EventArgs e)
{
cts = new CancellationTokenSource();
btnStart.Enabled = false;
btnStop.Enabled = true;
btnExport.Enabled = false;
btnOpen.Enabled = false;
btnClear.Enabled = false;
totalLinesCount = listBox_domains.Items.Count;
List<string> urls = new List<string>();
for (int i = 0; i < listBox_domains.Items.Count; i++)
{
urls.Add(listBox_domains.Items[i].ToString());
}
if (textBox_Proxies.Text != null)
{
for (int i = 0; i < textBox_Proxies.Lines.Length; i++)
{
proxyList.Add(textBox_Proxies.Lines[i]);
}
}
var maxThreads = (int)numericUpDown1.Value;
var q = new ConcurrentQueue<string>(urls);
tasks = new List<Task>();
for (int n = 0; n < maxThreads; n++)
{
tasks.Add(Task.Run(async () =>
{
while (q.TryDequeue(out string url))
{
await SendHttpRequestAsync(url, cts.Token);
Thread.Sleep(100);
}
}));
}
await Task.WhenAll(tasks).ContinueWith((FinalWork) =>
{
btnStart.Enabled = true;
btnExport.Enabled = true;
btnOpen.Enabled = true;
btnClear.Enabled = true;
MessageBox.Show(new Form { TopMost = true }, "Completed!", "Status", MessageBoxButtons.OK, MessageBoxIcon.Exclamation);
}, TaskContinuationOptions.OnlyOnRanToCompletion);
}
Main task
private async Task SendHttpRequestAsync(string url, CancellationToken ct)
{
var httpClientHandler = new HttpClientHandler
{
Proxy = new WebProxy(GetProxy(), false),
UseProxy = true
};
try
{
using (HttpClient client = new HttpClient(httpClientHandler))
{
client.Timeout = TimeSpan.FromMilliseconds(1000 * (int)numericUpDown_timeout.Value); //adjust based on your network
//var byteArray = Encoding.ASCII.GetBytes("username:password1234");
//client.DefaultRequestHeaders.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Basic", Convert.ToBase64String(byteArray));
try
{
using (HttpResponseMessage response = await client.GetAsync("URL" + url, ct))
{
using (HttpContent content = response.Content)
{
string result = await content.ReadAsStringAsync();
Regex match = new Regex(regex, RegexOptions.Singleline);
MatchCollection collection = Regex.Matches(result, regex);
try
{
await AddDataToDgv(url, collection[0].ToString(), collection[1].ToString(), collection[2].ToString());
}
catch (Exception ex)
{
await AddDataToDgv(url, "error", "error", "error");
}
}
}
}catch(Exception ex)
{
await SendHttpRequestAsync(url, ct);
retries++;
if (retries > (int)numericUpDown_Retries.Value)
{
retries = 1;
await AddDataToDgv(url, "timeout", "timeout", "timeout");
}
}
}
}
catch (Exception ex)
{
MessageBox.Show(ex.ToString());
}
}
Stop code
private void BtnStop_Click(object sender, EventArgs e)
{
if (cts != null)
{
cts.Cancel();
btnStart.Enabled = true;
btnStop.Enabled = false;
btnExport.Enabled = true;
btnOpen.Enabled = true;
btnClear.Enabled = true;
MessageBox.Show(new Form { TopMost = true }, "Cancelled!", "Status", MessageBoxButtons.OK, MessageBoxIcon.Exclamation);
}
}
WHen I click on the stop the tasks are getting stopped but generating high cpu usage 100%.
Tried examples:
https://binary-studio.com/2015/10/23/task-cancellation-in-c-and-things-you-should-know-about-it/
https://learn.microsoft.com/en-us/dotnet/standard/parallel-programming/task-cancellation
When client.GetAsync is cancelled, it throws a TaskCanceledException. In this case, you don't want to retry.
Also, do not show MessageBoxes in your async SendHttpRequestAsync.
I'm creating a backup application. I have 5 folders in different locations what needs to be copied to one backup folder. (solved via checkbox.checked)
The main issue is that some of the folders are more than 3-4GB in total (with all subfolders) Progressbar is not updated because "maxbyte" returns a negative value. (I assume int32 overflows after 2gb file copy)
(Sorry can't explain it more in details..I'm just a beginner C# programmer)
using System;
using System.ComponentModel;
using System.Linq;
using System.Windows.Forms;
using System.IO;
using memoQClientBackuTool;
namespace memoQBackupTool
{
public partial class BGWorker : Form
{
public BGWorker()
{
InitializeComponent();
}
BackgroundWorker bw;
string source = "";
string target = "";
bool isfile = false;
int filecount = 0;
int currentFileNr = 1;
string newFilename = "";
int maxbytes = 0;
public void ShowProgress(string from, string to, bool isf)
{
InitializeComponent();
source = from;
target = to;
isfile = isf;
bw = new BackgroundWorker();
bw.DoWork += new DoWorkEventHandler(bw_DoWork);
bw.RunWorkerCompleted += new RunWorkerCompletedEventHandler(bw_RunWorkerCompleted);
bw.ProgressChanged += new ProgressChangedEventHandler(bw_ProgressChanged);
bw.WorkerReportsProgress = true;
bw.WorkerSupportsCancellation = true;
GetFileData();
}
private void GetFileData()
{
if (isfile)
{
FileInfo fi = new FileInfo(source);
maxbytes = Convert.ToInt32(fi.Length);
//
//set progress bar length
//
progressBar1.Minimum = 0;
progressBar2.Minimum = 0;
progressBar1.Maximum = maxbytes;
progressBar2.Maximum = 1;
bw.RunWorkerAsync();
}
else
{
GetDirectoryInfo(source);
//
//set progress bar length
//
progressBar1.Minimum = 0;
progressBar2.Minimum = 0;
progressBar1.Maximum = maxbytes;
progressBar2.Maximum = filecount;
bw.RunWorkerAsync();
}
}
private void GetDirectoryInfo(string source)
{
string[] files = Directory.GetFiles(source);
foreach (string file in files)
{
FileInfo fi = new FileInfo(file);
maxbytes += Convert.ToInt32(fi.Length);
filecount += 1;
}
string[] folders = Directory.GetDirectories(source);
foreach (string folder in folders)
{
GetDirectoryInfo(folder);
}
}
void bw_DoWork(object sender, DoWorkEventArgs e)
{
if (isfile)
{
FileStream fs = new FileStream(source, FileMode.Open);
long FileSize = fs.Length;
FileInfo fi = new FileInfo(source);
byte[] bBuffer = new byte[(int)FileSize];
fs.Read(bBuffer, 0, (int)FileSize);
fs.Close();
UpdateLabels(fi.FullName);
newFilename = fi.Name;
try
{
FileStream fss = new FileStream(target + "\\" + newFilename, FileMode.CreateNew);
BinaryWriter biwr = new BinaryWriter(fss);
for (int i = 0; i < bBuffer.Length; i += 15000)
{
if (i + 15000 < bBuffer.Length)
{
biwr.Write(bBuffer, i, 15000);
bw.ReportProgress(15000);
}
else
{
biwr.Write(bBuffer, i, bBuffer.Length - i);
bw.ReportProgress(bBuffer.Length - i);
}
}
biwr.Close();
fss.Close();
} catch (IOException){
MessageBox.Show("Nincs olyan...");
}
}
else
{
string[] temp = source.Split('\\');
target += "\\" + temp[temp.Count() - 1];
DirectoryInfo s = new DirectoryInfo(source);
DirectoryInfo t = new DirectoryInfo(target);
CopyDirectory(s, t);
}
if (bw.CancellationPending)
{
e.Cancel = true;
return;
}
}
public void CopyDirectory(DirectoryInfo di_source, DirectoryInfo di_target)
{
if (Directory.Exists(di_target.FullName) == false)
{
Directory.CreateDirectory(di_target.FullName);
}
foreach (FileInfo fi in di_source.GetFiles())
{
newFilename = fi.Name;
FileStream fs = new FileStream(fi.FullName, FileMode.Open);
long FileSize = fs.Length;
byte[] bBuffer = new byte[(int)FileSize];
fs.Read(bBuffer, 0, (int)FileSize);
fs.Close();
UpdateLabels(fi.FullName);
if (File.Exists(di_target.ToString() + "\\" + fi.Name))
{
Random rand = new Random();
newFilename = newFilename + "_" + rand.Next(1, 10000);
}
FileStream fss = new FileStream(di_target.ToString() + "\\" + newFilename, FileMode.CreateNew);
BinaryWriter biwr = new BinaryWriter(fss);
for (int i = 0; i < bBuffer.Length; i += 500000)
{
if (i + 500000 < bBuffer.Length)
{
biwr.Write(bBuffer, i, 500000);
bw.ReportProgress(500000);
}
else
{
biwr.Write(bBuffer, i, bBuffer.Length - i);
bw.ReportProgress(bBuffer.Length - i);
}
}
biwr.Close();
fss.Close();
}
foreach (DirectoryInfo di_SourceSubDir in di_source.GetDirectories())
{
DirectoryInfo nextSubDir = di_target.CreateSubdirectory(di_SourceSubDir.Name);
CopyDirectory(di_SourceSubDir, nextSubDir);
}
}
void bw_ProgressChanged(object sender, ProgressChangedEventArgs e)
{
progressBar1.Value += e.ProgressPercentage;
int copied = progressBar1.Value / 1024;
int total = maxbytes / 1024;
lbl_kbscopied.Text = copied + "/" + total;
}
delegate void UpdateLabelsDelegate(string filename);
void UpdateLabels(string fname)
{
if (!InvokeRequired)
{
lbl_filename.Text = "Copying: " + fname;
lbl_filenr.Text = "File: " + currentFileNr + "/" + filecount;
currentFileNr++;
progressBar2.Value += 1;
}
else
{
Invoke(new UpdateLabelsDelegate(UpdateLabels), new object[] { fname });
}
}
private void button1_Click(object sender, EventArgs e)
{
Cursor.Current = Cursors.WaitCursor;
bw.CancelAsync();
}
private void bw_RunWorkerCompleted(object sender, RunWorkerCompletedEventArgs e)
{
if (e.Cancelled)
{
Cursor.Current = Cursors.Default;
CustomMsgBox.Show("The task has been canceled", "Error", "OK");
this.Close();
}
else if (e.Error != null)
{
MessageBox.Show("Error. Details: " + (e.Error as Exception).ToString());
}
else
{
this.Close();
}
}
}
}
Report progress not in absolute values, but in relative; percentage, for example.
Let say that 1MB file is 100% of progress bar. That means that you have to initialize it with 100: progressBar1.Maximum = 100. Then every 15000 bytes that were copied is a little bit more than 1%. Instead of bw.ReportProgress(15000) do bw.ReportProgress(1).
I will suggest to use long instead of int wherever you are getting total bytes copied/uploaded so far. If I understood your code you are using int in for loop where you call ReportProgress
In top of form1
Stopwatch sw = new Stopwatch();
Then
private void btnDownload_Click(object sender, EventArgs e)
{
//urll.Add("http://download.thinkbroadband.com/1GB.zip");
label7.Text = "Downloading images";
var v = lines.Where(s => s.Contains("Name")).Select(s => s.Substring(15));
var q = lines.Where(s => s.Contains("Code")).Select(s => s.Substring(15));
var r = q.Where(c => c == "is").Concat(q.Where(c => c != "is"));
var p = v.Where(c => c == "Israel").Concat(v.Where(c => c != "Israel"));
var n = r.Count();
int i = 0;
var results = p.ToList();
downloadFile(ExtractImages.imagesUrls);
}
private Queue<string> _downloadUrls = new Queue<string>();
private async void downloadFile(IEnumerable<string> urls)
{
foreach (var url in urls)
{
_downloadUrls.Enqueue(url);
}
await DownloadFile();
}
private async Task DownloadFile()
{
if (_downloadUrls.Any())
{
WebClient client = new WebClient();
client.DownloadProgressChanged += ProgressChanged;
client.DownloadFileCompleted += Completed;
var url = _downloadUrls.Dequeue();
await client.DownloadFileTaskAsync(new Uri(url), #"C:\Temp\TestingSatelliteImagesDownload\" + count + ".jpg");
return;
}
}
private void ProgressChanged(object sender, DownloadProgressChangedEventArgs e)
{
// Calculate download speed and output it to labelSpeed.
label3.Text = string.Format("{0} kb/s", (e.BytesReceived / 1024d / sw.Elapsed.TotalSeconds).ToString("0.00"));
}
And the completed event
long bytesFromCompletedFiles = 0;
// The event that will trigger when the WebClient is completed
private async void Completed(object sender, AsyncCompletedEventArgs e)
{
if (e.Cancelled == true)
{
MessageBox.Show("Download has been canceled.");
}
else
{
ProgressBar1.Value = 100;
count++;
bytesFromCompletedFiles += totalBytes[count -1];
await DownloadFile();
}
}
The problem is i never start the sw(Stopwatch) and never stop/reset it anywhere.
So in the line:
label3.Text = string.Format("{0} kb/s", (e.BytesReceived / 1024d / sw.Elapsed.TotalSeconds).ToString("0.00"));
I see the sign of infinity on the sw
The question is where should i start/stop/reset the sw ?
Now when i'm running the program i don't see anything on label3 i guess since the infinity and that i'm not starting the sw(Stopwatch).
It'd make sense to start the Stopwatch just before you start the download, since if you start it earlier this would yield wrong results
private async Task DownloadFile()
{
if (_downloadUrls.Any())
{
WebClient client = new WebClient();
[...]
sw = Stopwatch.StartNew();
await client.DownloadFileTaskAsync(new Uri(url), #"C:\Temp\TestingSatelliteImagesDownload\" + count + ".jpg");
return;
}
}
And to stop it after the download has been cancelled or is completed
// The event that will trigger when the WebClient is completed
private async void Completed(object sender, AsyncCompletedEventArgs e)
{
if ([...])
{
[...]
}
else
{
[...]
}
sw.Stop();
}
As title, how to release thread is required in multiple thread ?
Ex : I have 5 thread is waiting. I only want thread position 3 is released
I use autoresetevent/manualresetevent/monitor.wait and monitor.pulse but all release thread follow FIFO
help me !!!
UPDATED:
This is form1:
private BackgroundWorker[] threadArray;
public static ManualResetEvent _manualResetEvent = new ManualResetEvent(false);
private void btn_Start_Scraping_Click(object sender, EventArgs e)
{
threadArray = new BackgroundWorker[listView_Site.Items.Count];
for (var f = 0; f < listView_Site.Items.Count; f++)
{
threadArray[f] = new BackgroundWorker();
threadArray[f].DoWork += new DoWorkEventHandler(BackgroundWorkerFilesDoWork);
threadArray[f].RunWorkerCompleted += new RunWorkerCompletedEventHandler(BackgroundWorkerFilesRunWorkerCompleted);
threadArray[f].ProgressChanged += new ProgressChangedEventHandler(BackgroundWorkerFilesProgressChanged);
threadArray[f].WorkerReportsProgress = true;
threadArray[f].WorkerSupportsCancellation = true;
threadArray[f].RunWorkerAsync(listView_Site.Items[f].Tag.ToString());
}
}
private void BackgroundWorkerFilesDoWork(object sender, DoWorkEventArgs e)
{
....// all above code is fine
requestCaptcha = (HttpWebRequest)WebRequest.Create(uriCaptchaPage);
requestCaptcha.Pipelined = true;
requestCaptcha.KeepAlive = true;
requestCaptcha.AllowAutoRedirect = false;
//request.Proxy = null;
requestCaptcha.Timeout = 60000;
requestCaptcha.Headers.Add(HttpRequestHeader.AcceptEncoding, "gzip,deflate");
requestCaptcha.CookieContainer = sessionID;
request.ServicePoint.Expect100Continue = false;
requestCaptcha.Method = "GET";
requestCaptcha.Referer = uriloginPage.AbsoluteUri;
//get response.
responseCaptcha = (HttpWebResponse)requestCaptcha.GetResponse();
Stream imagestream = responseCaptcha.GetResponseStream();
if (imagestream != null)
{
Image image = Image.FromStream(imagestream);
if (Directory.Exists(Application.StartupPath + "\\Captcha") == false)
{
Directory.CreateDirectory(Application.StartupPath + "\\Captcha");
}
switch (responseCaptcha.ContentType)
{
case "image/jpeg":
{
saveLocation += ".jpg";
if (File.Exists(saveLocation))
{
File.Delete(saveLocation);
}
image.Save(saveLocation,ImageFormat.Jpeg);
break;
}
case "image/gif":
{
saveLocation += ".gif";
if (File.Exists(saveLocation))
{
File.Delete(saveLocation);
}
image.Save(saveLocation, ImageFormat.Gif);
break;
}
case "image/png":
{
saveLocation += ".png";
if (File.Exists(saveLocation))
{
File.Delete(saveLocation);
}
image.Save(saveLocation, ImageFormat.Png);
break;
}
}
//show form2 to enter captcha
lock (_lockObj)
{
if (Application.OpenForms.OfType<frmCaptchaQuestion>().Any() == false)
{
DoOnUIThread(delegate()
{
_formCaptchaQuestion.CreatePanelCaptcha(uriloginPage, saveLocation,idHomePage);
_formCaptchaQuestion.Show();
});
}
else
{
DoOnUIThread(() => _formCaptchaQuestion.CreatePanelCaptcha(uriloginPage, saveLocation,idHomePage));
}
}
//wait and get captcha from form2 and only run thread is required
//this is my problem <<<<========================================
lock (_lockObj)
{
//_manualResetEvent.WaitOne(30000);
//_manualResetEvent.Reset();
//if (clsValueStatic.CaptchaText != null)
//{
// foreach (var id in clsValueStatic.CaptchaText)
// {
while (!_go)
{
Monitor.Wait(_lockObj);
}
// }
//}
}
requestCaptcha = (HttpWebRequest)WebRequest.Create(uriActionLoginPage);
requestCaptcha.Pipelined = true;
requestCaptcha.KeepAlive = true;
requestCaptcha.AllowAutoRedirect = false;
//request.Proxy = null;
requestCaptcha.Timeout = 60000;
requestCaptcha.Headers.Add(HttpRequestHeader.AcceptEncoding, "gzip,deflate");
requestCaptcha.CookieContainer = sessionID;
request.ServicePoint.Expect100Continue = false;
requestCaptcha.Method = "GET";
}
Form2:
private void textBoxCaptcha_KeyDown(object sender, KeyEventArgs e)
{
if (e.KeyCode == Keys.Enter)
{
var textBox = sender as TextBoxX;
if (textBox != null)
{
clsValueStatic.CaptchaText = textBox.Text.Trim();
textBox.Parent.Parent.Dispose();
frmScrapingAnalysis._manualResetEvent.Set();
}
}
}
PS : Form1 have 1 button to start multiple backgroundworker and show form2 then all backgroundworker wait to get text captcha of textbox from form2
my way want when user enter text of backgroundworker is shown on form2 then only the backgroundworker is released. All other backgroundworker still wait