I am trying to use multithreading in my application. The method test5 tries to fetch some content from Internet, while the main thread waits for all threads to finish before continuing with other work.
But my main thread doesn't come back after calling test5, and my console lines Done Inside!! and thread all got back!! are never reached.
How can I resolve this issue?
class Program
{
static void Main(string[] args)
{
string[] url =
{
"http://...", "http://...", "http://...", "http://...", "http://..."
};
test5(url);
Console.WriteLine("thread all got back!!");
// Do some other work after all threads come back
Console.ReadLine();
}
private static void test5(string[] _url)
{
int numThreads = _url.Length;
ManualResetEvent resetEvent = new ManualResetEvent(false);
int toProcess = numThreads;
for (int i = 0; i < numThreads - 1; i++)
{
new Thread( delegate() {
testWebWorking(_url[i]);
if (Interlocked.Decrement(ref toProcess) == 0)
resetEvent.Set();
}).Start();
}
resetEvent.WaitOne();
Console.WriteLine("Done inside!!");
}
private static void test6(string[] _url)
{
int numThreads = _url.Length;
var countdownEvent = new CountdownEvent(numThreads);
for (int i = 0; i < numThreads - 1; i++)
{
new Thread(delegate() {
testWebWorking(_url[i]);
countdownEvent.Signal();
}).Start();
}
countdownEvent.Wait();
Console.WriteLine("Done inside!!");
}
private static void testWebWorking(object url)
{
Console.WriteLine("start {0}", Thread.CurrentThread.ManagedThreadId);
string uri = (string)url;
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(uri);
request.KeepAlive = true;
request.Timeout = 5000;
request.ReadWriteTimeout = 5000;
request.Proxy = null;
try
{
using (HttpWebResponse response = (HttpWebResponse)request.GetResponse())
{
//Console.WriteLine(response.ContentType + "; uri = " + uri);
Stream receiveStream = response.GetResponseStream();
Encoding encode = System.Text.Encoding.GetEncoding("utf-8");
// Pipes the stream to a higher level stream reader with the required encoding format.
StreamReader readStream = new StreamReader(receiveStream, encode);
//Console.WriteLine("\r\nResponse stream received.");
Char[] read = new Char[256];
// Reads 256 characters at a time.
int count = readStream.Read(read, 0, 256);
//Console.WriteLine("HTML...\r\n");
String str = "";
while (count > 0)
{
// Dumps the 256 characters on a string and displays the string to the console.
str = new String(read, 0, count);
//Console.Write(str);
count = readStream.Read(read, 0, 256);
}
//Console.WriteLine(str);
// Releases the resources of the response.
response.Close();
// Releases the resources of the Stream.
readStream.Close();
Console.WriteLine("end {0}", Thread.CurrentThread.ManagedThreadId);
}
}
catch (WebException ex)
{
//Console.WriteLine(ex.GetBaseException().ToString() );
//Console.WriteLine(url);
Console.WriteLine("time out !!");
}
finally
{
request.Abort();
request = null;
GC.Collect();
}
}
}
Look at this:
for (int i = 0; i < numThreads - 1; i++)
You're only starting numThreads - 1 threads. Your counter starts at numThreads and counts down, so it'll only ever reach 1, not 0.
As an aside, this is also broken::
for (int i = 0; i < numThreads - 1; i++)
{
new Thread( delegate()
{
testWebWorking(_url[i]);
...
}
...
}
Here, you're capturing the variable i within the delegate, so it will have whatever value i has when you execute it. You may very well test the same URL more than once, and skip other URLs. Instead, you should copy the value of i into a variable within the loop, so that you capture a different variable each time:
// Fixed the loop boundary as well
for (int i = 0; i < numThreads; i++)
{
int copy = i;
new Thread(() => // Use a lambda expression for brevity
{
// Fix naming convention for method name, too...
TestWebWorking(_url[copy]);
if (Interlocked.Decrement(ref toProcess) == 0))
{
resetEvent.Set();
}
}).Start()
}
Both of your methods have the same set of problems.
Personally I wouldn't use a for loop here anyway - I'd use a foreach loop:
private static void TestWithResetEvent(string[] urls)
{
ManualResetEvent resetEvent = new ManualResetEvent(false);
int counter = urls.Length;
foreach (string url in urls)
{
string copy = url;
Thread t = new Thread(() =>
{
TestWebWorking(copy);
if (Interlocked.Decrement(ref toProcess) == 0))
{
resetEvent.Set();
}
});
t.Start();
}
resetEvent.WaitOne();
Console.WriteLine("Done inside!!");
}
Alternatively, it would be simpler to use Parallel.ForEach, which is designed for exactly this sort of thing.
Related
I've written such producer/consumer code, which should generate big file filled with random data
class Program
{
static void Main(string[] args)
{
Random random = new Random();
String filename = #"d:\test_out";
long numlines = 1000000;
var buffer = new BlockingCollection<string[]>(10); //limit to not get OOM.
int arrSize = 100; //size of each string chunk in buffer;
String[] block = new string[arrSize];
Task producer = Task.Factory.StartNew(() =>
{
long blockNum = 0;
long lineStopped = 0;
for (long i = 0; i < numlines; i++)
{
if (blockNum == arrSize)
{
buffer.Add(block);
blockNum = 0;
lineStopped = i;
}
block[blockNum] = random.Next().ToString();
//null is sign to stop if last block is not fully filled
if (blockNum < arrSize - 1)
{
block[blockNum + 1] = null;
}
blockNum++;
};
if (lineStopped < numlines)
{
buffer.Add(block);
}
buffer.CompleteAdding();
}, TaskCreationOptions.LongRunning);
Task consumer = Task.Factory.StartNew(() =>
{
using (var outputFile = new StreamWriter(filename))
{
foreach (string[] chunk in buffer.GetConsumingEnumerable())
{
foreach (string value in chunk)
{
if (value == null) break;
outputFile.WriteLine(value);
}
}
}
}, TaskCreationOptions.LongRunning);
Task.WaitAll(producer, consumer);
}
}
And it does what is intended to do. But for some unknown reason it produces only ~550000 strings, not 1000000 and I can not understand why this is happening.
Can someone point on my mistake? I really don't get what's wrong with this code.
The buffer
String[] block = new string[arrSize];
is declared outside the Lambda. That means it is captured and re-used.
That would normally go unnoticed (you would just write out the wrong random data) but because your if (blockNum < arrSize - 1) is placed inside the for loop you regularly write a null into the shared buffer.
Exercise, instead of:
block[blockNum] = random.Next().ToString();
use
block[blockNum] = i.ToString();
and predict and verify the results.
How can I add System.Timer to System.Threading.RegisterWaitForSingleObject to run my thread after an interval time? In the code below, how do I specify the System.Timer[] timer object? Thanks.
int lvCtlrIdStart = 0;
int lvCtlrIdEnd = 0;
object s_range = null;
for (int i = 0; i < NbrOfThrds - 1; i++)
{
m_WorkerEvtThreads[i] = new ManualResetEvent(false);
int dueTime = int.Parse(lvTask.ctlrArray[i, 3].ToString());
lvCtlrIdStart = int.Parse(lvTask.ctlrArray[i, 0].ToString());
lvCtlrIdEnd = int.Parse(lvTask.ctlrArray[i, 1].ToString());
s_range = $"{lvCtlrIdStart},{lvCtlrIdEnd}";
dueTime = int.Parse(lvTask.ctlrArray[i, 3].ToString());
Action<int> m_workerThread = (sqm) =>
{
lvTask.WaitHandle = ThreadPool.RegisterWaitForSingleObject(m_SpecialWorkerEventThread,
new WaitOrTimerCallback(m_SendQueuedMsgs), s_range, 0, true);
};
timer[i].Change(dueTime, Timeout.Infinite);
}
I found a way to resolve this:
int dueTime = int.Parse(lvTask.ctlrArray[0, 3].ToString());
int lvCtlrIdStart = 0;
int lvCtlrIdEnd = 0;
object s_range = $"{lvCtlrIdStart},{lvCtlrIdEnd}";
bool bIntvlStart = false;
m_WorkerEvtThreads = new AutoResetEvent[NbrThrds]; // reset ALL events before looping into the next sets of Threadpool
CancellationTokenSource cts = new CancellationTokenSource(); // Create the token source
for (int i = 0; i < NbrThrds; i++) // Iterate based on the number of tasks to run
{
try
{
int intvlStart = 1000 * i; // delay before my callback method is invoked..
dueTime = bIntvlStart ? int.Parse(lvTask.ctlrArray[i, 3].ToString()) : intvlStart; // first loop is (1000 * loop_count) sec stagger run, otherwise kickoff on interval time
m_WorkerEvtThreads[i] = new AutoResetEvent(false); // initial each thread event
m_WorkerEvtThreads[i].Reset();
Thread.Sleep(WaitHandle.WaitTimeout); // sleep 258ms...
CurrentInstance = int.Parse(lvTask.ctlrArray[i, 2].ToString()) - 1; // get the current instance
lvCtlrIdStart = int.Parse(lvTask.ctlrArray[i, 0].ToString()); // start controller ID
lvCtlrIdEnd = int.Parse(lvTask.ctlrArray[i, 1].ToString()); // end controller ID
s_range = $"{lvCtlrIdStart},{lvCtlrIdEnd}"; // combine to pass to SendQueueMessage()
ThreadPool.QueueUserWorkItem(delegate
{ Thread.CurrentThread.Name = $"LV Instance {CurrentInstance}";
lvTask.TN = $"{Thread.CurrentThread.Name}"; // name the current instance task
m_ThreadName = $"{lvTask.TN}";
TimerSendQueueMsgCallback(s_range, intvlStart, dueTime); // Call the timer callback method
}, cts.Token); // cancellation token
}
catch (Exception ex)
{
log.WriteLineToBuf(ref logBuf, $"Exception: OnLvMessage() - {ex.Message}");
}
}
cts.Cancel(); // request cancellation...
I have main and working threads. These working threads must not stopped before main task is over (i.e. i cant re-creating them with Thread.Start()). These working threads reading data from main thread, then writing other data and then main thread writing it in file.
I must use NET 3.5, not including ThreadPool and BackgroundWorker
InFile -> RawData -> Working -> ReadyData -> OutFile.
public func(string InputFile,string outputFile) {
...
for(int i=0;i<ThreadCount;i++)
{
CompressorThreads[i] = new Thread(new ParameterizedThreadStart(CompressBlock));
ThreadParam thr = new ThreadParam(i);
CompressorThreads[i].Start(thr);
}
int count;
while (InputFile.Position < InputFile.Length)
{
for (int i = 0; i < ThreadCount; i++)
{
count = InputFile.Read(buffer, 0, BLOCK_SIZE);
RawData[i] = new byte[count];
/*here activating(start) thread*/
}
for (int i = 0; (i < ThreadCount) && (CompressorThreads[i] != null);)
{
/*prevent(pause) working thread from changing CompressedData*/
OutputFile.Write(CompressedData[i], 0, CompressedData[i].Length);
}
}
...
}
private static void CompressBlock(object BlockProperties)
{
ThreadParam Block = BlockProperties as ThreadParam;
int ThreadId = Block.ThreadId;
while (true)
{
if(RawData[ThreadId].Length == 0)
{
break;
}
...
/*working here*/
...
/*then save data to CompressedData*/
/*pause thread*/
}
}
I tried AutoResetEvent, but it seems inappropriate for this problem. What i'm looking for?
Its a pretty simple program. I'm trying to read out messages from a controller after I send a message that requests constant messages to be sent back. The first part is how I managed to get the reading working as intended, and every time a message is sent I can see it in the Console. I want to recreate this scenario with the task in the code below that. How ever I haven't been successful in doing so.
if (success)
{
if (fileStreamDeviceData.CanRead)
{
while(success)
{
await fileStreamDeviceData.ReadAsync(outputReportBuffer, 0, outputReportBuffer.Length);
for (int b = 0; b < 12; b++)
{
Console.Write(outputReportBuffer[b] + " ");
}
Console.WriteLine();
}
}
}
So the above part works as intended except for the ugly endless while loop that is currently happening. The following 2 Code Parts are the solution for this, but its not working as intended and I don't seem to get anything out of it. I really suck with async tasks and stuff so maybe someone can help me fix the below section.
if (success)
{
outputReportBuffer = null;
//start async reading
try
{
outputReportBuffer = await ReadByteAsync();
for (int b = 0; b < outputReportBuffer.Length; b++)
{
Console.Write(outputReportBuffer[b] + " ");
}
Console.WriteLine();
}
catch (Exception ex)
{
throw ex;
}
}
private async Task<Byte[]> ReadByteAsync()
{
Byte[] result = null;
Byte[] outputReportBuffer = null;
using (fileStreamDeviceData)
{
outputReportBuffer = new Byte[MyHid.Capabilities.OutputReportByteLength];
int byteRead;
while ((byteRead = await fileStreamDeviceData.ReadAsync(outputReportBuffer, 0, outputReportBuffer.Length)) != 0)
{
result = outputReportBuffer;
}
return result;
}
}
I get no exception or any output when I use this Code.
I can be due to many reasons, try to print something int he while-loop first.
Does the caller outputReportBuffer = await ReadByteAsync(); is inside an asyn method itself? If not I'll use ReadByteAsync().Result instead and set ConfigureAwait(false) in the ReadByteAsync() method.
const int BufferSize = 100;
private async Task<Byte[]> ReadByteAsync()
{
var memoryStream = new MemoryStream();
using (fileStreamDeviceData)
{
var outputReportBuffer = new Byte[BufferSize];
int offset = 0;
int byteRead;
while ((byteRead = await fileStreamDeviceData.ReadAsync(outputReportBuffer, offset, outputReportBuffer.Length)) != 0)
{
memoryStream.Write(outputReportBuffer, offset, byteRead);
offset += BufferSize;
}
return memoryStream.ToArray();
}
}
I'm trying to make a program like IDM, that can download parts of the file simultaneously.
The tool i'm using to achieve this is TPL in C# .Net4.5
But I'm having a problem when using Tasks to make the operation parallel.
The sequence function is functioning well and it is downloading the files correctly.
The parallel function using Tasks is working until something weird happens:
I've created 4 tasks, with Factory.StartNew(), in each task the start position and the end position are given, the task will download these files, then it'll return it in byte[], and everything is going well, the tasks are working fine, but at some point, the executing freezes and that's it, the program stops and nothing else happens.
the implementation of the parallel function:
static void DownloadPartsParallel()
{
string uriPath = "http://mschnlnine.vo.llnwd.net/d1/pdc08/PPTX/BB01.pptx";
Uri uri = new Uri(uriPath);
long l = GetFileSize(uri);
Console.WriteLine("Size={0}", l);
int granularity = 4;
byte[][] arr = new byte[granularity][];
Task<byte[]>[] tasks = new Task<byte[]>[granularity];
tasks[0] = Task<byte[]>.Factory.StartNew(() => DownloadPartOfFile(uri, 0, l / granularity));
tasks[1] = Task<byte[]>.Factory.StartNew(() => DownloadPartOfFile(uri, l / granularity + 1, l / granularity + l / granularity));
tasks[2] = Task<byte[]>.Factory.StartNew(() => DownloadPartOfFile(uri, l / granularity + l / granularity + 1, l / granularity + l / granularity + l / granularity));
tasks[3] = Task<byte[]>.Factory.StartNew(() => DownloadPartOfFile(uri, l / granularity + l / granularity + l / granularity + 1, l));//(l / granularity) + (l / granularity) + (l / granularity) + (l / granularity)
arr[0] = tasks[0].Result;
arr[1] = tasks[1].Result;
arr[2] = tasks[2].Result;
arr[3] = tasks[3].Result;
Stream localStream;
localStream = File.Create("E:\\a\\" + Path.GetFileName(uri.LocalPath));
for (int i = 0; i < granularity; i++)
{
if (i == granularity - 1)
{
for (int j = 0; j < arr[i].Length - 1; j++)
{
localStream.WriteByte(arr[i][j]);
}
}
else
for (int j = 0; j < arr[i].Length; j++)
{
localStream.WriteByte(arr[i][j]);
}
}
}
the DownloadPartOfFile function implementation:
public static byte[] DownloadPartOfFile(Uri fileUrl, long from, long to)
{
int bytesProcessed = 0;
BinaryReader reader = null;
WebResponse response = null;
byte[] bytes = new byte[(to - from) + 1];
try
{
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(fileUrl);
request.AddRange(from, to);
request.ReadWriteTimeout = int.MaxValue;
request.Timeout = int.MaxValue;
if (request != null)
{
response = request.GetResponse();
if (response != null)
{
reader = new BinaryReader(response.GetResponseStream());
int bytesRead;
do
{
byte[] buffer = new byte[1024];
bytesRead = reader.Read(buffer, 0, buffer.Length);
if (bytesRead == 0)
{
break;
}
Array.Resize<byte>(ref buffer, bytesRead);
buffer.CopyTo(bytes, bytesProcessed);
bytesProcessed += bytesRead;
Console.WriteLine(Thread.CurrentThread.ManagedThreadId + ",Downloading" + bytesProcessed);
} while (bytesRead > 0);
}
}
}
catch (Exception e)
{
Console.WriteLine(e.Message);
}
finally
{
if (response != null) response.Close();
if (reader != null) reader.Close();
}
return bytes;
}
I tried to solve it by setting int.MaxValue to the reading timeout, writing reading timeout, and timeout, that's why the program freezes, if i didn't do that, an exception of timeout will occur while in function DownloadPartsParallel
so is there a solution, or any other advice that may help, thanks.
I would use HttpClient.SendAsync rather than WebRequest (see "HttpClient is Here!").
I would not use any extra threads. The HttpClient.SendAsync API is naturally asynchronous and returns an awaitable Task<>, there is no need to offload it to a pool thread with Task.Run/Task.TaskFactory.StartNew (see this for a detailed discussion).
I would also limit the number of parallel downloads with SemaphoreSlim.WaitAsync(). Below is my take as a console app (not extensively tested):
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net.Http;
using System.Threading;
using System.Threading.Tasks;
namespace Console_21737681
{
class Program
{
const int MAX_PARALLEL = 4; // max parallel downloads
const int CHUNK_SIZE = 2048; // size of a single chunk
// a chunk of downloaded data
class Chunk
{
public long Start { get; set; }
public int Length { get; set; }
public byte[] Data { get; set; }
};
// throttle downloads
SemaphoreSlim _throttleSemaphore = new SemaphoreSlim(MAX_PARALLEL);
// get a chunk
async Task<Chunk> GetChunk(HttpClient client, long start, int length, string url)
{
await _throttleSemaphore.WaitAsync();
try
{
using (var request = new HttpRequestMessage(HttpMethod.Get, url))
{
request.Headers.Range = new System.Net.Http.Headers.RangeHeaderValue(start, start + length - 1);
using (var response = await client.SendAsync(request))
{
var data = await response.Content.ReadAsByteArrayAsync();
return new Chunk { Start = start, Length = length/*, Data = data*/ };
}
}
}
finally
{
_throttleSemaphore.Release();
}
}
// download the URL in parallel by chunks
async Task<Chunk[]> DownloadAsync(string url)
{
using (var client = new HttpClient())
{
var request = new HttpRequestMessage(HttpMethod.Head, url);
var response = await client.SendAsync(request);
var contentLength = response.Content.Headers.ContentLength;
if (!contentLength.HasValue)
throw new InvalidOperationException("ContentLength");
var numOfChunks = (int)((contentLength.Value + CHUNK_SIZE - 1) / CHUNK_SIZE);
var tasks = Enumerable.Range(0, numOfChunks).Select(i =>
{
// start a new chunk
long start = i * CHUNK_SIZE;
var length = (int)Math.Min(CHUNK_SIZE, contentLength.Value - start);
return GetChunk(client, start, length, url);
}).ToList();
await Task.WhenAll(tasks);
// the order of chunks is random
return tasks.Select(task => task.Result).ToArray();
}
}
static void Main(string[] args)
{
var program = new Program();
var chunks = program.DownloadAsync("http://flaglane.com/download/australian-flag/australian-flag-large.png").Result;
Console.WriteLine("Chunks: " + chunks.Count());
Console.ReadLine();
}
}
}
OK, here's how I would do what you're attempting. This is basically the same idea, just implemented differently.
public static void DownloadFileInPiecesAndSave()
{
//test
var uri = new Uri("http://www.w3.org/");
var bytes = DownloadInPieces(uri, 4);
File.WriteAllBytes(#"c:\temp\RangeDownloadSample.html", bytes);
}
/// <summary>
/// Donwload a file via HTTP in multiple pieces using a Range request.
/// </summary>
public static byte[] DownloadInPieces(Uri uri, uint numberOfPieces)
{
//I'm just fudging this for expository purposes. In reality you would probably want to do a HEAD request to get total file size.
ulong totalFileSize = 1003;
var pieceSize = totalFileSize / numberOfPieces;
List<Task<byte[]>> tasks = new List<Task<byte[]>>();
for (uint i = 0; i < numberOfPieces; i++)
{
var start = i * pieceSize;
var end = start + (i == numberOfPieces - 1 ? pieceSize + totalFileSize % numberOfPieces : pieceSize);
tasks.Add(DownloadFilePiece(uri, start, end));
}
Task.WaitAll(tasks.ToArray());
//This is probably not the single most efficient way to combine byte arrays, but it is succinct...
return tasks.SelectMany(t => t.Result).ToArray();
}
private static async Task<byte[]> DownloadFilePiece(Uri uri, ulong rangeStart, ulong rangeEnd)
{
try
{
var request = (HttpWebRequest)WebRequest.Create(uri);
request.AddRange((long)rangeStart, (long)rangeEnd);
request.Proxy = WebProxy.GetDefaultProxy();
using (var response = await request.GetResponseAsync())
using (var responseStream = response.GetResponseStream())
using (var memoryStream = new MemoryStream((int)(rangeEnd - rangeStart)))
{
await responseStream.CopyToAsync(memoryStream);
return memoryStream.ToArray();
}
}
catch (WebException wex)
{
//Do lots of error handling here, lots of things can go wrong
//In particular watch for 416 Requested Range Not Satisfiable
return null;
}
catch (Exception ex)
{
//handle the unexpected here...
return null;
}
}
Note that I glossed over a lot of stuff here, such as:
Detecting if the server supports range requests. If it doesn't then the server will return the entire content in each request, and we'll get several copies of it.
Handling any sort of HTTP errors. What if the third request fails?
Retry logic
Timeouts
Figuring out how big the file actually is
Checking whether the file is big enough to warrant multiple requests, and if so how many? It's probably not worth doing this in parallel for files under 1 or 2 MB, but you'd have to test
Most likely a bunch of other stuff.
So you've got a long way to go before I would use this in production. But it should give you an idea of where to start.