Develop Reference

Task process ending before finishing all the work

I've been having trouble running multiple tasks with heavy operations.
It seems as if the task processes is killed before all the operations are complete.
The code here is an example code I used to replicate the issue. If I add something like Debug.Write(), the added wait for writing fixes the issue. The issue is gone if I test on a smaller sample size too. The reason there is a class in the example below is to create complexity for the test.
The real case where I encountered the issue first is too complicated to explain for a post here.
public static class StaticRandom
{
static int seed = Environment.TickCount;
static readonly ThreadLocal<Random> random =
new ThreadLocal<Random>(() => new Random(Interlocked.Increment(ref seed)));
public static int Next()
{
return random.Value.Next();
}
public static int Next(int maxValue)
{
return random.Value.Next(maxValue);
}
public static double NextDouble()
{
return random.Value.NextDouble();
}
}
// this is the test function I run to recreate the problem:
static void tasktest()
{
var testlist = new List<ExampleClass>();
for (var index = 0; index < 10000; ++index)
{
var newClass = new ExampleClass();
newClass.Populate(Enumerable.Range(0, 1000).ToList());
testlist.Add(newClass);
}
var anotherClassList = new List<ExampleClass>();
var threadNumber = 5;
if (threadNumber > testlist.Count)
{
threadNumber = testlist.Count;
}
var taskList = new List<Task>();
var tokenSource = new CancellationTokenSource();
CancellationToken cancellationToken = tokenSource.Token;
int stuffPerThread = testlist.Count / threadNumber;
var stuffCounter = 0;
for (var count = 1; count <= threadNumber; ++count)
{
var toSkip = stuffCounter;
var threadWorkLoad = stuffPerThread;
var currentIndex = count;
// these ifs make sure all the indexes are covered
if (stuffCounter + threadWorkLoad > testlist.Count)
{
threadWorkLoad = testlist.Count - stuffCounter;
}
else if (count == threadNumber && stuffCounter + threadWorkLoad < testlist.Count)
{
threadWorkLoad = testlist.Count - stuffCounter;
}
taskList.Add(Task.Factory.StartNew(() => taskfunc(testlist, anotherClassList, toSkip, threadWorkLoad),
cancellationToken, TaskCreationOptions.None, TaskScheduler.Default));
stuffCounter += stuffPerThread;
}
Task.WaitAll(taskList.ToArray());
}
public class ExampleClass
{
public ExampleClassInner[] Inners { get; set; }
public ExampleClass()
{
Inners = new ExampleClassInner[5];
for (var index = 0; index < Inners.Length; ++index)
{
Inners[index] = new ExampleClassInner();
}
}
public void Populate(List<int> intlist) {/*adds random ints to the inner class*/}
public ExampleClass(ExampleClass copyFrom)
{
Inners = new ExampleClassInner[5];
for (var index = 0; index < Inners.Length; ++index)
{
Inners[index] = new ExampleClassInner(copyFrom.Inners[index]);
}
}
public class ExampleClassInner
{
public bool SomeBool { get; set; } = false;
public int SomeInt { get; set; } = -1;
public ExampleClassInner()
{
}
public ExampleClassInner(ExampleClassInner copyFrom)
{
SomeBool = copyFrom.SomeBool;
SomeInt = copyFrom.SomeInt;
}
}
}
static int expensivefunc(int theint)
{
/*a lot of pointless arithmetic and loops done only on primitives and with primitives,
just to increase the complexity*/
theint *= theint + 1;
var anotherlist = Enumerable.Range(0, 10000).ToList();
for (var index = 0; index < anotherlist.Count; ++index)
{
theint += index;
if (theint % 5 == 0)
{
theint *= index / 2;
}
}
var yetanotherlist = Enumerable.Range(0, 50000).ToList();
for (var index = 0; index < yetanotherlist.Count; ++index)
{
theint += index;
if (theint % 7 == 0)
{
theint -= index / 3;
}
}
while (theint > 8)
{
theint /= 2;
}
return theint;
}
// this function is intentionally creating a lot of objects, to simulate complexity
static void taskfunc(List<ExampleClass> intlist, List<ExampleClass> anotherClassList, int skip, int take)
{
if (take == 0)
{
take = intlist.Count;
}
var partial = intlist.Skip(skip).Take(take).ToList();
for (var index = 0; index < partial.Count; ++index)
{
var testint = expensivefunc(index);
var newClass = new ExampleClass(partial[index]);
newDna.Inners[StaticRandom.Next(5)].SomeInt = testint;
anotherClassList.Add(new ExampleClass(newClass));
}
}
The expected result is that the list anotherClassList will be the same size as testlist and this happens when the lists are smaller or the complexity of the task operations is smaller. However, when I increase the volume of operations, the anotherClassList has a few indexes missing and sometimes some of the indexes in the list are null objects.
Example result:
Why does this happen, I have Task.WaitAll?

Your problem is it's just not thread-safe; you just can't add to a list<T> in a multi-threaded environment and expect it to play nice.
One way is to use lock or a thread safe collection, but I feel this all should be refactored (my OCD is going off all over the place).
private static object _sync = new object();
...
private static void TaskFunc(List<ExampleClass> intlist, List<ExampleClass> anotherClassList, int skip, int take)
{
...
var partial = intlist.Skip(skip).Take(take).ToList();
...
// note that locking here will likely drastically decrease any performance threading gain
lock (_sync)
{
for (var index = 0; index < partial.Count; ++index)
{
// this is your problem, you are adding to a list from multiple threads
anotherClassList.Add(...);
}
}
}
In short, I think you need to better thinking about the threading logic of your method, identify what you are trying to achieve, and how to make it conceptually thread safe (while keeping your performance gains).

After TheGeneral enlightened me that Lists are not thread safe, I changed the List to which I was adding in a thread, to an Array type and this fixed my issue.

custom threads in my code?

I need to create a program scraping a website.
And I did use Thread to solve.
Example:
I have 100 pages and I need divide it, instead of get each page I need custom Thread number to get page:
2 threads - 50 pages/thread
4 threads - 25 pages/thread
I tried my code below, however when to the last page of each thread that very slow.
Before I ask I used to find the way to solve but I can't, therefore I need help.
int so_thread = 10;//thread number
int page_du = 0;
List<NameValueCollection> List_item = new List<NameValueCollection>();
Thread[] threads = new Thread[so_thread];
int dem = 0;
await Task.Run(() =>
{
for (int i = 1; i <= so_thread; i++)
{
if ((Int32.Parse(o_sopage.Text) % so_thread) != 0 && i == so_thread)
{
page_du = Int32.Parse(o_sopage.Text) % so_thread;//Int32.Parse(o_sopage.Text) == page number need get
}
threads[i - 1] = new Thread((object data) =>
{
Array New_Data = new object[2];
New_Data = (Array)data;
int _i = (int)New_Data.GetValue(0);
int _pagedu = (int)New_Data.GetValue(1);
int page_per_thread = Int32.Parse(o_sopage.Text) / so_thread;//Int32.Parse(o_sopage.Text) == page number need get
for (int j = ((page_per_thread * _i) - page_per_thread) + 1; j <= ((page_per_thread * _i) + _pagedu); j++)
{
//MessageBox.Show(j.ToString());
var TG = ebay.GetPage(j);
lock (List_item)
{
List_item.AddRange(TG);
dem++;
progressBar1.Invoke((MethodInvoker)delegate
{
progressBar1.Value = dem;
});
}
}
});
object DATA = new object[2] { i, page_du };
threads[i - 1].Start(DATA);
}
});

Use Parallel.ForEach instead of creating the threads on your own.
Parallel.ForEach(yourCollection, () => { /* your code here */});

Permutation on array of type "Location", from GoogleMapsAPI NuGet Package

this is my very first post here on StackOverflow so please tell me if I did anything wrong, also english is not my native language, forgive me if there is any gramatical errors.
My question is how can I permutate the items of an array of type "Location", I need to get all possible permutations of waypoints given by the user to then calculate the best route based on time or distance. (I don't want to use the normal route calculation)
I've searched for algorithms but all of them when I put the array of type "Location[]" in the function's parameter I get the error that the object needs to be IEnumerable and I don't know how to convert to that if is even possible, I never worked with IEnumerable.
If it is of any help this is my code for calculating the route:
//Gets the waypoints from a listBox provided by the user, "mode" selects between best time and best distance
//backgroundworker so the UI dont freezes, and return the optimal waypoint order
public Location[] CalcularRota(Location[] waypoints, int mode, BackgroundWorker work, DoWorkEventArgs e)
{
//Declarations
string origem = "";
string destino = "";
Rota[] prop = new Rota[100]; //this index is the number of times the algorithm will be executed, more equals accuracy but much more time to complete
Rota bestDist = new Rota();
Rota bestTime = new Rota();
DirectionService serv = new DirectionService();
DirectionRequest reqs = new DirectionRequest();
DirectionResponse resp;
Random rnd = new Random();
Location[] rndWays;
int dist = 0;
int ti = 0;
bestDist.Distance = 1000000000; //put higher values for the first comparation to be true (end of code)
bestTime.Time = 1000000000;
if (waypoints != null)
{
reqs.Sensor = false;
reqs.Mode = TravelMode.driving;
for (int i = 0; i < prop.Length; i++) //initializes prop
prop[i] = new Rota();
for (int i = 0; i < prop.Length; i++)
{
rndWays = waypoints.OrderBy(x => rnd.Next()).ToArray(); //randomizes the order, I want to get all permutations and then test
//but I dont know how so I've been using randomized
dist = ti = 0;
origem = prop[0].ToString(); //save this particular waypoint's origin and destination
destino = prop[1].ToString();
reqs.Origin = origem;
reqs.Destination = destino;
if (waypoints.Length > 0)
reqs.Waypoints = rndWays;
resp = serv.GetResponse(reqs); //request the route with X order of waypoints to google
if (resp.Status == ServiceResponseStatus.Ok) //wait the response otherwise the program crashes
{
for (int j = 0; j < resp.Routes[0].Legs.Length; j++) //gets the distance and time of this particular order
{
ti += int.Parse(resp.Routes[0].Legs[j].Duration.Value);
dist += int.Parse(resp.Routes[0].Legs[j].Distance.Value);
}
}
prop[i].Origem = origem; //saves this waypoints order details for further comparison
prop[i].Destino = destino;
prop[i].Distance = dist;
prop[i].Time = ti;
prop[i].Order = rndWays;
work.ReportProgress(i); //report the progress
}
for (int i = 0; i < prop.Length; i++) //gets the best distance and time
{
if (bestDist.Distance > prop[i].Distance)
{
bestDist.Distance = prop[i].Distance;
bestDist.Time = prop[i].Time;
bestDist.Order = prop[i].Order;
bestDist.Origem = prop[i].Origem;
bestDist.Destino = prop[i].Destino;
}
if (bestTime.Time > prop[i].Time)
{
bestTime.Distance = prop[i].Distance;
bestTime.Time = prop[i].Time;
bestTime.Order = prop[i].Order;
bestTime.Origem = prop[i].Origem;
bestTime.Destino = prop[i].Destino;
}
}
if (bestDist.Order == bestTime.Order) //if the same waypoint order has the same time and distance
return bestDist.Order; // returns whatever bestDist.Order or bestTime.Order
else if (bestDist.Order != bestTime.Order) //if different returns corresponding to the mode selected
{
if (mode == 1) return bestDist.Order;
if (mode == 2) return bestTime.Order;
}
}
return null;
}
What I want is to permutate the waypoints given and test each permutation, I've been struggling with this for a time, if u guys could help me with any way possible would be great.
Ty.
EDIT.
I found this function here on StackOverflow:
public static bool NextPermutation<T>(T[] elements) where T : IComparable<T>
{
var count = elements.Length;
var done = true;
for (var i = count - 1; i > 0; i--)
{
var curr = elements[i];
// Check if the current element is less than the one before it
if (curr.CompareTo(elements[i - 1]) < 0)
{
continue;
}
// An element bigger than the one before it has been found,
// so this isn't the last lexicographic permutation.
done = false;
// Save the previous (bigger) element in a variable for more efficiency.
var prev = elements[i - 1];
// Have a variable to hold the index of the element to swap
// with the previous element (the to-swap element would be
// the smallest element that comes after the previous element
// and is bigger than the previous element), initializing it
// as the current index of the current item (curr).
var currIndex = i;
// Go through the array from the element after the current one to last
for (var j = i + 1; j < count; j++)
{
// Save into variable for more efficiency
var tmp = elements[j];
// Check if tmp suits the "next swap" conditions:
// Smallest, but bigger than the "prev" element
if (tmp.CompareTo(curr) < 0 && tmp.CompareTo(prev) > 0)
{
curr = tmp;
currIndex = j;
}
}
// Swap the "prev" with the new "curr" (the swap-with element)
elements[currIndex] = prev;
elements[i - 1] = curr;
// Reverse the order of the tail, in order to reset it's lexicographic order
for (var j = count - 1; j > i; j--, i++)
{
var tmp = elements[j];
elements[j] = elements[i];
elements[i] = tmp;
}
// Break since we have got the next permutation
// The reason to have all the logic inside the loop is
// to prevent the need of an extra variable indicating "i" when
// the next needed swap is found (moving "i" outside the loop is a
// bad practice, and isn't very readable, so I preferred not doing
// that as well).
break;
}
// Return whether this has been the last lexicographic permutation.
return done;
}
The usage is:
NextPermutation(array);
Doing this and putting my array (rndWays) as overload I get the following error:
The type 'Google.Maps.Location' cannot be used as type parameter 'T' in the generic type or method 'Form1.NextPermutation< T >(T[])'. There is no implicit reference conversion from 'Google.Maps.Location' to 'System.IComparable< Google.Maps.Location >'.

The problem is that Location does not implement the IComparable interface.
Change:
public static bool NextPermutation<T>(T[] elements) where T : IComparable<T>
to:
public static bool NextPermutation(Location[] elements)
And replace each CompareTo() with your own comparison function.

How do i check if the List length is bigger than the last time?

I have this code:
int LRLength = LR.Count;
for (int i = 0; i < LR.Count; i++)
{
LRLength = LR.Count;
LR = merge(LR);
if (LR.Count < LRLength)
{
LR = merge(LR);
if (LR.Count == LRLength)
{
break;
}
}
}
And this is the function merge:
private List<Lightnings_Extractor.Lightnings_Region> merge(List<Lightnings_Extractor.Lightnings_Region> Merged)
{
List<Lightnings_Extractor.Lightnings_Region> NewMerged = new List<Lightnings_Extractor.Lightnings_Region>();
Lightnings_Extractor.Lightnings_Region reg;
int dealtWith = -1;
for (int i = 0; i < Merged.Count; i++)
{
if (i != dealtWith)
{
reg = new Lightnings_Extractor.Lightnings_Region();
if (i < Merged.Count - 1)
{
if (Merged[i].end + 1 >= Merged[i + 1].start)
{
reg.start = Merged[i].start;
reg.end = Merged[i + 1].end;
NewMerged.Add(reg);
dealtWith = i + 1;
}
else
{
reg.start = Merged[i].start;
reg.end = Merged[i].end;
NewMerged.Add(reg);
}
}
else
{
reg.start = Merged[i].start;
reg.end = Merged[i].end;
NewMerged.Add(reg);
}
}
}
return NewMerged;
}
In this class: Lightnings_Extractor.Lightnings_Region I have only two int variables.
The idea in this function is to get a List and merge areas that are congruent.
For example once im calling the function and the List LR Length is 8 now I will get it back less. For example if it needed to merge two indexs to one then the List I will get in return the Length will be 7. If it will need to merge another indexs then the Length will be 6 and so on.
What I want to check on the first code above is when I should stop calling the function to merge indexs.
If the length was 8 and the next time it's still 8 do nothing stop the loop.
If the length is 8 and the next time it's 7 then call the function again.
If the length is 7 stop the loop . But if the length is 6 keep calling it once again.
Untill the last length will be the same as the length before !!!
So I tried this code but it's not working good:
int LRLength = LR.Count;
for (int i = 0; i < LR.Count; i++)
{
LRLength = LR.Count;
LR = merge(LR);
if (LR.Count < LRLength)
{
LR = merge(LR);
if (LR.Count == LRLength)
{
break;
}
}
}

Trying to make some assumptions as to what you're trying to accomplish. The following will basically capture the original length of the list for comparison. It will run at least once, and keep running until the LRLength == LR.Count
int LRLength = LR.Count;
do{
LR = merge(LR);
} while(LR.Count != LRLength);
If you were trying to run the loop until you got the same count twice in a row:
int prevCount;
do{
prevCount = LR.Count;
LR = merge(LR);
} while(prevCount != LR.Count);

Mutithreading in C# queries

I am new to multithreading in C# . I have a 3D array of size (x)(y)(z) and say i want to calculate the average of all the z samples for every (x,y) values. I wish to do that using multithreading (say 2 threads) where i will send half the array of size (x/2)*y*z for processing to thread1 and the other half to thread2.
How to do it? How do I pass and retrieve arguments from individual threads? A code example will be helpful.
Regards

I would recommend using PLINQ for this instead of threading this yourself.
It will let you run your query using LINQ syntax, but parallelize it (across all of your cores) automatically.

There are many reasons why it makes sense to use something PLINQ (as mentioned by Reed) or Parallel.For as implementing a low-overhead scheduler for distributing jobs over several cpus is a bit challenging.
So if I understood you correctly maybe this could get you started (on my 4 core machine the parallel version is 3 times faster than the single core version):
using System;
using System.Diagnostics;
using System.Linq;
using System.Threading.Tasks;
class Program
{
static void AverageOfZ (
double[] input,
double[] result,
int x,
int y,
int z
)
{
Debug.Assert(input.Length == x*y*z);
Debug.Assert(result.Length == x*y);
//Replace Parallel with Sequential to compare with non-parallel loop
//Sequential.For(
Parallel.For(
0,
x*y,
i =>
{
var begin = i*z;
var end = begin + z;
var sum = 0.0;
for (var iter = begin; iter < end; ++iter)
{
sum += input[iter];
}
result[i] = sum/z;
});
}
static void Main(string[] args)
{
const int X = 64;
const int Y = 64;
const int Z = 64;
const int Repetitions = 40000;
var random = new Random(19740531);
var samples = Enumerable.Range(0, X*Y*Z).Select(x => random.NextDouble()).ToArray();
var result = new double[X*Y];
var then = DateTime.Now;
for (var iter = 0; iter < Repetitions; ++iter)
{
AverageOfZ(samples, result, X, Y, Z);
}
var diff = DateTime.Now - then;
Console.WriteLine(
"{0} samples processed {1} times in {2} seconds",
samples.Length,
Repetitions,
diff.TotalSeconds
);
}
}
static class Sequential
{
public static void For(int from, int to, Action<int> action)
{
for (var iter = from; iter < to; ++iter)
{
action(iter);
}
}
}
PS. When going for concurrent performance its important to consider how the different cores access memory as its very easy to get disappointing performance otherwise.

Dot Net 3.5 and onward introduced many shortcut keywords that abstract away the complexity of things like Parallel for multi threading or Async for Async IO. Unfortunately this also provides no opportunity for understanding whats involved in these tasks. For example a colleague of mine was recently trying to use Async for a login method which returned an authentication token.
Here is the full blown multi threaded sample code for your scenario. to make it more real the sample code pretends that:
X is Longitude
Y is Lattitude
and Z is Rainfall Samples at the coordinates
The sample code also follows the Unit of Work design pattern where Rainfall Samples at each coordinate becomes a work item. It also creates discrete foreground threads instead of using a background threadpool.
Due to the simplicity of the work item and short compute time involved I've split the thread synchronization locks into two locks. one for the work queue and one for the output data.
Note: I have not used any Dot net shortcuts such as Lync so this code should run on Dot Net 2.0 as well.
In real world app development something like whats below would only be needed in complex scenarios such as stream processing of a continuous stream of work items in which case you would also need to implement output data buffers cleared regularly as the threads would effectively run forever.
public static class MultiThreadSumRainFall
{
const int LongitudeSize = 64;
const int LattitudeSize = 64;
const int RainFallSamplesSize = 64;
const int SampleMinValue = 0;
const int SampleMaxValue = 1000;
const int ThreadCount = 4;
public static void SumRainfallAndOutputValues()
{
int[][][] SampleData;
SampleData = GenerateSampleRainfallData();
for (int Longitude = 0; Longitude < LongitudeSize; Longitude++)
{
for (int Lattitude = 0; Lattitude < LattitudeSize; Lattitude++)
{
QueueWork(new WorkItem(Longitude, Lattitude, SampleData[Longitude][Lattitude]));
}
}
System.Threading.ThreadStart WorkThreadStart;
System.Threading.Thread WorkThread;
List<System.Threading.Thread> RunningThreads;
WorkThreadStart = new System.Threading.ThreadStart(ParallelSum);
int NumThreads;
NumThreads = ThreadCount;
if (ThreadCount < 1)
{
NumThreads = 1;
}
else if (NumThreads > (Environment.ProcessorCount + 1))
{
NumThreads = Environment.ProcessorCount + 1;
}
OutputData = new int[LongitudeSize, LattitudeSize];
RunningThreads = new List<System.Threading.Thread>();
for (int I = 0; I < NumThreads; I++)
{
WorkThread = new System.Threading.Thread(WorkThreadStart);
WorkThread.Start();
RunningThreads.Add(WorkThread);
}
bool AllThreadsComplete;
AllThreadsComplete = false;
while (!AllThreadsComplete)
{
System.Threading.Thread.Sleep(100);
AllThreadsComplete = true;
foreach (System.Threading.Thread WorkerThread in RunningThreads)
{
if (WorkerThread.IsAlive)
{
AllThreadsComplete = false;
}
}
}
for (int Longitude = 0; Longitude < LongitudeSize; Longitude++)
{
for (int Lattitude = 0; Lattitude < LattitudeSize; Lattitude++)
{
Console.Write(string.Concat(OutputData[Longitude, Lattitude], #" "));
}
Console.WriteLine();
}
}
private class WorkItem
{
public WorkItem(int _Longitude, int _Lattitude, int[] _RainFallSamples)
{
Longitude = _Longitude;
Lattitude = _Lattitude;
RainFallSamples = _RainFallSamples;
}
public int Longitude { get; set; }
public int Lattitude { get; set; }
public int[] RainFallSamples { get; set; }
}
public static int[][][] GenerateSampleRainfallData()
{
int[][][] Result;
Random Rnd;
Rnd = new Random();
Result = new int[LongitudeSize][][];
for(int Longitude = 0; Longitude < LongitudeSize; Longitude++)
{
Result[Longitude] = new int[LattitudeSize][];
for (int Lattidude = 0; Lattidude < LattitudeSize; Lattidude++)
{
Result[Longitude][Lattidude] = new int[RainFallSamplesSize];
for (int Sample = 0; Sample < RainFallSamplesSize; Sample++)
{
Result[Longitude][Lattidude][Sample] = Rnd.Next(SampleMinValue, SampleMaxValue);
}
}
}
return Result;
}
private static object SyncRootWorkQueue = new object();
private static Queue<WorkItem> WorkQueue = new Queue<WorkItem>();
private static void QueueWork(WorkItem SamplesWorkItem)
{
lock(SyncRootWorkQueue)
{
WorkQueue.Enqueue(SamplesWorkItem);
}
}
private static WorkItem DeQueueWork()
{
WorkItem Samples;
Samples = null;
lock (SyncRootWorkQueue)
{
if (WorkQueue.Count > 0)
{
Samples = WorkQueue.Dequeue();
}
}
return Samples;
}
private static int QueueSize()
{
lock(SyncRootWorkQueue)
{
return WorkQueue.Count;
}
}
private static object SyncRootOutputData = new object();
private static int[,] OutputData;
private static void SetOutputData(int Longitude, int Lattitude, int SumSamples)
{
lock(SyncRootOutputData)
{
OutputData[Longitude, Lattitude] = SumSamples;
}
}
private static void ParallelSum()
{
WorkItem SamplesWorkItem;
int SummedResult;
SamplesWorkItem = DeQueueWork();
while (SamplesWorkItem != null)
{
SummedResult = 0;
foreach (int SampleValue in SamplesWorkItem.RainFallSamples)
{
SummedResult += SampleValue;
}
SetOutputData(SamplesWorkItem.Longitude, SamplesWorkItem.Lattitude, SummedResult);
SamplesWorkItem = DeQueueWork();
}
}
}