Develop Reference

When i try to use thread to rename my file name and close and flush the FileStream , it will give me error Cannot access a closed file

I have large amount of data, i need to make '.txt' file in every 5 sec and write next coming data in that newly created '.txt' file but when i try to use Timer thread for that for sometime program run properly but after sometime it will through exception cannot access a closed file . Please help me in my code and suggest me what has to be done.
public class Program
{
public static void Main(string[] args)
{
Service s = new Service();
s.init();
}
}
class Service
{
FileStream fs = null;
int filecount = 0;
long a = 1000000000000000;
// int j = 0;
public void init()
{
initialiseFile(filecount);
timer();
WriteINFile();
}
private void initialiseFile( int filecount)
{
fs = File.Create("C:\\Users\\yogesh.ghonate\\source\\repos\\ConsoleApp3\\ConsoleApp3\\NewFolder1\\Index_" + filecount + ".txt");
}
private void WriteINFile()
{
string sen = " write in file ";
for (int i = 0; i < a; i++)
{
Byte[] title = new UTF8Encoding(true).GetBytes(sen);
fs.Write(title, 0, title.Length);
}
}
public void timer()
{
System.Timers.Timer th = new System.Timers.Timer();
th.Interval = 5000;
th.Elapsed += new ElapsedEventHandler(run);
th.Start();
run(th, null);
}
public void run(object state, ElapsedEventArgs e)
{
Commitzfile();
}
private void Commitzfile()
{
//Stopwatch stopwatch = new Stopwatch();
//stopwatch.Start();
fs.Flush();
fs.Close();
// fs.Dispose();
//stopwatch.Stop();
filecount++;
initialiseFile(filecount);
}
}

The most strait forward solution is to add lock to your code. This will prevent any race condition, also will make your code a bit slower since it will block threads while the resource is under use.
You can use it like this:
// add new lock object
object lockObject = new object();
FileStream fs = null;
int filecount = 0;
long a = 1000000000000000;
…
private void WriteINFile()
{
string sen = " write in file ";
for (int i = 0; i < a; i++)
{
Byte[] title = new UTF8Encoding(true).GetBytes(sen);
lock (lockObject)
{
fs.Write(title, 0, title.Length);
}
}
}
…
private void Commitzfile()
{
lock(lockObject)
{
fs.Flush();
fs.Close();
filecount++;
initialiseFile(filecount);
}
}
I suggest you read the following documentation about lock statement
Update
In concurrent programming there are a number of problems that need to consider. One of these is the resource management. Usually the problem is presented with the Dining philosophers problem. The main issue happens when a resource is tried to be used by multiple processes/threads, but the resource is bound to be used by one at a time. In your case it is the FileStream since one thread is trying to write through it all the time while on the other thread it is close and changed. You can see that in some cases it can happen that one closes the stream while the other still trying to write through it at the same time. To overcome this issue you need to make sure that the resource is accessed by only one thread at a time. One tool to do this is the lock statement.
Other things to keep in mind:
Deadlocks.
How many concurrent thread worth running at the same time.
Hope this brief summary helps. Feel free to ask a new question (or look up if there one already) if you need more info.

Background worker stealing main thread

So I can't say for sure this is the issue but I'm just about positive it is. I have a recordset of IVR calls to make. I put the data for each one in a concurrent queue and start 5 background workers to start working from the queue. However, after making 2 calls, the calls stop coming until one person hangs up, then it moves on to call number 3,4,5 etc. Are the any issues with this code?
It seems like the background workers are blocking eachother from calling the same method...? Is that possible?
private ConcurrentQueue<DataTable> _ivrCallsQueue = new ConcurrentQueue<DataTable>();
private List<BackgroundWorker> _ivrCallers = new List<BackgroundWorker>();
public overrid void Process()
{
foreach(DataRow row in _tblRecordsToProcess.Rows)
{
_workingActionItem = actionItemDAL.GetActionItemFromId(Convert.ToInt32(row["FNActionItemId"].ToString()));
var workingActionItemsTable = actionItemDAL.GetActionItemParamValues(Convert.ToInt32(row["FNActionItemId"].ToString()));
ivrCallsQueue.Enqueue(workingActionItemsTable);
}
StartCalls();
while (_ivrCallers.Count != 0)
{
testurls = testurls;
}
}
private void StartCalls()
{
int maxLines = 5;
if (_ivrCallsQueue.Count < maxLines)
{
maxLines = _ivrCallsQueue.Count;
}
for (int i = 0; i < maxLines; i++)
{
DataTable workingCall = new DataTable();
_ivrCallsQueue.TryDequeue(out workingCall);
BackgroundWorker ivrCaller = new BackgroundWorker();
_ivrCallers.Add(ivrCaller);
ivrCaller.DoWork += delegate(object sender, DoWorkEventArgs e)
{
RequestIVR(workingCall, Convert.ToInt32(workingCall.Rows[2][0].ToString()));
_ivrCallers.Remove(ivrCaller);
};
ivrCaller.RunWorkerCompleted += (bw_AnalyzeResults);
ivrCaller.RunWorkerAsync();
}
}
private void bw_AnalyzeResults(object sender, RunWorkerCompletedEventArgs e)
{
DataTable workingCall = new DataTable();
if (_ivrCallsQueue.Count != 0)
{
_ivrCallsQueue.TryDequeue(out workingCall);
BackgroundWorker ivrCaller = new BackgroundWorker();
ivrCaller.DoWork += delegate(object completeSender, DoWorkEventArgs completeArgs)
{
RequestIVR(workingCall, Convert.ToInt32(workingCall.Rows[2][0].ToString()));
_ivrCallers.Remove(ivrCaller);
};
ivrCaller.RunWorkerCompleted += (bw_AnalyzeResults);
ivrCaller.RunWorkerAsync();
}
else
{
}
}
private void RequestIVR(DataTable workingTable,int fnActionItemID)
{
var urlRequest = "http://uccx_http_trigger:9080/test?strTestMode=1&strTaskID=" + fnActionItemID;
var webClient = new WebClient { UseDefaultCredentials = true, Proxy = WebRequest.DefaultWebProxy };
DecodeResponseType(GetValueFromElement("Response Code was ", webClient.DownloadString(urlRequest)));
}

This will spawn at most five threads that each attempt to pull the next item from the queue and process it. If the queue is empty the attempt will fail and the thread will simply exit:
private List<System.Threading.Thread> Threads = new List<System.Threading.Thread>();
private ConcurrentQueue<DataTable> _ivrCallsQueue = new ConcurrentQueue<DataTable>();
private void StartCalls()
{
int maxLines = Math.Min(5 , _ivrCallsQueue.Count);
for (int i = 0; i < maxLines; i++ )
{
System.Threading.Thread T = new System.Threading.Thread(delegate()
{
DataTable workingCall;
while (_ivrCallsQueue.TryDequeue(out workingCall))
{
RequestIVR(workingCall, Convert.ToInt32(workingCall.Rows[2][0].ToString()));
}
});
Threads.Add(T);
T.Start();
}
}
The threads will keep running until all the items have been processed.

It looks like bw_AnalyzeResults does pretty much the same thing that StartCalls() does. In other words, when the background worker has finished its work, you immediately enqueue the same work to happen again, recursively forever?
By the looks of it, you want bw_AnalyzeResults to analyze the results returned by calling your web service. That is not what is happening at the moment.
The code below taken from the bw_AnalyzeResults event handler is scheduling a background job and making itself handle the RunWorkerCompleted event. So, presumably the software keeps going around and around executing bw_AnalyzeResults forever until you kill the process?
private void bw_AnalyzeResults(object sender, RunWorkerCompletedEventArgs e)
{
ivrCaller.DoWork += delegate(object completeSender, DoWorkEventArgs completeArgs)
{
RequestIVR(workingCall, Convert.ToInt32(workingCall.Rows[2][0].ToString()));
_ivrCallers.Remove(ivrCaller);
};
ivrCaller.RunWorkerCompleted += (bw_AnalyzeResults);
}

Serial port data received handled too slowly

I am reading data from an arduino at a baud rate of 115200. The data comes in as a string on its own line in the format: <ID,Name/Data>.
I believe that the problem with my code is that it is not handling the incoming data fast enough and the incoming data is being forced to wait for old data to be processed.
The incoming string is split into the three separate categories (ID, Name, Data) and added to a data table called dtFromGrid which is bound to dataGridView1.
Is there any errors or suggestions for how to improve my code performance? Would a separate thread for the handling function work better than BeginInvoke?
serialPort1.DataReceived += new SerialDataReceivedEventHandler(serialPort1_DataReceived);
private void serialPort1_DataReceived(object sender, System.IO.Ports.SerialDataReceivedEventArgs e)
{
string inData = serialPort1.ReadLine();
if (PauseButton.Text == "Pause" && inData.StartsWith("<"))
{
try
{
this.BeginInvoke(new SetGridDeleg(DoUpdate), new object[] {inData});
}
catch
{
}
}
}
private void DoUpdate(string inData) //inData passed in so that Serial port read only once
{
if (dtFromGrid == null)
{
dtFromGrid = new DataTable();
dtFromGrid.Columns.Add("Time", typeof(String));
dtFromGrid.Columns.Add("ID", typeof(String));
dtFromGrid.Columns.Add("Name", typeof(String));
dtFromGrid.Columns.Add("Data", typeof(String));
}
DataRow dr = dtFromGrid.NewRow();
TimeSpan ts = stopWatch.Elapsed;
dr["Time"] = String.Format("{0:00}:{1:00}:{2:00}.{3:000}",
ts.Hours, ts.Minutes, ts.Seconds,
ts.Milliseconds);
dr["ID"] = inData.Split(new char[] { '<', ',' })[1];
dr["Name"] = inData.Split(new char[] { ',', '/' })[1];
dr["Data"] = inData.Split(new char[] { '/', '>' })[1];
dtFromGrid.Rows.InsertAt(dr, 0);
//Replace old data with new data if ID's are the same to showo list of only newest data per each ID
if (NewestButton.Text == "Chronological")
{
for (int i = 1; i < dataGridView1.Rows.Count; i++)
{
if (dtFromGrid.Rows[i].ItemArray[1].ToString() == dtFromGrid.Rows[0].ItemArray[1].ToString())
{
dtFromGrid.Rows[i].Delete();
break;
}
}
}
//Keep a maximum of 50 rows of data
if (dtFromGrid.Rows.Count == 51)
{
dtFromGrid.Rows[50].Delete();
}
dtFromGrid.AcceptChanges();
dataGridView1.DataSource = dtFromGrid;
//keep focus of dataGridView on top row
dataGridView1.CurrentCell = dataGridView1.Rows[0].Cells[0];
// add newest row to a logfile if the user has set one
if (logFile != "")
{
using (StreamWriter sw = File.AppendText(logFile))
{
DataRow row = dtFromGrid.Rows[0];
object[] array = row.ItemArray;
int col = 0;
for (col = 0; col < array.Length - 1; col++)
{
sw.Write(array[col].ToString() + "\t|\t");
}
sw.Write(array[col].ToString());
sw.WriteLine();
sw.Close();
}
}
}
Update
I am now using a separate thread as suggested but I am having errors with invoking inside of that thread. I get multiple errors at random but the most common is "Index out of range." My invoke code is as follows:
this.Invoke((MethodInvoker) delegate
{
dtFromGrid.AcceptChanges();
dataGridView1.DataSource = dtFromGrid;
dataGridView1.CurrentCell = dataGridView1.Rows[0].Cells[0];
});

You store the data in a queue and offload the work to a secondary thread. This only works if, on the average, you are able to process the data at the rate it is coming in. Otherwise, the size of the queue will keep growing as you fall behind.
First, start with a wrapper around Queue<T> that will allow one thread to write to the queue and another to read from it in a thread safe manner. Also, allows the reader thread to block waiting for data.
public class ThreadedQueue<T>
{
private readonly Queue<T> _queue = new Queue<T>();
private readonly ManualResetEvent _notEmptyEvt = new ManualResetEvent(false);
public WaitHandle WaitHandle { get { return _notEmptyEvt; } }
public void Enqueue(T obj)
{
lock (_queue)
{
_queue.Enqueue(obj);
_notEmptyEvt.Set();
}
}
public T Dequeue()
{
_notEmptyEvt.WaitOne(Timeout.Infinite);
lock (_queue)
{
var result = _queue.Dequeue();
if (_queue.Count == 0)
_notEmptyEvt.Reset();
return result;
}
}
}
In your serial port handler, write the data into the queue:
private void serialPort1_DataReceived(object sender, System.IO.Ports.SerialDataReceivedEventArgs e)
{
string inData = serialPort1.ReadLine();
if (PauseButton.Text == "Pause" && inData.StartsWith("<"))
{
_queue.Enqueue(inData);
}
}
In the secondary thread, read from the queue and do the invoke to the GUI thread:
private void ThreadProc()
{
while (true)
{
string inData = _queue.Dequeue();
this.Invoke(new SetGridDeleg(DoUpdate), new object[] {inData});
}
}
Start up the secondary thread like this:
Thread th = new Thread(ThreadProc);
th.IsBackground = true;
th.Start();
Of course you'll need to create an instance of the queue:
ThreadedQueue<string> _queue = new ThreadedQueue<string>();

I normally design a SerialService class to manage the SerialPort. Below is a simple version of the SerialService class.
The role of the SerialService class is to read the serial buffer as fast as possible. This clears the buffer and prevents any serial port errors. This raw data is then passed to the parser.
The trick for performance is in your parser. YourParser should also be fast in formatting the raw data into the string that you are expecting. Once your data is parse you may use a callback or an event. With a callback or event your parser will continue to parse new arriving data. YourParse is now a testable class.
Once you have your good data from the parser's callback use BeginInvoke to send the data to the main thread where your ui can then display it.
If you are not in the main UI thread and you try to update the UI from another thread you will have the cross theading problem.
Good luck.
class Program
{
private static YourDataParser _parser;
static void Main(string[] args)
{
_parser = new YourDataParser();
var serial = new SerialService("COM1");
serial.DataReceived += serial_DataReceived;
}
static void serial_DataReceived(object sender, DataReceivedEventArgs e)
{
_parser.HandleTheData(e.Data, good =>
{
// here is your good data
// This is not the main thread invoke your UI from here with the good data
// Use BeginInvoke to invoke the main thread
});
}
}
public class YourDataParser
{
private List<byte> _buffer = new List<byte>();
public void HandleTheData(byte[] rawdata, Action<string> goodData)
{
_buffer.AddRange(rawdata);
foreach (var b in _buffer)
{
var thechar = (char) b;
// handle your raw data... like look for the character '<'
// or look for the end of line this would be CR (0x0D) LF (0x0A)
// you can reference the ASCII table for the characters byte values
}
// and return the good data
var data = "your good data after parsing it";
goodData(data);
}
}
public class DataReceivedEventArgs : EventArgs
{
public DataReceivedEventArgs(byte[] data)
{
Data = data;
}
public byte[] Data { get; private set; }
}
class SerialService
{
public event EventHandler<DataReceivedEventArgs> DataReceived;
private SerialPort _port;
public SerialService(string comm)
{
_port = new SerialPort(comm)
{
// YOUR OTHER SETTINGS HERE...
ReceivedBytesThreshold = 1 // I think is better to increase this number if you know the minimum number of bytes that will arrive at the serial port's buffer
};
// Note that the ReceivedBytesThreshold is set to 1.
// That means that the port_DataReceived event will fire with a minimun of 1 byte in the serial buffer
_port.DataReceived += port_DataReceived;
}
void port_DataReceived(object sender, SerialDataReceivedEventArgs e)
{
if (e.EventType != SerialData.Chars) return;
while (_port.IsOpen & _port.BytesToRead != 0)
{
// important to get all the bytes off the buffer
var size = _port.BytesToRead;
var buffer = new byte[size];
var sizeRead = _port.Read(buffer, 0, size);
OnDataReceived(buffer);
}
}
protected virtual void OnDataReceived(byte[] data)
{
var ev = DataReceived;
if (ev != null) ev(this, new DataReceivedEventArgs(data));
}
}

As you said your code is slowing the data reception.
you can solve your problem by queuing your data to a queue list and a background process will be processing this list one by one.
another approach is to create a new thread on the reception of each data batch.
Example (second approach)
private void serialPort1_DataReceived(object sender, System.IO.Ports.SerialDataReceivedEventArgs e)
{
string inData = serialPort1.ReadLine();
System.Threading.Thread T = new System.Threading.Thread(new System.Threading.ParameterizedThreadStart(ProcessData));
T.Start(inData);
}
public void ProcessData(Object data)
{
....
}

Architecture of real-time, iteration application

Sorry for abstract question, but I'm looking for some samples/advices/articles on type of applications which does some equivalent operations in cycle, and every iteration of cycle should expose its result in certain portion of time (for instance, 10 seconds).
My application does synchronization of data between external WCF service and local database. In every iteration an application retrieves changes of data passing request to WCF service and puts changes to database and vice versa. One of most hard requirement for this application is that iterations should fire every ten seconds.
So here is the issues arises. How can I guarantee that iteration will finish for no more than 10 seconds?
I guess this type of applications called real-time applications (in maner of real-time OS).
DAL components that we use acts randomly on connection timeout behavior. So DB operations may take longer time than 10 seconds.
Here is the estimated code of one iteration:
Stopwatch s1 = new Stopwatch();
s1.Start();
Parallel.ForEach(Global.config.databases, new ParallelOptions { MaxDegreeOfParallelism = -1 }, (l) =>
{
Console.WriteLine("Started for {0}", l.key.name);
DB db = new DB(l.connectionString);
DateTime lastIterationTS = GetPreviousIterationTS(l.id);
ExternalService serv = new ExternalService(l.id);
List<ChangedData> ChangedDataDb = db.GetChangedData(DateTime.Now.AddSeconds((lastIterationTS == DateTime.MinValue) ? -300 : -1 * (DateTime.Now - lastIterationTS).Seconds));
List<Data> ChangedDataService = serv.GetModifiedData();
Action syncDBChanges = new Action(() =>
{
// Изменения в БД
foreach (ChangedData d in ChangedDataDb)
{
try
{
// ...
// analyzing & syncing
}
catch (Exception e)
{
logger.InfoEx("Exception_SyncDatabase", e.ToString());
}
}
}
);
Action syncService = new Action(() =>
{
foreach (Data d in ChangedDataService)
{
try
{
// ...
// analyzing & syncing
}
catch (Exception e)
{
logger.InfoEx("Exception_SyncService", e.ToString());
}
}
});
List<WaitHandle> handles = new List<WaitHandle>();
IAsyncResult ar1 = syncDBChanges.BeginInvoke(syncDBChanges.EndInvoke, null);
IAsyncResult ar2 = syncService.BeginInvoke(syncService.EndInvoke, null);
handles.Add(ar1.AsyncWaitHandle);
handles.Add(ar2.AsyncWaitHandle);
WaitHandle.WaitAll(handles.ToArray(), (int)((Global.config.syncModifiedInterval - 1) * 1000));
SetCurrentIterationTS(l.id);
}
catch (Exception e)
{
Console.WriteLine(e.Message);
logger.InfoEx("Exception_Iteration", e.ToString());
continue;
}
}
logger.InfoEx("end_Iteration", IterationContextParams);
}
);
s1.Stop();
Console.WriteLine("Main iteration done for {0}...", s1.Elapsed);

You can consider a couple of options...
Kill the iteration if it exceeds more than 10 seconds and hope that the next iteration can complete process. The issue with this approach is that there is a good possibility that the none of the iterations will complete and therefore the synchronization process will never occur. I would recommend the following option...
If the iteration takes more than 10 seconds, wait for it to complete and skip the next iteration(s). This way you ensure the process completes atleast once. The following is a simplified code sample for reference...
class Updater
{
Timer timer = new Timer();
public object StateLock = new object();
public string State;
public Updater()
{
timer.Elapsed += timer_Elapsed;
timer.Interval = 10000;
timer.AutoReset = true;
timer.Start();
}
void timer_Elapsed(object sender, ElapsedEventArgs e)
{
if (State != "Running")
{
Process();
}
}
private void Process()
{
try
{
lock (StateLock)
{
State = "Running";
}
// Process
lock (StateLock)
{
State = "";
}
}
catch
{
throw;
}
}
}
...
class Program
{
static void Main(string[] args)
{
Updater updater = new Updater();
Console.ReadLine();
}
}

Quartz.net is an excellent scheduler for the .NET platform, which I think could suit your needs.
If you want to kill a job, you can implement IInterruptableJob. You should be able to add some cleanup code in the Interupt method to dispose of any db connections.
If you want finish a job, but only start another job if the last one is completed (which I think is the better option), you can implement IStatefulJob interface

I usually separate the update cycle from the actual timer
The timer does two things:
1) if the update is not running starts it.
2) if the service is already running set a flag for it to continue running.
The update cycle:
1)set running flag
2) do the update
3) set running flag to false
4) if continue running is set go to 1).

You might want to read up on the variety of Timer objects available in .Net: http://msdn.microsoft.com/en-us/magazine/cc164015.aspx
I personally like System.Threading.Timer because you can easily use lambdas, and it allows a state object to be passed if you create a separate callback.
I would also recommend using the System.Threading.Tasks library, because it allows you to gracefully handle cancellations in the case that the timer elapses before your work is completed. Msdn example: http://msdn.microsoft.com/en-us/library/dd537607.aspx
Here's an example of using these components together in a 10 minute timer:
Note: to do this with your sql database you'll need to set Asynchronous Processing=true; and MultipleActiveResultSets=True;
CancellationTokenSource cancelSource = new CancellationTokenSource();
System.Threading.Timer timer = new System.Threading.Timer(callback =>
{
//start sync
Task syncTask = Task.Factory.StartNew(syncAction =>
{
using (SqlConnection conn =
new SqlConnection(
ConfigurationManager.ConnectionStrings["db"].ConnectionString))
{
conn.Open();
using (SqlCommand syncCommand = new SqlCommand
{
CommandText = "SELECT getdate() \n WAITFOR DELAY '00:11'; ",
CommandTimeout = 600,
Transaction = conn.BeginTransaction(),
Connection = conn
})
{
try
{
IAsyncResult t = syncCommand.BeginExecuteNonQuery();
SpinWait.SpinUntil(() =>
(t.IsCompleted || cancelSource.Token.IsCancellationRequested));
if (cancelSource.Token.IsCancellationRequested && !t.IsCompleted)
syncCommand.Transaction.Rollback();
}
catch (TimeoutException timeoutException)
{
syncCommand.Transaction.Rollback();
//log a failed sync attepmt here
Console.WriteLine(timeoutException.ToString());
}
finally
{
syncCommand.Connection.Close();
}
}
}
}, null, cancelSource.Token);
//set up a timer for processing in the interim, save some time for rollback
System.Threading.Timer spinTimer = new System.Threading.Timer(c => {
cancelSource.Cancel();
}, null, TimeSpan.FromMinutes(9), TimeSpan.FromSeconds(5));
//spin here until the spintimer elapses;
//this is optional, but would be useful for debugging.
SpinWait.SpinUntil(()=>(syncTask.IsCompleted || cancelSource.Token.IsCancellationRequested));
if (syncTask.IsCompleted || cancelSource.Token.IsCancellationRequested)
spinTimer.Dispose();
}, null, TimeSpan.FromMinutes(1), TimeSpan.FromMinutes(10));

Perhaps try this. Please make sure you do not create and use any new threads in DoWork() method.
class DatabaseUpdater
{
private readonly Timer _timer;
private List<Thread> _threads;
private readonly List<DatabaseConfig> _dbConfigs;
public DatabaseUpdater(int seconds, List<DatabaseConfig> dbConfigs)
{
_timer = new Timer(seconds * 1000);
_timer.Elapsed += TimerElapsed;
_dbConfigs = dbConfigs;
}
public void Start()
{
StartThreads();
_timer.Start();
}
public void Stop()
{
_timer.Stop();
StopThreads();
}
void TimerElapsed(object sender, ElapsedEventArgs e)
{
StopThreads();
StartThreads();
}
private void StartThreads()
{
var newThreads = new List<Thread>();
foreach (var config in _dbConfigs)
{
var thread = new Thread(DoWork);
thread.Start(config);
newThreads.Add(thread);
}
_threads = newThreads;
}
private void StopThreads()
{
if (_threads == null) return;
var oldThreads = _threads;
foreach (var thread in oldThreads)
{
thread.Abort();
}
}
static void DoWork(object objConfig)
{
var dbConfig = objConfig as DatabaseConfig;
if (null == dbConfig) return;
var n = GetRandomNumber();
try
{
ConsoleWriteLine("Sync started for : {0} - {1} sec work.", dbConfig.Id, n);
// update/sync db
Thread.Sleep(1000 * n);
ConsoleWriteLine("Sync finished for : {0} - {1} sec work.", dbConfig.Id, n);
}
catch (Exception ex)
{
// cancel/rollback db transaction
ConsoleWriteLine("Sync cancelled for : {0} - {1} sec work.",
dbConfig.Id, n);
}
}
static readonly Random Random = new Random();
[MethodImpl(MethodImplOptions.Synchronized)]
static int GetRandomNumber()
{
return Random.Next(5, 20);
}
[MethodImpl(MethodImplOptions.Synchronized)]
static void ConsoleWriteLine(string format, params object[] arg)
{
Console.WriteLine(format, arg);
}
}
static void Main(string[] args)
{
var configs = new List<DatabaseConfig>();
for (var i = 1; i <= 3; i++)
{
configs.Add(new DatabaseConfig() { Id = i });
}
var databaseUpdater = new DatabaseUpdater(10, configs);
databaseUpdater.Start();
Console.ReadKey();
databaseUpdater.Stop();
}

Issue looping through a DataTable Rows multiple times

::EDIT::
Ok folks, it seems that I'm an idiot after all. The problem had nothing at all to do with my code, and everything to do with Visual Studio having overwritten my SQLite Database with a previous (and empty) version. It does seem that there is a great discussion about thread safety going on, so I will stick around to read more!
::/EDIT::
I am attempting to use multiple background workers to loop through the rows of a database 100 records at a time while avoiding duplication, but I seem to be having some issues.
Basically, I start off by creating 10 background workers in a loop, and adding them to a List. Then I loop through the background workers in the List, and for each one I do RunWorkerAsync(), and then sleep the main thread for 5 seconds. Inside the DoWork method of each background worker, I have the worker select 100 rows from the database where a particular field is set to its default value.
From here, I want to first loop through every returned row and change that default value to an "In Progress" value, and then loop through the values again and actually do the processing required to find correct values for those fields. The problem I seem to be having is that I seem to be have an empty DataTable after the first iteration through the results, and I suspect that my issues stem from shallow copying.
Here is the code:
using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Data;
using System.IO;
using System.Text;
using DBFill.GeoCodeService;
using System.Diagnostics;
using System.Runtime.Serialization.Formatters.Binary;
namespace DBFill {
class Program {
public static int completedGeocodes = 0;
static void Main(string[] args) {
SQLiteDatabase db = new SQLiteDatabase("zipCodes.s3db");
List<BackgroundWorker> workers = new List<BackgroundWorker>();
for (int i = 0; i < 10; i++) {
BackgroundWorker b = new BackgroundWorker();
b.DoWork += new DoWorkEventHandler(worker_DoWork);
b.RunWorkerCompleted += new RunWorkerCompletedEventHandler(worker_RunWorkerCompleted);
b.WorkerReportsProgress = true;
b.ProgressChanged += new ProgressChangedEventHandler(b_ProgressChanged);
workers.Add(b);
}
int counter = 0;
foreach (BackgroundWorker b in workers) {
Debug.WriteLine("Worker {0} is starting.", counter);
b.RunWorkerAsync(b);
counter++;
System.Threading.Thread.Sleep(5000);
}
Boolean running = true;
while (running) {
running = false;
foreach (BackgroundWorker b in workers) {
Debug.WriteLine("Checking background Worker");
if (b.IsBusy) {
running = true;
}
}
System.Threading.Thread.Sleep(5000);
}
}
static void b_ProgressChanged(object sender, ProgressChangedEventArgs e) {
Console.WriteLine(".");
}
static void worker_RunWorkerCompleted(object sender, RunWorkerCompletedEventArgs e) {
}
static void worker_DoWork(object sender, DoWorkEventArgs e) {
BackgroundWorker b = (BackgroundWorker)e.Argument;
SQLiteDatabase db = new SQLiteDatabase("zipCodes.s3db");
DataTable results = get100Records();
DataTable temp = DeepClone<DataTable>(results);//results;
Dictionary<String, String> marker = new Dictionary<string, string>();
marker["LATITUDE"] = "In Progress";
foreach (DataRow row in temp.Rows) {
Debug.WriteLine("Working with zip {0}", row["ZIP_CODE"]);
db.Update("ZIP_CODES", marker, String.Format("ZIP_CODE = '{0}'", row["ZIP_CODE"]));
}
foreach (DataRow row in results.Rows) {
String geoCodeResponse = GeoCodeZip(row["ZIP_CODE"].ToString());
Debug.WriteLine(String.Format("Attempting Zip: {0}", row["ZIP_CODE"].ToString()));
if (geoCodeResponse != "There was an error") {
marker["LATITUDE"] = geoCodeResponse.Split(',')[0];
marker["LONGITUDE"] = geoCodeResponse.Split(',')[1];
Console.WriteLine(String.Format("#{0} updated successfully", completedGeocodes));
}
else {
marker["LATITUDE"] = "Not Set";
Console.WriteLine(String.Format("#{0} failed", completedGeocodes));
}
db.Update("ZIP_CODES", marker, String.Format("ZIP_CODE = '{0}'", row["ZIP_CODE"]));
db.ExecuteNonQuery("commit");
b.ReportProgress(1);
completedGeocodes++;
}
}
private static DataTable get100Records() {
SQLiteDatabase db = new SQLiteDatabase("zipCodes.s3db");
DataTable results = db.GetDataTable("select ZIP_CODE from ZIP_CODES where LATITUDE = 'Not Set' LIMIT 100");
return results;
}
private static String GeoCodeZip(String zip) {
try {
GeocodeRequest request = new GeocodeRequest();
request.Credentials = new GeoCodeService.Credentials();
request.Credentials.ApplicationId = "API_KEY";
request.Query = zip;
ConfidenceFilter[] filters = new ConfidenceFilter[1];
filters[0] = new ConfidenceFilter();
filters[0].MinimumConfidence = Confidence.High;
GeocodeOptions opts = new GeocodeOptions();
opts.Filters = filters;
request.Options = opts;
GeocodeServiceClient service = new GeocodeServiceClient("BasicHttpBinding_IGeocodeService");
GeocodeResponse response = service.Geocode(request);
if (response.Results.Length > 0) {
return String.Format("{0},{1}", response.Results[0].Locations[0].Latitude, response.Results[0].Locations[0].Longitude);
}
else {
Debug.WriteLine(String.Format("{0}", response.ResponseSummary.FaultReason));
return "There was an error";
}
}
catch (Exception e) {
Debug.WriteLine(e.Message);
return "There was an error";
}
}
public static T DeepClone<T>(T obj) {
using (var ms = new MemoryStream()) {
var formatter = new BinaryFormatter();
formatter.Serialize(ms, obj);
ms.Position = 0;
return (T)formatter.Deserialize(ms);
}
}
}
}
Any Ideas?

It seems that your delay and why you want to multi-thread it, is not to read records from the database, but rather calling out to the GeocodeServiceClient.
You could try reworking your main method to fetch all the records from the DB sequentially and parse them. You then split that list into even chunks and spin up background workers to run them through the Geocode service.
Another option would be to put the records into a queue and have each background worker pop one off the queue work on it, then go back to the queue while there are still unprocessed records. You will need to be careful of locking In C# would it be better to use Queue.Synchronized or lock() for thread safety?.