I have the following code:
static void Main(string[] args)
{
DateTime currentDay = DateTime.Now;
List<Task> taskList = new List<Task>();
List<string> markets = new List<string>() { "amex", "nasdaq", "nyse", "global" };
Parallel.ForEach(markets, market =>
{
Downloads.startInitialMarketSymbolsDownload(market);
}
);
Console.WriteLine("All downloads finished!");
}
public static void startInitialMarketSymbolsDownload(string market)
{
try
{
object valueTypeLock = new object();
List<string> symbolList = new List<string>() { "GOOG", "YHOO", "AAP" }
var historicalGroups = symbolList.AsParallel().Select((x, i) => new { x, i })
.GroupBy(x => x.i / 100)
.Select(g => g.Select(x => x.x).ToArray());
historicalGroups.AsParallel().ForAll(g => {
lock (valueTypeLock) {
getHistoricalStockData(g, market);
}
});
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
Console.WriteLine(ex.StackTrace);
}
}
public static void getHistoricalStockData(string[] symbols, string market)
{
// download data for list of symbols and then upload to db tables
Uri uri;
string url, line;
decimal open = 0, high = 0, low = 0, close = 0, adjClose = 0;
DateTime date;
Int64 volume = 0;
string[] lineArray;
List<string> symbolError = new List<string>();
Dictionary<string, string> badNameError = new Dictionary<string, string>();
System.Net.ServicePointManager.DefaultConnectionLimit = 1000;
Parallel.ForEach(symbols, symbol =>
{
url = "http://ichart.finance.yahoo.com/table.csv?s=" + symbol + "&a=00&b=1&c=1900&d=" + (DateTime.Now.Month - 1) + "&e=" + DateTime.Now.Day + "&f=" + DateTime.Now.Year + "&g=d&ignore=.csv";
uri = new Uri(url);
using (ooplesfinanceEntities entity = new ooplesfinanceEntities())
using (WebClient client = new WebClient())
using (Stream stream = client.OpenRead(uri))
using (StreamReader reader = new StreamReader(stream))
{
entity.Database.Connection.Open();
while (reader.EndOfStream == false)
{
line = reader.ReadLine();
lineArray = line.Split(',');
// if it isn't the very first line
if (lineArray[0] != "Date")
{
switch (market)
{
case "amex":
DailyAmexData amexData = new DailyAmexData();
var amexQuery = from r in entity.DailyAmexDatas.AsParallel().AsEnumerable()
where r.Date == date
select new StockData { Close = r.AdjustedClose };
List<StockData> amexResult = amexQuery.AsParallel().ToList();
if (amexResult.AsParallel().Count() > 0) // **never hits this breakpoint line**
{
// add the row to the table if it isn't there
// now checks for stock splits and updates if necessary
if (amexResult.AsParallel().FirstOrDefault().Close != adjClose)
{
// this means there is a stock split so it needs to have the other adjusted close prices updated
amexResult.AsParallel().FirstOrDefault().Close = adjClose;
}
else
{
continue;
}
}
else
{
// set the data then add it
amexData.Symbol = symbol;
entity.DailyAmexDatas.Add(amexData);
}
break;
default:
break;
}
}
}
// now save everything
entity.SaveChanges();
Console.WriteLine(symbol + " added to the " + market + " database!");
}
}
);
}
Everything starts fine and I get no exceptions but I'm getting no results and the code gets stuck one the line that I marked and the memory keeps shooting up. I figured the problem was with the above code because maybe I was doing something wrong. I just don't know where to start as this is my first time dealing with plinq/parallel processing and the tutorials don't show anything this complex.
Related
I am working on EF. I am trying to insert into a table, the insert function is in a thread.
private void DataReceivedHandler(object sender, SerialDataReceivedEventArgs e)
{
int bytes = port.BytesToRead;
//string indata = sp.ReadExisting();
Thread.Sleep(50);
try
{
receivedBytes = port.BaseStream.Read(buffer, 0, (int)buffer.Length);
}
catch (Exception ex)
{
Console.WriteLine(ex.Message.ToString());
}
var receiveData = BitConverter.ToString(buffer, 0, receivedBytes);
var finalData = receiveData.Replace("-", "");
//Thread.Sleep(100);
Console.WriteLine("Thread Going to Start");
new Thread(() => {
SaveData(finalData);
}).Start(); // starting the thread
port.DiscardOutBuffer();
port.DiscardInBuffer();
}
And this is my save data function
public void SaveData(string finalData)
{
Console.WriteLine(LineNumber() + "Data Transmiting...");
thread = Thread.CurrentThread;
mdc_dbEntities e = new mdc_dbEntities();
var msn = e.mdc_meter_config.Where(m => m.m_hex == sr).Select(s => new { s.msn, s.p_id, s.meter_id }).ToList();
var H = finalData.Substring(0, 2);
using (mdc_dbEntities u = new mdc_dbEntities())
{
foreach (var res in msn)
{
var cust_id = e.mdc_meter_cust_rel.Where(m => m.msn == res.msn)
.Select(s => s.cust_id)
.FirstOrDefault();
mdc_meters_data data = new mdc_meters_data()
{
msn = res.msn,
cust_id = cust_id,
device_id = res.meter_id.ToString(),
kwh = e_val.ToString(),
voltage_p1 = a_vol_val.ToString(),
voltage_p2 = b_vol_val.ToString(),
voltage_p3 = c_vol_val.ToString(),
current_p1 = a_curr_val.ToString(),
current_p2 = b_curr_val.ToString(),
current_p3 = c_curr_val.ToString(),
data_date_time = Convert.ToDateTime(theDate.ToString(format)),
d_type = d_type.ToString(),
pf1 = a_pf_val.ToString(),
pf2 = b_pf_val.ToString(),
pf3 = c_pf_val.ToString(),
p_id = res.p_id,
};
u.mdc_meters_data.Add(data);
}
u.SaveChanges();
}
Console.WriteLine(LineNumber() + "Data Saved");
Thread.Sleep(50);
}
try
{
thread.Abort(); // aborting it after insertion
//Thread.Sleep(50);
}
catch (Exception ex)
{
Console.WriteLine(ex.Message.ToString());
}
}
The above code runs for some time, but after that I encountered an error at u.SaveChanges();
System.Data.Entity.Core.EntityException: 'An error occurred while closing the provider connection. See the inner exception for details.'
MySqlException: Fatal error encountered during command execution.
MySqlException: Fatal error encountered attempting to read the resultset.
MySqlException: Reading from the stream has failed.
IOException: Unable to read data from the transport connection: A connection attempt failed because the connected party did not properly respond after a period of time, or established connection failed because connected host has failed to respond.
SocketException: A connection attempt failed because the connected party did not properly respond after a period of time, or established connection failed because connected host has failed to respond
I have looked into each solution and tried them but still unable to resolve this issue. I must be missing something that I don't know.
Update 1 My whole code
Calling constructor
public CommunicationEngine()
{
port.ReadTimeout = 500;
port.DataReceived += new SerialDataReceivedEventHandler(DataReceivedHandler);
port.Open();
Console.WriteLine("Port opened successfully");
Console.WriteLine("I am Recieving");
}
Calling handler
private void DataReceivedHandler(object sender, SerialDataReceivedEventArgs e)
{
int bytes = port.BytesToRead;
Thread.Sleep(50);
Console.WriteLine("Bytes are ok..." + port.BytesToRead + " Recieved ");
try
{
receivedBytes = port.BaseStream.Read(buffer, 0, (int)buffer.Length);
}
catch (Exception ex)
{
Console.WriteLine(ex.Message.ToString());
}
var receiveData = BitConverter.ToString(buffer, 0, receivedBytes);
var finalData = receiveData.Replace("-", "");
//Thread.Sleep(100);
Console.WriteLine("Thread Going to Start");
try
{
new Thread(() => {
SaveData(finalData);
}).Start();
}
catch(Exception ex)
{
Console.WriteLine(ex.Message.ToString());
}
port.DiscardOutBuffer(); port.DiscardInBuffer();
}
Saving data into DB
public void SaveData(string finalData)
{
Console.WriteLine(LineNumber() + "Data Transmiting...");
thread = Thread.CurrentThread;
if (finalData.Length == 138)
{
comm = true;
var H = finalData.Substring(0, 2);
var FC = finalData.Substring(2, 9);
var len = finalData.Substring(10, 2);
var sr = finalData.Substring(12, 12);
var energy_tag = finalData.Substring(24, 4);
var e_val = hexToDec(finalData.Substring(28, 8)) / 10;
var a_curr_tag = finalData.Substring(36, 4);
var a_curr_val = hexToDec(finalData.Substring(40, 8)) / 1000;
var b_curr_tag = finalData.Substring(48, 4);
var b_curr_val = hexToDec(finalData.Substring(52, 8)) / 1000;
var c_curr_tag = finalData.Substring(60, 4);
var c_curr_val = hexToDec(finalData.Substring(64, 8)) / 1000;
var a_vol_tag = finalData.Substring(72, 4);
var a_vol_val = hexToDec(finalData.Substring(76, 8)) / 10;
var b_vol_tag = finalData.Substring(84, 4);
var b_vol_val = hexToDec(finalData.Substring(88, 8)) / 10;
var c_vol_tag = finalData.Substring(96, 4);
var c_vol_val = hexToDec(finalData.Substring(100, 8)) / 10;
var a_pf_tag = finalData.Substring(108, 4);
var a_pf_val = hexToDec(finalData.Substring(112, 4)) / 1000;
var b_pf_tag = finalData.Substring(116, 4);
var b_pf_val = hexToDec(finalData.Substring(120, 4)) / 1000;
var c_pf_tag = finalData.Substring(124, 4);
var c_pf_val = hexToDec(finalData.Substring(128, 4)) / 1000;
var crc = finalData.Substring(132, 4);
var ftr = finalData.Substring(136, 2);
var d_type = "600";
DateTime theDate = DateTime.Now;
string format = "yyyy-MM-dd HH:mm:ss";
Console.WriteLine(LineNumber() + "Data Ready to be inserted in DB");
using (mdc_dbEntities u = new mdc_dbEntities())
{
var msnList = u.mdc_meter_config.Where(m => m.m_hex == sr)
.Select(s => new { s.msn, s.p_id, s.meter_id })
.ToList();
foreach (var res in msnList)
{
var cust_id = u.mdc_meter_cust_rel.Where(m => m.msn == res.msn)
.Select(s => s.cust_id)
.FirstOrDefault();
mdc_meters_data data = new mdc_meters_data()
{
msn = res.msn,
cust_id = cust_id,
device_id = res.meter_id.ToString(),
kwh = e_val.ToString(),
voltage_p1 = a_vol_val.ToString(),
voltage_p2 = b_vol_val.ToString(),
voltage_p3 = c_vol_val.ToString(),
current_p1 = a_curr_val.ToString(),
current_p2 = b_curr_val.ToString(),
current_p3 = c_curr_val.ToString(),
data_date_time = Convert.ToDateTime(theDate.ToString(format)),
d_type = d_type.ToString(),
pf1 = a_pf_val.ToString(),
pf2 = b_pf_val.ToString(),
pf3 = c_pf_val.ToString(),
p_id = res.p_id,
};
u.mdc_meters_data.Add(data);
}
try
{
u.SaveChanges();
}
catch(Exception ex)
{
Console.WriteLine(ex.Message.ToString());
}
}
Console.WriteLine(LineNumber() + "Data Saved");
Thread.Sleep(50);
}
else if(finalData.Length == 30)
{
var msn_no = finalData.Substring(12, 12);
mdc_dbEntities p = new mdc_dbEntities();
var update = p.meter_control.Where(c => (c.comm_executed == 0))
.Where(o => (o.m_hex == msn_no))
.SingleOrDefault();
if(update.comm_sent == "Disconnect")
{
update.comm_executed = 1;
update.comm = 0;
p.SaveChanges();
Console.WriteLine("Meter Disconnected....");
}
else if(update.comm_sent == "Connect")
{
update.comm_executed = 1;
update.comm = 1;
p.SaveChanges();
Console.WriteLine("Meter Connected....");
}
comm = true;
}
else
{
comm = true;
}
try
{
thread.Abort();
}
catch (Exception ex)
{
Console.WriteLine(ex.Message.ToString());
}
}
Any help would be highly appreciated.
Executing EF related changes in a manually initiated thread is not a good idea. Try to run the EF changes in the same thread. If you are bothered with processing incoming requests, use Async, and Await feature. I have modified your code to accommodate this feature. Please try this.
private void DataReceivedHandler(object sender, SerialDataReceivedEventArgs e)
{
int bytes = port.BytesToRead;
//string indata = sp.ReadExisting();
try
{
receivedBytes = port.BaseStream.Read(buffer, 0, (int)buffer.Length);
}
catch (Exception ex)
{
Console.WriteLine(ex.Message.ToString());
}
var receiveData = BitConverter.ToString(buffer, 0, receivedBytes);
var finalData = receiveData.Replace("-", "");
Console.WriteLine("Thread Going to Start");
SaveDataAsync(finalData).Wait(); // this call will become sync and runs under main thread.
port.DiscardOutBuffer();
port.DiscardInBuffer();
}
public async Task<bool> SaveDataAsync(string finalData)
{
mdc_dbEntities e = new mdc_dbEntities();
var msn = e.mdc_meter_config.Where(m => m.m_hex == sr).Select(s => new { s.msn, s.p_id, s.meter_id }).ToList();
var H = finalData.Substring(0, 2);
var isSaveSuccess = false;
using (mdc_dbEntities u = new mdc_dbEntities())
{
foreach (var res in msn)
{
var cust_id = e.mdc_meter_cust_rel.Where(m => m.msn == res.msn)
.Select(s => s.cust_id)
.FirstOrDefault();
mdc_meters_data data = new mdc_meters_data()
{
msn = res.msn,
cust_id = cust_id,
device_id = res.meter_id.ToString(),
kwh = e_val.ToString(),
voltage_p1 = a_vol_val.ToString(),
voltage_p2 = b_vol_val.ToString(),
voltage_p3 = c_vol_val.ToString(),
current_p1 = a_curr_val.ToString(),
current_p2 = b_curr_val.ToString(),
current_p3 = c_curr_val.ToString(),
data_date_time = Convert.ToDateTime(theDate.ToString(format)),
d_type = d_type.ToString(),
pf1 = a_pf_val.ToString(),
pf2 = b_pf_val.ToString(),
pf3 = c_pf_val.ToString(),
p_id = res.p_id,
};
u.mdc_meters_data.Add(data);
}
isSaveSuccess = (await u.SaveChangesAsync())>0; // if records inserted, the count will be more than 0
}
return isSaveSuccess;
}
}
this is easier to read and may help
saying all this please ensure that you can actually make a connection to the db
double check connectionString
public void SaveData(string finalData)
{
Console.WriteLine(LineNumber() + "Data Transmiting...");
using (mdc_dbEntities dbContext = new mdc_dbEntities())
{
var msnList = dbContext.mdc_meter_config.Where(m => m.m_hex == sr)
.Select(s => new { s.msn, s.p_id, s.meter_id })
.ToList();
//put debug point here and check that msnList is populated
foreach (var item in msnList)
{
//this is slow as it will be a db query for each loop
var cust_id = dbContext.mdc_meter_cust_rel.Where(m => m.msn == item.msn)
.Select(s => s.cust_id)
.FirstOrDefault();
var data = new mdc_meters_data()
{
msn = item.msn,
cust_id = cust_id,
device_id = item.meter_id.ToString(),
kwh = e_val.ToString(),
voltage_p1 = a_vol_val.ToString(),
voltage_p2 = b_vol_val.ToString(),
voltage_p3 = c_vol_val.ToString(),
current_p1 = a_curr_val.ToString(),
current_p2 = b_curr_val.ToString(),
current_p3 = c_curr_val.ToString(),
data_date_time = Convert.ToDateTime(theDate.ToString(format)),
d_type = d_type.ToString(),
pf1 = a_pf_val.ToString(),
pf2 = b_pf_val.ToString(),
pf3 = c_pf_val.ToString(),
p_id = item.p_id,
};
dbContext.mdc_meters_data.Add(data);
}
//depending on how many you added this may take some time.
dbContext.SaveChanges();
}
}
I have a calculation that does I/O I want to send to another thread so on the web site the user can goto the next page without waiting for the calculation to finish. From what I understand are I need to do is call the method like this
var backgroundTask = Task.Run(() => CalculateSet(uow, profile, userId, specialtyCode ));
But when I do this is seems to call one line in the method and goes away.. non of the work is done.
Am I missing something ?
[Fact]
public void Calculation_Xls()
{
string currentDirectory = Directory.GetCurrentDirectory();
string filesDirectory = currentDirectory + "\\Files";
System.Text.Encoding.RegisterProvider(CodePagesEncodingProvider.Instance);
var builder = new ConfigurationBuilder().SetBasePath(Directory.GetCurrentDirectory()).AddJsonFile("appsettings.json");
Configuration = builder.Build();
var optionsBuilder = new DbContextOptionsBuilder<RetContext>();
optionsBuilder.UseSqlServer(Configuration.GetConnectionString("DefaultConnection"));
int i = 0; //outer loop
UnitOfWork uow = new UnitOfWork(new RetContext(optionsBuilder.Options));
using (var stream = System.IO.File.Open(filesDirectory + "\\t2UserProfileDataTwoUserPerSpecialty.xlsx",
FileMode.Open, FileAccess.Read))
{
using (var reader = ExcelReaderFactory.CreateReader(stream))
{
do
{
while (reader.Read())
{
if (i != 0)
{
var userId = reader.GetString(0);
var specialtyCode = reader.GetString(1);
var userProfileElement1_WorkExp = reader.GetValue(2);
var userProfileElement2_VolExp = reader.GetValue(3);
var userProfileElement3_ResExp = reader.GetValue(4);
var userProfileElement4_Pubs = reader.GetValue(5);
var userProfileElement5_AOA = reader.GetValue(6);
var userProfileElement6_Nspecialties = reader.GetValue(7);
var userProfileElement7_PercentApps = reader.GetValue(8);
//Create profile
UserProfileData profile = new UserProfileData();
profile.UserProfileElement1_WorkExp = Convert.ToInt32(userProfileElement1_WorkExp);
profile.UserProfileElement2_VolExp = Convert.ToInt32(userProfileElement2_VolExp);
profile.UserProfileElement3_ResExp = Convert.ToInt32(userProfileElement3_ResExp);
profile.UserProfileElement4_Pubs = Convert.ToInt32(userProfileElement4_Pubs);
profile.UserProfileElement5_Aoa = Convert.ToBoolean(userProfileElement5_AOA);
profile.UserProfileElement6_Nspecialties = Convert.ToInt32(userProfileElement6_Nspecialties);
profile.UserProfileElement7_PercentApps = Convert.ToInt32(userProfileElement7_PercentApps);
//Calculate for set of programs. Selects one row at a time from XLS. BulkInsert into DB
//THIS ONLY RUNS ONE LINE OF THE METHOD
var backgroundTask = Task.Run(() => CalculateSet(uow, profile, userId, specialtyCode ));
//THIS WORKS
//CalculateSet(uow, profile, userId, specialtyCode);
}
i++;
Debug.WriteLine("Bulkcreate complete " + i);
//only process xxx rows
if (i > 1)
{
break;
}
}
} while (reader.NextResult());
}
}
Debug.WriteLine("Should get here quickly and not wait until task is done");
}
private void CalculateSet(UnitOfWork uow, UserProfileData profile, string userId, string specialtyCode)
{
//I CAN HIT THIS BREAKPOINT!
//get specialtyId from code
var specialtyId = uow.RefMedicalSpecialtyRepository
.Find(x => x.Code == specialtyCode).FirstOrDefault().Id;
//NEVER GET TO THIS BREAKPOINT
//loop through all programs for speciality
var programsForSpecialty = uow.RefProgramDetailDataRepository
.Find(x => x.RefMedicalSpecialtyId == specialtyId);
//List for bulk insert
// List<UserProgram> userPrograms = new List<UserProgram>();
//Write a row for each program
foreach (RefProgramDetailData rpdd in programsForSpecialty.ToList())
{
//Get program info
var programProfile = LoadData.Load_RefProgramProfileData(rpdd.Code);
//Calculate results
var userProgram = _calculator.CalculateAll(programProfile, profile, specialtyId, userId);
//If the Program can not be found in program detail then skip insert
if (userProgram == null)
{
Debug.WriteLine("Program NULL");
}
else
{
//Write results to UserProgram
uow.UserProgramRepository.Create(userProgram);
//userPrograms.Add(userProgram);
Debug.WriteLine("Program " + programProfile.ProgramCode);
}
}
//bulk insert
// uow.UserProgramRepository.BulkCreate(userPrograms);
}
}
EDIT 1: I put this in my controller. But it is waiting for the Task to finish before it redirects. Why isn't it redirecting immediately ?
await Task.Run(() => _calculator.CalculateAllSet(_unitOfWork, userProfileData, msId, null));
return RedirectToAction("Index", "Home");
Can you try like that.
[Fact]
public async Task Calculation_Xls()
{
string currentDirectory = Directory.GetCurrentDirectory();
string filesDirectory = currentDirectory + "\\Files";
System.Text.Encoding.RegisterProvider(CodePagesEncodingProvider.Instance);
var builder = new ConfigurationBuilder().SetBasePath(Directory.GetCurrentDirectory()).AddJsonFile("appsettings.json");
Configuration = builder.Build();
var optionsBuilder = new DbContextOptionsBuilder<RetContext>();
optionsBuilder.UseSqlServer(Configuration.GetConnectionString("DefaultConnection"));
int i = 0; //outer loop
UnitOfWork uow = new UnitOfWork(new RetContext(optionsBuilder.Options));
using (var stream = System.IO.File.Open(filesDirectory + "\\t2UserProfileDataTwoUserPerSpecialty.xlsx",
FileMode.Open, FileAccess.Read))
{
using (var reader = ExcelReaderFactory.CreateReader(stream))
{
do
{
while (reader.Read())
{
if (i != 0)
{
var userId = reader.GetString(0);
var specialtyCode = reader.GetString(1);
var userProfileElement1_WorkExp = reader.GetValue(2);
var userProfileElement2_VolExp = reader.GetValue(3);
var userProfileElement3_ResExp = reader.GetValue(4);
var userProfileElement4_Pubs = reader.GetValue(5);
var userProfileElement5_AOA = reader.GetValue(6);
var userProfileElement6_Nspecialties = reader.GetValue(7);
var userProfileElement7_PercentApps = reader.GetValue(8);
//Create profile
UserProfileData profile = new UserProfileData();
profile.UserProfileElement1_WorkExp = Convert.ToInt32(userProfileElement1_WorkExp);
profile.UserProfileElement2_VolExp = Convert.ToInt32(userProfileElement2_VolExp);
profile.UserProfileElement3_ResExp = Convert.ToInt32(userProfileElement3_ResExp);
profile.UserProfileElement4_Pubs = Convert.ToInt32(userProfileElement4_Pubs);
profile.UserProfileElement5_Aoa = Convert.ToBoolean(userProfileElement5_AOA);
profile.UserProfileElement6_Nspecialties = Convert.ToInt32(userProfileElement6_Nspecialties);
profile.UserProfileElement7_PercentApps = Convert.ToInt32(userProfileElement7_PercentApps);
//Calculate for set of programs. Selects one row at a time from XLS. BulkInsert into DB
//THIS ONLY RUNS ONE LINE OF THE METHOD
var backgroundTask = await Task.Run(() => CalculateSet(uow, profile, userId, specialtyCode ));
//THIS WORKS
//CalculateSet(uow, profile, userId, specialtyCode);
}
i++;
Debug.WriteLine("Bulkcreate complete " + i);
//only process xxx rows
if (i > 1)
{
break;
}
}
} while (reader.NextResult());
}
}
Debug.WriteLine("Should get here quickly and not wait until task is done");
}
private void CalculateSet(UnitOfWork uow, UserProfileData profile, string userId, string specialtyCode)
{
//I CAN HIT THIS BREAKPOINT!
//get specialtyId from code
var specialtyId = uow.RefMedicalSpecialtyRepository
.Find(x => x.Code == specialtyCode).FirstOrDefault().Id;
//NEVER GET TO THIS BREAKPOINT
//loop through all programs for speciality
var programsForSpecialty = uow.RefProgramDetailDataRepository
.Find(x => x.RefMedicalSpecialtyId == specialtyId);
//List for bulk insert
// List<UserProgram> userPrograms = new List<UserProgram>();
//Write a row for each program
foreach (RefProgramDetailData rpdd in programsForSpecialty.ToList())
{
//Get program info
var programProfile = LoadData.Load_RefProgramProfileData(rpdd.Code);
//Calculate results
var userProgram = _calculator.CalculateAll(programProfile, profile, specialtyId, userId);
//If the Program can not be found in program detail then skip insert
if (userProgram == null)
{
Debug.WriteLine("Program NULL");
}
else
{
//Write results to UserProgram
uow.UserProgramRepository.Create(userProgram);
//userPrograms.Add(userProgram);
Debug.WriteLine("Program " + programProfile.ProgramCode);
}
}
//bulk insert
// uow.UserProgramRepository.BulkCreate(userPrograms);
}
}
I have to store multiple xml files data into a single list of IEnumerable<SyncEntity>, but I don't get exact result as expected. Below is my sample code.
public IEnumerable<SyncEntity> GetUpdatedItemsOfType(DateTime? fromDate, string entityName, List<string> fieldsToRetrieve)
{
ConnLogger.WriteInfo("Dooors SyncConnector", "Run DOORS DXL for List of Getupdateditems of type ");
try
{
var dxlInput = $"{TmpRootFolder};{entityName};{fromDate.ToString()};{string.Join(",", fieldsToRetrieve)};{fieldsToRetrieve.Count.ToString()}";
string dxlPath = GetDxl("GetUpdatedItemsOfType.dxl");
//ActivateAsync(() =>
//{
// DoorsHandle.result = dxlInput;
// DoorsHandle.runFile(dxlPath);
//});
_doorsHandle.result = dxlInput;
_doorsHandle.runFile(dxlPath);
return GetUpdatedItemsOfTypePagination(TmpRootFolder);
}
catch (Exception ex)
{
ConnLogger.WriteException("Doors SyncConnector", ex, "Failed to get list of updateditems of type");
throw;
}
}
The above GetUpdatedItemsOfType(DateTime? fromDate, string entityName, List<string> fieldsToRetrieve) method is where I start one process of doors.
private IEnumerable<SyncEntity> GetUpdatedItemsOfTypePagination(string folderPath)
{
int currentPage = 1;
string finishFilePath = Path.Combine(folderPath, "GetUpdateItemsOfType_Finish.xml");
while (true)
{
string xmlFileFullPath = Path.Combine(folderPath, $"GetUpdateItemsOfType{currentPage}.xml");
bool pageReadCompleted = false;
for (int i = 0; i < 1000; i++) //wait max time of 1,000*0.1 = 100 seconds
{
if (!File.Exists(xmlFileFullPath))
{
if (File.Exists(finishFilePath))
{
yield break;
}
Thread.Sleep(TimeSpan.FromSeconds(0.1));
continue;
}
List<SyncEntity> pageItems = GetUpdatedItemsPage(xmlFileFullPath);
pageReadCompleted = true;
foreach (var syncEntity in pageItems)
{
yield return syncEntity;
}
break;
}
if (!pageReadCompleted)
{
throw new ApplicationException("Timeout reached for GetUpdatedItems method...");
}
currentPage++;
}
}
The above GetUpdatedItemsOfTypePagination(string folderPath) method checks the xml files in given folder.
private List<SyncEntity> GetUpdatedItemsPage(string xmlFilePath)
{
List<FileAttachment> fileAttachment=new List<FileAttachment>();
var xmlData = GenericSerializer.XmlDeSerialize<UpdatedItemsResult>(File.ReadAllText(xmlFilePath));
return xmlData.Items.Select(field => new SyncEntity
{
Name = field.ObjectName,
Id = field.Id,
Modified = Convert.ToDateTime(field.LastModifiedOn),
Fields = field.Attributes.Select(filed => new EntityField
{
Name = filed.Name,
Type = MetadataManager.FromDoorsDataType(filed.Type),
Value = fileAttachment
}).ToList()
}).ToList();
//copy here relevant code from the Execute method
}
The above GetUpdatedItemsPage(string xmlFilePath) method is the relevant code to deserialize the xml data.
The exception is happen most of the times but there are times it's working fine.
The exception is always the same
The remote server returned an error: (500) Internal Server Error.
Also the stacktrace is the same:
at System.Net.WebClient.DownloadDataInternal(Uri address, WebRequest& request)
at System.Net.WebClient.DownloadData(Uri address)
at System.Net.WebClient.DownloadData(String address)
at SatelliteImages.ExtractImages.ExtractDateAndTime(String baseAddress) in D:\C-Sharp\SatelliteImages\SatelliteImages\SatelliteImages\ExtractImages.cs:line 120
Line 120 is:
var temp = wc.DownloadData("/en");
The method:
public void ExtractDateAndTime(string baseAddress)
{
try
{
var wc = new WebClient();
wc.BaseAddress = baseAddress;
HtmlDocument doc = new HtmlDocument();
var temp = wc.DownloadData("/en");
doc.Load(new MemoryStream(temp));
var secTokenScript = doc.DocumentNode.Descendants()
.Where(e =>
String.Compare(e.Name, "script", true) == 0 &&
String.Compare(e.ParentNode.Name, "div", true) == 0 &&
e.InnerText.Length > 0 &&
e.InnerText.Trim().StartsWith("var region")
).FirstOrDefault().InnerText;
var securityToken = secTokenScript;
securityToken = securityToken.Substring(0, securityToken.IndexOf("arrayImageTimes.push"));
securityToken = secTokenScript.Substring(securityToken.Length).Replace("arrayImageTimes.push('", "").Replace("')", "");
var dates = securityToken.Trim().Split(new string[] { ";" }, StringSplitOptions.RemoveEmptyEntries);
var scriptDates = dates.Select(x => new ScriptDate { DateString = x });
foreach (var date in scriptDates)
{
DatesAndTimes.Add(date.DateString);
}
}
catch(WebException wex)
{
if (wex.Response != null)
{
using (var errorResponse = (HttpWebResponse)wex.Response)
{
using (var reader = new StreamReader(errorResponse.GetResponseStream()))
{
string error = reader.ReadToEnd();
}
}
}
countriescodes = new List<string>();
countriesnames = new List<string>();
DatesAndTimes = new List<string>();
imagesUrls = new List<string>();
this.Init();
}
}
When I use a breakpoint on the line:
string error = reader.ReadToEnd();
I see html content and in the content I see the text:
error occurred while processing your request. Return to the homepage of Sat24.com
What I want to do is somehow when the exception happen to start over again and try the download try the method ExtractDateAndTime.
I think using a timer some how and count back showing the user something like 30 seconds and try again. Will 30 seconds try will be consider as spam/flooding in the server site ?
This is the full class code but the exception is on this method ExtractDateAndTime.
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.IO;
using System.Net;
using System.Xml;
using HtmlAgilityPack;
using System.ComponentModel;
namespace SatelliteImages
{
class ExtractImages
{
static WebClient client;
static string htmltoextract;
public static List<string> countriescodes = new List<string>();
public static List<string> countriesnames = new List<string>();
public static List<string> DatesAndTimes = new List<string>();
public static List<string> imagesUrls = new List<string>();
static string firstUrlPart = "http://www.sat24.com/image2.ashx?region=";
static string secondUrlPart = "&time=";
static string thirdUrlPart = "&ir=";
public class ProgressEventArgs : EventArgs
{
public int Percentage { get; set; }
public string StateText { get; set; }
}
public event EventHandler<ProgressEventArgs> ProgressChanged;
public void Init()
{
object obj = null;
int index = 0;
ExtractCountires();
foreach (string cc in countriescodes)
{
// raise event here
ProgressChanged?.Invoke(obj,new ProgressEventArgs{ Percentage = 100 * index / countriescodes.Count, StateText = cc });
ExtractDateAndTime("http://www.sat24.com/image2.ashx?region=" + cc);
index +=1;
}
ImagesLinks();
}
public void ExtractCountires()
{
try
{
htmltoextract = "http://sat24.com/en/?ir=true";//"http://sat24.com/en/";// + regions;
client = new WebClient();
client.DownloadFile(htmltoextract, #"c:\temp\sat24.html");
client.Dispose();
string tag1 = "<li><a href=\"/en/";
string tag2 = "</a></li>";
string s = System.IO.File.ReadAllText(#"c:\temp\sat24.html");
s = s.Substring(s.IndexOf(tag1));
s = s.Substring(0, s.LastIndexOf(tag2) + tag2.ToCharArray().Length);
s = s.Replace("\r", "").Replace("\n", "").Replace(" ", "");
string[] parts = s.Split(new string[] { tag1, tag2 }, StringSplitOptions.RemoveEmptyEntries);
string tag3 = "<li><ahref=\"/en/";
for (int i = 0; i < parts.Length; i++)
{
if (i == 17)
{
//break;
}
string l = "";
if (parts[i].Contains(tag3))
l = parts[i].Replace(tag3, "");
string z1 = l.Substring(0, l.IndexOf('"'));
if (z1.Contains("</ul></li><liclass="))
{
z1 = z1.Replace("</ul></li><liclass=", "af");
}
countriescodes.Add(z1);
countriescodes.GroupBy(n => n).Any(c => c.Count() > 1);
string z2 = parts[i].Substring(parts[i].LastIndexOf('>') + 1);
if (z2.Contains("&"))
{
z2 = z2.Replace("&", " & ");
}
countriesnames.Add(z2);
countriesnames.GroupBy(n => n).Any(c => c.Count() > 1);
}
}
catch (Exception e)
{
if (countriescodes.Count == 0)
{
countriescodes = new List<string>();
countriesnames = new List<string>();
DatesAndTimes = new List<string>();
imagesUrls = new List<string>();
Init();
}
}
}
public void ExtractDateAndTime(string baseAddress)
{
try
{
var wc = new WebClient();
wc.BaseAddress = baseAddress;
HtmlDocument doc = new HtmlDocument();
var temp = wc.DownloadData("/en");
doc.Load(new MemoryStream(temp));
var secTokenScript = doc.DocumentNode.Descendants()
.Where(e =>
String.Compare(e.Name, "script", true) == 0 &&
String.Compare(e.ParentNode.Name, "div", true) == 0 &&
e.InnerText.Length > 0 &&
e.InnerText.Trim().StartsWith("var region")
).FirstOrDefault().InnerText;
var securityToken = secTokenScript;
securityToken = securityToken.Substring(0, securityToken.IndexOf("arrayImageTimes.push"));
securityToken = secTokenScript.Substring(securityToken.Length).Replace("arrayImageTimes.push('", "").Replace("')", "");
var dates = securityToken.Trim().Split(new string[] { ";" }, StringSplitOptions.RemoveEmptyEntries);
var scriptDates = dates.Select(x => new ScriptDate { DateString = x });
foreach (var date in scriptDates)
{
DatesAndTimes.Add(date.DateString);
}
}
catch(WebException wex)
{
if (wex.Response != null)
{
using (var errorResponse = (HttpWebResponse)wex.Response)
{
using (var reader = new StreamReader(errorResponse.GetResponseStream()))
{
string error = reader.ReadToEnd();
}
}
}
countriescodes = new List<string>();
countriesnames = new List<string>();
DatesAndTimes = new List<string>();
imagesUrls = new List<string>();
this.Init();
}
}
public class ScriptDate
{
public string DateString { get; set; }
public int Year
{
get
{
return Convert.ToInt32(this.DateString.Substring(0, 4));
}
}
public int Month
{
get
{
return Convert.ToInt32(this.DateString.Substring(4, 2));
}
}
public int Day
{
get
{
return Convert.ToInt32(this.DateString.Substring(6, 2));
}
}
public int Hours
{
get
{
return Convert.ToInt32(this.DateString.Substring(8, 2));
}
}
public int Minutes
{
get
{
return Convert.ToInt32(this.DateString.Substring(10, 2));
}
}
}
public void ImagesLinks()
{
int cnt = 0;
foreach (string countryCode in countriescodes)
{
cnt++;
for (; cnt < DatesAndTimes.Count(); cnt++)
{
string imageUrl = firstUrlPart + countryCode + secondUrlPart + DatesAndTimes[cnt] + thirdUrlPart + "true";
imagesUrls.Add(imageUrl);
if (cnt % 10 == 0) break;
}
}
}
}
}
What i want is in case of the exception happen to start over clean over again the whole class operation.
In Form1 i start the class operation first time once:
In top:
ExtractImages ei = new ExtractImages();
Then in constructor:
ei.Init();
The problem is this exception that sometimes happen.
You could go with something like Polly or the Transient Fault Handling Application Block to apply a retry strategy to your code.
Both packages provide multiple out-of-the-box components for various scenarios and you can always develop you own. Some of the included retry policies:
Incremental
Fixed interval
Exponential back-off
Retry
Retry for ever
Retry and wait
Wait and retry for ever
...
Parallel.ForEach keeps on running and my program does not end. I am unable to trace where it goes after the first iteration. My guess is that gets a deadlock and keeps on doing context switching.
private void ReadInputFile()
{
var collection = new ConcurrentBag<PropertyRecord>();
var lines = System.IO.File.ReadLines(InputFileName);
int i = 0;
int RecordsCount = lines.Count();
Parallel.ForEach(lines, line =>
{
if (string.IsNullOrWhiteSpace(line))
{
return;
}
var tokens = line.Split(',');
var postalCode = tokens[0];
var country = tokens.Length > 1 ? tokens[1] : "england";
SetLabelNotifyTwoText(
string.Format(
"Reading PostCode {0} out of {1}"
i,
lines.Length));
var tempRecord = GetAllAddesses(postalCode, country);
if (tempRecord != null)
{
foreach (PropertyRecord r in tempRecord)
{
collection.Add(r);
}
}
});
}
private List<PropertyRecord> GetAllAddesses(
string postalCode,
string country = "england")
{
SetLabelNotifyText("");
progressBar1.Value = 0;
progressBar1.Update();
var records = new List<PropertyRecord>();
using (WebClient w = new WebClient())
{
var url = CreateUrl(postalCode, country);
var document = w.DownloadString(url);
var pagesCount = GetPagesCount(document);
if (pagesCount == null)
{
return null;
}
for (int i = 0; i < pagesCount; i++)
{
SetLabelNotifyText(
string.Format(
"Reading Page {0} out of {1}",
i,
pagesCount - 1));
url = CreateUrl(postalcode,country, i);
document = w.DownloadString(url);
var collection = Regex.Matches(
document,
"<div class=\"soldDetails\">(.|\\n|\\r)*?class=" +
"\"soldAddress\".*?>(?<address>.*?)(</a>|</div>)" +
"(.|\\n|\\r)*?class=\\\"noBed\\\">(?<noBed>.*?)" +
"</td>|</tbody>");
foreach (var match in collection)
{
var r = new PropertyRecord();
var bedroomCount = match.Groups["noBed"].Value;
if(!string.IsNullOrEmpty(bedroomCount))
{
r.BedroomCount = bedroomCount;
}
else
{
r.BedroomCount = "-1";
}
r.address = match.Groups["address"].Value;
var line = string.Format(
"\"{0}\",{1}",
r.address
r.BedroomCount);
OutputLines.Add(line);
Records.Add(r);
}
}
}
return Records;
}
It runs fine without Parallel.ForEach, but using Parallel.ForEach is in requirements.
I have debugged it and after returning from GetAllAdresses-method first time, Step Next button halts and it just keep on debugging in the background. It doesn't come back on any bookmark I have placed.
As you said in comments, your SetLabelNotifyText and SetLabelNotifyTwoText methods calls Control.Invoke.
For Control.Invoke to work, Main thread has to be free, but in your case you seem to block the main thread by invoking Parallel.ForEach in it.
Here is a minimal reproduction:
private void button1_Click(object sender, EventArgs e)
{
Parallel.ForEach(Enumerable.Range(1, 100), (i) =>
{
Thread.Sleep(10);//Simulate some work
this.Invoke(new Action(() => SetText(i)));
});
}
private void SetText(int i)
{
textBox1.Text = i.ToString();
}
Main thread waits for Parallel.ForEach and worker threads waits for Main thread, and thus results in deadlock.
How to fix: Don't use Invoke simply use BeginInvoke or don't block the MainThread.
If this isn't the case post sscce, that will be helpful for us
Change your code like this, to use async and await. This is the modern alternative to using BeginInvoke and other asynchronous code models.
private async Task ReadInputFile()
{
var collection = new ConcurrentBag<PropertyRecord>();
var lines = System.IO.File.ReadLines(InputFileName);
int i = 0;
int RecordsCount = lines.Count();
Parallel.ForEach(lines, line =>
{
if (string.IsNullOrWhiteSpace(line))
{
return;
}
var tokens = line.Split(',');
var postalCode = tokens[0];
var country = tokens.Length > 1 ? tokens[1] : "england";
SetLabelNotifyTwoText(
string.Format(
"Reading PostCode {0} out of {1}"
i,
lines.Length));
var tempRecord = await GetAllAddesses(postalCode, country);
if (tempRecord != null)
{
foreach (PropertyRecord r in tempRecord)
{
collection.Add(r);
}
}
});
}
private async Task<List<PropertyRecord>> GetAllAddesses(
string postalCode,
string country = "england")
{
SetLabelNotifyText("");
progressBar1.Value = 0;
progressBar1.Update();
var records = new List<PropertyRecord>();
using (WebClient w = new WebClient())
{
var url = CreateUrl(postalCode, country);
var document = await w.DownloadStringTaskAsync(url);
var pagesCount = GetPagesCount(document);
if (pagesCount == null)
{
return null;
}
for (int i = 0; i < pagesCount; i++)
{
SetLabelNotifyText(
string.Format(
"Reading Page {0} out of {1}",
i,
pagesCount - 1));
url = CreateUrl(postalcode,country, i);
document = await w.DownloadStringTaskAsync(url);
var collection = Regex.Matches(
document,
"<div class=\"soldDetails\">(.|\\n|\\r)*?class=" +
"\"soldAddress\".*?>(?<address>.*?)(</a>|</div>)" +
"(.|\\n|\\r)*?class=\\\"noBed\\\">(?<noBed>.*?)" +
"</td>|</tbody>");
foreach (var match in collection)
{
var r = new PropertyRecord();
var bedroomCount = match.Groups["noBed"].Value;
if(!string.IsNullOrEmpty(bedroomCount))
{
r.BedroomCount = bedroomCount;
}
else
{
r.BedroomCount = "-1";
}
r.address = match.Groups["address"].Value;
var line = string.Format(
"\"{0}\",{1}",
r.address
r.BedroomCount);
OutputLines.Add(line);
Records.Add(r);
}
}
}
return Records;
}
Then call it like this
ReadInputFile.Wait();
or, even better, is the caller is async,
await ReadInputFile();