Develop Reference

Aspnet core Parallels operation DbContext problem

I've a problem when I use Pararrel function in aspnetcore, in particular when in the cycle i try to save something in database. I get my data from externarl api and deseserialize it in my class.
This is the Parallel code.
Root players = JsonConvert.DeserializeObject<Root>(responseStream);
var bulkhead = Policy.BulkheadAsync(10, Int32.MaxValue);
var tasks = new List<Task>();
foreach (var player in players.players)
{
var t = bulkhead.ExecuteAsync(async () =>
{
int wyId = Convert.ToInt32(player.wyId);
HttpRequestMessage secondRequest = createRequest("https://apirest.com/v2/players/" + wyId + "?details=currentTeam&imageDataURL=true");
var client2 = _clientFactory.CreateClient();
var response2 = await client2.SendAsync(secondRequest);
if (response2.IsSuccessStatusCode)
{
var responseStream2 = await response2.Content.ReadAsStringAsync();
dynamic playerFullDetails = JsonConvert.DeserializeObject<dynamic>(responseStream2);
int wyId2 = Convert.ToInt32(playerFullDetails.wyId);
int marketValue = 0;
HttpRequestMessage tirthRequest = createRequest("https://apirest.com/v2/players/" + wyId2 + "/marketvalue");
var client3 = _clientFactory.CreateClient();
var response3 = await client3.SendAsync(tirthRequest);
if (response3.IsSuccessStatusCode)
{
var responseStream3 = await response3.Content.ReadAsStringAsync();
dynamic marketValueResponse = JsonConvert.DeserializeObject<dynamic>(responseStream3);
if (marketValueResponse.marketValue != 0)
{
marketValue = Convert.ToInt32(marketValueResponse.marketValue);
}
}
DateTime birthday = Convert.ToDateTime(playerFullDetails.birthDate);
int age = DateTime.Now.Year - birthday.Year;
Player finalPlayer = new Player();
finalPlayer.PlayerId = wyId2;
finalPlayer.MarketValue = marketValue;
finalPlayer.Value = Convert.ToDouble(marketValue) / Convert.ToDouble(1000000);
finalPlayer.Firstname = playerFullDetails.firstName;
finalPlayer.Lastname = playerFullDetails.lastName;
finalPlayer.Name = playerFullDetails.shortName;
finalPlayer.Position = playerFullDetails.role.name;
finalPlayer.Height = playerFullDetails.height;
finalPlayer.Foot = playerFullDetails.foot;
finalPlayer.IsLocked = false;
finalPlayer.Team = playerFullDetails.currentTeam != null ? playerFullDetails.currentTeam.name : "";
finalPlayer.TeamId = playerFullDetails.currentTeam != null ? playerFullDetails.currentTeam.wyId : 0;
finalPlayer.CompetitionId = 524;
finalPlayer.UpdatedDay = DateTime.Now;
finalPlayer.League = "Serie A";
finalPlayer.Age = age;
Player playerExist = await _context.Player.Where(x => x.PlayerId == wyId2).SingleOrDefaultAsync();
if (playerExist == null)
{
if (finalPlayer.TeamId != 0)
{
await _context.Player.AddAsync(finalPlayer);
await _context.SaveChangesAsync();
}
}
if (finalPlayer.TeamId != 0)
{
Team teamExist = await _context.Team.Where(x => x.TeamId == finalPlayer.TeamId).SingleOrDefaultAsync();
if (teamExist == null)
{
Team team = new Team();
team.TeamId = finalPlayer.TeamId;
team.TeamName = finalPlayer.Team;
await _context.Team.AddAsync(team);
await _context.SaveChangesAsync();
}
}
}
});
tasks.Add(t);
}
await Task.WhenAll(tasks);
The function isert 50/60 (in total would be 500) element in db and finally i receive this error
A second operation was started on this context before a previous operation completed. This is usually caused by different threads concurrently using the same instance of DbContext. For more information on how to avoid threading issues with DbContext, see https://go.microsoft.com/fwlink/?linkid=2097913.
Thanks

It's best practice to use 1 dbcontext per unit of work, and the dbcontext is not thread safe
So either create a new dbcontext per thread or lock access with someting like a semaphore otherwise you will always get unstable code

How to inject a variable into every class or method in c#

I have the following code.
[HttpGet]
public async Task<List<TenantManagementWebApi.Entities.SiteCollection>> Get()
{
var tenant = await TenantHelper.GetActiveTenant();
var siteCollectionStore = CosmosStoreFactory.CreateForEntity<TenantManagementWebApi.Entities.SiteCollection>();
await siteCollectionStore.RemoveAsync(x => x.Title != string.Empty); // Removes all the entities that match the criteria
string domainUrl = tenant.TestSiteCollectionUrl;
string tenantName = domainUrl.Split('.')[0];
string tenantAdminUrl = tenantName + "-admin.sharepoint.com";
KeyVaultHelper keyVaultHelper = new KeyVaultHelper();
await keyVaultHelper.OnGetAsync(tenant.SecretIdentifier);
using (var context = new OfficeDevPnP.Core.AuthenticationManager().GetSharePointOnlineAuthenticatedContextTenant(tenantAdminUrl, tenant.Email, keyVaultHelper.SecretValue))
{
Tenant tenantOnline = new Tenant(context);
SPOSitePropertiesEnumerable siteProps = tenantOnline.GetSitePropertiesFromSharePoint("0", true);
context.Load(siteProps);
context.ExecuteQuery();
List<TenantManagementWebApi.Entities.SiteCollection> sites = new List<TenantManagementWebApi.Entities.SiteCollection>();
foreach (var site in siteProps)
{
if(site.Template.Contains("SITEPAGEPUBLISHING#0") || site.Template.Contains("GROUP#0"))
{
string strTemplate= default(string);
if(site.Template.Contains("SITEPAGEPUBLISHING#0"))
{
strTemplate = "CommunicationSite";
};
if (site.Template.Contains("GROUP#0"))
{
strTemplate = "Modern Team Site";
};
try
{
Guid id = Guid.NewGuid();
Entities.SiteCollection sc = new Entities.SiteCollection()
{
Id = id.ToString(),
Owner = site.Owner,
Template = strTemplate,
Title = site.Title,
Active = false,
Url = site.Url
};
var added = await siteCollectionStore.AddAsync(sc);
sites.Add(sc);
}
catch (System.Exception ex)
{
throw ex;
}
}
}
return sites;
};
}
However the following lines, I am repeating them on every method:
var tenant = await TenantHelper.GetActiveTenant();
var siteCollectionStore = CosmosStoreFactory.CreateForEntity<TenantManagementWebApi.Entities.SiteCollection>();
await siteCollectionStore.RemoveAsync(x => x.Title != string.Empty); // Removes all the entities that match the criteria
string domainUrl = tenant.TestSiteCollectionUrl;
string tenantName = domainUrl.Split('.')[0];
string tenantAdminUrl = tenantName + "-admin.sharepoint.com";
KeyVaultHelper keyVaultHelper = new KeyVaultHelper();
await keyVaultHelper.OnGetAsync(tenant.SecretIdentifier);
I will have lots of API controllers on my project
Is there an easy way (not refactor as a method), to make my code cleaner and inject the variables I need without copying and pasting every single time?

Async function in Converting object to another object in c#

I would like to implement an async function in converting the object to another object then in saving to the database.
public List<Order> GetOrdersFromTradeGeckoCount()
{
string orderLimit = base.StorePlugin.Store.OrderLimit.HasValue ? base.StorePlugin.Store.OrderLimit.Value.ToString() : "250";
string filters = string.Format("?status=finalized&limit={0}", orderLimit);
HttpResponseMessage response = _requestHelper.GetHttpResponse("orders" + filters);
var tgOrders = GetOrdersResponse(response);
//Async Convert and Save Order
ConvertToORouterOrdersAsync(tgOrders);
return ConvertToORouterOrdersCount(tgOrders);
}
I would like this method ConvertToORouterOrdersAsync(tgOrders); will run in the background and will return the Count of Orders from this ConvertToORouterOrdersCount(tgOrders) before the conversion is done.
Please help me to change the implementation to asynchronous.
public async void ConvertToORouterOrdersAsync(List<TGOrder> tgOrders)
{
var orderMgr = new OrderDAC();
var orders = new List<Order>(tgOrders.Count());
foreach (TGOrder tgOrder in tgOrders)
{
try
{
var order = new Order();
var orderId = TryConvertInt64(CleanUpOrderId(tgOrder.order_number));
if (orderId == null) continue;
var tempOrderId = string.Format("{0}{1}", base.StoreId, orderId.Value);
order.OrderId = TryConvertInt64(tempOrderId).Value;
order.StoreOrderId = tgOrder.id.ToString();
order.WarehouseOrderId = tgOrder.order_number;
var orderFromDb = orderMgr.GetOrder(order.OrderId, base.StoreId);
if (orderFromDb != null) continue; // make sure we only import new order(i.e. doesn't exists in database)
// shipping address
var tgShippingAddress = GetAddress(tgOrder.shipping_address_id);
if (tgShippingAddress == null) continue;
order.ShipFirstName = tgShippingAddress.first_name;
order.ShipLastName = tgShippingAddress.last_name;
order.ShipCompanyName = tgShippingAddress.company_name;
order.ShipAddress1 = tgShippingAddress.address1;
order.ShipAddress2 = tgShippingAddress.address2;
order.ShipCity = tgShippingAddress.suburb;
order.ShipState = tgShippingAddress.state;
order.ShipPostalCode = tgShippingAddress.zip_code;
order.ShipCountry = tgShippingAddress.country;
order.ShipPhoneNumber = tgShippingAddress.phone_number;
order.CustomerEmail = tgOrder.email;
// billing address
var tgBillingAddress = GetAddress(tgOrder.billing_address_id);
if (tgBillingAddress == null) continue;
// line items
var lineItems = GetOrderLineItems(tgOrder.id);
foreach (TGOrderLineItem lineItem in lineItems)
{
var ol = new OrderLine();
if (lineItem.variant_id.HasValue)
{
var variant = GetVariant(lineItem.variant_id.Value);
if (variant == null) continue;
ol.ProductName = variant.product_name;
ol.SKU = variant.sku;
ol.ThreePLSKU = ol.SKU;
ol.Qty = Convert.ToInt16(TryGetDecimal(lineItem.quantity));
ol.OrderId = order.OrderId;
ol.Price = TryGetDecimal(lineItem.price);
ol.SubTotal = (ol.Qty * ol.Price);
ol.StoreOrderLineId = Convert.ToString(lineItem.id);
order.OrderLines.Add(ol);
}
}
var validator = new Validator(base.Task);
if (validator.IsValidOrder(order))
{
orderMgr.Add(order);
}
}
catch (Exception ex)
{
AppendError(ex.Message);
}
}
}

To really have a advantage of the async and await the underlying calls should have async versions, for example the database calls should be async, don't know if you use EF or just plain SqlCommand but both have async versions of their calls.
Other calls that can be async is HTTP calls.
I have edited youre code with the assumption you are able to convert the underlying code to async.
public async Task<List<Order> GetOrdersFromTradeGeckoCount()
{
string orderLimit = base.StorePlugin.Store.OrderLimit.HasValue ? base.StorePlugin.Store.OrderLimit.Value.ToString() : "250";
string filters = string.Format("?status=finalized&limit={0}", orderLimit);
HttpResponseMessage response = await _requestHelper.GetHttpResponseAsync("orders" + filters).ConfigureAwait(false);
var tgOrders = GetOrdersResponse(response);
//Async Convert and Save Order
await ConvertToORouterOrdersAsync(tgOrders).ConfigureAwait(false);
return await ConvertToORouterOrdersCountAsync(tgOrders).ConfigureAwait(false);
}
public async Task ConvertToORouterOrdersAsync(List<TGOrder> tgOrders)
{
var orderMgr = new OrderDAC();
var orders = new List<Order>(tgOrders.Count());
foreach (TGOrder tgOrder in tgOrders)
{
try
{
var order = new Order();
var orderId = TryConvertInt64(CleanUpOrderId(tgOrder.order_number));
if (orderId == null) continue;
var tempOrderId = string.Format("{0}{1}", base.StoreId, orderId.Value);
order.OrderId = TryConvertInt64(tempOrderId).Value;
order.StoreOrderId = tgOrder.id.ToString();
order.WarehouseOrderId = tgOrder.order_number;
var orderFromDb = await orderMgr.GetOrderAsync(order.OrderId, base.StoreId).ConfigureAwait(false);
if (orderFromDb != null) continue; // make sure we only import new order(i.e. doesn't exists in database)
// shipping address
var tgShippingAddress = GetAddress(tgOrder.shipping_address_id);
if (tgShippingAddress == null) continue;
order.ShipFirstName = tgShippingAddress.first_name;
order.ShipLastName = tgShippingAddress.last_name;
order.ShipCompanyName = tgShippingAddress.company_name;
order.ShipAddress1 = tgShippingAddress.address1;
order.ShipAddress2 = tgShippingAddress.address2;
order.ShipCity = tgShippingAddress.suburb;
order.ShipState = tgShippingAddress.state;
order.ShipPostalCode = tgShippingAddress.zip_code;
order.ShipCountry = tgShippingAddress.country;
order.ShipPhoneNumber = tgShippingAddress.phone_number;
order.CustomerEmail = tgOrder.email;
// billing address
var tgBillingAddress = GetAddress(tgOrder.billing_address_id);
if (tgBillingAddress == null) continue;
// line items
var lineItems = GetOrderLineItems(tgOrder.id);
foreach (TGOrderLineItem lineItem in lineItems)
{
var ol = new OrderLine();
if (lineItem.variant_id.HasValue)
{
var variant = GetVariant(lineItem.variant_id.Value);
if (variant == null) continue;
ol.ProductName = variant.product_name;
ol.SKU = variant.sku;
ol.ThreePLSKU = ol.SKU;
ol.Qty = Convert.ToInt16(TryGetDecimal(lineItem.quantity));
ol.OrderId = order.OrderId;
ol.Price = TryGetDecimal(lineItem.price);
ol.SubTotal = (ol.Qty * ol.Price);
ol.StoreOrderLineId = Convert.ToString(lineItem.id);
order.OrderLines.Add(ol);
}
}
var validator = new Validator(base.Task);
if (validator.IsValidOrder(order))
{
await orderMgr.AddAsync(order).ConfigureAwait(false);
}
}
catch (Exception ex)
{
AppendError(ex.Message);
}
}
}
Or if you just want to run the code in the background you can also just wrap it in a Task.Run
public async Task<List<Order> GetOrdersFromTradeGeckoCount()
{
string orderLimit = base.StorePlugin.Store.OrderLimit.HasValue ? base.StorePlugin.Store.OrderLimit.Value.ToString() : "250";
string filters = string.Format("?status=finalized&limit={0}", orderLimit);
HttpResponseMessage response = _requestHelper.GetHttpResponse("orders" + filters);
var tgOrders = GetOrdersResponse(response);
//Async Convert and Save Order
await ConvertToORouterOrdersAsync(tgOrders).ConfigureAwait(false);
return ConvertToORouterOrdersCount(tgOrders);
}
public Task ConvertToORouterOrdersAsync(List<TGOrder> tgOrders)
{
return Task.Run(() =>
{
var orderMgr = new OrderDAC();
var orders = new List<Order>(tgOrders.Count());
foreach (TGOrder tgOrder in tgOrders)
{
try
{
var order = new Order();
var orderId = TryConvertInt64(CleanUpOrderId(tgOrder.order_number));
if (orderId == null) continue;
var tempOrderId = string.Format("{0}{1}", base.StoreId, orderId.Value);
order.OrderId = TryConvertInt64(tempOrderId).Value;
order.StoreOrderId = tgOrder.id.ToString();
order.WarehouseOrderId = tgOrder.order_number;
var orderFromDb = await orderMgr.GetOrder(order.OrderId, base.StoreId);
if (orderFromDb != null) continue; // make sure we only import new order(i.e. doesn't exists in database)
// shipping address
var tgShippingAddress = GetAddress(tgOrder.shipping_address_id);
if (tgShippingAddress == null) continue;
order.ShipFirstName = tgShippingAddress.first_name;
order.ShipLastName = tgShippingAddress.last_name;
order.ShipCompanyName = tgShippingAddress.company_name;
order.ShipAddress1 = tgShippingAddress.address1;
order.ShipAddress2 = tgShippingAddress.address2;
order.ShipCity = tgShippingAddress.suburb;
order.ShipState = tgShippingAddress.state;
order.ShipPostalCode = tgShippingAddress.zip_code;
order.ShipCountry = tgShippingAddress.country;
order.ShipPhoneNumber = tgShippingAddress.phone_number;
order.CustomerEmail = tgOrder.email;
// billing address
var tgBillingAddress = GetAddress(tgOrder.billing_address_id);
if (tgBillingAddress == null) continue;
// line items
var lineItems = GetOrderLineItems(tgOrder.id);
foreach (TGOrderLineItem lineItem in lineItems)
{
var ol = new OrderLine();
if (lineItem.variant_id.HasValue)
{
var variant = GetVariant(lineItem.variant_id.Value);
if (variant == null) continue;
ol.ProductName = variant.product_name;
ol.SKU = variant.sku;
ol.ThreePLSKU = ol.SKU;
ol.Qty = Convert.ToInt16(TryGetDecimal(lineItem.quantity));
ol.OrderId = order.OrderId;
ol.Price = TryGetDecimal(lineItem.price);
ol.SubTotal = (ol.Qty * ol.Price);
ol.StoreOrderLineId = Convert.ToString(lineItem.id);
order.OrderLines.Add(ol);
}
}
var validator = new Validator(base.Task);
if (validator.IsValidOrder(order))
{
orderMgr.Add(order);
}
}
catch (Exception ex)
{
AppendError(ex.Message);
}
}
});
}

Is my PLINQ code threadsafe?

I have the following code:
static void Main(string[] args)
{
DateTime currentDay = DateTime.Now;
List<Task> taskList = new List<Task>();
List<string> markets = new List<string>() { "amex", "nasdaq", "nyse", "global" };
Parallel.ForEach(markets, market =>
{
Downloads.startInitialMarketSymbolsDownload(market);
}
);
Console.WriteLine("All downloads finished!");
}
public static void startInitialMarketSymbolsDownload(string market)
{
try
{
object valueTypeLock = new object();
List<string> symbolList = new List<string>() { "GOOG", "YHOO", "AAP" }
var historicalGroups = symbolList.AsParallel().Select((x, i) => new { x, i })
.GroupBy(x => x.i / 100)
.Select(g => g.Select(x => x.x).ToArray());
historicalGroups.AsParallel().ForAll(g => {
lock (valueTypeLock) {
getHistoricalStockData(g, market);
}
});
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
Console.WriteLine(ex.StackTrace);
}
}
public static void getHistoricalStockData(string[] symbols, string market)
{
// download data for list of symbols and then upload to db tables
Uri uri;
string url, line;
decimal open = 0, high = 0, low = 0, close = 0, adjClose = 0;
DateTime date;
Int64 volume = 0;
string[] lineArray;
List<string> symbolError = new List<string>();
Dictionary<string, string> badNameError = new Dictionary<string, string>();
System.Net.ServicePointManager.DefaultConnectionLimit = 1000;
Parallel.ForEach(symbols, symbol =>
{
url = "http://ichart.finance.yahoo.com/table.csv?s=" + symbol + "&a=00&b=1&c=1900&d=" + (DateTime.Now.Month - 1) + "&e=" + DateTime.Now.Day + "&f=" + DateTime.Now.Year + "&g=d&ignore=.csv";
uri = new Uri(url);
using (ooplesfinanceEntities entity = new ooplesfinanceEntities())
using (WebClient client = new WebClient())
using (Stream stream = client.OpenRead(uri))
using (StreamReader reader = new StreamReader(stream))
{
entity.Database.Connection.Open();
while (reader.EndOfStream == false)
{
line = reader.ReadLine();
lineArray = line.Split(',');
// if it isn't the very first line
if (lineArray[0] != "Date")
{
switch (market)
{
case "amex":
DailyAmexData amexData = new DailyAmexData();
var amexQuery = from r in entity.DailyAmexDatas.AsParallel().AsEnumerable()
where r.Date == date
select new StockData { Close = r.AdjustedClose };
List<StockData> amexResult = amexQuery.AsParallel().ToList();
if (amexResult.AsParallel().Count() > 0) // **never hits this breakpoint line**
{
// add the row to the table if it isn't there
// now checks for stock splits and updates if necessary
if (amexResult.AsParallel().FirstOrDefault().Close != adjClose)
{
// this means there is a stock split so it needs to have the other adjusted close prices updated
amexResult.AsParallel().FirstOrDefault().Close = adjClose;
}
else
{
continue;
}
}
else
{
// set the data then add it
amexData.Symbol = symbol;
entity.DailyAmexDatas.Add(amexData);
}
break;
default:
break;
}
}
}
// now save everything
entity.SaveChanges();
Console.WriteLine(symbol + " added to the " + market + " database!");
}
}
);
}
Everything starts fine and I get no exceptions but I'm getting no results and the code gets stuck one the line that I marked and the memory keeps shooting up. I figured the problem was with the above code because maybe I was doing something wrong. I just don't know where to start as this is my first time dealing with plinq/parallel processing and the tutorials don't show anything this complex.

Multithreading issue ,Maybe a DeadLock using Foreach

Parallel.ForEach keeps on running and my program does not end. I am unable to trace where it goes after the first iteration. My guess is that gets a deadlock and keeps on doing context switching.
private void ReadInputFile()
{
var collection = new ConcurrentBag<PropertyRecord>();
var lines = System.IO.File.ReadLines(InputFileName);
int i = 0;
int RecordsCount = lines.Count();
Parallel.ForEach(lines, line =>
{
if (string.IsNullOrWhiteSpace(line))
{
return;
}
var tokens = line.Split(',');
var postalCode = tokens[0];
var country = tokens.Length > 1 ? tokens[1] : "england";
SetLabelNotifyTwoText(
string.Format(
"Reading PostCode {0} out of {1}"
i,
lines.Length));
var tempRecord = GetAllAddesses(postalCode, country);
if (tempRecord != null)
{
foreach (PropertyRecord r in tempRecord)
{
collection.Add(r);
}
}
});
}
private List<PropertyRecord> GetAllAddesses(
string postalCode,
string country = "england")
{
SetLabelNotifyText("");
progressBar1.Value = 0;
progressBar1.Update();
var records = new List<PropertyRecord>();
using (WebClient w = new WebClient())
{
var url = CreateUrl(postalCode, country);
var document = w.DownloadString(url);
var pagesCount = GetPagesCount(document);
if (pagesCount == null)
{
return null;
}
for (int i = 0; i < pagesCount; i++)
{
SetLabelNotifyText(
string.Format(
"Reading Page {0} out of {1}",
i,
pagesCount - 1));
url = CreateUrl(postalcode,country, i);
document = w.DownloadString(url);
var collection = Regex.Matches(
document,
"<div class=\"soldDetails\">(.|\\n|\\r)*?class=" +
"\"soldAddress\".*?>(?<address>.*?)(</a>|</div>)" +
"(.|\\n|\\r)*?class=\\\"noBed\\\">(?<noBed>.*?)" +
"</td>|</tbody>");
foreach (var match in collection)
{
var r = new PropertyRecord();
var bedroomCount = match.Groups["noBed"].Value;
if(!string.IsNullOrEmpty(bedroomCount))
{
r.BedroomCount = bedroomCount;
}
else
{
r.BedroomCount = "-1";
}
r.address = match.Groups["address"].Value;
var line = string.Format(
"\"{0}\",{1}",
r.address
r.BedroomCount);
OutputLines.Add(line);
Records.Add(r);
}
}
}
return Records;
}
It runs fine without Parallel.ForEach, but using Parallel.ForEach is in requirements.
I have debugged it and after returning from GetAllAdresses-method first time, Step Next button halts and it just keep on debugging in the background. It doesn't come back on any bookmark I have placed.

As you said in comments, your SetLabelNotifyText and SetLabelNotifyTwoText methods calls Control.Invoke.
For Control.Invoke to work, Main thread has to be free, but in your case you seem to block the main thread by invoking Parallel.ForEach in it.
Here is a minimal reproduction:
private void button1_Click(object sender, EventArgs e)
{
Parallel.ForEach(Enumerable.Range(1, 100), (i) =>
{
Thread.Sleep(10);//Simulate some work
this.Invoke(new Action(() => SetText(i)));
});
}
private void SetText(int i)
{
textBox1.Text = i.ToString();
}
Main thread waits for Parallel.ForEach and worker threads waits for Main thread, and thus results in deadlock.
How to fix: Don't use Invoke simply use BeginInvoke or don't block the MainThread.
If this isn't the case post sscce, that will be helpful for us

Change your code like this, to use async and await. This is the modern alternative to using BeginInvoke and other asynchronous code models.
private async Task ReadInputFile()
{
var collection = new ConcurrentBag<PropertyRecord>();
var lines = System.IO.File.ReadLines(InputFileName);
int i = 0;
int RecordsCount = lines.Count();
Parallel.ForEach(lines, line =>
{
if (string.IsNullOrWhiteSpace(line))
{
return;
}
var tokens = line.Split(',');
var postalCode = tokens[0];
var country = tokens.Length > 1 ? tokens[1] : "england";
SetLabelNotifyTwoText(
string.Format(
"Reading PostCode {0} out of {1}"
i,
lines.Length));
var tempRecord = await GetAllAddesses(postalCode, country);
if (tempRecord != null)
{
foreach (PropertyRecord r in tempRecord)
{
collection.Add(r);
}
}
});
}
private async Task<List<PropertyRecord>> GetAllAddesses(
string postalCode,
string country = "england")
{
SetLabelNotifyText("");
progressBar1.Value = 0;
progressBar1.Update();
var records = new List<PropertyRecord>();
using (WebClient w = new WebClient())
{
var url = CreateUrl(postalCode, country);
var document = await w.DownloadStringTaskAsync(url);
var pagesCount = GetPagesCount(document);
if (pagesCount == null)
{
return null;
}
for (int i = 0; i < pagesCount; i++)
{
SetLabelNotifyText(
string.Format(
"Reading Page {0} out of {1}",
i,
pagesCount - 1));
url = CreateUrl(postalcode,country, i);
document = await w.DownloadStringTaskAsync(url);
var collection = Regex.Matches(
document,
"<div class=\"soldDetails\">(.|\\n|\\r)*?class=" +
"\"soldAddress\".*?>(?<address>.*?)(</a>|</div>)" +
"(.|\\n|\\r)*?class=\\\"noBed\\\">(?<noBed>.*?)" +
"</td>|</tbody>");
foreach (var match in collection)
{
var r = new PropertyRecord();
var bedroomCount = match.Groups["noBed"].Value;
if(!string.IsNullOrEmpty(bedroomCount))
{
r.BedroomCount = bedroomCount;
}
else
{
r.BedroomCount = "-1";
}
r.address = match.Groups["address"].Value;
var line = string.Format(
"\"{0}\",{1}",
r.address
r.BedroomCount);
OutputLines.Add(line);
Records.Add(r);
}
}
}
return Records;
}
Then call it like this
ReadInputFile.Wait();
or, even better, is the caller is async,
await ReadInputFile();