Develop Reference

Dynamics Crm Bulk Update records in a transaction

Requirement
I have the requirement where I want to update few fields on account
and create contact for it reading data from an API.
The number of records to be updated is around 100,000 so I want to
use either ExecuteTransactionRequest or ExecuteMultipleRequest so that I can execute all in batches.
Since I want the contact record to be created for the account updated I used the ExecuteTransactionRequest.
Problem -
The problem is batch size.
I have added the condition if request batch size count equals 500 then execute all the requests. However my batch can include
Update request for account and
Create request for contact
So it may happen that the batch may not be exact 500 and it would skip the execute request. How can I do this and make sure that contact record is created for each Updated Account correctly.
Any help would be appreciated. Thanks in Advance
Below is my code --
var requests = new ExecuteTransactionRequest
{
Requests = new OrganizationRequestCollection(),
ReturnResponses = returnResponses
};
foreach (var customer in customerList)
{
string custNo = customer.GetAttributeValue<string>("customernumber");
// Gets customer details from another api
var custInfo = await CustomerService.Get(custNo);
// Update the values on customer table
Entity cust = new Entity("account");
cust.Id = customer.Id;
cust["companytypecode"] = custInfo.EntityTypeCode;
cust["companytypedescription"] = custInfo .EntityTypeDescription;
var roles = custInfo.Roles.Where(c => c.RoleStatus == "ACTIVE").ToArray();
//create contact for each account
foreach(var role in roles)
{
Entity contact = new Entity("contact");
contact["FirstName"] = role.RolePerson?.FirstName;
contact["MiddleName"] = role.RolePerson?.MiddleNames;
contact["LastName"] = role.RolePerson?.LastName;
contact["AccountId"] = new EntityReference("account", customer.Id);
CreateRequest createRequest = new CreateRequest { Target = contact };
requests.Requests.Add(createRequest);
}
UpdateRequest updateRequest = new UpdateRequest { Target = cust };
requests.Requests.Add(updateRequest);
if (requests.Requests.Count == 500) // Problem is the batch size will not execute per account since it also has create request of contact. How can i make sure that each request is executed correctly
{
service.Execute(requests);
requests.Requests.Clear();
}
}
// For the last remaining accounts
if (requests.Requests.Count > 0)
{
service.Execute(requests);
}

Thank you for helping out. I resolved this with below solution. Happy to be corrected.
EntityCollection requestsCollection = new EntityCollection();
foreach (var customer in customerList)
{
string custNo= customer.GetAttributeValue<string>("customernumber");
var custInfo = await businessService.Get(custNo);
Entity cust = new Entity("account");
cust.Id = customer.Id;
cust["companytypecode"] = custInfo.EntityTypeCode;
cust["companytypedescription"] = custInfo .EntityTypeDescription;
requestsCollection.Entities.Add(cust);
var roles = custInfo.Roles.Where(c => c.RoleStatus == "ACTIVE").ToArray();
foreach(var role in roles)
{
Entity contact = new Entity("contact");
contact["FirstName"] = role.RolePerson?.FirstName;
contact["MiddleName"] = role.RolePerson?.MiddleNames;
contact["LastName"] = role.RolePerson?.LastName;
contact["AccountId"] = new EntityReference("account", customer.Id);
requests.Entities.Add(contact);
}
if (requestsCollection.Entities.Count > 500)
{
ExecuteBulkUpdate(requestsCollection);
requestsCollection = new EntityCollection();
}
}
private void ExecuteBulkUpdate(EntityCollection requestsCollection)
{
var requests = new ExecuteTransactionRequest
{
Requests = new OrganizationRequestCollection(),
ReturnResponses = returnResponses
};
foreach (var request in requestsCollection.Entities)
{
if (request.Id != Guid.Empty)
{
UpdateRequest updateRequest = new UpdateRequest { Target = request };
requests.Requests.Add(updateRequest);
}
else
{
CreateRequest createRequest = new CreateRequest { Target = request };
requests.Requests.Add(createRequest);
}
}
try
{
var responseForCreateRecords = (ExecuteTransactionResponse)service.Execute(requests);
int i = 0;
// Display the results returned in the responses.
foreach (var responseItem in responseForCreateRecords.Responses)
{
if (responseItem != null)
log.LogInformation(responseItem.Results["id"].ToString());
i++;
}
requests.Requests.Clear();
}
catch (FaultException<OrganizationServiceFault> ex)
{
log.LogInformation("Request failed for the {0} and the reason being: {1}",
((ExecuteTransactionFault)(ex.Detail)).FaultedRequestIndex + 1, ex.Detail.Message);
throw;
}
}

Aspnet core Parallels operation DbContext problem

I've a problem when I use Pararrel function in aspnetcore, in particular when in the cycle i try to save something in database. I get my data from externarl api and deseserialize it in my class.
This is the Parallel code.
Root players = JsonConvert.DeserializeObject<Root>(responseStream);
var bulkhead = Policy.BulkheadAsync(10, Int32.MaxValue);
var tasks = new List<Task>();
foreach (var player in players.players)
{
var t = bulkhead.ExecuteAsync(async () =>
{
int wyId = Convert.ToInt32(player.wyId);
HttpRequestMessage secondRequest = createRequest("https://apirest.com/v2/players/" + wyId + "?details=currentTeam&imageDataURL=true");
var client2 = _clientFactory.CreateClient();
var response2 = await client2.SendAsync(secondRequest);
if (response2.IsSuccessStatusCode)
{
var responseStream2 = await response2.Content.ReadAsStringAsync();
dynamic playerFullDetails = JsonConvert.DeserializeObject<dynamic>(responseStream2);
int wyId2 = Convert.ToInt32(playerFullDetails.wyId);
int marketValue = 0;
HttpRequestMessage tirthRequest = createRequest("https://apirest.com/v2/players/" + wyId2 + "/marketvalue");
var client3 = _clientFactory.CreateClient();
var response3 = await client3.SendAsync(tirthRequest);
if (response3.IsSuccessStatusCode)
{
var responseStream3 = await response3.Content.ReadAsStringAsync();
dynamic marketValueResponse = JsonConvert.DeserializeObject<dynamic>(responseStream3);
if (marketValueResponse.marketValue != 0)
{
marketValue = Convert.ToInt32(marketValueResponse.marketValue);
}
}
DateTime birthday = Convert.ToDateTime(playerFullDetails.birthDate);
int age = DateTime.Now.Year - birthday.Year;
Player finalPlayer = new Player();
finalPlayer.PlayerId = wyId2;
finalPlayer.MarketValue = marketValue;
finalPlayer.Value = Convert.ToDouble(marketValue) / Convert.ToDouble(1000000);
finalPlayer.Firstname = playerFullDetails.firstName;
finalPlayer.Lastname = playerFullDetails.lastName;
finalPlayer.Name = playerFullDetails.shortName;
finalPlayer.Position = playerFullDetails.role.name;
finalPlayer.Height = playerFullDetails.height;
finalPlayer.Foot = playerFullDetails.foot;
finalPlayer.IsLocked = false;
finalPlayer.Team = playerFullDetails.currentTeam != null ? playerFullDetails.currentTeam.name : "";
finalPlayer.TeamId = playerFullDetails.currentTeam != null ? playerFullDetails.currentTeam.wyId : 0;
finalPlayer.CompetitionId = 524;
finalPlayer.UpdatedDay = DateTime.Now;
finalPlayer.League = "Serie A";
finalPlayer.Age = age;
Player playerExist = await _context.Player.Where(x => x.PlayerId == wyId2).SingleOrDefaultAsync();
if (playerExist == null)
{
if (finalPlayer.TeamId != 0)
{
await _context.Player.AddAsync(finalPlayer);
await _context.SaveChangesAsync();
}
}
if (finalPlayer.TeamId != 0)
{
Team teamExist = await _context.Team.Where(x => x.TeamId == finalPlayer.TeamId).SingleOrDefaultAsync();
if (teamExist == null)
{
Team team = new Team();
team.TeamId = finalPlayer.TeamId;
team.TeamName = finalPlayer.Team;
await _context.Team.AddAsync(team);
await _context.SaveChangesAsync();
}
}
}
});
tasks.Add(t);
}
await Task.WhenAll(tasks);
The function isert 50/60 (in total would be 500) element in db and finally i receive this error
A second operation was started on this context before a previous operation completed. This is usually caused by different threads concurrently using the same instance of DbContext. For more information on how to avoid threading issues with DbContext, see https://go.microsoft.com/fwlink/?linkid=2097913.
Thanks

It's best practice to use 1 dbcontext per unit of work, and the dbcontext is not thread safe
So either create a new dbcontext per thread or lock access with someting like a semaphore otherwise you will always get unstable code

Trying to run 100 parallel tasks on an iteration doesn't work

Well, I'm trying to run a task 100 times on each run (with paralellism) but I can't manage this to work.
I'm trying to bruteforce an API, for this the API allows me to concatenate as many IDS as possible (without exceeding the timeout)
// consts:
// idsNum = 1000
// maxTasks = 100
// We prepare the ids that we will process (in that case 100,000)
var ids = PrepareIds(i * idsNum * maxTasks, idsNum, maxTasks);
// This was my old approach (didn't work)
//var result = await Task.WhenAll(ids.AsParallel().Select(async x => (await client.GetItems(x)).ToArray()));
// This is my new approach (also this didn't worked...)
var items = new List<item[]>();
ids.AsParallel().Select(x => client.GetItems(x).GetAwaiter().GetResult()).ForAll(item =>
{
//Console.WriteLine("processed!");
items.Add(item.ToArray());
});
var result = items.ToArray();
As you can see I put Console.WriteLine("processed!"); statment, in order to check if anything worked... But I can't manage this to work.
Those are my other methods:
private static IEnumerable<ulong[]> PrepareIds(int startingId, int idsNum = 1000, int maxTasks = 100)
{
for (int i = 0; i < maxTasks; i++)
yield return Range((ulong)startingId, (ulong)(startingId + idsNum)).ToArray();
}
And...
public static async Task<IEnumerable<item>> GetItems(this HttpClient client, ulong[] ids, Action notFoundCallback = null)
{
var keys = PrepareDataInKeys(type, ids); // This prepares the data to be sent to the API server
var content = new FormUrlEncodedContent(keys);
content.Headers.ContentType =
new MediaTypeHeaderValue("application/x-www-form-urlencoded") { CharSet = "UTF-8" };
client.DefaultRequestHeaders.ExpectContinue = false;
// We create a post request
var response = await client.PostAsync(new Uri(FILE_URL), content);
string contents = null;
JObject jObject;
try
{
contents = await response.Content.ReadAsStringAsync();
jObject = JsonConvert.DeserializeObject<JObject>(contents);
}
catch (Exception ex)
{
Console.WriteLine(ex);
Console.WriteLine(contents);
return null;
}
// Then we read the items from the parsed JObject
JArray items;
try
{
items = jObject
.GetValue("...")
.ToObject<JObject>()
.GetValue("...").ToObject<JArray>();
}
catch (Exception ex)
{
Console.WriteLine(ex);
return null;
}
int notFoundItems = 0;
int nonxxxItems = 0;
int xxxItems = 0;
var all = items.BuildEnumerable(notFoundCallback, item =>
{
if (item.Result != 1)
++notFoundItems;
else if (item.id != 5)
++nonxxxItems;
else
++xxxItems;
});
CrawledItems += ids.Length;
NotFoundItems += notFoundItems;
NonXXXItems += nonxxxItems;
XXXItems += xxxItems;
return all;
}
private static IEnumerable<item> BuildEnumerable(this JArray items, Action notFoundCallback, Action<item> callback = null)
{
foreach (var item in items)
{
item _item;
try
{
_item = new item(item.ToObject<JObject>());
callback?.Invoke(_item);
}
catch (Exception ex)
{
if (notFoundCallback == null)
Console.WriteLine(ex, Color.Red);
else
notFoundCallback();
continue;
}
yield return _item;
}
}
So as you can see I create 100 parallel post requests using an HttpClient. But I can't manage it to work.
So the thing that I want to achieve is to retrieve as many items as possible because I need to crawl +2,000,000,000 items.
But any breakpoint is triggered, neither any caption is updated on Console (I'm using Konsole project in order to print values at a fixed position on console), so any advice can be given there?

ASP.NET Core Large Data Set Processing

I have an Asp.net core 2.1 application with a Postgresql database.
A large file (~100K lines) gets uploaded to the database and then each line is processed.
As each line is processed there are approximately 15 sub queries per line taking up about 10 seconds in total per line. I have tried optimizing the queries and loading certain datasets in memory to reduce database calls which has seen a slight improvement. I have also tried reducing the batch size but that has also had minimal reduction.
Are there any other ways I can speed this up? The queries are written in lambda.
The high level function:
public void CDR_Insert(List<InputTable> inputTable, IQueryable<Costing__Table> CostingTable, List<Patterns> PatternsTable, IQueryable<Country_Calling_Codes> CallingCodesTable, IQueryable<Country> CountryTable, string voicemail_number)
{
foreach (var row in inputTable)
{
//if (!string.IsNullOrEmpty(row.finalCalledPartyPattern))
//{
var rowexists = _context.CDR.Where(m => m.InputId == row.id).Count();
if (rowexists == 0)
{
if (ValidateIfOneNumberIsForCountry(row.callingPartyNumber, row.finalCalledPartyNumber, PatternsTable, CountryTable) == true)
{
var CallRingTime = TimeSpan.FromSeconds(Ring_Time(row.dateTimeOrigination, row.dateTimeConnect));
var Call_Direction = CallDirection(row.callingPartyNumber, row.finalCalledPartyNumber, row.finalCalledPartyPattern, voicemail_number);
var CallReason = "Destination Out Of Order";
if (row.dateTimeConnect != "0")
{
CallReason = Call_Reason(row.callingPartyNumber, row.originalCalledPartyNumber, row.finalCalledPartyNumber, voicemail_number);
}
var CallOutcome = Outcome(row.duration, row.dateTimeOrigination, row.dateTimeConnect, row.callingPartyNumber, row.originalCalledPartyNumber, row.finalCalledPartyNumber, voicemail_number);
var CallCategory = GetCategory(row.callingPartyNumber, row.finalCalledPartyNumber, PatternsTable, CallingCodesTable, CountryTable);
var CallCost = 0.00;
if (CallOutcome != "Not Connected" || CallOutcome != "No answer")
{
CallCost = Costing(row.callingPartyNumber, row.finalCalledPartyNumber, row.dateTimeConnect, row.dateTimeDisconnect, row.finalCalledPartyPattern, row.duration, Call_Direction, CostingTable, PatternsTable, voicemail_number, CallingCodesTable, CountryTable);
}
CDR cDR = new CDR()
{
Date_Time = UnixTimeStampToDateTimeConvertor2(row.dateTimeOrigination),
Call_Direction = Call_Direction,
Calling_Party = row.callingPartyNumber_uri,
Calling_Digits = row.callingPartyNumber,
Called_Party = row.finalCalledPartyNumber_uri,
Called_Digits = row.finalCalledPartyNumber,
Duration = TimeSpan.FromSeconds(Convert.ToDouble(row.duration)),
Cost = CallCost,
Reason = CallReason,
Outcome = CallOutcome,
Country_Code = "MUS",//_context.Country.Where(o => o.CountryCode == _CDRExtension.CountryCodeId(_CDRExtension.Call_Type_Id(row.callingPartyNumber))).Select(o => o.CountryCode).FirstOrDefault(),
Second_Ring_Time = CallRingTime,
Category = CallCategory,
CiscoInputId = row.id
};
_context.CDR.Add(cDR);
}
row.Processed = true;
row.DateAdded = DateTime.Now;
_context.Cisco_Input.Update(row);
// }
_context.SaveChanges();
}
}
}

Unable to find entity while under a transaction

I'm having an issue with this data layer method
public async Task<(bool HasFailed, IList<string> ErrorMessages)> ApproveBillCancelRequest(IEnumerable<string[]> billsToApprove,
string userId, string operationId)
{
var callerInfo = Shared.CommonAcross.Helper.GetCaller();
InfoScope($"{LayerName} -> {callerInfo.MethodName} -> Started", operationId, userId);
var errorMessages = new List<string>();
using (var context = new FocusConnection())
{
var transaction = context.Database.BeginTransaction();
try
{
foreach (var billToApprove in billsToApprove)
{
var targetBillNumber = billToApprove[0];
var targetPayModeId = int.Parse(billToApprove[1]);
var entityBill = context.BILL_INFO_CANCEL_REQUESTS
.SingleOrDefault(where =>
where.BILL_NUMBER == targetBillNumber &&
where.PAY_MODE_ID == targetPayModeId);
if (entityBill == null)
{
errorMessages.Add($"Bill #{billToApprove[0]}, payment #{billToApprove[1]} was not found for cancel approval");
continue;
}
entityBill.BILL_INFO.LAST_MODIFIED_BY = userId;
entityBill.BILL_INFO.STAMP_DATE = DateTime.Now;
entityBill.BILL_INFO.INPUT_STATUS = 0;
var cancelledBill = new BILL_INFO_CANCELED
{
BILL_NUMBER = entityBill.BILL_NUMBER,
PAY_MODE_ID = entityBill.PAY_MODE_ID,
CASHIER_ID = entityBill.CASHIER_ID,
CANCELED_DATE = entityBill.CANCEL_REQUEST_DATE,
CANCEL_REQUESTED_BY = entityBill.CANCEL_REQUESTED_BY,
CANCEL_APPROVED_BY = userId,
REMARKS = entityBill.CANCELATION_REASON
};
// Add cancelled bill
context.BILL_INFO_CANCELEDS.Add(cancelledBill);
// Remove cancellation request
context.BILL_INFO_CANCEL_REQUESTS.Remove(context.BILL_INFO_CANCEL_REQUESTS.Single(where =>
where.BILL_NUMBER == cancelledBill.BILL_NUMBER && where.PAY_MODE_ID == cancelledBill.PAY_MODE_ID));
await context.SaveChangesAsync();
}
transaction.Commit();
}
catch (Exception exp)
{
transaction?.Rollback();
ErrorScope($"{LayerName} -> {callerInfo.MethodName} -> Exception [{exp.Message}]", exp, operationId, userId);
errorMessages.Add($"{LayerName} -> {callerInfo.MethodName} -> Exception [{exp.Message}]");
return (true, errorMessages);
}
}
return (errorMessages.Any(), errorMessages);
}
The issue is that in the ForEach loop, the first record is retrieved properly and processed. But the remaining records are never found (entityBill == null) but they are in the database. I believe it has something to do with the running transaction.
Can anyone help out?

This was a hard one to crack. It turns out that while String.Splitting() on the UI layer, there was an extra space being appended to the first index of the array within the List of arrays. So I fixed that by:
var targetBillNumber = billToApprove[0].Trim();
var targetPayModeId = int.Parse(billToApprove[1].Trim());