Develop Reference

List folders from a directory which id is not a specific value in Google Drive API v3 and C#

I want to get list of sub-folders from folder id but except by specific ids (not all of them). I meant I want to get children which id is not in the list I provide
public async Task<FileListData> ListWithoutIds(string id, int itemsPerPage = 10, string nextPageToken = null, bool onlyFolders = false, bool onlyFiles = false, string[] excludeIds = null)
{
if (string.IsNullOrWhiteSpace(id))
{
throw new ArgumentException($"'{nameof(id)}' cannot be null or whitespace", nameof(id));
}
var request = PrepareListRequest(itemsPerPage, nextPageToken);
request.Q = $"'{id}' in parents";
if (onlyFolders)
{
request.Q = $"{request.Q} and mimeType='application/vnd.google-apps.folder'";
}
else if (onlyFiles)
{
request.Q = $"{request.Q} and not mimeType='application/vnd.google-apps.folder'";
}
if (excludeIds?.Length > 0)
{
request.Q = $"{request.Q} {string.Join(" ", excludeIds.Select(e => $"and not id='{e}'"))}";
}
return FileListData.From(await request.ExecuteAsync()); // here I got 400 error
}
and
private FilesResource.ListRequest PrepareListRequest(int itemsPerPage, string nextPageToken, string[] specificFields = null)
{
var request = driveService.Files.List();
request.PageSize = itemsPerPage;
request.SupportsAllDrives = true;
request.IncludeItemsFromAllDrives = true;
if (!string.IsNullOrWhiteSpace(nextPageToken))
{
request.PageToken = nextPageToken;
}
if (specificFields?.Length > 0)
{
var fields = $"{string.Join(",", specificFields)}";
request.Fields = $"nextPageToken, files({fields.Trim(',')})";
}
else
{
request.Fields = "nextPageToken, files";
}
return request;
}
The query executed looks like:
'parentFolderId' in parents and mimeType='application/vnd.google-apps.folder' and not id='Child1Id' and not id='Child2Id' and not id='Child3Id'
And I got 400 error.
How to achive that ?

The query funcitonality of the Drive API is quite limited, the id of the list results is not a possible query term.
You need to reconstruct the logic of your code.
First get a list of your folders with the query 'parentFolderId' in parents and mimeType='application/vnd.google-apps.folder' and set Fields to files/id
Subsequently implement programmatically a filter to substract from the total id list the ids you want to exclude.

ASP.NET Core Large Data Set Processing

I have an Asp.net core 2.1 application with a Postgresql database.
A large file (~100K lines) gets uploaded to the database and then each line is processed.
As each line is processed there are approximately 15 sub queries per line taking up about 10 seconds in total per line. I have tried optimizing the queries and loading certain datasets in memory to reduce database calls which has seen a slight improvement. I have also tried reducing the batch size but that has also had minimal reduction.
Are there any other ways I can speed this up? The queries are written in lambda.
The high level function:
public void CDR_Insert(List<InputTable> inputTable, IQueryable<Costing__Table> CostingTable, List<Patterns> PatternsTable, IQueryable<Country_Calling_Codes> CallingCodesTable, IQueryable<Country> CountryTable, string voicemail_number)
{
foreach (var row in inputTable)
{
//if (!string.IsNullOrEmpty(row.finalCalledPartyPattern))
//{
var rowexists = _context.CDR.Where(m => m.InputId == row.id).Count();
if (rowexists == 0)
{
if (ValidateIfOneNumberIsForCountry(row.callingPartyNumber, row.finalCalledPartyNumber, PatternsTable, CountryTable) == true)
{
var CallRingTime = TimeSpan.FromSeconds(Ring_Time(row.dateTimeOrigination, row.dateTimeConnect));
var Call_Direction = CallDirection(row.callingPartyNumber, row.finalCalledPartyNumber, row.finalCalledPartyPattern, voicemail_number);
var CallReason = "Destination Out Of Order";
if (row.dateTimeConnect != "0")
{
CallReason = Call_Reason(row.callingPartyNumber, row.originalCalledPartyNumber, row.finalCalledPartyNumber, voicemail_number);
}
var CallOutcome = Outcome(row.duration, row.dateTimeOrigination, row.dateTimeConnect, row.callingPartyNumber, row.originalCalledPartyNumber, row.finalCalledPartyNumber, voicemail_number);
var CallCategory = GetCategory(row.callingPartyNumber, row.finalCalledPartyNumber, PatternsTable, CallingCodesTable, CountryTable);
var CallCost = 0.00;
if (CallOutcome != "Not Connected" || CallOutcome != "No answer")
{
CallCost = Costing(row.callingPartyNumber, row.finalCalledPartyNumber, row.dateTimeConnect, row.dateTimeDisconnect, row.finalCalledPartyPattern, row.duration, Call_Direction, CostingTable, PatternsTable, voicemail_number, CallingCodesTable, CountryTable);
}
CDR cDR = new CDR()
{
Date_Time = UnixTimeStampToDateTimeConvertor2(row.dateTimeOrigination),
Call_Direction = Call_Direction,
Calling_Party = row.callingPartyNumber_uri,
Calling_Digits = row.callingPartyNumber,
Called_Party = row.finalCalledPartyNumber_uri,
Called_Digits = row.finalCalledPartyNumber,
Duration = TimeSpan.FromSeconds(Convert.ToDouble(row.duration)),
Cost = CallCost,
Reason = CallReason,
Outcome = CallOutcome,
Country_Code = "MUS",//_context.Country.Where(o => o.CountryCode == _CDRExtension.CountryCodeId(_CDRExtension.Call_Type_Id(row.callingPartyNumber))).Select(o => o.CountryCode).FirstOrDefault(),
Second_Ring_Time = CallRingTime,
Category = CallCategory,
CiscoInputId = row.id
};
_context.CDR.Add(cDR);
}
row.Processed = true;
row.DateAdded = DateTime.Now;
_context.Cisco_Input.Update(row);
// }
_context.SaveChanges();
}
}
}

Stripe API - C#/.NET List All Customers problems with pagination

this might be a simple answer due to my inexperience with C# and .NET. I have two Stripe Test Accounts. TL:DR; is I am essentially looking for a Customers.all solution.
The source account has all the customer, card, and charge data.
The destination account has the copied card and customer data done by Stripe Support.
I have code that loops through the pulls all the data from the source account. It then finds the customer data from the destination account using the collection of customer/card data from the source. After that it then recreates the charges from the source account into the destination account.
I am able to successfully copy the first 100 charges into the destination account using information from the source account but I am having the hardest time getting the rest of the customers.
This is what I have so far:
public static void GenerateDestinationChargeData()
{
// code to get collection of customer data from destination account
StripeConfiguration.SetApiKey(destinationTestKey);
var customerService = new StripeCustomerService();
IEnumerable<StripeCustomer> customerItems = customerService.List(
new StripeCustomerListOptions()
{
Limit = 100,
//this is what I cannot figure out, eventually to get all of the customers from the destination account
StartingAfter = customerItems.LastOrDefault().Id
}
);
// loop through collection of customers from destination acct to fetch customer charge data from source account
foreach (var i in customerItems)
{
bool isError = false;
var liveChargeService = new StripeChargeService();
StripeConfiguration.SetApiKey(sourceTestKey);
StripeList<StripeCharge> chargeItems = new StripeList<StripeCharge>();
chargeItems = liveChargeService.List(
new StripeChargeListOptions()
{
Limit = 100,
CustomerId = i.Id
}
);
// loop through customer charge data from source and re-create charge data on destination Acct
foreach (var c in chargeItems.Data)
{
StripeConfiguration.SetApiKey(sourceTestKey);
var emailReceipt = "";
Dictionary<string, string> chargeMetaData = new Dictionary<string, string>();
var onBehalfOf = "";
var transferGroup = "";
var chargeDescription = "";
var chargeCaptured = "";
var chargeCurrency = "";
var chargeStatementDescriptor = "";
if (c.ReceiptEmail != null)
{
emailReceipt = c.ReceiptEmail;
}
if (c.Metadata != null)
{
chargeMetaData = c.Metadata;
}
if (c.OnBehalfOf != null)
{
onBehalfOf = c.OnBehalfOf.ToString();
}
if (c.TransferGroup != null)
{
transferGroup = c.TransferGroup;
}
if (c.Description != null)
{
chargeDescription = c.Description;
}
if (c.Captured != null)
{
chargeCaptured = c.Captured.ToString();
}
if (c.Currency != null)
{
chargeCurrency = c.Currency;
}
if (c.StatementDescriptor != null)
{
chargeStatementDescriptor = c.StatementDescriptor;
}
try
{
var chargeOptions = new StripeChargeCreateOptions();
chargeOptions.CustomerId = i.Id;
chargeOptions.ReceiptEmail = emailReceipt;
chargeOptions.Metadata = chargeMetaData;
chargeOptions.Description = chargeDescription;
chargeOptions.Capture = c.Captured;
chargeOptions.Currency = chargeCurrency;
chargeOptions.Amount = c.Amount;
chargeOptions.StatementDescriptor = chargeStatementDescriptor;
StripeChargeService chargeService = new StripeChargeService(destinationTestKey);
StripeCharge stripeCharge = chargeService.Create(chargeOptions);
}
catch (Exception ex)
{
Utility.NotifyDevAdminException("test", ex);
isError = true;
}
if (isError) continue;
}
}
}
Thank you so much :)

Since we cannot do a Customers.all with this current Stripe API, the solution is to set an empty variable and assign it the last Customer ID in the first set of 100 that we get and continue the query from that last assigned value
var lastId = String.Empty;
if (String.IsNullOrEmpty(lastId))
{
StripeConfiguration.SetApiKey(sourceCustomerAccountAPIKey);
customerItems = customerService.List(
new StripeCustomerListOptions(){ Limit = 100 });
}
else
{
StripeConfiguration.SetApiKey(sourceCustomerAccountAPIKey);
customerItems = customerService.List(
new StripeCustomerListOptions() {
Limit = 100,
StartingAfter = lastId });
}
lastId = customerItems.LastOrDefault().Id;

CRM Dynamics 2013 SDK Update Current Accounts With 2 Values

I have a scenario in CRM where I need to update existing accounts with their Vat and Registration number. There is well over 30 thousand accounts in the system. I am trying to update using the CRM SDK API but I am battling to figure out how to perform the actual update. The vat number and reg have been provided to me in a spreadsheet with their corresponding number, please note that the accounts are already in CRM so I just need to update the correct account with its Vat and Reg number, How can I do this in CRM, please advice on my code below:
public static void UpdateAllCRMAccountsWithVATAndRegistrationNumber(IOrganizationService service)
{
QueryExpression qe = new QueryExpression();
qe.EntityName = "account";
qe.ColumnSet = new ColumnSet("account", "new_vatno", "new_registrationnumber");
qe.Criteria.AddCondition("accountnumber", ConditionOperator.In,"TA10024846", "TA10028471", "TA20014015", "TA4011652", "TA4011557");
EntityCollection response = service.RetrieveMultiple(qe);
foreach (var acc in response.Entities)
{
acc.Attributes["new_vatno"] = //this is where I am struggling to figure out how I am gong to match the records up,
acc.Attributes["new_registrationnumber"] = //this is where I am struggling to figure out how I am gong to match the records up,
service.Update(acc);
}
}
How am I going to ensure that I update the correct records. I have the vat and reg numbers for the accounts in a spreadsheet, please see example image below. Can I please get advised here. Thanks.

I would load the list of VAT updates from the spreadsheet into a dictionary and then load the 30k record from CRM into memory. Then I would match them up and use ExecuteMultipleRequest to do the updates. Alternatively, you could query CRM using the account numbers (if the list is small enough.) I made the assumption you had thousands of updates to do across the record set of 30k. Note, if the Account record size was very large and couldn't be loaded into memory you would need to do account number queries.
Here is the rough code for the basic solution (I haven't tested, method should be split up, and there is minimal error handling):
public class VatInfo
{
public string RegistrationNumber;
public string TaxNumber;
public static Dictionary<string, VatInfo> GetVatList()
{
//TODO: Implement logic to load CSV file into a list. Dictionary key value should be Account Number
throw new NotImplementedException();
}
}
public class UpdateVatDemo
{
public const int maxBatchSize = 100;
public static void RunVatUpdate(IOrganizationService conn)
{
var vats = VatInfo.GetVatList();
var pagingQuery = new QueryExpression("account");
pagingQuery.ColumnSet = new ColumnSet("accountnumber");
Queue<Entity> allEnts = new Queue<Entity>();
while (true)
{
var results = conn.RetrieveMultiple(pagingQuery);
if (results.Entities != null && results.Entities.Any())
results.Entities.ToList().ForEach(allEnts.Enqueue);
if (!results.MoreRecords) break;
pagingQuery.PageInfo.PageNumber++;
pagingQuery.PageInfo.PagingCookie = results.PagingCookie;
}
ExecuteMultipleRequest emr = null;
while (allEnts.Any())
{
if (emr == null)
emr = new ExecuteMultipleRequest()
{
Settings = new ExecuteMultipleSettings()
{
ContinueOnError = true,
ReturnResponses = true
},
Requests = new OrganizationRequestCollection()
};
var ent = allEnts.Dequeue();
if (vats.ContainsKey(ent.GetAttributeValue<string>("accountnumber")))
{
var newEnt = new Entity("account", ent.Id);
newEnt.Attributes.Add("new_vatno", vats[ent.GetAttributeValue<string>("accountnumber")].TaxNumber);
newEnt.Attributes.Add("new_registrationnumber", vats[ent.GetAttributeValue<string>("accountnumber")].RegistrationNumber);
emr.Requests.Add(new UpdateRequest() { Target = newEnt });
}
if (emr.Requests.Count >= maxBatchSize)
{
try
{
var emResponse = (ExecuteMultipleResponse) conn.Execute(emr);
foreach (
var responseItem in emResponse.Responses.Where(responseItem => responseItem.Fault != null))
DisplayFault(emr.Requests[responseItem.RequestIndex],
responseItem.RequestIndex, responseItem.Fault);
}
catch (Exception ex)
{
Console.WriteLine($"Exception during ExecuteMultiple: {ex.Message}");
throw;
}
emr = null;
}
}
}
private static void DisplayFault(OrganizationRequest organizationRequest, int count,
OrganizationServiceFault organizationServiceFault)
{
Console.WriteLine(
"A fault occurred when processing {1} request, at index {0} in the request collection with a fault message: {2}",
count + 1,
organizationRequest.RequestName,
organizationServiceFault.Message);
}
}

Updating the fetched entity is bound to fail because of its entity state, which would not be null.
To update the fetched entities, you need to new up the entity:
foreach (var acc in response.Entities)
{
var updateAccount = new Entity("account") { Id = acc.Id };
updateAccount .Attributes["new_vatno"] = null; //using null as an example.
updateAccount .Attributes["new_registrationnumber"] = null;
service.Update(acc);
}

Code below shows how I managed to get it righy. forst let me explain. I imported my records into a seperate SQL table, in my code I read that table into a list in memory, I then query CRM accounts that need to be updated, I then loop though each account and check if the account number in CRM matches the account number from my sql database, if it matches, I then update the relevant Reg no and Vat no, See code below:
List<Sheet1_> crmAccountList = new List<Sheet1_>();
//var crmAccount = db.Sheet1_.Select(x => x).ToList().Take(2);
var crmAccounts = db.Sheet1_.Select(x => x).ToList();
foreach (var dbAccount in crmAccounts)
{
CRMDataObject modelObject = new CRMDataObject()
{
ID = dbAccount.ID,
Account_No = dbAccount.Account_No,
Tax_No = dbAccount.Tax_No.ToString(),
Reg_No = dbAccount.Reg_No
//Tarsus_Country = dbAccount.Main_Phone
};
}
var officialDatabaseList = crmAccounts;
foreach (var crmAcc in officialDatabaseList)
{
QueryExpression qe = new QueryExpression();
qe.EntityName = "account";
qe.ColumnSet = new ColumnSet("accountnumber", "new_vatno", "new_registrationnumber");
qe.Criteria.AddCondition("accountnumber", ConditionOperator.In,'list of account numbers go here'
EntityCollection response = service.RetrieveMultiple(qe);
foreach (var acc in response.Entities)
{
if (crmAcc.Account_No == acc.Attributes["accountnumber"].ToString())
{
//acc.Attributes["new_vatno"] = crmAcc.VAT_No.ToString();
acc.Attributes["new_registrationnumber"] = crmAcc.Reg_No.ToString();
service.Update(acc);
}
}
}

Handle duplicate records when doing a EF SaveChanges

I have an web application where I parse a csv file that can have over 200k records in it. I parse each line for information, verify that the key does not exist in the database and then add it to the context. When the count reaches 10,000 records it calls SaveChanges routine. The problem is that there can be duplicates in the context and it errors out. This is running on a Azure VM communicating to an Azure SQL server.
Two questions, how do I handle the duplicate issue and is there any way I can improve the speed as it takes several hours to run?
using (LoanFileEntities db = new LoanFileEntities())
{
db.Configuration.AutoDetectChangesEnabled = false; // 1. this is a huge time saver
db.Configuration.ValidateOnSaveEnabled = false; // 2. this can also save time
while (parser.Read())
{
counter++;
int loan_code = 0;
string loan_code_string = parser["LoanId"];
string dateToParse = parser["PullDate"].Trim();
DateTime date_pulled;
try
{
date_pulled = DateTime.Parse(dateToParse, CultureInfo.InvariantCulture);
}
catch (Exception)
{
throw new Exception("No Pull Date for line " + counter);
}
string originationdate = parser["OriginationDate"].Trim();
DateTime date_originated;
try
{
date_originated = DateTime.Parse(originationdate, CultureInfo.InvariantCulture);
}
catch (Exception)
{
throw new Exception("No Origination Date for line " + counter);
}
dateToParse = parser["DueDate"].Trim();
DateTime date_due;
try
{
date_due = DateTime.Parse(dateToParse, CultureInfo.InvariantCulture);
}
catch (Exception)
{
throw new Exception("No Due Date for line " + counter);
}
string region = parser["Region"].Trim();
string source = parser["Channel"].Trim();
string password = parser["FilePass"].Trim();
decimal principalAmt = Convert.ToDecimal(parser["Principal"].Trim());
decimal totalDue = Convert.ToDecimal(parser["TotalDue"].Trim());
string vitaLoanId = parser["VitaLoanId"];
var toAdd =
db.dfc_LoanRecords.Any(
x => x.loan_code_string == loan_code_string);
if (!toAdd)
{
dfc_LoanRecords loan = new dfc_LoanRecords();
loan.loan_code = loan_code;
loan.loan_code_string = loan_code_string;
loan.loan_principal_amt = principalAmt;
loan.loan_due_date = date_due;
loan.date_pulled = date_pulled;
loan.date_originated = date_originated;
loan.region = region;
loan.source = source;
loan.password = password;
loan.loan_amt_due = totalDue;
loan.vitaLoanId = vitaLoanId;
loan.load_file = fileName;
loan.load_date = DateTime.Now;
switch (loan.region)
{
case "UK":
if (location.Equals("UK"))
{
//db.dfc_LoanRecords.Add(loan);
if (loan.source == "Online")
{
counter_new_uk_online++;
}
else
{
counter_new_uk_retail++;
}
}
break;
case "US":
if (location.Equals("US"))
{
db.dfc_LoanRecords.Add(loan);
if (loan.source == "Online")
{
counter_new_us_online++;
}
else
{
counter_new_us_retail++;
}
}
break;
case "Canada":
if (location.Equals("US"))
{
db.dfc_LoanRecords.Add(loan);
if (loan.source == "Online")
{
counter_new_cn_online++;
}
else
{
counter_new_cn_retail++;
}
}
break;
}
// delay save to speed up load. 3. also saves transactional time
if (counter % 10000 == 0)
{
db.SaveChanges();
}
}
} // end of parser read
db.SaveChanges();
}
}
}

I would suggest removing duplicates in the code before sending it over to .SaveChanges().
Instead of going into detail about duplicate removal, I've put together this list of links to existing questions and answers on StackOverflow that may help:
Delete duplicates using Lambda
Using DISTINCT on a subquery to remove duplicates in Entity Framework
Using LINQ to find / delete duplicates
Hope that helps!