Develop Reference

Using ValueTuple returns Item1,Item2 why?

The next api returns in Postmen and to the client item1,item2
While I am using ValueTuple to change the names (the names not so important, but I can’t return item1,item2)
public async Task<(List<CategoryFilterResponseDTO> categoryFilters, string MetaDataDescription)> GetCategoryFilterPage([FromBody]categoryFilterRequestDTO categoryFilterRequest)
{
var logItem = new LogDTO();
var result = await _service.GetCategoryFilterPage(categoryFilterRequest);
try
{
OnStart(logItem, new object[] { categoryFilterRequest });
var categoryFilters = result.categoryFilters;
var MetaDataDescription = result.MetaDataDescription;
return (categoryFilters: categoryFilters, MetaDataDescription: MetaDataDescription);
}
}
the method:
public async Task<(List<CategoryFilterResponseDTO> categoryFilters, string MetaDataDescription)> GetCategoryFilterPage(categoryFilterRequestDTO categoryFilterRequestDTO)
{
List<CategoryFilterResponseDTO> categoryFilter = new List<CategoryFilterResponseDTO>();
List<FavoriteDTO> isFavorite = null;
string MetaDataDescription = "";
(List<FilterSortDTO<FlatSearchCategory>>, int) searchCategory = await _clubRepo.CategoryFilterPage(categoryFilterRequestDTO);//BranchesCount
if (searchCategory.Item2 == 0)
{
MetaDataDescription = GetCategoryDetails(categoryFilterRequestDTO.CategoryFirstFatherID.Value).CategoryName;
return (categoryFilters: categoryFilter, MetaDataDescription: MetaDataDescription);
}

You can change your return to the following:
return Ok(new
{
categoryFilters = categoryFilter,
metaDataDescription = MetaDataDescription
});
You will need to change your return type to ActionResult or similar as well.

Because that's the name of the fields on a ValueTuple<,>.
The names given to the values are only available to source code.

Multithreaded c# console app to scrape data from sites

I have written an app that goes through our own properties and scraps the data. To make sure I don't run through the same URLs, I am using a MySQL database to store the URL, flag it once its processed. All this was being done in a single thread and it's fine if I had only few thousand entries. But I have few hundred thousand entries that I need to parse so I need to make changes in the code (I am newbie in multithreading in general). I found an example and was trying to copy the style but doesn't seem to work. Anyone know what the issue is with the following code?
EDIT: Sorry didn't mean to make people guess the issue but was stupid of me to include the exception. Here is the exception
"System.InValidCastException: 'Specified cast is not valid.'"
When I start the process it collects the URLs from the database and then never hits DoWork method
//This will get the entries from the database
List<Mappings> items = bot.GetUrlsToProcess(100);
if (items != null)
{
var tokenSource = new CancellationTokenSource();
var token = tokenSource.Token;
Worker.Done = new Worker.DoneDelegate(WorkerDone);
foreach (var item in items)
{
urls.Add(item.Url);
WaitingTasks.Enqueue(new Task(id => new Worker().DoWork((int)id, item.Url, token), item.Url, token));
}
LaunchTasks();
}
static async void LaunchTasks()
{
// keep checking until we're done
while ((WaitingTasks.Count > 0) || (RunningTasks.Count > 0))
{
// launch tasks when there's room
while ((WaitingTasks.Count > 0) && (RunningTasks.Count < MaxRunningTasks))
{
Task task = WaitingTasks.Dequeue();
lock (RunningTasks) RunningTasks.Add((int)task.AsyncState, task);
task.Start();
}
UpdateConsole();
await Task.Delay(300); // wait before checking again
}
UpdateConsole(); // all done
}
static void UpdateConsole()
{
Console.Write(string.Format("\rwaiting: {0,3:##0} running: {1,3:##0} ", WaitingTasks.Count, RunningTasks.Count));
}
static void WorkerDone(int id)
{
lock (RunningTasks) RunningTasks.Remove(id);
}
public class Worker
{
public delegate void DoneDelegate(int taskId);
public static DoneDelegate Done { private get; set; }
public async void DoWork(object id, string url, CancellationToken token)
{
if (token.IsCancellationRequested) return;
Content obj;
try
{
int tries = 0;
bool IsUrlProcessed = true;
DateTime dtStart = DateTime.Now;
string articleDate = string.Empty;
try
{
ScrapeWeb bot = new ScrapeWeb();
SearchApi searchApi = new SearchApi();
SearchHits searchHits = searchApi.Url(url, 5, 0);
if (searchHits.Hits.Count() == 0)
{
obj = await bot.ReturnArticleObject(url);
if (obj.Code != HttpStatusCode.OK)
{
Console.WriteLine(string.Format("\r Status is {0}", obj.Code));
tries = itemfound.UrlMaxTries + 1;
IsUrlProcessed = false;
itemfound.HttpCode = obj.Code;
}
else
{
string title = obj.Title;
string content = obj.Contents;
string description = obj.Description;
Articles article = new Articles();
article.Site = url.GetSite();
article.Content = content;
article.Title = title;
article.Url = url.ToLower();
article.Description = description;
string strThumbNail = HtmlHelper.GetImageUrl(url, obj.RawResponse);
article.Author = HtmlHelper.GetAuthor(url, obj.RawResponse);
if (!string.IsNullOrEmpty(strThumbNail))
{
//This condition needs to be added to remove ?n=<number> from EP thumbnails
if (strThumbNail.Contains("?"))
{
article.ImageUrl = strThumbNail.Substring(0, strThumbNail.IndexOf("?")).Replace("http:", "https:");
}
else
article.ImageUrl = strThumbNail.Replace("http:", "https:");
}
else
{
article.ImageUrl = string.IsNullOrEmpty(strThumbNail) ? article.Url.GetDefaultImageUrls() : strThumbNail.Replace("http:", "https:");
}
articleDate = HtmlHelper.GetPublishDate(url, obj.RawResponse);
if (string.IsNullOrEmpty(articleDate))
article.Pubdate = DateTime.Now;
else
article.Pubdate = DateTime.Parse(articleDate);
var client = new Index(searchApi);
var result = client.Upsert(article);
itemfound.HttpCode = obj.Code;
if (result)
{
itemfound.DateCreated = DateTime.Parse(articleDate);
itemfound.DateModified = DateTime.Parse(articleDate);
UpdateItem(itemfound);
}
else
{
tries = itemfound.UrlMaxTries + 1;
IsUrlProcessed = false;
itemfound.DateCreated = DateTime.Parse(articleDate);
itemfound.DateModified = DateTime.Parse(articleDate) == null ? DateTime.Now : DateTime.Parse(articleDate);
UpdateItem(itemfound, tries, IsUrlProcessed);
}
}
}
else
{
tries = itemfound.UrlMaxTries + 1;
IsUrlProcessed = true;
itemfound.HttpCode = HttpStatusCode.OK;
itemfound.DateCreated = DateTime.Parse(articleDate);
itemfound.DateModified = DateTime.Parse(articleDate) == null ? DateTime.Now : DateTime.Parse(articleDate);
}
}
catch (Exception e)
{
tries = itemfound.UrlMaxTries + 1;
IsUrlProcessed = false;
itemfound.DateCreated = DateTime.Parse(articleDate);
itemfound.DateModified = DateTime.Parse(articleDate) == null ? DateTime.Now : DateTime.Parse(articleDate);
}
finally
{
DateTime dtEnd = DateTime.Now;
Console.WriteLine(string.Format("\r Total time taken to process items is {0}", (dtEnd - dtStart).TotalSeconds));
}
}
catch (Exception e)
{
Console.WriteLine(e);
}
Done((int)id);
}
}
All this code is based from Best multi-thread approach for multiple web requests this link. Can someone tell me how to get this approach running?

I think the problem is in the way you're creating your tasks:
new Task(id => new Worker().DoWork((int)id, item.Url, token), item.Url, token)
This Task constructor overload expected Action<object> delegate. That means id will be typed as object and you need to cast it back to something useful first.
Parameters
action
Type: System.Action<Object>
The delegate that represents the code to execute in the task.
state
Type: System.Object
An object representing data to be used by the action.
cancellationToken
Type: System.Threading.CancellationToken
-The CancellationToken that that the new task will observe.
You decided to cast it to int by calling (int)id, but you're passing item.Url as the object itself. I can't tell you 100% what the type of Url is but I don't expect Url-named property to be of type int.

Based on what #MarcinJuraszek said I just went back to my code and added an int as I couldn't find another way to resolve it. Here is the change I made
int i=0
foreach (var item in items)
{
urls.Add(item.Url);
WaitingTasks.Enqueue(new Task(id => new Worker().DoWork((string)id, item.Url, token), item.Url, token));
i++;
}

How to incorporate criteria in Where method?

I want to ask about the WHERE condition in my search command. I'm calling web service (API) during searching and I want to put WHERE statement in my code but there's an error.
private async Task CallApi(string searchText = null)
{
long lastUpdatedTime = 0;
long.TryParse(AppSettings.ComplaintLastUpdatedTick, out lastUpdatedTime);
var currentTick = DateTime.UtcNow.Ticks;
var time = new TimeSpan(currentTick - lastUpdatedTime);
if (time.TotalSeconds > 1) {
int staffFk = Convert.ToInt32(StaffId);
var result = await mDataProvider.GetComplaintList(lastUpdatedTime, mCts.Token, staffFk);
if (result.IsSuccess)
{
// Save last updated time
AppSettings.ComplaintLastUpdatedTick = result.Data.Updated.ToString();
// Store data into database
if ((result.Data.Items != null) &&
(result.Data.Items.Count > 0))
{
var datas = new List<Complaint>(result.Data.Items);
**if (!string.IsNullOrEmpty(searchText))
{
datas = datas.Where(i => i.Description.Contains(searchText)
&& (i.SupervisorId.Equals(StaffId))
|| (i.ProblemTypeName.Contains(searchText)));
}
else
{
datas = datas.Where(i => i.SupervisorId.Equals(StaffId));
}**
Datas = new ObservableCollection<Complaint>(datas);
}
}
else if (result.HasError)
{
await mPageDialogService.DisplayAlertAsync("Error", result.ErrInfo.Message, "OK");
}
}
}
Both assignments of datas in the if ... else causes System.Collections.Generic.IEnumerable<ECS.Features.Complaints.Complaint>' to 'System.Collections.Generic.List<ECS.Features.Complaints.Complaint>'. An explicit conversions exists (are you missing a cast?) compilation errors:
I don't know how to use the WHERE condition there. Please help me. Thank you in advance for your concern.

datas is a List<Complaint> but you try to reassign it to IEnumerable<Complaint> with the Where statement. Add a ToList() after the Where to maintain type,
Or you could just declare datas as IEnumerable<Complaint>
IEnumerable<Complaint> datas = new List<Complaint>(result.Data.Items);

Issue is that datas is defined as being a List<Complaint>, and the return type of datas.Where(...) is an IEnumerable/IQueryable.
You could do:
datas = datas.Where(i => i.SupervisorId.Equals(StaffId)).ToList();
Complete code:
private async Task CallApi(string searchText = null)
{
long lastUpdatedTime = 0;
long.TryParse(AppSettings.ComplaintLastUpdatedTick, out lastUpdatedTime);
var currentTick = DateTime.UtcNow.Ticks;
var time = new TimeSpan(currentTick - lastUpdatedTime);
if (time.TotalSeconds > 1) {
int staffFk = Convert.ToInt32(StaffId);
var result = await mDataProvider.GetComplaintList(lastUpdatedTime, mCts.Token, staffFk);
if (result.IsSuccess)
{
// Save last updated time
AppSettings.ComplaintLastUpdatedTick = result.Data.Updated.ToString();
// Store data into database
if ((result.Data.Items != null) &&
(result.Data.Items.Count > 0))
{
var datas = new List<Complaint>(result.Data.Items);
if (!string.IsNullOrEmpty(searchText))
{
datas = datas.Where(i => i.Description.Contains(searchText)
&& (i.SupervisorId.Equals(StaffId))
|| (i.ProblemTypeName.Contains(searchText))).ToList();
}
else
{
datas = datas.Where(i => i.SupervisorId.Equals(StaffId)).ToList();
}
Datas = new ObservableCollection<Complaint>(datas);
}
}
else if (result.HasError)
{
await mPageDialogService.DisplayAlertAsync("Error", result.ErrInfo.Message, "OK");
}
}
}
You will then also have an error on the next line, Datas = new ObservableCollection becasue Datas is not defined, and if you meant datas, again, it will not be the List<> that you initially defined.

Entity.Contains(AttributeName) works for all annotation fields but not working for notetext

In following code snippet I am retrieving notes related to an order. It works fine only if notetext does contain data. Now, while debugging I found that, in other case it throws the exception that Object reference not set to an instance of an object.
I think following snippet looks good, but not sure what is missing, any idea to sort out the problem?
private void fetchDocument(IOrganizationService service, Guid vOrderId)
{
EntityCollection results = null;
string tempNote = string.Empty;
string tempFileName = string.Empty;
ColumnSet cols = new ColumnSet("subject", "filename", "documentbody", "mimetype","notetext");
QueryExpression query = new QueryExpression {
EntityName = "annotation" ,
ColumnSet = cols,
Criteria = new FilterExpression
{
Conditions = {
new ConditionExpression("objectid",ConditionOperator.Equal,vOrderId)
}
}
};
results = service.RetrieveMultiple(query);
Entity defaultRecord = results.Entities.ElementAtOrDefault(0);
if(defaultRecord.Contains("notetext"))
{
tempNote = defaultRecord.GetAttributeValue<string>("notetext");
}
if (defaultRecord.Contains("filename"))
{
tempFileName = defaultRecord.GetAttributeValue<string>("filename");
}
}

You haven't guarded defaultrecord against null.
results = service.RetrieveMultiple(query);
if (results.Entities == null || !results.Entities.Any()) return;
Entity defaultRecord = results.Entities.ElementAt(0);
Extending the answer to backup result.Entities == null check.
Retrieve multiple EntityCollection is not foolproof.
EntityCollection property:
Decomplied SDK retrieve multiple core:
protected internal virtual EntityCollection RetrieveMultipleCore(QueryBase query)
{
bool? retry = new bool?();
do
{
bool forceClose = false;
try
{
using (new OrganizationServiceContextInitializer(this))
return this.ServiceChannel.Channel.RetrieveMultiple(query);
}
catch (MessageSecurityException ex)
{
..
}
finally
{
this.CloseChannel(forceClose);
}
}
while (retry.HasValue && retry.Value);
return (EntityCollection) null;
}
Decomplied SDK Cached Organization Serivce Context Retrieve multiple:
public override EntityCollection RetrieveMultiple(QueryBase query)
{
RetrieveMultipleRequest retrieveMultipleRequest = new RetrieveMultipleRequest();
retrieveMultipleRequest.Query = query;
RetrieveMultipleResponse multipleResponse = this.Execute<RetrieveMultipleResponse>((OrganizationRequest) retrieveMultipleRequest);
if (multipleResponse == null)
return (EntityCollection) null;
else
return multipleResponse.EntityCollection;
}
public EntityCollection EntityCollection
{
get
{
if (this.Results.Contains("EntityCollection"))
return (EntityCollection) this.Results["EntityCollection"];
else
return (EntityCollection) null;
}
}

Your issue is actually at this line:
Entity defaultRecord = results.Entities.ElementAtOrDefault(0);
There are no results found, meaning
there is no Annotation that exists with an objectid of "vOrderId", or the user that is performing the query, doesn't have rights to read that record.
Regardless, you should just check for defaultRecord being null or not, and exiting if it is.
This check of null is a common occurrence, which is why I've written this ExtensionMethod:
public Entity GetFirstOrDefault(this IOrganizationService service, QueryBase qb) {
return service.RetrieveMultiple(qb)?.Entities.FirstOrDefault();
}
This would simplify your code to this:
private void fetchDocument(IOrganizationService service, Guid vOrderId)
{
EntityCollection results = null;
string tempNote = string.Empty;
string tempFileName = string.Empty;
ColumnSet cols = new ColumnSet("subject", "filename", "documentbody", "mimetype","notetext");
QueryExpression query = new QueryExpression {
EntityName = "annotation" ,
ColumnSet = cols,
Criteria = new FilterExpression
{
Conditions = {
new ConditionExpression("objectid",ConditionOperator.Equal,vOrderId)
}
}
};
var defaultRecord = service.GetFirstOrDefault(query);
if(defaultRecord != null)
{
if(defaultRecord.Contains("notetext"))
{
tempNote = defaultRecord.GetAttributeValue<string>("notetext");
}
if (defaultRecord.Contains("filename"))
{
tempFileName = defaultRecord.GetAttributeValue<string>("filename");
}
}
}

How to properly dispose objects created for Ldap search using ADODB ADsDSObject provider

I am looking for the best way how to lookup LDAP directory for users by given criteria. At the moment the best performance seems to offer usage of ADsDSObject provider. The code will run in ASP.NET web site.
I would like to confirm how to properly dispose the resources. Here is the code used at the moment. Is the code releasing resources correctly or need to be improved?
public static List<LookupValues> FindBy(LdapSearchCriteria criteria)
{
List<LookupValues> usersMatchingCriteria = new List<LookupValues>();
ADODB.Command adoCommand = new ADODB.Command();
ADODB.Connection adoConnection = new ADODB.Connection();
ADODB.Recordset adoResultSet = new ADODB.Recordset();
adoConnection.ConnectionString = connectionString;
adoConnection.Open();
adoCommand.ActiveConnection = adoConnection;
adoCommand.CommandText = BuildSelectStatmentFrom(criteria);
object dummy = Type.Missing;
try
{
adoResultSet = adoCommand.Execute(out dummy, ref dummy, 0);
if (adoResultSet != null)
{
while (adoResultSet.EOF == false)
{
LookupValues value = new LookupValues();
for (int i = 0; i < adoResultSet.Fields.Count; i++)
{
switch (adoResultSet.Fields[i].Name)
{
case "a-foreignGivenName":
value.FirstName = (adoResultSet.Fields[i].Value).ToString();
break;
case "a-foreignSn":
value.LastName = (adoResultSet.Fields[i].Value).ToString();
break;
}
}
usersMatchingCriteria.Add(value);
adoResultSet.MoveNext();
}
}
}
finally
{
if (adoResultSet != null)
{
adoResultSet.Close();
adoResultSet = null;
}
if (adoConnection != null)
{
adoConnection.Close();
adoConnection = null;
}
}
return usersMatchingCriteria;
}

I found equivalent and even a bit faster to use classes from System.DirectoryServices.Protocols namespace. Equivalent code using .NET classes
public List<LookupValues> FindBy(LdapSearchCriteria criteria)
{
List<LookupValues> usersMatchingCriteria = new List<LookupValues>();
NetworkCredential credentials = new NetworkCredential(connectionDetails.UserName, connectionDetails.Password, connectionDetails.Domain);
LdapDirectoryIdentifier directoryIdentifier = new LdapDirectoryIdentifier(connectionDetails.Server, connectionDetails.Port, false, false);
using (LdapConnection connection = CreateConnection(directoryIdentifier))
{
connection.Bind(credentials);
SearchRequest search = CreateSearchRequest(criteria);
SearchResponse response = connection.SendRequest(search) as SearchResponse;
foreach (SearchResultEntry entry in response.Entries)
{
LookupValues foundUser = GetUserDetailsFrom(entry);
usersMatchingCriteria.Add(foundUser);
}
}
return usersMatchingCriteria;
}