I have a cron-job method which constructs a user's story feed based on user's featured stories, followed categories and followed users.
The final feed is added in the below database table in correct order:
UserFeed table:
Uid StoryListIds
1 3, 23, 45, 3, 6, 234, .....
2 3, 23, 45, 6, 87, 44, .....
3 3, 23, 45, 32, 4, 62, .....
The method is below and contains comments.
The code:
public void ConstructUserFeed()
{
try
{
//get all user ids
var userIds = userBL.GetIds();
//get all featured stories
var featStories = storyBL.GetFeaturedStories();
if (userIds != null && userIds.Count > 0)
{
foreach (var userId in userIds)
{
//integer List to store the stories ids in correct order
List<int> storyIdsLst = new List<int>();
//integer List for excluding duplicates
List<int> exceptStoryIds = new List<int>();
//user feed List to store the stories
var userFeed = new List<UserFeedDTO>();
//string to store the id list so we can add it later in db
string storyIds = "";
if (featStories != null && featStories.Count > 0)
{
foreach (var featStory in featStories)
{
//first add all the ids of featured stories except own user stories, ordered by date
if (featStory.authorId != userId)
{
storyIdsLst.Add(featStory.id);
exceptStoryIds.Add(featStory.id);
}
}
}
//get user's followed categories ids
var followedCategoryIds = userCategoriesBL.GetFollowedCategoryIds(userId);
if (followedCategoryIds != null && followedCategoryIds.Count > 0)
{
foreach (var categoryId in followedCategoryIds)
{
//get the user's 5 latest stories for every followed category
//except own stories and previous stories
var storiesByCateg = storyBL.GetByCategory(5, categoryId, userId, exceptStoryIds);
if (storiesByCateg != null && storiesByCateg.Count > 0)
{
foreach (var storyByCateg in storiesByCateg)
{
userFeed.Add(storyByCateg);
exceptStoryIds.Add(storyByCateg.id);
}
}
}
}
//get user's followed users ids
var followedUserIds = userFollowersBL.GetFollowedUserIds(userId);
if (followedUserIds != null && followedUserIds.Count > 0)
{
foreach (var followedId in followedUserIds)
{
//get the user's 5 latest stories for every followed user
//except own stories and previous stories
var storiesByFollowedUsers = storyBL.GetByFollowedUser(5, followedId, userId, exceptStoryIds);
if (storiesByFollowedUsers != null && storiesByFollowedUsers.Count > 0)
{
foreach (var storyByFollowedUsers in storiesByFollowedUsers)
{
userFeed.Add(storyByFollowedUsers);
}
}
}
}
// order the stories by date
userFeed = userFeed.OrderByDescending(story => story.dateAdded).ToList();
if (userFeed != null && userFeed.Count > 0)
{
foreach (var story in userFeed)
{
//add the story ids after the featured story ids
storyIdsLst.Add(story.id);
}
}
//comma separated list of story ids as string so we can store it in db
storyIds = string.Join(",", storyIdsLst.Select(n => n.ToString()).ToArray());
//create the UserFeed model
UserFeed userFeedModel = new UserFeed();
userFeedModel.userId = userId;
userFeedModel.storyListId = storyIds;
userFeedModel.lastUpdateTime = DateTime.Now;
userFeedBL.AddOrUpdateUserFeed(userFeedModel);
}
uof.Save();
}
}
catch (Exception ex)
{
Console.WriteLine("Error occurred in processing job. Error : {0}", ex.Message);
}
}
The above method takes ~35 seconds to complete for 30 users.
Q: How can I improve my code and performance?
It's hard to say exactly what's causing it to execute so slowly, for that I recommend profiling your question with SQL Server Profiler. Check that your queries is properly asked and doesn't do anything unnecessary.
After that, I would consider asking fewer questions. Since you're doing it in a loop you may benefit from doing fewer, but heavier questions. For example (assuming userFollowersBL.* is querying the DB):
var followedCategoryIds = userCategoriesBL.GetFollowedCategoryIds(userId);
I assume that the signature is something like:
IEnumerable<int> GetFollowedCategoryIds(userId);
Consider changing it to:
IDictionary<int, IEnumerable<int>> GetFollowedCategoryIds(IEnumerable<int> userIds);
Then you'll have a dictionary with the userids and each of their followedCategoryIds in memory before you start your foreach. This way you can send all your result from userBL.GetIds() in one query. It may speed up performance to do it once, instead of 30 times. The same thing goes for userFollowersBL.GetFollowedUserIds(userId).
Now you have decreased the number of queries to you DB with approx 58 times.
public void ConstructUserFeed()
{
try
{
//get all user ids
var userIds = userBL.GetIds();
//get all featured stories
var featStories = storyBL.GetFeaturedStories();
// Fetch all in one query.
IDictionary<int,IEnumerable<int>> allFollowedCategoryIds= userCategoriesBL.GetFollowedCategoryIds(userIds);
// Fetch all in one query
IDictionary<int,IEnumerable<int>> allUserFollowers = userFollowersBL.GetFollowedUserIds(userIds);
if (userIds != null && userIds.Count > 0)
{
foreach (var userId in userIds)
{
//integer List to store the stories ids in correct order
List<int> storyIdsLst = new List<int>();
//integer List for excluding duplicates
List<int> exceptStoryIds = new List<int>();
//user feed List to store the stories
var userFeed = new List<UserFeedDTO>();
//string to store the id list so we can add it later in db
string storyIds = "";
if (featStories != null && featStories.Count > 0)
{
foreach (var featStory in featStories)
{
//first add all the ids of featured stories except own user stories, ordered by date
if (featStory.authorId != userId)
{
storyIdsLst.Add(featStory.id);
exceptStoryIds.Add(featStory.id);
}
}
}
//get user's followed categories ids
var followedCategoryIds = allFollowedCategoryIds[userId]
if (followedCategoryIds != null && followedCategoryIds.Count > 0)
{
foreach (var categoryId in followedCategoryIds)
{
//get the user's 5 latest stories for every followed category
//except own stories and previous stories
var storiesByCateg = storyBL.GetByCategory(5, categoryId, userId, exceptStoryIds);
if (storiesByCateg != null && storiesByCateg.Count > 0)
{
foreach (var storyByCateg in storiesByCateg)
{
userFeed.Add(storyByCateg);
exceptStoryIds.Add(storyByCateg.id);
}
}
}
}
//get user's followed users ids
var followedUserIds = allUserFollowers[userId];
if (followedUserIds != null && followedUserIds.Count > 0)
{
foreach (var followedId in followedUserIds)
{
//get the user's 5 latest stories for every followed user
//except own stories and previous stories
var storiesByFollowedUsers = storyBL.GetByFollowedUser(5, followedId, userId, exceptStoryIds);
if (storiesByFollowedUsers != null && storiesByFollowedUsers.Count > 0)
{
foreach (var storyByFollowedUsers in storiesByFollowedUsers)
{
userFeed.Add(storyByFollowedUsers);
}
}
}
}
// order the stories by date
userFeed = userFeed.OrderByDescending(story => story.dateAdded).ToList();
if (userFeed != null && userFeed.Count > 0)
{
foreach (var story in userFeed)
{
//add the story ids after the featured story ids
storyIdsLst.Add(story.id);
}
}
//comma separated list of story ids as string so we can store it in db
storyIds = string.Join(",", storyIdsLst.Select(n => n.ToString()).ToArray());
//create the UserFeed model
UserFeed userFeedModel = new UserFeed();
userFeedModel.userId = userId;
userFeedModel.storyListId = storyIds;
userFeedModel.lastUpdateTime = DateTime.Now;
userFeedBL.AddOrUpdateUserFeed(userFeedModel);
}
uof.Save();
}
}
catch (Exception ex)
{
Console.WriteLine("Error occurred in processing job. Error : {0}", ex.Message);
}
}
But honestly, this is mostly speculations and may differ from time to time. In my experience you tend to benefit from asking for more data once, than the same data several times.
Related
I have a list inside of another list. I need to add to the list only those elements whose internal list is not empty. The data is pulled from the server. The code contains 4 lists, Each inside of another - Companies - Shops - Campaigns - Coupons. I need to check if the Coupons list is not empty and add the elements related to that. I would be very thankfull if someone could help with that.
foreach (var row in Companies)
{
reportData.companies.Add(new Companies { ID = row.Id, name = row.BrandName });
var Shops = logicService.CompanyShopService.GetShopsByCompany(row.Id);
var Campaigns = logicService.CampaignService.GetCampaignsByCompany(row.Id);
foreach (var shop in Shops)
{
reportData.companies.Where(item => item.ID == shop.ClientID).FirstOrDefault().shops.Add(new Distribution_Shops { Id = shop.Id, Name = shop.Name });
var shop_campaigns = Campaigns.Where(item => item.ShopID == shop.Id);
foreach (var campaign in shop_campaigns)
{
var coupons = logicService.CouponsService.GetAllByCampaign(campaign.Id, false);
foreach (var company in reportData.companies)
{
foreach (var tmp_shop in company.shops)
{
if (tmp_shop.Id == shop.Id)
{
tmp_shop.campaigns.Add(new Distribution_Campaign { Id = campaign.Id, Name = campaign.CampaignName, coupons = coupons});
}
}
}
}
}
}
I have a result verification app. That matches candidates results against some API result and populates the data in a table called result, and inserts the candidate's data in table candidate with has a status column.
I want to use the CandidateNo to check result table to see if he has an unmatched result.
If true, update status on candidate table and set it to unverified
Otherwise set it to Verified
So, if a candidate has all his or her subjects = Match, then updateStatusOfCandidate(d.Candidate.CandidateNo, "VERIFIED");
enter code here
Else updateStatusOfCandidate(d.Candidate.CandidateNo, "Unverified");
private void Process()
{
//read the data from csv
List<CsvData> dataFromCsv = ReadCsvFile();
//if we dont get any data, lets return and sleep...
//if(dataFromCsv.Count < 1)
//{
// return;
//}
//lets save the data to the db
SaveCandidateData(dataFromCsv);
//now lets get the data, from the db, we are doing this because we will need the id of the students
List<Candidate> listFromDb = dbCtxt.Candidates.Where(c => c.Status == null).ToList();
List<CsvData> nonMatchedData = new List<CsvData>();
foreach(var l in listFromDb)
{
CsvData csvData = new CsvData();
csvData.Candidate = l;
csvData.Result = dbCtxt.Results.Where(c => c.CandidateNo.Trim() == l.CandidateNo.Trim()).ToList();
nonMatchedData.Add(csvData);
}
//loop through the data we have
foreach(var d in nonMatchedData)
{
//lets make the api call
var result = makeApiCall(d.Candidate.CandidateNo, Convert.ToInt32(d.Candidate.ExamDate), d.Candidate.ReferenceNo).Result;
if(result == null)
{
continue;
}
//lets convert the response to an object...
APIResults apiResults = JsonConvert.DeserializeObject<APIResults>(result);
//lets check the status of the result, 001 means its successful
if(apiResults.StatusCode != "001")
{
updateStatusOfCandidate(d.Candidate.CandidateNo, "NOT FOUND");
continue;
}
//lets do the compare it self
foreach(var s in apiResults.Result.SubjectCol)
{
//lets get the subject id, then use link to obtain the results...
int subId = getSubjectId(s.Subject);
if(subId == 0)
{
continue;
}
//since we have the subject id, lets check the data we have from the csv to be sure its there
var resultCsv = d.Result.Where(c => c.SubjectID == subId).FirstOrDefault();
//if the data is not there, we continue
if (resultCsv == null) continue;
//if the data exist, lets now match it with the data we have from the csv to be sure the correct grade is there
if (resultCsv.Grade.Trim().ToLower() != s.GradeScore.Trim().ToLower())
{
updateStatusOfResult(resultCsv.ResultID, "UNMATCHED");
//if the result do not match, lets now set the status of the result column to be unmatched...
}
else
{
updateStatusOfResult(resultCsv.ResultID, "MATCHED");
}
}
updateStatusOfCandidate(d.Candidate.CandidateNo, "COMPLETED");
}
}
So, if I understand correctly you want the status of any Candidate that is in the Result table (as identified by the CandidateNo) to have a Status of "Verified". And, any Candidate that is not in the Result table to have a Status of "Unverified"?
You can use a left join between Candidate and Result to find records that are in Candidate but not Result.
You can use an inner join between 'Candidate' and 'Result' to find records that are in both Candidate and Result.
I assume that dbCtxt is an Entity Framework DbContext, so for example...
private void SetUnverifiedCandidates()
{
var unverified = dbCtxt.Candidates
.GroupJoin(
dbCtxt.Results,
candidate => candidate.CandidateNo,
result => result.CandidateNo
(candidate, results) => new { candidate, results }
)
.SelectMany(
joined => joined.results.DefaultIfEmpty(),
(joined, result) => new { joined.candidate, result }
)
.Where(joined => joined.result == null)
.Select(joined => joined.candidate);
foreach (var candidate in unverified)
{
candidate.Status = "Unverified";
}
dbCtxt.SaveChanges();
}
private void SetVerifiedCandidates()
{
var verified = dbCtxt.Candidates
.Join(
dbCtxt.Results,
candidate => candidate.CandidateNo,
result => result.CandidateNo
(candidate, result) => candidate
);
foreach (var candidate in verified)
{
candidate.Status = "Verified";
}
dbCtxt.SaveChanges();
}
I have mission system in MVC. and I give a same mission or diffrent mission for a lot of users. and I show title, description, start date , finish date and who are/is in mission. These are showing view page in grid.mvc. but when I login I can see every mission. I dont want to every mission I just want to see only my mission. of course other users see their missions.
in my controller, I split names.
this is my Codes,
Controller:
TicketDbContext db = new TicketDbContext();
public ActionResult Index()
{
var result = db.Missions.OrderByDescending(x=>x.GivenDate).ToList();
string ad = string.Empty;
foreach (var item in result)
{
if (!string.IsNullOrEmpty(item.GivenUsers))
{
string[] Ids = item.GivenUsers.Split(',');
for (int i = 0; i < Ids.Length; i++)
{
int id = Convert.ToInt32(Ids[i]);
item.GivenUsers += db.Users.FirstOrDefault(x => x.Id == id).Name+ " " + db.Users.FirstOrDefault(x => x.Id == id).Surname+ ",";
}
}
}
return View(result);
}
ScreenShot of my Grid
firstly, I recommend that you keep the assigned users ids in separated table
This is can handle your case..
var currentUserId = "";// set current userId from Where are you holding (session,identity etc..)
var result = db.Missions.OrderByDescending(x=>x.GivenDate).ToList();
result = result.Where(x=> x.GivenUsers.Split(',').Contains(currentUserId));
foreach (var item in result)
{
if (!string.IsNullOrEmpty(item.GivenUsers))
{
int[] Ids = Array.ConvertAll(item.GivenUsers.Split(','),
delegate(string s) { return int.Parse(s); }) ;
string[] names = db.Users.Where(u => Ids.Contains(u.Id)).Select(u => u.Name+ " " + u.Surname).ToArray();
item.GivenUsers = string.Join(",",names);
}
}
I am able to filter the data with the following two parameters id1 and id2, and get accurate result of 10 records, from which have 9 with a price_type=cs and other with price-type=ms.
However, if I add price_type to the parameters id1 and id2 (id1=23456,567890&id2=6782345&price_type=ms), I get 3000 records instead of getting one record.
Am I missing something in the code. Any help would be very much appreciated.
var data = db.database_BWICs.AsQueryable();
var filteredData = new List<IQueryable<database_Data>>();
if (!string.IsNullOrEmpty(query.name))
{
var ids = query.name.Split(',');
foreach (string i in ids)
{
filteredData.Add(data.Where(c => c.Name != null && c.Name.Contains(i)));
}
}
if (!string.IsNullOrEmpty(query.id2))
{
var ids = query.id2.Split(',');
foreach (string i in ids)
{
filteredData.Add(data.Where(c => c.ID2!= null && c.ID2.Contains(i)));
}
}
if (!string.IsNullOrEmpty(query.id1))
{
var ids = query.id1.Split(',');
foreach (string i in ids)
{
filteredData.Add(data.Where(c => c.ID1!= null && c.ID1.Contains(i)));
}
}
if (query.price_type != null)
{
var ids = query.price_type.Split(',');
foreach (string i in ids)
{
filteredData.Add(data.Where(c => c.Type.Contains(i)));
}
}
if (filteredData.Count != 0)
{
data = filteredData.Aggregate(Queryable.Union);
}
Updated Code:
var data = db.database_BWICs.AsQueryable();
if (!string.IsNullOrEmpty(query.name))
{
var ids = query.name.Split(',');
data = data.Where(c => c.Name != null && ids.Contains(c.Name));
}
if (query.price_type != null)
{
var ids = query.price_type.Split(',');
data = data.Where(c => ids.Contains(c.Cover));
}
if (!String.IsNullOrEmpty(query.id1))
{
var ids = query.id1.Split(',');
data = data.Where(c => c.ID1!= null && ids.Contains(c.ID1));
}
Because you don't add filter to restrict, every filter adds datas to result.
It means you make OR between your filters, not AND.
And your usage of contains looks rather strange too : you're using String.Contains, while I would guess (maybe wrong) that you want to see if a value is in a list => Enumerable.Contains
You should rather go for something like this (withoud filteredData)
var data = db.database_BWICs.AsQueryable();
if (!string.IsNullOrEmpty(query.name))
{
var ids = query.name.Split(',');
data = data.Where(c => c.Name != null && ids.Contains(c.Name)));
}
//etc.
if (query.price_type != null)
{
var ids = query.price_type.Split(',');
data = data.Where(c => ids.Contains(c.Type));
}
EDIT
Well, if you wanna mix and or conditions, you could go for PredicateBuilder
Then your code should look like that (to be tested).
//manage the queries with OR clause first
var innerOr = Predicate.True<database_BWICs>();//or the real type of your entity
if (!String.IsNullOrEmpty(query.id1))
{
var ids = query.id1.Split(',');
innerOr = innerOr.Or(c => c.ID1!= null && ids.Contains(c.ID1));
}
if (!String.IsNullOrEmpty(query.id2))
{
var ids = query.id2.Split(',');
innerOr = innerOr.Or(c => c.ID2!= null && ids.Contains(c.ID2));
}
//now manage the queries with AND clauses
var innerAnd = Predicate.True<database_BWICs>();//or the real type of your entity
if (query.price_type != null)
{
var ids = query.price_type.Split(',');
innerAnd = innerAnd.And(c => ids.Contains(c.Type));
}
//etc.
innerAnd = innerAnd.And(innerOr);
var data = db.database_BWICs.AsQueryable().Where(innerAnd);
I have this query, it selects A LOT of user records from a table. This code block takes 16 seconds from my local/debug machine (more like 5 in production). Anything I do to make this more efficient doubles the amount of time the method takes to return the results. Examples of other things I've tried are below. I don't understand how selecting an anonymous type and having the extra middle section iterating through the anonymous type can possibly be faster than without.
This block takes 16 seconds:
List<BoAssetSecurityUser> userList = new List<BoAssetSecurityUser>();
using (var context = DataObjectFactory.CreateContext())
{
var query = from ui in context.User_Information
where (ui.AssetCustomerID == 1 &&
(ui.GlobalID != "1TPTEMPUSER" ||
ui.GlobalID == null))
select new { ui };
var result =
from q in query
select new
{
UserId = q.ui.UserID,
FirstName = q.ui.FirstName,
LastName = q.ui.LastName,
UserName = q.ui.Username,
Globalid = q.ui.GlobalID
};
foreach (var user in result)
{
BoAssetSecurityUser boAssetSecUser = new BoAssetSecurityUser();
boAssetSecUser.UserId = user.UserId;
boAssetSecUser.FirstName = user.FirstName;
boAssetSecUser.LastName = user.LastName;
boAssetSecUser.UserName = user.UserName;
boAssetSecUser.GlobalId = user.Globalid;
userList.Add(boAssetSecUser);
}
}
return userList;
This takes over 45 seconds to complete:
List<BoAssetSecurityUser> userList = new List<BoAssetSecurityUser>();
using (var context = DataObjectFactory.CreateContext())
{
var query = (from ui in context.User_Information
where (ui.AssetCustomerID == 1 &&
(ui.GlobalID != "1TPTEMPUSER" ||
ui.GlobalID == null))
select ui).ToList();
foreach (var user in query)
{
BoAssetSecurityUser boAssetSecUser = new BoAssetSecurityUser();
boAssetSecUser.UserId = user.UserID;
boAssetSecUser.FirstName = user.FirstName;
boAssetSecUser.LastName = user.LastName;
boAssetSecUser.UserName = user.Username;
boAssetSecUser.GlobalId = user.GlobalID;
userList.Add(boAssetSecUser);
}
}
return userList;
This example also takes over 45 seconds to complete:
List<BoAssetSecurityUser> userList = new List<BoAssetSecurityUser>();
using (var context = DataObjectFactory.CreateContext())
{
var query = from ui in context.User_Information
where (ui.AssetCustomerID == 1 &&
(ui.GlobalID != "1TPTEMPUSER" ||
ui.GlobalID == null))
select new { ui };
foreach (var user in query)
{
BoAssetSecurityUser boAssetSecUser = new BoAssetSecurityUser();
boAssetSecUser.UserId = user.ui.UserID;
boAssetSecUser.FirstName = user.ui.FirstName;
boAssetSecUser.LastName = user.ui.LastName;
boAssetSecUser.UserName = user.ui.Username;
boAssetSecUser.GlobalId = user.ui.GlobalID;
userList.Add(boAssetSecUser);
}
}
return userList;
This is most likely because ui's type has more properties than the 5 you're interested in. The new { ui } anonymous type is unnecessary; your first example is faster because you tell it before you iterate the list (and thus go to the DB) that you're only interested in those 5 fields. In the other examples, you iterate the list, thus pulling the whole ui objects, even though you only use 5 of its properties.
This code should only pull the 5 properties, and so be as fast as your first example, while being more concise:
List<BoAssetSecurityUser> userList = new List<BoAssetSecurityUser>();
using (var context = DataObjectFactory.CreateContext())
{
var query = from ui in context.User_Information
where (ui.AssetCustomerID == 1 && (ui.GlobalID != "1TPTEMPUSER" || ui.GlobalID == null))
select new
{
ui.UserID,
ui.FirstName,
ui.LastName,
ui.Username,
ui.GlobalID
};
foreach (var user in query)
{
BoAssetSecurityUser boAssetSecUser = new BoAssetSecurityUser();
boAssetSecUser.UserId = user.UserID;
boAssetSecUser.FirstName = user.FirstName;
boAssetSecUser.LastName = user.LastName;
boAssetSecUser.UserName = user.Username;
boAssetSecUser.GlobalId = user.GlobalID;
userList.Add(boAssetSecUser);
}
}
return userList;
It's more about the amount of data your bringing from the database. The first query selects only a few columns while the others bring all of them.
Do you have large columns on this table?
It is faster because you only fetch 5 properties per line with your anonymous type. I don't now how many fields you have in User_Information, but they're all fetched when you use .ToList() on your query, you probably get much more data than needed.