Develop Reference

C# Bad Performance OData when using extension

I have a Web API for OData services. I have a lot of table with many relations. Here is some of the table:
MSADDRESSCOUNTRY
public partial class MSADDRESSCOUNTRY
{
[System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Usage","CA2214:DoNotCallOverridableMethodsInConstructors")]
public MSADDRESSCOUNTRY()
{
this.MSADDRESSPROVINCEs = new HashSet<MSADDRESSPROVINCE>();
}
public int ID { get; set; }
public string CODE { get; set; }
public string COUNTRYNAME { get; set; }
[System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Usage","CA2227:CollectionPropertiesShouldBeReadOnly")]
public virtual ICollection<MSADDRESSPROVINCE> MSADDRESSPROVINCEs { get; set; }
}
MSADDRESSPROVINCE
public partial class MSADDRESSPROVINCE
{
[System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Usage", "CA2214:DoNotCallOverridableMethodsInConstructors")]
public MSADDRESSPROVINCE()
{
this.MSADDRESSDISTRICTs = new HashSet<MSADDRESSDISTRICT>();
}
public int ID { get; set; }
public Nullable<int> COUNTRYID { get; set; }
public string PROVINCENAME { get; set; }
public virtual MSADDRESSCOUNTRY MSADDRESSCOUNTRY { get; set; }
[System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Usage","CA2227:CollectionPropertiesShouldBeReadOnly")]
public virtual ICollection<MSADDRESSDISTRICT> MSADDRESSDISTRICTs { get; set; }
}
MSADDRESSDISTRICT
public partial class MSADDRESSDISTRICT
{
[System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Usage", "CA2214:DoNotCallOverridableMethodsInConstructors")]
public MSADDRESSDISTRICT()
{
this.MSADDRESSSUBDISTRICTs = new HashSet<MSADDRESSSUBDISTRICT>();
}
public int ID { get; set; }
public Nullable<int> PROVINCEID { get; set; }
public string DISTRICTNAME { get; set; }
public virtual MSADDRESSPROVINCE MSADDRESSPROVINCE { get; set; }
[System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Usage", "CA2227:CollectionPropertiesShouldBeReadOnly")]
public virtual ICollection<MSADDRESSSUBDISTRICT> MSADDRESSSUBDISTRICTs { get; set; }
}
I create DTO object model for every table with the property is the same with Database object model.
I want the client can use $expand keyword to get child data and/or parent data.
For MSADDRESSCOUNTRY I need to write the code like this.
[EnableQuery(MaxExpansionDepth = 4)]
public IQueryable<MsAddressCountryObject> Get()
{
return db.MSADDRESSCOUNTRies.Select(c => new MsAddressCountryObject
{
ID = c.ID,
CODE = c.CODE,
COUNTRYNAME = c.COUNTRYNAME,
MSADDRESSPROVINCEs = c.MSADDRESSPROVINCEs.Select(data => new MsAddressProvinceObject()
{
ID = data.ID,
COUNTRYID = data.COUNTRYID,
PROVINCENAME = data.PROVINCENAME,
MSADDRESSCOUNTRY = new MsAddressCountryObject()
{
ID = data.MSADDRESSCOUNTRY.ID,
CODE = data.MSADDRESSCOUNTRY.CODE,
COUNTRYNAME = data.MSADDRESSCOUNTRY.COUNTRYNAME,
},
MSADDRESSDISTRICTs = data.MSADDRESSDISTRICTs.Select(dist => new MsAddressDistrictObject()
{
ID = dist.ID,
PROVINCEID = dist.PROVINCEID,
DISTRICTNAME = dist.DISTRICTNAME,
})
})
});
}
For MSADDRESSPROVINCE I need to write the code like this.
[EnableQuery(MaxExpansionDepth = 4)]
public IQueryable<MsAddressProvinceObject> Get()
{
return db.MSADDRESSPROVINCEs.Select(data => new MsAddressProvinceObject()
{
ID = data.ID,
COUNTRYID = data.COUNTRYID,
PROVINCENAME = data.PROVINCENAME,
MSADDRESSCOUNTRY = new MsAddressCountryObject()
{
ID = data.MSADDRESSCOUNTRY.ID,
CODE = data.MSADDRESSCOUNTRY.CODE,
COUNTRYNAME = data.MSADDRESSCOUNTRY.COUNTRYNAME,
},
MSADDRESSDISTRICTs = data.MSADDRESSDISTRICTs.Select(dist => new MsAddressDistrictObject()
{
ID = dist.ID,
PROVINCEID = dist.PROVINCEID,
DISTRICTNAME = dist.DISTRICTNAME
})
});
}
That code works fast. But if I add/change/remove column, I have to modify the controller manually, one by one for all controller. For example, if I want to add geological coordinate in MSADDRESSDISTRICT, I have to change the code in Country Controller, Province Controller and District Controller.
So I decide to create extension method like this.
public static MsAddressCountryObject ToDTO(this MSADDRESSCOUNTRY data)
{
return new MsAddressCountryObject()
{
ID = data.ID,
CODE = data.CODE,
COUNTRYNAME = data.COUNTRYNAME,
};
}
public static IQueryable<MsAddressCountryObject ToDTO(this IEnumerable<MSADDRESSCOUNTRY datas)
{
return datas.Select(country =
{
var obj = country?.ToDTO();
obj.MSADDRESSPROVINCEs = country.MSADDRESSPROVINCEs?.ToDTO();
return obj;
}).AsQueryable();
}
public static MsAddressProvinceObject ToDTO(this MSADDRESSPROVINCE data)
{
return new MsAddressProvinceObject()
{
ID = data.ID,
COUNTRYID = data.COUNTRYID,
PROVINCENAME = data.PROVINCENAME,
MSADDRESSCOUNTRY = data.MSADDRESSCOUNTRY?.ToDTO()
};
}
public static IQueryable<MsAddressProvinceObject ToDTO(this IEnumerable<MSADDRESSPROVINCE datas)
{
return datas.Select(province =
{
var obj = province?.ToDTO();
obj.MSADDRESSDISTRICTs = province.MSADDRESSDISTRICTs.ToDTO();
return obj;
}).AsQueryable();
}
public static MsAddressDistrictObject ToDTO(this MSADDRESSDISTRICT data)
{
return new MsAddressDistrictObject()
{
ID = data.ID,
PROVINCEID = data.PROVINCEID,
DISTRICTNAME = data.DISTRICTNAME,
MSADDRESSPROVINCE = data.MSADDRESSPROVINCE?.ToDTO()
};
}
public static IQueryable<MsAddressDistrictObject ToDTO(this IEnumerable<MSADDRESSDISTRICT datas)
{
return datas.Select(district =
{
var obj = district?.ToDTO();
obj.MSADDRESSSUBDISTRICTs = district.MSADDRESSSUBDISTRICTs?.ToDTO();
return obj;
}).AsQueryable();
}
And the controller just like this.
[EnableQuery(MaxExpansionDepth = 4)]
public IQueryable<MsAddressCountryObject Get()
{
return db.MSADDRESSCOUNTRies.ToDTO()
}
And that makes the performance really bad. I think the extension is making a lot of memory allocation or some thing that make the result not being delivered directly to the client.
My goal is to create the code easy to maintain, and the performance not drop significantly.
I have many relation in other table. I want the $expand works without write all parent/child Select statement manually and one by one.
I have try to not calling ToDTO() from all the extension method. The result is the performance is fast. But I lost all the relation or I need to write the parent/child Select statement for all method.
Any suggestion will help.
Thanks.

Create query by dynamically pass the GroupBy() and create new class in Select() using Expression tree

I`m having simple method which builds IQueryable and returns it.
public IQueryable<ClassDTO> ReportByNestedProperty()
{
IQueryable<Class> query = this.dbSet;
IQueryable<ClassDTO> groupedQuery =
from opportunity in query
group new
{
ItemGroup = opportunity.OpportunityStage.Name,
EstimatedRevenue = opportunity.EstimatedRevenue,
CostOfLead = opportunity.CostOfLead
}
by new
{
opportunity.OpportunityStage.Name,
opportunity.OpportunityStage.Id
}
into item
select new ClassDTO()
{
ItemGroup = string.IsNullOrEmpty(item.Key.Name) ? "[Not Assigned]" : item.Key.Name,
Count = item.Select(z => z.ItemGroup.Name).Count(), // int
Commission = item.Sum(z => z.EstimatedRevenue), // decimal
Cost = item.Sum(z => z.CostOfLead), // decimal?
};
return groupedQuery;
}
This is fine. The thing i need is to create method with same return type, but groupby by different prperties dynamically. So from the above code I want to have 3 dynamic parts which will be passed as params:
ItemGroup = opportunity.OpportunityStage.Name
and
by new
{
opportunity.OpportunityStage.Name,
opportunity.OpportunityStage.Id
}
So the new method should be like this
public IQueryable<ClassDTO> ReportByNestedProperty(string firstNestedGroupByProperty, string secondNestedGroupByProperty)
{
// TODO: ExpressionTree
}
And call it like this:
ReportByNestedProperty("OpportunityStage.Name","OpportunityStage.Id")
ReportByNestedProperty("OtherNestedProperty.Name","OtherNestedProperty.Id")
ReportByNestedProperty("OpportunityStage.Name","OpportunityStage.Price")
So the main thing is to create expressions with these two selects:
opportunity.OpportunityStage.Name,
opportunity.OpportunityStage.Id
I have tried toe create the select expressions, groupby, the creation of Anonomoys classes and the DTO Class but I just cant get it right.
EDIT:
Here are the classes involved:
public class ClassDTO
{
public ClassDTO() { }
[Key]
public string ItemGroup { get; set; }
public decimal Commission { get; set; }
public decimal? Cost { get; set; }
public int Count { get; set; }
}
Class obj is a pretty big one so i`m posting just part of it
public partial class Class
{
public Class() { }
[Key]
public Guid Id { get; set; }
public Guid? OpportunityStageId { get; set; }
[ForeignKey(nameof(OpportunityStageId))]
[InverseProperty(nameof(Entities.OpportunityStage.Class))]
public virtual OpportunityStage OpportunityStage { get; set; }
}
public partial class OpportunityStage
{
public OpportunityStage()
{
this.Classes = new HashSet<Class>();
}
[Key]
public Guid Id { get; set; }
public string Name { get; set; }
[InverseProperty(nameof(Class.OpportunityStage))]
public virtual ICollection<TruckingCompanyOpportunity> Classes{ get; set; }
}

I have simplified your Grouping query and introduced private class IdName which should replace anonymous class usage:
class IdName
{
public int Id { get; set; }
public string Name { get; set; } = null!;
}
static Expression MakePropPath(Expression objExpression, string path)
{
return path.Split('.').Aggregate(objExpression, Expression.PropertyOrField);
}
IQueryable<ClassDTO> ReportByNestedProperty(IQueryable<Class> query, string nameProperty, string idProperty)
{
// Let compiler to do half of the work
Expression<Func<Class, string, int, IdName>> keySelectorTemplate = (opportunity, name, id) =>
new IdName { Name = name, Id = id };
var param = keySelectorTemplate.Parameters[0];
// generating expressions from prop path
var nameExpr = MakePropPath(param, nameProperty);
var idExpr = MakePropPath(param, idProperty);
var body = keySelectorTemplate.Body;
// substitute parameters
body = ReplacingExpressionVisitor.Replace(keySelectorTemplate.Parameters[1], nameExpr, body);
body = ReplacingExpressionVisitor.Replace(keySelectorTemplate.Parameters[2], idExpr, body);
var keySelectorLambda = Expression.Lambda<Func<Class, IdName>>(body, param);
// finalize query
IQueryable<ClassDTO> groupedQuery = query
.GroupBy(keySelectorLambda)
.Select(item => new ClassDTO()
{
ItemGroup = string.IsNullOrEmpty(item.Key.Name) ? "[Not Assigned]" : item.Key.Name,
Count = item.Count(x => x.Name), // int
Commission = item.Sum(x => x.EstimatedRevenue), // decimal
Cost = item.Sum(x => x.CostOfLead), // decimal?
});
return groupedQuery;
}

Expression-tree to build sub-select results

I'm trying to build a sub-query by using expression-trees. In linq I would write something like:
var single = MyTable
.AsExpandable()
.Select(c => new
{
Childs = Enumerable.Select(
MyTable.VisibleChilds.Invoke(c, dbContext),
cc => Convert(cfg.ChildsConfig).Invoke(dbContext, cc))
});
where the Convert is building an expression like
p => new MyTableSelect {
Id = p.Id,
Name = p.Name
}
depending on the given values from the config (to only read needed data from database).
but I'm struggeling with the second parameter to be passed to the Select call as I need cc to be passed to the Convert-call.
I guess I need something like Expression.Lambda<Func<>> but I don't see it.
Expression.Lambda>(Expression.Invoke(Instance.Convert(cfg.ChildOrganizersFilterConfig), ContextParameter, theEntity));

I am not familiar with your use of Invoke but if you just want to run a 'Converter' in a fluent syntax for use in a Linq Expression I could show you an example of that. Say I have three POCO classes, one parent container, a child container, and a container I want to convert to.
public class POC
{
public int Id { get; set; }
public string Name { get; set; }
public POC(int id, string name)
{
Id = id;
Name = name;
}
}
public class ChildPOC
{
public int ParentId { get; set; }
public string FirstName { get; set; }
public string LastName { get; set; }
public ChildPOC(int parentId, string firstName, string lastName)
{
ParentId = parentId;
FirstName = firstName;
LastName = lastName;
}
}
public class ChildPOCAlter
{
public int ParentId { get; set; }
public string Name { get; set; }
public ChildPOCAlter(string first, string last, int parentId)
{
ParentId = parentId;
Name = $"{first} {last}";
}
}
I could write a converter method to take ChildPOC to ChildPOCAlter like so:
public static Converter<ChildPOC, ChildPOCAlter> ChildPOCOAlter()
{
return new Converter<ChildPOC, ChildPOCAlter>((x) => { return new ChildPOCAlter(x.FirstName, x.LastName, x.ParentId); });
}
I could then populate some data:
var someParents = new List<POC> { new POC(1, "A"), new POC(2, "B") };
var somechildren = new List<ChildPOC> { new ChildPOC(1, "Brett", "x"), new ChildPOC(1, "Emily", "X"), new ChildPOC(2, "John", "Y") };
And then I may want to take these relationships and apply a converter directly on it:
var relationships = someParents.Select(x => new
{
Id = x.Id,
Name = x.Name,
Children = somechildren.Where(y => y.ParentId == x.Id).ToList().ConvertAll(ChildPOCOAlter())
});

Linq from list objects into one object with a list property

Im trying to use Linq to select the property ProductColor from a List of MyObject into the property AllProductColors
public class MyObject
{
public string ImgUrl { get; set; }
public string ProductName { get; set; }
public string ProductColor { get; set; }
}
public class ObjectToSelectInto
{
public string ImgUrl { get; set; }
public string ProductName { get; set; }
public List<string> AllProductColors { get; set; }
}
//*** CREATING EXAMPLE ***///
List<MyObject> MyObjectList = new List<MyObject>();
ObjectToSelectInto destinationObject = new ObjectToSelectInto();
//I can do it like this, but then I
// would have to do this for every list item, not good!
destinationObject.AllProductColors =
MyObjectList.Select(x => x.ProductColor).toList();
//*** This fails ***///
destinationObject = MyObjectList.Select( x =>
new destinationObject {
AllProductColors = x.ProductColor.ToList(),
ImgUrl = x.ImgUrl.First().toString(),
ProductName = x.ProductName .First().toString()
}
I want it to be like this.
destinationObject has a list with elements where one color equals one element.

You just need to put your first query that you tried into second like this:
destinationObject = MyObjectList.Select(x =>
new ObjectToSelectInto()
{
AllProductColors = MyObjectList.Select(y => y.ProductName).ToList(),
ImgUrl = x.ImgUrl,
ProductName = x.ProductName
}).First();

RavenDb how do I reduce group values into collection in reduce final result?

I hope it's more clear what I want to do from the code than the title. Basically I am grouping by 2 fields and want to reduce the results into a collection all the ProductKey's constructed in the Map phase.
public class BlockResult
{
public Client.Names ClientName;
public string Block;
public IEnumerable<ProductKey> ProductKeys;
}
public Block()
{
Map = products =>
from product in products
where product.Details.Block != null
select new
{
product.ClientName,
product.Details.Block,
ProductKeys = new List<ProductKey>(new ProductKey[]{
new ProductKey{
Id = product.Id,
Url = product.Url
}
})
};
Reduce = results =>
from result in results
group result by new {result.ClientName, result.Block} into g
select new BlockResult
{
ClientName = g.Key.ClientName,
Block = g.Key.Block,
ProductKeys = g.SelectMany(x=> x.ProductKeys)
};
}
I get some weird System.InvalidOperationException and a source code dump where basically it is trying to initialize the list with an int (?).
If I try replacing the ProductKey with just IEnumerable ProductIds (and make appropriate changes in the code). Then the code runs but I don't get any results in the reduce.

You probably don't want to do this. Are you really going to need to query in this manner? If you know the context, then you should probably just do this:
var q = session.Query<Product>()
.Where(x => x.ClientName == "Joe" && x.Details.Block == "A");
But, to answer your original question, the following index will work:
public class Products_GroupedByClientNameAndBlock : AbstractIndexCreationTask<Product, Products_GroupedByClientNameAndBlock.Result>
{
public class Result
{
public string ClientName { get; set; }
public string Block { get; set; }
public IList<ProductKey> ProductKeys { get; set; }
}
public class ProductKey
{
public string Id { get; set; }
public string Url { get; set; }
}
public Products_GroupedByClientNameAndBlock()
{
Map = products =>
from product in products
where product.Details.Block != null
select new {
product.ClientName,
product.Details.Block,
ProductKeys = new[] { new { product.Id, product.Url } }
};
Reduce = results =>
from result in results
group result by new { result.ClientName, result.Block }
into g
select new {
g.Key.ClientName,
g.Key.Block,
ProductKeys = g.SelectMany(x => x.ProductKeys)
};
}
}

When replicating I get the same InvalidOperationException, stating that it doesn't understand the index definition (stack trace omitted for brevity).
Url: "/indexes/Keys/ByNameAndBlock"
System.InvalidOperationException: Could not understand query:
I'm still not entirely sure what you're attempting here, so this may not be quite what you're after, but I managed to get the following working. In short, Map/Reduce deals in anonymous objects, so strongly typing to your custom types makes no sense to Raven.
public class Keys_ByNameAndBlock : AbstractIndexCreationTask<Product, BlockResult>
{
public Keys_ByNameAndBlock()
{
Map = products =>
from product in products
where product.Block != null
select new
{
product.Name,
product.Block,
ProductIds = product.ProductKeys.Select(x => x.Id)
};
Reduce = results =>
from result in results
group result by new {result.Name, result.Block}
into g
select new
{
g.Key.Name,
g.Key.Block,
ProductIds = g.SelectMany(x => x.ProductIds)
};
}
}
public class Product
{
public Product()
{
ProductKeys = new List<ProductKey>();
}
public int ProductId { get; set; }
public string Url { get; set; }
public string Name { get; set; }
public string Block { get; set; }
public IEnumerable<ProductKey> ProductKeys { get; set; }
}
public class ProductKey
{
public int Id { get; set; }
public string Url { get; set; }
}
public class BlockResult
{
public string Name { get; set; }
public string Block { get; set; }
public int[] ProductIds { get; set; }
}