Develop Reference

xUnit System.InvalidOperationException : The source 'IQueryable' doesn't implement 'IAsyncEnumerable<ApplicationUser>' [duplicate]

I have a MVC project on ASP.NET Core, my problem is connected with IQueryable and asynchronous. I wrote the following method for search in IQueryable<T>:
private IQueryable<InternalOrderInfo> WhereSearchTokens(IQueryable<InternalOrderInfo> query, SearchToken[] searchTokens)
{
if (searchTokens.Length == 0)
{
return query;
}
var results = new List<InternalOrderInfo>();
foreach (var searchToken in searchTokens)
{
//search logic, intermediate results are being added to `results` using `AddRange()`
}
return results.Count != 0 ? results.Distinct().AsQueryable() : query;
}
I call this in method ExecuteAsync():
public async Task<GetAllInternalOrderInfoResponse> ExecuteAsync(GetAllInternalOrderInfoRequest request)
{
//rest of the code
if (searchTokens != null && searchTokens.Any())
{
allInternalOrderInfo = WhereSearchTokens(allInternalOrderInfo, searchTokens);
}
var orders = await allInternalOrderInfo.Skip(offset).Take(limit).ToArrayAsync();
//rest of the code
}
When I test this I get an InvalidOperationException on line where I call ToArrayAsync()
The source IQueryable doesn't implement IAsyncEnumerable. Only sources that implement IAsyncEnumerable can be used for Entity Framework asynchronous operations.
I had changed ToArrayAsync() to ToListAsync() but nothing have changed. I have searched this problem for a while, but resolved questions are connected mostly with DbContext and entity creating. EntityFramework is not installed for this project and it's better not to do it because of application architecture. Hope someone has any ideas what to do in my situation.

I found I had to do a bit more work to get things to work nicely:
namespace TestDoubles
{
using Microsoft.EntityFrameworkCore.Query.Internal;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Linq.Expressions;
using System.Threading;
using System.Threading.Tasks;
public static class AsyncQueryable
{
/// <summary>
/// Returns the input typed as IQueryable that can be queried asynchronously
/// </summary>
/// <typeparam name="TEntity">The item type</typeparam>
/// <param name="source">The input</param>
public static IQueryable<TEntity> AsAsyncQueryable<TEntity>(this IEnumerable<TEntity> source)
=> new AsyncQueryable<TEntity>(source ?? throw new ArgumentNullException(nameof(source)));
}
public class AsyncQueryable<TEntity> : EnumerableQuery<TEntity>, IAsyncEnumerable<TEntity>, IQueryable<TEntity>
{
public AsyncQueryable(IEnumerable<TEntity> enumerable) : base(enumerable) { }
public AsyncQueryable(Expression expression) : base(expression) { }
public IAsyncEnumerator<TEntity> GetEnumerator() => new AsyncEnumerator(this.AsEnumerable().GetEnumerator());
public IAsyncEnumerator<TEntity> GetAsyncEnumerator(CancellationToken cancellationToken = default) => new AsyncEnumerator(this.AsEnumerable().GetEnumerator());
IQueryProvider IQueryable.Provider => new AsyncQueryProvider(this);
class AsyncEnumerator : IAsyncEnumerator<TEntity>
{
private readonly IEnumerator<TEntity> inner;
public AsyncEnumerator(IEnumerator<TEntity> inner) => this.inner = inner;
public void Dispose() => inner.Dispose();
public TEntity Current => inner.Current;
public ValueTask<bool> MoveNextAsync() => new ValueTask<bool>(inner.MoveNext());
#pragma warning disable CS1998 // Nothing to await
public async ValueTask DisposeAsync() => inner.Dispose();
#pragma warning restore CS1998
}
class AsyncQueryProvider : IAsyncQueryProvider
{
private readonly IQueryProvider inner;
internal AsyncQueryProvider(IQueryProvider inner) => this.inner = inner;
public IQueryable CreateQuery(Expression expression) => new AsyncQueryable<TEntity>(expression);
public IQueryable<TElement> CreateQuery<TElement>(Expression expression) => new AsyncQueryable<TElement>(expression);
public object Execute(Expression expression) => inner.Execute(expression);
public TResult Execute<TResult>(Expression expression) => inner.Execute<TResult>(expression);
public IAsyncEnumerable<TResult> ExecuteAsync<TResult>(Expression expression) => new AsyncQueryable<TResult>(expression);
TResult IAsyncQueryProvider.ExecuteAsync<TResult>(Expression expression, CancellationToken cancellationToken) => Execute<TResult>(expression);
}
}
}
This enables me to write tests like this:
[TestCase("", 3, 5)]
[TestCase("100", 2, 4)]
public async Task GetOrderStatusCounts_ReturnsCorrectNumberOfRecords(string query, int expectedCount, int expectedStatusProductionCount)
{
// omitted CreateOrder helper function
const int productionStatus = 6;
const int firstOtherStatus = 5;
const int otherOtherStatus = 7;
var items = new[]
{
CreateOrder(1, "100000", firstOtherStatus, 1),
CreateOrder(2, "100000", firstOtherStatus, 4),
CreateOrder(3, "100000", productionStatus, 4),
CreateOrder(4, "100001", productionStatus, 4),
CreateOrder(5, "100100", productionStatus, 4),
CreateOrder(6, "200000", otherOtherStatus, 4),
CreateOrder(7, "200001", productionStatus, 4),
CreateOrder(8, "200100", productionStatus, 4)
}.AsAsyncQueryable(); // this is where the magic happens
var mocker = new AutoMocker();
// IRepository implementation is also generic and calls DBCntext
// for easier testing
mocker.GetMock<IRepository<Order>>()
.Setup(m => m.BaseQuery()
.Returns(items);
// the base query is extended in the system under test.
// that's the behavior I'm testing here
var sut = mocker.CreateInstance<OrderService>();
var counts = await sut.GetOrderStatusCountsAsync(4, query);
counts.Should().HaveCount(expectedCount);
counts[OrderStatus.Production].Should().Be(expectedStatusProductionCount);
}

I wrote an ICollection extension AsAsyncQueryable that I use in my tests
using System;
using System.Collections;
using System.Collections.Generic;
using System.Linq;
using System.Linq.Expressions;
using System.Threading;
using System.Threading.Tasks;
namespace Whatevaaaaaaaa
{
public static class ICollectionExtensions
{
public static IQueryable<T> AsAsyncQueryable<T>(this ICollection<T> source) =>
new AsyncQueryable<T>(source.AsQueryable());
}
internal class AsyncQueryable<T> : IAsyncEnumerable<T>, IQueryable<T>
{
private IQueryable<T> Source;
public AsyncQueryable(IQueryable<T> source)
{
Source = source;
}
public Type ElementType => typeof(T);
public Expression Expression => Source.Expression;
public IQueryProvider Provider => new AsyncQueryProvider<T>(Source.Provider);
public IAsyncEnumerator<T> GetAsyncEnumerator(CancellationToken cancellationToken = default)
{
return new AsyncEnumeratorWrapper<T>(Source.GetEnumerator());
}
public IEnumerator<T> GetEnumerator() => Source.GetEnumerator();
IEnumerator IEnumerable.GetEnumerator() => GetEnumerator();
}
internal class AsyncQueryProvider<T> : IQueryProvider
{
private readonly IQueryProvider Source;
public AsyncQueryProvider(IQueryProvider source)
{
Source = source;
}
public IQueryable CreateQuery(Expression expression) =>
Source.CreateQuery(expression);
public IQueryable<TElement> CreateQuery<TElement>(Expression expression) =>
new AsyncQueryable<TElement>(Source.CreateQuery<TElement>(expression));
public object Execute(Expression expression) => Execute<T>(expression);
public TResult Execute<TResult>(Expression expression) =>
Source.Execute<TResult>(expression);
}
internal class AsyncEnumeratorWrapper<T> : IAsyncEnumerator<T>
{
private readonly IEnumerator<T> Source;
public AsyncEnumeratorWrapper(IEnumerator<T> source)
{
Source = source;
}
public T Current => Source.Current;
public ValueTask DisposeAsync()
{
return new ValueTask(Task.CompletedTask);
}
public ValueTask<bool> MoveNextAsync()
{
return new ValueTask<bool>(Source.MoveNext());
}
}
}

If you are not going to change your design - you have several options:
1) Change AsQueryable to another method which returns IQueryable which also implements IDbAsyncEnumerable. For example you can extend EnumerableQuery (which is returned by AsQueryable):
public class AsyncEnumerableQuery<T> : EnumerableQuery<T>, IDbAsyncEnumerable<T> {
public AsyncEnumerableQuery(IEnumerable<T> enumerable) : base(enumerable) {
}
public AsyncEnumerableQuery(Expression expression) : base(expression) {
}
public IDbAsyncEnumerator<T> GetAsyncEnumerator() {
return new InMemoryDbAsyncEnumerator<T>(((IEnumerable<T>) this).GetEnumerator());
}
IDbAsyncEnumerator IDbAsyncEnumerable.GetAsyncEnumerator() {
return GetAsyncEnumerator();
}
private class InMemoryDbAsyncEnumerator<T> : IDbAsyncEnumerator<T> {
private readonly IEnumerator<T> _enumerator;
public InMemoryDbAsyncEnumerator(IEnumerator<T> enumerator) {
_enumerator = enumerator;
}
public void Dispose() {
}
public Task<bool> MoveNextAsync(CancellationToken cancellationToken) {
return Task.FromResult(_enumerator.MoveNext());
}
public T Current => _enumerator.Current;
object IDbAsyncEnumerator.Current => Current;
}
}
Then you change
results.Distinct().AsQueryable()
to
new AsyncEnumerableQuery<InternalOrderInfo>(results.Distinct())
And later, ToArrayAsync will not throw exception any more (obviously you can create your own extension method like AsQueryable).
2) Change ToArrayAsync part:
public static class EfExtensions {
public static Task<TSource[]> ToArrayAsyncSafe<TSource>(this IQueryable<TSource> source) {
if (source == null)
throw new ArgumentNullException(nameof(source));
if (!(source is IDbAsyncEnumerable<TSource>))
return Task.FromResult(source.ToArray());
return source.ToArrayAsync();
}
}
And use ToArrayAsyncSafe instead of ToArrayAsync, which will fallback to synchronous enumeration in case IQueryable is not IDbAsyncEnumerable. In your case this only happens when query is really in-memory list and not query, so async execution does not make sense anyway.

For EF Core:
public static class QueryableExtensions
{
public static IQueryable<T> AsAsyncQueryable<T>(this IEnumerable<T> input)
{
return new NotInDbSet<T>( input );
}
}
public class NotInDbSet< T > : IQueryable<T>, IAsyncEnumerable< T >, IEnumerable< T >, IEnumerable
{
private readonly List< T > _innerCollection;
public NotInDbSet( IEnumerable< T > innerCollection )
{
_innerCollection = innerCollection.ToList();
}
public IAsyncEnumerator< T > GetAsyncEnumerator( CancellationToken cancellationToken = new CancellationToken() )
{
return new AsyncEnumerator( GetEnumerator() );
}
public IEnumerator< T > GetEnumerator()
{
return _innerCollection.GetEnumerator();
}
IEnumerator IEnumerable.GetEnumerator()
{
return GetEnumerator();
}
public class AsyncEnumerator : IAsyncEnumerator< T >
{
private readonly IEnumerator< T > _enumerator;
public AsyncEnumerator( IEnumerator< T > enumerator )
{
_enumerator = enumerator;
}
public ValueTask DisposeAsync()
{
return new ValueTask();
}
public ValueTask< bool > MoveNextAsync()
{
return new ValueTask< bool >( _enumerator.MoveNext() );
}
public T Current => _enumerator.Current;
}
public Type ElementType => typeof( T );
public Expression Expression => Expression.Empty();
public IQueryProvider Provider => new EnumerableQuery<T>( Expression );
}

For EFCore Bit late to the party but to others looking to resolve this type of problem, one of possible solution is to change code to use Task.FromResult() method in this way:
var result= await allInternalOrderInfo.Skip(offset).Take(limit);
var orders = await Task.FromResult(result.ToArray());

The AsQueryable() will not transform the result list into an Entity Framework IQueryable. And as the error states, the IQueryable that are used with ToArrayAsync() should implement IAsyncEnumerable, which is not what AsQueryable will return.
You can read more about the uses of AsQueryable on enumerables here.

As noted by #Titian Cernicova-Dragomir the exception means that List<InternalOrderInfo> doesn't implement IAsyncEnumerable
But here is a logical/design error. If your method works with IQueryable and returns IQueryable it should work with it as with IQueryable and not as with IEnumarable that assumes that collection is in a memory of app. You really need to read more about the difference between IQueryable and IEnumarable and what you should return from the method. A good point to start is to read answers here and here
So, since you already fetched results from db in WhereSearchTokens method or even before, there is no reason to do asynchronous request to db which is would be done by ToArrayAsync and return IQueryable.
You have two options here:
1) If your collection of InternalOrderInfo is fetched from db into memory before WhereSearchTokens make your all actions in synchronous mode i.e call ToArray instead of ToArrayAsync, and return IEnumerable instead of Taks<IQueryable> from both WhereSearchTokens and ExecuteAsync.
2) If your collection of InternalOrderInfo is fetched inside WhereSearchTokens and you want to do the async request to db you need to call async EF API only somewhere in //search logic, intermediate results are being added to results using AddRange() and again return Taks<IEnumerable> istead of Taks<IQueryable> from WhereSearchTokens

ERROR Message: 
System.InvalidOperationException : The source 'IQueryable' doesn't implement 'IAsyncEnumerable'. Only sources that implement 'IAsyncEnumerable' can be used for Entity Framework asynchronous operations.
For my case the solution: when you are mocking your dbContext and pass data from your mockSet to your context change .Returns to .ReturnsDbSet
Exemple:
var mockContext = new Mock<IWebApiDbContext>(); mockContext.Setup(m => m.User).ReturnsDbSet(mockSet.Object);
Full Code Mock db:
var mockSet = new Mock<DbSet<User>>();
mockSet.As<IDbAsyncEnumerable<User>>()
.Setup(m => m.GetAsyncEnumerator())
.Returns(new TestDbAsyncEnumerator<User>(data.GetEnumerator()));
mockSet.As<IQueryable<User>>()
.Setup(m => m.Provider)
.Returns(new TestDbAsyncQueryProvider<User>(data.Provider));
mockSet.As<IQueryable<User>>().Setup(m => m.Expression).Returns(data.Expression);
mockSet.As<IQueryable<User>>().Setup(m => m.ElementType).Returns(data.ElementType);
mockSet.As<IQueryable<User>>().Setup(m => m.GetEnumerator()).Returns(() => data.GetEnumerator());
var mockContext = new Mock<IWebApiDbContext>();
mockContext.Setup(m => m.User).ReturnsDbSet(mockSet.Object);

It's better to implement collection with IAsyncEnumerable<T> and IQueryable<T> rather than create your own ToListAsync extensions.
You can't apply your extensions in libraries.
For EF Core 5 and above check this implementation and tests.
Short version:
public sealed class FixedQuery<T> : IAsyncEnumerable<T>, IQueryable<T>
{
public static readonly IQueryable<T> Empty = Create(ArraySegment<T>.Empty);
public static IQueryable<T> Create(params T[] items)
{
return Create((IEnumerable<T>)items);
}
public static IQueryable<T> Create(IEnumerable<T> items)
{
return new FixedQuery<T>(items ?? ArraySegment<T>.Empty).AsQueryable();
}
private readonly IQueryable<T> _items;
private FixedQuery(IEnumerable<T> items)
{
_items = (items ?? throw new ArgumentNullException(nameof(items))).AsQueryable();
}
#pragma warning disable CS1998
public async IAsyncEnumerator<T> GetAsyncEnumerator(CancellationToken cancellationToken = default)
#pragma warning restore CS1998
{
foreach (var item in _items)
{
yield return item;
}
}
public IEnumerator<T> GetEnumerator()
{
return _items.GetEnumerator();
}
IEnumerator IEnumerable.GetEnumerator()
{
return GetEnumerator();
}
public Type ElementType => _items.ElementType;
public Expression Expression => _items.Expression;
public IQueryProvider Provider => _items.Provider;
}

Wire up ExpressionVisitor with EF Core Include

I have an ExpressionVisitor which I add to EF Core's IQueryable<T>. Everything works fine except the Include methods. Probably because they enforce your IQueryable<T>.Provider to be an EntityQueryProvider.
Whenever I try to Include now it results in multiple queries which in turn results in the error "A second operation started on this context before a previous operation completed. Any instance members are not guaranteed to be thread safe.".
How can I wire up my ExpressionVisitor so it still works with EF Core's Include functionality?
My issue is similar to this one except for EF Core instead of EF.
I hook up my ExpressionVisitor by calling it on the DbSet:
return new Translator<TEntity>(
_dbSet
.AsNoTracking());
This is my Translator class:
public class Translator<T> : IOrderedQueryable<T>
{
private readonly Expression _expression;
private readonly TranslatorProvider<T> _provider;
public Translator(IQueryable source)
{
_expression = Expression.Constant(this);
_provider = new TranslatorProvider<T>(source);
}
public Translator(IQueryable source, Expression expression)
{
if (expression == null)
{
throw new ArgumentNullException(nameof(expression));
}
_expression = expression;
_provider = new TranslatorProvider<T>(source);
}
public IEnumerator<T> GetEnumerator()
{
return ((IEnumerable<T>)_provider.ExecuteEnumerable(_expression)).GetEnumerator();
}
IEnumerator IEnumerable.GetEnumerator()
{
return _provider.ExecuteEnumerable(_expression).GetEnumerator();
}
public Type ElementType => typeof(T);
public Expression Expression => _expression;
public IQueryProvider Provider => _provider;
}
And this is my TranslatorProvider<T> class (I've taken out the non-relevant Visit methods to shorten the post):
public class TranslatorProvider<T> : ExpressionVisitor, IQueryProvider
{
private readonly IQueryable _source;
public TranslatorProvider(IQueryable source)
{
if (source == null)
{
throw new ArgumentNullException(nameof(source));
}
_source = source;
}
public IQueryable<TElement> CreateQuery<TElement>(Expression expression)
{
if (expression == null)
{
throw new ArgumentNullException(nameof(expression));
}
return new Translator<TElement>(_source, expression);
}
public IQueryable CreateQuery(Expression expression)
{
if (expression == null)
{
throw new ArgumentNullException(nameof(expression));
}
var elementType = expression.Type.GetGenericArguments().First();
var result = (IQueryable) Activator.CreateInstance(typeof(Translator<>).MakeGenericType(elementType),
_source, expression);
return result;
}
public TResult Execute<TResult>(Expression expression)
{
if (expression == null)
{
throw new ArgumentNullException(nameof(expression));
}
var result = (this as IQueryProvider).Execute(expression);
return (TResult) result;
}
public object Execute(Expression expression)
{
if (expression == null)
{
throw new ArgumentNullException(nameof(expression));
}
var translated = Visit(expression);
return _source.Provider.Execute(translated);
}
internal IEnumerable ExecuteEnumerable(Expression expression)
{
if (expression == null)
{
throw new ArgumentNullException(nameof(expression));
}
var translated = Visit(expression);
return _source.Provider.CreateQuery(translated);
}
protected override Expression VisitConstant(ConstantExpression node)
{
if (node.Type == typeof(Translator<T>))
{
return _source.Expression;
}
else
{
return base.VisitConstant(node);
}
}
}

Update (EF Core 3.x):
The internal query pipeline infrastructure has changed. The new query expression preprocessing extension point is QueryTranslationPreprocessor class - Process method. Plugging it in requires replacing the IQueryTranslationPreprocessorFactory. e.g.
using System.Linq.Expressions;
namespace Microsoft.EntityFrameworkCore.Query
{
public class CustomQueryTranslationPreprocessor : RelationalQueryTranslationPreprocessor
{
public CustomQueryTranslationPreprocessor(QueryTranslationPreprocessorDependencies dependencies, RelationalQueryTranslationPreprocessorDependencies relationalDependencies, QueryCompilationContext queryCompilationContext)
: base(dependencies, relationalDependencies, queryCompilationContext) { }
public override Expression Process(Expression query) => base.Process(Preprocess(query));
private Expression Preprocess(Expression query)
{
// query = new YourExpressionVisitor().Visit(query);
return query;
}
}
public class CustomQueryTranslationPreprocessorFactory : IQueryTranslationPreprocessorFactory
{
public CustomQueryTranslationPreprocessorFactory(QueryTranslationPreprocessorDependencies dependencies, RelationalQueryTranslationPreprocessorDependencies relationalDependencies)
{
Dependencies = dependencies;
RelationalDependencies = relationalDependencies;
}
protected QueryTranslationPreprocessorDependencies Dependencies { get; }
protected RelationalQueryTranslationPreprocessorDependencies RelationalDependencies;
public QueryTranslationPreprocessor Create(QueryCompilationContext queryCompilationContext)
=> new CustomQueryTranslationPreprocessor(Dependencies, RelationalDependencies, queryCompilationContext);
}
}
and
optionsBuilder.ReplaceService<IQueryTranslationPreprocessorFactory, CustomQueryTranslationPreprocessorFactory>();
Original:
Apparently custom query providers don't fit in the current EF Core queryable pipeline, since several methods (Include, AsNoTracking etc.) require provider to be EntityQueryProvider.
At the time of writing (EF Core 2.1.2), the query translation process involves several services - IAsyncQueryProvider, IQueryCompiler, IQueryModelGenerator and more. All they are replaceable, but the easiest place for interception I see is the IQueryModelGenerator service - ParseQuery method.
So, forget about custom IQueryable / IQueryProvider implementation, use the following class and plug your expression visitor inside Preprocess method:
using Microsoft.EntityFrameworkCore.Internal;
using Microsoft.EntityFrameworkCore.Query.Internal;
using Remotion.Linq;
using Remotion.Linq.Parsing.ExpressionVisitors.TreeEvaluation;
class CustomQueryModelGenerator : QueryModelGenerator
{
public CustomQueryModelGenerator(INodeTypeProviderFactory nodeTypeProviderFactory, IEvaluatableExpressionFilter evaluatableExpressionFilter, ICurrentDbContext currentDbContext)
: base(nodeTypeProviderFactory, evaluatableExpressionFilter, currentDbContext)
{ }
public override QueryModel ParseQuery(Expression query) => base.ParseQuery(Preprocess(query));
private Expression Preprocess(Expression query)
{
// return new YourExpressionVisitor().Visit(query);
return query;
}
}
and replace the corresponding EF Core service inside your derived context OnConfiguring override:
optionsBuilder.ReplaceService<IQueryModelGenerator, CustomQueryModelGenerator>();
The drawback is that this is using EF Core "internal" stuff, so you should keep monitoring for changes in the future updates.

Implementing a custom QueryProvider with in-memory query

I'm trying to create a wrapper around QueryableBase and INhQueryProvider that would receive a collection in the constructor and query it in-memory instead of going to a database. This is so I can mock the behavior of NHibernate's ToFuture() and properly unit test my classes.
The problem is that I'm facing a stack overflow due to infinite recursion and I'm struggling to find the reason.
Here's my implementation:
public class NHibernateQueryableProxy<T> : QueryableBase<T>, IOrderedQueryable<T>
{
public NHibernateQueryableProxy(IQueryable<T> data) : base(new NhQueryProviderProxy<T>(data))
{
}
public NHibernateQueryableProxy(IQueryParser queryParser, IQueryExecutor executor) : base(queryParser, executor)
{
}
public NHibernateQueryableProxy(IQueryProvider provider) : base(provider)
{
}
public NHibernateQueryableProxy(IQueryProvider provider, Expression expression) : base(provider, expression)
{
}
public new IEnumerator<T> GetEnumerator()
{
return Provider.Execute<IEnumerable<T>>(Expression).GetEnumerator();
}
IEnumerator IEnumerable.GetEnumerator()
{
return GetEnumerator();
}
}
internal class NhQueryProviderProxy<T> : INhQueryProvider
{
private readonly IQueryProvider provider;
public NhQueryProviderProxy(IQueryable<T> data)
{
provider = data.AsQueryable().Provider;
}
public IQueryable CreateQuery(Expression expression)
{
return new NHibernateQueryableProxy<T>(this, expression);
}
public IQueryable<TElement> CreateQuery<TElement>(Expression expression)
{
return new NHibernateQueryableProxy<TElement>(this, expression);
}
public object Execute(Expression expression)
{
return provider.Execute(expression);
}
public TResult Execute<TResult>(Expression expression)
{
return provider.Execute<TResult>(expression);
}
public object ExecuteFuture(Expression expression)
{
return provider.Execute(expression);
}
public void SetResultTransformerAndAdditionalCriteria(IQuery query, NhLinqExpression nhExpression, IDictionary<string, Tuple<object, IType>> parameters)
{
throw new NotImplementedException();
}
}
Edit: I've kind of figured out the problem. One of the arguments to expression is my custom queryable. When this expression is executed by the provider, it causes an infinite call loop between CreateQuery and Execute. Is it possible to change all the references to my custom queryable to the queryable wrapped by this class?

After a while I decided to give it another try and I guess I've managed to mock it. I didn't test it with real case scenarios but I don't think many tweaks will be necessary. Most of this code is either taken from or based on this tutorial. There are some caveats related to IEnumerable when dealing with those queries.
We need to implement QueryableBase since NHibernate asserts the type when using ToFuture.
public class NHibernateQueryableProxy<T> : QueryableBase<T>
{
public NHibernateQueryableProxy(IQueryable<T> data) : base(new NhQueryProviderProxy<T>(data))
{
}
public NHibernateQueryableProxy(IQueryProvider provider, Expression expression) : base(provider, expression)
{
}
}
Now we need to mock a QueryProvider since that's what LINQ queries depend on and it needs to implement INhQueryProvider because ToFuture() also uses it.
public class NhQueryProviderProxy<T> : INhQueryProvider
{
private readonly IQueryable<T> _data;
public NhQueryProviderProxy(IQueryable<T> data)
{
_data = data;
}
// These two CreateQuery methods get called by LINQ extension methods to build up the query
// and by ToFuture to return a queried collection and allow us to apply more filters
public IQueryable CreateQuery(Expression expression)
{
Type elementType = TypeSystem.GetElementType(expression.Type);
return (IQueryable)Activator.CreateInstance(typeof(NHibernateQueryableProxy<>)
.MakeGenericType(elementType), new object[] { this, expression });
}
public IQueryable<TElement> CreateQuery<TElement>(Expression expression)
{
return new NHibernateQueryableProxy<TElement>(this, expression);
}
// Those two Execute methods are called by terminal methods like .ToList() and .ToArray()
public object Execute(Expression expression)
{
return ExecuteInMemoryQuery(expression, false);
}
public TResult Execute<TResult>(Expression expression)
{
bool IsEnumerable = typeof(TResult).Name == "IEnumerable`1";
return (TResult)ExecuteInMemoryQuery(expression, IsEnumerable);
}
public object ExecuteFuture(Expression expression)
{
// Here we need to return a NhQueryProviderProxy so we can add more queries
// to the queryable and use another ToFuture if desired
return CreateQuery(expression);
}
private object ExecuteInMemoryQuery(Expression expression, bool isEnumerable)
{
var newExpr = new ExpressionTreeModifier<T>(_data).Visit(expression);
if (isEnumerable)
{
return _data.Provider.CreateQuery(newExpr);
}
return _data.Provider.Execute(newExpr);
}
public void SetResultTransformerAndAdditionalCriteria(IQuery query, NhLinqExpression nhExpression, IDictionary<string, Tuple<object, IType>> parameters)
{
throw new NotImplementedException();
}
}
The expression tree visitor will change the type of the query for us:
internal class ExpressionTreeModifier<T> : ExpressionVisitor
{
private IQueryable<T> _queryableData;
internal ExpressionTreeModifier(IQueryable<T> queryableData)
{
_queryableData = queryableData;
}
protected override Expression VisitConstant(ConstantExpression c)
{
// Here the magic happens: the expression types are all NHibernateQueryableProxy,
// so we replace them by the correct ones
if (c.Type == typeof(NHibernateQueryableProxy<T>))
return Expression.Constant(_queryableData);
else
return c;
}
}
And we also need a helper (taken from the tutorial) to get the type being queried:
internal static class TypeSystem
{
internal static Type GetElementType(Type seqType)
{
Type ienum = FindIEnumerable(seqType);
if (ienum == null) return seqType;
return ienum.GetGenericArguments()[0];
}
private static Type FindIEnumerable(Type seqType)
{
if (seqType == null || seqType == typeof(string))
return null;
if (seqType.IsArray)
return typeof(IEnumerable<>).MakeGenericType(seqType.GetElementType());
if (seqType.IsGenericType)
{
foreach (Type arg in seqType.GetGenericArguments())
{
Type ienum = typeof(IEnumerable<>).MakeGenericType(arg);
if (ienum.IsAssignableFrom(seqType))
{
return ienum;
}
}
}
Type[] ifaces = seqType.GetInterfaces();
if (ifaces != null && ifaces.Length > 0)
{
foreach (Type iface in ifaces)
{
Type ienum = FindIEnumerable(iface);
if (ienum != null) return ienum;
}
}
if (seqType.BaseType != null && seqType.BaseType != typeof(object))
{
return FindIEnumerable(seqType.BaseType);
}
return null;
}
}
To test the above code, I ran the following snippet:
var arr = new NHibernateQueryableProxy<int>(Enumerable.Range(1, 10000).AsQueryable());
var fluentQuery = arr.Where(x => x > 1 && x < 4321443)
.Take(1000)
.Skip(3)
.Union(new[] { 4235, 24543, 52 })
.GroupBy(x => x.ToString().Length)
.ToFuture()
.ToList();
var linqQuery = (from n in arr
where n > 40 && n < 50
select n.ToString())
.ToFuture()
.ToList();
As I said, no complex scenarios were tested but I guess only a few tweaks will be necessary for real-world usages.

Make a where or any clause in a IRepository pattern with entity framework

I have taken over a project from a collegue, who uses the IRepository pattern. I have never used it before, so I have some problems understanding how to make a WHERE clause or an ANY clause.
Previously I had the following code, which uses the DataContext repository (the actual implementation, where I can use where clauses:
IQueryable<Office> officeList = repository.Offices;
if (specification.CountryId > 0)
{
officeList = repository.Offices.Where(c => c.CountryId == specification.CountryId);
}
if (specification.LetterSize != null)
{
officeList =
officeList.Where(
c => c.OfficeProducts.Any(d => d.OfficeProductDetail.Size == (int)specification.LetterSize));
}
return officeList.ToList();
I would like to understand how to get a code snippet like the one above to work using the IRepository pattern. I have tried to make an implementation of a WHERE/QUERY, but could not get it to work.
My question:
How do you implement a WHERE/ANY statement in practice, so I can do something like the code snippet above?
My IRepository:
public interface IRepository
{
T GetById<T>(long id) where T : class;
void Create<T>(T entity) where T : class;
void Update<T>(T entity) where T : class;
void SaveOrUpdate<T>(T entity) where T : class;
void Delete<T>(T entity) where T : class;
IList<T> FindAll<T>(params OrderBy[] orders) where T : class;
int Count<T>(Expression<Func<T, bool>> whereExpression) where T : class;
bool Exists<T>(Expression<Func<T, bool>> whereExpression) where T : class;
T FindFirst<T>(Expression<Func<T, bool>> whereExpression, params OrderBy[] orders) where T : class;
PaginatedResult<T> Find<T>(Expression<Func<T, bool>> whereExpression, int pageIndex, int pageSize, params OrderBy[] orders) where T : class;
void ExecuteNativeSQL(string sql);
}
The implementation:
public class EFRepository : IRepository
{
private IDBFactory databaseFactory;
private LetterAmazerContext dataContext;
public EFRepository(IDBFactory databaseFactory)
{
this.databaseFactory = databaseFactory;
}
protected LetterAmazerContext DataContext
{
get { return dataContext ?? (dataContext = databaseFactory.Get()); }
}
public T GetById<T>(long id) where T : class
{
IDbSet<T> dbset = DataContext.Set<T>();
return dbset.Find(id);
}
public void Create<T>(T entity) where T : class
{
IDbSet<T> dbset = DataContext.Set<T>();
dbset.Add(entity);
}
public void Update<T>(T entity) where T : class
{
dataContext.Entry(entity).State = EntityState.Modified;
}
public void SaveOrUpdate<T>(T entity) where T : class
{
throw new NotImplementedException();
}
public void Delete<T>(T entity) where T : class
{
IDbSet<T> dbset = DataContext.Set<T>();
dbset.Remove(entity);
}
public IList<T> FindAll<T>(params OrderBy[] orders) where T : class
{
IDbSet<T> dbset = DataContext.Set<T>();
var query = dbset.Where(t => true);
query = ApplyOrders<T>(query, orders);
return query.ToList<T>();
}
public int Count<T>(Expression<Func<T, bool>> whereExpression) where T : class
{
IDbSet<T> dbset = DataContext.Set<T>();
return dbset.Count<T>(whereExpression);
}
public bool Exists<T>(Expression<Func<T, bool>> whereExpression) where T : class
{
IDbSet<T> dbset = DataContext.Set<T>();
return dbset.Count<T>(whereExpression) != 0;
}
public T FindFirst<T>(Expression<Func<T, bool>> whereExpression, params OrderBy[] orders) where T : class
{
IDbSet<T> dbset = DataContext.Set<T>();
var query = dbset.Where(whereExpression);
query = ApplyOrders<T>(query, orders);
return query.SingleOrDefault<T>();
}
public PaginatedResult<T> Find<T>(Expression<Func<T, bool>> whereExpression, int pageIndex, int pageSize, params OrderBy[] orders) where T : class
{
IDbSet<T> dbset = DataContext.Set<T>();
PaginatedResult<T> results = new PaginatedResult<T>();
var query = dbset.AsExpandable().Where(whereExpression);
query = ApplyOrders<T>(query, orders);
results.Results = query.Skip<T>(pageIndex * pageSize).Take<T>(pageSize).ToList<T>();
results.TotalItems = dbset.AsExpandable().LongCount(whereExpression);
return results;
}
public void ExecuteNativeSQL(string sql)
{
DataContext.Database.ExecuteSqlCommand(sql);
}
private IQueryable<T> ApplyOrders<T>(IQueryable<T> query, params OrderBy[] orders)
{
if (orders == null || orders.Length == 0) return query;
foreach (var order in orders)
{
query = query.OrderBy(order.ToString());
}
return query;
}
}

Your repository is at the moment open for arbitrary expressions, including potentially expressions that cannot be evaluated by the implementation.
On one hand this is a potential risk of not being able to deliver an implementation that matches so open contract.
On the other hand, why don't you just then expose it a bit more:
public interface IRepository
{
...
IQueryable<T> Query<T>();
}
and
public class EFRepository : IRepository
{
...
public IQueryable<T> Query<T>()
{
return DataContrxt.Set<T>();
}
}
Note that if you decide to go that way, most of your specific query methods doesn't make sense anymore as such open generic queryable interface satisfies most needs.
Note also that the same concern applies there, if you have multiple implementations you can't possibly guarantee that the contract is satisfied in the same way. Also, your potential client can easily create a query the provider will reject. If you accept these issues, the proposed solution solves your issue as you can now query the repository for almost anything.
And the last note, if you don't plan to have multiple implementations, remove the repository layer completely. The EF context is a unit of work with repositories inside. Starting from version 6, the context can be mocked so unit testing is possible.

ObjectSet wrapper not working with linqToEntities subquery

for access control purposes in a intensive DB use system I had to implement an objectset wrapper, where the AC will be checked.
The main objective is make this change preserving the existing code for database access, that is implemented with linq to entities all over the classes (there is no centralized layer for database).
The ObjectSetWrapper created is like that:
public class ObjectSetWrapper<TEntity> : IQueryable<TEntity> where TEntity : EntityObject
{
private IQueryable<TEntity> QueryableModel;
private ObjectSet<TEntity> ObjectSet;
public ObjectSetWrapper(ObjectSet<TEntity> objectSetModels)
{
this.QueryableModel = objectSetModels;
this.ObjectSet = objectSetModels;
}
public ObjectQuery<TEntity> Include(string path)
{
return this.ObjectSet.Include(path);
}
public void DeleteObject(TEntity #object)
{
this.ObjectSet.DeleteObject(#object);
}
public void AddObject(TEntity #object)
{
this.ObjectSet.AddObject(#object);
}
public IEnumerator<TEntity> GetEnumerator()
{
return QueryableModel.GetEnumerator();
}
public Type ElementType
{
get { return typeof(TEntity); }
}
public System.Linq.Expressions.Expression Expression
{
get { return this.QueryableModel.Expression; }
}
public IQueryProvider Provider
{
get { return this.QueryableModel.Provider; }
}
public void Attach(TEntity entity)
{
this.ObjectSet.Attach(entity);
}
public void Detach(TEntity entity)
{
this.ObjectSet.Detach(entity);
}
IEnumerator IEnumerable.GetEnumerator()
{
return this.QueryableModel.GetEnumerator();
}
}
It's really simple and works for simple queries, like that:
//db.Product is ObjectSetWrapper<Product>
var query = (from item in db.Product where item.Quantity > 0 select new { item.Id, item.Name, item.Value });
var itensList = query.Take(10).ToList();
But when I have subqueries like that:
//db.Product is ObjectSetWrapper<Product>
var query = (from item in db.Product
select new
{
Id = item.Id,
Name = item.Name,
SalesQuantity = (from sale in db.Sale where sale.ProductId == item.Id select sale.Id).Count()
}).OrderByDescending(x => x.SalesQuantity);
var productsList = query.Take(10).ToList();
I get NotSupportedException, saying I can't create a constant value of my inner query entity type:
Unable to create a constant value of type 'MyNamespace.Model.Sale'.
Only primitive types or enumeration types are supported in this
context.
How can I get my queries working? I don't really need to make my wrapper an ObjectSet type, I just need to use it in queries.
Updated
I have changed my class signature. Now it's also implementing IObjectSet<>, but I'm getting the same NotSupportedException:
public class ObjectSetWrapper<TEntity> : IQueryable<TEntity>, IObjectSet<TEntity> where TEntity : EntityObject

EDIT:
The problem is that the following LINQ construction is translated into LINQ expression containing your custom class inside (ObjectSetWrapper).
var query = (from item in db.Product
select new
{
Id = item.Id,
Name = item.Name,
SalesQuantity = (from sale in db.Sale where sale.ProductId == item.Id select sale.Id).Count()
}).OrderByDescending(x => x.SalesQuantity);
LINQ to Entities tries to convert this expression into SQL statement, but it has no idea how to deal with the custom classes (as well as custom methods).
The solution in such cases is to replace IQueryProvider with the custom one, which should intercept the query execution and translate LINQ expression, containing custom classes/methods into valid LINQ to Entities expression (which operates with entities and object sets).
Expression conversion is performed using the class, derived from ExpressionVisitor, which performs expression tree traversal, replacing relevant nodes, to the nodes which can be accepted by LINQ to Entities
Part 1 - IQueryWrapper
// Query wrapper interface - holds and underlying query
interface IQueryWrapper
{
IQueryable UnderlyingQueryable { get; }
}
Part 2 - Abstract QueryWrapperBase (not generic)
abstract class QueryWrapperBase : IQueryProvider, IQueryWrapper
{
public IQueryable UnderlyingQueryable { get; private set; }
class ObjectWrapperReplacer : ExpressionVisitor
{
public override Expression Visit(Expression node)
{
if (node == null || !typeof(IQueryWrapper).IsAssignableFrom(node.Type)) return base.Visit(node);
var wrapper = EvaluateExpression<IQueryWrapper>(node);
return Expression.Constant(wrapper.UnderlyingQueryable);
}
public static Expression FixExpression(Expression expression)
{
var replacer = new ObjectWrapperReplacer();
return replacer.Visit(expression);
}
private T EvaluateExpression<T>(Expression expression)
{
if (expression is ConstantExpression) return (T)((ConstantExpression)expression).Value;
var lambda = Expression.Lambda(expression);
return (T)lambda.Compile().DynamicInvoke();
}
}
protected QueryWrapperBase(IQueryable underlyingQueryable)
{
UnderlyingQueryable = underlyingQueryable;
}
public abstract IQueryable<TElement> CreateQuery<TElement>(Expression expression);
public abstract IQueryable CreateQuery(Expression expression);
public TResult Execute<TResult>(Expression expression)
{
return (TResult)Execute(expression);
}
public object Execute(Expression expression)
{
expression = ObjectWrapperReplacer.FixExpression(expression);
return typeof(IQueryable).IsAssignableFrom(expression.Type)
? ExecuteQueryable(expression)
: ExecuteNonQueryable(expression);
}
protected object ExecuteNonQueryable(Expression expression)
{
return UnderlyingQueryable.Provider.Execute(expression);
}
protected IQueryable ExecuteQueryable(Expression expression)
{
return UnderlyingQueryable.Provider.CreateQuery(expression);
}
}
Part 3 - Generic QueryWrapper<TElement>
class QueryWrapper<TElement> : QueryWrapperBase, IOrderedQueryable<TElement>
{
private static readonly MethodInfo MethodCreateQueryDef = GetMethodDefinition(q => q.CreateQuery<object>(null));
public QueryWrapper(IQueryable<TElement> underlyingQueryable) : this(null, underlyingQueryable)
{
}
protected QueryWrapper(Expression expression, IQueryable underlyingQueryable) : base(underlyingQueryable)
{
Expression = expression ?? Expression.Constant(this);
}
public virtual IEnumerator<TElement> GetEnumerator()
{
return ((IEnumerable<TElement>)Execute<IEnumerable>(Expression)).GetEnumerator();
}
IEnumerator IEnumerable.GetEnumerator()
{
return GetEnumerator();
}
public Expression Expression { get; private set; }
public Type ElementType
{
get { return typeof(TElement); }
}
public IQueryProvider Provider
{
get { return this; }
}
public override IQueryable CreateQuery(Expression expression)
{
var expressionType = expression.Type;
var elementType = expressionType
.GetInterfaces()
.Single(t => t.IsGenericType && t.GetGenericTypeDefinition() == typeof(IEnumerable<>))
.GetGenericArguments()
.Single();
var createQueryMethod = MethodCreateQueryDef.MakeGenericMethod(elementType);
return (IQueryable)createQueryMethod.Invoke(this, new object[] { expression });
}
public override IQueryable<TNewElement> CreateQuery<TNewElement>(Expression expression)
{
return new QueryWrapper<TNewElement>(expression, UnderlyingQueryable);
}
private static MethodInfo GetMethodDefinition(Expression<Action<QueryWrapper<TElement>>> methodSelector)
{
var methodCallExpression = (MethodCallExpression)methodSelector.Body;
return methodCallExpression.Method.GetGenericMethodDefinition();
}
}
Part 4 - finally your ObjectSetWrapper
public class ObjectSetWrapper<TEntity> : IQueryable<TEntity>, IQueryWrapper where TEntity : class
{
private IQueryable<TEntity> QueryableModel;
private ObjectSet<TEntity> ObjectSet;
public ObjectSetWrapper(ObjectSet<TEntity> objectSetModels)
{
this.QueryableModel = new QueryWrapper<TEntity>(objectSetModels);
this.ObjectSet = objectSetModels;
}
public ObjectQuery<TEntity> Include(string path)
{
return this.ObjectSet.Include(path);
}
public void DeleteObject(TEntity #object)
{
this.ObjectSet.DeleteObject(#object);
}
public void AddObject(TEntity #object)
{
this.ObjectSet.AddObject(#object);
}
public IEnumerator<TEntity> GetEnumerator()
{
return QueryableModel.GetEnumerator();
}
public Type ElementType
{
get { return typeof(TEntity); }
}
public System.Linq.Expressions.Expression Expression
{
get { return this.QueryableModel.Expression; }
}
public IQueryProvider Provider
{
get { return this.QueryableModel.Provider; }
}
public void Attach(TEntity entity)
{
this.ObjectSet.Attach(entity);
}
public void Detach(TEntity entity)
{
this.ObjectSet.Detach(entity);
}
IEnumerator IEnumerable.GetEnumerator()
{
return this.QueryableModel.GetEnumerator();
}
IQueryable IQueryWrapper.UnderlyingQueryable
{
get { return this.ObjectSet; }
}
}

Your inner query fails because you are referencing another dataset when you should be traversing foreign keys:
SalesQuantity = item.Sales.Count()