My masters thesis is about discovering bad database design by analyzing metadata and the data stored. We do this by extracting a metadata model from a given DBMS and then running a set of rules on this metadata.
To extend this process with data analysis, we need to allow rules to query the database directly, but we must retain DBMS independence, such that queries can be applied to PostgreSQL, MSSQL and MySQL.
We have discussed a sort of functional construction of queries such as:
new Query(new Select(columnID), new From(tableID), new Where(new Equality(columnID1, columnID2)))
And then using a DBMS-specific serializer.
Another approach is to let rules handle it all by themselves:
public Query QueryDatabase(DBMS dbms)
{
if (dbms == PostgreSQL) { return "select count(1) from Users"}
if (dbms == MSSQL) {return ....}
}
Are we missing something? Does all this in fact exist in a nice library somewhere? And yes, we have looked at Entity frameworks, but they seem to rely on a statically types model of the database, which for obvious reasons cannot be created.
I should mention that we maintain an extensible rule architecture, allowing end users to implement their own rules.
To clarify what we want to achieve, look at the following query (mssql), it needs two parameters, the name of the table (#table) and the name of the column (#column):
DECLARE #TotalCount FLOAT;
SELECT #TotalCount = COUNT(1) FROM [#table];
SELECT SUM(pcount * LOG10(#TotalCount / pcount)) / (LOG10(2) * #TotalCount)
FROM (SELECT (Count([#column])) as pcount
FROM [#table]
GROUP BY [#column]) as exp1
The query measures the amount of information stored in a given attribute, by estimating the entropy. It needs to access all rows in the table. To avoid extracting all rows from the database and transferring them over a slow network connection it is better to express them in SQL an only transfer a single number.
NOTE: We DO have all the metadata we need. This question is only for accessing data!
I was not very sure of whether to add this to my already long question, edit an existing answer or what todo. Please feel free to advise. ;)
Building on mrnye answer:
new Query()
.Variable(varname => FLOAT)
.Set(varname => new Query().Count(1).From(table) )
.Select(new Aggregate().Sum(varname => "pcount * LOG10(varname / pcount)"))
.From(
new Query()
.Select(pcount => new Aggregate().Count(column)
.From(table)
.GroupBy(column)
)
Syntax errors and misuse of lambda statements aside, i played with the idea of using some extension methods for building queries. It does seem as a fairly complex approach. How would you think about such an approach?
Building on the LINQ answer:
let totalCount = Table.Count
from uv un from r in Table
group r by r["attr"]
select r.Count
select r.Count * Log2((totalCount / r.Count))
Seems fairly nice, but a helluva lot to implement...
You could achieve the same by implementing a custom LINQ provider infrastructure. The queries are generic, but the AST tree visitors that generate the SQL queries can be made pluggable. You can even mock a database using a in memory data store and translating your custom LINQ query to a LINQ to objects query!
You would need to create a provider that would know how to extract the column name from the object's indexer. Here is a basic framework that you can extend:
// Runs in LinqPad!
public class TableQueryObject
{
private readonly Dictionary<string, object> _data = new Dictionary<string, object>();
public string TableName { get; set; }
public object this[string column]
{
get { return _data.ContainsKey(column) ? _data[column] : null; }
set { if (_data.ContainsKey(column)) _data[column] = value; else _data.Add(column, value); }
}
}
public interface ITableQuery : IEnumerable<TableQueryObject>
{
string TableName { get; }
string ConnectionString { get; }
Expression Expression { get; }
ITableQueryProvider Provider { get; }
}
public interface ITableQueryProvider
{
ITableQuery Query { get; }
IEnumerable<TableQueryObject> Execute(Expression expression);
}
public interface ITableQueryFactory
{
ITableQuery Query(string tableName);
}
public static class ExtensionMethods
{
class TableQueryContext : ITableQuery
{
private readonly ITableQueryProvider _queryProvider;
private readonly Expression _expression;
public TableQueryContext(ITableQueryProvider queryProvider, Expression expression)
{
_queryProvider = queryProvider;
_expression = expression;
}
public string TableName { get { return _queryProvider.Query.TableName; } }
public string ConnectionString { get { return _queryProvider.Query.ConnectionString; } }
public Expression Expression { get { return _expression; } }
public ITableQueryProvider Provider { get { return _queryProvider; } }
public IEnumerator<TableQueryObject> GetEnumerator() { return Provider.Execute(Expression).GetEnumerator(); }
IEnumerator IEnumerable.GetEnumerator() { return GetEnumerator(); }
}
public static MethodInfo MakeGeneric(MethodBase method, params Type[] parameters)
{
return ((MethodInfo)method).MakeGenericMethod(parameters);
}
public static Expression StaticCall(MethodInfo method, params Expression[] expressions)
{
return Expression.Call(null, method, expressions);
}
public static ITableQuery CreateQuery(this ITableQueryProvider source, Expression expression)
{
return new TableQueryContext(source, expression);
}
public static IEnumerable<TableQueryObject> Select<TSource>(this ITableQuery source, Expression<Func<TSource, TableQueryObject>> selector)
{
return source.Provider.CreateQuery(StaticCall(MakeGeneric(MethodBase.GetCurrentMethod(), typeof(TSource)), source.Expression, Expression.Quote(selector)));
}
public static ITableQuery Where(this ITableQuery source, Expression<Func<TableQueryObject, bool>> predicate)
{
return source.Provider.CreateQuery(StaticCall((MethodInfo)MethodBase.GetCurrentMethod(), source.Expression, Expression.Quote(predicate)));
}
}
class SqlTableQueryFactory : ITableQueryFactory
{
class SqlTableQuery : ITableQuery
{
private readonly string _tableName;
private readonly string _connectionString;
private readonly ITableQueryProvider _provider;
private readonly Expression _expression;
public SqlTableQuery(string tableName, string connectionString)
{
_connectionString = connectionString;
_tableName = tableName;
_provider = new SqlTableQueryProvider(this);
_expression = Expression.Constant(this);
}
public IEnumerator<TableQueryObject> GetEnumerator() { return Provider.Execute(Expression).GetEnumerator(); }
IEnumerator IEnumerable.GetEnumerator() { return GetEnumerator(); }
public string TableName { get { return _tableName; } }
public string ConnectionString { get { return _connectionString; } }
public Expression Expression { get { return _expression; } }
public ITableQueryProvider Provider { get { return _provider; } }
}
class SqlTableQueryProvider : ITableQueryProvider
{
private readonly ITableQuery _query;
public ITableQuery Query { get { return _query; } }
public SqlTableQueryProvider(ITableQuery query) { _query = query; }
public IEnumerable<TableQueryObject> Execute(Expression expression)
{
//var connecitonString = _query.ConnectionString;
//var tableName = _query.TableName;
// TODO visit expression AST (generate any sql dialect you want) and execute resulting sql
// NOTE of course the query can be easily parameterized!
// NOTE here the fun begins, just return some dummy data for now :)
for (int i = 0; i < 100; i++)
{
var obj = new TableQueryObject();
obj["a"] = i;
obj["b"] = "blah " + i;
yield return obj;
}
}
}
private readonly string _connectionString;
public SqlTableQueryFactory(string connectionString) { _connectionString = connectionString; }
public ITableQuery Query(string tableName)
{
return new SqlTableQuery(tableName, _connectionString);
}
}
static void Main()
{
ITableQueryFactory database = new SqlTableQueryFactory("SomeConnectionString");
var result = from row in database.Query("myTbl")
where row["someColumn"] == "1" && row["otherColumn"] == "2"
where row["thirdColumn"] == "2" && row["otherColumn"] == "4"
select row["a"]; // NOTE select executes as linq to objects! FTW
foreach(var a in result)
{
Console.WriteLine(a);
}
}
I think the LINQ route is the way to go, but for fun I tried to think of a solution. It needs some work, but the general idea is to have the query interface fluent and hide the implementation logic behind interfaces. Just throwing it out there as food for thought...
public interface IDBImplementation
{
public void ProcessQuery(Select query);
}
public class SqlServerImplementation : IDBImplementation
{
public void ProcessQuery(Select query)
{
string sqlQuery = "SELECT " + String.Join(", ", query.Columns)
+ " FROM " + query.TableName + " WHERE " + String.Join(" AND ", query.Conditions);
// execute query...
}
}
public class Select
{
public Select(params string[] columns)
{
Columns = columns;
}
public string[] Columns { get; set; }
public string TableName { get; set; }
public string[] Conditions { get; set; }
}
public static class Extensions
{
public static Select From(this Select select, string tableName)
{
select.TableName = tableName;
return select;
}
public static Select Where(this Select select, params string[] conditions)
{
select.Conditions = conditions;
return select;
}
}
public static class Main
{
public static void Example()
{
IDBImplementation database = new SqlServerImplementation();
var query = new Select("a", "b", "c").From("test").Where("c>5", "b<10");
database.ProcessQuery(query);
}
}
The most DBMS-independent way to retrieve information about a database is iNFORMATION_SCHEMA. See MySQL, SQL Server, PostgreSQL.
I'm kind of curious what type of rules you're thinking of. :)
What about NHibernate? http://community.jboss.org/wiki/NHibernateforNET
Related
I was trying to implement a wrapper in C# for SQL Server.
The normal workflow without wrapper is fetching the data into a datatable using direct SQL query and then mapping the columns by names into entities.
But as a wrapper is better to accept a mapping function which describes which column maps to which fields of an enumerable.
So, something like this :
public class UserInfo
{
public string FirstName{ get; set; }
public string LastName{ get; set; }
}
enumerableList = dbManager.Execute("** sql query **", /* some method to specify mapping */);
The enumerable will then contain the result from the database, mapped by the execute method. But I am unsure how to specify the mapping?
Even if I do then how to deal with the different data types for each column in the mapping?
If I correct understand, you want something like this:
public static List<T> ReadRows<T>(this SqlHelper sql, string query, SqlParameter[]
parameters, Func<SqlDataReader, T> projection)
{
var command = GetSqlCommand(query, CommandType.StoredProcedure, parameters);
return sql.ExecuteReader(command, reader => reader.Select(projection).ToList());
}
And use like:
var members = _unitOfWork.SqlHelper.ReadRows("spGetMembersByUserCompanies", parameters, _memberProjection);
readonly Func<SqlDataReader, MemberVm> _memberProjection = (r) => new MemberVm
{
InvitationId = r.Get<int?>("InvitationId"),
UserName = r.Get<string>("UserName"),
RoleName = r.Get<string>("RoleName"),
InvitationStatus = (InvitationStatus)r.Get<int>("InvitationStatus"),
LogoUrl = r.Get<string>("LogoUrl")
};
It is a piece of my code. I hope it is start to resolve your problem.
Implementing such a wrapper from bare bones is not that easy. But it is possible. There is an ADO wrapper library in Github : ADOWrapper
The implementation is pretty straightforward.
Short Answer
How to specify mapping between columns? - Use Func
How to deal with the different data types? You can write an extension
Long Answer
Make a generic method that takes as input the query and a Func Delegate (and optional third parameter to pass query parameters as dictionary)
public ICollection<T> Execute<T>(string query, Func<IDataReader, T> map, IDictionary<string, object> parameters = null)
{
ICollection<T> collection = new List<T>();
using (SqlConnection connection = CreateConnection())
{
connection.Open();
using (SqlCommand command = CreateCommand(connection, query, parameters))
{
using (IDataReader reader = await command.ExecuteReader())
{
while(reader.Read())
{
collection.Add(map.Invoke(reader));
}
}
}
connection.Close();
}
return collection;
}
Implementation of AddParameter AND CreateCommand:
private void AddParameter(IDbCommand command, string parameter, object value)
{
IDbDataParameter param = command.CreateParameter();
param.ParameterName = parameter;
param.Value = value;
command.Parameters.Add(param);
}
private SqlCommand CreateCommand(SqlConnection connection, string query,
IDictionary<string, object> parameters = null)
{
SqlCommand command = connection.CreateCommand();
command.CommandText = query;
if(parameters != null && parameters.Count > 0)
{
foreach(KeyValuePair<string, object> parameter in parameters)
{
AddParameter(command, parameter.Key, parameter.Value);
}
}
return command;
}
You can call the method like this :
public class UserInfo
{
public string FirstName{ get; set; }
public string LastName{ get; set; }
}
var enumerableList = manager.Execute("** query **",
(reader) =>
{
return new UserInfo()
{
FirstName = reader.Get<string>("FirstName"),
LastName = reader.Get<string>("LastName "),
};
})
The Get method makes it easy to manage different data types being fetched from column. But it is not an inbuilt method. So you need to write an extension for Data Reader:
public static class DataReaderExtension
{
public static T Get<T>(this IDataReader reader, string column) where T : IComparable
{
try
{
int index = reader.GetOrdinal(column);
if (!reader.IsDBNull(index))
{
return (T)reader[index];
}
}
catch (IndexOutOfRangeException) { throw new Exception($"Column, '{column}' not found."); }
return default(T);
}
public static IEnumerable<string> GetColumns(this IDataReader reader)
{
IEnumerable<string> columns = new List<string>();
if (reader != null && reader.FieldCount > 0)
{
columns = Enumerable.Range(0, reader.FieldCount)
.Select(index => reader.GetName(index))
.ToList();
}
return columns;
}
}
I have an abstract class like this:
public abstract class Records
{
public string Type;
public string Source;
public int Value;
protected Records(string type, string source, int value)
{
Type = type;
Source = source;
Value = value;
}
}
I would like to create many classes inheriting this class, and filling their Type field with a value coming from a static class like this:
public static class ContentTypesString
{
public static string DocumentNew { get { return "Document - New this Month"; }}
public static string HeadlinesNew { get { return "Headlines - New this Month"; }}
etc...
}
I would like to be able to create those child classes without having a test "if foo == "document" then type = ContentTypesString.DocumentNew" or an equivalent switch case (I really have a lot of cases)
Is there a design pattern that suits my needs?
EDIT : As several people pointed out, i should show how i create my instances.
private delegate SPListItemCollection Query(SPWeb web, DateTime startDate, DateTime endDate);
private readonly Query _queries;
#region Constructors
public QueryHandler(SPWeb web, DateTime startTimeSelectedDate, DateTime endTimeSelectedDate)
{
if (web == null) throw new ArgumentNullException("web");
_web = web;
_startTimeSelectedDate = startTimeSelectedDate;
_endTimeSelectedDate = endTimeSelectedDate;
RecordsList = new List<Records>();
// Query Invocation List
_queries = NumberPagePerMonthQuery.PreparedQuery;
_queries += NumberDocumentsPerMonthQuery.PreparedQuery;
_queries += NumberHeadlinesPerMonthQuery.PreparedQuery;
_queries += NumberLeaderboxPerMonthQuery.PreparedQuery;
_queries += NumberNewsPerMonthQuery.PreparedQuery;
_queries += NumberPagesModifiedPerMonthQuery.PreparedQuery;
_queries += NumberPicturesPerMonthQuery.PreparedQuery;
_queries += NumberTeasingPerMonthQuery.PreparedQuery;
}
#endregion Constructors
#region Public Methods
// what about NullReferenceException ? C#6 : item?.Foreach(item => {}); ?
/*** NO C#6 compiler in VS2012... ***/
public void Queries()
{
foreach (var del in _queries.GetInvocationList())
{
var queryresult =
(SPListItemCollection) del.DynamicInvoke(_web, _startTimeSelectedDate, _endTimeSelectedDate);
RecordsList.Add(new Records(del.Method.Name, _web.Title, queryresult.Count));
}
}
EDIT² :
The solution i chose
public List<IQuery> QueryList { get; } // no delegate anymore, and static classes became implementations of IQuery interface.
#region Constructors
public QueryHandler(SPWeb web, DateTime startTimeSelectedDate, DateTime endTimeSelectedDate)
{
if (web == null) throw new ArgumentNullException("web");
_web = web;
_startTimeSelectedDate = startTimeSelectedDate;
_endTimeSelectedDate = endTimeSelectedDate;
RecordsList = new List<Records>();
QueryList = new List<IQuery>
{
new NumberDocumentsPerMonthQuery(),
new NumberHeadlinesPerMonthQuery(),
new NumberLeaderboxPerMonthQuery(),
new NumberNewsPerMonthQuery(),
new NumberPagePerMonthQuery(),
new NumberPagesModifiedPerMonthQuery(),
new NumberPicturesPerMonthQuery(),
new NumberTeasingPerMonthQuery()
};
}
#endregion Constructors
#region Public Methods
// what about NullReferenceException ? C#6 : item?.Foreach(item => {}); ?
/*** NO C#6 compiler in VS2012... ***/
public void Queries()
{
foreach (var query in QueryList)
{
var queryresult = query.PreparedQuery(_web, _startTimeSelectedDate, _endTimeSelectedDate);
RecordsList.Add(query.CreateRecord(_web.Title, queryresult.Count));
}
}
Record class follow the implementation suggested by #dbraillon
Implementation of IQuery interface were added the method :
public Records CreateRecord(string source, int value)
{
return new ModifiedPagesPerMonthRecord(source, value); //or another child of Record class.
}
And voilà. Thank you all for the help.
You want to make collection of records, by string code of object type, and parameters.
One of many way to do it - use builder.
Firstly we need to configurate builder:
var builder = new RecordBuilder()
.RegisterBuilder("document", (source, value) => new Document(source, value))
.RegisterBuilder("headlines", (source, value) => new Headlines(source, value));
here we specify how to build record with code "document" and "headlines".
To build a record call:
builder.Build("document", "source", 1);
Builder code can by something like this
(here we look if we know how to build record of the passed type and make it):
public class RecordBuilder
{
public Records Build(string code, string source, int value)
{
Func<string, int, Records> buildAction;
if (recordBuilders.TryGetValue(code, out buildAction))
{
return buildAction(source, value);
}
return null;
}
public RecordBuilder RegisterBuilder(string code, Func<string, int, Records> buildAction)
{
recordBuilders.Add(code, buildAction);
return this;
}
private Dictionary<string, Func<string, int, Records>> recordBuilders = new Dictionary<string, Func<string, int, Records>> ();
}
public class Document : Records
{
public Document(string source, int value) : base(ContentTypesString.DocumentNew, source, value)
{
}
}
public class Headlines : Records
{
public Headlines(string source, int value) : base(ContentTypesString.HeadlinesNew, source, value)
{
}
}
Is that what you need ?
public abstract class Records
{
public string Type;
public string Source;
public int Value;
protected Records(string type, string source, int value)
{
Type = type;
Source = source;
Value = value;
}
}
public class DocumentRecords : Records
{
public DocumentRecords(string source, int value)
: base(ContentTypesString.DocumentNew, source, value) // use here
{
}
}
public class HeadlinesRecords : Records
{
public HeadlinesRecords(string source, int value)
: base(ContentTypesString.HeadlinesNew, source, value) // use here
{
}
}
public static class ContentTypesString
{
public static string DocumentNew { get { return "Document - New this Month"; } }
public static string HeadlinesNew { get { return "Headlines - New this Month"; } }
}
I want to use a pre-loaded lookup data from a list within a query. I need the query to return as IQueryable because it is used in a grid & being paged (not included here). I need the Labels to load from the lookup to avoid joins (more are in the actual code).
I know I could do a ToList() and post-process it, but I need the IQueryable. Here is the code:
// Run in intialization other code...
var contactLookups = new ContactsDbContext();
List<StatusType> _statusTypes = contactLookups.StatusTypes.ToList();
public IQueryable GetQuery()
{
var contactsCtx = new ContactsDbContext();
IQueryable query = contactsCtx.Select(contact => new
{
Id = contact.Id,
FullName = contact.FirstName + " " + contact.LastName,
CompanyName = contact.CompanyName,
Status = _statusTypes.FirstOrDefault(l => contact.StatusId == l.Id).Label
});
return query;
}
The code above throws an error because EF/LINQ can't form the in-memory list into SQL (for obvious reasons) - unless something is added/changed.
I want to somehow tell EF to apply the lookup after the SQL or something to that effect. I had read about doing something similar with EF Helper code and Expressions, but I can't find that article anymore.
Note, I'm flexible on the lookup itself. The only non-negotiable is the "contact.StatusId" (int), but the rest of the structure "_statusTypes.FirstOrDefault(l => contact.StatusId == l.Id)", like the List type, is open.
Any help is appreciated.
You can wrap EF's query into your own intercepting implementation of IQueryable, in which you can inject values of in-memory lookups prior to returning objects to application.
It may sound complex, but actually is not that hard to implement. The following needs to be done:
Mark the Status property in your entity as non-mapped (using Ignore() with Fluent API or [NotMapped] attribute on the property).
Write InterceptingQueryable<T> (let's name it so) implementation of IOrderedQueryable<T>, which wraps an IQueryable<T> object from the EF (returned by the Select method in your example).
Write InterceptingQueryProvider<T> implementation of IQueryProvider<T>, which in turn wraps query provider obtained from the EF.
Write InterceptingEnumerator<T> implementation of IEnumerator<T>, which relays to enumerator object returned by the EF. This enumerator will inject the value of Status property (can easily be generalized to populate any lookup property this way), right after it performs MoveNext, so that object returned by Current is fully populated.
The above chain is connected as follows:
InterceptingQueryable relays to EF's query object, passed in the constructor.
InterceptingQueryable.Provider property returns InterceptingQueryProvider.
InterceptingQueryable.GetEnumerator method returns InterceptingEnumerator.
InterceptingQueryProvider.CreateQuery methods obtain query object from the EF query provider, then return it wrapped in another instance of InterceptingQueryable.
InterceptingQueryProvider.Execute invokes Execute on the EF query provider, then in the case it gets an entity which is subject to lookup injection, it injects the lookup values in the same way as InterceptingEnumerator does (extract a method for reuse).
UPDATE
Here is the code:
using System;
using System.Collections;
using System.Collections.Generic;
using System.Linq;
using System.Linq.Expressions;
using System.Text;
using System.Threading.Tasks;
namespace Examples.Queryables
{
public class InterceptingQueryable<T> : IOrderedQueryable<T>
{
private readonly Action<T> _interceptor;
private readonly IQueryable<T> _underlyingQuery;
private InterceptingQueryProvider _provider;
public InterceptingQueryable(Action<T> interceptor, IQueryable<T> underlyingQuery)
{
_interceptor = interceptor;
_underlyingQuery = underlyingQuery;
_provider = null;
}
public IEnumerator<T> GetEnumerator()
{
return new InterceptingEnumerator(_interceptor, _underlyingQuery.GetEnumerator());
}
IEnumerator IEnumerable.GetEnumerator()
{
return GetEnumerator();
}
public Expression Expression
{
get { return _underlyingQuery.Expression; }
}
public Type ElementType
{
get { return _underlyingQuery.ElementType; }
}
public IQueryProvider Provider
{
get
{
if ( _provider == null )
{
_provider = new InterceptingQueryProvider(_interceptor, _underlyingQuery.Provider);
}
return _provider;
}
}
private class InterceptingQueryProvider : IQueryProvider
{
private readonly Action<T> _interceptor;
private readonly IQueryProvider _underlyingQueryProvider;
public InterceptingQueryProvider(Action<T> interceptor, IQueryProvider underlyingQueryProvider)
{
_interceptor = interceptor;
_underlyingQueryProvider = underlyingQueryProvider;
}
public IQueryable CreateQuery(Expression expression)
{
throw new NotImplementedException();
}
public IQueryable<TElement> CreateQuery<TElement>(Expression expression)
{
var query = _underlyingQueryProvider.CreateQuery<TElement>(expression);
if ( typeof(T).IsAssignableFrom(typeof(TElement)) )
{
return new InterceptingQueryable<TElement>((Action<TElement>)(object)_interceptor, query);
}
else
{
return query;
}
}
public object Execute(Expression expression)
{
throw new NotImplementedException();
}
public TResult Execute<TResult>(Expression expression)
{
var result = _underlyingQueryProvider.Execute<TResult>(expression);
if ( result is T )
{
_interceptor((T)(object)result);
}
return result;
}
}
private class InterceptingEnumerator : IEnumerator<T>
{
private readonly Action<T> _interceptor;
private readonly IEnumerator<T> _underlyingEnumerator;
private bool _hasCurrent;
public InterceptingEnumerator(Action<T> interceptor, IEnumerator<T> underlyingEnumerator)
{
_interceptor = interceptor;
_underlyingEnumerator = underlyingEnumerator;
_hasCurrent = false;
}
public void Dispose()
{
_underlyingEnumerator.Dispose();
}
public bool MoveNext()
{
_hasCurrent = _underlyingEnumerator.MoveNext();
if ( _hasCurrent )
{
_interceptor(_underlyingEnumerator.Current);
}
return _hasCurrent;
}
public void Reset()
{
_underlyingEnumerator.Reset();
}
public T Current
{
get
{
return _underlyingEnumerator.Current;
}
}
object IEnumerator.Current
{
get { return Current; }
}
}
}
public static class QueryableExtensions
{
public static IOrderedQueryable<T> InterceptWith<T>(this IQueryable<T> query, Action<T> interceptor)
{
return new InterceptingQueryable<T>(interceptor, query);
}
}
}
And here is test case/example. First, we should not forget to add non-mapped Status property to Contact entity:
public partial class Contact
{
[NotMapped]
public StatusType Status { get; set; }
}
Then, we can use the interceptor mechanism as follows:
var contactLookups = contactsCtx.StatusTypes.ToList();
Action<Contact> interceptor = contact => {
contact.Status = contactLookups.FirstOrDefault(l => contact.StatusId == l.Id);
};
// note that we add InterceptWith(...) to entity set
var someContacts =
from c in contactsCtx.Contacts.InterceptWith(interceptor)
where c.FullName.StartsWith("Jo")
orderby c.FullName, c.CompanyName
select c;
Console.WriteLine("--- SOME CONTACTS ---");
foreach ( var c in someContacts )
{
Console.WriteLine(
"{0}: {1}, {2}, {3}={4}",
c.Id, c.FullName, c.CompanyName, c.StatusId, c.Status.Name);
}
which prints:
--- SOME CONTACTS ---
1: John Smith, Acme Corp, 3=Status Three
3: Jon Snow, The Wall, 2=Status Two
and the query gets translated into:
SELECT
[Extent1].[Id] AS [Id],
[Extent1].[FullName] AS [FullName],
[Extent1].[CompanyName] AS [CompanyName],
[Extent1].[StatusId] AS [StatusId]
FROM [dbo].[Contacts] AS [Extent1]
WHERE [Extent1].[FullName] LIKE 'Jo%'
ORDER BY [Extent1].[FullName] ASC, [Extent1].[CompanyName] ASC
I'm not sure what is the benefit of avoiding the joins compared to the obvious drawback of not being able to process the whole query at the database side, but what you are asking for can be achieved by doing as much as possible (filtering, ordering, grouping, projection) with Linq to Entities, then turn it into IEnumerable and do the rest with Linq To Objects. You can always use Enumerable.AsQueryable to switch to IQueryable implementation over IEnumerable. Something like this
public IQueryable GetQuery()
{
var db = new ContactsDbContext();
var query = db.Contacts.Select(contact => new
{
Id = contact.Id,
FullName = contact.FirstName + " " + contact.LastName,
CompanyName = contact.CompanyName,
StatusId = contact.StatusId
})
.AsEnumerable()
.Select(contact => new
{
Id = contact.Id,
FullName = contact.FullName,
CompanyName = contact.CompanyName,
Status = _statusTypes.FirstOrDefault(l => contact.StatusId == l.Id).Label
})
.AsQueryable();
return query;
}
for access control purposes in a intensive DB use system I had to implement an objectset wrapper, where the AC will be checked.
The main objective is make this change preserving the existing code for database access, that is implemented with linq to entities all over the classes (there is no centralized layer for database).
The ObjectSetWrapper created is like that:
public class ObjectSetWrapper<TEntity> : IQueryable<TEntity> where TEntity : EntityObject
{
private IQueryable<TEntity> QueryableModel;
private ObjectSet<TEntity> ObjectSet;
public ObjectSetWrapper(ObjectSet<TEntity> objectSetModels)
{
this.QueryableModel = objectSetModels;
this.ObjectSet = objectSetModels;
}
public ObjectQuery<TEntity> Include(string path)
{
return this.ObjectSet.Include(path);
}
public void DeleteObject(TEntity #object)
{
this.ObjectSet.DeleteObject(#object);
}
public void AddObject(TEntity #object)
{
this.ObjectSet.AddObject(#object);
}
public IEnumerator<TEntity> GetEnumerator()
{
return QueryableModel.GetEnumerator();
}
public Type ElementType
{
get { return typeof(TEntity); }
}
public System.Linq.Expressions.Expression Expression
{
get { return this.QueryableModel.Expression; }
}
public IQueryProvider Provider
{
get { return this.QueryableModel.Provider; }
}
public void Attach(TEntity entity)
{
this.ObjectSet.Attach(entity);
}
public void Detach(TEntity entity)
{
this.ObjectSet.Detach(entity);
}
IEnumerator IEnumerable.GetEnumerator()
{
return this.QueryableModel.GetEnumerator();
}
}
It's really simple and works for simple queries, like that:
//db.Product is ObjectSetWrapper<Product>
var query = (from item in db.Product where item.Quantity > 0 select new { item.Id, item.Name, item.Value });
var itensList = query.Take(10).ToList();
But when I have subqueries like that:
//db.Product is ObjectSetWrapper<Product>
var query = (from item in db.Product
select new
{
Id = item.Id,
Name = item.Name,
SalesQuantity = (from sale in db.Sale where sale.ProductId == item.Id select sale.Id).Count()
}).OrderByDescending(x => x.SalesQuantity);
var productsList = query.Take(10).ToList();
I get NotSupportedException, saying I can't create a constant value of my inner query entity type:
Unable to create a constant value of type 'MyNamespace.Model.Sale'.
Only primitive types or enumeration types are supported in this
context.
How can I get my queries working? I don't really need to make my wrapper an ObjectSet type, I just need to use it in queries.
Updated
I have changed my class signature. Now it's also implementing IObjectSet<>, but I'm getting the same NotSupportedException:
public class ObjectSetWrapper<TEntity> : IQueryable<TEntity>, IObjectSet<TEntity> where TEntity : EntityObject
EDIT:
The problem is that the following LINQ construction is translated into LINQ expression containing your custom class inside (ObjectSetWrapper).
var query = (from item in db.Product
select new
{
Id = item.Id,
Name = item.Name,
SalesQuantity = (from sale in db.Sale where sale.ProductId == item.Id select sale.Id).Count()
}).OrderByDescending(x => x.SalesQuantity);
LINQ to Entities tries to convert this expression into SQL statement, but it has no idea how to deal with the custom classes (as well as custom methods).
The solution in such cases is to replace IQueryProvider with the custom one, which should intercept the query execution and translate LINQ expression, containing custom classes/methods into valid LINQ to Entities expression (which operates with entities and object sets).
Expression conversion is performed using the class, derived from ExpressionVisitor, which performs expression tree traversal, replacing relevant nodes, to the nodes which can be accepted by LINQ to Entities
Part 1 - IQueryWrapper
// Query wrapper interface - holds and underlying query
interface IQueryWrapper
{
IQueryable UnderlyingQueryable { get; }
}
Part 2 - Abstract QueryWrapperBase (not generic)
abstract class QueryWrapperBase : IQueryProvider, IQueryWrapper
{
public IQueryable UnderlyingQueryable { get; private set; }
class ObjectWrapperReplacer : ExpressionVisitor
{
public override Expression Visit(Expression node)
{
if (node == null || !typeof(IQueryWrapper).IsAssignableFrom(node.Type)) return base.Visit(node);
var wrapper = EvaluateExpression<IQueryWrapper>(node);
return Expression.Constant(wrapper.UnderlyingQueryable);
}
public static Expression FixExpression(Expression expression)
{
var replacer = new ObjectWrapperReplacer();
return replacer.Visit(expression);
}
private T EvaluateExpression<T>(Expression expression)
{
if (expression is ConstantExpression) return (T)((ConstantExpression)expression).Value;
var lambda = Expression.Lambda(expression);
return (T)lambda.Compile().DynamicInvoke();
}
}
protected QueryWrapperBase(IQueryable underlyingQueryable)
{
UnderlyingQueryable = underlyingQueryable;
}
public abstract IQueryable<TElement> CreateQuery<TElement>(Expression expression);
public abstract IQueryable CreateQuery(Expression expression);
public TResult Execute<TResult>(Expression expression)
{
return (TResult)Execute(expression);
}
public object Execute(Expression expression)
{
expression = ObjectWrapperReplacer.FixExpression(expression);
return typeof(IQueryable).IsAssignableFrom(expression.Type)
? ExecuteQueryable(expression)
: ExecuteNonQueryable(expression);
}
protected object ExecuteNonQueryable(Expression expression)
{
return UnderlyingQueryable.Provider.Execute(expression);
}
protected IQueryable ExecuteQueryable(Expression expression)
{
return UnderlyingQueryable.Provider.CreateQuery(expression);
}
}
Part 3 - Generic QueryWrapper<TElement>
class QueryWrapper<TElement> : QueryWrapperBase, IOrderedQueryable<TElement>
{
private static readonly MethodInfo MethodCreateQueryDef = GetMethodDefinition(q => q.CreateQuery<object>(null));
public QueryWrapper(IQueryable<TElement> underlyingQueryable) : this(null, underlyingQueryable)
{
}
protected QueryWrapper(Expression expression, IQueryable underlyingQueryable) : base(underlyingQueryable)
{
Expression = expression ?? Expression.Constant(this);
}
public virtual IEnumerator<TElement> GetEnumerator()
{
return ((IEnumerable<TElement>)Execute<IEnumerable>(Expression)).GetEnumerator();
}
IEnumerator IEnumerable.GetEnumerator()
{
return GetEnumerator();
}
public Expression Expression { get; private set; }
public Type ElementType
{
get { return typeof(TElement); }
}
public IQueryProvider Provider
{
get { return this; }
}
public override IQueryable CreateQuery(Expression expression)
{
var expressionType = expression.Type;
var elementType = expressionType
.GetInterfaces()
.Single(t => t.IsGenericType && t.GetGenericTypeDefinition() == typeof(IEnumerable<>))
.GetGenericArguments()
.Single();
var createQueryMethod = MethodCreateQueryDef.MakeGenericMethod(elementType);
return (IQueryable)createQueryMethod.Invoke(this, new object[] { expression });
}
public override IQueryable<TNewElement> CreateQuery<TNewElement>(Expression expression)
{
return new QueryWrapper<TNewElement>(expression, UnderlyingQueryable);
}
private static MethodInfo GetMethodDefinition(Expression<Action<QueryWrapper<TElement>>> methodSelector)
{
var methodCallExpression = (MethodCallExpression)methodSelector.Body;
return methodCallExpression.Method.GetGenericMethodDefinition();
}
}
Part 4 - finally your ObjectSetWrapper
public class ObjectSetWrapper<TEntity> : IQueryable<TEntity>, IQueryWrapper where TEntity : class
{
private IQueryable<TEntity> QueryableModel;
private ObjectSet<TEntity> ObjectSet;
public ObjectSetWrapper(ObjectSet<TEntity> objectSetModels)
{
this.QueryableModel = new QueryWrapper<TEntity>(objectSetModels);
this.ObjectSet = objectSetModels;
}
public ObjectQuery<TEntity> Include(string path)
{
return this.ObjectSet.Include(path);
}
public void DeleteObject(TEntity #object)
{
this.ObjectSet.DeleteObject(#object);
}
public void AddObject(TEntity #object)
{
this.ObjectSet.AddObject(#object);
}
public IEnumerator<TEntity> GetEnumerator()
{
return QueryableModel.GetEnumerator();
}
public Type ElementType
{
get { return typeof(TEntity); }
}
public System.Linq.Expressions.Expression Expression
{
get { return this.QueryableModel.Expression; }
}
public IQueryProvider Provider
{
get { return this.QueryableModel.Provider; }
}
public void Attach(TEntity entity)
{
this.ObjectSet.Attach(entity);
}
public void Detach(TEntity entity)
{
this.ObjectSet.Detach(entity);
}
IEnumerator IEnumerable.GetEnumerator()
{
return this.QueryableModel.GetEnumerator();
}
IQueryable IQueryWrapper.UnderlyingQueryable
{
get { return this.ObjectSet; }
}
}
Your inner query fails because you are referencing another dataset when you should be traversing foreign keys:
SalesQuantity = item.Sales.Count()
I have built a repository which only exposes IEnumerable based mostly on the examples in "Professional ASP.NET Design Patterns" by Scott Millett.
However because he mostly uses NHibernate his example of how to implement the Query Object Pattern, or rather how to best translate the query into something useful in EF, is a bit lacking.
I am looking for a good example of an implementation of the Query Object Pattern using EF4.
Edit: The main issues with the trivial example in the book are that CreateQueryAndObjectParameters only handles 2 cases, Equal & LesserThanOrEqual - not exactly a complete query solution. And it uses a string to build the criterion - which is a very crude way to handle this when compared to NHibernate. He says he'll provide the EF code for the chapter 10 example, but its not in the download. Hence looking for a real world example.
You're probably sick of hearing from me at this point, but the natively supported IQueryable is the only Query Object Pattern implementation you need for EF.
According to the book ( "Professional ASP.NET Design Patterns" by Scott Millett) you can use this codes [I have improved some lines]:
Infrastructure layer :
Criterion class : (each Query can contain some Criterion)
public class Criterion
{
private string _propertyName;
private object _value;
private CriteriaOperator _criteriaOperator;
public Criterion(string propertyName, object value,
CriteriaOperator criteriaOperator)
{
_propertyName = propertyName;
_value = value;
_criteriaOperator = criteriaOperator;
}
public string PropertyName
{
get { return _propertyName; }
}
public object Value
{
get { return _value; }
}
public CriteriaOperator criteriaOperator
{
get { return _criteriaOperator; }
}
public static Criterion Create<T>(Expression<Func<T, object>> expression, object value, CriteriaOperator criteriaOperator)
{
string propertyName = PropertyNameHelper.ResolvePropertyName<T>(expression);
Criterion myCriterion = new Criterion(propertyName, value, criteriaOperator);
return myCriterion;
}
}
CriteriaOperator Class:
public enum CriteriaOperator
{
Equal,
LesserThanOrEqual,
NotApplicable
// TODO: Add the remainder of the criteria operators as required.
}
OrderByClause Class:
public class OrderByClause
{
public string PropertyName { get; set; }
public bool Desc { get; set; }
}
Query class:
public class Query
{
private QueryName _name;
private IList<Query> _subQueries = new List<Query>();
private IList<Criterion> _criteria = new List<Criterion>();
public Query()
: this(QueryName.Dynamic, new List<Criterion>())
{ }
public Query(QueryName name, IList<Criterion> criteria)
{
_name = name;
_criteria = criteria;
}
public QueryName Name
{
get { return _name; }
}
public bool IsNamedQuery()
{
return Name != QueryName.Dynamic;
}
public IEnumerable<Criterion> Criteria
{
get {return _criteria ;}
}
public void Add(Criterion criterion)
{
if (!IsNamedQuery())
_criteria.Add(criterion);
else
throw new ApplicationException("You cannot add additional criteria to named queries");
}
public IEnumerable<Query> SubQueries
{
get { return _subQueries; }
}
public void AddSubQuery(Query subQuery)
{
_subQueries.Add(subQuery);
}
public QueryOperator QueryOperator { get; set; }
public OrderByClause OrderByProperty { get; set; }
}
PropertyNameHelper class:
public static class PropertyNameHelper
{
public static string ResolvePropertyName<T>(
Expression<Func<T, object>> expression)
{
var expr = expression.Body as MemberExpression;
if (expr == null)
{
var u = expression.Body as UnaryExpression;
expr = u.Operand as MemberExpression;
}
return expr.ToString().Substring(expr.ToString().IndexOf(".") + 1);
}
}
You need a QueryTranslator class too, to translate query (in Repository layer):
public class TimelogQueryTranslator : QueryTranslator
{
public ObjectQuery<Timelog> Translate(Query query)
{
ObjectQuery<Timelog> timelogQuery;
if (query.IsNamedQuery())
{
timelogQuery = FindEFQueryFor(query);
}
else
{
StringBuilder queryBuilder = new StringBuilder();
IList<ObjectParameter> paraColl = new List<ObjectParameter>();
CreateQueryAndObjectParameters(query, queryBuilder, paraColl);
//[Edited By= Iman] :
if (query.OrderByProperty == null)
{
timelogQuery = DataContextFactory.GetDataContext().Timelogs
.Where(queryBuilder.ToString(), paraColl.ToArray());
}
else if (query.OrderByProperty.Desc == true)
{
timelogQuery = DataContextFactory.GetDataContext().Timelogs
.Where(queryBuilder.ToString(), paraColl.ToArray()).OrderBy(String.Format("it.{0} desc", query.OrderByProperty.PropertyName));
}
else
{
timelogQuery = DataContextFactory.GetDataContext().Timelogs
.Where(queryBuilder.ToString(), paraColl.ToArray()).OrderBy(String.Format("it.{0} asc", query.OrderByProperty.PropertyName));
}
//[Edited By= Iman] .
}
return timelogQuery;
}
//-------------------------------------------------------------
public abstract class QueryTranslator
{
public void CreateQueryAndObjectParameters(Query query, StringBuilder queryBuilder, IList<ObjectParameter> paraColl)
{
bool _isNotFirstFilterClause = false;
foreach (Criterion criterion in query.Criteria)
{
if (_isNotFirstFilterClause)
{
queryBuilder.Append(" AND "); //TODO: select depending on query.QueryOperator
}
switch (criterion.criteriaOperator)
{
case CriteriaOperator.Equal:
queryBuilder.Append(String.Format("it.{0} = #{0}", criterion.PropertyName));
break;
case CriteriaOperator.LesserThanOrEqual:
queryBuilder.Append(String.Format("it.{0} <= #{0}", criterion.PropertyName));
break;
default:
throw new ApplicationException("No operator defined");
}
paraColl.Add(new ObjectParameter(criterion.PropertyName, criterion.Value));
_isNotFirstFilterClause = true;
}
}
}
Now in your service layer:
public IEnumerable<Timelog> GetAllTimelogsFor(int iadcId, byte workShift)
{
Query query = new Query(QueryName.Dynamic,new List<Criterion>());
query.Add(Criterion.Create<Timelog>(t=>t.IadcId, iadcId, CriteriaOperator.Equal));
query.QueryOperator = QueryOperator.And;
query.Add(Criterion.Create<Timelog>(t=>t.Shift, workShift, CriteriaOperator.Equal));
query.OrderByProperty = new OrderByClause { PropertyName = "FromTime", Desc = false };
IEnumerable<Timelog> timelogs = _timelogRepository.FindBy(query);
return timelogs;
}