Develop Reference

LINQ expression could not be translated and will be evaluated locally

When using an entity with property of a custom type, the type cannot be translated into SQL.
I have created an example to explain my approach to solve it:
A class takes place in a certain semester. The semester is stored as a DateTime value in the database.
The semester itself is a custom type, with additional properties.
public class Semester
{
public enum HalfYear
{
First = 1,
Second = 7
}
DateTime _dateTime;
public Semester (HalfYear halfYear, int year)
{
_dateTime = new DateTime(year, (int) halfYear, 1)
}
public int Year => _dateTime.Year;
public HalfYear HalfYear => (HalfYear) _dateTime.Month;
public DateTime FirstDay => new DateTime(Year, _dateTime.Month, 1);
public DateTime LastDay => new DateTime(Year, _dateTime.Month + 5, DateTime.DaysInMonth(Year, _dateTime.Month + 5));
}
public class Class
{
int Id { get; set; }
string Title { get; set; }
Semester Semester { get; set; }
}
The Semester type can be mapped to a DateTime using value converters.
This does not work in Where clause such as
db.Classes.Where(c = c.Semester.FirstDay <= DateTime.Now &&
c.Semester.LastDay >= DateTime.Now)
When Entity Framework Core tries to translate the expression tree to SQL, it does not know how to translate Semester.FirstDay or Semester.LastDay.
This is a known limitation of value conversions as the documentation states
Use of value conversions may impact the ability of EF Core to translate expressions to SQL. A warning will be logged for such cases. Removal of these limitations is being considered for a future release.
How to solve this issue?

EntityFrameworkCore has 3 extension points that can be used to translate custom types to SQL.
IMemberTranslator
IMethodCallTranslator
RelationalTypeMapping
These translators and mapppings can be registered using the corresponding plugins:
IMemberTranslatorPlugin
IMethodCallTranslatorPlugin
IRelationalTypeMappingSourcePlugin
The plugins are registered with a IDbContextOptionsExtension
The following example illustrates how I have implemented these interfaces to register the custom type Semester:
IMemberTranslator
public class SqlServerSemesterMemberTranslator : IMemberTranslator
{
public Expression Translate(MemberExpression memberExpression)
{
if (memberExpression.Member.DeclaringType != typeof(Semester)) {
return null;
}
var memberName = memberExpression.Member.Name;
if (memberName == nameof(Semester.FirstDay)) {
return new SqlFunctionExpression(
"DATEFROMPARTS",
typeof(DateTime),
new Expression[] {
new SqlFunctionExpression( "YEAR", typeof(int),new[] { memberExpression.Expression }),
new SqlFunctionExpression( "MONTH", typeof(int),new[] { memberExpression.Expression }),
Expression.Constant(1, typeof(int))
});
}
if (memberName == nameof(Semester.LastDay)) {
return new SqlFunctionExpression(
"EOMONTH",
typeof(DateTime),
new Expression[] {
memberExpression.Expression
});
}
if (memberName == nameof(Semester.HalfYear)) {
return Expression.Convert(
new SqlFunctionExpression(
"MONTH",
typeof(int),
new Expression[] {
memberExpression.Expression
}),
typeof(HalfYear));
}
if (memberName == nameof(Semester.Year)) {
return new SqlFunctionExpression(
"YEAR",
typeof(int),
new Expression[] {
memberExpression.Expression
});
}
return null;
}
}
IMethodCallTranslator
public class SqlServerSemesterMethodCallTranslator : IMethodCallTranslator
{
public Expression Translate(MethodCallExpression methodCallExpression)
{
if (methodCallExpression.Method.DeclaringType != typeof(Period)) {
return null;
}
var methodName = methodCallExpression.Method.Name;
// Implement your Method translations here
return null;
}
}
RelationalTypeMapping
public class SqlServerSemesterTypeMapping : DateTimeTypeMapping
{
public SqlServerSemesterTypeMapping(string storeType, DbType? dbType = null) :
base(storeType, dbType)
{
}
protected SqlServerSemesterTypeMapping(RelationalTypeMappingParameters parameters) : base(parameters)
{
}
protected override RelationalTypeMapping Clone(RelationalTypeMappingParameters parameters) => new SqlServerSemesterTypeMapping(parameters);
}
IMemberTranslatorPlugin
public class SqlServerCustomMemberTranslatorPlugin : IMemberTranslatorPlugin
{
public IEnumerable<IMemberTranslator> Translators => new IMemberTranslator[] { new SqlServerSemesterMemberTranslator() };
}
public class SqlServerCustomMethodCallTranslatorPlugin : IMethodCallTranslatorPlugin
{
public IEnumerable<IMethodCallTranslator> Translators => new IMethodCallTranslator[] { new SqlServerSemesterMethodCallTranslator() };
}
IRelationalTypeMappingSourcePlugin
public class SqlServerCustomTypeMappingSourcePlugin : IRelationalTypeMappingSourcePlugin
{
public RelationalTypeMapping FindMapping(in RelationalTypeMappingInfo mappingInfo)
=> mappingInfo.ClrType == typeof(Semester) || (mappingInfo.StoreTypeName == nameof(DateTime))
? new SqlServerSemesterTypeMapping(mappingInfo.StoreTypeName ?? "datetime")
: null;
}
After you have defined and registered the translators, you have to confgure them in the DbContext.
IDbContextOptionsExtension
public class SqlServerCustomTypeOptionsExtension : IDbContextOptionsExtensionWithDebugInfo
{
public string LogFragment => "using CustomTypes";
public bool ApplyServices(IServiceCollection services)
{
services.AddEntityFrameworkSqlServerCustomTypes();
return false;
}
public long GetServiceProviderHashCode() => 0;
public void PopulateDebugInfo(IDictionary<string, string> debugInfo)
=> debugInfo["SqlServer:" + nameof(SqlServerCustomDbContextOptionsBuilderExtensions.UseCustomTypes)] = "1";
public void Validate(IDbContextOptions options)
{
}
}
Extension Methods
public static class SqlServerCustomDbContextOptionsBuilderExtensions
{
public static object UseCustomTypes(this SqlServerDbContextOptionsBuilder optionsBuilder)
{
if (optionsBuilder == null) throw new ArgumentNullException(nameof(optionsBuilder));
// Registere die SqlServerDiamantOptionsExtension.
var coreOptionsBuilder = ((IRelationalDbContextOptionsBuilderInfrastructure)optionsBuilder).OptionsBuilder;
var extension = coreOptionsBuilder.Options.FindExtension<SqlServerCustomTypeOptionsExtension>()
?? new SqlServerCustomTypeOptionsExtension();
((IDbContextOptionsBuilderInfrastructure)coreOptionsBuilder).AddOrUpdateExtension(extension);
// Configure Warnings
coreOptionsBuilder
.ConfigureWarnings(warnings => warnings
.Log(RelationalEventId.QueryClientEvaluationWarning) // Should be thrown to prevent only warnings if a query is not fully evaluated on the db
.Ignore(RelationalEventId.ValueConversionSqlLiteralWarning)); // Ignore warnings for types that are using a ValueConverter
return optionsBuilder;
}
}
public static class SqlServerServiceCollectionExtensions
{
public static IServiceCollection AddEntityFrameworkSqlServerCustomTypes(
this IServiceCollection serviceCollection)
{
if (serviceCollection == null) throw new ArgumentNullException(nameof(serviceCollection));
new EntityFrameworkRelationalServicesBuilder(serviceCollection)
.TryAddProviderSpecificServices(
x => x.TryAddSingletonEnumerable<IRelationalTypeMappingSourcePlugin, SqlServerCustomTypeMappingSourcePlugin>()
.TryAddSingletonEnumerable<IMemberTranslatorPlugin, SqlServerCustomTypeMemberTranslatorPlugin>()
.TryAddSingletonEnumerable<IMethodCallTranslatorPlugin, SqlServerCustomTypeMethodCallTranslatorPlugin>());
return serviceCollection;
}
}
Register the option in the DbContext
dbOptionsBuilder.UseSqlServer(connectionString, builder => builder.UseCustomTypes())

Apply Custom Model Binder to Object Property in asp.net core

I am trying to apply custom model binder for DateTime type property of model.
Here is the IModelBinder and IModelBinderProvider implementations.
public class DateTimeModelBinderProvider : IModelBinderProvider
{
public IModelBinder GetBinder(ModelBinderProviderContext context)
{
if (context == null)
{
throw new ArgumentNullException(nameof(context));
}
if (context.Metadata.ModelType == typeof(DateTime))
{
return new BinderTypeModelBinder(typeof(DateTime));
}
return null;
}
}
public class DateTimeModelBinder : IModelBinder
{
private string[] _formats = new string[] { "yyyyMMdd", "yyyy-MM-dd", "yyyy/MM/dd"
, "yyyyMMddHHmm", "yyyy-MM-dd HH:mm", "yyyy/MM/dd HH:mm"
, "yyyyMMddHHmmss", "yyyy-MM-dd HH:mm:ss", "yyyy/MM/dd HH:mm:ss"};
private readonly IModelBinder baseBinder;
public DateTimeModelBinder()
{
baseBinder = new SimpleTypeModelBinder(typeof(DateTime), null);
}
public Task BindModelAsync(ModelBindingContext bindingContext)
{
if (bindingContext == null)
{
throw new ArgumentNullException(nameof(bindingContext));
}
var valueProviderResult = bindingContext.ValueProvider.GetValue(bindingContext.ModelName);
if (valueProviderResult != ValueProviderResult.None)
{
bindingContext.ModelState.SetModelValue(bindingContext.ModelName, valueProviderResult);
var value = valueProviderResult.FirstValue;
if (DateTime.TryParseExact(value, _formats, new CultureInfo("en-US"), DateTimeStyles.None, out DateTime dateTime))
{
bindingContext.Result = ModelBindingResult.Success(dateTime);
}
else
{
bindingContext.ModelState.TryAddModelError(bindingContext.ModelName, $"{bindingContext} property {value} format error.");
}
return Task.CompletedTask;
}
return baseBinder.BindModelAsync(bindingContext);
}
}
And here is the model class
public class Time
{
[ModelBinder(BinderType = typeof(DateTimeModelBinder))]
public DateTime? validFrom { get; set; }
[ModelBinder(BinderType = typeof(DateTimeModelBinder))]
public DateTime? validTo { get; set; }
}
And here is the controller action method.
[HttpPost("/test")]
public IActionResult test([FromBody]Time time)
{
return Ok(time);
}
When tested, the custom binder is not invoked but the default dotnet binder is invoked. According to the official documentation,
ModelBinder attribute could be applied to individual model properties
(such as on a viewmodel) or to action method parameters to specify a
certain model binder or model name for just that type or action.
But it seems not working with my code.

1. Reason
According to the [FromBody]Time time in your action, I guess you're sending a payload with Content-Type of application/json. In that case, when a json payload received, the Model Binding System will inspect the parameter time and then try to find a proper binder for it. Because the context.Metadata.ModelType equals typeof(Time) instead of the typeof(DateTime), and there's no custom ModelBinder for typeof(Time) , your GetBinder(context) method will return a null :
public class DateTimeModelBinderProvider : IModelBinderProvider
{
public IModelBinder GetBinder(ModelBinderProviderContext context)
{
if (context == null)
{
throw new ArgumentNullException(nameof(context));
}
if (context.Metadata.ModelType == typeof(DateTime)) // not typeof(Time)
{
return new BinderTypeModelBinder(typeof(DateTime));
}
return null;
}
}
Thus it falls back to the default model binder for application/json. The default json model binder uses Newtonsoft.Json under the hood and will simply deserialize the whole payload as an instance of Time. As a result, your DateTimeModelBinder is not invoked.
2. Quick Fix
One approach is to use application/x-www-form-urlencoded (avoid using the application/json)
Remove the [FromBody] attribute:
[HttpPost("/test2")]
public IActionResult test2(Time time)
{
return Ok(time);
}
and send the payload in the format of application/x-www-form-urlencoded
POST https://localhost:5001/test2
Content-Type: application/x-www-form-urlencoded
validFrom=2018-01-01&validTo=2018-02-02
It should work now.
3. Working with JSON
Create a custom converter as below :
public class CustomDateConverter : JsonConverter
{
public override bool CanConvert(Type objectType)
{
return true;
}
public static string[] _formats = new string[] {
"yyyyMMdd", "yyyy-MM-dd", "yyyy/MM/dd"
, "yyyyMMddHHmm", "yyyy-MM-dd HH:mm", "yyyy/MM/dd HH:mm"
, "yyyyMMddHHmmss", "yyyy-MM-dd HH:mm:ss", "yyyy/MM/dd HH:mm:ss"
};
public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer)
{
var dt= reader.Value;
if (DateTime.TryParseExact(dt as string, _formats, new CultureInfo("en-US"), DateTimeStyles.None, out DateTime dateTime))
return dateTime;
else
return null;
}
public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer)
{
serializer.Serialize(writer, value as string);
}
}
I simply copied your code to format the date.
Change your Model as below :
public class Time
{
[ModelBinder(BinderType = typeof(DateTimeModelBinder))]
[JsonConverter(typeof(CustomDateConverter))]
public DateTime? validFrom { get; set; }
[ModelBinder(BinderType = typeof(DateTimeModelBinder))]
[JsonConverter(typeof(CustomDateConverter))]
public DateTime? validTo { get; set; }
}
And now you can receive the time using [FromBody]
[HttpPost("/test")]
public IActionResult test([FromBody]Time time)
{
return Ok(time);
}

Model property bindings with [FromBody] attribute

I want to apply some preprocessing to raw data before it assigned to model properties. Namely to replace comma with dot to allow converting both this strings "324.32" and "324,32" into double. So I wrote this model binder
public class MoneyModelBinder: IModelBinder
{
private readonly Type _modelType;
public MoneyModelBinder(Type modelType)
{
_modelType = modelType;
}
public Task BindModelAsync(ModelBindingContext bindingContext)
{
if (bindingContext == null)
{
throw new ArgumentNullException(nameof(bindingContext));
}
string modelName = bindingContext.ModelName;
ValueProviderResult providerResult = bindingContext.ValueProvider.GetValue(modelName);
if (providerResult == ValueProviderResult.None)
{
return TaskCache.CompletedTask;
}
bindingContext.ModelState.SetModelValue(modelName, providerResult);
string value = providerResult.FirstValue;
if (string.IsNullOrEmpty(value))
{
return TaskCache.CompletedTask;
}
value = value.Replace(",", ".");
object result;
if(_modelType == typeof(double))
{
result = Convert.ToDouble(value, CultureInfo.InvariantCulture);
}
else if(_modelType == typeof(decimal))
{
result = Convert.ToDecimal(value, CultureInfo.InvariantCulture);
}
else if(_modelType == typeof(float))
{
result = Convert.ToSingle(value, CultureInfo.InvariantCulture);
}
else
{
throw new NotSupportedException($"binder doesn't implement this type {_modelType}");
}
bindingContext.Result = ModelBindingResult.Success(result);
return TaskCache.CompletedTask;
}
}
then appropriate provider
public class MoneyModelBinderProvider : IModelBinderProvider
{
public IModelBinder GetBinder(ModelBinderProviderContext context)
{
if (context == null)
{
throw new ArgumentNullException(nameof(context));
}
if(context.Metadata?.ModelType == null)
{
return null;
}
if (context.Metadata.ModelType.In(typeof(double), typeof(decimal), typeof(float)))
{
return new MoneyModelBinder(context.Metadata.ModelType);
}
return null;
}
}
and registering it inside Startup.cs
services.AddMvc(options =>
{
options.ModelBinderProviders.Insert(0, new MoneyModelBinderProvider());
});
but I noticed some strange behavior or maybe I missed something. If I use this kind of action
public class Model
{
public string Str { get; set; }
public double Number { get; set; }
}
[HttpPost]
public IActionResult Post(Model model)
{
return Ok("ok");
}
and supply parameters inside query string everything works fine: first provider is called for model itself then for every property of the model. But if I use [FromBody] attribute and supply parameters by JSON, provider is called for model but never called for properties of this model. But why? How can I use binders with FromBody?

I've found solution. As it described here [FromBody] behaves differently in comparing to other value providers - it converts complex objects all at once via JsonFormatters. So in addition model binders we should write separate logic just for FromBody. And of course we can catch some points during json processing:
public class MoneyJsonConverter : JsonConverter
{
public override bool CanWrite => false;
public override bool CanConvert(Type objectType)
{
return objectType == typeof(double);
}
public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer)
{
string value = (reader.Value ?? "").Replace(" ", "").Replace(",", ".");
TypeConverter converter = TypeDescriptor.GetConverter(modelType);
object result = converter.ConvertFromInvariantString(value);
return result;;
}
}
and using
services.AddMvc(options =>
{
options.ModelBinderProviders.Insert(0, new MoneyModelBinderProvider());
}).AddJsonOptions(options =>
{
options.SerializerSettings.Converters.Add(new MoneyJsonConverter());
});

Why can I not deserialize this custom struct using Json.Net?

I have a struct representing a DateTime which also has zone info as below:
public struct DateTimeWithZone
{
private readonly DateTime _utcDateTime;
private readonly TimeZoneInfo _timeZone;
public DateTimeWithZone(DateTime dateTime, TimeZoneInfo timeZone,
DateTimeKind kind = DateTimeKind.Utc)
{
dateTime = DateTime.SpecifyKind(dateTime, kind);
_utcDateTime = dateTime.Kind != DateTimeKind.Utc
? TimeZoneInfo.ConvertTimeToUtc(dateTime, timeZone)
: dateTime;
_timeZone = timeZone;
}
public DateTime UniversalTime { get { return _utcDateTime; } }
public TimeZoneInfo TimeZone { get { return _timeZone; } }
public DateTime LocalTime
{
get
{
return TimeZoneInfo.ConvertTime(_utcDateTime, _timeZone);
}
}
}
I can serialize the object using:
var now = DateTime.Now;
var dateTimeWithZone = new DateTimeWithZone(now, TimeZoneInfo.Local, DateTimeKind.Local);
var serializedDateTimeWithZone = JsonConvert.SerializeObject(dateTimeWithZone);
But when I deserialize it using the below, I get an invalid DateTime value (DateTime.MinValue)
var deserializedDateTimeWithZone = JsonConvert.DeserializeObject<DateTimeWithZone>(serializedDateTimeWithZone);
Any help is much appreciated.

Just declare the constructor as follows, that's all
[JsonConstructor]
public DateTimeWithZone(DateTime universalTime, TimeZoneInfo timeZone,
DateTimeKind kind = DateTimeKind.Utc)
{
universalTime = DateTime.SpecifyKind(universalTime, kind);
_utcDateTime = universalTime.Kind != DateTimeKind.Utc
? TimeZoneInfo.ConvertTimeToUtc(universalTime, timeZone)
: universalTime;
_timeZone = timeZone;
}
Note: I only added JsonConstructor attribute and changed the parameter name as universalTime

You need to write a custom JsonConverter to properly serialize and deserialize these values. Add this class to your project.
public class DateTimeWithZoneConverter : JsonConverter
{
public override bool CanConvert(Type objectType)
{
return objectType == typeof (DateTimeWithZone) || objectType == typeof (DateTimeWithZone?);
}
public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer)
{
var dtwz = (DateTimeWithZone) value;
writer.WriteStartObject();
writer.WritePropertyName("UniversalTime");
serializer.Serialize(writer, dtwz.UniversalTime);
writer.WritePropertyName("TimeZone");
serializer.Serialize(writer, dtwz.TimeZone.Id);
writer.WriteEndObject();
}
public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer)
{
var ut = default(DateTime);
var tz = default(TimeZoneInfo);
var gotUniversalTime = false;
var gotTimeZone = false;
while (reader.Read())
{
if (reader.TokenType != JsonToken.PropertyName)
break;
var propertyName = (string)reader.Value;
if (!reader.Read())
continue;
if (propertyName == "UniversalTime")
{
ut = serializer.Deserialize<DateTime>(reader);
gotUniversalTime = true;
}
if (propertyName == "TimeZone")
{
var tzid = serializer.Deserialize<string>(reader);
tz = TimeZoneInfo.FindSystemTimeZoneById(tzid);
gotTimeZone = true;
}
}
if (!(gotUniversalTime && gotTimeZone))
{
throw new InvalidDataException("An DateTimeWithZone must contain UniversalTime and TimeZone properties.");
}
return new DateTimeWithZone(ut, tz);
}
}
Then register it with the json settings you're using. For example, the default settings can be changed like this:
JsonConvert.DefaultSettings = () =>
{
var settings = new JsonSerializerSettings();
settings.Converters.Add(new DateTimeWithZoneConverter());
return settings;
};
Then it will properly serialize to a usable format. Example:
{
"UniversalTime": "2014-07-13T20:24:40.4664448Z",
"TimeZone": "Pacific Standard Time"
}
And it will deserialize properly as well.
If you want to include the local time, You would just add that to the WriteJson method, but it should probably be ignored when deserializing. Otherwise you'd have two different sources of truth. Only one can be authoritative.
Also, you might instead try Noda Time, which includes a ZonedDateTime struct for this exact purpose. There's already support for serialization via the NodaTime.Serialization.JsonNet NuGet package.

WCF DataContractJsonSerializer and setting DateTime objects to UTC?

Is there a universal way to instruct the DataContractJsonSerializer to use UTC for dates? Otherwise, I have to add .ToUniversalTime() to all of my date instances. Is this possible? The reason is that date values are defaulting DateTimeKind.Local and adding offsets to the JSON result. Making the dates universal does the trick, but can it be done at a global level? Thanks.

There's no way to do that directly at the global level - primitive types (such as DateTime) can't be "surrogated". A possible workaround is to use some kind of reflection along with a surrogate to change the DateTime fields (or properties) in an object when it's being serialized, as shown in the example below.
public class StackOverflow_6100587_751090
{
public class MyType
{
public MyTypeWithDates d1;
public MyTypeWithDates d2;
}
public class MyTypeWithDates
{
public DateTime Start;
public DateTime End;
}
public class MySurrogate : IDataContractSurrogate
{
public object GetCustomDataToExport(Type clrType, Type dataContractType)
{
throw new NotImplementedException();
}
public object GetCustomDataToExport(MemberInfo memberInfo, Type dataContractType)
{
throw new NotImplementedException();
}
public Type GetDataContractType(Type type)
{
return type;
}
public object GetDeserializedObject(object obj, Type targetType)
{
return obj;
}
public void GetKnownCustomDataTypes(Collection<Type> customDataTypes)
{
}
public object GetObjectToSerialize(object obj, Type targetType)
{
return ReplaceLocalDateWithUTC(obj);
}
public Type GetReferencedTypeOnImport(string typeName, string typeNamespace, object customData)
{
throw new NotImplementedException();
}
public CodeTypeDeclaration ProcessImportedType(CodeTypeDeclaration typeDeclaration, CodeCompileUnit compileUnit)
{
throw new NotImplementedException();
}
private object ReplaceLocalDateWithUTC(object obj)
{
if (obj == null) return null;
Type objType = obj.GetType();
foreach (var field in objType.GetFields())
{
if (field.FieldType == typeof(DateTime))
{
DateTime fieldValue = (DateTime)field.GetValue(obj);
if (fieldValue.Kind != DateTimeKind.Utc)
{
field.SetValue(obj, fieldValue.ToUniversalTime());
}
}
}
return obj;
}
}
public static void Test()
{
MemoryStream ms = new MemoryStream();
DataContractJsonSerializer dcjs = new DataContractJsonSerializer(typeof(MyType), null, int.MaxValue, true, new MySurrogate(), false);
MyType t = new MyType
{
d1 = new MyTypeWithDates { Start = DateTime.Now, End = DateTime.Now.AddMinutes(1) },
d2 = new MyTypeWithDates { Start = DateTime.Now.AddHours(1), End = DateTime.Now.AddHours(2) },
};
dcjs.WriteObject(ms, t);
Console.WriteLine(Encoding.UTF8.GetString(ms.ToArray()));
}
}