Develop Reference

Adding a filter on a $lookup stage in Mongo using C#

I have the following Mongo shell query:
db.getCollection('users').aggregate([
{ "$match": { "Email": "user1#example.com" } },
{ "$lookup": {
"from": "groups",
"localField": "_id",
"foreignField": "Users",
"as": "userGroups"
}},
{ "$graphLookup": {
"from": "groups",
"startWith": "$userGroups._id",
"connectFromField": "Groups",
"connectToField": "_id",
"as": "ancestorGroups"
}},
{ "$unwind": "$ancestorGroups" },
{ "$group": {
"_id": "$_id",
"Email": { "$first": "$email" },
"Groups": { "$addToSet": "$ancestorGroups._id" }
}}
])
The query matches a user by their email address, and then transitively finds all the groups that the user belongs to.
I converted this query to C# like so:
var opts = new AggregateLookupOptions<GroupDto, UserDto>();
var options = new AggregateGraphLookupOptions<GroupDto, GroupDto, UserDto>()
{
};
var filter = Builders<UserDto>.Filter.Eq(x => x.Email, email);
var query = _collection.Aggregate()
.Match(filter)
.Lookup(_groups, x => x.Id, x => x.Users, x => x.UserGroups, opts)
.GraphLookup(_groups, x => x.Groups, x => x.Id, x => x.UserGroups.Select(y => y.Id), x => x.AncestorGroups, options);
This works great, however, I want to filter the joined groups collection. In Mongo shell, I can throw in a pipeline parameter in the $lookup stage like so:
{ "$lookup": {
"from": "groups",
"localField": "_id",
"foreignField": "Users",
"as": "userGroups",
"pipeline": [
{ "$match": { "LinkId": "1337" } },
],
}},
However, I'm not sure how to achieve this in C#. The only overload for the Lookup method that takes a lookupPipeline parameter takes let and BsonDocument and won't support typed arguments.
Any ideas?

Formatting api JSON response to add attributes to an array

I have the following api controller that returns a json representation of the query you see here:
public async Task<ActionResult<IEnumerable<CarDto>>> GetCarData(Guid carID)
{
var carData = await (from cl in _context.CarList
join tl in _context.transmissionList
on cl.CId equals tl.CId
join to in _context.transmissionOptions
on tl.TId equals to.TId
where cl.CId == carID
select new CarDto
{
CarId = cl.CarId,
TransmissionId = tl.TId,
OptionId = to.OptionId,
GearId = to.GearId
})
.ToListAsync();
return carData;
}
The returned json data looks like this:
[
{
"carId": "351a",
"transmissionId": "ec7",
"optionId": "a1",
"gearId": "674532a"
},
{
"carId": "351a",
"transmissionId": "ec7",
"optionId": "b7",
"gearId": "5f9173f"
},
{
"carId": "351a",
"transmissionId": "ec7",
"optionId": "c5",
"gearId": "cf807"
}
]
However, I'd like for it to be formatted such that there is a property called transmissionChoices that contains an array of the possible options.
Like this:
{
"carId": "351a",
"transmissionId": "ec7",
"transmissionChoices": [
{
"optionId": "a1",
"gearId": "674532a"
},
{
"optionId": "b7",
"gearId": "5f9173f"
},
{
"optionId": "c5",
"gearId": "cf807"
}
]
}
Is there a way to get the controller to format it like that?

You can use the LINQ GroupBy method and then project the grouped results into the shape you want.
public async Task<ActionResult<IEnumerable<object>>> GetCarData(Guid carID)
{
var carData = await (from cl in _context.CarList
join tl in _context.transmissionList
on cl.CId equals tl.CId
join to in _context.transmissionOptions
on tl.TId equals to.TId
where cl.CId == carID
select new
{
CarId = cl.CarId,
TransmissionId = tl.TId,
OptionId = to.OptionId,
GearId = to.GearId
})
.GroupBy(x => x.CarId)
.Select(g => new
{
CarId = g.First().CarId,
TransmissionId = g.First().TransmissionId,
TransmissionChoices = g.Select(x => new
{
OptionId = x.OptionId,
GearId = x.GearId
})
})
.ToListAsync();
return carData;
}
Note that this is projecting the results into an anonymous type. Feel free to create a model that matches the schema you need and then use that model in the Select(...) projection.

NEST compound queries which must all be satisfied

var availableToField = Infer.Field<Project>(f => f.Availablity.AvailableTo);
var availableFromField = Infer.Field<Project>(f => f.Availablity.AvailableFrom);
var nameField = Infer.Field<Project>(f => f.Contact.Name);
var active_date_to = new DateRangeQuery(){
Name = "toDate",
Boost = 1.1,
Field = "availablity.availableTo",
GreaterThan = DateTime.Now,
TimeZone = "+01:00",
Format = "yyyy-MM-ddTHH:mm:SS||dd.MM.yyyy"
};
var active_date_from = new DateRangeQuery(){
Name = "from",
Boost = 1.1,
Field = "availablity.availableFrom",
LessThanOrEqualTo = DateTime.Now,
TimeZone = "+01:00",
Format = "yyyy-MM-ddTHH:mm:SS||dd.MM.yyyy"
};
public ISearchResult<Project> Search(SearchCriteria criteria)
{var ret = _client.Search<Project>(s =>
s.Query(q =>
active_date_from &&
active_date_to &&
q.Match(d => d.Query(criteria.FreeText))
).From(criteria.CurrentPage).Size(criteria.Take)
.From(criteria.CurrentPage)
.Take(criteria.Take)
);
result.Total = ret.Total;
result.Page = criteria.CurrentPage;
result.PerPage = criteria.Take;
result.Results = ret.Documents;
return result;
}
what im trying to do is get the results matching the freetext but are also withing the pricerange..
somehow though what i get is an invalid NEST response build from a unsuccessful low level call on POST... and in consequence an empty query.
there are no compiling errors.
does anyone have an idea where i could have gone wrong or what im missing?
the other thing i tried was
var mustClauses = new List<QueryContainer>();
mustClauses.Add(active_date_from);
mustClauses.Add(active_date_to);
mustClauses.Add(new TermQuery
{
Field = "contact.name",
Value = criteria.FreeText
});
var searchRequest = new SearchRequest<Project>()
{
Size = 10,
From = 0,
Query = new BoolQuery
{
Must = mustClauses
}
};
var ret = _client.Search<Project>(searchRequest);
result.Total = ret.Total;
result.Page = criteria.CurrentPage;
result.PerPage = criteria.Take;
result.Results = ret.Documents;
which got me pretty much the same results.. (read: none)
is there something im missing?
edit:
however.. this:
var ret = _client.Search<Project>(s => s.Query(q => q.Match(m => m.Field(f => f.DisplayName).Query(criteria.FreeText))));
gives me exactly what i want (without the validation of the dates of course and only looking at one field)

In your first example, the match query is missing a field property which is needed for the query. Because of NEST's conditionless query behaviour, the query is not serialized as part of the request. The two date range queries are serialized however.
Here's a simple example that you may find useful to get the correct query you're looking for
void Main()
{
var pool = new SingleNodeConnectionPool(new Uri("http://localhost:9200"));
var defaultIndex = "projects";
var connectionSettings = new ConnectionSettings(pool, new InMemoryConnection())
.DefaultIndex(defaultIndex )
.PrettyJson()
.DisableDirectStreaming()
.OnRequestCompleted(response =>
{
if (response.RequestBodyInBytes != null)
{
Console.WriteLine(
$"{response.HttpMethod} {response.Uri} \n" +
$"{Encoding.UTF8.GetString(response.RequestBodyInBytes)}");
}
else
{
Console.WriteLine($"{response.HttpMethod} {response.Uri}");
}
Console.WriteLine();
if (response.ResponseBodyInBytes != null)
{
Console.WriteLine($"Status: {response.HttpStatusCode}\n" +
$"{Encoding.UTF8.GetString(response.ResponseBodyInBytes)}\n" +
$"{new string('-', 30)}\n");
}
else
{
Console.WriteLine($"Status: {response.HttpStatusCode}\n" +
$"{new string('-', 30)}\n");
}
});
var client = new ElasticClient(connectionSettings);
var availableToField = Infer.Field<Project>(f => f.Availablity.AvailableTo);
var availableFromField = Infer.Field<Project>(f => f.Availablity.AvailableFrom);
var nameField = Infer.Field<Project>(f => f.Contact.Name);
var active_date_to = new DateRangeQuery
{
Name = "toDate",
Boost = 1.1,
Field = availableToField,
GreaterThan = DateTime.Now,
TimeZone = "+01:00",
Format = "yyyy-MM-ddTHH:mm:SS||dd.MM.yyyy"
};
var active_date_from = new DateRangeQuery
{
Name = "from",
Boost = 1.1,
Field = availableFromField,
LessThanOrEqualTo = DateTime.Now,
TimeZone = "+01:00",
Format = "yyyy-MM-ddTHH:mm:SS||dd.MM.yyyy"
};
var ret = client.Search<Project>(s => s
.Query(q =>
active_date_from &&
active_date_to && q
.Match(d => d
.Query("free text")
)
)
.From(0)
.Size(10)
);
}
public class Project
{
public Availibility Availablity { get; set; }
public Contact Contact { get; set; }
}
public class Contact
{
public string Name { get; set; }
}
public class Availibility
{
public DateTime AvailableFrom { get; set; }
public DateTime AvailableTo { get; set; }
}
Your current query generates
POST http://localhost:9200/projects/project/_search?pretty=true
{
"from": 0,
"size": 10,
"query": {
"bool": {
"must": [
{
"range": {
"availablity.availableFrom": {
"lte": "2017-07-21T10:01:01.456794+10:00",
"time_zone": "+01:00",
"format": "yyyy-MM-ddTHH:mm:SS||dd.MM.yyyy",
"_name": "from",
"boost": 1.1
}
}
},
{
"range": {
"availablity.availableTo": {
"gt": "2017-07-21T10:01:01.456794+10:00",
"time_zone": "+01:00",
"format": "yyyy-MM-ddTHH:mm:SS||dd.MM.yyyy",
"_name": "toDate",
"boost": 1.1
}
}
}
]
}
}
}
If a nameField is added as the field for the match query you get
POST http://localhost:9200/projects/project/_search?pretty=true
{
"from": 0,
"size": 10,
"query": {
"bool": {
"must": [
{
"range": {
"availablity.availableFrom": {
"lte": "2017-07-21T10:02:23.896385+10:00",
"time_zone": "+01:00",
"format": "yyyy-MM-ddTHH:mm:SS||dd.MM.yyyy",
"_name": "from",
"boost": 1.1
}
}
},
{
"range": {
"availablity.availableTo": {
"gt": "2017-07-21T10:02:23.896385+10:00",
"time_zone": "+01:00",
"format": "yyyy-MM-ddTHH:mm:SS||dd.MM.yyyy",
"_name": "toDate",
"boost": 1.1
}
}
},
{
"match": {
"contact.name": {
"query": "free text"
}
}
}
]
}
}
}
Remove InMemoryConnection from ConnectionSettings if you actually want to execute the query against Elasticsearch and see the results.
The range query is a structured query where a document either matches or doesn't match the predicate. Because of this, it can be wrapped in a bool query filter clause which will forgo calculating a score for it and perform better. Because no scoring occurs, boost is not needed.
Putting this together
var availableToField = Infer.Field<Project>(f => f.Availablity.AvailableTo);
var availableFromField = Infer.Field<Project>(f => f.Availablity.AvailableFrom);
var nameField = Infer.Field<Project>(f => f.Contact.Name);
var active_date_to = new DateRangeQuery
{
Name = "toDate",
Field = availableToField,
GreaterThan = DateTime.Now,
TimeZone = "+01:00",
Format = "yyyy-MM-ddTHH:mm:SS||dd.MM.yyyy"
};
var active_date_from = new DateRangeQuery
{
Name = "from",
Field = availableFromField,
LessThanOrEqualTo = DateTime.Now,
TimeZone = "+01:00",
Format = "yyyy-MM-ddTHH:mm:SS||dd.MM.yyyy"
};
var ret = client.Search<Project>(s => s
.Query(q =>
+active_date_from &&
+active_date_to && q
.Match(d => d
.Field(nameField)
.Query("free text")
)
)
.From(0)
.Size(10)
);
You may also want to explore modelling available from and to as a date_range type

Dynamically add named mapping to Index

I would like to add mappings to an index after I've created it. I've created the index as such:
client.CreateIndex("typeaheads", c => c
.Settings(t => t.Analysis(m => m.TokenFilters(fl => fl.EdgeNGram("edge_ngram_filter", ad => ad.MinGram(2).MaxGram(20)))
.Analyzers(anz => anz.Custom("edge_ngram_analyzer", an => an.Filters("lowercase", "edge_ngram_filter").Tokenizer("standard"))))));
The variable typeName, is the name I want for the mapping.
When I execute this:
var map = new CreateIndexDescriptor("typeaheads")
.Mappings(ms => ms
.Map(typeName, d => d.Properties(ps => ps.String(s => s.Name("countryCode")))
.Properties(ps => ps.String(s => s.Name("display_ID")))
.Properties(ps => ps.String(s => s.Name("display_String")))
.Properties(ps => ps.String(s => s.Name("id")))
.Properties(ps => ps.String(s => s.Name("languageCode")))
.Properties(ps => ps.String(s => s.Name("match_String").SearchAnalyzer("standard").Index(FieldIndexOption.Analyzed).Analyzer("edge_ngram_analyzer")))
.Properties(ps => ps.String(s => s.Name("type")))
.Properties(ps => ps.Number(s => s.Name("boostFactor").Type(NumberType.Long)))));
var response = client.Index(map);
I get this output on my ES service:
Wrong Mapping
I would like to get this: Correct Mapping
Any ideas?

If you have an existing index and wish to add a mapping to it, this can be done with the Put Mapping API, exposed in NEST as client.Map<T>() and client.MapAsync<T>()
var pool = new SingleNodeConnectionPool(new Uri("http://localhost:9200"));
var connectionSettings = new ConnectionSettings(pool);
var client = new ElasticClient(connectionSettings);
var typeName = "my-type";
var mappingResponse = client.Map<object>(d => d
.Index("typeaheads")
.Type(typeName)
.Properties(ps => ps
.String(s => s.Name("countryCode"))
.String(s => s.Name("display_ID"))
.String(s => s.Name("display_String"))
.String(s => s.Name("id"))
.String(s => s.Name("languageCode"))
.String(s => s
.Name("match_String")
.SearchAnalyzer("standard")
.Index(FieldIndexOption.Analyzed)
.Analyzer("edge_ngram_analyzer")
)
.String(s => s.Name("type"))
.Number(s => s
.Name("boostFactor")
.Type(NumberType.Long)
)
)
);
which sends the following request
PUT http://localhost:9200/typeaheads/my-type/_mapping?pretty=true
{
"properties": {
"countryCode": {
"type": "string"
},
"display_ID": {
"type": "string"
},
"display_String": {
"type": "string"
},
"id": {
"type": "string"
},
"languageCode": {
"type": "string"
},
"match_String": {
"type": "string",
"index": "analyzed",
"analyzer": "edge_ngram_analyzer",
"search_analyzer": "standard"
},
"type": {
"type": "string"
},
"boostFactor": {
"type": "long"
}
}
}

Linq query producing incorrect result

Consider the following linq query
var result = from a in
from b in filledTable
join c in distinctList on b[0].SerialNumber equals c.Field("SERIAL NUMBER")
select new { b, c }
group a by new { a.b[0].SerialNumber } into d
select new
{
Id = d.Select(x => x.b[0].Id),
SerialNumber = d.Select(x => x.b[0].SerialNumber),
// This part is not producing the correct output.
ImportTable = d.Select(w => w.c.Table
.AsEnumerable()
.GroupBy(y => y.Field("SERIAL NUMBER"))
.Select(z => z.First())
.CopyToData‌​Table())
};
filledTable in my linq query is a List<dynamic> which is populated by what the values are returned from a sproc and distinctList is a List<DataRow> which I distinct the values coming from the DataTable as follows:
List<DataRow> distinctList = dt.AsEnumerable().Distinct(DataRowComparer.Default).ToList();
My Linq query produces the following JSON
[
{
"FilledTableList":[
[
{
"Id":[
2
],
"SerialNumber":[
"1073410"
],
"ImportTable":[
[
{
"SERIAL NUMBER":"1073410",
"PRODUCT TYPE":"Product A"
},
{
"SERIAL NUMBER":"1073411",
"PRODUCT TYPE":"Product B"
}
]
]
},
{
"Id":[
-1
],
"SerialNumber":[
"1073411"
],
"ImportTable":[
[
{
"SERIAL NUMBER":"1073410",
"PRODUCT TYPE":"Proeduct A"
},
{
"SERIAL NUMBER":"1073411",
"PRODUCT TYPE":"Product B"
}
]
]
}
]
]
}]
But I would like the following JSON output
[
{
"FilledTableList":[
[
{
"Id":[
2
],
"SerialNumber":[
"1073410"
],
"ImportTable":[
[
{
"SERIAL NUMBER":"1073410",
"PRODUCT TYPE":"Product A"
}
]
]
},
{
"Id":[
-1
],
"SerialNumber":[
"1073411"
],
"ImporTable":[
[
{
"SERIAL NUMBER":"1073411",
"PRODUCT TYPE":"Product B"
}
]
]
}
]
]
}]
So the ImportTable node only contains the information matching to the serial number in the above FilleTabledList node. Everything else seems to work as expected by the Linq query apart from this. Can someone tell me where I'm going wrong please
Update:
My filledTable contains two items as follows:
{ Id = 2, SerialNumber = "1073410"}
{ Id = -1, SerialNumber = "1073411"}
Eventually I will have more items in the list but just to figure out why more linq query isn't working I have narrowed it down to just to items

I created a fiddle, which makes it easier to communicate the available data and the expected results.
When I understood it correctly you like to get a list of all products, listed in the filledTable and then find all elements with the same serial number from the dataTable.
If this is correct, than the LINQ query has to be:
var result = filledTable.GroupJoin(distinctList, product => product.SerialNumber, row => row.Field<string>("SERIAL NUMBER"), (Product, Rows) => new { Product, Rows })
.Select(group => new
{
Id = group.Product.Id,
SerialNumber = group.Product.SerialNumber,
ImportTable = group.Rows.CopyToDataTable()
});
and the result will be
[
{
"Id": 2,
"SerialNumber": "1073410",
"ImportTable": [
{
"SERIAL NUMBER": "1073410",
"PRODUCT TYPE": "Product A"
}
]
},
{
"Id": -1,
"SerialNumber": "1073411",
"ImportTable": [
{
"SERIAL NUMBER": "1073411",
"PRODUCT TYPE": "Product B"
}
]
}
]

I am not really sure, but would something like this work?
var result = (from a in (from b in filledTable join c in distinctList on b[0].SerialNumber equals c.Field<string>("SERIAL NUMBER") select new { b, c })
group a by new { a.b[0].SerialNumber } into d
select new
{
Id = d.Select(x => x.b[0].Id),
SerialNumber = d.Select(x => x.b[0].SerialNumber),
ImportTable = d.Select(w => w.c.Table.AsEnumerable()
.Where(y=>y.Field<string>("SERIAL NUMBER") == d.Key.ToString())
.GroupBy(y => y.Field<string>("SERIAL NUMBER")).Select(z => z.First()).CopyToData‌​Table())
});

Here is a simplified query that can be used:
var result =
from entry in filledTable
join row in distinctList on entry[0].SerialNumber equals row.Field<string>("SERIAL NUMBER")
group new { entry, row } by entry[0].SerialNumber into items
select new
{
Id = items.Select(x => x.entry[0].Id),
SerialNumber = new[] { items.Key }.AsEnumerable(),
ImportTable = items.Select(x => x.row).CopyToDataTable()
};
It should be equivalent to the desired output and deal with most strange data combinations that are handled by the original query.