Develop Reference

MongoDB C# Driver: Nested Lookups - How do I "join" nested relations?

I have 3 MongoDB collections that are related to each other:
Company
Store: a Company can have multiple Stores
Product: a Store can have multiple Products
Company
{
"_id": { "$oid": "1388445c0000000000000001" },
"name": "Company A",
"stores": [
{ "$oid": "1388445c0000000000000011" },
{ "$oid": "1388445c0000000000000012" }
]
}
Store
{
"_id": { "$oid": "1388445c0000000000000011" },
"name": "Store A",
"products": [
{ "$oid": "1388445c0000000000000021" },
{ "$oid": "1388445c0000000000000022" },
{ "$oid": "1388445c0000000000000023" }
]
}
Product
{
"_id": { "$oid": "1388445c0000000000000021" },
"name": "Product A"
}
If I use Lookup to "join" the first two collections, then the ObjectIds of the Stores are replaced with their corresponding objects from the Store collection:
db.GetCollection<BsonDocument>("Company")
.Aggregate()
.Lookup("Store", "stores", "_id", "stores")
.ToList();
{
"_id": { "$oid": "1388445c0000000000000001" },
"name": "Company A",
"stores": [
{
"_id": { "$oid": "1388445c0000000000000011" },
"name": "Store A",
"products": [
{ "$oid": "1388445c0000000000000021" },
{ "$oid": "1388445c0000000000000022" },
{ "$oid": "1388445c0000000000000023" }
]
},
...
]
}
But I'm struggling to "join" the Products on the nested Stores.
First I tried:
db.GetCollection<BsonDocument>("Company")
.Aggregate()
.Lookup("Store", "stores", "_id", "stores")
.Lookup("Product", "products", "_id", "products")
.ToList();
but obviously, it doesn't work as simple as that. Because the field products doesn't exist on Company, nothing happens.
If I try:
db.GetCollection<BsonDocument>("Company")
.Aggregate()
.Lookup("Store", "stores", "_id", "stores")
.Lookup("Product", "stores.products", "_id", "stores.products")
.ToList();
{
"_id": { "$oid": "1388445c0000000000000001" },
"name": "Company A",
"stores": {
"products": [
{
"_id": { "$oid": "1388445c0000000000000021" },
"name": "Product A"
},
...
]
}
}
then the products are "joined", but all other fields of the Store are gone. Furthermore the field stores is not an array anymore, but an object.
How do I correctly setup the aggregate pipeline with the MongoDB C# Driver to get the 3 collections "joined" so that I receive the following result:
{
"_id": { "$oid": "1388445c0000000000000001" },
"name": "Company A",
"stores": [
{
"_id": { "$oid": "1388445c0000000000000011" },
"name": "Store A",
"products": [
{
"_id": { "$oid": "1388445c0000000000000021" },
"name": "Product A"
},
...
]
}
]
}
Side note:
I'm working with BsonDocument and not a concrete C# type.

I think you should achieve with nested $lookup pipeline as below:
db.Company.aggregate([
{
"$lookup": {
"from": "Store",
"let": {
stores: "$stores"
},
"pipeline": [
{
$match: {
$expr: {
$in: [
"$_id",
"$$stores"
]
}
}
},
{
$lookup: {
"from": "Product",
let: {
products: { products: { $ifNull: [ "$products", [] ] } }
},
pipeline: [
{
$match: {
$expr: {
$in: [
"$_id",
"$$products"
]
}
}
}
],
as: "products"
}
}
],
"as": "stores"
}
}
])
Sample Mongo Playground
And convert the query to BsonDocument with MongoDB Compass.
var pipeline = new[]
{
new BsonDocument("$lookup",
new BsonDocument
{
{ "from", "Store" },
{ "let",
new BsonDocument("stores", "$stores")
},
{ "pipeline",
new BsonArray
{
new BsonDocument("$match",
new BsonDocument("$expr",
new BsonDocument("$in",
new BsonArray
{
"$_id",
"$$stores"
}
)
)
),
new BsonDocument("$lookup",
new BsonDocument
{
{ "from", "Product" },
{ "let",
new BsonDocument("products",
new BsonDocument("$ifNull",
new BsonArray
{
"$products",
new BsonArray()
}
)
)
},
{ "pipeline",
new BsonArray
{
new BsonDocument("$match",
new BsonDocument("$expr",
new BsonDocument("$in",
new BsonArray
{
"$_id",
"$$products"
}
)
)
)
}
},
{ "as", "products" }
}
)
}
},
{ "as", "stores" }
}
)
};
var result = _db.GetCollection<BsonDocument>("Company")
.Aggregate<BsonDocument>(pipeline)
.ToList();
Result

Thanx to #Yong Shun I have found the correct answer.
You can build the query also with MongoDB C# types as follows:
PipelineStageDefinition<BsonDocument, BsonDocument> stage = PipelineStageDefinitionBuilder.Lookup<BsonDocument, BsonDocument, BsonDocument, IEnumerable<BsonDocument>, BsonDocument>(
db.GetCollection<BsonDocument>("Store"),
new BsonDocument("stores", new BsonDocument("$ifNull", new BsonArray { "$stores", new BsonArray() })),
new PipelineStagePipelineDefinition<BsonDocument, BsonDocument>(new List<PipelineStageDefinition<BsonDocument, BsonDocument>>
{
PipelineStageDefinitionBuilder.Match(new BsonDocumentFilterDefinition<BsonDocument>(new BsonDocument("$expr", new BsonDocument("$in", new BsonArray { "$_id", "$$stores" })))),
PipelineStageDefinitionBuilder.Lookup<BsonDocument, BsonDocument, BsonDocument, IEnumerable<BsonDocument>, BsonDocument>(
db.GetCollection<BsonDocument>("Product"),
new BsonDocument("products", new BsonDocument("$ifNull", new BsonArray { "$products", new BsonArray() })),
new PipelineStagePipelineDefinition<BsonDocument, BsonDocument>(new List<PipelineStageDefinition<BsonDocument, BsonDocument>>
{
PipelineStageDefinitionBuilder.Match(new BsonDocumentFilterDefinition<BsonDocument>(new BsonDocument("$expr", new BsonDocument("$in", new BsonArray { "$_id", "$$products" })))),
}),
"products"
)
}),
"stores"
);
List<BsonDocument> result = db.GetCollection<BsonDocument>("Entity").Aggregate().AppendStage(stage).ToList();

Why do the NEST ElasticClient did not find a document?

When I use Kibana to execute the following Searchrequest to Elasticsearch
GET _search
{
"query": {
"query_string": {
"query": "PDB_W2237.docx",
"default_operator": "AND"
}
}
}
it returns:
{
"took": 14,
"timed_out": false,
"_shards": {
"total": 15,
"successful": 15,
"skipped": 0,
"failed": 0
},
"hits": {
"total": 1,
"max_score": 6.3527603,
"hits": [
{
"_index": "proconact",
"_type": "proconact",
"_id": "68cecf2c-7e5a-11e5-80fa-000c29bd9450",
"_score": 6.3527603,
"_source": {
"Id": "68cecf2c-7e5a-11e5-80fa-000c29bd9450",
"ActivityId": "1bad9115-7e5a-11e5-80fa-000c29bd9450",
"ProjectId": "08938a1d-2429-11e5-80f9-000c29bd9450",
"Filename": "PDB_W2237.docx"
}
}
]
}
}
When I use the NEST ElasticClient like
var client = new ElasticClient();
var searchResponse = client.Search<Hit>(new SearchRequest {
Query = new QueryStringQuery {
Query = "DB_W2237.docx",
DefaultOperator = Operator.And
}
});
it do return 0 Hits.
Here is the Indexmapping for the 4 Fields in the Hit:
{
"proconact": {
"mappings": {
"proconact": {
"properties": {
"ActivityId": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"Filename": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"Id": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"ProjectId": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
}
}
}
}
Are the two search-requests not the same?

The problem is your mapping doesn't allow a token different than whatever is present in your index.
In your kibana query:
GET _search
{
"query": {
"query_string": {
"query": "PDB_W2237.docx",
"default_operator": "AND"
}
}
}
You're querying PDB_W2237.docx but in your NEST you're querying DB_W2237.docx.
If you want to query DB_W2237.docx and expecting results then you might have to change the analyzer from standard analyzer which is applied by default to something else a possible candidate depends on your usecase.

Normalise JSON for flexible WebAPI response

I have a WebAPI method that returns Json in a flexible structure that depends on the request.
Part of the problem is that there could be any number of columns, and they could be any type. The 2 given below (Code and Count) are just one example.
This structure is based on the underlying classes but there could be any number of columns in the output. So, rather than the usual properties you might expect, these are objects in a collection with Name and Value properties.
The downside of this flexible approach is that it gives a non-standard format.
Is there a way to transform this into a more normalised shape? Are there maybe some attributes I can add to the class properties to change the way they are serialised?
For example, where there are 2 columns - Code (string) and Count (numeric):
Current Json:
{
"Rows": [
{
"Columns": [
{
"Value": "1",
"Name": "Code"
},
{
"Value": 13,
"Name": "Count"
}
]
},
{
"Columns": [
{
"Value": "2",
"Name": "Code"
},
{
"Value": 12,
"Name": "Count"
}
]
},
{
"Columns": [
{
"Value": "9",
"Name": "Code"
},
{
"Value": 1,
"Name": "Count"
}
]
},
{
"Columns": [
{
"Value": "5",
"Name": "Code"
},
{
"Value": 2,
"Name": "Count"
}
]
}
]
}
Ideally I'd like to transform it to this:
{
"Rows": [
{
"Code": "1",
"Count": 13
},
{
"Code": "2",
"Count": 12
},
{
"Code": "9",
"Count": 1
},
{
"Code": "5",
"Count": 2
}
]
}
The controller method (C#)
public ReportResponse Get(ReportRequest request)
{
var result = ReportLogic.GetReport(request);
return result;
}
The output classes
public class ReportResponse
{
public List<ReportRow> Rows { get; set; }
public ReportResponse()
{
Rows = new List<ReportRow>();
}
}
public class ReportRow
{
public List<ReportColumn> Columns { get; set; }
public ReportRow()
{
Columns = new List<ReportColumn>();
}
}
public class ReportColumn<T> : ReportColumn
{
public T Value { get; set; }
public ReportColumn(string name)
{
Name = name;
}
}
public abstract class ReportColumn
{
public string Name { get; internal set; }
}

I think the easiest way would be to map your class to a dictionary before serializing. Something like:
var dictionaries = List<Dictionary<string, object>();
foreach(var column in rows.Columns)
{
dictionaries.Add(new Dictionary<string, object>{{column.Name, column.Value}});
}
Then serialize the dictionaries variable should do the trick.

If you're using the output in JavaScript, you could translate as follows:
var
data = {
"Rows": [
{
"Columns": [
{
"Value": "1",
"Name": "Code"
},
{
"Value": 13,
"Name": "Count"
}
]
},
{
"Columns": [
{
"Value": "2",
"Name": "Code"
},
{
"Value": 12,
"Name": "Count"
}
]
},
{
"Columns": [
{
"Value": "9",
"Name": "Code"
},
{
"Value": 1,
"Name": "Count"
}
]
},
{
"Columns": [
{
"Value": "5",
"Name": "Code"
},
{
"Value": 2,
"Name": "Count"
}
]
}
]
},
output = [
];
data.Rows.forEach(function (row)
{
var
newRow = {};
row.Columns.forEach(function (column)
{
newRow[column.Name] = column.Value;
});
output.push(newRow);
})
console.log(JSON.stringify(output));

Extracting elements wherever JArray in json

Lets see, the json can be dynamic and can probably have number of nested arrays within any property.
Example:
{
"items": [
{
"id": "0001",
"name": "Cake",
"batters": {
"batter": [
{
"id": "1001",
"type": "Regular"
},
{
"id": "1002",
"type": "Chocolate"
},
{
"dry": [
{
"id": "1003",
"type": "Devil's Food"
}
]
}
],
"other": [
{
"id": "1004",
"type": "Home Food"
}
]
},
"topping": [
{
"id": "5002",
"type": "Glazed"
},
{
"id": "5005",
"type": "Sugar"
}
]
},
{
"id": "0002",
"name": "Sweets"
}
]
}
A simple list should return elements as:
[
{
"id": "1001",
"type": "Regular"
},
{
"id": "1002",
"type": "Chocolate"
},
{
"id": "1003",
"type": "Devil's Food"
},
{
"id": "1004",
"type": "Home Food"
},
{
"id": "5002",
"type": "Glazed"
},
{
"id": "5005",
"type": "Sugar"
},
{
"id": "0002",
"name": "Sweets"
}
]
Please note:
Json can by anything, no property can be used for extraction , just knowing that what needed is stuff inside an JArray.
What i have tried so far but its just a start:
public static bool ParseJsonArray(JToken token, List<string> extracts, string parentLocation = "")
{
if (token.HasValues)
{
foreach (JToken child in token.Children())
{
if (token.Type == JTokenType.Array)
{
parentLocation += ((JProperty)token).Name;
extracts.Add(token.ToString());
}
ParseJsonArray(child, extracts, parentLocation);
}
return true;
}
else
{
return false;
}
}
token here is the parsed dynamic json.

It appears as though you want to recursively find all JArray entries that do not themselves contain nested arrays. Let's call these "leaf" array entries. I say that because you don't include the following non-leaf entry in your results:
{
"id": "0001",
"name": "Cake"
}
That being said, you can find leaf array entries with the following extension method:
public static class JsonExtensions
{
public static IEnumerable<JToken> LeafArrayEntries(this JContainer container)
{
var nonLeafEntries = new HashSet<JToken>(container.DescendantsAndSelf()
.OfType<JArray>()
.SelectMany(a => a.Ancestors().Where(p => p.Type != JTokenType.Property)));
return container.DescendantsAndSelf().Where(c => c.Parent is JArray && !nonLeafEntries.Contains(c));
}
}
Then put the returned items in an array of their own with:
var leafItemArray = new JArray(rootJContainer.LeafArrayEntries());

Is Anything wrong in my query .Nest elastic C#

Is there something wrong in my .Nest libs query? My query will get all data, I need to get by multi term.
Query string elastic result i want:
{
"took": 2,
"timed_out": false,
"_shards": {
"total": 5,
"successful": 5,
"failed": 0
},
"hits": {
"total": 1000,
"max_score": 0,
"hits": []
},
"aggregations": {
"log_query": {
"doc_count": 2,
"histogram_Log": {
"buckets": [
{
"key_as_string": "06/02/2015 12:00:00",
"key": 1423180800000,
"doc_count": 1
},
{
"key_as_string": "21/02/2015 12:00:00",
"key": 1424476800000,
"doc_count": 1
}
]
}
}
}
}
My query string elastic:
{
"size": 0,
"aggs": {
"log_query": {
"filter": {
"bool": {
"must": [
{
"term": {
"cluster": "giauht1"
}
},
{
"term": {
"server": "hadoop0"
}
},
{
"term": {
"type": "Warn"
}
},
{
"range": {
"actionTime": {
"gte": "2015-02-01",
"lte": "2015-02-24"
}
}
}
]
}
},
"aggs": {
"histogram_Log": {
"date_histogram": {
"field": "actionTime",
"interval": "1d",
"format": "dd/MM/YYYY hh:mm:ss"
}
}
}
}
}
}
My .nest libs query:
Func<SearchDescriptor<LogInfoIndexView>, SearchDescriptor<LogInfoIndexView>> query =
que => que.Aggregations(aggs => aggs.Filter("log_query", fil =>
{
fil.Filter(fb => fb.Bool(fm => fm.Must(
ftm =>
{
ftm.Term(t => t.Cluster, cluster);
ftm.Term(t => t.Server, server);
ftm.Term(t => t.Type, logLevel);
ftm.Range(r => r.OnField("actionTime").GreaterOrEquals(from.Value).LowerOrEquals(to.Value));
return ftm;
}))).Aggregations(faggs => faggs.DateHistogram("histogram_Log", dr =>
{
dr.Field("actionTime");
dr.Interval("1d");
dr.Format("dd/MM/YYYY hh:mm:ss");
return dr;
}));
return fil;
})).Size(0).Type(new LogInfoIndexView().TypeName);
var result = client.Search(query);
My .nest result:
My model mapping:
{
"onef-sora": {
"mappings": {
"FPT.OneF.Api.Log": {
"properties": {
"actionTime": {
"type": "date",
"format": "dateOptionalTime"
},
"application": {
"type": "string",
"index": "not_analyzed"
},
"cluster": {
"type": "string",
"index": "not_analyzed"
},
"detail": {
"type": "string",
"index": "not_analyzed"
},
"iD": {
"type": "string"
},
"message": {
"type": "string",
"index": "not_analyzed"
},
"server": {
"type": "string",
"index": "not_analyzed"
},
"source": {
"type": "string",
"index": "not_analyzed"
},
"tags": {
"type": "string",
"index": "not_analyzed"
},
"type": {
"type": "string",
"index": "not_analyzed"
},
"typeLog": {
"type": "string"
},
"typeName": {
"type": "string"
},
"url": {
"type": "string",
"index": "not_analyzed"
},
"user": {
"type": "string",
"index": "not_analyzed"
}
}
}
}
}
}

The Must() condition passed to the Bool() filter takes a params Func<FilterDescriptor<T>, FilterContainer>[] but in your filter, the Term() and Range() filters are chained onto the same filter instance; unfortunately, this doesn't work as you might expect and the end result is actually an empty json object passed to the must clause in the query DSL for the filter i.e. you end up with
{
"size": 0,
"aggs": {
"log_query": {
"filter": {
"bool": {
"must": [
{} /* where are the filters?! */
]
}
},
"aggs": {
"histogram_Log": {
"date_histogram": {
"field": "actionTime",
"interval": "1d",
"format": "dd/MM/YYYY hh:mm:ss"
}
}
}
}
}
}
The solution is to pass an array of Func<FilterDescriptor<T>, FilterContainer>; The following matches your query DSL
void Main()
{
var settings = new ConnectionSettings(new Uri("http://localhost:9200"));
var connection = new InMemoryConnection(settings);
var client = new ElasticClient(connection: connection);
DateTime? from = new DateTime(2015, 2,1);
DateTime? to = new DateTime(2015, 2, 24);
var docs = client.Search<LogInfoIndexView>(s => s
.Size(0)
.Type("type")
.Aggregations(a => a
.Filter("log_query", f => f
.Filter(ff => ff
.Bool(b => b
.Must(m => m
.Term(t => t.Cluster, "giauht1"),
m => m
.Term(t => t.Server, "hadoop0"),
m => m
.Term(t => t.Type, "Warn"),
m => m
.Range(r => r.OnField("actionTime").GreaterOrEquals(from.Value).LowerOrEquals(to.Value))
)
)
)
.Aggregations(aa => aa
.DateHistogram("histogram_Log", da => da
.Field("actionTime")
.Interval("1d")
.Format("dd/MM/YYYY hh:mm:ss")
)
)
)
)
);
Console.WriteLine(Encoding.UTF8.GetString(docs.RequestInformation.Request));
}
public class LogInfoIndexView
{
public string Cluster { get; set; }
public string Server { get; set; }
public string Type { get; set; }
public DateTime ActionTime { get; set; }
}
returning
{
"size": 0,
"aggs": {
"log_query": {
"filter": {
"bool": {
"must": [
{
"term": {
"cluster": "giauht1"
}
},
{
"term": {
"server": "hadoop0"
}
},
{
"term": {
"type": "Warn"
}
},
{
"range": {
"actionTime": {
"lte": "2015-02-24T00:00:00.000",
"gte": "2015-02-01T00:00:00.000"
}
}
}
]
}
},
"aggs": {
"histogram_Log": {
"date_histogram": {
"field": "actionTime",
"interval": "1d",
"format": "dd/MM/YYYY hh:mm:ss"
}
}
}
}
}
}
EDIT:
In answer to your comment, the difference between a filtered query filter and a filter aggregation is that the former applies the filtering to all documents at the start of the query phase and filters are generally cached, improving performance on subsequent queries with those filters, whilst the latter applies in the scope of the aggregation to filter documents in the current context to a single bucket. If your query is only to perform the aggregation and you're likely to run the aggregation with the same filters, I think the filtered query filter should offer better performance.