Develop Reference

Applying Term, Filter and Aggregation in one Es Query using Nest

I'm using ElasticSearch's Nest 7.17.x to try and applying a Term, Filter and Aggregation all on single query.
The Term is a document id restriction which should restrict all following filters and aggregations. The Filter is to restriction the price to a specific range. The aggregation to is to make price buckets.
In the example below, is looks like both the price filter and term restriction arent being applied as I'm being returned documents with ids outside of the restriction and with prices larger than the filter.
var orderChangedArgFilter = client.Search<Product>(s => s
.Query(q => +q.Terms(p => p.Field("id").Terms(productIds)))
.Aggregations(aggs => aggs
.Filter("user filter with aggs", f => f
.Filter(q => q.Range(rf => rf.Field("price").GreaterThanOrEquals(0.01).LessThan(50.0)))
.Aggregations(childAggs => childAggs
.Range("0 to 50 price agg", r => r.Field("price").Ranges(rs => rs.From(0.0).To(50.0)))
.Range("50 to 100 price agg", r => r.Field("price").Ranges(rs => rs.From(50.0).To(100.0)))
.Range("100 to 150 price agg", r => r.Field("price").Ranges(rs => rs.From(100.0).To(150.0)))
)
)
)
);
How do I correct the query to have the Term restriction apply first and then the filter on top of it?
Edit 1:
It looks like the document Id term restriction is working as expected but the filter is not.
My engine was created via AppSearch. Looks like the datatype for the price field is actually text instead of number (on the abstracted elastic engine), even though I've specified number on the AppSearch engine. This seems to be the cause of the problem.
The raw index mappings
{
".ent-search-engine-documents-luke-test": {
"mappings": {
"dynamic": "true",
"properties": {
"price": {
"fields": {
"prefix": {
"search_analyzer": "q_prefix",
"type": "text",
"analyzer": "i_prefix",
"index_options": "docs"
},
"enum": {
"ignore_above": 2048,
"type": "keyword"
},
"float": {
"ignore_malformed": true,
"type": "double"
},
"joined": {
"search_analyzer": "q_text_bigram",
"type": "text",
"analyzer": "i_text_bigram",
"index_options": "freqs"
},
"stem": {
"type": "text",
"analyzer": "iq_text_stem"
},
"delimiter": {
"type": "text",
"analyzer": "iq_text_delimiter",
"index_options": "freqs"
},
"location": {
"ignore_malformed": true,
"type": "geo_point",
"ignore_z_value": false
},
"date": {
"ignore_malformed": true,
"type": "date",
"format": "strict_date_time||strict_date"
}
},
"type": "text",
"analyzer": "iq_text_base",
"index_options": "freqs"
},
"id": {
"type": "keyword"
},
"title": {
"fields": {
"prefix": {
"search_analyzer": "q_prefix",
"type": "text",
"analyzer": "i_prefix",
"index_options": "docs"
},
"enum": {
"ignore_above": 2048,
"type": "keyword"
},
"float": {
"ignore_malformed": true,
"type": "double"
},
"joined": {
"search_analyzer": "q_text_bigram",
"type": "text",
"analyzer": "i_text_bigram",
"index_options": "freqs"
},
"stem": {
"type": "text",
"analyzer": "iq_text_stem"
},
"delimiter": {
"type": "text",
"analyzer": "iq_text_delimiter",
"index_options": "freqs"
},
"location": {
"ignore_malformed": true,
"type": "geo_point",
"ignore_z_value": false
},
"date": {
"ignore_malformed": true,
"type": "date",
"format": "strict_date_time||strict_date"
}
},
"type": "text",
"analyzer": "iq_text_base",
"index_options": "freqs"
}
},
"dynamic_templates": [
{
"permissions": {
"mapping": {
"type": "keyword"
},
"match": "_*_permissions"
}
},
{
"thumbnails": {
"mapping": {
"type": "binary"
},
"match": "_thumbnail_*"
}
},
{
"data": {
"match_mapping_type": "*",
"mapping": {
"fields": {
"enum": {
"ignore_above": 2048,
"type": "keyword"
},
"float": {
"ignore_malformed": true,
"type": "double"
},
"delimiter": {
"type": "text",
"index_options": "freqs",
"analyzer": "iq_text_delimiter"
},
"joined": {
"search_analyzer": "q_text_bigram",
"type": "text",
"index_options": "freqs",
"analyzer": "i_text_bigram"
},
"prefix": {
"search_analyzer": "q_prefix",
"type": "text",
"index_options": "docs",
"analyzer": "i_prefix"
},
"location": {
"ignore_malformed": true,
"type": "geo_point",
"ignore_z_value": false
},
"date": {
"ignore_malformed": true,
"type": "date",
"format": "strict_date_time||strict_date"
},
"stem": {
"type": "text",
"analyzer": "iq_text_stem"
}
},
"type": "text",
"index_options": "freqs",
"analyzer": "iq_text_base"
}
}
}
]
}
}
}
Is it possible to still use the Nest client against an engine created via AppSearch?

My query had a couple of issues. Firstly I should have been using field name price.float which is the numeric representation of the price field on ES vs AppSearch. You could figure this out from the Explain endpoint offered by AppSearch.
The next issue was the structure of my query, I shouldn't have been using a FilteredAggregation. My intent was to not apply the filter specifically onto the aggregation, but to the query result.
Lastly, retrieving the results from the Aggregations was not that straight forward. Dumping the contents to console falsely shows an empty aggregation, however upon investigating the debugging info via the .EnableDebugMode() header, I was able to see that I was indeed receiving results. Applying a breakpoint and inspecting the results of the object allowed me to retrieve the object I wanted.
var orderChangedArgFilter = client.Search<Product>(s => s
.Query(q => +q.Range(rf => rf.Field("price.float").GreaterThanOrEquals(0.01).LessThan(50.0)) && +q.Terms(p => p.Field("id").Terms(productIds)))
.Aggregations(a => a
.Range("Price aggs", r => r
.Field("price.float")
.Ranges(rs =>
rs.From(0.01).To(50.0).Key("0 to 50 price agg"),
rs => rs.From(50.01).To(100).Key("50 to 100 price agg"),
rs => rs.From(100.01).Key("From 100 price agg")))
)
);
orderChangedArgFilter.Aggregations.Dump();
// vs
var agg = (Nest.BucketAggregate)orderChangedArgFilter.Aggregations.GetValueOrDefault("Price aggs");
var buckets = agg.Items.Select(b => (RangeBucket)b).ToList();
Console.WriteLine(buckets[0].Key + ", " + buckets[0].DocCount);
Note the line of text at the bottom of the below screen shot showing the Key and DocCount

Parsing Dynamic JSON in C#

I am working with an API that uses GraphQL and returns JSON data and I want to deserialize the JSON content dynamically rather than map the entire service structure in my application into C# data structures (.NET types/classes).
I've done this before but the JSON data was quite simple and straightforward. However, the this API I'm working with returns more complex/verbose JSON and I'm struggling to parse it properly.
Can anyone advise on how to parse the below JSON? That is, how to access the dynamic JSON's objects/properties.
I'm using Newtonsoft.Json. This is my code to deserialize:
byte bC = 0;
string location_text = string.Empty;
string location_value = string.Empty;
dynamic results = JsonConvert.DeserializeObject<dynamic>(json);
foreach (dynamic d in results)
{
bC = d.data.locationSuggestions.Count; // d.locationSuggestions.Count
for (byte b = 0; b < bC; b++)
{
location_text = d.data.locationSuggestions[b].location.contextualName;
location_value = d.data.locationSuggestions[b].location.id.value;
}
}
When I run this code I get an error that says "data" or "locationSuggestions" cannot be found.
The JSON I'm trying to parse is as follows:
{
"data": {
"locationSuggestions": [
{
"location": {
"id": {
"value": "seekAnzPublicTest:location:seek:2dkY4vZJF"
},
"name": "Brisbane",
"contextualName": "Brisbane QLD 4000 AU",
"countryCode": "AU",
"parent": {
"id": {
"value": "seekAnzPublicTest:location:seek:2FjxhQans"
},
"name": "CBD & Inner Suburbs",
"parent": {
"id": {
"value": "seekAnzPublicTest:location:seek:32XZdsa2K"
},
"name": "Brisbane"
}
}
}
},
{
"location": {
"id": {
"value": "seekAnzPublicTest:location:seek:2ckmimaF1"
},
"name": "Brisbane Grove",
"contextualName": "Brisbane Grove NSW 2580 AU",
"countryCode": "AU",
"parent": {
"id": {
"value": "seekAnzPublicTest:location:seek:2UC23LszP"
},
"name": "Southern Highlands & Tablelands",
"parent": {
"id": {
"value": "seekAnzPublicTest:location:seek:FTwZdE2K"
},
"name": "New South Wales"
}
}
}
}
]
},
"extensions": {
"requestLatency": 171
}
}

Regarding your JSON string, you can use dynamic to parse it and access the data as shown below:
You can also find a working example at: https://dotnetfiddle.net/lnhwJz
using System;
using Newtonsoft.Json;
public class Program
{
public static void Main()
{
var jsonString=#"{'data':{'locationSuggestions':[{'location':{'id':{'value':'seekAnzPublicTest:location:seek:2dkY4vZJF'},'name':'Brisbane','contextualName':'Brisbane QLD 4000 AU','countryCode':'AU','parent':{'id':{'value':'seekAnzPublicTest:location:seek:2FjxhQans'},'name':'CBD & Inner Suburbs','parent':{'id':{'value':'seekAnzPublicTest:location:seek:32XZdsa2K'},'name':'Brisbane'}}}},{'location':{'id':{'value':'seekAnzPublicTest:location:seek:2ckmimaF1'},'name':'Brisbane Grove','contextualName':'Brisbane Grove NSW 2580 AU','countryCode':'AU','parent':{'id':{'value':'seekAnzPublicTest:location:seek:2UC23LszP'},'name':'Southern Highlands & Tablelands','parent':{'id':{'value':'seekAnzPublicTest:location:seek:FTwZdE2K'},'name':'New South Wales'}}}}]},'extensions':{'requestLatency':171}}";
var parsedData=JsonConvert.DeserializeObject<dynamic>(jsonString);
foreach(var item in parsedData.data.locationSuggestions)
{
Console.WriteLine(item.location.name);
Console.WriteLine(item.location.id.value);
Console.WriteLine(item.location.contextualName);
}
}
}
Output:
Brisbane
seekAnzPublicTest:location:seek:2dkY4vZJF
Brisbane QLD 4000 AU
Brisbane Grove
seekAnzPublicTest:location:seek:2ckmimaF1
Brisbane Grove NSW 2580 AU

JSON - LINQ Query for object or an array to group two data with same key value pair

Consider I have a JSON data as:
{
"entities":[
{
"republish": false,
"OrgID": "",
"createdby": "730",
"questions": [
{
"sequence": "5",
"QuestionId": "57BB6DDC-A90A-10EE-E224-EC658A825871",
"metadata": [
{
"key": "Group",
"value": 0
},
{
"key": "Part",
"value": "0"
}
]
},
{
"sequence": "4",
"QuestionId": "57BB6DDC-A90A-10EE-E224-EC658A825871",
"metadata": [
{
"key": "Group",
"value": 1
},
{
"key": "Part",
"value": "A"
}
]
},
{
"sequence": "3",
"QuestionId": "57BB6DDC-A90A-10EE-E224-EC658A825871",
"metadata": [
{
"key": "Group",
"value": 1
},
{
"key": "Part",
"value": "B"
}
]
}
]
}
]
}
As you can see I have a list of questions available and in each question, I have a metadata which holds Key-Value pair.
Above example demonstrates, I have 3 questions and out of it 2 question, metadata key-value is "Group 1".
Now I want to do is combine the questions with a same key-value pair and treat it as one.
so in my final case, I will have 2 questions instead of 3. And out of that one question will have two separate questions inside.
And I want to achieve this using Linq query. And if possible please use Newtonsoft for parse if needed. I have been stuck for long onto this.
Things I have done:
public virtual HttpResponseMessage AddQuestionsToStandardMaster(TaxonomyMetaData objQuestion)
{
List<ResponseEntity> objResponseList = new List<ResponseEntity>();
try
{
if (ModelState.IsValid)
{
foreach (var objEntity in objQuestion.Entities)
{
EntityResponse objentityresponse = new EntityResponse();
ResponseEntity objResponse = new ResponseEntity();
}
List<Question> objQuestionList = new List<Question>();
if (objEntity.Questions.Length > 0)
{
foreach (var item in objEntity.Questions)
{
int questionTypeid = 0;
dynamic objQuestionJson = JObject.Parse(item.ToString())
}
}
}
Question objCurrentQuestion = new Question();
Question objQuestionforDelete = new Question();
JObject itemToParese = new JObject();
string SingleQuestionJson = objQuestionJson.GetValue("QuestionData").ToString();
string questionstem = "";
Regex rgx = new Regex("/\'");
objCurrentQuestion.Sequence = Convert.ToInt32(objQuestionJson.GetValue("sequence"));
objCurrentQuestion.tag = objQuestionJson.tag.ToObject<JToken[]>(); ;
objCurrentQuestion.metadata = objQuestionJson.metadata.ToObject<JToken[]>();
objCurrentQuestion.SingleQuestionJson = rgx.Replace(SingleQuestionJson, "'");
objCurrentQuestion.QuestionsType = questionTypeid;
objCurrentQuestion.QuestionsId = new Guid(objQuestionJson.GetValue("QuestionId").ToString());
objCurrentQuestion.VersionNo = Convert.ToInt32(objQuestionJson.GetValue("VersionNo"));
objCurrentQuestion.DisplayQuestionId = Convert.ToString(objQuestionJson.GetValue("DisplayQuestionId"));
objCurrentQuestion.OriginalQuestionId = Convert.ToString(objQuestionJson.GetValue("OriginalQuestionId"));
objCurrentQuestion.PassageText = Convert.ToString(objQuestionJson.GetValue("passage_text"));
objCurrentQuestion.PassageCode = Convert.ToString(objQuestionJson.GetValue("passage_id"));
objCurrentQuestion.PassageTitle = Convert.ToString(objQuestionJson.GetValue("passage_title"));
objCurrentQuestion.IsPublished = Convert.ToByte(true);
objCurrentQuestion.ProductId = objEntity.ProductID;
foreach (var metadata in objCurrentQuestion.metadata)
{
switch (metadata["key"].ToString())
{
case "Group":
objCurrentQuestion.Group = Convert.ToInt32(metadata["value"].ToString());
break;
case "Part":
objCurrentQuestion.Part = metadata["value"].ToString();
break;
}
}
objQuestionList.Add(objCurrentQuestion);
int counter = 1;
//Here I get the data in a group which needs to coverted to JSOn and then replace the original JSON data with this. But I know this is irrelevant to what we need to achieve.
var yui = objQuestionList.Where(tma => tma.Group == counter).Select(t => t).GroupBy(s => new { s.Group }).Where(p => p.Count() > 1).ToList();
//After proper conversion I need to enter this data to a database.

I am a bit unclear on what do you mean by make it
combine the questions with a same key-value pair and treat it as one
But here is how you can group by the JSON into groups in metadata you can do a custom select to what ever you want.
var text = #"{
""entities"":[
{
""republish"": false,
""OrgID"": """",
""createdby"": ""730"",
""questions"": [
{
""sequence"": ""5"",
""QuestionId"": ""57BB6DDC-A90A-10EE-E224-EC658A825871"",
""metadata"": [
{
""key"": ""Group"",
""value"": 0
},
{
""key"": ""Part"",
""value"": ""0""
}
]
},
{
""sequence"": ""4"",
""QuestionId"": ""57BB6DDC-A90A-10EE-E224-EC658A825871"",
""metadata"": [
{
""key"": ""Group"",
""value"": 1
},
{
""key"": ""Part"",
""value"": ""A""
}
]
},
{
""sequence"": ""3"",
""QuestionId"": ""57BB6DDC-A90A-10EE-E224-EC658A825871"",
""metadata"": [
{
""key"": ""Group"",
""value"": 1
},
{
""key"": ""Part"",
""value"": ""B""
}
]
}
]
}
]
}";
var json = JObject.Parse(text);
var groupedData = from entity in json["entities"]
from question in entity["questions"]
group question by question["metadata"][0]["value"] into questionGroup
select questionGroup;
foreach (var data in groupedData)
{
Console.WriteLine("_____________________");
Console.WriteLine("Group");
Console.WriteLine(data.Key);
foreach (var question in data)
{
Console.WriteLine(question["QuestionId"]);
}
Console.WriteLine("_____________________");
}
If "Group" is not always the fist item in the array you can do
question["metadata"].First(md => md.Value<string>("key") == "Group")["value"]
to get it.

get Specific Id from json file

this is my json file
{
{
"#odata.context": "https://api.onedrive.com/v1.0/$metadata#drives('me')/items('root')/children/$entity",
"createdBy": {
"application": {
"displayName": "Nopbackup",
"id": "4c190e01"
},
"user": {
"displayName": "pallav jha",
"id": "611c19eb038d5aa1"
}
},
"createdDateTime": "2016-05-12T07:25:36.463Z",
"cTag": "adDo2MTFDMTlFQjAzOEQ1QUExITEyNC42MzU5ODYzODk5MTI3MDAwMDA",
"eTag": "aNjExQzE5RUIwMzhENUFBMSExMjQuMw",
"id": "611C19EB038D5AA1!124",
"lastModifiedBy": {
"application": {
"displayName": "Nopbackup",
"id": "4c190e01"
},
"user": {
"displayName": "pallav jha",
"id": "611c19eb038d5aa1"
}
},
"lastModifiedDateTime": "2016-05-12T08:36:31.27Z",
"name": "Nopbackup",
"parentReference": {
"driveId": "611c19eb038d5aa1",
"id": "611C19EB038D5AA1!105",
"path": "/drive/root:"
},
"size": 0,
"webUrl": "https://onedrive.live.com/redir?resid=611C19EB038D5AA1!124",
"fileSystemInfo": {
"createdDateTime": "2016-05-12T07:25:36.463Z",
"lastModifiedDateTime": "2016-05-12T08:36:31.27Z"
},
"folder": {
"childCount": 0
}
}
}
i want to get this "id": "4c190e01" from json
dynamic value = Newtonsoft.Json.JsonConvert.DeserializeObject(result);
string id = Convert.ToString(value.id[0]);
this is my code but i am not getting 4c190e01 id

rewrite your code as following...
dynamic valuePoco = Newtonsoft.Json.JsonConvert.DeserializeObject(result);
string id = Convert.ToString(valuePoco.createdBy.application.id);

You can use the path to the desired property. If you want to select the element with 4c190e01 as id, you can use
string id = value.createdBy.application.id;
or
string id = value.lastModifiedBy.application.id;
depending on your needs.
As it was pointed out, you'd need to fix your JSON first and remove the first { and the last }

First thing first, Your json is invalid.
You've to remove one { from top and } from the bottom and then use the following code:
dynamic jsonObj = JsonConvert.DeserializeObject(result);
string id = jsonObj.createdBy.application.id.ToString();

how to boost nested object with NEST

I have mapped my index as below. Simply I have a product index with properties Id, Number,ManufactureNumber,shortDescription, name and nested object like SubProduct
"mappings": {
"Product": {
"properties": {
"id": { "index": "no","store": true,"type": "integer"},
"name": { "store": true,"type": "string"},
"image": { "properties": { "fileName": { "index": "no","store": true,"type": "string"},"virtualPath": { "index": "no","store": true,"type": "string"}}},
"number": { "index": "not_analyzed","store": true,"type": "string"},
"manufactureNumber": { "index": "not_analyzed","store": true,"type": "string"},
"subProduct": { "type": "nested","properties": { "name": { "store": true,"type": "string"},"number": { "index": "not_analyzed", "store": true,"type": "string"},"Id": { "index": "no","store": true,"type": "integer"}}}
}
}
}
what I want here is to search a keyword within properties of name, Number, ManufactureNumber,shortDescription and SubProduct.name, SubProduct.number. So if keyword is found any of this document should be returned with following priorities (scores)
manufactureNumber = 5.0
number =4.0
name=2.0
subproduct.number=2.0
subproduct.name =1.0
Based on those requirements and after my research, I thought multimatch query with boosting is my only option. Am I correct on this? or any query_string query can do it as well?
This is how I tried but I got stuck in the part with the nested object. I dont know how to boost them? below code will give error that number and name are not properties of subproduct.
var results = Client.Search<Product> (body => body.Query(query => query.MultiMatch(qs =>
qs.OnFieldsWithBoost(d => d.Add(entry => entry.manufactureNumber , 5.0)
.Add(entry => entry.number , 4.0)
.Add(entry => entry.name,3.0)
.Add(entry => entry.subproduct.number,2.0)
.Add(entry => entry.subproduct.name,1.0)
).Type(TextQueryType.BestFields).Query(key))));

You need to use Nested Query when querying on nested fields. In your case, they are subProduct.number and subProduct.name. The query you might be interested in is as under:
{
"query": {
"bool": {
"should": [
{
"multi_match": {
"type": "best_fields",
"query": "key",
"fields": [
"manufactureNumber^5",
"number^4",
"name^3"
]
}
},
{
"nested": {
"query": {
"multi_match": {
"type": "best_fields",
"query": "key",
"fields": [
"subProduct.number^2",
"subProduct.name^1"
]
}
},
"path": "subProduct"
}
}
]
}
}
}
The corresponding Nest query is as under:
var results = client.Search<Product>(s => s
.Query(q => q
.Bool(b => b
.Should(
sh => sh.MultiMatch(qs => qs
.OnFieldsWithBoost(d => d
.Add("manufactureNumber", 5.0)
.Add("number", 4.0)
.Add("name", 3.0))
.Type(TextQueryType.BestFields)
.Query(key)),
sh => sh.Nested(n => n
.Path("subProduct")
.Query(nq => nq
.MultiMatch(qs => qs
.OnFieldsWithBoost(d => d
.Add("subProduct.number", 2.0)
.Add("subProduct.name", 1.0))
.Type(TextQueryType.BestFields)
.Query(key))))))));