Is there something wrong in my .Nest libs query? My query will get all data, I need to get by multi term.
Query string elastic result i want:
{
"took": 2,
"timed_out": false,
"_shards": {
"total": 5,
"successful": 5,
"failed": 0
},
"hits": {
"total": 1000,
"max_score": 0,
"hits": []
},
"aggregations": {
"log_query": {
"doc_count": 2,
"histogram_Log": {
"buckets": [
{
"key_as_string": "06/02/2015 12:00:00",
"key": 1423180800000,
"doc_count": 1
},
{
"key_as_string": "21/02/2015 12:00:00",
"key": 1424476800000,
"doc_count": 1
}
]
}
}
}
}
My query string elastic:
{
"size": 0,
"aggs": {
"log_query": {
"filter": {
"bool": {
"must": [
{
"term": {
"cluster": "giauht1"
}
},
{
"term": {
"server": "hadoop0"
}
},
{
"term": {
"type": "Warn"
}
},
{
"range": {
"actionTime": {
"gte": "2015-02-01",
"lte": "2015-02-24"
}
}
}
]
}
},
"aggs": {
"histogram_Log": {
"date_histogram": {
"field": "actionTime",
"interval": "1d",
"format": "dd/MM/YYYY hh:mm:ss"
}
}
}
}
}
}
My .nest libs query:
Func<SearchDescriptor<LogInfoIndexView>, SearchDescriptor<LogInfoIndexView>> query =
que => que.Aggregations(aggs => aggs.Filter("log_query", fil =>
{
fil.Filter(fb => fb.Bool(fm => fm.Must(
ftm =>
{
ftm.Term(t => t.Cluster, cluster);
ftm.Term(t => t.Server, server);
ftm.Term(t => t.Type, logLevel);
ftm.Range(r => r.OnField("actionTime").GreaterOrEquals(from.Value).LowerOrEquals(to.Value));
return ftm;
}))).Aggregations(faggs => faggs.DateHistogram("histogram_Log", dr =>
{
dr.Field("actionTime");
dr.Interval("1d");
dr.Format("dd/MM/YYYY hh:mm:ss");
return dr;
}));
return fil;
})).Size(0).Type(new LogInfoIndexView().TypeName);
var result = client.Search(query);
My .nest result:
My model mapping:
{
"onef-sora": {
"mappings": {
"FPT.OneF.Api.Log": {
"properties": {
"actionTime": {
"type": "date",
"format": "dateOptionalTime"
},
"application": {
"type": "string",
"index": "not_analyzed"
},
"cluster": {
"type": "string",
"index": "not_analyzed"
},
"detail": {
"type": "string",
"index": "not_analyzed"
},
"iD": {
"type": "string"
},
"message": {
"type": "string",
"index": "not_analyzed"
},
"server": {
"type": "string",
"index": "not_analyzed"
},
"source": {
"type": "string",
"index": "not_analyzed"
},
"tags": {
"type": "string",
"index": "not_analyzed"
},
"type": {
"type": "string",
"index": "not_analyzed"
},
"typeLog": {
"type": "string"
},
"typeName": {
"type": "string"
},
"url": {
"type": "string",
"index": "not_analyzed"
},
"user": {
"type": "string",
"index": "not_analyzed"
}
}
}
}
}
}
The Must() condition passed to the Bool() filter takes a params Func<FilterDescriptor<T>, FilterContainer>[] but in your filter, the Term() and Range() filters are chained onto the same filter instance; unfortunately, this doesn't work as you might expect and the end result is actually an empty json object passed to the must clause in the query DSL for the filter i.e. you end up with
{
"size": 0,
"aggs": {
"log_query": {
"filter": {
"bool": {
"must": [
{} /* where are the filters?! */
]
}
},
"aggs": {
"histogram_Log": {
"date_histogram": {
"field": "actionTime",
"interval": "1d",
"format": "dd/MM/YYYY hh:mm:ss"
}
}
}
}
}
}
The solution is to pass an array of Func<FilterDescriptor<T>, FilterContainer>; The following matches your query DSL
void Main()
{
var settings = new ConnectionSettings(new Uri("http://localhost:9200"));
var connection = new InMemoryConnection(settings);
var client = new ElasticClient(connection: connection);
DateTime? from = new DateTime(2015, 2,1);
DateTime? to = new DateTime(2015, 2, 24);
var docs = client.Search<LogInfoIndexView>(s => s
.Size(0)
.Type("type")
.Aggregations(a => a
.Filter("log_query", f => f
.Filter(ff => ff
.Bool(b => b
.Must(m => m
.Term(t => t.Cluster, "giauht1"),
m => m
.Term(t => t.Server, "hadoop0"),
m => m
.Term(t => t.Type, "Warn"),
m => m
.Range(r => r.OnField("actionTime").GreaterOrEquals(from.Value).LowerOrEquals(to.Value))
)
)
)
.Aggregations(aa => aa
.DateHistogram("histogram_Log", da => da
.Field("actionTime")
.Interval("1d")
.Format("dd/MM/YYYY hh:mm:ss")
)
)
)
)
);
Console.WriteLine(Encoding.UTF8.GetString(docs.RequestInformation.Request));
}
public class LogInfoIndexView
{
public string Cluster { get; set; }
public string Server { get; set; }
public string Type { get; set; }
public DateTime ActionTime { get; set; }
}
returning
{
"size": 0,
"aggs": {
"log_query": {
"filter": {
"bool": {
"must": [
{
"term": {
"cluster": "giauht1"
}
},
{
"term": {
"server": "hadoop0"
}
},
{
"term": {
"type": "Warn"
}
},
{
"range": {
"actionTime": {
"lte": "2015-02-24T00:00:00.000",
"gte": "2015-02-01T00:00:00.000"
}
}
}
]
}
},
"aggs": {
"histogram_Log": {
"date_histogram": {
"field": "actionTime",
"interval": "1d",
"format": "dd/MM/YYYY hh:mm:ss"
}
}
}
}
}
}
EDIT:
In answer to your comment, the difference between a filtered query filter and a filter aggregation is that the former applies the filtering to all documents at the start of the query phase and filters are generally cached, improving performance on subsequent queries with those filters, whilst the latter applies in the scope of the aggregation to filter documents in the current context to a single bucket. If your query is only to perform the aggregation and you're likely to run the aggregation with the same filters, I think the filtered query filter should offer better performance.
Related
[
{
"_id": {
"$oid": "61c474fd740cd6a46a7e8166"
},
"GroupIds": [
{
"$oid": "31c482ff6836e438631995ed"
},
{
"$oid": "11c482ff6836e438631995ee"
},
{
"$oid": "61bb96fb4c3d7106f5b9587a"
}
],
"Username": "Test"
},
{
"_id": {
"$oid": "61c474fd740cd6a46a7e8166"
},
"GroupIds": [
{
"$oid": "15c482ff6836e438631995ed"
},
{
"$oid": "61c482ff6836e438631995ee"
},
{
"$oid": "61bb96ee4c3d7106f5b95879"
}
],
"Username": "Test1"
},
{
"_id": {
"$oid": "21c474fd740cd6a46a7e8166"
},
"GroupIds": [
{
"$oid": "61c482ff6836e438631995ed"
},
{
"$oid": "61c482ff6836e438631995ee"
},
{
"$oid": "61bb96ee4c3d7106f5b95879"
}
],
"Username": "Test2"
},
{
"_id": {
"$oid": "31c474fd740cd6a46a7e8166"
},
"GroupIds": [
{
"$oid": "61c482ff6836e438631995ed"
},
{
"$oid": "61c482ff6836e438631995ee"
},
{
"$oid": "61bb96fb4c3d7106f5b9587a"
}
],
"Username": "Test3"
}
]
Hello, I want to make multiple group based searches in mongo db,
I can make a single group based Search
public async Task<List<List<ObjectId>>> UsersByGroupIdV3(List<List<string>> targetGroupses)
{
List<FilterDefinition<User>> query= new List<FilterDefinition<User>>();
foreach (var targetGroups in targetGroupses)
{
if (targetGroups[0] != "allcountry")
{
query.Add(Builders<User>.Filter.Eq(x => x.GroupIds[0], new ObjectId(targetGroups[0])));
}
if (targetGroups[1] != "allCity")
{
query.Add(Builders<User>.Filter.Eq(x => x.GroupIds[1], new ObjectId(targetGroups[1])));
}
if (targetGroups[2] != "allDistrict")
{
query.Add(Builders<User>.Filter.Eq(x => x.GroupIds[2], new ObjectId(targetGroups[2])));
}
}
var match = Builders<User>.Filter.And(query);
var users = await _userRepository.Find(match);
var group = users.Result.Select(i => i.GroupIds);
return group.ToList();
}
There is a list of nested groups in the targetGroupses parameter
Example
[{["31c482ff6836e438631995ed","11c482ff6836e438631995ee","61bb96fb4c3d7106f5b9587a"]},{["15c482ff6836e438631995ed","61c482ff6836e438631995ee","61bb96ee4c3d7106f5b95879"]}]
var match = Builders.Filter.And(query);
If there is 1 list, it works fine, but if there is more than 1, how can I run the query?
I want to bring those in the America, Alaska, College or Germany Hessen Kreis groups
{[{"America", "Alaska", "College"}],[{"Germany", "Hessen", "Kreis"}]}
I want to fetch what's in this group from mongo db with c#
When I use Kibana to execute the following Searchrequest to Elasticsearch
GET _search
{
"query": {
"query_string": {
"query": "PDB_W2237.docx",
"default_operator": "AND"
}
}
}
it returns:
{
"took": 14,
"timed_out": false,
"_shards": {
"total": 15,
"successful": 15,
"skipped": 0,
"failed": 0
},
"hits": {
"total": 1,
"max_score": 6.3527603,
"hits": [
{
"_index": "proconact",
"_type": "proconact",
"_id": "68cecf2c-7e5a-11e5-80fa-000c29bd9450",
"_score": 6.3527603,
"_source": {
"Id": "68cecf2c-7e5a-11e5-80fa-000c29bd9450",
"ActivityId": "1bad9115-7e5a-11e5-80fa-000c29bd9450",
"ProjectId": "08938a1d-2429-11e5-80f9-000c29bd9450",
"Filename": "PDB_W2237.docx"
}
}
]
}
}
When I use the NEST ElasticClient like
var client = new ElasticClient();
var searchResponse = client.Search<Hit>(new SearchRequest {
Query = new QueryStringQuery {
Query = "DB_W2237.docx",
DefaultOperator = Operator.And
}
});
it do return 0 Hits.
Here is the Indexmapping for the 4 Fields in the Hit:
{
"proconact": {
"mappings": {
"proconact": {
"properties": {
"ActivityId": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"Filename": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"Id": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"ProjectId": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
}
}
}
}
Are the two search-requests not the same?
The problem is your mapping doesn't allow a token different than whatever is present in your index.
In your kibana query:
GET _search
{
"query": {
"query_string": {
"query": "PDB_W2237.docx",
"default_operator": "AND"
}
}
}
You're querying PDB_W2237.docx but in your NEST you're querying DB_W2237.docx.
If you want to query DB_W2237.docx and expecting results then you might have to change the analyzer from standard analyzer which is applied by default to something else a possible candidate depends on your usecase.
I'm trying to build a nested query that needs to filter by multiple terms on the nested object. I'm using Nest nuget package version 6.1
The query is build using Nest DSL syntax as follows:
queryContainer &= Query<PropertyDTO>.Nested(n => n
.Path(p => p.Publications)
.Query(q =>
q.Term(t => t
.Field(ff => ff.Publications.First().Id == publicationId.ToString(CultureInfo.InvariantCulture))
)
&& q.DateRange(r =>
r
.Field(f => f.Publications.First().PublishedOn)
.GreaterThanOrEquals(
from.HasValue
? DateMath.Anchored(from.Value)
: DateMath.Anchored(DateTime.MinValue)
)
.LessThanOrEquals(
to.HasValue
? DateMath.Anchored(to.Value)
: DateMath.Anchored(DateTime.Now)
)
)
)
);
The expected results should be:
{
"nested": {
"query":
{
"bool": {
"must": [
{
"range": {
"publications.publishedOn": {
"gte": "2018-06-13T00:00:00",
"lte": "2018-06-20T23:59:59"
}
}
},
{
"term": {
"publications.id": {
"value": "1.510136"
}
}
}
]
}
},
"path": "publications"
}
}
But on the contrary, I get:
{
"nested": {
"query": {
"term": {
"publications.id": {
"value": "1.510136"
}
}
},
"path": "publications"
}
},
{
"nested": {
"query": {
"range": {
"publications.publishedOn": {
"gte": "2018-06-14T00:00:00",
"lte": "2018-06-21T23:59:59"
}
}
},
"path": "publications"
}
}
What I'm doing wrong?
At the moment I'm using a workaround based on a raw query version, but I would like to use the Nest DSL syntax which is more refactoring friendly.
Try something like this:
lESResponse = lESClient.Search<PropertyDTO>(s => s
.Query(q => q
.Nested(n => n
.Path("publications")
.Query(qu => qu
.Bool(b => b
.Must(new QueryContainer[] {
new DateRangeQuery() {
Field = "publications.publishedOn",
GreaterThanOrEqualTo = DateMath.Anchored(DateTime.MinValue),
LessThanOrEqualTo = DateMath.Anchored(DateTime.Now)
},
new TermQuery() {
Field = "publications.id",
Value = "1.510136"
}
} ))))));
Below is the output:
{
"query": {
"nested": {
"query": {
"bool": {
"must": [
{
"range": {
"publications.publishedOn": {
"gte": "",
"lte": "2018-06-21T18:35:17.1274477+05:30"
}
}
},
{
"term": {
"publications.id": {
"value": "1.510136"
}
}
}
]
}
},
"path": "publications"
}
}
}
I have lots of product typed documents saved in elasticsearch but couldn't search by documents property values and need help.
//Product Class
public Guid productID { get; set; }
public string productName { get; set; }
public Guid productTypeID { get; set; }
public List<Guid> categoryIds {get; set; }
I'm trying to search like this:
//Search function
var esconn = Yaziylabir.Bukalemun.DataObjects.ElasticSearchConnectionManager.GetClient();
QueryContainer q = null;
if (!ProductType.HasValue || (ProductType.HasValue && ProductType.Value == B2CVariables.PRODUCTTYPE_PRODUCT))
{
q = Query<ProductModel>.Term(t => t.ProductTypeID, B2CVariables.PRODUCTTYPE_PRODUCT);
}
if (Category != null)
{
//catListZ is also List<Guid>
q &= Query<ProductModel>.Terms(u=>u.Field(z=>z.CategoryIDs).Terms<Guid>(catListZ));
}
// as a bonus I also have keyword search
if (!string.IsNullOrEmpty(Keyword))
{
q &= Query<ProductModel>.QueryString(t => t.Query(Keyword));
}
//do search
var pp = new SearchRequest(Yaziylabir.Bukalemun.DataObjects.ElasticSearchConnectionManager.DefaultIndex, "product");
pp.Query = q;
pp.Size = PageSize;
var res = esconn.Search<ProductModel>(pp);
rtnVal = res.Documents.ToList();
Now, I tried combinations of these (only producttypeID, only categoryIDs, only keyword, etc...) and watch what is happening with fiddler.
No result comes back, no errors are raised. Only 0 hits. Request body seems ok too.
When I check documents stored in that index, they are there and they have values required and should return in result.
What is wrong here? Do you have any ideas? Please help me here. I'm feeling ashamed to be the guy who couldn't search a database properly.
EDIT:
Search's body text:
{"size":12,"query":{"term":{"productTypeID":{"value":"cae344cf-8cfa-4960-8387-8ee89899c53f"}}}}
Example document:
{
"productID": "5687b8ac-c3fe-4f1a-9643-08b0bf6cede8",
"productName": "7011 Yelek",
"productCode": "701102Y001 ",
"currency": {
"currencyID": 1,
"sign": "TL",
"rate": 0
},
"normalPrice": 170,
"currentPrice": 84.9,
"isDiscounted": true,
"taxRate": 8,
"productTypeID": "cae344cf-8cfa-4960-8387-8ee89899c53f",
"defaultImagePath": "/contents/images/productimages/75681ee4-19b3-4c7d-a24b-b3566085a980.jpg",
"totalStockCount": 8,
"totalStockRecordCount": 4,
"isInStock": true,
"statusID": "9ad17471-2ff2-4eb0-9cb0-4b86922263ea",
"categoryIDs": [
"a8c83f54-b784-4866-89c3-cabc641490d5",
"9d5a9ab7-8edb-4d5a-800b-c48bf6575d78"
]
}
I didn't include all properties because it will make document very long.
Here is the mapping:
{
"mappings": {
"product": {
"properties": {
"categoryIDs": {
"type": "string"
},
"currentPrice": {
"type": "double"
},
"isDiscounted": {
"type": "boolean"
},
"isInStock": {
"type": "boolean"
},
"normalPrice": {
"type": "double"
},
"productCode": {
"type": "string"
},
"productID": {
"type": "string"
},
"productName": {
"type": "string"
},
"productTypeID": {
"type": "string"
},
"statusID": {
"type": "string"
},
"taxRate": {
"type": "double"
},
"totalStockCount": {
"type": "long"
},
"totalStockRecordCount": {
"type": "long"
}
}
}
}
}
I suspect the productTypeID field is using either the default analyzer - standard - or any other analyzer that's splitting it in the wrong places. What you need is for productTypeID to be index: not_analyzed or analyzed with something like keyword. And you need to create the mapping manually, otherwise you can't do it the way you want it.
The idea is that ES is tokenizing by default your productTypeID values and will split them at -, so in the index you'd have tokens not the entire value. You, instead, need to have that value unchanged so that term will match it fully.
For example, not to mess up with your previous mapping, you can add fields to define a sub-field that will be not_analyzed:
"productTypeID": {
"type": "string",
"fields": {
"raw": {
"type": "string",
"index": "not_analyzed"
}
}
}
And your query needs to change slightly: {"size":12,"query":{"term":{"productTypeID.raw":{"value":"cae344cf-8cfa-4960-8387-8ee89899c53f"}}}}
For completeness sake, this is the complete command to create the mapping by hand. You could do it while the index is still "alive", but you'd still have to re-index the documents:
curl -XPUT "http://localhost:9200/your_index" -d'
{
"mappings": {
"product": {
"properties": {
"categoryIDs": {
"type": "string"
},
"currentPrice": {
"type": "double"
},
"isDiscounted": {
"type": "boolean"
},
"isInStock": {
"type": "boolean"
},
"normalPrice": {
"type": "double"
},
"productCode": {
"type": "string"
},
"productID": {
"type": "string"
},
"productName": {
"type": "string"
},
"productTypeID": {
"type": "string",
"fields": {
"raw": {
"type": "string",
"index": "not_analyzed"
}
}
},
"statusID": {
"type": "string"
},
"taxRate": {
"type": "double"
},
"totalStockCount": {
"type": "long"
},
"totalStockRecordCount": {
"type": "long"
}
}
}
}
}'
I'm trying to use NEST to create an index with raw json and it produces different results than when I use that same json string interactively against elastic search. Is this a bug or am I using it incorrectly?
Posting directly to elastic search with the following command I get exactly the mappings I want for my index (results shown below)
POST /entities
{
"mappings": {
"sis_item" :
{
"properties":
{
"FullPath":
{
"type": "string",
"index":"not_analyzed"
},
"Ident":
{
"type": "nested",
"properties":
{
"ObjectGuid":
{
"type": "string",
"index":"not_analyzed"
}
}
}
}
}
}
Result when I check the index with: GET /entities/ : (which is correct)
{
"entities": {
"aliases": {},
"mappings": {
"sis_item": {
"properties": {
"FullPath": {
"type": "string",
"index": "not_analyzed"
},
"Ident": {
"type": "nested",
"properties": {
"ObjectGuid": {
"type": "string",
"index": "not_analyzed"
}
}
}
}
}
},
"settings": {
"index": {
"creation_date": "1453828294488",
"number_of_shards": "5",
"number_of_replicas": "1",
"version": {
"created": "1070499"
},
"uuid": "6_j4vRcqRwiTQw0E6bQokQ"
}
},
"warmers": {}
}
}
However I have to do this from code and using the following code the mappings I specify end up in the settings instead of the mappings as shown below.
var createIndexJson = #"
{
""mappings"": {
""sis_item"" :
{
""properties"":
{
""FullPath"":
{
""type"": ""string"",
""index"":""not_analyzed""
},
""Ident"":
{
""type"": ""nested"",
""properties"":
{
""ObjectGuid"":
{
""type"": ""string"",
""index"":""not_analyzed""
}
}
}
}
}
}
}";
var response = _client.Raw.IndicesCreatePost("entities_from_code", createIndexJson);
if (!response.Success || response.HttpStatusCode != 200)
{
throw new ElasticsearchServerException(response.ServerError);
}
Result (not correct, notice the mappings are nested inside the settings):
{
"entities_from_code": {
"aliases": {},
"mappings": {},
"settings": {
"index": {
"mappings": {
"sis_item": {
"properties": {
"FullPath": {
"type": "string",
"index": "not_analyzed"
},
"Ident": {
"type": "nested",
"properties": {
"ObjectGuid": {
"type": "string",
"index": "not_analyzed"
}
}
}
}
}
},
"creation_date": "1453828283162",
"number_of_shards": "5",
"number_of_replicas": "1",
"version": {
"created": "1070499"
},
"uuid": "fdmPqahGToCJw_mIbq0kNw"
}
},
"warmers": {}
}
}
There is a newline at the very top of the json string which cause the odd result, removing it gave me the expected behaviour.