Develop Reference

Elastic Search Nest With .NET Slow Scroll Speed

Firstly i sorry for my bad english. I have one node in my server.
My index stats below this :
{
"_shards" : {
"total" : 4,
"successful" : 4,
"failed" : 0
},
"_all" : {
"primaries" : {
"docs" : {
"count" : 1951593,
"deleted" : 0
},
"store" : {
"size_in_bytes" : 594806126,
"reserved_in_bytes" : 0
},
"indexing" : {
"index_total" : 0,
"index_time_in_millis" : 0,
"index_current" : 0,
"index_failed" : 0,
"delete_total" : 0,
"delete_time_in_millis" : 0,
"delete_current" : 0,
"noop_update_total" : 0,
"is_throttled" : false,
"throttle_time_in_millis" : 0
},
"get" : {
"total" : 0,
"time_in_millis" : 0,
"exists_total" : 0,
"exists_time_in_millis" : 0,
"missing_total" : 0,
"missing_time_in_millis" : 0,
"current" : 0
},
"search" : {
"open_contexts" : 0,
"query_total" : 5900,
"query_time_in_millis" : 5539,
"query_current" : 0,
"fetch_total" : 1464,
"fetch_time_in_millis" : 488659,
"fetch_current" : 0,
"scroll_total" : 44,
"scroll_time_in_millis" : 6017059,
"scroll_current" : 0,
"suggest_total" : 0,
"suggest_time_in_millis" : 0,
"suggest_current" : 0
},
"merges" : {
"current" : 0,
"current_docs" : 0,
"current_size_in_bytes" : 0,
"total" : 0,
"total_time_in_millis" : 0,
"total_docs" : 0,
"total_size_in_bytes" : 0,
"total_stopped_time_in_millis" : 0,
"total_throttled_time_in_millis" : 0,
"total_auto_throttle_in_bytes" : 83886080
},
"refresh" : {
"total" : 8,
"total_time_in_millis" : 0,
"external_total" : 8,
"external_total_time_in_millis" : 0,
"listeners" : 0
},
"flush" : {
"total" : 4,
"periodic" : 0,
"total_time_in_millis" : 0
},
"warmer" : {
"current" : 0,
"total" : 4,
"total_time_in_millis" : 0
},
"query_cache" : {
"memory_size_in_bytes" : 0,
"total_count" : 1668,
"hit_count" : 0,
"miss_count" : 1668,
"cache_size" : 0,
"cache_count" : 0,
"evictions" : 0
},
"fielddata" : {
"memory_size_in_bytes" : 0,
"evictions" : 0
},
"completion" : {
"size_in_bytes" : 0
},
"segments" : {
"count" : 45,
"memory_in_bytes" : 317076,
"terms_memory_in_bytes" : 174240,
"stored_fields_memory_in_bytes" : 22984,
"term_vectors_memory_in_bytes" : 0,
"norms_memory_in_bytes" : 25920,
"points_memory_in_bytes" : 0,
"doc_values_memory_in_bytes" : 93932,
"index_writer_memory_in_bytes" : 0,
"version_map_memory_in_bytes" : 0,
"fixed_bit_set_memory_in_bytes" : 0,
"max_unsafe_auto_id_timestamp" : -1,
"file_sizes" : { }
},
"translog" : {
"operations" : 0,
"size_in_bytes" : 220,
"uncommitted_operations" : 0,
"uncommitted_size_in_bytes" : 220,
"earliest_last_modified_age" : 5016454
},
"request_cache" : {
"memory_size_in_bytes" : 0,
"evictions" : 0,
"hit_count" : 0,
"miss_count" : 0
},
"recovery" : {
"current_as_source" : 0,
"current_as_target" : 0,
"throttle_time_in_millis" : 0
}
}
and my mappings below this :
{
"marka_listesi" : {
"mappings" : {
"properties" : {
"basvuruNo" : {
"type" : "keyword"
},
"basvuruTarihi" : {
"type" : "date"
},
"durum" : {
"type" : "text",
"analyzer" : "turkish_analyzer"
},
"evrakNo" : {
"type" : "keyword"
},
"ilanBultenNo" : {
"type" : "keyword"
},
"ilanBultenTarihi" : {
"type" : "date"
},
"korumaTarihi" : {
"type" : "date"
},
"markaAdi" : {
"type" : "text",
"analyzer" : "turkish_analyzer"
},
"markaID" : {
"type" : "integer"
},
"markaLogoUrl" : {
"type" : "text"
},
"sahip" : {
"type" : "text",
"analyzer" : "turkish_analyzer"
},
"siniflar" : {
"type" : "keyword"
},
"sonDegisiklik" : {
"type" : "date"
},
"sonIslem" : {
"type" : "text",
"analyzer" : "turkish_analyzer"
},
"sonIslemBaslik" : {
"type" : "text",
"analyzer" : "turkish_analyzer"
},
"sonIslemTarihi" : {
"type" : "date"
},
"tescilBultenNo" : {
"type" : "keyword"
},
"tescilBultenTarihi" : {
"type" : "date"
},
"tescilNo" : {
"type" : "keyword"
},
"tescilTarihi" : {
"type" : "date"
},
"tur" : {
"type" : "text",
"analyzer" : "turkish_analyzer"
},
"uTescilNo" : {
"type" : "keyword"
},
"vekil" : {
"type" : "text",
"analyzer" : "turkish_analyzer"
},
"vekilFirma" : {
"type" : "text",
"analyzer" : "turkish_analyzer"
}
}
}
}
}
This index have 2 million record but size average 400-500 mb.
I need get all document in index with dynamic filter and compare in my class method. I'am using scroll.all api from nest but documents coming and adding all documents to my model list very very slow(1 minutes). I seen Maximum cpu percentage for query processing 20%.
I need several second for query complete. This is my nest query :
public static List<Models.Model.Sonuc.MarkaListesi>GetDocumentsOnSelectedClasses(string _indexName,ElasticClient _client,string[] _siniflar)
{
var _docs = new List<Models.Model.Sonuc.MarkaListesi>();
// number of slices in slice scroll
var scrollObserver = _client.ScrollAll<Models.Model.Sonuc.MarkaListesi>("1m", 4, s => s
.MaxDegreeOfParallelism(4)
.Search(se => se
.Index(_indexName)
.Query(q => q.Bool(b => b.Filter(fq=>fq.Terms(t=>t.Field("siniflar").Terms(_siniflar)))))
.Size(1000)
)
);
var waitHandle = new ManualResetEvent(false);
Exception exception = null;
var scrollAllObserver = new ScrollAllObserver<Models.Model.Sonuc.MarkaListesi>(
onNext: response =>
{
// do something with the documents
_docs.AddRange(response.SearchResponse.Documents);
},
onError: e =>
{
exception = e;
waitHandle.Set();
},
onCompleted: () => waitHandle.Set()
);
scrollObserver.Subscribe(scrollAllObserver);
waitHandle.WaitOne();
if (exception != null)
{
throw exception;
}
return _docs;
}
Finally my question is : how can i increase my search speed ?

MongoDB sum values in document

I'm making an application with MongoDB and C#. I use the MongoDB driver.
{
_id: ObjectId("5099803df3f4948bd2f98391"),
Title: "Test",
Rating: {
1234 : 4.0,
4689 : 2.5,
1987 : 1.5
}
}
Now I want to get the average rating. So (4.0+2.5+1.5) / 3.
But how do I do this with MongoDB and the C# driver.
I'm trying to unwind and then count. But it's not working.

You can use $objectToArray (from mongo version 3.4.4) to convert object to array and calculate $avg
db.t71.aggregate([
{$addFields : {data : {$objectToArray : "$Rating"}}},
{$addFields : {avg : {$avg : "$data.v"}}}
]).pretty()
aggregation
> db.t71.aggregate([{$addFields : {data : {$objectToArray : "$Rating"}}}, {$addFields : {avg : {$avg : "$data.v"}}}]).pretty()
{
"_id" : 1,
"Title" : "Test",
"Rating" : {
"1234" : 4,
"1987" : 1.5,
"4689" : 2.5
},
"data" : [
{
"k" : "1234",
"v" : 4
},
{
"k" : "1987",
"v" : 1.5
},
{
"k" : "4689",
"v" : 2.5
}
],
"avg" : 2.6666666666666665
}

Is it possible to know how much time MongoDB has spent for parsing a query?

I have a query which filter a field against a large (> 100K) number of values using the $in operator. It is quite slow. There is an index on the field (IXSCAN is reported by profiler).
I was wondering if there was any way to see how much time MongoDB take to parse
the query (which is sent as BSON if i am not mistaken).
I think C# (since i'm using C# driver) is also taking some time to generate the BSON data but it cannot be reported by MongoDB since it's happening client side.
EDIT: here is the query plan:
{
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "Foo.Bar",
"indexFilterSet" : false,
"parsedQuery" : {
"ExternalKeys" : {
"$in" : [
{
"A" : 0,
"B" : 0,
"C" : 34228
},
{
"A" : 0,
"B" : 1,
"C" : 540830
},
... (many like that)
{
"A" : 5,
"B" : 3,
"C" : 773660
}
]
}
},
"winningPlan" : {
"stage" : "FETCH",
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"ExternalKeys" : 1
},
"indexName" : "ExternalKeys_1",
"isMultiKey" : true,
"multiKeyPaths" : {
"ExternalKeys" : [
"ExternalKeys"
]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"ExternalKeys" : [
"[{ A: 0.0, B: 0.0, C: 34228.0 }, { A: 0.0, B: 0.0, C: 34228.0 }]",
"[{ A: 0.0, B: 1.0, C: 540830.0 }, { A: 0.0, B: 1.0, C: 540830.0 }]",
... (many like that)
"[{ A: 5.0, B: 3.0, C: 773660.0 }, { A: 5.0, B: 3.0, C: 773660.0 }]"
]
}
}
},
"rejectedPlans" : [ ]
},
"serverInfo" : {
"host" : "....",
"port" : 27017,
"version" : "3.4.10",
"gitVersion" : "078f28920cb24de0dd479b5ea6c66c644f6326e9"
},
"ok" : 1
}

C# mongodb insert inside a document in a collection

Following is the structure of doc.
{
"name" : "Apparel & Accessories",
"description" : "Apparel & Accessories",
"logoPath" : "apparel_n_accessories.png",
"categoryCode" : "APP-N-ACC",
"isActive" : 1,
"subCategory" : [
{
"name" : "Clothing",
"description" : "Clothing",
"logoPath" : "clothing.png",
"categoryCode" : "CLOTH",
"isActive" : 1,
"subCategory" : [
{
"name" : "Outerwear",
"description" : "Outerwear",
"logoPath" : "outerwear.png",
"categoryCode" : "OUTWER",
"isActive" : 1,
"subCategory" : [
{
"name" : "Coats & Jackets",
"description" : "Coats & Jackets",
"logoPath" : "coats_n_jackets.png",
"categoryCode" : "COT-N-JACT",
"isActive" : 1,
"subCategory" : [ ]
}
]
},
{
"name" : "Jewelry",
"description" : "Jewelry",
"logoPath" : "jewelry.png",
"categoryCode" : "JEWL",
"subCategory" : [
{
"name" : "Rings",
"description" : "Rings",
"logoPath" : "rings.png",
"categoryCode" : "RINGS",
"isActive" : 1,
"subCategory" : [ ]
}
]
}
]
}
]
}
I want to insert into subcategory of "Apparel & Accessories" with following content :
{
"name" : "XYZ",
"description" : "XYZ",
"logoPath" : "XYZ.png",
"categoryCode" : "XYZ",
"isActive" : 1,
"subCategory" : [ ]
}
We are using c# ver 1.8 legacy drivers to connect mongodb.
Can anyone please suggest how to find any level object and add in it.

Do something like this:
var filter = Builders<Category>
.Filter.Eq(c => c.name, "Apparel & Accessories");
var update = Builders<Category>.Update
.Push<Category>(c => c.subCategory, mySubCategory);
await collection.FindOneAndUpdateAsync(filter, update);

mongoDB insert item in Array

{
"_id" : ObjectId("5431f38c4ba4dd20408b0432"),
"UserID" : "1",
"Status" : {
}
},
"ListFilterType" : 1,
"IssueCategories" : [
{
"_id" : ObjectId("000000000000000000000000"),
"IssueCategoryID" : 2,
"IssueCagetoryName" : "test",
"MatchKeyword" : "test",
"MatchKeywordID" : 2
}
]
}
edit :
my expactation is like this.
{
"_id" : ObjectId("5431f38c4ba4dd20408b0432"),
"UserID" : "1",
"Status" : {
}
},
"ListFilterType" : 1,
"IssueCategories" : [
{
"_id" : ObjectId("000000000000000000000000"),
"IssueCategoryID" : 2,
"IssueCagetoryName" : "test",
"MatchKeyword" : "test",
"MatchKeywordID" : 2
},
{
"_id" : ObjectId("000000000000000000000001"),
"IssueCategoryID" : 3,
"IssueCagetoryName" : "test2",
"MatchKeyword" : "test2",
"MatchKeywordID" : 3
},
{
"_id" : ObjectId("000000000000000000000004"),
"IssueCategoryID" : 4,
"IssueCagetoryName" : "test4",
"MatchKeyword" : "test34",
"MatchKeywordID" : 4
}
]
i have a type list as "IssueCategories". You can see my mongodb structure above. If root _id = my parameter, I want to add to mongodb.
I hope I explained right
thanks

You should use either $push or $addToSet to update document.
$push will add all element in IssueCategories including duplicates also.
While in $addToSet do not insert duplicate elements(object) in array if whole object is present in array.
Using $push:
db.collection.update({
"_id": ObjectId("5431f38c4ba4dd20408b0432") // you can add any other param
}, {
$push: {
SCSIssueCategories: {
"_id": ObjectId("11111111111111111"), //ur id here
"IssueCategoryID": 3,
"IssueCagetoryName": "Sözleşme1",
"MatchKeyword": "abonelik süresi1",
"MatchKeywordID": 3
}
}
})
Here is query using $addToSet:
db.collection.update({
"_id": ObjectId("5431f38c4ba4dd20408b0432")
}, {
$addToSet: {
SCSIssueCategories: {
"_id": ObjectId("11111111111111111"), //ur id here
"IssueCategoryID": 3,
"IssueCagetoryName": "Sözleşme1",
"MatchKeyword": "abonelik süresi1",
"MatchKeywordID": 3
}
}
})
**If single param have different value then it will be inserted in array using $addToSet. **