Develop Reference

Elastic Search Nest With .NET Slow Scroll Speed

Firstly i sorry for my bad english. I have one node in my server.
My index stats below this :
{
"_shards" : {
"total" : 4,
"successful" : 4,
"failed" : 0
},
"_all" : {
"primaries" : {
"docs" : {
"count" : 1951593,
"deleted" : 0
},
"store" : {
"size_in_bytes" : 594806126,
"reserved_in_bytes" : 0
},
"indexing" : {
"index_total" : 0,
"index_time_in_millis" : 0,
"index_current" : 0,
"index_failed" : 0,
"delete_total" : 0,
"delete_time_in_millis" : 0,
"delete_current" : 0,
"noop_update_total" : 0,
"is_throttled" : false,
"throttle_time_in_millis" : 0
},
"get" : {
"total" : 0,
"time_in_millis" : 0,
"exists_total" : 0,
"exists_time_in_millis" : 0,
"missing_total" : 0,
"missing_time_in_millis" : 0,
"current" : 0
},
"search" : {
"open_contexts" : 0,
"query_total" : 5900,
"query_time_in_millis" : 5539,
"query_current" : 0,
"fetch_total" : 1464,
"fetch_time_in_millis" : 488659,
"fetch_current" : 0,
"scroll_total" : 44,
"scroll_time_in_millis" : 6017059,
"scroll_current" : 0,
"suggest_total" : 0,
"suggest_time_in_millis" : 0,
"suggest_current" : 0
},
"merges" : {
"current" : 0,
"current_docs" : 0,
"current_size_in_bytes" : 0,
"total" : 0,
"total_time_in_millis" : 0,
"total_docs" : 0,
"total_size_in_bytes" : 0,
"total_stopped_time_in_millis" : 0,
"total_throttled_time_in_millis" : 0,
"total_auto_throttle_in_bytes" : 83886080
},
"refresh" : {
"total" : 8,
"total_time_in_millis" : 0,
"external_total" : 8,
"external_total_time_in_millis" : 0,
"listeners" : 0
},
"flush" : {
"total" : 4,
"periodic" : 0,
"total_time_in_millis" : 0
},
"warmer" : {
"current" : 0,
"total" : 4,
"total_time_in_millis" : 0
},
"query_cache" : {
"memory_size_in_bytes" : 0,
"total_count" : 1668,
"hit_count" : 0,
"miss_count" : 1668,
"cache_size" : 0,
"cache_count" : 0,
"evictions" : 0
},
"fielddata" : {
"memory_size_in_bytes" : 0,
"evictions" : 0
},
"completion" : {
"size_in_bytes" : 0
},
"segments" : {
"count" : 45,
"memory_in_bytes" : 317076,
"terms_memory_in_bytes" : 174240,
"stored_fields_memory_in_bytes" : 22984,
"term_vectors_memory_in_bytes" : 0,
"norms_memory_in_bytes" : 25920,
"points_memory_in_bytes" : 0,
"doc_values_memory_in_bytes" : 93932,
"index_writer_memory_in_bytes" : 0,
"version_map_memory_in_bytes" : 0,
"fixed_bit_set_memory_in_bytes" : 0,
"max_unsafe_auto_id_timestamp" : -1,
"file_sizes" : { }
},
"translog" : {
"operations" : 0,
"size_in_bytes" : 220,
"uncommitted_operations" : 0,
"uncommitted_size_in_bytes" : 220,
"earliest_last_modified_age" : 5016454
},
"request_cache" : {
"memory_size_in_bytes" : 0,
"evictions" : 0,
"hit_count" : 0,
"miss_count" : 0
},
"recovery" : {
"current_as_source" : 0,
"current_as_target" : 0,
"throttle_time_in_millis" : 0
}
}
and my mappings below this :
{
"marka_listesi" : {
"mappings" : {
"properties" : {
"basvuruNo" : {
"type" : "keyword"
},
"basvuruTarihi" : {
"type" : "date"
},
"durum" : {
"type" : "text",
"analyzer" : "turkish_analyzer"
},
"evrakNo" : {
"type" : "keyword"
},
"ilanBultenNo" : {
"type" : "keyword"
},
"ilanBultenTarihi" : {
"type" : "date"
},
"korumaTarihi" : {
"type" : "date"
},
"markaAdi" : {
"type" : "text",
"analyzer" : "turkish_analyzer"
},
"markaID" : {
"type" : "integer"
},
"markaLogoUrl" : {
"type" : "text"
},
"sahip" : {
"type" : "text",
"analyzer" : "turkish_analyzer"
},
"siniflar" : {
"type" : "keyword"
},
"sonDegisiklik" : {
"type" : "date"
},
"sonIslem" : {
"type" : "text",
"analyzer" : "turkish_analyzer"
},
"sonIslemBaslik" : {
"type" : "text",
"analyzer" : "turkish_analyzer"
},
"sonIslemTarihi" : {
"type" : "date"
},
"tescilBultenNo" : {
"type" : "keyword"
},
"tescilBultenTarihi" : {
"type" : "date"
},
"tescilNo" : {
"type" : "keyword"
},
"tescilTarihi" : {
"type" : "date"
},
"tur" : {
"type" : "text",
"analyzer" : "turkish_analyzer"
},
"uTescilNo" : {
"type" : "keyword"
},
"vekil" : {
"type" : "text",
"analyzer" : "turkish_analyzer"
},
"vekilFirma" : {
"type" : "text",
"analyzer" : "turkish_analyzer"
}
}
}
}
}
This index have 2 million record but size average 400-500 mb.
I need get all document in index with dynamic filter and compare in my class method. I'am using scroll.all api from nest but documents coming and adding all documents to my model list very very slow(1 minutes). I seen Maximum cpu percentage for query processing 20%.
I need several second for query complete. This is my nest query :
public static List<Models.Model.Sonuc.MarkaListesi>GetDocumentsOnSelectedClasses(string _indexName,ElasticClient _client,string[] _siniflar)
{
var _docs = new List<Models.Model.Sonuc.MarkaListesi>();
// number of slices in slice scroll
var scrollObserver = _client.ScrollAll<Models.Model.Sonuc.MarkaListesi>("1m", 4, s => s
.MaxDegreeOfParallelism(4)
.Search(se => se
.Index(_indexName)
.Query(q => q.Bool(b => b.Filter(fq=>fq.Terms(t=>t.Field("siniflar").Terms(_siniflar)))))
.Size(1000)
)
);
var waitHandle = new ManualResetEvent(false);
Exception exception = null;
var scrollAllObserver = new ScrollAllObserver<Models.Model.Sonuc.MarkaListesi>(
onNext: response =>
{
// do something with the documents
_docs.AddRange(response.SearchResponse.Documents);
},
onError: e =>
{
exception = e;
waitHandle.Set();
},
onCompleted: () => waitHandle.Set()
);
scrollObserver.Subscribe(scrollAllObserver);
waitHandle.WaitOne();
if (exception != null)
{
throw exception;
}
return _docs;
}
Finally my question is : how can i increase my search speed ?

MongoWriteException: A write operation resulted in an error. The positional operator did not find the match needed from the query

I have a collection Zonedetails shown below. below.
I am using C# to Insert or Update a Unit to Units array. If it is an insert I can insert Area blank. If it is an update It should only update the UnitName.
{
"Code" : "Zone1",
"Name" : "ZoneName1",
"Units" : [
{
"UnitCode" : "Unitcode1",
"UnitName" : "UnitCodeName",
"Areas" : [
{
"AreaCode" : "AreaCode1",
"AreaName" : "AreaName1"
}
]
}
]
}
{
"Code" : "Zone2",
"Name" : "ZoneName2",
"Units" : [
{
"UnitCode" : "UnitCode2",
"UnitName" : "UnityName2",
"Areas" : [
{
"AreaCode" : "Areacode2",
"AreaName" : "AreaName2"
}
]
}
]
}
{
"Code" : "Zone3",
"Name" : "ZoneName3",
"Units" : [
{
"UnitCode" : "UnitCode3",
"UnitName" : "UnitName3",
"Areas" : [
{
"AreaCode" : "Areadcode3",
"AreaName" : "AreaName3"
},
{
"AreaCode" : "AreaCode4",
"AreaName" : "Areaname4"
},
{
"AreaCode" : "AreaCode5",
"AreaName" : "Areaname5"
}
]
},
{
"UnitCode" : "UnitCode6",
"UnitName" : "UnitName6",
"Areas" : [
{
"AreaCode" : "AreaCode10",
"AreaName" : "AreaName10"
},
{
"AreaCode" : "AreaCOde11",
"AreaName" : "AreaName10"
},
{
"AreaCode" : "AreaCode12",
"AreaName" : "AreaName12"
}
]
}
]
}
I have writtent a C# code shown below. But getting "The positional operator did not find the match needed from the query"error if the Unit Code does not exist. Added not before unitCode check.
var mongoCollection = _context.GetCollection<Zone>("ZoneDetail");
var filter = Builders<Zone>.Filter.Where(x => x.Code == zoneCode && !x.Units.Any(u => u.UnitCode == unit.UnitCode));
var updateUnitCode = Builders<Zone>.Update.Set(x => x.Units.ElementAt(-1).UnitCode, unit.UnitCode);
var updateUnitName = Builders<Zone>.Update.Set(x => x.Units.ElementAt(-1).UnitName, unit.UnitName);
var result = await mongoCollection.UpdateOneAsync(filter, Builders<Zone>.Update.Combine(updateUnitCode, updateUnitName), new UpdateOptions { IsUpsert = true});

Your error message suggest that filter is not able to find matching element.
Either try to change your input or change your filter criteria.

MongoDB graphLookup using AggregateFluentExtensions

I've been trying to execute a MongoDB graphLookup using latest C# driver which uses AggregateFluentExtensions. According to the documentation:
the method receives a series of parameters that I can't find a way to make work.
Has anybody used it and could help me with an example?
This is the json version of my aggregation:
db.getCollection("Item").aggregate(
[
{
"$project" : {
"itemMasterId" : 1.0,
"parentItemId" : 1.0
}
},
{
"$graphLookup" : {
"from" : "Item",
"startWith" : "$itemMasterId",
"connectFromField" : "itemMasterId",
"connectToField" : "parentItemId",
"as" : "ancestors",
"maxDepth" : 10,
"depthField" : "depthField",
"restrictSearchWithMatch" : {
"locationId" : 26
}
}
},
{
"$project" : {
"itemMasterId" : 1.0,
"parentItemId" : 1.0,
"children.itemMasterId" : 1,
"children.parentItemId" : 1
}
}
]
);
Thanks!

(Late in the party, I hope it helps someone.)
I have used AppendStage() method, you can try this:
var graphLookupStage = new BsonDocument("$graphLookup",
new BsonDocument
{
{ "from", "someCollection" },
{ "startWith", "$reportsTo" },
{ "connectFromField", "reportsTo"},
{ "connectToField", "name" },
{ "as", "reportingHierarchy" },
{ "maxDepth", 1 },
{ "depthField", "depthField" } //optional
});
var result = collection.Aggregate().AppendStage<BsonDocument>(graphLookupStage);

MongoDB Create dynamic Query

Below is working code for 'And' Operation of Multiple Queries, I can able to do 'And' operation for list of Queries, 'Or' operation for list of Queries. But is there a way in MongoDB that i can do 'And' or 'Or' operation Dynamically for list of Queries?
public IQueryable<SocialRecord> GetWallSocialRecordsMongoQuery(Dictionary<string, List<string>> languagesPerTerms, string[] sources, DateTime fr, DateTime to)
{
try
{
var andList = new BindingList<IMongoQuery>
{
Query.And(Query<SocialRecord>.GTE(record => record.DateCreated, fr),
Query<SocialRecord>.LTE(record => record.DateCreated, to),
Query<SocialRecord>.In(record => record.SocialType, sources))
};
foreach (var languagesPerTerm in languagesPerTerms)
{
var term = languagesPerTerm;
andList.Add(Query.Or(Query.And((Query<SocialRecord>.Where(record => record.TermMonitorIds.Contains(term.Key))),
Query<SocialRecord>.In(record => record.Language, term.Value))));
}
return _collection.FindAs<SocialRecord>(Query.And(andList)).AsQueryable();
}
catch (Exception ex)
{
Log.Error("Exception in the Method GetWallSocialRecords, in the Class MongoSocialRecordRepository", ex);
}
return null;
}
Document Structure
{
"_id" : ObjectId("53a456b27f781d19f40ac76c"),
"DateCreated" : ISODate("2014-06-20T15:35:56.000Z"),
"SocialType" : "facebook",
"RecordId" : "1474971936_10202431655820767",
"UserId" : "1474971936",
"UserProfileUrl" : "********",
"UserProfilePictureUrl" : "/Downloads/v3/432bfeb8-901e-45a4-b739-1f3f48b69d61/facebook/2014-6/1946689/10492432_10202426005479512_740185019259071925_t.jpg",
"Description" : "******",
"MediaHiResUrl" : "",
"MediaLowResUrl" : "",
"MediaMedResUrl" : "",
"SocialCount" : NumberLong(354),
"SocialCountType" : "likes",
"Sentiment" : "",
"SentimentScore" : "0.0000000",
"IsLocalContent" : true,
"IsExactMatch" : true,
"IsHashTag" : false,
"IsActive" : false,
"MediaType" : "image",
"TermMonitorIds" : [
"432bfeb8-901e-45a4-b739-1f3f48b69d61"
],
"UserName" : "***",
"DisplayName" : "",
"DirectUrl" : "",
"IsUk" : true,
"IsEnglish" : true,
"Language" : "en",
"Location" : "GB",
"DataVersion" : "v3"
}
Edit
I need to get below Query:
query {
"DateCreated" : {
"$gte" : ISODate("2015-02-02T16:55:37.979Z"),
"$lte" : ISODate("2015-02-09T16:55:37.979Z")
},
"SocialType" : {
"$in" : ["twitter"]
},
"$or" : [{
"TermMonitorIds" : "b34b8bea-d1e6-4d05-bd25-5d07ad0b691e",
"Language" : {
"$in" : ["zh", "eo", "ja"]
}
}, {
"TermMonitorIds" : "c8497f52-70dd-47b6-8abe-afac42c3a009",
"Language" : {
"$in" : ["zh", "eo", "ja"]
}
}
]
}
Any suggestions will be appreciated.

I found a way to do that, Might help anyone :
public IQueryable<SocialRecord> GetWallSocialRecordsMongoQuery(Dictionary<string, List<string>> languagesPerTerms, string[] sources, DateTime fr, DateTime to)
{
try
{
var list = new BindingList<IMongoQuery>();
foreach (var languagesPerTerm in languagesPerTerms)
{
var term = languagesPerTerm;
list.Add(!term.Value.Contains("All")
? Query.And(
Query<SocialRecord>.Where(record => record.TermMonitorIds.Contains(term.Key)),
Query<SocialRecord>.In(record => record.Language, term.Value))
: Query<SocialRecord>.Where(record => record.TermMonitorIds.Contains(term.Key)));
}
var query = Query.And(Query<SocialRecord>.GTE(record => record.DateCreated, fr),
Query<SocialRecord>.LTE(record => record.DateCreated, to),
Query<SocialRecord>.In(record => record.SocialType, sources),
Query.Or(list));
//var x = _collection.FindAs<SocialRecord>(query).Explain();
return _collection.FindAs<SocialRecord>(query).AsQueryable();
}
catch (Exception ex)
{
Log.Error("Exception in the Method GetWallSocialRecords, in the Class MongoSocialRecordRepository", ex);
}
return null;
}
Please not that MongoDB Queries with the $or operator can use separate indexes on each clause of the $or expression.

MongoDb Query Array on C#

Im dba and development a C# Web App to get information from Mongodb, but when i try to get information from an array attribute, the C# show me all element of the document that contain the element that i find. I just want to get the specific element, not all.
I have this document:
{
"_id" : ObjectId("53b1b7dcb830980744687bd4"),
"i_nombre" : "Centro Comercial",
"i_direccion" : {
"i_d_pais" : "Panamá",
"i_d_ciudad" : "Panamá",
"i_d_provincia" : "Panamá",
"i_d_distrito" : "Chilibre",
"i_d_corregimiento" : "Alcalde Diaz",
"i_d_calle" : "Primera"
},
"i_correo_e" : "imqv#imqv.com.pa",
"i_telefono" : {
"i_t_iglesia" : "268-5000",
"i_t_colegio" : "268-5001",
"i_t_radio" : "268-5002"
},
"i_estado" : 1,
"i_sector" : [
{
"_id" : ObjectId("53b6d903b8309826e891eefe"),
"i_s_color" : "Amarillo",
"i_s_localizacion" : {
"i_s_l_provincia" : "Panamá",
"i_s_l_distrito" : "Chilibre",
"i_s_l_corregimiento" : "San Miguelito"
},
"i_s_supervisor" : []
},
{
"_id" : ObjectId("53b6d903b8309826e89100f0"),
"i_s_color" : "Rojo",
"i_s_localizacion" : {
"i_s_l_provincia" : "Panamá",
"i_s_l_distrito" : "Arraijan",
"i_s_l_corregimiento" : "Burunga"
},
"i_s_supervisor" : []
},
{
"_id" : ObjectId("53b6d903b8309826e89220f0"),
"i_s_color" : "Azul",
"i_s_localizacion" : {
"i_s_l_provincia" : "Panamá",
"i_s_l_distrito" : "Colon",
"i_s_l_corregimiento" : "Chilibre"
},
"i_s_supervisor" : []
}
]
};
When i execute this query on mongodb
db.iglesia.find
(
{ "_id" : ObjectId("53b1b7dcb830980744687bd4") }
, {
i_sector: {
$elemMatch: { "_id" : ObjectId("53b6d903b8309826e891eefe") }
}
}
);
i get:
{
"_id" : ObjectId("53b1b7dcb830980744687bd4"),
"i_sector" : [
{
"_id" : ObjectId("53b6d903b8309826e891eefe"),
"i_s_color" : "Amarillo",
"i_s_localizacion" : {
"i_s_l_provincia" : "Panamá",
"i_s_l_distrito" : "Chilibre",
"i_s_l_corregimiento" : "San Miguelito"
},
"i_s_supervisor" : []
}
]
}
Question: How i can get the same result with C#? Just with the $elemMatch that i set on Query.

The reason is that your Mongo shell query is not a find with 2 filters, by church id and sector, it is actually a find by church id and a projection by sector id. You have arranged the braces funny, that's all.
This has 2 filters and no projection:
db.iglesia.find ({"_id": "53b1b7dcb830980744687bd4", i_sector: { $elemMatch: {"_id" : "53b6d903b8309826e891eefe"}}}).pretty()
This has a filter and projection:
db.iglesia.find ({"_id": "53b1b7dcb830980744687bd4"}, {i_sector: { $elemMatch: { "_id": "53b6d903b8309826e891eefe"}}}).pretty()
A complete C# example, for these 2 cases using a newer driver, version 2.3, here on rextester.