Firstly i sorry for my bad english. I have one node in my server.
My index stats below this :
{
"_shards" : {
"total" : 4,
"successful" : 4,
"failed" : 0
},
"_all" : {
"primaries" : {
"docs" : {
"count" : 1951593,
"deleted" : 0
},
"store" : {
"size_in_bytes" : 594806126,
"reserved_in_bytes" : 0
},
"indexing" : {
"index_total" : 0,
"index_time_in_millis" : 0,
"index_current" : 0,
"index_failed" : 0,
"delete_total" : 0,
"delete_time_in_millis" : 0,
"delete_current" : 0,
"noop_update_total" : 0,
"is_throttled" : false,
"throttle_time_in_millis" : 0
},
"get" : {
"total" : 0,
"time_in_millis" : 0,
"exists_total" : 0,
"exists_time_in_millis" : 0,
"missing_total" : 0,
"missing_time_in_millis" : 0,
"current" : 0
},
"search" : {
"open_contexts" : 0,
"query_total" : 5900,
"query_time_in_millis" : 5539,
"query_current" : 0,
"fetch_total" : 1464,
"fetch_time_in_millis" : 488659,
"fetch_current" : 0,
"scroll_total" : 44,
"scroll_time_in_millis" : 6017059,
"scroll_current" : 0,
"suggest_total" : 0,
"suggest_time_in_millis" : 0,
"suggest_current" : 0
},
"merges" : {
"current" : 0,
"current_docs" : 0,
"current_size_in_bytes" : 0,
"total" : 0,
"total_time_in_millis" : 0,
"total_docs" : 0,
"total_size_in_bytes" : 0,
"total_stopped_time_in_millis" : 0,
"total_throttled_time_in_millis" : 0,
"total_auto_throttle_in_bytes" : 83886080
},
"refresh" : {
"total" : 8,
"total_time_in_millis" : 0,
"external_total" : 8,
"external_total_time_in_millis" : 0,
"listeners" : 0
},
"flush" : {
"total" : 4,
"periodic" : 0,
"total_time_in_millis" : 0
},
"warmer" : {
"current" : 0,
"total" : 4,
"total_time_in_millis" : 0
},
"query_cache" : {
"memory_size_in_bytes" : 0,
"total_count" : 1668,
"hit_count" : 0,
"miss_count" : 1668,
"cache_size" : 0,
"cache_count" : 0,
"evictions" : 0
},
"fielddata" : {
"memory_size_in_bytes" : 0,
"evictions" : 0
},
"completion" : {
"size_in_bytes" : 0
},
"segments" : {
"count" : 45,
"memory_in_bytes" : 317076,
"terms_memory_in_bytes" : 174240,
"stored_fields_memory_in_bytes" : 22984,
"term_vectors_memory_in_bytes" : 0,
"norms_memory_in_bytes" : 25920,
"points_memory_in_bytes" : 0,
"doc_values_memory_in_bytes" : 93932,
"index_writer_memory_in_bytes" : 0,
"version_map_memory_in_bytes" : 0,
"fixed_bit_set_memory_in_bytes" : 0,
"max_unsafe_auto_id_timestamp" : -1,
"file_sizes" : { }
},
"translog" : {
"operations" : 0,
"size_in_bytes" : 220,
"uncommitted_operations" : 0,
"uncommitted_size_in_bytes" : 220,
"earliest_last_modified_age" : 5016454
},
"request_cache" : {
"memory_size_in_bytes" : 0,
"evictions" : 0,
"hit_count" : 0,
"miss_count" : 0
},
"recovery" : {
"current_as_source" : 0,
"current_as_target" : 0,
"throttle_time_in_millis" : 0
}
}
and my mappings below this :
{
"marka_listesi" : {
"mappings" : {
"properties" : {
"basvuruNo" : {
"type" : "keyword"
},
"basvuruTarihi" : {
"type" : "date"
},
"durum" : {
"type" : "text",
"analyzer" : "turkish_analyzer"
},
"evrakNo" : {
"type" : "keyword"
},
"ilanBultenNo" : {
"type" : "keyword"
},
"ilanBultenTarihi" : {
"type" : "date"
},
"korumaTarihi" : {
"type" : "date"
},
"markaAdi" : {
"type" : "text",
"analyzer" : "turkish_analyzer"
},
"markaID" : {
"type" : "integer"
},
"markaLogoUrl" : {
"type" : "text"
},
"sahip" : {
"type" : "text",
"analyzer" : "turkish_analyzer"
},
"siniflar" : {
"type" : "keyword"
},
"sonDegisiklik" : {
"type" : "date"
},
"sonIslem" : {
"type" : "text",
"analyzer" : "turkish_analyzer"
},
"sonIslemBaslik" : {
"type" : "text",
"analyzer" : "turkish_analyzer"
},
"sonIslemTarihi" : {
"type" : "date"
},
"tescilBultenNo" : {
"type" : "keyword"
},
"tescilBultenTarihi" : {
"type" : "date"
},
"tescilNo" : {
"type" : "keyword"
},
"tescilTarihi" : {
"type" : "date"
},
"tur" : {
"type" : "text",
"analyzer" : "turkish_analyzer"
},
"uTescilNo" : {
"type" : "keyword"
},
"vekil" : {
"type" : "text",
"analyzer" : "turkish_analyzer"
},
"vekilFirma" : {
"type" : "text",
"analyzer" : "turkish_analyzer"
}
}
}
}
}
This index have 2 million record but size average 400-500 mb.
I need get all document in index with dynamic filter and compare in my class method. I'am using scroll.all api from nest but documents coming and adding all documents to my model list very very slow(1 minutes). I seen Maximum cpu percentage for query processing 20%.
I need several second for query complete. This is my nest query :
public static List<Models.Model.Sonuc.MarkaListesi>GetDocumentsOnSelectedClasses(string _indexName,ElasticClient _client,string[] _siniflar)
{
var _docs = new List<Models.Model.Sonuc.MarkaListesi>();
// number of slices in slice scroll
var scrollObserver = _client.ScrollAll<Models.Model.Sonuc.MarkaListesi>("1m", 4, s => s
.MaxDegreeOfParallelism(4)
.Search(se => se
.Index(_indexName)
.Query(q => q.Bool(b => b.Filter(fq=>fq.Terms(t=>t.Field("siniflar").Terms(_siniflar)))))
.Size(1000)
)
);
var waitHandle = new ManualResetEvent(false);
Exception exception = null;
var scrollAllObserver = new ScrollAllObserver<Models.Model.Sonuc.MarkaListesi>(
onNext: response =>
{
// do something with the documents
_docs.AddRange(response.SearchResponse.Documents);
},
onError: e =>
{
exception = e;
waitHandle.Set();
},
onCompleted: () => waitHandle.Set()
);
scrollObserver.Subscribe(scrollAllObserver);
waitHandle.WaitOne();
if (exception != null)
{
throw exception;
}
return _docs;
}
Finally my question is : how can i increase my search speed ?
Related
I have a collection Zonedetails shown below. below.
I am using C# to Insert or Update a Unit to Units array. If it is an insert I can insert Area blank. If it is an update It should only update the UnitName.
{
"Code" : "Zone1",
"Name" : "ZoneName1",
"Units" : [
{
"UnitCode" : "Unitcode1",
"UnitName" : "UnitCodeName",
"Areas" : [
{
"AreaCode" : "AreaCode1",
"AreaName" : "AreaName1"
}
]
}
]
}
{
"Code" : "Zone2",
"Name" : "ZoneName2",
"Units" : [
{
"UnitCode" : "UnitCode2",
"UnitName" : "UnityName2",
"Areas" : [
{
"AreaCode" : "Areacode2",
"AreaName" : "AreaName2"
}
]
}
]
}
{
"Code" : "Zone3",
"Name" : "ZoneName3",
"Units" : [
{
"UnitCode" : "UnitCode3",
"UnitName" : "UnitName3",
"Areas" : [
{
"AreaCode" : "Areadcode3",
"AreaName" : "AreaName3"
},
{
"AreaCode" : "AreaCode4",
"AreaName" : "Areaname4"
},
{
"AreaCode" : "AreaCode5",
"AreaName" : "Areaname5"
}
]
},
{
"UnitCode" : "UnitCode6",
"UnitName" : "UnitName6",
"Areas" : [
{
"AreaCode" : "AreaCode10",
"AreaName" : "AreaName10"
},
{
"AreaCode" : "AreaCOde11",
"AreaName" : "AreaName10"
},
{
"AreaCode" : "AreaCode12",
"AreaName" : "AreaName12"
}
]
}
]
}
I have writtent a C# code shown below. But getting "The positional operator did not find the match needed from the query"error if the Unit Code does not exist. Added not before unitCode check.
var mongoCollection = _context.GetCollection<Zone>("ZoneDetail");
var filter = Builders<Zone>.Filter.Where(x => x.Code == zoneCode && !x.Units.Any(u => u.UnitCode == unit.UnitCode));
var updateUnitCode = Builders<Zone>.Update.Set(x => x.Units.ElementAt(-1).UnitCode, unit.UnitCode);
var updateUnitName = Builders<Zone>.Update.Set(x => x.Units.ElementAt(-1).UnitName, unit.UnitName);
var result = await mongoCollection.UpdateOneAsync(filter, Builders<Zone>.Update.Combine(updateUnitCode, updateUnitName), new UpdateOptions { IsUpsert = true});
Your error message suggest that filter is not able to find matching element.
Either try to change your input or change your filter criteria.
I'm trying to receive new array with only sub fields filled using MongoDB C# Driver. For example I have the following document:
{
"_id" : "fca739d0-cddd-4762-b680-597d2996404b",
"Status" : 1,
"AccountId" : "1112",
"Timestamp" : ISODate("2016-04-27T13:46:01.888Z"),
"CartItems" : [
{
"ProductId" : "222",
"Price" : 100,
"ShippingPrice" : 20,
"Quantity" : 3
},
{
"ProductId" : "504",
"Price" : 200,
"ShippingPrice" : 20,
"Quantity" : 2
},
{
"ProductId" : "504",
"Price" : 200,
"ShippingPrice" : 20,
"Quantity" : 1
},
{
"ProductId" : "504",
"Price" : 200,
"ShippingPrice" : 20,
"Quantity" : 1
}
]
}
I'm trying to receive document with new array of CartItems with only ProductId so the response will look like:
{
"_id" : null,
"Status" : 0,
"AccountId" : null,
"Timestamp" : ISODate("2016-04-27T13:46:01.888Z"), (**default)
"CartItems" : [
{
"ProductId" : "222",
"Price" : 0,
"ShippingPrice" : 0,
"Quantity" : 0
},
{
"ProductId" : "504",
"Price" : 0,
"ShippingPrice" : 0,
"Quantity" : 0
},
{
"ProductId" : "504",
"Price" : 0,
"ShippingPrice" : 0,
"Quantity" : 0
},
{
"ProductId" : "504",
"Price" : 0,
"ShippingPrice" : 0,
"Quantity" : 0
}
]
}
The projection I tried (using C#) was
ProjectionDefinition<Cart, Cart> projectionDefinition = Builders<Cart>.Projection.Include(doc => doc.CartItems[0].ProductId) .Exclude(doc => doc.Id);
But the result is CartItems array with all default values (include ProductId). What I do wrong?
According to the documentation:
Use the dot notation to refer to the embedded field
https://docs.mongodb.com/manual/tutorial/project-fields-from-query-results/#return-specific-fields-in-embedded-documents
So your projection should look like this:
Builders<Cart>.Projection
.Include("CartItems.ProductId")
.Exclude(doc => doc.Id);
{
"_id" : ObjectId("5431f38c4ba4dd20408b0432"),
"UserID" : "1",
"Status" : {
}
},
"ListFilterType" : 1,
"IssueCategories" : [
{
"_id" : ObjectId("000000000000000000000000"),
"IssueCategoryID" : 2,
"IssueCagetoryName" : "test",
"MatchKeyword" : "test",
"MatchKeywordID" : 2
}
]
}
edit :
my expactation is like this.
{
"_id" : ObjectId("5431f38c4ba4dd20408b0432"),
"UserID" : "1",
"Status" : {
}
},
"ListFilterType" : 1,
"IssueCategories" : [
{
"_id" : ObjectId("000000000000000000000000"),
"IssueCategoryID" : 2,
"IssueCagetoryName" : "test",
"MatchKeyword" : "test",
"MatchKeywordID" : 2
},
{
"_id" : ObjectId("000000000000000000000001"),
"IssueCategoryID" : 3,
"IssueCagetoryName" : "test2",
"MatchKeyword" : "test2",
"MatchKeywordID" : 3
},
{
"_id" : ObjectId("000000000000000000000004"),
"IssueCategoryID" : 4,
"IssueCagetoryName" : "test4",
"MatchKeyword" : "test34",
"MatchKeywordID" : 4
}
]
i have a type list as "IssueCategories". You can see my mongodb structure above. If root _id = my parameter, I want to add to mongodb.
I hope I explained right
thanks
You should use either $push or $addToSet to update document.
$push will add all element in IssueCategories including duplicates also.
While in $addToSet do not insert duplicate elements(object) in array if whole object is present in array.
Using $push:
db.collection.update({
"_id": ObjectId("5431f38c4ba4dd20408b0432") // you can add any other param
}, {
$push: {
SCSIssueCategories: {
"_id": ObjectId("11111111111111111"), //ur id here
"IssueCategoryID": 3,
"IssueCagetoryName": "Sözleşme1",
"MatchKeyword": "abonelik süresi1",
"MatchKeywordID": 3
}
}
})
Here is query using $addToSet:
db.collection.update({
"_id": ObjectId("5431f38c4ba4dd20408b0432")
}, {
$addToSet: {
SCSIssueCategories: {
"_id": ObjectId("11111111111111111"), //ur id here
"IssueCategoryID": 3,
"IssueCagetoryName": "Sözleşme1",
"MatchKeyword": "abonelik süresi1",
"MatchKeywordID": 3
}
}
})
**If single param have different value then it will be inserted in array using $addToSet. **
Im dba and development a C# Web App to get information from Mongodb, but when i try to get information from an array attribute, the C# show me all element of the document that contain the element that i find. I just want to get the specific element, not all.
I have this document:
{
"_id" : ObjectId("53b1b7dcb830980744687bd4"),
"i_nombre" : "Centro Comercial",
"i_direccion" : {
"i_d_pais" : "Panamá",
"i_d_ciudad" : "Panamá",
"i_d_provincia" : "Panamá",
"i_d_distrito" : "Chilibre",
"i_d_corregimiento" : "Alcalde Diaz",
"i_d_calle" : "Primera"
},
"i_correo_e" : "imqv#imqv.com.pa",
"i_telefono" : {
"i_t_iglesia" : "268-5000",
"i_t_colegio" : "268-5001",
"i_t_radio" : "268-5002"
},
"i_estado" : 1,
"i_sector" : [
{
"_id" : ObjectId("53b6d903b8309826e891eefe"),
"i_s_color" : "Amarillo",
"i_s_localizacion" : {
"i_s_l_provincia" : "Panamá",
"i_s_l_distrito" : "Chilibre",
"i_s_l_corregimiento" : "San Miguelito"
},
"i_s_supervisor" : []
},
{
"_id" : ObjectId("53b6d903b8309826e89100f0"),
"i_s_color" : "Rojo",
"i_s_localizacion" : {
"i_s_l_provincia" : "Panamá",
"i_s_l_distrito" : "Arraijan",
"i_s_l_corregimiento" : "Burunga"
},
"i_s_supervisor" : []
},
{
"_id" : ObjectId("53b6d903b8309826e89220f0"),
"i_s_color" : "Azul",
"i_s_localizacion" : {
"i_s_l_provincia" : "Panamá",
"i_s_l_distrito" : "Colon",
"i_s_l_corregimiento" : "Chilibre"
},
"i_s_supervisor" : []
}
]
};
When i execute this query on mongodb
db.iglesia.find
(
{ "_id" : ObjectId("53b1b7dcb830980744687bd4") }
, {
i_sector: {
$elemMatch: { "_id" : ObjectId("53b6d903b8309826e891eefe") }
}
}
);
i get:
{
"_id" : ObjectId("53b1b7dcb830980744687bd4"),
"i_sector" : [
{
"_id" : ObjectId("53b6d903b8309826e891eefe"),
"i_s_color" : "Amarillo",
"i_s_localizacion" : {
"i_s_l_provincia" : "Panamá",
"i_s_l_distrito" : "Chilibre",
"i_s_l_corregimiento" : "San Miguelito"
},
"i_s_supervisor" : []
}
]
}
Question: How i can get the same result with C#? Just with the $elemMatch that i set on Query.
The reason is that your Mongo shell query is not a find with 2 filters, by church id and sector, it is actually a find by church id and a projection by sector id. You have arranged the braces funny, that's all.
This has 2 filters and no projection:
db.iglesia.find ({"_id": "53b1b7dcb830980744687bd4", i_sector: { $elemMatch: {"_id" : "53b6d903b8309826e891eefe"}}}).pretty()
This has a filter and projection:
db.iglesia.find ({"_id": "53b1b7dcb830980744687bd4"}, {i_sector: { $elemMatch: { "_id": "53b6d903b8309826e891eefe"}}}).pretty()
A complete C# example, for these 2 cases using a newer driver, version 2.3, here on rextester.
any one can help me,,
my brain stacking for make this collection to mapreduce,
how i can get PurchaseAmount, PurchaseReturnAmount, and TotalAmount, with key by "year".
{
"_id" : {
"OwnerId" : "coba#aja.com",
"SupplierId" : BinData(3,"AYC8In8bFkGYNx34poQLlg=="),
"GroupId" : BinData(3,"mzWBCilngEGd72YpeyijcQ=="),
"ProductId" : BinData(3,"gZ2g/syue06v8b88+0pqRA=="),
"Date" : ISODate("2013-01-15T00:00:00Z")
},
"value" : {
"OwnerId" : "oetawan#dokuku.com",
"Date" : ISODate("2013-01-15T00:00:00Z"),
"SupplierId" : BinData(3,"AYC8In8bFkGYNx34poQLlg=="),
"SupplierName" : "Matahari",
"GroupId" : BinData(3,"mzWBCilngEGd72YpeyijcQ=="),
"GroupCode" : "Umum",
"GroupName" : "Umum",
"ProductId" : BinData(3,"gZ2g/syue06v8b88+0pqRA=="),
"ProductCode" : null,
"ProductBarcode" : "IPAD2",
"ProductName" : "iPad 2",
"PurchaseAmount" : 19500000,
"PurchaseReturnAmount" : 0,
"TotalAmount" : 19500000
}
}
i want to use this map reduce to C#
Here's a shell example using MR. You could also do it with the aggregation framework of course, but presumably you want to run this in the background;
db.so.mapReduce(
function () { /* map */
var key = this._id.Date.getFullYear();
emit(key, { PurchaseAmount:this.value.PurchaseAmount, PurchaseReturnAmount:this.value.PurchaseReturnAmount, TotalAmount:this.value.TotalAmount } );
},
function (key, array) { /* reduce*/
var result = { PurchaseAmount: 0, PurchaseReturnAmount: 0, TotalAmount: 0 };
for (var i = 0; i < array.length; i++) {
{
result.PurchaseAmount += array[i].PurchaseAmount;
result.PurchaseReturnAmount += array[i].PurchaseReturnAmount;
result.TotalAmount += array[i].TotalAmount;
}
}
return result;
},
{
out: "so_mr_example",
query:{}
}
);
db.so_mr_example.find();
Produces something like;
{ "_id" : 2013, "value" : { "PurchaseAmount" : 39000000, "PurchaseReturnAmount" : 0, "TotalAmount" : 39000000 } }
{ "_id" : 2014, "value" : { "PurchaseAmount" : 19500000, "PurchaseReturnAmount" : 0, "TotalAmount" : 19500000 } }