I'm using Mongo 4 with the latest C# driver. My application creates DBs and Collections on the fly and I want to enable sharding. I'm using the following code:
if (!ShardingEnabled) return;
var database = collection.Database;
var databaseName = database.DatabaseNamespace.DatabaseName;
var collectionName = collection.CollectionNamespace.CollectionName;
var shardDbScript = $"{{ enableSharding: \"{databaseName}\" }}";
var shardDbResult = database.RunCommand<MongoDB.Bson.BsonDocument>(new MongoDB.Bson.BsonDocument() {
{ "eval",shardDbScript }
});
var adminDb = Client.GetDatabase("admin");
var shardScript = $"{{shardCollection: \"{databaseName}.{collectionName}\"}}";
var commandDoc = new BsonDocumentCommand<MongoDB.Bson.BsonDocument>(new MongoDB.Bson.BsonDocument() {
{ "eval",shardScript }
});
var response = adminDb.RunCommand(commandDoc);
I get an 'ok' response back from mongo, but my dbs arent sharded.
Output from sh.status()
{
"_id" : "uat_Test_0",
"primary" : "SynoviaShard2",
"partitioned" : false,
"version" : {
"uuid" : UUID("69576c3b-817c-4853-bb02-ea0a8e9813a4"),
"lastMod" : 1
}
}
How can I enable sharding from within C#?
I figured it out. This is how you shard a database and its collections from c#, note, that the sharding key index must already exist:
if (!ShardingEnabled) return;
var database = collection.Database;
var adminDb = Client.GetDatabase("admin");
var configDb = Client.GetDatabase("config");
//var dbs = Client.ListDatabaseNames().ToList();
var databaseName = database.DatabaseNamespace.DatabaseName;
var collectionName = collection.CollectionNamespace.CollectionName;
var shardDbResult = adminDb.RunCommand<MongoDB.Bson.BsonDocument>(new MongoDB.Bson.BsonDocument() {
{ "enableSharding",$"{databaseName}" }
});
var shardScript = $"{{shardCollection: \"{databaseName}.{collectionName}\"}}";
var commandDict = new Dictionary<string,object>();
commandDict.Add("shardCollection", $"{databaseName}.{collectionName}");
commandDict.Add("key",new Dictionary<string,object>(){{"_id","hashed"}});
var bsonDocument = new MongoDB.Bson.BsonDocument(commandDict);
var commandDoc = new BsonDocumentCommand<MongoDB.Bson.BsonDocument>(bsonDocument);
var response = adminDb.RunCommand(commandDoc);
Related
This bounty has ended. Answers to this question are eligible for a +50 reputation bounty. Bounty grace period ends in 9 hours.
Keppy is looking for an answer from a reputable source.
I have a Elastic Client from Elasticsearch.Net which fetching data from InMemoryConnection and add Query filter on the search but result is not filtering. Its returning entire data from responseBody as result.
Am I missing something or this is how InMemoryConnection is working?
CurrenciesDTO.cs
internal class CurrenciesDTO
{
[Keyword(Name = "CCY")]
public string CCY { get; set; }
}
Program.cs
using ConsoleApp_Elastic;
using Elasticsearch.Net;
using Nest;
using Newtonsoft.Json;
using System.Collections.Generic;
using System.Text;
using System.Threading;
List<CurrenciesDTO> listCurrencies = new List<CurrenciesDTO> { new CurrenciesDTO() { CCY = "GEL" }, new CurrenciesDTO() { CCY = "INR" }, new CurrenciesDTO() { CCY = "JPY" }, new CurrenciesDTO() { CCY = "USD" } };
var response = new
{
took = 1,
timed_out = false,
_shards = new
{
total = 1,
successful = 1,
skipped = 0,
failed = 0
},
hits = new
{
total = new
{
value = 193,
relation = "eq"
},
max_score = 1.0,
hits = Enumerable.Range(0, listCurrencies.Count).Select(i => (object)new
{
_index = "test.my.currencies",
_type = "_doc",
_id = listCurrencies[i].CCY,
_score = 1.0,
_source = new
{
CCY = listCurrencies[i].CCY,
}
})
}
};
string json = JsonConvert.SerializeObject(response);
var responseBody = Encoding.UTF8.GetBytes(json);
ConnectionSettings connectionSettings = new ConnectionSettings(new InMemoryConnection(responseBody, 200));
connectionSettings.OnRequestCompleted(apiCallDetails =>
{
if (apiCallDetails.RequestBodyInBytes != null)
{// not reaching here
Console.WriteLine(
$"{apiCallDetails.HttpMethod} {apiCallDetails.Uri} " +
$"{Encoding.UTF8.GetString(apiCallDetails.RequestBodyInBytes)}");
}
});
var client = new ElasticClient(connectionSettings);
var filterItems = new List<Func<QueryContainerDescriptor<CurrenciesDTO>, QueryContainer>>();
filterItems.Add(p => p.Term(v => v.Field(f=>f.CCY).Value("USD")));
var result = await client.SearchAsync<CurrenciesDTO>(s => s
.Index("test.my.currencies")
.Query(q => q.Bool(x => x.Filter(filterItems))), CancellationToken.None);
// .Query(q => q.Term(p => p.CCY, "USD")));
//expected 1 record but 4 records are returned.
foreach (var a in result.Documents.ToArray())
{
Console.WriteLine(a.CCY);
}
Console.ReadLine();
Yes, this is by design. InMemoryConnection was created to make unit testing easier and won't be much help with validating actual queries.
For making sure that Elasticsearch is configured the way you are expecting it to be and that queries sent to Elasticsearch are valid I would suggest using Testcontainers.
Simple test would look like:
spin up new docker instance of Elasticsearch with Testcontainers help
index some data
run you code against Elasticsearch running inside container
I have an api based on asp.net core 2.2 in which i am building up an array of ips(strings type) like this
[HttpGet ("{nsp}/geolocation")]
[ResponseCache (Duration = 0, Location = ResponseCacheLocation.None, NoStore = true)]
public async Task<dynamic> getLocation (string nsp) {
nsp = "/"+nsp;
// string ipInfoBaseUrl = "http://ip-api.com/json/";
string baseUrl = "http://ip-api.com/batch";
// string userIpAddress = "197.157.194.90";
// string ipUrl = ipInfoBaseUrl + userIpAddress;
// var client = _httpClientFactory.CreateClient();
// var result = await client.PostAsync(baseUrl,new StringContent(JsonConvert.SerializeObject(finals), System.Text.Encoding.UTF8, "application/json"));
// // var final = Newtonsoft.Json.JsonConvert.DeserializeObject<UserLocation>(result);
// Console.WriteLine(finals+" --result-----");
// return Ok(result);
var match = new BsonDocument ();
var group = new BsonDocument ();
var project = new BsonDocument ();
var sort = new BsonDocument ();
var addFields = new BsonDocument ();
var pipeline = new [] { new BsonDocument () };
/* #Aggregation : Stage-1 */
match = new BsonDocument ("$match",
new BsonDocument {
{
"nsp" , nsp
}
});
/* #Aggregation : Stage-2 */
group = new BsonDocument("$group",
new BsonDocument
{ {
"_id", "null"
},
{ "geoLocations",
new BsonDocument("$addToSet", "$visitor.ip")
}
});
/* #Aggregation : Stage-3 */
project = new BsonDocument ("$project", new BsonDocument { { "_id", 0 }});
pipeline = new [] { match, group,project};
var list = await DbService.tickets.AggregateAsync<BsonDocument> (pipeline, new AggregateOptions { UseCursor = true, BatchSize = batchCount });
while (await list.MoveNextAsync ()) {
var list_real = new List<BsonValue> ();
foreach (var data in list.Current.ToArray ()) {
list_real.Add (data);
}
return list_real.ToJson ();
}
return new BsonArray ().ToJson ();
}
It is returning result like this
[
{
" geoLocations": [
"122.8.208.9",
"196.62.107.243",
"182.188.38.219",
"39.50.244.198",
"39.51.40.251",
"103.20.134.56",
"103.228.156.83",
"202.143.125.21",
"196.62.151.47",
"45.116.232.50",
"39.57.128.75",
"103.18.8.60",
"202.143.125.20",
"182.190.252.96",
"119.153.56.2",
"46.101.89.227",
"196.194.172.211",
"192.168.20.186",
"64.233.173.146",
"104.236.195.147",
"39.50.156.242",
"103.255.5.58"
]
}
]
How can i get comma separated string from this result like
"111.92.158.82","202.142.168.162","122.8.157.172",.....
From very first i am getting all ips from all documents from my collection and forming an array of ips.But my ultimate goal is to form a comma separated string from that array because i have to pass that comma separated string of ips into an api to get ips locations.
I am using asp.net core and c#. How can i achieve this?
Assuming you want a single comma-separated result string containing all IP addresses, replace the method signature with IEnumerable<string>, and replace the bottom of the method with. Just use whatever you need for your result and get rid of the rest.
var list = await DbService.tickets.AggregateAsync<BsonDocument> (pipeline,
new AggregateOptions
{
UseCursor = true,
BatchSize = batchCount
});
var result = new List<string>();
while (await list.MoveNextAsync())
result.AddRange(list.Current.Cast<string>());
return string.Join(',', result);
I'm not sure why you're doing everything with BsonDocuments, you can just iterate over the data directly and return strings.
Also, consider upgrading to .NET Core 3, which you can then use C# 8's async enumerable. You'll also be able to use the new JSON functionality built-in .NET Core 3.
This will return:
"122.8.208.9,196.62.107.243,182.188.38.219,<all the rest>"
I have a collection in MongoDB which has document with names of collections I need to work. I need to query this collection, get all the collection names from the document inside this collection and then query those collections and join them based on ParentId references. Following is the collection which stores the name of other collection
db.AllInfoCollection.find()
{
"_id" : ObjectId("5b83b982a5e17c383c8424f3"),
"CollName" : "Collection1",
},
{
"_id" : ObjectId("5b83b9aaa5e17c383c8424f7"),
"CollName" : "Collection2",
},
{
"_id" : ObjectId("5b83b9afa5e17c383c8424f8"),
"CollName" : "Collection3",
},
{
"_id" : ObjectId("5b83b9b5a5e17c383c8424f9"),
"CollName" : "Collection4",
},
{
"_id" : ObjectId("5b83b9b9a5e17c383c8424fa"),
"CollName" : "Collection5",
},
{
"_id" : ObjectId("5b84f41bc5eb3f1f7c291f94"),
"CollName" : "Collection6",
}
All the above collections (Collection1, Collection2,.... Collection6) are created on run time with empty documents. They are connected to each other with Id and ParentId fields.
Now I need to query this AllInfoCollection, get the collection names and join them and generate the final joined ($lookup) output. I am able to query and get the collection list, but I am not sure how to add lookup projection inside the for loop. Any help would be appreciated.
public void RetrieveDynamicCollection()
{
IMongoDatabase _db = client.GetDatabase("MyDb");
var collectionList = _db.GetCollection<AllInfoCollection>("AllInfoCollection").AsQueryable().Distinct().Select(x => x.CollectionName).ToList();
for(int i = 0; i < collectionList.Count; i++)
{
var collectionName = collectionList[i];
IMongoCollection<BsonDocument> collection = _db.GetCollection<BsonDocument>(collectionName);
var options = new AggregateOptions()
{
AllowDiskUse = false
};
//not able to proceed here
}
}
Finally I was able to retrieve collections dynamically with all required joins(lookup aggregation) as below, hope it helps someone:
public async Task<string> RetrieveDynamicCollection()
{
try
{
IMongoDatabase _db = client.GetDatabase("MyDB");
var list = _db.GetCollection<HazopCollectionInfo>("AllCollectionInfo").AsQueryable().ToList();
var collectionList = list.OrderBy(x => x.CollectionOrder).Select(x => x.CollectionName).Distinct().ToList();
var listOfJoinDocuments = new List<BsonDocument>();
var firstCollection = _db.GetCollection<BsonDocument>(collectionList[0]);
var options = new AggregateOptions()
{
AllowDiskUse = false
};
var previousCollectionName = "";
for (int i = 0; i < collectionList.Count; i++)
{
var collectionName = collectionList[i];
IMongoCollection<BsonDocument> collection = _db.GetCollection<BsonDocument>(collectionName);
if (i == 0)
{
firstCollection = collection;
var firstarray = new BsonDocument("$project", new BsonDocument()
.Add("_id", 0)
.Add(collectionName, "$$ROOT"));
listOfJoinDocuments.Add(firstarray);
}
else
{
var remainingArray = new BsonDocument("$lookup", new BsonDocument()
.Add("localField", previousCollectionName + "." + "Id")
.Add("from", collectionName)
.Add("foreignField", "ParentId")
.Add("as", collectionName));
listOfJoinDocuments.Add(remainingArray);
remainingArray = new BsonDocument("$unwind", new BsonDocument()
.Add("path", "$" + collectionName)
.Add("preserveNullAndEmptyArrays", new BsonBoolean(true)));
listOfJoinDocuments.Add(remainingArray);
}
previousCollectionName = collectionName;
}
// Project the columns
list.OrderBy(x => x.ColumnOrder);
var docProjection = new BsonDocument();
for(int i=0;i<list.Count;i++)
{
docProjection.Add(list[i].ColumnName, "$"+list[i].CollectionName + "." + list[i].FieldName);
}
listOfJoinDocuments.Add(new BsonDocument("$project", docProjection));
PipelineDefinition<BsonDocument, BsonDocument> pipeline = listOfJoinDocuments;
var listOfDocs = new List<BsonDocument>();
using (var cursor = await firstCollection.AggregateAsync(pipeline, options))
{
while (await cursor.MoveNextAsync())
{
var batch = cursor.Current;
foreach (BsonDocument document in batch)
{
listOfDocs.Add(document);
}
}
}
var jsonString = listOfDocs.ToJson(new MongoDB.Bson.IO.JsonWriterSettings { OutputMode = MongoDB.Bson.IO.JsonOutputMode.Strict });
return jsonString;
}
catch(Exception ex)
{
throw ex;
}
}
I try to execute a search with NEST ElasticClient and getting only the _id of the hits.
Here is my Code:
var client = new ElasticClient();
var searchResponse = client.Search<ElasticResult>(new SearchRequest {
From = this.query.Page * 100,
Size = 100,
Source = new SourceFilter {
Includes = "_id"
},
Query = new QueryStringQuery {
Query = this.query.Querystring
}
});
public class ElasticResult {
public string _id;
}
But the _id of the Documents (ElasticResult-Objects) is always null. What am I doing wrong?
The _id is not part of the _source document, but part of the hit metadata for each hit in the hits array.
The most compact way to return just the _id fields would be with using response filtering which is exposed as FilterPath in NEST
private static void Main()
{
var defaultIndex = "documents";
var pool = new SingleNodeConnectionPool(new Uri("http://localhost:9200"));
var settings = new ConnectionSettings(pool)
.DefaultIndex(defaultIndex)
.DefaultTypeName("_doc");
var client = new ElasticClient(settings);
if (client.IndexExists(defaultIndex).Exists)
client.DeleteIndex(defaultIndex);
client.Bulk(b => b
.IndexMany<object>(new[] {
new { Message = "hello" },
new { Message = "world" }
})
.Refresh(Refresh.WaitFor)
);
var searchResponse = client.Search<object>(new SearchRequest<object>
{
From = 0 * 100,
Size = 100,
FilterPath = new [] { "hits.hits._id" },
Query = new QueryStringQuery
{
Query = ""
}
});
foreach(var id in searchResponse.Hits.Select(h => h.Id))
{
// do something with the ids
Console.WriteLine(id);
}
}
The JSON response from Elasticsearch to the search request looks like
{
"hits" : {
"hits" : [
{
"_id" : "6gs8lmQB_8sm1yFaJDlq"
},
{
"_id" : "6Qs8lmQB_8sm1yFaJDlq"
}
]
}
}
I have a search method that queries Solr for event items. I need to modify it to only get events where the date has not already passed (i.e. Where(x => x.EventDate.Date >= DateTime.Now.Date), but I'm not sure how to add this because I'm not very familiar with Solr. Here's my search function:
public SearchQueryResults Search(string keywords, int page,int perPage, List<Guid> contentTypeFilters, List<Guid> otherFilters, ISortBuilder<SearchResultItem> sortBuilder)
{
var searchFilters = new List<IPredicateBuilder<SearchResultItem>>()
{
new IsSearchablePredicateBuilder()
};
if (contentTypeFilters.Any())
{
var contentTypePredicateBuilder = new ContentTypePredicateBuilder();
contentTypePredicateBuilder.ContentTypes = contentTypeFilters;
searchFilters.Add(contentTypePredicateBuilder);
}
if (otherFilters.Any())
{
var tagFilterBuilder = new TagsAndPredicateBuilder(otherFilters,_sitecoreContext);
searchFilters.Add(tagFilterBuilder);
}
if (string.IsNullOrWhiteSpace(keywords))
{
keywords = "";
}
SearchRequest searchRequest = new SearchRequest();
var queryParams = new Dictionary<string, string>() { };
queryParams.Add("q", keywords);
searchRequest.QueryParameters = queryParams;
searchRequest.SortBy = "";
searchRequest.SortOrder = "";
SearchQuery<SearchResultItem> queryArguments = new SearchQuery<SearchResultItem>();
queryArguments.FilterBuilders = searchFilters;
queryArguments.Page = page;
queryArguments.PerPage = perPage;
queryArguments.FacetsBuilder = new SearchFacetBuilder<SearchResultItem>();
queryArguments.SearchRequest = searchRequest;
queryArguments.IndexName = _indexName;
if (string.IsNullOrWhiteSpace(keywords))
{
queryArguments.QueryBuilders =new List<IPredicateBuilder<SearchResultItem>>();
}
else
{
queryArguments.QueryBuilders = new[] { new KeywordPredicateBuilder<SearchResultItem>(new[] { keywords }) };
}
queryArguments.SortBuilder = sortBuilder;
try
{
var results = _searchManager.GetResults<SearchResultItem>(queryArguments);
SearchQueryResults queryResults = new SearchQueryResults();
queryResults.ResultItems = results.Results;
queryResults.CurrentPage = page;
queryResults.TotalResults = Int32.Parse(results.TotalResults.ToString());
queryResults.TotalPages = (queryResults.TotalResults + perPage - 1) / perPage; ;
return queryResults;
}
catch (Exception exc)
{
Sitecore.Diagnostics.Log.Error("Error with FilteredSearch, could be a loss of connection to the SOLR server: " + exc.Message, this);
return null;
}
}
and here is how it's being called:
Results = _searchService.Search(searchTerm, CurrentPage - 1, 10, contentTypes, searchFilters,
new GenericSortBuilder<SearchResultItem>(q => q.OrderByDescending(r => r.SearchDate)));
How do I add in date filtering so that it only returns items where the date is in the future?
I would add filter query to the list of existing ones filtering the date field. On the documentation page, I was able to find information about fluent API, which could help here
Query.Field("date").From(DateTime.Now)
I'm not C# developer, that this code could have some mistakes, but I think the main idea is clear what needs to be done.