I have a collection in MongoDB which has document with names of collections I need to work. I need to query this collection, get all the collection names from the document inside this collection and then query those collections and join them based on ParentId references. Following is the collection which stores the name of other collection
db.AllInfoCollection.find()
{
"_id" : ObjectId("5b83b982a5e17c383c8424f3"),
"CollName" : "Collection1",
},
{
"_id" : ObjectId("5b83b9aaa5e17c383c8424f7"),
"CollName" : "Collection2",
},
{
"_id" : ObjectId("5b83b9afa5e17c383c8424f8"),
"CollName" : "Collection3",
},
{
"_id" : ObjectId("5b83b9b5a5e17c383c8424f9"),
"CollName" : "Collection4",
},
{
"_id" : ObjectId("5b83b9b9a5e17c383c8424fa"),
"CollName" : "Collection5",
},
{
"_id" : ObjectId("5b84f41bc5eb3f1f7c291f94"),
"CollName" : "Collection6",
}
All the above collections (Collection1, Collection2,.... Collection6) are created on run time with empty documents. They are connected to each other with Id and ParentId fields.
Now I need to query this AllInfoCollection, get the collection names and join them and generate the final joined ($lookup) output. I am able to query and get the collection list, but I am not sure how to add lookup projection inside the for loop. Any help would be appreciated.
public void RetrieveDynamicCollection()
{
IMongoDatabase _db = client.GetDatabase("MyDb");
var collectionList = _db.GetCollection<AllInfoCollection>("AllInfoCollection").AsQueryable().Distinct().Select(x => x.CollectionName).ToList();
for(int i = 0; i < collectionList.Count; i++)
{
var collectionName = collectionList[i];
IMongoCollection<BsonDocument> collection = _db.GetCollection<BsonDocument>(collectionName);
var options = new AggregateOptions()
{
AllowDiskUse = false
};
//not able to proceed here
}
}
Finally I was able to retrieve collections dynamically with all required joins(lookup aggregation) as below, hope it helps someone:
public async Task<string> RetrieveDynamicCollection()
{
try
{
IMongoDatabase _db = client.GetDatabase("MyDB");
var list = _db.GetCollection<HazopCollectionInfo>("AllCollectionInfo").AsQueryable().ToList();
var collectionList = list.OrderBy(x => x.CollectionOrder).Select(x => x.CollectionName).Distinct().ToList();
var listOfJoinDocuments = new List<BsonDocument>();
var firstCollection = _db.GetCollection<BsonDocument>(collectionList[0]);
var options = new AggregateOptions()
{
AllowDiskUse = false
};
var previousCollectionName = "";
for (int i = 0; i < collectionList.Count; i++)
{
var collectionName = collectionList[i];
IMongoCollection<BsonDocument> collection = _db.GetCollection<BsonDocument>(collectionName);
if (i == 0)
{
firstCollection = collection;
var firstarray = new BsonDocument("$project", new BsonDocument()
.Add("_id", 0)
.Add(collectionName, "$$ROOT"));
listOfJoinDocuments.Add(firstarray);
}
else
{
var remainingArray = new BsonDocument("$lookup", new BsonDocument()
.Add("localField", previousCollectionName + "." + "Id")
.Add("from", collectionName)
.Add("foreignField", "ParentId")
.Add("as", collectionName));
listOfJoinDocuments.Add(remainingArray);
remainingArray = new BsonDocument("$unwind", new BsonDocument()
.Add("path", "$" + collectionName)
.Add("preserveNullAndEmptyArrays", new BsonBoolean(true)));
listOfJoinDocuments.Add(remainingArray);
}
previousCollectionName = collectionName;
}
// Project the columns
list.OrderBy(x => x.ColumnOrder);
var docProjection = new BsonDocument();
for(int i=0;i<list.Count;i++)
{
docProjection.Add(list[i].ColumnName, "$"+list[i].CollectionName + "." + list[i].FieldName);
}
listOfJoinDocuments.Add(new BsonDocument("$project", docProjection));
PipelineDefinition<BsonDocument, BsonDocument> pipeline = listOfJoinDocuments;
var listOfDocs = new List<BsonDocument>();
using (var cursor = await firstCollection.AggregateAsync(pipeline, options))
{
while (await cursor.MoveNextAsync())
{
var batch = cursor.Current;
foreach (BsonDocument document in batch)
{
listOfDocs.Add(document);
}
}
}
var jsonString = listOfDocs.ToJson(new MongoDB.Bson.IO.JsonWriterSettings { OutputMode = MongoDB.Bson.IO.JsonOutputMode.Strict });
return jsonString;
}
catch(Exception ex)
{
throw ex;
}
}
Related
I'm using Mongo 4 with the latest C# driver. My application creates DBs and Collections on the fly and I want to enable sharding. I'm using the following code:
if (!ShardingEnabled) return;
var database = collection.Database;
var databaseName = database.DatabaseNamespace.DatabaseName;
var collectionName = collection.CollectionNamespace.CollectionName;
var shardDbScript = $"{{ enableSharding: \"{databaseName}\" }}";
var shardDbResult = database.RunCommand<MongoDB.Bson.BsonDocument>(new MongoDB.Bson.BsonDocument() {
{ "eval",shardDbScript }
});
var adminDb = Client.GetDatabase("admin");
var shardScript = $"{{shardCollection: \"{databaseName}.{collectionName}\"}}";
var commandDoc = new BsonDocumentCommand<MongoDB.Bson.BsonDocument>(new MongoDB.Bson.BsonDocument() {
{ "eval",shardScript }
});
var response = adminDb.RunCommand(commandDoc);
I get an 'ok' response back from mongo, but my dbs arent sharded.
Output from sh.status()
{
"_id" : "uat_Test_0",
"primary" : "SynoviaShard2",
"partitioned" : false,
"version" : {
"uuid" : UUID("69576c3b-817c-4853-bb02-ea0a8e9813a4"),
"lastMod" : 1
}
}
How can I enable sharding from within C#?
I figured it out. This is how you shard a database and its collections from c#, note, that the sharding key index must already exist:
if (!ShardingEnabled) return;
var database = collection.Database;
var adminDb = Client.GetDatabase("admin");
var configDb = Client.GetDatabase("config");
//var dbs = Client.ListDatabaseNames().ToList();
var databaseName = database.DatabaseNamespace.DatabaseName;
var collectionName = collection.CollectionNamespace.CollectionName;
var shardDbResult = adminDb.RunCommand<MongoDB.Bson.BsonDocument>(new MongoDB.Bson.BsonDocument() {
{ "enableSharding",$"{databaseName}" }
});
var shardScript = $"{{shardCollection: \"{databaseName}.{collectionName}\"}}";
var commandDict = new Dictionary<string,object>();
commandDict.Add("shardCollection", $"{databaseName}.{collectionName}");
commandDict.Add("key",new Dictionary<string,object>(){{"_id","hashed"}});
var bsonDocument = new MongoDB.Bson.BsonDocument(commandDict);
var commandDoc = new BsonDocumentCommand<MongoDB.Bson.BsonDocument>(bsonDocument);
var response = adminDb.RunCommand(commandDoc);
I try to execute a search with NEST ElasticClient and getting only the _id of the hits.
Here is my Code:
var client = new ElasticClient();
var searchResponse = client.Search<ElasticResult>(new SearchRequest {
From = this.query.Page * 100,
Size = 100,
Source = new SourceFilter {
Includes = "_id"
},
Query = new QueryStringQuery {
Query = this.query.Querystring
}
});
public class ElasticResult {
public string _id;
}
But the _id of the Documents (ElasticResult-Objects) is always null. What am I doing wrong?
The _id is not part of the _source document, but part of the hit metadata for each hit in the hits array.
The most compact way to return just the _id fields would be with using response filtering which is exposed as FilterPath in NEST
private static void Main()
{
var defaultIndex = "documents";
var pool = new SingleNodeConnectionPool(new Uri("http://localhost:9200"));
var settings = new ConnectionSettings(pool)
.DefaultIndex(defaultIndex)
.DefaultTypeName("_doc");
var client = new ElasticClient(settings);
if (client.IndexExists(defaultIndex).Exists)
client.DeleteIndex(defaultIndex);
client.Bulk(b => b
.IndexMany<object>(new[] {
new { Message = "hello" },
new { Message = "world" }
})
.Refresh(Refresh.WaitFor)
);
var searchResponse = client.Search<object>(new SearchRequest<object>
{
From = 0 * 100,
Size = 100,
FilterPath = new [] { "hits.hits._id" },
Query = new QueryStringQuery
{
Query = ""
}
});
foreach(var id in searchResponse.Hits.Select(h => h.Id))
{
// do something with the ids
Console.WriteLine(id);
}
}
The JSON response from Elasticsearch to the search request looks like
{
"hits" : {
"hits" : [
{
"_id" : "6gs8lmQB_8sm1yFaJDlq"
},
{
"_id" : "6Qs8lmQB_8sm1yFaJDlq"
}
]
}
}
I want to make aggregate functions in Solr I found the way from this Post
But I can't implement it in SolrNet
How to Implement JSON Facet API in SolrNet
ISolrOperations<DeviceReadings> solr = connection.GetSolrInstance();
QueryOptions queryOption = new QueryOptions
{
Rows = 0,
FilterQueries = new ISolrQuery[] {
new SolrQueryByField("playerId", query.PlayerId.ToString()),
new SolrQueryByRange<DateTime>("dateTime", query.DateTimeFrom, query.DateTimeTo)
},
Facet = new FacetParameters
{
Queries = new List<ISolrFacetQuery>
{
new SolrFacetFieldQuery("heartRate")
}
}
};
queryOption.ExtraParams = new KeyValuePair<string, string>[] {
new KeyValuePair<string,string>("wt", "xml")
};
//Execute the query
solrResults = solr.Query(SolrQuery.All, queryOption);
Update
I did it using ExtraParams
queryOption.ExtraParams = new KeyValuePair<string, string>[] {
new KeyValuePair<string,string>("wt", "xml"),
new KeyValuePair<string,string>("json.facet", "{heartRateMin: 'min(heartRate)',heartRateMax: 'max(heartRate)',heartRateAvg: 'avg(heartRate)',distance: 'sum(distance)',calories: 'sum(calories)'}")
};
solrResults = await solr.QueryAsync(SolrQuery.All, queryOption);
ReadingsResponseExtraction extractResponse = new ReadingsResponseExtraction();
extractResponse.SetHeader(queryResponce, solrResults);
extractResponse.SetBody(queryResponce, solrResults);
extractResponse.SetFacets(queryResponce, solrResults);
//Return response;
return queryResponce;
ReadingsResponseExtraction.cs
internal class ReadingsResponseExtraction
{
//Extract parts of the SolrNet response and set them in QueryResponse class
internal void SetHeader(DeviceQueryResponse queryResponce, SolrQueryResults<DeviceReadings> solrResults)
{
queryResponce.QueryTime = solrResults.Header.QTime;
queryResponce.Status = solrResults.Header.Status;
queryResponce.TotalHits = solrResults.NumFound;
}
internal void SetBody(DeviceQueryResponse queryResponce, SolrQueryResults<DeviceReadings> solrResults)
{
queryResponce.Result = (List<DeviceReadings>)solrResults;
}
internal void SetFacets(DeviceQueryResponse queryResponse, SolrQueryResults<DeviceReadings> solrResults)
{
queryResponse.HeartRateMin = (int)solrResults.Stats["heartRate"].Min;
queryResponse.HeartRateMax = (int)solrResults.Stats["heartRate"].Max;
queryResponse.HeartRateAvg = (int)solrResults.Stats["heartRate"].Mean;
queryResponse.Distance = solrResults.Stats["distance"].Sum;
queryResponse.Calories = solrResults.Stats["calories"].Sum;
}
}
how to get these values from extraParames
To the best of my knowledge SolrNet does not yet have a .NET API supporting json.facet. However, you can always append extra query parameters via the QueryOptions.ExtraParams property. Based on your example:
queryOption.ExtraParams = new KeyValuePair<string, string>[] {
new KeyValuePair<string,string>("wt", "xml"),
new KeyValuePair<string,string("json.facet", "YOUR_JSON_FACET"),
};
YOUR_JSON_FACET can either just be a JSON string literal, or an object that is serialized into JSON. e.g.
var jsonFacet = new
{
heartRate = new {
type= "terms",
field= "heartRate",
}
};
JsonConvert.SerializeObject(jsonFacet, Formatting.None);
Next, you need to read the facet values out of the response from Solr. There are likely cleaner ways to do this, but one way that doesn't involve altering SolrNet internals is to write your own Query method that also outputs the raw XML. From that raw XML, you can just read the appropriate json.facet nodes.
public static SolrQueryResults<T> QueryWithRawXml<T>(this ISolrOperations<T> operations,
ISolrQuery query, QueryOptions queryOptions, out XDocument xml)
{
var executor = (SolrQueryExecuter<T>)ServiceLocator.Current.GetInstance<ISolrQueryExecuter<T>>();
var connectionKey = string.Format("{0}.{1}.{2}", typeof(SolrConnection), typeof(T), typeof(SolrConnection));
var connection = ServiceLocator.Current.GetInstance<ISolrConnection>(connectionKey);
var parser = ServiceLocator.Current.GetInstance<ISolrAbstractResponseParser<T>>();
var parameters = executor.GetAllParameters(query, queryOptions);
var responseXml = connection.Get(executor.Handler, parameters);
xml = XDocument.Parse(responseXml);
var results = new SolrQueryResults<T>();
parser.Parse(xml, results);
return results;
}
public IEnumerable<KeyValuePair<string, int> GetJsonFacets(
XDocument xml, string facetFieldName, string countFieldName = "count")
{
var response = xml.Element("response");
if (response == null)
{
yield break;
}
var mainFacetNode = response
.Elements("lst")
.FirstOrDefault(e => e.Attribute("name")?.Value == "facets");
if (mainFacetNode == null)
{
yield break;
}
var groupFacetElement = mainFacetNode
.Elements("lst")
.FirstOrDefault(x => x.Attribute("name")?.Value == facetFieldName);
if (groupFacetElement == null)
{
yield break;
}
var buckets = groupFacetElement.Elements("arr")
.FirstOrDefault(x => x.Attribute("name")?.Value == "buckets");
if (buckets == null)
{
yield break;
}
foreach (var bucket in buckets.Elements("lst"))
{
var valNode = bucket.Elements()
.FirstOrDefault(x => x.Attribute("name")?.Value == "val");
var countNode = bucket.Elements()
.FirstOrDefault(x => x.Attribute("name")?.Value == countFieldName);
int count;
if (valNode != null && countNode != null &&
int.TryParse(countNode.Value, out count))
{
yield return new KeyValuePair<string, int>(valNode.Value,count)
}
}
}
I have a search method that queries Solr for event items. I need to modify it to only get events where the date has not already passed (i.e. Where(x => x.EventDate.Date >= DateTime.Now.Date), but I'm not sure how to add this because I'm not very familiar with Solr. Here's my search function:
public SearchQueryResults Search(string keywords, int page,int perPage, List<Guid> contentTypeFilters, List<Guid> otherFilters, ISortBuilder<SearchResultItem> sortBuilder)
{
var searchFilters = new List<IPredicateBuilder<SearchResultItem>>()
{
new IsSearchablePredicateBuilder()
};
if (contentTypeFilters.Any())
{
var contentTypePredicateBuilder = new ContentTypePredicateBuilder();
contentTypePredicateBuilder.ContentTypes = contentTypeFilters;
searchFilters.Add(contentTypePredicateBuilder);
}
if (otherFilters.Any())
{
var tagFilterBuilder = new TagsAndPredicateBuilder(otherFilters,_sitecoreContext);
searchFilters.Add(tagFilterBuilder);
}
if (string.IsNullOrWhiteSpace(keywords))
{
keywords = "";
}
SearchRequest searchRequest = new SearchRequest();
var queryParams = new Dictionary<string, string>() { };
queryParams.Add("q", keywords);
searchRequest.QueryParameters = queryParams;
searchRequest.SortBy = "";
searchRequest.SortOrder = "";
SearchQuery<SearchResultItem> queryArguments = new SearchQuery<SearchResultItem>();
queryArguments.FilterBuilders = searchFilters;
queryArguments.Page = page;
queryArguments.PerPage = perPage;
queryArguments.FacetsBuilder = new SearchFacetBuilder<SearchResultItem>();
queryArguments.SearchRequest = searchRequest;
queryArguments.IndexName = _indexName;
if (string.IsNullOrWhiteSpace(keywords))
{
queryArguments.QueryBuilders =new List<IPredicateBuilder<SearchResultItem>>();
}
else
{
queryArguments.QueryBuilders = new[] { new KeywordPredicateBuilder<SearchResultItem>(new[] { keywords }) };
}
queryArguments.SortBuilder = sortBuilder;
try
{
var results = _searchManager.GetResults<SearchResultItem>(queryArguments);
SearchQueryResults queryResults = new SearchQueryResults();
queryResults.ResultItems = results.Results;
queryResults.CurrentPage = page;
queryResults.TotalResults = Int32.Parse(results.TotalResults.ToString());
queryResults.TotalPages = (queryResults.TotalResults + perPage - 1) / perPage; ;
return queryResults;
}
catch (Exception exc)
{
Sitecore.Diagnostics.Log.Error("Error with FilteredSearch, could be a loss of connection to the SOLR server: " + exc.Message, this);
return null;
}
}
and here is how it's being called:
Results = _searchService.Search(searchTerm, CurrentPage - 1, 10, contentTypes, searchFilters,
new GenericSortBuilder<SearchResultItem>(q => q.OrderByDescending(r => r.SearchDate)));
How do I add in date filtering so that it only returns items where the date is in the future?
I would add filter query to the list of existing ones filtering the date field. On the documentation page, I was able to find information about fluent API, which could help here
Query.Field("date").From(DateTime.Now)
I'm not C# developer, that this code could have some mistakes, but I think the main idea is clear what needs to be done.
I'm searching a sorted dictionary with a key of type datetime and values as list of objects. What I need to find is the latest value(based on a property on the object) for each object in the dictionary. My object has 3 properties : a name, a value and a date when it was created. My dictionary is sorted by latest date in descending order.
I have got this working somehow, but I'm sure there is a shortcut for this using LINQ. Please note that I'm using .NET 3.5. Could you please help? Please dont get put off by the huge amount code below as I have added it for clarity and i'm only looking for a linq query to query inside a list of list objects.
Code below:
public void Should_link_recent_data_together()
{
var data = TimeSeriesDataFactoryEx.GetData();
var allAttributes = new List<string>()
{
TimeSeriesConstants.TOTAL_COST_CODE,
TimeSeriesConstants.TOTAL_VALUE_CODE,
TimeSeriesConstants.SOURCE_CODE
};
var latestList = new List<TimeSeries>();
var allValues = data.Values.ToList();
#region HOW DO I DO THIS USING LINQ?
bool found = false;
foreach (var attribute in allAttributes)
{
found = false;
foreach (var tsData in allValues)
{
foreach (var ts in tsData)
{
if (ts.MetricName == attribute && !string.IsNullOrEmpty(ts.MetricValue))
{
latestList.Add(ts);
found = true;
break;
}
}
if (found)
break;
}
}
#endregion
Assert.IsTrue(latestList.Count == 3);
Assert.IsTrue(latestList.Where(x => x.MetricName == TimeSeriesConstants.TOTAL_COST_CODE).First().MetricValue == "1");
Assert.IsTrue(latestList.Where(x => x.MetricName == TimeSeriesConstants.TOTAL_VALUE_CODE).First().MetricValue == "2");
Assert.IsTrue(latestList.Where(x => x.MetricName == TimeSeriesConstants.SOURCE_CODE).First().MetricValue == "gp");
Assert.IsTrue(latestList.Where(x => x.MetricName == TimeSeriesConstants.SOURCE_CODE).First().Quarter == DateTime.Today.AddMonths(-3));
}
internal class TimeSeriesDataFactoryEx
{
public static SortedDictionary<DateTime?,List<TimeSeries>> GetData()
{
return new SortedDictionary<DateTime?, List<TimeSeries>>(new DateComparer())
{
{
DateTime.Today, new List<TimeSeries>()
{
new TimeSeries()
{
Quarter = DateTime.Today,
MetricValue = "1",
MetricName = TimeSeriesConstants.TOTAL_COST_CODE
},
new TimeSeries()
{
Quarter = DateTime.Today,
MetricValue = "2",
MetricName = TimeSeriesConstants.TOTAL_VALUE_CODE
},
new TimeSeries()
{
Quarter = DateTime.Today,
MetricValue = "",
MetricName = TimeSeriesConstants.SOURCE_CODE
}
}
},
{
DateTime.Today.AddMonths(-3), new List<TimeSeries>()
{
new TimeSeries()
{
Quarter = DateTime.Today.AddMonths(-3),
MetricValue = "3",
MetricName = TimeSeriesConstants.TOTAL_COST_CODE
},
new TimeSeries()
{
Quarter = DateTime.Today.AddMonths(-3),
MetricValue = "4",
MetricName = TimeSeriesConstants.TOTAL_VALUE_CODE
},
new TimeSeries()
{
Quarter = DateTime.Today.AddMonths(-3),
MetricValue = "gp",
MetricName =TimeSeriesConstants.SOURCE_CODE
}
}
}
};
}
}
So, assuming I understand your question right, say you have a dictionary like so:
{ Key = "1/1/1900", Value = List Of Objects, of which each has a DateTimeProperty }
...
{ Key = "1/4/1900", Value = List Of Objects, of which each has a DateTimeProperty }
And you are looking to find a set of objects from your dictionary, where it's the latest by time of each key, then you can do this pretty simply with linq:
var latestItems = data.SelectMany(kvp =>
kvp.Value.OrderByDescending(value => value.Quarter).Take(1)
);
This query finds the most recent object in each bucket and then returns that as a single set (not an enumerable of enumerables). You can change the selector inside the SelectMany to find elements in each set as much as you want, as long as you return an IEnumerable from that callback.