Develop Reference

AWS DynamoDB: query works on the console but not on .NET Code

I have a DynamoDB query that works fine on the AWS Console but it doesn't on code.
Here is my query on the console:
Now here is my c# code to query it:
var query = new QueryOperationConfig
{
KeyExpression = new Expression
{
ExpressionStatement = "#pkey = :v_pkey and #skey >= :v_skey",
ExpressionAttributeNames = {
{ "#pkey", "MailingId" },
{ "#skey", "RegistroCarteiraId" },
},
ExpressionAttributeValues = new Dictionary<string, DynamoDBEntry>()
{
{ ":v_pkey", new Primitive("62", true) },
{ ":v_skey", new Primitive("00e0bbfc-aed0-4f0e-acef-a3623a9f9694") },
},
},
BackwardSearch = false,
ConsistentRead = true,
Limit = 1,
FilterExpression = new Expression
{
ExpressionStatement = "#psituacao = :v_psituacao and attribute_not_exists(#pdisponibilidade)",
ExpressionAttributeNames =
{
{ "#psituacao", "Situacao" },
{ "#pdisponibilidade", "Disponibilidade" }
},
ExpressionAttributeValues =
{
{ ":v_psituacao", new Primitive("1", true) },
}
}
};
var search = table.Query(query);
var docs = await search.GetNextSetAsync();
I get no errors, only an empty array as the result. If I change the sort key to different values, it works, but for this particular value it does not...
I've been at it all day and couldn't figure it out what is wrong.
Any help will be much appreciated.
Thanks

The problem was the LIMIT 1.
As I found out, the filter only happens on the fetched items and, since I was only fetching 1 item, when the filter occurred, the result had no records that matched the criteria.
Removing the Limit 1 solved the problem.

Retrieve only internal _id with NEST ElasticClient

I try to execute a search with NEST ElasticClient and getting only the _id of the hits.
Here is my Code:
var client = new ElasticClient();
var searchResponse = client.Search<ElasticResult>(new SearchRequest {
From = this.query.Page * 100,
Size = 100,
Source = new SourceFilter {
Includes = "_id"
},
Query = new QueryStringQuery {
Query = this.query.Querystring
}
});
public class ElasticResult {
public string _id;
}
But the _id of the Documents (ElasticResult-Objects) is always null. What am I doing wrong?

The _id is not part of the _source document, but part of the hit metadata for each hit in the hits array.
The most compact way to return just the _id fields would be with using response filtering which is exposed as FilterPath in NEST
private static void Main()
{
var defaultIndex = "documents";
var pool = new SingleNodeConnectionPool(new Uri("http://localhost:9200"));
var settings = new ConnectionSettings(pool)
.DefaultIndex(defaultIndex)
.DefaultTypeName("_doc");
var client = new ElasticClient(settings);
if (client.IndexExists(defaultIndex).Exists)
client.DeleteIndex(defaultIndex);
client.Bulk(b => b
.IndexMany<object>(new[] {
new { Message = "hello" },
new { Message = "world" }
})
.Refresh(Refresh.WaitFor)
);
var searchResponse = client.Search<object>(new SearchRequest<object>
{
From = 0 * 100,
Size = 100,
FilterPath = new [] { "hits.hits._id" },
Query = new QueryStringQuery
{
Query = ""
}
});
foreach(var id in searchResponse.Hits.Select(h => h.Id))
{
// do something with the ids
Console.WriteLine(id);
}
}
The JSON response from Elasticsearch to the search request looks like
{
"hits" : {
"hits" : [
{
"_id" : "6gs8lmQB_8sm1yFaJDlq"
},
{
"_id" : "6Qs8lmQB_8sm1yFaJDlq"
}
]
}
}

MongoDB aggregation Shell script to MongoC# Driver

How can I convert this Mongo Shell script to MongoDB C# Driver?
var myItems = []
var myCursor = db.Tickets.aggregate(
[
{ $match : { TicketProjectID : 49 } },
{ $project: { TicketProjectID:1, TicketID:1, concatValue: { $concat: [ "$Status", " - ", "$Name" ] } } }
// I will have a list of fields that I need to concatenate at run time. So C# query should support concatenation for "N" number of fields at run-time.
//{ $group: { _id: null, count: { $sum: 1 } } }
],
{ allowDiskUse: true }
)
//This seems like a ugly performance approach when we are working against 100k results with above match
while (myCursor.hasNext()) {
var item = myCursor.next();
if(item.concatValue.search(/mysearchkey/i) > -1)
{
myItems.push(item.TicketID)
}
}
myItems
or is there a better way to do the string search in concatenated projection instead of foreach in cursor, as some quires might get 50k records.
This is what I have tried so far, (Not using Aggregation)
Note: Trimmed this code to suite for public Q&A sites. So please consider this as Pseudo-code
var tickets = ticketsCollection.FindSync(filter).ToList();
string concatinatedValue = string.Empty;
foreach (var ticket in tickets)
{
foreach (var field in customFieldsForThisProject)
concatinatedValue += ticket[field.Replace(" ", "_")];
if(concatinatedValue.StripHtml().contains("MysearchWord"))
{
TikectIdList.Add(ticket["TicketID"])
}
}

Thanks to #Nikola.Lukovic, working on his pseudo-code, I came up with this working solution.
Approach one: fully using C# Driver
var ticketsCollection = _mongoConnect.Database.GetCollection<BsonDocument>("Tickets");
var dbResult = from ticket in ticketsCollection.AsQueryable()
select new
{
TicketProjectID = ticket["TicketProjectID"],
TicketID = ticket["TicketID"],
ConcatValue = ticket["Status"] + (string) ticket["Name"]
};
var matches = from dbr in dbResult
where dbr.ConcatValue.Contains(searchKey)
where dbr.ConcatValue.StartsWith(searchKey)
select dbr;
This will not work for my scenario as fields I am trying to
concatenate are if type string, but $add will only work with
numeric and date types.
Approach two: using RunCommand and passing straight Shell command. This will work for all datatypes. And works for my need as well.
var projectCommand =
BsonDocument.Parse(
"{ $project: { _id: -1, TicketProjectID:1, TicketID:1, concatValue: { $concat: [ \"$Status\", \" - \", \"$Name\" ] } } }");
var matchCommand =
BsonDocument.Parse("{ $match: {concatValue: { $regex: '" + searchKey + "', $options: 'i'} } }");
var pipeline = new[] {projectCommand, matchCommand};
var result = ticketsCollection.Aggregate<BsonDocument>(pipeline).ToList();
if (result.Count > 0)
return result.Select(x => (int)x["TicketID"]).ToList();
return null;

Edited according to the given comment
If you can use AsQueryable() you can get the values like this:
var dbResult = from ticket in ticketsCollection.AsQueryable()
where ticket.TicketProjectID == 49
select new
{
TicketProjectID = ticket.TicketProjectID,
TicketID = ticket.TicketID,
ConcatValue = ticket.Status + " - " + ticket.Name
};
and than later you can do something like this:
var result = from dbr in dbResult
where dbr.ConcatValue.Contains("something") //or
where dbr.ConcatValue.StartsWith("something")//or you can use regex
select dbr;
Note: For some reason both Status and Name properties from type Ticket need to be of a type String for concatenation to work since mongo driver won't recognize the call to ToString() from some other type.
If you want to concatenate properties from some other types you could get them separately from the db and than concat them locally.
note, i'm not that good with mongo shell i could mess something up but you can see in which way you could go
Alternatively you could write your shell command like this and put it in a string:
var command = #"db.Tickets.aggregate(
[
{ $project: { TicketProjectID:1, TicketID:1, concatValue: { $concat: [ "$Status", " - ", "$Name" ] } } },
{ $match : { TicketProjectId : 49, concatValue : { $regex : /mysearchkey/i } } }
],
{ allowDiskUse : true }
);";
then execute it in c# with RunCommandAsync method from MongoDatabase.
var result = await mongoDatabase.RunCommandAsync<BsonDocument>(BsonDocument.Parse(command));

ElasticSearch C# client (NEST): access nested aggregation results

I have the following query in NEST (ElasticSearch C# client), note the nested aggregation:
var query = _elasticClient.Search<Auth5209>(s => s
.Size(0)
.Aggregations(a=> a
.Terms("incidentID", t=> t
.Field(f=>f.IncidentID)
.Size(5)
.Aggregations(a2 => a2
.Stats("authDateStats", s1=>s1.Field(f=>f.AuthEventDate))
)
)
)
);
This correctly generates the following query:
{
"size": 0,
"aggs": {
"incidentID": {
"terms": {
"field": "incidentID",
"size": 5
},
"aggs": {
"authDateStats": {
"stats": {
"field": "authEventDate"
}
}
}
}
}
}
Which gives me the following results:
"aggregations" : {
"incidentID" : {
"buckets" : [{
"key" : "0A631EB1-01EF-DC28-9503-FC28FE695C6D",
"doc_count" : 233,
"authDateStats" : {
"count" : 233,
"min" : 1401167036075,
"max" : 1401168969907,
"avg" : 1401167885682.6782,
"sum" : 326472117364064
}
}
]
}
}
What I can't figure out is how I access the "authDateStats" section. When I debug I don't see any way to access the data.

Neither the official documentation nor the answers here fully work for nest 2.0+. Although the answer from jhilden did get started me down the right path.
Here is a working example of a similar query which can be used with nest 2.0+:
const string termsAggregation = "device_number";
const string topHitsAggregation = "top_hits";
var response = await _elasticsearchClient.Client.SearchAsync<CustomerDeviceModel>(s => s
.Aggregations(a => a
.Terms(termsAggregation, ta => ta
.Field(o => o.DeviceNumber)
.Size(int.MaxValue)
.Aggregations(sa => sa
.TopHits(topHitsAggregation, th => th
.Size(1)
.Sort(x => x.Field(f => f.Modified).Descending())
)
)
)
)
);
if (!response.IsValid)
{
throw new ElasticsearchException(response.DebugInformation);
}
var results = new List<CustomerDeviceModel>();
var terms = response.Aggs.Terms(termsAggregation);
foreach (var bucket in terms.Buckets)
{
var hit = bucket.TopHits(topHitsAggregation);
var device = hit.Documents<CustomerDeviceModel>().First();
results.Add(device);
}

I'm guessing you already figured this out but you can access the nested aggregations, it's just in a base class, you can see it in Nest.KeyItem.base.base.Aggregations in the debugger.

Here is a full working sample of accessing an inner aggregation:
const string aggName = "LocationIDAgg";
const string aggNameTopHits = "LatestForLoc";
var response = await ElasticClient.SearchAsync<PlacementVerificationES>(s => s
.Query(BuildQuery(filter, null))
.Size(int.MaxValue)
.Aggregations(a=> a
.Terms(aggName, t=> t
.Field(f=>f.LocationID)
.Size(100)
.Aggregations(innerAgg => innerAgg
.TopHits(aggNameTopHits, th=> th
.Size(1)
.Sort(x=>x.OnField(f=> f.Date).Descending())
)
)
)
)
).VerifySuccessfulResponse();
//var debug = response.GetRequestString();
var agBucket = (Bucket)response.Aggregations[aggName];
var output = new List<PlacementVerificationForReporting>();
// ReSharper disable once LoopCanBeConvertedToQuery
// ReSharper disable once PossibleInvalidCastExceptionInForeachLoop
foreach (KeyItem i in agBucket.Items)
{
var topHits = (TopHitsMetric)i.Aggregations[aggNameTopHits];
var top1 = topHits.Hits<PlacementVerificationES>().Single();
var reportingObject = RepoToReporting(top1);
output.Add(reportingObject);
}
return output;

Linq query for searching an object inside a dictionary having list of lists as values

I'm searching a sorted dictionary with a key of type datetime and values as list of objects. What I need to find is the latest value(based on a property on the object) for each object in the dictionary. My object has 3 properties : a name, a value and a date when it was created. My dictionary is sorted by latest date in descending order.
I have got this working somehow, but I'm sure there is a shortcut for this using LINQ. Please note that I'm using .NET 3.5. Could you please help? Please dont get put off by the huge amount code below as I have added it for clarity and i'm only looking for a linq query to query inside a list of list objects.
Code below:
public void Should_link_recent_data_together()
{
var data = TimeSeriesDataFactoryEx.GetData();
var allAttributes = new List<string>()
{
TimeSeriesConstants.TOTAL_COST_CODE,
TimeSeriesConstants.TOTAL_VALUE_CODE,
TimeSeriesConstants.SOURCE_CODE
};
var latestList = new List<TimeSeries>();
var allValues = data.Values.ToList();
#region HOW DO I DO THIS USING LINQ?
bool found = false;
foreach (var attribute in allAttributes)
{
found = false;
foreach (var tsData in allValues)
{
foreach (var ts in tsData)
{
if (ts.MetricName == attribute && !string.IsNullOrEmpty(ts.MetricValue))
{
latestList.Add(ts);
found = true;
break;
}
}
if (found)
break;
}
}
#endregion
Assert.IsTrue(latestList.Count == 3);
Assert.IsTrue(latestList.Where(x => x.MetricName == TimeSeriesConstants.TOTAL_COST_CODE).First().MetricValue == "1");
Assert.IsTrue(latestList.Where(x => x.MetricName == TimeSeriesConstants.TOTAL_VALUE_CODE).First().MetricValue == "2");
Assert.IsTrue(latestList.Where(x => x.MetricName == TimeSeriesConstants.SOURCE_CODE).First().MetricValue == "gp");
Assert.IsTrue(latestList.Where(x => x.MetricName == TimeSeriesConstants.SOURCE_CODE).First().Quarter == DateTime.Today.AddMonths(-3));
}
internal class TimeSeriesDataFactoryEx
{
public static SortedDictionary<DateTime?,List<TimeSeries>> GetData()
{
return new SortedDictionary<DateTime?, List<TimeSeries>>(new DateComparer())
{
{
DateTime.Today, new List<TimeSeries>()
{
new TimeSeries()
{
Quarter = DateTime.Today,
MetricValue = "1",
MetricName = TimeSeriesConstants.TOTAL_COST_CODE
},
new TimeSeries()
{
Quarter = DateTime.Today,
MetricValue = "2",
MetricName = TimeSeriesConstants.TOTAL_VALUE_CODE
},
new TimeSeries()
{
Quarter = DateTime.Today,
MetricValue = "",
MetricName = TimeSeriesConstants.SOURCE_CODE
}
}
},
{
DateTime.Today.AddMonths(-3), new List<TimeSeries>()
{
new TimeSeries()
{
Quarter = DateTime.Today.AddMonths(-3),
MetricValue = "3",
MetricName = TimeSeriesConstants.TOTAL_COST_CODE
},
new TimeSeries()
{
Quarter = DateTime.Today.AddMonths(-3),
MetricValue = "4",
MetricName = TimeSeriesConstants.TOTAL_VALUE_CODE
},
new TimeSeries()
{
Quarter = DateTime.Today.AddMonths(-3),
MetricValue = "gp",
MetricName =TimeSeriesConstants.SOURCE_CODE
}
}
}
};
}
}

So, assuming I understand your question right, say you have a dictionary like so:
{ Key = "1/1/1900", Value = List Of Objects, of which each has a DateTimeProperty }
...
{ Key = "1/4/1900", Value = List Of Objects, of which each has a DateTimeProperty }
And you are looking to find a set of objects from your dictionary, where it's the latest by time of each key, then you can do this pretty simply with linq:
var latestItems = data.SelectMany(kvp =>
kvp.Value.OrderByDescending(value => value.Quarter).Take(1)
);
This query finds the most recent object in each bucket and then returns that as a single set (not an enumerable of enumerables). You can change the selector inside the SelectMany to find elements in each set as much as you want, as long as you return an IEnumerable from that callback.