Develop Reference

Read JSON output from AppInsights in C#

I want to read AppInsights API output in C# console application.
WebClient wc = new WebClient();
wc.BaseAddress = "https://api.applicationinsights.io/v1/apps/AppInsighID/query?query=requests|where timestamp>= ago(1h)|limit 100";
wc.Headers.Add("Host", "api.applicationinsights.io");
wc.Headers.Add("x-api-key", "key");
string json = wc.DownloadString("");
JObject jsonObject = JObject.Parse(json);
//With this, i got values for Rows
var rowsObject = jsonObject["tables"][0]["rows"];
Now values are in array, under rowObject, so how to read this?
I would also like to know the best practice we should follow when reading json string
I can see data like this
{
"tables": [
{
"name": "PrimaryResult",
"columns": [
{
"name": "timestamp",
"type": "datetime"
},
{
"name": "id",
"type": "string"
},
{
"name": "source",
"type": "string"
},
{
"name": "name",
"type": "string"
},
{
"name": "url",
"type": "string"
},
{
"name": "success",
"type": "string"
},
{
"name": "resultCode",
"type": "string"
},
{
"name": "duration",
"type": "real"
},
{
"name": "performanceBucket",
"type": "string"
},
{
"name": "itemType",
"type": "string"
},
{
"name": "customDimensions",
"type": "dynamic"
},
{
"name": "customMeasurements",
"type": "dynamic"
},
{
"name": "operation_Name",
"type": "string"
},
{
"name": "operation_Id",
"type": "string"
},
{
"name": "operation_ParentId",
"type": "string"
},
{
"name": "operation_SyntheticSource",
"type": "string"
},
{
"name": "session_Id",
"type": "string"
},
{
"name": "user_Id",
"type": "string"
},
{
"name": "user_AuthenticatedId",
"type": "string"
},
{
"name": "user_AccountId",
"type": "string"
},
{
"name": "application_Version",
"type": "string"
},
{
"name": "client_Type",
"type": "string"
},
{
"name": "client_Model",
"type": "string"
},
{
"name": "client_OS",
"type": "string"
},
{
"name": "client_IP",
"type": "string"
},
{
"name": "client_City",
"type": "string"
},
{
"name": "client_StateOrProvince",
"type": "string"
},
{
"name": "client_CountryOrRegion",
"type": "string"
},
{
"name": "client_Browser",
"type": "string"
},
{
"name": "cloud_RoleName",
"type": "string"
},
{
"name": "cloud_RoleInstance",
"type": "string"
},
{
"name": "appId",
"type": "string"
},
{
"name": "appName",
"type": "string"
},
{
"name": "iKey",
"type": "string"
},
{
"name": "sdkVersion",
"type": "string"
},
{
"name": "itemId",
"type": "string"
},
{
"name": "itemCount",
"type": "int"
}
],
"rows": [
[
"2020-01-16T07:07:35.8423912Z",
"ID",
"",
"POST ",
"https://",
"True",
"200",
57.679,
"<250ms",
"request",
"{\"Product Name\":\"Name\",\"Subscription Name\":\"Name\",\"Operation Name\":\"AdvancedSearch\",\"ApimanagementRegion\":\"Region\",\"ApimanagementServiceName\":\"Name\",\"Apim Request Id\":\"ID\",\"Request-Body\":\"{\\\"P1\\\":25,\\\"P2\\\":1,\\\"P3\\\":\\\"All \\\",\\\"P4\\\":\\\"Earliest\\\",\\\"P5\\\":\\\"Extended\\\",\\\"P6\\\":\\\"All \\\",\\\"P6\\\":\\\"Latest\\\",\\\"queryList\\\":[{\\\"P7\\\":\\\"physics\\\",\\\"P8\\\":\\\"A1\\\",\\\"operator\\\":\\\"\\\"}]}\",\"Cache\":\"None\",\"P9\":\"195.43.22.145\",\"API Name\":\"Name\",\"HTTP Method\":\"POST\"}",
"{\"Response Size\":776,\"Request Size\":1092,\"Client Time (in ms)\":0}",
"POST ",
"ID",
"ID",
"",
"",
"",
"1",
"",
"",
"PC",
"",
"",
"0.0.0.0",
"Milan",
"Milan",
"Italy",
"",
"Value1",
"Value2",
"ID1",
"AppInsight Name",
"Name",
"apim:0.12.885.0",
"ID",
1
]
]
}
]
}

You could deserialize the Json and fetch the Rows information. For example,
var result = JsonConvert.DeserializeObject<RootObject>(str);
var rowData = result.tables.SelectMany(x=>x.rows.SelectMany(c=>c));
If you do not want to Flatten the results, you could use
var rowData = result.tables.SelectMany(x=>x.rows.Select(c=>c));
Where RootObject is defined as
public class Column
{
public string name { get; set; }
public string type { get; set; }
}
public class Table
{
public string name { get; set; }
public List<Column> columns { get; set; }
public List<List<string>> rows { get; set; }
}
public class RootObject
{
public List<Table> tables { get; set; }
}
If you intend to eliminate empty values, you could filter them out using Linq
var rowData = result.tables.SelectMany(x=>x.rows.SelectMany(c=>c))
.Where(x=>!string.IsNullOrEmpty(x));
Or for non-Flatten results
var rowData = result.tables.SelectMany(x=>x.rows.Select(c=>c))
.Where(x=>!string.IsNullOrEmpty(x.ToString()));
Update
Based on comment, to retrieve the information and parse it to a Dto based on the position of values in array, you could do the following. The attribute could be used for handling inline Json Properties as well, as mentioned in the comment.
You could begin by defining an attribute as following.
public class DtoDefinitionAttribute:Attribute
{
public DtoDefinitionAttribute(int order)=>Order = order;
public DtoDefinitionAttribute(int order,bool isJson,Type jsonDataType)
{
Order = order;
JsonDataType = jsonDataType;
IsJson = isJson;
}
public bool IsJson{get;} = false;
public int Order{get;}
public Type JsonDataType {get;}
}
And, then you could decorate your Dto properties with the index of corresponding value in the array. Additionally, in case of Json string expected as Json, then you could use the attribute to indicate it as well, as shown in the ClientTime property For example,
public class Dto
{
[DtoDefinition(0)]
public DateTime CurrentDate{get;set;}
[DtoDefinition(1)]
public string ID{get;set;}
[DtoDefinition(2)]
public string Url{get;set;}
[DtoDefinition(11,true,typeof(Response))]
public Response Json1{get;set;}
}
public class Response
{
[JsonProperty("Response Size")]
public string ResponseSize{get;set;}
[JsonProperty("Request Size")]
public string RequestSize{get;set;}
[JsonProperty("Client Time (in ms)")]
public int ClientTime{get;set;}
}
Now you could use the rowData result obtained using
var listDto = new List<Dto>();
foreach(var row in rowData)
{
listDto.Add(AssignValues(row));
}
Where AssignValues are defined as
public Dto AssignValues(List<string> row)
{
var dto = new Dto();
var properties = typeof(Dto).GetProperties().Where(x=>x.GetCustomAttributes<DtoDefinitionAttribute>().Any());
foreach(var property in properties)
{
var attribute = property.GetCustomAttribute<DtoDefinitionAttribute>();
if(attribute.IsJson)
{
var jsonData = row[attribute.Order].ToString();
var deserializedData = JsonConvert.DeserializeObject(jsonData,attribute.JsonDataType);
property.SetValue(dto,deserializedData);
}
else
{
property.SetValue(dto,Convert.ChangeType(row[attribute.Order],property.PropertyType));
}
}
return dto;
}
The AssignValues method uses reflection to read the Attributes and create an instance of Dto based on it. In case, it finds the attribute defines it as Json, then it would deserialize the json value and use the result.
Demo Code

Change property type as exported by Swagger/Swashbuckle

I have a fairly complex object with nested objects; please note that in the example below I have simplified this object greatly.
Assume the following example object:
public class Result {
public string Name { get; set; }
public IpAddress IpAddress { get; set; }
}
I have implemented a JsonConverter<IPAddress> than (de)serializes the Ip as a string:
public class IPAddressConverter : JsonConverter<IPAddress>
{
public override IPAddress Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
=> IPAddress.Parse(reader.GetString());
public override void Write(Utf8JsonWriter writer, IPAddress value, JsonSerializerOptions options)
=> writer.WriteStringValue(value.ToString());
}
The IPAddressConverter was then 'registered' as a converter in the AddJsonOptions(...) method. This nicely returns results as:
{ "Name": "Foo", "IpAddress": "198.51.100.1" }
And, vice versa, my controller "understands" IP addresses specified as string:
public IEnumerable<Result> FindByIp(IpAddress ip) {
// ...
}
However, SwashBuckle exports this as:
{
"openapi": "3.0.1",
"info": {
"title": "Example",
"version": "v1"
},
"paths": {
"/FindByIp": {
"get": {
"parameters": [
{
"name": "q",
"in": "query",
"schema": {
"type": "array",
"items": {
"type": "string"
}
}
}
],
"responses": {
"200": {
"description": "Success",
"content": {
"application/json": {
"schema": {
"type": "object",
"additionalProperties": {
"$ref": "#/components/schemas/Result"
}
}
}
}
}
}
}
}
},
"components": {
"schemas": {
"AddressFamily": {
"enum": [
0,
1,
2,
3,
4,
5,
6,
6,
7,
7,
8,
9,
10,
11,
12,
13,
14,
15,
16,
17,
18,
19,
21,
22,
23,
24,
25,
26,
28,
29,
65536,
65537,
-1
],
"type": "integer",
"format": "int32"
},
"IPAddress": {
"type": "object",
"properties": {
"addressFamily": {
"$ref": "#/components/schemas/AddressFamily"
},
"scopeId": {
"type": "integer",
"format": "int64"
},
"isIPv6Multicast": {
"type": "boolean",
"readOnly": true
},
"isIPv6LinkLocal": {
"type": "boolean",
"readOnly": true
},
"isIPv6SiteLocal": {
"type": "boolean",
"readOnly": true
},
"isIPv6Teredo": {
"type": "boolean",
"readOnly": true
},
"isIPv4MappedToIPv6": {
"type": "boolean",
"readOnly": true
},
"address": {
"type": "integer",
"format": "int64"
}
},
"additionalProperties": false
},
"Result": {
"type": "object",
"properties": {
"ip": {
"$ref": "#/components/schemas/IPAddress"
},
"name": {
"type": "string",
"nullable": true
}
},
"additionalProperties": false
}
}
}
}
Which, for the more visually inclined, looks like:
What I'd like to achieve, however, is this:
{
"openapi": "3.0.1",
"info": {
"title": "Example",
"version": "v1"
},
"paths": {
"/FindByIp": {
"get": {
"parameters": [
{
"name": "q",
"in": "query",
"schema": {
"type": "array",
"items": {
"type": "string"
}
}
}
],
"responses": {
"200": {
"description": "Success",
"content": {
"application/json": {
"schema": {
"type": "object",
"additionalProperties": {
"$ref": "#/components/schemas/Result"
}
}
}
}
}
}
}
}
},
"components": {
"schemas": {
"Result": {
"type": "object",
"properties": {
"ip": {
"type": "string",
"nullable": true
},
"name": {
"type": "string",
"nullable": true
}
},
"additionalProperties": false
}
}
}
}
Again, visualized:
I was hoping to be able to add an annotation / attribute on some properties (so I looked at Swashbuckle.AspNetCore.Annotations) but that doesn't seem to be possible.
Also, because the object is fairly complex and comes from a 3rd party library it's hard for me to actually add annotations / attributes on properties because I can't change the model (easily).
I could resort to AutoMapper (or alike) to create another model with a string for IP adresses but that would mean having to model all objects in the original model. Besides, it requires extra code and maintenance when the model changes. I'd rather tell Swashbuckle, somehow, that IP adresses (and, so, the type IPAddress will be represented as a string (in- and outgoing to my API). I'm looking for options on how to accomplish this the best way possible within given limitations (preferably not introducing new models to map to, preferably no annotations/attributes because I can't easily access the 3rd party library). Is there a way to register a "type-converter-something" for Swashbuckle to handle this?
Update: Solved!
This is what I ended up with:
// This method gets called by the runtime. Use this method to add services to the container.
public void ConfigureServices(IServiceCollection services)
{
services
.AddResponseCompression()
.AddMemoryCache()
.AddControllers()
// etc...
// etc...
// Here's the interesting part:
services.AddSwaggerGen(c =>
{
c.SwaggerDoc("v1", new OpenApiInfo { Title = "Example", Version = "v1" });
c.MapType<IPAddress>(() => new OpenApiSchema { Type = typeof(string).Name });
// ...
});
}
Thank you strickt01

As you are converting to a non-complex type you should be able to use MapType for this IPAddress example:
swagger.MapType<IPAddress>(() => new Schema { Type = "string" });
If you are converting to a complex type then you'd need to use SchemaFilter.

Ignore base class properties during auto mapping in ElasticSearch using NEST

I'm working on Elastic Search 5.4 and using NEST to create index in C#, When I do a automap for a class which in Inheriting from SortedDictionary, The base class properties like Count,Comparer,Items are added to the Mapping, Can some one please help me how to avoid this.
Thanks
public class Document : SortedDictionary<string, dynamic>
{
public int Id { get; set; }
}
The actual mapping I'm getting is
{"DocumentMapping": {
"mappings": {
"Document": {
"dynamic": "strict",
"_all": {
"enabled": false
},
"properties": null,
"Documents": {
"properties": {
"Comparer": {
"type": "object"
},
"Count": {
"type": "integer"
},
"Item": {
"type": "object",
"include_in_all": false
},
"ID": {
"type": "int "
}
}
}
}
}}}
What I need is
{"DocumentMapping": {
"mappings": {
"Document": {
"dynamic": "strict",
"_all": {
"enabled": false
},
"properties": null,
"Documents": {
"properties":
"ID": {
"type": "int "
}
}
}
}
}}}
How do I ignore the the Comparer,Count and Item from my mapping JSON.

NEST ElasticSearch.Raw.IndiciesCreatePost does not get correct mappings for index

I'm trying to use NEST to create an index with raw json and it produces different results than when I use that same json string interactively against elastic search. Is this a bug or am I using it incorrectly?
Posting directly to elastic search with the following command I get exactly the mappings I want for my index (results shown below)
POST /entities
{
"mappings": {
"sis_item" :
{
"properties":
{
"FullPath":
{
"type": "string",
"index":"not_analyzed"
},
"Ident":
{
"type": "nested",
"properties":
{
"ObjectGuid":
{
"type": "string",
"index":"not_analyzed"
}
}
}
}
}
}
Result when I check the index with: GET /entities/ : (which is correct)
{
"entities": {
"aliases": {},
"mappings": {
"sis_item": {
"properties": {
"FullPath": {
"type": "string",
"index": "not_analyzed"
},
"Ident": {
"type": "nested",
"properties": {
"ObjectGuid": {
"type": "string",
"index": "not_analyzed"
}
}
}
}
}
},
"settings": {
"index": {
"creation_date": "1453828294488",
"number_of_shards": "5",
"number_of_replicas": "1",
"version": {
"created": "1070499"
},
"uuid": "6_j4vRcqRwiTQw0E6bQokQ"
}
},
"warmers": {}
}
}
However I have to do this from code and using the following code the mappings I specify end up in the settings instead of the mappings as shown below.
var createIndexJson = #"
{
""mappings"": {
""sis_item"" :
{
""properties"":
{
""FullPath"":
{
""type"": ""string"",
""index"":""not_analyzed""
},
""Ident"":
{
""type"": ""nested"",
""properties"":
{
""ObjectGuid"":
{
""type"": ""string"",
""index"":""not_analyzed""
}
}
}
}
}
}
}";
var response = _client.Raw.IndicesCreatePost("entities_from_code", createIndexJson);
if (!response.Success || response.HttpStatusCode != 200)
{
throw new ElasticsearchServerException(response.ServerError);
}
Result (not correct, notice the mappings are nested inside the settings):
{
"entities_from_code": {
"aliases": {},
"mappings": {},
"settings": {
"index": {
"mappings": {
"sis_item": {
"properties": {
"FullPath": {
"type": "string",
"index": "not_analyzed"
},
"Ident": {
"type": "nested",
"properties": {
"ObjectGuid": {
"type": "string",
"index": "not_analyzed"
}
}
}
}
}
},
"creation_date": "1453828283162",
"number_of_shards": "5",
"number_of_replicas": "1",
"version": {
"created": "1070499"
},
"uuid": "fdmPqahGToCJw_mIbq0kNw"
}
},
"warmers": {}
}
}

There is a newline at the very top of the json string which cause the odd result, removing it gave me the expected behaviour.

Extracting elements wherever JArray in json

Lets see, the json can be dynamic and can probably have number of nested arrays within any property.
Example:
{
"items": [
{
"id": "0001",
"name": "Cake",
"batters": {
"batter": [
{
"id": "1001",
"type": "Regular"
},
{
"id": "1002",
"type": "Chocolate"
},
{
"dry": [
{
"id": "1003",
"type": "Devil's Food"
}
]
}
],
"other": [
{
"id": "1004",
"type": "Home Food"
}
]
},
"topping": [
{
"id": "5002",
"type": "Glazed"
},
{
"id": "5005",
"type": "Sugar"
}
]
},
{
"id": "0002",
"name": "Sweets"
}
]
}
A simple list should return elements as:
[
{
"id": "1001",
"type": "Regular"
},
{
"id": "1002",
"type": "Chocolate"
},
{
"id": "1003",
"type": "Devil's Food"
},
{
"id": "1004",
"type": "Home Food"
},
{
"id": "5002",
"type": "Glazed"
},
{
"id": "5005",
"type": "Sugar"
},
{
"id": "0002",
"name": "Sweets"
}
]
Please note:
Json can by anything, no property can be used for extraction , just knowing that what needed is stuff inside an JArray.
What i have tried so far but its just a start:
public static bool ParseJsonArray(JToken token, List<string> extracts, string parentLocation = "")
{
if (token.HasValues)
{
foreach (JToken child in token.Children())
{
if (token.Type == JTokenType.Array)
{
parentLocation += ((JProperty)token).Name;
extracts.Add(token.ToString());
}
ParseJsonArray(child, extracts, parentLocation);
}
return true;
}
else
{
return false;
}
}
token here is the parsed dynamic json.

It appears as though you want to recursively find all JArray entries that do not themselves contain nested arrays. Let's call these "leaf" array entries. I say that because you don't include the following non-leaf entry in your results:
{
"id": "0001",
"name": "Cake"
}
That being said, you can find leaf array entries with the following extension method:
public static class JsonExtensions
{
public static IEnumerable<JToken> LeafArrayEntries(this JContainer container)
{
var nonLeafEntries = new HashSet<JToken>(container.DescendantsAndSelf()
.OfType<JArray>()
.SelectMany(a => a.Ancestors().Where(p => p.Type != JTokenType.Property)));
return container.DescendantsAndSelf().Where(c => c.Parent is JArray && !nonLeafEntries.Contains(c));
}
}
Then put the returned items in an array of their own with:
var leafItemArray = new JArray(rootJContainer.LeafArrayEntries());