Related
I need help, I need a function that recive a Json and returns a Datatable, any input can come in Json format so I can't use a object.
My problem is when the Json comes with multiple Arrays nested in the response (the Json can have any format, it can come without Arrays or it can come with 1 or more Arrays), any idea how to make this possible? I need the algorithm to be as generic as possible. Thanks
I leave an example input and what I expect to get
public const string ResponseThreeArrays = #"{
""key1"": ""val1"",
""key2"": {
""key2-1"":
[
{
""key2-arr1-1"": ""val2-arr1-1(1)"",
""key2-arr1-2"":
[
{
""key2-arr1-arr2-1"" : ""val2-arr1-arr2-1(1)(1)"",
""key2-arr1-arr2-2"" :
[
{
""key2-arr1-arr2-arr3-1"" : ""val2-arr1-arr2-arr3-1(1)(1)(1)"",
""key2-arr1-arr2-arr3-2"" : ""val2-arr1-arr2-arr3-2(1)(1)(1)"",
""key2-arr1-arr2-arr3-3"" : ""val2-arr1-arr2-arr3-3(1)(1)(1)""
},
{
""key2-arr1-arr2-arr3-1"" : ""val2-arr1-arr2-arr3-1(1)(1)(2)"",
""key2-arr1-arr2-arr3-2"" : ""val2-arr1-arr2-arr3-2(1)(1)(2)"",
""key2-arr1-arr2-arr3-3"" : ""val2-arr1-arr2-arr3-3(1)(1)(2)""
},
{
""key2-arr1-arr2-arr3-1"" : ""val2-arr1-arr2-arr3-1(1)(1)(3)"",
""key2-arr1-arr2-arr3-2"" : ""val2-arr1-arr2-arr3-2(1)(1)(3)"",
""key2-arr1-arr2-arr3-3"" : ""val2-arr1-arr2-arr3-3(1)(1)(3)""
}
],
""key2-arr1-arr2-3"" : ""val2-arr1-arr2-3(1)(1)""
},
{
""key2-arr1-arr2-1"" : ""val2-arr1-arr2-1(1)(2)"",
""key2-arr1-arr2-2"" :
[
{
""key2-arr1-arr2-arr3-1"" : ""val2-arr1-arr2-arr3-1(1)(2)(1)"",
""key2-arr1-arr2-arr3-2"" : ""val2-arr1-arr2-arr3-2(1)(2)(1)"",
""key2-arr1-arr2-arr3-3"" : ""val2-arr1-arr2-arr3-3(1)(2)(1)""
},
{
""key2-arr1-arr2-arr3-1"" : ""val2-arr1-arr2-arr3-1(1)(2)(2)"",
""key2-arr1-arr2-arr3-2"" : ""val2-arr1-arr2-arr3-2(1)(2)(2)"",
""key2-arr1-arr2-arr3-3"" : ""val2-arr1-arr2-arr3-3(1)(2)(2)""
},
{
""key2-arr1-arr2-arr3-1"" : ""val2-arr1-arr2-arr3-1(1)(2)(3)"",
""key2-arr1-arr2-arr3-2"" : ""val2-arr1-arr2-arr3-2(1)(2)(3)"",
""key2-arr1-arr2-arr3-3"" : ""val2-arr1-arr2-arr3-3(1)(2)(3)""
}
],
""key2-arr1-arr2-3"" : ""val2-arr1-arr2-3(1)(2)""
}
]
},
{
""key2-arr1-1"": ""val2-arr1-1(2)"",
""key2-arr1-2"":
[
{
""key2-arr1-arr2-1"" : ""val2-arr1-arr2-1(2)(1)"",
""key2-arr1-arr2-2"" :
[
{
""key2-arr1-arr2-arr3-1"" : ""val2-arr1-arr2-arr3-1(2)(1)(1)"",
""key2-arr1-arr2-arr3-2"" : ""val2-arr1-arr2-arr3-2(2)(1)(1)"",
""key2-arr1-arr2-arr3-3"" : ""val2-arr1-arr2-arr3-3(2)(1)(1)""
},
{
""key2-arr1-arr2-arr3-1"" : ""val2-arr1-arr2-arr3-1(2)(1)(2)"",
""key2-arr1-arr2-arr3-2"" : ""val2-arr1-arr2-arr3-2(2)(1)(2)"",
""key2-arr1-arr2-arr3-3"" : ""val2-arr1-arr2-arr3-3(2)(1)(2)""
},
{
""key2-arr1-arr2-arr3-1"" : ""val2-arr1-arr2-arr3-1(2)(1)(3)"",
""key2-arr1-arr2-arr3-2"" : ""val2-arr1-arr2-arr3-2(2)(1)(3)"",
""key2-arr1-arr2-arr3-3"" : ""val2-arr1-arr2-arr3-3(2)(1)(3)""
}
],
""key2-arr1-arr2-3"" : ""val2-arr1-arr2-3(2)(1)""
},
{
""key2-arr1-arr2-1"" : ""val2-arr1-arr2-1(2)(2)"",
""key2-arr1-arr2-2"" :
[
{
""key2-arr1-arr2-arr3-1"" : ""val2-arr1-arr2-arr3-1(2)(2)(1)"",
""key2-arr1-arr2-arr3-2"" : ""val2-arr1-arr2-arr3-2(2)(2)(1)"",
""key2-arr1-arr2-arr3-3"" : ""val2-arr1-arr2-arr3-3(2)(2)(1)""
},
{
""key2-arr1-arr2-arr3-1"" : ""val2-arr1-arr2-arr3-1(2)(2)(2)"",
""key2-arr1-arr2-arr3-2"" : ""val2-arr1-arr2-arr3-2(2)(2)(2)"",
""key2-arr1-arr2-arr3-3"" : ""val2-arr1-arr2-arr3-3(2)(2)(2)""
},
{
""key2-arr1-arr2-arr3-1"" : ""val2-arr1-arr2-arr3-1(2)(2)(3)"",
""key2-arr1-arr2-arr3-2"" : ""val2-arr1-arr2-arr3-2(2)(2)(3)"",
""key2-arr1-arr2-arr3-3"" : ""val2-arr1-arr2-arr3-3(2)(2)(3)""
}
],
""key2-arr1-arr2-3"" : ""val2-arr1-arr2-3(2)(2)""
}
]
}
],
""key2-2"" : ""val2-2""
},
""key3"": ""val3""
}";
And this is what I expect obtain:
Datatable expected
Use Cinchoo ETL - an open source library to do such conversion easily. (Not sure, this meets your requirement)
using (var r = ChoJSONReader.LoadText(ResponseThreeArrays)
.ErrorMode(ChoErrorMode.IgnoreAndContinue)
)
{
var dt = r.FlattenBy("key2", "key2-1", "key2-arr1-2", "key2-arr1-arr2-2").AsDataTable();
dt.Print();
}
Output:
key1,key3,key2-2,key2-arr1-1,key2-arr1-arr2-1,key2-arr1-arr2-3,key2-arr1-arr2-arr3-1,key2-arr1-arr2-arr3-2,key2-arr1-arr2-arr3-3
val1,val3,val2-2,val2-arr1-1(1),val2-arr1-arr2-1(1)(1),val2-arr1-arr2-3(1)(1),val2-arr1-arr2-arr3-1(1)(1)(1),val2-arr1-arr2-arr3-2(1)(1)(1),val2-arr1-arr2-arr3-3(1)(1)(1)
val1,val3,val2-2,val2-arr1-1(1),val2-arr1-arr2-1(1)(1),val2-arr1-arr2-3(1)(1),val2-arr1-arr2-arr3-1(1)(1)(2),val2-arr1-arr2-arr3-2(1)(1)(2),val2-arr1-arr2-arr3-3(1)(1)(2)
val1,val3,val2-2,val2-arr1-1(1),val2-arr1-arr2-1(1)(1),val2-arr1-arr2-3(1)(1),val2-arr1-arr2-arr3-1(1)(1)(3),val2-arr1-arr2-arr3-2(1)(1)(3),val2-arr1-arr2-arr3-3(1)(1)(3)
val1,val3,val2-2,val2-arr1-1(1),val2-arr1-arr2-1(1)(2),val2-arr1-arr2-3(1)(2),val2-arr1-arr2-arr3-1(1)(2)(1),val2-arr1-arr2-arr3-2(1)(2)(1),val2-arr1-arr2-arr3-3(1)(2)(1)
val1,val3,val2-2,val2-arr1-1(1),val2-arr1-arr2-1(1)(2),val2-arr1-arr2-3(1)(2),val2-arr1-arr2-arr3-1(1)(2)(2),val2-arr1-arr2-arr3-2(1)(2)(2),val2-arr1-arr2-arr3-3(1)(2)(2)
val1,val3,val2-2,val2-arr1-1(1),val2-arr1-arr2-1(1)(2),val2-arr1-arr2-3(1)(2),val2-arr1-arr2-arr3-1(1)(2)(3),val2-arr1-arr2-arr3-2(1)(2)(3),val2-arr1-arr2-arr3-3(1)(2)(3)
val1,val3,val2-2,val2-arr1-1(2),val2-arr1-arr2-1(2)(1),val2-arr1-arr2-3(2)(1),val2-arr1-arr2-arr3-1(2)(1)(1),val2-arr1-arr2-arr3-2(2)(1)(1),val2-arr1-arr2-arr3-3(2)(1)(1)
val1,val3,val2-2,val2-arr1-1(2),val2-arr1-arr2-1(2)(1),val2-arr1-arr2-3(2)(1),val2-arr1-arr2-arr3-1(2)(1)(2),val2-arr1-arr2-arr3-2(2)(1)(2),val2-arr1-arr2-arr3-3(2)(1)(2)
val1,val3,val2-2,val2-arr1-1(2),val2-arr1-arr2-1(2)(1),val2-arr1-arr2-3(2)(1),val2-arr1-arr2-arr3-1(2)(1)(3),val2-arr1-arr2-arr3-2(2)(1)(3),val2-arr1-arr2-arr3-3(2)(1)(3)
val1,val3,val2-2,val2-arr1-1(2),val2-arr1-arr2-1(2)(2),val2-arr1-arr2-3(2)(2),val2-arr1-arr2-arr3-1(2)(2)(1),val2-arr1-arr2-arr3-2(2)(2)(1),val2-arr1-arr2-arr3-3(2)(2)(1)
val1,val3,val2-2,val2-arr1-1(2),val2-arr1-arr2-1(2)(2),val2-arr1-arr2-3(2)(2),val2-arr1-arr2-arr3-1(2)(2)(2),val2-arr1-arr2-arr3-2(2)(2)(2),val2-arr1-arr2-arr3-3(2)(2)(2)
val1,val3,val2-2,val2-arr1-1(2),val2-arr1-arr2-1(2)(2),val2-arr1-arr2-3(2)(2),val2-arr1-arr2-arr3-1(2)(2)(3),val2-arr1-arr2-arr3-2(2)(2)(3),val2-arr1-arr2-arr3-3(2)(2)(3)
Sample fiddle: https://dotnetfiddle.net/wnacwN
Disclaimer: I'm the author of this library.
I leave here what I have so far, #Cinchoo
public DataTable MapJson(string json)
{
var jo = JToken.Parse(json);
var stringReader = new StringReader(MakeConfigString(jo));
var serializer = new XmlSerializer(typeof(DataTableConfig));
var config = (DataTableConfig)serializer.Deserialize(stringReader);
var dataTable = new DataTable();
foreach (var p in config.Property)
{
dataTable.Columns.Add(p.Name);
}
var regex = new Regex(Regex.Escape("*"));
var count = 0;
if (jo is JArray)
{
count = jo.Count();
}
else if (jo is JObject jsonLinq)
{
count = jsonLinq.Descendants().FirstOrDefault(d => d is JArray)?.Count() ?? 1;
}
var dataCollection = Enumerable.Range(0, count)
.Select(x => config.Property
.Select(prop => new KeyValuePair<string, List<string>>
(
prop.Name,
jo.SelectTokens(regex.Replace(prop.Path, x.ToString(), 1)).Select(c => c.Value<string>()).ToList())
));
foreach (var data in dataCollection)
{
var countMax = data.Select(x => x.Value).Max(x => x.Count);
for (var i = 0; i < countMax; i++)
{
var innerList = new List<object>();
foreach (var prop in config.Property)
{
var currentData = data.Where(x => x.Key == prop.Name).SelectMany(x => x.Value).ToList();
if (currentData.Any())
{
if (i >= currentData.Count)
{
innerList.Add(currentData.Last());
}
else
{
innerList.Add(currentData[i]);
}
}
else
{
innerList.Add("");
}
}
dataTable.Rows.Add(innerList.ToArray());
}
}
return dataTable;
}
private string MakeConfigString(JToken json)
{
var result = #"<DataTableConfig Name='DataTable'>";
foreach (var col in this.ApiConnectorsColumns)
{
var path = json is JArray ? col.PathNode.Replace(".Columns", "[*]") : col.PathNode.Remove(0, 9);
var temp = $"<Property Name='{col.ColumnName}' Path='{path}'/>";
result += temp;
}
result += "</DataTableConfig>";
return result;
}
[XmlRoot(ElementName = "Property")]
public class Property
{
[XmlAttribute(AttributeName = "Name")]
public string Name { get; set; }
[XmlAttribute(AttributeName = "Path")]
public string Path { get; set; }
}
[XmlRoot(ElementName = "DataTableConfig")]
public class DataTableConfig
{
[XmlElement(ElementName = "Property")]
public List<Property> Property { get; set; }
[XmlAttribute(AttributeName = "Name")]
public string Name { get; set; }
}
I'm trying to search Multiple indexes Elasticsearch with NEST Client, I just follow the below link
[stackover post ]How to search inside multiple indices using Nest ElasticSearch?
the only difference was my indexes are already existed but nothing returns
Sample code :
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Elasticsearch.Net;
using Nest;
namespace ElasticSearchDemo
{
public class ExceptionData
{
public bool HasException { get; set; }
public string ExceptionMessage { get; set; }
}
public class ElasticSearchResponse : ExceptionData
{
public ISearchResponse<dynamic> elasticSearchResponse { get; set; }
}
public class ComponentTypES
{
public string ComponentID { get; set; }
public string Componentname { get; set; }
public string Summary { get; set; }
}
public class ProjectTypES
{
public string ProjectID { get; set; }
public string Projectname { get; set; }
public string Summary { get; set; }
public string Description { get; set; }
}
class Program
{
static void Main(string[] args)
{
// calling the function
var response = GetAllSearchResults("test", 0, 10);
}
public static ElasticClient GetElasticSearchCommonSearch()
{
ElasticClient elasticClient = null;
try
{
const string strElasticSearchURL = "http://localhost:9200/";
const string componentIndex = "componenttypeindex";
const string projectIndex = "projecttypeindex";
if (!string.IsNullOrEmpty(strElasticSearchURL))
{
ConnectionSettings connectionSettings = new ConnectionSettings(new Uri(strElasticSearchURL))
.DefaultIndex(componentIndex)
.DefaultMappingFor<ComponentTypES>(i => i.IndexName(componentIndex).TypeName("Componenttype"))
.DefaultMappingFor<ProjectTypES>(j => j.IndexName(projectIndex).TypeName("Projecttype"))
.DisableDirectStreaming()
.PrettyJson()
.OnRequestCompleted(callDetails =>
{
if (callDetails.RequestBodyInBytes != null)
{
Console.WriteLine(
$"{callDetails.HttpMethod} {callDetails.Uri} \n" +
$"{Encoding.UTF8.GetString(callDetails.RequestBodyInBytes)}");
}
else
{
Console.WriteLine($"{callDetails.HttpMethod} {callDetails.Uri}");
}
Console.WriteLine();
if (callDetails.ResponseBodyInBytes != null)
{
Console.WriteLine($"Status: {callDetails.HttpStatusCode}\n" +
$"{Encoding.UTF8.GetString(callDetails.ResponseBodyInBytes)}\n" +
$"{new string('-', 30)}\n");
}
else
{
Console.WriteLine($"Status: {callDetails.HttpStatusCode}\n" +
$"{new string('-', 30)}\n");
}
}
);
elasticClient = new ElasticClient(connectionSettings);
}
}
catch (Exception ex)
{
throw new Exception(ex.Message + " ConnectionObject for : Common Search");
}
return elasticClient;
}
public static ElasticSearchResponse GetAllSearchResults(string query = "test", int
page = 1, int pagesize = 10)
{
ElasticSearchResponse combinedResponse = new ElasticSearchResponse();
try
{
ElasticClient elasticClient = GetElasticSearchCommonSearch();
var clusterHealth = elasticClient.ClusterHealth();
if (clusterHealth.IsValid && string.Compare(clusterHealth.Status.ToString(), "red", true) != 0 && clusterHealth.ServerError == null)
{
string Componentindex = "componenttypeindex";
string Projectindex = "projecttypeindex";
var indices = Indices.Index(typeof(ComponentTypES)).And(typeof(ProjectTypES));
//elasticClient.Refresh(indices);
//TODO : Development time coding
if (null != (indices))
{
var indexExists = elasticClient.IndexExists(Indices.Index(Componentindex));
var projectExists = elasticClient.IndexExists(Indices.Index(Projectindex));
if (indexExists.Exists && indexExists.IsValid && projectExists.Exists && projectExists.IsValid)
{
//full text example 1
combinedResponse.elasticSearchResponse = elasticClient.Search<object>(s => s
.Index(indices)
.Type(Types.Type(typeof(ComponentTypES), typeof(ProjectTypES)))
.Query(q => (q
.MultiMatch(m => m
.Fields(f => f
.Field(Infer.Field<ComponentTypES>(ff => ff.Componentname))
.Field(Infer.Field<ComponentTypES>(ff => ff.Summary, 1.1))
)
.Operator(Operator.Or)
.Query(query)
) && +q
.Term("_index", Componentindex)) || (q
.MultiMatch(m => m
.Fields(f => f
.Field(Infer.Field<ProjectTypES>(ff => ff.Projectname))
.Field(Infer.Field<ProjectTypES>(ff => ff.Summary, 0.3))
)
.Operator(Operator.Or)
.Query(query)
) && +q
.Term("_index", Projectindex))
).From(page - 1)
.Size(pagesize)
);
//free text example 2
combinedResponse.elasticSearchResponse = elasticClient.Search<object>(s => s
.Index(indices)
.Type(Types.Type(typeof(ComponentTypES), typeof(ProjectTypES)))
.Query(q => (q
.MatchPhrase(m => m
.Field(Infer.Field<ComponentTypES>(ff => ff.Componentname))
.Query(query)
) && +q
.Term("_index", Componentindex)) || (q
.MatchPhrase(m => m
.Field(Infer.Field<ProjectTypES>(ff => ff.Projectname))
.Query(query)
)
) && +q
.Term("_index", Projectindex)
).From(page - 1)
.Size(pagesize)
);
}
else
{
combinedResponse.HasException = true;
combinedResponse.ExceptionMessage = "Index Not Found";
}
}
else
{
combinedResponse.HasException = true;
combinedResponse.ExceptionMessage = "Index Not Found In Config File";
}
}
else
{
combinedResponse.HasException = true;
combinedResponse.ExceptionMessage = "Error on connecting with ElasticSearch";
}
}
catch (Exception ex)
{
combinedResponse.HasException = true;
combinedResponse.ExceptionMessage = ex.Message;
return combinedResponse;
}
return combinedResponse;
}
}
}
Elastic table schema:
PUT componenttypeindex
{
"mappings": {
"Componenttype":{
"properties":{
"ComponentID":{"type":"text"},
"Componentname":{"type":"text"},
"Summary":{"type":"text"}
}
}
}
}
PUT projecttypeindex
{
"mappings": {
"Projecttype":{
"properties":{
"ProjectID":{"type":"text"},
"Projectname":{"type":"text"},
"Summary":{"type":"text"},
"Description":{"type":"text"}
}
}
}
}
it should return query matched items, but nothing returns
sorry for my ugly code formatting I tried but the new editor won't change anything
UPDATE :
i've updated the Index values in the query as suggested by #RussCam but still no expected results , and also when expands the response objects and ran the URI parameter in directly in Browser it has all the results something weird not sure why this not shown in response count
Valid NEST response built from a successful low level call on POST: /componenttypeindex%2Cprojecttypeindex/Componenttype%2CProjecttype/_search?typed_keys=true
Audit trail of this API call:
[1] Healthy Response: Node: http://localhost:9200/ Took: 00:00:00.0620000
Request:
URI = "http://localhost:9200/componenttypeindex%2Cprojecttypeindex/Componenttype%2CProjecttype/_search?typed_keys=true"
My POCO Classes:
public class ComponentTypES
{
public string ComponentID { get; set; }
public string Componentname { get; set; }
public string Summary { get; set; }
}
public class ProjectTypES
{
public string ProjectID { get; set; }
public string Projectname { get; set; }
public string Summary { get; set; }
public string Description { get; set; }
}
sample Data :
PUT componenttypeindex/Componenttype/5342e739-1635-4021-baf2-55e25b95b8ec
{
"ComponentID":"5342e739-1635-4021-baf2-55e25b95b8ec",
"Componentname":"TestComponent1",
"Summary":"this is summary of test component1"
}
PUT componenttypeindex/Componenttype/90781386-8065-11e9-bc42-526af7764f64
{
"ComponentID":"90781386-8065-11e9-bc42-526af7764f64",
"Componentname":"TestComponent2",
"Summary":"this is summary of test component3"
}
PUT componenttypeindex/Componenttype/19871386-8065-11e9-bc42-526af7764f64
{
"ComponentID":"19871386-8065-11e9-bc42-526af7764f64",
"Componentname":"some xyz component test",
"Summary":"this is summary test of test xyz"
}
PUT projecttypeindex/Projecttype/5342e739-2019-4021-baf2-55e25b95b8ec
{
"ProjectID":"5342e739-2019-4021-baf2-55e25b95b8ec",
"Projectname":"Test Project1",
"Summary":"summary of Test Project1",
"Description":"Description of TestProject1"
}
PUT projecttypeindex/Projecttype/5342f739-2019-4021-baf2-55e25b95b8ba
{
"ProjectID":"5342f739-2019-4021-baf2-55e25b95b8ba",
"Projectname":"Test Project2",
"Summary":"summary of Test Project2",
"Description":"Description of TestProject1"
}
PUT projecttypeindex/Projecttype/6342f739-2020-4021-baf2-55e25b95b8ac
{
"ProjectID":"6342f739-2020-4021-baf2-55e25b95b8ac",
"Projectname":"some PQRS project",
"Summary":"summary of PQRS Project",
"Description":"Description of PQORS Project1"
}
There's a lot of superfluous information in your example that makes it tricky to work with, which raises the barrier of effort required by someone wishing to help. Could I suggest that you reduce an example down to the smallest, succinct but complete example that demonstrates the problem you are facing in future; it'll really help in getting to the crux of the issue quicker!
I think the fundamental issue is that the casing of properties within the fields in the index mappings, and the casing of properties that NEST will send by default are different, so the nested must clauses within the should clause of the bool query that is generated by NEST will never be matched by any documents because of the field casing difference.
NEST by default camel cases property names, but the fields in the index mappings and documents in your example are all pascal cased, so the field names generated by NEST will not match those within the mapping. You can easily change NEST's field casing behaviour by using the DefaultFieldNameInferrer(Func<string, string>) method on ConnectionSettings. A delegate that simply returns the string value passed will leave field names as they are on the POCOs.
Here's a complete but succinct, working example
private static void Main()
{
const string componentIndex = "componenttypeindex";
const string projectIndex = "projecttypeindex";
var pool = new SingleNodeConnectionPool(new Uri("http://localhost:9200"));
var settings = new ConnectionSettings(pool)
.DefaultIndex(componentIndex)
.DefaultMappingFor<ComponentTypES>(i => i.IndexName(componentIndex).TypeName("Componenttype").IdProperty(f => f.ComponentID))
.DefaultMappingFor<ProjectTypES>(j => j.IndexName(projectIndex).TypeName("Projecttype").IdProperty(f => f.ProjectID))
.DefaultFieldNameInferrer(f => f)
.DefaultTypeName("_doc")
.DisableDirectStreaming()
.PrettyJson()
.OnRequestCompleted(callDetails =>
{
if (callDetails.RequestBodyInBytes != null)
{
Console.WriteLine(
$"{callDetails.HttpMethod} {callDetails.Uri} \n" +
$"{Encoding.UTF8.GetString(callDetails.RequestBodyInBytes)}");
}
else
{
Console.WriteLine($"{callDetails.HttpMethod} {callDetails.Uri}");
}
Console.WriteLine();
if (callDetails.ResponseBodyInBytes != null)
{
Console.WriteLine($"Status: {callDetails.HttpStatusCode}\n" +
$"{Encoding.UTF8.GetString(callDetails.ResponseBodyInBytes)}\n" +
$"{new string('-', 30)}\n");
}
else
{
Console.WriteLine($"Status: {callDetails.HttpStatusCode}\n" +
$"{new string('-', 30)}\n");
}
});
var client = new ElasticClient(settings);
foreach (var index in new[] { componentIndex, projectIndex })
{
if (client.IndexExists(index).Exists)
client.DeleteIndex(index);
client.CreateIndex(index, c => c
.Mappings(m => {
if (index == projectIndex)
return m.Map<ProjectTypES>(mm => mm.AutoMap());
else
return m.Map<ComponentTypES>(mm => mm.AutoMap());
})
);
}
client.Bulk(b => b
.IndexMany(new [] {
new ComponentTypES
{
ComponentID = "5342e739-1635-4021-baf2-55e25b95b8ec",
Componentname = "TestComponent1",
Summary = "this is summary of test component1"
},
new ComponentTypES
{
ComponentID = "90781386-8065-11e9-bc42-526af7764f64",
Componentname = "TestComponent2",
Summary = "this is summary of test component3"
},
new ComponentTypES
{
ComponentID = "19871386-8065-11e9-bc42-526af7764f64",
Componentname = "some xyz component test",
Summary = "this is summary test of test xyz"
},
})
.IndexMany(new [] {
new ProjectTypES
{
ProjectID = "5342e739-2019-4021-baf2-55e25b95b8ec",
Projectname = "Test Project1",
Summary = "summary of Test Project1",
Description = "Description of TestProject1"
},
new ProjectTypES
{
ProjectID = "5342f739-2019-4021-baf2-55e25b95b8ba",
Projectname = "Test Project2",
Summary = "summary of Test Project2",
Description = "Description of TestProject1"
},
new ProjectTypES
{
ProjectID = "6342f739-2020-4021-baf2-55e25b95b8ac",
Projectname = "some PQRS project",
Summary = "summary of PQRS Project",
Description = "Description of PQORS Project1"
},
})
.Refresh(Refresh.WaitFor)
);
var query = "test";
var response = client.Search<object>(s => s
.Index(Indices.Index(typeof(ComponentTypES)).And(typeof(ProjectTypES)))
.Type(Types.Type(typeof(ComponentTypES), typeof(ProjectTypES)))
.Query(q =>
(q
.MultiMatch(m => m
.Fields(f => f
.Field(Infer.Field<ComponentTypES>(ff => ff.Componentname))
.Field(Infer.Field<ComponentTypES>(ff => ff.Summary, 1.1))
)
.Operator(Operator.Or)
.Query(query)
) && +q
.Term("_index", componentIndex)
) ||
(q
.MultiMatch(m => m
.Fields(f => f
.Field(Infer.Field<ProjectTypES>(ff => ff.Projectname))
.Field(Infer.Field<ProjectTypES>(ff => ff.Summary, 0.3))
)
.Operator(Operator.Or)
.Query(query)
) && +q
.Term("_index", projectIndex)
)
)
);
}
public class ComponentTypES
{
public string ComponentID { get; set; }
public string Componentname { get; set; }
public string Summary { get; set; }
}
public class ProjectTypES
{
public string ProjectID { get; set; }
public string Projectname { get; set; }
public string Summary { get; set; }
public string Description { get; set; }
}
The resulting JSON query for the search is
POST http://localhost:9200/componenttypeindex%2Cprojecttypeindex/Componenttype%2CProjecttype/_search?pretty=true&typed_keys=true
{
"query": {
"bool": {
"should": [
{
"bool": {
"filter": [
{
"term": {
"_index": {
"value": "componenttypeindex"
}
}
}
],
"must": [
{
"multi_match": {
"fields": [
"Componentname",
"Summary^1.1"
],
"operator": "or",
"query": "test"
}
}
]
}
},
{
"bool": {
"filter": [
{
"term": {
"_index": {
"value": "projecttypeindex"
}
}
}
],
"must": [
{
"multi_match": {
"fields": [
"Projectname",
"Summary^0.3"
],
"operator": "or",
"query": "test"
}
}
]
}
}
]
}
}
}
which returns 5 results
{
"took" : 53,
"timed_out" : false,
"_shards" : {
"total" : 10,
"successful" : 10,
"skipped" : 0,
"failed" : 0
},
"hits" : {
"total" : 5,
"max_score" : 0.7549128,
"hits" : [
{
"_index" : "projecttypeindex",
"_type" : "Projecttype",
"_id" : "5342e739-2019-4021-baf2-55e25b95b8ec",
"_score" : 0.7549128,
"_source" : {
"ProjectID" : "5342e739-2019-4021-baf2-55e25b95b8ec",
"Projectname" : "Test Project1",
"Summary" : "summary of Test Project1",
"Description" : "Description of TestProject1"
}
},
{
"_index" : "componenttypeindex",
"_type" : "Componenttype",
"_id" : "19871386-8065-11e9-bc42-526af7764f64",
"_score" : 0.5565415,
"_source" : {
"ComponentID" : "19871386-8065-11e9-bc42-526af7764f64",
"Componentname" : "some xyz component test",
"Summary" : "this is summary test of test xyz"
}
},
{
"_index" : "componenttypeindex",
"_type" : "Componenttype",
"_id" : "5342e739-1635-4021-baf2-55e25b95b8ec",
"_score" : 0.3164503,
"_source" : {
"ComponentID" : "5342e739-1635-4021-baf2-55e25b95b8ec",
"Componentname" : "TestComponent1",
"Summary" : "this is summary of test component1"
}
},
{
"_index" : "projecttypeindex",
"_type" : "Projecttype",
"_id" : "5342f739-2019-4021-baf2-55e25b95b8ba",
"_score" : 0.2876821,
"_source" : {
"ProjectID" : "5342f739-2019-4021-baf2-55e25b95b8ba",
"Projectname" : "Test Project2",
"Summary" : "summary of Test Project2",
"Description" : "Description of TestProject1"
}
},
{
"_index" : "componenttypeindex",
"_type" : "Componenttype",
"_id" : "90781386-8065-11e9-bc42-526af7764f64",
"_score" : 0.20706992,
"_source" : {
"ComponentID" : "90781386-8065-11e9-bc42-526af7764f64",
"Componentname" : "TestComponent2",
"Summary" : "this is summary of test component3"
}
}
]
}
}
I want to change the mapping behavior in C# NEST.
My class looks like:
public class Vector
{
public Guid Guid { get; set; } = Guid.Empty;
public Dictionary<string, double> Entries { get; set; }
}
The default auto mapping (e.g.)
var client = new ElasticClient(settings);
client.Map<Vector>(m => m.AutoMap());
// index sample data
client.Index(new Vector { Guid = Guid.NewGuid(),Entries = new Dictionary <string, double> { {"A", 5.3}, {"B", 1.3}, {"C", 7.7}}});
client.Index(new Vector { Guid = Guid.NewGuid(), Entries = new Dictionary<string, double> { {"D", 2.3}, {"B", 8.0}, {"F", 5.1}}});
client.Index(new Vector { Guid = Guid.NewGuid(), Entries = new Dictionary<string, double> { {"A", 2.2}, {"B", 0.3}, {"F", 5.9}}});
generates the following Elasticsearch-mapping:
"mappings" : {
"vector" : {
"properties" : {
"entries" : {
"properties" : {
"A" : {
"type" : "float"
},
"B" : {
"type" : "float"
},
"C" : {
"type" : "float"
},
"D" : {
"type" : "float"
},
"F" : {
"type" : "float"
}
}
},
"guid" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
}
}
}
}
As you can see - A, B, C, D, F are float-properties. If you have to many type properties in ES - it will slow down the whole server. So I want to map the dictionary to a key-value-pair, like this:
"mappings" : {
"vector" : {
"properties" : {
"entries" : {
"properties" : {
"key" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"value" : {
"type" : "float"
}
}
}
...
}
}
}
So my current solution is:
public class Vector
{
public Guid Guid { get; set; } = Guid.Empty;
public KeyValuePair<string, double>[] Entries { get; set; }
}
// ...
client.Index(new Vector { Guid = Guid.NewGuid(), entries = dictionary.ToArray() });
Is there a better solution without .ToArray(); or build a nested class?
I'm trying to get a list of SEDOL's & ADP values. Below is my json text:
{
"DataFeed" : {
"#FeedName" : "AdminData",
"Issuer" : [{
"id" : "1528",
"name" : "ZYZ.A a Test Company",
"clientCode" : "ZYZ.A",
"securities" : {
"Security" : {
"id" : "1537",
"sedol" : "SEDOL111",
"coverage" : {
"Coverage" : [{
"analyst" : {
"#id" : "164",
"#clientCode" : "SJ",
"#firstName" : "Steve",
"#lastName" : "Jobs",
"#rank" : "1"
}
}, {
"analyst" : {
"#id" : "261",
"#clientCode" : "BG",
"#firstName" : "Bill",
"#lastName" : "Gates",
"#rank" : "2"
}
}
]
},
"customFields" : {
"customField" : [{
"#name" : "ADP Security Code",
"#type" : "Textbox",
"values" : {
"value" : "ADPSC1111"
}
}, {
"#name" : "Top 10 - Select one or many",
"#type" : "Dropdown, multiple choice",
"values" : {
"value" : ["Large Cap", "Cdn Small Cap", "Income"]
}
}
]
}
}
}
}, {
"id" : "1519",
"name" : "ZVV Test",
"clientCode" : "ZVV=US",
"securities" : {
"Security" : [{
"id" : "1522",
"sedol" : "SEDOL112",
"coverage" : {
"Coverage" : {
"analyst" : {
"#id" : "79",
"#clientCode" : "MJ",
"#firstName" : "Michael",
"#lastName" : "Jordan",
"#rank" : "1"
}
}
},
"customFields" : {
"customField" : [{
"#name" : "ADP Security Code",
"#type" : "Textbox",
"values" : {
"value" : "ADPS1133"
}
}, {
"#name" : "Top 10 - Select one or many",
"#type" : "Dropdown, multiple choice",
"values" : {
"value" : ["Large Cap", "Cdn Small Cap", "Income"]
}
}
]
}
}, {
"id" : "1542",
"sedol" : "SEDOL112",
"customFields" : {
"customField" : [{
"#name" : "ADP Security Code",
"#type" : "Textbox",
"values" : {
"value" : "ADPS1133"
}
}, {
"#name" : "Top 10 - Select one or many",
"#type" : "Dropdown, multiple choice",
"values" : {
"value" : ["Large Cap", "Cdn Small Cap", "Income"]
}
}
]
}
}
]
}
}
]
}
}
Here's the code that I have so far:
var compInfo = feed["DataFeed"]["Issuer"]
.Select(p => new {
Id = p["id"],
CompName = p["name"],
SEDOL = p["securities"]["Security"].OfType<JArray>() ?
p["securities"]["Security"][0]["sedol"] :
p["securities"]["Security"]["sedol"]
ADP = p["securities"]["Security"].OfType<JArray>() ?
p["securities"]["Security"][0]["customFields"]["customField"][0]["values"]["value"] :
p["securities"]["Security"]["customFields"]["customField"][0]["values"]["value"]
});
The error I get is:
Accessed JArray values with invalid key value: "sedol". Int32 array
index expected
I think I'm really close to figuring this out. What should I do to fix the code? If there is an alternative to get the SEDOL and ADP value, please do let me know?
[UPDATE1] I've started working with dynamic ExpandoObject. Here's the code that I've used so far:
dynamic obj = JsonConvert.DeserializeObject<ExpandoObject>(json, new ExpandoObjectConverter());
foreach (dynamic element in obj)
{
Console.WriteLine(element.DataFeed.Issuer[0].id);
Console.WriteLine(element.DataFeed.Issuer[0].securities.Security.sedol);
Console.ReadLine();
}
But I'm now getting the error 'ExpandoObject' does not contain a definition for 'DataFeed' and no extension method 'DataFeed' accepting a first argument of type 'ExpandoObject' could be found. NOTE: I understand that this json text is malformed. One instance has an array & the other is an object. I want the code to be agile enough to handle both instances.
[UPDATE2] Thanks to #dbc for helping me with my code so far. I've updated the json text above to closely match my current environment. I'm now able to get the SEDOLs & ADP codes. However, when I'm trying to get the 1st analyst, my code only works on objects and produces nulls for the analysts that are part of an array. Here's my current code:
var compInfo = from issuer in feed.SelectTokens("DataFeed.Issuer").SelectMany(i => i.ObjectsOrSelf())
let security = issuer.SelectTokens("securities.Security").SelectMany(s => s.ObjectsOrSelf()).FirstOrDefault()
where security != null
select new
{
Id = (string)issuer["id"], // Change to (string)issuer["id"] if id is not necessarily numeric.
CompName = (string)issuer["name"],
SEDOL = (string)security["sedol"],
ADP = security["customFields"]
.DescendantsAndSelf()
.OfType<JObject>()
.Where(o => (string)o["#name"] == "ADP Security Code")
.Select(o => (string)o.SelectToken("values.value"))
.FirstOrDefault(),
Analyst = security["coverage"]
.DescendantsAndSelf()
.OfType<JObject>()
.Select(jo => (string)jo.SelectToken("Coverage.analyst.#lastName"))
.FirstOrDefault(),
};
What do I need to change to always select the 1st analyst?
If you want all SEDOL & ADP values with the associated issuer Id and CompName for each, you can do:
var compInfo = from issuer in feed.SelectTokens("DataFeed.Issuer").SelectMany(i => i.ObjectsOrSelf())
from security in issuer.SelectTokens("securities.Security").SelectMany(s => s.ObjectsOrSelf())
select new
{
Id = (long)issuer["id"], // Change to (string)issuer["id"] if id is not necessarily numeric.
CompName = (string)issuer["name"],
SEDOL = (string)security["sedol"],
ADP = security["customFields"]
.DescendantsAndSelf()
.OfType<JObject>()
.Where(o => (string)o["#name"] == "ADP Security Code")
.Select(o => (string)o.SelectToken("values.value"))
.FirstOrDefault(),
};
Using the extension methods:
public static class JsonExtensions
{
public static IEnumerable<JToken> DescendantsAndSelf(this JToken node)
{
if (node == null)
return Enumerable.Empty<JToken>();
var container = node as JContainer;
if (container != null)
return container.DescendantsAndSelf();
else
return new[] { node };
}
public static IEnumerable<JObject> ObjectsOrSelf(this JToken root)
{
if (root is JObject)
yield return (JObject)root;
else if (root is JContainer)
foreach (var item in ((JContainer)root).Children())
foreach (var child in item.ObjectsOrSelf())
yield return child;
else
yield break;
}
}
Then
Console.WriteLine(JsonConvert.SerializeObject(compInfo, Formatting.Indented));
Produces:
[
{
"Id": 1528,
"CompName": "ZYZ.A a Test Company",
"SEDOL": "SEDOL111",
"ADP": "ADPSC1111"
},
{
"Id": 1519,
"CompName": "ZVV Test",
"SEDOL": "SEDOL112",
"ADP": "ADPS1133"
},
{
"Id": 1519,
"CompName": "ZVV Test",
"SEDOL": "SEDOL112",
"ADP": "ADPS1133"
}
]
However, in the query you have written so far, you seem to be trying to return only the first SEDOL & ADP for each issuer. If that is really what you want, do:
var compInfo = from issuer in feed.SelectTokens("DataFeed.Issuer").SelectMany(i => i.ObjectsOrSelf())
let security = issuer.SelectTokens("securities.Security").SelectMany(s => s.ObjectsOrSelf()).FirstOrDefault()
where security != null
select new
{
Id = (long)issuer["id"], // Change to (string)issuer["id"] if id is not necessarily numeric.
CompName = (string)issuer["name"],
SEDOL = (string)security["sedol"],
ADP = security["customFields"]
.DescendantsAndSelf()
.OfType<JObject>()
.Where(o => (string)o["#name"] == "ADP Security Code")
.Select(o => (string)o.SelectToken("values.value"))
.FirstOrDefault(),
};
Which results in:
[
{
"Id": 1528,
"CompName": "ZYZ.A a Test Company",
"SEDOL": "SEDOL111",
"ADP": "ADPSC1111"
},
{
"Id": 1519,
"CompName": "ZVV Test",
"SEDOL": "SEDOL112",
"ADP": "ADPS1133"
}
]
As an aside, since your JSON is rather polymorphic (properties are sometimes arrays of objects and sometimes just objects) I don't think deserializing to a class hierarchy or ExpandoObject will be easier.
Update
Given your updated JSON, you can use SelectTokens() with the JSONPath recursive search operator .. to find the first analyst's last name, where the recursive search operator handles the fact that the analysts might or might not be contained in an array:
var compInfo = from issuer in feed.SelectTokens("DataFeed.Issuer").SelectMany(i => i.ObjectsOrSelf())
let security = issuer.SelectTokens("securities.Security").SelectMany(s => s.ObjectsOrSelf()).FirstOrDefault()
where security != null
select new
{
Id = (string)issuer["id"], // Change to (string)issuer["id"] if id is not necessarily numeric.
CompName = (string)issuer["name"],
SEDOL = (string)security["sedol"],
ADP = (string)security["customFields"]
.DescendantsAndSelf()
.OfType<JObject>()
.Where(o => (string)o["#name"] == "ADP Security Code")
.Select(o => o.SelectToken("values.value"))
.FirstOrDefault(),
Analyst = (string)security.SelectTokens("coverage.Coverage..analyst.#lastName").FirstOrDefault(),
};
Also note your JSON is malformed.
The proxy class created can't figure out the type of Security because in one instance it's an array, and in another it's a simple object.
Original answer:
I used json2csharp to help me get some classes, and now I can use the types:
var obj = JsonConvert.DeserializeObject<RootObject>(json);
var result = from a in obj.DataFeed.Issuer
select new
{
Sedol = a.securities.Security.sedol,
Name = a.name
};
Classes:
public class Values
{
public object value { get; set; }
}
public class CustomField
{
public string name { get; set; }
public string type { get; set; }
public Values values { get; set; }
}
public class CustomFields
{
public List<CustomField> customField { get; set; }
}
public class Security
{
public string id { get; set; }
public string sedol { get; set; }
public CustomFields customFields { get; set; }
}
public class Securities
{
public Security Security { get; set; }
}
public class Issuer
{
public string id { get; set; }
public string name { get; set; }
public string clientCode { get; set; }
public Securities securities { get; set; }
}
public class DataFeed
{
public string FeedName { get; set; }
public List<Issuer> Issuer { get; set; }
}
public class RootObject
{
public DataFeed DataFeed { get; set; }
}
Using newtonsoft.json
dynamic obj = JsonConvert.DeserializeObject<ExpandoObject>(json);
I dont have my ide in front of me but the value should be at:
obj.DataFeed.Issuer[0].securities.Security.sedol
Well I am trying to perform insert batch operation. For that I am creating an array of BsonDocuemt. To each BsonDocument In array, I am adding BsonElements.
Question Class (Model):
public class Question
{
[BsonId]
[BsonRepresentation(BsonType.ObjectId)]
public string Id { get; set; }
public string QuestionName { get; set; }
public Dictionary<string, VariableDetails> Rules { get; set; }
public List<Question> QuestionsList { get; set; }
}
public class VariableDetails
{
public string variableType { get; set; }
public string min { get; set; }
public string max { get; set; }
}
Now I am trying to form array of BsonDocuemts as follows:
public void batchInsert(Question Model)
{
_collection = _db.GetCollection<Question>("Question");
BsonDocument[] batch = new BsonDocument[Model.QuestionsList.Count];
int count = 0;
foreach (Question question in Model.QuestionsList )
{
BsonDocument bsonDoc = new BsonDocument();
bsonDoc.Add(new BsonElement("QuestionName", question.QuestionName ));
//Following line is in error
bsonDoc.Add(new BsonElement("Rules", question.Rules));
//Argument type 'Systems.Collections.Generic.Dictionary<string, VariableDetails>' is not assignable to parameter type 'MongoDB.Bson.BsonValue'.
batch[count] = bsonDoc;
count++;
}
_collection.InsertBatch(batch);
}
I am able to add property of type string as BsonElement in BsonDocument, Not able to do so with property of type Dictionary.
I want the final insert in db should be like:
{
"_id" : ObjectId("54757796bb63bc08b481ad86"),
"Name" : "Question1",
"Rules" : {
"a1" : {
"variableType" : "1dPI",
"min" : "1",
"max" : "9"
},
"a2" : {
"variableType" : "1dPI",
"min" : "1",
"max" : "9"
}
}
},
{
"_id" : ObjectId("54757796bb63bc08b481ad86"),
"Name" : "Question2",
"Rules" : {
"d1" : {
"variableType" : "1dPI",
"min" : "1",
"max" : "9"
},
"f3" : {
"variableType" : "1dPI",
"min" : "1",
"max" : "9"
}
}
}
My question is how can I add property of type Dictionary as BsonElement in BsonDocuemt?
I just needed to add nested BsonDocument as an element to parent BsonDocument. That's it!
public void batchInsert(Question Model)
{
_collection = _db.GetCollection<Question>("Question");
BsonDocument[] batch = new BsonDocument[Model.QuestionList.Count];
int count = 0;
foreach (Question question in Model.QuestionList)
{
BsonDocument rulesBsonDoc = new BsonDocument();
foreach (KeyValuePair<string, VariableDetails> qTemp in question.Rules)
{
string variableName = qTemp.Key;
VariableDetails variableDetails = qTemp.Value;
string variableType = variableDetails.variableType;
string min = variableDetails.min;
string max = variableDetails.max;
BsonDocument childBsonDoc = new BsonDocument();
childBsonDoc.Add(new BsonElement("variableType", variableType));
childBsonDoc.Add(new BsonElement("min", min));
childBsonDoc.Add(new BsonElement("max", max));
rulesBsonDoc.Add(new BsonElement(variableName, childBsonDoc));
}
BsonDocument bsonDoc = new BsonDocument();
bsonDoc.Add(new BsonElement("Name", question.Name));
bsonDoc.Add(new BsonElement("Rules", rulesBsonDoc));
batch[count] = bsonDoc;
count++;
}
_collection.InsertBatch(batch);
}