So I'm trying to configure my index to have certain mappings and filters, but whenever I try to create the index I get the following error:
"[amgindex] failed to create index]; nested: IllegalArgumentException[Custom Analyzer [amgsearch] failed to find filter under name [synonym]"
this is the code I'm using to create the index
public void newIndex() {
var amgBasic = new CustomAnalyzer {
Tokenizer = "edgeNGram",
Filter = new string[] { "lowercase", "worddelimiter", "stemmerEng", "stemmerNl", "stopper", "snowball" } //
};
var amgBasicText = new CustomAnalyzer {
Tokenizer = "standard",
Filter = new string[] { "lowercase", "worddelimiter" }
};
var amgSearch = new CustomAnalyzer {
Tokenizer = "whitespace",
Filter = new string[] { "lowercase", "synonym" }
};
var synonmyfilter = new SynonymTokenFilter() {
Format = "Solr",
SynonymsPath = "analysis/synonym.txt"
};
try {
var result = client.CreateIndex("amgindex", i => i
.Analysis(descriptor => descriptor
.Analyzers(bases => bases
//.Add("amgBasic", amgBasic)
//.Add("amgBasicText", amgBasicText)
.Add("amgsearch", amgSearch)
)
.TokenFilters(c => c.Add("stemmereng", new StemmerTokenFilter() { Language = "english" }))
.TokenFilters(c => c.Add("stemmernl", new StemmerTokenFilter() { Language = "english" }))
.TokenFilters(c => c.Add("stopper", new StopTokenFilter() { Stopwords = new List<string>() { "_english_", "_dutch_" } }))
.TokenFilters(c => c.Add("snowball", new SnowballTokenFilter() { Language = "english" }))
.TokenFilters(c => c.Add("worddelimiter ", new WordDelimiterTokenFilter() { }))
.TokenFilters(c => c.Add("synonym ", synonmyfilter))
)
.AddMapping<general_document>(m => m
.Properties(o => o
.String(p => p.Name(x => x.object_name).IndexAnalyzer("amgSearch"))
.String(p => p.Name(x => x.title).IndexAnalyzer("amgSearch"))
.String(p => p.Name(x => x.Text).IndexAnalyzer("amgSearch"))
)
)
);
Log.Info("Index created? " + result.Acknowledged);
} catch (Exception ex) {
Log.Error("[index-creation] " + ex.Message);
throw;
}
}
everytime I use one of my own filters the error pops up.
Any clue why this is happening?
After spending way to much time on this I found it!! :)
there was a space in the name of 2 Filters, and those were causing the issue;
▼
.TokenFilters(c => c.Add("worddelimiter ", new WordDelimiterTokenFilter() { }))
▼
.TokenFilters(c => c.Add("synonym ", synonmyfilter))
Related
I am trying to map the below class to a destination class using the below code. Here trying to map Employee class to
static void Main(string[] args)
{
var emp = new List<Employee>()
{
new Employee()
{
FirstName = "Test",
LastName = "Performance",
ID="1",
Availabities = new List<Availability>()
{
new Availability()
{
BeginDate = DateTime.Now,
EndDate = DateTime.Now.AddDays(1)
}
}
},
new Employee()
{
FirstName = "Test1",
LastName = "Performance1",
ID="2",
Availabities = new List<Availability>()
{
new Availability()
{
BeginDate = DateTime.Now,
EndDate = DateTime.Now.AddDays(10)
}
}
},
new Employee()
{
FirstName = "Test123",
LastName = "Performance1",
ID="3",
Availabities = new List<Availability>()
{
new Availability()
{
BeginDate = DateTime.Now,
EndDate = DateTime.Now.AddDays(5)
},
new Availability()
{
BeginDate = DateTime.Now,
EndDate = DateTime.Now.AddDays(3)
}
}
}
};
Here is the destination object mapping, here the Employee to be mapped with EmployeeDest.
List<AvailabilityDest> empAvailabilitiesDest = new List<AvailabilityDest>();
var results = new List<EmployeeDest>();
foreach (var token in emp)
{
empAvailabilitiesDest.Clear();
foreach (var item in token.Availabities)
{
var empAvailability = new AvailabilityDest
{
BeginDateDest = item.BeginDate,
EndDateDest = item.EndDate,
};
empAvailabilitiesDest.Add(empAvailability);
}
var employee = new EmployeeDest
{
FirstNameDest = token.FirstName,
LastNameDest = token.LastName,
IDDest =token.ID,
AvailabitiesDest = empAvailabilitiesDest
};
results.Add(employee);
}
Console.WriteLine(results);
}
Here the empAvailabilitiesDest.Clear() is not clearing the list and the availabilityDest is getting increased with each iteration.
I am missing something here .
How can I optimize the code here to get a better performance?
Presumably the real problem here is that all your employee objects are sharing the same list; it doesn't matter whether you clear it, add things, etc: if there's one list and it is shared between all the employees, then changes to that list will show against every employee, and it will appear incorrect.
Presumably, you really just want to move the list creation to inside the foreach per employee:
var results = new List<EmployeeDest>();
foreach (var token in emp)
{
List<AvailabilityDest> empAvailabilitiesDest = new List<AvailabilityDest>
();
// your previous code unchanged
}
1. Problem in your code
var employee = new EmployeeDest
{
FirstNameDest = token.FirstName,
LastNameDest = token.LastName,
IDDest = token.ID,
AvailabitiesDest = empAvailabilitiesDest // This is problematic code
};
Due to above code, every empolyee.AvailabitiesDest will reference to the same object because it is not a primitive type variable.
You have to assign a copy of the object to employee.AvailabitiesDest,
var employee = new EmployeeDest
{
FirstNameDest = token.FirstName,
LastNameDest = token.LastName,
IDDest = token.ID,
AvailabitiesDest = new List<AvailabilityDest>(empAvailabilitiesDest)
};
2. Simplifying code
Use Linq.
results = emp.Select(token => new EmployeeDest
{
FirstNameDest = token.FirstName,
LastNameDest = token.LastName,
IDDest = token.ID,
AvailabitiesDest = token.Availabities.Select(item => new AvailabilityDest
{
BeginDateDest = item.BeginDate,
EndDateDest = item.EndDate,
}).ToList()
}).ToList();
Or use AutoMapper
var mapper = new Mapper(new MapperConfiguration(cfg =>
{
cfg.CreateMap<Employee, EmployeeDest>()
.ForMember(x => x.FirstNameDest, opt => opt.MapFrom(src => src.FirstName))
.ForMember(x => x.LastNameDest, opt => opt.MapFrom(src => src.LastName))
.ForMember(x => x.IDDest, opt => opt.MapFrom(src => src.ID))
.ForMember(x=> x.AvailabitiesDest, opt => opt.MapFrom(src => src.Availabities));
cfg.CreateMap<Availability, AvailabilityDest>()
.ForMember(x => x.BeginDateDest, opt => opt.MapFrom(src => src.BeginDate))
.ForMember(x => x.EndDateDest, opt => opt.MapFrom(src => src.EndDate));
}));
var results = mapper.Map<List<EmployeeDest>>(emp);
I am using Elasticsearch 6.6.0 and NEST in a .NET MVC project.
I am indexing some products using this code:
var esSettings = new ConnectionSettings(node);
esSettings = esSettings.DefaultIndex(IndexInstanceName);
esSettings = esSettings
.DefaultMappingFor<SearchableProduct>(s => s.IdProperty("Id").IndexName(IndexInstanceName + "-products-" + ConfigurationManager.AppSettings["DefaultCulture"]));
var elastic = new ElasticClient(esSettings);
var mapResponse = elastic.Map<SearchableProduct>(x => x.AutoMap().Index(IndexInstanceName + "-products-" + culture));
var indexState = new IndexState
{
Settings = new IndexSettings()
};
indexState.Settings.Analysis = new Analysis
{
Analyzers = new Analyzers()
};
indexState.Settings.Analysis.Analyzers.Add("nospecialchars", new CustomAnalyzer
{
Tokenizer = "standard",
Filter = new List<string> { "standard", "lowercase", "stop", "asciifolding" }
});
//products
if (!elastic.IndexExists(IndexInstanceName + "-products-" + culture).Exists)
{
var response = elastic.CreateIndex(
IndexInstanceName + "-products-" + culture,
s => s.InitializeUsing(indexState)
.Mappings(m => m.Map<SearchableProduct>(sc => sc.AutoMap())));
}
await this.IndexProductsAsync(context, products, elastic, culture);
await elastic.RefreshAsync(new RefreshRequest(IndexInstanceName + "-products-" + culture));
and for the search I use the below code:
ISearchResponse<SearchableProduct> result = await elastic.SearchAsync<SearchableProduct>(s => s
.Index(elasticIndexName + "-products-" + culture)
.Take(DefaultPageSize)
.Source(src => src.IncludeAll())
.Query(query =>
query.QueryString(qs =>
qs.Query(q).DefaultOperator(Operator.And).Fuzziness(Fuzziness.EditDistance(0)).Fields(x => x.Field(d => d.Name, 2)
.Field(d => d.MetaTitle, 1)
.Field(d => d.Image, 1)
.Field(d => d.SystemId, 2)
.Field(d => d.Manufacturer, 1)
)
))
.Sort(d => d.Ascending(SortSpecialField.Score))
);
When i search for a word with accent in greek (eg παγωτό) I get results (Because in my index the product is indexed with accent), but when i use the same word without accent (eg παγωτο) i get no results.
Is anything wrong with the indexing settings or the search code?
Can I index my data without accents or alternatively index them as is but make the search or index accent insensitive?
Creating a field with a greek analyzer will make sure indexed text and query string pass the same analysis path. For παγωτό that means, during indexing, the text will be tokenized to παγωτ as well as during making the query request.
Please check my example which creates a field with greek analyzer and the example outputs both documents with παγωτό and παγωτο when looking for παγωτό or παγωτο.
class Program
{
static async Task Main(string[] args)
{
var connectionPool = new SingleNodeConnectionPool(new Uri("http://localhost:9200"));
var settings = new ConnectionSettings(connectionPool)
.DefaultIndex("index_name")
.DisableDirectStreaming()
.PrettyJson();
var client = new ElasticClient(settings);
await client.Indices.DeleteAsync("index_name");
var createIndexResponse = await client.Indices.CreateAsync("index_name",
c => c
.Map(map => map.AutoMap<Document>()));
await client.IndexManyAsync(new []
{new Document {Id = 1, Text = "παγωτό"}, new Document {Id = 2, Text = "παγωτο"},});
await client.Indices.RefreshAsync();
var query = "παγωτό";
var searchResponse = await client.SearchAsync<Document>(s => s
.Query(q => q.Match(m => m.Field(f => f.Text).Query(query))));
Console.OutputEncoding = Encoding.UTF8;
Print(query, searchResponse);
query = "παγωτο";
var searchResponse2 = await client.SearchAsync<Document>(s => s
.Query(q => q.Match(m => m.Field(f => f.Text).Query(query))));
Print(query, searchResponse2);
}
private static void Print(string query, ISearchResponse<Document> searchResponse)
{
Console.WriteLine($"For {query} found:");
foreach (var document in searchResponse.Documents)
{
Console.WriteLine($"Document {document.Id} {document.Text}");
}
}
}
public class Document
{
public int Id { get; set; }
[Text(Analyzer = "greek")]
public string Text { get; set; }
}
Prints:
For παγωτό found:
Document 1 παγωτό
Document 2 παγωτο
For παγωτο found:
Document 1 παγωτό
Document 2 παγωτο
Hope that helps.
I'm stuck on a basic problem. I need to convert a flat List to a Nested dictionary of type Dictionary<string,Dictionary<string,Dictionary<string,Dictionary<string,string>>>>
I've almost done except the fact that I need to group multiple properties, and I used an anonymous type for that. Here is an example :
List<ApprovalAction> Gen()
{
return new List<ApprovalAction>
{
new ApprovalAction { Name = "SendMailToApprover", Step="New", ApprovalRequestType = "Homeworking", ParameterName = "to", ParameterValue="Approver" },
new ApprovalAction { Name = "SendMailToApprover", Step="New", ApprovalRequestType = "Homeworking", ParameterName="subject", ParameterValue = "Aproval Request" },
new ApprovalAction { Name = "SendMailToApprover", Step="New", ApprovalRequestType = "Homeworking", ParameterName="body", ParameterValue = "I would like an approval request"},
new ApprovalAction { Name = "SendMailToApprover", Step="New", ApprovalRequestType = "Absence", ParameterName="to" , ParameterValue="Approver" },
new ApprovalAction { Name = "SendMailToApprover", Step="New", ApprovalRequestType = "Absence", ParameterName="subject", ParameterValue = "Aproval Request" },
new ApprovalAction { Name = "SendMailToApprover", Step="New", ApprovalRequestType = "Absence", ParameterName="body", ParameterValue = "I would like an approval request"}
};
}
var actions = Gen();
var dico = actions
.GroupBy(x => x.ApprovalRequestType)
.ToDictionary(
gdc => gdc.Key,
gdc => gdc.GroupBy(a => a.Step)
.ToDictionary(dd => dd.Key, dd => dd.GroupBy(x => new { x.Name, x.Step, x.ApprovalRequestType }, (key, group) => new
{
Key = key.Name,
Result = group.ToDictionary(k => k.ParameterName, v => v.ParameterValue)
})));
This is the output of Linqpad :
Do you know by which code I can replace to avoid the IEnumerable ?
Thank you !
IEnumerable<> comes from the final GroupBy. If you know that the innermost group would contain exactly one item, use Single(). Otherwise use First():
var dico = actions
.GroupBy(x => x.ApprovalRequestType)
.ToDictionary(
gdc => gdc.Key,
gdc => gdc.GroupBy(a => a.Step)
.ToDictionary(dd => dd.Key, dd => dd.GroupBy(x => new { x.Name, x.Step, x.ApprovalRequestType }, (key, group) => new {
Key = key.Name,
Result = group.ToDictionary(k => k.ParameterName, v => v.ParameterValue)
}).First()
)
);
Result of the query above looks like this:
I've got:
var result = _client.Search<ElasticFilm>(new SearchRequest("blaindex", "blatype")
{
From = 0,
Size = 100,
Query = titleQuery || pdfQuery,
Source = new SourceFilter
{
Include = new []
{
Property.Path<ElasticFilm>(p => p.Url),
Property.Path<ElasticFilm>(p => p.Title),
Property.Path<ElasticFilm>(p => p.Language),
Property.Path<ElasticFilm>(p => p.Details),
Property.Path<ElasticFilm>(p => p.Id)
}
},
Timeout = "20000"
});
And I'm trying to add a highlighter filter but I'm not that familiar with the Object Initializer (OIS) C# syntax. I've checked NEST official pages and SO but can't seem to return any results for specifically the (OIS).
I can see the Highlight property in the Nest.SearchRequest class but I'm not experienced enough (I guess) to simply construct what I need from there - some examples and explanations as to how to employ a highlighter with OIS would be hot!
This is the fluent syntax:
var response= client.Search<Document>(s => s
.Query(q => q.Match(m => m.OnField(f => f.Name).Query("test")))
.Highlight(h => h.OnFields(fields => fields.OnField(f => f.Name).PreTags("<tag>").PostTags("</tag>"))));
and this is by object initialization:
var searchRequest = new SearchRequest
{
Query = new QueryContainer(new MatchQuery{Field = Property.Path<Document>(p => p.Name), Query = "test"}),
Highlight = new HighlightRequest
{
Fields = new FluentDictionary<PropertyPathMarker, IHighlightField>
{
{
Property.Path<Document>(p => p.Name),
new HighlightField {PreTags = new List<string> {"<tag>"}, PostTags = new List<string> {"</tag>"}}
}
}
}
};
var searchResponse = client.Search<Document>(searchRequest);
UPDATE
NEST 7.x syntax:
var searchQuery = new SearchRequest
{
Highlight = new Highlight
{
Fields = new FluentDictionary<Field, IHighlightField>()
.Add(Nest.Infer.Field<Document>(d => d.Name),
new HighlightField {PreTags = new[] {"<tag>"}, PostTags = new[] {"<tag>"}})
}
};
My document class:
public class Document
{
public int Id { get; set; }
public string Name { get; set; }
}
How can I configure Index using NEST with such JSON:
"settings":{
"analysis":{
"filter":{
"name_ngrams":{
"side":"front",
"max_gram":50,
"min_gram":2,
"type":"edgeNGram"
}
},
"analyzer":{
"partial_name":{
"filter":[
"standard",
"lowercase",
"asciifolding",
"name_ngrams"
],
"type":"custom",
"tokenizer":"standard"
}
}
}
I could create my custom analyzer using CustomAnalyzer class, but I could'n find how to create custom filter and register it within my analyzer.
Thanks in advance!
After some searching I've found a solution:
var partialName = new CustomAnalyzer
{
Filter = new List<string> {"lowercase", "name_ngrams", "standard", "asciifolding"},
Tokenizer = "standard"
};
var fullName = new CustomAnalyzer
{
Filter = new List<string> {"standard", "lowercase", "asciifolding"},
Tokenizer = "standard"
};
client.CreateIndex("indexname", c => c
.Analysis(descriptor => descriptor
.TokenFilters(bases => bases.Add("name_ngrams", new EdgeNGramTokenFilter
{
MaxGram = 20,
MinGram = 2,
Side = "front"
}))
.Analyzers(bases => bases
.Add("partial_name", partialName)
.Add("full_name", fullName))
)
.AddMapping<Company>(m => m
.Properties(o => o
.String(i => i
.Name(x => x.Name)
.IndexAnalyzer("partial_name")
.SearchAnalyzer("full_name")
))));