Develop Reference

Lucene 4.8 facets usage

I have difficulties understanding this example on how to use facets :
https://lucenenet.apache.org/docs/4.8.0-beta00008/api/Lucene.Net.Demo/Lucene.Net.Demo.Facet.SimpleFacetsExample.html
My goal is to create an index in which each document field have a facet, so that at search time i can choose which facets use to navigate data.
What i am confused about is setup of facets in index creation, to
summarize my question : is index with facets compatibile with
ReferenceManager?
Need DirectoryTaxonomyWriter to be actually written and persisted
on disk or it will embedded into the index itself and is just
temporary? I mean given the code
indexWriter.AddDocument(config.Build(taxoWriter, doc)); of the
example i expect it's temporary and will be embedded into the index (but then the example also show you need the Taxonomy to drill down facet). So can the Taxonomy be tangled in some way with the index so that the are handled althogeter with ReferenceManager?
If is not may i just use the same folder i use for storing index?
Here is a more detailed list of point that confuse me :
In my scenario i am indexing the document asyncrhonously (background process) and then fetching the indext ASAP throught ReferenceManager in ASP.NET application. I hope this way to fetch the index is compatibile with DirectoryTaxonomyWriter needed by facets.
Then i modified the code i write introducing the taxonomy writer as indicated in the example, but i am a bit confused, seems like i can't store DirectoryTaxonomyWriter into the same folder of index because the folder is locked, need i to persist it or it will be embedded into the index (so a RAMDirectory is enougth)? if i need to persist it in a different direcotry, can i safely persist it into subdirectory?
Here the code i am actually using :
private static void BuildIndex (IndexEntry entry)
{
string targetFolder = ConfigurationManager.AppSettings["IndexFolder"] ?? string.Empty;
//** LOG
if (System.IO.Directory.Exists(targetFolder) == false)
{
string message = #"Index folder not found";
_fileLogger.Error(message);
_consoleLogger.Error(message);
return;
}
var metadata = JsonConvert.DeserializeObject<IndexMetadata>(File.ReadAllText(entry.MetdataPath) ?? "{}");
string[] header = new string[0];
List<dynamic> csvRecords = new List<dynamic>();
using (var reader = new StreamReader(entry.DataPath))
{
CsvConfiguration csvConfiguration = new CsvConfiguration(CultureInfo.InvariantCulture);
csvConfiguration.AllowComments = false;
csvConfiguration.CountBytes = false;
csvConfiguration.Delimiter = ",";
csvConfiguration.DetectColumnCountChanges = false;
csvConfiguration.Encoding = Encoding.UTF8;
csvConfiguration.HasHeaderRecord = true;
csvConfiguration.IgnoreBlankLines = true;
csvConfiguration.HeaderValidated = null;
csvConfiguration.MissingFieldFound = null;
csvConfiguration.TrimOptions = CsvHelper.Configuration.TrimOptions.None;
csvConfiguration.BadDataFound = null;
using (var csvReader = new CsvReader(reader, csvConfiguration))
{
csvReader.Read();
csvReader.ReadHeader();
csvReader.Read();
header = csvReader.HeaderRecord;
csvRecords = csvReader.GetRecords<dynamic>().ToList();
}
}
string targetDirectory = Path.Combine(targetFolder, "Index__" + metadata.Boundle + "__" + DateTime.Now.ToString("yyyyMMdd_HHmmss") + "__" + Path.GetRandomFileName().Substring(0, 6));
System.IO.Directory.CreateDirectory(targetDirectory);
//** LOG
{
string message = #"..creating index : {0}";
_fileLogger.Information(message, targetDirectory);
_consoleLogger.Information(message, targetDirectory);
}
using (var dir = FSDirectory.Open(targetDirectory))
{
using (DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(dir))
{
Analyzer analyzer = metadata.GetAnalyzer();
var indexConfig = new IndexWriterConfig(LuceneVersion.LUCENE_48, analyzer);
using (IndexWriter writer = new IndexWriter(dir, indexConfig))
{
long entryNumber = csvRecords.Count();
long index = 0;
long lastPercentage = 0;
foreach (dynamic csvEntry in csvRecords)
{
Document doc = new Document();
IDictionary<string, object> dynamicCsvEntry = (IDictionary<string, object>)csvEntry;
var indexedMetadataFiled = metadata.IdexedFields;
foreach (string headField in header)
{
if (indexedMetadataFiled.ContainsKey(headField) == false || (indexedMetadataFiled[headField].NeedToBeIndexed == false && indexedMetadataFiled[headField].NeedToBeStored == false))
continue;
var field = new Field(headField,
((string)dynamicCsvEntry[headField] ?? string.Empty).ToLower(),
indexedMetadataFiled[headField].NeedToBeStored ? Field.Store.YES : Field.Store.NO,
indexedMetadataFiled[headField].NeedToBeIndexed ? Field.Index.ANALYZED : Field.Index.NO
);
doc.Add(field);
var facetField = new FacetField(headField, (string)dynamicCsvEntry[headField]);
doc.Add(facetField);
}
long percentage = (long)(((decimal)index / (decimal)entryNumber) * 100m);
if (percentage > lastPercentage && percentage % 10 == 0)
{
_consoleLogger.Information($"..indexing {percentage}%..");
lastPercentage = percentage;
}
writer.AddDocument(doc);
index++;
}
writer.Commit();
}
}
}
//** LOG
{
string message = #"Index Created : {0}";
_fileLogger.Information(message, targetDirectory);
_consoleLogger.Information(message, targetDirectory);
}
}

Getting an MSFT_Partition for a given MSFT_Volume

I’m trying to get some details about a disk based on the drive letter and I’m having trouble getting the partition info for a given volume.
This is what I’m trying:
var scope = new ManagementScope(#"\\localhost\ROOT\Microsoft\Windows\Storage");
var query = new ObjectQuery("Associators of {MSFT_Volume.DriveLetter='" + upperCaseDiskLetter + "'} WHERE RESULTCLASS = MSFT_Partition");
using (var partitionSearcher = new ManagementObjectSearcher(
scope,
query))
{
var partition = partitionSearcher.Get().Cast<ManagementBaseObject>().Single();
And getting this exception:
Exception thrown: 'System.Management.ManagementException' in System.Management.dll
Additional information: Invalid object path
Maybe I’m approaching it wrong? I suppose a volume might exist on more than one partition? But still – I should get the “associators” query return some results unless MSFT_PartitionToVolume isn’t working for some reason?
This is what I’m trying to collect:
using (var volumeSearcher = new ManagementObjectSearcher(
#"\\localhost\ROOT\Microsoft\Windows\Storage",
$"SELECT Size, SizeRemaining, DriveType FROM MSFT_Volume WHERE DriveLetter='{ upperCaseDiskLetter }'"))
{
var volume = volumeSearcher.Get().Cast<ManagementBaseObject>().Single();
result = new DiskProperties
{
LastRetrieved = DateTime.Now,
DriveType = GetDriveTypeDescription((UInt32)volume["DriveType"]),
FreeSpace = (UInt64)volume["SizeRemaining"],
PartitionSize = (UInt64)volume["Size"]
};
}
var scope = new ManagementScope(#"\\localhost\ROOT\Microsoft\Windows\Storage");
var query = new ObjectQuery("Associators of {MSFT_Volume.DriveLetter='" + upperCaseDiskLetter + "'} WHERE RESULTCLASS = MSFT_Partition");
using (var partitionSearcher = new ManagementObjectSearcher(
scope,
query))
{
var partition = partitionSearcher.Get().Cast<ManagementBaseObject>().Single();
using (var physicalDiskSearcher = new ManagementObjectSearcher(
#"\\localhost\ROOT\Microsoft\Windows\Storage",
$"SELECT Size, Model, MediaType FROM MSFT_PhysicalDisk WHERE DeviceID='{ partition["DiskNumber"] }'"))
{
var physicalDisk = physicalDiskSearcher.Get().Cast<ManagementBaseObject>().Single();
result.DiskSize = (UInt64)physicalDisk["Size"];
result.Model = physicalDisk["Model"].ToString();
result.IsSsd =
(UInt16)physicalDisk["MediaType"] == 4 ||
SSDModelSubstrings.Any(substring => result.Model.ToLower().Contains(substring)); ;
return result;
}
}

I'm still not sure what's wrong with the associators query, but it seems like I overlooked the very simple solution - MSFT_Partition also has a DriveLetter column...
using (var partitionSearcher = new ManagementObjectSearcher(
#"\\localhost\ROOT\Microsoft\Windows\Storage",
$"SELECT DiskNumber FROM MSFT_Partition WHERE DriveLetter='{upperCaseDiskLetter}'"))
Note though that this won't be populated for a VHD drive.

Add and remove Ektron content items from taxonomies? (C#)

I have content items stored in Ektron that are assigned to taxonomies. I'm trying to create a method that will allow me to programmatically change the taxonomies. So far I find the content item by ID, and I'm able to retrieve its taxonomies, but I'm not sure how to change them.
var ektronItem = contentManager.GetItem((long) item.tctmd_id);
if (ektronItem != null) // item exists in Ektron
{
var newTaxonomies = item.taxonomy_ids;
var taxonomyAPI = new Taxonomy();
var taxData = taxonomyAPI.ReadAllAssignedCategory(ektronItem.Id);
foreach (var tax in taxData)
{
taxonomyAPI.RemoveTaxonomyItem(???);
// here I'm trying to remove the content item from the taxonomy
}
}
taxonomyAPI.RemoveTaxonomyItem() takes a Ektron.Cms.TaxonomyRequest object, but I'm not sure how to create this. I'm also not sure if this is even the method I should be using.

In case anyone else wants to know how to do this, here's the solution I came up with:
var contentManager = new Ektron.Cms.Framework.Content.ContentManager();
var criteria = new Ektron.Cms.Content.ContentCriteria(ContentProperty.Id, EkEnumeration.OrderByDirection.Ascending);
criteria.AddFilter(ContentProperty.FolderId, CriteriaFilterOperator.EqualTo, toUpdate.folder_id);
criteria.OrderByDirection = Ektron.Cms.Common.EkEnumeration.OrderByDirection.Descending;
criteria.OrderByField = Ektron.Cms.Common.ContentProperty.GoLiveDate;
criteria.FolderRecursive = true;
criteria.PagingInfo = new Ektron.Cms.PagingInfo(50, 1);
var ektronItem = contentManager.GetItem((long) item.tctmd_id);
if (ektronItem != null) // item exists in Ektron
{
// update taxonomy in Ektron
var taxIds = item.taxonomy_ids;
var taxonomyAPI = new Taxonomy();
var taxData = taxonomyAPI.ReadAllAssignedCategory(ektronItem.Id);
var taxManager = new Ektron.Cms.Framework.Organization.TaxonomyItemManager();
var taxCriteria = new TaxonomyItemCriteria();
// create a taxonomy criteria of the item ID
taxCriteria.AddFilter(TaxonomyItemProperty.ItemId, CriteriaFilterOperator.EqualTo, item.tctmd_id);
// get all taxonomy items with item ID
var taxItems = taxManager.GetList(taxCriteria);
// determine taxonomyItemType
var type = taxItems.FirstOrDefault().ItemType;
foreach (var tax in taxData)
{
// delete from old taxonomies
taxManager.Delete(tax.Id, (long)item.tctmd_id, type);
}
foreach (var tax in taxIds)
{
// add to new taxonomies
var taxonomyItemData = new TaxonomyItemData()
{
TaxonomyId = tax,
ItemType = type,
ItemId = (long)item.tctmd_id
};
try
{
taxManager.Add(taxonomyItemData);
}
catch (Exception ex)
{
}
}
}

Get total number of search results for paginated DirectorySearcher

I am using a DirectorySearch to search for objects in an Active Directory. I set the VirtualListView property to return just a few values:
using System;
using System.DirectoryServices;
namespace Testbed
{
internal class Testbed
{
private static void Main()
{
DoSearch();
Console.ReadKey();
}
private static void DoSearch()
{
var entry = new DirectoryEntry("LDAP://server/DC=mydomain,DC=com", #"USERNAME", "PASSWORD");
var searcher = new DirectorySearcher(entry)
{
PageSize = 0,
SearchScope = SearchScope.Subtree,
Filter = "(Description=J_*)",
Sort = new SortOption("Description", SortDirection.Ascending),
VirtualListView = new DirectoryVirtualListView(0, 9, 1)
};
SearchResultCollection results = searcher.FindAll();
foreach (SearchResult result in results)
{
Console.WriteLine(result.Properties["Description"][0]);
}
Console.WriteLine("Found: " + results.Count);
}
}
}
Is there a way to get the total number of objects in the AD for the query without doing an explicit DirectorySearch-"call" with an according PageSize (e.g. 500)?

There is an ApproximateTotal property inside DirectoryVirtualListView class. You can use it, but remember you should access it after the foreach block in your code.
Just change this line:
Console.WriteLine("Found: " + results.Count);
to this one:
Console.WriteLine("Found: " + searcher.VirtualListView.ApproximateTotal);
All done!

I've ran into a situation where VirtualListView.ApproximateTotal returns zero.
I've only encountered this running in netcoreapp3.1 however.
In the event you do get zero records returned, I discovered a hack to retrieve the value
Credit https://github.com/dotnet/runtime/issues/47169
using var searcher = new DirectorySearcher(rootEntry)
{
SearchScope = SearchScope.Subtree,
Filter = query,
Sort = new SortOption(criteria.Sort, sortDirection),
VirtualListView = new DirectoryVirtualListView(0, pageSize - 1, end),
PageSize = 0
};
var searchResults = searcher.FindAll();
foreach (SearchResult searchResult in searchResults)
{
//do something with entity here ...
}
//hack to properly get totals
var virtualListView = searchResults.GetType().GetProperty("VLVResponse", System.Reflection.BindingFlags.Instance | System.Reflection.BindingFlags.NonPublic)?.GetValue(searchResults) as DirectoryVirtualListView;
var count = virtualListView?.ApproximateTotal;
//Instead of simply
//var count = searchResults.Count;

Effective pagination with Active Directory searches

What would be an effective way to do pagination with Active Directory searches in .NET? There are many ways to search in AD but so far I couldn't find how to do it effectively. I want to be able to indicate Skip and Take parameters and be able to retrieve the total number of records matching my search criteria in the result.
I have tried searching with the PrincipalSearcher class:
using (var ctx = new PrincipalContext(ContextType.Domain, "FABRIKAM", "DC=fabrikam,DC=com"))
using (var criteria = new UserPrincipal(ctx))
{
criteria.SamAccountName = "*foo*";
using (var searcher = new PrincipalSearcher(criteria))
{
((DirectorySearcher)searcher.GetUnderlyingSearcher()).SizeLimit = 3;
var results = searcher.FindAll();
foreach (var found in results)
{
Console.WriteLine(found.Name);
}
}
}
Here I was able to limit the search results to 3 but I wasn't able to get the total number of records corresponding to my search criteria (SamAccountName contains foo) neither I was able to indicate to the searcher to skip the first 50 records for example.
I also tried using the System.DirectoryServices.DirectoryEntry and System.DirectoryServices.Protocols.SearchRequest but the only thing I can do is specify the page size.
So is the only way to fetch all the results on the client and do the Skip and Count there? I really hope that there are more effective ways to achieve this directly on the domain controller.

You may try the virtual list view search. The following sort the users by cn, and then get 51 users starting from the 100th one.
DirectoryEntry rootEntry = new DirectoryEntry("LDAP://domain.com/dc=domain,dc=com", "user", "pwd");
DirectorySearcher searcher = new DirectorySearcher(rootEntry);
searcher.SearchScope = SearchScope.Subtree;
searcher.Filter = "(&(objectCategory=person)(objectClass=user))";
searcher.Sort = new SortOption("cn", SortDirection.Ascending);
searcher.VirtualListView = new DirectoryVirtualListView(0, 50, 100);
foreach (SearchResult result in searcher.FindAll())
{
Console.WriteLine(result.Path);
}
For your use case you only need the BeforeCount, AfterCount and the Offset properties of DirectoryVirtualListView (the 3 in DirectoryVirtualListView ctor). The doc for DirectoryVirtualListView is very limited. You may need to do some experiments on how it behave.

If SizeLimit is set to zero and PageSize is set to 500, the search will return all 12,000 results in pages of 500 items, with the last page containing only 200 items. The paging occurs transparently to the application and the application does not have to perform any special processing other than setting the PageSize property to the proper value.
SizeLimit limits the amount of results that you can retrieve at once - so your PageSize needs to be less than or equal to 1000 (Active Directory limits the maximum number of search results to 1000. In this case, setting the SizeLimit property to a value greater than 1000 has no effect.). The paging is done automagically behind the scenes when you call FindAll() etc.
For more details please refer MSDN
https://msdn.microsoft.com/en-us/library/ms180880.aspx
https://msdn.microsoft.com/en-us/library/system.directoryservices.directorysearcher.pagesize.aspx
https://msdn.microsoft.com/en-us/library/system.directoryservices.directorysearcher.sizelimit.aspx

Waaaay late to the party, but this is what I'm doing:
I'm using FindOne() instead of FindAll() and member;range=<start>-<end> on PropertiesToLoad.
There's a catch on member;range: when it's the last page, even if you pass member;range=1000-1999 (for instance), it returns member;range=1000-*, so you have to check for the * at the end to know if there is more data.
public void List<string> PagedSearch()
{
var list = new List<string>();
bool lastPage = false;
int start = 0, end = 0, step = 1000;
var rootEntry = new DirectoryEntry("LDAP://domain.com/dc=domain,dc=com", "user", "pwd");
var filter = "(&(objectCategory=person)(objectClass=user)(samAccountName=*foo*))";
using (var memberSearcher = new DirectorySearcher(rootEntry, filter, null, SearchScope.Base))
{
while (!lastPage)
{
start = end;
end = start + step - 1;
memberSearcher.PropertiesToLoad.Clear();
memberSearcher.PropertiesToLoad.Add(string.Format("member;range={0}-{1}", start, end));
var memberResult = memberSearcher.FindOne();
var membersProperty = memberResult.Properties.PropertyNames.Cast<string>().FirstOrDefault(p => p.StartsWith("member;range="));
if (membersProperty != null)
{
lastPage = membersProperty.EndsWith("-*");
list.AddRange(memberResult.Properties[membersProperty].Cast<string>());
end = list.Count;
}
else
{
lastPage = true;
}
}
}
return list;
}

private static DirectoryEntry forestlocal = new DirectoryEntry(LocalGCUri, LocalGCUsername, LocalGCPassword);
private DirectorySearcher localSearcher = new DirectorySearcher(forestlocal);
public List<string> GetAllUsers()
{
List<string> users = new List<string>();
localSearcher.SizeLimit = 10000;
localSearcher.PageSize = 250;
string localFilter = string.Format(#"(&(objectClass=user)(objectCategory=person)(!(objectClass=contact))(msRTCSIP-PrimaryUserAddress=*))");
localSearcher.Filter = localFilter;
SearchResultCollection localForestResult;
try
{
localForestResult = localSearcher.FindAll();
if (resourceForestResult != null)
{
foreach (SearchResult result in localForestResult)
{
if (result.Properties.Contains("mail"))
users.Add((string)result.Properties["mail"][0]);
}
}
}
catch (Exception ex)
{
}
return users;
}