How to emit a cartesian product in TPL/Dataflow? - c#

I am trying to implement the following behaviour:
[TestMethod]
public async Task ProducesCartesianProductOfInputs()
{
var block = new CartesianProductBlock<int, string>();
var target = new BufferBlock<Tuple<int, string>>();
var left = block.Left;
var right = block.Right;
block.LinkTo(target);
var actual = new List<Tuple<int, string>>();
Assert.IsTrue(left.Post(1));
Assert.IsTrue(right.Post("a"));
Assert.IsTrue(left.Post(2));
Assert.IsTrue(right.Post("b"));
// PROBLEM?: These can run before messages have been processed and appear to abort further processing.
left.Complete();
right.Complete();
while (await target.OutputAvailableAsync())
{
actual.Add(target.Receive());
}
var expected = new List<Tuple<int, string>>()
{
Tuple.Create(1, "a"),
Tuple.Create(2, "a"),
Tuple.Create(1, "b"),
Tuple.Create(2, "b"),
};
CollectionAssert.AreEquivalent(expected, actual.ToList());
}
My current (partial) implementation does not work and I can't figure out why:
// A block that remembers every message it receives on two channels, and pairs every message on a channel to every message on the other channel
public class CartesianProductBlock<T1, T2> : ISourceBlock<Tuple<T1, T2>>
{
private TransformManyBlock<T1, Tuple<T1, T2>> left;
private TransformManyBlock<T2, Tuple<T1, T2>> right;
private List<T1> leftReceived = new List<T1>();
private List<T2> rightReceived = new List<T2>();
private List<ITargetBlock<Tuple<T1, T2>>> targets = new List<ITargetBlock<Tuple<T1, T2>>>();
private object lockObject = new object();
public ITargetBlock<T1> Left { get { return left; } }
public ITargetBlock<T2> Right { get { return right; } }
public CartesianProductBlock()
{
left = new TransformManyBlock<T1, Tuple<T1, T2>>(l =>
{
lock (lockObject)
{
leftReceived.Add(l);
// Pair this input up with all received alternatives
return rightReceived.Select(r => Tuple.Create(l, r));
}
});
right = new TransformManyBlock<T2, Tuple<T1, T2>>(r =>
{
lock(lockObject)
{
rightReceived.Add(r);
// Pair this input up with all received alternatives
return leftReceived.Select(l => Tuple.Create(l, r));
}
});
Task.WhenAll(Left.Completion, Right.Completion).ContinueWith(_ => {
// TODO: Respect propagate completion linkOptions. Defauting to propagation for now.
foreach (var target in targets)
{
target.Complete();
}
});
}
private TaskCompletionSource<int> completion = new TaskCompletionSource<int>();
public Task Completion => completion.Task;
public void Complete() { throw new NotImplementedException(); }
public void Fault(Exception exception) { throw new NotImplementedException(); }
public IDisposable LinkTo(ITargetBlock<Tuple<T1, T2>> target, DataflowLinkOptions linkOptions)
{
left.LinkTo(target);
right.LinkTo(target);
targets.Add(target);
return null; // TODO: Return something proper to allow unlinking
}
public void ReleaseReservation(DataflowMessageHeader messageHeader, ITargetBlock<Tuple<T1, T2>> target)
{
throw new NotImplementedException();
}
public bool ReserveMessage(DataflowMessageHeader messageHeader, ITargetBlock<Tuple<T1, T2>> target)
{
throw new NotImplementedException();
}
public Tuple<T1, T2> ConsumeMessage(DataflowMessageHeader messageHeader, ITargetBlock<Tuple<T1, T2>> target, out bool messageConsumed)
{
throw new NotImplementedException();
}
}
I'm experiencing the following (probably related) issues:
It is non-deterministic. The test fails in different ways.
It appears (from adding in logging, and also since I get anywhere from 3 to 6 output messages) that the Complete call to the two inputs is causing messages to not be processed, though my understanding is that it should allow all queues to drain first. (And if this is not the case, then I don't know how to write the test correctly.)
It's quite possible my locking scheme is wrong/suboptimal, though my goal was to have something big and coarse that worked before trying to fix.
My experiments with individual TransformManyBlocks has failed to come up with interesting surprising, and I can't figure out what's different in this case.

As suspected, this was related to completeness propagation. Here is a working version, including proper link disposable and respecting link options:
// A block that remembers every message it receives on two channels, and pairs every message on a channel to every message on the other channel
public class CartesianProductBlock<T1, T2> : ISourceBlock<Tuple<T1, T2>>
{
private TransformManyBlock<T1, Tuple<T1, T2>> left;
private TransformManyBlock<T2, Tuple<T1, T2>> right;
private List<T1> leftReceived = new List<T1>();
private List<T2> rightReceived = new List<T2>();
private List<ITargetBlock<Tuple<T1, T2>>> targets = new List<ITargetBlock<Tuple<T1, T2>>>();
private object lockObject = new object();
public ITargetBlock<T1> Left { get { return left; } }
public ITargetBlock<T2> Right { get { return right; } }
public CartesianProductBlock()
{
left = new TransformManyBlock<T1, Tuple<T1, T2>>(l =>
{
lock (lockObject)
{
leftReceived.Add(l);
return rightReceived.Select(r => Tuple.Create(l, r)).ToList();
}
});
right = new TransformManyBlock<T2, Tuple<T1, T2>>(r =>
{
lock(lockObject)
{
rightReceived.Add(r);
return leftReceived.Select(l => Tuple.Create(l, r)).ToList();
}
});
Task.WhenAll(Left.Completion, Right.Completion).ContinueWith(_ => {
completion.SetResult(VoidResult.Instance);
});
}
private TaskCompletionSource<VoidResult> completion = new TaskCompletionSource<VoidResult>();
public Task Completion => completion.Task;
public void Complete() {
Left.Complete();
Right.Complete();
}
public void Fault(Exception exception) { throw new NotImplementedException(); }
public IDisposable LinkTo(ITargetBlock<Tuple<T1, T2>> target, DataflowLinkOptions linkOptions)
{
var leftLink = left.LinkTo(target);
var rightLink = right.LinkTo(target);
var link = new Link(leftLink, rightLink);
Task task = Task.FromResult(0);
if (linkOptions.PropagateCompletion)
{
task = Task.WhenAny(Task.WhenAll(Left.Completion, Right.Completion), link.Completion).ContinueWith(_ =>
{
// If the link has been disposed of, we should not longer propagate completeness.
if (!link.Completion.IsCompleted)
{
target.Complete();
}
});
}
return link;
}
public void ReleaseReservation(DataflowMessageHeader messageHeader, ITargetBlock<Tuple<T1, T2>> target)
{
throw new NotImplementedException();
}
public bool ReserveMessage(DataflowMessageHeader messageHeader, ITargetBlock<Tuple<T1, T2>> target)
{
throw new NotImplementedException();
}
public Tuple<T1, T2> ConsumeMessage(DataflowMessageHeader messageHeader, ITargetBlock<Tuple<T1, T2>> target, out bool messageConsumed)
{
throw new NotImplementedException();
}
private class Link : IDisposable
{
private IDisposable leftLink;
private IDisposable rightLink;
public Link(IDisposable leftLink, IDisposable rightLink)
{
this.leftLink = leftLink;
this.rightLink = rightLink;
}
private TaskCompletionSource<VoidResult> completionSource = new TaskCompletionSource<VoidResult>();
public Task Completion { get { return completionSource.Task; } }
public void Dispose()
{
leftLink.Dispose();
rightLink.Dispose();
completionSource.SetResult(VoidResult.Instance);
}
}
private class VoidResult
{
public static VoidResult instance = new VoidResult();
public static VoidResult Instance { get { return instance; } }
protected VoidResult() { }
}
}

Related

TPL DataFlow Queue with Postponement

I am processing a queue concurrently using an ActionBlock.
The one catch here is that when processing an item in the queue, I may want to wait until a dependency is satisfied by the processing of another item in the queue.
I think I should be able to do this with the TPL DataFlow library with linking, postponement and release of postponement but I'm not sure what constructs to use.
In pseudocode:
public class Item
{
public string Name { get; set; }
public List<string> DependsOn = new List<string>();
}
ActionBlock<Item> block = null;
var block = new ActionBlock<Item>(o => {
if (!HasActionBlockProcessedAllDependencies(o.DependsOn))
{
// enqueue a callback when ALL dependencies have been completed
}
else
{
DoWork(o);
}
},
new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = resourceProcessorOptions.MaximumProviderConcurrency
});
var items = new[]
{
new Item { Name = "Apple", DependsOn = { "Pear" } },
new Item { Name = "Pear" }
}
I am not sure if this will be helpful to you, but here is a custom DependencyTransformBlock class that knows about the dependencies between the items it receives, and processes each one only after its dependencies have been successfully processed. This custom block supports all the built-in functionality of a normal TransformBlock, except from the EnsureOrdered option.
The constructors of this class accept a Func<TInput, TKey> lambda for retrieving the key of each item, and a Func<TInput, IReadOnlyCollection<TKey>> lambda for retrieving its dependencies. The keys are expected to be unique. In case a duplicate key is found, the block will complete with failure.
In case of circular dependencies between items, the affected items will remain unprocessed. The property TInput[] Unprocessed allows to retrieve the unprocessed items after the completion of the block. An item can also remain unprocessed in case any of its dependencies is not supplied.
public class DependencyTransformBlock<TInput, TKey, TOutput> :
ITargetBlock<TInput>, ISourceBlock<TOutput>
{
private readonly ITargetBlock<TInput> _inputBlock;
private readonly IPropagatorBlock<Item, TOutput> _transformBlock;
private readonly object _locker = new object();
private readonly Dictionary<TKey, Item> _items;
private int _pendingCount = 1;
// The initial 1 represents the completion of the _inputBlock
private class Item
{
public TKey Key;
public TInput Input;
public bool HasInput;
public bool IsCompleted;
public HashSet<Item> Dependencies;
public HashSet<Item> Dependents;
public Item(TKey key) => Key = key;
}
public DependencyTransformBlock(
Func<TInput, Task<TOutput>> transform,
Func<TInput, TKey> keySelector,
Func<TInput, IReadOnlyCollection<TKey>> dependenciesSelector,
ExecutionDataflowBlockOptions dataflowBlockOptions = null,
IEqualityComparer<TKey> keyComparer = null)
{
if (transform == null)
throw new ArgumentNullException(nameof(transform));
if (keySelector == null)
throw new ArgumentNullException(nameof(keySelector));
if (dependenciesSelector == null)
throw new ArgumentNullException(nameof(dependenciesSelector));
dataflowBlockOptions =
dataflowBlockOptions ?? new ExecutionDataflowBlockOptions();
keyComparer = keyComparer ?? EqualityComparer<TKey>.Default;
_items = new Dictionary<TKey, Item>(keyComparer);
_inputBlock = new ActionBlock<TInput>(async input =>
{
var key = keySelector(input);
var dependencyKeys = dependenciesSelector(input);
bool isReadyForProcessing = true;
Item item;
lock (_locker)
{
if (!_items.TryGetValue(key, out item))
{
item = new Item(key);
_items.Add(key, item);
}
if (item.HasInput)
throw new InvalidOperationException($"Duplicate key ({key}).");
item.Input = input;
item.HasInput = true;
if (dependencyKeys != null && dependencyKeys.Count > 0)
{
item.Dependencies = new HashSet<Item>();
foreach (var dependencyKey in dependencyKeys)
{
if (!_items.TryGetValue(dependencyKey, out var dependency))
{
dependency = new Item(dependencyKey);
_items.Add(dependencyKey, dependency);
}
if (!dependency.IsCompleted)
{
item.Dependencies.Add(dependency);
if (dependency.Dependents == null)
dependency.Dependents = new HashSet<Item>();
dependency.Dependents.Add(item);
}
}
isReadyForProcessing = item.Dependencies.Count == 0;
}
if (isReadyForProcessing) _pendingCount++;
}
if (isReadyForProcessing)
{
await _transformBlock.SendAsync(item);
}
}, new ExecutionDataflowBlockOptions()
{
CancellationToken = dataflowBlockOptions.CancellationToken,
BoundedCapacity = 1
});
var middleBuffer = new BufferBlock<Item>(new DataflowBlockOptions()
{
CancellationToken = dataflowBlockOptions.CancellationToken,
BoundedCapacity = DataflowBlockOptions.Unbounded
});
_transformBlock = new TransformBlock<Item, TOutput>(async item =>
{
try
{
TInput input;
lock (_locker)
{
Debug.Assert(item.HasInput && !item.IsCompleted);
input = item.Input;
}
var result = await transform(input).ConfigureAwait(false);
lock (_locker)
{
item.IsCompleted = true;
if (item.Dependents != null)
{
foreach (var dependent in item.Dependents)
{
Debug.Assert(dependent.Dependencies != null);
var removed = dependent.Dependencies.Remove(item);
Debug.Assert(removed);
if (dependent.HasInput
&& dependent.Dependencies.Count == 0)
{
middleBuffer.Post(dependent);
_pendingCount++;
}
}
}
item.Input = default; // Cleanup
item.Dependencies = null;
item.Dependents = null;
}
return result;
}
finally
{
lock (_locker)
{
_pendingCount--;
if (_pendingCount == 0) middleBuffer.Complete();
}
}
}, dataflowBlockOptions);
middleBuffer.LinkTo(_transformBlock);
PropagateCompletion(_inputBlock, middleBuffer,
condition: () => { lock (_locker) return --_pendingCount == 0; });
PropagateCompletion(middleBuffer, _transformBlock);
PropagateFailure(_transformBlock, middleBuffer);
PropagateFailure(_transformBlock, _inputBlock);
}
// Constructor with synchronous lambda
public DependencyTransformBlock(
Func<TInput, TOutput> transform,
Func<TInput, TKey> keySelector,
Func<TInput, IReadOnlyCollection<TKey>> dependenciesSelector,
ExecutionDataflowBlockOptions dataflowBlockOptions = null,
IEqualityComparer<TKey> keyComparer = null) : this(
input => Task.FromResult(transform(input)),
keySelector, dependenciesSelector, dataflowBlockOptions, keyComparer)
{
if (transform == null) throw new ArgumentNullException(nameof(transform));
}
public TInput[] Unprocessed
{
get
{
lock (_locker) return _items.Values
.Where(item => item.HasInput && !item.IsCompleted)
.Select(item => item.Input)
.ToArray();
}
}
public Task Completion => _transformBlock.Completion;
public void Complete() => _inputBlock.Complete();
void IDataflowBlock.Fault(Exception ex) => _inputBlock.Fault(ex);
DataflowMessageStatus ITargetBlock<TInput>.OfferMessage(
DataflowMessageHeader header, TInput value, ISourceBlock<TInput> source,
bool consumeToAccept)
{
return _inputBlock.OfferMessage(header, value, source, consumeToAccept);
}
TOutput ISourceBlock<TOutput>.ConsumeMessage(DataflowMessageHeader header,
ITargetBlock<TOutput> target, out bool messageConsumed)
{
return _transformBlock.ConsumeMessage(header, target, out messageConsumed);
}
bool ISourceBlock<TOutput>.ReserveMessage(DataflowMessageHeader header,
ITargetBlock<TOutput> target)
{
return _transformBlock.ReserveMessage(header, target);
}
void ISourceBlock<TOutput>.ReleaseReservation(DataflowMessageHeader header,
ITargetBlock<TOutput> target)
{
_transformBlock.ReleaseReservation(header, target);
}
public IDisposable LinkTo(ITargetBlock<TOutput> target,
DataflowLinkOptions linkOptions)
{
return _transformBlock.LinkTo(target, linkOptions);
}
private async void PropagateCompletion(IDataflowBlock source,
IDataflowBlock target, Func<bool> condition = null)
{
try { await source.Completion.ConfigureAwait(false); } catch { }
if (source.Completion.IsFaulted)
target.Fault(source.Completion.Exception.InnerException);
else
if (condition == null || condition()) target.Complete();
}
private async void PropagateFailure(IDataflowBlock source,
IDataflowBlock target)
{
try { await source.Completion.ConfigureAwait(false); } catch { }
if (source.Completion.IsFaulted)
target.Fault(source.Completion.Exception.InnerException);
}
}
Usage example:
var block = new DependencyTransformBlock<Item, string, Item>(item =>
{
DoWork(item);
return item;
},
keySelector: item => item.Name,
dependenciesSelector: item => item.DependsOn,
new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = Environment.ProcessorCount
},
keyComparer: StringComparer.OrdinalIgnoreCase);
//...
block.LinkTo(DataflowBlock.NullTarget<Item>());
In this example the block is linked to a NullTarget in order to discard its output, so that it becomes essentially an ActionBlock equivalent.

How to mock IFindFluent interface

I am writing unit tests with mongo db and I need to test somehow methods that works with data returned from mongo. For example method
IFindFluent<Transcript, Transcript> GetTranscriptByUserId(int userId);
Should return some transcript. But instead it returns interface that has couple of props - Filter, Options, Sort, and other.
I am going to test method Paginane of Paginator<T> class
public PaginatedObject<T> Paginate(IFindFluent<T,T> items, int limit, int page)
{
if (limit == 0)
{
limit = 10;
}
int count = (int)items.Count();
var lastPage = (count / limit) + 1;
if (page <= 0)
{
page = 1;
}
if (page > lastPage)
{
page = lastPage;
}
var request = items.Skip((page - 1) * limit).Limit(limit);
var itemsToReturn = request.ToList();
var pages = new PaginatedObject<T>
{
Entries = itemsToReturn,
Limit = limit,
Total = count,
Page = page
};
return pages;
}
First param is interface IFindFluent<T,T> items. So, I should mock it to return items when I call Count, Skip and Limit. But these methods could be easily mocked.
mockIfindFluent = new Mock<IFindFluent<Transcript, Transcript>>();
mockIfindFluent.Setup(s => s.Limit(It.IsAny<int>())).Returns(mockIfindFluent.Object);
mockIfindFluent.Setup(i => i.Skip(It.IsAny<int>())).Returns(mockIfindFluent.Object);
mockIfindFluent.Setup(i => i.Count(CancellationToken.None)).Returns(3);
Real problem I have when I call ToList().
I got exception that I can not mock property that does not belong to model and so on.
Took some inspiration from this https://gist.github.com/mizrael/a061331ff5849bf03bf2 and extended implementation which worked for me. I have created a fake implementation of the IFindFluent interface and it had a dependency on the IAsyncCursor interface, so I did a fake implementation of that also and using that in return of mock method I wanted to set up. In that fake implementation, I have initialized an enumerable and returned that in a method I am using. You can still be more creative and play around whatever you want to return. So far this worked for me.
Here is a fake implementation.
public class FakeFindFluent<TEntity, TProjection> : IFindFluent<TEntity, TEntity>
{
private readonly IEnumerable<TEntity> _items;
public FakeFindFluent(IEnumerable<TEntity> items)
{
_items = items ?? Enumerable.Empty<TEntity>();
}
public FilterDefinition<TEntity> Filter { get => throw new NotImplementedException(); set => throw new NotImplementedException(); }
public FindOptions<TEntity, TEntity> Options => throw new NotImplementedException();
public IFindFluent<TEntity, TResult> As<TResult>(MongoDB.Bson.Serialization.IBsonSerializer<TResult> resultSerializer = null)
{
throw new NotImplementedException();
}
public long Count(CancellationToken cancellationToken = default)
{
throw new NotImplementedException();
}
public Task<long> CountAsync(CancellationToken cancellationToken = default)
{
throw new NotImplementedException();
}
public long CountDocuments(CancellationToken cancellationToken = default)
{
throw new NotImplementedException();
}
public Task<long> CountDocumentsAsync(CancellationToken cancellationToken = default)
{
throw new NotImplementedException();
}
public IFindFluent<TEntity, TEntity> Limit(int? limit)
{
throw new NotImplementedException();
}
public IFindFluent<TEntity, TNewProjection> Project<TNewProjection>(ProjectionDefinition<TEntity, TNewProjection> projection)
{
throw new NotImplementedException();
}
public IFindFluent<TEntity, TEntity> Skip(int? skip)
{
throw new NotImplementedException();
}
public IFindFluent<TEntity, TEntity> Sort(SortDefinition<TEntity> sort)
{
throw new NotImplementedException();
}
public IAsyncCursor<TEntity> ToCursor(CancellationToken cancellationToken = default)
{
throw new NotImplementedException();
}
public Task<IAsyncCursor<TEntity>> ToCursorAsync(CancellationToken cancellationToken = default)
{
IAsyncCursor<TEntity> cursor = new FakeAsyncCursor<TEntity>(_items);
var task = Task.FromResult(cursor);
return task;
}
}
public class FakeAsyncCursor<TEntity> : IAsyncCursor<TEntity>
{
private IEnumerable<TEntity> items;
public FakeAsyncCursor(IEnumerable<TEntity> items)
{
this.items = items;
}
public IEnumerable<TEntity> Current => items;
public void Dispose()
{
//throw new NotImplementedException();
}
public bool MoveNext(CancellationToken cancellationToken = default)
{
throw new NotImplementedException();
}
public Task<bool> MoveNextAsync(CancellationToken cancellationToken = default)
{
return Task.FromResult(false);
}
}
Here is how I set up my mock method to return what I wanted for my unit testing.
mockParticipantRepository
.Setup(x => x.FindByFilter(It.IsAny<FilterDefinition<Participant>>()))
.Returns(new FakeFindFluent<Participant, Participant>(participantsByRelation));
I hope this is helpful.

BroadcastBlock with guaranteed delivery in TPL Dataflow

I have a stream of data that I process in several different ways... so I would like to send a copy of each message I get to multiple targets so that these targets may execute in parallel... however, I need to set BoundedCapacity on my blocks because the data is streamed in way faster than my targets can handle them and there is a ton of data. Without BoundedCapacity I would quickly run out of memory.
However the problem is BroadcastBlock will drop messages if a target cannot handle it (due to the BoundedCapacity).
What I need is a BroadcastBlock that will not drop messages, but will essentially refuse additional input until it can deliver messages to each target and then is ready for more.
Is there something like this, or has anybody written a custom block that behaves in this manner?
It is fairly simple to build what you're asking using ActionBlock and SendAsync(), something like:
public static ITargetBlock<T> CreateGuaranteedBroadcastBlock<T>(
IEnumerable<ITargetBlock<T>> targets)
{
var targetsList = targets.ToList();
return new ActionBlock<T>(
async item =>
{
foreach (var target in targetsList)
{
await target.SendAsync(item);
}
}, new ExecutionDataflowBlockOptions { BoundedCapacity = 1 });
}
This is the most basic version, but extending it to support mutable list of targets, propagating completion or cloning function should be easy.
Here is a polished version of svick's idea. The GuaranteedDeliveryBroadcastBlock class below is an (almost) complete substitute of the built-in BroadcastBlock. Linking and unlinking targets at any moment is supported.
public class GuaranteedDeliveryBroadcastBlock<T> :
ITargetBlock<T>, ISourceBlock<T>, IPropagatorBlock<T, T>
{
private class Subscription
{
public readonly ITargetBlock<T> Target;
public readonly bool PropagateCompletion;
public readonly CancellationTokenSource CancellationSource;
public Subscription(ITargetBlock<T> target,
bool propagateCompletion,
CancellationTokenSource cancellationSource)
{
Target = target;
PropagateCompletion = propagateCompletion;
CancellationSource = cancellationSource;
}
}
private readonly object _locker = new object();
private readonly Func<T, T> _cloningFunction;
private readonly CancellationToken _cancellationToken;
private readonly ITargetBlock<T> _actionBlock;
private readonly List<Subscription> _subscriptions = new List<Subscription>();
private readonly Task _completion;
private CancellationTokenSource _faultCTS
= new CancellationTokenSource(); // Is nullified on completion
public GuaranteedDeliveryBroadcastBlock(Func<T, T> cloningFunction,
DataflowBlockOptions dataflowBlockOptions = null)
{
_cloningFunction = cloningFunction
?? throw new ArgumentNullException(nameof(cloningFunction));
dataflowBlockOptions ??= new DataflowBlockOptions();
_cancellationToken = dataflowBlockOptions.CancellationToken;
_actionBlock = new ActionBlock<T>(async item =>
{
Task sendAsyncToAll;
lock (_locker)
{
var allSendAsyncTasks = _subscriptions
.Select(sub => sub.Target.SendAsync(
_cloningFunction(item), sub.CancellationSource.Token));
sendAsyncToAll = Task.WhenAll(allSendAsyncTasks);
}
await sendAsyncToAll;
}, new ExecutionDataflowBlockOptions()
{
CancellationToken = dataflowBlockOptions.CancellationToken,
BoundedCapacity = dataflowBlockOptions.BoundedCapacity,
MaxMessagesPerTask = dataflowBlockOptions.MaxMessagesPerTask,
TaskScheduler = dataflowBlockOptions.TaskScheduler,
});
var afterCompletion = _actionBlock.Completion.ContinueWith(t =>
{
lock (_locker)
{
// PropagateCompletion
foreach (var subscription in _subscriptions)
{
if (subscription.PropagateCompletion)
{
if (t.IsFaulted)
subscription.Target.Fault(t.Exception);
else
subscription.Target.Complete();
}
}
// Cleanup
foreach (var subscription in _subscriptions)
{
subscription.CancellationSource.Dispose();
}
_subscriptions.Clear();
_faultCTS.Dispose();
_faultCTS = null; // Prevent future subscriptions to occur
}
}, TaskScheduler.Default);
// Ensure that any exception in the continuation will be surfaced
_completion = Task.WhenAll(_actionBlock.Completion, afterCompletion);
}
public Task Completion => _completion;
public void Complete() => _actionBlock.Complete();
void IDataflowBlock.Fault(Exception ex)
{
_actionBlock.Fault(ex);
lock (_locker) _faultCTS?.Cancel();
}
public IDisposable LinkTo(ITargetBlock<T> target,
DataflowLinkOptions linkOptions)
{
if (linkOptions.MaxMessages != DataflowBlockOptions.Unbounded)
throw new NotSupportedException();
Subscription subscription;
lock (_locker)
{
if (_faultCTS == null) return new Unlinker(null); // Has completed
var cancellationSource = CancellationTokenSource
.CreateLinkedTokenSource(_cancellationToken, _faultCTS.Token);
subscription = new Subscription(target,
linkOptions.PropagateCompletion, cancellationSource);
_subscriptions.Add(subscription);
}
return new Unlinker(() =>
{
lock (_locker)
{
// The subscription may have already been removed
if (_subscriptions.Remove(subscription))
{
subscription.CancellationSource.Cancel();
subscription.CancellationSource.Dispose();
}
}
});
}
private class Unlinker : IDisposable
{
private readonly Action _action;
public Unlinker(Action disposeAction) => _action = disposeAction;
void IDisposable.Dispose() => _action?.Invoke();
}
DataflowMessageStatus ITargetBlock<T>.OfferMessage(
DataflowMessageHeader messageHeader, T messageValue,
ISourceBlock<T> source, bool consumeToAccept)
{
return _actionBlock.OfferMessage(messageHeader, messageValue, source,
consumeToAccept);
}
T ISourceBlock<T>.ConsumeMessage(DataflowMessageHeader messageHeader,
ITargetBlock<T> target, out bool messageConsumed)
=> throw new NotSupportedException();
bool ISourceBlock<T>.ReserveMessage(DataflowMessageHeader messageHeader,
ITargetBlock<T> target)
=> throw new NotSupportedException();
void ISourceBlock<T>.ReleaseReservation(DataflowMessageHeader messageHeader,
ITargetBlock<T> target)
=> throw new NotSupportedException();
}
Missing features: the IReceivableSourceBlock<T> interface is not implemented, and linking with the MaxMessages option is not supported.
This class is thread-safe.

TPL DataFlow, link blocks with priority?

Using TPL.DataFlow blocks, is it possible to link two or more sources to a single ITargetBlock(e.g. ActionBlock) and prioritize the sources?
e.g.
BufferBlock<string> b1 = new ...
BufferBlock<string> b2 = new ...
ActionBlock<string> a = new ...
//somehow force messages in b1 to be processed before any message of b2, always
b1.LinkTo (a);
b2.LinkTo (a);
As long as there are messages in b1, I want those to be fed to "a" and once b1 is empty, b2 messages are beeing pushed into "a"
Ideas?
There is nothing like that in TPL Dataflow itself.
The simplest way I can imagine doing this by yourself would be to create a structure that encapsulates three blocks: high priority input, low priority input and output. Those blocks would be simple BufferBlocks, along with a method forwarding messages from the two inputs to the output based on priority, running in background.
The code could look like this:
public class PriorityBlock<T>
{
private readonly BufferBlock<T> highPriorityTarget;
public ITargetBlock<T> HighPriorityTarget
{
get { return highPriorityTarget; }
}
private readonly BufferBlock<T> lowPriorityTarget;
public ITargetBlock<T> LowPriorityTarget
{
get { return lowPriorityTarget; }
}
private readonly BufferBlock<T> source;
public ISourceBlock<T> Source
{
get { return source; }
}
public PriorityBlock()
{
var options = new DataflowBlockOptions { BoundedCapacity = 1 };
highPriorityTarget = new BufferBlock<T>(options);
lowPriorityTarget = new BufferBlock<T>(options);
source = new BufferBlock<T>(options);
Task.Run(() => ForwardMessages());
}
private async Task ForwardMessages()
{
while (true)
{
await Task.WhenAny(
highPriorityTarget.OutputAvailableAsync(),
lowPriorityTarget.OutputAvailableAsync());
T item;
if (highPriorityTarget.TryReceive(out item))
{
await source.SendAsync(item);
}
else if (lowPriorityTarget.TryReceive(out item))
{
await source.SendAsync(item);
}
else
{
// both input blocks must be completed
source.Complete();
return;
}
}
}
}
Usage would look like this:
b1.LinkTo(priorityBlock.HighPriorityTarget);
b2.LinkTo(priorityBlock.LowPriorityTarget);
priorityBlock.Source.LinkTo(a);
For this to work, a also has to have BoundingCapacity set to one (or at least a very low number).
The caveat with this code is that it can introduce latency of two messages (one waiting in the output block, one waiting in SendAsync()). So, if you have a long list of low priority messages and suddenly a high priority message comes in, it will be processed only after those two low-priority messages that are already waiting.
If this is a problem for you, it can be solved. But I believe it would require more complicated code, that deals with the less public parts of TPL Dataflow, like OfferMessage().
Here is an implementation of a PriorityBufferBlock<T> class, that propagates high priority items more frequently than low priority items. The constructor of this class has a priorityPrecedence parameter, that defines how many high priority items will be propagated for each low priority item. If this parameter has the value 1.0 (the smallest valid value), there is no real priority to speak of. If this parameter has the value Double.PositiveInfinity, no low priority item will ever be propagated as long as there are high priority items in the queue. If this parameter has a more normal value, like 5.0 for example, one low priority item will be propagated for every 5 high priority items.
This class maintains internally two queues, one for high and one for low priority items. The number of items stored in each queue is not taken into account, unless one of the two lists is empty, in which case all items of the other queue are freely propagated on demand. The priorityPrecedence parameter influences the behavior of the class only when both internal queues are non-empty. Otherwise, if only one queue has items, the PriorityBufferBlock<T> behaves like a normal BufferBlock<T>.
public class PriorityBufferBlock<T> : IPropagatorBlock<T, T>,
IReceivableSourceBlock<T>
{
private readonly IPropagatorBlock<T, int> _block;
private readonly Queue<T> _highQueue = new();
private readonly Queue<T> _lowQueue = new();
private readonly Predicate<T> _hasPriorityPredicate;
private readonly double _priorityPrecedence;
private double _priorityCounter = 0;
private object Locker => _highQueue;
public PriorityBufferBlock(Predicate<T> hasPriorityPredicate,
double priorityPrecedence,
DataflowBlockOptions dataflowBlockOptions = null)
{
ArgumentNullException.ThrowIfNull(hasPriorityPredicate);
if (priorityPrecedence < 1.0)
throw new ArgumentOutOfRangeException(nameof(priorityPrecedence));
_hasPriorityPredicate = hasPriorityPredicate;
_priorityPrecedence = priorityPrecedence;
dataflowBlockOptions ??= new();
_block = new TransformBlock<T, int>(item =>
{
bool hasPriority = _hasPriorityPredicate(item);
Queue<T> selectedQueue = hasPriority ? _highQueue : _lowQueue;
lock (Locker) selectedQueue.Enqueue(item);
return 0;
}, new()
{
BoundedCapacity = dataflowBlockOptions.BoundedCapacity,
CancellationToken = dataflowBlockOptions.CancellationToken,
MaxMessagesPerTask = dataflowBlockOptions.MaxMessagesPerTask
});
this.Completion = _block.Completion.ContinueWith(completion =>
{
Debug.Assert(this.Count == 0 || !completion.IsCompletedSuccessfully);
lock (Locker) { _highQueue.Clear(); _lowQueue.Clear(); }
return completion;
}, default, TaskContinuationOptions.ExecuteSynchronously |
TaskContinuationOptions.DenyChildAttach, TaskScheduler.Default).Unwrap();
}
public Task Completion { get; private init; }
public void Complete() => _block.Complete();
void IDataflowBlock.Fault(Exception exception) => _block.Fault(exception);
public int Count
{
get { lock (Locker) return _highQueue.Count + _lowQueue.Count; }
}
private Queue<T> GetSelectedQueue(bool forDequeue)
{
Debug.Assert(Monitor.IsEntered(Locker));
Queue<T> selectedQueue;
if (_highQueue.Count == 0)
selectedQueue = _lowQueue;
else if (_lowQueue.Count == 0)
selectedQueue = _highQueue;
else if (_priorityCounter + 1 > _priorityPrecedence)
selectedQueue = _lowQueue;
else
selectedQueue = _highQueue;
if (forDequeue)
{
if (_highQueue.Count == 0 || _lowQueue.Count == 0)
_priorityCounter = 0;
else if (++_priorityCounter > _priorityPrecedence)
_priorityCounter -= _priorityPrecedence + 1;
}
return selectedQueue;
}
private T Peek()
{
Debug.Assert(Monitor.IsEntered(Locker));
Debug.Assert(_highQueue.Count > 0 || _lowQueue.Count > 0);
return GetSelectedQueue(false).Peek();
}
private T Dequeue()
{
Debug.Assert(Monitor.IsEntered(Locker));
Debug.Assert(_highQueue.Count > 0 || _lowQueue.Count > 0);
return GetSelectedQueue(true).Dequeue();
}
private class TargetProxy : ITargetBlock<int>
{
private readonly PriorityBufferBlock<T> _parent;
private readonly ITargetBlock<T> _realTarget;
public TargetProxy(PriorityBufferBlock<T> parent, ITargetBlock<T> target)
{
Debug.Assert(parent is not null);
_parent = parent;
_realTarget = target ?? throw new ArgumentNullException(nameof(target));
}
public Task Completion => throw new NotSupportedException();
public void Complete() => _realTarget.Complete();
void IDataflowBlock.Fault(Exception error) => _realTarget.Fault(error);
DataflowMessageStatus ITargetBlock<int>.OfferMessage(
DataflowMessageHeader messageHeader, int messageValue,
ISourceBlock<int> source, bool consumeToAccept)
{
Debug.Assert(messageValue == 0);
if (consumeToAccept) throw new NotSupportedException();
lock (_parent.Locker)
{
T realValue = _parent.Peek();
DataflowMessageStatus response = _realTarget.OfferMessage(
messageHeader, realValue, _parent, consumeToAccept);
if (response == DataflowMessageStatus.Accepted) _parent.Dequeue();
return response;
}
}
}
public IDisposable LinkTo(ITargetBlock<T> target,
DataflowLinkOptions linkOptions)
=> _block.LinkTo(new TargetProxy(this, target), linkOptions);
DataflowMessageStatus ITargetBlock<T>.OfferMessage(
DataflowMessageHeader messageHeader, T messageValue,
ISourceBlock<T> source, bool consumeToAccept)
=> _block.OfferMessage(messageHeader,
messageValue, source, consumeToAccept);
T ISourceBlock<T>.ConsumeMessage(DataflowMessageHeader messageHeader,
ITargetBlock<T> target, out bool messageConsumed)
{
_ = _block.ConsumeMessage(messageHeader, new TargetProxy(this, target),
out messageConsumed);
if (messageConsumed) lock (Locker) return Dequeue();
return default;
}
bool ISourceBlock<T>.ReserveMessage(DataflowMessageHeader messageHeader,
ITargetBlock<T> target)
=> _block.ReserveMessage(messageHeader, new TargetProxy(this, target));
void ISourceBlock<T>.ReleaseReservation(DataflowMessageHeader messageHeader,
ITargetBlock<T> target)
=> _block.ReleaseReservation(messageHeader, new TargetProxy(this, target));
public bool TryReceive(Predicate<T> filter, out T item)
{
if (filter is not null) throw new NotSupportedException();
if (((IReceivableSourceBlock<int>)_block).TryReceive(null, out _))
{
lock (Locker) item = Dequeue(); return true;
}
item = default; return false;
}
public bool TryReceiveAll(out IList<T> items)
{
if (((IReceivableSourceBlock<int>)_block).TryReceiveAll(out IList<int> items2))
{
T[] array = new T[items2.Count];
lock (Locker)
for (int i = 0; i < array.Length; i++)
array[i] = Dequeue();
items = array; return true;
}
items = default; return false;
}
}
Usage example:
var bufferBlock = new PriorityBufferBlock<SaleOrder>(x => x.HasPriority, 2.5);
The above implementation supports all the features of the built-in BufferBlock<T>, except from the TryReceive with not-null filter. The core functionality of the block is delegated to an internal TransformBlock<T, int>, that contains a dummy zero value for every item stored in one of the queues.

Need some help for planning class model

I want to declare some functions for tests. For example:
CountWords(string text)
ExistsWordInText(string text, string searchedWord)
CountOfWord(string text, string searchedWord)
Now I want to declare "testdefintions". These testdefinitions will be added to a collection and include a function and, depending on the function, different parameters.
A method will now execute all the tests from the testdefintions collection and the tests will return a result.
I want to add functions in the future without changing anything else.
At the moment I am at this point:
IFunction-interface
public interface IFunction
{
#region Methods
void Run();
#endregion
}
Any function
public class AttributeExistenceFunction : FunctionBase
{
public override void Run() {
// Do any test
throw new FormatException();
}
}
TestDefinition-Class
public class TestDefinition : ITestDefinition
{
#region Fields
public IFunction Function { get; set; }
#endregion
#region Constructor
public TestDefinition(IFunction function)
{
this.Function = function;
}
#endregion
#region Methods
public void Run(Uri site)
{
this.Function.Run();
}
#endregion
}
Has anybody an idea how to realize the dynamic paramters / results?
I started it out optimistic, but it turned out really ugly.
I'll post it anyways since it does the job after all.
You can easily add Func constructors to support Action and lose the VoidReturn hack.
public class Func
{
public readonly MethodInfo Method;
public readonly object Target;
#region Ctors
public static Func Get<TResult>(Func<TResult> func)
{
return new Func(func.Method, func.Target);
}
public static Func Get<T, TResult>(Func<T, TResult> func)
{
return new Func(func.Method, func.Target);
}
public static Func Get<T1, T2, TResult>(Func<T1, T2, TResult> func)
{
return new Func(func.Method, func.Target);
}
public static Func Get<T1, T2, T3, TResult>(Func<T1, T2, T3, TResult> func)
{
return new Func(func.Method, func.Target);
}
#endregion
private Func(MethodInfo method, object target)
{
this.Method = method;
this.Target = target;
}
public object Run(params object[] parameters)
{
return this.Method.Invoke(this.Target, parameters);
}
}
public class MyClass
{
public string Data { get; set; }
public int Add(int x, int y)
{
return x + y;
}
public bool IsZero(int i)
{
return i == 0;
}
public void Print(object msg)
{
Console.WriteLine(msg);
}
public bool ValidateData()
{
return string.IsNullOrEmpty(this.Data);
}
public void TestMethods()
{
var tests = new Dictionary<Func, object[][]>
{
{
Func.Get<int, int, int>(this.Add),
new[]
{
new object[] {2, 3},
new object[] {5, 0},
new object[] {10, -2}
}
},
{
Func.Get<int, bool>(this.IsZero),
new[]
{
new object[] {1},
new object[] {0},
new object[] {-1}
}
},
{
Func.Get<object, VoidReturn>(o =>
{
this.Print(o);
return VoidReturn.Blank;
}),
new[]
{
new object[] {"Msg1"},
new object[] {"Msg2"},
new object[] {"Msg3"}
}
},
{Func.Get(this.ValidateData), null}
};
foreach (var testFunc in tests)
{
Console.WriteLine("Testing method: " + testFunc.Key.Method.Name);
Console.WriteLine();
foreach (var parameters in testFunc.Value)
{
Console.WriteLine("Parameters: " + string.Join(", ", parameters));
var result = testFunc.Key.Run(parameters);
Console.WriteLine(result is VoidReturn ? "*void" : ("Returned: " + result));
Console.WriteLine();
}
Console.WriteLine("________________________________");
Console.WriteLine();
}
}
private enum VoidReturn
{
Blank
}
}

Categories

Resources