I'm using RabbitMq to process messages I receive on a bus. I was wondering if there's a better way to process the message I receive (maybe using async/await pattern)
Here's a snippet of my code
connection = connectionFactory.CreateConnection();
channel = connection.CreateModel();
channel.QueueDeclare(queue: Constants.RabbitListeningQueue,durable: false,exclusive: false,autoDelete: false,arguments: null);
channel.QueueDeclare(queue: Constants.RabbitMqRequestInsertedQueue,durable: false,exclusive: false,autoDelete: false,arguments: null);
var consumer = new EventingBasicConsumer(channel);
consumer.Received += (model, ea) =>
{
log.Debug($"[x] Received message :{ea}");
var body = ea.Body;
var message = Encoding.UTF8.GetString(body);
var dynamicObject = JObject.Parse(message);
queueMessageHandler.HandleMessage(dynamicObject);
};
The queueMessageHandler implementation is as follows
public class QueueMessageHandler : IQueueMessageHandler
{
private readonly IImportNucleoManager importNucleoManager;
public QueueMessageHandler(IImportNucleoManager importNucleoManager)
{
this.importNucleoManager = importNucleoManager;
}
public void HandleMessage(dynamic message)
{
switch ((string)message.Type)
{
case "T1":
{
importNucleoManager.Process(message);
break;
}
case "T3":
importNucleoManager.ProceedToInsertStep(message);
break;
}
}
}
I was wondering (since the T1/T3 events take a long time to process) should they be Task and so even the HandleMessage should be HandleMessageAsync? In this case, I also have to pass an async void which is not a best practice as I know
static async Task Main(string[] args)
{
var connectionFactory = new ConnectionFactory(DispatchConsumersAsync = true);
var connection = connectionFactory.CreateConnection();
var channel = connection.CreateModel();
var consumer = new AsyncEventingBasicConsumer(channel);
consumer.Received += Consumer_Received;
}
static async Task Consumer_Received(object sender, BasicDeliverEventArgs #event)
{
await DoSomethingAsync();
}
Related
I'm writing a simple producer/consumer application, but I'm noticing a really strange behaviour..This is the code:
private Thread _timelineThread = null;
private BufferBlock<RtpPacket> _queue = null;
private AutoResetEvent _stopping = new AutoResetEvent(false);
static void Main(string[] args)
{
// Start consumer thread
Consume();
// Produce
var t = new Thread(() =>
{
while (true)
{
var packet = RtpPacket.GetNext();
_queue.Post(packet);
Thread.Sleep(70);
}
}
t.Join();
}
static void Consume()
{
_timelineThread = new Thread(async () =>
{
while (_stopping.WaitOne(0) == false)
{
// Start consuming...
while (await _queue.OutputAvailableAsync())
{
var packet = await _queue.ReceiveAsync();
// Some processing...
}
}
});
_timelineThread.Start();
}
This is intended to be an infinite loop (until I route the _stopping signal). But, when _timelineThread hits the first await _queue.OutputAvailableAsync(), the thread changes state to 'Stopped'. There is something wrong that I'm not considering ?
If I change the Consume() function to this:
static void Consume()
{
_timelineThread = new Thread(() =>
{
while (_stopping.WaitOne(0) == false)
{
// Start consuming...
while (_queue.OutputAvailableAsync().GetAwaiter().GetResult())
{
var packet = _queue.ReceiveAsync().GetAwaiter().GetResult();
// Some processing...
}
}
});
_timelineThread.Start();
}
the thread runs without any problem..but the code is almost identical to the previous one..
EDIT: after one hour also this 'hack' doesn't seems to work..thread is 'Running' but I don't receive any data from the queue..
The Thread constructor does not understand async delegates. You can read about this here:
Is it OK to use "async" with a ThreadStart method?
Async thread body loop, It just works, but how?
My suggestion is to use a synchronous BlockingCollection<RtpPacket> instead of the BufferBlock<RtpPacket>, and consume it by enumerating the GetConsumingEnumerable method:
var _queue = new BlockingCollection<RtpPacket>();
var producer = new Thread(() =>
{
while (true)
{
var packet = RtpPacket.GetNext();
if (packet == null) { _queue.CompleteAdding(); break; }
_queue.Add(packet);
Thread.Sleep(70);
}
});
var consumer = new Thread(() =>
{
foreach (var packet in _queue.GetConsumingEnumerable())
{
// Some processing...
}
});
producer.Start();
consumer.Start();
producer.Join();
consumer.Join();
I have the following function that returns the standard output data, as an async stream, that results from running a System.Diagnostics.Process. Everything currently in the method works as intended; I can call it in an await foreach() loop and I get each line of output as its generated by the external exe.
private static async IAsyncEnumerable<string> ProcessAsyncStream (
ProcessStartInfo processStartInfo)
{
// Ensure that process is destroyed when this method exits
using var process = new Process() { StartInfo = processStartInfo };
// Buffer used to pass data from event-handler back to this method
BufferBlock<string> dataBuffer = new BufferBlock<string>();
process.OutputDataReceived += (s, e) =>
{
if (e.Data is null)
{
dataBuffer.Complete();
}
else
{
dataBuffer.Post(e.Data);
}
};
// Start process and redirect output streams
process.Start();
process.BeginOutputReadLine();
// Return data line by line
while (await dataBuffer.OutputAvailableAsync())
yield return dataBuffer.Receive();
}
My problem is that now I need it to return both the standard output and standard error results. I made this class to hold the data from each stream.
public class ProcessData
{
public string Error { get; set; } = "";
public string Output { get; set; } = "";
}
and changed ProcessAsyncStream() to look like this
private static async IAsyncEnumerable<ProcessData> ProcessAsyncStream (
ProcessStartInfo processStartInfo)
{
// Ensure that process is destroyed when this method exits
using var process = new Process() { StartInfo = processStartInfo };
// Buffer used to pass data from event-handlers back to this method
BufferBlock<string> outputDataBuffer = new BufferBlock<string>();
BufferBlock<string> errorDataBuffer = new BufferBlock<string>();
process.OutputDataReceived += (s, e) =>
{
if (e.Data is null)
{
outputDataBuffer.Complete();
}
else
{
outputDataBuffer.Post(e.Data);
}
};
process.ErrorDataReceived += (s, e) =>
{
if (e.Data is null)
{
errorDataBuffer.Complete();
}
else
{
errorDataBuffer.Post(e.Data);
}
};
// Start process and redirect output streams
process.Start();
process.BeginOutputReadLine();
// Return data line by line
while (await outputDataBuffer.OutputAvailableAsync()
|| await errorDataBuffer.OutputAvailableAsync())
yield return new ProcessData()
{
Error = errorDataBuffer.Receive(),
Output = outputDataBuffer.Receive()
}
}
The problem is that if either buffer completes before the other than the method hangs up because that buffer's .Receive() doesn't have any data to receive. If I change the while condition to && then I won't get all the data from the other buffer.
Any suggestions?
Regarding the actual problem, there is an issue with the process flow of reading the blocks. The easiest solution is to just use a single buffer with multiple producers and a single consumer combined with a message packet.
The conceptual issue that you are trying to solve with the DataFlow blocks is in the fundamental nature of events an async streams. Events are pushed, and async streams are pulled.
There are several solutions that would map them together, though I think the most elegant would be just to use an Unbounded Channel as the buffer.
Channels are more modern approach than DataFlow, have less degrees of freedom, less clunky then a BufferBlock, and very lightweight and highly optimized. Additionally, I would just pass a wrapper for the different response types.
Disregarding any other problem (conceptual or otherwise).
Given
public enum MessageType
{
Output,
Error
}
public class Message
{
public MessageType MessageType { get; set; }
public string Data { get; set; }
public Message(string data, MessageType messageType )
{
Data = data;
MessageType = messageType;
}
}
Usage
private async IAsyncEnumerable<Message> ProcessAsyncStreamAsync(
ProcessStartInfo processStartInfo,
CancellationToken cancellationToken)
{
using var process = new Process() { StartInfo = processStartInfo };
var ch = Channel.CreateUnbounded<Message>();
var completeCount = 0;
void OnReceived(string data, MessageType type)
{
// The Interlocked memory barrier is likely overkill here
if (data is null && Interlocked.Increment(ref completeCount) == 2)
ch?.Writer.Complete();
else
ch?.Writer.WriteAsync(new Message(data, type), cancellationToken);
}
process.OutputDataReceived += (_, args) => OnReceived(args.Data, MessageType.Output);
process.ErrorDataReceived += (_, args) => OnReceived(args.Data, MessageType.Error);
// start the process
// ...
await foreach (var message in ch.Reader
.ReadAllAsync(cancellationToken)
.ConfigureAwait(false))
yield return message;
// cleanup
// ...
}
Note : completely untested
Complete on exit instead.
void HandleData(object sender, DataReceivedEventArgs e)
{
if (e.Data != null) dataBuffer.Post(e.Data);
}
process.OutputDataReceived += HandleData;
process.ErrorDataReceived += HandleData;
process.Exited += (s,e) =>
{
process.WaitForExit();
dataBuffer.Complete();
};
You could use a single buffer of ProcessData items:
var buffer = new BufferBlock<ProcessData>();
Then use a custom Complete mechanism to complete the buffer when both events have propagated a null value:
process.OutputDataReceived += (s, e) =>
{
if (e.Data is null) Complete(1);
else buffer.Post(new ProcessData() { Output = e.Data });
};
process.ErrorDataReceived += (s, e) =>
{
if (e.Data is null) Complete(2);
else buffer.Post(new ProcessData() { Error = e.Data });
};
Here is an implementation of the Complete method:
bool[] completeState = new bool[2];
void Complete(int index)
{
bool completed;
lock (completeState.SyncRoot)
{
completeState[index - 1] = true;
completed = completeState.All(v => v);
}
if (completed) buffer.Complete();
}
I'm trying to port my code from an obsolete library called CastleMQ to NetMQ but I'm running into some problems.
I prefer to using polling with a timeout, for reliability - I just found that it worked best for me from trial and error compared to just sitting blocking the port indefinitely.
here is my CastleMQ code
public int ZeroPort;
private void ThreadProc()
{
var ctx = new Context();
try {
using (var repSocket = ctx.CreateSocket(SocketType.Rep))
{
string bindAddress = "tcp://*:"+ZeroPort;
repSocket.Bind(bindAddress);
print2("*** BINDING on {0} ***", bindAddress);
bool quit = false;
while (!quit) {
try {
var polling = new Polling(PollingEvents.RecvReady, repSocket);
polling.RecvReady += (socket) =>
{ // using socket.Recv() here is guaranted to return stuff
var msg = socket.Recv();
var msgStr = Encoding.UTF8.GetString(msg);
print2("[REP:{0}] {1}", bindAddress, msgStr);
switch (msgStr) {
case "positions": {
StringBuilder csv = new StringBuilder();
print2("csv: {0}", csv.ToString());
socket.Send(csv.ToString());
break;
}
default: {
socket.Send("Unrecognized Command: " + msgStr);
break;
}
}
};
polling.Poll(POLL_TIMEOUT_MS); // this returns once some socket event happens
} catch (Exception e) {
if (e is ThreadAbortException) {
quit = true;
print2("\n*** EXITED ***");
} else print2(e.ToString());
}
}
}
} catch (Exception e) {
print2(e.ToString());
} finally {
ctx.Dispose();
}
}
here is what I tried to do and then got lost with NetMQ
private void ThreadProc()
{
try {
string bindAddress = "#tcp://*:" + ZeroPort;
print2("*** BINDING on {0} ***", bindAddress);
using (var repSocket = new ResponseSocket(bindAddress))
using (var poller = new NetMQPoller { repSocket })
{
// bool quit = false;
// while (!quit)
// these event will be raised by the Poller
repSocket.ReceiveReady += (s, a) =>
{
// receive won't block as a message is ready
string msg = a.Socket.ReceiveString(); // defeinition for ReceiveString() can't be found
// send a response
a.Socket.Send("Response"); // it doesn't like "Response", do I need to wrap it in some object?
I'm especially confused as how to add a timeout so I can poll with a timeout in a loop the way my CastleMQ code does.
Any pointers would be much appreciated, thanks
I have a hardware device connected to my PC over serial port, when i signal something to it, it signals back that "Hi, here I am!" and then it is successfully detected the port on which it is connected. I wrote a Task in C# that waits for the response coming from the hardware device, but if it is not connected, then this task is waiting forever.. Where can i throw an exception to prevent this endless waiting?
My code:
public static Task<string> GetDevicePortName()
{
// Get all available serial ports on system.
var ports = SerialPort.GetPortNames();
var serialPort = new SerialPort();
serialPort.BaudRate = Constants.DeviceConstants.BaudRate;
serialPort.Parity = Constants.DeviceConstants.SerialPortParity;
serialPort.StopBits = Constants.DeviceConstants.SerialPortStopBits;
serialPort.WriteTimeout = Constants.DeviceConstants.WriteTimeoutInMilliseconds;
var taskCompletionSource = new TaskCompletionSource<string>();
serialPort.DataReceived += (s, e) =>
{
var dataIn = (byte)serialPort.ReadByte();
var receivedCharacter = Convert.ToChar(dataIn);
if (receivedCharacter == Constants.DeviceConstants.SignalYes)
{
serialPort.Dispose();
taskCompletionSource.SetResult(serialPort.PortName);
}
};
foreach (var port in ports)
{
serialPort.PortName = port;
try
{
serialPort.Open();
serialPort.Write(Constants.DeviceConstants.SignalDeviceDetect);
}
catch (IOException e) { }
}
return taskCompletionSource.Task;
}
You can create a "Custom timeout" combining Task.WhenAny with Task.Delay:
public async Task GetDevicePortNameAsync()
{
var cts = new CancellationTokenSource();
var timeOutTask = Task.Delay(5000, cts.Token);
var deviceNameTask = GetDevicePortName(cts.Token);
var finishedTask = await Task.WhenAny(timeOut, deviceNameTask);
if (finishedTask == timeOutTask)
{
// You've timed-out
}
// If you get here, the deviceName is available.
}
Note this won't cancel the underlying registration to SerialPort.
Edit:
#KDecker adds an idea to pass a CancellationToken which can be monitored in case we've already timed out before returning the TaskCompletionSource.Task. It would look like this:
public static Task<string> GetDevicePortName(CancellationToken cancellationToken)
{
// Get all available serial ports on system.
var ports = SerialPort.GetPortNames();
var serialPort = new SerialPort();
serialPort.BaudRate = Constants.DeviceConstants.BaudRate;
serialPort.Parity = Constants.DeviceConstants.SerialPortParity;
serialPort.StopBits = Constants.DeviceConstants.SerialPortStopBits;
serialPort.WriteTimeout = Constants.DeviceConstants.WriteTimeoutInMilliseconds;
var taskCompletionSource = new TaskCompletionSource<string>();
serialPort.DataReceived += (s, e) =>
{
var dataIn = (byte)serialPort.ReadByte();
var receivedCharacter = Convert.ToChar(dataIn);
if (receivedCharacter == Constants.DeviceConstants.SignalYes)
{
serialPort.Dispose();
taskCompletionSource.SetResult(serialPort.PortName);
}
};
foreach (var port in ports)
{
if (cancellationToken.IsCancellationRequested)
{
// Unregister from serialPort, and clean up whatever needs to be cleaned
taskCompletionSource.SetResult(null);
break;
}
serialPort.PortName = port;
try
{
serialPort.Open();
serialPort.Write(Constants.DeviceConstants.SignalDeviceDetect);
}
catch (IOException e) { }
finally
{
serialPort.Dispose();
}
}
return taskCompletionSource.Task;
}
Hello guys I'm pretty new to the whole async stuff and it would be nice if you could give me some advice. I'm not really sure if my approach is OK.
Lets say I have a bus device that is reading data and it fires an event if a telegram is completed. Now I want to check each telegram for its length. If length == expectation -> OK if not try until is OK or timeout. But it want to check for length 1, 2 and 5 at the same time.
UPDATE:
OK I changed my example to an async approach but I still can't figure out how this should help me with my problem? OK on the plus side I don't have threads anymore that are blocked most of the time, but this wasn't my problem :(
So I try to explain in a different way. I want a async method that listens on the bus and returns the telegram that match the defined length
async Task<byte[]> GetTelegramAsync(int length, Timespan timeout)
I want to do something like this
Task<byte[]> t1 = GetTelegramAsync(1);
Task<byte[]> t2 = GetTelegramAsync(6);
Task<byte[]> t4 = GetTelegramAsync(4);
Task t4 = DoOtherStuffAsync();
DoStuff();
Task.WaitAll(AsyncRsp(t1), AsyncRsp(t2), AsyncRsp(t3), t4);
/* Output
Get telegram with length of 1
Get telegram with length of 6
Get telegram with length of 4
Start doing other async stuff
Sync stuff done...
Telegram found 0x00 0x01 0x02 0x03 0x04 0x05
Async stuff done...
Telegram found 0xFF
Telegram with length 4 not found
*/
Here is my first BusDevice class. A thread starts that listens on the bus, if a telegram is received an event fires.
class BusDeviceThread
{
private readonly Random _r = new Random();
private Thread _t;
public event EventHandler<TelegramReceivedArgs> TelegramReceived;
public void Connect()
{
_t = new Thread(FetchBusData)
{
Name = "FetchBusData",
Priority = ThreadPriority.Normal
};
_t.Start();
}
public void Close()
{
_t.Abort();
_t.Join();
}
private void FetchBusData()
{
while (true)
{
Thread.Sleep(_r.Next(100, 1000));
var buffer = new byte[_r.Next(1, 10)];
_r.NextBytes(buffer);
OnTelegramReceived(new TelegramReceivedArgs(buffer));
}
}
private void OnTelegramReceived(TelegramReceivedArgs e)
{
var handler = TelegramReceived;
if (handler != null) handler(this, e);
}
}
And here is the changed BusDevice class utilizing async await.
class BusDeviceAsync
{
private readonly Random _r = new Random();
public event EventHandler<TelegramReceivedArgs> TelegramReceived;
public async Task Connect(CancellationToken token)
{
while (!token.IsCancellationRequested)
{
var telegram = await FetchBusData();
OnTelegramReceived(new TelegramReceivedArgs(telegram.ToArray()));
}
}
private async Task<IEnumerable<byte>> FetchBusData()
{
await Task.Delay(_r.Next(100, 1000));
var buffer = new byte[_r.Next(1, 10)];
_r.NextBytes(buffer);
return buffer;
}
private void OnTelegramReceived(TelegramReceivedArgs e)
{
var handler = TelegramReceived;
if (handler != null) handler(this, e);
}
}
Like I said it doesn't help me with my problem, the
async Task<byte[]> GetTelegramAsync(int length, Timespan timeout)
implementation stays the same or do I miss a point here?
byte[] GetTelegram(int length, TimeSpan timeout)
{
byte[] telegram = null;
using (var resetEvent = new AutoResetEvent(false))
{
EventHandler<TelegramReceivedArgs> handler = (sender, e) =>
{
var t = e.Telegram;
if (Check(t, length))
{
telegram = t;
resetEvent.Set();
}
};
_d.TelegramReceived += handler;
resetEvent.WaitOne(timeout.Milliseconds);
_d.TelegramReceived -= handler;
}
return telegram ?? new byte[0];
}
async Task<byte[]> GetTelegramAsync(int length, TimeSpan timeout)
{
return await Task.Run(() => GetTelegram(length, timeout));
}
I updated my example, but I can't figure out the difference regarding
my problem. Well I certainly have fixed the blocked thread "problem".
This is not exactly what I meant, you're still using the pull model for your data (now with help of Task.Delay), not the push model (where the notification is coming asynchronously from the bus driver, as shown here).
Anyway, I think the following implementation might be what you're looking for. Note it doesn't explicitly use threads at all, beside for async I/O bus read simulation. Substitute the real device APM API for readBus:
using System;
using System.Threading;
using System.Threading.Tasks;
namespace ConsoleApplication
{
class Program
{
public class TelegramEventArg: EventArgs
{
public byte[] Data { get; set; }
}
public EventHandler<TelegramEventArg> TelegramEvent = delegate { };
async Task<byte[]> ReadTelegramAsync(int size, CancellationToken token)
{
var tcs = new TaskCompletionSource<byte[]>();
EventHandler<TelegramEventArg> handler = null;
bool subscribed = false;
handler = (s, e) =>
{
if (e.Data.Length == size)
{
this.TelegramEvent -= handler;
subscribed = false;
tcs.TrySetResult(e.Data);
}
};
this.TelegramEvent += handler;
try
{
subscribed = true;
using (token.Register(() => tcs.TrySetCanceled()))
{
await tcs.Task.ConfigureAwait(false);
return tcs.Task.Result;
}
}
finally
{
if (subscribed)
this.TelegramEvent -= handler;
}
}
async Task ReadBusAsync(CancellationToken token)
{
while (true)
{
// get data from the bus
var data = await Task.Factory.FromAsync(
(asyncCallback, asyncState) =>
readBus.BeginInvoke(asyncCallback, asyncState),
(asyncResult) =>
readBus.EndInvoke(asyncResult),
state: null).ConfigureAwait(false);
token.ThrowIfCancellationRequested();
this.TelegramEvent(this, new TelegramEventArg { Data = data });
}
}
// simulate the async bus driver with BeginXXX/EndXXX APM API
static readonly Func<byte[]> readBus = () =>
{
var random = new Random(Environment.TickCount);
Thread.Sleep(random.Next(1, 500));
var data = new byte[random.Next(1, 5)];
Console.WriteLine("A bus message of {0} bytes", data.Length);
return data;
};
static void Main(string[] args)
{
try
{
var program = new Program();
var cts = new CancellationTokenSource(Timeout.Infinite); // cancel in 10s
var task1 = program.ReadTelegramAsync(1, cts.Token);
var task2 = program.ReadTelegramAsync(2, cts.Token);
var task3 = program.ReadTelegramAsync(3, cts.Token);
var busTask = program.ReadBusAsync(cts.Token);
Task.WaitAll(task1, task2, task3);
Console.WriteLine("All telegrams received");
cts.Cancel(); // stop ReadBusAsync
}
catch (Exception ex)
{
while (ex is AggregateException)
ex = ex.InnerException;
Console.WriteLine(ex.Message);
}
Console.ReadLine();
}
}
}
Also, this scenario appears to be an ideal candidate for implementation using Reactive Extensions (Rx). As time allows, I'll show how to do that.