Develop Reference

My .NET tcp server application will randomly use 100% CPU

I have a serious issue with my .NET server application. In production, the application will reach 100% CPU usage and get stuck there until I restart the server. It seems completely random. Sometimes it will happen 10 minutes after I start the server. Sometimes a week after I start the server. There are no logs that indicate what causes it either. But I am guessing I wrote the TCP client/server wrong and there is some edge case that can cause this. I believe this issue didn't start happening until I added this TCP client/server. I say client and server because this class does both and I actually have two different server applications that use it to communicate to each other and they both experience this issue randomly (not at the same time). Also side note user clients from all over the world use this same TCP client/server class: Bridge to connect to the server as well.
Is there anything I can do to try and figure out what's causing this? It's a .NET console app running on a Linux VM on Google Cloud Platform.
If you are knowledgable with .NET TCP classes then perhaps you can find an issue with this code?
using System;
using System.Collections.Generic;
using System.Net;
using System.Net.Sockets;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
namespace SBCommon
{
public class Bridge<T> where T : struct, IConvertible
{
public Dictionary<int, BridgeHandler> bridgeHandlers;
public ClientHandler clientHandler;
TcpListener server = null;
public Bridge(int port, Dictionary<int, BridgeHandler> bridgeHandlers)
{
server = new TcpListener(IPAddress.Any, port);
server.Start();
this.bridgeHandlers = bridgeHandlers;
Logging.Info($"bridge listener on address {IPAddress.Any} port {port}");
}
public void StartListener()
{
try
{
Logging.Info($"Starting to listen for TCP connections");
while (true)
{
Logging.Debug("TCP client ready");
TcpClient client = server.AcceptTcpClient();
Logging.Debug("TCP client connected");
Thread t = new Thread(new ParameterizedThreadStart(HandleConnection));
t.Start(client);
}
}
catch (SocketException e)
{
Logging.Error($"SocketException: {e}");
server.Stop();
}
}
public void HandleConnection(object obj)
{
TcpClient client = (TcpClient)obj;
client.ReceiveTimeout = 10000; // adding this to see if it fixes the server crashes
var stream = client.GetStream();
try
{
if (stream.CanRead)
{
byte[] messageLengthBytes = new byte[4];
int v = stream.Read(messageLengthBytes, 0, 4);
if (v != 4)
{
// this is happening and then the server runs at over 100% so adding lots of logging to figure out what's happening
StringBuilder sb = new StringBuilder($"could not read incoming message. Read {v} bytes");
try
{
sb.Append($"\nfrom {(IPEndPoint)client.Client.RemoteEndPoint}");
}
catch (Exception e)
{
sb.Append($"\ncould not get client's IP address because {e}");
}
sb.Append($"\nclient.Available: {client.Available}");
sb.Append($"\nclient.SendBufferSize: {client.SendBufferSize}");
Logging.Error(sb.ToString());
stream.Close();
client.Close();
stream.Dispose();
client.Dispose();
return;
}
int messageLength = BitConverter.ToInt32(messageLengthBytes, 0);
int readPos = 0;
byte[] recievedData = new byte[messageLength];
while (readPos < messageLength)
{
int size = Math.Min(messageLength - readPos, client.ReceiveBufferSize);
v = stream.Read(recievedData, readPos, size);
readPos += v;
}
Bits incoming = new Bits(recievedData);
incoming.InitReadableBuffer();
int packetType = incoming.ReadInt();
Bits outgoing;
if (packetType == int.MaxValue)
{
Logging.Info($"recieved client handler message");
outgoing = clientHandler(incoming, client);
}
else
{
if (bridgeHandlers.ContainsKey(packetType))
{
Logging.Info($"recieved {(T)(object)packetType}");
outgoing = bridgeHandlers[packetType](incoming);
}
else
{
Logging.Error($"recieved unhandled packetType {packetType}!!!!!");
return;
}
}
if (outgoing != null)
{
#region send response
byte[] sendData = new byte[outgoing.Length() + 4];
// first write the length of the message
BitConverter.GetBytes(outgoing.Length()).CopyTo(sendData, 0);
// then write the message
outgoing.ToArray().CopyTo(sendData, 4);
stream.Write(sendData, 0, sendData.Length);
outgoing.Dispose();
#endregion
}
else
{
byte[] sendData = new byte[4];
BitConverter.GetBytes(0).CopyTo(sendData, 0);
stream.Write(sendData, 0, sendData.Length);
}
}
else
{
Logging.Info("Sorry. You cannot read from this NetworkStream.");
}
}
catch (Exception e)
{
Logging.Error($"Exception: {e}");
stream.Close();
client.Close();
stream.Dispose();
client.Dispose();
}
}
public static void SendTCPmessageFireAndForget(IPEndPoint iPEndPoint, Bits bits)
{
Task.Run(() =>
{
SendTCPmessage(iPEndPoint, bits, out _);
bits.Dispose();
});
}
public static async Task<Bits> SendTCPmessageAsync(IPEndPoint iPEndPoint, Bits bits)
{
TcpClient client = new TcpClient();
client.Connect(iPEndPoint);
NetworkStream stream = client.GetStream();
stream.WriteTimeout = 5000;
stream.ReadTimeout = 5000;
// Send the message
byte[] bytes = new byte[bits.Length() + 4];
BitConverter.GetBytes(bits.Length()).CopyTo(bytes, 0); // write length of message
bits.ToArray().CopyTo(bytes, 4);
await stream.WriteAsync(bytes, 0, bytes.Length);
// Read the response
byte[] messageLengthBytes = new byte[4];
int v = await stream.ReadAsync(messageLengthBytes, 0, 4);
if (v != 4) throw new Exception("could not read incoming message");
int messageLength = BitConverter.ToInt32(messageLengthBytes, 0);
if (messageLength > 0)
{
int readPos = 0;
byte[] recievedData = new byte[messageLength];
while (readPos < messageLength)
{
int size = Math.Min(messageLength - readPos, client.ReceiveBufferSize);
v = await stream.ReadAsync(recievedData, readPos, size);
readPos += v;
}
stream.Close();
client.Close();
bits = new Bits(recievedData);
}
else bits = null;
return bits;
}
public static void SendTCPmessage(IPEndPoint iPEndPoint, Bits bits, out Bits responseBits)
{
try
{
TcpClient client = new TcpClient();
client.Connect(iPEndPoint);
NetworkStream stream = client.GetStream();
stream.WriteTimeout = 50000;
stream.ReadTimeout = 50000;
// Send the message
byte[] bytes = new byte[bits.Length() + 4];
BitConverter.GetBytes(bits.Length()).CopyTo(bytes, 0); // write length of message
bits.ToArray().CopyTo(bytes, 4);
stream.Write(bytes, 0, bytes.Length);
// Read the response
byte[] messageLengthBytes = new byte[4];
if (stream.Read(messageLengthBytes, 0, 4) != 4) throw new Exception("could not read incoming message");
int messageLength = BitConverter.ToInt32(messageLengthBytes, 0);
if (messageLength > 0)
{
int readPos = 0;
byte[] recievedData = new byte[messageLength];
while (readPos < messageLength)
{
int size = Math.Min(messageLength - readPos, client.ReceiveBufferSize);
int v = stream.Read(recievedData, readPos, size);
readPos += v;
}
stream.Close();
client.Close();
responseBits = new Bits(recievedData);
}
else responseBits = null;
}
catch (Exception e)
{
Logging.Error($"Exception: {e}");
responseBits = null;
}
}
}
public delegate Bits BridgeHandler(Bits incoming);
public delegate Bits ClientHandler(Bits incoming, TcpClient client);
}
Notes:
Bits is a class I use for serialization.
You can see in StartListener that I start a thread for every incoming connection
I also use while (true) and AcceptTcpClient to accept tcp connections. But maybe I shouldn't be doing it that way?
I read the first 4 bytes as an int from every packet to determine what kind of packet it is.
then I continue to to read the rest of the bytes until I have read all of it.

There is a lot wrong with your existing code, so it's hard to know what exactly is causing the issue.
Mix of async and non-async calls. Convert the whole thing to async and only use those, do not do sync-over-async.
Assuming stream.Read is actually returning the whole value in one call.
Lack of using in many places.
Repetitive code which should be refactored into functions.
Unsure what the use of bits.ToArray is and how efficient it is.
You may want to add CancellationToken to be able to cancel the operations.
Your code should look something like this:
public class Bridge<T> : IDisposable where T : struct, IConvertible
{
public Dictionary<int, BridgeHandler> bridgeHandlers;
public ClientHandler clientHandler;
TcpListener server;
public Bridge(int port, Dictionary<int, BridgeHandler> bridgeHandlers)
{
server = new TcpListener(IPAddress.Any, port);
this.bridgeHandlers = bridgeHandlers;
Logging.Info($"bridge listener on address {IPAddress.Any} port {port}");
}
public async Task StartListener()
{
try
{
Logging.Info($"Starting to listen for TCP connections");
server.Start();
while (true)
{
Logging.Debug("TCP client ready");
TcpClient client = await server.AcceptTcpClientAsync();
Logging.Debug("TCP client connected");
Task.Run(async () => await HandleConnection(client));
}
}
catch (SocketException e)
{
Logging.Error($"SocketException: {e}");
}
finally
{
if (listener.Active)
server.Stop();
}
}
public async Task HandleConnection(TcpClient client)
{
using client;
client.ReceiveTimeout = 10000; // adding this to see if it fixes the server crashes
using var stream = client.GetStream();
try
{
var incoming = await ReadMessageAsync(stream);
incoming.InitReadableBuffer();
int packetType = incoming.ReadInt();
Bits outgoing;
if (packetType == int.MaxValue)
{
Logging.Info($"recieved client handler message");
outgoing = clientHandler(incoming, client);
}
else
{
if (bridgeHandlers.TryGetValue(packetType, handler))
{
Logging.Info($"recieved {(T)(object)packetType}");
outgoing = handler(incoming);
}
else
{
Logging.Error($"recieved unhandled packetType {packetType}!!!!!");
return;
}
}
using (outgoing);
await SendMessageAsync(stream, outgoing);
}
catch (Exception e)
{
Logging.Error($"Exception: {e}");
}
}
public static void SendTCPmessageFireAndForget(IPEndPoint iPEndPoint, Bits bits)
{
Task.Run(async () =>
{
using (bits)
await SendTCPmessageAsync(iPEndPoint, bits);
});
}
public static async Task<Bits> SendTCPmessageAsync(IPEndPoint iPEndPoint, Bits bits)
{
using TcpClient client = new TcpClient();
await client.ConnectAsync(iPEndPoint);
using NetworkStream stream = client.GetStream();
stream.WriteTimeout = 5000;
stream.ReadTimeout = 5000;
await SendMessageAsync(stream, bits);
return await ReadMessageAsync(stream);
}
}
private async Task SendMessageAsync(Stream stream, Bits message)
{
var lengthArray = message == null ? new byte[4] : BitConverter.GetBytes(bits.Length());
await stream.WriteAsync(lengthArray, 0, lengthArray.Length); // write length of message
if (message == null)
return;
var bytes = bits.ToArray();
await stream.WriteAsync(bytes, 0, bytes.Length);
}
private async Task<Bits> ReadMessageAsync(Stream stream)
{
var lengthArray = new byte[4];
await FillBuffer(stream, lengthArray);
int messageLength = BitConverter.ToInt32(lengthArray, 0);
if (messageLength == 0)
return null;
byte[] receivedData = new byte[messageLength];
await FillBuffer(stream, receivedData);
bits = new Bits(receivedData);
return bits;
}
private async Task FillBuffer(Stream stream, byte[] buffer)
{
int totalRead = 0;
int bytesRead = 0;
while (totalRead < buffer.Length)
{
var bytesRead = await stream.ReadAsync(lengthArray, totalRead, buffer.Length - totalRead);
totalRead += bytesRead;
if(bytesRead <= 0)
throw new Exception("Unexpected end of stream");
}
}

Recovering from stream.Read() exception

I have a function to request an update from the server. I provide the query and also indicate the expected length of the response.
public Byte[] GetUpdate(Byte[] query, int expLength)
{
var response = new Byte[expLength];
lock(Client)
{
Stream s = Client.GetStream();
s.Write(query, 0, query.Length);
var totalBytesRead = 0;
var numAttempts = 0;
while(totalBytesRead < expLength && numAttempts < MAX_RETRIES)
{
numAttempts++;
int bytes;
try
{
bytes = s.Read(response, totalBytesRead, expLength);
}
catch (Exception ex)
{
throw new IOException();
}
totalBytesRead += bytes;
}
if(totalBytesRead < expLength)
{
// should probably throw something here
throw new IOException();
}
}
return response;
}
The function that calls the above function is given below. Both of them belong to class Connection.
public Byte[] GetData(string ip, int port ,Byte [] query, int responseLen)
{
Connection connection = GetConnection(ip,port);
Byte[] data = null;
try
{
data = connection.GetUpdate(query, responseLen);
}
catch(Exception e)
{
connection?.Disconnect();
return new Byte[0];
}
return data;
}
My question is as follows.
With the above code I am trying to read some values of a remote network endpoint. Sometimes the connections go down on the link. I test the scenario by manually unplugging the Ethernet cable. Once I unplug and plug the cable I sometimes find that s.Read() in function GetUpdate throws StreamReadFailed exception.It happens occasionally. What could be the possible reason for this and what is the best way to recover from this.?

C# / Sockets: Receiving data and constructing a message from it

I am trying to implement a system that can send and receive "Nessages" (a custom type) using async Sockets. Each Message consists of three elements:
First I send the Message-ID (4 bytes), next is the length of the data (4 bytes) and then the actual data.
The way TCP/IP works, I can not know if the full Message is in the buffer of data that I just received. I need to look for the InfoHeader (ID and length) first and use the remaining data to fill the Message's DataBuffer.
public void ReadCallback(IAsyncResult ar)
{
try
{
// How many bytes were received? They are now stored in the PacketBuffer
int bytesReceived = Socket.EndReceive(ar);
if (bytesReceived != 0)
{
// We did receive some data
if (MessageInfoHeader.Length < NetProtocol.NetInfoHeaderSize)
{
// The InfoHeader we have is not complete yet. How many bytes are missing from it?
int infoHeaderDataInCurrentBuffer = NetProtocol.NetInfoHeaderSize - (int) MessageInfoHeader.Length;
// Check if we have read enough data to fill the entire missing InfoHeader
if (infoHeaderDataInCurrentBuffer > bytesReceived)
{
// Nope, we dont. Fill as much as we can
infoHeaderDataInCurrentBuffer = bytesReceived;
Log.Info("We do not have enough data to complete the InfoHeader yet. Fill as much as we can");
}
else
{
Log.Info("We do have enough data to complete the InfoHeader");
}
// Now fill it from the chunk of data we just received
MessageInfoHeader.Write(ReceivedData, 0, infoHeaderDataInCurrentBuffer);
// See if there is any data remaining in the buffer that we can store in the MessageDataBuffer
int dataRemainingInPacket = bytesReceived - infoHeaderDataInCurrentBuffer;
Log.Info("DataRemainingInPacket: " + dataRemainingInPacket);
// Check again if we have the full header
if (MessageInfoHeader.Length == NetProtocol.NetInfoHeaderSize)
{
Log.Info("We now have assembled the full InfoHeader");
// We do have the full InfoHeader. Use the rest of the data to fill the DataBuffer
using (MemoryStream ms = new MemoryStream())
{
byte[] b = MessageInfoHeader.GetBuffer();
ms.Write(b, 0, b.Length);
ms.Seek(0, SeekOrigin.Begin);
try
{
using (BinaryReader br = new BinaryReader(ms))
{
CurrentMessageID = br.ReadInt32();
CurrentMessageDataLength = br.ReadInt32();
Log.Info("InfoHeader: MessageID=" + CurrentMessageID + " / DataLength=" + CurrentMessageDataLength);
}
}
catch (Exception ex)
{
throw ex;
}
}
if (dataRemainingInPacket > 0)
{
Log.Info("There are " + dataRemainingInPacket + " bytes remaining in the packet. Writing them to the PacketDataBuffer");
MessageDataBuffer.Write(ReceivedData, CurrentPacketBytesRead, ReceivedData.Length - CurrentPacketBytesRead);
}
}
}
else
{
MessageDataBuffer.Write(ReceivedData, 0, ReceivedData.Length);
if(ReceivedData.Length <= NetDataBufferSize)
{
Log.Info("WE HAVE RECEIVED THE ENTIRE MESSAGE");
}
}
// Start receiving the new TCP-Packet
Socket.BeginReceive(ReceivedData, 0, ReceivedData.Length, SocketFlags.None, new AsyncCallback(ReadCallback), null);
}
else
{
Log.Info("No data has been received");
}
}
catch (Exception ex)
{
throw ex;
}
}
private void AcceptCallback(IAsyncResult ar)
{
// Retrieve the listening Socket
Socket listener = (Socket)ar.AsyncState;
// Retrieve the Socket that is actually connected to the client
Socket csocket = listener.EndAccept(ar);
Log.Info("Accepted connection from: " + csocket.RemoteEndPoint.ToString());
// Create a new client with the CSocket and add it to the list
ClientConnection client = ClientManager.AddClient(csocket);
// Start reading data from the Client's Socket and then enter the "read-loop" inside the client
client.Socket.BeginReceive(client.ReceivedData, 0, (int) client.ReceivedData.Length, SocketFlags.None, new AsyncCallback(client.ReadCallback), null);
// Turn the listener back into listening mode
listener.BeginAccept(new AsyncCallback(AcceptCallback), listener);
}
public const int NetInfoHeaderSize = 8; // 2x int
public static byte[] FrameMessage(NetMessage message)
{
byte[] result;
using(MemoryStream ms = new MemoryStream())
{
using (BinaryWriter bw = new BinaryWriter(ms))
{
bw.Write(message.ID);
bw.Write(message.Data.Length);
bw.Write(message.Data);
}
result = ms.ToArray();
}
return result;
}
This piece of code obviously does not work correctly. I get the "RECEIVED ENTIRE MESSAGE" line multiple times.
Question: What exactly am I doing wrong? I'm failing to wrap my head around what is the best way to handle something like this? Thanks in advance for any input!

There could be quite a few problems. But the first one I can see for the loop on "RECEIVED ENTIRE MESSAGE" is that you are assuming the ReceivedData buffer has been filled entirely by Socket.EndReceive(ar); - which isn't the case, so a first troubleshooting start could be to rewrite this:
MessageDataBuffer.Write(ReceivedData, 0, ReceivedData.Length);
if (ReceivedData.Length <= NetDataBufferSize)
{
Log.Info("WE HAVE RECEIVED THE ENTIRE MESSAGE");
}
to:
MessageDataBuffer.Write(ReceivedData, 0, bytesReceived);
if (MessageDataBuffer.Length <= NetDataBufferSize)
{
Log.Info("WE HAVE RECEIVED THE ENTIRE MESSAGE");
}
Assuming that MessageDataBuffer is intended to contain the entire message

C# good practice waiting for TCP response

While making a c# application for remote controlling cisco routers using TCP, I got the problem of waiting for a response from the router.
For the application I have to connect to a Cisco router using a TCP connection. After the connection has been made a networkstream will push my command to the Cisco router. To let the router process the command, I am using Thread.Sleep. This is not the best solution.
Here is the complete code to get a idea what my program is doing.
string IPAddress = "192.168.1.1";
string message = "show running-config"; // command to run
int bytes;
string response = "";
byte[] responseInBytes = new byte[4096];
var client = new TcpClient();
client.ConnectAsync(IPAddress, 23).Wait(TimeSpan.FromSeconds(2));
if (client.Connected == true)
{
client.ReceiveTimeout = 3;
client.SendTimeout = 3;
byte[] messageInBytes = Encoding.ASCII.GetBytes(message);
NetworkStream stream = client.GetStream();
Console.WriteLine();
stream.Write(messageInBytes, 0, messageInBytes.Count()); //send data to router
Thread.Sleep(50); // temporary way to let the router fill his tcp response
bytes = stream.Read(responseInBytes, 0, responseInBytes.Length);
response = Encoding.ASCII.GetString(responseInBytes, 0, bytes);
return response; //whole command output
}
return null;
What is a good and reliable way to get the full response.
Thanks for any help or command.
More info:
The networksteam is always filled with something, most of the time it is filled with the cisco IOS login page. The biggest problem is to determine when the router is done filling up the response.
The response I most of the time get:
"??\u0001??\u0003??\u0018??\u001f\r\n\r\nUser Access Verification\r\n\r\nUsername: "
The return data will be diffent every time because it will be a result of a cisco command. This can vary from a short string to a very long string.
mrmathijs95 -

When reading from NetworkStream with Stream.Read it not 100% sure that you will read all expected data. Stream.Read can return when only few packet arrived and not waiting for others.
To be sure that you get all data use BinaryReader for reading.
BinaryReader.Read method will block current thread until all expected data arrived
private string GetResponse(string message)
{
const int RESPONSE_LENGTH = 4096;
byte[] messageInBytes = Encoding.ASCII.GetBytes(message);
bool leaveStreamOpen = true;
using(var writer = new BinaryWriter(client.GetStream()))
{
writer.Write(messageInBytes);
}
using(var reader = New BinaryReader(client.GetStream()))
{
byte[] bytes = reader.Read(RESPONSE_LENGTH );
return Encoding.ASCII.GetString(bytes);
}
}

Don't use Thread.Sleep. I would async/await the entire thing, given that you don't always know what the data is based on your recent edit. This is how I would do it (untested):
public class Program
{
// call Foo write with "show running-config"
}
public class Foo
{
private TcpClient _client;
private ConcurrentQueue<string> _responses;
private Task _continualRead;
private CancellationTokenSource _readCancellation;
public Foo()
{
this._responses = new ConcurrentQueue<string>();
this._readCancellation = new CancellationTokenSource();
this._continualRead = Task.Factory.StartNew(this.ContinualReadOperation, this._readCancellation.Token, this._readCancellation.Token);
}
public async Task<bool> Connect(string ip)
{
this._client = new TcpClient
{
ReceiveTimeout = 3, // probably shouldn't be 3ms.
SendTimeout = 3 // ^
};
int timeout = 1000;
return await this.AwaitTimeoutTask(this._client.ConnectAsync(ip, 23), timeout);
}
public async void StreamWrite(string message)
{
var messageBytes = Encoding.ASCII.GetBytes(message);
var stream = this._client.GetStream();
if (await this.AwaitTimeoutTask(stream.WriteAsync(messageBytes, 0, messageBytes.Length), 1000))
{
//write success
}
else
{
//write failure.
}
}
public async void ContinualReadOperation(object state)
{
var token = (CancellationToken)state;
var stream = this._client.GetStream();
var byteBuffer = new byte[4096];
while (!token.IsCancellationRequested)
{
int bytesLastRead = 0;
if (stream.DataAvailable)
{
bytesLastRead = await stream.ReadAsync(byteBuffer, 0, byteBuffer.Length, token);
}
if (bytesLastRead > 0)
{
var response = Encoding.ASCII.GetString(byteBuffer, 0, bytesLastRead);
this._responses.Enqueue(response);
}
}
}
private async Task<bool> AwaitTimeoutTask(Task task, int timeout)
{
return await Task.WhenAny(task, Task.Delay(timeout)) == task;
}
public void GetResponses()
{
//Do a TryDequeue etc... on this._responses.
}
}
I didn't expose the read cancellation publicly, but you could add this method to cancel the read operation:
public void Cancel()
{
this._readCancellation.Cancel();
}
And then dispose of your client and all that fun stuff.
Lastly, because you said there's always data available on the stream, where you're doing the read you may have to do some logic on the number of bytes last read to offset yourself within the stream if the data doesn't clear. You'll know if the responses you're getting is always the same.

This is the working code for me.
It uses the solution of Fabio,
combined with a while loop to check every X miliseconds if the response has changed.
client.ReceiveTimeout = 3;
client.SendTimeout = 3;
byte[] messageInBytes = Encoding.ASCII.GetBytes(message);
NetworkStream stream = client.GetStream();
Console.WriteLine();
using (var writer = new BinaryWriter(client.GetStream(),Encoding.ASCII,true))
{
writer.Write(messageInBytes);
}
using (var reader = new BinaryReader(client.GetStream(),Encoding.ASCII, true))
{
while (itIsTheEnd == false)
{
bytes = reader.Read(responseInBytes, 0, responseInBytes.Count());
if (lastBytesArray == responseInBytes)
{
itIsTheEnd = true;
}
lastBytesArray = responseInBytes;
Thread.Sleep(15);
}
}
response = Encoding.ASCII.GetString(responseInBytes);
Thanks for everyone who suggested a solution.
And thanks to Fabio for the given solution.

Read continous bytestream from Stream using TcpClient and Reactive Extensions

Consider the following code:
internal class Program
{
private static void Main(string[] args)
{
var client = new TcpClient();
client.ConnectAsync("localhost", 7105).Wait();
var stream = client.GetStream();
var observable = stream.ReadDataObservable().Repeat();
var s = from d in observable.Buffer(4)
let headerLength = IPAddress.NetworkToHostOrder(BitConverter.ToInt16(d.ToArray(), 2))
let b = observable.Take(headerLength)
select b.ToEnumerable().ToArray();
s.Subscribe(a => Console.WriteLine("{0}", a));
Console.ReadLine();
}
}
public static class Extensions
{
public static IObservable<byte> ReadDataObservable(this Stream stream)
{
return Observable.Defer(async () =>
{
var buffer = new byte[1024];
var readBytes = await stream.ReadAsync(buffer, 0, buffer.Length);
return buffer.Take(readBytes).ToObservable();
});
}
}
Basically I want to parse the messages I receive with Reactive Extensions. The header of the message is parsed correctly using the Buffer(4) and I get the length of the remainder of the message. The problem that arises is that when I do stream.Take(headerLength), the code reevaluates the whole "chain" and tries to get a new message from the stream instead of returning the rest of the bytes which already has been read from the stream. To be more exact, the first ReadAsync(...) returns 38 bytes, the Buffer(4) returns the first 4 of those, the observable.Take(headerLength) does not return the remainding 34 bytes but instead tries to read a new message with ReadAsync.
The question is, how can I make sure the observable.Take(headerLength) receives the already read 34 bytes and not try to read a new message from the stream? I've searched around for a solution, but I can't really figure out how to achieve this.
Edit: This solution (Using Reactive Extensions (Rx) for socket programming practical?) is not what I'm looking for. This isn't reading everything available in the stream (up to buffersize) and makes a continous bytestream out of it. To me this solution doesn't seem like a very efficient way to read from a stream, hence my question.

This approach isn't going to work. The problem is the way you are using the observable. Buffer will not read 4 bytes and quit, it will continually read 4 byte chunks. The Take forms a second subscription that will read overlapping bytes. You'll find it much easier to parse the stream directly into messages.
The following code makes a good deal of effort to clean up properly as well.
Assuming your Message is just this, (ToString added for testing):
public class Message
{
public byte[] PayLoad;
public override string ToString()
{
return Encoding.UTF8.GetString(PayLoad);
}
}
And you have acquired a Stream then you can parse it as follows. First, a method to read an exact number of bytes from a stream:
public async static Task ReadExactBytesAsync(
Stream stream, byte[] buffer, CancellationToken ct)
{
var count = buffer.Length;
var totalBytesRemaining = count;
var totalBytesRead = 0;
while (totalBytesRemaining != 0)
{
var bytesRead = await stream.ReadAsync(
buffer, totalBytesRead, totalBytesRemaining, ct);
ct.ThrowIfCancellationRequested();
totalBytesRead += bytesRead;
totalBytesRemaining -= bytesRead;
}
}
Then the conversion of a stream to IObservable<Message>:
public static IObservable<Message> ReadMessages(
Stream sourceStream,
IScheduler scheduler = null)
{
int subscribed = 0;
scheduler = scheduler ?? Scheduler.Default;
return Observable.Create<Message>(o =>
{
// first check there is only one subscriber
// (multiple stream readers would cause havoc)
int previous = Interlocked.CompareExchange(ref subscribed, 1, 0);
if (previous != 0)
o.OnError(new Exception(
"Only one subscriber is allowed for each stream."));
// we will return a disposable that cleans
// up both the scheduled task below and
// the source stream
var dispose = new CompositeDisposable
{
Disposable.Create(sourceStream.Dispose)
};
// use async scheduling to get nice imperative code
var schedule = scheduler.ScheduleAsync(async (ctrl, ct) =>
{
// store the header here each time
var header = new byte[4];
// loop until cancellation requested
while (!ct.IsCancellationRequested)
{
try
{
// read the exact number of bytes for a header
await ReadExactBytesAsync(sourceStream, header, ct);
}
catch (OperationCanceledException)
{
throw;
}
catch (Exception ex)
{
// pass through any problem in the stream and quit
o.OnError(new InvalidDataException("Error in stream.", ex));
return;
}
ct.ThrowIfCancellationRequested();
var bodyLength = IPAddress.NetworkToHostOrder(
BitConverter.ToInt16(header, 2));
// create buffer to read the message
var payload = new byte[bodyLength];
// read exact bytes as before
try
{
await ReadExactBytesAsync(sourceStream, payload, ct);
}
catch (OperationCanceledException)
{
throw;
}
catch (Exception ex)
{
o.OnError(new InvalidDataException("Error in stream.", ex));
return;
}
// create a new message and send it to client
var message = new Message { PayLoad = payload };
o.OnNext(message);
}
// wrap things up
ct.ThrowIfCancellationRequested();
o.OnCompleted();
});
// return the suscription handle
dispose.Add(schedule);
return dispose;
});
}
EDIT - Very hacky test code I used:
private static void Main(string[] args)
{
var listener = new TcpListener(IPAddress.Any, 12873);
listener.Start();
var listenTask = listener.AcceptTcpClientAsync();
listenTask.ContinueWith((Task<TcpClient> t) =>
{
var client = t.Result;
var stream = client.GetStream();
const string messageText = "Hello World!";
var body = Encoding.UTF8.GetBytes(messageText);
var header = BitConverter.GetBytes(
IPAddress.HostToNetworkOrder(body.Length));
for (int i = 0; i < 5; i++)
{
stream.Write(header, 0, 4);
stream.Write(body, 0, 4);
stream.Flush();
// deliberate nasty delay
Thread.Sleep(2000);
stream.Write(body, 4, body.Length - 4);
stream.Flush();
}
stream.Close();
listener.Stop();
});
var tcpClient = new TcpClient();
tcpClient.Connect(new IPEndPoint(IPAddress.Loopback, 12873));
var clientStream = tcpClient.GetStream();
ReadMessages(clientStream).Subscribe(
Console.WriteLine,
ex => Console.WriteLine("Error: " + ex.Message),
() => Console.WriteLine("Done!"));
Console.ReadLine();
}
Wrapping up
You need to think about setting a timeout for reads, in case the server dies, and some kind of "end message" should be sent by the server. Currently this method will just continually tries to receive bytes. As you haven't specced it, I haven't included anything like this - but if you do, then as I've written it just breaking out of the while loop will cause OnCompleted to be sent.

I guess what is needed here is Qactive: A Rx.Net based queryable reactive tcp server provider
Server
Observable
.Interval(TimeSpan.FromSeconds(1))
.ServeQbservableTcp(new IPEndPoint(IPAddress.Loopback, 3205))
.Subscribe();
Client
var datasourceAddress = new IPEndPoint(IPAddress.Loopback, 3205);
var datasource = new TcpQbservableClient<long>(datasourceAddress);
(
from value in datasource.Query()
//The code below is actually executed on the server
where value <= 5 || value >= 8
select value
)
.Subscribe(Console.WriteLine);
What´s mind blowing about this is that clients can say what and how frequently they want the data they receive and the server can still limit and control when, how frequent and how much data it returns.
For more info on this https://github.com/RxDave/Qactive
Another blog.sample
https://sachabarbs.wordpress.com/2016/12/23/rx-over-the-wire/