I'm using SharpPcap to dump packets to a .pcap file. My problem is, that it's working to slow to capture any amount of network traffic and I run out of memory eventually.
How can i speed up the file writing process?
Here is the code I'm using:
private void WriteToPCAPThread(object o)
{
this.WritePcapThreadDone.Reset();
string captureFileName = (string)o;
CaptureFileWriterDevice captureFileWriter = new CaptureFileWriterDevice(this.device, captureFileName);
captureFileWriter.Open();
RawCapture packet;
bool success;
while (this.capturing)
{
success = this.captures.TryDequeue(out packet);
if (success)
{
captureFileWriter.Write(packet);
}
else
{
// Queue emptied
Thread.Sleep(50);
}
}
}
Thanks in advance for any ideas.
I ended up writing my own StreamWriter. Now I get 100% performance out of my SSD.
public PcapStream
{
private Stream BaseStream;
public PcapStream(Stream BaseStream)
{
this.BaseStream=BaseStream;
}
public void Write(RawCapture packet)
{
byte[] arr = new byte[packet.Data.Length + 16];
byte[] sec = BitConverter.GetBytes((uint)packet.Timeval.Seconds);
byte[] msec = BitConverter.GetBytes((uint)packet.Timeval.MicroSeconds);
byte[] incllen = BitConverter.GetBytes((uint)packet.Data.Length);
byte[] origlen = BitConverter.GetBytes((uint)packet.Data.Length);
Array.Copy(sec, arr, sec.Length);
int offset = sec.Length;
Array.Copy(msec, 0, arr, offset, msec.Length);
offset += msec.Length;
Array.Copy(incllen, 0, arr, offset, incllen.Length);
offset += incllen.Length;
Array.Copy(origlen, 0, arr, offset, origlen.Length);
offset += origlen.Length;
Array.Copy(packet.Data, 0, arr, offset, packet.Data.Length);
BaseStream.Write(arr, 0, arr.Length);
}
Related
In my application every packet has 2 bytes length on the start. However after some time application starts receiving length less than zero. In synchronous client everything works correctly, but it's too slow. I'm 100% sure in Server everything is correct.
Connect:
public void Connect(IPAddress ip, int port)
{
tcpClient.Connect(ip, port);
stream = tcpClient.GetStream();
byte[] len_buffer = new byte[2];
stream.BeginRead(len_buffer, 0, len_buffer.Length, OnDataRead, len_buffer);
}
OnDataRead:
private void OnDataRead(IAsyncResult ar)
{
byte[] len = ar.AsyncState as byte[];
int length = BitConverter.ToInt16(len, 0);
byte[] buffer = new byte[length];
int remaining = length;
int pos = 0;
while (remaining != 0)
{
int add = stream.Read(buffer, pos, remaining);
pos += add;
remaining -= add;
}
Process(buffer);
len = new byte[2];
stream.EndRead(ar);
stream.BeginRead(len, 0, len.Length, OnDataRead, len);
}
As I can see, you're mixing up synchronious and asynchronious. That's a bad practice.
What you want is something like:
var header = ReadHeader(); // 2 bytes
var data = ReadData(header.DataSize);
I didn't use the network stream, but....
Here's an example of my async SocketReader:
public static class SocketReader
{
// This method will continues read until count bytes are read. (or socket is closed)
private static void DoReadFromSocket(Socket socket, int bytesRead, int count, byte[] buffer, Action<ArraySegment<byte>> endRead)
{
// Start a BeginReceive.
try
{
socket.BeginReceive(buffer, bytesRead, count - bytesRead, SocketFlags.None, (asyncResult) =>
{
// Get the bytes read.
int read = 0;
try
{
// if this goes wrong, the read remains 0
read = socket.EndReceive(asyncResult);
}
catch (ObjectDisposedException) { }
catch (Exception exception)
{
Trace.TraceError(exception.Message);
}
// if zero bytes received, the socket isn't available anymore.
if (read == 0)
{
endRead(new ArraySegment<byte>(buffer, 0, 0));
return;
}
// increase the bytesRead, (position within the buffer)
bytesRead += read;
// if all bytes are read, call the endRead with the buffer.
if (bytesRead == count)
// All bytes are read. Invoke callback.
endRead(new ArraySegment<byte>(buffer, 0, count));
else
// if not all bytes received, start another BeginReceive.
DoReadFromSocket(socket, bytesRead, count, buffer, endRead);
}, null);
}
catch (Exception exception)
{
Trace.TraceError(exception.Message);
endRead(new ArraySegment<byte>(buffer, 0, 0));
}
}
public static void ReadFromSocket(Socket socket, int count, Action<ArraySegment<byte>> endRead)
{
// read from socket, construct a new buffer.
DoReadFromSocket(socket, 0, count, new byte[count], endRead);
}
public static void ReadFromSocket(Socket socket, int count, byte[] buffer, Action<ArraySegment<byte>> endRead)
{
// if you do have a buffer available, you can pass that one. (this way you do not construct new buffers for receiving and able to reuse buffers)
// if the buffer is too small, raise an exception, the caller should check the count and size of the buffer.
if (count > buffer.Length)
throw new ArgumentOutOfRangeException(nameof(count));
DoReadFromSocket(socket, 0, count, buffer, endRead);
}
}
Usage:
SocketReader.ReadFromSocket(socket, 2, (headerData) =>
{
if(headerData.Count == 0)
{
// nothing/closed
return;
}
// Read the length of the data.
int length = BitConverter.ToInt16(headerData.Array, headerData.Offset);
SocketReader.ReadFromSocket(socket, length, (dataBufferSegment) =>
{
if(dataBufferSegment.Count == 0)
{
// nothing/closed
return;
}
Process(dataBufferSegment);
// extra: if you need a binaryreader..
using(var stream = new MemoryStream(dataBufferSegment.Array, dataBufferSegment.Offset, dataBufferSegment.Count))
using(var reader = new BinaryReader(stream))
{
var whatever = reader.ReadInt32();
}
}
});
You can optimize the receive buffer by passing a buffer (look at the overloads)
Continues receiving: (reusing receivebuffer)
public class PacketReader
{
private byte[] _receiveBuffer = new byte[2];
// This will run until the socket is closed.
public void StartReceiving(Socket socket, Action<ArraySegment<byte>> process)
{
SocketReader.ReadFromSocket(socket, 2, _receiveBuffer, (headerData) =>
{
if(headerData.Count == 0)
{
// nothing/closed
return;
}
// Read the length of the data.
int length = BitConverter.ToInt16(headerData.Array, headerData.Offset);
// if the receive buffer is too small, reallocate it.
if(_receiveBuffer.Length < length)
_receiveBuffer = new byte[length];
SocketReader.ReadFromSocket(socket, length, _receiveBuffer, (dataBufferSegment) =>
{
if(dataBufferSegment.Count == 0)
{
// nothing/closed
return;
}
try
{
process(dataBufferSegment);
}
catch { }
StartReceiving(socket, process);
});
});
}
}
Usage:
private PacketReader _reader;
public void Start()
{
_reader = new PacketReader(socket, HandlePacket);
}
private void HandlePacket(ArraySegment<byte> packet)
{
// do stuff.....
}
What I'm trying to do is to received a large number of bytes (about 5MB data) sent from the client side
Below is the code where data(byte[]) is received
byte[] receivedBytesRaw = new byte[4000];
//first, initialize network stream
NetworkStream stream = client.GetStream();
//The bytesNeeded is the size of bytes which is a protocol sent by the client side indicating the size of byte which will be sent
int bytesNeeded = 4000;
int bytesReceived = 0;
do
{
int bytesRead = stream.Read(receivedBytesRaw, bytesReceived, bytesNeeded - bytesReceived);
networkValidation.addBytesToList(receivedBytesRaw, ref receivedBytes);
bytesReceived += bytesRead;
} while (bytesReceived < bytesNeeded);
But now I'm stuck on a problem:
Everytime when data arrives, the do while loop loops for the first time, and the return value (i) is 26, then it loops again, this time, when it goes to " i = stream.Read(receivedBytesRaw, 0, receivedBytesRaw.Length);", the program seems waiting for the client side to send data and have no response, also, when I check "receivedBytesRaw", the data was incomplete, only the first 13 bytes was received, the remaining space in the byte array remains null, and the stream.DataAvailable is false
Why the server side received incomplete data?
Note: when I try to send small data (a string), it's ok
=====================================================================
Edited
Below is the client side code which sends data:
private int sendData(byte[] dataSend, string IP, int portNumber)
{
TcpClient clientSide = new TcpClient();
int result = -1;
try
{
clientSide.Connect(IP, portNumber);
}
catch (Exception ex)
{
return 2;
}
NetworkStream netStream = clientSide.GetStream();
if (netStream.CanWrite)
{
byte[] replyMsg = new byte[1024];
netStream.Write(dataSend, 0, dataSend.Length);
netStream.Flush();
result = 0;
}
else
{
result = 1;
}
return result;
}
Because it's a stream, and can be partial received. Are you sure you are always receiving packages with te size of 2048 bytes?
int i = 0;
int bytesNeeded = 200;
int bytesReceived = 0;
do
{
//read byte from client
int bytesRead = stream.Read(receivedBytesRaw, bytesReceived, bytesNeeded-bytesReceived);
bytesReceived += bytesRead;
// merge byte array to another byte array
} while (bytesReceived < bytesNeeded);
I think you need a frame protocol, try create a protocol like, writing the size of the data that follows.
example: (psuedo)
void SendData(byte[] data)
{
// get the 4 bytes of a int value.
byte[] dataLength = BitConverter.GetBytes(data.Lenght);
// write the length to the stream.
stream.Write(dataLength, 0, dataLength.Length);
// write the data bytes.
stream.Write(data, 0, data.Length);
}
void Receive()
{
// read 4 bytes from the stream.
ReadBuffer(buffer, 4);
// convert those 4 bytes to an int.
int dataLength = BitConverter.ToInt32(buffer, 0);
// read bytes with dataLength as count.
ReadBuffer(buffer, dataLength);
}
// read until the right amount of bytes are read.
void ReadBuffer(byte[] buffer, int length)
{
int i = 0;
int bytesNeeded = length;
int bytesReceived = 0;
do
{
//read byte from client
int bytesRead = stream.Read(buffer, bytesReceived, bytesNeeded-bytesReceived);
bytesReceived += bytesRead;
// merge byte array to another byte array
} while (bytesReceived < bytesNeeded); // <- you should do this async.
}
This is just an example..
Another solution you could try is using async reads.
I made a class that reads until all bytes are read. If it isn't a problem that the complete file is read, you could try this:
Example:
This example show that you can read a simple protocol. ReadPacket handles a length + data message. So the sender will first send an int value containing the length of data that follows.
The StartReading method reads a filename and the filedata. It will store up to 10mb max filesize. But this isn't originally designed for receiving files.
const int MaxFileSize = 10 * 1024 * 1024;
private void Example()
{
Socket socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
socket.Connect("localhost", 12345);
StartReading(socket);
}
private void StartReading(Socket socket)
{
ReadPacket(socket, (filenameData) =>
{
if (filenameData.Count == 0)
{
// disconnected
return;
}
// parse the filename
string filename = Encoding.UTF8.GetString(filenameData.Array, filenameData.Offset, filenameData.Count);
Trace.WriteLine("Receiving file :" + filename);
ReadPacket(socket, (fileData) =>
{
if (fileData.Count == 0)
{
// disconnected
return;
}
Trace.WriteLine("Writing file :" + filename);
// write to the file
using (FileStream stream = new FileStream(filename, FileMode.Create, FileAccess.Write))
stream.Write(fileData.Array, fileData.Offset, fileData.Count);
// start waiting for another packet.
StartReading(socket);
});
});
}
private void ReadPacket(Socket socket, Action<ArraySegment<byte>> endRead)
{
// read header. (length of data) (4 bytes)
EasySocketReader.ReadFromSocket(socket, 4, (headerBufferSegment) =>
{
// if the ReadFromSocket returns 0, the socket is closed.
if (headerBufferSegment.Count == 0)
{
// disconnected;
endRead(new ArraySegment<byte>());
return;
}
// Get the length of the data that follows
int length = BitConverter.ToInt32(headerBufferSegment.Array, headerBufferSegment.Offset);
// Check the length
if (length > MaxFileSize)
{
// disconnect
endRead(new ArraySegment<byte>());
return;
}
// Read bytes specified in length.
EasySocketReader.ReadFromSocket(socket, length, (dataBufferSegment) =>
{
// if the ReadFromSocket returns 0, the socket is closed.
if (dataBufferSegment.Count == 0)
{
endRead(new ArraySegment<byte>());
return;
}
endRead(dataBufferSegment);
});
});
}
The EasySocketReader class can be found on my blog: http://csharp.vanlangen.biz/network-programming/async-sockets/asyncsocketreader/
The original EasyPacketReader can be found here: http://csharp.vanlangen.biz/network-programming/async-sockets/easypacketreader/
For the sending part, you could use something like this:
private void SendFile(Socket socket, string filename)
{
byte[] filenameData = Encoding.UTF8.GetBytes(filename);
socket.Send(BitConverter.GetBytes(filenameData.Length));
socket.Send(filenameData);
int fileSize;
byte[] fileData;
using (FileStream stream = new FileStream(filename, FileMode.Open, FileAccess.Read))
{
fileSize = (int)stream.Length;
if (fileSize > MaxFileSize)
throw new ArgumentOutOfRangeException("File too big");
fileData = new byte[fileSize];
stream.Read(fileData, 0, fileSize);
}
socket.Send(BitConverter.GetBytes(fileSize));
socket.Send(fileData);
}
I'm building a network library to use in further projects, I know a bit of networking (in general and programming) but I still encounter problems when I want to transfer larger packets on a high frequency. How would this be possible in TCP?
One possible solution would be chuncking, that is not a problem at all I'll just put them in a buffer and have the whole packet again. This causes overhead at the server, handling lots of data is consuming lots of resources. And it affects the stability because the server get somewhere stuck and the client throws an ObjectDisposedException at BeginSend (connection is closed). Of course its probably my fault but consider a 'relative' small packet of 2048 bytes (2 kB) which get chunked in 512 bytes (4 chunks):
//main.cs
static void Main(string[] args)
{
client = new Client("82.72.201.150", 2345);
client.SendCompleted += OnSend;
Console.WriteLine(client.Connect());
var bytes = new byte[2048];
for (int i = 0; i < bytes.Length; i++)
{
bytes[i] = (byte)((i*i + 0x25)%byte.MaxValue);
}
while (true)
{
SendChuncked(bytes);
Thread.Sleep(500);
}
}
static void SendChuncked(byte[] buffer, int chunksize = 512)
{
var chunks = buffer.Length/chunksize;
var rest = buffer.Length%chunksize;
var size = BitConverter.GetBytes(buffer.Length);
client.Send(size, 0, size.Length);
for (int i = 0; i < chunks; i++)
{
client.Send(buffer, i * chunksize, chunksize);
}
if (rest > 0)
client.Send(buffer, chunks * chunksize, rest);
}
//....
//client.cs
public struct TransferState
{
public byte[] buffer;
public int offset;
public int count;
public int handled;
public ManualResetEvent waitHandle;
public bool wait;
}
public void Send(byte[] buffer, int offset, int count)
{
var size = BitConverter.GetBytes(count);
Send(size, 0, size.Length, true);
Send(buffer, offset, count, false);
}
private void Send(byte[] buffer, int offset, int count, bool wait, TransferState? state = null)
{
state = state ?? new TransferState
{
buffer = buffer,
offset = offset,
count = count,
handled = 0,
wait = wait,
waitHandle = new ManualResetEvent(false)
};
socket.BeginSend(buffer, offset, count, SocketFlags.None, SendCallback, state);
if (wait)
{
((TransferState) state).waitHandle.WaitOne();
}
}
private void Send(byte[] buffer, int offset, int count, bool wait, TransferState? state = null)
{
state = state ?? new TransferState
{
buffer = buffer,
offset = offset,
count = count,
handled = 0,
wait = wait,
waitHandle = new ManualResetEvent(false)
};
socket.BeginSend(buffer, offset, count, SocketFlags.None, SendCallback, state);
if (wait)
{
((TransferState) state).waitHandle.WaitOne();
}
}
private void SendCallback(IAsyncResult result)
{
if (result.AsyncState is TransferState == false)
throw new ArgumentException("Invalid type of state.", "state");
var state = (TransferState)result.AsyncState;
var sent = socket.EndSend(result);
state.handled += sent;
var tosent = state.count - state.handled;
var offset = state.offset + state.handled;
if (tosent > 0)
{
Send(state.buffer, offset, tosent, state.wait, state);
}
else
{
state.waitHandle.Set();
SendCompleted(this, new TransferCompletedArgs(this, state));
}
}
//....
public void Receive(TransferState? state = null)
{
if (state == null)
{
var buffer = new byte[sizeof(int)];
socket.BeginReceive(buffer, 0, sizeof (int), SocketFlags.None, ReceiveCallback, buffer);
}
else
{
var transferState = state.Value;
socket.BeginReceive(transferState.buffer, transferState.offset, transferState.count - transferState.handled, SocketFlags.None, ReceiveCallback, transferState);
}
}
private void ReceiveCallback(IAsyncResult result)
{
//receiving the amount to receive
if (result.AsyncState is byte[])
{
var buffer = (byte[])result.AsyncState;
var rec = socket.EndReceive(result);
if (rec != 4) //TODO: HANDLE MORE PROPERLY
throw new NotImplementedException("Error while receiving the amoount to receive.");
var toreceive = BitConverter.ToInt32(buffer, 0);
var state = new TransferState
{
buffer = new byte[toreceive],
count = toreceive,
wait = false
};
Receive(state);
}
//know we know the amount we can receive it till the end
else if (result.AsyncState is TransferState)
{
var state = (TransferState)result.AsyncState;
var rec = socket.EndReceive(result);
state.offset += rec;
state.handled += rec;
var toreceive = state.count - state.handled;
if (toreceive > 0)
{
Receive(state);
Debug.WriteLine("[{2}] size mismatch: {0}/{1}", state.handled, state.count, DateTime.Now.ToString("mm:ss.fff"));
}
else
{
ReceiveCompleted(this, new TransferCompletedArgs(this, state));
Receive();
}
}
else
{
throw new ArgumentException("State is not typeof byte[] or TransferState.");
}
}
So if its hard to follow:
The client tries to connect with the server
The server accepts
the client
The client first sends how long the buffer is
The
client starts sending the buffer (and loops this)
The server reads
how long the buffer is and allocates a buffer for it
The server
reads until the buffer is read and invokes an event (and loops this
proces as well)
So I don't know what actually happens, but the server stops receiving. While connected clients still sending data, after couple seconds an ObjectDisposedException is thrown at BeginSend. How would I resolve this from happening? (And have a stable server that can handle lots of traffic.)
Is there any way to use Stream.CopyTo to copy only certain number of bytes to destination stream? what is the best workaround?
Edit:
My workaround (some code omitted):
internal sealed class Substream : Stream
{
private readonly Stream stream;
private readonly long origin;
private readonly long length;
private long position;
public Substream(Stream stream, long length)
{
this.stream = stream;
this.origin = stream.Position;
this.position = stream.Position;
this.length = length;
}
public override int Read(byte[] buffer, int offset, int count)
{
var n = Math.Max(Math.Min(count, origin + length - position), 0);
int bytesRead = stream.Read(buffer, offset, (int) n);
position += bytesRead;
return bytesRead;
}
}
then to copy n bytes:
var substream = new Substream(stream, n);
substream.CopyTo(stm);
The implementation of copying streams is not overly complicated. If you want to adapt it to copy only a certain number of bytes then it shouldn't be too difficult to tweak the existing method, something like this
public static void CopyStream(Stream input, Stream output, int bytes)
{
byte[] buffer = new byte[32768];
int read;
while (bytes > 0 &&
(read = input.Read(buffer, 0, Math.Min(buffer.Length, bytes))) > 0)
{
output.Write(buffer, 0, read);
bytes -= read;
}
}
The check for bytes > 0 probably isn't strictly necessary, but can't do any harm.
What's wrong with just copying the bytes you need using a buffer?
long CopyBytes(long bytesRequired, Stream inStream, Stream outStream)
{
long readSoFar = 0L;
var buffer = new byte[64*1024];
do
{
var toRead = Math.Min(bytesRequired - readSoFar, buffer.Length);
var readNow = inStream.Read(buffer, 0, (int)toRead);
if (readNow == 0)
break; // End of stream
outStream.Write(buffer, 0, readNow);
readSoFar += readNow;
} while (readSoFar < bytesRequired);
return readSoFar;
}
I've created a simple buffer manager class to be used with asyncroneous sockets. This will protect against memory fragmentation and improve performance. Any suggestions for further improvements or other approaches?
public class BufferManager
{
private int[] free;
private byte[] buffer;
private readonly int blocksize;
public BufferManager(int count, int blocksize)
{
buffer = new byte[count * blocksize];
free = new int[count];
this.blocksize = blocksize;
for (int i = 0; i < count; i++)
free[i] = 1;
}
public void SetBuffer(SocketAsyncEventArgs args)
{
for (int i = 0; i < free.Length; i++)
{
if (1 == Interlocked.CompareExchange(ref free[i], 0, 1))
{
args.SetBuffer(buffer, i * blocksize, blocksize);
return;
}
}
args.SetBuffer(new byte[blocksize], 0, blocksize);
}
public void FreeBuffer(SocketAsyncEventArgs args)
{
int offset = args.Offset;
byte[] buff = args.Buffer;
args.SetBuffer(null, 0, 0);
if (buffer == buff)
free[offset / blocksize] = 1;
}
}
Edit:
The orignal answer below addresses a code construction issue of overly tight coupling. However, considering the solution as whole I would avoid using just one large buffer and handing over slices of it in this way. You expose your code to buffer overrun (and shall we call it buffer "underrun" issues). Instead I would manage an array of byte arrays each being a discrete buffer. Offset handed over is always 0 and size is always the length of the buffer. Any bad code that attempts to read/write parts beyond the boundaries will be caught.
Original answer
You've coupled the class to SocketAsyncEventArgs where in fact all it needs is a function to assign the buffer, change SetBuffer to:-
public void SetBuffer(Action<byte[], int, int> fnSet)
{
for (int i = 0; i < free.Length; i++)
{
if (1 == Interlocked.CompareExchange(ref free[i], 0, 1))
{
fnSet(buffer, i * blocksize, blocksize);
return;
}
}
fnSet(new byte[blocksize], 0, blocksize);
}
Now you can call from consuming code something like this:-
myMgr.SetBuffer((buf, offset, size) => myArgs.SetBuffer(buf, offset, size));
I'm not sure that type inference is clever enough to resolve the types of buf, offset, size in this case. If not you will have to place the types in the argument list:-
myMgr.SetBuffer((byte[] buf, int offset, int size) => myArgs.SetBuffer(buf, offset, size));
However now your class can be used to allocate a buffer for all manner of requirements that also use the byte[], int, int pattern which is very common.
Of course you need to decouple the free operation to but thats:-
public void FreeBuffer(byte[] buff, int offset)
{
if (buffer == buff)
free[offset / blocksize] = 1;
}
This requires you to call SetBuffer on the EventArgs in consuming code in the case for SocketAsyncEventArgs. If you are concerned that this approach reduces the atomicity of freeing the buffer and removing it from the sockets use, then sub-class this adjusted buffer manager and include SocketAsyncEventArgs specific code in the sub-class.
I've created a new class with a completely different approach.
I have a server class that receives byte arrays. It will then invoke different delegates handing them the buffer objects so that other classes can process them. When those classes are done they need a way to push the buffers back to the stack.
public class SafeBuffer
{
private static Stack bufferStack;
private static byte[][] buffers;
private byte[] buffer;
private int offset, lenght;
private SafeBuffer(byte[] buffer)
{
this.buffer = buffer;
offset = 0;
lenght = buffer.Length;
}
public static void Init(int count, int blocksize)
{
bufferStack = Stack.Synchronized(new Stack());
buffers = new byte[count][];
for (int i = 0; i < buffers.Length; i++)
buffers[i] = new byte[blocksize];
for (int i = 0; i < buffers.Length; i++)
bufferStack.Push(new SafeBuffer(buffers[i]));
}
public static SafeBuffer Get()
{
return (SafeBuffer)bufferStack.Pop();
}
public void Close()
{
bufferStack.Push(this);
}
public byte[] Buffer
{
get
{
return buffer;
}
}
public int Offset
{
get
{
return offset;
}
set
{
offset = value;
}
}
public int Lenght
{
get
{
return buffer.Length;
}
}
}