Develop Reference

Convert Rtsp Packet to bitmap in c#

Hello Every on I have implemented SharpRtsp in c# https://github.com/ngraziano/SharpRTSP
I am receiving rtsp Packets and it has h264 format It has following method in H264Payload Class
private List<byte[]> Process_H264_RTP_Frame(List<byte[]> rtp_payloads)
{
Console.WriteLine("RTP Data comprised of " + rtp_payloads.Count + " rtp packets");
List<byte[]> nal_units = new List<byte[]>(); // Stores the NAL units for a Video Frame. May be more than one NAL unit in a video frame.
for (int payload_index = 0; payload_index < rtp_payloads.Count; payload_index++)
{
// Examine the first rtp_payload and the first byte (the NAL header)
int nal_header_f_bit = (rtp_payloads[payload_index][0] >> 7) & 0x01;
int nal_header_nri = (rtp_payloads[payload_index][0] >> 5) & 0x03;
int nal_header_type = (rtp_payloads[payload_index][0] >> 0) & 0x1F;
// If the Nal Header Type is in the range 1..23 this is a normal NAL (not fragmented)
// So write the NAL to the file
if (nal_header_type >= 1 && nal_header_type <= 23)
{
Console.WriteLine("Normal NAL");
norm++;
nal_units.Add(rtp_payloads[payload_index]);
}
// There are 4 types of Aggregation Packet (split over RTP payloads)
else if (nal_header_type == 24)
{
Console.WriteLine("Agg STAP-A");
stap_a++;
// RTP packet contains multiple NALs, each with a 16 bit header
// Read 16 byte size
// Read NAL
try
{
int ptr = 1; // start after the nal_header_type which was '24'
// if we have at least 2 more bytes (the 16 bit size) then consume more data
while (ptr + 2 < (rtp_payloads[payload_index].Length - 1))
{
int size = (rtp_payloads[payload_index][ptr] << 8) + (rtp_payloads[payload_index][ptr + 1] << 0);
ptr = ptr + 2;
byte[] nal = new byte[size];
System.Array.Copy(rtp_payloads[payload_index], ptr, nal, 0, size); // copy the NAL
nal_units.Add(nal); // Add to list of NALs for this RTP frame. Start Codes like 00 00 00 01 get added later
ptr = ptr + size;
}
}
catch
{
// do nothing
}
}
else if (nal_header_type == 25)
{
Console.WriteLine("Agg STAP-B not supported");
stap_b++;
}
else if (nal_header_type == 26)
{
Console.WriteLine("Agg MTAP16 not supported");
mtap16++;
}
else if (nal_header_type == 27)
{
Console.WriteLine("Agg MTAP24 not supported");
mtap24++;
}
else if (nal_header_type == 28)
{
Console.WriteLine("Frag FU-A");
fu_a++;
// Parse Fragmentation Unit Header
int fu_header_s = (rtp_payloads[payload_index][1] >> 7) & 0x01; // start marker
int fu_header_e = (rtp_payloads[payload_index][1] >> 6) & 0x01; // end marker
int fu_header_r = (rtp_payloads[payload_index][1] >> 5) & 0x01; // reserved. should be 0
int fu_header_type = (rtp_payloads[payload_index][1] >> 0) & 0x1F; // Original NAL unit header
Console.WriteLine("Frag FU-A s=" + fu_header_s + "e=" + fu_header_e);
// Check Start and End flags
if (fu_header_s == 1 && fu_header_e == 0)
{
// Start of Fragment.
// Initiise the fragmented_nal byte array
// Build the NAL header with the original F and NRI flags but use the the Type field from the fu_header_type
byte reconstructed_nal_type = (byte)((nal_header_f_bit << 7) + (nal_header_nri << 5) + fu_header_type);
// Empty the stream
fragmented_nal.SetLength(0);
// Add reconstructed_nal_type byte to the memory stream
fragmented_nal.WriteByte(reconstructed_nal_type);
// copy the rest of the RTP payload to the memory stream
fragmented_nal.Write(rtp_payloads[payload_index], 2, rtp_payloads[payload_index].Length - 2);
}
if (fu_header_s == 0 && fu_header_e == 0)
{
// Middle part of Fragment
// Append this payload to the fragmented_nal
// Data starts after the NAL Unit Type byte and the FU Header byte
fragmented_nal.Write(rtp_payloads[payload_index], 2, rtp_payloads[payload_index].Length - 2);
}
if (fu_header_s == 0 && fu_header_e == 1)
{
// End part of Fragment
// Append this payload to the fragmented_nal
// Data starts after the NAL Unit Type byte and the FU Header byte
fragmented_nal.Write(rtp_payloads[payload_index], 2, rtp_payloads[payload_index].Length - 2);
// Add the NAL to the array of NAL units
nal_units.Add(fragmented_nal.ToArray());
}
}
else if (nal_header_type == 29)
{
Console.WriteLine("Frag FU-B not supported");
fu_b++;
}
else
{
Console.WriteLine("Unknown NAL header " + nal_header_type + " not supported");
}
}
// Output some statistics
Console.WriteLine("Norm=" + norm + " ST-A=" + stap_a + " ST-B=" + stap_b + " M16=" + mtap16 + " M24=" + mtap24 + " FU-A=" + fu_a + " FU-B=" + fu_b);
// Output all the NALs that form one RTP Frame (one frame of video)
return nal_units;
}
}
I have triew and done reading but i have not found any way to convert H264 Frame into bitmap I just want it to save as bitmaps can any body help me o convert this data to bitmap?

The end of that function returns data in H264 format
// Output all the NALs that form one RTP Frame (one frame of video)
return nal_units;
To get a bitmap you need to pass the H264 data into a Video Decompressor. If you are not familiar with digital video, then think about .ZIP or .RAR or .7Z files.
You cannot read the contents of these files without the right software to convert the compressed data back to a usable format.
The same applies to video. H264 is a compressed data format.
You need to use a H264 decompression codec to confer the nal_units into a bitmap (or another video format called YUV).
Some people use ffmpeg for this decompression. Others pass the H264 data into an Operating System API (like VideoToolbox on the Mac or MediaCodec on Android or whatever Windows supports).
But you need these extra steps.

C# how to send & receive from a hardware protocol?

I'm a C# programmer and I know nothing about hardware protocol things.
today I received a document that is some protocols of a Lock hardware, like this:
lock command
start 0x80
board address 0x01-0xf
lock address 0x00-18
command 0x33
verify code XX
sample:
machine send 0x80 0x01 0x01 0x33 0xB2
if recieve 0x80 0x01 0x01 0x01 0x91 (means unlock)
if receive 0x80 0x01 0x01 0x00 0x80 (means locked)
All I want to know is, if C# can handle these commands? if can, where I can have a quick start, or what should I search on google?
Thanks.

Yes. C# can handle this. This is call polling. Basically, the idea is
send command, receive reply, substring information that you need (in your case, most probably the last 2 bytes) and perform some function on it. I'm not sure if you understand but I'll give you an example on something I've done previously but this is event where data is send to machine whenever event is triggered.
public enum Transaction
{
LOCK = 0x01,
UNLOCK
};
private static string getTransactionDescription(Transaction transaction, string data = "")
{
string result = "";
switch (transaction)
{
case Transaction.UNLOCK:
case Transaction.LOCK:
var slot = ByteOperation.reverse4ByteBitPattern(data.Substring(32, 64));
for (int i = 8 - 1; i >= 0; i--)
{
for (int j = 0; j < 8; j++)
{
if ((Convert.ToInt32(ByteOperation.ToggleEndian_4Bytes(slot.Substring(i * 8, 8)), 16) & (1 << j)) > 0)
{
if (!string.IsNullOrWhiteSpace(result))
{
result += ", ";
}
result += "Slot " + (((7 - i) * 8) + j + 1).ToString("D3");
}
}
}
break;
}
return result;

Random arithmetic overflows when reading from serial port

What I'm doing is taking a user entered string, creating a packet with the data, then sending the string out to a serial port. I am then reading the data I send via a loopback connector. My send is working flawlessly, however my receive is randomly throwing an arithmetic overflow exception.
I say randomly because it is not happening consistently. For example, I send the message "hello" twice. The first time works fine, the second time outputs nothing and throws an exception. I restart my program, run the code again, and send hello only to receive "hell" and then an exception. On rare occasion, I'll receive the packet 3 or 4 times in a row without error before the exception.
Here is my relevant code:
public void receivePacket(object sender, SerialDataReceivedEventArgs e)
{
byte[] tempByte = new byte[2];
int byteCount = 0;
while (serialPort1.BytesToRead > 0)
{
if (byteCount <= 1)
{
tempByte[byteCount] = (byte)serialPort1.ReadByte();
}
if (byteCount == 1)
{
receivedString = new byte[tempByte[byteCount]];
receivedString[0] = tempByte[0];
receivedString[1] = tempByte[1];
}
else if (byteCount > 1)
{
byte b = (byte)serialPort1.ReadByte();
receivedString[byteCount] = b;
}
byteCount++;
}
int strLen = (byteCount - 3);
tempByte = new byte[strLen];
int newBit = 0;
for (int i = 2; i <= strLen+1; i++)
{
tempByte[newBit] = receivedString[i];
newBit++;
}
string receivedText = encoder.GetString(tempByte);
SetText(receivedText.ToString());
}
I'm well aware that my implementation using byteCount (which I use to traverse the byte array) is rather sloppy. When I step through the code, I find that when I get the error byteCount == 1, which is making strLen a negative number (since strLen is byteCount - 3, which is done because the packet contains a header, length, and CRC i.e. byteCount - 3 == # of actual data bytes received). This leads to by tempByte having a size of -2, which throws my exceptions. I, however, am having a very hard time figuring out why byteCount is being set to 1.
The code after this basically just traverses the data section of the array, copies it into the tempByte, then is sent off to a function to append the text in another thread.

I am guessing that byteCount is 1 because you only received one byte - or rather, you processed the first byte before the second one arrived in the buffer.
The ReadByte function will wait for a certain amount of time for a byte to arrive if there isn't one waiting.
Maybe if instead of checking BytesToRead, you did something more like this:
byte headerByte = serialPort1.ReadByte();
byte length = serialPort1.ReadByte();
receivedString = new byte[length];
receivedString[0] = headerByte;
receivedString[1] = length;
for (int i = 2; i < length; i++) {
receivedString[i] = serialPort1.ReadByte();
}

C# TcpClient closes the connection on the Server when sending world information with an exception

I am trying to send all the tiles (which are referred to as Particles, don't confuse them with client particle effects) in my world (500x350 2d array in my World class) through a NetworkStream hooked up with my Client both on the Server and Clientside.
The code is as follows :
public int ServerSendWorldData(NetworkStream networkStream, Point from, Point size, bool reportBytesSent, Networking.Client toClient)
{
int bytesOfAcknowledgementSent = 0;
int totalBytesSent = 0;
int tilesSentThisAcknowledgement = 0;
int tilesToSendForThisAcknowledgement = 20;
for (int x = from.X; x < size.X; x++)
{
for (int y = from.Y; y < size.Y; y++)
{
Thread.Sleep(0);
//Handle acknowledgement if needed.
if (tilesSentThisAcknowledgement >= tilesToSendForThisAcknowledgement)
{
//Wait for Client acknowledgement.
bytesOfAcknowledgementSent += Networking.NetworkingHelp.SendMessageFromByte((byte)Networking.MessageType.ServerSendWorldMapDataWaitAcknowledgement, networkStream);
//Handle here.
if (networkStream.ReadByte() == (byte)Networking.MessageType.ClientAcknowledgeWorldMapData)
tilesSentThisAcknowledgement = 0;
else
throw new Exception("Client did not acknowledge data!");
}
if (world.worldParticles[x, y] != null)
{
//Send Particle Data
totalBytesSent += Networking.NetworkingHelp.SendMessageFromByte((byte)Networking.MessageType.ServerSendWorldMapDataParticle, networkStream);
totalBytesSent += Networking.NetworkingHelp.SendMessageFromInt(x, networkStream);
totalBytesSent += Networking.NetworkingHelp.SendMessageFromInt(y, networkStream);
totalBytesSent += Networking.NetworkingHelp.SendMessageFromInt(world.worldParticles[x, y].ID, networkStream);
//totalBytesSent += Networking.NetworkingHelp.SendMessageFromInt(world.worldParticles[x, y].spriteIndex, networkStream);
totalBytesSent += Networking.NetworkingHelp.SendMessageFromBool(world.worldParticles[x, y].draw, networkStream);
totalBytesSent += Networking.NetworkingHelp.SendMessageFromBool(world.worldParticles[x, y].collisionWithEntities, networkStream);
tilesSentThisAcknowledgement++;
}
}
}
if (reportBytesSent)
{
Statistics.Console.WriteLine("Sent " + totalBytesSent + " bytes of World data to Client " + toClient.name + " and " + bytesOfAcknowledgementSent + " bytes of acknowledgement was exchanged!");
}
return totalBytesSent;
}
The NetworkingHelp class is basically just networkStream.Write with either a BitConverter converting data such as ints to bytes or an Encoding converting data such as strings to bytes in UTF8.
The code sends a total of 0.76 MB of map data to the Client.
The problem I am having is not present when connecting to the server using localhost / IPAddress.Loopback, but the problem appears when sending data to Clients through the internet - the data is sent extremely slowly (probably due to the file size, I'm unsure though) and without the "Handle Acknowledgement" bit of code ...
//Handle acknowledgement if needed.
if (tilesSentThisAcknowledgement >= tilesToSendForThisAcknowledgement)
{
//Wait for Client acknowledgement.
bytesOfAcknowledgementSent += Networking.NetworkingHelp.SendMessageFromByte((byte)Networking.MessageType.ServerSendWorldMapDataWaitAcknowledgement, networkStream);
//Handle here.
if (networkStream.ReadByte() == (byte)Networking.MessageType.ClientAcknowledgeWorldMapData)
tilesSentThisAcknowledgement = 0;
else
throw new Exception("Client did not acknowledge data!");
}
... the Client recieves about 20 world tiles and throws the exception "Unable to write data to the transport connection: An existing connection was forcibly closed by the remote host", this seems to happen because the Client gets a 1 byte message with byte 0 as its contents.
It seems that the pause created by the "Handle Acknowledgement" code solves this problem, but makes the world data transfer a lot slower.
Help would be much appreciated, and I'm aware my code is kind of messy (open to suggestions on how to improve it, it's my first time sending this much data). Here is the block of code for reading the particular message for the Client.
//WorldParticlesMapDataParticle
else if (recievedMessage[0] == (byte)Networking.MessageType.ServerSendWorldMapDataParticle)
{
//Read the position of the new data.
recievedMessageBytes = networkStream.Read(recievedMessage, 0, 4);
int newParticleX = BitConverter.ToInt32(recievedMessage, 0);
recievedMessageBytes = networkStream.Read(recievedMessage, 0, 4);
int newParticleY = BitConverter.ToInt32(recievedMessage, 0);
//Read the particle ParticleID.
recievedMessageBytes = networkStream.Read(recievedMessage, 0, 4);
int newParticleID = BitConverter.ToInt32(recievedMessage, 0);
//Read the particle SpriteID.
//recievedMessageBytes = networkStream.Read(recievedMessage, 0, 4);
//int newSpriteID = BitConverter.ToInt32(recievedMessage, 0);
//Read the particle draw.
recievedMessageBytes = networkStream.Read(recievedMessage, 0, 1);
bool newParticleDraw = BitConverter.ToBoolean(recievedMessage, 0);
//Read the particle collision.
recievedMessageBytes = networkStream.Read(recievedMessage, 0, 1);
bool newParticleCollision = BitConverter.ToBoolean(recievedMessage, 0);
//Set particle.
try
{
world.worldParticles[newParticleX, newParticleY] = World.Particle.ParticleManager.particleArray[newParticleID];
//world.worldParticles[newParticleX, newParticleY].spriteIndex = newSpriteID;
world.worldParticles[newParticleX, newParticleY].draw = newParticleDraw;
world.worldParticles[newParticleX, newParticleY].collisionWithEntities = newParticleCollision;
}
catch (Exception ex)
{
Statistics.Console.WriteLine("Server requested new Particle at " + newParticleX + "," + newParticleY + ", but Client failed to place it due to an exception : " + ex.Message);
try
{
world.worldParticles[newParticleX, newParticleY] = null;
}
catch
{ }
}
}

You ignore the return value from NetworkStream.Read. I think you are assuming that it will always read the number of bytes you asked it to read unless there's an error. In fact, the number of bytes you specify is just the maximum it will read and it will not read more bytes than are currently available when you call it.
"This method reads data into the buffer parameter and returns the number of bytes successfully read. If no data is available for reading, the Read method returns 0. The Read operation reads as much data as is available, up to the number of bytes specified by the size parameter. If the remote host shuts down the connection, and all available data has been received, the Read method completes immediately and return zero bytes." - NetworkStream.Read
You have made the classic mistake of assuming TCP will somehow 'glue together' data just because you sent it in a single call. In fact, TCP has no such mechanism.

you are sending one particle at a time try making a big array of what you want to send and send it in one chunk

To complement David Schwartz's answer, you would be wise to read Stephen Cleary's TCP/IP FAQ, especially the message framing section

What does a working example of dealing with packet fragmentation in C# look like when using the Begin* End* method?

After enough playing with asynchronous socket programming I noticed that the server was receiving chunked payloads (ie: more than one complete payload sitting in the same buffer). So I came up with the following:
if (bytes_to_read > 0)
{
while (bytes_to_read > 0)
// Get payload size as int.
// Get payload in byte format.
// Do something with payload.
// Decrease the amount of bytes to read.
}
// Wait for more data.
}
And then I noticed packet fragmentation (ie: what I thought were complete payloads chunked together wasn't always so) which changed the previous code to something like:
if (bytes_to_read > 0)
{
while (bytes_to_read > 0)
{
// Get payload size as int.
// Check if the payload size is less than or equal to the amount of bytes left to read.
if (payload_size <= bytes_to_read)
{
// Get payload in byte format.
// Do something with payload.
// Decrease the amount of bytes to read.
}
else
{
// We received a fragmented payload.
break;
}
}
if (bytes_to_read == 0)
{
// Wait for more data.
}
else if (bytes_to_read > 0)
{
// Wait for more data where we left off. ***
}
else
{
// Something awful happened.
}
}
*** I don't even know how to go about this and would like to see code for it. I had an idea that it involved copying the in-completed payload to the beginning of the buffer and then picking up from there.
The pseudo code I included is based on the Begin* End* method I am using (I'm aware that I should be using the *Async set of methods found here -> http://msdn.microsoft.com/en-us/library/system.net.sockets.socketasynceventargs.aspx <- but I think my overall question still applies).
I am seeking the answers to 2 questions--namely:
Is this approach correct or am I
missing something?
What does a working example of
dealing with packet fragmentation in
C# look like?
EDIT: I'm using raw sockets.
Thanks in advance for all your help.
EDIT: John Saunders and Greg Hewgill have brought up the point of treating data as a stream but that does not provide me with a concrete example of how to deal with the last chunked payload sometimes being fragmented.
EDIT: I have read Jon Skeet's answer here which is basically along the same lines as the other answers I have seen but it doesn't help me much as I already get what I have to do but not how to do it.
EDIT: To elaborate on what I mean by fragmentation, consider the following the receive buffers:
224TEST3foo3bar
224TEST3foo3bar224TEST3foo3bar
224TEST3foo3bar224TEST3foo
3bar224TEST3foo3bar
EDIT: I found this and this which lead me here. Vadym Stetsiak has cleared nearly everything up (his was one of the answers I was looking for).

This may or may not have anything to do with fragmentation.
In general, the socket will pass you as many bytes at a time as it feels like. Your job is to know how many bytes are in your overall message, and to read them all. Just keep looping until you have all the bytes you need, or until there's an exception.
The following code is untested right now. I thought I'd post it before writing the server side of it and testing both.
private static string ReceiveMessage(Socket socket)
{
const int BUFFER_SIZE = 1024;
var inputBuffer = new byte[BUFFER_SIZE];
var offset = 0;
var bytesReceived = socket.Receive(
inputBuffer, offset, BUFFER_SIZE - offset, SocketFlags.None);
if (bytesReceived < 2)
{
throw new InvalidOperationException("Receive error");
}
var inputMessageLength = inputBuffer[0]*256 + inputBuffer[1];
offset += bytesReceived;
var totalBytesReceived = bytesReceived;
while (bytesReceived > 0 &&
totalBytesReceived < inputMessageLength + 2)
{
bytesReceived = socket.Receive(
inputBuffer, offset, BUFFER_SIZE - offset, SocketFlags.None);
offset += bytesReceived;
totalBytesReceived += bytesReceived;
}
return Encoding.UTF8.GetString(
inputBuffer, 2, totalBytesReceived - 2);
}
Note that the receipt of the message length is wrong. The socket layer could give it to me a byte at a time. I'm going to revisit that as part of a refactoring that will receive the count into a separate two-byte buffer, and change the loop into a single do/while.

When you have to do it yourself, it can be done like so (reference here):
///
/// Server state holds current state of the client socket
///
class AsyncServerState
{
public byte[] Buffer = new byte[512]; //buffer for network i/o
public int DataSize = 0; //data size to be received by the server
//flag that indicates whether prefix was received
public bool DataSizeReceived = false;
public MemoryStream Data = new MemoryStream(); //place where data is stored
public SocketAsyncEventArgs ReadEventArgs = new SocketAsyncEventArgs();
public Socket Client;
}
///
/// Implements server receive logic
///
private void ProcessReceive(SocketAsyncEventArgs e)
{
//single message can be received using several receive operation
AsyncServerState state = e.UserToken as AsyncServerState;
if (e.BytesTransferred <= 0 || e.SocketError != SocketError.Success)
{
CloseConnection(e);
}
int dataRead = e.BytesTransferred;
int dataOffset = 0;
int restOfData = 0;
while (dataRead > 0)
{
if (!state.DataSizeReceived)
{
//there is already some data in the buffer
if (state.Data.Length > 0)
{
restOfData = PrefixSize - (int)state.Data.Length;
state.Data.Write(state.Buffer, dataOffset, restOfData);
dataRead -= restOfData;
dataOffset += restOfData;
}
else if (dataRead >= PrefixSize)
{ //store whole data size prefix
state.Data.Write(state.Buffer, dataOffset, PrefixSize);
dataRead -= PrefixSize;
dataOffset += PrefixSize;
}
else
{ // store only part of the size prefix
state.Data.Write(state.Buffer, dataOffset, dataRead);
dataOffset += dataRead;
dataRead = 0;
}
if (state.Data.Length == PrefixSize)
{ //we received data size prefix
state.DataSize = BitConverter.ToInt32(state.Data.GetBuffer(), 0);
state.DataSizeReceived = true;
state.Data.Position = 0;
state.Data.SetLength(0);
}
else
{ //we received just part of the headers information
//issue another read
if (!state.Client.ReceiveAsync(state.ReadEventArgs))
ProcessReceive(state.ReadEventArgs);
return;
}
}
//at this point we know the size of the pending data
if ((state.Data.Length + dataRead) >= state.DataSize)
{ //we have all the data for this message
restOfData = state.DataSize - (int)state.Data.Length;
state.Data.Write(state.Buffer, dataOffset, restOfData);
Console.WriteLine("Data message received. Size: {0}",
state.DataSize);
dataOffset += restOfData;
dataRead -= restOfData;
state.Data.SetLength(0);
state.Data.Position = 0;
state.DataSizeReceived = false;
state.DataSize = 0;
if (dataRead == 0)
{
if (!state.Client.ReceiveAsync(state.ReadEventArgs))
ProcessReceive(state.ReadEventArgs);
return;
}
else
continue;
}
else
{ //there is still data pending, store what we've
//received and issue another BeginReceive
state.Data.Write(state.Buffer, dataOffset, dataRead);
if (!state.Client.ReceiveAsync(state.ReadEventArgs))
ProcessReceive(state.ReadEventArgs);
dataRead = 0;
}
}
}
I did not do it exactly this way myself but it helped.