Well, I'm trying to convert a large information in bytes for string. (11076 length)
The problem in the end, the information is with missing characters. (length 10996)
Look:
The information is received by Winsock connection, look the proccess:
public static void UpdateClient(UserConnection client)
{
string data = null;
Decoder utf8Decoder = Encoding.UTF8.GetDecoder();
Console.WriteLine("Iniciando");
byte[] buffer = ReadFully(client.TCPClient.GetStream(), 0);
int charCount = utf8Decoder.GetCharCount(buffer, 0, buffer.Length);
Char[] chars = new Char[charCount];
int charsDecodedCount = utf8Decoder.GetChars(buffer, 0, buffer.Length, chars, 0);
foreach (Char c in chars)
{
data = data + String.Format("{0}", c);
}
int buffersize = buffer.Length;
Console.WriteLine("Chars is: " + chars.Length);
Console.WriteLine("Data is: " + data);
Console.WriteLine("Byte is: " + buffer.Length);
Console.WriteLine("Size is: " + data.Length);
Server.Network.ReceiveData.SelectPacket(client.Index, data);
}
public static byte[] ReadFully(Stream stream, int initialLength)
{
if (initialLength < 1)
{
initialLength = 32768;
}
byte[] buffer = new byte[initialLength];
int read = 0;
int chunk;
chunk = stream.Read(buffer, read, buffer.Length - read);
checkreach:
read += chunk;
if (read == buffer.Length)
{
int nextByte = stream.ReadByte();
if (nextByte == -1)
{
return buffer;
}
byte[] newBuffer = new byte[buffer.Length * 2];
Array.Copy(buffer, newBuffer, buffer.Length);
newBuffer[read] = (byte)nextByte;
buffer = newBuffer;
read++;
goto checkreach;
}
byte[] ret = new byte[read];
Array.Copy(buffer, ret, read);
return ret;
}
Anyone have tips or a solution?
It's perfectly normal for UTF-8 encoded text to be more bytes than the number of characters. In UTF-8 some characters (for example á and ã) are encoded into two or more bytes.
As the ReadFully method returns garbage if you try to use it to read more than fits in the initial buffer or if it can't read the entire stream with one Read call, you shouldn't use it. Also the way that the char array is converted to a string is extremely slow. Just use a StreamReader to read the stream and decode it to a string:
public static void UpdateClient(UserConnection client) {
string data;
using (StreamReader reader = new StreamReader(client.TCPClient.GetStream(), Encoding.UTF8)) {
data = reader.ReadToEnd();
}
Console.WriteLine("Data is: " + data);
Console.WriteLine("Size is: " + data.Length);
Server.Network.ReceiveData.SelectPacket(client.Index, data);
}
Related
I was looking around for a way to copy a portion of a file from a determined address to another address, is there a way to do that in C#?
For example let's say i have a file like this:
and I want to copy from 0xA0 to 0xB0 and then paste it to another file.
Something like this perhaps:
long start = 0xA0;
int length = 0xB0 - start;
byte[] data = new byte[length];
using (FileStream fs = File.OpenRead(#"C:\Temp\InputFile.txt"))
{
fs.Seek(start, SeekOrigin.Begin);
fs.Read(data, 0, length);
}
File.WriteAllBytes(#"C:\Temp\OutputFile.txt", data);
It should be something like:
// input data
string inputName = "input.bin";
long startInput = 0xa0;
long endInput = 0xb0; // excluding 0xb0 that is not copied
string outputName = "output.bin";
long startOutput = 0xa0;
// begin of code
long count = endInput - startInput;
using (var fs = File.OpenRead(inputName))
using (var fs2 = File.OpenWrite(outputName))
{
fs.Seek(startInput, SeekOrigin.Begin);
fs2.Seek(startOutput, SeekOrigin.Begin);
byte[] buf = new byte[4096];
while (count > 0)
{
int read = fs.Read(buf, 0, (int)Math.Min(buf.Length, count));
if (read == 0)
{
// end of file encountered
throw new IOException("end of file encountered");
}
fs2.Write(buf, 0, read);
count -= read;
}
}
public void SendFile(string remoteHostIP, int remoteHostPort, string longFileName, string shortFileName)
{
byte[] fileNameByte = Encoding.ASCII.GetBytes(shortFileName);
byte[] fileData = File.ReadAllBytes(longFileName);
byte[] clientData = new byte[4 + fileNameByte.Length + fileData.Length];
byte[] fileNameLen = BitConverter.GetBytes(fileNameByte.Length);
fileNameLen.CopyTo(clientData, 0);
fileNameByte.CopyTo(clientData, 4);
fileData.CopyTo(clientData, 4 + fileNameByte.Length);
TcpClient clientSocket = new TcpClient(remoteHostIP, remoteHostPort);
NetworkStream networkStream = clientSocket.GetStream();
networkStream.Write(clientData, 0, clientData.GetLength(0));
networkStream.Close();
}
It is possible to use this function to send a 1GB file because the maximum file size i had been try to send now is only up to 400MB. More than that is will cause an error of 'System.OutOfMemoryException'.
When i use another method to split the file into few parts but the server side can't receive the parts continuously and can only receive one of the part.
private void splitBigFile(string FileInputPath, byte[] inputArray)
{
int port = 1113;
double partSize = 104852000;
int partSize2 = 104852000;
string FolderOutputPath = "C:\\Users\\xx\\Desktop\\testing split";
string currPartPath;
string shortNameSplit;
FileStream fileStream = new FileStream(FileInputPath, FileMode.Open);
FileInfo fiSource = new FileInfo(txtFile.Text);
double sourceLength = fiSource.Length;
partNum = (int)Math.Ceiling((double)(sourceLength / partSize));
for (int i = 0; i < partNum; i++)
{
if (i == (partNum - 1))
{
partSize2 = (int)fiSource.Length - (i * 104852000);
}
currPartPath = FolderOutputPath + "\\" + fiSource.Name + "." + String.Format(#"{0:D4}", i) + ".part";
shortNameSplit = fiSource.Name + "." + String.Format(#"{0:D4}", i) + ".part";
byte[] fileNameByte = Encoding.ASCII.GetBytes(shortNameSplit);
byte[] readStream = new byte[partSize2];
byte[] concateFile = new byte[5 + fileNameByte.Length];
int ipSend = ((partNum - 1 - i) << 1);
ipSend |= 0; // for differentiate ip or file
byte[] byteSend = new byte[1];
byteSend[0] = (byte)ipSend;
fileStream.Read(readStream, 0, partSize2);
byte[] fileNameLen = BitConverter.GetBytes(fileNameByte.Length);
byteSend.CopyTo(concateFile, 0);
fileNameLen.CopyTo(concateFile, 1);
fileNameByte.CopyTo(concateFile, 5);
concateFile.CopyTo(inputArray, 0);
readStream.CopyTo(inputArray, concateFile.Length);
string ipAddress = "192.168.43.67";
int sendport = 1113;
//Task.Factory.StartNew(() => SendBigFileSize(ipAddress, sendport, inputArray[i]));
Array.Clear(readStream, 0, readStream.Length);
Array.Clear(fileNameLen, 0, fileNameLen.Length);
Array.Clear(fileNameByte, 0, fileNameByte.Length);
Array.Clear(byteSend, 0, byteSend.Length);
Array.Clear(byteSend, 0, byteSend.Length);
}
fileStream.Close();
}
Of course, just not all at once. Send chunks at a time.
Besides, the maximum TCP packet size is 64k, and the MTU is 1500 bytes. Your huge buffer is getting split up too, but you're using up a lot of memory doing it. Instead read 32MB (safe HDD I/O buffer size) at a time from your file and send it, then read the next bit.
I am trying to read all data present in the buffer of the Machine connected through TCP/IP but i don't know why i am not getting all data ,some data is getting Missed.
Here is the code that i am using ..
using (NetworkStream stream = client.GetStream())
{
byte[] data = new byte[1024];
int numBytesRead = stream.Read(data, 0, data.Length);
if (numBytesRead > 0)
{
string str= Encoding.ASCII.GetString(data, 0, numBytesRead);
}
}
Please tell me what i am missing to get all the data from the machine.
Thanks in advance..
The problem with your code is that you will not get all the data if the data size is bigger than the buffer size (1024 bytes in your case) so you have to Read the stream inside the loop. Then you can Write all the data inside a MemoryStream until the end of the NetworkStream.
string str;
using (NetworkStream stream = client.GetStream())
{
byte[] data = new byte[1024];
using (MemoryStream ms = new MemoryStream())
{
int numBytesRead ;
while ((numBytesRead = stream.Read(data, 0, data.Length)) > 0)
{
ms.Write(data, 0, numBytesRead);
}
str = Encoding.ASCII.GetString(ms.ToArray(), 0, (int)ms.Length);
}
}
This example from MSDN: NetworkStream.DataAvailable shows how you can use that property to do so:
// Examples for CanRead, Read, and DataAvailable.
// Check to see if this NetworkStream is readable.
if(myNetworkStream.CanRead)
{
byte[] myReadBuffer = new byte[1024];
StringBuilder myCompleteMessage = new StringBuilder();
int numberOfBytesRead = 0;
// Incoming message may be larger than the buffer size.
do{
numberOfBytesRead = myNetworkStream.Read(myReadBuffer, 0, myReadBuffer.Length);
myCompleteMessage.AppendFormat("{0}", Encoding.ASCII.GetString(myReadBuffer, 0, numberOfBytesRead));
}
while(myNetworkStream.DataAvailable);
// Print out the received message to the console.
Console.WriteLine("You received the following message : " +
myCompleteMessage);
}
else
{
Console.WriteLine("Sorry. You cannot read from this NetworkStream.");
}
Try this:
private string GetResponse(NetworkStream stream)
{
byte[] data = new byte[1024];
using (MemoryStream memoryStream = new MemoryStream())
{
do
{
stream.Read(data, 0, data.Length);
memoryStream.Write(data, 0, data.Length);
} while (stream.DataAvailable);
return Encoding.ASCII.GetString(memoryStream.ToArray(), 0, (int)memoryStream.Length);
}
}
Try this code:
using (NetworkStream stream = client.GetStream())
{
while (!stream.DataAvailable)
{
Thread.Sleep(20);
}
if (stream.DataAvailable && stream.CanRead)
{
Byte[] data = new Byte[1024];
List<byte> allData = new List<byte>();
do
{
int numBytesRead = stream.Read(data,0,data.Length);
if (numBytesRead == data.Length)
{
allData.AddRange(data);
}
else if (numBytesRead > 0)
{
allData.AddRange(data.Take(numBytesRead));
}
} while (stream.DataAvailable);
}
}
Hope this helps, it should prevent that you miss any data sended to you.
The synchronous method sometimes does not display the request body. Using the asynchronous method stably displays request body.
string request = default(string);
StringBuilder sb = new StringBuilder();
byte[] buffer = new byte[client.ReceiveBufferSize];
int bytesCount;
if (client.GetStream().CanRead)
{
do
{
bytesCount = client.GetStream().ReadAsync(buffer, 0, buffer.Length).Result;
sb.Append(Encoding.UTF8.GetString(buffer, 0, bytesCount));
}
while(client.GetStream().DataAvailable);
request = sb.ToString();
}
TCP itself does not have any ways to define "end of data" condition. This is responsibility of application level portocol.
For instance see HTTP request description:
A client request (consisting in this case of the request line and only one header field) is followed by a blank line, so that the request ends with a double newline
So, for request end of data is determined by two newline sequences. And for response:
Content-Type specifies the Internet media type of the data conveyed by the HTTP message, while Content-Length indicates its length in bytes.
The response content size is specified in header before data.
So, it's up to you how to encode amount of data transferred at once - it can be just first 2 or 4 bytes in the beginning of the data holding total size to read or more complex ways if needed.
for my scenario, the message itself was telling the length of subsequent message. here is the code
int lengthOfMessage=1024;
string message = "";
using (MemoryStream ms = new MemoryStream())
{
int numBytesRead;
while ((numBytesRead = memStream.Read(MessageBytes, 0, lengthOfMessage)) > 0)
{
lengthOfMessage = lengthOfMessage - numBytesRead;
ms.Write(MessageBytes, 0, numBytesRead);
}
message = Encoding.ASCII.GetString(ms.ToArray(), 0, (int)ms.Length);
}
#George Chondrompilas answer is correct but instead of writing it by yourself you can use CopyTo function which does the same :
https://stackoverflow.com/a/65188160/4120180
My requirement is to transfer a zip file of size 400MB or more; The following code works for at least 40MB; But for more I would have to change byte[] bytes = new byte[50000000]; to byte[] bytes = new byte[400000000]; and maxRequestLength to maxRequestLength="409600";
The problem is byte[] bytes = new byte[100000000]; returns an error regarding insufficient space. So how can I transfer large files using WebClient??
WebClient client = new WebClient();
client.AllowWriteStreamBuffering = true;
UriBuilder ub = new UriBuilder("http://localhost:57596/UploadImages.ashx");
ub.Query = "ImageName=" + "DataSet" + DataSetId + ".zip";
client.OpenWriteCompleted += (InputStream, eArguments) =>
{
try
{
using (Stream output = eArguments.Result)
{
output.Write(ImagesAux, 0, (int)ImagesAux.Length);
//numeroimagem++;
}
}
catch (Exception ex)
{
MessageBox.Show(ex.Message);
//throw;
}
};
client.OpenWriteAsync(ub.Uri);
in UploadImages.ashx
public void ProcessRequest(HttpContext context)
{
//context.Response.ContentType = "text/plain";
//context.Response.Write("Hello World");
string ImageName = context.Request.QueryString["ImageName"];
string UploadPath = context.Server.MapPath("~/ServerImages/");
using (FileStream stream = File.Create(UploadPath + ImageName))
{
byte[] bytes = new byte[50000000]; //
int bytesToRead = 0;
while ((bytesToRead =
context.Request.InputStream.Read(bytes, 0, bytes.Length)) != 0)
{
stream.Write(bytes, 0, bytesToRead);
stream.Close();
}
}
}
in Web.config
<httpRuntime targetFramework="4.5" maxRequestLength="40960"/>
You should never load everything in memory then write all back to disk, but instead you should load pieces and write them to disk while you are reading them.
When you've done reading you close the stream you are writing to.
Otherwise as soon as you reach sizes as GB you can get an OutOfMemory really quick.
So I would change the writing bytes to disk from this:
using (FileStream stream = File.Create(UploadPath + ImageName))
{
byte[] bytes = new byte[50000000]; //
int bytesToRead = 0;
while ((bytesToRead = context.Request.InputStream.Read(bytes, 0, bytes.Length)) != 0)
{
stream.Write(bytes, 0, bytesToRead);
stream.Close();
}
}
to this:
using (FileStream stream = File.Create(UploadPath + ImageName))
{
byte[] bytes = new byte[1024];
long totalBytes = context.Request.InputStream.Length;
long bytesRead = 0;
int bytesToRead = bytes.Length;
if (totalBytes - bytesRead < bytes.Length)
bytesToRead = (int)(totalBytes - bytesRead);
bytes = new byte[bytesToRead];
while ((bytesToRead = context.Request.InputStream.Read(bytes, bytesRead, bytes.Length)) != 0)
{
stream.Write(bytes, bytesRead, bytes.Length);
bytesRead += bytes.Length;
if (totalBytes - bytesRead < bytes.Length)
bytesToRead = (int)(totalBytes - bytesRead);
bytes = new byte[bytesToRead];
}
stream.Close();
}
1024 would be the buffer size.
I am trying to compress data using the zlib .net library. Regardless of the content of the uncompressed string I only seem to get two bytes of data in the raw[].
{
string uncompressed = "1234567890";
byte[] data = UTF8Encoding.Default.GetBytes(uncompressed);
MemoryStream input = new MemoryStream(data);
MemoryStream output = new MemoryStream();
Stream outZStream = new ZOutputStream(output,zlibConst.Z_DEFAULT_COMPRESSION);
CopyStream(input, outZStream);
output.Seek(0, SeekOrigin.Begin);
byte[] raw = output.ToArray();
string compressed = Convert.ToBase64String(raw);
}
public void CopyStream(System.IO.Stream input, System.IO.Stream output)
{
byte[] buffer = new byte[2000];
int len;
while ((len = input.Read(buffer, 0, 2000)) > 0)
{
output.Write(buffer, 0, len);
}
output.Flush();
}
The problem here is that the ZOutputStream actually writes some of the information into the stream in the finish() method (which is called by Close). The Close method also closes the base stream, so that is not much use in this situation.
Changing the code to the following should work:
{
string uncompressed = "1234567890";
byte[] data = UTF8Encoding.Default.GetBytes(uncompressed);
MemoryStream input = new MemoryStream(data);
MemoryStream output = new MemoryStream();
ZOutputStream outZStream = new ZOutputStream(output,zlibConst.Z_DEFAULT_COMPRESSION);
CopyStream(input, outZStream);
outZStream.finish();
output.Seek(0, SeekOrigin.Begin);
byte[] raw = output.ToArray();
string compressed = Convert.ToBase64String(raw);
}
public void CopyStream(System.IO.Stream input, System.IO.Stream output)
{
byte[] buffer = new byte[2000];
int len;
while ((len = input.Read(buffer, 0, 2000)) > 0)
{
output.Write(buffer, 0, len);
}
output.Flush();
}