I am writing a C# assembly to take advantage of the GZip functionality in newer versions of the .NET framework, and I am copying code I have used successfully elsewhere.
The way this works is that an application we are using feeds a stream to the assembly, the assembly reads it, compresses it, then returns the compressed result as a string. That string is then put back into the assembly through a stream to uncompress it (normally it will be stored, but I am simply running a basic test).
However, when I feed the string back to the assembly, it errors out while reading it from a byte array that is read from the application's stream. Here is the code:
private void ReadStream(IStream stream, out byte[] data)
{
using (MemoryStream writer = new MemoryStream())
{
IntPtr rwBytes = Marshal.AllocHGlobal(4);
int _rwBytes = 0;
data = new byte[0xafc8];
do
{
stream.Read(data, 0xafc8, rwBytes);
_rwBytes = Marshal.ReadInt32(rwBytes);
writer.Write(data, 0, _rwBytes);
}
while (_rwBytes > 0);
writer.Close();
Marshal.FreeHGlobal(rwBytes);
}
}
public string CompressString([In, MarshalAs(UnmanagedType.AsAny)] object appStream)
{
byte[] buffer = null;
ReadStream(appStream as IStream, out buffer);
MemoryStream ms = new MemoryStream();
using (GZipStream zip = new GZipStream(ms, CompressionMode.Compress, true))
{
zip.Write(buffer, 0, buffer.Length);
}
ms.Position = 0;
MemoryStream outStream = new MemoryStream();
byte[] compressed = new byte[ms.Length];
ms.Read(compressed, 0, compressed.Length);
byte[] gzBuffer = new byte[compressed.Length + 4];
System.Buffer.BlockCopy(compressed, 0, gzBuffer, 4, compressed.Length);
System.Buffer.BlockCopy(BitConverter.GetBytes(buffer.Length), 0, gzBuffer, 0, 4);
string str = Convert.ToBase64String(gzBuffer);
gzBuffer = Convert.FromBase64String(str);
return str;
}
public string DecompressString([In, MarshalAs(UnmanagedType.AsAny)] object appStream)
{
byte[] buffer = null;
ReadStream(appStream as IStream, out buffer);
string compressedText = Encoding.UTF8.GetString(buffer);
byte[] gzBuffer = Convert.FromBase64String(compressedText);
using (MemoryStream ms = new MemoryStream())
{
int msgLength = BitConverter.ToInt32(gzBuffer, 0);
ms.Write(gzBuffer, 4, gzBuffer.Length - 4);
buffer = new byte[msgLength];
ms.Position = 0;
using (GZipStream zip = new GZipStream(ms, CompressionMode.Decompress))
{
zip.Read(buffer, 0, buffer.Length);
}
}
return Encoding.UTF8.GetString(buffer);
}
The string is returned from the CompressString function as:
yK8AAB+LCAAAAAAABADtvQdgHEmWJSYvbcp7f0r1StfgdKEIgGATJNiQQBDswYjN5pLsHWlHIymrKoHKZVZlXWYWQMztnbz33nvvvffee++997o7nU4n99//P1xmZAFs9s5K2smeIYCqyB8/fnwfPyK+ndd5WjRplk6rxarOm6aolmmbN+04TZ/n7cdN2uTU4jwt2vSqqt8241/jR8+Pnh89P3p+9Pzo+dHzo+dHz4+eHz0/en70/Oj50fOj50fPj54fPT96fvT86PnR86PnR8+Pnh89P3p+9Pzo+dHz/67nR9j86PnR86PnR8+Pnh89P3p+9Pzo+dHzo+dHz4+eHz0/en70/Oj50fOj50fPj54fPT96fvT86PnR86PnR8+Pnh89/99/fjSCHz0/en70/Oj50fOj50fPj54fPT96fvT86PnR86PnR8+Pnh89P3p+9Pzo+dHzo+dHz4+eHz0/en70/Oj50fOj5//7z49G8KPnR8+Pnh89P3p+9Pzo+dHzo+dHz4+eHz0/en70/Oj50fOj50fPj54fPT96fvT86PnR86PnR8+Pnh89P3p+9Px///nRCH70/Oj50fOj50fPj54fPT96fvT86PnR86PnR8+Pnh89P3p+9Pzo+dHzo+dHz4+eHz0/en70/Oj50fOj50fPj57/7z8/GsGPnh89P3p+9Pzo+dHzo+dHz4+eHz0/en70/Oj50fP/nef/AYk/UbjIrwAA
When it is read back from the stream in the Encoding.UTF8.GetString call, it is:
yK8AAB+LCAAAAAAABADtvQdgHEmWJSYvbcp7f0r1StfgdKEIgGATJNiQQBDswYjN5pLsHWlHIymrKoHKZVZlXWYWQMztnbz33nvvvffee++997o7nU4n99//P1xmZAFs9s5K2smeIYCqyB8/fnwfPyK+ndd5WjRplk6rxarOm6aolmmbN+04TZ/n7cdN2uTU4jwt2vSqqt8241/jR8+Pnh89P3p+9Pzo+dHzo+dHz4+eHz0/en70/Oj50fOj50fPj54fPT96fvT86PnR86PnR8+Pnh89P3p+9Pzo+dHz/67nR9j86PnR86PnR8+Pnh89P3p+9Pzo+dHzo+dHz4+eHz0/en70/Oj50fOj50fPj54fPT96fvT86PnR86PnR8+Pnh89/99/fjSCHz0/en70/Oj50fOj50fPj54fPT96fvT86PnR86PnR8+Pnh89P3p+9Pzo+dHzo+dHz4+eHz0/en70/Oj50fOj5//7z49G8KPnR8+Pnh89P3p+9Pzo+dHzo+dHz4+eHz0/en70/Oj50fOj50fPj54fPT96fvT86PnR86PnR8+Pnh89P3p+9Px///nRCH70/Oj50fOj50fPj54fPT96fvT86PnR86PnR8+Pnh89P3p+9Pzo+dHzo+dHz4+eHz0/en70/Oj50fOj50fPj57/7z8/GsGPnh89P3p+9Pzo+dHzo+dHz4+eHz0/en70/Oj50fP/nef/AYk/UbjIrwAA
These strings are identical. I can even convert it back and forth in the CompressString function with no problem. Any thoughts out there? This is very strange.
Don't use UTF8, you should use the ascii text encoding instead. When you get the bytes from your string, you'll end up with some extra bytes due to the way UTF strings are encoded.
Related
I am trying to decompress the byte array using GZipStream but getting error of "Found invalid data while decoding."
// getting aadhaar sample qr code data from
// https://uidai.gov.in/images/resource/User_manulal_QR_Code_15032019.pdf
static void Main(string[] args)
{
string source = "6979414848205548481619299442879901900893978332594614407044767717485407280104077714658698163325401659212830920734233047578454701810567032015270223682917915825234703754712504887921309181789607809168884583848396456653007022479356336240198130363930881632367124738541517499494458139647378808680614169273221404741476596583953169248831376224396335169577064812987140578144885819479190173537644970232125142253963784979138011318798385442436099901621998283624816070080504830712594525760596934341576755626791590403636878139861665599383319429228364434183913197958738697001410493839281298692342829951566712530309758759364649701153639921979798429707566199261950037418171329283207372048014948669160666776198414040633384677104717697507521717586776709084200364956178863636105988867260929887577092955570407803783021397897341999914616790441029837229129746669225095633201097644321593502503404440714110515167034889128258965583435965030225845348564582051521348800742574442877087774194668983516629631073341202705453382780613775427336949283388084891654484225446940941660942440637784744293259916479841407088189462964489670231866481904237338494872813098890875845640034370370387108798950180220865436012752487216677041817312930119747601017807577565413977545693375480131324240696099879479436722576566447939593195590684591261809038023122178172006150499569185218838749337238281597037288924464009997530938336798176023597292328320965086990184531426188862965408313308973495924965144113396593829090645266653313774582036138982013368561474719154447134894466611560589758251829063226370300282175823479569847261439348404558251402273730865053482214589180028302043821438357583302818374143973997002745047526405755760407045006694423501337081780299815080324840337828812644300041900356816429114261098230198976752026002079876882796597235615015594486182057781476152918170746403157005216896239428521706033466061587608065036133153074432195952131368564234168005447770190345777024917629879639171161719929852078265309160759260989590618158889891835294735614366674503961584445497685736312628248483551986529867423016255476553691922054241686230968975229511700928171281549902682365302333677412951788839806869796040512235899311734337858684531156721416280114473368826463098485252394260075790386415875290922570568686439586036262465414002334117870088922801660529414759784318799843806130096998190881240404138869293309782335305296720666220243304175086358278211355789957998014801209332293458940463859106591986434520433810583569309224929264228263841477378949329312443958215939294432669464260216534074560882723006838459792812340253078330291135526952675203790833430237852831740601433198364243363569730205351077393441691141240055900819091229931605146865520183001810239708464322588389956036291760175558843819105418234580239610174323636606095262722940143706063698846499673285377621180570537788160304936809915237889489342387891057012783726694920184573202789672963922380028271124448024265644396686341508447830351380242127542393849410283830409594988503246799544444687606954881510597515686410993828907588979699141180160893062603338104857903239845856783130275935413569275439908789983311663211937449259444259898972766208";
BigInteger.TryParse(source, out BigInteger numBig);
byte[] bytes = numBig.ToByteArray();
string isoBytes2 = Decompress(bytes);
}
static string Decompress(byte[] gzBuffer)
{
using (MemoryStream ms = new MemoryStream())
{
int msgLength = BitConverter.ToInt32(gzBuffer, 0);
ms.Write(gzBuffer, 0, gzBuffer.Length);
byte[] buffer = new byte[msgLength];
ms.Position = 0;
int length;
using (GZipStream zip = new GZipStream(ms, CompressionMode.Decompress))
{
length = zip.Read(buffer, 0, buffer.Length);
}
var data = new byte[length];
Array.Copy(buffer, data, length);
return Encoding.UTF8.GetString(data);
}
}
In C#, we need to do reverse byte array after coverting byte array from biginteger and before decompression the byte array.
In Java, it not needed.
Does anyone know why I'm getting the "Unexpected end of data" error message when un-gzipping the gzip file?
To verify the bytes data is not corrupted, I use the FooTest4.csv to write to file and was able to opened the file successfully.
Both 'FooTest3.csv.gzand 'FooTest2.csv.gz ran into "Unexpected end of data" when un-gzipping.
public static List<byte> CompressFile(List<byte> parmRawBytes)
{
//Initialize variables...
List<byte> returnModifiedBytes = null;
File.WriteAllBytes(#"X:\FooTest4.csv", parmRawBytes.ToArray());
using (var memoryStream = new MemoryStream())
{
using (var gzipStream = new GZipStream(memoryStream, CompressionMode.Compress, false))
{
gzipStream.Write(parmRawBytes.ToArray(), 0, parmRawBytes.ToArray().Length);
gzipStream.Flush();
File.WriteAllBytes(#"X:\FooTest3.csv.gz", memoryStream.ToArray());
returnModifiedBytes = memoryStream.ToArray().ToList();
}
}
File.WriteAllBytes(#"X:\FooTest2.csv.gz", returnModifiedBytes.ToArray());
return returnModifiedBytes;
}
GZipStream needs to be closed so it can write some terminating data to the end of the buffer to complete the gzip encoding.
byte[] inputBytes = ...;
using (var compressedStream = new MemoryStream())
{
using (var compressor = new GZipStream(compressedStream, CompressionMode.Compress))
{
compressor.Write(inputBytes, 0, inputBytes.Length);
}
// get bytes after the gzip stream is closed
File.WriteAllBytes(pathToFile, compressedStream.ToArray());
}
Instead of loading the bytes, compressing and saving them you could do do compression and writing at once. Also I don't know why you're using List<Byte> instead of byte[], maybe this could be it.
void CompressFile(string inputPath, string outputPath)
{
Stream readStream = new FileStream(inputPath, Filemode.Open);
Stream writeStream = new FileStream(outputPath, FileMode.Create);
Stream compressionStream = new GZipStream(writeStream. CompressionMode.Compress);
byte[] data = new byte[readStream.Length];
readStream.Read(data, 0, data.Length);
compressionStream.Write(data, 0, data.Length);
readStream.Close();
writeStream.Close();
}
byte[] CompressFile(string inputPath)
{
byte[] data = File.ReadAllBytes(inputPath);
MemoryStream memStream = new MemoryStream(data);
var gzipStream = new GZipStream(memStream, CompressionMode.Compress);
gzipStream.Write(data, 0, data.Length);
gzipStream.Close();
return gzipStream.ToArray();
}
PS: I wrote the code in the text editor, so there might be errors. Also you say the error is on the "unzippiing", why no show us the unzip code?
I have following piece of code, it writes some data but not all. What can cut my data ?
using (var stream = new MemoryStream())
{
XmlTextWriter xwr = new XmlTextWriter(stream, Encoding.UTF8);
service.getXElement.WriteTo(xwr);
byte[] bytes = stream.ToArray();
writeStream.Write(bytes, 0, bytes.Length);
}
I have json string something like this
[ { "SectionDataId": "85247","Product": "0.00"},{ "SectionDataId": "85249","Product": "12.38"},{ "SectionDataId": "85264","Product": "0.00"}]
Which I have compressed using C# code found on this forum
private string encode(string json)
{
byte[] buffer = Encoding.UTF8.GetBytes(json);
MemoryStream ms = new MemoryStream();
using (GZipStream zip = new GZipStream(ms, CompressionMode.Compress, true))
{
zip.Write(buffer, 0, buffer.Length);
}
ms.Position = 0;
MemoryStream outStream = new MemoryStream();
byte[] compressed = new byte[ms.Length];
ms.Read(compressed, 0, compressed.Length);
byte[] gzBuffer = new byte[ms.Length + 4];
System.Buffer.BlockCopy(compressed, 0, gzBuffer, 4, compressed.Length);
System.Buffer.BlockCopy(BitConverter.GetBytes(buffer.Length), 0, gzBuffer, 0, 4);
return Convert.ToBase64String(gzBuffer);
}
This output string I have given as web service output so i can use this string in javascript. but I was not able to uncompress the string using javascript.I there any javascript library or code to solve this problem.
I am trying to compress data using the zlib .net library. Regardless of the content of the uncompressed string I only seem to get two bytes of data in the raw[].
{
string uncompressed = "1234567890";
byte[] data = UTF8Encoding.Default.GetBytes(uncompressed);
MemoryStream input = new MemoryStream(data);
MemoryStream output = new MemoryStream();
Stream outZStream = new ZOutputStream(output,zlibConst.Z_DEFAULT_COMPRESSION);
CopyStream(input, outZStream);
output.Seek(0, SeekOrigin.Begin);
byte[] raw = output.ToArray();
string compressed = Convert.ToBase64String(raw);
}
public void CopyStream(System.IO.Stream input, System.IO.Stream output)
{
byte[] buffer = new byte[2000];
int len;
while ((len = input.Read(buffer, 0, 2000)) > 0)
{
output.Write(buffer, 0, len);
}
output.Flush();
}
The problem here is that the ZOutputStream actually writes some of the information into the stream in the finish() method (which is called by Close). The Close method also closes the base stream, so that is not much use in this situation.
Changing the code to the following should work:
{
string uncompressed = "1234567890";
byte[] data = UTF8Encoding.Default.GetBytes(uncompressed);
MemoryStream input = new MemoryStream(data);
MemoryStream output = new MemoryStream();
ZOutputStream outZStream = new ZOutputStream(output,zlibConst.Z_DEFAULT_COMPRESSION);
CopyStream(input, outZStream);
outZStream.finish();
output.Seek(0, SeekOrigin.Begin);
byte[] raw = output.ToArray();
string compressed = Convert.ToBase64String(raw);
}
public void CopyStream(System.IO.Stream input, System.IO.Stream output)
{
byte[] buffer = new byte[2000];
int len;
while ((len = input.Read(buffer, 0, 2000)) > 0)
{
output.Write(buffer, 0, len);
}
output.Flush();
}