Develop Reference

Use compressed gzip file with UploadBlobAsync method

I want to compress a json string and save it als gzip file. I have the following code below which works but only saving it as a .json file. Using compression the files are smaller. If I compress the json string and change the filename to .gz I cannot open it using 7zip.
string containerEndpoint = string.Format("https://xxx.blob.core.windows.net/datafolder");
BlobContainerClient containerClient = new BlobContainerClient(new Uri(containerEndpoint), null);
byte[] byteArray = Encoding.ASCII.GetBytes(jsonString);
MemoryStream stream = new MemoryStream(byteArray);
string fileName = "test.gz";
string blobPath = string.Format("folder/{0}", fileName);
await containerClient.UploadBlobAsync(blobPath, stream);

I added your suggestions and it worked. My code is now as follows:
byte[] buffer = Encoding.UTF8.GetBytes(json);
using (var stream = new MemoryStream())
{
using (var gZipStream = new GZipStream(stream, CompressionMode.Compress, true))
{
gZipStream.Write(buffer, 0, buffer.Length);
}
var compressedData = new byte[stream.Length];
stream.Read(compressedData, 0, compressedData.Length);
var gZipBuffer = new byte[compressedData.Length + 4];
Buffer.BlockCopy(compressedData, 0, gZipBuffer, 4, compressedData.Length);
Buffer.BlockCopy(BitConverter.GetBytes(buffer.Length), 0, gZipBuffer, 0, 4);
stream.Position = 0;
string blobPath = string.Format("folder/{0}", fileName);
await containerClient.UploadBlobAsync(blobPath, stream);
}

Deserializing a byte array in multiple pieces

I have this function that splits a serialized byte array into pieces based on a max length:
public static List<byte[]> ZipStringList(List<string> curSpawn, int maxSize = -1) {
Environment.SetEnvironmentVariable("MONO_REFLECTION_SERIALIZER", "yes");
MemoryStream o = new MemoryStream();
BinaryFormatter bf = new BinaryFormatter();
bf.Serialize(o, curSpawn);
byte[] buffer = o.GetBuffer();
o.Close();
MemoryStream memoryStream = new MemoryStream();
using (GZipStream gZipStream = new GZipStream(memoryStream, CompressionMode.Compress, true))
{
gZipStream.Write(buffer, 0, buffer.Length);
}
memoryStream.Position = 0;
if (maxSize > 0 && memoryStream.Length > maxSize) {
int pieces = Mathf.CeilToInt (memoryStream.Length / (float)maxSize);
List<byte[]> listByte = new List<byte[]> ();
for (int i = 0; i < pieces; ++i) {
int curLength = (int)Mathf.Min (maxSize, memoryStream.Length - (i * maxSize));
byte[] compressedData = new byte[curLength];
memoryStream.Read (compressedData, 0, compressedData.Length);
if (i == pieces - 1) {
byte[] gZipBuffer = new byte[compressedData.Length + 4];
Buffer.BlockCopy (compressedData, 0, gZipBuffer, 4, compressedData.Length);
Buffer.BlockCopy (BitConverter.GetBytes (buffer.Length), 0, gZipBuffer, 0, 4);
listByte.Add (gZipBuffer);
}//if last piece
else {
listByte.Add (compressedData);
}//else normal
}//for each piece
return listByte;
} else {
byte[] compressedData = new byte[memoryStream.Length];
memoryStream.Read (compressedData, 0, compressedData.Length);
byte[] gZipBuffer = new byte[compressedData.Length + 4];
Buffer.BlockCopy (compressedData, 0, gZipBuffer, 4, compressedData.Length);
Buffer.BlockCopy (BitConverter.GetBytes (buffer.Length), 0, gZipBuffer, 0, 4);
List<byte[]> retList = new List<byte[]> ();
retList.Add (gZipBuffer);
return retList;
}
}//ZipStringList()
Then this function that deserializes it:
public static List<string> UnzipStringList(List<byte[]> byteList) {
byte[] gZipBuffer = byteList.SelectMany(b => b).ToArray();
Environment.SetEnvironmentVariable("MONO_REFLECTION_SERIALIZER", "yes");
byte[] buffer;
using (MemoryStream memoryStream = new MemoryStream())
{
int dataLength = BitConverter.ToInt32(gZipBuffer, 0);
memoryStream.Write(gZipBuffer, 4, gZipBuffer.Length - 4);
buffer = new byte[dataLength];
memoryStream.Position = 0;
using (GZipStream gZipStream = new GZipStream(memoryStream, CompressionMode.Decompress))
{
gZipStream.Read(buffer, 0, buffer.Length);
}
}
MemoryStream o = new MemoryStream(buffer);
BinaryFormatter bf = new BinaryFormatter();
return (List<string>)bf.Deserialize(o);
}//UnzipStringList()
When I run it though, I'm getting a "corrupted gzip header" error, so there's some simple thing I'm missing. What is it?
More details: there aren't really, I'm likely positioning the buffer wrong. Fun fact: I searched google for the error message and the only result was this question.
Here's some code you can test with. Make it a large enough file (like at least a kilobyte). My file is full of JSON arrays.
string[] allLines = File.ReadAllLines ("path_to_file.txt");
List<byte[]> byteArray = Utility.ZipStringList(new List<string>(allLines), 60000);
List<string> stringList = Utility.UnzipStringList (byteArray);
Utility.LogRealError ("I unzipped it?", stringList.Count, stringList [4]);

C# GZipStream suddenly not working

I was using GZipStream for some time without any problems. I didn't change any code at my function but now the compress and decompress doesn't work. I have extracted the following simple example (maybe I do something wrong generally...).
public static class ZipLib
{
#region Public Methods
public static string Unzip(byte[] compressedText)
{
try {
byte[] gzBuffer = compressedText;
int msgLength = BitConverter.ToInt32(gzBuffer, 0);
byte[] buffer = new byte[msgLength];
using (MemoryStream ms = new MemoryStream()) {
ms.Write(gzBuffer, 4, gzBuffer.Length - 4);
using (GZipStream zip = new GZipStream(ms, CompressionMode.Decompress, true)) {
zip.Read(buffer, 0, buffer.Length);
}
}
return Encoding.UTF8.GetString(buffer);
}
catch (Exception ex) {
Debug.WriteLine(ex.Message);
}
return "";
}
public static byte[] Zip(string text)
{
byte[] buffer = Encoding.UTF8.GetBytes(text);
MemoryStream ms = new MemoryStream();
using (GZipStream zip = new GZipStream(ms, CompressionMode.Compress, true)) {
zip.Write(buffer, 0, buffer.Length);
}
ms.Position = 0;
byte[] compressed = new byte[ms.Length];
ms.Read(compressed, 0, compressed.Length);
byte[] gzBuffer = new byte[compressed.Length + 4];
Buffer.BlockCopy(compressed, 0, gzBuffer, 4, compressed.Length);
Buffer.BlockCopy(BitConverter.GetBytes(buffer.Length), 0, gzBuffer, 0, 4);
return gzBuffer;
}
#endregion
}
internal class Program
{
#region Private Methods
private static void Main(string[] args)
{
string uncompressedIn = "Hallo Welt";
var compressed = ZipLib.Zip(uncompressedIn);
var uncompressed = ZipLib.Unzip(compressed);
Console.WriteLine("Uncompressed Input: " + uncompressedIn);
Console.WriteLine("Compressed value: " + string.Join(" | ", compressed));
Console.WriteLine("Uncompressed from compressed: " + string.Join(" | ", uncompressed.ToCharArray()));
Console.ReadKey();
}
#endregion
}
The value of uncompressed is "\0\0\0\0"...

In Unzip, you Write to a MemoryStream, then give it to the GZipStream to read from, but: it is still at the end of the stream, you didn't rewind it. Try adding:
ms.Position = 0;
after the Write. Note, however, that it might be more efficient to do:
using(MemoryStream ms = new MemoryStream(gzBuffer, 4, gzBuffer.Length - 4))
{
//...
}
this avoids duplicating any memory

C# Error - Invalid character in a Base-64 string

I am writing a C# assembly to take advantage of the GZip functionality in newer versions of the .NET framework, and I am copying code I have used successfully elsewhere.
The way this works is that an application we are using feeds a stream to the assembly, the assembly reads it, compresses it, then returns the compressed result as a string. That string is then put back into the assembly through a stream to uncompress it (normally it will be stored, but I am simply running a basic test).
However, when I feed the string back to the assembly, it errors out while reading it from a byte array that is read from the application's stream. Here is the code:
private void ReadStream(IStream stream, out byte[] data)
{
using (MemoryStream writer = new MemoryStream())
{
IntPtr rwBytes = Marshal.AllocHGlobal(4);
int _rwBytes = 0;
data = new byte[0xafc8];
do
{
stream.Read(data, 0xafc8, rwBytes);
_rwBytes = Marshal.ReadInt32(rwBytes);
writer.Write(data, 0, _rwBytes);
}
while (_rwBytes > 0);
writer.Close();
Marshal.FreeHGlobal(rwBytes);
}
}
public string CompressString([In, MarshalAs(UnmanagedType.AsAny)] object appStream)
{
byte[] buffer = null;
ReadStream(appStream as IStream, out buffer);
MemoryStream ms = new MemoryStream();
using (GZipStream zip = new GZipStream(ms, CompressionMode.Compress, true))
{
zip.Write(buffer, 0, buffer.Length);
}
ms.Position = 0;
MemoryStream outStream = new MemoryStream();
byte[] compressed = new byte[ms.Length];
ms.Read(compressed, 0, compressed.Length);
byte[] gzBuffer = new byte[compressed.Length + 4];
System.Buffer.BlockCopy(compressed, 0, gzBuffer, 4, compressed.Length);
System.Buffer.BlockCopy(BitConverter.GetBytes(buffer.Length), 0, gzBuffer, 0, 4);
string str = Convert.ToBase64String(gzBuffer);
gzBuffer = Convert.FromBase64String(str);
return str;
}
public string DecompressString([In, MarshalAs(UnmanagedType.AsAny)] object appStream)
{
byte[] buffer = null;
ReadStream(appStream as IStream, out buffer);
string compressedText = Encoding.UTF8.GetString(buffer);
byte[] gzBuffer = Convert.FromBase64String(compressedText);
using (MemoryStream ms = new MemoryStream())
{
int msgLength = BitConverter.ToInt32(gzBuffer, 0);
ms.Write(gzBuffer, 4, gzBuffer.Length - 4);
buffer = new byte[msgLength];
ms.Position = 0;
using (GZipStream zip = new GZipStream(ms, CompressionMode.Decompress))
{
zip.Read(buffer, 0, buffer.Length);
}
}
return Encoding.UTF8.GetString(buffer);
}
The string is returned from the CompressString function as:
yK8AAB+LCAAAAAAABADtvQdgHEmWJSYvbcp7f0r1StfgdKEIgGATJNiQQBDswYjN5pLsHWlHIymrKoHKZVZlXWYWQMztnbz33nvvvffee++997o7nU4n99//P1xmZAFs9s5K2smeIYCqyB8/fnwfPyK+ndd5WjRplk6rxarOm6aolmmbN+04TZ/n7cdN2uTU4jwt2vSqqt8241/jR8+Pnh89P3p+9Pzo+dHzo+dHz4+eHz0/en70/Oj50fOj50fPj54fPT96fvT86PnR86PnR8+Pnh89P3p+9Pzo+dHz/67nR9j86PnR86PnR8+Pnh89P3p+9Pzo+dHzo+dHz4+eHz0/en70/Oj50fOj50fPj54fPT96fvT86PnR86PnR8+Pnh89/99/fjSCHz0/en70/Oj50fOj50fPj54fPT96fvT86PnR86PnR8+Pnh89P3p+9Pzo+dHzo+dHz4+eHz0/en70/Oj50fOj5//7z49G8KPnR8+Pnh89P3p+9Pzo+dHzo+dHz4+eHz0/en70/Oj50fOj50fPj54fPT96fvT86PnR86PnR8+Pnh89P3p+9Px///nRCH70/Oj50fOj50fPj54fPT96fvT86PnR86PnR8+Pnh89P3p+9Pzo+dHzo+dHz4+eHz0/en70/Oj50fOj50fPj57/7z8/GsGPnh89P3p+9Pzo+dHzo+dHz4+eHz0/en70/Oj50fP/nef/AYk/UbjIrwAA
When it is read back from the stream in the Encoding.UTF8.GetString call, it is:
yK8AAB+LCAAAAAAABADtvQdgHEmWJSYvbcp7f0r1StfgdKEIgGATJNiQQBDswYjN5pLsHWlHIymrKoHKZVZlXWYWQMztnbz33nvvvffee++997o7nU4n99//P1xmZAFs9s5K2smeIYCqyB8/fnwfPyK+ndd5WjRplk6rxarOm6aolmmbN+04TZ/n7cdN2uTU4jwt2vSqqt8241/jR8+Pnh89P3p+9Pzo+dHzo+dHz4+eHz0/en70/Oj50fOj50fPj54fPT96fvT86PnR86PnR8+Pnh89P3p+9Pzo+dHz/67nR9j86PnR86PnR8+Pnh89P3p+9Pzo+dHzo+dHz4+eHz0/en70/Oj50fOj50fPj54fPT96fvT86PnR86PnR8+Pnh89/99/fjSCHz0/en70/Oj50fOj50fPj54fPT96fvT86PnR86PnR8+Pnh89P3p+9Pzo+dHzo+dHz4+eHz0/en70/Oj50fOj5//7z49G8KPnR8+Pnh89P3p+9Pzo+dHzo+dHz4+eHz0/en70/Oj50fOj50fPj54fPT96fvT86PnR86PnR8+Pnh89P3p+9Px///nRCH70/Oj50fOj50fPj54fPT96fvT86PnR86PnR8+Pnh89P3p+9Pzo+dHzo+dHz4+eHz0/en70/Oj50fOj50fPj57/7z8/GsGPnh89P3p+9Pzo+dHzo+dHz4+eHz0/en70/Oj50fP/nef/AYk/UbjIrwAA
These strings are identical. I can even convert it back and forth in the CompressString function with no problem. Any thoughts out there? This is very strange.

Don't use UTF8, you should use the ascii text encoding instead. When you get the bytes from your string, you'll end up with some extra bytes due to the way UTF strings are encoded.

ZLib decompression

I am trying to compress data using the zlib .net library. Regardless of the content of the uncompressed string I only seem to get two bytes of data in the raw[].
{
string uncompressed = "1234567890";
byte[] data = UTF8Encoding.Default.GetBytes(uncompressed);
MemoryStream input = new MemoryStream(data);
MemoryStream output = new MemoryStream();
Stream outZStream = new ZOutputStream(output,zlibConst.Z_DEFAULT_COMPRESSION);
CopyStream(input, outZStream);
output.Seek(0, SeekOrigin.Begin);
byte[] raw = output.ToArray();
string compressed = Convert.ToBase64String(raw);
}
public void CopyStream(System.IO.Stream input, System.IO.Stream output)
{
byte[] buffer = new byte[2000];
int len;
while ((len = input.Read(buffer, 0, 2000)) > 0)
{
output.Write(buffer, 0, len);
}
output.Flush();
}

The problem here is that the ZOutputStream actually writes some of the information into the stream in the finish() method (which is called by Close). The Close method also closes the base stream, so that is not much use in this situation.
Changing the code to the following should work:
{
string uncompressed = "1234567890";
byte[] data = UTF8Encoding.Default.GetBytes(uncompressed);
MemoryStream input = new MemoryStream(data);
MemoryStream output = new MemoryStream();
ZOutputStream outZStream = new ZOutputStream(output,zlibConst.Z_DEFAULT_COMPRESSION);
CopyStream(input, outZStream);
outZStream.finish();
output.Seek(0, SeekOrigin.Begin);
byte[] raw = output.ToArray();
string compressed = Convert.ToBase64String(raw);
}
public void CopyStream(System.IO.Stream input, System.IO.Stream output)
{
byte[] buffer = new byte[2000];
int len;
while ((len = input.Read(buffer, 0, 2000)) > 0)
{
output.Write(buffer, 0, len);
}
output.Flush();
}