Develop Reference

GzipStream blockwise compression results in a corrupted file

I am trying to implement block-by-block compression using GzipStream class. .NET Core 3.1, Visual Studio 2019, Console App. My OS is Windows 10. I believe it should be possible because gzip files consist of independent blocks one after another as per format specification. But resulting files that I get are corrupted. Here's the code:
using System;
using System.Globalization;
using System.IO;
using System.IO.Compression;
namespace GzipToMemoryStreamExample
{
class Program
{
private static int blockSize;
private static string sourceFileName = "e:\\SomeFolder\\SomeFile.ext";
private static byte[] currentBlock;
private static FileStream readingStream;
private static FileStream writingStream;
static void Main(string[] args)
{
Console.WriteLine("Enter block size:");
string blockSizeStr = Console.ReadLine();
int.TryParse(blockSizeStr, out blockSize);
readingStream = new FileStream(sourceFileName, FileMode.Open);
string resultingFileName = Path.ChangeExtension(sourceFileName, ".gz");
CreateAndOpenResultingFile(resultingFileName);
while (ReadBlock())
{
byte[] processedBlock = ProcessBlock(currentBlock);
writingStream.Write(processedBlock, 0, processedBlock.Length);
}
readingStream.Dispose();
writingStream.Dispose();
Console.WriteLine("Finished.");
Console.ReadKey();
}
private static bool ReadBlock()
{
bool result;
int bytesRead;
currentBlock = new byte[blockSize];
bytesRead = readingStream.Read(currentBlock, 0, blockSize);
result = bytesRead > 0;
return result;
}
private static byte[] ProcessBlock(byte[] sourceData)
{
byte[] result;
using (var outputStream = new MemoryStream())
{
using var compressionStream = new GZipStream(outputStream, CompressionMode.Compress);
compressionStream.Write(sourceData, 0, sourceData.Length);
result = outputStream.ToArray();
}
return result;
}
private static void CreateAndOpenResultingFile(string fileName)
{
if (File.Exists(fileName))
{
File.Delete(fileName);
}
writingStream = File.Create(fileName);
}
}
}
When I look at resulting files I see that result somehow depends on block size I choose. if it's smaller than ~100 Kb, resulting "compressed" blocks are of 10 bytes size each, which leads to extremely small useless file. If size of block is greater than ~100 Kb, then the size of file becomes reasonably large, about 80% of the original, but still corrupted.
Also I checked the block headers and it turns out they're strange. OS is set to TOPS-20 (0x0a value), ISIZE at the end of block is always totally wrong.
What is my mistake?

It's solved just with moving result = outputStream.ToArray(); line out of the compressionStream using scope as Mark Adler suggested in the comments.
private static byte[] ProcessBlock(byte[] sourceData)
{
byte[] result;
using (var outputStream = new MemoryStream())
{
using (var compressionStream = new GZipStream(outputStream, CompressionMode.Compress))
{
compressionStream.Write(sourceData, 0, sourceData.Length);
}
result = outputStream.ToArray();
}
return result;
}

Why does same static method behaves different in different controllers?

I have a public static method in Invoice.cs class:
public static string CalculateHash(Stream image)
{
using (var sha = SHA256.Create())
{
image.Seek(0, SeekOrigin.Begin);
var hash = sha.ComputeHash(image);
var hashStr = BitConverter.ToString(hash).Replace("-", "").ToLowerInvariant();
return hashStr;
}
}
I have two controllers: HookController.cs and DataController.cs. I try to check if same hash was created for same image, But, hashes which are created by these controllers are not same for same image.

I solved this way.I copied to a new memory stream.

Getting the same Hash value using the below code. Tested it on .Net Framework 4.6.1 console app.
class Program
{
static void Main(string[] args)
{
var fileStream = new FileStream(#"D:\Mukesh\Mukesh.jpg", FileMode.Open);
var result = CalculateHash(fileStream);
Console.ReadKey();
}
public static string CalculateHash(Stream image)
{
using (var sha = SHA256.Create())
{
image.Seek(0, SeekOrigin.Begin);
var hash = sha.ComputeHash(image);
var hashStr = BitConverter.ToString(hash).Replace("-", "").ToLowerInvariant();
return hashStr;
}
}

Testing checksum by modifying a file

A program in C# which copy a file or whole folders to another folder is made and in that application checksum SHA-512 has been used to verify that input and output of the copy process is/are identical, the program works fine but I need to test the whole program and especially test or verify the checksum. how can I, give the program an input e.g. a file and in the process modify the file somehow in order to see that checksum detect that error? thanks for your suggestions

Here's a simple example of testing the SHA512 hash. Here we have two tests, TestSHA512Modify and TestSHA512Append. One modifies bytes within the file, open appends bytes to the file. Both are useful tests of the Hash.
static void TestSHA512Modify()
{
var testFile = Path.GetTempFileName();
CreateRandomFile(testFile, 1024);
var sha12 = GetFileSHA512(testFile);
Console.WriteLine("TestSHA12Modify: Original file SHA512: " + ToHexString(sha12));
// Modify file bytes. Here we set byte offset [100] [101] [102]
WriteBytes(testFile, 100, new byte[] { 1, 2, 3 });
var modifiedSha12 = GetFileSHA512(testFile);
Console.WriteLine("TestSHA12Modify: Updated file SHA512: " + ToHexString(modifiedSha12));
Console.WriteLine("TestSHA12Modify: SHA12 Hashes are: " + (sha12.SequenceEqual(modifiedSha12) ? "EQUAL" : "NOT EQUAL"));
}
static void TestSHA512Append()
{
var testFile = Path.GetTempFileName();
CreateRandomFile(testFile, 1024);
var sha12 = GetFileSHA512(testFile);
Console.WriteLine("TestSHA12Append: Original file SHA512: " + ToHexString(sha12));
// Append bytes to the end of a file
AppendBytes(testFile, new byte[] { 1 });
var modifiedSha12 = GetFileSHA512(testFile);
Console.WriteLine("TestSHA12Append: Updated file SHA512: " + ToHexString(modifiedSha12));
Console.WriteLine("TestSHA12Append: SHA12 Hashes are: " + (sha12.SequenceEqual(modifiedSha12) ? "EQUAL" : "NOT EQUAL"));
}
static void CreateRandomFile(string path, int length)
{
// Make some random bytes.
var randomData = new byte[1024];
RNGCryptoServiceProvider p = new RNGCryptoServiceProvider();
p.GetBytes(randomData);
File.WriteAllBytes(path, randomData);
}
static void WriteBytes(string path, int fileOffset, byte[] data)
{
using (var fileStream = new FileStream(path, FileMode.Open))
{
fileStream.Seek(fileOffset, SeekOrigin.Begin);
fileStream.Write(data, 0, data.Length);
}
}
static void AppendBytes(string path, byte[] data)
{
using (var fileStream = new FileStream(path, FileMode.Append))
{
fileStream.Write(data, 0, data.Length);
}
}
static byte[] GetFileSHA512(string path)
{
using (SHA512 sha = new SHA512Managed())
{
return sha.ComputeHash(File.ReadAllBytes(path));
}
}
static string ToHexString(byte[] data)
{
return string.Join("", data.Select(b => b.ToString("X2")));
}

Saving the PgpPublicKey to string

I am generating a public key and secret Key and want to save the public key in a string field.Saving it to the file works fine. but how do I get it into a string...
private string SaveAndReturn(PgpPublicKey publicKey, bool armor = true)
{
var fileStream = new FileInfo($#"C:\encrypt\Keys\pub2.asc").OpenWrite() as Stream;
SaveFile(publicKey, fileStream, armor);
fileStream.Close();
var keyString = GetStringFromKey(publicKey);
return keyString;
}
public void SaveFile(PgpPublicKey publicKey, Stream outStream, bool armor = true)
{
if (armor)
{
outStream = new ArmoredOutputStream(outStream);
}
publicKey.Encode(outStream);
outStream.Close();
}
private string GetStringFromKey(PgpPublicKey publicKey)
{
var armor = true;
var stream = new MemoryStream() as Stream;
publicKey.Encode(stream);
stream.Position = 0;
var keyString = GetString(stream);
stream.Close();
Console.WriteLine(keyString);
return keyString;
}
private string GetString(Stream theStream, Encoding encoding = null)
{
encoding = encoding ?? Encoding.UTF8;
using (var reader = new StreamReader(theStream, encoding))
{
return reader.ReadToEnd();
}
}
The pub2.asc file is created fine and has the in there. Instead of saving the key into a file, I want it into a string.. and the following code gives me an empty string. What am I doing wrong?
So, I updated the code to set the position before reading the stream. But the string returned is not the same as the text I see in file.
I think ArmoredOutputStream does Base64 encoding. How to I get the same string as in text file
String from the above code:
???z}?.?u?l:?|?5 ?test?? Wh?? $???_??cH???L~??+??Bc$?rN??g?.??m?s????0?Z???k?-??G??mzBQ$
U?E??V|/?,l???=???p???? ??5?%%?9??ae?S?|8Lh?\???|pB??G*?8????N9'???+(?|R??Q#R??_9D
If I use ASCII encoding, the string is
??Wh{? ??;cgx??hO)???C?~????t?roS?4??:??????E?fF?U?r?+?=?p???#L?????a??B}????-.o??q?i5???Ff.??6???????6r?HdX??}?S???4?(j)3 ?test?? Wh?
?U?+U???????????v??????BJEwp*0?'?lz2??6??P,A???Q??2M??3?a?W??9???*????M%?|?U^>??????q?&??V??N
Text from Pub2.asc file
-----BEGIN PGP PUBLIC KEY BLOCK-----
Version: BCPG C# v1.8.1.0
mIsEV2h5lwEEAJakqTE+XcRBQ0b2P2vjNuDrgi9IcCsgEGdD3bpH6d1W8CSR+YkZ
EV/PwmNI1dyh8Ex+zqHQKwaPCIfEQhNjF2sIJKVyTtjpZ4YukoRto3PBjdse0zAB
2Freiv7Ha9ct0wSQRwID2haKbXpCUSQN5/j8xbx90y60ddxsOoV8C7A1AAUTtAR0
ZXN0iJwEEAECAAYFAldosdcACgkQPIdhk1pxnGAhxwQAg4NRYbuSDg5WVw3G1tVm
1xM746+O5J6B5RHnaQvFePpVVUm/tpLyCbHPNeYlJao5FeHRYWWpU4d8OExo4phc
iv7JfHBCx9tHKso4uccS3ROaTjknw+jwuLErKLN8UvjaUUBSn8wVXzlEDVXSRYCt
Vnwv/Sxsxe7mgT2qBJSYcKY=
=PYEC
-----END PGP PUBLIC KEY BLOCK-----

The position of the MemoryStream is still set to the end of the string you wrote into it. You need to reset the position to 0 before you try to read from it again. You can do this in the calling code or in GetString() directly. Here is the fix in the calling code:
private string GetStringFromKey(PgpPublicKey publicKey)
{
var armor = true;
var stream = new MemoryStream() as Stream;
publicKey.Encode(stream);
-->stream.Position = 0;
var keyString = GetString(stream);
stream.Close();
Console.WriteLine(keyString);
return keyString;
}
You may also find some useful information over at How do you get a string from a MemoryStream?

Decompress string in java from compressed string in C#

I was searching for the correct solution to decompress the string in java coming from c# code.I tried myself with lot of techniques in java like(gzip,inflatter etc.).but didn't get the solution.i got some error while trying to decompress the string in java from compressed string from c# code.
My C# code to compress the string is,
public static string CompressString(string text)
{
byte[] byteArray = Encoding.GetEncoding(1252).GetBytes(text);// Encoding.ASCII.GetBytes(text);
using (var ms = new MemoryStream())
{
// Compress the text
using (var ds = new DeflateStream(ms, CompressionMode.Compress))
{
ds.Write(byteArray, 0, byteArray.Length);
}
return Convert.ToBase64String(ms.ToArray());
}
}
And decompress the string in java using,
private static void compressAndDecompress(){
try {
// Encode a String into bytes
String string = "xxxxxxSAMPLECOMPRESSEDSTRINGxxxxxxxxxx";
// // Compress the bytes
byte[] decoded = Base64.decodeBase64(string.getBytes());
byte[] output = new byte[4096];
// Decompress the bytes
Inflater decompresser = new Inflater();
decompresser.setInput(decoded);
int resultLength = decompresser.inflate(output);
decompresser.end();
// Decode the bytes into a String
String outputString = new String(output, 0, resultLength, "UTF-8");
System.out.println(outputString);
} catch(java.io.UnsupportedEncodingException ex) {
ex.printStackTrace();
} catch (java.util.zip.DataFormatException ex) {
ex.printStackTrace();
}
}
I get this exception when running the above code:
java.util.zip.DataFormatException: incorrect header check
Kindly give me the sample code in java to decompress the string java.Thanks

My C# code to compress is
private string Compress(string text)
{
byte[] buffer = Encoding.UTF8.GetBytes(text);
MemoryStream ms = new MemoryStream();
using (GZipStream zip = new GZipStream(ms, CompressionMode.Compress, true))
{
zip.Write(buffer, 0, buffer.Length);
}
ms.Position = 0;
MemoryStream outStream = new MemoryStream();
byte[] compressed = new byte[ms.Length];
ms.Read(compressed, 0, compressed.Length);
byte[] gzBuffer = new byte[compressed.Length + 4];
System.Buffer.BlockCopy(compressed, 0, gzBuffer, 4, compressed.Length);
System.Buffer.BlockCopy(BitConverter.GetBytes(buffer.Length), 0, gzBuffer, 0, 4);
return Convert.ToBase64String(gzBuffer);
}
Java code to decompress the text is
private String Decompress(String compressedText)
{
byte[] compressed = compressedText.getBytes("UTF8");
compressed = org.apache.commons.codec.binary.Base64.decodeBase64(compressed);
byte[] buffer=new byte[compressed.length-4];
buffer = copyForDecompression(compressed,buffer, 4, 0);
final int BUFFER_SIZE = 32;
ByteArrayInputStream is = new ByteArrayInputStream(buffer);
GZIPInputStream gis = new GZIPInputStream(is, BUFFER_SIZE);
StringBuilder string = new StringBuilder();
byte[] data = new byte[BUFFER_SIZE];
int bytesRead;
while ((bytesRead = gis.read(data)) != -1)
{
string.append(new String(data, 0, bytesRead));
}
gis.close();
is.close();
return string.toString();
}
private byte[] copyForDecompression(byte[] b1,byte[] b2,int srcoffset,int dstoffset)
{
for(int i=0;i<b2.length && i<b1.length;i++)
{
b2[i]=b1[i+4];
}
return b2;
}
This code works perfectly fine for me.

Had exactly the same issue. Could solve it via
byte[] compressed = Base64Utils.decodeFromString("mybase64encodedandwithc#zippedcrap");
Inflater decompresser = new Inflater(true);
decompresser.setInput(compressed);
byte[] result = new byte[4096];
decompresser.inflate(result);
decompresser.end();
System.out.printf(new String(result));
The magic happens with the boolen parameter on instantiating the Inflator
BW Hubert

For beloved googlers,
As #dbw mentioned,
according to post How to decompress stream deflated with java.util.zip.Deflater in .NET?,
java.util.zip.deflater equivalent in c# the default deflater used in C#
is not having any java equivalent that's why users prefer Gzip, Ziplib
or some other zip techniques.
a relatively simple method would be using GZip.
And for the accepted answer, one problem is that in this method you should append the data size to the compressed string yourself, and more importantly as per my own experience in our production app, It is buggy when the string reaches ~2000 chars!
the bug is in the System.io.Compression.GZipStream
any way using SharpZipLib in c# the problem goes away and everything would be as simple as following snippets:
JAVA:
import android.util.Base64;
import com.google.android.gms.common.util.IOUtils;
import org.jetbrains.annotations.Nullable;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;
public class CompressionHelper {
#Nullable
public static String compress(#Nullable String data) {
if(data == null || data.length() == 0)
return null;
try {
// Create an output stream, and a gzip stream to wrap over.
ByteArrayOutputStream bos = new ByteArrayOutputStream(data.length());
GZIPOutputStream gzip = new GZIPOutputStream(bos);
// Compress the input string
gzip.write(data.getBytes());
gzip.close();
byte[] compressed;
// Convert to base64
compressed = Base64.encode(bos.toByteArray(),Base64.NO_WRAP);
bos.close();
// return the newly created string
return new String(compressed);
} catch(IOException e) {
return null;
}
}
#Nullable
public static String decompress(#Nullable String compressedText) {
if(compressedText == null || compressedText.length() == 0)
return null;
try {
// get the bytes for the compressed string
byte[] compressed = compressedText.getBytes("UTF-8");
// convert the bytes from base64 to normal string
compressed = Base64.decode(compressed, Base64.NO_WRAP);
ByteArrayInputStream bis = new ByteArrayInputStream(compressed);
GZIPInputStream gis = new GZIPInputStream(bis);
byte[] bytes = IOUtils.toByteArray(gis);
return new String(bytes, "UTF-8");
}catch (IOException e){
e.printStackTrace();
}
return null;
}
}
and c#:
using ICSharpCode.SharpZipLib.GZip; //PM> Install-Package SharpZipLib
using System;
using System.Collections.Generic;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace GeneralTools
{
public static class CompressionTools
{
public static string CompressString(string text)
{
if (string.IsNullOrEmpty(text))
return null;
byte[] buffer = Encoding.UTF8.GetBytes(text);
using (var compressedStream = new MemoryStream())
{
GZip.Compress(new MemoryStream(buffer), compressedStream, false);
byte[] compressedData = compressedStream.ToArray();
return Convert.ToBase64String(compressedData);
}
}
public static string DecompressString(string compressedText)
{
if (string.IsNullOrEmpty(compressedText))
return null;
byte[] gZipBuffer = Convert.FromBase64String(compressedText);
using (var memoryStream = new MemoryStream())
{
using (var compressedStream = new MemoryStream(gZipBuffer))
{
var decompressedStream = new MemoryStream();
GZip.Decompress(compressedStream, decompressedStream, false);
return Encoding.UTF8.GetString(decompressedStream.ToArray()).Trim();
}
}
}
}
}
you may also find the codes here

If anyone still interested, here's my full solution with outputstream to handle unknown string size. Using C# DeflateStream and Java Inflater (based on Hubert Ströbitzer answer).
C# Compression:
string CompressString(string raw)
{
byte[] uncompressedData = Encoding.UTF8.GetBytes(raw);
MemoryStream output = new MemoryStream();
using (DeflateStream dStream = new DeflateStream(output, CompressionLevel.Optimal))
{
dStream.Write(uncompressedData, 0, uncompressedData.Length);
}
string compressedString = Convert.ToBase64String(output.ToArray());
return compressedString;
}
Java decompress:
String decompressString(String compressedString) {
byte[] compressed = Base64Utils.decodeFromString(compressedString);
Inflater inflater = new Inflater(true);
inflater.setInput(compressed);
//Using output stream to handle unknown size of decompressed string
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
byte[] buffer = new byte[1024];
try {
while(!inflater.finished()){
int count = inflater.inflate(buffer);
outputStream.write(buffer, 0, count);
}
inflater.end();
outputStream.close();
} catch (DataFormatException e) {
//Handle DataFormatException
} catch (IOException e) {
//Handle IOException
}
return outputStream.toString();
}