I currently use the following code to retrieve and decompress string data from Amazon C#:
GetObjectRequest getObjectRequest = new GetObjectRequest().WithBucketName(bucketName).WithKey(key);
using (S3Response getObjectResponse = client.GetObject(getObjectRequest))
{
using (Stream s = getObjectResponse.ResponseStream)
{
using (GZipStream gzipStream = new GZipStream(s, CompressionMode.Decompress))
{
StreamReader Reader = new StreamReader(gzipStream, Encoding.Default);
string Html = Reader.ReadToEnd();
parseFile(Html);
}
}
}
I want to reverse this code so that I can compress and upload string data to S3 without being written to disk. I tried the following, but I am getting an Exception:
using (AmazonS3 client = Amazon.AWSClientFactory.CreateAmazonS3Client(AWSAccessKeyID, AWSSecretAccessKeyID))
{
string awsPath = AWSS3PrefixPath + "/" + keyName+ ".htm.gz";
byte[] buffer = Encoding.UTF8.GetBytes(content);
using (MemoryStream ms = new MemoryStream())
{
using (GZipStream zip = new GZipStream(ms, CompressionMode.Compress))
{
zip.Write(buffer, 0, buffer.Length);
PutObjectRequest request = new PutObjectRequest();
request.InputStream = ms;
request.Key = awsPath;
request.BucketName = AWSS3BuckenName;
using (S3Response putResponse = client.PutObject(request))
{
//process response
}
}
}
}
The exception I am getting is:
Cannot access a closed Stream.
What am I doing wrong?
EDIT:
The exception is occuring on the closing bracket of using (GZipStream zip
Stack trace:
at
System.IO.MemoryStream.Write(Byte[]
buffer, Int32 offset, Int32 count)
at
System.IO.Compression.DeflateStream.Dispose(Boolean
disposing) at
System.IO.Stream.Close() at
System.IO.Compression.GZipStream.Dispose(Boolean
disposing) at
System.IO.Stream.Close()
You need to flush and close the GZipStream and reset the Position of the MemoryStream to 0 before using it as input to the request:
MemoryStream ms = new MemoryStream();
using (GZipStream zip = new GZipStream(ms, CompressionMode.Compress, true))
{
byte[] buffer = Encoding.UTF8.GetBytes(content);
zip.Write(buffer, 0, buffer.Length);
zip.Flush();
}
ms.Position = 0;
PutObjectRequest request = new PutObjectRequest();
request.InputStream = ms;
request.Key = AWSS3PrefixPath + "/" + keyName+ ".htm.gz";
request.BucketName = AWSS3BuckenName;
using (AmazonS3 client = Amazon.AWSClientFactory.CreateAmazonS3Client(
AWSAccessKeyID, AWSSecretAccessKeyID))
using (S3Response putResponse = client.PutObject(request))
{
//process response
}
It might also be possible to use the GZipStream as input if you first fill the MemoryStream with the data, but I've never tried this yet.
Related
I am calling a third party service, which return a pdf file by IO.Stream. I need to change to MemoryStream, and save to a pdf file.
cRequestString = ".....";//You need to set up you own URL here.
//Make the API call
try
{
byte[] bHeaderBytes = System.Text.Encoding.UTF8.GetBytes(GetUserPasswordString()); //user and pa for third party call.
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(cRequestString);
request.Method = WebRequestMethods.Http.Get;
request.PreAuthenticate = true;
request.ContentType = "application/pdf";
request.Accept = "application/pdf";
request.Headers.Add("Authorization", "Basic " + Convert.ToBase64String(bHeaderBytes));
MemoryStream memStream;
WebResponse response = request.GetResponse();
using (Stream stream = response.GetResponseStream())
using (StreamReader reader = new StreamReader(stream))
{
memStream = new MemoryStream();
//read small blocks to show correctly
byte[] buffer = new Byte[1023];
int byteCount;
do
{
byteCount = stream.Read(buffer, 0, buffer.Length);
memStream.Write(buffer, 0, byteCount);
} while (byteCount > 0);
}
memStream.Seek(0, SeekOrigin.Begin);//set position to beginning
return memStream;
}
catch
{
return null;
}
//save MemoryStream to local pdf file
private void SavePDFFile(string cReportName, MemoryStream pdfStream)
{
//Check file exists, delete
if (File.Exists(cReportName))
{
File.Delete(cReportName);
}
using (FileStream file = new FileStream(cReportName, FileMode.Create, FileAccess.Write))
{
byte[] bytes = new byte[pdfStream.Length];
pdfStream.Read(bytes, 0, (int)pdfStream.Length);
file.Write(bytes, 0, bytes.Length);
pdfStream.Close();
}
}
You could do the following.
using (Stream stream = response.GetResponseStream())
using(MemoryStream memStream = new MemoryStream())
{
memStream = new MemoryStream();
stream.CopyTo(memoryStream);
// TODO : Rest of your task
}
More details on Stream.CopyTo on MSDN
Does anyone know why I'm getting the "Unexpected end of data" error message when un-gzipping the gzip file?
To verify the bytes data is not corrupted, I use the FooTest4.csv to write to file and was able to opened the file successfully.
Both 'FooTest3.csv.gzand 'FooTest2.csv.gz ran into "Unexpected end of data" when un-gzipping.
public static List<byte> CompressFile(List<byte> parmRawBytes)
{
//Initialize variables...
List<byte> returnModifiedBytes = null;
File.WriteAllBytes(#"X:\FooTest4.csv", parmRawBytes.ToArray());
using (var memoryStream = new MemoryStream())
{
using (var gzipStream = new GZipStream(memoryStream, CompressionMode.Compress, false))
{
gzipStream.Write(parmRawBytes.ToArray(), 0, parmRawBytes.ToArray().Length);
gzipStream.Flush();
File.WriteAllBytes(#"X:\FooTest3.csv.gz", memoryStream.ToArray());
returnModifiedBytes = memoryStream.ToArray().ToList();
}
}
File.WriteAllBytes(#"X:\FooTest2.csv.gz", returnModifiedBytes.ToArray());
return returnModifiedBytes;
}
GZipStream needs to be closed so it can write some terminating data to the end of the buffer to complete the gzip encoding.
byte[] inputBytes = ...;
using (var compressedStream = new MemoryStream())
{
using (var compressor = new GZipStream(compressedStream, CompressionMode.Compress))
{
compressor.Write(inputBytes, 0, inputBytes.Length);
}
// get bytes after the gzip stream is closed
File.WriteAllBytes(pathToFile, compressedStream.ToArray());
}
Instead of loading the bytes, compressing and saving them you could do do compression and writing at once. Also I don't know why you're using List<Byte> instead of byte[], maybe this could be it.
void CompressFile(string inputPath, string outputPath)
{
Stream readStream = new FileStream(inputPath, Filemode.Open);
Stream writeStream = new FileStream(outputPath, FileMode.Create);
Stream compressionStream = new GZipStream(writeStream. CompressionMode.Compress);
byte[] data = new byte[readStream.Length];
readStream.Read(data, 0, data.Length);
compressionStream.Write(data, 0, data.Length);
readStream.Close();
writeStream.Close();
}
byte[] CompressFile(string inputPath)
{
byte[] data = File.ReadAllBytes(inputPath);
MemoryStream memStream = new MemoryStream(data);
var gzipStream = new GZipStream(memStream, CompressionMode.Compress);
gzipStream.Write(data, 0, data.Length);
gzipStream.Close();
return gzipStream.ToArray();
}
PS: I wrote the code in the text editor, so there might be errors. Also you say the error is on the "unzippiing", why no show us the unzip code?
I'm trying on c# to download a zip file from a webservice and extract an entry in the memory but when I try to read the stream how is in the documentation of the dotnetzip I get the exception "This stream does not support seek operations” in the "ZipFile.Read(stream)" part.
Somebody could tell me what I'm doing wrong? Thanks in advance
urlAuthentication="https://someurl/?login=foo&token=faa"
var request = (HttpWebRequest)WebRequest.Create(urlAuthentication);
request.Proxy = WebRequest.DefaultWebProxy;
request.Credentials = System.Net.CredentialCache.DefaultCredentials; ;
request.Proxy.Credentials = System.Net.CredentialCache.DefaultCredentials;
using (var ms = new MemoryStream())
{
using (var response = (HttpWebResponse)request.GetResponse())
{
using (var stream =response.GetResponseStream())
{
using (ZipFile zipout = ZipFile.Read(stream))
{
ZipEntry entry = zipout["file1.xml"];
entry.Extract(ms);
}
}
}
}
Apparently dotnetzip requires a stream to support seek operations and the response stream of a HttpWebResponse does not support seeking.
You can solve this issue by first downloading the entire file in memory, and then accessing it:
using (var ms = new MemoryStream())
{
using (MemoryStream seekable = new MemoryStream())
{
using (var stream = response.GetResponseStream())
{
int bytes;
byte[] buffer = new byte[1024];
while ((bytes = stream.Read(buffer, 0, buffer.Length)) > 0)
{
seekable.Write(buffer, 0, bytes);
}
}
seekable.Position = 0;
using (ZipFile zipout = ZipFile.Read(seekable))
{
ZipEntry entry = zipout["file1.xml"];
entry.Extract(ms);
}
}
// access ms
}
I have an XML data which I would like to compress it using GZipStream and upload it to webservice. I would like to create the gzip file in memory instead of of creating it in local disk. I have tried the following:
public string class1(string url, string xml)
{
byte[] data = Encoding.ASCII.GetBytes(xml);
MemoryStream memory = new MemoryStream();
GZipStream gzip = new GZipStream(memory, CompressionMode.Compress, true);
gzip.Write(data, 0, data.Length);
byte[] zip=memory.ToArray();
HttpWebRequest wReq = (HttpWebRequest)WebRequest.Create(url);
wReq.Method = "POST";
wReq.ContentType = "application/zip";
var reqStream = wReq.GetRequestStream();
reqStream.Write(zip,0,zip.Length);
reqStream.Close();
var wRes = wReq.GetResponse();
var resStream = wRes.GetResponseStream();
var resgzip = new GZipStream(resStream, CompressionMode.Decompress);
var reader = new StreamReader(resgzip);
var textResponse = reader.ReadToEnd();
reader.Close();
resStream.Close();
wRes.Close();
return textResponse;
}
After writing data to webservice the server unzips the file and processess it. While the server decompresses the data an exception is thrown in server "Premature end of file". Please help me in this.
Add below method to convert Stream to MemoryStream
public static MemoryStream Read(Stream stream)
{
MemoryStream memStream = new MemoryStream();
byte[] readBuffer = new byte[4096];
int bytesRead;
while ((bytesRead = stream.Read(readBuffer, 0, readBuffer.Length)) > 0)
memStream.Write(readBuffer, 0, bytesRead);
return memStream;
}
then call as below.
var wRes = wReq.GetResponse();
var memstream = Read(wRes.GetResponseStream());
var resgzip = new GZipStream(memstream, CompressionMode.Decompress);
var reader = new StreamReader(resgzip);
var textResponse = reader.ReadToEnd();
I'm trying to download a .torrent file (not the contents of the torrent itself) in my .NET application.
Using the following code works for other files, but not .torrent. The resulting files is about 1-3kb smaller than if I download the file via a browser. When opening it in a torrent client, it says the torrent is corrupt.
WebClient web = new WebClient();
web.Headers.Add("Content-Type", "application/x-bittorrent");
web.DownloadFile("http://kat.ph/torrents/linux-mint-12-gnome-mate-dvd-64-bit-t6008958/", "test.torrent");
Opening the URL in a browser results in the file being downloaded correctly.
Any ideas as to why this would happen? Are there any good alternatives to WebClient that would download the file correctly?
EDIT: I've tried this as well as WebClient, and it results in the same thing:
private void DownloadFile(string url, string file)
{
byte[] result;
byte[] buffer = new byte[4096];
WebRequest wr = WebRequest.Create(url);
wr.ContentType = "application/x-bittorrent";
using (WebResponse response = wr.GetResponse())
{
using (Stream responseStream = response.GetResponseStream())
{
using (MemoryStream memoryStream = new MemoryStream())
{
int count = 0;
do
{
count = responseStream.Read(buffer, 0, buffer.Length);
memoryStream.Write(buffer, 0, count);
} while (count != 0);
result = memoryStream.ToArray();
using (BinaryWriter writer = new BinaryWriter(new FileStream(file, FileMode.Create)))
{
writer.Write(result);
}
}
}
}
}
The problem that server returns content compressed by gzip and you download this compressed content to file. For such cases you should check the "Content-Encoding" header and use proper stream reader to decompress the source.
I modified your function to handle gzipped content:
private void DownloadFile(string url, string file)
{
byte[] result;
byte[] buffer = new byte[4096];
WebRequest wr = WebRequest.Create(url);
wr.ContentType = "application/x-bittorrent";
using (WebResponse response = wr.GetResponse())
{
bool gzip = response.Headers["Content-Encoding"] == "gzip";
var responseStream = gzip
? new GZipStream(response.GetResponseStream(), CompressionMode.Decompress)
: response.GetResponseStream();
using (MemoryStream memoryStream = new MemoryStream())
{
int count = 0;
do
{
count = responseStream.Read(buffer, 0, buffer.Length);
memoryStream.Write(buffer, 0, count);
} while (count != 0);
result = memoryStream.ToArray();
using (BinaryWriter writer = new BinaryWriter(new FileStream(file, FileMode.Create)))
{
writer.Write(result);
}
}
}
}