I am trying to compress several excel files, but when generate the excel it generates corrupt. I don't know what could be any idea?
var Clients = getClients();
var carpetaUrl = string.Format(#"{0}/{1}/{2}.zip", IdCuenta, IdListado, "Folder");
try
{
using (MemoryStream zipFile = new MemoryStream())
{
using (var archive = new Archive())
{
foreach (var item in Clients)
{
var (fileUrl, stream) = GenerateFileExcel(item.archivoNombre,item.IdClient);
archive.CreateEntry(fileUrl, stream);
}
archive.Save(zipFile, new ArchiveSaveOptions() { Encoding = Encoding.ASCII });
}
var uri = B.Azure.Storages.UploadFile("tttt", zipFile, carpetaUrl, false).ToString();
return uri;
}
Is it possible for a API endpoint to stream data from an IQueryable object list populated from entity framework, and linked to SQL database to create and return a csv file?
What I have so far is I loop through all the items in my list and create a temporary file. I then stream that file as the result of my GET API call.
Directory.CreateDirectory($"{Environment.CurrentDirectory}/TmpData");
string tmpFileName = $"{Environment.CurrentDirectory}/TmpData/{Guid.NewGuid().ToString()}.csv";
using (FileStream file = new FileStream(tmpFileName, FileMode.CreateNew))
{
using (StreamWriter fileStream = new StreamWriter(file))
{
LookupItem liTmp = new();
await fileStream.WriteAsync(nameof(liTmp.LookupItemId));
await fileStream.WriteAsync(",");
await fileStream.WriteAsync(nameof(liTmp.Code));
await fileStream.WriteAsync(",");
await fileStream.WriteAsync(nameof(liTmp.Label));
await fileStream.WriteLineAsync();
foreach (var li in items)
{
await fileStream.WriteAsync(li.LookupItemId.ToString());
await fileStream.WriteAsync(",");
await fileStream.WriteAsync(li.Code?.ToString());
await fileStream.WriteAsync(",");
await fileStream.WriteAsync(li.Label?.ToString());
await fileStream.WriteLineAsync();
}
}
}
this.Response.StatusCode = 200;
this.Response.Headers.Add(HeaderNames.ContentDisposition, $"attachment; filename=\"{ request.LookupTableType.ToString() } Data { DateTime.Now.ToString("yyyy-mm-dd hh-MM-ss")}.csv\"");
this.Response.Headers.Add(HeaderNames.ContentType, "application/octet-stream");
var inputStream = new FileStream(tmpFileName, FileMode.Open, FileAccess.Read);
var outputStream = this.Response.Body;
const int bufferSize = 1 << 10;
var buffer = new byte[bufferSize];
while (true)
{
var bytesRead = await inputStream.ReadAsync(buffer, 0, bufferSize);
if (bytesRead == 0) break;
await outputStream.WriteAsync(buffer, 0, bytesRead);
}
await outputStream.FlushAsync();
System.IO.File.Delete(tmpFileName);
return new EmptyResult();
This works fine and I get a csv file back, but I was thinking is it possible to write the data straight to the output stream rather than creating a temp file? I don't want to load all the items in to memory as I would like to use this method on large data sets to allow users to download the table data.
You can add EPPlus package. below code I have tested, it works for me.
using dotnetcoreMVC.Models;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Logging;
using OfficeOpenXml;
using OfficeOpenXml.Style;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
namespace dotnetcoreMVC.Controllers
{
public class ForTestController : Controller
{
public IActionResult ExportData()
{
//TODO read data from db
// Mock data
List<TestModel> li = new List<TestModel>();
for (int i = 0; i < 10; i++)
{
TestModel m = new TestModel();
m.id = 1+i;
m.name = "test name"+ i;
li.Add(m);
}
var data = li;
if (data?.Any() != true)
{
return new ContentResult() { Content = "no data" };
}
ExcelPackage.LicenseContext = LicenseContext.NonCommercial;
using (var ep = new ExcelPackage())
{
using (var worksheet = ep.Workbook.Worksheets.Add("export data for test"))
{
var x = 1;
var y = 1;
var columnTitles = new List<string>()
{
"id",
"alias"
};
foreach (var columnTitle in columnTitles)
{
var cell = worksheet.Cells[x, y++];
cell.Style.Font.Bold = true;
cell.Style.HorizontalAlignment = ExcelHorizontalAlignment.Center;
cell.Style.VerticalAlignment = ExcelVerticalAlignment.Center;
cell.Value = columnTitle;
}
foreach (var item in data)
{
x++;
y = 1;
var cell = worksheet.Cells[x, y++];
cell.Value = item.id;
cell = worksheet.Cells[x, y++];
cell.Value = item.name;
}
using (var stream = new MemoryStream())
{
ep.SaveAs(stream);
return new FileContentResult(stream.ToArray(), "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet")
{
FileDownloadName = Guid.NewGuid()+"test.csv"
};
}
}
}
}
}
public class TestModel {
public int id { get; set; }
public string name { get; set; }
}
}
I'm generating a text file in a process which at the end loops through a list of strings that were fed to it, and through a MemoryStream and StreamWriter it converts that list to byte[]. The byte[] is then saved to an Oracle Database using a BLOB datatype. While it works for the majority of the data (typically thousands of lines. I've had anywhere between 5,000 and 40,000, and it's the same result regardless), I have a specific message that goes at the end, but it's always missing. Generally the last line that does end up in the file is cut off halfway.
The function that generates the byte[]:
public byte[] GenerateFileData()
{
var fileData = new byte[0];
using (var ms = new MemoryStream())
{
using (var sw = new StreamWriter(ms))
{
Messages.ForEach(x => sw.WriteLine(x)); // Messages is a list of strings in this class
fileData = ms.ToArray();
}
}
return fileData;
}
The function that saves the byte[] to the database:
public void SaveLogFile(int entityId, byte[] fileData)
{
using (var context = new SomeDBContext())
{
var entity= context.SomeEntity.FirstOrDefault(x => x.Id == runId);
if(entity != null)
{
entity.LOG_FILE = fileData;
context.SaveChanges();
}
}
}
And lastly, the function that turns the data into a file:
[HttpGet]
public FileResult GetLogFile(int id = 0)
{
var fileData = new byte[0];
using (var context = new SomeDbContext())
{
var entity = context.SomeEntity.FirstOrDefault(x => x.Id == id);
fileData = entity.LOG_FILE;
}
var fileName = "SomethingSomething" + id.ToString();
return File(fileData, "text/plain", fileName);
}
Try to get the MemoryStream content after the writer close asthis code:
public byte[] GenerateFileData()
{
var fileData = new byte[0];
using (var ms = new MemoryStream())
{
using (var sw = new StreamWriter(ms))
{
Messages.ForEach(x => sw.WriteLine(x)); // Messages is a list of strings in this class
}
ms.Flush();
fileData = ms.ToArray();
}
return fileData;
}
I am running a cron job in C# that takes 200,000 images and convert into 1 bit image. During running this job, sometimes the process crashes (even though I have global try catch), sometimes for some images(not all) it throws OutOfMemoryException and sometimes for some images it it throws A generic error occurred in GDI.
int pageSize = 1000;
for (int pageNumber = 0; pageNumber < 200; pageNumber++)
{
var imageUrls = allIMageUrls.Skip(pageSize * pageNumber).Take(pageSize).ToList();
var counter = 0;
var total = imageUrls.Count;
Logger.Log($"Page Number : {pageNumber}");
var failedImageUrls = new System.Collections.Concurrent.ConcurrentBag<string>();
Parallel.ForEach(imageUrls, imageUrl =>
{
try
{
Interlocked.Increment(ref counter);
var image = _httpService.DownloadImage(imageUrl);
if (image != null && image.Length > 0)
{
var oneBitImage = ConvertToOnebitFaxGroup4(contract);
_httpService.UploadImage(image, oneBitImage);
oneBitImage = null;
image = null;
}
}
catch (Exception ex)
{
failedImageUrls.Add(imageUrl);
Logger.Log(ex);
}
});
This is one time process. I added paging so that when it crashes I can restart from that page instead of start at beginning.
public static class ImageProcessor
{
static ImageCodecInfo _codecInfo;
static EncoderParameters _encoderParameters;
static ImageProcessor()
{
foreach (var codec in ImageCodecInfo.GetImageEncoders())
{
if (codec.MimeType == "image/tiff")
{
_codecInfo = codec;
break;
}
}
_encoderParameters = new EncoderParameters(2);
_encoderParameters.Param[0] = new EncoderParameter(Encoder.Compression, (long)EncoderValue.CompressionCCITT4);
_encoderParameters.Param[1] = new EncoderParameter(Encoder.ColorDepth, (long)1);
}
public static byte[] ConvertToOnebitFaxGroup4(byte[] bytes)
{
using (var memoryStream = new MemoryStream(bytes))
{
var image = Image.FromStream(memoryStream);
var pData = Marshal.AllocHGlobal(bytes.Length);
Marshal.Copy(bytes, 0, pData, bytes.Length);
var bytesPerLine = (image.Width + 31) / 32 * 4;
var img = new Bitmap(image.Width, image.Height, bytesPerLine, PixelFormat.Format1bppIndexed, pData);
using (var ms = new MemoryStream())
{
image.Save(ms, _codecInfo, _encoderParameters);
img.Dispose();
Marshal.FreeHGlobal(pData);
return ms.ToArray();
}
}
}
Updated:
public static byte[] ConvertToOnebitFaxGroup4(byte[] bytes)
{
using (var memoryStream = new MemoryStream(bytes))
{
using (var image = Image.FromStream(memoryStream))
{
var pData = Marshal.AllocHGlobal(bytes.Length);
Marshal.Copy(bytes, 0, pData, bytes.Length);
var bytesPerLine = (image.Width + 31) / 32 * 4;
using (var img = new Bitmap(image.Width, image.Height, bytesPerLine, PixelFormat.Format1bppIndexed, pData))
{
using (var ms = new MemoryStream())
{
img.Save(ms, _codecInfo, _encoderParameters);
Marshal.FreeHGlobal(pData);
return ms.ToArray();
}
}
}
}
}
Update2
public static byte[] ConvertToOnebitFaxGroup4(byte[] bytes)
{
using (var memoryStream = new MemoryStream(bytes))
{
using (var image = Image.FromStream(memoryStream))
{
using (var ms = new MemoryStream())
{
image.Save(ms, _codecInfo, _encoderParameters);
return ms.ToArray();
}
}
}
}
I have been trying to put together an in-memory public-key encryption infrastructure using OpenPGP via Bouncy Castle. One of our vendors uses OpenPGP public key encryption to encrypt all their feeds, and requires us to do the same, so I'm stuck with the technology and the implementation. So now I'm coding an OpenPGP encryption/ decryption toolkit for automating these feeds.
The examples at bouncycastle.org inexplicably default to writing encrypted data to and collecting keys from a file system; this is not what I want to do, so I've been trying to get everything stream-based.
I have gotten to the point where I can actually get my code to compile and run, but my encrypted payload is empty. I think I'm missing something silly, but after several days of trying this and that, I have lost the ability to objectively examine this.
My utility class contains these methods:
public static PgpPublicKey ImportPublicKey(
this Stream publicIn)
{
var pubRings =
new PgpPublicKeyRingBundle(PgpUtilities.GetDecoderStream(publicIn)).GetKeyRings().OfType<PgpPublicKeyRing>();
var pubKeys = pubRings.SelectMany(x => x.GetPublicKeys().OfType<PgpPublicKey>());
var pubKey = pubKeys.FirstOrDefault();
return pubKey;
}
public static Stream Streamify(this string theString, Encoding encoding = null)
{
encoding = encoding ?? Encoding.UTF8;
var stream = new MemoryStream(encoding.GetBytes(theString));
return stream;
}
public static string Stringify(this Stream theStream,
Encoding encoding = null)
{
encoding = encoding ?? Encoding.UTF8;
using (var reader = new StreamReader(theStream, encoding))
{
return reader.ReadToEnd();
}
}
public static byte[] ReadFully(this Stream stream)
{
if (!stream.CanRead) throw new ArgumentException("This is not a readable stream.");
var buffer = new byte[32768];
using (var ms = new MemoryStream())
{
while (true)
{
var read = stream.Read(buffer, 0, buffer.Length);
if (read <= 0)
return ms.ToArray();
ms.Write(buffer, 0, read);
}
}
}
public static void PgpEncrypt(
this Stream toEncrypt,
Stream outStream,
PgpPublicKey encryptionKey,
bool armor = true,
bool verify = true,
CompressionAlgorithmTag compressionAlgorithm = CompressionAlgorithmTag.Zip)
{
if (armor) outStream = new ArmoredOutputStream(outStream);
var compressor = new PgpCompressedDataGenerator(CompressionAlgorithmTag.Zip);
outStream = compressor.Open(outStream);
var data = toEncrypt.ReadFully();
var encryptor = new PgpEncryptedDataGenerator(SymmetricKeyAlgorithmTag.Cast5, verify, new SecureRandom());
encryptor.AddMethod(encryptionKey);
outStream = encryptor.Open(outStream, data.Length);
outStream.Write(data, 0, data.Length);
}
My test method looks like this:
private static void EncryptMessage()
{
var pubKey = #"<public key text>";
var clearText = "This is an encrypted message. There are many like it but this one is cryptic.";
using (var stream = pubKey.Streamify())
{
var key = stream.ImportPublicKey();
using (var clearStream = clearText.Streamify())
using (var cryptoStream = new MemoryStream())
{
clearStream.PgpEncrypt(cryptoStream,key);
cryptoStream.Position = 0;
Console.WriteLine(cryptoStream.Stringify());
Console.WriteLine("Press any key to continue.");
}
}
Console.ReadKey();
}
The result I get looks like this:
-----BEGIN PGP MESSAGE-----
Version: BCPG C# v1.7.4114.6378
Press any key to continue.
Can someone tell me what I am doing wrong?
OK, I managed to get this working. There were several problems with this implementation. One problem was that certain things had to be done in order. Here is what seems to need to happen:
The raw data needs to be put into a PgpLiteralData object
The literal data needs to be encrypted.
The encrypted data needs to be compressed.
The compressed data (optionally) needs to be armored.
The underlying streams need to be closed in order of usage.
There should be a more elegant way to do this, but the streams used by the BouncyCastle library are all frustratingly one-way, and at several points, I needed to convert the stream to a byte array to get another part to work. I include the code I used and independently verified; if someone has a verifyably better way of doing this, I would be quite interested.
public static class OpenPgpUtility
{
public static void ExportKeyPair(
Stream secretOut,
Stream publicOut,
AsymmetricKeyParameter publicKey,
AsymmetricKeyParameter privateKey,
string identity,
char[] passPhrase,
bool armor)
{
if (armor)
{
secretOut = new ArmoredOutputStream(secretOut);
}
var secretKey = new PgpSecretKey(
PgpSignature.DefaultCertification,
PublicKeyAlgorithmTag.RsaGeneral,
publicKey,
privateKey,
DateTime.UtcNow,
identity,
SymmetricKeyAlgorithmTag.Cast5,
passPhrase,
null,
null,
new SecureRandom()
);
secretKey.Encode(secretOut);
if (armor)
{
secretOut.Close();
publicOut = new ArmoredOutputStream(publicOut);
}
var key = secretKey.PublicKey;
key.Encode(publicOut);
if (armor)
{
publicOut.Close();
}
}
public static PgpPublicKey ImportPublicKey(
this Stream publicIn)
{
var pubRings =
new PgpPublicKeyRingBundle(PgpUtilities.GetDecoderStream(publicIn)).GetKeyRings().OfType<PgpPublicKeyRing>();
var pubKeys = pubRings.SelectMany(x => x.GetPublicKeys().OfType<PgpPublicKey>());
var pubKey = pubKeys.FirstOrDefault();
return pubKey;
}
public static PgpSecretKey ImportSecretKey(
this Stream secretIn)
{
var secRings =
new PgpSecretKeyRingBundle(PgpUtilities.GetDecoderStream(secretIn)).GetKeyRings().OfType<PgpSecretKeyRing>();
var secKeys = secRings.SelectMany(x => x.GetSecretKeys().OfType<PgpSecretKey>());
var secKey = secKeys.FirstOrDefault();
return secKey;
}
public static Stream Streamify(this string theString, Encoding encoding = null)
{
encoding = encoding ?? Encoding.UTF8;
var stream = new MemoryStream(encoding.GetBytes(theString));
return stream;
}
public static string Stringify(this Stream theStream,
Encoding encoding = null)
{
encoding = encoding ?? Encoding.UTF8;
using (var reader = new StreamReader(theStream, encoding))
{
return reader.ReadToEnd();
}
}
public static byte[] ReadFully(this Stream stream, int position = 0)
{
if (!stream.CanRead) throw new ArgumentException("This is not a readable stream.");
if (stream.CanSeek) stream.Position = 0;
var buffer = new byte[32768];
using (var ms = new MemoryStream())
{
while (true)
{
var read = stream.Read(buffer, 0, buffer.Length);
if (read <= 0)
return ms.ToArray();
ms.Write(buffer, 0, read);
}
}
}
public static void PgpEncrypt(
this Stream toEncrypt,
Stream outStream,
PgpPublicKey encryptionKey,
bool armor = true,
bool verify = false,
CompressionAlgorithmTag compressionAlgorithm = CompressionAlgorithmTag.Zip)
{
var encryptor = new PgpEncryptedDataGenerator(SymmetricKeyAlgorithmTag.Cast5, verify, new SecureRandom());
var literalizer = new PgpLiteralDataGenerator();
var compressor = new PgpCompressedDataGenerator(compressionAlgorithm);
encryptor.AddMethod(encryptionKey);
//it would be nice if these streams were read/write, and supported seeking. Since they are not,
//we need to shunt the data to a read/write stream so that we can control the flow of data as
//we go.
using (var stream = new MemoryStream()) // this is the read/write stream
using (var armoredStream = armor ? new ArmoredOutputStream(stream) : stream as Stream)
using (var compressedStream = compressor.Open(armoredStream))
{
//data is encrypted first, then compressed, but because of the one-way nature of these streams,
//other "interim" streams are required. The raw data is encapsulated in a "Literal" PGP object.
var rawData = toEncrypt.ReadFully();
var buffer = new byte[1024];
using (var literalOut = new MemoryStream())
using (var literalStream = literalizer.Open(literalOut, 'b', "STREAM", DateTime.UtcNow, buffer))
{
literalStream.Write(rawData, 0, rawData.Length);
literalStream.Close();
var literalData = literalOut.ReadFully();
//The literal data object is then encrypted, which flows into the compressing stream and
//(optionally) into the ASCII armoring stream.
using (var encryptedStream = encryptor.Open(compressedStream, literalData.Length))
{
encryptedStream.Write(literalData, 0, literalData.Length);
encryptedStream.Close();
compressedStream.Close();
armoredStream.Close();
//the stream processes are now complete, and our read/write stream is now populated with
//encrypted data. Convert the stream to a byte array and write to the out stream.
stream.Position = 0;
var data = stream.ReadFully();
outStream.Write(data, 0, data.Length);
}
}
}
}
}
My test method looked like this:
private static void EncryptMessage()
{
var pubKey = #"<public key text here>";
var clearText = #"<message text here>";
using (var stream = pubKey.Streamify())
{
var key = stream.ImportPublicKey();
using (var clearStream = clearText.Streamify())
using (var cryptoStream = new MemoryStream())
{
clearStream.PgpEncrypt(cryptoStream, key);
cryptoStream.Position = 0;
var cryptoString = cryptoStream.Stringify();
Console.WriteLine(cryptoString);
Console.WriteLine("Press any key to continue.");
}
}
Console.ReadKey();
}
Since someone asked, my decryption algorithm looked like this:
public static Stream PgpDecrypt(
this Stream encryptedData,
string armoredPrivateKey,
string privateKeyPassword,
Encoding armorEncoding = null)
{
armorEncoding = armorEncoding ?? Encoding.UTF8;
var stream = PgpUtilities.GetDecoderStream(encryptedData);
var layeredStreams = new List<Stream> { stream }; //this is to clean up/ dispose of any layered streams.
var dataObjectFactory = new PgpObjectFactory(stream);
var dataObject = dataObjectFactory.NextPgpObject();
Dictionary<long, PgpSecretKey> secretKeys;
using (var privateKeyStream = armoredPrivateKey.Streamify(armorEncoding))
{
var secRings =
new PgpSecretKeyRingBundle(PgpUtilities.GetDecoderStream(privateKeyStream)).GetKeyRings()
.OfType<PgpSecretKeyRing>();
var pgpSecretKeyRings = secRings as PgpSecretKeyRing[] ?? secRings.ToArray();
if (!pgpSecretKeyRings.Any()) throw new ArgumentException("No secret keys found.");
secretKeys = pgpSecretKeyRings.SelectMany(x => x.GetSecretKeys().OfType<PgpSecretKey>())
.ToDictionary(key => key.KeyId, value => value);
}
while (!(dataObject is PgpLiteralData) && dataObject != null)
{
try
{
var compressedData = dataObject as PgpCompressedData;
var listedData = dataObject as PgpEncryptedDataList;
//strip away the compression stream
if (compressedData != null)
{
stream = compressedData.GetDataStream();
layeredStreams.Add(stream);
dataObjectFactory = new PgpObjectFactory(stream);
}
//strip the PgpEncryptedDataList
if (listedData != null)
{
var encryptedDataList = listedData.GetEncryptedDataObjects()
.OfType<PgpPublicKeyEncryptedData>().First();
var decryptionKey = secretKeys[encryptedDataList.KeyId]
.ExtractPrivateKey(privateKeyPassword.ToCharArray());
stream = encryptedDataList.GetDataStream(decryptionKey);
layeredStreams.Add(stream);
dataObjectFactory = new PgpObjectFactory(stream);
}
dataObject = dataObjectFactory.NextPgpObject();
}
catch (Exception ex)
{
//Log exception here.
throw new PgpException("Failed to strip encapsulating streams.", ex);
}
}
foreach (var layeredStream in layeredStreams)
{
layeredStream.Close();
layeredStream.Dispose();
}
if (dataObject == null) return null;
var literalData = (PgpLiteralData)dataObject;
var ms = new MemoryStream();
using (var clearData = literalData.GetInputStream())
{
Streams.PipeAll(clearData, ms);
}
ms.Position = 0;
return ms;
}