Here is code which takes almost 3 to 8 sec for serialization depends on object type. I want to store this result into Redis for caching. But this operation is taking too long. Also same for deserialization.
public byte[] SerializeObject(object objectToSerialize)
{
try
{
//If object to serialize is null then return null
if (objectToSerialize == null)
return null;
byte[] result;
//Create memory stream and use it for Serializing object
using (var ms = new MemoryStream())
{
using (var zs = new GZipStream(ms, CompressionMode.Compress, true))
{
var bf = new BinaryFormatter();
bf.Serialize(zs, objectToSerialize);
}
result = ms.ToArray();
}
return result;
}
catch (Exception ex)
{
//Some code
return null;
}
}
Updated: How I have tried serializing a DataTable:
using(var client = m_oRedisClientsManager.GetClient()) {
//Serialize DataTable to byte
byteCachedDatatable = m_oSerializer.SerializeDataTable(oCacheObject);
//add Serialized bytes to redis and update expiration time
client.Set(sCacheKey, byteCachedDatatable, new TimeSpan(0, iExpiryTimeInMins, 0));
}
public byte[] SerializeDataTable(DataTable dataTable) {
if (dataTable == null) return null;
byte[] result;
using(var memoryStream = new MemoryStream()) {
using(var deflateStream = new DeflateStream(memoryStream, CompressionMode.Compress)) {
dataTable.WriteXml(deflateStream, XmlWriteMode.WriteSchema);
deflateStream.Flush();
deflateStream.Close();
result = memoryStream.ToArray();
}
}
return result;
}
Any pointers will be helpful.
Related
I am implementing the BLOB Storing capability provided through the Volo.Abp.BlobStoring.IBlobContainer interface.
I believe I have everything coded and configured correctly but a recent deprecation by Microsoft has me wondering if there is a better implementation than what I am attempting.
Here is my code:
public async Task StorePhotoAsync(Photo photo)
{
var photoBytes = ObjectToByteArray(photo);
await _blobContainer.SaveAsync(photo.Id.ToString(), photoBytes);
}
// TODO - Reimplement (deprecated serialization) - JLavallet 2022-03-09 10:41
private byte[] ObjectToByteArray(object obj)
{
if (obj == null)
{
return null;
}
var bf = new BinaryFormatter();
using var ms = new MemoryStream();
bf.Serialize(ms, obj);
return ms.ToArray();
}
// TODO - Reimplement (deprecated serialization) - JLavallet 2022-03-09 10:41
private object ByteArrayToObject(byte[] arrBytes)
{
using var memStream = new MemoryStream();
var binForm = new BinaryFormatter();
memStream.Write(arrBytes, 0, arrBytes.Length);
memStream.Seek(0, SeekOrigin.Begin);
var obj = binForm.Deserialize(memStream);
return obj;
}
As you can see from my to do comments and the screenshot below, there is a deprecation of the Serialize and Deserialize methods of the BinaryFormatter:
Could anyone suggest an alternative approach? The IBlobContainer only wants to save a byte array.
So after Mr. T set me straight, I read the documentation for JSON UTF8 serialization and deserialization, and here's what I came up with:
public async Task StorePhotoCacheItemAsync(PhotoCacheItem photoCacheItem)
{
var bytes = PhotoCacheItemToByteArray(photoCacheItem);
await _blobContainer.SaveAsync(photoCacheItem.Id.ToString(), bytes);
}
private byte[] PhotoCacheItemToByteArray(PhotoCacheItem photoCacheItem)
{
if (photoCacheItem == null)
{
return null;
}
// the old way
//var bf = new BinaryFormatter();
//using var ms = new MemoryStream();
//bf.Serialize(ms, obj);
// a new way
//using var ms = new MemoryStream();
//using var writer = new Utf8JsonWriter(ms);
//JsonSerializer.Serialize(writer, photoCacheItem);
//return ms.ToArray();
// a better new way
var bytes = JsonSerializer.SerializeToUtf8Bytes(photoCacheItem);
return bytes;
}
private async Task<PhotoCacheItem> GetPhotoCacheItemFromBlobStorage(Guid photoId)
{
var bytes = await _blobContainer.GetAllBytesOrNullAsync(photoId.ToString());
if (bytes == null)
{
return null;
}
var photoCacheItem = ByteArrayToPhotoCacheItem(bytes);
return photoCacheItem;
}
private PhotoCacheItem ByteArrayToPhotoCacheItem(byte[] bytes)
{
// the old way
//using var ms = new MemoryStream();
//var bf = new BinaryFormatter();
//ms.Write(bytes, 0, bytes.Length);
//ms.Seek(0, SeekOrigin.Begin);
//var obj = bf.Deserialize(ms);
// a new way
//var utf8Reader = new Utf8JsonReader(bytes);
//var photoCacheItem = JsonSerializer.Deserialize<PhotoCacheItem>(ref utf8Reader)!;
// a better new way
var readOnlySpan = new ReadOnlySpan<byte>(bytes);
var photoCacheItem = JsonSerializer.Deserialize<PhotoCacheItem>(readOnlySpan)!;
return photoCacheItem;
}
I have asp.net mvc application which has file upload functionality. While uploading the file, I am performing few validations on the uploaded content before moving it to database and file system location.
Here goes my code:
[HttpPost]
[ValidateAntiForgeryToken]
public ActionResult AddImage([Bind(Include = "image,ImageName,ImageType,CountryId,Keyword,Source,Copyright,Description")] CreateImageViewModel model)
{
if (!this.ModelState.IsValid)
{
return View("Images");
}
if (model != null && model.image.ContentType.Contains(Constants.Image) && !ValidateUploadedImageContent(model.image, model.image.FileName))
{
var dto = new ImageDTO();
model.FilePath = model.image.FileName;
dto.ImageFile = model.image;
dto.Name = model.ImageName;
dto.FilePath = model.image.FileName;
dto.FileType = Path.GetExtension(model.FilePath);
dto.ImageType = model.ImageType;
dto.CountryId = model.CountryId;
dto.Keyword = model.Keyword;
dto.Source = model.Source;
dto.Copyright = model.Copyright;
dto.Description = model.Description;
dto.CreatedBy = UserDto.emailId;
try
{
_imageService.SaveImage(dto);
}
catch (Exception ex)
{
if (ex.Message.Equals(Constants.InvalidImageType))
return GetSafeRedirect(Url.Action("AddImage", model) + "#onload-errors");
throw ex;
}
return RedirectToAction(Constants.Actions.Images.ToString());
}
else
{
return GetSafeRedirect(Url.Action("AddImage", model) + "#onload-errors");
}
}
private bool ValidateUploadedImageContent(HttpPostedFileBase uploadedFile, string imageFileName)
{
if (Path.GetExtension(imageFileName).Equals(".svg", StringComparison.OrdinalIgnoreCase))
{
if (uploadedFile.ContentLength > 0)
{
byte[] data;
//using (Stream inputStream = uploadedFile.InputStream)
//{
Stream inputStream = uploadedFile.InputStream;
var memoryStream = inputStream as MemoryStream;
if (memoryStream == null)
{
memoryStream = new MemoryStream();
inputStream.CopyTo(memoryStream);
}
data = memoryStream.ToArray();
//}
var parsedData = Encoding.UTF8.GetString(data, 0, data.Length).TrimEnd('\0');
var result = parsedData.ContainsAny(Constants.InsecureStrings, StringComparison.CurrentCultureIgnoreCase);
return result;
}
}
return false;
}
Here in the above method: ValidateUploadedImageContent(), I tried to dispose the stream object with the help of using statement but I found that if I keep the below code in the method: ValidateUploadedImageContent(), then in that case post validation process, I found on debugging that the ContentLength property is set with 0 value and finally corrupted image gets saved in the file system location.
Updated :
private bool ValidateUploadedImageContent(HttpPostedFileBase uploadedFile, string imageFileName)
{
if (Path.GetExtension(imageFileName).Equals(".svg", StringComparison.OrdinalIgnoreCase))
{
if (uploadedFile.ContentLength > 0)
{
byte[] data;
using (Stream inputStream = uploadedFile.InputStream)
{
Stream inputStream = uploadedFile.InputStream;
var memoryStream = inputStream as MemoryStream;
if (memoryStream == null)
{
memoryStream = new MemoryStream();
inputStream.CopyTo(memoryStream);
}
data = memoryStream.ToArray();
}
var parsedData = Encoding.UTF8.GetString(data, 0, data.Length).TrimEnd('\0');
var result = parsedData.ContainsAny(Constants.InsecureStrings, StringComparison.CurrentCultureIgnoreCase);
return result;
}
}
return false;
}
Can anyone help me to know how to fix this issue?
First to address your issue, which I now understand is that after the call to ValidateUploadedImageContent the image stream is invalid.
That is because the stream gained from the HttpPostedFileBase is "read-only sequential (non-seekable)" as detailed in this SO answer. This explains why the stream's ContentLength is 0 - the stream has been consumed by the validation call.
If you have flexibility with the ImageDTO class, modifying the validation method such that it returns the image bytes would be a workaround.
For example,
// on success, buffer contains the image data. Otherwise it is null.
private bool ValidateUploadedImageContent(
out byte[] buffer,
HttpPostedFileBase uploadedFile,
string imageFileName)
{
buffer = null;
if (Path.GetExtension(imageFileName).Equals(".svg", StringComparison.OrdinalIgnoreCase))
{
if (uploadedFile.ContentLength > 0)
{
var reader = new BinaryReader(inputStream);
buffer = reader.ReadBytes((int)uploadedFile.ContentLength);
var parsedData = Encoding.UTF8.GetString(buffer, 0, buffer.Length).TrimEnd('\0');
return parsedData.ContainsAny(Constants.InsecureStrings, StringComparison.CurrentCultureIgnoreCase);
}
}
return false;
}
I've used BinaryReader to simplify the code.
Then back to the calling method,
byte[] imageBuffer = null;
if (model != null && model.image.ContentType.Contains(Constants.Image)
&& !ValidateUploadedImageContent(out imageBuffer, model.image, model.image.FileName)) {
var dto = new ImageDTO();
using(var imageStream = new MemoryStream(imageBuffer)) {
// pass along imageStream to your ImageDTO and save.
}
}
Again, hopefully you have some flexibility with the ImageDTO class.
I'm generating a text file in a process which at the end loops through a list of strings that were fed to it, and through a MemoryStream and StreamWriter it converts that list to byte[]. The byte[] is then saved to an Oracle Database using a BLOB datatype. While it works for the majority of the data (typically thousands of lines. I've had anywhere between 5,000 and 40,000, and it's the same result regardless), I have a specific message that goes at the end, but it's always missing. Generally the last line that does end up in the file is cut off halfway.
The function that generates the byte[]:
public byte[] GenerateFileData()
{
var fileData = new byte[0];
using (var ms = new MemoryStream())
{
using (var sw = new StreamWriter(ms))
{
Messages.ForEach(x => sw.WriteLine(x)); // Messages is a list of strings in this class
fileData = ms.ToArray();
}
}
return fileData;
}
The function that saves the byte[] to the database:
public void SaveLogFile(int entityId, byte[] fileData)
{
using (var context = new SomeDBContext())
{
var entity= context.SomeEntity.FirstOrDefault(x => x.Id == runId);
if(entity != null)
{
entity.LOG_FILE = fileData;
context.SaveChanges();
}
}
}
And lastly, the function that turns the data into a file:
[HttpGet]
public FileResult GetLogFile(int id = 0)
{
var fileData = new byte[0];
using (var context = new SomeDbContext())
{
var entity = context.SomeEntity.FirstOrDefault(x => x.Id == id);
fileData = entity.LOG_FILE;
}
var fileName = "SomethingSomething" + id.ToString();
return File(fileData, "text/plain", fileName);
}
Try to get the MemoryStream content after the writer close asthis code:
public byte[] GenerateFileData()
{
var fileData = new byte[0];
using (var ms = new MemoryStream())
{
using (var sw = new StreamWriter(ms))
{
Messages.ForEach(x => sw.WriteLine(x)); // Messages is a list of strings in this class
}
ms.Flush();
fileData = ms.ToArray();
}
return fileData;
}
I am currently trying to serialize a List, it serializes (I think fine), but when it deserialize,
Sorry for the amount of code, but I am really stuck and have no idea why this is happening, i also tried to changed the struct into a class and no help.
THANKS.
i get the following error UPDATED
There was an error deserializing the object of type There was an error deserializing the object of type
`System.Collections.Generic.List`1[[A.B.C.DataValues, A.V, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null]]. Unexpected end of file. Following elements are not closed: Time, DataValues, ArrayOfDataValues.`
I am serializing like this UPDATED
public void SerializeDataValue(List<DataValues> values)
{
DataContractSerializer serializer = new DataContractSerializer(typeof(List<DataValues>));
using (MemoryStream stream = new MemoryStream())
{
using (GZipStream compress = new GZipStream(stream, CompressionMode.Compress))
{
XmlDictionaryWriter w = XmlDictionaryWriter.CreateBinaryWriter(compress);
serializer.WriteObject(w, values);
}
_serializedData = stream.ToArray();
}
}
I am deserializing like this UPDATED
public List<DataValues> DeserializeDataValue()
{
if (SerializedData == null || SerializedData.Length == 0)
{
return new List<DataValues> ();
}
else
{
DataContractSerializer serializer = new DataContractSerializer(typeof(List<DataValues>));
using (MemoryStream stream = new MemoryStream(SerializedData))
{
using (GZipStream decompress = new GZipStream(stream, CompressionMode.Decompress))
{
XmlDictionaryReader r = XmlDictionaryReader.CreateBinaryReader(decompress, XmlDictionaryReaderQuotas.Max);
return serializer.ReadObject(r, true) as List<DataValues>;
}
}
}
}
Properties
private byte[] _serializedData;
[DataMember]
[Browsable(false)]
public byte[] SerializedData
{
get { return _serializedData; }
set { _serializedData = value; }
}
helper Methods
public static byte[] ReadFully(Stream input)
{
byte[] buffer = new byte[16 * 1024];
input.Position = 0;
using (MemoryStream ms = new MemoryStream())
{
int read;
while ((read = input.Read(buffer, 0, buffer.Length)) > 0)
{
ms.Write(buffer, 0, read);
}
return ms.ToArray();
}
}
Struct
[DataContract(Name = "DataValues", Namespace = "A.B.C")]
public struct DataValues
{
[DataMember]
public DateTime Time { get; set; }
[DataMember]
public Single Value { get; set; }
public DataValues(DateTime dateTime, Single value)
{
Time = dateTime;
Value = value;
}
}
It’s because you are not serialising the object(s) completely. You need to close the stream(s) after writing, especially when using gzip. Recommended practice is to use using:
public void SerializeDataValue(List<DataValues> values)
{
DataContractSerializer serializer = new DataContractSerializer(typeof(List<DataValues>));
using (MemoryStream stream = new MemoryStream())
{
using (GZipStream compress = new GZipStream(stream, CompressionMode.Compress))
{
XmlDictionaryWriter w = XmlDictionaryWriter.CreateBinaryWriter(compress);
serializer.WriteObject(w, values);
}
_serializedData = stream.ToArray();
}
}
Sorry to be late to this question.
The problem with the initial approach was simply that you weren't flushing (read: disposing) the XmlDictionaryWriter.
This should work (note the 2nd using clause):
using (GZipStream compress = new GZipStream(stream, CompressionMode.Compress))
using (XmlDictionaryWriter w = XmlDictionaryWriter.CreateBinaryWriter(compress))
{
serializer.WriteObject(w, values);
}
Hope this helps someone.
I can get the sample to work by removing the XmlDictionaryReader and instead directly feeding the input/output stream into the DataContractSerializer. It may be a defect in the XmlDictionaryReader for large compressed collections but I'm not sure.
Hope this helps:
public void SerializeDataValue(List<DataValues> values)
{
DataContractSerializer serializer = new DataContractSerializer(typeof(List<DataValues>));
using (MemoryStream stream = new MemoryStream())
{
using (GZipStream compress = new GZipStream(stream, CompressionMode.Compress))
{
serializer.WriteObject(compress , values);
}
_serializedData = stream.ToArray();
}
}
public List<DataValues> DeserializeDataValue()
{
if (SerializedData == null || SerializedData.Length == 0)
{
return new List<DataValues> ();
}
else
{
DataContractSerializer serializer = new DataContractSerializer(typeof(List<DataValues>));
using (MemoryStream stream = new MemoryStream(SerializedData))
{
using (GZipStream decompress = new GZipStream(stream, CompressionMode.Decompress))
{
return serializer.ReadObject(decompress , true) as List<DataValues>;
}
}
}
}
I ran exactly into the same problem and I finally found the solution : the XmlDictionaryWriter needs to be disposed/closed before the Stream you are writing into is itself closed. I discovered that thanks to the thorough example found at http://www.albahari.com/nutshell/ch15.aspx whiche are more complete than the MSDN ones.
In your sample code, that would be :
using (XmlDictionaryWriter w = XmlDictionaryWriter.CreateBinaryWriter(compress))
{
serializer.WriteObject(w, values);
}
On my own example, using the XmlDictionaryWriter instead of the plain and by default Xml writer only gave me a ~25% decrease in file size but a factor 3 when reading back the object.
I am created a post before " Object to byte not working " .
I fixed problems that users said me , but there is still problem .
Error Message : The constructor to deserialize an object of type 'WindowsFormsApplication1.Form1+Item' was not found.;
void start()
{
Item item = new Item();
item.files.Add(#"test");
byte[] b = ObjectToByteArray(item);
Item k = Desriles(b);
}
[Serializable]
public class Item : ISerializable
{
public Item()
{
files = new List<string>();
Exclude = false;
CleanEmptyFolder = false;
}
public List<string> files;
public string MusicProfileName;
public bool Exclude;
#region ISerializable Members
public void GetObjectData(SerializationInfo info, StreamingContext context)
{
info.AddValue("files", files);
info.AddValue("MusicProfileName", MusicProfileName);
info.AddValue("Exclude", Exclude);
}
#endregion
}
public byte[] ObjectToByteArray(object _Object)
{
using (var stream = new MemoryStream())
{
// serialize object
var formatter = new BinaryFormatter();
formatter.Serialize(stream, _Object);
// get a byte array
var bytes = new byte[stream.Length];
using (BinaryReader br = new BinaryReader(stream))
{
bytes = br.ReadBytes(Convert.ToInt32(stream.Length));
}
return bytes;
}
}
public Item Desriles(byte[] items)
{
using (MemoryStream stream = new MemoryStream())
{
stream.SetLength(items.LongLength);
stream.write(items, 0, items.Length);
var formatter = new BinaryFormatter();
stream.Seek(0, SeekOrigin.Begin);
object item = formatter.Deserialize(stream); // Here I will get error
return (Item)item;
}
}
The serialization code can't work properly, you forgot to reset the stream back to the beginning. The better mouse trap:
public byte[] ObjectToByteArray(object _Object) {
using (var stream = new MemoryStream()) {
var formatter = new BinaryFormatter();
formatter.Serialize(stream, _Object);
return stream.ToArray();
}
}
The deserialization code can similarly be simplified:
public Item Desriles(byte[] items) {
using (MemoryStream stream = new MemoryStream(items)) {
var formatter = new BinaryFormatter();
return (Item)formatter.Deserialize(stream);
}
}
And you don't need GetObjectData().
In this section:
using (MemoryStream stream = new MemoryStream())
{
stream.SetLength(items.LongLength);
stream.Read(items, 0, items.Length);
[...]
object item = formatter.Deserialize(stream);
it seems you are creating a new, empty memory stream, then attempting to read from it, and Deserialize from it.
Of course it fails. The stream is empty.
abelenky makes a good point, but I don't think your:
public byte[] ObjectToByteArray(object _Object)
works either.
C# Code Snippet - Object to byte array
C# Code Snippet - Byte array to object
Thanks to All :
I found problem : I should base that to ISerializable