I want to put the contents of some files into the database to be read by a seperate process. This is a two step thing as the files will be uploaded to a java server but then processed by a seperate c# application that runs periodically.
I was planning on using a nvarchar(Max) column to represent the data but I can't see how to read from that sort of column in a sensible way. I don't want to use SqlDataReader.GetString as that will force me to hold all the data in memory at once. The files aren't massive but that just seems like a stupid thing to do - it'll give me it as a single string which will then need splitting up into lines, so the whole approach would be totally backwards.
I was assuming I'd just be able to use a normal stream reader but calling GetStream fails saying it doesn't work for this type of column.
Any ideas? Is it just going to be easier for the database to pretend this isn't really text and store it as bytes so I can stream it?
I wrote this extension method some time ago:
public static class DataExtensions
{
public static Stream GetStream(this IDataRecord record, int ordinal)
{
return new DbBinaryFieldStream(record, ordinal);
}
public static Stream GetStream(this IDataRecord record, string name)
{
int i = record.GetOrdinal(name);
return record.GetStream(i);
}
private class DbBinaryFieldStream : Stream
{
private readonly IDataRecord _record;
private readonly int _fieldIndex;
private long _position;
private long _length = -1;
public DbBinaryFieldStream(IDataRecord record, int fieldIndex)
{
_record = record;
_fieldIndex = fieldIndex;
}
public override bool CanRead
{
get { return true; }
}
public override bool CanSeek
{
get { return true; }
}
public override bool CanWrite
{
get { return false; }
}
public override void Flush()
{
throw new NotSupportedException();
}
public override long Length
{
get
{
if (_length < 0)
{
_length = _record.GetBytes(_fieldIndex, 0, null, 0, 0);
}
return _length;
}
}
public override long Position
{
get
{
return _position;
}
set
{
_position = value;
}
}
public override int Read(byte[] buffer, int offset, int count)
{
long nRead = _record.GetBytes(_fieldIndex, _position, buffer, offset, count);
_position += nRead;
return (int)nRead;
}
public override long Seek(long offset, SeekOrigin origin)
{
long newPosition = _position;
switch (origin)
{
case SeekOrigin.Begin:
newPosition = offset;
break;
case SeekOrigin.Current:
newPosition = _position + offset;
break;
case SeekOrigin.End:
newPosition = this.Length - offset;
break;
default:
break;
}
if (newPosition < 0)
throw new ArgumentOutOfRangeException("offset");
_position = newPosition;
return _position;
}
public override void SetLength(long value)
{
throw new NotSupportedException();
}
public override void Write(byte[] buffer, int offset, int count)
{
throw new NotSupportedException();
}
}
}
It's designed for a BLOB, but it works for a NVARCHAR(max) as well (at least on SQL Server).
You can use it like this:
using (var dataReader = command.ExecuteReader())
{
dataReader.Read();
using (var stream = dataReader.GetStream("Text"))
using (var streamReader = new StreamReader(stream))
{
// read the text using the StreamReader...
}
}
Related
Consider I have the following snippet:
public void Store(Stream s, object t)
{
var serializer = new DataContractSerializer(target.GetType(),
new DataContractSerializerSettings
{
PreserveObjectReferences = true
});
serializer.WriteObject(s, target);
}
where s is write-only and doesn't support seeking.
Is there any way to get bytes count that was written to stream by WriteObject? I know I can do it in the following way:
using (var memStream = new MemoryStream())
{
serializer.WriteObject(serializer, target);
Debug.WriteLine(memStream.Length);
memStream.CopyTo(s);
}
but I wonder wheter it is possible to avoid CopyTo - object is quite huge.
EDIT:
I've just came up with an idea: I can create a wrapper that counts bytes on write. It's the best solution so fat, but maybe there is another way.
DONE
I've implemented a wrapper: https://github.com/pwasiewicz/counted-stream - maybe it will be useful for someone.
Thanks!
Sample implementation of a wrapper I've made:
public class CountedStream : Stream
{
private readonly Stream stream;
public CountedStream(Stream stream)
{
if (stream == null) throw new ArgumentNullException("stream");
this.stream = stream;
}
public long WrittenBytes { get; private set; }
public override void Flush()
{
this.stream.Flush();
}
public override int Read(byte[] buffer, int offset, int count)
{
return this.stream.Read(buffer, offset, count);
}
public override long Seek(long offset, SeekOrigin origin)
{
return this.stream.Seek(offset, origin);
}
public override void SetLength(long value)
{
this.stream.SetLength(value);
}
public override void Write(byte[] buffer, int offset, int count)
{
if (buffer.Length >= offset + count)
throw new ArgumentException("Count exceeds buffer size");
this.stream.Write(buffer, offset, count);
this.WrittenBytes += count;
}
public override bool CanRead
{
get { return this.stream.CanRead; }
}
public override bool CanSeek
{
get { return this.stream.CanSeek; }
}
public override bool CanWrite
{
get { return this.stream.CanWrite; }
}
public override long Length
{
get { return this.stream.Length; }
}
public override bool CanTimeout
{
get { return this.stream.CanTimeout; }
}
public override long Position
{
get { return this.stream.Position; }
set { this.stream.Position = value; }
}
}
I'm currently searching for some way to extended SFML.Net to use MP3. Therefore ioctllr recommended NLayer, which I wanted to try. This is my approach:
namespace AudioCuesheetEditor.AudioBackend
{
public class SFMLMp3Stream : Stream
{
private static readonly Logfile log = Logfile.getLogfile(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType);
private MpegFile file;
public SFMLMp3Stream(String _filename)
{
//TODO:Check file existence
file = new MpegFile(_filename);
log.debug("file = " + file.ToString());
}
#region implemented abstract members of Stream
public override void Flush()
{
file.Position = 0;
}
public override long Seek(long offset, SeekOrigin origin)
{
switch (origin)
{
case SeekOrigin.Begin:
file.Position = offset;
break;
case SeekOrigin.Current:
file.Position = file.Position + offset;
break;
case SeekOrigin.End:
break;
}
return file.Position;
}
public override void SetLength(long value)
{
//TODO?
int i = 1;
}
public override int Read(byte[] buffer, int offset, int count)
{
return file.ReadSamples(buffer, offset, count);
}
public override void Write(byte[] buffer, int offset, int count)
{
throw new NotImplementedException();
}
public override bool CanRead
{
get
{
return true;
}
}
public override bool CanSeek
{
get
{
return file.CanSeek;
}
}
public override bool CanWrite
{
get
{
//Writing to stream is prohibited
return false;
}
}
public override long Length
{
get
{
return file.Length;
}
}
public override long Position
{
get
{
return file.Position;
}
set
{
file.Position = value;
}
}
#endregion
}
}
Used this way:
musicStream = new SFMLMp3Stream(this.objProgram.getObjCuesheet().getAudiofilePath(true));
try
{
music = new Music(musicStream);
}
catch(Exception ex)
{
log.fatal(ex.ToString());
}
I get the exception from SFML:
05.06.2014 10:34:28 AudioCuesheetEditor.AudioBackend.AudioManagerSFML: FATAL | SFML.LoadingFailedException: Failed to load music from memory
bei SFML.Audio.Music..ctor(Stream stream)
bei AudioCuesheetEditor.AudioBackend.AudioManagerSFML.setMusic() in d:\tmp\AudioCuesheetEditor\src\AudioCuesheetEditor\AudioBackend\AudioManagerSFML.cs:Zeile 50.
I read in SFML, that it needs PCM 16 Bit signed integers (http://en.sfml-dev.org/forums/index.php?topic=1406.0). Does NLayer support this? How do I need to change the code, that it works?
Thanks for your help.
Sven
Unhappiliy Music class doesn't offer reading an mp3 stream.
What are the possible causes for Stream not Writable Exception when serializing custom object over TCP using Network Stream in C#.
I am Sending the Mp3 data in the form of Packets.The Frame consists of Byte[] Buffer.I am Using Binary Formatter to serialize the object.
BinaryFormatter.Serialize(NetworkStream,Packet);
The Mp3 Played at client with distortion and jitters end for few seconds and then The above mentioned exception raised.I m using NAudio Open Source library for it.
Before doing this modification I was using
NetworkStream.Write(Byte[] Buffer,0,EncodedSizeofMp3);
and it was writing it successfully before giving any exception
If you are writing to a NetworkStream, the stream/socket could be closed
If you are writing to a NetworkStream, it could have been created with FileAccess.Read
If I had to guess, though, it sounds like something is closing the stream - this can be the case if, say, a "writer" along the route assumes it owns the stream, so closes the stream prematurely. It is pretty common to have to write and use some kind of wrapper Stream that ignores Close() requests (I have one in front of me right now, in fact, since I'm writing some TCP code).
As a small aside; I generally advise against BinaryFormatter for comms (except remoting) - most importantly: it doesn't "version" in a very friendly way, but it also tends to be a bit verbose in most cases.
Here's the wrapper I'm using currently, in case it helps (the Reset() method spoofs resetting the position, so the caller can read a relative position):
class NonClosingNonSeekableStream : Stream
{
public NonClosingNonSeekableStream(Stream tail)
{
if(tail == null) throw new ArgumentNullException("tail");
this.tail = tail;
}
private long position;
private readonly Stream tail;
public override bool CanRead
{
get { return tail.CanRead; }
}
public override bool CanWrite
{
get { return tail.CanWrite; }
}
public override bool CanSeek
{
get { return false; }
}
public override bool CanTimeout
{
get { return false; }
}
public override long Position
{
get { return position; }
set { throw new NotSupportedException(); }
}
public override void Flush()
{
tail.Flush();
}
public override void SetLength(long value)
{
throw new NotSupportedException();
}
public override long Seek(long offset, SeekOrigin origin)
{
throw new NotSupportedException();
}
public override long Length
{
get { throw new NotSupportedException(); }
}
public override int Read(byte[] buffer, int offset, int count)
{
int read = tail.Read(buffer, offset, count);
if (read > 0) position += read;
return read;
}
public override void Write(byte[] buffer, int offset, int count)
{
tail.Write(buffer, offset, count);
if (count > 0) position += count;
}
public override int ReadByte()
{
int result = tail.ReadByte();
if (result >= 0) position++;
return result;
}
public override void WriteByte(byte value)
{
tail.WriteByte(value);
position++;
}
public void Reset()
{
position = 0;
}
}
I am currently in the process of writing a BinaryReader that caches the BaseStream.Position and BaseStream.Length properties. Here is what I have so far:
public class FastBinaryReader
{
BinaryReader reader;
public long Length { get; private set; }
public long Position { get; private set; }
public FastBinaryReader(Stream stream)
{
reader = new BinaryReader(stream);
Length = stream.Length;
Position = 0;
}
public void Seek(long newPosition)
{
reader.BaseStream.Position = newPosition;
Position = newPosition;
}
public byte[] ReadBytes(int count)
{
if (Position + count >= Length)
Position = Length;
else
Position += count;
return reader.ReadBytes(count);
}
public void Close()
{
reader.Close();
}
}
Instead of providing a Length and Position property, I would like to create a BaseStream property that allows me to expose my Position and Length properties as FastBinaryReader.BaseStream.Position and FastBinaryReader.BaseStream.Length, so that my existing code will stay compatible with the original BinaryReader class.
How would I go about doing this?
Here's the final implementation, if anyone is interested. Passing this as a Stream object to a BinaryReader, instead of the usual FileStream object, yields about a 45% improvement in speed on my machine, when reading 1000 byte chunks.
Note that the Length param is only accurate when reading, since Length is read in at the start and doesn't change. If you are writing, it will not update as the length of the underlying stream changes.
public class FastFileStream : FileStream
{
private long _position;
private long _length;
public FastFileStream(string path, FileMode fileMode) : base(path, fileMode)
{
_position = base.Position;
_length = base.Length;
}
public override long Length
{
get { return _length; }
}
public override long Position
{
get { return _position; }
set
{
base.Position = value;
_position = value;
}
}
public override long Seek(long offset, SeekOrigin seekOrigin)
{
switch (seekOrigin)
{
case SeekOrigin.Begin:
_position = offset;
break;
case SeekOrigin.Current:
_position += offset;
break;
case SeekOrigin.End:
_position = Length + offset;
break;
}
return base.Seek(offset, seekOrigin);
}
public override int Read(byte[] array, int offset, int count)
{
_position += count;
return base.Read(array, offset, count);
}
public override int ReadByte()
{
_position += 1;
return base.ReadByte();
}
}
I wouldn't do this exactly the way you have it here.
Consider that you need to expose a property of type Stream (what BinaryReader.BaseStream is). So you 'll need to create your own class deriving from Stream. This class would need to:
take a reference to FastBinaryReader so that it can override Stream.Length and Stream.Offset by delegating to a FastBinaryReader member
take a reference to a Stream (the same one passed in the FastBinaryReader constructor) in order to delegate all other operations to that stream (you could have these throw new NotImplementedException() instead, but you never know which library method is going to call them!)
You can imagine how it'd look:
private class StreamWrapper : Stream
{
private readonly FastBinaryReader reader;
private readonly Stream baseStream;
public StreamWrapper(FastBinaryReader reader, Stream baseStream)
{
this.reader = reader;
this.baseStream = baseStream;
}
public override long Length
{
get { return reader.Length; }
}
public override long Position
{
get { return reader.Position; }
set { reader.Position = value; }
}
// Override all other Stream virtuals as well
}
This would work, but it seems to me to be slightly clumsy. The logical continuation would be to put the caching in StreamWrapper instead of inside FastBinaryReader itself:
private class StreamWrapper : Stream
{
private readonly Stream baseStream;
public StreamWrapper(Stream baseStream)
{
this.baseStream = baseStream;
}
public override long Length
{
get { /* caching implementation */ }
}
public override long Position
{
get { /* caching implementation */ }
set { /* caching implementation */ }
}
// Override all other Stream virtuals as well
}
This would allow you to use StreamWrapper transparently and keep the caching behavior. But it raises the question: is the Stream you work with so dumb that it doesn't cache this by itself?
And if it isn't, maybe the performance gain you see is the result of that if statement inside ReadBytes and not of caching Length and Position?
I have a stream that contains many pieces of data. I want to expose just a piece of that data in another stream. The piece of data I want to extract can often be over 100mb. Since I already have stream with the data in it it seems like a waste to copy that data to another stream and return that. What im looking for is a way to reference the data in the first stream while controlling how much of it the second stream can reference. Is this possible
There is a good implementation of this by Mark Gravell detailed here. The code posted there is:
using System.IO;
using System;
static class Program
{
// shows that we can read a subset of an existing stream...
static void Main()
{
byte[] buffer = new byte[255];
for (byte i = 0; i < 255; i++)
{
buffer[i] = i;
}
using(MemoryStream ms = new MemoryStream(buffer))
using (SubStream ss = new SubStream(ms, 10, 200))
{
const int BUFFER_SIZE = 17; // why not...
byte[] working = new byte[BUFFER_SIZE];
int read;
while ((read = ss.Read(working, 0, BUFFER_SIZE)) > 0)
{
for (int i = 0; i < read; i++)
{
Console.WriteLine(working[i]);
}
}
}
}
}
class SubStream : Stream
{
private Stream baseStream;
private readonly long length;
private long position;
public SubStream(Stream baseStream, long offset, long length)
{
if (baseStream == null) throw new ArgumentNullException("baseStream");
if (!baseStream.CanRead) throw new ArgumentException("can't read base stream");
if (offset < 0) throw new ArgumentOutOfRangeException("offset");
this.baseStream = baseStream;
this.length = length;
if (baseStream.CanSeek)
{
baseStream.Seek(offset, SeekOrigin.Current);
}
else
{ // read it manually...
const int BUFFER_SIZE = 512;
byte[] buffer = new byte[BUFFER_SIZE];
while (offset > 0)
{
int read = baseStream.Read(buffer, 0, offset < BUFFER_SIZE ? (int) offset : BUFFER_SIZE);
offset -= read;
}
}
}
public override int Read(byte[] buffer, int offset, int count)
{
CheckDisposed();
long remaining = length - position;
if (remaining <= 0) return 0;
if (remaining < count) count = (int) remaining;
int read = baseStream.Read(buffer, offset, count);
position += read;
return read;
}
private void CheckDisposed()
{
if (baseStream == null) throw new ObjectDisposedException(GetType().Name);
}
public override long Length
{
get { CheckDisposed(); return length; }
}
public override bool CanRead
{
get { CheckDisposed(); return true; }
}
public override bool CanWrite
{
get { CheckDisposed(); return false; }
}
public override bool CanSeek
{
get { CheckDisposed(); return false; }
}
public override long Position
{
get {
CheckDisposed();
return position;
}
set { throw new NotSupportedException(); }
}
public override long Seek(long offset, SeekOrigin origin)
{
throw new NotSupportedException();
}
public override void SetLength(long value)
{
throw new NotSupportedException();
}
public override void Flush()
{
CheckDisposed(); baseStream.Flush();
}
protected override void Dispose(bool disposing)
{
base.Dispose(disposing);
if (disposing)
{
if (baseStream != null)
{
try { baseStream.Dispose(); }
catch { }
baseStream = null;
}
}
}
public override void Write(byte[] buffer, int offset, int count)
{
throw new NotImplementedException();
}
}
You need to make your own Stream class that validates its position and returns the desired subset.
I'm not aware of any built-in classes that do this.
Looks like StreamMuxer project was created with similar purpose in mind.
I also needed a substream so that I could work on a ZIP archive inside of another file, the other answer didn't implement seeking though so here's one with seeking; be aware it will not dispose the original stream when disposed:
public class SubStream : Stream
{
Stream Vector;
long Offset, _Length, _Position = 0;
public SubStream(Stream vector, long offset, long length)
{
if (length < 1) throw new ArgumentException("Length must be greater than zero.");
this.Vector = vector;
this.Offset = offset;
this._Length = length;
vector.Seek(offset, SeekOrigin.Begin);
}
public override int Read(byte[] buffer, int offset, int count)
{
CheckDisposed();
long remaining = _Length - _Position;
if (remaining <= 0) return 0;
if (remaining < count) count = (int)remaining;
int read = Vector.Read(buffer, offset, count);
_Position += read;
return read;
}
private void CheckDisposed()
{
if (Vector == null) throw new ObjectDisposedException(GetType().Name);
}
public override long Seek(long offset, SeekOrigin origin)
{
long pos = _Position;
if (origin == SeekOrigin.Begin)
pos = offset;
else if (origin == SeekOrigin.End)
pos = _Length + offset;
else if (origin == SeekOrigin.Current)
pos += offset;
if (pos < 0) pos = 0;
else if (pos >= _Length) pos = _Length - 1;
_Position = Vector.Seek(this.Offset + pos, SeekOrigin.Begin) - this.Offset;
return pos;
}
public override bool CanRead => true;
public override bool CanSeek => true;
public override bool CanWrite => false;
public override long Length => _Length;
public override long Position { get => _Position; set { _Position = this.Seek(value, SeekOrigin.Begin); } }
public override void Flush()
{
throw new NotImplementedException();
}
public override void SetLength(long value)
{
throw new NotImplementedException();
}
public override void Write(byte[] buffer, int offset, int count)
{
throw new NotImplementedException();
}
}
What exactly are you scared of duplicating? I doubt you have anything super performance critical, parse your Stream on the fly and use a MemoryStream till you find you need something else.