Develop Reference

Write only stream - get written bytes count with DataContractSerializer

Consider I have the following snippet:
public void Store(Stream s, object t)
{
var serializer = new DataContractSerializer(target.GetType(),
new DataContractSerializerSettings
{
PreserveObjectReferences = true
});
serializer.WriteObject(s, target);
}
where s is write-only and doesn't support seeking.
Is there any way to get bytes count that was written to stream by WriteObject? I know I can do it in the following way:
using (var memStream = new MemoryStream())
{
serializer.WriteObject(serializer, target);
Debug.WriteLine(memStream.Length);
memStream.CopyTo(s);
}
but I wonder wheter it is possible to avoid CopyTo - object is quite huge.
EDIT:
I've just came up with an idea: I can create a wrapper that counts bytes on write. It's the best solution so fat, but maybe there is another way.
DONE
I've implemented a wrapper: https://github.com/pwasiewicz/counted-stream - maybe it will be useful for someone.
Thanks!

Sample implementation of a wrapper I've made:
public class CountedStream : Stream
{
private readonly Stream stream;
public CountedStream(Stream stream)
{
if (stream == null) throw new ArgumentNullException("stream");
this.stream = stream;
}
public long WrittenBytes { get; private set; }
public override void Flush()
{
this.stream.Flush();
}
public override int Read(byte[] buffer, int offset, int count)
{
return this.stream.Read(buffer, offset, count);
}
public override long Seek(long offset, SeekOrigin origin)
{
return this.stream.Seek(offset, origin);
}
public override void SetLength(long value)
{
this.stream.SetLength(value);
}
public override void Write(byte[] buffer, int offset, int count)
{
if (buffer.Length >= offset + count)
throw new ArgumentException("Count exceeds buffer size");
this.stream.Write(buffer, offset, count);
this.WrittenBytes += count;
}
public override bool CanRead
{
get { return this.stream.CanRead; }
}
public override bool CanSeek
{
get { return this.stream.CanSeek; }
}
public override bool CanWrite
{
get { return this.stream.CanWrite; }
}
public override long Length
{
get { return this.stream.Length; }
}
public override bool CanTimeout
{
get { return this.stream.CanTimeout; }
}
public override long Position
{
get { return this.stream.Position; }
set { this.stream.Position = value; }
}
}

Streaming data from a NVarchar(Max) column using c#

I want to put the contents of some files into the database to be read by a seperate process. This is a two step thing as the files will be uploaded to a java server but then processed by a seperate c# application that runs periodically.
I was planning on using a nvarchar(Max) column to represent the data but I can't see how to read from that sort of column in a sensible way. I don't want to use SqlDataReader.GetString as that will force me to hold all the data in memory at once. The files aren't massive but that just seems like a stupid thing to do - it'll give me it as a single string which will then need splitting up into lines, so the whole approach would be totally backwards.
I was assuming I'd just be able to use a normal stream reader but calling GetStream fails saying it doesn't work for this type of column.
Any ideas? Is it just going to be easier for the database to pretend this isn't really text and store it as bytes so I can stream it?

I wrote this extension method some time ago:
public static class DataExtensions
{
public static Stream GetStream(this IDataRecord record, int ordinal)
{
return new DbBinaryFieldStream(record, ordinal);
}
public static Stream GetStream(this IDataRecord record, string name)
{
int i = record.GetOrdinal(name);
return record.GetStream(i);
}
private class DbBinaryFieldStream : Stream
{
private readonly IDataRecord _record;
private readonly int _fieldIndex;
private long _position;
private long _length = -1;
public DbBinaryFieldStream(IDataRecord record, int fieldIndex)
{
_record = record;
_fieldIndex = fieldIndex;
}
public override bool CanRead
{
get { return true; }
}
public override bool CanSeek
{
get { return true; }
}
public override bool CanWrite
{
get { return false; }
}
public override void Flush()
{
throw new NotSupportedException();
}
public override long Length
{
get
{
if (_length < 0)
{
_length = _record.GetBytes(_fieldIndex, 0, null, 0, 0);
}
return _length;
}
}
public override long Position
{
get
{
return _position;
}
set
{
_position = value;
}
}
public override int Read(byte[] buffer, int offset, int count)
{
long nRead = _record.GetBytes(_fieldIndex, _position, buffer, offset, count);
_position += nRead;
return (int)nRead;
}
public override long Seek(long offset, SeekOrigin origin)
{
long newPosition = _position;
switch (origin)
{
case SeekOrigin.Begin:
newPosition = offset;
break;
case SeekOrigin.Current:
newPosition = _position + offset;
break;
case SeekOrigin.End:
newPosition = this.Length - offset;
break;
default:
break;
}
if (newPosition < 0)
throw new ArgumentOutOfRangeException("offset");
_position = newPosition;
return _position;
}
public override void SetLength(long value)
{
throw new NotSupportedException();
}
public override void Write(byte[] buffer, int offset, int count)
{
throw new NotSupportedException();
}
}
}
It's designed for a BLOB, but it works for a NVARCHAR(max) as well (at least on SQL Server).
You can use it like this:
using (var dataReader = command.ExecuteReader())
{
dataReader.Read();
using (var stream = dataReader.GetStream("Text"))
using (var streamReader = new StreamReader(stream))
{
// read the text using the StreamReader...
}
}

Stream to file while returning a WCF stream?

I have a WCF service that streams large files (100MB+) from a remote FTP server.
[ServiceContract]
public interface IStreamService
{
[OperationContract]
Stream GetDataFromFtp();
}
public class StreamService : IStreamService
{
public Stream GetDataFromFtp()
{
Stream ftpStream = Retr(...);
return ftpStream;
}
}
As WCF streams the file to the client, I want to stream it into a local cache, so that future requests don't need to go all the way back to the remote FTP - I can just serve them off disk.
I'm having problems achieving this without buffering the entire 100MB file stream in memory before returning it.
I tried using a simple capturing stream wrapper that does a write for each read:
public class CapturingStreamWrapper : Stream
{
private readonly Stream stream;
private readonly Stream captureStream;
public override int Read(byte[] buffer, int offset, int count)
{
int readBytes = stream.Read(buffer, offset, count);
captureStream.Write(buffer, offset, readBytes);
return readBytes;
}
}
public class StreamService : IStreamService
{
public Stream GetDataFromFtp()
{
Stream ftpStream = Retr(...);
Stream cacheStream = File.OpenWrite(...);
return new CapturingStreamWrapper(ftpStream, cacheStream);
}
}
But this didn't seem to work.
Additionally this doesn't provide any error handling - I need a catch block for the cache to delete any half-written file if the client transfer fails (i.e. transactional cache). I'm not sure how this would work because I don't know when streams are called/cleaned up in the WCF lifecycle.
Any ideas how I can stream into a file as I stream back to the client?

I ended up writing a couple of interconnected stream classes - one that pipes to the other as it reads. Apologies for the large code paste:
/// <summary>
/// A stream that, as it reads, makes those bytes available on an ouput
/// stream. Thread safe.
/// </summary>
public class CacheStream : Stream
{
private readonly Stream stream;
public CacheStream(Stream stream)
{
if (stream == null) throw new ArgumentNullException("stream");
this.stream = stream;
OutputStream = new CacheOutputStream(this);
}
public event EventHandler<BytesReadEventArgs> BytesRead = delegate { };
public event EventHandler Closing = delegate { };
public Stream OutputStream { get; private set; }
public override void Flush()
{
stream.Flush();
}
public override long Seek(long offset, SeekOrigin origin)
{
throw new InvalidOperationException("Cannot seek in CachingStream.");
}
public override void SetLength(long value)
{
stream.SetLength(value);
}
public override int Read(byte[] buffer, int offset, int count)
{
int numberOfBytesRead = stream.Read(buffer, offset, count);
if (numberOfBytesRead > 0)
PipeToOutputStream(buffer, offset, numberOfBytesRead);
return numberOfBytesRead;
}
private void PipeToOutputStream(byte[] buffer, int offset, int numberOfBytesRead)
{
var tmp = new byte[numberOfBytesRead];
Array.Copy(buffer, offset, tmp, 0, numberOfBytesRead);
BytesRead(this, new BytesReadEventArgs(tmp));
}
public override void Write(byte[] buffer, int offset, int count)
{
throw new InvalidOperationException("Cannot write in CachingStream.");
}
public override bool CanRead
{
get { return stream.CanRead; }
}
public override bool CanSeek
{
get { return false; }
}
public override bool CanWrite
{
get { return false; }
}
public override long Length
{
get { return stream.Length; }
}
public override long Position
{
get { return stream.Position; }
set { throw new InvalidOperationException("Cannot set position in CachingStream."); }
}
public override void Close()
{
Closing(this, EventArgs.Empty);
base.Close();
}
protected override void Dispose(bool disposing)
{
base.Dispose(disposing);
OutputStream.Dispose();
}
}
And
/// <summary>
/// Output portion of CacheStream. Streams bytes from a queue of buffers.
/// Thread safe.
/// </summary>
public class CacheOutputStream : Stream
{
private volatile int position;
private volatile int length;
private volatile bool sourceIsClosed;
// No Deque<T> in the BCL yet, but LinkedList is more or less the same.
private readonly LinkedList<byte[]> buffers = new LinkedList<byte[]>();
public CacheOutputStream(CacheStream stream)
{
if (stream == null) throw new ArgumentNullException("stream");
stream.BytesRead += (o, e) => AddToQueue(e.Buffer);
stream.Closing += (o, e) => sourceIsClosed = true;
}
private void AddToQueue(byte[] buffer)
{
if (buffer.Length == 0)
return;
lock (buffers)
{
buffers.AddLast(buffer);
length += buffer.Length;
}
}
public override int Read(byte[] buffer, int offset, int count)
{
if (buffer == null) throw new ArgumentNullException("buffer");
bool noMoreBuffersAvailable = HasNoMoreBuffersAvailable();
// Guard clause - closed and nothing more to write.
if (noMoreBuffersAvailable && sourceIsClosed)
return 0;
if (noMoreBuffersAvailable)
{
// Not closed yet! Block infinitely until we get closed or have some data.
while (HasNoMoreBuffersAvailable())
{
if (sourceIsClosed)
return 0;
Thread.Sleep(TimeSpan.FromMilliseconds(50));
}
}
byte[] currentBuffer = GetCurrentBuffer();
int numberOfBytesRead = DoRead(buffer, count, currentBuffer, offset);
PutLeftoverBytesAtFrontOfQueue(currentBuffer, numberOfBytesRead);
return numberOfBytesRead;
}
// Check if caller didn't have enough space to fit the buffer.
// Put the remaining bytes at the front of the queue.
private void PutLeftoverBytesAtFrontOfQueue(byte[] currentBuffer, int numberOfBytesRead)
{
if (currentBuffer == null) throw new ArgumentNullException("currentBuffer");
if (numberOfBytesRead == currentBuffer.Length)
return; // Clean read!
var remainingBuffer = new byte[currentBuffer.Length - numberOfBytesRead];
Array.Copy(currentBuffer, numberOfBytesRead, remainingBuffer, 0, remainingBuffer.Length);
lock (buffers)
buffers.AddFirst(remainingBuffer);
}
private int DoRead(byte[] buffer, int count, byte[] currentBuffer, int offset)
{
int maxNumberOfBytesWeCanWrite = Math.Min(count, currentBuffer.Length);
Array.Copy(currentBuffer, 0, buffer, offset, maxNumberOfBytesWeCanWrite);
position += maxNumberOfBytesWeCanWrite;
return maxNumberOfBytesWeCanWrite;
}
private byte[] GetCurrentBuffer()
{
byte[] currentBuffer;
lock (buffers)
{
currentBuffer = buffers.First.Value;
buffers.RemoveFirst();
}
return currentBuffer;
}
private bool HasNoMoreBuffersAvailable()
{
lock (buffers)
return buffers.Count == 0;
}
public override void Flush() { }
public override long Seek(long offset, SeekOrigin origin)
{
throw new InvalidOperationException("Cannot seek in CachingStream.");
}
public override void SetLength(long value)
{
throw new InvalidOperationException("Cannot set length in CachingStream.");
}
public override void Write(byte[] buffer, int offset, int count)
{
throw new InvalidOperationException("Cannot write in a CachingStream.");
}
public override bool CanRead
{
get { return true; }
}
public override bool CanSeek
{
get { return false; }
}
public override bool CanWrite
{
get { return false; }
}
public override long Length
{
get { return length; }
}
public override long Position
{
get { return position; }
set { throw new InvalidOperationException("Cannot set position in CachingStream."); }
}
}

C#: Implementing NetworkStream.Peek?

Currently, there isn't a NetworkStream.Peek method in C#. What is the best way of implementing such a method which functions just like NetworkStream.ReadByte except that the returned byte is not actually removed from the Stream?

I ran into the same 'peek for magic number and then decide which stream processor to send the stream to' requirement and unfortunately can't weasel my way out of that problem - as suggested in comments to Aaronaught's answer - by passing the already consumed bytes into the stream processing methods in separate parameters, as those methods are a given and they expect System.IO.Stream and nothing else.
I solved this by creating a more or less universal PeekableStream class that wraps a Stream. It works for NetworkStreams, but also for any other Stream, provided you Stream.CanRead it.
Edit
Alternatively, you could use the brand new ReadSeekableStream and do
var readSeekableStream = new ReadSeekableStream(networkStream, /* >= */ count);
...
readSeekableStream.Read(..., count);
readSeekableStream.Seek(-count, SeekOrigin.Current);
In any event, here comes PeekableStream:
/// <summary>
/// PeekableStream wraps a Stream and can be used to peek ahead in the underlying stream,
/// without consuming the bytes. In other words, doing Peek() will allow you to look ahead in the stream,
/// but it won't affect the result of subsequent Read() calls.
///
/// This is sometimes necessary, e.g. for peeking at the magic number of a stream of bytes and decide which
/// stream processor to hand over the stream.
/// </summary>
public class PeekableStream : Stream
{
private readonly Stream underlyingStream;
private readonly byte[] lookAheadBuffer;
private int lookAheadIndex;
public PeekableStream(Stream underlyingStream, int maxPeekBytes)
{
this.underlyingStream = underlyingStream;
lookAheadBuffer = new byte[maxPeekBytes];
}
protected override void Dispose(bool disposing)
{
if (disposing)
underlyingStream.Dispose();
base.Dispose(disposing);
}
/// <summary>
/// Peeks at a maximum of count bytes, or less if the stream ends before that number of bytes can be read.
///
/// Calls to this method do not influence subsequent calls to Read() and Peek().
///
/// Please note that this method will always peek count bytes unless the end of the stream is reached before that - in contrast to the Read()
/// method, which might read less than count bytes, even though the end of the stream has not been reached.
/// </summary>
/// <param name="buffer">An array of bytes. When this method returns, the buffer contains the specified byte array with the values between offset and
/// (offset + number-of-peeked-bytes - 1) replaced by the bytes peeked from the current source.</param>
/// <param name="offset">The zero-based byte offset in buffer at which to begin storing the data peeked from the current stream.</param>
/// <param name="count">The maximum number of bytes to be peeked from the current stream.</param>
/// <returns>The total number of bytes peeked into the buffer. If it is less than the number of bytes requested then the end of the stream has been reached.</returns>
public virtual int Peek(byte[] buffer, int offset, int count)
{
if (count > lookAheadBuffer.Length)
throw new ArgumentOutOfRangeException("count", "must be smaller than peekable size, which is " + lookAheadBuffer.Length);
while (lookAheadIndex < count)
{
int bytesRead = underlyingStream.Read(lookAheadBuffer, lookAheadIndex, count - lookAheadIndex);
if (bytesRead == 0) // end of stream reached
break;
lookAheadIndex += bytesRead;
}
int peeked = Math.Min(count, lookAheadIndex);
Array.Copy(lookAheadBuffer, 0, buffer, offset, peeked);
return peeked;
}
public override bool CanRead { get { return true; } }
public override long Position
{
get
{
return underlyingStream.Position - lookAheadIndex;
}
set
{
underlyingStream.Position = value;
lookAheadIndex = 0; // this needs to be done AFTER the call to underlyingStream.Position, as that might throw NotSupportedException,
// in which case we don't want to change the lookAhead status
}
}
public override int Read(byte[] buffer, int offset, int count)
{
int bytesTakenFromLookAheadBuffer = 0;
if (count > 0 && lookAheadIndex > 0)
{
bytesTakenFromLookAheadBuffer = Math.Min(count, lookAheadIndex);
Array.Copy(lookAheadBuffer, 0, buffer, offset, bytesTakenFromLookAheadBuffer);
count -= bytesTakenFromLookAheadBuffer;
offset += bytesTakenFromLookAheadBuffer;
lookAheadIndex -= bytesTakenFromLookAheadBuffer;
if (lookAheadIndex > 0) // move remaining bytes in lookAheadBuffer to front
// copying into same array should be fine, according to http://msdn.microsoft.com/en-us/library/z50k9bft(v=VS.90).aspx :
// "If sourceArray and destinationArray overlap, this method behaves as if the original values of sourceArray were preserved
// in a temporary location before destinationArray is overwritten."
Array.Copy(lookAheadBuffer, lookAheadBuffer.Length - bytesTakenFromLookAheadBuffer + 1, lookAheadBuffer, 0, lookAheadIndex);
}
return count > 0
? bytesTakenFromLookAheadBuffer + underlyingStream.Read(buffer, offset, count)
: bytesTakenFromLookAheadBuffer;
}
public override int ReadByte()
{
if (lookAheadIndex > 0)
{
lookAheadIndex--;
byte firstByte = lookAheadBuffer[0];
if (lookAheadIndex > 0) // move remaining bytes in lookAheadBuffer to front
Array.Copy(lookAheadBuffer, 1, lookAheadBuffer, 0, lookAheadIndex);
return firstByte;
}
else
{
return underlyingStream.ReadByte();
}
}
public override long Seek(long offset, SeekOrigin origin)
{
long ret = underlyingStream.Seek(offset, origin);
lookAheadIndex = 0; // this needs to be done AFTER the call to underlyingStream.Seek(), as that might throw NotSupportedException,
// in which case we don't want to change the lookAhead status
return ret;
}
// from here on, only simple delegations to underlyingStream
public override bool CanSeek { get { return underlyingStream.CanSeek; } }
public override bool CanWrite { get { return underlyingStream.CanWrite; } }
public override bool CanTimeout { get { return underlyingStream.CanTimeout; } }
public override int ReadTimeout { get { return underlyingStream.ReadTimeout; } set { underlyingStream.ReadTimeout = value; } }
public override int WriteTimeout { get { return underlyingStream.WriteTimeout; } set { underlyingStream.WriteTimeout = value; } }
public override void Flush() { underlyingStream.Flush(); }
public override long Length { get { return underlyingStream.Length; } }
public override void SetLength(long value) { underlyingStream.SetLength(value); }
public override void Write(byte[] buffer, int offset, int count) { underlyingStream.Write(buffer, offset, count); }
public override void WriteByte(byte value) { underlyingStream.WriteByte(value); }
}

If you don't need to actually retrieve the byte, you can refer to the DataAvailable property.
Otherwise, you can wrap it with a StreamReader and invoke its Peek method.
Note that neither of these are particularly reliable for reading from a network stream, due to latency issues. The data might become available (present in the read buffer) the very instant after you peek.
I'm not sure what it is that you intend to do with this, but the Read method on NetworkStream is a blocking call, so you don't really need to check for status, even if you are receiving in chunks. If you are trying to keep the application responsive while reading from the stream, you should use a thread or asynchronous call to receive the data instead.
Edit: According to this post, StreamReader.Peek is buggy on a NetworkStream, or at least has undocumented behaviour, so be careful if you choose to go that route.
Updated - response to comments
The notion of a "peek" on the actual stream itself is actually impossible; it's just a stream, and once the byte is received then it is no longer on the stream. Some streams support seeking so you could technically re-read that byte, but NetworkStream isn't one of them.
Peeking only applies when are reading the stream into a buffer; once the data is in a buffer then peeking is easy because you just check whatever's at the current position in the buffer. That's why a StreamReader is able to do this; no Stream class will generally have its own Peek method.
Now, for this problem specifically, I question whether or not this is really the right answer. I understand the idea of dynamically selecting a method for processing the stream, but do you really need to do this on the raw stream? Can you not read the stream into a byte array first, or even copy it into a MemoryStream, and process it from that point on?
The main issue I see is that if something bad happens when you're reading from a network stream, the data is gone. But if you read it into a temporary location first, you can debug this. You can find out what the data was and why the object that was trying to process the data failed halfway through.
In general, the very first thing you want to do with a NetworkStream is read it into a local buffer. The only reason I can think of not to do this is if you're reading an enormous amount of data - and even then, I might consider using the file system as an intermediate buffer if it won't fit in memory.
I don't know your exact requirements, but from what I've learned so far, my advice would be: Don't try to process your data directly from the NetworkStream unless there is a compelling reason to do so. Consider reading the data into memory or onto disk first, then processing the copy.

If you have access to the Socket object, you could try the Receive method, passing SocketFlags.Peek. This is analogous to the MSG_PEEK flag that can be passed to the recv call in BSD Sockets or Winsock.

Here is a very simple PeekStream implementation that allows you to peek a certain number of bytes at the start of the stream only (as opposed to being able to peek at any time). The peeked bytes are returned as a Stream themselves, to minimize changes to existing code.
Here's how you use it:
Stream nonSeekableStream = ...;
PeekStream peekStream = new PeekStream(nonSeekableStream, 30); // Peek max 30 bytes
Stream initialBytesStream = peekStream.GetInitialBytesStream();
ParseHeaders(initialBytesStream); // Work on initial bytes of nonSeekableStream
peekStream.Read(...) // Read normally, the read will start from the beginning
GetInitialBytesStream() returns a seekable stream that contains up to peekSize initial bytes of the underlying stream (less if the stream is shorter than peekSize).
Because of its simplicity, reading PeekStream should only be marginally slower (if at all) than reading underlying stream directly.
public class PeekStream : Stream
{
private Stream m_stream;
private byte[] m_buffer;
private int m_start;
private int m_end;
public PeekStream(Stream stream, int peekSize)
{
if (stream == null)
{
throw new ArgumentNullException("stream");
}
if (!stream.CanRead)
{
throw new ArgumentException("Stream is not readable.");
}
if (peekSize < 0)
{
throw new ArgumentOutOfRangeException("peekSize");
}
m_stream = stream;
m_buffer = new byte[peekSize];
m_end = stream.Read(m_buffer, 0, peekSize);
}
public override bool CanRead
{
get
{
return true;
}
}
public override bool CanWrite
{
get
{
return false;
}
}
public override bool CanSeek
{
get
{
return false;
}
}
public override long Length
{
get
{
throw new NotSupportedException();
}
}
public override long Position
{
get
{
throw new NotSupportedException();
}
set
{
throw new NotSupportedException();
}
}
public MemoryStream GetInitialBytesStream()
{
return new MemoryStream(m_buffer, 0, m_end, false);
}
public override long Seek(long offset, SeekOrigin origin)
{
throw new NotSupportedException();
}
public override void SetLength(long value)
{
throw new NotSupportedException();
}
public override int Read(byte[] buffer, int offset, int count)
{
// Validate arguments
if (buffer == null)
{
throw new ArgumentNullException("buffer");
}
if (offset < 0)
{
throw new ArgumentOutOfRangeException("offset");
}
if (offset + count > buffer.Length)
{
throw new ArgumentOutOfRangeException("count");
}
int totalRead = 0;
// Read from buffer
if (m_start < m_end)
{
int toRead = Math.Min(m_end - m_start, count);
Array.Copy(m_buffer, m_start, buffer, offset, toRead);
m_start += toRead;
offset += toRead;
count -= toRead;
totalRead += toRead;
}
// Read from stream
if (count > 0)
{
totalRead += m_stream.Read(buffer, offset, count);
}
// Return total bytes read
return totalRead;
}
public override void Write(byte[] buffer, int offset, int count)
{
throw new NotImplementedException();
}
public override int ReadByte()
{
if (m_start < m_end)
{
return m_buffer[m_start++];
}
else
{
return m_stream.ReadByte();
}
}
public override void Flush()
{
m_stream.Flush();
}
protected override void Dispose(bool disposing)
{
if (disposing)
{
m_stream.Dispose();
}
base.Dispose(disposing);
}
}
Disclaimer: PeekStream above is taken from a working program, but it's not comprehensively tested, so may contain bugs. It works for me, but you might uncover some corner cases where is fails.

FWIW, here is a peekable stream over a non-seekable one, optimized for just one byte ahead:
public class OneBytePeekableStream : Stream
{
private readonly bool _disposeStreamOnDispose;
private readonly Stream _stream;
private int _buffer; // byte or -1
private int _bufferLength; // 0 or 1
public OneBytePeekableStream(Stream stream, bool disposeStreamOnDispose)
{
if (stream == null)
throw new ArgumentNullException(nameof(stream));
_stream = stream;
_disposeStreamOnDispose = disposeStreamOnDispose;
}
public override long Length => _stream.Length;
public override bool CanRead => _stream.CanRead;
public override bool CanSeek => _stream.CanSeek;
public override bool CanWrite => _stream.CanWrite;
public override bool CanTimeout => _stream.CanTimeout;
public override int ReadTimeout { get => _stream.ReadTimeout; set => _stream.ReadTimeout = value; }
public override int WriteTimeout { get => _stream.WriteTimeout; set => _stream.WriteTimeout = value; }
public override long Position { get => _stream.Position - _bufferLength; set { _stream.Position = value; _bufferLength = 0; } }
public override int Read(byte[] buffer, int offset, int count)
{
if (buffer == null)
throw new ArgumentNullException(nameof(buffer));
if (offset < 0)
throw new ArgumentOutOfRangeException(nameof(offset));
if (count < 0)
throw new ArgumentOutOfRangeException(nameof(count));
if (buffer.Length - offset < count)
throw new ArgumentOutOfRangeException(nameof(count));
if (count == 0)
return 0;
if (_bufferLength == 0)
return _stream.Read(buffer, offset, count);
if (_buffer < 0)
return 0;
_bufferLength = 0;
buffer[offset] = (byte)_buffer;
if (count == 1)
return count;
var read = _stream.Read(buffer, offset + 1, count - 1);
return read + 1;
}
// this is the sole reason of this class
// returns -1 is stream is EOF
public virtual int PeekByte()
{
if (_bufferLength > 0)
return _buffer;
_buffer = _stream.ReadByte();
_bufferLength = 1;
return _buffer;
}
public override int ReadByte()
{
if (_bufferLength == 0)
return _stream.ReadByte();
if (_buffer < 0)
return -1;
_bufferLength = 0;
return _buffer;
}
public override long Seek(long offset, SeekOrigin origin)
{
var ret = _stream.Seek(offset, origin);
_bufferLength = 0;
return ret;
}
public override void Flush() => _stream.Flush();
public override void SetLength(long value) => _stream.SetLength(value);
public override void WriteByte(byte value) => _stream.WriteByte(value);
public override void Write(byte[] buffer, int offset, int count) => _stream.Write(buffer, offset, count);
protected override void Dispose(bool disposing)
{
if (disposing)
{
if (_disposeStreamOnDispose)
{
_stream.Dispose();
}
}
base.Dispose(disposing);
}
}

Is there a built-in way to handle multiple files as one stream?

I have a list of files, and i need to read them each in a specific order into byte[] of a given size. This in itself is not a problem for a single file, a simple while ((got = fs.Read(piece, 0, pieceLength)) > 0) gets the job done perfectly fine. The last piece of the file may be smaller than desired, which is fine.
Now, there is a tricky bit: If I have multiple files, I need to have one continous stream, which means that if the last piece of a file is smaller that pieceLength, then I need to read (pieceLength-got) of the next file, and then keep on going on until the end of the last file.
So essentially, given X files, I will always read pieces that are exactly pieceLength long, except for the very last piece of the very last file, which may be smaller.
I just wonder if there is already something build in .net (3.5 SP1) that does the trick. My current approach is to create a Class that takes a list of files and then exposes a Read(byte[] buffer, long index, long length) function, similar to FileStream.Read(). This should be pretty straight forward because I do not have to change my calling code that reads the data, but before I reinvent the wheel I'd just like to double check that the wheel is not already built into the BCL.
Thanks :)

I don't believe there's anything in the framework, but I'd suggest making it a bit more flexible - take an IEnumerable<Stream> in your constructor, and derive from Stream yourself. Then to get file streams you can (assuming C# 3.0) just do:
Stream combined = new CombinationStream(files.Select(file => File.Open(file));
The "ownership" part is slightly tricky here - the above would allow the combination stream to take ownership of any stream it reads from, but you may not want it to have to iterate through all the rest of the streams and close them all if it's closed prematurely.

Here is what I came up based on #jon skeet's idea.
It just implements Read which was quite sufficient for me. (but no i need help on implementing the BeginRead/EndRead method.) Here is the full code containing both sync and async - Read and BeginRead/EndRead
https://github.com/facebook-csharp-sdk/combination-stream/blob/master/src/CombinationStream-Net20/CombinationStream.cs
internal class CombinationStream : System.IO.Stream
{
private readonly System.Collections.Generic.IList<System.IO.Stream> _streams;
private int _currentStreamIndex;
private System.IO.Stream _currentStream;
private long _length = -1;
private long _postion;
public CombinationStream(System.Collections.Generic.IList<System.IO.Stream> streams)
{
if (streams == null)
{
throw new System.ArgumentNullException("streams");
}
_streams = streams;
if (streams.Count > 0)
{
_currentStream = streams[_currentStreamIndex++];
}
}
public override void Flush()
{
if (_currentStream != null)
{
_currentStream.Flush();
}
}
public override long Seek(long offset, System.IO.SeekOrigin origin)
{
throw new System.InvalidOperationException("Stream is not seekable.");
}
public override void SetLength(long value)
{
this._length = value;
}
public override int Read(byte[] buffer, int offset, int count)
{
int result = 0;
int buffPostion = offset;
while (count > 0)
{
int bytesRead = _currentStream.Read(buffer, buffPostion, count);
result += bytesRead;
buffPostion += bytesRead;
_postion += bytesRead;
if (bytesRead <= count)
{
count -= bytesRead;
}
if (count > 0)
{
if (_currentStreamIndex >= _streams.Count)
{
break;
}
_currentStream = _streams[_currentStreamIndex++];
}
}
return result;
}
public override long Length
{
get
{
if (_length == -1)
{
_length = 0;
foreach (var stream in _streams)
{
_length += stream.Length;
}
}
return _length;
}
}
public override long Position
{
get { return this._postion; }
set { throw new System.NotImplementedException(); }
}
public override void Write(byte[] buffer, int offset, int count)
{
throw new System.InvalidOperationException("Stream is not writable");
}
public override bool CanRead
{
get { return true; }
}
public override bool CanSeek
{
get { return false; }
}
public override bool CanWrite
{
get { return false; }
}
}
Also available as a NuGet package
Install-Package CombinationStream