Develop Reference

C# Download big file from Server with less memory consumption

I have a big file of memory size 42 mb. I want to download the file with less memory consumption.
Controller Code
public ActionResult Download()
{
var filePath = "file path in server";
FileInfo file = new FileInfo(filePath);
Response.ContentType = "application/zip";
Response.AppendHeader("Content-Disposition", "attachment; filename=folder.zip");
Response.TransmitFile(file.FullName);
Response.End();
}
alernative method tried with Stream
public ActionResult Download()
{
string failure = string.Empty;
Stream stream = null;
int bytesToRead = 10000;
long LengthToRead;
try
{
var path = "file path from server";
FileWebRequest fileRequest = (FileWebRequest)FileWebRequest.Create(path);
FileWebResponse fileResponse = (FileWebResponse)fileRequest.GetResponse();
if (fileRequest.ContentLength > 0)
fileResponse.ContentLength = fileRequest.ContentLength;
//Get the Stream returned from the response
stream = fileResponse.GetResponseStream();
LengthToRead = stream.Length;
//Indicate the type of data being sent
Response.ContentType = "application/octet-stream";
//Name the file
Response.AddHeader("Content-Disposition", "attachment; filename=SolutionWizardDesktopClient.zip");
Response.AddHeader("Content-Length", fileResponse.ContentLength.ToString());
int length;
do
{
// Verify that the client is connected.
if (Response.IsClientConnected)
{
byte[] buffer = new Byte[bytesToRead];
// Read data into the buffer.
length = stream.Read(buffer, 0, bytesToRead);
// and write it out to the response's output stream
Response.OutputStream.Write(buffer, 0, length);
// Flush the data
Response.Flush();
//Clear the buffer
LengthToRead = LengthToRead - length;
}
else
{
// cancel the download if client has disconnected
LengthToRead = -1;
}
} while (LengthToRead > 0); //Repeat until no data is read
}
finally
{
if (stream != null)
{
//Close the input stream
stream.Close();
}
Response.End();
Response.Close();
}
return View("Failed");
}
due to size of the file, it is consumpting more memory which leads to performance issue.
After checking in iis log, the download process is taking 42 mb and 64 mb each respectively.
Thanks in advance

A better option would be to use FileResult instead of ActionResult:
Using this method means you don't have to load the file/bytes in memory before serving.
public FileResult Download()
{
var filePath = "file path in server";
return new FilePathResult(Server.MapPath(filePath), "application/zip");
}
Edit: For larger files FilePathResult will also fail.
Your best bet is probably Response.TransmitFile() then. I've used this on larger files (GBs) and had no issues before
public ActionResult Download()
{
var filePath = #"file path from server";
Response.Clear();
Response.ContentType = "application/octet-stream";
Response.AppendHeader("Content-Disposition", "filename=" + filePath);
Response.TransmitFile(filePath);
Response.End();
return Index();
}
From MSDN:
Writes the specified file directly to an HTTP response output stream,
without buffering it in memory.

Try setting the Transfer-Encoding header to chunked, and return an HttpResponseMessage with a PushStreamContent. Transfer-Encoding of chunked means that the HTTP response will not have a Content-Length header, and so the client will have to parse the chunks of the HTTP response as a stream. Note, I've never run across a client (browser, etc) that didn't handle Transfer Encoding chunked. You can read more at the link below.
https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Transfer-Encoding
[HttpGet]
public async Task<HttpResponseMessage> Download(CancellationToken token)
{
var response = new HttpResponseMessage(System.Net.HttpStatusCode.OK)
{
Content = new PushStreamContent(async (stream, context, transportContext) =>
{
try
{
using (var fileStream = System.IO.File.OpenRead("some path to MyBigDownload.zip"))
{
await fileStream.CopyToAsync(stream);
}
}
finally
{
stream.Close();
}
}, "application/octet-stream"),
};
response.Headers.TransferEncodingChunked = true;
response.Content.Headers.ContentDisposition = new System.Net.Http.Headers.ContentDispositionHeaderValue("attachment")
{
FileName = "MyBigDownload.zip"
};
return response;
}

I had similar problem but I didn't have file on local disk, I had to download it from API (my MVC was like a proxy).
The key thing is to set Response.Buffer=false; on your MVC Action. I think #JanusPienaar's first solution should work with this.
My MVC action is:
public class HomeController : Controller
{
public async Task<FileStreamResult> Streaming(long RecordCount)
{
HttpClient Client;
System.IO.Stream Stream;
//This is the key thing
Response.Buffer=false;
Client = new HttpClient() { BaseAddress=new Uri("http://MyApi", };
Stream = await Client.GetStreamAsync("api/Streaming?RecordCount="+RecordCount);
return new FileStreamResult(Stream, "text/csv");
}
}
And my test WebApi (which generates the file) is:
public class StreamingController : ApiController
{
// GET: api/Streaming/5
public HttpResponseMessage Get(long RecordCount)
{
var response = Request.CreateResponse();
response.Content=new PushStreamContent((stream, http, transport) =>
{
RecordsGenerator Generator = new RecordsGenerator();
long i;
using(var writer = new System.IO.StreamWriter(stream, System.Text.Encoding.UTF8))
{
for(i=0; i<RecordCount; i++)
{
writer.Write(Generator.GetRecordString(i));
if(0==(i&0xFFFFF))
System.Diagnostics.Debug.WriteLine($"Record no: {i:N0}");
}
}
});
return response;
}
class RecordsGenerator
{
const string abc = "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
char[] Chars = new char[14];//Ceiling(log26(2^63))
public string GetRecordString(long Record)
{
int iLength = 0;
long Div = Record, Mod;
do
{
iLength++;
Div=Math.DivRem(Div, abc.Length, out Mod);
//Save from backwards
Chars[Chars.Length-iLength]=abc[(int)Mod];
}
while(Div!=0);
return $"{Record} {new string(Chars, Chars.Length-iLength, iLength)}\r\n";
}
}
}
}
If RecordCount is 100000000, the file generated by TestApi is 1.56 GB. Neither WebApi nor MVC consumes so much memory.

There is the Rizwan Ansari post that worked for me:
There are situation when you need to provide download option for a big file located somewhere on server or generated at runtime. Below function could be used to download files of any size. Sometimes downloading big file throws exception OutOfMemoryException showing “Insufficient memory to continue execution of the program”. So this function also handle this situation by breaking down file in 1 MB chunks (can be customized by changing bufferSize variable).
Usage:
DownloadLargeFile("A big file.pdf", "D:\\Big Files\\Big File.pdf", "application/pdf", System.Web.HttpContext.Current.Response);
You can change "application/pdf" by the right Mime type
Download Function:
public static void DownloadLargeFile(string DownloadFileName, string FilePath, string ContentType, HttpResponse response)
{
Stream stream = null;
// read buffer in 1 MB chunks
// change this if you want a different buffer size
int bufferSize = 1048576;
byte[] buffer = new Byte[bufferSize];
// buffer read length
int length;
// Total length of file
long lengthToRead;
try
{
// Open the file in read only mode
stream = new FileStream(FilePath, FileMode.Open, FileAccess.Read, FileShare.Read);
// Total length of file
lengthToRead = stream.Length;
response.ContentType = ContentType;
response.AddHeader("Content-Disposition", "attachment; filename=" + HttpUtility.UrlEncode(DownloadFileName, System.Text.Encoding.UTF8));
while (lengthToRead > 0)
{
// Verify that the client is connected.
if (response.IsClientConnected)
{
// Read the data in buffer
length = stream.Read(buffer, 0, bufferSize);
// Write the data to output stream.
response.OutputStream.Write(buffer, 0, length);
// Flush the data
response.Flush();
//buffer = new Byte[10000];
lengthToRead = lengthToRead - length;
}
else
{
// if user disconnects stop the loop
lengthToRead = -1;
}
}
}
catch (Exception exp)
{
// handle exception
response.ContentType = "text/html";
response.Write("Error : " + exp.Message);
}
finally
{
if (stream != null)
{
stream.Close();
}
response.End();
response.Close();
}
}

you just have to Using IIS to Enable HTTP Downloads look at this link
and you just need to return the HTTP path of the file it will be download so fast and so easy.

Network error downloading files on OSX

I have a site with a list of photos. The user has the option to download each of the photos. The download just writes the result to the output stream.
Here's the code:
[WebMethod]
public static void DownloadPhotoAsset(string assetId)
{
var photoAsset = GetPhotoAsset(assetId);
Stream stream = null;
int bytesToRead = 10000;
byte[] buffer = new Byte[bytesToRead];
try
{
HttpWebRequest fileReq =
(HttpWebRequest)HttpWebRequest.Create(photoAsset.FileAbsoluteUrl);
HttpWebResponse fileResp = (HttpWebResponse)fileReq.GetResponse();
if (fileReq.ContentLength > 0)
fileResp.ContentLength = fileReq.ContentLength;
stream = fileResp.GetResponseStream();
var resp = HttpContext.Current.Response;
resp.ContentType = "application/octet-stream";
resp.AddHeader("Content-Disposition",
"attachment; filename=\"" +
Path.GetFileName(photoAsset.FileAbsoluteUrl) + "\"");
resp.AddHeader("Content-Length", fileResp.ContentLength.ToString());
int length;
do
{
// verify that the client is connected.
if (resp.IsClientConnected)
{
// read data into the buffer and write it out
// to the response's output stream
length = stream.Read(buffer, 0, bytesToRead);
resp.OutputStream.Write(buffer, 0, length);
// flush the data and clear the buffer
resp.Flush();
buffer = new Byte[bytesToRead];
}
else
length = -1; // cancel the download if client has disconnected
} while (length > 0); //Repeat until no data is read
}
finally
{
if (stream != null)
stream.Close(); // close the input stream
}
}
This works fine in every browser on Windows, but I get network connection issues on Macs.
In Safari, the download stops after a second and says "The network connection was lost"
In Chrome, the error says "Failed - Network Error"
In Firefox, the error says "Download Error - image.jpeg.part could not be saved, because the source file could not be read"
I've checked on two different Macs with OSX 10.7.4 and and OSX 10.8.3
Anyone know what I'm doing wrong here?

Save streaming data to a WAV file using NAudio

I want to save the incoming stream data to a WAV file on my hard disk drive. How can I change the code below to be able to record the stream into a valid WAV file?
From the demo here:
private void StreamMP3(object state)
{
this.fullyDownloaded = false;
string url = (string)state;
webRequest = (HttpWebRequest)WebRequest.Create(url);
HttpWebResponse resp = null;
try
{
resp = (HttpWebResponse)webRequest.GetResponse();
}
catch(WebException e)
{
if (e.Status != WebExceptionStatus.RequestCanceled)
{
ShowError(e.Message);
}
return;
}
byte[] buffer = new byte[16384 * 4]; // Needs to be big enough to hold a decompressed frame
IMp3FrameDecompressor decompressor = null;
try
{
using (var responseStream = resp.GetResponseStream())
{
var readFullyStream = new ReadFullyStream(responseStream);
do
{
if (bufferedWaveProvider != null &&
bufferedWaveProvider.BufferLength - bufferedWaveProvider.BufferedBytes <
bufferedWaveProvider.WaveFormat.AverageBytesPerSecond / 4)
{
Debug.WriteLine("Buffer getting full, taking a break");
Thread.Sleep(500);
}
else
{
Mp3Frame frame = null;
try
{
frame = Mp3Frame.LoadFromStream(readFullyStream);
}
catch (EndOfStreamException)
{
this.fullyDownloaded = true;
// Reached the end of the MP3 file / stream
break;
}
catch (WebException)
{
// Probably we have aborted download from the GUI thread
break;
}
if (decompressor == null)
{
// I don't think these details matter too much - just help ACM select the right codec.
// However, the buffered provider doesn't know what sample rate it is working at
// until we have a frame.
WaveFormat waveFormat = new Mp3WaveFormat(
frame.SampleRate,
frame.ChannelMode == ChannelMode.Mono ? 1 : 2,
frame.FrameLength,
frame.BitRate);
decompressor = new AcmMp3FrameDecompressor(waveFormat);
this.bufferedWaveProvider = new BufferedWaveProvider(decompressor.OutputFormat);
this.bufferedWaveProvider.BufferDuration = TimeSpan.FromSeconds(20); // Allow us to get well ahead of ourselves
//this.bufferedWaveProvider.BufferedDuration = 250;
}
int decompressed = decompressor.DecompressFrame(frame, buffer, 0);
//Debug.WriteLine(String.Format("Decompressed a frame {0}", decompressed));
bufferedWaveProvider.AddSamples(buffer, 0, decompressed);
}
} while (playbackState != StreamingPlaybackState.Stopped);
Debug.WriteLine("Exiting");
// I was doing this in a finally block, but for some reason
// we are hanging on response stream .Dispose, so we never get there.
decompressor.Dispose();
}
}
finally
{
if (decompressor != null)
{
decompressor.Dispose();
}
}
}

I wouldn't take that particular approach to saving to disk. It's a bit too hands-on, because it has to deal with playing back at the right rate. Just buffer up the response, and then wrap it in an Mp3FileReader stream and use WaveFileWriter to write the WAV file:
MemoryStream mp3Buffered = new MemoryStream();
using (var responseStream = resp.GetResponseStream())
{
byte[] buffer = new byte[65536];
int bytesRead = responseStream.Read(buffer, 0, buffer.Length);
while (bytesRead > 0)
{
mp3Buffered.Write(buffer, 0, bytesRead);
bytesRead = responseStream.Read(buffer, 0, buffer.Length);
}
}
mp3Buffered.Position = 0;
using (var mp3Stream = new Mp3FileReader(mp3Buffered))
{
WaveFileWriter.CreateWaveFile("file.wav", mp3Stream);
}
That does, of course, assume that your MP3 file's wave format is compatible with WAV and in particular, your WAV player. If it isn't, you'll need to inject and add a WaveFormatConversion stream as well.

You can use following line to save to MemoryStream :
mp3Buffered.Write(frame.RawData, 0, frame.RawData.Length);
Saving stream to file is described in MattW's answer.

YouTube Direct Upload - OutOfMemory Exception

Whenever I try to upload a large video via Direct Upload using the YouTube API. I get an OutOfMemory Exception. Is there anything I can do to get rid of this? The YouTube API does not say anything about video size limit using direct upload.
I gave up on the Direct Upload. Now I trying the resumable upload way. My code is below.
YouTubeRequest request;
YouTubeRequestSettings settings = new YouTubeRequestSettings("YouTube Upload", Client Key, "Username", "Password");
request = new YouTubeRequest(settings);
Video newVideo = new Video();
ResumableUploader m_ResumableUploader = null;
Authenticator YouTubeAuthenticator;
m_ResumableUploader = new ResumableUploader(256); //chunksize 256 kilobyte
m_ResumableUploader.AsyncOperationCompleted += new AsyncOperationCompletedEventHandler(m_ResumableUploader_AsyncOperationCompleted);
m_ResumableUploader.AsyncOperationProgress += new AsyncOperationProgressEventHandler(m_ResumableUploader_AsyncOperationProgress);
YouTubeAuthenticator = new ClientLoginAuthenticator("YouTubeUploader", ServiceNames.YouTube, "kjohnson#resoluteinnovations.com", "password");
//AtomLink link = new AtomLink("http://uploads.gdata.youtube.com/resumable/feeds/api/users/uploads");
//link.Rel = ResumableUploader.CreateMediaRelation;
//newVideo.YouTubeEntry.Links.Add(link);
System.IO.FileStream stream = new System.IO.FileStream(filePath, System.IO.FileMode.Open, System.IO.FileAccess.Read);
byte[] chunk = new byte[256000];int count = 1;
while (true) {
int index = 0;
while (index < chunk.Length) {
int bytesRead = stream.Read(chunk, index, chunk.Length - index);
if (bytesRead == 0) {
break;
}
index += bytesRead;
}
if (index != 0) { // Our previous chunk may have been the last one
newVideo.MediaSource = new MediaFileSource(new MemoryStream(chunk), filePath, "video/quicktime");
if (count == 1) {
m_ResumableUploader.InsertAsync(YouTubeAuthenticator, newVideo.YouTubeEntry, new MemoryStream(chunk));
count++;
}
else
m_ResumableUploader.ResumeAsync(YouTubeAuthenticator, new Uri("http://uploads.gdata.youtube.com/resumable/feeds/api/users/uploads"), "POST", new MemoryStream(chunk), "video/quicktime", new object());
}
if (index != chunk.Length) { // We didn't read a full chunk: we're done
break;
}
}
Can anyone tell me what is wrong? My 2 GB video not uploading.

The reason I was getting a 403 Forbidden error was due to the fact that I was not passing in:
Username & Password
A developer key
The request variable in the code above is not being used/sent in the upload. Therefore I was doing an unauthorized upload.

Chances are that you are not disposing your objects. Ensure all disposable objects are within a using statement..
For example, this code will upload a large zip file to a server:
try
{
using (Stream ftpStream = FTPRequest.GetRequestStream())
{
using (FileStream file = File.OpenRead(ImagesZipFile))
{
// set up variables we'll use to read the file
int length = 1024;
byte[] buffer = new byte[length];
int bytesRead = 0;
// write the file to the request stream
do
{
bytesRead = file.Read(buffer, 0, length);
ftpStream.Write(buffer, 0, bytesRead);
}
while (bytesRead != 0);
}
}
}
catch (Exception e)
{
// throw the exception
throw e;
}

Download function not showing total size of file, while downloading

protected void downloadFunction(string filename)
{
string filepath = #"D:\XtraFiles\" + filename;
string contentType = "application/x-newton-compatible-pkg";
Stream iStream = null;
// Buffer to read 1024K bytes in chunk
byte[] buffer = new Byte[1048576];
// Length of the file:
int length;
// Total bytes to read:
long dataToRead;
try
{
// Open the file.
iStream = new FileStream(filepath, FileMode.Open, FileAccess.Read, FileShare.Read);
// Total bytes to read:
dataToRead = iStream.Length;
HttpContext.Current.Response.ContentType = contentType;
HttpContext.Current.Response.AddHeader("Content-Disposition", "attachment; filename=" + HttpUtility.UrlEncode(filename, System.Text.Encoding.UTF8));
// Read the bytes.
while (dataToRead > 0)
{
// Verify that the client is connected.
if (HttpContext.Current.Response.IsClientConnected)
{
// Read the data in buffer.
length = iStream.Read(buffer, 0, 10000);
// Write the data to the current output stream.
HttpContext.Current.Response.OutputStream.Write(buffer, 0, length);
// Flush the data to the HTML output.
HttpContext.Current.Response.Flush();
buffer = new Byte[10000];
dataToRead = dataToRead - length;
}
else
{
//prevent infinite loop if user disconnects
dataToRead = -1;
}
}
}
catch (Exception ex)
{
// Trap the error, if any.
HttpContext.Current.Response.Write("Error : " + ex.Message + "<br />");
HttpContext.Current.Response.ContentType = "text/html";
HttpContext.Current.Response.Write("Error : file not found");
}
finally
{
if (iStream != null)
{
//Close the file.
iStream.Close();
}
HttpContext.Current.Response.End();
HttpContext.Current.Response.Close();
}
}
My donwload function is working perfect, but when users are downloading the browser cant see the total file size of the download.
So now the browser says eq. Downloading 8mb of ?, insted of Downloading 8mb of 142mb.
What have i missed?

The Content-Length header seems to be what you are missing.
If you set this the browser will then know how much to expect. Otherwise it will just keep going til you stop sending data and it won't know how long it is until the end.
Response.AddHeader("Content-Length", iStream.Length);
You may also be interested in Response.WriteFile whcih can provide an easier way to send a file to a client without having to worry about streams yourself.

You need to send a ContentLength-Header:
HttpContext.Current.Response.AddHeader(HttpRequestHeader.ContentLength, iStream.Length);