Develop Reference

Read large files - 2GB+ for Google Drive API Upload

I'm currently working on a small backup tool written in C# that is supposed to upload files contained within a specified folder to Google Drive via its API. The program largely functions as it's supposed to, the only problem that it is unable to handle files larger than 2GB.
The problem is caused by the upload function itself which is attached down below, it uses a byte array to read the file to subsequently create a Memory Stream. As far as I'm aware (I'm still a beginner when it comes to c#), a byte array can only contain 2GB of information before returning an overflow exception. To combat this I've tried to utilize FileStream.Read (second bit of code attached below) instead of System.IO.File.ReadAllBytes, though this again lead to an overflow exception of the byte Array. I know that at this point I'd have to split the file up, however, due to the rather limited documentation of the GDrive API for C# - at least from what I've seen - and my limited knowledge of C# I've got little to no clue on how to tackle this problem.
I'm sorry for the long read, all help on this matter is highly appreciated.
Upload Function V1 (System.IO.File.ReadAllBytes):
private static Google.Apis.Drive.v3.Data.File UploadFile(Boolean useFolder, String mime, DriveService _service, string _uploadFile, string _parent, string _descrp = "")
{
if (System.IO.File.Exists(_uploadFile))
{
Google.Apis.Drive.v3.Data.File body = new Google.Apis.Drive.v3.Data.File
{
Name = System.IO.Path.GetFileName(_uploadFile),
Description = _descrp,
MimeType = mime
};
if (useFolder)
{
body.Parents = new List<string> { _parent };
}
byte[] byteArray = System.IO.File.ReadAllBytes(_uploadFile);
MemoryStream stream = new System.IO.MemoryStream(byteArray);
try
{
FilesResource.CreateMediaUpload request = _service.Files.Create(body, stream, mime);
request.SupportsTeamDrives = true;
request.Upload();
return request.ResponseBody;
}
catch (Exception e)
{
Console.WriteLine("Error Occured: " + e);
return null;
}
}
else
{
Console.WriteLine("The file does not exist. 404");
return null;
}
}
Upload Method V2 (FileStream):
private static Google.Apis.Drive.v3.Data.File UploadFile(Boolean useFolder, String mime, DriveService _service, string _uploadFile, string _parent, string _descrp = "")
{
if (System.IO.File.Exists(_uploadFile))
{
Google.Apis.Drive.v3.Data.File body = new Google.Apis.Drive.v3.Data.File
{
Name = System.IO.Path.GetFileName(_uploadFile),
Description = _descrp,
MimeType = mime
};
if (useFolder)
{
body.Parents = new List<string> { _parent };
}
//byte[] byteArray = System.IO.File.ReadAllBytes(_uploadFile);
using (FileStream fileStream = new FileStream(_uploadFile, FileMode.Open, FileAccess.Read))
{
Console.WriteLine("ByteArrayStart");
byte[] byteArray = new byte[fileStream.Length];
int bytesToRead = (int)fileStream.Length;
int bytesRead = 0;
while (bytesRead > 0)
{
int n = fileStream.Read(byteArray, bytesRead, bytesToRead);
if (n == 0)
{
break;
}
bytesRead += n;
Console.WriteLine("Bytes Read: " + bytesRead);
bytesToRead -= n;
Console.WriteLine("Bytes to Read: " + bytesToRead);
}
bytesToRead = byteArray.Length;
MemoryStream stream = new System.IO.MemoryStream(byteArray);
try
{
FilesResource.CreateMediaUpload request = _service.Files.Create(body, stream, mime);
request.SupportsTeamDrives = true;
request.Upload();
return request.ResponseBody;
}
catch (Exception e)
{
Console.WriteLine("Error Occured: " + e);
return null;
}
}
}
else
{
Console.WriteLine("The file does not exist. 404");
return null;
}
}

MemoryStream's constructors only work with byte arrays that are limited to Int32.MaxValue bytes. Why not just use your FileStream object directly?
var fileMetadata = new Google.Apis.Drive.v3.Data.File()
{
Name = "flag.jpg"
};
FilesResource.CreateMediaUpload request;
using (var stream = new System.IO.FileStream(#"C:\temp\flag.jpg", System.IO.FileMode.Open))
{
request = service.Files.Create(fileMetadata, stream, "image/jpeg");
request.Fields = "id";
request.Upload();
}
var file = request.ResponseBody;
Really a file that big you should be using resumable upload but im going to have to dig around for some sample code for that.

Send parts of a byte[] by webclient to create a large file in server side

So trying to Transfer a large byte[] for that chose to separate it in chunks of 20MB, and in the first chunk received create the file and add that,the rest open the existing file and add the remaining.The problem that I am having is instead of send the first part and reconnect to receive the second part its establishing the two connections and sending the two chunks at the same time.. how can send the second after the first as finished?
client.OpenWriteAsync(ub.Uri);
void client_OpenWriteCompleted(object sender, OpenWriteCompletedEventArgs e)
{
if (e.Cancelled)
{
MessageBox.Show("Cancelled");
}
else if (e.Error != null)
{
MessageBox.Show("Deu erro");
}
else
{
try
{
using (Stream output = e.Result)
{
int countbytes;
//for (int i = 0; i < max; i++)
//{
if ( (max+1) != maxAux)
{
countbytes = zippedMemoryStream.Read(PartOfDataSet, 0 , 20000000);//maxAux * 20000000
output.Write(PartOfDataSet, 0, (int)countbytes);
if (max != maxAux)
{
client.OpenWriteAsync(ub.Uri);
}
maxAux++;
}
//}
//numeroimagem++;
}
}
catch (Exception ex)
{
MessageBox.Show(ex.Message);
//throw;
}
}
}
public void ProcessRequest(HttpContext context)
{
//context.Response.ContentType = "text/plain";
//context.Response.Write("Hello World");
string ImageName = context.Request.QueryString["ImageName"];
string UploadPath = context.Server.MapPath("~/ServerImages/");
byte[] bytes = new byte[20000000];
int bytesToRead = 0;
if (!File.Exists(UploadPath + ImageName))
{
using (FileStream stream = File.Create(UploadPath + ImageName))
{
try
{
//List<byte> bytes = new List<byte>();
while ((bytesToRead =
context.Request.InputStream.Read(bytes, 0, bytes.Length)) != 0)
//context.Request.InputStream.Read(bytes, 0, 200000)) != 0)
{
stream.Write(bytes, 0, bytesToRead);
stream.Close();
}
bytes = null;
}
catch (Exception ex)
{
string error = ex.Message;
throw;
}
}
}
else
{
using (FileStream stream = File.Open(UploadPath + ImageName,FileMode.Append))
{
try
{
while ((bytesToRead =
context.Request.InputStream.Read(bytes, 0, bytes.Length)) != 0)
{
stream.Write(bytes, 0, bytesToRead);
stream.Close();
}
bytes = null;
}
catch (Exception ex)
{
string error = ex.Message;
throw;
}
}
}
}
public bool IsReusable
{
get
{
return false;
}
}

In fact you gain nothing by opening the client again. You can just loop over your zippedMemoryStream in chunks and write it to the output stream. Then the data will arrive in order on the server side.
Otherwise look into the UploadData methods, if you really want to create a new connection each time.

C# FTP Upload Server returned an error: 550 The process cannot access the file because it is being used by another process

Hi I'm trying to upload a file with a passed in byte[] and am getting a 550.
I'm using a Memory stream in a using statement so I thought it should close itself off nicely.
If I'm dealing with a byte array rather than a physical file what could be causing the lock as the byte array doesn't map to a file per-say it is generated by a csv helper.
Basically I build up a csv with a stringbuilder and the call Export which takes the string builder and puts it in to a byte[] which then gets passed in to this method..
Just wondering if anyone could see what I'm doing wrong.
Thanks!
bool PutFileToServer(string remoteFile, byte[] bytesToUpload, bool result)
{
try
{
LogStatus("Creating FTP Request");
var ftpRequest = CreateRequest(remoteFile, WebRequestMethods.Ftp.UploadFile);
using (var dataStream = new MemoryStream(bytesToUpload))
using (var requestStream = ftpRequest.GetRequestStream())
{
dataStream.Position = 0;
byte[] byteBuffer = new byte[bufferSize];
int bytesSent = dataStream.Read(byteBuffer, 0, bufferSize);
LogStatus("Reading File Stream");
try
{
while (bytesSent != 0)
{
requestStream.Write(byteBuffer, 0, bytesSent);
bytesSent = dataStream.Read(byteBuffer, 0, bufferSize);
}
}
catch (Exception ex)
{
LogStatus("Error Reading File Bytes");
LogStatus(ex.Message);
result = false;
Elmah.ErrorSignal.FromCurrentContext().Raise(ex);
if (dataStream != null)
{
dataStream.Close();
}
if (requestStream != null)
{
requestStream.Close();
}
if (ftpStream != null)
{
ftpStream.Close();
}
ftpRequest = null;
}
result = true;
}
}
catch (Exception ex)
{
LogStatus(ex.Message);
result = false;
Elmah.ErrorSignal.FromCurrentContext().Raise(ex);
if (ftpStream != null)
{
ftpStream.Close();
}
ftpRequest = null;
}
return result;
}

Save streaming data to a WAV file using NAudio

I want to save the incoming stream data to a WAV file on my hard disk drive. How can I change the code below to be able to record the stream into a valid WAV file?
From the demo here:
private void StreamMP3(object state)
{
this.fullyDownloaded = false;
string url = (string)state;
webRequest = (HttpWebRequest)WebRequest.Create(url);
HttpWebResponse resp = null;
try
{
resp = (HttpWebResponse)webRequest.GetResponse();
}
catch(WebException e)
{
if (e.Status != WebExceptionStatus.RequestCanceled)
{
ShowError(e.Message);
}
return;
}
byte[] buffer = new byte[16384 * 4]; // Needs to be big enough to hold a decompressed frame
IMp3FrameDecompressor decompressor = null;
try
{
using (var responseStream = resp.GetResponseStream())
{
var readFullyStream = new ReadFullyStream(responseStream);
do
{
if (bufferedWaveProvider != null &&
bufferedWaveProvider.BufferLength - bufferedWaveProvider.BufferedBytes <
bufferedWaveProvider.WaveFormat.AverageBytesPerSecond / 4)
{
Debug.WriteLine("Buffer getting full, taking a break");
Thread.Sleep(500);
}
else
{
Mp3Frame frame = null;
try
{
frame = Mp3Frame.LoadFromStream(readFullyStream);
}
catch (EndOfStreamException)
{
this.fullyDownloaded = true;
// Reached the end of the MP3 file / stream
break;
}
catch (WebException)
{
// Probably we have aborted download from the GUI thread
break;
}
if (decompressor == null)
{
// I don't think these details matter too much - just help ACM select the right codec.
// However, the buffered provider doesn't know what sample rate it is working at
// until we have a frame.
WaveFormat waveFormat = new Mp3WaveFormat(
frame.SampleRate,
frame.ChannelMode == ChannelMode.Mono ? 1 : 2,
frame.FrameLength,
frame.BitRate);
decompressor = new AcmMp3FrameDecompressor(waveFormat);
this.bufferedWaveProvider = new BufferedWaveProvider(decompressor.OutputFormat);
this.bufferedWaveProvider.BufferDuration = TimeSpan.FromSeconds(20); // Allow us to get well ahead of ourselves
//this.bufferedWaveProvider.BufferedDuration = 250;
}
int decompressed = decompressor.DecompressFrame(frame, buffer, 0);
//Debug.WriteLine(String.Format("Decompressed a frame {0}", decompressed));
bufferedWaveProvider.AddSamples(buffer, 0, decompressed);
}
} while (playbackState != StreamingPlaybackState.Stopped);
Debug.WriteLine("Exiting");
// I was doing this in a finally block, but for some reason
// we are hanging on response stream .Dispose, so we never get there.
decompressor.Dispose();
}
}
finally
{
if (decompressor != null)
{
decompressor.Dispose();
}
}
}

I wouldn't take that particular approach to saving to disk. It's a bit too hands-on, because it has to deal with playing back at the right rate. Just buffer up the response, and then wrap it in an Mp3FileReader stream and use WaveFileWriter to write the WAV file:
MemoryStream mp3Buffered = new MemoryStream();
using (var responseStream = resp.GetResponseStream())
{
byte[] buffer = new byte[65536];
int bytesRead = responseStream.Read(buffer, 0, buffer.Length);
while (bytesRead > 0)
{
mp3Buffered.Write(buffer, 0, bytesRead);
bytesRead = responseStream.Read(buffer, 0, buffer.Length);
}
}
mp3Buffered.Position = 0;
using (var mp3Stream = new Mp3FileReader(mp3Buffered))
{
WaveFileWriter.CreateWaveFile("file.wav", mp3Stream);
}
That does, of course, assume that your MP3 file's wave format is compatible with WAV and in particular, your WAV player. If it isn't, you'll need to inject and add a WaveFormatConversion stream as well.

You can use following line to save to MemoryStream :
mp3Buffered.Write(frame.RawData, 0, frame.RawData.Length);
Saving stream to file is described in MattW's answer.

File Upload in Silverlight

I need to update a file which is on a remote server, using Silverlight and C#.
I created the file on the current machine and after that I tried to upload it using this example. It returned no error, but it doesn't upload my file either.
Could you help me?
this is the ashx code
<%# WebHandler Language="C#" Class="receiver" %>
using System;
using System.Web;
using System.IO;
public class receiver : IHttpHandler {
public void ProcessRequest (HttpContext context) {
string filename = context.Request.QueryString["DB.xml"].ToString();
using (FileStream fs = File.Create(context.Server.MapPath("~/CLientBin" + filename)))
{
SaveFile(context.Request.InputStream, fs);
}
}
private void SaveFile(Stream stream, FileStream fs)
{
byte[] buffer = new byte[4096];
int bytesRead;
while ((bytesRead = stream.Read(buffer, 0, buffer.Length)) != 0)
{
fs.Write(buffer, 0, bytesRead);
}
}
public bool IsReusable {
get {
return false;
}
}
}
and this is the c# code:
OpenFileDialog dlg = new OpenFileDialog();
dlg.Multiselect = false;
dlg.Filter = "All files (*.*)|*.*|PNG Images (*.png)|*.png";
bool? retval = dlg.ShowDialog();
if (retval != null && retval == true)
{
try
{
UploadFile(dlg.File.Name, dlg.File.OpenRead());
titlu.Text = dlg.File.Name;
}
catch (Exception ex)
{
MessageBox.Show(ex.ToString());
}
}
else
{
titlu.Text = "No file selected...";
}
}
private void UploadFile(string fileName, Stream data)
{
UriBuilder ub = new UriBuilder("http://ganduri.elementfx.com/Handler.ashx");
ub.Query = string.Format("filename={0}", fileName);
WebClient c = new WebClient();
c.OpenWriteCompleted += (sender, e) =>
{
PushData(data, e.Result);
e.Result.Close();
data.Close();
};
c.OpenWriteAsync(ub.Uri);
}
private void PushData(Stream input, Stream output)
{
byte[] buffer = new byte[4096];
int bytesRead;
while ((bytesRead = input.Read(buffer, 0, buffer.Length)) != 0)
{
output.Write(buffer, 0, bytesRead);
}
MessageBox.Show("Writed");
}
I have also checked the permissions to the ClientBin folder and are read/write permissions
When I run this code it shows me the "Writed" messageBox, but if I look in the server, the file isn't anywhere.
I have set a breakpoint in af ashx file functions and id doesn't reach any of them.

Think you need check upload folder for existence, and write access rights