Develop Reference

Read large files - 2GB+ for Google Drive API Upload

I'm currently working on a small backup tool written in C# that is supposed to upload files contained within a specified folder to Google Drive via its API. The program largely functions as it's supposed to, the only problem that it is unable to handle files larger than 2GB.
The problem is caused by the upload function itself which is attached down below, it uses a byte array to read the file to subsequently create a Memory Stream. As far as I'm aware (I'm still a beginner when it comes to c#), a byte array can only contain 2GB of information before returning an overflow exception. To combat this I've tried to utilize FileStream.Read (second bit of code attached below) instead of System.IO.File.ReadAllBytes, though this again lead to an overflow exception of the byte Array. I know that at this point I'd have to split the file up, however, due to the rather limited documentation of the GDrive API for C# - at least from what I've seen - and my limited knowledge of C# I've got little to no clue on how to tackle this problem.
I'm sorry for the long read, all help on this matter is highly appreciated.
Upload Function V1 (System.IO.File.ReadAllBytes):
private static Google.Apis.Drive.v3.Data.File UploadFile(Boolean useFolder, String mime, DriveService _service, string _uploadFile, string _parent, string _descrp = "")
{
if (System.IO.File.Exists(_uploadFile))
{
Google.Apis.Drive.v3.Data.File body = new Google.Apis.Drive.v3.Data.File
{
Name = System.IO.Path.GetFileName(_uploadFile),
Description = _descrp,
MimeType = mime
};
if (useFolder)
{
body.Parents = new List<string> { _parent };
}
byte[] byteArray = System.IO.File.ReadAllBytes(_uploadFile);
MemoryStream stream = new System.IO.MemoryStream(byteArray);
try
{
FilesResource.CreateMediaUpload request = _service.Files.Create(body, stream, mime);
request.SupportsTeamDrives = true;
request.Upload();
return request.ResponseBody;
}
catch (Exception e)
{
Console.WriteLine("Error Occured: " + e);
return null;
}
}
else
{
Console.WriteLine("The file does not exist. 404");
return null;
}
}
Upload Method V2 (FileStream):
private static Google.Apis.Drive.v3.Data.File UploadFile(Boolean useFolder, String mime, DriveService _service, string _uploadFile, string _parent, string _descrp = "")
{
if (System.IO.File.Exists(_uploadFile))
{
Google.Apis.Drive.v3.Data.File body = new Google.Apis.Drive.v3.Data.File
{
Name = System.IO.Path.GetFileName(_uploadFile),
Description = _descrp,
MimeType = mime
};
if (useFolder)
{
body.Parents = new List<string> { _parent };
}
//byte[] byteArray = System.IO.File.ReadAllBytes(_uploadFile);
using (FileStream fileStream = new FileStream(_uploadFile, FileMode.Open, FileAccess.Read))
{
Console.WriteLine("ByteArrayStart");
byte[] byteArray = new byte[fileStream.Length];
int bytesToRead = (int)fileStream.Length;
int bytesRead = 0;
while (bytesRead > 0)
{
int n = fileStream.Read(byteArray, bytesRead, bytesToRead);
if (n == 0)
{
break;
}
bytesRead += n;
Console.WriteLine("Bytes Read: " + bytesRead);
bytesToRead -= n;
Console.WriteLine("Bytes to Read: " + bytesToRead);
}
bytesToRead = byteArray.Length;
MemoryStream stream = new System.IO.MemoryStream(byteArray);
try
{
FilesResource.CreateMediaUpload request = _service.Files.Create(body, stream, mime);
request.SupportsTeamDrives = true;
request.Upload();
return request.ResponseBody;
}
catch (Exception e)
{
Console.WriteLine("Error Occured: " + e);
return null;
}
}
}
else
{
Console.WriteLine("The file does not exist. 404");
return null;
}
}

MemoryStream's constructors only work with byte arrays that are limited to Int32.MaxValue bytes. Why not just use your FileStream object directly?
var fileMetadata = new Google.Apis.Drive.v3.Data.File()
{
Name = "flag.jpg"
};
FilesResource.CreateMediaUpload request;
using (var stream = new System.IO.FileStream(#"C:\temp\flag.jpg", System.IO.FileMode.Open))
{
request = service.Files.Create(fileMetadata, stream, "image/jpeg");
request.Fields = "id";
request.Upload();
}
var file = request.ResponseBody;
Really a file that big you should be using resumable upload but im going to have to dig around for some sample code for that.

c# push notification is not working with Turkish character

I want to send notification to IOS from c#. But it does not send message contain Turkish character.
Here is my Pushmessage Function :
public bool PushMessage(string Mess, string DeviceToken, int Badge, string Custom_Field)
{
ConnectToAPNS();
List<string> Key_Value_Custom_Field = new List<string>();
String cToken = DeviceToken;
String cAlert = Mess;
int iBadge = Badge;
// Ready to create the push notification
byte[] buf = new byte[256];
MemoryStream ms = new MemoryStream();
BinaryWriter bw = new BinaryWriter(ms);
bw.Write(new byte[] { 0, 0, 32 });
byte[] deviceToken = HexToData(cToken);
bw.Write(deviceToken);
bw.Write((byte)0);
// Create the APNS payload - new.caf is an audio file saved in the application bundle on the device
string msg = "";
msg = "{\"aps\":{\"alert\":\"" + cAlert + "\",\"badge\":\"" + iBadge.ToString() + "\",\"sound\":\"noti.aiff\",\"priority\":\"10\"}";
String PayloadMess = "";
if (string.IsNullOrWhiteSpace(Custom_Field) == false)
{
List<string> list_Custom_Field = Custom_Field.Split(';').ToList();
if (list_Custom_Field.Count > 0)
{
for (int indx = 0; indx < list_Custom_Field.Count; indx++)
{
Key_Value_Custom_Field = list_Custom_Field[indx].Split('=').ToList();
if (Key_Value_Custom_Field.Count > 1)
{
if (PayloadMess != "") PayloadMess += ", ";
PayloadMess += "\"" + Key_Value_Custom_Field[0].ToString() + "\":\"" + Key_Value_Custom_Field[1].ToString() + "\"";
}
}
}
}
if (PayloadMess != "")
{
msg += ", " + PayloadMess;
}
msg += "}";
bw.Write((byte)0);
bw.Write((byte)msg.Length);
byte[] b1 = System.Text.Encoding.UTF8.GetBytes(msg);
bw.Write(b1);
bw.Flush();
if (sslStream != null)
{
sslStream.Write(ms.ToArray());
return true;
}
return false;
}

Here is the solution:
I change
bw.Write((byte)msg.Length);
byte[] b1 = System.Text.Encoding.UTF8.GetBytes(msg);
bw.Write(b1);
to
byte[] bytes = Encoding.UTF8.GetBytes(msg);
// Write the data out to the stream
bw.Write((byte)bytes.Length);
bw.Write(msg.ToCharArray());
Now it works.The problem is character count was wrong.

Probably you just used
Encoding.UTF8.GetBytes(Message);
You can try the following:
Encoding iso = Encoding.GetEncoding("ISO-8859-9"); //Turkish
Encoding utf8 = Encoding.UTF8;
byte[] utfBytes = utf8.GetBytes(Message);
byte[] isoBytes = Encoding.Convert(utf8, iso, utfBytes);
string msg = iso.GetString(isoBytes);

tracking upload progress of multiple file uploads using multipart/body web request

I'm using HttpWebRequest to upload files to the server. The request sends 2 files to the server, a video file and an image file. I'm trying to track the progress of entire progress but the issue is, the progress log runs separately for each file upload. I want the progress to show only once for both the uploads but I can't quite figure out how to do it. Here's my client side code:
Dictionary<string, string> fields = new Dictionary<string, string>();
fields.Add("username", username);
HttpWebRequest hr = WebRequest.Create(url) as HttpWebRequest;
hr.Timeout = 500000;
string bound = "----------------------------" + DateTime.Now.Ticks.ToString("x");
hr.ContentType = "multipart/form-data; boundary=" + bound;
hr.Method = "POST";
hr.KeepAlive = true;
hr.Credentials = CredentialCache.DefaultCredentials;
byte[] boundBytes = Encoding.ASCII.GetBytes("\r\n--" + bound + "\r\n");
string formDataTemplate = "\r\n--" + bound + "\r\nContent-Disposition: form-data; name=\"{0}\";\r\n\r\n{1}";
Stream s = hr.GetRequestStreamWithTimeout(1000000);
foreach (string key in fields.Keys)
{
byte[] formItemBytes = Encoding.UTF8.GetBytes(
string.Format(formDataTemplate, key, fields[key]));
s.Write(formItemBytes, 0, formItemBytes.Length);
}
s.Write(boundBytes, 0, boundBytes.Length);
string headerTemplate =
"Content-Disposition: form-data; name=\"{0}\"; filename=\"{1}\"\r\n Content-Type: application/octet-stream\r\n\r\n";
List<string> files = new List<string> { fileUrl, thumbUrl };
List<string> type = new List<string> { "video", "thumb" };
int count = 0;
foreach (string f in files)
{
var m = Path.GetFileName(f);
var t = type[count];
var j = string.Format(headerTemplate, t, m);
byte[] headerBytes = Encoding.UTF8.GetBytes(
string.Format(headerTemplate, type[count], Path.GetFileName(f)));
s.Write(headerBytes, 0, headerBytes.Length);
FileStream fs = new FileStream(f, FileMode.Open, FileAccess.Read);
int bytesRead = 0;
long bytesSoFar = 0;
byte[] buffer = new byte[1024];
while ((bytesRead = fs.Read(buffer, 0, buffer.Length)) != 0)
{
bytesSoFar += bytesRead;
s.Write(buffer, 0, buffer.Length);
Console.WriteLine(string.Format("sending file data {0:0.000}%", (bytesSoFar * 100.0f) / fs.Length));
}
s.Write(boundBytes, 0, boundBytes.Length);
fs.Close();
count += 1;
}
s.Close();
string respString = "";
hr.BeginGetResponse((IAsyncResult res) =>
{
WebResponse resp = ((HttpWebRequest)res.AsyncState).EndGetResponse(res);
StreamReader respReader = new StreamReader(resp.GetResponseStream());
respString = respReader.ReadToEnd();
resp.Close();
resp = null;
}, hr);
while (!hr.HaveResponse)
{
Console.Write("hiya bob!");
Thread.Sleep(150);
}
Console.Write(respString);
hr = null;
How do I combine the progress log for both uploads into a single log? Any help is appreciated.

One option is to calculate the total number of bytes you need to send before doing any work:
// Calculate the total size to upload before starting work
long totalToUpload = 0;
foreach (var f in files)
{
totalToUpload += (new FileInfo(f)).Length;
}
Then keep track of the total number of bytes sent in any file, and use that in your calculation of progress:
int count = 0;
long bytesSoFar = 0;
foreach (string f in files)
{
// ... Your existing work ...
while ((bytesRead = fs.Read(buffer, 0, buffer.Length)) != 0)
{
bytesSoFar += bytesRead;
// Make sure to only write the number of bytes read from the file
s.Write(buffer, 0, bytesRead);
// Console.WriteLine takes a string.Format() style string
Console.WriteLine("sending file data {0:0.000}%", (bytesSoFar * 100.0f) / totalToUpload);
}

Can't download complete image file from skydrive using REST API

I'm working on a quick wrapper for the skydrive API in C#, but running into issues with downloading a file. For the first part of the file, everything comes through fine, but then there start to be differences in the file and shortly thereafter everything becomes null. I'm fairly sure that it's just me not reading the stream correctly.
This is the code I'm using to download the file:
public const string ApiVersion = "v5.0";
public const string BaseUrl = "https://apis.live.net/" + ApiVersion + "/";
public SkyDriveFile DownloadFile(SkyDriveFile file)
{
string uri = BaseUrl + file.ID + "/content";
byte[] contents = GetResponse(uri);
file.Contents = contents;
return file;
}
public byte[] GetResponse(string url)
{
checkToken();
Uri requestUri = new Uri(url + "?access_token=" + HttpUtility.UrlEncode(token.AccessToken));
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(requestUri);
request.Method = WebRequestMethods.Http.Get;
WebResponse response = request.GetResponse();
Stream responseStream = response.GetResponseStream();
byte[] contents = new byte[response.ContentLength];
responseStream.Read(contents, 0, (int)response.ContentLength);
return contents;
}
This is the image file I'm trying to download
And this is the image I am getting
These two images lead me to believe that I'm not waiting for the response to finish coming through, because the content-length is the same as the size of the image I'm expecting, but I'm not sure how to make my code wait for the entire response to come through or even really if that's the approach I need to take.
Here's my test code in case it's helpful
[TestMethod]
public void CanUploadAndDownloadFile()
{
var api = GetApi();
SkyDriveFolder folder = api.CreateFolder(null, "TestFolder", "Test Folder");
SkyDriveFile file = api.UploadFile(folder, TestImageFile, "TestImage.png");
file = api.DownloadFile(file);
api.DeleteFolder(folder);
byte[] contents = new byte[new FileInfo(TestImageFile).Length];
using (FileStream fstream = new FileStream(TestImageFile, FileMode.Open))
{
fstream.Read(contents, 0, contents.Length);
}
using (FileStream fstream = new FileStream(TestImageFile + "2", FileMode.CreateNew))
{
fstream.Write(file.Contents, 0, file.Contents.Length);
}
Assert.AreEqual(contents.Length, file.Contents.Length);
bool sameData = true;
for (int i = 0; i < contents.Length && sameData; i++)
{
sameData = contents[i] == file.Contents[i];
}
Assert.IsTrue(sameData);
}
It fails at Assert.IsTrue(sameData);

This is because you don't check the return value of responseStream.Read(contents, 0, (int)response.ContentLength);. Read doesn't ensure that it will read response.ContentLength bytes. Instead it returns the number of bytes read. You can use a loop or stream.CopyTo there.
Something like this:
WebResponse response = request.GetResponse();
MemoryStream m = new MemoryStream();
response.GetResponseStream().CopyTo(m);
byte[] contents = m.ToArray();

As LB already said, you need to continue to call Read() until you have read the entire stream.
Although Stream.CopyTo will copy the entire stream it does not ensure that read the number of bytes expected. The following method will solve this and raise an IOException if it does not read the length specified...
public static void Copy(Stream input, Stream output, long length)
{
byte[] bytes = new byte[65536];
long bytesRead = 0;
int len = 0;
while (0 != (len = input.Read(bytes, 0, Math.Min(bytes.Length, (int)Math.Min(int.MaxValue, length - bytesRead)))))
{
output.Write(bytes, 0, len);
bytesRead = bytesRead + len;
}
output.Flush();
if (bytesRead != length)
throw new IOException();
}

"500 error" in ASP.NET file-upload page?

I am calling this PHP code from Android to transfer file from Android to server and its working perfectly fine.
<?php
$target_path = "./";
$target_path = $target_path . basename( $_FILES['uploadedfile']['name']);
if(move_uploaded_file($_FILES['uploadedfile']['tmp_name'], $target_path)) {
echo "The file ". basename( $_FILES['uploadedfile']['name']).
" has been uploaded";
} else{
echo "There was an error uploading the file, please try again!";
}
?>
But my server application is in .net and I have to write this code in .net
I tried to write .net version code but it does not work and returns 500 internal server error.
public partial class uploadfiles : System.Web.UI.Page
{
protected void Page_Load(object sender, EventArgs e)
{
try
{
HttpFileCollection uploadFile = Request.Files;
if (uploadFile.Count > 0)
{
HttpPostedFile postedFile = uploadFile[0];
System.IO.Stream inStream = postedFile.InputStream;
byte[] fileData = new byte[postedFile.ContentLength];
inStream.Read(fileData, 0, postedFile.ContentLength);
postedFile.SaveAs(Server.MapPath("Data") + "\\" + postedFile.FileName);
}
}
catch (Exception ex)
{
StringBuilder sb = new StringBuilder();
sb.AppendLine("Message : " +ex.Message);
sb.AppendLine("Source : " + ex.Source);
sb.AppendLine("StackTrace : " + ex.StackTrace);
sb.AppendLine("InnerException : " + ex.InnerException);
sb.AppendLine("ToString : " + ex.ToString());
LogInToFile(sb.ToString());
}
}
}
It does not log any exception or even I think it does not reach to its first line. I checked it through Log file. as it does not work. please help.
Android code is below
HttpURLConnection connection = null;
DataOutputStream outputStream = null;
DataInputStream inputStream = null;
String pathToOurFile = "/data/file_to_send.mp3";
String urlServer = "http://192.168.1.1/handle_upload.php";
String lineEnd = "\r\n";
String twoHyphens = "--";
String boundary = "*****";
int bytesRead, bytesAvailable, bufferSize;
byte[] buffer;
int maxBufferSize = 1*1024*1024;
try
{
FileInputStream fileInputStream = new FileInputStream(new File(pathToOurFile) );
URL url = new URL(urlServer);
connection = (HttpURLConnection) url.openConnection();
// Allow Inputs & Outputs
connection.setDoInput(true);
connection.setDoOutput(true);
connection.setUseCaches(false);
// Enable POST method
connection.setRequestMethod("POST");
connection.setRequestProperty("Connection", "Keep-Alive");
connection.setRequestProperty("Content-Type", "multipart/form-data;boundary="+boundary);
outputStream = new DataOutputStream( connection.getOutputStream() );
outputStream.writeBytes(twoHyphens + boundary + lineEnd);
outputStream.writeBytes("Content-Disposition: form-data; name=\"uploadedfile\";filename=\"" + pathToOurFile +"\"" + lineEnd);
outputStream.writeBytes(lineEnd);
bytesAvailable = fileInputStream.available();
bufferSize = Math.min(bytesAvailable, maxBufferSize);
buffer = new byte[bufferSize];
// Read file
bytesRead = fileInputStream.read(buffer, 0, bufferSize);
while (bytesRead > 0)
{
outputStream.write(buffer, 0, bufferSize);
bytesAvailable = fileInputStream.available();
bufferSize = Math.min(bytesAvailable, maxBufferSize);
bytesRead = fileInputStream.read(buffer, 0, bufferSize);
}
outputStream.writeBytes(lineEnd);
outputStream.writeBytes(twoHyphens + boundary + twoHyphens + lineEnd);
// Responses from the server (code and message)
serverResponseCode = connection.getResponseCode();
serverResponseMessage = connection.getResponseMessage();
fileInputStream.close();
outputStream.flush();
outputStream.close();
}
catch (Exception ex)
{
//Exception handling
}

The default max file size limit in ASP.NET application is 4 MB. You can configure the application to accept bigger size by setting the below in web.config
<system.web>
<httpRuntime maxRequestLength="20480" />
</system.web>
To learn more about configuring the file size limit check out Large file uploads in ASP.NET

Try this:
postedFile.SaveAs(Server.MapPath("~/Data/" + postedFile.FileName ));
where ~/Data/ is the location where you want to save your file.
then + postedFile.FileName your filename.
Best Regards