I'm working on a split downloader for c#. It is downloading fine (so the logic is working) but the problem is that whatever file it downloads it corrupts. I have no idea on how to fix it. Here's the code:
private void mergeClean()
{
const int chunkSize = 1 * 1024; // 2KB
using (var output = File.Create("output.jpg"))
{
foreach (var file in Files)
{
using (var input = File.OpenRead(file))
{
var buffer = new byte[chunkSize];
int bytesRead;
while ((bytesRead = input.Read(buffer, 0, buffer.Length)) > 0)
{
output.Write(buffer, 0, bytesRead);
}
}
}
}
foreach (var file in Files)
{
File.Delete(file);
}
}
private void SaveFileStream(String path, Stream stream)
{
var fileStream = new FileStream(path, FileMode.Create, FileAccess.Write);
stream.CopyTo(fileStream);
fileStream.Dispose();
}
public void SplitDownload(string URL)
{
System.Net.WebRequest req = System.Net.HttpWebRequest.Create(URL);
req.Method = "HEAD";
System.Net.WebResponse resp = req.GetResponse();
var responseLength = double.Parse(resp.Headers.Get("Content-Length"));
var partSize = Math.Ceiling(responseLength / 10);
var previous = 0;
for (int i = (int)partSize; i <= responseLength; i = i + (int)partSize)
{
Thread t = new Thread(() => Download(URL, previous, i));
t.Start();
t.Join();
previous = i;
}
mergeClean();
}
private void Download(string URL, int Start, int End)
{
Console.WriteLine(String.Format("{0},{1}", Start, End));
HttpWebRequest myHttpWebRequest = (HttpWebRequest)WebRequest.Create(URL);
myHttpWebRequest.AddRange(Start, End);
HttpWebResponse myHttpWebResponse = (HttpWebResponse)myHttpWebRequest.GetResponse();
Stream streamResponse = myHttpWebResponse.GetResponseStream();
String name = GenerateTempName();
SaveFileStream(name, streamResponse);
Files.Add(name);
}
Here is an example of what it does:
UPDATED CODE:
static string GenerateTempName(int start)
{
String name = String.Format("{0:D6}.tmp", start);
return name;
}
static public List<string> Files = new List<string>();
static private void mergeClean()
{
Files.Sort();
const int chunkSize = 1 * 1024; // 2KB
using (var output = File.Create("output.jpg"))
{
foreach (var file in Files)
{
using (var input = File.OpenRead(file))
{
var buffer = new byte[chunkSize];
int bytesRead;
while ((bytesRead = input.Read(buffer, 0, buffer.Length)) > 0)
{
output.Write(buffer, 0, bytesRead);
}
}
}
}
foreach (var file in Files)
{
File.Delete(file);
}
}
You need to recombine file from pieces in correct order - current code create random file names and even if items are added to list of files they are added in random order due to unpredictable time when segment download finishes.
Possible fix: use block start offset as part of the file name String name = String.Format("file{0:D6}.tmp", Start) and sort files by name before combining them back.
Note that {0:D6} formatting is used to pad index with 0 to allow sorting by name to be easier and avoid need for natural sort code.
Related
I'm trying to use the following snippet in order download files via the SaveFileDialog in Silverlight:
public void SaveMediaLocal(string fileName)
{
FileInfo fInfo = new FileInfo(fileName);
if (fInfo.Exists)
{
if (fInfo.Length > 0)
{
string extension = fInfo.Extension;
SaveFileDialog dialog = new SaveFileDialog()
{
DefaultExt = extension,
Filter = String.Format("{1} files (*.{0})|*.{0}|All files (*.*)|*.*", extension, fInfo.Extension),
FilterIndex = 1,
DefaultFileName = fInfo.Name
};
if (dialog.ShowDialog() == true)
{
try
{
bool cancelFlag = false;
byte[] buffer = new byte[1024 * 1024]; // 1GB buffer
using (FileStream dest = (FileStream)dialog.OpenFile())
{
using (FileStream source = new FileStream(fInfo.FullName, FileMode.Open, FileAccess.Read))
{
long fileLength = source.Length;
long totalBytes = 0;
int currentBlockSize = 0;
while ((currentBlockSize = source.Read(buffer, 0, buffer.Length)) > 0)
{
totalBytes += currentBlockSize;
double percentage = (double)totalBytes * 100.0 / fileLength;
dest.Write(buffer, 0, currentBlockSize);
}
}
}
}
catch
{
}
}
}
else
{
//no results
}
}
}
When I use this snippet; Silverlight freezes until the download completes.
When I use this snippet instead, the UI is responsive, but doesn't work on bigger files.
using (Stream stream = dialog.OpenFile())
{
Byte[] bytes = File.ReadAllBytes(fileName);
stream.Write(bytes, 0, bytes.Length);
}
Is there something that I'm missing here?
Don't do the operation on the GUI thread. That is why it gets unresponsive. Either create a new thread or async process and do the operation in the background.
I have a function that creates a zip file a string array of files passed. The function does succeed in creating the zip file and the zip entry files inside it, but these zip entry files are empty. I've tried a couple of different methods - the function code below is the closest I've gotten to something working:
public static bool ZipFile(string[] arrFiles, string sZipToDirectory, string sZipFileName)
{
if (Directory.Exists(sZipToDirectory))
{
FileStream fNewZipFileStream;
ZipOutputStream zos;
try {
fNewZipFileStream = File.Create(sZipToDirectory + sZipFileName);
zos = new ZipOutputStream(fNewZipFileStream);
for (int i = 0; i < arrFiles.Length; i++) {
ZipEntry entry = new ZipEntry(arrFiles[i].Substring(arrFiles[i].LastIndexOf("/") + 1));
zos.PutNextEntry(entry);
FileStream fStream = File.OpenRead(arrFiles[i]);
BufferedStream bfStrm = new BufferedStream(fStream);
byte[] buffer = new byte[bfStrm.Length];
int count;
while ((count = bfStrm.Read(buffer, 0, 1024)) != -1) {
zos.Write(buffer);
}
bfStrm.Close();
fStream.Close();
zos.CloseEntry();
}
zos.Close();
fNewZipFileStream.Close();
return true;
}
catch (Exception ex)
{
string sErr = ex.Message;
return false;
}
finally
{
fNewZipFileStream = null;
zos = null;
}
}
else
{
return false;
}
}
I think it's got to do with the byte stream handling. I've tried this bit of code that handles the stream but it goes into an infinite loop:
while ((count = fStream.Read(buffer, 0, 1024)) != -1) {
zos.Write(buffer, 0, count);
}
fStream.Close();
I found a solution that is quite simple - I used the ReadAllBytes method of the static File class.
ZipEntry entry = new ZipEntry(arrFiles[i].Substring(arrFiles[i].LastIndexOf("/") + 1));
zos.PutNextEntry(entry);
byte[] fileContents = File.ReadAllBytes(arrFiles[i]);
zos.Write(fileContents);
zos.CloseEntry();
Using Read() on a FileStream returns the amount of bytes read into the stream or 0 if the end of the stream has been reached. It will never return a value of -1.
From MSDN:
The total number of bytes read into the buffer. This might be less than the number of bytes requested if that number of bytes are not currently available, orzero if the end of the stream is reached.
I'd modify your code to the following:
System.IO.FileStream fos = new System.IO.FileStream(sZipToDirectory + sZipFileName, FileMode.Create);
Java.Util.Zip.ZipOutputStream zos = new Java.Util.Zip.ZipOutputStream(fos);
byte[] buffer = new byte[1024];
for (int i = 0; i < arrFiles.Length; i++) {
FileInfo fi = new FileInfo (arrFiles[i]);
Java.IO.FileInputStream fis = new Java.IO.FileInputStream(fi.FullName);
ZipEntry entry = new ZipEntry(arrFiles[i].Substring(arrFiles[i].LastIndexOf("/") + 1));
zos.PutNextEntry(entry);
int count = 0;
while ((count = fis.Read(buffer)) > 0) {
zos.Write(buffer, 0, count);
}
fis.Close();
zos.CloseEntry();
}
This is nearly identical to the code I've used for creating zip archives on Android in the past.
Are you allowed to use SharpZip? It's really easy to use.
Here is a blog post I wrote to extract zip files
private static void upzip(string url)
{
WebClient wc = new WebClient();
wc.DownloadFile(url, "temp.zip");
//unzip
ZipFile zf = null;
try
{
zf = new ZipFile(File.OpenRead("temp.zip"));
foreach (ZipEntry zipEntry in zf)
{
string fileName = zipEntry.Name;
byte[] buffer = new byte[4096];
Stream zipStream = zf.GetInputStream(zipEntry);
using (FileStream streamWriter = File.Create( fileName))
{
StreamUtils.Copy(zipStream, streamWriter, buffer);
}
}
}
finally
{
if (zf != null)
{
zf.IsStreamOwner = true;
zf.Close();
}
}
}
private void ZipFolder(string[] _files, string zipFileName)
{
using var memoryStream = new MemoryStream();
using (var archive = new ZipArchive(memoryStream, ZipArchiveMode.Create, true))
{
foreach (var item in _files)
{
var demoFile = archive.CreateEntry(Path.GetFileName(item));
using var readStreamW = File.OpenRead(item);
using (var entryStream = demoFile.Open())
{
using (var streamWriter = new StreamWriter(entryStream))
{
readStreamW.Seek(0, SeekOrigin.Begin);
readStreamW.CopyTo(streamWriter.BaseStream);
}
}
}
}
using var fileStream = new FileStream(zipFileName, FileMode.Create);
memoryStream.Seek(0, SeekOrigin.Begin);
memoryStream.CopyTo(fileStream);
}
I want to create a TcpClient which automatically gets multiple files from server by their name.
I want to get some ideas how I can build such application.
My idea is:
Make a for loop which contains SwitchCase, where I specify my files names. I really don't know if this will work well.
To go out of for loop I can compare the index operator to numbers of files. If they are equal then I go out of for loop.
Example of my idea:
for (int i = 1; i <= 4; i++)
{
switch (----)
{
case 'file1':
code...
break;
case 'file2':
code...
case 'file3':
code...
break;
case 'file4':
code...
break;
default:
code...
break;
}
}
To download a file using ftp you could use the FtpWebRequest and for http use the HttpWebRequest.
Below is a simple example of how to request a file using http (the method is similar for ftp):
public void Download(string url, string localPath)
{
HttpWebRequest request = HttpWebRequest.Create(url);
HttpWebResponse response = request.GetResponse() as HttpWebResponse;
Stream stream = response.GetResponseStream();
FileStream fs = new FileStream(localPath, FileMode.Create);
int count;
byte[] buffer = new byte[8096];
while ((count = stream.Read(buffer, 0, 8096)) > 0)
fs.Write(buffer, 0, count);
fs.Dispose();
response.Close();
}
Instead of using a switch inside a for loop you should iterate an array:
string[] files = new string[]{ url1, url2, ...};
for(int i = 0; i < files.Length; i++)
{
Download(files[i], "file" + i);
}
I solved it like so:
MY app. gets 2 files from server and move files and rename them.
test = mytest
test111 = test2
static string myfile1 = #"C:\inbox\mytest.txt";
static string myfile2 = #"C:\inbox\test2.txt";
//files from server
static string myServerfile = #"C:\Users\me\Documents\file_client\bin\Debug\test.csv";
static string myServerfile1 = #"C:\Users\RH-T3\Documents\file_client\bin\Debug\test111.txt";
public static void Main(string[] args)
{
try
{
for (int i = 0; i < 2; i++)
{
if (i == 0)
{
Console.WriteLine("Downloading test.csv");
string fileName = "test.csv";
Console.WriteLine("Client starts...");
//args[0] = Console.ReadLine();
file_client client = new file_client(args);
Console.WriteLine("efter file_client...");
NetworkStream serverStream = client.clientSocket.GetStream();
LIB.writeTextTCP(serverStream, fileName);
long rest = long.Parse(LIB.readTextTCP(serverStream));
byte[] inStream = new byte[rest];
while (rest != 0)
{
rest = rest - serverStream.Read(inStream, 0, inStream.Length);
Console.WriteLine("REST: " + rest);
}
FileStream fs = new FileStream(fileName, FileMode.Create);
fs.Write(inStream, 0, inStream.Length);
{
fs.Close();
serverStream.Close();
}
if (File.Exists(myfile1))
{
File.Delete(myfile1);
}
File.Move(myServerfile, myfile1);
Console.WriteLine("Moved");
System.Threading.Thread.Sleep(500);
}
else
{
Console.WriteLine("Downloading .txt file");
string fileName = "test111.txt";
Console.WriteLine("Client starts...");
//args[0] = Console.ReadLine();
file_client client = new file_client(args);
Console.WriteLine("efter file_client...");
NetworkStream serverStream = client.clientSocket.GetStream();
LIB.writeTextTCP(serverStream, fileName);
long rest = long.Parse(LIB.readTextTCP(serverStream));
byte[] inStream = new byte[rest];
while (rest != 0)
{
rest = rest - serverStream.Read(inStream, 0, inStream.Length);
Console.WriteLine("REST: " + rest);
}
FileStream fs = new FileStream(fileName, FileMode.Create);
fs.Write(inStream, 0, inStream.Length);
{
fs.Close();
serverStream.Close();
}
if (File.Exists(myfile2))
{
File.Delete(myfile2);
}
File.Move(myServerfile1, myfile2);
Console.WriteLine("Moved");
System.Threading.Thread.Sleep(500);
}
}
}
catch (Exception ex)
{
Console.WriteLine(ex.ToString());
Console.WriteLine("Cannot be DONE!");
}
As part of an upcoming project at my university, I need to write a client that downloads a media file from a server and writes it to the local disk. Since these files can be very large, I need to implement partial download and serialization in order to avoid excessive memory use.
What I came up with:
namespace PartialDownloadTester
{
using System;
using System.Diagnostics.Contracts;
using System.IO;
using System.Net;
using System.Text;
public class DownloadClient
{
public static void Main(string[] args)
{
var dlc = new DownloadClient(args[0], args[1], args[2]);
dlc.DownloadAndSaveToDisk();
Console.ReadLine();
}
private WebRequest request;
// directory of file
private string dir;
// full file identifier
private string filePath;
public DownloadClient(string uri, string fileName, string fileType)
{
this.request = WebRequest.Create(uri);
this.request.Method = "GET";
var sb = new StringBuilder();
sb.Append("C:\\testdata\\DownloadedData\\");
this.dir = sb.ToString();
sb.Append(fileName + "." + fileType);
this.filePath = sb.ToString();
}
public void DownloadAndSaveToDisk()
{
// make sure directory exists
this.CreateDir();
var response = (HttpWebResponse)request.GetResponse();
Console.WriteLine("Content length: " + response.ContentLength);
var rStream = response.GetResponseStream();
int bytesRead = -1;
do
{
var buf = new byte[2048];
bytesRead = rStream.Read(buf, 0, buf.Length);
rStream.Flush();
this.SerializeFileChunk(buf);
}
while (bytesRead != 0);
}
private void CreateDir()
{
if (!Directory.Exists(dir))
{
Directory.CreateDirectory(dir);
}
}
private void SerializeFileChunk(byte[] bytes)
{
Contract.Requires(!Object.ReferenceEquals(bytes, null));
FileStream fs = File.Open(filePath, FileMode.Append);
fs.Write(bytes, 0, bytes.Length);
fs.Flush();
fs.Close();
}
}
}
For testing purposes, I've used the following parameters:
"http://itu.dk/people/janv/mufc_abc.jpg" "mufc_abc" "jpg"
However, the picture is incomplete (only the first ~10% look right) even though the content length prints 63780 which is the actual size of the image.
So my questions are:
Is this the right way to go for partial download and serialization or is there a better/easier approach?
Is the full content of the response stream stored in client memory? If this is the case, do I need to use HttpWebRequest.AddRange to partially download data from the server in order to conserve my client's memory?
How come the serialization fails and I get a broken image?
Do I introduce a lot of overhead when I use the FileMode.Append? (msdn states that this option "seeks to the end of the file")
Thanks in advance
You could definitely simplify your code using a WebClient:
class Program
{
static void Main()
{
DownloadClient("http://itu.dk/people/janv/mufc_abc.jpg", "mufc_abc.jpg");
}
public static void DownloadClient(string uri, string fileName)
{
using (var client = new WebClient())
{
using (var stream = client.OpenRead(uri))
{
// work with chunks of 2KB => adjust if necessary
const int chunkSize = 2048;
var buffer = new byte[chunkSize];
using (var output = File.OpenWrite(fileName))
{
int bytesRead;
while ((bytesRead = stream.Read(buffer, 0, buffer.Length)) > 0)
{
output.Write(buffer, 0, bytesRead);
}
}
}
}
}
}
Notice how I am writing only the number of bytes I have actually read from the socket to the output file and not the entire 2KB buffer.
I don't know if this is the source of the problem, however I would change the loop like this
const int ChunkSize = 2048;
var buf = new byte[ChunkSize];
var rStream = response.GetResponseStream();
do {
int bytesRead = rStream.Read(buf, 0, ChunkSize);
if (bytesRead > 0) {
this.SerializeFileChunk(buf, bytesRead);
}
} while (bytesRead == ChunkSize);
The serialize method would get an additional argument
private void SerializeFileChunk(byte[] bytes, int numBytes)
and then write the right number of bytes
fs.Write(bytes, 0, numBytes);
UPDATE:
I do not see the need for closing and reopening the file each time. I also would use the using statement, which closes the resources, even if an exception should occur. The using statement calls the Dispose() method of the resource at the end, which in turn calls Close() in the case of file streams. using can be applied to all types implementing IDisposable.
var buf = new byte[2048];
using (var rStream = response.GetResponseStream()) {
using (FileStream fs = File.Open(filePath, FileMode.Append)) {
do {
bytesRead = rStream.Read(buf, 0, buf.Length);
fs.Write(bytes, 0, bytesRead);
} while (...);
}
}
The using statement does something like this
{
var rStream = response.GetResponseStream();
try
{
// do some work with rStream here.
} finally {
if (rStream != null) {
rStream.Dispose();
}
}
}
Here is the solution from Microsoft: http://support.microsoft.com/kb/812406
Updated 2021-03-16: seems the original article is not available now. Here is the archived one: https://mskb.pkisolutions.com/kb/812406
I'm looking for a good way to get filesize from the Microsoft.SharePoint.Client.File object.
The Client object does not have a Length member.
I tried this:
foreach (SP.File file in files)
{
string path = file.Path;
path = path.Substring(this.getTeamSiteUrl().Length);
FileInformation fileInformation = SP.File.OpenBinaryDirect(this.Context, path);
using (MemoryStream memoryStream = new MemoryStream())
{
CopyStream(fileInformation.Stream, memoryStream);
file.Size = memoryStream.Length;
}
}
Which gave me a length through using the MemoryStream, but it's not good for performance. This file also does not belong to a document library. Since it's an attached file, I can't convert it to a ListItem object using ListItemAllFields. If I could convert it to a ListItem, I could get its size using: ListItem["File_x0020_Size"]
How do I get the filesize of the Client object in SharePoint using C#?
Load the File_x0020_Size field information to get it.
This is what I do when I want to list all the files in a Sharepoint 2010 folder:
//folderPath is something like /yoursite/yourlist/yourfolder
Microsoft.SharePoint.Client.Folder spFolder = _ctx.Web.GetFolderByServerRelativeUrl(folderPath);
_ctx.Load(spFolder);
_ctx.ExecuteQuery();
FileCollection fileCol = spFolder.Files;
_ctx.Load(fileCol);
_ctx.ExecuteQuery();
foreach (Microsoft.SharePoint.Client.File spFile in fileCol)
{
//In here, specify all the fields you want retrieved, including the file size one...
_ctx.Load(spFile, file => file.Author, file => file.TimeLastModified, file=>file.TimeCreated,
file => file.Name, file => file.ServerRelativeUrl, file => file.ListItemAllFields["File_x0020_Size"]);
_ctx.ExecuteQuery();
int fileSize = int.Parse((string)spFile.ListItemAllFields["File_x0020_Size"]);
}
_ctx is obviously the ClientContext you have initiated.
Here's an extended list of all Sharepoint internal fields
Can't you just use the Stream property's length?
file.Size = fileInformation.Stream.Length;
I don't know if this question was ever solved, but for the people who are looking for an answer (like I did) :
... CODE FOR GETTING THE SP.FILE ...
SP.FileInformation fileInfo = SP.File.OpenBinaryDirect(ctx, mySPFile.ServerRelativeUrl);
byte[] bodyString = ReadToEnd(fileInfo.Stream);
int length = bodyString.Length;
Console.Write(length.ToString());
... DO THE OTHER STUFF YOU NEED TO DO ....
public static byte[] ReadToEnd(System.IO.Stream stream)
{
long originalPosition = 0;
if (stream.CanSeek)
{
originalPosition = stream.Position;
stream.Position = 0;
}
try
{
byte[] readBuffer = new byte[4096];
int totalBytesRead = 0;
int bytesRead;
while ((bytesRead = stream.Read(readBuffer, totalBytesRead, readBuffer.Length - totalBytesRead)) > 0)
{
totalBytesRead += bytesRead;
if (totalBytesRead == readBuffer.Length)
{
int nextByte = stream.ReadByte();
if (nextByte != -1)
{
byte[] temp = new byte[readBuffer.Length * 2];
Buffer.BlockCopy(readBuffer, 0, temp, 0, readBuffer.Length);
Buffer.SetByte(temp, totalBytesRead, (byte)nextByte);
readBuffer = temp;
totalBytesRead++;
}
}
}
byte[] buffer = readBuffer;
if (readBuffer.Length != totalBytesRead)
{
buffer = new byte[totalBytesRead];
Buffer.BlockCopy(readBuffer, 0, buffer, 0, totalBytesRead);
}
return buffer;
}
finally
{
if (stream.CanSeek)
{
stream.Position = originalPosition;
}
}
}
Hope this will help other people, because I couldn't find a direct answer on the internet!