Develop Reference

Transfer files directly from FTP to Azure File Storage without keeping them locally in memory or disk

I have to transfer files from FTP to an Azure File Storage. My code works fine, but I'm transferring those files in memory which is not a best practice. So first I read the stream to an Byte array in memory. Then I upload the output to an Azure file storage.
Now I know it's better to do this asynchronicaly. But I don't know if this is possible and how to do it.
My code:
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Microsoft.WindowsAzure.Storage;
using System.Configuration;
using Microsoft.WindowsAzure.Storage.File;
using System.IO;
using Microsoft.Azure;
using System.Net;
namespace TransferFtpToAzure
{
class Program
{
public static void Main(string[] args)
{
List<FileName> sourceFileList = new List<FileName>();
List<FileName> targetFileList = new List<FileName>();
string targetShareReference = ConfigurationManager.AppSettings["AzureShare"];
string targetDirectoryReference = ConfigurationManager.AppSettings["Environment"] + "/" + Enums.AzureFolders.Mos + "/" + Enums.AzureFolders.In;
string sourceURI = (ConfigurationManager.AppSettings["FtpConnectionString"] + ConfigurationManager.AppSettings["Environment"].ToUpper() +"/"+ Enums.FtpFolders.Mos + "/").Replace("\\","/");
string sourceUser = ConfigurationManager.AppSettings["FtpServerUserName"];
string sourcePass = ConfigurationManager.AppSettings["FtpServerPassword"];
getFileLists(sourceURI, sourceUser, sourcePass, sourceFileList, targetShareReference, targetDirectoryReference, targetFileList);
Console.WriteLine(sourceFileList.Count + " files found!");
CheckLists(sourceFileList, targetFileList);
targetFileList.Sort();
Console.WriteLine(sourceFileList.Count + " unique files on sourceURI" + Environment.NewLine + "Attempting to move them.");
foreach (var file in sourceFileList)
{
try
{
CopyFile(file.fName, sourceURI, sourceUser, sourcePass, targetShareReference, targetDirectoryReference);
}
catch
{
Console.WriteLine("There was move error with : " + file.fName);
}
}
}
public class FileName : IComparable<FileName>
{
public string fName { get; set; }
public int CompareTo(FileName other)
{
return fName.CompareTo(other.fName);
}
}
public static void CheckLists(List<FileName> sourceFileList, List<FileName> targetFileList)
{
for (int i = 0; i < sourceFileList.Count; i++)
{
if (targetFileList.BinarySearch(sourceFileList[i]) > 0)
{
sourceFileList.RemoveAt(i);
i--;
}
}
}
public static void getFileLists(string sourceURI, string sourceUser, string sourcePass, List<FileName> sourceFileList, string targetShareReference, string targetDirectoryReference, List<FileName> targetFileList)
{
string line = "";
/////////Source FileList
FtpWebRequest sourceRequest;
sourceRequest = (FtpWebRequest)WebRequest.Create(sourceURI);
sourceRequest.Credentials = new NetworkCredential(sourceUser, sourcePass);
sourceRequest.Method = WebRequestMethods.Ftp.ListDirectory;
sourceRequest.UseBinary = true;
sourceRequest.KeepAlive = false;
sourceRequest.Timeout = -1;
sourceRequest.UsePassive = true;
FtpWebResponse sourceRespone = (FtpWebResponse)sourceRequest.GetResponse();
//Creates a list(fileList) of the file names
using (Stream responseStream = sourceRespone.GetResponseStream())
{
using (StreamReader reader = new StreamReader(responseStream))
{
line = reader.ReadLine();
while (line != null)
{
var fileName = new FileName
{
fName = line
};
sourceFileList.Add(fileName);
line = reader.ReadLine();
}
}
}
/////////////Target FileList
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(CloudConfigurationManager.GetSetting("StorageConnectionString"));
CloudFileClient fileClient = storageAccount.CreateCloudFileClient();
//var test = fileClient.ListShares();
CloudFileShare fileShare = fileClient.GetShareReference(targetShareReference);
if (fileShare.Exists())
{
CloudFileDirectory rootDirectory = fileShare.GetRootDirectoryReference();
if (rootDirectory.Exists())
{
CloudFileDirectory customDirectory = rootDirectory.GetDirectoryReference(targetDirectoryReference);
if (customDirectory.Exists())
{
var fileCollection = customDirectory.ListFilesAndDirectories().OfType<CloudFile>();
foreach (var item in fileCollection)
{
var fileName = new FileName
{
fName = item.Name
};
targetFileList.Add(fileName);
}
}
}
}
}
public static void CopyFile(string fileName, string sourceURI, string sourceUser, string sourcePass, string targetShareReference, string targetDirectoryReference)
{
try
{
FtpWebRequest request = (FtpWebRequest)WebRequest.Create(sourceURI + fileName);
request.Method = WebRequestMethods.Ftp.DownloadFile;
request.Credentials = new NetworkCredential(sourceUser, sourcePass);
FtpWebResponse response = (FtpWebResponse)request.GetResponse();
Stream responseStream = response.GetResponseStream();
Upload(fileName, ToByteArray(responseStream), targetShareReference, targetDirectoryReference);
responseStream.Close();
}
catch
{
Console.WriteLine("There was an error with :" + fileName);
}
}
public static Byte[] ToByteArray(Stream stream)
{
MemoryStream ms = new MemoryStream();
byte[] chunk = new byte[4096];
int bytesRead;
while ((bytesRead = stream.Read(chunk, 0, chunk.Length)) > 0)
{
ms.Write(chunk, 0, bytesRead);
}
return ms.ToArray();
}
public static bool Upload(string FileName, byte[] Image, string targetShareReference, string targetDirectoryReference)
{
try
{
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(CloudConfigurationManager.GetSetting("StorageConnectionString"));
CloudFileClient fileClient = storageAccount.CreateCloudFileClient();
//var test = fileClient.ListShares();
CloudFileShare fileShare = fileClient.GetShareReference(targetShareReference);
if (fileShare.Exists())
{
CloudFileDirectory rootDirectory = fileShare.GetRootDirectoryReference();
if (rootDirectory.Exists())
{
CloudFileDirectory customDirectory = rootDirectory.GetDirectoryReference(targetDirectoryReference);
if (customDirectory.Exists())
{
var cloudFile = customDirectory.GetFileReference(FileName);
using (var stream = new MemoryStream(Image, writable: false))
{
cloudFile.UploadFromStream(stream);
}
}
}
}
return true;
}
catch
{
return false;
}
}
}
}

If I understand you correctly, you want to avoid storing the file in memory between the download and upload.
For that see:
Azure function to copy files from FTP to blob storage.

Using Azure Storage File Share this is the only way it worked for me without loading the entire ZIP into Memory. I tested with a 3GB ZIP File (with thousands of files or with a big file inside) and Memory/CPU was low and stable. I hope it helps!
var zipFiles = _directory.ListFilesAndDirectories()
.OfType<CloudFile>()
.Where(x => x.Name.ToLower().Contains(".zip"))
.ToList();
foreach (var zipFile in zipFiles)
{
using (var zipArchive = new ZipArchive(zipFile.OpenRead()))
{
foreach (var entry in zipArchive.Entries)
{
if (entry.Length > 0)
{
CloudFile extractedFile = _directory.GetFileReference(entry.Name);
using (var entryStream = entry.Open())
{
byte[] buffer = new byte[16 * 1024];
using (var ms = extractedFile.OpenWrite(entry.Length))
{
int read;
while ((read = entryStream.Read(buffer, 0, buffer.Length)) > 0)
{
ms.Write(buffer, 0, read);
}
}
}
}
}
}
}

String too small to store standard output of process

I direct the standard output/error of a process and store it in a string which I log to a file.
I have noticed that not all of the standard output is stored in the text file so I must be hitting the size limit of the string.
Does anyone know of an alternative which would help satisfy this issue?
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.IO;
namespace Logging
{
class Logger
{
private FileStream file = null;
private StreamWriter writer = null;
public Logger(string _path)
{
if (!File.Exists(_path)) File.Create(_path).Dispose();
file = new FileStream(_path, FileMode.Append, FileAccess.Write, FileShare.Write);
writer = new StreamWriter(file);
}
public void Log(string _text)
{
Console.WriteLine(_text);
using (file)
{
writer.WriteLine("[" + DateTime.Now + "] : " + _text);
}
}
}
}

using (file)
You just disposed your file. You can't write to it anymore.

The string type has a limit of 2GB.
Your handling of the file creation and disposal is causing you some difficulty.
Try this for a little cleaner logging code:
const string log = #"logfile path";
public void Log(string _text)
{
try
{
using (TextWriter tw = new StreamWriter(_path, true))
{
tw.WriteLine("[" + DateTime.Now + "] : " + _text);
}
}
catch (Exception)
{
throw;
}
}

Zip complete folder using System.IO.Compression

I have one example where all the files in the folder are zipped but not the folder itself .[This code is from MSDN]
using System;
using System.IO;
using System.IO.Compression;
namespace zip
{
public class Program
{
public static void Main()
{
string directoryPath = #"c ------------------------------------------------------------------------ :\users\public\reports";
DirectoryInfo directorySelected = new DirectoryInfo(directoryPath);
foreach (FileInfo fileToCompress in directorySelected.GetFiles())
{
Compress(fileToCompress);
}
foreach (FileInfo fileToDecompress in directorySelected.GetFiles("*.gz"))
{
Decompress(fileToDecompress);
}
}
public static void Compress(FileInfo fileToCompress)
{
using (FileStream originalFileStream = fileToCompress.OpenRead())
{
if ((File.GetAttributes(fileToCompress.FullName) & FileAttributes.Hidden) != FileAttributes.Hidden & fileToCompress.Extension != ".gz")
{
using
(FileStream compressedFileStream = File.Create(fileToCompress.FullName + ".gz"))
{
using (GZipStream compressionStream = new GZipStream(compressedFileStream, CompressionMode.Compress))
{
originalFileStream.CopyTo(compressionStream);
Console.WriteLine("Compressed {0} from {1} to {2} bytes.", fileToCompress.Name, fileToCompress.Length.ToString(), compressedFileStream.Length.ToString());
}
}
}
}
}
public static void Decompress(FileInfo fileToDecompress)
{
using (FileStream originalFileStream = fileToDecompress.OpenRead())
{
string currentFileName = fileToDecompress.FullName;
string newFileName = currentFileName.Remove(currentFileName.Length - fileToDecompress.Extension.Length);
using (FileStream decompressedFileStream = File.Create(newFileName))
{
using (GZipStream decompressionStream = new GZipStream(originalFileStream, CompressionMode.Decompress))
{
decompressionStream.CopyTo(decompressedFileStream);
Console.WriteLine("Decompressed: {0}", fileToDecompress.Name);
}
}
}
}
}
}

I don't think you can zip a complete folder using System.IO.Compression, you can only compress files inside the folder. You can use DotNetZip instead. It is a 100% managed code library that can be used in any .NET application - Console, Winforms, WPF, ASP.NET, Sharepoint, Web services apps, and so on.
Download developer's kit package from http://dotnetzip.codeplex.com/Release/ProjectReleases.aspx.
Reference necessaries including DotNetZip DLL in your application and do follows:
string[] MainDirs = Directory.GetDirectories(""c:\users\public\reports");
for (int i = 0; i < MainDirs.Length; i++)
{
using (ZipFile zip = new ZipFile())
{
zip.UseUnicodeAsNecessary = true;
zip.AddDirectory(MainDirs[i]);
zip.CompressionLevel = Ionic.Zlib.CompressionLevel.BestCompression;
zip.Comment = "This zip was created at " + System.DateTime.Now.ToString("G");
zip.Save(string.Format("test{0}.zip", i));
}
}
Hope this helps,
Thanks

Insertion data into text file exception

I have written a code that crates a text file called contact and then try to store data and read data from it.
I have no problem in storing data but how can I store new data without override the oldest one?
and when I am finishing the insertion process an exception occurred : that my text file can not be accessed because it is being used by another process.
what should I do ??
here is my code:
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.IO;
using System.Runtime.Serialization.Formatters.Binary;
using System.Runtime.Serialization;
namespace ConsoleApplication9
{
[Serializable]
class contact
{
string name;
string address;
string phonenumber;
string emailaddress;
public override string ToString()
{
return name + " " + address + " " + phonenumber + " " + emailaddress;
}
public void AddContent(string cname, string caddress, string cphone, string cemail)
{
name = cname;
address = caddress;
phonenumber = cphone;
emailaddress = cemail;
FileStream file = new FileStream("contact.txt", FileMode.OpenOrCreate, FileAccess.Write);
BinaryFormatter bin = new BinaryFormatter();
contact person = new contact();
person.name = cname;
person.address = caddress;
person.phonenumber = cphone;
person.emailaddress = cemail;
bin.Serialize(file, person);
file.Close();
Console.WriteLine(" added ");
}
public void viewContact
{
FileStream file = new FileStream("contact.txt", FileMode.Open, FileAccess.Read);
BinaryFormatter bin = new BinaryFormatter();
List<string> contact_list = new List<string>();
//while (file != null)
using (StreamReader read = new StreamReader("contact.txt"))
{
string temp = read.ReadToEnd();
contact_list.Add(temp);
} file.Close();
for (int i = 0; i < contact_list.Count; i++)
{
Console.WriteLine(contact_list[i]);
}
}
}//end of class

Use FileMode.Append to append to existing file

Creating a Part Uri When Path Contains a #

I am zipping a file using the code below. I've found that when a folder in the path being zipped contains a hash (#), CreatePartUri(uri) throws an exception:
Part URI cannot contain a Fragment component.
Since I cannot alter the folder name, how can I escape the # symbol in the path so that the Uri will be created correctly?
using System;
using System.IO;
using System.IO.Packaging;
namespace ConsoleApplication1
{
class Program
{
static void Main(string[] args)
{
string targetFilePath = "C:\\TEMP.ZIP";
try
{
if (File.Exists(targetFilePath))
{
File.Delete(targetFilePath);
}
string packageRelationshipType =
#"http://schemas.openxmlformats.org/" +
#"package/2007/relationships/htmx/root-html";
CompressionOption compressionOption = CompressionOption.Maximum;
using (Package package = Package.Open(targetFilePath,
FileMode.OpenOrCreate))
{
string fileName = #"\#TestFolder\TestFile.txt";
string filePathOnServer = #"C:\" + fileName;
Uri uri = new Uri(fileName, UriKind.Relative);
Uri partUriDocument = PackUriHelper.CreatePartUri(uri);
PackagePart packagePartDocument =
package.CreatePart(partUriDocument,
System.Net.Mime.MediaTypeNames.Text.RichText,
compressionOption);
using (FileStream fileStream = new FileStream
(filePathOnServer,
FileMode.Open,
FileAccess.Read))
{
CopyStream(fileStream, packagePartDocument.GetStream());
}
package.CreateRelationship(packagePartDocument.Uri,
TargetMode.Internal,
packageRelationshipType);
}
}
catch (Exception e)
{
string exceptionText = e.ToString();
}
}
private static void CopyStream(Stream source, Stream target)
{
const int bufSize = 0x1000;
byte[] buf = new byte[bufSize];
int bytesRead = 0;
while ((bytesRead = source.Read(buf, 0, bufSize)) > 0)
target.Write(buf, 0, bytesRead);
}
}
}

System.IO.Packaging does not allow "#" in the names

As Rockstart says, '#' is not allowed in the Uri
Use this right before you create the Uri to remove # characters:
fileName = Regex.Replace(fileName, "#", "");