Develop Reference

Download GitHub files in directories

I have this function in my script:
private void DownloadFile(string file, string location)
{
var githubToken = "[token]";
var request = (HttpWebRequest)WebRequest.Create("https://api.github.com/repos/[owner]/[repo]/contents/" + file);
request.Headers.Add(HttpRequestHeader.Authorization, string.Concat("token ", githubToken));
request.Accept = "application/vnd.github.v3.raw";
request.UserAgent = "useragent";
try
{
using (var response = request.GetResponse())
{
var encoding = System.Text.ASCIIEncoding.UTF8;
using (var reader = new StreamReader(response.GetResponseStream(), encoding))
{
/*using (StreamWriter sw = File.CreateText(location))
{
sw.WriteLine(reader.ReadToEnd());
sw.Close();
}*/
using (FileStream fs = File.Create(location))
{
byte[] info = new UTF8Encoding(true).GetBytes(reader.ReadToEnd());
fs.Write(info, 0, info.Length);
}
reader.Close();
}
}
}
catch (Exception ex)
{
MessageBox.Show("ERROR: " + ex + "\n " + file);
}
}
So downloading a single file in the root directory like repo/text.txt, this works great! But when I want to download the file in files/file1.rar, that doesn't work. It gives me an 403 or 404. Can somebody explain why this is, and how to download files from a directory (the repository is private, therefore the large chunk of code)

recursive method to get ftp directoryList

I try to "create" a recursive method, which copy the ftp directory list to a treeView.
I allready tried to do that, but it is more quick & dirty as clean & simple.
Here you can see my Code-Snippets:
public void connectToServer(string pServerIP, string pServerPort, string pUsername, string pPassword)
{
_serverIP = pServerIP;
_serverPort = pServerPort;
_username = pUsername;
_password = pPassword;
string ftpServerPath = "ftp://" + pServerIP + ":" + pServerPort + "/";
try
{
FtpWebRequest request = (FtpWebRequest)WebRequest.Create(ftpServerPath);
request.Method = WebRequestMethods.Ftp.ListDirectory;
request.Credentials = new NetworkCredential(pUsername, pPassword);
FtpWebResponse response = (FtpWebResponse)request.GetResponse();
Stream responseStream = response.GetResponseStream();
StreamReader reader = new StreamReader(responseStream);
secondLevelDirectotyList = new List<string>();
int i = 0;
TreeNode rootTreeNode = tVDirectories.Nodes.Add("/");
Console.WriteLine("/\n");
while (!reader.EndOfStream)
{
secondLevelDirectotyList.Add(reader.ReadLine());
Console.WriteLine("...: " + secondLevelDirectotyList[i]);
i++;
}
reader.Close();
response.Close();
getFTPDirectoryList(secondLevelDirectotyList, 0);
}
catch (WebException ex)
{
MessageBox.Show("The following Exceptions occurs:\n" + ex.Message, "Exception occurs", MessageBoxButtons.OK, MessageBoxIcon.Error);
}
}
private void getFTPDirectoryList(List<string> pTopLevelDirectoryList, int pDirectoryListIndexer)//string pFTPPath)
{
//List<string>
string ftpServerPath = "ftp://" + _serverIP + ":" + _serverPort + "/" + pTopLevelDirectoryList[pDirectoryListIndexer];//pFTPPath;
try
{
FtpWebRequest request = (FtpWebRequest)WebRequest.Create(ftpServerPath);
request.Method = WebRequestMethods.Ftp.ListDirectory;
request.Credentials = new NetworkCredential(_username, _password);
FtpWebResponse response = (FtpWebResponse)request.GetResponse();
Stream responseStream = response.GetResponseStream();
StreamReader reader = new StreamReader(responseStream);
thirdLevelDirectoryList = new List<string>();
int i = 0;
TreeNode ftpServerDirectory = tVDirectories.Nodes[0].Nodes.Add(pTopLevelDirectoryList[pDirectoryListIndexer]);//pFTPPath);
while (!reader.EndOfStream)
{
string streamFTPPath = reader.ReadLine(); //Ließt die Zeile des Streams aus
thirdLevelDirectoryList.Add(streamFTPPath); //Fügt den gesamten Pfad in die String-Liste
Console.WriteLine("...........: " + thirdLevelDirectoryList[i]);
string newTreeNode = streamFTPPath.Substring(streamFTPPath.IndexOf(#"/") + 1);
ftpServerDirectory.Nodes.Add(newTreeNode); //Fügt nur den Unterordner- oder Unterdatei-Namen in die Ansicht ein
i++;
}
reader.Close();
response.Close();
//rekursiv
pDirectoryListIndexer++;
try
{
getFTPDirectoryList(pTopLevelDirectoryList, pDirectoryListIndexer);
}
catch (ArgumentOutOfRangeException ex)
{
// start next level Directory List
//pDirectoryListIndexer = 0;
//getFTPDirectoryList(thirdLevelDirectoryList, 0);
}
}
catch (WebException ex)
{
MessageBox.Show("The following Exceptions occurs:\n" + ex.Message, "Exception occurs", MessageBoxButtons.OK, MessageBoxIcon.Error);
}
}
As you can see, I don't get the point to browse through the ftp folders.
You also may noticed that I work with ftp-protocol the early first time.
For example: I don't know whether I should open a request every time the ftp-ListDirectory-Command is send.
I want something like this:
root/
...folder1
......file1infolder1
...file2
...folder2
......file3infolder2
I hope you can understand me :D
and I am also sry for my bad english.

This is my solution for a recursive method, which list all files and folders from a ftp-path,
BUT
it is unefficient and it is impossible to use!!!!
If you only have five folders, this method will work, but if you have more than ~five folders the method won't end - sure it will be finished SOME DAY..
So for everyone, who read this:
Think about your Idea to use a recursive method to list the ftp-directory!
You better should send the NLIST-ftp-command once the user "opens" a folder.
private void FtpNlistRecursive(string pPath)
{
try
{
DirectoryListOfCurrent = new List<string>();
_ftpServerFullPath = "ftp://" + _serverIP + ":" + _serverPort + "/" + pPath;
string newItem = "";
FtpWebRequest request = (FtpWebRequest)WebRequest.Create(_ftpServerFullPath);
request.Method = WebRequestMethods.Ftp.ListDirectory;
request.Credentials = new NetworkCredential(_username, _password);
FtpWebResponse response = (FtpWebResponse)request.GetResponse();
Stream responseStream = response.GetResponseStream();
StreamReader reader = new StreamReader(responseStream);
while (!reader.EndOfStream)
{
newItem = reader.ReadLine();
string shortItem = pPath.Substring(pPath.IndexOf(#"/") + 1); // Aus "Ornder1/Datei1.txt" wird "Datei1.txt"
if (!shortItem.Equals(newItem))
{
try
{
if (pPath.Equals("/"))
{
DirectoryListOfCurrent.Add(newItem);
directoryListOfAll.Add(newItem);
}
else
{
string completePath = pPath + newItem.Substring(newItem.IndexOf(#"/"));
DirectoryListOfCurrent.Add(completePath);
directoryListOfAll.Add(completePath);
}
}
catch (ArgumentOutOfRangeException ex)
{
//bei ZB "Datei3.txt" gibt es kein "/", somit einfach ignorieren
}
}
}
reader.Close();
response.Close();
foreach (string item in DirectoryListOfCurrent)
{
FtpNlistRecursive(item);
}
}
catch (Exception ex)
{
ExceptionOccurs(ex);
}
}

Upload Multiple files to FTP in c#

i'm using the below method to upload files from local server to FTP server, here i'm creating a new connection and initiating a new session each and every file uploading and closing the same. how to achieve this in single initiated session in c#.
this is my code
public bool UploadTempFilesToFTP()
{
string[] fileList;
try
{
ConfiguredValues conObj = new ConfiguredValues();
conObj.PickTheValuesFromConfigFile();
fileList = Directory.GetFiles(conObj.tempPath);
foreach (string FileName in fileList)
{
FtpWebRequest upldrequest = (FtpWebRequest)FtpWebRequest.Create(conObj.tempOutboundURL + FileName);
upldrequest.UseBinary = true;
upldrequest.KeepAlive = false;
upldrequest.Timeout = -1;
upldrequest.UsePassive = true;
upldrequest.Credentials = new NetworkCredential(conObj.user, conObj.pass);
upldrequest.Method = WebRequestMethods.Ftp.UploadFile;
string destinationAddress = conObj.tempPath;
FileStream fs = File.OpenRead(destinationAddress + FileName);
byte[] buffer = new byte[fs.Length];
fs.Read(buffer, 0, buffer.Length);
fs.Close();
Stream requestStr = upldrequest.GetRequestStream();
requestStr.Write(buffer, 0, buffer.Length);
requestStr.Close();
requestStr.Flush();
FtpWebResponse response = (FtpWebResponse)upldrequest.GetResponse();
response.Close();
File.Delete(destinationAddress + FileName);
}
Console.WriteLine("Uploaded Successfully to Temp folder");
return true;
}
catch (Exception ex)
{
Console.WriteLine("Upload failed. {0}", ex.Message);
return false;
}
}

it's weird that i answer an old question but i try almost everything to upload multiple files to ftp with no luck while the solution is very simple and effective, using LOOPING - foreach solved the issue for me i use the below function to Upload the files in one simple step..
public void Uploadbulkftpfiles(string[] list)
{
bool ife;// is folder exists
try
{
FtpWebRequest request = (FtpWebRequest)WebRequest.Create("ftp://ftpsite.com/folder");
request.Credentials = new NetworkCredential("Username", "Password");
request.Method = WebRequestMethods.Ftp.ListDirectory;
FtpWebResponse response = (FtpWebResponse)request.GetResponse();
ife = true;
}
catch (Exception)
{
ife = false;
}
/////////////////////////////////////////////begin of upload process
if (ife)//the folder is already exists
{
foreach (var str in list)
{
try
{
FtpWebRequest requestUP2 = (FtpWebRequest)WebRequest.Create("ftp://ftpsite.com/folder" + str);
requestUP2.Credentials = new NetworkCredential("UserName", "Password");
requestUP2.Method = WebRequestMethods.Ftp.UploadFile;
requestUP2.KeepAlive = false;
requestUP2.UsePassive = true;
using (Stream fileStream = File.OpenRead("ftp://ftpsite.com/folder" + str))
using (Stream ftpStream = requestUP2.GetRequestStream())
{
fileStream.CopyTo(ftpStream);
}
}
catch (Exception ex1)
{
MessageBox.Show(ex1.Message);
}
}
}
else if (!ife)
{
//CREATE THE FOLDER
try
{
FtpWebRequest request = (FtpWebRequest)WebRequest.Create("ftp:ftpsite/folder");
request.Credentials = new NetworkCredential("UserName", "Password");
request.Method = WebRequestMethods.Ftp.MakeDirectory;
FtpWebResponse response = (FtpWebResponse)request.GetResponse();
}
catch (Exception excr) { MessageBox.Show(excr.Message); }
//UPLOAD THE FILES
foreach (var str in list)
{
try
{
FtpWebRequest requestUP2 = (FtpWebRequest)WebRequest.Create("ftp://ftpsite.com/folder" + str);
requestUP2.Credentials = new NetworkCredential("UserName", "Password");
requestUP2.Method = WebRequestMethods.Ftp.UploadFile;
requestUP2.KeepAlive = false;
requestUP2.UsePassive = true;
using (Stream fileStream = File.OpenRead("ftp://ftpsite.com/folder" + str))
using (Stream ftpStream = requestUP2.GetRequestStream())
{
fileStream.CopyTo(ftpStream);
}
}
catch (Exception ex1)
{
MessageBox.Show(ex1.Message);
}
}
}
}

The ftp protocol is intended to works on request basis.
You start a request with a method (in your case UploadFile).
The only thing you can do is to KeepAlive your request to avoid connection closing
upldrequest.KeepAlive = true;
on every request you create except the last one. This will make a login only the first FTPWebRequest.
Then when you create the last FTPWebRequest, set
upldrequest.KeepAlive = false;
and it will close the connection when done.

Retry mechanism for downloading files

I have created a function that find files on a FTP folder.
public static List<string> GetFileList(NetworkCredential credential, string FTPSite, string FTPFolder, string extension)
{
string LSOutput = "";
List<string> files = new List<string>();
//Fetch LS
FtpWebRequest request = (FtpWebRequest)WebRequest.Create(#"ftp://" + FTPSite + FTPFolder);
request.Credentials = credential;
request.UseBinary = true;
request.EnableSsl = true;
request.Method = WebRequestMethods.Ftp.ListDirectory;
FtpWebResponse response = null;
response = (FtpWebResponse)request.GetResponse();
Stream responseStream = response.GetResponseStream();
StreamReader reader = new StreamReader(responseStream);
LSOutput = reader.ReadToEnd();
reader.Close();
response.Close();
PrepareLog("LS output while finding files:");
PrepareLog(LSOutput);
//Parse the LS
string[] LSOutputLines = LSOutput.Trim().Split(new string[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries);
foreach (string LSOutputLine in LSOutputLines)
files.Add(LSOutputLine);
//Filter files
files = files.Where(f => f.ToLower().EndsWith(extension.ToLower())).ToList();
PrepareLogAndEmail("Total " + extension.ToLower() + " files found: " + files.Count, LogMessageType.Simple);
return files;
}
Exception handling is done at the place where the function is being called.
Now the problem is that sometimes the FTP faces some issues(like timeout) and then we have to rerun the program.
I have been asked to add 3 times retry mechanism and here is what I did to retry it once.
public static List<string> GetFileList(NetworkCredential credential, string FTPSite, string FTPFolder, string extension)
{
string LSOutput = "";
List<string> files = new List<string>();
try
{
//Fetch LS
FtpWebRequest request = (FtpWebRequest)WebRequest.Create(#"ftp://" + FTPSite + FTPFolder);
request.Credentials = credential;
request.UseBinary = true;
request.EnableSsl = true;
request.Method = WebRequestMethods.Ftp.ListDirectory;
FtpWebResponse response = null;
response = (FtpWebResponse)request.GetResponse();
Stream responseStream = response.GetResponseStream();
StreamReader reader = new StreamReader(responseStream);
LSOutput = reader.ReadToEnd();
reader.Close();
response.Close();
PrepareLog("LS output while finding files:");
PrepareLog(LSOutput);
}
catch (Exception ex)
{
PrepareLogAndEmail("First attempt failed: Details: " + ex.Message, LogMessageType.AttemptFailed);
PrepareLogAndEmail("Second attempt started" + ex.Message, LogMessageType.Simple);
//Fetch LS
FtpWebRequest request = (FtpWebRequest)WebRequest.Create(#"ftp://" + FTPSite + FTPFolder);
request.Credentials = credential;
request.UseBinary = true;
request.EnableSsl = true;
request.Method = WebRequestMethods.Ftp.ListDirectory;
FtpWebResponse response = null;
response = (FtpWebResponse)request.GetResponse();
Stream responseStream = response.GetResponseStream();
StreamReader reader = new StreamReader(responseStream);
LSOutput = reader.ReadToEnd();
reader.Close();
response.Close();
PrepareLog("LS output while finding files(second attempt):");
PrepareLog(LSOutput);
}
//Parse the LS
string[] LSOutputLines = LSOutput.Trim().Split(new string[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries);
foreach (string LSOutputLine in LSOutputLines)
files.Add(LSOutputLine);
//Filter files
files = files.Where(f => f.ToLower().EndsWith(extension.ToLower())).ToList();
PrepareLogAndEmail("Total " + extension.ToLower() + " files found: " + files.Count, LogMessageType.Simple);
return files;
}
Now if I have to add 2 more tries, will I need to do this sort of daiys chaining or is there any other cleaner option available ?

I would go with a retry loop, something like below
You will notice that I make use of a bolean to flag out of a successfull download, or the retries count to terminate.
I would even go as far as changing the last section to check if the download was successfull, and notify the user if it was not, Maybe even logging the error messages.
public static List<string> GetFileList(NetworkCredential credential, string FTPSite, string FTPFolder, string extension)
{
List<string> files = new List<string>();
int retries = 0;
int maxRetries = 5;
bool downloaded = false;
string LSOutput = "";
while (!downloaded && retries < maxRetries)
{
try
{
LSOutput = "";
//Fetch LS
FtpWebRequest request = (FtpWebRequest) WebRequest.Create(#"ftp://" + FTPSite + FTPFolder);
request.Credentials = credential;
request.UseBinary = true;
request.EnableSsl = true;
request.Method = WebRequestMethods.Ftp.ListDirectory;
FtpWebResponse response = null;
response = (FtpWebResponse) request.GetResponse();
Stream responseStream = response.GetResponseStream();
StreamReader reader = new StreamReader(responseStream);
LSOutput = reader.ReadToEnd();
reader.Close();
response.Close();
PrepareLog("LS output while finding files:");
PrepareLog(LSOutput);
downloaded = true;
}
catch (Exception ex)
{
retries++;
}
}
if (downloaded)
{
//Parse the LS
string[] LSOutputLines = LSOutput.Trim().Split(new string[] {Environment.NewLine}, StringSplitOptions.RemoveEmptyEntries);
foreach (string LSOutputLine in LSOutputLines)
files.Add(LSOutputLine);
//Filter files
files = files.Where(f => f.ToLower().EndsWith(extension.ToLower())).ToList();
PrepareLogAndEmail("Total " + extension.ToLower() + " files found: " + files.Count, LogMessageType.Simple);
return files;
}
else
{
PrepareLogAndEmail("Failed to download file", LogMessageType.Simple);
return null;
}
}

How do I pull the file locations (path and filename) for all files at a given FTP address?

Hi and thanks for looking!
Background
I need to pull the file locations (path and filename) for all files at a given FTP address.
For files on a mapped network or local drive, this code would work:
foreach(string fileName f in Directory.GetFiles("C\\:SomeDirectory"), "*.*",
SearchOption.AllDirectories)
{
//do stuff with each file found
}
But this will NOT work over an FTP connection. I have already found this MS documentation which covers the establishing of an FTPWebRequest, but it does not show me how to loop through each file found (in all nested directories as well).
I am using C# within a forms app.
Question
How do I accomplish this:
foreach(string fileName f in Directory.GetFiles("C\\:SomeDirectory"), "*.*",
SearchOption.AllDirectories)
{
//do stuff with each file found
}
With an FTP connection?
Many thanks!!
UPDATE / Final Answer
Special thanks to #sunk for getting this going. I made a minor tweek to his code that makes it fully recursive so that it can drill into nested folders. Here is the final code:
//A list that holds all file locations in all folders of a given FTP address:
List<string> fnl= new List<string>();
//A string to hold the base FTP address:
string ftpBase = "ftp://[SOME FTP ADDRESS]";
//A button-click event. Can be a stand alone method as well
private void GetFileLocations(object sender, EventArgs e)
{
//Get the file names from the FTP location:
DownloadFileNames(ftpBase);
//Once 'DownloadFileNames' has run, we have populated 'fnl'
foreach(var f in fnl)
{
//do stuff
}
}
//Gets all files in a given FTP address. RECURSIVE METHOD:
public void DownloadFileNames(string ftpAddress)
{
string uri = ftpAddress;
FtpWebRequest reqFTP = (FtpWebRequest)FtpWebRequest.Create(uri);
reqFTP.Credentials = new NetworkCredential("pella", "PellaWA01!");
reqFTP.EnableSsl = false;
reqFTP.KeepAlive = false;
reqFTP.UseBinary = true;
reqFTP.UsePassive = true;
reqFTP.Method = WebRequestMethods.Ftp.ListDirectory;
FtpWebResponse response = (FtpWebResponse)reqFTP.GetResponse();
Stream responseStream = response.GetResponseStream();
List<string> files = new List<string>();
StreamReader reader = new StreamReader(responseStream);
while (!reader.EndOfStream)
files.Add(reader.ReadLine());
reader.Close();
responseStream.Dispose();
//Loop through the resulting file names.
foreach (var fileName in files)
{
var parentDirectory = "";
//If the filename has an extension, then it actually is
//a file and should be added to 'fnl'.
if (fileName.IndexOf(".") > 0)
{
fnl.Add(ftpAddress.Replace("ftp://pella.upload.akamai.com/140607/pella/", "http://media.pella.com/") + fileName);
}
else
{
//If the filename has no extension, then it is just a folder.
//Run this method again as a recursion of the original:
parentDirectory += fileName + "/";
try
{
DownloadFileNames(ftpAddress + parentDirectory);
}
catch (Exception)
{
//throw;
}
}
}
}

First of all, you have to get the files name local on your machine using FTPWebRequest.
WebRequestMethods.Ftp.ListDirectory;
then use foreach {};
Here is the code:-
public List<string> DownloadFileNames()
{
string uri = "ftp://" + ftpServerIP + "/";
FtpWebRequest reqFTP = (FtpWebRequest)FtpWebRequest.Create(uri);
reqFTP.Credentials = new NetworkCredential(ftpUserID, ftpPassword);
reqFTP.EnableSsl = true;
reqFTP.KeepAlive = false;
reqFTP.UseBinary = true;
reqFTP.UsePassive = Settings.UsePassive;
reqFTP.Method = WebRequestMethods.Ftp.ListDirectory;
ServicePointManager.ServerCertificateValidationCallback = new System.Net.Security.RemoteCertificateValidationCallback(AcceptAllCertifications);
FtpWebResponse response = (FtpWebResponse)reqFTP.GetResponse();
Stream responseStream = response.GetResponseStream();
List<string> files = new List<string>();
StreamReader reader = new StreamReader(responseStream);
while (!reader.EndOfStream)
files.Add(reader.ReadLine());
reader.Close();
responseStream.Dispose();
return files;
}
Now you have the List:-
List<string> FileNameList = DownloadFileNames();
foreach (var fileName in FileNameList)
{
}

the ListDirectoryDetails command used in the example just returns a string. You will have to manually parse it to build a list of files and subdirectories.

Found at http://social.msdn.microsoft.com/Forums/en/ncl/thread/079fb811-3c55-4959-85c4-677e4b20bea3
string[] files = GetFileList();
foreach (string file in files)
{
Download(file);
}
public string[] GetFileList()
{
string[] downloadFiles;
StringBuilder result = new StringBuilder();
WebResponse response = null;
StreamReader reader = null;
try
{
FtpWebRequest reqFTP;
reqFTP = (FtpWebRequest)FtpWebRequest.Create(new Uri("ftp://" + ftpServerIP + "/"));
reqFTP.UseBinary = true;
reqFTP.Credentials = new NetworkCredential(ftpUserID, ftpPassword);
reqFTP.Method = WebRequestMethods.Ftp.ListDirectory;
reqFTP.Proxy = null;
reqFTP.KeepAlive = false;
reqFTP.UsePassive = false;
response = reqFTP.GetResponse();
reader = new StreamReader(response.GetResponseStream());
string line = reader.ReadLine();
while (line != null)
{
result.Append(line);
result.Append("\n");
line = reader.ReadLine();
}
// to remove the trailing '\n'
result.Remove(result.ToString().LastIndexOf('\n'), 1);
return result.ToString().Split('\n');
}
catch (Exception ex)
{
if (reader != null)
{
reader.Close();
}
if (response != null)
{
response.Close();
}
downloadFiles = null;
return downloadFiles;
}
}
private void Download(string file)
{
try
{
string uri = "ftp://" + ftpServerIP + "/" + remoteDir + "/" + file;
Uri serverUri = new Uri(uri);
if (serverUri.Scheme != Uri.UriSchemeFtp)
{
return;
}
FtpWebRequest reqFTP;
reqFTP = (FtpWebRequest)FtpWebRequest.Create(new Uri("ftp://" + ftpServerIP + "/" + remoteDir + "/" + file));
reqFTP.Credentials = new NetworkCredential(ftpUserID, ftpPassword);
reqFTP.KeepAlive = false;
reqFTP.Method = WebRequestMethods.Ftp.DownloadFile;
reqFTP.UseBinary = true;
reqFTP.Proxy = null;
reqFTP.UsePassive = false;
FtpWebResponse response = (FtpWebResponse)reqFTP.GetResponse();
Stream responseStream = response.GetResponseStream();
FileStream writeStream = new FileStream(localDestnDir + "\" + file, FileMode.Create);
int Length = 2048;
Byte[] buffer = new Byte[Length];
int bytesRead = responseStream.Read(buffer, 0, Length);
while (bytesRead > 0)
{
writeStream.Write(buffer, 0, bytesRead);
bytesRead = responseStream.Read(buffer, 0, Length);
}
writeStream.Close();
response.Close();
}
catch (WebException wEx)
{
MessageBox.Show(wEx.Message, "Download Error");
}
catch (Exception ex)
{
MessageBox.Show(ex.Message, "Download Error");
}
}