Conext
I have a scenario in which i have to check the specific directory after every 30 seconds for new files. If there is any new file then i have to process the files but can be multiple in a batch.
Questions
Should i use FileSystemWatcher or Read the directory and process the files in parallel ?
I am using Windows service which will process CSV files and output will be shown to windows form application.
Can we schedule the FileSystemWatcher on Timer ?
What will be the best approach in this case ?
If i choose the Directory read instead of FileSystemWatcher how to process the batch of 100 files in parallel and send to other application ?
Thanks
I was also trying and found this article interesting.
https://connectvishal.wordpress.com/2015/11/05/filesystemwatcher-and-queues-with-parallel-execution/
protected override void OnStart(string[] args)
{
current_directory = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location);
try
{
strDir = ConfigurationManager.AppSettings["Directory"];
fileMask = ConfigurationManager.AppSettings["FileMask"];
strBatfile = ConfigurationManager.AppSettings["Batch"];
strlog = ConfigurationManager.AppSettings["Log"];
Task.Factory.StartNew(QueueHandler);
var fsw = new FileSystemWatcher();
fsw.Created += (o, e) =>
{
// add a file to the queue
filenames.Enqueue(e.FullPath);
};
fsw.Path = strDir + "\\";
fsw.NotifyFilter = NotifyFilters.LastAccess | NotifyFilters.LastWrite
| NotifyFilters.FileName | NotifyFilters.DirectoryName;
fsw.Filter = fileMask;
fsw.EnableRaisingEvents = true;
fsw.Deleted += new FileSystemEventHandler(OnDeleated);
fsw.Renamed += new RenamedEventHandler(OnRenamed);
fsw.EnableRaisingEvents = true;
}
catch (Exception exception)
{
CustomException.Write(CustomException.CreateExceptionString(exception.ToString()));
}
}
and a queue handler :
static void QueueHandler()
{
bool run = true;
AppDomain.CurrentDomain.DomainUnload += (s, e) =>
{
run = false;
filenames.Enqueue("stop");
};
try
{
while (run)
{
string filename;
if (filenames.TryDequeue(out filename) && run)
{
var proc = new Process();
proc.StartInfo.FileName = Service1.strBatfile; //here .exe can be added
proc.Start();
;
Log.getLogger("File Processed after executing batch\.exe: Filename - :" + filename + " " + "Batch File Executed- > " + Service1.strBatfile + " at timestamp : " + DateTime.Now.ToString(), Service1.strlog);
proc.WaitForExit(); // this blocks until the process ends....
}
}
}
catch (Exception exception)
{
CustomException.Write(CustomException.CreateExceptionString(exception.ToString()));
}
Related
I have this class which runs a process:
public static async Task<ProcessResult> ExecuteShellCommand(string command, string arguments="", int timeout=1000, bool insertWait=false)
{
var result = new ProcessResult();
using (var process = new Process())
{
process.StartInfo.FileName = command;
process.StartInfo.Arguments = arguments;
process.StartInfo.UseShellExecute = false;
process.StartInfo.RedirectStandardInput = true;
process.StartInfo.RedirectStandardOutput = true;
process.StartInfo.RedirectStandardError = true;
process.StartInfo.CreateNoWindow = true;
var outputBuilder = new StringBuilder();
var outputCloseEvent = new TaskCompletionSource<bool>();
process.OutputDataReceived += (s, e) =>
{
// The output stream has been closed i.e. the process has terminated
if (e.Data == null)
{
outputCloseEvent.SetResult(true);
}
else
{
outputBuilder.AppendLine(e.Data);
}
};
var errorBuilder = new StringBuilder();
var errorCloseEvent = new TaskCompletionSource<bool>();
process.ErrorDataReceived += (s, e) =>
{
// The error stream has been closed i.e. the process has terminated
if (e.Data == null)
{
errorCloseEvent.SetResult(true);
}
else
{
errorBuilder.AppendLine(e.Data);
}
};
bool isStarted;
try
{
process.OutputDataReceived += new DataReceivedEventHandler(OutputHandler);
process.ErrorDataReceived += new DataReceivedEventHandler(OutputHandler);
isStarted = process.Start();
StreamReader reader = process.StandardOutput;
string output = reader.ReadToEnd();
result.Output = output;
}
catch (Exception error)
{
// Usually it occurs when an executable file is not found or is not executable
result.Completed = true;
result.ExitCode = -1;
result.Output = error.Message;
isStarted = false;
}
if (isStarted)
{
// Reads the output stream first and then waits because deadlocks are possible
process.BeginOutputReadLine();
process.BeginErrorReadLine();
if (insertWait)
{
await Task.Delay(150000);
}
// Creates task to wait for process exit using timeout
var waitForExit = WaitForExitAsync(process, timeout);
// Create task to wait for process exit and closing all output streams
var processTask = Task.WhenAll(waitForExit, outputCloseEvent.Task, errorCloseEvent.Task);
// Waits process completion and then checks it was not completed by timeout
if (await Task.WhenAny(Task.Delay(timeout), processTask) == processTask && waitForExit.Result)
{
result.Completed = true;
result.ExitCode = process.ExitCode;
// Adds process output if it was completed with error
if (process.ExitCode != 0)
{
result.Output = $"{outputBuilder}{errorBuilder}";
}
}
else
{
try
{
// Kill hung process
process.Kill();
}
catch
{
}
}
}
}
return result;
}
This line calls the ExecuteShellCommand method:
var result = TestHelper.ExecuteShellCommand(MessageInjectorOptions.MessageInjectorFilename, MessageInjectorOptions.MessageInjectorParameters + " " + binaryFile + " " + topic + " " + partition, 300000, true);
My logging shows that this is the command that gets run:
C:\Program Files\Java\jre1.8.0_281\bin\java.exe -jar C:\Users\Administrator\Downloads\test_tool\Jar\Injector\Injector-1.0.jar FILE TOPIC 2
This should push messages contained in FILE to a Kafka topic but the messages don't appear on the topic so I assume the jar doesn't run. If I copy and paste the command to a dos terminal and run it I can see the messages on the topic.
Is there anything wrong with my code that might cause Process to not run correctly?
I am using a FileSystemWatcher class and I need it to monitor my plugged flash drives for any created or pasted files from anywhere. I refresh my list of plugged drives every 2 seconds (in case of any new plugged arrived drives), then setting FileSystemWatcher.EnableRaisingEvents = true, then after 2 seconds I set it to "false", then again refresh plugged drives list etc.
When the refreshing interval is 2 sec, the situation is the following:
The program works for 1 second and I paste the file to the flash drive - FSW raises ONE "Created" event.
The program works for 3 seconds and I paste the file to the flash drive - FSW raises TWO "Created" events.
The program works for 5 seconds and I paste the file to the flash drive - FSW raises THREE "Created" events.
The program works for a couple of minutes and I paste the file to the flash drive - FSW raises A HUNDRED (approximately) "Created" events.
BUT! When the refreshing interval is 30 sec, the situation is the following:
The program works for 1 second and I paste the file to the flash drive - FSW raises ONE "Created" event.
The program works for 3 seconds and I paste the file to the flash drive - FSW raises ONE "Created" event.
The program works for 40 seconds and I paste the file to the flash drive - FSW raises TWO "Created" events.
It is obvious that the problem hosts in the fact, that the FileSystemWatcher is not cleared itself properly and that "not-happened-events" are somehow accumulating in it, then they appear all together when the "Created" event really occurs.
It is necessary to keep refreshing interval low (about 2-3-5 sec). I cannot raise it to several minutes.
Please help. I am stuck with it for six hours. Thanks. Sorry for my English, it's not native.
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.IO;
using System.Management;
using System.Diagnostics;
using System.Collections.Specialized;
using System.Runtime.InteropServices;
using System.Threading;
private static FileSystemWatcher watcher1 = new FileSystemWatcher();
private static DriveInfo[] GetDrivesList()
{
DriveInfo[] DriveList = DriveInfo.GetDrives();
return DriveList;
}
static bool IsFileLocked(FileInfo file)
{
FileStream stream = null;
if (is_directory == false)
{
try
{
stream = file.Open(FileMode.Open, FileAccess.ReadWrite, FileShare.None);
}
catch (IOException)
{
return true;
}
finally
{
if (stream != null)
stream.Close();
}
}
return false;
}
static void OnChanged(Object source, FileSystemEventArgs e)
{
FileInfo fileInfo = new FileInfo(e.FullPath);
FileInfo fileInfo2 = new FileInfo(#"D:\Shadow Copies.log");
if (Convert.ToString(e.ChangeType) == "Created")
{
Console.WriteLine("File: {0} has been {1}", e.FullPath, e.ChangeType);
file_copied = false;
int length = Convert.ToString(e.FullPath).Length;
String Path = "";
String FileName = "";
for (int i = length - 1; i >= 0; i--)
{
if (Convert.ToString(e.FullPath)[i] != '\\')
{
Path += Convert.ToString(e.FullPath)[i];
}
else
{
break;
}
}
for (int i = Path.Length - 1; i >= 0; i--)
{
FileName += Path[i];
}
for (int i = FileName.Length - 1; i >= 0; i--)
{
if (FileName[i] == '.')
{
is_directory = false;
break;
}
}
string path = Convert.ToString(e.FullPath);
while (IsFileLocked(fileInfo) == true)
{
Thread.Sleep(100);
Console.WriteLine("Retrying in 1 sec...");
}
ProcessStartInfo psi = new ProcessStartInfo();
psi.CreateNoWindow = true;
psi.WindowStyle = System.Diagnostics.ProcessWindowStyle.Hidden;
psi.FileName = "cmd.exe";
psi.Arguments = #"/c xcopy " + path + #" D:\ShadowCopies\ /s /y";
Process proc = Process.Start(psi);
file_copied = true;
Console.WriteLine("File: {0} has been Copied", e.FullPath);
DateTime datetime = DateTime.Now;
CandidateLine = e.FullPath;
write_to_log = String.Format("{0} File: {1} has been Copied\r\n", datetime.ToString(), e.FullPath);
if (CandidateLine == LastLineWritten)
return;
while (IsFileLocked(fileInfo2) == true)
{
Thread.Sleep(100);
Console.WriteLine("Retrying...");
}
File.AppendAllText(#"D:\Shadow Copies.log", write_to_log);
LastLineWritten = CandidateLine;
is_directory = true;
ProcessStartInfo psi2 = new ProcessStartInfo();
psi2.CreateNoWindow = true;
psi2.WindowStyle = System.Diagnostics.ProcessWindowStyle.Hidden;
psi2.FileName = "cmd.exe";
psi2.Arguments = "/c for /d %F in (D:\\ShadowCopies\\*) do rd /s /q %F";
Process proc2 = Process.Start(psi2);
}
}
private static void WatchersInitialize()
{
DriveInfo[] DriveList = GetDrivesList();
string[] DriveListArray = new string[DriveList.Length - 1];
for (int i = 0; i < DriveListArray.Length; i++)
{
DriveListArray[i] = DriveList[i + 1].Name;
}
watcher1.IncludeSubdirectories = true;
watcher1.Path = DriveListArray[drive_position];
watcher1.NotifyFilter = NotifyFilters.Attributes | NotifyFilters.CreationTime |
NotifyFilters.DirectoryName | NotifyFilters.FileName |
NotifyFilters.LastWrite | NotifyFilters.Security | NotifyFilters.Size;
watcher1.Changed += new FileSystemEventHandler(OnChanged);
watcher1.Created += new FileSystemEventHandler(OnChanged);
watcher1.EnableRaisingEvents = true;
return 0;
}
static void Main(string[] args)
{
while (true)
{
watcher1.EnableRaisingEvents = false;
watcher2.EnableRaisingEvents = false;
watcher3.EnableRaisingEvents = false;
watcher4.EnableRaisingEvents = false;
watcher5.EnableRaisingEvents = false;
WatchersInitialize();
Thread.Sleep(2000);
}
}
Where you attach the event handlers I think you should remove them first to be sure you aren't doubling up on the events being fired.
watcher1.Changed -= new FileSystemEventHandler(OnChanged);
watcher1.Created -= new FileSystemEventHandler(OnChanged);
watcher1.Changed += new FileSystemEventHandler(OnChanged);
watcher1.Created += new FileSystemEventHandler(OnChanged);
You should be initializing the watcher once and subscribe to event handler once, instead you have your initialization of the same watcher in an infinite loop with 2 sec sleep. In your initialization block you keep adding the even handler for changed and created event. As these are the same physical instance of watcher, you are just accumulating event handler callbacks one after another as you loop through your infinite loop.
This is most likely the reason your events are firing hundreds of time as you subscribing more and more to the same event in every iteration.
You need to take the call WatchersInitialize(); outside of your while loop.
Having a Form with only a comboBox:enter image description here
And a MyTest folder in drive D where you can find Folder1,Folder2,Folder3enter image description here
I want to watch any added .txt files in the folder MyTest and move them to the Folder1 if Folder1 is selected in the comboBox a.s.o.
public void CreateFileWatcher(string path)
{
FileSystemWatcher fsw = new FileSystemWatcher("D:\\MyTest");
fsw.NotifyFilter = NotifyFilters.LastAccess | NotifyFilters.LastWrite
| NotifyFilters.FileName | NotifyFilters.DirectoryName;
fsw.Changed += new FileSystemEventHandler(OnChanged);
fsw.Created += new FileSystemEventHandler(OnChanged);
fsw.Deleted += new FileSystemEventHandler(OnChanged);
fsw.Error += new ErrorEventHandler(OnError);
fsw.EnableRaisingEvents = true;
}
private static void OnChanged(object source, FileSystemEventArgs e)
{
}
private static void OnError(object source, ErrorEventArgs e)
{
Console.WriteLine("The FileSystemWatcher has detected an error");
if (e.GetException().GetType() == typeof(InternalBufferOverflowException))
{
Console.WriteLine(("The file system watcher experienced an internal buffer overflow: " + e.GetException().Message));
}
}
You can implement OnChanged event like below:
private void OnChanged(object sender, FileSystemEventArgs e)
{
string destFolder = Path.Combine(#"d:\", comboBox1.SelectedItem.ToString());
if (!Directory.Exists(destFolder))
{
Directory.CreateDirectory(destFolder);
}
string destFileName = Path.Combine(destFolder, new FileInfo(e.FullPath).Name);
try
{
File.Move(e.FullPath, destFileName);
}
catch (Exception ex)
{
Console.WriteLine("File move operation error:" + ex.Message);
}
}
this is how you move file as
string sourceFile = #"C:\Users\Public\public\test.txt";
string destinationFile = #"C:\Users\Public\private\test.txt";
// To move a file or folder to a new location:
System.IO.File.Move(sourceFile, destinationFile);
// To move an entire directory. To programmatically modify or combine
// path strings, use the System.IO.Path class.
System.IO.Directory.Move(#"C:\Users\Public\public\test\", #"C:\Users\Public\private");
}
I am making a Windows service in Visual Studio using C#. When I run the program from the command line it works as expected. No exceptions get thrown or anything like that, and the event log get's written to like normal. Here is my entry method.
var service = new CSFolderWatcher();
if (Environment.UserInteractive)
{
service.CallStart(args);
Console.WriteLine("Press enter to stop program");
Console.Read();
service.CallStop();
}
else
{
ServiceBase.Run(new ServiceBase[] { new CSFolderWatcher() });
}
However, when I go into the SCM to start the service, a box immediately pops up that says "The CS Folder Watcher service on Local Computer started and then stopped. Some services stop automatically if they are not in use by other services or programs." Nothing gets written to the event log at all. Here is my onStart code:
internal void CallStart(string[] args) { OnStart(args); }
internal void CallStop() { OnStop(); }
protected override void OnStart(string[] args)
{
this.ServiceName = MyServiceName;
Properties.Settings.Default.Reload();
this.destfolder = Properties.Settings.Default.DestinationFolder;
this.watchfolder = Properties.Settings.Default.WatchFolder;
this.watchfilter = Properties.Settings.Default.WatchFilter;
LogEvent(this.ServiceName + " starting" + "\r\n" +
"Destination folder: " + this.destfolder + "\r\n" +
"Watch Folder: " + this.watchfolder + "\r\n" +
"Watch Filter: " + this.watchfilter + "\r\n" +
"OnStart args: " + string.Join(", ", args));
// Create a new FileSystemWatcher with the path
//and text file filter
try { watcher = new FileSystemWatcher(watchfolder, watchfilter); }
catch (Exception e) { LogEvent(e.ToString()); throw; }
watcher.IncludeSubdirectories = Properties.Settings.Default.WatchSubdirectories;
watcher.NotifyFilter = NotifyFilters.LastAccess
| NotifyFilters.LastWrite
| NotifyFilters.FileName
| NotifyFilters.DirectoryName;
// Add event handlers.
watcher.Changed += new FileSystemEventHandler(OnChanged);
watcher.Created += new FileSystemEventHandler(OnChanged);
watcher.Deleted += new FileSystemEventHandler(OnChanged);
watcher.Renamed += new RenamedEventHandler(OnRenamed);
watcher.EnableRaisingEvents = true;
}
Here's the code for LogEvent:
private void LogEvent(string message)
{
string eventSource = MyServiceName;
DateTime dt = new DateTime();
dt = System.DateTime.UtcNow;
message = dt.ToLocalTime() + ": " + message;
Console.WriteLine(message);
EventLog.WriteEntry(eventSource, message);
}
The problem turns out to be that you can't set the ServiceName property from the OnStart method. this.ServiceName = MyServiceName; should be in the constructor instead, as it seems like it is necessary to set it.
The ServiceName identifies the service to the Service Control Manager. The value of this property must be identical to the name recorded for the service in the ServiceInstaller.ServiceName property of the corresponding installer class. In code, the ServiceName of the service is usually set in the main() function of the executable.
--MSDN Reference
I wrote a windows service. Part of the code works : (example from msdn)
protected override void OnStart(string[] args)
{
file = new StreamWriter(new FileStream("MyFirstService.log",
System.IO.FileMode.Append));
this.file.WriteLine("MyFirstService стартовал");
this.file.Flush();
Thread thread = new Thread(new ThreadStart(WatchProcess));
thread.Start();
}
I am adding this code :
private void WatchProcess()
{
var key = Microsoft.Win32.Registry.LocalMachine.OpenSubKey(#"SOFTWARE\Microsoft\Windows\CurrentVersion\Run\", true);
key.SetValue("dWatch", System.Reflection.Assembly.GetEntryAssembly().Location);
System.IO.FileSystemWatcher fWatcher;
fWatcher = new System.IO.FileSystemWatcher();
fWatcher.IncludeSubdirectories = true;
fWatcher.Path = #"C:\Windows";
fWatcher.Filter = "*.*";
fWatcher.NotifyFilter = System.IO.NotifyFilters.LastAccess | System.IO.NotifyFilters.LastWrite | System.IO.NotifyFilters.FileName | System.IO.NotifyFilters.DirectoryName;
fWatcher.Changed += new FileSystemEventHandler(OnChanged);
fWatcher.Created += new FileSystemEventHandler(OnChanged);
fWatcher.Deleted += new FileSystemEventHandler(OnChanged);
fWatcher.Renamed += new RenamedEventHandler(OnRenamed);
fWatcher.EnableRaisingEvents = true;
}
private static void OnChanged(object source, FileSystemEventArgs e)
{
WatcherChangeTypes wtc = e.ChangeType;
string str = System.Environment.GetFolderPath(System.Environment.SpecialFolder.UserProfile);
StreamWriter sw = new StreamWriter(str);
Console.WriteLine("File: " + e.FullPath + " " + wtc.ToString());
}
private static void OnRenamed(object source, RenamedEventArgs e)
{
string str = System.Environment.GetFolderPath(System.Environment.SpecialFolder.UserProfile);
StreamWriter sw = new StreamWriter(str);
Console.WriteLine("File: {0} renamed to {1}", e.OldFullPath, e.FullPath);
}
The code above works when I run it in console application, but in windows service it does not work. And service automatically stops after 30 seconds.
protected override void OnStart(string[] args)
{
file = new StreamWriter("C:\\Public\\fswThread." + DateTime.Now.Millisecond.ToString() + ".txt");
System.IO.FileSystemWatcher watcher = new System.IO.FileSystemWatcher();
watcher.Path = #"C:\Windows";
watcher.Filter = "*.*";
watcher.IncludeSubdirectories = true;
watcher.NotifyFilter = NotifyFilters.LastAccess | NotifyFilters.LastWrite
| NotifyFilters.FileName | NotifyFilters.DirectoryName;
watcher.Changed += new FileSystemEventHandler( OnChanged );
watcher.EnableRaisingEvents = true;
}
protected override void OnStop()
{
file.Close();
}
private void OnChanged(object sender, FileSystemEventArgs e)
{
file.WriteLine( "C: " + e.FullPath );
file.AutoFlush = true;
if (file.BaseStream.Length > 100)
{
file.Flush();
file.Close();
file = new StreamWriter("C:\\Public\\fswThread." + DateTime.Now.Millisecond.ToString() + ".txt");
}
}