So I was thinking of writing online c# compiler and execution environment. And of course problem #1 is security. I ended up creating a little-privileged appdomain for user code and starting it in a new process which is tightly monitored for cpu and memory consumption. Standard console application namespaces are available. So my question is this: can you think of ways of breaking something in some way? You can try your ideas on the spot rundotnet.
Edit2 If anyone cares about the code, there is now open source fork of this project: rextester at github
Edit1 As a response to one of the comments here are some code samples.
So basically you create a console application. I'll just post a big chunk of it:
class Sandboxer : MarshalByRefObject
{
private static object[] parameters = { new string[] { "parameter for the curious" } };
static void Main(string[] args)
{
Console.OutputEncoding = Encoding.UTF8;
string pathToUntrusted = args[0].Replace("|_|", " ");
string untrustedAssembly = args[1];
string entryPointString = args[2];
string[] parts = entryPointString.Split(new string[] { "|" }, StringSplitOptions.RemoveEmptyEntries);
string name_space = parts[0];
string class_name = parts[1];
string method_name = parts[2];
//Setting the AppDomainSetup. It is very important to set the ApplicationBase to a folder
//other than the one in which the sandboxer resides.
AppDomainSetup adSetup = new AppDomainSetup();
adSetup.ApplicationBase = Path.GetFullPath(pathToUntrusted);
//Setting the permissions for the AppDomain. We give the permission to execute and to
//read/discover the location where the untrusted code is loaded.
PermissionSet permSet = new PermissionSet(PermissionState.None);
permSet.AddPermission(new SecurityPermission(SecurityPermissionFlag.Execution));
//Now we have everything we need to create the AppDomain, so let's create it.
AppDomain newDomain = AppDomain.CreateDomain("Sandbox", null, adSetup, permSet, null);
//Use CreateInstanceFrom to load an instance of the Sandboxer class into the
//new AppDomain.
ObjectHandle handle = Activator.CreateInstanceFrom(
newDomain, typeof(Sandboxer).Assembly.ManifestModule.FullyQualifiedName,
typeof(Sandboxer).FullName
);
//Unwrap the new domain instance into a reference in this domain and use it to execute the
//untrusted code.
Sandboxer newDomainInstance = (Sandboxer)handle.Unwrap();
Job job = new Job(newDomainInstance, untrustedAssembly, name_space, class_name, method_name, parameters);
Thread thread = new Thread(new ThreadStart(job.DoJob));
thread.Start();
thread.Join(10000);
if (thread.ThreadState != ThreadState.Stopped)
{
thread.Abort();
Console.Error.WriteLine("Job taking too long. Aborted.");
}
AppDomain.Unload(newDomain);
}
public void ExecuteUntrustedCode(string assemblyName, string name_space, string class_name, string method_name, object[] parameters)
{
MethodInfo target = null;
try
{
target = Assembly.Load(assemblyName).GetType(name_space+"."+class_name).GetMethod(method_name);
if (target == null)
throw new Exception();
}
catch (Exception)
{
Console.Error.WriteLine("Entry method '{0}' in class '{1}' in namespace '{2}' not found.", method_name, class_name, name_space);
return;
}
...
//Now invoke the method.
try
{
target.Invoke(null, parameters);
}
catch (Exception e)
{
...
}
}
}
class Job
{
Sandboxer sandboxer = null;
string assemblyName;
string name_space;
string class_name;
string method_name;
object[] parameters;
public Job(Sandboxer sandboxer, string assemblyName, string name_space, string class_name, string method_name, object[] parameters)
{
this.sandboxer = sandboxer;
this.assemblyName = assemblyName;
this.name_space = name_space;
this.class_name = class_name;
this.method_name = method_name;
this.parameters = parameters;
}
public void DoJob()
{
try
{
sandboxer.ExecuteUntrustedCode(assemblyName, name_space, class_name, method_name, parameters);
}
catch (Exception e)
{
Console.Error.WriteLine(e.Message);
}
}
}
You compile the above and have executable which you start and monitor in a new process:
using (Process process = new Process())
{
try
{
double TotalMemoryInBytes = 0;
double TotalThreadCount = 0;
int samplesCount = 0;
process.StartInfo.FileName = /*path to sandboxer*/;
process.StartInfo.Arguments = folder.Replace(" ", "|_|") + " " + assemblyName + " Rextester|Program|Main"; //assemblyName - assembly that contains compiled user code
process.StartInfo.UseShellExecute = false;
process.StartInfo.CreateNoWindow = true;
process.StartInfo.RedirectStandardOutput = true;
process.StartInfo.RedirectStandardError = true;
DateTime start = DateTime.Now;
process.Start();
OutputReader output = new OutputReader(process.StandardOutput);
Thread outputReader = new Thread(new ThreadStart(output.ReadOutput));
outputReader.Start();
OutputReader error = new OutputReader(process.StandardError);
Thread errorReader = new Thread(new ThreadStart(error.ReadOutput));
errorReader.Start();
do
{
// Refresh the current process property values.
process.Refresh();
if (!process.HasExited)
{
try
{
var proc = process.TotalProcessorTime;
// Update the values for the overall peak memory statistics.
var mem1 = process.PagedMemorySize64;
var mem2 = process.PrivateMemorySize64;
//update stats
TotalMemoryInBytes += (mem1 + mem2);
TotalThreadCount += (process.Threads.Count);
samplesCount++;
if (proc.TotalSeconds > 5 || mem1 + mem2 > 100000000 || process.Threads.Count > 100 || start + TimeSpan.FromSeconds(10) < DateTime.Now)
{
var time = proc.TotalSeconds;
var mem = mem1 + mem2;
process.Kill();
...
}
}
catch (InvalidOperationException)
{
break;
}
}
}
while (!process.WaitForExit(10)); //check process every 10 milliseconds
process.WaitForExit();
...
}
...
class OutputReader
{
StreamReader reader;
public string Output
{
get;
set;
}
StringBuilder sb = new StringBuilder();
public StringBuilder Builder
{
get
{
return sb;
}
}
public OutputReader(StreamReader reader)
{
this.reader = reader;
}
public void ReadOutput()
{
try
{
int bufferSize = 40000;
byte[] buffer = new byte[bufferSize];
int outputLimit = 200000;
int count;
bool addMore = true;
while (true)
{
Thread.Sleep(10);
count = reader.BaseStream.Read(buffer, 0, bufferSize);
if (count != 0)
{
if (addMore)
{
sb.Append(Encoding.UTF8.GetString(buffer, 0, count));
if (sb.Length > outputLimit)
{
sb.Append("\n\n...");
addMore = false;
}
}
}
else
break;
}
Output = sb.ToString();
}
catch (Exception e)
{
...
}
}
}
Assemblies that user code can use are added at compile time:
CompilerParameters cp = new CompilerParameters();
cp.GenerateExecutable = false;
cp.OutputAssembly = ...
cp.GenerateInMemory = false;
cp.TreatWarningsAsErrors = false;
cp.WarningLevel = 4;
cp.IncludeDebugInformation = false;
cp.ReferencedAssemblies.Add("System.dll");
cp.ReferencedAssemblies.Add("System.Core.dll");
cp.ReferencedAssemblies.Add("System.Data.dll");
cp.ReferencedAssemblies.Add("System.Data.DataSetExtensions.dll");
cp.ReferencedAssemblies.Add("System.Xml.dll");
cp.ReferencedAssemblies.Add("System.Xml.Linq.dll");
using (CodeDomProvider provider = CodeDomProvider.CreateProvider(/*language*/))
{
cr = provider.CompileAssemblyFromSource(cp, new string[] { data.Program });
}
Have you looked at Mono's Compiler as a service? I think that is pretty cool what they are doing, perhaps something there could be useful to you for this project.
For a good example of something similar to this already in existence, there is a place at http://www.topcoder.com, which has an "Algorithm Arena", where code is submitted and automatically scored. There are restrictions against using certain types of classes, such as Exception, but it may be a good idea to examine their application for a proof of concept.
Related
Unresponsive UI using background worker, processing files are very slow.
Two different issues encountered here. GDpicture SDK is used for image processing. CPU Utilization is bare minimum, how can I maximize performance, ultimately have responsive and fast wpf application.
namespace OCR
{
public partial class MainWindow : Window
{
BackgroundWorker bw;/*= new BackgroundWorker();*/
private SynchronizationContext threadSyn = null;
string log_cap = string.Empty;
List<string> log_list = new List<string>();
string value = "Merged";
public MainWindow()
{
try
{
InitializeComponent();
InitializeBackgroundWorker();
File_process();
string configpath = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, #"path.txt");
string[] configfile = File.ReadAllLines(configpath);
if (configfile.Length > 1)
{
ip.Text = configfile[0];
op.Text = configfile[1];
ex_tb.Text = configfile[2];
Protb.Text = configfile[3];
}
cbPDFConform.Items.Clear();
for (int i = 0; i < Enum.GetNames(typeof(PdfConformance)).Length - 1; i++)
{
ComboBoxItem cbi = new ComboBoxItem();
cbi.Content = Enum.GetName(typeof(PdfConformance), (PdfConformance)i);
PdfConformance test = (PdfConformance)i;
cbi.Tag = (PdfConformance)i;
cbPDFConform.Items.Add(cbi);
}
cbPDFConform.SelectedIndex = 0;
cbProcessorCount.Items.Clear();
for (int i = 1; i <= Environment.ProcessorCount; i++)
{
cbProcessorCount.Items.Add(i.ToString());
if (Environment.ProcessorCount / 2 == i) { cbProcessorCount.SelectedIndex = i - 1; }
}
LicenseManager oLicenseManager = new LicenseManager();
oLicenseManager.RegisterKEY("");
configpath = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, #"path.txt");
configfile = File.ReadAllLines(configpath);
if (configfile.Length > 1)
{
ip.Text = configfile[0];
op.Text = configfile[1];
ex_tb.Text = configfile[2];
Protb.Text = configfile[3];
}
GrantAccess(ip.Text);
GrantAccess(op.Text);
GrantAccess(ex_tb.Text);
GrantAccess(Protb.Text);
threadSyn = SynchronizationContext.Current;
}
catch (Exception e1)
{ MessageBox.Show("e1" + e1.Message); }
}
private void InitializeBackgroundWorker()
{
bw = new BackgroundWorker();
bw.DoWork += Bw_DoWork;
bw.WorkerSupportsCancellation = true;
}
public async void File_process()
{
await Task.Run(() => converttiffpdfreducer());
}
private void Bw_DoWork(object sender, DoWorkEventArgs e)
{
this.Dispatcher.Invoke(() =>
{
try
{
using (StreamWriter sw = new StreamWriter(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, #"path.txt"), false))
{
sw.WriteLine(ip.Text);
sw.WriteLine(op.Text);
sw.WriteLine(ex_tb.Text);
sw.WriteLine(Protb.Text);
sw.Close();
}
ParallelOptions parallelOptions = new ParallelOptions();
parallelOptions.MaxDegreeOfParallelism = int.Parse(cbProcessorCount.SelectedItem.ToString());
var watch1 = new System.Diagnostics.Stopwatch();
watch1.Start();
converttiffpdfreducer();
//deletenew();
watch1.Stop();
TimeSpan ts1 = watch1.Elapsed;
ts1.ToString("mm\\:ss");
if (MergeChk.IsChecked == false)
{
value = "OCRed";
}
WriteLn("All documents have been successfully " + value + " " + ts1 + " " + DateTime.Now +" "+Environment.UserName);
}
catch (Exception DOwork)
{ MessageBox.Show("e2 " + DOwork.Message); }
});
}
private void GrantAccess(string fullPath)
{
DirectoryInfo dInfo = new DirectoryInfo(fullPath);
DirectorySecurity dSecurity = dInfo.GetAccessControl();
dSecurity.AddAccessRule(new FileSystemAccessRule(new SecurityIdentifier(WellKnownSidType.WorldSid, null), FileSystemRights.FullControl, InheritanceFlags.ObjectInherit | InheritanceFlags.ContainerInherit, PropagationFlags.NoPropagateInherit, AccessControlType.Allow));
dInfo.SetAccessControl(dSecurity);
}
private string[] mutliocr(string[] arr)
{
string box = string.Empty;
string box1 = string.Empty;
try
{
string filepath = string.Empty;
string outpath = ex_tb.Text;
if (MergeChk.IsChecked == true)
{ filepath = op.Text; }
else if (MergeChk.IsChecked == false)
{ filepath = Protb.Text; }
System.Windows.Threading.Dispatcher.CurrentDispatcher.Invoke((Action)(() =>
{
Thread.CurrentThread.IsBackground = true;
var watch2 = new System.Diagnostics.Stopwatch();
watch2.Start();
string[] getfilearray = arr;
for (int f = 0; f < getfilearray.Length; f++)
{
string dirName = Directory.GetParent(getfilearray[f]).FullName;
string folder = Directory.GetParent(getfilearray[f]).FullName;
box = Path.GetDirectoryName(getfilearray[f]);
box1 = Path.GetDirectoryName(box);
string getextension = Path.GetExtension(getfilearray[f]);
string[] newF = Directory.EnumerateFiles(dirName, "*.*", SearchOption.AllDirectories).ToArray();
string FN = Directory.GetParent(getfilearray[f]).Name;
string ocrfolder = (new FileInfo(getfilearray[f]).Directory.FullName);
string filen = Path.Combine(ocrfolder, folder, FN + "-ocr" + getextension);
string dict = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "Redist", "OCR");
if (!Directory.Exists(ocrfolder))
{
Directory.CreateDirectory(ocrfolder);
}
GrantAccess(ocrfolder);
GdPicturePDF oGdPicturePDF = new GdPicturePDF();
oGdPicturePDF.OcrPagesDone += OcrPagesDone;
void OcrPagesDone(GdPictureStatus status1)
{
if (oGdPicturePDF.SaveToFile(filen) == GdPictureStatus.OK)
{ }
else
MessageBox.Show("PDF: The OCR-ed file has failed to save. Status: " + oGdPicturePDF.GetStat().ToString());
}
GdPictureStatus status = GdPictureStatus.OK;
if (oGdPicturePDF.LoadFromFile(getfilearray[f], false) == GdPictureStatus.OK)
if (status == GdPictureStatus.OK)
{
if (oGdPicturePDF.OcrPages_4("*", 0, "eng", dict, "", 300, OCRMode.FavorSpeed, 1, true) == GdPictureStatus.OK)
if (status == GdPictureStatus.OK)
{ }
else
{ MessageBox.Show("PDF: The OCR process has failed. Status: " + status.ToString()); }
}
else
{ MessageBox.Show("PDF: The PDF file has failed to load. Status: " + status.ToString()); }
oGdPicturePDF.Dispose();
GrantAccess(getfilearray[f]);
File.Delete(getfilearray[f]);
watch2.Stop();
TimeSpan ts2 = watch2.Elapsed;
ts2.ToString("mm\\:ss");
WriteLn(" OCR pages " + filen.Replace(op.Text, "") + " " + ts2 + " " + DateTime.Now);
}
if (MergeChk.IsChecked == true)
{
foreach (string str in Directory.EnumerateFiles(op.Text, "*.pdf", SearchOption.AllDirectories).ToArray())
{
if (Path.GetFileNameWithoutExtension(str).EndsWith("-ocr"))
File.Move(str, Path.Combine(Path.GetDirectoryName(str), Path.GetFileNameWithoutExtension(str).Substring(0, Path.GetFileNameWithoutExtension(str).Length - 4) + ".pdf"));
}
}
if (MergeChk.IsChecked == false)
{
FileSystem.MoveDirectory(Protb.Text, op.Text, UIOption.AllDialogs);
Directory.CreateDirectory(Protb.Text);
string FF = string.Empty;
foreach (string str in Directory.EnumerateFiles(op.Text, "*.pdf", SearchOption.AllDirectories))
{
if (Path.GetFileNameWithoutExtension(str).EndsWith("-ocr"))
File.Move(str, Path.Combine(Path.GetDirectoryName(str), Path.GetFileNameWithoutExtension(str).Substring(0, Path.GetFileNameWithoutExtension(str).Length - 4) + ".pdf"));
}
}
}));
}
catch (Exception mul)
{
}
return arr;
}
public static string browseFolder()
{
Microsoft.Win32.OpenFileDialog dlg = new Microsoft.Win32.OpenFileDialog();
System.Windows.Forms.FolderBrowserDialog fbd = new System.Windows.Forms.FolderBrowserDialog();
System.Windows.Forms.DialogResult result = fbd.ShowDialog();
string path = string.Empty;
if (result == (System.Windows.Forms.DialogResult)MessageBoxResult.OK)
{
path = fbd.SelectedPath;
if (path[path.Length - 1] != '\\')
{
path = path + "\\";
}
}
return path;
}
private string[] converttiffpdfreducer()
{
string[] dir = null;
string box = string.Empty;
string box1 = string.Empty;
string[] gg = null;
try
{
string filepath = ip.Text;
string outpath = Protb.Text;
System.Windows.Threading.Dispatcher.CurrentDispatcher.Invoke((Action)(() =>
{
PdfConformance optPDFConform = PdfConformance.Unknown;
dir = Directory.EnumerateDirectories(filepath, "*.*", SearchOption.AllDirectories).Where(l => l.Length != 0).OrderBy(f => f).ToArray();
for (int ad = 0; ad < dir.Length; ad++)
{ string[] getfilearray = Directory.EnumerateFiles(dir[ad], "*.*", SearchOption.AllDirectories).ToArray();
if (getfilearray.Length == 0)
break;
if (getfilearray.Length != 0)
for (int f = 0; f < getfilearray.Length; f++)
{
string getext = Path.GetExtension(getfilearray[f]);
string fd = Path.GetDirectoryName(getfilearray[f]);
string op_path = fd.Replace(filepath, Protb.Text);
string getextension = Path.GetExtension(getfilearray[f]);
string dict = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "Redist", "OCR");
string outputPath = fd.Replace(filepath, outpath);
string FNAME = Path.GetFileNameWithoutExtension(getfilearray[f]);
string fn = Path.GetDirectoryName(getfilearray[f]).Replace(filepath, outpath);
string filen = Path.Combine(outputPath, fn, FNAME + ".pdf");
string savefile = Path.Combine(op_path, filen);
string pathString = getfilearray[f];
box = Path.GetDirectoryName(getfilearray[f]);
box1 = Path.GetDirectoryName(box);
using (GdPictureDocumentConverter oConverter = new GdPictureDocumentConverter()) {
GdPictureStatus status = new GdPictureStatus();
if (Path.GetExtension(getfilearray[f]).ToUpper() == ".PDF")
{
status = oConverter.LoadFromFile(getfilearray[f], GdPicture14.DocumentFormat.DocumentFormatPDF);
}
else if (Path.GetExtension(getfilearray[f]).ToUpper() == ".TIF" || Path.GetExtension(getfilearray[f]).ToUpper() == ".TIFF")
{
status = oConverter.LoadFromFile(getfilearray[f], GdPicture14.DocumentFormat.DocumentFormatTIFF);
}
else if (Path.GetExtension(getfilearray[f]).ToUpper() == ".JPG")
{
status = oConverter.LoadFromFile(getfilearray[f], GdPicture14.DocumentFormat.DocumentFormatJPEG);
}
if (status == GdPictureStatus.OK)
{
if (!Directory.Exists(op_path))
{
Directory.CreateDirectory(op_path);
}
GrantAccess(op_path);
optPDFConform = (PdfConformance)((ComboBoxItem)cbPDFConform.SelectedItem).Tag;
status = oConverter.SaveAsPDF(savefile, optPDFConform);
if (status == GdPictureStatus.OK)
{ }
else
{ }
}
else
{ }
}
}
string BOXX = box.Replace(ip.Text, Protb.Text);
string[] Arr = Directory.EnumerateFiles(BOXX, "*.pdf", SearchOption.AllDirectories).ToArray();
if (MergeChk.IsChecked == true)
{ merge(Arr); }
else if (MergeChk.IsChecked == false)
{
mutliocr(Arr);
}
}
}));
}
catch (Exception ee)
{ }
return dir;
}
private string[] merge(string[] arr)
{
string box = string.Empty;
string box1 = string.Empty; string[] gg = null;
System.Windows.Threading.Dispatcher.CurrentDispatcher.Invoke((Action)(() =>
{
box = Path.GetDirectoryName(arr[0]);
box1 = Path.GetDirectoryName(box);
string dirName = Directory.GetParent(arr[0]).FullName;
string BOXFILES = Path.GetDirectoryName(dirName);
string folder = Directory.GetParent(arr[0]).FullName.Replace(Protb.Text, op.Text);
string ocrfolder = (new FileInfo(arr[0]).Directory.FullName).Replace(Protb.Text, op.Text);
string fn = Directory.GetParent(arr[0]).Name;
string filen = Path.Combine(ocrfolder, folder, fn + ".pdf");
if (!Directory.Exists(ocrfolder))
{
Directory.CreateDirectory(ocrfolder);
}
GrantAccess(ocrfolder);
using (GdPicturePDF oGdPicturePDF = new GdPicturePDF())
{
GdPictureStatus status = oGdPicturePDF.MergeDocuments(ref arr, filen);
if (status == GdPictureStatus.OK)
{ }
else
{ }
oGdPicturePDF.Dispose();
}
Directory.Delete(box, true);
string BOXX = box.Replace(Protb.Text, op.Text);//op
string[] files = Directory.EnumerateFiles(BOXX, "*.pdf", SearchOption.AllDirectories).ToArray();
if (MergeChk.IsChecked == true)
{ mutliocr(files); }
}));
return gg;
}
private void inbtn_Click(object sender, RoutedEventArgs e)
{
try
{ ip.Text = browseFolder(); }
catch (Exception e7)
{ MessageBox.Show("e7" + e7.Message); }
}
private void obtn_Click(object sender, RoutedEventArgs e)
{
try
{ op.Text = browseFolder(); }
catch (Exception e8)
{ MessageBox.Show("e8" + e8.Message); }
}
private void start_btn_Click(object sender, RoutedEventArgs e)
{
if (!bw.IsBusy)
{
// Cancel the asynchronous operation.
this.bw.CancelAsync();
// Disable the Cancel button.
bw.RunWorkerAsync();
start_btn.Content = "Stop";
//this.Status.Content = "Running....";
}
else
{
bw.CancelAsync();
start_btn.Content = "Start";
//this.Status.Content = "Stopped....";
}
}
private void pro_btn_Click(object sender, RoutedEventArgs e)
{
try
{ Protb.Text = browseFolder(); }
catch (Exception e10)
{ MessageBox.Show("e10" + e10.Message); }
}
private void excep_Click(object sender, RoutedEventArgs e)
{
try
{ ex_tb.Text = browseFolder(); }
catch (Exception e11)
{ MessageBox.Show("e111" + e11.Message); }
}
private void WriteLn(string text)
{
logtb.Dispatcher.BeginInvoke(new Action(() =>
{
logtb.Text += text + Environment.NewLine;
}));
log_list.Add(text);
log_cap = text + Environment.NewLine + log_cap;
using (StreamWriter sw = new StreamWriter(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, #"FileProcessing_log.txt"), false))
{
foreach (string l in log_list)
sw.WriteLine(l);
sw.Close();
}
}
}
}
The point of a BackgroundWorker is to push things off the UI thread. You do that, but then in Bw_DoWork you immediately push things back the UI thread, with this.Dispatcher.Invoke, which blocks the worker thread until the work now running on the UI thread completes.
Basically: remove that this.Dispatcher.Invoke call, and just run the code on the worker. If you need to touch the UI, then you'll need to be on the UI thread for those bits, but only those buts.
Likewise, I doubt that mutliocr should be using dispatch invoke, and it certainly shouldn't change the UI thread to being a background thread (Thread.CurrentThread.IsBackground = true;).
Your code is very very smelly and ugly. You seem to follow absolutely no naming convention at all. Locals and class members are camelCase and PascalCase and some use no casing at all and others use underscores. You should really review you code carefully with many aspects in mind and clean it up. There are some redundancies too.
A really bad habbit of yours is the excessive use of the Dispatcher. For example, you create a background thread and post the complete! work of this thread back to the Dispatcher/UI thread. A huge waste of resources and elimination of any multithreading benefit.
You don't want to put all your work on the Dispatcher. You want to offload CPU intensive eork to a background thread. You want to make use of asynchronous APIs where possible. Because you want to keep the UI responsive. Dispatcher means UI thread.
Some highly important points of interest
GrantAccess implementation is a severe security violation of user rights. Don't modify access rules. Rather filter and ignore resources where the current user is not authorized for access. What is especially critical, is that you never revert the access rights to the original state.
As a general rule: don't execute IO related code (e.g., database, HTTP streams, file IO) on a background thread. Use async APIs instead. Use threading only for CPU bound work (e.g. computations, conversions).
File has an async API you should always use. For example File.OpenRead returns a FileStream which exposes a FileStream.ReadAsnyc member. if you need more convenience when handling files like read line by line, then wrap the FileStream into a StreamReader/StreamWriter and use members like StreamReader.ReadLineAsync
To avoid Dispatcher calls, pass the required UI values to the concurent methods as argument. Better use data binding (which won't eliminate the cross threading issues writing, but would make your code more readable and eliminates Dispatcher invocation to read the values). Take alook at the refactored File_process method below. It shows how to pass UI values as argument to the converttiffpdfreducer method which is then executed on a background thread.
Consider to add cancellation support to to the longrunning converttiffpdfreducer()
Avoid calling ToArray or ToList on an IEnumerable. Those methods are finalizers that immediately execute the originally deferred LINQ queries.
Don't call ToArray or ToList on the result of EnumerateFiles and its likes. Those methods are used to improve performance as they return the filesystem objects item by item. This is especially important if you recursively iterate over the complete filesystem structure. If the filesystem tree is very deep and broad, calling ToArray will force the itereation to complete and then to return all results at once. ToArray on EnumerateFiles is like using GetFiles. You should review your complete code and refactor it properly. You always use EnumerateXYZ wrong!
"The EnumerateFiles and GetFiles methods differ as follows: When you
use EnumerateFiles, you can start enumerating the collection of names
before the whole collection is returned. When you use GetFiles, you
must wait for the whole array of names to be returned before you can
access the array. Therefore, when you are working with many files and
directories, EnumerateFiles can be more efficient."
Use data binding instead of directly accessing UI elements. This allows you to read the properties on the background thread without using the Dispatcher.
Never execute long running operation from the constructor
Never call async code from the constructor
Always keep object instantiation cheep and fast and without hidden performance/resource costs
Never catch Exception. Always catch a specialized exception type.
Don't use empty catch blocks. Either handle the exception if you can or laet it crash your application to give you a chance to fix bugs. When you swallow exceptions bugs will silently creep into your application. You will have a really hard time to discover them. Logging exceptions is not considered handling - rethrow in such case.
You don't have to close a resource explicitly if you declare the resource using the using statement. The implicit call to Dispose once the instruction pointer leaves the using scope will close the resource automatically.
Implementing all the suggestion will significantly speed up your application.
I have refactored only some of your code to show how to properly use async APIs and Task.Run instead of the BackgroundWorker. I have removed every Dispatcher invocation. Instead of direct access to UI elements in order to read their values from the background thread, I have extracted those values before invoking the concurent method and passed those prefetched values as method arguments. If you would use data binding you could read the property values directly and therefore ommit the method parameters.
The MainWindow should be shown manually from App.xaml.cs to allow asynchronous and longrunning initialization of the instance. For this pattern let the class that requires such initialization implement a public InitializeAsync method that can be awaited from the caller's context. Alternatively use Lazy<T> to defer initialization when required e.g., when initialization is depending on explicit access to members.
Although the refactored code will significantly improve the applicatoin's performance, you will have do to do some important refactoring yourself (following the pattern of the already refactored code sections).
Take a look at
InitializeAsync and WriteLnAsync to learn how to use the async file IO API.
converttiffpdfreducer to learn how to use the EnumerateFiles and EnumerateDirectories methods properly in order to significantly improve the performance.
mutliocr, merge and converttiffpdfreducer to learn how pass UI element values as argument in order to avoid Dispatcher invocations.
start_btn_Click and converttiffpdfreducer to learn how to implemnent cancellation and to guard your API against calls during an uninitialized state
App.xaml
<Application Startup="Application_Startup">
</Application>
App.xaml.cs
class App : Application
{
private async void Application_Startup(object sender, StartupEventArgs e)
{
var mainWindow = new MainWindow();
// Because InitializeAsync depends on UI elements,
// we have to wait until the Ui is loaded.
mainWindow.Loaded += OnMainWindowLoaded;
// Either call Show() before initialization or after.
// If before, ensure access to uninitialized members and resources is denied
// e.g. by querying the MainWindow.IsInitialized property in public members and event handlers.
mainWindow.Show();
}
private async void OnMainWindowLoaded(object sender, EventArgs args)
=> await mainWindow.InitializeAsync();
}
MainWindow.xaml.cs
public partial class MainWindow : Window
{
public bool IsInitialized { get; private set; }
private bool IsBusy { get; set; }
private CancellationTokenSource CancellationTokenSource { get; set; }
public MainWindow()
{
InitializeComponent();
CancellationTokenSource = new CancellationTokenSource();
}
// Execute blocking initialization routines asynchronously
public async Task InitializeAsync()
{
if (IsInitialized)
{
return;
}
// Will execute the intesive CPU bound work on a background thread.
await File_process(Cancellationtoken.None);
string configpath = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, #"path.txt");
// Use async API to read/write from/to files and other IO resources
using (FileStream configfile = File.OpenRead(configpath))
{
using (var fileReader = new StreamReader(configfile))
{
var configFileContent = new List<string>();
while (!fileReader.EndOfStream)
{
string lineOfFile = await fileReader.ReadLineAsync();
configFileContent.Add(lineOfFile);
}
if (configFileContent.Any())
{
ip.Text = configFileContent[0];
GrantAccess(configFileContent[0]);
op.Text = configFileContent[1];
GrantAccess(configFileContent[1]);
ex_tb.Text = configFileContent[2];
GrantAccess(configFileContent[2]);
Protb.Text = configFileContent[3];
GrantAccess(configFileContent[3]);
}
}
}
cbPDFConform.Items.Clear();
for (int i = 0; i < Enum.GetNames(typeof(PdfConformance)).Length - 1; i++)
{
ComboBoxItem cbi = new ComboBoxItem();
cbi.Content = Enum.GetName(typeof(PdfConformance), (PdfConformance)i);
PdfConformance test = (PdfConformance)i;
cbi.Tag = (PdfConformance)i;
cbPDFConform.Items.Add(cbi);
}
cbPDFConform.SelectedIndex = 0;
cbProcessorCount.Items.Clear();
for (int i = 1; i <= Environment.ProcessorCount; i++)
{
cbProcessorCount.Items.Add(i.ToString());
if (Environment.ProcessorCount / 2 == i) { cbProcessorCount.SelectedIndex = i - 1; }
}
LicenseManager oLicenseManager = new LicenseManager();
oLicenseManager.RegisterKEY("");
threadSyn = SynchronizationContext.Current;
IsInitialiezd = true;
}
public async Task File_process(CancellationToken cancellationToken)
{
// Read UI values to avoid Dispatcher calls from the background thread
string ipText = ip.Text;
string protbText = Protb.Text;
string opText = op.Text;
// Execute the intesive CPU bound work on a background thread.
await Task.Run(() => converttiffpdfreducer(ipText, protbText, opText, cancellationToken), cancellationToken);
}
private async Task DoWorkAsync(CancellationToken cancellationToken)
{
IsBusy = true;
using (var sw = new StreamWriter(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, #"path.txt"), false))
{
await sw.WriteLineAsync(ip.Text);
await sw.WriteLineAsync(op.Text);
await sw.WriteLineAsync(ex_tb.Text);
await sw.WriteLineAsync(Protb.Text);
}
try
{
cancellationToken.ThrowIfCancellationRequested();
var watch1 = new System.Diagnostics.Stopwatch();
watch1.Start();
// Consider to add cancellation support to File_process
await File_process(cancellationToken);
watch1.Stop();
TimeSpan ts1 = watch1.Elapsed;
ts1.ToString("mm\\:ss");
if (MergeChk.IsChecked == false)
{
value = "OCRed";
}
await WriteLnAsync("All documents have been successfully " + value + " " + ts1 + " " + DateTime.Now + " " + Environment.UserName, cancellationToken);
IsBusy = false;
}
catch (OperationCanceledException)
{
IsBusy = false;
throw;
}
}
private async Task WriteLnAsync(string text, CancellationToken cancellationToken)
{
logtb.Text += text + Environment.NewLine;
log_list.Add(text);
log_cap = text + Environment.NewLine + log_cap;
using (var sw = new StreamWriter(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, #"FileProcessing_log.txt"), false))
{
foreach (string l in log_list)
{
cancellationToken.ThrowIfCancellationRequested();
await sw.WriteLineAsync(l);
}
}
}
private async void start_btn_Click(object sender, RoutedEventArgs e)
{
if (!IsInitialized)
{
return;
}
if (IsBusy)
{
// Cancel the longrunning operation.
this.CancellationTokenSource.Cancel();
}
start_btn.Content = "Start";
try
{
await DoWorkAsync(CancellationTokenSource.Token);
}
catch (OperationCanceledException)
{
CancellationTokenSource?.Dispose();
CancellationTokenSource = new CancellationTokenSource();
}
}
private void converttiffpdfreducer(
string ipText,
string protbText,
string opText,
CancellationToken cancellationToken)
{
string[] dir = null;
string box = string.Empty;
string box1 = string.Empty;
string[] gg = null;
PdfConformance optPDFConform = PdfConformance.Unknown;
foreach (var directoryPath in Directory.EnumerateDirectories(ipText, "*.*", SearchOption.AllDirectories).Where(l => l.Length != 0))
{
cancellationToken.ThrowIfCancellationRequested();
foreach (var filePath in Directory.EnumerateFiles(directoryPath, "*.*", SearchOption.AllDirectories))
{
cancellationToken.ThrowIfCancellationRequested();
string getext = Path.GetExtension(filePath);
string fd = Path.GetDirectoryName(filePath);
string op_path = fd.Replace(ipText, protbText);
string getextension = Path.GetExtension(filePath);
string dict = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "Redist", "OCR");
string outputPath = fd.Replace(ipText, protbText);
string FNAME = Path.GetFileNameWithoutExtension(filePath);
string fn = Path.GetDirectoryName(filePath).Replace(ipText, protbText);
string filen = Path.Combine(outputPath, fn, FNAME + ".pdf");
string savefile = Path.Combine(op_path, filen);
box = Path.GetDirectoryName(filePath);
box1 = Path.GetDirectoryName(box);
using (GdPictureDocumentConverter oConverter = new GdPictureDocumentConverter())
{
GdPictureStatus status = new GdPictureStatus();
if (Path.GetExtension(filePath).ToUpper() == ".PDF")
{
status = oConverter.LoadFromFile(filePath, GdPicture14.DocumentFormat.DocumentFormatPDF);
}
else if (Path.GetExtension(filePath).ToUpper() == ".TIF" || Path.GetExtension(filePath).ToUpper() == ".TIFF")
{
status = oConverter.LoadFromFile(filePath, GdPicture14.DocumentFormat.DocumentFormatTIFF);
}
else if (Path.GetExtension(filePath).ToUpper() == ".JPG")
{
status = oConverter.LoadFromFile(filePath, GdPicture14.DocumentFormat.DocumentFormatJPEG);
}
if (status == GdPictureStatus.OK)
{
if (!Directory.Exists(op_path))
{
Directory.CreateDirectory(op_path);
}
GrantAccess(op_path);
optPDFConform = (PdfConformance)((ComboBoxItem)cbPDFConform.SelectedItem).Tag;
status = oConverter.SaveAsPDF(savefile, optPDFConform);
if (status == GdPictureStatus.OK)
{ }
else
{ }
}
else
{ }
}
}
}
string BOXX = box.Replace(ipText, protbText);
// TODO::Refactor 'merge' and replace 'ToArray' with 'foreach'
string[] Arr = Directory.EnumerateFiles(BOXX, "*.pdf", SearchOption.AllDirectories).ToArray();
if (MergeChk.IsChecked == true)
{ merge(Arr, protbText); }
else if (MergeChk.IsChecked == false)
{
mutliocr(Arr);
}
}
private string[] merge(string[] arr, string protbText, string opText)
{
string box = string.Empty;
string box1 = string.Empty; string[] gg = null;
System.Windows.Threading.Dispatcher.CurrentDispatcher.Invoke((Action)(() =>
{
box = Path.GetDirectoryName(arr[0]);
box1 = Path.GetDirectoryName(box);
string dirName = Directory.GetParent(arr[0]).FullName;
string BOXFILES = Path.GetDirectoryName(dirName);
string folder = Directory.GetParent(arr[0]).FullName.Replace(protbText, opText);
string ocrfolder = (new FileInfo(arr[0]).Directory.FullName).Replace(protbText, opText);
string fn = Directory.GetParent(arr[0]).Name;
string filen = Path.Combine(ocrfolder, folder, fn + ".pdf");
if (!Directory.Exists(ocrfolder))
{
Directory.CreateDirectory(ocrfolder);
}
GrantAccess(ocrfolder);
using (GdPicturePDF oGdPicturePDF = new GdPicturePDF())
{
GdPictureStatus status = oGdPicturePDF.MergeDocuments(ref arr, filen);
if (status == GdPictureStatus.OK)
{ }
else
{ }
oGdPicturePDF.Dispose();
}
Directory.Delete(box, true);
string BOXX = box.Replace(protbText, opText);//op
string[] files = Directory.EnumerateFiles(BOXX, "*.pdf", SearchOption.AllDirectories).ToArray();
if (MergeChk.IsChecked == true)
{ mutliocr(files, protbText, opText); }
}));
return gg;
}
private string[] mutliocr(string[] arr, string protbText, string opText)
{
string box = string.Empty;
string box1 = string.Empty;
try
{
string filepath = string.Empty;
if (MergeChk.IsChecked == true)
{ filepath = opText; }
else if (MergeChk.IsChecked == false)
{ filepath = protbText; }
System.Windows.Threading.Dispatcher.CurrentDispatcher.Invoke((Action)(() =>
{
Thread.CurrentThread.IsBackground = true;
var watch2 = new System.Diagnostics.Stopwatch();
watch2.Start();
string[] getfilearray = arr;
for (int f = 0; f < getfilearray.Length; f++)
{
string dirName = Directory.GetParent(getfilearray[f]).FullName;
string folder = Directory.GetParent(getfilearray[f]).FullName;
box = Path.GetDirectoryName(getfilearray[f]);
box1 = Path.GetDirectoryName(box);
string getextension = Path.GetExtension(getfilearray[f]);
string[] newF = Directory.EnumerateFiles(dirName, "*.*", SearchOption.AllDirectories).ToArray();
string FN = Directory.GetParent(getfilearray[f]).Name;
string ocrfolder = (new FileInfo(getfilearray[f]).Directory.FullName);
string filen = Path.Combine(ocrfolder, folder, FN + "-ocr" + getextension);
string dict = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "Redist", "OCR");
if (!Directory.Exists(ocrfolder))
{
Directory.CreateDirectory(ocrfolder);
}
GrantAccess(ocrfolder);
GdPicturePDF oGdPicturePDF = new GdPicturePDF();
oGdPicturePDF.OcrPagesDone += OcrPagesDone;
void OcrPagesDone(GdPictureStatus status1)
{
if (oGdPicturePDF.SaveToFile(filen) == GdPictureStatus.OK)
{ }
else
MessageBox.Show("PDF: The OCR-ed file has failed to save. Status: " + oGdPicturePDF.GetStat().ToString());
}
GdPictureStatus status = GdPictureStatus.OK;
if (oGdPicturePDF.LoadFromFile(getfilearray[f], false) == GdPictureStatus.OK)
if (status == GdPictureStatus.OK)
{
if (oGdPicturePDF.OcrPages_4("*", 0, "eng", dict, "", 300, OCRMode.FavorSpeed, 1, true) == GdPictureStatus.OK)
if (status == GdPictureStatus.OK)
{ }
else
{ MessageBox.Show("PDF: The OCR process has failed. Status: " + status.ToString()); }
}
else
{ MessageBox.Show("PDF: The PDF file has failed to load. Status: " + status.ToString()); }
oGdPicturePDF.Dispose();
GrantAccess(getfilearray[f]);
File.Delete(getfilearray[f]);
watch2.Stop();
TimeSpan ts2 = watch2.Elapsed;
ts2.ToString("mm\\:ss");
WriteLn(" OCR pages " + filen.Replace(opText, "") + " " + ts2 + " " + DateTime.Now);
}
if (MergeChk.IsChecked == true)
{
foreach (string str in Directory.EnumerateFiles(opText, "*.pdf", SearchOption.AllDirectories).ToArray())
{
if (Path.GetFileNameWithoutExtension(str).EndsWith("-ocr"))
File.Move(str, Path.Combine(Path.GetDirectoryName(str), Path.GetFileNameWithoutExtension(str).Substring(0, Path.GetFileNameWithoutExtension(str).Length - 4) + ".pdf"));
}
}
if (MergeChk.IsChecked == false)
{
FileSystem.MoveDirectory(protbText, opText, UIOption.AllDialogs);
Directory.CreateDirectory(protbText);
string FF = string.Empty;
foreach (string str in Directory.EnumerateFiles(opText, "*.pdf", SearchOption.AllDirectories))
{
if (Path.GetFileNameWithoutExtension(str).EndsWith("-ocr"))
File.Move(str, Path.Combine(Path.GetDirectoryName(str), Path.GetFileNameWithoutExtension(str).Substring(0, Path.GetFileNameWithoutExtension(str).Length - 4) + ".pdf"));
}
}
}));
}
catch (Exception mul)
{
}
return arr;
}
}
I'm helping a colleague debug a tool he is creating for ArcGis. It is used to create Replicas, and is done through selecting some different inputs from dropdowns in a winform.
The issue is that when running all the code on the UI-thread, our ui freezes. This is what he wants me to solve. The code causing this is the following, the relevant code is just the button1_Click_1() method, but i provided the second class for contex:
public partial class FrmReplicaAdmin : Form
{
private void button1_Click_1(object sender, EventArgs e)
{
DataConnectionConfig selectedDatabase = cmboxDatabase.SelectedItem as DataConnectionConfig;
if (selectedDatabase.PlWorkspace == null)
{
statusLabel.Text = "Could not open GeoDatabase (PL)";
return;
}
if (selectedDatabase.DataWorkspace == null)
{
statusLabel.Text = "Could not open GeoDatabase (NIS)";
return;
}
int scaleBand = int.Parse(cmboxScale.SelectedItem.ToString());
string gridName = cmboxGridNr.SelectedItem as string;
IGeometry shapeOfSelectedAOI = getSelectedPolygon(gridName, scaleBand, selectedDatabase);
(ArcMap.Application as IMxApplication2).PauseDrawing = true;
replica.CheckOutReplica(selectedDatabase.PlWorkspace, selectedDatabase.DataWorkspace, selectedDatabase.TemplateGdb, gridName, scaleBand, shapeOfSelectedAOI, selectedDatabase.DatasetName);
(ArcMap.Application as IMxApplication2).PauseDrawing = false;
}
}
public class Replica
{
public void CheckOutReplica(IWorkspace plWorkspace, IWorkspace parentWorkspace, string pathToDatabaseTemplate, string gridName, int scaleBand, IGeometry shapeOfSelectedAOI, string datasetName = "NIS.Nautical")
{
try
{
string replicaName = string.Format("{0}_{1}_r", Environment.UserName, gridName);
string versionName = string.Format("{0}_{1}", Environment.UserName, gridName);
string pathToLocalChildDatabase = System.IO.Path.Combine(ReplicationPath, $"{replicaName}.gdb");
Directory.CreateDirectory(pathToLocalChildDatabase);
foreach (string newPath in Directory.GetFiles(pathToDatabaseTemplate, "*.*", SearchOption.AllDirectories))
File.Copy(newPath, newPath.Replace(pathToDatabaseTemplate, pathToLocalChildDatabase), true);
IWorkspace childWorkspace = OpenWorkspace(pathToLocalChildDatabase);
// Create Version in ParentDatabase
IEnumVersionInfo versionEnum = (parentWorkspace as IVersionedWorkspace).Versions;
versionEnum.Reset();
for (IVersionInfo versionInfo = versionEnum.Next(); versionInfo != null; versionInfo = versionEnum.Next())
{
if (versionInfo.VersionName.EndsWith(versionName))
{
System.Windows.Forms.MessageBox.Show("A version named '" + versionName + "' has already been created", "map...", System.Windows.Forms.MessageBoxButtons.OK);
return;
}
}
Marshal.ReleaseComObject(versionEnum);
IVersion newVersion = (parentWorkspace as IVersionedWorkspace).DefaultVersion.CreateVersion(versionName);
newVersion.Access = esriVersionAccess.esriVersionAccessPublic;
string defQuery = "((IS_CONFLATE=1 AND PLTS_COMP_SCALE >= " + scaleBand + ") OR ((IS_CONFLATE=0 OR IS_CONFLATE IS NULL) AND PLTS_COMP_SCALE = " + scaleBand + "))";
ReplicateData(parentWorkspace, newVersion.VersionInfo, replicaName, shapeOfSelectedAOI, childWorkspace, defQuery, datasetName);
// Update map. Show replica data
ILayerFile nauticalLyrFile = new LayerFileClass();
nauticalLyrFile.Open(ReplicationPath + #"\Nautical.lyr");
AddDataToMap((ArcMap.Application.Document as IMxDocument), childWorkspace as IFeatureWorkspace, nauticalLyrFile, gridName, datasetName);
(ArcMap.Application.Document as IMxDocument).ActiveView.Extent = shapeOfSelectedAOI.Envelope;
Marshal.ReleaseComObject(childWorkspace);
}
catch (Exception err)
{
System.Windows.Forms.MessageBox.Show($"Unexpected error. {Environment.NewLine}{err.Message}", "map...", System.Windows.Forms.MessageBoxButtons.OK);
}
}
}
private void ReplicateData(IWorkspace parentWorkspace, IVersionInfo versionInfo, string replicaName, IGeometry area, IWorkspace childWorkspace, string definitionQuery, string featureDataset)
{
if (childWorkspace == null)
throw new ArgumentNullException("Child workspace is null.");
if (parentWorkspace == null)
throw new ArgumentNullException("Parent workspace is null.");
if (versionInfo == null)
throw new ArgumentNullException("Version name is null.");
if (string.IsNullOrEmpty(replicaName))
throw new ArgumentNullException("Replica name is null.");
if (area == null)
throw new ArgumentNullException("Area geometry is null.");
IVersion oVersion = (parentWorkspace as IVersionedWorkspace).FindVersion(versionInfo.VersionName);
IWorkspace sdeVersionWorkspace = oVersion as IWorkspace;
IGeoDataServer parentGds = InitGeoDataServer(sdeVersionWorkspace),
childGds = InitGeoDataServer(childWorkspace);
CreateFeatureDatasetReplica(parentGds, childGds, versionInfo, replicaName, parentWorkspace, childWorkspace, area, definitionQuery, featureDataset);
Marshal.ReleaseComObject(parentGds);
Marshal.ReleaseComObject(childGds);
}
//Function to create the replica, based on this link http://help.arcgis.com/en/sdk/10.0/arcobjects_net/conceptualhelp/index.html#//0001000003r5000000
private void CreateFeatureDatasetReplica(IGeoDataServer parentGDS, IGeoDataServer childGDS, IVersionInfo versionInfo, string replicaName, IWorkspace parentWorkspace, IWorkspace childWorkspace, IGeometry geometry, string definitionQuery, string featureDatasetName)
{
IList<string> existingReplicas = ReadExistingReplicas(parentGDS);
if (existingReplicas.Contains(replicaName.ToUpper()))
{
throw new Exception("A replica with the following name has already been created: " + replicaName);
}
IEnumDataset datasets = null;
if (!string.IsNullOrEmpty(featureDatasetName))
{
IEnumDataset featureDatasets = parentWorkspace.get_Datasets(esriDatasetType.esriDTFeatureDataset);
IFeatureDataset featureDataset;
while ((featureDataset = featureDatasets.Next() as IFeatureDataset) != null)
{
if (featureDataset.Name == featureDatasetName)
{
datasets = featureDataset.Subsets;
break;
}
}
if (datasets == null)
throw new Exception("Didn't find FeatureDataset " + featureDatasetName + " in the db");
}
else
{
datasets = parentWorkspace.get_Datasets(esriDatasetType.esriDTFeatureClass);
}
IGPReplicaDatasets gpReplicaDatasets = new GPReplicaDatasetsClass();
IDataset dataset;
while ((dataset = datasets.Next()) != null)
{
//temporary workaround to not include a view that is on the feature classes :^)
if (dataset.Name.Contains("VW_") || dataset.Name.Contains("_EVW"))
continue;
if (m_ListExcludedTables.Contains(dataset.Name.Substring(dataset.Name.LastIndexOf(".") + 1).ToUpper()))
continue;
if (!(childWorkspace as IWorkspace2).NameExists[dataset.Type, dataset.Name.Substring(dataset.Name.LastIndexOf(".") + 1)])
continue;
IGPReplicaDataset gpReplicaDataset = new GPReplicaDatasetClass();
gpReplicaDataset.DatasetType = dataset.Type;
gpReplicaDataset.Name = dataset.Name.ToUpper();
gpReplicaDataset.IsPrivate = false;
gpReplicaDataset.UseGeometry = true;
gpReplicaDataset.RowsType = esriRowsType.esriRowsTypeFilter;
if ((dataset as ITable).Fields.FindField("PLTS_COMP_SCALE") != -1)
gpReplicaDataset.DefQuery = definitionQuery; //DefQuery here
else
gpReplicaDataset.DefQuery = "";
gpReplicaDatasets.Add(gpReplicaDataset);
}
IGPReplicaDescription gpReplicaDesc = new GPReplicaDescriptionClass();
gpReplicaDesc.QueryGeometry = geometry;
gpReplicaDesc.SpatialRelation = esriSpatialRelEnum.esriSpatialRelIntersects;
gpReplicaDesc.ModelType = esriReplicaModelType.esriModelTypeSimple;
gpReplicaDesc.SingleGeneration = true;
gpReplicaDesc.ReplicaDatasets = gpReplicaDatasets;
IGPReplicaOptions2 replicaOptions = new GPReplicaOptionsClass();
replicaOptions.AccessType = esriReplicaAccessType.esriReplicaAccessNone;
replicaOptions.RegisterReplicaOnly = true;
ExtractData(datasets, childWorkspace, geometry, definitionQuery);
IReplicationAgent replicationAgent = new ReplicationAgentClass();
replicationAgent.CreateReplica(versionInfo.VersionName, parentGDS, childGDS, replicaName, gpReplicaDesc, replicaOptions);
}
}
For fixing the UI freeze i made the following changes to the button1_Click_1() method:
public partial class FrmReplicaAdmin : Form
{
private void button1_Click_1(object sender, EventArgs e)
{
DataConnectionConfig selectedDatabase = cmboxDatabase.SelectedItem as DataConnectionConfig;
if (selectedDatabase.PlWorkspace == null)
{
statusLabel.Text = "Could not open GeoDatabase (PL)";
return;
}
if (selectedDatabase.DataWorkspace == null)
{
statusLabel.Text = "Could not open GeoDatabase (NIS)";
return;
}
int scaleBand = int.Parse(cmboxScale.SelectedItem.ToString());
string gridName = cmboxGridNr.SelectedItem as string;
IGeometry shapeOfSelectedAOI = getSelectedPolygon(gridName, scaleBand, selectedDatabase);
// adding inputs to list i can pass onto the backgroundWorker
List<object> arguments = new List<object>();
arguments.Add(selectedDatabase.PlWorkspace);
arguments.Add(selectedDatabase.DataWorkspace);
arguments.Add(selectedDatabase.TemplateGdb);
arguments.Add(gridName);
arguments.Add(scaleBand);
arguments.Add(shapeOfSelectedAOI);
arguments.Add(selectedDatabase.DatasetName);
backgroundWorker1.RunWorkerAsync(arguments);
// starting progress bar
progressBarReplica.Visible = true;
lblReplica.Text = "Checking out replica...";
}
private void backgroundWorker1_DoWork(object sender, DoWorkEventArgs e)
{
List<object> genericlist = e.Argument as List<object>;
IWorkspace ws = (IWorkspace)genericlist[0];
IWorkspace pWs = (IWorkspace)genericlist[1];
string pathToDbTemplate = genericlist[2].ToString();
string gName = genericlist[3].ToString();
int sBand = (int)genericlist[4];
IGeometry shape = (IGeometry)genericlist[5];
string dsName = genericlist[6].ToString();
(ArcMap.Application as IMxApplication2).PauseDrawing = true;
replica.CheckOutReplica(ws, pWs, pathToDbTemplate, gName, sBand, shape, dsName);
(ArcMap.Application as IMxApplication2).PauseDrawing = false;
}
}
This is causing a: "RPC_E_SERVERFAULT(0X80010105)", but the UI isn't freezing anymore. My guess is that it's because i'm initiating the database in the first thread, and then using it in the second. I also sort of get that i can't use a backgroundWorker due to the entire STA and COM-object things with ArcGis, but i'm still not getting all this 100%.
Any help of a possible solution to making my UI responsive, or at least showing some sort of progressbar while the task is running would be nice. The entire process can take a few minutes at times, and the program feels like it's crashed meanwhile due to the freeze.
Edit: I'm referring to ArcMap, forgot to mention that.
Using .NET 4, wpf c#, I am passing method return values and parameters between two processes.
As I need the connection open and active a all times, I have tried my best to minimize the code that is recurring (within the loop) but unless I put this whole code inside the loop it did not succeed (after the first transfer the connection to server was closed), so as it is here, it does work repeatedly.
I was wondering first is this the way it should be coded, all the process including the new instance, dispose, close... within the loop?
Is the only available datatype for inter-process communication to pass as a string (inefficient)?
public void client()
{
for (int i = 0; i < 2; i++)
{
System.IO.Pipes.NamedPipeClientStream pipeClient =
new System.IO.Pipes.NamedPipeClientStream(".", "testpipe",
System.IO.Pipes.PipeDirection.InOut, System.IO.Pipes.PipeOptions.None);
if (pipeClient.IsConnected != true)
{
pipeClient.Connect(550);
}
System.IO.StreamReader sr = new System.IO.StreamReader(pipeClient);
System.IO.StreamWriter sw = new System.IO.StreamWriter(pipeClient);
string status;
status = sr.ReadLine();
if (status == "Waiting")
{
try
{
sw.WriteLine("param1fileName.cs,33" + i);
sw.Flush();
pipeClient.Close();
}
catch (Exception ex) { throw ex; }
}
}
}
public string server()
{
NamedPipeServerStream pipeServer = null;
do
{
try
{
pipeServer = new NamedPipeServerStream("testpipe", PipeDirection.InOut, 4);
StreamReader sr = new StreamReader(pipeServer);
StreamWriter sw = new StreamWriter(pipeServer);
System.Threading.Thread.Sleep(100);
pipeServer.WaitForConnection();
string test;
sw.WriteLine("Waiting");
sw.Flush();
pipeServer.WaitForPipeDrain();
test = sr.ReadLine();
if (!string.IsNullOrEmpty(test))
try
{
System.Windows.Application.Current.Dispatcher.Invoke(new Action(() => MbxTw.Show(Convert.ToInt32(test.Split(',')[1]), test.Split(',')[0], "method()", "Warning!! - " + "content")), System.Windows.Threading.DispatcherPriority.Normal);
}
catch (Exception e)
{
}
}
catch (Exception ex) {
throw ex; }
finally
{
pipeServer.WaitForPipeDrain();
if (pipeServer.IsConnected) { pipeServer.Disconnect(); }
}
} while (true);
}
after a thorough research on inter process communication I have changed the approach from using Named pipes to Memory mapped files,
as it is all rounds winner, no need to recreate it and faster
I present the ultimate partitioned-global-app-inter-process communication
any thoughts on the code will be greatly appreciated!
public class MMFinterComT
{
public EventWaitHandle flagCaller1, flagCaller2, flagReciver1, flagReciver2;
private System.IO.MemoryMappedFiles.MemoryMappedFile mmf;
private System.IO.MemoryMappedFiles.MemoryMappedViewAccessor accessor;
public virtual string DepositChlName { get; set; }
public virtual string DepositThrdName { get; set; }
public virtual int DepositSize { get; set; }
private System.Threading.Thread writerThread;
private bool writerThreadRunning;
public int ReadPosition { get; set; }
public List<string> statusSet;
private int writePosition;
public int WritePosition
{
get { return writePosition; }
set
{
if (value != writePosition)
{
this.writePosition = value;
this.accessor.Write(WritePosition + READ_CONFIRM_OFFSET, true);
}
}
}
private List<byte[]> dataToSend;
private const int DATA_AVAILABLE_OFFSET = 0;
private const int READ_CONFIRM_OFFSET = DATA_AVAILABLE_OFFSET + 1;
private const int DATA_LENGTH_OFFSET = READ_CONFIRM_OFFSET + 1;
private const int DATA_OFFSET = DATA_LENGTH_OFFSET + 10;
public IpcMMFinterComSF.MMFinterComTStatus IntercomStatus;
public MMFinterComT(string ctrIpcChannelNameStr, string ctrIpcThreadName, int ctrMMFSize)
{
this.DepositChlName = ctrIpcChannelNameStr;
this.Deposit Size = ctrMMFSize;
this.DepositThrdName = ctrIpcThreadName;
mmf = MemoryMappedFile.CreateOrOpen(DepositChlName, DepositSize);
accessor = mmf.CreateViewAccessor(0, DepositSize, System.IO.MemoryMappedFiles.MemoryMappedFileAccess.ReadWrite);//if (started)
//smLock = new System.Threading.Mutex(true, IpcMutxName, out locked);
ReadPosition = -1;
writePosition = -1;
this.dataToSend = new List<byte[]>();
this.statusSet = new List<string>();
}
public bool reading;
public byte[] ReadData;
public void StartReader()
{
if (this.IntercomStatus != IpcMMFinterComSF.MMFinterComTStatus._Null || ReadPosition < 0 || writePosition < 0)
return;
this.IntercomStatus = IpcMMFinterComSF.MMFinterComTStatus.PreparingReader;
System.Threading.Thread t = new System.Threading.Thread(ReaderThread);
t.IsBackground = true;
t.Start();
}
private void ReaderThread(object stateInfo)
{
// Checks if there is something to read.
this.IntercomStatus = IpcMMFinterComSF.MMFinterComTStatus.TryingToRead;
this.reading = accessor.ReadBoolean(ReadPosition + DATA_AVAILABLE_OFFSET);
if (this.reading)
{
this.IntercomStatus = IpcMMFinterComSF.MMFinterComTStatus.ReadingData;
// Checks how many bytes to read.
int availableBytes = accessor.ReadInt32(ReadPosition + DATA_LENGTH_OFFSET);
this.ReadData = new byte[availableBytes];
// Reads the byte array.
int read = accessor.ReadArray<byte>(ReadPosition + DATA_OFFSET, this.ReadData, 0, availableBytes);
// Sets the flag used to signal that there aren't available data anymore.
accessor.Write(ReadPosition + DATA_AVAILABLE_OFFSET, false);
// Sets the flag used to signal that data has been read.
accessor.Write(ReadPosition + READ_CONFIRM_OFFSET, true);
this.IntercomStatus = IpcMMFinterComSF.MMFinterComTStatus.FinishedReading;
}
else this.IntercomStatus = IpcMMFinterComSF.MMFinterComTStatus._Null;
}
public void Write(byte[] data)
{
if (ReadPosition < 0 || writePosition < 0)
throw new ArgumentException();
this.statusSet.Add("ReadWrite:-> " + ReadPosition + "-" + writePosition);
lock (this.dataToSend)
this.dataToSend.Add(data);
if (!writerThreadRunning)
{
writerThreadRunning = true;
writerThread = new System.Threading.Thread(WriterThread);
writerThread.IsBackground = true;
writerThread.Name = this.DepositThrdName;
writerThread.Start();
}
}
public void WriterThread(object stateInfo)
{
while (dataToSend.Count > 0 && !this.disposed)
{
byte[] data = null;
lock (dataToSend)
{
data = dataToSend[0];
dataToSend.RemoveAt(0);
}
while (!this.accessor.ReadBoolean(WritePosition + READ_CONFIRM_OFFSET))
System.Threading.Thread.Sleep(133);
// Sets length and write data.
this.accessor.Write(writePosition + DATA_LENGTH_OFFSET, data.Length);
this.accessor.WriteArray<byte>(writePosition + DATA_OFFSET, data, 0, data.Length);
// Resets the flag used to signal that data has been read.
this.accessor.Write(writePosition + READ_CONFIRM_OFFSET, false);
// Sets the flag used to signal that there are data avaibla.
this.accessor.Write(writePosition + DATA_AVAILABLE_OFFSET, true);
}
writerThreadRunning = false;
}
public virtual void Close()
{
if (accessor != null)
{
try
{
accessor.Dispose();
accessor = null;
}
catch { }
}
if (this.mmf != null)
{
try
{
mmf.Dispose();
mmf = null;
}
catch { }
}
disposed = true;
GC.SuppressFinalize(this);
}
private bool disposed;
}
and usage
instaciant once !
public static bool StartCurProjInterCom(IpcAccessorSetting curSrv, int DepoSize)
{
if(CurProjMMF ==null)
CurProjMMF = new MMFinterComT(curSrv.Channel.ToString(), curSrv.AccThreadName.ToString(), DepoSize);
CurProjMMF.flagCaller1 = new EventWaitHandle(false, EventResetMode.ManualReset, CurProjMMF.DepositThrdName);
CurProjMMF.flagCaller2 = new EventWaitHandle(false, EventResetMode.ManualReset, CurProjMMF.DepositThrdName);
CurProjMMF.flagReciver1 = new EventWaitHandle(false, EventResetMode.ManualReset, IpcAccessorThreadNameS.DebuggerThrd.ToString());
CurProjMMF.ReadPosition = curSrv.AccessorSectorsSets.DepoSects.Setter.Read;
CurProjMMF.WritePosition = curSrv.AccessorSectorsSets.DepoSects.Setter.Write;
Console.WriteLine("MMFInterComSetter.ReadPosition " + CurProjMMF.ReadPosition);
Console.WriteLine("MMFInterComSetter.WritePosition " + CurProjMMF.WritePosition);
CurProjMMF.StartReader();
return true;
}
use many
public static void StartADebugerInterComCall(IpcCarier SetterDataObj)
{
IpcAccessorSetting curSrv = new IpcAccessorSetting(IpcMMf.IPChannelS.Debugger, IpcAccessorThreadNameS.DebuggerThrdCurProj, 0, 5000);
StartCurProjInterCom(curSrv, 10000);
var dataW = SetterDataObj.IpcCarierToByteArray();//System.Text.Encoding.UTF8.GetBytes(msg);
CurProjMMF.Write(dataW);
CurProjMMF.flagReciver1.Set();
CurProjMMF.flagCaller1.WaitOne();
CurProjMMF.flagCaller1.Reset();
}
I want to free a TCP port during startup of my application (asking confirmation to user), how to get the PID number and then, if the user confirm, kill it?
I know I can get this information by netstat, but how to do it in a script or better in a C# method.
You can run netstat then redirect the output to a text stream so you can parse and get the info you want.
Here is what i did.
Run netstat -a -n -o as a Process
redirect the standard out put and capture the output text
capture the result, parse and return all the processes in use
check if the port is being used
find the process using linq
Run Process.Kill()
you will have to do the exception handling.
namespace test
{
static class Program
{
/// <summary>
/// The main entry point for the application.
/// </summary>
static void Main()
{
Console.WriteLine("Port number you want to clear");
var input = Console.ReadLine();
//var port = int.Parse(input);
var prc = new ProcManager();
prc.KillByPort(7972); //prc.KillbyPort(port);
}
}
public class PRC
{
public int PID { get; set; }
public int Port { get; set; }
public string Protocol { get; set; }
}
public class ProcManager
{
public void KillByPort(int port)
{
var processes = GetAllProcesses();
if (processes.Any(p => p.Port == port))
try{
Process.GetProcessById(processes.First(p => p.Port == port).PID).Kill();
}
catch(Exception ex)
{
Console.WriteLine(ex.Message);
}
else
{
Console.WriteLine("No process to kill!");
}
}
public List<PRC> GetAllProcesses()
{
var pStartInfo = new ProcessStartInfo();
pStartInfo.FileName = "netstat.exe";
pStartInfo.Arguments = "-a -n -o";
pStartInfo.WindowStyle = ProcessWindowStyle.Maximized;
pStartInfo.UseShellExecute = false;
pStartInfo.RedirectStandardInput = true;
pStartInfo.RedirectStandardOutput = true;
pStartInfo.RedirectStandardError = true;
var process = new Process()
{
StartInfo = pStartInfo
};
process.Start();
var soStream = process.StandardOutput;
var output = soStream.ReadToEnd();
if(process.ExitCode != 0)
throw new Exception("somethign broke");
var result = new List<PRC>();
var lines = Regex.Split(output, "\r\n");
foreach (var line in lines)
{
if(line.Trim().StartsWith("Proto"))
continue;
var parts = line.Split(new char[]{' '}, StringSplitOptions.RemoveEmptyEntries);
var len = parts.Length;
if(len > 2)
result.Add(new PRC
{
Protocol = parts[0],
Port = int.Parse(parts[1].Split(':').Last()),
PID = int.Parse(parts[len - 1])
});
}
return result;
}
}
}
I'm writing a function which creates a Process and runs it.
Here is the original function:
public static void LaunchApplication(string runCommand, string commandArguments, string workingDir, string informMessage)
{
try
{
Process applicationProcess = new Process();
applicationProcess.StartInfo.FileName = runCommand;
applicationProcess.StartInfo.WorkingDirectory = workingDir;
applicationProcess.StartInfo.Arguments = commandArguments;
applicationProcess.StartInfo.UseShellExecute = false;
WriteHeadlineToConsole(informMessage);
applicationProcess.Start();
while (Process.GetProcesses().Any(runningProcess => runningProcess.Id == applicationProcess.Id)) { }
WriteHeadlineToConsole("Finished " + informMessage);
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
return;
}
}
I would like to change the functions' signature to something like:
public static void LaunchApplication(string runCommand, string commandArguments, string workingDir, string informMessage, Func<object, DataReceivedEventArgs> myMethodName)
And then inside the function, redirect the output, something like:
applicationProcess.OutputDataReceived += new DataReceivedEventHandler(myMethodName);
Can this be done?
Try this:
public static void LaunchApplication(string runCommand, string commandArguments, string workingDir, string informMessage, Action<string> redirectOutput, Action<string> redirectError)
{
try
{
Process applicationProcess = new Process();
applicationProcess.StartInfo.FileName = runCommand;
applicationProcess.StartInfo.WorkingDirectory = workingDir;
applicationProcess.StartInfo.Arguments = commandArguments;
applicationProcess.StartInfo.UseShellExecute = false;
applicationProcess.EnableRaisingEvents = true;
applicationProcess.StartInfo.CreateNoWindow = true;
if (redirectOutput != null)
applicationProcess.OutputDataReceived += (s, data) => redirectOutput(data.Data);
if (redirectError != null)
applicationProcess.ErrorDataReceived += (sErr, errData) => redirectError(errData.Data);
GeneralUtils.WriteHeadlineToConsole(informMessage);
applicationProcess.Start();
if (redirectOutput != null)
applicationProcess.BeginOutputReadLine();
if (redirectError != null)
applicationProcess.BeginErrorReadLine();
applicationProcess.WaitForExit();
GeneralUtils.WriteHeadlineToConsole("Finished " + informMessage);
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
return;
}
}
Usage:
LaunchApplication("cmd.exe", "/C ipconfig /all", "C:\\Windows", "Test",
s => Console.WriteLine("HELLO: " + s));
PS:
The following code is awful! AccessDeniedException might be thrown, CPU is going crazy!
while (Process.GetProcesses().Any(runningProcess => runningProcess.Id == applicationProcess.Id)) { }
Use applicationProcess.WaitForExit(); instead!