Develop Reference

Can you check python code in csharp before pythonNet execution?

i have recently started using pythonNet for executing scripts from Csharp, on an algorithm i was doing in csharp up until now, it works pretty well:
using (Py.GIL())
{
PythonEngine.Initialize();
using (var scope = Py.CreateScope())
{
string code = File.ReadAllText(fileName);
var scriptCompiled = PythonEngine.Compile(code, "Analyze.py");
scope.Execute(scriptCompiled);
dynamic func = scope.Get("predictFromData");
PyList Pydata = new PyList(data.ToPython());
PyTuple rettp = new PyTuple(func(Pydata));
PyList pyIndexList = new PyList(rettp[0]);
foreach (PyObject intobj in pyIndexList)
{
indexList.Add(intobj.As<int>());
}
}
}
But i'd like to know if there is a way to check if the code can be executed before actually running it, since it works with compiled code, and since PythonNet does require an external python installation to see if every modules are here ect... And then switch back to my previous csharp algorithm if it is not possible in python.
For now i'm thinking about simply executing a python unit test importing modules and testing functions with dummy values and returning exceptions and units tests values to csharp code, but i'd prefer a cleaner way if anyone has an idea.
Cheers.

There are few things you can check here:
first is to see if Python code has correct syntax, it can be done with the code like this:
public static IReadOnlyList<ScriptCompilationDiagnostic> CheckErrors(ScriptEngine engine, string script, string fileName, RunFlagType mode)
{
try
{
PythonEngine.Compile(script, fileName, mode);
}
catch (PythonException e)
{
dynamic error = e.Value;
return new[]
{
new ScriptCompilationDiagnostic
{
Kind = ScriptCompilationDiagnosticKind.Error,
Line = error.lineno - 1,
Column = error.offset - 1,
Message = error.msg,
Code = error.text,
FileName = error.filename,
},
};
}
return new ScriptCompilationDiagnostic[0];
}
second is that you can check if Python is installed on a target machine, with the code like this:
var pythonHome = TryGetFullPathFromPathEnvironmentVariable("python.exe");
private static string? TryGetFullPathFromPathEnvironmentVariable(string fileName)
{
if (fileName.Length >= MAXPATH)
throw new ArgumentException($"The executable name '{fileName}' must have less than {MAXPATH} characters.", nameof(fileName));
var sb = new StringBuilder(fileName, MAXPATH);
return PathFindOnPath(sb, null) ? sb.ToString() : null;
}
[DllImport("shlwapi.dll", CharSet = CharSet.Unicode, SetLastError = false)]
private static extern bool PathFindOnPath([In, Out] StringBuilder pszFile, [In] string[]? ppszOtherDirs);
If your script is using third-party modules, you may check that they're installed as well:
public bool IsModuleInstalled(string module)
{
string moduleDir = Path.Combine(PythonHome, "Lib", "site-packages", module);
return Directory.Exists(moduleDir) && File.Exists(Path.Combine(moduleDir, "__init__.py"));
}
Please note that Python.NET does not officially support the latest Python version 3.9, so alternatively you can distribute and install embedded python with your application from here:
https://www.python.org/ftp/python/3.7.3/
alongside with all required third-party modules as wheels.
We use this approach in our AlterNET Studio product to check if Python is installed for our Python debugger based on Debug Adapter Protocol, and install embedded Python with wheels for our Python.NET based scripter/debugger.

How to build multiple c# projects using MSBuild engine without using command prompt?

I am working on windows application project and from that project want to build different multiple c# projects which are in one solution of visual studio 2015 and also want them to be build programmatically individually using MSBuild tool without using command prompt and finally want to show the output in log file not in command prompt (means those project is building successfully or having any errors like this message in log file)
Do I need to use any MSBuild API and how to add in this project?
I have seen many questions like this (not exactly same) but it didn't work for me. please can anybody help me with this?

using Microsoft.Build.Evaluation;
using Microsoft.Build.Execution;
using Microsoft.Build.Logging;
...
public static BuildResult Compile(string solution_name, out string buildLog)
{
buildLog = "";
string projectFilePath = solution_name;
ProjectCollection pc = new ProjectCollection();
Dictionary<string, string> globalProperty = new Dictionary<string, string>();
globalProperty.Add("nodeReuse", "false");
BuildParameters bp = new BuildParameters(pc);
bp.Loggers = new List<Microsoft.Build.Framework.ILogger>()
{
new FileLogger() {Parameters = #"logfile=buildresult.txt"}
};
BuildRequestData buildRequest = new BuildRequestData(projectFilePath, globalProperty, "4.0",
new string[] {"Clean", "Build"}, null);
BuildResult buildResult = BuildManager.DefaultBuildManager.Build(bp, buildRequest);
BuildManager.DefaultBuildManager.Dispose();
pc = null;
bp = null;
buildRequest = null;
if (buildResult.OverallResult == BuildResultCode.Success)
{
Console.ForegroundColor = ConsoleColor.Green;
}
else
{
if (Directory.Exists("C:\\BuildResults") == false)
{
Directory.CreateDirectory("C:\\BuildResults");
}
buildLog = File.ReadAllText("buildresult.txt");
Console.WriteLine(buildLog);
string fileName = "C:\\BuildResults\\" + DateTime.Now.Ticks + ".txt";
File.Move("buildresult.txt", fileName);
Console.ForegroundColor = ConsoleColor.Red;
Thread.Sleep(5000);
}
Console.WriteLine("Build Result " + buildResult.OverallResult.ToString());
Console.ForegroundColor = ConsoleColor.Gray;
Console.WriteLine("================================");
return buildResult;
}
This is some old code I had lying around.
I use this to programatically build solutions and C# Projects. The output will be a BuildResult.Success or BuildResult.Failure.
The variable buildLog will contain the build output.
Note - the only way to access the build output that I am aware of is to use the above methodology of having a log file generated and then reading it in your C# code.
One thing to be aware of and I never did find a fix for this, is that the application that runs this code, may keep dll's it loads into memory from nuget package directories in memory. This makes deleting those directories problematic. I found a work around by having my application run as a MS Service - it seems when it runs as a local service, it has enough permissions to delete files held in memory.

IronPython 2.7.7 C# Integration Memory Leak

I am integrating IronPython scripts to run under a C# engine. The C# engine builds a dictionary of 'ScriptValue' objects and passes it to the IronPython Script which then uses the objects to do calculations. The 'ScriptValue' object is in a separate class library and implements 'MarshalByRefObject' and is a simple .net object (stores double and bool values only). The script run happens frequently.
First Attempt:
I instantiated the IronPython engine and ran the scripts. As the runs progress I could see that memory usage was increasing at a fast rate. Eventually after a day or running the application crashed with an out of memory exception. I tried both keeping an instance of the IronPythonEngine alive and restarting a new instance on each run. I also tried shutting down the IronPython Engine but memory would increase consistently.
Second Attempt:
After researching this a lot, suggestions came up to try running the engine in a separate AppDomain and unloading the AppDomain once you are done running the scripts. I then implemented this and create a new AppDomain and unload it once a run has completed. This appears to help to a certain degree but the memory leak persists albeit it creeps up at a slower rate.
I did various memory profiling and it seems like IronPython or somewhere in DLR land unmanaged memory is not getting freed and this creeps up overtime. Managed memory seems to be getting cleared as the AppDomain is unloaded.
The C# engine itself is rather complex and interacts with MS SQL, IronPython, a Data Historian and an Asset Database. I won't go into the specifics of this as I have been able to recreate the issue by taking out all the additional components into a simple Windows Forms application.
The code I am running at the moment under a timer is:
private void RunEngine()
{
ScriptEngine pythonEngine = null;
AppDomain sandbox = null;
ScriptSource source = null;
ScriptScope scope = null;
dynamic subClass = null;
ObjectOperations ops = null;
dynamic instance = null;
dynamic result = null;
Dictionary<string, ScriptValue> scriptInputValues = GetIronPythonScriptInputAttributeValues();
Dictionary<string, ScriptValue> scriptOutputValues = GetIronPythonScriptOutputAttributes();
// Setup PythonEngine options
Dictionary<string, object> options = new Dictionary<string, object>();
//options["Debug"] = ScriptingRuntimeHelpers.True;
options["ExceptionDetail"] = ScriptingRuntimeHelpers.True;
options["ShowClrExceptions"] = ScriptingRuntimeHelpers.True;
// Create a sandbox to run the IronPython scripts in
sandbox = AppDomain.CreateDomain("IronPythonSandbox",
AppDomain.CurrentDomain.Evidence,
new AppDomainSetup() { ApplicationBase = AppDomain.CurrentDomain.BaseDirectory, ApplicationName = "IronPythonSandbox" },
new PermissionSet(PermissionState.Unrestricted));
// Create the python engine
pythonEngine = Python.CreateEngine(sandbox, options);
source = pythonEngine.CreateScriptSourceFromFile(#"\\server2\Projects\Customer\Development\Scripts\calculation.py");
var compiled = source.Compile();
scope = pythonEngine.CreateScope();
//source.Execute(scope);
compiled.Execute(scope);
subClass = scope.GetVariableHandle("Calculate");
ops = pythonEngine.Operations;
instance = ops.Invoke(subClass, scriptInputValues, scriptOutputValues);
result = instance.Unwrap();
if (scriptInputValues?.Count > 0) { scriptInputValues.Clear(); scriptInputValues = null; }
if (scriptOutputValues?.Count > 0) { scriptOutputValues.Clear(); scriptOutputValues = null; }
result = null;
instance = null;
ops = null;
subClass = null;
scope = null;
source = null;
pythonEngine?.Runtime?.Shutdown();
pythonEngine = null;
if (sandbox != null) { AppDomain.Unload(sandbox); }
sandbox = null;
}
I have stripped down the script into bare bones now to test the memory issue and it is like this and does not carry out any actual calculations as such.
import clr
import sys
# Import integration library to allow for access to the required .Net object types
sys.path.append(r"C:\Program Files\Company\RTCM Worker") # Include the path to the .Net Library
clr.AddReference('RTCM.Worker.IPy.Integration.Library.dll')
import RTCM.Worker.IPy.Integration.Library
import System
from System.Collections.Generic import Dictionary
sys.path.append(r"\\server2\Projects\Customer\Development\Scripts") # Include the path to the module
from constants import *
from sharedfunctions import *
import math
def Calculate(scriptInputValues, scriptOutputValues):
returnValue = True
try:
# Parameter validations
if returnValue: # Only proceed with the calculation if all inputs are valid
## Script logging related objects
#ENABLE_SCRIPTLOGGING = scriptOutputValues[C_EnableScriptLogging].Value
#SCRIPT_LOG = scriptOutputValues[C_ScriptLog].Value
# Get all the required input parameter values
AMB_TEMP = scriptInputValues[C_AmbientTemperature].Value
GND_AIR = scriptInputValues[C_GroundAir].Value
MAX_DESIGN_TEMP = scriptInputValues[C_MaximumDesignTemperature].Value
g = scriptInputValues[C_RatingCalculationConstants_g].Value
CONDUCTOR_DIA = scriptInputValues[C_ConductorDIA].Value
WIND_SPEED = scriptInputValues[C_WindSpeed].Value # From lookup table and no conversion needed as this is in m/s
DEFAULT_WIND_ANGLE = scriptInputValues[C_WindBearing].Value
SIGMA = scriptInputValues[C_Rating_Calculation_Constants_SIGMA].Value
CONDUCTOR_EMISSIVITY = scriptInputValues[C_ConductorEmissivity].Value
SOLAR_ABSORPTION = scriptInputValues[C_SolarAbsorption].Value
SOLAR_DIRECT = scriptInputValues[C_SolarDirect].Value
GROUND_REFLECTIVITY = scriptInputValues[C_GroundReflectivity].Value
SOLAR_DIFFUSE = scriptInputValues[C_SolarDiffuse].Value
CONDUCTOR_SKIN_EFFECT = scriptInputValues[C_ConductorSkinEffect].Value
CONDUCTOR_MAG_EFFECT = scriptInputValues[C_ConductorMAGEffect].Value
CONDUCTOR_DC_RESISTANCE = scriptInputValues[C_ConductorDCResistance].Value
CONDUCTOR_ALPHA = scriptInputValues[C_ConductorAlpha].Value
# Destroy all referenced objects
del AMB_TEMP
del GND_AIR
del MAX_DESIGN_TEMP
del g
del CONDUCTOR_DIA
del WIND_SPEED
del DEFAULT_WIND_ANGLE
del SIGMA
del CONDUCTOR_EMISSIVITY
del SOLAR_ABSORPTION
del SOLAR_DIRECT
del GROUND_REFLECTIVITY
del SOLAR_DIFFUSE
del CONDUCTOR_SKIN_EFFECT
del CONDUCTOR_MAG_EFFECT
del CONDUCTOR_DC_RESISTANCE
del CONDUCTOR_ALPHA
del scriptInputValues
del scriptOutputValues
returnValue = True
except System.Exception as ex:
returnValue = False
return returnValue;
Some screenshots of memory creeping up over time and you will notice unmanaged memory is creeping up:
Start of run
some time later
some time later
I am running out of options now. Are there any other suggestions on things to try?
A few other things I tried:
Setting LightweightScopes to true and it did not help.
Deleting objects referenced in the IronPython script using the del keyword and it did not help.
Let me know if you want to know any additional details around my setup.

I had the exact same issue with memory leakage per execution of an IronPython 2.7.5 script within a C# engine.
You should manually disconnect from your MarshalByRef objects at the end of each script execution or else you may be holding on to objects. If within your MarshalByRef object(s), you have overriden the InitializeLifetimeService to prevent remoting exceptions, it is mandatory to manually disconnect as shown here:
System.Runtime.Remoting.RemotingServices.Disconnect(MarshalByRefObj obj)
Hopefully you've had success with removing Debug options from the engine, I'd like to know if that worked for you.

Running Closure Compiler to get warning messages

I'm using Closure Compiler running on my local machine to generate client files. I'm also using the online closure tool at http://closure-compiler.appspot.com/home to check my work: I paste my JavaScript code, press compile and if there's a warning the compiler shows me the warnings with the actual lines and line numbers.
I'm wondering if it's possible to get the same output using the local version of Closure Compiler. This is the code I use to compile the files and get the compiled JavaScript but this time, I want the warnings:
System.IO.File.WriteAllText(FileNameInput, TheJS);
string JavaArgument = "-jar ClosureCompiler/compiler.jar --js ClosureCompiler/initial" + FileNameSuffix + ".js --js_output_file ClosureCompiler/compiled" + FileNameSuffix + ".js --compilation_level ADVANCED_OPTIMIZATIONS --externs ExternalFiles/JqueryExtern.js";
System.Diagnostics.Process clientProcess = new System.Diagnostics.Process();
clientProcess.StartInfo.FileName = "java.exe";
clientProcess.StartInfo.Arguments = JavaArgument;
clientProcess.StartInfo.CreateNoWindow = true;
clientProcess.StartInfo.UseShellExecute = false;
clientProcess.StartInfo.WorkingDirectory = HttpRuntime.AppDomainAppPath;
clientProcess.Start();
clientProcess.WaitForExit();
string TheOutputScript = System.IO.File.ReadAllText(FileNameOutput);
What do I need to change?

The compiler writes the warnings and errors to standard error which isn't going to show in your output file.
Option 1 - Redirect Standard Error to a File
Add 2> errorfile.txt to the end of your execution command to redirect standard error to a file. You'll then need to read that file.
Option 2 - Read the Standard Error Property of the Process
This should be as simple as:
clientProcess.StandardError

Chad's answer was great in that in pointed me towards the code. For those who need the actual code, this is what I have:
clientProcess.StartInfo.FileName = "java.exe";
clientProcess.StartInfo.Arguments = JavaArgument;
clientProcess.StartInfo.CreateNoWindow = true;
clientProcess.StartInfo.UseShellExecute = false;
clientProcess.StartInfo.WorkingDirectory = HttpRuntime.AppDomainAppPath;
clientProcess.StartInfo.RedirectStandardError = true; //add this line
clientProcess.Start();
string CompilerErrors = clientProcess.StandardError.ReadToEnd(); //add this line
clientProcess.WaitForExit();
return System.IO.File.ReadAllText(FileNameOutput); //add breakpoint here
Now all you have to do is add a break point at the end and watch the variable CompilerErrors.

RDotNet vs R scripting

When is it an advantage/disadvantage to be using RDotNet for making statistical calculations as opposed to generating an R script text file and running it from the application using e.g. Process.Start? Or is there some other better way?
I need to execute a large number of commands and have a feeling that sending them one by one to R takes a lot of time.

I'd say the following two scenario's are stereotypical:
.NET code and R code are quite separate, not a lot of interaction is needed between the R code and the .NET code. For example, the .NET code gathers some information, and launches a processing script on that, after which the .NET code picks up the results. In this case spawning an R process (Process.Start) is a simple way to get this working.
A lot of interaction is needed between the .NET code and the R code, the workflow consists of going back and forth between .NET and R often. In this case, a more heavy weight, flexible solution such as RDotNet makes a lot of sense. RDotNet allows more easy integration of the .NET code and the R code, with the price being that it is often harder to learn, harder to debug, and often needs to be updated for new versions of R etc.

R.NET currently can initilize once. Parallel execution will be problematic.
Would suggest use RScript.
Our solution based on this answer on stackoverflow Call R (programming language) from .net
With monor change, we send R code from string and save it to temp file, since user run custom R code when needed.
public static void RunFromCmd(string batch, params string[] args)
{
// Not required. But our R scripts use allmost all CPU resources if run multiple instances
lock (typeof(REngineRunner))
{
string file = string.Empty;
string result = string.Empty;
try
{
// Save R code to temp file
file = TempFileHelper.CreateTmpFile();
using (var streamWriter = new StreamWriter(new FileStream(file, FileMode.Open, FileAccess.Write)))
{
streamWriter.Write(batch);
}
// Get path to R
var rCore = Registry.LocalMachine.OpenSubKey(#"SOFTWARE\R-core") ??
Registry.CurrentUser.OpenSubKey(#"SOFTWARE\R-core");
var is64Bit = Environment.Is64BitProcess;
if (rCore != null)
{
var r = rCore.OpenSubKey(is64Bit ? "R64" : "R");
var installPath = (string)r.GetValue("InstallPath");
var binPath = Path.Combine(installPath, "bin");
binPath = Path.Combine(binPath, is64Bit ? "x64" : "i386");
binPath = Path.Combine(binPath, "Rscript");
string strCmdLine = #"/c """ + binPath + #""" " + file;
if (args.Any())
{
strCmdLine += " " + string.Join(" ", args);
}
var info = new ProcessStartInfo("cmd", strCmdLine);
info.RedirectStandardInput = false;
info.RedirectStandardOutput = true;
info.UseShellExecute = false;
info.CreateNoWindow = true;
using (var proc = new Process())
{
proc.StartInfo = info;
proc.Start();
result = proc.StandardOutput.ReadToEnd();
}
}
else
{
result += "R-Core not found in registry";
}
Console.WriteLine(result);
}
catch (Exception ex)
{
throw new Exception("R failed to compute. Output: " + result, ex);
}
finally
{
if (!string.IsNullOrWhiteSpace(file))
{
TempFileHelper.DeleteTmpFile(file, false);
}
}
}
}
Full blog post: http://kostylizm.blogspot.ru/2014/05/run-r-code-from-c-sharp.html

With Process.Start you will start up a new R session. This can take some time, especially if you are using different packages in your script which you need to load.
If you use R.NET, you can create an R instance, and keep on talking with it. So if you have created a webservice to connect R with ASP you don't want to start up R all the time as this will be very time costly. You need it just once and you can work with it interactively.