I'm getting a OutOfMemory exception when running the following code, it happens on the File.ReadLines line, it processes most files fine until it hits larger files.
It's consistantly using tons of memory and cpu during the whole process though.
The file it crashed on is only 156,000KB, which is 156mb
static void Main(string[] args)
{
Console.CursorVisible = false;
Console.ForegroundColor = ConsoleColor.Green;
Console.WriteLine();
Console.WriteLine(" [" + DateTime.Now.ToShortTimeString() + "]" + " Connected to the Cassandra Database");
Console.WriteLine();
Console.ForegroundColor = ConsoleColor.White;
string filepath = #"C:\Users\admin\Desktop\wecrack lists";
DirectoryInfo directory = new DirectoryInfo(filepath);
int fileCount = 0;
var client = new MongoClient("mongodb://localhost:27017");
var database = client.GetDatabase("cracking");
var collection = database.GetCollection<Password>("passwords");
foreach (var file in directory.GetFiles("*"))
{
fileCount++;
Console.WriteLine(" [" + DateTime.Now.ToShortTimeString() + "]" + " Working through file: {" + file + "} {" + fileCount + "/" + directory.GetFiles("*").Count() + "}");
List<Password> entitys = new List<Password>();
foreach (string line in File.ReadLines(filepath + #"\" + file.ToString()))
{
entitys.Add(new Password { password = line });
}
collection.InsertManyAsync(entitys);
}
Console.WriteLine();
Console.WriteLine(" [" + DateTime.Now.ToShortTimeString() + "]" + " Finished inserting records, press any key to get the count.");
Console.ReadKey(true);
while (true)
{
Console.ReadKey(true);
}
}
Try batching your updates. That way you won't have all that data in memory at the same time. It may also help you not totally lock up your database.
...
foreach (var file in directory.GetFiles("*"))
{
fileCount++;
Console.WriteLine(" [" + DateTime.Now.ToShortTimeString() + "]" + " Working through file: {" + file + "} {" + fileCount + "/" + directory.GetFiles("*").Count() + "}");
System.IO.StreamReader file = new System.IO.StreamReader(filepath + #"\" + file.ToString());
while(!file.EndOfStream)
{
int passwordBatchCount = 0;
List<Password> entitysBatch = new List<Password>();
while ((string line = file.ReadLine()) != null && passwordBatchCount < BATCH_SIZE)
{
entitysBatch.Add(new Password { password = line });
passwordBatchCount++;
}
collection.InsertManyAsync(entitysBatch);
}
file.Close();
}
}
...
Related
This question already has answers here:
IOException: The process cannot access the file 'file path' because it is being used by another process
(12 answers)
Closed 1 year ago.
I'm a student who just started learning C#,
I'm trying to create Sign-Up and Login functions using text files, and save details that the user inputs into the textfile, but when I run my function again it rewrites what is already there instead of starting a new line.
This is a bit of my code that is important
using System;
using System.IO;
namespace SDD_Assessment_2
{
class Program
{
static void Main(string[] args)
{
//Write();
//GetTime();
var privInfo = SignUp();
var FullName = Profile(privInfo);
Profile(privInfo);
//Test(FullName);
}
const string filename = "Workout.txt";
static string[] SignUp()
{
string[] privinfo = new string[4];
Console.Write("First Name: ");
privinfo[0] = Console.ReadLine();
Console.Write("Last Name: ");
privinfo[1] = Console.ReadLine();
Console.Write("Email: ");
privinfo[2] = Console.ReadLine();
Console.Write("Password: ");
privinfo[3] = Console.ReadLine();
StreamWriter NewAccount = new StreamWriter(filename);
//NewAccount.WriteLine(privinfo[0] + "," + privinfo[1] + "," + privinfo[2] + "," + privinfo[3] + Environment.NewLine);
File.AppendAllText(filename, privinfo[0] + ", " + privinfo[1] + "," + privinfo[2] + "," + privinfo[3] + Environment.NewLine);
NewAccount.Close();
return privinfo;
}
}
}
When I run it right now, it says "System.IO.IOException: The process cannot access the file 'filename' because it is being used by another process."
Your code is mostly correct. You need to change
StreamWriter NewAccount = new StreamWriter(filename);
//NewAccount.WriteLine(privinfo[0] + "," + privinfo[1] + "," + privinfo[2] + "," + privinfo[3] + Environment.NewLine);
File.AppendAllText(filename, privinfo[0] + ", " + privinfo[1] + "," + privinfo[2] + "," + privinfo[3] + Environment.NewLine);
NewAccount.Close();
to just
File.AppendAllText(filename, privinfo[0] + ", " + privinfo[1] + "," + privinfo[2] + "," + privinfo[3] + Environment.NewLine);
Your allocation of NewAccount opens a stream to the file, which keeps the file handle open. That means your File.AppendAllText can't complete, because the file was previously opened in a non-sharing mode.
If you had deleted File.AppendAllText you wouldn't get the error, but you'd end up with a zero byte file as you wrote nothing to the stream.
As you are not using NewAccount at all, removing those lines will allow File.AppendAllText to complete without error, and... append all the text you expect, which would be the values for first and last names, email and password.
I try, it work
Don't open stream when you not used
File name include directory
class Program
{
static void Main(string[] args)
{
SignUp();
}
const string filename = "Workout.txt";
static string[] SignUp()
{
string[] privinfo = new string[4];
privinfo[0] = "First Name:";
privinfo[1] = "Last Name: ";
privinfo[2] = "Email: ";
privinfo[3] = "Password: ";
File.AppendAllText(Path.Combine(Environment.CurrentDirectory, filename), privinfo[0] + ", " + privinfo[1] + "," + privinfo[2] + "," + privinfo[3] + Environment.NewLine);
return privinfo;
}
}
Run 2 times
file content is
First Name:, Last Name: ,Email: ,Password:
First Name:, Last Name: ,Email: ,Password:
I know you are going to tell me that this question is stupid but I really cannot find a solution to delete my file.
In fact, I open a connection to an .add file (same style as SQL in a way) but afterwards I cannot delete it because it is used by another process which is the process of my application.
While doing some research on the internet I was able to find the solution to kill the process however if I do this it stops my application. Then I also found the GC collect but it doesn't work : /
There is my code :
try
{
string idClient = "parfilux";
string tableName = "ACT,ACF";
string dataSourceDBF = "C:/winbooks/data/parfilux";
string path = dataSourceDBF ;
string pathTemp = dataSourceDBF + "/CopyTempWebService/";
if (!Directory.Exists(pathTemp)) Directory.CreateDirectory(pathTemp);
string addFile = path + "/" + idClient + ".add";
File.Copy(addFile, pathTemp + idClient.ToUpper() + ".add");
File.Copy(addFile.Replace(".add", ".ai"), pathTemp + "/" + idClient.ToUpper() + ".ai");
File.Copy(addFile.Replace(".add", ".am"), pathTemp + "/" + idClient.ToUpper() + ".am");
tableName = tableName.Replace(" ", "");
string[] tables = tableName.Split(',');
string pathTable = null;
foreach (string tab in tables)
{
pathTable = path + "/" + idClient.ToUpper() + "_" + tab.ToUpper() + ".dbf";
File.Copy(pathTable, pathTemp + idClient.ToUpper() + "_" + tab.ToUpper() + ".dbf");
File.Copy(pathTable.Replace(".dbf", ".cdx"), pathTemp + idClient.ToUpper() + "_" + tab.ToUpper() + ".cdx");
}
AdsConnection dbfCo;
//dbfCo.Close();
dbfCo = new AdsConnection(#"data Source=" + dataSourceDBF + "/CopyTempWebService/" + idClient + ".add;User ID=admin;Password=;ServerType=Local;ReadOnly=true;pooling=true;TrimTrailingSpaces=true;ShowDeleted=TRUE;TableType=CDX;LockMode=COMPATIBLE");
dbfCo.Open();
//QueryDataDBF(tableName, idClient, false);
dbfCo.Close();
dbfCo.Dispose();
//Process process = Process.GetCurrentProcess();
//Console.WriteLine(process.MainModule);
//process.Kill();
//foreach(Process pro in process)
//{
// Console.WriteLine(pro);
// if(pro.ProcessName == pathTemp + idClient.ToUpper() + ".add")
// {
// pro.Kill();
// }
//}
//System.GC.Collect();
//System.GC.WaitForPendingFinalizers();
//File.Delete(pathTemp + idClient.ToUpper() + ".add");
Directory.Delete(dataSourceDBF + "/CopyTempWebService", true);
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
}
I open a connection to my .add with the Advantage Data Provider library.
Do you have any idea to fix this problem ? thank you in advance ;)
I am calling following method inside thread. inside method i have written logs which writes some variable values in notepad file.
Problem : Sometime last log of method do not log anything. ie last line not exucuting sometime. i am not able to understand problem. please guide me if there are issue somewhere.
This is web application hosted in iis server.
Function :
public bool ResetEmployeeAssignedCoursesByRole()
{
bool bReturn = false;
DbTransactionHelper dbTransactionHelper = new DbTransactionHelper();
dbTransactionHelper.BeginTransaction();
try
{
// this line execuete fine
ErrorLog.createRoleLog("Method sp_Reset_EmpAssignedCoursesByRole Started " + this.RoleID + " Course ID " + this.CourseID + " External Message " + sMessage);
StringBuilder sbQueryEmployeeCourse = new StringBuilder();
sbQueryEmployeeCourse.Append(" set #roleID = " + roleID + "; ");
sbQueryEmployeeCourse.Append(" set #courseID = " + courseID + "; ");
sbQueryEmployeeCourse.Append(" set #inActiveReason = " + inActiveReason + "; ");
sbQueryEmployeeCourse.Append(" set #lastRecordUpdateSource = " + lastRecordUpdateSource + "; ");
sbQueryEmployeeCourse.Append(" call sp_Reset_EmpAssignedCoursesByRole (#roleID, #courseID, #inActiveReason, #lastRecordUpdateSource); ");
DataTable dtEmployeeCourse = dbTransactionHelper.GetDataTable(BusinessUtility.GetString(sbQueryEmployeeCourse), CommandType.Text, null
);
dbTransactionHelper.CommitTransaction();
bReturn = true;
// this line not execuete sometime.
ErrorLog.createRoleLog("Method sp_Reset_EmpAssignedCoursesByRole Ended " + this.RoleID + " Course ID " + this.CourseID + " External Message " + sMessage);
}
catch (Exception ex)
{
ErrorLog.createRoleLog("Add Role ID " + BusinessUtility.GetString(roleID));
dbTransactionHelper.RollBackTransaction();
ErrorLog.createRoleLog("Add Course ID " + BusinessUtility.GetString(courseID));
ErrorLog.createRoleLog(ex.ToString());
}
return bReturn;
}
Calls method like below :
Thread t = new Thread(ResetEmployeeAssignedCoursesByRole);
t.Start();
FUNCTION createRoleLog :
public static void createRoleLog(string errorMessage, int empHdrID = 0)
{
try
{
//string path = BusinessUtility.GetString(AppConfig.GetAppConfigValue("LogsDiractory")) + "Log" + DateTime.Now.ToString("yyyy-MM-dd") + ".txt";
string path = BusinessUtility.GetString(ErrorLog.ErrorLogPath) + "RoleLog" + DateTime.Now.ToString("yyyy-MM-dd") + ".txt";
if (BusinessUtility.GetString(AppConfig.GetAppConfigValue("LogError")).ToString().ToUpper() == "TRUE")
{
if (!File.Exists(path))
{
StreamWriter sw = File.CreateText(path);
sw.Close();
}
//using (System.IO.StreamWriter sw = System.IO.File.AppendText(path))
//{
// sw.WriteLine("-------- " + DateTime.Now + " --------");
// sw.WriteLine(errorMessage);
// sw.WriteLine("------------------------");
// sw.Close();
//}
if (!IsFileLocked(new FileInfo(path)))
{
using (FileStream stream = new FileStream(path, FileMode.Append, FileAccess.Write, FileShare.ReadWrite))
{
using (System.IO.StreamWriter sw = new StreamWriter(stream))
{
if (empHdrID != 0)
{
sw.WriteLine("-------- [empHdrID: " + empHdrID + "] " + DateTime.Now + " --------");
}
else
{
sw.WriteLine("-------- " + DateTime.Now + " --------");
}
sw.WriteLine(errorMessage);
sw.WriteLine("------------------------");
sw.Close();
}
}
}
else
{
}
}
}
catch (Exception ex)
{
//throw ex;
}
}
Last line of function which not execution sometime is :
** this line not execuete sometime.**
ErrorLog.createRoleLog("Method sp_Reset_EmpAssignedCoursesByRole Ended " + this.RoleID + " Course ID " + this.CourseID + " External Message " + sMessage);
The process cannot access the file 'file path' because it is being used by another process.
i have found these 2 question
File being used by another process after using File.Create()
and
Does File.AppendAllText close the file after the operation
this is an API that i have and need to save every request that comes in and the result that goes out,
there might be more than one request that a give time
my code
public static void SaveTheRequestAndResponse(string type, SearchRequest searchRequest = null, dynamic result = null)
{
var FilePath = AppDomain.CurrentDomain.BaseDirectory + #"SearchRequest";
bool exists = Directory.Exists(FilePath);
if (!exists)
{
var stream = Directory.CreateDirectory(FilePath);
}
if (type == "request")
{
string Space = ", ";
StringBuilder request = new StringBuilder();
request.Append("Search Id : " + searchRequest.ID);
request.Append(Space + "Company Name : " + searchRequest.CompanyName);
request.Append(Space + "Country Code : " + searchRequest.CountryCode);
request.Append(Space + "Search Type : " + searchRequest.SeacrhType);
request.Append(Space + "Request Time : " + DateTime.Now + Environment.NewLine);
var DataToBeSave = request.ToString();
System.IO.File.AppendAllText(FilePath + #"\" + "FileNAme" + DateTime.Now.ToString("dd-MM-yyyy") + ".txt", DataToBeSave + Environment.NewLine);
}
else
{
string Space = ", ";
StringBuilder SearchResult = new StringBuilder();
SearchResult.Append("The result for Request" + Space);
SearchResult.Append("Search Id : " + searchRequest.ID + Space);
SearchResult.Append("States Code : " + result.StatusCode + Space);
SearchResult.Append("Result Time : " + DateTime.Now + Environment.NewLine);
var DataToBeSave = SearchResult.ToString();
System.IO.File.AppendAllText(FilePath + #"\" + "FileNAme" + DateTime.Now.ToString("dd-MM-yyyy") + ".txt", DataToBeSave + Environment.NewLine);
}
}
my understanding is that the File.AppendAllText will close after the Operation so why do i get the this error
my code is having an race condition, and this is because the API is being call by more than one user at each given time, even that
System.IO.File.AppendAllText(FilePath + #"\" + "FileNAme" + DateTime.Now.ToString("dd-MM-yyyy") + ".txt", DataToBeSave + Environment.NewLine);
will close after the Operation, it still need time do its work and only one connection can be open at each give time so the thread need to be lock and that can be done by
private static Object thisLock = new Object();
lock (thisLock)
{
System.IO.File.AppendAllText(FilePath + #"\" + "DandB" + DateTime.Now.ToString("dd-MM-yyyy") + ".txt", DataToBeSave + Environment.NewLine);
}
Thanks to Abydal
So I'm trying to exit a console application IF a parameter check fails, however, I still want it to log to a file. The logging to a file works fine as long as all the parameters are good. However, when a parameter check fails and hits the System.Environment.Exit(0) portion, the log file is still completely empty. Here's the code so far. Please help, I've tried everything I could think of.
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.IO;
namespace I2C_File_Splitter
{
class Program
{
static void Main(string[] args)
{
//get command line input paramaters
using (StreamWriter log = File.AppendText("Splitter_log.txt"))
{
log.WriteLine(DateTime.Now + " ******************************************** SPLITTER STARTED ****************************************************************");
log.WriteLine(DateTime.Now + " FILE: " + args[0] + " DESTINATION: " + args[1] + " MAX COUNT PER FILE: " + args[2]);
if (args.Length == 0)
System.Environment.Exit(0);
string originalFile = args[0];
string destination = args[1];
int fileLength = Convert.ToInt32(args[2]);
string fileName;
string fileExtension;
string line;
int fileNumber = 1;
if (!File.Exists(originalFile))
{
log.WriteLine(DateTime.Now + " Error: Transfund file not found for: " + args[0]);
log.WriteLine(DateTime.Now + " ******************************************** SPLITTER ENDED ****************************************************************");
System.Environment.Exit(0);
}
if (!Directory.Exists(destination))
{
log.WriteLine(DateTime.Now + " Error: destination directory not found for: " + args[1] );
log.WriteLine(DateTime.Now + " ******************************************** SPLITTER ENDED ****************************************************************");
System.Environment.Exit(0);
}
if (fileLength < 0)
{
log.WriteLine(DateTime.Now + " Error: file length must be greater than 0. Incorrect value " + args[2]);
log.WriteLine(DateTime.Now + " ******************************************** SPLITTER ENDED ****************************************************************");
System.Environment.Exit(0);
}
//get file name and file extension
fileName = Path.GetFileNameWithoutExtension(originalFile);
fileExtension = Path.GetExtension(originalFile);
StreamReader file = new StreamReader(originalFile);
log.WriteLine(DateTime.Now + " processing: " + fileName);
string header = file.ReadLine(); //get first line
while ((line = file.ReadLine()) != null)
{
StreamWriter newFile = new StreamWriter(destination + "\\" + fileName + "_" + fileNumber.ToString() + fileExtension);
newFile.WriteLine(header);
newFile.WriteLine(line);
int counter = 1;
while (counter < fileLength)
{
line = file.ReadLine();
if (line == null)
break;
newFile.WriteLine(line);
counter++;
}
newFile.Close();
log.WriteLine(DateTime.Now + " " + fileName + "_" + fileNumber.ToString() + fileExtension + " created. Card count: " + counter);
fileNumber++;
}
log.WriteLine(DateTime.Now + " Processing completed: " + fileName);
log.WriteLine(DateTime.Now + " ******************************************** SPLITTER ENDED ****************************************************************");
}
}
}
}
When you call Environment.Exit, you're telling it to terminate your program immediately.
Your "log" stream never gets flushed (which would happen when you reach the end of the using block), and so nothing gets a chance to be written to the file.
Try flushing the stream before calling exit.
if (!Directory.Exists(destination))
{
log.WriteLine(DateTime.Now + " Error: destination directory not found for: " + args[1] );
log.WriteLine(DateTime.Now + " ******************************************** SPLITTER ENDED ****************************************************************");
// write all pending log messages to the file
log.Flush();
System.Environment.Exit(0);
}