Develop Reference

Appending to log file [closed]

Closed. This question needs debugging details. It is not currently accepting answers.
Edit the question to include desired behavior, a specific problem or error, and the shortest code necessary to reproduce the problem. This will help others answer the question.
Closed 5 years ago.
Improve this question
I am trying to to write data to a log file but nothing gets written to the file.
Aim of the program is to run a continuous loop and keep looking for file, if file is valid, process it and move it. I am logging for any errors and items that are created.
Also, how can I make my log file access able while the loop is running so that I can see that values got appended.
static void Main(string[] args)
{
var logFile = File.Create(filePath + "\\log_" + DateTime.Today.ToString("MMMM") + ".txt").ToString();
while (true)
{
try
{
var moveTo = Directory.CreateDirectory(#"" + directoryPath + "Processed_" + DateTime.Today.ToString("MMMM"));
var files = Directory.GetFiles(filePath);
var todaysDate = DateTime.Now.Date;
var firstOfMonth = new DateTime(todaysDate.Year, todaysDate.Month, 1);
var monthEnd = firstOfMonth.AddMonths(1).AddDays(-1);
if (todaysDate == monthEnd)
{
File.Move(logFile, #"" + moveToNewPath + logFile);
}
foreach (var fileName in files)
{
if (fileName.Contains("myFile.csv"))
{
var fileValues = File.ReadAllLines(filePath + fileName.Substring(44)).Skip(1).Select(v => new myFile(v)).ToList();
foreach (var i in fileValues)
{
try
{
var jsonValues = ValueFromFile(i);
var response = UploadData(url, username, password, values);
this should be written to a log file ===> .File.AppendAllText(logFile, Environment.NewLine + DateTime.Now + "\t" + response);
}
catch (Exception exception)
{
File.AppendAllText(logFile, Environment.NewLine + DateTime.Now + "\t" + exception.Message.Replace("\n", " "));
}
}
File.Move(fileName, #"" + directoryPath + "\\" + moveTo + "\\" + "processedMyFile" + DateTime.Now.Date.ToString("MM-dd-yy") + ".csv");
}
}
}
catch (Exception exception)
{
File.AppendAllText(logFile, Environment.NewLine + DateTime.Now + "\t" + exception.Message.Replace("\n", " "));
}
}
}

Let's start with this line at the top of the program:
var logFile = File.Create(filePath + "\\log_" + DateTime.Today.ToString("MMMM") + ".txt").ToString();
I'm not sure what you're doing with that ToString() call hanging off the end. It almost certainly doesn't do what you think it does. But I really want to take a closer look at the documentation for the File.Create() method here. Specifically, this excerpt:
The FileStream object created by this method has a default FileShare value of None; no other process or code can access the created file until the original file handle is closed.
Uh oh. That means the File.AppendAllText() call later on will be out of luck. But let's look at the AppendAllText() documentation. Specifically this:
The method creates the file if it doesn’t exist
Meaning you can just remove the problem line at the top. You neither need nor want it. Or maybe you just want to create the file name there, like this:
var logFile = Path.Combine(filePath, "log_" + DateTime.Today.ToString("MMMM") + ".txt");
As a bonus, I'd explore changing this code to use System.Diagnostics.Trace in conjunction with FileTraceListener and maybe a ConsoleTraceListener attached.
static string moveToNewPath = "...";
static string filePath = "...";
static string logFormat = "\n{0:s}\t{1}";
static string logFile = "";
static string directoryPath = "...";
static void LogMessage(string FilePath, string Message)
{
File.AppendAllText(Path.Combine(filePath, logFile),
string.Format(logFormat, DateTime.Now, Message.Replace("\n", " ")));
}
static void Main(string[] args)
{
logFile = "log_" + DateTime.Today.ToString("MMMM") + ".txt";
//Rotate log file on last day of month
try
{
var todaysDate = DateTime.Now.Date;
var firstOfMonth = new DateTime(todaysDate.Year, todaysDate.Month, 1);
var monthEnd = firstOfMonth.AddMonths(1).AddDays(-1);
if (todaysDate == monthEnd)
{
File.Move(Path.Combe(filePath, logFile), Path.Combine(moveToNewPath, logFile));
}
}
catch (Exception ex)
{
LogMessage(ex.Message);
}
while (true)
{
// !!!!!!!!!!!!!!
//Log rotate code used to be here... but... you need something to be sure this only happens once per day.
// I STRONGLY suspect this code should be setup to run as
// a SCHEDULED TASK set to run once per day or maybe once per hour, rather than an always-on background program.
// !!!!!!!!!!!!!!!!
try
{
var moveTo = Path.Combine(directoryPath, "Processed_" + DateTime.Today.ToString("MMMM"));
Directory.CreateDirectory(moveTo);
var files = Directory.GetFiles(filePath).Where(f => f.EndsWith("myFile.csv"));
foreach (var fileName in files)
{
var fileValues = File.ReadAllLines(filePath + fileName.Substring(44)).Skip(1).Select(v => new myFile(v));
foreach (var i in fileValues)
{
try
{
var jsonValues = ValueFromFile(i);
var response = UploadData(url, username, password, jsonValues);
LogMessage(response);
}
catch (Exception ex)
{
LogMessage(ex.Message);
}
}
File.Move(fileName, Path.Combine(moveTo, "processedMyFile" + DateTime.Now.Date.ToString("MM-dd-yy") + ".csv"));
}
}
catch (Exception ex)
{
LogMessage(ex.Message);
}
}
}

C# out of memory with file

I'm getting a OutOfMemory exception when running the following code, it happens on the File.ReadLines line, it processes most files fine until it hits larger files.
It's consistantly using tons of memory and cpu during the whole process though.
The file it crashed on is only 156,000KB, which is 156mb
static void Main(string[] args)
{
Console.CursorVisible = false;
Console.ForegroundColor = ConsoleColor.Green;
Console.WriteLine();
Console.WriteLine(" [" + DateTime.Now.ToShortTimeString() + "]" + " Connected to the Cassandra Database");
Console.WriteLine();
Console.ForegroundColor = ConsoleColor.White;
string filepath = #"C:\Users\admin\Desktop\wecrack lists";
DirectoryInfo directory = new DirectoryInfo(filepath);
int fileCount = 0;
var client = new MongoClient("mongodb://localhost:27017");
var database = client.GetDatabase("cracking");
var collection = database.GetCollection<Password>("passwords");
foreach (var file in directory.GetFiles("*"))
{
fileCount++;
Console.WriteLine(" [" + DateTime.Now.ToShortTimeString() + "]" + " Working through file: {" + file + "} {" + fileCount + "/" + directory.GetFiles("*").Count() + "}");
List<Password> entitys = new List<Password>();
foreach (string line in File.ReadLines(filepath + #"\" + file.ToString()))
{
entitys.Add(new Password { password = line });
}
collection.InsertManyAsync(entitys);
}
Console.WriteLine();
Console.WriteLine(" [" + DateTime.Now.ToShortTimeString() + "]" + " Finished inserting records, press any key to get the count.");
Console.ReadKey(true);
while (true)
{
Console.ReadKey(true);
}
}

Try batching your updates. That way you won't have all that data in memory at the same time. It may also help you not totally lock up your database.
...
foreach (var file in directory.GetFiles("*"))
{
fileCount++;
Console.WriteLine(" [" + DateTime.Now.ToShortTimeString() + "]" + " Working through file: {" + file + "} {" + fileCount + "/" + directory.GetFiles("*").Count() + "}");
System.IO.StreamReader file = new System.IO.StreamReader(filepath + #"\" + file.ToString());
while(!file.EndOfStream)
{
int passwordBatchCount = 0;
List<Password> entitysBatch = new List<Password>();
while ((string line = file.ReadLine()) != null && passwordBatchCount < BATCH_SIZE)
{
entitysBatch.Add(new Password { password = line });
passwordBatchCount++;
}
collection.InsertManyAsync(entitysBatch);
}
file.Close();
}
}
...

C# compare id from text file in filestream

I need to fill a text file with information about workers. Then I need to read from the file and search for an ID that user tries to find. For example my file contains ids 1,2,3 and if I try to find id 3 and it matches, then this worker's all information is written in console. Otherwise it writes a text A worker cannot be found.
using System;
using System.IO;
class Program
{
static void Main(string[] args)
{
string file = "C:\\Temp\\registery.txt";
FileStream fOutStream = File.Open(file, FileMode.Append, FileAccess.Write);
StreamWriter sWriter = new StreamWriter(fOutStream);
int[] id = { 1, 2, 3 };
string[] name = { "John", "Carl", "Thomas" };
float[] salary = { 3500, 4800, 2100 };
for (int i = 0; i < id.Length; i++)
{
sWriter.WriteLine(id[i] + " " + name[i] + " " + salary[i]);
}
sWriter.Flush();
sWriter.Close();
FileStream fInStream = File.OpenRead(file);
StreamReader sReader = new StreamReader(fInStream);
int id2;
Console.WriteLine("Type worker's id");
id2 = int.Parse(Console.ReadLine());
bool a;
a = sReader.ReadToEnd().Contains(id2);
Console.WriteLine(a);
sReader.Close();
}
}

If you want to create a text file to be searchable, it should be delimited by a separator like comma /TAB
so modify your code:
sWriter.WriteLine(id[i] + "," + name[i] + "," + salary[i]);
To search your text file by id/name/..whatever and use AND/OR, you can use the method described here:
How would I convert data in a .txt file into xml? c#
BTW: Re-factor your code to create the file in a separate method, and the search in other one.

I found a solution myself to my problem and it worked good enough. It might not be the best solution. I removed bool things and I replaced the whole thing with this:
string line;
while ((line = sReader.ReadLine()) != null)
{
if (line.Contains("id: " + id2))
{
Console.WriteLine(line);
break;
}
else if ((line = sReader.ReadLine()) == null)
{
Console.WriteLine("Worker not found with id " + id2);
}
}
And I fixed the upper for loop to look like this:
sWriter.WriteLine("id: " + id[i] + " name: " + name[i] + " salary: " + salary[i]);

Reading a large CSV file and processing in C#. Any suggestions?

I have a large CSV file around 25G. I need to parse each line which has around 10 columns and do some processing and finally save it to a new file with parsed data.
I am using dictionary as my datastructure. To avoid the memory overflow I am writing the file after 500,000 records and clearing the dictionary.
Can anyone suggest whether is this good way of doing. If not, any other better way of doing this? Right now it is taking 30 mins to process 25G file.
Here is the code
private static void ReadData(string filename, FEnum fileType)
{
var resultData = new ResultsData
{
DataColumns = new List<string>(),
DataRows = new List<Dictionary<string, Results>>()
};
resultData.DataColumns.Add("count");
resultData.DataColumns.Add("userid");
Console.WriteLine("Start Processing : " + DateTime.Now);
const long processLimit = 100000;
//ProcessLimit : 500000, TimeElapsed : 30 Mins;
//ProcessLimit : 100000, TimeElaspsed - Overflow
Stopwatch stopwatch = new Stopwatch();
stopwatch.Start();
Dictionary<string, Results> parsedData = new Dictionary<string, Results>();
FileStream fileStream = new FileStream(filename, FileMode.Open, FileAccess.Read);
using (StreamReader streamReader = new StreamReader(fileStream))
{
string charsRead = streamReader.ReadLine();
int count = 0;
long linesProcessed = 0;
while (!String.IsNullOrEmpty(charsRead))
{
string[] columns = charsRead.Split(',');
string eventsList = columns[0] + ";" + columns[1] + ";" + columns[2] + ";" + columns[3] + ";" +
columns[4] + ";" + columns[5] + ";" + columns[6] + ";" + columns[7];
if (parsedData.ContainsKey(columns[0]))
{
Results results = parsedData[columns[0]];
results.Count = results.Count + 1;
results.Conversion = results.Count;
results.EventList.Add(eventsList);
parsedData[columns[0]] = results;
}
else
{
Results results = new Results {
Count = 1, Hash_Person_Id = columns[0], Tag_Id = columns[1], Conversion = 1,
Campaign_Id = columns[2], Inventory_Placement = columns[3], Action_Id = columns[4],
Creative_Group_Id = columns[5], Creative_Id = columns[6], Record_Time = columns[7]
};
results.EventList = new List<string> {eventsList};
parsedData.Add(columns[0], results);
}
charsRead = streamReader.ReadLine();
linesProcessed++;
if (linesProcessed == processLimit)
{
linesProcessed = 0;
SaveParsedValues(filename, fileType, parsedData);
//Clear Dictionary
parsedData.Clear();
}
}
}
stopwatch.Stop();
Console.WriteLine(#"File : {0} Batch Limit : {1} Time elapsed : {2} ", filename + Environment.NewLine, processLimit + Environment.NewLine, stopwatch.Elapsed + Environment.NewLine);
}
Thank you

The Microsoft.VisualBasic.FileIO.TextFieldParser class looks like it could do the job. Try it, it may speed things up.

Store sensitive information inside keepass database from c#

I have a project where I have to handle sensitive data.
How do I open a keepass database from C# to use the data?
I have downloaded the source. I will look in it to get what I need. Any other idea?

I thought about reading a KeyPass 2 database so I added a reference to KeyPass.exe in Linqpad and started to experiment. To my surprise and without any outside help (a testament to a good API), I was reading the database after only a few minutes. Here's how I did it:
var dbpath = #"C:\path\to\passwords.kdbx";
var masterpw = "Your$uper$tr0ngMst3rP#ssw0rd";
var ioConnInfo = new IOConnectionInfo { Path = dbpath };
var compKey = new CompositeKey();
compKey.AddUserKey(new KcpPassword(masterpw));
var db = new KeePassLib.PwDatabase();
db.Open(ioConnInfo, compKey, null);
var kpdata = from entry in db.RootGroup.GetEntries(true)
select new
{
Group = entry.ParentGroup.Name,
Title = entry.Strings.ReadSafe("Title"),
Username = entry.Strings.ReadSafe("UserName"),
Password = entry.Strings.ReadSafe("Password"),
URL = entry.Strings.ReadSafe("URL"),
Notes = entry.Strings.ReadSafe("Notes")
};
kpdata.Dump(); // this is how Linqpad outputs stuff
db.Close();

Here is an extension of the original answer from Ronnie - walking the keepass tree recursively. This outputs a format that jsTree can use by the way.
public static void JsonData() {
var dbpath = Web.MapPath(#"your-password-file.kdbx");
var masterpw = "Your$uper$tr0ngMst3rP#ssw0rd";
var ioConnInfo = new IOConnectionInfo { Path = dbpath };
var compKey = new CompositeKey();
compKey.AddUserKey(new KcpPassword(masterpw));
var db = new KeePassLib.PwDatabase();
db.Open(ioConnInfo, compKey, null);
//get everything
var kpdata = from entry in db.RootGroup.GetEntries(true)
select new {
Group = entry.ParentGroup.Name,
Title = entry.Strings.ReadSafe("Title"),
Username = entry.Strings.ReadSafe("UserName"),
Password = entry.Strings.ReadSafe("Password"),
URL = entry.Strings.ReadSafe("URL"),
Notes = entry.Strings.ReadSafe("Notes")
};
var kproot = db.RootGroup.Groups;
string lastGroup = "#";
uint sc = 0;
int depth = 0;
var parent = "#"; //root is # parent
foreach (var entry in kproot) {
PwGroup pwGroup = db.RootGroup.Groups.GetAt(sc);
Web.Write(" { \"id\" : \"" + (sc) + "\", \"parent\" : \"" + parent + "\", \"text\" : \"" + pwGroup.Name.HtmlEncode() + "\" },\n");
WriteChildren(pwGroup,sc+"", depth + 1);
sc++;
}
db.Close();
}
public static void WriteChildren(PwGroup pwGroup, string parentID,int depth) {
uint sc = 0;
//if(depth>3)return; //used to prevent too much recursion
foreach (var entry in pwGroup.Groups) {
var subGroup = pwGroup.Groups.GetAt(sc);
var curID = (parentID+"_"+sc);
Web.Write(" { \"id\" : \"" + curID + "\", \"parent\" : \"" + parentID + "\", \"text\" : \"" + subGroup.Name.JsEncode() + "\"},\n");
WriteChildren(subGroup, curID, depth+1);
WriteLeaves(subGroup, curID, depth);
sc++;
}
}
public static void WriteLeaves(PwGroup pwGroup, string parentID,int depth) {
uint sc = 0;
//if(depth>3)return;
var entryList = pwGroup.GetEntries(false);
foreach (var entry in entryList) {
var curID = (parentID+"_"+sc);
Web.Write(" { \"id\" : \"" + curID + "\", \"parent\" : \"" + parentID + "\", \"text\" : \"" + entry.Strings.ReadSafe("Title").JsEncode() + "\", \"password\" : \"" + entry.Strings.ReadSafe("Password").JsEncode() + "\", \"type\" : \"file\"},\n");
sc++;
}
}

Check : KeePass Password Safe (For how keepass works)
Rather use the C# System.Cryptography classes and store you data enrypted in a database or txt file...
There is a KeePass-2.05-Alpha-Source.zip,The latest version of KeePass. C# source code,1919KB
http://s.pudn.com/upload_log_en.asp?e=1781366
http://en.pudn.com/downloads175/sourcecode/windows/other/detail816102_en.html