Develop Reference

System.OutOfMemoryException in C# when Generating huge amount of byte[] objects

I'm using this code to modify a pdf tmeplate to add specific details to it,
private static byte[] GeneratePdfFromPdfFile(byte[] file, string landingPage, string code)
{
try
{
using (var ms = new MemoryStream())
{
using (var reader = new PdfReader(file))
{
using (var stamper = new PdfStamper(reader, ms))
{
string _embeddedURL = "http://" + landingPage + "/Default.aspx?code=" + code + "&m=" + eventCode18;
PdfAction act = new PdfAction(_embeddedURL);
stamper.Writer.SetOpenAction(act);
stamper.Close();
reader.Close();
return ms.ToArray();
}
}
}
}
catch(Exception ex)
{
File.WriteAllText(HttpRuntime.AppDomainAppPath + #"AttachmentException.txt", ex.Message + ex.StackTrace);
return null;
}
}
this Method is being called from this Method:
public static byte[] GenerateAttachment(AttachmentExtenstion type, string Contents, string FileName, string code, string landingPage, bool zipped, byte[] File = null)
{
byte[] finalVal = null;
try
{
switch (type)
{
case AttachmentExtenstion.PDF:
finalVal = GeneratePdfFromPdfFile(File, landingPage, code);
break;
case AttachmentExtenstion.WordX:
case AttachmentExtenstion.Word:
finalVal = GenerateWordFromDocFile(File, code, landingPage);
break;
case AttachmentExtenstion.HTML:
finalVal = GenerateHtmlFile(Contents, code, landingPage);
break;
}
return zipped ? _getZippedFile(finalVal, FileName) : finalVal;
}
catch(Exception ex)
{
return null;
}
}
and here is the main caller,
foreach (var item in Recipients)
{
//...
//....
item.EmailAttachment = AttachmentGeneratorEngine.GenerateAttachment(_type, "", item.AttachmentName, item.CMPRCode, _cmpTmp.LandingDomain, _cmpTmp.AttachmentZip.Value, _cmpTmp.getFirstAttachment(item.Language, item.DefaultLanguage));
}
The AttachmentGeneratorEngine.GenerateAttachment method is being called approx. 4k times, because I'm adding a specific PDF file from a PDF template for every element in my List.
recently I started having this exception:
Exception of type 'System.OutOfMemoryException' was thrown. at System.IO.MemoryStream.ToArray()
I already implemented IDisposible in the classes and and I made sure that all of them are being released.
Note: it was running before very smoothely and also I double checked the system's resources - 9 GB is used out of 16 GB, so I had enough memory available.
==========================================
Update:
Here is the code that loops through the list
public static bool ProcessGroupLaunch(string groupCode, int customerId, string UilangCode)
{
CampaignGroup cmpGList = GetCampaignGroup(groupCode, customerId, UilangCode)[0];
_campaigns = GetCampaigns(groupCode, customerId);
List<CampaignRecipientLib> Recipients = GetGroupRcipientsToLaunch(cmpGList.ID, customerId);
try
{
foreach (var item in _campaigns)
item.Details = GetCampaignDetails(item.CampaignId.Value, UilangCode);
Stopwatch stopWatch = new Stopwatch();
#region single-threaded ForEach
foreach (var item in Recipients)
{
CampaignLib _cmpTmp = _campaigns.FirstOrDefault(x => x.CampaignId.Value == item.CampaignId);
bool IncludeAttachment = _cmpTmp.IncludeAttachment ?? false;
bool IncludeAttachmentDoubleBarrel = _cmpTmp.IncludeAttachmentDoubleBarrel ?? false;
if (IncludeAttachment)
{
if (_cmpTmp.AttachmentExtension.ToLower().Equals("doc") || (_cmpTmp.AttachmentExtension.ToLower().Equals("docx")))
_type = AttachmentGeneratorEngine.AttachmentExtenstion.Word;
else if (_cmpTmp.AttachmentExtension.ToLower().Equals("ppt") || (_cmpTmp.AttachmentExtension.ToLower().Equals("pptx")))
_type = AttachmentGeneratorEngine.AttachmentExtenstion.PowePoint;
else if (_cmpTmp.AttachmentExtension.ToLower().Equals("xls") || (_cmpTmp.AttachmentExtension.ToLower().Equals("xlsx")))
_type = AttachmentGeneratorEngine.AttachmentExtenstion.Excel;
else if (_cmpTmp.AttachmentExtension.ToLower().Equals("pdf"))
_type = AttachmentGeneratorEngine.AttachmentExtenstion.PDF;
else if (_cmpTmp.AttachmentExtension.ToLower().Equals("html"))
_type = AttachmentGeneratorEngine.AttachmentExtenstion.HTML;
}
//set "recpient" details
item.EmailFrom = _cmpTmp.EmailFromPrefix + "#" + _cmpTmp.EmailFromDomain;
item.EmailBody = GetChangedPlaceHolders((_cmpTmp.getBodybyLangCode(string.IsNullOrEmpty(item.Language) ? item.DefaultLanguage : item.Language, item.DefaultLanguage)), item.ID, _cmpTmp.CustomerId.Value, _cmpTmp.CampaignId.Value);
if (item.EmailBody.Contains("[T-LandingPageLink]"))
{
//..
}
if (item.EmailBody.Contains("[T-FeedbackLink]"))
{
//..
}
if (item.EmailBody.Contains("src=\".."))
{
//..
}
//set flags to be used by the SMTP Queue and Scheduler
item.ReadyTobeSent = true;
item.PickupReady = false;
//add attachment to the recipient, if any.
if (IncludeAttachment)
{
item.AttachmentName = _cmpTmp.getAttachmentSubjectbyLangCode(string.IsNullOrEmpty(item.Language) ? item.DefaultLanguage : item.Language, item.DefaultLanguage) + "." + _cmpTmp.AttachmentExtension.ToLower();
try
{
if (_type == AttachmentGeneratorEngine.AttachmentExtenstion.PDF || _type == AttachmentGeneratorEngine.AttachmentExtenstion.WordX || _type == AttachmentGeneratorEngine.AttachmentExtenstion.Word)
item.EmailAttachment = AttachmentGeneratorEngine.GenerateAttachment(_type, "", item.AttachmentName, item.CMPRCode, _cmpTmp.LandingDomain, _cmpTmp.AttachmentZip.Value, _cmpTmp.getFirstAttachment(item.Language, item.DefaultLanguage));
else item.EmailAttachment = AttachmentGeneratorEngine.GenerateAttachment(_type, value, item.AttachmentName, item.CMPRCode, _cmpTmp.LandingDomain, _cmpTmp.AttachmentZip.Value);
item.AttachmentName = _cmpTmp.AttachmentZip.Value ? (_cmpTmp.getAttachmentSubjectbyLangCode(string.IsNullOrEmpty(item.Language) ? item.DefaultLanguage : item.Language, item.DefaultLanguage) + ".zip") :
_cmpTmp.getAttachmentSubjectbyLangCode(string.IsNullOrEmpty(item.Language) ? item.DefaultLanguage : item.Language, item.DefaultLanguage) + "." + _cmpTmp.AttachmentExtension.ToLower();
}
catch (Exception ex)
{
}
}
else
{
item.EmailAttachment = null;
item.AttachmentName = null;
}
}
#endregion
stopWatch.Stop();
bool res = WriteCampaignRecipientsLaunch(ref Recipients);
return res;
}
catch (Exception ex)
{
Recipients.ForEach(i => i.Dispose());
cmpGList.Dispose();
Recipients = null;
cmpGList = null;
return false;
}
finally
{
Recipients.ForEach(i => i.Dispose());
cmpGList.Dispose();
Recipients = null;
cmpGList = null;
}
}

Cannot open file c#

Can someone tell me why I have this error on the below code?
The procces cannot access the file "..." beucase it's using by another procces.
I closed the first StreamReader and after, when I intialize the StreamWriter, it crashed.
private static void removeSetting(string _class)
{
try
{
string[] allSettings = new string[20];
int iSettings = 0;
using (StreamReader FILE_READER = new StreamReader("DATA.properties"))
{
string line = FILE_READER.ReadLine();
while (line != null)
{
if (!line.Equals(""))
{
allSettings[iSettings] = line;
iSettings++;
}
line = FILE_READER.ReadLine();
}
}
using (StreamWriter FILE_WRITER = new StreamWriter("DATA.properties", false))
{
for (int i = 0; i <= iSettings; i++)
{
if (!allSettings[i].Split('=')[0].Equals(_class))
{
FILE_WRITER.WriteLine('\n' + allSettings[i] + '\n');
//i--;
}
}
}
}
catch (Exception ex)
{
}
}
public static void saveSetting(string _class, string value)
{
removeSetting(_class);
try
{
StreamWriter FILE_WRITER = new StreamWriter("DATA.properties", true);
FILE_WRITER.WriteLine( _class.ToString() +'='+ value);
FILE_WRITER.Close();
}
catch (Exception ex)
{
}
}

Solution: you need find a program which open this file and kill it or reload OS.
I suggest you want change "_class=oldValue" on "_class=newValue" by using _class like this
oldText:
dog=lock
cat=bob
use saveSetting(dog, fork)
newText:
cat=bob
dog=fork
you can use this method
private static void removeSetting(string _class,string value)
{
try
{
//read all lines from file
string[] allSettings = File.ReadAllLines("DATA.properties");
//divide string on string[] by '='
List<List<string>> strings = allSettings.Select(x => x.Split(new char[] { '=' }, StringSplitOptions.RemoveEmptyEntries).ToList()).ToList();
//find all lines where first element equals _class
List<List<string>> result = strings.Where(x => x.First().Equals(_class)).Select(x=>new List<string>{_class, value}).ToList();
// convert string[] => string
string[] secondResult = result.Select(x => String.Join("=",x.ToArray())).ToArray();
List<List<string>> otherResult = strings.Where(x => !x.First().Equals(_class)).Select(x => x).ToList();
string[] firstResult = otherResult.Select(x => String.Join("=", x.ToArray())).ToArray();
//wrtie all lines in file
File.WriteAllLines("DATA.properties",firstResult);
File.AppendAllLines("DATA.properties", secondResult);
}
catch (Exception ex)
{
}
}

C# : line = StreamReader.ReadLine <= item

Working with reading a file that's formatted like this
2014/11/03 14:31:03 PID:8696 UUID:2ae855da-37d1-4a99-8510-27539afa09d0 Start E:/I3/IC/Logs/2014-11-03/SIPEngine_2.ininlog
2014/11/04 00:00:01 PID:8696 UUID:2ae855da-37d1-4a99-8510-27539afa09d0 End E:/I3/IC/Logs/2014-11-03/SIPEngine_2.ininlog
I am already stripping out the time in a variable but need to get be able to search the file for anything that is less than or equal to the value
Thanks to #dbc
Now I need to get the match on date
while (reader.next(msg, control))
{
ActiveAttributesVect_t attribs = msg.get_active_context_attributes();
ActiveAttributeValuesVect_t attribVals = msg.get_active_context_attribute_values();
int numAttribs = ((attribs != null) && (attribVals != null)) ? Math.Min(attribs.Count, attribVals.Count) : 0;
for (int i = 0; i < numAttribs; ++i)
{
if (attribVals[i].ToString().Contains(strCallID))
{
callidList.Add(msg.expand_format_message());
// Setting the date to match the log format
strCallDate = msg.timestamp().as_creator_time(header.tz_offset()).ToString().Substring(0, 10).Replace("-", "/");
strCallTime = msg.timestamp().as_creator_time(header.tz_offset()).ToString().Substring(11, 8);
Console.WriteLine("Call Time : {0}", strCallTime);
Console.WriteLine("Call Date : {0}", strCallDate);
Console.WriteLine("");
foreach (var entry in callidList)
{
Console.WriteLine(entry);
}
Console.WriteLine("");
OutputLogLinesBeforeTime(strLogDirectory, msg.timestamp().as_creator_time(header.tz_offset()), strCallTime);
}
}

You can use TimeSpan to extract and parse a time of day:
static TimeSpan? ExtractTime(string logLine)
{
var tokens = logLine.Split(new char [] { ' ', '\t' }, StringSplitOptions.RemoveEmptyEntries);
if (tokens.Length < 2)
return null;
TimeSpan time;
if (!TimeSpan.TryParse(tokens[1], out time))
return null;
return time;
}
static DateTime? ExtractDate(string logLine)
{
var tokens = logLine.Split(new char[] { ' ', '\t' }, StringSplitOptions.RemoveEmptyEntries);
if (tokens.Length < 1)
return null;
DateTime date;
if (!DateTime.TryParse(tokens[0], out date))
return null;
return date;
}
static void OutputLogLinesBeforeTime(string strLogDirectory, string strLogDate, string strCallTime)
{
try
{
var time = TimeSpan.Parse(strCallTime); // Throws a format exception if invalid.
DirectoryInfo d = new DirectoryInfo(strLogDirectory + "\\" + strLogDate + "\\");
foreach (var file in d.GetFiles("*.ininlog_journal"))
{
try
{
using (Stream stream = new FileStream(file.FullName, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
using (StreamReader sReader = new StreamReader(stream))
{
foreach (var line in sReader.EnumerateLines().Where(l => ExtractTime(l) < time))
Console.WriteLine(line);
}
}
catch (UnauthorizedAccessException ae)
{
Console.WriteLine(ae.Message);
}
catch (SystemException se)
{
Console.WriteLine(se.Message);
}
catch (ApplicationException ape)
{
Console.WriteLine(ape.Message);
}
catch (Exception e)
{
Console.WriteLine(e.Message);
}
}
}
catch (UnauthorizedAccessException ae)
{
Console.WriteLine(ae.Message);
}
catch (SystemException se)
{
Console.WriteLine(se.Message);
}
catch (ApplicationException ape)
{
Console.WriteLine(ape.Message);
}
catch (Exception e)
{
Console.WriteLine(e.Message);
}
}
For convenience I extracted the following:
public static class TextReaderExtensions
{
public static IEnumerable<string> ReadLines(this TextReader sReader)
{
if (sReader == null)
throw new ArgumentNullException();
string line;
while ((line = sReader.ReadLine()) != null)
yield return line;
}
}

How can I fix the following method?

I made a method which can scan an ftp server. It has two parts in an if...else statement. The first part in the if runs when the list of directories equals to 0, and separate the folders from the files( and put them into that list). Then(because the list is no more empty) the else statement should runs. It has a foreach loop which check all elements of the list and concat them to the ftp address and scan that folder. And here's the problem. Looks like it turn into an infinite loop. I only want to check the folders on the server and break the loop but looks like I cannot find a useful solution.
Here's the code:
internal void ListFilesOnServer()
{
ArrayList files = new ArrayList();
if (directories.Count == 0)
{
try
{
FtpWebRequest ftpwrq = (FtpWebRequest)WebRequest.Create(server);
ftpwrq.Credentials = new NetworkCredential(user, passw);
ftpwrq.Method = WebRequestMethods.Ftp.ListDirectory;
ftpwrq.KeepAlive = false;
FtpWebResponse fresponse = (FtpWebResponse)ftpwrq.GetResponse();
StreamReader sr = new StreamReader(fresponse.GetResponseStream());
string temp = "";
while ((temp = sr.ReadLine()) != null)
{
files.Add(temp);
}
temp = String.Empty;
sr.Close();
fresponse.Close();
DirOrFile(files);
}
catch (Exception e)
{
MessageBox.Show(e.Message);
}
}
else
{
foreach (string file in directories.ToArray())
{
try
{
FtpWebRequest ftpwrq = (FtpWebRequest)WebRequest.Create(server+"/"+file);
ftpwrq.Credentials = new NetworkCredential(user, passw);
ftpwrq.Method = WebRequestMethods.Ftp.ListDirectory;
ftpwrq.KeepAlive = false;
FtpWebResponse fresponse = (FtpWebResponse)ftpwrq.GetResponse();
StreamReader sr = new StreamReader(fresponse.GetResponseStream());
string temp = "";
while ((temp = sr.ReadLine()) != null)
{
files.Add(temp);
}
temp = String.Empty;
sr.Close();
fresponse.Close();
DirOrFile(files);
}
catch(ArgumentException)
{
}
catch (Exception e)
{
MessageBox.Show(e.Message);
}
}
}
ListFilesOnServer();
}

The infinite loop is because you don't have a way to break out of the recursion.
The pattern for recursion looks like this
MyRecursiveMethod()
{
if (conditions)
{
}
else
{
MyRecursiveMethod()
}
}
Here's how I might go about rewriting this code to make it work for you.
using System;
using System.Collections.Generic;
using System.IO;
using System.Net;
using System.Security;
using System.Windows.Forms;
namespace ConsoleApplication5
{
public class FtpTest
{
string server = "servier/";
string user = "user";
SecureString passw = new SecureString();
public List<string> GetFilesOnServer()
{
return GetFilesOnServer(server);
}
public List<string> GetFilesOnServer(string dir)
{
var root = GetDirectoryContents(dir);
var files = new List<string>();
foreach (string name in root)
{
var path = GetFullPath(dir, name);
if (IsDirectory(name))
{
var subFiles = GetFilesOnServer(path);
files.AddRange(subFiles);
}
else
{
files.Add(path);
}
}
return files;
}
public List<string> GetDirectoryContents(string dir)
{
var files = new List<string>();
try
{
var ftpwrq = (FtpWebRequest)WebRequest.Create(dir);
ftpwrq.Credentials = new NetworkCredential(user, passw);
ftpwrq.Method = WebRequestMethods.Ftp.ListDirectory;
ftpwrq.KeepAlive = false;
var fresponse = (FtpWebResponse)ftpwrq.GetResponse();
var sr = new StreamReader(fresponse.GetResponseStream());
string fileName = "";
while ((fileName = sr.ReadLine()) != null)
{
files.Add(fileName);
}
sr.Close();
fresponse.Close();
return files;
}
catch (ArgumentException)
{
}
catch (Exception e)
{
MessageBox.Show(e.Message);
}
return files;
}
public static string GetFullPath(string dir, string file)
{
string path = dir;
if (!path.EndsWith("/"))
path += "/";
path += file;
return path;
}
public static bool IsDirectory(string name)
{
return name.IndexOf(".") > 0;
}
}
}
Note that I only recursively call into GetFilesOnServer if the item retrieved is a directory. I've also refactored out the code that grabs the contents on the FTP server into a non-recursive method.
Hope this helps you out.

MySQL OdbcCommand commands sometimes hangs?

I am running a loop in C# that reads a file and make updates to the MySQL database with MySQL ODBC 5.1 driver in a Windows 8 64-bit environment.
The operations is simple
Count +1
See if file exists
Load XML file(XDocument)
Fetch data from XDocument
Open ODBCConnection
Run a couple of Stored Procedures against the MySQL database to store data
Close ODBCConnection
The problem is that after a while it will hang on for example a OdbcCommand.ExecuteNonQuery. It is not always the same SP that it will hang on?
This is a real problem, I need to loop 60 000 files but it only last around 1000 at a time.
Edit 1:
The problem seemse to accure here hever time :
public bool addPublisherToGame(int inPublisherId, int inGameId)
{
string sqlStr;
OdbcCommand commandObj;
try
{
sqlStr = "INSERT INTO games_publisher_binder (gameId, publisherId) VALUE(?,?)";
commandObj = new OdbcCommand(sqlStr, mainConnection);
commandObj.Parameters.Add("#gameId", OdbcType.Int).Value = inGameId;
commandObj.Parameters.Add("#publisherId", OdbcType.Int).Value = inPublisherId;
if (Convert.ToInt32(executeNonQuery(commandObj)) > 0)
return true;
else
return false;
}
catch (Exception ex)
{
throw (loggErrorMessage(this.ToString(), "addPublisherToGame", ex, -1, "", ""));
}
finally
{
}
}
protected object executeNonQuery(OdbcCommand inCommandObj)
{
try
{
//FileStream file = new FileStream("d:\\test.txt", FileMode.Append, FileAccess.Write);
//System.IO.StreamWriter stream = new System.IO.StreamWriter(file);
//stream.WriteLine(DateTime.Now.ToString() + " - " + inCommandObj.CommandText);
//stream.Close();
//file.Close();
//mainConnection.Open();
return inCommandObj.ExecuteNonQuery();
}
catch (Exception ex)
{
throw (ex);
}
}
I can see that the in parameters is correct
The open and close of the connection is done in a top method for ever loop (with finally).
Edit 2:
This is the method that will extract the information and save to database :
public Boolean addBoardgameToDatabase(XElement boardgame, GameFactory gameFactory)
{
int incomingGameId = -1;
XElement tmpElement;
string primaryName = string.Empty;
List<string> names = new List<string>();
GameStorage externalGameStorage;
int retry = 3;
try
{
if (boardgame.FirstAttribute != null &&
boardgame.FirstAttribute.Value != null)
{
while (retry > -1)
{
try
{
incomingGameId = int.Parse(boardgame.FirstAttribute.Value);
#region Find primary name
tmpElement = boardgame.Elements("name").Where(c => c.Attribute("primary") != null).FirstOrDefault(a => a.Attribute("primary").Value.Equals("true"));
if (tmpElement != null)
primaryName = tmpElement.Value;
else
return false;
#endregion
externalGameStorage = new GameStorage(incomingGameId,
primaryName,
string.Empty,
getDateTime("1/1/" + boardgame.Element("yearpublished").Value),
getInteger(boardgame.Element("minplayers").Value),
getInteger(boardgame.Element("maxplayers").Value),
boardgame.Element("playingtime").Value,
0, 0, false);
gameFactory.updateGame(externalGameStorage);
gameFactory.updateGameGrade(incomingGameId);
gameFactory.removeDesignersFromGame(externalGameStorage.id);
foreach (XElement designer in boardgame.Elements("boardgamedesigner"))
{
gameFactory.updateDesigner(int.Parse(designer.FirstAttribute.Value), designer.Value);
gameFactory.addDesignerToGame(int.Parse(designer.FirstAttribute.Value), externalGameStorage.id);
}
gameFactory.removePublishersFromGame(externalGameStorage.id);
foreach (XElement publisher in boardgame.Elements("boardgamepublisher"))
{
gameFactory.updatePublisher(int.Parse(publisher.FirstAttribute.Value), publisher.Value, string.Empty);
gameFactory.addPublisherToGame(int.Parse(publisher.FirstAttribute.Value), externalGameStorage.id);
}
foreach (XElement element in boardgame.Elements("name").Where(c => c.Attribute("primary") == null))
names.Add(element.Value);
gameFactory.removeGameNames(incomingGameId);
foreach (string name in names)
if (name != null && name.Length > 0)
gameFactory.addGameName(incomingGameId, name);
return true;
}
catch (Exception)
{
retry--;
if (retry < 0)
return false;
}
}
}
return false;
}
catch (Exception ex)
{
throw (new Exception(this.ToString() + ".addBoardgameToDatabase : " + ex.Message, ex));
}
}
And then we got one step higher, the method that will trigger addBoardgameToDatabase :
private void StartThreadToHandleXmlFile(int gameId)
{
FileInfo fileInfo;
XDocument xmlDoc;
Boolean gameAdded = false;
GameFactory gameFactory = new GameFactory();
try
{
fileInfo = new FileInfo(_directory + "\\" + gameId.ToString() + ".xml");
if (fileInfo.Exists)
{
xmlDoc = XDocument.Load(fileInfo.FullName);
if (addBoardgameToDatabase(xmlDoc.Element("boardgames").Element("boardgame"), gameFactory))
{
gameAdded = true;
fileInfo.Delete();
}
else
return;
}
if (!gameAdded)
{
gameFactory.InactivateGame(gameId);
fileInfo.Delete();
}
}
catch (Exception)
{ throw; }
finally
{
if(gameFactory != null)
gameFactory.CloseConnection();
}
}
And then finally the top level :
public void UpdateGames(string directory)
{
DirectoryInfo dirInfo;
FileInfo fileInfo;
Thread thread;
int gameIdToStartOn = 1;
dirInfo = new DirectoryInfo(directory);
if(dirInfo.Exists)
{
_directory = directory;
fileInfo = dirInfo.GetFiles("*.xml").OrderBy(c=> int.Parse(c.Name.Replace(".xml",""))).FirstOrDefault();
gameIdToStartOn = int.Parse(fileInfo.Name.Replace(".xml", ""));
for (int gameId = gameIdToStartOn; gameId < 500000; gameId++)
{
try
{ StartThreadToHandleXmlFile(gameId); }
catch(Exception){}
}
}
}

Use SQL connection pooling by adding "Pooling=true" to your connectionstring.
Make sure you properly close the connection AND the file.
You can create one large query and execute it only once, I think it is a lot faster then 60.000 loose queries!
Can you show a bit of your code?