Develop Reference

Download websites web pages in parallel

I have the following code and it works fine when I loop through one by one using foreach, but when I change it to use Parallel.ForEach I'm getting errors. Trying to figure out how I can correct this. FYI the dParms list contain unique id's.
The error I'm getting is
Error writing to path..... ErrorMessage:Item has already been added.
Key in dictionary: 'IIS_WasUrlRewritten' Key being added:
'IIS_WasUrlRewritten',
private void GeneratePages(RequestStatus response, string localDir, List<Parameters> dParms, int total, DateTime generatedTime)
{
int current = 0;
Parallel.ForEach(dParms, curJob =>
{
try
{
DownloadPage(localDir, curJob, generatedTime);
}
catch (Exception ex)
{
response.Status = false;
response.Message = ex.Message;
}
finally
{
Interlocked.Increment(ref current);
if (current % 10 == 0)
//to do: Send Progress to UI
}
//});
}
private string DownloadPage(string localDir, Parameters p, DateTime generatedTime)
{
string strExtension = "html";
string url = string.Empty;
url = this.Url.Action("MyAction", "Home", new { area = "", #id = p.MyId, #generatedTime = generatedTime }, this.Request.Url.Scheme);
var document = new HtmlWeb().Load(url);
string strFileName = url.Substring(url.LastIndexOf("/") + 1);
strFileName = strFileName.Substring(0, strFileName.IndexOf("generatedTime") - 1);
string strDiskFileName = strFileName.Replace(".aspx?", "");
strDiskFileName = strDiskFileName.Replace("?", "");
strDiskFileName = strDiskFileName.Replace(".aspx", "");
strDiskFileName = strDiskFileName.Replace("&", "");
strDiskFileName = strDiskFileName.Replace("=", "");
strDiskFileName = strDiskFileName.Replace("%20", "");
strDiskFileName += "." + strExtension;
document.Save(localDir + strDiskFileName);
return url;
}

Thread in a web service doesn't start when called it from windows service

I have a windows service,the codes in that body are in a elapsed timer method. I call some web services in a loop,that those web services run in a thread.
I checked the services on servers, they dont start.
And when I take out the codes from thread body,those worked.
This is my windows service code that calls webservices from servers.
private void CallServiceForTrancoding(object sender, System.Timers.ElapsedEventArgs e)
{
DataSet.TranscodingVideosDataTable oTranscodingVideosDataTable = new DataSet.TranscodingVideosDataTable();
DataSetTableAdapters.TranscodingVideoTableAdapter oTranscodingVideoTableAdapter = new DataSetTableAdapters.TranscodingVideoTableAdapter();
oTranscodingVideoTableAdapter.FillVideosForTranscoding(oTranscodingVideosDataTable);
DataSet.ServersDataTable oServersDataTable = new DataSet.ServersDataTable();
DataSetTableAdapters.ServersTableAdapter oServersTableAdapter = new DataSetTableAdapters.ServersTableAdapter();
oServersTableAdapter.FillBaseOnConvertAction(oServersDataTable);
for (int i = 0; i < oTranscodingVideosDataTable.Count; i++)
{
if (oTranscodingVideosDataTable[i].IsQualityNull() == false)
{
var Qualities = oTranscodingVideosDataTable[i].Quality.Split(new string[] { "," }, StringSplitOptions.RemoveEmptyEntries);
string path = oTranscodingVideosDataTable[i].Path + oTranscodingVideosDataTable[i].ObjectUniqueName;
string[] Url = new string[Qualities.Length];
for (int j = 0; j < Qualities.Length; j++)
{
if (j == 0)
{
Url[j] = path;
}
else
{
Url[j] = path.Replace(path.Split('/').Last(), "") + path.Split('/').Last().Split('.')[0] + "-" + Qualities[j] + "." + path.Split('/').Last().Split('.')[1];
}
}
string parameters = "Token=" + "#HelloWorld" +
"&Url=" + new JavaScriptSerializer().Serialize(Url) +
"&FileUniqueName=" + oTranscodingVideosDataTable[i].ObjectUniqueName +
"&ip=" + oTranscodingVideosDataTable[i].IPValid +
"&GregoreanDate=" + oTranscodingVideosDataTable[i].Gorean_Date.ToString();
DataSet.VideoDataTable oVideoDataTable = new DataSet.VideoDataTable();
DataSetTableAdapters.VideoTableAdapter oVideoTableAdapter = new DataSetTableAdapters.VideoTableAdapter();
oVideoTableAdapter.UpdateVideoStatus(oTranscodingVideosDataTable[i].ObjectUniqueName);
string WebServiceOfServer = oServersDataTable[i].WebService;
WebServiceOfServer = WebServiceOfServer.Replace(WebServiceOfServer.Split('/').Last(), "RS");
JavaScriptSerializer objSerializer = new JavaScriptSerializer();
//Task a = Task.Factory.StartNew(() => {
Task ResponseResult = GetPostMethods.POST(WebServiceOfServer, parameters);
string JSONResult = new System.Web.Script.Serialization.JavaScriptSerializer().Serialize(ResponseResult);
var ResponseString = objSerializer.Deserialize<dynamic>(JSONResult);
var UserToken = ResponseString["Result"]["UserToken"];
var Successfully = ResponseString["Result"]["Successfully"];
//});
}
}
}
Here is my web method :
[WebMethod]
public void RS(string Token, string Url, string FileUniqueName, string ip,string GregoreanDate)
{
new Thread(() =>
{
if (Token != "HelloWorld")
{
}
else
{
try
{
var VideoUrl = new JavaScriptSerializer().Deserialize<string[]>(Url);
string[] FileName = new string[VideoUrl.Length];
string[] FilePath = new string[VideoUrl.Length];
t.VideoUniqueName = FileUniqueName.Split('.')[0];
t.mediaOutPath = HttpContext.Current.Server.MapPath("/video/") + DateTime.Parse(GregoreanDate).Year + "\\" + DateTime.Parse(GregoreanDate).Month + "\\" + FileUniqueName.Split('.')[0] + "\\";
t.mediaOutFolderPath = "/video/" + DateTime.Parse(GregoreanDate).Year + "/" + DateTime.Parse(GregoreanDate).Month + "/";
if (!Directory.Exists(t.mediaOutPath))
{
Directory.CreateDirectory(t.mediaOutPath);
Directory.CreateDirectory(t.mediaOutPath + "m3u8\\");
}
t.CreateFTPPathForVideoScheduler(t.VideoTypeFormKey, t.mediaOutPath);
string Paths = "";
for (int i = 0; i < VideoUrl.Length; i++)
{
FileName[i] = VideoUrl[i].Split('/').Last();
if (FileName[i] != FileUniqueName)
{
FilePath[i] = t.mediaOutPath + FileName[i];
}
else
{
FilePath[i] = t.ffmpegFolderPath + FileName[i];
}
using (WebClient client = new WebClientWithAwesomeTimeouts { Timeout = new TimeSpan(10, 0, 0, 0) })
{
try
{
client.DownloadFile(VideoUrl[i], FilePath[i]);
client.Dispose();
}
catch (Exception ex)
{
StreamWriter sw = new StreamWriter(t.LogPath, true);
sw.WriteLine(ex.Message + "----" + ex.StackTrace != null ? ex.StackTrace : "" + "----" + DateTime.Now);
sw.Close();
}
}
}
DirectoryInfo FilesDirectory = new DirectoryInfo(t.ffmpegFolderPath);
FileInfo[] fInfo = FilesDirectory.GetFiles("*" + FileUniqueName.Split('.')[0] + "*.*");
fInfo[0].CopyTo(t.mediaOutPath + FileUniqueName, true);
t.SendFileWithFTPScheduler(t.VideoTypeFormKey, t.mediaOutPath + FileUniqueName);
FilesDirectory = new DirectoryInfo(t.mediaOutPath);
fInfo = FilesDirectory.GetFiles("*" + FileUniqueName.Split('.')[0] + "*.*");
foreach (FileInfo file in fInfo)
{
t.SendFileWithFTPScheduler(t.VideoTypeFormKey, file.FullName);
}
string UniqueFilePath = FilePath[0].Replace(FilePath[0].Split('/').Last(), FileUniqueName);
JavaScriptSerializer objSerializer = new JavaScriptSerializer();
var VideoDimensions = objSerializer.Deserialize<Dictionary<string, string>>(t.getVideoHeightAndWidth(t.ffmpegFolderPath + UniqueFilePath));
string Width = VideoDimensions["Width"];
string Height = VideoDimensions["Height"];
Transcoding.VideoResolutions vr = new Transcoding.VideoResolutions();
var VideoResolutionsInfo = objSerializer.Deserialize<Dictionary<object, object>>(t.getVideoResolution(Height));
vr.OrginalResolution = VideoResolutionsInfo["OrginalResolution"].ToString();
vr.ResCount = Convert.ToInt32(VideoResolutionsInfo["ResCount"]);
vr.OrginalResIndex = Convert.ToInt32(VideoResolutionsInfo["OrginalResIndex"]);
vr.UpperResolutionHeight = VideoResolutionsInfo["UpperResolutionHeight"].ToString();
t.MakeM3U8FormatScheduler(FileUniqueName, vr.OrginalResolution, vr.ResCount, vr.OrginalResIndex, false, "", Height);
t.ProcessFilesScheduler(t.mediaOutPath + "m3u8\\", t.VideoTypeFormKey);
DataSet.VideoDataTable oVideoDataTable = new DataSet.VideoDataTable();
DataSetTableAdapters.VideoTableAdapter oVideoTableAdapter = new DataSetTableAdapters.VideoTableAdapter();
oVideoTableAdapter.UpdateVideoForAcceptation(1, true, "mp4,m3u8", "http://" + t.mediaOutFolderPath + t.VideoUniqueName + "/", FileUniqueName);
oVideoTableAdapter.UpdateStatus("resfinish", FileUniqueName);
foreach (string path in VideoUrl)
{
if (path.ToLower().Contains("mp4videos"))
{
t.FTPDelete(new Uri(path.Replace(new Uri(path).Host, ip).Replace("http", "ftp")));
}
}
System.IO.File.Delete(t.ffmpegFolderPath + UniqueFilePath);
System.IO.Directory.Delete(t.mediaOutPath, true);
}
catch (Exception ex)
{
}
}
}).Start();
}

Task ResponseResult = GetPostMethods.POST(WebServiceOfServer, parameters);
string JSONResult = new System.Web.Script.Serialization.JavaScriptSerializer().Serialize(ResponseResult);
You are serializing a task. You surely want to wait for the result, don't you?

I debugged my web service and I found that the issue is related to httpcontext.current in thread is null .
For that I defined HttpContext.Current.Server.MapPath("/video/") as a variable in a class and I recalled it from the webservice.

alchemy api in c# for text extractor

I want to extract text from web page at a run time then, use alchemy api in asp.net with c# but I don't know how to use this api in c#. I am trying to find out what the parameter of text extractor is. If needed I can also try to regular expression for extracting the web page but this was not clean html tags.
private void Form3_Load(object sender, EventArgs e)
{
}
void GetPosition(Uri url, string searchTerm)
{
string raw = "http://www.google.co.in/search?num=39&q={0}&btnG=Search"; string search = string.Format(raw,
HttpUtility.UrlEncode(searchTerm)); HttpWebRequest request = (HttpWebRequest)WebRequest.Create(search); using (HttpWebResponse
response = (HttpWebResponse)request.GetResponse())
{
using (StreamReader
reader = new StreamReader(response.GetResponseStream(),
Encoding.ASCII))
{
string html = reader.ReadToEnd();
//return FindPosition(html, url);
fillgoogle(html);
}
}
}
//New Fill Google
void fillgoogle(string html)
{
listBox1.Items.Clear();
// string pattern = #"h3 class=";
string pattern = "<h3 class=\"r\"><a href=";
/*for (int i = 0; i < 10; i++)
{
int start;
int end;
int pos;
pos = html.IndexOf(pattern);
start = html.IndexOf("href=", pos);
end = html.IndexOf("/", start + 15);
ListBox1.Items.Add(prepare(html.Substring(start + 6, end - start)));
html = html.Substring(end);
}*/
// int start;
int end;
int pos;
// string[] strUrl;
pos = html.IndexOf(pattern);
string[] Arr = Regex.Split(html, pattern);
for (int x = 1; x <= Arr.Length - 1; x++)
{
//string find = Arr[x].ToString();
//string RealData=find.Substring
// listBox1.Items.Add(Arr[x].ToString());
end = Arr[x].IndexOf("/", 38);
str1 = Arr[x].Substring(0, end);
// strUrl = Regex.Split(Arr[1], "&");
//string n = string.Join("/url?q=", Arr);
str1 = str1.Replace('"', ' ');
str1 = str1.Trim();
str1 = str1.Remove(0, 7).ToString();
listBox1.Items.Add(str1);
// ListBox1.Items.Add(Arr[x].Substring(0, end));
if (x == 10)
{
break;
}
}
}
void finalList()
{
listBox2.Items.Clear();
for (int i = 0; i < listBox1.Items.Count; i++)
{
string Link = listBox1.Items[i].ToString();
if (Link.IndexOf("&") != -1)
{
int end = Link.IndexOf("&");
string real = Link.Substring(0, end);
listBox2.Items.Add(real);
//MessageBox.Show(real);
}
}
}
string prepare(string url)
{
string temp;
int i;
i = url.IndexOf("//");
int j;
j = url.IndexOf("/", i + 3);
temp = url.Substring(0, j);
return (temp);
}
private static int FindPosition(
string html, Uri url)
{// h3 class=\"r\"><a href=\"http://www.godaddy.com/\
string lookup = "(<h3 class=r><a href=\")(\\*)";
MatchCollection matches = Regex.Matches(html, lookup);
for (int i = 0; i < matches.Count; i++)
{
string match = matches[i].Groups[2].Value;
if (match.Contains(url.Host))
return i + 1;
} return 0;
}
private void button1_Click(object sender, EventArgs e)
{
richTextBox1.Text = "";
Uri url = new Uri("http://www.godaddy.com");
GetPosition(url, textBox1.Text);
finalList();
webPage page = new webPage();
page.URL = listBox2.Items[0].ToString();
page.Load(); //Load the text from the specified URL
label3.Visible = true;
linkLabel1.Visible = true;
label3.Text = listBox2.Items[0].ToString();
//Display the page TITLE on the screen
//richTextBox1.Text = "Title: " + page.Title + Environment.NewLine + Environment.NewLine;
//Display a list of INTERNAL links on the screen (to include external links, see below)
//richTextBox1.Text += "LINKS" + Environment.NewLine + "=====" + Environment.NewLine;
//foreach (String link in page.LinksArray)
//{
// richTextBox1.Text += link + Environment.NewLine;
//}
//Display the BODY TEXT on the screen
richTextBox1.Text += Environment.NewLine + page.Body;
//richTextBox1.Text += Environment.NewLine + page.Paragraph;
}
public class webPage
{
public String URL;
private String sTitle;
private String sBody;
private String sParagraph;
private ArrayList aList;
public String Title
{
get
{
return sTitle;
}
}
public ArrayList LinksArray
{
get
{
return aList;
}
}
public String Body
{
get
{
return sBody;
}
}
public String Paragraph
{
get
{
return sParagraph;
}
}
public void Load()
{
try
{
WebRequest objRequest = WebRequest.Create(this.URL);
WebResponse objResponse = objRequest.GetResponse();
StreamReader oSR = new StreamReader(objResponse.GetResponseStream());
string strContent = oSR.ReadToEnd();
this.sTitle = getTitle(strContent);
this.aList = fetchLinks(strContent, URL);
this.sBody = fetchText(strContent);
this.sParagraph = GetFirstParagraph(strContent);
}
catch (Exception e)
{
MessageBox.Show(e.ToString());
}
}
private String getTitle(String sHTMLContent)
{
//Retrieve the title from the HTML code
return Regex.Match(sHTMLContent, "<title>(?<title>[^<]+)</title>", RegexOptions.IgnoreCase).Groups["title"].ToString();
}
private ArrayList fetchLinks(String sHTMLContent, String sURL)
{
//Find all the links in the HTML code and put them
//into an array
Match mMatch;
ArrayList aMatch = new ArrayList();
mMatch = Regex.Match(sHTMLContent, "href\\s*=\\s*(?:\"(?<1>[^\"]*)\"|(?<1>\\S+))", RegexOptions.IgnoreCase);
while (mMatch.Success)
{
String sMatch = processURL(mMatch.Groups[1].ToString(), sURL);
//Currently, this code only lists INTERNAL URLs. If you would
//like to include EXTERNAL URLs as well, comment out the fol-
//lowing IF statement EXCEPT the "aMatch.Add(sMatch);" line
if (sMatch.IndexOf(sURL) >= 0 && checkFormat(sMatch))
{
aMatch.Add(sMatch);
}
mMatch = mMatch.NextMatch();
}
return aMatch;
}
static string GetFirstParagraph(string s)
{
Match m = Regex.Match(s, #"<p>\s*(.+?)\s*</p>");
if (m.Success)
{
return m.Groups[1].Value;
}
else
{
return "";
}
}
private String fetchText(String s)
{
//Filter out HTML and JavaScript from the page, leaving only body text
s = Convert.ToString(Regex.Match(s, #"<body.+?</body>", RegexOptions.Singleline | RegexOptions.IgnoreCase)); //strip everything but <BODY>
s = Regex.Replace(s, "<script[^>]*?>.*?</script>", "", RegexOptions.Singleline | RegexOptions.IgnoreCase); //strip JavaScript
s = Regex.Replace(s, "<[^>]*>", ""); //strip HTML tags
s = Regex.Replace(s, "&(copy|#169);|&(quot|#34);|&(amp|#38);|&(lt|#60);&(gt|#62);|&(nbsp|#160);|&(iexcl|#161);|&(cent|#162);|&(pound|#163);|·", " "); //strip symbols
s = s.Replace("\t", " "); //strip tabs
s = Regex.Replace(s, "([\r\n])+", " "); //strip carriage returns
s = Regex.Replace(s, "\\s\\s+", " "); //strip white space (must be last)
return s.Trim();
}
private String processURL(String sInput, String sURL)
{
sURL = "http://" + Convert.ToString(sURL.Split('/').GetValue(2));
if (sInput.IndexOf("http://") < 0)
{
if (!sInput.StartsWith("/") && !sURL.EndsWith("/"))
{
return sURL + "/" + sInput;
}
else
{
if (sInput.StartsWith("/") && sURL.EndsWith("/"))
{
return sURL.Substring(0, sURL.Length - 1) + sInput;
}
else
{
return sURL + sInput;
}
}
}
else
{
return sInput;
}
}
private bool checkFormat(String sURL)
{
//List only pages ending with valid extensions
String[] validExt = { ".html", ".php", ".asp", ".htm", ".jsp", ".shtml", ".php3", ".aspx", ".pl", ".cfm" };
sURL = Convert.ToString(sURL.Split('?').GetValue(0));
foreach (String ext in validExt)
{
if (sURL.Substring(sURL.Length - ext.Length, ext.Length).ToLower() == ext) { return true; }
}
return false;
}
}
private void linkLabel1_LinkClicked(object sender, LinkLabelLinkClickedEventArgs e)
{
System.Diagnostics.Process.Start(label3.Text);
}
}
}

I do not see any attempts to call Alchemy API in your example but here's what you need to know:
Alchemy API uses standard Web Service calls with XML responses as default. However you can specify the response you want (JSON / RDP).
Here's the start-up documentation, Text Extraction and Requirements and URLGetText Endpoint (but check on the documentation website for the endpoint you need).

Dynamic Crystal Report using Dataset, Microsoft Access Database

I'm trying to get a report from a custom query accessing to a Microsoft Access Database filling a Dataset and then a CrystalReportViwer but I just got an empty report with the column names.
This is my connection:
public String ConectionString = #"Provider=Microsoft.ACE.OLEDB.12.0;Data";
public OleDbConnection miconexion;
public Int32 retorna = 0;
public Int32 conectar(String location){
try
{
ConectionString += #" Source="+location;
miconexion = new OleDbConnection(ConectionString);
//Open Database Conexion
miconexion.Open();
retorna = 1;
}
catch (OleDbException ex){
retorna = 0;
MessageBox.Show("Database Error: "+ex.ToString());
}
return retorna;
}
public void desconectar() {
if(miconexion.State == ConnectionState.Open){
miconexion.Close();
}
}
Method that process the sql //---------------------------------
//txt_CustomConsult is the textbox that contains the custom query.
public String ExecuteSQl()
{
string sql = null;
string inSql = null;
string firstPart = null;
string LastPart = null;
int SelectStart = 0;
int fromStart = 0;
string[] fields = null;
string[] sep = { "," };
int i = 0;
TextObject MyText;
inSql = txt_CustomConsult.Text;
inSql = inSql.ToUpper();
SelectStart = inSql.IndexOf("SELECT");
fromStart = inSql.IndexOf("FROM");
SelectStart = SelectStart + 6;
firstPart = inSql.Substring(SelectStart, (fromStart - SelectStart));
LastPart = inSql.Substring(fromStart, inSql.Length - fromStart);
fields = firstPart.Split(',');
firstPart = "";
for (i = 0; i <= fields.Length - 1; i++) {
if (i > 0)
{
firstPart = firstPart + ", " + fields[i].ToString() + " AS COLUMN" + (i + 1);
firstPart.Trim();
MyText = (TextObject)ObjRep.ReportDefinition.ReportObjects[i + 1];
MyText.Text = fields[i].ToString();
}
else {
firstPart = firstPart + fields[i].ToString() + " AS COLUMN"+(i+1);
firstPart.Trim();
MyText = (TextObject)ObjRep.ReportDefinition.ReportObjects[i + 1];
MyText.Text = fields[i].ToString();
}
}
sql = "SELECT "+firstPart+" "+LastPart;
return sql;
}
// ---------------------------------------------------
Method that exceute report. lstbx_Tablas is the ListBox that contains all tables,
I got all the tables but report doesn't work.
public void DynamicRep() {
try {
//Windows Form where the rpt is located
ReporteResult mirepres = new ReporteResult();
//Connection
conexion miconect = new conexion();
miconect.conectar(Environment.GetEnvironmentVariable("dbUbicacion"));
String sql = ExecuteSQl();
OleDbDataAdapter dscm = new OleDbDataAdapter(sql,miconect.miconexion);
MisConsultas Dataset1 = new MisConsultas();
dscm.Fill(Dataset1,lstbx_Tablas.SelectedItem.ToString());
ObjRep.SetDataSource(Dataset1.Tables[0]);
mirepres.crv_Reporte.ReportSource = ObjRep;
mirepres.crv_Reporte.Refresh();
mirepres.Show();
}
catch (Exception ex) { MessageBox.Show(ex.ToString()); }
}

C# 2.0 Fastest way to parse Excel spreadsheet [duplicate]

This question already has answers here:
Closed 10 years ago.
Possible Duplicate:
Reading Excel files from C#
What is the fastest way to read large sets of data from excel from Csharp. Example code would be great . .

In our desktop environment, I have reached the best mix between performance, flexibility and stability by using Excel via COM.
Access to Excel is always via the same thread.
I use late-binding (in VB.Net) to make my app version independent.
The rest of the application is developed in C#, only this part and some other small parts are in VB, because they are easier in VB.Net.
Dim workBook As Object = GetObject(fileName)
Dim workSheet As Object = workBook.WorkSheets.Item(WorkSheetNr)
Dim range As Object = workSheet.Cells.Item(1, 1)
Dim range2 As Object = range.CurrentRegion
Dim rrow As Integer = range2.Row ' For XL97, first convert to integer. XL97 will generate an error '
Dim rcolumn As Integer = range2.Column
Dim top As Object = workSheet.Cells.Item(rrow, rcolumn)
Dim bottom As Object = top.Offset(range2.Rows.Count - 1, range2.Columns.Count - 1)
range = workSheet.Range(top, bottom)
Dim values As Object(,)
values = range.Value
Here you have a 2-dimensional array containing the values from Excel. The last statement gets the data from Excel to .Net.
Since the limits on the size of a Excel sheet, these cannot get very large, so memory should not be a problem.
We have done some tests on performance, on multiple systems. It is optimized to create as few as possible (out-of-process) COM calls.
This way was the one that has given us the best performance, specially since the data is directly in an array, and access to this data is faster as going through a dataset.
Slow in this solution is starting Excel. But if you need to process multiple files, right after each other, the cost of starting Excel is made only once.
Also I would not use this solution in a server environment.

public class ExcelHeaderValues
{
public static string CUSIP = "CUSIP";
public static string ORIG = "ORIG";
public static string PRICE = "PRICE";
public static int COLUMNCOUNT = 3;
}
public class ExcelParser
{
private ArrayList collOutput = null;
string sSheetName = String.Empty;
string[] strValidColumn;
int validRowCnt = 0;
string sColumnPositions = String.Empty;
OleDbCommand ExcelCommand;
OleDbDataAdapter ExcelAdapter;
OleDbConnection ExcelConnection = null;
DataSet dsSheet = null;
string path = string.Empty;
string identifier = string.Empty;
public ExcelParser()
{
collOutput = new ArrayList();
}
public void Extract()
{
bool headermatch = false;
string strCusip = string.Empty, strOrig = string.Empty, strPrice = string.Empty, strData = string.Empty;
string strCusipPos = string.Empty, strPricePos = string.Empty, strOrigPos = string.Empty;
string strColumnHeader = String.Empty;
int reqColcount = 0;
string[] strTemp;
bool bTemp = false;
bool validRow = false;
DataTable schemaTable = GetSchemaTable();
validRowCnt = 0;
foreach (DataRow dr in schemaTable.Rows)
{
if (dsSheet != null)
{
dsSheet.Reset();
dsSheet = null;
}
strCusipPos = string.Empty;
strOrigPos = string.Empty;
strPricePos = string.Empty;
if (isValidSheet(dr))
{
sColumnPositions = string.Empty;
validRowCnt = 0;
foreach (DataRow dataRow in dsSheet.Tables[0].Rows)
{
sColumnPositions = string.Empty;
if (headermatch == false)
{
sColumnPositions = string.Empty;
foreach (DataColumn column in dsSheet.Tables[0].Columns)
{
strColumnHeader = dataRow[column.ColumnName].ToString().ToUpper().Trim();
strColumnHeader = strColumnHeader.ToUpper();
if (strColumnHeader == ExcelHeaderValues.ORIG.ToUpper() || strColumnHeader == ExcelHeaderValues.CUSIP.ToUpper() || strColumnHeader == ExcelHeaderValues.PRICE.ToUpper())
{
bTemp = true;
validRow = true;
reqColcount = ExcelHeaderValues.COLUMNCOUNT;
}
if (bTemp)
{
bTemp = false;
sColumnPositions += column.ColumnName + "^" + strColumnHeader + ";";
}
}
strValidColumn = sColumnPositions.Trim().Split(';');
if (validRow == true && reqColcount == strValidColumn.Length - 1)
{
headermatch = true;
break;
}
validRowCnt++;
}
}
if (headermatch == true)
{
try
{
if (dsSheet.Tables[0].Rows.Count > 0)
{
if (strValidColumn.Length > 0)
{
for (int i = 0; i < strValidColumn.Length - 1; i++)
{
if (strValidColumn[i].ToUpper().Contains("CUSIP"))
{
strTemp = strValidColumn[i].ToString().Split('^');
strCusipPos = strTemp[0].ToString();
strTemp = null;
}
if (strValidColumn[i].ToUpper().Contains("PRICE"))
{
strTemp = strValidColumn[i].ToString().Split('^');
strPricePos = strTemp[0].ToString();
strTemp = null;
}
if (strValidColumn[i].ToUpper().Contains("ORIG"))
{
strTemp = strValidColumn[i].ToString().Split('^');
strOrigPos = strTemp[0].ToString();
strTemp = null;
}
}
}
for (int iData = validRowCnt; iData < dsSheet.Tables[0].Rows.Count; iData++)
{
if (strCusipPos.Trim() != "")
strCusip = dsSheet.Tables[0].Rows[iData][strCusipPos].ToString().Trim();
if (strOrigPos.Trim() != "")
strOrig = dsSheet.Tables[0].Rows[iData][strOrigPos].ToString().Trim();
if (strPricePos.Trim() != "")
strPrice = dsSheet.Tables[0].Rows[iData][strPricePos].ToString().Trim().ToUpper();
strData = "";
if (strCusip.Length == 9 && strCusip != "" && strPrice != "" && strOrig != "" && !strPrice.ToUpper().Contains("SOLD"))
strData = strCusip + "|" + Convert.ToDouble(strOrig) * 1000000 + "|" + strPrice + "|||||";
if (strData != null && strData != "")
collOutput.Add(strData);
strCusip = string.Empty;
strOrig = string.Empty;
strPrice = string.Empty;
strData = string.Empty;
}
}
}
catch (Exception ex)
{
throw ex;
}
}
headermatch = false;
sColumnPositions = string.Empty;
strColumnHeader = string.Empty;
}
}
}
private bool isValidSheet(DataRow dr)
{
bool isValidSheet = false;
sSheetName = dr[2].ToString().ToUpper();
if (!(sSheetName.Contains("$_")) && !(sSheetName.Contains("$'_")) && (!sSheetName.Contains("Print_Titles".ToUpper())) && (dr[3].ToString() == "TABLE" && ((!sSheetName.Contains("Print_Area".ToUpper())))) && !(sSheetName.ToUpper() == "DLOFFERLOOKUP"))
{
if (sSheetName.Trim().ToUpper() != "Disclaimer$".ToUpper() && sSheetName.Trim().ToUpper() != "Summary$".ToUpper() && sSheetName.Trim().ToUpper() != "FORMULAS$" )
{
string sQry = string.Empty;
sQry = "SELECT * FROM [" + sSheetName + "]";
ExcelCommand = new OleDbCommand(sQry, ExcelConnection);
dsSheet = new DataSet();
ExcelAdapter = new OleDbDataAdapter(ExcelCommand);
isValidSheet = false;
try
{
ExcelAdapter.Fill(dsSheet, sSheetName);
isValidSheet = true;
}
catch (Exception ex)
{
isValidSheet = false;
throw new Exception(ex.Message.ToString());
}
finally
{
ExcelAdapter = null;
ExcelCommand = null;
}
}
}
return isValidSheet;
}
private DataTable GetSchemaTable()
{
DataTable dt = null;
string connectionString = String.Empty;
connectionString = GetConnectionString();
ExcelConnection = new OleDbConnection(connectionString);
try
{
ExcelConnection.Open();
dt = ExcelConnection.GetOleDbSchemaTable(OleDbSchemaGuid.Tables, new object[] { null, null, null, "TABLE" });
}
catch (Exception ex)
{
throw ex;
}
return dt;
}
private string GetConnectionString()
{
string connStr = String.Empty;
try
{
if (path.ToLower().Contains(".xlsx"))
{
connStr = "Provider=Microsoft.ACE.OLEDB.12.0;" + "Data Source='" + path + "';" + "Extended Properties='Excel 12.0;HDR=No;IMEX=1;'";
}
else if (path.ToLower().Contains(".xlsm"))
{
connStr = "Provider=Microsoft.ACE.OLEDB.12.0;" + "Data Source='" + path + "';" + "Extended Properties='Excel 12.0 Macro;HDR=No;IMEX=1;'";
}
else if (path.ToLower().Contains(".xls"))
{
connStr = "provider=Microsoft.Jet.OLEDB.4.0;Data Source='" + path + "';Extended Properties='Excel 8.0;HDR=No;IMEX=1;'";
}
else
{connStr = "Provider=Microsoft.ACE.OLEDB.12.0;" + "Data Source='" + path + "';" + "Extended Properties='HTML Import;IMEX=1;HDR=No;'";
}
}
catch (Exception ex)
{
throw ex;
}
return connStr;
}
}