I'm using an MDF database with the OpenFileDialog class to import a single CSV file to a database. That works fine for a single CSV file, but I need to open and process multiple CSV files in bulk.
How can I improve my code? I've tried using a for loop.
Here is my code to process a single CSV file:
ofd.Filter = "CSV files (*.csv) | *.csv; | CSV PRN files (*.prn,) |*.prn;";
ofd.FileName = "";
ofd.ShowDialog();
DataTable dt = new DataTable();
string line = null;
int i = 0;
using (StreamReader sr = File.OpenText(ofd.FileName))
{
while ((line = sr.ReadLine()) != null)
{
string[] data = line.Split(',');
if (data.Length > 0)
{
if (i == 0)
{
foreach (var item in data)
{
dt.Columns.Add(new DataColumn());
}
i++;
}
DataRow row = dt.NewRow();
row.ItemArray = data;
dt.Rows.Add(row);
}
}
}
string symbolName = dt.Rows[1][0].ToString();
string strConnection =
#"Data Source =.\SQLEXPRESS; AttachDbFilename = C:\USERS\JEF\DOCUMENTS\DATABASE1.MDF; Integrated Security = True; Connect Timeout = 30; User Instance = True";
SqlConnection condb2 = new SqlConnection(strConnection);
string createTablerow ="create table ["+symbolName+"] (code1 VARCHAR(100) COLLATE Arabic_CI_AI_KS_WS,date1 varchar(50),open1 varchar(50),high1 varchar(50),low1 varchar(50),close1 varchar(50),vol1 varchar(50))";
using (SqlConnection connection = new SqlConnection(strConnection))
{
SqlCommand command1 = new SqlCommand(createTablerow, connection);
connection.Open();
command1.ExecuteNonQuery();
}
using (SqlConnection cn = new SqlConnection(strConnection))
{
cn.Open();
using (SqlBulkCopy copy = new SqlBulkCopy(cn))
{
copy.ColumnMappings.Add(0, "code1");
copy.ColumnMappings.Add(1, "date1");
copy.ColumnMappings.Add(2, "open1");
copy.ColumnMappings.Add(3, "high1");
copy.ColumnMappings.Add(4, "low1");
copy.ColumnMappings.Add(5, "close1");
copy.ColumnMappings.Add(6, "vol1");
copy.DestinationTableName = "[" + symbolName + "]";
copy.WriteToServer(dt);
}
}
I moved some of you code around to make it more efficient :
using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Data;
using System.Drawing;
using System.Linq;
using System.Text;
using System.Windows.Forms;
using System.IO;
using System.Data.SqlClient;
namespace WindowsFormsApplication1
{
public partial class Form1 : Form
{
public Form1()
{
InitializeComponent();
ofd.FileName = "";
ofd.ShowDialog();
string line = null;
int i = 0;
string strConnection =
#"Data Source =.\SQLEXPRESS; AttachDbFilename = C:\USERS\JEF\DOCUMENTS\DATABASE1.MDF; Integrated Security = True; Connect Timeout = 30; User Instance = True";
using (SqlConnection connection = new SqlConnection(strConnection))
{
connection.Open();
SqlBulkCopy copy = new SqlBulkCopy(connection);
copy.ColumnMappings.Add(0, "code1");
copy.ColumnMappings.Add(1, "date1");
copy.ColumnMappings.Add(2, "open1");
copy.ColumnMappings.Add(3, "high1");
copy.ColumnMappings.Add(4, "low1");
copy.ColumnMappings.Add(5, "close1");
copy.ColumnMappings.Add(6, "vol1");
foreach (string file in ofd.FileNames)
{
using (StreamReader sr = File.OpenText(file))
{
DataTable dt = new DataTable();
while ((line = sr.ReadLine()) != null)
{
string[] data = line.Split(',');
if (data.Length > 0)
{
if (i == 0)
{
foreach (var item in data)
{
dt.Columns.Add(new DataColumn());
}
i++;
}
DataRow row = dt.NewRow();
row.ItemArray = data;
dt.Rows.Add(row);
}
}
string symbolName = dt.Rows[1][0].ToString();
string createTable = string.Format("create table [{0}] (code1 VARCHAR(100) COLLATE Arabic_CI_AI_KS_WS,date1 varchar(50),open1 varchar(50),high1 varchar(50),low1 varchar(50),close1 varchar(50),vol1 varchar(50))",
symbolName);
using (SqlCommand command1 = new SqlCommand(createTable, connection))
{
command1.ExecuteNonQuery();
copy.DestinationTableName = "[" + symbolName + "]";
copy.WriteToServer(dt);
}
}
}
}
}
}
}
In order to process multiple files at once you will need to use a parallel code. Your use of "for" was a nice try, but "for" loops still run concurrently, which means one at a time.
The easiest way to achieve this parallel processing in this case would be to use Parallel.Foreach, here is a Microsoft guide about it: https://learn.microsoft.com/en-us/dotnet/standard/parallel-programming/how-to-write-a-simple-parallel-foreach-loop
Related
I have a .csv file and I'd like to read it into a datagridview (each value into an each column).
I read this file with block note and I see that each value is divided by ";"
I tried to set a datatable but it's not working. This is my code:
string FileName = #"C:\mydir\testcsv.csv";
OleDbConnection conn = new OleDbConnection("Provider=Microsoft.Jet.OleDb.4.0; Data Source = " + Path.GetDirectoryName(FileName) + "; Extended Properties = \"Text;HDR=YES;FMT=Delimited\"");
conn.Open();
OleDbDataAdapter adapter = new OleDbDataAdapter("SELECT * FROM " + Path.GetFileName(FileName), conn);
DataSet ds = new DataSet("Temp");
adapter.Fill(ds);
conn.Close();
dataGridView2.DataSource = ds;
I don't understand where's the error.
Your code worked for me as it is.
I just added one line to the datasource assignment after looking inside the dataset, I saw just one table is inside with name "Table" so I assigned the datamember of the datagridview:
dataGridView1.DataSource = ds;
dataGridView1.DataMember = "Table";
Anyway if I used ';' separator, all the values were in one column... With ',' comma separator it works ok.
The complete code of the form:
using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Data;
using System.Data.OleDb;
using System.Drawing;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.Windows.Forms;
namespace WindowsFormsApplication1
{
public partial class Form1 : Form
{
public Form1()
{
InitializeComponent();
}
private void Form1_Load(object sender, EventArgs e)
{
string FileName = #"C:\mydir\testcsv.csv";
OleDbConnection conn = new OleDbConnection
("Provider=Microsoft.Jet.OleDb.4.0; Data Source = " +
Path.GetDirectoryName(FileName) +
"; Extended Properties = \"Text;HDR=YES;FMT=Delimited\"");
conn.Open();
OleDbDataAdapter adapter = new OleDbDataAdapter
("SELECT * FROM " + Path.GetFileName(FileName), conn);
DataSet ds = new DataSet("Temp");
adapter.Fill(ds);
conn.Close();
dataGridView1.DataSource = ds;
dataGridView1.DataMember = "Table";
}
}
}
Contents of the csv file:
abc,123
def,456
ijk,789
lmn,111213
For semicolon delimited files you need to add an ini file in your folder containing the csv file. How to do it exactly is described here:
How to specify the delimiter when importing CSV files via OLEDB in C#
For decimal delimiter symbol you have to add the
DecimalSymbol
directive to your Jet ini file.
See the full ini file capabilities documented in MSDN (https://msdn.microsoft.com/en-us/library/ms709353(v=vs.85).aspx)
I use this function by long, long time, after: yourgrid.datasource = function result.
public static DataTable CsvDb(string filename, string separatorChar)
{
var table = new DataTable("Filecsv");
using (var sr = new StreamReader(filename, Encoding.Default))
{
string line;
var i = 0;
while (sr.Peek() >= 0)
{
try
{
line = sr.ReadLine();
if (string.IsNullOrEmpty(line)) continue;
var values = line.Split(new[] { separatorChar }, StringSplitOptions.None);
var row = table.NewRow();
for (var colNum = 0; colNum < values.Length; colNum++)
{
var value = values[colNum];
if (i == 0)
{
table.Columns.Add(value, typeof(String));
}
else
{ row[table.Columns[colNum]] = value; }
}
if (i != 0) table.Rows.Add(row);
}
catch (Exception ex)
{
string cErr = ex.Message;
//if you need the message error
}
i++;
}
}
return table;
}
Try...
So far I can do an easy data grab from the spreadsheet, just getting the ref number row but I currently don't know how to match the row to the data section before getting the correct data out.
I currently have to extract some data from the excel spreadsheet example below:
Start date Ref number
29/07/2015 2342326
01/07/2016 5697455
02/08/2016 3453787
02/08/2016 5345355
02/08/2015 8364456
03/08/2016 1479789
04/07/2015 9334578
The main question is would it be possible to read in the data from a set date, from the row and get the ref number form the set date e.g. Start date.
For example if i just wanted the data from the date set to the 1st of last month and upwards.
How would this be best implemented.
Example of current code used to get the column, using basic OleDb:
using System;
using System.Data.OleDb;
using System.Text.RegularExpressions;
namespace Number_Cleaner
{
public class NumberCleanerReport
{
public void runExcel_Report()
{
Console.ForegroundColor = ConsoleColor.Green;
Console.WriteLine("[*][START OF: NumberExt.xls, Number Extraction]");
Console.ResetColor();
string con =
#"Provider=Microsoft.Jet.OLEDB.4.0;Data Source=NumberExt.xls;" +
#"Extended Properties='Excel 8.0;HDR=Yes;'";
string connectionString = ExcelWriter.GetConnectionString();
using (OleDbConnection conn = new OleDbConnection(connectionString))
{
conn.Open();
OleDbCommand cmd = new OleDbCommand();
cmd.Connection = conn;
using (OleDbConnection connection = new OleDbConnection(con))
{
connection.Open();
OleDbCommand command = new OleDbCommand("select * from [Sheet1$]", connection);
System.IO.StreamWriter files = new System.IO.StreamWriter(Controller.fpath + "NumberExtOutput.txt");
using (OleDbDataReader dr = command.ExecuteReader())
{
while (dr.Read())
{
var row1Col0 = dr[0];
string ExcelData = row1Col0.ToString();
string subStr = "null";
try
{
subStr = ExcelData.Substring(0, 6);
}
catch
{
//Console.WriteLine("Found Nulls.");
}
if (subStr == "00")
{
string result = Regex.Replace(ExcelData, "^00", "0");
Console.WriteLine(result);
files.WriteLine(result);
cmd.CommandText = "INSERT INTO [table1]('MainNmbers') VALUES(" + result + ");";
cmd.ExecuteNonQuery();
}
}
files.Close();
conn.Close();
Console.ForegroundColor = ConsoleColor.Green;
Console.WriteLine("[*][END OF: NumberExt.xls, RefNumber Extraction]");
Console.ResetColor();
}
}
}
}
}
}
You could load it into a DataTable and then filter it with Linq-To-DataTable. Here's a method that reads all as strings. You could modify it so that it parse the first column to DateTime and the second to int with DateTime.Parse or DateTime.ParseExact and int.Parse:
public static DataTable GetDataTableFromExcel(string path, bool hasHeader = true)
{
using (var pck = new OfficeOpenXml.ExcelPackage())
{
using (var stream = File.OpenRead(path))
{
pck.Load(stream);
}
var ws = pck.Workbook.Worksheets.First();
DataTable tbl = new DataTable();
foreach (var firstRowCell in ws.Cells[1, 1, 1, ws.Dimension.End.Column])
{
tbl.Columns.Add(hasHeader ? firstRowCell.Text : string.Format("Column {0}", firstRowCell.Start.Column));
}
var startRow = hasHeader ? 2 : 1;
for (int rowNum = startRow; rowNum <= ws.Dimension.End.Row; rowNum++)
{
var wsRow = ws.Cells[rowNum, 1, rowNum, ws.Dimension.End.Column];
DataRow row = tbl.Rows.Add();
foreach (var cell in wsRow)
{
row[cell.Start.Column - 1] = cell.Text;
}
}
return tbl;
}
}
Once you have the correct types in the table the query is simple:
var rowsOfInterest = table.AsEnumerable()
.Where(row => row.Field<DateTime>("Start date") >= new DateTime(2016, 7, 1))
.ToList();
If you need it as a DataTable:
DataTable resultTable = table.Clone(); // empty table with correct columns
if(rowsOfInterest.Count > 0)
resultTable = rowsOfInterest.CopyToDataTable();
I got the error message below when I tried to upload a csv file to a SQL Server table using C# (csv file has no header).
Error Message : "A column named ' ' already belongs to this DataTable"
I tried to find some solutions somewhere on the web but I'm really stuck with it.
My code :
SqlConnection con = new SqlConnection(#"server=.;Initial Catalog=myDtabase;Integrated Security=SSPI;");
string filepath = #"c:\\my_CSV_file.csv";
StreamReader sr = new StreamReader(filepath);
string line = sr.ReadLine();
string[] value = line.Split(',');
DataTable dt = new DataTable();
DataRow row;
foreach (string dc in value)
{
dt.Columns.Add(new DataColumn(dc));
}
while (!sr.EndOfStream)
{
value = sr.ReadLine().Split(',');
if (value.Length == dt.Columns.Count)
{
row = dt.NewRow();
row.ItemArray = value;
dt.Rows.Add(row);
}
}
SqlBulkCopy bc = new SqlBulkCopy(con.ConnectionString, SqlBulkCopyOptions.TableLock);
bc.DestinationTableName = "my_SQLServer_Table";
bc.BatchSize = dt.Rows.Count;
con.Open();
bc.WriteToServer(dt);
bc.Close();
con.Close();
I think this link will help you get this done.
http://forums.asp.net/t/1695615.aspx
As usual, there is more than one way to skin a cat. So, if yo don't like the solution listed above, try this script, which I know will work for you.
using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Data;
using System.Drawing;
using System.Linq;
using System.Text;
using System.Windows.Forms;
using System.IO;
using System.Data.SqlClient;
using System.Data.OleDb;
using System.Configuration;
namespace WindowsFormsApplication1
{
public partial class Form1 : Form
{
public Form1()
{
InitializeComponent();
}
private void button1_Click(object sender, EventArgs e)
{
string server = "EXCEL-PC\\EXCELDEVELOPER";
string database = "AdventureWorksLT2012";
string SQLServerConnectionString = String.Format("Data Source={0};Initial Catalog={1};Integrated Security=SSPI", server, database);
string CSVpath = #"C:\Users\Ryan\Documents\Visual Studio 2010\Projects\Bulk Copy from CSV to SQL Server Table\WindowsFormsApplication1\bin"; // CSV file Path
string CSVFileConnectionString = String.Format("Provider=Microsoft.Jet.OLEDB.4.0;Data Source={0};;Extended Properties=\"text;HDR=Yes;FMT=Delimited\";", CSVpath);
var AllFiles = new DirectoryInfo(CSVpath).GetFiles("*.CSV");
string File_Name = string.Empty;
foreach (var file in AllFiles)
{
try
{
DataTable dt = new DataTable();
using (OleDbConnection con = new OleDbConnection(CSVFileConnectionString))
{
con.Open();
var csvQuery = string.Format("select * from [{0}]", file.Name);
using (OleDbDataAdapter da = new OleDbDataAdapter(csvQuery, con))
{
da.Fill(dt);
}
}
using (SqlBulkCopy bulkCopy = new SqlBulkCopy(SQLServerConnectionString))
{
bulkCopy.ColumnMappings.Add(0, "MyGroup");
bulkCopy.ColumnMappings.Add(1, "ID");
bulkCopy.ColumnMappings.Add(2, "Name");
bulkCopy.ColumnMappings.Add(3, "Address");
bulkCopy.ColumnMappings.Add(4, "Country");
bulkCopy.DestinationTableName = "AllEmployees";
bulkCopy.BatchSize = 0;
bulkCopy.WriteToServer(dt);
bulkCopy.Close();
}
}
catch(Exception ex)
{
MessageBox.Show(ex.Message, "Warning!", MessageBoxButtons.OK, MessageBoxIcon.Exclamation);
}
}
}
}
}
The CsvHelper NuGet library has an implementation for IDataReader which you can pass into the BulkCopy WriteToServer method. This makes for really simple code and allows you to customize the data import.
using CsvHelper;
using Microsoft.Data.SqlClient;
using Microsoft.EntityFrameworkCore;
using System.Globalization;
using System.IO;
public int ReplaceTableWithFile(string table, FileInfo csvFile)
{
using var fileReader = new StreamReader(csvFile.OpenRead());
using var csv = new CsvReader(fileReader, CultureInfo.InvariantCulture);
using var csvDataReader = new CsvDataReader(csv);
var connection = GetDbConnection();
using var command = new SqlBulkCopy(connection);
command.EnableStreaming = true;
command.DestinationTableName = table;
command.WriteToServer(csvDataReader);
return command.RowsCopied;
}
CsvDataReader from CsvHelper NuGet package
I have the following code:
using MySql.Data.MySqlClient;
using System;
using System.Collections.Generic;
using System.Data;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace TimeClock
{
class Company
{
DataTable rows = new DataTable();
public Company()
{
MySqlConnection connection = null;
try
{
string connectionString = TimeClock.Properties.Settings.Default.timeclockConnectionString;
connection = new MySqlConnection(connectionString);
connection.Open();
MySqlCommand command = new MySqlCommand("SELECT * FROM companies WHERE ID = #ID LIMIT 1", connection);
command.Parameters.AddWithValue("#ID", TimeClock.Properties.Settings.Default.CompanyID);
MySqlDataAdapter da = new MySqlDataAdapter(command);
da.Fill(rows);
}
catch (MySql.Data.MySqlClient.MySqlException ex)
{
Console.WriteLine(ex);
}
finally
{
if (connection != null)
{
connection.Close();
}
}
}
public String getName()
{
DataRow row = rows.Rows[0];
return row["company_name"].ToString();
}
}
}
I know why I'm getting this error: say that no records were found in the database, the of course row[0] will no exist, hence the exception. But how do I deal with when there are no records to store in the Datatable? By the way, I'm quite new to C# and any input would be great; feel free to criticize.
Before you access a collection
DataRow row = rows.Rows[0];
you'll have to make sure that the item exists:
if(rows.Count > 0)
DataRow row = rows.Rows[0];
always
You must change getName function.
public String getName()
{
if (rows.Rows != null && rows.Rows.Count > 0)
{
DataRow row = rows.Rows[0];
return row["company_name"].ToString();
}
return string.Empty;
}
To read data from SQL I'm doing it like so:
using (SqlCommand SelectCommand = new SqlCommand(strbSelect.ToString()))
{
SelectCommand.Parameters.AddWithValue("Asset", AssetNumber);
SelectCommand.Parameters.AddWithValue("Subnumber", Subnumber);
SelectCommand.Connection = new SqlConnection(GetConnectionString());
SelectCommand.Connection.Open();
using (SqlDataReader Reader = SelectCommand.ExecuteReader())
{
if (Reader.HasRows)
{
while (Reader.Read())
{
if (Reader[0] != DBNull.Value)
{
ReturnValue = Reader.GetBoolean(0);
}
}
}
else
return false;
}
SelectCommand.Connection.Close();
}
StrbSelect is a StringBuilder.
I'm currently extracting data from an xml file and copying it to store in the database. Is there a way for me to add my own values during SqlBulkCopy? I want to add my own value itemsTotalPrice as I want to calculated the total price of the items but can't get it to work. How do I get around this?
DataTable sourceData = ds.Tables["Invoice"];
sourceData.Columns.Add("itemsTotalPrice", typeof(System.Decimal));
sourceData.Columns.Add("invoiceTotal", typeof(System.Decimal));
sqlConn.Open();
using (SqlBulkCopy sqlbc = new SqlBulkCopy(sqlConn))
{
sqlbc.DestinationTableName = "Invoice";
sqlbc.ColumnMappings.Add("invoiceDate", "invoiceDate");
sqlbc.ColumnMappings.Add("orderID", "orderID");
sqlbc.ColumnMappings.Add("sellerID", "sellerID");
sqlbc.ColumnMappings.Add("itemsTotalPrice", itemsTotalPrice);
//sqlbc.ColumnMappings.Add("shippingCharges", "shippingCharges");
sqlbc.ColumnMappings.Add("invoiceTotal", "invoiceTotal");
sqlbc.WriteToServer(sourceData);
}
You can try by loading the Xml into a DataTable, modify values in columns according to your choice, and try use WriteToServer.
reference code.
if (FileUploadControl.HasFile)
{
try
{
if (FileUploadControl.PostedFile.ContentType == "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" || FileUploadControl.PostedFile.ContentType == "application/vnd.ms-excel")
{
if (FileUploadControl.PostedFile.ContentLength >= 0)
{
string filename = Path.GetFileNameWithoutExtension(FileUploadControl.FileName);
string fileExt = Path.GetExtension(FileUploadControl.FileName);
FileUploadControl.SaveAs(Server.MapPath("~/DownloadedExcelFilesOp4/myfile" + fileExt));
if (FileUploadControl.PostedFile.ContentLength == 0)
{
lblMsg.Text = fun.UploadStatus;
lblMsg.ForeColor = Color.Green;
}
else
lblMsg.Text = fun.UploadStatus;
lblMsg.ForeColor = Color.Green;
string excelConnectionString = string.Format("Provider=Microsoft.ACE.OLEDB.12.0;Data Source={0};Extended Properties='Excel 8.0;HRD=YES;IMEX=1'", Server.MapPath(#"~\DownloadedExcelFilesOp4\myfile" + fileExt));// + "\\" + FileUploadControl.PostedFile.FileName.ToString());
using (OleDbConnection connection = new OleDbConnection(excelConnectionString))
{
OleDbCommand command = new OleDbCommand(("Select [Customer] ,[InvoiceDate] , [InvoiceNo] , [CustomerPo],[SoLine] ,[VendorName] ,[Category] ,[Item] ,[PickQty] ,[Price] ,[PriceExtentio] FROM [Sheet1$]"), connection);
connection.Open();
using (DbDataReader dr = command.ExecuteReader())
{
using (SqlBulkCopy bulkCopy = new SqlBulkCopy(ConfigurationManager.ConnectionStrings["ConString"].ConnectionString))
{
bulkCopy.DestinationTableName = "TempExamQuation4Master";
bulkCopy.ColumnMappings.Add("Customer", "Customer");
bulkCopy.ColumnMappings.Add("InvoiceDate", "InvoiceDate");
bulkCopy.ColumnMappings.Add("InvoiceNo", "InvoiceNo");
bulkCopy.ColumnMappings.Add("CustomerPo", "CustomerPo");
bulkCopy.ColumnMappings.Add("SoLine", "SoLine");
bulkCopy.ColumnMappings.Add("VendorName", "VendorName");
bulkCopy.ColumnMappings.Add("Categor", "Categor");
bulkCopy.ColumnMappings.Add("Item", "Item");
bulkCopy.ColumnMappings.Add("PickQty", "PickQty");
bulkCopy.ColumnMappings.Add("Price", "Price");
bulkCopy.ColumnMappings.Add("PriceExtentio", "PriceExtentio");
bulkCopy.WriteToServer(dr);
}
}
}