Related
I've been developing a C# WPF project with VS2015 using SQL Server Express LocalDb with Entity Framework. I have built a custom seeder for the database, that reads test data from an Excel file, that simply combines the Excel data into a command string, and this is inserted using context.Database.ExecuteSQLCommand.
Now, I was thinking of launching the project with SQL Server Compact Edition 4.0, but I find this command is not working anymore. Do I have to write my uploader again using SqlCeConnection and SqlCeCommand or am I missing something?
Also, from somewhere I have understood that with EF you can switch the SQL provider and the code would not need other changes. Am I in for more surprises down the road?
Example of the uploader command:
string cmd = "INSERT INTO Venues(Name, City, Telephone) Values ('X','Y','Z')"
context.Database.ExecuteSqlCommand(cmd);
The error:
There was an error parsing the query. [ Token line number = 2,Token line offset = 1,Token in error = INSERT ]
This is not just a testing issue, as I would want to include this uploader in the production version, too, for quick inserting of master data (e.g. employee list).
EDIT: Uploader code. If this can be done without resorting to raw SQL, that would be a good solution, too.
This loops through Excel sheets (named after entities) and columns (first row has property name) and rows 2->n (data). This handles the upload of basically any amount of data within Excel limitations. The point is that the code has no knowledge of the entities (might have been possible to parameterize DataContext too). Code might not be optimal, as I'm just a beginner, but has worked for me, except not with SQL CE. Editing to suit CE is not a big issue, but I wanted to ask for possibly better ways.
public static class ExcelUploader
{
static ArrayList data;
static List<string> tableNames;
public static string Upload(string filePath)
{
string result = "";
data = new ArrayList();
tableNames = new List<string>();
ArrayList upLoadData = ReadFile(filePath);
List<string> dataList = ArrayListToStringList(upLoadData);
using (var db = new DataContext())
{
using (var trans = db.Database.BeginTransaction())
{
try
{
foreach (var cmd in dataList)
{
Console.WriteLine(cmd);
db.Database.ExecuteSqlCommand(cmd);
}
db.SaveChanges();
trans.Commit();
}
catch (Exception e)
{
trans.Rollback();
result = e.Message;
MessageBox.Show(result);
}
}
}
return result;
}
private static ArrayList ReadFile(string fileName)
{
List<string> commands = new List<string>();
var xlApp = new Microsoft.Office.Interop.Excel.Application();
var wb = xlApp.Workbooks.Open(fileName, ReadOnly: true);
xlApp.Visible = false;
foreach (Worksheet ws in wb.Worksheets)
{
var r = ws.UsedRange;
var array = r.Value;
data.Add(array);
tableNames.Add(ws.Name);
}
wb.Close(SaveChanges: false);
xlApp.Quit();
return data;
}
private static List<string> ArrayListToStringList(ArrayList arrList)
{
List<string> result = new List<string>();
for(int tableAmount = 0;tableAmount<data.Count;tableAmount++)
{
result.Add(ArrayToSqlCommand(arrList[tableAmount] as Array, tableNames[tableAmount]));
}
return result;
}
private static string ArrayToSqlCommand(Array arr, string tableName)
{
int propertyRow = 1;
int firstDataRow = 2;
string command = "";
// loop rows
for (int rowIndex = firstDataRow; rowIndex <= arr.GetUpperBound(0); rowIndex++)
{
command += "INSERT INTO " + tableName + "(";
//add column names
for (int colIndex = 1; colIndex <= arr.GetUpperBound(1); colIndex++)
{
//get property name
command += arr.GetValue(propertyRow, colIndex);
//add comma if not last column, otherwise close bracket
if (colIndex == arr.GetUpperBound(1))
{
command += ") Values (";
}
else
{
command += ", ";
}
}
//add values
for (int colIndex = 1; colIndex <= arr.GetUpperBound(1); colIndex++)
{
//get property value
command += "'" + arr.GetValue(rowIndex, colIndex) + "'";
//add comma if not last column, otherwise close bracket
if (colIndex == arr.GetUpperBound(1))
{
command += ");";
}
else
{
command += ", ";
}
}
command += "\n";
}
return command;
}
}
There are two ways to use raw SQL queries I'd offer.
Initial data
1) Excel table
+=======+=======+===========+
| Name | City | Telephone |
|===========================|
| Adam | Addr1 | 111-11-11 |
|-------|-------|-----------|
| Peter | Addr2 | 222-22-22 |
+-------+-------+-----------+
2) SQL Server CE table
CREATE TABLE Venues
(
Id int identity primary key,
[Name] nvarchar(100) null,
City nvarchar(100) null,
Telephone nvarchar(100) null
);
3) Getting data from Excel
Here we're interested in getting array from Excel sheet. As soon as we get it, we can safely close Excel. The code assumes file "Employees.xlsx" to be next to executable file.
private object[,] GetExcelData()
{
xlApp = new Excel.Application { Visible = false };
var xlBook =
xlApp.Workbooks.Open(System.IO.Path.Combine(
Environment.CurrentDirectory,
"Employees.xlsx"));
var xlSheet = xlBook.Sheets[1] as Excel.Worksheet;
// For process termination
var xlHwnd = new IntPtr(xlApp.Hwnd);
var xlProc = Process.GetProcesses()
.Where(p => p.MainWindowHandle == xlHwnd)
.First();
// Get Excel data: it's 2-D array with lower bounds as 1.
object[,] arr = xlSheet.Range["A1"].CurrentRegion.Value;
// Shutdown Excel
xlBook.Close();
xlApp.Quit();
xlProc.Kill();
GC.Collect();
GC.WaitForFullGCComplete();
return arr;
}
Now you can use one of the ways to generate query.
Option 1. Use ExecuteSqlCommand
When using ExecuteSqlCommand, it's advisable to use parameterized queries to avoid errors. You can pass explicitly created SqlCeParameter or just pass a value.
private void UseExecuteSqlCommand()
{
object[,] arr = GetExcelData();
using (var db = new EmpContext())
{
db.Database.Initialize(true);
int count = 0;
string sql = "INSERT INTO Venues (Name, City, Telephone) " +
"VALUES (#name, #city, #phone);";
// Start from 2-nd row since we need to skip header
for (int r = 2; r <= arr.GetUpperBound(0); ++r)
{
db.Database.ExecuteSqlCommand(
sql,
new SqlCeParameter("#name", (string)arr[r, 1]),
new SqlCeParameter("#city", (string)arr[r, 2]),
new SqlCeParameter("#phone", (string)arr[r, 3])
);
++count;
}
conn.Close();
MessageBox.Show($"{count} records were saved.");
}
}
Option 2. Use DbConnection
If you want your code to be more generic, you can create method which would accept DbConnection. This will allow to pass either SqlConnection or SqlCeConnection. But the code becomes more verbose because we can't use constructors since these classes are abstract.
private void UseDbConnection()
{
object[,] arr = GetExcelData();
using (var db = new EmpContext())
{
db.Database.Initialize(true);
int count = 0;
string sql = "INSERT INTO Venues (Name, City, Telephone) " +
"VALUES (#name, #city, #phone);";
DbParameter param = null;
DbConnection conn = db.Database.Connection;
conn.Open();
DbCommand command = conn.CreateCommand();
command.CommandText = sql;
command.CommandType = CommandType.Text;
// Create parameters
// Name
param = command.CreateParameter();
param.ParameterName = "#name";
command.Parameters.Add(param);
// City
param = command.CreateParameter();
param.ParameterName = "#city";
command.Parameters.Add(param);
// Telephone
param = command.CreateParameter();
param.ParameterName = "#phone";
command.Parameters.Add(param);
// Start from 2-nd row since we need to skip header
for (int r = 2; r <= arr.GetUpperBound(0); ++r)
{
command.Parameters["#name"].Value = (string)arr[r, 1];
command.Parameters["#city"].Value = (string)arr[r, 2];
command.Parameters["#phone"].Value = (string)arr[r, 3];
command.ExecuteNonQuery();
++count;
}
conn.Close();
MessageBox.Show($"{count} records were saved.");
}
}
You can also use ordinal positions for parameters which eliminates creating parameters names and makes code much shorter:
private void UseDbConnection()
{
object[,] arr = GetExcelData();
using (var db = new EmpContext())
{
db.Database.Initialize(true);
int count = 0;
// Take a note - use '?' as parameters
string sql = "INSERT INTO Venues (Name, City, Telephone) " +
"VALUES (?, ?, ?);";
DbConnection conn = db.Database.Connection;
conn.Open();
DbCommand command = conn.CreateCommand();
command.CommandText = sql;
command.CommandType = CommandType.Text;
// Create parameters
command.Parameters.Add(command.CreateParameter());
command.Parameters.Add(command.CreateParameter());
command.Parameters.Add(command.CreateParameter());
for (int r = 2; r <= arr.GetUpperBound(0); ++r)
{
// Access parameters by position
command.Parameters[0].Value = (string)arr[r, 1];
command.Parameters[1].Value = (string)arr[r, 2];
command.Parameters[2].Value = (string)arr[r, 3];
command.ExecuteNonQuery();
++count;
}
conn.Close();
MessageBox.Show($"{count} records were saved.");
}
}
P.S.
I didn't check whether the underlying connection is opened, but it's a good idea to do so.
Based on JohnyL's excellent input, I was able to modify my code so that it works with either SQL Server Express and and SQL Server CE. I'll put my new code as an answer, as I had to parameterize it further, as I couldn't write the property names in the code either. But this was a simple step, once I got the idea from JohnyL. Not sure though, if the database writing operation should be wrapped inside a DbTransaction, but this worked for now.
public static class ExcelUploader
{
static ArrayList data;
static List<string> tableNames;
static List<DbCommand> cmdList = new List<DbCommand>();
static DbConnection conn;
public static void Upload(string filePath)
{
data = new ArrayList();
tableNames = new List<string>();
//get Excel data to array list
ArrayList upLoadData = ReadFile(filePath);
using (var db = new DataContext())
{
conn = db.Database.Connection;
//transform arraylist into a list of DbCommands
ArrayListToCommandList(upLoadData);
conn.Open();
try
{
foreach (var cmd in cmdList)
{
//Console.WriteLine(cmd.CommandText);
cmd.ExecuteNonQuery();
}
}
catch (Exception e)
{
var result = e.Message;
MessageBox.Show(result);
}
}
}
//opens Excel file and reads worksheets to arraylist
private static ArrayList ReadFile(string fileName)
{
List<string> commands = new List<string>();
var xlApp = new Microsoft.Office.Interop.Excel.Application();
var wb = xlApp.Workbooks.Open(fileName, ReadOnly: true);
xlApp.Visible = false;
foreach (Worksheet ws in wb.Worksheets)
{
var r = ws.UsedRange;
var array = r.Value;
data.Add(array);
tableNames.Add(ws.Name);
}
wb.Close(SaveChanges: false);
xlApp.Quit();
return data;
}
//transforms arraylist to a list of DbCommands
private static void ArrayListToCommandList(ArrayList arrList)
{
List<DbCommand> result = new List<DbCommand>();
for (int tableAmount = 0; tableAmount < data.Count; tableAmount++)
{
ArrayToSqlCommands(arrList[tableAmount] as Array, tableNames[tableAmount]);
}
}
private static void ArrayToSqlCommands(Array arr, string tableName)
{
//Excel row which holds property names
int propertyRow = 1;
//First Excel row with values
int firstDataRow = 2;
string sql = "";
DbCommand cmd = conn.CreateCommand();
sql += "INSERT INTO " + tableName + "(";
//add column names to command text
for (int colIndex = 1; colIndex <= arr.GetUpperBound(1); colIndex++)
{
//get property name
sql += arr.GetValue(propertyRow, colIndex);
//add comma if not last column, otherwise close bracket
if (colIndex == arr.GetUpperBound(1))
{
sql += ") Values (";
}
else
{
sql += ", ";
}
}
//add value parameter names to command text
for (int colIndex = 1; colIndex <= arr.GetUpperBound(1); colIndex++)
{
//get property name
sql += "#" + arr.GetValue(propertyRow, colIndex);
//add comma if not last column, otherwise close bracket
if (colIndex == arr.GetUpperBound(1))
{
sql += ");";
}
else
{
sql += ", ";
}
}
//add data elements as command parameter values
for (int rowIndex = firstDataRow; rowIndex <= arr.GetUpperBound(0); rowIndex++)
{
//initialize command
cmd = conn.CreateCommand();
cmd.CommandText = sql;
cmd.CommandType = CommandType.Text;
for (int colIndex = 1; colIndex <= arr.GetUpperBound(1); colIndex++)
{
//set parameter values
DbParameter param = null;
param = cmd.CreateParameter();
param.ParameterName = "#" + (string)arr.GetValue(propertyRow, colIndex);
cmd.Parameters.Add(param);
cmd.Parameters[param.ParameterName].Value = arr.GetValue(rowIndex, colIndex);
}
//add command to command list
cmdList.Add(cmd);
}
}
}
I am getting data from excel and showing it in DataGridWiew.
I have two textboxes, one is for starting index for first record and other is for last record.
Code works fine. But lets suppose starting record is 1 and ending is 10 when I change 10 to 1 or 2 it gives me an error in this line:
adapter.Fill(dataTable);
Full Code is below:
public DataSet Parse(string fileName)
{
string connectionString = string.Format("provider = Microsoft.Jet.OLEDB.4.0; data source = {0}; Extended Properties = Excel 8.0;", fileName);
DataSet data = new DataSet();
foreach (var sheetName in GetExcelSheetNames(connectionString))
{
using (OleDbConnection con = new OleDbConnection(connectionString))
{
string query = "";
var dataTable = new DataTable();
if(tbStarting.Text.Trim()=="" && tbEnding.Text.Trim() == "")
{
query = string.Format("SELECT * FROM [{0}]", sheetName);
}
else
{
query = string.Format("SELECT * FROM [{0}] where SrNo between " + int.Parse(tbStarting.Text.Trim()) + " and " + int.Parse(tbEnding.Text.Trim()) + " order by SrNo", sheetName);
}
con.Open();
OleDbDataAdapter adapter = new OleDbDataAdapter(query, con);
adapter.Fill(dataTable);
data.Tables.Add(dataTable);
con.Close();
}
}
return data;
}
static string[] GetExcelSheetNames(string connectionString)
{
OleDbConnection con = null;
DataTable dt = null;
con = new OleDbConnection(connectionString);
con.Open();
dt = con.GetOleDbSchemaTable(OleDbSchemaGuid.Tables, null);
if (dt == null)
{
return null;
}
String[] excelSheetNames = new String[dt.Rows.Count];
int i = 0;
foreach (DataRow row in dt.Rows)
{
excelSheetNames[i] = row["TABLE_NAME"].ToString();
i++;
}
return excelSheetNames;
}
Why this is happening please help me?
Looking at the code, it seems that your procedure is working when you ask to retrieve all the record in each table. But you are not showing which table (Sheet) is actually used afterwars.
Chances are, you are using the first one only.
When you submit some parameters, only one of the tables (Sheets) can fulfill those requirements. The other(s) don't, possibly because a field named [SrNo] is not present.
This causes the More Parameters Required error when trying to apply a filter.
Not related to the error, but worth noting: you don't need to recreate the whole DataSet + DataTables to filter your DataSources.
The DataSet.Tables[N].DefaultView.RowFilter can be used to get the same result without destroying all the objects each time a filter is required.
RowFilter has some limitations in the language (e.g. does not support BETWEEN, Field >= Value1 AND Field <= Value2 must be used), but it's quite effective.
This is a possible setup:
(xDataSet is a placeholder for your actual DataSet)
//Collect the values in the TextBoxes in a string array
private void button1_Click(object sender, EventArgs e)
{
string[] Ranges = new string[] { tbStarting.Text.Trim(), tbEnding.Text.Trim() };
if (xDataSet != null)
FilterDataset(Ranges);
}
private void FilterDataset(string[] Ranges)
{
if (string.IsNullOrEmpty(Ranges[0]) & string.IsNullOrEmpty(Ranges[1]))
xDataSet.Tables[0].DefaultView.RowFilter = null;
else if (string.IsNullOrEmpty(Ranges[0]) | string.IsNullOrEmpty(Ranges[1]))
return;
else if (int.Parse(Ranges[0]) < int.Parse(Ranges[1]))
xDataSet.Tables[0].DefaultView.RowFilter = string.Format("SrNo >= {0} AND SrNo <= {1}", Ranges[0], Ranges[1]);
else
xDataSet.Tables[0].DefaultView.RowFilter = string.Format("SrNo = {0}", Ranges[0]);
this.dataGridView1.Update();
}
I've modified your code you code a bit to handle those requirements.
(I've left here those filters anyway; they're not used, but if you still want them, they are in a working condition)
DataSet xDataSet = new DataSet();
string WorkBookPath = #"[Excel WorkBook Path]";
//Query one Sheet only. More can be added if necessary
string[] WBSheetsNames = new string[] { "Sheet1" };
//Open the Excel document and assign the DataSource to a dataGridView
xDataSet = Parse(WorkBookPath, WBSheetsNames, null);
dataGridView1.DataSource = xDataSet.Tables[0];
dataGridView1.Refresh();
public DataSet Parse(string fileName, string[] WorkSheets, string[] ranges)
{
if (!File.Exists(fileName)) return null;
string connectionString = string.Format("provider = Microsoft.ACE.OLEDB.12.0; " +
"data source = {0}; " +
"Extended Properties = \"Excel 12.0;HDR=YES\"",
fileName);
DataSet data = new DataSet();
string query = string.Empty;
foreach (string sheetName in GetExcelSheetNames(connectionString))
{
foreach (string WorkSheet in WorkSheets)
if (sheetName == (WorkSheet + "$"))
{
using (OleDbConnection con = new OleDbConnection(connectionString))
{
DataTable dataTable = new DataTable();
if ((ranges == null) ||
(string.IsNullOrEmpty(ranges[0]) || string.IsNullOrEmpty(ranges[1])) ||
(int.Parse(ranges[0]) > int.Parse(ranges[1])))
query = string.Format("SELECT * FROM [{0}]", sheetName);
else if ((int.Parse(ranges[0]) == int.Parse(ranges[1])))
query = string.Format("SELECT * FROM [{0}] WHERE SrNo = {1}", sheetName, ranges[0]);
else
query = string.Format("SELECT * FROM [{0}] WHERE (SrNo BETWEEN {1} AND {2}) " +
"ORDER BY SrNo", sheetName, ranges[0], ranges[1]);
con.Open();
OleDbDataAdapter adapter = new OleDbDataAdapter(query, con);
adapter.Fill(dataTable);
data.Tables.Add(dataTable);
};
}
}
return data;
}
static string[] GetExcelSheetNames(string connectionString)
{
string[] excelSheetNames = null;
using (OleDbConnection con = new OleDbConnection(connectionString))
{
con.Open();
using (DataTable dt = con.GetOleDbSchemaTable(OleDbSchemaGuid.Tables, null))
{
if (dt != null)
{
excelSheetNames = new string[dt.Rows.Count];
for (int i = 0; i < dt.Rows.Count; i++)
{
excelSheetNames[i] = dt.Rows[i]["TABLE_NAME"].ToString();
}
}
}
}
return excelSheetNames;
}
I am trying to bulk copy from one table to another by mapping the column names as the source and destination may not have same columns always.
Source can have 8 columns and destination can have 10 .. I need to map the columns and bulk copy.
Tried the below code..didn't work..getting Error: The given ColumnName 'moduleid' does not match up with any column in data source.
Source: existingtablecolumnsPresent has [collection time],[logtime],[moduleid],[node],[reason],[time],[timestamp],[usecaseid]
Destination: dataTable.Columns has [Node],[Time],[Reason],[Moduleid],[Usecaseid]
Please advise
public static void BatchBulkCopy(DataTable dataTable, string DestinationTbl, List<string> columnMapping,string filename)
{
var program = new Program();
// Get the DataTable
DataTable dtInsertRows = dataTable;
using (SqlBulkCopy sbc = new SqlBulkCopy(program.connectionStr.ToString()))
{
try {
sbc.DestinationTableName = DestinationTbl.ToLower();
string sourceTableQuery = "Select top 1 * from " + "[" + dataTable.TableName + "]";
DataTable dtSource = SqlHelper.ExecuteDataset(program.connectionStr.ToString(), CommandType.Text, sourceTableQuery).Tables[0];
for (int i = 0; i < dataTable.Columns.Count; i++)
{ //check if destination Column Exists in Source table
if (dtSource.Columns.Cast<DataColumn>().Select(a => "[" + a.ColumnName.ToLower() + "]").Contains(dataTable.Columns[i].ToString().ToLower()))//contain method is not case sensitive
{
List<string> existingtablecolumnsPresent = dtSource.Columns.Cast<DataColumn>().Select(a => "[" + a.ColumnName.ToLower() + "]").Distinct().OrderBy(t => t).ToList();
int sourceColumnIndex = existingtablecolumnsPresent.IndexOf(dataTable.Columns[i].ToString().ToLower());//Once column matched get its index
sbc.ColumnMappings.Add(dtSource.Columns[sourceColumnIndex].ToString(), dtSource.Columns[sourceColumnIndex].ToString());//give coluns name of source table rather then destination table so that it would avoid case sensitivity
}
}
sbc.WriteToServer(dtInsertRows);
sbc.Close();
}
catch (Exception ex)
{
Log.WriteLog("BatchBulkCopy" + " - " + filename, dataTable.TableName, ex.Message.ToString());
// To move a file or folder to a new location:
//if (File.Exists(program.sourceFile + filename))
// System.IO.File.Move(program.sourceFile + filename, program.failedfiles + filename);
}
As requested (create a DataTable with the columns you want to insert in them- leave the others out. Make sure any columns you leave out are marked in the table for NULL or have a DEFAULT VALUE constraint (I can't show you how to do that unless you show me your table);
//This first method is psuedoCode to explain how to create your datatable. You need to do it in the way that makes sense for you.
public DataTable createDataTable(){
List<string> excludedColumns = new List<string>();
excludedColumns.Add("FieldToExclude");
//...
DataTable dt = new DataTable();
foreach(string col in getColumns(myTable)){
if(!excludedColumns.Contains(name)){
DataColumn dC = new DataColumn(name,type);
DataTable.Add(dC);
}
return dt;
}
public List<string> getColumns(string tableName)
{
List<string> ret = new List<string>();
using (SqlConnection conn = getConn())
{
conn.Open();
using (SqlCommand com = conn.CreateCommand())
{
com.CommandText = "select column_Name from information_schema.COLUMNS where table_name = #tab";
com.Parameters.AddWithValue("#tab", tableName);
SqlDataReader read = com.ExecuteReader();
While(read.Read()){
ret.Add(Convert.ToString(read[0]);
}
conn.Close();
}
return ret;
}
//Now, you have a DataTable that has all the columns you want to insert. Map them yourself in code by adding to the appropriate column in your datatable.
public bool isCopyInProgess = false;//not necessary - just part of my code
public void saveDataTable(string tableName, DataTable table)
{
using (SqlConnection conn = getConn())
{
conn.Open();
using (var bulkCopy = new SqlBulkCopy(conn))//, SqlBulkCopyOptions.KeepIdentity))//un-comment if you want to use your own identity column
{
// my DataTable column names match my SQL Column names, so I simply made this loop. However if your column names don't match, just pass in which datatable name matches the SQL column name in Column Mappings
foreach (DataColumn col in table.Columns)
{
//Console.WriteLine("mapping " + col.ColumnName+" ("+does_Column_Exist(col.ColumnName,"Item").ToString()+")");
bulkCopy.ColumnMappings.Add(col.ColumnName, "["+col.ColumnName+"]");
// Console.WriteLine("ok\n");
}
bulkCopy.BulkCopyTimeout = 8000;
bulkCopy.DestinationTableName = tableName;
bulkCopy.BatchSize = 10000;
bulkCopy.EnableStreaming = true;
//bulkCopy.SqlRowsCopied += BulkCopy_SqlRowsCopied;
//bulkCopy.NotifyAfter = 10000;
isCopyInProgess = true;
bulkCopy.WriteToServer(table);
}
conn.Close();
}
}
Also, use this as your bolumn checker:
public bool does_Column_Exist(string colName,string tableName)
{
bool ret = false;
using (SqlConnection conn = getConn())
{
conn.Open();
using (SqlCommand com = conn.CreateCommand())
{
com.CommandText = "select count(*) from information_schema.COLUMNS where column_name = #col and table_name = #tab";
com.Parameters.AddWithValue("#tab", tableName);
com.Parameters.AddWithValue("#col", colName);
ret = Convert.ToInt32(com.ExecuteScalar()) == 0 ? false : true;
}
conn.Close();
}
return ret;
}
Is there a specific reason you need C# for this? It seems like the path of least resistance would be to use SQL to do the job.
INSERT INTO table2
(column_name(s))
SELECT column_name(s)
FROM table1;
I already have this code to make stringBuilder for every employee, I get all the employeesId from another table . But if I get more than 1000 employees, I get the error ORA-07195 , I know this error is related to a Maximum of expression in a list . Therefore how can I send every 500 employees to my query in Data Access Objects.
Public List<GraphModel> countRequestCreatedByTypeDefaulPage(int year, int month, String employeeID)
{
int count = 0;
int countEmployeess = 0;
string employeesid = "";
DataView dv = _employeeOverrideBO.getRelatedEmployees(year, month, employeeID);
StringBuilder listEmployees = new StringBuilder();
for (int i = 0; i < countEmployees; i += 500)
{
foreach (DataRowView rowView in dv)
{
DataRow row = rowView.Row;
String employee = row["EMPLOYEE_ID"].ToString();
if (count > 0)
listEmployees.Append(",");
listEmployees.Append("'").Append(employee).Append("'");
count++;
}
}
countEmployeess++;
employeesid = listEmployees.ToString();
return _requestDAO.countRequestCreatedByTypeDefaulPage(employeesid);
Also this is my query in Data Access Object
public List<GraphModel> countRequestCreatedByTypeDefaulPage(string employeesIds)
{
String sql = " select NVL(TO_CHAR(RR.REASON_NM_NEW), 'Total') as SERIES1, count(*) AS VAL" +
" from REQUEST R, REQUEST_PERSON RP, REQUEST_REASON RR " +
" WHERE R.STATUS IN ('CREATED', 'PENDING APPROVAL', 'APPROVED BY MANAGER', 'APPROVED', 'IN PROCESS') " +
" AND R.REQUEST_ID = RP.REQUEST_ID" +
" AND RP.REQUEST_ROLE = 'REQUESTOR' " +
" AND RR.REASON_ID = R.REASON_ID" +
" AND RP.EMPLOYEE_ID IN (" + employeesIds + ") " +
" group by rollup (RR.REASON_NM_NEW) " +
" ORDER BY count(*) DESC";
OracleCommand cmd = new OracleCommand(sql);
try
{
DataTable dataTable = Data_base_Access.executeSQL(cmd, ConfigurationManager.ConnectionStrings["stage"].ToString());
return (GraphModel.convertToList(dataTable));
}
catch (Exception ex)
{
Log.writeError("Request DAO", ex);
throw new DataAccessException("There was an error counting the open requests");
}
}
Also this query get the count to list called GraphModel
public static List<GraphModel> convertToList(System.Data.DataTable dataTable)
{
List<GraphModel> list = new List<GraphModel>();
foreach (DataRow dtRow in dataTable.Rows)
{
list.Add(convertToGraphModel(dtRow));
}
return list;
}
public static GraphModel convertToGraphModel(DataRow dtRow)
{
GraphModel graphModel = new GraphModel();
if (dtRow.Table.Columns.Contains("SERIES1") && dtRow["SERIES1"] != DBNull.Value)
{
graphModel.SERIES1 = Convert.ToString(dtRow["SERIES1"]);
}
if (dtRow.Table.Columns.Contains("SERIES2") && dtRow["SERIES2"] != DBNull.Value)
{
graphModel.SERIES2 = Convert.ToString(dtRow["SERIES2"]);
}
if (dtRow.Table.Columns.Contains("VAL") && dtRow["VAL"] != DBNull.Value)
{
graphModel.VAL = Convert.ToInt32(dtRow["VAL"]);
}
return graphModel;
}
}
I really appreciate your help because I am research a lot and I dont know what can I do
Split the list into 1000 item lists and change the query into this:
" AND (RP.EMPLOYEE_ID IN (" + ids_1_1000 + ") OR RP.EMPLOYEE_ID IN (" + ids_1001_2000 + "))" +
One of the features I love most about Oracle is the Oracle Call Interface (OCI) that lets you access some of the more powerful features or Oracle with programming languages. In particular, for this example, the ability to do bulk inserts should prove very helpful.
If, instead of the approach you have above, which is trying to insert thousands of literals into a single SQL statement, you put those values into a table and do a join, I think you will:
Spare the shared pool from having to compile a really nasty SQL statement
Eliminate the need to escape strings or worry about SQL Injection
Have a lightning fast query that substitutes a join (bread and butter for a database) for a giant in-list
Step 1: Create a GTT:
create global temporary table employee_list (
employee_id varchar2(100) not null
) on commit preserve rows;
The GTT is based on a session, so even if you have this code running in multiple instances, each GTT will act as a blank slate for each instance -- no possibility of collisions of data.
Step 2: Within your code, create a transaction to handle the fact that you need the insert to the table and the select on the data to occur as part of the same transaction:
OracleTransaction trans = conn.BeginTransaction(IsolationLevel.ReadCommitted);
Step 3: Use ODP.net's bulk insert capabilities to insert all of your employee Ids at once. I encourage you to benchmark this versus inserting them one at a time. You'll be amazed. If you have more than 50,000, then maybe you need to break the up into chunks, but with a single field, I think this should be more than adequate:
// string[] employeesIds
OracleCommand cmd = new OracleCommand("insert into employee_list values (:EMPLOYEE)",
conn);
cmd.Transaction = trans;
cmd.Parameters.Add(new OracleParameter("EMPLOYEE", OracleDbType.Varchar2));
cmd.Parameters[0].Value = employeesIds;
cmd.ArrayBindCount = employeesIds.Length;
cmd.ExecuteNonQuery();
Note employeeIds should be an array.
Step 4: Change your SQL from an in-list to a join:
select NVL(TO_CHAR(RR.REASON_NM_NEW), 'Total') as SERIES1, count(*) AS VAL
from
REQUEST R,
REQUEST_PERSON RP,
REQUEST_REASON RR,
employee_list e -- added this
WHERE R.STATUS IN ('CREATED', 'PENDING APPROVAL', 'APPROVED BY MANAGER',
'APPROVED', 'IN PROCESS')
AND R.REQUEST_ID = RP.REQUEST_ID
AND RP.REQUEST_ROLE = 'REQUESTOR'
AND RR.REASON_ID = R.REASON_ID
AND RP.EMPLOYEE_ID = e.employee_id -- changed this
group by rollup (RR.REASON_NM_NEW)
ORDER BY count(*) DESC
And here's what it would all look like together:
public List<GraphModel> countRequestCreatedByTypeDefaulPage(string[] employeesIds)
{
OracleTransaction trans = conn.BeginTransaction(IsolationLevel.ReadCommitted);
OracleCommand cmd = new OracleCommand("insert into employee_list values (:EMPLOYEE)",
conn);
cmd.Transaction = trans;
cmd.Parameters.Add(new OracleParameter("EMPLOYEE", OracleDbType.Varchar2));
cmd.Parameters[0].Value = employeesIds;
cmd.ArrayBindCount = employeesIds.Length;
cmd.ExecuteNonQuery();
String sql = ""; // code from above goes here
cmd = new OracleCommand(sql, conn);
cmd.Transaction = trans;
DataTable dataTable = null;
try
{
dataTable = Data_base_Access.executeSQL(cmd,
ConfigurationManager.ConnectionStrings["stage"].ToString());
return (GraphModel.convertToList(dataTable));
}
catch (Exception ex)
{
Log.writeError("Request DAO", ex);
throw new DataAccessException("There was an error counting the open requests");
}
finally
{
trans.Rollback();
}
return dataTable;
}
I resolved my problem with this code..
public List<GraphModel> countRequestCreatedByTypeDefaulPage(int year, int month, String employeeID)
{
int count = 0;
int countEmployees = 0;
Dictionary<string, int> dataChart = new Dictionary<string, int>();
DataView dv = _employeeOverrideBO.getRelatedEmployeesRequests(year, month, employeeID);
StringBuilder listEmployees = new StringBuilder();
foreach (DataRowView rowView in dv)
{
if (countEmployees == 500)
{
List<GraphModel> listReturn = _requestDAO.countRequestCreatedByTypeDefaulPage(listEmployees.ToString());
foreach(GraphModel model in listReturn){
if (dataChart.ContainsKey(model.SERIES1))
{
dataChart[model.SERIES1] = dataChart[model.SERIES1] + model.VAL;
}
else
{
dataChart[model.SERIES1] = model.VAL;
}
}
listEmployees = new StringBuilder();
count = 0;
countEmployees = 0;
}
DataRow row = rowView.Row;
String employee = row["EMPLOYEE_ID"].ToString();
if (count > 0)
listEmployees.Append(",");
listEmployees.Append("'").Append(employee).Append("'");
count++;
countEmployees++;
}
//Last Call
List<GraphModel> listReturnLast = _requestDAO.countRequestCreatedByTypeDefaulPage(listEmployees.ToString());
foreach (GraphModel model in listReturnLast) {
if (dataChart.ContainsKey(model.SERIES1))
{
dataChart[model.SERIES1] = dataChart[model.SERIES1] + model.VAL;
}
else
{
dataChart[model.SERIES1] = model.VAL;
}
}
List<GraphModel> list = new List<GraphModel>();
foreach (KeyValuePair<string, int> entry in dataChart)
{
GraphModel model = new GraphModel();
model.SERIES1 = entry.Key;
model.VAL = entry.Value;
list.Add(model);
}
return list;
}
I have an Excel file that originally had about 600 rows, and I was able to convert the excel file to a data table and everything got inserted into the sql table correctly.
The Excel file now has 3,600 rows and is having some type of issues that is not throwing an error but after 5 mins or so all the rows are still not inserted into the sql table.
Now, when converting the Excel file to a in memory datatable this happens very quickly, but when looping the datatable and inserting into the sql table is where I'm loosing data and is very slow, but I'm am receiving no errors what so ever.
For one, on each insert I've got to make a new connection to the database and insert the record, and I already know this is very VERY wrong, and I'm hoping to get some guidance from one of the sql pros on this one.
What is the correct way to process a in memory datatable with 3,600 records / rows with-out making 3,600 new connections?
--Here is the code the processes the excel file, and This happens very quickly.--
public static async Task<DataTable> ProcessExcelToDataTableAsync(string pathAndNewFileName, string hasHeader/*Yes or No*/)
{
return await Task.Run(() =>
{
string conStr = "", SheetName = "";
switch (Path.GetExtension(pathAndNewFileName))
{
case ".xls": //Excel 97-03
conStr = ConfigurationManager.ConnectionStrings["Excel03ConString"].ConnectionString;
break;
case ".xlsx":
conStr = ConfigurationManager.ConnectionStrings["Excel07ConString"].ConnectionString;
break;
}
conStr = String.Format(conStr, pathAndNewFileName, hasHeader);
OleDbConnection connExcel = new OleDbConnection(conStr);
OleDbCommand cmdExcel = new OleDbCommand();
OleDbDataAdapter oda = new OleDbDataAdapter();
DataTable dt = new DataTable();
cmdExcel.Connection = connExcel;
//Get the name of First Sheet
connExcel.Open();
DataTable dtExcelSchema;
dtExcelSchema = connExcel.GetOleDbSchemaTable(OleDbSchemaGuid.Tables, null);
SheetName = dtExcelSchema.Rows[0]["TABLE_NAME"].ToString();
connExcel.Close();
//Read Data from First Sheet
connExcel.Open();
cmdExcel.CommandText = "SELECT * From [" + SheetName + "]";
oda.SelectCommand = cmdExcel;
oda.Fill(dt);
connExcel.Close();
cmdExcel.Dispose();
oda.Dispose();
if (File.Exists(pathAndNewFileName))
{
File.Delete(pathAndNewFileName);
}
return dt;
});
}
--Here is the code that processes the in memory datatable and inserts each new record into the sql table, and this is where things stop working, no visible errors, but just does not return or work--
**I am in need of a better way to optimize this function where the records get inserted into the the sql table.
static async Task<ProcessDataTablePartsResult> ProcessDataTablePartsAsync(int genericCatalogID, DataTable initialExcelData)
{
//#GenericCatalogID INT,
//#Number VARCHAR(50),
//#Name VARCHAR(200),
//#Length DECIMAL(8,4),
//#Width DECIMAL(8,4),
//#Height DECIMAL(8,4),
//#ProfileID TINYINT,
//#PackageQty DECIMAL(9,4),
//#CategoryID INT,
//#UnitMeasure VARCHAR(10),
//#Cost MONEY,
//#PartID INT OUT
return await Task.Run(() =>
{
DataTable badDataTable = null,
goodDataTable = initialExcelData.Clone();
goodDataTable.Clear();
int newPartID = 0,
currIx = 0,
numGoodRows = initialExcelData.Rows.Count,
numBadRows = 0;
List<int> badIndexes = new List<int>();
List<int> goodIndexes = new List<int>();
List<Profile> profiles = GenericCatalogManagerBL.GetProfiles(_genericCNN);
List<Category> categories = GenericCatalogManagerBL.GetAllCategoryNameID(_genericCNN);
Func<string, byte> getProfileID = delegate(string x)
{
return profiles.Where(p => p.TheProfile.ToLower().Replace(" ", "") == x.ToLower().Replace(" ", "")).FirstOrDefault().ID;
};
Func<string, int> getCategoryID = delegate(string x)
{
return categories.Where(c => c.Name.ToLower().Replace(" ", "") == x.ToLower().Replace(" ", "")).FirstOrDefault().ID;
};
foreach (DataRow r in initialExcelData.Rows)
{
try
{
IPart p = new Part
{
GenericCatalogID = genericCatalogID,
Number = r["Number"].ToString(),
Name = r["Name"].ToString(),
Length = decimal.Parse(r["Length"].ToString()),
Width = decimal.Parse(r["Width"].ToString()),
Height = decimal.Parse(r["Height"].ToString()),
ProfileID = getProfileID(r["Profile"].ToString()),
CategoryID = getCategoryID(r["Category"].ToString()),
PackageQty = int.Parse(r["PackageQty"].ToString()),
UnitMeasure = r["UnitMeasure"].ToString(),
Cost = decimal.Parse(r["Cost"].ToString())
};
GenericCatalogManagerBL.InsertPart(_genericCNN, p, out newPartID);
goodIndexes.Add(currIx);
}
catch (Exception)
{
numBadRows++;
numGoodRows--;
badIndexes.Add(currIx);
}
currIx++;
}
for (int i = 0; i < goodIndexes.Count; i++)
{
goodDataTable.ImportRow(initialExcelData.Rows[goodIndexes[i]]);
initialExcelData.Rows[goodIndexes[i]].Delete();
}
initialExcelData.AcceptChanges();
goodDataTable.AcceptChanges();
if (initialExcelData.Rows.Count > 0)
{
badDataTable = initialExcelData;
}
return new ProcessDataTablePartsResult(numGoodRows, numBadRows, badDataTable, goodDataTable);
});
}
**--Here is the entire flow of the function--**
public static async Task<GenericPartsReport> ProcessGenericPartsAsync(int genericCatalogID, MembershipUser user, HttpRequest request, bool emailReport, bool hasHeaders)
{
byte[] fbytes = new byte[request.ContentLength];
request.InputStream.Read(fbytes, 0, fbytes.Length);
string pathAndNewFileName = Path.GetRandomFileName() + Path.GetExtension(request.Headers["X-FILE-NAME"]),
badReportTableString = "",
goodReportTableString = "";
GenericPartsReport report = new GenericPartsReport();
//get the users temp folder
pathAndNewFileName = UtilCommon.SiteHelper.GetUserTempFolder(user, request) + pathAndNewFileName;
File.WriteAllBytes(pathAndNewFileName, fbytes);
//process the excel file first
DataTable excelDataTable = await ProcessExcelToDataTableAsync(pathAndNewFileName, hasHeaders ? "Yes" : "No");
ProcessDataTablePartsResult processedResult = await ProcessDataTablePartsAsync(genericCatalogID, excelDataTable);
if (processedResult.BadDataTable != null)
{
if (processedResult.BadDataTable.Rows.Count > 0)
{
badReportTableString = await BuildTableReportAsync(processedResult.BadDataTable, "AlumCloud Parts Not Added Report");
processedResult.BadDataTable.Dispose();
}
}
if (processedResult.GoodDataTable != null)
{
if (processedResult.GoodDataTable.Rows.Count > 0)
{
goodReportTableString = await BuildTableReportAsync(processedResult.GoodDataTable, "AlumCloud Parts Added Report");
processedResult.GoodDataTable.Dispose();
}
}
report.Report = "A total number of (" + processedResult.NumberOfGoodRows + ") records was added to your generic catalog.<br/><br/>A total number of (" + processedResult.NumberOfBadRows + ") records were excluded from being added to your generic catalog.";
if (processedResult.NumberOfBadRows > 0)
{
report.Report += "<br/><br/>You can review an excel file that meets the standards here: <a href='" + _exampleExcelFile + "'>How to format a part excel file</a>.";
report.HasBadRows = true;
}
if (processedResult.NumberOfGoodRows > 0)
{
report.Report += "<br/><br/><b>Below is all of the parts that were added to your generic catalog<b/><br/><br/>" + goodReportTableString;
}
if (processedResult.NumberOfBadRows > 0)
{
report.Report += "<br/><br/><b>Below is all of the parts that were not added to your generic catalog</b><br/><br/>" + badReportTableString;
}
if (emailReport)
{
AFCCIncCommonUtil.EmailUtil.SendMailToThreadPool(user.Email, _supportEmail, report.Report, "AlumCloud Generic Catalog Parts Report", true);
}
excelDataTable.Dispose();
return report;
}
--This is the function that never returns or is in some state of limbo--
ProcessDataTablePartsResult processedResult = await ProcessDataTablePartsAsync(genericCatalogID, excelDataTable);