// conn is read from handydrive
//conn2 read from C:\
this code is for write new record in to DB in C:\ by check exist first.
my problem is too slow for alot of records. and how to improve it to be faster...
SqlCeCommand cmd1 = new SqlCeCommand("Select * from bill_discount ", conn);
SqlCeDataReader dr1 = cmd1.ExecuteReader();
while (dr1.Read() != false)
{
SqlCeCommand cmd4 = new SqlCeCommand("Select * from bill_discount where bill_no='" + dr1.GetInt32(0) + "' AND bill_shopdc='" + dr1.GetString(2) + "' ", conn2);
SqlCeDataReader dr4 = cmd4.ExecuteReader();
if (dr4.Read() == false)
{
SqlCeCommand cmd2 = new SqlCeCommand("INSERT INTO bill_discount (bill_no,bill_value,bill_shopdc) VALUES ('" + dr1.GetInt32(0) + "','" + dr1.GetDouble(1) + "','" + dr1.GetString(2) + "') ", conn2);
// SqlCeDataReader dr2 = cmd2.ExecuteReader();
cmd2.ExecuteNonQuery();
}
}
//-------------------------------------------------------------------
I would take a look at the SqlBulkCopy Class:
Lets you efficiently bulk load a SQL
Server table with data from another
source.
BTW: In your code above, selecting the entire bill_discount table is not really a good idea, especially if the table is large.
[Also, it appears you could perform a single TSQL statement rather than looping through each row and round-tripping to the database.]
This example should be of help: SqlBulkCopy - Copy Table Data Between SQL Servers at High Speeds - ADO.NET 2.0 New Feature
Let's start by make the code more readable. Here's the result:
SqlCeCommand getAllBills = new SqlCeCommand("select * from bill_discount", primaryConnection);
SqlCeDataReader allBillsReader = getAllBills.ExecuteReader();
while (allBillsReader.Read())
{
SqlCeCommand getBill = new SqlCeCommand("select * from bill_discount where bill_no = '" + allBillsReader.GetInt32(0) + "' and bill_shopdc = '" + allBillsReader.GetString(2) + "' ", secondaryConnection);
SqlCeDataReader billReader = getBill.ExecuteReader();
if (!billReader.Read())
{
SqlCeCommand addMissingBill = new SqlCeCommand("insert into bill_discount (bill_no, bill_value, bill_shopdc) values ('" + allBillsReader.GetInt32(0) + "', '" + allBillsReader.GetDouble(1) + "', '" + allBillsReader.GetString(2) + "')", secondaryConnection);
addMissingBill.ExecuteNonQuery();
}
}
Disposable objects must be disposed. Let's do it.
Let's also remove SQL Injections.
Finally, let's optimize the second query: you don't need to select something and executing the reader if you just want to check if the value exists in the database.
using (SqlCeCommand getAllBills = new SqlCeCommand("select bill_no, bill_value, bill_shopdc from [bill_discount]", primaryConnection))
{
using (SqlCeDataReader allBillsReader = getAllBills.ExecuteReader())
{
while (allBillsReader.Read())
{
using (SqlCeCommand getBill = new SqlCeCommand("if exists(select * from [bill_discount] where [bill_no] = #billNumber and bill_shopdc = #billShop) select 1 else select 0", secondaryConnection))
{
getBill.Parameters.AddWithValue("#billNumber", allBillsReader["bill_no"]);
getBill.Parameters.AddWithValue("#billShop", allBillsReader["bill_shopdc"]);
bool billExists = Convert.ToBoolean(getBill.ExecuteScalar());
if (!billExists)
{
using (SqlCeCommand addMissingBill = new SqlCeCommand("insert into [bill_discount] ([bill_no], [bill_value], [bill_shopdc]) values (#billNumber, #billValue, #billShop)", secondaryConnection))
{
addMissingBill.Parameters.AddWithValue("#billNumber", allBillsReader["bill_no"]);
addMissingBill.Parameters.AddWithValue("#billValue", allBillsReader["bill_value"]);
addMissingBill.Parameters.AddWithValue("#billShop", allBillsReader["bill_shopdc"]);
int countAffectedRows = addMissingBill.ExecuteNonQuery();
Debug.Assert(countAffectedRows == 1, "The data was not inserted.");
}
}
}
}
}
}
So here we are.
Now, it's still a low performance solution. To be more effective, you might want to do the same thing in a single SQL query with joins. Since two tables are probably situated on different servers, you may look at linked servers: a feature that enables to execute a single query over several tables from several servers.
I see you are using SqlCe, which has number of limitations when inserting bulk data. The main limitation is the actual SqlCe Engine. You can however bypass this by using direct table inserts:
using (var command = connection.CreateCommand())
{
command.Transaction = transaction;
command.CommandType = CommandType.TableDirect;
command.CommandText = TABLE_NAME_IN_SQL;
using (var rs = command.ExecuteResultSet(ResultSetOptions.Updatable))
{
var rec = rs.CreateRecord();
rec.SetInt32(0, value0); // the index represents the column numbering
rec.SetString(1, value1);
rec.SetInt32(2, value2);
rs.Insert(rec);
}
}
Related
Hello guys, i am trying to get all records from tblInvoiceItemsTemp table and save all the records inside the tblInvoiceItems table but not able to solve. Any help would be appreciated, thank you.
I have added following code on btnSave_Click() event.
string connetionString1 = "server=localhost;database=billingDB;uid=root;pwd=root;integrated security=true";
using (MySqlConnection cnn1 = new MySqlConnection(connetionString1))
{
cnn1.Open();
string load_temp_table_rec_qry = "SELECT * FROM tblInvoiceItemsTemp";
using (MySqlCommand sqlcmd = new MySqlCommand(load_temp_table_rec_qry, cnn1))
{
MySqlDataReader temp_reader = sqlcmd.ExecuteReader();
while (temp_reader.Read())
{
string insert_invoice_items_qry = "INSERT INTO tblInvoiceItems(invoiceID, particulars, qty, rate) VALUES('" + 12 + "', '" + temp_reader["particulars"] + "', '" + temp_reader["qty"] + "', '" + temp_reader["rate"] + "')";
using (MySqlCommand itemsCmd = new MySqlCommand(insert_invoice_items_qry, cnn1))
{
itemsCmd.ExecuteNonQuery();
}
}
}
cnn1.Close();
}
I am getting following error messages.
An unhandled exception of type 'MySql.Data.MySqlClient.MySqlException' occurred in MySql.Data.dll
Additional Information: There is already an open DataReader associated with this Connection which must be closed first.
The error message is pretty clear: while you have a DataReader open (you haven't called Close/Dispose), the Connection cannot be used for anything else. One way to do this is to open a second connection:
using (MySqlCommand sqlcmd = new MySqlCommand(load_temp_table_rec_qry, cnn1))
{
MySqlDataReader temp_reader = sqlcmd.ExecuteReader();
using (var secondConnection = new MySqlConnection(connetionString1))
{
secondConnection.Open();
while (temp_reader.Read())
{
string insert_invoice_items_qry = "INSERT INTO tblInvoiceItems(invoiceID, particulars, qty, rate) VALUES('" + 12 + "', '" + temp_reader["particulars"] + "', '" + temp_reader["qty"] + "', '" + temp_reader["rate"] + "')";
using (MySqlCommand itemsCmd = new MySqlCommand(insert_invoice_items_qry, secondConnection))
{
itemsCmd.ExecuteNonQuery();
}
}
}
}
Another way is to use the disconnected model and load the records to a DataTable using a MySqlDataAdapter, so that the connection is free for using for itemsCmd.
However, you don't need to download into memory all the records for this, you can do an INSERT INTO SELECT, for much better performance:
INSERT INTO tblInvoiceItems(invoiceID, particulars, qty, rate)
SELECT 12, tblInvoiceItemsTemp.particulars, tblInvoiceItemsTemp.qty, tblInvoiceItemsTemp.rate
FROM tblInvoiceItemsTemp
Apologies in advance if I missed an answer to this somewhere but I wasn't quite finding it anywhere. So I'm building an application that scans PDF's of service orders our company gets, parses it, and inserts it into a SQL DB. The problem is at the end of this code. It successfully :
saves the original pdf in the proper folder
scans the pdf and parses it
inserts the correct data into the service order table
grabs PK of service order just created as we need that for the next batch of inserts
Here is where it gets hung up with a Exception thrown: 'System.Data.SqlClient.SqlException' in System.Data.dll
I foreach through all the instruments as there are multiples per Service Order, but it is erroring on this somewhere. to be clear I put a break point on the insert statement and all of the data is good and in the proper format ("string" int)
I feel like its in my connection maybe?
Anyways, thanks in advance for the help.
string filename = Path.GetFileName(FileUpload1.FileName);
FileUpload1.SaveAs(Server.MapPath("~/PDF/") + filename);
// Now we parse the PDF by creating a new ServiceOrder object and parsing from it.
ServiceOrder so = new ServiceOrder();
// Make sure we load the PDF from the correct path on the server
so.LoadPDF(Server.MapPath("~/PDF/") + filename);
String strConnString = "Data Source=127.0.0.0;Initial Catalog=SOMECATALOG;User ID=SOMEUSER;Password=SOMEPASSWORD";
// Insert Into Service Orders Table
string defaultdate = DateTime.Now.ToString("yyyy-MM-dd");
String strQuery = "insert into TServiceOrders (strServiceOrderNo, intStatusCodeID, strCustomerName, strCustomerNo, strCustomerAddress1, strCustomerAddress2, strCustomerAddress3, intRepID, strServiceDescription, strServiceRequestDate, strServiceOrderDate, strNotes) values ('"
+ so.ServiceOrderNumber.ToString() + "', 2, '"
+ so.CustomerContactName.ToString() + "', '"
+ so.CustomerNumber.ToString() + "', '"
+ so.CustomerContactAddress1.ToString() + "', '"
+ so.CustomerContactAddress2.ToString() + "', '"
+ so.CustomerContactAddress3.ToString() + "', 1, '', '"
+ defaultdate + "', '" + defaultdate + "', '')";
SqlConnection conn = new SqlConnection(strConnString);
SqlCommand cmd = new SqlCommand(strQuery, conn);
conn.Open();
cmd.ExecuteNonQuery();
conn.Close();
// Grabbing latest primary key od service order just added for next instrument inserts
int lastid = 999999;
String strPKquery = "select top 1 intServiceOrderID from TServiceOrders order by intServiceOrderID desc";
SqlDataReader rdr = null;
SqlConnection conn2 = new SqlConnection(strConnString);
SqlCommand cmd2 = new SqlCommand(strPKquery, conn2);
try
{
conn2.Open();
rdr = cmd2.ExecuteReader();
while (rdr.Read())
{
lastid = (int)rdr["intServiceOrderID"];
}
}
finally
{
if (rdr != null)
{
rdr.Close();
}
if (conn2 != null)
{
conn2.Close();
}
}
// Insert Into Service Instruments Tables
SqlConnection conn3 = new SqlConnection(strConnString);
conn3.Open();
foreach (ServiceInstrument sin in so.ServiceInstruments)
{
string sim = "";
sim = sin.ServiceInstrumentModel;
if (String.IsNullOrEmpty(sim))
{
sim = "";
}
else
{
sim = sin.ServiceInstrumentModel.ToString();
}
string sid = "";
sid = sin.ServiceInstrumentDescription;
if (String.IsNullOrEmpty(sid))
{
sid = "";
}
else
{
sid = sin.ServiceInstrumentDescription.ToString();
}
string sis = "";
sis = sin.ServiceInstrumentSerial;
if (String.IsNullOrEmpty(sis))
{
sis = "";
}
else
{
sis = sin.ServiceInstrumentSerial.ToString();
}
string sih = "";
sih = sin.ServiceInstrumentHandle;
if (String.IsNullOrEmpty(sih))
{
sih = "";
}
else
{
sih = sin.ServiceInstrumentHandle.ToString();
}
string sip = "";
sip = sin.ServiceInstrumentParentAsset;
if (String.IsNullOrEmpty(sip))
{
sip = "";
}
else
{
sip = sin.ServiceInstrumentParentAsset.ToString();
}
String strQuery3 = "insert into TServiceInstruments values ('" + sim.ToString() + "', '" + sid.ToString() + "', '" + sis.ToString() + "', '" + sih.ToString() + "', " + sip.ToString() + ", " + lastid + ")";
SqlCommand cmd3 = new SqlCommand(strQuery3, conn3);
cmd3.ExecuteNonQuery();
}
conn3.Close();
When writing insert statements you should always specify the column names. This will protect the code from changes in the order of the columns in the table schema.
You are not using parameters in your sql statements, this leaves your code vulnerable to Sql Injection.
You should use using statements around your SqlConnection instances to ensure they are closed even when an Exception occurs.
Your logic is very difficult to follow, split your code until methods with meaningful names instead of having 1 "God" method that does everything.
If you follow those guidelines the problem will most likely solve itself in your refactoring.
Update Code Fragment
Note that you should always specify the correct types for your columns and the length if applicable. Also pass the actual value and never the string value.
const String strQuery3 = "INSERT INTO TServiceInstruments (sim, sid, sis, sih, sip, lid) VALUES (#sim, #sid, #sis, #sih, #sip, #lid)";
using(var conection = new SqlConnection(strConnString))
using(SqlCommand command = new SqlCommand(strQuery3, connection))
{
command.Parameters.Add(new SqlParameter("#sim", SqlDbType.VarChar, 200){Value = sim});
command.Parameters.Add(new SqlParameter("#sid", SqlDbType.VarChar, 200){Value = sid});
command.Parameters.Add(new SqlParameter("#sis", SqlDbType.VarChar, 200){Value = sis});
command.Parameters.Add(new SqlParameter("#sih", SqlDbType.VarChar, 200){Value = sih});
command.Parameters.Add(new SqlParameter("#sip", SqlDbType.Int){Value = sip});
command.Parameters.Add(new SqlParameter("#lid", SqlDbType.Int){Value = lid});
connection.Open();
command.ExecuteNonQuery();
}
Final note: You really need to learn how to read Exceptions and this includes the Stack Trace which points directly to the line in the call stack where the Exception originated. If you can understand this then debugging becomes much easier.
Maybe this doesn't deserve to be an answer, but I'm trying to build some reputation, so here goes :).
I suspect that your error lies in the "insert into TServiceInstruments ..." statement. Namely, you are giving the table more (or less) columns. As a good practice, always specify the columns, like this:
insert into TServiceInstruments (column1, column2, column3)
values (1, 2, 3)
I need to insert 388 datas per minute to local Database.
At first when the table is Empty, I only need 5 second to Insert to database.
But when the table gets larger, the program efficacy slow down to more than one minute when the amount of rows comes to 1,026,558.
And the useage of CPU is 100%. It's unusual.
here is my code:
public static void dataToDB(String[] routeIDArray,String[] levelArray,String[] valueArray,String[] travelTimeArray, int amountOfData)
{
MySqlConnection con = new MySqlConnection(connStr);
MySqlCommand cmd = null;
MySqlDataReader rdr = null;
String sqlCmd, updateSqlCmd = "UPDATE `datetimetable` SET ";
for(int counter = 0; counter < amountOfData; counter++)
{
sqlCmd = "ALTER TABLE `datetimetable` ADD COLUMN IF NOT EXISTS `" + routeIDArray[counter] + "` INT NULL;"
+ "INSERT INTO `roadvalue`.`data` (`level`,`value`,`traveltime`) VALUES ("
+ levelArray[counter] + ","
+ valueArray[counter] + ","
+ travelTimeArray[counter] + ");"
+ "SELECT LAST_INSERT_ID() FROM `data`;";
cmd = new MySqlCommand(sqlCmd, con);
con.Open();
rdr = cmd.ExecuteReader();
rdr.Read();
updateSqlCmd += "`" + routeIDArray[counter] + "` = " + rdr[0] + ",";
rdr.Close();
}
updateSqlCmd = updateSqlCmd.TrimEnd(',');
updateSqlCmd += " WHERE EXISTS (SELECT * WHERE dateTime = '" + dateTime.ToString("yyyy-MM-dd HH:mm:00") + "');";
cmd = new MySqlCommand(updateSqlCmd, con);//update data key to datetimetable
cmd.ExecuteNonQuery();
Console.WriteLine("Done.");
con.Close();
}
public static void checkDateTimeExisted()
{
MySqlConnection con = new MySqlConnection(connStr);
MySqlCommand cmd;
String sqlCmd;
sqlCmd = "INSERT INTO `datetimetable` (`dateTime`) SELECT * FROM (SELECT '" + dateTime.ToString("yyyy-MM-dd HH:mm:00")
+ "') AS tmp WHERE NOT EXISTS(SELECT `dateTime` FROM `datetimetable` WHERE `dateTime` = '" + dateTime.ToString("yyyy-MM-dd HH:mm:00") + "') LIMIT 1; ";
con.Open();
cmd = new MySqlCommand(sqlCmd, con);
cmd.ExecuteNonQuery();
con.Close();
}
And Mysql Engine is InooDB, table "data" has one Auto_Increment Primary key, table "datetimetable" has an Auto_Increment Primary key and a not duplicate datetime as index.
What have I done wrong?
I find the answer, the command "SELECT LAST_INSERT_ID() FROM data;" should add LIMIT 1 or it will get all the ID kill the performance.
Do not use ALTER TABLE in a loop -- Plan ahead and provide all the columns before starting.
Do not use multiple statements in a single string. This has security implications, etc.
Do not use WHERE EXISTS... when (I think) a simple WHERE would work.
If there is UNIQUE(datetime), then the final INSERT can be simply
INSERT IGNORE INTO datetimetable
(datetime)
VALUE
('...');
Do batch inserts unless you need the LAST_INSERT_ID(). LIMIT 1 should not be necessary.
Do not 'Normalize' datetime values; it only slows things down. Just put the datetime as is in the main table.
I've built an app in vs 2012 that is supposed to be able to select, insert, update, delete info from a sql server 2012 database. At first I put some data into the db using sql server. I built my select queries to test them out and they worked. After this I built my insert query and tested it out. It also works. But if I try to retrieve data that I have inserted, it doesn't retrieve anything.
Here's an example of one of my select queries:
query = "SELECT P.Denumire, P.Pret, P.Cantitate, P.Reducere, P.Pret_redus, " +
"S.Stoc_magazin, S.Stoc_depozit " +
"FROM Produse P, Stoc_intern S " +
"WHERE S.ID_produs IN " +
"(SELECT P.ID_produs " +
"FROM Produse " +
"WHERE P.Denumire LIKE '%" + tb_s_name.Text + "%')";
SqlDataAdapter da = new SqlDataAdapter(query, c);
SqlCommandBuilder cb = new SqlCommandBuilder(da);
DataTable dt = new DataTable();
da.Fill(dt);
q_res.DataSource = dt;
And here's and example of one of my insert queries:
query = "Insert INTO Produse " +
"(Denumire, Pret, Cantitate, Reducere) " +
"Values(#Denumire, #Pret, #Cantitate, #Reducere)";
SqlCommand cmd = c.CreateCommand();
cmd.Connection = c;
cmd.CommandText = query;
cmd.Parameters.Add("#Denumire", SqlDbType.NVarChar, 50).Value = tb_op_name.Text;
cmd.Parameters.Add("#Pret", SqlDbType.Float).Value = tb_op_pv.Text;
cmd.Parameters.Add("#Cantitate", SqlDbType.NVarChar, 50).Value = tb_op_cantitate.Text;
cmd.Parameters.Add("#Reducere", SqlDbType.Float).Value = tb_op_red.Text;
cmd.ExecuteNonQuery();
Both queries work, it's just that if I submit the insert query and after try to select something with the select query, it does not retrieve anything although the information exists in the database. I checked by opening the db in sql server after executing the insert query from my vs built app.
Here's the connection string also:
SqlConnection c = new SqlConnection(#"Data Source=(LocalDB)\v11.0;AttachDbFilename=" + Application.StartupPath + #"\DB\Supermarket.mdf;Integrated Security=True;MultipleActiveResultSets=true;Connect Timeout=30");
Please use different SqlConnections for each procedure.
I am querying for data, if the data does not exist, I insert it. if it does, I do something else:
SqlCommand checkHead = new SqlCommand("SELECT * FROM TABLE WHERE ORDER_NO = '" + orderNo + "';", connection);
SqlDataReader checkHeadReader = checkHead.ExecuteReader(CommandBehavior.SingleRow);
if (!checkHeadReader.HasRows)
{
checkHeadReader.Close();
addHead.ExecuteNonQuery();
}
But I wonder if there's a shorter way to code this? would the code below work?
SqlCommand checkHead = new SqlCommand("SELECT * FROM TABLE WHERE ORDER_NO = ' + orderNo + "';", connection);
if(checkHead.ExecuteReader(CommandBehavior.SingleRow).HasRows)
addHead.ExecuteNonQuery();
else //this order already exists
Server.Transfer(#"~/Views/Error.aspx");
ExecuteScalar is great for this, E.g.
using (SqlCommand cmdCheck = new SqlCommand("Select Count(*) From Table Where Order_No = '" + orderNo + "'", connection))
{
int nExists = (int)cmdCheck.ExecuteScalar();
if (nExists==0) addHead.ExecuteNonQuery();
}