Develop Reference

How to execute INSERT INTO in SQLite with ODBC in c#

The function of application is to select some values from one database (PSQL) and insert it into another database (SQLite). But code below does not work, it stops at executing line and shows no error, but last forever (also if I use SELECT TOP 1 ...).
//... odbc conection to DSN, this works fine
odbc.dbsqlite.Open();
odbc.dbpsql.Open();
//sql command
OdbcCommand comsqlite = odbc.dbsqlite.CreateCommand();
OdbcCommand compsql = odbc.dbpsql.CreateCommand();
//SQL for select ... this works
compsql.CommandText = "SELECT DISTINCT ..."
compsql.Parameters.AddWithValue("#sifra", "VP");
...
// from here is problem
try
{
OdbcDataReader dbReader = compsql.ExecuteReader();
OdbcTransaction transaction = odbc.dbsqlite.BeginTransaction();
var ordinal = new
{
cenik = dbReader.GetOrdinal("sifra"),
ident = dbReader.GetOrdinal("ident"),
klasi = dbReader.GetOrdinal("klasi"),
cena = dbReader.GetOrdinal("cena"),
eankoda = dbReader.GetOrdinal("eankoda"),
};
int count = 0;
while (dbReader.Read())
{
//here single variable gets results
var cena = Convert.ToDouble(dbReader.GetDouble(ordinal.cena));
var ident = Convert.ToString(dbReader.GetString(ordinal.ident));
var cenik = Convert.ToString(dbReader.GetString(ordinal.cenik));
var klasi = Convert.ToString(dbReader.GetString(ordinal.klasi));
var eanko = Convert.ToString(dbReader.GetString(ordinal.eankoda));
comsqlite.CommandText = "INSERT INTO ARTIKLI (KLASI, CENA, BARKODA, CENIK, IDENT) VALUES (?,?,?,?,?);";
comsqlite.Parameters.AddWithValue("#KLASI", klasi);
comsqlite.Parameters.AddWithValue("#CENA", cena);
comsqlite.Parameters.AddWithValue("#BARKODA", eanko);
comsqlite.Parameters.AddWithValue("#CENIK", cenik);
comsqlite.Parameters.AddWithValue("#IDENT", ident);
if (count % 1000 == 0)
{
transaction.Commit();
transaction.Dispose();
**comsqlite.ExecuteNonQuery(); //here it stops and give no results**
transaction = odbc.dbsqlite.BeginTransaction();
}
count++;
}
comsqlite.Dispose();
odbc.dbsqlite.Close();
transaction.Commit();
transaction.Dispose();
dbReader.Close();
compsql.Dispose();
odbc.dbpsql.Close();
}
catch (Exception e)
{
Console.WriteLine("Error: "+ e);
throw;
}

I am not sure what your CommandText looks like at this point, but you should try to set some single quotation marks around the values being strings/characters in your database.
comsqlite.CommandText = "INSERT INTO ARTIKLI (KLASI, CENA, BARKODA, CENIK, IDENT) VALUES ('?','?','?','?','?');";

SQL Transaction not committing to database

I am writing my first SQL transaction code and have scoured stackoverflow to make sure I've done it right. The basic idea of the project is to get an email, parse it into a RawDatum object, then write the contents of the object to the SQL database. I was hoping to commit the new rows in batches of 100.
I have the code running without errors, but nothing is being actually written to the database.
I have a local DB (MS SQL Server Express). I have a few things commented out for testing purposes. What am I doing wrong?
using (SqlConnection connection = new SqlConnection(Properties.Settings.Default.MissionMetricsConnectionString))
{
connection.Open();
SqlCommand command = connection.CreateCommand();
command.CommandType = CommandType.Text;
command.CommandText = #"INSERT INTO RawData (MessageUID, DateTime, Attribute, DigitalValue, AnalogValue) VALUES (#MessageUID, #DateTime, #Attribute, #DigitalValue, #AnalogValue)";
command.Parameters.Add("#MessageUID", SqlDbType.Int);
command.Parameters.Add("#DateTime", SqlDbType.DateTime);
command.Parameters.Add("#Attribute", SqlDbType.Text);
command.Parameters.Add("#DigitalValue", SqlDbType.Bit);
command.Parameters.Add("#AnalogValue", SqlDbType.Float);
command.Connection = connection;
RawDatum currentDatum = null;
int uid;
List<int> uidsToDelete = new List<int>();
int numLoops = 1;//(int)(ids.Length / loopSize) + 1;
for (var loopIndex = 0; loopIndex < numLoops; loopIndex++)
{
int startIndex = loopIndex * loopSize;
int endIndex = 10;// Math.Min((loopIndex + 1) * loopSize, ids.Length);
using (SqlTransaction transaction = connection.BeginTransaction())
{
try
{
command.Transaction = transaction;
for (int i = startIndex; i < endIndex; i++)
{
msg = inbox.Fetch.MessageObject(ids[i]);
uid = inbox.Fetch.Uid(ids[i]);
if (msg.Subject == "[METRICS]")
{
currentDatum = new RawDatum(uid, msg.Date, msg.BodyText.TextStripped);
command.Parameters["#MessageUID"].Value = currentDatum.MessageUid;
command.Parameters["#DateTime"].Value = currentDatum.DateTime;
command.Parameters["#Attribute"].Value = currentDatum.Attribute;
command.Parameters["#DigitalValue"].Value = currentDatum.DigitalValue;
command.Parameters["#AnalogValue"].Value = currentDatum.AnalogValue;
int queryResult = command.ExecuteNonQuery();
if (queryResult != 1)
{
throw new InvalidProgramException();
}
}
uidsToDelete.Add(uid);
}
transaction.Commit();
//Delete(uidsToDelete);
output[1] += uidsToDelete.Count;
uidsToDelete.Clear();
}
catch
{
transaction.Rollback();
output[0] += 1;
output[2] += endIndex - startIndex;
}
}
}
connection.Close();
}

inserting to SqlLite takes too long

I insert data like this but it takes too long
For 34,000 records it took 20 minutes !! (for example if I insert into SQL Server CE it took only 3 minutes)
conn_sql = new SQLiteConnection(conn_str2);
conn_sql.Open();
cmd_sql = conn_sql.CreateCommand();
for (int i = 0; i < iTotalRows; i++)
{
try
{
Makat = dsView.Tables["Items"].Rows[i]["Makat"].ToString().Trim();
}
catch { Makat = ""; }
try
{
Barcode = dsView.Tables["Items"].Rows[i]["Barcode"].ToString().Trim();
}
catch { Barcode = ""; }
try
{
Des = dsView.Tables["Items"].Rows[i]["Des"].ToString().Trim();
}
catch { Des = ""; }
try
{
Price = dsView.Tables["Items"].Rows[i]["Price"].ToString().Trim();
}
catch { Price = ""; }
SQL = "INSERT INTO Catalog(Makat,Barcode,Des,Price)VALUES('" + Makat + "','" + Barcode + "','" + Des + "','" + Price + "')";
cmd_sql.CommandText = SQL;
cmd_sql.CommandType = CommandType.Text;
cmd_sql.ExecuteNonQuery();
//cmd_sql.Dispose();
}
How to insert faster ?

SQLite implicitly wraps queries within a transaction. Beginning and committing transactions in a loop could be slowing things down. I think you should get a significant boost of speed if you start a transaction and commit it once the loop is completed:
conn_sql.Open();
using(var tran = conn_sql.BeginTransaction()) // <--- create a transaction
{
cmd_sql = conn_sql.CreateCommand();
cmd_sql.Transaction = tran;   // <--- assign the transaction to the command
              
for (int i = 0; i < iTotalRows; i++)
{
// ...
cmd_sql.CommandText = SQL;
cmd_sql.CommandType = CommandType.Text;
cmd_sql.ExecuteNonQuery();
//cmd_sql.Dispose();
}
tran.Commit(); // <--- commit the transaction
} // <--- transaction will rollback if not committed already

If you do it in a single transaction it should be faster:
SqlTransaction transaction;
try
{
conn_sql = new SQLiteConnection(conn_str2);
conn_sql.Open();
cmd_sql = conn_sql.CreateCommand();
transaction = conn_sql.BeginTransaction();
for (int i = 0; i < iTotalRows; i++)
{
// create SQL string
cmd_sql.CommandText = SQL;
cmd_sql.CommandType = CommandType.Text;
cmd_sql.ExecuteNonQuery();
}
transaction.Commit();
}
catch
{
transaction.Rollback();
}

At first, try to use StringBuilder for the string concatenation. Secondly, you are making 34k requests and this is slow, try to make less amount of requests. Let say using StringBuilder concatenate 5k insert statements in one and fire it in transaction, do this until all your data will not be saved.

Parallel reading from SQL Server

I was looking through dozens of articles, but couldn't find a solution.
Here is the logic :
I have a Winform (VS2010) application, that needs to read data from SQL Server 2008 R2 Express table A, process some calculations and store in a different table B.
I want to use parallel ForEach in order to shorten execution time (otherwise the calculation + SQL process takes days.....)
I have to read from SQL, because the database has over 5 million rows, each read returns a few hundreds rows.
Lists are defined as :
BindingList<ItemsClass> etqM = new BindingList<ItemsClass>();
BindingList<ItemsClass> etqC = new BindingList<ItemsClass>();
The parallel execution :
Parallel.ForEach(etqC, cv => {
readData(ref etqM, "tableA", "WHERE ID LIKE '" + cv.Name + "%'");
IList<ItemsClass> eResults = etqM.OrderBy(f => f.ID).ToList();
foreach (ItemsClass R in eResults)
{
//calculations comes here
etqM[rID] = R;
}
Parallel.ForEach(etqM, r => {
// part 2 of calculations comes here
}
});
exportList(etqM, "tableB", true);
});
The SQL Read function :
The function gets a List, Table name + conditions for the SQL
read from SQL the records, and transform them to the List format.
public void readData<T>(ref BindingList<T> etqList, string tableName, string conditions)
{
SqlConnection myConnection = new SqlConnection();
SqlCommand myCommand = new SqlCommand();
myCommand.CommandTimeout = 0;
myCommand.Connection = myConnection;
etqList.Clear();
openConn(myConnection);
SqlDataReader myReader = null;
try
{
int totalResults;
myCommand.CommandText = "SELECT COUNT (*) FROM " + tableName + " " + conditions;
totalResults = (int)myCommand.ExecuteScalar();
if (totalResults > 0)
{
myCommand.CommandText = "SELECT * FROM " + tableName + " " + conditions;
myReader = myCommand.ExecuteReader();
etqList = ConvertTo<T>(convertReaderToDT(myReader));
}
}
catch (Exception ex) { }
finally
{
try { myReader.Close(); }
catch { }
}
closeConn(myConnection);
}
The SQL export function : this function exports the given list to the table name.
private void exportListToSql<T>(IEnumerable<T> etqList, string tableName)
{
SqlConnection myConnection = new SqlConnection();
SqlCommand myCommand = new SqlCommand();
myCommand.CommandTimeout = 0;
myCommand.Connection = myConnection;
openConn(myConnection);
try
{
actionTotalCount++;
DataTable dt = new DataTable(tableName);
dt = ToDataTable(etqList);//List Name
var bulkCopy = new SqlBulkCopy(myConnection,
SqlBulkCopyOptions.TableLock |
SqlBulkCopyOptions.FireTriggers |
SqlBulkCopyOptions.UseInternalTransaction,
null
);
bulkCopy.DestinationTableName = tableName;
bulkCopy.BatchSize = BATCH_SIZE;
bulkCopy.WriteToServer(dt);
}
catch (Exception ex) { }
finally { closeConn(myConnection); }
}
SQL openConn and closeConn :
void openConn(SqlConnection myConnection)
{
if (myConnection.State == ConnectionState.Open) return;
myConnection.ConnectionString = "Data Source=" + DB_NAME + ";Initial Catalog=APPDB;Integrated Security=True;Connect Timeout=120;Asynchronous Processing=true;";
try { myConnection.Open(); actionTotalCount++; }
catch (System.Exception ex) { MessageBox.Show(ex.Message); }
}
void closeConn(SqlConnection myConnection)
{
if (myConnection.State == ConnectionState.Fetching || myConnection.State == ConnectionState.Executing || myConnection.State == ConnectionState.Connecting) return;
try { myConnection.Dispose(); }
catch (System.Exception ex) { MessageBox.Show(ex.Message); }
}
The problem is : once I execute, I get this message :
ExecuteScalar requires an open and available connection. The connection's current state is closed.
This message arrives for all threads, except the first one.
Any ideas ?

Apparently the problem was not in the SQL, but the calculations.
Since the List was defined outside the 'Parallel.Foreach', it was accesses by different threads simultaneously, causing a crash.
Once I changed the code as followed, all worked excellent :
Parallel.ForEach(etqC, cv => {
BindingList<ItemsClass> etqM = new BindingList<ItemsClass>();
readData(ref etqM, "tableA", "WHERE ID LIKE '" + cv.Name + "%'");
IList<ItemsClass> eResults = etqM.OrderBy(f => f.ID).ToList();
foreach (ItemsClass R in eResults)
{
//calculations comes here
etqM[rID] = R;
}
Parallel.ForEach(etqM, r => {
// part 2 of calculations comes here
}
});
exportList(etqM, "tableB", true);
});

ExecuteNonQuery requires the command to have a transaction error in my code

I get the following error on cmd.ExecuteNonQuery.
"ExecuteNonQuery requires the command to have a transaction when the
connection assigned to the command is in a pending local transaction.
The Transaction property of the command has not been initialized."
Here is my code:
//if (hdRefresh.Value.Length > done.Value.Length || done.Value == "1")
//{
// //Write Your Add Customer Code here > Response.Write("true")
// done.Value = hdRefresh.Value;
//}
//else
//{
// Response.Redirect("~/Cashier/BTBill.aspx");
// return;
//}
if (IsClosedToDay())
{
ScriptManager.RegisterClientScriptBlock(Page, typeof(Page), "Warning", "<script>alert('Day Closing has been Performed ')</script>", false);
return;
}
DateTime dateFeomDB = getdate();
// by atizaz
if (HDD.Value == "" || HDD.Value == null)
{
ScriptManager.RegisterClientScriptBlock(Page, typeof(Page), "Warning", "<script>alert('No Transaction Found')</script>", false);
return;
}
//
SqlConnection scon = new SqlConnection(ConfigurationManager.ConnectionStrings["SQLCONN"].ToString());
Common.BillTransaction bill1 = new Common.BillTransaction();
ProcessUpdateBalandUnAuthBal insertBalance = new ProcessUpdateBalandUnAuthBal();
Common.Currency currencyy = new Common.Currency();
ProcessAuthorizeTokenByBillNo authorize = new ProcessAuthorizeTokenByBillNo();
BillTransaction bill = new BillTransaction();
scon.Open();
SqlTransaction sqlTrans = scon.BeginTransaction();
try
{
string strforxml = HDD.Value;
XmlDocument docXml = new XmlDocument();
#region Read In To Sender Controlls
#region Common Information
Contact con = new Contact();
con.Title = ddlTitle.SelectedItem.Text;
con.FirstName = TextBox1.Text.Trim();
con.LastName = TextBox9.Text.Trim();
con.ConTactNo = txtCell.Text == "" ? SqlString.Null : txtCell.Text;
con.Country = ddlCountry.SelectedItem.Text;
con.CustomerType = ddlCustomerType.SelectedItem.Text;
con.CustTypeID = int.Parse(ddlCustomerType.SelectedValue);
con.CountryID = Int32.Parse(ddlCountry.SelectedValue);
con.sqlTransaction = sqlTrans;
if (Scitytxt.Value != "")
{
try
{
con.City = Scitytxt.Value;
con.CityID = Int32.Parse(Scityval.Value);
}
catch (Exception)
{ }
}
else
{
con.City = SqlString.Null;// Scitytxt.Value;
con.CityID = SqlInt32.Null;// Int32.Parse(Scityval.Value);
con.Address = "";
}
//con.City = ddlCity.SelectedItem.Text;
//con.CityID = int.Parse(ddlCity.SelectedValue);
con.Address = TextBox10.Text;
#endregion
#region Check For NIC and Passport
if (txtNIC.Text != "" || txtPassport.Text != "")
{
SqlDataReader rdrsender;
if (txtNIC.Text != "")
{
con.NIC = txtNIC.Text;
}
else
{
con.NIC = SqlString.Null;
}
if (txtPassport.Text != "")
{
con.Passport = txtPassport.Text;
}
else
{
con.Passport = SqlString.Null;
}
ProcessSearchContactInContactInfo srchSender = new ProcessSearchContactInContactInfo();
srchSender.Contact = con;
srchSender.Invokewith5parameters();
rdrsender = srchSender.ResultSet;
#region If record Doesnot Exist In response of NIC Passport
if (!rdrsender.Read())
{
rdrsender.Close();
rdrsender.Dispose();
// con.sqlTransaction = sqlTrans;
ProcessAddContact InsertnewSenderInfo = new ProcessAddContact();
// InsertnewSenderInfo.sqlTransaction = sqlTrans;
InsertnewSenderInfo.Contact = con;
InsertnewSenderInfo.Invoke();
// sender1 = InsertnewSenderInfo.ResultSet;
// Sender_ID.Value = sender1[13].ToString();
}
#endregion
#region If Record Exists
else
{
con.CustomerID = Int32.Parse(rdrsender["Customer_ID"].ToString());
rdrsender.Close();
rdrsender.Dispose();
}
#endregion
}
#endregion
#region If Customer Donot Have NIC And/OR Passport
else// this executes when both Pasport and NIC are Null
{
con.NIC = SqlString.Null;
con.Passport = SqlString.Null;
ProcessAddContact InsertnewSenderInfo = new ProcessAddContact();
InsertnewSenderInfo.Contact = con;
InsertnewSenderInfo.Invoke();
DataSet ds = new DataSet();
int a = con.CustomerID;
StringReader inforeader = new StringReader("<CusTable><CusInfo><Relation_Type></Relation_Type><HusbandFather_Name></HusbandFather_Name><Address_Present></Address_Present><Address_Other></Address_Other><Phone_No_Office></Phone_No_Office><Cell_No></Cell_No><Fax_No></Fax_No><Date_Of_Birth></Date_Of_Birth><NTN_No></NTN_No><Nationality></Nationality><Occupation></Occupation><Relation_With_Financial_Institution></Relation_With_Financial_Institution><Other_Relation_With_Financial_Institution></Other_Relation_With_Financial_Institution><Business_Relation></Business_Relation></CusInfo></CusTable>");
ds.ReadXml(inforeader);
ds.GetXml();
SqlCommand cmd = new SqlCommand("update Contact_Info set CustInfo=" + ds.GetXml() + " WHERE Customer_ID=" + a + "", scon);
cmd.ExecuteNonQuery();
// sender1 = InsertnewSenderInfo.ResultSet;
// Sender_ID.Value = sender1[13].ToString();
}
tell me what is problem in my code and how to solve it.

You need to change this line
SqlCommand cmd = new SqlCommand("update Contact_Info set CustInfo=" + ds.GetXml() +
" WHERE Customer_ID=" + a + "", scon);
in this way
SqlCommand cmd = new SqlCommand("update Contact_Info set CustInfo=" + ds.GetXml() +
" WHERE Customer_ID=" + a + "", scon, sqlTrans);
The error message states exactly the problem.
Before code reaches that line you have opened a transaction and it is still open at the point of error
.....
scon.Open();
SqlTransaction sqlTrans = scon.BeginTransaction();
.....
Now, every SqlCommand executed when the connection has an opened transaction need to be informed of this. The transaction is not automatically set by the Framework.
You can use the SqlCommand constructor, as explained above, or you can set the cmd.Transaction property before executing the command.
Warning 1
The need to set the transaction for the current command is true even if you create the SqlCommand directly from the connection itself.
SqlCommand cmd = scon.CreateCommand();
cmd.Transaction = sqlTrans; // Required when inside a transaction
Warning 2
Avoid at all cost the use of string concatenation when using query text to update/insert/delete/select on a database. Use parameters. This will remove problems with strange or invalid characters and, most important, will prevent SqlInjection Attacks
string sqlText = "update Contact_Info set CustInfo=#info WHERE Customer_ID=#id";
SqlCommand cmd = new SqlCommand(sqlText, scon, sqlTrans);
cmd.Parameters.AddWithValue("#info", ds.GetXml());
cmd.Parameters.AddWithValue("#id",a);
cmd.ExecuteNonQuery();
Also, another recommendation is to NOT use AddWithValue, while handy, this method has many problems as explained in my answer here

You have started a transaction that is not commited before you called cmd.ExecuteNonQuery().
Just write down cmd.Transaction = sqlTrans; just before cmd.ExecuteNonQuery();
it will ensure that Now ExecuteNonQuery() will be executed in same transaction and also will be able to see all the modification done to database in the same transaction.

If you have this code:
SqlTransaction sqlTrans = scon.BeginTransaction();
then you should also have this:
cmd.Transaction = sqlTrans;
they work together.

For me dapper's ExecuteAsync takes 2 optional parameters and transaction wasn't being recognized so I had to name the parameters like this:
var result = await a_mConn.ExecuteAsync(sql: a_sSqlQuery,transaction: a_mTransaction);