I'm completly new in donet and I'm trying to find a way to write database from c# into a db provider. The data stems from json files, which hold all necessary information for the insertion process (tablesnames, columnnames, types (as strings) and data). Since the amount of data can get quite big I'd like to circumvent creating a new insert statement for every row. Is there an approach that works with all db providers?
From the deserialized json File I'm first creating a DbType array, which holds the datatypes for each column.
With this I tried to get a satisfactory insert approach by
Building the INSERT Statement by using a stringbuilder
Building a DbCommand and supply it with all information
Execute the created DbCommand
using (IDbConnection connection = dbFactory.CreateConnection())
{
foreach (TableDTO table in dbTables)
{
//build the insert statement
StringBuilder insertSQLBuilder = new StringBuilder();
insertSQLBuilder.Append("INSERT INTO " + table.Name + "(");
foreach (ColumnDTO column in table.Columns)
{
insertSQLBuilder.Append(column.Name + ", ");
}
insertSQLBuilder.Length = insertSQLBuilder.Length - 2;
insertSQLBuilder.Append(") VALUES (");
for (int i = 0; i < table.Columns.Length; i++) {
insertSQLBuilder.Append("#param" + i + ", ");
}
insertSQLBuilder.Length = insertSQLBuilder.Length - 2;
insertSQLBuilder.Append(")");
//prepare the insert command
using (IDbCommand dbCommand = connection.CreateCommand()) {
dbCommand.CommandText = insertSQLBuilder.ToString();
IDbDataParameter[] dbParameters = new DbParameter[table.Columns.Length];
for (int i = 0; i < table.Columns.Length; i++)
{
IDbDataParameter dbParameter = dbCommand.CreateParameter();
dbParameter.DbType = typeArray[i]; //DbType Array, which holds the types for each column
dbParameter.ParameterName = "param" + i;
dbParameters[i] = dbParameter;
}
while (dataDeserializer.MoveNext())
{
// get new row from json file, each element of ColumnData holds the value and extra information which is not needed here
ColumnData[] columnData = dataDeserializer.Current;
for (int i = 0; i < dbParameters.Length; i++)
{
bool isNotGuid = typeArray[i] != DbType.Guid;
object value = null;
//TODO stupid conversion workaround
if (isNotGuid)
{
value = columnData[i].Value;
}
else
{
value = Guid.Parse(columnData[i].Value);
}
dbParameters[i].Value = value ?? DBNull.Value;
dbCommand.Parameters.Add(dbParameters[i]);
}
//execute statement and close connection
dbCommand.Connection.Open();
dbCommand.ExecuteNonQuery();
dbCommand.Connection.Close();
dbCommand.Parameters.Clear();
}
}
}
}
What I'd like to do is to circumvent single insert calls to the database without specifying a database provider. Is there a library which support a bulk insert for multiple providers? A perfect scenario would be a library, which allows me to just change the value of a parameter and also a maximum count for rows which should be inserted at once (to keep memory usage in check).
I am doing a simple select with a date filter on it with a months range where only 32 records are present however its taking 15 seconds to query and return the data I am using sage 50 as you can probally tell and c#. I am using odbc to create the query the seem speeds can be found if i use the odbc query tool.
This is for a stright forward select and it should not be taking that long to return the data through odbc.
String SQL = string.Format("SELECT 'ORDER_NUMBER', 'ORDER_OR_QUOTE',
'ANALYSIS_1','ACCOUNT_REF','ORDER_DATE','NAME',
'COURIER_NUMBER','COURIER_NAME','CUST_TEL_NUMBER'
,'DESPATCH_DATE','ACCOUNT_REF', 'DEL_NAME', 'DEL_ADDRESS_1',
'DEL_ADDRESS_2', 'DEL_ADDRESS_3', 'DEL_ADDRESS_4', 'DEL_ADDRESS_5',
'INVOICE_NUMBER','ORDER_DATE','INVOICE_NUMBER_NUMERIC',
'CONTACT_NAME','CONSIGNMENT', 'NOTES_1', 'ITEMS_NET'
,'ITEMS_GROSS','QUOTE_STATUS' FROM SALES_ORDER WHERE ORDER_DATE
='{0}' and ORDER_DATE <='{1}'", fromD, toD);
public List<SalesOrders> GetSalesOrders()
{
List<SalesOrders> _salesOrdersList = new List<SalesOrders>();
try
{
string sageDsn = ConfigurationManager.AppSettings["SageDSN"];
string sageUsername = ConfigurationManager.AppSettings["SageUsername"];
string sagePassword = ConfigurationManager.AppSettings["SagePassword"];
//int totalRecords = GetSalesOrdersount();
int counter = 0;
//using (var connection = new OdbcConnection("DSN=SageLine50v24;Uid=Manager;Pwd=;"))
using (var connection = new OdbcConnection(String.Format("DSN={0};Uid={1};Pwd={2};", sageDsn, sageUsername, sagePassword)))
{
connection.Open();
//string sql = string.Format(getInvoiceSql, customerCode, DateTime.Today.AddMonths(-1).ToString("yyyy-MM-dd"));
string fromD = dtpFrom.Value.ToString("yyyy-MM-dd");
string toD = dtpTo.Value.ToString("yyyy-MM-dd");
String SQL = string.Format("SELECT 'ORDER_NUMBER', 'ORDER_OR_QUOTE', 'ANALYSIS_1','ACCOUNT_REF','ORDER_DATE','NAME', 'COURIER_NUMBER','COURIER_NAME','CUST_TEL_NUMBER' ,'DESPATCH_DATE','ACCOUNT_REF', 'DEL_NAME', 'DEL_ADDRESS_1', 'DEL_ADDRESS_2', 'DEL_ADDRESS_3', 'DEL_ADDRESS_4', 'DEL_ADDRESS_5', 'INVOICE_NUMBER','ORDER_DATE','INVOICE_NUMBER_NUMERIC', 'CONTACT_NAME','CONSIGNMENT', 'NOTES_1', 'ITEMS_NET' ,'ITEMS_GROSS','QUOTE_STATUS' FROM SALES_ORDER WHERE ORDER_DATE >='{0}' and ORDER_DATE <='{1}'", fromD, toD);
using (var command = new OdbcCommand(SQL, connection))
{
using (var reader = command.ExecuteReader())
{
while (reader.Read())
{
counter++;
backgroundWorker1.ReportProgress(counter);
var salesOrders = new SalesOrders();
salesOrders.ACCOUNT_REF = Convert.ToString(reader["ACCOUNT_REF"]);
salesOrders.RecordIdentifier = "";
salesOrders.ShipmmentId = Convert.ToString(reader["ORDER_NUMBER"]);
salesOrders.OrderDate = Convert.ToDateTime(reader["ORDER_DATE"]);
salesOrders.OrderNumber = Convert.ToString(reader["ORDER_NUMBER"]);
salesOrders.Company = "";
salesOrders.Carrier = Convert.ToString(reader["COURIER_NUMBER"]);
salesOrders.CarrierService = Convert.ToString(reader["COURIER_NAME"]);
salesOrders.CustomerName = Convert.ToString(reader["NAME"]);
salesOrders.ShipToAddress1 = Convert.ToString(reader["DEL_ADDRESS_1"]);
salesOrders.ShipToAddress2 = Convert.ToString(reader["DEL_ADDRESS_2"]);
salesOrders.ShipToAddress3 = Convert.ToString(reader["DEL_ADDRESS_3"]);
salesOrders.ShipToAddress4 = Convert.ToString(reader["DEL_ADDRESS_4"]);
salesOrders.ShipToAddress5 = Convert.ToString(reader["DEL_ADDRESS_5"]);
salesOrders.ShiptoAttention = Convert.ToString(reader["DEL_NAME"]);
salesOrders.ShiptoPhoneNo = Convert.ToString(reader["CUST_TEL_NUMBER"]);
salesOrders.Country = Convert.ToString(reader["ANALYSIS_1"]);
salesOrders.ShiptoEmail = "";
salesOrders.MakeAddressDefault = "Y";
bool isProcessed = _sqlManager.hasbeenProcessed(salesOrders.OrderNumber);
if (isProcessed == true)
salesOrders.Exported = true;
_salesOrdersList.Add(salesOrders);
}
}
}
}
return _salesOrdersList.OrderByDescending(o => o.OrderDate).ToList();
}
don't use {0}, {1} for embedding values in strings... ADD via Parameters
String SQL =
#"SELECT
ORDER_NUMBER,
ORDER_OR_QUOTE,
ANALYSIS_1,
ACCOUNT_REF,
ORDER_DATE,
`NAME`,
COURIER_NUMBER,'
OURIER_NAME,
CUST_TEL_NUMBER,
DESPATCH_DATE,
ACCOUNT_REF,
DEL_NAME,
DEL_ADDRESS_1,
DEL_ADDRESS_2,
DEL_ADDRESS_3,
DEL_ADDRESS_4,
DEL_ADDRESS_5,
INVOICE_NUMBER,
ORDER_DATE,
INVOICE_NUMBER_NUMERIC,
CONTACT_NAME,
CONSIGNMENT,
NOTES_1,
ITEMS_NET,
ITEMS_GROSS,
QUOTE_STATUS
FROM
SALES_ORDER
WHERE
ORDER_DATE >= ?
and ORDER_DATE <= ?
ORDER BY
ORDER_DATE DESC";
using (var command = new OdbcCommand(SQL, connection))
{
// assuming fields are actually date data types fields
command.Parameters.Add( "parmFromDate", fromD );
command.Parameters.Add( "parmToDate", toD );
The "?" in the query are place-holders for the parameter values which are handled by the ODBC process. The Parameters being added within the using() portion are added in the same ordinal position as their respective place-holder parts. I just assigned the parameter name to give context to whoever is looking at it after.
The query itself SHOULD be very quick depending on the date range you are pulling. Even added the SQL Order by descending order so it is pre-pulled down in the order you intended it too.
I'm trying to have a method return a count statement.
public int getSettingsCount(string UserId, string Setting)
{
int LastSetting;
//var user = new SqlDataLayer();
using (var db = new SQLite.SQLiteConnection(this.DBPath))
{
{
List<int> _setting = db.Query<int>("SELECT COUNT(*) FROM QTabSettings WHERE UserId = 1058 AND Setting = 'ServerDropdown' GROUP BY UserId;");
LastSetting = Convert.ToInt32(_setting.SingleOrDefault());
}
return LastSetting;
}
}
When I execute the query it returns the correct value (6). However I am getting the value (0) from my above query.
How can I get the method to return the count as an int?
You are using LIMIT 1 and still using List<int> that's strange and unnecessary. Also since it's count(*) there is no need of LIMIT 1 since the result would be a scalar data. Should change it to
int _setting = db.Query<int>("SELECT COUNT(*) FROM QTabSettings WHERE UserId = 1058 AND Setting = 'ServerDropdown';");
LastSetting = _setting;
If I understand your question properly I think this is what you want:
LastSetting = _setting.FirstOrDefault();
or:
LastSetting = _setting[0];
I need to modify and existing Azure Table Storage query, assuming i is an integer query retrieves latest report:
string rowCompare = String.Format(CommonDefs.inverseTimeStampRowKeyFormat, DateTime.MaxValue.Ticks - DateTime.UtcNow.Ticks);
var result = (from er in this.serviceContext.EntityReportsTable
where er.PartitionKey.Equals(i.ToString(), StringComparison.OrdinalIgnoreCase) && er.RowKey.CompareTo(rowCompare) > 0
select er).Take(1)).FirstOrDefault();
I need to modify it to retrieve latest reports for several known entities, replacing single integer i with array of integers - like int[]{1, 6, 10}.
Apart from running existing query sequentially for the each parameter in array, is there a way to do it in one query? Like IN clause in Sql?
You can use the lastest version of the Azure Storage Client Library this is the complete pseudo code for your task:
var rowCompare = String.Format("{0}", DateTime.MaxValue.Ticks - DateTime.UtcNow.Ticks);
var items = new []{"1", "6", "10"};
var filters =
items.Select(key => TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, key)).ToArray();
var combine =
filters.Length > 0
? filters[0]
: null;
for (var k = 0; k < filters.Length; k++)
combine = TableQuery.CombineFilters(combine, TableOperators.Or, filters[k]);
var final = TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.GreaterThan, rowCompare);
if (!string.IsNullOrEmpty(combine))
final = TableQuery.CombineFilters(final, TableOperators.And, combine);
var query = new TableQuery<EntityReport>().Where(final);
var client = CloudStorageAccount.DevelopmentStorageAccount.CreateCloudTableClient();
var table = client.GetTableReference("EntityReports");
var result = table.ExecuteQuery(query);
Azure Table Storage does not support IN clause like SQL. However instead of doing a query sequentially, you could fire queries in parallel and compare the result. For example look at the pseudo code below:
List<Task<T>> tasks = new List<Task<T>>();
foreach (var i in integerArray)
{
tasks.Add(Task.Factory.StartNew<T>(() => {
string rowCompare = String.Format(CommonDefs.inverseTimeStampRowKeyFormat, DateTime.MaxValue.Ticks - DateTime.UtcNow.Ticks);
var result = (from er in this.serviceContext.EntityReportsTable
where er.PartitionKey.Equals(i.ToString(), StringComparison.OrdinalIgnoreCase) && er.RowKey.CompareTo(rowCompare) > 0
select er).Take(1)).FirstOrDefault();
}));
}
Task.WaitAll(tasks.ToArray());
foreach (var task in tasks)
{
var queryResult = task.Result;
//Work on the query result
}
Having something similar to:
SELECT (SELECT COUNT(*) from Table1),(SELECT COUNT(*) from Table2 )
How do I write it in linq? Or is it simple not possible?
Limitations:
Can only hit the database one time:
var result = new {
Sum1 = db.Table1.Count(),
Sum2 = db.Table2.Count()
}); // is not valid.....
I do not want to use something similar to (using a "helping" table):
var result = (from t3 in db.Table3
select new {
Sum1 = db.Table1.Count(),
Sum2 = db.Table2.Count()
}).firstOrDefault();
//In order to get only the first row
//but it will not return nothing if the table 3 has no entries......
Not using db.Database.ExecuteSqlCommand
I cannot see a solution which solves all your limitations. This is one of the caveats with using an ORM-mapper, you are not in control of the generated SQL.
In this case, if it is utterly unacceptable for you to send more than one query to the database, the harsh truth is that you will have to write the query yourself.
Update
I got curious and created an extension method that can do this! Of course it constructs its own SQL command, and it just works for Linq2SQL. Also massive disclaimer: It's fairly dirty code, if I have some time I'll fix it up in the weekend :)
public static TOut CountMany<TContext, TOut>(this TContext db, Expression<Func<TContext, TOut>> tableSelector)
where TContext: DataContext
{
var newExpression = (NewExpression) tableSelector.Body;
var tables =
newExpression.Arguments.OfType<MethodCallExpression>()
.SelectMany(mce => mce.Arguments.OfType<MemberExpression>())
.ToList();
var command = new string[tables.Count];
for(var i = 0; i < tables.Count; i++)
{
var table = tables[i];
var tableType = ((PropertyInfo) table.Member).PropertyType.GetGenericArguments()[0];
var tableName = tableType.GetCustomAttribute<TableAttribute>().Name;
command[i] = string.Format("(SELECT COUNT(*) FROM {0}) AS T{1}", tableName, i);
}
var dbCommand = db.Connection.CreateCommand();
dbCommand.CommandText = string.Format("SELECT {0}", String.Join(",", command));
db.Connection.Open();
IDataRecord result;
try
{
result = dbCommand.ExecuteReader().OfType<IDataRecord>().First();
}
finally
{
db.Connection.Close();
}
var results = new object[tables.Count];
for (var i = 0; i < tables.Count; i++)
results[i] = result.GetInt32(i);
var ctor = typeof(TOut).GetConstructor(Enumerable.Repeat(typeof(int), tables.Count).ToArray());
return (TOut) ctor.Invoke(results);
}
the code is called like this:
var counts = dbContext.CountMany(db => new
{
table1Count = db.Table1.Count(),
table2Count = db.Table2.Count()
//etc.
});