I am tryign to do bulk inserts/updates efficiently from c# code to oracle database.
If I done it by statement, then it doesn’t take much time.
I am using the ODP.NET
Currently insert of 6000 records via below stored proc is taking 15 mins.
I have to use this stored proc, because it generates unique user_id.
Is this proc doing auto-commit ? Is there any autocommit setting I should turn off ?
Please suggest ways to do it efficiently.
CREATE OR REPLACE
PROCEDURE sbx_staging_insert_user(client IN varchar2,
username IN varchar2,
comm_type IN varchar2,
email_addr IN varchar2,
buddy_name IN varchar2,
--default_flag IN char,
user_id OUT INT)
AS
BEGIN
select sbx_staging_user_id_seq.nextval into user_id from dual;
insert into sbx_staging_user
(user_id,
client,
username,
comm_type,
email_addr,
buddy_name,
default_flag)
values
(user_id,
client,
username,
comm_type,
email_addr,
buddy_name,
'Y');
end sbx_staging_insert_user;
and C# code is:
cmd.Transaction = conn.BeginTransaction();
cmd.CommandType = CommandType.Text;
cmd.CommandText = "sbx_staging_insert_user";
cmd.CommandType = CommandType.StoredProcedure;
var userClientParam = new OracleParameter(":client", OracleDbType.Varchar2);
var usernameParam = new OracleParameter(":username", OracleDbType.Varchar2);
var commTypeParam = new OracleParameter(":comm_type", OracleDbType.Varchar2);
var defaultParam = new OracleParameter(":default_flag", OracleDbType.Char) { Size = 1};
var emailParam = new OracleParameter(":email_addr", OracleDbType.Varchar2) { IsNullable = true };
var buddyParam = new OracleParameter(":buddy_name", OracleDbType.Varchar2) { IsNullable = true };
var userIdParam = new OracleParameter(":user_id", OracleDbType.Int32) { Direction = ParameterDirection.Output };
cmd.Parameters.Add(userClientParam);
cmd.Parameters.Add(usernameParam);
cmd.Parameters.Add(commTypeParam);
//cmd.Parameters.Add(defaultParam);
cmd.Parameters.Add(emailParam);
cmd.Parameters.Add(buddyParam);
cmd.Parameters.Add(userIdParam);
var cuList = new List<string>(Users.Count);
var uList = new List<string>(Users.Count);
var ctList = new List<string>(Users.Count);
var dfList = new List<char>(Users.Count);
var eaList = new List<string>(Users.Count);
var bnList = new List<string>(Users.Count);
var uiList = new List<decimal>(Users.Count);
int loopCnt = 0;
foreach (var ud in Users)
{
cuList.Add(ud.User.Client);
uList.Add(ud.User.Username);
ctList.Add(ud.User.CommType);
dfList.Add(ud.User.Default ? 'Y' : 'N');
eaList.Add(ud.User.Email);
bnList.Add(ud.User.BuddyName);
uiList.Add(-1);
}
userClientParam.Value = cuList.ToArray();
usernameParam.Value = uList.ToArray();
commTypeParam.Value = ctList.ToArray();
//defaultParam.Value = dfList.ToArray();
emailParam.Value = eaList.ToArray();
buddyParam.Value = bnList.ToArray();
userIdParam.Value = uiList.ToArray();
cmd.ArrayBindCount = cuList.Count;//Users.Count;//
cmd.ExecuteNonQuery();
cmd.Transaction.Commit();
Try something like this (untested, shortened for brevity):
public void insertRows()
{
if (Users.Count > 0)
{
OracleTransaction trans=_conn.BeginTransaction();
try
{
// create insert statement with bind vars
Stringbuilder sb = new Stringbuilder();
sb.Append("INSERT into sbx_staging_user(");
sb.Append("client,");
sb.Append("username,");
sb.Append("user_id");
sb.Append(") VALUES (");
sb.Append(":client,");
sb.Append(":username,");
sb.Append("seq_user_id.nextval");
sb.Append(") ");
OracleCommand cmd = new OracleCommand(sb.ToString(), _conn);
string[] ary_client = new string[Users.Count];
string[] ary_username = new string[Users.Count];
for (int i=0; i<Users.Count; i++)
{
User row=Users[i];
ary_client[i]=row.client;
ary_username[i]=row.username;
}
// prepare bind vars(bind in bulk using arrays)
OracleParameter prm=new OracleParameter();
cmd.Parameters.Clear();
cmd.ArrayBindCount=Users.Count;
cmd.BindByName=true;
prm=new OracleParameter("client", OracleDbType.Varchar2); prm.Value=ary_client; cmd.Parameters.Add(prm);
prm=new OracleParameter("username", OracleDbType.Varchar2); prm.Value=ary_username; cmd.Parameters.Add(prm);
cmd.ExecuteNonQuery();
trans.Commit();
trans.Dispose();
}
catch {
trans.Rollback();
trans.Dispose();
throw;
}
}
}
Maybe you should moove the bulk logic to the plsql.
See an example here:
http://dotnetslackers.com/articles/ado_net/BulkOperationsUsingOracleDataProviderForNETODPNET.aspx
Related
using (SqlConnection conn = new SqlConnection(connectionString))
{
conn.Open();
using (SqlCommand cmd = new SqlCommand("user_storedproc_name", conn))
{
cmd.CommandType = CommandType.StoredProcedure;
cmd.CommandTimeout = 0;
cmd.Parameters.Clear();
SqlParameter[] parameters = new SqlParameter[14];
parameters[0] = new SqlParameter("#p1", vlaue);
String ts = String.IsNullOrEmpty(value1)?"0":value1;
parameters[1] = new SqlParameter("p2", Convert.ToInt64(ts));
parameters[2] = new SqlParameter("p3", value3);
String strDate1 = String.IsNullOrEmpty(value4)? "01/01/1900":value4;
parameters[3] = new SqlParameter("#p4", ConvertDate(strDate1).ToString("dd/mm/yyyy"));
String strDate2 = String.IsNullOrEmpty(value5)? "01/01/1900":value5;
parameters[4] = new SqlParameter("#p5", ConvertDate(strDate2).ToShortDateString());
parameters[5] = new SqlParameter("#p6", value6);
String strDate3 = String.IsNullOrEmpty(value7)? "01/01/1900":value7;
parameters[6] = new SqlParameter("#p7", ConvertDate(strDate3).ToShortDateString());
String strDate4 = String.IsNullOrEmpty(value8)? "01/01/1900":value8;
parameters[7] = new SqlParameter("#p8", ConvertDate(strDate4).ToShortDateString());
parameters[8] = new SqlParameter("#p9", value9);
parameters[9] = new SqlParameter("#p10", value10);
parameters[10] = new SqlParameter("#p11", value11);
parameters[11] = new SqlParameter("#p12", value12);
String strDate5 = String.IsNullOrEmpty(value13)?"01/01/1900":value13;
parameters[12] = new SqlParameter("#p13", ConvertDate(strDate5).ToShortDateString());
string strp14val = String.IsNullOrEmpty(Convert.ToString(value14)) ? "0" : value14;
parameters[13] = new SqlParameter("#p14",DbType.Int32);
parameters[13].Direction = ParameterDirection.Output;
int r_retval = SQLHelper.ExecuteNonQuery(SQLHelper.GetConnString(), CommandType.StoredProcedure, "user_storedproc_name", "DB_NAME_STRING", parameters);
private DateTime ConvertDate(String date)
{
String str = String.Empty;
try
{
if (date.Contains("/") || date.Contains("-"))
{
str = date.Split('/')[2] + "-" + date.Split('/')[1] + "-" + date.Split('/')[0];
}
}
catch (Exception ex)
{
throw ex;
}
return Convert.ToDateTime(str);
}
Code behind throwing error nvarchar to datetime.
Note: This working fine with SqlCommand object and adding parameters with binding values. but the calling function was sqlParameter[] (parameters array) but the SqlCommand object parameters is a SqlParameterCollection. I spent almost 48 hours of time till now debugging and troubleshooting with no fair success. Throw some light, Something that my sight is not able to get in code that is wrong. awaiting for comments/answers what is possible to guide.
Note: Stored procedure is working fine
ALTER PROCEDURE [dbo].[User_StoredProc]
(#p1 nvarchar(500)
,#p2 nvarchar(50)=null
,#p3 bigint=null
,#p4 nvarchar(50)=null
,#p5 nvarchar(50)=null
,#p6 varchar(40)=null
,#p7 nvarchar(50)=null
,#p8 nvarchar(50)=null
,#p9 nvarchar(50)=null
,#p10 nvarchar(50)=null
,#p11 nvarchar(50)=null
,#p12 nvarchar(50)=null
,#p13 nvarchar(50)=null
,#p14 int OUTPUT)
AS
--DECLARE #USP_PROJECT_ID INT
BEGIN
IF (#p14=0)
--print 'enter'
SET dateformat 'dmy';
INSERT INTO Project_Details
(col1
,col2
,col3
,col4
,col5
,col6
,col7
,col8
,col9
,col10
,col11
,col12
,col13)
VALUES(#p1
,#p2
,#p3
,Convert(Datetime,#p4,103)
,Convert(Datetime,#p5,103)
,#p6
,Convert(Datetime,#p7,103)
,Convert(Datetime,#p8,103)
,#p9
,#p10
,#p11
,#p12
,#p13)
-- print SCOPE_IDENTITY()
SET #p13=SCOPE_IDENTITY()
SELECT Project_Details.Project_Id FROM Project_Details WHERE Project_Id=#Project_id
END
In this way, I retrieve data using the stored procedure:
var returnParameter = new SqlParameter("#RV", SqlDbType.Int) { Direction = ParameterDirection.Output };
var xmlParameter = new SqlParameter("#XML", XML) { DbType = DbType.Xml };
var familySentParameter = new SqlParameter("#FamilySent", SqlDbType.Int) { Direction = ParameterDirection.Output };
//var resultParameter = new SqlParameter("Result", SqlDbType.Text) { Direction = ParameterDirection.ReturnValue };
var sql = "exec #RV = uspConsumeSomething #XML, #FamilySent OUT";
var result = myContext.Database.ExecuteSqlCommand(sql, returnParameter, xmlParameter, familySentParameter);
RV = (int)returnParameter.Value;
Below the fragment of the stored procedure (termination of the procedure):
BEGIN CATCH
SELECT ERROR_MESSAGE();
IF ##TRANCOUNT > 0 ROLLBACK TRAN ENROLL;
SELECT * FROM #tblResult;
RETURN -400
END CATCH
SELECT Result FROM #tblResult;
RETURN 0 --ALL OK
END
How to obtain in C# data obtained from:
SELECT * FROM #tblResult;
SELECT Result FROM #tblResult;
They are not needed for the operation of the procedure but contain information about the type of error that arises in the stored procedure.
The use of Return in a stored procedure is supported by #RV
EDIT:
I'm trying a different approach but I still do not know how to get the values from SELECT here
DbConnection connection = myContext.Database.GetDbConnection();
using (DbCommand cmd = connection.CreateCommand())
{
cmd.CommandText = sql;
cmd.Parameters.AddRange(new[] { xmlParameter, returnParameter, familySentParameter });
if (connection.State.Equals(ConnectionState.Closed)) connection.Open();
using (var reader = cmd.ExecuteReader())
{
var entities = new List<string>();
while (reader.Read())
{
Debug.WriteLine("Reader: " + reader.GetValue(0));
}
}
}
As a result, there are 3 SELECT. How do you get access to 2 and 3?
I'm familiar with SQL Server but new to Oracle. I'm working on doing a bulk insert or update using the Array technique I've found in some examples. However, I must be doing this incorrect because I'm getting errors about "invalid table.column". I have a feeling it has to do with the way I'm trying to assign parameter names in the in-line SQL and command parameters. In the example I've seen they use numeric values for the parameters like ":1, :2, :3" but if possible I would like to use ":paramtername1, :parametername2, :parametername3".
Here is what I'm doing
sbQuery.Append("update SAP_EMPLOYEE set ");
sbQuery.Append(" EMP_ID = :p_EmployeeId, EVENT_FROM_DT = :p_EventFromDate, EVENT_TYP = :p_EventType, EVENT_RSN = :p_EventRsn,");
sbQuery.Append(" where Emp = :p_Employee");
Then I create arrays to add the values to
string[] arrEmployee = new string[listEmployee.Count];
string[] arrEmployeeId = new string[listEmployee.Count];
DateTime[] arrEventFromDt = new DateTime[listEmployee.Count];
string[] arrEventTyp = new string[listEmployee.Count];
string[] arrEventRsn = new string[listEmployee.Count];
Populate the arrays
int i = 0;
foreach (SAP_EMPLOYEE item in listEmployee)
{
arrEmployee[i] = item.EMP;
arrEmployeeId[i] = item.EMP_ID;
if(item.EVENT_FROM_DT.HasValue)
{
arrEventFromDt[i] = item.EVENT_FROM_DT.Value;
}
arrEventTyp[i] = item.EVENT_TYP;
arrEventRsn[i] = item.EVENT_RSN;
i++;
}
Then call the connection, cmd, query
OracleConnection objConnection = new OracleConnection(connString);
using (objConnection)
{
objConnection.Open();
//create Oracle parameters and pass arrays of data
OracleParameter p_Employee = new OracleParameter();
p_Employee.OracleDbType = OracleDbType.Varchar2;
p_Employee.Value = arrEmployee;
OracleParameter p_EmployeeId = new OracleParameter();
p_EmployeeId.OracleDbType = OracleDbType.Varchar2;
p_EmployeeId.Value = arrEmployeeId;
OracleParameter p_EventFromDate = new OracleParameter();
p_EventFromDate.OracleDbType = OracleDbType.Date;
p_EventFromDate.Value = arrEventFromDt;
OracleParameter p_EventType = new OracleParameter();
p_EventType.OracleDbType = OracleDbType.Char;
p_EventType.Value = arrEventTyp;
OracleParameter p_EventRsn = new OracleParameter();
p_EventType.OracleDbType = OracleDbType.Char;
p_EventType.Value = arrEventRsn;
OracleCommand objCmd = objConnection.CreateCommand();
objCmd.CommandText = sbQuery.ToString();
objCmd.ArrayBindCount = arrEmployee.Length;
objCmd.Parameters.Add(p_Employee);
objCmd.Parameters.Add(p_EmployeeId);
objCmd.Parameters.Add(p_EventFromDate);
objCmd.Parameters.Add(p_EventType);
objCmd.Parameters.Add(p_EventRsn);
objCmd.ExecuteNonQuery();
}
[WebMethod]
public List<reports> getMyReports( int user_id )
{
string cs = ConfigurationManager.ConnectionStrings["ReportDB"].ConnectionString;
using (SqlConnection con = new SqlConnection(cs))
{
SqlCommand cmd = new SqlCommand("getAllReportsByUserID", con);
cmd.CommandType = CommandType.StoredProcedure;
List<reports> repers = new List<reports>();
//users[][] liser = new users[][];
SqlParameter user_id_parameter = new SqlParameter("#user_id", user_id);
cmd.Parameters.Add(user_id_parameter);
reports report = new reports();
con.Open();
SqlDataReader reader = cmd.ExecuteReader();
while (reader.Read())
{
report.id = Convert.ToInt32(reader["id"]);
report.title = reader["title"].ToString();
report.description = reader["description"].ToString();
report.anonymous = (bool)reader["anonymous"];
report.location = reader["location"].ToString();
report.status = reader["status"].ToString();
report.category = reader["category"].ToString();
report.date = (DateTime)reader["date"];
report.picture_url = reader["picture_url"].ToString();
report.admin_id = Convert.ToInt32(reader["admin_id"]);
repers.Add(report);
}
return repers;
}
}
I have the top function that calls the following stored procedure:
CREATE Proc [dbo].[getAllReportsByUserID]
#user_id int
as
Begin
Select
id,
title,
description,
anonymous,
location,
status,
category,
date,
picture_url,
admin_id
from reports
where user_id = #user_id
End
I have tested the procedure individually and it works fine. Yet, when I test the WebService created above I get a list with the last value duplicated along the whole list.
Can someone please help me figure out why do I get the same (last)value repeated over and over again?
By creating the report object before the loop and reusing it repeatedly, you insert a reference to that same object multiple times in your list.
You should create the report object inside your loop:
SqlDataReader reader = cmd.ExecuteReader();
while (reader.Read())
{
reports report = new reports();
report.id = Convert.ToInt32(reader["id"]);
report.title = reader["title"].ToString();
report.description = reader["description"].ToString();
report.anonymous = (bool)reader["anonymous"];
report.location = reader["location"].ToString();
report.status = reader["status"].ToString();
report.category = reader["category"].ToString();
report.date = (DateTime)reader["date"];
report.picture_url = reader["picture_url"].ToString();
report.admin_id = Convert.ToInt32(reader["admin_id"]);
repers.Add(report);
}
return repers;
I'm writing a code generator and am getting stuck on determining the nullable status of a stored procedure result set Column. I can query the DataType just fine but neither the datareader object nor a data table column contain the correct nullable value of my column.
public List<DataColumn> GetColumnInfoFromStoredProcResult(string schema, string storedProcName)
{
//build sql text
var sb = new StringBuilder();
sb.Append("SET FMTONLY OFF; SET FMTONLY ON; \n");//this is how EF4.1 did so I copied..not sure why the repeat
sb.Append(String.Format("exec {0}.{1} ", schema, storedProcName));
var prms = GetStoredProcedureParameters(schema: schema, sprocName: storedProcName);
var count = 1;
foreach (var param in prms)
{
sb.Append(String.Format("{0}=null", param.Name));
if (count < prms.Count)
{
sb.Append(", ");
}
count++;
}
sb.Append("\n SET FMTONLY OFF; SET FMTONLY OFF;");
var dataTable = new DataTable();
//var list = new List<DataColumn>();
using (var sqlConnection = this.SqlConnection)
{
using (var sqlAdapter = new SqlDataAdapter(sb.ToString(), sqlConnection))
{
if (sqlConnection.State != ConnectionState.Open) sqlConnection.Open();
sqlAdapter.SelectCommand.ExecuteReader(CommandBehavior.KeyInfo);
sqlConnection.Close();
sqlAdapter.Fill(dataTable);
}
//using (var sqlCommand = new SqlCommand())
//{
// sqlCommand.CommandText = sb.ToString();
// sqlCommand.CommandType = CommandType.Text;
// sqlCommand.Connection = sqlConnection;
// if (sqlConnection.State != ConnectionState.Open) sqlConnection.Open();
// var dr = sqlCommand.ExecuteReader(CommandBehavior.SchemaOnly);
// var whateva = dr.GetSchemaTable();
// foreach (DataColumn col in whateva.Columns)
// {
// list.Add(col);
// }
//}
}
var list = dataTable.Columns.Cast<DataColumn>().ToList();
return list;
}
I'm trying to end up with something similar to the the Entities Framework creation of a complex type from a stored procedure. Can I hijack that functionality?
On this example the Id column.. tblJobId (not my naming convention) would never be null.. But I selected null as ImNull and it has all the same properties so how does EF determine if the corresponding C# data type should be nullable or not?
Has anybody done this..
Ideas are appreciated.
The secret was to use Schema Only and fill a dataset not datatable. Now the AllowDbNull property on the datacolumn properly displays the nullable status of the return value.
This was it...
public List<DataColumn> GetColumnInfoFromStoredProcResult(string schema, string storedProcName)
{
//build sql text
var sb = new StringBuilder();
sb.Append("SET FMTONLY OFF; SET FMTONLY ON; \n");//this is how EF4.1 did so I copied..not sure why the repeat
sb.Append(String.Format("exec {0}.{1} ", schema, storedProcName));
var prms = GetStoredProcedureParameters(schema: schema, sprocName: storedProcName);
var count = 1;
foreach (var param in prms)
{
sb.Append(String.Format("{0}=null", param.Name));
if (count < prms.Count)
{
sb.Append(", ");
}
count++;
}
sb.Append("\n SET FMTONLY OFF; SET FMTONLY OFF;");
var ds = new DataSet();
using (var sqlConnection = this.SqlConnection)
{
using (var sqlAdapter = new SqlDataAdapter(sb.ToString(), sqlConnection))
{
if (sqlConnection.State != ConnectionState.Open) sqlConnection.Open();
sqlAdapter.SelectCommand.ExecuteReader(CommandBehavior.SchemaOnly);
sqlConnection.Close();
sqlAdapter.FillSchema(ds, SchemaType.Source, "MyTable");
}
}
var list = ds.Tables[0].Columns.Cast<DataColumn>().ToList();
return list;
}
public List<SqlParamInfo> GetStoredProcedureParameters(string schema, string sprocName)
{
var sqlText = String.Format(
#"SELECT
[Name] = N'#RETURN_VALUE',
[ID] = 0,
[Direction] = 6,
[UserType] = NULL,
[SystemType] = N'int',
[Size] = 4,
[Precision] = 10,
[Scale] = 0
WHERE
OBJECTPROPERTY(OBJECT_ID(N'{0}.{1}'), 'IsProcedure') = 1
UNION
SELECT
[Name] = CASE WHEN p.name <> '' THEN p.name ELSE '#RETURN_VALUE' END,
[ID] = p.parameter_id,
[Direction] = CASE WHEN p.is_output = 0 THEN 1 WHEN p.parameter_id > 0 AND p.is_output = 1 THEN 3 ELSE 6 END,
[UserType] = CASE WHEN ut.is_assembly_type = 1 THEN SCHEMA_NAME(ut.schema_id) + '.' + ut.name ELSE NULL END,
[SystemType] = CASE WHEN ut.is_assembly_type = 0 AND ut.user_type_id = ut.system_type_id THEN ut.name WHEN ut.is_user_defined = 1 OR ut.is_assembly_type = 0 THEN st.name WHEN ut.is_table_type =1 Then 'STRUCTURED' ELSE 'UDT' END,
[Size] = CONVERT(int, CASE WHEN st.name IN (N'text', N'ntext', N'image') AND p.max_length = 16 THEN -1 WHEN st.name IN (N'nchar', N'nvarchar', N'sysname') AND p.max_length >= 0 THEN p.max_length/2 ELSE p.max_length END),
[Precision] = p.precision,
[Scale] = p.scale
FROM
sys.all_parameters p
INNER JOIN sys.types ut ON p.user_type_id = ut.user_type_id
LEFT OUTER JOIN sys.types st ON ut.system_type_id = st.user_type_id AND ut.system_type_id = st.system_type_id
WHERE
object_id = OBJECT_ID(N'{0}.{1}')
ORDER BY 2", schema, sprocName);
using (var sqlConnection = this.SqlConnection)
{
using (var sqlCommand = new SqlCommand())
{
if (sqlConnection.State != ConnectionState.Open) sqlConnection.Open();
sqlCommand.Connection = sqlConnection;
sqlCommand.CommandType = CommandType.Text;
sqlCommand.CommandText = sqlText;
var dr = sqlCommand.ExecuteReader();
var result = new List<SqlParamInfo>();
while (dr.Read())
{
if (Convert.ToString(dr["Name"]) != "#RETURN_VALUE")
{
result.Add(new SqlParamInfo(dr));
}
}
return result;
}
}
}
Assume, that every column which comes from SP can be null - this is a valid assumption because stored procedure - its a kind of data abstraction layer and thus its code can change but still produce valid results.
If column was non-nullable yesterday it means nothing for today. So - all the columns which come from SP resultsets are nullable by design.
Update.
Assuming that table t1 has column Id INT IDENTITY PRIMARY KEY
Your stored proc looks like this:
CREATE PROC p1
AS
BEGIN
SELECT Id FROM t1
END
So it will never return an Id = NULL, but this is the SP - an abstraction of data, so - tomorrow i'll modify it like this:
CREATE PROC p1
AS
BEGIN
SELECT Id FROM t1
UNION
SELECT NULL
END
So, now it returns NULL - think about this. The difference in understanding of data abstraction