Related
I've been trying to figure this out for the past few days, but I just can't seem to get it work.
So I have a txt file which has this format:
id;könyvcím;szerző;kiadó;kiadási év;
I am using a structs and a list such as this:
public static List<Books> BooksList = new List<Books>();
public struct Books
{
public int id;
public string title;
public string writer;
public string publisher;
public int published_year;
}
And I'm also putting all these into a List based on the struct like this:
StreamReader booksRead = new StreamReader("konyvek.txt", Encoding.UTF8);
booksRead.ReadLine();
while (!booksRead.EndOfStream)
{
string[] split = booksRead.ReadLine().Split(';');
Books inRead = new Books();
inRead.id = Convert.ToInt32(split[0]);
inRead.title = split[1];
inRead.writer = split[2];
inRead.publisher = split[3];
inRead.published_year = Convert.ToInt32(split[4]);
BooksList.Add(inRead);
}
booksRead.Close();
All I want is, for example, to find where the line with ID 2 is, and remove that line from my textfile. I've tried to get the index of the line I want, and remove it like that from my textfile, but it even fails to get the index, I tried using IndexOf, FindIndex and trying to go on a loop. I'm pretty sure my struct is not happy with me for using it like that because I get errors such as this when I run my code:
System.InvalidCastException: 'Unable to cast object of type 'Books' to
type 'System.IConvertible'.'
Here is the way I'm trying to get the index of the line I want to remove
Books item = new Books();
for (int i = 0; i < BooksList.Count; i++)
{
if (Convert.ToInt32(textBox_id_delete.Text) == item.id)
{
RemoveAt = item.id;
}
}
int index = BooksList.FindIndex(x => Convert.ToInt32(x) == RemoveAt);
MessageBox.Show(Convert.ToString(index));
I'm pretty sure I'm approaching this extremely wrong, and I'd accept any kind of help.
You are doing it completely wrong for a number of reasons.
First, how would you do that the way you are doing:
void Main()
{
var filename = #"c:\myFolder\mybooklist.txt";
// read into an enumerable
var books = File.ReadAllLines(filename)
.Select(x => x.Split(';'))
.Select(x => new Book {
Id = int.TryParse(x[0], out int bookId)?bookId:0,
Title = x[1],
Writer = x[2],
Publisher = x[3],
Published_year=int.TryParse(x[4], out int year)?year:0
});
// remove the one with id 2
// and save back
var otherBooks = books.Where(b => b.Id != 2);
File.WriteAllLines(filename, otherBooks.Select(b => $"{b.Id};{b.Title};{b.Writer};{b.Publisher};{b.Published_year}"));
}
public struct Book
{
public int Id;
public string Title;
public string Writer;
public string Publisher;
public int Published_year;
}
And now what is wrong with this.
A text file is not a database but you are trying to use a text file as a database.
With a text file, you are not actually doing any control here, if the ID is unique or not (there might be N books with the ID 2).
(Side matter) You are using C#, but looks like you are coming from another language and not using the naming conventions at all.
IMHO, instead you should simply use a database, an embedded one for example like LiteDb or Sqlite. If you care to see a sample with LiteDb or Sqlite, let me know.
EDIT: I am adding SQLite and LiteDb samples. In either case, you would need to add Sqlite.Data.Sqlite and LiteDB respectively from Nuget and add using statements.
In case of SQLite, please note that you could use Linq adding some drivers. I directly used the ADO.Net commands and didn't use a Book class for mapping.
LiteDB, being a NoSQL database written in C# for C#, can directly use objects and support Linq out of the box.
Samples show only the surface for both.
SQLite sample:
private static readonly string dataFile = #"d:\temp\books.s3db";
void Main()
{
CreateDb(dataFile);
SeedSampleData(dataFile);
// List the current data
Console.WriteLine("Current Data");
Console.WriteLine("".PadRight(100, '='));
ListData(dataFile);
Console.WriteLine("".PadRight(100, '='));
DeleteSampleRow(dataFile);
// List the current data
Console.WriteLine("After deleting");
Console.WriteLine("".PadRight(100, '='));
ListData(dataFile);
Console.WriteLine("".PadRight(100, '='));
}
void DeleteSampleRow(string dbName)
{
string deleteById = "delete from books where id = #id";
string deleteByTitle = "delete from books where Title = #title";
string deleteByWriter = "delete from books where Writer = #writer";
using (SQLiteConnection cn = new SQLiteConnection($"Data Source={dbName}"))
using (SQLiteCommand cmdById = new SQLiteCommand(deleteById, cn))
using (SQLiteCommand cmdByTitle = new SQLiteCommand(deleteByTitle, cn))
using (SQLiteCommand cmdByWriter = new SQLiteCommand(deleteByWriter, cn))
{
cmdById.Parameters.Add("#id", DbType.Int32).Value = 2; // delete the book with id = 2
cmdByTitle.Parameters.Add("#title", DbType.String).Value = $"Sample Title #5"; // delete all books having title "Sample Title #5"
cmdByWriter.Parameters.Add("#writer", DbType.String).Value = $"Sample Writer #3"; // delete all books written by "Sample Writer #3"
cn.Open();
cmdById.ExecuteNonQuery();
cmdByTitle.ExecuteNonQuery();
cmdByWriter.ExecuteNonQuery();
cn.Close();
}
}
void ListData(string dbName)
{
string selectCommand = "select * from books";
using (SQLiteConnection cn = new SQLiteConnection($"Data Source={dbName}"))
using (SQLiteCommand cmd = new SQLiteCommand(selectCommand, cn))
{
cn.Open();
var r = cmd.ExecuteReader();
while (r.Read())
{
Console.WriteLine($"{r["id"]},{r["title"]},{r["writer"]},{r["publisher"]},{r["published_year"]}");
}
cn.Close();
}
}
private void CreateDb(string dbName)
{
if (File.Exists(dbName)) // if it exists, delete and create afresh, just for sampling
{
File.Delete(dbName);
}
string createTable = #"Create Table books (
id int primary key not null,
title varchar(500) not null,
writer varchar(100) not null,
publisher varchar(100) not null,
published_year int not null
)";
using (SQLiteConnection cn = new SQLiteConnection($"Data Source={dbName}"))
using (SQLiteCommand cmd = new SQLiteCommand(createTable, cn))
{
cn.Open();
cmd.ExecuteNonQuery();
cn.Close();
}
}
private void SeedSampleData(string dbName)
{
string insertCommand = #"insert into books
(id, title, writer, publisher, published_year)
values
(#id, #title, #writer, #publisher, #year);";
using (SQLiteConnection cn = new SQLiteConnection($"Data Source={dbName}"))
using (SQLiteCommand cmd = new SQLiteCommand(insertCommand, cn))
{
cmd.Parameters.Add("#id", DbType.Int32);
cmd.Parameters.Add("#title", DbType.String);
cmd.Parameters.Add("#writer", DbType.String);
cmd.Parameters.Add("#publisher", DbType.String);
cmd.Parameters.Add("#year", DbType.Int32);
Random r = new Random();
cn.Open();
int id = 1;
using (SQLiteTransaction transaction = cn.BeginTransaction())
{
cmd.Parameters["#id"].Value = id++;
cmd.Parameters["#title"].Value = $"Around the World in Eighty Days";
cmd.Parameters["#writer"].Value = $"Jules Verne";
cmd.Parameters["#publisher"].Value = $"Le Temps, Pierre-Jules Hetzel";
cmd.Parameters["#year"].Value = 1873;
cmd.ExecuteNonQuery();
cmd.Parameters["#id"].Value = id++;
cmd.Parameters["#title"].Value = $"A Tale of Two Cities";
cmd.Parameters["#writer"].Value = $"Charles Dickens";
cmd.Parameters["#publisher"].Value = $"Chapman & Hall";
cmd.Parameters["#year"].Value = 1859;
cmd.ExecuteNonQuery();
// add dummy 10 more rows
for (int i = 0; i < 10; i++)
{
cmd.Parameters["#id"].Value = id++;
cmd.Parameters["#title"].Value = $"Sample Title #{i}";
cmd.Parameters["#writer"].Value = $"Sample Writer #{r.Next(1, 5)}";
cmd.Parameters["#publisher"].Value = $"Sample Publisher #{i}";
cmd.Parameters["#year"].Value = r.Next(1980, 2022);
cmd.ExecuteNonQuery();
}
transaction.Commit();
}
// databases generally use some indexes
new SQLiteCommand(#"Create Index if not exists ixId on books (id);", cn).ExecuteNonQuery();
new SQLiteCommand(#"Create Index if not exists ixTitle on books (title);", cn).ExecuteNonQuery();
new SQLiteCommand(#"Create Index if not exists ixWriter on books (writer);", cn).ExecuteNonQuery();
new SQLiteCommand(#"Create Index if not exists ixPublisher on books (publisher);", cn).ExecuteNonQuery();
cn.Close();
}
}
LiteDb sample:
private static readonly string dataFile = #"d:\temp\books.litedb";
void Main()
{
//CreateDb(dataFile); // this step is not needed with LiteDB
// instead we just simply delete the datafile if it exists
// for starting afresh
// if it exists, delete and create afresh, just for sampling
// so you can run this same sample over and over if you wish
if (File.Exists(dataFile))
{
File.Delete(dataFile);
}
SeedSampleData(dataFile);
// List the current data
Console.WriteLine("Current Data");
Console.WriteLine("".PadRight(100, '='));
ListData(dataFile);
Console.WriteLine("".PadRight(100, '='));
DeleteSampleRow(dataFile);
// List the current data
Console.WriteLine("After deleting");
Console.WriteLine("".PadRight(100, '='));
ListData(dataFile);
Console.WriteLine("".PadRight(100, '='));
}
void DeleteSampleRow(string dbName)
{
using (var db = new LiteDatabase(dbName))
{
var bookCollection = db.GetCollection<Book>("Books");
// by ID
bookCollection.Delete(2);
// by Title
bookCollection.DeleteMany(c => c.Title == "Sample Title #5");
// by Writer
bookCollection.DeleteMany(c => c.Writer == "Sample Writer #3");
}
}
void ListData(string dbName)
{
using (var db = new LiteDatabase(dbName))
{
var bookCollection = db.GetCollection<Book>("Books");
foreach (var book in bookCollection.FindAll())
{
Console.WriteLine($"{book.Id},{book.Title},{book.Writer},{book.Publisher},{book.Published_year}");
}
}
}
private void SeedSampleData(string dbName)
{
Random r = new Random();
var books = new List<Book> {
new Book {Title="Around the World in Eighty Days",Writer = "Jules Verne",Publisher = "Le Temps, Pierre-Jules Hetzel",Published_year= 1873},
new Book {Title="A Tale of Two Cities",Writer = "Charles Dickens",Publisher = "Chapman & Hall",Published_year= 1859},
};
// add dummy 10 more rows
books.AddRange(Enumerable.Range(0, 10).Select(i => new Book
{
Title = $"Sample Title #{i}",
Writer = $"Sample Writer #{r.Next(1, 5)}",
Publisher = $"Sample Publisher #{i}",
Published_year = r.Next(1980, 2022)
}));
using (var db = new LiteDatabase(dbName))
{
var bookCollection = db.GetCollection<Book>("Books");
bookCollection.InsertBulk(books);
// databases generally use some indexes
// create the same indexes that we created in SQLite sample
bookCollection.EnsureIndex(c => c.Id);
bookCollection.EnsureIndex(c => c.Title);
bookCollection.EnsureIndex(c => c.Writer);
bookCollection.EnsureIndex(c => c.Publisher);
}
}
public class Book
{
public int Id {get;set;}
public string Title {get;set;}
public string Writer {get;set;}
public string Publisher {get;set;}
public int Published_year {get;set;}
}
welcome to SO. I'm going to assume you've got a reason for keeping the data in a text file. As several answers have suggested if you need it in a text file the easiest thing to do is to simply create a new file with the lines you want.
One way to do that is to make use of a interator function to filter the lines. This lets you easily use the .NET File class to do the rest - creating the new file and removing the old if you want to. Often keeping the old file and archiving it can be useful too but anyway, here's a way to filter the lines.
static void Main(string[] _)
{
var filteredLines = FilterOnID(File.ReadAllLines("datafile.txt"), "2");
File.WriteAllLines("updated.datafile.txt", filteredLines);
// rename if necessary
File.Delete("datafile.txt");
File.Move("updated.datafile.txt", "datafile.txt");
}
static IEnumerable<string> FilterOnID(IEnumerable<string> lines, string id)
{
foreach (var line in lines)
{
var fields = line.Split(';');
if (fields.Length != 0 || !string.IsNullOrEmpty(fields[0]))
{
if (id == fields[0])
continue;
}
yield return line;
}
}
To test I added simple file like so:
1;field1;field2;field3
2;field1;field2;field3
3;field1;field2;field3
4;field1;field2;field3
5;field1;field2;field3
6;field1;field2;field3
And after running you get this:
1;field1;field2;field3
3;field1;field2;field3
4;field1;field2;field3
5;field1;field2;field3
6;field1;field2;field3
When you put books into a list from a file, you can search the book for remove from BooksList.
Delete it and save BooksList into a file.
var removeBook = BookList.FirstOrDefault(book => book.id == removeId);
if (removeBook != null)
{
BookList.Remove(removeBook);
}
var booksAsString = BookList.Select(book => $"{book.id};{book.title};{book.writer};{book.publisher};{book.published_year}");
File.WriteAllLines("konyvek.txt", booksAsString, Encoding.UTF8);
I have problem
I need to get list from list in linq. But it type of list is unknown on compilation stage.
using(var context = MyDbContext())
{
var list = (from p in context.Employee select p).ToList()
}
I dont know what just property (change Employee)
I want to do something that
public IList<T> GetAll(string propName)
{
using (var context = new ModelContext())
{
return (from p in context.GetType().GetProperty(propName) select p).ToList();
}
}
The DbContext.Database property provides an API that allows you to perform ADO.NET operations directly. The GetDbConnection method returns a DbConnection object representing the context's underlying connection. From that point, you can revert to the familiar ADO.NET APIs:
public dynamic GetAll(string propName)
{
using (var context = new SampleContext())
using (var command = context.Database.GetDbConnection().CreateCommand())
{
//Escape propName to prevent SQL injection
var builder = new SqlCommandBuilder();
string escapedTableName = builder.QuoteIdentifier(propName);
command.CommandText = $"SELECT * From {escapedTableName}";
context.Database.OpenConnection();
using (var dataReader= command.ExecuteReader())
{
var dataTable = new DataTable();
dataTable.Load(dataReader);
return dataTable ;
}
}
}
I am doing a simple select with a date filter on it with a months range where only 32 records are present however its taking 15 seconds to query and return the data I am using sage 50 as you can probally tell and c#. I am using odbc to create the query the seem speeds can be found if i use the odbc query tool.
This is for a stright forward select and it should not be taking that long to return the data through odbc.
String SQL = string.Format("SELECT 'ORDER_NUMBER', 'ORDER_OR_QUOTE',
'ANALYSIS_1','ACCOUNT_REF','ORDER_DATE','NAME',
'COURIER_NUMBER','COURIER_NAME','CUST_TEL_NUMBER'
,'DESPATCH_DATE','ACCOUNT_REF', 'DEL_NAME', 'DEL_ADDRESS_1',
'DEL_ADDRESS_2', 'DEL_ADDRESS_3', 'DEL_ADDRESS_4', 'DEL_ADDRESS_5',
'INVOICE_NUMBER','ORDER_DATE','INVOICE_NUMBER_NUMERIC',
'CONTACT_NAME','CONSIGNMENT', 'NOTES_1', 'ITEMS_NET'
,'ITEMS_GROSS','QUOTE_STATUS' FROM SALES_ORDER WHERE ORDER_DATE
='{0}' and ORDER_DATE <='{1}'", fromD, toD);
public List<SalesOrders> GetSalesOrders()
{
List<SalesOrders> _salesOrdersList = new List<SalesOrders>();
try
{
string sageDsn = ConfigurationManager.AppSettings["SageDSN"];
string sageUsername = ConfigurationManager.AppSettings["SageUsername"];
string sagePassword = ConfigurationManager.AppSettings["SagePassword"];
//int totalRecords = GetSalesOrdersount();
int counter = 0;
//using (var connection = new OdbcConnection("DSN=SageLine50v24;Uid=Manager;Pwd=;"))
using (var connection = new OdbcConnection(String.Format("DSN={0};Uid={1};Pwd={2};", sageDsn, sageUsername, sagePassword)))
{
connection.Open();
//string sql = string.Format(getInvoiceSql, customerCode, DateTime.Today.AddMonths(-1).ToString("yyyy-MM-dd"));
string fromD = dtpFrom.Value.ToString("yyyy-MM-dd");
string toD = dtpTo.Value.ToString("yyyy-MM-dd");
String SQL = string.Format("SELECT 'ORDER_NUMBER', 'ORDER_OR_QUOTE', 'ANALYSIS_1','ACCOUNT_REF','ORDER_DATE','NAME', 'COURIER_NUMBER','COURIER_NAME','CUST_TEL_NUMBER' ,'DESPATCH_DATE','ACCOUNT_REF', 'DEL_NAME', 'DEL_ADDRESS_1', 'DEL_ADDRESS_2', 'DEL_ADDRESS_3', 'DEL_ADDRESS_4', 'DEL_ADDRESS_5', 'INVOICE_NUMBER','ORDER_DATE','INVOICE_NUMBER_NUMERIC', 'CONTACT_NAME','CONSIGNMENT', 'NOTES_1', 'ITEMS_NET' ,'ITEMS_GROSS','QUOTE_STATUS' FROM SALES_ORDER WHERE ORDER_DATE >='{0}' and ORDER_DATE <='{1}'", fromD, toD);
using (var command = new OdbcCommand(SQL, connection))
{
using (var reader = command.ExecuteReader())
{
while (reader.Read())
{
counter++;
backgroundWorker1.ReportProgress(counter);
var salesOrders = new SalesOrders();
salesOrders.ACCOUNT_REF = Convert.ToString(reader["ACCOUNT_REF"]);
salesOrders.RecordIdentifier = "";
salesOrders.ShipmmentId = Convert.ToString(reader["ORDER_NUMBER"]);
salesOrders.OrderDate = Convert.ToDateTime(reader["ORDER_DATE"]);
salesOrders.OrderNumber = Convert.ToString(reader["ORDER_NUMBER"]);
salesOrders.Company = "";
salesOrders.Carrier = Convert.ToString(reader["COURIER_NUMBER"]);
salesOrders.CarrierService = Convert.ToString(reader["COURIER_NAME"]);
salesOrders.CustomerName = Convert.ToString(reader["NAME"]);
salesOrders.ShipToAddress1 = Convert.ToString(reader["DEL_ADDRESS_1"]);
salesOrders.ShipToAddress2 = Convert.ToString(reader["DEL_ADDRESS_2"]);
salesOrders.ShipToAddress3 = Convert.ToString(reader["DEL_ADDRESS_3"]);
salesOrders.ShipToAddress4 = Convert.ToString(reader["DEL_ADDRESS_4"]);
salesOrders.ShipToAddress5 = Convert.ToString(reader["DEL_ADDRESS_5"]);
salesOrders.ShiptoAttention = Convert.ToString(reader["DEL_NAME"]);
salesOrders.ShiptoPhoneNo = Convert.ToString(reader["CUST_TEL_NUMBER"]);
salesOrders.Country = Convert.ToString(reader["ANALYSIS_1"]);
salesOrders.ShiptoEmail = "";
salesOrders.MakeAddressDefault = "Y";
bool isProcessed = _sqlManager.hasbeenProcessed(salesOrders.OrderNumber);
if (isProcessed == true)
salesOrders.Exported = true;
_salesOrdersList.Add(salesOrders);
}
}
}
}
return _salesOrdersList.OrderByDescending(o => o.OrderDate).ToList();
}
don't use {0}, {1} for embedding values in strings... ADD via Parameters
String SQL =
#"SELECT
ORDER_NUMBER,
ORDER_OR_QUOTE,
ANALYSIS_1,
ACCOUNT_REF,
ORDER_DATE,
`NAME`,
COURIER_NUMBER,'
OURIER_NAME,
CUST_TEL_NUMBER,
DESPATCH_DATE,
ACCOUNT_REF,
DEL_NAME,
DEL_ADDRESS_1,
DEL_ADDRESS_2,
DEL_ADDRESS_3,
DEL_ADDRESS_4,
DEL_ADDRESS_5,
INVOICE_NUMBER,
ORDER_DATE,
INVOICE_NUMBER_NUMERIC,
CONTACT_NAME,
CONSIGNMENT,
NOTES_1,
ITEMS_NET,
ITEMS_GROSS,
QUOTE_STATUS
FROM
SALES_ORDER
WHERE
ORDER_DATE >= ?
and ORDER_DATE <= ?
ORDER BY
ORDER_DATE DESC";
using (var command = new OdbcCommand(SQL, connection))
{
// assuming fields are actually date data types fields
command.Parameters.Add( "parmFromDate", fromD );
command.Parameters.Add( "parmToDate", toD );
The "?" in the query are place-holders for the parameter values which are handled by the ODBC process. The Parameters being added within the using() portion are added in the same ordinal position as their respective place-holder parts. I just assigned the parameter name to give context to whoever is looking at it after.
The query itself SHOULD be very quick depending on the date range you are pulling. Even added the SQL Order by descending order so it is pre-pulled down in the order you intended it too.
I am having difficulty with the following method:
public override List<Team> Search(Dictionary<string, string> prms,
int pageSize, int page, out int results)
{
List<Team> t = null;
//Team t = null;
var tresults = new List<Team>();
using (SqlConnection conn = DB.GetSqlConnection())
{
using (SqlCommand cmd = conn.CreateCommand())
{
cmd.CommandText = #"SearchForTeam";
cmd.CommandType = System.Data.CommandType.StoredProcedure;
foreach (var key in prms.Keys)
{
cmd.Parameters.Add(key, prms[key]);
}
SqlDataReader reader
= cmd.ExecuteReader(CommandBehavior.CloseConnection);
while (reader.Read())
{
var temp = Load(reader);
if (t == null)
{
t = temp;
}
else
{
t.CityHistory.Add(temp.CityHistory[0]);
}
}
}
}
results = 0;
return t;
}
The error lies mainly with the if and else statement where the temp in the if block is claiming that it "cannot implicitly convert type DataLayer.Team to System.Collections.GenericList"
EDIT:
Here is my load method:
public Team Load(SqlDataReader reader)
{
var team = new Team()
{
TeamID = Int32.Parse(reader["TeamID"].ToString()),
TeamName = reader["TeamName"].ToString()
};
team.CityHistory.Add(
new TeamCity(
Int32.Parse(reader["TeamCitiesID"].ToString()),
team.TeamID,
Int32.Parse(reader["CityID"].ToString()),
reader["CityName"].ToString(),
Int32.Parse(reader["YearStart"].ToString()),
Int32.Parse(reader["YearEnd"].ToString())
)
);
return team;
}
t is defined as List<Team>, yet you later say t.CityHistory. CityHistory is clearly not a property of List<>. I'd guess it's a property of Team, but since you never show us that we can't say.
Show us the definition of Team and the method signature of Load() and perhaps we can give an answer.
UPDATE (from OP's update)
Now, I'm going to assume that you are getting multiple rows, one for each City, with the team info repeating. So, which you want is:
var temp = Load(reader);
// remove t definition above
var t = tresults.FirstOrDefault(team=> team.TeamId == temp.TeamId);
if (t == null)
{
t = temp;
tresults.Add(t);
}
else
t.CityHistory.Add(temp.CityHistory[0]);
(Updated again, based on Steve's comment)
You have to wrap temp into a List first to assign it to t:
if (t == null) {
t = new List<Team>() { temp }
} else {
t.add(temp);
}
VERY LONG COMMENT:
Consider the following refactoring of your method:
private IEnumerable<Team> LoadTeams()
{
using (SqlConnection conn = DB.GetSqlConnection())
{
using (SqlCommand cmd = conn.CreateCommand())
{
cmd.CommandText = #"SearchForTeam";
cmd.CommandType = System.Data.CommandType.StoredProcedure;
foreach (var key in prms.Keys)
{
cmd.Parameters.Add(key, prms[key]);
}
SqlDataReader reader
= cmd.ExecuteReader(CommandBehavior.CloseConnection);
while (reader.Read())
{
yield return Load(reader);
}
}
}
}
public override List<Team> Search(Dictionary<string, string> prms,
int pageSize, int page, out int results)
{
List<Team> searchResult = new List<Team>;
//Team t = null;
var tresults = new List<Team>();
foreach(Team team in LoadTeams())
{
if (team .....)
searchResult.Add(team);
}
results = 0;
return searchResult;
}
Take into account this NEEDED initialization:
List<Team> searchResult = new List<Team>;
And ask yourself: What should the following excerpt do? :
if (team .....)
searchResult.Add(team);
P.S.: Also the line
results = 0;
should probably look like:
results = searchResult.Count;
I'm having to guess a little here, but I assume that your Load() method is returning a data type 'DataLayer.Team', which sounds like its one Team, and you're trying to assign it to 't', which is a list of teams.
Try:
t.Add(temp)
or
t.Add(temp as Team).
It doesn't help that you're using 'var' declarations all the time.
I have a table which contains around 500 Million records. I am reading the data from table and storing those in a Dictionary.
EDIT: I am loading the data into dictionary because these data needs to be compared with another volume of data coming from an indexing server.
My code is as below:
public static void GetDetailsFromDB()
{
string sqlStr = "SELECT ID, Name ,Age, email ,DOB ,Address ,Affiliation ,Interest ,Homepage FROM Author WITH (NOLOCK) ORDER BY ID";
SqlCommand cmd = new SqlCommand(sqlStr, _con);
cmd.CommandTimeout = 0;
using (SqlDataReader reader = cmd.ExecuteReader())
{
while (reader.Read())
{
//Author Class
Author author = new Author();
author.id = Convert.ToInt32(reader["ID"].ToString());
author.Name = reader["Name"].ToString().Trim();
author.age = Convert.ToInt32(reader["Age"].ToString());
author.email = reader["email"].ToString().Trim();
author.DOB = reader["DOB"].ToString().Trim();
author.Address = reader["Address"].ToString().Trim();
author.Affiliation = reader["Affiliation"].ToString().Trim();
author.Homepage = reader["Homepage"].ToString().Trim();
string interests = reader["Interest"].ToString().Trim();
author.interest = interests.Split(new char[] { ',' }, StringSplitOptions.RemoveEmptyEntries).Select(p => p.Trim()).ToList();
if (!AuthorDict.ContainsKey(author.id))
{
AuthorDict.Add(author.id, author);
}
if (AuthorDict.Count % 1000000 == 0)
{
Console.WriteLine("{0}M author loaded.", AuthorDict.Count / 1000000);
}
}
}
}
This process is taking long time to read and store all 500 Million records from DB. Also, the RAM usage is very high.
Can this be optimized ? also, can the running time be decreased ? any help is appreciated.
If I hold my nose I can come up with the following optimisations:
Store the ordinal positions of your fields in local variables and reference the fields in your reader using these ordinal variables.
Do not call ToString on the reader and convert - just get the value out in the correct type.
Check for the existence of the author id in the AuthorDict as soon as you have the id. Don't even create the Author instance if you don't need it.
using (SqlDataReader reader = cmd.ExecuteReader())
{
var idOrdinal = reader.GetOrdinal("ID");
//extract other ordinal positions and store here
while (reader.Read())
{
var id = reader.GetInt32(idOrdinal);
if (!AuthorDict.ContainsKey(id))
{
Author author = new Author();
author.id = reader.GetInt32(idOrdinal);
...
}
}
}