I have a program that writes a huge DataTable (2.000.000 to 70.000.000 rows, depends on the configuration) to a database using a SqlBulkCopy.
I decided to change the loop that populates this table into a IDataReader, because the amount of rows often causes an OutOfMemoryException.
The table is populated like this
// int[] firsts;
// string[] seconds;
// byte[] thirds;
var table = new DataTable();
foreach(var f in firsts)
{
foreach(var s in seconds)
{
foreach(var t in thirds)
{
var row = table.NewRow();
row[0] = f;
row[1] = s;
row[2] = t;
table.Rows.Add(row);
}
}
// here I also bulk load the table and clear it
}
so in my IDataReader class I will loop by index. This is my attempt.
class TableReader : IDataReader
{
bool Eof = false;
int FirstIndex;
int SecondIndex;
int ThirdIndex;
//those are populated via constructor
int[] firsts;
string[] seconds;
byte[] thirds;
// this will be retrieved automatically via indexer
object[] Values;
public bool Read()
{
if(ThirdIndex != thirds.Length
&& SecondIndex < seconds.Length
&& FirstIndex < firsts.Length)
{
Values[0] = firsts[FirstIndex];
Values[1] = seconds[SecondIndex];
Values[2] = thirds[ThirdIndex++];
}
else if(SecondIndex != seconds.Length)
{
ThirdIndex = 0;
SecondIndex++;
}
else if(FirstIndex != firsts.Length)
{
SecondIndex = 0;
FirstIndex++;
}
else
{
Eof = true;
}
return !Eof;
}
}
I've created this code using a while(true) loop with a break instead of the Eof, but I can't seem to figure out how to do this.
Anyone can help?
This is actually possible if you implement IDataReader and use the "yield return" keyword to provide rows. IDataReader is a bit of a pain to implement, but it isn't complex at all. The code below can be adapted to load a terabyte worth of data to the database and never run out of memory.
I replaced the DataRow objects with a single object array that is reused throughout the data read.
Because there's no DataTable object to represent the columns, I had to do this myself by storing the data types and column names separately.
class TestDataReader : IDataReader {
int[] firsts = { 1, 2, 3, 4 };
string[] seconds = { "abc", "def", "ghi" };
byte[] thirds = { 0x30, 0x31, 0x32 };
// The data types of each column.
Type[] dataTypes = { typeof(int), typeof(string), typeof(byte) };
// The names of each column.
string[] names = { "firsts", "seconds", "thirds" };
// This function uses coroutines to turn the "push" approach into a "pull" approach.
private IEnumerable<object[]> GetRows() {
// Just re-use the same array.
object[] row = new object[3];
foreach (var f in firsts) {
foreach (var s in seconds) {
foreach (var t in thirds) {
row[0] = f;
row[1] = s;
row[2] = t;
yield return row;
}
}
// here I also bulk load he table and clear it
}
}
// Everything below basically wraps this.
IEnumerator<object[]> rowProvider;
public TestDataReader() {
rowProvider = GetRows().GetEnumerator();
}
public object this[int i] {
get {
return GetValue(i);
}
}
public object this[string name] {
get {
return GetValue(GetOrdinal(name));
}
}
public int Depth { get { return 0; } }
public int FieldCount { get { return dataTypes.Length; } }
public bool IsClosed { get { return false; } }
public int RecordsAffected { get { return 0; } }
// These don't really do anything.
public void Close() { Dispose(); }
public void Dispose() { rowProvider.Dispose(); }
public string GetDataTypeName(int i) { return dataTypes[i].Name; }
public Type GetFieldType(int i) { return dataTypes[i]; }
// These functions get basic data types.
public bool GetBoolean(int i) { return (bool) rowProvider.Current[i]; }
public byte GetByte(int i) { return (byte) rowProvider.Current[i]; }
public char GetChar(int i) { return (char) rowProvider.Current[i]; }
public DateTime GetDateTime(int i) { return (DateTime) rowProvider.Current[i]; }
public decimal GetDecimal(int i) { return (decimal) rowProvider.Current[i]; }
public double GetDouble(int i) { return (double) rowProvider.Current[i]; }
public float GetFloat(int i) { return (float) rowProvider.Current[i]; }
public Guid GetGuid(int i) { return (Guid) rowProvider.Current[i]; }
public short GetInt16(int i) { return (short) rowProvider.Current[i]; }
public int GetInt32(int i) { return (int) rowProvider.Current[i]; }
public long GetInt64(int i) { return (long) rowProvider.Current[i]; }
public string GetString(int i) { return (string) rowProvider.Current[i]; }
public object GetValue(int i) { return (object) rowProvider.Current[i]; }
public string GetName(int i) { return names[i]; }
public bool IsDBNull(int i) {
object obj = rowProvider.Current[i];
return obj == null || obj is DBNull;
}
// Looks up a field number given its name.
public int GetOrdinal(string name) {
return Array.FindIndex(names, x => x.Equals(name, StringComparison.OrdinalIgnoreCase));
}
// Populate "values" given the current row of data.
public int GetValues(object[] values) {
if (values == null) {
return 0;
} else {
int len = Math.Min(values.Length, rowProvider.Current.Length);
Array.Copy(rowProvider.Current, values, len);
return len;
}
}
// This reader only supports a single result set.
public bool NextResult() {
return false;
}
// Move to the next row.
public bool Read() {
return rowProvider.MoveNext();
}
// Don't bother implementing these in any meaningful way.
public long GetBytes(int i, long fieldOffset, byte[] buffer, int bufferoffset, int length) {
throw new NotImplementedException();
}
public long GetChars(int i, long fieldoffset, char[] buffer, int bufferoffset, int length) {
throw new NotImplementedException();
}
public IDataReader GetData(int i) {
throw new NotImplementedException();
}
public DataTable GetSchemaTable() {
return null;
}
}
Related
The following script is working fine. But need to add one more item in array without built-in function. Is it possible to do without Resize() ?
string[] data = {"item-1", "item-2"};
Array.Resize(ref data, 3);
data[2] = "item-3";
foreach(string i in data) {
Console.WriteLine(i);
}
No. Arrays are fixed size, so the only way to add more stuff to them is to resize them.
If you have a scenario where you need to add a dynamic number of elements, use List<T> instead.
Assuming you're trying to increase the size of an array without any built-in libraries or data structures, you just need to create a new larger array, copy the old elements, then add your new element.
string[] data = {"item-1", "item-2"};
string[] newData = new string[data.Length + 1];
int i;
for (i = 0; i < data.Length; i++) {
newData[i] = data[i];
}
newData[i] = "item-3";
Console.WriteLine(newData[2]);
The short answer is no. But if you want you can always implement a logic that does the resizing. You could do something like this implementation of a List:
public class List<T> : IAbstractList<T>
{
private const int DEFAULT_CAPACITY = 4;
private T[] _items;
public List()
: this(DEFAULT_CAPACITY) {
}
public List(int capacity)
{
if (capacity < 0)
{
throw new ArgumentOutOfRangeException(nameof(capacity));
}
this._items = new T[capacity];
}
public T this[int index]
{
get
{
this.ValidateIndex(index);
return this._items[index];
}
set
{
this.ValidateIndex(index);
this._items[index] = value;
}
}
public int Count { get; private set; }
public void Add(T item)
{
this.GrowIfNecessary();
this._items[this.Count++] = item;
}
public bool Contains(T item)
{
for (int i = 0; i < this.Count; i++)
{
if (this._items[i].Equals(item))
{
return true;
}
}
return false;
}
public int IndexOf(T item)
{
for (int i = 0; i < this.Count; i++)
{
if (this._items[i].Equals(item))
{
return i;
}
}
return -1;
}
public void Insert(int index, T item)
{
this.ValidateIndex(index);
this.GrowIfNecessary();
for (int i = this.Count - 1; i > index; i--)
{
this._items[i] = this._items[i - 1];
}
this._items[index] = item;
this.Count++;
}
public bool Remove(T item)
{
var index = this.IndexOf(item);
if (index == - 1)
{
return false;
}
this.RemoveAt(index);
return true;
}
public void RemoveAt(int index)
{
this.ValidateIndex(index);
for(int i = index; i < this.Count - 1; i++)
{
this._items[i] = this._items[i + 1];
}
this._items[this.Count - 1] = default;
this.Count--;
}
public IEnumerator<T> GetEnumerator()
{
for (int i = 0; i < this.Count; i++)
{
yield return this._items[i];
}
}
IEnumerator IEnumerable.GetEnumerator()
=> this.GetEnumerator();
private void ValidateIndex(int index)
{
if (index < 0 || index >= this.Count)
{
throw new IndexOutOfRangeException(nameof(index));
}
}
private void GrowIfNecessary()
{
if (this.Count == this._items.Length)
{
var array = new T[this.Count * 2];
Array.Copy(this._items, array, this._items.Length);
this._items = array;
}
}
}
The above sample code can give you ideas about implementing other methods that need array resizing as well.
I have uncommon question/request. I am learning about hash tables and have a hash table with open addressing when using the square hash function, and I'm trying to figure out how could I transform it to hash table with list? Is it possible to do from the code provided? Sorry if the question is too broad or unclear, trying to learn:
`
class HMap<K, V>
{
public static int DEFAULT_INITIAL_CAPACITY = 16;
public static float DEFAULT_LOAD_FACTOR = 0.75f;
Entry<K, V>[] table;
protected int size = 0;
protected float loadFactor;
protected int rehashesCounter = 0;
protected int index;
public HMap()
{
loadFactor = DEFAULT_LOAD_FACTOR;
this.table = new Entry<K, V>[DEFAULT_INITIAL_CAPACITY];
}
public HMap(int initialCapacity)
{
loadFactor = 0.75F;
this.table = new Entry<K, V>[initialCapacity];
}
public Boolean IsEmpty()
{
return size == 0;
}
public int Size()
{
return size;
}
public void Clear()
{
for(int i = 0; i < table.Length; i++)
{
table[i] = null;
}
size = 0;
rehashesCounter = 0;
}
public V Put(K key, V value)
{
if (key == null || value == null)
{
throw new Exception("Key or value is null in put(Key key, Value value)");
}
index = FindPosition(key);
if (index == -1)
{
return default(V);
}
table[index] = new Entry<K, V>(key, value);
size++;
if (size > table.Length * loadFactor)
{
Rehash(table[index]);
}
return value;
}
public V Get(K key)
{
if (key == null)
{
throw new Exception("Key is null in get(Key key)");
}
index = FindPosition(key);
return (table[index] != null) ? table[index].value : default(V);
}
public Boolean Contains(K key)
{
return Get(key) != null;
}
public Boolean ContainsValue(V value)
{
foreach (Entry<K, V> e in table)
{
if (e != null)
{
if (e.value.Equals(value))
{
return true;
}
}
}
return false;
}
private void Rehash(Entry<K, V> entry)
{
HMap<K, V> map = new HMap<K, V>(table.Length * 2);
for (int i = 0; i < table.Length; i++)
{
if (table[i] != null)
{
map.Put(table[i].key, table[i].value);
}
}
table = map.table;
rehashesCounter++;
}
private int Hash(K key)
{
int h = key.GetHashCode();
return Math.Abs(h) % table.Length;
}
private int FindPosition(K key)
{
index = Hash(key);
int indexO = index;
int i = 0;
for (int j = 0; j < table.Length; j++)
{
if (table[index] == null || table[index].key.Equals(key))
{
return index;
}
i++;
index = (indexO + i * Hash2(key)) % table.Length;
}
return -1;
}
private int Hash2(K key)
{
return 7 - (Math.Abs(key.GetHashCode()) % 7);
}
public override String ToString()
{
StringBuilder result = new StringBuilder();
foreach (Entry<K, V> entry in table)
{
if (entry == null)
{
result.Append("null").Append("\n");
}
else
{
result.Append(entry.toString()).Append("\n");
}
}
return result.ToString();
}
public void ToFile(string fileName)
{
using(StreamWriter file = new StreamWriter(#fileName))
{
foreach (Entry<K, V> entry in table)
{
if (entry == null)
{
file.WriteLine("null");
}
else
{
file.WriteLine(entry.toString());
}
}
}
}
class Entry<Key, Value>
{
public Key key;
public Value value;
public Entry(Key key, Value value)
{
this.key = key;
this.value = value;
}
public String toString()
{
return key + "=" + value;
}
}
}
`
I have a problem where my custom IComparer is not being called. I am thinking that maybe it might have to do with me expecting for it to work with different objects? Why is it not being called?
The Icomparer:
public class MyComparer : IComparer<object>
{
public string[] needles { get; set; }
public MyComparer(string[] argument)
{
needles = argument;
}
public int Compare(object x, object y)
{
int rankA = x.getRankedResults(needles);
int rankB = y.getRankedResults(needles);
if (rankA > rankB) { return 1; }
else if (rankA < rankB) { return -1; }
else { return 0; }
}
}
public static class MyComparerExtensions
{
public static int getRankedResults(this object o, string[] needle)
{
int result = 0;
if (o.GetType() == typeof(StaticPage))
{
var orig = o as StaticPage;
result = needle.countOccurences(orig.PageTitle.StripHTML(), orig.BodyCopy.StripHTML());
}
else if (o.GetType() == typeof(CustomerNewsArticle))
{
var orig = o as CustomerNewsArticle;
result = needle.countOccurences(orig.Title.StripHTML(), orig.BodyCopy.StripHTML());
}
else if (o.GetType() == typeof(Blog))
{
var orig = o as Blog;
result = needle.countOccurences(orig.Title.StripHTML(), orig.Body.StripHTML());
}
else if (o.GetType() == typeof(PressRelease))
{
var orig = o as PressRelease;
result = needle.countOccurences(orig.title.StripHTML(), orig.body.StripHTML());
}
return result;
}
//count the total occurences of the needles inside the haystack
private static int countOccurences(this string[] needle, params string[] haystack)
{
int occurences = 0;
foreach (var n in needle)
{
foreach (var h in haystack)
{
var nh = h;
int lenghtDif = nh.Length - nh.Replace(n, "").Length;
occurences = occurences + (lenghtDif / n.Length);
}
}
return occurences;
}
}
And how it is being called:
string[] needles = new string[] { "one", "two", "three" };
Haystacks.OrderBy(o=> o, new MyComparer(needles));
So "Haystacks" has 42 objects inside of different types (they are rows from different tables, and inside "getRankedResults" I detect the type of the object and choose the fields used to calculate the rank values). When i call the MyComparer on the sort, I can see the argument being set in the constructor, but then "Compare(object x, object y)" is never being called.
I know I can read write list of string like below using nhibernate
HasMany(x => x.Attachments)
.KeyColumn("RowId")
.Table("PostTable").Element("PostKey");
But this creates an extra table, is there a way e.g. UserType or something else, so that we can directly write to list... if yes any example of custom UserType using nhibernate?? with sample code...
I also want that If i add value to list that also should be saved. I have seen below example code which breaks in case we add value to list...
private virtual string StringValue
public virtual IList<string> valueList
{
get { return StringValue; }
set { StringValue = string.Join(value, "|"); }
}
You can do this with IUserType like so:
public class DelimitedList : IUserType
{
private const string delimiter = "|";
public new bool Equals(object x, object y)
{
return object.Equals(x, y);
}
public int GetHashCode(object x)
{
return x.GetHashCode();
}
public object NullSafeGet(IDataReader rs, string[] names, object owner)
{
var r = rs[names[0]];
return r == DBNull.Value
? new List<string>()
: ((string)r).SplitAndTrim(new [] { delimiter });
}
public void NullSafeSet(IDbCommand cmd, object value, int index)
{
object paramVal = DBNull.Value;
if (value != null)
{
paramVal = ((IEnumerable<string>)value).Join(delimiter);
}
var parameter = (IDataParameter)cmd.Parameters[index];
parameter.Value = paramVal;
}
public object DeepCopy(object value)
{
return value;
}
public object Replace(object original, object target, object owner)
{
return original;
}
public object Assemble(object cached, object owner)
{
return cached;
}
public object Disassemble(object value)
{
return value;
}
public SqlType[] SqlTypes
{
get { return new SqlType[] { new StringSqlType() }; }
}
public Type ReturnedType
{
get { return typeof(IList<string>); }
}
public bool IsMutable
{
get { return false; }
}
}
Then define the IList<string> property as type="MyApp.DelimitedList, MyApp".
NOTE: SplitAndTrim is a string extension with various overrides that I created. Here is the core method:
public static IList<string> SplitAndTrim(this string s, StringSplitOptions options, params string[] delimiters)
{
if (s == null)
{
return null;
}
var query = s.Split(delimiters, StringSplitOptions.None).Select(x => x.Trim());
if (options == StringSplitOptions.RemoveEmptyEntries)
{
query = query.Where(x => x.Trim() != string.Empty);
}
return query.ToList();
}
How can I get only values when I use a JsonConvert.SerializeObject? I don't need repeat words like id, name, etc...
Example:{ id: 189, name:'Paul', age:31, } x { [189, 'Paul', 31] }
Thanks!
I need to use with PageList class
public class PageList {
IEnumerable _rows;
int _total;
int _page;
int _records;
object _userData;
public PageList(IEnumerable rows, int page, int total, int records, object userData) {
_rows = rows;
_page = page;
_total = total;
_records = records;
_userData = userData;
}
public PageList(IEnumerable rows, int page, int total, int records)
: this(rows, page, total, records, null) {
}
public int total { get { return _total; } }
public int page { get { return _page; } }
public int records { get { return _records; } }
public IEnumerable rows { get { return _rows; } }
[JsonIgnore]
public object userData { get { return _userData; } }
public override string ToString() {
return Newtonsoft.Json.JsonConvert.SerializeObject(this, new IsoDateTimeConverter() { DateTimeFormat = "dd-MM-yyyy hh:mm:ss" });
}
}
The closest thing I can think of is
var yourObjectList = List<YourObject>(){.....}
string s = JsonConvert.SerializeObject(GetObjectArray(yourObjectList));
public static IEnumerable<object> GetObjectArray<T>(IEnumerable<T> obj)
{
return obj.Select(o => o.GetType().GetProperties().Select(p => p.GetValue(o, null)));
}
The second one is not valid JSON ( { 189, 'Paul', 31 } ). Maybe you want an array instead ([ 189, 'Paul', 31 ]), in which case you can instead of using the serializer directly, first load the object into a JObject, then take only its values.
public class Foo
{
public int id;
public string name;
public int age;
}
public class Test
{
public static void Main()
{
Foo foo = new Foo { id = 189, name = "Paul", age = 31 };
JObject jo = JObject.FromObject(foo);
JArray ja = new JArray();
foreach (var value in jo.Values())
{
ja.Add(value);
}
Console.WriteLine(ja);
}
}
Or if you really want the non-JSON format, you can also use the JObject enumeration and print the values yourself.