I'm working on a bit-based B/W/Greyscale Pre-Compiled font format, and was having issues with either reading or writing the format, (I've not been able to determine where the issue was. (I do have a B/W bit-based version working, but an Aliased font doesn't look too good, as you can imagine, especially when working with a 320x200 pixel screen) ) but decided that just using a BinaryWriter would be much easier than writing to a bool[] when I pulled the image data.
The basic format of a pixel in the file looks like this:
1 - White Pixel (Shortest, as this would be most of the pixels)
00 - Black Pixel (No reason to write 10-bits for a pure black pixel, which there are a reasonable number of)
01 - Greyscale Pixel, and is followed by 1 byte describing the shade of the pixel
Now, everything is fine and dandy with writing the required info, as that's all full bytes, but the default .Net 4.0 BinaryWriter writes a Boolean value as a full byte, and as you can imagine, that negates the use of a bit-based format. So I was wondering, is there a BinaryWriter, (and BinaryReader) implementation out there that's bit-based
Edit:
I ended up creating my own. (See the answer for the code.)
I ended up writing my own, so here they are.
The BinaryWriter (I've only overridden the ones that I needed)
private class BinaryWriter : System.IO.BinaryWriter
{
private bool[] curByte = new bool[8];
private byte curBitIndx = 0;
private System.Collections.BitArray ba;
public BinaryWriter(Stream s) : base(s) { }
public override void Flush()
{
base.Write(ConvertToByte(curByte));
base.Flush();
}
public override void Write(bool value)
{
curByte[curBitIndx] = value;
curBitIndx++;
if (curBitIndx == 8)
{
base.Write(ConvertToByte(curByte));
this.curBitIndx = 0;
this.curByte = new bool[8];
}
}
public override void Write(byte value)
{
ba = new BitArray(new byte[] { value });
for (byte i = 0; i < 8; i++)
{
this.Write(ba[i]);
}
ba = null;
}
public override void Write(byte[] buffer)
{
for (int i = 0; i < buffer.Length; i++)
{
this.Write((byte)buffer[i]);
}
}
public override void Write(uint value)
{
ba = new BitArray(BitConverter.GetBytes(value));
for (byte i = 0; i < 32; i++)
{
this.Write(ba[i]);
}
ba = null;
}
public override void Write(ulong value)
{
ba = new BitArray(BitConverter.GetBytes(value));
for (byte i = 0; i < 64; i++)
{
this.Write(ba[i]);
}
ba = null;
}
public override void Write(ushort value)
{
ba = new BitArray(BitConverter.GetBytes(value));
for (byte i = 0; i < 16; i++)
{
this.Write(ba[i]);
}
ba = null;
}
private static byte ConvertToByte(bool[] bools)
{
byte b = 0;
byte bitIndex = 0;
for (int i = 0; i < 8; i++)
{
if (bools[i])
{
b |= (byte)(((byte)1) << bitIndex);
}
bitIndex++;
}
return b;
}
}
And, the BinaryReader, once again, I've only overridden the methods that I needed.
private class BinaryReader : System.IO.BinaryReader
{
private bool[] curByte = new bool[8];
private byte curBitIndx = 0;
private BitArray ba;
public BinaryReader(Stream s) : base(s)
{
ba = new BitArray(new byte[] { base.ReadByte() });
ba.CopyTo(curByte, 0);
ba = null;
}
public override bool ReadBoolean()
{
if (curBitIndx == 8)
{
ba = new BitArray(new byte[] { base.ReadByte() });
ba.CopyTo(curByte, 0);
ba = null;
this.curBitIndx = 0;
}
bool b = curByte[curBitIndx];
curBitIndx++;
return b;
}
public override byte ReadByte()
{
bool[] bar = new bool[8];
byte i;
for (i = 0; i < 8; i++)
{
bar[i] = this.ReadBoolean();
}
byte b = 0;
byte bitIndex = 0;
for (i = 0; i < 8; i++)
{
if (bar[i])
{
b |= (byte)(((byte)1) << bitIndex);
}
bitIndex++;
}
return b;
}
public override byte[] ReadBytes(int count)
{
byte[] bytes = new byte[count];
for (int i = 0; i < count; i++)
{
bytes[i] = this.ReadByte();
}
return bytes;
}
public override ushort ReadUInt16()
{
byte[] bytes = ReadBytes(2);
return BitConverter.ToUInt16(bytes, 0);
}
public override uint ReadUInt32()
{
byte[] bytes = ReadBytes(4);
return BitConverter.ToUInt32(bytes, 0);
}
public override ulong ReadUInt64()
{
byte[] bytes = ReadBytes(8);
return BitConverter.ToUInt64(bytes, 0);
}
}
I don't believe there's anything in the framework for this, no. Basically you'd need to write a class to wrap a BinaryWriter (or just a stream) and "the byte written so far" and the number of bits written. When the number of bits gets to 8, write the byte to the underlying stream and clear.
EDIT: the OP posted a possible and working implementation of the above suggestion below.
If you keep your data in a byte array (bools are of no use and take too much space, if you use them for bits, they take up a byte in memory) or in an array of a particular struct that fits your dataformat.
Once you have an internal memory representation, you don't need a bit-based binary writer anymore. You can simply write the data to a BinaryWriter and you're done with it.
...but the default .Net 4.0 BinaryWriter writes a Boolean value as a full
byte, and as you can imagine, that negates the use of a bit-based
format....
The reason for this is: the bool is, by definition, of 1 byte size in C#. The BinaryWriter simply writes what you give it.
I found myself in need of this as well, so I built upon OP and filled in all the read/writes (except char & string since those are a bit special).
I also made a quick unit test try it out. For streams containing only boolean (or other custom sub-byte value types) it's obviously 87.5% cheaper, and for a random mixed stream containing 75% boolean values, it was about 33% cheaper. So could be useful for some scenarios.
Here are the both classes in case anyone else needs them, use at your own risk:
/// <summary>
/// A binary writer that packs data into bits, to preserve space when using many bit/boolean values. Up to about 87.5% cheaper for streams that only contains boolean values.
/// By: jsmars#gmail.com, based on posters classes in this post: https://stackoverflow.com/questions/7051939/bit-based-binarywriter-in-c-sharp
/// </summary>
public class BinaryBitWriter : BinaryWriter
{
public byte BitPosition { get; private set; } = 0;
private bool[] curByte = new bool[8];
private System.Collections.BitArray ba;
public BinaryBitWriter(Stream s) : base(s) { }
public override void Flush()
{
flushBitBuffer();
base.Flush();
}
public override void Write(byte[] buffer, int index, int count)
{
for (int i = index; i < index + count; i++)
Write((byte)buffer[i]);
}
public override void Write(byte value)
{
ba = new BitArray(new byte[] { value });
for (byte i = 0; i < 8; i++)
Write(ba[i]);
}
public override void Write(bool value)
{
curByte[BitPosition] = value;
BitPosition++;
if (BitPosition == 8)
flushBitBuffer();
}
public override void Write(char[] chars, int index, int count)
{
for (int i = index; i < index + count; i++)
Write(chars[i]);
}
public override void Write(string value)
{
// write strings as normal for now, so flush the bits first
flushBitBuffer();
base.Write(value);
}
public override void Write(decimal value)
{
var ints = decimal.GetBits(value);
for (int i = 0; i < ints.Length; i++)
Write(ints[i]);
}
public override void Write(float value) => Write(BitConverter.GetBytes(value));
public override void Write(ulong value) => Write(BitConverter.GetBytes(value));
public override void Write(long value) => Write(BitConverter.GetBytes(value));
public override void Write(uint value) => Write(BitConverter.GetBytes(value));
public override void Write(int value) => Write(BitConverter.GetBytes(value));
public override void Write(ushort value) => Write(BitConverter.GetBytes(value));
public override void Write(short value) => Write(BitConverter.GetBytes(value));
public override void Write(double value) => Write(BitConverter.GetBytes(value));
public override void Write(char[] value) => Write(value, 0, value.Length);
public override void Write(char value)
{
// write strings as normal for now, so flush the bits first
flushBitBuffer();
base.Write(value);
//var b = BitConverter.GetBytes(value);
//Write(b);
}
public override void Write(byte[] buffer) => Write(buffer, 0, buffer.Length);
public override void Write(sbyte value) => Write((byte)value);
void flushBitBuffer()
{
if (BitPosition == 0) // Nothing to flush
return;
base.Write(ConvertToByte(curByte));
BitPosition = 0;
curByte = new bool[8];
}
private static byte ConvertToByte(bool[] bools)
{
byte b = 0;
byte bitIndex = 0;
for (int i = 0; i < 8; i++)
{
if (bools[i])
b |= (byte)(((byte)1) << bitIndex);
bitIndex++;
}
return b;
}
}
public class BinaryBitReader : BinaryReader
{
public byte BitPosition { get; private set; } = 8;
private bool[] curByte = new bool[8];
public BinaryBitReader(Stream s) : base(s)
{
}
public override bool ReadBoolean()
{
if (BitPosition == 8)
{
var ba = new BitArray(new byte[] { base.ReadByte() });
ba.CopyTo(curByte, 0);
BitPosition = 0;
}
bool b = curByte[BitPosition];
BitPosition++;
return b;
}
public override byte ReadByte()
{
bool[] bar = new bool[8];
byte i;
for (i = 0; i < 8; i++)
{
bar[i] = this.ReadBoolean();
}
byte b = 0;
byte bitIndex = 0;
for (i = 0; i < 8; i++)
{
if (bar[i])
{
b |= (byte)(((byte)1) << bitIndex);
}
bitIndex++;
}
return b;
}
public override byte[] ReadBytes(int count)
{
byte[] bytes = new byte[count];
for (int i = 0; i < count; i++)
{
bytes[i] = this.ReadByte();
}
return bytes;
}
//public override int Read() => BitConverter.ToUInt64(ReadBytes(8), 0);
public override int Read(byte[] buffer, int index, int count)
{
for (int i = index; i < index + count; i++)
buffer[i] = ReadByte();
return count; // we can return this here, it will die at the above row if anything is off
}
public override int Read(char[] buffer, int index, int count)
{
for (int i = index; i < index + count; i++)
buffer[i] = ReadChar();
return count; // we can return this here, it will die at the above row if anything is off
}
public override char ReadChar()
{
BitPosition = 8;
return base.ReadChar();
//BitConverter.ToChar(ReadBytes(2), 0);
}
public override char[] ReadChars(int count)
{
var chars = new char[count];
Read(chars, 0, count);
return chars;
}
public override decimal ReadDecimal()
{
int[] ints = new int[4];
for (int i = 0; i < ints.Length; i++)
ints[i] = ReadInt32();
return new decimal(ints);
}
public override double ReadDouble() => BitConverter.ToDouble(ReadBytes(8), 0);
public override short ReadInt16() => BitConverter.ToInt16(ReadBytes(2), 0);
public override int ReadInt32() => BitConverter.ToInt32(ReadBytes(4), 0);
public override long ReadInt64() => BitConverter.ToInt64(ReadBytes(8), 0);
public override sbyte ReadSByte() => (sbyte)ReadByte();
public override float ReadSingle() => BitConverter.ToSingle(ReadBytes(4), 0);
public override string ReadString()
{
BitPosition = 8; // Make sure we read a new byte when we start reading the string
return base.ReadString();
}
public override ushort ReadUInt16() => BitConverter.ToUInt16(ReadBytes(2), 0);
public override uint ReadUInt32() => BitConverter.ToUInt32(ReadBytes(4), 0);
public override ulong ReadUInt64() => BitConverter.ToUInt64(ReadBytes(8), 0);
}
And the unit tests:
public static bool UnitTest()
{
const int testPairs = 512;
var bitstream = new MemoryStream();
var bitwriter = new BinaryBitWriter(bitstream);
var bitreader = new BinaryBitReader(bitstream);
byte[] bytes = new byte[] { 1, 2, 3, 4, 255 };
byte Byte = 128;
bool Bool = true;
char[] chars = new char[] { 'a', 'b', 'c' };
string str = "hello";
var Float = 2.5f;
ulong Ulong = 12345678901234567890;
long Long = 1122334455667788;
uint Uint = 1234567890;
int Int = 999998888;
ushort UShort = 12345;
short Short = 4321;
double Double = 9.9;
char Char = 'A';
sbyte Sbyte = -128;
decimal Decimal = 10000.00001m;
List<BBTest> pairs = new List<BBTest>();
// Make pairs of write and read tests
pairs.Add(new BBTest(Bool, (w) => w.Write(Bool), (r) => { if (r.ReadBoolean() != Bool) throw new Exception(); }));
pairs.Add(new BBTest(bytes, (w) => w.Write(bytes, 0, 5), (r) => { if (arrayCompare(r.ReadBytes(5), bytes)) throw new Exception(); }));
pairs.Add(new BBTest(Byte, (w) => w.Write(Byte), (r) => { if (r.ReadByte() != Byte) throw new Exception(); }));
pairs.Add(new BBTest(chars, (w) => w.Write(chars, 0, 3), (r) => { if (arrayCompare(r.ReadChars(3), chars)) throw new Exception(); })); /////////////
pairs.Add(new BBTest(str, (w) => w.Write(str), (r) => { string s; if ((s = r.ReadString()) != str) throw new Exception(); }));
pairs.Add(new BBTest(Decimal, (w) => w.Write(Decimal), (r) => { if (r.ReadDecimal() != Decimal) throw new Exception(); }));
pairs.Add(new BBTest(Float, (w) => w.Write(Float), (r) => { if (r.ReadSingle() != Float) throw new Exception(); }));
pairs.Add(new BBTest(Ulong, (w) => w.Write(Ulong), (r) => { if (r.ReadUInt64() != Ulong) throw new Exception(); }));
pairs.Add(new BBTest(Long, (w) => w.Write(Long), (r) => { if (r.ReadInt64() != Long) throw new Exception(); }));
pairs.Add(new BBTest(Uint, (w) => w.Write(Uint), (r) => { if (r.ReadUInt32() != Uint) throw new Exception(); }));
pairs.Add(new BBTest(Int, (w) => w.Write(Int), (r) => { if (r.ReadInt32() != Int) throw new Exception(); }));
pairs.Add(new BBTest(UShort, (w) => w.Write(UShort), (r) => { if (r.ReadUInt16() != UShort) throw new Exception(); }));
pairs.Add(new BBTest(Short, (w) => w.Write(Short), (r) => { if (r.ReadInt16() != Short) throw new Exception(); }));
pairs.Add(new BBTest(Double, (w) => w.Write(Double), (r) => { if (r.ReadDouble() != Double) throw new Exception(); }));
pairs.Add(new BBTest(Char, (w) => w.Write(Char), (r) => { if (r.ReadChar() != Char) throw new Exception(); })); ///////////////
pairs.Add(new BBTest(bytes, (w) => w.Write(bytes), (r) => { if (arrayCompare(r.ReadBytes(5), bytes)) throw new Exception(); }));
pairs.Add(new BBTest(Sbyte, (w) => w.Write(Sbyte), (r) => { if (r.ReadSByte() != Sbyte) throw new Exception(); }));
// Now add all tests, and then a bunch of randomized tests, to make sure we test lots of combinations incase there is some offsetting error
List<BBTest> test = new List<BBTest>();
test.AddRange(pairs);
var rnd = new Random();
for (int i = 0; i < testPairs - test.Count; i++)
{
if (rnd.NextDouble() < 0.75)
test.Add(pairs[0]);
else
test.Add(pairs[rnd.Next(pairs.Count)]);
}
// now write all the tests
for (int i = 0; i < test.Count; i++)
test[i].Writer(bitwriter);
bitwriter.Flush();
// now reset the stream and test to see that they are the same
bitstream.Position = 0;
for (int i = 0; i < test.Count; i++)
test[i].ReadTest(bitreader);
// As comparison, lets write the same stuff to a normal binarywriter and compare sized
var binstream = new MemoryStream();
var binwriter = new BinaryWriter(binstream);
for (int i = 0; i < test.Count; i++)
test[i].Writer(binwriter);
binwriter.Flush();
var saved = 1 - bitstream.Length / (float)binstream.Length;
var result = $"BinaryBitWriter was {(saved * 100).ToString("0.00")}% cheaper than a normal BinaryWriter with random data";
bool arrayCompare(IEnumerable a, IEnumerable b)
{
var B = b.GetEnumerator();
B.MoveNext();
foreach (var item in a)
{
if (item != B.Current)
return false;
B.MoveNext();
}
return true;
}
return true;
}
delegate void writer(BinaryWriter w);
delegate void reader(BinaryReader r);
class BBTest
{
public object Object;
public writer Writer;
public reader ReadTest;
public BBTest(object obj, writer w, reader r) { Object = obj; Writer = w; ReadTest = r; }
public override string ToString() => Object.ToString();
}
Related
I have a class that internally is just an array of integers. Once constructed the array never changes. I'd like to pre-compute a good hashcode so that this class can be very efficiently used as a key in a Dictionary. The length of the array is less than about 30 items, and the integers are between -1000 and 1000 in general.
Not very clever, but sufficient for most practical purposes:
EDIT: changed due to comment of Henk Holterman, thanks for that.
int hc = array.Length;
foreach (int val in array)
{
hc = unchecked(hc * 314159 + val);
}
If you need something more sophisticated, look here.
For an array of values generally between -1000 and 1000, I would probably use something like this:
static int GetHashCode(int[] values)
{
int result = 0;
int shift = 0;
for (int i = 0; i < values.Length; i++)
{
shift = (shift + 11) % 21;
result ^= (values[i]+1024) << shift;
}
return result;
}
You may use CRC32 checksum. Here is the code:
[CLSCompliant(false)]
public class Crc32 {
uint[] table = new uint[256];
uint[] Table { get { return table; } }
public Crc32() {
MakeCrcTable();
}
void MakeCrcTable() {
for (uint n = 0; n < 256; n++) {
uint value = n;
for (int i = 0; i < 8; i++) {
if ((value & 1) != 0)
value = 0xedb88320 ^ (value >> 1);
else
value = value >> 1;
}
Table[n] = value;
}
}
public uint UpdateCrc(uint crc, byte[] buffer, int length) {
uint result = crc;
for (int n = 0; n < length; n++) {
result = Table[(result ^ buffer[n]) & 0xff] ^ (result >> 8);
}
return result;
}
public uint Calculate(Stream stream) {
long pos = stream.Position;
const int size = 0x32000;
byte[] buf = new byte[size];
int bytes = 0;
uint result = 0xffffffff;
do {
bytes = stream.Read(buf, 0, size);
result = UpdateCrc(result, buf, bytes);
}
while (bytes == size);
stream.Position = pos;
return ~result;
}
}
I think choosing a good hash-algorithm would have to be based on the distribution (in a probability sense) of the integer values.
Have a look at Wikipedia for a list of algorithms
Any CRC (or even XOR) should be ok.
You could take a different approach and use a recursive dictionary for each value in your int array. This way you can leave .net to do primitive type hashing.
internal class DictionaryEntry<TKey, TValue>
{
public Dictionary<TKey, DictionaryEntry<TKey, TValue>> Children { get; private set; }
public TValue Value { get; private set; }
public bool HasValue { get; private set; }
public void SetValue(TValue value)
{
Value = value;
HasValue = true;
}
public DictionaryEntry()
{
Children = new Dictionary<TKey, DictionaryEntry<TKey, TValue>>();
}
}
internal class KeyStackDictionary<TKey, TValue>
{
// Helper dictionary to work with a stack of keys
// Usage:
// var dict = new KeyStackDictionary<int, string>();
// int[] keyStack = new int[] {23, 43, 54};
// dict.SetValue(keyStack, "foo");
// string value;
// if (dict.GetValue(keyStack, out value))
// {
// }
private DictionaryEntry<TKey, TValue> _dict;
public KeyStackDictionary()
{
_dict = new DictionaryEntry<TKey, TValue>();
}
public void SetValue(TKey[] keyStack, TValue value)
{
DictionaryEntry<TKey, TValue> dict = _dict;
for (int i = 0; i < keyStack.Length; i++)
{
TKey key = keyStack[i];
if (dict.Children.ContainsKey(key))
{
dict = dict.Children[key];
}
else
{
var child = new DictionaryEntry<TKey, TValue>();
dict.Children.Add(key, child);
dict = child;
}
if (i == keyStack.Length - 1)
{
dict.SetValue(value);
}
}
}
// returns false if the value is not found using the key stack
public bool GetValue(TKey[] keyStack, out TValue value)
{
DictionaryEntry<TKey, TValue> dict = _dict;
for (int i = 0; i < keyStack.Length; i++)
{
TKey key = keyStack[i];
if (dict.Children.ContainsKey(key))
{
dict = dict.Children[key];
}
else
{
break;
}
if (i == keyStack.Length - 1 && dict.HasValue)
{
value = dict.Value;
return true;
}
}
value = default(TValue);
return false;
}
}
You can use Linq methods too:
var array = new int[10];
var hashCode = array.Aggregate(0, (a, v) =>
HashCode.Combine(a, v.GetHashCode()));
I'm using this here
var arrayHash = string.Join(string.Empty, array).GetHashCode();
If a element changed in the array, you will get a new hash.
I would recommend:
HashCode.Combine(array)
For .NET Core 2.1 / .NET Standard 2.1 / .NET 5 and later.
I have a socket.io client which sends data to each other where encryption is based on ARC4.
I tried multiple different scenarios but it keeps failing to decrypt anything and I'm not sure why.
The class: ARC4_New
public class ARC4_New
{
private int i;
private int j;
private byte[] bytes;
public const int POOLSIZE = 256;
public ARC4_New()
{
bytes = new byte[POOLSIZE];
}
public ARC4_New(byte[] key)
{
bytes = new byte[POOLSIZE];
this.Initialize(key);
}
public void Initialize(byte[] key)
{
this.i = 0;
this.j = 0;
for (i = 0; i < POOLSIZE; ++i)
{
this.bytes[i] = (byte)i;
}
for (i = 0; i < POOLSIZE; ++i)
{
j = (j + bytes[i] + key[i % key.Length]) & (POOLSIZE - 1);
this.Swap(i, j);
}
this.i = 0;
this.j = 0;
}
private void Swap(int a, int b)
{
byte t = this.bytes[a];
this.bytes[a] = this.bytes[b];
this.bytes[b] = t;
}
public byte Next()
{
this.i = ++this.i & (POOLSIZE - 1);
this.j = (this.j + this.bytes[i]) & (POOLSIZE - 1);
this.Swap(i, j);
return this.bytes[(this.bytes[i] + this.bytes[j]) & 255];
}
public void Encrypt(ref byte[] src)
{
for (int k = 0; k < src.Length; k++)
{
src[k] ^= this.Next();
}
}
public void Decrypt(ref byte[] src)
{
this.Encrypt(ref src);
}
}
public System.Numerics.BigInteger RandomInteger(int bitSize)
{
var integerData = new byte[bitSize / 8];
_numberGenerator.NextBytes(integerData);
integerData[integerData.Length - 1] &= 0x7f;
return new System.Numerics.BigInteger(integerData);
}
My script which generates a key:
System.Numerics.BigInteger DHPrivate = RandomInteger(256);
System.Numerics.BigInteger DHPrimal = RandomInteger(256);
System.Numerics.BigInteger DHGenerated = RandomInteger(256);
if (DHGenerated > DHPrimal)
{
System.Numerics.BigInteger tempG = DHGenerated;
DHGenerated= DHPrimal;
DHPrimal = tempG;
}
Then with those values I generate a public key:
System.Numerics.BigInteger DHPublic = System.Numerics.BigInteger.ModPow(DHGenerated, DHPrivate, DHPrimal);
Then I encrypt this key:
string pkey = EncryptY(CalculatePublic, DHPublic);
(Additional code for the encryption below)
protected virtual string EncryptY(Func<System.Numerics.BigInteger, System.Numerics.BigInteger> calculator, System.Numerics.BigInteger value)
{
byte[] valueData = Encoding.UTF8.GetBytes(value.ToString());
valueData = PKCSPad(valueData);
Array.Reverse(valueData);
var paddedInteger = new System.Numerics.BigInteger(valueData);
System.Numerics.BigInteger calculatedInteger = calculator(paddedInteger);
byte[] paddedData = calculatedInteger.ToByteArray();
Array.Reverse(paddedData);
string encryptedValue = Utils.Converter.BytesToHexString(paddedData).ToLower();
return encryptedValue.StartsWith("00") ? encryptedValue.Substring(2) : encryptedValue;
}
protected virtual byte[] PKCSPad(byte[] data)
{
var buffer = new byte[128 - 1];
int dataStartPos = (buffer.Length - data.Length);
buffer[0] = (byte)Padding;
Buffer.BlockCopy(data, 0, buffer, dataStartPos, data.Length);
int paddingEndPos = (dataStartPos - 1);
bool isRandom = (Padding == PKCSPadding.RandomByte);
for (int i = 1; i < paddingEndPos; i++)
{
buffer[i] = (byte)(isRandom ?
_numberGenerator.Next(1, 256) : byte.MaxValue);
}
return buffer;
}
After all that I sent the string PKEY to the server.
And after decrypting the string, the server gets the public key which is for example: 127458393
When I connect both my client and server using: 127458393
Like:
BigInteger key = System.Numerics.BigInteger.Parse("127458393");
client = new ARC4_New(PrimalDing.ToByteArray());
My client sends a string like:
client.Encrypt(BYTE_HERE);
And my server reads it like:
client.Decrypt(BYTE_HERE);
But it fails, and gets a random unreadable string.
What am I doing wrong here?
I managed to fix the issue
For some reason, my server was and is reversing the bytes i used in the ARC4 client..
So i simple reverse it now as a hotfix
System.Numerics.BigInteger temp = System.Numerics.BigInteger.Parse(textBox1.Text);
client = new ARC4_New(temp.ToByteArray().Reverse().ToArray());
I want to know a fastest way to count all byte in file ? I need to work on large binary file
I want to know the quantity of all byte in file (Quantity of 0x00, 0x01, .. 0xff)
It's for add a graph with file representation in my WPF Hexeditor usercontrol https://github.com/abbaye/WPFHexEditorControl like in HxD hexeditor.
This code work fine but it's to slow for large file.
public Dictionary<int, long> GetByteCount()
{
if (IsOpen)
{
Position = 0;
int currentByte = 0;
// Build dictionary
Dictionary<int, long> cd = new Dictionary<int, long>();
for (int i = 0; i <= 255; i++) cd.Add(i, 0);
//
for (int i = 0; i <= Length; i++)
{
//if (EOF) break;
currentByte = ReadByte();
if (currentByte != -1) cd[currentByte]++;
Position++;
}
return cd;
}
return new Dictionary<int, long>();
}
/// <summary>
/// Get an array of long computing the total of each byte in the file.
/// The position of the array makes it possible to obtain the sum of the desired byte
/// </summary>
public long[] GetByteCount()
{
if (IsOpen)
{
const int bufferLenght = 1048576; //1mb
var storedCnt = new long[256];
Position = 0;
while (!Eof)
{
var testLenght = Length - Position;
var buffer = testLenght <= bufferLenght ? new byte[testLenght] : new byte[bufferLenght];
Read(buffer, 0, buffer.Length);
foreach (var b in buffer)
storedCnt[b]++;
Position += bufferLenght;
}
return storedCnt;
}
return null;
}
I have optimized David's solution a bit. The "Position" - calls are not necessary. I've found that the buffer length and the unbuffered read mode are not very important, but the "for"- instead of "foreach" - construct in the calculation made a big difference.
Results with
foreach (var b in buffer.Take(count))
{
storedCnt[b]++;
}
file length is 4110217216
duration 00:00:51.1686821
Results with
for(var i = 0; i < count; i++)
{
storedCnt[buffer[i]]++;
}
file length 4110217216
duration 00:00:05.9695418
Here the program
private static void Main(
{
const string fileForCheck = #"D:\Data\System\en_visual_studio_enterprise_2015_x86_x64_dvd_6850497.iso";
Debug.Assert(File.Exists(fileForCheck));
var watch = new Stopwatch();
var counter = new FileBytesCounter(fileForCheck);
watch.Start();
var results = counter.GetByteCount();
watch.Stop();
counter.Dispose();
Console.WriteLine("results:");
Console.WriteLine(string.Join(", ", results.Select((c, b) => $"{b} -> {c}")));
var sumBytes = results.Sum(c => c);
Debug.Assert((new FileInfo(fileForCheck)).Length == sumBytes); // here's the proof
Console.WriteLine();
Console.WriteLine($"file length {sumBytes}");
Console.WriteLine($"duration {watch.Elapsed}");
}
and here the class
internal class FileBytesCounter
: FileStream
{
private const FileOptions FileFlagNoBuffering = (FileOptions)0x20000000;
private const int CopyBufferSize = 1024 * 1024;
//private const int CopyBufferSize = 4 * 1024 * 16;
public FileBytesCounter(string path, FileShare share = FileShare.Read)
: base(path, FileMode.Open, FileAccess.Read, share, CopyBufferSize/*, FileFlagNoBuffering*/)
{
}
public long[] GetByteCount()
{
var buffer = new byte[CopyBufferSize];
var storedCnt = new long[256];
int count;
Position = 0;
while ((count = Read(buffer, 0, CopyBufferSize)) > 0)
{
for(var i = 0; i < count; i++)
{
storedCnt[buffer[i]]++;
}
}
return storedCnt;
}
}
See also https://www.codeproject.com/Articles/172613/Fast-File-Copy-With-Managed-Code-UBCopy-update for FileFlagNoBuffering
It seems like you want something like this:
public Dictionary<char, long> GetCharCount(string filePath)
{
var result = new Dictionary<char, long>();
var content = File.ReadAllText(filePath);
foreach(var c in content)
{
if (result.ContainsKey(c))
{
result[c] = result[c] + 1;
}
else
{
result.Add(c, 1);
}
}
return result;
}
Was wondering how can I convert an int to a List in reverse order padded with zeroes and vice versa?
Have a byte that represents List(8), sometimes 2 bytes for List(16), 8 bytes for List(64); so looking for a good solution to handle converting to an int list, manipulate then back again.
e.g. Input of 3 to a List of 1,1,0,0,0,0,0,0
Or input of 42 to a List of 0,1,0,1,0,1,0,0
And vice-versa, take a List of 1,1,0,0,0,0,0,0 and return 3 or List of 0,1,0,1,0,1,0,0 and return 42
What I have done at present is build a couple of functions to handle both scenarios, all works fine, just wondering if there is a better / more elegant solution that I've completelt overlooked?
private List<int> IntToList(int _Input)
{
string _Binary = ReverseString(Convert.ToString(_Input, 2).PadLeft(8, '0'));
List<int> _List = new List<int>(8);
for (int i = 0; i < _Binary.Length; i++)
{
_List.Add(Convert.ToInt32(_Binary.Substring(i, 1)));
}
return _List;
}
private int IntsToByte(List<int> _List)
{
string _Binary = "";
for (int i = 7; i > -1; i--)
{
_Binary += _List[i];
}
return Convert.ToInt32(_Binary, 2);
}
You can work with bitwise operations. They might be fast.
Warning : Be aware of Little/Big Endian (More here)
The following code works :
private List<int> IntToList(int _Input, int _MaxSize = 8)
{
int padding = 1;
List<int> resultList = new List<int>(_MaxSize);
while (padding < 1 << _MaxSize)
{
resultList.Add((_Input & padding) == padding ? 1 : 0);
padding = padding << 1;
}
return resultList;
}
private int IntsToByte(List<int> _List)
{
int result = 0, padding = 0;
foreach (int i in _List)
{
result = result | (i << padding++);
}
return result;
}
This should work
int number = 42
char[] reverse = Convert.ToString(number, 2).PadLeft(8, '0').ToCharArray();
Array.Reverse(reverse);
Try this
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.IO;
namespace ConsoleApplication1
{
class Program
{
static void Main(string[] args)
{
List<ulong> results = null;
List<byte> output = null;
List<byte> input1 = new List<byte>() { 1, 1, 0, 0, 0, 0, 0, 0 };
results = ReadList(input1, 1);
output = WriteList(results,1);
List<byte> input2 = new List<byte>() { 0, 1, 0, 1, 0, 1, 0, 0 };
results = ReadList(input2, 1);
output = WriteList(results,1);
}
static List<ulong> ReadList(List<byte> input, int size)
{
List<ulong> results = new List<ulong>();
input.Reverse();
MemoryStream stream = new MemoryStream(input.ToArray());
BinaryReader reader = new BinaryReader(stream);
int count = 0;
ulong newValue = 0;
while (reader.PeekChar() != -1)
{
switch (size)
{
case 1:
newValue = ((ulong)Math.Pow(2, size) * newValue) + (ulong)reader.ReadByte();
break;
case 2:
newValue = ((ulong)Math.Pow(2, size) * newValue) + (ulong)reader.ReadInt16();
break;
}
if (++count == size)
{
results.Add(newValue);
newValue = 0;
count = 0;
}
}
return results;
}
static List<byte> WriteList(List<ulong> input, int size)
{
List<byte> results = new List<byte>();
foreach (ulong num in input)
{
ulong result = num;
for (int count = 0; count < size; count++)
{
if (result > 0)
{
byte bit = (byte)(result % Math.Pow(2, size));
results.Add(bit);
result = (ulong)(result / Math.Pow(2, size));
}
else
{
results.Add(0);
}
}
}
results.Reverse();
return results;
}
}
}
Solution from OP.
Have gone with Jean Bob's suggestion of using BitWise.
For anyone elses benefit, here is my modified version to read / write in blocks of 8 to/from the list.
private List<int> IntToList(List<int> _List, int _Input)
{
int _Padding = 1;
while (_Padding < 1 << 8)
{
_List.Add((_Input & _Padding) == _Padding ? 1 : 0);
_Padding = _Padding << 1;
}
return _List;
}
private int IntsToByte(List<int> _List, int l)
{
int _Result = 0, _Padding = 0;
for (int i = l; i < (l + 8); i++)
{
_Result = _Result | (_List[i] << _Padding++);
}
return _Result;
}
My requirements are simple:
Be able to define dimensions of bit array, i.e: 5 bytes.
bool Get(bitIndex: int)
Set(bitIndex: int)
Is there a c# equivalent which provides similar functionality to BitSet in Java?
Here's the scenario:
Initialize 5 bytes, all bits are 0(false).
Set byte 3, bit 8 to TRUE.
Get status of byte 3, bit 8.
UPDATE: Solution from Michael Bray:
static void Main(string[] args)
{
// Set for 5 bytes
BitArray ba = new BitArray(8 * 5);
// Set bit #1 on byte #4
ba.Set(GetBitNum(4, 1), true);
// Get bit #1 on byte #4
bool v = ba.Get(GetBitNum(4, 1));
}
static int GetBitNum(int byteNum, int bitNum) // Assumes index starts at 1
{
return (byteNum - 1) * 8 + (bitNum - 1);
}
System.Collections.BitArray is pretty close, but it's a bit lacking in features. I wrote a helper class that implements a lot of features you might need for BitArray some time ago, but I'd have to dig it up. Let me know if you think you need it.
EDIT: As requested in the comments below, I've posted the code at http://pastebin.com/GLyzcUZC. As I discuss, though, it's benefits over stock BitArray are minimal, as I wrote it for some specific needs that I had. Do with it as you wish.
EDIT 2: As Miguel pointed out in the comments, there are some implementation issues that make my 'BitArray' code not so good... I had already realized the deficiencies and had re-written a new version called BoolArray (to distinguish from BitArray) that doesn't suffer from those problems :
namespace Utils
{
using System;
using System.Collections;
using System.Collections.Generic;
using System.ComponentModel;
using System.Globalization;
using System.Linq;
using System.Threading;
/// <summary>
/// A replacement for BitArray
/// </summary>
public class BoolArray : IEnumerable, ICollection, ICloneable
{
private UInt32[] bits = null;
private int _length = 0;
private static UInt32 ONE = (UInt32)1 << 31;
private object _syncRoot;
private static Func<byte[], byte[]> EndianFixer = null;
#region Constructors
static BoolArray()
{
if (BitConverter.IsLittleEndian) EndianFixer = (a) => a.Reverse().ToArray();
else EndianFixer = (a) => a;
}
public BoolArray(BoolArray srcBits)
{
this.InitializeFrom(srcBits.ToArray());
}
public BoolArray(BitArray srcBits)
{
this._length = srcBits.Count;
this.bits = new UInt32[RequiredSize(this._length)];
for (int i = 0; i < srcBits.Count; i++) this[i] = srcBits[i];
}
public BoolArray(int v)
{
ICollection<byte> bytes = EndianFixer(BitConverter.GetBytes(v)).ToList();
InitializeFrom(bytes);
}
public BoolArray(ICollection<bool> srcBits)
{
this.InitializeFrom(srcBits.ToArray());
}
public BoolArray(ICollection<byte> srcBits)
{
InitializeFrom(srcBits);
}
public BoolArray(ICollection<short> srcBits)
{
ICollection<byte> bytes = srcBits.SelectMany(v => EndianFixer(BitConverter.GetBytes(v))).ToList();
InitializeFrom(bytes);
}
public BoolArray(ICollection<ushort> srcBits)
{
ICollection<byte> bytes = srcBits.SelectMany(v => EndianFixer(BitConverter.GetBytes(v))).ToList();
InitializeFrom(bytes);
}
public BoolArray(ICollection<int> srcBits)
{
ICollection<byte> bytes = srcBits.SelectMany(v => EndianFixer(BitConverter.GetBytes(v))).ToList();
InitializeFrom(bytes);
}
public BoolArray(ICollection<uint> srcBits)
{
ICollection<byte> bytes = srcBits.SelectMany(v => EndianFixer(BitConverter.GetBytes(v))).ToList();
InitializeFrom(bytes);
}
public BoolArray(ICollection<long> srcBits)
{
ICollection<byte> bytes = srcBits.SelectMany(v => EndianFixer(BitConverter.GetBytes(v))).ToList();
InitializeFrom(bytes);
}
public BoolArray(ICollection<ulong> srcBits)
{
ICollection<byte> bytes = srcBits.SelectMany(v => EndianFixer(BitConverter.GetBytes(v))).ToList();
InitializeFrom(bytes);
}
public BoolArray(int capacity, bool defaultValue = false)
{
this.bits = new UInt32[RequiredSize(capacity)];
this._length = capacity;
// Only need to do this if true, because default for all bits is false
if (defaultValue) for (int i = 0; i < this._length; i++) this[i] = true;
}
private void InitializeFrom(ICollection<byte> srcBits)
{
this._length = srcBits.Count * 8;
this.bits = new UInt32[RequiredSize(this._length)];
for (int i = 0; i < srcBits.Count; i++)
{
uint bv = srcBits.Skip(i).Take(1).Single();
for (int b = 0; b < 8; b++)
{
bool bitVal = ((bv << b) & 0x0080) != 0;
int bi = 8 * i + b;
this[bi] = bitVal;
}
}
}
private void InitializeFrom(ICollection<bool> srcBits)
{
this._length = srcBits.Count;
this.bits = new UInt32[RequiredSize(this._length)];
int index = 0;
foreach (var b in srcBits) this[index++] = b;
}
private static int RequiredSize(int bitCapacity)
{
return (bitCapacity + 31) >> 5;
}
#endregion
public bool this[int index]
{
get
{
if (index >= _length) throw new IndexOutOfRangeException();
int byteIndex = index >> 5;
int bitIndex = index & 0x1f;
return ((bits[byteIndex] << bitIndex) & ONE) != 0;
}
set
{
if (index >= _length) throw new IndexOutOfRangeException();
int byteIndex = index >> 5;
int bitIndex = index & 0x1f;
if (value) bits[byteIndex] |= (ONE >> bitIndex);
else bits[byteIndex] &= ~(ONE >> bitIndex);
}
}
#region Interfaces implementation
#region IEnumerable
public IEnumerator GetEnumerator()
{
//for (int i = 0; i < _length; i++) yield return this[i];
return this.ToArray().GetEnumerator();
}
#endregion
#region ICollection
public void CopyTo(Array array, int index)
{
if (array == null) throw new ArgumentNullException("array");
if (index < 0) throw new ArgumentOutOfRangeException("index");
if (array.Rank != 1) throw new ArgumentException("Multidimensional array not supported");
if (array is UInt32[]) Array.Copy(this.bits, 0, array, index, (this.Count + sizeof(UInt32) - 1) / sizeof(UInt32));
else if (array is bool[]) Array.Copy(this.ToArray(), 0, array, index, this.Count);
else throw new ArgumentException("Array type not supported (UInt32[] or bool[] only)");
}
public int Count
{
get { return this._length; }
private set
{
if (value > this._length) Extend(value - this._length);
else this._length = Math.Max(0, value);
}
}
public bool IsSynchronized
{
get { return false; }
}
public object SyncRoot
{
get
{
if (this._syncRoot == null) Interlocked.CompareExchange<object>(ref this._syncRoot, new object(), null);
return _syncRoot;
}
}
#endregion
#region ICloneable
public object Clone()
{
return new BoolArray(this);
}
// Not part of ICloneable, but better - returns a strongly-typed result
public BoolArray Dup()
{
return new BoolArray(this);
}
#endregion
#endregion
#region String Conversions
public override string ToString()
{
return ToBinaryString();
//return ToHexString(" ", " ■ ");
}
public static BoolArray FromHexString(string hex)
{
if (hex == null) throw new ArgumentNullException("hex");
List<bool> bits = new List<bool>();
for (int i = 0; i < hex.Length; i++)
{
int b = byte.Parse(hex[i].ToString(), NumberStyles.HexNumber);
bits.Add((b >> 3) == 1);
bits.Add(((b & 0x7) >> 2) == 1);
bits.Add(((b & 0x3) >> 1) == 1);
bits.Add((b & 0x1) == 1);
}
BoolArray ba = new BoolArray(bits.ToArray());
return ba;
}
public string ToHexString(string bitSep8 = null, string bitSep128 = null)
{
string s = string.Empty;
int b = 0;
bool[] bbits = this.ToArray();
for (int i = 1; i <= bbits.Length; i++)
{
b = (b << 1) | (bbits[i - 1] ? 1 : 0);
if (i % 4 == 0)
{
s = s + string.Format("{0:x}", b);
b = 0;
}
if (i % (8 * 16) == 0)
{
s = s + bitSep128;
}
else if (i % 8 == 0)
{
s = s + bitSep8;
}
}
int ebits = bbits.Length % 4;
if (ebits != 0)
{
b = b << (4 - ebits);
s = s + string.Format("{0:x}", b);
}
return s;
}
public static BoolArray FromBinaryString(string bin, char[] trueChars = null)
{
if (trueChars == null) trueChars = new char[] { '1', 'Y', 'y', 'T', 't' };
if (bin == null) throw new ArgumentNullException("bin");
BoolArray ba = new BoolArray(bin.Length);
for (int i = 0; i < bin.Length; i++) ba[i] = bin[i].In(trueChars);
return ba;
}
public string ToBinaryString(char setChar = '1', char unsetChar = '0')
{
return new string(this.ToArray().Select(v => v ? setChar : unsetChar).ToArray());
}
#endregion
#region Class Methods
public bool[] ToArray()
{
bool[] vbits = new bool[this._length];
for (int i = 0; i < _length; i++) vbits[i] = this[i];
return vbits;
}
public BoolArray Append(ICollection<bool> addBits)
{
int startPos = this._length;
Extend(addBits.Count);
bool[] bitArray = addBits.ToArray();
for (int i = 0; i < bitArray.Length; i++) this[i + startPos] = bitArray[i];
return this;
}
public BoolArray Append(BoolArray addBits)
{
return this.Append(addBits.ToArray());
}
public static BoolArray Concatenate(params BoolArray[] bArrays)
{
return new BoolArray(bArrays.SelectMany(ba => ba.ToArray()).ToArray());
}
private void Extend(int numBits)
{
numBits += this._length;
int reqBytes = RequiredSize(numBits);
if (reqBytes > this.bits.Length)
{
UInt32[] newBits = new UInt32[reqBytes];
this.bits.CopyTo(newBits, 0);
this.bits = newBits;
}
this._length = numBits;
}
public bool Get(int index)
{
return this[index];
}
public BoolArray GetBits(int startBit = 0, int numBits = -1)
{
if (numBits == -1) numBits = bits.Length;
return new BoolArray(this.ToArray().Skip(startBit).Take(numBits).ToArray());
}
public BoolArray Repeat(int numReps)
{
bool[] oBits = this.ToArray();
List<bool> nBits = new List<bool>();
for(int i=0; i<numReps; i++) nBits.AddRange(oBits);
this.InitializeFrom(nBits);
return this;
}
public BoolArray Reverse()
{
int n = this.Count;
for(int i=0; i<n/2; i++)
{
bool b1 = this[i];
this[i] = this[n - i - 1];
this[n - i - 1] = b1;
}
return this;
}
public BoolArray Set(int index, bool v)
{
this[index] = v;
return this;
}
public BoolArray SetAll(bool v)
{
for (int i = 0; i < this.Count; i++) this[i] = v;
return this;
}
public BoolArray SetBits(ICollection<bool> setBits, int destStartBit = 0, int srcStartBit = 0, int numBits = -1, bool allowExtend = false)
{
if (setBits == null) throw new ArgumentNullException("setBits");
if ((destStartBit < 0) || (destStartBit >= this.Count)) throw new ArgumentOutOfRangeException("destStartBit");
if ((srcStartBit < 0) || (srcStartBit >= setBits.Count)) throw new ArgumentOutOfRangeException("srcStartBit");
bool[] sBits;
if (setBits is bool[]) sBits = (bool[])setBits;
else sBits = setBits.ToArray();
if (numBits == -1) numBits = setBits.Count;
if (numBits > (setBits.Count - srcStartBit)) numBits = setBits.Count - srcStartBit;
int diffSize = numBits - (this.Count - destStartBit);
if (diffSize > 0)
{
if (allowExtend) Extend(diffSize);
else numBits = this.Count - destStartBit;
}
for (int i = 0; i < numBits; i++) this[destStartBit + i] = sBits[srcStartBit + i];
return this;
}
public List<BoolArray> SplitEvery(int numBits)
{
int i = 0;
List<BoolArray> bitSplits = new List<BoolArray>();
while (i < this.Count)
{
bitSplits.Add(this.GetBits(i, numBits));
i += numBits;
}
return bitSplits;
}
public byte[] ToBytes(int startBit = 0, int numBits = -1)
{
if (numBits == -1) numBits = this._length - startBit;
BoolArray ba = GetBits(startBit, numBits);
int nb = (numBits + 7) / 8;
byte[] bb = new byte[nb];
for (int i = 0; i < ba.Count; i++)
{
if (!ba[i]) continue;
int bp = 7 - (i % 8);
bb[i / 8] = (byte)((int)bb[i / 8] | (1 << bp));
}
return bb;
}
#endregion
#region Logical Bitwise Operations
public BoolArray BinaryBitwiseOp(Func<bool, bool, bool> op, BoolArray ba, int start = 0)
{
for (int i = 0; i < ba.Count; i++)
{
if (start + i >= this.Count) break;
this[start + i] = op(this[start + i], ba[i]);
}
return this;
}
public BoolArray Xor(BoolArray xor, int start = 0)
{
return BinaryBitwiseOp((a, b) => (a ^ b), xor, start);
}
public BoolArray And(BoolArray and, int start = 0)
{
return BinaryBitwiseOp((a, b) => (a & b), and, start);
}
public BoolArray Or(BoolArray or, int start = 0)
{
return BinaryBitwiseOp((a, b) => (a | b), or, start);
}
public BoolArray Not(int start = 0, int len = -1)
{
for (int i = start; i < this.Count; i++)
{
if (--len == -1) break;
this[i] = !this[i];
}
return this;
}
#endregion
#region Class Operators
public static BoolArray operator +(BoolArray a, BoolArray b)
{
return a.Dup().Append(b);
}
public static BoolArray operator |(BoolArray a, BoolArray b)
{
return a.Dup().Or(b);
}
public static BoolArray operator &(BoolArray a, BoolArray b)
{
return a.Dup().And(b);
}
public static BoolArray operator ^(BoolArray a, BoolArray b)
{
return a.Dup().Xor(b);
}
public static BoolArray operator ~(BoolArray a)
{
return a.Dup().Not();
}
public static BoolArray operator <<(BoolArray a, int shift)
{
return a.Dup().Append(new bool[shift]);
}
public static BoolArray operator >>(BoolArray a, int shift)
{
return new BoolArray(a.ToArray().Take(Math.Max(0, a.Count - shift)).ToArray());
}
public static bool operator ==(BoolArray a, BoolArray b)
{
if (a.Count != b.Count) return false;
for (int i = 0; i < a.Count; i++) if (a[i] != b[i]) return false;
return true;
}
public override bool Equals(object obj)
{
if (!(obj is BoolArray)) return false;
return (this == (BoolArray)obj);
}
public override int GetHashCode()
{
return this.ToHexString().GetHashCode();
}
public static bool operator !=(BoolArray a, BoolArray b)
{
return !(a == b);
}
#endregion
}
}
I had the same issue, but had more than just the one Cardinality method to convert. So, I opted to port the entire BitSet class. Fortunately it was self-contained.
Here is the Gist of the C# port.
I have also added it to the open source BoboBrowse.Net project.
I have also ported the version from Apache Harmony (which is essentially a carbon copy of the JDK) and added it to the general library J2N so it is easy to consume.