In C#, is there a way to right/left shift an entire byte array (and subsequently adding a byte to a particular side for the last bit isn't lost)?
I know this sounds like a weird request, but I'd still like to know if its possible and/or how to begin doing it.
Just for grins. shifting and rotating bytes in a byte array. (not bitshifting)
shift left, zero fill:
mybytes.Skip(1).Concat(new byte[] { 0 }).ToArray();
shift right, zero fill:
(new byte[] {0}).Concat(mybytes.Take(mybytes.Length - 1)).ToArray();
rotate left:
mybytes.Skip(1).Concat(mybytes.Take(1)).ToArray();
rotate right:
mybytes.Skip(mbytes.Length - 1).Concat(mbytes.Take(mbytes.Length - 1)).ToArray();
Yes, you can. See the following methods I wrote:
/// <summary>
/// Rotates the bits in an array of bytes to the left.
/// </summary>
/// <param name="bytes">The byte array to rotate.</param>
public static void RotateLeft(byte[] bytes)
{
bool carryFlag = ShiftLeft(bytes);
if (carryFlag == true)
{
bytes[bytes.Length - 1] = (byte)(bytes[bytes.Length - 1] | 0x01);
}
}
/// <summary>
/// Rotates the bits in an array of bytes to the right.
/// </summary>
/// <param name="bytes">The byte array to rotate.</param>
public static void RotateRight(byte[] bytes)
{
bool carryFlag = ShiftRight(bytes);
if (carryFlag == true)
{
bytes[0] = (byte)(bytes[0] | 0x80);
}
}
/// <summary>
/// Shifts the bits in an array of bytes to the left.
/// </summary>
/// <param name="bytes">The byte array to shift.</param>
public static bool ShiftLeft(byte[] bytes)
{
bool leftMostCarryFlag = false;
// Iterate through the elements of the array from left to right.
for (int index = 0; index < bytes.Length; index++)
{
// If the leftmost bit of the current byte is 1 then we have a carry.
bool carryFlag = (bytes[index] & 0x80) > 0;
if (index > 0)
{
if (carryFlag == true)
{
// Apply the carry to the rightmost bit of the current bytes neighbor to the left.
bytes[index - 1] = (byte)(bytes[index - 1] | 0x01);
}
}
else
{
leftMostCarryFlag = carryFlag;
}
bytes[index] = (byte)(bytes[index] << 1);
}
return leftMostCarryFlag;
}
/// <summary>
/// Shifts the bits in an array of bytes to the right.
/// </summary>
/// <param name="bytes">The byte array to shift.</param>
public static bool ShiftRight(byte[] bytes)
{
bool rightMostCarryFlag = false;
int rightEnd = bytes.Length - 1;
// Iterate through the elements of the array right to left.
for (int index = rightEnd; index >= 0; index--)
{
// If the rightmost bit of the current byte is 1 then we have a carry.
bool carryFlag = (bytes[index] & 0x01) > 0;
if (index < rightEnd)
{
if (carryFlag == true)
{
// Apply the carry to the leftmost bit of the current bytes neighbor to the right.
bytes[index + 1] = (byte)(bytes[index + 1] | 0x80);
}
}
else
{
rightMostCarryFlag = carryFlag;
}
bytes[index] = (byte)(bytes[index] >> 1);
}
return rightMostCarryFlag;
}
It seems you are performing bit operations on large amount of bits storing them in a byte array. Consider using BitArray class and BitVector32 Structure. Depending on what you are doing with bits you can create a class like this. Note that shifting works in O(1) instead of O(n).
public class BitRing : IEnumerable<bool>
{
private readonly BitArray m_InnerBitArray;
private int m_StarIndex;
public BitRing(byte[] bytes)
{
m_InnerBitArray = new BitArray(bytes);
m_StarIndex = 0;
}
public void ShiftLeft()
{
m_StarIndex++;
}
public void ShiftRight()
{
m_StarIndex--;
}
public bool this[int i]
{
get
{
int index = GetIndex(i);
return m_InnerBitArray[index];
}
set
{
int index = GetIndex(i);
m_InnerBitArray[index] = value;
}
}
private int GetIndex(int i)
{
return i - m_StarIndex%m_InnerBitArray.Count;
}
public IEnumerator<bool> GetEnumerator()
{
for (int i = m_StarIndex; i < m_InnerBitArray.Count; i++)
{
yield return m_InnerBitArray[i];
}
for (int i = 0; i < m_StarIndex; i++)
{
yield return m_InnerBitArray[i];
}
}
IEnumerator IEnumerable.GetEnumerator()
{
return GetEnumerator();
}
}
I've given it some more thought and realized that this probably fits the question better:
public static void Main()
{
byte[] bytes = new byte[] { 0xFF, 0x01, 0x80, 0x81 };
Stack<bool> bitStack = CreateBitStack(bytes);
ShiftLeftExpand(bitStack, 1);
byte[] newBytes = CreateByteArray(bitStack);
}
public static void ShiftLeftExpand(Stack<bool> bitStack, int count)
{
while (count-- > 0)
{
bitStack.Push(false);
}
}
public static Stack<bool> CreateBitStack(byte[] bytes)
{
Stack<bool> bitStack = new Stack<bool>(bytes.Length * 8);
for (int bytePosition = 0; bytePosition < bytes.Length; bytePosition++)
{
for (int bitPosition = 7; bitPosition >= 0; bitPosition--)
{
int bitMask = 0x01 << bitPosition;
bitStack.Push((bytes[bytePosition] & bitMask) > 0);
}
}
return bitStack;
}
public static byte[] CreateByteArray(Stack<bool> bitStack)
{
int newArrayLength = (int)Math.Ceiling(bitStack.Count / 8.0);
byte[] bytes = new byte[newArrayLength];
int bitCounter = 0;
while (bitStack.Count > 0)
{
bool? bitValue = bitStack.Pop();
int bitPosition = bitCounter % 8;
int bytePosition = newArrayLength - 1 - bitCounter / 8;
if (bitValue == true)
{
bytes[bytePosition] = (byte)(bytes[bytePosition] | (0x01 << bitPosition));
}
bitCounter++;
}
return bytes;
}
A similar technique can be applied to perform the right shift.
Linq way:
static class Shifter
{
public static byte[] ShiftLeft(this byte[] array, int n)
{
var a = array.Select(x => (byte)(x >> 8 - n % 8)).Concat(new byte[(7 + n) / 8]).Select((x, i) => new Tuple<int, byte>(i - (n % 8 == 0 ? 0 : 1), x));
var b = array.Select(x => (byte)(x << n % 8)).Concat(new byte[n / 8]).Select((x, i) => new Tuple<int, byte>(i, x));
return (from x in a
join y in b on x.Item1 equals y.Item1 into yy
from y in yy.DefaultIfEmpty()
select (byte)(x.Item2 | (y == null ? 0 : y.Item2))).ToArray();
}
public static byte[] ShiftRight(this byte[] array, int n)
{
return (new byte[n/8]).Concat(ShiftLeft(array, (8 - (n%8))%8)).ToArray();
}
}
I don't think there's a built-in way. I implemented the shift-left operation you described below (assuming little endian). It's not quite as elegant as you can do with x86 assembly (shift with carry instructions), but pretty close to what you could do with C.
Alternately, you can almost use the BigInteger struct (.NET 4 and above) which has a constructor that takes a byte array and a ToByteArray method. But its shift left operation sign-extends the high byte and its shift right operation truncates. So you'd need to compensate for both to get the exact behavior you described.
// Left-shifts a byte array in place. Assumes little-endian. Throws on overflow.
static public void ShiftByteArrayLeft(byte[] array)
{
if (array == null)
throw new ArgumentNullException("array");
if (array[array.Length - 1] >= 0x80)
throw new OverflowException();
// move left-to-right, left-shifting each byte
for (int i = array.Length - 1; i >= 1; --i)
{
// left-shift current byte
array[i] <<= 1;
// carry the bit from the next/right byte if needed
if (array[i - 1] >= 0x80)
++array[i];
}
// finally shift the left-shift the right-most byte
array[0] <<= 1;
}
// Left-shifts a byte array in place. Assumes little-endian. Grows array as needed.
static public void ShiftByteArrayLeftAutoGrow(ref byte[] array)
{
if (array == null)
throw new ArgumentNullException("array");
if (array[array.Length - 1] >= 0x80)
{
// allocate a bigger array and do the left-shift on it
byte[] oldArray = array;
array = new byte[oldArray.Length + 1];
Array.Copy(oldArray, 0, array, 0, oldArray.Length);
}
ShiftByteArrayLeft(array);
}
Shift left:
for (int i = byteArray.Length - 1; i >= 0; i--) byteArray[i] = (byte)((byteArray[i] << 1) | ((i == 0) ? 0 : byteArray[i - 1] >> 7));
Shift right:
for (int i = 0; i <= byteArray.Length - 1; i++) byteArray[i] = (byte)((byteArray[i] >> 1) | ((i == byteArray.Length - 1) ? 0 : byteArray[i + 1] << 7));
These two functions will shift the bits in an array of bytes the specified amount, shifting them into neighboring bytes as needed. Optionally, they can wrap the bits from one end of the array to the other. Note that they create a new array, but the code can be easily changed to modify the passed 'array' instead...
public static byte[] ShiftRight(byte[] array, int shift, bool wrap = false) {
if(shift < 0) {
throw new ArgumentOutOfRangeException("shift");
} else if(shift == 0) {
return (byte[])array.Clone();
} else {
if(wrap) shift %= (array.Length * 8);
if(shift >= (array.Length * 8)) return new byte[array.Length];
var offset = (shift / 8);
shift %= 8;
var ʀ = new byte[array.Length];
for(var ɪ = 0; ɪ < ʀ.Length; ɪ++) {
var indexL_ɪ = (ɪ + offset);
var indexH_ɪ = (ɪ + offset + 1);
if(wrap) {
if(indexL_ɪ >= array.Length) indexL_ɪ -= array.Length;
if(indexH_ɪ >= array.Length) indexH_ɪ -= array.Length;
}
if(indexL_ɪ < array.Length) ʀ[ɪ] = (byte)(array[indexL_ɪ] >> shift);
if(indexH_ɪ < array.Length) ʀ[ɪ] |= (byte)(array[indexH_ɪ] << (8 - shift));
}
return ʀ;
}
}
public static byte[] ShiftLeft(byte[] array, int shift, bool wrap = false) {
if(shift < 0) {
throw new ArgumentOutOfRangeException("shift");
} else if(shift == 0) {
return (byte[])array.Clone();
} else {
if(wrap) shift %= (array.Length * 8);
if(shift >= (array.Length * 8)) return new byte[array.Length];
var offset = (shift / 8);
shift %= 8;
for(var ɪ = 0; ɪ < ʀ.Length; ɪ++) {
var indexL_ɪ = (ɪ - offset - 1);
var indexH_ɪ = (ɪ - offset);
if(wrap) {
if(indexL_ɪ < 0) indexL_ɪ += array.Length;
if(indexH_ɪ < 0) indexH_ɪ += array.Length;
}
if(indexL_ɪ >= 0) ʀ[ɪ] = (byte)(array[indexL_ɪ] >> (8 - shift));
if(indexH_ɪ >= 0) ʀ[ɪ] |= (byte)(array[indexH_ɪ] << shift);
}
return ʀ;
}
}
Related
is there a way to Ror an entire byte[] by a specific amount?
I've already done some research and found a solution to Rol a byte[] :
public static byte[] ROL_ByteArray(byte[] arr, int nShift)
{
//Performs bitwise circular shift of 'arr' by 'nShift' bits to the left
//RETURN:
// = Result
byte[] resArr = new byte[arr.Length];
if(arr.Length > 0)
{
int nByteShift = nShift / (sizeof(byte) * 8); //Adjusted after #dasblinkenlight's correction
int nBitShift = nShift % (sizeof(byte) * 8);
if (nByteShift >= arr.Length)
nByteShift %= arr.Length;
int s = arr.Length - 1;
int d = s - nByteShift;
for (int nCnt = 0; nCnt < arr.Length; nCnt++, d--, s--)
{
while (d < 0)
d += arr.Length;
while (s < 0)
s += arr.Length;
byte byteS = arr[s];
resArr[d] |= (byte)(byteS << nBitShift);
resArr[d > 0 ? d - 1 : resArr.Length - 1] |= (byte)(byteS >> (sizeof(byte) * 8 - nBitShift));
}
}
return resArr;
}
The author of this code can be found here: Is there a function to do circular bitshift for a byte array in C#?
Any idea how I can do the same thing but perform a Ror operation instead of a Rol operation on a byte[] ?
static byte[] ROR_ByteArray(byte[] arr, int nShift)
{
return ROL_ByteArray(arr, arr.Length*8-nShift);
}
I'm trying to decode bitcoin address from Base58 string into byte array, and to do that I rewrited original function from Satoshi repository (https://github.com/bitcoin/bitcoin/blob/master/src/base58.cpp), written in c++, to c# (which I'm using).
Original code
static const char* pszBase58 = "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz";
bool DecodeBase58(const char *psz, std::vector<unsigned char>& vch) {
// Skip leading spaces.
while (*psz && isspace(*psz))
psz++;
// Skip and count leading '1's.
int zeroes = 0;
while (*psz == '1') {
zeroes++;
psz++;
}
// Allocate enough space in big-endian base256 representation.
std::vector<unsigned char> b256(strlen(psz) * 733 / 1000 + 1); // log(58) / log(256), rounded up.
// Process the characters.
while (*psz && !isspace(*psz)) {
// Decode base58 character
const char *ch = strchr(pszBase58, *psz);
if (ch == NULL)
return false;
// Apply "b256 = b256 * 58 + ch".
int carry = ch - pszBase58;
for (std::vector<unsigned char>::reverse_iterator it = b256.rbegin(); it != b256.rend(); it++) {
carry += 58 * (*it);
*it = carry % 256;
carry /= 256;
}
assert(carry == 0);
psz++;
}
// Skip trailing spaces.
while (isspace(*psz))
psz++;
if (*psz != 0)
return false;
// Skip leading zeroes in b256.
std::vector<unsigned char>::iterator it = b256.begin();
while (it != b256.end() && *it == 0)
it++;
// Copy result into output vector.
vch.reserve(zeroes + (b256.end() - it));
vch.assign(zeroes, 0x00);
while (it != b256.end())
vch.push_back(*(it++));
return true;
}
Mine rewrited c# version
private static string Base58characters = "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz";
public static bool Decode(string source, ref byte[] destination)
{
int i = 0;
while (i < source.Length)
{
if (source[i] == 0 || !Char.IsWhiteSpace(source[i]))
{
break;
}
i++;
}
int zeros = 0;
while (source[i] == '1')
{
zeros++;
i++;
}
byte[] b256 = new byte[(source.Length - i) * 733 / 1000 + 1];
while (i < source.Length && !Char.IsWhiteSpace(source[i]))
{
int ch = Base58characters.IndexOf(source[i]);
if (ch == -1) //null
{
return false;
}
int carry = Base58characters.IndexOf(source[i]);
for (int k = b256.Length - 1; k > 0; k--)
{
carry += 58 * b256[k];
b256[k] = (byte)(carry % 256);
carry /= 256;
}
i++;
}
while (i < source.Length && Char.IsWhiteSpace(source[i]))
{
i++;
}
if (i != source.Length)
{
return false;
}
int j = 0;
while (j < b256.Length && b256[j] == 0)
{
j++;
}
destination = new byte[zeros + (b256.Length - j)];
for (int kk = 0; kk < destination.Length; kk++)
{
if (kk < zeros)
{
destination[kk] = 0x00;
}
else
{
destination[kk] = b256[j++];
}
}
return true;
}
Function that I'm using for converting from byte-array to HexString
public static string ByteArrayToHexString(byte[] source)
{
return BitConverter.ToString(source).Replace("-", "");
}
To test if everything is working correctly I've used test cases found online here (https://github.com/ThePiachu/Bitcoin-Unit-Tests/blob/master/Address/Address%20Generation%20Test%201.txt). Good thing is that 97% of this test are passed correctly but for 3 there is a little error and I do not know where it is coming from. So my ask to you is to point me what could for these test go wrong or where in rewriting I've made an error. Thank you in advance.
The test cases where errors occures are 1, 21 and 25.
1.
Input:
16UwLL9Risc3QfPqBUvKofHmBQ7wMtjvM
Output:
000966776006953D5567439E5E39F86A0D273BEED61967F6
Should be:
00010966776006953D5567439E5E39F86A0D273BEED61967F6
21.
Input:
1v3VUYGogXD7S1E8kipahj7QXgC568dz1
Output:
0008201462985DF5255E4A6C9D493C932FAC98EF791E2F22
Should be:
000A08201462985DF5255E4A6C9D493C932FAC98EF791E2F22
25.
Input:
1axVFjCkMWDFCHjQHf99AsszXTuzxLxxg
Output:
006C0B8995C7464E89F6760900EA6978DF18157388421561
Should be:
00066C0B8995C7464E89F6760900EA6978DF18157388421561
In your for-loop:
for (int k = b256.Length - 1; k > 0; k--)
The loop condition should be k >= 0 so that you don't skip the first byte in b256.
Could anyone help me optimize this piece of code? Its currently a large bottleneck as it gets called very often. Even a 25% speed improvement would be significant.
public int ReadInt(int length)
{
if (Position + length > Length)
throw new BitBufferException("Not enough bits remaining.");
int result = 0;
while (length > 0)
{
int off = Position & 7;
int count = 8 - off;
if (count > length)
count = length;
int mask = (1 << count) - 1;
int bits = (Data[Position >> 3] >> off);
result |= (bits & mask) << (length - count);
length -= count;
Position += count;
}
return result;
}
Best answer would go to fastest solution. Benchmarks done with dottrace. Currently this block of code takes up about 15% of the total cpu time. Lowest number wins best answer.
EDIT: Sample usage:
public class Auth : Packet
{
int Field0;
int ProtocolHash;
int Field1;
public override void Parse(buffer)
{
Field0 = buffer.ReadInt(9);
ProtocolHash = buffer.ReadInt(32);
Field1 = buffer.ReadInt(8);
}
}
Size of Data is variable but in most cases 512 bytes;
How about using pointers and unsafe context? You didn't say anything about your input data, method context, etc. so I tried to deduct all of these by myself.
public class BitTest
{
private int[] _data;
public BitTest(int[] data)
{
Length = data.Length * 4 * 8;
// +2, because we use byte* and long* later
// and don't want to read outside the array memory
_data = new int[data.Length + 2];
Array.Copy(data, _data, data.Length);
}
public int Position { get; private set; }
public int Length { get; private set; }
and ReadInt method. Hope comments give a little light on the solution:
public unsafe int ReadInt(int length)
{
if (Position + length > Length)
throw new ArgumentException("Not enough bits remaining.");
// method returns int, so getting more then 32 bits is pointless
if (length > 4 * 8)
throw new ArgumentException();
//
int bytePosition = Position / 8;
int bitPosition = Position % 8;
Position += length;
// get int* on array to start with
fixed (int* array = _data)
{
// change pointer to byte*
byte* bt = (byte*)array;
// skip already read bytes and change pointer type to long*
long* ptr = (long*)(bt + bytePosition);
// read value from current pointer position
long value = *ptr;
// take only necessary bits
value &= (1L << (length + bitPosition)) - 1;
value >>= bitPosition;
// cast value to int before returning
return (int)value;
}
}
}
I didn't test the method, but would bet it's much faster then your approach.
My simple test code:
var data = new[] { 1 | (1 << 8 + 1) | (1 << 16 + 2) | (1 << 24 + 3) };
var test = new BitTest(data);
var bytes = Enumerable.Range(0, 4)
.Select(x => test.ReadInt(8))
.ToArray();
bytes contains { 1, 2, 4, 8}, as expected.
I Don't know if this give you a significant improvements but it should give you some numbers.
Instead of creating new int variables inside the loop (this requires a time to create) let reserved those variables before entering the loop.
public int ReadInt(int length)
{
if (Position + length > Length)
throw new BitBufferException("Not enough bits remaining.");
int result = 0;
int off = 0;
int count = 0;
int mask = 0;
int bits = 0
while (length > 0)
{
off = Position & 7;
count = 8 - off;
if (count > length)
count = length;
mask = (1 << count) - 1;
bits = (Data[Position >> 3] >> off);
result |= (bits & mask) << (length - count);
length -= count;
Position += count;
}
return result;
}
HOPE THIS increase your performance even a bit
So Say I have an array of bytes that is 16 long, with each 8 bits representing my data and an array that is 8 long with each 4 bits (so 2 per byte) representing my data.
If I wanted to loop over these and get the values, what would be the easiest way of doing so?
My poor attempt would be something like this, but this doesn't appear to be working as I expect.
for(int i = 0; i < bigByteArray.Length; i++)
{
byte BigByteInfo = bigByteArray[i];
byte SmallByteInfo;
if(i % 2 == 0)
{
SmallByteInfo = smallByteArray[i / 2] % 16;
}
else
{
SmallByteInfo = smallByteArray[i / 2] / 16;
}
//Use of data Here.
}
you can use this class as helper class
public class FoutBitsArrayEnumerator : IEnumeable<byte>
{
FoutBitsArrayEnumerator(byte[] array)
{
this.array = array;
}
public IEnumerator<byte> GetEnumerator
{
foreach (byte i in array)
{
yield return i & 15;
yield return (i >> 4) & 15;
}
}
byte[] array;
}
If I understand right (bigByteArray is 16 long, smallByteArray is 8 long, packed):
for(int i = 0; i < bigByteArray.Length; i++)
{
bigByteArray[i] = (byte)((smallByteArray[i / 2] >> (i % 2 == 0 ? 4 : 0)) & 0xF);
}
I need to write an integer to a byte array such that leading zeros are omitted and the bytes are written in big endian order.
Example:
int original = 0x00123456;
byte[] encoded = Encode(original); // == new byte[] { 0x12, 0x34, 0x56 };
int decoded = Decode(encoded); // == 0x123456
My Decode method:
private static int Decode(byte[] buffer, int index, int length)
{
int result = 0;
while (length > 0)
{
result = (result << 8) | buffer[index];
index++;
length--;
}
return result;
}
I'm struggling to come up with an Encode method that doesn't require a temporary buffer or reverses the bytes after writing them in little endian order. Can anyone help?
private static int Encode(int value, byte[] buffer, int index)
{
}
As per OP's request, here is a version without loops for a 32-bit number:
private static int Encode(int value, byte[] buffer, int index)
{
byte temp;
bool leading = true;
temp = (value >> 24) & 0xFF;
if (temp > 0) {
buffer[index++] = temp;
leading = false;
}
temp = (value >> 16) & 0xFF;
if (temp > 0 || leading == false) {
buffer[index++] = temp;
leading = false;
}
temp = (value >> 8) & 0xFF;
if (temp > 0 || leading == false) {
buffer[index++] = temp;
leading = false;
}
temp = value & 0xFF;
buffer[index++] = temp;
return index;
}
Version using a loop for 32-bit numbers:
private static int Encode(int value, byte[] buffer, int index)
{
int length = 0;
for (int i = 3; i >= 0; i++) {
byte temp = (byte)(value >> (8 * i));
if (temp > 0 || length > 0) {
buffer[index++] = temp;
length++;
}
}
return length;
}
Note that this version doesn't write anything if the input is just 0.
private static int Encode(int value, byte[] buffer, int index)
{
int length = 0;
int valueCopy = value;
while (valueCopy != 0)
{
valueCopy >>= 8;
length++;
}
for (int i = 0; i < length; i++)
{
buffer[index + length - i - 1] = (byte)value;
value >>= 8;
}
return length;
}
Note that you are saving the value to a variable Length byte array. If you save these byteArray, you need save also the length.
You can see the protected Functions Write7BitEncodedInt from BinaryWriter and Read7BitEncodedInt from BinaryReader.
These functions save storage on disk for positive numbers. A number 0-128 needs only one byte.
Microsoft uses these functions to store/retrieve the String length prefix when save to Stream.
To use these Functions, you can create own Class derived from BinaryReader / BinaryWriter.