Develop Reference

Finding largest sequence of bytes in two byte arrays

Example:
{ 54, 87, 23, 87, 45, 67, 7, 85, 65, 65, 3, 4, 55, 76, 65, 64, 5, 6, 4, 54, 45, 6, 4 };
{ 76, 57, 65, 3, 4, 55, 76, 65, 64, 5, 6, 4, 54, 45, 8, 65, 66, 57, 6, 7, 7, 56, 6, 7, 44, 57, 8, 76, 54, 67 };
Basically, I have two byte[], and need to find the largest identical sequence of bytes in both.
I have tried the obvious thing and wrote some code that bruteforces the result:
var bestIndex = 0;
var bestCount = 0;
for (var i1 = 0; i1 + bestCount < data1.Length; i1++)
{
var currentCount = 0;
for (var i2 = 0; i2 < data2.Length; i2++)
{
if (data1[i1 + currentCount] == data2[i2])
{
currentCount++;
if (i1 + currentCount == data1.Length)
{
bestCount = currentCount;
bestIndex = i1;
break;
}
}
else
{
if (currentCount > bestCount)
{
bestCount = currentCount;
bestIndex = i1;
}
currentCount = 0;
}
}
if (currentCount > bestCount)
{
bestCount = currentCount;
bestIndex = i1;
}
}
However, in my application the byte arrays will be much larger, up to a GB even. So basically I need a hint / code on how to be more efficient than that.

I had a couple thoughts on this. I'm not sure if this helps or hurts, but have you considered working backwards through the largest possibilities first, so you can terminate as soon as you've found a match.
byte[] b1 = { 54, 87, 23, 87, 45, 67, 7, 85, 65, 65, 3, 4, 55, 76, 65, 64, 5, 6, 4, 54, 45, 6, 4 };
byte[] b2 = { 76, 57, 65, 3, 4, 55, 76, 65, 64, 5, 6, 4, 54, 45, 8, 65, 66, 57, 6, 7, 7, 56, 6, 7, 44, 57, 8, 76, 54, 67 };
//figure out which one is smaller, since that one will limit the range options
byte[] smaller;
byte[] bigger;
if (b1.Count() > b2.Count())
{
bigger = b1;
smaller = b2;
}
else
{
bigger = b2;
smaller = b1;
}
// doesn't matter what order we put these in, since they will be ordered later by length
List<Tuple<int, int>> ranges = new List<Tuple<int, int>>();
Parallel.For(0, smaller.Count(), (i1) => {
Parallel.For(i1 + 1, smaller.Count(), (i2) =>
{
ranges.Add(new Tuple<int, int>(i1, i2));
});
});
// order by length of slice produced by range in descending order
// this way, once we get an answer, we know nothing else can be longer
ranges = ranges.OrderByDescending(x => x.Item2 - x.Item1).ToList();
Tuple<int, int> largestMatchingRange = new Tuple<int, int>(0, 0);
foreach (Tuple<int, int> range in ranges)
{
bool match = true; // set in outer loop to allow for break
for (int i1 = 0; i1 < bigger.Count(); i1++)
{
if (bigger.Count() <= i1 + (range.Item2 - range.Item1))
{
//short cut if the available slice from the bigger array is shorter than the range length
match = false;
continue;
}
match = true; // reset to true to allow for new attempt for each larger array slice
for (int i2 = range.Item1, i1Temp = i1; i2 < range.Item2; i2++, i1Temp++)
{
if (bigger[i1Temp] != smaller[i2])
{
match = false;
break;
}
}
if (match)
{
largestMatchingRange = range;
break;
}
}
if (match)
{
break;
}
}
byte[] largestMatchingBytes = smaller.Skip(largestMatchingRange.Item1).Take(largestMatchingRange.Item2 - largestMatchingRange.Item1).ToArray();

Instead of checking the bytes one by one, you can save the index locations for each byte value in a dictionary of lists. In your case arrays of 256 lists might be better.
List<int>[] index(byte[] a) { // List<long> if the array can be more than 2GB
var lists = new List<int>[256];
for(int i = 0; i < a.Length; i++) {
var b = a[i];
if (lists[b] == null) lists[b] = new List<int>();
lists[b].Add(i);
}
return lists;
}
then you can loop over the 256 possible byte values
byte[] data1 = { 54, 87, 23, 87, 45, 67, 7, 85, 65, 65, 3, 4, 55, 76, 65, 64, 5, 6, 4, 54, 45, 6, 4 };
byte[] data2 = { 76, 57, 65, 3, 4, 55, 76, 65, 64, 5, 6, 4, 54, 45, 8, 65, 66, 57, 6, 7, 7, 56, 6, 7, 44, 57, 8, 76, 54, 67 };
var indexes1 = index(data1);
var indexes2 = index(data2);
var bestIndex = 0;
var bestCount = 0;
for (var b = 0; b < 256; b++)
{
var list1 = indexes1[b]; if (list1 == null) continue;
var list2 = indexes1[b]; if (list2 == null) continue;
foreach(var index1 in list1)
{
foreach (var index2 in list2)
{
// your code here
for (var i1 = index1; i1 < data1.Length - bestCount; i1++)
{
var currentCount = 0;
for (var i2 = index2; i2 < data2.Length; i2++)
{
if (data1[i1 + currentCount] == data2[i2])
{
currentCount++;
if (i1 + currentCount == data1.Length)
{
bestCount = currentCount;
bestIndex = i1;
break;
}
}
else
{
if (currentCount > bestCount)
{
bestCount = currentCount;
bestIndex = i1;
}
currentCount = 0;
}
}
if (currentCount > bestCount)
{
bestCount = currentCount;
bestIndex = i1;
}
}
}
}
}
var best = data1.Skip(bestIndex).Take(bestCount);
Debug.Print(bestIndex + ", " + bestCount + ": " + string.Join(", ", best));
In theory this feels like it will take less comparisons for bigger arrays, but in practice it will have more memory cache misses, so I am not sure if it will be faster than a more linear parallel version like in the other answer. I did not think into this too much but hopefully it can give you some ideas in case I got it wrong.
Update
I just realized how bad this idea is for a regular machine with less than 32 GB of memory as the list of indexes will take more than 4 times the memory of the byte array.

I figured out the loops, this one should be faster.
byte[] data1 = { 54, 87, 23, 87, 45, 67, 7, 85, 65, 65, 3, 4, 55, 76, 65, 64, 5, 6, 4, 54, 45, 6, 4 };
byte[] data2 = { 76, 57, 65, 3, 4, 55, 76, 65, 64, 5, 6, 4, 54, 45, 8, 65, 66, 57, 6, 7, 7, 56, 6, 7, 44, 57, 8, 76, 54, 67 };
//figure out which one is smaller, since that one will limit the range options
byte[] smaller;
byte[] bigger;
if (data1.Count() > data2.Count())
{
bigger = data1;
smaller = data2;
}
else
{
bigger = data2;
smaller = data1;
}
Tuple<int, int> largestMatchingRange = new Tuple<int, int>(0, 0);
//iterate over slices in reverse length order
for (int length = smaller.Count() - 1; length > 0; length--)
{
int numberOfSlicesForLength = smaller.Count() - length;
bool match = true; // set in outer loop to allow for break
for (int start = 0; start < numberOfSlicesForLength; start++)
{
//within a collection of similarly sized slices, we start with the slice found first within the array
Tuple<int, int> range = new Tuple<int, int>(start, start + length);
for (int i1 = 0; i1 < bigger.Count(); i1++)
{
if (bigger.Count() <= i1 + (range.Item2 - range.Item1))
{
//short cut if the available slice from the bigger array is shorter than the range length
match = false;
continue;
}
match = true; // reset to true to allow for new attempt for each larger array slice
for (int i2 = range.Item1, i1Temp = i1; i2 < range.Item2; i2++, i1Temp++)
{
if (bigger[i1Temp] != smaller[i2])
{
match = false;
break;
}
}
if (match)
{
largestMatchingRange = range;
break;
}
}
if (match)
{
break;
}
}
if (match)
{
break;
}
}
byte[] largestMatchingBytes = smaller.Skip(largestMatchingRange.Item1).Take(largestMatchingRange.Item2 - largestMatchingRange.Item1).ToArray();

ToLookup IEqualityComparer arguments always null

I've stumbled into a weird problem, which does not really make sense to me.
I've got a business object Address with a property Location (with the type SqlGeography). For the sake of my requirement i have to do a lookup on the location, because there may be multiple addresses per exact location.
Since SqlGeography is a complex type i suspected that maybe the lookup isn't working because it isn't based on location coordinates for some reason so i did this:
public class Address
{
public Address(byte[] location)
{
Location = SqlGeography.Deserialize(new SqlBytes(location));
}
public SqlGeography Location { get; set; }
}
public class SqlGeographyComparer : IEqualityComparer<SqlGeography>
{
public bool Equals(SqlGeography x, SqlGeography y)
{
// !!! always entered but for some reason x + y always null
if (x == null && y == null)
return true;
if (x == null ^ y == null)
return false;
return x.STEquals(y).IsTrue;
}
public int GetHashCode(SqlGeography obj)
{
return obj.GetHashCode();
}
}
class Program
{
static void Main(string[] args)
{
var addresses = GetAddresses();
// should be 2 but it's 3 results
var addressesLookup = addresses.ToLookup(d => d.Location);
// should be 2 but it's 3 results
var addressesLookup2 = addresses.ToLookup(d => d.Location, new SqlGeographyComparer());
Console.ReadLine();
}
private static IList<Address> GetAddresses()
{
// 230,16,0,0,1,12,213,97,212,23,126,78,72,64,109,51,198,37,82,163,32,64
var result = new List<Address>();
result.Add(new Address(new byte[] { 230, 16, 0, 0, 1, 12, 213, 97, 212, 23, 126, 78, 72, 64, 109, 51, 198, 37, 82, 163, 32, 64 }));
result.Add(new Address(new byte[] { 230, 16, 0, 0, 1, 12, 213, 97, 212, 23, 126, 78, 72, 64, 109, 51, 198, 37, 82, 163, 32, 64 }));
result.Add(new Address(new byte[] { 230, 16, 0, 0, 1, 12, 213, 97, 212, 23, 126, 78, 72, 64, 109, 51, 198, 37, 82, 163, 32, 63 }));
return result;
}
}
Is this some weird bug i haven't heared about where ToLookup just doesn't pass objects into the given comparer instance?!

Once i changed SqlGeographyComparer to override its method like this:
public int GetHashCode(SqlGeography obj)
{
unchecked
{
return ((obj.Lat.GetHashCode() * 397) ^ obj.Long.GetHashCode());
}
}
it worked correctly.
Turns out GetHashCode has to mimic the required equality comparison (as written in Equals) in order for arguments to work properly.
Reference for implementing GetHashCode in case someone else stumbles upon this question and is wondering about 397:
What is the best algorithm for an overridden System.Object.GetHashCode?

Adding many lists

If I have 100 lists (eg x1 to x100), is there a better way of expressing the final line of code?
var x1 = new List<int>() { 75 };
var x2 = new List<int>() { 95, 64 };
var x3 = new List<int>() { 17, 47, 82 };
var x4 = new List<int>() { 18, 35, 87, 10 };
var x5 = new List<int>() { 20, 04, 82, 47, 65 };
var x6 = new List<int>() { 19, 01, 23, 75, 03, 34 };
var x7 = new List<int>() { 88, 02, 77, 73, 07, 63, 67 };
//etc..
var listOfListOfInts = new List<List<int>>() { x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, x11, x12, x13, x14, x15 };
possibly a Dictionary and a for loop to reference all the x1..100.

As long as x1 etc. aren't referenced elsewhere, then write:
var listOfListOfInts = new List<List<int>>()
{
new List<int>() { 75 },
new List<int>() { 95, 64 },
//etc.
};
In fact you shouldn't need to reference the individual variables elsewhere since listOfListOfInts[0] is just as good as x1 for example.

Do you really need these to be of type List<T>? It looks like you're setting up preinitialized data. If you're never going to change the length of any of these "lists", you could use arrays instead; the syntax is more compact:
var listOfListOfInts = new[] {
new[] { 75 },
new[] { 95, 64 },
new[] { 17, 47, 82 },
new[] { 18, 35, 87, 10 },
new[] { 20, 04, 82, 47, 65 },
new[] { 19, 01, 23, 75, 03, 34 },
new[] { 88, 02, 77, 73, 07, 63, 67 },
// ...
};

Perhaps i'm over complicating things but you could do something like
public interface IClass1
{
IList<IList<int>> ListList { get; set; }
void AddList(List<int> nList);
}
public class Class1 : IClass1
{
public IList<IList<int>> ListList { get; set; }
public void AddList(List<int> nList)
{
ListList.Add(nList);
}
}
and then use it like:
public class Create1
{
public Create1()
{
IClass1 iClass1 = new Class1();
iClass1.AddList(new List<int>() { 75 });
}
}

How to group the same values in a sequence with LINQ?

I have a sequence. For example:
new [] { 10, 1, 1, 5, 25, 45, 45, 45, 40, 100, 1, 1, 2, 2, 3 }
Now I have to remove duplicated values without changing the overall order. For the sequence above:
new [] { 10, 1, 5, 25, 45, 40, 100, 1, 2, 3 }
How to do this with LINQ?

var list = new List<int> { 10, 1, 1, 5, 25, 45, 45, 45, 40, 100, 1, 1, 2, 2, 3 };
List<int> result = list.Where((x, index) =>
{
return index == 0 || x != list.ElementAt(index - 1) ? true : false;
}).ToList();
This returns what you want. Hope it helped.

var list = new List<int> { 10, 1, 1, 5, 25, 45, 45, 45, 40, 100, 1, 1, 2, 2, 3 };
var result = list.Where((item, index) => index == 0 || list[index - 1] != item);

It may be technically possible (though I don't think you can with a one-liner) to solve this with LINQ, but I think it's more elegant to write it yourself.
public static class ExtensionMethods
{
public static IEnumerable<T> PackGroups<T>(this IEnumerable<T> e)
{
T lastItem = default(T);
bool first = true;
foreach(T item in e)
{
if (!first && EqualityComparer<T>.Default.Equals(item, lastItem))
continue;
first = false;
yield return item;
lastItem = item;
}
}
}
You can use it like this:
int[] packed = myArray.PackGroups().ToArray();
It's unclear from the question what should be returned in the case of 1,1,2,3,3,1. Most answers given return 1,2,3, whereas mine returns 1,2,3,1.

You can use Contains and preserve order
List<int> newList = new List<int>();
foreach (int n in numbers)
if (newList.Count == 0 || newList.Last() != n)
newList.Add(n);
var newArray = newList.ToArray();
OUTPUT:
10, 1, 5, 25, 45, 40, 100, 1, 2, 3

Did you try Distinct?
var list = new [] { 10, 20, 20, 5, 25, 45, 45, 45, 40, 100, 1, 1, 2, 2, 3 };
list = list.Distinct();
Edit: Since you apparently only want to group items with the same values when consecutive, you could use the following:
var list = new[] { 10, 1, 1, 5, 25, 45, 45, 45, 40, 100, 1, 1, 2, 2, 3 };
List<int> result = new List<int>();
foreach (int item in list)
if (result.Any() == false || result.Last() != item)
result.Add(item);

Using a string to generate a byte array of data from a lookup table

I have this program that is reading the test string from a textbox and convert it to a byte array that makeup data to be diplayed on a screen. I am getting very close. The code can currently pull the text, convert it to a char array, and then replace the zeros in the byte array with useful data from a 2 dimensional array that contains 5 bits for all the letters of the alphabet. I am have a problem though. The code only seems to run once. If I click the button a second time I end up with an "indexOutOfRange exception unhandled." Also it only seems to work for one letter at a time
EX: if I type "A" it will display, but if I type "AA" I get the same error.
Here is the WordArray[]
byte[,] Letters = new byte[18, 5] { { 63, 72, 72, 63, 0 },
{ 127, 73, 73, 54, 0 },
{ 63, 72, 72, 63, 0 },
{ 127, 73, 73, 54, 0 },
{ 63, 72, 72, 63, 0 },
{ 127, 73, 73, 54, 0 },
{ 63, 72, 72, 63, 0 },
{ 127, 73, 73, 54, 0 },
{ 63, 72, 72, 63, 0 },
{ 127, 73, 73, 54, 0 },
{ 63, 72, 72, 63, 0 },
{ 127, 73, 73, 54, 0 },
{ 63, 72, 72, 63, 0 },
{ 127, 73, 73, 54, 0 },
{ 63, 72, 72, 63, 0 },
{ 127, 73, 73, 54, 0 },
{ 63, 72, 72, 63, 0 },
{ 127, 73, 73, 54, 0 } };
Here is the click_button method:
int Aindex = 0;
int LettersIndex = 0;
private void SendButton_Click(object sender, EventArgs e)
{
WordIndex = 0;
if (Aindex > 0)
{
Aindex = 0;
}
string CurrentTextString = textBox1.Text;
char[] charArray = CurrentTextString.ToCharArray();
if (textBox1.Text == "")
{
}
else
{
foreach (char c in charArray)
{
int index = 0;
CharAsciiArray[index] = Convert.ToChar((Convert.ToInt32(c)));
textBox2.Text += CharAsciiArray[index] + " ";
charCount++;
index++;
}
for (int NumberofBytes = 0; NumberofBytes < charCount; NumberofBytes++)
{
LettersIndex = 0;
// int currentChar = CharAsciiArray[index] - 65;
//textBox2.Text += currentChar;
int currentCharAscii = (CharAsciiArray[Aindex]);
int currentChar = currentCharAscii - 'A';
for (int NumberofBits = 0; NumberofBits < 5; NumberofBits++)
{
// textBox2.Text += currentChar;
WordArray[WordIndex + 3] = Letters[currentChar, LettersIndex];
textBox2.Text += WordArray[WordIndex] + " ";
LettersIndex++;
WordIndex++;
}
Aindex++;
}
SendingData = true;
//SendNextByte();
serialPort1.Write(WordArray, 0, WordArray.Length);
}
}

In the following loop
foreach (char c in charArray)
{
int index = 0;
CharAsciiArray[index] = Convert.ToChar((Convert.ToInt32(c)));
textBox2.Text += CharAsciiArray[index] + " ";
charCount++;
index++;
}
You apparently want to increase the index on each iteration but you're resetting index to 0 every time.
Besides that, make up your mind on what pattern you want to follow when naming your variables. For instance:
Why is AIndex not AnIndex and how would you name the next index? AnotherIndex? Does it need to be global? Why does charArray start with a lowercase c and NumberOfBytes with an uppercase N? Write code as if you would have to explain it to your wife / husband (who knows?) and it'll be easier to maintain / debug.