Develop Reference

c# 1D-byte array to 2D-double array

I'm dealing with c# concurrent-queue and multi-threading in socket-programming tcp/ip
First, I've already done with socket-programming itself. That means, I've already finished coding about client, server and stuffs about communication itself
basic structure is pipe-lined(producer-consumer problem) and now I'm doing with bit conversion
below is brief summary about my code
client-socket ->server-socket -> concurrent_queue_1(with type byte[65536],Thread_1 process this) -> concurrent_queue_2(with type double[40,3500], Thread_2 process this) -> display-data or other work(It can be gpu-work)
*(double[40,3500] can be changed to other size)
Till now,I've implemented putting_data into queue1(Thread1) and just dequeuing all(Thread2) and, its speed is about 700Mbps
The reason I used two concurrent_queue is, I want communication,and type conversion work to be processed in background regardless of main procedure about control things.
Here is the code about my own concurrent_queue with Blocking
public class BlockingConcurrentQueue<T> : IDisposable
{
private readonly ConcurrentQueue<T> _internalQueue;
private AutoResetEvent _autoResetEvent;
private long _consumed;
private long _isAddingCompleted = 0;
private long _produced;
private long _sleeping;
public BlockingConcurrentQueue()
{
_internalQueue = new ConcurrentQueue<T>();
_produced = 0;
_consumed = 0;
_sleeping = 0;
_autoResetEvent = new AutoResetEvent(false);
}
public bool IsAddingCompleted
{
get
{
return Interlocked.Read(ref _isAddingCompleted) == 1;
}
}
public bool IsCompleted
{
get
{
if (Interlocked.Read(ref _isAddingCompleted) == 1 && _internalQueue.IsEmpty)
return true;
else
return false;
}
}
public void CompleteAdding()
{
Interlocked.Exchange(ref _isAddingCompleted, 1);
}
public void Dispose()
{
_autoResetEvent.Dispose();
}
public void Enqueue(T item)
{
_internalQueue.Enqueue(item);
if (Interlocked.Read(ref _isAddingCompleted) == 1)
throw new InvalidOperationException("Adding Completed.");
Interlocked.Increment(ref _produced);
if (Interlocked.Read(ref _sleeping) == 1)
{
Interlocked.Exchange(ref _sleeping, 0);
_autoResetEvent.Set();
}
}
public bool TryDequeue(out T result)
{
if (Interlocked.Read(ref _consumed) == Interlocked.Read(ref _produced))
{
Interlocked.Exchange(ref _sleeping, 1);
_autoResetEvent.WaitOne();
}
if (_internalQueue.TryDequeue(out result))
{
Interlocked.Increment(ref _consumed);
return true;
}
return false;
}
}
My question is here
As I mentioned above, concurrent_queue1's type is byte[65536] and 65536 bytes = 8192 double data.
(40 * 3500=8192 * 17.08984375)
I want merge multiple 8192 double data into form of double[40,3500](size can be changed)and enqueue to concurrent_queue2 with Thread2
It's easy to do it with naive-approach(using many complex for loop) but it's slow cuz, It copys all the
data and expose to upper class or layer.
I'm searching method automatically enqueuing with matched size like foreach loop automatically iterates through 2D-array in row-major way, not yet found
Is there any fast way to merge 1D-byte array into form of 2D-double array and enqueue it?
Thanks for your help!

I try to understand your conversion rule, so I write this conversion code. Use Parallel to speed up the calculation.
int maxSize = 65536;
byte[] dim1Array = new byte[maxSize];
for (int i = 0; i < maxSize; ++i)
{
dim1Array[i] = byte.Parse((i % 256).ToString());
}
int dim2Row = 40;
int dim2Column = 3500;
int byteToDoubleRatio = 8;
int toDoubleSize = maxSize / byteToDoubleRatio;
double[,] dim2Array = new double[dim2Row, dim2Column];
Parallel.For(0, toDoubleSize, i =>
{
int row = i / dim2Column;
int col = i % dim2Column;
int originByteIndex = row * dim2Column * byteToDoubleRatio + col * byteToDoubleRatio;
dim2Array[row, col] = BitConverter.ToDouble(
dim1Array,
originByteIndex);
});

ImageMagic can't catch exception

I generating millions of bmp by Imagemagick in linux with multi-threaded code but for some reason I get one of these errors(without stack trace) which I can't handle:
dotnet: MagickCore/pixel.c:138: ClonePixelChannelMap: Assertion 'channel_map != (PixelChannelMap *) NULL' failed.
or
dotnet: MagickCore/semaphore.c:452: UnlockSemaphoreInfo: Assertion 'semaphore_info != (SemaphoreInfo *) NULL' failed.
or
dotnet: MagickCore/option.c:2137: CloneImageOptions: Assertion 'clone_info->signature == MagickCoreSignature' failed.
My try - catch block doesn't handle these errors so my program closes immediately.
The errors are very random because the same input for Imagemagick in another attempt passes without error.
I use Include="Magick.NET-Q16-x64" Version="7.11.0.0"
I also try "Magick.NET-Q8-x64" and previous versions of imagemagic with the same results.
Linux 4.18.0-3-amd64 #1 SMP Debian 4.18.20-2 (2018-11-23) x86_64 GNU/Linux
Here is code from thread(I run app with 9 threads)
public virtual void GeneratePictureForCurrSpin()
{
do
{
GenerateInternal();
} while (!isAllPicturesGenerated());
}
public override void GenerateInternal()
{
symbols = symbolParser.ParseSymbols(config, spins[currLocalSpin].symbols);
img = (MagickImage)SpinGenerator.rl.emptyImg.Clone();
bgImg = SpinGenerator.rl.bg;
img.Composite(bgImg, new PointD(config.BgX, config.BgY),ImageMagick.CompositeOperator.Over);
reelsBgImg = SpinGenerator.rl.reelsBg;
img.Composite(reelsBgImg, CompositeOperator.Over);
PointD startPoint = new PointD(config.StartOffsetX, config.StartOffsetY);
MagickImage sym;
for(int i = 0; i < symbols.Count; i++)
{
List<int> symbolsOnReel = symbols[i];
for(int j = 0; j < symbolsOnReel.Count; j++)
{
var newPoint = new PointD(startPoint.X + i * (config.SymbolWidth + config.OffsetX), startPoint.Y + j * (config.SymbolHeight + config.OffsetY));
sym = SpinGenerator.rl.symbols[symbolsOnReel[j]-1];
img.Composite(sym, newPoint, ImageMagick.CompositeOperator.Over);
}
}
bool isFreeSpin = (spins[currLocalSpin].betValue == 0)?true:false;
GenerateBottomBar(isFreeSpin);
ShowTokensValue(spins[currLocalSpin].coins);
ResizeImage();
int fileNr = spins[currLocalSpin].spinNr;
String dir = "Resources/Result/" + GAME_NAME + "/" + betValue;
SaveImg(dir , fileNr);
img.Dispose();
IncrementSpinValues();
}
public virtual void GenerateBottomBar(bool isFreeSpin)
{
var pos = (isFreeSpin)?1:0;
bottomBar = SpinGenerator.rl.bottomBars[pos];
PointD startPoint = new PointD(0,config.BgHeight);
img.Composite(bottomBar,startPoint);
}
private void ShowTokensValue(int tokens)
{
if(tokens < 10)
{
var tokenValueImg = SpinGenerator.rl.tokensValues[tokens];
tokenValueImg.Resize(90,60);
img.Composite(tokenValueImg, new PointD(tokenValueStartX + tokenValueImgWidth, tokenValueStartY),ImageMagick.CompositeOperator.Over);
}
else{
int a = tokens / 10;
int b = tokens % 10;
var tokenValueImgA = SpinGenerator.rl.tokensValues[a];
tokenValueImgA.Resize(90,55);
img.Composite(tokenValueImgA, new PointD(tokenValueStartX, tokenValueStartY),ImageMagick.CompositeOperator.Over);
var tokenValueImgB = SpinGenerator.rl.tokensValues[b];
tokenValueImgB.Resize(90,55);
img.Composite(tokenValueImgB, new PointD(tokenValueStartX+ tokenValueImgWidth, tokenValueStartY),ImageMagick.CompositeOperator.Over);
}
var tokensTxtImg = SpinGenerator.rl.tokensTxt;
img.Composite(tokensTxtImg, new PointD(tokenTxtStartX, tokenTxtStartY),ImageMagick.CompositeOperator.Over);
}
internal void ResizeImage()
{
img.Format = ImageMagick.MagickFormat.Bmp3;
img.Depth = 16;
img.Resize(new MagickGeometry("600, 450!"));
//img.Quality = 100;
}
internal void SaveImg(String dir , int fileNr)
{
System.IO.Directory.CreateDirectory(dir);
img.Write(dir + "/" +"s" + fileNr + ".bmp");
ClearImg();
}
internal virtual void IncrementSpinValues()
{
currLocalSpin++;
currSpin++;
}

Percolating down method for heap

I have read a file of integers into an array and I have to turn that one array into 3 separate heaps. Here is my current code but I am having trouble with a percolating down method. I have managed to create an array with the first integer n and create a single array.
For now I have only implemented the program to read from the file and based on the first value (n) read in the n values listed in the file into an array - I'm not quite sure how to make a min heap from that single array as I need 3 heaps. Example of the file is:
12 4 2 10 3 10 2 3 4 2 11 1 4 10
So 12 is n number of values in the array and 4 is # of values in each heap therefore 3 heaps.
public class realheap{
private static final int DEFAULT_CAPACITY = 10;
private int[] heap1;
private int size;
private int maxSize;
private static final int START = 1;
private static int numofval;
private static int valinheap;
public realheap(int[] array){
heap1 = new int[valinheap+1];
heap1[0] = Integer.MIN_VALUE;
size = array.length;
buildHeap();
}
public void buildHeap(){
for(int k = size/2; k > 0; k--)
{
percolatingDown(k);
}
}
public void percolatingDown(int k){
int temp = heap1[k];
int child;
for(; 2*k <= size; k = child)
{
child = 2*k;
if(child != size &&
heap1[child].compareTo(heap1[child + 1]) > 0) child++;
if(tmp.compareTo(heap[child]) > 0) heap[k] = heap[child];
else
break;
}
heap[k] = tmp;
}
/*
public static void loadFile(String file) {
try {
Scanner sc = new Scanner(new File(file));
numofval = sc.nextInt();
valinheap = sc.nextInt();
int[] ar1 = new int[numofval+1];
ar1[0] = 3;
// 3 arrays to load textfile data into arrays will be later transformed to heaps
while(sc.hasNextInt()){
// for (int i=1;i<valinheap+1;i++){
//ar1[i] = sc.nextInt(); }
for (int i=1;i<numofval+1;i++){
ar1[i] = sc.nextInt();
}
}
sc.close();
}
catch (FileNotFoundException e){
System.out.println("File not found");
}
}
*/
//locates parent of index
private int getParent(int index){
return index/2;
}
//locates index of left
private int getLeftChild(int index){
return 2*index;
}
//locates the index
private int getRightChild(int index){
return (2*index)+1;
}
private void swap(int index1,int index2){
int temp = heap1[index1];
heap1[index1] = heap1[index2];
heap1[index2] = temp;
}
/**
* #param args the command line arguments
*/
public static void main(String[] args) {
// TODO code application logic here
String files;
Scanner input= new Scanner(System.in);
System.out.println("Please enter the name of the file");
files=input.next();
input.close();
try {
Scanner sc = new Scanner(new File(files));
numofval = sc.nextInt();
valinheap = sc.nextInt();
int k = 1;
int[] ar1 = new int[numofval+1];
// 3 arrays to load textfile data into arrays will be later transformed to heaps
while(sc.hasNext()){
// for (int i=1;i<valinheap+1;i++){
//ar1[i] = sc.nextInt(); }
ar1[k] = sc.nextInt();
k++;
}
for (int i=1;i<numofval+1;i++){
System.out.println(ar1[i]);
}
sc.close();
}
catch (FileNotFoundException e){
System.out.println("File not found");
}
}
}

SystemAccessViolationException in OpenTK.dll using Shaders

Using OpenTK to attempt to create shaders in C#, repeatedly getting memory access violation errors. Any help would be appreciated. Here is the class I am having errors with, full code will be able to be found at https://github.com/Autodesk/synthesis/engine-research/OpenTKBuild
Error at Line 53 :
An unhandled exception of type 'System.AccessViolationException'
occurred in OpenTK.dll
Additional information: Attempted to read or write protected memory.
This is often an indication that other memory is corrupt.
Source code:
using OpenTK.Graphics.OpenGL;
namespace OpenTKbuild
{
class Shader
{
public int Program;
public Shader(string vertexPath, string fragmentPath)
{
string vertexCode, fragmentCode;
int vertex, fragment;
try
{
System.IO.FileStream vShaderFile = new System.IO.FileStream(vertexPath, System.IO.FileMode.Open);
System.IO.FileStream fShaderFile = new System.IO.FileStream(fragmentPath, System.IO.FileMode.Open);
System.IO.StreamReader vShaderRead = new System.IO.StreamReader(vShaderFile);
System.IO.StreamReader fShaderRead = new System.IO.StreamReader(fShaderFile);
vShaderFile.Close();
fShaderFile.Close();
vertexCode = vShaderRead.ToString();
fragmentCode = fShaderRead.ToString();
char[] vShaderCodeChar = vertexCode.ToCharArray();
char[] fShaderCodeChar = fragmentCode.ToCharArray();
string[] vShaderCode = new string[vShaderCodeChar.Length];
string[] fShaderCode = new string[fShaderCodeChar.Length];
for (int i = 0; i < vShaderCodeChar.Length; i++)
{
vShaderCode[i] = vShaderCodeChar[i].ToString();
}
for (int i = 0; i < vShaderCodeChar.Length; i++)
{
vShaderCode[i] = vShaderCodeChar[i].ToString();
}
int vcount = vShaderCode.Length;
int fcount = fShaderCode.Length;
vertex = GL.CreateShader(ShaderType.VertexShader);
GL.ShaderSource(vertex, 1, vShaderCode, ref vcount);
GL.CompileShader(vertex);
fragment = GL.CreateShader(ShaderType.FragmentShader);
// --------> error occurs onthe next line. when commented out, error does not persist.
GL.ShaderSource(fragment, 1, fShaderCode, ref fcount);
GL.CompileShader(fragment);
Program = GL.CreateProgram();
GL.AttachShader(Program, vertex);
GL.AttachShader(Program, fragment);
GL.LinkProgram(Program);
GL.DeleteShader(vertex);
GL.DeleteShader(fragment);
}
catch
{
throw new System.Exception("wut");
}
}
public void Use()
{
GL.UseProgram(Program);
}
}
}

I'm not sure why you convert the shader source string into a character array and then into a string array with just one character per string. But even if you do so, the length and the count parameter of GL.ShaderSource have to fit your data. At the moment, you tell OpenGL that there is one element in the array and that this element has fcount characters, which is not the case. This is then the reason why OpenGL tries to read outside of your allocated memory.
To solve your problem, I would suggest the following code:
GL.ShaderSource(vertex, 1, new string[] { vertexCode }, vertexCode.Length);

solved using the following code:
using OpenTK;
using OpenTK.Graphics.OpenGL;
namespace MeshImport
{
class Shader
{
public int ShaderProgram = 0;
public static int vertex, fragment, geometry;
public Shader() : base()
{
}
public Shader(string vertexPath, string fragmentPath, string geometryPath)
{
#region OLD
//string vertexCode = null, fragmentCode = null;
//System.IO.FileStream vShaderFile = new System.IO.FileStream(vertexPath, System.IO.FileMode.Open);
//System.IO.FileStream fShaderFile = new System.IO.FileStream(fragmentPath, System.IO.FileMode.Open);
#endregion
string vCode = null, fCode = null, gCode = null;
try
{
#region OLD
//System.IO.StreamReader vShaderRead = new System.IO.StreamReader(vShaderFile);
//System.IO.StreamReader fShaderRead = new System.IO.StreamReader(fShaderFile);
//vShaderFile.Close();
//fShaderFile.Close();
//vertexCode = vShaderRead.ToString();
//fragmentCode = fShaderRead.ToString();
#endregion
vCode = System.IO.File.ReadAllText(vertexPath);
fCode = System.IO.File.ReadAllText(fragmentPath);
gCode = System.IO.File.ReadAllText(geometryPath);
}
catch
{
System.Console.WriteLine("FILE NOT READ SUCCESSFULLY\n");
}
#region OLD
//char[] vShaderCodeChar = vertexCode.ToCharArray();
//char[] fShaderCodeChar = fragmentCode.ToCharArray();
//string[] vShaderCode = new string[vShaderCodeChar.Length];
//string[] fShaderCode = new string[fShaderCodeChar.Length];
//for (int i = 0; i < vShaderCodeChar.Length; i++)
//{
// vShaderCode[i] = vShaderCodeChar[i].ToString();
//}
//for (int i = 0; i < vShaderCodeChar.Length; i++)
//{
// vShaderCode[i] = vShaderCodeChar[i].ToString();
//}
#endregion
vertex = GL.CreateShader(ShaderType.VertexShader);
GL.CompileShader(vertex);
fragment = GL.CreateShader(ShaderType.FragmentShader);
GL.CompileShader(fragment);
geometry = GL.CreateShader(ShaderType.GeometryShaderExt);
GL.CompileShader(geometry);
compileShader(vertex, vCode);
compileShader(fragment, fCode);
compileShader(geometry, gCode);
ShaderProgram = GL.CreateProgram();
GL.AttachShader(ShaderProgram, vertex);
GL.AttachShader(ShaderProgram, fragment);
GL.LinkProgram(ShaderProgram);
string info;
GL.GetProgramInfoLog(ShaderProgram, out info);
System.Console.WriteLine(info);
GL.ProgramParameter(ShaderProgram, Version32.GeometryInputType, (int)All.Lines);
GL.ProgramParameter(ShaderProgram, Version32.GeometryOutputType, (int)All.LineStrip);
int tmp;
GL.GetInteger((GetPName)ExtGeometryShader4.MaxGeometryOutputVerticesExt, out tmp);
#region OLD
//int success;
//char[] infolog = new char[512];
//int[] vcount = null;
//int[] fcount = null;
//int nullint = 0;
//System.Text.StringBuilder sb = new System.Text.StringBuilder();
//sb.Append(infolog);
//vertex = GL.CreateShader(ShaderType.VertexShader);
//GL.CompileShader(vertex);
//GL.GetShader(vertex, ShaderParameter.CompileStatus, out success);
//if (success == 0)
//{
// GL.GetShaderInfoLog(vertex, 512, out nullint, sb);
// System.Console.WriteLine("Error: Shader : Vertex : Compilation Failed\n" + infolog);
//}
//fragment = GL.CreateShader(ShaderType.FragmentShader);
//GL.CompileShader(fragment);
//GL.GetShader(fragment, ShaderParameter.CompileStatus, out success);
//if (success == 0)
//{
// GL.GetShaderInfoLog(fragment, 512, out nullint, sb);
// System.Console.WriteLine("Error: Shader : Fragment : Compilation Failed\n" + infolog);
//}
//Program = GL.CreateProgram();
//GL.AttachShader(Program, vertex);
//GL.AttachShader(Program, fragment);
//GL.LinkProgram(Program);
//GL.DeleteShader(vertex);
//GL.DeleteShader(fragment);
#endregion
}
public void Use()
{
GL.UseProgram(ShaderProgram);
}
private void compileShader(int shader, string source)
{
GL.ShaderSource(shader, source);
GL.CompileShader(shader);
string info;
GL.GetShaderInfoLog(shader, out info);
System.Console.WriteLine(info);
int compileResult;
GL.GetShader(shader, ShaderParameter.CompileStatus, out compileResult);
if (compileResult != 1)
{
System.Console.WriteLine("CompileError : " + source);
}
}
public void cleanUp()
{
if (fragment != 0)
GL.DeleteShader(fragment);
if (vertex != 0)
GL.DeleteShader(vertex);
if (geometry != 0)
GL.DeleteShader(geometry);
}
}
}

Implementing a fast DBSCAN in C#

I tried to implement a DBSCAN in C# using kd-trees. I followed the implementation from:
http://www.yzuzun.com/2015/07/dbscan-clustering-algorithm-and-c-implementation/
public class DBSCANAlgorithm
{
private readonly Func<PointD, PointD, double> _metricFunc;
public DBSCANAlgorithm(Func<PointD, PointD, double> metricFunc)
{
_metricFunc = metricFunc;
}
public void ComputeClusterDbscan(ScanPoint[] allPoints, double epsilon, int minPts, out HashSet<ScanPoint[]> clusters)
{
clusters = null;
var allPointsDbscan = allPoints.Select(x => new DbscanPoint(x)).ToArray();
var tree = new KDTree.KDTree<DbscanPoint>(2);
for (var i = 0; i < allPointsDbscan.Length; ++i)
{
tree.AddPoint(new double[] { allPointsDbscan[i].ClusterPoint.point.X, allPointsDbscan[i].ClusterPoint.point.Y }, allPointsDbscan[i]);
}
var C = 0;
for (int i = 0; i < allPointsDbscan.Length; i++)
{
var p = allPointsDbscan[i];
if (p.IsVisited)
continue;
p.IsVisited = true;
DbscanPoint[] neighborPts = null;
RegionQuery(tree, p.ClusterPoint.point, epsilon, out neighborPts);
if (neighborPts.Length < minPts)
p.ClusterId = (int)ClusterIds.NOISE;
else
{
C++;
ExpandCluster(tree, p, neighborPts, C, epsilon, minPts);
}
}
clusters = new HashSet<ScanPoint[]>(
allPointsDbscan
.Where(x => x.ClusterId > 0)
.GroupBy(x => x.ClusterId)
.Select(x => x.Select(y => y.ClusterPoint).ToArray())
);
return;
}
private void ExpandCluster(KDTree.KDTree<DbscanPoint> tree, DbscanPoint p, DbscanPoint[] neighborPts, int c, double epsilon, int minPts)
{
p.ClusterId = c;
for (int i = 0; i < neighborPts.Length; i++)
{
var pn = neighborPts[i];
if (!pn.IsVisited)
{
pn.IsVisited = true;
DbscanPoint[] neighborPts2 = null;
RegionQuery(tree, pn.ClusterPoint.point, epsilon, out neighborPts2);
if (neighborPts2.Length >= minPts)
{
neighborPts = neighborPts.Union(neighborPts2).ToArray();
}
}
if (pn.ClusterId == (int)ClusterIds.UNCLASSIFIED)
pn.ClusterId = c;
}
}
private void RegionQuery(KDTree.KDTree<DbscanPoint> tree, PointD p, double epsilon, out DbscanPoint[] neighborPts)
{
int totalCount = 0;
var pIter = tree.NearestNeighbors(new double[] { p.X, p.Y }, 10, epsilon);
while (pIter.MoveNext())
{
totalCount++;
}
neighborPts = new DbscanPoint[totalCount];
int currCount = 0;
pIter.Reset();
while (pIter.MoveNext())
{
neighborPts[currCount] = pIter.Current;
currCount++;
}
return;
}
}
//Dbscan clustering identifiers
public enum ClusterIds
{
UNCLASSIFIED = 0,
NOISE = -1
}
//Point container for Dbscan clustering
public class DbscanPoint
{
public bool IsVisited;
public ScanPoint ClusterPoint;
public int ClusterId;
public DbscanPoint(ScanPoint point)
{
ClusterPoint = point;
IsVisited = false;
ClusterId = (int)ClusterIds.UNCLASSIFIED;
}
}
, and modifying the regionQuery(P, eps) to invoke the nearest neighbour function of a kd-tree. To do so, I used the kd-sharp library for C#, which is one of the fastest kd-tree implementations out there.
However, when given a dataset of about 20000 2d points, its performance is in the region of 40s, as compared to the scikit-learn python implementation of DBSCAN, which given the same parameters, takes about 2s.
Since this algorithm is for a C# program that I am writing, I am stuck using C#. As such, I would like to find out what am I still missing out in terms of optimization of the algorithm?