Develop Reference

Cannot convert between a TensorFlowLite buffer with 150528 bytes and a ByteBuffer with 602112 bytes

Im using xamarin form to create Image Classification which i got the documentation in xamarin blog.
but i got an error when passing image to bytebuffer, once the tensorflow-lite interpreter run the output i got an error saying "Cannot convert between a TensorFlowLite buffer with 150528 bytes and a ByteBuffer with 602112 bytes."
Hope someone help me in this error. im little frustrated, thank you in advance. below is my code
public class TensorflowClassifier : IClassify
{
//FloatSize is a constant with the value of 4 because a float value is 4 bytes
const int FloatSize = 4;
//PixelSize is a constant with the value of 3 because a pixel has three color channels: Red Green and Blue
const int PixelSize = 3;
public List<ImageClassificationModel> Classify(byte[] image)
{
var mappedByteBuffer = GetModelAsMappedByteBuffer();
var interpreter = new Xamarin.TensorFlow.Lite.Interpreter(mappedByteBuffer);
//To resize the image, we first need to get its required width and height
var tensor = interpreter.GetInputTensor(0);
var shape = tensor.Shape();
var width = shape[1];
var height = shape[2];
var byteBuffer = GetPhotoAsByteBuffer(image, width, height);
//use StreamReader to import the labels from labels.txt
var streamReader = new StreamReader(Application.Context.Assets.Open("labels.txt"));
//Transform labels.txt into List<string>
var labels = streamReader.ReadToEnd().Split('\n').Select(s => s.Trim()).Where(s => !string.IsNullOrEmpty(s)).ToList();
//Convert our two-dimensional array into a Java.Lang.Object, the required input for Xamarin.TensorFlow.List.Interpreter
var outputLocations = new float[1][] { new float[labels.Count] };
var outputs = Java.Lang.Object.FromArray(outputLocations);
interpreter.Run(byteBuffer, outputs);
var classificationResult = outputs.ToArray<float[]>();
//Map the classificationResult to the labels and sort the result to find which label has the highest probability
var classificationModelList = new List<ImageClassificationModel>();
for (var i = 0; i < labels.Count; i++)
{
var label = labels[i]; classificationModelList.Add(new ImageClassificationModel(label, classificationResult[0][i]));
}
return classificationModelList;
}
//Convert model.tflite to Java.Nio.MappedByteBuffer , the require type for Xamarin.TensorFlow.Lite.Interpreter
private MappedByteBuffer GetModelAsMappedByteBuffer()
{
var assetDescriptor = Application.Context.Assets.OpenFd("model.tflite");
var inputStream = new FileInputStream(assetDescriptor.FileDescriptor);
var mappedByteBuffer = inputStream.Channel.Map(FileChannel.MapMode.ReadOnly, assetDescriptor.StartOffset, assetDescriptor.DeclaredLength);
return mappedByteBuffer;
}
//Resize the image for the TensorFlow interpreter
private ByteBuffer GetPhotoAsByteBuffer(byte[] image, int width, int height)
{
var bitmap = BitmapFactory.DecodeByteArray(image, 0, image.Length);
var resizedBitmap = Bitmap.CreateScaledBitmap(bitmap, width, height, true);
var modelInputSize = FloatSize * height * width * PixelSize;
var byteBuffer = ByteBuffer.AllocateDirect(modelInputSize);
byteBuffer.Order(ByteOrder.NativeOrder());
var pixels = new int[width * height];
resizedBitmap.GetPixels(pixels, 0, resizedBitmap.Width, 0, 0, resizedBitmap.Width, resizedBitmap.Height);
var pixel = 0;
//Loop through each pixels to create a Java.Nio.ByteBuffer
for (var i = 0; i < width; i++)
{
for (var j = 0; j < height; j++)
{
var pixelVal = pixels[pixel++];
byteBuffer.PutFloat(pixelVal >> 16 & 0xFF / 4);
byteBuffer.PutFloat(pixelVal >> 8 & 0xFF / 4);
byteBuffer.PutFloat(pixelVal & 0xFF / 4);
}
}
bitmap.Recycle();
return byteBuffer;
}
}
On my project i passed the byte array to the classifier
private async void SaveImagePad(object sender, EventArgs e)
{
Stream image = await PadView.GetImageStreamAsync(SignatureImageFormat.Jpeg);
//byte[] bytes = ReadFully(image);
byte[] bytes = new byte[image.Length];
image.Read(bytes, 0, bytes.Length);
List<ImageClassificationModel> classifyImage = DependencyService.Get<IClassify>().Classify(bytes);
}

Reading Datastream sharpDX Error all values are 0

I followed this solution for my project : How to create bitmap from Surface (SharpDX)
I don't have enough reputation to comment so I'm opening a new question here.
My project is basically in Direct 2D, I have a Surface buffer, a swapchain. I want to put my buffer into a datastream and reads it's value to put it into a bitmap and save it on disk ( like a screen capture), but my code won't work since all the bytes values are 0 (which is black) and this doesn't make sense since my image is fully white with a bit of blue.
Here is my code :
SwapChainDescription description = new SwapChainDescription()
{
ModeDescription = new ModeDescription(this.Width, this.Height, new Rational(60, 1), Format.B8G8R8A8_UNorm),
SampleDescription = new SampleDescription(1, 0),
Usage = Usage.RenderTargetOutput,
BufferCount = 1,
SwapEffect = SwapEffect.Discard,
IsWindowed = true,
OutputHandle = this.Handle
};
Device.CreateWithSwapChain(DriverType.Hardware, DeviceCreationFlags.Debug | DeviceCreationFlags.BgraSupport, description, out device, out swapChain);
SharpDX.DXGI.Device dxgiDevice = device.QueryInterface<SharpDX.DXGI.Device>();
SharpDX.DXGI.Adapter dxgiAdapter = dxgiDevice.Adapter;
SharpDX.Direct2D1.Device d2dDevice = new SharpDX.Direct2D1.Device(dxgiDevice);
d2dContext = new SharpDX.Direct2D1.DeviceContext(d2dDevice, SharpDX.Direct2D1.DeviceContextOptions.None);
SharpDX.Direct3D11.DeviceContext d3DeviceContext = new SharpDX.Direct3D11.DeviceContext(device);
properties = new BitmapProperties(new SharpDX.Direct2D1.PixelFormat(SharpDX.DXGI.Format.B8G8R8A8_UNorm, SharpDX.Direct2D1.AlphaMode.Premultiplied),
96, 96);
Surface backBuffer = swapChain.GetBackBuffer<Surface>(0);
d2dTarget = new SharpDX.Direct2D1.Bitmap(d2dContext, backBuffer, properties);
d2dContext.Target = d2dTarget;
playerBitmap = this.LoadBitmapFromContentFile(#"C:\Users\ndesjardins\Desktop\wave.png");
//System.Drawing.Bitmap bitmapCanva = new System.Drawing.Bitmap(1254, 735);
d2dContext.BeginDraw();
d2dContext.Clear(SharpDX.Color.White);
d2dContext.DrawBitmap(playerBitmap, new SharpDX.RectangleF(0, 0, playerBitmap.Size.Width, playerBitmap.Size.Height), 1f, SharpDX.Direct2D1.BitmapInterpolationMode.NearestNeighbor);
SharpDX.Direct2D1.SolidColorBrush brush = new SharpDX.Direct2D1.SolidColorBrush(d2dContext, SharpDX.Color.Green);
d2dContext.DrawRectangle(new SharpDX.RectangleF(200, 200, 100, 100), brush);
d2dContext.EndDraw();
swapChain.Present(1, PresentFlags.None);
Texture2D backBuffer3D = backBuffer.QueryInterface<SharpDX.Direct3D11.Texture2D>();
Texture2DDescription desc = backBuffer3D.Description;
desc.CpuAccessFlags = CpuAccessFlags.Read;
desc.Usage = ResourceUsage.Staging;
desc.OptionFlags = ResourceOptionFlags.None;
desc.BindFlags = BindFlags.None;
var texture = new Texture2D(device, desc);
d3DeviceContext.CopyResource(backBuffer3D, texture);
byte[] data = null;
using (Surface surface = texture.QueryInterface<Surface>())
{
DataStream dataStream;
var map = surface.Map(SharpDX.DXGI.MapFlags.Read, out dataStream);
int lines = (int)(dataStream.Length / map.Pitch);
data = new byte[surface.Description.Width * surface.Description.Height * 4];
dataStream.Position = 0;
int dataCounter = 0;
// width of the surface - 4 bytes per pixel.
int actualWidth = surface.Description.Width * 4;
for (int y = 0; y < lines; y++)
{
for (int x = 0; x < map.Pitch; x++)
{
if (x < actualWidth)
{
data[dataCounter++] = dataStream.Read<byte>();
}
else
{
dataStream.Read<byte>();
}
}
}
dataStream.Dispose();
surface.Unmap();
int width = surface.Description.Width;
int height = surface.Description.Height;
byte[] bytewidth = BitConverter.GetBytes(width);
byte[] byteheight = BitConverter.GetBytes(height);
Array.Copy(bytewidth, 0, data, 0, 4);
Array.Copy(byteheight, 0, data, 4, 4);
}
Do you guys have any idea why the byte array that is returned at the end is full of 0 since it should be mostly 255? All I did in my backbuffer was to draw a bitmap image and a rectangle form. Array.Copy is to add the width and height header to the byte array, therefore I could create a bitmap out of it.

I answered in a comment but formatting is horrible so apologies!
https://gamedev.stackexchange.com/a/112978/29920 This looks promising but as you said in reply to mine, this was some time ago and I'm pulling this out of thin air, if it doesn't work either someone with more current knowledge will have to answer or I'll have to grab some source code and try myself.
SharpDX.Direct2D1.Bitmap dxbmp = new SharpDX.Direct2D1.Bitmap(renderTarget,
new SharpDX.Size2(bmpWidth, bmpHeight), new
BitmapProperties(renderTarget.PixelFormat));
dxbmp.CopyFromMemory(bmpBits, bmpWidth * 4);
This looks kind of like what you need. I'm assuming bmpBits in here is either a byte array or a memory stream either of which could then be saved off or at least give you something to look at to see if you're actually getting pixel data

A generic error occurred in GDI+, Getting this exception in my code when i try to save optimized image on server

Here is my code below which i am using to redraw and save the image.
Bitmap bitmap = new Bitmap(HttpContext.Current.Server.MapPath(
filePath + Path.GetFileName(fileName)));
int newWidth = 100;
int newHeight = 100;
int iwidth = bitmap.Width;
int iheight = bitmap.Height;
if (iwidth <= 100)
newWidth = iwidth;
if (iheight <= 100)
newHeight = iheight;
bitmap.Dispose();
// ONCE WE GOT ALL THE INFORMATION, WE'll NOW PROCESS IT.
// CREATE AN IMAGE OBJECT USING ORIGINAL WIDTH AND HEIGHT.
// ALSO DEFINE A PIXEL FORMAT (FOR RICH RGB COLOR).
System.Drawing.Image objOptImage = new System.Drawing.Bitmap(newWidth, newHeight,
System.Drawing.Imaging.PixelFormat.Format16bppRgb555);
// GET THE ORIGINAL IMAGE.
using (System.Drawing.Image objImg =
System.Drawing.Image.FromFile(HttpContext.Current.Server.MapPath(filePath + fileName)))
{
// RE-DRAW THE IMAGE USING THE NEWLY OBTAINED PIXEL FORMAT.
using (System.Drawing.Graphics oGraphic = System.Drawing.Graphics.FromImage(objOptImage))
{
var _1 = oGraphic;
System.Drawing.Rectangle oRectangle = new System.Drawing.Rectangle(0, 0, newWidth, newHeight);
_1.DrawImage(objImg, oRectangle);
}
// On the below lines of code i am getting error while i try to execute
// this code after uploading image. If image size is around 10 kb it will
// not throw any error but if it exceeds then 10 kb then it will.
//SAVE THE OPTIMIZED IMAGE
objOptImage.Save(HttpContext.Current.Server.MapPath(filePath + "TempSubImages/" + fileName),
System.Drawing.Imaging.ImageFormat.Png);**
objImg.Dispose();
}
objOptImage.Dispose();
// FINALLY SHOW THE OPTIMIZED IMAGE DETAILS WITH SIZE.
Bitmap bitmap_Opt = new Bitmap(HttpContext.Current.Server.MapPath(filePath + "TempSubImages/" + Path.GetFileName(fileName)));
byte[] bytes = System.IO.File.ReadAllBytes(HttpContext.Current.Server.MapPath(filePath + "TempSubImages/" + Path.GetFileName(fileName)));
UI.BasePage.TemporaryFile = bytes;
String[] ArrContentType = new String[] { postedFile.ContentType, fileName };
UI.BasePage.TemporaryStrings = ArrContentType;
int iwidth_Opt = bitmap_Opt.Width;
int iheight_Opt = bitmap_Opt.Height;
bitmap_Opt.Dispose();

FileLabels are not visible

I try to capture desktop screenshot using SharpDX. My application is able to capture screenshot but without labels in Windows Explorer.
I tryed 2 solutions but without change. I tried find in documentation any information, but without change.
Here is mi code:
public void SCR()
{
uint numAdapter = 0; // # of graphics card adapter
uint numOutput = 0; // # of output device (i.e. monitor)
// create device and factory
var device = new SharpDX.Direct3D11.Device(SharpDX.Direct3D.DriverType.Hardware);
var factory = new Factory1();
// creating CPU-accessible texture resource
var texdes = new SharpDX.Direct3D11.Texture2DDescription
{
CpuAccessFlags = SharpDX.Direct3D11.CpuAccessFlags.Read,
BindFlags = SharpDX.Direct3D11.BindFlags.None,
Format = Format.B8G8R8A8_UNorm,
Height = factory.Adapters1[numAdapter].Outputs[numOutput].Description.DesktopBounds.Height,
Width = factory.Adapters1[numAdapter].Outputs[numOutput].Description.DesktopBounds.Width,
OptionFlags = SharpDX.Direct3D11.ResourceOptionFlags.None,
MipLevels = 1,
ArraySize = 1
};
texdes.SampleDescription.Count = 1;
texdes.SampleDescription.Quality = 0;
texdes.Usage = SharpDX.Direct3D11.ResourceUsage.Staging;
var screenTexture = new SharpDX.Direct3D11.Texture2D(device, texdes);
// duplicate output stuff
var output = new Output1(factory.Adapters1[numAdapter].Outputs[numOutput].NativePointer);
var duplicatedOutput = output.DuplicateOutput(device);
SharpDX.DXGI.Resource screenResource = null;
SharpDX.DataStream dataStream;
Surface screenSurface;
var i = 0;
var miliseconds = 2500000;
while (true)
{
i++;
// try to get duplicated frame within given time
try
{
SharpDX.DXGI.OutputDuplicateFrameInformation duplicateFrameInformation;
duplicatedOutput.AcquireNextFrame(miliseconds, out duplicateFrameInformation, out screenResource);
}
catch (SharpDX.SharpDXException e)
{
if (e.ResultCode.Code == SharpDX.DXGI.ResultCode.WaitTimeout.Result.Code)
{
// this has not been a successful capture
// thanks #Randy
// keep retrying
continue;
}
else
{
throw e;
}
}
device.ImmediateContext.CopyResource(screenResource.QueryInterface<SharpDX.Direct3D11.Resource>(), screenTexture);
screenSurface = screenTexture.QueryInterface<Surface>();
// screenSurface.Map(SharpDX.DXGI.MapFlags.Read, out dataStream);
int width = output.Description.DesktopBounds.Width;
int height = output.Description.DesktopBounds.Height;
var boundsRect = new System.Drawing.Rectangle(0, 0, width, height);
var mapSource = device.ImmediateContext.MapSubresource(screenTexture, 0, MapMode.Read, SharpDX.Direct3D11.MapFlags.None);
using (var bitmap = new System.Drawing.Bitmap(width, height, PixelFormat.Format32bppArgb))
{
// Copy pixels from screen capture Texture to GDI bitmap
var bitmapData = bitmap.LockBits(boundsRect, ImageLockMode.WriteOnly, bitmap.PixelFormat);
var sourcePtr = mapSource.DataPointer;
var destinationPtr = bitmapData.Scan0;
for (int y = 0; y < height; y++)
{
// Copy a single line
Utilities.CopyMemory(destinationPtr, sourcePtr, width * 4);
// Advance pointers
sourcePtr = IntPtr.Add(sourcePtr, mapSource.RowPitch);
destinationPtr = IntPtr.Add(destinationPtr, bitmapData.Stride);
}
// Release source and dest locks
bitmap.UnlockBits(bitmapData);
device.ImmediateContext.UnmapSubresource(screenTexture, 0);
bitmap.Save(string.Format(#"d:\scr\{0}.png", i));
}
// var image = FromByte(ToByte(dataStream));
//var image = getImageFromDXStream(1920, 1200, dataStream);
//image.Save(string.Format(#"d:\scr\{0}.png", i));
// dataStream.Close();
//screenSurface.Unmap();
screenSurface.Dispose();
screenResource.Dispose();
duplicatedOutput.ReleaseFrame();
}
}

After few hours of research and googling i found working solution:
From:
PixelFormat.Format32bppArgb
To:
PixelFormat.Format32bppRgb

How can I find a template image in another image? Kinect and AForge preferred

I copied the AForge-Sample from here:
http://www.aforgenet.com/framework/features/template_matching.html
And hoped, it would work with 2 Bitmaps as sources as in the following code:
Bitmap findTemplate (Bitmap sourceImage, Bitmap template)
{
// create template matching algorithm's instance
// (set similarity threshold to x.y%, 1.0f = 100%)
ExhaustiveTemplateMatching tm = new ExhaustiveTemplateMatching( 0.4f );
// find all matchings with specified above similarity
TemplateMatch[] matchings = tm.ProcessImage( sourceImage, template ); **// "Unsupported pixel format of the source or template image." as error message**
// highlight found matchings
BitmapData data = sourceImage.LockBits(
new Rectangle( 0, 0, sourceImage.Width, sourceImage.Height ),
ImageLockMode.ReadWrite, sourceImage.PixelFormat );
foreach ( TemplateMatch m in matchings )
{
AForge.Imaging.Drawing.Rectangle( data, m.Rectangle, System.Drawing.Color.White );
// do something else with matching
}
sourceImage.UnlockBits( data );
return sourceImage;
}
But when calling TemplateMatch[] matchings = tm.P.... it gives the error mentioned above.
The template is generated this way:
Bitmap templatebitmap=(Bitmap)AForge.Imaging.Image.FromFile("template.jpg");
the source is generated with the kinect-webcam, where the PlanarImage is formatted as Bitmap (method copied from somewhere, but it was working up to now)
Bitmap PImageToBitmap(PlanarImage PImage)
{
Bitmap bmap = new Bitmap(
PImage.Width,
PImage.Height,
System.Drawing.Imaging.PixelFormat.Format32bppRgb);
BitmapData bmapdata = bmap.LockBits(
new Rectangle(0, 0, PImage.Width,
PImage.Height),
ImageLockMode.WriteOnly,
bmap.PixelFormat);
IntPtr ptr = bmapdata.Scan0;
Marshal.Copy(PImage.Bits,
0,
ptr,
PImage.Width *
PImage.BytesPerPixel *
PImage.Height);
bmap.UnlockBits(bmapdata);
return bmap;
}
So, is anbody able to help me, where my mistake might be?
Or maybe anyone knows a better way to match a template with a Kinect?
The overall job is to detect a known object with the kinect, in my case a rubberduck.
Thank you in advamce.

here's the solution using AForge
but it is slow take around 5 seconds but it works
As usuall u need to introduce AForge framework download and install it.
specify using AForge namespace
and copy paste to make it work
System.Drawing.Bitmap sourceImage = (Bitmap)Bitmap.FromFile(#"C:\SavedBMPs\1.jpg");
System.Drawing.Bitmap template = (Bitmap)Bitmap.FromFile(#"C:\SavedBMPs\2.jpg");
// create template matching algorithm's instance
// (set similarity threshold to 92.5%)
ExhaustiveTemplateMatching tm = new ExhaustiveTemplateMatching(0.921f);
// find all matchings with specified above similarity
TemplateMatch[] matchings = tm.ProcessImage(sourceImage, template);
// highlight found matchings
BitmapData data = sourceImage.LockBits(
new Rectangle(0, 0, sourceImage.Width, sourceImage.Height),
ImageLockMode.ReadWrite, sourceImage.PixelFormat);
foreach (TemplateMatch m in matchings)
{
Drawing.Rectangle(data, m.Rectangle, Color.White);
MessageBox.Show(m.Rectangle.Location.ToString());
// do something else with matching
}
sourceImage.UnlockBits(data);

So, I just implemented it myself. But it is so slow - so if anyone has an idea to improve, feel free to criticize my code:
public class Position
{
public int bestRow { get; set; }
public int bestCol { get; set; }
public double bestSAD { get; set; }
public Position(int row, int col, double sad)
{
bestRow = row;
bestCol = col;
bestSAD = sad;
}
}
Position element_position = new Position(0, 0, double.PositiveInfinity);
Position ownSearch(Bitmap search, Bitmap template) {
Position position = new Position(0,0,double.PositiveInfinity);
double minSAD = double.PositiveInfinity;
// loop through the search image
for (int x = 0; x <= search.PhysicalDimension.Width - template.PhysicalDimension.Width; x++)
{
for (int y = 0; y <= search.PhysicalDimension.Height - template.PhysicalDimension.Height; y++)
{
position_label2.Content = "Running: X=" + x + " Y=" + y;
double SAD = 0.0;
// loop through the template image
for (int i = 0; i < template.PhysicalDimension.Width; i++)
{
for (int j = 0; j < template.PhysicalDimension.Height; j++)
{
int r = Math.Abs(search.GetPixel(x + i, y + j).R - template.GetPixel(i, j).R);
int g = Math.Abs(search.GetPixel(x + i, y + j).G - template.GetPixel(i, j).G);
int b = Math.Abs(search.GetPixel(x + i, y + j).B - template.GetPixel(i, j).B);
int a = template.GetPixel(i, j).A;
SAD = SAD + ((r + g + b)*a/255 );
}
}
// save the best found position
if (minSAD > SAD)
{
minSAD = SAD;
// give me VALUE_MAX
position.bestRow = x;
position.bestCol = y;
position.bestSAD = SAD;
}
}
}
return position;
}