Develop Reference

Cannot convert between a TensorFlowLite buffer with 150528 bytes and a ByteBuffer with 602112 bytes

Im using xamarin form to create Image Classification which i got the documentation in xamarin blog.
but i got an error when passing image to bytebuffer, once the tensorflow-lite interpreter run the output i got an error saying "Cannot convert between a TensorFlowLite buffer with 150528 bytes and a ByteBuffer with 602112 bytes."
Hope someone help me in this error. im little frustrated, thank you in advance. below is my code
public class TensorflowClassifier : IClassify
{
//FloatSize is a constant with the value of 4 because a float value is 4 bytes
const int FloatSize = 4;
//PixelSize is a constant with the value of 3 because a pixel has three color channels: Red Green and Blue
const int PixelSize = 3;
public List<ImageClassificationModel> Classify(byte[] image)
{
var mappedByteBuffer = GetModelAsMappedByteBuffer();
var interpreter = new Xamarin.TensorFlow.Lite.Interpreter(mappedByteBuffer);
//To resize the image, we first need to get its required width and height
var tensor = interpreter.GetInputTensor(0);
var shape = tensor.Shape();
var width = shape[1];
var height = shape[2];
var byteBuffer = GetPhotoAsByteBuffer(image, width, height);
//use StreamReader to import the labels from labels.txt
var streamReader = new StreamReader(Application.Context.Assets.Open("labels.txt"));
//Transform labels.txt into List<string>
var labels = streamReader.ReadToEnd().Split('\n').Select(s => s.Trim()).Where(s => !string.IsNullOrEmpty(s)).ToList();
//Convert our two-dimensional array into a Java.Lang.Object, the required input for Xamarin.TensorFlow.List.Interpreter
var outputLocations = new float[1][] { new float[labels.Count] };
var outputs = Java.Lang.Object.FromArray(outputLocations);
interpreter.Run(byteBuffer, outputs);
var classificationResult = outputs.ToArray<float[]>();
//Map the classificationResult to the labels and sort the result to find which label has the highest probability
var classificationModelList = new List<ImageClassificationModel>();
for (var i = 0; i < labels.Count; i++)
{
var label = labels[i]; classificationModelList.Add(new ImageClassificationModel(label, classificationResult[0][i]));
}
return classificationModelList;
}
//Convert model.tflite to Java.Nio.MappedByteBuffer , the require type for Xamarin.TensorFlow.Lite.Interpreter
private MappedByteBuffer GetModelAsMappedByteBuffer()
{
var assetDescriptor = Application.Context.Assets.OpenFd("model.tflite");
var inputStream = new FileInputStream(assetDescriptor.FileDescriptor);
var mappedByteBuffer = inputStream.Channel.Map(FileChannel.MapMode.ReadOnly, assetDescriptor.StartOffset, assetDescriptor.DeclaredLength);
return mappedByteBuffer;
}
//Resize the image for the TensorFlow interpreter
private ByteBuffer GetPhotoAsByteBuffer(byte[] image, int width, int height)
{
var bitmap = BitmapFactory.DecodeByteArray(image, 0, image.Length);
var resizedBitmap = Bitmap.CreateScaledBitmap(bitmap, width, height, true);
var modelInputSize = FloatSize * height * width * PixelSize;
var byteBuffer = ByteBuffer.AllocateDirect(modelInputSize);
byteBuffer.Order(ByteOrder.NativeOrder());
var pixels = new int[width * height];
resizedBitmap.GetPixels(pixels, 0, resizedBitmap.Width, 0, 0, resizedBitmap.Width, resizedBitmap.Height);
var pixel = 0;
//Loop through each pixels to create a Java.Nio.ByteBuffer
for (var i = 0; i < width; i++)
{
for (var j = 0; j < height; j++)
{
var pixelVal = pixels[pixel++];
byteBuffer.PutFloat(pixelVal >> 16 & 0xFF / 4);
byteBuffer.PutFloat(pixelVal >> 8 & 0xFF / 4);
byteBuffer.PutFloat(pixelVal & 0xFF / 4);
}
}
bitmap.Recycle();
return byteBuffer;
}
}
On my project i passed the byte array to the classifier
private async void SaveImagePad(object sender, EventArgs e)
{
Stream image = await PadView.GetImageStreamAsync(SignatureImageFormat.Jpeg);
//byte[] bytes = ReadFully(image);
byte[] bytes = new byte[image.Length];
image.Read(bytes, 0, bytes.Length);
List<ImageClassificationModel> classifyImage = DependencyService.Get<IClassify>().Classify(bytes);
}

Create movie from images c# using leadTools

I am trying to create movie from images.
I am following following links :
https://www.leadtools.com/support/forum/posts/t11084- // Here I am trying option 2 mentioned & https://social.msdn.microsoft.com/Forums/sqlserver/en-US/b61726a4-4b87-49c7-b4fc-8949cd1366ac/visual-c-visual-studio-2017-how-do-you-convert-jpg-images-to-video-in-visual-c?forum=csharpgeneral
void convert()
{
bmp = new Bitmap(320, 240, System.Drawing.Imaging.PixelFormat.Format24bppRgb);
// create sample source object
SampleSource smpsrc = new SampleSource();
ConvertCtrl convertCtrl = new ConvertCtrl();
// create a new media type wrapper
MediaType mt = new MediaType();
double AvgTimePerFrame = (10000000 / 15);
// set the type to 24-bit RGB video
mt.Type = Constants.MEDIATYPE_Video;
mt.SubType = Constants.MEDIASUBTYPE_RGB24;
// set the format
mt.FormatType = Constants.FORMAT_VideoInfo;
VideoInfoHeader vih = new VideoInfoHeader();
int bmpSize = GetBitmapSize(bmp);
// setup the video info header
vih.bmiHeader.biCompression = 0; // BI_RGB
vih.bmiHeader.biBitCount = 24;
vih.bmiHeader.biWidth = bmp.Width;
vih.bmiHeader.biHeight = bmp.Height;
vih.bmiHeader.biPlanes = 1;
vih.bmiHeader.biSizeImage = bmpSize;
vih.bmiHeader.biClrImportant = 0;
vih.AvgTimePerFrame.lowpart = (int)AvgTimePerFrame;
vih.dwBitRate = bmpSize * 8 * 15;
mt.SetVideoFormatData(vih, null, 0);
// set fixed size samples matching the bitmap size
mt.SampleSize = bmpSize;
mt.FixedSizeSamples = true;
// assign the source media type
smpsrc.SetMediaType(mt);
// select the LEAD compressor
convertCtrl.VideoCompressors.MCmpMJpeg.Selected = true;
convertCtrl.SourceObject = smpsrc;
convertCtrl.TargetFile = #"D:\Projects\LEADTool_Movie_fromImage\ImageToVideo_LeadTool\ImageToVideo_LeadTool\Images\Out\aa.avi";
//convertCtrl.TargetFile = "C:\\Users\\vipul.langalia\\Documents\\count.avi";
convertCtrl.TargetFormat = TargetFormatType.WMVMux;
convertCtrl.StartConvert();
BitmapData bmpData;
int i = 1;
byte[] a = new byte[bmpSize];
System.Drawing.Rectangle rect = new System.Drawing.Rectangle(0, 0, bmp.Width, bmp.Height);
var imgs = GetAllFiles();
foreach (var item in imgs)
{
bmpSize = GetBitmapSize(item);
MediaSample ms = smpsrc.GetSampleBuffer(30000);
ms.SyncPoint = true;
bmpData = item.LockBits(rect, ImageLockMode.ReadWrite, item.PixelFormat);
Marshal.Copy(bmpData.Scan0, a, 0, bmpSize);
item.UnlockBits(bmpData);
ms.SetData(bmpSize, a);
SetSampleTime(ms, i, AvgTimePerFrame);
smpsrc.DeliverSample(1000, ms);
i++;
}
smpsrc.DeliverEndOfStream(1000);
}
byte[] GetByteArrayFroMWritableBitmap(WriteableBitmap bitmapSource)
{
var width = bitmapSource.PixelWidth;
var height = bitmapSource.PixelHeight;
var stride = width * ((bitmapSource.Format.BitsPerPixel + 7) / 8);
var bitmapData = new byte[height * stride];
bitmapSource.CopyPixels(bitmapData, stride, 0);
return bitmapData;
}
private int GetBitmapSize(WriteableBitmap bmp)
{
int BytesPerLine = (((int)bmp.Width * 24 + 31) & ~31) / 8;
return BytesPerLine * (int)bmp.Height;
}
private int GetBitmapSize(Bitmap bmp)
{
int BytesPerLine = ((bmp.Width * 24 + 31) & ~31) / 8;
return BytesPerLine * bmp.Height;
}
It is throwing out of memory exception when execute ms.SetData(bmpSize, a); statement. Plus If I directly pass byte[] by var a = System.IO.File.ReadAllBytes(imagePath); in ms.SetData(bmpSize, a); statement then it will not throw error but video file is not properly created.
Can anybody please help me?

There are a couple of problems with your code:
Are all your images 320x240 pixels? If not, you should resize them to these exact dimensions before delivering them as video samples to the Convert control.
If you want to use a different size, you can, but it should be the same size for all images, and you should modify the code accordingly.
You are setting the TargetFormat property to WMVMux, but the name of the output file has “.avi” extension. If you want to save AVI files, set TargetFormat = TargetFormatType.AVI.
If you still face problems after this, feel free to contact support#leadtools.com and provide full details about what you tried and what errors you’re getting. Email support is free for LEADTOOLS SDK owners and also for free evaluation users.

FileLabels are not visible

I try to capture desktop screenshot using SharpDX. My application is able to capture screenshot but without labels in Windows Explorer.
I tryed 2 solutions but without change. I tried find in documentation any information, but without change.
Here is mi code:
public void SCR()
{
uint numAdapter = 0; // # of graphics card adapter
uint numOutput = 0; // # of output device (i.e. monitor)
// create device and factory
var device = new SharpDX.Direct3D11.Device(SharpDX.Direct3D.DriverType.Hardware);
var factory = new Factory1();
// creating CPU-accessible texture resource
var texdes = new SharpDX.Direct3D11.Texture2DDescription
{
CpuAccessFlags = SharpDX.Direct3D11.CpuAccessFlags.Read,
BindFlags = SharpDX.Direct3D11.BindFlags.None,
Format = Format.B8G8R8A8_UNorm,
Height = factory.Adapters1[numAdapter].Outputs[numOutput].Description.DesktopBounds.Height,
Width = factory.Adapters1[numAdapter].Outputs[numOutput].Description.DesktopBounds.Width,
OptionFlags = SharpDX.Direct3D11.ResourceOptionFlags.None,
MipLevels = 1,
ArraySize = 1
};
texdes.SampleDescription.Count = 1;
texdes.SampleDescription.Quality = 0;
texdes.Usage = SharpDX.Direct3D11.ResourceUsage.Staging;
var screenTexture = new SharpDX.Direct3D11.Texture2D(device, texdes);
// duplicate output stuff
var output = new Output1(factory.Adapters1[numAdapter].Outputs[numOutput].NativePointer);
var duplicatedOutput = output.DuplicateOutput(device);
SharpDX.DXGI.Resource screenResource = null;
SharpDX.DataStream dataStream;
Surface screenSurface;
var i = 0;
var miliseconds = 2500000;
while (true)
{
i++;
// try to get duplicated frame within given time
try
{
SharpDX.DXGI.OutputDuplicateFrameInformation duplicateFrameInformation;
duplicatedOutput.AcquireNextFrame(miliseconds, out duplicateFrameInformation, out screenResource);
}
catch (SharpDX.SharpDXException e)
{
if (e.ResultCode.Code == SharpDX.DXGI.ResultCode.WaitTimeout.Result.Code)
{
// this has not been a successful capture
// thanks #Randy
// keep retrying
continue;
}
else
{
throw e;
}
}
device.ImmediateContext.CopyResource(screenResource.QueryInterface<SharpDX.Direct3D11.Resource>(), screenTexture);
screenSurface = screenTexture.QueryInterface<Surface>();
// screenSurface.Map(SharpDX.DXGI.MapFlags.Read, out dataStream);
int width = output.Description.DesktopBounds.Width;
int height = output.Description.DesktopBounds.Height;
var boundsRect = new System.Drawing.Rectangle(0, 0, width, height);
var mapSource = device.ImmediateContext.MapSubresource(screenTexture, 0, MapMode.Read, SharpDX.Direct3D11.MapFlags.None);
using (var bitmap = new System.Drawing.Bitmap(width, height, PixelFormat.Format32bppArgb))
{
// Copy pixels from screen capture Texture to GDI bitmap
var bitmapData = bitmap.LockBits(boundsRect, ImageLockMode.WriteOnly, bitmap.PixelFormat);
var sourcePtr = mapSource.DataPointer;
var destinationPtr = bitmapData.Scan0;
for (int y = 0; y < height; y++)
{
// Copy a single line
Utilities.CopyMemory(destinationPtr, sourcePtr, width * 4);
// Advance pointers
sourcePtr = IntPtr.Add(sourcePtr, mapSource.RowPitch);
destinationPtr = IntPtr.Add(destinationPtr, bitmapData.Stride);
}
// Release source and dest locks
bitmap.UnlockBits(bitmapData);
device.ImmediateContext.UnmapSubresource(screenTexture, 0);
bitmap.Save(string.Format(#"d:\scr\{0}.png", i));
}
// var image = FromByte(ToByte(dataStream));
//var image = getImageFromDXStream(1920, 1200, dataStream);
//image.Save(string.Format(#"d:\scr\{0}.png", i));
// dataStream.Close();
//screenSurface.Unmap();
screenSurface.Dispose();
screenResource.Dispose();
duplicatedOutput.ReleaseFrame();
}
}

After few hours of research and googling i found working solution:
From:
PixelFormat.Format32bppArgb
To:
PixelFormat.Format32bppRgb

How can I find a template image in another image? Kinect and AForge preferred

I copied the AForge-Sample from here:
http://www.aforgenet.com/framework/features/template_matching.html
And hoped, it would work with 2 Bitmaps as sources as in the following code:
Bitmap findTemplate (Bitmap sourceImage, Bitmap template)
{
// create template matching algorithm's instance
// (set similarity threshold to x.y%, 1.0f = 100%)
ExhaustiveTemplateMatching tm = new ExhaustiveTemplateMatching( 0.4f );
// find all matchings with specified above similarity
TemplateMatch[] matchings = tm.ProcessImage( sourceImage, template ); **// "Unsupported pixel format of the source or template image." as error message**
// highlight found matchings
BitmapData data = sourceImage.LockBits(
new Rectangle( 0, 0, sourceImage.Width, sourceImage.Height ),
ImageLockMode.ReadWrite, sourceImage.PixelFormat );
foreach ( TemplateMatch m in matchings )
{
AForge.Imaging.Drawing.Rectangle( data, m.Rectangle, System.Drawing.Color.White );
// do something else with matching
}
sourceImage.UnlockBits( data );
return sourceImage;
}
But when calling TemplateMatch[] matchings = tm.P.... it gives the error mentioned above.
The template is generated this way:
Bitmap templatebitmap=(Bitmap)AForge.Imaging.Image.FromFile("template.jpg");
the source is generated with the kinect-webcam, where the PlanarImage is formatted as Bitmap (method copied from somewhere, but it was working up to now)
Bitmap PImageToBitmap(PlanarImage PImage)
{
Bitmap bmap = new Bitmap(
PImage.Width,
PImage.Height,
System.Drawing.Imaging.PixelFormat.Format32bppRgb);
BitmapData bmapdata = bmap.LockBits(
new Rectangle(0, 0, PImage.Width,
PImage.Height),
ImageLockMode.WriteOnly,
bmap.PixelFormat);
IntPtr ptr = bmapdata.Scan0;
Marshal.Copy(PImage.Bits,
0,
ptr,
PImage.Width *
PImage.BytesPerPixel *
PImage.Height);
bmap.UnlockBits(bmapdata);
return bmap;
}
So, is anbody able to help me, where my mistake might be?
Or maybe anyone knows a better way to match a template with a Kinect?
The overall job is to detect a known object with the kinect, in my case a rubberduck.
Thank you in advamce.

here's the solution using AForge
but it is slow take around 5 seconds but it works
As usuall u need to introduce AForge framework download and install it.
specify using AForge namespace
and copy paste to make it work
System.Drawing.Bitmap sourceImage = (Bitmap)Bitmap.FromFile(#"C:\SavedBMPs\1.jpg");
System.Drawing.Bitmap template = (Bitmap)Bitmap.FromFile(#"C:\SavedBMPs\2.jpg");
// create template matching algorithm's instance
// (set similarity threshold to 92.5%)
ExhaustiveTemplateMatching tm = new ExhaustiveTemplateMatching(0.921f);
// find all matchings with specified above similarity
TemplateMatch[] matchings = tm.ProcessImage(sourceImage, template);
// highlight found matchings
BitmapData data = sourceImage.LockBits(
new Rectangle(0, 0, sourceImage.Width, sourceImage.Height),
ImageLockMode.ReadWrite, sourceImage.PixelFormat);
foreach (TemplateMatch m in matchings)
{
Drawing.Rectangle(data, m.Rectangle, Color.White);
MessageBox.Show(m.Rectangle.Location.ToString());
// do something else with matching
}
sourceImage.UnlockBits(data);

So, I just implemented it myself. But it is so slow - so if anyone has an idea to improve, feel free to criticize my code:
public class Position
{
public int bestRow { get; set; }
public int bestCol { get; set; }
public double bestSAD { get; set; }
public Position(int row, int col, double sad)
{
bestRow = row;
bestCol = col;
bestSAD = sad;
}
}
Position element_position = new Position(0, 0, double.PositiveInfinity);
Position ownSearch(Bitmap search, Bitmap template) {
Position position = new Position(0,0,double.PositiveInfinity);
double minSAD = double.PositiveInfinity;
// loop through the search image
for (int x = 0; x <= search.PhysicalDimension.Width - template.PhysicalDimension.Width; x++)
{
for (int y = 0; y <= search.PhysicalDimension.Height - template.PhysicalDimension.Height; y++)
{
position_label2.Content = "Running: X=" + x + " Y=" + y;
double SAD = 0.0;
// loop through the template image
for (int i = 0; i < template.PhysicalDimension.Width; i++)
{
for (int j = 0; j < template.PhysicalDimension.Height; j++)
{
int r = Math.Abs(search.GetPixel(x + i, y + j).R - template.GetPixel(i, j).R);
int g = Math.Abs(search.GetPixel(x + i, y + j).G - template.GetPixel(i, j).G);
int b = Math.Abs(search.GetPixel(x + i, y + j).B - template.GetPixel(i, j).B);
int a = template.GetPixel(i, j).A;
SAD = SAD + ((r + g + b)*a/255 );
}
}
// save the best found position
if (minSAD > SAD)
{
minSAD = SAD;
// give me VALUE_MAX
position.bestRow = x;
position.bestCol = y;
position.bestSAD = SAD;
}
}
}
return position;
}

I need to rotate an image

I need to rotate an image that is passed to a function as an array of BitmapFrames. the finished product needs to be saved as a BitmapFrame as well so I can send it to my Export-Image function. Help?
[Cmdlet(VerbsData.ConvertTo, "Rotate")]
public class RotateCmdlet : PSCmdlet
{
private BitmapFrame[] bFrame, outFrame;
private BitmapSource src;
private double pixelsize;
private int degrees;
private byte[] pixels, outPixels;
[Parameter(ValueFromPipeline = true,
ValueFromPipelineByPropertyName = true), ValidateNotNullOrEmpty]
public BitmapFrame[] Bitmap
{
get
{
return bFrame;
}
set
{
bFrame = value;
}
}
[Parameter(Position = 0), ValidateNotNullOrEmpty]
public int Degrees
{
get
{
return degrees;
}
set
{
degrees = value;
}
}
protected override void ProcessRecord()
{
base.ProcessRecord();
Console.Write("Rotating the image {0} degrees...\n\n", degrees);
outFrame = new BitmapFrame[bFrame.Length];
for (int c = 0; c < bFrame.Length; c++)
{
Image image;
pixelsize = bFrame[c].PixelWidth * bFrame[c].PixelHeight;
pixels = new byte[(int)pixelsize];
outPixels = new byte[(int)pixelsize];
bFrame[c].CopyPixels(pixels, (int)(bFrame[c].Width * (bFrame[c].Format.BitsPerPixel / 8.0)), 0);
Stream strm = new MemoryStream(pixels);
image = Image.FromStream(strm);
var newBitmap = new Bitmap((int)bFrame[c].PixelWidth, (int)bFrame[c].PixelHeight);
var graphics = Graphics.FromImage(newBitmap);
graphics.TranslateTransform((float)bFrame[c].PixelWidth / 2, (float)bFrame[c].PixelHeight / 2);
graphics.RotateTransform(degrees);
graphics.TranslateTransform(-(float)bFrame[c].PixelWidth / 2, -(float)bFrame[c].PixelHeight / 2);
graphics.DrawImage(image, new System.Drawing.Point(0, 0));
for (int i = 0; i < pixelsize; i++)
{
outPixels[i] = pixels[i];
}
src = BitmapSource.Create(bFrame[c].PixelWidth, bFrame[c].PixelHeight, bFrame[c].DpiX, bFrame[c].DpiY,
bFrame[c].Format, bFrame[c].Palette, outPixels, (int)(bFrame[c].Width * (bFrame[c].Format.BitsPerPixel / 8)));
outFrame[c] = BitmapFrame.Create(src);
}
WriteObject(outFrame);
}
}

It seems that the error you're getting on the FromStream call might be because the image format is invalid. This could be for many reasons and since I'm not sure exactly how this Cmdlet is being used I will make an assumption:
I may be incorrect here, but since you're passing in an array of BitMapFrame's, I'm wondering if the ProcessRecord is being called once for each array element. The only way to truly tell is to see how you're Cmdlet is being invoked. For example, if your BitMap parameter is coming from the pipeline then there is a good chance that ProcessRecord is being called once for each BitMapFrame in the array.
Does that make sense?