I'm working on pedestrians detection algorithm and got stuck on using ConnectedComponentsWithStats() method. I can't get to the values of stats. Here is what I wrote so far:
var labels = new Mat();
var stats = new Mat();
var centroids = new Mat();
var nLabels = CvInvoke.ConnectedComponentsWithStats(greyImage, labels, stats, centroids);
var centroidPoints = new MCvPoint2D64f[nLabels];
centroids.CopyTo(centroidPoints);
foreach (MCvPoint2D64f point in centroidPoints)
{
var x = point.X;
var y = point.Y;
}
The stats are stored as int32's in opencv c++ and have 5 fields x,y,width,height and area.
You can see the opencv implementation of ConnectedComponentsWithStats here.
Also you can see the struct CCStatsOp opencv returns the stats as here.
Sadly EMGUCV has not created the data structure CCStatsOp so you will need to just store the raw integer data like such.
int[] statsData = new int[stats.Rows * stats.Cols];
stats.CopyTo(statsData);
for (int i = 0; i < stats.Rows; i++)
{
var x = statsData[i * stats.Cols + 0];
var y = statsData[i * stats.Cols + 1];
var width = statsData[i * stats.Cols + 2];
var height = statsData[i * stats.Cols + 3];
var area = statsData[i * stats.Cols + 4];
}
You can also roll your own CCStats op with this code here.
Struct
public struct CCStatsOp
{
public Rectangle Rectangle;
public int Area;
}
Code
var labels = new Mat();
var stats = new Mat();
var centroids = new Mat();
var nLabels = CvInvoke.ConnectedComponentsWithStats(greyImage, labels, stats, centroids);
var centroidPoints = new MCvPoint2D64f[nLabels];
centroids.CopyTo(centroidPoints);
CCStatsOp[] statsOp = new CCStatsOp[stats.Rows];
stats.CopyTo(statsOp);
foreach (var statop in statsOp)
{
Console.WriteLine($"Rectangle: {statop.Rectangle} Area: {statop.Area}");
}
Related
Im using xamarin form to create Image Classification which i got the documentation in xamarin blog.
but i got an error when passing image to bytebuffer, once the tensorflow-lite interpreter run the output i got an error saying "Cannot convert between a TensorFlowLite buffer with 150528 bytes and a ByteBuffer with 602112 bytes."
Hope someone help me in this error. im little frustrated, thank you in advance. below is my code
public class TensorflowClassifier : IClassify
{
//FloatSize is a constant with the value of 4 because a float value is 4 bytes
const int FloatSize = 4;
//PixelSize is a constant with the value of 3 because a pixel has three color channels: Red Green and Blue
const int PixelSize = 3;
public List<ImageClassificationModel> Classify(byte[] image)
{
var mappedByteBuffer = GetModelAsMappedByteBuffer();
var interpreter = new Xamarin.TensorFlow.Lite.Interpreter(mappedByteBuffer);
//To resize the image, we first need to get its required width and height
var tensor = interpreter.GetInputTensor(0);
var shape = tensor.Shape();
var width = shape[1];
var height = shape[2];
var byteBuffer = GetPhotoAsByteBuffer(image, width, height);
//use StreamReader to import the labels from labels.txt
var streamReader = new StreamReader(Application.Context.Assets.Open("labels.txt"));
//Transform labels.txt into List<string>
var labels = streamReader.ReadToEnd().Split('\n').Select(s => s.Trim()).Where(s => !string.IsNullOrEmpty(s)).ToList();
//Convert our two-dimensional array into a Java.Lang.Object, the required input for Xamarin.TensorFlow.List.Interpreter
var outputLocations = new float[1][] { new float[labels.Count] };
var outputs = Java.Lang.Object.FromArray(outputLocations);
interpreter.Run(byteBuffer, outputs);
var classificationResult = outputs.ToArray<float[]>();
//Map the classificationResult to the labels and sort the result to find which label has the highest probability
var classificationModelList = new List<ImageClassificationModel>();
for (var i = 0; i < labels.Count; i++)
{
var label = labels[i]; classificationModelList.Add(new ImageClassificationModel(label, classificationResult[0][i]));
}
return classificationModelList;
}
//Convert model.tflite to Java.Nio.MappedByteBuffer , the require type for Xamarin.TensorFlow.Lite.Interpreter
private MappedByteBuffer GetModelAsMappedByteBuffer()
{
var assetDescriptor = Application.Context.Assets.OpenFd("model.tflite");
var inputStream = new FileInputStream(assetDescriptor.FileDescriptor);
var mappedByteBuffer = inputStream.Channel.Map(FileChannel.MapMode.ReadOnly, assetDescriptor.StartOffset, assetDescriptor.DeclaredLength);
return mappedByteBuffer;
}
//Resize the image for the TensorFlow interpreter
private ByteBuffer GetPhotoAsByteBuffer(byte[] image, int width, int height)
{
var bitmap = BitmapFactory.DecodeByteArray(image, 0, image.Length);
var resizedBitmap = Bitmap.CreateScaledBitmap(bitmap, width, height, true);
var modelInputSize = FloatSize * height * width * PixelSize;
var byteBuffer = ByteBuffer.AllocateDirect(modelInputSize);
byteBuffer.Order(ByteOrder.NativeOrder());
var pixels = new int[width * height];
resizedBitmap.GetPixels(pixels, 0, resizedBitmap.Width, 0, 0, resizedBitmap.Width, resizedBitmap.Height);
var pixel = 0;
//Loop through each pixels to create a Java.Nio.ByteBuffer
for (var i = 0; i < width; i++)
{
for (var j = 0; j < height; j++)
{
var pixelVal = pixels[pixel++];
byteBuffer.PutFloat(pixelVal >> 16 & 0xFF / 4);
byteBuffer.PutFloat(pixelVal >> 8 & 0xFF / 4);
byteBuffer.PutFloat(pixelVal & 0xFF / 4);
}
}
bitmap.Recycle();
return byteBuffer;
}
}
On my project i passed the byte array to the classifier
private async void SaveImagePad(object sender, EventArgs e)
{
Stream image = await PadView.GetImageStreamAsync(SignatureImageFormat.Jpeg);
//byte[] bytes = ReadFully(image);
byte[] bytes = new byte[image.Length];
image.Read(bytes, 0, bytes.Length);
List<ImageClassificationModel> classifyImage = DependencyService.Get<IClassify>().Classify(bytes);
}
I want to make image stitching with openCvSharp but something goes wrong. Since I do not get the right output of 2 image stitched together.
I was fallowing the tutorial in python link and converting it in to C# code.
Here are 2 photo examples
The output where image should be stitched I get wrong result. It looks the same as image 2.
I gues something goes wrong in line, or somewhere after the line
Cv2.WarpPerspective(trainImg, result, H, new OpenCvSharp.Size(width, height));
Here is the full code.
public partial class Form1 : Form
{
public Form1()
{
InitializeComponent();
}
private void Form1_Load(object sender, EventArgs e)
{
bool debugImages = true;
string locationFolder = "";
OpenFileDialog dlg = new OpenFileDialog();
dlg.CheckFileExists = true;
dlg.Multiselect = true;
if (dlg.ShowDialog() == System.Windows.Forms.DialogResult.OK)
{
locationFolder = Path.GetDirectoryName(dlg.FileNames[0]) + "\\output\\";
List<Mat> imagesMat = new List<Mat>();
for (var i = 0; i < dlg.FileNames.Length; i++)
{
using (Bitmap fromFile = new Bitmap(dlg.FileNames[i]))
{
Mat source = BitmapConverter.ToMat(fromFile);
imagesMat.Add(source);
}
}
if (imagesMat.Count != 2)
throw new Exception("Select only 2 images!!!");
int imageCounter = 0;
Mat trainImg = imagesMat[0];
Mat queryImg = imagesMat[1];
Mat trainImg_gray = new Mat();
Mat queryImg_gray = new Mat();
Cv2.CvtColor(trainImg, trainImg_gray, ColorConversionCodes.BGRA2GRAY);
Cv2.CvtColor(queryImg, queryImg_gray, ColorConversionCodes.BGRA2GRAY);
// detecting keypoints
// FastFeatureDetector, StarDetector, SIFT, SURF, ORB, BRISK, MSER, GFTTDetector, DenseFeatureDetector, SimpleBlobDetector
string method = "SURF";
string feature_matching = "bf"; //bf, knn
var descriptor = SURF.Create(500, 4, 2, true);
Mat descriptors1 = new Mat();
Mat descriptors2 = new Mat();
KeyPoint[] kpsA;
KeyPoint[] kpsB;
descriptor.DetectAndCompute(trainImg_gray, null, out kpsA, descriptors1);
descriptor.DetectAndCompute(queryImg_gray, null, out kpsB, descriptors2);
// Match descriptor vectors
//var flannMatcher = new FlannBasedMatcher();
DMatch[] matches;
if (feature_matching == "bf")
matches = matchKeyPointsBF(descriptors1, descriptors2, method);
else
matches = matchKeyPointsKNN(descriptors1, descriptors2, 0.75, method);
var bfView = new Mat();
Cv2.DrawMatches(trainImg, kpsA, queryImg, kpsB, matches, bfView, null, flags: DrawMatchesFlags.NotDrawSinglePoints);
if (debugImages)
{
using (Bitmap resultBitmap = BitmapConverter.ToBitmap(bfView))
resultBitmap.Save(locationFolder + (imageCounter++).ToString().PadLeft(3, '0') + ".png", ImageFormat.Png); //1
}
Mat H = getHomography(kpsA, kpsB, descriptors1, descriptors2, matches, 4);
if (H == null)
throw new Exception("No Homography!!!");
//for (var i = 0; i < H.Cols; i++)
//{
// for (var j = 0; j < H.Rows; j++)
// Console.Write(H.At<float>(i, j) + " ");
// Console.WriteLine("");
//}
double width = trainImg.Size().Width + queryImg.Size().Width;
double height = trainImg.Size().Height + queryImg.Size().Height;
Mat result = new Mat();
Cv2.WarpPerspective(trainImg, result, H, new OpenCvSharp.Size(width, height));
if (debugImages)
{
using (Bitmap resultBitmap = BitmapConverter.ToBitmap(result))
resultBitmap.Save(locationFolder + (imageCounter++).ToString().PadLeft(3, '0') + ".png", ImageFormat.Png); //1
}
result[new Rect(new OpenCvSharp.Point(0, 0), new OpenCvSharp.Size(queryImg.Size().Width, queryImg.Size().Height))] = queryImg;
if (debugImages)
{
using (Bitmap resultBitmap = BitmapConverter.ToBitmap(result))
resultBitmap.Save(locationFolder + (imageCounter++).ToString().PadLeft(3, '0') + ".png", ImageFormat.Png); //2
}
//# transform the panorama image to grayscale and threshold it
Mat gray = result.Clone();
Cv2.CvtColor(result, gray, ColorConversionCodes.BGR2GRAY);
Mat thresh = new Mat();
double thresh2 = Cv2.Threshold(gray, thresh, 0, 255, ThresholdTypes.Binary);
//# Finds contours from the binary image
OpenCvSharp.Point[][] cnts;
HierarchyIndex[] hierarchy;
Cv2.FindContours(thresh, out cnts, out hierarchy, RetrievalModes.External, ContourApproximationModes.ApproxSimple);
OpenCvSharp.Point[] cnts2 = new OpenCvSharp.Point[cnts[0].Length];
for (var k = 0; k < cnts[0].Length; k++)
cnts2[k] = cnts[0][k];
//InputArray ptsA = InputArray.Create(cnts2);
//var c = Cv2.ContourArea(ptsA, true);
OpenCvSharp.Rect xywh = Cv2.BoundingRect(cnts2);
result = result[new Rect(new OpenCvSharp.Point(xywh.X, xywh.Y), new OpenCvSharp.Size(xywh.Width, xywh.Height))];
//result = result[new Rect(new OpenCvSharp.Point(0, 0), new OpenCvSharp.Size(256, 256))];
Bitmap endResultBitmap = BitmapConverter.ToBitmap(result);
endResultBitmap.Save(locationFolder + (imageCounter++).ToString().PadLeft(3, '0') + ".png", ImageFormat.Png); //4
Environment.Exit(-1);
}
}
private BFMatcher createMatcher(string method, bool crossCheck)
{
//"Create and return a Matcher Object"
if (method == "SURF" || method == "SIFT")
return new BFMatcher(NormTypes.L2, crossCheck);
else //if (method == "ORB" || method == "BRISK")
return new BFMatcher(NormTypes.Hamming, crossCheck);
}
private DMatch[] matchKeyPointsBF(Mat featuresA, Mat featuresB, string method)
{
BFMatcher bf = createMatcher(method, crossCheck: true);
// # Match descriptors.
DMatch[] bfMatches = bf.Match(featuresA, featuresB);
//# Sort the features in order of distance.
//# The points with small distance (more similarity) are ordered first in the vector
DMatch[] rawMatches = bfMatches.OrderBy(a => a.Distance).ToArray();
if (rawMatches.Length > 100)
Array.Resize(ref rawMatches, 100);
return rawMatches;
}
private DMatch[] matchKeyPointsKNN(Mat featuresA, Mat featuresB, double ratio, string method)
{
BFMatcher bf = createMatcher(method, crossCheck: false);
// # compute the raw matches and initialize the list of actual matches
DMatch[][] rawMatches = bf.KnnMatch(featuresA, featuresB, 2);
List<DMatch> rawMatches2 = new List<DMatch>();
//# loop over the raw matches
DMatch prevmatchN = rawMatches[0][0];
rawMatches2.Add(prevmatchN);
for (int m = 0; m < rawMatches.Length; m++)
{
for (int n = 0; n < rawMatches[m].Length; n++)
{
//# ensure the distance is within a certain ratio of each
//# other (i.e. Lowe's ratio test)
DMatch matchN = rawMatches[m][n];
if (n == 0)
prevmatchN = matchN;
if (prevmatchN.Distance < matchN.Distance * (ratio))
rawMatches2.Add(matchN);
if (rawMatches2.Count >= 100)
break;
}
}
return rawMatches2.ToArray();
}
private Mat getHomography(KeyPoint[] kpsA, KeyPoint[] kpsB, Mat featuresA, Mat featuresB, DMatch[] matches, int reprojThresh)
{
//# convert the keypoints to numpy arrays
Point2f[] PtA = new Point2f[matches.Length];
Point2f[] PtB = new Point2f[matches.Length];
for (int i = 0; i < matches.Length; i++)
{
KeyPoint kpsAI = kpsA[matches[i].QueryIdx];
KeyPoint kpsBI = kpsB[matches[i].TrainIdx];
PtA[i] = new Point2f(kpsAI.Pt.X, kpsAI.Pt.Y);
PtB[i] = new Point2f(kpsBI.Pt.X, kpsBI.Pt.Y);
}
InputArray ptsA = InputArray.Create(PtA);
InputArray ptsB = InputArray.Create(PtB);
if (matches.Length > 4)
{
//You get the homography matrix usin
Mat H = Cv2.FindHomography(ptsA, ptsB, HomographyMethods.Ransac, reprojThresh);
//and then to get any point on the target picture from the original picture:
//Mat targetPoint = new Mat();
//Cv2.PerspectiveTransform(ptsA, targetPoint, H);
return H;
}
else
return null;
}
}
I have c# TensorFlow.NET working in Unity. But it using an image from the file system. I want to be able to use an image from memory (Texture2D).
I tried to follow some examples of people using TensorFlowSharp. But that didn't work.
What am I doing wrong?
Note: with both functions, I am using the same image. The image is 512x512. but the result of both pictures is different.
// Doesn't work
private NDArray FromTextureToNDArray(Texture2D texture) {
Color32[] pixels = texture.GetPixels32();
byte[] floatValues = new byte[(texture.width * texture.height) * 3];
for (int i = 0; i < pixels.Length; i++) {
var color = pixels[i];
floatValues[i * 3] = color.r;
floatValues[i * 3 + 1] = color.g;
floatValues[i * 3 + 2] = color.b;
}
Shape shape = new Shape(1, texture.width, texture.height, 3);
NDArray image = new NDArray(floatValues, shape);
return image;
}
// Works
private NDArray ReadFromFile(string fileName) {
var graph = new Graph().as_default();
// Change image
var file_reader = tf.read_file(fileName, "file_reader");
var decodeJpeg = tf.image.decode_jpeg(file_reader, channels: 3, name: "DecodeJpeg");
var casted = tf.cast(decodeJpeg, TF_DataType.TF_UINT8);
var dims_expander = tf.expand_dims(casted, 0);
using (var sess = tf.Session(graph)) {
return sess.run(dims_expander);
}
}
I ended up using this code from Shaqian: https://github.com/shaqian/TF-Unity/blob/master/TensorFlow/Utils.cs
Add this script to your project and then you could use it like this:
// Get image
byte[] imageData = Utils.DecodeTexture(texture, texture.width, texture.height, 0, Flip.VERTICAL);
Shape shape = new Shape(1, texture.width, texture.height, 3);
NDArray image = new NDArray(imageData, shape);
Use Barracuda as step in between.
var encoder = new Unity.Barracuda.TextureAsTensorData(your_2d_texture);
My first time working with OpenCV. The code below uses the Emgu package, but I've also tried it using OpenCVSharp and have got exactly the same results.
I am calibrating the camera using 6 chessboard photos, then using the calibration to Undistort an image. The test image is a photo of some squared paper. The camera really isn't very distorted - I can't tell that it is distorted by eye, only when I draw a straight line over the lines in a paint program.
However - the resulting image comes out way more distorted than the original - adding a lot of barrel distortion.
When I run the commented out lines, the "DrawChessboardCorners" show that it is identifying the points perfectly.
What am I doing wrong here?
static void Main(string[] args)
{
Size patternSize = new Size(9, 6);
List<VectorOfPointF> ListOfCornerPoints = new List<VectorOfPointF>();
DirectoryInfo dir = new DirectoryInfo(#"Z:\Simon\Dropbox\Apps\OpenCVPlay\Image Processing\Chessboard Pattern Photos With Pixel");
Size calibrationImageSize = new Size(0,0) ;
foreach (var file in dir.GetFiles())
{
VectorOfPointF corners = new VectorOfPointF();
var calibrationImage = new Image<Rgb, byte>(file.FullName);
bool find = CvInvoke.FindChessboardCorners(calibrationImage, patternSize, corners, CalibCbType.AdaptiveThresh | CalibCbType.FilterQuads);
//CvInvoke.DrawChessboardCorners(calibrationImage, patternSize, corners, find);
//calibrationImage.Save(#"Z:\Simon\Dropbox\Apps\OpenCVPlay\Image Processing\Chessboard Pattern Photos With Pixel\test\" + file.Name);
ListOfCornerPoints.Add(corners);
calibrationImageSize = calibrationImage.Size;
}
PointF[][] arrayOfCornerPoints = ListOfCornerPoints.Select(a => a.ToArray()).ToArray();
var modelPoints = CreateModelPoints(ListOfCornerPoints.Count, patternSize.Width, patternSize.Height);
var arrayOfModelPoints = modelPoints.Select(a => a.ToArray()).ToArray();
Matrix<double> cameraDistortionCoeffs = new Matrix<double>(5, 1);
Matrix<double> cameraMatrix = new Matrix<double>(3, 3);
Mat[] rotationVectors;
Mat[] translationVectors;
CvInvoke.CalibrateCamera(
arrayOfModelPoints,
arrayOfCornerPoints,
calibrationImageSize,
cameraMatrix,
cameraDistortionCoeffs,
CalibType.Default,
new MCvTermCriteria(),
out rotationVectors,
out translationVectors);
var imageToProcess = new Image<Rgb, byte>(#"Z:\Simon\Dropbox\Apps\OpenCVPlay\Image Processing\Test data\squarePaper.jpg");
Rectangle Rect2 = new Rectangle();
var newCameraMatrix = CvInvoke.GetOptimalNewCameraMatrix(cameraMatrix, cameraDistortionCoeffs, calibrationImageSize, 1, imageToProcess.Size, ref Rect2, true);
Image<Rgb, byte> processedImage = imageToProcess.Clone();
CvInvoke.Undistort(imageToProcess, processedImage, cameraMatrix, cameraDistortionCoeffs);
processedImage.Save(#"Z:\Simon\Dropbox\Apps\OpenCVPlay\Image Processing\Test data\squarePaperFixed.jpg");
}
static List<List<MCvPoint3D32f>> CreateModelPoints(int length, int chessboardCols, int chessboardRows)
{
var modelPoints = new List<List<MCvPoint3D32f>>();
for (var k = 0; k < length; k++)
{
var chessboard = new List<MCvPoint3D32f>();
for (var y = 0; y < chessboardRows; y++)
{
for (var x = 0; x < chessboardCols; x++)
{
chessboard.Add(new MCvPoint3D32f(x, y, 0));
}
}
modelPoints.Add(chessboard);
}
return modelPoints;
}
Calculating area:
var coordinates = new List<Coordinate> {
new Coordinate(55, 35),
new Coordinate(55, 35.1),
new Coordinate(55.1, 35.1),
new Coordinate(55.1, 35),
};
Console.WriteLine(new Polygon(coordinates).Area); // ~0.01
Calculation is right, because it's happen in orthogonal coordinate system.
But how to mark that coordinates are in WGS?
It seems that task is more complicated that I've expected. I found this useful discussion on google groups
Firstly we need to found projection system, that is most suitable for our region where we need to compute area. For example you can take one of UTM zones
using DotSpatial.Projections;
using DotSpatial.Topology;
public static double CalculateArea(IEnumerable<double> latLonPoints)
{
// source projection is WGS1984
var projFrom = KnownCoordinateSystems.Geographic.World.WGS1984;
// most complicated problem - you have to find most suitable projection
var projTo = KnownCoordinateSystems.Projected.UtmWgs1984.WGS1984UTMZone37N;
// prepare for ReprojectPoints (it mutates array)
var z = new double[latLonPoints.Count() / 2];
var pointsArray = latLonPoints.ToArray();
Reproject.ReprojectPoints(pointsArray, z, projFrom, projTo, 0, pointsArray.Length / 2);
// assemblying new points array to create polygon
var points = new List<Coordinate>(pointsArray.Length / 2);
for (int i = 0; i < pointsArray.Length / 2; i++)
points.Add(new Coordinate(pointsArray[i * 2], pointsArray[i * 2 + 1]));
var poly = new Polygon(points);
return poly.Area;
}
You can get the area directly from IGeometry or from Feature.Geometry. Also You need to repeat the first coordinate to close your polygon.
FeatureSet fs = new FeatureSet(FeatureType.Polygon);
Coordinate[] coord = new Coordinate[]
{
new Coordinate(55, 35),
new Coordinate(55, 35.1),
new Coordinate(55.1, 35.1),
new Coordinate(55.1, 35),
new Coordinate(55, 35)
};
fs.AddFeature(new Polygon(new LinearRing(coord)));
var area = fs.Features.First().Geometry.Area;