Develop Reference

Capture image from screen and get colors

I'm making a program which can capture a small area on screen and will run something if there is any color on image that match the target colors. My program run as the following Sequence:
Get image from a specific area from screen
Save to a folder
using CountPixel to detect any target_color
However, after I click the button5 twice times (not double click), it through an exception at below line :
b.Save(#"C:\Applications\CaptureImage000.jpg", ImageFormat.Jpeg);
Exception :
An unhandled exception of type
'System.Runtime.InteropServices.ExternalException' occurred in
System.Drawing.dll
Additional information: A generic error occurred in GDI+
My questions are :
How can i fix this exception ?
I want to use another method instead of CountPixel() to improve performance, because I just need to detect only one target color to rise event.
Step 2 is troublesome. I wonder if i can skip it and use the other way to call: (#"C:\Applications\CaptureImage000.jpg", ImageFormat.Jpeg) , because using this long string isn't comfortable and result error when im trying to use with GetPixel,... or add it into some "value example" code on internet for improvement purpose.
private int CountPixels(Bitmap bm, Color target_color)
{
// Loop through the pixels.
int matches = 0;
for (int y = 0; y < bm.Height; y++)
{
for (int x = 0; x < bm.Width; x++)
{
if (bm.GetPixel(x, y) == target_color) matches++;
}
}
return matches;
}
private Bitmap CapturedImage(int x, int y)
{
Bitmap b = new Bitmap(XX, YY);
Graphics g = Graphics.FromImage(b);
g.CopyFromScreen(x, y, 0, 0, new Size(XX, YY));
b.Save(#"C:\Applications\CaptureImage000.jpg", ImageFormat.Jpeg);
/* Run 3 line below will lead to question 1 - through exception
Bitmap bm = new Bitmap(#"C:\Applications\CaptureImage000.jpg");
int black_pixels = CountPixels(b, Color.FromArgb(255, 0, 0, 0));
textBox3.Text = black_pixels + " black pixels";
*/
return b;
}
private void button5_Click(object sender, EventArgs e)// Do screen cap
{
Bitmap bmp = null;
bmp = CapturedImage(X0, Y0);
}

[EDIT] Worked on this tonight with OP, made some improvements
This now accounts for endianness of the machine and correctly compares colors by converting them to integers with the Color.ToArgb() function
the below code will work, I have added comments for clarity and given you some options. I wrote the code without an IDE but I am confident it is fine.
In both cases below, just keep the handle to the bitmap, don't need to save and reopen regardless of if you need a copy.
Exception issue and improvements to CapturedImage function
option A (recommended)
Don't save the bitmap, you already have a handle, the graphics object just modified the BMP. Just leave the below code as is for this function and it will work fine without un-commenting one of the other options.
Code and other options:
private Bitmap CapturedImage(Bitmap bm, int x, int y)
{
Bitmap b = new Bitmap(XX, YY);
Graphics g = Graphics.FromImage(b);
g.CopyFromScreen(x, y, 0, 0, new Size(XX, YY));
//option B - If you DO need to keep a copy of the image use PNG and delete the old image
/*
try
{
if(System.IO.File.Exists(#"C:\Applications\CaptureImage.png"))
{
System.IO.File.Delete(#"C:\Applications\CaptureImage.png");
}
b.Save(#"C:\Applications\CaptureImage.png", ImageFormat.Png);
}
catch (System.Exception ex)
{
MessageBox.Show("There was a problem trying to save the image, is the file in open in another program?\r\nError:\r\n\r\n" + ex.Message);
}
*/
//option C - If you DO need to keep a copy of the image AND keep all copies of all images when you click the button use PNG and generate unique filename
/*
int id = 0;
while(System.IO.File.Exists(#"C:\Applications\CaptureImage" + id.ToString().PadLeft('0',4) + ".png"))
{
//increment the id until a unique file name is found
id++;
}
b.Save(#"C:\Applications\CaptureImage" + id.ToString().PadLeft('0',4) + ".png", ImageFormat.Png);
*/
int black_pixels = CountPixels(b, Color.FromArgb(255, 0, 0, 0));
textBox3.Text = black_pixels + " black pixels";
return b;
}
Now for the CountPixels function, you have 3 options but really, you have one really solid option, so I am omitting the others.
This locks the bits in the BMP, uses marshalling to copy the data into an array and scans the array for data, very, very fast, and you will likely not even need to remove the count. If you do STILL want to remove the count, just add "return 1;" right underneath where it increments the matches variable.
Speed issue and improvements to CountPixels function
private int CountPixels(Bitmap bm, Color target_color)
{
int matches = 0;
Bitmap bmp = (Bitmap)bm.Clone();
BitmapData bmpDat = bmp.LockBits(new Rectangle(0, 0, bmp.Width, bmp.Height), ImageLockMode.ReadWrite, bmp.PixelFormat);
int size = bmpDat.Stride * bmpDat.Height;
byte[] subPx = new byte[size];
System.Runtime.InteropServices.Marshal.Copy(bmpDat.Scan0, subPx, 0, size);
//change the 4 (ARGB) to a 3 (RGB) if you don't have an alpha channel, this is for 32bpp images
//ternary operator to check endianess of machine and organise pixel colors as A,R,G,B or B,G,R,A (little endian is reversed);
Color temp = BitConverter.IsLittleEndian ? Color.FromArgb(subPx[i + 2], subPx[i + 1], subPx[i]) : Color.FromArgb(subPx[i + 1], subPx[i + 2], subPx[i + 3]);
for (int i = 0; i < size; i += 4 ) //4 bytes per pixel A, R, G, B
{
if(temp.ToArgb() == target_color.ToArgb())
{
matches++;
}
}
System.Runtime.InteropServices.Marshal.Copy(subPx, 0, bmpDat.Scan0, subPx.Length);
bmp.UnlockBits(bmpDat);
return matches;
}
Finally the same function but allowing for a tolerance percent
private int CountPixels(Bitmap bm, Color target_color, float tolerancePercent)
{
int matches = 0;
Bitmap bmp = (Bitmap)bm.Clone();
BitmapData bmpDat = bmp.LockBits(new Rectangle(0, 0, bmp.Width, bmp.Height), ImageLockMode.ReadWrite, bmp.PixelFormat);
int size = bmpDat.Stride * bmpDat.Height;
byte[] subPx = new byte[size];
System.Runtime.InteropServices.Marshal.Copy(bmpDat.Scan0, subPx, 0, size);
for (int i = 0; i < size; i += 4 )
{
byte r = BitConverter.IsLittleEndian ? subPx[i+2] : subPx[i+3];
byte g = BitConverter.IsLittleEndian ? subPx[i+1] : subPx[i+2];
byte b = BitConverter.IsLittleEndian ? subPx[i] : subPx[i+1];
float distancePercent = (float)Math.Sqrt(
Math.Abs(target_color.R-r) +
Math.Abs(target_color.G-g) +
Math.Abs(target_color.B-b)
) / 7.65f;
if(distancePercent < tolerancePercent)
{
matches++;
}
}
System.Runtime.InteropServices.Marshal.Copy(subPx, 0, bmpDat.Scan0, subPx.Length);
bmp.UnlockBits(bmpDat);
return matches;
}

How to access information in a thread with a timer?

I created a form in C# where there is a image which is being converted into a boolean array. I am trying to generate a thread with a timer where the image is converted 4 times a second.
When I debug it, it works, I can trace the ticks from the timer. But when the form is running the application quits without giving a bug.
This is the initialize script:
form = new LoadForm();
form.Show();
form.BringToFront();
timer = new System.Threading.Timer(new TimerCallback(camTick), null, 0, 250);
This is the tick that works:
private void camTick(Object myObject)
{
if (form.isRunning)
{
bool[,] ar = form.getBoolBitmap(100);
}
}
This is the function that loads and saves the bitmap. In form1.cs
public bool[,] getBoolBitmap(uint threshold) {
unsafe {
Bitmap b = getBitmap();
BitmapData originalData = b.LockBits(new Rectangle(0, 0, b.Width, b.Height), ImageLockMode.ReadOnly, PixelFormat.Format24bppRgb);
bool[,] ar = new bool[b.Width, b.Height];
for (int y = 0; y < b.Height; y++) {
byte* Row = (byte*)originalData.Scan0 + (y * originalData.Stride);
for (int x = 0; x < b.Width; x++) {
if ((byte)Row[x * 3] < threshold) {
ar[x, y] = false;
} else {
ar[x, y] = true;
}
}
}
b.Dispose();
return ar;
}
}
public Bitmap getBitmap() {
if (!panelVideoPreview.IsDisposed) {
Bitmap b = new Bitmap(panelVideoPreview.Width, panelVideoPreview.Height, PixelFormat.Format24bppRgb);
using (Graphics g = Graphics.FromImage(b)) {
Rectangle rectanglePanelVideoPreview = panelVideoPreview.Bounds;
Point sourcePoints = panelVideoPreview.PointToScreen(new Point(panelVideoPreview.ClientRectangle.X, panelVideoPreview.ClientRectangle.Y));
g.CopyFromScreen(sourcePoints, Point.Empty, rectanglePanelVideoPreview.Size);
}
return b;
} else {
Bitmap b = new Bitmap(panelVideoPreview.Width, panelVideoPreview.Height);
return b;
}
}

Is this image being stored in a Control like PictureBox? If so, make sure you're only accessing it on the control's thread. Check out Control.Invoke().

The callback from System.Threading.Timer runs on a ThreadPool thread. In that callback you are accessing a Form instance. You simply cannot do this. Well, you can, but it will not work correctly. It will fail unpredictable and sometimes spectacularly.
Use System.Windows.Forms.Timer instead. Have it tick every 4 seconds. In the Tick event get whatever data you need from the form and then pass it off to another thread for further processing.
In the following code I get the Bitmap object by calling DrawToBitmap on the UI thread. I then pass the Bitmap off to a Task where it can be converted into a bool[] in the background. Optionally, you can return that bool[] from the Task and then call ContinueWith to transfer it back to the UI thread if necessary (sounds like you probably do not need to do that though).
private void YourWindowsFormsTimer_Tick(object sender, EventArgs args)
{
// Get the bitmap object while still on the UI thread.
var bm = new Bitmap(panelVideoPreview.ClientSize.Width, panelVideoPreview.ClientSize.Height, PixelFormat.Format24bppRgb);
panelVideoPreview.DrawToBitmap(bm, panelVideoPreview.ClientRectangle);
Task.Factory
.StartNew(() =>
{
// It is not safe to access the UI here.
bool[,] ar = ConvertBitmapToByteArray(bm);
DoSomethingWithTheArrayHereIfNecessary(ar);
// Optionally return the byte array if you need to transfer it to the UI thread.
return ar;
})
.ContinueWith((task) =>
{
// It is safe to access the UI here.
// Get the returned byte array.
bool[,] ar = task.Result;
UpdateSomeControlIfNecessaryHere(ar);
}, TaskScheduler.FromCurrentSynchronizationContext());
}
And ConvertBitmapToByteArray would look like this now.
public unsafe bool[,] ConvertBitmapToBoolArray(Bitmap b, uint threshold)
{
BitmapData originalData = b.LockBits(new Rectangle(0, 0, b.Width, b.Height), ImageLockMode.ReadOnly, PixelFormat.Format24bppRgb);
bool[,] ar = new bool[b.Width, b.Height];
for (int y = 0; y < b.Height; y++)
{
byte* Row = (byte*)originalData.Scan0 + (y * originalData.Stride);
for (int x = 0; x < b.Width; x++)
{
if ((byte)Row[x * 3] < treshold)
{
ar[x, y] = false;
} else
{
ar[x, y] = true;
}
}
}
b.Dispose();
return ar;
}

After some searching on the internet I found this out, I needed to make another function to return the bool[,], this is what I came up with:
public bool[,] invokeBitmap(uint treshold) {
if (this.InvokeRequired) {
return (bool[,])this.Invoke((Func<bool[,]>)delegate {
return getBoolBitmap(treshold);
});
} else {
return getBoolBitmap(treshold);
}
}

How can I find a template image in another image? Kinect and AForge preferred

I copied the AForge-Sample from here:
http://www.aforgenet.com/framework/features/template_matching.html
And hoped, it would work with 2 Bitmaps as sources as in the following code:
Bitmap findTemplate (Bitmap sourceImage, Bitmap template)
{
// create template matching algorithm's instance
// (set similarity threshold to x.y%, 1.0f = 100%)
ExhaustiveTemplateMatching tm = new ExhaustiveTemplateMatching( 0.4f );
// find all matchings with specified above similarity
TemplateMatch[] matchings = tm.ProcessImage( sourceImage, template ); **// "Unsupported pixel format of the source or template image." as error message**
// highlight found matchings
BitmapData data = sourceImage.LockBits(
new Rectangle( 0, 0, sourceImage.Width, sourceImage.Height ),
ImageLockMode.ReadWrite, sourceImage.PixelFormat );
foreach ( TemplateMatch m in matchings )
{
AForge.Imaging.Drawing.Rectangle( data, m.Rectangle, System.Drawing.Color.White );
// do something else with matching
}
sourceImage.UnlockBits( data );
return sourceImage;
}
But when calling TemplateMatch[] matchings = tm.P.... it gives the error mentioned above.
The template is generated this way:
Bitmap templatebitmap=(Bitmap)AForge.Imaging.Image.FromFile("template.jpg");
the source is generated with the kinect-webcam, where the PlanarImage is formatted as Bitmap (method copied from somewhere, but it was working up to now)
Bitmap PImageToBitmap(PlanarImage PImage)
{
Bitmap bmap = new Bitmap(
PImage.Width,
PImage.Height,
System.Drawing.Imaging.PixelFormat.Format32bppRgb);
BitmapData bmapdata = bmap.LockBits(
new Rectangle(0, 0, PImage.Width,
PImage.Height),
ImageLockMode.WriteOnly,
bmap.PixelFormat);
IntPtr ptr = bmapdata.Scan0;
Marshal.Copy(PImage.Bits,
0,
ptr,
PImage.Width *
PImage.BytesPerPixel *
PImage.Height);
bmap.UnlockBits(bmapdata);
return bmap;
}
So, is anbody able to help me, where my mistake might be?
Or maybe anyone knows a better way to match a template with a Kinect?
The overall job is to detect a known object with the kinect, in my case a rubberduck.
Thank you in advamce.

here's the solution using AForge
but it is slow take around 5 seconds but it works
As usuall u need to introduce AForge framework download and install it.
specify using AForge namespace
and copy paste to make it work
System.Drawing.Bitmap sourceImage = (Bitmap)Bitmap.FromFile(#"C:\SavedBMPs\1.jpg");
System.Drawing.Bitmap template = (Bitmap)Bitmap.FromFile(#"C:\SavedBMPs\2.jpg");
// create template matching algorithm's instance
// (set similarity threshold to 92.5%)
ExhaustiveTemplateMatching tm = new ExhaustiveTemplateMatching(0.921f);
// find all matchings with specified above similarity
TemplateMatch[] matchings = tm.ProcessImage(sourceImage, template);
// highlight found matchings
BitmapData data = sourceImage.LockBits(
new Rectangle(0, 0, sourceImage.Width, sourceImage.Height),
ImageLockMode.ReadWrite, sourceImage.PixelFormat);
foreach (TemplateMatch m in matchings)
{
Drawing.Rectangle(data, m.Rectangle, Color.White);
MessageBox.Show(m.Rectangle.Location.ToString());
// do something else with matching
}
sourceImage.UnlockBits(data);

So, I just implemented it myself. But it is so slow - so if anyone has an idea to improve, feel free to criticize my code:
public class Position
{
public int bestRow { get; set; }
public int bestCol { get; set; }
public double bestSAD { get; set; }
public Position(int row, int col, double sad)
{
bestRow = row;
bestCol = col;
bestSAD = sad;
}
}
Position element_position = new Position(0, 0, double.PositiveInfinity);
Position ownSearch(Bitmap search, Bitmap template) {
Position position = new Position(0,0,double.PositiveInfinity);
double minSAD = double.PositiveInfinity;
// loop through the search image
for (int x = 0; x <= search.PhysicalDimension.Width - template.PhysicalDimension.Width; x++)
{
for (int y = 0; y <= search.PhysicalDimension.Height - template.PhysicalDimension.Height; y++)
{
position_label2.Content = "Running: X=" + x + " Y=" + y;
double SAD = 0.0;
// loop through the template image
for (int i = 0; i < template.PhysicalDimension.Width; i++)
{
for (int j = 0; j < template.PhysicalDimension.Height; j++)
{
int r = Math.Abs(search.GetPixel(x + i, y + j).R - template.GetPixel(i, j).R);
int g = Math.Abs(search.GetPixel(x + i, y + j).G - template.GetPixel(i, j).G);
int b = Math.Abs(search.GetPixel(x + i, y + j).B - template.GetPixel(i, j).B);
int a = template.GetPixel(i, j).A;
SAD = SAD + ((r + g + b)*a/255 );
}
}
// save the best found position
if (minSAD > SAD)
{
minSAD = SAD;
// give me VALUE_MAX
position.bestRow = x;
position.bestCol = y;
position.bestSAD = SAD;
}
}
}
return position;
}

Obtain RGB value of image and potentially move image file in C#

I'm looking at doing a project in C# that looks at an image file not sure of extension yet, and notes the RGB value and if its too dark moves it to another folder for me to look at later
So here it is in block form
Load multiple images from directory > Check RGB value of every file > if too dark > move to different folder. if not ignore (leave in original folder)
I know the basics like get files from dir but checking RGB value of whole picture and then moving it or ignoring it I'm stumped.
I have this code:
private void button1_Click(object sender, EventArgs e)
{
CompareImages(Environment.GetFolderPath(Environment.SpecialFolder.MyPictures),
Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.DesktopDirectory), "checked"), 127.0, new string[] {"*.jpg", "*.png"});
}
private void CompareImages(string sourceFolder, string disposedImgFolder, double threshold, string[] extensions)
{
if (Directory.Exists(sourceFolder))
{
DirectoryInfo dir = new DirectoryInfo(sourceFolder);
List<FileInfo> pictures = new List<FileInfo>();
foreach (string ext in extensions)
{
FileInfo[] fi = dir.GetFiles(ext);
pictures.AddRange(fi);
}
Directory.CreateDirectory(disposedImgFolder);
int j = 0;
if (pictures.Count > 0)
{
for (int i = 0; i < pictures.Count; i++)
{
Image img = null;
Bitmap bmp = null;
try
{
img = Image.FromFile(pictures[i].FullName);
bmp = new Bitmap(img);
img.Dispose();
double avg = GetAveragePixelValue(bmp);
bmp.Dispose();
if (avg < threshold)
{
string dest = Path.Combine(disposedImgFolder, pictures[i].Name);
if (File.Exists(dest) == false)
{
pictures[i].MoveTo(dest);
j++;
}
else
{
}
}
else
{
}
}
catch
{
if (img != null)
img.Dispose();
if (bmp != null)
bmp.Dispose();
}
}
MessageBox.Show("Done, " + j.ToString() + " files moved.");
}
}
}
private unsafe double GetAveragePixelValue(Bitmap bmp)
{
BitmapData bmData = null;
try
{
bmData = bmp.LockBits(new Rectangle(0, 0, bmp.Width, bmp.Height), ImageLockMode.ReadOnly, PixelFormat.Format32bppArgb);
int stride = bmData.Stride;
IntPtr scan0 = bmData.Scan0;
int w = bmData.Width;
int h = bmData.Height;
double sum = 0;
long pixels = bmp.Width * bmp.Height;
byte* p = (byte*)scan0.ToPointer();
for (int y = 0; y < h; y++)
{
p = (byte*)scan0.ToPointer();
p += y * stride;
for (int x = 0; x < w; x++)
{
double i = ((double)p[0] + p[1] + p[2]) / 3.0;
sum += i;
p += 4;
}
}
bmp.UnlockBits(bmData);
double result = sum / (double)pixels;
return result;
}
catch
{
try
{
bmp.UnlockBits(bmData);
}
catch
{
}
}
return -1;
}
How do I define the threashold?

If you want to read every pixel of an image, it must be converted to a bitmap. Then you can use the GetPixel method. But, this process is very slow and it takes a lot of CPU. If you do so, you really should run some test before.
using (var m = new MemoryStream())
{
using (var img = Image.FromFile(args[0]))
{
img.Save(m, ImageFormat.Bmp);
}
m.Position = 0;
using (var bitmap = (Bitmap)Bitmap.FromStream(m))
{
for (var x = 0; x < bitmap.Width; x++)
{
for (var y = 0; y < bitmap.Height; y++)
{
var color = bitmap.GetPixel(x, y);
// TODO: Do what ever you want
}
}
}
}

I think you need to read up a bit on RGB. Every pixel will have an Red, Green and Blue value associated with it and i guess you are looking for a way to get some meassure of how bright the "average" pixel is? If so you need to loop over all pixels. While doing so calculate brightness of each pixel. "Brightness" of each pixels can be calculated in several ways, you could simply do (R + G + B)/3, or you could try to mimic that the human eye isn't equaly sensitive to R, G and B.
Then you will have to decide how to use your "brightness" of pixel. Mean, Median, something else? It depends on what you want to do..
Update: After reading more of your comments I'm still not really sure what you mean by "bright" or "dark". It also seems that you have your terminology a bit confused, you keep talking about a RGB value for an entire image, but RGB values in an image refer to individual pixel values.
I believe that this page could help you doing what you want:
http://www.nbdtech.com/Blog/archive/2008/04/27/Calculating-the-Perceived-Brightness-of-a-Color.aspx
Also, some complimentary reading to understand RGB:
http://en.wikipedia.org/wiki/Luma_(video)
http://en.wikipedia.org/wiki/RGB_color_space

Unsafe Per Pixel access, 30ms access for 1756000 pixels

So I've been sharing some thoughts on the above topic title on my website about fast, unsafe pixel access. A gentlemen gave me a rough example of how he'd do it in C++, but that doesn't help me in C# unless I can interop it, and the interop is fast as well. I had found a class in the internet that was written using MSDN help, to unsafely access pixels. The class is exceptionally fast, but it's not fast enough. Here's the class:
using System;
using System.Collections.Generic;
using System.Text;
using System.Drawing;
using System.Drawing.Imaging;
namespace DCOMProductions.Desktop.ScreenViewer {
public unsafe class UnsafeBitmap {
Bitmap bitmap;
// three elements used for MakeGreyUnsafe
int width;
BitmapData bitmapData = null;
Byte* pBase = null;
public UnsafeBitmap(Bitmap bitmap) {
this.bitmap = new Bitmap(bitmap);
}
public UnsafeBitmap(int width, int height) {
this.bitmap = new Bitmap(width, height, PixelFormat.Format32bppArgb);
}
public void Dispose() {
bitmap.Dispose();
}
public Bitmap Bitmap {
get {
return (bitmap);
}
}
private Point PixelSize {
get {
GraphicsUnit unit = GraphicsUnit.Pixel;
RectangleF bounds = bitmap.GetBounds(ref unit);
return new Point((int)bounds.Width, (int)bounds.Height);
}
}
public void LockBitmap() {
GraphicsUnit unit = GraphicsUnit.Pixel;
RectangleF boundsF = bitmap.GetBounds(ref unit);
Rectangle bounds = new Rectangle((int)boundsF.X,
(int)boundsF.Y,
(int)boundsF.Width,
(int)boundsF.Height);
// Figure out the number of bytes in a row
// This is rounded up to be a multiple of 4
// bytes, since a scan line in an image must always be a multiple of 4 bytes
// in length.
width = (int)boundsF.Width * sizeof(Pixel);
if (width % 4 != 0) {
width = 4 * (width / 4 + 1);
}
bitmapData =
bitmap.LockBits(bounds, ImageLockMode.ReadWrite, PixelFormat.Format32bppArgb);
pBase = (Byte*)bitmapData.Scan0.ToPointer();
}
public Pixel GetPixel(int x, int y) {
Pixel returnValue = *PixelAt(x, y);
return returnValue;
}
public void SetPixel(int x, int y, Pixel colour) {
Pixel* pixel = PixelAt(x, y);
*pixel = colour;
}
public void UnlockBitmap() {
bitmap.UnlockBits(bitmapData);
bitmapData = null;
pBase = null;
}
public Pixel* PixelAt(int x, int y) {
return (Pixel*)(pBase + y * width + x * sizeof(Pixel));
}
}
}
Basically what I am doing is copying the entire screen and comparing each pixel to and old copy. On a 1680x1050 bitmap, this takes approximately 300 milliseconds using the following code.
private Bitmap GetInvalidFrame(Bitmap frame) {
Stopwatch sp = new Stopwatch();
sp.Start();
if (m_FrameBackBuffer == null) {
return frame;
}
Int32 pixelsToRead = frame.Width * frame.Height;
Int32 x = 0, y = 0;
UnsafeBitmap unsafeBitmap = new UnsafeBitmap(frame);
UnsafeBitmap unsafeBuffBitmap = new UnsafeBitmap(m_FrameBackBuffer);
UnsafeBitmap retVal = new UnsafeBitmap(frame.Width, frame.Height);
unsafeBitmap.LockBitmap();
unsafeBuffBitmap.LockBitmap();
retVal.LockBitmap();
do {
for (x = 0; x < frame.Width; x++) {
Pixel newPixel = unsafeBitmap.GetPixel(x, y);
Pixel oldPixel = unsafeBuffBitmap.GetPixel(x, y);
if (newPixel.Alpha != oldPixel.Alpha || newPixel.Red != oldPixel.Red || newPixel.Green != oldPixel.Green || newPixel.Blue != oldPixel.Blue) {
retVal.SetPixel(x, y, newPixel);
}
else {
// Skip pixel
}
}
y++;
} while (y != frame.Height);
unsafeBitmap.UnlockBitmap();
unsafeBuffBitmap.UnlockBitmap();
retVal.UnlockBitmap();
sp.Stop();
System.Diagnostics.Debug.WriteLine(sp.Elapsed.Milliseconds.ToString());
sp.Reset();
return retVal.Bitmap;
}
Is there any possible method/means/approach that I could speed this up to about 30ms? I can copy the screen in about 30ms using Graphics.CopyFromScreen(), so that produces approximately 30 frames each second. However, a program only runs as fast as its slower counterpart, so the 300ms delay in GetInvalidFrame, slows this down to about 1 - 3 frames each second. This isn't good for a meeting software.
Any advice, approaches, pointers in the right direction would be absolutely wonderful! Also, the code that is used to draw the bitmap on the client-side is below as well.
To comment on Dmitriy's answer/comment:
#region RootWorkItem
private ScreenClient m_RootWorkItem;
/// <summary>
/// Gets the RootWorkItem
/// </summary>
public ScreenClient RootWorkItem {
get {
if (m_RootWorkItem == null) {
m_RootWorkItem = new ScreenClient();
m_RootWorkItem.FrameRead += new EventHandler<FrameEventArgs>(RootWorkItem_FrameRead);
}
return m_RootWorkItem;
}
}
#endregion
private void RootWorkItem_FrameRead(Object sender, FrameEventArgs e) {
if (e.Frame != null) {
if (uxSurface.Image != null) {
Bitmap frame = (Bitmap)uxSurface.Image;
Graphics g = Graphics.FromImage(frame);
g.DrawImage(e.Frame, 0, 0); // Draw only updated pixels
uxSurface.Image = frame;
}
else {
uxSurface.Image = e.Frame; // Draw initial, full image
}
}
else {
uxSurface.Image = null;
}
}

Unsafe approach with usage of integers instead of Pixels and single loop:
private static Bitmap GetInvalidFrame(Bitmap oldFrame, Bitmap newFrame)
{
if (oldFrame.Size != newFrame.Size)
{
throw new ArgumentException();
}
Bitmap result = new Bitmap(oldFrame.Width, oldFrame.Height, oldFrame.PixelFormat);
Rectangle lockArea = new Rectangle(Point.Empty, oldFrame.Size);
PixelFormat format = PixelFormat.Format32bppArgb;
BitmapData oldData = oldFrame.LockBits(lockArea, ImageLockMode.ReadOnly, format);
BitmapData newData = newFrame.LockBits(lockArea, ImageLockMode.ReadOnly, format);
BitmapData resultData = result.LockBits(lockArea, ImageLockMode.WriteOnly, format);
int len = resultData.Height * Math.Abs(resultData.Stride) / 4;
unsafe
{
int* pOld = (int*)oldData.Scan0;
int* pNew = (int*)newData.Scan0;
int* pResult = (int*)resultData.Scan0;
for (int i = 0; i < len; i++)
{
int oldValue = *pOld++;
int newValue = *pNew++;
*pResult++ = oldValue != newValue ? newValue : 0 /* replace with 0xff << 24 if you need non-transparent black pixel */;
// *pResult++ = *pOld++ ^ *pNew++; // if you can use XORs.
}
}
oldFrame.UnlockBits(oldData);
newFrame.UnlockBits(newData);
result.UnlockBits(resultData);
return result;
}
I think you really can use XORed frames here and I hope that this can have better performance on both sides.
private static void XorFrames(Bitmap leftFrame, Bitmap rightFrame)
{
if (leftFrame.Size != rightFrame.Size)
{
throw new ArgumentException();
}
Rectangle lockArea = new Rectangle(Point.Empty, leftFrame.Size);
PixelFormat format = PixelFormat.Format32bppArgb;
BitmapData leftData = leftFrame.LockBits(lockArea, ImageLockMode.ReadWrite, format);
BitmapData rightData = rightFrame.LockBits(lockArea, ImageLockMode.ReadOnly, format);
int len = leftData.Height * Math.Abs(rightData.Stride) / 4;
unsafe
{
int* pLeft = (int*)leftData.Scan0;
int* pRight = (int*)rightData.Scan0;
for (int i = 0; i < len; i++)
{
*pLeft++ ^= *pRight++;
}
}
leftFrame.UnlockBits(leftData);
rightFrame.UnlockBits(rightData);
}
You can use this procedure on both sides in following way:
On server side you need to evaluate difference between old and new frame, send it to client and replace old frame by new. The server code should look something like this:
XorFrames(oldFrame, newFrame); // oldFrame ^= newFrame
Send(oldFrame); // send XOR of two frames
oldFrame = newFrame;
On client side you need to update your current frame with xor frame recieved from server:
XorFrames((Bitmap)uxSurface.Image, e.Frame);

Here: Utilizing the GPU with c# there are mentioned some librarys for using the GPU from C#.

Yes, you can do so by using unsafe code.
BitmapData d = l.LockBits(new Rectangle(0, 0, l.Width, l.Height), ImageLockMode.ReadOnly,l.PixelFormat);
IntPtr scan = d.Scan0;
unsafe
{
byte* p = (byte*)(void*)scan;
//dostuff
}
Check out http://www.codeproject.com/KB/GDI-plus/csharpgraphicfilters11.aspx for some basic examples of this kind of stuff. My code is based on that.
Note:
One of the reasons this will be much faster than yours is that you are separately comparing each channel instead of just comparing the entire byte using one operation. Similarly, changing PixelAt to give you a byte to facilitate this would probably give you an improvement.

Instead of checking each and every pixel, you can just perform a basic memory compare of the 2 bitmaps. In C, something like memcmp().
This would give you a much quicker test to let you know that the images are the same or not. Only when you know they are different do you need to resort to the more expensive code that will help you determine where they are different (if you even need to know that).
I am not a C# person though, so I don't know how easy it is to get access to the raw memory.

Was able to slice off about 60ms. I think this is going to require the GPU. I'm not seeing any solution to this utilizing the CPU, even by comparing more than one byte/pixel at a time, unless someone can whip up a code sample to show me otherwise. Still sits at about 200-260ms, far too slow for 30fps.
private static BitmapData m_OldData;
private static BitmapData m_NewData;
private static unsafe Byte* m_OldPBase;
private static unsafe Byte* m_NewPBase;
private static unsafe Pixel* m_OldPixel;
private static unsafe Pixel* m_NewPixel;
private static Int32 m_X;
private static Int32 m_Y;
private static Stopwatch m_Watch = new Stopwatch();
private static GraphicsUnit m_GraphicsUnit = GraphicsUnit.Pixel;
private static RectangleF m_OldBoundsF;
private static RectangleF m_NewBoundsF;
private static Rectangle m_OldBounds;
private static Rectangle m_NewBounds;
private static Pixel m_TransparentPixel = new Pixel() { Alpha = 0x00, Red = 0, Green = 0, Blue = 0 };
private Bitmap GetInvalidFrame(Bitmap frame) {
if (m_FrameBackBuffer == null) {
return frame;
}
m_Watch.Start();
unsafe {
m_OldBoundsF = m_FrameBackBuffer.GetBounds(ref m_GraphicsUnit);
m_OldBounds = new Rectangle((Int32)m_OldBoundsF.X, (Int32)m_OldBoundsF.Y, (Int32)m_OldBoundsF.Width, (Int32)m_OldBoundsF.Height);
m_OldData = m_FrameBackBuffer.LockBits(m_OldBounds, ImageLockMode.ReadOnly, PixelFormat.Format32bppArgb);
m_NewBoundsF = m_FrameBackBuffer.GetBounds(ref m_GraphicsUnit);
m_NewBounds = new Rectangle((Int32)m_NewBoundsF.X, (Int32)m_NewBoundsF.Y, (Int32)m_NewBoundsF.Width, (Int32)m_NewBoundsF.Height);
m_NewData = frame.LockBits(m_NewBounds, ImageLockMode.ReadWrite, PixelFormat.Format32bppArgb);
m_OldPBase = (Byte*)m_OldData.Scan0.ToPointer();
m_NewPBase = (Byte*)m_NewData.Scan0.ToPointer();
do {
for (m_X = 0; m_X < frame.Width; m_X++) {
m_OldPixel = (Pixel*)(m_OldPBase + m_Y * m_OldData.Stride + 1 + m_X * sizeof(Pixel));
m_NewPixel = (Pixel*)(m_NewPBase + m_Y * m_NewData.Stride + 1 + m_X * sizeof(Pixel));
if (m_OldPixel->Alpha == m_NewPixel->Alpha // AccessViolationException accessing Property in get {}
|| m_OldPixel->Red == m_NewPixel->Red
|| m_OldPixel->Green == m_NewPixel->Green
|| m_OldPixel->Blue == m_NewPixel->Blue) {
// Set the transparent pixel
*m_NewPixel = m_TransparentPixel;
}
}
m_Y++; //Debug.WriteLine(String.Format("X: {0}, Y: {1}", m_X, m_Y));
} while (m_Y < frame.Height);
}
m_Y = 0;
m_Watch.Stop();
Debug.WriteLine("Time elapsed: " + m_Watch.ElapsedMilliseconds.ToString());
m_Watch.Reset();
return frame;
}