Develop Reference

Capture image from screen and get colors

I'm making a program which can capture a small area on screen and will run something if there is any color on image that match the target colors. My program run as the following Sequence:
Get image from a specific area from screen
Save to a folder
using CountPixel to detect any target_color
However, after I click the button5 twice times (not double click), it through an exception at below line :
b.Save(#"C:\Applications\CaptureImage000.jpg", ImageFormat.Jpeg);
Exception :
An unhandled exception of type
'System.Runtime.InteropServices.ExternalException' occurred in
System.Drawing.dll
Additional information: A generic error occurred in GDI+
My questions are :
How can i fix this exception ?
I want to use another method instead of CountPixel() to improve performance, because I just need to detect only one target color to rise event.
Step 2 is troublesome. I wonder if i can skip it and use the other way to call: (#"C:\Applications\CaptureImage000.jpg", ImageFormat.Jpeg) , because using this long string isn't comfortable and result error when im trying to use with GetPixel,... or add it into some "value example" code on internet for improvement purpose.
private int CountPixels(Bitmap bm, Color target_color)
{
// Loop through the pixels.
int matches = 0;
for (int y = 0; y < bm.Height; y++)
{
for (int x = 0; x < bm.Width; x++)
{
if (bm.GetPixel(x, y) == target_color) matches++;
}
}
return matches;
}
private Bitmap CapturedImage(int x, int y)
{
Bitmap b = new Bitmap(XX, YY);
Graphics g = Graphics.FromImage(b);
g.CopyFromScreen(x, y, 0, 0, new Size(XX, YY));
b.Save(#"C:\Applications\CaptureImage000.jpg", ImageFormat.Jpeg);
/* Run 3 line below will lead to question 1 - through exception
Bitmap bm = new Bitmap(#"C:\Applications\CaptureImage000.jpg");
int black_pixels = CountPixels(b, Color.FromArgb(255, 0, 0, 0));
textBox3.Text = black_pixels + " black pixels";
*/
return b;
}
private void button5_Click(object sender, EventArgs e)// Do screen cap
{
Bitmap bmp = null;
bmp = CapturedImage(X0, Y0);
}

[EDIT] Worked on this tonight with OP, made some improvements
This now accounts for endianness of the machine and correctly compares colors by converting them to integers with the Color.ToArgb() function
the below code will work, I have added comments for clarity and given you some options. I wrote the code without an IDE but I am confident it is fine.
In both cases below, just keep the handle to the bitmap, don't need to save and reopen regardless of if you need a copy.
Exception issue and improvements to CapturedImage function
option A (recommended)
Don't save the bitmap, you already have a handle, the graphics object just modified the BMP. Just leave the below code as is for this function and it will work fine without un-commenting one of the other options.
Code and other options:
private Bitmap CapturedImage(Bitmap bm, int x, int y)
{
Bitmap b = new Bitmap(XX, YY);
Graphics g = Graphics.FromImage(b);
g.CopyFromScreen(x, y, 0, 0, new Size(XX, YY));
//option B - If you DO need to keep a copy of the image use PNG and delete the old image
/*
try
{
if(System.IO.File.Exists(#"C:\Applications\CaptureImage.png"))
{
System.IO.File.Delete(#"C:\Applications\CaptureImage.png");
}
b.Save(#"C:\Applications\CaptureImage.png", ImageFormat.Png);
}
catch (System.Exception ex)
{
MessageBox.Show("There was a problem trying to save the image, is the file in open in another program?\r\nError:\r\n\r\n" + ex.Message);
}
*/
//option C - If you DO need to keep a copy of the image AND keep all copies of all images when you click the button use PNG and generate unique filename
/*
int id = 0;
while(System.IO.File.Exists(#"C:\Applications\CaptureImage" + id.ToString().PadLeft('0',4) + ".png"))
{
//increment the id until a unique file name is found
id++;
}
b.Save(#"C:\Applications\CaptureImage" + id.ToString().PadLeft('0',4) + ".png", ImageFormat.Png);
*/
int black_pixels = CountPixels(b, Color.FromArgb(255, 0, 0, 0));
textBox3.Text = black_pixels + " black pixels";
return b;
}
Now for the CountPixels function, you have 3 options but really, you have one really solid option, so I am omitting the others.
This locks the bits in the BMP, uses marshalling to copy the data into an array and scans the array for data, very, very fast, and you will likely not even need to remove the count. If you do STILL want to remove the count, just add "return 1;" right underneath where it increments the matches variable.
Speed issue and improvements to CountPixels function
private int CountPixels(Bitmap bm, Color target_color)
{
int matches = 0;
Bitmap bmp = (Bitmap)bm.Clone();
BitmapData bmpDat = bmp.LockBits(new Rectangle(0, 0, bmp.Width, bmp.Height), ImageLockMode.ReadWrite, bmp.PixelFormat);
int size = bmpDat.Stride * bmpDat.Height;
byte[] subPx = new byte[size];
System.Runtime.InteropServices.Marshal.Copy(bmpDat.Scan0, subPx, 0, size);
//change the 4 (ARGB) to a 3 (RGB) if you don't have an alpha channel, this is for 32bpp images
//ternary operator to check endianess of machine and organise pixel colors as A,R,G,B or B,G,R,A (little endian is reversed);
Color temp = BitConverter.IsLittleEndian ? Color.FromArgb(subPx[i + 2], subPx[i + 1], subPx[i]) : Color.FromArgb(subPx[i + 1], subPx[i + 2], subPx[i + 3]);
for (int i = 0; i < size; i += 4 ) //4 bytes per pixel A, R, G, B
{
if(temp.ToArgb() == target_color.ToArgb())
{
matches++;
}
}
System.Runtime.InteropServices.Marshal.Copy(subPx, 0, bmpDat.Scan0, subPx.Length);
bmp.UnlockBits(bmpDat);
return matches;
}
Finally the same function but allowing for a tolerance percent
private int CountPixels(Bitmap bm, Color target_color, float tolerancePercent)
{
int matches = 0;
Bitmap bmp = (Bitmap)bm.Clone();
BitmapData bmpDat = bmp.LockBits(new Rectangle(0, 0, bmp.Width, bmp.Height), ImageLockMode.ReadWrite, bmp.PixelFormat);
int size = bmpDat.Stride * bmpDat.Height;
byte[] subPx = new byte[size];
System.Runtime.InteropServices.Marshal.Copy(bmpDat.Scan0, subPx, 0, size);
for (int i = 0; i < size; i += 4 )
{
byte r = BitConverter.IsLittleEndian ? subPx[i+2] : subPx[i+3];
byte g = BitConverter.IsLittleEndian ? subPx[i+1] : subPx[i+2];
byte b = BitConverter.IsLittleEndian ? subPx[i] : subPx[i+1];
float distancePercent = (float)Math.Sqrt(
Math.Abs(target_color.R-r) +
Math.Abs(target_color.G-g) +
Math.Abs(target_color.B-b)
) / 7.65f;
if(distancePercent < tolerancePercent)
{
matches++;
}
}
System.Runtime.InteropServices.Marshal.Copy(subPx, 0, bmpDat.Scan0, subPx.Length);
bmp.UnlockBits(bmpDat);
return matches;
}

Crop image out of two points (C#)

Update:
Updated code added.
However for some reason it still doesn't work properly. If the coordinates are Bitmap coordinates, what could be the reason? The first code sample I put here doesn't work properly and the second gives me an OutOfMemoryException.
I've ran into a problem trying to crop an image between two points. In my project I have a pictureBox (named AP), and the general idea is that the user clicks on two points and the program crops the image between these two corners. I've tried two methods, one with Bitmap.Crop and the other one with Graphics.DrawImage, but both seemed to fail for the same reason and didn't work at all (cropped a much smaller portion of the image).
Code:
private void AP_Click(object sender, EventArgs e)
{
// Setting the corners
else if (mark_shape == 0)
{
var mouseEventArgs = e as MouseEventArgs;
if (picture_corners_set == 0)
{
northEast = AP.PointToClient(new Point(mouseEventArgs.X, mouseEventArgs.Y));
picture_corners_set = 1;
}
else if (picture_corners_set == 1)
{
southWest = AP.PointToClient(new Point(mouseEventArgs.X, mouseEventArgs.Y));
Rectangle imageRectangle = new Rectangle(southWest.X, northEast.Y, (northEast.X - southWest.X), (southWest.Y - northEast.Y));
var bmp = new Bitmap(imageRectangle.Width, imageRectangle.Height);
using (var gr = Graphics.FromImage(bmp))
{
gr.DrawImage(AP.Image, 0, 0, imageRectangle, GraphicsUnit.Pixel);
}
AP.Image = bmp;
enableAllButtons();
}
}
}

Since your cropped Bitmap image size is the same as the width/height of the user selection, I'm guessing you want that cropped image to be in the top/left of the new Bitmap and fill it. As it is, you're telling the DrawImage() method to draw that portion of the Bitmap in the same location, albeit in a smaller size Bitmap.
The correct way to do this is to draw the source rectangle image at (0, 0):
private Point pt1, pt2;
private void AP_Click(object sender, EventArgs e)
{
// ... obviously other code here ...
else if (mark_shape == 0) // Setting the corners
{
Point pt = AP.PointToClient(Cursor.Position);
if (picture_corners_set == 0)
{
pt1 = new Point(pt.X, pt.Y);
picture_corners_set = 1;
}
else if (picture_corners_set == 1)
{
pt2 = new Point(pt.X, pt.Y);
picture_corners_set = 0;
Rectangle imageRectangle = new Rectangle(new Point(Math.Min(pt1.X, pt2.X), Math.Min(pt1.Y, pt2.Y)), new Size(Math.Abs(pt2.X - pt1.X) + 1, Math.Abs(pt2.Y - pt1.Y) + 1));
var bmp = new Bitmap(imageRectangle.Width, imageRectangle.Height);
using (var gr = Graphics.FromImage(bmp))
{
gr.DrawImage(AP.Image, 0, 0, imageRectangle, GraphicsUnit.Pixel);
}
AP.Image = bmp;
enableAllButtons();
}
}
}
There are several other overloads you could use to do this, but the one above makes it pretty clear that imageRectangle is being drawn at (0, 0).

Image processing with lockbits, alternative to getpixel?

I am trying to increase my image detection class using lockbits, yet this cause problems with the code and thus it does not run. How can i go about using lockbits and getpixel at the same time in order to speed up image detection, or can someone show me an alternative which is just as fast?
code:
static IntPtr Iptr = IntPtr.Zero;
static BitmapData bitmapData = null;
static public byte[] Pixels { get; set; }
static public int Depth { get; private set; }
static public int Width { get; private set; }
static public int Height { get; private set; }
static public void LockBits(Bitmap source)
{
// Get width and height of bitmap
Width = source.Width;
Height = source.Height;
// get total locked pixels count
int PixelCount = Width * Height;
// Create rectangle to lock
Rectangle rect = new Rectangle(0, 0, Width, Height);
// get source bitmap pixel format size
Depth = System.Drawing.Bitmap.GetPixelFormatSize(source.PixelFormat);
// Lock bitmap and return bitmap data
bitmapData = source.LockBits(rect, ImageLockMode.ReadWrite,
source.PixelFormat);
// create byte array to copy pixel values
int step = Depth / 8;
Pixels = new byte[PixelCount * step];
Iptr = bitmapData.Scan0;
// Copy data from pointer to array
Marshal.Copy(Iptr, Pixels, 0, Pixels.Length);
}
static public bool SimilarColors(int R1, int G1, int B1, int R2, int G2, int B2, int Tolerance)
{
bool returnValue = true;
if (Math.Abs(R1 - R2) > Tolerance || Math.Abs(G1 - G2) > Tolerance || Math.Abs(B1 - B2) > Tolerance)
{
returnValue = false;
}
return returnValue;
}
public bool findImage(Bitmap small, Bitmap large, out Point location)
{
unsafe
{
LockBits(small);
LockBits(large);
//Loop through large images width
for (int largeX = 0; largeX < large.Width; largeX++)
{
//And height
for (int largeY = 0; largeY < large.Height; largeY++)
{
//Loop through the small width
for (int smallX = 0; smallX < small.Width; smallX++)
{
//And height
for (int smallY = 0; smallY < small.Height; smallY++)
{
//Get current pixels for both image
Color currentSmall = small.GetPixel(smallX, smallY);
Color currentLarge = large.GetPixel(largeX + smallX, largeY + smallY);
//If they dont match (i.e. the image is not there)
if (!colorsMatch(currentSmall, currentLarge))
//Goto the next pixel in the large image
goto nextLoop;
}
}
//If all the pixels match up, then return true and change Point location to the top left co-ordinates where it was found
location = new Point(largeX, largeY);
return true;
//Go to next pixel on large image
nextLoop:
continue;
}
}
//Return false if image is not found, and set an empty point
location = Point.Empty;
return false;
}
}

You wouldn't want to rely on getPixel() for image processing; it's okay to make an occasional call to get a point value (e.g. on mouseover), but in general it's preferable to do image processing in image memory or in some 2D array that you can convert to a Bitmap when necessary.
To start, you might try writing a method that using LockBits/UnlockBits to extract an array that is convenient to manipulate. Once you're done manipulating the array, you can write it back to a bitmap using a different LockBits/UnlockBits function.
Here's some sample code I've used in the past. The first function returns a 1D array of values from a Bitmap. Since you know the bitmap's width, you can convert this 1D array to a 2D array for further processing. Once you're done processing, you can call the second function to convert the (modified) 1D array into a bitmap again.
public static byte[] Array1DFromBitmap(Bitmap bmp){
if (bmp == null) throw new NullReferenceException("Bitmap is null");
Rectangle rect = new Rectangle(0, 0, bmp.Width, bmp.Height);
BitmapData data = bmp.LockBits(rect, ImageLockMode.ReadWrite, bmp.PixelFormat);
IntPtr ptr = data.Scan0;
//declare an array to hold the bytes of the bitmap
int numBytes = data.Stride * bmp.Height;
byte[] bytes = new byte[numBytes];
//copy the RGB values into the array
System.Runtime.InteropServices.Marshal.Copy(ptr, bytes, 0, numBytes);
bmp.UnlockBits(data);
return bytes;
}
public static Bitmap BitmapFromArray1D(byte[] bytes, int width, int height)
{
Bitmap grayBmp = new Bitmap(width, height, PixelFormat.Format8bppIndexed);
Rectangle grayRect = new Rectangle(0, 0, grayBmp.Width, grayBmp.Height);
BitmapData grayData = grayBmp.LockBits(grayRect, ImageLockMode.ReadWrite, grayBmp.PixelFormat);
IntPtr grayPtr = grayData.Scan0;
int grayBytes = grayData.Stride * grayBmp.Height;
ColorPalette pal = grayBmp.Palette;
for (int g = 0; g < 256; g++){
pal.Entries[g] = Color.FromArgb(g, g, g);
}
grayBmp.Palette = pal;
System.Runtime.InteropServices.Marshal.Copy(bytes, 0, grayPtr, grayBytes);
grayBmp.UnlockBits(grayData);
return grayBmp;
}
These methods makes assumptions about the Bitmap pixel format that may not work for you, but I hope the general idea is clear: use LockBits/UnlockBits to extract an array of bytes from a Bitmap so that you can write and debug algorithms most easily, and then use LockBits/UnlockBits again to write the array to a Bitmap again.
For portability, I would recommend that your methods return the desired data types rather than manipulating global variables within the methods themselves.
If you've been using getPixel(), then converting to/from arrays as described above could speed up your code considerably for a small investment of coding effort.

Ok where to start. Better you understand what you are doing with lockBits.
First of all make sure, that you dont overwrite your byte array with.
LockBits(small);
LockBits(large);
due to the second call all the first call does is locking your image and that is not good since you doesn't unlock it again.
So add another byte array that represents the image.
You can do something like this
LockBits(small, true);
LockBits(large, false);
and change your Lockbits method
static public void LockBits(Bitmap source, bool flag)
{
...
Marshal.Copy(Iptr, Pixels, 0, Pixels.Length);
if(flag)
PixelsSmall=Pixels;
else
PixelsLarge=Pixels;
}
where PixelsLarge and PixelsSmall are globals and Pixels isn't
Those 2 contain your image. Now you have to compare it.
Now you have to compare each "set of bytes" therefore you have to know the Pixelformat.
Is it 32b/pix 24 or only 8 (ARGB,RGB,grayscale)
Let's take ARGB images. In this case a set would consist of 4 bytes (=32/8)
I am not sure about the order but I think the order of one set is ABGR or BGRA.
Hope this may help you. If you don't figure out how to compare the right pixels then ask again. Ah and dont forget to use the UnlockBits command.

How to access information in a thread with a timer?

I created a form in C# where there is a image which is being converted into a boolean array. I am trying to generate a thread with a timer where the image is converted 4 times a second.
When I debug it, it works, I can trace the ticks from the timer. But when the form is running the application quits without giving a bug.
This is the initialize script:
form = new LoadForm();
form.Show();
form.BringToFront();
timer = new System.Threading.Timer(new TimerCallback(camTick), null, 0, 250);
This is the tick that works:
private void camTick(Object myObject)
{
if (form.isRunning)
{
bool[,] ar = form.getBoolBitmap(100);
}
}
This is the function that loads and saves the bitmap. In form1.cs
public bool[,] getBoolBitmap(uint threshold) {
unsafe {
Bitmap b = getBitmap();
BitmapData originalData = b.LockBits(new Rectangle(0, 0, b.Width, b.Height), ImageLockMode.ReadOnly, PixelFormat.Format24bppRgb);
bool[,] ar = new bool[b.Width, b.Height];
for (int y = 0; y < b.Height; y++) {
byte* Row = (byte*)originalData.Scan0 + (y * originalData.Stride);
for (int x = 0; x < b.Width; x++) {
if ((byte)Row[x * 3] < threshold) {
ar[x, y] = false;
} else {
ar[x, y] = true;
}
}
}
b.Dispose();
return ar;
}
}
public Bitmap getBitmap() {
if (!panelVideoPreview.IsDisposed) {
Bitmap b = new Bitmap(panelVideoPreview.Width, panelVideoPreview.Height, PixelFormat.Format24bppRgb);
using (Graphics g = Graphics.FromImage(b)) {
Rectangle rectanglePanelVideoPreview = panelVideoPreview.Bounds;
Point sourcePoints = panelVideoPreview.PointToScreen(new Point(panelVideoPreview.ClientRectangle.X, panelVideoPreview.ClientRectangle.Y));
g.CopyFromScreen(sourcePoints, Point.Empty, rectanglePanelVideoPreview.Size);
}
return b;
} else {
Bitmap b = new Bitmap(panelVideoPreview.Width, panelVideoPreview.Height);
return b;
}
}

Is this image being stored in a Control like PictureBox? If so, make sure you're only accessing it on the control's thread. Check out Control.Invoke().

The callback from System.Threading.Timer runs on a ThreadPool thread. In that callback you are accessing a Form instance. You simply cannot do this. Well, you can, but it will not work correctly. It will fail unpredictable and sometimes spectacularly.
Use System.Windows.Forms.Timer instead. Have it tick every 4 seconds. In the Tick event get whatever data you need from the form and then pass it off to another thread for further processing.
In the following code I get the Bitmap object by calling DrawToBitmap on the UI thread. I then pass the Bitmap off to a Task where it can be converted into a bool[] in the background. Optionally, you can return that bool[] from the Task and then call ContinueWith to transfer it back to the UI thread if necessary (sounds like you probably do not need to do that though).
private void YourWindowsFormsTimer_Tick(object sender, EventArgs args)
{
// Get the bitmap object while still on the UI thread.
var bm = new Bitmap(panelVideoPreview.ClientSize.Width, panelVideoPreview.ClientSize.Height, PixelFormat.Format24bppRgb);
panelVideoPreview.DrawToBitmap(bm, panelVideoPreview.ClientRectangle);
Task.Factory
.StartNew(() =>
{
// It is not safe to access the UI here.
bool[,] ar = ConvertBitmapToByteArray(bm);
DoSomethingWithTheArrayHereIfNecessary(ar);
// Optionally return the byte array if you need to transfer it to the UI thread.
return ar;
})
.ContinueWith((task) =>
{
// It is safe to access the UI here.
// Get the returned byte array.
bool[,] ar = task.Result;
UpdateSomeControlIfNecessaryHere(ar);
}, TaskScheduler.FromCurrentSynchronizationContext());
}
And ConvertBitmapToByteArray would look like this now.
public unsafe bool[,] ConvertBitmapToBoolArray(Bitmap b, uint threshold)
{
BitmapData originalData = b.LockBits(new Rectangle(0, 0, b.Width, b.Height), ImageLockMode.ReadOnly, PixelFormat.Format24bppRgb);
bool[,] ar = new bool[b.Width, b.Height];
for (int y = 0; y < b.Height; y++)
{
byte* Row = (byte*)originalData.Scan0 + (y * originalData.Stride);
for (int x = 0; x < b.Width; x++)
{
if ((byte)Row[x * 3] < treshold)
{
ar[x, y] = false;
} else
{
ar[x, y] = true;
}
}
}
b.Dispose();
return ar;
}

After some searching on the internet I found this out, I needed to make another function to return the bool[,], this is what I came up with:
public bool[,] invokeBitmap(uint treshold) {
if (this.InvokeRequired) {
return (bool[,])this.Invoke((Func<bool[,]>)delegate {
return getBoolBitmap(treshold);
});
} else {
return getBoolBitmap(treshold);
}
}

Unsafe Per Pixel access, 30ms access for 1756000 pixels

So I've been sharing some thoughts on the above topic title on my website about fast, unsafe pixel access. A gentlemen gave me a rough example of how he'd do it in C++, but that doesn't help me in C# unless I can interop it, and the interop is fast as well. I had found a class in the internet that was written using MSDN help, to unsafely access pixels. The class is exceptionally fast, but it's not fast enough. Here's the class:
using System;
using System.Collections.Generic;
using System.Text;
using System.Drawing;
using System.Drawing.Imaging;
namespace DCOMProductions.Desktop.ScreenViewer {
public unsafe class UnsafeBitmap {
Bitmap bitmap;
// three elements used for MakeGreyUnsafe
int width;
BitmapData bitmapData = null;
Byte* pBase = null;
public UnsafeBitmap(Bitmap bitmap) {
this.bitmap = new Bitmap(bitmap);
}
public UnsafeBitmap(int width, int height) {
this.bitmap = new Bitmap(width, height, PixelFormat.Format32bppArgb);
}
public void Dispose() {
bitmap.Dispose();
}
public Bitmap Bitmap {
get {
return (bitmap);
}
}
private Point PixelSize {
get {
GraphicsUnit unit = GraphicsUnit.Pixel;
RectangleF bounds = bitmap.GetBounds(ref unit);
return new Point((int)bounds.Width, (int)bounds.Height);
}
}
public void LockBitmap() {
GraphicsUnit unit = GraphicsUnit.Pixel;
RectangleF boundsF = bitmap.GetBounds(ref unit);
Rectangle bounds = new Rectangle((int)boundsF.X,
(int)boundsF.Y,
(int)boundsF.Width,
(int)boundsF.Height);
// Figure out the number of bytes in a row
// This is rounded up to be a multiple of 4
// bytes, since a scan line in an image must always be a multiple of 4 bytes
// in length.
width = (int)boundsF.Width * sizeof(Pixel);
if (width % 4 != 0) {
width = 4 * (width / 4 + 1);
}
bitmapData =
bitmap.LockBits(bounds, ImageLockMode.ReadWrite, PixelFormat.Format32bppArgb);
pBase = (Byte*)bitmapData.Scan0.ToPointer();
}
public Pixel GetPixel(int x, int y) {
Pixel returnValue = *PixelAt(x, y);
return returnValue;
}
public void SetPixel(int x, int y, Pixel colour) {
Pixel* pixel = PixelAt(x, y);
*pixel = colour;
}
public void UnlockBitmap() {
bitmap.UnlockBits(bitmapData);
bitmapData = null;
pBase = null;
}
public Pixel* PixelAt(int x, int y) {
return (Pixel*)(pBase + y * width + x * sizeof(Pixel));
}
}
}
Basically what I am doing is copying the entire screen and comparing each pixel to and old copy. On a 1680x1050 bitmap, this takes approximately 300 milliseconds using the following code.
private Bitmap GetInvalidFrame(Bitmap frame) {
Stopwatch sp = new Stopwatch();
sp.Start();
if (m_FrameBackBuffer == null) {
return frame;
}
Int32 pixelsToRead = frame.Width * frame.Height;
Int32 x = 0, y = 0;
UnsafeBitmap unsafeBitmap = new UnsafeBitmap(frame);
UnsafeBitmap unsafeBuffBitmap = new UnsafeBitmap(m_FrameBackBuffer);
UnsafeBitmap retVal = new UnsafeBitmap(frame.Width, frame.Height);
unsafeBitmap.LockBitmap();
unsafeBuffBitmap.LockBitmap();
retVal.LockBitmap();
do {
for (x = 0; x < frame.Width; x++) {
Pixel newPixel = unsafeBitmap.GetPixel(x, y);
Pixel oldPixel = unsafeBuffBitmap.GetPixel(x, y);
if (newPixel.Alpha != oldPixel.Alpha || newPixel.Red != oldPixel.Red || newPixel.Green != oldPixel.Green || newPixel.Blue != oldPixel.Blue) {
retVal.SetPixel(x, y, newPixel);
}
else {
// Skip pixel
}
}
y++;
} while (y != frame.Height);
unsafeBitmap.UnlockBitmap();
unsafeBuffBitmap.UnlockBitmap();
retVal.UnlockBitmap();
sp.Stop();
System.Diagnostics.Debug.WriteLine(sp.Elapsed.Milliseconds.ToString());
sp.Reset();
return retVal.Bitmap;
}
Is there any possible method/means/approach that I could speed this up to about 30ms? I can copy the screen in about 30ms using Graphics.CopyFromScreen(), so that produces approximately 30 frames each second. However, a program only runs as fast as its slower counterpart, so the 300ms delay in GetInvalidFrame, slows this down to about 1 - 3 frames each second. This isn't good for a meeting software.
Any advice, approaches, pointers in the right direction would be absolutely wonderful! Also, the code that is used to draw the bitmap on the client-side is below as well.
To comment on Dmitriy's answer/comment:
#region RootWorkItem
private ScreenClient m_RootWorkItem;
/// <summary>
/// Gets the RootWorkItem
/// </summary>
public ScreenClient RootWorkItem {
get {
if (m_RootWorkItem == null) {
m_RootWorkItem = new ScreenClient();
m_RootWorkItem.FrameRead += new EventHandler<FrameEventArgs>(RootWorkItem_FrameRead);
}
return m_RootWorkItem;
}
}
#endregion
private void RootWorkItem_FrameRead(Object sender, FrameEventArgs e) {
if (e.Frame != null) {
if (uxSurface.Image != null) {
Bitmap frame = (Bitmap)uxSurface.Image;
Graphics g = Graphics.FromImage(frame);
g.DrawImage(e.Frame, 0, 0); // Draw only updated pixels
uxSurface.Image = frame;
}
else {
uxSurface.Image = e.Frame; // Draw initial, full image
}
}
else {
uxSurface.Image = null;
}
}

Unsafe approach with usage of integers instead of Pixels and single loop:
private static Bitmap GetInvalidFrame(Bitmap oldFrame, Bitmap newFrame)
{
if (oldFrame.Size != newFrame.Size)
{
throw new ArgumentException();
}
Bitmap result = new Bitmap(oldFrame.Width, oldFrame.Height, oldFrame.PixelFormat);
Rectangle lockArea = new Rectangle(Point.Empty, oldFrame.Size);
PixelFormat format = PixelFormat.Format32bppArgb;
BitmapData oldData = oldFrame.LockBits(lockArea, ImageLockMode.ReadOnly, format);
BitmapData newData = newFrame.LockBits(lockArea, ImageLockMode.ReadOnly, format);
BitmapData resultData = result.LockBits(lockArea, ImageLockMode.WriteOnly, format);
int len = resultData.Height * Math.Abs(resultData.Stride) / 4;
unsafe
{
int* pOld = (int*)oldData.Scan0;
int* pNew = (int*)newData.Scan0;
int* pResult = (int*)resultData.Scan0;
for (int i = 0; i < len; i++)
{
int oldValue = *pOld++;
int newValue = *pNew++;
*pResult++ = oldValue != newValue ? newValue : 0 /* replace with 0xff << 24 if you need non-transparent black pixel */;
// *pResult++ = *pOld++ ^ *pNew++; // if you can use XORs.
}
}
oldFrame.UnlockBits(oldData);
newFrame.UnlockBits(newData);
result.UnlockBits(resultData);
return result;
}
I think you really can use XORed frames here and I hope that this can have better performance on both sides.
private static void XorFrames(Bitmap leftFrame, Bitmap rightFrame)
{
if (leftFrame.Size != rightFrame.Size)
{
throw new ArgumentException();
}
Rectangle lockArea = new Rectangle(Point.Empty, leftFrame.Size);
PixelFormat format = PixelFormat.Format32bppArgb;
BitmapData leftData = leftFrame.LockBits(lockArea, ImageLockMode.ReadWrite, format);
BitmapData rightData = rightFrame.LockBits(lockArea, ImageLockMode.ReadOnly, format);
int len = leftData.Height * Math.Abs(rightData.Stride) / 4;
unsafe
{
int* pLeft = (int*)leftData.Scan0;
int* pRight = (int*)rightData.Scan0;
for (int i = 0; i < len; i++)
{
*pLeft++ ^= *pRight++;
}
}
leftFrame.UnlockBits(leftData);
rightFrame.UnlockBits(rightData);
}
You can use this procedure on both sides in following way:
On server side you need to evaluate difference between old and new frame, send it to client and replace old frame by new. The server code should look something like this:
XorFrames(oldFrame, newFrame); // oldFrame ^= newFrame
Send(oldFrame); // send XOR of two frames
oldFrame = newFrame;
On client side you need to update your current frame with xor frame recieved from server:
XorFrames((Bitmap)uxSurface.Image, e.Frame);

Here: Utilizing the GPU with c# there are mentioned some librarys for using the GPU from C#.

Yes, you can do so by using unsafe code.
BitmapData d = l.LockBits(new Rectangle(0, 0, l.Width, l.Height), ImageLockMode.ReadOnly,l.PixelFormat);
IntPtr scan = d.Scan0;
unsafe
{
byte* p = (byte*)(void*)scan;
//dostuff
}
Check out http://www.codeproject.com/KB/GDI-plus/csharpgraphicfilters11.aspx for some basic examples of this kind of stuff. My code is based on that.
Note:
One of the reasons this will be much faster than yours is that you are separately comparing each channel instead of just comparing the entire byte using one operation. Similarly, changing PixelAt to give you a byte to facilitate this would probably give you an improvement.

Instead of checking each and every pixel, you can just perform a basic memory compare of the 2 bitmaps. In C, something like memcmp().
This would give you a much quicker test to let you know that the images are the same or not. Only when you know they are different do you need to resort to the more expensive code that will help you determine where they are different (if you even need to know that).
I am not a C# person though, so I don't know how easy it is to get access to the raw memory.

Was able to slice off about 60ms. I think this is going to require the GPU. I'm not seeing any solution to this utilizing the CPU, even by comparing more than one byte/pixel at a time, unless someone can whip up a code sample to show me otherwise. Still sits at about 200-260ms, far too slow for 30fps.
private static BitmapData m_OldData;
private static BitmapData m_NewData;
private static unsafe Byte* m_OldPBase;
private static unsafe Byte* m_NewPBase;
private static unsafe Pixel* m_OldPixel;
private static unsafe Pixel* m_NewPixel;
private static Int32 m_X;
private static Int32 m_Y;
private static Stopwatch m_Watch = new Stopwatch();
private static GraphicsUnit m_GraphicsUnit = GraphicsUnit.Pixel;
private static RectangleF m_OldBoundsF;
private static RectangleF m_NewBoundsF;
private static Rectangle m_OldBounds;
private static Rectangle m_NewBounds;
private static Pixel m_TransparentPixel = new Pixel() { Alpha = 0x00, Red = 0, Green = 0, Blue = 0 };
private Bitmap GetInvalidFrame(Bitmap frame) {
if (m_FrameBackBuffer == null) {
return frame;
}
m_Watch.Start();
unsafe {
m_OldBoundsF = m_FrameBackBuffer.GetBounds(ref m_GraphicsUnit);
m_OldBounds = new Rectangle((Int32)m_OldBoundsF.X, (Int32)m_OldBoundsF.Y, (Int32)m_OldBoundsF.Width, (Int32)m_OldBoundsF.Height);
m_OldData = m_FrameBackBuffer.LockBits(m_OldBounds, ImageLockMode.ReadOnly, PixelFormat.Format32bppArgb);
m_NewBoundsF = m_FrameBackBuffer.GetBounds(ref m_GraphicsUnit);
m_NewBounds = new Rectangle((Int32)m_NewBoundsF.X, (Int32)m_NewBoundsF.Y, (Int32)m_NewBoundsF.Width, (Int32)m_NewBoundsF.Height);
m_NewData = frame.LockBits(m_NewBounds, ImageLockMode.ReadWrite, PixelFormat.Format32bppArgb);
m_OldPBase = (Byte*)m_OldData.Scan0.ToPointer();
m_NewPBase = (Byte*)m_NewData.Scan0.ToPointer();
do {
for (m_X = 0; m_X < frame.Width; m_X++) {
m_OldPixel = (Pixel*)(m_OldPBase + m_Y * m_OldData.Stride + 1 + m_X * sizeof(Pixel));
m_NewPixel = (Pixel*)(m_NewPBase + m_Y * m_NewData.Stride + 1 + m_X * sizeof(Pixel));
if (m_OldPixel->Alpha == m_NewPixel->Alpha // AccessViolationException accessing Property in get {}
|| m_OldPixel->Red == m_NewPixel->Red
|| m_OldPixel->Green == m_NewPixel->Green
|| m_OldPixel->Blue == m_NewPixel->Blue) {
// Set the transparent pixel
*m_NewPixel = m_TransparentPixel;
}
}
m_Y++; //Debug.WriteLine(String.Format("X: {0}, Y: {1}", m_X, m_Y));
} while (m_Y < frame.Height);
}
m_Y = 0;
m_Watch.Stop();
Debug.WriteLine("Time elapsed: " + m_Watch.ElapsedMilliseconds.ToString());
m_Watch.Reset();
return frame;
}