I try to do some stuff with PDF for a project. I'm trying to do some interop with Pdfium: https://pdfium.googlesource.com/pdfium/.
I want to convert a JPG image to a PDF (just for the start). So I found some resources on the web and I came to this:
///// My main file
// Create the stream
using FileStream image = new FileStream(#"test.jpg", FileMode.Open, FileAccess.Read);
using FileStream pdf = new FileStream(#"test.pdf", FileMode.OpenOrCreate, FileAccess.Write);
// Init the library to use it
FPDF.InitLibrary();
// Create the PDF
IntPtr document = FPDF.CreateNewDocument();
IntPtr page = FPDFPage.New(document, 0, 3000, 4000);
IntPtr pageObject = FPDFPageObj.NewImageObj(document);
// Create a copy of the delegate
FPDF.m_GetBlockDelegate del = Delegates.m_GetBlockDelegate;
FPDF.FILEACCESS fileAccess = new FPDF.FILEACCESS
{
m_FileLen = (ulong)image.Length,
m_Param = image.SafeFileHandle.DangerousGetHandle(),
m_GetBlock = del,
};
// Create the FILEWRITE struct
byte[] buffer = null;
FPDF.FILEWRITE fileWrite = new FPDF.FILEWRITE
{
WriteBlock = delegate (ref FPDF.FILEWRITE pThis, byte[] pData, ulong size)
{
if (buffer == null || (ulong)buffer.Length < size)
buffer = new byte[size];
pData.CopyTo(buffer, 0);
pdf.Write(buffer, 0, (int)size);
return true;
}
};
// Load the image
FPDPImageObj.LoadJpegFile(IntPtr.Zero, 0, pageObject, ref fileAccess);
FPDFPage.InsertObject(page, pageObject);
// Save the PDF
FPDF.SaveAsCopy(document, ref fileWrite, 0);
// Destroy the library
FPDF.DestroyLibrary();
The problem is with this method: FPDPImageObj.LoadJpegFile(IntPtr.Zero, 0, pageObject, ref fileAccess);.
The struct has a delegate as a callback function to do some stuff (it does a sort of stream implementation in C).
Here is the code of my delegate, struct, and the implementation of the delegate:
public struct FILEACCESS
{
public ulong m_FileLen { get; set; }
public m_GetBlockDelegate m_GetBlock { get; set; }
public IntPtr m_Param { get; set; }
}
public struct FILEWRITE
{
// Must be 1
public int version = 1;
public WriteBlockDelegate WriteBlock { get; set; }
}
public delegate int m_GetBlockDelegate(IntPtr param, ulong position, [In] byte[] pBuf, ulong size);
public delegate bool WriteBlockDelegate(ref FILEWRITE pThis, [In] byte[] pData, ulong size);
// The implementation of my delegate in another class
public static int m_GetBlockDelegate(IntPtr param, ulong position, [In] byte[] pBuf, ulong size)
{
// Original source code for the delegate
//FILE* fileHandle = (FILE*)param;
//fseek(imageFile, position, SEEK_SET);
//fread(pBuf, size, 1, fileHandle);
using FileStream fs = new FileStream(new SafeFileHandle(param, false), FileAccess.Read);
fs.Seek((long)position, SeekOrigin.Begin);
fs.Read(pBuf, 0, (int)size);
return (int)size;
}
The current problem is that pBuf should be a buffer to hold all the data to read. So it must be the same size as size. In my case pBuf has a size of 1. I have tested a C variant of my code and it works well (it reads all the data). Here is the code:
#include <stdio.h>
#include <fpdfview.h>
#include <fpdf_edit.h>
#include <fpdf_save.h>
FILE* pdfFile = nullptr;
int m_GetBlockImpl(void* param, unsigned long position, unsigned char* pBuf, unsigned long size)
{
FILE* fileHandle = (FILE*)param;
fseek(fileHandle, position, SEEK_SET);
fread(pBuf, size, 1, fileHandle);
return size;
}
int WriteBlockImpl(FPDF_FILEWRITE* pThis, const void* pData, unsigned long size)
{
fwrite(pData, size, 1, pdfFile);
return TRUE;
}
int main()
{
FILE* jpgFile = fopen("C:\\Users\\tcroi\\Desktop\\test.jpg", "rb");
pdfFile = fopen("C:\\Users\\tcroi\\Desktop\\test.pdf", "wb");
if (jpgFile == nullptr || pdfFile == nullptr)
return 1;
FPDF_InitLibrary();
// Create the PDF
FPDF_DOCUMENT pdf_doc = FPDF_CreateNewDocument();
FPDF_PAGE pdf_page = FPDFPage_New(pdf_doc, 0, 3000, 4000);
// Seek the stream and get the size
fseek(jpgFile, 0, SEEK_END);
long fileSize = ftell(jpgFile);
fseek(jpgFile, 0, SEEK_SET);
// Create the FILEACCESS
FPDF_FILEACCESS fileReadObj;
fileReadObj.m_FileLen = fileSize;
fileReadObj.m_Param = jpgFile;
fileReadObj.m_GetBlock = m_GetBlockImpl;
// Load the image to the PDF
FPDF_PAGEOBJECT page_obj_image = FPDFPageObj_NewImageObj(pdf_doc);
FPDFImageObj_LoadJpegFile(nullptr, 0, page_obj_image, &fileReadObj);
FPDFPage_InsertObject(pdf_page, page_obj_image);
// Create the FILEWRITE
FPDF_FILEWRITE fileWriteObj;
fileWriteObj.version = 1;
fileWriteObj.WriteBlock = WriteBlockImpl;
FPDF_SaveAsCopy(pdf_doc, &fileWriteObj, 0);
FPDF_DestroyLibrary();
fclose(jpgFile);
fclose(pdfFile);
return 0;
}
This code works well until I want to save the PDF, I only get a blank page, maybe I have missed something but according to example no. Here is the example: https://groups.google.com/g/pdfium/c/P73DgvJtgTs
You probably have to FPDFPage_GenerateContent the page after inserting the object and perhaps also FPDF_ClosePage the page.
Related
I'm trying to create a PInvoke for FbwfFindFirst and am struggling with the struct FbwfCacheDetail.
In short, I'm not sure how to marshal WCHAR fileName[1]; seeing as it's a variable length array and a non-null terminated.
Any help would be welcomed
Since the whole structure is of variable size, one way to do this is like this (I can't test it because I don't have this dll on my system):
string volume = "";
int size = 0;
// ask for whole structure size
FbwfFindFirst(volume, IntPtr.Zero, ref size); // this call should return ERROR_MORE_DATA which is ok
// allocate for the structure
var ptr = Marshal.AllocHGlobal(size);
try
{
FbwfFindFirst(volume, ptr, ref size); // should not return error
// get the easy part
var detail = Marshal.PtrToStructure<FbwfCacheDetail>(ptr);
// compute filename offset and get the string
// file name length is in bytes, per documentation
var fileName = Marshal.PtrToStringUni(ptr + Marshal.OffsetOf<FbwfCacheDetail>("fileName").ToInt32(), detail.fileNameLength / 2);
}
finally
{
Marshal.FreeHGlobal(ptr);
}
[DllImport("fbwflib", CharSet = CharSet.Unicode)]
static extern int FbwfFindFirst(string volume, IntPtr cacheDetail, ref int size);
[StructLayout(LayoutKind.Sequential)]
struct FbwfCacheDetail
{
public int cacheSize;
public int openHandleCount;
public int fileNameLength;
byte fileName; // don't use this member
}
Simon Mourier's answer is 99% correct, and with normal APIs it would have definitely worked, but it appears as if this particular API doesn't follow "normal rules", whatever that might be ;). As such, I needed to modify a couple things and this is what worked for me:
const int ERROR_INSUFFICIENT_BUFFER = 122;
const int ERROR_MORE_DATA = 234;
var volume = "C:";
var fileName = string.Empty;
var size = 0;
while (true)
{
var ptr = Marshal.AllocHGlobal(size); // FbwfFindFirst fails if given IntPtr.Zero - regardless of what the value of size is.
try
{
var result = FbwfFindFirst(volume, ptr, ref size);
// Despite documentation saying otherwise, it can return either of these
if (result == ERROR_MORE_DATA || result == ERROR_INSUFFICIENT_BUFFER)
{
continue;
}
if (result != 0)
{
throw new Exception($"Failed with {result}");
}
// get the easy part
var detail = (FbwfCacheDetail) Marshal.PtrToStructure(ptr, typeof(FbwfCacheDetail));
// compute filename offset and get the string
// file name length is in bytes, per documentation
fileName = Marshal.PtrToStringUni(ptr + Marshal.OffsetOf(typeof(FbwfCacheDetail), "fileName").ToInt32(), detail.fileNameLength / 2);
break;
}
finally
{
Marshal.FreeHGlobal(ptr);
}
}
EDIT
Forgot to say that the pinvoke for FbwfFindFirst needs to be as follows, else it will return ERROR_INVALID_PARAMETER:
[DllImport("fbwflib.dll")]
public static extern uint FbwfFindFirst(
[MarshalAs(UnmanagedType.LPWStr)]
string volume,
IntPtr cacheDetail,
ref int size
);
I want to record the audio stream from Kinect and save it in my computer , so I refer the sample code here.
http://channel9.msdn.com/Series/KinectQuickstart/Audio-Fundamentals
Here is my partial code.
public static void WriteWavHeader(Stream stream, int dataLength)
{
using (var memStream = new MemoryStream(64))
{
int cbFormat = 18; //sizeof(WAVEFORMATEX)
WAVEFORMATEX format = new WAVEFORMATEX()
{
//wFormatTag = 0,
nChannels = 1,
nSamplesPerSec = 16000,
nAvgBytesPerSec = 32000,
nBlockAlign = 2,
wBitsPerSample = 16,
cbSize = 0
};
using (var bw = new BinaryWriter(memStream))
{
//RIFF header
WriteString(memStream, "data");
bw.Write(cbFormat);
WriteString(memStream, "RIFF");
bw.Write(dataLength + cbFormat + 4); //File size - 8
WriteString(memStream, "WAVE");
WriteString(memStream, "fmt ");
bw.Write(cbFormat);
//WAVEFORMATEX
bw.Write(format.wFormatTag);
bw.Write(format.nChannels);
bw.Write(format.nSamplesPerSec);
bw.Write(format.nAvgBytesPerSec);
bw.Write(format.nBlockAlign);
bw.Write(format.wBitsPerSample);
bw.Write(format.cbSize);
//data header
WriteString(memStream, "data");
bw.Write(dataLength);
memStream.WriteTo(stream);
}
}
}
There's an error occur when in line "bw.Write(format.wFormatTag);" when I compile the code , but no error with other line in WAVEFORMATEX , I have add "Bass.Net" to my reference and using "Un4seen.Bass" , but I still can't fix it.
It seems you are mixing several different SDKs. In the link you provided, the WAVEFORMATEX is defined as follows:
struct WAVEFORMATEX
{
public ushort wFormatTag;
public ushort nChannels;
public uint nSamplesPerSec;
public uint nAvgBytesPerSec;
public ushort nBlockAlign;
public ushort wBitsPerSample;
public ushort cbSize;
}
So, no enum values or Un4seen.Bass references. The function to write the header file is slighty different (only differs in the wFormatTag value):
/// <summary>
/// A bare bones WAV file header writer
/// </summary>
static void WriteWavHeader(Stream stream, int dataLength)
{
//We need to use a memory stream because the BinaryWriter will close the underlying stream when it is closed
using (var memStream = new MemoryStream(64))
{
int cbFormat = 18; //sizeof(WAVEFORMATEX)
WAVEFORMATEX format = new WAVEFORMATEX()
{
wFormatTag = 1,
nChannels = 1,
nSamplesPerSec = 16000,
nAvgBytesPerSec = 32000,
nBlockAlign = 2,
wBitsPerSample = 16,
cbSize = 0
};
using (var bw = new BinaryWriter(memStream))
{
//RIFF header
WriteString(memStream, "RIFF");
bw.Write(dataLength + cbFormat + 4); //File size - 8
WriteString(memStream, "WAVE");
WriteString(memStream, "fmt ");
bw.Write(cbFormat);
//WAVEFORMATEX
bw.Write(format.wFormatTag);
bw.Write(format.nChannels);
bw.Write(format.nSamplesPerSec);
bw.Write(format.nAvgBytesPerSec);
bw.Write(format.nBlockAlign);
bw.Write(format.wBitsPerSample);
bw.Write(format.cbSize);
//data header
WriteString(memStream, "data");
bw.Write(dataLength);
memStream.WriteTo(stream);
}
}
}
An WriteString also takes care of converting it to ASCII. I'm refering to the .zip file which is referenced in the presentation above ("Download the Quickstart slides and samples", KinectforWindowsSDKV1\6.Audio\AudioRecorder\AudioRecorder\MainWindow.xaml.cs)
I have been banging my head against a problem for days now. I would like your help.
I am trying to interface to I2C from a board running Windows CE7. The board is a Boundary Devices Nitrogen6X.
I am trying to code this in C#.
After a lot of googling and trial and error I can now do almost everything with the I2C (by that I mean I wrapped most commands in methods that work). Of course, the one thing that I cannot do yet is Reading/Writing. I have been trying a few different implementations, porting C and C++ code that supposedly worked. To no avail. Currently I am putting more effort in the two implementations I will copy here.
Neither of these implementations work for me. Both enter the error management portion, and both report error number 87 (ERROR_INVALID_PARAMETER).
Does anyone have experience on these kind of issues? could someone point out what I am doing wrong?
Edit 1: I should probably mention that I am trying to "see" some signals on the SDA and SCL pins of I2C3: of the board by simply plugging an oscilloscope to them. There is no actual device connected on the I2C bus. I would expect this to give me some sort of error after the first byte (addres+Read/Write) has been sent, because no acknowledge bit would be received. However, I see that error 87 in my code, and no change on the signals as seen from the scope (both remain high on idle).
(Code snippets follow)
The first one uses pointers and stuff, and is probably closer to C++ code:
StructLayout(LayoutKind.Sequential)]
unsafe public struct UNSAFE_I2C_PACKET
{
//[MarshalAs(UnmanagedType.U1)]
public byte byAddr; //I2C slave device address
//[MarshalAs(UnmanagedType.U1)]
public byte byRw; //Read = I2C_Read or Write = I2C_Write
//[MarshalAs(UnmanagedType.LPArray)]
public byte* pbyBuf; //Message Buffer
//[MarshalAs(UnmanagedType.U2)]
public Int16 wLen; //Message Buffer Length
//[MarshalAs(UnmanagedType.LPStruct)]
public int* lpiResult; //Contain the result of last operation
}
[StructLayout(LayoutKind.Sequential)]
unsafe public struct UNSAFE_I2C_TRANSFER_BLOCK
{
//public I2C_PACKET pI2CPackets;
public UNSAFE_I2C_PACKET* pI2CPackets;
public Int32 iNumPackets;
}
[DllImport("coredll.dll", EntryPoint = "DeviceIoControl", SetLastError = true)]
unsafe internal static extern int DeviceIoControlCE(
IntPtr hDevice, //file handle to driver
uint dwIoControlCode, //a correct call to CTL_CODE
[In, Out]byte* lpInBuffer,
uint nInBufferSize,
byte* lpOutBuffer,
uint nOutBufferSize,
uint* lpBytesReturned,
int* lpOverlapped);
unsafe public void ReadI2C(byte* pBuf, int count)
{
int ret;
int iResult;
UNSAFE_I2C_TRANSFER_BLOCK I2CXferBlock = new UNSAFE_I2C_TRANSFER_BLOCK();
UNSAFE_I2C_PACKET i2cPckt = new UNSAFE_I2C_PACKET();
//fixed (byte* p = pBuf)
{
i2cPckt.pbyBuf = pBuf;// p;
i2cPckt.wLen = (Int16)count;
i2cPckt.byRw = I2C_READ;
i2cPckt.byAddr = BASE;
i2cPckt.lpiResult = &iResult;
I2CXferBlock.iNumPackets = 1;
//fixed (I2C_PACKET* pck = &i2cPckt)
{
I2CXferBlock.pI2CPackets = &i2cPckt; // pck;
//fixed (I2C_TRANSFER_BLOCK* tBlock = &I2CXferBlock)
{
if (DeviceIoControlCE(_i2cFile,
I2C_IOCTL_TRANSFER,
(byte*)&I2CXferBlock,//tBlock,
(uint)sizeof(UNSAFE_I2C_TRANSFER_BLOCK),//Marshal.SizeOf(I2CXferBlock),
null,
0,
null,
null) == 0)
{
int error = GetLastError();
diag("Errore nella TRANSFER");
diag(error.ToString());
}
}
}
}
}
The second option I am working on marshals stuff around between managed and unmanaged:
[StructLayout(LayoutKind.Sequential)]
public struct I2C_PACKET
//public class I2C_PACKET
{
//[MarshalAs(UnmanagedType.U1)]
public Byte byAddr; //I2C slave device address
//[MarshalAs(UnmanagedType.U1)]
public Byte byRw; //Read = I2C_Read or Write = I2C_Write
//[MarshalAs(UnmanagedType.LPArray)]
public IntPtr pbyBuf; //Message Buffer
//[MarshalAs(UnmanagedType.U2)]
public Int16 wLen; //Message Buffer Length
//[MarshalAs(UnmanagedType.LPStruct)]
public IntPtr lpiResult; //Contain the result of last operation
}
[StructLayout(LayoutKind.Sequential)]
public struct I2C_TRANSFER_BLOCK
{
//public I2C_PACKET pI2CPackets;
//[MarshalAs(UnmanagedType.LPArray)]
public IntPtr pI2CPackets;
//[MarshalAs(UnmanagedType.U4)]
public Int32 iNumPackets;
}
[DllImport("coredll.dll", EntryPoint = "DeviceIoControl", SetLastError = true)]
internal static extern int DeviceIoControlCE(
IntPtr hDevice, //file handle to driver
uint dwIoControlCode, //a correct call to CTL_CODE
[In] IntPtr lpInBuffer,
uint nInBufferSize,
[Out] IntPtr lpOutBuffer,
uint nOutBufferSize,
out uint lpBytesReturned,
IntPtr lpOverlapped);
unsafe public void ReadI2C(byte[] buffer)
{
int count = buffer.Length;
I2C_TRANSFER_BLOCK I2CXFerBlock = new I2C_TRANSFER_BLOCK();
I2C_PACKET I2CPckt = new I2C_PACKET();
I2CPckt.byAddr = BASE;
I2CPckt.byRw = I2C_READ;
I2CPckt.wLen = (Int16)count;
I2CPckt.lpiResult = Marshal.AllocHGlobal(sizeof(int));
I2CPckt.pbyBuf = Marshal.AllocHGlobal(count);
//GCHandle packet = GCHandle.Alloc(I2CPckt, GCHandleType.Pinned);
I2CXFerBlock.iNumPackets = 1;
I2CXFerBlock.pI2CPackets = Marshal.AllocHGlobal(Marshal.SizeOf(I2CPckt)); //(Marshal.SizeOf(typeof(I2C_PACKET)));// //(sizeof(I2C_PACKET));//
Marshal.StructureToPtr(I2CPckt, I2CXFerBlock.pI2CPackets, false);
//I2CXFerBlock.pI2CPackets = packet.AddrOfPinnedObject();
//GCHandle xferBlock = GCHandle.Alloc(I2CXFerBlock, GCHandleType.Pinned);
IntPtr xfer = Marshal.AllocHGlobal(Marshal.SizeOf(I2CXFerBlock)); //(sizeof(I2C_TRANSFER_BLOCK)); //
Marshal.StructureToPtr(I2CXFerBlock, xfer, false);
//IntPtr xfer = xferBlock.AddrOfPinnedObject();
uint size = (uint)sizeof(I2C_TRANSFER_BLOCK);//Marshal.SizeOf(I2CXFerBlock);
uint getCnt = 0;
if ((DeviceIoControlCE(_i2cFile,
I2C_IOCTL_TRANSFER,
xfer,
size,
xfer, //IntPtr.Zero,
size, //0,
out getCnt,
IntPtr.Zero)) == 0)
{
int error = GetLastError();
diag("Errore nella TRANSFER.");
diag(error.ToString());
}
else
{
//success
I2CXFerBlock = (I2C_TRANSFER_BLOCK)Marshal.PtrToStructure(xfer, typeof(I2C_TRANSFER_BLOCK));
I2CPckt = (I2C_PACKET)Marshal.PtrToStructure(I2CXFerBlock.pI2CPackets, typeof(I2C_PACKET));
Marshal.Copy(I2CPckt.pbyBuf, buffer, 0, count);
diag("Success in TRANSFER: " + buffer[0].ToString());
}
Marshal.FreeHGlobal(I2CPckt.pbyBuf);
Marshal.FreeHGlobal(I2CXFerBlock.pI2CPackets);
Marshal.FreeHGlobal(xfer);
//packet.Free();
//xferBlock.Free();
}
Edit 2: The (supposedly) working code I have (which I am unable to run) comes from drivers I have been given, which might be partially proprietary (hence I cannot share). However I found online the header for an I2C bus, that contains the following definition:
#define I2C_MACRO_TRANSFER(hDev, pI2CTransferBlock) \
(DeviceIoControl(hDev, I2C_IOCTL_TRANSFER, (PBYTE) pI2CTransferBlock, sizeof(I2C_TRANSFER_BLOCK), NULL, 0, NULL, NULL))
I initially tried giving "null" to parameters as it's done here, but I still got the same error code.
Edit 3: From the same driver, the struct definitions:
// I2C Packet
typedef struct
{
BYTE byAddr; // I2C slave device address for this I2C operation
BYTE byRW; // Read = I2C_READ or Write = I2C_WRITE
PBYTE pbyBuf; // Message Buffer
WORD wLen; // Message Buffer Length
LPINT lpiResult; // Contains the result of last operation
} I2C_PACKET, *PI2C_PACKET;
// I2C Transfer Block
typedef struct
{
I2C_PACKET *pI2CPackets;
INT32 iNumPackets;
} I2C_TRANSFER_BLOCK, *PI2C_TRANSFER_BLOCK;
Edit 4: I tried implementing version passing a ref to my structures, as #ctacke suggested in his comment. I still get the same error, so I guess I must have done womthing different from the way he thought it. Here is the snippet:
[StructLayout(LayoutKind.Sequential)]
public struct REF_I2C_PACKET //public class REF_I2C_PACKET //
{
//[MarshalAs(UnmanagedType.U1)]
public Byte byAddr; //I2C slave device address
//[MarshalAs(UnmanagedType.U1)]
public Byte byRw; //Read = I2C_Read or Write = I2C_Write
//[MarshalAs(UnmanagedType.LPArray)]
public IntPtr pbyBuf; //Message Buffer
//[MarshalAs(UnmanagedType.U2)]
public Int16 wLen; //Message Buffer Length
//[MarshalAs(UnmanagedType.LPStruct)]
public IntPtr lpiResult; //Contain the result of last operation
}
[StructLayout(LayoutKind.Sequential)]
public struct REF_I2C_TRANSFER_BLOCK //public class REF_I2C_TRANSFER_BLOCK //
{
//public I2C_PACKET pI2CPackets;
public IntPtr pI2CPackets;
//[MarshalAs(UnmanagedType.U4)]
public Int32 iNumPackets;
//[MarshalAs(UnmanagedType.LPArray)]
//[MarshalAs(UnmanagedType.ByValArray, ArraySubType = UnmanagedType.LPStruct, SizeConst = 2)]
//public REF_I2C_PACKET[] pI2CPackets;
}
[DllImport("coredll.dll", EntryPoint = "DeviceIoControl", SetLastError = true)]
unsafe internal static extern int DeviceIoControlCE(
IntPtr hDevice, //file handle to driver
uint dwIoControlCode, //a correct call to CTL_CODE
//[MarshalAs(UnmanagedType.AsAny)]
//[In] object lpInBuffer, //
ref REF_I2C_TRANSFER_BLOCK lpInBuffer,
uint nInBufferSize,
byte* lpOutBuffer, //ref REF_I2C_TRANSFER_BLOCK lpOutBuffer,
uint nOutBufferSize,
out uint lpBytesReturned, //uint* lpBytesReturned,
int* lpOverlapped);
unsafe public void RefReadI2C(byte[] buffer)
{
int count = buffer.Length;
REF_I2C_TRANSFER_BLOCK I2CXFerBlock = new REF_I2C_TRANSFER_BLOCK();
REF_I2C_PACKET[] I2CPckt = new REF_I2C_PACKET[2];
I2CPckt[0] = new REF_I2C_PACKET();
I2CPckt[1] = new REF_I2C_PACKET();
I2CPckt[0].byAddr = BASE;
I2CPckt[0].byRw = I2C_READ;
I2CPckt[0].wLen = (Int16)count;
I2CPckt[0].lpiResult = Marshal.AllocHGlobal(sizeof(int));
I2CPckt[0].pbyBuf = Marshal.AllocHGlobal(count);
Marshal.Copy(buffer, 0, I2CPckt[0].pbyBuf, count);
I2CPckt[1].byAddr = BASE;
I2CPckt[1].byRw = I2C_READ;
I2CPckt[1].wLen = (Int16)count;
I2CPckt[1].lpiResult = Marshal.AllocHGlobal(sizeof(int));
I2CPckt[1].pbyBuf = Marshal.AllocHGlobal(count);
Marshal.Copy(buffer, 0, I2CPckt[0].pbyBuf, count);
I2CXFerBlock.iNumPackets = 2;
I2CXFerBlock.pI2CPackets = Marshal.AllocHGlobal(Marshal.SizeOf(I2CPckt[0])*I2CPckt.Length);
uint size = (uint)Marshal.SizeOf(I2CXFerBlock); //sizeof(REF_I2C_TRANSFER_BLOCK);//Marshal.SizeOf(I2CXFerBlock);
//size += (uint)(Marshal.SizeOf(I2CPckt[0]) * I2CPckt.Length);
uint getCnt = 0;
if ((DeviceIoControlCE(_i2cFile,
I2C_IOCTL_TRANSFER,
ref I2CXFerBlock,
size,
null, //IntPtr.Zero,
0, //0,
out getCnt,
null)) == 0)
{
int error = GetLastError();
diag("Errore nella TRANSFER.");
diag(error.ToString());
}
else
{
//success
I2CPckt = (REF_I2C_PACKET[])Marshal.PtrToStructure(I2CXFerBlock.pI2CPackets, typeof(REF_I2C_PACKET[]));
Marshal.Copy(I2CPckt[0].pbyBuf, buffer, 0, count);
diag("Success in TRANSFER: " + buffer[0].ToString());
}
Marshal.FreeHGlobal(I2CPckt[0].pbyBuf);
Marshal.FreeHGlobal(I2CPckt[0].lpiResult);
}
Edit 5:
I found online (http://em-works.googlecode.com/svn/trunk/WINCE600/PLATFORM/COMMON/SRC/SOC/COMMON_FSL_V2_PDK1_9/I2C/PDK/i2c_io.cpp) The following code:
BOOL I2C_IOControl(DWORD hOpenContext, DWORD dwCode, PBYTE pBufIn,
DWORD dwLenIn, PBYTE pBufOut, DWORD dwLenOut,
PDWORD pdwActualOut)
{
/*stuff*/
case I2C_IOCTL_TRANSFER:
{
#define MARSHAL 1
#if MARSHAL
DuplicatedBuffer_t Marshalled_pInBuf(pBufIn, dwLenIn, ARG_I_PTR);
pBufIn = reinterpret_cast<PBYTE>( Marshalled_pInBuf.ptr() );
if( (dwLenIn > 0) && (NULL == pBufIn) )
{
return FALSE;
}
#endif
I2C_TRANSFER_BLOCK *pXferBlock = (I2C_TRANSFER_BLOCK *) pBufIn;
if (pXferBlock->iNumPackets<=0)
{
return FALSE;
}
#if MARSHAL
MarshalledBuffer_t Marshalled_pPackets(pXferBlock->pI2CPackets,
pXferBlock->iNumPackets*sizeof(I2C_PACKET),
ARG_I_PTR);
I2C_PACKET *pPackets = reinterpret_cast<I2C_PACKET *>(Marshalled_pPackets.ptr());
if( (NULL == pPackets) )
{
return FALSE;
}
#else
I2C_PACKET *pPackets = pXferBlock->pI2CPackets;
#endif
#if MARSHAL
struct Marshalled_I2C_PACKET
{
MarshalledBuffer_t *pbyBuf;
MarshalledBuffer_t *lpiResult;
} *Marshalled_of_pPackets;
Marshalled_of_pPackets = new Marshalled_I2C_PACKET[pXferBlock->iNumPackets];
if (Marshalled_of_pPackets==0)
{
return FALSE;
}
MarshalledBuffer_t *pMarshalled_ptr;
int i;
// Map pointers for each packet in the array
for (i = 0; i < pXferBlock->iNumPackets; i++)
{
switch( pPackets[i].byRW & I2C_METHOD_MASK )
{
case I2C_RW_WRITE:
pMarshalled_ptr = new MarshalledBuffer_t(
pPackets[i].pbyBuf,
pPackets[i].wLen,
ARG_I_PTR,
FALSE, FALSE);
if (pMarshalled_ptr ==0)
{
bRet = FALSE;
goto cleanupPass1;
}
if (pMarshalled_ptr->ptr()==0)
{
bRet = FALSE;
delete pMarshalled_ptr;
goto cleanupPass1;
}
break;
case I2C_RW_READ:
pMarshalled_ptr = new MarshalledBuffer_t(
pPackets[i].pbyBuf,
pPackets[i].wLen,
ARG_O_PTR, FALSE, FALSE);
if (pMarshalled_ptr ==0)
{
bRet = FALSE;
goto cleanupPass1;
}
if (pMarshalled_ptr->ptr()==0)
{
bRet = FALSE;
delete pMarshalled_ptr;
goto cleanupPass1;
}
break;
default:
{
bRet = FALSE;
goto cleanupPass1;
}
}
pPackets[i].pbyBuf = reinterpret_cast<PBYTE>(pMarshalled_ptr->ptr());
Marshalled_of_pPackets[i].pbyBuf = pMarshalled_ptr;
}
for (i = 0; i < pXferBlock->iNumPackets; i++)
{
pMarshalled_ptr = new MarshalledBuffer_t(
pPackets[i].lpiResult, sizeof(INT),
ARG_O_PDW, FALSE, FALSE);
if (pMarshalled_ptr ==0)
{
bRet = FALSE;
goto cleanupPass2;
}
if (pMarshalled_ptr->ptr()==0)
{
bRet = FALSE;
delete pMarshalled_ptr;
goto cleanupPass2;
}
pPackets[i].lpiResult = reinterpret_cast<LPINT>(pMarshalled_ptr->ptr());
Marshalled_of_pPackets[i].lpiResult = pMarshalled_ptr;
}
#endif
bRet = pI2C->ProcessPackets(pPackets, pXferBlock->iNumPackets);
#if MARSHAL
DEBUGMSG (ZONE_IOCTL|ZONE_FUNCTION, (TEXT("I2C_IOControl:I2C_IOCTL_TRANSFER -\r\n")));
i = pXferBlock->iNumPackets;
cleanupPass2:
for (--i; i>=0; --i)
{
delete Marshalled_of_pPackets[i].lpiResult;
}
i = pXferBlock->iNumPackets;
cleanupPass1:
for (--i; i>=0; --i)
{
delete Marshalled_of_pPackets[i].pbyBuf;
}
delete[] Marshalled_of_pPackets;
#endif
break;
}
/*stuff*/
}
I cannot claim to understand 100% of it, but from the Windows naming conventions (http://msdn.microsoft.com/en-us/library/windows/desktop/aa378932(v=vs.85).aspx) it would appear that the size parameter I should send is the total number of bytes of my transfer, including everything. I have tried to figure that number out myself, but I have so far not been able to. Alternatively, I guess it would be possible to try and do something to the structures I have to turn them into a byte array. Only I guess that it would need to have a specific order of the bytes in it for the system to understand it.
Can anyone pitch in on that?
I've written a memory editor class that can read and write memory, but what I need to do is to scan the program's memory, and find the list of memory addresses that contain the memory I'm searching for.
This is the memory editor class.
class MemoryEditor
{
public const uint DELETE = 0x00010000;
public const uint READ_CONTROL = 0x00020000;
public const uint WRITE_DAC = 0x00040000;
public const uint WRITE_OWNER = 0x00080000;
public const uint SYNCHRONIZE = 0x00100000;
public const uint END = 0xFFF;
public const uint PROCESS_ALL_ACCESS = (DELETE | READ_CONTROL | WRITE_DAC | WRITE_OWNER | SYNCHRONIZE | END);
public Process targetedProcess;
[DllImport("kernel32.dll")]
public static extern int OpenProcess(uint dwDesiredAccess, bool bInheritHandle, int dwProcessId);
[DllImport("kernel32.dll")]
public static extern bool ReadProcessMemory(int hProcess, int lpBaseAddress, byte[] buffer, int size, int lpNumberOfBytesRead);
[DllImport("kernel32.dll")]
public static extern bool WriteProcessMemory(int hProcess, int lpBaseAddress, byte[] buffer, int size, int lpNumberOfBytesWritten);
public Process targetProcess(string name, int index = 0)
{
return (targetedProcess = Process.GetProcessesByName(name)[index]);
}
public int getHandle(Process proc, uint access = PROCESS_ALL_ACCESS)
{
return OpenProcess(access, false, proc.Id);
}
public byte[] getBytesFromString(string str)
{
return Encoding.Unicode.GetBytes(str);
}
public string getStringFromBytes(byte[] byteArr)
{
return Encoding.Unicode.GetString(byteArr);
}
public int makeHex(string str)
{
return (int.Parse(str, System.Globalization.NumberStyles.HexNumber));
}
public byte[] ReadMemory(int address, int processSize)
{
byte[] buffer = new byte[processSize];
ReadProcessMemory(getHandle(targetedProcess), address, buffer, processSize, 0);
return buffer;
}
public List<int> GetAddress(byte[] memory, int index = 0)
{
List<int> buf = new List<int>();
for (int i = 0; i < int.MaxValue; i++)
if (ReadMemory(makeHex(i.ToString()), 1) == memory)
buf.Add(i);
return buf;
}
public void WriteMemory(int address, byte[] processBytes)
{
WriteProcessMemory(getHandle(targetedProcess), address, processBytes, processBytes.Length, 0);
}
public int GetObjectSize(object TestObject)
{
BinaryFormatter bf = new BinaryFormatter();
MemoryStream ms = new MemoryStream();
byte[] Array;
bf.Serialize(ms, TestObject);
Array = ms.ToArray();
return Array.Length;
}
}
And here is the function to try to find the memory addresses
public List<int> GetAddress(byte[] memory, int index = 0)
{
List<int> buf = new List<int>();
for (int i = 0; i < int.MaxValue; i++)
if (ReadMemory(makeHex(i.ToString()), 1) == memory)
buf.Add(i);
return buf;
}
It lags extremely badly, and I'm only targeting notepad. When I scan memory in Cheat Engine, it finds it immediately, without any lag. My program scans from 0, to the max value of an int, but Cheat Engine does it 0 to the max value of a long, so I don't know what I'm doing wrong.
Any way I can do this efficiently?
There are several problems I see right away.
1. You're only reading one byte at a time:
ReadMemory(makeHex(i.ToString()), 1)
I'm not certain, but I would assume that a call to ReadProcessMemory requires a system call to execute, and doing this for every byte is certainly going to be one source of slowdown. Instead, you should read some size of "block" and then scan through that block in your process. (Doing one page at a time may be the most efficient.)
2. Why in the world are you doing all of the conversions back-and-forth to strings?!
public int makeHex(string str)
{
return (int.Parse(str, System.Globalization.NumberStyles.HexNumber));
}
....
for (int i = 0; i < int.MaxValue; i++)
if (ReadMemory(makeHex(i.ToString()), 1) == memory)
Every iteration of that loop, you're converting i to a string (the default, decimal - not hex), and then immediately passing it to makeHex which parses it (as hex, always) back to an integer. What's the point of this? Just pass the integer! These conversions can be very expensive.
<pedantic>By the way, the name "makeHex" doesn't make sense - it's going from hex, and making an integer.</pedantic>
I wish to allocate more than MaxInteger bytes of memory.
Marshall.AllocHGlobal() expects an integer - so I cannot use this. Is there another way?
Update
I changed the platform to x64, and then I ran the code below.
myp appears to have the right length: about 3.0G. But stubbornly "buffer" maxes out at 2.1G.
Any idea why?
var fileStream = new FileStream(
"C:\\big.BC2",
FileMode.Open,
FileAccess.Read,
FileShare.Read,
16 * 1024,
FileOptions.SequentialScan);
Int64 length = fileStream.Length;
Console.WriteLine(length);
Console.WriteLine(Int64.MaxValue);
IntPtr myp = new IntPtr(length);
//IntPtr buffer = Marshal.AllocHGlobal(myp);
IntPtr buffer = VirtualAllocEx(
Process.GetCurrentProcess().Handle,
IntPtr.Zero,
new IntPtr(length),
AllocationType.Commit | AllocationType.Reserve,
MemoryProtection.ReadWrite);
unsafe
{
byte* pBytes = (byte*)myp.ToPointer();
var memoryStream = new UnmanagedMemoryStream(pBytes, (long)length, (long)length, FileAccess.ReadWrite);
fileStream.CopyTo(memoryStream);
That's not possible on current mainstream hardware. Memory buffers are restricted to 2 gigabytes, even on 64-bit machines. Indexed addressing of the buffer is still done with a 32-bit signed offset. It is technically possible to generate machine code that can index more, using a register to store the offset, but that's expensive and slows down all array indexing, even for the ones that aren't larger than 2 GB.
Furthermore, you can't get a buffer larger than about 650MB out of the address space available to a 32-bit process. There aren't enough contiguous memory pages available because virtual memory contains both code and data at various addresses.
Companies like IBM and Sun sell hardware that can do this.
I have been involved in one of the other questions you asked, and I honestly think you are fighting a losing battle here. You need to possibly explore other avenues of processing this data other than reading everything into memory.
If I understand correctly, you have multiple threads that process the data concurrently and that is why you do not want to work off the file directly because of I/O contention I assume.
Have you considered or would the possibility exist to reading a block of data into memory, have the threads process the block and then read the next block or processing by the threads? This way, at any one time, you never have more than a block in memory, but all threads can access the block. This is not optimal, but I put it out there as a starting point. If this is feasible then options to optimize this can be explored.
Update: Example using platform invoke to allocate unmanaged memory and use it from .NET.
Since you are are so certain you need to load this much data into memory I thought I would write a small test application to verify that this will work. For this you will need the following
Compile with the /unsafe compile option
If you want to allocate more that 2 GB you will also need to switch your target platform to x64
*Point 2 above is a little more complicated, on a 64-bit OS you could still target the x86 platform and get access to the full 4 GB memory. This will require you to use a tool like EDITBIN.EXE to set the LargeAddressAware flag in the PE header.
This code uses VirtualAllocEx to allocate unmanaged memory and UnmanagedMemoryStream to access the unmanaged memory using the .NET stream metaphor. Note this code has only had some very basic quick tests done and only on the target 64-bit environment with 4 GB RAM. And most importantly I only went up to about 2.6 GB memory utilization for the process.
using System;
using System.IO;
using System.Runtime.InteropServices;
using System.Diagnostics;
using System.ComponentModel;
namespace MemoryMappedFileTests
{
class Program
{
static void Main(string[] args)
{
IntPtr ptr = IntPtr.Zero;
try
{
// Allocate and Commit the memory directly.
ptr = VirtualAllocEx(
Process.GetCurrentProcess().Handle,
IntPtr.Zero,
new IntPtr(0xD0000000L),
AllocationType.Commit | AllocationType.Reserve,
MemoryProtection.ReadWrite);
if (ptr == IntPtr.Zero)
{
throw new Win32Exception(Marshal.GetLastWin32Error());
}
// Query some information about the allocation, used for testing.
MEMORY_BASIC_INFORMATION mbi = new MEMORY_BASIC_INFORMATION();
IntPtr result = VirtualQueryEx(
Process.GetCurrentProcess().Handle,
ptr,
out mbi,
new IntPtr(Marshal.SizeOf(mbi)));
if (result == IntPtr.Zero)
{
throw new Win32Exception(Marshal.GetLastWin32Error());
}
// Use unsafe code to get a pointer to the unmanaged memory.
// This requires compiling with /unsafe option.
unsafe
{
// Pointer to the allocated memory
byte* pBytes = (byte*)ptr.ToPointer();
// Create Read/Write stream to access the memory.
UnmanagedMemoryStream stm = new UnmanagedMemoryStream(
pBytes,
mbi.RegionSize.ToInt64(),
mbi.RegionSize.ToInt64(),
FileAccess.ReadWrite);
// Create a StreamWriter to write to the unmanaged memory.
StreamWriter sw = new StreamWriter(stm);
sw.Write("Everything seems to be working!\r\n");
sw.Flush();
// Reset the stream position and create a reader to check that the
// data was written correctly.
stm.Position = 0;
StreamReader rd = new StreamReader(stm);
Console.WriteLine(rd.ReadLine());
}
}
catch (Exception ex)
{
Console.WriteLine(ex.ToString());
}
finally
{
if (ptr != IntPtr.Zero)
{
VirtualFreeEx(
Process.GetCurrentProcess().Handle,
ptr,
IntPtr.Zero,
FreeType.Release);
}
}
Console.ReadKey();
}
[DllImport("kernel32.dll", SetLastError = true, ExactSpelling = true)]
static extern IntPtr VirtualAllocEx(
IntPtr hProcess,
IntPtr lpAddress,
IntPtr dwSize,
AllocationType dwAllocationType,
MemoryProtection flProtect);
[DllImport("kernel32.dll", SetLastError = true, ExactSpelling = true)]
static extern bool VirtualFreeEx(
IntPtr hProcess,
IntPtr lpAddress,
IntPtr dwSize,
FreeType dwFreeType);
[DllImport("kernel32.dll", SetLastError = true, ExactSpelling = true)]
static extern IntPtr VirtualQueryEx(
IntPtr hProcess,
IntPtr lpAddress,
out MEMORY_BASIC_INFORMATION lpBuffer,
IntPtr dwLength);
[StructLayout(LayoutKind.Sequential)]
public struct MEMORY_BASIC_INFORMATION
{
public IntPtr BaseAddress;
public IntPtr AllocationBase;
public int AllocationProtect;
public IntPtr RegionSize;
public int State;
public int Protect;
public int Type;
}
[Flags]
public enum AllocationType
{
Commit = 0x1000,
Reserve = 0x2000,
Decommit = 0x4000,
Release = 0x8000,
Reset = 0x80000,
Physical = 0x400000,
TopDown = 0x100000,
WriteWatch = 0x200000,
LargePages = 0x20000000
}
[Flags]
public enum MemoryProtection
{
Execute = 0x10,
ExecuteRead = 0x20,
ExecuteReadWrite = 0x40,
ExecuteWriteCopy = 0x80,
NoAccess = 0x01,
ReadOnly = 0x02,
ReadWrite = 0x04,
WriteCopy = 0x08,
GuardModifierflag = 0x100,
NoCacheModifierflag = 0x200,
WriteCombineModifierflag = 0x400
}
[Flags]
public enum FreeType
{
Decommit = 0x4000,
Release = 0x8000
}
}
}
This is not possible from managed code without a pinvoke call and for good reason. Allocating that much memory is usually a sign of a bad solution that needs revisiting.
Can you tell us why you think you need this much memory?
Use Marshal.AllocHGlobal(IntPtr). This overload treats the value of the IntPtr as the amount of memory to allocate and IntPtr can hold a 64 bit value.
From a comment:
How do I create a second binaryreader that can read the same memorystream independantly?
var fileStream = new FileStream("C:\\big.BC2",
FileMode.Open,
FileAccess.Read,
FileShare.Read,
16 * 1024,
FileOptions.SequentialScan);
Int64 length = fileStream.Length;
IntPtr buffer = Marshal.AllocHGlobal(length);
unsafe
{
byte* pBytes = (byte*)myp.ToPointer();
var memoryStream = new UnmanagedMemoryStream(pBytes, (long)length, (long)length, FileAccess.ReadWrite);
var binaryReader = new BinaryReader(memoryStream);
fileStream.CopyTo(memoryStream);
memoryStream.Seek(0, SeekOrigin.Begin);
// Create a second UnmanagedMemoryStream on the _same_ memory buffer
var memoryStream2 = new UnmanagedMemoryStream(pBytes, (long)length, (long)length, FileAccess.Read);
var binaryReader2 = new BinaryReader(memoryStream);
}
If you can't make it work the way you want it to directly, create a class to provide the type of behaviour you want. So, to use big arrays:
using System;
using System.Collections.Generic;
using System.Text;
using System.IO;
namespace BigBuffer
{
class Storage
{
public Storage (string filename)
{
m_buffers = new SortedDictionary<int, byte []> ();
m_file = new FileStream (filename, FileMode.Open, FileAccess.Read, FileShare.Read);
}
public byte [] GetBuffer (long address)
{
int
key = GetPageIndex (address);
byte []
buffer;
if (!m_buffers.TryGetValue (key, out buffer))
{
System.Diagnostics.Trace.WriteLine ("Allocating a new array at " + key);
buffer = new byte [1 << 24];
m_buffers [key] = buffer;
m_file.Seek (address, SeekOrigin.Begin);
m_file.Read (buffer, 0, buffer.Length);
}
return buffer;
}
public void FillBuffer (byte [] destination_buffer, int offset, int count, long position)
{
do
{
byte []
source_buffer = GetBuffer (position);
int
start = GetPageOffset (position),
length = Math.Min (count, (1 << 24) - start);
Array.Copy (source_buffer, start, destination_buffer, offset, length);
position += length;
offset += length;
count -= length;
} while (count > 0);
}
public int GetPageIndex (long address)
{
return (int) (address >> 24);
}
public int GetPageOffset (long address)
{
return (int) (address & ((1 << 24) - 1));
}
public long Length
{
get { return m_file.Length; }
}
public int PageSize
{
get { return 1 << 24; }
}
FileStream
m_file;
SortedDictionary<int, byte []>
m_buffers;
}
class BigStream : Stream
{
public BigStream (Storage source)
{
m_source = source;
m_position = 0;
}
public override bool CanRead
{
get { return true; }
}
public override bool CanSeek
{
get { return true; }
}
public override bool CanTimeout
{
get { return false; }
}
public override bool CanWrite
{
get { return false; }
}
public override long Length
{
get { return m_source.Length; }
}
public override long Position
{
get { return m_position; }
set { m_position = value; }
}
public override void Flush ()
{
}
public override long Seek (long offset, SeekOrigin origin)
{
switch (origin)
{
case SeekOrigin.Begin:
m_position = offset;
break;
case SeekOrigin.Current:
m_position += offset;
break;
case SeekOrigin.End:
m_position = Length + offset;
break;
}
return m_position;
}
public override void SetLength (long value)
{
}
public override int Read (byte [] buffer, int offset, int count)
{
int
bytes_read = (int) (m_position + count > Length ? Length - m_position : count);
m_source.FillBuffer (buffer, offset, bytes_read, m_position);
m_position += bytes_read;
return bytes_read;
}
public override void Write(byte[] buffer, int offset, int count)
{
}
Storage
m_source;
long
m_position;
}
class IntBigArray
{
public IntBigArray (Storage storage)
{
m_storage = storage;
m_current_page = -1;
}
public int this [long index]
{
get
{
int
value = 0;
index <<= 2;
for (int offset = 0 ; offset < 32 ; offset += 8, ++index)
{
int
page = m_storage.GetPageIndex (index);
if (page != m_current_page)
{
m_current_page = page;
m_array = m_storage.GetBuffer (m_current_page);
}
value |= (int) m_array [m_storage.GetPageOffset (index)] << offset;
}
return value;
}
}
Storage
m_storage;
int
m_current_page;
byte []
m_array;
}
class Program
{
static void Main (string [] args)
{
Storage
storage = new Storage (#"<some file>");
BigStream
stream = new BigStream (storage);
StreamReader
reader = new StreamReader (stream);
string
line = reader.ReadLine ();
IntBigArray
array = new IntBigArray (storage);
int
value = array [0];
BinaryReader
binary = new BinaryReader (stream);
binary.BaseStream.Seek (0, SeekOrigin.Begin);
int
another_value = binary.ReadInt32 ();
}
}
}
I split the problem into three classes:
Storage - where the actual data is stored, uses a paged system
BigStream - a stream class that uses the Storage class for its data source
IntBigArray - a wrapper around the Storage type that provides an int array interface
The above can be improved significantly but it should give you ideas about how to solve your problems.