I am currently exploring neural networks and machine learning and I implemented a basic neural network in c#. Now I wanted to test my back propagation training algorithm with the MNIST database. Although I am having serious trouble reading the files correctly.
Spoiler the code is currently very badly optimised for performance. My aim currently is to grasp the subject and get a structured view how things work before I start throwing out my data structures for faster ones.
To train the network I want to feed it a custom TrainingSet data structure:
[Serializable]
public class TrainingSet
{
public Dictionary<List<double>, List<double>> data = new Dictionary<List<double>, List<double>>();
}
Keys will be my input data (784 pixels per entry(image) which will represent the greyscale values in range from 0 to 1). Values will be my output data (10 entries representing the digits from 0-9 with all entries on 0 except the exspected one at 1)
Now I want to read the MNIST database according to this contract. I am currentl on my 2nd try which is inspired by this blogpost: https://jamesmccaffrey.wordpress.com/2013/11/23/reading-the-mnist-data-set-with-c/ . Sadly it is still producing the same nonsense as my first try scattering the pixels in a strange pattern:
My current reading algorithm:
public static TrainingSet GenerateTrainingSet(FileInfo imagesFile, FileInfo labelsFile)
{
MnistImageView imageView = new MnistImageView();
imageView.Show();
TrainingSet trainingSet = new TrainingSet();
List<List<double>> labels = new List<List<double>>();
List<List<double>> images = new List<List<double>>();
using (BinaryReader brLabels = new BinaryReader(new FileStream(labelsFile.FullName, FileMode.Open)))
{
using (BinaryReader brImages = new BinaryReader(new FileStream(imagesFile.FullName, FileMode.Open)))
{
int magic1 = brImages.ReadBigInt32(); //Reading as BigEndian
int numImages = brImages.ReadBigInt32();
int numRows = brImages.ReadBigInt32();
int numCols = brImages.ReadBigInt32();
int magic2 = brLabels.ReadBigInt32();
int numLabels = brLabels.ReadBigInt32();
byte[] pixels = new byte[numRows * numCols];
// each image
for (int imageCounter = 0; imageCounter < numImages; imageCounter++)
{
List<double> imageInput = new List<double>();
List<double> exspectedOutput = new List<double>();
for (int i = 0; i < 10; i++) //generate empty exspected output
exspectedOutput.Add(0);
//read image
for (int p = 0; p < pixels.Length; p++)
{
byte b = brImages.ReadByte();
pixels[p] = b;
imageInput.Add(b / 255.0f); //scale in 0 to 1 range
}
//read label
byte lbl = brLabels.ReadByte();
exspectedOutput[lbl] = 1; //modify exspected output
labels.Add(exspectedOutput);
images.Add(imageInput);
//Debug view showing parsed image.......................
Bitmap image = new Bitmap(numCols, numRows);
for (int y = 0; y < numRows; y++)
{
for (int x = 0; x < numCols; x++)
{
image.SetPixel(x, y, Color.FromArgb(255 - pixels[x * y], 255 - pixels[x * y], 255 - pixels[x * y])); //invert colors to have 0,0,0 be white as specified by mnist
}
}
imageView.SetImage(image);
imageView.Refresh();
//.......................................................
}
brImages.Close();
brLabels.Close();
}
}
for (int i = 0; i < images.Count; i++)
{
trainingSet.data.Add(images[i], labels[i]);
}
return trainingSet;
}
All images produce a pattern as shown above. It's never the exact same pattern but always seems to have the pixels "pulled" down to the right corner.
That is how I did it:
public static class MnistReader
{
private const string TrainImages = "mnist/train-images.idx3-ubyte";
private const string TrainLabels = "mnist/train-labels.idx1-ubyte";
private const string TestImages = "mnist/t10k-images.idx3-ubyte";
private const string TestLabels = "mnist/t10k-labels.idx1-ubyte";
public static IEnumerable<Image> ReadTrainingData()
{
foreach (var item in Read(TrainImages, TrainLabels))
{
yield return item;
}
}
public static IEnumerable<Image> ReadTestData()
{
foreach (var item in Read(TestImages, TestLabels))
{
yield return item;
}
}
private static IEnumerable<Image> Read(string imagesPath, string labelsPath)
{
BinaryReader labels = new BinaryReader(new FileStream(labelsPath, FileMode.Open));
BinaryReader images = new BinaryReader(new FileStream(imagesPath, FileMode.Open));
int magicNumber = images.ReadBigInt32();
int numberOfImages = images.ReadBigInt32();
int width = images.ReadBigInt32();
int height = images.ReadBigInt32();
int magicLabel = labels.ReadBigInt32();
int numberOfLabels = labels.ReadBigInt32();
for (int i = 0; i < numberOfImages; i++)
{
var bytes = images.ReadBytes(width * height);
var arr = new byte[height, width];
arr.ForEach((j,k) => arr[j, k] = bytes[j * height + k]);
yield return new Image()
{
Data = arr,
Label = labels.ReadByte()
};
}
}
}
Image class:
public class Image
{
public byte Label { get; set; }
public byte[,] Data { get; set; }
}
Some extension methods:
public static class Extensions
{
public static int ReadBigInt32(this BinaryReader br)
{
var bytes = br.ReadBytes(sizeof(Int32));
if (BitConverter.IsLittleEndian) Array.Reverse(bytes);
return BitConverter.ToInt32(bytes, 0);
}
public static void ForEach<T>(this T[,] source, Action<int, int> action)
{
for (int w = 0; w < source.GetLength(0); w++)
{
for (int h = 0; h < source.GetLength(1); h++)
{
action(w, h);
}
}
}
}
Usage:
foreach (var image in MnistReader.ReadTrainingData())
{
//use image here
}
or
foreach (var image in MnistReader.ReadTestData())
{
//use image here
}
Why not use a nuget package:
MNIST.IO Just a datareader (disclaimer: my package)
Accord.DataSets Contains classes to download and parse machine learning datasets such as MNIST, News20, Iris. This package is part of the Accord.NET Framework.
Related
I'm want to implement the k-means clustering algorithm to clusterize a list of text files loaded from disk, and also using the silhouette method to determine the number of clusters.
The error I'm getting is System.ArgumentOutOfRangeException: 'Schema mismatch for input column 'Features': expected scalar or vector of String, got VarVector<Single> (Parameter 'inputSchema')'
How can I solve the problem?
Code
static void Main(string[] args)
{
// Load text files from disk
string[] filePaths = Directory.GetFiles("C:\\ExportedEmails\\", "*.txt");
string[] textFiles = filePaths.Select(File.ReadAllText).ToArray();
// Extract features from text files
var textData = textFiles.Select((text, index) =>
{
int startIndex = text.IndexOf("Description: ") + "Description: ".Length;
string description = text.Substring(startIndex);
return new TextData { Text = description, Index = index };
}).ToArray();
// Initialize the MLContext
var context = new MLContext();
// Convert data to IDataView
var data = context.Data.LoadFromEnumerable(textData);
// Initialize the variables to store the silhouette scores
var silhouetteScores = new double[10];
// Iter
for (int k = 2; k <= 10; k++)
{
// Create a new KMeansTrainer
var pipeline = context.Transforms.Text.FeaturizeText("Text", "Features")
.Append(context.Clustering.Trainers.KMeans(featureColumnName: "Features", numberOfClusters: k));
var model = pipeline.Fit(data);
var transformedData = model.Transform(data);
// Compute the silhouette score
var clusterAssignments = transformedData.GetColumn<uint>("PredictedLabel").ToArray();
var clusterCenters = transformedData.GetColumn<float[]>("Centroid").ToArray();
var features = transformedData.GetColumn<float[]>("Features").ToArray();
var silhouetteScore = ComputeSilhouetteScore(features, clusterAssignments, clusterCenters);
silhouetteScores[k - 2] = silhouetteScore;
}
// Find the number of clusters that maximizes the average silhouette score
int optimalClusters = 0;
double maxScore = double.MinValue;
for (int i = 0; i < silhouetteScores.Length; i++)
{
if (silhouetteScores[i] > maxScore)
{
maxScore = silhouetteScores[i];
optimalClusters = i + 2;
}
}
Console.WriteLine($"The optimal number of clusters is {optimalClusters}.");
}
private static double ComputeSilhouetteScore(float[][] features, uint[] clusterAssignments, float[][] clusterCenters)
{
double silhouetteScore = 0;
for (int i = 0; i < features.Length; i++)
{
var a = ComputeAverageDistance(features[i], clusterAssignments[i], features, clusterAssignments);
var b = ComputeMinimumDistance(features[i], clusterAssignments[i], clusterCenters);
silhouetteScore += (b - a) / Math.Max(a, b);
}
return silhouetteScore / features.Length;
}
private static double ComputeAverageDistance(float[] feature, uint clusterAssignment, float[][] features, uint[] clusterAssignments)
{
double distance = 0;
int count = 0;
for (int i = 0; i < features.Length; i++)
{
if (clusterAssignments[i] == clusterAssignment)
{
distance += Distance(feature, features[i]);
count++;
}
}
return distance / count;
}
private static double ComputeMinimumDistance(float[] feature, uint clusterAssignment, float[][] clusterCenters)
{
double minDistance = double.MaxValue;
for (int i = 0; i < clusterCenters.Length; i++)
{
if (i != clusterAssignment)
{
minDistance = Math.Min(minDistance, Distance(feature, clusterCenters[i]));
}
}
return minDistance;
}
private static double Distance(float[] feature1, float[] feature2)
{
double distance = 0;
for (int i = 0; i < feature1.Length; i++)
{
distance += Math.Pow(feature1[i] - feature2[i], 2);
}
return Math.Sqrt(distance);
}
class TextData
{
public string? Text { get; set; }
public float[] Features { get; set; }
public int Index { get; set; }
}
I believe the reason you're getting this error is you set the output column name to Text and input column name as Features in the FeaturizeText transform. I suspect you want the opposite. Take the Text column as input and generate a feature vector called Features which you can do by swapping the values in FeaturizeText or you can explicitly add parameter names (FeaturizeText(outputColumnName: "Features", inputColumnName:"Text")).
Check out the FeaturizeText documentation for more details.
Im trying to get value from the struct and use the value in OverSized to use it in the if statement. but the program doesnt seem like its getting any information from the oversized. by writing on console I can see the value of overSized is true(for example) but that heavy in the third code section doesnt really have any value. Am I calling overSized in a wrong way?
This is my struct for shipping box:
public struct ShippingBox
{
public int Length;
public int Width;
public int Height;
public bool OverSized;
public ShippingBox(
int length,
int width,
int height,
bool overSized)
{
this.Length = length;
this.Width = width;
this.Height = height;
this.OverSized = overSized;
}
}
and this is how I give value to the struct:(typewiseboxes is already defined)
bool overSized = false;
if (some condition)
{
overSized = true;
}
int l = 0;
int w = 0;
int h = 0;
if (overSized)
{
l = 100;
w = 100;
h = 100;
}
if (boxTypeWiseNumOfBoxes > 0)
{
typeWiseBoxes.Add(
new ShippingBox(
l,
w,
h,
overSized));
}
now trying to get the value of oversized with this:
ShippingBox specialBox = new ShippingBox();
var heavy = specialBox.OverSized; //also tried bool heavy
int tempLength = 0;
int tempWidth = 0;
int tempHeight = 0;
if (heavy)
{
tempLength = 101;
tempWidth = 8;
tempHeight = 12;
}
else if (!heavy)
{
tempLength = 77;
tempWidth = 8;
tempHeight = 12;
}
I guess you have to change:
ShippingBox specialBox = new ShippingBox(); // this assigns specialBox an instance of a brand new ShippingBox object
var heavy = specialBox.OverSized; //heavy will be undefined
...
to:
ShippingBox specialBox = typeWiseBoxes.Last(); // this assigns specialBox the instance of the last ShippingBox object added to typeWiseBoxes List
var heavy = specialBox.OverSized; // heavy is set to the value of OverSized property of last typeWiseBoxes List element
...
and thus assigning specialBox the last element added to the list instead of a brand new one
I am trying to work out how to use ConvNetShar correctly to learn some shapes in some images. I have test data consisting of black shapes on white backgrounds, all these shapes belong to 1 class (road) and I have a mixture of images with that and without. I have tried to adapt the 2d demo and it does seem to learn but when it tests it fails...
Below is my code...just wondering if anyone has a working example with images?
Thanks
(code just updated) seems to be learning..still failing tests though...
using System;
using System.Collections.Generic;
using ConvNetSharp.Core;
using ConvNetSharp.Core.Layers.Double;
using ConvNetSharp.Core.Training;
using ConvNetSharp.Volume;
using ConvNetSharp.Volume.Double;
using System.IO;
using System.Windows.Forms;
using System.Drawing;
using NDNBackpropNnTrainer;
namespace ClassifyImageDemo
{
internal class Program
{
private static void ClassifyImage()
{
string inputFolder = GetInputFolder();
var filelist = Directory.EnumerateFiles(inputFolder, "*.png");
List<List<double>> matrix = new List<List<double>>();
List<int> expetedList = new List<int>();
int looper = 0;
int width = 1;
int height = 1;
foreach (var fileInImage in filelist)
{
matrix.Add(new List<double>());
Bitmap source = (Bitmap) Image.FromFile(fileInImage);
List<double> innerList = new List<double>();
using (var bmp = new LockBitmap(source))
{
width = bmp.Width;
height = bmp.Height;
for (var y = 0; y < bmp.Height; y++)
{
for (var x = 0; x < bmp.Width; x++)
{
var color = bmp.GetPixel(x, y);
int myR = color.R;
int myG = color.G;
int myB = color.B;
int total = myR + myG + myB;
//Adds new sub List
innerList.Add(total);
}
}
}
matrix[looper]=innerList; //Add values to the sub List at index 0
looper = looper + 1 ;
byte[] myImageRaw = File.ReadAllBytes(fileInImage);
int len = myImageRaw.Length;
var lastOneByte = (byte) myImageRaw[len - 1];
expetedList.Add(lastOneByte);
}
var net = new Net<double>();
net.AddLayer(new InputLayer(1, 1, (width * height)));
net.AddLayer(new FullyConnLayer(6));
net.AddLayer(new TanhLayer());
net.AddLayer(new FullyConnLayer(2));
net.AddLayer(new TanhLayer());
net.AddLayer(new FullyConnLayer(2));
net.AddLayer(new SoftmaxLayer(2));
// Data
var data = new List<double[]>();
var labels = new List<int>();
foreach (var lstInputs in matrix)
{
double[] arrayIn = lstInputs.ToArray();
data.Add(arrayIn);
}
foreach (var lbl in expetedList)
{
labels.Add(lbl);
}
var n = labels.Count;
var trainer = new SgdTrainer<double>(net) { LearningRate = 0.01, L2Decay = 0.001, BatchSize = n };
double loss = 1.0;
// Training
while (loss > 0.00001)
{
loss = ClassifyImageUpdate(width, height, n, data, trainer, labels);
}
// Testing
var netx = BuilderInstance.Volume.From(new double[(width * height) * n], new Shape(1, 1, (width*height), n));
for (var ix = 0; ix < n; ix++)
{
int subLooper2 = 0;
foreach (var dtA2 in data[ix])
{
netx.Set(0, 0, subLooper2, ix, dtA2);
subLooper2 = subLooper2 + 1;
}
}
var result = net.Forward(netx);
var c = net.GetPrediction();
var accurate = c[0] == labels[0];
Console.ReadLine();
}
private static string GetInputFolder()
{
return #"D:\temp\filtered\blackroads\subset_500";
}
private static double ClassifyImageUpdate(int width, int height, int n, List<double[]> data, TrainerBase<double> trainer, List<int> labels)
{
var avloss = 0.0;
int dimensions = width * height;
var netx = BuilderInstance.Volume.SameAs(new Shape(1, 1, dimensions, n));
var hotLabels = BuilderInstance.Volume.SameAs(new Shape(1, 1, 1, n));
for (var ix = 0; ix < n; ix++)
{
hotLabels.Set(0, 0, 0, ix, labels[ix]);
int subLooper = 0;
foreach (var dtA in data[ix])
{
netx.Set(0, 0, subLooper, ix, dtA);
subLooper = subLooper + 1;
}
// netx.Set(0, 0, 1, ix, data[ix][1]);
}
for (var iters = 0; iters < 50; iters++)
{
trainer.Train(netx, hotLabels);
avloss += trainer.Loss;
}
avloss /= 50.0;
Console.WriteLine(" Loss:" + avloss);
return avloss;
}
private static void Main(string[] args)
{
ClassifyImage();
}
}
}
I am trying to get the max intensity value and min intensity from an image called btm to get the average from the "max, min", then use this average as a threshold to convert image to binary image.
So I used histogram class from aforge library which takes an int array, so I am trying to convert my image btm to the array but the function ImageToByteArray that I used to convert image return array from byte data type.
System.Drawing.Image img = (System.Drawing.Image)btm;
byte[] imgarr = ImageToByteArray(img);
Histogram h = new Histogram(imgarr);
int Maxval= h.max();
int Minval= h.min();
.
public static byte[] ImageToByteArray(System.Drawing.Image imageIn)
{
using (var ms = new MemoryStream())
{
imageIn.Save(ms, imageIn.RawFormat);
return ms.ToArray();
}
}
I am posting two routines. You can examine them to get an idea regarding how to achieve your task.
Step 1. Convert Bitmap to int[,]:
public static int[,] ToInteger(Bitmap input)
{
//// We are presuming that the image is grayscale.
//// A color image is impossible to convert to 2D.
int Width = input.Width;
int Height = input.Height;
int[,] array2d = new int[Width, Height];
for (int y = 0; y < Height; y++)
{
for (int x = 0; x < Width; x++)
{
Color cl = input.GetPixel(x, y);
// image is Grayscale
// three elements are averaged.
int gray = (int)Convert.ChangeType(cl.R * 0.3 + cl.G * 0.59 + cl.B * 0.11, typeof(int));
array2d[x, y] = gray;
}
}
return array2d;
}
Step 2. Seach for Max and Min.
public int Max(int[,] values)
{
int max = 0;
for (int i = 1; i < values.GetLength(0); i++)
{
for (int j = 1; j < values.GetLength(1); j++)
{
if (values[i,j] > 0)
{
max = values[i, j];
}
}
}
return max;
}
public int Min(int[,] values)
{
... ... ...
if (values[i,j] < 0)
{
min = values[i];
}
... ... ...
return min;
}
You can combine last two.
Hope you get the idea.
I want to know a fastest way to count all byte in file ? I need to work on large binary file
I want to know the quantity of all byte in file (Quantity of 0x00, 0x01, .. 0xff)
It's for add a graph with file representation in my WPF Hexeditor usercontrol https://github.com/abbaye/WPFHexEditorControl like in HxD hexeditor.
This code work fine but it's to slow for large file.
public Dictionary<int, long> GetByteCount()
{
if (IsOpen)
{
Position = 0;
int currentByte = 0;
// Build dictionary
Dictionary<int, long> cd = new Dictionary<int, long>();
for (int i = 0; i <= 255; i++) cd.Add(i, 0);
//
for (int i = 0; i <= Length; i++)
{
//if (EOF) break;
currentByte = ReadByte();
if (currentByte != -1) cd[currentByte]++;
Position++;
}
return cd;
}
return new Dictionary<int, long>();
}
/// <summary>
/// Get an array of long computing the total of each byte in the file.
/// The position of the array makes it possible to obtain the sum of the desired byte
/// </summary>
public long[] GetByteCount()
{
if (IsOpen)
{
const int bufferLenght = 1048576; //1mb
var storedCnt = new long[256];
Position = 0;
while (!Eof)
{
var testLenght = Length - Position;
var buffer = testLenght <= bufferLenght ? new byte[testLenght] : new byte[bufferLenght];
Read(buffer, 0, buffer.Length);
foreach (var b in buffer)
storedCnt[b]++;
Position += bufferLenght;
}
return storedCnt;
}
return null;
}
I have optimized David's solution a bit. The "Position" - calls are not necessary. I've found that the buffer length and the unbuffered read mode are not very important, but the "for"- instead of "foreach" - construct in the calculation made a big difference.
Results with
foreach (var b in buffer.Take(count))
{
storedCnt[b]++;
}
file length is 4110217216
duration 00:00:51.1686821
Results with
for(var i = 0; i < count; i++)
{
storedCnt[buffer[i]]++;
}
file length 4110217216
duration 00:00:05.9695418
Here the program
private static void Main(
{
const string fileForCheck = #"D:\Data\System\en_visual_studio_enterprise_2015_x86_x64_dvd_6850497.iso";
Debug.Assert(File.Exists(fileForCheck));
var watch = new Stopwatch();
var counter = new FileBytesCounter(fileForCheck);
watch.Start();
var results = counter.GetByteCount();
watch.Stop();
counter.Dispose();
Console.WriteLine("results:");
Console.WriteLine(string.Join(", ", results.Select((c, b) => $"{b} -> {c}")));
var sumBytes = results.Sum(c => c);
Debug.Assert((new FileInfo(fileForCheck)).Length == sumBytes); // here's the proof
Console.WriteLine();
Console.WriteLine($"file length {sumBytes}");
Console.WriteLine($"duration {watch.Elapsed}");
}
and here the class
internal class FileBytesCounter
: FileStream
{
private const FileOptions FileFlagNoBuffering = (FileOptions)0x20000000;
private const int CopyBufferSize = 1024 * 1024;
//private const int CopyBufferSize = 4 * 1024 * 16;
public FileBytesCounter(string path, FileShare share = FileShare.Read)
: base(path, FileMode.Open, FileAccess.Read, share, CopyBufferSize/*, FileFlagNoBuffering*/)
{
}
public long[] GetByteCount()
{
var buffer = new byte[CopyBufferSize];
var storedCnt = new long[256];
int count;
Position = 0;
while ((count = Read(buffer, 0, CopyBufferSize)) > 0)
{
for(var i = 0; i < count; i++)
{
storedCnt[buffer[i]]++;
}
}
return storedCnt;
}
}
See also https://www.codeproject.com/Articles/172613/Fast-File-Copy-With-Managed-Code-UBCopy-update for FileFlagNoBuffering
It seems like you want something like this:
public Dictionary<char, long> GetCharCount(string filePath)
{
var result = new Dictionary<char, long>();
var content = File.ReadAllText(filePath);
foreach(var c in content)
{
if (result.ContainsKey(c))
{
result[c] = result[c] + 1;
}
else
{
result.Add(c, 1);
}
}
return result;
}