Develop Reference

'Schema mismatch for input column 'Features': expected scalar or vector of String, got VarVector<Single> (Parameter 'inputSchema')'

I'm want to implement the k-means clustering algorithm to clusterize a list of text files loaded from disk, and also using the silhouette method to determine the number of clusters.
The error I'm getting is System.ArgumentOutOfRangeException: 'Schema mismatch for input column 'Features': expected scalar or vector of String, got VarVector<Single> (Parameter 'inputSchema')'
How can I solve the problem?
Code
static void Main(string[] args)
{
// Load text files from disk
string[] filePaths = Directory.GetFiles("C:\\ExportedEmails\\", "*.txt");
string[] textFiles = filePaths.Select(File.ReadAllText).ToArray();
// Extract features from text files
var textData = textFiles.Select((text, index) =>
{
int startIndex = text.IndexOf("Description: ") + "Description: ".Length;
string description = text.Substring(startIndex);
return new TextData { Text = description, Index = index };
}).ToArray();
// Initialize the MLContext
var context = new MLContext();
// Convert data to IDataView
var data = context.Data.LoadFromEnumerable(textData);
// Initialize the variables to store the silhouette scores
var silhouetteScores = new double[10];
// Iter
for (int k = 2; k <= 10; k++)
{
// Create a new KMeansTrainer
var pipeline = context.Transforms.Text.FeaturizeText("Text", "Features")
.Append(context.Clustering.Trainers.KMeans(featureColumnName: "Features", numberOfClusters: k));
var model = pipeline.Fit(data);
var transformedData = model.Transform(data);
// Compute the silhouette score
var clusterAssignments = transformedData.GetColumn<uint>("PredictedLabel").ToArray();
var clusterCenters = transformedData.GetColumn<float[]>("Centroid").ToArray();
var features = transformedData.GetColumn<float[]>("Features").ToArray();
var silhouetteScore = ComputeSilhouetteScore(features, clusterAssignments, clusterCenters);
silhouetteScores[k - 2] = silhouetteScore;
}
// Find the number of clusters that maximizes the average silhouette score
int optimalClusters = 0;
double maxScore = double.MinValue;
for (int i = 0; i < silhouetteScores.Length; i++)
{
if (silhouetteScores[i] > maxScore)
{
maxScore = silhouetteScores[i];
optimalClusters = i + 2;
}
}
Console.WriteLine($"The optimal number of clusters is {optimalClusters}.");
}
private static double ComputeSilhouetteScore(float[][] features, uint[] clusterAssignments, float[][] clusterCenters)
{
double silhouetteScore = 0;
for (int i = 0; i < features.Length; i++)
{
var a = ComputeAverageDistance(features[i], clusterAssignments[i], features, clusterAssignments);
var b = ComputeMinimumDistance(features[i], clusterAssignments[i], clusterCenters);
silhouetteScore += (b - a) / Math.Max(a, b);
}
return silhouetteScore / features.Length;
}
private static double ComputeAverageDistance(float[] feature, uint clusterAssignment, float[][] features, uint[] clusterAssignments)
{
double distance = 0;
int count = 0;
for (int i = 0; i < features.Length; i++)
{
if (clusterAssignments[i] == clusterAssignment)
{
distance += Distance(feature, features[i]);
count++;
}
}
return distance / count;
}
private static double ComputeMinimumDistance(float[] feature, uint clusterAssignment, float[][] clusterCenters)
{
double minDistance = double.MaxValue;
for (int i = 0; i < clusterCenters.Length; i++)
{
if (i != clusterAssignment)
{
minDistance = Math.Min(minDistance, Distance(feature, clusterCenters[i]));
}
}
return minDistance;
}
private static double Distance(float[] feature1, float[] feature2)
{
double distance = 0;
for (int i = 0; i < feature1.Length; i++)
{
distance += Math.Pow(feature1[i] - feature2[i], 2);
}
return Math.Sqrt(distance);
}
class TextData
{
public string? Text { get; set; }
public float[] Features { get; set; }
public int Index { get; set; }
}

I believe the reason you're getting this error is you set the output column name to Text and input column name as Features in the FeaturizeText transform. I suspect you want the opposite. Take the Text column as input and generate a feature vector called Features which you can do by swapping the values in FeaturizeText or you can explicitly add parameter names (FeaturizeText(outputColumnName: "Features", inputColumnName:"Text")).
Check out the FeaturizeText documentation for more details.

My single layer perceptrone is not working

Here is code.
public class Adaline
{
private int _layer;
public int Layer { get { return _layer; } }
private int _epoch;
public int Epoch { get { return _epoch; } }
private double _error;
public double Error { get { return _error; } }
private double[] _weights;
public Adaline(int layer)
{
_layer = layer;
_weights = new double[layer];
Reset();
}
public void Reset()
{
Random r = new Random();
for (int i = 0; i < _layer; i++)
_weights[i] = r.NextDouble() - 0.5;
_error = 1;
}
public void Train(BasicTrainSet<double> trainset, double learnRate)
{
double ers = 0;
for(int p = 0; p < trainset.DataCount; p++)
{
double result = Compute(trainset.Input[p], true);
double error = trainset.Output[p] - result;
for (int i = 0; i < _weights.Length; i++)
{
_weights[i] += error * trainset.Input[p][i] * learnRate;
}
ers += Math.Abs(error);
}
_epoch++;
_error = ers;
}
public double Compute(double[] input, bool quan)
{
double result = 0;
for (int i = 0; i < _layer; i++)
result += Math.Tanh(_weights[i] * input[i]);
//double result = _weights.Zip(input, (a, b) => Math.Tanh(a * b)).Sum();
return quan ? (result >= 0 ? 1 : 0) : result;
}
}
When I tried to train and gate like this, it works like this.
Up four results are from this code
This is pretty weird, because there is not any problem with algorithm.
Weights are getting bigger and bigger. Where did i mistake?

In your code to compute the output of each neuron, you aren't applying the activation function correctly. You need to find the dot product between the weights and the inputs into each neuron then apply the activation function after. You are applying the activation function after each weighted accumulation, which is not correct.
Accumulate, then apply the activation function:
public double Compute(double[] input, bool quan)
{
double result = 0;
for (int i = 0; i < _layer; i++)
result += _weights[i] * input[i]; // Change - accumulate first
result = Math.Tanh(result); // Change - now apply activation function
return quan ? (result >= 0 ? 1 : 0) : result;
}

Unit Test these methods?

Hi I was wondering how I could unit test the methods below:
public float calcNP()
{
int rowcount = dataGridView1.Rows.Count;
int[] colB = new int[rowcount];
float[] NP = new float[rowcount];
float avgNP = 0;
for (int i = 0; i < rowcount; i++)
{
colB[i] = Convert.ToInt32(dataGridView1.Rows[i].Cells[5].Value);
}
float min = colB.Min();
float max = colB.Max();
float a = 1, b = 10;
for (int i = 0; i < rowcount; i++)
{
NP[i] = (a + (colB[i] - min) * (b - a)) / (max - min);
avgNP = NP[i] + avgNP;
}
avgNP = (avgNP / rowcount) * 100;
return avgNP;
}
and also the following
public float calcIFact()
{
float ftp = calcFTP();
float NP = calcNP();
float IntFact = NP / ftp;
return IntFact;
}
and this
public float calcFTP()
{
float ftp = 0;
if (chkFTP.Checked == true)
{
// ftp =(float)Convert.ToDouble(txtFTP.Text)/100;
ftp = (float)Convert.ToDouble(txtFTP.Text);
}
if (chkFTP.Checked == false)
{
int rowcount = dataGridView1.Rows.Count;
int[] colB = new int[rowcount];
for (int i = 0; i < rowcount; i++)
{
colB[i] = Convert.ToInt32(dataGridView1.Rows[i].Cells[5].Value);
}
// ftp = colB.Max() / 100;
ftp = colB.Max();
}
If someone could give me some examples that would be great.
Basically just need some simple tests for the calculations

We are telling you to do this:
// New method, not depending on a gridview. Testable.
public float calcNP_pure(int[] colB)
{
float[] NP = new float[colB.Length];
float avgNP = 0;
float min = colB.Min();
float max = colB.Max();
float a = 1, b = 10;
for (int i = 0; i < rowcount; i++)
{
NP[i] = (a + (colB[i] - min) * (b - a)) / (max - min);
avgNP = NP[i] + avgNP;
}
avgNP = (avgNP / rowcount) * 100;
return avgNP;
}
// Not testable
public float calcNP()
{
int rowcount = dataGridView1.Rows.Count;
int[] colB = new int[rowcount];
for (int i = 0; i < rowcount; i++)
{
colB[i] = Convert.ToInt32(dataGridView1.Rows[i].Cells[5].Value);
}
return calcNP_pure(colB);
}
This new method should also go into a new class and not sit on a form or some other UI-element. The best practice is to program against interfaces and to inject required dependencies into the objects:
public interface INpCalculator
{
float CalcNP(int[] values);
}
public class NpCalculator : INpCalculator
{
public float CalcNP(int[] values)
{
...
return ...;
}
}
public partial class Form1 : Form
{
private INpCalculator _npCalculator;
public Form1(INpCalculator npCalculator)
{
InitializeComponent();
_npCalculator = npCalculator;
}
private float CalcNP()
{
int[] colB = dataGridView1.Rows
.Cast<DataGridViewRow>()
.Select(row => row.Cells[5].Value)
.ToArray();
return _npCalculator.CalcNP(colB);
}
}
Now the calculator is testable without the need to open any form:
[TestMethod]
public void TestNpCalculator()
{
// Arrange
var sut = new NpCalculator();
var values = new int[] { 12, 7, 15, 3 };
// Act
float result = sut.CalcNP(values);
// Assert
Assert.AreEqual(123.456f, result);
}
Btw: sut stands for "system under test".

In current state of the method, your unit tests are going to require manually create datagrid object and populate values as per your testing scenario. As suggested by Daniel, your method "public float calcNP()" should not use data grid, instead of that you can pass an array of integers to the method (pass only values which you are going to use in calculation)

Matrix of objects class

I have written 3 classes: KompresorSilnik, KompresorObiekt and Mrowka.
I want to create a matrix of objects of class KompresorObiekt ( it's a GrafObiektow method ). In the constructor of KompresorObiekt, I create 4 instances of KompresorSilnik. I have a method, zwrocListe() which returns a list of int[] from 0000 to 2222.
The question is: was that correct or not implementation?
public class KompresorSilnik
{
public double minFlow;
public double maxFlow;
public bool typ;
public int stan;
public KompresorSilnik()
{
}
public KompresorSilnik(double minFlow, double maxFlow, bool typ, int stan)
{
this.minFlow = minFlow;
this.maxFlow = maxFlow;
this.typ = typ;
this.stan = stan;
}
}
This is my second class KompresorObiekt
public class KompresorObiekt
{
public KompresorObiekt() { }
public KompresorObiekt( List<int[]> list, int x, int y)
{
double k1Max = 3000000.0;
double k1Min = 10000.0;
double k2Max = 200000.0;
double k2Min = 5000.0;
double K3Max = 100000000.0;
double k3Min = 800.0;
double k4Max = 10000.0;
double k4Min = 2000.0;
KompresorSilnik komp1 = new KompresorSilnik(k1Min, k1Max, false, list[x][y]);
KompresorSilnik komp2 = new KompresorSilnik(k2Min, k2Max, true, list[x][y+1]);
KompresorSilnik komp3 = new KompresorSilnik(k3Min, K3Max, false, list[x][y+2]);
KompresorSilnik komp4 = new KompresorSilnik(k4Min, k4Max, true, list[x][y+3]);
}
}
and the last one Mrowka
public class Mrowka
{
List<int[]> zwrocListe()
{
List<int[]> arrList = new List<int[]>();
int[] ArrayOfInts = new int[4];
for (int i = 0; i < 3; i++)
{
for (int j = 0; j < 3; j++)
{
for (int k = 0; k < 3; k++)
{
for (int m = 0; m < 3; m++)
{
ArrayOfInts[0] = i;
ArrayOfInts[1] = j;
ArrayOfInts[2] = k;
ArrayOfInts[3] = m;
int[] arc = (int[])ArrayOfInts.Clone();
arrList.Add(arc);
}
}
}
}
return arrList;
}
KompresorObiekt[][] GrafObiektow(int time)
{
KompresorObiekt[][] graf = new KompresorObiekt[time][];
int numNodes = zwrocListe().Count;
for (int i = 0; i < graf.Length; ++i)
graf[i] = new KompresorObiekt[numNodes];
for (int i = 0; i < time; ++i)
for (int j = 0; j < numNodes; ++j)
{
graf[i][j] = new KompresorObiekt(zwrocListe(), j, 0);
}
return graf;
}
}

How to parameterize a method with a given dimension during multidimensional array iteration?

I have a function which is applied on each element of a 2D array (double[,]), but only along a given dimension.
I had to create two functions because I don't know how to pass the desired dimension to the method as a parameter. I ended up with a "vertical_foo" and a "horizontal_foo" functions, which are almost identical to each other:
private double[,] vertical_foo (double[,] a) {
int height = a.GetLength(0);
int width = a.GetLength(1);
var result = new double[height, weight];
for (int i = 0; i < height; i++) {
for (int j = 0; j < width; j++) {
// Here I use first ("i") dimension
int before = Math.Max(i-1, 0);
int after = Math.Min(i+1, height-1);
result[i,j] = (a[after, j] - a[before, j]) * 0.5;
}
}
return result;
}
private double[,] horizontal_foo (double[,] a) {
int height = a.GetLength(0);
int width = a.GetLength(1);
var result = new double[height, weight];
for (int i = 0; i < height; i++) {
for (int j = 0; j < width; j++) {
// Here I use second ("j") dimension
int before = Math.Max(j-1, 0);
int after = Math.Min(j+1, height-1);
result[i,j] = (a[i, after] - a[i, before]) * 0.5;
}
}
return result;
}
I would like to have a signature like this, where the second parameter is the dimension on which I want to apply the indexing:
private double[,] general_foo (double[,] a, int dimension) {}
Any suggestion is much welcome!

I'll take a stab at this:
private double[,] general_foo(double[,] a, int dimension)
{
var w = a.GetLength(0);
var h = a.GetLength(1);
var result = new double[w, h];
var otherDimension = 1 - dimension; // NOTE only works for 2D arrays
var otherDimensionLength = a.GetLength(otherDimension);
var dimensionLength = a.GetLength(dimension);
for (int i = 0; i < dimensionLength; i++)
{
for (int j = 0; j < otherDimensionLength; j++)
{
var setIndexes = new int[2] { j, j };
setIndexes[dimension] = i;
var beforeIndexes = new int[2] { j, j };
beforeIndexes[dimension] = Math.Max(i - 1, 0);
var afterIndexes = new int[2] { j, j };
afterIndexes[dimension] = Math.Min(i + 1, dimensionLength - 1);
var beforeValue = (double)a.GetValue(beforeIndexes);
var afterValue = (double)a.GetValue(afterIndexes);
result.SetValue((afterValue - beforeValue) * 0.5, setIndexes);
}
}
return result;
}
Here's a more generic method. It uses a few lambdas, so the it might also help you understand the use of lambdas a bit also.
// Iterates through every item in a multidementional array array
private Array MutateArray<T>(Array a, Func<T, int[], T> selector)
{
var rank = a.Rank;
var lengths = Enumerable.Range(0, a.Rank)
.Select(r => a.GetLength(r))
.ToArray(); // Get length of a in each dimension
var result = Array.CreateInstance(typeof(T), lengths);
var index = new int[a.Rank];
foreach (T item in a) // flattens array
{
result.SetValue(selector(item, index), index);
// Get next index value (I'm sure this could be improved)
for (var d = 0; d < rank; d++)
{
if (index[d] == lengths[d] - 1)
{
index[d] = 0;
}
else
{
index[d]++;
break;
}
}
}
return result;
}
// Your "foo" method
private double[,] generic_foo(double[,] a, int d)
{
var upperD = a.GetUpperBound(d);
return (double[,])MutateArray<double>(a, (x, i) =>
{
var prev = i.ToArray(); // clone
prev[d] = Math.Max(prev[d] - 1, 0);
var next = i.ToArray(); // clone
next[d] = Math.Min(next[d] + 1, upperD);
var prevVal = (double)a.GetValue(prev);
var nextVal = (double)a.GetValue(next);
return (nextVal - prevVal) * 0.5;
});
}

Would it be acceptable to do something along these lines?
int before_i = i, after_i = i;
int before_j = j, after_j = j;
switch( dimension ) {
case 0:
before_i = Math.max(i-1,0);
after_i = Math.min(i+1, width-1);
break;
case 1:
before_j = Math.max(j-1,0);
after_j = Math.min(j+1, height-1);
break;
}
result[ i, j ] = (a[after_i, after_j] - a[before_i,before_j]) * 0.5
It's not terribly pretty, but at least this way you don't need two functions.

You could pass in a delegate to extract the dimension you're interested in? (or a lambda)
Func<int[,],int,int[]> accessor here indicates the signature of a function (where the last template parameter is the return type)
private void Working()
{
DoSomething(GetRow,1);
}
So, in this example, you want the "DoSomething" worker to work on a row.
private void DoSomething(Func<int[,],int,int[]> accessor, int Idx)
{
int[,] theData = {{1,1,1,1,1},{2,2,2,2,2}};
int[] someData = accessor(theData,Idx);
}
public int[] GetRow(int[,] data,int index)
{
List<int> numbers = new List<int>();
for (int i = 0; i < data.GetLength(1); i++)
{
numbers.Add(data[index, i]);
}
return numbers.ToArray();
}
In the above example, you get a one dimensional array of 2,2,2,2,2
I'm addressing the general case of extracting a particular part of a multidimensional array here... The method/ lambda you pass in extracts the meaningful part of data...