Develop Reference

It's possible to write neural network for different size of training data inputs and outputs

It's possible to write neural network for different size of training data inputs and outputs
for example:
inputs are 1. (1,2,3,4) , 2. (2,3,1), 3. (1,2,3,4,5) and so on...
and the same for outputs 1. (0,0,1,1) 2. (1,1,1) 3. (0,0,1,1,1)
So far I have managed to write the one which only works with the same size of training data which mean that all my training data needs to have the same length.
So far I'm stuck with this
NeuralNetwork net;
int[] layers = new int[3]
{
3/*Always the same*/,
1/*Always the same*/,
3 /*Always the same*/
};
string[] activation = new string[2] { "leakyrelu", "leakyrelu" };
net = new NeuralNetwork(layers, activation);
What I need
NeuralNetwork net1;
int[] layers1 = new int[3]
{
input.Length /*Based on input's Length*/,
1/*Always the same*/,
output.Length /*Based on output's Length*/
};
string[] activation1 = new string[2] { "leakyrelu", "leakyrelu" };
net = new NeuralNetwork(layers, activation);
// BackPropagate
public void BackPropagate(float[] inputs, float[] expected)
{
float[] output = FeedForward(inputs);
cost = 0;
for (int i = 0; i < output.Length; i++) cost += (float)Math.Pow(output[i] - expected[i], 2);
cost = cost / 2;
float[][] gamma;
List<float[]> gammaList = new List<float[]>();
for (int i = 0; i < layers.Length; i++)
{
gammaList.Add(new float[layers[i]]);
}
gamma = gammaList.ToArray();
int layer = layers.Length - 2;
for (int i = 0; i < output.Length; i++)
gamma[layers.Length-1][i] = (output[i] - expected[i]) * activateDer(output[i],layer);
for (int i = 0; i < neurons[layers.Length - 1].Length; i++)
{
biases[layers.Length - 1][i] -= gamma[layers.Length - 1][i] * learningRate;
for (int j = 0; j < neurons[layers.Length - 2].Length; j++)
{
weights[layers.Length - 2][i][j] -= gamma[layers.Length - 1][i] * neurons[layers.Length-2][j] * learningRate;
}
}
for (int i = layers.Length - 2; i > 0; i--)
{
layer = i - 1;
for (int j = 0; j < neurons[i].Length; j++)
{
gamma[i][j] = 0;
for (int k = 0; k < gamma[i+1].Length; k++)
{
gamma[i][j] = gamma[i + 1][k] * weights[i][k][j];
}
gamma[i][j] *= activateDer(neurons[i][j],layer);
}
for (int j = 0; j < neurons[i].Length; j++)
{
biases[i][j] -= gamma[i][j] * learningRate;
for (int k = 0; k < neurons[i-1].Length; k++)
{
weights[i - 1][j][k] -= gamma[i][j] * neurons[i-1][k] * learningRate;
}
}
}
}

Resilient Propagation (RProp) training slowly

I am creating a library for Neural Networks; I have successfully implemented the Back Propagation algorithm, but I'm having trouble with Resilient Propagation.
I have been using the XOR scenario to test my implementation; the error seems to decrease after a few epochs, then it increases and eventually stops changing.
The network I am testing on has 3 layers - 2 input, 2 hidden and 1 output neuron/s. The synapse/dendrite weights are on the following layer's neurons, for example, the input-hidden weights would be stored on the hidden neurons and the hidden-output weights would be stored on the output neurons.
This is my gradient calculation (EDIT 4):
private void CalculateGradient()
{
for (int t = 0; t < TrainingInputs.Count; ++t) //loop through training data
{
Network.Run(TrainingInputs[t]);
for (int l = Network.Layers.Count - 1; l > 0; l--)
{
for (int i = 0; i < Network.Layers[l].Neurons.Count; i++)
{
Network.Layers[l].Neurons[i].Delta = l < Network.Layers.Count - 1
? CalculateNonLastGradient(l + 1, i, Network.Layers[l].Neurons[i].Value)
: CalculateLastGradient(TrainingOutputs[t][i], Network.Layers[l].Neurons[i].Value);
}
}
for (int l = Network.Layers.Count - 1; l > 0; l--)
{
for (int j = 0; j < Network.Layers[l].Neurons.Count; ++j)
{
double grad = Network.Layers[l].Neurons[j].Delta;
biasGrads[l][j] += grad;
for (int i = 0; i < Network.Layers[l - 1].Neurons.Count; ++i)
{
grad = Network.Layers[l].Neurons[j].Delta * Network.Layers[l - 1].Neurons[i].Value;
grads[l][j][i] += grad;
}
}
}
int o = 0;
Layer layer = Network.Layers[Network.Layers.Count - 1];
errors.Add(layer.Neurons.Sum(n => Math.Abs(TrainingOutputs[t][o++] - n.Value)));
}
Error = errors.Average();
}
private double CalculateLastGradient(double ideal, double nValue)
{
return Network.Activation.Derivitive(nValue) * (ideal - nValue);
}
private double CalculateNonLastGradient(int nextLayer, int j, double nValue)
{
double sum = 0.0;
for (int i = 0; i < Network.Layers[nextLayer].Neurons.Count; i++)
{
sum += Network.Layers[nextLayer].Neurons[i].Delta * Network.Layers[nextLayer].Neurons[i].Dendrites[j].Weight;
}
return Network.Activation.Derivitive(nValue) * sum;
}
My RProp implementation (EDIT 4):
public bool Algorithm()
{
//initialise matrices
if (!initializedMatrices)
{
InitializeMatrices();
}
ZeroOut();
//calculate gradients
CalculateGradient();
for (int l = 1; l < Network.Layers.Count; l++) //layers
{
for (int i = 0; i < Network.Layers[l - 1].Neurons.Count; ++i) //prev layer neurons
{
for (int j = 0; j < Network.Layers[l].Neurons.Count; ++j) //current layer neurons
{
double delta = prevDeltas[l][j][i];
int change = Math.Sign(prevGrads[l][j][i] * grads[l][j][i]);
if (change > 0)
{
delta = Math.Min(delta * etaPlus, deltaMax);
double deltaWeight = -Math.Sign(grads[l][j][i]) * delta;
Network.Layers[l].Neurons[j].Dendrites[i].Weight += deltaWeight;
}
else if (change < 0)
{
delta = Math.Max(delta * etaMinus, deltaMin);
Network.Layers[l].Neurons[j].Dendrites[i].Weight -= prevDeltas[l][j][i];
prevGrads[l][j][i] = 0;
}
else if (change == 0)
{
double deltaWeight = -Math.Sign(grads[l][j][i]) * delta;
Network.Layers[l].Neurons[j].Dendrites[i].Weight += deltaWeight;
}
prevGrads[l][j][i] = grads[l][j][i];
prevDeltas[l][j][i] = delta;
} //j
} //i
for (int i = 1; i < Network.Layers[l].Neurons.Count; ++i)
{
double delta = prevBiasDeltas[l][i];
int change = Math.Sign(prevBiasGrads[l][i] * biasGrads[l][i]);
if (change > 0)
{
delta = Math.Min(prevBiasDeltas[l][i] * etaPlus, deltaMax);
double biasDeltaWeight = -Math.Sign(biasGrads[l][i]) * delta;
Network.Layers[l].Neurons[i].Bias += biasDeltaWeight;
}
else if (change < 0)
{
delta = Math.Max(prevBiasDeltas[l][i] * etaMinus, deltaMin);
Network.Layers[l].Neurons[i].Bias -= prevBiasDeltas[l][i];
prevBiasGrads[l][i] = 0;
}
else if (change == 0)
{
double biasDeltaWeight = -Math.Sign(biasGrads[l][i]) * delta;
Network.Layers[l].Neurons[i].Bias += biasDeltaWeight;
}
prevBiasGrads[l][i] = biasGrads[l][i];
prevBiasDeltas[l][i] = delta;
}
}
return true;
}
Could anyone point me as to what could be going wrong?
EDIT:
The issue seems to be that the delta is not changing.
EDIT 2:
I have fixed the delta not changing by initializing the first previous deltas to 0.01 and Zero'ing the gradients before each call. The error increases very fast, then decreases very slowly with TANH; with Sigmoid it does the opposite.
EDIT 3:
The loop for the bias' was starting at 0, when it should have been 1. Fixing this fixed the original issue, but has brought up at new one - the error stops decreasing after a certain point.
EDIT 4: I realised that I had not accumulated the gradients for each neuron over the training set. The error now decreases, but very slowly.

Chess board positions

How can i make this so the user types positions in console (table[4,5]) i want that user types that?
int[,] table = new int[8, 8];
for (int i = 0; i < 8; i++)
{
for (int j = 0; j < 8; j++)
{
if ((i + j) % 2 == 0)
{
table[i, j] = 0;
}
else
{
table[i, j] = 1;
}
}
}
Console.WriteLine("4 - king");
Console.WriteLine("3 - queen");
Console.WriteLine("4 - hunter");
table[4,5] = 2;
table[6,7] = 3;
table[2,2] = 4;
for (int i = 0; i < 8; i++)
{
for (int j = 0; j < 8; j++)
{
Console.Write(table[i, j] + " ");
}
Console.WriteLine();
}
What should i do to make this work?And if i type this it doesnt work:
Thats why i have to put figures in different rows or columns how do i fix this
table[4,5] = 2;
table[4,7] = 3;
table[2,2] = 4;

You are confusing the assignment operator "=" with the logical comparison operator "==". Your second line is just comparing table[4,5] with 2 and probably returning false.
Change it to:
table[4,5] = 2;
Also, even if you manage to assign a value to table[4,5], you will overwrite it in the next lines. You should move that line to the end of the first nested loop. Just before the second "for (int i = 0; i < 8; i++)"

C# LevenshteinDistance algorithm for spellchecker

Hi i'm using the levenshtein algorithm to calculate the difference between two strings, using the below code. It currently provides the total number of changes which need to be made to get from 'answer' to 'target', but i'd like to split these up into the types of errors being made. So classifying an error as a deletion, substitution or insertion.
I've tried adding a simple count but i'm new at this and don't really understand how the code works so not sure how to go about it.
static class LevenshteinDistance
{
/// <summary>
/// Compute the distance between two strings.
/// </summary>
public static int Compute(string s, string t)
{
int n = s.Length;
int m = t.Length;
int[,] d = new int[n + 1, m + 1];
// Step 1
if (n == 0)
{
return m;
}
if (m == 0)
{
return n;
}
// Step 2
for (int i = 0; i <= n; d[i, 0] = i++)
{
}
for (int j = 0; j <= m; d[0, j] = j++)
{
}
// Step 3
for (int i = 1; i <= n; i++)
{
//Step 4
for (int j = 1; j <= m; j++)
{
// Step 5
int cost = (t[j - 1] == s[i - 1]) ? 0 : 1;
// Step 6
d[i, j] = Math.Min(
Math.Min(d[i - 1, j] + 1, d[i, j - 1] + 1),
d[i - 1, j - 1] + cost);
}
}
// Step 7
return d[n, m];
}
}
Thanks in advance.

Simple DFT Lowpass

I'm having some trouble making a simple lowpass filter with a DFT. In the end, I hope to be able to pitch-shift audio in real time, but as it stands I can't even get this right. I have no training in this area, I only know that FFTs change waves to frequencies and iFFTs do that back, and a couple of other things I've read. To bo honest I'm surprised it works as well as it does so far. Anyway here's the code:
byte[] samples = new byte[20000000];
int spos = 0;
samples is filled here with 8Bit Unsigned PCM. spos <- number of samples
int samplesize = 128;
int sampleCount = spos / samplesize;
frequencies = new System.Numerics.Complex[sampleCount][];
for (int i = 0; i < sampleCount; i++)
{
Console.WriteLine("Sample " + i + " / " + sampleCount);
frequencies[i] = new System.Numerics.Complex[samplesize];
for (int j = 0; j < samplesize; j++)
{
frequencies[i][j] = (float)(samples[i * samplesize + j] - 128) / 128.0f;
}
dft.Radix2Forward(frequencies[i], MathNet.Numerics.IntegralTransforms.FourierOptions.Default);
}
int shiftUp = 1000; //1khz
int fade = 2; //8 sample fade.
int kick = frequencies[0].Length * shiftUp / rate;
So now I've calculated a bunch of DFTs for 128 sample portions of the input. kick is (I hope) the number of samples in the DFT that span 1000Hz. I.E since frequencies.Length / 2 contains frequency amplitude data up to rate/2 Hz, then frequencies[0].Length / 2 * shiftUp / (rate / 2) = frequencies[0].Length * shiftUp / rate should give me the right value
for (int i = 0; i < sampleCount; i++)
{
This is the part I have trouble with. Without it, the output sounds great! This skips both index 0 and index 64. Both of these have a complex component of 0, and I recall reading somewhere that the value at index 0 was important...
for (int j = 0; j < frequencies[i].Length; j++)
{
if (j == 0 || j == 64)
continue;
if (j < 64)
{
if (!(j < kick + 1))
{
frequencies[i][j] = 0;
}
}
else
{
if (!(j - 64 > 63 - kick))
{
frequencies[i][j] = 0;
}
}
}
Finally it undoes the transform
dft.Radix2Inverse(frequencies[i], MathNet.Numerics.IntegralTransforms.FourierOptions.Default);
...tosses it back in the samples array
for (int j=0; j<samplesize; j++)
samples[i * samplesize + j] = (byte)(frequencies[i][j].Real * 128.0f + 128.0f);
}
...chucks it into a file
BinaryWriter bw = new BinaryWriter(File.OpenWrite("sound"));
for (int i = 0; i < spos; i++)
{
bw.Write(samples[i]);
}
bw.Close();
...then I import it into Audacity to murder my ears with artifacts.
The spectral display shows that the code works, to an extent
However there's these annoying highpitched crackling sounds that occur throughout the entire song. I've heard something about the Gibbs phenomenon and a window function, but I don't really know how to apply that here. The fade variable is my best attempt at a window function: everything past the 1000hz mark fades to 0 in 2 samples.
Any ideas?
Thanks!

So it turns out that I was right (yay): Every 1024 samples I was getting a click sound which made the audio sound awful. To fix this, I faded inbetween many short overlapping chunks of filtered audio. It's not fast, but it works, and I'm pretty sure this is what they mean by "windowing"
public class OggDFT
{
int sample_length;
byte[] samples;
DragonOgg.MediaPlayer.OggFile f;
int rate = 0;
System.Numerics.Complex[][] frequencies;
DiscreteFourierTransform dft = new DiscreteFourierTransform();
int samplespacing = 128;
int samplesize = 1024;
int sampleCount;
public void ExampleLowpass()
{
int shiftUp = 1000; //1khz
int fade = 2; //8 sample fade.
int halfsize = samplesize / 2;
int kick = frequencies[0].Length * shiftUp / rate;
for (int i = 0; i < sampleCount; i++)
{
for (int j = 0; j < frequencies[i].Length; j++)
{
if (j == 0 || j == halfsize)
continue;
if (j < halfsize)
{
if (!(j < kick + 1))
{
frequencies[i][j] = 0;
}
}
else
{
if (!(j - halfsize > halfsize - 1 - kick))
{
frequencies[i][j] = 0;
}
}
}
dft.BluesteinInverse(frequencies[i], MathNet.Numerics.IntegralTransforms.FourierOptions.Default);
}
}
public OggDFT(DragonOgg.MediaPlayer.OggFile f)
{
Complex[] c = new Complex[10];
for (int i = 0; i < 10; i++)
c[i] = i;
ShiftComplex(-2, c, 5, 10);
this.f = f;
//Make a 20MB buffer.
samples = new byte[20000000];
int sample_length = 0;
//This block here simply loads the uncompressed data from the ogg file into a nice n' large 20MB buffer. If you want to use the same library as I've used, It's called DragonOgg (If you cant tell by the namespace)
while (sample_length < samples.Length)
{
var bs = f.GetBufferSegment(4096); //Get ~4096 bytes (does not gurantee that 4096 bytes will be returned.
if (bs.ReturnValue == 0)
break; //End of file
//Set the rate
rate = bs.RateHz;
//Display some loading info:
Console.WriteLine("seconds: " + sample_length / rate);
//It's stereo so we want half the data.
int max = bs.ReturnValue / 2;
//Buffer overflow care.
if (samples.Length - sample_length < max)
max = samples.Length - sample_length;
//The copier.
for (int j = 0; j < max; j++)
{
//I'm using j * 2 here because I know that the input audio is 8Bit Stereo, and we want just one mono channel. So we skip every second one.
samples[sample_length + j] = bs.Buffer[j * 2];
}
sample_length += max;
if (max == 0)
break;
}
sampleCount = (sample_length - 1) / samplespacing + 1;
frequencies = new System.Numerics.Complex[sampleCount][];
for (int i = 0; i < sample_length; i += samplespacing)
{
Console.WriteLine("Sample---" + i + " / " + sample_length);
System.Numerics.Complex[] sample;
if (i + samplesize > sample_length)
sample = new System.Numerics.Complex[sample_length - i];
else
sample = new System.Numerics.Complex[samplesize];
for (int j = 0; j < sample.Length; j++)
{
sample[j] = (float)(samples[i + j] - 128) / 128.0f;
}
dft.BluesteinForward(sample, MathNet.Numerics.IntegralTransforms.FourierOptions.Default);
frequencies[i / samplespacing] = sample;
}
//Perform the filters to the frequencies
ExampleLowpass();
//Make window kernel thingy
float[] kernel = new float[samplesize / samplespacing * 2];
for (int i=0; i<kernel.Length; i++)
{
kernel[i] = (float)((1-Math.Cos(2*Math.PI*i/(kernel.Length - 1)))/2);
}
//Apply window kernel thingy
for (int i = 0; i < sample_length; i++)
{
int jstart = i / samplespacing - samplesize / samplespacing + 1;
int jend = i / samplespacing;
if (jstart < 0) jstart = 0;
float ktotal = 0;
float stotal = 0;
for (int j = jstart; j <= jend; j++)
{
float kernelHere = 1.0f;
if (jstart != jend)
kernelHere = kernel[(j - jstart) * kernel.Length / (jend + 1 - jstart)];
int index = i - j * samplespacing;
stotal += (float)frequencies[j][index].Real * kernelHere;
ktotal += kernelHere;
}
if (ktotal != 0)
{
stotal /= ktotal;
samples[i] = (byte)(stotal * 128 * 0.9f + 128);
}
else
{
Console.WriteLine("BAD " + jstart + " " + jend + " sec: " + ((float)i / rate));
samples[i] = (byte)(stotal * 128 * 0.9f + 128);
}
}
BinaryWriter bw = new BinaryWriter(File.OpenWrite("sound"));
for (int i = 0; i < sample_length; i++)
{
bw.Write(samples[i]);
}
bw.Close();
}
}
If you want to compile this, you'll need DragonOgg (http://sourceforge.net/projects/dragonogg/) and MathNet.Numerics (http://mathnetnumerics.codeplex.com/)
I hope it helps someone - I don't know about how StackOverflow licences by default, but this code as it stands is public domain.
On further thought, I decided I could achieve an approximated effect much easier by simply "blurring" the samples to get a basic low pass filter. A high pass filter can be made by subtracting the result of the low pass.