Develop Reference

How should weights be optimized with gradient descent algorithm in order to work?

I have a neural network in visual studio. for the loss function I am using a basic cost function (pred-target)**2 and after I finish an epoch I optimize the parameter functions afterwards, but the algorithm doesn't work.
No matter what is my network configuration, the predictions are not write (it is the same output for all the inputs) and the loss function is not optimized. It stays the same through all the epochs.
void calc_lyr(int x, int y, int idx, float target) // thus function calculates the neuron value based on the previous layer
{
if (x == -1 || y == 0) // if its the first layer, get the data from input nodes
{
for (int i = 0; i < neurons[y]; i++)
{
float sum = 0;
for (int j = 0; j < inputTypes.Count; j++)
{
sum += weights[x+1][j][i] * training_test[idx][j];
}
sum = relu(sum);
vals[y+1][i] = sum;
}
}
else
{
for(int i = 0; i < neurons[y]; i++)
{
float sum = 0;
for(int j = 0; j < neurons[x]; j++)
{
sum += weights[x+1][j][i] * vals[x+1][j] + biases[y][i];
}
sum = relu(sum);
vals[y+1][i] = sum;
}
}
}
void train()
{
log("Proces de antrenare inceput ----------------- " + DateTime.Now.ToString());
vals = new List<List<float>>();
weights = new List<List<List<float>>>();
biases = new List<List<float>>();
Random randB = new Random(DateTime.Now.Millisecond);
Random randW = new Random(DateTime.Now.Millisecond);
for (int i = 0; i <= nrLayers; i++)
{
progressEpochs.Value =(int)(((float)i * (float)nrLayers) / 100.0f);
vals.Add(new List<float>());
weights.Add(new List<List<float>>());
if (i == 0)
{
for (int j = 0; j < inputTypes.Count; j++)
{
vals[i].Add(0);
}
}
else
{
biases.Add(new List<float>());
for (int j = 0; j < neurons[i-1]; j++)
{
vals[i].Add(0);
float valB = (float)randB.NextDouble();
biases[i-1].Add(valB - ((int)valB));
}
}
}
float valLB = (float)randB.NextDouble();
biases.Add(new List<float>());
biases[nrLayers].Add(valLB - ((int)valLB));
for (int i = 0; i <= nrLayers; i++)
{
if (i == 0)
{
for (int j = 0; j < inputTypes.Count; j++)
{
weights[i].Add(new List<float>());
for (int x = 0; x < neurons[i]; x++)
{
float valW = (float)randW.NextDouble();
weights[i][j].Add(valW);
}
}
}
else if (i == nrLayers)
{
for (int j = 0; j < neurons[i-1]; j++) {
weights[i].Add(new List<float>());
weights[i][j].Add(0);
}
}
else
{
for (int j = 0; j < neurons[i - 1]; j++)
{
weights[i].Add(new List<float>());
for (int x = 0; x < neurons[i]; x++)
{
float valW = (float)randW.NextDouble();
weights[i][j].Add(valW);
}
}
}
}
Random rand = new Random(DateTime.Now.Millisecond);
log("\n\n");
for (int i = 0; i < epochs; i++)
{
log("Epoch " + (i + 1).ToString() + " inceput ---> " + DateTime.Now.ToString());
int idx = rand.Next() % training_test.Count;
float target = outputsPossible.IndexOf(training_labels[idx]);
for (int j = 0; j < nrLayers; j++)
{
calc_lyr(j - 1, j, idx, target);
}
float total_val = 0;
for(int x = 0; x < neurons[nrLayers - 1]; x++)
{
float val = relu(weights[nrLayers][x][0] * vals[nrLayers][x] + biases[nrLayers][0]);
total_val += val;
}
total_val = sigmoid(total_val);
float cost_res = cost(total_val, target);
log("Epoch " + (i+1).ToString() + " terminat ----- " + DateTime.Now.ToString() + "\n");
log("Eroare epoch ---> " + (cost_res<1?"0":"") + cost_res.ToString("##.##") + "\n\n\n");
float cost_der = cost_d(total_val, target);
for (int a = 0; a < weights.Count; a++)
{
for (int b = 0; b < weights[a].Count; b++)
{
for (int c = 0; c < weights[a][b].Count; c++)
{
weights[a][b][c]-=cost_der*learning_rate * sigmoid_d(weights[a][b][c]);
}
}
}
for (int a = 0; a < nrLayers; a++)
{
for (int b = 0; b < neurons[a]; b++)
{
biases[a][b] -= cost_der * learning_rate;
}
}
}
hasTrained = true;
testBut.Enabled = hasTrained;
MessageBox.Show("Antrenament complet!");
SavePrompt sp = new SavePrompt();
sp.Show();
}
How can it be changed to optimize the weights, biases and loss function? For now, when I try to debug, the weights are changing, but it is the same value for the loss function.

I solved it by using AForge.NET: link

C#: Build 3D-Array and fill it with data

The code I tried so far below:
private static List<List<List<int>>> threeDArrayToThreeDList(int [,,] letters) {
// 3d-array to 3d-list
List<List<List<int>>> letterslist = new List<List<List<int>>>();
List<List<int>> sublist = new List<List<int>> ();
List<int> subsublist = new List<int> ();
for (int i = 0; i < 2; i++) {
letterslist.Add (sublist);
for (int j = 0; j < 2; j++) {
letterslist[i].Add (subsublist);
for (int k = 0; k < 2; k++) {
Console.WriteLine (letterslist [i][j][k]); // Element not found
Console.WriteLine (letters [i,j,k]);
letterslist [i] [j] [k] = letters [i,j,k];
}
}
}
return letterslist;
}
Why letterslist [i][j][k] isn't found?

Your code is wrong. You need to create a list for each "index". You're code only creates 3 lists altogether.
Here's how it should work:
private static List<List<List<int>>> threeDArrayToThreeDList(int [,,] letters) {
// 3d-array to 3d-list
List<List<List<int>>> letterslist = new List<List<List<int>>>();
for (int i = 0; i < 2; i++) {
letterslist.Add (new List<List<int>> ());
for (int j = 0; j < 2; j++) {
letterslist[i].Add (new List<int> ());
for (int k = 0; k < 2; k++) {
Console.WriteLine (letters [i,j,k]);
letterslist [i] [j].Add(letters [i,j,k]);
}
}
}
return letterslist;
}

This because the letterslist [i][j] list has no elements in it.
Add an element to it, and it will get you through the line that causes the exception.
Change the code in the innermost loop as follows:
for (int k = 0; k < 2; k++) {
letterslist[i][j].Add (letters[i, j, k]);
Console.WriteLine(letterslist[i][j][k]); // Should work fine
Console.WriteLine(letters[i, j, k]);
}

Converting Codility solution to C# (Grocery-store, Hydrogenium 2013)

In order to learn and understand how Dijkstra's algorithm is used to solve the "Grocery Store" ([Hydrogenium 2013]: https://codility.com/programmers/challenges/hydrogenium2013) problem on codility, I'm trying to rewrite the #2, O(n^2) solution(https://codility.com/media/train/solution-grocery-store.pdf) in C#.
1) What language are those solutions written in?
2) What would be the C# equivalent to this bit of code?
G = [[]] *N
for i in xrange(M):
G[A[i]] = G[A[i]] + [(B[i], C[i])]
G[B[i]] = G[B[i]] + [(A[i], C[i])]
This is what I have so far
int[] G = new int[N];
for (int i = 0; i < M; i++)
{
G[A[i]] = G[A[i]];
G[B[i]] = G[B[i]];
}
Thanks in advance,
Gregory

Downloaded IDLE (a python IDE...kind of) and figured it out. It appears to be adding pairs to each array element. Here's the code I came up with if anyone else happens to stumble across the same problem.
private struct nodePair
{
public int node;
public int time;
public nodePair(int node, int time)
{
this.node = node;
this.time = time;
}
}
public int solution(int[] A, int[] B, int[] C, int[] D)
{
int M = A.Length;
int N = D.Length;
//build the graph
List<nodePair>[] G = new List<nodePair>[N];
for (int i = 0; i < N; i++)
{
G[i] = new List<nodePair>();
}
for (int i = 0; i < M; i++)
{
G[A[i]].Add(new nodePair(B[i], C[i]));
G[B[i]].Add(new nodePair(A[i], C[i]));
}
//initialize the distance table
int[] dist = new int[N];
for (int i = 0; i < N; i++)
{
dist[i] = int.MaxValue;
}
bool[] visited = new bool[N];
for (int i = 0; i < N; i++)
{
visited[i] = false;
}
//look for the minimum value
int ii = 0; ;
dist[0] = 0;
for (int k = 0; k < N; k++)
{
int s = int.MaxValue;
//find the minimum
for (int j = 0; j < N; j++)
{
if ((dist[j] < s) && (visited[j] == false))
{
s = dist[j];
ii = j;
}
}
visited[ii] = true;
if (s < D[ii])
{
return s;
}
List<nodePair> thisNodeLIst = G[ii];
foreach (nodePair oneNode in thisNodeLIst)
{
dist[oneNode.node] = Math.Min(dist[oneNode.node], s + oneNode.time);
}
}//for
return -1;
}
}

To solve upside-down rendering?

I have written this segment of C# to help me understand how nested for loops can be used to render 2 dimensional data.
Here is what the output looks like.
████
███
██
█
I would like to make it so that the 4 blocks up top are rendered at the bottom, basically in the reverse order so that the steps ascend. However the console window only renders downward, so the conventional thinking won't be right. The following is my code.
static void Main(string[] args)
{
int i = 0;
int j = 0;
for (i = 0; i < 4; i++)
{
Console.Write('\n');
for (j = i; j < 4; j++)
{
Console.Write("█");
}
}
Console.ReadKey();
}
This is what I'd like the output to look like.
█
██
███
████

You need to reverse your loop condition from inremant to decremant..
for (i = 0; i < 4; i++)
{
Console.Write('\n');
for (j = i; j >= 0; j--)
{
Console.Write("█");
}
}
Output will be;
Here is a DEMO.
UPDATE: Since you change your mind, you need to add space every column (column number isi) 4 - 1 times.
public static void Main(string[] args)
{
int i = 0;
int j = 0;
for ( i = 0; i < 4; i++ )
{
for ( j = 0; j < 4; j++ )
{
if ( j < 3 - i )
Console.Write(" ");
else
Console.Write("█");
}
Console.Write('\n');
}
Console.ReadKey();
}
Here is a DEMO.

class Program
{
const int Dimension = 4;
static void Main(string[] args)
{
char[] blocks = new char[Dimension];
for (int j = 0; j < Dimension; j++)
blocks[j] = ' ';
for (int i = 0; i < Dimension; i++)
{
blocks[Dimension - i - 1] = '█';
for (int j = 0; j < Dimension; j++)
Console.Write(blocks[j]);
Console.WriteLine();
}
Console.ReadKey();
}
}

Should be:
for (j = 3 - i; j < 4; j++)
{
Console.Write("█");
}

The easiest way would be: just reverse your inner loop condition and decrement the counter instead of incrementing it:
for (i = 0; i < 4; i++)
{
Console.Write('\n');
for (j = i; j >= 0; j--)
{
Console.Write("█");
}
}
Console.ReadKey();
returning:
█
██
███
████
And for right-to-left version:
for (i = 0; i < 4; i++)
{
for(j = 0; j < 4; j++)
{
if(j < 3 - i)
Console.Write(" ");
else
Console.Write("█");
}
Console.Write('\n');
}
Console.ReadKey();
with a result:
█
██
███
████

How to sort a 2D array filld with random numbers

I'm doing a class assignment,
I need to create an 2D array of random numbers and sort them either bubble or other sorting codes. I'm fine with single array, but the problem is a 2D array filled with random numbers, I just don't get it.
Random numbers should be made of (-I,I) interval it's a user input. Sorry for bad english, haven't gotten any degree. In working on visual C# windows form.
looking for simple couple cicles method.
example. : A[MxN] ->>> B[MxN] (Sorted 1.....n)

Getting the random numbers is trivial:
Random rnd;
for (int y = 0; y < h; y++)
for (int x = 0; x < w; x++)
array[y][x] = l - rnd.Next(2 * l + 1);
Random.Next() will return a random value between 0 and the given parameter (excluding the parameter; which is the reason for the + 1).
A 2D array can be sorted just like a 1D array, it only depends how you want to handle multiple lines, e.g. is it just for display or do you want to sort every line for itself, etc.

Here's a solution which first sorts each row's columns into order, then sorts each row comparing by first column, then second, etc:
namespace StackOverflow.Demos
{
class Program
{
public static void Main(string[] args)
{
new Program();
Console.WriteLine("Done");
Console.ReadKey();
}
Program()
{
double[,] data = GenerateData();
OutputData(data, "Before");
SortData(ref data);
OutputData(data, "After");
}
double[,] GenerateData()
{
Random randomGenerator = new Random(DateTime.UtcNow.Millisecond);
double[,] data = new double[5, 5];
for (int i = 0; i < data.GetLength(0); i++)
{
for (int j = 0; j < data.GetLength(1); j++)
{
data[i, j] = (randomGenerator.NextDouble() * 2) - 1;
}
}
return data;
}
void OutputData(double[,] data, string message)
{
Console.WriteLine("=====================");
Console.WriteLine(message);
Console.WriteLine("=====================");
for (int i = 0; i < data.GetLength(0); i++)
{
for (int j = 0; j < data.GetLength(1); j++)
{
Console.Write(data[i, j]);
Console.Write("\t");
}
Console.WriteLine();
}
}
void SortData(ref double[,] data)
{
//sort sub arrays
SortDataRows(ref data);
//sort this array
for (int i = 0; i < data.GetLength(0)-1; i++)
{
for (int j = i; j < data.GetLength(0); j++)
{
for (int k = 0; k < data.GetLength(1); k++)
{
if (data[i, k].CompareTo(data[j, k]) < 0) //if already in order exit loop
{
break;
} else if (data[i, k].CompareTo(data[j, k]) > 0) //if out of order switch and loop
{
SwapRows(ref data, i, j);
break;
}//else orders are equal so far; continue to loop
}
}
}
}
void SortDataRows(ref double[,] data)
{
for (int row = 0; row < data.GetLength(0); row++)
{
for (int i = 0; i < data.GetLength(1) - 1; i++)
{
for (int j = i; j < data.GetLength(1); j++)
{
if (data[row, i].CompareTo(data[row, j]) > 0)
{
Swap<double>(ref data[row, i], ref data[row, j]);
}
}
}
}
}
void Swap<T>(ref T a, ref T b)
{
T temp = a;
a = b;
b = temp;
}
void SwapRows(ref double[,]data, int i, int j)
{
for (int k = 0; k < data.GetLength(1); k++)
{
Swap<double>(ref data[i, k], ref data[j, k]);
}
}
}
}
The code's not the best (haven't had a cup of tea yet), but should do what you're after.
Here's a better solution (not using a 2D array as such, but using a structure which can easily be converted to/from such an array):
sing System.Diagnostics;
namespace StackOverflow.Demos
{
class Program
{
public static void Main(string[] args)
{
new Program();
Console.WriteLine("Done");
Console.ReadKey();
}
Program()
{
List<List<double>> data = GenerateData(5, 5).ToList<List<double>>();
OutputData(data,"Before");
foreach (List<double> item in data)
{
item.Sort();
}
data.Sort(CompareListOfDoubles);
OutputData(data,"After");
}
private IEnumerable<List<double>> GenerateData(int index1, int index2)
{
Random rnd = new Random(DateTime.UtcNow.Millisecond);
List<double> result;
for (int i = 0; i < index1; i++)
{
result = new List<double>(index2);
for (int j = 0; j < index2; j++)
{
result.Add(rnd.NextDouble() * 2 - 1);
}
yield return result;
}
}
private void OutputData(List<List<double>> data, string message)
{
Console.WriteLine(message);
foreach (List<double> list in data)
{
foreach (double datum in list)
{
Console.Write(datum);
Console.Write("\t");
}
Console.WriteLine();
}
}
static int CompareListOfDoubles(List<double> a, List<double> b)
{
for (int i = 0; i < a.Count; i++)
{
if (i > b.Count) return -1;
if (a[i] > b[i]) return -1;
if (a[i] < b[i]) return 1;
}
if (b.Count > a.Count) return 1;
return 0;
}
double[,] ConvertListListDoubleTo2DArray(List<List<double>> data)
{
double[,] result = new double[data.Count, data[0].Count];
for (int i = 0; i < result.GetLength(0); i++)
{
for (int j = 0; j < result.GetLength(1); j++)
{
result[i, j] = data[i][j];
}
}
return result;
}
}