+ * This method performs an element-wise subtraction between two matrices.
+ * It requires that both matrices have the same dimensions. If the matrices
+ * do not have the same dimensions, an IllegalArgumentException is thrown.
+ *
+ *
+ * @param matrix The matrix to be subtracted from this matrix.
+ * @return A new Matrix object representing the result of the subtraction.
+ * @throws IllegalArgumentException if the input matrix and this matrix do not have the same dimensions.
+ */
+ public Matrix subtract(Matrix matrix) {
+ if (this.rows != matrix.rows || this.cols != matrix.cols) {
+ throw new IllegalArgumentException("Matrices must have the same size.");
+ }
+
+ Matrix result = new Matrix(this.rows, this.cols);
+ for (int i = 0; i < this.data.length; i++) {
+ result.data[i] = this.data[i] - matrix.getData()[i];
+ }
+ return result;
+ }
+
+ public Matrix relu() {
+ return this.apply((index, value) -> Math.max(0, value));
+ }
+
+ public Matrix reluDerivative(Matrix input) {
+ return this.apply((index, value) -> input.get(index) > 0 ? value : 0);
+ }
+
+ public Matrix print() {
+ System.out.println(this);
+ return this;
+ }
+
+ public void set(int row, int col, double value) {
+ data[row * cols + col] = value;
+ }
+
+ public double get(int row, int col) {
+ return data[row * cols + col];
+ }
+
+ public Matrix addIncrement(int row, int col, double increment) {
+ Matrix result = apply((index, value) -> data[index]);
+ double originalValue = result.get(row, col);
+ double newValue = originalValue + increment;
+ result.set(row, col, newValue);
+
+ return result;
+ }
+
+ public Matrix transpose() {
+ Matrix result = new Matrix(cols, rows);
+ for (int row = 0; row < rows; row++) {
+ for (int col = 0; col < cols; col++) {
+ result.data[col * rows + row] = data[row * cols + col];
+ }
+ }
+ return result;
+ }
+
+ public Matrix transposeParallel() {
+ Matrix result = new Matrix(cols, rows);
+
+ IntStream.range(0, rows).parallel().forEach(row -> {
+ for (int col = 0; col < cols; col++) {
+ result.data[col * rows + row] = data[row * cols + col];
+ }
+ });
+
+ return result;
+ }
+
+ public double get(int index) {
+ return this.getData()[index];
+ }
+
+ public Matrix multiply(double rate) {
+ return this.apply((index, value) -> value * rate);
+ }
+
+ public Matrix copy() {
+ Matrix result = new Matrix(rows, cols);
+ System.arraycopy(data, 0, result.data, 0, data.length);
+ return result;
+ }
+
+
+ public interface RowColumnProducer {
+ double produce(int row, int col, double value);
+ }
+
+ public interface Producer {
+ double produce(int index);
+ }
+
+ public interface IndexValueProducer {
+ double produce(int index, double value);
+ }
+
+ public interface ValueProducer {
+ double produce(double value);
+ }
+
+ public interface IndexValueConsumer {
+ void consume(int index, double value);
+ }
+
+ public interface RowColValueConsumer {
+ void consume(int row, int col, double value);
+ }
+
+
+ public interface RowColIndexValueConsumer {
+ void consume(int row, int col, int index, double value);
+ }
+
+ public double[] getData() {
+ return data;
+ }
+
+
+
+ public Matrix(int rows, int cols) {
+ data = new double[rows * cols];
+ this.rows = rows;
+ this.cols = cols;
+ }
+
+ public Matrix(int rows, int cols, Producer producer) {
+ this(rows, cols);
+ for (int i = 0; i < data.length; i++) {
+ data[i] = producer.produce(i);
+ }
+ }
+
+ public Matrix(int rows, int cols, double[] values) {
+ this.rows = rows;
+ this.cols = cols;
+
+ Matrix temp = new Matrix(cols, rows);
+ temp.data = values;
+ Matrix transposed = temp.transpose();
+ data = transposed.data;
+ }
+
+ public Matrix(double[][]values){
+ this.rows = values.length;
+ this.cols = values[0].length;
+ this.data = new double[rows*cols];
+ for(int i = 0; i < rows; i++){
+ for(int j = 0; j < cols; j++){
+ this.data[i*cols+j] = values[i][j];
+ }
+ }
+ }
+
+ public Matrix apply(IndexValueProducer function) {
+ Matrix result = new Matrix(rows, cols);
+ for (int i = 0; i < data.length; i++) {
+ result.data[i] = function.produce(i, data[i]);
+ }
+ return result;
+ }
+
+ public Matrix multiply(Matrix other) {
+ if (cols != other.rows) {
+ throw new IllegalArgumentException("Matrix dimensions do not match");
+ }
+ Matrix result = new Matrix(rows, other.cols);
+ for (int row = 0; row < rows; row++) {
+ for (int col = 0; col < other.cols; col++) {
+ double sum = 0;
+ for (int i = 0; i < cols; i++) {
+ sum += data[row * cols + i] * other.data[i * other.cols + col];
+ }
+ result.data[row * other.cols + col] = sum;
+ }
+ }
+ return result;
+ }
+
+ public Matrix multiplyParallel(Matrix other) {
+ if (cols != other.rows) {
+ throw new IllegalArgumentException("Matrix dimensions do not match");
+ }
+ Matrix result = new Matrix(rows, other.cols);
+ IntStream.range(0, rows)
+ .parallel()
+ .forEach(
+ row -> {
+ for (int col = 0; col < other.cols; col++) {
+ double sum = 0;
+ for (int i = 0; i < cols; i++) {
+ sum += data[row * cols + i] * other.data[i * other.cols + col];
+ }
+ result.data[row * other.cols + col] = sum;
+ }
+ });
+
+ return result;
+ }
+
+ public Matrix averageColumn() {
+ Matrix result = new Matrix(rows, 1);
+ forEach((row, col, value) -> {
+ result.data[row] += value / cols;
+ });
+ return result;
+ }
+
+ public Matrix add(Matrix other) {
+ // Überprüfen, ob die andere Matrix eine Spaltenmatrix ist, die als Bias verwendet werden kann
+ if (this.cols != other.cols && other.cols != 1) {
+ throw new IllegalArgumentException(
+ "Für die Addition muss die zweite Matrix entweder dieselbe Größe haben oder eine Spaltenmatrix sein.");
+ }
+
+ Matrix result = new Matrix(rows, cols);
+ for (int row = 0; row < this.rows; row++) {
+ for (int col = 0; col < this.cols; col++) {
+ if (other.cols == 1) {
+ // Addiere den Bias, wenn die zweite Matrix eine Spaltenmatrix ist
+ result.data[row * cols + col] = this.data[row * cols + col] + other.data[row];
+ } else {
+ // Normale elementweise Addition, wenn die zweite Matrix dieselbe Größe hat
+ result.data[row * cols + col] =
+ this.data[row * cols + col] + other.data[row * cols + col];
+ }
+ }
+ }
+
+ return result;
+ }
+
+ public Matrix modify(RowColumnProducer function) {
+ int index = 0;
+ for (int row = 0; row < rows; row++) {
+ for (int col = 0; col < cols; col++, index++) {
+ data[index] = function.produce(row, col, data[index]);
+ }
+ }
+ return this;
+ }
+
+ public Matrix modify(ValueProducer function) {
+ for (int i = 0; i < data.length; i++) {
+ data[i] = function.produce(data[i]);
+ }
+ return this;
+ }
+
+ public Matrix modify(IndexValueProducer function) {
+ for (int i = 0; i < data.length; i++) {
+ data[i] = function.produce(i, data[i]);
+ }
+ return this;
+ }
+
+
+ public void forEach(IndexValueConsumer consumer) {
+ for (int i = 0; i < data.length; i++) {
+ consumer.consume(i, data[i]);
+ }
+ }
+
+ public void forEach(RowColIndexValueConsumer consumer) {
+ int index = 0;
+ for (int row = 0; row < rows; row++) {
+ for (int col = 0; col < cols; col++) {
+ consumer.consume(row, col, index, data[index++]);
+ }
+ }
+ }
+
+ public void forEach(RowColValueConsumer consumer) {
+ int index = 0;
+ for (int row = 0; row < rows; row++) {
+ for (int col = 0; col < cols; col++) {
+ consumer.consume(row, col, data[index++]);
+ }
+ }
+ }
+
+ public void setTolerance(double tolerance) {
+ this.tolerance = tolerance;
+ }
+
+ public Matrix sumColumns() {
+ Matrix result = new Matrix(1, cols);
+ int index = 0;
+ for (int row = 0; row < rows; row++) {
+ for (int col = 0; col < cols; col++) {
+ result.data[col] += data[index++];
+ }
+ }
+
+ return result;
+ }
+
+ public Matrix softmax() {
+ Matrix result = new Matrix(rows, cols, i -> Math.exp(data[i]));
+ Matrix colSum = result.sumColumns();
+
+ result.modify((row, col, value) -> {
+ return value / colSum.getData()[col];
+ });
+ return result;
+ }
+
+ public Matrix getGreatestRowNumber() {
+ Matrix result = new Matrix(1, cols);
+ double[] greatest = new double[cols];
+ for (int i = 0; i < cols; i++) {
+ greatest[i] = Double.MIN_VALUE;
+ }
+
+ forEach((row, col, value) -> {
+ if (value > greatest[col]) {
+ greatest[col] = value;
+ result.data[col] = row;
+ }
+ });
+ return result;
+ }
+
+ public int getRows() {
+ return rows;
+ }
+
+ public int getCols() {
+ return cols;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ Matrix matrix = (Matrix) o;
+
+ for (int i = 0; i < data.length; i++) {
+ if (Math.abs(data[i] - matrix.data[i]) > tolerance) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ int result = Objects.hash(rows, cols);
+ result = 31 * result + Arrays.hashCode(data);
+ return result;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+
+ // Berechnen der maximalen Breite jeder Spalte
+ int[] maxWidth = new int[cols];
+ for (int row = 0; row < rows; row++) {
+ for (int col = 0; col < cols; col++) {
+ int length = String.format(NUMBER_FORMAT, data[row * cols + col]).length();
+ if (length > maxWidth[col]) {
+ maxWidth[col] = length;
+ }
+ }
+ }
+
+ // Hinzufügen der Rahmenlinien und der Daten
+ String rowSeparator =
+ "+"
+ + Arrays.stream(maxWidth)
+ .mapToObj(width -> "-".repeat(width + 2))
+ .collect(Collectors.joining("+"))
+ + "+\n";
+
+ for (int row = 0; row < rows; row++) {
+ sb.append(rowSeparator);
+ sb.append("|");
+ for (int col = 0; col < cols; col++) {
+ String formattedNumber =
+ String.format(
+ "%" + maxWidth[col] + "s", String.format(NUMBER_FORMAT, data[row * cols + col]));
+ sb.append(" ").append(formattedNumber).append(" |");
+ }
+ sb.append("\n");
+ }
+ sb.append(rowSeparator);
+
+ return sb.toString();
+ }
+
+ public String toString(boolean showValues) {
+ if (showValues) {
+ return toString();
+ } else {
+ return "{" +
+ "rows=" + rows +
+ ", cols=" + cols +
+ '}';
+ }
+ }
+}
diff --git a/lib/src/main/java/de/edux/ml/mlp/core/transformer/Transform.java b/lib/src/main/java/de/edux/ml/mlp/core/transformer/Transform.java
new file mode 100644
index 0000000..16caad6
--- /dev/null
+++ b/lib/src/main/java/de/edux/ml/mlp/core/transformer/Transform.java
@@ -0,0 +1,5 @@
+package de.edux.ml.mlp.core.transformer;
+
+public enum Transform {
+ DENSE, RELU, SOFTMAX;
+}
diff --git a/lib/src/main/java/de/edux/ml/mlp/exceptions/LoaderException.java b/lib/src/main/java/de/edux/ml/mlp/exceptions/LoaderException.java
new file mode 100644
index 0000000..f6ae931
--- /dev/null
+++ b/lib/src/main/java/de/edux/ml/mlp/exceptions/LoaderException.java
@@ -0,0 +1,7 @@
+package de.edux.ml.mlp.exceptions;
+
+public class LoaderException extends RuntimeException{
+ public LoaderException(String message) {
+ super(message);
+ }
+}
diff --git a/lib/src/main/java/de/edux/ml/mlp/exceptions/UnsupportedLayerException.java b/lib/src/main/java/de/edux/ml/mlp/exceptions/UnsupportedLayerException.java
new file mode 100644
index 0000000..a1eb422
--- /dev/null
+++ b/lib/src/main/java/de/edux/ml/mlp/exceptions/UnsupportedLayerException.java
@@ -0,0 +1,28 @@
+package de.edux.ml.mlp.exceptions;
+
+/**
+ * This class represents an exception that is thrown when an unsupported
+ * layer type is encountered in the MLP context. It extends RuntimeException
+ * to indicate that this is an unchecked exception that might occur during
+ * the runtime of the application, particularly when configuring or building
+ * MLP models with incompatible or unsupported layer types.
+ */
+public class UnsupportedLayerException extends RuntimeException {
+
+ /**
+ * Constructs a new UnsupportedLayerException with the default message.
+ */
+ public UnsupportedLayerException() {
+ super("The specified layer type is not supported.");
+ }
+
+ /**
+ * Constructs a new UnsupportedLayerException with a custom message.
+ *
+ * @param message the detail message. The detail message is saved for
+ * later retrieval by the Throwable.getMessage() method.
+ */
+ public UnsupportedLayerException(String message) {
+ super(message);
+ }
+}
diff --git a/lib/src/main/java/de/edux/ml/mlp/exceptions/UnsupportedLossFunction.java b/lib/src/main/java/de/edux/ml/mlp/exceptions/UnsupportedLossFunction.java
new file mode 100644
index 0000000..79203d9
--- /dev/null
+++ b/lib/src/main/java/de/edux/ml/mlp/exceptions/UnsupportedLossFunction.java
@@ -0,0 +1,28 @@
+package de.edux.ml.mlp.exceptions;
+
+/**
+ * Represents an exception that is thrown when an unsupported loss function is encountered.
+ * This class is part of the MLP (Multi-Layer Perceptron) framework and is used to signal
+ * that the specified loss function is not supported by the current implementation.
+ *
+ * It extends RuntimeException, which allows this exception to be thrown and propagated
+ * through the call stack without being explicitly declared in method signatures.
+ */
+public class UnsupportedLossFunction extends RuntimeException {
+
+ /**
+ * Constructs a new UnsupportedLossFunction exception with the default message.
+ */
+ public UnsupportedLossFunction() {
+ super("Unsupported loss function.");
+ }
+
+ /**
+ * Constructs a new UnsupportedLossFunction exception with a custom message.
+ *
+ * @param message The custom message that describes this exception.
+ */
+ public UnsupportedLossFunction(String message) {
+ super(message);
+ }
+}
diff --git a/lib/src/main/java/de/edux/ml/mlp/util/TrainingArrays.java b/lib/src/main/java/de/edux/ml/mlp/util/TrainingArrays.java
new file mode 100644
index 0000000..070f6ac
--- /dev/null
+++ b/lib/src/main/java/de/edux/ml/mlp/util/TrainingArrays.java
@@ -0,0 +1,27 @@
+package de.edux.ml.mlp.util;
+
+public class TrainingArrays {
+ private double[] input;
+ private double[] output;
+
+ public TrainingArrays(double[] input, double[] output) {
+ this.input = input;
+ this.output = output;
+ }
+
+ public double[] getInput() {
+ return input;
+ }
+
+ public void setInput(double[] input) {
+ this.input = input;
+ }
+
+ public double[] getOutput() {
+ return output;
+ }
+
+ public void setOutput(double[] output) {
+ this.output = output;
+ }
+}
diff --git a/lib/src/main/java/de/edux/ml/mlp/util/TrainingMatrices.java b/lib/src/main/java/de/edux/ml/mlp/util/TrainingMatrices.java
new file mode 100644
index 0000000..b79ee07
--- /dev/null
+++ b/lib/src/main/java/de/edux/ml/mlp/util/TrainingMatrices.java
@@ -0,0 +1,29 @@
+package de.edux.ml.mlp.util;
+
+import de.edux.ml.mlp.core.tensor.Matrix;
+
+public class TrainingMatrices {
+ private Matrix input;
+ private Matrix output;
+
+ public TrainingMatrices(Matrix input, Matrix output) {
+ this.input = input;
+ this.output = output;
+ }
+
+ public Matrix getInput() {
+ return input;
+ }
+
+ public Matrix getOutput() {
+ return output;
+ }
+
+ public void setInput(Matrix input) {
+ this.input = input;
+ }
+
+ public void setOutput(Matrix output) {
+ this.output = output;
+ }
+}
diff --git a/lib/src/main/java/de/edux/ml/mlp/util/Util.java b/lib/src/main/java/de/edux/ml/mlp/util/Util.java
new file mode 100644
index 0000000..0be9be0
--- /dev/null
+++ b/lib/src/main/java/de/edux/ml/mlp/util/Util.java
@@ -0,0 +1,81 @@
+package de.edux.ml.mlp.util;
+
+import de.edux.ml.mlp.core.tensor.Matrix;
+import java.util.Random;
+
+public class Util {
+
+ private static final Random random = new Random();
+
+ public static Matrix generateInputMatrix(int rows, int cols) {
+ return new Matrix(rows, cols, i -> random.nextGaussian());
+ }
+
+ public static Matrix generateExpectedMatrix(int rows, int cols) {
+ Matrix expected = new Matrix(rows, cols, i -> 0);
+ for (int col = 0; col < cols; col++) {
+ int randowmRow = random.nextInt(rows);
+ expected.set(randowmRow, col, 1);
+ }
+
+ return expected;
+ }
+
+ public static Matrix generateTrainableExpectedMatrix(int outputRows, Matrix input) {
+ Matrix expected = new Matrix(outputRows, input.getCols());
+
+ Matrix columnSum = input.sumColumns();
+ columnSum.forEach((row, col, value) -> {
+ int rowIndex = (int) (outputRows * (Math.sin(value) + 1) / 2.0);
+ expected.set(rowIndex, col, 1);
+ });
+
+ return expected;
+
+ }
+
+ /**
+ * Generates a matrix with a gaussian distribution and a radius between 0 and outputRows
+ *
+ * @param inputRows number of rows
+ * @param outputRows number of rows
+ * @param cols number of columns
+ * @return a matrix with a gaussian distribution and a radius between 0 and outputRows
+ */
+ public static TrainingMatrices generateTrainingMatrices(int inputRows, int outputRows, int cols) {
+ var io = generateTrainingArrays(inputRows, outputRows, cols);
+ Matrix input = new Matrix(inputRows, cols, io.getInput());
+ Matrix output = new Matrix(outputRows, cols, io.getOutput());
+
+ return new TrainingMatrices(input, output);
+ }
+
+ public static TrainingArrays generateTrainingArrays(int inputSize, int outputSize, int numberItems) {
+ double[] input = new double[inputSize * numberItems];
+ double[] output = new double[outputSize * numberItems];
+
+ int inputPos = 0;
+ int outputPos = 0;
+ for (int col = 0; col < numberItems; col++) {
+ int radius = random.nextInt(outputSize);
+
+ double[] values = new double[inputSize];
+ double initialRadius = 0;
+ for (int row = 0; row < inputSize; row++) {
+ values[row] = random.nextGaussian();
+ initialRadius += values[row] * values[row];
+ }
+ initialRadius = Math.sqrt(initialRadius);
+
+ for (int row = 0; row < inputSize; row++) {
+ input[inputPos++] = values[row] * radius / initialRadius;
+
+ }
+ output[outputPos + radius] = 1;
+ outputPos += outputSize;
+
+ }
+ return new TrainingArrays(input, output);
+ }
+
+}
diff --git a/lib/src/main/java/de/edux/ml/nn/config/NetworkConfiguration.java b/lib/src/main/java/de/edux/ml/nn/config/NetworkConfiguration.java
deleted file mode 100644
index 4864d01..0000000
--- a/lib/src/main/java/de/edux/ml/nn/config/NetworkConfiguration.java
+++ /dev/null
@@ -1,18 +0,0 @@
-package de.edux.ml.nn.config;
-
-import de.edux.functions.activation.ActivationFunction;
-import de.edux.functions.initialization.Initialization;
-import de.edux.functions.loss.LossFunction;
-import java.util.List;
-
-public record NetworkConfiguration(
- int inputSize,
- List hiddenLayersSize,
- int outputSize,
- double learningRate,
- int epochs,
- ActivationFunction hiddenLayerActivationFunction,
- ActivationFunction outputLayerActivationFunction,
- LossFunction lossFunction,
- Initialization hiddenLayerWeightInitialization,
- Initialization outputLayerWeightInitialization) {}
diff --git a/lib/src/main/java/de/edux/ml/nn/config/package-info.java b/lib/src/main/java/de/edux/ml/nn/config/package-info.java
deleted file mode 100644
index 6bbb23d..0000000
--- a/lib/src/main/java/de/edux/ml/nn/config/package-info.java
+++ /dev/null
@@ -1,2 +0,0 @@
-/** Classes for the configuration of the neural network. */
-package de.edux.ml.nn.config;
diff --git a/lib/src/main/java/de/edux/ml/nn/network/MultilayerPerceptron.java b/lib/src/main/java/de/edux/ml/nn/network/MultilayerPerceptron.java
deleted file mode 100644
index 5fa8b2f..0000000
--- a/lib/src/main/java/de/edux/ml/nn/network/MultilayerPerceptron.java
+++ /dev/null
@@ -1,273 +0,0 @@
-package de.edux.ml.nn.network;
-
-import de.edux.api.Classifier;
-import de.edux.functions.activation.ActivationFunction;
-import de.edux.ml.nn.config.NetworkConfiguration;
-import java.util.ArrayList;
-import java.util.List;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * The {@code MultilayerPerceptron} class represents a simple feedforward neural network, which
- * consists of input, hidden, and output layers. It implements the {@code Classifier} interface,
- * facilitating both the training and prediction processes on a given dataset.
- *
- * This implementation utilizes a backpropagation algorithm for training the neural network to
- * adjust weights and biases, considering a set configuration defined by {@link
- * NetworkConfiguration}. The network's architecture is multi-layered, comprising one or more hidden
- * layers in addition to the input and output layers. Neurons within these layers utilize activation
- * functions defined per layer through the configuration.
- *
- *
The training process adjusts the weights and biases of neurons within the network based on the
- * error between predicted and expected outputs. Additionally, the implementation provides
- * functionality to save and restore the best model achieved during training based on accuracy.
- * Early stopping is applied during training to prevent overfitting and unnecessary computational
- * expense by monitoring the performance improvement across epochs.
- *
- *
Usage example:
- *
- *
- * NetworkConfiguration config = ... ;
- * double[][] testFeatures = ... ;
- * double[][] testLabels = ... ;
- *
- * MultilayerPerceptron mlp = new MultilayerPerceptron(config, testFeatures, testLabels);
- * mlp.train(features, labels);
- *
- * double accuracy = mlp.evaluate(testFeatures, testLabels);
- * double[] prediction = mlp.predict(singleInput);
- *
- *
- * Note: This implementation logs informative messages, such as accuracy per epoch, using SLF4J
- * logging.
- *
- * @see de.edux.api.Classifier
- * @see de.edux.ml.nn.network.Neuron
- * @see de.edux.ml.nn.config.NetworkConfiguration
- * @see de.edux.functions.activation.ActivationFunction
- */
-public class MultilayerPerceptron implements Classifier {
- private static final Logger LOG = LoggerFactory.getLogger(MultilayerPerceptron.class);
-
- private final NetworkConfiguration config;
- private final ActivationFunction hiddenLayerActivationFunction;
- private final ActivationFunction outputLayerActivationFunction;
- private final double[][] testFeatures;
- private final double[][] testLabels;
- private List hiddenLayers;
- private Neuron[] outputLayer;
- private double bestAccuracy;
- private ArrayList bestHiddenLayers;
- private Neuron[] bestOutputLayer;
-
- public MultilayerPerceptron(
- NetworkConfiguration config, double[][] testFeatures, double[][] testLabels) {
- this.config = config;
- this.testFeatures = testFeatures;
- this.testLabels = testLabels;
-
- hiddenLayerActivationFunction = config.hiddenLayerActivationFunction();
- outputLayerActivationFunction = config.outputLayerActivationFunction();
-
- hiddenLayers = new ArrayList<>();
-
- int inputSizeForCurrentLayer = config.inputSize();
- for (int layerSize : config.hiddenLayersSize()) {
- Neuron[] hiddenLayer = new Neuron[layerSize];
- for (int i = 0; i < layerSize; i++) {
- hiddenLayer[i] =
- new Neuron(
- inputSizeForCurrentLayer,
- hiddenLayerActivationFunction,
- this.config.hiddenLayerWeightInitialization());
- }
- hiddenLayers.add(hiddenLayer);
- inputSizeForCurrentLayer = layerSize;
- }
-
- outputLayer = new Neuron[config.outputSize()];
- for (int i = 0; i < config.outputSize(); i++) {
- outputLayer[i] =
- new Neuron(
- inputSizeForCurrentLayer,
- outputLayerActivationFunction,
- this.config.outputLayerWeightInitialization());
- }
- }
-
- private double[] feedforward(double[] input) {
-
- double[] currentInput = passInputThroughAllHiddenLayers(input);
-
- double[] output = passInputTroughOutputLayer(currentInput);
-
- return outputLayerActivationFunction.calculateActivation(output);
- }
-
- private double[] passInputThroughAllHiddenLayers(double[] input) {
- double[] currentInput = input;
- for (Neuron[] layer : hiddenLayers) {
- double[] hiddenOutputs = new double[layer.length];
- for (int i = 0; i < layer.length; i++) {
- hiddenOutputs[i] = layer[i].calculateOutput(currentInput);
- }
- currentInput = hiddenOutputs;
- }
- return currentInput;
- }
-
- private double[] passInputTroughOutputLayer(double[] currentInput) {
- double[] output = new double[config.outputSize()];
- for (int i = 0; i < config.outputSize(); i++) {
- output[i] = outputLayer[i].calculateOutput(currentInput);
- }
- return output;
- }
-
- @Override
- public boolean train(double[][] features, double[][] labels) {
- bestAccuracy = 0;
- int epochsWithoutImprovement = 0;
- final int PATIENCE = 10;
-
- for (int epoch = 0; epoch < config.epochs(); epoch++) {
- for (int i = 0; i < features.length; i++) {
- double[] output = feedforward(features[i]);
-
- double[] output_error_signal = new double[config.outputSize()];
- for (int j = 0; j < config.outputSize(); j++) {
- output_error_signal[j] = labels[i][j] - output[j];
- }
-
- List hidden_error_signals = new ArrayList<>();
- for (int j = hiddenLayers.size() - 1; j >= 0; j--) {
- double[] hidden_error_signal = new double[hiddenLayers.get(j).length];
- for (int k = 0; k < hiddenLayers.get(j).length; k++) {
- for (int l = 0; l < output_error_signal.length; l++) {
- hidden_error_signal[k] +=
- output_error_signal[l]
- * (j == hiddenLayers.size() - 1
- ? outputLayer[l].getWeight(k)
- : hiddenLayers.get(j + 1)[l].getWeight(k));
- }
- }
- hidden_error_signals.add(0, hidden_error_signal);
- output_error_signal = hidden_error_signal;
- }
-
- updateWeights(i, output_error_signal, hidden_error_signals, features);
- }
-
- double accuracy = evaluate(testFeatures, testLabels);
- LOG.info("Epoch: {} - Accuracy: {}%", epoch, String.format("%.2f", accuracy * 100));
-
- if (accuracy > bestAccuracy) {
- bestAccuracy = accuracy;
- epochsWithoutImprovement = 0;
- saveBestModel(hiddenLayers, outputLayer);
- } else {
- epochsWithoutImprovement++;
- }
-
- if (epochsWithoutImprovement >= PATIENCE) {
- LOG.info(
- "Early stopping: Stopping training as the model has not improved in the last {} epochs.",
- PATIENCE);
- loadBestModel();
- LOG.info(
- "Best accuracy after restoring best MLP model: {}%",
- String.format("%.2f", bestAccuracy * 100));
- break;
- }
- }
- return true;
- }
-
- private void loadBestModel() {
- this.hiddenLayers = this.bestHiddenLayers;
- this.outputLayer = this.bestOutputLayer;
- }
-
- private void saveBestModel(List hiddenLayers, Neuron[] outputLayer) {
- this.bestHiddenLayers = new ArrayList<>();
- this.bestOutputLayer = new Neuron[outputLayer.length];
- for (int i = 0; i < hiddenLayers.size(); i++) {
- Neuron[] layer = hiddenLayers.get(i);
- Neuron[] newLayer = new Neuron[layer.length];
- for (int j = 0; j < layer.length; j++) {
- newLayer[j] =
- new Neuron(
- layer[j].getWeights().length,
- layer[j].getActivationFunction(),
- layer[j].getInitialization());
- newLayer[j].setBias(layer[j].getBias());
- for (int k = 0; k < layer[j].getWeights().length; k++) {
- newLayer[j].getWeights()[k] = layer[j].getWeight(k);
- }
- }
- this.bestHiddenLayers.add(newLayer);
- }
- for (int i = 0; i < outputLayer.length; i++) {
- this.bestOutputLayer[i] =
- new Neuron(
- outputLayer[i].getWeights().length,
- outputLayer[i].getActivationFunction(),
- outputLayer[i].getInitialization());
- this.bestOutputLayer[i].setBias(outputLayer[i].getBias());
- for (int j = 0; j < outputLayer[i].getWeights().length; j++) {
- this.bestOutputLayer[i].getWeights()[j] = outputLayer[i].getWeight(j);
- }
- }
- }
-
- private void updateWeights(
- int i,
- double[] output_error_signal,
- List hidden_error_signals,
- double[][] features) {
- double[] currentInput = features[i];
-
- for (int j = 0; j < hiddenLayers.size(); j++) {
- Neuron[] layer = hiddenLayers.get(j);
- double[] errorSignal = hidden_error_signals.get(j);
- for (int k = 0; k < layer.length; k++) {
- layer[k].adjustBias(errorSignal[k], config.learningRate());
- layer[k].adjustWeights(currentInput, errorSignal[k], config.learningRate());
- }
- currentInput = new double[layer.length];
- for (int k = 0; k < layer.length; k++) {
- currentInput[k] = layer[k].calculateOutput(features[i]);
- }
- }
-
- for (int j = 0; j < config.outputSize(); j++) {
- outputLayer[j].adjustBias(output_error_signal[j], config.learningRate());
- outputLayer[j].adjustWeights(currentInput, output_error_signal[j], config.learningRate());
- }
- }
-
- @Override
- public double evaluate(double[][] testInputs, double[][] testTargets) {
- int correctCount = 0;
-
- for (int i = 0; i < testInputs.length; i++) {
- double[] predicted = predict(testInputs[i]);
- int predictedIndex = 0;
- int targetIndex = 0;
-
- for (int j = 0; j < predicted.length; j++) {
- if (predicted[j] > predicted[predictedIndex]) predictedIndex = j;
- if (testTargets[i][j] > testTargets[i][targetIndex]) targetIndex = j;
- }
-
- if (predictedIndex == targetIndex) correctCount++;
- }
-
- return (double) correctCount / testInputs.length;
- }
-
- public double[] predict(double[] input) {
- return feedforward(input);
- }
-}
diff --git a/lib/src/main/java/de/edux/ml/nn/network/Neuron.java b/lib/src/main/java/de/edux/ml/nn/network/Neuron.java
deleted file mode 100644
index 6fdb4bc..0000000
--- a/lib/src/main/java/de/edux/ml/nn/network/Neuron.java
+++ /dev/null
@@ -1,66 +0,0 @@
-package de.edux.ml.nn.network;
-
-import de.edux.functions.activation.ActivationFunction;
-import de.edux.functions.initialization.Initialization;
-
-class Neuron {
- private final Initialization initialization;
- private final ActivationFunction activationFunction;
- private double[] weights;
- private double bias;
-
- public Neuron(
- int inputSize, ActivationFunction activationFunction, Initialization initialization) {
- this.weights = new double[inputSize];
- this.activationFunction = activationFunction;
- this.initialization = initialization;
- this.bias = initialization.weightInitialization(inputSize, new double[1])[0];
- this.weights = initialization.weightInitialization(inputSize, weights);
- }
-
- public Initialization getInitialization() {
- return initialization;
- }
-
- public double calculateOutput(double[] input) {
- double output = bias;
- for (int i = 0; i < input.length; i++) {
- output += input[i] * weights[i];
- }
- return activationFunction.calculateActivation(output);
- }
-
- public void adjustWeights(double[] input, double error, double learningRate) {
- for (int i = 0; i < weights.length; i++) {
- weights[i] += learningRate * input[i] * error;
- }
- }
-
- public void adjustBias(double error, double learningRate) {
- bias += learningRate * error;
- }
-
- public double getWeight(int index) {
- return weights[index];
- }
-
- public double[] getWeights() {
- return weights;
- }
-
- public void setWeights(double[] weights) {
- this.weights = weights;
- }
-
- public double getBias() {
- return bias;
- }
-
- public void setBias(double bias) {
- this.bias = bias;
- }
-
- public ActivationFunction getActivationFunction() {
- return activationFunction;
- }
-}
diff --git a/lib/src/main/java/de/edux/ml/nn/network/api/Dataset.java b/lib/src/main/java/de/edux/ml/nn/network/api/Dataset.java
deleted file mode 100644
index e4a159c..0000000
--- a/lib/src/main/java/de/edux/ml/nn/network/api/Dataset.java
+++ /dev/null
@@ -1,5 +0,0 @@
-package de.edux.ml.nn.network.api;
-
-import java.util.List;
-
-public record Dataset(List trainData, List testData) {}
diff --git a/lib/src/main/java/de/edux/ml/nn/network/api/INeuron.java b/lib/src/main/java/de/edux/ml/nn/network/api/INeuron.java
deleted file mode 100644
index 88c63aa..0000000
--- a/lib/src/main/java/de/edux/ml/nn/network/api/INeuron.java
+++ /dev/null
@@ -1,9 +0,0 @@
-package de.edux.ml.nn.network.api;
-
-public interface INeuron {
- double calculateOutput(double[] inputs);
-
- double calculateError(double targetOutput);
-
- void updateWeights(double[] inputs, double error);
-}
diff --git a/lib/src/main/java/de/edux/ml/nn/network/api/IPerceptron.java b/lib/src/main/java/de/edux/ml/nn/network/api/IPerceptron.java
deleted file mode 100644
index 486fcb1..0000000
--- a/lib/src/main/java/de/edux/ml/nn/network/api/IPerceptron.java
+++ /dev/null
@@ -1,11 +0,0 @@
-package de.edux.ml.nn.network.api;
-
-public interface IPerceptron {
- void train(double[][] inputs, double[][] targetOutputs);
-
- double[] predict(double[] inputs);
-
- void backpropagate(double[] inputs, double target);
-
- double evaluate(double[][] inputs, double[][] targetOutputs);
-}
diff --git a/lib/src/test/java/de/edux/core/BackpropagationTest.java b/lib/src/test/java/de/edux/core/BackpropagationTest.java
new file mode 100644
index 0000000..e594ded
--- /dev/null
+++ b/lib/src/test/java/de/edux/core/BackpropagationTest.java
@@ -0,0 +1,60 @@
+package de.edux.core;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import de.edux.ml.mlp.core.network.loss.LossFunctions;
+import de.edux.ml.mlp.core.network.optimizer.Approximator;
+import de.edux.ml.mlp.core.tensor.Matrix;
+import java.util.Random;
+import org.junit.jupiter.api.Test;
+
+public class BackpropagationTest {
+ private final Random random = new Random();
+
+ @Test
+ void shouldBackpropagate() {
+
+ interface NeuralNetwork {
+ Matrix apply(Matrix m);
+ }
+ final int inputRows = 4;
+ final int cols = 5;
+ final int outputRows = 4;
+
+ Matrix input =new Matrix(inputRows, cols, i->random.nextGaussian());
+ Matrix expected = new Matrix(outputRows, cols, i->0);
+
+ Matrix weights = new Matrix(outputRows, inputRows, i->random.nextGaussian());
+ Matrix biases = new Matrix(outputRows, 1, i->random.nextGaussian());
+
+ for(int col =0; col {
+ Matrix out = m.relu(); //input
+ out = weights.multiply(out).add(biases); // Dense
+ out = out.softmax(); // Softmax
+ return out;
+ };
+
+ Matrix softmaxOutput = neuralNet.apply(input);
+
+ Matrix approximatedResult = Approximator.gradient(input, in-> {
+ Matrix out = neuralNet.apply(in);
+ return LossFunctions.crossEntropy(expected, out);
+ });
+
+ Matrix calculatedResult = softmaxOutput.subtract(expected); //Softmax backward
+ calculatedResult = weights.transpose().multiply(calculatedResult);
+ calculatedResult = calculatedResult.reluDerivative(input);
+
+ System.out.println("Approximated Result");
+ System.out.println(approximatedResult);
+ System.out.println("Backpropagated Result");
+ System.out.println(calculatedResult);
+
+ assertEquals(approximatedResult, calculatedResult);
+ }
+}
diff --git a/lib/src/test/java/de/edux/core/CategoricalCrossEntropyLossTest.java b/lib/src/test/java/de/edux/core/CategoricalCrossEntropyLossTest.java
new file mode 100644
index 0000000..ffcf94e
--- /dev/null
+++ b/lib/src/test/java/de/edux/core/CategoricalCrossEntropyLossTest.java
@@ -0,0 +1,32 @@
+package de.edux.core;
+
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import de.edux.ml.mlp.core.network.loss.LossFunctions;
+import de.edux.ml.mlp.core.tensor.Matrix;
+import org.junit.jupiter.api.Test;
+
+public class CategoricalCrossEntropyLossTest {
+
+ private static final double DELTA = 0.05;
+
+ @Test
+ public void shouldCalculateCategoricalCrossEntropyLoss() {
+ double[] expectedValues = {1, 0, 0, 0, 0, 1, 0, 1, 0};
+
+ Matrix expected = new Matrix(3, 3, i -> expectedValues[i]);
+ Matrix actual = new Matrix(3, 3, i -> DELTA * i * i).softmax();
+ Matrix result = LossFunctions.crossEntropy(expected, actual);
+
+ actual.forEach((row, col, index, value) -> {
+ double expectedValue = expected.getData()[index];
+ double loss = result.getData()[col];
+
+ if (expectedValue > 0.9) {
+ assertTrue(Math.abs(-Math.log(value) - loss) < 0.001, String.format("expected: %f, actual: %f", -Math.log(value), loss));
+ }
+ });
+
+
+ }
+}
diff --git a/lib/src/test/java/de/edux/core/MatrixTest.java b/lib/src/test/java/de/edux/core/MatrixTest.java
new file mode 100644
index 0000000..e483b54
--- /dev/null
+++ b/lib/src/test/java/de/edux/core/MatrixTest.java
@@ -0,0 +1,283 @@
+package de.edux.core;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+import de.edux.ml.mlp.core.tensor.Matrix;
+import java.util.Random;
+import org.junit.jupiter.api.Test;
+
+class MatrixTest {
+ private static final double TOLERANCE = 1e-6;
+ private final Random random = new Random();
+
+ @Test
+ public void testMultiply() {
+ // Creating a matrix with elements initialized to their index - 1
+ Matrix a = new Matrix(2, 3, (index) -> index - 1);
+ double x = 0.5;
+
+ // Applying the multiplication
+ Matrix result = a.apply((index, value) -> value * x);
+
+ // Testing the result
+ for (int i = 0; i < 2 * 3; i++) {
+ assertEquals((i - 1) * x, result.getData()[i]);
+ }
+ }
+
+ @Test
+ public void testEquals() {
+ Matrix a = new Matrix(2, 3, (index) -> index - 6);
+ Matrix b = new Matrix(2, 3, (index) -> index - 6);
+ Matrix c = new Matrix(2, 3, (index) -> index - 6.2);
+
+ assertEquals(a, b);
+ assertNotEquals(a, c);
+ }
+
+ @Test
+ public void testAddMatrices() {
+ Matrix a = new Matrix(2, 3, (index) -> index);
+ Matrix b = new Matrix(2, 3, (index) -> index * 2);
+ Matrix expected = a.apply((index, value) -> value + b.getData()[index]);
+
+ Matrix result = a.add(b);
+
+ assertEquals(expected, result);
+ }
+
+ @Test
+ public void shouldNotMultiplyMatricesWithWrongDimensions() {
+ Matrix a = new Matrix(2, 2);
+ Matrix b = new Matrix(3, 2);
+ assertThrows(IllegalArgumentException.class, () -> a.multiply(b));
+ }
+
+ @Test
+ public void shouldMultiplyMatrices() {
+ Matrix a = new Matrix(2, 2, (index) -> index);
+ Matrix b = new Matrix(2, 2, (index) -> index * 2);
+
+ /*+-------+-------+
+ | 0,000 | 1,000 |
+ +-------+-------+
+ | 2,000 | 3,000 |
+ +-------+-------+
+ multiply
+ +-------+-------+
+ | 0,000 | 2,000 |
+ +-------+-------+
+ | 4,000 | 6,000 |
+ +-------+-------+*/
+
+ Matrix expected = new Matrix(2, 2);
+ expected.getData()[0] = 4;
+ expected.getData()[1] = 6;
+ expected.getData()[2] = 12;
+ expected.getData()[3] = 22;
+
+ Matrix result = a.multiply(b);
+ assertEquals(expected, result);
+ }
+
+ @Test
+ public void shouldMultiplyWithDifferentColsAndRows() {
+ Matrix a = new Matrix(2, 3, (index) -> index);
+ Matrix b = new Matrix(3, 2, (index) -> index);
+
+ /*+-------+-------+-------+
+ | 0,000 | 1,000 | 2,000 |
+ +-------+-------+-------+
+ | 3,000 | 4,000 | 5,000 |
+ +-------+-------+-------+
+ multiply
+ +-------+--------+
+ | 0,000 | 2,000 |
+ +-------+--------+
+ | 4,000 | 6,000 |
+ +-------+--------+
+ | 8,000 | 10,000 |
+ +-------+--------+*/
+
+ Matrix expected = new Matrix(2, 2);
+ expected.getData()[0] = 10;
+ expected.getData()[1] = 13;
+ expected.getData()[2] = 28;
+ expected.getData()[3] = 40;
+
+ Matrix result = a.multiply(b);
+ /*
+ *+--------+--------+
+ | 10,000 | 13,000 |
+ +--------+--------+
+ | 28,000 | 40,000 |
+ +--------+--------+
+ */
+ System.out.println(result);
+ assertEquals(expected, result);
+ }
+
+ @Test
+ public void shouldRunWithoutOutOfMemory() {
+ try {
+ var matrixSize = 1000;
+ Matrix a = new Matrix(matrixSize, matrixSize, (index) -> index);
+ Matrix b = new Matrix(matrixSize, matrixSize, (index) -> index);
+
+ long startTime = System.nanoTime();
+ a.multiply(b);
+ long endTime = System.nanoTime();
+
+ System.out.println("Time: " + (endTime - startTime) / 1e9 + "s");
+ } catch (OutOfMemoryError e) {
+ fail("Test failed due to insufficient memory: " + e.getMessage());
+ }
+ }
+
+ @Test
+ public void shouldRunWithoutOutOfMemoryOnParallelMultiplication() {
+ try {
+ Matrix a = new Matrix(1500, 1500, (index) -> index);
+ Matrix b = new Matrix(1500, 1500, (index) -> index);
+
+ long startTime = System.nanoTime();
+ a.multiplyParallel(b);
+ long endTime = System.nanoTime();
+
+ System.out.println("Time: " + (endTime - startTime) / 1e9 + "s");
+ } catch (OutOfMemoryError e) {
+ fail("Test failed due to insufficient memory: " + e.getMessage());
+ }
+ }
+
+ @Test
+ void shouldMultiplyAndAdd() {
+ Matrix inout = new Matrix(3, 3, (index) -> index + 1);
+ Matrix weights = new Matrix(3, 3, (index) -> index + 1);
+ Matrix bias = new Matrix(3, 1, (index) -> index + 1);
+
+ Matrix result = inout.multiply(weights).add(bias);
+
+ Matrix expected = new Matrix(3, 3);
+
+ expected.getData()[0] = 31;
+ expected.getData()[1] = 37;
+ expected.getData()[2] = 43;
+ expected.getData()[3] = 68;
+ expected.getData()[4] = 83;
+ expected.getData()[5] = 98;
+ expected.getData()[6] = 105;
+ expected.getData()[7] = 129;
+ expected.getData()[8] = 153;
+
+ assert result.equals(expected);
+ }
+
+ @Test
+ void shouldSumColumns() {
+ Matrix a = new Matrix(4, 5, (index) -> index);
+ Matrix result = a.sumColumns();
+ Matrix expected = new Matrix(1, 5);
+ expected.getData()[0] = 30;
+ expected.getData()[1] = 34;
+ expected.getData()[2] = 38;
+ expected.getData()[3] = 42;
+ expected.getData()[4] = 46;
+ assert result.equals(expected);
+ }
+
+ @Test
+ void shouldAddIncrement() {
+ Matrix a = new Matrix(5, 8, (index) -> random.nextGaussian());
+ int row = 3;
+ int col = 2;
+ double increment = 10;
+
+ Matrix result = a.addIncrement(row, col, increment);
+
+ double incrementedValue = result.get(row, col);
+ double originalValue = a.get(row, col);
+
+ assertEquals(a.get(row, col) + increment, result.get(row, col));
+ assertTrue(Math.abs(incrementedValue - (originalValue + increment)) < TOLERANCE);
+ }
+
+ @Test
+ void shouldTranspose() {
+ Matrix m = new Matrix(2, 3, i -> i);
+ System.out.println(m);
+ Matrix result = m.transpose();
+
+ System.out.println(result);
+ double[] expectedData = {0, 3, 1, 4, 2, 5};
+ Matrix expected = new Matrix(3, 2, i -> expectedData[i]);
+
+ assertEquals(expected, result);
+ }
+
+ @Test
+ public void shouldCalculateAverageColumn() {
+ int rows = 3;
+ int cols = 4;
+ Matrix m = new Matrix(rows, cols, i -> 2 * i - 3);
+ double averageIndex = (cols - 1) / 2.0;
+
+ Matrix expected = new Matrix(rows, 1);
+ expected.modify((row, col, value) -> 2 * (row * cols + averageIndex) - 3);
+
+ Matrix result = m.averageColumn();
+ assertEquals(expected, result);
+
+ }
+
+ @Test
+ void shouldFindGreatestRowNumber() {
+ double[] values = {7, -6, -6, 7, 2, 10, 3, -1, 1};
+ Matrix m = new Matrix(3,3, i->values[i]);
+
+ Matrix result = m.getGreatestRowNumber();
+
+ double[] expectedValues = {0, 1, 1};
+ Matrix expected = new Matrix(3,1, i->expectedValues[i]);
+
+ assertEquals(expected, result);
+
+ System.out.println(m);
+ System.out.println(result);
+ }
+
+ @Test
+ void testDivideNormalCase() {
+ Matrix matrix = new Matrix(4, 4, i -> (double)i);
+ int batches = 2;
+ Matrix result = matrix.divide(batches);
+
+ assertNotNull(result, "The resulting matrix should not be null");
+ assertEquals(matrix.getRows(), result.getRows(), "The number of rows should remain the same");
+ assertEquals(matrix.getCols(), result.getCols(), "The number of columns should remain the same");
+
+ for (int row = 0; row < matrix.getRows(); row++) {
+ for (int col = 0; col < matrix.getCols(); col++) {
+ double expectedValue = matrix.get(row, col) / batches;
+ assertEquals(expectedValue, result.get(row, col), "The value at position [" + row + "][" + col + "] is not correct");
+ }
+ }
+ }
+
+
+ @Test
+ void testDivideWithInvalidBatches() {
+ Matrix matrix = new Matrix(4, 4);
+ int invalidBatches = 0;
+ assertThrows(IllegalArgumentException.class, () -> matrix.divide(invalidBatches));
+ }
+
+ @Test
+ void testDivideWithBoundaryBatches() {
+ Matrix matrix = new Matrix(4, 4);
+ int batches = 1; // Oder ein anderer Grenzwert
+ Matrix result = matrix.divide(batches);
+ assertNotNull(result);
+
+ }
+}
diff --git a/lib/src/test/java/de/edux/core/NeuralNetTest.java b/lib/src/test/java/de/edux/core/NeuralNetTest.java
new file mode 100644
index 0000000..0626e25
--- /dev/null
+++ b/lib/src/test/java/de/edux/core/NeuralNetTest.java
@@ -0,0 +1,86 @@
+package de.edux.core;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import de.edux.ml.mlp.core.network.Engine;
+import de.edux.ml.mlp.core.network.layers.DenseLayer;
+import de.edux.ml.mlp.core.network.layers.ReLuLayer;
+import de.edux.ml.mlp.core.network.layers.SoftmaxLayer;
+import de.edux.ml.mlp.core.network.loss.LossFunctions;
+import de.edux.ml.mlp.core.network.optimizer.Approximator;
+import de.edux.ml.mlp.core.tensor.Matrix;
+import de.edux.ml.mlp.util.Util;
+import java.util.Random;
+import org.junit.jupiter.api.Test;
+
+public class NeuralNetTest {
+
+ private final Random random = new Random();
+
+
+ @Test
+ void testWeightGradient() {
+ int inputRows = 4;
+ int outputRows = 5;
+ Matrix weights = new Matrix(outputRows, inputRows, i -> random.nextGaussian());
+ Matrix input = Util.generateInputMatrix(inputRows, 1);
+ Matrix expected = Util.generateExpectedMatrix(outputRows, 1);
+ Matrix output = weights.multiply(input).softmax();
+
+ Matrix loss = LossFunctions.crossEntropy(expected, output);
+
+ Matrix calculatedGradient = output.apply((index, value) -> value - expected.get(index));
+
+ Matrix calculatedWeightGradients = calculatedGradient.multiply(input.transpose());
+
+ Matrix approximatedWeightGradients = Approximator.weightGradient(weights, in -> {
+ Matrix out = in.multiply(input).softmax();
+ return LossFunctions.crossEntropy(expected, out);
+ });
+
+ calculatedWeightGradients.setTolerance(0.01);
+ assertEquals(approximatedWeightGradients, calculatedWeightGradients);
+ }
+
+ @Test
+ void testEngineLayerbased() {
+ int rows = 5;
+ int cols = 6;
+ int outputRows = 4;
+
+ Engine engine = new Engine(10);
+ engine.addLayer(new DenseLayer(5, 8));
+ engine.addLayer(new ReLuLayer());
+ engine.addLayer(new DenseLayer(8, 5));
+ engine.addLayer(new ReLuLayer());
+ engine.addLayer(new DenseLayer(5, 4));
+ engine.addLayer(new SoftmaxLayer());
+
+ Matrix input = Util.generateInputMatrix(rows, cols);
+ Matrix expected = Util.generateTrainableExpectedMatrix(outputRows, input);
+
+ //Forward pass
+ Matrix softmaxOutput = engine.forwardLayerbased(input);
+
+ //Loss function
+ Matrix approximatedError = Approximator.gradient(input, in -> {
+ Matrix forwardResult = engine.forwardLayerbased(in);
+ return LossFunctions.crossEntropy(expected, forwardResult);
+ });
+
+
+ //Backward pass
+ Matrix calculatedError = engine.backwardLayerBased(expected, 0.01f);
+ System.out.println("Approximated Error");
+ System.out.println(approximatedError);
+ System.out.println("Calculated Error");
+ System.out.println(calculatedError);
+
+ calculatedError.setTolerance(0.0001);
+
+ assertEquals(approximatedError, calculatedError);
+
+ }
+
+
+}
diff --git a/lib/src/test/java/de/edux/core/ReLuTest.java b/lib/src/test/java/de/edux/core/ReLuTest.java
new file mode 100644
index 0000000..d1569a9
--- /dev/null
+++ b/lib/src/test/java/de/edux/core/ReLuTest.java
@@ -0,0 +1,35 @@
+package de.edux.core;
+
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import de.edux.ml.mlp.core.tensor.Matrix;
+import java.util.Random;
+import org.junit.jupiter.api.Test;
+
+public class ReLuTest {
+
+ private Random random = new Random();
+
+ @Test
+ void shouldReLu() {
+ final int numberOfNeurons = 50;
+ final int numberOfInputs = 60;
+ final int inputSize = 50;
+ Matrix inout = new Matrix(inputSize, numberOfInputs, (index) -> random.nextDouble());
+ Matrix weights = new Matrix(numberOfNeurons, inputSize, (index) -> random.nextGaussian());
+ Matrix bias = new Matrix(numberOfNeurons, 1, (index) -> random.nextGaussian());
+
+ Matrix result1 = weights.multiply(inout).add(bias);
+ Matrix result2 = weights.multiply(inout).add(bias).relu();
+
+ result2.forEach(
+ (index, value) -> {
+ double originalValue = result1.getData()[index];
+ if (originalValue <= 0) {
+ assertTrue(Math.abs(originalValue - value) < 1e-6 || Math.abs(value) < 1e-6);
+ } else {
+ assertTrue(Math.abs(value) < 1e-6 || Math.abs(originalValue - value) < 1e-6);
+ }
+ });
+ }
+}
diff --git a/lib/src/test/java/de/edux/core/SoftmaxTest.java b/lib/src/test/java/de/edux/core/SoftmaxTest.java
new file mode 100644
index 0000000..3b6dbb5
--- /dev/null
+++ b/lib/src/test/java/de/edux/core/SoftmaxTest.java
@@ -0,0 +1,32 @@
+package de.edux.core;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import de.edux.ml.mlp.core.tensor.Matrix;
+import java.util.Random;
+import org.junit.jupiter.api.Test;
+
+public class SoftmaxTest {
+ final Random random = new Random();
+
+ @Test
+ void shouldSoftmax() {
+
+ Matrix a = new Matrix(5, 8, (index) -> random.nextGaussian());
+ Matrix result = a.softmax();
+ double[] colSums = new double[8];
+ result.forEach((row, col, value) -> {
+ assertTrue(value >= 0 && value <= 1);
+ colSums[col] += value;
+ });
+
+ for (int i = 0; i < colSums.length; i++) {
+ assertEquals(1, colSums[i], 1e-6);
+ }
+
+
+
+
+ }
+}
diff --git a/lib/src/test/java/de/edux/core/network/loader/LoaderTest.java b/lib/src/test/java/de/edux/core/network/loader/LoaderTest.java
new file mode 100644
index 0000000..785eec7
--- /dev/null
+++ b/lib/src/test/java/de/edux/core/network/loader/LoaderTest.java
@@ -0,0 +1,51 @@
+package de.edux.core.network.loader;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+import de.edux.ml.mlp.core.network.loader.BatchData;
+import de.edux.ml.mlp.core.network.loader.Loader;
+import de.edux.ml.mlp.core.network.loader.MetaData;
+import de.edux.ml.mlp.core.network.loader.test.TestLoader;
+import de.edux.ml.mlp.core.tensor.Matrix;
+import org.junit.jupiter.api.Test;
+
+class LoaderTest {
+
+ @Test
+ void shouldOpen() {
+ var batchSize = 33;
+ Loader loader = new TestLoader(600, batchSize, 33);
+ MetaData metaData = loader.open();
+
+ int numberItems = metaData.getNumberItems();
+ int lastBatchSize = numberItems % batchSize;
+ int numberBatches = metaData.getNumberBatches();
+
+ for (int i = 0; i random.nextGaussian()).softmax();
+
+ Matrix expected = new Matrix(rows, cols, i -> 0);
+ for (int col = 0; col < cols; col++) {
+ int randowmRow = random.nextInt(rows);
+ expected.set(randowmRow, col, 1);
+ }
+
+
+ Matrix result = Approximator.gradient(input, in -> LossFunctions.crossEntropy(expected, in));
+
+ input.forEach((index, value) -> {
+ double resultValue = result.getData()[index];
+ double expectedValue = expected.getData()[index];
+
+ if (expectedValue < 0.001) {
+ assertTrue(Math.abs(resultValue) < 0.01);
+ } else {
+ assertTrue(Math.abs(resultValue + 1.0 / value) < 0.01);
+ }
+ });
+ }
+
+ @Test
+ void shouldSoftmaxCrossEntropyGradient() {
+ final int rows = 4;
+ final int cols = 5;
+
+ Matrix input = new Matrix(rows, cols, i -> random.nextGaussian());
+ Matrix expected = new Matrix(rows, cols, i -> 0);
+ for (int col = 0; col < cols; col++) {
+ int randowmRow = random.nextInt(rows);
+ expected.set(randowmRow, col, 1);
+ }
+
+ Matrix softmaxOutput = input.softmax();
+ Matrix result = Approximator.gradient(input, in -> LossFunctions.crossEntropy(expected, in.softmax()));
+
+ result.forEach((index, value) -> {
+ double softmaxValue = softmaxOutput.getData()[index];
+ double expectedValue = expected.getData()[index];
+ assertTrue(Math.abs(value - (softmaxValue - expectedValue)) < 0.01);
+ });
+
+ }
+}
\ No newline at end of file
diff --git a/lib/src/test/java/de/edux/ml/nn/network/NeuronTest.java b/lib/src/test/java/de/edux/ml/nn/network/NeuronTest.java
deleted file mode 100644
index 8ddbad7..0000000
--- a/lib/src/test/java/de/edux/ml/nn/network/NeuronTest.java
+++ /dev/null
@@ -1,50 +0,0 @@
-package de.edux.ml.nn.network;
-
-import static org.junit.jupiter.api.Assertions.assertEquals;
-
-import de.edux.functions.activation.ActivationFunction;
-import de.edux.functions.initialization.Initialization;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
-
-public class NeuronTest {
-
- private final int inputSize = 3;
- private final ActivationFunction dummyActivationFunction = ActivationFunction.SOFTMAX;
- private Neuron neuron;
-
- @BeforeEach
- public void setUp() {
- neuron = new Neuron(inputSize, dummyActivationFunction, Initialization.XAVIER);
- }
-
- @Test
- public void testAdjustWeights() {
- double[] initialWeights = new double[inputSize];
- for (int i = 0; i < inputSize; i++) {
- initialWeights[i] = neuron.getWeight(i);
- }
-
- double[] input = {1.0, 2.0, 3.0};
- double error = 0.5;
- double learningRate = 0.1;
- neuron.adjustWeights(input, error, learningRate);
-
- for (int i = 0; i < inputSize; i++) {
- double expectedWeight = initialWeights[i] + learningRate * input[i] * error;
- assertEquals(expectedWeight, neuron.getWeight(i));
- }
- }
-
- @Test
- public void testAdjustBias() {
- double initialBias = neuron.getBias();
-
- double error = 0.5;
- double learningRate = 0.1;
- neuron.adjustBias(error, learningRate);
-
- double expectedBias = initialBias + learningRate * error;
- assertEquals(expectedBias, neuron.getBias());
- }
-}