diff --git a/src/main b/src/main index 45b2c56..aa7ca69 100755 Binary files a/src/main and b/src/main differ diff --git a/src/main.cpp b/src/main.cpp index 2f4187f..3fed565 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -38,7 +38,7 @@ public: : Layer(layer), inputFeatures(inputFeatures), weights(layer.getOuputFeatures(), inputFeatures) {} ConnectedLayer(const Layer &a, const Layer &b) - : ConnectedLayer(b.getOuputFeatures(), a) {} + : ConnectedLayer(a.getOuputFeatures(), b) {} int getInputFeatures() const { return inputFeatures; } const Matrix &getWeights() const { return weights; } @@ -47,7 +47,6 @@ public: class LearnLayer : public ConnectedLayer { protected: - // Matrix gradients; Matrix internal; Matrix outputs; @@ -57,7 +56,7 @@ public: internal(layer.getOuputFeatures(), inputFeatures, false), outputs(layer.getOuputFeatures(), inputFeatures, false) {} LearnLayer(const Layer &a, const Layer &b) - : LearnLayer(b.getOuputFeatures(), a) {} + : LearnLayer(a.getOuputFeatures(), b) {} const Matrix &getInternal() const { return internal; } const Matrix &getOutputs() const { return outputs; } @@ -101,16 +100,16 @@ public: // employ back layers.push_back(LearnLayer(inputFeatures, l[0])); for (size_t i = 1; i < l.size(); i++) - layers.push_back(LearnLayer(l[i - 1].getOuputFeatures(), l[i])); + layers.push_back(LearnLayer(l[i - 1], l[i])); } Matrix learn(Matrix inputs, Matrix target, float speed = 1.0f) { MatrixMath mm; VectorMath vm; for (size_t i = 0; i < layers.size(); i++) { - layers[i].setInternal(mm.dot(i == 0 ? inputs : layers[i - 1].getOutputs(), - layers[i].getWeights(), false, true, - &layers[i].getBias())); + layers[i].setInternal(mm.dot(layers[i].getWeights(), + i == 0 ? inputs : layers[i - 1].getOutputs(), + false, false, &layers[i].getBias())); layers[i].setOutputs(mm.activate(layers[i].getInternal(), layers[i].getActivation(), layers[i].getAlpha())); @@ -120,17 +119,26 @@ public: std::vector io = inputs.toVector(); std::cout << "I: "; for (size_t i = 0; i < io.size(); ++i) - printf("%5.3f ", io[i]); + printf("%4.2f ", io[i]); + + std::vector ni = layers[layers.size() - 1].getInternal().toVector(); + std::cout << "| NNI: "; + for (size_t i = 0; i < ni.size(); ++i) + printf("%4.2f ", ni[i]); + std::vector no = layers[layers.size() - 1].getOutputs().toVector(); - std::cout << "| NN: "; + std::cout << "| NNO: "; for (size_t i = 0; i < no.size(); ++i) - printf("%5.3f ", no[i]); + printf("%4.2f ", no[i]); + std::vector to = target.toVector(); std::cout << "| T: "; for (size_t i = 0; i < to.size(); ++i) - printf("%5.3f ", to[i]); + printf("%4.2f ", to[i]); + Matrix mse = mm.loss(layers[layers.size() - 1].getOutputs(), target, Loss::MSE); + std::vector lo = mse.toVector(); std::cout << "| L: "; for (size_t i = 0; i < lo.size(); ++i) @@ -139,25 +147,48 @@ public: Matrix dAnl = mm.d_loss(layers[layers.size() - 1].getOutputs(), target, Loss::MSE); + for (int i = layers.size() - 1; i >= 0; --i) { + printf("=== Layer %d ===\n", i + 1); + printf("dAnl: "); + dAnl.print(); + Matrix dZl = mm.mult(dAnl, mm.d_activate(layers[i].getInternal())); - Matrix dWl = mm.mult( - mm.dot(dZl, i == 0 ? inputs : layers[i - 1].getOutputs(), true), - 1.0f / (float)inputs.getRows()); + printf("dZl: "); + dZl.print(); + + Matrix dWl = + mm.mult(mm.dot(dZl, i == 0 ? inputs : layers[i - 1].getOutputs(), + false, true), + 1.0f / (float)inputs.getRows()); + printf("dWl: "); + dWl.print(); + Vector dbl = mm.axis_sum(mm.mult(dZl, 1.0f / (float)inputs.getRows())); - dAnl = mm.dot(dZl, layers[i].getWeights(), false, false); // false true?! + printf("dbl: "); + dbl.print(); + + dAnl = mm.dot(layers[i].getWeights(), dZl, true); // false true?! mm.await(); layers[i].setWeights(mm.add(layers[i].getWeights(), dWl, -speed)); + printf("Weights %d: ", i + 1); + layers[i].getWeights().print(); + layers[i].setBias( vm.add(layers[i].getBias(), dbl, -speed / (float)inputs.getRows())); + printf("Bias %d: ", i + 1); + layers[i].getBias().print(); } return mse; } const LearnLayer &getLayer(int i) const { return layers[i]; } + + // delete + LearnLayer &getLayer(int i) { return layers[i]; } }; #ifndef NOGPU @@ -165,41 +196,80 @@ OpenCL openCL; #endif int main() { + // LearnNerualNetrowk nn( + // 3, {Layer(3, Activation::SIGMOID), Layer(3, Activation::SIGMOID)}); + // + // Matrix weights1(3, 3, + // {0.88f, 0.39f, 0.9f, 0.37f, 0.14f, 0.41f, 0.96f, 0.5f, + // 0.6f}); + // Matrix weights2( + // 3, 3, {0.29f, 0.57f, 0.36f, 0.73f, 0.53f, 0.68f, 0.01f, 0.02f, 0.58f}); + // + // Vector bias1(std::vector{0.23f, 0.89f, 0.08f}); + // Vector bias2(std::vector{0.78f, 0.83f, 0.8f}); + // + // nn.getLayer(0).setWeights(weights1); + // nn.getLayer(0).setBias(bias1); + // + // nn.getLayer(1).setWeights(weights2); + // nn.getLayer(1).setBias(bias2); + // + // std::cout << std::endl; + // + // Matrix input(3, 1, {0.03f, 0.72f, 0.49f}); + // Matrix target(3, 1, {0.93f, 0.74f, 0.17f}); + // + // // for (int i = 0; i < 1000; i++) + // nn.learn(input, target, 0.01f); + LearnNerualNetrowk nn( - 2, {Layer(2, Activation::TANH), Layer(1, Activation::SIGMOID)}); - std::cout << std::endl; + 2, {Layer(3, Activation::SIGMOID), Layer(1, Activation::SIGMOID)}); - // Matrix input(4, 2); - // Matrix target(4, 1); - // - // for (int batch = 0; batch < 4; batch++) { - // for (int i = 0; i < 4; i++) { - // int v1 = (i / 2) % 2; - // int v2 = i % 2; - // - // input(i, 0) = static_cast(v1); - // input(i, 1) = static_cast(v2); - // target(i, 0) = static_cast(v1 ^ v2); - // } - // } - // - // for (int i = 0; i < 10; i++) { - // printf("%4d | ", i + 1); - // Matrix mse = nn.learn(input, target, 0.1f * std::pow(0.99, i)); - // } + Matrix input(2, 4); + Matrix target(1, 4); - for (int i = 0; i < 4 * 1000; i++) { - int v1 = (i / 2) % 2; - int v2 = i % 2; + float min = 100.0f; + for (int batch = 0; batch < 4; batch++) { + for (int i = 0; i < 4; i++) { + int v1 = (i / 2) % 2; + int v2 = i % 2; - Matrix input(1, 2, {static_cast(v1), static_cast(v2)}); - Matrix target(1, 1, static_cast(v1 ^ v2)); - - printf("%5d | ", i + 1); - Matrix mse = nn.learn(input, target, 0.00003f); - if (i % 4 == 3) - std::cout << std::endl; + input(0, i) = static_cast(v1); + input(1, i) = static_cast(v2); + target(0, i) = static_cast(v1 ^ v2); + } } + for (int i = 0; i < 1000; i++) { + printf("%4d | ", i + 1); + Matrix mse = nn.learn(input, target, 0.0001f * std::pow(0.99f, i)); + std::vector lv = mse.toVector(); + float loss = 0.0f; + for (size_t i = 0; i < lv.size(); ++i) + loss += lv[i]; + if (loss < min) + min = loss; + } + std::cout << min << std::endl; + + // LearnNerualNetrowk nn( + // 2, {Layer(3, Activation::SIGMOID), Layer(1, Activation::SIGMOID)}); + // float min = 100.0f; + // for (int i = 0; i < 4 * 10000; i++) { + // int v1 = (i / 2) % 2; + // int v2 = i % 2; + // + // Matrix input(2, 1, {static_cast(v1), static_cast(v2)}); + // Matrix target(1, 1, static_cast(v1 ^ v2)); + // + // printf("%5d | ", i + 1); + // Matrix mse = nn.learn(input, target, 0.0001f * std::pow(0.95f, i)); + // if (i % 4 == 3) + // std::cout << std::endl; + // if (mse[0] < min) + // min = mse[0]; + // } + // std::cout << min << std::endl; + return 0; } diff --git a/src/math/tensor/cpu/math.hpp b/src/math/tensor/cpu/math.hpp index cb34ed6..ad4f586 100644 --- a/src/math/tensor/cpu/math.hpp +++ b/src/math/tensor/cpu/math.hpp @@ -116,13 +116,13 @@ private: Tensor2 mse(const Tensor2 &a, const Tensor2 &b) { Tensor2 result(a.getShape(), false); for (size_t i = 0; i < result.getSize(); ++i) - result[i] += (a[i] - b[i]) * (a[i] - b[i]) / (float)a.getCols(); + result[i] = (a[i] - b[i]) * (a[i] - b[i]) / (float)a.getCols(); return result; } - Tensor2 dmse(const Tensor2 &a, const Tensor2 &b) { + Tensor2 d_mse(const Tensor2 &a, const Tensor2 &b) { Tensor2 result(a.getShape(), false); for (size_t i = 0; i < result.getSize(); ++i) - result[i] += 2 * (a[i] - b[i]) / (float)a.getCols(); + result[i] = 2 * (a[i] - b[i]) / (float)a.getCols(); return result; } @@ -133,7 +133,7 @@ public: float alpha = 0.01f) override { validateMultDimensions(a, b, transpose_a, transpose_b); if (bias != nullptr) - validateBiasDimensions(b, *bias, transpose_b); + validateBiasDimensions(a, *bias, transpose_a); Tensor2 result(transpose_a ? a.getCols() : a.getRows(), transpose_b ? b.getRows() : b.getCols(), 0.0f); for (int i = 0; i < result.getRows(); ++i) { @@ -143,7 +143,7 @@ public: sum += (transpose_a ? a(k, i) : a(i, k)) * (transpose_b ? b(j, k) : b(k, j)); result(i, j) = - activateX(sum + (bias == nullptr ? 0.0f : (*bias)(j)), type, alpha); + activateX(sum + (bias == nullptr ? 0.0f : (*bias)(i)), type, alpha); } } return result; @@ -162,18 +162,18 @@ public: this->validateSameDimensions(a, b); switch (type) { case Loss::MSE: - return dmse(a, b); + return d_mse(a, b); default: throw std::invalid_argument("Unknown loss type"); } } Tensor1 axis_sum(const Tensor2 &m) override { - Tensor1 result(m.getCols(), 0.0f); - for (int i = 0; i < m.getCols(); ++i) { + Tensor1 result(m.getRows(), 0.0f); + for (int i = 0; i < m.getRows(); ++i) { float sum = 0.0f; - for (int j = 0; j < m.getRows(); ++j) - sum += m(j, i); + for (int j = 0; j < m.getCols(); ++j) + sum += m(i, j); result(i) = sum; } return result; diff --git a/src/math/tensor/math.hpp b/src/math/tensor/math.hpp index b0c435f..1a20bca 100644 --- a/src/math/tensor/math.hpp +++ b/src/math/tensor/math.hpp @@ -64,9 +64,9 @@ public: throw std::invalid_argument( "Invalid matrix dimensions for multiplication"); }; - void validateBiasDimensions(const M &a, const V &b, bool transpose) const { - if ((!transpose && a.getCols() != b.getSize()) || - (transpose && a.getRows() != b.getSize())) + void validateBiasDimensions(const M &m, const V &v, bool transpose) const { + if ((transpose && (size_t)m.getCols() != v.getSize()) || + (!transpose && (size_t)m.getRows() != v.getSize())) throw std::invalid_argument("Invalid matrix bias"); }; };