simple-deep-learning-cpp/Initializers.cpp at main · tte0/simple-deep-learning-cpp · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
#include <vector>
#include <random>
#include "Initializers.h"

namespace initializers {
    // RandomInitializer
    RandomInitializer::RandomInitializer(const double& parameter_initialization_scale=0.01):
        param_init_scale(parameter_initialization_scale),
        rng(rd()),
        distribution(0, parameter_initialization_scale){}

    std::vector<std::vector<double>> RandomInitializer::initialize_weights(const size_t& num_neurons,const size_t& num_neurons_previous){
        std::vector<std::vector<double>> weights(num_neurons, std::vector<double>(num_neurons_previous));
        for(int i = 0; i < num_neurons; i++){
            for(int j = 0; j < num_neurons_previous; j++){
                weights[i][j] = distribution(rng);
            }
        }
        return weights;
    }

    std::vector<double> RandomInitializer::initialize_biases(const int& num_neurons) {
        return std::vector<double>(num_neurons, 0);
    }


    // HeInitializer
    HeInitializer::HeInitializer():rng(rd()){}

    std::vector<std::vector<double>> HeInitializer::initialize_weights(const size_t& num_neurons,const size_t& num_neurons_previous){
        double std = sqrt(2.0 / double(num_neurons_previous));
        std::vector<std::vector<double>> weights(num_neurons, std::vector<double>(num_neurons_previous));
        for(int i = 0; i < num_neurons; i++){
            for(int j = 0; j < num_neurons_previous; j++){
                weights[i][j] = distribution(rng);
            }
        }
        return weights;
    }

    std::vector<double> HeInitializer::initialize_biases(const size_t& num_neurons) {
        return std::vector<double>(num_neurons, 0);
    }
} // namespace initializers