simple-deep-learning-cpp/Optimizers.cpp at main · tte0/simple-deep-learning-cpp · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
#include <assert.h>
#include <tuple>
#include <vector>
#include "VectorOps.h"
#include "Optimizers.h"

namespace optimizers {
    // GradientDescent
    GradientDescent::GradientDescent(double _learning_rate):learning_rate(_learning_rate){}

    void GradientDescent::build(std::pair<size_t,size_t> weights_shape, std::pair<size_t,size_t> biases_shape){
        // No initialization needed for GradientDescent
    }

    std::tuple<std::vector<std::vector<double>>,std::vector<std::vector<double>>>
    GradientDescent::update_parameters(std::vector<std::vector<double>> weights_gradient, std::vector<std::vector<double>> biases_gradient){
        return{
            VectorOps::multiply(weights_gradient, -learning_rate),
            VectorOps::multiply(biases_gradient, -learning_rate)
        };
    }


    // MomentumOptimizer
    MomentumOptimizer::MomentumOptimizer(double _learning_rate, double _beta):
        learning_rate(_learning_rate),
        beta(_beta){}

    void MomentumOptimizer::build(std::pair<size_t,size_t> weights_shape, std::pair<size_t,size_t> biases_shape){
        if(_built) return; // Avoid re-initialization
        moment_W.assign(weights_shape.first, std::vector<double>(weights_shape.second));
        moment_b.assign(biases_shape.first, std::vector<double>(biases_shape.second));
        _built = true;
    }

    std::tuple<std::vector<std::vector<double>>,std::vector<std::vector<double>>>
    MomentumOptimizer::update_parameters(std::vector<std::vector<double>> weights_gradient, std::vector<std::vector<double>> biases_gradient){
        assert(_built);
        moment_W = VectorOps::add(VectorOps::multiply(moment_W, beta), VectorOps::multiply(weights_gradient, 1 - beta));
        moment_b = VectorOps::add(VectorOps::multiply(moment_b, beta), VectorOps::multiply(biases_gradient, 1 - beta));
        return{
            VectorOps::multiply(weights_gradient, -learning_rate),
            VectorOps::multiply(biases_gradient, -learning_rate)
        };
    }


    // AdamOptimizer
    AdamOptimizer::AdamOptimizer(double _learning_rate, double _beta1, double _beta2, double _epsilon):
        learning_rate(_learning_rate),
        beta1(_beta1),
        beta2(_beta2),
        epsilon(_epsilon){}

    void AdamOptimizer::build(std::pair<size_t,size_t> weights_shape, std::pair<size_t,size_t> biases_shape){
        if(_built) return; // Avoid re-initialization
        moment1_W.assign(weights_shape.first, std::vector<double>(weights_shape.second));
        moment2_W.assign(weights_shape.first, std::vector<double>(weights_shape.second));
        moment1_b.assign(biases_shape.first, std::vector<double>(biases_shape.second));
        moment2_b.assign(biases_shape.first, std::vector<double>(biases_shape.second));
        _built = true;
    }

    std::tuple<std::vector<std::vector<double>>,std::vector<std::vector<double>>>
    AdamOptimizer::update_parameters(std::vector<std::vector<double>> weights_gradient, std::vector<std::vector<double>> biases_gradient){
        assert(_built);
        timestep++;
        p_beta1 /= beta1;
        p_beta2 /= beta2;

        moment1_W = VectorOps::add(VectorOps::multiply(moment1_W, beta1), VectorOps::multiply(weights_gradient, 1 - beta1));
        moment1_b = VectorOps::add(VectorOps::multiply(moment1_b, beta1), VectorOps::multiply(biases_gradient, 1 - beta1));

        moment2_W = VectorOps::add(VectorOps::multiply(moment2_W, beta2), VectorOps::multiply(VectorOps::square(weights_gradient), 1 - beta2));
        moment2_b = VectorOps::add(VectorOps::multiply(moment2_b, beta2), VectorOps::multiply(VectorOps::square(biases_gradient), 1 - beta2));

        std::vector<std::vector<double>> corrected_moment1_W = VectorOps::multiply(moment1_W, p_beta1);
        std::vector<std::vector<double>> corrected_moment1_b = VectorOps::multiply(moment1_b, p_beta1);

        std::vector<std::vector<double>> corrected_moment2_W = VectorOps::multiply(moment2_W, p_beta2);
        std::vector<std::vector<double>> corrected_moment2_b = VectorOps::multiply(moment2_b, p_beta2);

        return{
            VectorOps::multiply(VectorOps::divide(corrected_moment1_W, VectorOps::add(VectorOps::sqrt(corrected_moment2_W), epsilon)), -learning_rate),
            VectorOps::multiply(VectorOps::divide(corrected_moment1_b, VectorOps::add(VectorOps::sqrt(corrected_moment2_b), epsilon)), -learning_rate),
        };
    }


    // RMSProp
    RMSProp::RMSProp(double _learning_rate, double _beta, double _epsilon):
        learning_rate(_learning_rate),
        beta(_beta),
        epsilon(_epsilon){}

    void RMSProp::build(std::pair<size_t,size_t> weights_shape, std::pair<size_t,size_t> biases_shape){
        if(_built) return; // Avoid re-initialization
        squared_W.assign(weights_shape.first, std::vector<double>(weights_shape.second));
        squared_b.assign(biases_shape.first, std::vector<double>(biases_shape.second));
        _built = true;
    }

    std::tuple<std::vector<std::vector<double>>,std::vector<std::vector<double>>>
    RMSProp::update_parameters(std::vector<std::vector<double>> weights_gradient, std::vector<std::vector<double>> biases_gradient){
        assert(_built);
        timestep++;
        squared_W = VectorOps::add(VectorOps::multiply(squared_W, beta), VectorOps::multiply(VectorOps::square(weights_gradient), 1 - beta));
        squared_b = VectorOps::add(VectorOps::multiply(squared_b, beta), VectorOps::multiply(VectorOps::square(biases_gradient), 1 - beta));

        // the lion does not include bias correction in RMSprop

        return {
            VectorOps::multiply(VectorOps::divide(weights_gradient, VectorOps::add(VectorOps::sqrt(squared_W), epsilon)), -learning_rate),
            VectorOps::multiply(VectorOps::divide(biases_gradient, VectorOps::add(VectorOps::sqrt(squared_b), epsilon)), -learning_rate)
        };
    }
} // namespace optimizers