Optimizer.hpp

Go to the documentation of this file.

#pragma once
 
#include <Eigen/Dense>
#include "../Utilities/clue.hpp"
#include "../Layer/Dense.hpp"
 
namespace NNFS
{
    class Optimizer
    {
    public:
        Optimizer(double lr, double decay) : _lr(lr), _current_lr(lr), _iterations(0), _decay(decay) {}
 
        virtual ~Optimizer() = default;
 
        virtual void update_params(std::shared_ptr<Dense> &layer) = 0;
 
        void pre_update_params()
        {
            if (_decay > 0)
            {
                _current_lr = _lr * (1. / (1. + _decay * _iterations));
            }
        }
 
        void post_update_params()
        {
            _iterations += 1;
        }
 
        double &current_lr()
        {
            return _current_lr;
        }
 
        int &iterations()
        {
            return _iterations;
        }
 
    protected:
        const double _lr;   // Learning rate (constant)
        double _current_lr; // Current learning rate
        int _iterations;    // Iteration count
        double _decay;      // Learning rate decay
    };
} // namespace NNFS