225 lines
6.4 KiB
C
225 lines
6.4 KiB
C
/******************************************************//**
|
||
* C++ library of the Stochastic Gradient Descent (SGD) methods.
|
||
*
|
||
* Copyright (c) 2020-2031 Yi Zhang (zhangyiss@icloud.com)
|
||
* All rights reserved.
|
||
*
|
||
* Permission is hereby granted, free of charge, to any person obtaining a copy
|
||
* of this software and associated documentation files (the "Software"), to deal
|
||
* in the Software without restriction, including without limitation the rights
|
||
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||
* copies of the Software, and to permit persons to whom the Software is
|
||
* furnished to do so, subject to the following conditions:
|
||
*
|
||
* The above copyright notice and this permission notice shall be included in
|
||
* all copies or substantial portions of the Software.
|
||
*
|
||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||
* THE SOFTWARE.
|
||
*********************************************************/
|
||
|
||
#ifndef _SGD_H
|
||
#define _SGD_H
|
||
|
||
#ifndef _cplusplus
|
||
extern "C"
|
||
{
|
||
|
||
#include "stddef.h"
|
||
#endif
|
||
|
||
/**
|
||
* @brief A simple definition of the float type we use here.
|
||
* Easy to change in the future. For now it is just an alias of the double type.
|
||
*/
|
||
typedef double sgd_float;
|
||
|
||
/**
|
||
* @brief Types of method that could be recognized by the sgd_solver() function.
|
||
*/
|
||
typedef enum
|
||
{
|
||
/**
|
||
* Classic momentum.
|
||
*/
|
||
SGD_MOMENTUM,
|
||
|
||
/**
|
||
* Nesterov’s accelerated gradient (NAG)
|
||
*/
|
||
SGD_NAG,
|
||
|
||
/**
|
||
* AdaGrad method.
|
||
*/
|
||
SGD_ADAGRAD,
|
||
|
||
/**
|
||
* RMSProp method.
|
||
*/
|
||
SGD_RMSPROP,
|
||
|
||
/**
|
||
* Adam method.
|
||
*/
|
||
SGD_ADAM,
|
||
|
||
/**
|
||
* Nadam method.
|
||
*/
|
||
SGD_NADAM,
|
||
|
||
/**
|
||
* AdaMax method.
|
||
*/
|
||
SGD_ADAMAX,
|
||
|
||
/**
|
||
* AdaBelief method.
|
||
*/
|
||
SGD_ADABELIEF,
|
||
} sgd_solver_enum;
|
||
|
||
/**
|
||
* @brief Parameters of the Adam method.
|
||
*/
|
||
typedef struct
|
||
{
|
||
/**
|
||
* Iteration times for the entire observation set. The default is 100.
|
||
*/
|
||
int iteration;
|
||
|
||
/**
|
||
* Epsilon for convergence test. This parameter determines the accuracy
|
||
* with which the solution is to be found. Must be bigger than zero and
|
||
* the default is 1e-6.
|
||
*/
|
||
sgd_float epsilon;
|
||
|
||
/**
|
||
* Damping rate of the classic momentum method and the NAG method, which
|
||
* is typically given between 0 and 1. The default is 0.01.
|
||
*/
|
||
sgd_float mu;
|
||
|
||
/**
|
||
* Step size of the iteration. The default value is 0.001 for Adam and 0.002
|
||
* for AdaMax.
|
||
*/
|
||
sgd_float alpha;
|
||
|
||
/**
|
||
* Exponential decay rates for the first order moment estimates. The range of this
|
||
* parameter is [0, 1) and the default value is 0.9.
|
||
*/
|
||
sgd_float beta_1;
|
||
|
||
/**
|
||
* Exponential decay rates for the second order moment estimates. The range of this
|
||
* parameter is [0, 1) and the default value is 0.999.
|
||
*/
|
||
sgd_float beta_2;
|
||
|
||
/**
|
||
* A small positive number validates the algorithm. The default value is 1e-8.
|
||
*/
|
||
sgd_float sigma;
|
||
} sgd_para;
|
||
|
||
/**
|
||
* @brief Callback interface for calculating the value of objective function
|
||
* and the corresponding model gradients.
|
||
*
|
||
* @param instance The user data sent for the sgd_solver() functions by the client.
|
||
* @param x Pointer of the solution.
|
||
* @param g Pointer of the model gradient.
|
||
* @param n_size Length of the solution.
|
||
* @param m Index of the observation.
|
||
*
|
||
* @return Value of objective function.
|
||
*/
|
||
typedef sgd_float (*sgd_evaulate_ptr)(void *instance, const sgd_float *x, sgd_float *g,
|
||
const int n_size, const int m);
|
||
|
||
/**
|
||
* @brief Callback interface for monitoring the progress and terminate the iteration
|
||
* if necessary.
|
||
*
|
||
* @param instance The user data sent for the sgd_solver() functions by the client.
|
||
* @param fx Current value of the objective function.
|
||
* @param x Current solution.
|
||
* @param g Current model gradients.
|
||
* @param param User defined iteration parameters.
|
||
* @param n_size Length of the solution array.
|
||
* @param k Times of the iteration.
|
||
*
|
||
* @return int Zero to continue the optimization process. Otherwise, the optimization
|
||
* process will be terminated.
|
||
*/
|
||
typedef int (*sgd_progress_ptr)(void *instance, sgd_float fx, const sgd_float *x, const sgd_float *g,
|
||
const sgd_para *param, const int n_size, const int k);
|
||
|
||
/**
|
||
* @brief Locate memory for a sgd_float pointer type.
|
||
*
|
||
* @param[in] n_size Size of the sgd_float array.
|
||
*
|
||
* @return Pointer of the data
|
||
*/
|
||
sgd_float *sgd_malloc(const int n_size);
|
||
|
||
/**
|
||
* @brief Destroy memory used by the sgd_float type array.
|
||
*
|
||
* @param x Pointer of the array.
|
||
*/
|
||
void sgd_free(sgd_float *x);
|
||
|
||
/**
|
||
* @brief Return a sgd_para type instance with default values.
|
||
*
|
||
* @return A sgd_para type instance.
|
||
*/
|
||
sgd_para sgd_default_parameters();
|
||
|
||
/**
|
||
* @brief Return a string explanation for the sgd_solver() function's return values.
|
||
*
|
||
* @param[in] er_index The error index returned by the sgd_solver() function.
|
||
*
|
||
* @return A string explanation of the error.
|
||
*/
|
||
const char* sgd_error_str(int er_index);
|
||
|
||
/**
|
||
* @brief An Adam solver function.
|
||
*
|
||
* @note The size of all arrays must be equal to n_size.
|
||
*
|
||
* @param[in] Evafp Callback function for calculating the objective function and its gradient.
|
||
* @param[in] Profp Callback function for monitoring the optimization process.
|
||
* @param fx Returned best value of the objective function by now.
|
||
* @param m Pointer of the solution array.
|
||
* @param[in] n_size Length of the solution array.
|
||
* @param[in] m_size Length of the observation.
|
||
* @param[in] param Parameters of optimization process.
|
||
* @param instance The user data sent for the function by the client.
|
||
* @param solver_id Solver type used to solve the objective. The default value is SGD_ADAM.
|
||
*
|
||
* @return Status of the function.
|
||
*/
|
||
int sgd_solver(sgd_evaulate_ptr Evafp, sgd_progress_ptr Profp, sgd_float *fx, sgd_float *m,
|
||
const int n_size, const int m_size, const sgd_para *param, void *instance,
|
||
sgd_solver_enum solver_id = SGD_ADAM);
|
||
|
||
#ifndef _cplusplus
|
||
}
|
||
#endif
|
||
|
||
#endif // _SGD_H
|