[mlpack-git] master: Divide the root square of the second decay value by the first decay value; clarification of the beta parameter. (66cb285)
gitdub at mlpack.org
gitdub at mlpack.org
Fri Apr 1 12:17:57 EDT 2016
Repository : https://github.com/mlpack/mlpack
On branch : master
Link : https://github.com/mlpack/mlpack/compare/6359987ecb0cbf762dc2b2167e574ae595a120d8...66cb285dc7d80068ac9ba6a5f40d239f1a6f672d
>---------------------------------------------------------------
commit 66cb285dc7d80068ac9ba6a5f40d239f1a6f672d
Author: marcus <marcus.edel at fu-berlin.de>
Date: Fri Apr 1 18:17:57 2016 +0200
Divide the root square of the second decay value by the first decay value; clarification of the beta parameter.
>---------------------------------------------------------------
66cb285dc7d80068ac9ba6a5f40d239f1a6f672d
src/mlpack/core/optimizers/adam/adam.hpp | 9 +++++----
src/mlpack/core/optimizers/adam/adam_impl.hpp | 6 +++---
2 files changed, 8 insertions(+), 7 deletions(-)
diff --git a/src/mlpack/core/optimizers/adam/adam.hpp b/src/mlpack/core/optimizers/adam/adam.hpp
index 965ecfb..9894664 100644
--- a/src/mlpack/core/optimizers/adam/adam.hpp
+++ b/src/mlpack/core/optimizers/adam/adam.hpp
@@ -67,8 +67,9 @@ class Adam
*
* @param function Function to be optimized (minimized).
* @param stepSize Step size for each iteration.
- * @param beta1 The first moment coefficient.
- * @param beta2 The second moment coefficient.
+ * @param beta1 Exponential decay rate for the first moment estimates.
+ * @param beta2 Exponential decay rate for the weighted infinity norm
+ estimates.
* @param eps Value used to initialise the mean squared gradient parameter.
* @param maxIterations Maximum number of iterations allowed (0 means no
* limit).
@@ -142,10 +143,10 @@ class Adam
//! The step size for each example.
double stepSize;
- //! The value used as first moment coefficient.
+ //! Exponential decay rate for the first moment estimates.
double beta1;
- //! The value used as second moment coefficient.
+ //! Exponential decay rate for the weighted infinity norm estimates.
double beta2;
//! The value used to initialise the mean squared gradient parameter.
diff --git a/src/mlpack/core/optimizers/adam/adam_impl.hpp b/src/mlpack/core/optimizers/adam/adam_impl.hpp
index bd1b953..878cddc 100644
--- a/src/mlpack/core/optimizers/adam/adam_impl.hpp
+++ b/src/mlpack/core/optimizers/adam/adam_impl.hpp
@@ -111,10 +111,10 @@ double Adam<DecomposableFunctionType>::Optimize(arma::mat& iterate)
variance *= beta2;
variance += (1 - beta2) * (gradient % gradient);
- double biasCorrection1 = 1.0 - std::pow(beta1, (double) i);
- double biasCorrection2 = 1.0 - std::pow(beta2, (double) i);
+ const double biasCorrection1 = 1.0 - std::pow(beta1, (double) i);
+ const double biasCorrection2 = 1.0 - std::pow(beta2, (double) i);
- iterate -= (stepSize * std::sqrt(biasCorrection1) / biasCorrection2) *
+ iterate -= (stepSize * std::sqrt(biasCorrection2) / biasCorrection1) *
mean / (arma::sqrt(variance) + eps);
// Now add that to the overall objective function.
More information about the mlpack-git
mailing list