[mlpack-git] master: Added LeakyReLU Layer and its tests; Change the Cmakelists (ea703b9)
gitdub at mlpack.org
gitdub at mlpack.org
Wed Mar 9 14:19:13 EST 2016
Repository : https://github.com/mlpack/mlpack
On branch : master
Link : https://github.com/mlpack/mlpack/compare/c75652ba2e49a284f2b932b66e28491f64103bc2...e6f7ffe266faea0705ad19c7aada8ded99f92706
>---------------------------------------------------------------
commit ea703b9e80bfe7d40f9636317469d686aa111059
Author: Dhawal Arora <d.p.arora1 at gmail.com>
Date: Thu Mar 10 00:49:13 2016 +0530
Added LeakyReLU Layer and its tests; Change the Cmakelists
>---------------------------------------------------------------
ea703b9e80bfe7d40f9636317469d686aa111059
src/mlpack/methods/ann/layer/CMakeLists.txt | 1 +
.../{hard_tanh_layer.hpp => leaky_relu_layer.hpp} | 91 +++++++++-------------
src/mlpack/tests/activation_functions_test.cpp | 63 +++++++++++++++
3 files changed, 100 insertions(+), 55 deletions(-)
diff --git a/src/mlpack/methods/ann/layer/CMakeLists.txt b/src/mlpack/methods/ann/layer/CMakeLists.txt
index 75a6075..04e41df 100644
--- a/src/mlpack/methods/ann/layer/CMakeLists.txt
+++ b/src/mlpack/methods/ann/layer/CMakeLists.txt
@@ -7,6 +7,7 @@ set(SOURCES
bias_layer.hpp
dropout_layer.hpp
hard_tanh_layer.hpp
+ leaky_relu_layer.hpp
linear_layer.hpp
conv_layer.hpp
pooling_layer.hpp
diff --git a/src/mlpack/methods/ann/layer/hard_tanh_layer.hpp b/src/mlpack/methods/ann/layer/leaky_relu_layer.hpp
similarity index 68%
copy from src/mlpack/methods/ann/layer/hard_tanh_layer.hpp
copy to src/mlpack/methods/ann/layer/leaky_relu_layer.hpp
index ca59bb0..120e979 100644
--- a/src/mlpack/methods/ann/layer/hard_tanh_layer.hpp
+++ b/src/mlpack/methods/ann/layer/leaky_relu_layer.hpp
@@ -1,11 +1,13 @@
/**
- * @file hard_tanh_layer.hpp
+ * @file leaky_relu_layer.hpp
* @author Dhawal Arora
*
- * Definition and implementation of the HardTanHLayer layer.
+ * Definition and implementation of LeakyReLULayer layer first introduced
+ * in the acoustic model, Andrew L. Maas, Awni Y. Hannun, Andrew Y. Ng,
+ * "Rectifier Nonlinearities Improve Neural Network Acoustic Models", 2014
*/
-#ifndef __MLPACK_METHODS_ANN_LAYER_HARD_TANH_LAYER_HPP
-#define __MLPACK_METHODS_ANN_LAYER_HARD_TANH_LAYER_HPP
+#ifndef __MLPACK_METHODS_ANN_LAYER_LEAKYRELU_LAYER_HPP
+#define __MLPACK_METHODS_ANN_LAYER_LEAKYRELU_LAYER_HPP
#include <mlpack/core.hpp>
@@ -13,21 +15,14 @@ namespace mlpack {
namespace ann /** Artificial Neural Network. */ {
/**
- * The Hard Tanh activation function, defined by
+ * The LeakyReLU activation function, defined by
*
* @f{eqnarray*}{
- * f(x) &=& \left\{
- * \begin{array}{lr}
- * max & : x > maxValue \\
- * min & : x \le minValue \\
- * x & : otherwise
- * \end{array}
- * \right.
+ * f(x) &=& \max(x, alpha*x) \\
* f'(x) &=& \left\{
* \begin{array}{lr}
- * 0 & : x > maxValue \\
- * 0 & : x \le minValue \\
- * 1 & : otherwise
+ * 1 & : x > 0 \\
+ * alpha & : x \le 0
* \end{array}
* \right.
* @f}
@@ -41,19 +36,17 @@ template <
typename InputDataType = arma::mat,
typename OutputDataType = arma::mat
>
-class HardTanHLayer
+class LeakyReLULayer
{
public:
/**
- * Create the HardTanHLayer object using the specified parameters. The range
- * of the linear region can be adjusted by specifying the maxValue and
- * minValue. Default (maxValue = 1, minValue = -1).
+ * Create the LeakyReLULayer object using the specified parameters.
+ * The non zero gradient can be adjusted by specifying tha parameter
+ * alpha in the range 0 to 1. Default (alpha = 0.03)
*
- * @param maxValue Range of the linear region maximum value.
- * @param minValue Range of the linear region minimum value.
+ * @param alpha Non zero gradient
*/
- HardTanHLayer(const double maxValue = 1, const double minValue = -1) :
- maxValue(maxValue), minValue(minValue)
+ LeakyReLULayer(const double alpha = 0.03) : alpha(alpha)
{
// Nothing to do here.
}
@@ -141,15 +134,10 @@ class HardTanHLayer
//! Modify the delta.
OutputDataType& Delta() { return delta; }
- //! Get the maximum value.
- double const& MaxValue() const { return maxValue; }
- //! Modify the maximum value.
- double& MaxValue() { return maxValue; }
-
- //! Get the minimum value.
- double const& MinValue() const { return minValue; }
- //! Modify the minimum value.
- double& MinValue() { return minValue; }
+ //! Get the non zero gradient
+ double const& Alpha() const { return alpha; }
+ //! Modify the non zero gradient
+ double& Alpha() { return alpha; }
/**
* Serialize the layer.
@@ -157,43 +145,35 @@ class HardTanHLayer
template<typename Archive>
void Serialize(Archive& ar, const unsigned int /* version */)
{
- ar & data::CreateNVP(maxValue, "maxValue");
- ar & data::CreateNVP(minValue, "minValue");
+ ar & data::CreateNVP(alpha, "alpha");
}
private:
/**
- * Computes the HardTanH function.
+ * Computes the LeakReLU function
*
* @param x Input data.
* @return f(x).
*/
double Fn(const double x)
{
- if (x > maxValue)
- return maxValue;
- else if (x < minValue)
- return minValue;
- return x;
+ return std::max(x, alpha * x);
}
/**
- * Computes the HardTanH function using a dense matrix as input.
+ * Computes the Leaky ReLU function using a dense matrix as input.
*
* @param x Input data.
* @param y The resulting output activation.
*/
-
template<typename eT>
void Fn(const arma::Mat<eT>& x, arma::Mat<eT>& y)
{
- y = x;
- y.transform( [&](eT val) { return std::min(
- std::max( val, minValue ), maxValue ); } );
+ y = arma::max(x, alpha * x);
}
/**
- * Computes the HardTanH function using a 3rd-order tensor as input.
+ * Computes the LeakyReLU function using a 3rd-order tensor as input.
*
* @param x Input data.
* @param y The resulting output activation.
@@ -203,26 +183,27 @@ class HardTanHLayer
{
y = x;
for (size_t s = 0; s < x.n_slices; s++)
- Fn(x.slice(s), y.slice(s));
+ fn(x.slice(s), y.slice(s));
}
/**
- * Computes the first derivative of the HardTanH function.
+ * Computes the first derivative of the LeakyReLU function.
*
* @param x Input data.
* @return f'(x)
*/
double Deriv(const double x)
{
- return (x > maxValue || x < minValue) ? 0 : 1;
+ return (x >= 0) ? 1 : alpha;
}
/**
- * Computes the first derivative of the HardTanH function.
+ * Computes the first derivative of the LeakyReLU function.
*
* @param y Input activations.
* @param x The resulting derivatives.
*/
+
template<typename InputType, typename OutputType>
void Deriv(const InputType& x, OutputType& y)
{
@@ -232,6 +213,8 @@ class HardTanHLayer
y(i) = Deriv(x(i));
}
+
+
//! Locally-stored delta object.
OutputDataType delta;
@@ -241,12 +224,10 @@ class HardTanHLayer
//! Locally-stored output parameter object.
OutputDataType outputParameter;
- //! Maximum value for the HardTanH function.
- double maxValue;
+ //! Leakyness Parameter in the range 0 <alpha< 1
+ double alpha;
- //! Minimum value for the HardTanH function.
- double minValue;
-}; // class HardTanHLayer
+}; // class LeakyReLULayer
} // namespace ann
} // namespace mlpack
diff --git a/src/mlpack/tests/activation_functions_test.cpp b/src/mlpack/tests/activation_functions_test.cpp
index 61e28cb..7e7b883 100644
--- a/src/mlpack/tests/activation_functions_test.cpp
+++ b/src/mlpack/tests/activation_functions_test.cpp
@@ -22,6 +22,7 @@
#include <mlpack/methods/ann/layer/linear_layer.hpp>
#include <mlpack/methods/ann/layer/base_layer.hpp>
#include <mlpack/methods/ann/layer/binary_classification_layer.hpp>
+#include <mlpack/methods/ann/layer/leaky_relu_layer.hpp>
#include <mlpack/methods/ann/layer/hard_tanh_layer.hpp>
#include <boost/test/unit_test.hpp>
@@ -167,6 +168,53 @@ void CheckHardTanHDerivativeCorrect(const arma::colvec input,
}
}
+/*
+ * Implementation of the LeakyReLU activation function test. The function is
+ * implemented as LeakyReLU layer in the file leaky_relu_layer.hpp
+ *
+ * @param input Input data used for evaluating the LeakyReLU activation function.
+ * @param target Target data used to evaluate the LeakyReLU activation.
+ */
+void CheckLeakyReLUActivationCorrect(const arma::colvec input,
+ const arma::colvec target)
+{
+ LeakyReLULayer<> lrf;
+
+ // Test the activation function using the entire vector as input.
+ arma::colvec activations;
+ lrf.Forward(input, activations);
+ for (size_t i = 0; i < activations.n_elem; i++)
+ {
+ BOOST_REQUIRE_CLOSE(activations.at(i), target.at(i), 1e-3);
+ }
+}
+
+/*
+ * Implementation of the LeakyReLU activation function derivative test.
+ * The derivative function is implemented as LeakyReLU layer in the file
+ * leaky_relu_layer.hpp
+ *
+ * @param input Input data used for evaluating the LeakyReLU activation function.
+ * @param target Target data used to evaluate the LeakyReLU activation.
+ */
+
+void CheckLeakyReLUDerivativeCorrect(const arma::colvec input,
+ const arma::colvec target)
+{
+ LeakyReLULayer<> lrf;
+
+ // Test the calculation of the derivatives using the entire vector as input.
+ arma::colvec derivatives;
+
+ // This error vector will be set to 1 to get the derivatives.
+ arma::colvec error(input.n_elem);
+ lrf.Backward(input, (arma::colvec)error.ones(), derivatives);
+ for (size_t i = 0; i < derivatives.n_elem; i++)
+ {
+ BOOST_REQUIRE_CLOSE(derivatives.at(i), target.at(i), 1e-3);
+ }
+}
+
/**
* Basic test of the tanh function.
*/
@@ -247,6 +295,21 @@ BOOST_AUTO_TEST_CASE(RectifierFunctionTest)
}
/**
+ * Basic test of the LeakyReLU function.
+ */
+BOOST_AUTO_TEST_CASE(LeakyReLUFunctionTest)
+{
+ const arma::colvec desiredActivations("-0.06 3.2 4.5 -3.006 \
+ 1 -0.03 2 0");
+
+ const arma::colvec desiredDerivatives("0.03 1 1 0.03 \
+ 1 0.03 1 1");
+
+ CheckLeakyReLUActivationCorrect(activationData, desiredActivations);
+ CheckLeakyReLUDerivativeCorrect(desiredActivations, desiredDerivatives);
+}
+
+/**
* Basic test of the HardTanH function.
*/
BOOST_AUTO_TEST_CASE(HardTanHFunctionTest)
More information about the mlpack-git
mailing list