[mlpack-git] master: Added HardTanH Layer and its tests (0a68185)
gitdub at mlpack.org
gitdub at mlpack.org
Thu Mar 3 16:35:49 EST 2016
Repository : https://github.com/mlpack/mlpack
On branch : master
Link : https://github.com/mlpack/mlpack/compare/d5841b1f7204434f39f4887b342335e195a009f7...3e366ec5016cbf3e48607b585cf5c6554121fc84
>---------------------------------------------------------------
commit 0a681856edd57ab2d8e84ca9d7704df92e475b26
Author: Dhawal Arora <d.p.arora1 at gmail.com>
Date: Fri Mar 4 03:05:49 2016 +0530
Added HardTanH Layer and its tests
>---------------------------------------------------------------
0a681856edd57ab2d8e84ca9d7704df92e475b26
.../layer/{base_layer.hpp => hard_tanh_layer.hpp} | 212 ++++++++++++---------
src/mlpack/tests/activation_functions_test.cpp | 74 +++++++
2 files changed, 198 insertions(+), 88 deletions(-)
diff --git a/src/mlpack/methods/ann/layer/base_layer.hpp b/src/mlpack/methods/ann/layer/hard_tanh_layer.hpp
similarity index 52%
copy from src/mlpack/methods/ann/layer/base_layer.hpp
copy to src/mlpack/methods/ann/layer/hard_tanh_layer.hpp
index 876d87e..7e71675 100644
--- a/src/mlpack/methods/ann/layer/base_layer.hpp
+++ b/src/mlpack/methods/ann/layer/hard_tanh_layer.hpp
@@ -1,54 +1,128 @@
/**
- * @file base_layer.hpp
- * @author Marcus Edel
+ * @file hard_tanh_layer.hpp
+ * @author Dhawal Arora
*
- * Definition of the BaseLayer class, which attaches various functions to the
- * embedding layer.
+ * Implementation of hard_tanh activation function. The function is mentioned below.
*/
-#ifndef __MLPACK_METHODS_ANN_LAYER_BASE_LAYER_HPP
-#define __MLPACK_METHODS_ANN_LAYER_BASE_LAYER_HPP
+
+#ifndef __MLPACK_METHODS_ANN_LAYER_HARD_TANH_LAYER_HPP
+#define __MLPACK_METHODS_ANN_LAYER_HARD_TANH_LAYER_HPP
#include <mlpack/core.hpp>
-#include <mlpack/methods/ann/activation_functions/logistic_function.hpp>
-#include <mlpack/methods/ann/activation_functions/identity_function.hpp>
-#include <mlpack/methods/ann/activation_functions/rectifier_function.hpp>
-#include <mlpack/methods/ann/activation_functions/tanh_function.hpp>
namespace mlpack {
namespace ann /** Artificial Neural Network. */ {
/**
- * Implementation of the base layer. The base layer works as a metaclass which
- * attaches various functions to the embedding layer.
- *
- * A few convenience typedefs are given:
- *
- * - SigmoidLayer
- * - IdentityLayer
- * - ReLULayer
- * - TanHLayer
- * - BaseLayer2D
+ * The Hard Tanh activation function, defined by
*
- * @tparam ActivationFunction Activation function used for the embedding layer.
- * @tparam InputDataType Type of the input data (arma::colvec, arma::mat,
- * arma::sp_mat or arma::cube).
- * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat,
- * arma::sp_mat or arma::cube).
+ * @f{eqnarray*}{
+ * f(x) &=& \left\{
+ * \begin{array}{lr}
+ * max & : x > maxValue \\
+ * min & : x \le minValue \\
+ * x & : otherwise
+ * \end{array}
+ * \right.
+ * f'(x) &=& \left\{
+ * \begin{array}{lr}
+ * 0 & : x > maxValue \\
+ * 0 & : x \le minValue \\
+ * 1 & : otherwise
+ * \end{array}
+ * \right.
+ * @f}
*/
+
template <
- class ActivationFunction = LogisticFunction,
typename InputDataType = arma::mat,
typename OutputDataType = arma::mat
>
-class BaseLayer
+
+class HardTanHLayer
{
public:
+
+ /**
+ * Constructor. Default maxValue is set to 1 and default minValue is set to -1.
+ *
+ */
+
+ HardTanHLayer(const double maxValue = 1.00, const double minValue = -1.00) : maxValue(maxValue), minValue(minValue)
+ {
+ // Nothing to do here.
+ }
+
+ /**
+ * Computes the HardTanH function.
+ *
+ * @param x Input data.
+ * @return f(x).
+ */
+ double fn(const double x)
+ {
+ double val;
+ val = x;
+ if (x > maxValue)
+ val = maxValue;
+ else if (x < minValue)
+ val = minValue;
+ return val;
+ }
+
+ /**
+ * Computes the HardTanH function using a dense matrix as input.
+ *
+ * @param x Input data.
+ * @param y The resulting output activation.
+ */
+
+ template<typename eT>
+ void fn(const arma::Mat<eT>& x, arma::Mat<eT>& y)
+ {
+ arma::Mat<eT> t;
+ t = x;
+ y = t.transform( [&](eT val) { return std::min( std::max( val, minValue ), maxValue ); } );
+ }
+
+ /**
+ * Computes the HardTanH function using a 3rd-order tensor as input.
+ *
+ * @param x Input data.
+ * @param y The resulting output activation.
+ */
+ template<typename eT>
+ void fn(const arma::Cube<eT>& x, arma::Cube<eT>& y)
+ {
+ y = x;
+ for (size_t s = 0; s < x.n_slices; s++)
+ fn(x.slice(s), y.slice(s));
+ }
+
+ /**
+ * Computes the first derivative of the HardTanH function.
+ *
+ * @param x Input data.
+ * @return f'(x)
+ */
+ double deriv(const double x)
+ {
+ return (x > maxValue || x < minValue) ? 0 : 1;
+ }
+
/**
- * Create the BaseLayer object.
+ * Computes the first derivative of the HardTanH function.
+ *
+ * @param y Input activations.
+ * @param x The resulting derivatives.
*/
- BaseLayer()
+ template<typename InputType, typename OutputType>
+ void deriv(const InputType& x, OutputType& y)
{
- // Nothing to do here.
+ y = x;
+
+ for (size_t i = 0; i < x.n_elem; i++)
+ y(i) = deriv(x(i));
}
/**
@@ -58,10 +132,11 @@ class BaseLayer
* @param input Input data used for evaluating the specified function.
* @param output Resulting output activation.
*/
+
template<typename InputType, typename OutputType>
void Forward(const InputType& input, OutputType& output)
{
- ActivationFunction::fn(input, output);
+ fn(input, output);
}
/**
@@ -79,7 +154,7 @@ class BaseLayer
DataType& g)
{
DataType derivative;
- ActivationFunction::deriv(input, derivative);
+ deriv(input, derivative);
g = gy % derivative;
}
@@ -115,7 +190,7 @@ class BaseLayer
}
arma::Cube<eT> derivative;
- ActivationFunction::deriv(input, derivative);
+ deriv(input, derivative);
g = mappedError % derivative;
}
@@ -134,6 +209,17 @@ class BaseLayer
//! Modify the delta.
OutputDataType& Delta() { return delta; }
+ //! Get the Maximum value.
+ double const& getmaxValue() const { return maxValue; }
+ //! Modify the Maximum value.
+ double& setmaxValue() { return maxValue; }
+
+ //! Get the Minimum value.
+ double const& getminValue() const { return minValue; }
+ //! Modify the Minimum value.
+ double& setminValue() { return minValue; }
+
+
/**
* Serialize the layer.
*/
@@ -152,65 +238,15 @@ class BaseLayer
//! Locally-stored output parameter object.
OutputDataType outputParameter;
-}; // class BaseLayer
-// Convenience typedefs.
+ //! Maximum value for the HardTanH function.
+ double maxValue;
-/**
- * Standard Sigmoid-Layer using the logistic activation function.
- */
-template <
- class ActivationFunction = LogisticFunction,
- typename InputDataType = arma::mat,
- typename OutputDataType = arma::mat
->
-using SigmoidLayer = BaseLayer<
- ActivationFunction, InputDataType, OutputDataType>;
+ //! Minimum value for the HardTanH function.
+ double minValue;
-/**
- * Standard Identity-Layer using the identity activation function.
- */
-template <
- class ActivationFunction = IdentityFunction,
- typename InputDataType = arma::mat,
- typename OutputDataType = arma::mat
->
-using IdentityLayer = BaseLayer<
- ActivationFunction, InputDataType, OutputDataType>;
-
-/**
- * Standard rectified linear unit non-linearity layer.
- */
-template <
- class ActivationFunction = RectifierFunction,
- typename InputDataType = arma::mat,
- typename OutputDataType = arma::mat
->
-using ReLULayer = BaseLayer<
- ActivationFunction, InputDataType, OutputDataType>;
-
-/**
- * Standard hyperbolic tangent layer.
- */
-template <
- class ActivationFunction = TanhFunction,
- typename InputDataType = arma::mat,
- typename OutputDataType = arma::mat
->
-using TanHLayer = BaseLayer<
- ActivationFunction, InputDataType, OutputDataType>;
-
-/**
- * Standard Base-Layer2D using the logistic activation function.
- */
-template <
- class ActivationFunction = LogisticFunction,
- typename InputDataType = arma::cube,
- typename OutputDataType = arma::cube
->
-using BaseLayer2D = BaseLayer<
- ActivationFunction, InputDataType, OutputDataType>;
+}; // class HardTanHLayer
} // namespace ann
} // namespace mlpack
diff --git a/src/mlpack/tests/activation_functions_test.cpp b/src/mlpack/tests/activation_functions_test.cpp
index 5c40e2e..26bedb5 100644
--- a/src/mlpack/tests/activation_functions_test.cpp
+++ b/src/mlpack/tests/activation_functions_test.cpp
@@ -21,6 +21,7 @@
#include <mlpack/methods/ann/layer/linear_layer.hpp>
#include <mlpack/methods/ann/layer/base_layer.hpp>
#include <mlpack/methods/ann/layer/binary_classification_layer.hpp>
+#include <mlpack/methods/ann/layer/hard_tanh_layer.hpp>
#include <boost/test/unit_test.hpp>
#include "old_boost_test_definitions.hpp"
@@ -120,6 +121,62 @@ void CheckInverseCorrect(const arma::colvec input)
}
}
+
+/*
+ * Implementation of the HardTanH activation function test. The function is implemented as a HardTanH Layer
+ * in file hard_tanh_layer.hpp
+ * @param input Input data used for evaluating the HardTanH activation function.
+ * @param target Target data used to evaluate the HardTanH activation.
+ *
+ */
+void CheckHardTanHActivationCorrect(const arma::colvec input, const arma::colvec target)
+{
+ HardTanHLayer<> htf;
+ // Test the activation function using a single value as input.
+ for (size_t i = 0; i < target.n_elem; i++)
+ {
+ BOOST_REQUIRE_CLOSE(htf.fn(input.at(i)),
+ target.at(i), 1e-3);
+ }
+
+ // Test the activation function using the entire vector as input.
+ arma::colvec activations;
+ htf.fn(input, activations);
+ for (size_t i = 0; i < activations.n_elem; i++)
+ {
+ BOOST_REQUIRE_CLOSE(activations.at(i), target.at(i), 1e-3);
+ }
+}
+
+/*
+ * Implementation of the HardTanH activation function derivative test. The derivative is implemented in HardTanH Layer
+ * in file hard_tanh_layer.hpp
+ * @param input Input data used for evaluating the HardTanH activation function.
+ * @param target Target data used to evaluate the HardTanH activation.
+ *
+ */
+
+void CheckHardTanHDerivativeCorrect(const arma::colvec input, const arma::colvec target)
+{
+ HardTanHLayer<> htf;
+ // Test the calculation of the derivatives using a single value as input.
+ for (size_t i = 0; i < target.n_elem; i++)
+ {
+ BOOST_REQUIRE_CLOSE(htf.deriv(input.at(i)),
+ target.at(i), 1e-3);
+ }
+
+ // Test the calculation of the derivatives using the entire vector as input.
+ arma::colvec derivatives;
+ htf.deriv(input, derivatives);
+ for (size_t i = 0; i < derivatives.n_elem; i++)
+ {
+ BOOST_REQUIRE_CLOSE(derivatives.at(i), target.at(i), 1e-3);
+ }
+}
+
+
+
/**
* Basic test of the tanh function.
*/
@@ -199,4 +256,21 @@ BOOST_AUTO_TEST_CASE(RectifierFunctionTest)
desiredDerivatives);
}
+/**
+ * Basic test of the HardTanH function.
+ */
+BOOST_AUTO_TEST_CASE(HardTanHFunctionTest)
+{
+ const arma::colvec desiredActivations("-1 1 1 -1 \
+ 1 -1 1 0");
+
+ const arma::colvec desiredDerivatives("0 0 0 0 \
+ 1 1 0 1");
+
+ CheckHardTanHActivationCorrect(activationData, desiredActivations);
+ CheckHardTanHDerivativeCorrect(activationData, desiredDerivatives);
+}
+
+
+
BOOST_AUTO_TEST_SUITE_END();
More information about the mlpack-git
mailing list