[mlpack-git] master: Add 3rd order tensors support (weight init methods). (8394ec0)
gitdub at big.cc.gt.atl.ga.us
gitdub at big.cc.gt.atl.ga.us
Sun Apr 26 06:55:57 EDT 2015
Repository : https://github.com/mlpack/mlpack
On branch : master
Link : https://github.com/mlpack/mlpack/compare/fbd6b1f878ec3b2fa365254a22daf3add743ee51...2eb28bcc2ada2fe09a7ad7073c0bbcbb96aac0c5
>---------------------------------------------------------------
commit 8394ec0a6bf587ec457316bf5dd10aeb66e94a80
Author: Marcus Edel <marcus.edel at fu-berlin.de>
Date: Sat Apr 25 19:53:00 2015 +0200
Add 3rd order tensors support (weight init methods).
>---------------------------------------------------------------
8394ec0a6bf587ec457316bf5dd10aeb66e94a80
.../kathirvalavakumar_subavathi_init.hpp | 50 ++++++++++++++++------
.../methods/ann/init_rules/nguyen_widrow_init.hpp | 37 ++++++++++++----
src/mlpack/methods/ann/init_rules/oivs_init.hpp | 47 ++++++++++++++------
.../methods/ann/init_rules/orthogonal_init.hpp | 42 ++++++++++++------
src/mlpack/methods/ann/init_rules/random_init.hpp | 31 +++++++++++---
5 files changed, 152 insertions(+), 55 deletions(-)
diff --git a/src/mlpack/methods/ann/init_rules/kathirvalavakumar_subavathi_init.hpp b/src/mlpack/methods/ann/init_rules/kathirvalavakumar_subavathi_init.hpp
index dcac6b2..8c3f18e 100644
--- a/src/mlpack/methods/ann/init_rules/kathirvalavakumar_subavathi_init.hpp
+++ b/src/mlpack/methods/ann/init_rules/kathirvalavakumar_subavathi_init.hpp
@@ -46,10 +46,7 @@ namespace ann /** Artificial Neural Network. */ {
* considered in training, f is the transfer function and \={s} is the active
* region in which the derivative of the activation function is greater than 4%
* of the maximum derivatives.
- *
- * @tparam MatType Type of matrix (should be arma::mat or arma::spmat).
*/
-template<typename MatType = arma::mat>
class KathirvalavakumarSubavathiInitialization
{
public:
@@ -59,29 +56,54 @@ class KathirvalavakumarSubavathiInitialization
* @param data The input patterns.
* @param s Parameter that defines the active region.
*/
- KathirvalavakumarSubavathiInitialization(const MatType& data, const double s)
- : data(data), s(s) { }
+ template<typename eT>
+ KathirvalavakumarSubavathiInitialization(const arma::Mat<eT>& data,
+ const double s) : s(s)
+ {
+ dataSum = arma::sum(data + data);
+ }
/**
* Initialize the elements of the specified weight matrix with the
* Kathirvalavakumar-Subavathi method.
*
* @param W Weight matrix to initialize.
- * @param n_rows Number of rows.
- * @return n_cols Number of columns.
+ * @param rows Number of rows.
+ * @param cols Number of columns.
+ */
+ template<typename eT>
+ void Initialize(arma::Mat<eT>& W, const size_t rows, const size_t cols)
+ {
+ arma::Row<eT> b = s * arma::sqrt(3 / (rows * dataSum));
+ const double theta = b.min();
+
+ RandomInitialization randomInit(-theta, theta);
+ randomInit.Initialize(W, rows, cols);
+ }
+
+ /**
+ * Initialize the elements of the specified weight 3rd order tensor with the
+ * Kathirvalavakumar-Subavathi method.
+ *
+ * @param W Weight matrix to initialize.
+ * @param rows Number of rows.
+ * @param cols Number of columns.
*/
- void Initialize(MatType& W, const size_t n_rows, const size_t n_cols)
+ template<typename eT>
+ void Initialize(arma::Cube<eT>& W,
+ const size_t rows,
+ const size_t cols,
+ const size_t slices)
{
- arma::rowvec b = s * arma::sqrt(3 / (n_rows * sum(data + data)));
- double theta = b.min();
+ W = arma::Cube<eT>(rows, cols, slices);
- RandomInitialization<MatType> randomInit(-theta, theta);
- randomInit.Initialize(W, n_rows, n_cols);
+ for (size_t i = 0; i < slices; i++)
+ Initialize(W.slice(i), rows, cols);
}
private:
- //! The input patterns.
- MatType data;
+ //! Parameter that defines the sum of elements in each column.
+ arma::colvec dataSum;
//! Parameter that defines the active region.
const double s;
diff --git a/src/mlpack/methods/ann/init_rules/nguyen_widrow_init.hpp b/src/mlpack/methods/ann/init_rules/nguyen_widrow_init.hpp
index 4cb8a62..5c46343 100644
--- a/src/mlpack/methods/ann/init_rules/nguyen_widrow_init.hpp
+++ b/src/mlpack/methods/ann/init_rules/nguyen_widrow_init.hpp
@@ -43,10 +43,7 @@ namespace ann /** Artificial Neural Network. */ {
* number of neurons in the ingoing layer and gamma defines the random interval
* that is used to initialize the weights with a random value in a specific
* range.
- *
- * @tparam MatType Type of matrix (should be arma::mat or arma::spmat).
*/
-template<typename MatType = arma::mat>
class NguyenWidrowInitialization
{
public:
@@ -66,18 +63,40 @@ class NguyenWidrowInitialization
* Nguyen-Widrow method.
*
* @param W Weight matrix to initialize.
- * @param n_rows Number of rows.
- * @return n_cols Number of columns.
+ * @param rows Number of rows.
+ * @param cols Number of columns.
*/
- void Initialize(MatType& W, const size_t n_rows, const size_t n_cols)
+ template<typename eT>
+ void Initialize(arma::Mat<eT>& W, const size_t rows, const size_t cols)
{
- RandomInitialization<MatType> randomInit(lowerBound, upperBound);
- randomInit.Initialize(W, n_rows, n_cols);
+ RandomInitialization randomInit(lowerBound, upperBound);
+ randomInit.Initialize(W, rows, cols);
- double beta = 0.7 * std::pow(n_cols, 1 / n_rows);
+ double beta = 0.7 * std::pow(cols, 1 / rows);
W *= (beta / arma::norm(W));
}
+ /**
+ * Initialize the elements of the specified weight 3rd order tensor with the
+ * Nguyen-Widrow method.
+ *
+ * @param W Weight matrix to initialize.
+ * @param rows Number of rows.
+ * @param cols Number of columns.
+ * @param slices Number of slices.
+ */
+ template<typename eT>
+ void Initialize(arma::Cube<eT>& W,
+ const size_t rows,
+ const size_t cols,
+ const size_t slices)
+ {
+ W = arma::Cube<eT>(rows, cols, slices);
+
+ for (size_t i = 0; i < slices; i++)
+ Initialize(W.slice(i), rows, cols);
+ }
+
private:
//! The number used as lower bound.
const double lowerBound;
diff --git a/src/mlpack/methods/ann/init_rules/oivs_init.hpp b/src/mlpack/methods/ann/init_rules/oivs_init.hpp
index 85f2c9a..028a41b 100644
--- a/src/mlpack/methods/ann/init_rules/oivs_init.hpp
+++ b/src/mlpack/methods/ann/init_rules/oivs_init.hpp
@@ -47,11 +47,9 @@ namespace ann /** Artificial Neural Network. */ {
* interval.
*
* @tparam ActivationFunction The activation function used for the oivs method.
- * @tparam MatType Type of matrix (should be arma::mat or arma::spmat).
*/
template<
- class ActivationFunction = LogisticFunction,
- typename MatType = arma::mat
+ class ActivationFunction = LogisticFunction
>
class OivsInitialization
{
@@ -66,29 +64,52 @@ class OivsInitialization
OivsInitialization(const double epsilon = 0.1,
const int k = 5,
const double gamma = 0.9) :
- epsilon(epsilon), k(k), gamma(gamma) { }
+ epsilon(epsilon), k(k), gamma(gamma)
+ {
+ double b = std::abs(ActivationFunction::inv(1 - epsilon) -
+ ActivationFunction::inv(epsilon));
+ }
/**
* Initialize the elements of the specified weight matrix with the oivs method.
*
* @param W Weight matrix to initialize.
- * @param n_rows Number of rows.
- * @return n_cols Number of columns.
+ * @param rows Number of rows.
+ * @param cols Number of columns.
*/
- void Initialize(MatType& W, const size_t n_rows, const size_t n_cols)
+ template<typename eT>
+ void Initialize(arma::Mat<eT>& W, const size_t rows, const size_t cols)
{
- double b = std::abs(ActivationFunction::inv(1 - epsilon) -
- ActivationFunction::inv(epsilon));
+ RandomInitialization randomInit(-gamma, gamma);
+ randomInit.Initialize(W, rows, cols);
+
+ W = (b / (k * rows)) * arma::sqrt(W + 1);
+ }
- RandomInitialization<MatType> randomInit(-gamma, gamma);
- randomInit.Initialize(W, n_rows, n_cols);
+ /**
+ * Initialize the elements of the specified weight 3rd order tensor with the
+ * oivs method.
+ *
+ * @param W 3rd order tensor to initialize.
+ * @param rows Number of rows.
+ * @param cols Number of columns.
+ * @param slices Number of slices.
+ */
+ template<typename eT>
+ void Initialize(arma::Cube<eT>& W,
+ const size_t rows,
+ const size_t cols,
+ const size_t slices)
+ {
+ W = arma::Cube<eT>(rows, cols, slices);
- W = (b / (k * n_rows)) * arma::sqrt(W + 1);
+ for (size_t i = 0; i < slices; i++)
+ Initialize(W.slice(i), rows, cols);
}
private:
//! Parameter to control the activation region.
- const double epsilon;
+ const double b;
//! Parameter to control the activation region width.
const int k;
diff --git a/src/mlpack/methods/ann/init_rules/orthogonal_init.hpp b/src/mlpack/methods/ann/init_rules/orthogonal_init.hpp
index 41ade2e..ca2da85 100644
--- a/src/mlpack/methods/ann/init_rules/orthogonal_init.hpp
+++ b/src/mlpack/methods/ann/init_rules/orthogonal_init.hpp
@@ -15,11 +15,7 @@ namespace ann /** Artificial Neural Network. */ {
/**
* This class is used to initialize the weight matrix with the orthogonal
* matrix initialization
- *
- * @tparam MatType Type of matrix (should be arma::mat or arma::spmat).
- * @tparam VecType Type of vector (arma::colvec, arma::mat or arma::sp_mat).
*/
-template<typename MatType = arma::mat, typename VecType = arma::colvec>
class OrthogonalInitialization
{
public:
@@ -31,22 +27,44 @@ class OrthogonalInitialization
OrthogonalInitialization(const double gain = 1.0) : gain(gain) { }
/**
- * Initialize the elements of the specified weight matrix with the
- * orthogonal matrix initialization method.
+ * Initialize the elements of the specified weight matrix with the orthogonal
+ * matrix initialization method.
*
* @param W Weight matrix to initialize.
- * @param n_rows Number of rows.
- * @return n_cols Number of columns.
+ * @param rows Number of rows.
+ * @param cols Number of columns.
*/
- void Initialize(MatType& W, const size_t n_rows, const size_t n_cols)
+ template<typename eT>
+ void Initialize(arma::Mat<eT>& W, const size_t rows, const size_t cols)
{
- MatType V;
- VecType s;
+ arma::Mat<eT> V;
+ arma::Col<eT> s;
- arma::svd_econ(W, s, V, arma::randu<MatType>(n_rows, n_cols));
+ arma::svd_econ(W, s, V, arma::randu<arma::Mat<eT> >(rows, cols));
W *= gain;
}
+ /**
+ * Initialize the elements of the specified weight 3rd order tensor with the
+ * orthogonal matrix initialization method.
+ *
+ * @param W Weight matrix to initialize.
+ * @param rows Number of rows.
+ * @param cols Number of columns.
+ * @param slices Number of slices.
+ */
+ template<typename eT>
+ void Initialize(arma::Cube<eT>& W,
+ const size_t rows,
+ const size_t cols,
+ const size_t slices)
+ {
+ W = arma::Cube<eT>(rows, cols, slices);
+
+ for (size_t i = 0; i < slices; i++)
+ Initialize(W.slice(i), rows, cols);
+ }
+
private:
//! The number used as gain.
const double gain;
diff --git a/src/mlpack/methods/ann/init_rules/random_init.hpp b/src/mlpack/methods/ann/init_rules/random_init.hpp
index 6811e8e..daf3fb6 100644
--- a/src/mlpack/methods/ann/init_rules/random_init.hpp
+++ b/src/mlpack/methods/ann/init_rules/random_init.hpp
@@ -15,10 +15,7 @@ namespace ann /** Artificial Neural Network. */ {
/**
* This class is used to initialize randomly the weight matrix.
- *
- * @tparam MatType Type of matrix (should be arma::mat or arma::spmat).
*/
-template<typename MatType = arma::mat>
class RandomInitialization
{
public:
@@ -47,15 +44,35 @@ class RandomInitialization
* Initialize randomly the elements of the specified weight matrix.
*
* @param W Weight matrix to initialize.
- * @param n_rows Number of rows.
- * @param n_cols Number of columns.
+ * @param rows Number of rows.
+ * @param cols Number of columns.
*/
- void Initialize(MatType& W, const size_t n_rows, const size_t n_cols)
+ template<typename eT>
+ void Initialize(arma::Mat<eT>& W, const size_t rows, const size_t cols)
{
- W = lowerBound + arma::randu<MatType>(n_rows, n_cols) *
+ W = lowerBound + arma::randu<arma::Mat<eT>>(rows, cols) *
(upperBound - lowerBound);
}
+ /**
+ * Initialize randomly the elements of the specified weight 3rd order tensor.
+ *
+ * @param W Weight matrix to initialize.
+ * @param rows Number of rows.
+ * @param cols Number of columns.
+ */
+ template<typename eT>
+ void Initialize(arma::Cube<eT>& W,
+ const size_t rows,
+ const size_t cols,
+ const size_t slices)
+ {
+ W = arma::Cube<eT>(rows, cols, slices);
+
+ for (size_t i = 0; i < slices; i++)
+ Initialize(W.slice(i), rows, cols);
+ }
+
private:
//! The number used as lower bound.
const double lowerBound;
More information about the mlpack-git
mailing list