[mlpack-svn] r16434 - mlpack/trunk/src/mlpack/methods/sparse_autoencoder
fastlab-svn at coffeetalk-1.cc.gatech.edu
fastlab-svn at coffeetalk-1.cc.gatech.edu
Wed Apr 16 14:35:16 EDT 2014
Author: rcurtin
Date: Wed Apr 16 14:35:16 2014
New Revision: 16434
Log:
First pass: add file headers, include guards, and then my vimrc auto-stripped
all trailing spaces.
Modified:
mlpack/trunk/src/mlpack/methods/sparse_autoencoder/sparse_autoencoder.hpp
mlpack/trunk/src/mlpack/methods/sparse_autoencoder/sparse_autoencoder_function.cpp
mlpack/trunk/src/mlpack/methods/sparse_autoencoder/sparse_autoencoder_function.hpp
mlpack/trunk/src/mlpack/methods/sparse_autoencoder/sparse_autoencoder_impl.hpp
Modified: mlpack/trunk/src/mlpack/methods/sparse_autoencoder/sparse_autoencoder.hpp
==============================================================================
--- mlpack/trunk/src/mlpack/methods/sparse_autoencoder/sparse_autoencoder.hpp (original)
+++ mlpack/trunk/src/mlpack/methods/sparse_autoencoder/sparse_autoencoder.hpp Wed Apr 16 14:35:16 2014
@@ -1,3 +1,12 @@
+/**
+ * @file sparse_autoencoder.hpp
+ * @author Siddharth Agrawal
+ *
+ * An implementation of sparse autoencoders.
+ */
+#ifndef __MLPACK_METHODS_SPARSE_AUTOENCODER_SPARSE_AUTOENCODER_HPP
+#define __MLPACK_METHODS_SPARSE_AUTOENCODER_SPARSE_AUTOENCODER_HPP
+
#include <mlpack/core.hpp>
#include <mlpack/core/optimizers/lbfgs/lbfgs.hpp>
@@ -10,7 +19,7 @@
* Autoencoders can be stacked together to learn a hierarchy of features, which
* provide a better representation of the data for classification. This is a
* method used in the recently developed field of Deep Learning. More technical
- * details about the model can be found on the following webpage:
+ * details about the model can be found on the following webpage:
* http://deeplearning.stanford.edu/wiki/index.php/UFLDL_Tutorial
*
* An example of how to use the interface is shown below:
@@ -66,7 +75,7 @@
const double lambda = 0.0001,
const double beta = 3,
const double rho = 0.01);
-
+
/**
* Construct the Sparse Autoencoder model with the given training data. This
* will train the model. This overload takes an already instantiated optimizer
@@ -77,7 +86,7 @@
* @param optimizer Instantiated optimizer with instantiated error function.
*/
SparseAutoencoder(OptimizerType<SparseAutoencoderFunction>& optimizer);
-
+
/**
* Transforms the provided data into a more meaningful and compact
* representation. The function basically performs a feedforward computation
@@ -87,11 +96,11 @@
* @param features The hidden layer representation of the provided data.
*/
void GetNewFeatures(arma::mat& data, arma::mat& features);
-
+
/**
* Returns the elementwise sigmoid of the passed matrix, where the sigmoid
* function of a real number 'x' is [1 / (1 + exp(-x))].
- *
+ *
* @param x Matrix of real values for which we require the sigmoid activation.
*/
arma::mat Sigmoid(const arma::mat& x) const
@@ -104,7 +113,7 @@
{
this->visibleSize = visible;
}
-
+
//! Gets size of the visible layer.
size_t VisibleSize() const
{
@@ -117,42 +126,42 @@
this->hiddenSize = hidden;
}
- //! Gets the size of the hidden layer.
+ //! Gets the size of the hidden layer.
size_t HiddenSize() const
{
return hiddenSize;
}
-
+
//! Sets the L2-regularization parameter.
void Lambda(const double l)
{
this->lambda = l;
}
-
+
//! Gets the L2-regularization parameter.
double Lambda() const
{
return lambda;
}
-
+
//! Sets the KL divergence parameter.
void Beta(const double b)
{
this->beta = b;
}
-
+
//! Gets the KL divergence parameter.
double Beta() const
{
return beta;
}
-
+
//! Sets the sparsity parameter.
void Rho(const double r)
{
this->rho = r;
}
-
+
//! Gets the sparsity parameter.
double Rho() const
{
@@ -180,3 +189,5 @@
// Include implementation.
#include "sparse_autoencoder_impl.hpp"
+
+#endif
Modified: mlpack/trunk/src/mlpack/methods/sparse_autoencoder/sparse_autoencoder_function.cpp
==============================================================================
--- mlpack/trunk/src/mlpack/methods/sparse_autoencoder/sparse_autoencoder_function.cpp (original)
+++ mlpack/trunk/src/mlpack/methods/sparse_autoencoder/sparse_autoencoder_function.cpp Wed Apr 16 14:35:16 2014
@@ -1,3 +1,9 @@
+/**
+ * @file sparse_autoencoder_function.cpp
+ * @author Siddharth Agrawal
+ *
+ * Implementation of function to be optimized for sparse autoencoders.
+ */
#include "sparse_autoencoder_function.hpp"
using namespace mlpack;
@@ -23,7 +29,7 @@
/** Initializes the parameter weights if the initial point is not passed to the
* constructor. The weights w1, w2 are initialized to randomly in the range
- * [-r, r] where 'r' is decided using the sizes of the visible and hidden
+ * [-r, r] where 'r' is decided using the sizes of the visible and hidden
* layers. The biases b1, b2 are initialized to 0.
*/
const arma::mat SparseAutoencoderFunction::InitializeWeights()
@@ -44,22 +50,22 @@
arma::mat parameters;
parameters.zeros(2*hiddenSize + 1, visibleSize + 1);
-
+
// Initialize w1 and w2 to random values in the range [0, 1].
parameters.submat(0, 0, 2*hiddenSize - 1, visibleSize - 1).randu();
-
+
// Decide the parameter 'r' depending on the size of the visible and hidden
// layers. The formula used is r = sqrt(6) / sqrt(vSize + hSize + 1).
const double range = sqrt(6) / sqrt(visibleSize + hiddenSize + 1);
-
+
//Shift range of w1 and w2 values from [0, 1] to [-r, r].
parameters.submat(0, 0, 2*hiddenSize - 1, visibleSize - 1) = 2 * range *
(parameters.submat(0, 0, 2*hiddenSize - 1, visibleSize - 1) - 0.5);
-
+
return parameters;
}
-/** Evaluates the objective function given the parameters.
+/** Evaluates the objective function given the parameters.
*/
double SparseAutoencoderFunction::Evaluate(const arma::mat& parameters) const
{
@@ -75,7 +81,7 @@
const size_t l1 = hiddenSize;
const size_t l2 = visibleSize;
const size_t l3 = 2*hiddenSize;
-
+
// w1, w2, b1 and b2 are not extracted separately, 'parameters' is directly
// used in their place to avoid copying data. The following representations
// are used:
@@ -83,31 +89,31 @@
// w2 <- parameters.submat(l1, 0, l3-1, l2-1).t()
// b1 <- parameters.submat(0, l2, l1-1, l2)
// b2 <- parameters.submat(l3, 0, l3, l2-1).t()
-
+
arma::mat hiddenLayer, outputLayer;
-
+
// Compute activations of the hidden and output layers.
hiddenLayer = Sigmoid(parameters.submat(0, 0, l1-1, l2-1) * data +
arma::repmat(parameters.submat(0, l2, l1-1, l2), 1, data.n_cols));
-
+
outputLayer = Sigmoid(parameters.submat(l1, 0, l3-1, l2-1).t() * hiddenLayer +
arma::repmat(parameters.submat(l3, 0, l3, l2-1).t(), 1, data.n_cols));
-
+
arma::mat rhoCap, diff;
-
+
// Average activations of the hidden layer.
rhoCap = arma::sum(hiddenLayer, 1) / data.n_cols;
// Difference between the reconstructed data and the original data.
diff = outputLayer - data;
-
+
double wL2SquaredNorm;
-
+
// Calculate squared L2-norms of w1 and w2.
wL2SquaredNorm = arma::accu(parameters.submat(0, 0, l3-1, l2-1) %
parameters.submat(0, 0, l3-1, l2-1));
-
+
double sumOfSquaresError, weightDecay, klDivergence, cost;
-
+
// Calculate the reconstruction error, the regularization cost and the KL
// divergence cost terms. 'sumOfSquaresError' is the average squared l2-norm
// of the reconstructed data difference. 'weightDecay' is the squared l2-norm
@@ -118,10 +124,10 @@
weightDecay = 0.5 * lambda * wL2SquaredNorm;
klDivergence = beta * arma::accu(rho * arma::log(rho / rhoCap) + (1 - rho) *
arma::log((1 - rho) / (1 - rhoCap)));
-
+
// The cost is the sum of the terms calculated above.
cost = sumOfSquaresError + weightDecay + klDivergence;
-
+
return cost;
}
@@ -140,7 +146,7 @@
const size_t l1 = hiddenSize;
const size_t l2 = visibleSize;
const size_t l3 = 2*hiddenSize;
-
+
// w1, w2, b1 and b2 are not extracted separately, 'parameters' is directly
// used in their place to avoid copying data. The following representations
// are used:
@@ -148,25 +154,25 @@
// w2 <- parameters.submat(l1, 0, l3-1, l2-1).t()
// b1 <- parameters.submat(0, l2, l1-1, l2)
// b2 <- parameters.submat(l3, 0, l3, l2-1).t()
-
+
arma::mat hiddenLayer, outputLayer;
-
+
// Compute activations of the hidden and output layers.
hiddenLayer = Sigmoid(parameters.submat(0, 0, l1-1, l2-1) * data +
arma::repmat(parameters.submat(0, l2, l1-1, l2), 1, data.n_cols));
-
+
outputLayer = Sigmoid(parameters.submat(l1, 0, l3-1, l2-1).t() * hiddenLayer +
arma::repmat(parameters.submat(l3, 0, l3, l2-1).t(), 1, data.n_cols));
-
+
arma::mat rhoCap, diff;
-
+
// Average activations of the hidden layer.
rhoCap = arma::sum(hiddenLayer, 1) / data.n_cols;
// Difference between the reconstructed data and the original data.
diff = outputLayer - data;
-
+
arma::mat klDivGrad, delOut, delHid;
-
+
// The delta vector for the output layer is given by diff * f'(z), where z is
// the preactivation and f is the activation function. The derivative of the
// sigmoid function turns out to be f(z) * (1 - f(z)). For every other layer
@@ -177,9 +183,9 @@
delOut = diff % outputLayer % (1 - outputLayer);
delHid = (parameters.submat(l1, 0, l3-1, l2-1) * delOut +
arma::repmat(klDivGrad, 1, data.n_cols)) % hiddenLayer % (1-hiddenLayer);
-
+
gradient.zeros(2*hiddenSize + 1, visibleSize + 1);
-
+
// Compute the gradient values using the activations and the delta values. The
// formula also accounts for the regularization terms in the objective.
// function.
Modified: mlpack/trunk/src/mlpack/methods/sparse_autoencoder/sparse_autoencoder_function.hpp
==============================================================================
--- mlpack/trunk/src/mlpack/methods/sparse_autoencoder/sparse_autoencoder_function.hpp (original)
+++ mlpack/trunk/src/mlpack/methods/sparse_autoencoder/sparse_autoencoder_function.hpp Wed Apr 16 14:35:16 2014
@@ -1,3 +1,13 @@
+/**
+ * @file sparse_autoencoder_function.hpp
+ * @author Siddharth Agrawal
+ *
+ * The function to be optimized for sparse autoencoders. Any mlpack optimizer
+ * can be used.
+ */
+#ifndef __MLPACK_METHODS_SPARSE_AUTOENCODER_SPARSE_AUTOENCODER_FUNCTION_HPP
+#define __MLPACK_METHODS_SPARSE_AUTOENCODER_SPARSE_AUTOENCODER_FUNCTION_HPP
+
#include <mlpack/core.hpp>
namespace mlpack {
@@ -11,17 +21,17 @@
class SparseAutoencoderFunction
{
public:
-
+
SparseAutoencoderFunction(const arma::mat& data,
const size_t visibleSize,
const size_t hiddenSize,
const double lambda = 0.0001,
const double beta = 3,
const double rho = 0.01);
-
+
//Initializes the parameters of the model to suitable values.
const arma::mat InitializeWeights();
-
+
/**
* Evaluates the objective function of the Sparse Autoencoder model using the
* given parameters. The cost function has terms for the reconstruction
@@ -33,29 +43,29 @@
* @param parameters Current values of the model parameters.
*/
double Evaluate(const arma::mat& parameters) const;
-
+
/**
* Evaluates the gradient values of the parameters given the current set of
- * parameters. The function performs a feedforward pass and computes the error
- * in reconstructing the data points. It then uses the backpropagation
+ * parameters. The function performs a feedforward pass and computes the error
+ * in reconstructing the data points. It then uses the backpropagation
* algorithm to compute the gradient values.
*
* @param parameters Current values of the model parameters.
* @param gradient Pointer to matrix where gradient values are stored.
*/
void Gradient(const arma::mat& parameters, arma::mat& gradient) const;
-
+
/**
* Returns the elementwise sigmoid of the passed matrix, where the sigmoid
* function of a real number 'x' is [1 / (1 + exp(-x))].
- *
+ *
* @param x Matrix of real values for which we require the sigmoid activation.
*/
arma::mat Sigmoid(const arma::mat& x) const
{
return (1.0 / (1 + arma::exp(-x)));
}
-
+
//! Return the initial point for the optimization.
const arma::mat& GetInitialPoint() const { return initialPoint; }
@@ -64,7 +74,7 @@
{
this->visibleSize = visible;
}
-
+
//! Gets size of the visible layer.
size_t VisibleSize() const
{
@@ -77,42 +87,42 @@
this->hiddenSize = hidden;
}
- //! Gets the size of the hidden layer.
+ //! Gets the size of the hidden layer.
size_t HiddenSize() const
{
return hiddenSize;
}
-
+
//! Sets the L2-regularization parameter.
void Lambda(const double l)
{
this->lambda = l;
}
-
+
//! Gets the L2-regularization parameter.
double Lambda() const
{
return lambda;
}
-
+
//! Sets the KL divergence parameter.
void Beta(const double b)
{
this->beta = b;
}
-
+
//! Gets the KL divergence parameter.
double Beta() const
{
return beta;
}
-
+
//! Sets the sparsity parameter.
void Rho(const double r)
{
this->rho = r;
}
-
+
//! Gets the sparsity parameter.
double Rho() const
{
@@ -120,7 +130,7 @@
}
private:
-
+
//! The matrix of data points.
const arma::mat& data;
//! Intial parameter vector.
@@ -139,3 +149,5 @@
}; // namespace nn
}; // namespace mlpack
+
+#endif
Modified: mlpack/trunk/src/mlpack/methods/sparse_autoencoder/sparse_autoencoder_impl.hpp
==============================================================================
--- mlpack/trunk/src/mlpack/methods/sparse_autoencoder/sparse_autoencoder_impl.hpp (original)
+++ mlpack/trunk/src/mlpack/methods/sparse_autoencoder/sparse_autoencoder_impl.hpp Wed Apr 16 14:35:16 2014
@@ -1,3 +1,15 @@
+/**
+ * @file sparse_autoencoder_impl.hpp
+ * @author Siddharth Agrawal
+ *
+ * Implementation of sparse autoencoders.
+ */
+#ifndef __MLPACK_METHODS_SPARSE_AUTOENCODER_SPARSE_AUTOENCODER_IMPL_HPP
+#define __MLPACK_METHODS_SPARSE_AUTOENCODER_SPARSE_AUTOENCODER_IMPL_HPP
+
+// In case it hasn't been included yet.
+#include "sparse_autoencoder.hpp"
+
namespace mlpack {
namespace nn {
@@ -17,9 +29,9 @@
SparseAutoencoderFunction encoderFunction(data, visibleSize, hiddenSize,
lambda, beta, rho);
OptimizerType<SparseAutoencoderFunction> optimizer(encoderFunction);
-
+
parameters = encoderFunction.GetInitialPoint();
-
+
// Train the model.
Timer::Start("sparse_autoencoder_optimization");
const double out = optimizer.Optimize(parameters);
@@ -60,3 +72,5 @@
}; // namespace nn
}; // namespace mlpack
+
+#endif
More information about the mlpack-svn
mailing list