[mlpack-git] master: Refactor bias layer for new network API. (dac2a6f)
gitdub at big.cc.gt.atl.ga.us
gitdub at big.cc.gt.atl.ga.us
Sat Aug 29 08:23:25 EDT 2015
Repository : https://github.com/mlpack/mlpack
On branch : master
Link : https://github.com/mlpack/mlpack/compare/ea45ace1ff744390a4c35183528eda881eda5c61...fd336238de224ed72fc23b84e1e2f02ae3c879d6
>---------------------------------------------------------------
commit dac2a6f1f7889b9c6dad7d1b587604df44af0d89
Author: Marcus Edel <marcus.edel at fu-berlin.de>
Date: Wed Aug 26 02:06:09 2015 +0200
Refactor bias layer for new network API.
>---------------------------------------------------------------
dac2a6f1f7889b9c6dad7d1b587604df44af0d89
src/mlpack/methods/ann/layer/bias_layer.hpp | 142 +++++++++++++++++++++-------
1 file changed, 106 insertions(+), 36 deletions(-)
diff --git a/src/mlpack/methods/ann/layer/bias_layer.hpp b/src/mlpack/methods/ann/layer/bias_layer.hpp
index 987fe51..20ff71d 100644
--- a/src/mlpack/methods/ann/layer/bias_layer.hpp
+++ b/src/mlpack/methods/ann/layer/bias_layer.hpp
@@ -19,15 +19,22 @@ namespace ann /** Artificial Neural Network. */ {
* An implementation of a standard bias layer. The BiasLayer class represents a
* single layer of a neural network.
*
+ * A convenient typedef is given:
+ *
+ * - 2DBiasLayer
+ *
* @tparam OptimizerType Type of the optimizer used to update the weights.
* @tparam WeightInitRule Rule used to initialize the weight matrix.
- * @tparam DataType Type of data (arma::colvec, arma::mat arma::sp_mat or
- * arma::cube).
+ * @tparam InputDataType Type of the input data (arma::colvec, arma::mat,
+ * arma::sp_mat or arma::cube).
+ * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat,
+ * arma::sp_mat or arma::cube).
*/
template <
template<typename, typename> class OptimizerType = mlpack::ann::RMSPROP,
class WeightInitRule = NguyenWidrowInitialization,
- typename DataType = arma::mat
+ typename InputDataType = arma::mat,
+ typename OutputDataType = arma::mat
>
class BiasLayer
{
@@ -36,22 +43,21 @@ class BiasLayer
* Create the BiasLayer object using the specified number of units and bias
* parameter.
*
- * @param inSize The number of input units.
* @param outSize The number of output units.
* @param bias The bias value.
* @param WeightInitRule The weight initialization rule used to initialize the
* weight matrix.
*/
- BiasLayer(const size_t inSize,
- const size_t outSize,
+ BiasLayer(const size_t outSize,
const double bias = 1,
WeightInitRule weightInitRule = WeightInitRule()) :
- inSize(inSize),
outSize(outSize),
bias(bias),
optimizer(new OptimizerType<BiasLayer<OptimizerType,
- WeightInitRule,
- DataType>, DataType>(*this)),
+ WeightInitRule,
+ InputDataType,
+ OutputDataType>,
+ InputDataType>(*this)),
ownsOptimizer(true)
{
weightInitRule.Initialize(weights, outSize, 1);
@@ -80,6 +86,23 @@ class BiasLayer
}
/**
+ * Ordinary feed forward pass of a neural network, evaluating the function
+ * f(x) by propagating the activity forward through f.
+ *
+ * @param input Input data used for evaluating the specified function.
+ * @param output Resulting output activation.
+ */
+ template<typename eT>
+ void Forward(const arma::Cube<eT>& input, arma::Cube<eT>& output)
+ {
+ output = input;
+ for (size_t s = 0; s < input.n_slices; s++)
+ {
+ output.slice(s) += weights(s) * bias;
+ }
+ }
+
+ /**
* Ordinary feed backward pass of a neural network, calculating the function
* f(x) by propagating x backwards trough f. Using the results from the feed
* forward pass.
@@ -88,10 +111,10 @@ class BiasLayer
* @param gy The backpropagated error.
* @param g The calculated gradient.
*/
- template<typename eT>
- void Backward(const arma::Mat<eT>& /* unused */,
- const arma::Mat<eT>& gy,
- arma::Mat<eT>& g)
+ template<typename DataType, typename ErrorType>
+ void Backward(const DataType& /* unused */,
+ const ErrorType& gy,
+ ErrorType& g)
{
g = gy;
}
@@ -100,47 +123,73 @@ class BiasLayer
* Calculate the gradient using the output delta and the bias.
*
* @param g The calculated gradient.
+ * @param d The calculated error.
*/
- template<typename eT>
- void Gradient(arma::Mat<eT>& gradient)
+ template<typename GradientDataType, typename eT>
+ void Gradient(GradientDataType& g, arma::Cube<eT>& d)
+ {
+ g = arma::Mat<eT>(weights.n_rows, weights.n_cols);
+ for (size_t s = 0; s < d.n_slices; s++)
+ {
+ g(s) = arma::accu(d.slice(s)) * bias;
+ }
+ }
+
+ /*
+ * Calculate the gradient using the output delta and the bias.
+ *
+ * @param g The calculated gradient.
+ * @param d The calculated error.
+ */
+ template<typename GradientDataType, typename eT>
+ void Gradient(GradientDataType& g, arma::Mat<eT>& d)
{
- gradient = delta * bias;
+ g = d * bias;
}
//! Get the optimizer.
OptimizerType<BiasLayer<OptimizerType,
WeightInitRule,
- DataType>, DataType>& Optimizer() const
+ InputDataType,
+ OutputDataType>, InputDataType>& Optimizer() const
{
return *optimizer;
}
//! Modify the optimizer.
OptimizerType<BiasLayer<OptimizerType,
WeightInitRule,
- DataType>, DataType>& Optimizer()
+ InputDataType,
+ OutputDataType>, InputDataType>& Optimizer()
{
return *optimizer;
}
//! Get the weights.
- DataType& Weights() const { return weights; }
+ InputDataType& Weights() const { return weights; }
//! Modify the weights.
- DataType& Weights() { return weights; }
+ InputDataType& Weights() { return weights; }
- //! Get the parameter.
- DataType& Parameter() const {return parameter; }
- //! Modify the parameter.
- DataType& Parameter() { return parameter; }
+ //! Get the input parameter.
+ InputDataType& InputParameter() const {return inputParameter; }
+ //! Modify the input parameter.
+ InputDataType& InputParameter() { return inputParameter; }
+
+ //! Get the output parameter.
+ OutputDataType& OutputParameter() const {return outputParameter; }
+ //! Modify the output parameter.
+ OutputDataType& OutputParameter() { return outputParameter; }
//! Get the delta.
- DataType& Delta() const {return delta; }
+ OutputDataType& Delta() const {return delta; }
//! Modify the delta.
- DataType& Delta() { return delta; }
+ OutputDataType& Delta() { return delta; }
- private:
- //! Locally-stored number of input units.
- const size_t inSize;
+ //! Get the gradient.
+ InputDataType& Gradient() const {return gradient; }
+ //! Modify the gradient.
+ InputDataType& Gradient() { return gradient; }
+ private:
//! Locally-stored number of output units.
const size_t outSize;
@@ -148,18 +197,25 @@ class BiasLayer
double bias;
//! Locally-stored weight object.
- DataType weights;
+ InputDataType weights;
+
+ //! Locally-stored delta object.
+ OutputDataType delta;
//! Locally-stored delta object.
- DataType delta;
+ InputDataType gradient;
+
+ //! Locally-stored input parameter object.
+ InputDataType inputParameter;
- //! Locally-stored parameter object.
- DataType parameter;
+ //! Locally-stored output parameter object.
+ OutputDataType outputParameter;
//! Locally-stored pointer to the optimzer object.
OptimizerType<BiasLayer<OptimizerType,
WeightInitRule,
- DataType>, DataType>* optimizer;
+ InputDataType,
+ OutputDataType>, InputDataType>* optimizer;
//! Parameter that indicates if the class owns a optimizer object.
bool ownsOptimizer;
@@ -169,9 +225,11 @@ class BiasLayer
template<
template<typename, typename> class OptimizerType,
typename WeightInitRule,
- typename DataType
+ typename InputDataType,
+ typename OutputDataType
>
-class LayerTraits<BiasLayer<OptimizerType, WeightInitRule, DataType> >
+class LayerTraits<BiasLayer<
+ OptimizerType, WeightInitRule, InputDataType, OutputDataType> >
{
public:
static const bool IsBinary = false;
@@ -181,6 +239,18 @@ class LayerTraits<BiasLayer<OptimizerType, WeightInitRule, DataType> >
static const bool IsConnection = true;
};
+/**
+ * Standard 2D-Bias-Layer.
+ */
+template <
+ template<typename, typename> class OptimizerType = mlpack::ann::RMSPROP,
+ class WeightInitRule = NguyenWidrowInitialization,
+ typename InputDataType = arma::mat,
+ typename OutputDataType = arma::cube
+>
+using BiasLayer2D = BiasLayer<
+ OptimizerType, WeightInitRule, InputDataType, OutputDataType>;
+
}; // namespace ann
}; // namespace mlpack
More information about the mlpack-git
mailing list