[mlpack-svn] r11460 - mlpack/trunk/src/mlpack/core/optimizers/aug_lagrangian
fastlab-svn at coffeetalk-1.cc.gatech.edu
fastlab-svn at coffeetalk-1.cc.gatech.edu
Fri Feb 10 12:55:34 EST 2012
Author: rcurtin
Date: 2012-02-10 12:55:33 -0500 (Fri, 10 Feb 2012)
New Revision: 11460
Modified:
mlpack/trunk/src/mlpack/core/optimizers/aug_lagrangian/aug_lagrangian.hpp
mlpack/trunk/src/mlpack/core/optimizers/aug_lagrangian/aug_lagrangian_function.hpp
mlpack/trunk/src/mlpack/core/optimizers/aug_lagrangian/aug_lagrangian_function_impl.hpp
mlpack/trunk/src/mlpack/core/optimizers/aug_lagrangian/aug_lagrangian_impl.hpp
Log:
Revamp the AugLagrangian API to deal with the new L-BFGS API, and take a
pre-constructed L-BFGS object at construction time.
Modified: mlpack/trunk/src/mlpack/core/optimizers/aug_lagrangian/aug_lagrangian.hpp
===================================================================
--- mlpack/trunk/src/mlpack/core/optimizers/aug_lagrangian/aug_lagrangian.hpp 2012-02-10 17:55:10 UTC (rev 11459)
+++ mlpack/trunk/src/mlpack/core/optimizers/aug_lagrangian/aug_lagrangian.hpp 2012-02-10 17:55:33 UTC (rev 11460)
@@ -11,7 +11,10 @@
#define __MLPACK_CORE_OPTIMIZERS_AUG_LAGRANGIAN_AUG_LAGRANGIAN_HPP
#include <mlpack/core.hpp>
+#include <mlpack/core/optimizers/lbfgs/lbfgs.hpp>
+#include "aug_lagrangian_function.hpp"
+
namespace mlpack {
namespace optimization {
@@ -41,18 +44,33 @@
class AugLagrangian
{
public:
+ //! Shorthand for the type of the L-BFGS optimizer we'll be using.
+ typedef L_BFGS<AugLagrangianFunction<LagrangianFunction> >
+ L_BFGSType;
+
/**
- * Construct the Augmented Lagrangian optimizer with an instance of the given
- * function.
+ * Initialize the Augmented Lagrangian with the default L-BFGS optimizer.
*
- * @param function Function to be optimizer.
- * @param numBasis Number of points of memory for L-BFGS.
+ * @param function The function to be optimized.
*/
- AugLagrangian(LagrangianFunction& function, size_t numBasis = 5);
+ AugLagrangian(LagrangianFunction& function);
/**
- * Optimize the function.
+ * Initialize the Augmented Lagrangian with a custom L-BFGS optimizer.
*
+ * @param function The function to be optimized. This must be a pre-created
+ * utility AugLagrangianFunction.
+ * @param lbfgs The custom L-BFGS optimizer to be used. This should have
+ * already been initialized with the given AugLagrangianFunction.
+ */
+ AugLagrangian(AugLagrangianFunction<LagrangianFunction>& augfunc,
+ L_BFGSType& lbfgs);
+
+ /**
+ * Optimize the function. The value '1' is used for the initial value of each
+ * Lagrange multiplier. To set the Lagrange multipliers yourself, use the
+ * other overload of Optimize().
+ *
* @param coordinates Output matrix to store the optimized coordinates in.
* @param maxIterations Maximum number of iterations of the Augmented
* Lagrangian algorithm. 0 indicates no maximum.
@@ -60,6 +78,23 @@
*/
bool Optimize(arma::mat& coordinates,
const size_t maxIterations = 1000,
+ const double sigma = 0.5);
+
+ /**
+ * Optimize the function, giving initial estimates for the Lagrange
+ * multipliers. The vector of Lagrange multipliers will be modified to
+ * contain the Lagrange multipliers of the final solution (if one is found).
+ *
+ * @param coordinates Output matrix to store the optimized coordinates in.
+ * @param lambda Vector of initial Lagrange multipliers. Should have length
+ * equal to the number of constraints.
+ * @param maxIterations Maximum number of iterations of the Augmented
+ * Lagrangian algorithm. 0 indicates no maximum.
+ * @param sigma Initial penalty parameter.
+ */
+ bool Optimize(arma::mat& coordinates,
+ arma::vec& lambda,
+ const size_t maxIterations = 1000,
double sigma = 0.5);
//! Get the LagrangianFunction.
@@ -67,16 +102,25 @@
//! Modify the LagrangianFunction.
LagrangianFunction& Function() { return function; }
- //! Get the number of memory points used by L-BFGS.
- size_t NumBasis() const { return numBasis; }
- //! Modify the number of memory points used by L-BFGS.
- size_t& NumBasis() { return numBasis; }
+ //! Get the L-BFGS object used for the actual optimization.
+ const L_BFGSType& LBFGS() const { return lbfgs; }
+ //! Modify the L-BFGS object used for the actual optimization.
+ L_BFGSType& LBFGS() { return lbfgs; }
private:
//! Function to be optimized.
LagrangianFunction& function;
- //! Number of memory points for L-BFGS.
- size_t numBasis;
+
+ //! Internally used AugLagrangianFunction which holds the function we are
+ //! optimizing. This isn't publically accessible.
+ AugLagrangianFunction<LagrangianFunction> augfunc;
+
+ //! If the user did not pass an L_BFGS object, we'll use our own internal one.
+ L_BFGSType lbfgsInternal;
+
+ //! The L-BFGS optimizer that we will use.
+ L_BFGSType& lbfgs;
+
};
}; // namespace optimization
Modified: mlpack/trunk/src/mlpack/core/optimizers/aug_lagrangian/aug_lagrangian_function.hpp
===================================================================
--- mlpack/trunk/src/mlpack/core/optimizers/aug_lagrangian/aug_lagrangian_function.hpp 2012-02-10 17:55:10 UTC (rev 11459)
+++ mlpack/trunk/src/mlpack/core/optimizers/aug_lagrangian/aug_lagrangian_function.hpp 2012-02-10 17:55:33 UTC (rev 11460)
@@ -34,6 +34,15 @@
{
public:
/**
+ * Initialize the AugLagrangianFunction, but don't set the Lagrange
+ * multipliers or penalty parameters yet. Make sure you set the Lagrange
+ * multipliers before you use this...
+ *
+ * @param function Lagrangian function.
+ */
+ AugLagrangianFunction(LagrangianFunction& function);
+
+ /**
* Initialize the AugLagrangianFunction with the given LagrangianFunction,
* Lagrange multipliers, and initial penalty parameter.
*
Modified: mlpack/trunk/src/mlpack/core/optimizers/aug_lagrangian/aug_lagrangian_function_impl.hpp
===================================================================
--- mlpack/trunk/src/mlpack/core/optimizers/aug_lagrangian/aug_lagrangian_function_impl.hpp 2012-02-10 17:55:10 UTC (rev 11459)
+++ mlpack/trunk/src/mlpack/core/optimizers/aug_lagrangian/aug_lagrangian_function_impl.hpp 2012-02-10 17:55:33 UTC (rev 11460)
@@ -18,6 +18,15 @@
// Initialize the AugLagrangianFunction.
template<typename LagrangianFunction>
AugLagrangianFunction<LagrangianFunction>::AugLagrangianFunction(
+ LagrangianFunction& function) :
+ function(function)
+{
+ // Nothing else to do.
+}
+
+// Initialize the AugLagrangianFunction.
+template<typename LagrangianFunction>
+AugLagrangianFunction<LagrangianFunction>::AugLagrangianFunction(
LagrangianFunction& function,
const arma::vec& lambda,
const double sigma) :
Modified: mlpack/trunk/src/mlpack/core/optimizers/aug_lagrangian/aug_lagrangian_impl.hpp
===================================================================
--- mlpack/trunk/src/mlpack/core/optimizers/aug_lagrangian/aug_lagrangian_impl.hpp 2012-02-10 17:55:10 UTC (rev 11459)
+++ mlpack/trunk/src/mlpack/core/optimizers/aug_lagrangian/aug_lagrangian_impl.hpp 2012-02-10 17:55:33 UTC (rev 11460)
@@ -16,33 +16,54 @@
namespace optimization {
template<typename LagrangianFunction>
-AugLagrangian<LagrangianFunction>::AugLagrangian(
- LagrangianFunction& function, size_t numBasis) :
+AugLagrangian<LagrangianFunction>::AugLagrangian(LagrangianFunction& function) :
function(function),
- numBasis(numBasis)
+ augfunc(function),
+ lbfgsInternal(augfunc),
+ lbfgs(lbfgsInternal)
{
// Not sure what to do here (if anything).
}
template<typename LagrangianFunction>
+AugLagrangian<LagrangianFunction>::AugLagrangian(
+ AugLagrangianFunction<LagrangianFunction>& augfunc,
+ L_BFGSType& lbfgs) :
+ function(augfunc.Function()),
+ augfunc(augfunc),
+ lbfgs(lbfgs)
+{
+ // Nothing to do. lbfgsInternal isn't used in this case.
+}
+
+// This overload just makes the lambda and calls the other overload.
+template<typename LagrangianFunction>
bool AugLagrangian<LagrangianFunction>::Optimize(arma::mat& coordinates,
const size_t maxIterations,
+ const double sigma)
+{
+ arma::vec lambda = arma::ones<arma::vec>(function.NumConstraints());
+
+ return Optimize(coordinates, lambda, maxIterations, sigma);
+}
+
+template<typename LagrangianFunction>
+bool AugLagrangian<LagrangianFunction>::Optimize(arma::mat& coordinates,
+ arma::vec& lambda,
+ const size_t maxIterations,
double sigma)
{
- // Choose initial lambda parameters (vector of zeros, for simplicity).
- arma::vec lambda(function.NumConstraints());
- lambda.ones();
- lambda *= -1;
- lambda[0] = -double(coordinates.n_cols);
- double penalty_threshold = DBL_MAX; // Ensure we update lambda immediately.
+ // Ensure that we update lambda immediately.
+ double penalty_threshold = DBL_MAX;
// Track the last objective to compare for convergence.
double last_objective = function.Evaluate(coordinates);
- // First, we create an instance of the utility function class.
- AugLagrangianFunction<LagrangianFunction> f(function, lambda, sigma);
+ // First, update the utility function class.
+ augfunc.Lambda() = lambda;
+ augfunc.Sigma() = sigma;
- // First, calculate the current penalty.
+ // Then, calculate the current penalty.
double penalty = 0;
for (size_t i = 0; i < function.NumConstraints(); i++)
penalty += std::pow(function.EvaluateConstraint(i, coordinates), 2);
@@ -62,11 +83,7 @@
// Log::Warn << trans(coordinates) * coordinates << std::endl;
- // Use L-BFGS to optimize this function for the given lambda and sigma.
- L_BFGS<AugLagrangianFunction<LagrangianFunction> >
- lbfgs(f, numBasis, 1e-4, 0.9, 1e-10, 100, 1e-20, 1e20);
-
- if (!lbfgs.Optimize(1000, coordinates))
+ if (!lbfgs.Optimize(coordinates))
Log::Warn << "L-BFGS reported an error during optimization."
<< std::endl;
@@ -108,7 +125,7 @@
// but we have to write a loop to do this for each constraint.
for (size_t i = 0; i < function.NumConstraints(); i++)
lambda[i] -= sigma * function.EvaluateConstraint(i, coordinates);
- f.Lambda() = lambda;
+ augfunc.Lambda() = lambda;
// We also update the penalty threshold to be a factor of the current
// penalty. TODO: this factor should be a parameter (from CLI). The
@@ -122,7 +139,7 @@
// parameter (from CLI). The value of 10 is taken from Burer and Monteiro
// (2002).
sigma *= 10;
- f.Sigma() = sigma;
+ augfunc.Sigma() = sigma;
Log::Warn << "Updated sigma to " << sigma << "." << std::endl;
}
}
More information about the mlpack-svn
mailing list