[mlpack-svn] r11475 - mlpack/trunk/src/mlpack/core/optimizers/aug_lagrangian
fastlab-svn at coffeetalk-1.cc.gatech.edu
fastlab-svn at coffeetalk-1.cc.gatech.edu
Sun Feb 12 01:29:31 EST 2012
Author: rcurtin
Date: 2012-02-12 01:29:30 -0500 (Sun, 12 Feb 2012)
New Revision: 11475
Modified:
mlpack/trunk/src/mlpack/core/optimizers/aug_lagrangian/aug_lagrangian.hpp
mlpack/trunk/src/mlpack/core/optimizers/aug_lagrangian/aug_lagrangian_function_impl.hpp
mlpack/trunk/src/mlpack/core/optimizers/aug_lagrangian/aug_lagrangian_impl.hpp
Log:
Further refactor AugLagrangian API to make it simpler.
Modified: mlpack/trunk/src/mlpack/core/optimizers/aug_lagrangian/aug_lagrangian.hpp
===================================================================
--- mlpack/trunk/src/mlpack/core/optimizers/aug_lagrangian/aug_lagrangian.hpp 2012-02-11 22:46:02 UTC (rev 11474)
+++ mlpack/trunk/src/mlpack/core/optimizers/aug_lagrangian/aug_lagrangian.hpp 2012-02-12 06:29:30 UTC (rev 11475)
@@ -49,7 +49,9 @@
L_BFGSType;
/**
- * Initialize the Augmented Lagrangian with the default L-BFGS optimizer.
+ * Initialize the Augmented Lagrangian with the default L-BFGS optimizer. We
+ * limit the number of L-BFGS iterations to 1000, rather than the unlimited
+ * default L-BFGS.
*
* @param function The function to be optimized.
*/
@@ -77,8 +79,7 @@
* @param sigma Initial penalty parameter.
*/
bool Optimize(arma::mat& coordinates,
- const size_t maxIterations = 1000,
- const double sigma = 0.5);
+ const size_t maxIterations = 1000);
/**
* Optimize the function, giving initial estimates for the Lagrange
@@ -86,16 +87,16 @@
* contain the Lagrange multipliers of the final solution (if one is found).
*
* @param coordinates Output matrix to store the optimized coordinates in.
- * @param lambda Vector of initial Lagrange multipliers. Should have length
- * equal to the number of constraints.
+ * @param initLambda Vector of initial Lagrange multipliers. Should have
+ * length equal to the number of constraints.
+ * @param initSigma Initial penalty parameter.
* @param maxIterations Maximum number of iterations of the Augmented
* Lagrangian algorithm. 0 indicates no maximum.
- * @param sigma Initial penalty parameter.
*/
bool Optimize(arma::mat& coordinates,
- arma::vec& lambda,
- const size_t maxIterations = 1000,
- double sigma = 0.5);
+ const arma::vec& initLambda,
+ const double initSigma,
+ const size_t maxIterations = 1000);
//! Get the LagrangianFunction.
const LagrangianFunction& Function() const { return function; }
@@ -107,12 +108,23 @@
//! Modify the L-BFGS object used for the actual optimization.
L_BFGSType& LBFGS() { return lbfgs; }
+ //! Get the Lagrange multipliers.
+ const arma::vec& Lambda() const { return augfunc.Lambda(); }
+ //! Modify the Lagrange multipliers (i.e. set them before optimization).
+ arma::vec& Lambda() { return augfunc.Lambda(); }
+
+ //! Get the penalty parameter.
+ double Sigma() const { return augfunc.Sigma(); }
+ //! Modify the penalty parameter.
+ double& Sigma() { return augfunc.Sigma(); }
+
private:
//! Function to be optimized.
LagrangianFunction& function;
//! Internally used AugLagrangianFunction which holds the function we are
- //! optimizing. This isn't publically accessible.
+ //! optimizing. This isn't publically accessible, but we provide ways to get
+ //! to the Lagrange multipliers and the penalty parameter sigma.
AugLagrangianFunction<LagrangianFunction> augfunc;
//! If the user did not pass an L_BFGS object, we'll use our own internal one.
@@ -120,7 +132,6 @@
//! The L-BFGS optimizer that we will use.
L_BFGSType& lbfgs;
-
};
}; // namespace optimization
Modified: mlpack/trunk/src/mlpack/core/optimizers/aug_lagrangian/aug_lagrangian_function_impl.hpp
===================================================================
--- mlpack/trunk/src/mlpack/core/optimizers/aug_lagrangian/aug_lagrangian_function_impl.hpp 2012-02-11 22:46:02 UTC (rev 11474)
+++ mlpack/trunk/src/mlpack/core/optimizers/aug_lagrangian/aug_lagrangian_function_impl.hpp 2012-02-12 06:29:30 UTC (rev 11475)
@@ -19,7 +19,9 @@
template<typename LagrangianFunction>
AugLagrangianFunction<LagrangianFunction>::AugLagrangianFunction(
LagrangianFunction& function) :
- function(function)
+ function(function),
+ lambda(function.NumConstraints()),
+ sigma(0)
{
// Nothing else to do.
}
Modified: mlpack/trunk/src/mlpack/core/optimizers/aug_lagrangian/aug_lagrangian_impl.hpp
===================================================================
--- mlpack/trunk/src/mlpack/core/optimizers/aug_lagrangian/aug_lagrangian_impl.hpp 2012-02-11 22:46:02 UTC (rev 11474)
+++ mlpack/trunk/src/mlpack/core/optimizers/aug_lagrangian/aug_lagrangian_impl.hpp 2012-02-12 06:29:30 UTC (rev 11475)
@@ -22,7 +22,7 @@
lbfgsInternal(augfunc),
lbfgs(lbfgsInternal)
{
- // Not sure what to do here (if anything).
+ lbfgs.MaxIterations() = 1000;
}
template<typename LagrangianFunction>
@@ -36,39 +36,35 @@
// Nothing to do. lbfgsInternal isn't used in this case.
}
-// This overload just makes the lambda and calls the other overload.
+// This overload just sets the lambda and sigma and calls the other overload.
template<typename LagrangianFunction>
bool AugLagrangian<LagrangianFunction>::Optimize(arma::mat& coordinates,
- const size_t maxIterations,
- const double sigma)
+ const arma::vec& initLambda,
+ const double initSigma,
+ const size_t maxIterations)
{
- arma::vec lambda = arma::ones<arma::vec>(function.NumConstraints());
+ augfunc.Lambda() = initLambda;
+ augfunc.Sigma() = initSigma;
- return Optimize(coordinates, lambda, maxIterations, sigma);
+ return Optimize(coordinates, maxIterations);
}
template<typename LagrangianFunction>
bool AugLagrangian<LagrangianFunction>::Optimize(arma::mat& coordinates,
- arma::vec& lambda,
- const size_t maxIterations,
- double sigma)
+ const size_t maxIterations)
{
// Ensure that we update lambda immediately.
- double penalty_threshold = DBL_MAX;
+ double penaltyThreshold = DBL_MAX;
// Track the last objective to compare for convergence.
- double last_objective = function.Evaluate(coordinates);
+ double lastObjective = function.Evaluate(coordinates);
- // First, update the utility function class.
- augfunc.Lambda() = lambda;
- augfunc.Sigma() = sigma;
-
// Then, calculate the current penalty.
double penalty = 0;
for (size_t i = 0; i < function.NumConstraints(); i++)
penalty += std::pow(function.EvaluateConstraint(i, coordinates), 2);
- Log::Info << "Penalty is " << penalty << " (threshold " << penalty_threshold
+ Log::Debug << "Penalty is " << penalty << " (threshold " << penaltyThreshold
<< ")." << std::endl;
// The odd comparison allows user to pass maxIterations = 0 (i.e. no limit on
@@ -77,7 +73,7 @@
for (it = 0; it != (maxIterations - 1); it++)
{
Log::Warn << "AugLagrangian on iteration " << it
- << ", starting with objective " << last_objective << "." << std::endl;
+ << ", starting with objective " << lastObjective << "." << std::endl;
// Log::Warn << coordinates << std::endl;
@@ -89,11 +85,11 @@
// Check if we are done with the entire optimization (the threshold we are
// comparing with is arbitrary).
- if (std::abs(last_objective - function.Evaluate(coordinates)) < 1e-10 &&
- sigma > 500000)
+ if (std::abs(lastObjective - function.Evaluate(coordinates)) < 1e-10 &&
+ augfunc.Sigma() > 500000)
return true;
- last_objective = function.Evaluate(coordinates);
+ lastObjective = function.Evaluate(coordinates);
// Assuming that the optimization has converged to a new set of coordinates,
// we now update either lambda or sigma. We update sigma if the penalty
@@ -109,7 +105,7 @@
}
Log::Warn << "Penalty is " << penalty << " (threshold "
- << penalty_threshold << ")." << std::endl;
+ << penaltyThreshold << ")." << std::endl;
for (size_t i = 0; i < function.NumConstraints(); ++i)
{
@@ -119,18 +115,18 @@
// Log::Debug << tmpgrad << std::endl;
}
- if (penalty < penalty_threshold) // We update lambda.
+ if (penalty < penaltyThreshold) // We update lambda.
{
- // We use the update: lambda{k + 1} = lambdak - sigma * c(coordinates),
+ // We use the update: lambda_{k + 1} = lambda_k - sigma * c(coordinates),
// but we have to write a loop to do this for each constraint.
for (size_t i = 0; i < function.NumConstraints(); i++)
- lambda[i] -= sigma * function.EvaluateConstraint(i, coordinates);
- augfunc.Lambda() = lambda;
+ augfunc.Lambda()[i] -= augfunc.Sigma() *
+ function.EvaluateConstraint(i, coordinates);
// We also update the penalty threshold to be a factor of the current
// penalty. TODO: this factor should be a parameter (from CLI). The
// value of 0.25 is taken from Burer and Monteiro (2002).
- penalty_threshold = 0.25 * penalty;
+ penaltyThreshold = 0.25 * penalty;
Log::Warn << "Lagrange multiplier estimates updated." << std::endl;
}
else
@@ -138,9 +134,8 @@
// We multiply sigma by a constant value. TODO: this factor should be a
// parameter (from CLI). The value of 10 is taken from Burer and Monteiro
// (2002).
- sigma *= 10;
- augfunc.Sigma() = sigma;
- Log::Warn << "Updated sigma to " << sigma << "." << std::endl;
+ augfunc.Sigma() *= 10;
+ Log::Warn << "Updated sigma to " << augfunc.Sigma() << "." << std::endl;
}
}
More information about the mlpack-svn
mailing list