[mlpack-svn] r11475 - mlpack/trunk/src/mlpack/core/optimizers/aug_lagrangian

fastlab-svn at coffeetalk-1.cc.gatech.edu fastlab-svn at coffeetalk-1.cc.gatech.edu
Sun Feb 12 01:29:31 EST 2012


Author: rcurtin
Date: 2012-02-12 01:29:30 -0500 (Sun, 12 Feb 2012)
New Revision: 11475

Modified:
   mlpack/trunk/src/mlpack/core/optimizers/aug_lagrangian/aug_lagrangian.hpp
   mlpack/trunk/src/mlpack/core/optimizers/aug_lagrangian/aug_lagrangian_function_impl.hpp
   mlpack/trunk/src/mlpack/core/optimizers/aug_lagrangian/aug_lagrangian_impl.hpp
Log:
Further refactor AugLagrangian API to make it simpler.


Modified: mlpack/trunk/src/mlpack/core/optimizers/aug_lagrangian/aug_lagrangian.hpp
===================================================================
--- mlpack/trunk/src/mlpack/core/optimizers/aug_lagrangian/aug_lagrangian.hpp	2012-02-11 22:46:02 UTC (rev 11474)
+++ mlpack/trunk/src/mlpack/core/optimizers/aug_lagrangian/aug_lagrangian.hpp	2012-02-12 06:29:30 UTC (rev 11475)
@@ -49,7 +49,9 @@
       L_BFGSType;
 
   /**
-   * Initialize the Augmented Lagrangian with the default L-BFGS optimizer.
+   * Initialize the Augmented Lagrangian with the default L-BFGS optimizer.  We
+   * limit the number of L-BFGS iterations to 1000, rather than the unlimited
+   * default L-BFGS.
    *
    * @param function The function to be optimized.
    */
@@ -77,8 +79,7 @@
    * @param sigma Initial penalty parameter.
    */
   bool Optimize(arma::mat& coordinates,
-                const size_t maxIterations = 1000,
-                const double sigma = 0.5);
+                const size_t maxIterations = 1000);
 
   /**
    * Optimize the function, giving initial estimates for the Lagrange
@@ -86,16 +87,16 @@
    * contain the Lagrange multipliers of the final solution (if one is found).
    *
    * @param coordinates Output matrix to store the optimized coordinates in.
-   * @param lambda Vector of initial Lagrange multipliers.  Should have length
-   *     equal to the number of constraints.
+   * @param initLambda Vector of initial Lagrange multipliers.  Should have
+   *     length equal to the number of constraints.
+   * @param initSigma Initial penalty parameter.
    * @param maxIterations Maximum number of iterations of the Augmented
    *     Lagrangian algorithm.  0 indicates no maximum.
-   * @param sigma Initial penalty parameter.
    */
   bool Optimize(arma::mat& coordinates,
-                arma::vec& lambda,
-                const size_t maxIterations = 1000,
-                double sigma = 0.5);
+                const arma::vec& initLambda,
+                const double initSigma,
+                const size_t maxIterations = 1000);
 
   //! Get the LagrangianFunction.
   const LagrangianFunction& Function() const { return function; }
@@ -107,12 +108,23 @@
   //! Modify the L-BFGS object used for the actual optimization.
   L_BFGSType& LBFGS() { return lbfgs; }
 
+  //! Get the Lagrange multipliers.
+  const arma::vec& Lambda() const { return augfunc.Lambda(); }
+  //! Modify the Lagrange multipliers (i.e. set them before optimization).
+  arma::vec& Lambda() { return augfunc.Lambda(); }
+
+  //! Get the penalty parameter.
+  double Sigma() const { return augfunc.Sigma(); }
+  //! Modify the penalty parameter.
+  double& Sigma() { return augfunc.Sigma(); }
+
  private:
   //! Function to be optimized.
   LagrangianFunction& function;
 
   //! Internally used AugLagrangianFunction which holds the function we are
-  //! optimizing.  This isn't publically accessible.
+  //! optimizing.  This isn't publically accessible, but we provide ways to get
+  //! to the Lagrange multipliers and the penalty parameter sigma.
   AugLagrangianFunction<LagrangianFunction> augfunc;
 
   //! If the user did not pass an L_BFGS object, we'll use our own internal one.
@@ -120,7 +132,6 @@
 
   //! The L-BFGS optimizer that we will use.
   L_BFGSType& lbfgs;
-
 };
 
 }; // namespace optimization

Modified: mlpack/trunk/src/mlpack/core/optimizers/aug_lagrangian/aug_lagrangian_function_impl.hpp
===================================================================
--- mlpack/trunk/src/mlpack/core/optimizers/aug_lagrangian/aug_lagrangian_function_impl.hpp	2012-02-11 22:46:02 UTC (rev 11474)
+++ mlpack/trunk/src/mlpack/core/optimizers/aug_lagrangian/aug_lagrangian_function_impl.hpp	2012-02-12 06:29:30 UTC (rev 11475)
@@ -19,7 +19,9 @@
 template<typename LagrangianFunction>
 AugLagrangianFunction<LagrangianFunction>::AugLagrangianFunction(
     LagrangianFunction& function) :
-    function(function)
+    function(function),
+    lambda(function.NumConstraints()),
+    sigma(0)
 {
   // Nothing else to do.
 }

Modified: mlpack/trunk/src/mlpack/core/optimizers/aug_lagrangian/aug_lagrangian_impl.hpp
===================================================================
--- mlpack/trunk/src/mlpack/core/optimizers/aug_lagrangian/aug_lagrangian_impl.hpp	2012-02-11 22:46:02 UTC (rev 11474)
+++ mlpack/trunk/src/mlpack/core/optimizers/aug_lagrangian/aug_lagrangian_impl.hpp	2012-02-12 06:29:30 UTC (rev 11475)
@@ -22,7 +22,7 @@
     lbfgsInternal(augfunc),
     lbfgs(lbfgsInternal)
 {
-  // Not sure what to do here (if anything).
+  lbfgs.MaxIterations() = 1000;
 }
 
 template<typename LagrangianFunction>
@@ -36,39 +36,35 @@
   // Nothing to do.  lbfgsInternal isn't used in this case.
 }
 
-// This overload just makes the lambda and calls the other overload.
+// This overload just sets the lambda and sigma and calls the other overload.
 template<typename LagrangianFunction>
 bool AugLagrangian<LagrangianFunction>::Optimize(arma::mat& coordinates,
-                                                 const size_t maxIterations,
-                                                 const double sigma)
+                                                 const arma::vec& initLambda,
+                                                 const double initSigma,
+                                                 const size_t maxIterations)
 {
-  arma::vec lambda = arma::ones<arma::vec>(function.NumConstraints());
+  augfunc.Lambda() = initLambda;
+  augfunc.Sigma() = initSigma;
 
-  return Optimize(coordinates, lambda, maxIterations, sigma);
+  return Optimize(coordinates, maxIterations);
 }
 
 template<typename LagrangianFunction>
 bool AugLagrangian<LagrangianFunction>::Optimize(arma::mat& coordinates,
-                                                 arma::vec& lambda,
-                                                 const size_t maxIterations,
-                                                 double sigma)
+                                                 const size_t maxIterations)
 {
   // Ensure that we update lambda immediately.
-  double penalty_threshold = DBL_MAX;
+  double penaltyThreshold = DBL_MAX;
 
   // Track the last objective to compare for convergence.
-  double last_objective = function.Evaluate(coordinates);
+  double lastObjective = function.Evaluate(coordinates);
 
-  // First, update the utility function class.
-  augfunc.Lambda() = lambda;
-  augfunc.Sigma() = sigma;
-
   // Then, calculate the current penalty.
   double penalty = 0;
   for (size_t i = 0; i < function.NumConstraints(); i++)
     penalty += std::pow(function.EvaluateConstraint(i, coordinates), 2);
 
-  Log::Info << "Penalty is " << penalty << " (threshold " << penalty_threshold
+  Log::Debug << "Penalty is " << penalty << " (threshold " << penaltyThreshold
       << ")." << std::endl;
 
   // The odd comparison allows user to pass maxIterations = 0 (i.e. no limit on
@@ -77,7 +73,7 @@
   for (it = 0; it != (maxIterations - 1); it++)
   {
     Log::Warn << "AugLagrangian on iteration " << it
-        << ", starting with objective "  << last_objective << "." << std::endl;
+        << ", starting with objective "  << lastObjective << "." << std::endl;
 
  //   Log::Warn << coordinates << std::endl;
 
@@ -89,11 +85,11 @@
 
     // Check if we are done with the entire optimization (the threshold we are
     // comparing with is arbitrary).
-    if (std::abs(last_objective - function.Evaluate(coordinates)) < 1e-10 &&
-        sigma > 500000)
+    if (std::abs(lastObjective - function.Evaluate(coordinates)) < 1e-10 &&
+        augfunc.Sigma() > 500000)
       return true;
 
-    last_objective = function.Evaluate(coordinates);
+    lastObjective = function.Evaluate(coordinates);
 
     // Assuming that the optimization has converged to a new set of coordinates,
     // we now update either lambda or sigma.  We update sigma if the penalty
@@ -109,7 +105,7 @@
     }
 
     Log::Warn << "Penalty is " << penalty << " (threshold "
-        << penalty_threshold << ")." << std::endl;
+        << penaltyThreshold << ")." << std::endl;
 
     for (size_t i = 0; i < function.NumConstraints(); ++i)
     {
@@ -119,18 +115,18 @@
 //      Log::Debug << tmpgrad << std::endl;
     }
 
-    if (penalty < penalty_threshold) // We update lambda.
+    if (penalty < penaltyThreshold) // We update lambda.
     {
-      // We use the update: lambda{k + 1} = lambdak - sigma * c(coordinates),
+      // We use the update: lambda_{k + 1} = lambda_k - sigma * c(coordinates),
       // but we have to write a loop to do this for each constraint.
       for (size_t i = 0; i < function.NumConstraints(); i++)
-        lambda[i] -= sigma * function.EvaluateConstraint(i, coordinates);
-      augfunc.Lambda() = lambda;
+        augfunc.Lambda()[i] -= augfunc.Sigma() *
+            function.EvaluateConstraint(i, coordinates);
 
       // We also update the penalty threshold to be a factor of the current
       // penalty.  TODO: this factor should be a parameter (from CLI).  The
       // value of 0.25 is taken from Burer and Monteiro (2002).
-      penalty_threshold = 0.25 * penalty;
+      penaltyThreshold = 0.25 * penalty;
       Log::Warn << "Lagrange multiplier estimates updated." << std::endl;
     }
     else
@@ -138,9 +134,8 @@
       // We multiply sigma by a constant value.  TODO: this factor should be a
       // parameter (from CLI).  The value of 10 is taken from Burer and Monteiro
       // (2002).
-      sigma *= 10;
-      augfunc.Sigma() = sigma;
-      Log::Warn << "Updated sigma to " << sigma << "." << std::endl;
+      augfunc.Sigma() *= 10;
+      Log::Warn << "Updated sigma to " << augfunc.Sigma() << "." << std::endl;
     }
   }
 




More information about the mlpack-svn mailing list