[mlpack-svn] r12200 - mlpack/trunk/src/mlpack/core/optimizers/lbfgs

[fastlab-svn at coffeetalk-1.cc.gatech.edu]

Author: rcurtin
Date: 2012-04-04 13:00:58 -0400 (Wed, 04 Apr 2012)
New Revision: 12200

Modified:
   mlpack/trunk/src/mlpack/core/optimizers/lbfgs/lbfgs.hpp
   mlpack/trunk/src/mlpack/core/optimizers/lbfgs/lbfgs_impl.hpp
Log:
Handle a case where the step size ends up being 0 but the gradient is not yet
the minimum gradient size.  Maybe it is a little slow (elementwise
comparison is not incredibly fast)...


Modified: mlpack/trunk/src/mlpack/core/optimizers/lbfgs/lbfgs.hpp
===================================================================
--- mlpack/trunk/src/mlpack/core/optimizers/lbfgs/lbfgs.hpp	2012-04-04 16:58:54 UTC (rev 12199)
+++ mlpack/trunk/src/mlpack/core/optimizers/lbfgs/lbfgs.hpp	2012-04-04 17:00:58 UTC (rev 12200)
@@ -207,8 +207,7 @@
   bool LineSearch(double& functionValue,
                   arma::mat& iterate,
                   arma::mat& gradient,
-                  const arma::mat& searchDirection,
-                  double& stepSize);
+                  const arma::mat& searchDirection);
 
   /**
    * Find the L-BFGS search direction.

Modified: mlpack/trunk/src/mlpack/core/optimizers/lbfgs/lbfgs_impl.hpp
===================================================================
--- mlpack/trunk/src/mlpack/core/optimizers/lbfgs/lbfgs_impl.hpp	2012-04-04 16:58:54 UTC (rev 12199)
+++ mlpack/trunk/src/mlpack/core/optimizers/lbfgs/lbfgs_impl.hpp	2012-04-04 17:00:58 UTC (rev 12200)
@@ -90,9 +90,11 @@
 bool L_BFGS<FunctionType>::LineSearch(double& functionValue,
                                       arma::mat& iterate,
                                       arma::mat& gradient,
-                                      const arma::mat& searchDirection,
-                                      double& stepSize)
+                                      const arma::mat& searchDirection)
 {
+  // Default first step size of 1.0.
+  double stepSize = 1.0;
+
   // The initial linear term approximation in the direction of the
   // search direction.
   double initialSearchDirectionDotGradient =
@@ -100,7 +102,11 @@
 
   // If it is not a descent direction, just report failure.
   if (initialSearchDirectionDotGradient > 0.0)
+  {
+    Log::Warn << "L-BFGS line search direction is not a descent direction "
+        << "(terminating)!" << std::endl;
     return false;
+  }
 
   // Save the initial function value.
   double initialFunctionValue = functionValue;
@@ -354,24 +360,17 @@
   // The initial gradient value.
   function.Gradient(iterate, gradient);
 
-  // The flag denoting whether or not the optimization has been successful.
-  bool success = false;
-
-  // The main optimization loop.  Start from 1 to allow running forever.
+  // The main optimization loop.
   for (size_t itNum = 0; optimizeUntilConvergence || (itNum != maxIterations);
-       itNum++)
+       ++itNum)
   {
     Log::Debug << "L-BFGS iteration " << itNum << "; objective " <<
         function.Evaluate(iterate) << "." << std::endl;
-//    Log::Debug << "Coordinates " << std::endl << iterate << std::endl;
 
-//    Log::Debug << "Gradient " << std::endl << gradient << std::endl;
-
     // Break when the norm of the gradient becomes too small.
     if (GradientNormTooSmall(gradient))
     {
-      success = true; // We have found the minimum.
-      Log::Info << "L-BFGS gradient norm too small (terminating)."
+      Log::Debug << "L-BFGS gradient norm too small (terminating successfully)."
           << std::endl;
       break;
     }
@@ -388,13 +387,21 @@
     oldGradient = gradient;
 
     // Do a line search and take a step.
-    double stepSize = 1.0;
-    success = LineSearch(functionValue, iterate, gradient, searchDirection,
-        stepSize);
-
-    if (!success)
+    if (!LineSearch(functionValue, iterate, gradient, searchDirection))
+    {
+      Log::Debug << "Line search failed.  Stopping optimization." << std::endl;
       break; // The line search failed; nothing else to try.
+    }
 
+    // It is possible that the difference between the two coordinates is zero.
+    // In this case we terminate successfully.
+    if (accu(iterate != oldIterate) == 0)
+    {
+      Log::Debug << "L-BFGS step size of 0 (terminating successfully)."
+          << std::endl;
+      break;
+    }
+
     // Overwrite an old basis set.
     UpdateBasisSet(itNum, iterate, oldIterate, gradient, oldGradient);