[mlpack-svn] r16031 - mlpack/trunk/src/mlpack/methods/logistic_regression

fastlab-svn at coffeetalk-1.cc.gatech.edu fastlab-svn at coffeetalk-1.cc.gatech.edu
Thu Nov 14 11:07:09 EST 2013


Author: rcurtin
Date: Thu Nov 14 11:07:09 2013
New Revision: 16031

Log:
Revamp Evaluate(parameters, i) for SGD and related optimizers.  NumFunctions()
also needed to be fixed.


Modified:
   mlpack/trunk/src/mlpack/methods/logistic_regression/logistic_regression_function.cpp
   mlpack/trunk/src/mlpack/methods/logistic_regression/logistic_regression_function.hpp

Modified: mlpack/trunk/src/mlpack/methods/logistic_regression/logistic_regression_function.cpp
==============================================================================
--- mlpack/trunk/src/mlpack/methods/logistic_regression/logistic_regression_function.cpp	(original)
+++ mlpack/trunk/src/mlpack/methods/logistic_regression/logistic_regression_function.cpp	Thu Nov 14 11:07:09 2013
@@ -77,6 +77,30 @@
   return -(result + regularization);
 }
 
+/**
+ * Evaluate the logistic regression objective function, but with only one point.
+ * This is useful for optimizers that use a separable objective function, such
+ * as SGD.
+ */
+double LogisticRegressionFunction::Evaluate(const arma::mat& parameters,
+                                            const size_t i) const
+{
+  // Calculate the regularization term.  We must divide by the number of points,
+  // so that sum(Evaluate(parameters, [1:points])) == Evaluate(parameters).
+  const double regularization = lambda * (1.0 / (2.0 * predictors.n_cols)) *
+      arma::dot(parameters.col(0).subvec(1, parameters.n_elem - 1),
+                parameters.col(0).subvec(1, parameters.n_elem - 1));
+
+  // Calculate sigmoid.
+  const double sigmoid = 1.0 /
+      (1.0 + std::exp(-arma::dot(predictors.col(i), parameters)));
+
+  if (responses[i] == 1)
+    return -(log(sigmoid) + regularization);
+  else
+    return -(log(1.0 - sigmoid) + regularization);
+}
+
 void LogisticRegressionFunction::Gradient(const arma::mat& parameters,
                                           arma::mat& gradient) const
 {

Modified: mlpack/trunk/src/mlpack/methods/logistic_regression/logistic_regression_function.hpp
==============================================================================
--- mlpack/trunk/src/mlpack/methods/logistic_regression/logistic_regression_function.hpp	(original)
+++ mlpack/trunk/src/mlpack/methods/logistic_regression/logistic_regression_function.hpp	Thu Nov 14 11:07:09 2013
@@ -55,6 +55,22 @@
   double Evaluate(const arma::mat& parameters) const;
 
   /**
+   * Evaluate the logistic regression log-likelihood function with the given
+   * parameters, but using only one data point.  This is useful for optimizers
+   * such as SGD, that require a separable objective function.  Note that if the
+   * point has 0 probability of being classified correctly with the given
+   * parameters, then Evaluate() will return nan (this is kind of a corner case
+   * and should not happen for reasonable models).
+   *
+   * The optimum (minimum) of this function is 0.0, and occurs when the point is
+   * classified correctly with very high probability.
+   *
+   * @param parameters Vector of logistic regression parameters.
+   * @param i Index of point to use for objective function evaluation.
+   */
+  double Evaluate(const arma::mat& values, const size_t i) const;
+
+  /**
    * Evaluate the gradient of the logistic regression log-likelihood function
    * with the given parameters.
    *
@@ -66,11 +82,6 @@
   //! Return the initial point for the optimization.
   const arma::mat& GetInitialPoint() const { return initialPoint; }
 
-  //functions to optimize by sgd
-  double Evaluate(const arma::mat& values, const size_t i) const
-  {
-    return Evaluate(values);
-  }
   void Gradient(const arma::mat& values,
                 const size_t i,
                 arma::mat& gradient)
@@ -78,7 +89,7 @@
     Gradient(values,gradient);
   }
 
-  size_t NumFunctions() { return 1; }
+  size_t NumFunctions() const { return predictors.n_cols; }
 
  private:
   //! The initial point, from which to start the optimization.



More information about the mlpack-svn mailing list