[mlpack-svn] r16034 - mlpack/trunk/src/mlpack/methods/logistic_regression

[fastlab-svn at coffeetalk-1.cc.gatech.edu]

Author: rcurtin
Date: Thu Nov 14 11:31:41 2013
New Revision: 16034

Log:
Update Gradient() for the separable case.


Modified:
   mlpack/trunk/src/mlpack/methods/logistic_regression/logistic_regression_function.cpp
   mlpack/trunk/src/mlpack/methods/logistic_regression/logistic_regression_function.hpp

Modified: mlpack/trunk/src/mlpack/methods/logistic_regression/logistic_regression_function.cpp
==============================================================================
--- mlpack/trunk/src/mlpack/methods/logistic_regression/logistic_regression_function.cpp	(original)
+++ mlpack/trunk/src/mlpack/methods/logistic_regression/logistic_regression_function.cpp	Thu Nov 14 11:31:41 2013
@@ -101,6 +101,7 @@
     return -(log(1.0 - sigmoid) + regularization);
 }
 
+//! Evaluate the gradient of the logistic regression objective function.
 void LogisticRegressionFunction::Gradient(const arma::mat& parameters,
                                           arma::mat& gradient) const
 {
@@ -113,3 +114,23 @@
       - (1 / (1 + arma::exp(-predictors.t() * parameters))))
       - regularization;
 }
+
+/**
+ * Evaluate the individual gradients of the logistic regression objective
+ * function with respect to individual points.  This is useful for optimizers
+ * that use a separable objective function, such as SGD.
+ */
+void LogisticRegressionFunction::Gradient(const arma::mat& parameters,
+                                          const size_t i,
+                                          arma::mat& gradient) const
+{
+  // Calculate the regularization term.
+  arma::mat regularization = arma::zeros<arma::mat>(predictors.n_rows, 1);
+  regularization.rows(1, predictors.n_rows - 1) = lambda *
+      parameters.col(0).subvec(1, predictors.n_rows - 1) / predictors.n_cols;
+
+  const double sigmoid = 1.0 /
+      (1.0 + std::exp(-arma::dot(predictors.col(i), parameters)));
+
+  gradient = -predictors.col(i) * (responses[i] - sigmoid) - regularization;
+}

Modified: mlpack/trunk/src/mlpack/methods/logistic_regression/logistic_regression_function.hpp
==============================================================================
--- mlpack/trunk/src/mlpack/methods/logistic_regression/logistic_regression_function.hpp	(original)
+++ mlpack/trunk/src/mlpack/methods/logistic_regression/logistic_regression_function.hpp	Thu Nov 14 11:31:41 2013
@@ -57,8 +57,8 @@
   /**
    * Evaluate the logistic regression log-likelihood function with the given
    * parameters, but using only one data point.  This is useful for optimizers
-   * such as SGD, that require a separable objective function.  Note that if the
-   * point has 0 probability of being classified correctly with the given
+   * such as SGD, which require a separable objective function.  Note that if
+   * the point has 0 probability of being classified correctly with the given
    * parameters, then Evaluate() will return nan (this is kind of a corner case
    * and should not happen for reasonable models).
    *
@@ -68,7 +68,7 @@
    * @param parameters Vector of logistic regression parameters.
    * @param i Index of point to use for objective function evaluation.
    */
-  double Evaluate(const arma::mat& values, const size_t i) const;
+  double Evaluate(const arma::mat& parameters, const size_t i) const;
 
   /**
    * Evaluate the gradient of the logistic regression log-likelihood function
@@ -79,16 +79,24 @@
    */
   void Gradient(const arma::mat& parameters, arma::mat& gradient) const;
 
+  /**
+   * Evaluate the gradient of the logistic regression log-likelihood function
+   * with the given parameters, and with respect to only one point in the
+   * dataset.  This is useful for optimizers such as SGD, which require a
+   * separable objective function.
+   *
+   * @param parameters Vector of logistic regression parameters.
+   * @param i Index of points to use for objective function gradient evaluation.
+   * @param gradient Vector to output gradient into.
+   */
+  void Gradient(const arma::mat& parameters,
+                const size_t i,
+                arma::mat& gradient) const;
+
   //! Return the initial point for the optimization.
   const arma::mat& GetInitialPoint() const { return initialPoint; }
 
-  void Gradient(const arma::mat& values,
-                const size_t i,
-                arma::mat& gradient)
-  {
-    Gradient(values,gradient);
-  }
-
+  //! Return the number of separable functions (the number of predictor points).
   size_t NumFunctions() const { return predictors.n_cols; }
 
  private: