[mlpack-svn] r16020 - mlpack/trunk/src/mlpack/methods/logistic_regression
fastlab-svn at coffeetalk-1.cc.gatech.edu
fastlab-svn at coffeetalk-1.cc.gatech.edu
Wed Nov 13 11:42:30 EST 2013
Author: rcurtin
Date: Wed Nov 13 11:42:29 2013
New Revision: 16020
Log:
Refactor Evaluate() function so that it works alright, and eliminate unnecessary
parameters to the function. Move LogisticRegressionFunction implementation into
a .cpp file because it is not templatized (for now).
Added:
mlpack/trunk/src/mlpack/methods/logistic_regression/logistic_regression_function.cpp
- copied, changed from r16015, /mlpack/trunk/src/mlpack/methods/logistic_regression/logistic_regression_function_impl.hpp
Removed:
mlpack/trunk/src/mlpack/methods/logistic_regression/logistic_regression_function_impl.hpp
Modified:
mlpack/trunk/src/mlpack/methods/logistic_regression/CMakeLists.txt
mlpack/trunk/src/mlpack/methods/logistic_regression/logistic_regression_function.hpp
mlpack/trunk/src/mlpack/methods/logistic_regression/logistic_regression_impl.hpp
Modified: mlpack/trunk/src/mlpack/methods/logistic_regression/CMakeLists.txt
==============================================================================
--- mlpack/trunk/src/mlpack/methods/logistic_regression/CMakeLists.txt (original)
+++ mlpack/trunk/src/mlpack/methods/logistic_regression/CMakeLists.txt Wed Nov 13 11:42:29 2013
@@ -5,7 +5,7 @@
logistic_regression.hpp
logistic_regression_impl.hpp
logistic_regression_function.hpp
- logistic_regression_function_impl.hpp
+ logistic_regression_function.cpp
)
# add directory name to sources
@@ -17,10 +17,10 @@
# the parent scope)
set(MLPACK_SRCS ${MLPACK_SRCS} ${DIR_SRCS} PARENT_SCOPE)
-add_executable(logistic_regression
- logistic_regression_main.cpp
-)
-target_link_libraries(logistic_regression
- mlpack
-)
-install(TARGETS logistic_regression RUNTIME DESTINATION bin)
+#add_executable(logistic_regression
+# logistic_regression_main.cpp
+#)
+#target_link_libraries(logistic_regression
+# mlpack
+#)
+#install(TARGETS logistic_regression RUNTIME DESTINATION bin)
Copied: mlpack/trunk/src/mlpack/methods/logistic_regression/logistic_regression_function.cpp (from r16015, /mlpack/trunk/src/mlpack/methods/logistic_regression/logistic_regression_function_impl.hpp)
==============================================================================
--- /mlpack/trunk/src/mlpack/methods/logistic_regression/logistic_regression_function_impl.hpp (original)
+++ mlpack/trunk/src/mlpack/methods/logistic_regression/logistic_regression_function.cpp Wed Nov 13 11:42:29 2013
@@ -1,17 +1,13 @@
/**
- * @file logistic_regression_function_impl.hpp
+ * @file logistic_regression_function.cpp
* @author Sumedh Ghaisas
*
* Implementation of hte LogisticRegressionFunction class.
*/
-#ifndef __MLPACK_METHODS_LOGISTIC_REGRESSION_FUNCTION_IMPL_HPP
-#define __MLPACK_METHODS_LOGISTIC_REGRESSION_FUNCTION_IMPL_HPP
-
-// In case it hasn't been done yet.
#include "logistic_regression_function.hpp"
-namespace mlpack {
-namespace regression {
+using namespace mlpack;
+using namespace mlpack::regression;
LogisticRegressionFunction::LogisticRegressionFunction(
arma::mat& predictors,
@@ -40,6 +36,7 @@
this->initialPoint = arma::zeros<arma::mat>(predictors.n_rows + 1, 1);
}
+/*
arma::vec LogisticRegressionFunction::getSigmoid(const arma::vec& values,
arma::vec& output) const
{
@@ -47,42 +44,59 @@
(arma::ones<arma::vec>(values.n_rows,1) + arma::exp(-values));
return out;
}
+*/
-double LogisticRegressionFunction::Evaluate(const arma::mat& values) const
+/**
+ * Evaluate the logistic regression objective function given the estimated
+ * parameters.
+ */
+double LogisticRegressionFunction::Evaluate(const arma::mat& parameters)
+ const
{
- const size_t nCols = predictors.n_cols;
+ // The objective function is the log-likelihood function (w is the parameters
+ // vector for the model; y is the responses; x is the predictors; sig() is the
+ // sigmoid function):
+ // f(w) = sum(y log(sig(w'x)) + (1 - y) log(sig(1 - w'x))).
+ // We want to minimize this function. L2-regularization is just lambda
+ // multiplied by the squared l2-norm of the parameters then divided by two.
+
+ // For the regularization, we ignore the first term, which is the intercept
+ // term.
+ const double regularization = 0.5 * lambda *
+ arma::dot(parameters.col(0).subvec(1, parameters.n_elem - 1),
+ parameters.col(0).subvec(1, parameters.n_elem - 1));
+
+ // Calculate vectors of sigmoids.
+ const arma::vec exponents = predictors.t() * parameters;
+ const arma::vec sigmoid = 1.0 / (1.0 + arma::exp(-exponents));
+
+ // Assemble full objective function. Often the objective function and the
+ // regularization as given are divided by the number of features, but this
+ // doesn't actually affect the optimization result, so we'll just ignore those
+ // terms for computational efficiency.
+ double result = 0.0;
+ for (size_t i = 0; i < responses.n_elem; ++i)
+ {
+ if (responses[i] == 1)
+ result += responses[i] * log(sigmoid[i]);
+ else
+ result += (1 - responses[i]) * log(1.0 - sigmoid[i]);
+ }
- //sigmoid = Sigmoid(X' * values)
- arma::vec sigmoid = 1 / (1 + arma::exp(-(arma::trans(predictors) * values)));
-
- //l2-regularization(considering only values(2:end) in regularization
- arma::vec temp = arma::trans(values) * values;
- double regularization = lambda * (temp(0,0) - values(0,0) * values(0,0)) /
- (2 * responses.n_rows);
-
- //J = -(sum(y' * log(sigmoid)) + sum((ones(m,1) - y)' * log(ones(m,1)
- // - sigmoid))) + regularization
- return -(sum(arma::trans(responses) * arma::log(sigmoid)) +
- sum(arma::trans(arma::ones<arma::vec>(nCols, 1) - responses) *
- arma::log(arma::ones<arma::vec>(nCols,1) - sigmoid))) /
- predictors.n_cols + regularization;
+ // Invert the result, because it's a minimization.
+ return -(result + regularization);
}
void LogisticRegressionFunction::Gradient(const arma::mat& values,
arma::mat& gradient)
{
//regularization
- arma::mat regularization = arma::zeros<arma::mat>(predictors.n_rows, 1);
- regularization.rows(1, predictors.n_rows - 1) = lambda *
- values.rows(1, predictors.n_rows - 1) / responses.n_rows;
+// arma::mat regularization = arma::zeros<arma::mat>(predictors.n_rows, 1);
+// regularization.rows(1, predictors.n_rows - 1) = lambda *
+// values.rows(1, predictors.n_rows - 1) / responses.n_rows;
//gradient =
- gradient = -(predictors * (responses
- - (1 / (1 + arma::exp(-(arma::trans(predictors) * values))))
- / responses.n_rows + regularization;
+// gradient = -(predictors * (responses
+// - (1 / (1 + arma::exp(-(arma::trans(predictors) * values))))
+// / responses.n_rows + regularization;
}
-
-}; // namespace regression
-}; // namespace mlpack
-
-#endif
Modified: mlpack/trunk/src/mlpack/methods/logistic_regression/logistic_regression_function.hpp
==============================================================================
--- mlpack/trunk/src/mlpack/methods/logistic_regression/logistic_regression_function.hpp (original)
+++ mlpack/trunk/src/mlpack/methods/logistic_regression/logistic_regression_function.hpp Wed Nov 13 11:42:29 2013
@@ -38,8 +38,18 @@
//! Modify the lambda
double& Lambda() { return lambda; }
- //functions to optimize by l-bfgs
- double Evaluate(const arma::mat& values) const;
+ /**
+ * Evaluate the logistic regression log-likelihood function with the given
+ * parameters. Note that if a point has 0 probability of being classified
+ * directly with the given parameters, then Evaluate() will return nan (this
+ * is kind of a corner case and should not happen for reasonable models).
+ *
+ * The optimum (minimum) of this function is 0.0, and occurs when each point
+ * is classified correctly with very high probability.
+ *
+ * @param parameters Vector of logistic regression parameters.
+ */
+ double Evaluate(const arma::mat& parameters) const;
void Gradient(const arma::mat& values, arma::mat& gradient);
Modified: mlpack/trunk/src/mlpack/methods/logistic_regression/logistic_regression_impl.hpp
==============================================================================
--- mlpack/trunk/src/mlpack/methods/logistic_regression/logistic_regression_impl.hpp (original)
+++ mlpack/trunk/src/mlpack/methods/logistic_regression/logistic_regression_impl.hpp Wed Nov 13 11:42:29 2013
@@ -72,11 +72,12 @@
ones.ones(predictors.n_cols);
predictors.insert_rows(0, ones);
- double out = errorFunction.Evaluate(predictors, responses, parameters);
+// double out = errorFunction.Evaluate(predictors, responses, parameters);
predictors.shed_row(0);
- return out;
+// return out;
+ return 0.0;
}
template <template<typename> class OptimizerType>
More information about the mlpack-svn
mailing list