[mlpack-svn] r16011 - mlpack/trunk/src/mlpack/methods/logistic_regression
fastlab-svn at coffeetalk-1.cc.gatech.edu
fastlab-svn at coffeetalk-1.cc.gatech.edu
Fri Nov 8 13:52:11 EST 2013
Author: rcurtin
Date: Fri Nov 8 13:52:11 2013
New Revision: 16011
Log:
Initial commit of logistic regression by Sumedh Ghaisas (#305).
Added:
mlpack/trunk/src/mlpack/methods/logistic_regression/
mlpack/trunk/src/mlpack/methods/logistic_regression/CMakeLists.txt
mlpack/trunk/src/mlpack/methods/logistic_regression/logistic_regression.hpp
mlpack/trunk/src/mlpack/methods/logistic_regression/logistic_regression_function.hpp
mlpack/trunk/src/mlpack/methods/logistic_regression/logistic_regression_function_impl.hpp
mlpack/trunk/src/mlpack/methods/logistic_regression/logistic_regression_impl.hpp
Added: mlpack/trunk/src/mlpack/methods/logistic_regression/CMakeLists.txt
==============================================================================
--- (empty file)
+++ mlpack/trunk/src/mlpack/methods/logistic_regression/CMakeLists.txt Fri Nov 8 13:52:11 2013
@@ -0,0 +1,26 @@
+# Define the files we need to compile
+# Anything not in this list will not be compiled into the output library
+# Do not include test programs here
+set(SOURCES
+ logistic_regression.hpp
+ logistic_regression_impl.hpp
+ logistic_regression_function.hpp
+ logistic_regression_function_impl.hpp
+)
+
+# add directory name to sources
+set(DIR_SRCS)
+foreach(file ${SOURCES})
+ set(DIR_SRCS ${DIR_SRCS} ${CMAKE_CURRENT_SOURCE_DIR}/${file})
+endforeach()
+# append sources (with directory name) to list of all MLPACK sources (used at
+# the parent scope)
+set(MLPACK_SRCS ${MLPACK_SRCS} ${DIR_SRCS} PARENT_SCOPE)
+
+add_executable(logistic_regression
+ logistic_regression_main.cpp
+)
+target_link_libraries(logistic_regression
+ mlpack
+)
+install(TARGETS logistic_regression RUNTIME DESTINATION bin)
Added: mlpack/trunk/src/mlpack/methods/logistic_regression/logistic_regression.hpp
==============================================================================
--- (empty file)
+++ mlpack/trunk/src/mlpack/methods/logistic_regression/logistic_regression.hpp Fri Nov 8 13:52:11 2013
@@ -0,0 +1,71 @@
+/**
+ * @file logistic_regression.hpp
+ * @author Sumedh Ghaisas
+ *
+ * The LogisticRegression class, which implements logistic regression.
+ */
+#ifndef __MLPACK_METHODS_LOGISTIC_REGRESSION_LOGISTIC_REGRESSION_HPP
+#define __MLPACK_METHODS_LOGISTIC_REGRESSION_LOGISTIC_REGRESSION_HPP
+
+#include <mlpack/core.hpp>
+#include <mlpack/core/optimizers/lbfgs/lbfgs.hpp>
+
+#include "logistic_regression_function.hpp"
+
+namespace mlpack {
+namespace regression {
+
+template<
+ template<typename> class OptimizerType = mlpack::optimization::L_BFGS
+>
+class LogisticRegression
+{
+ public:
+ LogisticRegression(arma::mat& predictors,
+ arma::vec& responses,
+ const double lambda = 0);
+
+ LogisticRegression(arma::mat& predictors,
+ arma::vec& responses,
+ const arma::mat& initialPoint,
+ const double lambda = 0);
+
+ //! Return the parameters (the b vector).
+ const arma::vec& Parameters() const { return parameters; }
+ //! Modify the parameters (the b vector).
+ arma::vec& Parameters() { return parameters; }
+
+ //! Return the lambda value
+ const double& Lambda() const { return lambda; }
+ //! Modify the lambda value
+ double& Lambda() { return lambda; }
+
+ double LearnModel();
+
+ //predict functions
+ void Predict(arma::mat& predictors,
+ arma::vec& responses,
+ const double decisionBoundary = 0.5);
+
+ double ComputeAccuracy(arma::mat& predictors,
+ const arma::vec& responses,
+ const double decisionBoundary = 0.5);
+
+ double ComputeError(arma::mat& predictors,const arma::vec& responses);
+
+ private:
+ arma::vec parameters;
+ arma::mat& predictors;
+ arma::vec& responses;
+ LogisticRegressionFunction errorFunction;
+ OptimizerType<LogisticRegressionFunction> optimizer;
+ double lambda;
+};
+
+}; // namespace regression
+}; // namespace mlpack
+
+// Include implementation.
+#include "logistic_regression_impl.hpp"
+
+#endif // __MLPACK_METHODS_LOGISTIC_REGRESSION_LOGISTIC_REGRESSION_HPP
Added: mlpack/trunk/src/mlpack/methods/logistic_regression/logistic_regression_function.hpp
==============================================================================
--- (empty file)
+++ mlpack/trunk/src/mlpack/methods/logistic_regression/logistic_regression_function.hpp Fri Nov 8 13:52:11 2013
@@ -0,0 +1,80 @@
+/**
+ * @file logistic_regression_function.hpp
+ * @author Sumedh Ghaisas
+ *
+ * Implementation of the logistic regression function, which is meant to be
+ * optimized by a separate optimizer class that takes LogisticRegressionFunction
+ * as its FunctionType class.
+ */
+#ifndef __MLPACK_METHODS_LOGISTIC_REGRESSION_LOGISTIC_REGRESSION_FUNCTION_HPP
+#define __MLPACK_METHODS_LOGISTIC_REGRESSION_LOGISTIC_REGRESSION_FUNCTION_HPP
+
+#include <mlpack/core.hpp>
+
+namespace mlpack {
+namespace regression {
+
+class LogisticRegressionFunction
+{
+ public:
+ LogisticRegressionFunction(arma::mat& predictors,
+ arma::vec& responses,
+ const double lambda = 0);
+
+ LogisticRegressionFunction(arma::mat& predictors,
+ arma::vec& responses,
+ const arma::mat& initialPoint,
+ const double lambda = 0);
+
+ arma::vec getSigmoid(const arma::vec& values) const;
+
+ //evaluates the logistic function with given parameters
+ double Evaluate(const arma::mat& predictors,
+ const arma::vec& responses,
+ const arma::mat& values) const;
+
+ //!Return the initial point
+ const arma::mat& InitialPoint() const { return initialPoint; }
+ //! Modify the initial point
+ arma::mat& InitialPoint() { return initialPoint; }
+
+ //!Return the lambda
+ const double& Lambda() const { return lambda; }
+ //! Modify the lambda
+ double& Lambda() { return lambda; }
+
+ //functions to optimize by l-bfgs
+ double Evaluate(const arma::mat& values) const
+ {
+ return Evaluate(predictors, responses, values);
+ }
+
+ void Gradient(const arma::mat& values, arma::mat& gradient);
+
+ const arma::mat& GetInitialPoint() const { return initialPoint; }
+
+ //functions to optimize by sgd
+ double Evaluate(const arma::mat& values, const size_t i) const
+ {
+ return Evaluate(values);
+ }
+ void Gradient(const arma::mat& values,
+ const size_t i,
+ arma::mat& gradient)
+ {
+ Gradient(values,gradient);
+ }
+
+ size_t NumFunctions() { return 1; }
+
+ private:
+ arma::mat initialPoint;
+ arma::mat& predictors;
+ arma::vec& responses;
+ double lambda;
+};
+
+}; // namespace regression
+}; // namespace mlpack
+
+#endif // __MLPACK_METHODS_LOGISTIC_REGRESSION_LOGISTIC_REGRESSION_FUNCTION_HPP
Added: mlpack/trunk/src/mlpack/methods/logistic_regression/logistic_regression_function_impl.hpp
==============================================================================
--- (empty file)
+++ mlpack/trunk/src/mlpack/methods/logistic_regression/logistic_regression_function_impl.hpp Fri Nov 8 13:52:11 2013
@@ -0,0 +1,89 @@
+/**
+ * @file logistic_regression_function_impl.hpp
+ * @author Sumedh Ghaisas
+ *
+ * Implementation of hte LogisticRegressionFunction class.
+ */
+#ifndef __MLPACK_METHODS_LOGISTIC_REGRESSION_LOGISTIC_REGRESSION_FUNCTION_IMPL_HPP
+#define __MLPACK_METHODS_LOGISTIC_REGRESSION_LOGISTIC_REGRESSION_FUNCTION_IMPL_HPP
+
+// In case it hasn't been done yet.
+#include "logistic_regression_function.hpp"
+
+namespace mlpack {
+namespace regression {
+
+LogisticRegressionFunction::LogisticRegressionFunction(
+ arma::mat& predictors,
+ arma::vec& responses,
+ const double lambda) :
+ predictors(predictors),
+ responses(responses),
+ lambda(lambda)
+{
+ initialPoint = arma::zeros<arma::mat>(predictors.n_rows + 1, 1);
+}
+
+LogisticRegressionFunction::LogisticRegressionFunction(
+ arma::mat& predictors,
+ arma::vec& responses,
+ const arma::mat& initialPoint,
+ const double lambda) :
+ initialPoint(initialPoint),
+ predictors(predictors),
+ responses(responses),
+ lambda(lambda)
+{
+ //to check if initialPoint is compatible with predictors
+ if(initialPoint.n_rows != (predictors.n_rows + 1) || initialPoint.n_cols != 1)
+ this->initialPoint = arma::zeros<arma::mat>(predictors.n_rows + 1,1);
+}
+
+arma::vec LogisticRegressionFunction::getSigmoid(const arma::vec& values) const
+{
+ arma::vec out = arma::ones<arma::vec>(values.n_rows,1) /
+ (arma::ones<arma::vec>(values.n_rows,1) + arma::exp(-values));
+ return out;
+}
+
+double LogisticRegressionFunction::Evaluate(
+ const arma::mat& predictors,
+ const arma::vec& responses,
+ const arma::mat& values) const
+{
+ size_t nCols = predictors.n_cols;
+
+ //sigmoid = Sigmoid(X' * values)
+ arma::vec sigmoid = getSigmoid(arma::trans(predictors) * values);
+
+ //l2-regularization(considering only values(2:end) in regularization
+ arma::vec temp = arma::trans(values) * values;
+ double regularization = lambda * (temp(0,0) - values(0,0) * values(0,0)) /
+ (2 * responses.n_rows);
+
+ //J = -(sum(y' * log(sigmoid)) + sum((ones(m,1) - y)' * log(ones(m,1)
+ // - sigmoid))) + regularization
+ return -(sum(arma::trans(responses) * arma::log(sigmoid)) +
+ sum(arma::trans(arma::ones<arma::vec>(nCols, 1) - responses) *
+ arma::log(arma::ones<arma::vec>(nCols,1) - sigmoid))) /
+ predictors.n_cols + regularization;
+}
+
+void LogisticRegressionFunction::Gradient(
+ const arma::mat& values,
+ arma::mat& gradient)
+{
+ //regularization
+ arma::mat regularization = arma::zeros<arma::mat>(predictors.n_rows, 1);
+ regularization.rows(1, predictors.n_rows - 1) = lambda *
+ values.rows(1, predictors.n_rows - 1) / responses.n_rows;
+
+ //gradient =
+ gradient = -(predictors * (responses - getSigmoid(arma::trans(predictors) *
+ values))) / responses.n_rows + regularization;
+}
+
+}; // namespace regression
+}; // namespace mlpack
+
+#endif
Added: mlpack/trunk/src/mlpack/methods/logistic_regression/logistic_regression_impl.hpp
==============================================================================
--- (empty file)
+++ mlpack/trunk/src/mlpack/methods/logistic_regression/logistic_regression_impl.hpp Fri Nov 8 13:52:11 2013
@@ -0,0 +1,120 @@
+/**
+ * @file logistic_regression_impl.hpp
+ * @author Sumedh Ghaisas
+ *
+ * Implementation of the LogisticRegression class.
+ */
+#ifndef __MLPACK_METHODS_LOGISTIC_REGRESSION_LOGISTIC_REGRESSION_IMPL_HPP
+#define __MLPACK_METHODS_LOGISTIC_REGRESSION_LOGISTIC_REGRESSION_IMPL_HPP
+
+// In case it hasn't been included yet.
+#include "logistic_regression.hpp"
+
+namespace mlpack {
+namespace regression {
+
+template<template<typename> class OptimizerType>
+LogisticRegression<OptimizerType>::LogisticRegression(
+ arma::mat& predictors,
+ arma::vec& responses,
+ const double lambda) :
+ predictors(predictors),
+ responses(responses),
+ errorFunction(LogisticRegressionFunction(predictors,responses,lambda)),
+ optimizer(OptimizerType<LogisticRegressionFunction>(errorFunction)),
+ lambda(lambda)
+{
+ parameters.zeros(predictors.n_rows + 1);
+}
+
+template<template<typename> class OptimizerType>
+LogisticRegression<OptimizerType>::LogisticRegression(
+ arma::mat& predictors,
+ arma::vec& responses,
+ const arma::mat& initialPoint,
+ const double lambda) :
+ predictors(predictors),
+ responses(responses),
+ errorFunction(LogisticRegressionFunction(predictors,responses)),
+ optimizer(OptimizerType<LogisticRegressionFunction>(errorFunction)),
+ lambda(lambda)
+{
+ parameters.zeros(predictors.n_rows + 1);
+}
+
+template <template<typename> class OptimizerType>
+double LogisticRegression<OptimizerType>::LearnModel()
+{
+ //add rows of ones to predictors
+ arma::rowvec ones;
+ ones.ones(predictors.n_cols);
+ predictors.insert_rows(0, ones);
+
+ double out;
+ Timer::Start("logistic_regression_optimization");
+ out = optimizer.Optimize(parameters);
+ Timer::Stop("logistic_regression_optimization");
+
+ //shed the added rows
+ predictors.shed_row(0);
+
+ return out;
+}
+
+template <template<typename> class OptimizerType>
+double LogisticRegression<OptimizerType>::ComputeError(
+ arma::mat& predictors,
+ const arma::vec& responses)
+{
+ // Here we add the row of ones to the predictors.
+ arma::rowvec ones;
+ ones.ones(predictors.n_cols);
+ predictors.insert_rows(0, ones);
+
+ double out = errorFunction.Evaluate(predictors,responses,parameters);
+
+ predictors.shed_row(0);
+
+ return out;
+}
+
+template <template<typename> class OptimizerType>
+double LogisticRegression<OptimizerType>::ComputeAccuracy(
+ arma::mat& predictors,
+ const arma::vec& responses,
+ const double decisionBoundary)
+{
+ arma::vec temp_responses;
+ Predict(predictors,temp_responses,decisionBoundary);
+ int count = 0;
+ for (size_t i = 0; i < responses.n_rows; i++)
+ if (responses(i, 0) == temp_responses(i, 0))
+ count++;
+
+ return (double) (count * 100) / responses.n_rows;
+}
+
+template <template<typename> class OptimizerType>
+void LogisticRegression<OptimizerType>::Predict(
+ arma::mat& predictors,
+ arma::vec& responses,
+ const double decisionBoundary)
+{
+ //add rows of ones to predictors
+ arma::rowvec ones;
+ ones.ones(predictors.n_cols);
+ predictors.insert_rows(0, ones);
+
+ responses = arma::floor(
+ errorFunction.getSigmoid(arma::trans(predictors) * parameters) -
+ decisionBoundary * arma::ones<arma::vec>(predictors.n_cols, 1)) +
+ arma::ones<arma::vec>(predictors.n_cols);
+
+ //shed the added rows
+ predictors.shed_row(0);
+}
+
+}; // namespace regression
+}; // namespace mlpack
+
+#endif // __MLPACK_METHODS_LOGISTIC_REGRESSION_LOGISTIC_REGRESSION_IMPL_HPP
More information about the mlpack-svn
mailing list