[mlpack-svn] r15781 - in mlpack/conf/jenkins-conf/benchmark: . benchmark util
fastlab-svn at coffeetalk-1.cc.gatech.edu
fastlab-svn at coffeetalk-1.cc.gatech.edu
Sat Sep 14 07:47:01 EDT 2013
Author: marcus
Date: Sat Sep 14 07:47:00 2013
New Revision: 15781
Log:
Add option to update records in the database.
Modified:
mlpack/conf/jenkins-conf/benchmark/Makefile
mlpack/conf/jenkins-conf/benchmark/benchmark/run_benchmark.py
mlpack/conf/jenkins-conf/benchmark/util/database.py
Modified: mlpack/conf/jenkins-conf/benchmark/Makefile
==============================================================================
--- mlpack/conf/jenkins-conf/benchmark/Makefile (original)
+++ mlpack/conf/jenkins-conf/benchmark/Makefile Sat Sep 14 07:47:00 2013
@@ -25,6 +25,8 @@
BENCHMARKDDIR := benchmark
LOG:=False
BLOCK:=""
+METHODBLOCK:=""
+UPDATE:=False
# Specify the path for the libraries.
export MLPACK_BIN=/usr/local/bin/
@@ -89,7 +91,7 @@
$(PYTHON_BIN) $(BENCHMARKDDIR)/test_config.py -c $(CONFIG)
.run:
- $(PYTHON_BIN) $(BENCHMARKDDIR)/run_benchmark.py -c $(CONFIG) -b $(BLOCK) -l $(LOG)
+ $(PYTHON_BIN) $(BENCHMARKDDIR)/run_benchmark.py -c $(CONFIG) -b $(BLOCK) -l $(LOG) -u $(UPDATE) -m $(METHODBLOCK)
.memory:
$(PYTHON_BIN) $(BENCHMARKDDIR)/memory_benchmark.py -c $(CONFIG) -b $(BLOCK) -l $(LOG)
Modified: mlpack/conf/jenkins-conf/benchmark/benchmark/run_benchmark.py
==============================================================================
--- mlpack/conf/jenkins-conf/benchmark/benchmark/run_benchmark.py (original)
+++ mlpack/conf/jenkins-conf/benchmark/benchmark/run_benchmark.py Sat Sep 14 07:47:00 2013
@@ -100,8 +100,10 @@
@param configfile - Start the benchmark with this configuration file.
@param blocks - Run only the specified blocks.
@param log - If True save the reports otherwise use stdout and print the reports.
+ at param methodBlocks - Run only the specified methods.
+ at param update - Update the records in the database.
'''
-def Main(configfile, blocks, log):
+def Main(configfile, blocks, log, methodBlocks, update):
# Benchmark settings.
timeout = 23000
database = "reports/benchmark.db"
@@ -137,131 +139,144 @@
for method, sets in streamData.items():
if method == "general":
continue
- Log.Info("Method: " + method)
- for options, libraries in sets.items():
- Log.Info('Options: ' + (options if options != '' else 'None'))
-
- if log:
- methodId = db.GetMethod(method, options)
- methodId = methodId[0][0] if methodId else db.NewMethod(method, options)
-
- # Create the Table.
- table = []
- header = ['']
- table.append(header)
-
- # Count the Datasets.
- datasetCount = CountLibrariesDatasets(libraries)
-
- # Create the matrix which contains the time and dataset informations.
- dataMatrix = [['-' for x in range(len(libraries) + 1)] for x in
- range(datasetCount)]
-
- col = 1
- run = 0
- for libary in libraries:
- name = libary[0]
- datsets = libary[1]
- trials = libary[2]
- script = libary[3]
- format = libary[4]
-
- header.append(name)
-
- if not blocks or name in blocks:
- run += 1
- Log.Info("Libary: " + name)
-
- # Logging: create a new build and library record for this library.
- if log and name not in build:
- libaryId = db.GetLibrary(name)
- libaryId = libaryId[0][0] if libaryId else db.NewLibrary(name)
-
- build[name] = (db.NewBuild(libaryId), libaryId)
-
- # Load script.
- try:
- module = Loader.ImportModuleFromPath(script)
- methodCall = getattr(module, method)
- except Exception as e:
- Log.Fatal("Could not load the script: " + script)
- Log.Fatal("Exception: " + str(e))
- else:
-
- for dataset in datsets:
- datasetName = NormalizeDatasetName(dataset)
- row = FindRightRow(dataMatrix, datasetName, datasetCount)
-
- # Logging: Create a new dataset record fot this dataset.
- if log:
- datasetId = db.GetDataset(datasetName)
- datasetId = datasetId[0][0] if datasetId else db.NewDataset(*DatasetInfo(dataset))
-
- dataMatrix[row][0] = datasetName
- Log.Info("Dataset: " + dataMatrix[row][0])
-
- modifiedDataset = GetDataset(dataset, format)
-
- try:
- instance = methodCall(modifiedDataset[0], timeout=timeout,
- verbose=False)
- except Exception as e:
- Log.Fatal("Could not call the constructor: " + script)
- Log.Fatal("Exception: " + str(e))
- continue
+ if not methodBlocks or method in methodBlocks:
+ Log.Info("Method: " + method)
+ for options, libraries in sets.items():
+ Log.Info('Options: ' + (options if options != '' else 'None'))
+
+ if log:
+ methodId = db.GetMethod(method, options)
+ methodId = methodId[0][0] if methodId else db.NewMethod(method, options)
+
+ # Create the Table.
+ table = []
+ header = ['']
+ table.append(header)
+
+ # Count the Datasets.
+ datasetCount = CountLibrariesDatasets(libraries)
+
+ # Create the matrix which contains the time and dataset informations.
+ dataMatrix = [['-' for x in range(len(libraries) + 1)] for x in
+ range(datasetCount)]
- # Add method information record.
- if log:
- try:
- methodDescription = instance.description
- except AttributeError:
- pass
+ col = 1
+ run = 0
+ for libary in libraries:
+ name = libary[0]
+ datsets = libary[1]
+ trials = libary[2]
+ script = libary[3]
+ format = libary[4]
+
+ header.append(name)
+
+ if not blocks or name in blocks:
+ run += 1
+ Log.Info("Libary: " + name)
+
+ # Logging: create a new build and library record for this library.
+ if log and name not in build:
+ libaryId = db.GetLibrary(name)
+ libaryId = libaryId[0][0] if libaryId else db.NewLibrary(name)
+
+ if update:
+ buildId = db.GetLatestBuildFromLibary(libaryId)
+ if buildId >= 0:
+ build[name] = (buildId, libaryId)
else:
- if methodDescription and not db.GetMethodInfo(methodId):
- db.NewMethodInfo(methodId, methodDescription)
+ Log.Warn("Nothing to update.")
+ continue
+ else:
+ build[name] = (db.NewBuild(libaryId), libaryId)
- time = []
- for trial in range(trials + 1):
- if trial > 0:
- try:
- time.append(instance.RunMethod(options));
+ # Load script.
+ try:
+ module = Loader.ImportModuleFromPath(script)
+ methodCall = getattr(module, method)
+ except Exception as e:
+ Log.Fatal("Could not load the script: " + script)
+ Log.Fatal("Exception: " + str(e))
+ else:
+
+ for dataset in datsets:
+ datasetName = NormalizeDatasetName(dataset)
+ row = FindRightRow(dataMatrix, datasetName, datasetCount)
+
+ # Logging: Create a new dataset record fot this dataset.
+ if log:
+ datasetId = db.GetDataset(datasetName)
+ datasetId = datasetId[0][0] if datasetId else db.NewDataset(*DatasetInfo(dataset))
- # Method unsuccessful.
- if sum(time) < 0:
- break
- except Exception as e:
- Log.Fatal("Exception: " + str(e))
-
- # Set time.
- if sum(time) == -2:
- dataMatrix[row][col] = ">" + str(timeout)
- elif sum(time) == -1:
- dataMatrix[row][col] = "failure"
- else:
- dataMatrix[row][col] = "{0:.6f}".format(sum(time) / trials)
+ dataMatrix[row][0] = datasetName
+ Log.Info("Dataset: " + dataMatrix[row][0])
- # Save results in the databse if the user asked for.
- if log:
- # Get the variance.
- var = 0
- if len(time) != 0:
- avg = sum(time) / len(time)
- var = sum((avg - value) ** 2 for value in time) / len(time)
-
- buildId, libaryId = build[name]
- db.NewResult(buildId, libaryId, dataMatrix[row][col], var,
- datasetId, methodId)
-
- # Remove temporary datasets.
- RemoveDataset(modifiedDataset[1])
- col += 1
-
- # Show results in a table.
- if not log and run > 0:
- Log.Notice("\n\n")
- Log.PrintTable(AddMatrixToTable(dataMatrix, table))
- Log.Notice("\n\n")
- run = 0
+ modifiedDataset = GetDataset(dataset, format)
+
+ try:
+ instance = methodCall(modifiedDataset[0], timeout=timeout,
+ verbose=False)
+ except Exception as e:
+ Log.Fatal("Could not call the constructor: " + script)
+ Log.Fatal("Exception: " + str(e))
+ continue
+
+ # Add method information record.
+ if log:
+ try:
+ methodDescription = instance.description
+ except AttributeError:
+ pass
+ else:
+ if methodDescription and not db.GetMethodInfo(methodId):
+ db.NewMethodInfo(methodId, methodDescription)
+
+ time = []
+ for trial in range(trials + 1):
+ if trial > 0:
+ try:
+ time.append(instance.RunMethod(options));
+
+ # Method unsuccessful.
+ if sum(time) < 0:
+ break
+ except Exception as e:
+ Log.Fatal("Exception: " + str(e))
+
+ # Set time.
+ if sum(time) == -2:
+ dataMatrix[row][col] = ">" + str(timeout)
+ elif sum(time) == -1:
+ dataMatrix[row][col] = "failure"
+ else:
+ dataMatrix[row][col] = "{0:.6f}".format(sum(time) / trials)
+
+ # Save results in the databse if the user asked for.
+ if log:
+ # Get the variance.
+ var = 0
+ if len(time) != 0:
+ avg = sum(time) / len(time)
+ var = sum((avg - value) ** 2 for value in time) / len(time)
+
+ buildId, libaryId = build[name]
+ if update:
+ db.UpdateResult(buildId, libaryId, dataMatrix[row][col],
+ var, datasetId, methodId)
+ else:
+ db.NewResult(buildId, libaryId, dataMatrix[row][col], var,
+ datasetId, methodId)
+
+ # Remove temporary datasets.
+ RemoveDataset(modifiedDataset[1])
+ col += 1
+
+ # Show results in a table.
+ if not log and run > 0:
+ Log.Notice("\n\n")
+ Log.PrintTable(AddMatrixToTable(dataMatrix, table))
+ Log.Notice("\n\n")
+ run = 0
if __name__ == '__main__':
parser = argparse.ArgumentParser(description="""Perform the benchmark with the
@@ -272,10 +287,15 @@
required=False)
parser.add_argument('-l','--log', help='Save the results in the logfile.',
required=False)
+ parser.add_argument('-u','--update', help="""Update the results in the
+ database.""", required=False)
+ parser.add_argument('-m','--methodBlocks', help="""Run only the specified
+ method blocks.""", required=False)
args = parser.parse_args()
if args:
SystemInformation()
log = True if args.log == "True" else False
- Main(args.config, args.blocks, log)
+ update = True if args.update == "True" else False
+ Main(args.config, args.blocks, log, args.methodBlocks, update)
Modified: mlpack/conf/jenkins-conf/benchmark/util/database.py
==============================================================================
--- mlpack/conf/jenkins-conf/benchmark/util/database.py (original)
+++ mlpack/conf/jenkins-conf/benchmark/util/database.py Sat Sep 14 07:47:00 2013
@@ -236,6 +236,23 @@
(buildId, libaryId, time, var, datasetId, methodId))
'''
+ Update the given result record in the results table.
+
+ @param buildId - The id of the build.
+ @param libaryId - The if ot the library.
+ @param time - The mesured time of the build.
+ @param var - The variance of the build.
+ @param datasetId - The id of the dataset.
+ @param methodId - The id of the method.
+ '''
+ def UpdateResult(self, buildId, libaryId, time, var, datasetId, methodId):
+ with self.con:
+ self.cur.execute("UPDATE results SET time=" + str(time) + ",var="
+ + str(var) + " WHERE build_id=" + str(buildId) + " AND libary_id="
+ + str(libaryId) + " AND dataset_id=" + str(datasetId)
+ + " AND method_id=" + str(methodId))
+
+ '''
Get the method id from the methods table with the given name and parameters.
@param name - The name of the method.
@@ -294,13 +311,17 @@
Get the latest build id for the specified libary id.
@param libaryId - Get the build id for the libary id.
- @param The latest build id.
+ @param The latest build id if there is a latest build otherwise -1.
'''
def GetLatestBuildFromLibary(self, libaryId):
with self.con:
self.cur.execute("SELECT id FROM builds WHERE libary_id=" + str(libaryId)
+ " ORDER BY build DESC LIMIT 1")
- return self.cur.fetchall()[0][0]
+ res = self.cur.fetchall()
+ if res:
+ return res[0][0]
+ else:
+ return -1
'''
Get a list of all methods.
More information about the mlpack-svn
mailing list