From f9643d8541f639620b6854bb8fa7d93001490979 Mon Sep 17 00:00:00 2001 From: erickson Date: Sun, 11 Apr 2010 14:51:35 +0000 Subject: [PATCH] added option to bypass inserting timing data into the db. removed old db.py module, which was replaced by data.py git-svn-id: svn://svn.open-ils.org/ILS-Contrib/constrictor/trunk@860 6d9bc8c9-1ec2-4278-b937-99fde70a366f --- constrictor.py | 11 ++- constrictor/data.py | 68 ++++++++++++------- constrictor/db.py | 188 --------------------------------------------------- constrictor/utils.py | 3 +- 4 files changed, 52 insertions(+), 218 deletions(-) delete mode 100644 constrictor/db.py diff --git a/constrictor.py b/constrictor.py index 6ad12604e..0b6e60097 100755 --- a/constrictor.py +++ b/constrictor.py @@ -25,6 +25,7 @@ import constrictor.data props = None props_filename = PROPS_FILENAME drone_controller = None +store_data = True def usage(): print ''' @@ -43,6 +44,7 @@ python %s [options] -p port to listen for controller connections on -l listen address for incoming controller connections -x clear the local cache file cache + -n do not store the results in the database ''' % sys.argv[0] sys.exit(0) @@ -51,9 +53,10 @@ python %s [options] def read_args_and_props(): global props global props_filename + global store_data # see if we have any command-line args that override the properties file - ops, args = getopt.getopt(sys.argv[1:], 's:t:i:d:p:l:f:hx') + ops, args = getopt.getopt(sys.argv[1:], 's:t:i:d:p:l:f:hxn') options = dict( (k,v) for k,v in ops ) if options.has_key('-f'): @@ -76,6 +79,8 @@ def read_args_and_props(): props.set_property('constrictor.port', options['-p']) if options.has_key('-l'): props.set_property('constrictor.listenAddress', options['-l']) + if options.has_key('-n'): + store_data = False if options.has_key('-x'): # delete the cache directory @@ -130,7 +135,7 @@ if props.get_property('constrictor.listen') == 'true': log_info('running ' + script) f = open_script(scriptDirs, script) if f: - init_db() + init_db(store_data) try: exec(f) except Exception, e: @@ -143,7 +148,7 @@ if props.get_property('constrictor.listen') == 'true': drone_controller.shutdown() else: - init_db() + init_db(store_data) script = props.get_property('constrictor.script') # execute the requested script ScriptThread.reset_thread_seed() f = open_script(scriptDirs, script) diff --git a/constrictor/data.py b/constrictor/data.py index ec463eca3..946f8fc06 100644 --- a/constrictor/data.py +++ b/constrictor/data.py @@ -29,6 +29,7 @@ class Data(object): self.meta.bind = self.engine self.runtime_data = [] self.task_set_id = None + self.insert_data = True Data.singleton = self def disconnect(self): @@ -67,28 +68,34 @@ class Data(object): def create_task_set(self): + self.task_set_start_time = time.time() - res = self.engine.execute( - self.task_set_table.insert().values(start_time = self.task_set_start_time) - ) - #self.task_set_id = res.inserted_primary_key - self.task_set_id = res.last_inserted_ids()[0] - res.close() + + if self.insert_data: + + res = self.engine.execute( + self.task_set_table.insert().values(start_time = self.task_set_start_time) + ) + #self.task_set_id = res.inserted_primary_key + self.task_set_id = res.last_inserted_ids()[0] + res.close() def store_data(self): end_time = time.time() # close out the task set - self.engine.execute( - self.task_set_table.update( - ).where( - self.task_set_table.c.id == self.task_set_id - ).values( - end_time = end_time - ) - ).close() + if self.insert_data: + self.engine.execute( + self.task_set_table.update( + ).where( + self.task_set_table.c.id == self.task_set_id + ).values( + end_time = end_time + ) + ).close() + task_counts = {} task_times = 0 total_tasks = len(self.runtime_data) counter = 0 @@ -99,29 +106,38 @@ class Data(object): # TODO: Create a batch insert SQL string to insert, since # one row at a time is pretty slow for task in self.runtime_data: - + task_times += task['duration'] log.log_debug("Storing " + task['name']) - self.engine.execute( - self.task_table.insert().values( - task_name = task['name'], - task_set = self.task_set_id, - run_time = task['run_time'], - thread_id = task['thread_id'], - duration = '%0.3f' % task['duration'], - success = task['success'] - ) - ).close() - + if task['name'] not in task_counts: + task_counts[task['name']] = 0; + + task_counts[task['name']] += 1 + + if self.insert_data: + self.engine.execute( + self.task_table.insert().values( + task_name = task['name'], + task_set = self.task_set_id, + run_time = task['run_time'], + thread_id = task['thread_id'], + duration = '%0.3f' % task['duration'], + success = task['success'] + ) + ).close() + counter += 1 if counter %100 == 0: log.log_info("Inserted %d rows" % counter) # log some basic stats task_set_time = end_time - self.task_set_start_time + log.log_info("-"*70) + log.log_info("Task Counts -> %s" % str(task_counts)) log.log_info("Total Tasks -> %d" % len(self.runtime_data)) log.log_info("Total time -> %0.3f seconds" % task_set_time) log.log_info("Average task time -> %0.3f seconds" % (task_times / total_tasks)) log.log_info("Real Time / Total Tasks -> %0.3f seconds/task" % (task_set_time / total_tasks)) + log.log_info("-"*70) diff --git a/constrictor/db.py b/constrictor/db.py deleted file mode 100644 index 058cb7768..000000000 --- a/constrictor/db.py +++ /dev/null @@ -1,188 +0,0 @@ -# ----------------------------------------------------------------------- -# Copyright (C) 2007-2008 King County Library System -# Bill Erickson -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 3 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# ----------------------------------------------------------------------- - -from pysqlite2 import dbapi2 as sqlite -import sys, threading, thread, time -from log import * -from properties import Properties - -# wrap DB access in a semaphore -dbSema = threading.BoundedSemaphore(value=1) - - -class DBConnection(object): - - # the ID of the currently executing task set - taskSetID = 0 - - def __init__(self, dbFile): - self.conn = sqlite.connect(dbFile) - - def createTaskSet(self): - SQL = ''' - insert into task_set (start_time) values (%f) - ''' % time.time() - cur = self.execute(SQL, True) - DBConnection.taskSetID = cur.lastrowid - - def finishTaskSet(self): - SQL = ''' - update task_set set end_time = %f where id = %d - ''' % (time.time(), DBConnection.taskSetID) - self.execute(SQL, True) - - - def createTables(self): - taskSQL = ''' - CREATE TABLE IF NOT EXISTS task ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - task_name TEXT, - task_set INTEGER, - runtime REAL, - runner INTEGER, - thread_id INTEGER, - duration INTEGER, - success INTEGER - ); - ''' - - taskSetSQL = ''' - CREATE TABLE IF NOT EXISTS task_set ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - start_time REAL, - end_time REAL - ); - ''' - - self.execute(taskSQL) - self.execute(taskSetSQL, True) - - - def createTaskSummary(self): - summary = {} - - cur = self.execute( - 'select end_time - start_time from task_set where id = %d' % DBConnection.taskSetID) - row = cur.fetchone() - summary['task_set_duration'] = float(row[0]) or '' - - cur = self.execute( - 'select avg(duration) from task where success = 1 and task_set = %d' % DBConnection.taskSetID) - row = cur.fetchone() - summary['avg_task_duration'] = row[0] or '' - - cur = self.execute( - 'select count(*) from task where success = 1 and task_set = %d' % DBConnection.taskSetID) - row = cur.fetchone() - summary['num_task_success'] = row[0] - - cur = self.execute( - 'select count(*) from task where success = 0 and task_set = %d' % DBConnection.taskSetID) - row = cur.fetchone() - summary['num_task_failed'] = row[0] - - summary['amortized_task_duration'] = '' - if summary['num_task_success'] > 0: - summary['amortized_task_duration'] = \ - float(summary['task_set_duration']) / float(summary['num_task_success']) - - summary['amortized_tasks_per_second'] = '' - if summary['num_task_success'] > 0: - summary['amortized_tasks_per_second'] = 1.0 / float(summary['amortized_task_duration']) - - props = Properties.get_properties() - summary['thread_count'] = props.get_property('constrictor.numThreads') - summary['iteration_count'] = props.get_property('constrictor.numIterations') - - self.makeTaskTypeSummary(summary) - - log_info('created summary %s' % summary) - return summary - - def makeTaskTypeSummary(self, summary): - ''' build a summary string for each task type of ::; ''' - - summary['task_type_summary'] = '' - tasks = self.execute( - 'select distinct(task_name) from task where task_set = %d' % DBConnection.taskSetID) - - for t in tasks: - t = t[0] - - # grab the average duration for this type of task - cur = self.execute( - 'select avg(duration) from task where task_set = %d and task_name = "%s"' % ( - DBConnection.taskSetID, t)) - avg = cur.fetchone()[0] - - # grab the number of times this task was run - cur = self.execute( - 'select count(*) from task where task_set = %d and task_name = "%s"' % ( - DBConnection.taskSetID, t)) - row = cur.fetchone() - count = row[0] - - summary['task_type_summary'] += '%s:%d:%f;' % (t, count, float(avg)) - - - - - def dropTables(self): - SQL = 'drop table task;' - self.execute(SQL, True) - - - def disconnect(self): - if self.conn is not None: - self.conn.close() - - - def execute(self, sql, commit=False): - log_debug('SQL ' + sql) - from script import ScriptThread - cur = None - try: - dbSema.acquire() - cur = self.conn.cursor() - cur.execute(sql) - if commit: - self.conn.commit() - except Exception, e: - sys.stderr.write('DB error: thread = %d : %s\n' % (ScriptThread.get_thread_id(), str(e))) - sys.stderr.flush() - dbSema.release() - sys.exit(1) - - dbSema.release() - return cur - - def insertTask(self, task): - from script import ScriptThread - - SQL = """ - insert into task (task_name, runtime, runner, thread_id, task_set, duration, success) - values ('%s', %f, %d, %d, %d, %f, %d) """ % ( - task.name, - time.time(), - 1, # XXX get me from the task runner ? - ScriptThread.get_thread_id(), - DBConnection.taskSetID, - task.duration, - task.success - ) - self.execute(SQL, True) - - - diff --git a/constrictor/utils.py b/constrictor/utils.py index 54c048d2c..9576ccc98 100755 --- a/constrictor/utils.py +++ b/constrictor/utils.py @@ -48,11 +48,12 @@ def save_props(): print "WARNING: Unable to store properties to file\n%s" % str(e) -def init_db(): +def init_db(store_data = True): ''' connect to the db and make sure the tables exist ''' data = constrictor.data.Data( os.path.join(os.getcwd(), props.get_property('constrictor.dbFile')) ) + data.insert_data = store_data data.create_schema() data.create_task_set() data.disconnect() -- 2.11.0