From 8ccc1583872076700cda1944d22252985fe2ec5d Mon Sep 17 00:00:00 2001 From: erickson Date: Thu, 8 Apr 2010 18:04:09 +0000 Subject: [PATCH] some more expressive logging during data inserts git-svn-id: svn://svn.open-ils.org/ILS-Contrib/constrictor/trunk@855 6d9bc8c9-1ec2-4278-b937-99fde70a366f --- constrictor/data.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/constrictor/data.py b/constrictor/data.py index ea55007d0..ec463eca3 100644 --- a/constrictor/data.py +++ b/constrictor/data.py @@ -78,7 +78,6 @@ class Data(object): def store_data(self): end_time = time.time() - log.log_info("Inserting data into data store...") # close out the task set self.engine.execute( @@ -91,11 +90,19 @@ class Data(object): ).close() task_times = 0 + total_tasks = len(self.runtime_data) + counter = 0 + + log.log_info("Inserting %d rows of data into data store..." % total_tasks) # insert all of the task data + # TODO: Create a batch insert SQL string to insert, since + # one row at a time is pretty slow for task in self.runtime_data: + task_times += task['duration'] log.log_debug("Storing " + task['name']) + self.engine.execute( self.task_table.insert().values( task_name = task['name'], @@ -107,10 +114,14 @@ class Data(object): ) ).close() + counter += 1 + if counter %100 == 0: + log.log_info("Inserted %d rows" % counter) + # log some basic stats task_set_time = end_time - self.task_set_start_time log.log_info("Total Tasks -> %d" % len(self.runtime_data)) log.log_info("Total time -> %0.3f seconds" % task_set_time) - log.log_info("Average task time -> %0.3f seconds" % (task_times / len(self.runtime_data))) - log.log_info("Time / Total Tasks -> %0.3f seconds/task" % (task_set_time / len(self.runtime_data))) + log.log_info("Average task time -> %0.3f seconds" % (task_times / total_tasks)) + log.log_info("Real Time / Total Tasks -> %0.3f seconds/task" % (task_set_time / total_tasks)) -- 2.11.0