From 3321256943227cedfd69ff3cf6a458056b6dffc3 Mon Sep 17 00:00:00 2001 From: erickson Date: Tue, 13 Apr 2010 12:39:10 +0000 Subject: [PATCH] plugged in some initial dynamic data fetching (todo: move to eg_data or related); additional python style tweaks; fleshing out more seed data for batch call git-svn-id: svn://svn.open-ils.org/ILS-Contrib/constrictor/trunk@865 6d9bc8c9-1ec2-4278-b937-99fde70a366f --- contrib/evergreen/batch_api.txt | 37 +++++++++++---------------- contrib/evergreen/eg_batch.py | 55 ++++++++++++++++++++++++++++++++--------- contrib/evergreen/eg_data.py | 32 ++++++++++++++---------- 3 files changed, 77 insertions(+), 47 deletions(-) diff --git a/contrib/evergreen/batch_api.txt b/contrib/evergreen/batch_api.txt index 1c1d61a4c..b64ce8e83 100644 --- a/contrib/evergreen/batch_api.txt +++ b/contrib/evergreen/batch_api.txt @@ -1,20 +1,20 @@ 259 open-ils.search.biblio.record.mods_slim.retrieve TITLE_ID 164 open-ils.search.biblio.record.copy_count ORG_ID, TITLE_ID -100 open-ils.actor.user.checked_out.count.authoritative AUTHTOKEN, USER_ID +100 open-ils.actor.user.checked_out.count.authoritative AUTHTOKEN, PATRON_ID 88 open-ils.search.authority.crossref.batch [["subject",DICT_SUBJECT],["subject",DICT_SUBJECT]] -84 open-ils.actor.user.fines.summary.authoritative AUTHTOKEN, USER_ID +84 open-ils.actor.user.fines.summary.authoritative AUTHTOKEN, PATRON_ID 83 open-ils.search.callnumber.retrieve.authoritative CALLNUMBER_ID 64 open-ils.search.biblio.record.copy_count.staff ORG_ID, TITLE_ID -57 open-ils.actor.note.retrieve.all.authoritative AUTHTOKEN, {"patronid":USER_ID} -51 open-ils.actor.user.fleshed.retrieve AUTHTOKEN, USER_ID +57 open-ils.actor.note.retrieve.all.authoritative AUTHTOKEN, {"patronid":PATRON_ID} +51 open-ils.actor.user.fleshed.retrieve AUTHTOKEN, PATRON_ID 46 open-ils.auth.session.retrieve AUTHTOKEN -45 open-ils.actor.patron.settings.retrieve AUTHTOKEN, USER_ID -45 open-ils.actor.user.hold_requests.count.authoritative AUTHTOKEN, USER_ID +45 open-ils.actor.patron.settings.retrieve AUTHTOKEN, PATRON_ID +45 open-ils.actor.user.hold_requests.count.authoritative AUTHTOKEN, PATRON_ID 44 open-ils.search.biblio.mods_from_copy TITLE_ID 42 open-ils.circ.retrieve AUTHTOKEN, CIRC_ID 31 open-ils.search.biblio.multiclass.query {"org_unit":1,"depth":0,"limit":1000,"offset":0,"visibility_limit":3000,"default_class":"keyword"}, DICT_KEYWORD, 1 29 open-ils.circ.money.billable_xact_summary.retrieve.authoritative AUTHTOKEN, XACT_ID -27 open-ils.circ.open_non_cataloged_circulation.user.authoritative AUTHTOKEN, USER_ID +27 open-ils.circ.open_non_cataloged_circulation.user.authoritative AUTHTOKEN, PATRON_ID 24 open-ils.circ.fleshed.retrieve.authoritative CIRC_ID 21 open-ils.actor.user.retrieve_id_by_barcode_or_username AUTHTOKEN, PATRON_BARCODE 18 open-ils.actor.ou_setting.ancestor_default ORG_ID, "opac.barcode_regex" @@ -30,23 +30,18 @@ 11 open-ils.search.spellcheck DICT_KEYWORD 10 open-ils.circ.hold.status.retrieve AUTHTOKEN, HOLD_ID 10 open-ils.search.biblio.record.mods_slim.retrieve.authoritative TITLE_ID -9 open-ils.circ.copy_location.retrieve.authoritative COPY_LOCATION_ID -9 open-ils.actor.container.retrieve_by_class AUTHTOKEN, CONTAINER_ID, "biblio", "bookbag" 8 open-ils.cat.biblio.record.metadata.retrieve.authoritative AUTHTOKEN, [TITLE_ID] 8 open-ils.cat.asset.copy_tree.retrieve.authoritative AUTHTOKEN, TITLE_ID, [ORG_ID] 7 open-ils.circ.copy_details.retrieve.barcode.authoritative AUTHTOKEN, COPY_BARCODE -6 open-ils.actor.org_unit.retrieve_by_shorname ORG_SHORTNAME 6 open-ils.search.biblio.item_type_map.retrieve.all -6 open-ils.actor.org_unit.address.retrieve ORG_ADDRESS_ID 6 open-ils.circ.copy_location.retrieve.all ORG_ID -6 open-ils.actor.user.perm.highest_org.batch AUTHTOKEN, USER_ID, ["HOLD_EXISTS.override"] +6 open-ils.actor.user.perm.highest_org.batch AUTHTOKEN, PATRON_ID, ["HOLD_EXISTS.override"] 5 open-ils.search.biblio.item_form_map.retrieve.all 5 open-ils.search.biblio.lit_form_map.retrieve.all 5 open-ils.search.biblio.audience_map.retrieve.all 5 open-ils.search.biblio.bib_level_map.retrieve.all -5 open-ils.actor.address.members AUTHTOKEN, USER_ADDRESS_ID -5 open-ils.actor.user.checked_out.authoritative AUTHTOKEN, USER_ID -4 open-ils.actor.user.checked_in_with_fines.authoritative AUTHTOKEN, USER_ID +5 open-ils.actor.user.checked_out.authoritative AUTHTOKEN, PATRON_ID +4 open-ils.actor.user.checked_in_with_fines.authoritative AUTHTOKEN, PATRON_ID 4 open-ils.cat.actor.org_unit.retrieve_by_title.authoritative TITLE_ID 4 open-ils.search.biblio.record.copy_count.staff.authoritative ORG_ID, TITLE_ID 4 open-ils.search.asset.copy.fleshed.custom COPY_ID, ["location"] @@ -54,18 +49,16 @@ 4 open-ils.search.callnumber.retrieve.authoritative CALLNUMBER_ID 3 open-ils.search.metabib.record_to_descriptors {"record":TITLE_ID} 3 open-ils.circ.open_non_cataloged_circulation.user AUTHTOKEN -3 open-ils.actor.user.checked_out AUTHTOKEN, USER_ID -3 open-ils.actor.note.retrieve.all AUTHTOKEN, {"pub":1,"patronid":USER_ID} -2 open-ils.search.biblio.metarecord_to_records.staff METARECORD_ID, {"format":null,"org":ORG_ID,"depth":0} -2 open-ils.actor.user.perm.check.multi_org AUTHTOKEN, USER_ID, [ORG_ID], ["CREATE_VOLUME","CREATE_COPY"] +3 open-ils.actor.user.checked_out AUTHTOKEN, PATRON_ID +3 open-ils.actor.note.retrieve.all AUTHTOKEN, {"pub":1,"patronid":PATRON_ID} +2 open-ils.actor.user.perm.check.multi_org AUTHTOKEN, PATRON_ID, [ORG_ID], ["CREATE_VOLUME","CREATE_COPY"] 2 open-ils.circ.copy_details.retrieve.barcode.authoritative AUTHTOKEN, COPY_BARCODE 2 open-ils.search.asset.copy.fleshed2.find_by_barcode COPY_BARCODE -2 open-ils.actor.user.penalties.update AUTHTOKEN, USER_ID -2 open-ils.circ.holds.id_list.retrieve.authoritative AUTHTOKEN, USER_ID +2 open-ils.actor.user.penalties.update AUTHTOKEN, PATRON_ID +2 open-ils.circ.holds.id_list.retrieve.authoritative AUTHTOKEN, PATRON_ID 2 open-ils.circ.circulation.count COPY_ID 1 open-ils.circ.non_cat_types.retrieve.all ORG_ID 1 open-ils.circ.holds.retrieve AUTHTOKEN, HOLD_ID -1 open-ils.actor.container.flesh AUTHTOKEN, "biblio", CONTAINER_ID 1 open-ils.circ.copy_checkout_history.retrieve AUTHTOKEN, COPY_ID, 1 1 open-ils.actor.user.fleshed.retrieve_by_barcode.authoritative AUTHTOKEN, PATRON_BARCODE 1 open-ils.circ.money.payment.retrieve.all.authoritative AUTHTOKEN, XACT_ID diff --git a/contrib/evergreen/eg_batch.py b/contrib/evergreen/eg_batch.py index 143b548fe..ac538f8ee 100755 --- a/contrib/evergreen/eg_batch.py +++ b/contrib/evergreen/eg_batch.py @@ -30,16 +30,50 @@ class BatchScript(Script): Script.__init__(self) self.calls = [] self.loaded = False - + self.data_mgr = None + self.bibs = [] + self.copies = [] + self.call_numbers = [] + self.org_unis = [] + + + def fetch_dynamic_data(self): + pcrud = 'open-ils.pcrud' + method = 'open-ils.pcrud.search.%s.atomic' + + auth = eg_utils.authtoken() + thread_count = props.get_property('constrictor.numThreads') + + self.bibs = BatchMethodTask(pcrud, method % 'bre').start( + params = [auth, {'deleted' : 'f', 'id' : {'>' : 0}}, {'limit' : thread_count}]) + + self.copies = BatchMethodTask(pcrud, method % 'acp').start( + params = [auth, {'deleted' : 'f', 'id' : {'>' : 0}}, {'limit' : thread_count}]) + + self.call_numbers = BatchMethodTask(pcrud, method % 'acn').start( + params = [auth, {'deleted' : 'f', 'id' : {'>' : 0}}, {'limit' : thread_count}]) + + self.org_units = BatchMethodTask(pcrud, method % 'aou').start( + params = [auth, {'id' : {'>' : 0}}, {'limit' : thread_count}]) + + self.data_mgr.insert_prop_data(eg_data.PROP_TITLE_ID, [obj.id() for obj in self.bibs]) + self.data_mgr.insert_prop_data(eg_data.PROP_COPY_ID, [obj.id() for obj in self.copies]) + self.data_mgr.insert_prop_data(eg_data.PROP_COPY_BARCODE, [obj.id() for obj in self.copies]) + self.data_mgr.insert_prop_data(eg_data.PROP_CALLNUMBER_ID, [obj.id() for obj in self.call_numbers]) + self.data_mgr.insert_prop_data(eg_data.PROP_ORG_ID, [obj.id() for obj in self.org_units]) + self.data_mgr.insert_prop_data(eg_data.PROP_ORG_SHORTNAME, [obj.shortname() for obj in self.org_units]) + def load_file(self): batch_file = props.get_property('evergreen.batchAPIFile') log.log_info("loading " + batch_file) - file = open(batch_file) - dm = eg_data.DataManager() + self.data_mgr = eg_data.DataManager() + self.fetch_dynamic_data() + file = open(batch_file) for line in file.readlines(): line = line[0:len(line)-1] # chomp + parts = line.split(None, 2) weight = parts[0] method = parts[1] @@ -65,25 +99,22 @@ class BatchScript(Script): for prop in [ 'COPY_BARCODE', 'PATRON_BARCODE', - 'CONTAINER_ID', 'CIRC_ID', 'COPY_ID', 'HOLD_ID', 'ORG_ID', - 'USER_ID', + 'PATRON_ID', 'TITLE_ID', - 'USER_ADDRESS_ID', 'XACT_ID', - 'CALLNUMBER_ID', - 'COPY_LOCATION_ID', - 'METARECORD_ID', - 'ORG_ADDRESS_ID', - 'ORG_SHORTNAME' ]: + 'CALLNUMBER_ID' ]: value = 1 prop_name = 'PROP_%s' % prop if hasattr(eg_data, prop_name): - value = dm.get_thread_data(getattr(eg_data, prop_name)) + value = self.data_mgr.get_thread_data(getattr(eg_data, prop_name)) + else: + #log.log_error("PROP NEEDED: %s" % prop_name) + pass params = params.replace(prop, '"%s"' % value) diff --git a/contrib/evergreen/eg_data.py b/contrib/evergreen/eg_data.py index bc1153b30..e754485df 100644 --- a/contrib/evergreen/eg_data.py +++ b/contrib/evergreen/eg_data.py @@ -6,11 +6,14 @@ from oils.utils.utils import unique PROP_USERNAME = 'evergreen.username' PROP_PASSWORD = 'evergreen.password' PROP_WORKSTATION = 'evergreen.workstation' -PROP_COPY_BARCODE = 'evergreen.copyBarcodes' -PROP_TITLE_ID = 'evergreen.titleIDs' PROP_PATRON_BARCODE = 'evergreen.patronBarcodes' PROP_ORG_ID = 'evergreen.orgIDs' +PROP_ORG_SHORTNAME = 'evergreen.orgShortnames' +PROP_TITLE_ID = 'evergreen.titleIDs' PROP_PATRON_ID = 'evergreen.patronIDs' +PROP_COPY_ID = 'evergreen.copyIds' +PROP_COPY_BARCODE = 'evergreen.copyBarcodes' +PROP_CALLNUMBER_ID = 'evergreen.callnumberIds' PROP_CONSTRICTOR_THREADS = 'constrictor.numThreads' @@ -23,7 +26,7 @@ class DataManager(object): def __init__(self): self.data = {} self.props = Properties.get_properties() - self.readProps() + self.read_props() log_debug(self) def __str__(self): @@ -41,23 +44,26 @@ class DataManager(object): return s - def readProps(self): - self.readProp(PROP_USERNAME) - self.readProp(PROP_PASSWORD) - self.readProp(PROP_WORKSTATION) - self.readProp(PROP_COPY_BARCODE, True) - self.readProp(PROP_TITLE_ID, True) - self.readProp(PROP_PATRON_BARCODE, True) - self.readProp(PROP_PATRON_ID, True) - self.readProp(PROP_ORG_ID, True) + def read_props(self): + self.read_prop(PROP_USERNAME) + self.read_prop(PROP_PASSWORD) + self.read_prop(PROP_WORKSTATION) + self.read_prop(PROP_COPY_BARCODE, True) + self.read_prop(PROP_TITLE_ID, True) + self.read_prop(PROP_PATRON_BARCODE, True) + self.read_prop(PROP_PATRON_ID, True) + self.read_prop(PROP_ORG_ID, True) - def readProp(self, prop, split=False): + def read_prop(self, prop, split=False): v = self.props.get_property(prop) if split and v: v = unique(v.split(',')) self.data[prop] = v log_debug("DataManager set property %s => %s" % (prop, str(v))) + def insert_prop_data(self, prop, array): + self.data[prop] = array + def get_thread_data(self, prop, noSharing=False): ''' If the caller is requesting array-based data, we want to de-multiplex(?) the requested data based on the currently running thread. In other -- 2.11.0