From: erickson Date: Sun, 11 Apr 2010 19:33:44 +0000 (+0000) Subject: added a batch file processor. the batch file is a set of API calls, with param macro... X-Git-Url: https://old-git.evergreen-ils.org/?a=commitdiff_plain;h=60d7b6275c6a42fb75e2986edc8eaee2694ec086;p=working%2Frandom.git added a batch file processor. the batch file is a set of API calls, with param macros, and a weight which defines how many times the call should be made relative to other calls git-svn-id: svn://svn.open-ils.org/ILS-Contrib/constrictor/trunk@862 6d9bc8c9-1ec2-4278-b937-99fde70a366f --- diff --git a/constrictor.properties b/constrictor.properties index 4bb1298be..a93c497bd 100644 --- a/constrictor.properties +++ b/constrictor.properties @@ -62,6 +62,7 @@ evergreen.username=demo evergreen.password=demo123 evergreen.workstation=demo evergreen.dictionary=/usr/share/dict/words +evergreen.batchAPIFile=contrib/evergreen/batch_api.txt #evergreen.titleIDs= #evergreen.patronIDs= #evergreen.orgIDs= diff --git a/contrib/evergreen/batch_api.txt b/contrib/evergreen/batch_api.txt new file mode 100644 index 000000000..bfa14faf0 --- /dev/null +++ b/contrib/evergreen/batch_api.txt @@ -0,0 +1,75 @@ +259 open-ils.search.biblio.record.mods_slim.retrieve TITLE_ID +164 open-ils.search.biblio.record.copy_count ORG_ID, TITLE_ID +100 open-ils.actor.user.checked_out.count.authoritative AUTHTOKEN, USER_ID +88 open-ils.search.authority.crossref.batch [["subject",DICT_SUBJECT],["subject",DICT_SUBJECT]] +84 open-ils.actor.user.fines.summary.authoritative AUTHTOKEN, USER_ID +83 open-ils.search.callnumber.retrieve.authoritative CALLNUMBER_ID +64 open-ils.search.biblio.record.copy_count.staff ORG_ID, TITLE_ID +57 open-ils.actor.note.retrieve.all.authoritative AUTHTOKEN, {"patronid":USER_ID} +51 open-ils.actor.user.fleshed.retrieve AUTHTOKEN, USER_ID +46 open-ils.auth.session.retrieve AUTHTOKEN +45 open-ils.actor.patron.settings.retrieve AUTHTOKEN, USER_ID +45 open-ils.actor.user.hold_requests.count.authoritative AUTHTOKEN, USER_ID +44 open-ils.search.biblio.mods_from_copy TITLE_ID +42 open-ils.circ.retrieve AUTHTOKEN, CIRC_ID +31 open-ils.search.biblio.multiclass.query {"org_unit":1,"depth":0,"limit":1000,"offset":0,"visibility_limit":3000,"default_class":"keyword"}, DICT_KEYWORD, 1 +29 open-ils.circ.money.billable_xact_summary.retrieve.authoritative AUTHTOKEN, XACT_ID +27 open-ils.circ.open_non_cataloged_circulation.user.authoritative AUTHTOKEN, USER_ID +24 open-ils.circ.fleshed.retrieve.authoritative CIRC_ID +21 open-ils.actor.user.retrieve_id_by_barcode_or_username AUTHTOKEN, USER_BARCODE +18 open-ils.actor.ou_setting.ancestor_default ORG_ID, "opac.barcode_regex" +17 open-ils.actor.user.fleshed.retrieve_by_barcode.authoritative AUTHTOKEN, USER_BARCODE +17 open-ils.search.asset.copy.fleshed2.retrieve 5379864 +15 open-ils.search.biblio.copy_counts.summary.retrieve TITLE_ID, 1, 0 +15 open-ils.search.biblio.multiclass.query.staff {"org_unit":1,"depth":0,"limit":20,"offset":0,"visibility_limit":3000,"default_class":"keyword"}, DICT_KEYWORD +14 open-ils.search.config.copy_status.retrieve.all +14 open-ils.actor.user.retrieve_id_by_barcode_or_username AUTHTOKEN, USER_BARCODE +13 open-ils.search.biblio.record_entry.slim.retrieve [TITLE_ID] +12 open-ils.circ.hold.details.retrieve.authoritative AUTHTOKEN, HOLD_ID +11 open-ils.search.authority.crossref.batch [["subject",DICT_SUBJECT],["subject",DICT_SUBJECT]] +11 open-ils.search.spellcheck DICT_KEYWORD +10 open-ils.circ.hold.status.retrieve AUTHTOKEN, HOLD_ID +10 open-ils.search.biblio.record.mods_slim.retrieve.authoritative TITLE_ID +9 open-ils.circ.copy_location.retrieve.authoritative COPY_LOCATION_ID +9 open-ils.actor.container.retrieve_by_class AUTHTOKEN, CONTAINER_ID, "biblio", "bookbag" +8 open-ils.cat.biblio.record.metadata.retrieve.authoritative AUTHTOKEN, [TITLE_ID] +8 open-ils.cat.asset.copy_tree.retrieve.authoritative AUTHTOKEN, TITLE_ID, [ORG_ID] +7 open-ils.circ.copy_details.retrieve.barcode.authoritative AUTHTOKEN, COPY_BARCODE +6 open-ils.actor.org_unit.retrieve_by_shorname ORG_SHORTNAME +6 open-ils.search.biblio.item_type_map.retrieve.all +6 open-ils.actor.org_unit.address.retrieve ORG_ADDRESS_ID +6 open-ils.circ.copy_location.retrieve.all ORG_ID +6 open-ils.actor.user.perm.highest_org.batch AUTHTOKEN, USER_ID, ["HOLD_EXISTS.override"] +5 open-ils.search.biblio.item_form_map.retrieve.all +5 open-ils.search.biblio.lit_form_map.retrieve.all +5 open-ils.search.biblio.audience_map.retrieve.all +5 open-ils.search.biblio.bib_level_map.retrieve.all +5 open-ils.actor.address.members AUTHTOKEN, USER_ADDRESS_ID +5 open-ils.actor.user.checked_out.authoritative AUTHTOKEN, USER_ID +4 open-ils.actor.user.checked_in_with_fines.authoritative AUTHTOKEN, USER_ID +4 open-ils.cat.actor.org_unit.retrieve_by_title.authoritative TITLE_ID +4 open-ils.search.biblio.record.copy_count.staff.authoritative ORG_ID, TITLE_ID +4 open-ils.search.asset.copy.fleshed.custom COPY_ID, ["location"] +4 open-ils.actor.patron.search.advanced AUTHTOKEN, {"family_name":{"value":DICT_USER,"group":0},"first_given_name":{"value":DICT_USER,"group":0}}, 51, ["family_name ASC","first_given_name ASC","second_given_name ASC","dob DESC"], 0, "0" +4 open-ils.search.callnumber.retrieve.authoritative CALLNUMBER_ID +3 open-ils.search.metabib.record_to_descriptors {"record":TITLE_ID} +3 open-ils.circ.open_non_cataloged_circulation.user AUTHTOKEN +3 open-ils.actor.user.checked_out AUTHTOKEN, USER_ID +3 open-ils.actor.note.retrieve.all AUTHTOKEN, {"pub":1,"patronid":USER_ID} +2 open-ils.search.biblio.metarecord_to_records.staff METARECORD_ID, {"format":null,"org":ORG_ID,"depth":0} +2 open-ils.actor.user.perm.check.multi_org AUTHTOKEN, USER_ID, [ORG_ID], ["CREATE_VOLUME","CREATE_COPY"] +2 open-ils.circ.copy_details.retrieve.barcode.authoritative AUTHTOKEN, COPY_BARCODE +2 open-ils.search.asset.copy.fleshed2.find_by_barcode COPY_BARCODE +2 open-ils.actor.user.penalties.update AUTHTOKEN, USER_ID +2 open-ils.circ.holds.id_list.retrieve.authoritative AUTHTOKEN, USER_ID +2 open-ils.circ.circulation.count COPY_ID +1 open-ils.circ.non_cat_types.retrieve.all ORG_ID +1 open-ils.circ.holds.retrieve AUTHTOKEN, HOLD_ID +1 open-ils.actor.container.flesh AUTHTOKEN, "biblio", CONTAINER_ID +1 open-ils.circ.copy_checkout_history.retrieve AUTHTOKEN, COPY_ID, 1 +1 open-ils.actor.user.fleshed.retrieve_by_barcode.authoritative AUTHTOKEN, USER_BARCODE +1 open-ils.circ.money.payment.retrieve.all.authoritative AUTHTOKEN, XACT_ID +1 open-ils.circ.transit.retrieve_by_lib AUTHTOKEN, ORG_ID, "1969-12-31T19:00:00-05:00" +1 open-ils.search.biblio.isbn DICT_ISBN +1 open-ils.actor.barcode.exists AUTHTOKEN, USER_BARCODE +1 open-ils.cat.biblio.record.marc_cn.retrieve TITLE_ID diff --git a/contrib/evergreen/eg_batch.py b/contrib/evergreen/eg_batch.py new file mode 100755 index 000000000..ac4a0f495 --- /dev/null +++ b/contrib/evergreen/eg_batch.py @@ -0,0 +1,114 @@ +#!/usr/bin/python +import random, traceback, os +from constrictor.script import Script, ScriptThread +from constrictor.properties import Properties +import constrictor.log as log +import eg_utils +import eg_data +import eg_workflow +import eg_tasks +import osrf.json + +eg_utils.init() +props = Properties.get_properties() + + +class BatchMethodTask(eg_tasks.AbstractMethodTask): + def __init__(self, service, method): + eg_tasks.AbstractMethodTask.__init__(self) + self.service = service + self.method = method + + def run(self, **kw): + log.log_info("Batch call: %s %s" % (self.method, osrf.json.to_json(kw['params']))) + return self.runMethod(*kw['params']) + + +class BatchScript(Script): + + def __init__(self): + Script.__init__(self) + self.calls = [] + self.loaded = False + + def load_file(self): + + batch_file = props.get_property('evergreen.batchAPIFile') + log.log_info("loading " + batch_file) + file = open(batch_file) + dm = eg_data.DataManager() + + for line in file.readlines(): + line = line[0:len(line)-1] # chomp + parts = line.split(None, 2) + weight = parts[0] + method = parts[1] + params = '[]' + if len(parts) > 2: + params = "[%s]" % parts[2] + + params = params.replace('AUTHTOKEN', '"%s"' % eg_utils.authtoken()) + + trying = True + while trying: + try: # get past some encoding oddities in the dict files + params = params.replace('DICT_KEYWORD', '"%s"' % eg_utils.random_phrase(1)) + params = params.replace('DICT_SUBJECT', '"%s"' % eg_utils.random_phrase(1)) + params = params.replace('DICT_USER', '"%s"' % eg_utils.random_phrase(1)) + params = params.replace('DICT_ISBN', '"%s"' % eg_utils.random_phrase(1)) + params = params.replace('DICT_KEYWORD', '"%s"' % eg_utils.random_phrase(1)) + except: + continue + trying = False + + # TODO: fill in props values for all of these, pre-collecting data as necessary + for prop in [ + 'COPY_BARCODE', + 'USER_BARCODE', + 'CONTAINER_ID', + 'CIRC_ID', + 'COPY_ID', + 'HOLD_ID', + 'ORG_ID', + 'USER_ID', + 'TITLE_ID', + 'USER_ADDRESS_ID', + 'XACT_ID', + 'CALLNUMBER_ID', + 'COPY_LOCATION_ID', + 'METARECORD_ID', + 'ORG_ADDRESS_ID', + 'ORG_SHORTNAME' ]: + + value = dm.get_thread_data(prop) + if value is None: + value = 1 # sane default + + params = params.replace(prop, '"%s"' % value) + + params = osrf.json.to_object(params) + for idx in range(0, int(weight)): + self.calls.append({ + 'service' : '.'.join(method.split('.')[0:2]), + 'method' : method, + 'params' : params + }) + + file.close() + + def run(self): + + if not self.loaded: + self.load_file() + + for idx in range(0, len(self.calls)): + call = self.calls[int(random.random() * len(self.calls))] + try: + BatchMethodTask(call['service'], call['method']).start(params=call['params']) + except: + log.log_error("Batch Method Error") + traceback.print_exc() + + + +ScriptManager.go(BatchScript())