From 1015de4ba0b82d923ef069c426128f0243a14001 Mon Sep 17 00:00:00 2001 From: gfawcett Date: Mon, 23 Mar 2009 01:06:07 +0000 Subject: [PATCH] Z39.50 + Evergreen demo: are items available, holdable? see /syrup/graham_z3950test/ . I am using Evergreen in this example, but it could be replaced by SIP or another backend that can take a bib ID and return availability information. (If SIP needs a barcode, not bib ID (which I suspect it might), then we may need another lookup function in the interface. Will investigate. git-svn-id: svn://svn.open-ils.org/ILS-Contrib/servres/trunk@210 6d9bc8c9-1ec2-4278-b937-99fde70a366f --- conifer/libsystems/__init__.py | 0 conifer/libsystems/evergreen/__init__.py | 0 conifer/libsystems/evergreen/item_status.py | 26 +++++++ conifer/libsystems/evergreen/support.py | 83 +++++++++++++++++++++ conifer/libsystems/z3950/__init__.py | 0 conifer/libsystems/z3950/marctools.py | 112 ++++++++++++++++++++++++++++ conifer/libsystems/z3950/yaz_search.py | 85 +++++++++++++++++++++ conifer/syrup/urls.py | 1 + conifer/syrup/views.py | 15 +++- conifer/templates/graham_z3950_test.xhtml | 48 ++++++++++++ 10 files changed, 369 insertions(+), 1 deletion(-) create mode 100644 conifer/libsystems/__init__.py create mode 100644 conifer/libsystems/evergreen/__init__.py create mode 100644 conifer/libsystems/evergreen/item_status.py create mode 100644 conifer/libsystems/evergreen/support.py create mode 100644 conifer/libsystems/z3950/__init__.py create mode 100644 conifer/libsystems/z3950/marctools.py create mode 100644 conifer/libsystems/z3950/yaz_search.py create mode 100644 conifer/templates/graham_z3950_test.xhtml diff --git a/conifer/libsystems/__init__.py b/conifer/libsystems/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/conifer/libsystems/evergreen/__init__.py b/conifer/libsystems/evergreen/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/conifer/libsystems/evergreen/item_status.py b/conifer/libsystems/evergreen/item_status.py new file mode 100644 index 0000000..f8b725b --- /dev/null +++ b/conifer/libsystems/evergreen/item_status.py @@ -0,0 +1,26 @@ +import warnings +from support import ER, E1 +from pprint import pprint + +# Proposing this as an interface method. Given a bib ID, return a dict +# giving the item's bibid, barcode, availability (boolean), +# holdability (boolean), and location (a string description). If the +# bib ID is invalid, return None. + +def lookup_availability(bib_id): + rec = E1('open-ils.search.asset.copy.fleshed2.retrieve', bib_id) + if 'stacktrace' in rec: + warnings.warn(repr(('no such bib id', bib_id, repr(rec)))) + return None + resp = { + 'bibid': bib_id, + 'barcode': rec['barcode'], + 'available': rec['status']['name'] == 'Available', + 'holdable': rec['status']['holdable'] == 't', + 'location': rec['location']['name']} + return resp + + +if __name__ == '__main__': + DYLAN = 1321798 + print lookup_availability(DYLAN) diff --git a/conifer/libsystems/evergreen/support.py b/conifer/libsystems/evergreen/support.py new file mode 100644 index 0000000..df87491 --- /dev/null +++ b/conifer/libsystems/evergreen/support.py @@ -0,0 +1,83 @@ +import warnings +import urllib2 +from urllib import quote +import simplejson as json +from xml.etree import ElementTree +import re +import sys + +#------------------------------------------------------------ +# Configuration + +# where is our evergreen server's opensrf http gateway? + +BASE = 'http://dwarf.cs.uoguelph.ca/osrf-gateway-v1' +LOCALE = 'en-US' + +# where can I find a copy of fm_IDL.xml from Evergreen? + +# # This will work always, though maybe you want to up the rev number... +# FM_IDL_LOCATION = ('http://svn.open-ils.org/trac/ILS/export/12640' +# '/trunk/Open-ILS/examples/fm_IDL.xml') + +# # or, if you have a local copy... +# FM_IDL_LOCATION = 'file:fm_IDL.xml' + +FM_IDL_LOCATION = 'http://dwarf.cs.uoguelph.ca/reports/fm_IDL.xml' + +#------------------------------------------------------------ +# parse fm_IDL, to build a field-name-lookup service. + +def _fields(): + tree = ElementTree.parse(urllib2.urlopen(FM_IDL_LOCATION)) + NS = '{http://opensrf.org/spec/IDL/base/v1}' + for c in tree.findall('%sclass' % NS): + cid = c.attrib['id'] + fields = [f.attrib['name'] \ + for f in c.findall('%sfields/%sfield' % (NS,NS))] + yield (cid, fields) + +fields_for_class = dict(_fields()) + +#------------------------------------------------------------ + +def evergreen_object(rec): + """Where possible, add field-names to an Evergreen return-value.""" + if isinstance(rec, list): + return map(evergreen_object, rec) + if not (isinstance(rec, dict) and '__c' in rec): + return rec + else: + kls = rec['__c'] + data = rec['__p'] + fields = fields_for_class[kls] + #print '----', (kls, fields) + return dict(zip(fields, map(evergreen_object, data))) + +def evergreen_request(method, *args, **kwargs): + service = '.'.join(method.split('.')[:2]) + kwargs.setdefault('locale', LOCALE) + kwargs.update({'service':service, 'method':method}) + params = ['%s=%s' % (k,quote(v)) for k,v in kwargs.items()] + params += ['param=%s' % quote(str(a)) for a in args] + url = '%s?%s' % (BASE, '&'.join(params)) + req = urllib2.urlopen(url) + resp = json.load(req) + assert resp['status'] == 200, 'error during evergreen request' + payload = resp['payload'] + #print '----', payload + return evergreen_object(payload) + +def evergreen_request_single_result(method, *args): + resp = evergreen_request(method, *args) + if len(resp) > 1: + warnings.warn('taking single value from multivalue evergreen response') + print >> sys.stderr, repr(resp) + return resp[0] + + +#------------------------------------------------------------ +# Abbreviations + +ER = evergreen_request +E1 = evergreen_request_single_result diff --git a/conifer/libsystems/z3950/__init__.py b/conifer/libsystems/z3950/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/conifer/libsystems/z3950/marctools.py b/conifer/libsystems/z3950/marctools.py new file mode 100644 index 0000000..e6ed72c --- /dev/null +++ b/conifer/libsystems/z3950/marctools.py @@ -0,0 +1,112 @@ +""" + MARC utlities + Public Domain 2007 public.resource.org + + Author: Joel Hardi +""" + +class locToUTF8(object): + "Changes text from LOC into unicode, using replace() method" + + dict = {} + charmap = {} + + def __init__(self): + "Sets self.dict and search character index self.charmap" + self.dict = { + "\X20":"\u0020", # "HARD SPACE - represented by a space" + "\XC2\XA1":"\u00A1", # "INVERTED EXCLAMATION MARK" + "\XC2\XA3":"\u00A3", # "BRITISH POUND / POUND SIGN" + "\XC2\XA9":"\u00A9", # "COPYRIGHT SIGN" + "\XC2\XAE":"\u00AE", # "PATENT MARK / REGISTERED SIGN" + "\XC2\XB0":"\u00B0", # "DEGREE SIGN" + "\XC2\XB1":"\u00B1", # "PLUS OR MINUS / PLUS-MINUS SIGN" + "\XC2\XB7":"\u00B7", # "MIDDLE DOT" + "\XC2\XBF":"\u00BF", # "INVERTED QUESTION MARK" + "\XC3\X86":"\u00C6", # "UPPERCASE DIGRAPH AE / LATIN CAPITAL LIGATURE AE" + "\XC3\X98":"\u00D8", # "UPPERCASE SCANDINAVIAN O / LATIN CAPITAL LETTER O WITH STROKE" + "\XC3\X9E":"\u00DE", # "UPPERCASE ICELANDIC THORN / LATIN CAPITAL LETTER THORN (Icelandic)" + "\XC3\XA6":"\u00E6", # "LOWERCASE DIGRAPH AE / LATIN SMALL LIGATURE AE" + "\XC3\XB0":"\u00F0", # "LOWERCASE ETH / LATIN SMALL LETTER ETH (Icelandic)" + "\XC3\XB8":"\u00F8", # "LOWERCASE SCANDINAVIAN O / LATIN SMALL LETTER O WITH STROKE" + "\XC3\XBE":"\u00FE", # "LOWERCASE ICELANDIC THORN / LATIN SMALL LETTER THORN (Icelandic)" + "\XC4\X90":"\u0110", # "UPPERCASE D WITH CROSSBAR / LATIN CAPITAL LETTER D WITH STROKE" + "\XC4\X91":"\u0111", # "LOWERCASE D WITH CROSSBAR / LATIN SMALL LETTER D WITH STROKE" + "\XC4\XB1":"\u0131", # "LOWERCASE TURKISH I / LATIN SMALL LETTER DOTLESS I" + "\XC5\X81":"\u0141", # "UPPERCASE POLISH L / LATIN CAPITAL LETTER L WITH STROKE" + "\XC5\X82":"\u0142", # "LOWERCASE POLISH L / LATIN SMALL LETTER L WITH STROKE" + "\XC5\X92":"\u0152", # "UPPERCASE DIGRAPH OE / LATIN CAPITAL LIGATURE OE" + "\XC5\X93":"\u0153", # "LOWERCASE DIGRAPH OE / LATIN SMALL LIGATURE OE" + "\XC6\XA0":"\u01A0", # "UPPERCASE O-HOOK / LATIN CAPITAL LETTER O WITH HORN" + "\XC6\XA1":"\u01A1", # "LOWERCASE O-HOOK / LATIN SMALL LETTER O WITH HORN" + "\XC6\XAF":"\u01AF", # "UPPERCASE U-HOOK / LATIN CAPITAL LETTER U WITH HORN" + "\XC6\XB0":"\u01B0", # "LOWERCASE U-HOOK / LATIN SMALL LETTER U WITH HORN" + "\XCA\XB9":"\u02B9", # "SOFT SIGN, PRIME / MODIFIER LETTER PRIME" + "\XCA\XBA":"\u02BA", # "HARD SIGN, DOUBLE PRIME / MODIFIER LETTER DOUBLE PRIME" + "\XCA\XBB":"\u02BB", # "AYN / MODIFIER LETTER TURNED COMMA" + "\XCA\XBE":"\u02BE", # "ALIF / MODIFIER LETTER RIGHT HALF RING" + "\XCC\X80":"\u0300", # "GRAVE / COMBINING GRAVE ACCENT (Varia)" + "\XCC\X81":"\u0301", # "ACUTE / COMBINING ACUTE ACCENT (Oxia)" + "\XCC\X82":"\u0302", # "CIRCUMFLEX / COMBINING CIRCUMFLEX ACCENT" + "\XCC\X83":"\u0303", # "TILDE / COMBINING TILDE" + "\XCC\X84":"\u0304", # "MACRON / COMBINING MACRON" + "\XCC\X86":"\u0306", # "BREVE / COMBINING BREVE (Vrachy)" + "\XCC\X87":"\u0307", # "SUPERIOR DOT / COMBINING DOT ABOVE" + "\XCC\X88":"\u0308", # "UMLAUT, DIAERESIS / COMBINING DIAERESIS (Dialytika)" + "\XCC\X89":"\u0309", # "PSEUDO QUESTION MARK / COMBINING HOOK ABOVE" + "\XCC\X8A":"\u030A", # "CIRCLE ABOVE, ANGSTROM / COMBINING RING ABOVE" + "\XCC\X8B":"\u030B", # "DOUBLE ACUTE / COMBINING DOUBLE ACUTE ACCENT" + "\XCC\X8C":"\u030C", # "HACEK / COMBINING CARON" + "\XCC\X90":"\u0310", # "CANDRABINDU / COMBINING CANDRABINDU" + "\XCC\X93":"\u0313", # "HIGH COMMA, CENTERED / COMBINING COMMA ABOVE (Psili)" + "\XCC\X95":"\u0315", # "HIGH COMMA, OFF CENTER / COMBINING COMMA ABOVE RIGHT" + "\XCC\X9C":"\u031C", # "RIGHT CEDILLA / COMBINING LEFT HALF RING BELOW" + "\XCC\XA3":"\u0323", # "DOT BELOW / COMBINING DOT BELOW" + "\XCC\XA4":"\u0324", # "DOUBLE DOT BELOW / COMBINING DIAERESIS BELOW" + "\XCC\XA5":"\u0325", # "CIRCLE BELOW / COMBINING RING BELOW" + "\XCC\XA6":"\u0326", # "LEFT HOOK (COMMA BELOW) / COMBINING COMMA BELOW" + "\XCC\XA7":"\u0327", # "CEDILLA / COMBINING CEDILLA" + "\XCC\XA8":"\u0328", # "RIGHT HOOK, OGONEK / COMBINING OGONEK" + "\XCC\XAE":"\u032E", # "UPADHMANIYA / COMBINING BREVE BELOW" + "\XCC\XB2":"\u0332", # "UNDERSCORE / COMBINING LOW LINE" + "\XCC\XB3":"\u0333", # "DOUBLE UNDERSCORE / COMBINING DOUBLE LOW LINE" + "\XE2\X84\X93":"\u2113", # "SCRIPT SMALL L" + "\XE2\X84\X97":"\u2117", # "SOUND RECORDING COPYRIGHT" + "\XE2\X99\XAD":"\u266D", # "MUSIC FLAT SIGN" + "\XE2\X99\XAF":"\u266F", # "MUSIC SHARP SIGN" + "\XEF\XB8\XA0":"\uFE20", # "LIGATURE, FIRST HALF / COMBINING LIGATURE LEFT HALF" + "\XEF\XB8\XA1":"\uFE21", # "LIGATURE, SECOND HALF / COMBINING LIGATURE RIGHT HALF" + "\XEF\XB8\XA2":"\uFE22", # "DOUBLE TILDE, FIRST HALF / COMBINING DOUBLE TILDE LEFT HALF" + "\XEF\XB8\XA3":"\uFE23", # "DOUBLE TILDE, SECOND HALF / COMBINING DOUBLE TILDE RIGHT HALF" + } + + # build self.charmap to map each first char of a search string to a list of its search strings + firstchars = [] + self.charmap = {} + for i in self.dict.iterkeys(): + if firstchars.count(i[0]) == 0: + firstchars.append(i[0]) + self.charmap[i[0]] = [] + self.charmap[i[0]].append(i) + + def replace(self, str): + "Given string str, returns unicode string with correct character replcements" + searchchars = [] + # build subset of search/replace pairs to use based on if first char of search appears in str + prev = range(0,3) + for c in str: + prev[0] = prev[1] + prev[1] = prev[2] + prev[2] = c + if self.charmap.has_key(c): + if searchchars.count(c) == 0: + searchchars.append(c) + elif ord(c) > 127 and prev.count(c) == 0: + str = str.replace(c, '\\X%x' % ord(c)) + + # perform search/replaces + for c in searchchars: + for i in self.charmap[c]: + str = str.replace(i, self.dict[i]) + + return unicode(str, 'raw-unicode-escape') diff --git a/conifer/libsystems/z3950/yaz_search.py b/conifer/libsystems/z3950/yaz_search.py new file mode 100644 index 0000000..724d235 --- /dev/null +++ b/conifer/libsystems/z3950/yaz_search.py @@ -0,0 +1,85 @@ +# z39.50 search using yaz-client. +# dependencies: yaz-client, pexpect + +# I found that pyz3950.zoom seemed wonky when testing against conifer +# z3950, so I whipped up this expect-based version instead. + +import warnings +import re +from xml.etree import ElementTree +import pexpect +import marctools +loc_to_unicode = marctools.locToUTF8().replace + +LOG = None # for pexpect debugging, try LOG = sys.stderr +YAZ_CLIENT = 'yaz-client' +GENERAL_TIMEOUT = 3 +PRESENT_TIMEOUT = 30 + + +def search(host, database, query, start=1, limit=None): + + server = pexpect.spawn('yaz-client', timeout=GENERAL_TIMEOUT, logfile=LOG) + for line in ('open %s' % host, 'base %s' % database, 'format xml'): + server.sendline(line) + server.expect('Z>') + + # send the query + # note, we're using prefix queries for the moment. + server.sendline('find %s' % query) + server.expect(r'Number of hits: (\d+).*') + numhits = int(server.match.group(1)) + if start > numhits: + warnings.warn('asked z3950 to start at %d, but only %d results.' % (start, numhits)) + return [] + + # how many to present? At most 10 for now. + to_show = min(numhits-1, 10) # minus 1 for dwarf ?? + if limit: + to_show = min(to_show, limit) + server.expect('Z>') + server.sendline('show %s + %d' % (start, to_show)) + err = server.expect_list([re.compile(r'Records: (\d+)'), + re.compile('Target closed connection')]) + if err: + warnings.warn('error during z3950 conversation.') + server.close() + return [] + + raw_records = [] + for x in range(to_show): + server.expect(r'Record type: XML', timeout=PRESENT_TIMEOUT) + server.expect('') + raw_records.append(server.match.group(0)) + + server.expect('nextResultSetPosition') + server.expect('Z>') + server.sendline('quit') + server.close() + + parsed = [] + for rec in raw_records: + dct = {} + parsed.append(dct) + tree = ElementTree.fromstring(rec) + for df in tree.findall('{http://www.loc.gov/MARC21/slim}datafield'): + t = df.attrib['tag'] + for sf in df.findall('{http://www.loc.gov/MARC21/slim}subfield'): + c = sf.attrib['code'] + v = sf.text + dct[t+c] = loc_to_unicode(v) + + return parsed + +#------------------------------------------------------------ +# some tests + +if __name__ == '__main__': + print loc_to_unicode('A\\XCC\\X81n') + tests = [ + ('dwarf.cs.uoguelph.ca:2210', 'conifer', '@and "Musson" "Evil"'), + ('dwarf.cs.uoguelph.ca:2210', 'conifer', '@and "Denis" "Gravel"'), + ('z3950.loc.gov:7090', 'VOYAGER', '@attr 1=4 @attr 4=1 "dylan"')] + for host, db, query in tests: + print (host, db, query) + print search(host, db, query, limit=1) diff --git a/conifer/syrup/urls.py b/conifer/syrup/urls.py index a4e9d5a..6499a85 100644 --- a/conifer/syrup/urls.py +++ b/conifer/syrup/urls.py @@ -16,6 +16,7 @@ urlpatterns = patterns('conifer.syrup.views', (r'^browse/(?P.*)/$', 'browse'), (r'^prefs/$', 'user_prefs'), (r'^z3950test/$', 'z3950_test'), + (r'^graham_z3950test/$', 'graham_z3950_test'), #MARK: propose we kill open_courses, we have browse. (r'^opencourse/$', 'open_courses'), (r'^search/$', 'search'), diff --git a/conifer/syrup/views.py b/conifer/syrup/views.py index 84189cf..196653e 100644 --- a/conifer/syrup/views.py +++ b/conifer/syrup/views.py @@ -241,9 +241,22 @@ def z3950_test(request): # print("done searching...") res_str = "" . join(collector) # print(res_str) - return g.render('z3950_test.xhtml', res_str=res_str) +def graham_z3950_test(request): + from conifer.libsystems.z3950 import yaz_search + from conifer.libsystems.evergreen.item_status import lookup_availability + host, db, query = ('dwarf.cs.uoguelph.ca:2210', 'conifer', '@and "Denis" "Gravel"') + #host, db, query = ('z3950.loc.gov:7090', 'VOYAGER', '@attr 1=4 @attr 4=1 "dylan"') + results = yaz_search.search(host, db, query) + for result in results: + bibid = result.get('901c') + if bibid: + avail = lookup_availability(bibid) + if avail: + result['avail'] = avail + return g.render('graham_z3950_test.xhtml', results=results) + def browse(request, browse_option=''): #the defaults should be moved into a config file or something... page_num = int(request.GET.get('page', 1)) diff --git a/conifer/templates/graham_z3950_test.xhtml b/conifer/templates/graham_z3950_test.xhtml new file mode 100644 index 0000000..5dfd989 --- /dev/null +++ b/conifer/templates/graham_z3950_test.xhtml @@ -0,0 +1,48 @@ + + + + + + ${title} + + + + +

${title}

+

show more detail

+ + + + + + + + + + + + + + + +
${title}${res[k]}
Availability +
+ ${'%s = %s' % (k,v)} +
+
${k}${res[k]}
+ + -- 2.11.0