From: erickson Date: Fri, 5 Jan 2007 19:14:58 +0000 (+0000) Subject: adding python libs X-Git-Url: https://old-git.evergreen-ils.org/?a=commitdiff_plain;h=d323cac30d954aa0bceaff8b961663da67bdce72;p=opensrf%2Fbjwebb.git adding python libs git-svn-id: svn://svn.open-ils.org/OpenSRF/trunk@811 9efc2488-bf62-4759-914b-345cdb29e865 --- diff --git a/src/python/osrf/__init__.py b/src/python/osrf/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/python/osrf/conf.py b/src/python/osrf/conf.py new file mode 100644 index 0000000..34e8df3 --- /dev/null +++ b/src/python/osrf/conf.py @@ -0,0 +1,48 @@ +# ----------------------------------------------------------------------- +# Copyright (C) 2007 Georgia Public Library Service +# Bill Erickson +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# ----------------------------------------------------------------------- + + +from osrf.utils import * +from osrf.ex import * + +class osrfConfig(object): + """Loads and parses the bootstrap config file""" + + config = None + + def __init__(self, file=None): + self.file = file + self.data = {} + + def parseConfig(self,file=None): + self.data = osrfXMLFileToObject(file or self.file) + osrfConfig.config = self + + def getValue(self, key, idx=None): + val = osrfObjectFindPath(self.data, key, idx) + if not val: + raise osrfConfigException("Config value not found: " + key) + return val + + +def osrfConfigValue(key, idx=None): + """Returns a bootstrap config value. + + key -- A string representing the path to the value in the config object + e.g. "domains.domain", "username" + idx -- Optional array index if the searched value is an array member + """ + return osrfConfig.config.getValue(key, idx) + diff --git a/src/python/osrf/const.py b/src/python/osrf/const.py new file mode 100644 index 0000000..eacf479 --- /dev/null +++ b/src/python/osrf/const.py @@ -0,0 +1,74 @@ +# ----------------------------------------------------------------------- +# Copyright (C) 2007 Georgia Public Library Service +# Bill Erickson +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# +# Collection of global constants +# ----------------------------------------------------------------------- + +# ----------------------------------------------------------------------- +# log levels +# ----------------------------------------------------------------------- +OSRF_LOG_ERR = 1 +OSRF_LOG_WARN = 2 +OSRF_LOG_INFO = 3 +OSRF_LOG_DEBUG = 4 +OSRF_LOG_INTERNAL = 5 + +# ----------------------------------------------------------------------- +# Session states +# ----------------------------------------------------------------------- +OSRF_APP_SESSION_CONNECTED = 0 +OSRF_APP_SESSION_CONNECTING = 1 +OSRF_APP_SESSION_DISCONNECTED = 2 + +# ----------------------------------------------------------------------- +# OpenSRF message types +# ----------------------------------------------------------------------- +OSRF_MESSAGE_TYPE_REQUEST = 'REQUEST' +OSRF_MESSAGE_TYPE_STATUS = 'STATUS' +OSRF_MESSAGE_TYPE_RESULT = 'RESULT' +OSRF_MESSAGE_TYPE_CONNECT = 'CONNECT' +OSRF_MESSAGE_TYPE_DISCONNECT = 'DISCONNECT' + +# ----------------------------------------------------------------------- +# OpenSRF message statuses +# ----------------------------------------------------------------------- +OSRF_STATUS_CONTINUE = 100 +OSRF_STATUS_OK = 200 +OSRF_STATUS_ACCEPTED = 202 +OSRF_STATUS_COMPLETE = 205 +OSRF_STATUS_REDIRECTED = 307 +OSRF_STATUS_BADREQUEST = 400 +OSRF_STATUS_UNAUTHORIZED = 401 +OSRF_STATUS_FORBIDDEN = 403 +OSRF_STATUS_NOTFOUND = 404 +OSRF_STATUS_NOTALLOWED = 405 +OSRF_STATUS_TIMEOUT = 408 +OSRF_STATUS_EXPFAILED = 417 +OSRF_STATUS_INTERNALSERVERERROR = 500 +OSRF_STATUS_NOTIMPLEMENTED = 501 +OSRF_STATUS_VERSIONNOTSUPPORTED = 505 + + +# ----------------------------------------------------------------------- +# Some well-known services +# ----------------------------------------------------------------------- +OSRF_APP_SETTINGS = 'opensrf.settings' +OSRF_APP_MATH = 'opensrf.math' + + +# where do we find the settings config +OSRF_METHOD_GET_HOST_CONFIG = 'opensrf.settings.host_config.get' + + diff --git a/src/python/osrf/ex.py b/src/python/osrf/ex.py new file mode 100644 index 0000000..4c160f3 --- /dev/null +++ b/src/python/osrf/ex.py @@ -0,0 +1,55 @@ +# ----------------------------------------------------------------------- +# Copyright (C) 2007 Georgia Public Library Service +# Bill Erickson +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# +# This modules define the exception classes. In general, an +# exception is little more than a name. +# ----------------------------------------------------------------------- + +class osrfException(Exception): + """Root class for exceptions.""" + def __init__(self, info=None): + self.info = info; + def __str__(self): + return self.info + + +class osrfNetworkException(osrfException): + def __str__(self): + str = "\nUnable to communicate with the OpenSRF network" + if self.info: + str = str + '\n' + repr(self.info) + return str + +class osrfProtocolException(osrfException): + """Raised when something happens during opensrf network stack processing.""" + pass + +class osrfServiceException(osrfException): + """Raised when there was an error communicating with a remote service.""" + pass + +class osrfConfigException(osrfException): + """Invalid config option requested.""" + pass + +class osrfNetworkObjectException(osrfException): + pass + +class osrfJSONParseException(osrfException): + """Raised when a JSON parsing error occurs.""" + pass + + + diff --git a/src/python/osrf/json.py b/src/python/osrf/json.py new file mode 100644 index 0000000..615b0fd --- /dev/null +++ b/src/python/osrf/json.py @@ -0,0 +1,233 @@ +# ----------------------------------------------------------------------- +# Copyright (C) 2007 Georgia Public Library Service +# Bill Erickson +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# ----------------------------------------------------------------------- + + +import simplejson, types + +JSON_PAYLOAD_KEY = '__p' +JSON_CLASS_KEY = '__c' + +class osrfNetworkObject(object): + """Base class for serializable network objects.""" + def getData(self): + """Returns a dict of data contained by this object""" + return self.data + + +class __unknown(osrfNetworkObject): + """Default class for un-registered network objects.""" + def __init__(self, data=None): + self.data = data + +setattr(__unknown,'__keys', []) +setattr(osrfNetworkObject,'__unknown', __unknown) + + +def osrfNetworkRegisterHint(hint, keys, type='hash'): + """Register a network hint. + + This creates a new class at osrfNetworkObject. with + methods for accessing/mutating the object's data. + Method names will match the names found in the keys array + + hint - The hint name to encode with the object + type - The data container type. + keys - An array of data keys. If type is an 'array', the order of + the keys will determine how the data is accessed + """ + + estr = "class %s(osrfNetworkObject):\n" % hint + estr += "\tdef __init__(self, data=None):\n" + estr += "\t\tself.data = data\n" + estr += "\t\tif data:\n" + + if type == 'hash': + estr += "\t\t\tpass\n" + else: + # we have to make sure the array is large enough + estr += "\t\t\twhile len(data) < %d:\n" % len(keys) + estr += "\t\t\t\tdata.append(None)\n" + + estr += "\t\telse:\n" + + if type == 'array': + estr += "\t\t\tself.data = []\n" + estr += "\t\t\tfor i in range(%s):\n" % len(keys) + estr += "\t\t\t\tself.data.append(None)\n" + for i in range(len(keys)): + estr += "\tdef %s(self, *args):\n"\ + "\t\tif len(args) != 0:\n"\ + "\t\t\tself.data[%s] = args[0]\n"\ + "\t\treturn self.data[%s]\n" % (keys[i], i, i) + + if type == 'hash': + estr += "\t\t\tself.data = {}\n" + estr += "\t\t\tfor i in %s:\n" % str(keys) + estr += "\t\t\t\tself.data[i] = None\n" + for i in keys: + estr += "\tdef %s(self, *args):\n"\ + "\t\tif len(args) != 0:\n"\ + "\t\t\tself.data['%s'] = args[0]\n"\ + "\t\tval = None\n"\ + "\t\ttry: val = self.data['%s']\n"\ + "\t\texcept: return None\n"\ + "\t\treturn val\n" % (i, i, i) + + estr += "setattr(osrfNetworkObject, '%s', %s)\n" % (hint,hint) + estr += "setattr(osrfNetworkObject.%s, '__keys', keys)" % hint + exec(estr) + + + +# ------------------------------------------------------------------- +# Define the custom object parsing behavior +# ------------------------------------------------------------------- +def __parseNetObject(obj): + hint = None + islist = False + try: + hint = obj[JSON_CLASS_KEY] + obj = obj[JSON_PAYLOAD_KEY] + except: pass + if isinstance(obj,list): + islist = True + for i in range(len(obj)): + obj[i] = __parseNetObject(obj[i]) + else: + if isinstance(obj,dict): + for k,v in obj.iteritems(): + obj[k] = __parseNetObject(v) + + if hint: # Now, "bless" the object into an osrfNetworkObject + estr = 'obj = osrfNetworkObject.%s(obj)' % hint + try: + exec(estr) + except AttributeError: + # this object has not been registered, shove it into the default container + obj = osrfNetworkObject.__unknown(obj) + + return obj; + + +# ------------------------------------------------------------------- +# Define the custom object encoding behavior +# ------------------------------------------------------------------- +class osrfJSONNetworkEncoder(simplejson.JSONEncoder): + def default(self, obj): + if isinstance(obj, osrfNetworkObject): + return { + JSON_CLASS_KEY: obj.__class__.__name__, + JSON_PAYLOAD_KEY: self.default(obj.getData()) + } + return obj + + +def osrfObjectToJSON(obj): + """Turns a python object into a wrapped JSON object""" + return simplejson.dumps(obj, cls=osrfJSONNetworkEncoder) + + +def osrfJSONToObject(json): + """Turns a JSON string into python objects""" + obj = simplejson.loads(json) + return __parseNetObject(obj) + +def osrfParseJSONRaw(json): + """Parses JSON the old fashioned way.""" + return simplejson.loads(json) + +def osrfToJSONRaw(obj): + """Stringifies an object as JSON with no additional logic.""" + return simplejson.dumps(obj) + +def __tabs(t): + r='' + for i in range(t): r += ' ' + return r + +def osrfDebugNetworkObject(obj, t=1): + """Returns a debug string for a given object. + + If it's an osrfNetworkObject and has registered keys, key/value p + pairs are returned. Otherwise formatted JSON is returned""" + + s = '' + if isinstance(obj, osrfNetworkObject) and len(obj.__keys): + obj.__keys.sort() + + for k in obj.__keys: + + key = k + while len(key) < 24: key += '.' # pad the names to make the values line up somewhat + val = getattr(obj, k)() + + subobj = val and not (isinstance(val,unicode) or \ + isinstance(val, int) or isinstance(val, float) or isinstance(val, long)) + + + s += __tabs(t) + key + ' = ' + + if subobj: + s += '\n' + val = osrfDebugNetworkObject(val, t+1) + + s += str(val) + + if not subobj: s += '\n' + + else: + s = osrfFormatJSON(osrfObjectToJSON(obj)) + return s + +def osrfFormatJSON(json): + """JSON pretty-printer""" + r = '' + t = 0 + instring = False + inescape = False + done = False + + for c in json: + + done = False + if (c == '{' or c == '[') and not instring: + t += 1 + r += c + '\n' + __tabs(t) + done = True + + if (c == '}' or c == ']') and not instring: + t -= 1 + r += '\n' + __tabs(t) + c + done = True + + if c == ',' and not instring: + r += c + '\n' + __tabs(t) + done = True + + if c == '"' and not inescape: + instring = not instring + + if inescape: + inescape = False + + if c == '\\': + inescape = True + + if not done: + r += c + + return r + + diff --git a/src/python/osrf/log.py b/src/python/osrf/log.py new file mode 100644 index 0000000..083d513 --- /dev/null +++ b/src/python/osrf/log.py @@ -0,0 +1,88 @@ +# ----------------------------------------------------------------------- +# Copyright (C) 2007 Georgia Public Library Service +# Bill Erickson +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# ----------------------------------------------------------------------- + +from syslog import * +import traceback, sys, os, re +from osrf.const import * + +loglevel = 4 + +def osrfInitLog(level, facility=None, file=None): + """Initialize the logging subsystem.""" + global loglevel + if facility: osrfInitSyslog(facility, level) + loglevel = level + syslog(LOG_DEBUG, "syslog initialized") + + +# ----------------------------------------------------------------------- +# Define wrapper functions for the log levels +# ----------------------------------------------------------------------- +def osrfLogInternal(s): __osrfLog(OSRF_LOG_INTERNAL,s) +def osrfLogDebug(s): __osrfLog(OSRF_LOG_DEBUG,s) +def osrfLogInfo(s): __osrfLog(OSRF_LOG_INFO,s) +def osrfLogWarn(s): __osrfLog(OSRF_LOG_WARN,s) +def osrfLogErr(s): __osrfLog(OSRF_LOG_ERR,s) + + +frgx = re.compile('/.*/') + +def __osrfLog(level, msg): + """Builds the log message and passes the message off to the logger.""" + global loglevel + if int(level) > int(loglevel): return + + # find the caller info for logging the file and line number + tb = traceback.extract_stack(limit=3) + tb = tb[0] + lvl = 'DEBG' + slvl = LOG_DEBUG + + if level == OSRF_LOG_INTERNAL: lvl = 'INT '; slvl=LOG_DEBUG + if level == OSRF_LOG_INFO: lvl = 'INFO'; slvl=LOG_INFO + if level == OSRF_LOG_WARN: lvl = 'WARN'; slvl=LOG_WARNING + if level == OSRF_LOG_ERR: lvl = 'ERR '; slvl=LOG_ERR + + file = frgx.sub('',tb[0]) + msg = '[%s:%d:%s:%s] %s' % (lvl, os.getpid(), file, tb[1], msg) + syslog(slvl, msg) + + if level == OSRF_LOG_ERR: + sys.stderr.write(msg + '\n') + + +def osrfInitSyslog(facility, level): + """Connect to syslog and set the logmask based on the level provided.""" + + level = int(level) + + if facility == 'local0': facility = LOG_LOCAL0 + if facility == 'local1': facility = LOG_LOCAL1 + if facility == 'local2': facility = LOG_LOCAL2 + if facility == 'local3': facility = LOG_LOCAL3 + if facility == 'local4': facility = LOG_LOCAL4 + if facility == 'local5': facility = LOG_LOCAL5 + if facility == 'local6': facility = LOG_LOCAL6 + # XXX add other facility maps if necessary + openlog(sys.argv[0], 0, facility) + + # this is redundant... + mask = LOG_UPTO(LOG_ERR) + if level >= 1: mask |= LOG_MASK(LOG_WARNING) + if level >= 2: mask |= LOG_MASK(LOG_NOTICE) + if level >= 3: mask |= LOG_MASK(LOG_INFO) + if level >= 4: mask |= LOG_MASK(LOG_DEBUG) + setlogmask(mask) + diff --git a/src/python/osrf/net.py b/src/python/osrf/net.py new file mode 100644 index 0000000..db0b131 --- /dev/null +++ b/src/python/osrf/net.py @@ -0,0 +1,147 @@ +# ----------------------------------------------------------------------- +# Copyright (C) 2007 Georgia Public Library Service +# Bill Erickson +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# ----------------------------------------------------------------------- + + +from pyxmpp.jabber.client import JabberClient +from pyxmpp.message import Message +from pyxmpp.jid import JID +from socket import gethostname +from osrf.log import * +import os, time +import logging + +# - log jabber activity (for future reference) +#logger=logging.getLogger() +#logger.addHandler(logging.StreamHandler()) +#logger.addHandler(logging.FileHandler('j.log')) +#logger.setLevel(logging.DEBUG) + +__network = None +def osrfSetNetworkHandle(handle): + """Sets the global network connection handle.""" + global __network + __network = handle + +def osrfGetNetworkHandle(): + """Returns the global network connection handle.""" + global __network + return __network + + +class osrfNetworkMessage(object): + """Network message + + attributes: + + sender - message sender + to - message recipient + body - the body of the message + thread - the message thread + """ + + def __init__(self, message=None, **args): + if message: + self.body = message.get_body() + self.thread = message.get_thread() + self.to = message.get_to() + if message.xmlnode.hasProp('router_from') and message.xmlnode.prop('router_from') != '': + self.sender = message.xmlnode.prop('router_from') + else: self.sender = message.get_from().as_utf8() + else: + if args.has_key('sender'): self.sender = args['sender'] + if args.has_key('to'): self.to = args['to'] + if args.has_key('body'): self.body = args['body'] + if args.has_key('thread'): self.thread = args['thread'] + + +class osrfNetwork(JabberClient): + def __init__(self, **args): + self.isconnected = False + + # Create a unique jabber resource + resource = 'osrf_client' + if args.has_key('resource'): + resource = args['resource'] + resource += '_' + gethostname()+':'+ str(os.getpid()) + self.jid = JID(args['username'], args['host'], resource) + + osrfLogDebug("initializing network with JID %s and host=%s, port=%s, username=%s" % + (self.jid.as_utf8(), args['host'], args['port'], args['username'])) + + #initialize the superclass + JabberClient.__init__(self, self.jid, args['password'], args['host']) + self.queue = [] + + def connect(self): + JabberClient.connect(self) + while not self.isconnected: + stream = self.get_stream() + act = stream.loop_iter(10) + if not act: self.idle() + + def setRecvCallback(self, func): + """The callback provided is called when a message is received. + + The only argument to the function is the received message. """ + self.recvCallback = func + + def session_started(self): + osrfLogInfo("Successfully connected to the opensrf network") + self.authenticated() + self.stream.set_message_handler("normal",self.message_received) + self.isconnected = True + + def send(self, message): + """Sends the provided network message.""" + osrfLogInternal("jabber sending to %s: %s" % (message.to, message.body)) + msg = Message(None, None, message.to, None, None, None, message.body, message.thread) + self.stream.send(msg) + + def message_received(self, stanza): + """Handler for received messages.""" + osrfLogInternal("jabber received a message of type %s" % stanza.get_type()) + if stanza.get_type()=="headline": + return True + # check for errors + osrfLogInternal("jabber received message from %s : %s" + % (stanza.get_from().as_utf8(), stanza.get_body())) + self.queue.append(osrfNetworkMessage(stanza)) + return True + + def recv(self, timeout=120): + """Attempts to receive a message from the network. + + timeout - max number of seconds to wait for a message. + If no message is received in 'timeout' seconds, None is returned. """ + + msg = None + if len(self.queue) == 0: + while timeout >= 0 and len(self.queue) == 0: + starttime = time.time() + osrfLogInternal("going into stream loop at " + str(starttime)) + act = self.get_stream().loop_iter(timeout) + endtime = time.time() - starttime + timeout -= endtime + osrfLogInternal("exiting stream loop after %s seconds" % str(endtime)) + osrfLogInternal("act = %s : queue length = %d" % (act, len(self.queue)) ) + if not act: self.idle() + + # if we've acquired a message, handle it + if len(self.queue) > 0: + self.recvCallback(self.queue.pop(0)) + return None + + + diff --git a/src/python/osrf/ses.py b/src/python/osrf/ses.py new file mode 100644 index 0000000..2fbb502 --- /dev/null +++ b/src/python/osrf/ses.py @@ -0,0 +1,307 @@ +# ----------------------------------------------------------------------- +# Copyright (C) 2007 Georgia Public Library Service +# Bill Erickson +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# ----------------------------------------------------------------------- + +from osrf.json import * +from osrf.conf import osrfConfigValue +from osrf.net import osrfNetworkMessage, osrfGetNetworkHandle +from osrf.log import * +from osrf.const import * +import random, sys, os, time + + +# ----------------------------------------------------------------------- +# Go ahead and register the common network objects +# ----------------------------------------------------------------------- +osrfNetworkRegisterHint('osrfMessage', ['threadTrace', 'type', 'payload'], 'hash') +osrfNetworkRegisterHint('osrfMethod', ['method', 'params'], 'hash') +osrfNetworkRegisterHint('osrfResult', ['status', 'statusCode', 'content'], 'hash') +osrfNetworkRegisterHint('osrfConnectStatus', ['status','statusCode'], 'hash') +osrfNetworkRegisterHint('osrfMethodException', ['status', 'statusCode'], 'hash') + + +class osrfSession(object): + """Abstract session superclass.""" + + def __init__(self): + # by default, we're connected to no one + self.state = OSRF_APP_SESSION_DISCONNECTED + + + def wait(self, timeout=120): + """Wait up to seconds for data to arrive on the network""" + osrfLogInternal("osrfSession.wait(%d)" % timeout) + handle = osrfGetNetworkHandle() + handle.recv(timeout) + + def send(self, omessage): + """Sends an OpenSRF message""" + netMessage = osrfNetworkMessage( + to = self.remoteId, + body = osrfObjectToJSON([omessage]), + thread = self.thread ) + + handle = osrfGetNetworkHandle() + handle.send(netMessage) + + def cleanup(self): + """Removes the session from the global session cache.""" + del osrfClientSession.sessionCache[self.thread] + +class osrfClientSession(osrfSession): + """Client session object. Use this to make server requests.""" + + def __init__(self, service): + + # call superclass constructor + osrfSession.__init__(self) + + # the remote service we want to make requests of + self.service = service + + # find the remote service handle @/ + domain = osrfConfigValue('domains.domain', 0) + router = osrfConfigValue('router_name') + self.remoteId = "%s@%s/%s" % (router, domain, service) + self.origRemoteId = self.remoteId + + # generate a random message thread + self.thread = "%s%s%s" % (os.getpid(), str(random.randint(100,100000)), str(time.time())) + + # how many requests this session has taken part in + self.nextId = 0 + + # cache of request objects + self.requests = {} + + # cache this session in the global session cache + osrfClientSession.sessionCache[self.thread] = self + + def resetRequestTimeout(self, rid): + req = self.findRequest(rid) + if req: + req.resetTimeout = True + + + def request2(self, method, arr): + """Creates a new request and sends the request to the server using a python array as the params.""" + return self.__request(method, arr) + + def request(self, method, *args): + """Creates a new request and sends the request to the server using a variable argument list as params""" + arr = list(args) + return self.__request(method, arr) + + def __request(self, method, arr): + """Builds the request object and sends it.""" + if self.state != OSRF_APP_SESSION_CONNECTED: + self.resetRemoteId() + + osrfLogDebug("Sending request %s -> %s " % (self.service, method)) + req = osrfRequest(self, self.nextId, method, arr) + self.requests[str(self.nextId)] = req + self.nextId += 1 + req.send() + return req + + + def connect(self, timeout=10): + """Connects to a remote service""" + + if self.state == OSRF_APP_SESSION_CONNECTED: + return True + self.state == OSRF_APP_SESSION_CONNECTING + + # construct and send a CONNECT message + self.send( + osrfNetworkObject.osrfMessage( + { 'threadTrace' : 0, + 'type' : OSRF_MESSAGE_TYPE_CONNECT + } + ) + ) + + while timeout >= 0 and not self.state == OSRF_APP_SESSION_CONNECTED: + start = time.time() + self.wait(timeout) + timeout -= time.time() - start + + if self.state != OSRF_APP_SESSION_CONNECTED: + raise osrfServiceException("Unable to connect to " + self.service) + + return True + + def disconnect(self): + """Disconnects from a remote service""" + + if self.state == OSRF_APP_SESSION_DISCONNECTED: + return True + + self.send( + osrfNetworkObject.osrfMessage( + { 'threadTrace' : 0, + 'type' : OSRF_MESSAGE_TYPE_DISCONNECT + } + ) + ) + + self.state = OSRF_APP_SESSION_DISCONNECTED + + + + + def setRemoteId(self, remoteid): + self.remoteId = remoteid + osrfLogInternal("Setting request remote ID to %s" % self.remoteId) + + def resetRemoteId(self): + """Recovers the original remote id""" + self.remoteId = self.origRemoteId + osrfLogInternal("Resetting remote ID to %s" % self.remoteId) + + def pushResponseQueue(self, message): + """Pushes the message payload onto the response queue + for the request associated with the message's ID.""" + osrfLogDebug("pushing %s" % message.payload()) + try: + self.findRequest(message.threadTrace()).pushResponse(message.payload()) + except Exception, e: + osrfLogWarn("pushing respond to non-existent request %s : %s" % (message.threadTrace(), e)) + + def findRequest(self, rid): + """Returns the original request matching this message's threadTrace.""" + try: + return self.requests[str(rid)] + except KeyError: + osrfLogDebug('findRequest(): non-existent request %s' % str(rid)) + return None + + + +osrfSession.sessionCache = {} +def osrfFindSession(thread): + """Finds a session in the global cache.""" + try: + return osrfClientSession.sessionCache[thread] + except: return None + +class osrfRequest(object): + """Represents a single OpenSRF request. + A request is made and any resulting respones are + collected for the client.""" + + def __init__(self, session, id, method=None, params=[]): + + self.session = session # my session handle + self.id = id # my unique request ID + self.method = method # method name + self.params = params # my method params + self.queue = [] # response queue + self.resetTimeout = False # resets the recv timeout? + self.complete = False # has the server told us this request is done? + self.sendTime = 0 # local time the request was put on the wire + self.completeTime = 0 # time the server told us the request was completed + self.firstResponseTime = 0 # time it took for our first reponse to be received + + def send(self): + """Sends a request message""" + + # construct the method object message with params and method name + method = osrfNetworkObject.osrfMethod( { + 'method' : self.method, + 'params' : self.params + } ) + + # construct the osrf message with our method message embedded + message = osrfNetworkObject.osrfMessage( { + 'threadTrace' : self.id, + 'type' : OSRF_MESSAGE_TYPE_REQUEST, + 'payload' : method + } ) + + self.sendTime = time.time() + self.session.send(message) + + def recv(self, timeout=120): + """Waits up to seconds for a response to this request. + + If a message is received in time, the response message is returned. + Returns None otherwise.""" + + self.session.wait(0) + + origTimeout = timeout + while not self.complete and timeout >= 0 and len(self.queue) == 0: + s = time.time() + self.session.wait(timeout) + timeout -= time.time() - s + if self.resetTimeout: + self.resetTimeout = False + timeout = origTimeout + + now = time.time() + + # ----------------------------------------------------------------- + # log some statistics + if len(self.queue) > 0: + if not self.firstResponseTime: + self.firstResponseTime = now + osrfLogDebug("time elapsed before first response: %f" \ + % (self.firstResponseTime - self.sendTime)) + + if self.complete: + if not self.completeTime: + self.completeTime = now + osrfLogDebug("time elapsed before complete: %f" \ + % (self.completeTime - self.sendTime)) + # ----------------------------------------------------------------- + + + if len(self.queue) > 0: + # we have a reponse, return it + return self.queue.pop(0) + + return None + + def pushResponse(self, content): + """Pushes a method response onto this requests response queue.""" + self.queue.append(content) + + def cleanup(self): + """Cleans up request data from the cache. + + Do this when you are done with a request to prevent "leaked" cache memory.""" + del self.session.requests[str(self.id)] + + def setComplete(self): + """Sets me as complete. This means the server has sent a 'request complete' message""" + self.complete = True + + +class osrfServerSession(osrfSession): + """Implements a server-side session""" + pass + + +def osrfAtomicRequest(service, method, *args): + ses = osrfClientSession(service) + req = ses.request2('open-ils.cstore.direct.actor.user.retrieve', list(args)) # grab user with ID 1 + resp = req.recv() + data = resp.content() + req.cleanup() + ses.cleanup() + return data + + + diff --git a/src/python/osrf/set.py b/src/python/osrf/set.py new file mode 100644 index 0000000..a075bc1 --- /dev/null +++ b/src/python/osrf/set.py @@ -0,0 +1,43 @@ +# ----------------------------------------------------------------------- +# Copyright (C) 2007 Georgia Public Library Service +# Bill Erickson +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# ----------------------------------------------------------------------- + +from osrf.utils import * +from osrf.const import * +from osrf.ex import * + +# global settings config object +__conifg = None + +def osrfSettingsValue(path, idx=0): + global __config + val = osrfObjectFindPath(__config, path, idx) + if not val: + raise osrfConfigException("Config value not found: " + path) + return val + + +def osrfLoadSettings(hostname): + global __config + + from osrf.system import osrfConnect + from osrf.ses import osrfClientSession + + ses = osrfClientSession(OSRF_APP_SETTINGS) + req = ses.request(OSRF_METHOD_GET_HOST_CONFIG, hostname) + resp = req.recv(timeout=30) + __config = resp.content() + req.cleanup() + ses.cleanup() + diff --git a/src/python/osrf/stack.py b/src/python/osrf/stack.py new file mode 100644 index 0000000..0bf4892 --- /dev/null +++ b/src/python/osrf/stack.py @@ -0,0 +1,95 @@ +# ----------------------------------------------------------------------- +# Copyright (C) 2007 Georgia Public Library Service +# Bill Erickson +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# ----------------------------------------------------------------------- + +from osrf.json import * +from osrf.log import * +from osrf.ex import * +from osrf.ses import osrfFindSession, osrfClientSession, osrfServerSession +from osrf.const import * +from time import time + + +def osrfPushStack(netMessage): + ses = osrfFindSession(netMessage.thread) + + if not ses: + # This is an incoming request from a client, create a new server session + pass + + ses.setRemoteId(netMessage.sender) + + oMessages = osrfJSONToObject(netMessage.body) + + osrfLogInternal("osrfPushStack(): received %d messages" % len(oMessages)) + + # Pass each bundled opensrf message to the message handler + t = time() + for m in oMessages: + osrfHandleMessage(ses, m) + t = time() - t + + if isinstance(ses, osrfServerSession): + osrfLogInfo("Message processing duration %f" % t) + +def osrfHandleMessage(session, message): + + osrfLogInternal("osrfHandleMessage(): processing message of type %s" % message.type()) + + if isinstance(session, osrfClientSession): + + if message.type() == OSRF_MESSAGE_TYPE_RESULT: + session.pushResponseQueue(message) + return + + if message.type() == OSRF_MESSAGE_TYPE_STATUS: + + statusCode = int(message.payload().statusCode()) + statusText = message.payload().status() + osrfLogInternal("osrfHandleMessage(): processing STATUS, statusCode = %d" % statusCode) + + if statusCode == OSRF_STATUS_COMPLETE: + # The server has informed us that this request is complete + req = session.findRequest(message.threadTrace()) + if req: + osrfLogInternal("marking request as complete: %d" % req.id) + req.setComplete() + return + + if statusCode == OSRF_STATUS_OK: + # We have connected successfully + osrfLogDebug("Successfully connected to " + session.service) + session.state = OSRF_APP_SESSION_CONNECTED + return + + if statusCode == OSRF_STATUS_CONTINUE: + # server is telling us to reset our wait timeout and keep waiting for a response + session.resetRequestTimeout(message.threadTrace()) + return; + + if statusCode == OSRF_STATUS_TIMEOUT: + osrfLogDebug("The server did not receive a request from us in time...") + session.state = OSRF_APP_SESSION_DISCONNECTED + return + + if statusCode == OSRF_STATUS_NOTFOUND: + osrfLogErr("Requested method was not found on the server: %s" % statusText) + session.state = OSRF_APP_SESSION_DISCONNECTED + raise osrfServiceException(statusText) + + raise osrfProtocolException("Unknown message status: %d" % statusCode) + + + + diff --git a/src/python/osrf/system.py b/src/python/osrf/system.py new file mode 100644 index 0000000..4a978a5 --- /dev/null +++ b/src/python/osrf/system.py @@ -0,0 +1,48 @@ +# ----------------------------------------------------------------------- +# Copyright (C) 2007 Georgia Public Library Service +# Bill Erickson +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# ----------------------------------------------------------------------- + +from osrf.conf import osrfConfig, osrfConfigValue +from osrf.net import osrfNetwork, osrfSetNetworkHandle +from osrf.stack import osrfPushStack +from osrf.log import * +from osrf.set import osrfLoadSettings +import sys + + +def osrfConnect(configFile): + """ Connects to the opensrf network """ + + # parse the config file + configParser = osrfConfig(configFile) + configParser.parseConfig() + + # set up logging + osrfInitLog(osrfConfigValue('loglevel'), osrfConfigValue('syslog')) + + # connect to the opensrf network + network = osrfNetwork( + host=osrfConfigValue('domains.domain'), + port=osrfConfigValue('port'), + username=osrfConfigValue('username'), + password=osrfConfigValue('passwd')) + network.setRecvCallback(osrfPushStack) + osrfSetNetworkHandle(network) + network.connect() + + # load the domain-wide settings file + osrfLoadSettings(osrfConfigValue('domains.domain')) + + + diff --git a/src/python/osrf/utils.py b/src/python/osrf/utils.py new file mode 100644 index 0000000..1d9d7aa --- /dev/null +++ b/src/python/osrf/utils.py @@ -0,0 +1,116 @@ +# ----------------------------------------------------------------------- +# Copyright (C) 2007 Georgia Public Library Service +# Bill Erickson +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# ----------------------------------------------------------------------- + +import libxml2, re + +def osrfXMLFileToObject(filename): + """Turns the contents of an XML file into a Python object""" + doc = libxml2.parseFile(filename) + xmlNode = doc.children.children + return osrfXMLNodeToObject(xmlNode) + +def osrfXMLStringToObject(string): + """Turns an XML string into a Python object""" + doc = libxml2.parseString(string) + xmlNode = doc.children.children + return osrfXMLNodeToObject(xmlNode) + +def osrfXMLNodeToObject(xmlNode): + """Turns an XML node into a Python object""" + obj = {} + + while xmlNode: + if xmlNode.type == 'element': + nodeChild = xmlNode.children + done = False + nodeName = xmlNode.name + + while nodeChild: + if nodeChild.type == 'element': + + # If a node has element children, create a new sub-object + # for this node, attach an array for each type of child + # and recursively collect the children data into the array(s) + + if not obj.has_key(nodeName): + obj[nodeName] = {} + + sub_obj = osrfXMLNodeToObject(nodeChild); + + if not obj[nodeName].has_key(nodeChild.name): + # we've encountered 1 sub-node with nodeChild's name + obj[nodeName][nodeChild.name] = sub_obj[nodeChild.name] + + else: + if isinstance(obj[nodeName][nodeChild.name], list): + # we already have multiple sub-nodes with nodeChild's name + obj[nodeName][nodeChild.name].append(sub_obj[nodeChild.name]) + + else: + # we already have 1 sub-node with nodeChild's name, make + # it a list and append the current node + val = obj[nodeName][nodeChild.name] + obj[nodeName][nodeChild.name] = [ val, sub_obj[nodeChild.name] ] + + done = True + + nodeChild = nodeChild.next + + if not done: + # If the node has no children, clean up the text content + # and use that as the data + data = re.compile('^\s*').sub('', xmlNode.content) + data = re.compile('\s*$').sub('', data) + + obj[nodeName] = data + + xmlNode = xmlNode.next + + return obj + + +def osrfObjectFindPath(obj, path, idx=None): + """Searches an object along the given path for a value to return. + + Path separaters can be '/' or '.', '/' is tried first.""" + + parts = [] + + if re.compile('/').search(path): + parts = path.split('/') + else: + parts = path.split('.') + + for part in parts: + try: + o = obj[part] + except Exception: + return None + if isinstance(o,str): + return o + if isinstance(o,list): + if( idx != None ): + return o[idx] + return o + if isinstance(o,dict): + obj = o + else: + return o + + return obj + + + + diff --git a/src/python/srfsh.py b/src/python/srfsh.py new file mode 100755 index 0000000..4742246 --- /dev/null +++ b/src/python/srfsh.py @@ -0,0 +1,280 @@ +#!/usr/bin/python2.4 +import os, sys, time, readline, atexit, re +from string import * +from osrf.system import osrfConnect +from osrf.json import * +from osrf.ses import osrfClientSession +from osrf.conf import osrfConfigValue + + +# ------------------------------------------------------------------- +# main listen loop +# ------------------------------------------------------------------- +def do_loop(): + while True: + + try: + #line = raw_input("srfsh% ") + line = raw_input("\033[01;32msrfsh\033[01;34m% \033[00m") + if not len(line): + continue + if lower(line) == 'exit' or lower(line) == 'quit': + break + parts = split(line) + + command = parts[0] + + if command == 'request': + parts.pop(0) + handle_request(parts) + continue + + if command == 'math_bench': + parts.pop(0) + handle_math_bench(parts) + continue + + if command == 'help': + handle_help() + continue + + if command == 'set': + parts.pop(0) + handle_set(parts) + + if command == 'get': + parts.pop(0) + handle_get(parts) + + + + except KeyboardInterrupt: + print "" + + except EOFError: + print "exiting..." + sys.exit(0) + + +# ------------------------------------------------------------------- +# Set env variables to control behavior +# ------------------------------------------------------------------- +def handle_set(parts): + m = re.compile('(.*)=(.*)').match(parts[0]) + key = m.group(1) + val = m.group(2) + set_var(key, val) + print "%s = %s" % (key, val) + +def handle_get(parts): + try: + print get_var(parts[0]) + except: + print "" + + +# ------------------------------------------------------------------- +# Prints help info +# ------------------------------------------------------------------- +def handle_help(): + print """ + help + - show this menu + + math_bench + - runs opensrf.math requests and reports the average time + + request [, , ...] + - performs an opensrf request + + set VAR= + - sets an environment variable + + Environment variables: + SRFSH_OUTPUT = pretty - print pretty JSON and key/value pairs for network objects + = raw - print formatted JSON + """ + + + + +# ------------------------------------------------------------------- +# performs an opesnrf request +# ------------------------------------------------------------------- +def handle_request(parts): + service = parts.pop(0) + method = parts.pop(0) + jstr = '[%s]' % join(parts) + params = None + + try: + params = osrfJSONToObject(jstr) + except: + print "Error parsing JSON: %s" % jstr + return + + ses = osrfClientSession(service) + + end = None + start = time.time() + + req = ses.request2(method, tuple(params)) + + + while True: + resp = req.recv(timeout=120) + if not end: + total = time.time() - start + if not resp: break + + otp = get_var('SRFSH_OUTPUT') + if otp == 'pretty': + print osrfDebugNetworkObject(resp.content()) + else: + print osrfFormatJSON(osrfObjectToJSON(resp.content())) + + req.cleanup() + ses.cleanup() + + print '-'*60 + print "Total request time: %f" % total + print '-'*60 + + +def handle_math_bench(parts): + + count = int(parts.pop(0)) + ses = osrfClientSession('opensrf.math') + times = [] + + for i in range(100): + if i % 10: sys.stdout.write('.') + else: sys.stdout.write( str( i / 10 ) ) + print ""; + + + for i in range(count): + + starttime = time.time() + req = ses.request('add', 1, 2) + resp = req.recv(timeout=2) + endtime = time.time() + + if resp.content() == 3: + sys.stdout.write("+") + sys.stdout.flush() + times.append( endtime - starttime ) + else: + print "What happened? %s" % str(resp.content()) + + req.cleanup() + if not ( (i+1) % 100): + print ' [%d]' % (i+1) + + ses.cleanup() + total = 0 + for i in times: total += i + print "\naverage time %f" % (total / len(times)) + + + + +# ------------------------------------------------------------------- +# Defines the tab-completion handling and sets up the readline history +# ------------------------------------------------------------------- +def setup_readline(): + class SrfshCompleter(object): + def __init__(self, words): + self.words = words + self.prefix = None + + def complete(self, prefix, index): + if prefix != self.prefix: + # find all words that start with this prefix + self.matching_words = [ + w for w in self.words if w.startswith(prefix) + ] + self.prefix = prefix + try: + return self.matching_words[index] + except IndexError: + return None + + words = 'request', 'help', 'exit', 'quit', 'opensrf.settings', 'opensrf.math', 'set' + completer = SrfshCompleter(words) + readline.parse_and_bind("tab: complete") + readline.set_completer(completer.complete) + + histfile = os.path.join(get_var('HOME'), ".srfsh_history") + try: + readline.read_history_file(histfile) + except IOError: + pass + atexit.register(readline.write_history_file, histfile) + +def do_connect(): + file = os.path.join(get_var('HOME'), ".srfsh.xml") + + print_green("Connecting to opensrf...") + osrfConnect(file) + print_red('OK\n') + +def load_plugins(): + # Load the user defined external plugins + # XXX Make this a real module interface, with tab-complete words, commands, etc. + plugins = osrfConfigValue('plugins') + plugins = osrfConfigValue('plugins.plugin') + if not isinstance(plugins, list): + plugins = [plugins] + + for module in plugins: + name = module['module'] + init = module['init'] + print_green("Loading module %s..." % name) + + try: + str = 'from %s import %s\n%s()' % (name, init, init) + exec(str) + print_red('OK\n') + + except Exception, e: + sys.stderr.write("\nError importing plugin %s, with init symbol %s: \n%s\n" % (name, init, e)) + +def set_vars(): + if not get_var('SRFSH_OUTPUT'): + set_var('SRFSH_OUTPUT', 'pretty') + + +def set_var(key, val): + os.environ[key] = val + + +def get_var(key): + try: return os.environ[key] + except: return '' + + +def print_green(str): + sys.stdout.write("\033[01;32m") + sys.stdout.write(str) + sys.stdout.write("\033[00m") + sys.stdout.flush() + +def print_red(str): + sys.stdout.write("\033[01;31m") + sys.stdout.write(str) + sys.stdout.write("\033[00m") + sys.stdout.flush() + + + + +# Kick it off +set_vars() +setup_readline() +do_connect() +load_plugins() +do_loop() + + +