diff options
Diffstat (limited to 'indra/lib')
| -rw-r--r-- | indra/lib/python/indra/base/cllsd_test.py | 2 | ||||
| -rw-r--r-- | indra/lib/python/indra/base/llsd.py | 22 | ||||
| -rw-r--r-- | indra/lib/python/indra/base/metrics.py | 94 | ||||
| -rw-r--r-- | indra/lib/python/indra/ipc/servicebuilder.py | 22 | ||||
| -rw-r--r-- | indra/lib/python/indra/ipc/siesta.py | 132 | ||||
| -rw-r--r-- | indra/lib/python/indra/util/named_query.py | 20 |
6 files changed, 229 insertions, 63 deletions
diff --git a/indra/lib/python/indra/base/cllsd_test.py b/indra/lib/python/indra/base/cllsd_test.py index 3af59e741a..0b20d99d80 100644 --- a/indra/lib/python/indra/base/cllsd_test.py +++ b/indra/lib/python/indra/base/cllsd_test.py @@ -10,7 +10,7 @@ values = ( '&<>', u'\u81acj', llsd.uri('http://foo<'), - lluuid.LLUUID(), + lluuid.UUID(), llsd.LLSD(['thing']), 1, myint(31337), diff --git a/indra/lib/python/indra/base/llsd.py b/indra/lib/python/indra/base/llsd.py index 9534d5935e..1190d88663 100644 --- a/indra/lib/python/indra/base/llsd.py +++ b/indra/lib/python/indra/base/llsd.py @@ -72,8 +72,11 @@ BOOL_FALSE = ('0', '0.0', 'false', '') def format_datestr(v): - """ Formats a datetime object into the string format shared by xml and notation serializations.""" - return v.isoformat() + 'Z' + """ Formats a datetime or date object into the string format shared by xml and notation serializations.""" + if hasattr(v, 'microsecond'): + return v.isoformat() + 'Z' + else: + return v.strftime('%Y-%m-%dT%H:%M:%SZ') def parse_datestr(datestr): """Parses a datetime object from the string format shared by xml and notation serializations.""" @@ -183,6 +186,7 @@ class LLSDXMLFormatter(object): unicode : self.STRING, uri : self.URI, datetime.datetime : self.DATE, + datetime.date : self.DATE, list : self.ARRAY, tuple : self.ARRAY, types.GeneratorType : self.ARRAY, @@ -347,6 +351,7 @@ class LLSDNotationFormatter(object): unicode : self.STRING, uri : self.URI, datetime.datetime : self.DATE, + datetime.date : self.DATE, list : self.ARRAY, tuple : self.ARRAY, types.GeneratorType : self.ARRAY, @@ -924,12 +929,13 @@ def _format_binary_recurse(something): (type(something), something)) -def parse_binary(something): - header = '<?llsd/binary?>\n' - if not something.startswith(header): - raise LLSDParseError('LLSD binary encoding header not found') - return LLSDBinaryParser().parse(something[len(header):]) - +def parse_binary(binary): + if binary.startswith('<?llsd/binary?>'): + just_binary = binary.split('\n', 1)[1] + else: + just_binary = binary + return LLSDBinaryParser().parse(just_binary) + def parse_xml(something): try: return to_python(fromstring(something)[0]) diff --git a/indra/lib/python/indra/base/metrics.py b/indra/lib/python/indra/base/metrics.py index 8f2a85cf0e..ff8380265f 100644 --- a/indra/lib/python/indra/base/metrics.py +++ b/indra/lib/python/indra/base/metrics.py @@ -29,25 +29,93 @@ $/LicenseInfo$ """ import sys -from indra.base import llsd +try: + import syslog +except ImportError: + # Windows + import sys + class syslog(object): + # wrap to a lame syslog for windows + _logfp = sys.stderr + def syslog(msg): + _logfp.write(msg) + if not msg.endswith('\n'): + _logfp.write('\n') + syslog = staticmethod(syslog) -_sequence_id = 0 +from indra.base.llsd import format_notation -def record_metrics(table, stats, dest=None): +def record_metrics(table, stats): "Write a standard metrics log" - _log("LLMETRICS", table, stats, dest) + _log("LLMETRICS", table, stats) -def record_event(table, data, dest=None): +def record_event(table, data): "Write a standard logmessage log" - _log("LLLOGMESSAGE", table, data, dest) + _log("LLLOGMESSAGE", table, data) + +def set_destination(dest): + """Set the destination of metrics logs for this process. -def _log(header, table, data, dest): + If you do not call this function prior to calling a logging + method, that function will open sys.stdout as a destination. + Attempts to set dest to None will throw a RuntimeError. + @param dest a file-like object which will be the destination for logs.""" if dest is None: - # do this check here in case sys.stdout changes at some - # point. as a default parameter, it will never be - # re-evaluated. - dest = sys.stdout + raise RuntimeError("Attempt to unset metrics destination.") + global _destination + _destination = dest + +def destination(): + """Get the destination of the metrics logs for this process. + Returns None if no destination is set""" + global _destination + return _destination + +class SysLogger(object): + "A file-like object which writes to syslog." + def __init__(self, ident='indra', logopt = None, facility = None): + try: + if logopt is None: + logopt = syslog.LOG_CONS | syslog.LOG_PID + if facility is None: + facility = syslog.LOG_LOCAL0 + syslog.openlog(ident, logopt, facility) + import atexit + atexit.register(syslog.closelog) + except AttributeError: + # No syslog module on Windows + pass + + def write(str): + syslog.syslog(str) + write = staticmethod(write) + + def flush(): + pass + flush = staticmethod(flush) + +# +# internal API +# +_sequence_id = 0 +_destination = None + +def _next_id(): global _sequence_id - print >>dest, header, "(" + str(_sequence_id) + ")", - print >>dest, table, llsd.format_notation(data) + next = _sequence_id _sequence_id += 1 + return next + +def _dest(): + global _destination + if _destination is None: + # this default behavior is documented in the metrics functions above. + _destination = sys.stdout + return _destination + +def _log(header, table, data): + log_line = "%s (%d) %s %s" \ + % (header, _next_id(), table, format_notation(data)) + dest = _dest() + dest.write(log_line) + dest.flush() diff --git a/indra/lib/python/indra/ipc/servicebuilder.py b/indra/lib/python/indra/ipc/servicebuilder.py index cb43bcb26f..0a0ce2b4e2 100644 --- a/indra/lib/python/indra/ipc/servicebuilder.py +++ b/indra/lib/python/indra/ipc/servicebuilder.py @@ -39,6 +39,12 @@ except: pass _g_builder = None +def _builder(): + global _g_builder + if _g_builder is None: + _g_builder = ServiceBuilder() + return _g_builder + def build(name, context={}, **kwargs): """ Convenience method for using a global, singleton, service builder. Pass arguments either via a dict or via python keyword arguments, or both! @@ -56,6 +62,11 @@ def build(name, context={}, **kwargs): _g_builder = ServiceBuilder() return _g_builder.buildServiceURL(name, context, **kwargs) +def build_path(name, context={}, **kwargs): + context = context.copy() # shouldn't modify the caller's dictionary + context.update(kwargs) + return _builder().buildPath(name, context) + class ServiceBuilder(object): def __init__(self, services_definition = services_config): """\ @@ -73,12 +84,21 @@ class ServiceBuilder(object): continue if isinstance(service_builder, dict): # We will be constructing several builders - for name, builder in service_builder.items(): + for name, builder in service_builder.iteritems(): full_builder_name = service['name'] + '-' + name self.builders[full_builder_name] = builder else: self.builders[service['name']] = service_builder + def buildPath(self, name, context): + """\ + @brief given the environment on construction, return a service path. + @param name The name of the service. + @param context A dict of name value lookups for the service. + @returns Returns the + """ + return russ.format(self.builders[name], context) + def buildServiceURL(self, name, context={}, **kwargs): """\ @brief given the environment on construction, return a service URL. diff --git a/indra/lib/python/indra/ipc/siesta.py b/indra/lib/python/indra/ipc/siesta.py index b206f181c4..d867e71537 100644 --- a/indra/lib/python/indra/ipc/siesta.py +++ b/indra/lib/python/indra/ipc/siesta.py @@ -1,3 +1,32 @@ +"""\ +@file siesta.py +@brief A tiny llsd based RESTful web services framework + +$LicenseInfo:firstyear=2008&license=mit$ + +Copyright (c) 2008, Linden Research, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +$/LicenseInfo$ +""" + +from indra.base import config from indra.base import llsd from webob import exc import webob @@ -37,11 +66,11 @@ def mime_type(content_type): return content_type.split(';', 1)[0].strip().lower() class BodyLLSD(object): - '''Give a webob Request or Response an llsd property. + '''Give a webob Request or Response an llsd based "content" property. - Getting the llsd property parses the body, and caches the result. + Getting the content property parses the body, and caches the result. - Setting the llsd property formats a payload, and the body property + Setting the content property formats a payload, and the body property is set.''' def _llsd__get(self): @@ -80,7 +109,7 @@ class BodyLLSD(object): if hasattr(self, '_llsd'): del self._llsd - llsd = property(_llsd__get, _llsd__set, _llsd__del) + content = property(_llsd__get, _llsd__set, _llsd__del) class Response(webob.Response, BodyLLSD): @@ -114,10 +143,10 @@ class Request(webob.Request, BodyLLSD): Sensible content type and accept headers are used by default. - Setting the llsd property also sets the body. Getting the llsd + Setting the content property also sets the body. Getting the content property parses the body if necessary. - If you set the body property directly, the llsd property will be + If you set the body property directly, the content property will be deleted.''' default_content_type = 'application/llsd+xml' @@ -149,11 +178,11 @@ class Request(webob.Request, BodyLLSD): body = property(webob.Request._body__get, _body__set, webob.Request._body__del, webob.Request._body__get.__doc__) - def create_response(self, llsd=None, status='200 OK', + def create_response(self, content=None, status='200 OK', conditional_response=webob.NoDefault): resp = self.ResponseClass(status=status, request=self, conditional_response=conditional_response) - resp.llsd = llsd + resp.content = content return resp def curl(self): @@ -196,12 +225,18 @@ llsd_formatters = { 'application/xml': llsd.format_xml, } +formatter_qualities = ( + ('application/llsd+xml', 1.0), + ('application/llsd+notation', 0.5), + ('application/llsd+binary', 0.4), + ('application/xml', 0.3), + ('application/json', 0.2), + ) def formatter_for_mime_type(mime_type): '''Return a formatter that encodes to the given MIME type. The result is a pair of function and MIME type.''' - try: return llsd_formatters[mime_type], mime_type except KeyError: @@ -214,21 +249,19 @@ def formatter_for_request(req): '''Return a formatter that encodes to the preferred type of the client. The result is a pair of function and actual MIME type.''' - - for ctype in req.accept.best_matches('application/llsd+xml'): - try: - return llsd_formatters[ctype], ctype - except KeyError: - pass - else: + ctype = req.accept.best_match(formatter_qualities) + try: + return llsd_formatters[ctype], ctype + except KeyError: raise exc.HTTPNotAcceptable().exception def wsgi_adapter(func, environ, start_response): '''Adapt a Siesta callable to act as a WSGI application.''' - + # Process the request as appropriate. try: req = Request(environ) + #print req.urlvars resp = func(req, **req.urlvars) if not isinstance(resp, webob.Response): try: @@ -281,7 +314,8 @@ def llsd_class(cls): allowed = [m for m in http11_methods if hasattr(instance, 'handle_' + m.lower())] raise exc.HTTPMethodNotAllowed( - headers={'Allowed': ', '.join(allowed)}).exception + headers={'Allow': ', '.join(allowed)}).exception + #print "kwargs: ", kwargs return handler(req, **kwargs) def replacement(environ, start_response): @@ -336,7 +370,7 @@ def curl(reqs): route_re = re.compile(r''' \{ # exact character "{" - (\w+) # variable name (restricted to a-z, 0-9, _) + (\w*) # "config" or variable (restricted to a-z, 0-9, _) (?:([:~])([^}]+))? # optional :type or ~regex part \} # exact character "}" ''', re.VERBOSE) @@ -344,27 +378,37 @@ route_re = re.compile(r''' predefined_regexps = { 'uuid': r'[a-f0-9][a-f0-9-]{31,35}', 'int': r'\d+', + 'host': r'[a-z0-9][a-z0-9\-\.]*', } def compile_route(route): fp = StringIO() last_pos = 0 for match in route_re.finditer(route): + #print "matches: ", match.groups() fp.write(re.escape(route[last_pos:match.start()])) var_name = match.group(1) sep = match.group(2) expr = match.group(3) - if expr: - if sep == ':': - expr = predefined_regexps[expr] - # otherwise, treat what follows '~' as a regexp + if var_name == 'config': + expr = re.escape(str(config.get(var_name))) else: - expr = '[^/]+' - expr = '(?P<%s>%s)' % (var_name, expr) + if expr: + if sep == ':': + expr = predefined_regexps[expr] + # otherwise, treat what follows '~' as a regexp + else: + expr = '[^/]+' + if var_name != '': + expr = '(?P<%s>%s)' % (var_name, expr) + else: + expr = '(%s)' % (expr,) fp.write(expr) last_pos = match.end() fp.write(re.escape(route[last_pos:])) - return '^%s$' % fp.getvalue() + compiled_route = '^%s$' % fp.getvalue() + #print route, "->", compiled_route + return compiled_route class Router(object): '''WSGI routing class. Parses a URL and hands off a request to @@ -372,21 +416,43 @@ class Router(object): responds with a 404.''' def __init__(self): - self.routes = [] - self.paths = [] + self._new_routes = [] + self._routes = [] + self._paths = [] def add(self, route, app, methods=None): - self.paths.append(route) - self.routes.append((re.compile(compile_route(route)), app, - methods and dict.fromkeys(methods))) + self._new_routes.append((route, app, methods)) + + def _create_routes(self): + for route, app, methods in self._new_routes: + self._paths.append(route) + self._routes.append( + (re.compile(compile_route(route)), + app, + methods and dict.fromkeys(methods))) + self._new_routes = [] def __call__(self, environ, start_response): + # load up the config from the config file. Only needs to be + # done once per interpreter. This is the entry point of all + # siesta applications, so this is where we trap it. + _conf = config.get_config() + if _conf is None: + import os.path + fname = os.path.join( + environ.get('ll.config_dir', '/local/linden/etc'), + 'indra.xml') + config.load(fname) + + # proceed with handling the request + self._create_routes() path_info = environ['PATH_INFO'] request_method = environ['REQUEST_METHOD'] allowed = [] - for regex, app, methods in self.routes: + for regex, app, methods in self._routes: m = regex.match(path_info) if m: + #print "groupdict:",m.groupdict() if not methods or request_method in methods: environ['paste.urlvars'] = m.groupdict() return app(environ, start_response) @@ -396,7 +462,7 @@ class Router(object): allowed = dict.fromkeys(allows).keys() allowed.sort() resp = exc.HTTPMethodNotAllowed( - headers={'Allowed': ', '.join(allowed)}) + headers={'Allow': ', '.join(allowed)}) else: resp = exc.HTTPNotFound() return resp(environ, start_response) diff --git a/indra/lib/python/indra/util/named_query.py b/indra/lib/python/indra/util/named_query.py index 59c37a7218..693b483f79 100644 --- a/indra/lib/python/indra/util/named_query.py +++ b/indra/lib/python/indra/util/named_query.py @@ -48,9 +48,8 @@ from indra.base import llsd from indra.base import config DEBUG = False - -NQ_FILE_SUFFIX = config.get('named-query-file-suffix', '.nq') -NQ_FILE_SUFFIX_LEN = len(NQ_FILE_SUFFIX) +NQ_FILE_SUFFIX = None +NQ_FILE_SUFFIX_LEN = None _g_named_manager = None @@ -60,6 +59,11 @@ def _init_g_named_manager(sql_dir = None): This function is intended entirely for testing purposes, because it's tricky to control the config from inside a test.""" + global NQ_FILE_SUFFIX + NQ_FILE_SUFFIX = config.get('named-query-file-suffix', '.nq') + global NQ_FILE_SUFFIX_LEN + NQ_FILE_SUFFIX_LEN = len(NQ_FILE_SUFFIX) + if sql_dir is None: sql_dir = config.get('named-query-base-dir') @@ -73,11 +77,11 @@ def _init_g_named_manager(sql_dir = None): _g_named_manager = NamedQueryManager( os.path.abspath(os.path.realpath(sql_dir))) -def get(name): +def get(name, schema = None): "Get the named query object to be used to perform queries" if _g_named_manager is None: _init_g_named_manager() - return _g_named_manager.get(name) + return _g_named_manager.get(name).for_schema(schema) def sql(connection, name, params): # use module-global NamedQuery object to perform default substitution @@ -330,6 +334,8 @@ class NamedQuery(object): def for_schema(self, db_name): "Look trough the alternates and return the correct query" + if db_name is None: + return self try: return self._alternative[db_name] except KeyError, e: @@ -359,10 +365,10 @@ class NamedQuery(object): if DEBUG: print "SQL:", self.sql(connection, params) rows = cursor.execute(full_query, params) - + # *NOTE: the expect_rows argument is a very cheesy way to get some # validation on the result set. If you want to add more expectation - # logic, do something more object-oriented and flexible. Or use an ORM. + # logic, do something more object-oriented and flexible. Or use an ORM. if(self._return_as_map): expect_rows = 1 if expect_rows is not None and rows != expect_rows: |