Result of svn merge -r119432:120464 svn+ssh://svn/svn/linden/branches/http_database/merge-03 into trunk. QAR-1462

6 files changed, 229 insertions, 64 deletions

diff --git a/indra/lib/python/indra/base/cllsd_test.py b/indra/lib/python/indra/base/cllsd_test.py
index 3af59e741a..0b20d99d80 100644
--- a/indra/lib/python/indra/base/cllsd_test.py
+++ b/indra/lib/python/indra/base/cllsd_test.py
@@ -10,7 +10,7 @@ values = (
     '&<>',
     u'\u81acj',
     llsd.uri('http://foo<'),
-    lluuid.LLUUID(),
+    lluuid.UUID(),
     llsd.LLSD(['thing']),
     1,
     myint(31337),
diff --git a/indra/lib/python/indra/base/llsd.py b/indra/lib/python/indra/base/llsd.py
index 9534d5935e..1190d88663 100644
--- a/indra/lib/python/indra/base/llsd.py
+++ b/indra/lib/python/indra/base/llsd.py
@@ -72,8 +72,11 @@ BOOL_FALSE = ('0', '0.0', 'false', '')
 
 
 def format_datestr(v):
-    """ Formats a datetime object into the string format shared by xml and notation serializations."""
-    return v.isoformat() + 'Z'
+    """ Formats a datetime or date object into the string format shared by xml and notation serializations."""
+    if hasattr(v, 'microsecond'):
+        return v.isoformat() + 'Z'
+    else:
+        return v.strftime('%Y-%m-%dT%H:%M:%SZ')
 
 def parse_datestr(datestr):
     """Parses a datetime object from the string format shared by xml and notation serializations."""
@@ -183,6 +186,7 @@ class LLSDXMLFormatter(object):
             unicode : self.STRING,
             uri : self.URI,
             datetime.datetime : self.DATE,
+            datetime.date : self.DATE,
             list : self.ARRAY,
             tuple : self.ARRAY,
             types.GeneratorType : self.ARRAY,
@@ -347,6 +351,7 @@ class LLSDNotationFormatter(object):
             unicode : self.STRING,
             uri : self.URI,
             datetime.datetime : self.DATE,
+            datetime.date : self.DATE,
             list : self.ARRAY,
             tuple : self.ARRAY,
             types.GeneratorType : self.ARRAY,
@@ -924,12 +929,13 @@ def _format_binary_recurse(something):
                 (type(something), something))
 
 
-def parse_binary(something):
-    header = '<?llsd/binary?>\n'
-    if not something.startswith(header):
-        raise LLSDParseError('LLSD binary encoding header not found')
-    return LLSDBinaryParser().parse(something[len(header):])
-    
+def parse_binary(binary):
+    if binary.startswith('<?llsd/binary?>'):
+        just_binary = binary.split('\n', 1)[1]
+    else:
+        just_binary = binary
+    return LLSDBinaryParser().parse(just_binary)
+
 def parse_xml(something):
     try:
         return to_python(fromstring(something)[0])
diff --git a/indra/lib/python/indra/base/metrics.py b/indra/lib/python/indra/base/metrics.py
index 8f2a85cf0e..ff8380265f 100644
--- a/indra/lib/python/indra/base/metrics.py
+++ b/indra/lib/python/indra/base/metrics.py
@@ -29,25 +29,93 @@ $/LicenseInfo$
 """
 
 import sys
-from indra.base import llsd
+try:
+    import syslog
+except ImportError:
+    # Windows
+    import sys
+    class syslog(object):
+        # wrap to a lame syslog for windows
+        _logfp = sys.stderr
+        def syslog(msg):
+            _logfp.write(msg)
+            if not msg.endswith('\n'):
+                _logfp.write('\n')
+        syslog = staticmethod(syslog)
 
-_sequence_id = 0
+from indra.base.llsd import format_notation
 
-def record_metrics(table, stats, dest=None):
+def record_metrics(table, stats):
     "Write a standard metrics log"
-    _log("LLMETRICS", table, stats, dest)
+    _log("LLMETRICS", table, stats)
 
-def record_event(table, data, dest=None):
+def record_event(table, data):
     "Write a standard logmessage log"
-    _log("LLLOGMESSAGE", table, data, dest)
+    _log("LLLOGMESSAGE", table, data)
+
+def set_destination(dest):
+    """Set the destination of metrics logs for this process.
 
-def _log(header, table, data, dest):
+    If you do not call this function prior to calling a logging
+    method, that function will open sys.stdout as a destination.
+    Attempts to set dest to None will throw a RuntimeError.
+    @param dest a file-like object which will be the destination for logs."""
     if dest is None:
-        # do this check here in case sys.stdout changes at some
-        # point. as a default parameter, it will never be
-        # re-evaluated.
-        dest = sys.stdout
+        raise RuntimeError("Attempt to unset metrics destination.")
+    global _destination
+    _destination = dest
+
+def destination():
+    """Get the destination of the metrics logs for this process.
+    Returns None if no destination is set"""
+    global _destination
+    return _destination
+
+class SysLogger(object):
+    "A file-like object which writes to syslog."
+    def __init__(self, ident='indra', logopt = None, facility = None):
+        try:
+            if logopt is None:
+                logopt = syslog.LOG_CONS | syslog.LOG_PID
+            if facility is None:
+                facility = syslog.LOG_LOCAL0
+            syslog.openlog(ident, logopt, facility)
+            import atexit
+            atexit.register(syslog.closelog)
+        except AttributeError:
+            # No syslog module on Windows
+            pass
+
+    def write(str):
+        syslog.syslog(str)
+    write = staticmethod(write)
+
+    def flush():
+        pass
+    flush = staticmethod(flush)
+
+#
+# internal API
+#
+_sequence_id = 0
+_destination = None
+
+def _next_id():
     global _sequence_id
-    print >>dest, header, "(" + str(_sequence_id) + ")",
-    print >>dest, table, llsd.format_notation(data)
+    next = _sequence_id
     _sequence_id += 1
+    return next
+
+def _dest():
+    global _destination
+    if _destination is None:
+        # this default behavior is documented in the metrics functions above.
+        _destination = sys.stdout
+    return _destination
+    
+def _log(header, table, data):
+    log_line = "%s (%d) %s %s" \
+               % (header, _next_id(), table, format_notation(data))
+    dest = _dest()
+    dest.write(log_line)
+    dest.flush()
diff --git a/indra/lib/python/indra/ipc/servicebuilder.py b/indra/lib/python/indra/ipc/servicebuilder.py
index cb43bcb26f..0a0ce2b4e2 100644
--- a/indra/lib/python/indra/ipc/servicebuilder.py
+++ b/indra/lib/python/indra/ipc/servicebuilder.py
@@ -39,6 +39,12 @@ except:
     pass
 
 _g_builder = None
+def _builder():
+    global _g_builder
+    if _g_builder is None:
+        _g_builder = ServiceBuilder()
+    return _g_builder
+
 def build(name, context={}, **kwargs):
     """ Convenience method for using a global, singleton, service builder.  Pass arguments either via a dict or via python keyword arguments, or both!
 
@@ -56,6 +62,11 @@ def build(name, context={}, **kwargs):
         _g_builder = ServiceBuilder()
     return _g_builder.buildServiceURL(name, context, **kwargs)
 
+def build_path(name, context={}, **kwargs):
+    context = context.copy()  # shouldn't modify the caller's dictionary
+    context.update(kwargs)
+    return _builder().buildPath(name, context)
+
 class ServiceBuilder(object):
     def __init__(self, services_definition = services_config):
         """\
@@ -73,12 +84,21 @@ class ServiceBuilder(object):
                 continue
             if isinstance(service_builder, dict):
                 # We will be constructing several builders
-                for name, builder in service_builder.items():
+                for name, builder in service_builder.iteritems():
                     full_builder_name = service['name'] + '-' + name
                     self.builders[full_builder_name] = builder
             else:
                 self.builders[service['name']] = service_builder
 
+    def buildPath(self, name, context):
+        """\
+        @brief given the environment on construction, return a service path.
+        @param name The name of the service.
+        @param context A dict of name value lookups for the service.
+        @returns Returns the 
+        """
+        return russ.format(self.builders[name], context)
+
     def buildServiceURL(self, name, context={}, **kwargs):
         """\
         @brief given the environment on construction, return a service URL.
diff --git a/indra/lib/python/indra/ipc/siesta.py b/indra/lib/python/indra/ipc/siesta.py
index b206f181c4..d867e71537 100644
--- a/indra/lib/python/indra/ipc/siesta.py
+++ b/indra/lib/python/indra/ipc/siesta.py
@@ -1,3 +1,32 @@
+"""\
+@file siesta.py
+@brief A tiny llsd based RESTful web services framework
+
+$LicenseInfo:firstyear=2008&license=mit$
+
+Copyright (c) 2008, Linden Research, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+$/LicenseInfo$
+"""
+
+from indra.base import config
 from indra.base import llsd
 from webob import exc
 import webob
@@ -37,11 +66,11 @@ def mime_type(content_type):
     return content_type.split(';', 1)[0].strip().lower()
     
 class BodyLLSD(object):
-    '''Give a webob Request or Response an llsd property.
+    '''Give a webob Request or Response an llsd based "content" property.
 
-    Getting the llsd property parses the body, and caches the result.
+    Getting the content property parses the body, and caches the result.
 
-    Setting the llsd property formats a payload, and the body property
+    Setting the content property formats a payload, and the body property
     is set.'''
 
     def _llsd__get(self):
@@ -80,7 +109,7 @@ class BodyLLSD(object):
         if hasattr(self, '_llsd'):
             del self._llsd
 
-    llsd = property(_llsd__get, _llsd__set, _llsd__del)
+    content = property(_llsd__get, _llsd__set, _llsd__del)
 
 
 class Response(webob.Response, BodyLLSD):
@@ -114,10 +143,10 @@ class Request(webob.Request, BodyLLSD):
 
     Sensible content type and accept headers are used by default.
 
-    Setting the llsd property also sets the body.  Getting the llsd
+    Setting the content property also sets the body. Getting the content
     property parses the body if necessary.
 
-    If you set the body property directly, the llsd property will be
+    If you set the body property directly, the content property will be
     deleted.'''
     
     default_content_type = 'application/llsd+xml'
@@ -149,11 +178,11 @@ class Request(webob.Request, BodyLLSD):
     body = property(webob.Request._body__get, _body__set,
                     webob.Request._body__del, webob.Request._body__get.__doc__)
 
-    def create_response(self, llsd=None, status='200 OK',
+    def create_response(self, content=None, status='200 OK',
                         conditional_response=webob.NoDefault):
         resp = self.ResponseClass(status=status, request=self,
                                   conditional_response=conditional_response)
-        resp.llsd = llsd
+        resp.content = content
         return resp
 
     def curl(self):
@@ -196,12 +225,18 @@ llsd_formatters = {
     'application/xml': llsd.format_xml,
     }
 
+formatter_qualities = (
+    ('application/llsd+xml', 1.0),
+    ('application/llsd+notation', 0.5),
+    ('application/llsd+binary', 0.4),
+    ('application/xml', 0.3),
+    ('application/json', 0.2),
+    )
 
 def formatter_for_mime_type(mime_type):
     '''Return a formatter that encodes to the given MIME type.
 
     The result is a pair of function and MIME type.'''
-
     try:
         return llsd_formatters[mime_type], mime_type
     except KeyError:
@@ -214,21 +249,19 @@ def formatter_for_request(req):
     '''Return a formatter that encodes to the preferred type of the client.
 
     The result is a pair of function and actual MIME type.'''
-
-    for ctype in req.accept.best_matches('application/llsd+xml'):
-        try:
-            return llsd_formatters[ctype], ctype
-        except KeyError:
-            pass
-    else:
+    ctype = req.accept.best_match(formatter_qualities)
+    try:
+        return llsd_formatters[ctype], ctype
+    except KeyError:
         raise exc.HTTPNotAcceptable().exception
 
 
 def wsgi_adapter(func, environ, start_response):
     '''Adapt a Siesta callable to act as a WSGI application.'''
-
+    # Process the request as appropriate.
     try:
         req = Request(environ)
+        #print req.urlvars
         resp = func(req, **req.urlvars)
         if not isinstance(resp, webob.Response):
             try:
@@ -281,7 +314,8 @@ def llsd_class(cls):
             allowed = [m for m in http11_methods
                        if hasattr(instance, 'handle_' + m.lower())]
             raise exc.HTTPMethodNotAllowed(
-                headers={'Allowed': ', '.join(allowed)}).exception
+                headers={'Allow': ', '.join(allowed)}).exception
+        #print "kwargs: ", kwargs
         return handler(req, **kwargs)
 
     def replacement(environ, start_response):
@@ -336,7 +370,7 @@ def curl(reqs):
 
 route_re = re.compile(r'''
     \{                 # exact character "{"
-    (\w+)              # variable name (restricted to a-z, 0-9, _)
+    (\w*)              # "config" or variable (restricted to a-z, 0-9, _)
     (?:([:~])([^}]+))? # optional :type or ~regex part
     \}                 # exact character "}"
     ''', re.VERBOSE)
@@ -344,27 +378,37 @@ route_re = re.compile(r'''
 predefined_regexps = {
     'uuid': r'[a-f0-9][a-f0-9-]{31,35}',
     'int': r'\d+',
+    'host': r'[a-z0-9][a-z0-9\-\.]*',
     }
 
 def compile_route(route):
     fp = StringIO()
     last_pos = 0
     for match in route_re.finditer(route):
+        #print "matches: ", match.groups()
         fp.write(re.escape(route[last_pos:match.start()]))
         var_name = match.group(1)
         sep = match.group(2)
         expr = match.group(3)
-        if expr:
-            if sep == ':':
-                expr = predefined_regexps[expr]
-            # otherwise, treat what follows '~' as a regexp
+        if var_name == 'config':
+            expr = re.escape(str(config.get(var_name)))
         else:
-            expr = '[^/]+'
-        expr = '(?P<%s>%s)' % (var_name, expr)
+            if expr:
+                if sep == ':':
+                    expr = predefined_regexps[expr]
+                # otherwise, treat what follows '~' as a regexp
+            else:
+                expr = '[^/]+'
+            if var_name != '':
+                expr = '(?P<%s>%s)' % (var_name, expr)
+            else:
+                expr = '(%s)' % (expr,)
         fp.write(expr)
         last_pos = match.end()
     fp.write(re.escape(route[last_pos:]))
-    return '^%s$' % fp.getvalue()
+    compiled_route = '^%s$' % fp.getvalue()
+    #print route, "->", compiled_route
+    return compiled_route
 
 class Router(object):
     '''WSGI routing class.  Parses a URL and hands off a request to
@@ -372,21 +416,43 @@ class Router(object):
     responds with a 404.'''
 
     def __init__(self):
-        self.routes = []
-        self.paths = []
+        self._new_routes = []
+        self._routes = []
+        self._paths = []
 
     def add(self, route, app, methods=None):
-        self.paths.append(route)
-        self.routes.append((re.compile(compile_route(route)), app,
-                            methods and dict.fromkeys(methods)))
+        self._new_routes.append((route, app, methods))
+
+    def _create_routes(self):
+        for route, app, methods in self._new_routes:
+            self._paths.append(route)
+            self._routes.append(
+                (re.compile(compile_route(route)),
+                 app,
+                 methods and dict.fromkeys(methods)))
+        self._new_routes = []
 
     def __call__(self, environ, start_response):
+        # load up the config from the config file. Only needs to be
+        # done once per interpreter. This is the entry point of all
+        # siesta applications, so this is where we trap it.
+        _conf = config.get_config()
+        if _conf is None:
+            import os.path
+            fname = os.path.join(
+                environ.get('ll.config_dir', '/local/linden/etc'),
+                'indra.xml')
+            config.load(fname)
+
+        # proceed with handling the request
+        self._create_routes()
         path_info = environ['PATH_INFO']
         request_method = environ['REQUEST_METHOD']
         allowed = []
-        for regex, app, methods in self.routes:
+        for regex, app, methods in self._routes:
             m = regex.match(path_info)
             if m:
+                #print "groupdict:",m.groupdict()
                 if not methods or request_method in methods:
                     environ['paste.urlvars'] = m.groupdict()
                     return app(environ, start_response)
@@ -396,7 +462,7 @@ class Router(object):
             allowed = dict.fromkeys(allows).keys()
             allowed.sort()
             resp = exc.HTTPMethodNotAllowed(
-                headers={'Allowed': ', '.join(allowed)})
+                headers={'Allow': ', '.join(allowed)})
         else:
             resp = exc.HTTPNotFound()
         return resp(environ, start_response)
diff --git a/indra/lib/python/indra/util/named_query.py b/indra/lib/python/indra/util/named_query.py
index 59c37a7218..cdce8237c8 100644
--- a/indra/lib/python/indra/util/named_query.py
+++ b/indra/lib/python/indra/util/named_query.py
@@ -47,10 +47,8 @@ except NameError:
 from indra.base import llsd
 from indra.base import config
 
-DEBUG = False
-
-NQ_FILE_SUFFIX = config.get('named-query-file-suffix', '.nq')
-NQ_FILE_SUFFIX_LEN  = len(NQ_FILE_SUFFIX)
+NQ_FILE_SUFFIX = None
+NQ_FILE_SUFFIX_LEN = None
 
 _g_named_manager = None
 
@@ -60,6 +58,11 @@ def _init_g_named_manager(sql_dir = None):
 
     This function is intended entirely for testing purposes,
     because it's tricky to control the config from inside a test."""
+    global NQ_FILE_SUFFIX
+    NQ_FILE_SUFFIX = config.get('named-query-file-suffix', '.nq')
+    global NQ_FILE_SUFFIX_LEN
+    NQ_FILE_SUFFIX_LEN  = len(NQ_FILE_SUFFIX)
+
     if sql_dir is None:
         sql_dir = config.get('named-query-base-dir')
 
@@ -73,11 +76,11 @@ def _init_g_named_manager(sql_dir = None):
     _g_named_manager = NamedQueryManager(
         os.path.abspath(os.path.realpath(sql_dir)))
 
-def get(name):
+def get(name, schema = None):
     "Get the named query object to be used to perform queries"
     if _g_named_manager is None:
         _init_g_named_manager()
-    return _g_named_manager.get(name)
+    return _g_named_manager.get(name).for_schema(schema)
 
 def sql(connection, name, params):
     # use module-global NamedQuery object to perform default substitution
@@ -330,6 +333,8 @@ class NamedQuery(object):
 
     def for_schema(self, db_name):
         "Look trough the alternates and return the correct query"
+        if db_name is None:
+            return self
         try:
             return self._alternative[db_name]
         except KeyError, e:
@@ -359,10 +364,10 @@ class NamedQuery(object):
         if DEBUG:
             print "SQL:", self.sql(connection, params)
         rows = cursor.execute(full_query, params)
-        
+
         # *NOTE: the expect_rows argument is a very cheesy way to get some
         # validation on the result set.  If you want to add more expectation
-        # logic, do something more object-oriented and flexible.  Or use an ORM.
+        # logic, do something more object-oriented and flexible. Or use an ORM.
         if(self._return_as_map):
             expect_rows = 1
         if expect_rows is not None and rows != expect_rows: