svn merge -r88066:88786 svn+ssh://svn.lindenlab.com/svn/linden/branches/cmake-9-merge

dataserver-is-deprecated
for-fucks-sake-whats-with-these-commit-markers

4 files changed, 801 insertions, 85 deletions

diff --git a/indra/lib/python/indra/base/cllsd_test.py b/indra/lib/python/indra/base/cllsd_test.py
new file mode 100644
index 0000000000..3af59e741a
--- /dev/null
+++ b/indra/lib/python/indra/base/cllsd_test.py
@@ -0,0 +1,51 @@
+from indra.base import llsd, lluuid
+from datetime import datetime
+import cllsd
+import time, sys
+
+class myint(int):
+    pass
+
+values = (
+    '&<>',
+    u'\u81acj',
+    llsd.uri('http://foo<'),
+    lluuid.LLUUID(),
+    llsd.LLSD(['thing']),
+    1,
+    myint(31337),
+    sys.maxint + 10,
+    llsd.binary('foo'),
+    [],
+    {},
+    {u'f&\u1212': 3},
+    3.1,
+    True,
+    None,
+    datetime.fromtimestamp(time.time()),
+    )
+
+def valuator(values):
+    for v in values:
+        yield v
+
+longvalues = () # (values, list(values), iter(values), valuator(values))
+
+for v in values + longvalues:
+    print '%r => %r' % (v, cllsd.llsd_to_xml(v))
+
+a = [[{'a':3}]] * 1000000
+
+s = time.time()
+print hash(cllsd.llsd_to_xml(a))
+e = time.time()
+t1 = e - s
+print t1
+
+s = time.time()
+print hash(llsd.LLSDXMLFormatter()._format(a))
+e = time.time()
+t2 = e - s
+print t2
+
+print 'Speedup:', t2 / t1
diff --git a/indra/lib/python/indra/ipc/siesta.py b/indra/lib/python/indra/ipc/siesta.py
new file mode 100644
index 0000000000..5fbea29339
--- /dev/null
+++ b/indra/lib/python/indra/ipc/siesta.py
@@ -0,0 +1,402 @@
+from indra.base import llsd
+from webob import exc
+import webob
+import re, socket
+
+try:
+    from cStringIO import StringIO
+except ImportError:
+    from StringIO import StringIO
+
+try:
+    import cjson
+    json_decode = cjson.decode
+    json_encode = cjson.encode
+    JsonDecodeError = cjson.DecodeError
+    JsonEncodeError = cjson.EncodeError
+except ImportError:
+    import simplejson
+    json_decode = simplejson.loads
+    json_encode = simplejson.dumps
+    JsonDecodeError = ValueError
+    JsonEncodeError = TypeError
+
+
+llsd_parsers = {
+    'application/json': json_decode,
+    'application/llsd+binary': llsd.parse_binary,
+    'application/llsd+notation': llsd.parse_notation,
+    'application/llsd+xml': llsd.parse_xml,
+    'application/xml': llsd.parse_xml,
+    }
+
+
+def mime_type(content_type):
+    '''Given a Content-Type header, return only the MIME type.'''
+
+    return content_type.split(';', 1)[0].strip().lower()
+    
+class BodyLLSD(object):
+    '''Give a webob Request or Response an llsd property.
+
+    Getting the llsd property parses the body, and caches the result.
+
+    Setting the llsd property formats a payload, and the body property
+    is set.'''
+
+    def _llsd__get(self):
+        '''Get, set, or delete the LLSD value stored in this object.'''
+
+        try:
+            return self._llsd
+        except AttributeError:
+            if not self.body:
+                raise AttributeError('No llsd attribute has been set')
+            else:
+                mtype = mime_type(self.content_type)
+                try:
+                    parser = llsd_parsers[mtype]
+                except KeyError:
+                    raise exc.HTTPUnsupportedMediaType(
+                        'Content type %s not supported' % mtype).exception
+                try:
+                    self._llsd = parser(self.body)
+                except (llsd.LLSDParseError, JsonDecodeError, TypeError), err:
+                    raise exc.HTTPBadRequest(
+                        'Could not parse body: %r' % err.args).exception
+            return self._llsd
+
+    def _llsd__set(self, val):
+        req = getattr(self, 'request', None)
+        if req is not None:
+            formatter, ctype = formatter_for_request(req)
+            self.content_type = ctype
+        else:
+            formatter, ctype = formatter_for_mime_type(
+                mime_type(self.content_type))
+        self.body = formatter(val)
+
+    def _llsd__del(self):
+        if hasattr(self, '_llsd'):
+            del self._llsd
+
+    llsd = property(_llsd__get, _llsd__set, _llsd__del)
+
+
+class Response(webob.Response, BodyLLSD):
+    '''Response class with LLSD support.
+
+    A sensible default content type is used.
+
+    Setting the llsd property also sets the body.  Getting the llsd
+    property parses the body if necessary.
+
+    If you set the body property directly, the llsd property will be
+    deleted.'''
+
+    default_content_type = 'application/llsd+xml'
+
+    def _body__set(self, body):
+        if hasattr(self, '_llsd'):
+            del self._llsd
+        super(Response, self)._body__set(body)
+
+    def cache_forever(self):
+        self.cache_expires(86400 * 365)
+
+    body = property(webob.Response._body__get, _body__set,
+                    webob.Response._body__del,
+                    webob.Response._body__get.__doc__)
+
+
+class Request(webob.Request, BodyLLSD):
+    '''Request class with LLSD support.
+
+    Sensible content type and accept headers are used by default.
+
+    Setting the llsd property also sets the body.  Getting the llsd
+    property parses the body if necessary.
+
+    If you set the body property directly, the llsd property will be
+    deleted.'''
+    
+    default_content_type = 'application/llsd+xml'
+    default_accept = ('application/llsd+xml; q=0.5, '
+                      'application/llsd+notation; q=0.3, '
+                      'application/llsd+binary; q=0.2, '
+                      'application/xml; q=0.1, '
+                      'application/json; q=0.0')
+
+    def __init__(self, environ=None, *args, **kwargs):
+        if environ is None:
+            environ = {}
+        else:
+            environ = environ.copy()
+        if 'CONTENT_TYPE' not in environ:
+            environ['CONTENT_TYPE'] = self.default_content_type
+        if 'HTTP_ACCEPT' not in environ:
+            environ['HTTP_ACCEPT'] = self.default_accept
+        super(Request, self).__init__(environ, *args, **kwargs)
+
+    def _body__set(self, body):
+        if hasattr(self, '_llsd'):
+            del self._llsd
+        super(Request, self)._body__set(body)
+
+    def path_urljoin(self, *parts):
+        return '/'.join([path_url.rstrip('/')] + list(parts))
+
+    body = property(webob.Request._body__get, _body__set,
+                    webob.Request._body__del, webob.Request._body__get.__doc__)
+
+    def create_response(self, llsd=None, status='200 OK',
+                        conditional_response=webob.NoDefault):
+        resp = self.ResponseClass(status=status, request=self,
+                                  conditional_response=conditional_response)
+        resp.llsd = llsd
+        return resp
+
+    def curl(self):
+        '''Create and fill out a pycurl easy object from this request.'''
+ 
+        import pycurl
+        c = pycurl.Curl()
+        c.setopt(pycurl.URL, self.url())
+        if self.headers:
+            c.setopt(pycurl.HTTPHEADER,
+                     ['%s: %s' % (k, self.headers[k]) for k in self.headers])
+        c.setopt(pycurl.FOLLOWLOCATION, True)
+        c.setopt(pycurl.AUTOREFERER, True)
+        c.setopt(pycurl.MAXREDIRS, 16)
+        c.setopt(pycurl.NOSIGNAL, True)
+        c.setopt(pycurl.READFUNCTION, self.body_file.read)
+        c.setopt(pycurl.SSL_VERIFYHOST, 2)
+        
+        if self.method == 'POST':
+            c.setopt(pycurl.POST, True)
+            post301 = getattr(pycurl, 'POST301', None)
+            if post301 is not None:
+                # Added in libcurl 7.17.1.
+                c.setopt(post301, True)
+        elif self.method == 'PUT':
+            c.setopt(pycurl.PUT, True)
+        elif self.method != 'GET':
+            c.setopt(pycurl.CUSTOMREQUEST, self.method)
+        return c
+
+Request.ResponseClass = Response
+Response.RequestClass = Request
+
+
+llsd_formatters = {
+    'application/json': json_encode,
+    'application/llsd+binary': llsd.format_binary,
+    'application/llsd+notation': llsd.format_notation,
+    'application/llsd+xml': llsd.format_xml,
+    'application/xml': llsd.format_xml,
+    }
+
+
+def formatter_for_mime_type(mime_type):
+    '''Return a formatter that encodes to the given MIME type.
+
+    The result is a pair of function and MIME type.'''
+
+    try:
+        return llsd_formatters[mime_type], mime_type
+    except KeyError:
+        raise exc.HTTPInternalServerError(
+            'Could not use MIME type %r to format response' %
+            mime_type).exception
+
+
+def formatter_for_request(req):
+    '''Return a formatter that encodes to the preferred type of the client.
+
+    The result is a pair of function and actual MIME type.'''
+
+    for ctype in req.accept.best_matches('application/llsd+xml'):
+        try:
+            return llsd_formatters[ctype], ctype
+        except KeyError:
+            pass
+    else:
+        raise exc.HTTPNotAcceptable().exception
+
+
+def wsgi_adapter(func, environ, start_response):
+    '''Adapt a Siesta callable to act as a WSGI application.'''
+
+    try:
+        req = Request(environ)
+        resp = func(req, **req.urlvars)
+        if not isinstance(resp, webob.Response):
+            try:
+                formatter, ctype = formatter_for_request(req)
+                resp = req.ResponseClass(formatter(resp), content_type=ctype)
+                resp._llsd = resp
+            except (JsonEncodeError, TypeError), err:
+                resp = exc.HTTPInternalServerError(
+                    detail='Could not format response')
+    except exc.HTTPException, e:
+        resp = e
+    except socket.error, e:
+        resp = exc.HTTPInternalServerError(detail=e.args[1])
+    return resp(environ, start_response)
+
+
+def llsd_callable(func):
+    '''Turn a callable into a Siesta application.'''
+
+    def replacement(environ, start_response):
+        return wsgi_adapter(func, environ, start_response)
+
+    return replacement
+
+
+def llsd_method(http_method, func):
+    def replacement(environ, start_response):
+        if environ['REQUEST_METHOD'] == http_method:
+            return wsgi_adapter(func, environ, start_response)
+        return exc.HTTPMethodNotAllowed()(environ, start_response)
+
+    return replacement
+
+
+http11_methods = 'OPTIONS GET HEAD POST PUT DELETE TRACE CONNECT'.split()
+http11_methods.sort()
+
+def llsd_class(cls):
+    '''Turn a class into a Siesta application.
+
+    A new instance is created for each request.  A HTTP method FOO is
+    turned into a call to the handle_foo method of the instance.'''
+
+    def foo(req, **kwargs):
+        instance = cls()
+        method = req.method.lower()
+        try:
+            handler = getattr(instance, 'handle_' + method)
+        except AttributeError:
+            allowed = [m for m in http11_methods
+                       if hasattr(instance, 'handle_' + m.lower())]
+            raise exc.HTTPMethodNotAllowed(
+                headers={'Allowed': ', '.join(allowed)}).exception
+        return handler(req, **kwargs)
+
+    def replacement(environ, start_response):
+        return wsgi_adapter(foo, environ, start_response)
+
+    return replacement
+
+
+def curl(reqs):
+    import pycurl
+
+    m = pycurl.CurlMulti()
+    curls = [r.curl() for r in reqs]
+    io = {}
+    for c in curls:
+        fp = StringIO()
+        hdr = StringIO()
+        c.setopt(pycurl.WRITEFUNCTION, fp.write)
+        c.setopt(pycurl.HEADERFUNCTION, hdr.write)
+        io[id(c)] = fp, hdr
+    m.handles = curls
+    try:
+        while True:
+            ret, num_handles = m.perform()
+            if ret != pycurl.E_CALL_MULTI_PERFORM:
+                break
+    finally:
+        m.close()
+
+    for req, c in zip(reqs, curls):
+        fp, hdr = io[id(c)]
+        hdr.seek(0)
+        status = hdr.readline().rstrip()
+        headers = []
+        name, values = None, None
+
+        # XXX We don't currently handle bogus header data.
+
+        for line in hdr.readlines():
+            if not line[0].isspace():
+                if name:
+                    headers.append((name, ' '.join(values)))
+                name, value = line.strip().split(':', 1)
+                value = [value]
+            else:
+                values.append(line.strip())
+        if name:
+            headers.append((name, ' '.join(values)))
+
+        resp = c.ResponseClass(fp.getvalue(), status, headers, request=req)
+
+
+route_re = re.compile(r'''
+    \{                 # exact character "{"
+    (\w+)              # variable name (restricted to a-z, 0-9, _)
+    (?:([:~])([^}]+))? # optional :type or ~regex part
+    \}                 # exact character "}"
+    ''', re.VERBOSE)
+
+predefined_regexps = {
+    'uuid': r'[a-f0-9][a-f0-9-]{31,35}',
+    'int': r'\d+',
+    }
+
+def compile_route(route):
+    fp = StringIO()
+    last_pos = 0
+    for match in route_re.finditer(route):
+        fp.write(re.escape(route[last_pos:match.start()]))
+        var_name = match.group(1)
+        sep = match.group(2)
+        expr = match.group(3)
+        if expr:
+            if sep == ':':
+                expr = predefined_regexps[expr]
+            # otherwise, treat what follows '~' as a regexp
+        else:
+            expr = '[^/]+'
+        expr = '(?P<%s>%s)' % (var_name, expr)
+        fp.write(expr)
+        last_pos = match.end()
+    fp.write(re.escape(route[last_pos:]))
+    return '^%s$' % fp.getvalue()
+
+class Router(object):
+    '''WSGI routing class.  Parses a URL and hands off a request to
+    some other WSGI application.  If no suitable application is found,
+    responds with a 404.'''
+
+    def __init__(self):
+        self.routes = []
+        self.paths = []
+
+    def add(self, route, app, methods=None):
+        self.paths.append(route)
+        self.routes.append((re.compile(compile_route(route)), app,
+                            methods and dict.fromkeys(methods)))
+
+    def __call__(self, environ, start_response):
+        path_info = environ['PATH_INFO']
+        request_method = environ['REQUEST_METHOD']
+        allowed = []
+        for regex, app, methods in self.routes:
+            m = regex.match(path_info)
+            if m:
+                if not methods or request_method in methods:
+                    environ['paste.urlvars'] = m.groupdict()
+                    return app(environ, start_response)
+                else:
+                    allowed += methods
+        if allowed:
+            allowed = dict.fromkeys(allows).keys()
+            allowed.sort()
+            resp = exc.HTTPMethodNotAllowed(
+                headers={'Allowed': ', '.join(allowed)})
+        else:
+            resp = exc.HTTPNotFound()
+        return resp(environ, start_response)
diff --git a/indra/lib/python/indra/ipc/siesta_test.py b/indra/lib/python/indra/ipc/siesta_test.py
new file mode 100644
index 0000000000..177ea710d1
--- /dev/null
+++ b/indra/lib/python/indra/ipc/siesta_test.py
@@ -0,0 +1,214 @@
+from indra.base import llsd, lluuid
+from indra.ipc import siesta
+import datetime, math, unittest
+from webob import exc
+
+
+class ClassApp(object):
+    def handle_get(self, req):
+        pass
+
+    def handle_post(self, req):
+        return req.llsd
+    
+
+def callable_app(req):
+    if req.method == 'UNDERPANTS':
+        raise exc.HTTPMethodNotAllowed()
+    elif req.method == 'GET':
+        return None
+    return req.llsd
+
+
+class TestBase:
+    def test_basic_get(self):
+        req = siesta.Request.blank('/')
+        self.assertEquals(req.get_response(self.server).body,
+                          llsd.format_xml(None))
+        
+    def test_bad_method(self):
+        req = siesta.Request.blank('/')
+        req.environ['REQUEST_METHOD'] = 'UNDERPANTS'
+        self.assertEquals(req.get_response(self.server).status_int,
+                          exc.HTTPMethodNotAllowed.code)
+        
+    json_safe = {
+        'none': None,
+        'bool_true': True,
+        'bool_false': False,
+        'int_zero': 0,
+        'int_max': 2147483647,
+        'int_min': -2147483648,
+        'long_zero': 0,
+        'long_max': 2147483647L,
+        'long_min': -2147483648L,
+        'float_zero': 0,
+        'float': math.pi,
+        'float_huge': 3.14159265358979323846e299,
+        'str_empty': '',
+        'str': 'foo',
+        u'unic\u1e51de_empty': u'',
+        u'unic\u1e51de': u'\u1e4exx\u10480',
+        }
+    json_safe['array'] = json_safe.values()
+    json_safe['tuple'] = tuple(json_safe.values())
+    json_safe['dict'] = json_safe.copy()
+
+    json_unsafe = {
+        'uuid_empty': lluuid.UUID(),
+        'uuid_full': lluuid.UUID('dc61ab0530200d7554d23510559102c1a98aab1b'),
+        'binary_empty': llsd.binary(),
+        'binary': llsd.binary('f\0\xff'),
+        'uri_empty': llsd.uri(),
+        'uri': llsd.uri('http://www.secondlife.com/'),
+        'datetime_empty': datetime.datetime(1970,1,1),
+        'datetime': datetime.datetime(1999,9,9,9,9,9),
+        }
+    json_unsafe.update(json_safe)
+    json_unsafe['array'] = json_unsafe.values()
+    json_unsafe['tuple'] = tuple(json_unsafe.values())
+    json_unsafe['dict'] = json_unsafe.copy()
+    json_unsafe['iter'] = iter(json_unsafe.values())
+
+    def _test_client_content_type_good(self, content_type, ll):
+        def run(ll):
+            req = siesta.Request.blank('/')
+            req.environ['REQUEST_METHOD'] = 'POST'
+            req.content_type = content_type
+            req.llsd = ll
+            req.accept = content_type
+            resp = req.get_response(self.server)
+            self.assertEquals(resp.status_int, 200)
+            return req, resp
+        
+        if False and isinstance(ll, dict):
+            def fixup(v):
+                if isinstance(v, float):
+                    return '%.5f' % v
+                if isinstance(v, long):
+                    return int(v)
+                if isinstance(v, (llsd.binary, llsd.uri)):
+                    return v
+                if isinstance(v, (tuple, list)):
+                    return [fixup(i) for i in v]
+                if isinstance(v, dict):
+                    return dict([(k, fixup(i)) for k, i in v.iteritems()])
+                return v
+            for k, v in ll.iteritems():
+                l = [k, v]
+                req, resp = run(l)
+                self.assertEquals(fixup(resp.llsd), fixup(l))
+
+        run(ll)
+
+    def test_client_content_type_json_good(self):
+        self._test_client_content_type_good('application/json', self.json_safe)
+
+    def test_client_content_type_llsd_xml_good(self):
+        self._test_client_content_type_good('application/llsd+xml',
+                                            self.json_unsafe)
+
+    def test_client_content_type_llsd_notation_good(self):
+        self._test_client_content_type_good('application/llsd+notation',
+                                            self.json_unsafe)
+
+    def test_client_content_type_llsd_binary_good(self):
+        self._test_client_content_type_good('application/llsd+binary',
+                                            self.json_unsafe)
+
+    def test_client_content_type_xml_good(self):
+        self._test_client_content_type_good('application/xml',
+                                            self.json_unsafe)
+
+    def _test_client_content_type_bad(self, content_type):
+        req = siesta.Request.blank('/')
+        req.environ['REQUEST_METHOD'] = 'POST'
+        req.body = '\0invalid nonsense under all encodings'
+        req.content_type = content_type
+        self.assertEquals(req.get_response(self.server).status_int,
+                          exc.HTTPBadRequest.code)
+        
+    def test_client_content_type_json_bad(self):
+        self._test_client_content_type_bad('application/json')
+
+    def test_client_content_type_llsd_xml_bad(self):
+        self._test_client_content_type_bad('application/llsd+xml')
+
+    def test_client_content_type_llsd_notation_bad(self):
+        self._test_client_content_type_bad('application/llsd+notation')
+
+    def test_client_content_type_llsd_binary_bad(self):
+        self._test_client_content_type_bad('application/llsd+binary')
+
+    def test_client_content_type_xml_bad(self):
+        self._test_client_content_type_bad('application/xml')
+
+    def test_client_content_type_bad(self):
+        req = siesta.Request.blank('/')
+        req.environ['REQUEST_METHOD'] = 'POST'
+        req.body = 'XXX'
+        req.content_type = 'application/nonsense'
+        self.assertEquals(req.get_response(self.server).status_int,
+                          exc.HTTPUnsupportedMediaType.code)
+
+    def test_request_default_content_type(self):
+        req = siesta.Request.blank('/')
+        self.assertEquals(req.content_type, req.default_content_type)
+
+    def test_request_default_accept(self):
+        req = siesta.Request.blank('/')
+        from webob import acceptparse
+        self.assertEquals(str(req.accept).replace(' ', ''),
+                          req.default_accept.replace(' ', ''))
+
+    def test_request_llsd_auto_body(self):
+        req = siesta.Request.blank('/')
+        req.llsd = {'a': 2}
+        self.assertEquals(req.body, '<?xml version="1.0" ?><llsd><map>'
+                          '<key>a</key><integer>2</integer></map></llsd>')
+
+    def test_request_llsd_mod_body_changes_llsd(self):
+        req = siesta.Request.blank('/')
+        req.llsd = {'a': 2}
+        req.body = '<?xml version="1.0" ?><llsd><integer>1337</integer></llsd>'
+        self.assertEquals(req.llsd, 1337)
+
+    def test_request_bad_llsd_fails(self):
+        def crashme(ctype):
+            def boom():
+                class foo(object): pass
+                req = siesta.Request.blank('/')
+                req.content_type = ctype
+                req.llsd = foo()
+        for mime_type in siesta.llsd_parsers:
+            self.assertRaises(TypeError, crashme(mime_type))
+
+
+class ClassServer(TestBase, unittest.TestCase):
+    def __init__(self, *args, **kwargs):
+        unittest.TestCase.__init__(self, *args, **kwargs)
+        self.server = siesta.llsd_class(ClassApp)
+
+
+class CallableServer(TestBase, unittest.TestCase):
+    def __init__(self, *args, **kwargs):
+        unittest.TestCase.__init__(self, *args, **kwargs)
+        self.server = siesta.llsd_callable(callable_app)
+
+
+class RouterServer(unittest.TestCase):
+    def test_router(self):
+        def foo(req, quux):
+            print quux
+
+        r = siesta.Router()
+        r.add('/foo/{quux:int}', siesta.llsd_callable(foo), methods=['GET'])
+        req = siesta.Request.blank('/foo/33')
+        req.get_response(r)
+
+        req = siesta.Request.blank('/foo/bar')
+        self.assertEquals(req.get_response(r).status_int,
+                          exc.HTTPNotFound.code)
+    
+if __name__ == '__main__':
+    unittest.main()
diff --git a/indra/lib/python/indra/util/llmanifest.py b/indra/lib/python/indra/util/llmanifest.py
index 9679650104..467517756a 100644
--- a/indra/lib/python/indra/util/llmanifest.py
+++ b/indra/lib/python/indra/util/llmanifest.py
@@ -5,7 +5,7 @@
 
 $LicenseInfo:firstyear=2007&license=mit$
 
-Copyright (c) 2007, Linden Research, Inc.
+Copyright (c) 2007-2008, Linden Research, Inc.
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
@@ -34,7 +34,6 @@ import fnmatch
 import getopt
 import glob
 import os
-import os.path
 import re
 import shutil
 import sys
@@ -42,10 +41,10 @@ import tarfile
 import errno
 
 def path_ancestors(path):
-    path = os.path.normpath(path)
+    drive, path = os.path.splitdrive(os.path.normpath(path))
     result = []
-    while len(path) > 0:
-        result.append(path)
+    while len(path) > 0 and path != os.path.sep:
+        result.append(drive+path)
         path, sub = os.path.split(path)
     return result
 
@@ -57,13 +56,13 @@ def proper_windows_path(path, current_platform = sys.platform):
     drive_letter = None
     rel = None
     match = re.match("/cygdrive/([a-z])/(.*)", path)
-    if(not match):
+    if not match:
         match = re.match('([a-zA-Z]):\\\(.*)', path)
-    if(not match):
+    if not match:
         return None         # not an absolute path
     drive_letter = match.group(1)
     rel = match.group(2)
-    if(current_platform == "cygwin"):
+    if current_platform == "cygwin":
         return "/cygdrive/" + drive_letter.lower() + '/' + rel.replace('\\', '/')
     else:
         return drive_letter.upper() + ':\\' + rel.replace('/', '\\')
@@ -98,6 +97,7 @@ def get_channel(srctree):
             return channel
     
 
+DEFAULT_SRCTREE = os.path.dirname(sys.argv[0])
 DEFAULT_CHANNEL = 'Second Life Release'
 
 ARGUMENTS=[
@@ -118,10 +118,12 @@ ARGUMENTS=[
         Example use: %(name)s --arch=i686
         On Linux this would try to use Linux_i686Manifest.""",
          default=""),
+    dict(name='build', description='Build directory.', default=DEFAULT_SRCTREE),
     dict(name='configuration',
          description="""The build configuration used. Only used on OS X for
         now, but it could be used for other platforms as well.""",
          default="Universal"),
+    dict(name='dest', description='Destination directory.', default=DEFAULT_SRCTREE),
     dict(name='grid',
          description="""Which grid the client will try to connect to. Even
         though it's not strictly a grid, 'firstlook' is also an acceptable
@@ -144,6 +146,15 @@ ARGUMENTS=[
          description="""The current platform, to be used for looking up which
         manifest class to run.""",
          default=get_default_platform),
+    dict(name='source',
+         description='Source directory.',
+         default=DEFAULT_SRCTREE),
+    dict(name='artwork', description='Artwork directory.', default=DEFAULT_SRCTREE),
+    dict(name='touch',
+         description="""File to touch when action is finished. Touch file will
+        contain the name of the final package in a form suitable
+        for use by a .bat file.""",
+         default=None),
     dict(name='version',
          description="""This specifies the version of Second Life that is
         being packaged up.""",
@@ -167,63 +178,75 @@ def usage(srctree=""):
             default,
             arg['description'] % nd)
 
-def main(argv=None, srctree='.', dsttree='./dst'):
-    if(argv == None):
-        argv = sys.argv
-
+def main():
     option_names = [arg['name'] + '=' for arg in ARGUMENTS]
     option_names.append('help')
-    options, remainder = getopt.getopt(argv[1:], "", option_names)
-    if len(remainder) >= 1:
-        dsttree = remainder[0]
-
-    print "Source tree:", srctree
-    print "Destination tree:", dsttree
+    options, remainder = getopt.getopt(sys.argv[1:], "", option_names)
 
     # convert options to a hash
-    args = {}
+    args = {'source':  DEFAULT_SRCTREE,
+            'artwork': DEFAULT_SRCTREE,
+            'build':   DEFAULT_SRCTREE,
+            'dest':    DEFAULT_SRCTREE }
     for opt in options:
         args[opt[0].replace("--", "")] = opt[1]
 
+    for k in 'artwork build dest source'.split():
+        args[k] = os.path.normpath(args[k])
+
+    print "Source tree:", args['source']
+    print "Artwork tree:", args['artwork']
+    print "Build tree:", args['build']
+    print "Destination tree:", args['dest']
+
     # early out for help
-    if args.has_key('help'):
+    if 'help' in args:
         # *TODO: it is a huge hack to pass around the srctree like this
-        usage(srctree)
+        usage(args['source'])
         return
 
     # defaults
     for arg in ARGUMENTS:
-        if not args.has_key(arg['name']):
+        if arg['name'] not in args:
             default = arg['default']
             if hasattr(default, '__call__'):
-                default = default(srctree)
+                default = default(args['source'])
             if default is not None:
                 args[arg['name']] = default
 
     # fix up version
-    if args.has_key('version') and type(args['version']) == str:
+    if isinstance(args.get('version'), str):
         args['version'] = args['version'].split('.')
         
     # default and agni are default
     if args['grid'] in ['default', 'agni']:
         args['grid'] = ''
 
-    if args.has_key('actions'):
+    if 'actions' in args:
         args['actions'] = args['actions'].split()
 
     # debugging
     for opt in args:
         print "Option:", opt, "=", args[opt]
 
-    wm = LLManifest.for_platform(args['platform'], args.get('arch'))(srctree, dsttree, args)
+    wm = LLManifest.for_platform(args['platform'], args.get('arch'))(args)
     wm.do(*args['actions'])
+
+    # Write out the package file in this format, so that it can easily be called
+    # and used in a .bat file - yeah, it sucks, but this is the simplest...
+    touch = args.get('touch')
+    if touch:
+        fp = open(touch, 'w')
+        fp.write('set package_file=%s\n' % wm.package_file)
+        fp.close()
+        print 'touched', touch
     return 0
 
 class LLManifestRegistry(type):
     def __init__(cls, name, bases, dct):
         super(LLManifestRegistry, cls).__init__(name, bases, dct)
         match = re.match("(\w+)Manifest", name)
-        if(match):
+        if match:
            cls.manifests[match.group(1).lower()] = cls
 
 class LLManifest(object):
@@ -235,15 +258,18 @@ class LLManifest(object):
         return self.manifests[platform.lower()]
     for_platform = classmethod(for_platform)
 
-    def __init__(self, srctree, dsttree, args):
+    def __init__(self, args):
         super(LLManifest, self).__init__()
         self.args = args
         self.file_list = []
         self.excludes = []
         self.actions = []
-        self.src_prefix = [srctree]
-        self.dst_prefix = [dsttree]
+        self.src_prefix = [args['source']]
+        self.artwork_prefix = [args['artwork']]
+        self.build_prefix = [args['build']]
+        self.dst_prefix = [args['dest']]
         self.created_paths = []
+        self.package_name = "Unknown"
         
     def default_grid(self):
         return self.args.get('grid', None) == ''
@@ -260,16 +286,20 @@ class LLManifest(object):
         in the file list by path()."""
         self.excludes.append(glob)
 
-    def prefix(self, src='', dst=None):
+    def prefix(self, src='', build=None, dst=None):
         """ Pushes a prefix onto the stack.  Until end_prefix is
         called, all relevant method calls (esp. to path()) will prefix
         paths with the entire prefix stack.  Source and destination
         prefixes can be different, though if only one is provided they
         are both equal.  To specify a no-op, use an empty string, not
         None."""
-        if(dst == None):
+        if dst is None:
             dst = src
+        if build is None:
+            build = src
         self.src_prefix.append(src)
+        self.artwork_prefix.append(src)
+        self.build_prefix.append(build)
         self.dst_prefix.append(dst)
         return True  # so that you can wrap it in an if to get indentation
 
@@ -281,14 +311,24 @@ class LLManifest(object):
         exception is raised."""
         # as an error-prevention mechanism, check the prefix and see if it matches the source or destination prefix.  If not, improper nesting may have occurred.
         src = self.src_prefix.pop()
+        artwork = self.artwork_prefix.pop()
+        build = self.build_prefix.pop()
         dst = self.dst_prefix.pop()
-        if descr and not(src == descr or dst == descr):
+        if descr and not(src == descr or build == descr or dst == descr):
             raise ValueError, "End prefix '" + descr + "' didn't match '" +src+ "' or '" +dst + "'"
 
     def get_src_prefix(self):
         """ Returns the current source prefix."""
         return os.path.join(*self.src_prefix)
 
+    def get_artwork_prefix(self):
+        """ Returns the current artwork prefix."""
+        return os.path.join(*self.artwork_prefix)
+
+    def get_build_prefix(self):
+        """ Returns the current build prefix."""
+        return os.path.join(*self.build_prefix)
+
     def get_dst_prefix(self):
         """ Returns the current destination prefix."""
         return os.path.join(*self.dst_prefix)
@@ -298,6 +338,11 @@ class LLManifest(object):
         relative to the source directory."""
         return os.path.join(self.get_src_prefix(), relpath)
 
+    def build_path_of(self, relpath):
+        """Returns the full path to a file or directory specified
+        relative to the build directory."""
+        return os.path.join(self.get_build_prefix(), relpath)
+
     def dst_path_of(self, relpath):
         """Returns the full path to a file or directory specified
         relative to the destination directory."""
@@ -329,13 +374,13 @@ class LLManifest(object):
         lines = []
         while True:
             lines.append(fd.readline())
-            if(lines[-1] == ''):
+            if lines[-1] == '':
                 break
             else:
                 print lines[-1],
         output = ''.join(lines)
         status = fd.close()
-        if(status):
+        if status:
             raise RuntimeError(
                 "Command %s returned non-zero status (%s) \noutput:\n%s"
                 % (command, status, output) )
@@ -356,7 +401,7 @@ class LLManifest(object):
         f.close()
 
     def replace_in(self, src, dst=None, searchdict={}):
-        if(dst == None):
+        if dst == None:
             dst = src
         # read src
         f = open(self.src_path_of(src), "rbU")
@@ -369,11 +414,11 @@ class LLManifest(object):
         self.created_paths.append(dst)
 
     def copy_action(self, src, dst):
-        if(src and (os.path.exists(src) or os.path.islink(src))):
+        if src and (os.path.exists(src) or os.path.islink(src)):
             # ensure that destination path exists
             self.cmakedirs(os.path.dirname(dst))
             self.created_paths.append(dst)
-            if(not os.path.isdir(src)):
+            if not os.path.isdir(src):
                 self.ccopy(src,dst)
             else:
                 # src is a dir
@@ -408,7 +453,7 @@ class LLManifest(object):
             print "Cleaning up " + c
 
     def process_file(self, src, dst):
-        if(self.includes(src, dst)):
+        if self.includes(src, dst):
 #            print src, "=>", dst
             for action in self.actions:
                 methodname = action + "_action"
@@ -416,26 +461,29 @@ class LLManifest(object):
                 if method is not None:
                     method(src, dst)
             self.file_list.append([src, dst])
+            return 1
         else:
-            print "Excluding: ", src, dst
-
+            sys.stdout.write(" (excluding %r, %r)" % (src, dst))
+            sys.stdout.flush()
+            return 0
 
     def process_directory(self, src, dst):
-        if(not self.includes(src, dst)):
-            print "Excluding: ", src, dst
-            return
+        if not self.includes(src, dst):
+            sys.stdout.write(" (excluding %r, %r)" % (src, dst))
+            sys.stdout.flush()
+            return 0
         names = os.listdir(src)
         self.cmakedirs(dst)
         errors = []
+        count = 0
         for name in names:
             srcname = os.path.join(src, name)
             dstname = os.path.join(dst, name)
             if os.path.isdir(srcname):
-                self.process_directory(srcname, dstname)
+                count += self.process_directory(srcname, dstname)
             else:
-                self.process_file(srcname, dstname)
-
-
+                count += self.process_file(srcname, dstname)
+        return count
 
     def includes(self, src, dst):
         if src:
@@ -446,9 +494,9 @@ class LLManifest(object):
 
     def remove(self, *paths):
         for path in paths:
-            if(os.path.exists(path)):
+            if os.path.exists(path):
                 print "Removing path", path
-                if(os.path.isdir(path)):
+                if os.path.isdir(path):
                     shutil.rmtree(path)
                 else:
                     os.remove(path)
@@ -457,17 +505,17 @@ class LLManifest(object):
         """ Copy a single file or symlink.  Uses filecmp to skip copying for existing files."""
         if os.path.islink(src):
             linkto = os.readlink(src)
-            if(os.path.islink(dst) or os.path.exists(dst)):
+            if os.path.islink(dst) or os.path.exists(dst):
                 os.remove(dst)  # because symlinking over an existing link fails
             os.symlink(linkto, dst)
         else:
             # Don't recopy file if it's up-to-date.
             # If we seem to be not not overwriting files that have been
             # updated, set the last arg to False, but it will take longer.
-            if(os.path.exists(dst) and filecmp.cmp(src, dst, True)):
+            if os.path.exists(dst) and filecmp.cmp(src, dst, True):
                 return
             # only copy if it's not excluded
-            if(self.includes(src, dst)):
+            if self.includes(src, dst):
                 try:
                     os.unlink(dst)
                 except OSError, err:
@@ -481,7 +529,7 @@ class LLManifest(object):
         feature that the destination directory can exist.  It
         is so dumb that Python doesn't come with this. Also it
         implements the excludes functionality."""
-        if(not self.includes(src, dst)):
+        if not self.includes(src, dst):
             return
         names = os.listdir(src)
         self.cmakedirs(dst)
@@ -512,7 +560,7 @@ class LLManifest(object):
 
     def find_existing_file(self, *list):
         for f in list:
-            if(os.path.exists(f)):
+            if os.path.exists(f):
                 return f
         # didn't find it, return last item in list
         if len(list) > 0:
@@ -535,62 +583,63 @@ class LLManifest(object):
 
 
     def wildcard_regex(self, src_glob, dst_glob):
- #       print "regex_pair:", src_glob, dst_glob
         src_re = re.escape(src_glob)
         src_re = src_re.replace('\*', '([-a-zA-Z0-9._ ]+)')
         dst_temp = dst_glob
         i = 1
-        while(dst_temp.count("*") > 0):
+        while dst_temp.count("*") > 0:
             dst_temp = dst_temp.replace('*', '\g<' + str(i) + '>', 1)
             i = i+1
- #       print "regex_result:", src_re, dst_temp
         return re.compile(src_re), dst_temp
 
     def check_file_exists(self, path):
-        if(not os.path.exists(path) and not os.path.islink(path)):
+        if not os.path.exists(path) and not os.path.islink(path):
             raise RuntimeError("Path %s doesn't exist" % (
                 os.path.normpath(os.path.join(os.getcwd(), path)),))
 
 
     wildcard_pattern = re.compile('\*')
     def expand_globs(self, src, dst):
-        def fw_slash(str):
-            return str.replace('\\', '/')
-        def os_slash(str):
-            return str.replace('/', os.path.sep)
-        dst = fw_slash(dst)
-        src = fw_slash(src)
         src_list = glob.glob(src)
-        src_re, d_template = self.wildcard_regex(src, dst)
+        src_re, d_template = self.wildcard_regex(src.replace('\\', '/'),
+                                                 dst.replace('\\', '/'))
         for s in src_list:
-            s = fw_slash(s)
-            d = src_re.sub(d_template, s)
-            #print "s:",s, "d_t", d_template, "dst", dst, "d", d
-            yield os_slash(s), os_slash(d)
+            d = src_re.sub(d_template, s.replace('\\', '/'))
+            yield os.path.normpath(s), os.path.normpath(d)
 
     def path(self, src, dst=None):
-        print "Processing", src, "=>", dst
+        sys.stdout.write("Processing %s => %s ... " % (src, dst))
+        sys.stdout.flush()
         if src == None:
             raise RuntimeError("No source file, dst is " + dst)
         if dst == None:
             dst = src
         dst = os.path.join(self.get_dst_prefix(), dst)
-        src = os.path.join(self.get_src_prefix(), src)
 
-        # expand globs
-        if(self.wildcard_pattern.search(src)):
-            for s,d in self.expand_globs(src, dst):
-                self.process_file(s, d)
-        else:
-            # if we're specifying a single path (not a glob),
-            # we should error out if it doesn't exist
-            self.check_file_exists(src)
-            # if it's a directory, recurse through it
-            if(os.path.isdir(src)):
-                self.process_directory(src, dst)
+        def try_path(src):
+            # expand globs
+            count = 0
+            if self.wildcard_pattern.search(src):
+                for s,d in self.expand_globs(src, dst):
+                    count += self.process_file(s, d)
             else:
-                self.process_file(src, dst)
-
+                # if we're specifying a single path (not a glob),
+                # we should error out if it doesn't exist
+                self.check_file_exists(src)
+                # if it's a directory, recurse through it
+                if os.path.isdir(src):
+                    count += self.process_directory(src, dst)
+                else:
+                    count += self.process_file(src, dst)
+            return count
+        try:
+            count = try_path(os.path.join(self.get_src_prefix(), src))
+        except RuntimeError:
+            try:
+                count = try_path(os.path.join(self.get_artwork_prefix(), src))
+            except RuntimeError:
+                count = try_path(os.path.join(self.get_build_prefix(), src))
+        print "%d files" % count
 
     def do(self, *actions):
         self.actions = actions