From 68832f56ba09541f01a246574b141e3ab5fb62b8 Mon Sep 17 00:00:00 2001 From: Glenn Glazer Date: Thu, 14 Jul 2016 08:37:23 -0700 Subject: add dependent modules --- indra/newview/viewer_manifest.py | 4 +- indra/viewer_components/manager/base/llsd.py | 1052 ++++++++++++++++++++ indra/viewer_components/manager/base/lluuid.py | 319 ++++++ indra/viewer_components/manager/llsd.py | 1052 -------------------- .../manager/util/fastest_elementtree.py | 64 ++ 5 files changed, 1438 insertions(+), 1053 deletions(-) create mode 100755 indra/viewer_components/manager/base/llsd.py create mode 100755 indra/viewer_components/manager/base/lluuid.py delete mode 100755 indra/viewer_components/manager/llsd.py create mode 100755 indra/viewer_components/manager/util/fastest_elementtree.py (limited to 'indra') diff --git a/indra/newview/viewer_manifest.py b/indra/newview/viewer_manifest.py index 9194a2161a..ac9b8d8ebe 100755 --- a/indra/newview/viewer_manifest.py +++ b/indra/newview/viewer_manifest.py @@ -712,9 +712,11 @@ class Darwin_i386_Manifest(ViewerManifest): if self.prefix(dst="MacOS"): self.path2basename("../viewer_components/updater/scripts/darwin", "*.py") - #this copies over the python wrapper script, see SL-322 + #this copies over the python wrapper script and associated utilities, see SL-321, SL-322 and SL-323 self.path2basename("../viewer_components/manager","SL_Launcher") self.path2basename("../viewer_components/manager","*.py") + self.path2basename("../viewer_components/manager/base","*py") + self.path2basename("../viewer_components/manager/util","*py") self.end_prefix() # most everything goes in the Resources directory diff --git a/indra/viewer_components/manager/base/llsd.py b/indra/viewer_components/manager/base/llsd.py new file mode 100755 index 0000000000..4527b115f9 --- /dev/null +++ b/indra/viewer_components/manager/base/llsd.py @@ -0,0 +1,1052 @@ +"""\ +@file llsd.py +@brief Types as well as parsing and formatting functions for handling LLSD. + +$LicenseInfo:firstyear=2006&license=mit$ + +Copyright (c) 2006-2009, Linden Research, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +$/LicenseInfo$ +""" + +import datetime +import base64 +import string +import struct +import time +import types +import re + +from indra.util.fastest_elementtree import ElementTreeError, fromstring +from indra.base import lluuid + +# cllsd.c in server/server-1.25 has memory leaks, +# so disabling cllsd for now +#try: +# import cllsd +#except ImportError: +# cllsd = None +cllsd = None + +int_regex = re.compile(r"[-+]?\d+") +real_regex = re.compile(r"[-+]?(\d+(\.\d*)?|\d*\.\d+)([eE][-+]?\d+)?") +alpha_regex = re.compile(r"[a-zA-Z]+") +date_regex = re.compile(r"(?P\d{4})-(?P\d{2})-(?P\d{2})T" + r"(?P\d{2}):(?P\d{2}):(?P\d{2})" + r"(?P(\.\d+)?)Z") +#date: d"YYYY-MM-DDTHH:MM:SS.FFFFFFZ" + +class LLSDParseError(Exception): + pass + +class LLSDSerializationError(TypeError): + pass + + +class binary(str): + pass + +class uri(str): + pass + + +BOOL_TRUE = ('1', '1.0', 'true') +BOOL_FALSE = ('0', '0.0', 'false', '') + + +def format_datestr(v): + """ Formats a datetime or date object into the string format shared by xml and notation serializations.""" + if hasattr(v, 'microsecond'): + return v.isoformat() + 'Z' + else: + return v.strftime('%Y-%m-%dT%H:%M:%SZ') + +def parse_datestr(datestr): + """Parses a datetime object from the string format shared by xml and notation serializations.""" + if datestr == "": + return datetime.datetime(1970, 1, 1) + + match = re.match(date_regex, datestr) + if not match: + raise LLSDParseError("invalid date string '%s'." % datestr) + + year = int(match.group('year')) + month = int(match.group('month')) + day = int(match.group('day')) + hour = int(match.group('hour')) + minute = int(match.group('minute')) + second = int(match.group('second')) + seconds_float = match.group('second_float') + microsecond = 0 + if seconds_float: + microsecond = int(float('0' + seconds_float) * 1e6) + return datetime.datetime(year, month, day, hour, minute, second, microsecond) + + +def bool_to_python(node): + val = node.text or '' + if val in BOOL_TRUE: + return True + else: + return False + +def int_to_python(node): + val = node.text or '' + if not val.strip(): + return 0 + return int(val) + +def real_to_python(node): + val = node.text or '' + if not val.strip(): + return 0.0 + return float(val) + +def uuid_to_python(node): + return lluuid.UUID(node.text) + +def str_to_python(node): + return node.text or '' + +def bin_to_python(node): + return binary(base64.decodestring(node.text or '')) + +def date_to_python(node): + val = node.text or '' + if not val: + val = "1970-01-01T00:00:00Z" + return parse_datestr(val) + + +def uri_to_python(node): + val = node.text or '' + if not val: + return None + return uri(val) + +def map_to_python(node): + result = {} + for index in range(len(node))[::2]: + result[node[index].text] = to_python(node[index+1]) + return result + +def array_to_python(node): + return [to_python(child) for child in node] + + +NODE_HANDLERS = dict( + undef=lambda x: None, + boolean=bool_to_python, + integer=int_to_python, + real=real_to_python, + uuid=uuid_to_python, + string=str_to_python, + binary=bin_to_python, + date=date_to_python, + uri=uri_to_python, + map=map_to_python, + array=array_to_python, + ) + +def to_python(node): + return NODE_HANDLERS[node.tag](node) + +class Nothing(object): + pass + + +class LLSDXMLFormatter(object): + def __init__(self): + self.type_map = { + type(None) : self.UNDEF, + bool : self.BOOLEAN, + int : self.INTEGER, + long : self.INTEGER, + float : self.REAL, + lluuid.UUID : self.UUID, + binary : self.BINARY, + str : self.STRING, + unicode : self.STRING, + uri : self.URI, + datetime.datetime : self.DATE, + datetime.date : self.DATE, + list : self.ARRAY, + tuple : self.ARRAY, + types.GeneratorType : self.ARRAY, + dict : self.MAP, + LLSD : self.LLSD + } + + def elt(self, name, contents=None): + if(contents is None or contents is ''): + return "<%s />" % (name,) + else: + if type(contents) is unicode: + contents = contents.encode('utf-8') + return "<%s>%s" % (name, contents, name) + + def xml_esc(self, v): + if type(v) is unicode: + v = v.encode('utf-8') + return v.replace('&', '&').replace('<', '<').replace('>', '>') + + def LLSD(self, v): + return self.generate(v.thing) + def UNDEF(self, v): + return self.elt('undef') + def BOOLEAN(self, v): + if v: + return self.elt('boolean', 'true') + else: + return self.elt('boolean', 'false') + def INTEGER(self, v): + return self.elt('integer', v) + def REAL(self, v): + return self.elt('real', v) + def UUID(self, v): + if(v.isNull()): + return self.elt('uuid') + else: + return self.elt('uuid', v) + def BINARY(self, v): + return self.elt('binary', base64.encodestring(v)) + def STRING(self, v): + return self.elt('string', self.xml_esc(v)) + def URI(self, v): + return self.elt('uri', self.xml_esc(str(v))) + def DATE(self, v): + return self.elt('date', format_datestr(v)) + def ARRAY(self, v): + return self.elt('array', ''.join([self.generate(item) for item in v])) + def MAP(self, v): + return self.elt( + 'map', + ''.join(["%s%s" % (self.elt('key', self.xml_esc(str(key))), self.generate(value)) + for key, value in v.items()])) + + typeof = type + def generate(self, something): + t = self.typeof(something) + if self.type_map.has_key(t): + return self.type_map[t](something) + else: + raise LLSDSerializationError("Cannot serialize unknown type: %s (%s)" % ( + t, something)) + + def _format(self, something): + return '' + self.elt("llsd", self.generate(something)) + + def format(self, something): + if cllsd: + return cllsd.llsd_to_xml(something) + return self._format(something) + +_g_xml_formatter = None +def format_xml(something): + global _g_xml_formatter + if _g_xml_formatter is None: + _g_xml_formatter = LLSDXMLFormatter() + return _g_xml_formatter.format(something) + +class LLSDXMLPrettyFormatter(LLSDXMLFormatter): + def __init__(self, indent_atom = None): + # Call the super class constructor so that we have the type map + super(LLSDXMLPrettyFormatter, self).__init__() + + # Override the type map to use our specialized formatters to + # emit the pretty output. + self.type_map[list] = self.PRETTY_ARRAY + self.type_map[tuple] = self.PRETTY_ARRAY + self.type_map[types.GeneratorType] = self.PRETTY_ARRAY, + self.type_map[dict] = self.PRETTY_MAP + + # Private data used for indentation. + self._indent_level = 1 + if indent_atom is None: + self._indent_atom = ' ' + else: + self._indent_atom = indent_atom + + def _indent(self): + "Return an indentation based on the atom and indentation level." + return self._indent_atom * self._indent_level + + def PRETTY_ARRAY(self, v): + rv = [] + rv.append('\n') + self._indent_level = self._indent_level + 1 + rv.extend(["%s%s\n" % + (self._indent(), + self.generate(item)) + for item in v]) + self._indent_level = self._indent_level - 1 + rv.append(self._indent()) + rv.append('') + return ''.join(rv) + + def PRETTY_MAP(self, v): + rv = [] + rv.append('\n') + self._indent_level = self._indent_level + 1 + keys = v.keys() + keys.sort() + rv.extend(["%s%s\n%s%s\n" % + (self._indent(), + self.elt('key', key), + self._indent(), + self.generate(v[key])) + for key in keys]) + self._indent_level = self._indent_level - 1 + rv.append(self._indent()) + rv.append('') + return ''.join(rv) + + def format(self, something): + data = [] + data.append('\n') + data.append(self.generate(something)) + data.append('\n') + return '\n'.join(data) + +def format_pretty_xml(something): + """@brief Serialize a python object as 'pretty' llsd xml. + + The output conforms to the LLSD DTD, unlike the output from the + standard python xml.dom DOM::toprettyxml() method which does not + preserve significant whitespace. + This function is not necessarily suited for serializing very large + objects. It is not optimized by the cllsd module, and sorts on + dict (llsd map) keys alphabetically to ease human reading. + """ + return LLSDXMLPrettyFormatter().format(something) + +class LLSDNotationFormatter(object): + def __init__(self): + self.type_map = { + type(None) : self.UNDEF, + bool : self.BOOLEAN, + int : self.INTEGER, + long : self.INTEGER, + float : self.REAL, + lluuid.UUID : self.UUID, + binary : self.BINARY, + str : self.STRING, + unicode : self.STRING, + uri : self.URI, + datetime.datetime : self.DATE, + datetime.date : self.DATE, + list : self.ARRAY, + tuple : self.ARRAY, + types.GeneratorType : self.ARRAY, + dict : self.MAP, + LLSD : self.LLSD + } + + def LLSD(self, v): + return self.generate(v.thing) + def UNDEF(self, v): + return '!' + def BOOLEAN(self, v): + if v: + return 'true' + else: + return 'false' + def INTEGER(self, v): + return "i%s" % v + def REAL(self, v): + return "r%s" % v + def UUID(self, v): + return "u%s" % v + def BINARY(self, v): + return 'b64"' + base64.encodestring(v) + '"' + def STRING(self, v): + if isinstance(v, unicode): + v = v.encode('utf-8') + return "'%s'" % v.replace("\\", "\\\\").replace("'", "\\'") + def URI(self, v): + return 'l"%s"' % str(v).replace("\\", "\\\\").replace('"', '\\"') + def DATE(self, v): + return 'd"%s"' % format_datestr(v) + def ARRAY(self, v): + return "[%s]" % ','.join([self.generate(item) for item in v]) + def MAP(self, v): + def fix(key): + if isinstance(key, unicode): + return key.encode('utf-8') + return key + return "{%s}" % ','.join(["'%s':%s" % (fix(key).replace("\\", "\\\\").replace("'", "\\'"), self.generate(value)) + for key, value in v.items()]) + + def generate(self, something): + t = type(something) + handler = self.type_map.get(t) + if handler: + return handler(something) + else: + try: + return self.ARRAY(iter(something)) + except TypeError: + raise LLSDSerializationError( + "Cannot serialize unknown type: %s (%s)" % (t, something)) + + def format(self, something): + return self.generate(something) + +def format_notation(something): + return LLSDNotationFormatter().format(something) + +def _hex_as_nybble(hex): + if (hex >= '0') and (hex <= '9'): + return ord(hex) - ord('0') + elif (hex >= 'a') and (hex <='f'): + return 10 + ord(hex) - ord('a') + elif (hex >= 'A') and (hex <='F'): + return 10 + ord(hex) - ord('A'); + +class LLSDBinaryParser(object): + def __init__(self): + pass + + def parse(self, buffer, ignore_binary = False): + """ + This is the basic public interface for parsing. + + @param buffer the binary data to parse in an indexable sequence. + @param ignore_binary parser throws away data in llsd binary nodes. + @return returns a python object. + """ + self._buffer = buffer + self._index = 0 + self._keep_binary = not ignore_binary + return self._parse() + + def _parse(self): + cc = self._buffer[self._index] + self._index += 1 + if cc == '{': + return self._parse_map() + elif cc == '[': + return self._parse_array() + elif cc == '!': + return None + elif cc == '0': + return False + elif cc == '1': + return True + elif cc == 'i': + # 'i' = integer + idx = self._index + self._index += 4 + return struct.unpack("!i", self._buffer[idx:idx+4])[0] + elif cc == ('r'): + # 'r' = real number + idx = self._index + self._index += 8 + return struct.unpack("!d", self._buffer[idx:idx+8])[0] + elif cc == 'u': + # 'u' = uuid + idx = self._index + self._index += 16 + return lluuid.uuid_bits_to_uuid(self._buffer[idx:idx+16]) + elif cc == 's': + # 's' = string + return self._parse_string() + elif cc in ("'", '"'): + # delimited/escaped string + return self._parse_string_delim(cc) + elif cc == 'l': + # 'l' = uri + return uri(self._parse_string()) + elif cc == ('d'): + # 'd' = date in seconds since epoch + idx = self._index + self._index += 8 + seconds = struct.unpack("!d", self._buffer[idx:idx+8])[0] + return datetime.datetime.fromtimestamp(seconds) + elif cc == 'b': + binary = self._parse_string() + if self._keep_binary: + return binary + # *NOTE: maybe have a binary placeholder which has the + # length. + return None + else: + raise LLSDParseError("invalid binary token at byte %d: %d" % ( + self._index - 1, ord(cc))) + + def _parse_map(self): + rv = {} + size = struct.unpack("!i", self._buffer[self._index:self._index+4])[0] + self._index += 4 + count = 0 + cc = self._buffer[self._index] + self._index += 1 + key = '' + while (cc != '}') and (count < size): + if cc == 'k': + key = self._parse_string() + elif cc in ("'", '"'): + key = self._parse_string_delim(cc) + else: + raise LLSDParseError("invalid map key at byte %d." % ( + self._index - 1,)) + value = self._parse() + rv[key] = value + count += 1 + cc = self._buffer[self._index] + self._index += 1 + if cc != '}': + raise LLSDParseError("invalid map close token at byte %d." % ( + self._index,)) + return rv + + def _parse_array(self): + rv = [] + size = struct.unpack("!i", self._buffer[self._index:self._index+4])[0] + self._index += 4 + count = 0 + cc = self._buffer[self._index] + while (cc != ']') and (count < size): + rv.append(self._parse()) + count += 1 + cc = self._buffer[self._index] + if cc != ']': + raise LLSDParseError("invalid array close token at byte %d." % ( + self._index,)) + self._index += 1 + return rv + + def _parse_string(self): + size = struct.unpack("!i", self._buffer[self._index:self._index+4])[0] + self._index += 4 + rv = self._buffer[self._index:self._index+size] + self._index += size + return rv + + def _parse_string_delim(self, delim): + list = [] + found_escape = False + found_hex = False + found_digit = False + byte = 0 + while True: + cc = self._buffer[self._index] + self._index += 1 + if found_escape: + if found_hex: + if found_digit: + found_escape = False + found_hex = False + found_digit = False + byte <<= 4 + byte |= _hex_as_nybble(cc) + list.append(chr(byte)) + byte = 0 + else: + found_digit = True + byte = _hex_as_nybble(cc) + elif cc == 'x': + found_hex = True + else: + if cc == 'a': + list.append('\a') + elif cc == 'b': + list.append('\b') + elif cc == 'f': + list.append('\f') + elif cc == 'n': + list.append('\n') + elif cc == 'r': + list.append('\r') + elif cc == 't': + list.append('\t') + elif cc == 'v': + list.append('\v') + else: + list.append(cc) + found_escape = False + elif cc == '\\': + found_escape = True + elif cc == delim: + break + else: + list.append(cc) + return ''.join(list) + +class LLSDNotationParser(object): + """ Parse LLSD notation: + map: { string:object, string:object } + array: [ object, object, object ] + undef: ! + boolean: true | false | 1 | 0 | T | F | t | f | TRUE | FALSE + integer: i#### + real: r#### + uuid: u#### + string: "g\'day" | 'have a "nice" day' | s(size)"raw data" + uri: l"escaped" + date: d"YYYY-MM-DDTHH:MM:SS.FFZ" + binary: b##"ff3120ab1" | b(size)"raw data" + """ + def __init__(self): + pass + + def parse(self, buffer, ignore_binary = False): + """ + This is the basic public interface for parsing. + + @param buffer the notation string to parse. + @param ignore_binary parser throws away data in llsd binary nodes. + @return returns a python object. + """ + if buffer == "": + return False + + self._buffer = buffer + self._index = 0 + return self._parse() + + def _parse(self): + cc = self._buffer[self._index] + self._index += 1 + if cc == '{': + return self._parse_map() + elif cc == '[': + return self._parse_array() + elif cc == '!': + return None + elif cc == '0': + return False + elif cc == '1': + return True + elif cc in ('F', 'f'): + self._skip_alpha() + return False + elif cc in ('T', 't'): + self._skip_alpha() + return True + elif cc == 'i': + # 'i' = integer + return self._parse_integer() + elif cc == ('r'): + # 'r' = real number + return self._parse_real() + elif cc == 'u': + # 'u' = uuid + return self._parse_uuid() + elif cc in ("'", '"', 's'): + return self._parse_string(cc) + elif cc == 'l': + # 'l' = uri + delim = self._buffer[self._index] + self._index += 1 + val = uri(self._parse_string(delim)) + if len(val) == 0: + return None + return val + elif cc == ('d'): + # 'd' = date in seconds since epoch + return self._parse_date() + elif cc == 'b': + return self._parse_binary() + else: + raise LLSDParseError("invalid token at index %d: %d" % ( + self._index - 1, ord(cc))) + + def _parse_binary(self): + i = self._index + if self._buffer[i:i+2] == '64': + q = self._buffer[i+2] + e = self._buffer.find(q, i+3) + try: + return base64.decodestring(self._buffer[i+3:e]) + finally: + self._index = e + 1 + else: + raise LLSDParseError('random horrible binary format not supported') + + def _parse_map(self): + """ map: { string:object, string:object } """ + rv = {} + cc = self._buffer[self._index] + self._index += 1 + key = '' + found_key = False + while (cc != '}'): + if not found_key: + if cc in ("'", '"', 's'): + key = self._parse_string(cc) + found_key = True + elif cc.isspace() or cc == ',': + cc = self._buffer[self._index] + self._index += 1 + else: + raise LLSDParseError("invalid map key at byte %d." % ( + self._index - 1,)) + elif cc.isspace() or cc == ':': + cc = self._buffer[self._index] + self._index += 1 + continue + else: + self._index += 1 + value = self._parse() + rv[key] = value + found_key = False + cc = self._buffer[self._index] + self._index += 1 + + return rv + + def _parse_array(self): + """ array: [ object, object, object ] """ + rv = [] + cc = self._buffer[self._index] + while (cc != ']'): + if cc.isspace() or cc == ',': + self._index += 1 + cc = self._buffer[self._index] + continue + rv.append(self._parse()) + cc = self._buffer[self._index] + + if cc != ']': + raise LLSDParseError("invalid array close token at index %d." % ( + self._index,)) + self._index += 1 + return rv + + def _parse_uuid(self): + match = re.match(lluuid.UUID.uuid_regex, self._buffer[self._index:]) + if not match: + raise LLSDParseError("invalid uuid token at index %d." % self._index) + + (start, end) = match.span() + start += self._index + end += self._index + self._index = end + return lluuid.UUID(self._buffer[start:end]) + + def _skip_alpha(self): + match = re.match(alpha_regex, self._buffer[self._index:]) + if match: + self._index += match.end() + + def _parse_date(self): + delim = self._buffer[self._index] + self._index += 1 + datestr = self._parse_string(delim) + return parse_datestr(datestr) + + def _parse_real(self): + match = re.match(real_regex, self._buffer[self._index:]) + if not match: + raise LLSDParseError("invalid real token at index %d." % self._index) + + (start, end) = match.span() + start += self._index + end += self._index + self._index = end + return float( self._buffer[start:end] ) + + def _parse_integer(self): + match = re.match(int_regex, self._buffer[self._index:]) + if not match: + raise LLSDParseError("invalid integer token at index %d." % self._index) + + (start, end) = match.span() + start += self._index + end += self._index + self._index = end + return int( self._buffer[start:end] ) + + def _parse_string(self, delim): + """ string: "g\'day" | 'have a "nice" day' | s(size)"raw data" """ + rv = "" + + if delim in ("'", '"'): + rv = self._parse_string_delim(delim) + elif delim == 's': + rv = self._parse_string_raw() + else: + raise LLSDParseError("invalid string token at index %d." % self._index) + + return rv + + + def _parse_string_delim(self, delim): + """ string: "g'day 'un" | 'have a "nice" day' """ + list = [] + found_escape = False + found_hex = False + found_digit = False + byte = 0 + while True: + cc = self._buffer[self._index] + self._index += 1 + if found_escape: + if found_hex: + if found_digit: + found_escape = False + found_hex = False + found_digit = False + byte <<= 4 + byte |= _hex_as_nybble(cc) + list.append(chr(byte)) + byte = 0 + else: + found_digit = True + byte = _hex_as_nybble(cc) + elif cc == 'x': + found_hex = True + else: + if cc == 'a': + list.append('\a') + elif cc == 'b': + list.append('\b') + elif cc == 'f': + list.append('\f') + elif cc == 'n': + list.append('\n') + elif cc == 'r': + list.append('\r') + elif cc == 't': + list.append('\t') + elif cc == 'v': + list.append('\v') + else: + list.append(cc) + found_escape = False + elif cc == '\\': + found_escape = True + elif cc == delim: + break + else: + list.append(cc) + return ''.join(list) + + def _parse_string_raw(self): + """ string: s(size)"raw data" """ + # Read the (size) portion. + cc = self._buffer[self._index] + self._index += 1 + if cc != '(': + raise LLSDParseError("invalid string token at index %d." % self._index) + + rparen = self._buffer.find(')', self._index) + if rparen == -1: + raise LLSDParseError("invalid string token at index %d." % self._index) + + size = int(self._buffer[self._index:rparen]) + + self._index = rparen + 1 + delim = self._buffer[self._index] + self._index += 1 + if delim not in ("'", '"'): + raise LLSDParseError("invalid string token at index %d." % self._index) + + rv = self._buffer[self._index:(self._index + size)] + self._index += size + cc = self._buffer[self._index] + self._index += 1 + if cc != delim: + raise LLSDParseError("invalid string token at index %d." % self._index) + + return rv + +def format_binary(something): + return '\n' + _format_binary_recurse(something) + +def _format_binary_recurse(something): + def _format_list(something): + array_builder = [] + array_builder.append('[' + struct.pack('!i', len(something))) + for item in something: + array_builder.append(_format_binary_recurse(item)) + array_builder.append(']') + return ''.join(array_builder) + + if something is None: + return '!' + elif isinstance(something, LLSD): + return _format_binary_recurse(something.thing) + elif isinstance(something, bool): + if something: + return '1' + else: + return '0' + elif isinstance(something, (int, long)): + return 'i' + struct.pack('!i', something) + elif isinstance(something, float): + return 'r' + struct.pack('!d', something) + elif isinstance(something, lluuid.UUID): + return 'u' + something._bits + elif isinstance(something, binary): + return 'b' + struct.pack('!i', len(something)) + something + elif isinstance(something, str): + return 's' + struct.pack('!i', len(something)) + something + elif isinstance(something, unicode): + something = something.encode('utf-8') + return 's' + struct.pack('!i', len(something)) + something + elif isinstance(something, uri): + return 'l' + struct.pack('!i', len(something)) + something + elif isinstance(something, datetime.datetime): + seconds_since_epoch = time.mktime(something.timetuple()) + return 'd' + struct.pack('!d', seconds_since_epoch) + elif isinstance(something, (list, tuple)): + return _format_list(something) + elif isinstance(something, dict): + map_builder = [] + map_builder.append('{' + struct.pack('!i', len(something))) + for key, value in something.items(): + if isinstance(key, unicode): + key = key.encode('utf-8') + map_builder.append('k' + struct.pack('!i', len(key)) + key) + map_builder.append(_format_binary_recurse(value)) + map_builder.append('}') + return ''.join(map_builder) + else: + try: + return _format_list(list(something)) + except TypeError: + raise LLSDSerializationError( + "Cannot serialize unknown type: %s (%s)" % + (type(something), something)) + + +def parse_binary(binary): + if binary.startswith(''): + just_binary = binary.split('\n', 1)[1] + else: + just_binary = binary + return LLSDBinaryParser().parse(just_binary) + +def parse_xml(something): + try: + return to_python(fromstring(something)[0]) + except ElementTreeError, err: + raise LLSDParseError(*err.args) + +def parse_notation(something): + return LLSDNotationParser().parse(something) + +def parse(something): + try: + something = string.lstrip(something) #remove any pre-trailing whitespace + if something.startswith(''): + return parse_binary(something) + # This should be better. + elif something.startswith('<'): + return parse_xml(something) + else: + return parse_notation(something) + except KeyError, e: + raise Exception('LLSD could not be parsed: %s' % (e,)) + +class LLSD(object): + def __init__(self, thing=None): + self.thing = thing + + def __str__(self): + return self.toXML(self.thing) + + parse = staticmethod(parse) + toXML = staticmethod(format_xml) + toPrettyXML = staticmethod(format_pretty_xml) + toBinary = staticmethod(format_binary) + toNotation = staticmethod(format_notation) + + +undef = LLSD(None) + +XML_MIME_TYPE = 'application/llsd+xml' +BINARY_MIME_TYPE = 'application/llsd+binary' + +# register converters for llsd in mulib, if it is available +try: + from mulib import stacked, mu + stacked.NoProducer() # just to exercise stacked + mu.safe_load(None) # just to exercise mu +except: + # mulib not available, don't print an error message since this is normal + pass +else: + mu.add_parser(parse, XML_MIME_TYPE) + mu.add_parser(parse, 'application/llsd+binary') + + def llsd_convert_xml(llsd_stuff, request): + request.write(format_xml(llsd_stuff)) + + def llsd_convert_binary(llsd_stuff, request): + request.write(format_binary(llsd_stuff)) + + for typ in [LLSD, dict, list, tuple, str, int, long, float, bool, unicode, type(None)]: + stacked.add_producer(typ, llsd_convert_xml, XML_MIME_TYPE) + stacked.add_producer(typ, llsd_convert_xml, 'application/xml') + stacked.add_producer(typ, llsd_convert_xml, 'text/xml') + + stacked.add_producer(typ, llsd_convert_binary, 'application/llsd+binary') + + stacked.add_producer(LLSD, llsd_convert_xml, '*/*') + + # in case someone is using the legacy mu.xml wrapper, we need to + # tell mu to produce application/xml or application/llsd+xml + # (based on the accept header) from raw xml. Phoenix 2008-07-21 + stacked.add_producer(mu.xml, mu.produce_raw, XML_MIME_TYPE) + stacked.add_producer(mu.xml, mu.produce_raw, 'application/xml') + + + +# mulib wsgi stuff +# try: +# from mulib import mu, adapters +# +# # try some known attributes from mulib to be ultra-sure we've imported it +# mu.get_current +# adapters.handlers +# except: +# # mulib not available, don't print an error message since this is normal +# pass +# else: +# def llsd_xml_handler(content_type): +# def handle_llsd_xml(env, start_response): +# llsd_stuff, _ = mu.get_current(env) +# result = format_xml(llsd_stuff) +# start_response("200 OK", [('Content-Type', content_type)]) +# env['mu.negotiated_type'] = content_type +# yield result +# return handle_llsd_xml +# +# def llsd_binary_handler(content_type): +# def handle_llsd_binary(env, start_response): +# llsd_stuff, _ = mu.get_current(env) +# result = format_binary(llsd_stuff) +# start_response("200 OK", [('Content-Type', content_type)]) +# env['mu.negotiated_type'] = content_type +# yield result +# return handle_llsd_binary +# +# adapters.DEFAULT_PARSERS[XML_MIME_TYPE] = parse + +# for typ in [LLSD, dict, list, tuple, str, int, float, bool, unicode, type(None)]: +# for content_type in (XML_MIME_TYPE, 'application/xml'): +# adapters.handlers.set_handler(typ, llsd_xml_handler(content_type), content_type) +# +# adapters.handlers.set_handler(typ, llsd_binary_handler(BINARY_MIME_TYPE), BINARY_MIME_TYPE) +# +# adapters.handlers.set_handler(LLSD, llsd_xml_handler(XML_MIME_TYPE), '*/*') diff --git a/indra/viewer_components/manager/base/lluuid.py b/indra/viewer_components/manager/base/lluuid.py new file mode 100755 index 0000000000..7413ffe10d --- /dev/null +++ b/indra/viewer_components/manager/base/lluuid.py @@ -0,0 +1,319 @@ +"""\ +@file lluuid.py +@brief UUID parser/generator. + +$LicenseInfo:firstyear=2004&license=mit$ + +Copyright (c) 2004-2009, Linden Research, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +$/LicenseInfo$ +""" + +import random, socket, string, time, re +import uuid +try: + # Python 2.6 + from hashlib import md5 +except ImportError: + # Python 2.5 and earlier + from md5 import new as md5 + +def _int2binstr(i,l): + s='' + for a in range(l): + s=chr(i&0xFF)+s + i>>=8 + return s + +def _binstr2int(s): + i = long(0) + for c in s: + i = (i<<8) + ord(c) + return i + +class UUID(object): + """ + A class which represents a 16 byte integer. Stored as a 16 byte 8 + bit character string. + + The string version is to be of the form: + AAAAAAAA-AAAA-BBBB-BBBB-BBBBBBCCCCCC (a 128-bit number in hex) + where A=network address, B=timestamp, C=random. + """ + + NULL_STR = "00000000-0000-0000-0000-000000000000" + + # the UUIDREGEX_STRING is helpful for parsing UUID's in text + hex_wildcard = r"[0-9a-fA-F]" + word = hex_wildcard + r"{4,4}-" + long_word = hex_wildcard + r"{8,8}-" + very_long_word = hex_wildcard + r"{12,12}" + UUID_REGEX_STRING = long_word + word + word + word + very_long_word + uuid_regex = re.compile(UUID_REGEX_STRING) + + rand = random.Random() + ip = '' + try: + ip = socket.gethostbyname(socket.gethostname()) + except(socket.gaierror, socket.error): + # no ip address, so just default to somewhere in 10.x.x.x + ip = '10' + for i in range(3): + ip += '.' + str(rand.randrange(1,254)) + hexip = ''.join(["%04x" % long(i) for i in ip.split('.')]) + lastid = '' + + def __init__(self, possible_uuid=None): + """ + Initialize to first valid UUID in argument (if a string), + or to null UUID if none found or argument is not supplied. + + If the argument is a UUID, the constructed object will be a copy of it. + """ + self._bits = "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0" + if possible_uuid is None: + return + + if isinstance(possible_uuid, type(self)): + self.set(possible_uuid) + return + + uuid_match = UUID.uuid_regex.search(possible_uuid) + if uuid_match: + uuid_string = uuid_match.group() + s = string.replace(uuid_string, '-', '') + self._bits = _int2binstr(string.atol(s[:8],16),4) + \ + _int2binstr(string.atol(s[8:16],16),4) + \ + _int2binstr(string.atol(s[16:24],16),4) + \ + _int2binstr(string.atol(s[24:],16),4) + + def __len__(self): + """ + Used by the len() builtin. + """ + return 36 + + def __nonzero__(self): + return self._bits != "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0" + + def __str__(self): + uuid_string = self.toString() + return uuid_string + + __repr__ = __str__ + + def __getitem__(self, index): + return str(self)[index] + + def __eq__(self, other): + if isinstance(other, (str, unicode)): + return other == str(self) + return self._bits == getattr(other, '_bits', '') + + def __ne__(self, other): + return not self.__eq__(other) + + def __le__(self, other): + return self._bits <= other._bits + + def __ge__(self, other): + return self._bits >= other._bits + + def __lt__(self, other): + return self._bits < other._bits + + def __gt__(self, other): + return self._bits > other._bits + + def __hash__(self): + return hash(self._bits) + + def set(self, uuid): + self._bits = uuid._bits + + def setFromString(self, uuid_string): + """ + Given a string version of a uuid, set self bits + appropriately. Returns self. + """ + s = string.replace(uuid_string, '-', '') + self._bits = _int2binstr(string.atol(s[:8],16),4) + \ + _int2binstr(string.atol(s[8:16],16),4) + \ + _int2binstr(string.atol(s[16:24],16),4) + \ + _int2binstr(string.atol(s[24:],16),4) + return self + + def setFromMemoryDump(self, gdb_string): + """ + We expect to get gdb_string as four hex units. eg: + 0x147d54db 0xc34b3f1b 0x714f989b 0x0a892fd2 + Which will be translated to: + db547d14-1b3f4bc3-9b984f71-d22f890a + Returns self. + """ + s = string.replace(gdb_string, '0x', '') + s = string.replace(s, ' ', '') + t = '' + for i in range(8,40,8): + for j in range(0,8,2): + t = t + s[i-j-2:i-j] + self.setFromString(t) + + def toString(self): + """ + Return as a string matching the LL standard + AAAAAAAA-AAAA-BBBB-BBBB-BBBBBBCCCCCC (a 128-bit number in hex) + where A=network address, B=timestamp, C=random. + """ + return uuid_bits_to_string(self._bits) + + def getAsString(self): + """ + Return a different string representation of the form + AAAAAAAA-AAAABBBB-BBBBBBBB-BBCCCCCC (a 128-bit number in hex) + where A=network address, B=timestamp, C=random. + """ + i1 = _binstr2int(self._bits[0:4]) + i2 = _binstr2int(self._bits[4:8]) + i3 = _binstr2int(self._bits[8:12]) + i4 = _binstr2int(self._bits[12:16]) + return '%08lx-%08lx-%08lx-%08lx' % (i1,i2,i3,i4) + + def generate(self): + """ + Generate a new uuid. This algorithm is slightly different + from c++ implementation for portability reasons. + Returns self. + """ + m = md5() + m.update(uuid.uuid1().bytes) + self._bits = m.digest() + return self + + def isNull(self): + """ + Returns 1 if the uuid is null - ie, equal to default uuid. + """ + return (self._bits == "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0") + + def xor(self, rhs): + """ + xors self with rhs. + """ + v1 = _binstr2int(self._bits[0:4]) ^ _binstr2int(rhs._bits[0:4]) + v2 = _binstr2int(self._bits[4:8]) ^ _binstr2int(rhs._bits[4:8]) + v3 = _binstr2int(self._bits[8:12]) ^ _binstr2int(rhs._bits[8:12]) + v4 = _binstr2int(self._bits[12:16]) ^ _binstr2int(rhs._bits[12:16]) + self._bits = _int2binstr(v1,4) + \ + _int2binstr(v2,4) + \ + _int2binstr(v3,4) + \ + _int2binstr(v4,4) + + +# module-level null constant +NULL = UUID() + +def printTranslatedMemory(four_hex_uints): + """ + We expect to get the string as four hex units. eg: + 0x147d54db 0xc34b3f1b 0x714f989b 0x0a892fd2 + Which will be translated to: + db547d14-1b3f4bc3-9b984f71-d22f890a + """ + uuid = UUID() + uuid.setFromMemoryDump(four_hex_uints) + print uuid.toString() + +def isUUID(id_str): + """ + This function returns: + - 1 if the string passed is a UUID + - 0 is the string passed is not a UUID + - None if it neither of the if's below is satisfied + """ + if not id_str or len(id_str) < 5 or len(id_str) > 36: + return 0 + + if isinstance(id_str, UUID) or UUID.uuid_regex.match(id_str): + return 1 + + return None + +def isPossiblyID(id_str): + """ + This function returns 1 if the string passed has some uuid-like + characteristics. Otherwise returns 0. + """ + + is_uuid = isUUID(id_str) + if is_uuid is not None: + return is_uuid + + # build a string which matches every character. + hex_wildcard = r"[0-9a-fA-F]" + chars = len(id_str) + next = min(chars, 8) + matcher = hex_wildcard+"{"+str(next)+","+str(next)+"}" + chars = chars - next + if chars > 0: + matcher = matcher + "-" + chars = chars - 1 + for block in range(3): + next = max(min(chars, 4), 0) + if next: + matcher = matcher + hex_wildcard+"{"+str(next)+","+str(next)+"}" + chars = chars - next + if chars > 0: + matcher = matcher + "-" + chars = chars - 1 + if chars > 0: + next = min(chars, 12) + matcher = matcher + hex_wildcard+"{"+str(next)+","+str(next)+"}" + #print matcher + uuid_matcher = re.compile(matcher) + if uuid_matcher.match(id_str): + return 1 + return 0 + +def uuid_bits_to_string(bits): + i1 = _binstr2int(bits[0:4]) + i2 = _binstr2int(bits[4:6]) + i3 = _binstr2int(bits[6:8]) + i4 = _binstr2int(bits[8:10]) + i5 = _binstr2int(bits[10:12]) + i6 = _binstr2int(bits[12:16]) + return '%08lx-%04lx-%04lx-%04lx-%04lx%08lx' % (i1,i2,i3,i4,i5,i6) + +def uuid_bits_to_uuid(bits): + return UUID(uuid_bits_to_string(bits)) + + +try: + from mulib import stacked + stacked.NoProducer() # just to exercise stacked +except: + #print "Couldn't import mulib.stacked, not registering UUID converter" + pass +else: + def convertUUID(uuid, req): + req.write(str(uuid)) + + stacked.add_producer(UUID, convertUUID, "*/*") + stacked.add_producer(UUID, convertUUID, "text/html") diff --git a/indra/viewer_components/manager/llsd.py b/indra/viewer_components/manager/llsd.py deleted file mode 100755 index 4527b115f9..0000000000 --- a/indra/viewer_components/manager/llsd.py +++ /dev/null @@ -1,1052 +0,0 @@ -"""\ -@file llsd.py -@brief Types as well as parsing and formatting functions for handling LLSD. - -$LicenseInfo:firstyear=2006&license=mit$ - -Copyright (c) 2006-2009, Linden Research, Inc. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -$/LicenseInfo$ -""" - -import datetime -import base64 -import string -import struct -import time -import types -import re - -from indra.util.fastest_elementtree import ElementTreeError, fromstring -from indra.base import lluuid - -# cllsd.c in server/server-1.25 has memory leaks, -# so disabling cllsd for now -#try: -# import cllsd -#except ImportError: -# cllsd = None -cllsd = None - -int_regex = re.compile(r"[-+]?\d+") -real_regex = re.compile(r"[-+]?(\d+(\.\d*)?|\d*\.\d+)([eE][-+]?\d+)?") -alpha_regex = re.compile(r"[a-zA-Z]+") -date_regex = re.compile(r"(?P\d{4})-(?P\d{2})-(?P\d{2})T" - r"(?P\d{2}):(?P\d{2}):(?P\d{2})" - r"(?P(\.\d+)?)Z") -#date: d"YYYY-MM-DDTHH:MM:SS.FFFFFFZ" - -class LLSDParseError(Exception): - pass - -class LLSDSerializationError(TypeError): - pass - - -class binary(str): - pass - -class uri(str): - pass - - -BOOL_TRUE = ('1', '1.0', 'true') -BOOL_FALSE = ('0', '0.0', 'false', '') - - -def format_datestr(v): - """ Formats a datetime or date object into the string format shared by xml and notation serializations.""" - if hasattr(v, 'microsecond'): - return v.isoformat() + 'Z' - else: - return v.strftime('%Y-%m-%dT%H:%M:%SZ') - -def parse_datestr(datestr): - """Parses a datetime object from the string format shared by xml and notation serializations.""" - if datestr == "": - return datetime.datetime(1970, 1, 1) - - match = re.match(date_regex, datestr) - if not match: - raise LLSDParseError("invalid date string '%s'." % datestr) - - year = int(match.group('year')) - month = int(match.group('month')) - day = int(match.group('day')) - hour = int(match.group('hour')) - minute = int(match.group('minute')) - second = int(match.group('second')) - seconds_float = match.group('second_float') - microsecond = 0 - if seconds_float: - microsecond = int(float('0' + seconds_float) * 1e6) - return datetime.datetime(year, month, day, hour, minute, second, microsecond) - - -def bool_to_python(node): - val = node.text or '' - if val in BOOL_TRUE: - return True - else: - return False - -def int_to_python(node): - val = node.text or '' - if not val.strip(): - return 0 - return int(val) - -def real_to_python(node): - val = node.text or '' - if not val.strip(): - return 0.0 - return float(val) - -def uuid_to_python(node): - return lluuid.UUID(node.text) - -def str_to_python(node): - return node.text or '' - -def bin_to_python(node): - return binary(base64.decodestring(node.text or '')) - -def date_to_python(node): - val = node.text or '' - if not val: - val = "1970-01-01T00:00:00Z" - return parse_datestr(val) - - -def uri_to_python(node): - val = node.text or '' - if not val: - return None - return uri(val) - -def map_to_python(node): - result = {} - for index in range(len(node))[::2]: - result[node[index].text] = to_python(node[index+1]) - return result - -def array_to_python(node): - return [to_python(child) for child in node] - - -NODE_HANDLERS = dict( - undef=lambda x: None, - boolean=bool_to_python, - integer=int_to_python, - real=real_to_python, - uuid=uuid_to_python, - string=str_to_python, - binary=bin_to_python, - date=date_to_python, - uri=uri_to_python, - map=map_to_python, - array=array_to_python, - ) - -def to_python(node): - return NODE_HANDLERS[node.tag](node) - -class Nothing(object): - pass - - -class LLSDXMLFormatter(object): - def __init__(self): - self.type_map = { - type(None) : self.UNDEF, - bool : self.BOOLEAN, - int : self.INTEGER, - long : self.INTEGER, - float : self.REAL, - lluuid.UUID : self.UUID, - binary : self.BINARY, - str : self.STRING, - unicode : self.STRING, - uri : self.URI, - datetime.datetime : self.DATE, - datetime.date : self.DATE, - list : self.ARRAY, - tuple : self.ARRAY, - types.GeneratorType : self.ARRAY, - dict : self.MAP, - LLSD : self.LLSD - } - - def elt(self, name, contents=None): - if(contents is None or contents is ''): - return "<%s />" % (name,) - else: - if type(contents) is unicode: - contents = contents.encode('utf-8') - return "<%s>%s" % (name, contents, name) - - def xml_esc(self, v): - if type(v) is unicode: - v = v.encode('utf-8') - return v.replace('&', '&').replace('<', '<').replace('>', '>') - - def LLSD(self, v): - return self.generate(v.thing) - def UNDEF(self, v): - return self.elt('undef') - def BOOLEAN(self, v): - if v: - return self.elt('boolean', 'true') - else: - return self.elt('boolean', 'false') - def INTEGER(self, v): - return self.elt('integer', v) - def REAL(self, v): - return self.elt('real', v) - def UUID(self, v): - if(v.isNull()): - return self.elt('uuid') - else: - return self.elt('uuid', v) - def BINARY(self, v): - return self.elt('binary', base64.encodestring(v)) - def STRING(self, v): - return self.elt('string', self.xml_esc(v)) - def URI(self, v): - return self.elt('uri', self.xml_esc(str(v))) - def DATE(self, v): - return self.elt('date', format_datestr(v)) - def ARRAY(self, v): - return self.elt('array', ''.join([self.generate(item) for item in v])) - def MAP(self, v): - return self.elt( - 'map', - ''.join(["%s%s" % (self.elt('key', self.xml_esc(str(key))), self.generate(value)) - for key, value in v.items()])) - - typeof = type - def generate(self, something): - t = self.typeof(something) - if self.type_map.has_key(t): - return self.type_map[t](something) - else: - raise LLSDSerializationError("Cannot serialize unknown type: %s (%s)" % ( - t, something)) - - def _format(self, something): - return '' + self.elt("llsd", self.generate(something)) - - def format(self, something): - if cllsd: - return cllsd.llsd_to_xml(something) - return self._format(something) - -_g_xml_formatter = None -def format_xml(something): - global _g_xml_formatter - if _g_xml_formatter is None: - _g_xml_formatter = LLSDXMLFormatter() - return _g_xml_formatter.format(something) - -class LLSDXMLPrettyFormatter(LLSDXMLFormatter): - def __init__(self, indent_atom = None): - # Call the super class constructor so that we have the type map - super(LLSDXMLPrettyFormatter, self).__init__() - - # Override the type map to use our specialized formatters to - # emit the pretty output. - self.type_map[list] = self.PRETTY_ARRAY - self.type_map[tuple] = self.PRETTY_ARRAY - self.type_map[types.GeneratorType] = self.PRETTY_ARRAY, - self.type_map[dict] = self.PRETTY_MAP - - # Private data used for indentation. - self._indent_level = 1 - if indent_atom is None: - self._indent_atom = ' ' - else: - self._indent_atom = indent_atom - - def _indent(self): - "Return an indentation based on the atom and indentation level." - return self._indent_atom * self._indent_level - - def PRETTY_ARRAY(self, v): - rv = [] - rv.append('\n') - self._indent_level = self._indent_level + 1 - rv.extend(["%s%s\n" % - (self._indent(), - self.generate(item)) - for item in v]) - self._indent_level = self._indent_level - 1 - rv.append(self._indent()) - rv.append('') - return ''.join(rv) - - def PRETTY_MAP(self, v): - rv = [] - rv.append('\n') - self._indent_level = self._indent_level + 1 - keys = v.keys() - keys.sort() - rv.extend(["%s%s\n%s%s\n" % - (self._indent(), - self.elt('key', key), - self._indent(), - self.generate(v[key])) - for key in keys]) - self._indent_level = self._indent_level - 1 - rv.append(self._indent()) - rv.append('') - return ''.join(rv) - - def format(self, something): - data = [] - data.append('\n') - data.append(self.generate(something)) - data.append('\n') - return '\n'.join(data) - -def format_pretty_xml(something): - """@brief Serialize a python object as 'pretty' llsd xml. - - The output conforms to the LLSD DTD, unlike the output from the - standard python xml.dom DOM::toprettyxml() method which does not - preserve significant whitespace. - This function is not necessarily suited for serializing very large - objects. It is not optimized by the cllsd module, and sorts on - dict (llsd map) keys alphabetically to ease human reading. - """ - return LLSDXMLPrettyFormatter().format(something) - -class LLSDNotationFormatter(object): - def __init__(self): - self.type_map = { - type(None) : self.UNDEF, - bool : self.BOOLEAN, - int : self.INTEGER, - long : self.INTEGER, - float : self.REAL, - lluuid.UUID : self.UUID, - binary : self.BINARY, - str : self.STRING, - unicode : self.STRING, - uri : self.URI, - datetime.datetime : self.DATE, - datetime.date : self.DATE, - list : self.ARRAY, - tuple : self.ARRAY, - types.GeneratorType : self.ARRAY, - dict : self.MAP, - LLSD : self.LLSD - } - - def LLSD(self, v): - return self.generate(v.thing) - def UNDEF(self, v): - return '!' - def BOOLEAN(self, v): - if v: - return 'true' - else: - return 'false' - def INTEGER(self, v): - return "i%s" % v - def REAL(self, v): - return "r%s" % v - def UUID(self, v): - return "u%s" % v - def BINARY(self, v): - return 'b64"' + base64.encodestring(v) + '"' - def STRING(self, v): - if isinstance(v, unicode): - v = v.encode('utf-8') - return "'%s'" % v.replace("\\", "\\\\").replace("'", "\\'") - def URI(self, v): - return 'l"%s"' % str(v).replace("\\", "\\\\").replace('"', '\\"') - def DATE(self, v): - return 'd"%s"' % format_datestr(v) - def ARRAY(self, v): - return "[%s]" % ','.join([self.generate(item) for item in v]) - def MAP(self, v): - def fix(key): - if isinstance(key, unicode): - return key.encode('utf-8') - return key - return "{%s}" % ','.join(["'%s':%s" % (fix(key).replace("\\", "\\\\").replace("'", "\\'"), self.generate(value)) - for key, value in v.items()]) - - def generate(self, something): - t = type(something) - handler = self.type_map.get(t) - if handler: - return handler(something) - else: - try: - return self.ARRAY(iter(something)) - except TypeError: - raise LLSDSerializationError( - "Cannot serialize unknown type: %s (%s)" % (t, something)) - - def format(self, something): - return self.generate(something) - -def format_notation(something): - return LLSDNotationFormatter().format(something) - -def _hex_as_nybble(hex): - if (hex >= '0') and (hex <= '9'): - return ord(hex) - ord('0') - elif (hex >= 'a') and (hex <='f'): - return 10 + ord(hex) - ord('a') - elif (hex >= 'A') and (hex <='F'): - return 10 + ord(hex) - ord('A'); - -class LLSDBinaryParser(object): - def __init__(self): - pass - - def parse(self, buffer, ignore_binary = False): - """ - This is the basic public interface for parsing. - - @param buffer the binary data to parse in an indexable sequence. - @param ignore_binary parser throws away data in llsd binary nodes. - @return returns a python object. - """ - self._buffer = buffer - self._index = 0 - self._keep_binary = not ignore_binary - return self._parse() - - def _parse(self): - cc = self._buffer[self._index] - self._index += 1 - if cc == '{': - return self._parse_map() - elif cc == '[': - return self._parse_array() - elif cc == '!': - return None - elif cc == '0': - return False - elif cc == '1': - return True - elif cc == 'i': - # 'i' = integer - idx = self._index - self._index += 4 - return struct.unpack("!i", self._buffer[idx:idx+4])[0] - elif cc == ('r'): - # 'r' = real number - idx = self._index - self._index += 8 - return struct.unpack("!d", self._buffer[idx:idx+8])[0] - elif cc == 'u': - # 'u' = uuid - idx = self._index - self._index += 16 - return lluuid.uuid_bits_to_uuid(self._buffer[idx:idx+16]) - elif cc == 's': - # 's' = string - return self._parse_string() - elif cc in ("'", '"'): - # delimited/escaped string - return self._parse_string_delim(cc) - elif cc == 'l': - # 'l' = uri - return uri(self._parse_string()) - elif cc == ('d'): - # 'd' = date in seconds since epoch - idx = self._index - self._index += 8 - seconds = struct.unpack("!d", self._buffer[idx:idx+8])[0] - return datetime.datetime.fromtimestamp(seconds) - elif cc == 'b': - binary = self._parse_string() - if self._keep_binary: - return binary - # *NOTE: maybe have a binary placeholder which has the - # length. - return None - else: - raise LLSDParseError("invalid binary token at byte %d: %d" % ( - self._index - 1, ord(cc))) - - def _parse_map(self): - rv = {} - size = struct.unpack("!i", self._buffer[self._index:self._index+4])[0] - self._index += 4 - count = 0 - cc = self._buffer[self._index] - self._index += 1 - key = '' - while (cc != '}') and (count < size): - if cc == 'k': - key = self._parse_string() - elif cc in ("'", '"'): - key = self._parse_string_delim(cc) - else: - raise LLSDParseError("invalid map key at byte %d." % ( - self._index - 1,)) - value = self._parse() - rv[key] = value - count += 1 - cc = self._buffer[self._index] - self._index += 1 - if cc != '}': - raise LLSDParseError("invalid map close token at byte %d." % ( - self._index,)) - return rv - - def _parse_array(self): - rv = [] - size = struct.unpack("!i", self._buffer[self._index:self._index+4])[0] - self._index += 4 - count = 0 - cc = self._buffer[self._index] - while (cc != ']') and (count < size): - rv.append(self._parse()) - count += 1 - cc = self._buffer[self._index] - if cc != ']': - raise LLSDParseError("invalid array close token at byte %d." % ( - self._index,)) - self._index += 1 - return rv - - def _parse_string(self): - size = struct.unpack("!i", self._buffer[self._index:self._index+4])[0] - self._index += 4 - rv = self._buffer[self._index:self._index+size] - self._index += size - return rv - - def _parse_string_delim(self, delim): - list = [] - found_escape = False - found_hex = False - found_digit = False - byte = 0 - while True: - cc = self._buffer[self._index] - self._index += 1 - if found_escape: - if found_hex: - if found_digit: - found_escape = False - found_hex = False - found_digit = False - byte <<= 4 - byte |= _hex_as_nybble(cc) - list.append(chr(byte)) - byte = 0 - else: - found_digit = True - byte = _hex_as_nybble(cc) - elif cc == 'x': - found_hex = True - else: - if cc == 'a': - list.append('\a') - elif cc == 'b': - list.append('\b') - elif cc == 'f': - list.append('\f') - elif cc == 'n': - list.append('\n') - elif cc == 'r': - list.append('\r') - elif cc == 't': - list.append('\t') - elif cc == 'v': - list.append('\v') - else: - list.append(cc) - found_escape = False - elif cc == '\\': - found_escape = True - elif cc == delim: - break - else: - list.append(cc) - return ''.join(list) - -class LLSDNotationParser(object): - """ Parse LLSD notation: - map: { string:object, string:object } - array: [ object, object, object ] - undef: ! - boolean: true | false | 1 | 0 | T | F | t | f | TRUE | FALSE - integer: i#### - real: r#### - uuid: u#### - string: "g\'day" | 'have a "nice" day' | s(size)"raw data" - uri: l"escaped" - date: d"YYYY-MM-DDTHH:MM:SS.FFZ" - binary: b##"ff3120ab1" | b(size)"raw data" - """ - def __init__(self): - pass - - def parse(self, buffer, ignore_binary = False): - """ - This is the basic public interface for parsing. - - @param buffer the notation string to parse. - @param ignore_binary parser throws away data in llsd binary nodes. - @return returns a python object. - """ - if buffer == "": - return False - - self._buffer = buffer - self._index = 0 - return self._parse() - - def _parse(self): - cc = self._buffer[self._index] - self._index += 1 - if cc == '{': - return self._parse_map() - elif cc == '[': - return self._parse_array() - elif cc == '!': - return None - elif cc == '0': - return False - elif cc == '1': - return True - elif cc in ('F', 'f'): - self._skip_alpha() - return False - elif cc in ('T', 't'): - self._skip_alpha() - return True - elif cc == 'i': - # 'i' = integer - return self._parse_integer() - elif cc == ('r'): - # 'r' = real number - return self._parse_real() - elif cc == 'u': - # 'u' = uuid - return self._parse_uuid() - elif cc in ("'", '"', 's'): - return self._parse_string(cc) - elif cc == 'l': - # 'l' = uri - delim = self._buffer[self._index] - self._index += 1 - val = uri(self._parse_string(delim)) - if len(val) == 0: - return None - return val - elif cc == ('d'): - # 'd' = date in seconds since epoch - return self._parse_date() - elif cc == 'b': - return self._parse_binary() - else: - raise LLSDParseError("invalid token at index %d: %d" % ( - self._index - 1, ord(cc))) - - def _parse_binary(self): - i = self._index - if self._buffer[i:i+2] == '64': - q = self._buffer[i+2] - e = self._buffer.find(q, i+3) - try: - return base64.decodestring(self._buffer[i+3:e]) - finally: - self._index = e + 1 - else: - raise LLSDParseError('random horrible binary format not supported') - - def _parse_map(self): - """ map: { string:object, string:object } """ - rv = {} - cc = self._buffer[self._index] - self._index += 1 - key = '' - found_key = False - while (cc != '}'): - if not found_key: - if cc in ("'", '"', 's'): - key = self._parse_string(cc) - found_key = True - elif cc.isspace() or cc == ',': - cc = self._buffer[self._index] - self._index += 1 - else: - raise LLSDParseError("invalid map key at byte %d." % ( - self._index - 1,)) - elif cc.isspace() or cc == ':': - cc = self._buffer[self._index] - self._index += 1 - continue - else: - self._index += 1 - value = self._parse() - rv[key] = value - found_key = False - cc = self._buffer[self._index] - self._index += 1 - - return rv - - def _parse_array(self): - """ array: [ object, object, object ] """ - rv = [] - cc = self._buffer[self._index] - while (cc != ']'): - if cc.isspace() or cc == ',': - self._index += 1 - cc = self._buffer[self._index] - continue - rv.append(self._parse()) - cc = self._buffer[self._index] - - if cc != ']': - raise LLSDParseError("invalid array close token at index %d." % ( - self._index,)) - self._index += 1 - return rv - - def _parse_uuid(self): - match = re.match(lluuid.UUID.uuid_regex, self._buffer[self._index:]) - if not match: - raise LLSDParseError("invalid uuid token at index %d." % self._index) - - (start, end) = match.span() - start += self._index - end += self._index - self._index = end - return lluuid.UUID(self._buffer[start:end]) - - def _skip_alpha(self): - match = re.match(alpha_regex, self._buffer[self._index:]) - if match: - self._index += match.end() - - def _parse_date(self): - delim = self._buffer[self._index] - self._index += 1 - datestr = self._parse_string(delim) - return parse_datestr(datestr) - - def _parse_real(self): - match = re.match(real_regex, self._buffer[self._index:]) - if not match: - raise LLSDParseError("invalid real token at index %d." % self._index) - - (start, end) = match.span() - start += self._index - end += self._index - self._index = end - return float( self._buffer[start:end] ) - - def _parse_integer(self): - match = re.match(int_regex, self._buffer[self._index:]) - if not match: - raise LLSDParseError("invalid integer token at index %d." % self._index) - - (start, end) = match.span() - start += self._index - end += self._index - self._index = end - return int( self._buffer[start:end] ) - - def _parse_string(self, delim): - """ string: "g\'day" | 'have a "nice" day' | s(size)"raw data" """ - rv = "" - - if delim in ("'", '"'): - rv = self._parse_string_delim(delim) - elif delim == 's': - rv = self._parse_string_raw() - else: - raise LLSDParseError("invalid string token at index %d." % self._index) - - return rv - - - def _parse_string_delim(self, delim): - """ string: "g'day 'un" | 'have a "nice" day' """ - list = [] - found_escape = False - found_hex = False - found_digit = False - byte = 0 - while True: - cc = self._buffer[self._index] - self._index += 1 - if found_escape: - if found_hex: - if found_digit: - found_escape = False - found_hex = False - found_digit = False - byte <<= 4 - byte |= _hex_as_nybble(cc) - list.append(chr(byte)) - byte = 0 - else: - found_digit = True - byte = _hex_as_nybble(cc) - elif cc == 'x': - found_hex = True - else: - if cc == 'a': - list.append('\a') - elif cc == 'b': - list.append('\b') - elif cc == 'f': - list.append('\f') - elif cc == 'n': - list.append('\n') - elif cc == 'r': - list.append('\r') - elif cc == 't': - list.append('\t') - elif cc == 'v': - list.append('\v') - else: - list.append(cc) - found_escape = False - elif cc == '\\': - found_escape = True - elif cc == delim: - break - else: - list.append(cc) - return ''.join(list) - - def _parse_string_raw(self): - """ string: s(size)"raw data" """ - # Read the (size) portion. - cc = self._buffer[self._index] - self._index += 1 - if cc != '(': - raise LLSDParseError("invalid string token at index %d." % self._index) - - rparen = self._buffer.find(')', self._index) - if rparen == -1: - raise LLSDParseError("invalid string token at index %d." % self._index) - - size = int(self._buffer[self._index:rparen]) - - self._index = rparen + 1 - delim = self._buffer[self._index] - self._index += 1 - if delim not in ("'", '"'): - raise LLSDParseError("invalid string token at index %d." % self._index) - - rv = self._buffer[self._index:(self._index + size)] - self._index += size - cc = self._buffer[self._index] - self._index += 1 - if cc != delim: - raise LLSDParseError("invalid string token at index %d." % self._index) - - return rv - -def format_binary(something): - return '\n' + _format_binary_recurse(something) - -def _format_binary_recurse(something): - def _format_list(something): - array_builder = [] - array_builder.append('[' + struct.pack('!i', len(something))) - for item in something: - array_builder.append(_format_binary_recurse(item)) - array_builder.append(']') - return ''.join(array_builder) - - if something is None: - return '!' - elif isinstance(something, LLSD): - return _format_binary_recurse(something.thing) - elif isinstance(something, bool): - if something: - return '1' - else: - return '0' - elif isinstance(something, (int, long)): - return 'i' + struct.pack('!i', something) - elif isinstance(something, float): - return 'r' + struct.pack('!d', something) - elif isinstance(something, lluuid.UUID): - return 'u' + something._bits - elif isinstance(something, binary): - return 'b' + struct.pack('!i', len(something)) + something - elif isinstance(something, str): - return 's' + struct.pack('!i', len(something)) + something - elif isinstance(something, unicode): - something = something.encode('utf-8') - return 's' + struct.pack('!i', len(something)) + something - elif isinstance(something, uri): - return 'l' + struct.pack('!i', len(something)) + something - elif isinstance(something, datetime.datetime): - seconds_since_epoch = time.mktime(something.timetuple()) - return 'd' + struct.pack('!d', seconds_since_epoch) - elif isinstance(something, (list, tuple)): - return _format_list(something) - elif isinstance(something, dict): - map_builder = [] - map_builder.append('{' + struct.pack('!i', len(something))) - for key, value in something.items(): - if isinstance(key, unicode): - key = key.encode('utf-8') - map_builder.append('k' + struct.pack('!i', len(key)) + key) - map_builder.append(_format_binary_recurse(value)) - map_builder.append('}') - return ''.join(map_builder) - else: - try: - return _format_list(list(something)) - except TypeError: - raise LLSDSerializationError( - "Cannot serialize unknown type: %s (%s)" % - (type(something), something)) - - -def parse_binary(binary): - if binary.startswith(''): - just_binary = binary.split('\n', 1)[1] - else: - just_binary = binary - return LLSDBinaryParser().parse(just_binary) - -def parse_xml(something): - try: - return to_python(fromstring(something)[0]) - except ElementTreeError, err: - raise LLSDParseError(*err.args) - -def parse_notation(something): - return LLSDNotationParser().parse(something) - -def parse(something): - try: - something = string.lstrip(something) #remove any pre-trailing whitespace - if something.startswith(''): - return parse_binary(something) - # This should be better. - elif something.startswith('<'): - return parse_xml(something) - else: - return parse_notation(something) - except KeyError, e: - raise Exception('LLSD could not be parsed: %s' % (e,)) - -class LLSD(object): - def __init__(self, thing=None): - self.thing = thing - - def __str__(self): - return self.toXML(self.thing) - - parse = staticmethod(parse) - toXML = staticmethod(format_xml) - toPrettyXML = staticmethod(format_pretty_xml) - toBinary = staticmethod(format_binary) - toNotation = staticmethod(format_notation) - - -undef = LLSD(None) - -XML_MIME_TYPE = 'application/llsd+xml' -BINARY_MIME_TYPE = 'application/llsd+binary' - -# register converters for llsd in mulib, if it is available -try: - from mulib import stacked, mu - stacked.NoProducer() # just to exercise stacked - mu.safe_load(None) # just to exercise mu -except: - # mulib not available, don't print an error message since this is normal - pass -else: - mu.add_parser(parse, XML_MIME_TYPE) - mu.add_parser(parse, 'application/llsd+binary') - - def llsd_convert_xml(llsd_stuff, request): - request.write(format_xml(llsd_stuff)) - - def llsd_convert_binary(llsd_stuff, request): - request.write(format_binary(llsd_stuff)) - - for typ in [LLSD, dict, list, tuple, str, int, long, float, bool, unicode, type(None)]: - stacked.add_producer(typ, llsd_convert_xml, XML_MIME_TYPE) - stacked.add_producer(typ, llsd_convert_xml, 'application/xml') - stacked.add_producer(typ, llsd_convert_xml, 'text/xml') - - stacked.add_producer(typ, llsd_convert_binary, 'application/llsd+binary') - - stacked.add_producer(LLSD, llsd_convert_xml, '*/*') - - # in case someone is using the legacy mu.xml wrapper, we need to - # tell mu to produce application/xml or application/llsd+xml - # (based on the accept header) from raw xml. Phoenix 2008-07-21 - stacked.add_producer(mu.xml, mu.produce_raw, XML_MIME_TYPE) - stacked.add_producer(mu.xml, mu.produce_raw, 'application/xml') - - - -# mulib wsgi stuff -# try: -# from mulib import mu, adapters -# -# # try some known attributes from mulib to be ultra-sure we've imported it -# mu.get_current -# adapters.handlers -# except: -# # mulib not available, don't print an error message since this is normal -# pass -# else: -# def llsd_xml_handler(content_type): -# def handle_llsd_xml(env, start_response): -# llsd_stuff, _ = mu.get_current(env) -# result = format_xml(llsd_stuff) -# start_response("200 OK", [('Content-Type', content_type)]) -# env['mu.negotiated_type'] = content_type -# yield result -# return handle_llsd_xml -# -# def llsd_binary_handler(content_type): -# def handle_llsd_binary(env, start_response): -# llsd_stuff, _ = mu.get_current(env) -# result = format_binary(llsd_stuff) -# start_response("200 OK", [('Content-Type', content_type)]) -# env['mu.negotiated_type'] = content_type -# yield result -# return handle_llsd_binary -# -# adapters.DEFAULT_PARSERS[XML_MIME_TYPE] = parse - -# for typ in [LLSD, dict, list, tuple, str, int, float, bool, unicode, type(None)]: -# for content_type in (XML_MIME_TYPE, 'application/xml'): -# adapters.handlers.set_handler(typ, llsd_xml_handler(content_type), content_type) -# -# adapters.handlers.set_handler(typ, llsd_binary_handler(BINARY_MIME_TYPE), BINARY_MIME_TYPE) -# -# adapters.handlers.set_handler(LLSD, llsd_xml_handler(XML_MIME_TYPE), '*/*') diff --git a/indra/viewer_components/manager/util/fastest_elementtree.py b/indra/viewer_components/manager/util/fastest_elementtree.py new file mode 100755 index 0000000000..4fcf662dd9 --- /dev/null +++ b/indra/viewer_components/manager/util/fastest_elementtree.py @@ -0,0 +1,64 @@ +"""\ +@file fastest_elementtree.py +@brief Concealing some gnarly import logic in here. This should export the interface of elementtree. + +$LicenseInfo:firstyear=2008&license=mit$ + +Copyright (c) 2008-2009, Linden Research, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +$/LicenseInfo$ +""" + +# The parsing exception raised by the underlying library depends +# on the ElementTree implementation we're using, so we provide an +# alias here. +# +# Use ElementTreeError as the exception type for catching parsing +# errors. + + +# Using cElementTree might cause some unforeseen problems, so here's a +# convenient off switch. +use_celementree = True + +try: + if not use_celementree: + raise ImportError() + # Python 2.3 and 2.4. + from cElementTree import * + ElementTreeError = SyntaxError +except ImportError: + try: + if not use_celementree: + raise ImportError() + # Python 2.5 and above. + from xml.etree.cElementTree import * + ElementTreeError = SyntaxError + except ImportError: + # Pure Python code. + try: + # Python 2.3 and 2.4. + from elementtree.ElementTree import * + except ImportError: + # Python 2.5 and above. + from xml.etree.ElementTree import * + + # The pure Python ElementTree module uses Expat for parsing. + from xml.parsers.expat import ExpatError as ElementTreeError -- cgit v1.2.3