diff options
33 files changed, 7 insertions, 6610 deletions
@@ -1,6 +1,6 @@ Second Life Viewer ==================== - + This project manages the source code for the [Second Life](https://www.secondlife.com) Viewer. diff --git a/indra/lib/python/indra/base/__init__.py b/indra/lib/python/indra/base/__init__.py deleted file mode 100755 index 2904fd3380..0000000000 --- a/indra/lib/python/indra/base/__init__.py +++ /dev/null @@ -1,27 +0,0 @@ -"""\ -@file __init__.py -@brief Initialization file for the indra.base module. - -$LicenseInfo:firstyear=2007&license=mit$ - -Copyright (c) 2007-2009, Linden Research, Inc. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -$/LicenseInfo$ -""" diff --git a/indra/lib/python/indra/base/cllsd_test.py b/indra/lib/python/indra/base/cllsd_test.py deleted file mode 100755 index 1f06898ffd..0000000000 --- a/indra/lib/python/indra/base/cllsd_test.py +++ /dev/null @@ -1,73 +0,0 @@ -#!/usr/bin/python -## -## $LicenseInfo:firstyear=2011&license=viewerlgpl$ -## Second Life Viewer Source Code -## Copyright (C) 2011, Linden Research, Inc. -## -## This library is free software; you can redistribute it and/or -## modify it under the terms of the GNU Lesser General Public -## License as published by the Free Software Foundation; -## version 2.1 of the License only. -## -## This library is distributed in the hope that it will be useful, -## but WITHOUT ANY WARRANTY; without even the implied warranty of -## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -## Lesser General Public License for more details. -## -## You should have received a copy of the GNU Lesser General Public -## License along with this library; if not, write to the Free Software -## Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA -## -## Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA -## $/LicenseInfo$ -from indra.base import llsd, lluuid -from datetime import datetime -import cllsd -import time, sys - -class myint(int): - pass - -values = ( - '&<>', - u'\u81acj', - llsd.uri('http://foo<'), - lluuid.UUID(), - llsd.LLSD(['thing']), - 1, - myint(31337), - sys.maxint + 10, - llsd.binary('foo'), - [], - {}, - {u'f&\u1212': 3}, - 3.1, - True, - None, - datetime.fromtimestamp(time.time()), - ) - -def valuator(values): - for v in values: - yield v - -longvalues = () # (values, list(values), iter(values), valuator(values)) - -for v in values + longvalues: - print '%r => %r' % (v, cllsd.llsd_to_xml(v)) - -a = [[{'a':3}]] * 1000000 - -s = time.time() -print hash(cllsd.llsd_to_xml(a)) -e = time.time() -t1 = e - s -print t1 - -s = time.time() -print hash(llsd.LLSDXMLFormatter()._format(a)) -e = time.time() -t2 = e - s -print t2 - -print 'Speedup:', t2 / t1 diff --git a/indra/lib/python/indra/base/config.py b/indra/lib/python/indra/base/config.py deleted file mode 100755 index adafa29b51..0000000000 --- a/indra/lib/python/indra/base/config.py +++ /dev/null @@ -1,266 +0,0 @@ -"""\ -@file config.py -@brief Utility module for parsing and accessing the indra.xml config file. - -$LicenseInfo:firstyear=2006&license=mit$ - -Copyright (c) 2006-2009, Linden Research, Inc. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -$/LicenseInfo$ -""" - -import copy -import errno -import os -import traceback -import time -import types - -from os.path import dirname, getmtime, join, realpath -from indra.base import llsd - -_g_config = None - -class IndraConfig(object): - """ - IndraConfig loads a 'indra' xml configuration file - and loads into memory. This representation in memory - can get updated to overwrite values or add new values. - - The xml configuration file is considered a live file and changes - to the file are checked and reloaded periodically. If a value had - been overwritten via the update or set method, the loaded values - from the file are ignored (the values from the update/set methods - override) - """ - def __init__(self, indra_config_file): - self._indra_config_file = indra_config_file - self._reload_check_interval = 30 # seconds - self._last_check_time = 0 - self._last_mod_time = 0 - - self._config_overrides = {} - self._config_file_dict = {} - self._combined_dict = {} - - self._load() - - def _load(self): - # if you initialize the IndraConfig with None, no attempt - # is made to load any files - if self._indra_config_file is None: - return - - config_file = open(self._indra_config_file) - self._config_file_dict = llsd.parse(config_file.read()) - self._combine_dictionaries() - config_file.close() - - self._last_mod_time = self._get_last_modified_time() - self._last_check_time = time.time() # now - - def _get_last_modified_time(self): - """ - Returns the mtime (last modified time) of the config file, - if such exists. - """ - if self._indra_config_file is not None: - return os.path.getmtime(self._indra_config_file) - - return 0 - - def _combine_dictionaries(self): - self._combined_dict = {} - self._combined_dict.update(self._config_file_dict) - self._combined_dict.update(self._config_overrides) - - def _reload_if_necessary(self): - now = time.time() - - if (now - self._last_check_time) > self._reload_check_interval: - self._last_check_time = now - try: - modtime = self._get_last_modified_time() - if modtime > self._last_mod_time: - self._load() - except OSError, e: - if e.errno == errno.ENOENT: # file not found - # someone messed with our internal state - # or removed the file - - print 'WARNING: Configuration file has been removed ' + (self._indra_config_file) - print 'Disabling reloading of configuration file.' - - traceback.print_exc() - - self._indra_config_file = None - self._last_check_time = 0 - self._last_mod_time = 0 - else: - raise # pass the exception along to the caller - - def __getitem__(self, key): - self._reload_if_necessary() - - return self._combined_dict[key] - - def get(self, key, default = None): - try: - return self.__getitem__(key) - except KeyError: - return default - - def __setitem__(self, key, value): - """ - Sets the value of the config setting of key to be newval - - Once any key/value pair is changed via the set method, - that key/value pair will remain set with that value until - change via the update or set method - """ - self._config_overrides[key] = value - self._combine_dictionaries() - - def set(self, key, newval): - return self.__setitem__(key, newval) - - def update(self, new_conf): - """ - Load an XML file and apply its map as overrides or additions - to the existing config. Update can be a file or a dict. - - Once any key/value pair is changed via the update method, - that key/value pair will remain set with that value until - change via the update or set method - """ - if isinstance(new_conf, dict): - overrides = new_conf - else: - # assuming that it is a filename - config_file = open(new_conf) - overrides = llsd.parse(config_file.read()) - config_file.close() - - self._config_overrides.update(overrides) - self._combine_dictionaries() - - def as_dict(self): - """ - Returns immutable copy of the IndraConfig as a dictionary - """ - return copy.deepcopy(self._combined_dict) - -def load(config_xml_file = None): - global _g_config - - load_default_files = config_xml_file is None - if load_default_files: - ## going from: - ## "/opt/linden/indra/lib/python/indra/base/config.py" - ## to: - ## "/opt/linden/etc/indra.xml" - config_xml_file = realpath( - dirname(realpath(__file__)) + "../../../../../../etc/indra.xml") - - try: - _g_config = IndraConfig(config_xml_file) - except IOError: - # Failure to load passed in file - # or indra.xml default file - if load_default_files: - try: - config_xml_file = realpath( - dirname(realpath(__file__)) + "../../../../../../etc/globals.xml") - _g_config = IndraConfig(config_xml_file) - return - except IOError: - # Failure to load globals.xml - # fall to code below - pass - - # Either failed to load passed in file - # or failed to load all default files - _g_config = IndraConfig(None) - -def dump(indra_xml_file, indra_cfg = None, update_in_mem=False): - ''' - Dump config contents into a file - Kindof reverse of load. - Optionally takes a new config to dump. - Does NOT update global config unless requested. - ''' - global _g_config - - if not indra_cfg: - if _g_config is None: - return - - indra_cfg = _g_config.as_dict() - - if not indra_cfg: - return - - config_file = open(indra_xml_file, 'w') - _config_xml = llsd.format_xml(indra_cfg) - config_file.write(_config_xml) - config_file.close() - - if update_in_mem: - update(indra_cfg) - -def update(new_conf): - global _g_config - - if _g_config is None: - # To keep with how this function behaved - # previously, a call to update - # before the global is defined - # make a new global config which does not - # load data from a file. - _g_config = IndraConfig(None) - - return _g_config.update(new_conf) - -def get(key, default = None): - global _g_config - - if _g_config is None: - load() - - return _g_config.get(key, default) - -def set(key, newval): - """ - Sets the value of the config setting of key to be newval - - Once any key/value pair is changed via the set method, - that key/value pair will remain set with that value until - change via the update or set method or program termination - """ - global _g_config - - if _g_config is None: - _g_config = IndraConfig(None) - - _g_config.set(key, newval) - -def get_config(): - global _g_config - return _g_config diff --git a/indra/lib/python/indra/base/llsd.py b/indra/lib/python/indra/base/llsd.py deleted file mode 100755 index 4527b115f9..0000000000 --- a/indra/lib/python/indra/base/llsd.py +++ /dev/null @@ -1,1052 +0,0 @@ -"""\ -@file llsd.py -@brief Types as well as parsing and formatting functions for handling LLSD. - -$LicenseInfo:firstyear=2006&license=mit$ - -Copyright (c) 2006-2009, Linden Research, Inc. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -$/LicenseInfo$ -""" - -import datetime -import base64 -import string -import struct -import time -import types -import re - -from indra.util.fastest_elementtree import ElementTreeError, fromstring -from indra.base import lluuid - -# cllsd.c in server/server-1.25 has memory leaks, -# so disabling cllsd for now -#try: -# import cllsd -#except ImportError: -# cllsd = None -cllsd = None - -int_regex = re.compile(r"[-+]?\d+") -real_regex = re.compile(r"[-+]?(\d+(\.\d*)?|\d*\.\d+)([eE][-+]?\d+)?") -alpha_regex = re.compile(r"[a-zA-Z]+") -date_regex = re.compile(r"(?P<year>\d{4})-(?P<month>\d{2})-(?P<day>\d{2})T" - r"(?P<hour>\d{2}):(?P<minute>\d{2}):(?P<second>\d{2})" - r"(?P<second_float>(\.\d+)?)Z") -#date: d"YYYY-MM-DDTHH:MM:SS.FFFFFFZ" - -class LLSDParseError(Exception): - pass - -class LLSDSerializationError(TypeError): - pass - - -class binary(str): - pass - -class uri(str): - pass - - -BOOL_TRUE = ('1', '1.0', 'true') -BOOL_FALSE = ('0', '0.0', 'false', '') - - -def format_datestr(v): - """ Formats a datetime or date object into the string format shared by xml and notation serializations.""" - if hasattr(v, 'microsecond'): - return v.isoformat() + 'Z' - else: - return v.strftime('%Y-%m-%dT%H:%M:%SZ') - -def parse_datestr(datestr): - """Parses a datetime object from the string format shared by xml and notation serializations.""" - if datestr == "": - return datetime.datetime(1970, 1, 1) - - match = re.match(date_regex, datestr) - if not match: - raise LLSDParseError("invalid date string '%s'." % datestr) - - year = int(match.group('year')) - month = int(match.group('month')) - day = int(match.group('day')) - hour = int(match.group('hour')) - minute = int(match.group('minute')) - second = int(match.group('second')) - seconds_float = match.group('second_float') - microsecond = 0 - if seconds_float: - microsecond = int(float('0' + seconds_float) * 1e6) - return datetime.datetime(year, month, day, hour, minute, second, microsecond) - - -def bool_to_python(node): - val = node.text or '' - if val in BOOL_TRUE: - return True - else: - return False - -def int_to_python(node): - val = node.text or '' - if not val.strip(): - return 0 - return int(val) - -def real_to_python(node): - val = node.text or '' - if not val.strip(): - return 0.0 - return float(val) - -def uuid_to_python(node): - return lluuid.UUID(node.text) - -def str_to_python(node): - return node.text or '' - -def bin_to_python(node): - return binary(base64.decodestring(node.text or '')) - -def date_to_python(node): - val = node.text or '' - if not val: - val = "1970-01-01T00:00:00Z" - return parse_datestr(val) - - -def uri_to_python(node): - val = node.text or '' - if not val: - return None - return uri(val) - -def map_to_python(node): - result = {} - for index in range(len(node))[::2]: - result[node[index].text] = to_python(node[index+1]) - return result - -def array_to_python(node): - return [to_python(child) for child in node] - - -NODE_HANDLERS = dict( - undef=lambda x: None, - boolean=bool_to_python, - integer=int_to_python, - real=real_to_python, - uuid=uuid_to_python, - string=str_to_python, - binary=bin_to_python, - date=date_to_python, - uri=uri_to_python, - map=map_to_python, - array=array_to_python, - ) - -def to_python(node): - return NODE_HANDLERS[node.tag](node) - -class Nothing(object): - pass - - -class LLSDXMLFormatter(object): - def __init__(self): - self.type_map = { - type(None) : self.UNDEF, - bool : self.BOOLEAN, - int : self.INTEGER, - long : self.INTEGER, - float : self.REAL, - lluuid.UUID : self.UUID, - binary : self.BINARY, - str : self.STRING, - unicode : self.STRING, - uri : self.URI, - datetime.datetime : self.DATE, - datetime.date : self.DATE, - list : self.ARRAY, - tuple : self.ARRAY, - types.GeneratorType : self.ARRAY, - dict : self.MAP, - LLSD : self.LLSD - } - - def elt(self, name, contents=None): - if(contents is None or contents is ''): - return "<%s />" % (name,) - else: - if type(contents) is unicode: - contents = contents.encode('utf-8') - return "<%s>%s</%s>" % (name, contents, name) - - def xml_esc(self, v): - if type(v) is unicode: - v = v.encode('utf-8') - return v.replace('&', '&').replace('<', '<').replace('>', '>') - - def LLSD(self, v): - return self.generate(v.thing) - def UNDEF(self, v): - return self.elt('undef') - def BOOLEAN(self, v): - if v: - return self.elt('boolean', 'true') - else: - return self.elt('boolean', 'false') - def INTEGER(self, v): - return self.elt('integer', v) - def REAL(self, v): - return self.elt('real', v) - def UUID(self, v): - if(v.isNull()): - return self.elt('uuid') - else: - return self.elt('uuid', v) - def BINARY(self, v): - return self.elt('binary', base64.encodestring(v)) - def STRING(self, v): - return self.elt('string', self.xml_esc(v)) - def URI(self, v): - return self.elt('uri', self.xml_esc(str(v))) - def DATE(self, v): - return self.elt('date', format_datestr(v)) - def ARRAY(self, v): - return self.elt('array', ''.join([self.generate(item) for item in v])) - def MAP(self, v): - return self.elt( - 'map', - ''.join(["%s%s" % (self.elt('key', self.xml_esc(str(key))), self.generate(value)) - for key, value in v.items()])) - - typeof = type - def generate(self, something): - t = self.typeof(something) - if self.type_map.has_key(t): - return self.type_map[t](something) - else: - raise LLSDSerializationError("Cannot serialize unknown type: %s (%s)" % ( - t, something)) - - def _format(self, something): - return '<?xml version="1.0" ?>' + self.elt("llsd", self.generate(something)) - - def format(self, something): - if cllsd: - return cllsd.llsd_to_xml(something) - return self._format(something) - -_g_xml_formatter = None -def format_xml(something): - global _g_xml_formatter - if _g_xml_formatter is None: - _g_xml_formatter = LLSDXMLFormatter() - return _g_xml_formatter.format(something) - -class LLSDXMLPrettyFormatter(LLSDXMLFormatter): - def __init__(self, indent_atom = None): - # Call the super class constructor so that we have the type map - super(LLSDXMLPrettyFormatter, self).__init__() - - # Override the type map to use our specialized formatters to - # emit the pretty output. - self.type_map[list] = self.PRETTY_ARRAY - self.type_map[tuple] = self.PRETTY_ARRAY - self.type_map[types.GeneratorType] = self.PRETTY_ARRAY, - self.type_map[dict] = self.PRETTY_MAP - - # Private data used for indentation. - self._indent_level = 1 - if indent_atom is None: - self._indent_atom = ' ' - else: - self._indent_atom = indent_atom - - def _indent(self): - "Return an indentation based on the atom and indentation level." - return self._indent_atom * self._indent_level - - def PRETTY_ARRAY(self, v): - rv = [] - rv.append('<array>\n') - self._indent_level = self._indent_level + 1 - rv.extend(["%s%s\n" % - (self._indent(), - self.generate(item)) - for item in v]) - self._indent_level = self._indent_level - 1 - rv.append(self._indent()) - rv.append('</array>') - return ''.join(rv) - - def PRETTY_MAP(self, v): - rv = [] - rv.append('<map>\n') - self._indent_level = self._indent_level + 1 - keys = v.keys() - keys.sort() - rv.extend(["%s%s\n%s%s\n" % - (self._indent(), - self.elt('key', key), - self._indent(), - self.generate(v[key])) - for key in keys]) - self._indent_level = self._indent_level - 1 - rv.append(self._indent()) - rv.append('</map>') - return ''.join(rv) - - def format(self, something): - data = [] - data.append('<?xml version="1.0" ?>\n<llsd>') - data.append(self.generate(something)) - data.append('</llsd>\n') - return '\n'.join(data) - -def format_pretty_xml(something): - """@brief Serialize a python object as 'pretty' llsd xml. - - The output conforms to the LLSD DTD, unlike the output from the - standard python xml.dom DOM::toprettyxml() method which does not - preserve significant whitespace. - This function is not necessarily suited for serializing very large - objects. It is not optimized by the cllsd module, and sorts on - dict (llsd map) keys alphabetically to ease human reading. - """ - return LLSDXMLPrettyFormatter().format(something) - -class LLSDNotationFormatter(object): - def __init__(self): - self.type_map = { - type(None) : self.UNDEF, - bool : self.BOOLEAN, - int : self.INTEGER, - long : self.INTEGER, - float : self.REAL, - lluuid.UUID : self.UUID, - binary : self.BINARY, - str : self.STRING, - unicode : self.STRING, - uri : self.URI, - datetime.datetime : self.DATE, - datetime.date : self.DATE, - list : self.ARRAY, - tuple : self.ARRAY, - types.GeneratorType : self.ARRAY, - dict : self.MAP, - LLSD : self.LLSD - } - - def LLSD(self, v): - return self.generate(v.thing) - def UNDEF(self, v): - return '!' - def BOOLEAN(self, v): - if v: - return 'true' - else: - return 'false' - def INTEGER(self, v): - return "i%s" % v - def REAL(self, v): - return "r%s" % v - def UUID(self, v): - return "u%s" % v - def BINARY(self, v): - return 'b64"' + base64.encodestring(v) + '"' - def STRING(self, v): - if isinstance(v, unicode): - v = v.encode('utf-8') - return "'%s'" % v.replace("\\", "\\\\").replace("'", "\\'") - def URI(self, v): - return 'l"%s"' % str(v).replace("\\", "\\\\").replace('"', '\\"') - def DATE(self, v): - return 'd"%s"' % format_datestr(v) - def ARRAY(self, v): - return "[%s]" % ','.join([self.generate(item) for item in v]) - def MAP(self, v): - def fix(key): - if isinstance(key, unicode): - return key.encode('utf-8') - return key - return "{%s}" % ','.join(["'%s':%s" % (fix(key).replace("\\", "\\\\").replace("'", "\\'"), self.generate(value)) - for key, value in v.items()]) - - def generate(self, something): - t = type(something) - handler = self.type_map.get(t) - if handler: - return handler(something) - else: - try: - return self.ARRAY(iter(something)) - except TypeError: - raise LLSDSerializationError( - "Cannot serialize unknown type: %s (%s)" % (t, something)) - - def format(self, something): - return self.generate(something) - -def format_notation(something): - return LLSDNotationFormatter().format(something) - -def _hex_as_nybble(hex): - if (hex >= '0') and (hex <= '9'): - return ord(hex) - ord('0') - elif (hex >= 'a') and (hex <='f'): - return 10 + ord(hex) - ord('a') - elif (hex >= 'A') and (hex <='F'): - return 10 + ord(hex) - ord('A'); - -class LLSDBinaryParser(object): - def __init__(self): - pass - - def parse(self, buffer, ignore_binary = False): - """ - This is the basic public interface for parsing. - - @param buffer the binary data to parse in an indexable sequence. - @param ignore_binary parser throws away data in llsd binary nodes. - @return returns a python object. - """ - self._buffer = buffer - self._index = 0 - self._keep_binary = not ignore_binary - return self._parse() - - def _parse(self): - cc = self._buffer[self._index] - self._index += 1 - if cc == '{': - return self._parse_map() - elif cc == '[': - return self._parse_array() - elif cc == '!': - return None - elif cc == '0': - return False - elif cc == '1': - return True - elif cc == 'i': - # 'i' = integer - idx = self._index - self._index += 4 - return struct.unpack("!i", self._buffer[idx:idx+4])[0] - elif cc == ('r'): - # 'r' = real number - idx = self._index - self._index += 8 - return struct.unpack("!d", self._buffer[idx:idx+8])[0] - elif cc == 'u': - # 'u' = uuid - idx = self._index - self._index += 16 - return lluuid.uuid_bits_to_uuid(self._buffer[idx:idx+16]) - elif cc == 's': - # 's' = string - return self._parse_string() - elif cc in ("'", '"'): - # delimited/escaped string - return self._parse_string_delim(cc) - elif cc == 'l': - # 'l' = uri - return uri(self._parse_string()) - elif cc == ('d'): - # 'd' = date in seconds since epoch - idx = self._index - self._index += 8 - seconds = struct.unpack("!d", self._buffer[idx:idx+8])[0] - return datetime.datetime.fromtimestamp(seconds) - elif cc == 'b': - binary = self._parse_string() - if self._keep_binary: - return binary - # *NOTE: maybe have a binary placeholder which has the - # length. - return None - else: - raise LLSDParseError("invalid binary token at byte %d: %d" % ( - self._index - 1, ord(cc))) - - def _parse_map(self): - rv = {} - size = struct.unpack("!i", self._buffer[self._index:self._index+4])[0] - self._index += 4 - count = 0 - cc = self._buffer[self._index] - self._index += 1 - key = '' - while (cc != '}') and (count < size): - if cc == 'k': - key = self._parse_string() - elif cc in ("'", '"'): - key = self._parse_string_delim(cc) - else: - raise LLSDParseError("invalid map key at byte %d." % ( - self._index - 1,)) - value = self._parse() - rv[key] = value - count += 1 - cc = self._buffer[self._index] - self._index += 1 - if cc != '}': - raise LLSDParseError("invalid map close token at byte %d." % ( - self._index,)) - return rv - - def _parse_array(self): - rv = [] - size = struct.unpack("!i", self._buffer[self._index:self._index+4])[0] - self._index += 4 - count = 0 - cc = self._buffer[self._index] - while (cc != ']') and (count < size): - rv.append(self._parse()) - count += 1 - cc = self._buffer[self._index] - if cc != ']': - raise LLSDParseError("invalid array close token at byte %d." % ( - self._index,)) - self._index += 1 - return rv - - def _parse_string(self): - size = struct.unpack("!i", self._buffer[self._index:self._index+4])[0] - self._index += 4 - rv = self._buffer[self._index:self._index+size] - self._index += size - return rv - - def _parse_string_delim(self, delim): - list = [] - found_escape = False - found_hex = False - found_digit = False - byte = 0 - while True: - cc = self._buffer[self._index] - self._index += 1 - if found_escape: - if found_hex: - if found_digit: - found_escape = False - found_hex = False - found_digit = False - byte <<= 4 - byte |= _hex_as_nybble(cc) - list.append(chr(byte)) - byte = 0 - else: - found_digit = True - byte = _hex_as_nybble(cc) - elif cc == 'x': - found_hex = True - else: - if cc == 'a': - list.append('\a') - elif cc == 'b': - list.append('\b') - elif cc == 'f': - list.append('\f') - elif cc == 'n': - list.append('\n') - elif cc == 'r': - list.append('\r') - elif cc == 't': - list.append('\t') - elif cc == 'v': - list.append('\v') - else: - list.append(cc) - found_escape = False - elif cc == '\\': - found_escape = True - elif cc == delim: - break - else: - list.append(cc) - return ''.join(list) - -class LLSDNotationParser(object): - """ Parse LLSD notation: - map: { string:object, string:object } - array: [ object, object, object ] - undef: ! - boolean: true | false | 1 | 0 | T | F | t | f | TRUE | FALSE - integer: i#### - real: r#### - uuid: u#### - string: "g\'day" | 'have a "nice" day' | s(size)"raw data" - uri: l"escaped" - date: d"YYYY-MM-DDTHH:MM:SS.FFZ" - binary: b##"ff3120ab1" | b(size)"raw data" - """ - def __init__(self): - pass - - def parse(self, buffer, ignore_binary = False): - """ - This is the basic public interface for parsing. - - @param buffer the notation string to parse. - @param ignore_binary parser throws away data in llsd binary nodes. - @return returns a python object. - """ - if buffer == "": - return False - - self._buffer = buffer - self._index = 0 - return self._parse() - - def _parse(self): - cc = self._buffer[self._index] - self._index += 1 - if cc == '{': - return self._parse_map() - elif cc == '[': - return self._parse_array() - elif cc == '!': - return None - elif cc == '0': - return False - elif cc == '1': - return True - elif cc in ('F', 'f'): - self._skip_alpha() - return False - elif cc in ('T', 't'): - self._skip_alpha() - return True - elif cc == 'i': - # 'i' = integer - return self._parse_integer() - elif cc == ('r'): - # 'r' = real number - return self._parse_real() - elif cc == 'u': - # 'u' = uuid - return self._parse_uuid() - elif cc in ("'", '"', 's'): - return self._parse_string(cc) - elif cc == 'l': - # 'l' = uri - delim = self._buffer[self._index] - self._index += 1 - val = uri(self._parse_string(delim)) - if len(val) == 0: - return None - return val - elif cc == ('d'): - # 'd' = date in seconds since epoch - return self._parse_date() - elif cc == 'b': - return self._parse_binary() - else: - raise LLSDParseError("invalid token at index %d: %d" % ( - self._index - 1, ord(cc))) - - def _parse_binary(self): - i = self._index - if self._buffer[i:i+2] == '64': - q = self._buffer[i+2] - e = self._buffer.find(q, i+3) - try: - return base64.decodestring(self._buffer[i+3:e]) - finally: - self._index = e + 1 - else: - raise LLSDParseError('random horrible binary format not supported') - - def _parse_map(self): - """ map: { string:object, string:object } """ - rv = {} - cc = self._buffer[self._index] - self._index += 1 - key = '' - found_key = False - while (cc != '}'): - if not found_key: - if cc in ("'", '"', 's'): - key = self._parse_string(cc) - found_key = True - elif cc.isspace() or cc == ',': - cc = self._buffer[self._index] - self._index += 1 - else: - raise LLSDParseError("invalid map key at byte %d." % ( - self._index - 1,)) - elif cc.isspace() or cc == ':': - cc = self._buffer[self._index] - self._index += 1 - continue - else: - self._index += 1 - value = self._parse() - rv[key] = value - found_key = False - cc = self._buffer[self._index] - self._index += 1 - - return rv - - def _parse_array(self): - """ array: [ object, object, object ] """ - rv = [] - cc = self._buffer[self._index] - while (cc != ']'): - if cc.isspace() or cc == ',': - self._index += 1 - cc = self._buffer[self._index] - continue - rv.append(self._parse()) - cc = self._buffer[self._index] - - if cc != ']': - raise LLSDParseError("invalid array close token at index %d." % ( - self._index,)) - self._index += 1 - return rv - - def _parse_uuid(self): - match = re.match(lluuid.UUID.uuid_regex, self._buffer[self._index:]) - if not match: - raise LLSDParseError("invalid uuid token at index %d." % self._index) - - (start, end) = match.span() - start += self._index - end += self._index - self._index = end - return lluuid.UUID(self._buffer[start:end]) - - def _skip_alpha(self): - match = re.match(alpha_regex, self._buffer[self._index:]) - if match: - self._index += match.end() - - def _parse_date(self): - delim = self._buffer[self._index] - self._index += 1 - datestr = self._parse_string(delim) - return parse_datestr(datestr) - - def _parse_real(self): - match = re.match(real_regex, self._buffer[self._index:]) - if not match: - raise LLSDParseError("invalid real token at index %d." % self._index) - - (start, end) = match.span() - start += self._index - end += self._index - self._index = end - return float( self._buffer[start:end] ) - - def _parse_integer(self): - match = re.match(int_regex, self._buffer[self._index:]) - if not match: - raise LLSDParseError("invalid integer token at index %d." % self._index) - - (start, end) = match.span() - start += self._index - end += self._index - self._index = end - return int( self._buffer[start:end] ) - - def _parse_string(self, delim): - """ string: "g\'day" | 'have a "nice" day' | s(size)"raw data" """ - rv = "" - - if delim in ("'", '"'): - rv = self._parse_string_delim(delim) - elif delim == 's': - rv = self._parse_string_raw() - else: - raise LLSDParseError("invalid string token at index %d." % self._index) - - return rv - - - def _parse_string_delim(self, delim): - """ string: "g'day 'un" | 'have a "nice" day' """ - list = [] - found_escape = False - found_hex = False - found_digit = False - byte = 0 - while True: - cc = self._buffer[self._index] - self._index += 1 - if found_escape: - if found_hex: - if found_digit: - found_escape = False - found_hex = False - found_digit = False - byte <<= 4 - byte |= _hex_as_nybble(cc) - list.append(chr(byte)) - byte = 0 - else: - found_digit = True - byte = _hex_as_nybble(cc) - elif cc == 'x': - found_hex = True - else: - if cc == 'a': - list.append('\a') - elif cc == 'b': - list.append('\b') - elif cc == 'f': - list.append('\f') - elif cc == 'n': - list.append('\n') - elif cc == 'r': - list.append('\r') - elif cc == 't': - list.append('\t') - elif cc == 'v': - list.append('\v') - else: - list.append(cc) - found_escape = False - elif cc == '\\': - found_escape = True - elif cc == delim: - break - else: - list.append(cc) - return ''.join(list) - - def _parse_string_raw(self): - """ string: s(size)"raw data" """ - # Read the (size) portion. - cc = self._buffer[self._index] - self._index += 1 - if cc != '(': - raise LLSDParseError("invalid string token at index %d." % self._index) - - rparen = self._buffer.find(')', self._index) - if rparen == -1: - raise LLSDParseError("invalid string token at index %d." % self._index) - - size = int(self._buffer[self._index:rparen]) - - self._index = rparen + 1 - delim = self._buffer[self._index] - self._index += 1 - if delim not in ("'", '"'): - raise LLSDParseError("invalid string token at index %d." % self._index) - - rv = self._buffer[self._index:(self._index + size)] - self._index += size - cc = self._buffer[self._index] - self._index += 1 - if cc != delim: - raise LLSDParseError("invalid string token at index %d." % self._index) - - return rv - -def format_binary(something): - return '<?llsd/binary?>\n' + _format_binary_recurse(something) - -def _format_binary_recurse(something): - def _format_list(something): - array_builder = [] - array_builder.append('[' + struct.pack('!i', len(something))) - for item in something: - array_builder.append(_format_binary_recurse(item)) - array_builder.append(']') - return ''.join(array_builder) - - if something is None: - return '!' - elif isinstance(something, LLSD): - return _format_binary_recurse(something.thing) - elif isinstance(something, bool): - if something: - return '1' - else: - return '0' - elif isinstance(something, (int, long)): - return 'i' + struct.pack('!i', something) - elif isinstance(something, float): - return 'r' + struct.pack('!d', something) - elif isinstance(something, lluuid.UUID): - return 'u' + something._bits - elif isinstance(something, binary): - return 'b' + struct.pack('!i', len(something)) + something - elif isinstance(something, str): - return 's' + struct.pack('!i', len(something)) + something - elif isinstance(something, unicode): - something = something.encode('utf-8') - return 's' + struct.pack('!i', len(something)) + something - elif isinstance(something, uri): - return 'l' + struct.pack('!i', len(something)) + something - elif isinstance(something, datetime.datetime): - seconds_since_epoch = time.mktime(something.timetuple()) - return 'd' + struct.pack('!d', seconds_since_epoch) - elif isinstance(something, (list, tuple)): - return _format_list(something) - elif isinstance(something, dict): - map_builder = [] - map_builder.append('{' + struct.pack('!i', len(something))) - for key, value in something.items(): - if isinstance(key, unicode): - key = key.encode('utf-8') - map_builder.append('k' + struct.pack('!i', len(key)) + key) - map_builder.append(_format_binary_recurse(value)) - map_builder.append('}') - return ''.join(map_builder) - else: - try: - return _format_list(list(something)) - except TypeError: - raise LLSDSerializationError( - "Cannot serialize unknown type: %s (%s)" % - (type(something), something)) - - -def parse_binary(binary): - if binary.startswith('<?llsd/binary?>'): - just_binary = binary.split('\n', 1)[1] - else: - just_binary = binary - return LLSDBinaryParser().parse(just_binary) - -def parse_xml(something): - try: - return to_python(fromstring(something)[0]) - except ElementTreeError, err: - raise LLSDParseError(*err.args) - -def parse_notation(something): - return LLSDNotationParser().parse(something) - -def parse(something): - try: - something = string.lstrip(something) #remove any pre-trailing whitespace - if something.startswith('<?llsd/binary?>'): - return parse_binary(something) - # This should be better. - elif something.startswith('<'): - return parse_xml(something) - else: - return parse_notation(something) - except KeyError, e: - raise Exception('LLSD could not be parsed: %s' % (e,)) - -class LLSD(object): - def __init__(self, thing=None): - self.thing = thing - - def __str__(self): - return self.toXML(self.thing) - - parse = staticmethod(parse) - toXML = staticmethod(format_xml) - toPrettyXML = staticmethod(format_pretty_xml) - toBinary = staticmethod(format_binary) - toNotation = staticmethod(format_notation) - - -undef = LLSD(None) - -XML_MIME_TYPE = 'application/llsd+xml' -BINARY_MIME_TYPE = 'application/llsd+binary' - -# register converters for llsd in mulib, if it is available -try: - from mulib import stacked, mu - stacked.NoProducer() # just to exercise stacked - mu.safe_load(None) # just to exercise mu -except: - # mulib not available, don't print an error message since this is normal - pass -else: - mu.add_parser(parse, XML_MIME_TYPE) - mu.add_parser(parse, 'application/llsd+binary') - - def llsd_convert_xml(llsd_stuff, request): - request.write(format_xml(llsd_stuff)) - - def llsd_convert_binary(llsd_stuff, request): - request.write(format_binary(llsd_stuff)) - - for typ in [LLSD, dict, list, tuple, str, int, long, float, bool, unicode, type(None)]: - stacked.add_producer(typ, llsd_convert_xml, XML_MIME_TYPE) - stacked.add_producer(typ, llsd_convert_xml, 'application/xml') - stacked.add_producer(typ, llsd_convert_xml, 'text/xml') - - stacked.add_producer(typ, llsd_convert_binary, 'application/llsd+binary') - - stacked.add_producer(LLSD, llsd_convert_xml, '*/*') - - # in case someone is using the legacy mu.xml wrapper, we need to - # tell mu to produce application/xml or application/llsd+xml - # (based on the accept header) from raw xml. Phoenix 2008-07-21 - stacked.add_producer(mu.xml, mu.produce_raw, XML_MIME_TYPE) - stacked.add_producer(mu.xml, mu.produce_raw, 'application/xml') - - - -# mulib wsgi stuff -# try: -# from mulib import mu, adapters -# -# # try some known attributes from mulib to be ultra-sure we've imported it -# mu.get_current -# adapters.handlers -# except: -# # mulib not available, don't print an error message since this is normal -# pass -# else: -# def llsd_xml_handler(content_type): -# def handle_llsd_xml(env, start_response): -# llsd_stuff, _ = mu.get_current(env) -# result = format_xml(llsd_stuff) -# start_response("200 OK", [('Content-Type', content_type)]) -# env['mu.negotiated_type'] = content_type -# yield result -# return handle_llsd_xml -# -# def llsd_binary_handler(content_type): -# def handle_llsd_binary(env, start_response): -# llsd_stuff, _ = mu.get_current(env) -# result = format_binary(llsd_stuff) -# start_response("200 OK", [('Content-Type', content_type)]) -# env['mu.negotiated_type'] = content_type -# yield result -# return handle_llsd_binary -# -# adapters.DEFAULT_PARSERS[XML_MIME_TYPE] = parse - -# for typ in [LLSD, dict, list, tuple, str, int, float, bool, unicode, type(None)]: -# for content_type in (XML_MIME_TYPE, 'application/xml'): -# adapters.handlers.set_handler(typ, llsd_xml_handler(content_type), content_type) -# -# adapters.handlers.set_handler(typ, llsd_binary_handler(BINARY_MIME_TYPE), BINARY_MIME_TYPE) -# -# adapters.handlers.set_handler(LLSD, llsd_xml_handler(XML_MIME_TYPE), '*/*') diff --git a/indra/lib/python/indra/base/lluuid.py b/indra/lib/python/indra/base/lluuid.py deleted file mode 100755 index 7413ffe10d..0000000000 --- a/indra/lib/python/indra/base/lluuid.py +++ /dev/null @@ -1,319 +0,0 @@ -"""\ -@file lluuid.py -@brief UUID parser/generator. - -$LicenseInfo:firstyear=2004&license=mit$ - -Copyright (c) 2004-2009, Linden Research, Inc. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -$/LicenseInfo$ -""" - -import random, socket, string, time, re -import uuid -try: - # Python 2.6 - from hashlib import md5 -except ImportError: - # Python 2.5 and earlier - from md5 import new as md5 - -def _int2binstr(i,l): - s='' - for a in range(l): - s=chr(i&0xFF)+s - i>>=8 - return s - -def _binstr2int(s): - i = long(0) - for c in s: - i = (i<<8) + ord(c) - return i - -class UUID(object): - """ - A class which represents a 16 byte integer. Stored as a 16 byte 8 - bit character string. - - The string version is to be of the form: - AAAAAAAA-AAAA-BBBB-BBBB-BBBBBBCCCCCC (a 128-bit number in hex) - where A=network address, B=timestamp, C=random. - """ - - NULL_STR = "00000000-0000-0000-0000-000000000000" - - # the UUIDREGEX_STRING is helpful for parsing UUID's in text - hex_wildcard = r"[0-9a-fA-F]" - word = hex_wildcard + r"{4,4}-" - long_word = hex_wildcard + r"{8,8}-" - very_long_word = hex_wildcard + r"{12,12}" - UUID_REGEX_STRING = long_word + word + word + word + very_long_word - uuid_regex = re.compile(UUID_REGEX_STRING) - - rand = random.Random() - ip = '' - try: - ip = socket.gethostbyname(socket.gethostname()) - except(socket.gaierror, socket.error): - # no ip address, so just default to somewhere in 10.x.x.x - ip = '10' - for i in range(3): - ip += '.' + str(rand.randrange(1,254)) - hexip = ''.join(["%04x" % long(i) for i in ip.split('.')]) - lastid = '' - - def __init__(self, possible_uuid=None): - """ - Initialize to first valid UUID in argument (if a string), - or to null UUID if none found or argument is not supplied. - - If the argument is a UUID, the constructed object will be a copy of it. - """ - self._bits = "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0" - if possible_uuid is None: - return - - if isinstance(possible_uuid, type(self)): - self.set(possible_uuid) - return - - uuid_match = UUID.uuid_regex.search(possible_uuid) - if uuid_match: - uuid_string = uuid_match.group() - s = string.replace(uuid_string, '-', '') - self._bits = _int2binstr(string.atol(s[:8],16),4) + \ - _int2binstr(string.atol(s[8:16],16),4) + \ - _int2binstr(string.atol(s[16:24],16),4) + \ - _int2binstr(string.atol(s[24:],16),4) - - def __len__(self): - """ - Used by the len() builtin. - """ - return 36 - - def __nonzero__(self): - return self._bits != "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0" - - def __str__(self): - uuid_string = self.toString() - return uuid_string - - __repr__ = __str__ - - def __getitem__(self, index): - return str(self)[index] - - def __eq__(self, other): - if isinstance(other, (str, unicode)): - return other == str(self) - return self._bits == getattr(other, '_bits', '') - - def __ne__(self, other): - return not self.__eq__(other) - - def __le__(self, other): - return self._bits <= other._bits - - def __ge__(self, other): - return self._bits >= other._bits - - def __lt__(self, other): - return self._bits < other._bits - - def __gt__(self, other): - return self._bits > other._bits - - def __hash__(self): - return hash(self._bits) - - def set(self, uuid): - self._bits = uuid._bits - - def setFromString(self, uuid_string): - """ - Given a string version of a uuid, set self bits - appropriately. Returns self. - """ - s = string.replace(uuid_string, '-', '') - self._bits = _int2binstr(string.atol(s[:8],16),4) + \ - _int2binstr(string.atol(s[8:16],16),4) + \ - _int2binstr(string.atol(s[16:24],16),4) + \ - _int2binstr(string.atol(s[24:],16),4) - return self - - def setFromMemoryDump(self, gdb_string): - """ - We expect to get gdb_string as four hex units. eg: - 0x147d54db 0xc34b3f1b 0x714f989b 0x0a892fd2 - Which will be translated to: - db547d14-1b3f4bc3-9b984f71-d22f890a - Returns self. - """ - s = string.replace(gdb_string, '0x', '') - s = string.replace(s, ' ', '') - t = '' - for i in range(8,40,8): - for j in range(0,8,2): - t = t + s[i-j-2:i-j] - self.setFromString(t) - - def toString(self): - """ - Return as a string matching the LL standard - AAAAAAAA-AAAA-BBBB-BBBB-BBBBBBCCCCCC (a 128-bit number in hex) - where A=network address, B=timestamp, C=random. - """ - return uuid_bits_to_string(self._bits) - - def getAsString(self): - """ - Return a different string representation of the form - AAAAAAAA-AAAABBBB-BBBBBBBB-BBCCCCCC (a 128-bit number in hex) - where A=network address, B=timestamp, C=random. - """ - i1 = _binstr2int(self._bits[0:4]) - i2 = _binstr2int(self._bits[4:8]) - i3 = _binstr2int(self._bits[8:12]) - i4 = _binstr2int(self._bits[12:16]) - return '%08lx-%08lx-%08lx-%08lx' % (i1,i2,i3,i4) - - def generate(self): - """ - Generate a new uuid. This algorithm is slightly different - from c++ implementation for portability reasons. - Returns self. - """ - m = md5() - m.update(uuid.uuid1().bytes) - self._bits = m.digest() - return self - - def isNull(self): - """ - Returns 1 if the uuid is null - ie, equal to default uuid. - """ - return (self._bits == "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0") - - def xor(self, rhs): - """ - xors self with rhs. - """ - v1 = _binstr2int(self._bits[0:4]) ^ _binstr2int(rhs._bits[0:4]) - v2 = _binstr2int(self._bits[4:8]) ^ _binstr2int(rhs._bits[4:8]) - v3 = _binstr2int(self._bits[8:12]) ^ _binstr2int(rhs._bits[8:12]) - v4 = _binstr2int(self._bits[12:16]) ^ _binstr2int(rhs._bits[12:16]) - self._bits = _int2binstr(v1,4) + \ - _int2binstr(v2,4) + \ - _int2binstr(v3,4) + \ - _int2binstr(v4,4) - - -# module-level null constant -NULL = UUID() - -def printTranslatedMemory(four_hex_uints): - """ - We expect to get the string as four hex units. eg: - 0x147d54db 0xc34b3f1b 0x714f989b 0x0a892fd2 - Which will be translated to: - db547d14-1b3f4bc3-9b984f71-d22f890a - """ - uuid = UUID() - uuid.setFromMemoryDump(four_hex_uints) - print uuid.toString() - -def isUUID(id_str): - """ - This function returns: - - 1 if the string passed is a UUID - - 0 is the string passed is not a UUID - - None if it neither of the if's below is satisfied - """ - if not id_str or len(id_str) < 5 or len(id_str) > 36: - return 0 - - if isinstance(id_str, UUID) or UUID.uuid_regex.match(id_str): - return 1 - - return None - -def isPossiblyID(id_str): - """ - This function returns 1 if the string passed has some uuid-like - characteristics. Otherwise returns 0. - """ - - is_uuid = isUUID(id_str) - if is_uuid is not None: - return is_uuid - - # build a string which matches every character. - hex_wildcard = r"[0-9a-fA-F]" - chars = len(id_str) - next = min(chars, 8) - matcher = hex_wildcard+"{"+str(next)+","+str(next)+"}" - chars = chars - next - if chars > 0: - matcher = matcher + "-" - chars = chars - 1 - for block in range(3): - next = max(min(chars, 4), 0) - if next: - matcher = matcher + hex_wildcard+"{"+str(next)+","+str(next)+"}" - chars = chars - next - if chars > 0: - matcher = matcher + "-" - chars = chars - 1 - if chars > 0: - next = min(chars, 12) - matcher = matcher + hex_wildcard+"{"+str(next)+","+str(next)+"}" - #print matcher - uuid_matcher = re.compile(matcher) - if uuid_matcher.match(id_str): - return 1 - return 0 - -def uuid_bits_to_string(bits): - i1 = _binstr2int(bits[0:4]) - i2 = _binstr2int(bits[4:6]) - i3 = _binstr2int(bits[6:8]) - i4 = _binstr2int(bits[8:10]) - i5 = _binstr2int(bits[10:12]) - i6 = _binstr2int(bits[12:16]) - return '%08lx-%04lx-%04lx-%04lx-%04lx%08lx' % (i1,i2,i3,i4,i5,i6) - -def uuid_bits_to_uuid(bits): - return UUID(uuid_bits_to_string(bits)) - - -try: - from mulib import stacked - stacked.NoProducer() # just to exercise stacked -except: - #print "Couldn't import mulib.stacked, not registering UUID converter" - pass -else: - def convertUUID(uuid, req): - req.write(str(uuid)) - - stacked.add_producer(UUID, convertUUID, "*/*") - stacked.add_producer(UUID, convertUUID, "text/html") diff --git a/indra/lib/python/indra/base/metrics.py b/indra/lib/python/indra/base/metrics.py deleted file mode 100755 index ff8380265f..0000000000 --- a/indra/lib/python/indra/base/metrics.py +++ /dev/null @@ -1,121 +0,0 @@ -"""\ -@file metrics.py -@author Phoenix -@date 2007-11-27 -@brief simple interface for logging metrics - -$LicenseInfo:firstyear=2007&license=mit$ - -Copyright (c) 2007-2009, Linden Research, Inc. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -$/LicenseInfo$ -""" - -import sys -try: - import syslog -except ImportError: - # Windows - import sys - class syslog(object): - # wrap to a lame syslog for windows - _logfp = sys.stderr - def syslog(msg): - _logfp.write(msg) - if not msg.endswith('\n'): - _logfp.write('\n') - syslog = staticmethod(syslog) - -from indra.base.llsd import format_notation - -def record_metrics(table, stats): - "Write a standard metrics log" - _log("LLMETRICS", table, stats) - -def record_event(table, data): - "Write a standard logmessage log" - _log("LLLOGMESSAGE", table, data) - -def set_destination(dest): - """Set the destination of metrics logs for this process. - - If you do not call this function prior to calling a logging - method, that function will open sys.stdout as a destination. - Attempts to set dest to None will throw a RuntimeError. - @param dest a file-like object which will be the destination for logs.""" - if dest is None: - raise RuntimeError("Attempt to unset metrics destination.") - global _destination - _destination = dest - -def destination(): - """Get the destination of the metrics logs for this process. - Returns None if no destination is set""" - global _destination - return _destination - -class SysLogger(object): - "A file-like object which writes to syslog." - def __init__(self, ident='indra', logopt = None, facility = None): - try: - if logopt is None: - logopt = syslog.LOG_CONS | syslog.LOG_PID - if facility is None: - facility = syslog.LOG_LOCAL0 - syslog.openlog(ident, logopt, facility) - import atexit - atexit.register(syslog.closelog) - except AttributeError: - # No syslog module on Windows - pass - - def write(str): - syslog.syslog(str) - write = staticmethod(write) - - def flush(): - pass - flush = staticmethod(flush) - -# -# internal API -# -_sequence_id = 0 -_destination = None - -def _next_id(): - global _sequence_id - next = _sequence_id - _sequence_id += 1 - return next - -def _dest(): - global _destination - if _destination is None: - # this default behavior is documented in the metrics functions above. - _destination = sys.stdout - return _destination - -def _log(header, table, data): - log_line = "%s (%d) %s %s" \ - % (header, _next_id(), table, format_notation(data)) - dest = _dest() - dest.write(log_line) - dest.flush() diff --git a/indra/lib/python/indra/ipc/httputil.py b/indra/lib/python/indra/ipc/httputil.py deleted file mode 100755 index d53f34a771..0000000000 --- a/indra/lib/python/indra/ipc/httputil.py +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/python -## $LicenseInfo:firstyear=2011&license=viewerlgpl$ -## Second Life Viewer Source Code -## Copyright (C) 2011, Linden Research, Inc. -## -## This library is free software; you can redistribute it and/or -## modify it under the terms of the GNU Lesser General Public -## License as published by the Free Software Foundation; -## version 2.1 of the License only. -## -## This library is distributed in the hope that it will be useful, -## but WITHOUT ANY WARRANTY; without even the implied warranty of -## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -## Lesser General Public License for more details. -## -## You should have received a copy of the GNU Lesser General Public -## License along with this library; if not, write to the Free Software -## Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA -## -## Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA -## $/LicenseInfo$ - -import warnings - -warnings.warn("indra.ipc.httputil has been deprecated; use eventlet.httpc instead", DeprecationWarning, 2) - -from eventlet.httpc import * - - -makeConnection = make_connection diff --git a/indra/lib/python/indra/ipc/llsdhttp.py b/indra/lib/python/indra/ipc/llsdhttp.py deleted file mode 100755 index cbe8ee1eca..0000000000 --- a/indra/lib/python/indra/ipc/llsdhttp.py +++ /dev/null @@ -1,100 +0,0 @@ -"""\ -@file llsdhttp.py -@brief Functions to ease moving llsd over http - -$LicenseInfo:firstyear=2006&license=mit$ - -Copyright (c) 2006-2009, Linden Research, Inc. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -$/LicenseInfo$ -""" - -import os.path -import os -import urlparse - -from indra.base import llsd - -from eventlet import httpc - -suite = httpc.HttpSuite(llsd.format_xml, llsd.parse, 'application/llsd+xml') -delete = suite.delete -delete_ = suite.delete_ -get = suite.get -get_ = suite.get_ -head = suite.head -head_ = suite.head_ -post = suite.post -post_ = suite.post_ -put = suite.put -put_ = suite.put_ -request = suite.request -request_ = suite.request_ - -# import every httpc error exception into our namespace for convenience -for x in httpc.status_to_error_map.itervalues(): - globals()[x.__name__] = x -ConnectionError = httpc.ConnectionError -Retriable = httpc.Retriable - -for x in (httpc.ConnectionError,): - globals()[x.__name__] = x - - -def postFile(url, filename): - f = open(filename) - body = f.read() - f.close() - llsd_body = llsd.parse(body) - return post_(url, llsd_body) - - -# deprecated in favor of get_ -def getStatus(url, use_proxy=False): - status, _headers, _body = get_(url, use_proxy=use_proxy) - return status - -# deprecated in favor of put_ -def putStatus(url, data): - status, _headers, _body = put_(url, data) - return status - -# deprecated in favor of delete_ -def deleteStatus(url): - status, _headers, _body = delete_(url) - return status - -# deprecated in favor of post_ -def postStatus(url, data): - status, _headers, _body = post_(url, data) - return status - - -def postFileStatus(url, filename): - status, _headers, body = postFile(url, filename) - return status, body - - -def getFromSimulator(path, use_proxy=False): - return get('http://' + simulatorHostAndPort + path, use_proxy=use_proxy) - - -def postToSimulator(path, data=None): - return post('http://' + simulatorHostAndPort + path, data) diff --git a/indra/lib/python/indra/ipc/mysql_pool.py b/indra/lib/python/indra/ipc/mysql_pool.py deleted file mode 100755 index e5855a3091..0000000000 --- a/indra/lib/python/indra/ipc/mysql_pool.py +++ /dev/null @@ -1,81 +0,0 @@ -"""\ -@file mysql_pool.py -@brief Thin wrapper around eventlet.db_pool that chooses MySQLdb and Tpool. - -$LicenseInfo:firstyear=2007&license=mit$ - -Copyright (c) 2007-2009, Linden Research, Inc. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -$/LicenseInfo$ -""" - -import MySQLdb -from eventlet import db_pool - -class DatabaseConnector(db_pool.DatabaseConnector): - def __init__(self, credentials, *args, **kwargs): - super(DatabaseConnector, self).__init__(MySQLdb, credentials, - conn_pool=db_pool.ConnectionPool, - *args, **kwargs) - - # get is extended relative to eventlet.db_pool to accept a port argument - def get(self, host, dbname, port=3306): - key = (host, dbname, port) - if key not in self._databases: - new_kwargs = self._kwargs.copy() - new_kwargs['db'] = dbname - new_kwargs['host'] = host - new_kwargs['port'] = port - new_kwargs.update(self.credentials_for(host)) - dbpool = ConnectionPool(*self._args, **new_kwargs) - self._databases[key] = dbpool - - return self._databases[key] - -class ConnectionPool(db_pool.TpooledConnectionPool): - """A pool which gives out saranwrapped MySQLdb connections from a pool - """ - - def __init__(self, *args, **kwargs): - super(ConnectionPool, self).__init__(MySQLdb, *args, **kwargs) - - def get(self): - conn = super(ConnectionPool, self).get() - # annotate the connection object with the details on the - # connection; this is used elsewhere to check that you haven't - # suddenly changed databases in midstream while making a - # series of queries on a connection. - arg_names = ['host','user','passwd','db','port','unix_socket','conv','connect_timeout', - 'compress', 'named_pipe', 'init_command', 'read_default_file', 'read_default_group', - 'cursorclass', 'use_unicode', 'charset', 'sql_mode', 'client_flag', 'ssl', - 'local_infile'] - # you could have constructed this connectionpool with a mix of - # keyword and non-keyword arguments, but we want to annotate - # the connection object with a dict so it's easy to check - # against so here we are converting the list of non-keyword - # arguments (in self._args) into a dict of keyword arguments, - # and merging that with the actual keyword arguments - # (self._kwargs). The arg_names variable lists the - # constructor arguments for MySQLdb Connection objects. - converted_kwargs = dict([ (arg_names[i], arg) for i, arg in enumerate(self._args) ]) - converted_kwargs.update(self._kwargs) - conn.connection_parameters = converted_kwargs - return conn - diff --git a/indra/lib/python/indra/ipc/russ.py b/indra/lib/python/indra/ipc/russ.py deleted file mode 100755 index ac780f128b..0000000000 --- a/indra/lib/python/indra/ipc/russ.py +++ /dev/null @@ -1,165 +0,0 @@ -"""\ -@file russ.py -@brief Recursive URL Substitution Syntax helpers -@author Phoenix - -Many details on how this should work is available on the wiki: -https://wiki.secondlife.com/wiki/Recursive_URL_Substitution_Syntax - -Adding features to this should be reflected in that page in the -implementations section. - -$LicenseInfo:firstyear=2007&license=mit$ - -Copyright (c) 2007-2009, Linden Research, Inc. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -$/LicenseInfo$ -""" - -import urllib -from indra.ipc import llsdhttp - -class UnbalancedBraces(Exception): - pass - -class UnknownDirective(Exception): - pass - -class BadDirective(Exception): - pass - -def format_value_for_path(value): - if type(value) in [list, tuple]: - # *NOTE: treat lists as unquoted path components so that the quoting - # doesn't get out-of-hand. This is a workaround for the fact that - # russ always quotes, even if the data it's given is already quoted, - # and it's not safe to simply unquote a path directly, so if we want - # russ to substitute urls parts inside other url parts we always - # have to do so via lists of unquoted path components. - return '/'.join([urllib.quote(str(item)) for item in value]) - else: - return urllib.quote(str(value)) - -def format(format_str, context): - """@brief Format format string according to rules for RUSS. -@see https://osiris.lindenlab.com/mediawiki/index.php/Recursive_URL_Substitution_Syntax -@param format_str The input string to format. -@param context A map used for string substitutions. -@return Returns the formatted string. If no match, the braces remain intact. -""" - while True: - #print "format_str:", format_str - all_matches = _find_sub_matches(format_str) - if not all_matches: - break - substitutions = 0 - while True: - matches = all_matches.pop() - # we work from right to left to make sure we do not - # invalidate positions earlier in format_str - matches.reverse() - for pos in matches: - # Use index since _find_sub_matches should have raised - # an exception, and failure to find now is an exception. - end = format_str.index('}', pos) - #print "directive:", format_str[pos+1:pos+5] - if format_str[pos + 1] == '$': - value = context[format_str[pos + 2:end]] - if value is not None: - value = format_value_for_path(value) - elif format_str[pos + 1] == '%': - value = _build_query_string( - context.get(format_str[pos + 2:end])) - elif format_str[pos+1:pos+5] == 'http' or format_str[pos+1:pos+5] == 'file': - value = _fetch_url_directive(format_str[pos + 1:end]) - else: - raise UnknownDirective, format_str[pos:end + 1] - if value is not None: - format_str = format_str[:pos]+str(value)+format_str[end+1:] - substitutions += 1 - - # If there were any substitutions at this depth, re-parse - # since this may have revealed new things to substitute - if substitutions: - break - if not all_matches: - break - - # If there were no substitutions at all, and we have exhausted - # the possible matches, bail. - if not substitutions: - break - return format_str - -def _find_sub_matches(format_str): - """@brief Find all of the substitution matches. -@param format_str the RUSS conformant format string. -@return Returns an array of depths of arrays of positional matches in input. -""" - depth = 0 - matches = [] - for pos in range(len(format_str)): - if format_str[pos] == '{': - depth += 1 - if not len(matches) == depth: - matches.append([]) - matches[depth - 1].append(pos) - continue - if format_str[pos] == '}': - depth -= 1 - continue - if not depth == 0: - raise UnbalancedBraces, format_str - return matches - -def _build_query_string(query_dict): - """\ - @breif given a dict, return a query string. utility wrapper for urllib. - @param query_dict input query dict - @returns Returns an urlencoded query string including leading '?'. - """ - if query_dict: - keys = query_dict.keys() - keys.sort() - def stringize(value): - if type(value) in (str,unicode): - return value - else: - return str(value) - query_list = [urllib.quote(str(key)) + '=' + urllib.quote(stringize(query_dict[key])) for key in keys] - return '?' + '&'.join(query_list) - else: - return '' - -def _fetch_url_directive(directive): - "*FIX: This only supports GET" - commands = directive.split('|') - resource = llsdhttp.get(commands[0]) - if len(commands) == 3: - resource = _walk_resource(resource, commands[2]) - return resource - -def _walk_resource(resource, path): - path = path.split('/') - for child in path: - if not child: - continue - resource = resource[child] - return resource diff --git a/indra/lib/python/indra/ipc/servicebuilder.py b/indra/lib/python/indra/ipc/servicebuilder.py deleted file mode 100755 index 0a0ce2b4e2..0000000000 --- a/indra/lib/python/indra/ipc/servicebuilder.py +++ /dev/null @@ -1,134 +0,0 @@ -"""\ -@file servicebuilder.py -@author Phoenix -@brief Class which will generate service urls. - -$LicenseInfo:firstyear=2007&license=mit$ - -Copyright (c) 2007-2009, Linden Research, Inc. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -$/LicenseInfo$ -""" - -from indra.base import config -from indra.ipc import llsdhttp -from indra.ipc import russ - -# *NOTE: agent presence relies on this variable existing and being current, it is a huge hack -services_config = {} -try: - services_config = llsdhttp.get(config.get('services-config')) -except: - pass - -_g_builder = None -def _builder(): - global _g_builder - if _g_builder is None: - _g_builder = ServiceBuilder() - return _g_builder - -def build(name, context={}, **kwargs): - """ Convenience method for using a global, singleton, service builder. Pass arguments either via a dict or via python keyword arguments, or both! - - Example use: - > context = {'channel':'Second Life Release', 'version':'1.18.2.0'} - > servicebuilder.build('version-manager-version', context) - 'http://int.util.vaak.lindenlab.com/channel/Second%20Life%20Release/1.18.2.0' - > servicebuilder.build('version-manager-version', channel='Second Life Release', version='1.18.2.0') - 'http://int.util.vaak.lindenlab.com/channel/Second%20Life%20Release/1.18.2.0' - > servicebuilder.build('version-manager-version', context, version='1.18.1.2') - 'http://int.util.vaak.lindenlab.com/channel/Second%20Life%20Release/1.18.1.2' - """ - global _g_builder - if _g_builder is None: - _g_builder = ServiceBuilder() - return _g_builder.buildServiceURL(name, context, **kwargs) - -def build_path(name, context={}, **kwargs): - context = context.copy() # shouldn't modify the caller's dictionary - context.update(kwargs) - return _builder().buildPath(name, context) - -class ServiceBuilder(object): - def __init__(self, services_definition = services_config): - """\ - @brief - @brief Create a ServiceBuilder. - @param services_definition Complete services definition, services.xml. - """ - # no need to keep a copy of the services section of the - # complete services definition, but it doesn't hurt much. - self.services = services_definition['services'] - self.builders = {} - for service in self.services: - service_builder = service.get('service-builder') - if not service_builder: - continue - if isinstance(service_builder, dict): - # We will be constructing several builders - for name, builder in service_builder.iteritems(): - full_builder_name = service['name'] + '-' + name - self.builders[full_builder_name] = builder - else: - self.builders[service['name']] = service_builder - - def buildPath(self, name, context): - """\ - @brief given the environment on construction, return a service path. - @param name The name of the service. - @param context A dict of name value lookups for the service. - @returns Returns the - """ - return russ.format(self.builders[name], context) - - def buildServiceURL(self, name, context={}, **kwargs): - """\ - @brief given the environment on construction, return a service URL. - @param name The name of the service. - @param context A dict of name value lookups for the service. - @param kwargs Any keyword arguments are treated as members of the - context, this allows you to be all 31337 by writing shit like: - servicebuilder.build('name', param=value) - @returns Returns the - """ - context = context.copy() # shouldn't modify the caller's dictionary - context.update(kwargs) - base_url = config.get('services-base-url') - svc_path = russ.format(self.builders[name], context) - return base_url + svc_path - - -def on_in(query_name, host_key, schema_key): - """\ - @brief Constructs an on/in snippet (for running named queries) - from a schema name and two keys referencing values stored in - indra.xml. - - @param query_name Name of the query. - @param host_key Logical name of destination host. Will be - looked up in indra.xml. - @param schema_key Logical name of destination schema. Will - be looked up in indra.xml. - """ - return "on/config:%s/in/config:%s/%s" % (host_key.strip('/'), - schema_key.strip('/'), - query_name.lstrip('/')) - diff --git a/indra/lib/python/indra/ipc/siesta.py b/indra/lib/python/indra/ipc/siesta.py deleted file mode 100755 index d867e71537..0000000000 --- a/indra/lib/python/indra/ipc/siesta.py +++ /dev/null @@ -1,468 +0,0 @@ -"""\ -@file siesta.py -@brief A tiny llsd based RESTful web services framework - -$LicenseInfo:firstyear=2008&license=mit$ - -Copyright (c) 2008, Linden Research, Inc. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -$/LicenseInfo$ -""" - -from indra.base import config -from indra.base import llsd -from webob import exc -import webob -import re, socket - -try: - from cStringIO import StringIO -except ImportError: - from StringIO import StringIO - -try: - import cjson - json_decode = cjson.decode - json_encode = cjson.encode - JsonDecodeError = cjson.DecodeError - JsonEncodeError = cjson.EncodeError -except ImportError: - import simplejson - json_decode = simplejson.loads - json_encode = simplejson.dumps - JsonDecodeError = ValueError - JsonEncodeError = TypeError - - -llsd_parsers = { - 'application/json': json_decode, - llsd.BINARY_MIME_TYPE: llsd.parse_binary, - 'application/llsd+notation': llsd.parse_notation, - llsd.XML_MIME_TYPE: llsd.parse_xml, - 'application/xml': llsd.parse_xml, - } - - -def mime_type(content_type): - '''Given a Content-Type header, return only the MIME type.''' - - return content_type.split(';', 1)[0].strip().lower() - -class BodyLLSD(object): - '''Give a webob Request or Response an llsd based "content" property. - - Getting the content property parses the body, and caches the result. - - Setting the content property formats a payload, and the body property - is set.''' - - def _llsd__get(self): - '''Get, set, or delete the LLSD value stored in this object.''' - - try: - return self._llsd - except AttributeError: - if not self.body: - raise AttributeError('No llsd attribute has been set') - else: - mtype = mime_type(self.content_type) - try: - parser = llsd_parsers[mtype] - except KeyError: - raise exc.HTTPUnsupportedMediaType( - 'Content type %s not supported' % mtype).exception - try: - self._llsd = parser(self.body) - except (llsd.LLSDParseError, JsonDecodeError, TypeError), err: - raise exc.HTTPBadRequest( - 'Could not parse body: %r' % err.args).exception - return self._llsd - - def _llsd__set(self, val): - req = getattr(self, 'request', None) - if req is not None: - formatter, ctype = formatter_for_request(req) - self.content_type = ctype - else: - formatter, ctype = formatter_for_mime_type( - mime_type(self.content_type)) - self.body = formatter(val) - - def _llsd__del(self): - if hasattr(self, '_llsd'): - del self._llsd - - content = property(_llsd__get, _llsd__set, _llsd__del) - - -class Response(webob.Response, BodyLLSD): - '''Response class with LLSD support. - - A sensible default content type is used. - - Setting the llsd property also sets the body. Getting the llsd - property parses the body if necessary. - - If you set the body property directly, the llsd property will be - deleted.''' - - default_content_type = 'application/llsd+xml' - - def _body__set(self, body): - if hasattr(self, '_llsd'): - del self._llsd - super(Response, self)._body__set(body) - - def cache_forever(self): - self.cache_expires(86400 * 365) - - body = property(webob.Response._body__get, _body__set, - webob.Response._body__del, - webob.Response._body__get.__doc__) - - -class Request(webob.Request, BodyLLSD): - '''Request class with LLSD support. - - Sensible content type and accept headers are used by default. - - Setting the content property also sets the body. Getting the content - property parses the body if necessary. - - If you set the body property directly, the content property will be - deleted.''' - - default_content_type = 'application/llsd+xml' - default_accept = ('application/llsd+xml; q=0.5, ' - 'application/llsd+notation; q=0.3, ' - 'application/llsd+binary; q=0.2, ' - 'application/xml; q=0.1, ' - 'application/json; q=0.0') - - def __init__(self, environ=None, *args, **kwargs): - if environ is None: - environ = {} - else: - environ = environ.copy() - if 'CONTENT_TYPE' not in environ: - environ['CONTENT_TYPE'] = self.default_content_type - if 'HTTP_ACCEPT' not in environ: - environ['HTTP_ACCEPT'] = self.default_accept - super(Request, self).__init__(environ, *args, **kwargs) - - def _body__set(self, body): - if hasattr(self, '_llsd'): - del self._llsd - super(Request, self)._body__set(body) - - def path_urljoin(self, *parts): - return '/'.join([path_url.rstrip('/')] + list(parts)) - - body = property(webob.Request._body__get, _body__set, - webob.Request._body__del, webob.Request._body__get.__doc__) - - def create_response(self, content=None, status='200 OK', - conditional_response=webob.NoDefault): - resp = self.ResponseClass(status=status, request=self, - conditional_response=conditional_response) - resp.content = content - return resp - - def curl(self): - '''Create and fill out a pycurl easy object from this request.''' - - import pycurl - c = pycurl.Curl() - c.setopt(pycurl.URL, self.url()) - if self.headers: - c.setopt(pycurl.HTTPHEADER, - ['%s: %s' % (k, self.headers[k]) for k in self.headers]) - c.setopt(pycurl.FOLLOWLOCATION, True) - c.setopt(pycurl.AUTOREFERER, True) - c.setopt(pycurl.MAXREDIRS, 16) - c.setopt(pycurl.NOSIGNAL, True) - c.setopt(pycurl.READFUNCTION, self.body_file.read) - c.setopt(pycurl.SSL_VERIFYHOST, 2) - - if self.method == 'POST': - c.setopt(pycurl.POST, True) - post301 = getattr(pycurl, 'POST301', None) - if post301 is not None: - # Added in libcurl 7.17.1. - c.setopt(post301, True) - elif self.method == 'PUT': - c.setopt(pycurl.PUT, True) - elif self.method != 'GET': - c.setopt(pycurl.CUSTOMREQUEST, self.method) - return c - -Request.ResponseClass = Response -Response.RequestClass = Request - - -llsd_formatters = { - 'application/json': json_encode, - 'application/llsd+binary': llsd.format_binary, - 'application/llsd+notation': llsd.format_notation, - 'application/llsd+xml': llsd.format_xml, - 'application/xml': llsd.format_xml, - } - -formatter_qualities = ( - ('application/llsd+xml', 1.0), - ('application/llsd+notation', 0.5), - ('application/llsd+binary', 0.4), - ('application/xml', 0.3), - ('application/json', 0.2), - ) - -def formatter_for_mime_type(mime_type): - '''Return a formatter that encodes to the given MIME type. - - The result is a pair of function and MIME type.''' - try: - return llsd_formatters[mime_type], mime_type - except KeyError: - raise exc.HTTPInternalServerError( - 'Could not use MIME type %r to format response' % - mime_type).exception - - -def formatter_for_request(req): - '''Return a formatter that encodes to the preferred type of the client. - - The result is a pair of function and actual MIME type.''' - ctype = req.accept.best_match(formatter_qualities) - try: - return llsd_formatters[ctype], ctype - except KeyError: - raise exc.HTTPNotAcceptable().exception - - -def wsgi_adapter(func, environ, start_response): - '''Adapt a Siesta callable to act as a WSGI application.''' - # Process the request as appropriate. - try: - req = Request(environ) - #print req.urlvars - resp = func(req, **req.urlvars) - if not isinstance(resp, webob.Response): - try: - formatter, ctype = formatter_for_request(req) - resp = req.ResponseClass(formatter(resp), content_type=ctype) - resp._llsd = resp - except (JsonEncodeError, TypeError), err: - resp = exc.HTTPInternalServerError( - detail='Could not format response') - except exc.HTTPException, e: - resp = e - except socket.error, e: - resp = exc.HTTPInternalServerError(detail=e.args[1]) - return resp(environ, start_response) - - -def llsd_callable(func): - '''Turn a callable into a Siesta application.''' - - def replacement(environ, start_response): - return wsgi_adapter(func, environ, start_response) - - return replacement - - -def llsd_method(http_method, func): - def replacement(environ, start_response): - if environ['REQUEST_METHOD'] == http_method: - return wsgi_adapter(func, environ, start_response) - return exc.HTTPMethodNotAllowed()(environ, start_response) - - return replacement - - -http11_methods = 'OPTIONS GET HEAD POST PUT DELETE TRACE CONNECT'.split() -http11_methods.sort() - -def llsd_class(cls): - '''Turn a class into a Siesta application. - - A new instance is created for each request. A HTTP method FOO is - turned into a call to the handle_foo method of the instance.''' - - def foo(req, **kwargs): - instance = cls() - method = req.method.lower() - try: - handler = getattr(instance, 'handle_' + method) - except AttributeError: - allowed = [m for m in http11_methods - if hasattr(instance, 'handle_' + m.lower())] - raise exc.HTTPMethodNotAllowed( - headers={'Allow': ', '.join(allowed)}).exception - #print "kwargs: ", kwargs - return handler(req, **kwargs) - - def replacement(environ, start_response): - return wsgi_adapter(foo, environ, start_response) - - return replacement - - -def curl(reqs): - import pycurl - - m = pycurl.CurlMulti() - curls = [r.curl() for r in reqs] - io = {} - for c in curls: - fp = StringIO() - hdr = StringIO() - c.setopt(pycurl.WRITEFUNCTION, fp.write) - c.setopt(pycurl.HEADERFUNCTION, hdr.write) - io[id(c)] = fp, hdr - m.handles = curls - try: - while True: - ret, num_handles = m.perform() - if ret != pycurl.E_CALL_MULTI_PERFORM: - break - finally: - m.close() - - for req, c in zip(reqs, curls): - fp, hdr = io[id(c)] - hdr.seek(0) - status = hdr.readline().rstrip() - headers = [] - name, values = None, None - - # XXX We don't currently handle bogus header data. - - for line in hdr.readlines(): - if not line[0].isspace(): - if name: - headers.append((name, ' '.join(values))) - name, value = line.strip().split(':', 1) - value = [value] - else: - values.append(line.strip()) - if name: - headers.append((name, ' '.join(values))) - - resp = c.ResponseClass(fp.getvalue(), status, headers, request=req) - - -route_re = re.compile(r''' - \{ # exact character "{" - (\w*) # "config" or variable (restricted to a-z, 0-9, _) - (?:([:~])([^}]+))? # optional :type or ~regex part - \} # exact character "}" - ''', re.VERBOSE) - -predefined_regexps = { - 'uuid': r'[a-f0-9][a-f0-9-]{31,35}', - 'int': r'\d+', - 'host': r'[a-z0-9][a-z0-9\-\.]*', - } - -def compile_route(route): - fp = StringIO() - last_pos = 0 - for match in route_re.finditer(route): - #print "matches: ", match.groups() - fp.write(re.escape(route[last_pos:match.start()])) - var_name = match.group(1) - sep = match.group(2) - expr = match.group(3) - if var_name == 'config': - expr = re.escape(str(config.get(var_name))) - else: - if expr: - if sep == ':': - expr = predefined_regexps[expr] - # otherwise, treat what follows '~' as a regexp - else: - expr = '[^/]+' - if var_name != '': - expr = '(?P<%s>%s)' % (var_name, expr) - else: - expr = '(%s)' % (expr,) - fp.write(expr) - last_pos = match.end() - fp.write(re.escape(route[last_pos:])) - compiled_route = '^%s$' % fp.getvalue() - #print route, "->", compiled_route - return compiled_route - -class Router(object): - '''WSGI routing class. Parses a URL and hands off a request to - some other WSGI application. If no suitable application is found, - responds with a 404.''' - - def __init__(self): - self._new_routes = [] - self._routes = [] - self._paths = [] - - def add(self, route, app, methods=None): - self._new_routes.append((route, app, methods)) - - def _create_routes(self): - for route, app, methods in self._new_routes: - self._paths.append(route) - self._routes.append( - (re.compile(compile_route(route)), - app, - methods and dict.fromkeys(methods))) - self._new_routes = [] - - def __call__(self, environ, start_response): - # load up the config from the config file. Only needs to be - # done once per interpreter. This is the entry point of all - # siesta applications, so this is where we trap it. - _conf = config.get_config() - if _conf is None: - import os.path - fname = os.path.join( - environ.get('ll.config_dir', '/local/linden/etc'), - 'indra.xml') - config.load(fname) - - # proceed with handling the request - self._create_routes() - path_info = environ['PATH_INFO'] - request_method = environ['REQUEST_METHOD'] - allowed = [] - for regex, app, methods in self._routes: - m = regex.match(path_info) - if m: - #print "groupdict:",m.groupdict() - if not methods or request_method in methods: - environ['paste.urlvars'] = m.groupdict() - return app(environ, start_response) - else: - allowed += methods - if allowed: - allowed = dict.fromkeys(allows).keys() - allowed.sort() - resp = exc.HTTPMethodNotAllowed( - headers={'Allow': ', '.join(allowed)}) - else: - resp = exc.HTTPNotFound() - return resp(environ, start_response) diff --git a/indra/lib/python/indra/ipc/siesta_test.py b/indra/lib/python/indra/ipc/siesta_test.py deleted file mode 100755 index a35eed2460..0000000000 --- a/indra/lib/python/indra/ipc/siesta_test.py +++ /dev/null @@ -1,235 +0,0 @@ -#!/usr/bin/python -## $LicenseInfo:firstyear=2011&license=viewerlgpl$ -## Second Life Viewer Source Code -## Copyright (C) 2011, Linden Research, Inc. -## -## This library is free software; you can redistribute it and/or -## modify it under the terms of the GNU Lesser General Public -## License as published by the Free Software Foundation; -## version 2.1 of the License only. -## -## This library is distributed in the hope that it will be useful, -## but WITHOUT ANY WARRANTY; without even the implied warranty of -## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -## Lesser General Public License for more details. -## -## You should have received a copy of the GNU Lesser General Public -## License along with this library; if not, write to the Free Software -## Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA -## -## Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA -## $/LicenseInfo$ -from indra.base import llsd, lluuid -from indra.ipc import siesta -import datetime, math, unittest -from webob import exc - - -class ClassApp(object): - def handle_get(self, req): - pass - - def handle_post(self, req): - return req.llsd - - -def callable_app(req): - if req.method == 'UNDERPANTS': - raise exc.HTTPMethodNotAllowed() - elif req.method == 'GET': - return None - return req.llsd - - -class TestBase: - def test_basic_get(self): - req = siesta.Request.blank('/') - self.assertEquals(req.get_response(self.server).body, - llsd.format_xml(None)) - - def test_bad_method(self): - req = siesta.Request.blank('/') - req.environ['REQUEST_METHOD'] = 'UNDERPANTS' - self.assertEquals(req.get_response(self.server).status_int, - exc.HTTPMethodNotAllowed.code) - - json_safe = { - 'none': None, - 'bool_true': True, - 'bool_false': False, - 'int_zero': 0, - 'int_max': 2147483647, - 'int_min': -2147483648, - 'long_zero': 0, - 'long_max': 2147483647L, - 'long_min': -2147483648L, - 'float_zero': 0, - 'float': math.pi, - 'float_huge': 3.14159265358979323846e299, - 'str_empty': '', - 'str': 'foo', - u'unic\u1e51de_empty': u'', - u'unic\u1e51de': u'\u1e4exx\u10480', - } - json_safe['array'] = json_safe.values() - json_safe['tuple'] = tuple(json_safe.values()) - json_safe['dict'] = json_safe.copy() - - json_unsafe = { - 'uuid_empty': lluuid.UUID(), - 'uuid_full': lluuid.UUID('dc61ab0530200d7554d23510559102c1a98aab1b'), - 'binary_empty': llsd.binary(), - 'binary': llsd.binary('f\0\xff'), - 'uri_empty': llsd.uri(), - 'uri': llsd.uri('http://www.secondlife.com/'), - 'datetime_empty': datetime.datetime(1970,1,1), - 'datetime': datetime.datetime(1999,9,9,9,9,9), - } - json_unsafe.update(json_safe) - json_unsafe['array'] = json_unsafe.values() - json_unsafe['tuple'] = tuple(json_unsafe.values()) - json_unsafe['dict'] = json_unsafe.copy() - json_unsafe['iter'] = iter(json_unsafe.values()) - - def _test_client_content_type_good(self, content_type, ll): - def run(ll): - req = siesta.Request.blank('/') - req.environ['REQUEST_METHOD'] = 'POST' - req.content_type = content_type - req.llsd = ll - req.accept = content_type - resp = req.get_response(self.server) - self.assertEquals(resp.status_int, 200) - return req, resp - - if False and isinstance(ll, dict): - def fixup(v): - if isinstance(v, float): - return '%.5f' % v - if isinstance(v, long): - return int(v) - if isinstance(v, (llsd.binary, llsd.uri)): - return v - if isinstance(v, (tuple, list)): - return [fixup(i) for i in v] - if isinstance(v, dict): - return dict([(k, fixup(i)) for k, i in v.iteritems()]) - return v - for k, v in ll.iteritems(): - l = [k, v] - req, resp = run(l) - self.assertEquals(fixup(resp.llsd), fixup(l)) - - run(ll) - - def test_client_content_type_json_good(self): - self._test_client_content_type_good('application/json', self.json_safe) - - def test_client_content_type_llsd_xml_good(self): - self._test_client_content_type_good('application/llsd+xml', - self.json_unsafe) - - def test_client_content_type_llsd_notation_good(self): - self._test_client_content_type_good('application/llsd+notation', - self.json_unsafe) - - def test_client_content_type_llsd_binary_good(self): - self._test_client_content_type_good('application/llsd+binary', - self.json_unsafe) - - def test_client_content_type_xml_good(self): - self._test_client_content_type_good('application/xml', - self.json_unsafe) - - def _test_client_content_type_bad(self, content_type): - req = siesta.Request.blank('/') - req.environ['REQUEST_METHOD'] = 'POST' - req.body = '\0invalid nonsense under all encodings' - req.content_type = content_type - self.assertEquals(req.get_response(self.server).status_int, - exc.HTTPBadRequest.code) - - def test_client_content_type_json_bad(self): - self._test_client_content_type_bad('application/json') - - def test_client_content_type_llsd_xml_bad(self): - self._test_client_content_type_bad('application/llsd+xml') - - def test_client_content_type_llsd_notation_bad(self): - self._test_client_content_type_bad('application/llsd+notation') - - def test_client_content_type_llsd_binary_bad(self): - self._test_client_content_type_bad('application/llsd+binary') - - def test_client_content_type_xml_bad(self): - self._test_client_content_type_bad('application/xml') - - def test_client_content_type_bad(self): - req = siesta.Request.blank('/') - req.environ['REQUEST_METHOD'] = 'POST' - req.body = 'XXX' - req.content_type = 'application/nonsense' - self.assertEquals(req.get_response(self.server).status_int, - exc.HTTPUnsupportedMediaType.code) - - def test_request_default_content_type(self): - req = siesta.Request.blank('/') - self.assertEquals(req.content_type, req.default_content_type) - - def test_request_default_accept(self): - req = siesta.Request.blank('/') - from webob import acceptparse - self.assertEquals(str(req.accept).replace(' ', ''), - req.default_accept.replace(' ', '')) - - def test_request_llsd_auto_body(self): - req = siesta.Request.blank('/') - req.llsd = {'a': 2} - self.assertEquals(req.body, '<?xml version="1.0" ?><llsd><map>' - '<key>a</key><integer>2</integer></map></llsd>') - - def test_request_llsd_mod_body_changes_llsd(self): - req = siesta.Request.blank('/') - req.llsd = {'a': 2} - req.body = '<?xml version="1.0" ?><llsd><integer>1337</integer></llsd>' - self.assertEquals(req.llsd, 1337) - - def test_request_bad_llsd_fails(self): - def crashme(ctype): - def boom(): - class foo(object): pass - req = siesta.Request.blank('/') - req.content_type = ctype - req.llsd = foo() - for mime_type in siesta.llsd_parsers: - self.assertRaises(TypeError, crashme(mime_type)) - - -class ClassServer(TestBase, unittest.TestCase): - def __init__(self, *args, **kwargs): - unittest.TestCase.__init__(self, *args, **kwargs) - self.server = siesta.llsd_class(ClassApp) - - -class CallableServer(TestBase, unittest.TestCase): - def __init__(self, *args, **kwargs): - unittest.TestCase.__init__(self, *args, **kwargs) - self.server = siesta.llsd_callable(callable_app) - - -class RouterServer(unittest.TestCase): - def test_router(self): - def foo(req, quux): - print quux - - r = siesta.Router() - r.add('/foo/{quux:int}', siesta.llsd_callable(foo), methods=['GET']) - req = siesta.Request.blank('/foo/33') - req.get_response(r) - - req = siesta.Request.blank('/foo/bar') - self.assertEquals(req.get_response(r).status_int, - exc.HTTPNotFound.code) - -if __name__ == '__main__': - unittest.main() diff --git a/indra/lib/python/indra/ipc/webdav.py b/indra/lib/python/indra/ipc/webdav.py deleted file mode 100755 index 98b8499b6a..0000000000 --- a/indra/lib/python/indra/ipc/webdav.py +++ /dev/null @@ -1,597 +0,0 @@ -""" -@file webdav.py -@brief Classes to make manipulation of a webdav store easier. - -$LicenseInfo:firstyear=2007&license=mit$ - -Copyright (c) 2007-2009, Linden Research, Inc. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -$/LicenseInfo$ -""" - -import sys, os, httplib, urlparse -import socket, time -import xml.dom.minidom -import syslog -# import signal - -__revision__ = '0' - -dav_debug = False - - -# def urlsafe_b64decode (enc): -# return base64.decodestring (enc.replace ('_', '/').replace ('-', '+')) - -# def urlsafe_b64encode (str): -# return base64.encodestring (str).replace ('+', '-').replace ('/', '_') - - -class DAVError (Exception): - """ Base class for exceptions in this module. """ - def __init__ (self, status=0, message='', body='', details=''): - self.status = status - self.message = message - self.body = body - self.details = details - Exception.__init__ (self, '%d:%s:%s%s' % (self.status, self.message, - self.body, self.details)) - - def print_to_stderr (self): - """ print_to_stderr docstring """ - print >> sys.stderr, str (self.status) + ' ' + self.message - print >> sys.stderr, str (self.details) - - -class Timeout (Exception): - """ Timeout docstring """ - def __init__ (self, arg=''): - Exception.__init__ (self, arg) - - -def alarm_handler (signum, frame): - """ alarm_handler docstring """ - raise Timeout ('caught alarm') - - -class WebDAV: - """ WebDAV docstring """ - def __init__ (self, url, proxy=None, retries_before_fail=6): - self.init_url = url - self.init_proxy = proxy - self.retries_before_fail = retries_before_fail - url_parsed = urlparse.urlsplit (url) - - self.top_path = url_parsed[ 2 ] - # make sure top_path has a trailing / - if self.top_path == None or self.top_path == '': - self.top_path = '/' - elif len (self.top_path) > 1 and self.top_path[-1:] != '/': - self.top_path += '/' - - if dav_debug: - syslog.syslog ('new WebDAV %s : %s' % (str (url), str (proxy))) - - if proxy: - proxy_parsed = urlparse.urlsplit (proxy) - self.host_header = url_parsed[ 1 ] - host_and_port = proxy_parsed[ 1 ].split (':') - self.host = host_and_port[ 0 ] - if len (host_and_port) > 1: - self.port = int(host_and_port[ 1 ]) - else: - self.port = 80 - else: # no proxy - host_and_port = url_parsed[ 1 ].split (':') - self.host_header = None - self.host = host_and_port[ 0 ] - if len (host_and_port) > 1: - self.port = int(host_and_port[ 1 ]) - else: - self.port = 80 - - self.connection = False - self.connect () - - - def log (self, msg, depth=0): - """ log docstring """ - if dav_debug and depth == 0: - host = str (self.init_url) - if host == 'http://int.tuco.lindenlab.com:80/asset/': - host = 'tuco' - if host == 'http://harriet.lindenlab.com/asset-keep/': - host = 'harriet/asset-keep' - if host == 'http://harriet.lindenlab.com/asset-flag/': - host = 'harriet/asset-flag' - if host == 'http://harriet.lindenlab.com/asset/': - host = 'harriet/asset' - if host == 'http://ozzy.lindenlab.com/asset/': - host = 'ozzy/asset' - if host == 'http://station11.lindenlab.com:12041/:': - host = 'station11:12041' - proxy = str (self.init_proxy) - if proxy == 'None': - proxy = '' - if proxy == 'http://int.tuco.lindenlab.com:3128/': - proxy = 'tuco' - syslog.syslog ('WebDAV (%s:%s) %s' % (host, proxy, str (msg))) - - - def connect (self): - """ connect docstring """ - self.log ('connect') - self.connection = httplib.HTTPConnection (self.host, self.port) - - def __err (self, response, details): - """ __err docstring """ - raise DAVError (response.status, response.reason, response.read (), - str (self.init_url) + ':' + \ - str (self.init_proxy) + ':' + str (details)) - - def request (self, method, path, body=None, headers=None, - read_all=True, body_hook = None, recurse=0, allow_cache=True): - """ request docstring """ - # self.log ('request %s %s' % (method, path)) - if headers == None: - headers = {} - if not allow_cache: - headers['Pragma'] = 'no-cache' - headers['cache-control'] = 'no-cache' - try: - if method.lower () != 'purge': - if path.startswith ('/'): - path = path[1:] - if self.host_header: # use proxy - headers[ 'host' ] = self.host_header - fullpath = 'http://%s%s%s' % (self.host_header, - self.top_path, path) - else: # no proxy - fullpath = self.top_path + path - else: - fullpath = path - - self.connection.request (method, fullpath, body, headers) - if body_hook: - body_hook () - - # signal.signal (signal.SIGALRM, alarm_handler) - # try: - # signal.alarm (120) - # signal.alarm (0) - # except Timeout, e: - # if recurse < 6: - # return self.retry_request (method, path, body, headers, - # read_all, body_hook, recurse) - # else: - # raise DAVError (0, 'timeout', self.host, - # (method, path, body, headers, recurse)) - - response = self.connection.getresponse () - - if read_all: - while len (response.read (1024)) > 0: - pass - if (response.status == 500 or \ - response.status == 503 or \ - response.status == 403) and \ - recurse < self.retries_before_fail: - return self.retry_request (method, path, body, headers, - read_all, body_hook, recurse) - return response - except (httplib.ResponseNotReady, - httplib.BadStatusLine, - socket.error): - # if the server hangs up on us (keepalive off, broken pipe), - # we need to reconnect and try again. - if recurse < self.retries_before_fail: - return self.retry_request (method, path, body, headers, - read_all, body_hook, recurse) - raise DAVError (0, 'reconnect failed', self.host, - (method, path, body, headers, recurse)) - - - def retry_request (self, method, path, body, headers, - read_all, body_hook, recurse): - """ retry_request docstring """ - time.sleep (10.0 * recurse) - self.connect () - return self.request (method, path, body, headers, - read_all, body_hook, recurse+1) - - - - def propfind (self, path, body=None, depth=1): - """ propfind docstring """ - # self.log ('propfind %s' % path) - headers = {'Content-Type':'text/xml; charset="utf-8"', - 'Depth':str(depth)} - response = self.request ('PROPFIND', path, body, headers, False) - if response.status == 207: - return response # Multi-Status - self.__err (response, ('PROPFIND', path, body, headers, 0)) - - - def purge (self, path): - """ issue a squid purge command """ - headers = {'Accept':'*/*'} - response = self.request ('PURGE', path, None, headers) - if response.status == 200 or response.status == 404: - # 200 if it was purge, 404 if it wasn't there. - return response - self.__err (response, ('PURGE', path, None, headers)) - - - def get_file_size (self, path): - """ - Use propfind to ask a webdav server what the size of - a file is. If used on a directory (collection) return 0 - """ - self.log ('get_file_size %s' % path) - # "getcontentlength" property - # 8.1.1 Example - Retrieving Named Properties - # http://docs.python.org/lib/module-xml.dom.html - nsurl = 'http://apache.org/dav/props/' - doc = xml.dom.minidom.Document () - propfind_element = doc.createElementNS (nsurl, "D:propfind") - propfind_element.setAttributeNS (nsurl, 'xmlns:D', 'DAV:') - doc.appendChild (propfind_element) - prop_element = doc.createElementNS (nsurl, "D:prop") - propfind_element.appendChild (prop_element) - con_len_element = doc.createElementNS (nsurl, "D:getcontentlength") - prop_element.appendChild (con_len_element) - - response = self.propfind (path, doc.toxml ()) - doc.unlink () - - resp_doc = xml.dom.minidom.parseString (response.read ()) - cln = resp_doc.getElementsByTagNameNS ('DAV:','getcontentlength')[ 0 ] - try: - content_length = int (cln.childNodes[ 0 ].nodeValue) - except IndexError: - return 0 - resp_doc.unlink () - return content_length - - - def file_exists (self, path): - """ - do an http head on the given file. return True if it succeeds - """ - self.log ('file_exists %s' % path) - expect_gzip = path.endswith ('.gz') - response = self.request ('HEAD', path) - got_gzip = response.getheader ('Content-Encoding', '').strip () - if got_gzip.lower () == 'x-gzip' and expect_gzip == False: - # the asset server fakes us out if we ask for the non-gzipped - # version of an asset, but the server has the gzipped version. - return False - return response.status == 200 - - - def mkdir (self, path): - """ mkdir docstring """ - self.log ('mkdir %s' % path) - headers = {} - response = self.request ('MKCOL', path, None, headers) - if response.status == 201: - return # success - if response.status == 405: - return # directory already existed? - self.__err (response, ('MKCOL', path, None, headers, 0)) - - - def delete (self, path): - """ delete docstring """ - self.log ('delete %s' % path) - headers = {'Depth':'infinity'} # collections require infinity - response = self.request ('DELETE', path, None, headers) - if response.status == 204: - return # no content - if response.status == 404: - return # hmm - self.__err (response, ('DELETE', path, None, headers, 0)) - - - def list_directory (self, path, dir_filter=None, allow_cache=True, - minimum_cache_time=False): - """ - Request an http directory listing and parse the filenames out of lines - like: '<LI><A HREF="X"> X</A>'. If a filter function is provided, - only return filenames that the filter returns True for. - - This is sort of grody, but it seems faster than other ways of getting - this information from an isilon. - """ - self.log ('list_directory %s' % path) - - def try_match (lline, before, after): - """ try_match docstring """ - try: - blen = len (before) - asset_start_index = lline.index (before) - asset_end_index = lline.index (after, asset_start_index + blen) - asset = line[ asset_start_index + blen : asset_end_index ] - - if not dir_filter or dir_filter (asset): - return [ asset ] - return [] - except ValueError: - return [] - - if len (path) > 0 and path[-1:] != '/': - path += '/' - - response = self.request ('GET', path, None, {}, False, - allow_cache=allow_cache) - - if allow_cache and minimum_cache_time: # XXX - print response.getheader ('Date') - # s = "2005-12-06T12:13:14" - # from datetime import datetime - # from time import strptime - # datetime(*strptime(s, "%Y-%m-%dT%H:%M:%S")[0:6]) - # datetime.datetime(2005, 12, 6, 12, 13, 14) - - if response.status != 200: - self.__err (response, ('GET', path, None, {}, 0)) - assets = [] - for line in response.read ().split ('\n'): - lline = line.lower () - if lline.find ("parent directory") == -1: - # isilon file - assets += try_match (lline, '<li><a href="', '"> ') - # apache dir - assets += try_match (lline, 'alt="[dir]"> <a href="', '/">') - # apache file - assets += try_match (lline, 'alt="[ ]"> <a href="', '">') - return assets - - - def __tmp_filename (self, path_and_file): - """ __tmp_filename docstring """ - head, tail = os.path.split (path_and_file) - if head != '': - return head + '/.' + tail + '.' + str (os.getpid ()) - else: - return head + '.' + tail + '.' + str (os.getpid ()) - - - def __put__ (self, filesize, body_hook, remotefile): - """ __put__ docstring """ - headers = {'Content-Length' : str (filesize)} - remotefile_tmp = self.__tmp_filename (remotefile) - response = self.request ('PUT', remotefile_tmp, None, - headers, True, body_hook) - if not response.status in (201, 204): # created, no content - self.__err (response, ('PUT', remotefile, None, headers, 0)) - if filesize != self.get_file_size (remotefile_tmp): - try: - self.delete (remotefile_tmp) - except: - pass - raise DAVError (0, 'tmp upload error', remotefile_tmp) - # move the file to its final location - try: - self.rename (remotefile_tmp, remotefile) - except DAVError, exc: - if exc.status == 403: # try to clean up the tmp file - try: - self.delete (remotefile_tmp) - except: - pass - raise - if filesize != self.get_file_size (remotefile): - raise DAVError (0, 'file upload error', str (remotefile_tmp)) - - - def put_string (self, strng, remotefile): - """ put_string docstring """ - self.log ('put_string %d -> %s' % (len (strng), remotefile)) - filesize = len (strng) - def body_hook (): - """ body_hook docstring """ - self.connection.send (strng) - self.__put__ (filesize, body_hook, remotefile) - - - def put_file (self, localfile, remotefile): - """ - Send a local file to a remote webdav store. First, upload to - a temporary filename. Next make sure the file is the size we - expected. Next, move the file to its final location. Next, - check the file size at the final location. - """ - self.log ('put_file %s -> %s' % (localfile, remotefile)) - filesize = os.path.getsize (localfile) - def body_hook (): - """ body_hook docstring """ - handle = open (localfile) - while True: - data = handle.read (1300) - if len (data) == 0: - break - self.connection.send (data) - handle.close () - self.__put__ (filesize, body_hook, remotefile) - - - def create_empty_file (self, remotefile): - """ create an empty file """ - self.log ('touch_file %s' % (remotefile)) - headers = {'Content-Length' : '0'} - response = self.request ('PUT', remotefile, None, headers) - if not response.status in (201, 204): # created, no content - self.__err (response, ('PUT', remotefile, None, headers, 0)) - if self.get_file_size (remotefile) != 0: - raise DAVError (0, 'file upload error', str (remotefile)) - - - def __get_file_setup (self, remotefile, check_size=True): - """ __get_file_setup docstring """ - if check_size: - remotesize = self.get_file_size (remotefile) - response = self.request ('GET', remotefile, None, {}, False) - if response.status != 200: - self.__err (response, ('GET', remotefile, None, {}, 0)) - try: - content_length = int (response.getheader ("Content-Length")) - except TypeError: - content_length = None - if check_size: - if content_length != remotesize: - raise DAVError (0, 'file DL size error', remotefile) - return (response, content_length) - - - def __get_file_read (self, writehandle, response, content_length): - """ __get_file_read docstring """ - if content_length != None: - so_far_length = 0 - while so_far_length < content_length: - data = response.read (content_length - so_far_length) - if len (data) == 0: - raise DAVError (0, 'short file download') - so_far_length += len (data) - writehandle.write (data) - while len (response.read ()) > 0: - pass - else: - while True: - data = response.read () - if (len (data) < 1): - break - writehandle.write (data) - - - def get_file (self, remotefile, localfile, check_size=True): - """ - Get a remote file from a webdav server. Download to a local - tmp file, then move into place. Sanity check file sizes as - we go. - """ - self.log ('get_file %s -> %s' % (remotefile, localfile)) - (response, content_length) = \ - self.__get_file_setup (remotefile, check_size) - localfile_tmp = self.__tmp_filename (localfile) - handle = open (localfile_tmp, 'w') - self.__get_file_read (handle, response, content_length) - handle.close () - if check_size: - if content_length != os.path.getsize (localfile_tmp): - raise DAVError (0, 'file DL size error', - remotefile+','+localfile) - os.rename (localfile_tmp, localfile) - - - def get_file_as_string (self, remotefile, check_size=True): - """ - download a file from a webdav server and return it as a string. - """ - self.log ('get_file_as_string %s' % remotefile) - (response, content_length) = \ - self.__get_file_setup (remotefile, check_size) - # (tmp_handle, tmp_filename) = tempfile.mkstemp () - tmp_handle = os.tmpfile () - self.__get_file_read (tmp_handle, response, content_length) - tmp_handle.seek (0) - ret = tmp_handle.read () - tmp_handle.close () - # os.unlink (tmp_filename) - return ret - - - def get_post_as_string (self, remotefile, body): - """ - Do an http POST, send body, get response and return it. - """ - self.log ('get_post_as_string %s' % remotefile) - # headers = {'Content-Type':'application/x-www-form-urlencoded'} - headers = {'Content-Type':'text/xml; charset="utf-8"'} - # b64body = urlsafe_b64encode (asset_url) - response = self.request ('POST', remotefile, body, headers, False) - if response.status != 200: - self.__err (response, ('POST', remotefile, body, headers, 0)) - try: - content_length = int (response.getheader ('Content-Length')) - except TypeError: - content_length = None - tmp_handle = os.tmpfile () - self.__get_file_read (tmp_handle, response, content_length) - tmp_handle.seek (0) - ret = tmp_handle.read () - tmp_handle.close () - return ret - - - def __destination_command (self, verb, remotesrc, dstdav, remotedst): - """ - self and dstdav should point to the same http server. - """ - if len (remotedst) > 0 and remotedst[ 0 ] == '/': - remotedst = remotedst[1:] - headers = {'Destination': 'http://%s:%d%s%s' % (dstdav.host, - dstdav.port, - dstdav.top_path, - remotedst)} - response = self.request (verb, remotesrc, None, headers) - if response.status == 201: - return # created - if response.status == 204: - return # no content - self.__err (response, (verb, remotesrc, None, headers, 0)) - - - def rename (self, remotesrc, remotedst): - """ rename a file on a webdav server """ - self.log ('rename %s -> %s' % (remotesrc, remotedst)) - self.__destination_command ('MOVE', remotesrc, self, remotedst) - def xrename (self, remotesrc, dstdav, remotedst): - """ rename a file on a webdav server """ - self.log ('xrename %s -> %s' % (remotesrc, remotedst)) - self.__destination_command ('MOVE', remotesrc, dstdav, remotedst) - - - def copy (self, remotesrc, remotedst): - """ copy a file on a webdav server """ - self.log ('copy %s -> %s' % (remotesrc, remotedst)) - self.__destination_command ('COPY', remotesrc, self, remotedst) - def xcopy (self, remotesrc, dstdav, remotedst): - """ copy a file on a webdav server """ - self.log ('xcopy %s -> %s' % (remotesrc, remotedst)) - self.__destination_command ('COPY', remotesrc, dstdav, remotedst) - - -def put_string (data, url): - """ - upload string s to a url - """ - url_parsed = urlparse.urlsplit (url) - dav = WebDAV ('%s://%s/' % (url_parsed[ 0 ], url_parsed[ 1 ])) - dav.put_string (data, url_parsed[ 2 ]) - - -def get_string (url, check_size=True): - """ - return the contents of a url as a string - """ - url_parsed = urlparse.urlsplit (url) - dav = WebDAV ('%s://%s/' % (url_parsed[ 0 ], url_parsed[ 1 ])) - return dav.get_file_as_string (url_parsed[ 2 ], check_size) diff --git a/indra/lib/python/indra/ipc/xml_rpc.py b/indra/lib/python/indra/ipc/xml_rpc.py deleted file mode 100755 index 47536c10c3..0000000000 --- a/indra/lib/python/indra/ipc/xml_rpc.py +++ /dev/null @@ -1,273 +0,0 @@ -"""\ -@file xml_rpc.py -@brief An implementation of a parser/generator for the XML-RPC xml format. - -$LicenseInfo:firstyear=2006&license=mit$ - -Copyright (c) 2006-2009, Linden Research, Inc. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -$/LicenseInfo$ -""" - - -from greenlet import greenlet - -from mulib import mu - -from xml.sax import handler -from xml.sax import parseString - - -# States -class Expected(object): - def __init__(self, tag): - self.tag = tag - - def __getattr__(self, name): - return type(self)(name) - - def __repr__(self): - return '%s(%r)' % ( - type(self).__name__, self.tag) - - -class START(Expected): - pass - - -class END(Expected): - pass - - -class STR(object): - tag = '' - - -START = START('') -END = END('') - - -class Malformed(Exception): - pass - - -class XMLParser(handler.ContentHandler): - def __init__(self, state_machine, next_states): - handler.ContentHandler.__init__(self) - self.state_machine = state_machine - if not isinstance(next_states, tuple): - next_states = (next_states, ) - self.next_states = next_states - self._character_buffer = '' - - def assertState(self, state, name, *rest): - if not isinstance(self.next_states, tuple): - self.next_states = (self.next_states, ) - for next in self.next_states: - if type(state) == type(next): - if next.tag and next.tag != name: - raise Malformed( - "Expected %s, got %s %s %s" % ( - next, state, name, rest)) - break - else: - raise Malformed( - "Expected %s, got %s %s %s" % ( - self.next_states, state, name, rest)) - - def startElement(self, name, attrs): - self.assertState(START, name.lower(), attrs) - self.next_states = self.state_machine.switch(START, (name.lower(), dict(attrs))) - - def endElement(self, name): - if self._character_buffer.strip(): - characters = self._character_buffer.strip() - self._character_buffer = '' - self.assertState(STR, characters) - self.next_states = self.state_machine.switch(characters) - self.assertState(END, name.lower()) - self.next_states = self.state_machine.switch(END, name.lower()) - - def error(self, exc): - self.bozo = 1 - self.exc = exc - - def fatalError(self, exc): - self.error(exc) - raise exc - - def characters(self, characters): - self._character_buffer += characters - - -def parse(what): - child = greenlet(xml_rpc) - me = greenlet.getcurrent() - startup_states = child.switch(me) - parser = XMLParser(child, startup_states) - try: - parseString(what, parser) - except Malformed: - print what - raise - return child.switch() - - -def xml_rpc(yielder): - yielder.switch(START.methodcall) - yielder.switch(START.methodname) - methodName = yielder.switch(STR) - yielder.switch(END.methodname) - - yielder.switch(START.params) - - root = None - params = [] - while True: - state, _ = yielder.switch(START.param, END.params) - if state == END: - break - - yielder.switch(START.value) - - params.append( - handle(yielder)) - - yielder.switch(END.value) - yielder.switch(END.param) - - yielder.switch(END.methodcall) - ## Resume parse - yielder.switch() - ## Return result to parse - return methodName.strip(), params - - -def handle(yielder): - _, (tag, attrs) = yielder.switch(START) - if tag in ['int', 'i4']: - result = int(yielder.switch(STR)) - elif tag == 'boolean': - result = bool(int(yielder.switch(STR))) - elif tag == 'string': - result = yielder.switch(STR) - elif tag == 'double': - result = float(yielder.switch(STR)) - elif tag == 'datetime.iso8601': - result = yielder.switch(STR) - elif tag == 'base64': - result = base64.b64decode(yielder.switch(STR)) - elif tag == 'struct': - result = {} - while True: - state, _ = yielder.switch(START.member, END.struct) - if state == END: - break - - yielder.switch(START.name) - key = yielder.switch(STR) - yielder.switch(END.name) - - yielder.switch(START.value) - result[key] = handle(yielder) - yielder.switch(END.value) - - yielder.switch(END.member) - ## We already handled </struct> above, don't want to handle it below - return result - elif tag == 'array': - result = [] - yielder.switch(START.data) - while True: - state, _ = yielder.switch(START.value, END.data) - if state == END: - break - - result.append(handle(yielder)) - - yielder.switch(END.value) - - yielder.switch(getattr(END, tag)) - - return result - - -VALUE = mu.tag_factory('value') -BOOLEAN = mu.tag_factory('boolean') -INT = mu.tag_factory('int') -STRUCT = mu.tag_factory('struct') -MEMBER = mu.tag_factory('member') -NAME = mu.tag_factory('name') -ARRAY = mu.tag_factory('array') -DATA = mu.tag_factory('data') -STRING = mu.tag_factory('string') -DOUBLE = mu.tag_factory('double') -METHODRESPONSE = mu.tag_factory('methodResponse') -PARAMS = mu.tag_factory('params') -PARAM = mu.tag_factory('param') - -mu.inline_elements['string'] = True -mu.inline_elements['boolean'] = True -mu.inline_elements['name'] = True - - -def _generate(something): - if isinstance(something, dict): - result = STRUCT() - for key, value in something.items(): - result[ - MEMBER[ - NAME[key], _generate(value)]] - return VALUE[result] - elif isinstance(something, list): - result = DATA() - for item in something: - result[_generate(item)] - return VALUE[ARRAY[[result]]] - elif isinstance(something, basestring): - return VALUE[STRING[something]] - elif isinstance(something, bool): - if something: - return VALUE[BOOLEAN['1']] - return VALUE[BOOLEAN['0']] - elif isinstance(something, int): - return VALUE[INT[something]] - elif isinstance(something, float): - return VALUE[DOUBLE[something]] - -def generate(*args): - params = PARAMS() - for arg in args: - params[PARAM[_generate(arg)]] - return METHODRESPONSE[params] - - -if __name__ == '__main__': - print parse("""<?xml version="1.0"?> <methodCall> <methodName>examples.getStateName</methodName> <params> <param> <value><i4>41</i4></value> </param> </params> </methodCall> -""") - - - - - - - - - diff --git a/indra/lib/python/indra/util/fastest_elementtree.py b/indra/lib/python/indra/util/fastest_elementtree.py deleted file mode 100755 index 4fcf662dd9..0000000000 --- a/indra/lib/python/indra/util/fastest_elementtree.py +++ /dev/null @@ -1,64 +0,0 @@ -"""\ -@file fastest_elementtree.py -@brief Concealing some gnarly import logic in here. This should export the interface of elementtree. - -$LicenseInfo:firstyear=2008&license=mit$ - -Copyright (c) 2008-2009, Linden Research, Inc. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -$/LicenseInfo$ -""" - -# The parsing exception raised by the underlying library depends -# on the ElementTree implementation we're using, so we provide an -# alias here. -# -# Use ElementTreeError as the exception type for catching parsing -# errors. - - -# Using cElementTree might cause some unforeseen problems, so here's a -# convenient off switch. -use_celementree = True - -try: - if not use_celementree: - raise ImportError() - # Python 2.3 and 2.4. - from cElementTree import * - ElementTreeError = SyntaxError -except ImportError: - try: - if not use_celementree: - raise ImportError() - # Python 2.5 and above. - from xml.etree.cElementTree import * - ElementTreeError = SyntaxError - except ImportError: - # Pure Python code. - try: - # Python 2.3 and 2.4. - from elementtree.ElementTree import * - except ImportError: - # Python 2.5 and above. - from xml.etree.ElementTree import * - - # The pure Python ElementTree module uses Expat for parsing. - from xml.parsers.expat import ExpatError as ElementTreeError diff --git a/indra/lib/python/indra/util/helpformatter.py b/indra/lib/python/indra/util/helpformatter.py deleted file mode 100755 index ba5c9b67d1..0000000000 --- a/indra/lib/python/indra/util/helpformatter.py +++ /dev/null @@ -1,52 +0,0 @@ -"""\ -@file helpformatter.py -@author Phoenix -@brief Class for formatting optparse descriptions. - -$LicenseInfo:firstyear=2007&license=mit$ - -Copyright (c) 2007-2009, Linden Research, Inc. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -$/LicenseInfo$ -""" - -import optparse -import textwrap - -class Formatter(optparse.IndentedHelpFormatter): - def __init__( - self, - p_indentIncrement = 2, - p_maxHelpPosition = 24, - p_width = 79, - p_shortFirst = 1) : - optparse.HelpFormatter.__init__( - self, - p_indentIncrement, - p_maxHelpPosition, - p_width, - p_shortFirst) - def format_description(self, p_description): - t_descWidth = self.width - self.current_indent - t_indent = " " * (self.current_indent + 2) - return "\n".join( - [textwrap.fill(descr, t_descWidth, initial_indent = t_indent, - subsequent_indent = t_indent) - for descr in p_description.split("\n")] ) diff --git a/indra/lib/python/indra/util/iterators.py b/indra/lib/python/indra/util/iterators.py deleted file mode 100755 index 9013fa6303..0000000000 --- a/indra/lib/python/indra/util/iterators.py +++ /dev/null @@ -1,63 +0,0 @@ -"""\ -@file iterators.py -@brief Useful general-purpose iterators. - -$LicenseInfo:firstyear=2008&license=mit$ - -Copyright (c) 2008-2009, Linden Research, Inc. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -$/LicenseInfo$ -""" - -from __future__ import nested_scopes - -def iter_chunks(rows, aggregate_size=100): - """ - Given an iterable set of items (@p rows), produces lists of up to @p - aggregate_size items at a time, for example: - - iter_chunks([1,2,3,4,5,6,7,8,9,10], 3) - - Values for @p aggregate_size < 1 will raise ValueError. - - Will return a generator that produces, in the following order: - - [1, 2, 3] - - [4, 5, 6] - - [7, 8, 9] - - [10] - """ - if aggregate_size < 1: - raise ValueError() - - def iter_chunks_inner(): - row_iter = iter(rows) - done = False - agg = [] - while not done: - try: - row = row_iter.next() - agg.append(row) - except StopIteration: - done = True - if agg and (len(agg) >= aggregate_size or done): - yield agg - agg = [] - - return iter_chunks_inner() diff --git a/indra/lib/python/indra/util/iterators_test.py b/indra/lib/python/indra/util/iterators_test.py deleted file mode 100755 index 66928c8e7d..0000000000 --- a/indra/lib/python/indra/util/iterators_test.py +++ /dev/null @@ -1,72 +0,0 @@ -"""\ -@file iterators_test.py -@brief Test cases for iterators module. - -$LicenseInfo:firstyear=2008&license=mit$ - -Copyright (c) 2008-2009, Linden Research, Inc. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -$/LicenseInfo$ -""" - -import unittest - -from indra.util.iterators import iter_chunks - -class TestIterChunks(unittest.TestCase): - """Unittests for iter_chunks""" - def test_bad_agg_size(self): - rows = [1,2,3,4] - self.assertRaises(ValueError, iter_chunks, rows, 0) - self.assertRaises(ValueError, iter_chunks, rows, -1) - - try: - for i in iter_chunks(rows, 0): - pass - except ValueError: - pass - else: - self.fail() - - try: - result = list(iter_chunks(rows, 0)) - except ValueError: - pass - else: - self.fail() - def test_empty(self): - rows = [] - result = list(iter_chunks(rows)) - self.assertEqual(result, []) - def test_small(self): - rows = [[1]] - result = list(iter_chunks(rows, 2)) - self.assertEqual(result, [[[1]]]) - def test_size(self): - rows = [[1],[2]] - result = list(iter_chunks(rows, 2)) - self.assertEqual(result, [[[1],[2]]]) - def test_multi_agg(self): - rows = [[1],[2],[3],[4],[5]] - result = list(iter_chunks(rows, 2)) - self.assertEqual(result, [[[1],[2]],[[3],[4]],[[5]]]) - -if __name__ == "__main__": - unittest.main() diff --git a/indra/lib/python/indra/util/llperformance.py b/indra/lib/python/indra/util/llperformance.py deleted file mode 100755 index 57dd64de3f..0000000000 --- a/indra/lib/python/indra/util/llperformance.py +++ /dev/null @@ -1,182 +0,0 @@ -#!/usr/bin/env python -"""\ -@file llperformance.py - -$LicenseInfo:firstyear=2010&license=viewerlgpl$ -Second Life Viewer Source Code -Copyright (C) 2010-2011, Linden Research, Inc. - -This library is free software; you can redistribute it and/or -modify it under the terms of the GNU Lesser General Public -License as published by the Free Software Foundation; -version 2.1 of the License only. - -This library is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -Lesser General Public License for more details. - -You should have received a copy of the GNU Lesser General Public -License along with this library; if not, write to the Free Software -Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA -$/LicenseInfo$ -""" - -# ------------------------------------------------ -# Sim metrics utility functions. - -import glob, os, time, sys, stat, exceptions - -from indra.base import llsd - -gBlockMap = {} #Map of performance metric data with function hierarchy information. -gCurrentStatPath = "" - -gIsLoggingEnabled=False - -class LLPerfStat: - def __init__(self,key): - self.mTotalTime = 0 - self.mNumRuns = 0 - self.mName=key - self.mTimeStamp = int(time.time()*1000) - self.mUTCTime = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) - - def __str__(self): - return "%f" % self.mTotalTime - - def start(self): - self.mStartTime = int(time.time() * 1000000) - self.mNumRuns += 1 - - def stop(self): - execution_time = int(time.time() * 1000000) - self.mStartTime - self.mTotalTime += execution_time - - def get_map(self): - results={} - results['name']=self.mName - results['utc_time']=self.mUTCTime - results['timestamp']=self.mTimeStamp - results['us']=self.mTotalTime - results['count']=self.mNumRuns - return results - -class PerfError(exceptions.Exception): - def __init__(self): - return - - def __Str__(self): - print "","Unfinished LLPerfBlock" - -class LLPerfBlock: - def __init__( self, key ): - global gBlockMap - global gCurrentStatPath - global gIsLoggingEnabled - - #Check to see if we're running metrics right now. - if gIsLoggingEnabled: - self.mRunning = True #Mark myself as running. - - self.mPreviousStatPath = gCurrentStatPath - gCurrentStatPath += "/" + key - if gCurrentStatPath not in gBlockMap: - gBlockMap[gCurrentStatPath] = LLPerfStat(key) - - self.mStat = gBlockMap[gCurrentStatPath] - self.mStat.start() - - def finish( self ): - global gBlockMap - global gIsLoggingEnabled - - if gIsLoggingEnabled: - self.mStat.stop() - self.mRunning = False - gCurrentStatPath = self.mPreviousStatPath - -# def __del__( self ): -# if self.mRunning: -# #SPATTERS FIXME -# raise PerfError - -class LLPerformance: - #-------------------------------------------------- - # Determine whether or not we want to log statistics - - def __init__( self, process_name = "python" ): - self.process_name = process_name - self.init_testing() - self.mTimeStamp = int(time.time()*1000) - self.mUTCTime = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) - - def init_testing( self ): - global gIsLoggingEnabled - - host_performance_file = "/dev/shm/simperf/simperf_proc_config.llsd" - - #If file exists, open - if os.path.exists(host_performance_file): - file = open (host_performance_file,'r') - - #Read serialized LLSD from file. - body = llsd.parse(file.read()) - - #Calculate time since file last modified. - stats = os.stat(host_performance_file) - now = time.time() - mod = stats[stat.ST_MTIME] - age = now - mod - - if age < ( body['duration'] ): - gIsLoggingEnabled = True - - - def get ( self ): - global gIsLoggingEnabled - return gIsLoggingEnabled - - #def output(self,ptr,path): - # if 'stats' in ptr: - # stats = ptr['stats'] - # self.mOutputPtr[path] = stats.get_map() - - # if 'children' in ptr: - # children=ptr['children'] - - # curptr = self.mOutputPtr - # curchildren={} - # curptr['children'] = curchildren - - # for key in children: - # curchildren[key]={} - # self.mOutputPtr = curchildren[key] - # self.output(children[key],path + '/' + key) - - def done(self): - global gBlockMap - - if not self.get(): - return - - output_name = "/dev/shm/simperf/%s_proc.%d.llsd" % (self.process_name, os.getpid()) - output_file = open(output_name, 'w') - process_info = { - "name" : self.process_name, - "pid" : os.getpid(), - "ppid" : os.getppid(), - "timestamp" : self.mTimeStamp, - "utc_time" : self.mUTCTime, - } - output_file.write(llsd.format_notation(process_info)) - output_file.write('\n') - - for key in gBlockMap.keys(): - gBlockMap[key] = gBlockMap[key].get_map() - output_file.write(llsd.format_notation(gBlockMap)) - output_file.write('\n') - output_file.close() - diff --git a/indra/lib/python/indra/util/llsubprocess.py b/indra/lib/python/indra/util/llsubprocess.py deleted file mode 100755 index 7e0e115d14..0000000000 --- a/indra/lib/python/indra/util/llsubprocess.py +++ /dev/null @@ -1,117 +0,0 @@ -"""\ -@file llsubprocess.py -@author Phoenix -@date 2008-01-18 -@brief The simplest possible wrapper for a common sub-process paradigm. - -$LicenseInfo:firstyear=2007&license=mit$ - -Copyright (c) 2007-2009, Linden Research, Inc. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -$/LicenseInfo$ -""" - -import os -import popen2 -import time -import select - -class Timeout(RuntimeError): - "Exception raised when a subprocess times out." - pass - -def run(command, args=None, data=None, timeout=None): - """\ -@brief Run command with arguments - -This is it. This is the function I want to run all the time when doing -subprocces, but end up copying the code everywhere. none of the -standard commands are secure and provide a way to specify input, get -all the output, and get the result. -@param command A string specifying a process to launch. -@param args Arguments to be passed to command. Must be list, tuple or None. -@param data input to feed to the command. -@param timeout Maximum number of many seconds to run. -@return Returns (result, stdout, stderr) from process. -""" - cmd = [command] - if args: - cmd.extend([str(arg) for arg in args]) - #print "cmd: ","' '".join(cmd) - child = popen2.Popen3(cmd, True) - #print child.pid - out = [] - err = [] - result = -1 - time_left = timeout - tochild = [child.tochild.fileno()] - while True: - time_start = time.time() - #print "time:",time_left - p_in, p_out, p_err = select.select( - [child.fromchild.fileno(), child.childerr.fileno()], - tochild, - [], - time_left) - if p_in: - new_line = os.read(child.fromchild.fileno(), 32 * 1024) - if new_line: - #print "line:",new_line - out.append(new_line) - new_line = os.read(child.childerr.fileno(), 32 * 1024) - if new_line: - #print "error:", new_line - err.append(new_line) - if p_out: - if data: - #print "p_out" - bytes = os.write(child.tochild.fileno(), data) - data = data[bytes:] - if len(data) == 0: - data = None - tochild = [] - child.tochild.close() - result = child.poll() - if result != -1: - # At this point, the child process has exited and result - # is the return value from the process. Between the time - # we called select() and poll() the process may have - # exited so read all the data left on the child process - # stdout and stderr. - last = child.fromchild.read() - if last: - out.append(last) - last = child.childerr.read() - if last: - err.append(last) - child.tochild.close() - child.fromchild.close() - child.childerr.close() - break - if time_left is not None: - time_left -= (time.time() - time_start) - if time_left < 0: - raise Timeout - #print "result:",result - out = ''.join(out) - #print "stdout:", out - err = ''.join(err) - #print "stderr:", err - return result, out, err diff --git a/indra/lib/python/indra/util/named_query.py b/indra/lib/python/indra/util/named_query.py deleted file mode 100755 index 6bf956107d..0000000000 --- a/indra/lib/python/indra/util/named_query.py +++ /dev/null @@ -1,592 +0,0 @@ -"""\ -@file named_query.py -@author Ryan Williams, Phoenix -@date 2007-07-31 -@brief An API for running named queries. - -$LicenseInfo:firstyear=2007&license=mit$ - -Copyright (c) 2007-2009, Linden Research, Inc. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -$/LicenseInfo$ -""" - -import errno -import MySQLdb -import MySQLdb.cursors -import os -import os.path -import re -import time - -from indra.base import llsd -from indra.base import config - -DEBUG = False -NQ_FILE_SUFFIX = config.get('named-query-file-suffix', '.nq') -NQ_FILE_SUFFIX_LEN = len(NQ_FILE_SUFFIX) - -_g_named_manager = None - -def _init_g_named_manager(sql_dir = None): - """Initializes a global NamedManager object to point at a - specified named queries hierarchy. - - This function is intended entirely for testing purposes, - because it's tricky to control the config from inside a test.""" - global NQ_FILE_SUFFIX - NQ_FILE_SUFFIX = config.get('named-query-file-suffix', '.nq') - global NQ_FILE_SUFFIX_LEN - NQ_FILE_SUFFIX_LEN = len(NQ_FILE_SUFFIX) - - if sql_dir is None: - sql_dir = config.get('named-query-base-dir') - - # extra fallback directory in case config doesn't return what we want - if sql_dir is None: - sql_dir = os.path.abspath( - os.path.join( - os.path.realpath(os.path.dirname(__file__)), "..", "..", "..", "..", "web", "dataservice", "sql")) - - global _g_named_manager - _g_named_manager = NamedQueryManager( - os.path.abspath(os.path.realpath(sql_dir))) - -def get(name, schema = None): - "Get the named query object to be used to perform queries" - if _g_named_manager is None: - _init_g_named_manager() - return _g_named_manager.get(name).for_schema(schema) - -def sql(connection, name, params): - # use module-global NamedQuery object to perform default substitution - return get(name).sql(connection, params) - -def run(connection, name, params, expect_rows = None): - """\ -@brief given a connection, run a named query with the params - -Note that this function will fetch ALL rows. -@param connection The connection to use -@param name The name of the query to run -@param params The parameters passed into the query -@param expect_rows The number of rows expected. Set to 1 if return_as_map is true. Raises ExpectationFailed if the number of returned rows doesn't exactly match. Kind of a hack. -@return Returns the result set as a list of dicts. -""" - return get(name).run(connection, params, expect_rows) - -class ExpectationFailed(Exception): - """ Exception that is raised when an expectation for an sql query - is not met.""" - def __init__(self, message): - Exception.__init__(self, message) - self.message = message - -class NamedQuery(object): - def __init__(self, name, filename): - """ Construct a NamedQuery object. The name argument is an - arbitrary name as a handle for the query, and the filename is - a path to a file or a file-like object containing an llsd named - query document.""" - self._stat_interval_seconds = 5 # 5 seconds - self._name = name - if (filename is not None and isinstance(filename, (str, unicode)) - and NQ_FILE_SUFFIX != filename[-NQ_FILE_SUFFIX_LEN:]): - filename = filename + NQ_FILE_SUFFIX - self._location = filename - self._alternative = dict() - self._last_mod_time = 0 - self._last_check_time = 0 - self.deleted = False - self.load_contents() - - def name(self): - """ The name of the query. """ - return self._name - - def get_modtime(self): - """ Returns the mtime (last modified time) of the named query - filename. For file-like objects, expect a modtime of 0""" - if self._location and isinstance(self._location, (str, unicode)): - return os.path.getmtime(self._location) - return 0 - - def load_contents(self): - """ Loads and parses the named query file into self. Does - nothing if self.location is nonexistant.""" - if self._location: - if isinstance(self._location, (str, unicode)): - contents = llsd.parse(open(self._location).read()) - else: - # we probably have a file-like object. Godspeed! - contents = llsd.parse(self._location.read()) - self._reference_contents(contents) - # Check for alternative implementations - try: - for name, alt in self._contents['alternative'].items(): - nq = NamedQuery(name, None) - nq._reference_contents(alt) - self._alternative[name] = nq - except KeyError, e: - pass - self._last_mod_time = self.get_modtime() - self._last_check_time = time.time() - - def _reference_contents(self, contents): - "Helper method which builds internal structure from parsed contents" - self._contents = contents - self._ttl = int(self._contents.get('ttl', 0)) - self._return_as_map = bool(self._contents.get('return_as_map', False)) - self._legacy_dbname = self._contents.get('legacy_dbname', None) - - # reset these before doing the sql conversion because we will - # read them there. reset these while loading so we pick up - # changes. - self._around = set() - self._append = set() - self._integer = set() - self._options = self._contents.get('dynamic_where', {}) - for key in self._options: - if isinstance(self._options[key], basestring): - self._options[key] = self._convert_sql(self._options[key]) - elif isinstance(self._options[key], list): - lines = [] - for line in self._options[key]: - lines.append(self._convert_sql(line)) - self._options[key] = lines - else: - moreopt = {} - for kk in self._options[key]: - moreopt[kk] = self._convert_sql(self._options[key][kk]) - self._options[key] = moreopt - self._base_query = self._convert_sql(self._contents['base_query']) - self._query_suffix = self._convert_sql( - self._contents.get('query_suffix', '')) - - def _convert_sql(self, sql): - """convert the parsed sql into a useful internal structure. - - This function has to turn the named query format into a pyformat - style. It also has to look for %:name% and :name% and - ready them for use in LIKE statements""" - if sql: - # This first sub is to properly escape any % signs that - # are meant to be literally passed through to mysql in the - # query. It leaves any %'s that are used for - # like-expressions. - expr = re.compile("(?<=[^a-zA-Z0-9_-])%(?=[^:])") - sql = expr.sub('%%', sql) - - # This should tackle the rest of the %'s in the query, by - # converting them to LIKE clauses. - expr = re.compile("(%?):([a-zA-Z][a-zA-Z0-9_-]*)%") - sql = expr.sub(self._prepare_like, sql) - expr = re.compile("#:([a-zA-Z][a-zA-Z0-9_-]*)") - sql = expr.sub(self._prepare_integer, sql) - expr = re.compile(":([a-zA-Z][a-zA-Z0-9_-]*)") - sql = expr.sub("%(\\1)s", sql) - return sql - - def _prepare_like(self, match): - """This function changes LIKE statement replace behavior - - It works by turning %:name% to %(_name_around)s and :name% to - %(_name_append)s. Since a leading '_' is not a valid keyname - input (enforced via unit tests), it will never clash with - existing keys. Then, when building the statement, the query - runner will generate corrected strings.""" - if match.group(1) == '%': - # there is a leading % so this is treated as prefix/suffix - self._around.add(match.group(2)) - return "%(" + self._build_around_key(match.group(2)) + ")s" - else: - # there is no leading %, so this is suffix only - self._append.add(match.group(2)) - return "%(" + self._build_append_key(match.group(2)) + ")s" - - def _build_around_key(self, key): - return "_" + key + "_around" - - def _build_append_key(self, key): - return "_" + key + "_append" - - def _prepare_integer(self, match): - """This function adjusts the sql for #:name replacements - - It works by turning #:name to %(_name_as_integer)s. Since a - leading '_' is not a valid keyname input (enforced via unit - tests), it will never clash with existing keys. Then, when - building the statement, the query runner will generate - corrected strings.""" - self._integer.add(match.group(1)) - return "%(" + self._build_integer_key(match.group(1)) + ")s" - - def _build_integer_key(self, key): - return "_" + key + "_as_integer" - - def _strip_wildcards_to_list(self, value): - """Take string, and strip out the LIKE special characters. - - Technically, this is database dependant, but postgresql and - mysql use the same wildcards, and I am not aware of a general - way to handle this. I think you need a sql statement of the - form: - - LIKE_STRING( [ANY,ONE,str]... ) - - which would treat ANY as their any string, and ONE as their - single glyph, and str as something that needs database - specific encoding to not allow any % or _ to affect the query. - - As it stands, I believe it's impossible to write a named query - style interface which uses like to search the entire space of - text available. Imagine the query: - - % of brain used by average linden - - In order to search for %, it must be escaped, so once you have - escaped the string to not do wildcard searches, and be escaped - for the database, and then prepended the wildcard you come - back with one of: - - 1) %\% of brain used by average linden - 2) %%% of brain used by average linden - - Then, when passed to the database to be escaped to be database - safe, you get back: - - 1) %\\% of brain used by average linden - : which means search for any character sequence, followed by a - backslash, followed by any sequence, followed by ' of - brain...' - 2) %%% of brain used by average linden - : which (I believe) means search for a % followed by any - character sequence followed by 'of brain...' - - Neither of which is what we want! - - So, we need a vendor (or extention) for LIKE_STRING. Anyone - want to write it?""" - if isinstance(value, unicode): - utf8_value = value - else: - utf8_value = unicode(value, "utf-8") - esc_list = [] - remove_chars = set(u"%_") - for glyph in utf8_value: - if glyph in remove_chars: - continue - esc_list.append(glyph.encode("utf-8")) - return esc_list - - def delete(self): - """ Makes this query unusable by deleting all the members and - setting the deleted member. This is desired when the on-disk - query has been deleted but the in-memory copy remains.""" - # blow away all members except _name, _location, and deleted - name, location = self._name, self._location - for key in self.__dict__.keys(): - del self.__dict__[key] - self.deleted = True - self._name, self._location = name, location - - def ttl(self): - """ Estimated time to live of this query. Used for web - services to set the Expires header.""" - return self._ttl - - def legacy_dbname(self): - return self._legacy_dbname - - def return_as_map(self): - """ Returns true if this query is configured to return its - results as a single map (as opposed to a list of maps, the - normal behavior).""" - - return self._return_as_map - - def for_schema(self, db_name): - "Look trough the alternates and return the correct query" - if db_name is None: - return self - try: - return self._alternative[db_name] - except KeyError, e: - pass - return self - - def run(self, connection, params, expect_rows = None, use_dictcursor = True): - """given a connection, run a named query with the params - - Note that this function will fetch ALL rows. We do this because it - opens and closes the cursor to generate the values, and this - isn't a generator so the cursor has no life beyond the method call. - - @param cursor The connection to use (this generates its own cursor for the query) - @param name The name of the query to run - @param params The parameters passed into the query - @param expect_rows The number of rows expected. Set to 1 if return_as_map is true. Raises ExpectationFailed if the number of returned rows doesn't exactly match. Kind of a hack. - @param use_dictcursor Set to false to use a normal cursor and manually convert the rows to dicts. - @return Returns the result set as a list of dicts, or, if the named query has return_as_map set to true, returns a single dict. - """ - if use_dictcursor: - cursor = connection.cursor(MySQLdb.cursors.DictCursor) - else: - cursor = connection.cursor() - - full_query, params = self._construct_sql(params) - if DEBUG: - print "SQL:", self.sql(connection, params) - rows = cursor.execute(full_query, params) - - # *NOTE: the expect_rows argument is a very cheesy way to get some - # validation on the result set. If you want to add more expectation - # logic, do something more object-oriented and flexible. Or use an ORM. - if(self._return_as_map): - expect_rows = 1 - if expect_rows is not None and rows != expect_rows: - cursor.close() - raise ExpectationFailed("Statement expected %s rows, got %s. Sql: '%s' %s" % ( - expect_rows, rows, full_query, params)) - - # convert to dicts manually if we're not using a dictcursor - if use_dictcursor: - result_set = cursor.fetchall() - else: - if cursor.description is None: - # an insert or something - x = cursor.fetchall() - cursor.close() - return x - - names = [x[0] for x in cursor.description] - - result_set = [] - for row in cursor.fetchall(): - converted_row = {} - for idx, col_name in enumerate(names): - converted_row[col_name] = row[idx] - result_set.append(converted_row) - - cursor.close() - if self._return_as_map: - return result_set[0] - return result_set - - def _construct_sql(self, params): - """ Returns a query string and a dictionary of parameters, - suitable for directly passing to the execute() method.""" - self.refresh() - - # build the query from the options available and the params - base_query = [] - base_query.append(self._base_query) - for opt, extra_where in self._options.items(): - if type(extra_where) in (dict, list, tuple): - if opt in params: - base_query.append(extra_where[params[opt]]) - else: - if opt in params and params[opt]: - base_query.append(extra_where) - if self._query_suffix: - base_query.append(self._query_suffix) - full_query = '\n'.join(base_query) - - # Go through the query and rewrite all of the ones with the - # @:name syntax. - rewrite = _RewriteQueryForArray(params) - expr = re.compile("@%\(([a-zA-Z][a-zA-Z0-9_-]*)\)s") - full_query = expr.sub(rewrite.operate, full_query) - params.update(rewrite.new_params) - - # build out the params for like. We only have to do this - # parameters which were detected to have ued the where syntax - # during load. - # - # * treat the incoming string as utf-8 - # * strip wildcards - # * append or prepend % as appropriate - new_params = {} - for key in params: - if key in self._around: - new_value = ['%'] - new_value.extend(self._strip_wildcards_to_list(params[key])) - new_value.append('%') - new_params[self._build_around_key(key)] = ''.join(new_value) - if key in self._append: - new_value = self._strip_wildcards_to_list(params[key]) - new_value.append('%') - new_params[self._build_append_key(key)] = ''.join(new_value) - if key in self._integer: - new_params[self._build_integer_key(key)] = int(params[key]) - params.update(new_params) - - return full_query, params - - def sql(self, connection, params): - """ Generates an SQL statement from the named query document - and a dictionary of parameters. - - *NOTE: Only use for debugging, because it uses the - non-standard MySQLdb 'literal' method. - """ - if not DEBUG: - import warnings - warnings.warn("Don't use named_query.sql() when not debugging. Used on %s" % self._location) - # do substitution using the mysql (non-standard) 'literal' - # function to do the escaping. - full_query, params = self._construct_sql(params) - return full_query % connection.literal(params) - - - def refresh(self): - """ Refresh self from the file on the filesystem. - - This is optimized to be callable as frequently as you wish, - without adding too much load. It does so by only stat-ing the - file every N seconds, where N defaults to 5 and is - configurable through the member _stat_interval_seconds. If the stat - reveals that the file has changed, refresh will re-parse the - contents of the file and use them to update the named query - instance. If the stat reveals that the file has been deleted, - refresh will call self.delete to make the in-memory - representation unusable.""" - now = time.time() - if(now - self._last_check_time > self._stat_interval_seconds): - self._last_check_time = now - try: - modtime = self.get_modtime() - if(modtime > self._last_mod_time): - self.load_contents() - except OSError, e: - if e.errno == errno.ENOENT: # file not found - self.delete() # clean up self - raise # pass the exception along to the caller so they know that this query disappeared - -class NamedQueryManager(object): - """ Manages the lifespan of NamedQuery objects, drawing from a - directory hierarchy of named query documents. - - In practice this amounts to a memory cache of NamedQuery objects.""" - - def __init__(self, named_queries_dir): - """ Initializes a manager to look for named queries in a - directory.""" - self._dir = os.path.abspath(os.path.realpath(named_queries_dir)) - self._cached_queries = {} - - def sql(self, connection, name, params): - nq = self.get(name) - return nq.sql(connection, params) - - def get(self, name): - """ Returns a NamedQuery instance based on the name, either - from memory cache, or by parsing from disk. - - The name is simply a relative path to the directory associated - with the manager object. Before returning the instance, the - NamedQuery object is cached in memory, so that subsequent - accesses don't have to read from disk or do any parsing. This - means that NamedQuery objects returned by this method are - shared across all users of the manager object. - NamedQuery.refresh is used to bring the NamedQuery objects in - sync with the actual files on disk.""" - nq = self._cached_queries.get(name) - if nq is None: - nq = NamedQuery(name, os.path.join(self._dir, name)) - self._cached_queries[name] = nq - else: - try: - nq.refresh() - except OSError, e: - if e.errno == errno.ENOENT: # file not found - del self._cached_queries[name] - raise # pass exception along to caller so they know that the query disappeared - - return nq - -class _RewriteQueryForArray(object): - "Helper class for rewriting queries with the @:name syntax" - def __init__(self, params): - self.params = params - self.new_params = dict() - - def operate(self, match): - "Given a match, return the string that should be in use" - key = match.group(1) - value = self.params[key] - if type(value) in (list,tuple): - rv = [] - for idx in range(len(value)): - # if the value@idx is array-like, we are - # probably dealing with a VALUES - new_key = "_%s_%s"%(key, str(idx)) - val_item = value[idx] - if type(val_item) in (list, tuple, dict): - if type(val_item) is dict: - # this is because in Python, the order of - # key, value retrieval from the dict is not - # guaranteed to match what the input intended - # and for VALUES, order is important. - # TODO: Implemented ordered dict in LLSD parser? - raise ExpectationFailed('Only lists/tuples allowed,\ - received dict') - values_keys = [] - for value_idx, item in enumerate(val_item): - # we want a key of the format : - # key_#replacement_#value_row_#value_col - # ugh... so if we are replacing 10 rows in user_note, - # the first values clause would read (for @:user_notes) :- - # ( :_user_notes_0_1_1, :_user_notes_0_1_2, :_user_notes_0_1_3 ) - # the input LLSD for VALUES will look like: - # <llsd>... - # <map> - # <key>user_notes</key> - # <array> - # <array> <!-- row 1 for VALUES --> - # <string>...</string> - # <string>...</string> - # <string>...</string> - # </array> - # ... - # </array> - # </map> - # ... </llsd> - values_key = "%s_%s"%(new_key, value_idx) - self.new_params[values_key] = item - values_keys.append("%%(%s)s"%values_key) - # now collapse all these new place holders enclosed in () - # from [':_key_0_1_1', ':_key_0_1_2', ':_key_0_1_3,...] - # rv will have [ '(:_key_0_1_1, :_key_0_1_2, :_key_0_1_3)', ] - # which is flattened a few lines below join(rv) - rv.append('(%s)' % ','.join(values_keys)) - else: - self.new_params[new_key] = val_item - rv.append("%%(%s)s"%new_key) - return ','.join(rv) - else: - # not something that can be expanded, so just drop the - # leading @ in the front of the match. This will mean that - # the single value we have, be it a string, int, whatever - # (other than dict) will correctly show up, eg: - # - # where foo in (@:foobar) -- foobar is a string, so we get - # where foo in (:foobar) - return match.group(0)[1:] diff --git a/indra/lib/python/indra/util/shutil2.py b/indra/lib/python/indra/util/shutil2.py deleted file mode 100755 index 9e2e7a6ded..0000000000 --- a/indra/lib/python/indra/util/shutil2.py +++ /dev/null @@ -1,84 +0,0 @@ -''' -@file shutil2.py -@brief a better shutil.copytree replacement - -$LicenseInfo:firstyear=2007&license=mit$ - -Copyright (c) 2007-2009, Linden Research, Inc. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -$/LicenseInfo$ -''' - -# -# shutil2.py -# Taken from http://www.scons.org/wiki/AccumulateBuilder -# the stock copytree sucks because it insists that the -# target dir not exist -# - -import os.path -import shutil - -def copytree(src, dest, symlinks=False): - """My own copyTree which does not fail if the directory exists. - - Recursively copy a directory tree using copy2(). - - If the optional symlinks flag is true, symbolic links in the - source tree result in symbolic links in the destination tree; if - it is false, the contents of the files pointed to by symbolic - links are copied. - - Behavior is meant to be identical to GNU 'cp -R'. - """ - def copyItems(src, dest, symlinks=False): - """Function that does all the work. - - It is necessary to handle the two 'cp' cases: - - destination does exist - - destination does not exist - - See 'cp -R' documentation for more details - """ - for item in os.listdir(src): - srcPath = os.path.join(src, item) - if os.path.isdir(srcPath): - srcBasename = os.path.basename(srcPath) - destDirPath = os.path.join(dest, srcBasename) - if not os.path.exists(destDirPath): - os.makedirs(destDirPath) - copyItems(srcPath, destDirPath) - elif os.path.islink(item) and symlinks: - linkto = os.readlink(item) - os.symlink(linkto, dest) - else: - shutil.copy2(srcPath, dest) - - # case 'cp -R src/ dest/' where dest/ already exists - if os.path.exists(dest): - destPath = os.path.join(dest, os.path.basename(src)) - if not os.path.exists(destPath): - os.makedirs(destPath) - # case 'cp -R src/ dest/' where dest/ does not exist - else: - os.makedirs(dest) - destPath = dest - # actually copy the files - copyItems(src, destPath) diff --git a/indra/lib/python/indra/util/simperf_host_xml_parser.py b/indra/lib/python/indra/util/simperf_host_xml_parser.py deleted file mode 100755 index 672c1050c2..0000000000 --- a/indra/lib/python/indra/util/simperf_host_xml_parser.py +++ /dev/null @@ -1,338 +0,0 @@ -#!/usr/bin/env python -"""\ -@file simperf_host_xml_parser.py -@brief Digest collector's XML dump and convert to simple dict/list structure - -$LicenseInfo:firstyear=2008&license=mit$ - -Copyright (c) 2008-2009, Linden Research, Inc. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -$/LicenseInfo$ -""" - -import sys, os, getopt, time -import simplejson -from xml import sax - - -def usage(): - print "Usage:" - print sys.argv[0] + " [options]" - print " Convert RRD's XML dump to JSON. Script to convert the simperf_host_collector-" - print " generated RRD dump into JSON. Steps include converting selected named" - print " fields from GAUGE type to COUNTER type by computing delta with preceding" - print " values. Top-level named fields are:" - print - print " lastupdate Time (javascript timestamp) of last data sample" - print " step Time in seconds between samples" - print " ds Data specification (name/type) for each column" - print " database Table of data samples, one time step per row" - print - print "Options:" - print " -i, --in Input settings filename. (Default: stdin)" - print " -o, --out Output settings filename. (Default: stdout)" - print " -h, --help Print this message and exit." - print - print "Example: %s -i rrddump.xml -o rrddump.json" % sys.argv[0] - print - print "Interfaces:" - print " class SimPerfHostXMLParser() # SAX content handler" - print " def simperf_host_xml_fixup(parser) # post-parse value fixup" - -class SimPerfHostXMLParser(sax.handler.ContentHandler): - - def __init__(self): - pass - - def startDocument(self): - self.rrd_last_update = 0 # public - self.rrd_step = 0 # public - self.rrd_ds = [] # public - self.rrd_records = [] # public - self._rrd_level = 0 - self._rrd_parse_state = 0 - self._rrd_chars = "" - self._rrd_capture = False - self._rrd_ds_val = {} - self._rrd_data_row = [] - self._rrd_data_row_has_nan = False - - def endDocument(self): - pass - - # Nasty little ad-hoc state machine to extract the elements that are - # necessary from the 'rrdtool dump' XML output. The same element - # name '<ds>' is used for two different data sets so we need to pay - # some attention to the actual structure to get the ones we want - # and ignore the ones we don't. - - def startElement(self, name, attrs): - self._rrd_level = self._rrd_level + 1 - self._rrd_capture = False - if self._rrd_level == 1: - if name == "rrd" and self._rrd_parse_state == 0: - self._rrd_parse_state = 1 # In <rrd> - self._rrd_capture = True - self._rrd_chars = "" - elif self._rrd_level == 2: - if self._rrd_parse_state == 1: - if name == "lastupdate": - self._rrd_parse_state = 2 # In <rrd><lastupdate> - self._rrd_capture = True - self._rrd_chars = "" - elif name == "step": - self._rrd_parse_state = 3 # In <rrd><step> - self._rrd_capture = True - self._rrd_chars = "" - elif name == "ds": - self._rrd_parse_state = 4 # In <rrd><ds> - self._rrd_ds_val = {} - self._rrd_chars = "" - elif name == "rra": - self._rrd_parse_state = 5 # In <rrd><rra> - elif self._rrd_level == 3: - if self._rrd_parse_state == 4: - if name == "name": - self._rrd_parse_state = 6 # In <rrd><ds><name> - self._rrd_capture = True - self._rrd_chars = "" - elif name == "type": - self._rrd_parse_state = 7 # In <rrd><ds><type> - self._rrd_capture = True - self._rrd_chars = "" - elif self._rrd_parse_state == 5: - if name == "database": - self._rrd_parse_state = 8 # In <rrd><rra><database> - elif self._rrd_level == 4: - if self._rrd_parse_state == 8: - if name == "row": - self._rrd_parse_state = 9 # In <rrd><rra><database><row> - self._rrd_data_row = [] - self._rrd_data_row_has_nan = False - elif self._rrd_level == 5: - if self._rrd_parse_state == 9: - if name == "v": - self._rrd_parse_state = 10 # In <rrd><rra><database><row><v> - self._rrd_capture = True - self._rrd_chars = "" - - def endElement(self, name): - self._rrd_capture = False - if self._rrd_parse_state == 10: - self._rrd_capture = self._rrd_level == 6 - if self._rrd_level == 5: - if self._rrd_chars == "NaN": - self._rrd_data_row_has_nan = True - else: - self._rrd_data_row.append(self._rrd_chars) - self._rrd_parse_state = 9 # In <rrd><rra><database><row> - elif self._rrd_parse_state == 9: - if self._rrd_level == 4: - if not self._rrd_data_row_has_nan: - self.rrd_records.append(self._rrd_data_row) - self._rrd_parse_state = 8 # In <rrd><rra><database> - elif self._rrd_parse_state == 8: - if self._rrd_level == 3: - self._rrd_parse_state = 5 # In <rrd><rra> - elif self._rrd_parse_state == 7: - if self._rrd_level == 3: - self._rrd_ds_val["type"] = self._rrd_chars - self._rrd_parse_state = 4 # In <rrd><ds> - elif self._rrd_parse_state == 6: - if self._rrd_level == 3: - self._rrd_ds_val["name"] = self._rrd_chars - self._rrd_parse_state = 4 # In <rrd><ds> - elif self._rrd_parse_state == 5: - if self._rrd_level == 2: - self._rrd_parse_state = 1 # In <rrd> - elif self._rrd_parse_state == 4: - if self._rrd_level == 2: - self.rrd_ds.append(self._rrd_ds_val) - self._rrd_parse_state = 1 # In <rrd> - elif self._rrd_parse_state == 3: - if self._rrd_level == 2: - self.rrd_step = long(self._rrd_chars) - self._rrd_parse_state = 1 # In <rrd> - elif self._rrd_parse_state == 2: - if self._rrd_level == 2: - self.rrd_last_update = long(self._rrd_chars) - self._rrd_parse_state = 1 # In <rrd> - elif self._rrd_parse_state == 1: - if self._rrd_level == 1: - self._rrd_parse_state = 0 # At top - - if self._rrd_level: - self._rrd_level = self._rrd_level - 1 - - def characters(self, content): - if self._rrd_capture: - self._rrd_chars = self._rrd_chars + content.strip() - -def _make_numeric(value): - try: - value = float(value) - except: - value = "" - return value - -def simperf_host_xml_fixup(parser, filter_start_time = None, filter_end_time = None): - # Fixup for GAUGE fields that are really COUNTS. They - # were forced to GAUGE to try to disable rrdtool's - # data interpolation/extrapolation for non-uniform time - # samples. - fixup_tags = [ "cpu_user", - "cpu_nice", - "cpu_sys", - "cpu_idle", - "cpu_waitio", - "cpu_intr", - # "file_active", - # "file_free", - # "inode_active", - # "inode_free", - "netif_in_kb", - "netif_in_pkts", - "netif_in_errs", - "netif_in_drop", - "netif_out_kb", - "netif_out_pkts", - "netif_out_errs", - "netif_out_drop", - "vm_page_in", - "vm_page_out", - "vm_swap_in", - "vm_swap_out", - #"vm_mem_total", - #"vm_mem_used", - #"vm_mem_active", - #"vm_mem_inactive", - #"vm_mem_free", - #"vm_mem_buffer", - #"vm_swap_cache", - #"vm_swap_total", - #"vm_swap_used", - #"vm_swap_free", - "cpu_interrupts", - "cpu_switches", - "cpu_forks" ] - - col_count = len(parser.rrd_ds) - row_count = len(parser.rrd_records) - - # Process the last row separately, just to make all values numeric. - for j in range(col_count): - parser.rrd_records[row_count - 1][j] = _make_numeric(parser.rrd_records[row_count - 1][j]) - - # Process all other row/columns. - last_different_row = row_count - 1 - current_row = row_count - 2 - while current_row >= 0: - # Check for a different value than the previous row. If everything is the same - # then this is probably just a filler/bogus entry. - is_different = False - for j in range(col_count): - parser.rrd_records[current_row][j] = _make_numeric(parser.rrd_records[current_row][j]) - if parser.rrd_records[current_row][j] != parser.rrd_records[last_different_row][j]: - # We're good. This is a different row. - is_different = True - - if not is_different: - # This is a filler/bogus entry. Just ignore it. - for j in range(col_count): - parser.rrd_records[current_row][j] = float('nan') - else: - # Some tags need to be converted into deltas. - for j in range(col_count): - if parser.rrd_ds[j]["name"] in fixup_tags: - parser.rrd_records[last_different_row][j] = \ - parser.rrd_records[last_different_row][j] - parser.rrd_records[current_row][j] - last_different_row = current_row - - current_row -= 1 - - # Set fixup_tags in the first row to 'nan' since they aren't useful anymore. - for j in range(col_count): - if parser.rrd_ds[j]["name"] in fixup_tags: - parser.rrd_records[0][j] = float('nan') - - # Add a timestamp to each row and to the catalog. Format and name - # chosen to match other simulator logging (hopefully). - start_time = parser.rrd_last_update - (parser.rrd_step * (row_count - 1)) - # Build a filtered list of rrd_records if we are limited to a time range. - filter_records = False - if filter_start_time is not None or filter_end_time is not None: - filter_records = True - filtered_rrd_records = [] - if filter_start_time is None: - filter_start_time = start_time * 1000 - if filter_end_time is None: - filter_end_time = parser.rrd_last_update * 1000 - - for i in range(row_count): - record_timestamp = (start_time + (i * parser.rrd_step)) * 1000 - parser.rrd_records[i].insert(0, record_timestamp) - if filter_records: - if filter_start_time <= record_timestamp and record_timestamp <= filter_end_time: - filtered_rrd_records.append(parser.rrd_records[i]) - - if filter_records: - parser.rrd_records = filtered_rrd_records - - parser.rrd_ds.insert(0, {"type": "GAUGE", "name": "javascript_timestamp"}) - - -def main(argv=None): - opts, args = getopt.getopt(sys.argv[1:], "i:o:h", ["in=", "out=", "help"]) - input_file = sys.stdin - output_file = sys.stdout - for o, a in opts: - if o in ("-i", "--in"): - input_file = open(a, 'r') - if o in ("-o", "--out"): - output_file = open(a, 'w') - if o in ("-h", "--help"): - usage() - sys.exit(0) - - # Using the SAX parser as it is at least 4X faster and far, far - # smaller on this dataset than the DOM-based interface in xml.dom.minidom. - # With SAX and a 5.4MB xml file, this requires about seven seconds of - # wall-clock time and 32MB VSZ. With the DOM interface, about 22 seconds - # and over 270MB VSZ. - - handler = SimPerfHostXMLParser() - sax.parse(input_file, handler) - if input_file != sys.stdin: - input_file.close() - - # Various format fixups: string-to-num, gauge-to-counts, add - # a time stamp, etc. - simperf_host_xml_fixup(handler) - - # Create JSONable dict with interesting data and format/print it - print >>output_file, simplejson.dumps({ "step" : handler.rrd_step, - "lastupdate": handler.rrd_last_update * 1000, - "ds" : handler.rrd_ds, - "database" : handler.rrd_records }) - - return 0 - -if __name__ == "__main__": - sys.exit(main()) diff --git a/indra/lib/python/indra/util/simperf_oprof_interface.py b/indra/lib/python/indra/util/simperf_oprof_interface.py deleted file mode 100755 index 547d2f9980..0000000000 --- a/indra/lib/python/indra/util/simperf_oprof_interface.py +++ /dev/null @@ -1,167 +0,0 @@ -#!/usr/bin/env python -"""\ -@file simperf_oprof_interface.py -@brief Manage OProfile data collection on a host - -$LicenseInfo:firstyear=2008&license=mit$ - -Copyright (c) 2008-2009, Linden Research, Inc. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -$/LicenseInfo$ -""" - -import sys, os, getopt -import simplejson - - -def usage(): - print "Usage:" - print sys.argv[0] + " [options]" - print " Digest the OProfile report forms that come out of the" - print " simperf_oprof_ctl program's -r/--report command. The result" - print " is an array of dictionaires with the following keys:" - print - print " symbol Name of sampled, calling, or called procedure" - print " file Executable or library where symbol resides" - print " percentage Percentage contribution to profile, calls or called" - print " samples Sample count" - print " calls Methods called by the method in question (full only)" - print " called_by Methods calling the method (full only)" - print - print " For 'full' reports the two keys 'calls' and 'called_by' are" - print " themselves arrays of dictionaries based on the first four keys." - print - print "Return Codes:" - print " None. Aggressively digests everything. Will likely mung results" - print " if a program or library has whitespace in its name." - print - print "Options:" - print " -i, --in Input settings filename. (Default: stdin)" - print " -o, --out Output settings filename. (Default: stdout)" - print " -h, --help Print this message and exit." - print - print "Interfaces:" - print " class SimPerfOProfileInterface()" - -class SimPerfOProfileInterface: - def __init__(self): - self.isBrief = True # public - self.isValid = False # public - self.result = [] # public - - def parse(self, input): - in_samples = False - for line in input: - if in_samples: - if line[0:6] == "------": - self.isBrief = False - self._parseFull(input) - else: - self._parseBrief(input, line) - self.isValid = True - return - try: - hd1, remain = line.split(None, 1) - if hd1 == "samples": - in_samples = True - except ValueError: - pass - - def _parseBrief(self, input, line1): - try: - fld1, fld2, fld3, fld4 = line1.split(None, 3) - self.result.append({"samples" : fld1, - "percentage" : fld2, - "file" : fld3, - "symbol" : fld4.strip("\n")}) - except ValueError: - pass - for line in input: - try: - fld1, fld2, fld3, fld4 = line.split(None, 3) - self.result.append({"samples" : fld1, - "percentage" : fld2, - "file" : fld3, - "symbol" : fld4.strip("\n")}) - except ValueError: - pass - - def _parseFull(self, input): - state = 0 # In 'called_by' section - calls = [] - called_by = [] - current = {} - for line in input: - if line[0:6] == "------": - if len(current): - current["calls"] = calls - current["called_by"] = called_by - self.result.append(current) - state = 0 - calls = [] - called_by = [] - current = {} - else: - try: - fld1, fld2, fld3, fld4 = line.split(None, 3) - tmp = {"samples" : fld1, - "percentage" : fld2, - "file" : fld3, - "symbol" : fld4.strip("\n")} - except ValueError: - continue - if line[0] != " ": - current = tmp - state = 1 # In 'calls' section - elif state == 0: - called_by.append(tmp) - else: - calls.append(tmp) - if len(current): - current["calls"] = calls - current["called_by"] = called_by - self.result.append(current) - - -def main(argv=None): - opts, args = getopt.getopt(sys.argv[1:], "i:o:h", ["in=", "out=", "help"]) - input_file = sys.stdin - output_file = sys.stdout - for o, a in opts: - if o in ("-i", "--in"): - input_file = open(a, 'r') - if o in ("-o", "--out"): - output_file = open(a, 'w') - if o in ("-h", "--help"): - usage() - sys.exit(0) - - oprof = SimPerfOProfileInterface() - oprof.parse(input_file) - if input_file != sys.stdin: - input_file.close() - - # Create JSONable dict with interesting data and format/print it - print >>output_file, simplejson.dumps(oprof.result) - - return 0 - -if __name__ == "__main__": - sys.exit(main()) diff --git a/indra/lib/python/indra/util/simperf_proc_interface.py b/indra/lib/python/indra/util/simperf_proc_interface.py deleted file mode 100755 index de061f68cc..0000000000 --- a/indra/lib/python/indra/util/simperf_proc_interface.py +++ /dev/null @@ -1,191 +0,0 @@ -#!/usr/bin/env python -"""\ -@file simperf_proc_interface.py -@brief Utility to extract log messages from *.<pid>.llsd files containing performance statistics. - -$LicenseInfo:firstyear=2008&license=mit$ - -Copyright (c) 2008-2009, Linden Research, Inc. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -$/LicenseInfo$ -""" - -# ---------------------------------------------------- -# Utility to extract log messages from *.<pid>.llsd -# files that contain performance statistics. - -# ---------------------------------------------------- -import sys, os - -if os.path.exists("setup-path.py"): - execfile("setup-path.py") - -from indra.base import llsd - -DEFAULT_PATH="/dev/shm/simperf/" - - -# ---------------------------------------------------- -# Pull out the stats and return a single document -def parse_logfile(filename, target_column=None, verbose=False): - full_doc = [] - # Open source temp log file. Let exceptions percolate up. - sourcefile = open( filename,'r') - - if verbose: - print "Reading " + filename - - # Parse and output all lines from the temp file - for line in sourcefile.xreadlines(): - partial_doc = llsd.parse(line) - if partial_doc is not None: - if target_column is None: - full_doc.append(partial_doc) - else: - trim_doc = { target_column: partial_doc[target_column] } - if target_column != "fps": - trim_doc[ 'fps' ] = partial_doc[ 'fps' ] - trim_doc[ '/total_time' ] = partial_doc[ '/total_time' ] - trim_doc[ 'utc_time' ] = partial_doc[ 'utc_time' ] - full_doc.append(trim_doc) - - sourcefile.close() - return full_doc - -# Extract just the meta info line, and the timestamp of the first/last frame entry. -def parse_logfile_info(filename, verbose=False): - # Open source temp log file. Let exceptions percolate up. - sourcefile = open(filename, 'rU') # U is to open with Universal newline support - - if verbose: - print "Reading " + filename - - # The first line is the meta info line. - info_line = sourcefile.readline() - if not info_line: - sourcefile.close() - return None - - # The rest of the lines are frames. Read the first and last to get the time range. - info = llsd.parse( info_line ) - info['start_time'] = None - info['end_time'] = None - first_frame = sourcefile.readline() - if first_frame: - try: - info['start_time'] = int(llsd.parse(first_frame)['timestamp']) - except: - pass - - # Read the file backwards to find the last two lines. - sourcefile.seek(0, 2) - file_size = sourcefile.tell() - offset = 1024 - num_attempts = 0 - end_time = None - if file_size < offset: - offset = file_size - while 1: - sourcefile.seek(-1*offset, 2) - read_str = sourcefile.read(offset) - # Remove newline at the end - if read_str[offset - 1] == '\n': - read_str = read_str[0:-1] - lines = read_str.split('\n') - full_line = None - if len(lines) > 2: # Got two line - try: - end_time = llsd.parse(lines[-1])['timestamp'] - except: - # We couldn't parse this line. Try once more. - try: - end_time = llsd.parse(lines[-2])['timestamp'] - except: - # Nope. Just move on. - pass - break - if len(read_str) == file_size: # Reached the beginning - break - offset += 1024 - - info['end_time'] = int(end_time) - - sourcefile.close() - return info - - -def parse_proc_filename(filename): - try: - name_as_list = filename.split(".") - cur_stat_type = name_as_list[0].split("_")[0] - cur_pid = name_as_list[1] - except IndexError, ValueError: - return (None, None) - return (cur_pid, cur_stat_type) - -# ---------------------------------------------------- -def get_simstats_list(path=None): - """ Return stats (pid, type) listed in <type>_proc.<pid>.llsd """ - if path is None: - path = DEFAULT_PATH - simstats_list = [] - for file_name in os.listdir(path): - if file_name.endswith(".llsd") and file_name != "simperf_proc_config.llsd": - simstats_info = parse_logfile_info(path + file_name) - if simstats_info is not None: - simstats_list.append(simstats_info) - return simstats_list - -def get_log_info_list(pid=None, stat_type=None, path=None, target_column=None, verbose=False): - """ Return data from all llsd files matching the pid and stat type """ - if path is None: - path = DEFAULT_PATH - log_info_list = {} - for file_name in os.listdir ( path ): - if file_name.endswith(".llsd") and file_name != "simperf_proc_config.llsd": - (cur_pid, cur_stat_type) = parse_proc_filename(file_name) - if cur_pid is None: - continue - if pid is not None and pid != cur_pid: - continue - if stat_type is not None and stat_type != cur_stat_type: - continue - log_info_list[cur_pid] = parse_logfile(path + file_name, target_column, verbose) - return log_info_list - -def delete_simstats_files(pid=None, stat_type=None, path=None): - """ Delete *.<pid>.llsd files """ - if path is None: - path = DEFAULT_PATH - del_list = [] - for file_name in os.listdir(path): - if file_name.endswith(".llsd") and file_name != "simperf_proc_config.llsd": - (cur_pid, cur_stat_type) = parse_proc_filename(file_name) - if cur_pid is None: - continue - if pid is not None and pid != cur_pid: - continue - if stat_type is not None and stat_type != cur_stat_type: - continue - del_list.append(cur_pid) - # Allow delete related exceptions to percolate up if this fails. - os.unlink(os.path.join(DEFAULT_PATH, file_name)) - return del_list - diff --git a/indra/lib/python/indra/util/term.py b/indra/lib/python/indra/util/term.py deleted file mode 100755 index 8c316a1f12..0000000000 --- a/indra/lib/python/indra/util/term.py +++ /dev/null @@ -1,222 +0,0 @@ -''' -@file term.py -@brief a better shutil.copytree replacement - -$LicenseInfo:firstyear=2007&license=mit$ - -Copyright (c) 2007-2009, Linden Research, Inc. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -$/LicenseInfo$ -''' - -#http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/475116 - -import sys, re - -class TerminalController: - """ - A class that can be used to portably generate formatted output to - a terminal. - - `TerminalController` defines a set of instance variables whose - values are initialized to the control sequence necessary to - perform a given action. These can be simply included in normal - output to the terminal: - - >>> term = TerminalController() - >>> print 'This is '+term.GREEN+'green'+term.NORMAL - - Alternatively, the `render()` method can used, which replaces - '${action}' with the string required to perform 'action': - - >>> term = TerminalController() - >>> print term.render('This is ${GREEN}green${NORMAL}') - - If the terminal doesn't support a given action, then the value of - the corresponding instance variable will be set to ''. As a - result, the above code will still work on terminals that do not - support color, except that their output will not be colored. - Also, this means that you can test whether the terminal supports a - given action by simply testing the truth value of the - corresponding instance variable: - - >>> term = TerminalController() - >>> if term.CLEAR_SCREEN: - ... print 'This terminal supports clearning the screen.' - - Finally, if the width and height of the terminal are known, then - they will be stored in the `COLS` and `LINES` attributes. - """ - # Cursor movement: - BOL = '' #: Move the cursor to the beginning of the line - UP = '' #: Move the cursor up one line - DOWN = '' #: Move the cursor down one line - LEFT = '' #: Move the cursor left one char - RIGHT = '' #: Move the cursor right one char - - # Deletion: - CLEAR_SCREEN = '' #: Clear the screen and move to home position - CLEAR_EOL = '' #: Clear to the end of the line. - CLEAR_BOL = '' #: Clear to the beginning of the line. - CLEAR_EOS = '' #: Clear to the end of the screen - - # Output modes: - BOLD = '' #: Turn on bold mode - BLINK = '' #: Turn on blink mode - DIM = '' #: Turn on half-bright mode - REVERSE = '' #: Turn on reverse-video mode - NORMAL = '' #: Turn off all modes - - # Cursor display: - HIDE_CURSOR = '' #: Make the cursor invisible - SHOW_CURSOR = '' #: Make the cursor visible - - # Terminal size: - COLS = None #: Width of the terminal (None for unknown) - LINES = None #: Height of the terminal (None for unknown) - - # Foreground colors: - BLACK = BLUE = GREEN = CYAN = RED = MAGENTA = YELLOW = WHITE = '' - - # Background colors: - BG_BLACK = BG_BLUE = BG_GREEN = BG_CYAN = '' - BG_RED = BG_MAGENTA = BG_YELLOW = BG_WHITE = '' - - _STRING_CAPABILITIES = """ - BOL=cr UP=cuu1 DOWN=cud1 LEFT=cub1 RIGHT=cuf1 - CLEAR_SCREEN=clear CLEAR_EOL=el CLEAR_BOL=el1 CLEAR_EOS=ed BOLD=bold - BLINK=blink DIM=dim REVERSE=rev UNDERLINE=smul NORMAL=sgr0 - HIDE_CURSOR=cinvis SHOW_CURSOR=cnorm""".split() - _COLORS = """BLACK BLUE GREEN CYAN RED MAGENTA YELLOW WHITE""".split() - _ANSICOLORS = "BLACK RED GREEN YELLOW BLUE MAGENTA CYAN WHITE".split() - - def __init__(self, term_stream=sys.stdout): - """ - Create a `TerminalController` and initialize its attributes - with appropriate values for the current terminal. - `term_stream` is the stream that will be used for terminal - output; if this stream is not a tty, then the terminal is - assumed to be a dumb terminal (i.e., have no capabilities). - """ - # Curses isn't available on all platforms - try: import curses - except: return - - # If the stream isn't a tty, then assume it has no capabilities. - if not term_stream.isatty(): return - - # Check the terminal type. If we fail, then assume that the - # terminal has no capabilities. - try: curses.setupterm() - except: return - - # Look up numeric capabilities. - self.COLS = curses.tigetnum('cols') - self.LINES = curses.tigetnum('lines') - - # Look up string capabilities. - for capability in self._STRING_CAPABILITIES: - (attrib, cap_name) = capability.split('=') - setattr(self, attrib, self._tigetstr(cap_name) or '') - - # Colors - set_fg = self._tigetstr('setf') - if set_fg: - for i,color in zip(range(len(self._COLORS)), self._COLORS): - setattr(self, color, curses.tparm(set_fg, i) or '') - set_fg_ansi = self._tigetstr('setaf') - if set_fg_ansi: - for i,color in zip(range(len(self._ANSICOLORS)), self._ANSICOLORS): - setattr(self, color, curses.tparm(set_fg_ansi, i) or '') - set_bg = self._tigetstr('setb') - if set_bg: - for i,color in zip(range(len(self._COLORS)), self._COLORS): - setattr(self, 'BG_'+color, curses.tparm(set_bg, i) or '') - set_bg_ansi = self._tigetstr('setab') - if set_bg_ansi: - for i,color in zip(range(len(self._ANSICOLORS)), self._ANSICOLORS): - setattr(self, 'BG_'+color, curses.tparm(set_bg_ansi, i) or '') - - def _tigetstr(self, cap_name): - # String capabilities can include "delays" of the form "$<2>". - # For any modern terminal, we should be able to just ignore - # these, so strip them out. - import curses - cap = curses.tigetstr(cap_name) or '' - return re.sub(r'\$<\d+>[/*]?', '', cap) - - def render(self, template): - """ - Replace each $-substitutions in the given template string with - the corresponding terminal control string (if it's defined) or - '' (if it's not). - """ - return re.sub(r'\$\$|\${\w+}', self._render_sub, template) - - def _render_sub(self, match): - s = match.group() - if s == '$$': return s - else: return getattr(self, s[2:-1]) - -####################################################################### -# Example use case: progress bar -####################################################################### - -class ProgressBar: - """ - A 3-line progress bar, which looks like:: - - Header - 20% [===========----------------------------------] - progress message - - The progress bar is colored, if the terminal supports color - output; and adjusts to the width of the terminal. - """ - BAR = '%3d%% ${GREEN}[${BOLD}%s%s${NORMAL}${GREEN}]${NORMAL}\n' - HEADER = '${BOLD}${CYAN}%s${NORMAL}\n\n' - - def __init__(self, term, header): - self.term = term - if not (self.term.CLEAR_EOL and self.term.UP and self.term.BOL): - raise ValueError("Terminal isn't capable enough -- you " - "should use a simpler progress dispaly.") - self.width = self.term.COLS or 75 - self.bar = term.render(self.BAR) - self.header = self.term.render(self.HEADER % header.center(self.width)) - self.cleared = 1 #: true if we haven't drawn the bar yet. - self.update(0, '') - - def update(self, percent, message): - if self.cleared: - sys.stdout.write(self.header) - self.cleared = 0 - n = int((self.width-10)*percent) - sys.stdout.write( - self.term.BOL + self.term.UP + self.term.CLEAR_EOL + - (self.bar % (100*percent, '='*n, '-'*(self.width-10-n))) + - self.term.CLEAR_EOL + message.center(self.width)) - - def clear(self): - if not self.cleared: - sys.stdout.write(self.term.BOL + self.term.CLEAR_EOL + - self.term.UP + self.term.CLEAR_EOL + - self.term.UP + self.term.CLEAR_EOL) - self.cleared = 1 diff --git a/indra/lib/python/uuid.py b/indra/lib/python/uuid.py deleted file mode 100755 index e956383cca..0000000000 --- a/indra/lib/python/uuid.py +++ /dev/null @@ -1,508 +0,0 @@ -#!/usr/bin/python -## $LicenseInfo:firstyear=2011&license=viewerlgpl$ -## Second Life Viewer Source Code -## Copyright (C) 2011, Linden Research, Inc. -## -## This library is free software; you can redistribute it and/or -## modify it under the terms of the GNU Lesser General Public -## License as published by the Free Software Foundation; -## version 2.1 of the License only. -## -## This library is distributed in the hope that it will be useful, -## but WITHOUT ANY WARRANTY; without even the implied warranty of -## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -## Lesser General Public License for more details. -## -## You should have received a copy of the GNU Lesser General Public -## License along with this library; if not, write to the Free Software -## Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA -## -## Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA -## $/LicenseInfo$ -r"""UUID objects (universally unique identifiers) according to RFC 4122. - -This module provides immutable UUID objects (class UUID) and the functions -uuid1(), uuid3(), uuid4(), uuid5() for generating version 1, 3, 4, and 5 -UUIDs as specified in RFC 4122. - -If all you want is a unique ID, you should probably call uuid1() or uuid4(). -Note that uuid1() may compromise privacy since it creates a UUID containing -the computer's network address. uuid4() creates a random UUID. - -Typical usage: - - >>> import uuid - - # make a UUID based on the host ID and current time - >>> uuid.uuid1() - UUID('a8098c1a-f86e-11da-bd1a-00112444be1e') - - # make a UUID using an MD5 hash of a namespace UUID and a name - >>> uuid.uuid3(uuid.NAMESPACE_DNS, 'python.org') - UUID('6fa459ea-ee8a-3ca4-894e-db77e160355e') - - # make a random UUID - >>> uuid.uuid4() - UUID('16fd2706-8baf-433b-82eb-8c7fada847da') - - # make a UUID using a SHA-1 hash of a namespace UUID and a name - >>> uuid.uuid5(uuid.NAMESPACE_DNS, 'python.org') - UUID('886313e1-3b8a-5372-9b90-0c9aee199e5d') - - # make a UUID from a string of hex digits (braces and hyphens ignored) - >>> x = uuid.UUID('{00010203-0405-0607-0809-0a0b0c0d0e0f}') - - # convert a UUID to a string of hex digits in standard form - >>> str(x) - '00010203-0405-0607-0809-0a0b0c0d0e0f' - - # get the raw 16 bytes of the UUID - >>> x.bytes - '\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f' - - # make a UUID from a 16-byte string - >>> uuid.UUID(bytes=x.bytes) - UUID('00010203-0405-0607-0809-0a0b0c0d0e0f') - -This module works with Python 2.3 or higher.""" - -__author__ = 'Ka-Ping Yee <ping@zesty.ca>' -__date__ = '$Date: 2006/06/12 23:15:40 $'.split()[1].replace('/', '-') -__version__ = '$Revision: 1.30 $'.split()[1] - -RESERVED_NCS, RFC_4122, RESERVED_MICROSOFT, RESERVED_FUTURE = [ - 'reserved for NCS compatibility', 'specified in RFC 4122', - 'reserved for Microsoft compatibility', 'reserved for future definition'] - -class UUID(object): - """Instances of the UUID class represent UUIDs as specified in RFC 4122. - UUID objects are immutable, hashable, and usable as dictionary keys. - Converting a UUID to a string with str() yields something in the form - '12345678-1234-1234-1234-123456789abc'. The UUID constructor accepts - four possible forms: a similar string of hexadecimal digits, or a - string of 16 raw bytes as an argument named 'bytes', or a tuple of - six integer fields (with 32-bit, 16-bit, 16-bit, 8-bit, 8-bit, and - 48-bit values respectively) as an argument named 'fields', or a single - 128-bit integer as an argument named 'int'. - - UUIDs have these read-only attributes: - - bytes the UUID as a 16-byte string - - fields a tuple of the six integer fields of the UUID, - which are also available as six individual attributes - and two derived attributes: - - time_low the first 32 bits of the UUID - time_mid the next 16 bits of the UUID - time_hi_version the next 16 bits of the UUID - clock_seq_hi_variant the next 8 bits of the UUID - clock_seq_low the next 8 bits of the UUID - node the last 48 bits of the UUID - - time the 60-bit timestamp - clock_seq the 14-bit sequence number - - hex the UUID as a 32-character hexadecimal string - - int the UUID as a 128-bit integer - - urn the UUID as a URN as specified in RFC 4122 - - variant the UUID variant (one of the constants RESERVED_NCS, - RFC_4122, RESERVED_MICROSOFT, or RESERVED_FUTURE) - - version the UUID version number (1 through 5, meaningful only - when the variant is RFC_4122) - """ - - def __init__(self, hex=None, bytes=None, fields=None, int=None, - version=None): - r"""Create a UUID from either a string of 32 hexadecimal digits, - a string of 16 bytes as the 'bytes' argument, a tuple of six - integers (32-bit time_low, 16-bit time_mid, 16-bit time_hi_version, - 8-bit clock_seq_hi_variant, 8-bit clock_seq_low, 48-bit node) as - the 'fields' argument, or a single 128-bit integer as the 'int' - argument. When a string of hex digits is given, curly braces, - hyphens, and a URN prefix are all optional. For example, these - expressions all yield the same UUID: - - UUID('{12345678-1234-5678-1234-567812345678}') - UUID('12345678123456781234567812345678') - UUID('urn:uuid:12345678-1234-5678-1234-567812345678') - UUID(bytes='\x12\x34\x56\x78'*4) - UUID(fields=(0x12345678, 0x1234, 0x5678, 0x12, 0x34, 0x567812345678)) - UUID(int=0x12345678123456781234567812345678) - - Exactly one of 'hex', 'bytes', 'fields', or 'int' must be given. - The 'version' argument is optional; if given, the resulting UUID - will have its variant and version number set according to RFC 4122, - overriding bits in the given 'hex', 'bytes', 'fields', or 'int'. - """ - - if [hex, bytes, fields, int].count(None) != 3: - raise TypeError('need just one of hex, bytes, fields, or int') - if hex is not None: - hex = hex.replace('urn:', '').replace('uuid:', '') - hex = hex.strip('{}').replace('-', '') - if len(hex) != 32: - raise ValueError('badly formed hexadecimal UUID string') - int = long(hex, 16) - if bytes is not None: - if len(bytes) != 16: - raise ValueError('bytes is not a 16-char string') - int = long(('%02x'*16) % tuple(map(ord, bytes)), 16) - if fields is not None: - if len(fields) != 6: - raise ValueError('fields is not a 6-tuple') - (time_low, time_mid, time_hi_version, - clock_seq_hi_variant, clock_seq_low, node) = fields - if not 0 <= time_low < 1<<32L: - raise ValueError('field 1 out of range (need a 32-bit value)') - if not 0 <= time_mid < 1<<16L: - raise ValueError('field 2 out of range (need a 16-bit value)') - if not 0 <= time_hi_version < 1<<16L: - raise ValueError('field 3 out of range (need a 16-bit value)') - if not 0 <= clock_seq_hi_variant < 1<<8L: - raise ValueError('field 4 out of range (need an 8-bit value)') - if not 0 <= clock_seq_low < 1<<8L: - raise ValueError('field 5 out of range (need an 8-bit value)') - if not 0 <= node < 1<<48L: - raise ValueError('field 6 out of range (need a 48-bit value)') - clock_seq = (clock_seq_hi_variant << 8L) | clock_seq_low - int = ((time_low << 96L) | (time_mid << 80L) | - (time_hi_version << 64L) | (clock_seq << 48L) | node) - if int is not None: - if not 0 <= int < 1<<128L: - raise ValueError('int is out of range (need a 128-bit value)') - if version is not None: - if not 1 <= version <= 5: - raise ValueError('illegal version number') - # Set the variant to RFC 4122. - int &= ~(0xc000 << 48L) - int |= 0x8000 << 48L - # Set the version number. - int &= ~(0xf000 << 64L) - int |= version << 76L - self.__dict__['int'] = int - - def __cmp__(self, other): - if isinstance(other, UUID): - return cmp(self.int, other.int) - return NotImplemented - - def __hash__(self): - return hash(self.int) - - def __int__(self): - return self.int - - def __repr__(self): - return 'UUID(%r)' % str(self) - - def __setattr__(self, name, value): - raise TypeError('UUID objects are immutable') - - def __str__(self): - hex = '%032x' % self.int - return '%s-%s-%s-%s-%s' % ( - hex[:8], hex[8:12], hex[12:16], hex[16:20], hex[20:]) - - def get_bytes(self): - bytes = '' - for shift in range(0, 128, 8): - bytes = chr((self.int >> shift) & 0xff) + bytes - return bytes - - bytes = property(get_bytes) - - def get_fields(self): - return (self.time_low, self.time_mid, self.time_hi_version, - self.clock_seq_hi_variant, self.clock_seq_low, self.node) - - fields = property(get_fields) - - def get_time_low(self): - return self.int >> 96L - - time_low = property(get_time_low) - - def get_time_mid(self): - return (self.int >> 80L) & 0xffff - - time_mid = property(get_time_mid) - - def get_time_hi_version(self): - return (self.int >> 64L) & 0xffff - - time_hi_version = property(get_time_hi_version) - - def get_clock_seq_hi_variant(self): - return (self.int >> 56L) & 0xff - - clock_seq_hi_variant = property(get_clock_seq_hi_variant) - - def get_clock_seq_low(self): - return (self.int >> 48L) & 0xff - - clock_seq_low = property(get_clock_seq_low) - - def get_time(self): - return (((self.time_hi_version & 0x0fffL) << 48L) | - (self.time_mid << 32L) | self.time_low) - - time = property(get_time) - - def get_clock_seq(self): - return (((self.clock_seq_hi_variant & 0x3fL) << 8L) | - self.clock_seq_low) - - clock_seq = property(get_clock_seq) - - def get_node(self): - return self.int & 0xffffffffffff - - node = property(get_node) - - def get_hex(self): - return '%032x' % self.int - - hex = property(get_hex) - - def get_urn(self): - return 'urn:uuid:' + str(self) - - urn = property(get_urn) - - def get_variant(self): - if not self.int & (0x8000 << 48L): - return RESERVED_NCS - elif not self.int & (0x4000 << 48L): - return RFC_4122 - elif not self.int & (0x2000 << 48L): - return RESERVED_MICROSOFT - else: - return RESERVED_FUTURE - - variant = property(get_variant) - - def get_version(self): - # The version bits are only meaningful for RFC 4122 UUIDs. - if self.variant == RFC_4122: - return int((self.int >> 76L) & 0xf) - - version = property(get_version) - -def _ifconfig_getnode(): - """Get the hardware address on Unix by running ifconfig.""" - import os - for dir in ['', '/sbin/', '/usr/sbin']: - try: - path = os.path.join(dir, 'ifconfig') - if os.path.exists(path): - pipe = os.popen(path) - else: - continue - except IOError: - continue - for line in pipe: - words = line.lower().split() - for i in range(len(words)): - if words[i] in ['hwaddr', 'ether']: - return int(words[i + 1].replace(':', ''), 16) - -def _ipconfig_getnode(): - """Get the hardware address on Windows by running ipconfig.exe.""" - import os, re - dirs = ['', r'c:\windows\system32', r'c:\winnt\system32'] - try: - import ctypes - buffer = ctypes.create_string_buffer(300) - ctypes.windll.kernel32.GetSystemDirectoryA(buffer, 300) - dirs.insert(0, buffer.value.decode('mbcs')) - except: - pass - for dir in dirs: - try: - pipe = os.popen(os.path.join(dir, 'ipconfig') + ' /all') - except IOError: - continue - for line in pipe: - value = line.split(':')[-1].strip().lower() - if re.match('([0-9a-f][0-9a-f]-){5}[0-9a-f][0-9a-f]', value): - return int(value.replace('-', ''), 16) - -def _netbios_getnode(): - """Get the hardware address on Windows using NetBIOS calls. - See http://support.microsoft.com/kb/118623 for details.""" - import win32wnet, netbios - ncb = netbios.NCB() - ncb.Command = netbios.NCBENUM - ncb.Buffer = adapters = netbios.LANA_ENUM() - adapters._pack() - if win32wnet.Netbios(ncb) != 0: - return - adapters._unpack() - for i in range(adapters.length): - ncb.Reset() - ncb.Command = netbios.NCBRESET - ncb.Lana_num = ord(adapters.lana[i]) - if win32wnet.Netbios(ncb) != 0: - continue - ncb.Reset() - ncb.Command = netbios.NCBASTAT - ncb.Lana_num = ord(adapters.lana[i]) - ncb.Callname = '*'.ljust(16) - ncb.Buffer = status = netbios.ADAPTER_STATUS() - if win32wnet.Netbios(ncb) != 0: - continue - status._unpack() - bytes = map(ord, status.adapter_address) - return ((bytes[0]<<40L) + (bytes[1]<<32L) + (bytes[2]<<24L) + - (bytes[3]<<16L) + (bytes[4]<<8L) + bytes[5]) - -# Thanks to Thomas Heller for ctypes and for his help with its use here. - -# If ctypes is available, use it to find system routines for UUID generation. -_uuid_generate_random = _uuid_generate_time = _UuidCreate = None -try: - import ctypes, ctypes.util - _buffer = ctypes.create_string_buffer(16) - - # The uuid_generate_* routines are provided by libuuid on at least - # Linux and FreeBSD, and provided by libc on Mac OS X. - for libname in ['uuid', 'c']: - try: - lib = ctypes.CDLL(ctypes.util.find_library(libname)) - except: - continue - if hasattr(lib, 'uuid_generate_random'): - _uuid_generate_random = lib.uuid_generate_random - if hasattr(lib, 'uuid_generate_time'): - _uuid_generate_time = lib.uuid_generate_time - - # On Windows prior to 2000, UuidCreate gives a UUID containing the - # hardware address. On Windows 2000 and later, UuidCreate makes a - # random UUID and UuidCreateSequential gives a UUID containing the - # hardware address. These routines are provided by the RPC runtime. - try: - lib = ctypes.windll.rpcrt4 - except: - lib = None - _UuidCreate = getattr(lib, 'UuidCreateSequential', - getattr(lib, 'UuidCreate', None)) -except: - pass - -def _unixdll_getnode(): - """Get the hardware address on Unix using ctypes.""" - _uuid_generate_time(_buffer) - return UUID(bytes=_buffer.raw).node - -def _windll_getnode(): - """Get the hardware address on Windows using ctypes.""" - if _UuidCreate(_buffer) == 0: - return UUID(bytes=_buffer.raw).node - -def _random_getnode(): - """Get a random node ID, with eighth bit set as suggested by RFC 4122.""" - import random - return random.randrange(0, 1<<48L) | 0x010000000000L - -_node = None - -def getnode(): - """Get the hardware address as a 48-bit integer. The first time this - runs, it may launch a separate program, which could be quite slow. If - all attempts to obtain the hardware address fail, we choose a random - 48-bit number with its eighth bit set to 1 as recommended in RFC 4122.""" - - global _node - if _node is not None: - return _node - - import sys - if sys.platform == 'win32': - getters = [_windll_getnode, _netbios_getnode, _ipconfig_getnode] - else: - getters = [_unixdll_getnode, _ifconfig_getnode] - - for getter in getters + [_random_getnode]: - try: - _node = getter() - except: - continue - if _node is not None: - return _node - -def uuid1(node=None, clock_seq=None): - """Generate a UUID from a host ID, sequence number, and the current time. - If 'node' is not given, getnode() is used to obtain the hardware - address. If 'clock_seq' is given, it is used as the sequence number; - otherwise a random 14-bit sequence number is chosen.""" - - # When the system provides a version-1 UUID generator, use it (but don't - # use UuidCreate here because its UUIDs don't conform to RFC 4122). - if _uuid_generate_time and node is clock_seq is None: - _uuid_generate_time(_buffer) - return UUID(bytes=_buffer.raw) - - import time - nanoseconds = int(time.time() * 1e9) - # 0x01b21dd213814000 is the number of 100-ns intervals between the - # UUID epoch 1582-10-15 00:00:00 and the Unix epoch 1970-01-01 00:00:00. - timestamp = int(nanoseconds/100) + 0x01b21dd213814000L - if clock_seq is None: - import random - clock_seq = random.randrange(1<<14L) # instead of stable storage - time_low = timestamp & 0xffffffffL - time_mid = (timestamp >> 32L) & 0xffffL - time_hi_version = (timestamp >> 48L) & 0x0fffL - clock_seq_low = clock_seq & 0xffL - clock_seq_hi_variant = (clock_seq >> 8L) & 0x3fL - if node is None: - node = getnode() - return UUID(fields=(time_low, time_mid, time_hi_version, - clock_seq_hi_variant, clock_seq_low, node), version=1) - -def uuid3(namespace, name): - """Generate a UUID from the MD5 hash of a namespace UUID and a name.""" - try: - # Python 2.6 - from hashlib import md5 - except ImportError: - # Python 2.5 and earlier - from md5 import new as md5 - - hash = md5(namespace.bytes + name).digest() - return UUID(bytes=hash[:16], version=3) - -def uuid4(): - """Generate a random UUID.""" - - # When the system provides a version-4 UUID generator, use it. - if _uuid_generate_random: - _uuid_generate_random(_buffer) - return UUID(bytes=_buffer.raw) - - # Otherwise, get randomness from urandom or the 'random' module. - try: - import os - return UUID(bytes=os.urandom(16), version=4) - except: - import random - bytes = [chr(random.randrange(256)) for i in range(16)] - return UUID(bytes=bytes, version=4) - -def uuid5(namespace, name): - """Generate a UUID from the SHA-1 hash of a namespace UUID and a name.""" - import sha - hash = sha.sha(namespace.bytes + name).digest() - return UUID(bytes=hash[:16], version=5) - -# The following standard UUIDs are for use with uuid3() or uuid5(). - -NAMESPACE_DNS = UUID('6ba7b810-9dad-11d1-80b4-00c04fd430c8') -NAMESPACE_URL = UUID('6ba7b811-9dad-11d1-80b4-00c04fd430c8') -NAMESPACE_OID = UUID('6ba7b812-9dad-11d1-80b4-00c04fd430c8') -NAMESPACE_X500 = UUID('6ba7b814-9dad-11d1-80b4-00c04fd430c8') diff --git a/indra/llcommon/tests/llleap_test.cpp b/indra/llcommon/tests/llleap_test.cpp index d342dece84..c387da6c48 100644 --- a/indra/llcommon/tests/llleap_test.cpp +++ b/indra/llcommon/tests/llleap_test.cpp @@ -110,10 +110,7 @@ namespace tut // finding indra/lib/python. Use our __FILE__, with // raw-string syntax to deal with Windows pathnames. "mydir = os.path.dirname(r'" << __FILE__ << "')\n" - // We expect mydir to be .../indra/llcommon/tests. - "sys.path.insert(0,\n" - " os.path.join(mydir, os.pardir, os.pardir, 'lib', 'python'))\n" - "from indra.base import llsd\n" + "from llbase import llsd\n" "\n" "class ProtocolError(Exception):\n" " def __init__(self, msg, data):\n" diff --git a/indra/llcommon/tests/llsdserialize_test.cpp b/indra/llcommon/tests/llsdserialize_test.cpp index 6fbb9abfc0..81b930e1e2 100644 --- a/indra/llcommon/tests/llsdserialize_test.cpp +++ b/indra/llcommon/tests/llsdserialize_test.cpp @@ -1518,10 +1518,7 @@ namespace tut // scanner. import_llsd("import os.path\n" "import sys\n" - "sys.path.insert(0,\n" - " os.path.join(os.path.dirname(r'" __FILE__ "'),\n" - " os.pardir, os.pardir, 'lib', 'python'))\n" - "from indra.base import llsd\n") + "from llbase import llsd\n") {} ~TestPythonCompatible() {} diff --git a/indra/llcorehttp/tests/test_llcorehttp_peer.py b/indra/llcorehttp/tests/test_llcorehttp_peer.py index 04cde651c4..6c5f37d407 100755 --- a/indra/llcorehttp/tests/test_llcorehttp_peer.py +++ b/indra/llcorehttp/tests/test_llcorehttp_peer.py @@ -42,10 +42,8 @@ except ImportError: from BaseHTTPServer import HTTPServer, BaseHTTPRequestHandler from SocketServer import ThreadingMixIn -mydir = os.path.dirname(__file__) # expected to be .../indra/llcorehttp/tests/ -sys.path.insert(0, os.path.join(mydir, os.pardir, os.pardir, "lib", "python")) -from indra.util.fastest_elementtree import parse as xml_parse -from indra.base import llsd +from llbase.fastest_elementtree import parse as xml_parse +from llbase import llsd from testrunner import freeport, run, debug, VERBOSE class TestHTTPRequestHandler(BaseHTTPRequestHandler): diff --git a/indra/llmessage/tests/test_llsdmessage_peer.py b/indra/llmessage/tests/test_llsdmessage_peer.py index e45249b1cb..bac18fa374 100755 --- a/indra/llmessage/tests/test_llsdmessage_peer.py +++ b/indra/llmessage/tests/test_llsdmessage_peer.py @@ -34,10 +34,8 @@ import sys from threading import Thread from BaseHTTPServer import HTTPServer, BaseHTTPRequestHandler -mydir = os.path.dirname(__file__) # expected to be .../indra/llmessage/tests/ -sys.path.insert(0, os.path.join(mydir, os.pardir, os.pardir, "lib", "python")) -from indra.util.fastest_elementtree import parse as xml_parse -from indra.base import llsd +from llbase.fastest_elementtree import parse as xml_parse +from llbase import llsd from testrunner import freeport, run, debug, VERBOSE import time |