diff options
Diffstat (limited to 'indra/lib')
| -rw-r--r-- | indra/lib/python/indra/base/config.py | 231 | ||||
| -rw-r--r-- | indra/lib/python/indra/base/llsd.py | 104 | ||||
| -rw-r--r-- | indra/lib/python/indra/base/lluuid.py | 32 | ||||
| -rw-r--r-- | indra/lib/python/indra/ipc/llsdhttp.py | 7 | ||||
| -rw-r--r-- | indra/lib/python/indra/ipc/mysql_pool.py | 88 | ||||
| -rw-r--r-- | indra/lib/python/indra/util/fastest_elementtree.py | 52 | ||||
| -rw-r--r-- | indra/lib/python/indra/util/named_query.py | 5 |
7 files changed, 349 insertions, 170 deletions
diff --git a/indra/lib/python/indra/base/config.py b/indra/lib/python/indra/base/config.py index a28c59c702..9d8da7dd15 100644 --- a/indra/lib/python/indra/base/config.py +++ b/indra/lib/python/indra/base/config.py @@ -26,74 +26,219 @@ THE SOFTWARE. $/LicenseInfo$ """ -from os.path import dirname, join, realpath +import copy +import os +import traceback +import time import types + +from os.path import dirname, getmtime, join, realpath from indra.base import llsd -_g_config_dict = None - -def load(indra_xml_file=None): - global _g_config_dict - if _g_config_dict == None: - if indra_xml_file is None: - ## going from: - ## "/opt/linden/indra/lib/python/indra/base/config.py" - ## to: - ## "/opt/linden/etc/indra.xml" - indra_xml_file = realpath( - dirname(realpath(__file__)) + "../../../../../../etc/indra.xml") - config_file = file(indra_xml_file) - _g_config_dict = llsd.LLSD().parse(config_file.read()) +_g_config = None + +class IndraConfig(object): + """ + IndraConfig loads a 'indra' xml configuration file + and loads into memory. This representation in memory + can get updated to overwrite values or add new values. + + The xml configuration file is considered a live file and changes + to the file are checked and reloaded periodically. If a value had + been overwritten via the update or set method, the loaded values + from the file are ignored (the values from the update/set methods + override) + """ + def __init__(self, indra_config_file): + self._indra_config_file = indra_config_file + self._reload_check_interval = 30 # seconds + self._last_check_time = 0 + self._last_mod_time = 0 + + self._config_overrides = {} + self._config_file_dict = {} + self._combined_dict = {} + + self._load() + + def _load(self): + if self._indra_config_file is None: + return + + config_file = open(self._indra_config_file) + self._config_file_dict = llsd.parse(config_file.read()) + self._combine_dictionaries() config_file.close() - #print "loaded config from",indra_xml_file,"into",_g_config_dict -def dump(indra_xml_file, indra_cfg={}, update_in_mem=False): + self._last_mod_time = self._get_last_modified_time() + self._last_check_time = time.time() # now + + def _get_last_modified_time(self): + """ + Returns the mtime (last modified time) of the config file, + if such exists. + """ + if self._indra_config_file is not None: + return os.path.getmtime(self._indra_config_file) + + return 0 + + def _combine_dictionaries(self): + self._combined_dict = {} + self._combined_dict.update(self._config_file_dict) + self._combined_dict.update(self._config_overrides) + + def _reload_if_necessary(self): + now = time.time() + + if (now - self._last_check_time) > self._reload_check_interval: + self._last_check_time = now + try: + modtime = self._get_last_modified_time() + if modtime > self._last_mod_time: + self._load() + except OSError, e: + if e.errno == errno.ENOENT: # file not found + # someone messed with our internal state + # or removed the file + + print 'WARNING: Configuration file has been removed ' + (self._indra_config_file) + print 'Disabling reloading of configuration file.' + + traceback.print_exc() + + self._indra_config_file = None + self._last_check_time = 0 + self._last_mod_time = 0 + else: + raise # pass the exception along to the caller + + def __getitem__(self, key): + self._reload_if_necessary() + + return self._combined_dict[key] + + def get(self, key, default = None): + try: + return self.__getitem__(key) + except KeyError: + return default + + def __setitem__(self, key, value): + """ + Sets the value of the config setting of key to be newval + + Once any key/value pair is changed via the set method, + that key/value pair will remain set with that value until + change via the update or set method + """ + self._config_overrides[key] = value + self._combine_dictionaries() + + def set(self, key, newval): + return self.__setitem__(key, newval) + + def update(self, new_conf): + """ + Load an XML file and apply its map as overrides or additions + to the existing config. Update can be a file or a dict. + + Once any key/value pair is changed via the update method, + that key/value pair will remain set with that value until + change via the update or set method + """ + if isinstance(new_conf, dict): + overrides = new_conf + else: + # assuming that it is a filename + config_file = open(new_conf) + overrides = llsd.parse(config_file.read()) + config_file.close() + + self._config_overrides.update(overrides) + self._combine_dictionaries() + + def as_dict(self): + """ + Returns immutable copy of the IndraConfig as a dictionary + """ + return copy.deepcopy(self._combined_dict) + +def load(indra_xml_file = None): + global _g_config + + if indra_xml_file is None: + ## going from: + ## "/opt/linden/indra/lib/python/indra/base/config.py" + ## to: + ## "/opt/linden/etc/indra.xml" + indra_xml_file = realpath( + dirname(realpath(__file__)) + "../../../../../../etc/indra.xml") + + _g_config = IndraConfig(indra_xml_file) + +def dump(indra_xml_file, indra_cfg = None, update_in_mem=False): ''' Dump config contents into a file Kindof reverse of load. Optionally takes a new config to dump. Does NOT update global config unless requested. ''' - global _g_config_dict + global _g_config + if not indra_cfg: - indra_cfg = _g_config_dict + if _g_config is None: + return + + indra_cfg = _g_config.as_dict() + if not indra_cfg: return + config_file = open(indra_xml_file, 'w') _config_xml = llsd.format_xml(indra_cfg) config_file.write(_config_xml) config_file.close() + if update_in_mem: update(indra_cfg) def update(new_conf): - """Load an XML file and apply its map as overrides or additions - to the existing config. The dataserver does this with indra.xml - and dataserver.xml.""" - global _g_config_dict - if _g_config_dict == None: - _g_config_dict = {} - if isinstance(new_conf, dict): - overrides = new_conf - else: - config_file = file(new_conf) - overrides = llsd.LLSD().parse(config_file.read()) - config_file.close() - - _g_config_dict.update(overrides) + global _g_config + + if _g_config is None: + # To keep with how this function behaved + # previously, a call to update + # before the global is defined + # make a new global config which does not + # load data from a file. + _g_config = IndraConfig(None) + + return _g_config.update(new_conf) def get(key, default = None): - global _g_config_dict - if _g_config_dict == None: + global _g_config + + if _g_config is None: load() - return _g_config_dict.get(key, default) + + return _g_config.get(key, default) def set(key, newval): - global _g_config_dict - if _g_config_dict == None: - load() - _g_config_dict[key] = newval + """ + Sets the value of the config setting of key to be newval + + Once any key/value pair is changed via the set method, + that key/value pair will remain set with that value until + change via the update or set method or program termination + """ + global _g_config + + if _g_config is None: + _g_config = IndraConfig(None) + + _g_config.set(key, newval) -def as_dict(): - global _g_config_dict - return _g_config_dict +def get_config(): + global _g_config + return _g_config diff --git a/indra/lib/python/indra/base/llsd.py b/indra/lib/python/indra/base/llsd.py index 9e636ea423..0faf4df57f 100644 --- a/indra/lib/python/indra/base/llsd.py +++ b/indra/lib/python/indra/base/llsd.py @@ -33,14 +33,7 @@ import time import types import re -#from cElementTree import fromstring ## This does not work under Windows -try: - ## This is the old name of elementtree, for use with 2.3 - from elementtree.ElementTree import fromstring -except ImportError: - ## This is the name of elementtree under python 2.5 - from xml.etree.ElementTree import fromstring - +from indra.util.fastest_elementtree import fromstring from indra.base import lluuid int_regex = re.compile("[-+]?\d+") @@ -67,6 +60,39 @@ BOOL_TRUE = ('1', '1.0', 'true') BOOL_FALSE = ('0', '0.0', 'false', '') +def format_datestr(v): + """ Formats a datetime object into the string format shared by xml and notation serializations.""" + second_str = "" + if v.microsecond > 0: + seconds = v.second + float(v.microsecond) / 1000000 + second_str = "%05.2f" % seconds + else: + second_str = "%d" % v.second + return '%s%sZ' % (v.strftime('%Y-%m-%dT%H:%M:'), second_str) + + +def parse_datestr(datestr): + """Parses a datetime object from the string format shared by xml and notation serializations.""" + if datestr == "": + return datetime.datetime(1970, 1, 1) + + match = re.match(date_regex, datestr) + if not match: + raise LLSDParseError("invalid date string '%s'." % datestr) + + year = int(match.group('year')) + month = int(match.group('month')) + day = int(match.group('day')) + hour = int(match.group('hour')) + minute = int(match.group('minute')) + second = int(match.group('second')) + seconds_float = match.group('second_float') + microsecond = 0 + if seconds_float: + microsecond = int(seconds_float[1:]) * 10000 + return datetime.datetime(year, month, day, hour, minute, second, microsecond) + + def bool_to_python(node): val = node.text or '' if val in BOOL_TRUE: @@ -99,8 +125,7 @@ def date_to_python(node): val = node.text or '' if not val: val = "1970-01-01T00:00:00Z" - return datetime.datetime( - *time.strptime(val, '%Y-%m-%dT%H:%M:%SZ')[:6]) + return parse_datestr(val) def uri_to_python(node): val = node.text or '' @@ -194,7 +219,7 @@ class LLSDXMLFormatter(object): def URI(self, v): return self.elt('uri', self.xml_esc(str(v))) def DATE(self, v): - return self.elt('date', v.strftime('%Y-%m-%dT%H:%M:%SZ')) + return self.elt('date', format_datestr(v)) def ARRAY(self, v): return self.elt('array', ''.join([self.generate(item) for item in v])) def MAP(self, v): @@ -261,13 +286,7 @@ class LLSDNotationFormatter(object): def URI(self, v): return 'l"%s"' % str(v).replace("\\", "\\\\").replace('"', '\\"') def DATE(self, v): - second_str = "" - if v.microsecond > 0: - seconds = v.second + float(v.microsecond) / 1000000 - second_str = "%05.2f" % seconds - else: - second_str = "%d" % v.second - return 'd"%s%sZ"' % (v.strftime('%Y-%m-%dT%H:%M:'), second_str) + return 'd"%s"' % format_datestr(v) def ARRAY(self, v): return "[%s]" % ','.join([self.generate(item) for item in v]) def MAP(self, v): @@ -476,10 +495,11 @@ class LLSDNotationParser(object): integer: i#### real: r#### uuid: u#### - string: "g'day" | 'have a "nice" day' | s(size)"raw data" + string: "g\'day" | 'have a "nice" day' | s(size)"raw data" uri: l"escaped" date: d"YYYY-MM-DDTHH:MM:SS.FFZ" - binary: b##"ff3120ab1" | b(size)"raw data" """ + binary: b##"ff3120ab1" | b(size)"raw data" + """ def __init__(self): pass @@ -542,7 +562,6 @@ class LLSDNotationParser(object): elif cc == 'b': raise LLSDParseError("binary notation not yet supported") else: - print cc raise LLSDParseError("invalid token at index %d: %d" % ( self._index - 1, ord(cc))) @@ -623,25 +642,7 @@ class LLSDNotationParser(object): delim = self._buffer[self._index] self._index += 1 datestr = self._parse_string(delim) - - if datestr == "": - return datetime.datetime(1970, 1, 1) - - match = re.match(date_regex, datestr) - if not match: - raise LLSDParseError("invalid date string '%s'." % datestr) - - year = int(match.group('year')) - month = int(match.group('month')) - day = int(match.group('day')) - hour = int(match.group('hour')) - minute = int(match.group('minute')) - second = int(match.group('second')) - seconds_float = match.group('second_float') - microsecond = 0 - if seconds_float: - microsecond = int(seconds_float[1:]) * 10000 - return datetime.datetime(year, month, day, hour, minute, second, microsecond) + return parse_datestr(datestr) def _parse_real(self): match = re.match(real_regex, self._buffer[self._index:]) @@ -666,7 +667,7 @@ class LLSDNotationParser(object): return int( self._buffer[start:end] ) def _parse_string(self, delim): - """ string: "g'day" | 'have a "nice" day' | s(size)"raw data" """ + """ string: "g\'day" | 'have a "nice" day' | s(size)"raw data" """ rv = "" if delim in ("'", '"'): @@ -835,22 +836,17 @@ class LLSD(object): undef = LLSD(None) -# register converters for stacked, if stacked is available +# register converters for llsd in mulib, if it is available try: - from mulib import stacked + from mulib import stacked, mu stacked.NoProducer() # just to exercise stacked + mu.safe_load(None) # just to exercise mu except: - print "Couldn't import mulib.stacked, not registering LLSD converters" + # mulib not available, don't print an error message since this is normal + pass else: - def llsd_convert_json(llsd_stuff, request): - callback = request.get_header('callback') - if callback is not None: - ## See Yahoo's ajax documentation for information about using this - ## callback style of programming - ## http://developer.yahoo.com/common/json.html#callbackparam - req.write("%s(%s)" % (callback, simplejson.dumps(llsd_stuff))) - else: - req.write(simplejson.dumps(llsd_stuff)) + mu.add_parser(parse, 'application/llsd+xml') + mu.add_parser(parse, 'application/llsd+binary') def llsd_convert_xml(llsd_stuff, request): request.write(format_xml(llsd_stuff)) @@ -859,8 +855,6 @@ else: request.write(format_binary(llsd_stuff)) for typ in [LLSD, dict, list, tuple, str, int, float, bool, unicode, type(None)]: - stacked.add_producer(typ, llsd_convert_json, 'application/json') - stacked.add_producer(typ, llsd_convert_xml, 'application/llsd+xml') stacked.add_producer(typ, llsd_convert_xml, 'application/xml') stacked.add_producer(typ, llsd_convert_xml, 'text/xml') diff --git a/indra/lib/python/indra/base/lluuid.py b/indra/lib/python/indra/base/lluuid.py index 019ccfc215..bd6c4320f3 100644 --- a/indra/lib/python/indra/base/lluuid.py +++ b/indra/lib/python/indra/base/lluuid.py @@ -74,21 +74,29 @@ class UUID(object): hexip = ''.join(["%04x" % long(i) for i in ip.split('.')]) lastid = '' - def __init__(self, string_with_uuid=None): + def __init__(self, possible_uuid=None): """ - Initialize to first valid UUID in string argument, - or to null UUID if none found or string is not supplied. + Initialize to first valid UUID in argument (if a string), + or to null UUID if none found or argument is not supplied. + + If the argument is a UUID, the constructed object will be a copy of it. """ self._bits = "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0" - if string_with_uuid: - uuid_match = UUID.uuid_regex.search(string_with_uuid) - if uuid_match: - uuid_string = uuid_match.group() - s = string.replace(uuid_string, '-', '') - self._bits = _int2binstr(string.atol(s[:8],16),4) + \ - _int2binstr(string.atol(s[8:16],16),4) + \ - _int2binstr(string.atol(s[16:24],16),4) + \ - _int2binstr(string.atol(s[24:],16),4) + if possible_uuid is None: + return + + if isinstance(possible_uuid, type(self)): + self.set(possible_uuid) + return + + uuid_match = UUID.uuid_regex.search(possible_uuid) + if uuid_match: + uuid_string = uuid_match.group() + s = string.replace(uuid_string, '-', '') + self._bits = _int2binstr(string.atol(s[:8],16),4) + \ + _int2binstr(string.atol(s[8:16],16),4) + \ + _int2binstr(string.atol(s[16:24],16),4) + \ + _int2binstr(string.atol(s[24:],16),4) def __len__(self): """ diff --git a/indra/lib/python/indra/ipc/llsdhttp.py b/indra/lib/python/indra/ipc/llsdhttp.py index 0561cfd520..12d759d3a0 100644 --- a/indra/lib/python/indra/ipc/llsdhttp.py +++ b/indra/lib/python/indra/ipc/llsdhttp.py @@ -60,21 +60,22 @@ def postFile(url, filename): return post_(url, llsd_body) +# deprecated in favor of get_ def getStatus(url, use_proxy=False): status, _headers, _body = get_(url, use_proxy=use_proxy) return status - +# deprecated in favor of put_ def putStatus(url, data): status, _headers, _body = put_(url, data) return status - +# deprecated in favor of delete_ def deleteStatus(url): status, _headers, _body = delete_(url) return status - +# deprecated in favor of post_ def postStatus(url, data): status, _headers, _body = post_(url, data) return status diff --git a/indra/lib/python/indra/ipc/mysql_pool.py b/indra/lib/python/indra/ipc/mysql_pool.py index 827b6d42e9..2a5a916e74 100644 --- a/indra/lib/python/indra/ipc/mysql_pool.py +++ b/indra/lib/python/indra/ipc/mysql_pool.py @@ -1,6 +1,6 @@ """\ @file mysql_pool.py -@brief Uses saranwrap to implement a pool of nonblocking database connections to a mysql server. +@brief Thin wrapper around eventlet.db_pool that chooses MySQLdb and Tpool. $LicenseInfo:firstyear=2007&license=mit$ @@ -26,44 +26,14 @@ THE SOFTWARE. $/LicenseInfo$ """ -import os - -from eventlet.pools import Pool -from eventlet.processes import DeadProcess -from indra.ipc import saranwrap - import MySQLdb +from eventlet import db_pool -# method 2: better -- admits the existence of the pool -# dbp = my_db_connector.get() -# dbh = dbp.get() -# dbc = dbh.cursor() -# dbc.execute(named_query) -# dbc.close() -# dbp.put(dbh) - -class DatabaseConnector(object): - """\ -@brief This is an object which will maintain a collection of database -connection pools keyed on host,databasename""" +class DatabaseConnector(db_pool.DatabaseConnector): def __init__(self, credentials, min_size = 0, max_size = 4, *args, **kwargs): - """\ - @brief constructor - @param min_size the minimum size of a child pool. - @param max_size the maximum size of a child pool.""" - self._min_size = min_size - self._max_size = max_size - self._args = args - self._kwargs = kwargs - self._credentials = credentials # this is a map of hostname to username/password - self._databases = {} - - def credentials_for(self, host): - if host in self._credentials: - return self._credentials[host] - else: - return self._credentials.get('default', None) + super(DatabaseConnector, self).__init__(MySQLdb, credentials, min_size, max_size, conn_pool=db_pool.ConnectionPool, *args, **kwargs) + # get is extended relative to eventlet.db_pool to accept a port argument def get(self, host, dbname, port=3306): key = (host, dbname, port) if key not in self._databases: @@ -77,28 +47,32 @@ connection pools keyed on host,databasename""" return self._databases[key] - -class ConnectionPool(Pool): +class ConnectionPool(db_pool.TpooledConnectionPool): """A pool which gives out saranwrapped MySQLdb connections from a pool """ - def __init__(self, min_size = 0, max_size = 4, *args, **kwargs): - self._args = args - self._kwargs = kwargs - Pool.__init__(self, min_size, max_size) - def create(self): - return saranwrap.wrap(MySQLdb).connect(*self._args, **self._kwargs) - - def put(self, conn): - # rollback any uncommitted changes, so that the next process - # has a clean slate. This also pokes the process to see if - # it's dead or None - try: - conn.rollback() - except (AttributeError, DeadProcess), e: - conn = self.create() - # TODO figure out if we're still connected to the database - if conn is not None: - Pool.put(self, conn) - else: - self.current_size -= 1 + def __init__(self, min_size = 0, max_size = 4, *args, **kwargs): + super(ConnectionPool, self).__init__(MySQLdb, min_size, max_size, *args, **kwargs) + + def get(self): + conn = super(ConnectionPool, self).get() + # annotate the connection object with the details on the + # connection; this is used elsewhere to check that you haven't + # suddenly changed databases in midstream while making a + # series of queries on a connection. + arg_names = ['host','user','passwd','db','port','unix_socket','conv','connect_timeout', + 'compress', 'named_pipe', 'init_command', 'read_default_file', 'read_default_group', + 'cursorclass', 'use_unicode', 'charset', 'sql_mode', 'client_flag', 'ssl', + 'local_infile'] + # you could have constructed this connectionpool with a mix of + # keyword and non-keyword arguments, but we want to annotate + # the connection object with a dict so it's easy to check + # against so here we are converting the list of non-keyword + # arguments (in self._args) into a dict of keyword arguments, + # and merging that with the actual keyword arguments + # (self._kwargs). The arg_names variable lists the + # constructor arguments for MySQLdb Connection objects. + converted_kwargs = dict([ (arg_names[i], arg) for i, arg in enumerate(self._args) ]) + converted_kwargs.update(self._kwargs) + conn.connection_parameters = converted_kwargs + return conn diff --git a/indra/lib/python/indra/util/fastest_elementtree.py b/indra/lib/python/indra/util/fastest_elementtree.py new file mode 100644 index 0000000000..6661580463 --- /dev/null +++ b/indra/lib/python/indra/util/fastest_elementtree.py @@ -0,0 +1,52 @@ +"""\ +@file fastest_elementtree.py +@brief Concealing some gnarly import logic in here. This should export the interface of elementtree. + +$LicenseInfo:firstyear=2006&license=mit$ + +Copyright (c) 2006-2007, Linden Research, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +$/LicenseInfo$ +""" + +# Using celementree might cause some unforeseen problems so here's a +# convenient off switch. + +# *NOTE: turned off cause of problems. :-( *TODO: debug +use_celementree = False + +try: + if not use_celementree: + raise ImportError() + from cElementTree import * ## This does not work under Windows +except ImportError: + try: + if not use_celementree: + raise ImportError() + ## This is the name of cElementTree under python 2.5 + from xml.etree.cElementTree import * + except ImportError: + try: + ## This is the old name of elementtree, for use with 2.3 + from elementtree.ElementTree import * + except ImportError: + ## This is the name of elementtree under python 2.5 + from xml.etree.ElementTree import * + diff --git a/indra/lib/python/indra/util/named_query.py b/indra/lib/python/indra/util/named_query.py index 17f25f46d2..063ef7932e 100644 --- a/indra/lib/python/indra/util/named_query.py +++ b/indra/lib/python/indra/util/named_query.py @@ -60,6 +60,11 @@ def _init_g_named_manager(sql_dir = None): because it's tricky to control the config from inside a test.""" if sql_dir is None: sql_dir = config.get('named-query-base-dir') + + # extra fallback directory in case config doesn't return what we want + if sql_dir is None: + sql_dir = os.path.dirname(__file__) + "../../../../web/dataservice/sql" + global _g_named_manager _g_named_manager = NamedQueryManager( os.path.abspath(os.path.realpath(sql_dir))) |