From 987d14fd466a645cf32dff7bdea699b34325196c Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Fri, 20 Jul 2007 21:49:31 +0000 Subject: SL-49406: template_verifier.py requires network access to build. Implemented a 4-hour cache so it hits the network less often overall (saving approx .5 seconds every build), and added fault tolerance for when it fails to fetch the master over the network. It'll whine at you if it can't fetch it, but even if it can't get the master from the network and doesn't have a cached copy at all, it will still at least do a syntax check on the local template. --- scripts/template_verifier.py | 82 +++++++++++++++++++++++++++++++++++++------- 1 file changed, 70 insertions(+), 12 deletions(-) (limited to 'scripts/template_verifier.py') diff --git a/scripts/template_verifier.py b/scripts/template_verifier.py index 67728d73d8..6fbb787f6f 100755 --- a/scripts/template_verifier.py +++ b/scripts/template_verifier.py @@ -54,6 +54,7 @@ DEVELOPMENT_ACCEPTABLE = ( compatibility.Same, compatibility.Newer, compatibility.Older, compatibility.Mixed) +MAX_MASTER_AGE = 60 * 60 * 4 # refresh master cache every 4 hours def compare(base, current, mode): """Compare the current template against the base template using the given @@ -68,6 +69,8 @@ def compare(base, current, mode): Returns a tuple of (bool, Compatibility) Return True if they are compatible in this mode, False if not. """ + + # catch this exception so we can print a message explaining which template is scr0d try: base = llmessage.parseTemplateString(base) except tokenstream.ParseError, e: @@ -90,12 +93,45 @@ def compare(base, current, mode): return True, compat return False, compat +def fetch(url): + # *FIX: this doesn't throw an exception for a 404, and oddly enough the sl.com 404 page actually gets parsed successfully + return ''.join(urllib.urlopen(url).readlines()) + +def cache_master(master_url): + """Using the url for the master, updates the local cache, and returns an url to the local cache.""" + master_cache = local_master_cache_filename() + master_cache_url = 'file://' + master_cache + # decide whether to refresh the master cache based on its age + import time + if (os.path.exists(master_cache) + and time.time() - os.path.getmtime(master_cache) < MAX_MASTER_AGE): + return master_cache_url # our cache is fresh + # new master doesn't exist or isn't fresh + print "Refreshing master cache from %s" % master_url + try: + new_master_contents = fetch(master_url) + except IOError, e: + # the refresh failed, so we should just soldier on + print "WARNING: unable to download new master, probably due to network error. Your message template compatibility may be suspect." + return master_cache_url + mc = open(master_cache, 'wb') + mc.write(new_master_contents) + mc.close() + return master_cache_url + def local_template_filename(): """Returns the message template's default location relative to template_verifier.py: ./messages/message_template.msg.""" d = os.path.dirname(os.path.realpath(__file__)) return os.path.join(d, 'messages', MESSAGE_TEMPLATE) +def local_master_cache_filename(): + """Returns the location of the master template cache relative to template_verifier.py + ./messages/master_message_template_cache.msg""" + d = os.path.dirname(os.path.realpath(__file__)) + return os.path.join(d, 'messages', 'master_message_template_cache.msg') + + def run(sysargs): parser = optparse.OptionParser( usage="usage: %prog [FILE] [FILE]", @@ -112,43 +148,65 @@ http://wiki.secondlife.com/wiki/Template_verifier.py '-u', '--master_url', type='string', dest='master_url', default='http://secondlife.com/app/message_template/master_message_template.msg', help="""The url of the master message template.""") + parser.add_option( + '-c', '--cache_master', action='store_true', dest='cache_master', + default=False, help="""Set to true to attempt use local cached copy of the master template.""") options, args = parser.parse_args(sysargs) + if options.mode == 'production': + options.cache_master = False + # both current and master supplied in positional params if len(args) == 2: master_filename, current_filename = args print "base:", master_filename print "current:", current_filename - master = file(master_filename).read() - current = file(current_filename).read() + master_url = 'file://%s' % master_filename + current_url = 'file://%s' % current_filename # only current supplied in positional param elif len(args) == 1: - master = None + master_url = None current_filename = args[0] print "base: " print "current:", current_filename - current = file(current_filename).read() + current_url = 'file://%s' % current_filename # nothing specified, use defaults for everything elif len(args) == 0: - master = None - current = None + master_url = None + current_url = None else: die("Too many arguments") - # fetch the master from the url (default or supplied) - if master is None: - master = ''.join(urllib.urlopen(options.master_url).readlines()) - + if master_url is None: + master_url = options.master_url + # fetch the template for this build - if current is None: + if current_url is None: current_filename = local_template_filename() print "base: " print "current:", current_filename - current = file(current_filename).read() + current_url = 'file://%s' % current_filename + + if options.cache_master: + # optionally return a url to a locally-cached master so we don't hit the network all the time + master_url = cache_master(master_url) + + current = fetch(current_url) + try: + master = fetch(master_url) + except IOError, e: + if options.mode == 'production': + raise e + else: + print "WARNING: problems fetching the master from %s. Syntax-checking the local template ONLY, no compatibility check is being run." % master_url + llmessage.parseTemplateString(current) + return 0 + acceptable, compat = compare( master, current, options.mode) + def explain(header, compat): print header -- cgit v1.2.3