SL-15742: Convert build scripts to Python 3

This changeset makes it possible to build the Second Life viewer using
Python 3. It is designed to be used with an equivalent Autobuild branch
so that a developer can compile without needing Python 2 on their
machine.

Breaking change: Python 2 support ending

Rather than supporting two versions of Python, including one that was
discontinued at the beginning of the year, this branch focuses on
pouring future effort into Python 3 only. As a result, scripts do not
need to be backwards compatible. This means that build environments,
be they on personal computers and on build agents, need to have a
compatible interpreter.

Notes

- SLVersionChecker will still use Python 2 on macOS
- Fixed the message template url used by template_verifier.py

1 files changed, 39 insertions, 36 deletions

diff --git a/scripts/template_verifier.py b/scripts/template_verifier.py
index b44410cdd8..0f5135fae6 100755
--- a/scripts/template_verifier.py
+++ b/scripts/template_verifier.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 """\
 @file template_verifier.py
 @brief Message template compatibility verifier.
@@ -58,14 +58,14 @@ def add_indra_lib_path():
                 sys.path.insert(0, dir)
             break
     else:
-        print >>sys.stderr, "This script is not inside a valid installation."
+        print("This script is not inside a valid installation.", file=sys.stderr)
         sys.exit(1)
 
 add_indra_lib_path()
 
 import optparse
 import os
-import urllib
+import urllib.request, urllib.parse, urllib.error
 import hashlib
 
 from indra.ipc import compatibility
@@ -90,7 +90,7 @@ def getstatusoutput(command):
 
 
 def die(msg):
-    print >>sys.stderr, msg
+    print(msg, file=sys.stderr)
     sys.exit(1)
 
 MESSAGE_TEMPLATE = 'message_template.msg'
@@ -106,7 +106,7 @@ def retry(times, function, *args, **kwargs):
     for i in range(times):
         try:
             return function(*args, **kwargs)
-        except Exception, e:
+        except Exception as e:
             if i == times - 1:
                 raise e  # we retried all the times we could
 
@@ -138,10 +138,14 @@ def fetch(url):
     if url.startswith('file://'):
         # just open the file directly because urllib is dumb about these things
         file_name = url[len('file://'):]
-        return open(file_name).read()
+        with open(file_name, 'rb') as f:
+            return f.read()
     else:
-        # *FIX: this doesn't throw an exception for a 404, and oddly enough the sl.com 404 page actually gets parsed successfully
-        return ''.join(urllib.urlopen(url).readlines())   
+        with urllib.request.urlopen(url) as res:
+            body = res.read()
+            if res.status > 299:
+                sys.exit("ERROR: Unable to download %s. HTTP status %d.\n%s" % (url, res.status, body.decode("utf-8")))
+            return body
 
 def cache_master(master_url):
     """Using the url for the master, updates the local cache, and returns an url to the local cache."""
@@ -153,23 +157,22 @@ def cache_master(master_url):
         and time.time() - os.path.getmtime(master_cache) < MAX_MASTER_AGE):
         return master_cache_url  # our cache is fresh
     # new master doesn't exist or isn't fresh
-    print "Refreshing master cache from %s" % master_url
+    print("Refreshing master cache from %s" % master_url)
     def get_and_test_master():
         new_master_contents = fetch(master_url)
-        llmessage.parseTemplateString(new_master_contents)
+        llmessage.parseTemplateString(new_master_contents.decode("utf-8"))
         return new_master_contents
     try:
         new_master_contents = retry(3, get_and_test_master)
-    except IOError, e:
+    except IOError as e:
         # the refresh failed, so we should just soldier on
-        print "WARNING: unable to download new master, probably due to network error.  Your message template compatibility may be suspect."
-        print "Cause: %s" % e
+        print("WARNING: unable to download new master, probably due to network error.  Your message template compatibility may be suspect.")
+        print("Cause: %s" % e)
         return master_cache_url
     try:
         tmpname = '%s.%d' % (master_cache, os.getpid())
-        mc = open(tmpname, 'wb')
-        mc.write(new_master_contents)
-        mc.close()
+        with open(tmpname, "wb") as mc:
+            mc.write(new_master_contents)
         try:
             os.rename(tmpname, master_cache)
         except OSError:
@@ -180,9 +183,9 @@ def cache_master(master_url):
             # a single day.
             os.unlink(master_cache)
             os.rename(tmpname, master_cache)
-    except IOError, e:
-        print "WARNING: Unable to write master message template to %s, proceeding without cache." % master_cache
-        print "Cause: %s" % e
+    except IOError as e:
+        print("WARNING: Unable to write master message template to %s, proceeding without cache." % master_cache)
+        print("Cause: %s" % e)
         return master_url
     return master_cache_url
 
@@ -229,7 +232,7 @@ http://wiki.secondlife.com/wiki/Template_verifier.py
 """)
     parser.add_option(
         '-u', '--master_url', type='string', dest='master_url',
-        default='http://bitbucket.org/lindenlab/master-message-template/raw/tip/message_template.msg',
+        default='https://bitbucket.org/lindenlab/master-message-template-git/raw/master/message_template.msg',
         help="""The url of the master message template.""")
     parser.add_option(
         '-c', '--cache_master', action='store_true', dest='cache_master',
@@ -246,16 +249,16 @@ http://wiki.secondlife.com/wiki/Template_verifier.py
     # both current and master supplied in positional params
     if len(args) == 2:
         master_filename, current_filename = args
-        print "master:", master_filename
-        print "current:", current_filename
+        print("master:", master_filename)
+        print("current:", current_filename)
         master_url = 'file://%s' % master_filename
         current_url = 'file://%s' % current_filename
     # only current supplied in positional param
     elif len(args) == 1:
         master_url = None
         current_filename = args[0]
-        print "master:", options.master_url 
-        print "current:", current_filename
+        print("master:", options.master_url) 
+        print("current:", current_filename)
         current_url = 'file://%s' % current_filename
     # nothing specified, use defaults for everything
     elif len(args) == 0:
@@ -269,8 +272,8 @@ http://wiki.secondlife.com/wiki/Template_verifier.py
         
     if current_url is None:
         current_filename = local_template_filename()
-        print "master:", options.master_url
-        print "current:", current_filename
+        print("master:", options.master_url)
+        print("current:", current_filename)
         current_url = 'file://%s' % current_filename
 
     # retrieve the contents of the local template
@@ -281,42 +284,42 @@ http://wiki.secondlife.com/wiki/Template_verifier.py
         sha_url = "%s.sha1" % current_url
         current_sha = fetch(sha_url)
         if hexdigest == current_sha:
-            print "Message template SHA_1 has not changed."
+            print("Message template SHA_1 has not changed.")
             sys.exit(0)
 
     # and check for syntax
-    current_parsed = llmessage.parseTemplateString(current)
+    current_parsed = llmessage.parseTemplateString(current.decode("utf-8"))
 
     if options.cache_master:
         # optionally return a url to a locally-cached master so we don't hit the network all the time
         master_url = cache_master(master_url)
 
     def parse_master_url():
-        master = fetch(master_url)
+        master = fetch(master_url).decode("utf-8")
         return llmessage.parseTemplateString(master)
     try:
         master_parsed = retry(3, parse_master_url)
-    except (IOError, tokenstream.ParseError), e:
+    except (IOError, tokenstream.ParseError) as e:
         if options.mode == 'production':
             raise e
         else:
-            print "WARNING: problems retrieving the master from %s."  % master_url
-            print "Syntax-checking the local template ONLY, no compatibility check is being run."
-            print "Cause: %s\n\n" % e
+            print("WARNING: problems retrieving the master from %s."  % master_url)
+            print("Syntax-checking the local template ONLY, no compatibility check is being run.")
+            print("Cause: %s\n\n" % e)
             return 0
         
     acceptable, compat = compare(
         master_parsed, current_parsed, options.mode)
 
     def explain(header, compat):
-        print header
+        print(header)
         # indent compatibility explanation
-        print '\n\t'.join(compat.explain().split('\n'))
+        print('\n\t'.join(compat.explain().split('\n')))
 
     if acceptable:
         explain("--- PASS ---", compat)
         if options.force_verification == False:
-            print "Updating sha1 to %s" % hexdigest
+            print("Updating sha1 to %s" % hexdigest)
             sha_filename = "%s.sha1" % current_filename
             sha_file = open(sha_filename, 'w')
             sha_file.write(hexdigest)