Merge from viewer-development

5 files changed, 313 insertions, 28 deletions

diff --git a/indra/llmessage/tests/commtest.h b/indra/llmessage/tests/commtest.h
index 32035783e2..0d149b5258 100644
--- a/indra/llmessage/tests/commtest.h
+++ b/indra/llmessage/tests/commtest.h
@@ -34,7 +34,67 @@
 #include "llsd.h"
 #include "llhost.h"
 #include "stringize.h"
+#include <map>
 #include <string>
+#include <stdexcept>
+#include <boost/lexical_cast.hpp>
+
+struct CommtestError: public std::runtime_error
+{
+    CommtestError(const std::string& what): std::runtime_error(what) {}
+};
+
+static bool query_verbose()
+{
+    const char* cbose = getenv("INTEGRATION_TEST_VERBOSE");
+    if (! cbose)
+    {
+        cbose = "1";
+    }
+    std::string strbose(cbose);
+    return (! (strbose == "0" || strbose == "off" ||
+               strbose == "false" || strbose == "quiet"));
+}
+
+bool verbose()
+{
+    // This should only be initialized once.
+    static bool vflag = query_verbose();
+    return vflag;
+}
+
+static int query_port(const std::string& var)
+{
+    const char* cport = getenv(var.c_str());
+    if (! cport)
+    {
+        throw CommtestError(STRINGIZE("missing environment variable" << var));
+    }
+    // This will throw, too, if the value of PORT isn't numeric.
+    int port(boost::lexical_cast<int>(cport));
+    if (verbose())
+    {
+        std::cout << "getport('" << var << "') = " << port << std::endl;
+    }
+    return port;
+}
+
+static int getport(const std::string& var)
+{
+    typedef std::map<std::string, int> portsmap;
+    static portsmap ports;
+    // We can do this with a single map lookup with map::insert(). Either it
+    // returns an existing entry and 'false' (not newly inserted), or it
+    // inserts the specified value and 'true'.
+    std::pair<portsmap::iterator, bool> inserted(ports.insert(portsmap::value_type(var, 0)));
+    if (inserted.second)
+    {
+        // We haven't yet seen this var. Remember its value.
+        inserted.first->second = query_port(var);
+    }
+    // Return the (existing or new) iterator's value.
+    return inserted.first->second;
+}
 
 /**
  * This struct is shared by a couple of standalone comm tests (ADD_COMM_BUILD_TEST).
@@ -55,13 +115,21 @@ struct commtest_data
         replyPump("reply"),
         errorPump("error"),
         success(false),
-        host("127.0.0.1", 8000),
+        host("127.0.0.1", getport("PORT")),
         server(STRINGIZE("http://" << host.getString() << "/"))
     {
         replyPump.listen("self", boost::bind(&commtest_data::outcome, this, _1, true));
         errorPump.listen("self", boost::bind(&commtest_data::outcome, this, _1, false));
     }
 
+    static int getport(const std::string& var)
+    {
+        // We have a couple consumers of commtest_data::getport(). But we've
+        // since moved it out to the global namespace. So this is just a
+        // facade.
+        return ::getport(var);
+    }
+
     bool outcome(const LLSD& _result, bool _success)
     {
 //      std::cout << "commtest_data::outcome(" << _result << ", " << _success << ")\n";
diff --git a/indra/llmessage/tests/llhost_test.cpp b/indra/llmessage/tests/llhost_test.cpp
index b20bceae1d..eadf83c428 100644
--- a/indra/llmessage/tests/llhost_test.cpp
+++ b/indra/llmessage/tests/llhost_test.cpp
@@ -151,8 +151,9 @@ namespace tut
 	template<> template<>
 	void host_object::test<9>()
 	{
+		skip("this test is flaky, but we should figure out why...");
 //		skip("setHostByName(\"google.com\"); getHostName() -> (e.g.) \"yx-in-f100.1e100.net\"");
-		std::string hostStr = "linux.org";		
+		std::string hostStr = "lindenlab.com";		
 		LLHost host;
 		host.setHostByName(hostStr);	
 
diff --git a/indra/llmessage/tests/llsdmessage_test.cpp b/indra/llmessage/tests/llsdmessage_test.cpp
index 9998a1b8bb..0f2c069303 100644
--- a/indra/llmessage/tests/llsdmessage_test.cpp
+++ b/indra/llmessage/tests/llsdmessage_test.cpp
@@ -61,6 +61,7 @@ namespace tut
         llsdmessage_data():
             httpPump(pumps.obtain("LLHTTPClient"))
         {
+            LLCurl::initClass();
             LLSDMessage::link();
         }
     };
diff --git a/indra/llmessage/tests/test_llsdmessage_peer.py b/indra/llmessage/tests/test_llsdmessage_peer.py
index 580ee7f8b4..22edd9dad8 100644
--- a/indra/llmessage/tests/test_llsdmessage_peer.py
+++ b/indra/llmessage/tests/test_llsdmessage_peer.py
@@ -38,7 +38,7 @@ mydir = os.path.dirname(__file__)       # expected to be .../indra/llmessage/tes
 sys.path.insert(0, os.path.join(mydir, os.pardir, os.pardir, "lib", "python"))
 from indra.util.fastest_elementtree import parse as xml_parse
 from indra.base import llsd
-from testrunner import run, debug
+from testrunner import freeport, run, debug, VERBOSE
 
 class TestHTTPRequestHandler(BaseHTTPRequestHandler):
     """This subclass of BaseHTTPRequestHandler is to receive and echo
@@ -72,10 +72,10 @@ class TestHTTPRequestHandler(BaseHTTPRequestHandler):
 ##         # assuming that the underlying XML parser reads its input file
 ##         # incrementally. Unfortunately I haven't been able to make it work.
 ##         tree = xml_parse(self.rfile)
-##         debug("Finished raw parse\n")
-##         debug("parsed XML tree %s\n" % tree)
-##         debug("parsed root node %s\n" % tree.getroot())
-##         debug("root node tag %s\n" % tree.getroot().tag)
+##         debug("Finished raw parse")
+##         debug("parsed XML tree %s", tree)
+##         debug("parsed root node %s", tree.getroot())
+##         debug("root node tag %s", tree.getroot().tag)
 ##         return llsd.to_python(tree.getroot())
 
     def do_GET(self):
@@ -88,8 +88,10 @@ class TestHTTPRequestHandler(BaseHTTPRequestHandler):
         self.answer(self.read_xml())
 
     def answer(self, data):
+        debug("%s.answer(%s): self.path = %r", self.__class__.__name__, data, self.path)
         if "fail" not in self.path:
             response = llsd.format_xml(data.get("reply", llsd.LLSD("success")))
+            debug("success: %s", response)
             self.send_response(200)
             self.send_header("Content-type", "application/llsd+xml")
             self.send_header("Content-Length", str(len(response)))
@@ -97,27 +99,48 @@ class TestHTTPRequestHandler(BaseHTTPRequestHandler):
             self.wfile.write(response)
         else:                           # fail requested
             status = data.get("status", 500)
+            # self.responses maps an int status to a (short, long) pair of
+            # strings. We want the longer string. That's why we pass a string
+            # pair to get(): the [1] will select the second string, whether it
+            # came from self.responses or from our default pair.
             reason = data.get("reason",
                                self.responses.get(status,
                                                   ("fail requested",
                                                    "Your request specified failure status %s "
                                                    "without providing a reason" % status))[1])
+            debug("fail requested: %s: %r", status, reason)
             self.send_error(status, reason)
 
-    def log_request(self, code, size=None):
-        # For present purposes, we don't want the request splattered onto
-        # stderr, as it would upset devs watching the test run
-        pass
+    if not VERBOSE:
+        # When VERBOSE is set, skip both these overrides because they exist to
+        # suppress output.
 
-    def log_error(self, format, *args):
-        # Suppress error output as well
-        pass
+        def log_request(self, code, size=None):
+            # For present purposes, we don't want the request splattered onto
+            # stderr, as it would upset devs watching the test run
+            pass
 
-class TestHTTPServer(Thread):
-    def run(self):
-        httpd = HTTPServer(('127.0.0.1', 8000), TestHTTPRequestHandler)
-        debug("Starting HTTP server...\n")
-        httpd.serve_forever()
+        def log_error(self, format, *args):
+            # Suppress error output as well
+            pass
+
+class Server(HTTPServer):
+    # This pernicious flag is on by default in HTTPServer. But proper
+    # operation of freeport() absolutely depends on it being off.
+    allow_reuse_address = False
 
 if __name__ == "__main__":
-    sys.exit(run(server=TestHTTPServer(name="httpd"), *sys.argv[1:]))
+    # Instantiate a Server(TestHTTPRequestHandler) on the first free port
+    # in the specified port range. Doing this inline is better than in a
+    # daemon thread: if it blows up here, we'll get a traceback. If it blew up
+    # in some other thread, the traceback would get eaten and we'd run the
+    # subject test program anyway.
+    httpd, port = freeport(xrange(8000, 8020),
+                           lambda port: Server(('127.0.0.1', port), TestHTTPRequestHandler))
+    # Pass the selected port number to the subject test program via the
+    # environment. We don't want to impose requirements on the test program's
+    # command-line parsing -- and anyway, for C++ integration tests, that's
+    # performed in TUT code rather than our own.
+    os.environ["PORT"] = str(port)
+    debug("$PORT = %s", port)
+    sys.exit(run(server=Thread(name="httpd", target=httpd.serve_forever), *sys.argv[1:]))
diff --git a/indra/llmessage/tests/testrunner.py b/indra/llmessage/tests/testrunner.py
index b70ce91ee7..f2c841532a 100644
--- a/indra/llmessage/tests/testrunner.py
+++ b/indra/llmessage/tests/testrunner.py
@@ -27,14 +27,118 @@ Linden Research, Inc., 945 Battery Street, San Francisco, CA  94111  USA
 $/LicenseInfo$
 """
 
+from __future__ import with_statement
+
 import os
 import sys
+import re
+import errno
+import socket
+
+VERBOSE = os.environ.get("INTEGRATION_TEST_VERBOSE", "1") # default to verbose
+# Support usage such as INTEGRATION_TEST_VERBOSE=off -- distressing to user if
+# that construct actually turns on verbosity...
+VERBOSE = not re.match(r"(0|off|false|quiet)$", VERBOSE, re.IGNORECASE)
+
+if VERBOSE:
+    def debug(fmt, *args):
+        print fmt % args
+        sys.stdout.flush()
+else:
+    debug = lambda *args: None
+
+def freeport(portlist, expr):
+    """
+    Find a free server port to use. Specifically, evaluate 'expr' (a
+    callable(port)) until it stops raising EADDRINUSE exception.
+
+    Pass:
+
+    portlist: an iterable (e.g. xrange()) of ports to try. If you exhaust the
+    range, freeport() lets the socket.error exception propagate. If you want
+    unbounded, you could pass itertools.count(baseport), though of course in
+    practice the ceiling is 2^16-1 anyway. But it seems prudent to constrain
+    the range much more sharply: if we're iterating an absurd number of times,
+    probably something else is wrong.
+
+    expr: a callable accepting a port number, specifically one of the items
+    from portlist. If calling that callable raises socket.error with
+    EADDRINUSE, freeport() retrieves the next item from portlist and retries.
+
+    Returns: (expr(port), port)
+
+    port: the value from portlist for which expr(port) succeeded
+
+    Raises:
+
+    Any exception raised by expr(port) other than EADDRINUSE.
+
+    socket.error if, for every item from portlist, expr(port) raises
+    socket.error. The exception you see is the one from the last item in
+    portlist.
+
+    StopIteration if portlist is completely empty.
+
+    Example:
+
+    class Server(HTTPServer):
+        # If you use BaseHTTPServer.HTTPServer, turning off this flag is
+        # essential for proper operation of freeport()!
+        allow_reuse_address = False
+    # ...
+    server, port = freeport(xrange(8000, 8010),
+                            lambda port: Server(("localhost", port),
+                                                MyRequestHandler))
+    # pass 'port' to client code
+    # call server.serve_forever()
+    """
+    try:
+        # If portlist is completely empty, let StopIteration propagate: that's an
+        # error because we can't return meaningful values. We have no 'port',
+        # therefore no 'expr(port)'.
+        portiter = iter(portlist)
+        port = portiter.next()
+
+        while True:
+            try:
+                # If this value of port works, return as promised.
+                value = expr(port)
+
+            except socket.error, err:
+                # Anything other than 'Address already in use', propagate
+                if err.args[0] != errno.EADDRINUSE:
+                    raise
+
+                # Here we want the next port from portiter. But on StopIteration,
+                # we want to raise the original exception rather than
+                # StopIteration. So save the original exc_info().
+                type, value, tb = sys.exc_info()
+                try:
+                    try:
+                        port = portiter.next()
+                    except StopIteration:
+                        raise type, value, tb
+                finally:
+                    # Clean up local traceback, see docs for sys.exc_info()
+                    del tb
+
+            else:
+                debug("freeport() returning %s on port %s", value, port)
+                return value, port
 
-def debug(*args):
-    sys.stdout.writelines(args)
-    sys.stdout.flush()
-# comment out the line below to enable debug output
-debug = lambda *args: None
+            # Recap of the control flow above:
+            # If expr(port) doesn't raise, return as promised.
+            # If expr(port) raises anything but EADDRINUSE, propagate that
+            # exception.
+            # If portiter.next() raises StopIteration -- that is, if the port
+            # value we just passed to expr(port) was the last available -- reraise
+            # the EADDRINUSE exception.
+            # If we've actually arrived at this point, portiter.next() delivered a
+            # new port value. Loop back to pass that to expr(port).
+
+    except Exception, err:
+        debug("*** freeport() raising %s: %s", err.__class__.__name__, err)
+        raise
 
 def run(*args, **kwds):
     """All positional arguments collectively form a command line, executed as
@@ -63,8 +167,96 @@ def run(*args, **kwds):
     # - [no p] don't use the PATH because we specifically want to invoke the
     #   executable passed as our first arg,
     # - [no e] child should inherit this process's environment.
-    debug("Running %s...\n" % (" ".join(args)))
-    sys.stdout.flush()
+    debug("Running %s...", " ".join(args))
     rc = os.spawnv(os.P_WAIT, args[0], args)
-    debug("%s returned %s\n" % (args[0], rc))
+    debug("%s returned %s", args[0], rc)
     return rc
+
+# ****************************************************************************
+#   test code -- manual at this point, see SWAT-564
+# ****************************************************************************
+def test_freeport():
+    # ------------------------------- Helpers --------------------------------
+    from contextlib import contextmanager
+    # helper Context Manager for expecting an exception
+    # with exc(SomeError):
+    #     raise SomeError()
+    # raises AssertionError otherwise.
+    @contextmanager
+    def exc(exception_class, *args):
+        try:
+            yield
+        except exception_class, err:
+            for i, expected_arg in enumerate(args):
+                assert expected_arg == err.args[i], \
+                       "Raised %s, but args[%s] is %r instead of %r" % \
+                       (err.__class__.__name__, i, err.args[i], expected_arg)
+            print "Caught expected exception %s(%s)" % \
+                  (err.__class__.__name__, ', '.join(repr(arg) for arg in err.args))
+        else:
+            assert False, "Failed to raise " + exception_class.__class__.__name__
+
+    # helper to raise specified exception
+    def raiser(exception):
+        raise exception
+
+    # the usual
+    def assert_equals(a, b):
+        assert a == b, "%r != %r" % (a, b)
+
+    # ------------------------ Sanity check the above ------------------------
+    class SomeError(Exception): pass
+    # Without extra args, accept any err.args value
+    with exc(SomeError):
+        raiser(SomeError("abc"))
+    # With extra args, accept only the specified value
+    with exc(SomeError, "abc"):
+        raiser(SomeError("abc"))
+    with exc(AssertionError):
+        with exc(SomeError, "abc"):
+            raiser(SomeError("def"))
+    with exc(AssertionError):
+        with exc(socket.error, errno.EADDRINUSE):
+            raiser(socket.error(errno.ECONNREFUSED, 'Connection refused'))
+
+    # ----------- freeport() without engaging socket functionality -----------
+    # If portlist is empty, freeport() raises StopIteration.
+    with exc(StopIteration):
+        freeport([], None)
+
+    assert_equals(freeport([17], str), ("17", 17))
+
+    # This is the magic exception that should prompt us to retry
+    inuse = socket.error(errno.EADDRINUSE, 'Address already in use')
+    # Get the iterator to our ports list so we can check later if we've used all
+    ports = iter(xrange(5))
+    with exc(socket.error, errno.EADDRINUSE):
+        freeport(ports, lambda port: raiser(inuse))
+    # did we entirely exhaust 'ports'?
+    with exc(StopIteration):
+        ports.next()
+
+    ports = iter(xrange(2))
+    # Any exception but EADDRINUSE should quit immediately
+    with exc(SomeError):
+        freeport(ports, lambda port: raiser(SomeError()))
+    assert_equals(ports.next(), 1)
+
+    # ----------- freeport() with platform-dependent socket stuff ------------
+    # This is what we should've had unit tests to begin with (see CHOP-661).
+    def newbind(port):
+        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        sock.bind(('127.0.0.1', port))
+        return sock
+
+    bound0, port0 = freeport(xrange(7777, 7780), newbind)
+    assert_equals(port0, 7777)
+    bound1, port1 = freeport(xrange(7777, 7780), newbind)
+    assert_equals(port1, 7778)
+    bound2, port2 = freeport(xrange(7777, 7780), newbind)
+    assert_equals(port2, 7779)
+    with exc(socket.error, errno.EADDRINUSE):
+        bound3, port3 = freeport(xrange(7777, 7780), newbind)
+
+if __name__ == "__main__":
+    test_freeport()