CHOP-661: add and use code to listen on next available server port.

In indra/llmessage/tests/testrunner.py, introduce new freeport() function to
try a caller-specified expression (such as instantiating an object that will
listen on a server port) with a range of candidate port numbers until the
expression produces a value instead of EADDRINUSE exception.
Change test_llsdmessage_peer.py and test_llxmlrpc_peer.py to use freeport() to
construct their server class inline BEFORE launching the thread that will run
it, then pass that server's serve_forever method to daemon thread. Also set
os.environ["PORT"] to selected environment variable before running subject
test program.
In indra/llmessage/tests/commtest.h, introduce commtest_data::getport() to
read port number from specified environment variable, throwing exception if
variable not set or non-numeric. Construct default LLHost from getport("PORT")
instead of hardcoded constant.
Change indra/newview/tests/llxmlrpclistener_test.cpp to use commtest_data::
getport("PORT") instead of hardcoded constant. Also use LLSD::with() rather
than older LLSD::insert() syntax.
HOWEVER -- I am irritated to discover that llxmlrpclistener_test IS NOT RUN or
even built by newview/CMakeLists.txt! It's not even commented out -- it's
entirely deleted! I am determined to restore this test. However, as it will
take some fiddling with new link-time dependencies, that will be a separate
commit.

5 files changed, 138 insertions, 21 deletions

diff --git a/indra/llmessage/tests/commtest.h b/indra/llmessage/tests/commtest.h
index 32035783e2..0fef596df2 100644
--- a/indra/llmessage/tests/commtest.h
+++ b/indra/llmessage/tests/commtest.h
@@ -35,6 +35,13 @@
 #include "llhost.h"
 #include "stringize.h"
 #include <string>
+#include <stdexcept>
+#include <boost/lexical_cast.hpp>
+
+struct CommtestError: public std::runtime_error
+{
+    CommtestError(const std::string& what): std::runtime_error(what) {}
+};
 
 /**
  * This struct is shared by a couple of standalone comm tests (ADD_COMM_BUILD_TEST).
@@ -55,13 +62,24 @@ struct commtest_data
         replyPump("reply"),
         errorPump("error"),
         success(false),
-        host("127.0.0.1", 8000),
+        host("127.0.0.1", getport("PORT")),
         server(STRINGIZE("http://" << host.getString() << "/"))
     {
         replyPump.listen("self", boost::bind(&commtest_data::outcome, this, _1, true));
         errorPump.listen("self", boost::bind(&commtest_data::outcome, this, _1, false));
     }
 
+    static int getport(const std::string& var)
+    {
+        const char* port = getenv(var.c_str());
+        if (! port)
+        {
+            throw CommtestError("missing $PORT environment variable");
+        }
+        // This will throw, too, if the value of PORT isn't numeric.
+        return boost::lexical_cast<int>(port);
+    }
+
     bool outcome(const LLSD& _result, bool _success)
     {
 //      std::cout << "commtest_data::outcome(" << _result << ", " << _success << ")\n";
diff --git a/indra/llmessage/tests/test_llsdmessage_peer.py b/indra/llmessage/tests/test_llsdmessage_peer.py
index 580ee7f8b4..cea5032111 100644
--- a/indra/llmessage/tests/test_llsdmessage_peer.py
+++ b/indra/llmessage/tests/test_llsdmessage_peer.py
@@ -38,7 +38,7 @@ mydir = os.path.dirname(__file__)       # expected to be .../indra/llmessage/tes
 sys.path.insert(0, os.path.join(mydir, os.pardir, os.pardir, "lib", "python"))
 from indra.util.fastest_elementtree import parse as xml_parse
 from indra.base import llsd
-from testrunner import run, debug
+from testrunner import freeport, run, debug
 
 class TestHTTPRequestHandler(BaseHTTPRequestHandler):
     """This subclass of BaseHTTPRequestHandler is to receive and echo
@@ -97,6 +97,10 @@ class TestHTTPRequestHandler(BaseHTTPRequestHandler):
             self.wfile.write(response)
         else:                           # fail requested
             status = data.get("status", 500)
+            # self.responses maps an int status to a (short, long) pair of
+            # strings. We want the longer string. That's why we pass a string
+            # pair to get(): the [1] will select the second string, whether it
+            # came from self.responses or from our default pair.
             reason = data.get("reason",
                                self.responses.get(status,
                                                   ("fail requested",
@@ -113,11 +117,17 @@ class TestHTTPRequestHandler(BaseHTTPRequestHandler):
         # Suppress error output as well
         pass
 
-class TestHTTPServer(Thread):
-    def run(self):
-        httpd = HTTPServer(('127.0.0.1', 8000), TestHTTPRequestHandler)
-        debug("Starting HTTP server...\n")
-        httpd.serve_forever()
-
 if __name__ == "__main__":
-    sys.exit(run(server=TestHTTPServer(name="httpd"), *sys.argv[1:]))
+    # Instantiate an HTTPServer(TestHTTPRequestHandler) on the first free port
+    # in the specified port range. Doing this inline is better than in a
+    # daemon thread: if it blows up here, we'll get a traceback. If it blew up
+    # in some other thread, the traceback would get eaten and we'd run the
+    # subject test program anyway.
+    httpd, port = freeport(xrange(8000, 8020),
+                           lambda port: HTTPServer(('127.0.0.1', port), TestHTTPRequestHandler))
+    # Pass the selected port number to the subject test program via the
+    # environment. We don't want to impose requirements on the test program's
+    # command-line parsing -- and anyway, for C++ integration tests, that's
+    # performed in TUT code rather than our own.
+    os.environ["PORT"] = str(port)
+    sys.exit(run(server=Thread(name="httpd", target=httpd.serve_forever), *sys.argv[1:]))
diff --git a/indra/llmessage/tests/testrunner.py b/indra/llmessage/tests/testrunner.py
index b70ce91ee7..8ff13e0426 100644
--- a/indra/llmessage/tests/testrunner.py
+++ b/indra/llmessage/tests/testrunner.py
@@ -29,6 +29,8 @@ $/LicenseInfo$
 
 import os
 import sys
+import errno
+import socket
 
 def debug(*args):
     sys.stdout.writelines(args)
@@ -36,6 +38,85 @@ def debug(*args):
 # comment out the line below to enable debug output
 debug = lambda *args: None
 
+def freeport(portlist, expr):
+    """
+    Find a free server port to use. Specifically, evaluate 'expr' (a
+    callable(port)) until it stops raising EADDRINUSE exception.
+
+    Pass:
+
+    portlist: an iterable (e.g. xrange()) of ports to try. If you exhaust the
+    range, freeport() lets the socket.error exception propagate. If you want
+    unbounded, you could pass itertools.count(baseport), though of course in
+    practice the ceiling is 2^16-1 anyway. But it seems prudent to constrain
+    the range much more sharply: if we're iterating an absurd number of times,
+    probably something else is wrong.
+
+    expr: a callable accepting a port number, specifically one of the items
+    from portlist. If calling that callable raises socket.error with
+    EADDRINUSE, freeport() retrieves the next item from portlist and retries.
+
+    Returns: (expr(port), port)
+
+    port: the value from portlist for which expr(port) succeeded
+
+    Raises:
+
+    Any exception raised by expr(port) other than EADDRINUSE.
+
+    socket.error if, for every item from portlist, expr(port) raises
+    socket.error. The exception you see is the one from the last item in
+    portlist.
+
+    StopIteration if portlist is completely empty.
+
+    Example:
+
+    server, port = freeport(xrange(8000, 8010),
+                            lambda port: HTTPServer(("localhost", port),
+                                                    MyRequestHandler))
+    # pass 'port' to client code
+    # call server.serve_forever()
+    """
+    # If portlist is completely empty, let StopIteration propagate: that's an
+    # error because we can't return meaningful values. We have no 'port',
+    # therefore no 'expr(port)'.
+    portiter = iter(portlist)
+    port = portiter.next()
+
+    while True:
+        try:
+            # If this value of port works, return as promised.
+            return expr(port), port
+
+        except socket.error, err:
+            # Anything other than 'Address already in use', propagate
+            if err.args[0] != errno.EADDRINUSE:
+                raise
+
+            # Here we want the next port from portiter. But on StopIteration,
+            # we want to raise the original exception rather than
+            # StopIteration. So save the original exc_info().
+            type, value, tb = sys.exc_info()
+            try:
+                try:
+                    port = portiter.next()
+                except StopIteration:
+                    raise type, value, tb
+            finally:
+                # Clean up local traceback, see docs for sys.exc_info()
+                del tb
+
+        # Recap of the control flow above:
+        # If expr(port) doesn't raise, return as promised.
+        # If expr(port) raises anything but EADDRINUSE, propagate that
+        # exception.
+        # If portiter.next() raises StopIteration -- that is, if the port
+        # value we just passed to expr(port) was the last available -- reraise
+        # the EADDRINUSE exception.
+        # If we've actually arrived at this point, portiter.next() delivered a
+        # new port value. Loop back to pass that to expr(port).
+
 def run(*args, **kwds):
     """All positional arguments collectively form a command line, executed as
     a synchronous child process.
diff --git a/indra/newview/tests/llxmlrpclistener_test.cpp b/indra/newview/tests/llxmlrpclistener_test.cpp
index 4d5df1043e..711c2a3d51 100644
--- a/indra/newview/tests/llxmlrpclistener_test.cpp
+++ b/indra/newview/tests/llxmlrpclistener_test.cpp
@@ -40,8 +40,10 @@
 #include "llevents.h"
 #include "lleventfilter.h"
 #include "llsd.h"
+#include "llhost.h"
 #include "llcontrol.h"
 #include "tests/wrapllerrs.h"
+#include "tests/commtest.h"
 
 LLControlGroup gSavedSettings("Global");
 
@@ -54,7 +56,8 @@ namespace tut
     {
         data():
             pumps(LLEventPumps::instance()),
-            uri("http://127.0.0.1:8000")
+            uri(std::string("http://") +
+                LLHost("127.0.0.1", commtest_data::getport("PORT")).getString())
         {
             // These variables are required by machinery used by
             // LLXMLRPCTransaction. The values reflect reality for this test
@@ -145,7 +148,7 @@ namespace tut
         pumps.obtain("LLXMLRPCTransaction").post(request);
         // Set the timer
         F32 timeout(10);
-        watchdog.eventAfter(timeout, LLSD().insert("timeout", 0));
+        watchdog.eventAfter(timeout, LLSD().with("timeout", 0));
         // and pump "mainloop" until we get something, whether from
         // LLXMLRPCListener or from the watchdog filter.
         LLTimer timer;
@@ -182,7 +185,7 @@ namespace tut
         pumps.obtain("LLXMLRPCTransaction").post(request);
         // Set the timer
         F32 timeout(10);
-        watchdog.eventAfter(timeout, LLSD().insert("timeout", 0));
+        watchdog.eventAfter(timeout, LLSD().with("timeout", 0));
         // and pump "mainloop" until we get something, whether from
         // LLXMLRPCListener or from the watchdog filter.
         LLTimer timer;
@@ -218,7 +221,7 @@ namespace tut
         pumps.obtain("LLXMLRPCTransaction").post(request);
         // Set the timer
         F32 timeout(10);
-        watchdog.eventAfter(timeout, LLSD().insert("timeout", 0));
+        watchdog.eventAfter(timeout, LLSD().with("timeout", 0));
         // and pump "mainloop" until we get something, whether from
         // LLXMLRPCListener or from the watchdog filter.
         LLTimer timer;
diff --git a/indra/newview/tests/test_llxmlrpc_peer.py b/indra/newview/tests/test_llxmlrpc_peer.py
index 1c7204a6b6..281b72a058 100644
--- a/indra/newview/tests/test_llxmlrpc_peer.py
+++ b/indra/newview/tests/test_llxmlrpc_peer.py
@@ -37,7 +37,7 @@ from SimpleXMLRPCServer import SimpleXMLRPCServer
 mydir = os.path.dirname(__file__)       # expected to be .../indra/newview/tests/
 sys.path.insert(0, os.path.join(mydir, os.pardir, os.pardir, "lib", "python"))
 sys.path.insert(1, os.path.join(mydir, os.pardir, os.pardir, "llmessage", "tests"))
-from testrunner import run, debug
+from testrunner import freeport, run, debug
 
 class TestServer(SimpleXMLRPCServer):
     def _dispatch(self, method, params):
@@ -66,11 +66,16 @@ class TestServer(SimpleXMLRPCServer):
         # Suppress error output as well
         pass
 
-class ServerRunner(Thread):
-    def run(self):
-        server = TestServer(('127.0.0.1', 8000))
-        debug("Starting XMLRPC server...\n")
-        server.serve_forever()
-
 if __name__ == "__main__":
-    sys.exit(run(server=ServerRunner(name="xmlrpc"), *sys.argv[1:]))
+    # Instantiate a TestServer on the first free port in the specified port
+    # range. Doing this inline is better than in a daemon thread: if it blows
+    # up here, we'll get a traceback. If it blew up in some other thread, the
+    # traceback would get eaten and we'd run the subject test program anyway.
+    xmlrpcd, port = freeport(xrange(8000, 8020),
+                             lambda port: TestServer(('127.0.0.1', port)))
+    # Pass the selected port number to the subject test program via the
+    # environment. We don't want to impose requirements on the test program's
+    # command-line parsing -- and anyway, for C++ integration tests, that's
+    # performed in TUT code rather than our own.
+    os.environ["PORT"] = str(port)
+    sys.exit(run(server=Thread(name="xmlrpc", target=xmlrpcd.serve_forever), *sys.argv[1:]))