From 73a7b14013059eee3b010ae271515d7492654f9b Mon Sep 17 00:00:00 2001 From: Nat Goodspeed Date: Thu, 17 Nov 2016 17:46:50 -0500 Subject: DRTVWR-418: Fold redundant testrunner.py modules together again. llcorehttp/tests had a clone of llmessage/tests/testrunner.py that was almost identical save for recognizing an extra optional parameter. Migrate those few lines into llmessage/tests/testrunner.py; eliminate the copy in llcorehttp; help test_llcorehttp_peer.py find the testrunner.py in llmessage/tests. --- indra/llcorehttp/tests/test_llcorehttp_peer.py | 5 + indra/llcorehttp/tests/testrunner.py | 265 ------------------------- 2 files changed, 5 insertions(+), 265 deletions(-) delete mode 100755 indra/llcorehttp/tests/testrunner.py (limited to 'indra/llcorehttp/tests') diff --git a/indra/llcorehttp/tests/test_llcorehttp_peer.py b/indra/llcorehttp/tests/test_llcorehttp_peer.py index 6c5f37d407..3ec9cd7d4c 100755 --- a/indra/llcorehttp/tests/test_llcorehttp_peer.py +++ b/indra/llcorehttp/tests/test_llcorehttp_peer.py @@ -44,6 +44,11 @@ from SocketServer import ThreadingMixIn from llbase.fastest_elementtree import parse as xml_parse from llbase import llsd + +# we're in llcorehttp/tests ; testrunner.py is found in llmessage/tests +sys.path.append(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, + "llmessage", "tests")) + from testrunner import freeport, run, debug, VERBOSE class TestHTTPRequestHandler(BaseHTTPRequestHandler): diff --git a/indra/llcorehttp/tests/testrunner.py b/indra/llcorehttp/tests/testrunner.py deleted file mode 100755 index 9a2de71142..0000000000 --- a/indra/llcorehttp/tests/testrunner.py +++ /dev/null @@ -1,265 +0,0 @@ -#!/usr/bin/env python -"""\ -@file testrunner.py -@author Nat Goodspeed -@date 2009-03-20 -@brief Utilities for writing wrapper scripts for ADD_COMM_BUILD_TEST unit tests - -$LicenseInfo:firstyear=2009&license=viewerlgpl$ -Second Life Viewer Source Code -Copyright (C) 2010, Linden Research, Inc. - -This library is free software; you can redistribute it and/or -modify it under the terms of the GNU Lesser General Public -License as published by the Free Software Foundation; -version 2.1 of the License only. - -This library is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -Lesser General Public License for more details. - -You should have received a copy of the GNU Lesser General Public -License along with this library; if not, write to the Free Software -Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA -$/LicenseInfo$ -""" - -from __future__ import with_statement - -import os -import sys -import re -import errno -import socket - -VERBOSE = os.environ.get("INTEGRATION_TEST_VERBOSE", "0") # default to quiet -# Support usage such as INTEGRATION_TEST_VERBOSE=off -- distressing to user if -# that construct actually turns on verbosity... -VERBOSE = not re.match(r"(0|off|false|quiet)$", VERBOSE, re.IGNORECASE) - -if VERBOSE: - def debug(fmt, *args): - print fmt % args - sys.stdout.flush() -else: - debug = lambda *args: None - -def freeport(portlist, expr): - """ - Find a free server port to use. Specifically, evaluate 'expr' (a - callable(port)) until it stops raising EADDRINUSE exception. - - Pass: - - portlist: an iterable (e.g. xrange()) of ports to try. If you exhaust the - range, freeport() lets the socket.error exception propagate. If you want - unbounded, you could pass itertools.count(baseport), though of course in - practice the ceiling is 2^16-1 anyway. But it seems prudent to constrain - the range much more sharply: if we're iterating an absurd number of times, - probably something else is wrong. - - expr: a callable accepting a port number, specifically one of the items - from portlist. If calling that callable raises socket.error with - EADDRINUSE, freeport() retrieves the next item from portlist and retries. - - Returns: (expr(port), port) - - port: the value from portlist for which expr(port) succeeded - - Raises: - - Any exception raised by expr(port) other than EADDRINUSE. - - socket.error if, for every item from portlist, expr(port) raises - socket.error. The exception you see is the one from the last item in - portlist. - - StopIteration if portlist is completely empty. - - Example: - - class Server(HTTPServer): - # If you use BaseHTTPServer.HTTPServer, turning off this flag is - # essential for proper operation of freeport()! - allow_reuse_address = False - # ... - server, port = freeport(xrange(8000, 8010), - lambda port: Server(("localhost", port), - MyRequestHandler)) - # pass 'port' to client code - # call server.serve_forever() - """ - try: - # If portlist is completely empty, let StopIteration propagate: that's an - # error because we can't return meaningful values. We have no 'port', - # therefore no 'expr(port)'. - portiter = iter(portlist) - port = portiter.next() - - while True: - try: - # If this value of port works, return as promised. - value = expr(port) - - except socket.error, err: - # Anything other than 'Address already in use', propagate - if err.args[0] != errno.EADDRINUSE: - raise - - # Here we want the next port from portiter. But on StopIteration, - # we want to raise the original exception rather than - # StopIteration. So save the original exc_info(). - type, value, tb = sys.exc_info() - try: - try: - port = portiter.next() - except StopIteration: - raise type, value, tb - finally: - # Clean up local traceback, see docs for sys.exc_info() - del tb - - else: - debug("freeport() returning %s on port %s", value, port) - return value, port - - # Recap of the control flow above: - # If expr(port) doesn't raise, return as promised. - # If expr(port) raises anything but EADDRINUSE, propagate that - # exception. - # If portiter.next() raises StopIteration -- that is, if the port - # value we just passed to expr(port) was the last available -- reraise - # the EADDRINUSE exception. - # If we've actually arrived at this point, portiter.next() delivered a - # new port value. Loop back to pass that to expr(port). - - except Exception, err: - debug("*** freeport() raising %s: %s", err.__class__.__name__, err) - raise - -def run(*args, **kwds): - """All positional arguments collectively form a command line, executed as - a synchronous child process. - In addition, pass server=new_thread_instance as an explicit keyword (to - differentiate it from an additional command-line argument). - new_thread_instance should be an instantiated but not yet started Thread - subclass instance, e.g.: - run("python", "-c", 'print "Hello, world!"', server=TestHTTPServer(name="httpd")) - """ - # If there's no server= keyword arg, don't start a server thread: simply - # run a child process. - try: - thread = kwds.pop("server") - except KeyError: - pass - else: - # Start server thread. Note that this and all other comm server - # threads should be daemon threads: we'll let them run "forever," - # confident that the whole process will terminate when the main thread - # terminates, which will be when the child process terminates. - thread.setDaemon(True) - thread.start() - # choice of os.spawnv(): - # - [v vs. l] pass a list of args vs. individual arguments, - # - [no p] don't use the PATH because we specifically want to invoke the - # executable passed as our first arg, - # - [no e] child should inherit this process's environment. - debug("Running %s...", " ".join(args)) - if kwds.get("use_path", False): - rc = os.spawnvp(os.P_WAIT, args[0], args) - else: - rc = os.spawnv(os.P_WAIT, args[0], args) - debug("%s returned %s", args[0], rc) - return rc - -# **************************************************************************** -# test code -- manual at this point, see SWAT-564 -# **************************************************************************** -def test_freeport(): - # ------------------------------- Helpers -------------------------------- - from contextlib import contextmanager - # helper Context Manager for expecting an exception - # with exc(SomeError): - # raise SomeError() - # raises AssertionError otherwise. - @contextmanager - def exc(exception_class, *args): - try: - yield - except exception_class, err: - for i, expected_arg in enumerate(args): - assert expected_arg == err.args[i], \ - "Raised %s, but args[%s] is %r instead of %r" % \ - (err.__class__.__name__, i, err.args[i], expected_arg) - print "Caught expected exception %s(%s)" % \ - (err.__class__.__name__, ', '.join(repr(arg) for arg in err.args)) - else: - assert False, "Failed to raise " + exception_class.__class__.__name__ - - # helper to raise specified exception - def raiser(exception): - raise exception - - # the usual - def assert_equals(a, b): - assert a == b, "%r != %r" % (a, b) - - # ------------------------ Sanity check the above ------------------------ - class SomeError(Exception): pass - # Without extra args, accept any err.args value - with exc(SomeError): - raiser(SomeError("abc")) - # With extra args, accept only the specified value - with exc(SomeError, "abc"): - raiser(SomeError("abc")) - with exc(AssertionError): - with exc(SomeError, "abc"): - raiser(SomeError("def")) - with exc(AssertionError): - with exc(socket.error, errno.EADDRINUSE): - raiser(socket.error(errno.ECONNREFUSED, 'Connection refused')) - - # ----------- freeport() without engaging socket functionality ----------- - # If portlist is empty, freeport() raises StopIteration. - with exc(StopIteration): - freeport([], None) - - assert_equals(freeport([17], str), ("17", 17)) - - # This is the magic exception that should prompt us to retry - inuse = socket.error(errno.EADDRINUSE, 'Address already in use') - # Get the iterator to our ports list so we can check later if we've used all - ports = iter(xrange(5)) - with exc(socket.error, errno.EADDRINUSE): - freeport(ports, lambda port: raiser(inuse)) - # did we entirely exhaust 'ports'? - with exc(StopIteration): - ports.next() - - ports = iter(xrange(2)) - # Any exception but EADDRINUSE should quit immediately - with exc(SomeError): - freeport(ports, lambda port: raiser(SomeError())) - assert_equals(ports.next(), 1) - - # ----------- freeport() with platform-dependent socket stuff ------------ - # This is what we should've had unit tests to begin with (see CHOP-661). - def newbind(port): - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.bind(('127.0.0.1', port)) - return sock - - bound0, port0 = freeport(xrange(7777, 7780), newbind) - assert_equals(port0, 7777) - bound1, port1 = freeport(xrange(7777, 7780), newbind) - assert_equals(port1, 7778) - bound2, port2 = freeport(xrange(7777, 7780), newbind) - assert_equals(port2, 7779) - with exc(socket.error, errno.EADDRINUSE): - bound3, port3 = freeport(xrange(7777, 7780), newbind) - -if __name__ == "__main__": - test_freeport() -- cgit v1.2.3 From e9a9e3d4bafab0e40a8ed3a65dfd4474ab7bb938 Mon Sep 17 00:00:00 2001 From: Nat Goodspeed Date: Tue, 6 Dec 2016 09:32:36 -0500 Subject: DRTVWR-418: Try for more llcorehttp tests error diagnostics. --- indra/llcorehttp/tests/test_llcorehttp_peer.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'indra/llcorehttp/tests') diff --git a/indra/llcorehttp/tests/test_llcorehttp_peer.py b/indra/llcorehttp/tests/test_llcorehttp_peer.py index 3ec9cd7d4c..cc636d8d87 100755 --- a/indra/llcorehttp/tests/test_llcorehttp_peer.py +++ b/indra/llcorehttp/tests/test_llcorehttp_peer.py @@ -284,7 +284,8 @@ class Server(ThreadingMixIn, HTTPServer): # to stderr which annoys some. Disable this override to get # default behavior which *shouldn't* cause the program to return # a failure status. - def handle_error(self, request, client_address): + if not VERBOSE: + def handle_error(self, request, client_address): print '-'*40 print 'Ignoring exception during processing of request from', print client_address -- cgit v1.2.3 From 780120dc46e6b99135bfd68dfdc05bd3e133208c Mon Sep 17 00:00:00 2001 From: Nat Goodspeed Date: Tue, 6 Dec 2016 12:19:01 -0500 Subject: DRTVWR-418: Remove ThreadingMixin from our HTTPServer subclass. It's possible that raising an exception in a worker thread -- even though we're TRYING to suppress it -- is what's causing the process to terminate with nonzero rc. --- indra/llcorehttp/tests/test_llcorehttp_peer.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) (limited to 'indra/llcorehttp/tests') diff --git a/indra/llcorehttp/tests/test_llcorehttp_peer.py b/indra/llcorehttp/tests/test_llcorehttp_peer.py index cc636d8d87..caa204b519 100755 --- a/indra/llcorehttp/tests/test_llcorehttp_peer.py +++ b/indra/llcorehttp/tests/test_llcorehttp_peer.py @@ -40,7 +40,6 @@ try: except ImportError: from StringIO import StringIO from BaseHTTPServer import HTTPServer, BaseHTTPRequestHandler -from SocketServer import ThreadingMixIn from llbase.fastest_elementtree import parse as xml_parse from llbase import llsd @@ -274,7 +273,7 @@ class TestHTTPRequestHandler(BaseHTTPRequestHandler): # Suppress error output as well pass -class Server(ThreadingMixIn, HTTPServer): +class Server(HTTPServer): # This pernicious flag is on by default in HTTPServer. But proper # operation of freeport() absolutely depends on it being off. allow_reuse_address = False @@ -284,8 +283,7 @@ class Server(ThreadingMixIn, HTTPServer): # to stderr which annoys some. Disable this override to get # default behavior which *shouldn't* cause the program to return # a failure status. - if not VERBOSE: - def handle_error(self, request, client_address): + def handle_error(self, request, client_address): print '-'*40 print 'Ignoring exception during processing of request from', print client_address -- cgit v1.2.3 From 4aae3e8eb4e6872b62e15b721ee7f6c34b80d3c8 Mon Sep 17 00:00:00 2001 From: Nat Goodspeed Date: Tue, 6 Dec 2016 16:07:05 -0500 Subject: DRTVWR-418: Try harder to ignore errors in llcorehttp's dummy server. --- indra/llcorehttp/tests/test_llcorehttp_peer.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) (limited to 'indra/llcorehttp/tests') diff --git a/indra/llcorehttp/tests/test_llcorehttp_peer.py b/indra/llcorehttp/tests/test_llcorehttp_peer.py index caa204b519..b91cd6bcb4 100755 --- a/indra/llcorehttp/tests/test_llcorehttp_peer.py +++ b/indra/llcorehttp/tests/test_llcorehttp_peer.py @@ -284,10 +284,16 @@ class Server(HTTPServer): # default behavior which *shouldn't* cause the program to return # a failure status. def handle_error(self, request, client_address): - print '-'*40 - print 'Ignoring exception during processing of request from', - print client_address - print '-'*40 + print >>sys.stderr, '-'*40 + print >>sys.stderr, 'Ignoring exception during processing of request from', client_address + print >>sys.stderr, '-'*40 + + def shutdown_request(self, *args, **kwds): + try: + # just forward to base-class method, but wrap in try/except + HTTPServer.shutdown_request(*args, **kwds) + except Exception as err: + print >>sys.stderr, "Once more ignoring: %s" % err if __name__ == "__main__": do_valgrind = False -- cgit v1.2.3 From 40b1913af318f58f2a56e4bf4049437748405033 Mon Sep 17 00:00:00 2001 From: Nat Goodspeed Date: Tue, 6 Dec 2016 16:20:43 -0500 Subject: DRTVWR-418: Fix minor error in forwarding shutdown_request() call. --- indra/llcorehttp/tests/test_llcorehttp_peer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'indra/llcorehttp/tests') diff --git a/indra/llcorehttp/tests/test_llcorehttp_peer.py b/indra/llcorehttp/tests/test_llcorehttp_peer.py index b91cd6bcb4..4dfb60bddb 100755 --- a/indra/llcorehttp/tests/test_llcorehttp_peer.py +++ b/indra/llcorehttp/tests/test_llcorehttp_peer.py @@ -291,7 +291,7 @@ class Server(HTTPServer): def shutdown_request(self, *args, **kwds): try: # just forward to base-class method, but wrap in try/except - HTTPServer.shutdown_request(*args, **kwds) + HTTPServer.shutdown_request(self, *args, **kwds) except Exception as err: print >>sys.stderr, "Once more ignoring: %s" % err -- cgit v1.2.3 From 50a3f19f1a0421419155cab099b63e3436a24c7f Mon Sep 17 00:00:00 2001 From: Nat Goodspeed Date: Wed, 7 Dec 2016 22:49:32 -0500 Subject: DRTVWR-418: Overriding shutdown_request() wasn't the issue. Remove. --- indra/llcorehttp/tests/test_llcorehttp_peer.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) (limited to 'indra/llcorehttp/tests') diff --git a/indra/llcorehttp/tests/test_llcorehttp_peer.py b/indra/llcorehttp/tests/test_llcorehttp_peer.py index b2af8a6c9c..493143641b 100755 --- a/indra/llcorehttp/tests/test_llcorehttp_peer.py +++ b/indra/llcorehttp/tests/test_llcorehttp_peer.py @@ -283,16 +283,10 @@ class Server(HTTPServer): # default behavior which *shouldn't* cause the program to return # a failure status. def handle_error(self, request, client_address): - print >>sys.stderr, '-'*40 - print >>sys.stderr, 'Ignoring exception during processing of request from', client_address - print >>sys.stderr, '-'*40 - - def shutdown_request(self, *args, **kwds): - try: - # just forward to base-class method, but wrap in try/except - HTTPServer.shutdown_request(self, *args, **kwds) - except Exception as err: - print >>sys.stderr, "Once more ignoring: %s" % err + print '-'*40 + print 'Ignoring exception during processing of request from', + print client_address + print '-'*40 if __name__ == "__main__": do_valgrind = False -- cgit v1.2.3 From a4b0159d5710200b6284cce545f35c73505537e1 Mon Sep 17 00:00:00 2001 From: Nat Goodspeed Date: Fri, 16 Dec 2016 19:05:59 -0500 Subject: DRTVWR-418: Adjust HttpStatus::toHex() test for 64-bit result. --- indra/llcorehttp/tests/test_httpstatus.hpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'indra/llcorehttp/tests') diff --git a/indra/llcorehttp/tests/test_httpstatus.hpp b/indra/llcorehttp/tests/test_httpstatus.hpp index 4502d32fe1..10cda799e7 100644 --- a/indra/llcorehttp/tests/test_httpstatus.hpp +++ b/indra/llcorehttp/tests/test_httpstatus.hpp @@ -244,7 +244,11 @@ void HttpStatusTestObjectType::test<7>() HttpStatus status(404); std::string msg = status.toHex(); // std::cout << "Result: " << msg << std::endl; - ensure(msg == "01940001"); +#if ADDRESS_SIZE == 32 + ensure_equals(msg, "01940001"); +#else + ensure_equals(msg, "19400000001"); +#endif } -- cgit v1.2.3 From 40fb9d3e58fcb28778c57a835795ff4a1ef90b98 Mon Sep 17 00:00:00 2001 From: Nat Goodspeed Date: Mon, 19 Dec 2016 16:30:19 -0500 Subject: DRTVWR-418: Use U32 for int (and hex) of HttpStatus in 64-bit too. Turns out that Monty didn't intend for the int-flavored representation of HttpStatus to expand to 64 bits even when unsigned long is that wide. So change the implicit conversion operator, and its uses, to U32 instead. That produces a consistent toHex() result for both 32-bit and 64-bit builds. --- indra/llcorehttp/tests/test_httpstatus.hpp | 4 ---- 1 file changed, 4 deletions(-) (limited to 'indra/llcorehttp/tests') diff --git a/indra/llcorehttp/tests/test_httpstatus.hpp b/indra/llcorehttp/tests/test_httpstatus.hpp index 10cda799e7..cbe3f574d4 100644 --- a/indra/llcorehttp/tests/test_httpstatus.hpp +++ b/indra/llcorehttp/tests/test_httpstatus.hpp @@ -244,11 +244,7 @@ void HttpStatusTestObjectType::test<7>() HttpStatus status(404); std::string msg = status.toHex(); // std::cout << "Result: " << msg << std::endl; -#if ADDRESS_SIZE == 32 ensure_equals(msg, "01940001"); -#else - ensure_equals(msg, "19400000001"); -#endif } -- cgit v1.2.3 From a971909a3429a8944387ede557ed46e939cf0e8f Mon Sep 17 00:00:00 2001 From: Nat Goodspeed Date: Mon, 13 Feb 2017 16:07:38 -0500 Subject: DRTVWR-418: Reluctantly skip llcorehttp 503-with-retry test on W64. --- indra/llcorehttp/tests/test_httprequest.hpp | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'indra/llcorehttp/tests') diff --git a/indra/llcorehttp/tests/test_httprequest.hpp b/indra/llcorehttp/tests/test_httprequest.hpp index 6cd7960ecd..8b689e8c83 100644 --- a/indra/llcorehttp/tests/test_httprequest.hpp +++ b/indra/llcorehttp/tests/test_httprequest.hpp @@ -3089,6 +3089,10 @@ void HttpRequestTestObjectType::test<23>() set_test_name("HttpRequest GET 503s with 'Retry-After'"); +#if LL_WINDOWS && ADDRESS_SIZE == 64: + skip("llcorehttp 503-with-retry test hangs on Windows 64"); +#endif + // This tests mainly that the code doesn't fall over if // various well- and mis-formed Retry-After headers are // sent along with the response. Direct inspection of -- cgit v1.2.3 From a0c18425958f34b8c373ffc3b20b6ba710b1d8c8 Mon Sep 17 00:00:00 2001 From: Nat Goodspeed Date: Mon, 13 Feb 2017 16:53:18 -0500 Subject: DRTVWR-418: Fix syntax for previous test skip. --- indra/llcorehttp/tests/test_httprequest.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'indra/llcorehttp/tests') diff --git a/indra/llcorehttp/tests/test_httprequest.hpp b/indra/llcorehttp/tests/test_httprequest.hpp index 8b689e8c83..a9c192e141 100644 --- a/indra/llcorehttp/tests/test_httprequest.hpp +++ b/indra/llcorehttp/tests/test_httprequest.hpp @@ -3089,7 +3089,7 @@ void HttpRequestTestObjectType::test<23>() set_test_name("HttpRequest GET 503s with 'Retry-After'"); -#if LL_WINDOWS && ADDRESS_SIZE == 64: +#if LL_WINDOWS && ADDRESS_SIZE == 64 skip("llcorehttp 503-with-retry test hangs on Windows 64"); #endif -- cgit v1.2.3