diff options
| -rw-r--r-- | indra/llcorehttp/CMakeLists.txt | 2 | ||||
| -rw-r--r-- | indra/llcorehttp/tests/llcorehttp_test.cpp | 36 | ||||
| -rw-r--r-- | indra/llcorehttp/tests/llcorehttp_test.h | 5 | ||||
| -rw-r--r-- | indra/llcorehttp/tests/test_httprequest.hpp | 119 | ||||
| -rw-r--r-- | indra/llcorehttp/tests/test_llcorehttp_peer.py | 146 | ||||
| -rw-r--r-- | indra/llcorehttp/tests/testrunner.py | 262 |
6 files changed, 568 insertions, 2 deletions
diff --git a/indra/llcorehttp/CMakeLists.txt b/indra/llcorehttp/CMakeLists.txt index 3fda524ddf..a0827286e3 100644 --- a/indra/llcorehttp/CMakeLists.txt +++ b/indra/llcorehttp/CMakeLists.txt @@ -125,6 +125,8 @@ if (LL_TESTS) LL_ADD_INTEGRATION_TEST(llcorehttp "${llcorehttp_TEST_SOURCE_FILES}" "${test_libs}" + ${PYTHON_EXECUTABLE} + "${CMAKE_CURRENT_SOURCE_DIR}/tests/test_llcorehttp_peer.py" ) endif (LL_TESTS) diff --git a/indra/llcorehttp/tests/llcorehttp_test.cpp b/indra/llcorehttp/tests/llcorehttp_test.cpp index 2d48bca443..f59361ab53 100644 --- a/indra/llcorehttp/tests/llcorehttp_test.cpp +++ b/indra/llcorehttp/tests/llcorehttp_test.cpp @@ -27,6 +27,7 @@ #include "llcorehttp_test.h" #include <iostream> +#include <sstream> // These are not the right way in viewer for some reason: // #include <tut/tut.hpp> @@ -130,3 +131,38 @@ void ssl_locking_callback(int mode, int type, const char * /* file */, int /* li } +std::string get_base_url() +{ + const char * env(getenv("LL_TEST_PORT")); + + if (! env) + { + std::cerr << "LL_TEST_PORT environment variable missing." << std::endl; + std::cerr << "Test expects to run in test_llcorehttp_peer.py script." << std::endl; + tut::ensure("LL_TEST_PORT set in environment", NULL != env); + } + + int port(atoi(env)); + std::ostringstream out; + out << "http://localhost:" << port << "/"; + return out.str(); +} + + +void stop_thread(LLCore::HttpRequest * req) +{ + if (req) + { + req->requestStopThread(NULL); + + int count = 0; + int limit = 10; + while (count++ < limit && ! HttpService::isStopped()) + { + req->update(1000); + usleep(100000); + } + } +} + + diff --git a/indra/llcorehttp/tests/llcorehttp_test.h b/indra/llcorehttp/tests/llcorehttp_test.h index 1550881a00..a9567435ce 100644 --- a/indra/llcorehttp/tests/llcorehttp_test.h +++ b/indra/llcorehttp/tests/llcorehttp_test.h @@ -32,6 +32,9 @@ #include <curl/curl.h> #include <openssl/crypto.h> +#include <string> + +#include "httprequest.h" // Initialization and cleanup for libcurl. Mainly provides // a mutex callback for SSL and a thread ID hash for libcurl. @@ -40,6 +43,8 @@ // operations. extern void init_curl(); extern void term_curl(); +extern std::string get_base_url(); +extern void stop_thread(LLCore::HttpRequest * req); class ScopedCurlInit { diff --git a/indra/llcorehttp/tests/test_httprequest.hpp b/indra/llcorehttp/tests/test_httprequest.hpp index 68da9e2dc7..81f8fe4a85 100644 --- a/indra/llcorehttp/tests/test_httprequest.hpp +++ b/indra/llcorehttp/tests/test_httprequest.hpp @@ -84,8 +84,10 @@ public: if (response && mState) { const HttpStatus actual_status(response->getStatus()); - - ensure("Expected HttpStatus received in response", actual_status == mState->mStatus); + std::ostringstream test; + test << "Expected HttpStatus received in response. Wanted: " + << mState->mStatus.toHex() << " Received: " << actual_status.toHex(); + ensure(test.str().c_str(), actual_status == mState->mStatus); } if (mState) { @@ -184,6 +186,7 @@ void HttpRequestTestObjectType::test<2>() } catch (...) { + stop_thread(req); delete req; HttpRequest::destroyService(); throw; @@ -275,6 +278,7 @@ void HttpRequestTestObjectType::test<3>() } catch (...) { + stop_thread(req); delete req; HttpRequest::destroyService(); throw; @@ -377,6 +381,7 @@ void HttpRequestTestObjectType::test<4>() } catch (...) { + stop_thread(req1); delete req1; delete req2; HttpRequest::destroyService(); @@ -483,6 +488,116 @@ void HttpRequestTestObjectType::test<5>() } catch (...) { + stop_thread(req); + delete req; + HttpRequest::destroyService(); + throw; + } +} + +template <> template <> +void HttpRequestTestObjectType::test<6>() +{ + ScopedCurlInit ready; + + std::string url_base(get_base_url()); + std::cerr << "Base: " << url_base << std::endl; + + set_test_name("HttpRequest GET to real service"); + + // Handler can be stack-allocated *if* there are no dangling + // references to it after completion of this method. + // Create before memory record as the string copy will bump numbers. + TestHandler2 handler(this, "handler"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + mHandlerCalls = 0; + + HttpRequest * req = NULL; + + try + { + // Get singletons created + HttpRequest::createService(); + + // Start threading early so that thread memory is invariant + // over the test. + HttpRequest::startThread(); + + // create a new ref counted object with an implicit reference + req = new HttpRequest(); + ensure("Memory allocated on construction", mMemTotal < GetMemTotal()); + + // Issue a GET that *can* connect + mStatus = HttpStatus(200); + HttpHandle handle = req->requestGetByteRange(HttpRequest::DEFAULT_POLICY_ID, + 0U, + url_base, + 0, + 0, + NULL, + NULL, + &handler); + ensure("Valid handle returned for ranged request", handle != LLCORE_HTTP_HANDLE_INVALID); + + // Run the notification pump. + int count(0); + int limit(10); + while (count++ < limit && mHandlerCalls < 1) + { + req->update(1000); + usleep(100000); + } + ensure("Request executed in reasonable time", count < limit); + ensure("One handler invocation for request", mHandlerCalls == 1); + + // Okay, request a shutdown of the servicing thread + mStatus = HttpStatus(); + handle = req->requestStopThread(&handler); + ensure("Valid handle returned for second request", handle != LLCORE_HTTP_HANDLE_INVALID); + + // Run the notification pump again + count = 0; + limit = 10; + while (count++ < limit && mHandlerCalls < 2) + { + req->update(1000); + usleep(100000); + } + ensure("Second request executed in reasonable time", count < limit); + ensure("Second handler invocation", mHandlerCalls == 2); + + // See that we actually shutdown the thread + count = 0; + limit = 10; + while (count++ < limit && ! HttpService::isStopped()) + { + usleep(100000); + } + ensure("Thread actually stopped running", HttpService::isStopped()); + + // release the request object + delete req; + req = NULL; + + // Shut down service + HttpRequest::destroyService(); + + ensure("Two handler calls on the way out", 2 == mHandlerCalls); + +#if defined(WIN32) + // Can only do this memory test on Windows. On other platforms, + // the LL logging system holds on to memory and produces what looks + // like memory leaks... + + // printf("Old mem: %d, New mem: %d\n", mMemTotal, GetMemTotal()); + ensure("Memory usage back to that at entry", mMemTotal == GetMemTotal()); +#endif + } + catch (...) + { + stop_thread(req); delete req; HttpRequest::destroyService(); throw; diff --git a/indra/llcorehttp/tests/test_llcorehttp_peer.py b/indra/llcorehttp/tests/test_llcorehttp_peer.py new file mode 100644 index 0000000000..3e200a5c19 --- /dev/null +++ b/indra/llcorehttp/tests/test_llcorehttp_peer.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python +"""\ +@file test_llsdmessage_peer.py +@author Nat Goodspeed +@date 2008-10-09 +@brief This script asynchronously runs the executable (with args) specified on + the command line, returning its result code. While that executable is + running, we provide dummy local services for use by C++ tests. + +$LicenseInfo:firstyear=2008&license=viewerlgpl$ +Second Life Viewer Source Code +Copyright (C) 2010, Linden Research, Inc. + +This library is free software; you can redistribute it and/or +modify it under the terms of the GNU Lesser General Public +License as published by the Free Software Foundation; +version 2.1 of the License only. + +This library is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public +License along with this library; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA +$/LicenseInfo$ +""" + +import os +import sys +from threading import Thread +from BaseHTTPServer import HTTPServer, BaseHTTPRequestHandler + +mydir = os.path.dirname(__file__) # expected to be .../indra/llmessage/tests/ +sys.path.insert(0, os.path.join(mydir, os.pardir, os.pardir, "lib", "python")) +from indra.util.fastest_elementtree import parse as xml_parse +from indra.base import llsd +from testrunner import freeport, run, debug, VERBOSE + +class TestHTTPRequestHandler(BaseHTTPRequestHandler): + """This subclass of BaseHTTPRequestHandler is to receive and echo + LLSD-flavored messages sent by the C++ LLHTTPClient. + """ + def read(self): + # The following logic is adapted from the library module + # SimpleXMLRPCServer.py. + # Get arguments by reading body of request. + # We read this in chunks to avoid straining + # socket.read(); around the 10 or 15Mb mark, some platforms + # begin to have problems (bug #792570). + try: + size_remaining = int(self.headers["content-length"]) + except (KeyError, ValueError): + return "" + max_chunk_size = 10*1024*1024 + L = [] + while size_remaining: + chunk_size = min(size_remaining, max_chunk_size) + chunk = self.rfile.read(chunk_size) + L.append(chunk) + size_remaining -= len(chunk) + return ''.join(L) + # end of swiped read() logic + + def read_xml(self): + # This approach reads the entire POST data into memory first + return llsd.parse(self.read()) +## # This approach attempts to stream in the LLSD XML from self.rfile, +## # assuming that the underlying XML parser reads its input file +## # incrementally. Unfortunately I haven't been able to make it work. +## tree = xml_parse(self.rfile) +## debug("Finished raw parse") +## debug("parsed XML tree %s", tree) +## debug("parsed root node %s", tree.getroot()) +## debug("root node tag %s", tree.getroot().tag) +## return llsd.to_python(tree.getroot()) + + def do_GET(self): + # Of course, don't attempt to read data. + self.answer(dict(reply="success", status=200, + reason="Your GET operation worked")) + + def do_POST(self): + # Read the provided POST data. + self.answer(self.read()) + + def answer(self, data): + debug("%s.answer(%s): self.path = %r", self.__class__.__name__, data, self.path) + if "fail" not in self.path: + response = llsd.format_xml(data.get("reply", llsd.LLSD("success"))) + debug("success: %s", response) + self.send_response(200) + self.send_header("Content-type", "application/llsd+xml") + self.send_header("Content-Length", str(len(response))) + self.end_headers() + self.wfile.write(response) + else: # fail requested + status = data.get("status", 500) + # self.responses maps an int status to a (short, long) pair of + # strings. We want the longer string. That's why we pass a string + # pair to get(): the [1] will select the second string, whether it + # came from self.responses or from our default pair. + reason = data.get("reason", + self.responses.get(status, + ("fail requested", + "Your request specified failure status %s " + "without providing a reason" % status))[1]) + debug("fail requested: %s: %r", status, reason) + self.send_error(status, reason) + + if not VERBOSE: + # When VERBOSE is set, skip both these overrides because they exist to + # suppress output. + + def log_request(self, code, size=None): + # For present purposes, we don't want the request splattered onto + # stderr, as it would upset devs watching the test run + pass + + def log_error(self, format, *args): + # Suppress error output as well + pass + +class Server(HTTPServer): + # This pernicious flag is on by default in HTTPServer. But proper + # operation of freeport() absolutely depends on it being off. + allow_reuse_address = False + +if __name__ == "__main__": + # Instantiate a Server(TestHTTPRequestHandler) on the first free port + # in the specified port range. Doing this inline is better than in a + # daemon thread: if it blows up here, we'll get a traceback. If it blew up + # in some other thread, the traceback would get eaten and we'd run the + # subject test program anyway. + httpd, port = freeport(xrange(8000, 8020), + lambda port: Server(('127.0.0.1', port), TestHTTPRequestHandler)) + # Pass the selected port number to the subject test program via the + # environment. We don't want to impose requirements on the test program's + # command-line parsing -- and anyway, for C++ integration tests, that's + # performed in TUT code rather than our own. + os.environ["LL_TEST_PORT"] = str(port) + debug("$LL_TEST_PORT = %s", port) + sys.exit(run(server=Thread(name="httpd", target=httpd.serve_forever), *sys.argv[1:])) diff --git a/indra/llcorehttp/tests/testrunner.py b/indra/llcorehttp/tests/testrunner.py new file mode 100644 index 0000000000..f2c841532a --- /dev/null +++ b/indra/llcorehttp/tests/testrunner.py @@ -0,0 +1,262 @@ +#!/usr/bin/env python +"""\ +@file testrunner.py +@author Nat Goodspeed +@date 2009-03-20 +@brief Utilities for writing wrapper scripts for ADD_COMM_BUILD_TEST unit tests + +$LicenseInfo:firstyear=2009&license=viewerlgpl$ +Second Life Viewer Source Code +Copyright (C) 2010, Linden Research, Inc. + +This library is free software; you can redistribute it and/or +modify it under the terms of the GNU Lesser General Public +License as published by the Free Software Foundation; +version 2.1 of the License only. + +This library is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public +License along with this library; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA +$/LicenseInfo$ +""" + +from __future__ import with_statement + +import os +import sys +import re +import errno +import socket + +VERBOSE = os.environ.get("INTEGRATION_TEST_VERBOSE", "1") # default to verbose +# Support usage such as INTEGRATION_TEST_VERBOSE=off -- distressing to user if +# that construct actually turns on verbosity... +VERBOSE = not re.match(r"(0|off|false|quiet)$", VERBOSE, re.IGNORECASE) + +if VERBOSE: + def debug(fmt, *args): + print fmt % args + sys.stdout.flush() +else: + debug = lambda *args: None + +def freeport(portlist, expr): + """ + Find a free server port to use. Specifically, evaluate 'expr' (a + callable(port)) until it stops raising EADDRINUSE exception. + + Pass: + + portlist: an iterable (e.g. xrange()) of ports to try. If you exhaust the + range, freeport() lets the socket.error exception propagate. If you want + unbounded, you could pass itertools.count(baseport), though of course in + practice the ceiling is 2^16-1 anyway. But it seems prudent to constrain + the range much more sharply: if we're iterating an absurd number of times, + probably something else is wrong. + + expr: a callable accepting a port number, specifically one of the items + from portlist. If calling that callable raises socket.error with + EADDRINUSE, freeport() retrieves the next item from portlist and retries. + + Returns: (expr(port), port) + + port: the value from portlist for which expr(port) succeeded + + Raises: + + Any exception raised by expr(port) other than EADDRINUSE. + + socket.error if, for every item from portlist, expr(port) raises + socket.error. The exception you see is the one from the last item in + portlist. + + StopIteration if portlist is completely empty. + + Example: + + class Server(HTTPServer): + # If you use BaseHTTPServer.HTTPServer, turning off this flag is + # essential for proper operation of freeport()! + allow_reuse_address = False + # ... + server, port = freeport(xrange(8000, 8010), + lambda port: Server(("localhost", port), + MyRequestHandler)) + # pass 'port' to client code + # call server.serve_forever() + """ + try: + # If portlist is completely empty, let StopIteration propagate: that's an + # error because we can't return meaningful values. We have no 'port', + # therefore no 'expr(port)'. + portiter = iter(portlist) + port = portiter.next() + + while True: + try: + # If this value of port works, return as promised. + value = expr(port) + + except socket.error, err: + # Anything other than 'Address already in use', propagate + if err.args[0] != errno.EADDRINUSE: + raise + + # Here we want the next port from portiter. But on StopIteration, + # we want to raise the original exception rather than + # StopIteration. So save the original exc_info(). + type, value, tb = sys.exc_info() + try: + try: + port = portiter.next() + except StopIteration: + raise type, value, tb + finally: + # Clean up local traceback, see docs for sys.exc_info() + del tb + + else: + debug("freeport() returning %s on port %s", value, port) + return value, port + + # Recap of the control flow above: + # If expr(port) doesn't raise, return as promised. + # If expr(port) raises anything but EADDRINUSE, propagate that + # exception. + # If portiter.next() raises StopIteration -- that is, if the port + # value we just passed to expr(port) was the last available -- reraise + # the EADDRINUSE exception. + # If we've actually arrived at this point, portiter.next() delivered a + # new port value. Loop back to pass that to expr(port). + + except Exception, err: + debug("*** freeport() raising %s: %s", err.__class__.__name__, err) + raise + +def run(*args, **kwds): + """All positional arguments collectively form a command line, executed as + a synchronous child process. + In addition, pass server=new_thread_instance as an explicit keyword (to + differentiate it from an additional command-line argument). + new_thread_instance should be an instantiated but not yet started Thread + subclass instance, e.g.: + run("python", "-c", 'print "Hello, world!"', server=TestHTTPServer(name="httpd")) + """ + # If there's no server= keyword arg, don't start a server thread: simply + # run a child process. + try: + thread = kwds.pop("server") + except KeyError: + pass + else: + # Start server thread. Note that this and all other comm server + # threads should be daemon threads: we'll let them run "forever," + # confident that the whole process will terminate when the main thread + # terminates, which will be when the child process terminates. + thread.setDaemon(True) + thread.start() + # choice of os.spawnv(): + # - [v vs. l] pass a list of args vs. individual arguments, + # - [no p] don't use the PATH because we specifically want to invoke the + # executable passed as our first arg, + # - [no e] child should inherit this process's environment. + debug("Running %s...", " ".join(args)) + rc = os.spawnv(os.P_WAIT, args[0], args) + debug("%s returned %s", args[0], rc) + return rc + +# **************************************************************************** +# test code -- manual at this point, see SWAT-564 +# **************************************************************************** +def test_freeport(): + # ------------------------------- Helpers -------------------------------- + from contextlib import contextmanager + # helper Context Manager for expecting an exception + # with exc(SomeError): + # raise SomeError() + # raises AssertionError otherwise. + @contextmanager + def exc(exception_class, *args): + try: + yield + except exception_class, err: + for i, expected_arg in enumerate(args): + assert expected_arg == err.args[i], \ + "Raised %s, but args[%s] is %r instead of %r" % \ + (err.__class__.__name__, i, err.args[i], expected_arg) + print "Caught expected exception %s(%s)" % \ + (err.__class__.__name__, ', '.join(repr(arg) for arg in err.args)) + else: + assert False, "Failed to raise " + exception_class.__class__.__name__ + + # helper to raise specified exception + def raiser(exception): + raise exception + + # the usual + def assert_equals(a, b): + assert a == b, "%r != %r" % (a, b) + + # ------------------------ Sanity check the above ------------------------ + class SomeError(Exception): pass + # Without extra args, accept any err.args value + with exc(SomeError): + raiser(SomeError("abc")) + # With extra args, accept only the specified value + with exc(SomeError, "abc"): + raiser(SomeError("abc")) + with exc(AssertionError): + with exc(SomeError, "abc"): + raiser(SomeError("def")) + with exc(AssertionError): + with exc(socket.error, errno.EADDRINUSE): + raiser(socket.error(errno.ECONNREFUSED, 'Connection refused')) + + # ----------- freeport() without engaging socket functionality ----------- + # If portlist is empty, freeport() raises StopIteration. + with exc(StopIteration): + freeport([], None) + + assert_equals(freeport([17], str), ("17", 17)) + + # This is the magic exception that should prompt us to retry + inuse = socket.error(errno.EADDRINUSE, 'Address already in use') + # Get the iterator to our ports list so we can check later if we've used all + ports = iter(xrange(5)) + with exc(socket.error, errno.EADDRINUSE): + freeport(ports, lambda port: raiser(inuse)) + # did we entirely exhaust 'ports'? + with exc(StopIteration): + ports.next() + + ports = iter(xrange(2)) + # Any exception but EADDRINUSE should quit immediately + with exc(SomeError): + freeport(ports, lambda port: raiser(SomeError())) + assert_equals(ports.next(), 1) + + # ----------- freeport() with platform-dependent socket stuff ------------ + # This is what we should've had unit tests to begin with (see CHOP-661). + def newbind(port): + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.bind(('127.0.0.1', port)) + return sock + + bound0, port0 = freeport(xrange(7777, 7780), newbind) + assert_equals(port0, 7777) + bound1, port1 = freeport(xrange(7777, 7780), newbind) + assert_equals(port1, 7778) + bound2, port2 = freeport(xrange(7777, 7780), newbind) + assert_equals(port2, 7779) + with exc(socket.error, errno.EADDRINUSE): + bound3, port3 = freeport(xrange(7777, 7780), newbind) + +if __name__ == "__main__": + test_freeport() |