From 75242eab8f8a892c792681fca080d86cfbb3e061 Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Mon, 11 Jun 2012 19:06:52 -0400 Subject: Bring in the testrunner/http server scaffold for better integration testing. This brings in a copy of llmessage's llsdmessage testing server. We run a mocked HTTP service to handle requests and the integration tests run against it by picking up the LL_TEST_PORT environment variable when running. Add some checks and output to produce useful info when run in the wrong environment and when bad status is received. Later will add a dead port as well so we can test that rather than use 'localhost:2'. --- indra/llcorehttp/tests/test_llcorehttp_peer.py | 146 +++++++++++++++++++++++++ 1 file changed, 146 insertions(+) create mode 100644 indra/llcorehttp/tests/test_llcorehttp_peer.py (limited to 'indra/llcorehttp/tests/test_llcorehttp_peer.py') diff --git a/indra/llcorehttp/tests/test_llcorehttp_peer.py b/indra/llcorehttp/tests/test_llcorehttp_peer.py new file mode 100644 index 0000000000..3e200a5c19 --- /dev/null +++ b/indra/llcorehttp/tests/test_llcorehttp_peer.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python +"""\ +@file test_llsdmessage_peer.py +@author Nat Goodspeed +@date 2008-10-09 +@brief This script asynchronously runs the executable (with args) specified on + the command line, returning its result code. While that executable is + running, we provide dummy local services for use by C++ tests. + +$LicenseInfo:firstyear=2008&license=viewerlgpl$ +Second Life Viewer Source Code +Copyright (C) 2010, Linden Research, Inc. + +This library is free software; you can redistribute it and/or +modify it under the terms of the GNU Lesser General Public +License as published by the Free Software Foundation; +version 2.1 of the License only. + +This library is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public +License along with this library; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA +$/LicenseInfo$ +""" + +import os +import sys +from threading import Thread +from BaseHTTPServer import HTTPServer, BaseHTTPRequestHandler + +mydir = os.path.dirname(__file__) # expected to be .../indra/llmessage/tests/ +sys.path.insert(0, os.path.join(mydir, os.pardir, os.pardir, "lib", "python")) +from indra.util.fastest_elementtree import parse as xml_parse +from indra.base import llsd +from testrunner import freeport, run, debug, VERBOSE + +class TestHTTPRequestHandler(BaseHTTPRequestHandler): + """This subclass of BaseHTTPRequestHandler is to receive and echo + LLSD-flavored messages sent by the C++ LLHTTPClient. + """ + def read(self): + # The following logic is adapted from the library module + # SimpleXMLRPCServer.py. + # Get arguments by reading body of request. + # We read this in chunks to avoid straining + # socket.read(); around the 10 or 15Mb mark, some platforms + # begin to have problems (bug #792570). + try: + size_remaining = int(self.headers["content-length"]) + except (KeyError, ValueError): + return "" + max_chunk_size = 10*1024*1024 + L = [] + while size_remaining: + chunk_size = min(size_remaining, max_chunk_size) + chunk = self.rfile.read(chunk_size) + L.append(chunk) + size_remaining -= len(chunk) + return ''.join(L) + # end of swiped read() logic + + def read_xml(self): + # This approach reads the entire POST data into memory first + return llsd.parse(self.read()) +## # This approach attempts to stream in the LLSD XML from self.rfile, +## # assuming that the underlying XML parser reads its input file +## # incrementally. Unfortunately I haven't been able to make it work. +## tree = xml_parse(self.rfile) +## debug("Finished raw parse") +## debug("parsed XML tree %s", tree) +## debug("parsed root node %s", tree.getroot()) +## debug("root node tag %s", tree.getroot().tag) +## return llsd.to_python(tree.getroot()) + + def do_GET(self): + # Of course, don't attempt to read data. + self.answer(dict(reply="success", status=200, + reason="Your GET operation worked")) + + def do_POST(self): + # Read the provided POST data. + self.answer(self.read()) + + def answer(self, data): + debug("%s.answer(%s): self.path = %r", self.__class__.__name__, data, self.path) + if "fail" not in self.path: + response = llsd.format_xml(data.get("reply", llsd.LLSD("success"))) + debug("success: %s", response) + self.send_response(200) + self.send_header("Content-type", "application/llsd+xml") + self.send_header("Content-Length", str(len(response))) + self.end_headers() + self.wfile.write(response) + else: # fail requested + status = data.get("status", 500) + # self.responses maps an int status to a (short, long) pair of + # strings. We want the longer string. That's why we pass a string + # pair to get(): the [1] will select the second string, whether it + # came from self.responses or from our default pair. + reason = data.get("reason", + self.responses.get(status, + ("fail requested", + "Your request specified failure status %s " + "without providing a reason" % status))[1]) + debug("fail requested: %s: %r", status, reason) + self.send_error(status, reason) + + if not VERBOSE: + # When VERBOSE is set, skip both these overrides because they exist to + # suppress output. + + def log_request(self, code, size=None): + # For present purposes, we don't want the request splattered onto + # stderr, as it would upset devs watching the test run + pass + + def log_error(self, format, *args): + # Suppress error output as well + pass + +class Server(HTTPServer): + # This pernicious flag is on by default in HTTPServer. But proper + # operation of freeport() absolutely depends on it being off. + allow_reuse_address = False + +if __name__ == "__main__": + # Instantiate a Server(TestHTTPRequestHandler) on the first free port + # in the specified port range. Doing this inline is better than in a + # daemon thread: if it blows up here, we'll get a traceback. If it blew up + # in some other thread, the traceback would get eaten and we'd run the + # subject test program anyway. + httpd, port = freeport(xrange(8000, 8020), + lambda port: Server(('127.0.0.1', port), TestHTTPRequestHandler)) + # Pass the selected port number to the subject test program via the + # environment. We don't want to impose requirements on the test program's + # command-line parsing -- and anyway, for C++ integration tests, that's + # performed in TUT code rather than our own. + os.environ["LL_TEST_PORT"] = str(port) + debug("$LL_TEST_PORT = %s", port) + sys.exit(run(server=Thread(name="httpd", target=httpd.serve_forever), *sys.argv[1:])) -- cgit v1.2.3 From 7adeb3923728ca84a309a6af141c148ce38066fc Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Tue, 12 Jun 2012 17:42:33 -0400 Subject: HTTP Proxy, PUT & POST, unit tests and refactoring. Implemented/modified PUT & POST to not used chunked encoding for the request. Made the unit test much happier and probably a better thing for the pipeline. Have a cheesy static & dynamic proxy capability using both local options and a way to wire into LLProxy in llmessages. Not a clean thing but it will get the proxy path working with both socks5 & http proxies. Refactoring to get rid of unneeded library handler and unified an HttpStatus return for all requests. Big batch of code removed as a result of that and more is possible as well as some syscall avoidance with a bit more work. Boosted the unit tests for simple PUT & POST test which revealed the test harness does *not* like chunked encoding so we'll avoid it for now (and don't really need it in any of our schemes). --- indra/llcorehttp/tests/test_llcorehttp_peer.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) (limited to 'indra/llcorehttp/tests/test_llcorehttp_peer.py') diff --git a/indra/llcorehttp/tests/test_llcorehttp_peer.py b/indra/llcorehttp/tests/test_llcorehttp_peer.py index 3e200a5c19..8c3ad805b3 100644 --- a/indra/llcorehttp/tests/test_llcorehttp_peer.py +++ b/indra/llcorehttp/tests/test_llcorehttp_peer.py @@ -85,7 +85,15 @@ class TestHTTPRequestHandler(BaseHTTPRequestHandler): def do_POST(self): # Read the provided POST data. - self.answer(self.read()) + # self.answer(self.read()) + self.answer(dict(reply="success", status=200, + reason=self.read())) + + def do_PUT(self): + # Read the provided PUT data. + # self.answer(self.read()) + self.answer(dict(reply="success", status=200, + reason=self.read())) def answer(self, data): debug("%s.answer(%s): self.path = %r", self.__class__.__name__, data, self.path) -- cgit v1.2.3 From f0353abe7605778048d69ce3acb8f5ddd5693083 Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Tue, 19 Jun 2012 15:43:29 -0400 Subject: Implement timeout and retry count options for requests. Pretty straightforward. Still don't like how I'm managing the options block. Struct? Accessors? Can't decide. But the options now speed up the unit test runs even as I add tests. --- indra/llcorehttp/tests/test_llcorehttp_peer.py | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'indra/llcorehttp/tests/test_llcorehttp_peer.py') diff --git a/indra/llcorehttp/tests/test_llcorehttp_peer.py b/indra/llcorehttp/tests/test_llcorehttp_peer.py index 8c3ad805b3..5f0116d384 100644 --- a/indra/llcorehttp/tests/test_llcorehttp_peer.py +++ b/indra/llcorehttp/tests/test_llcorehttp_peer.py @@ -31,6 +31,7 @@ $/LicenseInfo$ import os import sys +import time from threading import Thread from BaseHTTPServer import HTTPServer, BaseHTTPRequestHandler @@ -97,6 +98,9 @@ class TestHTTPRequestHandler(BaseHTTPRequestHandler): def answer(self, data): debug("%s.answer(%s): self.path = %r", self.__class__.__name__, data, self.path) + if self.path.find("/sleep/") != -1: + time.sleep(30) + if "fail" not in self.path: response = llsd.format_xml(data.get("reply", llsd.LLSD("success"))) debug("success: %s", response) -- cgit v1.2.3 From a066bc1994fccae7967921980332505aac97953f Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Wed, 20 Jun 2012 18:43:28 -0400 Subject: SH-3181 More interface options for API. Also includes returned headers. Only thing interesting in this changeset is the discovery that a sleep in the fake HTTP server ties up tests. Need to thread that or fail on client disconnect or something to speed that up and make it usable for bigger test scenarios. But good enough for now... --- indra/llcorehttp/tests/test_llcorehttp_peer.py | 1 + 1 file changed, 1 insertion(+) (limited to 'indra/llcorehttp/tests/test_llcorehttp_peer.py') diff --git a/indra/llcorehttp/tests/test_llcorehttp_peer.py b/indra/llcorehttp/tests/test_llcorehttp_peer.py index 5f0116d384..aedd8acd83 100644 --- a/indra/llcorehttp/tests/test_llcorehttp_peer.py +++ b/indra/llcorehttp/tests/test_llcorehttp_peer.py @@ -107,6 +107,7 @@ class TestHTTPRequestHandler(BaseHTTPRequestHandler): self.send_response(200) self.send_header("Content-type", "application/llsd+xml") self.send_header("Content-Length", str(len(response))) + self.send_header("X-LL-Special", "Mememememe"); self.end_headers() self.wfile.write(response) else: # fail requested -- cgit v1.2.3 From 398d78a77348fb4c5ee4e96b7ed6f981b1042627 Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Mon, 9 Jul 2012 13:28:50 -0400 Subject: Rework the 'sleep' logic in the test HTTP server so that the 30-second hang doesn't break subsequent tests. Did this by introducing threads into the HTTP server as I can't find the magic to detect that my client has gone away. --- indra/llcorehttp/tests/test_llcorehttp_peer.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'indra/llcorehttp/tests/test_llcorehttp_peer.py') diff --git a/indra/llcorehttp/tests/test_llcorehttp_peer.py b/indra/llcorehttp/tests/test_llcorehttp_peer.py index 0e38e5a87f..9e1847c66e 100644 --- a/indra/llcorehttp/tests/test_llcorehttp_peer.py +++ b/indra/llcorehttp/tests/test_llcorehttp_peer.py @@ -32,10 +32,12 @@ $/LicenseInfo$ import os import sys import time +import select from threading import Thread from BaseHTTPServer import HTTPServer, BaseHTTPRequestHandler +from SocketServer import ThreadingMixIn -mydir = os.path.dirname(__file__) # expected to be .../indra/llmessage/tests/ +mydir = os.path.dirname(__file__) # expected to be .../indra/llcorehttp/tests/ sys.path.insert(0, os.path.join(mydir, os.pardir, os.pardir, "lib", "python")) from indra.util.fastest_elementtree import parse as xml_parse from indra.base import llsd @@ -144,7 +146,7 @@ class TestHTTPRequestHandler(BaseHTTPRequestHandler): # Suppress error output as well pass -class Server(HTTPServer): +class Server(ThreadingMixIn, HTTPServer): # This pernicious flag is on by default in HTTPServer. But proper # operation of freeport() absolutely depends on it being off. allow_reuse_address = False -- cgit v1.2.3 From 7010459f04177aef1875a110b3d33e10c8ec5cad Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Wed, 11 Jul 2012 15:53:57 -0400 Subject: SH-3240 Capture Content-Type and Content-Encoding headers. HttpResponse object now has two strings for these content headers. Either or both may be empty. Tidied up the cross-platform string code and got more defensive about the length of a header line. Integration test for the new response object. --- indra/llcorehttp/tests/test_llcorehttp_peer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'indra/llcorehttp/tests/test_llcorehttp_peer.py') diff --git a/indra/llcorehttp/tests/test_llcorehttp_peer.py b/indra/llcorehttp/tests/test_llcorehttp_peer.py index 9e1847c66e..58b5bedd09 100644 --- a/indra/llcorehttp/tests/test_llcorehttp_peer.py +++ b/indra/llcorehttp/tests/test_llcorehttp_peer.py @@ -9,7 +9,7 @@ $LicenseInfo:firstyear=2008&license=viewerlgpl$ Second Life Viewer Source Code -Copyright (C) 2010, Linden Research, Inc. +Copyright (C) 2012, Linden Research, Inc. This library is free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General Public -- cgit v1.2.3 From d45b2e7caece787dce4be501b103432c0f06c0f2 Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Thu, 12 Jul 2012 17:46:53 +0000 Subject: SH-3183 Use valgrind on the library. Using http_texture_load as the test subject, library looks clean. Did some better shutdown in the program itself and it looks better. Libcurl itself is making a lot of noise. Adapted testrunner to run valgrind as well but the memory allocation tester in the tools themselves grossly interferes with Valgrind operations. --- indra/llcorehttp/tests/test_llcorehttp_peer.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) (limited to 'indra/llcorehttp/tests/test_llcorehttp_peer.py') diff --git a/indra/llcorehttp/tests/test_llcorehttp_peer.py b/indra/llcorehttp/tests/test_llcorehttp_peer.py index 58b5bedd09..489e8b2979 100644 --- a/indra/llcorehttp/tests/test_llcorehttp_peer.py +++ b/indra/llcorehttp/tests/test_llcorehttp_peer.py @@ -33,6 +33,7 @@ import os import sys import time import select +import getopt from threading import Thread from BaseHTTPServer import HTTPServer, BaseHTTPRequestHandler from SocketServer import ThreadingMixIn @@ -152,6 +153,13 @@ class Server(ThreadingMixIn, HTTPServer): allow_reuse_address = False if __name__ == "__main__": + do_valgrind = False + path_search = False + options, args = getopt.getopt(sys.argv[1:], "V", ["valgrind"]) + for option, value in options: + if option == "-V" or option == "--valgrind": + do_valgrind = True + # Instantiate a Server(TestHTTPRequestHandler) on the first free port # in the specified port range. Doing this inline is better than in a # daemon thread: if it blows up here, we'll get a traceback. If it blew up @@ -159,10 +167,14 @@ if __name__ == "__main__": # subject test program anyway. httpd, port = freeport(xrange(8000, 8020), lambda port: Server(('127.0.0.1', port), TestHTTPRequestHandler)) + # Pass the selected port number to the subject test program via the # environment. We don't want to impose requirements on the test program's # command-line parsing -- and anyway, for C++ integration tests, that's # performed in TUT code rather than our own. os.environ["LL_TEST_PORT"] = str(port) debug("$LL_TEST_PORT = %s", port) - sys.exit(run(server=Thread(name="httpd", target=httpd.serve_forever), *sys.argv[1:])) + if do_valgrind: + args = ["valgrind", "--log-file=./valgrind.log"] + args + path_search = True + sys.exit(run(server=Thread(name="httpd", target=httpd.serve_forever), use_path=path_search, *args)) -- cgit v1.2.3 From 5eb5dc6b27c57f1c3e77fc04b471614968620068 Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Fri, 13 Jul 2012 18:24:49 -0400 Subject: SH-3241 validate that request headers are correct First round of integration tests. Added a request header 'reflector' to the web server to sent the client's headers back with a 'X-Reflect-' prefix. Use boost::regex to check various headers. Run a test on a simple GET and a byte-ranged GET a la texture fetch. --- indra/llcorehttp/tests/test_llcorehttp_peer.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) (limited to 'indra/llcorehttp/tests/test_llcorehttp_peer.py') diff --git a/indra/llcorehttp/tests/test_llcorehttp_peer.py b/indra/llcorehttp/tests/test_llcorehttp_peer.py index 489e8b2979..c527ce6ce0 100644 --- a/indra/llcorehttp/tests/test_llcorehttp_peer.py +++ b/indra/llcorehttp/tests/test_llcorehttp_peer.py @@ -104,7 +104,7 @@ class TestHTTPRequestHandler(BaseHTTPRequestHandler): def answer(self, data, withdata=True): debug("%s.answer(%s): self.path = %r", self.__class__.__name__, data, self.path) - if self.path.find("/sleep/") != -1: + if "/sleep/" in self.path: time.sleep(30) if "fail" not in self.path: @@ -114,6 +114,8 @@ class TestHTTPRequestHandler(BaseHTTPRequestHandler): response = llsd.format_xml(data) debug("success: %s", response) self.send_response(200) + if "/reflect/" in self.path: + self.reflect_headers() self.send_header("Content-type", "application/llsd+xml") self.send_header("Content-Length", str(len(response))) self.send_header("X-LL-Special", "Mememememe"); @@ -133,6 +135,13 @@ class TestHTTPRequestHandler(BaseHTTPRequestHandler): "without providing a reason" % status))[1]) debug("fail requested: %s: %r", status, reason) self.send_error(status, reason) + if "/reflect/" in self.path: + self.reflect_headers() + self.end_headers() + + def reflect_headers(self): + for name in self.headers.keys(): + self.send_header("X-Reflect-" + name, self.headers[name]) if not VERBOSE: # When VERBOSE is set, skip both these overrides because they exist to -- cgit v1.2.3 From 8d3e5f3959b071226c4a5c2c68d9fe8664ae1ffd Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Mon, 16 Jul 2012 17:35:44 -0400 Subject: SH-3241 Validate header correctness, SH-3243 more unit tests Define expectations for headers for GET, POST, PUT requests. Document those in the interface, test those with integration tests. Verify that header overrides work as expected. --- indra/llcorehttp/tests/test_llcorehttp_peer.py | 1 + 1 file changed, 1 insertion(+) (limited to 'indra/llcorehttp/tests/test_llcorehttp_peer.py') diff --git a/indra/llcorehttp/tests/test_llcorehttp_peer.py b/indra/llcorehttp/tests/test_llcorehttp_peer.py index c527ce6ce0..75a3c39ef2 100644 --- a/indra/llcorehttp/tests/test_llcorehttp_peer.py +++ b/indra/llcorehttp/tests/test_llcorehttp_peer.py @@ -141,6 +141,7 @@ class TestHTTPRequestHandler(BaseHTTPRequestHandler): def reflect_headers(self): for name in self.headers.keys(): + # print "Header: %s: %s" % (name, self.headers[name]) self.send_header("X-Reflect-" + name, self.headers[name]) if not VERBOSE: -- cgit v1.2.3