SH-3241 validate that request headers are correct

First round of integration tests.  Added a request header 'reflector'
to the web server to sent the client's headers back with a 'X-Reflect-'
prefix.  Use boost::regex to check various headers.  Run a test on
a simple GET and a byte-ranged GET a la texture fetch.

4 files changed, 232 insertions, 20 deletions

diff --git a/indra/llcorehttp/httpoptions.cpp b/indra/llcorehttp/httpoptions.cpp
index f2771c1f29..68f7277ed3 100644
--- a/indra/llcorehttp/httpoptions.cpp
+++ b/indra/llcorehttp/httpoptions.cpp
@@ -46,9 +46,9 @@ HttpOptions::~HttpOptions()
 {}
 
 
-void HttpOptions::setWantHeaders()
+void HttpOptions::setWantHeaders(bool wanted)
 {
-	mWantHeaders = true;
+	mWantHeaders = wanted;
 }
 
 
diff --git a/indra/llcorehttp/httpoptions.h b/indra/llcorehttp/httpoptions.h
index a0b2253c11..97e46a8cd3 100644
--- a/indra/llcorehttp/httpoptions.h
+++ b/indra/llcorehttp/httpoptions.h
@@ -68,7 +68,7 @@ protected:
 	void operator=(const HttpOptions &);		// Not defined
 
 public:
-	void				setWantHeaders();
+	void				setWantHeaders(bool wanted);
 	bool				getWantHeaders() const
 		{
 			return mWantHeaders;
diff --git a/indra/llcorehttp/tests/test_httprequest.hpp b/indra/llcorehttp/tests/test_httprequest.hpp
index 9edf3d19ec..0f9eb93996 100644
--- a/indra/llcorehttp/tests/test_httprequest.hpp
+++ b/indra/llcorehttp/tests/test_httprequest.hpp
@@ -36,6 +36,8 @@
 #include "_httprequestqueue.h"
 
 #include <curl/curl.h>
+#include <boost/regex.hpp>
+#include <sstream>
 
 #include "test_allocator.h"
 #include "llcorehttp_test.h"
@@ -74,8 +76,7 @@ public:
 				 const std::string & name)
 		: mState(state),
 		  mName(name),
-		  mExpectHandle(LLCORE_HTTP_HANDLE_INVALID),
-		  mCheckHeader(false)
+		  mExpectHandle(LLCORE_HTTP_HANDLE_INVALID)
 		{}
 	
 	virtual void onCompleted(HttpHandle handle, HttpResponse * response)
@@ -97,24 +98,50 @@ public:
 			{
 				mState->mHandlerCalls++;
 			}
-			if (mCheckHeader)
+			if (! mHeadersRequired.empty() || ! mHeadersDisallowed.empty())
 			{
 				ensure("Response required with header check", response != NULL);
 				HttpHeaders * header(response->getHeaders());	// Will not hold onto this
 				ensure("Some quantity of headers returned", header != NULL);
-				bool found_special(false);
-				
-				for (HttpHeaders::container_t::const_iterator iter(header->mHeaders.begin());
-					 header->mHeaders.end() != iter;
-					 ++iter)
+
+				if (! mHeadersRequired.empty())
+				{
+					for (int i(0); i < mHeadersRequired.size(); ++i)
+					{
+						bool found = false;
+						for (HttpHeaders::container_t::const_iterator iter(header->mHeaders.begin());
+							 header->mHeaders.end() != iter;
+							 ++iter)
+						{
+							if (boost::regex_match(*iter, mHeadersRequired[i]))
+							{
+								found = true;
+								break;
+							}
+						}
+						std::ostringstream str;
+						str << "Required header # " << i << " found in response";
+						ensure(str.str(), found);
+					}
+				}
+
+				if (! mHeadersDisallowed.empty())
 				{
-					if (std::string::npos != (*iter).find("X-LL-Special"))
+					for (int i(0); i < mHeadersDisallowed.size(); ++i)
 					{
-						found_special = true;
-						break;
+						for (HttpHeaders::container_t::const_iterator iter(header->mHeaders.begin());
+							 header->mHeaders.end() != iter;
+							 ++iter)
+						{
+							if (boost::regex_match(*iter, mHeadersDisallowed[i]))
+							{
+								std::ostringstream str;
+								str << "Disallowed header # " << i << " not found in response";
+								ensure(str.str(), false);
+							}
+						}
 					}
 				}
-				ensure("Special header X-LL-Special in response", found_special);
 			}
 			
 			if (! mCheckContentType.empty())
@@ -132,8 +159,9 @@ public:
 	HttpRequestTestData * mState;
 	std::string mName;
 	HttpHandle mExpectHandle;
-	bool mCheckHeader;
 	std::string mCheckContentType;
+	std::vector<boost::regex> mHeadersRequired;
+	std::vector<boost::regex> mHeadersDisallowed;
 };
 
 typedef test_group<HttpRequestTestData> HttpRequestTestGroupType;
@@ -1268,6 +1296,10 @@ void HttpRequestTestObjectType::test<13>()
 {
 	ScopedCurlInit ready;
 
+	// Warmup boost::regex to pre-alloc memory for memory size tests
+	boost::regex warmup("askldjflasdj;f", boost::regex::icase);
+	boost::regex_match("akl;sjflajfk;ajsk", warmup);
+	
 	std::string url_base(get_base_url());
 	// std::cerr << "Base:  "  << url_base << std::endl;
 	
@@ -1277,6 +1309,7 @@ void HttpRequestTestObjectType::test<13>()
 	// references to it after completion of this method.
 	// Create before memory record as the string copy will bump numbers.
 	TestHandler2 handler(this, "handler");
+	handler.mHeadersRequired.reserve(20);				// Avoid memory leak test failure
 		
 	// record the total amount of dynamically allocated memory
 	mMemTotal = GetMemTotal();
@@ -1302,11 +1335,11 @@ void HttpRequestTestObjectType::test<13>()
 		ensure("Memory allocated on construction", mMemTotal < GetMemTotal());
 
 		opts = new HttpOptions();
-		opts->setWantHeaders();
+		opts->setWantHeaders(true);
 		
 		// Issue a GET that succeeds
 		mStatus = HttpStatus(200);
-		handler.mCheckHeader = true;
+		handler.mHeadersRequired.push_back(boost::regex("\\W*X-LL-Special:.*", boost::regex::icase));
 		HttpHandle handle = req->requestGetByteRange(HttpRequest::DEFAULT_POLICY_ID,
 													 0U,
 													 url_base,
@@ -1334,7 +1367,7 @@ void HttpRequestTestObjectType::test<13>()
 
 		// Okay, request a shutdown of the servicing thread
 		mStatus = HttpStatus();
-		handler.mCheckHeader = false;
+		handler.mHeadersRequired.clear();
 		handle = req->requestStopThread(&handler);
 		ensure("Valid handle returned for second request", handle != LLCORE_HTTP_HANDLE_INVALID);
 	
@@ -1628,6 +1661,176 @@ void HttpRequestTestObjectType::test<15>()
 }
 
 
+// Test header generation on GET requests
+template <> template <>
+void HttpRequestTestObjectType::test<16>()
+{
+	ScopedCurlInit ready;
+
+	// Warmup boost::regex to pre-alloc memory for memory size tests
+	boost::regex warmup("askldjflasdj;f", boost::regex::icase);
+	boost::regex_match("akl;sjflajfk;ajsk", warmup);
+
+	std::string url_base(get_base_url());
+	
+	set_test_name("Header generation for HttpRequest GET");
+
+	// Handler can be stack-allocated *if* there are no dangling
+	// references to it after completion of this method.
+	// Create before memory record as the string copy will bump numbers.
+	TestHandler2 handler(this, "handler");
+
+	// record the total amount of dynamically allocated memory
+	mMemTotal = GetMemTotal();
+	mHandlerCalls = 0;
+
+	HttpRequest * req = NULL;
+	HttpOptions * options = NULL;
+	HttpHeaders * headers = NULL;
+
+	try
+	{
+		// Get singletons created
+		HttpRequest::createService();
+		
+		// Start threading early so that thread memory is invariant
+		// over the test.
+		HttpRequest::startThread();
+
+		// create a new ref counted object with an implicit reference
+		req = new HttpRequest();
+
+		// options set
+		options = new HttpOptions();
+		options->setWantHeaders(true);
+		
+		// Issue a GET that *can* connect
+		mStatus = HttpStatus(200);
+		handler.mHeadersRequired.push_back(boost::regex("X-Reflect-connection:\\W*keep-alive", boost::regex::icase));
+		handler.mHeadersRequired.push_back(boost::regex("X-Reflect-accept:\\W*\\*/\\*", boost::regex::icase));
+		handler.mHeadersDisallowed.push_back(boost::regex("\\W*X-Reflect-cache-control:.*", boost::regex::icase));
+		handler.mHeadersDisallowed.push_back(boost::regex("\\W*X-Reflect-pragma:.*", boost::regex::icase));
+		handler.mHeadersDisallowed.push_back(boost::regex("\\W*X-Reflect-range:.*", boost::regex::icase));
+		HttpHandle handle = req->requestGet(HttpRequest::DEFAULT_POLICY_ID,
+											0U,
+											url_base + "reflect/",
+											options,
+											NULL,
+											&handler);
+		ensure("Valid handle returned for get request", handle != LLCORE_HTTP_HANDLE_INVALID);
+
+		// Run the notification pump.
+		int count(0);
+		int limit(10);
+		while (count++ < limit && mHandlerCalls < 1)
+		{
+			req->update(1000000);
+			usleep(100000);
+		}
+		ensure("Request executed in reasonable time", count < limit);
+		ensure("One handler invocation for request", mHandlerCalls == 1);
+
+		// Do a texture-style fetch
+		headers = new HttpHeaders;
+		headers->mHeaders.push_back("Accept: image/x-j2c");
+		
+		mStatus = HttpStatus(200);
+		handler.mHeadersRequired.clear();
+		handler.mHeadersRequired.push_back(boost::regex("X-Reflect-connection:\\W*keep-alive", boost::regex::icase));
+		handler.mHeadersRequired.push_back(boost::regex("X-Reflect-accept:\\W*\\image/x-j2c", boost::regex::icase));
+		handler.mHeadersDisallowed.push_back(boost::regex("\\W*X-Reflect-range:.*", boost::regex::icase));
+		handler.mHeadersDisallowed.clear();
+		handler.mHeadersDisallowed.push_back(boost::regex("\\W*X-Reflect-cache-control:.*", boost::regex::icase));
+		handler.mHeadersDisallowed.push_back(boost::regex("\\W*X-Reflect-pragma:.*", boost::regex::icase));
+		handle = req->requestGetByteRange(HttpRequest::DEFAULT_POLICY_ID,
+										  0U,
+										  url_base + "reflect/",
+										  0,
+										  47,
+										  options,
+										  headers,
+										  &handler);
+		ensure("Valid handle returned for ranged request", handle != LLCORE_HTTP_HANDLE_INVALID);
+
+		// Run the notification pump.
+		count = 0;
+		limit = 10;
+		while (count++ < limit && mHandlerCalls < 2)
+		{
+			req->update(1000000);
+			usleep(100000);
+		}
+		ensure("Request executed in reasonable time", count < limit);
+		ensure("One handler invocation for request", mHandlerCalls == 2);
+
+
+		// Okay, request a shutdown of the servicing thread
+		mStatus = HttpStatus();
+		handler.mHeadersRequired.clear();
+		handler.mHeadersDisallowed.clear();
+		handle = req->requestStopThread(&handler);
+		ensure("Valid handle returned for second request", handle != LLCORE_HTTP_HANDLE_INVALID);
+	
+		// Run the notification pump again
+		count = 0;
+		limit = 10;
+		while (count++ < limit && mHandlerCalls < 3)
+		{
+			req->update(1000000);
+			usleep(100000);
+		}
+		ensure("Second request executed in reasonable time", count < limit);
+		ensure("Second handler invocation", mHandlerCalls == 3);
+
+		// See that we actually shutdown the thread
+		count = 0;
+		limit = 10;
+		while (count++ < limit && ! HttpService::isStopped())
+		{
+			usleep(100000);
+		}
+		ensure("Thread actually stopped running", HttpService::isStopped());
+	
+		// release options & headers
+		if (options)
+		{
+			options->release();
+		}
+		options = NULL;
+
+		if (headers)
+		{
+			headers->release();
+		}
+		headers = NULL;
+		
+		// release the request object
+		delete req;
+		req = NULL;
+
+		// Shut down service
+		HttpRequest::destroyService();
+	}
+	catch (...)
+	{
+		stop_thread(req);
+		if (options)
+		{
+			options->release();
+			options = NULL;
+		}
+		if (headers)
+		{
+			headers->release();
+			headers = NULL;
+		}
+		delete req;
+		HttpRequest::destroyService();
+		throw;
+	}
+}
+
+
 }  // end namespace tut
 
 namespace
diff --git a/indra/llcorehttp/tests/test_llcorehttp_peer.py b/indra/llcorehttp/tests/test_llcorehttp_peer.py
index 489e8b2979..c527ce6ce0 100644
--- a/indra/llcorehttp/tests/test_llcorehttp_peer.py
+++ b/indra/llcorehttp/tests/test_llcorehttp_peer.py
@@ -104,7 +104,7 @@ class TestHTTPRequestHandler(BaseHTTPRequestHandler):
 
     def answer(self, data, withdata=True):
         debug("%s.answer(%s): self.path = %r", self.__class__.__name__, data, self.path)
-        if self.path.find("/sleep/") != -1:
+        if "/sleep/" in self.path:
             time.sleep(30)
 
         if "fail" not in self.path:
@@ -114,6 +114,8 @@ class TestHTTPRequestHandler(BaseHTTPRequestHandler):
             response = llsd.format_xml(data)
             debug("success: %s", response)
             self.send_response(200)
+            if "/reflect/" in self.path:
+                self.reflect_headers()
             self.send_header("Content-type", "application/llsd+xml")
             self.send_header("Content-Length", str(len(response)))
             self.send_header("X-LL-Special", "Mememememe");
@@ -133,6 +135,13 @@ class TestHTTPRequestHandler(BaseHTTPRequestHandler):
                                                    "without providing a reason" % status))[1])
             debug("fail requested: %s: %r", status, reason)
             self.send_error(status, reason)
+            if "/reflect/" in self.path:
+                self.reflect_headers()
+            self.end_headers()
+
+    def reflect_headers(self):
+        for name in self.headers.keys():
+            self.send_header("X-Reflect-" + name, self.headers[name])
 
     if not VERBOSE:
         # When VERBOSE is set, skip both these overrides because they exist to