SH-3240 Capture Content-Type and Content-Encoding headers.

HttpResponse object now has two strings for these content headers.
Either or both may be empty.  Tidied up the cross-platform string
code and got more defensive about the length of a header line.
Integration test for the new response object.

5 files changed, 263 insertions, 46 deletions

diff --git a/indra/llcorehttp/_httpoprequest.cpp b/indra/llcorehttp/_httpoprequest.cpp
index bec82d8449..1854d7ada4 100644
--- a/indra/llcorehttp/_httpoprequest.cpp
+++ b/indra/llcorehttp/_httpoprequest.cpp
@@ -74,16 +74,16 @@ void escape_libcurl_debug_data(char * buffer, size_t len, bool scrub,
 							   std::string & safe_line);
 
 
-#if LL_WINDOWS
+// OS-neutral string comparisons of various types
+int os_strncasecmp(const char *s1, const char *s2, size_t n);
+int os_strcasecmp(const char *s1, const char *s2);
+char * os_strtok_r(char *str, const char *delim, char **saveptr);
 
-// Not available on windows where the legacy strtok interface
-// is thread-safe.
-char *strtok_r(char *str, const char *delim, char **saveptr);
 
-#endif // LL_WINDOWS
+static const char * const hdr_whitespace(" \t");
+static const char * const hdr_separator(": \t");
 
-
-}
+} // end anonymous namespace
 
 
 namespace LLCore
@@ -228,7 +228,8 @@ void HttpOpRequest::visitNotifier(HttpRequest * request)
 			// Got an explicit offset/length in response
 			response->setRange(mReplyOffset, mReplyLength, mReplyFullLength);
 		}
-
+		response->setContent(mReplyConType, mReplyConEncode);
+		
 		mUserHandler->onCompleted(static_cast<HttpHandle>(this), response);
 
 		response->release();
@@ -315,7 +316,7 @@ void HttpOpRequest::setupCommon(HttpRequest::policy_t policy_id,
 								HttpOptions * options,
 								HttpHeaders * headers)
 {
-	mProcFlags = 0;
+	mProcFlags = PF_SCAN_CONTENT_HEADERS;		// Always scan for content headers
 	mReqPolicy = policy_id;
 	mReqPriority = priority;
 	mReqURL = url;
@@ -377,6 +378,8 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service)
 		mReplyHeaders->release();
 		mReplyHeaders = NULL;
 	}
+	mReplyConType.clear();
+	mReplyConEncode.clear();
 	
 	// *FIXME:  better error handling later
 	HttpStatus status;
@@ -539,7 +542,7 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service)
 	}
 	curl_easy_setopt(mCurlHandle, CURLOPT_HTTPHEADER, mCurlHeaders);
 
-	if (mProcFlags & (PF_SCAN_RANGE_HEADER | PF_SAVE_HEADERS))
+	if (mProcFlags & (PF_SCAN_RANGE_HEADER | PF_SAVE_HEADERS | PF_SCAN_CONTENT_HEADERS))
 	{
 		curl_easy_setopt(mCurlHandle, CURLOPT_HEADERFUNCTION, headerCallback);
 		curl_easy_setopt(mCurlHandle, CURLOPT_HEADERDATA, this);
@@ -598,12 +601,18 @@ size_t HttpOpRequest::headerCallback(void * data, size_t size, size_t nmemb, voi
 
 	static const char con_ran_line[] = "content-range:";
 	static const size_t con_ran_line_len = sizeof(con_ran_line) - 1;
+
+	static const char con_type_line[] = "content-type:";
+	static const size_t con_type_line_len = sizeof(con_type_line) - 1;
+	
+	static const char con_enc_line[] = "content-encoding:";
+	static const size_t con_enc_line_len = sizeof(con_enc_line) - 1;
 	
 	HttpOpRequest * op(static_cast<HttpOpRequest *>(userdata));
 
 	const size_t hdr_size(size * nmemb);
 	const char * hdr_data(static_cast<const char *>(data));		// Not null terminated
-	
+
 	if (hdr_size >= status_line_len && ! strncmp(status_line, hdr_data, status_line_len))
 	{
 		// One of possibly several status lines.  Reset what we know and start over
@@ -611,24 +620,47 @@ size_t HttpOpRequest::headerCallback(void * data, size_t size, size_t nmemb, voi
 		op->mReplyOffset = 0;
 		op->mReplyLength = 0;
 		op->mReplyFullLength = 0;
+		op->mReplyConType.clear();
+		op->mReplyConEncode.clear();
 		op->mStatus = HttpStatus();
 		if (op->mReplyHeaders)
 		{
 			op->mReplyHeaders->mHeaders.clear();
 		}
 	}
-	else if (op->mProcFlags & PF_SCAN_RANGE_HEADER)
+
+	// Nothing in here wants a final CR/LF combination.  Remove
+	// it as much as possible.
+	size_t wanted_hdr_size(hdr_size);
+	if (wanted_hdr_size && '\n' == hdr_data[wanted_hdr_size - 1])
+	{
+		if (--wanted_hdr_size && '\r' == hdr_data[wanted_hdr_size - 1])
+		{
+			--wanted_hdr_size;
+		}
+	}
+	
+	// Save header if caller wants them in the response
+	if (op->mProcFlags & PF_SAVE_HEADERS)
+	{
+		// Save headers in response
+		if (! op->mReplyHeaders)
+		{
+			op->mReplyHeaders = new HttpHeaders;
+		}
+		op->mReplyHeaders->mHeaders.push_back(std::string(hdr_data, wanted_hdr_size));
+	}
+
+	// Detect and parse 'Content-Range' headers
+	if (op->mProcFlags & PF_SCAN_RANGE_HEADER)
 	{
 		char hdr_buffer[128];			// Enough for a reasonable header
-		size_t frag_size((std::min)(hdr_size, sizeof(hdr_buffer) - 1));
+		size_t frag_size((std::min)(wanted_hdr_size, sizeof(hdr_buffer) - 1));
 		
 		memcpy(hdr_buffer, hdr_data, frag_size);
 		hdr_buffer[frag_size] = '\0';
-#if LL_WINDOWS
-		if (! _strnicmp(hdr_buffer, con_ran_line, (std::min)(frag_size, con_ran_line_len)))
-#else
-		if (! strncasecmp(hdr_buffer, con_ran_line, (std::min)(frag_size, con_ran_line_len)))
-#endif	// LL_WINDOWS
+		if (frag_size > con_ran_line_len &&
+			! os_strncasecmp(hdr_buffer, con_ran_line, con_ran_line_len))
 		{
 			unsigned int first(0), last(0), length(0);
 			int status;
@@ -656,22 +688,43 @@ size_t HttpOpRequest::headerCallback(void * data, size_t size, size_t nmemb, voi
 		}
 	}
 
-	if (op->mProcFlags & PF_SAVE_HEADERS)
+	// Detect and parse 'Content-Type' and 'Content-Encoding' headers
+	if (op->mProcFlags & PF_SCAN_CONTENT_HEADERS)
 	{
-		// Save headers in response
-		if (! op->mReplyHeaders)
+		if (wanted_hdr_size > con_type_line_len &&
+			! os_strncasecmp(hdr_data, con_type_line, con_type_line_len))
 		{
-			op->mReplyHeaders = new HttpHeaders;
+			// Found 'Content-Type:', extract single-token value
+			std::string rhs(hdr_data + con_type_line_len, wanted_hdr_size - con_type_line_len);
+			std::string::size_type begin(0), end(rhs.size()), pos;
+
+			if ((pos = rhs.find_first_not_of(hdr_whitespace)) != std::string::npos)
+			{
+				begin = pos;
+			}
+			if ((pos = rhs.find_first_of(hdr_whitespace, begin)) != std::string::npos)
+			{
+				end = pos;
+			}
+			op->mReplyConType.assign(rhs, begin, end - begin);
 		}
-		size_t wanted_size(hdr_size);
-		if (wanted_size && '\n' == hdr_data[wanted_size - 1])
+		else if (wanted_hdr_size > con_enc_line_len &&
+				 ! os_strncasecmp(hdr_data, con_enc_line, con_enc_line_len))
 		{
-			if (--wanted_size && '\r' == hdr_data[wanted_size - 1])
+			// Found 'Content-Encoding:', extract single-token value
+			std::string rhs(hdr_data + con_enc_line_len, wanted_hdr_size - con_enc_line_len);
+			std::string::size_type begin(0), end(rhs.size()), pos;
+
+			if ((pos = rhs.find_first_not_of(hdr_whitespace)) != std::string::npos)
 			{
-				--wanted_size;
+				begin = pos;
 			}
+			if ((pos = rhs.find_first_of(hdr_whitespace, begin)) != std::string::npos)
+			{
+				end = pos;
+			}
+			op->mReplyConEncode.assign(rhs, begin, end - begin);
 		}
-		op->mReplyHeaders->mHeaders.push_back(std::string(hdr_data, wanted_size));
 	}
 
 	return hdr_size;
@@ -788,15 +841,11 @@ int parse_content_range_header(char * buffer,
 	char * tok_state(NULL), * tok(NULL);
 	bool match(true);
 			
-	if (! strtok_r(buffer, ": \t", &tok_state))
+	if (! os_strtok_r(buffer, hdr_separator, &tok_state))
 		match = false;
-	if (match && (tok = strtok_r(NULL, " \t", &tok_state)))
-#if LL_WINDOWS
-		match = 0 == _stricmp("bytes", tok);
-#else
-		match = 0 == strcasecmp("bytes", tok);
-#endif // LL_WINDOWS
-	if (match && ! (tok = strtok_r(NULL, " \t", &tok_state)))
+	if (match && (tok = os_strtok_r(NULL, hdr_whitespace, &tok_state)))
+		match = 0 == os_strcasecmp("bytes", tok);
+	if (match && ! (tok = os_strtok_r(NULL, " \t", &tok_state)))
 		match = false;
 	if (match)
 	{
@@ -834,15 +883,6 @@ int parse_content_range_header(char * buffer,
 	return 1;
 }
 
-#if LL_WINDOWS
-
-char *strtok_r(char *str, const char *delim, char ** savestate)
-{
-	return strtok_s(str, delim, savestate);
-}
-
-#endif // LL_WINDOWS
-
 
 void escape_libcurl_debug_data(char * buffer, size_t len, bool scrub, std::string & safe_line)
 {
@@ -880,6 +920,36 @@ void escape_libcurl_debug_data(char * buffer, size_t len, bool scrub, std::strin
 }
 
 
+int os_strncasecmp(const char *s1, const char *s2, size_t n)
+{
+#if LL_WINDOWS
+	return _strnicmp(s1, s2, n);
+#else
+	return strncasecmp(s1, s2, n);
+#endif	// LL_WINDOWS
+}
+
+
+int os_strcasecmp(const char *s1, const char *s2)
+{
+#if LL_WINDOWS
+	return _stricmp(s1, s2);
+#else
+	return strcasecmp(s1, s2);
+#endif // LL_WINDOWS
+}
+
+
+char * os_strtok_r(char *str, const char *delim, char ** savestate)
+{
+#if LL_WINDOWS
+	return strtok_s(str, delim, savestate);
+#else
+	return strtok_r(str, delim, savestate);
+#endif
+}
+
+
 }  // end anonymous namespace
 
 		
diff --git a/indra/llcorehttp/_httpoprequest.h b/indra/llcorehttp/_httpoprequest.h
index a4c5fbb3c2..200b925c4e 100644
--- a/indra/llcorehttp/_httpoprequest.h
+++ b/indra/llcorehttp/_httpoprequest.h
@@ -30,6 +30,7 @@
 
 #include "linden_common.h"		// Modifies curl/curl.h interfaces
 
+#include <string>
 #include <curl/curl.h>
 
 #include "httpcommon.h"
@@ -137,6 +138,7 @@ protected:
 	unsigned int		mProcFlags;
 	static const unsigned int	PF_SCAN_RANGE_HEADER = 0x00000001U;
 	static const unsigned int	PF_SAVE_HEADERS = 0x00000002U;
+	static const unsigned int	PF_SCAN_CONTENT_HEADERS = 0x00000004U;
 
 public:
 	// Request data
@@ -162,6 +164,8 @@ public:
 	size_t				mReplyLength;
 	size_t				mReplyFullLength;
 	HttpHeaders *		mReplyHeaders;
+	std::string			mReplyConType;
+	std::string			mReplyConEncode;
 
 	// Policy data
 	int					mPolicyRetries;
diff --git a/indra/llcorehttp/httpresponse.h b/indra/llcorehttp/httpresponse.h
index 65e403cec8..3d35050c17 100644
--- a/indra/llcorehttp/httpresponse.h
+++ b/indra/llcorehttp/httpresponse.h
@@ -28,6 +28,8 @@
 #define	_LLCORE_HTTP_RESPONSE_H_
 
 
+#include <string>
+
 #include "httpcommon.h"
 
 #include "_refcounted.h"
@@ -130,7 +132,20 @@ public:
 			mReplyLength = length;
 			mReplyFullLength = full_length;
 		}
-			
+
+	///
+	void getContent(std::string & con_type, std::string & con_encode) const
+		{
+			con_type = mContentType;
+			con_encode = mContentEncoding;
+		}
+
+	void setContent(const std::string & con_type, const std::string & con_encode)
+		{
+			mContentType = con_type;
+			mContentEncoding = con_encode;
+		}
+
 protected:
 	// Response data here
 	HttpStatus			mStatus;
@@ -139,6 +154,8 @@ protected:
 	unsigned int		mReplyFullLength;
 	BufferArray *		mBufferArray;
 	HttpHeaders *		mHeaders;
+	std::string			mContentType;
+	std::string			mContentEncoding;
 };
 
 
diff --git a/indra/llcorehttp/tests/test_httprequest.hpp b/indra/llcorehttp/tests/test_httprequest.hpp
index ba7c757af4..9edf3d19ec 100644
--- a/indra/llcorehttp/tests/test_httprequest.hpp
+++ b/indra/llcorehttp/tests/test_httprequest.hpp
@@ -117,6 +117,15 @@ public:
 				ensure("Special header X-LL-Special in response", found_special);
 			}
 			
+			if (! mCheckContentType.empty())
+			{
+				ensure("Response required with content type check", response != NULL);
+				std::string con_type, con_enc;
+				response->getContent(con_type, con_enc);
+				ensure("Content-Type as expected (" + mCheckContentType + ")",
+					   mCheckContentType == con_type);
+			}
+
 			// std::cout << "TestHandler2::onCompleted() invoked" << std::endl;
 		}
 
@@ -124,6 +133,7 @@ public:
 	std::string mName;
 	HttpHandle mExpectHandle;
 	bool mCheckHeader;
+	std::string mCheckContentType;
 };
 
 typedef test_group<HttpRequestTestData> HttpRequestTestGroupType;
@@ -1502,6 +1512,122 @@ void HttpRequestTestObjectType::test<14>()
 	}
 }
 
+// Test retrieval of Content-Type/Content-Encoding headers
+template <> template <>
+void HttpRequestTestObjectType::test<15>()
+{
+	ScopedCurlInit ready;
+
+	std::string url_base(get_base_url());
+	// std::cerr << "Base:  "  << url_base << std::endl;
+	
+	set_test_name("HttpRequest GET with Content-Type");
+
+	// Handler can be stack-allocated *if* there are no dangling
+	// references to it after completion of this method.
+	// Create before memory record as the string copy will bump numbers.
+	TestHandler2 handler(this, "handler");
+
+	// Load and clear the string setting to preload std::string object
+	// for memory return tests.
+	handler.mCheckContentType = "application/llsd+xml";
+	handler.mCheckContentType.clear();
+		
+	// record the total amount of dynamically allocated memory
+	mMemTotal = GetMemTotal();
+	mHandlerCalls = 0;
+
+	HttpRequest * req = NULL;
+
+	try
+	{
+		// Get singletons created
+		HttpRequest::createService();
+		
+		// Start threading early so that thread memory is invariant
+		// over the test.
+		HttpRequest::startThread();
+
+		// create a new ref counted object with an implicit reference
+		req = new HttpRequest();
+		ensure("Memory allocated on construction", mMemTotal < GetMemTotal());
+
+		// Issue a GET that *can* connect
+		mStatus = HttpStatus(200);
+		handler.mCheckContentType = "application/llsd+xml";
+		HttpHandle handle = req->requestGet(HttpRequest::DEFAULT_POLICY_ID,
+											0U,
+											url_base,
+											NULL,
+											NULL,
+											&handler);
+		ensure("Valid handle returned for ranged request", handle != LLCORE_HTTP_HANDLE_INVALID);
+
+		// Run the notification pump.
+		int count(0);
+		int limit(10);
+		while (count++ < limit && mHandlerCalls < 1)
+		{
+			req->update(1000000);
+			usleep(100000);
+		}
+		ensure("Request executed in reasonable time", count < limit);
+		ensure("One handler invocation for request", mHandlerCalls == 1);
+
+		// Okay, request a shutdown of the servicing thread
+		mStatus = HttpStatus();
+		handler.mCheckContentType.clear();
+		handle = req->requestStopThread(&handler);
+		ensure("Valid handle returned for second request", handle != LLCORE_HTTP_HANDLE_INVALID);
+	
+		// Run the notification pump again
+		count = 0;
+		limit = 10;
+		while (count++ < limit && mHandlerCalls < 2)
+		{
+			req->update(1000000);
+			usleep(100000);
+		}
+		ensure("Second request executed in reasonable time", count < limit);
+		ensure("Second handler invocation", mHandlerCalls == 2);
+
+		// See that we actually shutdown the thread
+		count = 0;
+		limit = 10;
+		while (count++ < limit && ! HttpService::isStopped())
+		{
+			usleep(100000);
+		}
+		ensure("Thread actually stopped running", HttpService::isStopped());
+	
+		// release the request object
+		delete req;
+		req = NULL;
+
+		// Shut down service
+		HttpRequest::destroyService();
+	
+		ensure("Two handler calls on the way out", 2 == mHandlerCalls);
+
+#if defined(WIN32)
+		// Can only do this memory test on Windows.  On other platforms,
+		// the LL logging system holds on to memory and produces what looks
+		// like memory leaks...
+	
+		// printf("Old mem:  %d, New mem:  %d\n", mMemTotal, GetMemTotal());
+		ensure("Memory usage back to that at entry", mMemTotal == GetMemTotal());
+#endif
+	}
+	catch (...)
+	{
+		stop_thread(req);
+		delete req;
+		HttpRequest::destroyService();
+		throw;
+	}
+}
+
+
 }  // end namespace tut
 
 namespace
diff --git a/indra/llcorehttp/tests/test_llcorehttp_peer.py b/indra/llcorehttp/tests/test_llcorehttp_peer.py
index 9e1847c66e..58b5bedd09 100644
--- a/indra/llcorehttp/tests/test_llcorehttp_peer.py
+++ b/indra/llcorehttp/tests/test_llcorehttp_peer.py
@@ -9,7 +9,7 @@
 
 $LicenseInfo:firstyear=2008&license=viewerlgpl$
 Second Life Viewer Source Code
-Copyright (C) 2010, Linden Research, Inc.
+Copyright (C) 2012, Linden Research, Inc.
 
 This library is free software; you can redistribute it and/or
 modify it under the terms of the GNU Lesser General Public