SH-4257 Preparation for a new cap grant: GetMesh2

Mesh repo is using three policy classes now:  one for
large objects, one for GetMesh2 regions, one for
GetMesh regions.  It's also detecting the presence
of the cap and using the correct class.  Class
initialization cleaned up significantly in llappcorehttp
using data-directed code.  Pulled in the changes to
HttpHeader done for sunshine-internal then did a
refactoring pass on the header callback which now
uses a unified approach to clean up and deliver
header information to all interested parties.  Added
support for using Retry-After header information on
503 retries.

9 files changed, 222 insertions, 71 deletions

diff --git a/indra/llcorehttp/_httpinternal.h b/indra/llcorehttp/_httpinternal.h
index d60996756f..f085ca3b91 100755
--- a/indra/llcorehttp/_httpinternal.h
+++ b/indra/llcorehttp/_httpinternal.h
@@ -98,7 +98,7 @@ namespace LLCore
 
 // Maxium number of policy classes that can be defined.
 // *TODO:  Currently limited to the default class + 1, extend.
-const int HTTP_POLICY_CLASS_LIMIT = 4;
+const int HTTP_POLICY_CLASS_LIMIT = 8;
 
 // Debug/informational tracing.  Used both
 // as a global option and in per-request traces.
@@ -138,6 +138,10 @@ const int HTTP_CONNECTION_LIMIT_DEFAULT = 8;
 const int HTTP_CONNECTION_LIMIT_MIN = 1;
 const int HTTP_CONNECTION_LIMIT_MAX = 256;
 
+// Miscellaneous defaults
+const long HTTP_PIPELINING_DEFAULT = 0L;
+const bool HTTP_USE_RETRY_AFTER_DEFAULT = true;
+
 // Tuning parameters
 
 // Time worker thread sleeps after a pass through the
diff --git a/indra/llcorehttp/_httpoperation.cpp b/indra/llcorehttp/_httpoperation.cpp
index 5cf5bc5930..7acd728bbd 100755
--- a/indra/llcorehttp/_httpoperation.cpp
+++ b/indra/llcorehttp/_httpoperation.cpp
@@ -4,7 +4,7 @@
  *
  * $LicenseInfo:firstyear=2012&license=viewerlgpl$
  * Second Life Viewer Source Code
- * Copyright (C) 2012, Linden Research, Inc.
+ * Copyright (C) 2012-2013, Linden Research, Inc.
  *
  * This library is free software; you can redistribute it and/or
  * modify it under the terms of the GNU Lesser General Public
@@ -53,7 +53,7 @@ HttpOperation::HttpOperation()
 	  mUserHandler(NULL),
 	  mReqPolicy(HttpRequest::DEFAULT_POLICY_ID),
 	  mReqPriority(0U),
-	  mTracing(0)
+	  mTracing(HTTP_TRACE_OFF)
 {
 	mMetricCreated = totalTime();
 }
diff --git a/indra/llcorehttp/_httpoprequest.cpp b/indra/llcorehttp/_httpoprequest.cpp
index a4c0a12fdc..8cb7fee701 100755
--- a/indra/llcorehttp/_httpoprequest.cpp
+++ b/indra/llcorehttp/_httpoprequest.cpp
@@ -64,6 +64,15 @@ int parse_content_range_header(char * buffer,
 							   unsigned int * last,
 							   unsigned int * length);
 
+// Similar for Retry-After headers.  Only parses the delta form
+// of the header, HTTP time formats aren't interesting for client
+// purposes.
+//
+// @return		0 if successfully parsed and seconds time delta
+//				returned in time argument.
+//
+int parse_retry_after_header(char * buffer, int * time);
+
 
 // Take data from libcurl's CURLOPT_DEBUGFUNCTION callback and
 // escape and format it for a tracing line in logging.  Absolutely
@@ -74,14 +83,12 @@ void escape_libcurl_debug_data(char * buffer, size_t len, bool scrub,
 							   std::string & safe_line);
 
 
-// OS-neutral string comparisons of various types
-int os_strncasecmp(const char *s1, const char *s2, size_t n);
-int os_strcasecmp(const char *s1, const char *s2);
-char * os_strtok_r(char *str, const char *delim, char **saveptr);
-
-
-static const char * const hdr_whitespace(" \t");
-static const char * const hdr_separator(": \t");
+// OS-neutral string comparisons of various types.
+int os_strcasecmp(const char * s1, const char * s2);
+char * os_strtok_r(char * str, const char * delim, char ** saveptr);
+char * os_strtrim(char * str);
+char * os_strltrim(char * str);
+void os_strlower(char * str);
 
 } // end anonymous namespace
 
@@ -104,6 +111,8 @@ HttpOpRequest::HttpOpRequest()
 	  mCurlService(NULL),
 	  mCurlHeaders(NULL),
 	  mCurlBodyPos(0),
+	  mCurlTemp(NULL),
+	  mCurlTempLen(0),
 	  mReplyBody(NULL),
 	  mReplyOffset(0),
 	  mReplyLength(0),
@@ -154,6 +163,10 @@ HttpOpRequest::~HttpOpRequest()
 		mCurlHeaders = NULL;
 	}
 
+	delete [] mCurlTemp;
+	mCurlTemp = NULL;
+	mCurlTempLen = 0;
+	
 	if (mReplyBody)
 	{
 		mReplyBody->release();
@@ -207,6 +220,11 @@ void HttpOpRequest::stageFromActive(HttpService * service)
 		mCurlHeaders = NULL;
 	}
 
+	// Also not needed on the other side
+	delete [] mCurlTemp;
+	mCurlTemp = NULL;
+	mCurlTempLen = 0;
+	
 	addAsReply();
 }
 
@@ -335,6 +353,10 @@ void HttpOpRequest::setupCommon(HttpRequest::policy_t policy_id,
 		{
 			mProcFlags |= PF_SAVE_HEADERS;
 		}
+		if (options->getUseRetryAfter())
+		{
+			mProcFlags |= PF_USE_RETRY_AFTER;
+		}
 		mPolicyRetryLimit = options->getRetries();
 		mPolicyRetryLimit = llclamp(mPolicyRetryLimit, HTTP_RETRY_COUNT_MIN, HTTP_RETRY_COUNT_MAX);
 		mTracing = (std::max)(mTracing, llclamp(options->getTrace(), HTTP_TRACE_MIN, HTTP_TRACE_MAX));
@@ -549,7 +571,7 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service)
 	}
 	curl_easy_setopt(mCurlHandle, CURLOPT_HTTPHEADER, mCurlHeaders);
 
-	if (mProcFlags & (PF_SCAN_RANGE_HEADER | PF_SAVE_HEADERS))
+	if (mProcFlags & (PF_SCAN_RANGE_HEADER | PF_SAVE_HEADERS | PF_USE_RETRY_AFTER))
 	{
 		curl_easy_setopt(mCurlHandle, CURLOPT_HEADERFUNCTION, headerCallback);
 		curl_easy_setopt(mCurlHandle, CURLOPT_HEADERDATA, this);
@@ -610,10 +632,9 @@ size_t HttpOpRequest::headerCallback(void * data, size_t size, size_t nmemb, voi
 {
 	static const char status_line[] = "HTTP/";
 	static const size_t status_line_len = sizeof(status_line) - 1;
-
-	static const char con_ran_line[] = "content-range:";
-	static const size_t con_ran_line_len = sizeof(con_ran_line) - 1;
-
+	static const char con_ran_line[] = "content-range";
+	static const char con_retry_line[] = "retry-after";
+	
 	HttpOpRequest * op(static_cast<HttpOpRequest *>(userdata));
 
 	const size_t hdr_size(size * nmemb);
@@ -627,6 +648,7 @@ size_t HttpOpRequest::headerCallback(void * data, size_t size, size_t nmemb, voi
 		op->mReplyOffset = 0;
 		op->mReplyLength = 0;
 		op->mReplyFullLength = 0;
+		op->mReplyRetryAfter = 0;
 		op->mStatus = HttpStatus();
 		if (op->mReplyHeaders)
 		{
@@ -645,6 +667,53 @@ size_t HttpOpRequest::headerCallback(void * data, size_t size, size_t nmemb, voi
 			--wanted_hdr_size;
 		}
 	}
+
+	// Copy and normalize header fragments for the following
+	// stages.  Would like to modify the data in-place but that
+	// may not be allowed and we need one byte extra for NUL.
+	// At the end of this we will have:
+	//
+	// If ':' present in header:
+	//   1.  name points to text to left of colon which
+	//       will be ascii lower-cased and left and right
+	//       trimmed of whitespace.
+	//   2.  value points to text to right of colon which
+	//       will be left trimmed of whitespace.
+	// Otherwise:
+	//   1.  name points to header which will be left
+	//       trimmed of whitespace.
+	//   2.  value is NULL
+	// Any non-NULL pointer may point to a zero-length string.
+	//
+	if (wanted_hdr_size >= op->mCurlTempLen)
+	{
+		delete [] op->mCurlTemp;
+		op->mCurlTempLen = 2 * wanted_hdr_size + 1;
+		op->mCurlTemp = new char [op->mCurlTempLen];
+	}
+	memcpy(op->mCurlTemp, hdr_data, wanted_hdr_size);
+	op->mCurlTemp[wanted_hdr_size] = '\0';
+	char * name(op->mCurlTemp);
+	char * value(strchr(name, ':'));
+	if (value)
+	{
+		*value++ = '\0';
+		os_strlower(name);
+		name = os_strtrim(name);
+		value = os_strltrim(value);
+	}
+	else
+	{
+		// Doesn't look well-formed, do minimal normalization on it
+		name = os_strltrim(name);
+	}
+
+	// Normalized, now reject headers with empty names.
+	if (! *name)
+	{
+		// No use continuing
+		return hdr_size;
+	}
 	
 	// Save header if caller wants them in the response
 	if (is_header && op->mProcFlags & PF_SAVE_HEADERS)
@@ -654,43 +723,53 @@ size_t HttpOpRequest::headerCallback(void * data, size_t size, size_t nmemb, voi
 		{
 			op->mReplyHeaders = new HttpHeaders;
 		}
-		op->mReplyHeaders->appendNormal(hdr_data, wanted_hdr_size);
+		op->mReplyHeaders->append(name, value ? value : "");
 	}
 
+	// From this point, header-specific processors are free to
+	// modify the header value.
+	
 	// Detect and parse 'Content-Range' headers
-	if (is_header && op->mProcFlags & PF_SCAN_RANGE_HEADER)
+	if (is_header
+		&& op->mProcFlags & PF_SCAN_RANGE_HEADER
+		&& value && *value
+		&& ! strcmp(name, con_ran_line))
 	{
-		char hdr_buffer[128];			// Enough for a reasonable header
-		size_t frag_size((std::min)(wanted_hdr_size, sizeof(hdr_buffer) - 1));
-		
-		memcpy(hdr_buffer, hdr_data, frag_size);
-		hdr_buffer[frag_size] = '\0';
-		if (frag_size > con_ran_line_len &&
-			! os_strncasecmp(hdr_buffer, con_ran_line, con_ran_line_len))
+		unsigned int first(0), last(0), length(0);
+		int status;
+
+		if (! (status = parse_content_range_header(value, &first, &last, &length)))
+		{
+			// Success, record the fragment position
+			op->mReplyOffset = first;
+			op->mReplyLength = last - first + 1;
+			op->mReplyFullLength = length;
+		}
+		else if (-1 == status)
 		{
-			unsigned int first(0), last(0), length(0);
-			int status;
+			// Response is badly formed and shouldn't be accepted
+			op->mStatus = HttpStatus(HttpStatus::LLCORE, HE_INV_CONTENT_RANGE_HDR);
+		}
+		else
+		{
+			// Ignore the unparsable.
+			LL_INFOS_ONCE("CoreHttp") << "Problem parsing odd Content-Range header:  '"
+									  << std::string(hdr_data, wanted_hdr_size)
+									  << "'.  Ignoring."
+									  << LL_ENDL;
+		}
+	}
 
-			if (! (status = parse_content_range_header(hdr_buffer, &first, &last, &length)))
-			{
-				// Success, record the fragment position
-				op->mReplyOffset = first;
-				op->mReplyLength = last - first + 1;
-				op->mReplyFullLength = length;
-			}
-			else if (-1 == status)
-			{
-				// Response is badly formed and shouldn't be accepted
-				op->mStatus = HttpStatus(HttpStatus::LLCORE, HE_INV_CONTENT_RANGE_HDR);
-			}
-			else
-			{
-				// Ignore the unparsable.
-				LL_INFOS_ONCE("CoreHttp") << "Problem parsing odd Content-Range header:  '"
-										  << std::string(hdr_data, frag_size)
-										  << "'.  Ignoring."
-										  << LL_ENDL;
-			}
+	// Detect and parse 'Retry-After' headers
+	if (is_header
+		&& op->mProcFlags & PF_USE_RETRY_AFTER
+		&& value && *value
+		&& ! strcmp(name, con_retry_line))
+	{
+		int time(0);
+		if (! parse_retry_after_header(value, &time))
+		{
+			op->mReplyRetryAfter = time;
 		}
 	}
 
@@ -805,14 +884,16 @@ int parse_content_range_header(char * buffer,
 							   unsigned int * last,
 							   unsigned int * length)
 {
+	static const char * const hdr_whitespace(" \t");
+
 	char * tok_state(NULL), * tok(NULL);
 	bool match(true);
 			
-	if (! os_strtok_r(buffer, hdr_separator, &tok_state))
+	if (! (tok = os_strtok_r(buffer, hdr_whitespace, &tok_state)))
 		match = false;
-	if (match && (tok = os_strtok_r(NULL, hdr_whitespace, &tok_state)))
-		match = 0 == os_strcasecmp("bytes", tok);
-	if (match && ! (tok = os_strtok_r(NULL, " \t", &tok_state)))
+	else
+		match = (0 == os_strcasecmp("bytes", tok));
+	if (match && ! (tok = os_strtok_r(NULL, hdr_whitespace, &tok_state)))
 		match = false;
 	if (match)
 	{
@@ -851,6 +932,25 @@ int parse_content_range_header(char * buffer,
 }
 
 
+int parse_retry_after_header(char * buffer, int * time)
+{
+	char * endptr(buffer);
+	long lcl_time(strtol(buffer, &endptr, 10));
+	if (*endptr == '\0' && endptr != buffer && lcl_time > 0)
+	{
+		*time = lcl_time;
+		return 0;
+	}
+
+	// Could attempt to parse HTTP time here but we're not really
+	// interested in it.  Scheduling based on wallclock time on
+	// user hardware will lead to tears.
+	
+	// Header is there but badly/unexpectedly formed, try to ignore it.
+	return 1;
+}
+
+
 void escape_libcurl_debug_data(char * buffer, size_t len, bool scrub, std::string & safe_line)
 {
 	std::string out;
@@ -887,15 +987,6 @@ void escape_libcurl_debug_data(char * buffer, size_t len, bool scrub, std::strin
 }
 
 
-int os_strncasecmp(const char *s1, const char *s2, size_t n)
-{
-#if LL_WINDOWS
-	return _strnicmp(s1, s2, n);
-#else
-	return strncasecmp(s1, s2, n);
-#endif	// LL_WINDOWS
-}
-
 
 int os_strcasecmp(const char *s1, const char *s2)
 {
@@ -917,6 +1008,45 @@ char * os_strtok_r(char *str, const char *delim, char ** savestate)
 }
 
 
+void os_strlower(char * str)
+{
+	for (char c(0); (c = *str); ++str)
+	{
+		*str = tolower(c);
+	}
+}
+
+
+char * os_strtrim(char * lstr)
+{
+	while (' ' == *lstr || '\t' == *lstr)
+	{
+		++lstr;
+	}
+	if (*lstr)
+	{
+		for (char * rstr(lstr + strlen(lstr)); *--rstr;)
+		{
+			if (' ' == *rstr || '\t' == *rstr)
+			{
+				*rstr = '\0';
+			}
+		}
+	}
+	return lstr;
+}
+
+
+char * os_strltrim(char * lstr)
+{
+	while (' ' == *lstr || '\t' == *lstr)
+	{
+		++lstr;
+	}
+	return lstr;
+}
+
+
 }  // end anonymous namespace
 
 		
diff --git a/indra/llcorehttp/_httpoprequest.h b/indra/llcorehttp/_httpoprequest.h
index 831e5bebf7..2e737cf1cc 100755
--- a/indra/llcorehttp/_httpoprequest.h
+++ b/indra/llcorehttp/_httpoprequest.h
@@ -158,6 +158,7 @@ protected:
 	unsigned int		mProcFlags;
 	static const unsigned int	PF_SCAN_RANGE_HEADER = 0x00000001U;
 	static const unsigned int	PF_SAVE_HEADERS = 0x00000002U;
+	static const unsigned int	PF_USE_RETRY_AFTER = 0x00000004U;
 
 public:
 	// Request data
@@ -175,6 +176,8 @@ public:
 	HttpService *		mCurlService;
 	curl_slist *		mCurlHeaders;
 	size_t				mCurlBodyPos;
+	char *				mCurlTemp;				// Scratch buffer for header processing
+	size_t				mCurlTempLen;
 	
 	// Result data
 	HttpStatus			mStatus;
@@ -184,6 +187,7 @@ public:
 	size_t				mReplyFullLength;
 	HttpHeaders *		mReplyHeaders;
 	std::string			mReplyConType;
+	int					mReplyRetryAfter;
 
 	// Policy data
 	int					mPolicyRetries;
diff --git a/indra/llcorehttp/_httppolicy.cpp b/indra/llcorehttp/_httppolicy.cpp
index 54c9c6bb1b..5f303dd0fe 100755
--- a/indra/llcorehttp/_httppolicy.cpp
+++ b/indra/llcorehttp/_httppolicy.cpp
@@ -160,8 +160,12 @@ void HttpPolicy::retryOp(HttpOpRequest * op)
 	
 	const HttpTime now(totalTime());
 	const int policy_class(op->mReqPolicy);
-	
-	const HttpTime delta(retry_deltas[llclamp(op->mPolicyRetries, 0, delta_max)]);
+	HttpTime delta(retry_deltas[llclamp(op->mPolicyRetries, 0, delta_max)]);
+
+	if (op->mReplyRetryAfter > 0 && op->mReplyRetryAfter < 30)
+	{
+		delta = op->mReplyRetryAfter * U64L(1000000);
+	}
 	op->mPolicyRetryAt = now + delta;
 	++op->mPolicyRetries;
 	if (error_503 == op->mStatus)
@@ -170,10 +174,10 @@ void HttpPolicy::retryOp(HttpOpRequest * op)
 	}
 	LL_WARNS("CoreHttp") << "HTTP request " << static_cast<HttpHandle>(op)
 						 << " retry " << op->mPolicyRetries
-						 << " scheduled for +" << (delta / HttpTime(1000))
+						 << " scheduled in " << (delta / HttpTime(1000))
 						 << " mS.  Status:  " << op->mStatus.toHex()
 						 << LL_ENDL;
-	if (op->mTracing > 0)
+	if (op->mTracing > HTTP_TRACE_OFF)
 	{
 		LL_INFOS("CoreHttp") << "TRACE, ToRetryQueue, Handle:  "
 							 << static_cast<HttpHandle>(op)
diff --git a/indra/llcorehttp/_httppolicyclass.cpp b/indra/llcorehttp/_httppolicyclass.cpp
index a23b81322c..1a55ab1ac6 100755
--- a/indra/llcorehttp/_httppolicyclass.cpp
+++ b/indra/llcorehttp/_httppolicyclass.cpp
@@ -4,7 +4,7 @@
  *
  * $LicenseInfo:firstyear=2012&license=viewerlgpl$
  * Second Life Viewer Source Code
- * Copyright (C) 2012, Linden Research, Inc.
+ * Copyright (C) 2012-2013, Linden Research, Inc.
  *
  * This library is free software; you can redistribute it and/or
  * modify it under the terms of the GNU Lesser General Public
@@ -37,7 +37,7 @@ HttpPolicyClass::HttpPolicyClass()
 	: mSetMask(0UL),
 	  mConnectionLimit(HTTP_CONNECTION_LIMIT_DEFAULT),
 	  mPerHostConnectionLimit(HTTP_CONNECTION_LIMIT_DEFAULT),
-	  mPipelining(0)
+	  mPipelining(HTTP_PIPELINING_DEFAULT)
 {}
 
 
diff --git a/indra/llcorehttp/_httpservice.cpp b/indra/llcorehttp/_httpservice.cpp
index 0825888d0f..0821401289 100755
--- a/indra/llcorehttp/_httpservice.cpp
+++ b/indra/llcorehttp/_httpservice.cpp
@@ -55,9 +55,6 @@ HttpService::HttpService()
 {
 	// Create the default policy class
 	HttpPolicyClass pol_class;
-	pol_class.set(HttpRequest::CP_CONNECTION_LIMIT, HTTP_CONNECTION_LIMIT_DEFAULT);
-	pol_class.set(HttpRequest::CP_PER_HOST_CONNECTION_LIMIT, HTTP_CONNECTION_LIMIT_DEFAULT);
-	pol_class.set(HttpRequest::CP_ENABLE_PIPELINING, 0L);
 	mPolicyClasses.push_back(pol_class);
 }
 
diff --git a/indra/llcorehttp/httpoptions.cpp b/indra/llcorehttp/httpoptions.cpp
index 4dcd862ca4..5bf1ecb4a5 100755
--- a/indra/llcorehttp/httpoptions.cpp
+++ b/indra/llcorehttp/httpoptions.cpp
@@ -39,7 +39,8 @@ HttpOptions::HttpOptions()
 	  mTracing(HTTP_TRACE_OFF),
 	  mTimeout(HTTP_REQUEST_TIMEOUT_DEFAULT),
 	  mTransferTimeout(HTTP_REQUEST_XFER_TIMEOUT_DEFAULT),
-	  mRetries(HTTP_RETRY_COUNT_DEFAULT)
+	  mRetries(HTTP_RETRY_COUNT_DEFAULT),
+	  mUseRetryAfter(HTTP_USE_RETRY_AFTER_DEFAULT)
 {}
 
 
@@ -76,5 +77,10 @@ void HttpOptions::setRetries(unsigned int retries)
 	mRetries = retries;
 }
 
+void HttpOptions::setUseRetryAfter(bool use_retry)
+{
+	mUseRetryAfter = use_retry;
+}
+
 
 }   // end namespace LLCore
diff --git a/indra/llcorehttp/httpoptions.h b/indra/llcorehttp/httpoptions.h
index 623d71d3e6..04531425d8 100755
--- a/indra/llcorehttp/httpoptions.h
+++ b/indra/llcorehttp/httpoptions.h
@@ -98,13 +98,19 @@ public:
 			return mRetries;
 		}
 	
+	void				setUseRetryAfter(bool use_retry);
+	bool				getUseRetryAfter() const
+		{
+			return mUseRetryAfter;
+		}
+	
 protected:
 	bool				mWantHeaders;
 	int					mTracing;
 	unsigned int		mTimeout;
 	unsigned int		mTransferTimeout;
 	unsigned int		mRetries;
-	
+	bool				mUseRetryAfter;
 }; // end class HttpOptions