From 5cca78e718f15522cc3db9aec76aa910dd696aa8 Mon Sep 17 00:00:00 2001
From: Monty Brandenberg <monty@lindenlab.com>
Date: Mon, 23 Jun 2014 14:23:33 -0400
Subject: First HTTP pipelining viewer.  Enable pipelining for GetTexture and
 GetMesh2 at a pipeline depth of 5.  Create global debug option,
 HttpPipelining, to enable and disable HTTP pipelining (defaults to true). 
 Tweak texture and mesh low- and high-water request levels based on pipelining
 status and depth.  Fixup texture console which was damaged in a recent
 release.  Split logging of the no-request HTTP error case into two cases: 
 one for missing URL in HTTP request, one for HTTP request not created.  A
 refactor in llcorehttp is coming:  I will be moving all libcurl- using code
 into libcurl-specific modules.

---
 indra/llcorehttp/_httpinternal.h      |  5 ++-
 indra/llcorehttp/_httplibcurl.cpp     | 69 ++++++++++++++++++++++++++++++++++-
 indra/llcorehttp/_httppolicy.cpp      | 11 ++++--
 indra/llcorehttp/_httppolicyclass.cpp |  8 ++--
 indra/llcorehttp/httprequest.h        | 35 ++++++++++++++++--
 5 files changed, 114 insertions(+), 14 deletions(-)

(limited to 'indra/llcorehttp')

diff --git a/indra/llcorehttp/_httpinternal.h b/indra/llcorehttp/_httpinternal.h
index f80d7f60f5..a2a60ca056 100755
--- a/indra/llcorehttp/_httpinternal.h
+++ b/indra/llcorehttp/_httpinternal.h
@@ -145,8 +145,11 @@ const int HTTP_CONNECTION_LIMIT_DEFAULT = 8;
 const int HTTP_CONNECTION_LIMIT_MIN = 1;
 const int HTTP_CONNECTION_LIMIT_MAX = 256;
 
-// Miscellaneous defaults
+// Pipelining limits
 const long HTTP_PIPELINING_DEFAULT = 0L;
+const long HTTP_PIPELINING_MAX = 20L;
+
+// Miscellaneous defaults
 const bool HTTP_USE_RETRY_AFTER_DEFAULT = true;
 const long HTTP_THROTTLE_RATE_DEFAULT = 0L;
 
diff --git a/indra/llcorehttp/_httplibcurl.cpp b/indra/llcorehttp/_httplibcurl.cpp
index e56bc84174..fb907f6318 100755
--- a/indra/llcorehttp/_httplibcurl.cpp
+++ b/indra/llcorehttp/_httplibcurl.cpp
@@ -4,7 +4,7 @@
  *
  * $LicenseInfo:firstyear=2012&license=viewerlgpl$
  * Second Life Viewer Source Code
- * Copyright (C) 2012-2013, Linden Research, Inc.
+ * Copyright (C) 2012-2014, Linden Research, Inc.
  *
  * This library is free software; you can redistribute it and/or
  * modify it under the terms of the GNU Lesser General Public
@@ -33,6 +33,15 @@
 
 #include "llhttpconstants.h"
 
+namespace
+{
+
+// Error testing and reporting for libcurl status codes
+void check_curl_multi_code(CURLMcode code);
+void check_curl_multi_code(CURLMcode code, int curl_setopt_option);
+
+} // end anonymous namespace
+
 
 namespace LLCore
 {
@@ -92,14 +101,44 @@ void HttpLibcurl::start(int policy_count)
 	llassert_always(policy_count <= HTTP_POLICY_CLASS_LIMIT);
 	llassert_always(! mMultiHandles);					// One-time call only
 	
+	HttpPolicy & policy(mService->getPolicy());
 	mPolicyCount = policy_count;
 	mMultiHandles = new CURLM * [mPolicyCount];
 	mActiveHandles = new int [mPolicyCount];
 	
 	for (int policy_class(0); policy_class < mPolicyCount; ++policy_class)
 	{
-		mMultiHandles[policy_class] = curl_multi_init();
+		HttpPolicyClass & options(policy.getClassOptions(policy_class));
+
 		mActiveHandles[policy_class] = 0;
+		if (NULL == (mMultiHandles[policy_class] = curl_multi_init()))
+		{
+			LL_ERRS("CoreHttp") << "Failed to allocate multi handle in libcurl."
+								<< LL_ENDL;
+		}
+				
+		if (options.mPipelining > 1)
+		{
+			CURLMcode code;
+			
+			// We'll try to do pipelining on this multihandle
+			code = curl_multi_setopt(mMultiHandles[policy_class],
+									 CURLMOPT_PIPELINING,
+									 1L);
+			check_curl_multi_code(code, CURLMOPT_PIPELINING);
+			code = curl_multi_setopt(mMultiHandles[policy_class],
+									 CURLMOPT_MAX_PIPELINE_LENGTH,
+									 long(options.mPipelining));
+			check_curl_multi_code(code, CURLMOPT_MAX_PIPELINE_LENGTH);
+			code = curl_multi_setopt(mMultiHandles[policy_class],
+									 CURLMOPT_MAX_HOST_CONNECTIONS,
+									 long(options.mPerHostConnectionLimit));
+			check_curl_multi_code(code, CURLMOPT_MAX_HOST_CONNECTIONS);
+			code = curl_multi_setopt(mMultiHandles[policy_class],
+									 CURLMOPT_MAX_TOTAL_CONNECTIONS,
+									 long(options.mConnectionLimit));
+			check_curl_multi_code(code, CURLMOPT_MAX_TOTAL_CONNECTIONS);
+		}
 	}
 }
 
@@ -376,3 +415,29 @@ struct curl_slist * append_headers_to_slist(const HttpHeaders * headers, struct
 
 
 }  // end namespace LLCore
+
+
+namespace 
+{
+	
+void check_curl_multi_code(CURLMcode code, int curl_setopt_option)
+{
+	if (CURLM_OK != code)
+	{
+		LL_WARNS("CoreHttp") << "libcurl multi error detected:  " << curl_multi_strerror(code)
+							 << ", curl_multi_setopt option:  " << curl_setopt_option
+							 << LL_ENDL;
+	}
+}
+
+
+void check_curl_multi_code(CURLMcode code)
+{
+	if (CURLM_OK != code)
+	{
+		LL_WARNS("CoreHttp") << "libcurl multi error detected:  " << curl_multi_strerror(code)
+							 << LL_ENDL;
+	}
+}
+
+}  // end anonymous namespace
diff --git a/indra/llcorehttp/_httppolicy.cpp b/indra/llcorehttp/_httppolicy.cpp
index fd5a93e192..bb7959b578 100755
--- a/indra/llcorehttp/_httppolicy.cpp
+++ b/indra/llcorehttp/_httppolicy.cpp
@@ -4,7 +4,7 @@
  *
  * $LicenseInfo:firstyear=2012&license=viewerlgpl$
  * Second Life Viewer Source Code
- * Copyright (C) 2012-2013, Linden Research, Inc.
+ * Copyright (C) 2012-2014, Linden Research, Inc.
  *
  * This library is free software; you can redistribute it and/or
  * modify it under the terms of the GNU Lesser General Public
@@ -128,7 +128,8 @@ void HttpPolicy::shutdown()
 
 
 void HttpPolicy::start()
-{}
+{
+}
 
 
 void HttpPolicy::addOp(HttpOpRequest * op)
@@ -234,7 +235,11 @@ HttpService::ELoopSpeed HttpPolicy::processReadyQueue()
 		}
 
 		int active(transport.getActiveCountInClass(policy_class));
-		int needed(state.mOptions.mConnectionLimit - active);		// Expect negatives here
+		int active_limit(state.mOptions.mPipelining > 1L
+						 ? (state.mOptions.mPerHostConnectionLimit
+							* state.mOptions.mPipelining)
+						 : state.mOptions.mConnectionLimit);
+		int needed(active_limit - active);		// Expect negatives here
 
 		if (needed > 0)
 		{
diff --git a/indra/llcorehttp/_httppolicyclass.cpp b/indra/llcorehttp/_httppolicyclass.cpp
index f34a8e9f1e..2c0f650155 100755
--- a/indra/llcorehttp/_httppolicyclass.cpp
+++ b/indra/llcorehttp/_httppolicyclass.cpp
@@ -4,7 +4,7 @@
  *
  * $LicenseInfo:firstyear=2012&license=viewerlgpl$
  * Second Life Viewer Source Code
- * Copyright (C) 2012-2013, Linden Research, Inc.
+ * Copyright (C) 2012-2014, Linden Research, Inc.
  *
  * This library is free software; you can redistribute it and/or
  * modify it under the terms of the GNU Lesser General Public
@@ -78,8 +78,8 @@ HttpStatus HttpPolicyClass::set(HttpRequest::EPolicyOption opt, long value)
 		mPerHostConnectionLimit = llclamp(value, long(HTTP_CONNECTION_LIMIT_MIN), mConnectionLimit);
 		break;
 
-	case HttpRequest::PO_ENABLE_PIPELINING:
-		mPipelining = llclamp(value, 0L, 1L);
+	case HttpRequest::PO_PIPELINING_DEPTH:
+		mPipelining = llclamp(value, 0L, HTTP_PIPELINING_MAX);
 		break;
 
 	case HttpRequest::PO_THROTTLE_RATE:
@@ -106,7 +106,7 @@ HttpStatus HttpPolicyClass::get(HttpRequest::EPolicyOption opt, long * value) co
 		*value = mPerHostConnectionLimit;
 		break;
 
-	case HttpRequest::PO_ENABLE_PIPELINING:
+	case HttpRequest::PO_PIPELINING_DEPTH:
 		*value = mPipelining;
 		break;
 
diff --git a/indra/llcorehttp/httprequest.h b/indra/llcorehttp/httprequest.h
index 651654844a..7f23723b0b 100755
--- a/indra/llcorehttp/httprequest.h
+++ b/indra/llcorehttp/httprequest.h
@@ -4,7 +4,7 @@
  *
  * $LicenseInfo:firstyear=2012&license=viewerlgpl$
  * Second Life Viewer Source Code
- * Copyright (C) 2012-2013, Linden Research, Inc.
+ * Copyright (C) 2012-2014, Linden Research, Inc.
  *
  * This library is free software; you can redistribute it and/or
  * modify it under the terms of the GNU Lesser General Public
@@ -183,11 +183,38 @@ public:
 		/// Global only
 		PO_TRACE,
 
-		/// Suitable requests are allowed to pipeline on their
-		/// connections when they ask for it.
+		/// If greater than 1, suitable requests are allowed to
+		/// pipeline on their connections when they ask for it.
+		/// Value gives the maximum number of outstanding requests
+		/// on a connection.
+		///
+		/// There is some interaction between PO_CONNECTION_LIMIT,
+		/// PO_PER_HOST_CONNECTION_LIMIT, and PO_PIPELINING_DEPTH.
+		/// When PIPELINING_DEPTH is 0 or 1 (no pipelining), this
+		/// library manages connection lifecycle and honors the
+		/// PO_CONNECTION_LIMIT setting as the maximum in-flight
+		/// request limit.  Libcurl itself may be caching additional
+		/// connections under its connection cache policy.
+		///
+		/// When PIPELINING_DEPTH is 2 or more, libcurl performs
+		/// connection management and both PO_CONNECTION_LIMIT and
+		/// PO_PER_HOST_CONNECTION_LIMIT should be set and non-zero.
+		/// In this case (as of libcurl 7.37.0), libcurl will
+		/// open new connections in preference to pipelining, up
+		/// to the above limits at which time pipelining begins.
+		/// And as usual, an additional cache of open but inactive
+		/// connections may still be maintained within libcurl.
+		/// For SL, a good rule-of-thumb is to set
+		/// PO_PER_HOST_CONNECTION_LIMIT to the user-visible
+		/// concurrency value and PO_CONNECTION_LIMIT to twice
+		/// that for baked texture loads and region crossings where
+		/// additional connection load will be tolerated.  If
+		/// either limit is 0, libcurl will prefer pipelining
+		/// over connection creation, which is still interesting,
+		/// but won't be pursued at this time.
 		///
 		/// Per-class only
-		PO_ENABLE_PIPELINING,
+		PO_PIPELINING_DEPTH,
 
 		/// Controls whether client-side throttling should be
 		/// performed on this policy class.  Positive values
-- 
cgit v1.2.3


From 9fb96b416f0612882ddcaa658945a5b3f73ff331 Mon Sep 17 00:00:00 2001
From: Monty Brandenberg <monty@lindenlab.com>
Date: Thu, 3 Jul 2014 19:44:02 -0400
Subject: Add pipelining and tracing command line options to the test program.

---
 indra/llcorehttp/examples/http_texture_load.cpp | 64 +++++++++++++++++++++++--
 1 file changed, 59 insertions(+), 5 deletions(-)

(limited to 'indra/llcorehttp')

diff --git a/indra/llcorehttp/examples/http_texture_load.cpp b/indra/llcorehttp/examples/http_texture_load.cpp
index 73c49687d7..88692c3f69 100755
--- a/indra/llcorehttp/examples/http_texture_load.cpp
+++ b/indra/llcorehttp/examples/http_texture_load.cpp
@@ -4,7 +4,7 @@
  *
  * $LicenseInfo:firstyear=2012&license=viewerlgpl$
  * Second Life Viewer Source Code
- * Copyright (C) 2012-2013, Linden Research, Inc.
+ * Copyright (C) 2012-2014, Linden Research, Inc.
  * 
  * This library is free software; you can redistribute it and/or
  * modify it under the terms of the GNU Lesser General Public
@@ -59,11 +59,13 @@ void usage(std::ostream & out);
 // Default command line settings
 static int concurrency_limit(40);
 static int highwater(100);
+static int pipeline_depth(0);
+static int tracing(0);
 static char url_format[1024] = "http://example.com/some/path?texture_id=%s.texture";
 
 #if defined(WIN32)
 
-#define	strncpy(_a, _b, _c)   strncpy_s(_a, _b, _c)
+#define	strncpy(_a, _b, _c)			strncpy_s(_a, _b, _c)
 #define strtok_r(_a, _b, _c)		strtok_s(_a, _b, _c)
 
 int getopt(int argc, char * const argv[], const char *optstring);
@@ -163,7 +165,7 @@ int main(int argc, char** argv)
 	bool do_verbose(false);
 	
 	int option(-1);
-	while (-1 != (option = getopt(argc, argv, "u:c:h?RvH:")))
+	while (-1 != (option = getopt(argc, argv, "u:c:h?RvH:p:t:")))
 	{
 		switch (option)
 		{
@@ -193,7 +195,7 @@ int main(int argc, char** argv)
 				char * end;
 
 				value = strtoul(optarg, &end, 10);
-				if (value < 1 || value > 100 || *end != '\0')
+				if (value < 1 || value > 200 || *end != '\0')
 				{
 					usage(std::cerr);
 					return 1;
@@ -202,6 +204,36 @@ int main(int argc, char** argv)
 			}
 			break;
 
+		case 'p':
+		    {
+				unsigned long value;
+				char * end;
+
+				value = strtoul(optarg, &end, 10);
+				if (value < 0 || value > 100 || *end != '\0')
+				{
+					usage(std::cerr);
+					return 1;
+				}
+				pipeline_depth = value;
+			}
+			break;
+
+		case '5':
+		    {
+				unsigned long value;
+				char * end;
+
+				value = strtoul(optarg, &end, 10);
+				if (value < 0 || value > 3 || *end != '\0')
+				{
+					usage(std::cerr);
+					return 1;
+				}
+				tracing = value;
+			}
+			break;
+
 		case 'R':
 			do_random = true;
 			break;
@@ -240,6 +272,24 @@ int main(int argc, char** argv)
 											   LLCore::HttpRequest::DEFAULT_POLICY_ID,
 											   concurrency_limit,
 											   NULL);
+	LLCore::HttpRequest::setStaticPolicyOption(LLCore::HttpRequest::PO_PER_HOST_CONNECTION_LIMIT,
+											   LLCore::HttpRequest::DEFAULT_POLICY_ID,
+											   concurrency_limit,
+											   NULL);
+	if (pipeline_depth)
+	{
+		LLCore::HttpRequest::setStaticPolicyOption(LLCore::HttpRequest::PO_PIPELINING_DEPTH,
+												   LLCore::HttpRequest::DEFAULT_POLICY_ID,
+												   pipeline_depth,
+												   NULL);
+	}
+	if (tracing)
+	{
+		LLCore::HttpRequest::setStaticPolicyOption(LLCore::HttpRequest::PO_TRACE,
+												   LLCore::HttpRequest::DEFAULT_POLICY_ID,
+												   tracing,
+												   NULL);
+	}
 	LLCore::HttpRequest::startThread();
 	
 	// Get service point
@@ -334,7 +384,11 @@ void usage(std::ostream & out)
 		" -c <limit>            Maximum connection concurrency.  Range:  [1..100]\n"
 		"                       Default:  " << concurrency_limit << "\n"
 		" -H <limit>            HTTP request highwater (requests fed to llcorehttp).\n"
-		"                       Range:  [1..100]  Default:  " << highwater << "\n"
+		"                       Range:  [1..200]  Default:  " << highwater << "\n"
+		" -p <depth>            If <depth> is positive, enables and sets pipelineing\n"
+		"                       depth on HTTP requests.  Default:  " << pipeline_depth << "\n"
+		" -t <level>            If <level> is positive ([1..3]), enables and sets HTTP\n"
+		"                       tracing on HTTP requests.  Default:  " << tracing << "\n"
 		" -v                    Verbose mode.  Issue some chatter while running\n"
 		" -h                    print this help\n"
 		"\n"
-- 
cgit v1.2.3


From e79a88c8ccfadcd260892000d4dec2ae921b26de Mon Sep 17 00:00:00 2001
From: Monty Brandenberg <monty@lindenlab.com>
Date: Tue, 12 Aug 2014 18:21:26 -0400
Subject: Better support for dynamic option changes in llcorehttp.  Libcurl has
 some problems disabling pipelining on a multi handle with outstanding
 requests so build a more conservative system that allows requests to drain
 before setting curl multi options.  Would rather not have this but it is
 significantly safer.  "HttpPipelining" debug setting is now fully dynamic. 
 Connection limits can also be made dynamic in the near future.  Upped the
 default connection count back to 8 for now but will revisit this in the
 tuning phase.  It might be time to combine mesh and textures into a single
 asset class.  For normal server operations that would be a clear path, but
 for server under load, the current scheme may be better.  Minor cleanup in
 logging to elminate some redundant strings.  Might add some more tracing to
 the stall logic 'just in case'.

---
 indra/llcorehttp/_httplibcurl.cpp   | 198 +++++++++++++++++++++++++-----------
 indra/llcorehttp/_httplibcurl.h     |  12 ++-
 indra/llcorehttp/_httpoperation.cpp |  28 +++--
 indra/llcorehttp/_httpoprequest.cpp |  46 +++++----
 indra/llcorehttp/_httppolicy.cpp    |  83 ++++++++++-----
 indra/llcorehttp/_httppolicy.h      |   8 ++
 indra/llcorehttp/_httpservice.cpp   |  30 ++++--
 7 files changed, 276 insertions(+), 129 deletions(-)

(limited to 'indra/llcorehttp')

diff --git a/indra/llcorehttp/_httplibcurl.cpp b/indra/llcorehttp/_httplibcurl.cpp
index fb907f6318..b46833a1f3 100755
--- a/indra/llcorehttp/_httplibcurl.cpp
+++ b/indra/llcorehttp/_httplibcurl.cpp
@@ -40,6 +40,8 @@ namespace
 void check_curl_multi_code(CURLMcode code);
 void check_curl_multi_code(CURLMcode code, int curl_setopt_option);
 
+static const char * const LOG_CORE("CoreHttp");
+
 } // end anonymous namespace
 
 
@@ -51,7 +53,8 @@ HttpLibcurl::HttpLibcurl(HttpService * service)
 	: mService(service),
 	  mPolicyCount(0),
 	  mMultiHandles(NULL),
-	  mActiveHandles(NULL)
+	  mActiveHandles(NULL),
+	  mDirtyPolicy(NULL)
 {}
 
 
@@ -90,6 +93,9 @@ void HttpLibcurl::shutdown()
 
 		delete [] mActiveHandles;
 		mActiveHandles = NULL;
+
+		delete [] mDirtyPolicy;
+		mDirtyPolicy = NULL;
 	}
 
 	mPolicyCount = 0;
@@ -101,44 +107,21 @@ void HttpLibcurl::start(int policy_count)
 	llassert_always(policy_count <= HTTP_POLICY_CLASS_LIMIT);
 	llassert_always(! mMultiHandles);					// One-time call only
 	
-	HttpPolicy & policy(mService->getPolicy());
 	mPolicyCount = policy_count;
 	mMultiHandles = new CURLM * [mPolicyCount];
 	mActiveHandles = new int [mPolicyCount];
+	mDirtyPolicy = new bool [mPolicyCount];
 	
 	for (int policy_class(0); policy_class < mPolicyCount; ++policy_class)
 	{
-		HttpPolicyClass & options(policy.getClassOptions(policy_class));
-
-		mActiveHandles[policy_class] = 0;
 		if (NULL == (mMultiHandles[policy_class] = curl_multi_init()))
 		{
-			LL_ERRS("CoreHttp") << "Failed to allocate multi handle in libcurl."
-								<< LL_ENDL;
-		}
-				
-		if (options.mPipelining > 1)
-		{
-			CURLMcode code;
-			
-			// We'll try to do pipelining on this multihandle
-			code = curl_multi_setopt(mMultiHandles[policy_class],
-									 CURLMOPT_PIPELINING,
-									 1L);
-			check_curl_multi_code(code, CURLMOPT_PIPELINING);
-			code = curl_multi_setopt(mMultiHandles[policy_class],
-									 CURLMOPT_MAX_PIPELINE_LENGTH,
-									 long(options.mPipelining));
-			check_curl_multi_code(code, CURLMOPT_MAX_PIPELINE_LENGTH);
-			code = curl_multi_setopt(mMultiHandles[policy_class],
-									 CURLMOPT_MAX_HOST_CONNECTIONS,
-									 long(options.mPerHostConnectionLimit));
-			check_curl_multi_code(code, CURLMOPT_MAX_HOST_CONNECTIONS);
-			code = curl_multi_setopt(mMultiHandles[policy_class],
-									 CURLMOPT_MAX_TOTAL_CONNECTIONS,
-									 long(options.mConnectionLimit));
-			check_curl_multi_code(code, CURLMOPT_MAX_TOTAL_CONNECTIONS);
+			LL_ERRS(LOG_CORE) << "Failed to allocate multi handle in libcurl."
+							  << LL_ENDL;
 		}
+		mActiveHandles[policy_class] = 0;
+		mDirtyPolicy[policy_class] = false;
+		policyUpdated(policy_class);
 	}
 }
 
@@ -156,8 +139,19 @@ HttpService::ELoopSpeed HttpLibcurl::processTransport()
 	// Give libcurl some cycles to do I/O & callbacks
 	for (int policy_class(0); policy_class < mPolicyCount; ++policy_class)
 	{
-		if (! mActiveHandles[policy_class] || ! mMultiHandles[policy_class])
+		if (! mMultiHandles[policy_class])
 		{
+			// No handle, nothing to do.
+			continue;
+		}
+		if (! mActiveHandles[policy_class])
+		{
+			// If we've gone quiet and there's a dirty update, apply it,
+			// otherwise we're done.
+			if (mDirtyPolicy[policy_class])
+			{
+				policyUpdated(policy_class);
+			}
 			continue;
 		}
 		
@@ -192,9 +186,9 @@ HttpService::ELoopSpeed HttpLibcurl::processTransport()
 			}
 			else
 			{
-				LL_WARNS_ONCE("CoreHttp") << "Unexpected message from libcurl.  Msg code:  "
-										  << msg->msg
-										  << LL_ENDL;
+				LL_WARNS_ONCE(LOG_CORE) << "Unexpected message from libcurl.  Msg code:  "
+										<< msg->msg
+										<< LL_ENDL;
 			}
 			msgs_in_queue = 0;
 		}
@@ -230,11 +224,11 @@ void HttpLibcurl::addOp(HttpOpRequest * op)
 	{
 		HttpPolicy & policy(mService->getPolicy());
 		
-		LL_INFOS("CoreHttp") << "TRACE, ToActiveQueue, Handle:  "
-							 << static_cast<HttpHandle>(op)
-							 << ", Actives:  " << mActiveOps.size()
-							 << ", Readies:  " << policy.getReadyCount(op->mReqPolicy)
-							 << LL_ENDL;
+		LL_INFOS(LOG_CORE) << "TRACE, ToActiveQueue, Handle:  "
+						   << static_cast<HttpHandle>(op)
+						   << ", Actives:  " << mActiveOps.size()
+						   << ", Readies:  " << policy.getReadyCount(op->mReqPolicy)
+						   << LL_ENDL;
 	}
 	
 	// On success, make operation active
@@ -286,10 +280,10 @@ void HttpLibcurl::cancelRequest(HttpOpRequest * op)
 	// Tracing
 	if (op->mTracing > HTTP_TRACE_OFF)
 	{
-		LL_INFOS("CoreHttp") << "TRACE, RequestCanceled, Handle:  "
-							 << static_cast<HttpHandle>(op)
-							 << ", Status:  " << op->mStatus.toTerseString()
-							 << LL_ENDL;
+		LL_INFOS(LOG_CORE) << "TRACE, RequestCanceled, Handle:  "
+						   << static_cast<HttpHandle>(op)
+						   << ", Status:  " << op->mStatus.toTerseString()
+						   << LL_ENDL;
 	}
 
 	// Cancel op and deliver for notification
@@ -306,18 +300,18 @@ bool HttpLibcurl::completeRequest(CURLM * multi_handle, CURL * handle, CURLcode
 
 	if (handle != op->mCurlHandle || ! op->mCurlActive)
 	{
-		LL_WARNS("CoreHttp") << "libcurl handle and HttpOpRequest handle in disagreement or inactive request."
-							 << "  Handle:  " << static_cast<HttpHandle>(handle)
-							 << LL_ENDL;
+		LL_WARNS(LOG_CORE) << "libcurl handle and HttpOpRequest handle in disagreement or inactive request."
+						   << "  Handle:  " << static_cast<HttpHandle>(handle)
+						   << LL_ENDL;
 		return false;
 	}
 
 	active_set_t::iterator it(mActiveOps.find(op));
 	if (mActiveOps.end() == it)
 	{
-		LL_WARNS("CoreHttp") << "libcurl completion for request not on active list.  Continuing."
-							 << "  Handle:  " << static_cast<HttpHandle>(handle)
-							 << LL_ENDL;
+		LL_WARNS(LOG_CORE) << "libcurl completion for request not on active list.  Continuing."
+						   << "  Handle:  " << static_cast<HttpHandle>(handle)
+						   << LL_ENDL;
 		return false;
 	}
 
@@ -348,9 +342,9 @@ bool HttpLibcurl::completeRequest(CURLM * multi_handle, CURL * handle, CURLcode
 		}
 		else
 		{
-			LL_WARNS("CoreHttp") << "Invalid HTTP response code ("
-								 << http_status << ") received from server."
-								 << LL_ENDL;
+			LL_WARNS(LOG_CORE) << "Invalid HTTP response code ("
+							   << http_status << ") received from server."
+							   << LL_ENDL;
 			op->mStatus = HttpStatus(HttpStatus::LLCORE, HE_INVALID_HTTP_STATUS);
 		}
 	}
@@ -363,10 +357,10 @@ bool HttpLibcurl::completeRequest(CURLM * multi_handle, CURL * handle, CURLcode
 	// Tracing
 	if (op->mTracing > HTTP_TRACE_OFF)
 	{
-		LL_INFOS("CoreHttp") << "TRACE, RequestComplete, Handle:  "
-							 << static_cast<HttpHandle>(op)
-							 << ", Status:  " << op->mStatus.toTerseString()
-							 << LL_ENDL;
+		LL_INFOS(LOG_CORE) << "TRACE, RequestComplete, Handle:  "
+						   << static_cast<HttpHandle>(op)
+						   << ", Status:  " << op->mStatus.toTerseString()
+						   << LL_ENDL;
 	}
 
 	// Dispatch to next stage
@@ -390,6 +384,88 @@ int HttpLibcurl::getActiveCountInClass(int policy_class) const
 	return mActiveHandles ? mActiveHandles[policy_class] : 0;
 }
 
+void HttpLibcurl::policyUpdated(int policy_class)
+{
+	if (policy_class < 0 || policy_class >= mPolicyCount || ! mMultiHandles)
+	{
+		return;
+	}
+	
+	HttpPolicy & policy(mService->getPolicy());
+	
+	if (! mActiveHandles[policy_class])
+	{
+		// Clear to set options.  As of libcurl 7.37.0, if a pipelining
+		// multi handle has active requests and you try to set the
+		// multi handle to non-pipelining, the library gets very angry
+		// and goes off the rails corrupting memory.  A clue that you're
+		// about to crash is that you'll get a missing server response
+		// error (curl code 9).  So, if options are to be set, we let
+		// the multi handle run out of requests, then set options, and
+		// re-enable request processing.
+		//
+		// All of this stall mechanism exists for this reason.  If
+		// libcurl becomes more resilient later, it should be possible
+		// to remove all of this.  The connection limit settings are fine,
+		// it's just that pipelined-to-non-pipelined transition that
+		// is fatal at the moment.
+		
+		HttpPolicyClass & options(policy.getClassOptions(policy_class));
+		CURLM * multi_handle(mMultiHandles[policy_class]);
+		CURLMcode code;
+
+		// Enable policy if stalled
+		policy.stallPolicy(policy_class, false);
+		mDirtyPolicy[policy_class] = false;
+		
+		if (options.mPipelining > 1)
+		{
+			// We'll try to do pipelining on this multihandle
+			code = curl_multi_setopt(multi_handle,
+									 CURLMOPT_PIPELINING,
+									 1L);
+			check_curl_multi_code(code, CURLMOPT_PIPELINING);
+			code = curl_multi_setopt(multi_handle,
+									 CURLMOPT_MAX_PIPELINE_LENGTH,
+									 long(options.mPipelining));
+			check_curl_multi_code(code, CURLMOPT_MAX_PIPELINE_LENGTH);
+			code = curl_multi_setopt(multi_handle,
+									 CURLMOPT_MAX_HOST_CONNECTIONS,
+									 long(options.mPerHostConnectionLimit));
+			check_curl_multi_code(code, CURLMOPT_MAX_HOST_CONNECTIONS);
+			code = curl_multi_setopt(multi_handle,
+									 CURLMOPT_MAX_TOTAL_CONNECTIONS,
+									 long(options.mConnectionLimit));
+			check_curl_multi_code(code, CURLMOPT_MAX_TOTAL_CONNECTIONS);
+		}
+		else
+		{
+			code = curl_multi_setopt(multi_handle,
+									 CURLMOPT_PIPELINING,
+									 0L);
+			check_curl_multi_code(code, CURLMOPT_PIPELINING);
+			code = curl_multi_setopt(multi_handle,
+									 CURLMOPT_MAX_HOST_CONNECTIONS,
+									 0L);
+			check_curl_multi_code(code, CURLMOPT_MAX_HOST_CONNECTIONS);
+			code = curl_multi_setopt(multi_handle,
+									 CURLMOPT_MAX_TOTAL_CONNECTIONS,
+									 long(options.mConnectionLimit));
+			check_curl_multi_code(code, CURLMOPT_MAX_TOTAL_CONNECTIONS);
+		}
+	}
+	else if (! mDirtyPolicy[policy_class])
+	{
+		// Mark policy dirty and request a stall in the policy.
+		// When policy goes idle, we'll re-invoke this method
+		// and perform the change.  Don't allow this thread to
+		// sleep while we're waiting for quiescence, we'll just
+		// stop processing.
+		mDirtyPolicy[policy_class] = true;
+		policy.stallPolicy(policy_class, true);
+	}
+}
+
 
 // ---------------------------------------
 // Free functions
@@ -424,9 +500,9 @@ void check_curl_multi_code(CURLMcode code, int curl_setopt_option)
 {
 	if (CURLM_OK != code)
 	{
-		LL_WARNS("CoreHttp") << "libcurl multi error detected:  " << curl_multi_strerror(code)
-							 << ", curl_multi_setopt option:  " << curl_setopt_option
-							 << LL_ENDL;
+		LL_WARNS(LOG_CORE) << "libcurl multi error detected:  " << curl_multi_strerror(code)
+						   << ", curl_multi_setopt option:  " << curl_setopt_option
+						   << LL_ENDL;
 	}
 }
 
@@ -435,8 +511,8 @@ void check_curl_multi_code(CURLMcode code)
 {
 	if (CURLM_OK != code)
 	{
-		LL_WARNS("CoreHttp") << "libcurl multi error detected:  " << curl_multi_strerror(code)
-							 << LL_ENDL;
+		LL_WARNS(LOG_CORE) << "libcurl multi error detected:  " << curl_multi_strerror(code)
+						   << LL_ENDL;
 	}
 }
 
diff --git a/indra/llcorehttp/_httplibcurl.h b/indra/llcorehttp/_httplibcurl.h
index 67f98dd4f0..2c7ad1fa8e 100755
--- a/indra/llcorehttp/_httplibcurl.h
+++ b/indra/llcorehttp/_httplibcurl.h
@@ -4,7 +4,7 @@
  *
  * $LicenseInfo:firstyear=2012&license=viewerlgpl$
  * Second Life Viewer Source Code
- * Copyright (C) 2012-2013, Linden Research, Inc.
+ * Copyright (C) 2012-2014, Linden Research, Inc.
  *
  * This library is free software; you can redistribute it and/or
  * modify it under the terms of the GNU Lesser General Public
@@ -116,6 +116,14 @@ public:
 	/// Threading:  called by worker thread.
 	bool cancel(HttpHandle handle);
 
+	/// Informs transport that a particular policy class has had
+	/// options changed and so should effect any transport state
+	/// change necessary to effect those changes.  Used mainly for
+	/// initialization and dynamic option setting.
+	///
+	/// Threading:  called by worker thread.
+	void policyUpdated(int policy_class);
+
 protected:
 	/// Invoked when libcurl has indicated a request has been processed
 	/// to completion and we need to move the request to a new state.
@@ -134,6 +142,8 @@ protected:
 	int					mPolicyCount;
 	CURLM **			mMultiHandles;			// One handle per policy class
 	int *				mActiveHandles;			// Active count per policy class
+	bool *				mDirtyPolicy;			// Dirty policy update waiting for stall (per pc)
+	
 }; // end class HttpLibcurl
 
 }  // end namespace LLCore
diff --git a/indra/llcorehttp/_httpoperation.cpp b/indra/llcorehttp/_httpoperation.cpp
index 5bb0654652..fefe561f80 100755
--- a/indra/llcorehttp/_httpoperation.cpp
+++ b/indra/llcorehttp/_httpoperation.cpp
@@ -4,7 +4,7 @@
  *
  * $LicenseInfo:firstyear=2012&license=viewerlgpl$
  * Second Life Viewer Source Code
- * Copyright (C) 2012-2013, Linden Research, Inc.
+ * Copyright (C) 2012-2014, Linden Research, Inc.
  *
  * This library is free software; you can redistribute it and/or
  * modify it under the terms of the GNU Lesser General Public
@@ -38,6 +38,14 @@
 #include "lltimer.h"
 
 
+namespace
+{
+
+static const char * const LOG_CORE("CoreHttp");
+
+} // end anonymous namespace
+
+
 namespace LLCore
 {
 
@@ -94,8 +102,8 @@ void HttpOperation::stageFromRequest(HttpService *)
 	// Default implementation should never be called.  This
 	// indicates an operation making a transition that isn't
 	// defined.
-	LL_ERRS("CoreHttp") << "Default stageFromRequest method may not be called."
-						<< LL_ENDL;
+	LL_ERRS(LOG_CORE) << "Default stageFromRequest method may not be called."
+					  << LL_ENDL;
 }
 
 
@@ -104,8 +112,8 @@ void HttpOperation::stageFromReady(HttpService *)
 	// Default implementation should never be called.  This
 	// indicates an operation making a transition that isn't
 	// defined.
-	LL_ERRS("CoreHttp") << "Default stageFromReady method may not be called."
-						<< LL_ENDL;
+	LL_ERRS(LOG_CORE) << "Default stageFromReady method may not be called."
+					  << LL_ENDL;
 }
 
 
@@ -114,8 +122,8 @@ void HttpOperation::stageFromActive(HttpService *)
 	// Default implementation should never be called.  This
 	// indicates an operation making a transition that isn't
 	// defined.
-	LL_ERRS("CoreHttp") << "Default stageFromActive method may not be called."
-						<< LL_ENDL;
+	LL_ERRS(LOG_CORE) << "Default stageFromActive method may not be called."
+					  << LL_ENDL;
 }
 
 
@@ -145,9 +153,9 @@ void HttpOperation::addAsReply()
 {
 	if (mTracing > HTTP_TRACE_OFF)
 	{
-		LL_INFOS("CoreHttp") << "TRACE, ToReplyQueue, Handle:  "
-							 << static_cast<HttpHandle>(this)
-							 << LL_ENDL;
+		LL_INFOS(LOG_CORE) << "TRACE, ToReplyQueue, Handle:  "
+						   << static_cast<HttpHandle>(this)
+						   << LL_ENDL;
 	}
 	
 	if (mReplyQueue)
diff --git a/indra/llcorehttp/_httpoprequest.cpp b/indra/llcorehttp/_httpoprequest.cpp
index 43dd069bc6..eb664fdced 100755
--- a/indra/llcorehttp/_httpoprequest.cpp
+++ b/indra/llcorehttp/_httpoprequest.cpp
@@ -4,7 +4,7 @@
  *
  * $LicenseInfo:firstyear=2012&license=viewerlgpl$
  * Second Life Viewer Source Code
- * Copyright (C) 2012-2013, Linden Research, Inc.
+ * Copyright (C) 2012-2014, Linden Research, Inc.
  *
  * This library is free software; you can redistribute it and/or
  * modify it under the terms of the GNU Lesser General Public
@@ -94,6 +94,8 @@ void os_strlower(char * str);
 void check_curl_easy_code(CURLcode code);
 void check_curl_easy_code(CURLcode code, int curl_setopt_option);
 
+static const char * const LOG_CORE("CoreHttp");
+
 } // end anonymous namespace
 
 
@@ -416,8 +418,8 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service)
 	if (! mCurlHandle)
 	{
 		// We're in trouble.  We'll continue but it won't go well.
-		LL_WARNS("CoreHttp") << "Failed to allocate libcurl easy handle.  Continuing."
-							 << LL_ENDL;
+		LL_WARNS(LOG_CORE) << "Failed to allocate libcurl easy handle.  Continuing."
+						   << LL_ENDL;
 		return HttpStatus(HttpStatus::LLCORE, HE_BAD_ALLOC);
 	}
 	code = curl_easy_setopt(mCurlHandle, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4);
@@ -538,9 +540,9 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service)
 		break;
 		
 	default:
-		LL_ERRS("CoreHttp") << "Invalid HTTP method in request:  "
-							<< int(mReqMethod)  << ".  Can't recover."
-							<< LL_ENDL;
+		LL_ERRS(LOG_CORE) << "Invalid HTTP method in request:  "
+						  << int(mReqMethod)  << ".  Can't recover."
+						  << LL_ENDL;
 		break;
 	}
 
@@ -652,8 +654,8 @@ size_t HttpOpRequest::readCallback(void * data, size_t size, size_t nmemb, void
 		{
 			// Warn but continue if the read position moves beyond end-of-body
 			// for some reason.
-			LL_WARNS("CoreHttp") << "Request body position beyond body size.  Truncating request body."
-								 << LL_ENDL;
+			LL_WARNS(LOG_CORE) << "Request body position beyond body size.  Truncating request body."
+							   << LL_ENDL;
 		}
 		return 0;
 	}
@@ -790,10 +792,10 @@ size_t HttpOpRequest::headerCallback(void * data, size_t size, size_t nmemb, voi
 		else
 		{
 			// Ignore the unparsable.
-			LL_INFOS_ONCE("CoreHttp") << "Problem parsing odd Content-Range header:  '"
-									  << std::string(hdr_data, wanted_hdr_size)
-									  << "'.  Ignoring."
-									  << LL_ENDL;
+			LL_INFOS_ONCE(LOG_CORE) << "Problem parsing odd Content-Range header:  '"
+									<< std::string(hdr_data, wanted_hdr_size)
+									<< "'.  Ignoring."
+									<< LL_ENDL;
 		}
 	}
 
@@ -895,11 +897,11 @@ int HttpOpRequest::debugCallback(CURL * handle, curl_infotype info, char * buffe
 
 	if (logit)
 	{
-		LL_INFOS("CoreHttp") << "TRACE, LibcurlDebug, Handle:  "
-							 << static_cast<HttpHandle>(op)
-							 << ", Type:  " << tag
-							 << ", Data:  " << safe_line
-							 << LL_ENDL;
+		LL_INFOS(LOG_CORE) << "TRACE, LibcurlDebug, Handle:  "
+						   << static_cast<HttpHandle>(op)
+						   << ", Type:  " << tag
+						   << ", Data:  " << safe_line
+						   << LL_ENDL;
 	}
 		
 	return 0;
@@ -1094,9 +1096,9 @@ void check_curl_easy_code(CURLcode code, int curl_setopt_option)
 		//
 		// linux appears to throw a curl error once per session for a bad initialization
 		// at a pretty random time (when enabling cookies).
-		LL_WARNS("CoreHttp") << "libcurl error detected:  " << curl_easy_strerror(code)
-							 << ", curl_easy_setopt option:  " << curl_setopt_option
-							 << LL_ENDL;
+		LL_WARNS(LOG_CORE) << "libcurl error detected:  " << curl_easy_strerror(code)
+						   << ", curl_easy_setopt option:  " << curl_setopt_option
+						   << LL_ENDL;
 	}
 }
 
@@ -1109,8 +1111,8 @@ void check_curl_easy_code(CURLcode code)
 		//
 		// linux appears to throw a curl error once per session for a bad initialization
 		// at a pretty random time (when enabling cookies).
-		LL_WARNS("CoreHttp") << "libcurl error detected:  " << curl_easy_strerror(code)
-							 << LL_ENDL;
+		LL_WARNS(LOG_CORE) << "libcurl error detected:  " << curl_easy_strerror(code)
+						   << LL_ENDL;
 	}
 }
 
diff --git a/indra/llcorehttp/_httppolicy.cpp b/indra/llcorehttp/_httppolicy.cpp
index bb7959b578..09b9206f63 100755
--- a/indra/llcorehttp/_httppolicy.cpp
+++ b/indra/llcorehttp/_httppolicy.cpp
@@ -35,6 +35,13 @@
 
 #include "lltimer.h"
 
+namespace
+{
+
+static const char * const LOG_CORE("CoreHttp");
+
+} // end anonymous namespace
+
 
 namespace LLCore
 {
@@ -51,7 +58,8 @@ public:
 	ClassState()
 		: mThrottleEnd(0),
 		  mThrottleLeft(0L),
-		  mRequestCount(0L)
+		  mRequestCount(0L),
+		  mStallStaging(false)
 		{}
 	
 	HttpReadyQueue		mReadyQueue;
@@ -61,6 +69,7 @@ public:
 	HttpTime			mThrottleEnd;
 	long				mThrottleLeft;
 	long				mRequestCount;
+	bool				mStallStaging;
 };
 
 
@@ -171,19 +180,19 @@ void HttpPolicy::retryOp(HttpOpRequest * op)
 	{
 		++op->mPolicy503Retries;
 	}
-	LL_DEBUGS("CoreHttp") << "HTTP request " << static_cast<HttpHandle>(op)
-						  << " retry " << op->mPolicyRetries
-						  << " scheduled in " << (delta / HttpTime(1000))
-						  << " mS (" << (external_delta ? "external" : "internal")
-						  << ").  Status:  " << op->mStatus.toTerseString()
-						  << LL_ENDL;
+	LL_DEBUGS(LOG_CORE) << "HTTP request " << static_cast<HttpHandle>(op)
+						<< " retry " << op->mPolicyRetries
+						<< " scheduled in " << (delta / HttpTime(1000))
+						<< " mS (" << (external_delta ? "external" : "internal")
+						<< ").  Status:  " << op->mStatus.toTerseString()
+						<< LL_ENDL;
 	if (op->mTracing > HTTP_TRACE_OFF)
 	{
-		LL_INFOS("CoreHttp") << "TRACE, ToRetryQueue, Handle:  "
-							 << static_cast<HttpHandle>(op)
-							 << ", Delta:  " << (delta / HttpTime(1000))
-							 << ", Retries:  " << op->mPolicyRetries
-							 << LL_ENDL;
+		LL_INFOS(LOG_CORE) << "TRACE, ToRetryQueue, Handle:  "
+						   << static_cast<HttpHandle>(op)
+						   << ", Delta:  " << (delta / HttpTime(1000))
+						   << ", Retries:  " << op->mPolicyRetries
+						   << LL_ENDL;
 	}
 	mClasses[policy_class]->mRetryQueue.push(op);
 }
@@ -219,6 +228,15 @@ HttpService::ELoopSpeed HttpPolicy::processReadyQueue()
 		HttpRetryQueue & retryq(state.mRetryQueue);
 		HttpReadyQueue & readyq(state.mReadyQueue);
 
+		if (state.mStallStaging)
+		{
+			// Stalling but don't sleep.  Need to complete operations
+			// and get back to servicing queues.  Do this test before
+			// the retryq/readyq test or you'll get stalls until you
+			// click a setting or an asset request comes in.
+			result = HttpService::NORMAL;
+			continue;
+		}
 		if (retryq.empty() && readyq.empty())
 		{
 			continue;
@@ -262,9 +280,9 @@ HttpService::ELoopSpeed HttpPolicy::processReadyQueue()
 					if (now >= state.mThrottleEnd)
 					{
 						// Throttle expired, move to next window
-						LL_DEBUGS("CoreHttp") << "Throttle expired with " << state.mThrottleLeft
-											  << " requests to go and " << state.mRequestCount
-											  << " requests issued." << LL_ENDL;
+						LL_DEBUGS(LOG_CORE) << "Throttle expired with " << state.mThrottleLeft
+											<< " requests to go and " << state.mRequestCount
+											<< " requests issued." << LL_ENDL;
 						state.mThrottleLeft = state.mOptions.mThrottleRate;
 						state.mThrottleEnd = now + HttpTime(1000000);
 					}
@@ -291,9 +309,9 @@ HttpService::ELoopSpeed HttpPolicy::processReadyQueue()
 					if (now >= state.mThrottleEnd)
 					{
 						// Throttle expired, move to next window
-						LL_DEBUGS("CoreHttp") << "Throttle expired with " << state.mThrottleLeft
-											  << " requests to go and " << state.mRequestCount
-											  << " requests issued." << LL_ENDL;
+						LL_DEBUGS(LOG_CORE) << "Throttle expired with " << state.mThrottleLeft
+											<< " requests to go and " << state.mRequestCount
+											<< " requests issued." << LL_ENDL;
 						state.mThrottleLeft = state.mOptions.mThrottleRate;
 						state.mThrottleEnd = now + HttpTime(1000000);
 					}
@@ -408,17 +426,17 @@ bool HttpPolicy::stageAfterCompletion(HttpOpRequest * op)
 	// This op is done, finalize it delivering it to the reply queue...
 	if (! op->mStatus)
 	{
-		LL_WARNS("CoreHttp") << "HTTP request " << static_cast<HttpHandle>(op)
-							 << " failed after " << op->mPolicyRetries
-							 << " retries.  Reason:  " << op->mStatus.toString()
-							 << " (" << op->mStatus.toTerseString() << ")"
-							 << LL_ENDL;
+		LL_WARNS(LOG_CORE) << "HTTP request " << static_cast<HttpHandle>(op)
+						   << " failed after " << op->mPolicyRetries
+						   << " retries.  Reason:  " << op->mStatus.toString()
+						   << " (" << op->mStatus.toTerseString() << ")"
+						   << LL_ENDL;
 	}
 	else if (op->mPolicyRetries)
 	{
-		LL_DEBUGS("CoreHttp") << "HTTP request " << static_cast<HttpHandle>(op)
-							  << " succeeded on retry " << op->mPolicyRetries << "."
-							  << LL_ENDL;
+		LL_DEBUGS(LOG_CORE) << "HTTP request " << static_cast<HttpHandle>(op)
+							<< " succeeded on retry " << op->mPolicyRetries << "."
+							<< LL_ENDL;
 	}
 
 	op->stageFromActive(mService);
@@ -446,4 +464,17 @@ int HttpPolicy::getReadyCount(HttpRequest::policy_t policy_class) const
 }
 
 
+bool HttpPolicy::stallPolicy(HttpRequest::policy_t policy_class, bool stall)
+{
+	bool ret(false);
+	
+	if (policy_class < mClasses.size())
+	{
+		ret = mClasses[policy_class]->mStallStaging;
+		mClasses[policy_class]->mStallStaging = stall;
+	}
+	return ret;
+}
+
+
 }  // end namespace LLCore
diff --git a/indra/llcorehttp/_httppolicy.h b/indra/llcorehttp/_httppolicy.h
index bf1aa74267..11cd89bbd1 100755
--- a/indra/llcorehttp/_httppolicy.h
+++ b/indra/llcorehttp/_httppolicy.h
@@ -158,6 +158,14 @@ public:
 	/// Threading:  called by worker thread
 	int getReadyCount(HttpRequest::policy_t policy_class) const;
 	
+	/// Stall (or unstall) a policy class preventing requests from
+	/// transitioning to an active state.  Used to allow an HTTP
+	/// request policy to empty prior to changing settings or state
+	/// that isn't tolerant of changes when work is outstanding.
+	///
+	/// Threading:  called by worker thread
+	bool stallPolicy(HttpRequest::policy_t policy_class, bool stall);
+	
 protected:
 	struct ClassState;
 	typedef std::vector<ClassState *>	class_list_t;
diff --git a/indra/llcorehttp/_httpservice.cpp b/indra/llcorehttp/_httpservice.cpp
index c94249dc2d..c673e1be1d 100755
--- a/indra/llcorehttp/_httpservice.cpp
+++ b/indra/llcorehttp/_httpservice.cpp
@@ -4,7 +4,7 @@
  *
  * $LicenseInfo:firstyear=2012&license=viewerlgpl$
  * Second Life Viewer Source Code
- * Copyright (C) 2012-2013, Linden Research, Inc.
+ * Copyright (C) 2012-2014, Linden Research, Inc.
  *
  * This library is free software; you can redistribute it and/or
  * modify it under the terms of the GNU Lesser General Public
@@ -40,6 +40,14 @@
 #include "llthread.h"
 
 
+namespace
+{
+
+static const char * const LOG_CORE("CoreHttp");
+
+} // end anonymous namespace
+
+
 namespace LLCore
 {
 
@@ -87,8 +95,8 @@ HttpService::~HttpService()
 				// Failed to join, expect problems ahead so do a hard termination.
 				mThread->cancel();
 
-				LL_WARNS("CoreHttp") << "Destroying HttpService with running thread.  Expect problems."
-									 << LL_ENDL;
+				LL_WARNS(LOG_CORE) << "Destroying HttpService with running thread.  Expect problems."
+								   << LL_ENDL;
 			}
 		}
 	}
@@ -328,9 +336,9 @@ HttpService::ELoopSpeed HttpService::processRequestQueue(ELoopSpeed loop)
 
 			if (op->mTracing > HTTP_TRACE_OFF)
 			{
-				LL_INFOS("CoreHttp") << "TRACE, FromRequestQueue, Handle:  "
-									 << static_cast<HttpHandle>(op)
-									 << LL_ENDL;
+				LL_INFOS(LOG_CORE) << "TRACE, FromRequestQueue, Handle:  "
+								   << static_cast<HttpHandle>(op)
+								   << LL_ENDL;
 			}
 
 			// Stage
@@ -437,9 +445,13 @@ HttpStatus HttpService::setPolicyOption(HttpRequest::EPolicyOption opt, HttpRequ
 		HttpPolicyClass & opts(mPolicy->getClassOptions(pclass));
 
 		status = opts.set(opt, value);
-		if (status && ret_value)
+		if (status)
 		{
-			status = opts.get(opt, ret_value);
+			mTransport->policyUpdated(pclass);
+			if (ret_value)
+			{
+				status = opts.get(opt, ret_value);
+			}
 		}
 	}
 
@@ -463,7 +475,7 @@ HttpStatus HttpService::setPolicyOption(HttpRequest::EPolicyOption opt, HttpRequ
 		return status;
 	}
 
-	// Only string values are global at this time
+	// String values are always global (at this time).
 	if (pclass == HttpRequest::GLOBAL_POLICY_ID)
 	{
 		HttpPolicyGlobal & opts(mPolicy->getGlobalOptions());
-- 
cgit v1.2.3


From 0c20beda6800149ee71a307ca4e943b5bba56908 Mon Sep 17 00:00:00 2001
From: Monty Brandenberg <monty@lindenlab.com>
Date: Thu, 4 Sep 2014 16:57:44 -0400
Subject: Pipelining work.  Extend transfer timeout by the pipeline depth as
 transfers can appear delayed with deep pipelining and more requests in the
 pool.  Added bad HTTP status error (typically getting a 0 back as HTTP status
 from libcurl) to the list of retryable errors.  There's a response stream
 problem with libcurl and pipelining that induces this problem.  Retrying
 helps but may not be entirely safe.  Watch bug 1420 on the libcurl
 sourceforge bug tracker.  Extend options of test/example program to include
 un-ranged requests.  Document the excessive data transfer induced when ranged
 requests are disabled.  This is an abnormal mode for very rare users so we'll
 just eat that for now.

---
 indra/llcorehttp/_httplibcurl.cpp               | 15 +++++++----
 indra/llcorehttp/_httpoprequest.cpp             | 34 ++++++++++++++++++-------
 indra/llcorehttp/examples/http_texture_load.cpp | 21 ++++++++++++---
 indra/llcorehttp/httpcommon.cpp                 | 13 ++++++++--
 4 files changed, 64 insertions(+), 19 deletions(-)

(limited to 'indra/llcorehttp')

diff --git a/indra/llcorehttp/_httplibcurl.cpp b/indra/llcorehttp/_httplibcurl.cpp
index b46833a1f3..cfbe0fd2bb 100755
--- a/indra/llcorehttp/_httplibcurl.cpp
+++ b/indra/llcorehttp/_httplibcurl.cpp
@@ -217,8 +217,17 @@ void HttpLibcurl::addOp(HttpOpRequest * op)
 	}
 
 	// Make the request live
-	curl_multi_add_handle(mMultiHandles[op->mReqPolicy], op->mCurlHandle);
+	CURLMcode code;
+	code = curl_multi_add_handle(mMultiHandles[op->mReqPolicy], op->mCurlHandle);
+	if (CURLM_OK != code)
+	{
+		// *TODO:  Better cleanup and recovery but not much we can do here.
+		check_curl_multi_code(code);
+		return;
+	}
 	op->mCurlActive = true;
+	mActiveOps.insert(op);
+	++mActiveHandles[op->mReqPolicy];
 	
 	if (op->mTracing > HTTP_TRACE_OFF)
 	{
@@ -230,10 +239,6 @@ void HttpLibcurl::addOp(HttpOpRequest * op)
 						   << ", Readies:  " << policy.getReadyCount(op->mReqPolicy)
 						   << LL_ENDL;
 	}
-	
-	// On success, make operation active
-	mActiveOps.insert(op);
-	++mActiveHandles[op->mReqPolicy];
 }
 
 
diff --git a/indra/llcorehttp/_httpoprequest.cpp b/indra/llcorehttp/_httpoprequest.cpp
index eb664fdced..38c1f1e78a 100755
--- a/indra/llcorehttp/_httpoprequest.cpp
+++ b/indra/llcorehttp/_httpoprequest.cpp
@@ -378,6 +378,7 @@ void HttpOpRequest::setupCommon(HttpRequest::policy_t policy_id,
 // Junk may be left around from a failed request and that
 // needs to be cleaned out.
 //
+// *TODO:  Move this to _httplibcurl where it belongs.
 HttpStatus HttpOpRequest::prepareRequest(HttpService * service)
 {
 	CURLcode code;
@@ -411,8 +412,9 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service)
 	// *FIXME:  better error handling later
 	HttpStatus status;
 
-	// Get global policy options
-	HttpPolicyGlobal & policy(service->getPolicy().getGlobalOptions());
+	// Get global and class policy options
+	HttpPolicyGlobal & gpolicy(service->getPolicy().getGlobalOptions());
+	HttpPolicyClass & cpolicy(service->getPolicy().getClassOptions(mReqPolicy));
 	
 	mCurlHandle = LLCurl::createStandardCurlHandle();
 	if (! mCurlHandle)
@@ -462,30 +464,30 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service)
 	code = curl_easy_setopt(mCurlHandle, CURLOPT_SSL_VERIFYHOST, 0);
 	check_curl_easy_code(code, CURLOPT_SSL_VERIFYHOST);
 
-	if (policy.mUseLLProxy)
+	if (gpolicy.mUseLLProxy)
 	{
 		// Use the viewer-based thread-safe API which has a
 		// fast/safe check for proxy enable.  Would like to
 		// encapsulate this someway...
 		LLProxy::getInstance()->applyProxySettings(mCurlHandle);
 	}
-	else if (policy.mHttpProxy.size())
+	else if (gpolicy.mHttpProxy.size())
 	{
 		// *TODO:  This is fine for now but get fuller socks5/
 		// authentication thing going later....
-		code = curl_easy_setopt(mCurlHandle, CURLOPT_PROXY, policy.mHttpProxy.c_str());
+		code = curl_easy_setopt(mCurlHandle, CURLOPT_PROXY, gpolicy.mHttpProxy.c_str());
 		check_curl_easy_code(code, CURLOPT_PROXY);
 		code = curl_easy_setopt(mCurlHandle, CURLOPT_PROXYTYPE, CURLPROXY_HTTP);
 		check_curl_easy_code(code, CURLOPT_PROXYTYPE);
 	}
-	if (policy.mCAPath.size())
+	if (gpolicy.mCAPath.size())
 	{
-		code = curl_easy_setopt(mCurlHandle, CURLOPT_CAPATH, policy.mCAPath.c_str());
+		code = curl_easy_setopt(mCurlHandle, CURLOPT_CAPATH, gpolicy.mCAPath.c_str());
 		check_curl_easy_code(code, CURLOPT_CAPATH);
 	}
-	if (policy.mCAFile.size())
+	if (gpolicy.mCAFile.size())
 	{
-		code = curl_easy_setopt(mCurlHandle, CURLOPT_CAINFO, policy.mCAFile.c_str());
+		code = curl_easy_setopt(mCurlHandle, CURLOPT_CAINFO, gpolicy.mCAFile.c_str());
 		check_curl_easy_code(code, CURLOPT_CAINFO);
 	}
 	
@@ -594,6 +596,20 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service)
 	{
 		xfer_timeout = timeout;
 	}
+	if (cpolicy.mPipelining > 1L)
+	{
+		// Pipelining affects both connection and transfer timeout values.
+		// Requests that are added to a pipeling immediately have completed
+		// their connection so the connection delay tends to be less than
+		// the non-pipelined value.  Transfers are the opposite.  Transfer
+		// timeout starts once the connection is established and completion
+		// can be delayed due to the pipelined requests ahead.  So, it's
+		// a handwave but bump the transfer timeout up by the pipelining
+		// depth to give some room.
+		//
+		// *TODO:  Find a better scheme than timeouts to guarantee liveness.
+		xfer_timeout *= cpolicy.mPipelining;
+	}
 	code = curl_easy_setopt(mCurlHandle, CURLOPT_TIMEOUT, xfer_timeout);
 	check_curl_easy_code(code, CURLOPT_TIMEOUT);
 	code = curl_easy_setopt(mCurlHandle, CURLOPT_CONNECTTIMEOUT, timeout);
diff --git a/indra/llcorehttp/examples/http_texture_load.cpp b/indra/llcorehttp/examples/http_texture_load.cpp
index 88692c3f69..b76c874557 100755
--- a/indra/llcorehttp/examples/http_texture_load.cpp
+++ b/indra/llcorehttp/examples/http_texture_load.cpp
@@ -102,6 +102,7 @@ public:
 public:
 	bool						mVerbose;
 	bool						mRandomRange;
+	bool						mNoRange;
 	int							mRequestLowWater;
 	int							mRequestHighWater;
 	handle_set_t				mHandles;
@@ -162,10 +163,11 @@ int main(int argc, char** argv)
 {
 	LLCore::HttpStatus status;
 	bool do_random(false);
+	bool do_whole(false);
 	bool do_verbose(false);
 	
 	int option(-1);
-	while (-1 != (option = getopt(argc, argv, "u:c:h?RvH:p:t:")))
+	while (-1 != (option = getopt(argc, argv, "u:c:h?RwvH:p:t:")))
 	{
 		switch (option)
 		{
@@ -236,6 +238,12 @@ int main(int argc, char** argv)
 
 		case 'R':
 			do_random = true;
+			do_whole = false;
+			break;
+
+		case 'w':
+			do_whole = true;
+			do_random = false;
 			break;
 
 		case 'v':
@@ -307,6 +315,7 @@ int main(int argc, char** argv)
 	ws.mUrl = url_format;
 	ws.loadAssetUuids(uuids);
 	ws.mRandomRange = do_random;
+	ws.mNoRange = do_whole;
 	ws.mVerbose = do_verbose;
 	ws.mRequestHighWater = highwater;
 	ws.mRequestLowWater = ws.mRequestHighWater / 2;
@@ -381,6 +390,7 @@ void usage(std::ostream & out)
 		" -u <url_format>       printf-style format string for URL generation\n"
 		"                       Default:  " << url_format << "\n"
 		" -R                    Issue GETs with random Range: headers\n"
+		" -w                    Issue GETs without Range: headers to get whole object\n"
 		" -c <limit>            Maximum connection concurrency.  Range:  [1..100]\n"
 		"                       Default:  " << concurrency_limit << "\n"
 		" -H <limit>            HTTP request highwater (requests fed to llcorehttp).\n"
@@ -400,6 +410,7 @@ WorkingSet::WorkingSet()
 	: LLCore::HttpHandler(),
 	  mVerbose(false),
 	  mRandomRange(false),
+	  mNoRange(false),
 	  mRemaining(200),
 	  mLimit(200),
 	  mAt(0),
@@ -449,8 +460,12 @@ bool WorkingSet::reload(LLCore::HttpRequest * hr, LLCore::HttpOptions * opt)
 #else
 		snprintf(buffer, sizeof(buffer), mUrl.c_str(), mAssets[mAt].mUuid.c_str());
 #endif
-		int offset(mRandomRange ? ((unsigned long) rand()) % 1000000UL : mAssets[mAt].mOffset);
-		int length(mRandomRange ? ((unsigned long) rand()) % 1000000UL : mAssets[mAt].mLength);
+		int offset(mNoRange
+				   ? 0
+				   : (mRandomRange ? ((unsigned long) rand()) % 1000000UL : mAssets[mAt].mOffset));
+		int length(mNoRange
+				   ? 0
+				   : (mRandomRange ? ((unsigned long) rand()) % 1000000UL : mAssets[mAt].mLength));
 
 		LLCore::HttpHandle handle;
 		if (offset || length)
diff --git a/indra/llcorehttp/httpcommon.cpp b/indra/llcorehttp/httpcommon.cpp
index c2f15155ac..9bcf7ac5e3 100755
--- a/indra/llcorehttp/httpcommon.cpp
+++ b/indra/llcorehttp/httpcommon.cpp
@@ -4,7 +4,7 @@
  *
  * $LicenseInfo:firstyear=2012&license=viewerlgpl$
  * Second Life Viewer Source Code
- * Copyright (C) 2012-2013, Linden Research, Inc.
+ * Copyright (C) 2012-2014, Linden Research, Inc.
  *
  * This library is free software; you can redistribute it and/or
  * modify it under the terms of the GNU Lesser General Public
@@ -219,6 +219,13 @@ std::string HttpStatus::toTerseString() const
 // Pass true on statuses that might actually be cleared by a
 // retry.  Library failures, calling problems, etc. aren't
 // going to be fixed by squirting bits all over the Net.
+//
+// HE_INVALID_HTTP_STATUS is special.  As of 7.37.0, there are
+// some scenarios where response processing in libcurl appear
+// to go wrong and response data is corrupted.  A side-effect
+// of this is that the HTTP status is read as 0 from the library.
+// See libcurl bug report 1420 (https://sourceforge.net/p/curl/bugs/1420/)
+// for details.
 bool HttpStatus::isRetryable() const
 {
 	static const HttpStatus cant_connect(HttpStatus::EXT_CURL_EASY, CURLE_COULDNT_CONNECT);
@@ -231,6 +238,7 @@ bool HttpStatus::isRetryable() const
 	static const HttpStatus post_error(HttpStatus::EXT_CURL_EASY, CURLE_HTTP_POST_ERROR);
 	static const HttpStatus partial_file(HttpStatus::EXT_CURL_EASY, CURLE_PARTIAL_FILE);
 	static const HttpStatus inv_cont_range(HttpStatus::LLCORE, HE_INV_CONTENT_RANGE_HDR);
+	static const HttpStatus inv_status(HttpStatus::LLCORE, HE_INVALID_HTTP_STATUS);
 
 	return ((isHttpStatus() && mType >= 499 && mType <= 599) ||	// Include special 499 in retryables
 			*this == cant_connect ||	// Connection reset/endpoint problems
@@ -242,7 +250,8 @@ bool HttpStatus::isRetryable() const
 			*this == op_timedout ||		// Timer expired
 			*this == post_error ||		// Transport problem
 			*this == partial_file ||	// Data inconsistency in response
-			*this == inv_cont_range);	// Short data read disagrees with content-range
+			*this == inv_cont_range ||	// Short data read disagrees with content-range
+			*this == inv_status);		// Inv status can reflect internal state problem in libcurl
 }
 
 } // end namespace LLCore
-- 
cgit v1.2.3


From 3057d246f0c63be1b3015313b9ad9824a66f4e0f Mon Sep 17 00:00:00 2001
From: Monty Brandenberg <monty@lindenlab.com>
Date: Thu, 18 Sep 2014 18:42:30 -0400
Subject: Documentation.  Describe curl bug 1420 testing and how to reproduce
 data corruption via timeouts.

---
 indra/llcorehttp/_httpoprequest.cpp | 15 +++++++++++++++
 indra/llcorehttp/httpcommon.cpp     |  4 ++++
 2 files changed, 19 insertions(+)

(limited to 'indra/llcorehttp')

diff --git a/indra/llcorehttp/_httpoprequest.cpp b/indra/llcorehttp/_httpoprequest.cpp
index 38c1f1e78a..4453bf2922 100755
--- a/indra/llcorehttp/_httpoprequest.cpp
+++ b/indra/llcorehttp/_httpoprequest.cpp
@@ -47,6 +47,19 @@
 #include "llhttpconstants.h"
 #include "llproxy.h"
 
+// *DEBUG:  "[curl:bugs] #1420" problem and testing.
+//
+// A pipelining problem, https://sourceforge.net/p/curl/bugs/1420/,
+// was a source of Core_9 failures.  Code related to this can be
+// identified and tested by:
+// * Looking for '[curl:bugs]' strings in source and following
+//   instructions there.
+// * Set 'QAModeHttpTrace' to 2 or 3 in settings.xml and look for
+//   'timed out' events in the log.
+// * Enable the HttpRangeRequestsDisable debug setting which causes
+//   full asset fetches.  These slow the pipelines down a bit.
+//
+
 namespace
 {
 
@@ -610,6 +623,8 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service)
 		// *TODO:  Find a better scheme than timeouts to guarantee liveness.
 		xfer_timeout *= cpolicy.mPipelining;
 	}
+	// *DEBUG:  Useful for timeout handling and "[curl:bugs] #1420" tests
+	// xfer_timeout = 3L;
 	code = curl_easy_setopt(mCurlHandle, CURLOPT_TIMEOUT, xfer_timeout);
 	check_curl_easy_code(code, CURLOPT_TIMEOUT);
 	code = curl_easy_setopt(mCurlHandle, CURLOPT_CONNECTTIMEOUT, timeout);
diff --git a/indra/llcorehttp/httpcommon.cpp b/indra/llcorehttp/httpcommon.cpp
index 9bcf7ac5e3..8714915fa2 100755
--- a/indra/llcorehttp/httpcommon.cpp
+++ b/indra/llcorehttp/httpcommon.cpp
@@ -240,6 +240,10 @@ bool HttpStatus::isRetryable() const
 	static const HttpStatus inv_cont_range(HttpStatus::LLCORE, HE_INV_CONTENT_RANGE_HDR);
 	static const HttpStatus inv_status(HttpStatus::LLCORE, HE_INVALID_HTTP_STATUS);
 
+	// *DEBUG:  For "[curl:bugs] #1420" tests.
+	// Disable the '*this == inv_status' test and look for 'Core_9'
+	// failures in log files.
+
 	return ((isHttpStatus() && mType >= 499 && mType <= 599) ||	// Include special 499 in retryables
 			*this == cant_connect ||	// Connection reset/endpoint problems
 			*this == cant_res_proxy ||	// DNS problems
-- 
cgit v1.2.3


From 79ab7c20703c092a4416a4f9a885e0246fc17ee0 Mon Sep 17 00:00:00 2001
From: Monty Brandenberg <monty@lindenlab.com>
Date: Fri, 19 Sep 2014 15:34:09 -0400
Subject: Introduce libcurl handle cache.  Create a private cache of used
 handles and a fast handle factory that's thread- correct.

---
 indra/llcorehttp/_httplibcurl.cpp   | 83 +++++++++++++++++++++++++++++++++++--
 indra/llcorehttp/_httplibcurl.h     | 78 ++++++++++++++++++++++++++++++++--
 indra/llcorehttp/_httpoprequest.cpp |  5 ++-
 3 files changed, 158 insertions(+), 8 deletions(-)

(limited to 'indra/llcorehttp')

diff --git a/indra/llcorehttp/_httplibcurl.cpp b/indra/llcorehttp/_httplibcurl.cpp
index cfbe0fd2bb..81b44ab90b 100755
--- a/indra/llcorehttp/_httplibcurl.cpp
+++ b/indra/llcorehttp/_httplibcurl.cpp
@@ -51,6 +51,7 @@ namespace LLCore
 
 HttpLibcurl::HttpLibcurl(HttpService * service)
 	: mService(service),
+	  mHandleCache(),
 	  mPolicyCount(0),
 	  mMultiHandles(NULL),
 	  mActiveHandles(NULL),
@@ -61,7 +62,7 @@ HttpLibcurl::HttpLibcurl(HttpService * service)
 HttpLibcurl::~HttpLibcurl()
 {
 	shutdown();
-	
+
 	mService = NULL;
 }
 
@@ -279,7 +280,7 @@ void HttpLibcurl::cancelRequest(HttpOpRequest * op)
 
 	// Detach from multi and recycle handle
 	curl_multi_remove_handle(mMultiHandles[op->mReqPolicy], op->mCurlHandle);
-	curl_easy_cleanup(op->mCurlHandle);
+	mHandleCache.freeHandle(op->mCurlHandle);
 	op->mCurlHandle = NULL;
 
 	// Tracing
@@ -356,7 +357,7 @@ bool HttpLibcurl::completeRequest(CURLM * multi_handle, CURL * handle, CURLcode
 
 	// Detach from multi and recycle handle
 	curl_multi_remove_handle(multi_handle, handle);
-	curl_easy_cleanup(handle);
+	mHandleCache.freeHandle(op->mCurlHandle);
 	op->mCurlHandle = NULL;
 
 	// Tracing
@@ -471,6 +472,82 @@ void HttpLibcurl::policyUpdated(int policy_class)
 	}
 }
 
+// ---------------------------------------
+// HttpLibcurl::HandleCache
+// ---------------------------------------
+
+HttpLibcurl::HandleCache::HandleCache()
+	: mHandleTemplate(NULL)
+{
+	mCache.reserve(50);
+}
+
+
+HttpLibcurl::HandleCache::~HandleCache()
+{
+	if (mHandleTemplate)
+	{
+		curl_easy_cleanup(mHandleTemplate);
+		mHandleTemplate = NULL;
+	}
+
+	for (handle_cache_t::iterator it(mCache.begin()); mCache.end() != it; ++it)
+	{
+		curl_easy_cleanup(*it);
+	}
+	mCache.clear();
+}
+
+
+CURL * HttpLibcurl::HandleCache::getHandle()
+{
+	CURL * ret(NULL);
+	
+	if (! mCache.empty())
+	{
+		// Fastest path to handle
+		ret = mCache.back();
+		mCache.pop_back();
+	}
+	else if (mHandleTemplate)
+	{
+		// Still fast path
+		ret = curl_easy_duphandle(mHandleTemplate);
+	}
+	else
+	{
+		// When all else fails
+		ret = curl_easy_init();
+	}
+
+	return ret;
+}
+
+
+void HttpLibcurl::HandleCache::freeHandle(CURL * handle)
+{
+	if (! handle)
+	{
+		return;
+	}
+
+	curl_easy_reset(handle);
+	if (! mHandleTemplate)
+	{
+		// Save the first freed handle as a template.
+		mHandleTemplate = handle;
+	}
+	else
+	{
+		// Otherwise add it to the cache
+		if (mCache.size() >= mCache.capacity())
+		{
+			mCache.reserve(mCache.capacity() + 50);
+		}
+		mCache.push_back(handle);
+	}
+}
+
 
 // ---------------------------------------
 // Free functions
diff --git a/indra/llcorehttp/_httplibcurl.h b/indra/llcorehttp/_httplibcurl.h
index 2c7ad1fa8e..ffc24c63a8 100755
--- a/indra/llcorehttp/_httplibcurl.h
+++ b/indra/llcorehttp/_httplibcurl.h
@@ -124,6 +124,23 @@ public:
 	/// Threading:  called by worker thread.
 	void policyUpdated(int policy_class);
 
+	/// Allocate a curl handle for caller.  May be freed using
+	/// either the freeHandle() method or calling curl_easy_cleanup()
+	/// directly.
+	///
+	/// @return			Libcurl handle (CURL *) or NULL on allocation
+	///					problem.  Handle will be in curl_easy_reset()
+	///					condition.
+	///
+	/// Threading:  callable by worker thread.
+	///
+	/// Deprecation:  Expect this to go away after _httpoprequest is
+	/// refactored bringing code into this class.
+	CURL * getHandle()
+		{
+			return mHandleCache.getHandle();
+		}
+
 protected:
 	/// Invoked when libcurl has indicated a request has been processed
 	/// to completion and we need to move the request to a new state.
@@ -135,14 +152,67 @@ protected:
 	
 protected:
 	typedef std::set<HttpOpRequest *> active_set_t;
+
+	/// Simple request handle cache for libcurl.
+	///
+	/// Handle creation is somewhat slow and chunky in libcurl and there's
+	/// a pretty good speedup to be had from handle re-use.  So, a simple
+	/// vector is kept of 'freed' handles to be reused as needed.  When
+	/// that is empty, the first freed handle is kept as a template for
+	/// handle duplication.  This is still faster than creation from nothing.
+	/// And when that fails, we init fresh from curl_easy_init().
+	///
+	/// Handles allocated with getHandle() may be freed with either
+	/// freeHandle() or curl_easy_cleanup().  Choice may be dictated
+	/// by thread constraints.
+	///
+	/// Threading:  Single-threaded.  May only be used by a single thread,
+	/// typically the worker thread.  If freeing requests' handles in an
+	/// unknown threading context, use curl_easy_cleanup() for safety.
+
+	class HandleCache
+	{
+	public:
+		HandleCache();
+		~HandleCache();
+
+	private:
+		HandleCache(const HandleCache &);				// Not defined
+		void operator=(const HandleCache &);			// Not defined
+
+	public:
+		/// Allocate a curl handle for caller.  May be freed using
+		/// either the freeHandle() method or calling curl_easy_cleanup()
+		/// directly.
+		///
+		/// @return			Libcurl handle (CURL *) or NULL on allocation
+		///					problem.
+		///
+		/// Threading:  Single-thread (worker) only.
+		CURL * getHandle();
+
+		/// Free a libcurl handle acquired by whatever means.  Thread
+		/// safety is left to the caller.
+		///
+		/// Threading:  Single-thread (worker) only.
+		void freeHandle(CURL * handle);
+
+	protected:
+		typedef std::vector<CURL *> handle_cache_t;
+	
+	protected:
+		CURL *				mHandleTemplate;		// Template for duplicating new handles
+		handle_cache_t		mCache;					// Cache of old handles
+	}; // end class HandleCache
 	
 protected:
-	HttpService *		mService;				// Simple reference, not owner
+	HttpService *		mService;			// Simple reference, not owner
+	HandleCache			mHandleCache;		// Handle allocator, owner
 	active_set_t		mActiveOps;
 	int					mPolicyCount;
-	CURLM **			mMultiHandles;			// One handle per policy class
-	int *				mActiveHandles;			// Active count per policy class
-	bool *				mDirtyPolicy;			// Dirty policy update waiting for stall (per pc)
+	CURLM **			mMultiHandles;		// One handle per policy class
+	int *				mActiveHandles;		// Active count per policy class
+	bool *				mDirtyPolicy;		// Dirty policy update waiting for stall (per pc)
 	
 }; // end class HttpLibcurl
 
diff --git a/indra/llcorehttp/_httpoprequest.cpp b/indra/llcorehttp/_httpoprequest.cpp
index 4453bf2922..bbda0b82fd 100755
--- a/indra/llcorehttp/_httpoprequest.cpp
+++ b/indra/llcorehttp/_httpoprequest.cpp
@@ -170,6 +170,8 @@ HttpOpRequest::~HttpOpRequest()
 
 	if (mCurlHandle)
 	{
+		// Uncertain of thread context so free using
+		// safest method.
 		curl_easy_cleanup(mCurlHandle);
 		mCurlHandle = NULL;
 	}
@@ -429,7 +431,7 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service)
 	HttpPolicyGlobal & gpolicy(service->getPolicy().getGlobalOptions());
 	HttpPolicyClass & cpolicy(service->getPolicy().getClassOptions(mReqPolicy));
 	
-	mCurlHandle = LLCurl::createStandardCurlHandle();
+	mCurlHandle = service->getTransport().getHandle();
 	if (! mCurlHandle)
 	{
 		// We're in trouble.  We'll continue but it won't go well.
@@ -437,6 +439,7 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service)
 						   << LL_ENDL;
 		return HttpStatus(HttpStatus::LLCORE, HE_BAD_ALLOC);
 	}
+
 	code = curl_easy_setopt(mCurlHandle, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4);
 	check_curl_easy_code(code, CURLOPT_IPRESOLVE);
 	code = curl_easy_setopt(mCurlHandle, CURLOPT_NOSIGNAL, 1);
-- 
cgit v1.2.3


From 329608d24668b044e16b54ff7a7d0ac592b2b88d Mon Sep 17 00:00:00 2001
From: Monty Brandenberg <monty@lindenlab.com>
Date: Mon, 22 Sep 2014 18:49:45 -0400
Subject: Tuning and documentation.  Use a fast poll frequency (0.05S) on the
 HTTP requests for inventory.  We'll benchmark with that and see how it goes. 
 Document some of the history of the background fetcher for future devs. 
 Suggest some future projects to make things faster.  Pointers on using LLSD
 with the llcorehttp library in the readme.  And restructured the LLSD
 onCompleted() processing phases using do{}while(false) which produced a code
 flow that is fairly attractive.

---
 indra/llcorehttp/README.Linden | 8 ++++++++
 1 file changed, 8 insertions(+)

(limited to 'indra/llcorehttp')

diff --git a/indra/llcorehttp/README.Linden b/indra/llcorehttp/README.Linden
index eb6ccab3bc..c3aaa9158d 100644
--- a/indra/llcorehttp/README.Linden
+++ b/indra/llcorehttp/README.Linden
@@ -529,6 +529,14 @@ HttpOperation::addAsReply: TRACE, ToReplyQueue, Handle:  086D3148
         data = NULL;
 
 
+    There are now helper functions in llmessage/llcorehttputil.h to
+    assist with LLSD usage.  requestPostWithLLSD(...) provides a
+    requestPost()-like interface that takes an LLSD object rather than
+    a BufferArray.  And responseToLLSD(...) attempts to convert a
+    BufferArray received from a server into an LLSD object.  You can
+    find examples in llmeshrepository.cpp, llinventorymodel.cpp,
+    llinventorymodelbackgroundfetch.cpp and lltexturefetch.cpp.
+
     LLSD will often go hand-in-hand with BufferArray and data
     transport.  But you can also do all the streaming I/O you'd expect
     of a std::iostream object:
-- 
cgit v1.2.3


From ec4fd2f0e226bb2cae5982760317e1d6ea2d2d69 Mon Sep 17 00:00:00 2001
From: Monty Brandenberg <monty@lindenlab.com>
Date: Fri, 10 Oct 2014 16:43:04 -0400
Subject: MAINT-4564 HTTP Pipelining is not happening in Drano HTTP Phase 4
 Incorporate the new libcurl 7.38.0 build with curl bug 1420 workaround.  Add
 developer-centric testing code to evaluate the workaround or a future fix for
 1420.

---
 indra/llcorehttp/_httpoprequest.cpp |  4 ++--
 indra/llcorehttp/_httppolicy.cpp    | 12 ++++++++++++
 indra/llcorehttp/httpcommon.cpp     |  5 +++--
 3 files changed, 17 insertions(+), 4 deletions(-)

(limited to 'indra/llcorehttp')

diff --git a/indra/llcorehttp/_httpoprequest.cpp b/indra/llcorehttp/_httpoprequest.cpp
index bbda0b82fd..fbbb1614fb 100755
--- a/indra/llcorehttp/_httpoprequest.cpp
+++ b/indra/llcorehttp/_httpoprequest.cpp
@@ -626,8 +626,8 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service)
 		// *TODO:  Find a better scheme than timeouts to guarantee liveness.
 		xfer_timeout *= cpolicy.mPipelining;
 	}
-	// *DEBUG:  Useful for timeout handling and "[curl:bugs] #1420" tests
-	// xfer_timeout = 3L;
+	// *DEBUG:  Enable following override for timeout handling and "[curl:bugs] #1420" tests
+	// xfer_timeout = 1L;
 	code = curl_easy_setopt(mCurlHandle, CURLOPT_TIMEOUT, xfer_timeout);
 	check_curl_easy_code(code, CURLOPT_TIMEOUT);
 	code = curl_easy_setopt(mCurlHandle, CURLOPT_CONNECTTIMEOUT, timeout);
diff --git a/indra/llcorehttp/_httppolicy.cpp b/indra/llcorehttp/_httppolicy.cpp
index 09b9206f63..e5d6321401 100755
--- a/indra/llcorehttp/_httppolicy.cpp
+++ b/indra/llcorehttp/_httppolicy.cpp
@@ -414,6 +414,18 @@ bool HttpPolicy::stageAfterCompletion(HttpOpRequest * op)
 	// Retry or finalize
 	if (! op->mStatus)
 	{
+		// *DEBUG:  For "[curl:bugs] #1420" tests.  This will interfere
+		// with unit tests due to allocation retention by logging code.
+		// But you won't be checking this in enabled.
+#if 0
+		if (op->mStatus == HttpStatus(HttpStatus::EXT_CURL_EASY, CURLE_OPERATION_TIMEDOUT))
+		{
+			LL_WARNS(LOG_CORE) << "HTTP request " << static_cast<HttpHandle>(op)
+							   << " timed out."
+							   << LL_ENDL;
+		}
+#endif
+		
 		// If this failed, we might want to retry.
 		if (op->mPolicyRetries < op->mPolicyRetryLimit && op->mStatus.isRetryable())
 		{
diff --git a/indra/llcorehttp/httpcommon.cpp b/indra/llcorehttp/httpcommon.cpp
index 8714915fa2..7907e958a4 100755
--- a/indra/llcorehttp/httpcommon.cpp
+++ b/indra/llcorehttp/httpcommon.cpp
@@ -254,8 +254,9 @@ bool HttpStatus::isRetryable() const
 			*this == op_timedout ||		// Timer expired
 			*this == post_error ||		// Transport problem
 			*this == partial_file ||	// Data inconsistency in response
-			*this == inv_cont_range ||	// Short data read disagrees with content-range
-			*this == inv_status);		// Inv status can reflect internal state problem in libcurl
+			// *DEBUG:  Comment out 'inv_status' test for [curl:bugs] #1420 testing.
+			*this == inv_status ||		// Inv status can reflect internal state problem in libcurl
+			*this == inv_cont_range);	// Short data read disagrees with content-range
 }
 
 } // end namespace LLCore
-- 
cgit v1.2.3