Better support for dynamic option changes in llcorehttp. Libcurl has

some problems disabling pipelining on a multi handle with outstanding
requests so build a more conservative system that allows requests
to drain before setting curl multi options.  Would rather not have
this but it is significantly safer.  "HttpPipelining" debug setting
is now fully dynamic.  Connection limits can also be made dynamic
in the near future.  Upped the default connection count back to 8 for
now but will revisit this in the tuning phase.  It might be time to
combine mesh and textures into a single asset class.  For normal
server operations that would be a clear path, but for server under
load, the current scheme may be better.  Minor cleanup in logging
to elminate some redundant strings.  Might add some more tracing to the
stall logic 'just in case'.

1 files changed, 57 insertions, 26 deletions

diff --git a/indra/llcorehttp/_httppolicy.cpp b/indra/llcorehttp/_httppolicy.cpp
index bb7959b578..09b9206f63 100755
--- a/indra/llcorehttp/_httppolicy.cpp
+++ b/indra/llcorehttp/_httppolicy.cpp
@@ -35,6 +35,13 @@
 
 #include "lltimer.h"
 
+namespace
+{
+
+static const char * const LOG_CORE("CoreHttp");
+
+} // end anonymous namespace
+
 
 namespace LLCore
 {
@@ -51,7 +58,8 @@ public:
 	ClassState()
 		: mThrottleEnd(0),
 		  mThrottleLeft(0L),
-		  mRequestCount(0L)
+		  mRequestCount(0L),
+		  mStallStaging(false)
 		{}
 	
 	HttpReadyQueue		mReadyQueue;
@@ -61,6 +69,7 @@ public:
 	HttpTime			mThrottleEnd;
 	long				mThrottleLeft;
 	long				mRequestCount;
+	bool				mStallStaging;
 };
 
 
@@ -171,19 +180,19 @@ void HttpPolicy::retryOp(HttpOpRequest * op)
 	{
 		++op->mPolicy503Retries;
 	}
-	LL_DEBUGS("CoreHttp") << "HTTP request " << static_cast<HttpHandle>(op)
-						  << " retry " << op->mPolicyRetries
-						  << " scheduled in " << (delta / HttpTime(1000))
-						  << " mS (" << (external_delta ? "external" : "internal")
-						  << ").  Status:  " << op->mStatus.toTerseString()
-						  << LL_ENDL;
+	LL_DEBUGS(LOG_CORE) << "HTTP request " << static_cast<HttpHandle>(op)
+						<< " retry " << op->mPolicyRetries
+						<< " scheduled in " << (delta / HttpTime(1000))
+						<< " mS (" << (external_delta ? "external" : "internal")
+						<< ").  Status:  " << op->mStatus.toTerseString()
+						<< LL_ENDL;
 	if (op->mTracing > HTTP_TRACE_OFF)
 	{
-		LL_INFOS("CoreHttp") << "TRACE, ToRetryQueue, Handle:  "
-							 << static_cast<HttpHandle>(op)
-							 << ", Delta:  " << (delta / HttpTime(1000))
-							 << ", Retries:  " << op->mPolicyRetries
-							 << LL_ENDL;
+		LL_INFOS(LOG_CORE) << "TRACE, ToRetryQueue, Handle:  "
+						   << static_cast<HttpHandle>(op)
+						   << ", Delta:  " << (delta / HttpTime(1000))
+						   << ", Retries:  " << op->mPolicyRetries
+						   << LL_ENDL;
 	}
 	mClasses[policy_class]->mRetryQueue.push(op);
 }
@@ -219,6 +228,15 @@ HttpService::ELoopSpeed HttpPolicy::processReadyQueue()
 		HttpRetryQueue & retryq(state.mRetryQueue);
 		HttpReadyQueue & readyq(state.mReadyQueue);
 
+		if (state.mStallStaging)
+		{
+			// Stalling but don't sleep.  Need to complete operations
+			// and get back to servicing queues.  Do this test before
+			// the retryq/readyq test or you'll get stalls until you
+			// click a setting or an asset request comes in.
+			result = HttpService::NORMAL;
+			continue;
+		}
 		if (retryq.empty() && readyq.empty())
 		{
 			continue;
@@ -262,9 +280,9 @@ HttpService::ELoopSpeed HttpPolicy::processReadyQueue()
 					if (now >= state.mThrottleEnd)
 					{
 						// Throttle expired, move to next window
-						LL_DEBUGS("CoreHttp") << "Throttle expired with " << state.mThrottleLeft
-											  << " requests to go and " << state.mRequestCount
-											  << " requests issued." << LL_ENDL;
+						LL_DEBUGS(LOG_CORE) << "Throttle expired with " << state.mThrottleLeft
+											<< " requests to go and " << state.mRequestCount
+											<< " requests issued." << LL_ENDL;
 						state.mThrottleLeft = state.mOptions.mThrottleRate;
 						state.mThrottleEnd = now + HttpTime(1000000);
 					}
@@ -291,9 +309,9 @@ HttpService::ELoopSpeed HttpPolicy::processReadyQueue()
 					if (now >= state.mThrottleEnd)
 					{
 						// Throttle expired, move to next window
-						LL_DEBUGS("CoreHttp") << "Throttle expired with " << state.mThrottleLeft
-											  << " requests to go and " << state.mRequestCount
-											  << " requests issued." << LL_ENDL;
+						LL_DEBUGS(LOG_CORE) << "Throttle expired with " << state.mThrottleLeft
+											<< " requests to go and " << state.mRequestCount
+											<< " requests issued." << LL_ENDL;
 						state.mThrottleLeft = state.mOptions.mThrottleRate;
 						state.mThrottleEnd = now + HttpTime(1000000);
 					}
@@ -408,17 +426,17 @@ bool HttpPolicy::stageAfterCompletion(HttpOpRequest * op)
 	// This op is done, finalize it delivering it to the reply queue...
 	if (! op->mStatus)
 	{
-		LL_WARNS("CoreHttp") << "HTTP request " << static_cast<HttpHandle>(op)
-							 << " failed after " << op->mPolicyRetries
-							 << " retries.  Reason:  " << op->mStatus.toString()
-							 << " (" << op->mStatus.toTerseString() << ")"
-							 << LL_ENDL;
+		LL_WARNS(LOG_CORE) << "HTTP request " << static_cast<HttpHandle>(op)
+						   << " failed after " << op->mPolicyRetries
+						   << " retries.  Reason:  " << op->mStatus.toString()
+						   << " (" << op->mStatus.toTerseString() << ")"
+						   << LL_ENDL;
 	}
 	else if (op->mPolicyRetries)
 	{
-		LL_DEBUGS("CoreHttp") << "HTTP request " << static_cast<HttpHandle>(op)
-							  << " succeeded on retry " << op->mPolicyRetries << "."
-							  << LL_ENDL;
+		LL_DEBUGS(LOG_CORE) << "HTTP request " << static_cast<HttpHandle>(op)
+							<< " succeeded on retry " << op->mPolicyRetries << "."
+							<< LL_ENDL;
 	}
 
 	op->stageFromActive(mService);
@@ -446,4 +464,17 @@ int HttpPolicy::getReadyCount(HttpRequest::policy_t policy_class) const
 }
 
 
+bool HttpPolicy::stallPolicy(HttpRequest::policy_t policy_class, bool stall)
+{
+	bool ret(false);
+	
+	if (policy_class < mClasses.size())
+	{
+		ret = mClasses[policy_class]->mStallStaging;
+		mClasses[policy_class]->mStallStaging = stall;
+	}
+	return ret;
+}
+
+
 }  // end namespace LLCore