1 files changed, 260 insertions, 37 deletions

diff --git a/indra/llcorehttp/_httplibcurl.cpp b/indra/llcorehttp/_httplibcurl.cpp
index e56bc84174..81b44ab90b 100755
--- a/indra/llcorehttp/_httplibcurl.cpp
+++ b/indra/llcorehttp/_httplibcurl.cpp
@@ -4,7 +4,7 @@
  *
  * $LicenseInfo:firstyear=2012&license=viewerlgpl$
  * Second Life Viewer Source Code
- * Copyright (C) 2012-2013, Linden Research, Inc.
+ * Copyright (C) 2012-2014, Linden Research, Inc.
  *
  * This library is free software; you can redistribute it and/or
  * modify it under the terms of the GNU Lesser General Public
@@ -33,6 +33,17 @@
 
 #include "llhttpconstants.h"
 
+namespace
+{
+
+// Error testing and reporting for libcurl status codes
+void check_curl_multi_code(CURLMcode code);
+void check_curl_multi_code(CURLMcode code, int curl_setopt_option);
+
+static const char * const LOG_CORE("CoreHttp");
+
+} // end anonymous namespace
+
 
 namespace LLCore
 {
@@ -40,16 +51,18 @@ namespace LLCore
 
 HttpLibcurl::HttpLibcurl(HttpService * service)
 	: mService(service),
+	  mHandleCache(),
 	  mPolicyCount(0),
 	  mMultiHandles(NULL),
-	  mActiveHandles(NULL)
+	  mActiveHandles(NULL),
+	  mDirtyPolicy(NULL)
 {}
 
 
 HttpLibcurl::~HttpLibcurl()
 {
 	shutdown();
-	
+
 	mService = NULL;
 }
 
@@ -81,6 +94,9 @@ void HttpLibcurl::shutdown()
 
 		delete [] mActiveHandles;
 		mActiveHandles = NULL;
+
+		delete [] mDirtyPolicy;
+		mDirtyPolicy = NULL;
 	}
 
 	mPolicyCount = 0;
@@ -95,11 +111,18 @@ void HttpLibcurl::start(int policy_count)
 	mPolicyCount = policy_count;
 	mMultiHandles = new CURLM * [mPolicyCount];
 	mActiveHandles = new int [mPolicyCount];
+	mDirtyPolicy = new bool [mPolicyCount];
 	
 	for (int policy_class(0); policy_class < mPolicyCount; ++policy_class)
 	{
-		mMultiHandles[policy_class] = curl_multi_init();
+		if (NULL == (mMultiHandles[policy_class] = curl_multi_init()))
+		{
+			LL_ERRS(LOG_CORE) << "Failed to allocate multi handle in libcurl."
+							  << LL_ENDL;
+		}
 		mActiveHandles[policy_class] = 0;
+		mDirtyPolicy[policy_class] = false;
+		policyUpdated(policy_class);
 	}
 }
 
@@ -117,8 +140,19 @@ HttpService::ELoopSpeed HttpLibcurl::processTransport()
 	// Give libcurl some cycles to do I/O & callbacks
 	for (int policy_class(0); policy_class < mPolicyCount; ++policy_class)
 	{
-		if (! mActiveHandles[policy_class] || ! mMultiHandles[policy_class])
+		if (! mMultiHandles[policy_class])
+		{
+			// No handle, nothing to do.
+			continue;
+		}
+		if (! mActiveHandles[policy_class])
 		{
+			// If we've gone quiet and there's a dirty update, apply it,
+			// otherwise we're done.
+			if (mDirtyPolicy[policy_class])
+			{
+				policyUpdated(policy_class);
+			}
 			continue;
 		}
 		
@@ -153,9 +187,9 @@ HttpService::ELoopSpeed HttpLibcurl::processTransport()
 			}
 			else
 			{
-				LL_WARNS_ONCE("CoreHttp") << "Unexpected message from libcurl.  Msg code:  "
-										  << msg->msg
-										  << LL_ENDL;
+				LL_WARNS_ONCE(LOG_CORE) << "Unexpected message from libcurl.  Msg code:  "
+										<< msg->msg
+										<< LL_ENDL;
 			}
 			msgs_in_queue = 0;
 		}
@@ -184,23 +218,28 @@ void HttpLibcurl::addOp(HttpOpRequest * op)
 	}
 
 	// Make the request live
-	curl_multi_add_handle(mMultiHandles[op->mReqPolicy], op->mCurlHandle);
+	CURLMcode code;
+	code = curl_multi_add_handle(mMultiHandles[op->mReqPolicy], op->mCurlHandle);
+	if (CURLM_OK != code)
+	{
+		// *TODO:  Better cleanup and recovery but not much we can do here.
+		check_curl_multi_code(code);
+		return;
+	}
 	op->mCurlActive = true;
+	mActiveOps.insert(op);
+	++mActiveHandles[op->mReqPolicy];
 	
 	if (op->mTracing > HTTP_TRACE_OFF)
 	{
 		HttpPolicy & policy(mService->getPolicy());
 		
-		LL_INFOS("CoreHttp") << "TRACE, ToActiveQueue, Handle:  "
-							 << static_cast<HttpHandle>(op)
-							 << ", Actives:  " << mActiveOps.size()
-							 << ", Readies:  " << policy.getReadyCount(op->mReqPolicy)
-							 << LL_ENDL;
+		LL_INFOS(LOG_CORE) << "TRACE, ToActiveQueue, Handle:  "
+						   << static_cast<HttpHandle>(op)
+						   << ", Actives:  " << mActiveOps.size()
+						   << ", Readies:  " << policy.getReadyCount(op->mReqPolicy)
+						   << LL_ENDL;
 	}
-	
-	// On success, make operation active
-	mActiveOps.insert(op);
-	++mActiveHandles[op->mReqPolicy];
 }
 
 
@@ -241,16 +280,16 @@ void HttpLibcurl::cancelRequest(HttpOpRequest * op)
 
 	// Detach from multi and recycle handle
 	curl_multi_remove_handle(mMultiHandles[op->mReqPolicy], op->mCurlHandle);
-	curl_easy_cleanup(op->mCurlHandle);
+	mHandleCache.freeHandle(op->mCurlHandle);
 	op->mCurlHandle = NULL;
 
 	// Tracing
 	if (op->mTracing > HTTP_TRACE_OFF)
 	{
-		LL_INFOS("CoreHttp") << "TRACE, RequestCanceled, Handle:  "
-							 << static_cast<HttpHandle>(op)
-							 << ", Status:  " << op->mStatus.toTerseString()
-							 << LL_ENDL;
+		LL_INFOS(LOG_CORE) << "TRACE, RequestCanceled, Handle:  "
+						   << static_cast<HttpHandle>(op)
+						   << ", Status:  " << op->mStatus.toTerseString()
+						   << LL_ENDL;
 	}
 
 	// Cancel op and deliver for notification
@@ -267,18 +306,18 @@ bool HttpLibcurl::completeRequest(CURLM * multi_handle, CURL * handle, CURLcode
 
 	if (handle != op->mCurlHandle || ! op->mCurlActive)
 	{
-		LL_WARNS("CoreHttp") << "libcurl handle and HttpOpRequest handle in disagreement or inactive request."
-							 << "  Handle:  " << static_cast<HttpHandle>(handle)
-							 << LL_ENDL;
+		LL_WARNS(LOG_CORE) << "libcurl handle and HttpOpRequest handle in disagreement or inactive request."
+						   << "  Handle:  " << static_cast<HttpHandle>(handle)
+						   << LL_ENDL;
 		return false;
 	}
 
 	active_set_t::iterator it(mActiveOps.find(op));
 	if (mActiveOps.end() == it)
 	{
-		LL_WARNS("CoreHttp") << "libcurl completion for request not on active list.  Continuing."
-							 << "  Handle:  " << static_cast<HttpHandle>(handle)
-							 << LL_ENDL;
+		LL_WARNS(LOG_CORE) << "libcurl completion for request not on active list.  Continuing."
+						   << "  Handle:  " << static_cast<HttpHandle>(handle)
+						   << LL_ENDL;
 		return false;
 	}
 
@@ -309,25 +348,25 @@ bool HttpLibcurl::completeRequest(CURLM * multi_handle, CURL * handle, CURLcode
 		}
 		else
 		{
-			LL_WARNS("CoreHttp") << "Invalid HTTP response code ("
-								 << http_status << ") received from server."
-								 << LL_ENDL;
+			LL_WARNS(LOG_CORE) << "Invalid HTTP response code ("
+							   << http_status << ") received from server."
+							   << LL_ENDL;
 			op->mStatus = HttpStatus(HttpStatus::LLCORE, HE_INVALID_HTTP_STATUS);
 		}
 	}
 
 	// Detach from multi and recycle handle
 	curl_multi_remove_handle(multi_handle, handle);
-	curl_easy_cleanup(handle);
+	mHandleCache.freeHandle(op->mCurlHandle);
 	op->mCurlHandle = NULL;
 
 	// Tracing
 	if (op->mTracing > HTTP_TRACE_OFF)
 	{
-		LL_INFOS("CoreHttp") << "TRACE, RequestComplete, Handle:  "
-							 << static_cast<HttpHandle>(op)
-							 << ", Status:  " << op->mStatus.toTerseString()
-							 << LL_ENDL;
+		LL_INFOS(LOG_CORE) << "TRACE, RequestComplete, Handle:  "
+						   << static_cast<HttpHandle>(op)
+						   << ", Status:  " << op->mStatus.toTerseString()
+						   << LL_ENDL;
 	}
 
 	// Dispatch to next stage
@@ -351,6 +390,164 @@ int HttpLibcurl::getActiveCountInClass(int policy_class) const
 	return mActiveHandles ? mActiveHandles[policy_class] : 0;
 }
 
+void HttpLibcurl::policyUpdated(int policy_class)
+{
+	if (policy_class < 0 || policy_class >= mPolicyCount || ! mMultiHandles)
+	{
+		return;
+	}
+	
+	HttpPolicy & policy(mService->getPolicy());
+	
+	if (! mActiveHandles[policy_class])
+	{
+		// Clear to set options.  As of libcurl 7.37.0, if a pipelining
+		// multi handle has active requests and you try to set the
+		// multi handle to non-pipelining, the library gets very angry
+		// and goes off the rails corrupting memory.  A clue that you're
+		// about to crash is that you'll get a missing server response
+		// error (curl code 9).  So, if options are to be set, we let
+		// the multi handle run out of requests, then set options, and
+		// re-enable request processing.
+		//
+		// All of this stall mechanism exists for this reason.  If
+		// libcurl becomes more resilient later, it should be possible
+		// to remove all of this.  The connection limit settings are fine,
+		// it's just that pipelined-to-non-pipelined transition that
+		// is fatal at the moment.
+		
+		HttpPolicyClass & options(policy.getClassOptions(policy_class));
+		CURLM * multi_handle(mMultiHandles[policy_class]);
+		CURLMcode code;
+
+		// Enable policy if stalled
+		policy.stallPolicy(policy_class, false);
+		mDirtyPolicy[policy_class] = false;
+		
+		if (options.mPipelining > 1)
+		{
+			// We'll try to do pipelining on this multihandle
+			code = curl_multi_setopt(multi_handle,
+									 CURLMOPT_PIPELINING,
+									 1L);
+			check_curl_multi_code(code, CURLMOPT_PIPELINING);
+			code = curl_multi_setopt(multi_handle,
+									 CURLMOPT_MAX_PIPELINE_LENGTH,
+									 long(options.mPipelining));
+			check_curl_multi_code(code, CURLMOPT_MAX_PIPELINE_LENGTH);
+			code = curl_multi_setopt(multi_handle,
+									 CURLMOPT_MAX_HOST_CONNECTIONS,
+									 long(options.mPerHostConnectionLimit));
+			check_curl_multi_code(code, CURLMOPT_MAX_HOST_CONNECTIONS);
+			code = curl_multi_setopt(multi_handle,
+									 CURLMOPT_MAX_TOTAL_CONNECTIONS,
+									 long(options.mConnectionLimit));
+			check_curl_multi_code(code, CURLMOPT_MAX_TOTAL_CONNECTIONS);
+		}
+		else
+		{
+			code = curl_multi_setopt(multi_handle,
+									 CURLMOPT_PIPELINING,
+									 0L);
+			check_curl_multi_code(code, CURLMOPT_PIPELINING);
+			code = curl_multi_setopt(multi_handle,
+									 CURLMOPT_MAX_HOST_CONNECTIONS,
+									 0L);
+			check_curl_multi_code(code, CURLMOPT_MAX_HOST_CONNECTIONS);
+			code = curl_multi_setopt(multi_handle,
+									 CURLMOPT_MAX_TOTAL_CONNECTIONS,
+									 long(options.mConnectionLimit));
+			check_curl_multi_code(code, CURLMOPT_MAX_TOTAL_CONNECTIONS);
+		}
+	}
+	else if (! mDirtyPolicy[policy_class])
+	{
+		// Mark policy dirty and request a stall in the policy.
+		// When policy goes idle, we'll re-invoke this method
+		// and perform the change.  Don't allow this thread to
+		// sleep while we're waiting for quiescence, we'll just
+		// stop processing.
+		mDirtyPolicy[policy_class] = true;
+		policy.stallPolicy(policy_class, true);
+	}
+}
+
+// ---------------------------------------
+// HttpLibcurl::HandleCache
+// ---------------------------------------
+
+HttpLibcurl::HandleCache::HandleCache()
+	: mHandleTemplate(NULL)
+{
+	mCache.reserve(50);
+}
+
+
+HttpLibcurl::HandleCache::~HandleCache()
+{
+	if (mHandleTemplate)
+	{
+		curl_easy_cleanup(mHandleTemplate);
+		mHandleTemplate = NULL;
+	}
+
+	for (handle_cache_t::iterator it(mCache.begin()); mCache.end() != it; ++it)
+	{
+		curl_easy_cleanup(*it);
+	}
+	mCache.clear();
+}
+
+
+CURL * HttpLibcurl::HandleCache::getHandle()
+{
+	CURL * ret(NULL);
+	
+	if (! mCache.empty())
+	{
+		// Fastest path to handle
+		ret = mCache.back();
+		mCache.pop_back();
+	}
+	else if (mHandleTemplate)
+	{
+		// Still fast path
+		ret = curl_easy_duphandle(mHandleTemplate);
+	}
+	else
+	{
+		// When all else fails
+		ret = curl_easy_init();
+	}
+
+	return ret;
+}
+
+
+void HttpLibcurl::HandleCache::freeHandle(CURL * handle)
+{
+	if (! handle)
+	{
+		return;
+	}
+
+	curl_easy_reset(handle);
+	if (! mHandleTemplate)
+	{
+		// Save the first freed handle as a template.
+		mHandleTemplate = handle;
+	}
+	else
+	{
+		// Otherwise add it to the cache
+		if (mCache.size() >= mCache.capacity())
+		{
+			mCache.reserve(mCache.capacity() + 50);
+		}
+		mCache.push_back(handle);
+	}
+}
+
 
 // ---------------------------------------
 // Free functions
@@ -376,3 +573,29 @@ struct curl_slist * append_headers_to_slist(const HttpHeaders * headers, struct
 
 
 }  // end namespace LLCore
+
+
+namespace 
+{
+	
+void check_curl_multi_code(CURLMcode code, int curl_setopt_option)
+{
+	if (CURLM_OK != code)
+	{
+		LL_WARNS(LOG_CORE) << "libcurl multi error detected:  " << curl_multi_strerror(code)
+						   << ", curl_multi_setopt option:  " << curl_setopt_option
+						   << LL_ENDL;
+	}
+}
+
+
+void check_curl_multi_code(CURLMcode code)
+{
+	if (CURLM_OK != code)
+	{
+		LL_WARNS(LOG_CORE) << "libcurl multi error detected:  " << curl_multi_strerror(code)
+						   << LL_ENDL;
+	}
+}
+
+}  // end anonymous namespace