1 files changed, 282 insertions, 54 deletions

diff --git a/indra/llcorehttp/_httplibcurl.cpp b/indra/llcorehttp/_httplibcurl.cpp
index 6fe0bfc7d1..81b44ab90b 100644..100755
--- a/indra/llcorehttp/_httplibcurl.cpp
+++ b/indra/llcorehttp/_httplibcurl.cpp
@@ -4,7 +4,7 @@
  *
  * $LicenseInfo:firstyear=2012&license=viewerlgpl$
  * Second Life Viewer Source Code
- * Copyright (C) 2012, Linden Research, Inc.
+ * Copyright (C) 2012-2014, Linden Research, Inc.
  *
  * This library is free software; you can redistribute it and/or
  * modify it under the terms of the GNU Lesser General Public
@@ -31,7 +31,18 @@
 #include "_httpoprequest.h"
 #include "_httppolicy.h"
 
-#include "llhttpstatuscodes.h"
+#include "llhttpconstants.h"
+
+namespace
+{
+
+// Error testing and reporting for libcurl status codes
+void check_curl_multi_code(CURLMcode code);
+void check_curl_multi_code(CURLMcode code, int curl_setopt_option);
+
+static const char * const LOG_CORE("CoreHttp");
+
+} // end anonymous namespace
 
 
 namespace LLCore
@@ -40,15 +51,18 @@ namespace LLCore
 
 HttpLibcurl::HttpLibcurl(HttpService * service)
 	: mService(service),
+	  mHandleCache(),
 	  mPolicyCount(0),
-	  mMultiHandles(NULL)
+	  mMultiHandles(NULL),
+	  mActiveHandles(NULL),
+	  mDirtyPolicy(NULL)
 {}
 
 
 HttpLibcurl::~HttpLibcurl()
 {
 	shutdown();
-	
+
 	mService = NULL;
 }
 
@@ -77,6 +91,12 @@ void HttpLibcurl::shutdown()
 
 		delete [] mMultiHandles;
 		mMultiHandles = NULL;
+
+		delete [] mActiveHandles;
+		mActiveHandles = NULL;
+
+		delete [] mDirtyPolicy;
+		mDirtyPolicy = NULL;
 	}
 
 	mPolicyCount = 0;
@@ -90,9 +110,19 @@ void HttpLibcurl::start(int policy_count)
 	
 	mPolicyCount = policy_count;
 	mMultiHandles = new CURLM * [mPolicyCount];
+	mActiveHandles = new int [mPolicyCount];
+	mDirtyPolicy = new bool [mPolicyCount];
+	
 	for (int policy_class(0); policy_class < mPolicyCount; ++policy_class)
 	{
-		mMultiHandles[policy_class] = curl_multi_init();
+		if (NULL == (mMultiHandles[policy_class] = curl_multi_init()))
+		{
+			LL_ERRS(LOG_CORE) << "Failed to allocate multi handle in libcurl."
+							  << LL_ENDL;
+		}
+		mActiveHandles[policy_class] = 0;
+		mDirtyPolicy[policy_class] = false;
+		policyUpdated(policy_class);
 	}
 }
 
@@ -111,7 +141,20 @@ HttpService::ELoopSpeed HttpLibcurl::processTransport()
 	for (int policy_class(0); policy_class < mPolicyCount; ++policy_class)
 	{
 		if (! mMultiHandles[policy_class])
+		{
+			// No handle, nothing to do.
 			continue;
+		}
+		if (! mActiveHandles[policy_class])
+		{
+			// If we've gone quiet and there's a dirty update, apply it,
+			// otherwise we're done.
+			if (mDirtyPolicy[policy_class])
+			{
+				policyUpdated(policy_class);
+			}
+			continue;
+		}
 		
 		int running(0);
 		CURLMcode status(CURLM_CALL_MULTI_PERFORM);
@@ -132,12 +175,10 @@ HttpService::ELoopSpeed HttpLibcurl::processTransport()
 				CURL * handle(msg->easy_handle);
 				CURLcode result(msg->data.result);
 
-				if (completeRequest(mMultiHandles[policy_class], handle, result))
-				{
-					// Request is still active, don't get too sleepy
-					ret = HttpService::NORMAL;
-				}
-				handle = NULL;			// No longer valid on return
+				completeRequest(mMultiHandles[policy_class], handle, result);
+				handle = NULL;					// No longer valid on return
+				ret = HttpService::NORMAL;		// If anything completes, we may have a free slot.
+												// Turning around quickly reduces connection gap by 7-10mS.
 			}
 			else if (CURLMSG_NONE == msg->msg)
 			{
@@ -146,9 +187,9 @@ HttpService::ELoopSpeed HttpLibcurl::processTransport()
 			}
 			else
 			{
-				LL_WARNS_ONCE("CoreHttp") << "Unexpected message from libcurl.  Msg code:  "
-										  << msg->msg
-										  << LL_ENDL;
+				LL_WARNS_ONCE(LOG_CORE) << "Unexpected message from libcurl.  Msg code:  "
+										<< msg->msg
+										<< LL_ENDL;
 			}
 			msgs_in_queue = 0;
 		}
@@ -177,22 +218,28 @@ void HttpLibcurl::addOp(HttpOpRequest * op)
 	}
 
 	// Make the request live
-	curl_multi_add_handle(mMultiHandles[op->mReqPolicy], op->mCurlHandle);
+	CURLMcode code;
+	code = curl_multi_add_handle(mMultiHandles[op->mReqPolicy], op->mCurlHandle);
+	if (CURLM_OK != code)
+	{
+		// *TODO:  Better cleanup and recovery but not much we can do here.
+		check_curl_multi_code(code);
+		return;
+	}
 	op->mCurlActive = true;
+	mActiveOps.insert(op);
+	++mActiveHandles[op->mReqPolicy];
 	
 	if (op->mTracing > HTTP_TRACE_OFF)
 	{
 		HttpPolicy & policy(mService->getPolicy());
 		
-		LL_INFOS("CoreHttp") << "TRACE, ToActiveQueue, Handle:  "
-							 << static_cast<HttpHandle>(op)
-							 << ", Actives:  " << mActiveOps.size()
-							 << ", Readies:  " << policy.getReadyCount(op->mReqPolicy)
-							 << LL_ENDL;
+		LL_INFOS(LOG_CORE) << "TRACE, ToActiveQueue, Handle:  "
+						   << static_cast<HttpHandle>(op)
+						   << ", Actives:  " << mActiveOps.size()
+						   << ", Readies:  " << policy.getReadyCount(op->mReqPolicy)
+						   << LL_ENDL;
 	}
-	
-	// On success, make operation active
-	mActiveOps.insert(op);
 }
 
 
@@ -214,6 +261,7 @@ bool HttpLibcurl::cancel(HttpHandle handle)
 
 	// Drop references
 	mActiveOps.erase(it);
+	--mActiveHandles[op->mReqPolicy];
 	op->release();
 
 	return true;
@@ -232,16 +280,16 @@ void HttpLibcurl::cancelRequest(HttpOpRequest * op)
 
 	// Detach from multi and recycle handle
 	curl_multi_remove_handle(mMultiHandles[op->mReqPolicy], op->mCurlHandle);
-	curl_easy_cleanup(op->mCurlHandle);
+	mHandleCache.freeHandle(op->mCurlHandle);
 	op->mCurlHandle = NULL;
 
 	// Tracing
 	if (op->mTracing > HTTP_TRACE_OFF)
 	{
-		LL_INFOS("CoreHttp") << "TRACE, RequestCanceled, Handle:  "
-							 << static_cast<HttpHandle>(op)
-							 << ", Status:  " << op->mStatus.toHex()
-							 << LL_ENDL;
+		LL_INFOS(LOG_CORE) << "TRACE, RequestCanceled, Handle:  "
+						   << static_cast<HttpHandle>(op)
+						   << ", Status:  " << op->mStatus.toTerseString()
+						   << LL_ENDL;
 	}
 
 	// Cancel op and deliver for notification
@@ -258,23 +306,24 @@ bool HttpLibcurl::completeRequest(CURLM * multi_handle, CURL * handle, CURLcode
 
 	if (handle != op->mCurlHandle || ! op->mCurlActive)
 	{
-		LL_WARNS("CoreHttp") << "libcurl handle and HttpOpRequest handle in disagreement or inactive request."
-							 << "  Handle:  " << static_cast<HttpHandle>(handle)
-							 << LL_ENDL;
+		LL_WARNS(LOG_CORE) << "libcurl handle and HttpOpRequest handle in disagreement or inactive request."
+						   << "  Handle:  " << static_cast<HttpHandle>(handle)
+						   << LL_ENDL;
 		return false;
 	}
 
 	active_set_t::iterator it(mActiveOps.find(op));
 	if (mActiveOps.end() == it)
 	{
-		LL_WARNS("CoreHttp") << "libcurl completion for request not on active list.  Continuing."
-							 << "  Handle:  " << static_cast<HttpHandle>(handle)
-							 << LL_ENDL;
+		LL_WARNS(LOG_CORE) << "libcurl completion for request not on active list.  Continuing."
+						   << "  Handle:  " << static_cast<HttpHandle>(handle)
+						   << LL_ENDL;
 		return false;
 	}
 
 	// Deactivate request
 	mActiveOps.erase(it);
+	--mActiveHandles[op->mReqPolicy];
 	op->mCurlActive = false;
 
 	// Set final status of request if it hasn't failed by other mechanisms yet
@@ -299,25 +348,25 @@ bool HttpLibcurl::completeRequest(CURLM * multi_handle, CURL * handle, CURLcode
 		}
 		else
 		{
-			LL_WARNS("CoreHttp") << "Invalid HTTP response code ("
-								 << http_status << ") received from server."
-								 << LL_ENDL;
+			LL_WARNS(LOG_CORE) << "Invalid HTTP response code ("
+							   << http_status << ") received from server."
+							   << LL_ENDL;
 			op->mStatus = HttpStatus(HttpStatus::LLCORE, HE_INVALID_HTTP_STATUS);
 		}
 	}
 
 	// Detach from multi and recycle handle
 	curl_multi_remove_handle(multi_handle, handle);
-	curl_easy_cleanup(handle);
+	mHandleCache.freeHandle(op->mCurlHandle);
 	op->mCurlHandle = NULL;
 
 	// Tracing
 	if (op->mTracing > HTTP_TRACE_OFF)
 	{
-		LL_INFOS("CoreHttp") << "TRACE, RequestComplete, Handle:  "
-							 << static_cast<HttpHandle>(op)
-							 << ", Status:  " << op->mStatus.toHex()
-							 << LL_ENDL;
+		LL_INFOS(LOG_CORE) << "TRACE, RequestComplete, Handle:  "
+						   << static_cast<HttpHandle>(op)
+						   << ", Status:  " << op->mStatus.toTerseString()
+						   << LL_ENDL;
 	}
 
 	// Dispatch to next stage
@@ -336,19 +385,167 @@ int HttpLibcurl::getActiveCount() const
 
 int HttpLibcurl::getActiveCountInClass(int policy_class) const
 {
-	int count(0);
+	llassert_always(policy_class < mPolicyCount);
+
+	return mActiveHandles ? mActiveHandles[policy_class] : 0;
+}
+
+void HttpLibcurl::policyUpdated(int policy_class)
+{
+	if (policy_class < 0 || policy_class >= mPolicyCount || ! mMultiHandles)
+	{
+		return;
+	}
 	
-	for (active_set_t::const_iterator iter(mActiveOps.begin());
-		 mActiveOps.end() != iter;
-		 ++iter)
+	HttpPolicy & policy(mService->getPolicy());
+	
+	if (! mActiveHandles[policy_class])
 	{
-		if ((*iter)->mReqPolicy == policy_class)
+		// Clear to set options.  As of libcurl 7.37.0, if a pipelining
+		// multi handle has active requests and you try to set the
+		// multi handle to non-pipelining, the library gets very angry
+		// and goes off the rails corrupting memory.  A clue that you're
+		// about to crash is that you'll get a missing server response
+		// error (curl code 9).  So, if options are to be set, we let
+		// the multi handle run out of requests, then set options, and
+		// re-enable request processing.
+		//
+		// All of this stall mechanism exists for this reason.  If
+		// libcurl becomes more resilient later, it should be possible
+		// to remove all of this.  The connection limit settings are fine,
+		// it's just that pipelined-to-non-pipelined transition that
+		// is fatal at the moment.
+		
+		HttpPolicyClass & options(policy.getClassOptions(policy_class));
+		CURLM * multi_handle(mMultiHandles[policy_class]);
+		CURLMcode code;
+
+		// Enable policy if stalled
+		policy.stallPolicy(policy_class, false);
+		mDirtyPolicy[policy_class] = false;
+		
+		if (options.mPipelining > 1)
 		{
-			++count;
+			// We'll try to do pipelining on this multihandle
+			code = curl_multi_setopt(multi_handle,
+									 CURLMOPT_PIPELINING,
+									 1L);
+			check_curl_multi_code(code, CURLMOPT_PIPELINING);
+			code = curl_multi_setopt(multi_handle,
+									 CURLMOPT_MAX_PIPELINE_LENGTH,
+									 long(options.mPipelining));
+			check_curl_multi_code(code, CURLMOPT_MAX_PIPELINE_LENGTH);
+			code = curl_multi_setopt(multi_handle,
+									 CURLMOPT_MAX_HOST_CONNECTIONS,
+									 long(options.mPerHostConnectionLimit));
+			check_curl_multi_code(code, CURLMOPT_MAX_HOST_CONNECTIONS);
+			code = curl_multi_setopt(multi_handle,
+									 CURLMOPT_MAX_TOTAL_CONNECTIONS,
+									 long(options.mConnectionLimit));
+			check_curl_multi_code(code, CURLMOPT_MAX_TOTAL_CONNECTIONS);
 		}
+		else
+		{
+			code = curl_multi_setopt(multi_handle,
+									 CURLMOPT_PIPELINING,
+									 0L);
+			check_curl_multi_code(code, CURLMOPT_PIPELINING);
+			code = curl_multi_setopt(multi_handle,
+									 CURLMOPT_MAX_HOST_CONNECTIONS,
+									 0L);
+			check_curl_multi_code(code, CURLMOPT_MAX_HOST_CONNECTIONS);
+			code = curl_multi_setopt(multi_handle,
+									 CURLMOPT_MAX_TOTAL_CONNECTIONS,
+									 long(options.mConnectionLimit));
+			check_curl_multi_code(code, CURLMOPT_MAX_TOTAL_CONNECTIONS);
+		}
+	}
+	else if (! mDirtyPolicy[policy_class])
+	{
+		// Mark policy dirty and request a stall in the policy.
+		// When policy goes idle, we'll re-invoke this method
+		// and perform the change.  Don't allow this thread to
+		// sleep while we're waiting for quiescence, we'll just
+		// stop processing.
+		mDirtyPolicy[policy_class] = true;
+		policy.stallPolicy(policy_class, true);
+	}
+}
+
+// ---------------------------------------
+// HttpLibcurl::HandleCache
+// ---------------------------------------
+
+HttpLibcurl::HandleCache::HandleCache()
+	: mHandleTemplate(NULL)
+{
+	mCache.reserve(50);
+}
+
+
+HttpLibcurl::HandleCache::~HandleCache()
+{
+	if (mHandleTemplate)
+	{
+		curl_easy_cleanup(mHandleTemplate);
+		mHandleTemplate = NULL;
+	}
+
+	for (handle_cache_t::iterator it(mCache.begin()); mCache.end() != it; ++it)
+	{
+		curl_easy_cleanup(*it);
 	}
+	mCache.clear();
+}
+
+
+CURL * HttpLibcurl::HandleCache::getHandle()
+{
+	CURL * ret(NULL);
 	
-	return count;
+	if (! mCache.empty())
+	{
+		// Fastest path to handle
+		ret = mCache.back();
+		mCache.pop_back();
+	}
+	else if (mHandleTemplate)
+	{
+		// Still fast path
+		ret = curl_easy_duphandle(mHandleTemplate);
+	}
+	else
+	{
+		// When all else fails
+		ret = curl_easy_init();
+	}
+
+	return ret;
+}
+
+
+void HttpLibcurl::HandleCache::freeHandle(CURL * handle)
+{
+	if (! handle)
+	{
+		return;
+	}
+
+	curl_easy_reset(handle);
+	if (! mHandleTemplate)
+	{
+		// Save the first freed handle as a template.
+		mHandleTemplate = handle;
+	}
+	else
+	{
+		// Otherwise add it to the cache
+		if (mCache.size() >= mCache.capacity())
+		{
+			mCache.reserve(mCache.capacity() + 50);
+		}
+		mCache.push_back(handle);
+	}
 }
 
 
@@ -359,15 +556,46 @@ int HttpLibcurl::getActiveCountInClass(int policy_class) const
 
 struct curl_slist * append_headers_to_slist(const HttpHeaders * headers, struct curl_slist * slist)
 {
-	for (HttpHeaders::container_t::const_iterator it(headers->mHeaders.begin());
-
-		headers->mHeaders.end() != it;
-		 ++it)
+	const HttpHeaders::const_iterator end(headers->end());
+	for (HttpHeaders::const_iterator it(headers->begin()); end != it; ++it)
 	{
-		slist = curl_slist_append(slist, (*it).c_str());
+		static const char sep[] = ": ";
+		std::string header;
+		header.reserve((*it).first.size() + (*it).second.size() + sizeof(sep));
+		header.append((*it).first);
+		header.append(sep);
+		header.append((*it).second);
+		
+		slist = curl_slist_append(slist, header.c_str());
 	}
 	return slist;
 }
 
 
 }  // end namespace LLCore
+
+
+namespace 
+{
+	
+void check_curl_multi_code(CURLMcode code, int curl_setopt_option)
+{
+	if (CURLM_OK != code)
+	{
+		LL_WARNS(LOG_CORE) << "libcurl multi error detected:  " << curl_multi_strerror(code)
+						   << ", curl_multi_setopt option:  " << curl_setopt_option
+						   << LL_ENDL;
+	}
+}
+
+
+void check_curl_multi_code(CURLMcode code)
+{
+	if (CURLM_OK != code)
+	{
+		LL_WARNS(LOG_CORE) << "libcurl multi error detected:  " << curl_multi_strerror(code)
+						   << LL_ENDL;
+	}
+}
+
+}  // end anonymous namespace