LLMutex recursive lock, global & per-request tracing, simple GET request, LLProxy support, HttpOptions starting to work, HTTP resource waiting fixed.

Non-LLThread-based threads need to do some registration or LLMutex locks taken out in these
threads will not work as expected (SH-3154).  We'll get a better solution later, this fixes
some things for now.  Tracing of operations now supported.  Global and per-request (via
HttpOptions) tracing levels of [0..3].  The 2 and 3 levels use libcurl's VERBOSE mode
combined with CURLOPT_DEBUGFUNCTION to stream high levels of detail into the log.  *Very*
laggy but useful.  Simple GET request supported (no Range: header).  Really just a
degenrate case of a ranged get but supplied an API anyway.  Global option to use the
LLProxy interface to setup CURL handles for either socks5 or http proxy usage.  This
isn't really the most encapsulated way to do this but a better solution will have to
come later.  The wantHeaders and tracing options are now supported in HttpOptions giving
per-request controls.  Big refactoring of the HTTP resource waiter in lltexturefetch.
What I was doing before wasn't correct.  Instead, I'm implementing the resource wait
after the Semaphore model (though not using system semaphores).  So instead of having
a sequence like:  SEND_HTTP_REQ -> WAIT_HTTP_RESOURCE -> SEND_HTTP_REQ, we now
do WAIT_HTTP_RESOURCE -> WAIT_HTTP_RESOURCE2 (actual wait) -> SEND_HTTP_REQ.  Works
well but the prioritized filling of the corehttp library needs some performance
work later.

16 files changed, 691 insertions, 107 deletions

diff --git a/indra/llcorehttp/_httplibcurl.cpp b/indra/llcorehttp/_httplibcurl.cpp
index e134a28401..a176dd5b2a 100644
--- a/indra/llcorehttp/_httplibcurl.cpp
+++ b/indra/llcorehttp/_httplibcurl.cpp
@@ -159,6 +159,17 @@ void HttpLibcurl::addOp(HttpOpRequest * op)
 	curl_multi_add_handle(mMultiHandles[op->mReqPolicy], op->mCurlHandle);
 	op->mCurlActive = true;
 	
+	if (op->mTracing > 0)
+	{
+		HttpPolicy & policy(mService->getPolicy());
+		
+		LL_INFOS("CoreHttp") << "TRACE, ToActiveQueue, Handle:  "
+							 << static_cast<HttpHandle>(op)
+							 << ", Actives:  " << mActiveOps.size()
+							 << ", Readies:  " << policy.getReadyCount(op->mReqPolicy)
+							 << LL_ENDL;
+	}
+	
 	// On success, make operation active
 	mActiveOps.insert(op);
 }
@@ -190,10 +201,9 @@ bool HttpLibcurl::completeRequest(CURLM * multi_handle, CURL * handle, CURLcode
 	// Deactivate request
 	op->mCurlActive = false;
 
-	// Set final status of request
+	// Set final status of request if it hasn't failed by other mechanisms yet
 	if (op->mStatus)
 	{
-		// Only set if it hasn't failed by other mechanisms yet
 		op->mStatus = HttpStatus(HttpStatus::EXT_CURL_EASY, status);
 	}
 	if (op->mStatus)
@@ -209,6 +219,16 @@ bool HttpLibcurl::completeRequest(CURLM * multi_handle, CURL * handle, CURLcode
 	curl_easy_cleanup(handle);
 	op->mCurlHandle = NULL;
 
+	// Tracing
+	if (op->mTracing > 0)
+	{
+		LL_INFOS("CoreHttp") << "TRACE, RequestComplete, Handle:  "
+							 << static_cast<HttpHandle>(op)
+							 << ", Status:  " << op->mStatus.toHex()
+							 << LL_ENDL;
+	}
+
+	// Dispatch to next stage
 	HttpPolicy & policy(mService->getPolicy());
 	bool still_active(policy.stageAfterCompletion(op));
 
diff --git a/indra/llcorehttp/_httpoperation.cpp b/indra/llcorehttp/_httpoperation.cpp
index b5c58013d4..5a31bf90e7 100644
--- a/indra/llcorehttp/_httpoperation.cpp
+++ b/indra/llcorehttp/_httpoperation.cpp
@@ -34,6 +34,8 @@
 #include "_httpreplyqueue.h"
 #include "_httpservice.h"
 
+#include "lltimer.h"
+
 
 namespace LLCore
 {
@@ -49,8 +51,10 @@ HttpOperation::HttpOperation()
 	  mReplyQueue(NULL),
 	  mUserHandler(NULL),
 	  mReqPolicy(HttpRequest::DEFAULT_POLICY_ID),
-	  mReqPriority(0U)
+	  mReqPriority(0U),
+	  mTracing(0)
 {
+	mMetricCreated = totalTime();
 }
 
 
@@ -113,7 +117,7 @@ void HttpOperation::stageFromActive(HttpService *)
 	llassert_always(false);
 }
 
-	
+
 void HttpOperation::visitNotifier(HttpRequest *)
 {
 	if (mUserHandler)
@@ -138,6 +142,13 @@ HttpStatus HttpOperation::cancel()
 
 void HttpOperation::addAsReply()
 {
+	if (mTracing > 0)
+	{
+		LL_INFOS("CoreHttp") << "TRACE, ToReplyQueue, Handle:  "
+							 << static_cast<HttpHandle>(this)
+							 << LL_ENDL;
+	}
+	
 	if (mReplyQueue)
 	{
 		addRef();
diff --git a/indra/llcorehttp/_httpoperation.h b/indra/llcorehttp/_httpoperation.h
index c93aa2def9..de4939a0ac 100644
--- a/indra/llcorehttp/_httpoperation.h
+++ b/indra/llcorehttp/_httpoperation.h
@@ -110,6 +110,10 @@ public:
 
 	// Reply Data
 	HttpStatus					mStatus;
+
+	// Tracing, debug and metrics
+	HttpTime					mMetricCreated;
+	int							mTracing;
 };  // end class HttpOperation
 
 
diff --git a/indra/llcorehttp/_httpoprequest.cpp b/indra/llcorehttp/_httpoprequest.cpp
index e2550d057e..f78971d8f2 100644
--- a/indra/llcorehttp/_httpoprequest.cpp
+++ b/indra/llcorehttp/_httpoprequest.cpp
@@ -63,6 +63,16 @@ int parse_content_range_header(char * buffer,
 							   unsigned int * last,
 							   unsigned int * length);
 
+
+// Take data from libcurl's CURLOPT_DEBUGFUNCTION callback and
+// escape and format it for a tracing line in logging.  Absolutely
+// anything including NULs can be in the data.  If @scrub is true,
+// non-printing or non-ascii characters are replaced with spaces
+// otherwise a %XX form of escaping is used.
+void escape_libcurl_debug_data(char * buffer, size_t len, bool scrub,
+							   std::string & safe_line);
+
+
 #if defined(WIN32)
 
 // Not available on windows where the legacy strtok interface
@@ -78,11 +88,6 @@ namespace LLCore
 {
 
 
-// ==================================
-// HttpOpRequest
-// ==================================
-
-
 HttpOpRequest::HttpOpRequest()
 	: HttpOperation(),
 	  mProcFlags(0U),
@@ -237,6 +242,19 @@ HttpStatus HttpOpRequest::cancel()
 }
 
 
+HttpStatus HttpOpRequest::setupGet(HttpRequest::policy_t policy_id,
+								   HttpRequest::priority_t priority,
+								   const std::string & url,
+								   HttpOptions * options,
+								   HttpHeaders * headers)
+{
+	setupCommon(policy_id, priority, url, NULL, options, headers);
+	mReqMethod = HOR_GET;
+	
+	return HttpStatus();
+}
+
+
 HttpStatus HttpOpRequest::setupGetByteRange(HttpRequest::policy_t policy_id,
 											HttpRequest::priority_t priority,
 											const std::string & url,
@@ -245,30 +263,16 @@ HttpStatus HttpOpRequest::setupGetByteRange(HttpRequest::policy_t policy_id,
 											HttpOptions * options,
 											HttpHeaders * headers)
 {
-	HttpStatus status;
-
-	mProcFlags = 0;
-	mReqPolicy = policy_id;
-	mReqPriority = priority;
+	setupCommon(policy_id, priority, url, NULL, options, headers);
 	mReqMethod = HOR_GET;
-	mReqURL = url;
 	mReqOffset = offset;
 	mReqLength = len;
 	if (offset || len)
 	{
 		mProcFlags |= PF_SCAN_RANGE_HEADER;
 	}
-	if (headers && ! mReqHeaders)
-	{
-		headers->addRef();
-		mReqHeaders = headers;
-	}
-	if (options && ! mReqOptions)
-	{
-		mReqOptions = new HttpOptions(*options);
-	}
 	
-	return status;
+	return HttpStatus();
 }
 
 
@@ -279,29 +283,10 @@ HttpStatus HttpOpRequest::setupPost(HttpRequest::policy_t policy_id,
 									HttpOptions * options,
 									HttpHeaders * headers)
 {
-	HttpStatus status;
-
-	mProcFlags = 0;
-	mReqPolicy = policy_id;
-	mReqPriority = priority;
+	setupCommon(policy_id, priority, url, body, options, headers);
 	mReqMethod = HOR_POST;
-	mReqURL = url;
-	if (body)
-	{
-		body->addRef();
-		mReqBody = body;
-	}
-	if (headers && ! mReqHeaders)
-	{
-		headers->addRef();
-		mReqHeaders = headers;
-	}
-	if (options && ! mReqOptions)
-	{
-		mReqOptions = new HttpOptions(*options);
-	}
 	
-	return status;
+	return HttpStatus();
 }
 
 
@@ -312,12 +297,23 @@ HttpStatus HttpOpRequest::setupPut(HttpRequest::policy_t policy_id,
 								   HttpOptions * options,
 								   HttpHeaders * headers)
 {
-	HttpStatus status;
+	setupCommon(policy_id, priority, url, body, options, headers);
+	mReqMethod = HOR_PUT;
+	
+	return HttpStatus();
+}
 
+
+void HttpOpRequest::setupCommon(HttpRequest::policy_t policy_id,
+								HttpRequest::priority_t priority,
+								const std::string & url,
+								BufferArray * body,
+								HttpOptions * options,
+								HttpHeaders * headers)
+{
 	mProcFlags = 0;
 	mReqPolicy = policy_id;
 	mReqPriority = priority;
-	mReqMethod = HOR_PUT;
 	mReqURL = url;
 	if (body)
 	{
@@ -331,10 +327,14 @@ HttpStatus HttpOpRequest::setupPut(HttpRequest::policy_t policy_id,
 	}
 	if (options && ! mReqOptions)
 	{
-		mReqOptions = new HttpOptions(*options);
+		options->addRef();
+		mReqOptions = options;
+		if (options->getWantHeaders())
+		{
+			mProcFlags |= PF_SAVE_HEADERS;
+		}
+		mTracing = (std::max)(mTracing, llclamp(options->getTrace(), 0, 3));
 	}
-	
-	return status;
 }
 
 
@@ -394,30 +394,29 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service)
 	curl_easy_setopt(mCurlHandle, CURLOPT_SSL_VERIFYHOST, 0);
 
 	const std::string * opt_value(NULL);
-	if (policy.get(HttpRequest::GP_CA_PATH, opt_value))
+	long opt_long(0L);
+	policy.get(HttpRequest::GP_LLPROXY, &opt_long);
+	if (opt_long)
 	{
-		curl_easy_setopt(mCurlHandle, CURLOPT_CAPATH, opt_value->c_str());
+		// Use the viewer-based thread-safe API which has a
+		// fast/safe check for proxy enable.  Would like to
+		// encapsulate this someway...
+		LLProxy::getInstance()->applyProxySettings(mCurlHandle);
 	}
-	if (policy.get(HttpRequest::GP_CA_FILE, opt_value))
+	else if (policy.get(HttpRequest::GP_HTTP_PROXY, &opt_value))
 	{
-		curl_easy_setopt(mCurlHandle, CURLOPT_CAINFO, opt_value->c_str());
+		// *TODO:  This is fine for now but get fuller socks/
+		// authentication thing going later....
+		curl_easy_setopt(mCurlHandle, CURLOPT_PROXY, opt_value->c_str());
+		curl_easy_setopt(mCurlHandle, CURLOPT_PROXYTYPE, CURLPROXY_HTTP);
 	}
-	if (policy.get(HttpRequest::GP_HTTP_PROXY, opt_value))
+	if (policy.get(HttpRequest::GP_CA_PATH, &opt_value))
 	{
-		if (*opt_value == "LLProxy")
-		{
-			// Use the viewer-based thread-safe API which has a
-			// fast/safe check for proxy enable.  Would like to
-			// encapsulate this someway...
-			LLProxy::getInstance()->applyProxySettings(mCurlHandle);
-		}
-		else
-		{
-			// *TODO:  This is fine for now but get fuller socks/
-			// authentication thing going later....
-			curl_easy_setopt(mCurlHandle, CURLOPT_PROXY, opt_value->c_str());
-			curl_easy_setopt(mCurlHandle, CURLOPT_PROXYTYPE, CURLPROXY_HTTP);
-		}
+		curl_easy_setopt(mCurlHandle, CURLOPT_CAPATH, opt_value->c_str());
+	}
+	if (policy.get(HttpRequest::GP_CA_FILE, &opt_value))
+	{
+		curl_easy_setopt(mCurlHandle, CURLOPT_CAINFO, opt_value->c_str());
 	}
 	
 	switch (mReqMethod)
@@ -463,6 +462,14 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service)
 		// *FIXME:  fail out here
 		break;
 	}
+
+	// Tracing
+	if (mTracing > 1)
+	{
+		curl_easy_setopt(mCurlHandle, CURLOPT_VERBOSE, 1);
+		curl_easy_setopt(mCurlHandle, CURLOPT_DEBUGDATA, mCurlHandle);
+		curl_easy_setopt(mCurlHandle, CURLOPT_DEBUGFUNCTION, debugCallback);
+	}
 	
 	// There's a CURLOPT for this now...
 	if ((mReqOffset || mReqLength) && HOR_GET == mReqMethod)
@@ -621,6 +628,101 @@ size_t HttpOpRequest::headerCallback(void * data, size_t size, size_t nmemb, voi
 	return hdr_size;
 }
 
+
+int HttpOpRequest::debugCallback(CURL * handle, curl_infotype info, char * buffer, size_t len, void * userdata)
+{
+	HttpOpRequest * op(NULL);
+	curl_easy_getinfo(handle, CURLINFO_PRIVATE, &op);
+	// *FIXME:  check the pointer
+
+	std::string safe_line;
+	std::string tag;
+	bool logit(false);
+	len = (std::min)(len, size_t(256));					// Keep things reasonable in all cases
+	
+	switch (info)
+	{
+	case CURLINFO_TEXT:
+		if (op->mTracing > 1)
+		{
+			tag = "TEXT";
+			escape_libcurl_debug_data(buffer, len, true, safe_line);
+			logit = true;
+		}
+		break;
+			
+	case CURLINFO_HEADER_IN:
+		if (op->mTracing > 1)
+		{
+			tag = "HEADERIN";
+			escape_libcurl_debug_data(buffer, len, true, safe_line);
+			logit = true;
+		}
+		break;
+			
+	case CURLINFO_HEADER_OUT:
+		if (op->mTracing > 1)
+		{
+			tag = "HEADEROUT";
+			escape_libcurl_debug_data(buffer, len, true, safe_line);
+			logit = true;
+		}
+		break;
+			
+	case CURLINFO_DATA_IN:
+		if (op->mTracing > 1)
+		{
+			tag = "DATAIN";
+			logit = true;
+			if (op->mTracing > 2)
+			{
+				escape_libcurl_debug_data(buffer, len, false, safe_line);
+			}
+			else
+			{
+				std::ostringstream out;
+				out << len << " Bytes";
+				safe_line = out.str();
+			}
+		}
+		break;
+			
+	case CURLINFO_DATA_OUT:
+		if (op->mTracing > 1)
+		{
+			tag = "DATAOUT";
+			logit = true;
+			if (op->mTracing > 2)
+			{
+				escape_libcurl_debug_data(buffer, len, false, safe_line);
+			}
+			else
+			{
+				std::ostringstream out;
+				out << len << " Bytes";
+				safe_line = out.str();
+			}
+		}
+		break;
+			
+	default:
+		logit = false;
+		break;
+	}
+
+	if (logit)
+	{
+		LL_INFOS("CoreHttp") << "TRACE, LibcurlDebug, Handle:  "
+							 << static_cast<HttpHandle>(op)
+							 << ", Type:  " << tag
+							 << ", Data:  " << safe_line
+							 << LL_ENDL;
+	}
+		
+	return 0;
+}
+
+
 }   // end namespace LLCore
 
 
@@ -694,6 +796,43 @@ char *strtok_r(char *str, const char *delim, char ** savestate)
 
 #endif
 
+
+void escape_libcurl_debug_data(char * buffer, size_t len, bool scrub, std::string & safe_line)
+{
+	std::string out;
+	len = (std::min)(len, size_t(200));
+	out.reserve(3 * len);
+	for (int i(0); i < len; ++i)
+	{
+		unsigned char uc(static_cast<unsigned char>(buffer[i]));
+
+		if (uc < 32 || uc > 126)
+		{
+			if (scrub)
+			{
+				out.append(1, ' ');
+			}
+			else
+			{
+				static const char hex[] = "0123456789ABCDEF";
+				char convert[4];
+
+				convert[0] = '%';
+				convert[1] = hex[(uc >> 4) % 16];
+				convert[2] = hex[uc % 16];
+				convert[3] = '\0';
+				out.append(convert);
+			}
+		}
+		else
+		{
+			out.append(1, buffer[i]);
+		}
+	}
+	safe_line.swap(out);
+}
+
+
 }  // end anonymous namespace
 
 		
diff --git a/indra/llcorehttp/_httpoprequest.h b/indra/llcorehttp/_httpoprequest.h
index 80893beb40..fc2301057c 100644
--- a/indra/llcorehttp/_httpoprequest.h
+++ b/indra/llcorehttp/_httpoprequest.h
@@ -76,6 +76,12 @@ public:
 			
 public:
 	// Setup Methods
+	HttpStatus setupGet(HttpRequest::policy_t policy_id,
+						HttpRequest::priority_t priority,
+						const std::string & url,
+						HttpOptions * options,
+						HttpHeaders * headers);
+	
 	HttpStatus setupGetByteRange(HttpRequest::policy_t policy_id,
 								 HttpRequest::priority_t priority,
 								 const std::string & url,
@@ -103,15 +109,23 @@ public:
 	virtual HttpStatus cancel();
 
 protected:
+	void setupCommon(HttpRequest::policy_t policy_id,
+					 HttpRequest::priority_t priority,
+					 const std::string & url,
+					 BufferArray * body,
+					 HttpOptions * options,
+					 HttpHeaders * headers);
+	
 	static size_t writeCallback(void * data, size_t size, size_t nmemb, void * userdata);
 	static size_t readCallback(void * data, size_t size, size_t nmemb, void * userdata);
 	static size_t headerCallback(void * data, size_t size, size_t nmemb, void * userdata);
+	static int debugCallback(CURL *, curl_infotype info, char * buffer, size_t len, void * userdata);
 
 protected:
 	unsigned int		mProcFlags;
 	static const unsigned int	PF_SCAN_RANGE_HEADER = 0x00000001U;
 	static const unsigned int	PF_SAVE_HEADERS = 0x00000002U;
-	
+
 public:
 	// Request data
 	EMethod				mReqMethod;
diff --git a/indra/llcorehttp/_httpopsetget.cpp b/indra/llcorehttp/_httpopsetget.cpp
index 21e058b2be..c1357f9ae5 100644
--- a/indra/llcorehttp/_httpopsetget.cpp
+++ b/indra/llcorehttp/_httpopsetget.cpp
@@ -89,8 +89,8 @@ void HttpOpSetGet::stageFromRequest(HttpService * service)
 	}
 	if (mStatus)
 	{
-		const std::string * value;
-		if ((mStatus = pol_opt.get(setting, value)))
+		const std::string * value(NULL);
+		if ((mStatus = pol_opt.get(setting, &value)))
 		{
 			mStrValue = *value;
 		}
diff --git a/indra/llcorehttp/_httppolicy.cpp b/indra/llcorehttp/_httppolicy.cpp
index 8ee3f88658..0e08d88276 100644
--- a/indra/llcorehttp/_httppolicy.cpp
+++ b/indra/llcorehttp/_httppolicy.cpp
@@ -107,6 +107,12 @@ void HttpPolicy::retryOp(HttpOpRequest * op)
 	LL_WARNS("CoreHttp") << "URL op retry #" << op->mPolicyRetries
 						 << " being scheduled for " << delta << " uSecs from now."
 						 << LL_ENDL;
+	if (op->mTracing > 0)
+	{
+		LL_INFOS("CoreHttp") << "TRACE, ToRetryQueue, Handle:  "
+							 << static_cast<HttpHandle>(op)
+							 << LL_ENDL;
+	}
 	mState[policy_class].mRetryQueue.push(op);
 }
 
@@ -224,8 +230,8 @@ bool HttpPolicy::stageAfterCompletion(HttpOpRequest * op)
 	}
 	else if (op->mPolicyRetries)
 	{
-		LL_WARNS("CoreHttp") << "URL op succeeded after " << op->mPolicyRetries << " retries."
-							 << LL_ENDL;
+		LL_DEBUGS("CoreHttp") << "URL op succeeded after " << op->mPolicyRetries << " retries."
+							  << LL_ENDL;
 	}
 
 	op->stageFromActive(mService);
@@ -234,4 +240,15 @@ bool HttpPolicy::stageAfterCompletion(HttpOpRequest * op)
 }
 
 
+int HttpPolicy::getReadyCount(HttpRequest::policy_t policy_class)
+{
+	if (policy_class < HttpRequest::POLICY_CLASS_LIMIT)
+	{
+		return (mState[policy_class].mReadyQueue.size()
+				+ mState[policy_class].mRetryQueue.size());
+	}
+	return 0;
+}
+
+
 }  // end namespace LLCore
diff --git a/indra/llcorehttp/_httppolicy.h b/indra/llcorehttp/_httppolicy.h
index 73c22bab78..4114f64848 100644
--- a/indra/llcorehttp/_httppolicy.h
+++ b/indra/llcorehttp/_httppolicy.h
@@ -97,7 +97,10 @@ public:
 		}
 
 	void setPolicies(const HttpPolicyGlobal & global);
-			
+
+	// Get ready counts for a particular class
+	int getReadyCount(HttpRequest::policy_t policy_class);
+	
 protected:
 	struct State
 	{
diff --git a/indra/llcorehttp/_httppolicyglobal.cpp b/indra/llcorehttp/_httppolicyglobal.cpp
index d95d73cfba..6b1de38fd6 100644
--- a/indra/llcorehttp/_httppolicyglobal.cpp
+++ b/indra/llcorehttp/_httppolicyglobal.cpp
@@ -33,7 +33,9 @@ namespace LLCore
 
 HttpPolicyGlobal::HttpPolicyGlobal()
 	: mSetMask(0UL),
-	  mConnectionLimit(32L)
+	  mConnectionLimit(32L),
+	  mTrace(0),
+	  mUseLLProxy(0)
 {}
 
 
@@ -50,6 +52,8 @@ HttpPolicyGlobal & HttpPolicyGlobal::operator=(const HttpPolicyGlobal & other)
 		mCAPath = other.mCAPath;
 		mCAFile = other.mCAFile;
 		mHttpProxy = other.mHttpProxy;
+		mTrace = other.mTrace;
+		mUseLLProxy = other.mUseLLProxy;
 	}
 	return *this;
 }
@@ -63,6 +67,14 @@ HttpStatus HttpPolicyGlobal::set(HttpRequest::EGlobalPolicy opt, long value)
 		mConnectionLimit = value;
 		break;
 
+	case HttpRequest::GP_TRACE:
+		mTrace = llclamp(value, 0L, 3L);
+		break;
+
+	case HttpRequest::GP_LLPROXY:
+		mUseLLProxy = llclamp(value, 0L, 1L);
+		break;
+
 	default:
 		return HttpStatus(HttpStatus::LLCORE, HE_INVALID_ARG);
 	}
@@ -97,54 +109,64 @@ HttpStatus HttpPolicyGlobal::set(HttpRequest::EGlobalPolicy opt, const std::stri
 }
 
 
-HttpStatus HttpPolicyGlobal::get(HttpRequest::EGlobalPolicy opt, long & value)
+HttpStatus HttpPolicyGlobal::get(HttpRequest::EGlobalPolicy opt, long * value)
 {
 	static const HttpStatus not_set(HttpStatus::LLCORE, HE_OPT_NOT_SET);
+	long * src(NULL);
 	
 	switch (opt)
 	{
 	case HttpRequest::GP_CONNECTION_LIMIT:
-		if (! (mSetMask & (1UL << int(opt))))
-			return not_set;
-		value = mConnectionLimit;
+		src = &mConnectionLimit;
+		break;
+
+	case HttpRequest::GP_TRACE:
+		src = &mTrace;
+		break;
+
+	case HttpRequest::GP_LLPROXY:
+		src = &mUseLLProxy;
 		break;
 
 	default:
 		return HttpStatus(HttpStatus::LLCORE, HE_INVALID_ARG);
 	}
 
+	if (! (mSetMask & (1UL << int(opt))))
+		return not_set;
+
+	*value = *src;
 	return HttpStatus();
 }
 
 
-HttpStatus HttpPolicyGlobal::get(HttpRequest::EGlobalPolicy opt, const std::string *& value)
+HttpStatus HttpPolicyGlobal::get(HttpRequest::EGlobalPolicy opt, const std::string ** value)
 {
 	static const HttpStatus not_set(HttpStatus::LLCORE, HE_OPT_NOT_SET);
+	const std::string * src(NULL);
 
 	switch (opt)
 	{
 	case HttpRequest::GP_CA_PATH:
-		if (! (mSetMask & (1UL << int(opt))))
-			return not_set;
-		value = &mCAPath;
+		src = &mCAPath;
 		break;
 
 	case HttpRequest::GP_CA_FILE:
-		if (! (mSetMask & (1UL << int(opt))))
-			return not_set;
-		value = &mCAFile;
+		src = &mCAFile;
 		break;
 
 	case HttpRequest::GP_HTTP_PROXY:
-		if (! (mSetMask & (1UL << int(opt))))
-			return not_set;
-		value = &mHttpProxy;
+		src = &mHttpProxy;
 		break;
 
 	default:
 		return HttpStatus(HttpStatus::LLCORE, HE_INVALID_ARG);
 	}
 	
+	if (! (mSetMask & (1UL << int(opt))))
+		return not_set;
+
+	*value = src;
 	return HttpStatus();
 }
 
diff --git a/indra/llcorehttp/_httppolicyglobal.h b/indra/llcorehttp/_httppolicyglobal.h
index f4bb4d4b25..a50d0e4188 100644
--- a/indra/llcorehttp/_httppolicyglobal.h
+++ b/indra/llcorehttp/_httppolicyglobal.h
@@ -48,8 +48,8 @@ private:
 public:
 	HttpStatus set(HttpRequest::EGlobalPolicy opt, long value);
 	HttpStatus set(HttpRequest::EGlobalPolicy opt, const std::string & value);
-	HttpStatus get(HttpRequest::EGlobalPolicy opt, long & value);
-	HttpStatus get(HttpRequest::EGlobalPolicy opt, const std::string *& value);
+	HttpStatus get(HttpRequest::EGlobalPolicy opt, long * value);
+	HttpStatus get(HttpRequest::EGlobalPolicy opt, const std::string ** value);
 	
 public:
 	unsigned long		mSetMask;
@@ -57,6 +57,8 @@ public:
 	std::string			mCAPath;
 	std::string			mCAFile;
 	std::string			mHttpProxy;
+	long				mTrace;
+	long				mUseLLProxy;
 };  // end class HttpPolicyGlobal
 
 }  // end namespace LLCore
diff --git a/indra/llcorehttp/_httpservice.cpp b/indra/llcorehttp/_httpservice.cpp
index 920a3f3b6d..beba8f08f4 100644
--- a/indra/llcorehttp/_httpservice.cpp
+++ b/indra/llcorehttp/_httpservice.cpp
@@ -36,6 +36,7 @@
 #include "_thread.h"
 
 #include "lltimer.h"
+#include "llthread.h"
 
 
 // Tuning parameters
@@ -186,8 +187,10 @@ void HttpService::shutdown()
 void HttpService::threadRun(LLCoreInt::HttpThread * thread)
 {
 	boost::this_thread::disable_interruption di;
-	ELoopSpeed loop(REQUEST_SLEEP);
+
+	LLThread::registerThreadID();
 	
+	ELoopSpeed loop(REQUEST_SLEEP);
 	while (! mExitRequested)
 	{
 		loop = processRequestQueue(loop);
@@ -226,6 +229,19 @@ HttpService::ELoopSpeed HttpService::processRequestQueue(ELoopSpeed loop)
 		// Process operation
 		if (! mExitRequested)
 		{
+			// Setup for subsequent tracing
+			long tracing(0);
+			mPolicy->getGlobalOptions().get(HttpRequest::GP_TRACE, &tracing);
+			op->mTracing = (std::max)(op->mTracing, int(tracing));
+
+			if (op->mTracing > 0)
+			{
+				LL_INFOS("CoreHttp") << "TRACE, FromRequestQueue, Handle:  "
+									 << static_cast<HttpHandle>(op)
+									 << LL_ENDL;
+			}
+
+			// Stage
 			op->stageFromRequest(this);
 		}
 				
diff --git a/indra/llcorehttp/httpoptions.cpp b/indra/llcorehttp/httpoptions.cpp
index 15b505f5bf..155fbda7f1 100644
--- a/indra/llcorehttp/httpoptions.cpp
+++ b/indra/llcorehttp/httpoptions.cpp
@@ -33,13 +33,15 @@ namespace LLCore
 
 HttpOptions::HttpOptions()
 	: RefCounted(true),
-	  mWantHeaders(false)
+	  mWantHeaders(false),
+	  mTracing(0)
 {}
 
 
 HttpOptions::HttpOptions(const HttpOptions & rhs)
 	: RefCounted(true),
-	  mWantHeaders(rhs.mWantHeaders)
+	  mWantHeaders(rhs.mWantHeaders),
+	  mTracing(rhs.mTracing)
 {}
 
 
@@ -47,4 +49,16 @@ HttpOptions::~HttpOptions()
 {}
 
 
+void HttpOptions::setWantHeaders()
+{
+	mWantHeaders = true;
+}
+
+
+void HttpOptions::setTrace(long level)
+{
+	mTracing = int(level);
+}
+
+
 }   // end namespace LLCore
diff --git a/indra/llcorehttp/httpoptions.h b/indra/llcorehttp/httpoptions.h
index 267a982dd5..0b9dfdc1de 100644
--- a/indra/llcorehttp/httpoptions.h
+++ b/indra/llcorehttp/httpoptions.h
@@ -67,11 +67,21 @@ protected:
 	void operator=(const HttpOptions &);		// Not defined
 
 public:
+	void		setWantHeaders();
+	bool		getWantHeaders() const
+		{
+			return mWantHeaders;
+		}
+	
+	void		setTrace(long level);
+	int			getTrace() const
+		{
+			return mTracing;
+		}
 	
 protected:
-	// *TODO:  add some options
 	bool		mWantHeaders;
-	
+	long		int mTracing;
 }; // end class HttpOptions
 
 
diff --git a/indra/llcorehttp/httprequest.cpp b/indra/llcorehttp/httprequest.cpp
index 089eee76f3..2036ecfd1c 100644
--- a/indra/llcorehttp/httprequest.cpp
+++ b/indra/llcorehttp/httprequest.cpp
@@ -135,6 +135,33 @@ HttpStatus HttpRequest::getStatus() const
 }
 
 
+HttpHandle HttpRequest::requestGet(policy_t policy_id,
+								   priority_t priority,
+								   const std::string & url,
+								   HttpOptions * options,
+								   HttpHeaders * headers,
+								   HttpHandler * user_handler)
+{
+	HttpStatus status;
+	HttpHandle handle(LLCORE_HTTP_HANDLE_INVALID);
+
+	HttpOpRequest * op = new HttpOpRequest();
+	if (! (status = op->setupGet(policy_id, priority, url, options, headers)))
+	{
+		op->release();
+		mLastReqStatus = status;
+		return handle;
+	}
+	op->setReplyPath(mReplyQueue, user_handler);
+	mRequestQueue->addOp(op);			// transfers refcount
+	
+	mLastReqStatus = status;
+	handle = static_cast<HttpHandle>(op);
+	
+	return handle;
+}
+
+
 HttpHandle HttpRequest::requestGetByteRange(policy_t policy_id,
 											priority_t priority,
 											const std::string & url,
diff --git a/indra/llcorehttp/httprequest.h b/indra/llcorehttp/httprequest.h
index a953aa28d0..4e78ed3719 100644
--- a/indra/llcorehttp/httprequest.h
+++ b/indra/llcorehttp/httprequest.h
@@ -111,10 +111,43 @@ public:
 		/// Maximum number of connections the library will use to
 		/// perform operations.  This is somewhat soft as the underlying
 		/// transport will cache some connections (up to 5).
-		GP_CONNECTION_LIMIT,		///< Takes long giving number of connections
-		GP_CA_PATH,					///< System path/directory where SSL certs are stored.
-		GP_CA_FILE,					///< System path/file containing certs.
-		GP_HTTP_PROXY				///< String giving host/port to use for HTTP proxy
+
+		/// A long value setting the maximum number of connections
+		/// allowed over all policy classes.  Note that this will be
+		/// a somewhat soft value.  There may be an additional five
+		/// connections per policy class depending upon runtime
+		/// behavior.
+		GP_CONNECTION_LIMIT,
+
+		/// String containing a system-appropriate directory name
+		/// where SSL certs are stored.
+		GP_CA_PATH,
+
+		/// String giving a full path to a file containing SSL certs.
+		GP_CA_FILE,
+
+		/// String of host/port to use as simple HTTP proxy.  This is
+		/// going to change in the future into something more elaborate
+		/// that may support richer schemes.
+		GP_HTTP_PROXY,
+
+		/// Long value that if non-zero enables the use of the
+		/// traditional LLProxy code for http/socks5 support.  If
+		/// enabled, has priority over GP_HTTP_PROXY.
+		GP_LLPROXY,
+
+		/// Long value setting the logging trace level for the
+		/// library.  Possible values are:
+		/// 0 - No tracing (default)
+		/// 1 - Basic tracing of request start, stop and major events.
+		/// 2 - Connection, header and payload size information from
+		///     HTTP transactions.
+		/// 3 - Partial logging of payload itself.
+		///
+		/// These values are also used in the trace modes for
+		/// individual requests in HttpOptions.  Also be aware that
+		/// tracing tends to impact performance of the viewer.
+		GP_TRACE
 	};
 
 	/// Set a parameter on a global policy option.  Calls
@@ -133,7 +166,7 @@ public:
 	///					the class in other methods.  If -1, an error
 	///					occurred and @see getStatus() may provide more
 	///					detail on the reason.
-	policy_t createPolicyClass();
+	static policy_t createPolicyClass();
 
 	enum EClassPolicy
 	{
@@ -157,7 +190,7 @@ public:
 	/// @param opt				Enum of option to be set.
 	/// @param value			Desired value of option.
 	/// @return					Standard status code.
-	HttpStatus setPolicyClassOption(policy_t policy_id, EClassPolicy opt, long value);
+	static HttpStatus setPolicyClassOption(policy_t policy_id, EClassPolicy opt, long value);
 
 	/// @}
 
@@ -176,6 +209,36 @@ public:
 	///
 	HttpStatus getStatus() const;
 
+	/// Queue a full HTTP GET request to be issued for entire entity.
+	/// The request is queued and serviced by the working thread and
+	/// notification of completion delivered to the optional HttpHandler
+	/// argument during @see update() calls.
+	///
+	/// With a valid handle returned, it can be used to reference the
+	/// request in other requests (like cancellation) and will be an
+	/// argument when any HttpHandler object is invoked.
+	///
+	/// @param	policy_id		Default or user-defined policy class under
+	///							which this request is to be serviced.
+	/// @param	priority		Standard priority scheme inherited from
+	///							Indra code base (U32-type scheme).
+	/// @param	url
+	/// @param	options			(optional)
+	/// @param	headers			(optional)
+	/// @param	handler			(optional)
+	/// @return					The handle of the request if successfully
+	///							queued or LLCORE_HTTP_HANDLE_INVALID if the
+	///							request could not be queued.  In the latter
+	///							case, @see getStatus() will return more info.
+	///
+	HttpHandle requestGet(policy_t policy_id,
+						  priority_t priority,
+						  const std::string & url,
+						  HttpOptions * options,
+						  HttpHeaders * headers,
+						  HttpHandler * handler);
+
+
 	/// Queue a full HTTP GET request to be issued with a 'Range' header.
 	/// The request is queued and serviced by the working thread and
 	/// notification of completion delivered to the optional HttpHandler
diff --git a/indra/llcorehttp/tests/test_httprequest.hpp b/indra/llcorehttp/tests/test_httprequest.hpp
index 61698f34d8..5b04796c8a 100644
--- a/indra/llcorehttp/tests/test_httprequest.hpp
+++ b/indra/llcorehttp/tests/test_httprequest.hpp
@@ -395,7 +395,7 @@ void HttpRequestTestObjectType::test<5>()
 {
 	ScopedCurlInit ready;
 
-	set_test_name("HttpRequest GET + Stop execution");
+	set_test_name("HttpRequest GET to dead port + Stop execution");
 
 	// Handler can be stack-allocated *if* there are no dangling
 	// references to it after completion of this method.
@@ -496,6 +496,7 @@ void HttpRequestTestObjectType::test<5>()
 	}
 }
 
+
 template <> template <>
 void HttpRequestTestObjectType::test<6>()
 {
@@ -532,6 +533,114 @@ void HttpRequestTestObjectType::test<6>()
 
 		// Issue a GET that *can* connect
 		mStatus = HttpStatus(200);
+		HttpHandle handle = req->requestGet(HttpRequest::DEFAULT_POLICY_ID,
+											0U,
+											url_base,
+											NULL,
+											NULL,
+											&handler);
+		ensure("Valid handle returned for ranged request", handle != LLCORE_HTTP_HANDLE_INVALID);
+
+		// Run the notification pump.
+		int count(0);
+		int limit(10);
+		while (count++ < limit && mHandlerCalls < 1)
+		{
+			req->update(1000);
+			usleep(100000);
+		}
+		ensure("Request executed in reasonable time", count < limit);
+		ensure("One handler invocation for request", mHandlerCalls == 1);
+
+		// Okay, request a shutdown of the servicing thread
+		mStatus = HttpStatus();
+		handle = req->requestStopThread(&handler);
+		ensure("Valid handle returned for second request", handle != LLCORE_HTTP_HANDLE_INVALID);
+	
+		// Run the notification pump again
+		count = 0;
+		limit = 10;
+		while (count++ < limit && mHandlerCalls < 2)
+		{
+			req->update(1000);
+			usleep(100000);
+		}
+		ensure("Second request executed in reasonable time", count < limit);
+		ensure("Second handler invocation", mHandlerCalls == 2);
+
+		// See that we actually shutdown the thread
+		count = 0;
+		limit = 10;
+		while (count++ < limit && ! HttpService::isStopped())
+		{
+			usleep(100000);
+		}
+		ensure("Thread actually stopped running", HttpService::isStopped());
+	
+		// release the request object
+		delete req;
+		req = NULL;
+
+		// Shut down service
+		HttpRequest::destroyService();
+	
+		ensure("Two handler calls on the way out", 2 == mHandlerCalls);
+
+#if defined(WIN32)
+		// Can only do this memory test on Windows.  On other platforms,
+		// the LL logging system holds on to memory and produces what looks
+		// like memory leaks...
+	
+		// printf("Old mem:  %d, New mem:  %d\n", mMemTotal, GetMemTotal());
+		ensure("Memory usage back to that at entry", mMemTotal == GetMemTotal());
+#endif
+	}
+	catch (...)
+	{
+		stop_thread(req);
+		delete req;
+		HttpRequest::destroyService();
+		throw;
+	}
+}
+
+
+template <> template <>
+void HttpRequestTestObjectType::test<7>()
+{
+	ScopedCurlInit ready;
+
+	std::string url_base(get_base_url());
+	std::cerr << "Base:  "  << url_base << std::endl;
+	
+	set_test_name("HttpRequest GET with Range: header to real service");
+
+	// Handler can be stack-allocated *if* there are no dangling
+	// references to it after completion of this method.
+	// Create before memory record as the string copy will bump numbers.
+	TestHandler2 handler(this, "handler");
+		
+	// record the total amount of dynamically allocated memory
+	mMemTotal = GetMemTotal();
+	mHandlerCalls = 0;
+
+	HttpRequest * req = NULL;
+
+	try
+	{
+		// Get singletons created
+		HttpRequest::createService();
+		
+		// Start threading early so that thread memory is invariant
+		// over the test.
+		HttpRequest::startThread();
+
+		// create a new ref counted object with an implicit reference
+		req = new HttpRequest();
+		ensure("Memory allocated on construction", mMemTotal < GetMemTotal());
+
+		// Issue a GET that *can* connect
+		mStatus = HttpStatus(200);
 		HttpHandle handle = req->requestGetByteRange(HttpRequest::DEFAULT_POLICY_ID,
 													 0U,
 													 url_base,
@@ -605,8 +714,9 @@ void HttpRequestTestObjectType::test<6>()
 	}
 }
 
+
 template <> template <>
-void HttpRequestTestObjectType::test<7>()
+void HttpRequestTestObjectType::test<8>()
 {
 	ScopedCurlInit ready;
 
@@ -725,7 +835,7 @@ void HttpRequestTestObjectType::test<7>()
 }
 
 template <> template <>
-void HttpRequestTestObjectType::test<8>()
+void HttpRequestTestObjectType::test<9>()
 {
 	ScopedCurlInit ready;
 
@@ -843,6 +953,118 @@ void HttpRequestTestObjectType::test<8>()
 	}
 }
 
+template <> template <>
+void HttpRequestTestObjectType::test<10>()
+{
+	ScopedCurlInit ready;
+
+	std::string url_base(get_base_url());
+	std::cerr << "Base:  "  << url_base << std::endl;
+	
+	set_test_name("HttpRequest GET with some tracing");
+
+	// Handler can be stack-allocated *if* there are no dangling
+	// references to it after completion of this method.
+	// Create before memory record as the string copy will bump numbers.
+	TestHandler2 handler(this, "handler");
+		
+	// record the total amount of dynamically allocated memory
+	mMemTotal = GetMemTotal();
+	mHandlerCalls = 0;
+
+	HttpRequest * req = NULL;
+
+	try
+	{
+		// Get singletons created
+		HttpRequest::createService();
+
+		// Enable tracing
+		HttpRequest::setPolicyGlobalOption(LLCore::HttpRequest::GP_TRACE, 2);
+
+		// Start threading early so that thread memory is invariant
+		// over the test.
+		HttpRequest::startThread();
+
+		// create a new ref counted object with an implicit reference
+		req = new HttpRequest();
+		ensure("Memory allocated on construction", mMemTotal < GetMemTotal());
+
+		// Issue a GET that *can* connect
+		mStatus = HttpStatus(200);
+		HttpHandle handle = req->requestGetByteRange(HttpRequest::DEFAULT_POLICY_ID,
+													 0U,
+													 url_base,
+													 0,
+													 0,
+													 NULL,
+													 NULL,
+													 &handler);
+		ensure("Valid handle returned for ranged request", handle != LLCORE_HTTP_HANDLE_INVALID);
+
+		// Run the notification pump.
+		int count(0);
+		int limit(10);
+		while (count++ < limit && mHandlerCalls < 1)
+		{
+			req->update(1000);
+			usleep(100000);
+		}
+		ensure("Request executed in reasonable time", count < limit);
+		ensure("One handler invocation for request", mHandlerCalls == 1);
+
+		// Okay, request a shutdown of the servicing thread
+		mStatus = HttpStatus();
+		handle = req->requestStopThread(&handler);
+		ensure("Valid handle returned for second request", handle != LLCORE_HTTP_HANDLE_INVALID);
+	
+		// Run the notification pump again
+		count = 0;
+		limit = 10;
+		while (count++ < limit && mHandlerCalls < 2)
+		{
+			req->update(1000);
+			usleep(100000);
+		}
+		ensure("Second request executed in reasonable time", count < limit);
+		ensure("Second handler invocation", mHandlerCalls == 2);
+
+		// See that we actually shutdown the thread
+		count = 0;
+		limit = 10;
+		while (count++ < limit && ! HttpService::isStopped())
+		{
+			usleep(100000);
+		}
+		ensure("Thread actually stopped running", HttpService::isStopped());
+	
+		// release the request object
+		delete req;
+		req = NULL;
+
+		// Shut down service
+		HttpRequest::destroyService();
+	
+		ensure("Two handler calls on the way out", 2 == mHandlerCalls);
+
+#if defined(WIN32)
+		// Can only do this memory test on Windows.  On other platforms,
+		// the LL logging system holds on to memory and produces what looks
+		// like memory leaks...
+	
+		// printf("Old mem:  %d, New mem:  %d\n", mMemTotal, GetMemTotal());
+		ensure("Memory usage back to that at entry", mMemTotal == GetMemTotal());
+#endif
+	}
+	catch (...)
+	{
+		stop_thread(req);
+		delete req;
+		HttpRequest::destroyService();
+		throw;
+	}
+}
+
 }  // end namespace tut
 
 namespace