Policy + caching fixes + https support + POST working

Implemented first global policy definitions to support SSL CA certificate configuration
to support https: operations.  Fixed HTTP 206 status handling to match what is currently
being done by grid services and to lay a foundation for fixes that will be a response
to ER-1824.  More libcurl CURLOPT options set on easy handles to do peer verification
in the traditional way.  HTTP POST working and now reporting asset metrics back to
grid for the viewer's asset system.  This uses LLSD so that is also showing as compatible
with the new library.

12 files changed, 126 insertions, 38 deletions

diff --git a/indra/llcorehttp/CMakeLists.txt b/indra/llcorehttp/CMakeLists.txt
index 9a073eb850..3fda524ddf 100644
--- a/indra/llcorehttp/CMakeLists.txt
+++ b/indra/llcorehttp/CMakeLists.txt
@@ -33,6 +33,7 @@ set(llcorehttp_SOURCE_FILES
     _httpoprequest.cpp
     _httpopsetpriority.cpp
     _httppolicy.cpp
+    _httppolicyglobal.cpp
     _httpreplyqueue.cpp
     _httprequestqueue.cpp
     _httpservice.cpp
@@ -55,6 +56,7 @@ set(llcorehttp_HEADER_FILES
     _httpoprequest.h
     _httpopsetpriority.h
     _httppolicy.h
+    _httppolicyglobal.h
     _httpreadyqueue.h
     _httpreplyqueue.h
     _httprequestqueue.h
diff --git a/indra/llcorehttp/_httplibcurl.cpp b/indra/llcorehttp/_httplibcurl.cpp
index 704f9baac9..5272c391e8 100644
--- a/indra/llcorehttp/_httplibcurl.cpp
+++ b/indra/llcorehttp/_httplibcurl.cpp
@@ -27,6 +27,7 @@
 #include "_httplibcurl.h"
 
 #include "httpheaders.h"
+#include "bufferarray.h"
 #include "_httpoprequest.h"
 
 
diff --git a/indra/llcorehttp/_httpoprequest.cpp b/indra/llcorehttp/_httpoprequest.cpp
index f52ff5a44c..4bdc4a5257 100644
--- a/indra/llcorehttp/_httpoprequest.cpp
+++ b/indra/llcorehttp/_httpoprequest.cpp
@@ -40,8 +40,10 @@
 #include "_httpreplyqueue.h"
 #include "_httpservice.h"
 #include "_httppolicy.h"
+#include "_httppolicyglobal.h"
 #include "_httplibcurl.h"
 
+#include "llhttpstatuscodes.h"
 
 namespace
 {
@@ -153,14 +155,14 @@ HttpOpRequest::~HttpOpRequest()
 void HttpOpRequest::stageFromRequest(HttpService * service)
 {
 	addRef();
-	service->getPolicy()->addOp(this);			// transfers refcount
+	service->getPolicy().addOp(this);			// transfers refcount
 }
 
 
 void HttpOpRequest::stageFromReady(HttpService * service)
 {
 	addRef();
-	service->getTransport()->addOp(this);		// transfers refcount
+	service->getTransport().addOp(this);		// transfers refcount
 }
 
 
@@ -195,6 +197,8 @@ void HttpOpRequest::stageFromActive(HttpService * service)
 
 void HttpOpRequest::visitNotifier(HttpRequest * request)
 {
+	static const HttpStatus partial_content(HTTP_PARTIAL_CONTENT, HE_SUCCESS);
+	
 	if (mLibraryHandler)
 	{
 		HttpResponse * response = new HttpResponse();
@@ -208,9 +212,15 @@ void HttpOpRequest::visitNotifier(HttpRequest * request)
 			offset = mReplyOffset;
 			length = mReplyLength;
 		}
-		else if (mReplyBody)
+		else if (mReplyBody && partial_content == mStatus)
 		{
-			// Provide implicit offset/length from request/response
+			// Legacy grid services did not provide a 'Content-Range'
+			// header in responses to full- or partly-satisfyiable
+			// 'Range' requests.  For these, we have to hope that
+			// the data starts where requested and the length is simply
+			// whatever we received.  A bit of sanity could be provided
+			// by overlapping ranged requests and verifying that the
+			// overlap matches.
 			offset = mReqOffset;
 			length = mReplyBody->size();
 		}
@@ -306,6 +316,9 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service)
 	// *FIXME:  better error handling later
 	HttpStatus status;
 
+	// Get policy options
+	HttpPolicyGlobal & policy(service->getPolicy().getGlobalOptions());
+	
 	mCurlHandle = curl_easy_init();
 	// curl_easy_setopt(mCurlHandle, CURLOPT_VERBOSE, 1);
 	curl_easy_setopt(mCurlHandle, CURLOPT_TIMEOUT, 30);
@@ -322,21 +335,40 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service)
 	curl_easy_setopt(mCurlHandle, CURLOPT_DNS_CACHE_TIMEOUT, 0);
 	curl_easy_setopt(mCurlHandle, CURLOPT_AUTOREFERER, 1);
 	curl_easy_setopt(mCurlHandle, CURLOPT_FOLLOWLOCATION, 1);
-	curl_easy_setopt(mCurlHandle, CURLOPT_MAXREDIRS, 10);
+	curl_easy_setopt(mCurlHandle, CURLOPT_MAXREDIRS, 10);		// *FIXME:  parameterize this later
 	curl_easy_setopt(mCurlHandle, CURLOPT_WRITEFUNCTION, writeCallback);
 	curl_easy_setopt(mCurlHandle, CURLOPT_WRITEDATA, mCurlHandle);
 	curl_easy_setopt(mCurlHandle, CURLOPT_READFUNCTION, readCallback);
 	curl_easy_setopt(mCurlHandle, CURLOPT_READDATA, mCurlHandle);
+	curl_easy_setopt(mCurlHandle, CURLOPT_SSL_VERIFYPEER, 1);
+	curl_easy_setopt(mCurlHandle, CURLOPT_SSL_VERIFYHOST, 0);
 
+	std::string opt_value;
+	if (policy.get(HttpRequest::GP_CA_PATH, opt_value))
+	{
+		curl_easy_setopt(mCurlHandle, CURLOPT_CAPATH, opt_value.c_str());
+	}
+	if (policy.get(HttpRequest::GP_CA_FILE, opt_value))
+	{
+		curl_easy_setopt(mCurlHandle, CURLOPT_CAINFO, opt_value.c_str());
+	}
+	if (policy.get(HttpRequest::GP_HTTP_PROXY, opt_value))
+	{
+		curl_easy_setopt(mCurlHandle, CURLOPT_PROXY, opt_value.c_str());
+	}
+	
 	switch (mReqMethod)
 	{
 	case HOR_GET:
 		curl_easy_setopt(mCurlHandle, CURLOPT_HTTPGET, 1);
+		mCurlHeaders = curl_slist_append(mCurlHeaders, "Connection: keep-alive");
+		mCurlHeaders = curl_slist_append(mCurlHeaders, "Keep-alive: 300");
 		break;
 		
 	case HOR_POST:
 		{
 			curl_easy_setopt(mCurlHandle, CURLOPT_POST, 1);
+			curl_easy_setopt(mCurlHandle, CURLOPT_ENCODING, "");
 			long data_size(0);
 			if (mReqBody)
 			{
@@ -358,8 +390,11 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service)
 				data_size = mReqBody->size();
 			}
 			curl_easy_setopt(mCurlHandle, CURLOPT_INFILESIZE, data_size);
+			curl_easy_setopt(mCurlHandle, CURLOPT_POSTFIELDS, (void *) NULL);
 			mCurlHeaders = curl_slist_append(mCurlHeaders, "Transfer-Encoding: chunked");
 			mCurlHeaders = curl_slist_append(mCurlHeaders, "Expect:");
+			mCurlHeaders = curl_slist_append(mCurlHeaders, "Connection: keep-alive");
+			mCurlHeaders = curl_slist_append(mCurlHeaders, "Keep-alive: 300");
 		}
 		break;
 		
diff --git a/indra/llcorehttp/_httppolicy.cpp b/indra/llcorehttp/_httppolicy.cpp
index 1d28f23d56..51f5e487dc 100644
--- a/indra/llcorehttp/_httppolicy.cpp
+++ b/indra/llcorehttp/_httppolicy.cpp
@@ -76,12 +76,12 @@ void HttpPolicy::addOp(HttpOpRequest * op)
 HttpService::ELoopSpeed HttpPolicy::processReadyQueue()
 {
 	HttpService::ELoopSpeed result(HttpService::REQUEST_SLEEP);
-	HttpLibcurl * pTransport(mService->getTransport());
+	HttpLibcurl & transport(mService->getTransport());
 	
 	for (int policy_class(0); policy_class < HttpRequest::POLICY_CLASS_LIMIT; ++policy_class)
 	{
 		HttpReadyQueue & readyq(mReadyQueue[policy_class]);
-		int active(pTransport->getActiveCountInClass(policy_class));
+		int active(transport.getActiveCountInClass(policy_class));
 		int needed(8 - active);
 
 		if (needed > 0 && mReadyInClass[policy_class] > 0)
diff --git a/indra/llcorehttp/_httppolicy.h b/indra/llcorehttp/_httppolicy.h
index 2bc03c531f..425079ec63 100644
--- a/indra/llcorehttp/_httppolicy.h
+++ b/indra/llcorehttp/_httppolicy.h
@@ -31,6 +31,7 @@
 #include "httprequest.h"
 #include "_httpservice.h"
 #include "_httpreadyqueue.h"
+#include "_httppolicyglobal.h"
 
 
 namespace LLCore
@@ -68,11 +69,20 @@ public:
 
 	// Shadows HttpService's method
 	bool changePriority(HttpHandle handle, HttpRequest::priority_t priority);
+
+	// Get pointer to global policy options.  Caller is expected
+	// to do context checks like no setting once running.
+	HttpPolicyGlobal &	getGlobalOptions()
+		{
+			return mGlobalOptions;
+		}
+	
 	
 protected:
 	int					mReadyInClass[HttpRequest::POLICY_CLASS_LIMIT];
 	HttpReadyQueue		mReadyQueue[HttpRequest::POLICY_CLASS_LIMIT];
 	HttpService *		mService;				// Naked pointer, not refcounted, not owner
+	HttpPolicyGlobal	mGlobalOptions;
 	
 };  // end class HttpPolicy
 
diff --git a/indra/llcorehttp/_httpservice.h b/indra/llcorehttp/_httpservice.h
index 095316c8a7..748354a8e4 100644
--- a/indra/llcorehttp/_httpservice.h
+++ b/indra/llcorehttp/_httpservice.h
@@ -148,14 +148,14 @@ public:
 	/// Threading:  callable by worker thread.
 	bool changePriority(HttpHandle handle, HttpRequest::priority_t priority);
 	
-	HttpPolicy * getPolicy()
+	HttpPolicy & getPolicy()
 		{
-			return mPolicy;
+			return *mPolicy;
 		}
 
-	HttpLibcurl * getTransport()
+	HttpLibcurl & getTransport()
 		{
-			return mTransport;
+			return *mTransport;
 		}
 	
 protected:
diff --git a/indra/llcorehttp/httpcommon.cpp b/indra/llcorehttp/httpcommon.cpp
index a01182cf23..9f17b5c842 100644
--- a/indra/llcorehttp/httpcommon.cpp
+++ b/indra/llcorehttp/httpcommon.cpp
@@ -66,7 +66,10 @@ std::string HttpStatus::toString() const
 			"Services shutting down",
 			"Operation canceled",
 			"Invalid Content-Range header encountered",
-			"Request handle not found"
+			"Request handle not found",
+			"Invalid datatype for argument or option",
+			"Option has not been explicitly set",
+			"Option is not dynamic and must be set early"
 		};
 	static const int llcore_errors_count(sizeof(llcore_errors) / sizeof(llcore_errors[0]));
 
diff --git a/indra/llcorehttp/httpcommon.h b/indra/llcorehttp/httpcommon.h
index c01a5f85d3..fd2661b700 100644
--- a/indra/llcorehttp/httpcommon.h
+++ b/indra/llcorehttp/httpcommon.h
@@ -137,7 +137,16 @@ enum HttpError
 	HE_INV_CONTENT_RANGE_HDR = 4,
 	
 	// Request handle not found
-	HE_HANDLE_NOT_FOUND = 5
+	HE_HANDLE_NOT_FOUND = 5,
+
+	// Invalid datatype for option/setting
+	HE_INVALID_ARG = 6,
+
+	// Option hasn't been explicitly set
+	HE_OPT_NOT_SET = 7,
+	
+	// Option not dynamic, must be set during init phase
+	HE_OPT_NOT_DYNAMIC = 8
 	
 }; // end enum HttpError
 
diff --git a/indra/llcorehttp/httprequest.cpp b/indra/llcorehttp/httprequest.cpp
index 0e512d97ed..baa0fe1a84 100644
--- a/indra/llcorehttp/httprequest.cpp
+++ b/indra/llcorehttp/httprequest.cpp
@@ -29,6 +29,7 @@
 #include "_httprequestqueue.h"
 #include "_httpreplyqueue.h"
 #include "_httpservice.h"
+#include "_httppolicy.h"
 #include "_httpoperation.h"
 #include "_httpoprequest.h"
 #include "_httpopsetpriority.h"
@@ -127,9 +128,17 @@ HttpRequest::~HttpRequest()
 
 HttpStatus HttpRequest::setPolicyGlobalOption(EGlobalPolicy opt, long value)
 {
-	HttpStatus status;
+	// *FIXME:  Fail if thread is running.
 
-	return status;
+	return HttpService::instanceOf()->getPolicy().getGlobalOptions().set(opt, value);
+}
+
+
+HttpStatus HttpRequest::setPolicyGlobalOption(EGlobalPolicy opt, const std::string & value)
+{
+	// *FIXME:  Fail if thread is running.
+
+	return HttpService::instanceOf()->getPolicy().getGlobalOptions().set(opt, value);
 }
 
 
diff --git a/indra/llcorehttp/httprequest.h b/indra/llcorehttp/httprequest.h
index 57d2da245b..3592d5c6a3 100644
--- a/indra/llcorehttp/httprequest.h
+++ b/indra/llcorehttp/httprequest.h
@@ -111,7 +111,10 @@ public:
 		/// Maximum number of connections the library will use to
 		/// perform operations.  This is somewhat soft as the underlying
 		/// transport will cache some connections (up to 5).
-		GLOBAL_CONNECTION_LIMIT
+		GP_CONNECTION_LIMIT,		///< Takes long giving number of connections
+		GP_CA_PATH,					///< System path/directory where SSL certs are stored.
+		GP_CA_FILE,					///< System path/file containing certs.
+		GP_HTTP_PROXY				///< String giving host/port to use for HTTP proxy
 	};
 
 	/// Set a parameter on a global policy option.  Calls
@@ -122,6 +125,7 @@ public:
 	/// @param value	Desired value of option.
 	/// @return			Standard status code.
 	HttpStatus setPolicyGlobalOption(EGlobalPolicy opt, long value);
+	HttpStatus setPolicyGlobalOption(EGlobalPolicy opt, const std::string & value);
 
 	/// Create a new policy class into which requests can be made.
 	///
@@ -134,15 +138,15 @@ public:
 	enum EClassPolicy
 	{
 		/// Limits the number of connections used for the class.
-		CLASS_CONNECTION_LIMIT,
+		CP_CONNECTION_LIMIT,
 
 		/// Limits the number of connections used for a single
 		/// literal address/port pair within the class.
-		PER_HOST_CONNECTION_LIMIT,
+		CP_PER_HOST_CONNECTION_LIMIT,
 
 		/// Suitable requests are allowed to pipeline on their
 		/// connections when they ask for it.
-		ENABLE_PIPELINING
+		CP_ENABLE_PIPELINING
 	};
 	
 	/// Set a parameter on a class-based policy option.  Calls
@@ -153,9 +157,7 @@ public:
 	/// @param opt				Enum of option to be set.
 	/// @param value			Desired value of option.
 	/// @return					Standard status code.
-	HttpStatus setPolicyClassOption(policy_t policy_id,
-									EClassPolicy opt,
-									long value);
+	HttpStatus setPolicyClassOption(policy_t policy_id, EClassPolicy opt, long value);
 
 	/// @}
 
diff --git a/indra/newview/llappviewer.cpp b/indra/newview/llappviewer.cpp
index 8e6deb9cce..7a44415fba 100644
--- a/indra/newview/llappviewer.cpp
+++ b/indra/newview/llappviewer.cpp
@@ -5356,6 +5356,17 @@ void CoreHttp::init()
 						<< LL_ENDL;
 	}
 
+	mRequest = new LLCore::HttpRequest;
+
+	status = mRequest->setPolicyGlobalOption(LLCore::HttpRequest::GP_CA_FILE,
+											 gDirUtilp->getCAFile());
+	if (! status)
+	{
+		LL_ERRS("Init") << "Failed to set CA File for HTTP services.  Reason:  "
+						<< status.toString()
+						<< LL_ENDL;
+	}
+	
 	status = LLCore::HttpRequest::startThread();
 	if (! status)
 	{
@@ -5364,7 +5375,6 @@ void CoreHttp::init()
 						<< LL_ENDL;
 	}
 
-	mRequest = new LLCore::HttpRequest;
 }
 
 
diff --git a/indra/newview/lltexturefetch.cpp b/indra/newview/lltexturefetch.cpp
index 34fb21798f..f9294b4cd1 100644
--- a/indra/newview/lltexturefetch.cpp
+++ b/indra/newview/lltexturefetch.cpp
@@ -816,14 +816,6 @@ bool LLTextureFetchWorker::doWork(S32 param)
 		mFetchTimer.reset();
 	}
 
-	static LLUUID last_id;
-	if (mID != last_id)
-	{
-		// LL_WARNS("Texture") << "DOWORK SWITCH: " << last_id << " to: " << mID
-		// << LL_ENDL;
-		last_id = mID;
-	}
-	
 	if (mState == INIT)
 	{		
 		mRawImage = NULL ;
@@ -1109,10 +1101,6 @@ bool LLTextureFetchWorker::doWork(S32 param)
 									 << " Bytes: " << mRequestedSize
 									 << " Bandwidth(kbps): " << mFetcher->getTextureBandwidth() << "/" << mFetcher->mMaxBandwidth
 									 << LL_ENDL;
-// 				LL_WARNS("Texture") << "HTTP GET: " << mID << " Offset: " << mRequestedOffset
-// 									<< " Bytes: " << mRequestedSize
-// 									<< " Bandwidth(kbps): " << mFetcher->getTextureBandwidth() << "/" << mFetcher->mMaxBandwidth
-// 									<< LL_ENDL;
 
 				// Will call callbackHttpGet when curl request completes
 				// *FIXME:  enable redirection follow
@@ -1241,7 +1229,7 @@ bool LLTextureFetchWorker::doWork(S32 param)
 				}
 			}
 						
-			if (mHaveAllData && mRequestedDiscard == 0) //the image file is fully loaded.
+			if (mHaveAllData /* && mRequestedDiscard == 0*/) //the image file is fully loaded.
 			{
 				mFileSize = total_size;
 			}
@@ -1692,13 +1680,32 @@ S32 LLTextureFetchWorker::callbackHttpGet(LLCore::HttpResponse * response,
 			body->addRef();
 			mHttpBufferArray = body;
 
-			if (data_size < mRequestedSize && mRequestedDiscard == 0)
+			if (! partial)
+			{
+				// Response indicates this is the entire asset regardless
+				// of our asking for a byte range.  Mark it so and drop
+				// any partial data we might have so that the current
+				// response body becomes the entire dataset.
+				if (data_size <= mRequestedOffset)
+				{
+					LL_WARNS("Texture") << "Fetched entire texture " << mID
+										<< " when it was expected to be marked complete.  mImageSize:  "
+										<< mFileSize << " datasize:  " << mFormattedImage->getDataSize()
+										<< LL_ENDL;
+				}
+				mHaveAllData = TRUE;
+				llassert_always(mDecodeHandle == 0);
+				mFormattedImage = NULL; // discard any previous data we had
+			}
+			else if (data_size < mRequestedSize && mRequestedDiscard == 0)
 			{
+				// *FIXME:  I think we can treat this as complete regardless
+				// of requested discard level.  Revisit this...
 				mHaveAllData = TRUE;
 			}
 			else if (data_size > mRequestedSize)
 			{
-				// *TODO: This shouldn't be happening any more
+				// *TODO: This shouldn't be happening any more  (REALLY don't expect this anymore)
 				llwarns << "data_size = " << data_size << " > requested: " << mRequestedSize << llendl;
 				mHaveAllData = TRUE;
 				llassert_always(mDecodeHandle == 0);