From 5611cb6d476540e6a1c654c1f9acdce2787b3505 Mon Sep 17 00:00:00 2001
From: Monty Brandenberg <monty@lindenlab.com>
Date: Mon, 23 Apr 2012 16:19:39 -0400
Subject: Okay, imported the core-http library and got it compiling
 suspiciously easily. The unit/integration tests don't work yet as I'm still
 battling cmake/autobuild as usual but first milestone passed.

---
 indra/llcorehttp/_httpoprequest.cpp | 473 ++++++++++++++++++++++++++++++++++++
 1 file changed, 473 insertions(+)
 create mode 100644 indra/llcorehttp/_httpoprequest.cpp

(limited to 'indra/llcorehttp/_httpoprequest.cpp')

diff --git a/indra/llcorehttp/_httpoprequest.cpp b/indra/llcorehttp/_httpoprequest.cpp
new file mode 100644
index 0000000000..d71ace5d57
--- /dev/null
+++ b/indra/llcorehttp/_httpoprequest.cpp
@@ -0,0 +1,473 @@
+/**
+ * @file _httpoprequest.cpp
+ * @brief Definitions for internal class HttpOpRequest
+ *
+ * $LicenseInfo:firstyear=2012&license=viewerlgpl$
+ * Second Life Viewer Source Code
+ * Copyright (C) 2012, Linden Research, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation;
+ * version 2.1 of the License only.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ * Linden Research, Inc., 945 Battery Street, San Francisco, CA  94111  USA
+ * $/LicenseInfo$
+ */
+
+#include "_httpoprequest.h"
+
+#include <cstdio>
+#include <algorithm>
+
+#include "httpcommon.h"
+#include "httphandler.h"
+#include "httpresponse.h"
+#include "bufferarray.h"
+#include "httpheaders.h"
+#include "httpoptions.h"
+
+#include "_httprequestqueue.h"
+#include "_httpreplyqueue.h"
+#include "_httpservice.h"
+#include "_httppolicy.h"
+#include "_httplibcurl.h"
+
+
+namespace
+{
+
+// Attempts to parse a 'Content-Range:' header.  Caller must already
+// have verified that the header tag is present.  The 'buffer' argument
+// will be processed by strtok_r calls which will modify the buffer.
+//
+// @return		-1 if invalid and response should be dropped, 0 if valid an
+//				correct, 1 if couldn't be parsed.  If 0, the first, last,
+//				and length arguments are also written.  'length' may be
+//				0 if the length wasn't available to the server.
+//
+int parse_content_range_header(char * buffer,
+							   unsigned int * first,
+							   unsigned int * last,
+							   unsigned int * length);
+
+#if defined(WIN32)
+
+// Not available on windows where the legacy strtok interface
+// is thread-safe.
+char *strtok_r(char *str, const char *delim, char **saveptr);
+
+#endif
+
+}
+
+
+namespace LLCore
+{
+
+
+// ==================================
+// HttpOpRequest
+// ==================================
+
+
+HttpOpRequest::HttpOpRequest()
+	: HttpOperation(),
+	  mProcFlags(0U),
+	  mReqMethod(HOR_GET),
+	  mReqBody(NULL),
+	  mReqOffset(0),
+	  mReqLen(0),
+	  mReqHeaders(NULL),
+	  mReqOptions(NULL),
+	  mCurlActive(false),
+	  mCurlHandle(NULL),
+	  mCurlHeaders(NULL),
+	  mCurlService(NULL),
+	  mReplyStatus(200),
+	  mReplyBody(NULL),
+	  mReplyOffset(0),
+	  mReplyLen(0),
+	  mReplyHeaders(NULL)
+{}
+
+
+HttpOpRequest::~HttpOpRequest()
+{
+	if (mCurlHandle)
+	{
+		curl_easy_cleanup(mCurlHandle);
+		mCurlHandle = NULL;
+	}
+
+	if (mCurlHeaders)
+	{
+		curl_slist_free_all(mCurlHeaders);
+		mCurlHeaders = NULL;
+	}
+
+	mCurlService = NULL;
+	
+	if (mReqBody)
+	{
+		mReqBody->release();
+		mReqBody = NULL;
+	}
+	
+	if (mReqHeaders)
+	{
+		curl_slist_free_all(mReqHeaders);
+		mReqHeaders = NULL;
+	}
+
+	if (mReqOptions)
+	{
+		mReqOptions->release();
+		mReqOptions = NULL;
+	}
+
+	if (mReplyBody)
+	{
+		mReplyBody->release();
+		mReplyBody = NULL;
+	}
+
+	if (mReplyHeaders)
+	{
+		mReplyHeaders->release();
+		mReplyHeaders = NULL;
+	}
+}
+
+
+void HttpOpRequest::stageFromRequest(HttpService * service)
+{
+	addRef();
+	service->getPolicy()->addOp(this);			// transfers refcount
+}
+
+
+void HttpOpRequest::stageFromReady(HttpService * service)
+{
+	addRef();
+	service->getTransport()->addOp(this);		// transfers refcount
+}
+
+
+void HttpOpRequest::stageFromActive(HttpService * service)
+{
+	if (mReplyLen)
+	{
+		// If non-zero, we received and processed a Content-Range
+		// header with the response.  Verify that what it says
+		// is consistent with the received data.
+		if (mReplyLen != mReplyBody->size())
+		{
+			// Not as expected, fail the request
+			mStatus = HttpStatus(HttpStatus::LLCORE, HE_INV_CONTENT_RANGE_HDR);
+		}
+	}
+	
+	if (mReqHeaders)
+	{
+		// We take these headers out of the request now as they were
+		// allocated originally in this thread and the notifier doesn't
+		// need them.  This eliminates one source of heap moving across
+		// threads.
+
+		curl_slist_free_all(mReqHeaders);
+		mReqHeaders = NULL;
+	}
+	addAsReply();
+}
+
+
+void HttpOpRequest::visitNotifier(HttpRequest * request)
+{
+	if (mLibraryHandler)
+	{
+		HttpResponse * response = new HttpResponse();
+
+		// *FIXME:  add http status, offset, length
+		response->setStatus(mStatus);
+		response->setReplyStatus(mReplyStatus);
+		response->setBody(mReplyBody);
+		response->setHeaders(mReplyHeaders);
+		mLibraryHandler->onCompleted(static_cast<HttpHandle>(this), response);
+
+		response->release();
+	}
+}
+
+
+HttpStatus HttpOpRequest::cancel()
+{
+	mStatus = HttpStatus(HttpStatus::LLCORE, HE_OP_CANCELED);
+
+	addAsReply();
+
+	return HttpStatus();
+}
+
+
+HttpStatus HttpOpRequest::setupGetByteRange(unsigned int policy_id,
+											float priority,
+											const std::string & url,
+											size_t offset,
+											size_t len,
+											HttpOptions * options,
+											HttpHeaders * headers)
+{
+	HttpStatus status;
+
+	mProcFlags = 0;
+	mReqPolicy = policy_id;
+	mReqPriority = priority;
+	mReqMethod = HOR_GET;
+	mReqURL = url;
+	mReqOffset = offset;
+	mReqLen = len;
+	if (offset || len)
+	{
+		mProcFlags |= PF_SCAN_RANGE_HEADER;
+	}
+	if (headers && ! mReqHeaders)
+	{
+		mReqHeaders = append_headers_to_slist(headers, mReqHeaders);
+	}
+	if (options && ! mReqOptions)
+	{
+		mReqOptions = new HttpOptions(*options);
+	}
+	
+	return status;
+}
+
+
+HttpStatus HttpOpRequest::prepareForGet(HttpService * service)
+{
+	// *FIXME:  better error handling later
+	HttpStatus status;
+	
+	mCurlHandle = curl_easy_init();
+	curl_easy_setopt(mCurlHandle, CURLOPT_NOSIGNAL, 1);
+	curl_easy_setopt(mCurlHandle, CURLOPT_NOPROGRESS, 1);
+	curl_easy_setopt(mCurlHandle, CURLOPT_URL, mReqURL.c_str());
+	curl_easy_setopt(mCurlHandle, CURLOPT_PRIVATE, this);
+	curl_easy_setopt(mCurlHandle, CURLOPT_ENCODING, "");
+	// curl_easy_setopt(handle, CURLOPT_PROXY, "");
+
+	mCurlHeaders = curl_slist_append(mCurlHeaders, "Pragma:");
+	curl_easy_setopt(mCurlHandle, CURLOPT_DNS_CACHE_TIMEOUT, 0);
+	curl_easy_setopt(mCurlHandle, CURLOPT_HTTPHEADER, mCurlHeaders);
+	curl_easy_setopt(mCurlHandle, CURLOPT_FOLLOWLOCATION, 1);
+	curl_easy_setopt(mCurlHandle, CURLOPT_WRITEFUNCTION, writeCallback);
+	curl_easy_setopt(mCurlHandle, CURLOPT_WRITEDATA, mCurlHandle);
+
+	if (mReqOffset || mReqLen)
+	{
+		static const char * fmt1("Range: bytes=%d-%d");
+		static const char * fmt2("Range: bytes=%d-");
+
+		char range_line[64];
+
+#if defined(WIN32)
+		_snprintf_s(range_line, sizeof(range_line), sizeof(range_line) - 1,
+					(mReqLen ? fmt1 : fmt2),
+					mReqOffset, mReqOffset + mReqLen - 1);
+#else
+		snprintf(range_line, sizeof(range_line),
+				 (mReqLen ? fmt1 : fmt2),
+				 mReqOffset, mReqOffset + mReqLen - 1);
+#endif // defined(WIN32)
+		range_line[sizeof(range_line) - 1] = '\0';
+		mCurlHeaders = curl_slist_append(mCurlHeaders, range_line);
+	}
+	
+	if (mProcFlags & (PF_SCAN_RANGE_HEADER | PF_SAVE_HEADERS))
+	{
+		curl_easy_setopt(mCurlHandle, CURLOPT_HEADERFUNCTION, headerCallback);
+		curl_easy_setopt(mCurlHandle, CURLOPT_HEADERDATA, mCurlHandle);
+	}
+	
+	if (status)
+	{
+		mCurlService = service;
+	}
+	return status;
+}
+
+
+size_t HttpOpRequest::writeCallback(void * data, size_t size, size_t nmemb, void * userdata)
+{
+	CURL * handle(static_cast<CURL *>(userdata));
+	HttpOpRequest * op(NULL);
+	curl_easy_getinfo(handle, CURLINFO_PRIVATE, &op);
+	// *FIXME:  check the pointer
+
+	if (! op->mReplyBody)
+	{
+		op->mReplyBody = new BufferArray();
+	}
+	const size_t req_size(size * nmemb);
+	char * lump(op->mReplyBody->appendBufferAlloc(req_size));
+	memcpy(lump, data, req_size);
+
+	return req_size;
+}
+
+		
+size_t HttpOpRequest::headerCallback(void * data, size_t size, size_t nmemb, void * userdata)
+{
+	static const char status_line[] = "HTTP/";
+	static const size_t status_line_len = sizeof(status_line) - 1;
+
+	static const char con_ran_line[] = "content-range:";
+	static const size_t con_ran_line_len = sizeof(con_ran_line) - 1;
+	
+	CURL * handle(static_cast<CURL *>(userdata));
+	HttpOpRequest * op(NULL);
+	curl_easy_getinfo(handle, CURLINFO_PRIVATE, &op);
+	// *FIXME:  check the pointer
+
+	const size_t hdr_size(size * nmemb);
+	const char * hdr_data(static_cast<const char *>(data));		// Not null terminated
+	
+	if (hdr_size >= status_line_len && ! strncmp(status_line, hdr_data, status_line_len))
+	{
+		// One of possibly several status lines.  Reset what we know and start over
+		op->mReplyOffset = 0;
+		op->mReplyLen = 0;
+		op->mStatus = HttpStatus();
+	}
+	else if (op->mProcFlags & PF_SCAN_RANGE_HEADER)
+	{
+		char hdr_buffer[128];
+		size_t frag_size((std::min)(hdr_size, sizeof(hdr_buffer) - 1));
+		
+		memcpy(hdr_buffer, hdr_data, frag_size);
+		hdr_buffer[frag_size] = '\0';
+#if defined(WIN32)
+		if (! _strnicmp(hdr_buffer, con_ran_line, (std::min)(frag_size, con_ran_line_len)))
+#else
+		if (! strncasecmp(hdr_buffer, con_ran_line, (std::min)(frag_size, con_ran_line_len)))
+#endif
+		{
+			unsigned int first(0), last(0), length(0);
+			int status;
+
+			if (! (status = parse_content_range_header(hdr_buffer, &first, &last, &length)))
+			{
+				// Success, record the fragment position
+				op->mReplyOffset = first;
+				op->mReplyLen = last - first + 1;
+			}
+			else if (-1 == status)
+			{
+				// Response is badly formed and shouldn't be accepted
+				op->mStatus = HttpStatus(HttpStatus::LLCORE, HE_INV_CONTENT_RANGE_HDR);
+			}
+			else
+			{
+				// Ignore the unparsable.
+				// *FIXME:  Maybe issue a warning into the log here
+				;
+			}
+		}
+	}
+
+	if (op->mProcFlags & PF_SAVE_HEADERS)
+	{
+		// Save headers in response
+		;
+		
+	}
+
+	return hdr_size;
+}
+
+}   // end namespace LLCore
+
+
+// =======================================
+// Anonymous Namespace
+// =======================================
+
+namespace
+{
+
+int parse_content_range_header(char * buffer,
+							   unsigned int * first,
+							   unsigned int * last,
+							   unsigned int * length)
+{
+	char * tok_state(NULL), * tok;
+	bool match(true);
+			
+	if (! strtok_r(buffer, ": \t", &tok_state))
+		match = false;
+	if (match && (tok = strtok_r(NULL, " \t", &tok_state)))
+#if defined(WIN32)
+		match = 0 == _stricmp("bytes", tok);
+#else
+		match = 0 == strcasecmp("bytes", tok);
+#endif
+	if (match && ! (tok = strtok_r(NULL, " \t", &tok_state)))
+		match = false;
+	if (match)
+	{
+		unsigned int lcl_first(0), lcl_last(0), lcl_len(0);
+
+#if defined(WIN32)
+		if (3 == sscanf_s(tok, "%u-%u/%u", &lcl_first, &lcl_last, &lcl_len))
+#else
+		if (3 == sscanf(tok, "%u-%u/%u", &lcl_first, &lcl_last, &lcl_len))
+#endif
+		{
+			if (lcl_first > lcl_last || lcl_last >= lcl_len)
+				return -1;
+			*first = lcl_first;
+			*last = lcl_last;
+			*length = lcl_len;
+			return 0;
+		}
+#if defined(WIN32)
+		if (2 == sscanf_s(tok, "%u-%u/*", &lcl_first, &lcl_last))
+#else
+		if (2 == sscanf(tok, "%u-%u/*", &lcl_first, &lcl_last))
+#endif
+		{
+			if (lcl_first > lcl_last)
+				return -1;
+			*first = lcl_first;
+			*last = lcl_last;
+			*length = 0;
+			return 0;
+		}
+	}
+
+	// Header is there but badly/unexpectedly formed, try to ignore it.
+	return 1;
+}
+
+#if defined(WIN32)
+
+char *strtok_r(char *str, const char *delim, char ** savestate)
+{
+	return strtok_s(str, delim, savestate);
+}
+
+#endif
+
+}  // end anonymous namespace
+
+		
-- 
cgit v1.2.3


From a54e9c379550f7901404990ea7fddb99c0188048 Mon Sep 17 00:00:00 2001
From: Monty Brandenberg <monty@lindenlab.com>
Date: Wed, 25 Apr 2012 15:47:30 -0400
Subject: Another fix for Mac warnings.  Uninitialized auto check.  Not an
 actual problem but this will quiet the compiler.

---
 indra/llcorehttp/_httpoprequest.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'indra/llcorehttp/_httpoprequest.cpp')

diff --git a/indra/llcorehttp/_httpoprequest.cpp b/indra/llcorehttp/_httpoprequest.cpp
index d71ace5d57..316df8bd49 100644
--- a/indra/llcorehttp/_httpoprequest.cpp
+++ b/indra/llcorehttp/_httpoprequest.cpp
@@ -410,7 +410,7 @@ int parse_content_range_header(char * buffer,
 							   unsigned int * last,
 							   unsigned int * length)
 {
-	char * tok_state(NULL), * tok;
+	char * tok_state(NULL), * tok(NULL);
 	bool match(true);
 			
 	if (! strtok_r(buffer, ": \t", &tok_state))
-- 
cgit v1.2.3


From 74d59e7128bb02a4b49af99e44f437a736a3f62b Mon Sep 17 00:00:00 2001
From: Monty Brandenberg <monty@lindenlab.com>
Date: Mon, 7 May 2012 15:16:31 -0400
Subject: Build llcorehttp as part of a viewer dependency with unit tests. 
 This required boost::thread and the easiest path to that was to go with the
 1.48 Boost release in the 3P tree (eliminating a fork for a modified 1.45
 packaging).  One unit test, the most important one, is failing in
 test_httprequest but that can be attended to later.  This test issues a GET
 to http://localhost:2/ and that is hitting the wire but the libcurl plumbing
 isn't delivering the failure, only the eventual timeout.  An unexpected
 change in behavior.

---
 indra/llcorehttp/_httpoprequest.cpp | 2 ++
 1 file changed, 2 insertions(+)

(limited to 'indra/llcorehttp/_httpoprequest.cpp')

diff --git a/indra/llcorehttp/_httpoprequest.cpp b/indra/llcorehttp/_httpoprequest.cpp
index 316df8bd49..3c9eb71b9a 100644
--- a/indra/llcorehttp/_httpoprequest.cpp
+++ b/indra/llcorehttp/_httpoprequest.cpp
@@ -259,6 +259,8 @@ HttpStatus HttpOpRequest::prepareForGet(HttpService * service)
 	HttpStatus status;
 	
 	mCurlHandle = curl_easy_init();
+	curl_easy_setopt(mCurlHandle, CURLOPT_TIMEOUT, 30);
+	curl_easy_setopt(mCurlHandle, CURLOPT_CONNECTTIMEOUT, 30);
 	curl_easy_setopt(mCurlHandle, CURLOPT_NOSIGNAL, 1);
 	curl_easy_setopt(mCurlHandle, CURLOPT_NOPROGRESS, 1);
 	curl_easy_setopt(mCurlHandle, CURLOPT_URL, mReqURL.c_str());
-- 
cgit v1.2.3


From 8fc350125c671baeae6b7f8b1814251009f4f50a Mon Sep 17 00:00:00 2001
From: Monty Brandenberg <monty@lindenlab.com>
Date: Wed, 23 May 2012 19:12:09 -0400
Subject: Integrate llcorehttp library into lltexturefetch design. This is the
 first functional viewer pass with the HTTP work of the texture fetch code
 performed by the llcorehttp library.  Not exactly a 'drop-in' replacement but
 a work-alike with some changes (e.g. handler notification in consumer thread
 versus responder notification in worker thread).

This also includes some temporary changes in the priority scheme to prevent
the kind of priority inversion found in VWR-28996.  Scheme used here does
provide liveness if not optimal responsiveness or order-of-operation.

The llcorehttp library at this point is far from optimally performing.
Its worker thread is making relatively poor use of cycles it gets and
it doesn't idle or sleep intelligently yet.  This early integration step
helps shake out the interfaces, implementation niceties will be covered
soon.
---
 indra/llcorehttp/_httpoprequest.cpp | 105 +++++++++++++++++++++---------------
 1 file changed, 63 insertions(+), 42 deletions(-)

(limited to 'indra/llcorehttp/_httpoprequest.cpp')

diff --git a/indra/llcorehttp/_httpoprequest.cpp b/indra/llcorehttp/_httpoprequest.cpp
index 3c9eb71b9a..521bd5b879 100644
--- a/indra/llcorehttp/_httpoprequest.cpp
+++ b/indra/llcorehttp/_httpoprequest.cpp
@@ -86,53 +86,53 @@ HttpOpRequest::HttpOpRequest()
 	  mReqMethod(HOR_GET),
 	  mReqBody(NULL),
 	  mReqOffset(0),
-	  mReqLen(0),
+	  mReqLength(0),
 	  mReqHeaders(NULL),
 	  mReqOptions(NULL),
 	  mCurlActive(false),
 	  mCurlHandle(NULL),
-	  mCurlHeaders(NULL),
 	  mCurlService(NULL),
-	  mReplyStatus(200),
+	  mCurlHeaders(NULL),
 	  mReplyBody(NULL),
 	  mReplyOffset(0),
-	  mReplyLen(0),
+	  mReplyLength(0),
 	  mReplyHeaders(NULL)
 {}
 
 
 HttpOpRequest::~HttpOpRequest()
 {
-	if (mCurlHandle)
-	{
-		curl_easy_cleanup(mCurlHandle);
-		mCurlHandle = NULL;
-	}
-
-	if (mCurlHeaders)
-	{
-		curl_slist_free_all(mCurlHeaders);
-		mCurlHeaders = NULL;
-	}
-
-	mCurlService = NULL;
-	
 	if (mReqBody)
 	{
 		mReqBody->release();
 		mReqBody = NULL;
 	}
 	
+	if (mReqOptions)
+	{
+		mReqOptions->release();
+		mReqOptions = NULL;
+	}
+
 	if (mReqHeaders)
 	{
-		curl_slist_free_all(mReqHeaders);
+		mReqHeaders->release();
 		mReqHeaders = NULL;
 	}
 
-	if (mReqOptions)
+	if (mCurlHandle)
 	{
-		mReqOptions->release();
-		mReqOptions = NULL;
+		curl_easy_cleanup(mCurlHandle);
+		mCurlHandle = NULL;
+	}
+
+	mCurlService = NULL;
+	
+
+	if (mCurlHeaders)
+	{
+		curl_slist_free_all(mCurlHeaders);
+		mCurlHeaders = NULL;
 	}
 
 	if (mReplyBody)
@@ -165,28 +165,29 @@ void HttpOpRequest::stageFromReady(HttpService * service)
 
 void HttpOpRequest::stageFromActive(HttpService * service)
 {
-	if (mReplyLen)
+	if (mReplyLength)
 	{
 		// If non-zero, we received and processed a Content-Range
 		// header with the response.  Verify that what it says
 		// is consistent with the received data.
-		if (mReplyLen != mReplyBody->size())
+		if (mReplyLength != mReplyBody->size())
 		{
 			// Not as expected, fail the request
 			mStatus = HttpStatus(HttpStatus::LLCORE, HE_INV_CONTENT_RANGE_HDR);
 		}
 	}
 	
-	if (mReqHeaders)
+	if (mCurlHeaders)
 	{
 		// We take these headers out of the request now as they were
 		// allocated originally in this thread and the notifier doesn't
 		// need them.  This eliminates one source of heap moving across
 		// threads.
 
-		curl_slist_free_all(mReqHeaders);
-		mReqHeaders = NULL;
+		curl_slist_free_all(mCurlHeaders);
+		mCurlHeaders = NULL;
 	}
+
 	addAsReply();
 }
 
@@ -196,12 +197,24 @@ void HttpOpRequest::visitNotifier(HttpRequest * request)
 	if (mLibraryHandler)
 	{
 		HttpResponse * response = new HttpResponse();
-
-		// *FIXME:  add http status, offset, length
 		response->setStatus(mStatus);
-		response->setReplyStatus(mReplyStatus);
 		response->setBody(mReplyBody);
 		response->setHeaders(mReplyHeaders);
+		unsigned int offset(0), length(0);
+		if (mReplyOffset || mReplyLength)
+		{
+			// Got an explicit offset/length in response
+			offset = mReplyOffset;
+			length = mReplyLength;
+		}
+		else if (mReplyBody)
+		{
+			// Provide implicit offset/length from request/response
+			offset = mReqOffset;
+			length = mReplyBody->size();
+		}
+		response->setRange(offset, length);
+
 		mLibraryHandler->onCompleted(static_cast<HttpHandle>(this), response);
 
 		response->release();
@@ -235,14 +248,15 @@ HttpStatus HttpOpRequest::setupGetByteRange(unsigned int policy_id,
 	mReqMethod = HOR_GET;
 	mReqURL = url;
 	mReqOffset = offset;
-	mReqLen = len;
+	mReqLength = len;
 	if (offset || len)
 	{
 		mProcFlags |= PF_SCAN_RANGE_HEADER;
 	}
 	if (headers && ! mReqHeaders)
 	{
-		mReqHeaders = append_headers_to_slist(headers, mReqHeaders);
+		headers->addRef();
+		mReqHeaders = headers;
 	}
 	if (options && ! mReqOptions)
 	{
@@ -257,7 +271,7 @@ HttpStatus HttpOpRequest::prepareForGet(HttpService * service)
 {
 	// *FIXME:  better error handling later
 	HttpStatus status;
-	
+
 	mCurlHandle = curl_easy_init();
 	curl_easy_setopt(mCurlHandle, CURLOPT_TIMEOUT, 30);
 	curl_easy_setopt(mCurlHandle, CURLOPT_CONNECTTIMEOUT, 30);
@@ -268,14 +282,18 @@ HttpStatus HttpOpRequest::prepareForGet(HttpService * service)
 	curl_easy_setopt(mCurlHandle, CURLOPT_ENCODING, "");
 	// curl_easy_setopt(handle, CURLOPT_PROXY, "");
 
-	mCurlHeaders = curl_slist_append(mCurlHeaders, "Pragma:");
 	curl_easy_setopt(mCurlHandle, CURLOPT_DNS_CACHE_TIMEOUT, 0);
-	curl_easy_setopt(mCurlHandle, CURLOPT_HTTPHEADER, mCurlHeaders);
 	curl_easy_setopt(mCurlHandle, CURLOPT_FOLLOWLOCATION, 1);
 	curl_easy_setopt(mCurlHandle, CURLOPT_WRITEFUNCTION, writeCallback);
 	curl_easy_setopt(mCurlHandle, CURLOPT_WRITEDATA, mCurlHandle);
 
-	if (mReqOffset || mReqLen)
+	if (mReqHeaders)
+	{
+		mCurlHeaders = append_headers_to_slist(mReqHeaders, mCurlHeaders);
+	}
+	mCurlHeaders = curl_slist_append(mCurlHeaders, "Pragma:");
+	
+	if (mReqOffset || mReqLength)
 	{
 		static const char * fmt1("Range: bytes=%d-%d");
 		static const char * fmt2("Range: bytes=%d-");
@@ -284,16 +302,17 @@ HttpStatus HttpOpRequest::prepareForGet(HttpService * service)
 
 #if defined(WIN32)
 		_snprintf_s(range_line, sizeof(range_line), sizeof(range_line) - 1,
-					(mReqLen ? fmt1 : fmt2),
-					mReqOffset, mReqOffset + mReqLen - 1);
+					(mReqLength ? fmt1 : fmt2),
+					mReqOffset, mReqOffset + mReqLength - 1);
 #else
 		snprintf(range_line, sizeof(range_line),
-				 (mReqLen ? fmt1 : fmt2),
-				 mReqOffset, mReqOffset + mReqLen - 1);
+				 (mReqLength ? fmt1 : fmt2),
+				 mReqOffset, mReqOffset + mReqLength - 1);
 #endif // defined(WIN32)
 		range_line[sizeof(range_line) - 1] = '\0';
 		mCurlHeaders = curl_slist_append(mCurlHeaders, range_line);
 	}
+	curl_easy_setopt(mCurlHandle, CURLOPT_HTTPHEADER, mCurlHeaders);
 	
 	if (mProcFlags & (PF_SCAN_RANGE_HEADER | PF_SAVE_HEADERS))
 	{
@@ -347,8 +366,9 @@ size_t HttpOpRequest::headerCallback(void * data, size_t size, size_t nmemb, voi
 	if (hdr_size >= status_line_len && ! strncmp(status_line, hdr_data, status_line_len))
 	{
 		// One of possibly several status lines.  Reset what we know and start over
+		// taking results from the last header stanza we receive.
 		op->mReplyOffset = 0;
-		op->mReplyLen = 0;
+		op->mReplyLength = 0;
 		op->mStatus = HttpStatus();
 	}
 	else if (op->mProcFlags & PF_SCAN_RANGE_HEADER)
@@ -371,7 +391,7 @@ size_t HttpOpRequest::headerCallback(void * data, size_t size, size_t nmemb, voi
 			{
 				// Success, record the fragment position
 				op->mReplyOffset = first;
-				op->mReplyLen = last - first + 1;
+				op->mReplyLength = last - first + 1;
 			}
 			else if (-1 == status)
 			{
@@ -390,6 +410,7 @@ size_t HttpOpRequest::headerCallback(void * data, size_t size, size_t nmemb, voi
 	if (op->mProcFlags & PF_SAVE_HEADERS)
 	{
 		// Save headers in response
+		// *FIXME:  Implement this...
 		;
 		
 	}
-- 
cgit v1.2.3


From b8edacd0bb4feacc3ac1d61421e600c75ab87f7c Mon Sep 17 00:00:00 2001
From: Monty Brandenberg <monty@lindenlab.com>
Date: Fri, 1 Jun 2012 14:07:34 -0400
Subject: Major steps towards implementing the policy component. Identified and
 reacted to the priority inversion problem we have in texturefetch.  Includes
 the introduction of a priority_queue for the requests that are ready.  Start
 some parameterization in anticipation of having policy_class everywhere. 
 Removed _assert.h which isn't really needed in indra codebase.  Implemented
 async setPriority request (which I hope I can get rid of eventually along
 with all priorities in this library).  Converted to using unsigned int for
 priority rather than float.  Implemented POST and did groundwork for PUT.

---
 indra/llcorehttp/_httpoprequest.cpp | 113 +++++++++++++++++++++++++++++++++++-
 1 file changed, 111 insertions(+), 2 deletions(-)

(limited to 'indra/llcorehttp/_httpoprequest.cpp')

diff --git a/indra/llcorehttp/_httpoprequest.cpp b/indra/llcorehttp/_httpoprequest.cpp
index 521bd5b879..54b9990057 100644
--- a/indra/llcorehttp/_httpoprequest.cpp
+++ b/indra/llcorehttp/_httpoprequest.cpp
@@ -93,6 +93,7 @@ HttpOpRequest::HttpOpRequest()
 	  mCurlHandle(NULL),
 	  mCurlService(NULL),
 	  mCurlHeaders(NULL),
+	  mCurlBodyPos(0),
 	  mReplyBody(NULL),
 	  mReplyOffset(0),
 	  mReplyLength(0),
@@ -267,12 +268,46 @@ HttpStatus HttpOpRequest::setupGetByteRange(unsigned int policy_id,
 }
 
 
-HttpStatus HttpOpRequest::prepareForGet(HttpService * service)
+HttpStatus HttpOpRequest::setupPost(unsigned int policy_id,
+									float priority,
+									const std::string & url,
+									BufferArray * body,
+									HttpOptions * options,
+									HttpHeaders * headers)
+{
+	HttpStatus status;
+
+	mProcFlags = 0;
+	mReqPolicy = policy_id;
+	mReqPriority = priority;
+	mReqMethod = HOR_POST;
+	mReqURL = url;
+	if (body)
+	{
+		body->addRef();
+		mReqBody = body;
+	}
+	if (headers && ! mReqHeaders)
+	{
+		headers->addRef();
+		mReqHeaders = headers;
+	}
+	if (options && ! mReqOptions)
+	{
+		mReqOptions = new HttpOptions(*options);
+	}
+	
+	return status;
+}
+
+
+HttpStatus HttpOpRequest::prepareRequest(HttpService * service)
 {
 	// *FIXME:  better error handling later
 	HttpStatus status;
 
 	mCurlHandle = curl_easy_init();
+	// curl_easy_setopt(mCurlHandle, CURLOPT_VERBOSE, 1);
 	curl_easy_setopt(mCurlHandle, CURLOPT_TIMEOUT, 30);
 	curl_easy_setopt(mCurlHandle, CURLOPT_CONNECTTIMEOUT, 30);
 	curl_easy_setopt(mCurlHandle, CURLOPT_NOSIGNAL, 1);
@@ -280,20 +315,68 @@ HttpStatus HttpOpRequest::prepareForGet(HttpService * service)
 	curl_easy_setopt(mCurlHandle, CURLOPT_URL, mReqURL.c_str());
 	curl_easy_setopt(mCurlHandle, CURLOPT_PRIVATE, this);
 	curl_easy_setopt(mCurlHandle, CURLOPT_ENCODING, "");
+	// *FIXME:  Need to deal with proxy setup...
 	// curl_easy_setopt(handle, CURLOPT_PROXY, "");
 
+	// *FIXME:  Revisit this old DNS timeout setting - may no longer be valid
 	curl_easy_setopt(mCurlHandle, CURLOPT_DNS_CACHE_TIMEOUT, 0);
+	curl_easy_setopt(mCurlHandle, CURLOPT_AUTOREFERER, 1);
 	curl_easy_setopt(mCurlHandle, CURLOPT_FOLLOWLOCATION, 1);
+	curl_easy_setopt(mCurlHandle, CURLOPT_MAXREDIRS, 10);
 	curl_easy_setopt(mCurlHandle, CURLOPT_WRITEFUNCTION, writeCallback);
 	curl_easy_setopt(mCurlHandle, CURLOPT_WRITEDATA, mCurlHandle);
+	curl_easy_setopt(mCurlHandle, CURLOPT_READFUNCTION, readCallback);
+	curl_easy_setopt(mCurlHandle, CURLOPT_READDATA, mCurlHandle);
 
+	switch (mReqMethod)
+	{
+	case HOR_GET:
+		curl_easy_setopt(mCurlHandle, CURLOPT_HTTPGET, 1);
+		break;
+		
+	case HOR_POST:
+		{
+			curl_easy_setopt(mCurlHandle, CURLOPT_POST, 1);
+			long data_size(0);
+			if (mReqBody)
+			{
+				mReqBody->seek(0);
+				data_size = mReqBody->size();
+			}
+			curl_easy_setopt(mCurlHandle, CURLOPT_POSTFIELDS, static_cast<void *>(NULL));
+			curl_easy_setopt(mCurlHandle, CURLOPT_POSTFIELDSIZE, data_size);
+			mCurlHeaders = curl_slist_append(mCurlHeaders, "Transfer-Encoding: chunked");
+			mCurlHeaders = curl_slist_append(mCurlHeaders, "Expect:");
+		}
+		break;
+		
+	case HOR_PUT:
+		{
+			curl_easy_setopt(mCurlHandle, CURLOPT_UPLOAD, 1);
+			long data_size(0);
+			if (mReqBody)
+			{
+				mReqBody->seek(0);
+				data_size = mReqBody->size();
+			}
+			curl_easy_setopt(mCurlHandle, CURLOPT_INFILESIZE, data_size);
+			mCurlHeaders = curl_slist_append(mCurlHeaders, "Transfer-Encoding: chunked");
+			mCurlHeaders = curl_slist_append(mCurlHeaders, "Expect:");
+		}
+		break;
+		
+	default:
+		// *FIXME:  fail out here
+		break;
+	}
+	
 	if (mReqHeaders)
 	{
 		mCurlHeaders = append_headers_to_slist(mReqHeaders, mCurlHeaders);
 	}
 	mCurlHeaders = curl_slist_append(mCurlHeaders, "Pragma:");
 	
-	if (mReqOffset || mReqLength)
+	if ((mReqOffset || mReqLength) && HOR_GET == mReqMethod)
 	{
 		static const char * fmt1("Range: bytes=%d-%d");
 		static const char * fmt2("Range: bytes=%d-");
@@ -347,6 +430,32 @@ size_t HttpOpRequest::writeCallback(void * data, size_t size, size_t nmemb, void
 }
 
 		
+size_t HttpOpRequest::readCallback(void * data, size_t size, size_t nmemb, void * userdata)
+{
+	CURL * handle(static_cast<CURL *>(userdata));
+	HttpOpRequest * op(NULL);
+	curl_easy_getinfo(handle, CURLINFO_PRIVATE, &op);
+	// *FIXME:  check the pointer
+
+	if (! op->mReqBody)
+	{
+		return 0;
+	}
+	const size_t req_size(size * nmemb);
+	const size_t body_size(op->mReqBody->size());
+	if (body_size <= op->mCurlBodyPos)
+	{
+		// *FIXME:  should probably log this event - unexplained
+		return 0;
+	}
+
+	const size_t do_size((std::min)(req_size, body_size - op->mCurlBodyPos));
+	op->mReqBody->read(static_cast<char *>(data), do_size);
+	op->mCurlBodyPos += do_size;
+	return do_size;
+}
+
+		
 size_t HttpOpRequest::headerCallback(void * data, size_t size, size_t nmemb, void * userdata)
 {
 	static const char status_line[] = "HTTP/";
-- 
cgit v1.2.3


From 7b9da4eeda7505162f37cbfa52591f7adff032e7 Mon Sep 17 00:00:00 2001
From: Monty Brandenberg <monty@lindenlab.com>
Date: Fri, 1 Jun 2012 17:23:51 -0400
Subject: Missed two instances of priority typed as 'float'.  Became an excuse
 to go through an use a typedef for priority and policy class id.

---
 indra/llcorehttp/_httpoprequest.cpp | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

(limited to 'indra/llcorehttp/_httpoprequest.cpp')

diff --git a/indra/llcorehttp/_httpoprequest.cpp b/indra/llcorehttp/_httpoprequest.cpp
index 54b9990057..791ee5f860 100644
--- a/indra/llcorehttp/_httpoprequest.cpp
+++ b/indra/llcorehttp/_httpoprequest.cpp
@@ -233,8 +233,8 @@ HttpStatus HttpOpRequest::cancel()
 }
 
 
-HttpStatus HttpOpRequest::setupGetByteRange(unsigned int policy_id,
-											float priority,
+HttpStatus HttpOpRequest::setupGetByteRange(HttpRequest::policy_t policy_id,
+											HttpRequest::priority_t priority,
 											const std::string & url,
 											size_t offset,
 											size_t len,
@@ -268,8 +268,8 @@ HttpStatus HttpOpRequest::setupGetByteRange(unsigned int policy_id,
 }
 
 
-HttpStatus HttpOpRequest::setupPost(unsigned int policy_id,
-									float priority,
+HttpStatus HttpOpRequest::setupPost(HttpRequest::policy_t policy_id,
+									HttpRequest::priority_t priority,
 									const std::string & url,
 									BufferArray * body,
 									HttpOptions * options,
-- 
cgit v1.2.3


From 4155301015525a242a79b9b3134e66d366bc0ebd Mon Sep 17 00:00:00 2001
From: Monty Brandenberg <monty@lindenlab.com>
Date: Fri, 1 Jun 2012 21:30:45 -0400
Subject: Do some work on BufferArray to make it a bit less naive about
 chunking data.  Remove the stateful use of a seek pointer so that shared read
 is possible (though maybe not interesting).

---
 indra/llcorehttp/_httpoprequest.cpp | 14 +++++---------
 1 file changed, 5 insertions(+), 9 deletions(-)

(limited to 'indra/llcorehttp/_httpoprequest.cpp')

diff --git a/indra/llcorehttp/_httpoprequest.cpp b/indra/llcorehttp/_httpoprequest.cpp
index 791ee5f860..196011f953 100644
--- a/indra/llcorehttp/_httpoprequest.cpp
+++ b/indra/llcorehttp/_httpoprequest.cpp
@@ -340,7 +340,6 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service)
 			long data_size(0);
 			if (mReqBody)
 			{
-				mReqBody->seek(0);
 				data_size = mReqBody->size();
 			}
 			curl_easy_setopt(mCurlHandle, CURLOPT_POSTFIELDS, static_cast<void *>(NULL));
@@ -356,7 +355,6 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service)
 			long data_size(0);
 			if (mReqBody)
 			{
-				mReqBody->seek(0);
 				data_size = mReqBody->size();
 			}
 			curl_easy_setopt(mCurlHandle, CURLOPT_INFILESIZE, data_size);
@@ -423,10 +421,8 @@ size_t HttpOpRequest::writeCallback(void * data, size_t size, size_t nmemb, void
 		op->mReplyBody = new BufferArray();
 	}
 	const size_t req_size(size * nmemb);
-	char * lump(op->mReplyBody->appendBufferAlloc(req_size));
-	memcpy(lump, data, req_size);
-
-	return req_size;
+	const size_t write_size(op->mReplyBody->append(static_cast<char *>(data), req_size));
+	return write_size;
 }
 
 		
@@ -450,9 +446,9 @@ size_t HttpOpRequest::readCallback(void * data, size_t size, size_t nmemb, void
 	}
 
 	const size_t do_size((std::min)(req_size, body_size - op->mCurlBodyPos));
-	op->mReqBody->read(static_cast<char *>(data), do_size);
-	op->mCurlBodyPos += do_size;
-	return do_size;
+	const size_t read_size(op->mReqBody->read(op->mCurlBodyPos, static_cast<char *>(data), do_size));
+	op->mCurlBodyPos += read_size;
+	return read_size;
 }
 
 		
-- 
cgit v1.2.3


From 6c6d1c8338b15828278d27912bb9fe3b0d133b12 Mon Sep 17 00:00:00 2001
From: Monty Brandenberg <monty@lindenlab.com>
Date: Tue, 5 Jun 2012 16:49:39 -0400
Subject: Format/data type mismatch lead to a 'Range: bytes=0-0' header which
 gave me 1 byte of data. Shouldn't be making that kind of mistake.

---
 indra/llcorehttp/_httpoprequest.cpp | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

(limited to 'indra/llcorehttp/_httpoprequest.cpp')

diff --git a/indra/llcorehttp/_httpoprequest.cpp b/indra/llcorehttp/_httpoprequest.cpp
index 196011f953..f52ff5a44c 100644
--- a/indra/llcorehttp/_httpoprequest.cpp
+++ b/indra/llcorehttp/_httpoprequest.cpp
@@ -376,19 +376,19 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service)
 	
 	if ((mReqOffset || mReqLength) && HOR_GET == mReqMethod)
 	{
-		static const char * fmt1("Range: bytes=%d-%d");
-		static const char * fmt2("Range: bytes=%d-");
+		static const char * const fmt1("Range: bytes=%lu-%lu");
+		static const char * const fmt2("Range: bytes=%lu-");
 
 		char range_line[64];
 
 #if defined(WIN32)
 		_snprintf_s(range_line, sizeof(range_line), sizeof(range_line) - 1,
 					(mReqLength ? fmt1 : fmt2),
-					mReqOffset, mReqOffset + mReqLength - 1);
+					(unsigned long) mReqOffset, (unsigned long) (mReqOffset + mReqLength - 1));
 #else
 		snprintf(range_line, sizeof(range_line),
 				 (mReqLength ? fmt1 : fmt2),
-				 mReqOffset, mReqOffset + mReqLength - 1);
+				 (unsigned long) mReqOffset, (unsigned long) (mReqOffset + mReqLength - 1));
 #endif // defined(WIN32)
 		range_line[sizeof(range_line) - 1] = '\0';
 		mCurlHeaders = curl_slist_append(mCurlHeaders, range_line);
-- 
cgit v1.2.3


From 05af16a23abe37210e0b880aa27387d8994419dd Mon Sep 17 00:00:00 2001
From: Monty Brandenberg <monty@lindenlab.com>
Date: Wed, 6 Jun 2012 13:52:38 -0400
Subject: Policy + caching fixes + https support + POST working Implemented
 first global policy definitions to support SSL CA certificate configuration
 to support https: operations.  Fixed HTTP 206 status handling to match what
 is currently being done by grid services and to lay a foundation for fixes
 that will be a response to ER-1824.  More libcurl CURLOPT options set on easy
 handles to do peer verification in the traditional way.  HTTP POST working
 and now reporting asset metrics back to grid for the viewer's asset system. 
 This uses LLSD so that is also showing as compatible with the new library.

---
 indra/llcorehttp/_httpoprequest.cpp | 45 ++++++++++++++++++++++++++++++++-----
 1 file changed, 40 insertions(+), 5 deletions(-)

(limited to 'indra/llcorehttp/_httpoprequest.cpp')

diff --git a/indra/llcorehttp/_httpoprequest.cpp b/indra/llcorehttp/_httpoprequest.cpp
index f52ff5a44c..4bdc4a5257 100644
--- a/indra/llcorehttp/_httpoprequest.cpp
+++ b/indra/llcorehttp/_httpoprequest.cpp
@@ -40,8 +40,10 @@
 #include "_httpreplyqueue.h"
 #include "_httpservice.h"
 #include "_httppolicy.h"
+#include "_httppolicyglobal.h"
 #include "_httplibcurl.h"
 
+#include "llhttpstatuscodes.h"
 
 namespace
 {
@@ -153,14 +155,14 @@ HttpOpRequest::~HttpOpRequest()
 void HttpOpRequest::stageFromRequest(HttpService * service)
 {
 	addRef();
-	service->getPolicy()->addOp(this);			// transfers refcount
+	service->getPolicy().addOp(this);			// transfers refcount
 }
 
 
 void HttpOpRequest::stageFromReady(HttpService * service)
 {
 	addRef();
-	service->getTransport()->addOp(this);		// transfers refcount
+	service->getTransport().addOp(this);		// transfers refcount
 }
 
 
@@ -195,6 +197,8 @@ void HttpOpRequest::stageFromActive(HttpService * service)
 
 void HttpOpRequest::visitNotifier(HttpRequest * request)
 {
+	static const HttpStatus partial_content(HTTP_PARTIAL_CONTENT, HE_SUCCESS);
+	
 	if (mLibraryHandler)
 	{
 		HttpResponse * response = new HttpResponse();
@@ -208,9 +212,15 @@ void HttpOpRequest::visitNotifier(HttpRequest * request)
 			offset = mReplyOffset;
 			length = mReplyLength;
 		}
-		else if (mReplyBody)
+		else if (mReplyBody && partial_content == mStatus)
 		{
-			// Provide implicit offset/length from request/response
+			// Legacy grid services did not provide a 'Content-Range'
+			// header in responses to full- or partly-satisfyiable
+			// 'Range' requests.  For these, we have to hope that
+			// the data starts where requested and the length is simply
+			// whatever we received.  A bit of sanity could be provided
+			// by overlapping ranged requests and verifying that the
+			// overlap matches.
 			offset = mReqOffset;
 			length = mReplyBody->size();
 		}
@@ -306,6 +316,9 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service)
 	// *FIXME:  better error handling later
 	HttpStatus status;
 
+	// Get policy options
+	HttpPolicyGlobal & policy(service->getPolicy().getGlobalOptions());
+	
 	mCurlHandle = curl_easy_init();
 	// curl_easy_setopt(mCurlHandle, CURLOPT_VERBOSE, 1);
 	curl_easy_setopt(mCurlHandle, CURLOPT_TIMEOUT, 30);
@@ -322,21 +335,40 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service)
 	curl_easy_setopt(mCurlHandle, CURLOPT_DNS_CACHE_TIMEOUT, 0);
 	curl_easy_setopt(mCurlHandle, CURLOPT_AUTOREFERER, 1);
 	curl_easy_setopt(mCurlHandle, CURLOPT_FOLLOWLOCATION, 1);
-	curl_easy_setopt(mCurlHandle, CURLOPT_MAXREDIRS, 10);
+	curl_easy_setopt(mCurlHandle, CURLOPT_MAXREDIRS, 10);		// *FIXME:  parameterize this later
 	curl_easy_setopt(mCurlHandle, CURLOPT_WRITEFUNCTION, writeCallback);
 	curl_easy_setopt(mCurlHandle, CURLOPT_WRITEDATA, mCurlHandle);
 	curl_easy_setopt(mCurlHandle, CURLOPT_READFUNCTION, readCallback);
 	curl_easy_setopt(mCurlHandle, CURLOPT_READDATA, mCurlHandle);
+	curl_easy_setopt(mCurlHandle, CURLOPT_SSL_VERIFYPEER, 1);
+	curl_easy_setopt(mCurlHandle, CURLOPT_SSL_VERIFYHOST, 0);
 
+	std::string opt_value;
+	if (policy.get(HttpRequest::GP_CA_PATH, opt_value))
+	{
+		curl_easy_setopt(mCurlHandle, CURLOPT_CAPATH, opt_value.c_str());
+	}
+	if (policy.get(HttpRequest::GP_CA_FILE, opt_value))
+	{
+		curl_easy_setopt(mCurlHandle, CURLOPT_CAINFO, opt_value.c_str());
+	}
+	if (policy.get(HttpRequest::GP_HTTP_PROXY, opt_value))
+	{
+		curl_easy_setopt(mCurlHandle, CURLOPT_PROXY, opt_value.c_str());
+	}
+	
 	switch (mReqMethod)
 	{
 	case HOR_GET:
 		curl_easy_setopt(mCurlHandle, CURLOPT_HTTPGET, 1);
+		mCurlHeaders = curl_slist_append(mCurlHeaders, "Connection: keep-alive");
+		mCurlHeaders = curl_slist_append(mCurlHeaders, "Keep-alive: 300");
 		break;
 		
 	case HOR_POST:
 		{
 			curl_easy_setopt(mCurlHandle, CURLOPT_POST, 1);
+			curl_easy_setopt(mCurlHandle, CURLOPT_ENCODING, "");
 			long data_size(0);
 			if (mReqBody)
 			{
@@ -358,8 +390,11 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service)
 				data_size = mReqBody->size();
 			}
 			curl_easy_setopt(mCurlHandle, CURLOPT_INFILESIZE, data_size);
+			curl_easy_setopt(mCurlHandle, CURLOPT_POSTFIELDS, (void *) NULL);
 			mCurlHeaders = curl_slist_append(mCurlHeaders, "Transfer-Encoding: chunked");
 			mCurlHeaders = curl_slist_append(mCurlHeaders, "Expect:");
+			mCurlHeaders = curl_slist_append(mCurlHeaders, "Connection: keep-alive");
+			mCurlHeaders = curl_slist_append(mCurlHeaders, "Keep-alive: 300");
 		}
 		break;
 		
-- 
cgit v1.2.3


From 28a04400b4160dd34166483ddcf0c12637bcc363 Mon Sep 17 00:00:00 2001
From: Monty Brandenberg <monty@lindenlab.com>
Date: Fri, 8 Jun 2012 20:21:54 -0400
Subject: Implemented HTTP retry for requests.  Went in rather easily which
 surprised me.  Added a retry queue similar to ready queue to the policy
 object which is sorted by retry time.  Currently do five retries (after the
 initial try) delayed by .25, .5, 1, 2 and 5 seconds.  Removed the retry logic
 from the lltexturefetch module. Upped the waiting time in the unit test for
 the retries.  People won't like this but tough, need tests.

---
 indra/llcorehttp/_httpoprequest.cpp | 51 ++++++++++++++++++++++++++++++-------
 1 file changed, 42 insertions(+), 9 deletions(-)

(limited to 'indra/llcorehttp/_httpoprequest.cpp')

diff --git a/indra/llcorehttp/_httpoprequest.cpp b/indra/llcorehttp/_httpoprequest.cpp
index 4bdc4a5257..895629c514 100644
--- a/indra/llcorehttp/_httpoprequest.cpp
+++ b/indra/llcorehttp/_httpoprequest.cpp
@@ -99,8 +99,15 @@ HttpOpRequest::HttpOpRequest()
 	  mReplyBody(NULL),
 	  mReplyOffset(0),
 	  mReplyLength(0),
-	  mReplyHeaders(NULL)
-{}
+	  mReplyHeaders(NULL),
+	  mPolicyRetries(0),
+	  mPolicyRetryAt(HttpTime(0)),
+	  mPolicyRetryLimit(5)				// *FIXME:  Get from policy definitions
+{
+	// *NOTE:  As members are added, retry initialization/cleanup
+	// may need to be extended in @prepareRequest().
+}
+
 
 
 HttpOpRequest::~HttpOpRequest()
@@ -130,7 +137,6 @@ HttpOpRequest::~HttpOpRequest()
 	}
 
 	mCurlService = NULL;
-	
 
 	if (mCurlHeaders)
 	{
@@ -313,6 +319,30 @@ HttpStatus HttpOpRequest::setupPost(HttpRequest::policy_t policy_id,
 
 HttpStatus HttpOpRequest::prepareRequest(HttpService * service)
 {
+	// Scrub transport and result data for retried op case
+	mCurlActive = false;
+	mCurlHandle = NULL;
+	mCurlService = NULL;
+	if (mCurlHeaders)
+	{
+		curl_slist_free_all(mCurlHeaders);
+		mCurlHeaders = NULL;
+	}
+	mCurlBodyPos = 0;
+
+	if (mReplyBody)
+	{
+		mReplyBody->release();
+		mReplyBody = NULL;
+	}
+	mReplyOffset = 0;
+	mReplyLength = 0;
+	if (mReplyHeaders)
+	{
+		mReplyHeaders->release();
+		mReplyHeaders = NULL;
+	}
+	
 	// *FIXME:  better error handling later
 	HttpStatus status;
 
@@ -321,6 +351,7 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service)
 	
 	mCurlHandle = curl_easy_init();
 	// curl_easy_setopt(mCurlHandle, CURLOPT_VERBOSE, 1);
+	curl_easy_setopt(mCurlHandle, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4);
 	curl_easy_setopt(mCurlHandle, CURLOPT_TIMEOUT, 30);
 	curl_easy_setopt(mCurlHandle, CURLOPT_CONNECTTIMEOUT, 30);
 	curl_easy_setopt(mCurlHandle, CURLOPT_NOSIGNAL, 1);
@@ -403,12 +434,7 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service)
 		break;
 	}
 	
-	if (mReqHeaders)
-	{
-		mCurlHeaders = append_headers_to_slist(mReqHeaders, mCurlHeaders);
-	}
-	mCurlHeaders = curl_slist_append(mCurlHeaders, "Pragma:");
-	
+	// There's a CURLOPT for this now...
 	if ((mReqOffset || mReqLength) && HOR_GET == mReqMethod)
 	{
 		static const char * const fmt1("Range: bytes=%lu-%lu");
@@ -428,6 +454,13 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service)
 		range_line[sizeof(range_line) - 1] = '\0';
 		mCurlHeaders = curl_slist_append(mCurlHeaders, range_line);
 	}
+
+	mCurlHeaders = curl_slist_append(mCurlHeaders, "Pragma:");
+	if (mReqHeaders)
+	{
+		// Caller's headers last to override
+		mCurlHeaders = append_headers_to_slist(mReqHeaders, mCurlHeaders);
+	}
 	curl_easy_setopt(mCurlHandle, CURLOPT_HTTPHEADER, mCurlHeaders);
 	
 	if (mProcFlags & (PF_SCAN_RANGE_HEADER | PF_SAVE_HEADERS))
-- 
cgit v1.2.3


From 89187229dd630845177ecd7a16e2b9cb01dc47ce Mon Sep 17 00:00:00 2001
From: Monty Brandenberg <monty@lindenlab.com>
Date: Mon, 11 Jun 2012 15:28:06 -0400
Subject: Refactoring of the request completion thread and removal of
 206/content-range hack in xport. Retry/response handling is decided in policy
 so moved that there.  Removed special case 206-without-content-range response
 in transport.  Have this sitation recognizable in the API and let callers
 deal with it as needed.

---
 indra/llcorehttp/_httpoprequest.cpp | 19 +++----------------
 1 file changed, 3 insertions(+), 16 deletions(-)

(limited to 'indra/llcorehttp/_httpoprequest.cpp')

diff --git a/indra/llcorehttp/_httpoprequest.cpp b/indra/llcorehttp/_httpoprequest.cpp
index 895629c514..ea0b99303e 100644
--- a/indra/llcorehttp/_httpoprequest.cpp
+++ b/indra/llcorehttp/_httpoprequest.cpp
@@ -144,6 +144,8 @@ HttpOpRequest::~HttpOpRequest()
 		mCurlHeaders = NULL;
 	}
 
+	mReplyOffset = 0;
+	mReplyLength = 0;
 	if (mReplyBody)
 	{
 		mReplyBody->release();
@@ -211,26 +213,11 @@ void HttpOpRequest::visitNotifier(HttpRequest * request)
 		response->setStatus(mStatus);
 		response->setBody(mReplyBody);
 		response->setHeaders(mReplyHeaders);
-		unsigned int offset(0), length(0);
 		if (mReplyOffset || mReplyLength)
 		{
 			// Got an explicit offset/length in response
-			offset = mReplyOffset;
-			length = mReplyLength;
-		}
-		else if (mReplyBody && partial_content == mStatus)
-		{
-			// Legacy grid services did not provide a 'Content-Range'
-			// header in responses to full- or partly-satisfyiable
-			// 'Range' requests.  For these, we have to hope that
-			// the data starts where requested and the length is simply
-			// whatever we received.  A bit of sanity could be provided
-			// by overlapping ranged requests and verifying that the
-			// overlap matches.
-			offset = mReqOffset;
-			length = mReplyBody->size();
+			response->setRange(mReplyOffset, mReplyLength);
 		}
-		response->setRange(offset, length);
 
 		mLibraryHandler->onCompleted(static_cast<HttpHandle>(this), response);
 
-- 
cgit v1.2.3


From 7adeb3923728ca84a309a6af141c148ce38066fc Mon Sep 17 00:00:00 2001
From: Monty Brandenberg <monty@lindenlab.com>
Date: Tue, 12 Jun 2012 17:42:33 -0400
Subject: HTTP Proxy, PUT & POST, unit tests and refactoring.
 Implemented/modified PUT & POST to not used chunked encoding for the request.
 Made the unit test much happier and probably a better thing for the pipeline.
 Have a cheesy static & dynamic proxy capability using both local options and
 a way to wire into LLProxy in llmessages.  Not a clean thing but it will get
 the proxy path working with both socks5 & http proxies.  Refactoring to get
 rid of unneeded library handler and unified an HttpStatus return for all
 requests.  Big batch of code removed as a result of that and more is possible
 as well as some syscall avoidance with a bit more work.  Boosted the unit
 tests for simple PUT & POST test which revealed the test harness does *not*
 like chunked encoding so we'll avoid it for now (and don't really need it in
 any of our schemes).

---
 indra/llcorehttp/_httpoprequest.cpp | 63 +++++++++++++++++++++++++++++++------
 1 file changed, 53 insertions(+), 10 deletions(-)

(limited to 'indra/llcorehttp/_httpoprequest.cpp')

diff --git a/indra/llcorehttp/_httpoprequest.cpp b/indra/llcorehttp/_httpoprequest.cpp
index ea0b99303e..e2550d057e 100644
--- a/indra/llcorehttp/_httpoprequest.cpp
+++ b/indra/llcorehttp/_httpoprequest.cpp
@@ -44,6 +44,7 @@
 #include "_httplibcurl.h"
 
 #include "llhttpstatuscodes.h"
+#include "llproxy.h"
 
 namespace
 {
@@ -207,7 +208,7 @@ void HttpOpRequest::visitNotifier(HttpRequest * request)
 {
 	static const HttpStatus partial_content(HTTP_PARTIAL_CONTENT, HE_SUCCESS);
 	
-	if (mLibraryHandler)
+	if (mUserHandler)
 	{
 		HttpResponse * response = new HttpResponse();
 		response->setStatus(mStatus);
@@ -219,7 +220,7 @@ void HttpOpRequest::visitNotifier(HttpRequest * request)
 			response->setRange(mReplyOffset, mReplyLength);
 		}
 
-		mLibraryHandler->onCompleted(static_cast<HttpHandle>(this), response);
+		mUserHandler->onCompleted(static_cast<HttpHandle>(this), response);
 
 		response->release();
 	}
@@ -304,6 +305,39 @@ HttpStatus HttpOpRequest::setupPost(HttpRequest::policy_t policy_id,
 }
 
 
+HttpStatus HttpOpRequest::setupPut(HttpRequest::policy_t policy_id,
+								   HttpRequest::priority_t priority,
+								   const std::string & url,
+								   BufferArray * body,
+								   HttpOptions * options,
+								   HttpHeaders * headers)
+{
+	HttpStatus status;
+
+	mProcFlags = 0;
+	mReqPolicy = policy_id;
+	mReqPriority = priority;
+	mReqMethod = HOR_PUT;
+	mReqURL = url;
+	if (body)
+	{
+		body->addRef();
+		mReqBody = body;
+	}
+	if (headers && ! mReqHeaders)
+	{
+		headers->addRef();
+		mReqHeaders = headers;
+	}
+	if (options && ! mReqOptions)
+	{
+		mReqOptions = new HttpOptions(*options);
+	}
+	
+	return status;
+}
+
+
 HttpStatus HttpOpRequest::prepareRequest(HttpService * service)
 {
 	// Scrub transport and result data for retried op case
@@ -346,8 +380,6 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service)
 	curl_easy_setopt(mCurlHandle, CURLOPT_URL, mReqURL.c_str());
 	curl_easy_setopt(mCurlHandle, CURLOPT_PRIVATE, this);
 	curl_easy_setopt(mCurlHandle, CURLOPT_ENCODING, "");
-	// *FIXME:  Need to deal with proxy setup...
-	// curl_easy_setopt(handle, CURLOPT_PROXY, "");
 
 	// *FIXME:  Revisit this old DNS timeout setting - may no longer be valid
 	curl_easy_setopt(mCurlHandle, CURLOPT_DNS_CACHE_TIMEOUT, 0);
@@ -361,18 +393,31 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service)
 	curl_easy_setopt(mCurlHandle, CURLOPT_SSL_VERIFYPEER, 1);
 	curl_easy_setopt(mCurlHandle, CURLOPT_SSL_VERIFYHOST, 0);
 
-	std::string opt_value;
+	const std::string * opt_value(NULL);
 	if (policy.get(HttpRequest::GP_CA_PATH, opt_value))
 	{
-		curl_easy_setopt(mCurlHandle, CURLOPT_CAPATH, opt_value.c_str());
+		curl_easy_setopt(mCurlHandle, CURLOPT_CAPATH, opt_value->c_str());
 	}
 	if (policy.get(HttpRequest::GP_CA_FILE, opt_value))
 	{
-		curl_easy_setopt(mCurlHandle, CURLOPT_CAINFO, opt_value.c_str());
+		curl_easy_setopt(mCurlHandle, CURLOPT_CAINFO, opt_value->c_str());
 	}
 	if (policy.get(HttpRequest::GP_HTTP_PROXY, opt_value))
 	{
-		curl_easy_setopt(mCurlHandle, CURLOPT_PROXY, opt_value.c_str());
+		if (*opt_value == "LLProxy")
+		{
+			// Use the viewer-based thread-safe API which has a
+			// fast/safe check for proxy enable.  Would like to
+			// encapsulate this someway...
+			LLProxy::getInstance()->applyProxySettings(mCurlHandle);
+		}
+		else
+		{
+			// *TODO:  This is fine for now but get fuller socks/
+			// authentication thing going later....
+			curl_easy_setopt(mCurlHandle, CURLOPT_PROXY, opt_value->c_str());
+			curl_easy_setopt(mCurlHandle, CURLOPT_PROXYTYPE, CURLPROXY_HTTP);
+		}
 	}
 	
 	switch (mReqMethod)
@@ -394,7 +439,6 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service)
 			}
 			curl_easy_setopt(mCurlHandle, CURLOPT_POSTFIELDS, static_cast<void *>(NULL));
 			curl_easy_setopt(mCurlHandle, CURLOPT_POSTFIELDSIZE, data_size);
-			mCurlHeaders = curl_slist_append(mCurlHeaders, "Transfer-Encoding: chunked");
 			mCurlHeaders = curl_slist_append(mCurlHeaders, "Expect:");
 		}
 		break;
@@ -409,7 +453,6 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service)
 			}
 			curl_easy_setopt(mCurlHandle, CURLOPT_INFILESIZE, data_size);
 			curl_easy_setopt(mCurlHandle, CURLOPT_POSTFIELDS, (void *) NULL);
-			mCurlHeaders = curl_slist_append(mCurlHeaders, "Transfer-Encoding: chunked");
 			mCurlHeaders = curl_slist_append(mCurlHeaders, "Expect:");
 			mCurlHeaders = curl_slist_append(mCurlHeaders, "Connection: keep-alive");
 			mCurlHeaders = curl_slist_append(mCurlHeaders, "Keep-alive: 300");
-- 
cgit v1.2.3


From b08125a5874a89ce5210f8fb2c961ae17fb80fde Mon Sep 17 00:00:00 2001
From: Monty Brandenberg <monty@lindenlab.com>
Date: Thu, 14 Jun 2012 16:31:48 -0400
Subject: LLMutex recursive lock, global & per-request tracing, simple GET
 request, LLProxy support, HttpOptions starting to work, HTTP resource waiting
 fixed. Non-LLThread-based threads need to do some registration or LLMutex
 locks taken out in these threads will not work as expected (SH-3154).  We'll
 get a better solution later, this fixes some things for now.  Tracing of
 operations now supported.  Global and per-request (via HttpOptions) tracing
 levels of [0..3].  The 2 and 3 levels use libcurl's VERBOSE mode combined
 with CURLOPT_DEBUGFUNCTION to stream high levels of detail into the log. 
 *Very* laggy but useful.  Simple GET request supported (no Range: header). 
 Really just a degenrate case of a ranged get but supplied an API anyway. 
 Global option to use the LLProxy interface to setup CURL handles for either
 socks5 or http proxy usage.  This isn't really the most encapsulated way to
 do this but a better solution will have to come later.  The wantHeaders and
 tracing options are now supported in HttpOptions giving per-request controls.
  Big refactoring of the HTTP resource waiter in lltexturefetch. What I was
 doing before wasn't correct.  Instead, I'm implementing the resource wait
 after the Semaphore model (though not using system semaphores).  So instead
 of having a sequence like:  SEND_HTTP_REQ -> WAIT_HTTP_RESOURCE ->
 SEND_HTTP_REQ, we now do WAIT_HTTP_RESOURCE -> WAIT_HTTP_RESOURCE2 (actual
 wait) -> SEND_HTTP_REQ.  Works well but the prioritized filling of the
 corehttp library needs some performance work later.

---
 indra/llcorehttp/_httpoprequest.cpp | 271 +++++++++++++++++++++++++++---------
 1 file changed, 205 insertions(+), 66 deletions(-)

(limited to 'indra/llcorehttp/_httpoprequest.cpp')

diff --git a/indra/llcorehttp/_httpoprequest.cpp b/indra/llcorehttp/_httpoprequest.cpp
index e2550d057e..f78971d8f2 100644
--- a/indra/llcorehttp/_httpoprequest.cpp
+++ b/indra/llcorehttp/_httpoprequest.cpp
@@ -63,6 +63,16 @@ int parse_content_range_header(char * buffer,
 							   unsigned int * last,
 							   unsigned int * length);
 
+
+// Take data from libcurl's CURLOPT_DEBUGFUNCTION callback and
+// escape and format it for a tracing line in logging.  Absolutely
+// anything including NULs can be in the data.  If @scrub is true,
+// non-printing or non-ascii characters are replaced with spaces
+// otherwise a %XX form of escaping is used.
+void escape_libcurl_debug_data(char * buffer, size_t len, bool scrub,
+							   std::string & safe_line);
+
+
 #if defined(WIN32)
 
 // Not available on windows where the legacy strtok interface
@@ -78,11 +88,6 @@ namespace LLCore
 {
 
 
-// ==================================
-// HttpOpRequest
-// ==================================
-
-
 HttpOpRequest::HttpOpRequest()
 	: HttpOperation(),
 	  mProcFlags(0U),
@@ -237,6 +242,19 @@ HttpStatus HttpOpRequest::cancel()
 }
 
 
+HttpStatus HttpOpRequest::setupGet(HttpRequest::policy_t policy_id,
+								   HttpRequest::priority_t priority,
+								   const std::string & url,
+								   HttpOptions * options,
+								   HttpHeaders * headers)
+{
+	setupCommon(policy_id, priority, url, NULL, options, headers);
+	mReqMethod = HOR_GET;
+	
+	return HttpStatus();
+}
+
+
 HttpStatus HttpOpRequest::setupGetByteRange(HttpRequest::policy_t policy_id,
 											HttpRequest::priority_t priority,
 											const std::string & url,
@@ -245,30 +263,16 @@ HttpStatus HttpOpRequest::setupGetByteRange(HttpRequest::policy_t policy_id,
 											HttpOptions * options,
 											HttpHeaders * headers)
 {
-	HttpStatus status;
-
-	mProcFlags = 0;
-	mReqPolicy = policy_id;
-	mReqPriority = priority;
+	setupCommon(policy_id, priority, url, NULL, options, headers);
 	mReqMethod = HOR_GET;
-	mReqURL = url;
 	mReqOffset = offset;
 	mReqLength = len;
 	if (offset || len)
 	{
 		mProcFlags |= PF_SCAN_RANGE_HEADER;
 	}
-	if (headers && ! mReqHeaders)
-	{
-		headers->addRef();
-		mReqHeaders = headers;
-	}
-	if (options && ! mReqOptions)
-	{
-		mReqOptions = new HttpOptions(*options);
-	}
 	
-	return status;
+	return HttpStatus();
 }
 
 
@@ -279,29 +283,10 @@ HttpStatus HttpOpRequest::setupPost(HttpRequest::policy_t policy_id,
 									HttpOptions * options,
 									HttpHeaders * headers)
 {
-	HttpStatus status;
-
-	mProcFlags = 0;
-	mReqPolicy = policy_id;
-	mReqPriority = priority;
+	setupCommon(policy_id, priority, url, body, options, headers);
 	mReqMethod = HOR_POST;
-	mReqURL = url;
-	if (body)
-	{
-		body->addRef();
-		mReqBody = body;
-	}
-	if (headers && ! mReqHeaders)
-	{
-		headers->addRef();
-		mReqHeaders = headers;
-	}
-	if (options && ! mReqOptions)
-	{
-		mReqOptions = new HttpOptions(*options);
-	}
 	
-	return status;
+	return HttpStatus();
 }
 
 
@@ -312,12 +297,23 @@ HttpStatus HttpOpRequest::setupPut(HttpRequest::policy_t policy_id,
 								   HttpOptions * options,
 								   HttpHeaders * headers)
 {
-	HttpStatus status;
+	setupCommon(policy_id, priority, url, body, options, headers);
+	mReqMethod = HOR_PUT;
+	
+	return HttpStatus();
+}
 
+
+void HttpOpRequest::setupCommon(HttpRequest::policy_t policy_id,
+								HttpRequest::priority_t priority,
+								const std::string & url,
+								BufferArray * body,
+								HttpOptions * options,
+								HttpHeaders * headers)
+{
 	mProcFlags = 0;
 	mReqPolicy = policy_id;
 	mReqPriority = priority;
-	mReqMethod = HOR_PUT;
 	mReqURL = url;
 	if (body)
 	{
@@ -331,10 +327,14 @@ HttpStatus HttpOpRequest::setupPut(HttpRequest::policy_t policy_id,
 	}
 	if (options && ! mReqOptions)
 	{
-		mReqOptions = new HttpOptions(*options);
+		options->addRef();
+		mReqOptions = options;
+		if (options->getWantHeaders())
+		{
+			mProcFlags |= PF_SAVE_HEADERS;
+		}
+		mTracing = (std::max)(mTracing, llclamp(options->getTrace(), 0, 3));
 	}
-	
-	return status;
 }
 
 
@@ -394,30 +394,29 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service)
 	curl_easy_setopt(mCurlHandle, CURLOPT_SSL_VERIFYHOST, 0);
 
 	const std::string * opt_value(NULL);
-	if (policy.get(HttpRequest::GP_CA_PATH, opt_value))
+	long opt_long(0L);
+	policy.get(HttpRequest::GP_LLPROXY, &opt_long);
+	if (opt_long)
 	{
-		curl_easy_setopt(mCurlHandle, CURLOPT_CAPATH, opt_value->c_str());
+		// Use the viewer-based thread-safe API which has a
+		// fast/safe check for proxy enable.  Would like to
+		// encapsulate this someway...
+		LLProxy::getInstance()->applyProxySettings(mCurlHandle);
 	}
-	if (policy.get(HttpRequest::GP_CA_FILE, opt_value))
+	else if (policy.get(HttpRequest::GP_HTTP_PROXY, &opt_value))
 	{
-		curl_easy_setopt(mCurlHandle, CURLOPT_CAINFO, opt_value->c_str());
+		// *TODO:  This is fine for now but get fuller socks/
+		// authentication thing going later....
+		curl_easy_setopt(mCurlHandle, CURLOPT_PROXY, opt_value->c_str());
+		curl_easy_setopt(mCurlHandle, CURLOPT_PROXYTYPE, CURLPROXY_HTTP);
 	}
-	if (policy.get(HttpRequest::GP_HTTP_PROXY, opt_value))
+	if (policy.get(HttpRequest::GP_CA_PATH, &opt_value))
 	{
-		if (*opt_value == "LLProxy")
-		{
-			// Use the viewer-based thread-safe API which has a
-			// fast/safe check for proxy enable.  Would like to
-			// encapsulate this someway...
-			LLProxy::getInstance()->applyProxySettings(mCurlHandle);
-		}
-		else
-		{
-			// *TODO:  This is fine for now but get fuller socks/
-			// authentication thing going later....
-			curl_easy_setopt(mCurlHandle, CURLOPT_PROXY, opt_value->c_str());
-			curl_easy_setopt(mCurlHandle, CURLOPT_PROXYTYPE, CURLPROXY_HTTP);
-		}
+		curl_easy_setopt(mCurlHandle, CURLOPT_CAPATH, opt_value->c_str());
+	}
+	if (policy.get(HttpRequest::GP_CA_FILE, &opt_value))
+	{
+		curl_easy_setopt(mCurlHandle, CURLOPT_CAINFO, opt_value->c_str());
 	}
 	
 	switch (mReqMethod)
@@ -463,6 +462,14 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service)
 		// *FIXME:  fail out here
 		break;
 	}
+
+	// Tracing
+	if (mTracing > 1)
+	{
+		curl_easy_setopt(mCurlHandle, CURLOPT_VERBOSE, 1);
+		curl_easy_setopt(mCurlHandle, CURLOPT_DEBUGDATA, mCurlHandle);
+		curl_easy_setopt(mCurlHandle, CURLOPT_DEBUGFUNCTION, debugCallback);
+	}
 	
 	// There's a CURLOPT for this now...
 	if ((mReqOffset || mReqLength) && HOR_GET == mReqMethod)
@@ -621,6 +628,101 @@ size_t HttpOpRequest::headerCallback(void * data, size_t size, size_t nmemb, voi
 	return hdr_size;
 }
 
+
+int HttpOpRequest::debugCallback(CURL * handle, curl_infotype info, char * buffer, size_t len, void * userdata)
+{
+	HttpOpRequest * op(NULL);
+	curl_easy_getinfo(handle, CURLINFO_PRIVATE, &op);
+	// *FIXME:  check the pointer
+
+	std::string safe_line;
+	std::string tag;
+	bool logit(false);
+	len = (std::min)(len, size_t(256));					// Keep things reasonable in all cases
+	
+	switch (info)
+	{
+	case CURLINFO_TEXT:
+		if (op->mTracing > 1)
+		{
+			tag = "TEXT";
+			escape_libcurl_debug_data(buffer, len, true, safe_line);
+			logit = true;
+		}
+		break;
+			
+	case CURLINFO_HEADER_IN:
+		if (op->mTracing > 1)
+		{
+			tag = "HEADERIN";
+			escape_libcurl_debug_data(buffer, len, true, safe_line);
+			logit = true;
+		}
+		break;
+			
+	case CURLINFO_HEADER_OUT:
+		if (op->mTracing > 1)
+		{
+			tag = "HEADEROUT";
+			escape_libcurl_debug_data(buffer, len, true, safe_line);
+			logit = true;
+		}
+		break;
+			
+	case CURLINFO_DATA_IN:
+		if (op->mTracing > 1)
+		{
+			tag = "DATAIN";
+			logit = true;
+			if (op->mTracing > 2)
+			{
+				escape_libcurl_debug_data(buffer, len, false, safe_line);
+			}
+			else
+			{
+				std::ostringstream out;
+				out << len << " Bytes";
+				safe_line = out.str();
+			}
+		}
+		break;
+			
+	case CURLINFO_DATA_OUT:
+		if (op->mTracing > 1)
+		{
+			tag = "DATAOUT";
+			logit = true;
+			if (op->mTracing > 2)
+			{
+				escape_libcurl_debug_data(buffer, len, false, safe_line);
+			}
+			else
+			{
+				std::ostringstream out;
+				out << len << " Bytes";
+				safe_line = out.str();
+			}
+		}
+		break;
+			
+	default:
+		logit = false;
+		break;
+	}
+
+	if (logit)
+	{
+		LL_INFOS("CoreHttp") << "TRACE, LibcurlDebug, Handle:  "
+							 << static_cast<HttpHandle>(op)
+							 << ", Type:  " << tag
+							 << ", Data:  " << safe_line
+							 << LL_ENDL;
+	}
+		
+	return 0;
+}
+
+
 }   // end namespace LLCore
 
 
@@ -694,6 +796,43 @@ char *strtok_r(char *str, const char *delim, char ** savestate)
 
 #endif
 
+
+void escape_libcurl_debug_data(char * buffer, size_t len, bool scrub, std::string & safe_line)
+{
+	std::string out;
+	len = (std::min)(len, size_t(200));
+	out.reserve(3 * len);
+	for (int i(0); i < len; ++i)
+	{
+		unsigned char uc(static_cast<unsigned char>(buffer[i]));
+
+		if (uc < 32 || uc > 126)
+		{
+			if (scrub)
+			{
+				out.append(1, ' ');
+			}
+			else
+			{
+				static const char hex[] = "0123456789ABCDEF";
+				char convert[4];
+
+				convert[0] = '%';
+				convert[1] = hex[(uc >> 4) % 16];
+				convert[2] = hex[uc % 16];
+				convert[3] = '\0';
+				out.append(convert);
+			}
+		}
+		else
+		{
+			out.append(1, buffer[i]);
+		}
+	}
+	safe_line.swap(out);
+}
+
+
 }  // end anonymous namespace
 
 		
-- 
cgit v1.2.3


From f0353abe7605778048d69ce3acb8f5ddd5693083 Mon Sep 17 00:00:00 2001
From: Monty Brandenberg <monty@lindenlab.com>
Date: Tue, 19 Jun 2012 15:43:29 -0400
Subject: Implement timeout and retry count options for requests. Pretty
 straightforward.  Still don't like how I'm managing the options block. 
 Struct?  Accessors?  Can't decide.  But the options now speed up the unit
 test runs even as I add tests.

---
 indra/llcorehttp/_httpoprequest.cpp | 21 ++++++++++++++++-----
 1 file changed, 16 insertions(+), 5 deletions(-)

(limited to 'indra/llcorehttp/_httpoprequest.cpp')

diff --git a/indra/llcorehttp/_httpoprequest.cpp b/indra/llcorehttp/_httpoprequest.cpp
index f78971d8f2..ce41ebcce0 100644
--- a/indra/llcorehttp/_httpoprequest.cpp
+++ b/indra/llcorehttp/_httpoprequest.cpp
@@ -108,7 +108,7 @@ HttpOpRequest::HttpOpRequest()
 	  mReplyHeaders(NULL),
 	  mPolicyRetries(0),
 	  mPolicyRetryAt(HttpTime(0)),
-	  mPolicyRetryLimit(5)				// *FIXME:  Get from policy definitions
+	  mPolicyRetryLimit(5)
 {
 	// *NOTE:  As members are added, retry initialization/cleanup
 	// may need to be extended in @prepareRequest().
@@ -333,6 +333,8 @@ void HttpOpRequest::setupCommon(HttpRequest::policy_t policy_id,
 		{
 			mProcFlags |= PF_SAVE_HEADERS;
 		}
+		mPolicyRetryLimit = options->getRetries();
+		mPolicyRetryLimit = llclamp(mPolicyRetryLimit, 0, 100);
 		mTracing = (std::max)(mTracing, llclamp(options->getTrace(), 0, 3));
 	}
 }
@@ -371,10 +373,7 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service)
 	HttpPolicyGlobal & policy(service->getPolicy().getGlobalOptions());
 	
 	mCurlHandle = curl_easy_init();
-	// curl_easy_setopt(mCurlHandle, CURLOPT_VERBOSE, 1);
 	curl_easy_setopt(mCurlHandle, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4);
-	curl_easy_setopt(mCurlHandle, CURLOPT_TIMEOUT, 30);
-	curl_easy_setopt(mCurlHandle, CURLOPT_CONNECTTIMEOUT, 30);
 	curl_easy_setopt(mCurlHandle, CURLOPT_NOSIGNAL, 1);
 	curl_easy_setopt(mCurlHandle, CURLOPT_NOPROGRESS, 1);
 	curl_easy_setopt(mCurlHandle, CURLOPT_URL, mReqURL.c_str());
@@ -493,13 +492,25 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service)
 	}
 
 	mCurlHeaders = curl_slist_append(mCurlHeaders, "Pragma:");
+
+	// Request options
+	long timeout(30);
+	if (mReqOptions)
+	{
+		timeout = mReqOptions->getTimeout();
+		timeout = llclamp(timeout, 0L, 3600L);
+	}
+	curl_easy_setopt(mCurlHandle, CURLOPT_TIMEOUT, timeout);
+	curl_easy_setopt(mCurlHandle, CURLOPT_CONNECTTIMEOUT, timeout);
+
+	// Request headers
 	if (mReqHeaders)
 	{
 		// Caller's headers last to override
 		mCurlHeaders = append_headers_to_slist(mReqHeaders, mCurlHeaders);
 	}
 	curl_easy_setopt(mCurlHandle, CURLOPT_HTTPHEADER, mCurlHeaders);
-	
+
 	if (mProcFlags & (PF_SCAN_RANGE_HEADER | PF_SAVE_HEADERS))
 	{
 		curl_easy_setopt(mCurlHandle, CURLOPT_HEADERFUNCTION, headerCallback);
-- 
cgit v1.2.3


From a50944e078b98435685f04eda0ba93e95d4c61f2 Mon Sep 17 00:00:00 2001
From: Monty Brandenberg <monty@lindenlab.com>
Date: Tue, 19 Jun 2012 17:01:02 -0400
Subject: Cleanup:  move magic nubmers to new _httpinternal.h header file.

---
 indra/llcorehttp/_httpoprequest.cpp | 62 ++++++++++++++++++-------------------
 1 file changed, 31 insertions(+), 31 deletions(-)

(limited to 'indra/llcorehttp/_httpoprequest.cpp')

diff --git a/indra/llcorehttp/_httpoprequest.cpp b/indra/llcorehttp/_httpoprequest.cpp
index ce41ebcce0..04a8e0baff 100644
--- a/indra/llcorehttp/_httpoprequest.cpp
+++ b/indra/llcorehttp/_httpoprequest.cpp
@@ -42,6 +42,7 @@
 #include "_httppolicy.h"
 #include "_httppolicyglobal.h"
 #include "_httplibcurl.h"
+#include "_httpinternal.h"
 
 #include "llhttpstatuscodes.h"
 #include "llproxy.h"
@@ -73,13 +74,14 @@ void escape_libcurl_debug_data(char * buffer, size_t len, bool scrub,
 							   std::string & safe_line);
 
 
-#if defined(WIN32)
+#if LL_WINDOWS
 
 // Not available on windows where the legacy strtok interface
 // is thread-safe.
 char *strtok_r(char *str, const char *delim, char **saveptr);
 
-#endif
+#endif // LL_WINDOWS
+
 
 }
 
@@ -108,7 +110,7 @@ HttpOpRequest::HttpOpRequest()
 	  mReplyHeaders(NULL),
 	  mPolicyRetries(0),
 	  mPolicyRetryAt(HttpTime(0)),
-	  mPolicyRetryLimit(5)
+	  mPolicyRetryLimit(DEFAULT_RETRY_COUNT)
 {
 	// *NOTE:  As members are added, retry initialization/cleanup
 	// may need to be extended in @prepareRequest().
@@ -334,8 +336,8 @@ void HttpOpRequest::setupCommon(HttpRequest::policy_t policy_id,
 			mProcFlags |= PF_SAVE_HEADERS;
 		}
 		mPolicyRetryLimit = options->getRetries();
-		mPolicyRetryLimit = llclamp(mPolicyRetryLimit, 0, 100);
-		mTracing = (std::max)(mTracing, llclamp(options->getTrace(), 0, 3));
+		mPolicyRetryLimit = llclamp(mPolicyRetryLimit, LIMIT_RETRY_MIN, LIMIT_RETRY_MAX);
+		mTracing = (std::max)(mTracing, llclamp(options->getTrace(), TRACE_MIN, TRACE_MAX));
 	}
 }
 
@@ -384,7 +386,7 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service)
 	curl_easy_setopt(mCurlHandle, CURLOPT_DNS_CACHE_TIMEOUT, 0);
 	curl_easy_setopt(mCurlHandle, CURLOPT_AUTOREFERER, 1);
 	curl_easy_setopt(mCurlHandle, CURLOPT_FOLLOWLOCATION, 1);
-	curl_easy_setopt(mCurlHandle, CURLOPT_MAXREDIRS, 10);		// *FIXME:  parameterize this later
+	curl_easy_setopt(mCurlHandle, CURLOPT_MAXREDIRS, DEFAULT_HTTP_REDIRECTS);	// *FIXME:  parameterize this later
 	curl_easy_setopt(mCurlHandle, CURLOPT_WRITEFUNCTION, writeCallback);
 	curl_easy_setopt(mCurlHandle, CURLOPT_WRITEDATA, mCurlHandle);
 	curl_easy_setopt(mCurlHandle, CURLOPT_READFUNCTION, readCallback);
@@ -452,8 +454,6 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service)
 			curl_easy_setopt(mCurlHandle, CURLOPT_INFILESIZE, data_size);
 			curl_easy_setopt(mCurlHandle, CURLOPT_POSTFIELDS, (void *) NULL);
 			mCurlHeaders = curl_slist_append(mCurlHeaders, "Expect:");
-			mCurlHeaders = curl_slist_append(mCurlHeaders, "Connection: keep-alive");
-			mCurlHeaders = curl_slist_append(mCurlHeaders, "Keep-alive: 300");
 		}
 		break;
 		
@@ -463,7 +463,7 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service)
 	}
 
 	// Tracing
-	if (mTracing > 1)
+	if (mTracing >= TRACE_CURL_HEADERS)
 	{
 		curl_easy_setopt(mCurlHandle, CURLOPT_VERBOSE, 1);
 		curl_easy_setopt(mCurlHandle, CURLOPT_DEBUGDATA, mCurlHandle);
@@ -478,7 +478,7 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service)
 
 		char range_line[64];
 
-#if defined(WIN32)
+#if LL_WINDOWS
 		_snprintf_s(range_line, sizeof(range_line), sizeof(range_line) - 1,
 					(mReqLength ? fmt1 : fmt2),
 					(unsigned long) mReqOffset, (unsigned long) (mReqOffset + mReqLength - 1));
@@ -486,7 +486,7 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service)
 		snprintf(range_line, sizeof(range_line),
 				 (mReqLength ? fmt1 : fmt2),
 				 (unsigned long) mReqOffset, (unsigned long) (mReqOffset + mReqLength - 1));
-#endif // defined(WIN32)
+#endif // LL_WINDOWS
 		range_line[sizeof(range_line) - 1] = '\0';
 		mCurlHeaders = curl_slist_append(mCurlHeaders, range_line);
 	}
@@ -494,11 +494,11 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service)
 	mCurlHeaders = curl_slist_append(mCurlHeaders, "Pragma:");
 
 	// Request options
-	long timeout(30);
+	long timeout(DEFAULT_TIMEOUT);
 	if (mReqOptions)
 	{
 		timeout = mReqOptions->getTimeout();
-		timeout = llclamp(timeout, 0L, 3600L);
+		timeout = llclamp(timeout, LIMIT_TIMEOUT_MIN, LIMIT_TIMEOUT_MAX);
 	}
 	curl_easy_setopt(mCurlHandle, CURLOPT_TIMEOUT, timeout);
 	curl_easy_setopt(mCurlHandle, CURLOPT_CONNECTTIMEOUT, timeout);
@@ -599,11 +599,11 @@ size_t HttpOpRequest::headerCallback(void * data, size_t size, size_t nmemb, voi
 		
 		memcpy(hdr_buffer, hdr_data, frag_size);
 		hdr_buffer[frag_size] = '\0';
-#if defined(WIN32)
+#if LL_WINDOWS
 		if (! _strnicmp(hdr_buffer, con_ran_line, (std::min)(frag_size, con_ran_line_len)))
 #else
 		if (! strncasecmp(hdr_buffer, con_ran_line, (std::min)(frag_size, con_ran_line_len)))
-#endif
+#endif	// LL_WINDOWS
 		{
 			unsigned int first(0), last(0), length(0);
 			int status;
@@ -654,7 +654,7 @@ int HttpOpRequest::debugCallback(CURL * handle, curl_infotype info, char * buffe
 	switch (info)
 	{
 	case CURLINFO_TEXT:
-		if (op->mTracing > 1)
+		if (op->mTracing >= TRACE_CURL_HEADERS)
 		{
 			tag = "TEXT";
 			escape_libcurl_debug_data(buffer, len, true, safe_line);
@@ -663,7 +663,7 @@ int HttpOpRequest::debugCallback(CURL * handle, curl_infotype info, char * buffe
 		break;
 			
 	case CURLINFO_HEADER_IN:
-		if (op->mTracing > 1)
+		if (op->mTracing >= TRACE_CURL_HEADERS)
 		{
 			tag = "HEADERIN";
 			escape_libcurl_debug_data(buffer, len, true, safe_line);
@@ -672,20 +672,20 @@ int HttpOpRequest::debugCallback(CURL * handle, curl_infotype info, char * buffe
 		break;
 			
 	case CURLINFO_HEADER_OUT:
-		if (op->mTracing > 1)
+		if (op->mTracing >= TRACE_CURL_HEADERS)
 		{
 			tag = "HEADEROUT";
-			escape_libcurl_debug_data(buffer, len, true, safe_line);
+			escape_libcurl_debug_data(buffer, 2 * len, true, safe_line);		// Goes out as one line
 			logit = true;
 		}
 		break;
 			
 	case CURLINFO_DATA_IN:
-		if (op->mTracing > 1)
+		if (op->mTracing >= TRACE_CURL_HEADERS)
 		{
 			tag = "DATAIN";
 			logit = true;
-			if (op->mTracing > 2)
+			if (op->mTracing >= TRACE_CURL_BODIES)
 			{
 				escape_libcurl_debug_data(buffer, len, false, safe_line);
 			}
@@ -699,11 +699,11 @@ int HttpOpRequest::debugCallback(CURL * handle, curl_infotype info, char * buffe
 		break;
 			
 	case CURLINFO_DATA_OUT:
-		if (op->mTracing > 1)
+		if (op->mTracing >= TRACE_CURL_HEADERS)
 		{
 			tag = "DATAOUT";
 			logit = true;
-			if (op->mTracing > 2)
+			if (op->mTracing >= TRACE_CURL_BODIES)
 			{
 				escape_libcurl_debug_data(buffer, len, false, safe_line);
 			}
@@ -755,22 +755,22 @@ int parse_content_range_header(char * buffer,
 	if (! strtok_r(buffer, ": \t", &tok_state))
 		match = false;
 	if (match && (tok = strtok_r(NULL, " \t", &tok_state)))
-#if defined(WIN32)
+#if LL_WINDOWS
 		match = 0 == _stricmp("bytes", tok);
 #else
 		match = 0 == strcasecmp("bytes", tok);
-#endif
+#endif // LL_WINDOWS
 	if (match && ! (tok = strtok_r(NULL, " \t", &tok_state)))
 		match = false;
 	if (match)
 	{
 		unsigned int lcl_first(0), lcl_last(0), lcl_len(0);
 
-#if defined(WIN32)
+#if LL_WINDOWS
 		if (3 == sscanf_s(tok, "%u-%u/%u", &lcl_first, &lcl_last, &lcl_len))
 #else
 		if (3 == sscanf(tok, "%u-%u/%u", &lcl_first, &lcl_last, &lcl_len))
-#endif
+#endif // LL_WINDOWS
 		{
 			if (lcl_first > lcl_last || lcl_last >= lcl_len)
 				return -1;
@@ -779,11 +779,11 @@ int parse_content_range_header(char * buffer,
 			*length = lcl_len;
 			return 0;
 		}
-#if defined(WIN32)
+#if LL_WINDOWS
 		if (2 == sscanf_s(tok, "%u-%u/*", &lcl_first, &lcl_last))
 #else
 		if (2 == sscanf(tok, "%u-%u/*", &lcl_first, &lcl_last))
-#endif
+#endif	// LL_WINDOWS
 		{
 			if (lcl_first > lcl_last)
 				return -1;
@@ -798,14 +798,14 @@ int parse_content_range_header(char * buffer,
 	return 1;
 }
 
-#if defined(WIN32)
+#if LL_WINDOWS
 
 char *strtok_r(char *str, const char *delim, char ** savestate)
 {
 	return strtok_s(str, delim, savestate);
 }
 
-#endif
+#endif // LL_WINDOWS
 
 
 void escape_libcurl_debug_data(char * buffer, size_t len, bool scrub, std::string & safe_line)
-- 
cgit v1.2.3


From 6b4fe9fadc2301eb13a2854457b67fbebe045b08 Mon Sep 17 00:00:00 2001
From: Monty Brandenberg <monty@lindenlab.com>
Date: Tue, 19 Jun 2012 17:12:20 -0400
Subject: When a Content-Range header is received, make available the full
 triplet of <offset, length, fulllength>.

---
 indra/llcorehttp/_httpoprequest.cpp | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

(limited to 'indra/llcorehttp/_httpoprequest.cpp')

diff --git a/indra/llcorehttp/_httpoprequest.cpp b/indra/llcorehttp/_httpoprequest.cpp
index 04a8e0baff..516daadf9b 100644
--- a/indra/llcorehttp/_httpoprequest.cpp
+++ b/indra/llcorehttp/_httpoprequest.cpp
@@ -107,6 +107,7 @@ HttpOpRequest::HttpOpRequest()
 	  mReplyBody(NULL),
 	  mReplyOffset(0),
 	  mReplyLength(0),
+	  mReplyFullLength(0),
 	  mReplyHeaders(NULL),
 	  mPolicyRetries(0),
 	  mPolicyRetryAt(HttpTime(0)),
@@ -154,6 +155,7 @@ HttpOpRequest::~HttpOpRequest()
 
 	mReplyOffset = 0;
 	mReplyLength = 0;
+	mReplyFullLength = 0;
 	if (mReplyBody)
 	{
 		mReplyBody->release();
@@ -224,7 +226,7 @@ void HttpOpRequest::visitNotifier(HttpRequest * request)
 		if (mReplyOffset || mReplyLength)
 		{
 			// Got an explicit offset/length in response
-			response->setRange(mReplyOffset, mReplyLength);
+			response->setRange(mReplyOffset, mReplyLength, mReplyFullLength);
 		}
 
 		mUserHandler->onCompleted(static_cast<HttpHandle>(this), response);
@@ -362,6 +364,7 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service)
 	}
 	mReplyOffset = 0;
 	mReplyLength = 0;
+	mReplyFullLength = 0;
 	if (mReplyHeaders)
 	{
 		mReplyHeaders->release();
@@ -590,6 +593,7 @@ size_t HttpOpRequest::headerCallback(void * data, size_t size, size_t nmemb, voi
 		// taking results from the last header stanza we receive.
 		op->mReplyOffset = 0;
 		op->mReplyLength = 0;
+		op->mReplyFullLength = 0;
 		op->mStatus = HttpStatus();
 	}
 	else if (op->mProcFlags & PF_SCAN_RANGE_HEADER)
@@ -613,6 +617,7 @@ size_t HttpOpRequest::headerCallback(void * data, size_t size, size_t nmemb, voi
 				// Success, record the fragment position
 				op->mReplyOffset = first;
 				op->mReplyLength = last - first + 1;
+				op->mReplyFullLength = length;
 			}
 			else if (-1 == status)
 			{
-- 
cgit v1.2.3


From a066bc1994fccae7967921980332505aac97953f Mon Sep 17 00:00:00 2001
From: Monty Brandenberg <monty@lindenlab.com>
Date: Wed, 20 Jun 2012 18:43:28 -0400
Subject: SH-3181  More interface options for API.  Also includes returned
 headers. Only thing interesting in this changeset is the discovery that a
 sleep in the fake HTTP server ties up tests.  Need to thread that or fail on
 client disconnect or something to speed that up and make it usable for bigger
 test scenarios.  But good enough for now...

---
 indra/llcorehttp/_httpoprequest.cpp | 20 +++++++++++++++++---
 1 file changed, 17 insertions(+), 3 deletions(-)

(limited to 'indra/llcorehttp/_httpoprequest.cpp')

diff --git a/indra/llcorehttp/_httpoprequest.cpp b/indra/llcorehttp/_httpoprequest.cpp
index 516daadf9b..86ecee5b26 100644
--- a/indra/llcorehttp/_httpoprequest.cpp
+++ b/indra/llcorehttp/_httpoprequest.cpp
@@ -595,6 +595,10 @@ size_t HttpOpRequest::headerCallback(void * data, size_t size, size_t nmemb, voi
 		op->mReplyLength = 0;
 		op->mReplyFullLength = 0;
 		op->mStatus = HttpStatus();
+		if (op->mReplyHeaders)
+		{
+			op->mReplyHeaders->mHeaders.clear();
+		}
 	}
 	else if (op->mProcFlags & PF_SCAN_RANGE_HEADER)
 	{
@@ -636,9 +640,19 @@ size_t HttpOpRequest::headerCallback(void * data, size_t size, size_t nmemb, voi
 	if (op->mProcFlags & PF_SAVE_HEADERS)
 	{
 		// Save headers in response
-		// *FIXME:  Implement this...
-		;
-		
+		if (! op->mReplyHeaders)
+		{
+			op->mReplyHeaders = new HttpHeaders;
+		}
+		size_t wanted_size(hdr_size);
+		if (wanted_size && '\n' == hdr_data[wanted_size - 1])
+		{
+			if (--wanted_size && '\r' == hdr_data[wanted_size - 1])
+			{
+				--wanted_size;
+			}
+		}
+		op->mReplyHeaders->mHeaders.push_back(std::string(hdr_data, wanted_size));
 	}
 
 	return hdr_size;
-- 
cgit v1.2.3


From f37b90df5046fbe50309beada01022e35c5aa424 Mon Sep 17 00:00:00 2001
From: Monty Brandenberg <monty@lindenlab.com>
Date: Fri, 6 Jul 2012 18:09:17 -0400
Subject: SH-3222  Slow loading textures on Lag Me 1 Think I have found the
 major factor that causes the Linksys WRT54G V5 to fall over in testing
 scenarios:  DNS.  For some historical reason, we're trying to use libcurl
 without any DNS caching.  My implementation echoed that and implemented it
 correctly and I was seeing a DNS request per request on the wire.  The
 existing implementation tries to do that and has bugs because it is clearing
 caching DNS data querying only once every few seconds.  Once I started
 emulating the bug, comms through the WRT became much, much more reliable.

---
 indra/llcorehttp/_httpoprequest.cpp | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

(limited to 'indra/llcorehttp/_httpoprequest.cpp')

diff --git a/indra/llcorehttp/_httpoprequest.cpp b/indra/llcorehttp/_httpoprequest.cpp
index 86ecee5b26..a19b6bd0dc 100644
--- a/indra/llcorehttp/_httpoprequest.cpp
+++ b/indra/llcorehttp/_httpoprequest.cpp
@@ -385,8 +385,15 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service)
 	curl_easy_setopt(mCurlHandle, CURLOPT_PRIVATE, this);
 	curl_easy_setopt(mCurlHandle, CURLOPT_ENCODING, "");
 
-	// *FIXME:  Revisit this old DNS timeout setting - may no longer be valid
-	curl_easy_setopt(mCurlHandle, CURLOPT_DNS_CACHE_TIMEOUT, 0);
+	if (ENABLE_LINKSYS_WRT54G_V5_DNS_FIX)
+	{
+		curl_easy_setopt(mCurlHandle, CURLOPT_DNS_CACHE_TIMEOUT, 10);
+	}
+	else
+	{
+		// *FIXME:  Revisit this old DNS timeout setting - may no longer be valid
+		curl_easy_setopt(mCurlHandle, CURLOPT_DNS_CACHE_TIMEOUT, 0);
+	}
 	curl_easy_setopt(mCurlHandle, CURLOPT_AUTOREFERER, 1);
 	curl_easy_setopt(mCurlHandle, CURLOPT_FOLLOWLOCATION, 1);
 	curl_easy_setopt(mCurlHandle, CURLOPT_MAXREDIRS, DEFAULT_HTTP_REDIRECTS);	// *FIXME:  parameterize this later
-- 
cgit v1.2.3


From f5f51d3cda8861b3b3a380cc96aaca98e572c377 Mon Sep 17 00:00:00 2001
From: Monty Brandenberg <monty@lindenlab.com>
Date: Sat, 7 Jul 2012 19:35:32 -0400
Subject: SH-3185  Fill in some FIXME/TODO cases Also added some comments and
 changed the callback userdata argument to be an HttpOpRequest rather than a
 libcurl handle.  Less code, less clutter.

---
 indra/llcorehttp/_httpoprequest.cpp | 62 +++++++++++++++++++++----------------
 1 file changed, 36 insertions(+), 26 deletions(-)

(limited to 'indra/llcorehttp/_httpoprequest.cpp')

diff --git a/indra/llcorehttp/_httpoprequest.cpp b/indra/llcorehttp/_httpoprequest.cpp
index a19b6bd0dc..bec82d8449 100644
--- a/indra/llcorehttp/_httpoprequest.cpp
+++ b/indra/llcorehttp/_httpoprequest.cpp
@@ -344,6 +344,13 @@ void HttpOpRequest::setupCommon(HttpRequest::policy_t policy_id,
 }
 
 
+// Sets all libcurl options and data for a request.
+//
+// Used both for initial requests and to 'reload' for
+// a retry, generally with a different CURL handle.
+// Junk may be left around from a failed request and that
+// needs to be cleaned out.
+//
 HttpStatus HttpOpRequest::prepareRequest(HttpService * service)
 {
 	// Scrub transport and result data for retried op case
@@ -387,20 +394,29 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service)
 
 	if (ENABLE_LINKSYS_WRT54G_V5_DNS_FIX)
 	{
+		// The Linksys WRT54G V5 router has an issue with frequent
+		// DNS lookups from LAN machines.  If they happen too often,
+		// like for every HTTP request, the router gets annoyed after
+		// about 700 or so requests and starts issuing TCP RSTs to
+		// new connections.  Reuse the DNS lookups for even a few
+		// seconds and no RSTs.
 		curl_easy_setopt(mCurlHandle, CURLOPT_DNS_CACHE_TIMEOUT, 10);
 	}
 	else
 	{
-		// *FIXME:  Revisit this old DNS timeout setting - may no longer be valid
+		// *TODO:  Revisit this old DNS timeout setting - may no longer be valid
+		// I don't think this is valid anymore, the Multi shared DNS
+		// cache is working well.  For the case of naked easy handles,
+		// consider using a shared DNS object.
 		curl_easy_setopt(mCurlHandle, CURLOPT_DNS_CACHE_TIMEOUT, 0);
 	}
 	curl_easy_setopt(mCurlHandle, CURLOPT_AUTOREFERER, 1);
 	curl_easy_setopt(mCurlHandle, CURLOPT_FOLLOWLOCATION, 1);
-	curl_easy_setopt(mCurlHandle, CURLOPT_MAXREDIRS, DEFAULT_HTTP_REDIRECTS);	// *FIXME:  parameterize this later
+	curl_easy_setopt(mCurlHandle, CURLOPT_MAXREDIRS, DEFAULT_HTTP_REDIRECTS);
 	curl_easy_setopt(mCurlHandle, CURLOPT_WRITEFUNCTION, writeCallback);
-	curl_easy_setopt(mCurlHandle, CURLOPT_WRITEDATA, mCurlHandle);
+	curl_easy_setopt(mCurlHandle, CURLOPT_WRITEDATA, this);
 	curl_easy_setopt(mCurlHandle, CURLOPT_READFUNCTION, readCallback);
-	curl_easy_setopt(mCurlHandle, CURLOPT_READDATA, mCurlHandle);
+	curl_easy_setopt(mCurlHandle, CURLOPT_READDATA, this);
 	curl_easy_setopt(mCurlHandle, CURLOPT_SSL_VERIFYPEER, 1);
 	curl_easy_setopt(mCurlHandle, CURLOPT_SSL_VERIFYHOST, 0);
 
@@ -468,7 +484,9 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service)
 		break;
 		
 	default:
-		// *FIXME:  fail out here
+		LL_ERRS("CoreHttp") << "Invalid HTTP method in request:  "
+							<< int(mReqMethod)  << ".  Can't recover."
+							<< LL_ENDL;
 		break;
 	}
 
@@ -476,7 +494,7 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service)
 	if (mTracing >= TRACE_CURL_HEADERS)
 	{
 		curl_easy_setopt(mCurlHandle, CURLOPT_VERBOSE, 1);
-		curl_easy_setopt(mCurlHandle, CURLOPT_DEBUGDATA, mCurlHandle);
+		curl_easy_setopt(mCurlHandle, CURLOPT_DEBUGDATA, this);
 		curl_easy_setopt(mCurlHandle, CURLOPT_DEBUGFUNCTION, debugCallback);
 	}
 	
@@ -524,7 +542,7 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service)
 	if (mProcFlags & (PF_SCAN_RANGE_HEADER | PF_SAVE_HEADERS))
 	{
 		curl_easy_setopt(mCurlHandle, CURLOPT_HEADERFUNCTION, headerCallback);
-		curl_easy_setopt(mCurlHandle, CURLOPT_HEADERDATA, mCurlHandle);
+		curl_easy_setopt(mCurlHandle, CURLOPT_HEADERDATA, this);
 	}
 	
 	if (status)
@@ -537,10 +555,7 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service)
 
 size_t HttpOpRequest::writeCallback(void * data, size_t size, size_t nmemb, void * userdata)
 {
-	CURL * handle(static_cast<CURL *>(userdata));
-	HttpOpRequest * op(NULL);
-	curl_easy_getinfo(handle, CURLINFO_PRIVATE, &op);
-	// *FIXME:  check the pointer
+	HttpOpRequest * op(static_cast<HttpOpRequest *>(userdata));
 
 	if (! op->mReplyBody)
 	{
@@ -554,10 +569,7 @@ size_t HttpOpRequest::writeCallback(void * data, size_t size, size_t nmemb, void
 		
 size_t HttpOpRequest::readCallback(void * data, size_t size, size_t nmemb, void * userdata)
 {
-	CURL * handle(static_cast<CURL *>(userdata));
-	HttpOpRequest * op(NULL);
-	curl_easy_getinfo(handle, CURLINFO_PRIVATE, &op);
-	// *FIXME:  check the pointer
+	HttpOpRequest * op(static_cast<HttpOpRequest *>(userdata));
 
 	if (! op->mReqBody)
 	{
@@ -567,7 +579,8 @@ size_t HttpOpRequest::readCallback(void * data, size_t size, size_t nmemb, void
 	const size_t body_size(op->mReqBody->size());
 	if (body_size <= op->mCurlBodyPos)
 	{
-		// *FIXME:  should probably log this event - unexplained
+		LL_WARNS("HttpCore") << "Request body position beyond body size.  Aborting request."
+							 << LL_ENDL;
 		return 0;
 	}
 
@@ -586,10 +599,7 @@ size_t HttpOpRequest::headerCallback(void * data, size_t size, size_t nmemb, voi
 	static const char con_ran_line[] = "content-range:";
 	static const size_t con_ran_line_len = sizeof(con_ran_line) - 1;
 	
-	CURL * handle(static_cast<CURL *>(userdata));
-	HttpOpRequest * op(NULL);
-	curl_easy_getinfo(handle, CURLINFO_PRIVATE, &op);
-	// *FIXME:  check the pointer
+	HttpOpRequest * op(static_cast<HttpOpRequest *>(userdata));
 
 	const size_t hdr_size(size * nmemb);
 	const char * hdr_data(static_cast<const char *>(data));		// Not null terminated
@@ -609,7 +619,7 @@ size_t HttpOpRequest::headerCallback(void * data, size_t size, size_t nmemb, voi
 	}
 	else if (op->mProcFlags & PF_SCAN_RANGE_HEADER)
 	{
-		char hdr_buffer[128];
+		char hdr_buffer[128];			// Enough for a reasonable header
 		size_t frag_size((std::min)(hdr_size, sizeof(hdr_buffer) - 1));
 		
 		memcpy(hdr_buffer, hdr_data, frag_size);
@@ -638,8 +648,10 @@ size_t HttpOpRequest::headerCallback(void * data, size_t size, size_t nmemb, voi
 			else
 			{
 				// Ignore the unparsable.
-				// *FIXME:  Maybe issue a warning into the log here
-				;
+				LL_INFOS_ONCE("CoreHttp") << "Problem parsing odd Content-Range header:  '"
+										  << std::string(hdr_data, frag_size)
+										  << "'.  Ignoring."
+										  << LL_ENDL;
 			}
 		}
 	}
@@ -668,9 +680,7 @@ size_t HttpOpRequest::headerCallback(void * data, size_t size, size_t nmemb, voi
 
 int HttpOpRequest::debugCallback(CURL * handle, curl_infotype info, char * buffer, size_t len, void * userdata)
 {
-	HttpOpRequest * op(NULL);
-	curl_easy_getinfo(handle, CURLINFO_PRIVATE, &op);
-	// *FIXME:  check the pointer
+	HttpOpRequest * op(static_cast<HttpOpRequest *>(userdata));
 
 	std::string safe_line;
 	std::string tag;
-- 
cgit v1.2.3


From 7010459f04177aef1875a110b3d33e10c8ec5cad Mon Sep 17 00:00:00 2001
From: Monty Brandenberg <monty@lindenlab.com>
Date: Wed, 11 Jul 2012 15:53:57 -0400
Subject: SH-3240  Capture Content-Type and Content-Encoding headers.
 HttpResponse object now has two strings for these content headers. Either or
 both may be empty.  Tidied up the cross-platform string code and got more
 defensive about the length of a header line. Integration test for the new
 response object.

---
 indra/llcorehttp/_httpoprequest.cpp | 158 ++++++++++++++++++++++++++----------
 1 file changed, 114 insertions(+), 44 deletions(-)

(limited to 'indra/llcorehttp/_httpoprequest.cpp')

diff --git a/indra/llcorehttp/_httpoprequest.cpp b/indra/llcorehttp/_httpoprequest.cpp
index bec82d8449..1854d7ada4 100644
--- a/indra/llcorehttp/_httpoprequest.cpp
+++ b/indra/llcorehttp/_httpoprequest.cpp
@@ -74,16 +74,16 @@ void escape_libcurl_debug_data(char * buffer, size_t len, bool scrub,
 							   std::string & safe_line);
 
 
-#if LL_WINDOWS
+// OS-neutral string comparisons of various types
+int os_strncasecmp(const char *s1, const char *s2, size_t n);
+int os_strcasecmp(const char *s1, const char *s2);
+char * os_strtok_r(char *str, const char *delim, char **saveptr);
 
-// Not available on windows where the legacy strtok interface
-// is thread-safe.
-char *strtok_r(char *str, const char *delim, char **saveptr);
 
-#endif // LL_WINDOWS
+static const char * const hdr_whitespace(" \t");
+static const char * const hdr_separator(": \t");
 
-
-}
+} // end anonymous namespace
 
 
 namespace LLCore
@@ -228,7 +228,8 @@ void HttpOpRequest::visitNotifier(HttpRequest * request)
 			// Got an explicit offset/length in response
 			response->setRange(mReplyOffset, mReplyLength, mReplyFullLength);
 		}
-
+		response->setContent(mReplyConType, mReplyConEncode);
+		
 		mUserHandler->onCompleted(static_cast<HttpHandle>(this), response);
 
 		response->release();
@@ -315,7 +316,7 @@ void HttpOpRequest::setupCommon(HttpRequest::policy_t policy_id,
 								HttpOptions * options,
 								HttpHeaders * headers)
 {
-	mProcFlags = 0;
+	mProcFlags = PF_SCAN_CONTENT_HEADERS;		// Always scan for content headers
 	mReqPolicy = policy_id;
 	mReqPriority = priority;
 	mReqURL = url;
@@ -377,6 +378,8 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service)
 		mReplyHeaders->release();
 		mReplyHeaders = NULL;
 	}
+	mReplyConType.clear();
+	mReplyConEncode.clear();
 	
 	// *FIXME:  better error handling later
 	HttpStatus status;
@@ -539,7 +542,7 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service)
 	}
 	curl_easy_setopt(mCurlHandle, CURLOPT_HTTPHEADER, mCurlHeaders);
 
-	if (mProcFlags & (PF_SCAN_RANGE_HEADER | PF_SAVE_HEADERS))
+	if (mProcFlags & (PF_SCAN_RANGE_HEADER | PF_SAVE_HEADERS | PF_SCAN_CONTENT_HEADERS))
 	{
 		curl_easy_setopt(mCurlHandle, CURLOPT_HEADERFUNCTION, headerCallback);
 		curl_easy_setopt(mCurlHandle, CURLOPT_HEADERDATA, this);
@@ -598,12 +601,18 @@ size_t HttpOpRequest::headerCallback(void * data, size_t size, size_t nmemb, voi
 
 	static const char con_ran_line[] = "content-range:";
 	static const size_t con_ran_line_len = sizeof(con_ran_line) - 1;
+
+	static const char con_type_line[] = "content-type:";
+	static const size_t con_type_line_len = sizeof(con_type_line) - 1;
+	
+	static const char con_enc_line[] = "content-encoding:";
+	static const size_t con_enc_line_len = sizeof(con_enc_line) - 1;
 	
 	HttpOpRequest * op(static_cast<HttpOpRequest *>(userdata));
 
 	const size_t hdr_size(size * nmemb);
 	const char * hdr_data(static_cast<const char *>(data));		// Not null terminated
-	
+
 	if (hdr_size >= status_line_len && ! strncmp(status_line, hdr_data, status_line_len))
 	{
 		// One of possibly several status lines.  Reset what we know and start over
@@ -611,24 +620,47 @@ size_t HttpOpRequest::headerCallback(void * data, size_t size, size_t nmemb, voi
 		op->mReplyOffset = 0;
 		op->mReplyLength = 0;
 		op->mReplyFullLength = 0;
+		op->mReplyConType.clear();
+		op->mReplyConEncode.clear();
 		op->mStatus = HttpStatus();
 		if (op->mReplyHeaders)
 		{
 			op->mReplyHeaders->mHeaders.clear();
 		}
 	}
-	else if (op->mProcFlags & PF_SCAN_RANGE_HEADER)
+
+	// Nothing in here wants a final CR/LF combination.  Remove
+	// it as much as possible.
+	size_t wanted_hdr_size(hdr_size);
+	if (wanted_hdr_size && '\n' == hdr_data[wanted_hdr_size - 1])
+	{
+		if (--wanted_hdr_size && '\r' == hdr_data[wanted_hdr_size - 1])
+		{
+			--wanted_hdr_size;
+		}
+	}
+	
+	// Save header if caller wants them in the response
+	if (op->mProcFlags & PF_SAVE_HEADERS)
+	{
+		// Save headers in response
+		if (! op->mReplyHeaders)
+		{
+			op->mReplyHeaders = new HttpHeaders;
+		}
+		op->mReplyHeaders->mHeaders.push_back(std::string(hdr_data, wanted_hdr_size));
+	}
+
+	// Detect and parse 'Content-Range' headers
+	if (op->mProcFlags & PF_SCAN_RANGE_HEADER)
 	{
 		char hdr_buffer[128];			// Enough for a reasonable header
-		size_t frag_size((std::min)(hdr_size, sizeof(hdr_buffer) - 1));
+		size_t frag_size((std::min)(wanted_hdr_size, sizeof(hdr_buffer) - 1));
 		
 		memcpy(hdr_buffer, hdr_data, frag_size);
 		hdr_buffer[frag_size] = '\0';
-#if LL_WINDOWS
-		if (! _strnicmp(hdr_buffer, con_ran_line, (std::min)(frag_size, con_ran_line_len)))
-#else
-		if (! strncasecmp(hdr_buffer, con_ran_line, (std::min)(frag_size, con_ran_line_len)))
-#endif	// LL_WINDOWS
+		if (frag_size > con_ran_line_len &&
+			! os_strncasecmp(hdr_buffer, con_ran_line, con_ran_line_len))
 		{
 			unsigned int first(0), last(0), length(0);
 			int status;
@@ -656,22 +688,43 @@ size_t HttpOpRequest::headerCallback(void * data, size_t size, size_t nmemb, voi
 		}
 	}
 
-	if (op->mProcFlags & PF_SAVE_HEADERS)
+	// Detect and parse 'Content-Type' and 'Content-Encoding' headers
+	if (op->mProcFlags & PF_SCAN_CONTENT_HEADERS)
 	{
-		// Save headers in response
-		if (! op->mReplyHeaders)
+		if (wanted_hdr_size > con_type_line_len &&
+			! os_strncasecmp(hdr_data, con_type_line, con_type_line_len))
 		{
-			op->mReplyHeaders = new HttpHeaders;
+			// Found 'Content-Type:', extract single-token value
+			std::string rhs(hdr_data + con_type_line_len, wanted_hdr_size - con_type_line_len);
+			std::string::size_type begin(0), end(rhs.size()), pos;
+
+			if ((pos = rhs.find_first_not_of(hdr_whitespace)) != std::string::npos)
+			{
+				begin = pos;
+			}
+			if ((pos = rhs.find_first_of(hdr_whitespace, begin)) != std::string::npos)
+			{
+				end = pos;
+			}
+			op->mReplyConType.assign(rhs, begin, end - begin);
 		}
-		size_t wanted_size(hdr_size);
-		if (wanted_size && '\n' == hdr_data[wanted_size - 1])
+		else if (wanted_hdr_size > con_enc_line_len &&
+				 ! os_strncasecmp(hdr_data, con_enc_line, con_enc_line_len))
 		{
-			if (--wanted_size && '\r' == hdr_data[wanted_size - 1])
+			// Found 'Content-Encoding:', extract single-token value
+			std::string rhs(hdr_data + con_enc_line_len, wanted_hdr_size - con_enc_line_len);
+			std::string::size_type begin(0), end(rhs.size()), pos;
+
+			if ((pos = rhs.find_first_not_of(hdr_whitespace)) != std::string::npos)
 			{
-				--wanted_size;
+				begin = pos;
 			}
+			if ((pos = rhs.find_first_of(hdr_whitespace, begin)) != std::string::npos)
+			{
+				end = pos;
+			}
+			op->mReplyConEncode.assign(rhs, begin, end - begin);
 		}
-		op->mReplyHeaders->mHeaders.push_back(std::string(hdr_data, wanted_size));
 	}
 
 	return hdr_size;
@@ -788,15 +841,11 @@ int parse_content_range_header(char * buffer,
 	char * tok_state(NULL), * tok(NULL);
 	bool match(true);
 			
-	if (! strtok_r(buffer, ": \t", &tok_state))
+	if (! os_strtok_r(buffer, hdr_separator, &tok_state))
 		match = false;
-	if (match && (tok = strtok_r(NULL, " \t", &tok_state)))
-#if LL_WINDOWS
-		match = 0 == _stricmp("bytes", tok);
-#else
-		match = 0 == strcasecmp("bytes", tok);
-#endif // LL_WINDOWS
-	if (match && ! (tok = strtok_r(NULL, " \t", &tok_state)))
+	if (match && (tok = os_strtok_r(NULL, hdr_whitespace, &tok_state)))
+		match = 0 == os_strcasecmp("bytes", tok);
+	if (match && ! (tok = os_strtok_r(NULL, " \t", &tok_state)))
 		match = false;
 	if (match)
 	{
@@ -834,15 +883,6 @@ int parse_content_range_header(char * buffer,
 	return 1;
 }
 
-#if LL_WINDOWS
-
-char *strtok_r(char *str, const char *delim, char ** savestate)
-{
-	return strtok_s(str, delim, savestate);
-}
-
-#endif // LL_WINDOWS
-
 
 void escape_libcurl_debug_data(char * buffer, size_t len, bool scrub, std::string & safe_line)
 {
@@ -880,6 +920,36 @@ void escape_libcurl_debug_data(char * buffer, size_t len, bool scrub, std::strin
 }
 
 
+int os_strncasecmp(const char *s1, const char *s2, size_t n)
+{
+#if LL_WINDOWS
+	return _strnicmp(s1, s2, n);
+#else
+	return strncasecmp(s1, s2, n);
+#endif	// LL_WINDOWS
+}
+
+
+int os_strcasecmp(const char *s1, const char *s2)
+{
+#if LL_WINDOWS
+	return _stricmp(s1, s2);
+#else
+	return strcasecmp(s1, s2);
+#endif // LL_WINDOWS
+}
+
+
+char * os_strtok_r(char *str, const char *delim, char ** savestate)
+{
+#if LL_WINDOWS
+	return strtok_s(str, delim, savestate);
+#else
+	return strtok_r(str, delim, savestate);
+#endif
+}
+
+
 }  // end anonymous namespace
 
 		
-- 
cgit v1.2.3


From d238341afaecedfe227141126c4c35dcde4a0671 Mon Sep 17 00:00:00 2001
From: Monty Brandenberg <monty@lindenlab.com>
Date: Mon, 16 Jul 2012 11:53:04 -0400
Subject: SH-3189  Remove/improve naive data structures When releasing HTTP
 waiters, avoid unnecessary sort activity. For Content-Type in responses, let
 libcurl do the work and removed my parsing of headers.  Drop Content-Encoding
 as libcurl will deal with that.  If anyone is interested, they can parse.

---
 indra/llcorehttp/_httpoprequest.cpp | 48 +++----------------------------------
 1 file changed, 3 insertions(+), 45 deletions(-)

(limited to 'indra/llcorehttp/_httpoprequest.cpp')

diff --git a/indra/llcorehttp/_httpoprequest.cpp b/indra/llcorehttp/_httpoprequest.cpp
index 1854d7ada4..1a770f67be 100644
--- a/indra/llcorehttp/_httpoprequest.cpp
+++ b/indra/llcorehttp/_httpoprequest.cpp
@@ -228,7 +228,7 @@ void HttpOpRequest::visitNotifier(HttpRequest * request)
 			// Got an explicit offset/length in response
 			response->setRange(mReplyOffset, mReplyLength, mReplyFullLength);
 		}
-		response->setContent(mReplyConType, mReplyConEncode);
+		response->setContentType(mReplyConType);
 		
 		mUserHandler->onCompleted(static_cast<HttpHandle>(this), response);
 
@@ -316,7 +316,7 @@ void HttpOpRequest::setupCommon(HttpRequest::policy_t policy_id,
 								HttpOptions * options,
 								HttpHeaders * headers)
 {
-	mProcFlags = PF_SCAN_CONTENT_HEADERS;		// Always scan for content headers
+	mProcFlags = 0U;
 	mReqPolicy = policy_id;
 	mReqPriority = priority;
 	mReqURL = url;
@@ -379,7 +379,6 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service)
 		mReplyHeaders = NULL;
 	}
 	mReplyConType.clear();
-	mReplyConEncode.clear();
 	
 	// *FIXME:  better error handling later
 	HttpStatus status;
@@ -542,7 +541,7 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service)
 	}
 	curl_easy_setopt(mCurlHandle, CURLOPT_HTTPHEADER, mCurlHeaders);
 
-	if (mProcFlags & (PF_SCAN_RANGE_HEADER | PF_SAVE_HEADERS | PF_SCAN_CONTENT_HEADERS))
+	if (mProcFlags & (PF_SCAN_RANGE_HEADER | PF_SAVE_HEADERS))
 	{
 		curl_easy_setopt(mCurlHandle, CURLOPT_HEADERFUNCTION, headerCallback);
 		curl_easy_setopt(mCurlHandle, CURLOPT_HEADERDATA, this);
@@ -620,8 +619,6 @@ size_t HttpOpRequest::headerCallback(void * data, size_t size, size_t nmemb, voi
 		op->mReplyOffset = 0;
 		op->mReplyLength = 0;
 		op->mReplyFullLength = 0;
-		op->mReplyConType.clear();
-		op->mReplyConEncode.clear();
 		op->mStatus = HttpStatus();
 		if (op->mReplyHeaders)
 		{
@@ -688,45 +685,6 @@ size_t HttpOpRequest::headerCallback(void * data, size_t size, size_t nmemb, voi
 		}
 	}
 
-	// Detect and parse 'Content-Type' and 'Content-Encoding' headers
-	if (op->mProcFlags & PF_SCAN_CONTENT_HEADERS)
-	{
-		if (wanted_hdr_size > con_type_line_len &&
-			! os_strncasecmp(hdr_data, con_type_line, con_type_line_len))
-		{
-			// Found 'Content-Type:', extract single-token value
-			std::string rhs(hdr_data + con_type_line_len, wanted_hdr_size - con_type_line_len);
-			std::string::size_type begin(0), end(rhs.size()), pos;
-
-			if ((pos = rhs.find_first_not_of(hdr_whitespace)) != std::string::npos)
-			{
-				begin = pos;
-			}
-			if ((pos = rhs.find_first_of(hdr_whitespace, begin)) != std::string::npos)
-			{
-				end = pos;
-			}
-			op->mReplyConType.assign(rhs, begin, end - begin);
-		}
-		else if (wanted_hdr_size > con_enc_line_len &&
-				 ! os_strncasecmp(hdr_data, con_enc_line, con_enc_line_len))
-		{
-			// Found 'Content-Encoding:', extract single-token value
-			std::string rhs(hdr_data + con_enc_line_len, wanted_hdr_size - con_enc_line_len);
-			std::string::size_type begin(0), end(rhs.size()), pos;
-
-			if ((pos = rhs.find_first_not_of(hdr_whitespace)) != std::string::npos)
-			{
-				begin = pos;
-			}
-			if ((pos = rhs.find_first_of(hdr_whitespace, begin)) != std::string::npos)
-			{
-				end = pos;
-			}
-			op->mReplyConEncode.assign(rhs, begin, end - begin);
-		}
-	}
-
 	return hdr_size;
 }
 
-- 
cgit v1.2.3


From 8d3e5f3959b071226c4a5c2c68d9fe8664ae1ffd Mon Sep 17 00:00:00 2001
From: Monty Brandenberg <monty@lindenlab.com>
Date: Mon, 16 Jul 2012 17:35:44 -0400
Subject: SH-3241 Validate header correctness, SH-3243 more unit tests Define
 expectations for headers for GET, POST, PUT requests. Document those in the
 interface, test those with integration tests. Verify that header overrides
 work as expected.

---
 indra/llcorehttp/_httpoprequest.cpp | 4 ++++
 1 file changed, 4 insertions(+)

(limited to 'indra/llcorehttp/_httpoprequest.cpp')

diff --git a/indra/llcorehttp/_httpoprequest.cpp b/indra/llcorehttp/_httpoprequest.cpp
index 1a770f67be..a18a164f0d 100644
--- a/indra/llcorehttp/_httpoprequest.cpp
+++ b/indra/llcorehttp/_httpoprequest.cpp
@@ -468,6 +468,8 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service)
 			curl_easy_setopt(mCurlHandle, CURLOPT_POSTFIELDS, static_cast<void *>(NULL));
 			curl_easy_setopt(mCurlHandle, CURLOPT_POSTFIELDSIZE, data_size);
 			mCurlHeaders = curl_slist_append(mCurlHeaders, "Expect:");
+			mCurlHeaders = curl_slist_append(mCurlHeaders, "Connection: keep-alive");
+			mCurlHeaders = curl_slist_append(mCurlHeaders, "Keep-alive: 300");
 		}
 		break;
 		
@@ -482,6 +484,8 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service)
 			curl_easy_setopt(mCurlHandle, CURLOPT_INFILESIZE, data_size);
 			curl_easy_setopt(mCurlHandle, CURLOPT_POSTFIELDS, (void *) NULL);
 			mCurlHeaders = curl_slist_append(mCurlHeaders, "Expect:");
+			mCurlHeaders = curl_slist_append(mCurlHeaders, "Connection: keep-alive");
+			mCurlHeaders = curl_slist_append(mCurlHeaders, "Keep-alive: 300");
 		}
 		break;
 		
-- 
cgit v1.2.3


From 85e69b043b098dbe5a09f2eac6ff541123089f13 Mon Sep 17 00:00:00 2001
From: Monty Brandenberg <monty@lindenlab.com>
Date: Mon, 23 Jul 2012 23:40:07 +0000
Subject: Big comment and naming cleanup.  Ready for prime-time. Add to-do list
 to _httpinternal.h to guide anyone who wants to pitch in and help.

---
 indra/llcorehttp/_httpoprequest.cpp | 47 ++++++++++++++-----------------------
 1 file changed, 18 insertions(+), 29 deletions(-)

(limited to 'indra/llcorehttp/_httpoprequest.cpp')

diff --git a/indra/llcorehttp/_httpoprequest.cpp b/indra/llcorehttp/_httpoprequest.cpp
index a18a164f0d..7db19b1841 100644
--- a/indra/llcorehttp/_httpoprequest.cpp
+++ b/indra/llcorehttp/_httpoprequest.cpp
@@ -111,10 +111,10 @@ HttpOpRequest::HttpOpRequest()
 	  mReplyHeaders(NULL),
 	  mPolicyRetries(0),
 	  mPolicyRetryAt(HttpTime(0)),
-	  mPolicyRetryLimit(DEFAULT_RETRY_COUNT)
+	  mPolicyRetryLimit(HTTP_RETRY_COUNT_DEFAULT)
 {
 	// *NOTE:  As members are added, retry initialization/cleanup
-	// may need to be extended in @prepareRequest().
+	// may need to be extended in @see prepareRequest().
 }
 
 
@@ -153,9 +153,6 @@ HttpOpRequest::~HttpOpRequest()
 		mCurlHeaders = NULL;
 	}
 
-	mReplyOffset = 0;
-	mReplyLength = 0;
-	mReplyFullLength = 0;
 	if (mReplyBody)
 	{
 		mReplyBody->release();
@@ -215,8 +212,6 @@ void HttpOpRequest::stageFromActive(HttpService * service)
 
 void HttpOpRequest::visitNotifier(HttpRequest * request)
 {
-	static const HttpStatus partial_content(HTTP_PARTIAL_CONTENT, HE_SUCCESS);
-	
 	if (mUserHandler)
 	{
 		HttpResponse * response = new HttpResponse();
@@ -339,8 +334,8 @@ void HttpOpRequest::setupCommon(HttpRequest::policy_t policy_id,
 			mProcFlags |= PF_SAVE_HEADERS;
 		}
 		mPolicyRetryLimit = options->getRetries();
-		mPolicyRetryLimit = llclamp(mPolicyRetryLimit, LIMIT_RETRY_MIN, LIMIT_RETRY_MAX);
-		mTracing = (std::max)(mTracing, llclamp(options->getTrace(), TRACE_MIN, TRACE_MAX));
+		mPolicyRetryLimit = llclamp(mPolicyRetryLimit, HTTP_RETRY_COUNT_MIN, HTTP_RETRY_COUNT_MAX);
+		mTracing = (std::max)(mTracing, llclamp(options->getTrace(), HTTP_TRACE_MIN, HTTP_TRACE_MAX));
 	}
 }
 
@@ -394,7 +389,7 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service)
 	curl_easy_setopt(mCurlHandle, CURLOPT_PRIVATE, this);
 	curl_easy_setopt(mCurlHandle, CURLOPT_ENCODING, "");
 
-	if (ENABLE_LINKSYS_WRT54G_V5_DNS_FIX)
+	if (HTTP_ENABLE_LINKSYS_WRT54G_V5_DNS_FIX)
 	{
 		// The Linksys WRT54G V5 router has an issue with frequent
 		// DNS lookups from LAN machines.  If they happen too often,
@@ -402,7 +397,7 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service)
 		// about 700 or so requests and starts issuing TCP RSTs to
 		// new connections.  Reuse the DNS lookups for even a few
 		// seconds and no RSTs.
-		curl_easy_setopt(mCurlHandle, CURLOPT_DNS_CACHE_TIMEOUT, 10);
+		curl_easy_setopt(mCurlHandle, CURLOPT_DNS_CACHE_TIMEOUT, 15);
 	}
 	else
 	{
@@ -414,7 +409,7 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service)
 	}
 	curl_easy_setopt(mCurlHandle, CURLOPT_AUTOREFERER, 1);
 	curl_easy_setopt(mCurlHandle, CURLOPT_FOLLOWLOCATION, 1);
-	curl_easy_setopt(mCurlHandle, CURLOPT_MAXREDIRS, DEFAULT_HTTP_REDIRECTS);
+	curl_easy_setopt(mCurlHandle, CURLOPT_MAXREDIRS, HTTP_REDIRECTS_DEFAULT);
 	curl_easy_setopt(mCurlHandle, CURLOPT_WRITEFUNCTION, writeCallback);
 	curl_easy_setopt(mCurlHandle, CURLOPT_WRITEDATA, this);
 	curl_easy_setopt(mCurlHandle, CURLOPT_READFUNCTION, readCallback);
@@ -434,7 +429,7 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service)
 	}
 	else if (policy.get(HttpRequest::GP_HTTP_PROXY, &opt_value))
 	{
-		// *TODO:  This is fine for now but get fuller socks/
+		// *TODO:  This is fine for now but get fuller socks5/
 		// authentication thing going later....
 		curl_easy_setopt(mCurlHandle, CURLOPT_PROXY, opt_value->c_str());
 		curl_easy_setopt(mCurlHandle, CURLOPT_PROXYTYPE, CURLPROXY_HTTP);
@@ -497,7 +492,7 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service)
 	}
 
 	// Tracing
-	if (mTracing >= TRACE_CURL_HEADERS)
+	if (mTracing >= HTTP_TRACE_CURL_HEADERS)
 	{
 		curl_easy_setopt(mCurlHandle, CURLOPT_VERBOSE, 1);
 		curl_easy_setopt(mCurlHandle, CURLOPT_DEBUGDATA, this);
@@ -528,11 +523,11 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service)
 	mCurlHeaders = curl_slist_append(mCurlHeaders, "Pragma:");
 
 	// Request options
-	long timeout(DEFAULT_TIMEOUT);
+	long timeout(HTTP_REQUEST_TIMEOUT_DEFAULT);
 	if (mReqOptions)
 	{
 		timeout = mReqOptions->getTimeout();
-		timeout = llclamp(timeout, LIMIT_TIMEOUT_MIN, LIMIT_TIMEOUT_MAX);
+		timeout = llclamp(timeout, HTTP_REQUEST_TIMEOUT_MIN, HTTP_REQUEST_TIMEOUT_MAX);
 	}
 	curl_easy_setopt(mCurlHandle, CURLOPT_TIMEOUT, timeout);
 	curl_easy_setopt(mCurlHandle, CURLOPT_CONNECTTIMEOUT, timeout);
@@ -605,12 +600,6 @@ size_t HttpOpRequest::headerCallback(void * data, size_t size, size_t nmemb, voi
 	static const char con_ran_line[] = "content-range:";
 	static const size_t con_ran_line_len = sizeof(con_ran_line) - 1;
 
-	static const char con_type_line[] = "content-type:";
-	static const size_t con_type_line_len = sizeof(con_type_line) - 1;
-	
-	static const char con_enc_line[] = "content-encoding:";
-	static const size_t con_enc_line_len = sizeof(con_enc_line) - 1;
-	
 	HttpOpRequest * op(static_cast<HttpOpRequest *>(userdata));
 
 	const size_t hdr_size(size * nmemb);
@@ -705,7 +694,7 @@ int HttpOpRequest::debugCallback(CURL * handle, curl_infotype info, char * buffe
 	switch (info)
 	{
 	case CURLINFO_TEXT:
-		if (op->mTracing >= TRACE_CURL_HEADERS)
+		if (op->mTracing >= HTTP_TRACE_CURL_HEADERS)
 		{
 			tag = "TEXT";
 			escape_libcurl_debug_data(buffer, len, true, safe_line);
@@ -714,7 +703,7 @@ int HttpOpRequest::debugCallback(CURL * handle, curl_infotype info, char * buffe
 		break;
 			
 	case CURLINFO_HEADER_IN:
-		if (op->mTracing >= TRACE_CURL_HEADERS)
+		if (op->mTracing >= HTTP_TRACE_CURL_HEADERS)
 		{
 			tag = "HEADERIN";
 			escape_libcurl_debug_data(buffer, len, true, safe_line);
@@ -723,7 +712,7 @@ int HttpOpRequest::debugCallback(CURL * handle, curl_infotype info, char * buffe
 		break;
 			
 	case CURLINFO_HEADER_OUT:
-		if (op->mTracing >= TRACE_CURL_HEADERS)
+		if (op->mTracing >= HTTP_TRACE_CURL_HEADERS)
 		{
 			tag = "HEADEROUT";
 			escape_libcurl_debug_data(buffer, 2 * len, true, safe_line);		// Goes out as one line
@@ -732,11 +721,11 @@ int HttpOpRequest::debugCallback(CURL * handle, curl_infotype info, char * buffe
 		break;
 			
 	case CURLINFO_DATA_IN:
-		if (op->mTracing >= TRACE_CURL_HEADERS)
+		if (op->mTracing >= HTTP_TRACE_CURL_HEADERS)
 		{
 			tag = "DATAIN";
 			logit = true;
-			if (op->mTracing >= TRACE_CURL_BODIES)
+			if (op->mTracing >= HTTP_TRACE_CURL_BODIES)
 			{
 				escape_libcurl_debug_data(buffer, len, false, safe_line);
 			}
@@ -750,11 +739,11 @@ int HttpOpRequest::debugCallback(CURL * handle, curl_infotype info, char * buffe
 		break;
 			
 	case CURLINFO_DATA_OUT:
-		if (op->mTracing >= TRACE_CURL_HEADERS)
+		if (op->mTracing >= HTTP_TRACE_CURL_HEADERS)
 		{
 			tag = "DATAOUT";
 			logit = true;
-			if (op->mTracing >= TRACE_CURL_BODIES)
+			if (op->mTracing >= HTTP_TRACE_CURL_BODIES)
 			{
 				escape_libcurl_debug_data(buffer, len, false, safe_line);
 			}
-- 
cgit v1.2.3