merge -r94900 linden/branches/kelly/qar-825 to linden/release

QAR-825 DEV-18489 Event poll is brittle and doesn't parse status correctly

6 files changed, 243 insertions, 38 deletions

diff --git a/indra/llcommon/CMakeLists.txt b/indra/llcommon/CMakeLists.txt
index 72d267d794..8810549883 100644
--- a/indra/llcommon/CMakeLists.txt
+++ b/indra/llcommon/CMakeLists.txt
@@ -114,6 +114,7 @@ set(llcommon_HEADER_FILES
     llframetimer.h
     llhash.h
     llheartbeat.h
+    llhttpstatuscodes.h
     llindexedqueue.h
     llindraconfigfile.h
     llkeythrottle.h
diff --git a/indra/llcommon/llhttpstatuscodes.h b/indra/llcommon/llhttpstatuscodes.h
new file mode 100644
index 0000000000..ed85dd5148
--- /dev/null
+++ b/indra/llcommon/llhttpstatuscodes.h
@@ -0,0 +1,94 @@
+/** 
+ * @file llhttpstatuscodes.h
+ * @brief Constants for HTTP status codes
+ *
+ * $LicenseInfo:firstyear=2001&license=viewergpl$
+ * 
+ * Copyright (c) 2001-2007, Linden Research, Inc.
+ * 
+ * Second Life Viewer Source Code
+ * The source code in this file ("Source Code") is provided by Linden Lab
+ * to you under the terms of the GNU General Public License, version 2.0
+ * ("GPL"), unless you have obtained a separate licensing agreement
+ * ("Other License"), formally executed by you and Linden Lab.  Terms of
+ * the GPL can be found in doc/GPL-license.txt in this distribution, or
+ * online at http://secondlife.com/developers/opensource/gplv2
+ * 
+ * There are special exceptions to the terms and conditions of the GPL as
+ * it is applied to this Source Code. View the full text of the exception
+ * in the file doc/FLOSS-exception.txt in this software distribution, or
+ * online at http://secondlife.com/developers/opensource/flossexception
+ * 
+ * By copying, modifying or distributing this software, you acknowledge
+ * that you have read and understood your obligations described above,
+ * and agree to abide by those obligations.
+ * 
+ * ALL LINDEN LAB SOURCE CODE IS PROVIDED "AS IS." LINDEN LAB MAKES NO
+ * WARRANTIES, EXPRESS, IMPLIED OR OTHERWISE, REGARDING ITS ACCURACY,
+ * COMPLETENESS OR PERFORMANCE.
+ * $/LicenseInfo$
+ */
+
+#ifndef LL_HTTP_STATUS_CODES_H
+#define LL_HTTP_STATUS_CODES_H
+
+#include "stdtypes.h"
+
+// Standard errors from HTTP spec:
+// http://www.w3.org/Protocols/rfc2616/rfc2616-sec6.html#sec6.1
+const S32 HTTP_CONTINUE = 100;
+const S32 HTTP_SWITCHING_PROTOCOLS = 101;
+
+// Success
+const S32 HTTP_OK = 200;
+const S32 HTTP_CREATED = 201;
+const S32 HTTP_ACCEPTED = 202;
+const S32 HTTP_NON_AUTHORITATIVE_INFORMATION = 203;
+const S32 HTTP_NO_CONTENT = 204;
+const S32 HTTP_RESET_CONTENT = 205;
+const S32 HTTP_PARTIAL_CONTENT = 206;
+
+// Redirection
+const S32 HTTP_MULTIPLE_CHOICES = 300;
+const S32 HTTP_MOVED_PERMANENTLY = 301;
+const S32 HTTP_FOUND = 302;
+const S32 HTTP_SEE_OTHER = 303;
+const S32 HTTP_NOT_MODIFIED = 304;
+const S32 HTTP_USE_PROXY = 305;
+const S32 HTTP_TEMPORARY_REDIRECT = 307;
+
+// Client Error
+const S32 HTTP_BAD_REQUEST = 400;
+const S32 HTTP_UNAUTHORIZED = 401;
+const S32 HTTP_PAYMENT_REQUIRED = 402;
+const S32 HTTP_FORBIDDEN = 403;
+const S32 HTTP_NOT_FOUND = 404;
+const S32 HTTP_METHOD_NOT_ALLOWED = 405;
+const S32 HTTP_NOT_ACCEPTABLE = 406;
+const S32 HTTP_PROXY_AUTHENTICATION_REQUIRED = 407;
+const S32 HTTP_REQUEST_TIME_OUT = 408;
+const S32 HTTP_CONFLICT = 409;
+const S32 HTTP_GONE = 410;
+const S32 HTTP_LENGTH_REQUIRED = 411;
+const S32 HTTP_PRECONDITION_FAILED = 412;
+const S32 HTTP_REQUEST_ENTITY_TOO_LARGE = 413;
+const S32 HTTP_REQUEST_URI_TOO_LARGE = 414;
+const S32 HTTP_UNSUPPORTED_MEDIA_TYPE = 415;
+const S32 HTTP_REQUESTED_RANGE_NOT_SATISFIABLE = 416;
+const S32 HTTP_EXPECTATION_FAILED = 417;
+
+// Server Error
+const S32 HTTP_INTERNAL_SERVER_ERROR = 500;
+const S32 HTTP_NOT_IMPLEMENTED = 501;
+const S32 HTTP_BAD_GATEWAY = 502;
+const S32 HTTP_SERVICE_UNAVAILABLE = 503;
+const S32 HTTP_GATEWAY_TIME_OUT = 504;
+const S32 HTTP_VERSION_NOT_SUPPORTED = 505;
+
+// We combine internal process errors with status codes
+// These status codes should not be sent over the wire
+//   and indicate something went wrong internally.
+// If you get these they are not normal.
+const S32 HTTP_INTERNAL_ERROR = 499;
+
+#endif
diff --git a/indra/llmessage/llhttpclient.cpp b/indra/llmessage/llhttpclient.cpp
index 7a6add6282..2f822aee87 100644
--- a/indra/llmessage/llhttpclient.cpp
+++ b/indra/llmessage/llhttpclient.cpp
@@ -56,7 +56,7 @@ namespace
 	{
 	public:
 		LLHTTPClientURLAdaptor(LLCurl::ResponderPtr responder)
-			: mResponder(responder), mStatus(499),
+			: LLURLRequestComplete(), mResponder(responder), mStatus(499),
 			  mReason("LLURLRequest complete w/no status")
 		{
 		}
@@ -67,6 +67,8 @@ namespace
 
 		virtual void httpStatus(U32 status, const std::string& reason)
 		{
+			LLURLRequestComplete::httpStatus(status,reason);
+
 			mStatus = status;
 			mReason = reason;
 		}
diff --git a/indra/llmessage/llurlrequest.cpp b/indra/llmessage/llurlrequest.cpp
index 5ac55bc83d..6e5fa6def8 100644
--- a/indra/llmessage/llurlrequest.cpp
+++ b/indra/llmessage/llurlrequest.cpp
@@ -461,58 +461,76 @@ size_t LLURLRequest::upCallback(
 
 static size_t headerCallback(void* data, size_t size, size_t nmemb, void* user)
 {
-	const char* headerLine = (const char*)data;
-	size_t headerLen = size * nmemb;
+	const char* header_line = (const char*)data;
+	size_t header_len = size * nmemb;
 	LLURLRequestComplete* complete = (LLURLRequestComplete*)user;
 
+	if (!complete || !header_line)
+	{
+		return header_len;
+	}
+
 	// *TODO: This should be a utility in llstring.h: isascii()
-	for (size_t i = 0; i < headerLen; ++i)
+	for (size_t i = 0; i < header_len; ++i)
 	{
-		if (headerLine[i] < 0)
+		if (header_line[i] < 0)
 		{
-			return headerLen;
+			return header_len;
 		}
 	}
 
-	size_t sep;
-	for (sep = 0; sep < headerLen  &&  headerLine[sep] != ':'; ++sep) { }
-
-	if (sep < headerLen && complete)
-	{
-		std::string key(headerLine, sep);
-		std::string value(headerLine + sep + 1, headerLen - sep - 1);
-
-		key = utf8str_tolower(utf8str_trim(key));
-		value = utf8str_trim(value);
+	std::string header(header_line, header_len);
 
-		complete->header(key, value);
-	}
-	else
+	// Per HTTP spec the first header line must be the status line.
+	if (!complete->haveHTTPStatus())
 	{
-		std::string s(headerLine, headerLen);
-
-		std::string::iterator end = s.end();
-		std::string::iterator pos1 = std::find(s.begin(), end, ' ');
+		std::string::iterator end = header.end();
+		std::string::iterator pos1 = std::find(header.begin(), end, ' ');
 		if (pos1 != end) ++pos1;
 		std::string::iterator pos2 = std::find(pos1, end, ' ');
 		if (pos2 != end) ++pos2;
 		std::string::iterator pos3 = std::find(pos2, end, '\r');
 
-		std::string version(s.begin(), pos1);
+		std::string version(header.begin(), pos1);
 		std::string status(pos1, pos2);
 		std::string reason(pos2, pos3);
 
 		int statusCode = atoi(status.c_str());
 		if (statusCode > 0)
 		{
-			if (complete)
-			{
-				complete->httpStatus((U32)statusCode, reason);
-			}
+			complete->httpStatus((U32)statusCode, reason);
+		}
+		else
+		{
+			llwarns << "Unable to parse http response status line: "
+					<< header << llendl;
+			complete->httpStatus(499,"Unable to parse status line.");
+		}
+		return header_len;
+	}
+
+	std::string::iterator sep = std::find(header.begin(),header.end(),':');
+
+	if (sep != header.end())
+	{
+		std::string key(header.begin(), sep);
+		std::string value(sep + 1, header.end());
+
+		key = utf8str_tolower(utf8str_trim(key));
+		value = utf8str_trim(value);
+
+		complete->header(key, value);
+	}
+	else
+	{
+		LLStringUtil::trim(header);
+		if (!header.empty())
+		{
+			llwarns << "Unable to parse header: " << header << llendl;
 		}
 	}
 
-	return headerLen;
+	return header_len;
 }
 
 /**
@@ -553,7 +571,8 @@ LLIOPipe::EStatus LLContextURLExtractor::process_impl(
  * LLURLRequestComplete
  */
 LLURLRequestComplete::LLURLRequestComplete() :
-	mRequestStatus(LLIOPipe::STATUS_ERROR)
+	mRequestStatus(LLIOPipe::STATUS_ERROR),
+	mHaveHTTPStatus(false)
 {
 	LLMemType m1(LLMemType::MTYPE_IO_URL_REQUEST);
 }
@@ -572,6 +591,7 @@ void LLURLRequestComplete::header(const std::string& header, const std::string&
 //virtual 
 void LLURLRequestComplete::httpStatus(U32 status, const std::string& reason)
 {
+	mHaveHTTPStatus = true;
 }
 
 //virtual 
diff --git a/indra/llmessage/llurlrequest.h b/indra/llmessage/llurlrequest.h
index 579706f83b..c0af746116 100644
--- a/indra/llmessage/llurlrequest.h
+++ b/indra/llmessage/llurlrequest.h
@@ -327,6 +327,9 @@ public:
 	LLURLRequestComplete();
 	virtual ~LLURLRequestComplete();
 
+	// The first line of an http response must be the status line
+	// true if we have already parsed this line.
+	bool haveHTTPStatus() const { return mHaveHTTPStatus; }
 protected:
 	/* @name LLIOPipe virtual implementations
 	 */
@@ -345,6 +348,8 @@ protected:
 	// value to note if we actually got the response. This value
 	// depends on correct useage from the LLURLRequest instance.
 	EStatus mRequestStatus;
+
+	bool mHaveHTTPStatus;
 };
 
 
diff --git a/indra/newview/lleventpoll.cpp b/indra/newview/lleventpoll.cpp
index 745d0f7be8..49862e8112 100644
--- a/indra/newview/lleventpoll.cpp
+++ b/indra/newview/lleventpoll.cpp
@@ -31,16 +31,26 @@
 
 #include "llviewerprecompiledheaders.h"
 
+#include "llappviewer.h"
 #include "llagent.h"
 #include "lleventpoll.h"
 
 #include "llhttpclient.h"
+#include "llhttpstatuscodes.h"
 #include "llsdserialize.h"
+#include "lltimer.h"
 #include "llviewerregion.h"
 #include "message.h"
 
 namespace
 {
+	// We will wait RETRY_SECONDS + (errorCount * RETRY_SECONDS_INC) before retrying after an error.
+	// This means we attempt to recover relatively quickly but back off giving more time to recover
+	// until we finally give up after MAX_EVENT_POLL_HTTP_ERRORS attempts.
+	const F32 EVENT_POLL_ERROR_RETRY_SECONDS = 15.f; // ~ half of a normal timeout.
+	const F32 EVENT_POLL_ERROR_RETRY_SECONDS_INC = 5.f; // ~ half of a normal timeout.
+	const S32 MAX_EVENT_POLL_HTTP_ERRORS = 10; // ~5 minutes, by the above rules.
+
 	class LLEventPollResponder : public LLHTTPClient::Responder
 	{
 	public:
@@ -48,15 +58,21 @@ namespace
 		static LLHTTPClient::ResponderPtr start(const std::string& pollURL, const LLHost& sender);
 		void stop();
 		
+		void makeRequest();
+
 	private:
 		LLEventPollResponder(const std::string&	pollURL, const LLHost& sender);
 		~LLEventPollResponder();
 
-		void makeRequest();
+		
 		void handleMessage(const LLSD& content);
 		virtual	void error(U32 status, const std::string& reason);
 		virtual	void result(const LLSD&	content);
 
+		virtual void completedRaw(U32 status,
+									const std::string& reason,
+									const LLChannelDescriptors& channels,
+									const LLIOPipe::buffer_ptr_t& buffer);
 	private:
 
 		bool	mDone;
@@ -69,6 +85,27 @@ namespace
 		// these are only here for debugging so	we can see which poller	is which
 		static int sCount;
 		int	mCount;
+		S32 mErrorCount;
+	};
+
+	class LLEventPollEventTimer : public LLEventTimer
+	{
+		typedef boost::intrusive_ptr<LLEventPollResponder> EventPollResponderPtr;
+
+	public:
+		LLEventPollEventTimer(F32 period, EventPollResponderPtr responder)
+			: LLEventTimer(period), mResponder(responder)
+		{ }
+
+		virtual BOOL tick()
+		{
+			mResponder->makeRequest();
+			return TRUE;	// Causes this instance to be deleted.
+		}
+
+	private:
+		
+		EventPollResponderPtr mResponder;
 	};
 
 	//static
@@ -94,7 +131,8 @@ namespace
 	LLEventPollResponder::LLEventPollResponder(const std::string& pollURL, const LLHost& sender)
 		: mDone(false),
 		  mPollURL(pollURL),
-		  mCount(++sCount)
+		  mCount(++sCount),
+		  mErrorCount(0)
 	{
 		//extract host and port of simulator to set as sender
 		LLViewerRegion *regionp = gAgent.getRegion();
@@ -114,6 +152,24 @@ namespace
 				 <<	mPollURL <<	llendl;
 	}
 
+	// virtual 
+	void LLEventPollResponder::completedRaw(U32 status,
+									const std::string& reason,
+									const LLChannelDescriptors& channels,
+									const LLIOPipe::buffer_ptr_t& buffer)
+	{
+		if (status == HTTP_BAD_GATEWAY)
+		{
+			// These errors are not parsable as LLSD, 
+			// which LLHTTPClient::Responder::completedRaw will try to do.
+			completed(status, reason, LLSD());
+		}
+		else
+		{
+			LLHTTPClient::Responder::completedRaw(status,reason,channels,buffer);
+		}
+	}
+
 	void LLEventPollResponder::makeRequest()
 	{
 		LLSD request;
@@ -139,16 +195,37 @@ namespace
 	{
 		if (mDone) return;
 
-		if(status != 499)
+		// A HTTP_BAD_GATEWAY (502) error is our standard timeout response
+		// we get this when there are no events.
+		if ( status == HTTP_BAD_GATEWAY )	
+		{
+			mErrorCount = 0;
+			makeRequest();
+		}
+		else if (mErrorCount < MAX_EVENT_POLL_HTTP_ERRORS)
+		{
+			++mErrorCount;
+			
+			// The 'tick' will return TRUE causing the timer to delete this.
+			new LLEventPollEventTimer(EVENT_POLL_ERROR_RETRY_SECONDS
+										+ mErrorCount * EVENT_POLL_ERROR_RETRY_SECONDS_INC
+									, this);
+
+			llwarns << "Unexpected HTTP error.  status: " << status << ", reason: " << reason << llendl;
+		}
+		else
 		{
 			llwarns <<	"LLEventPollResponder::error: <" << mCount << "> got "
 					<<	status << ": " << reason
 					<<	(mDone ? " -- done"	: "") << llendl;
 			stop();
-			return;
-		}
 
-		makeRequest();
+			// At this point we have given up and the viewer will not receive HTTP messages from the simulator.
+			// IMs, teleports, about land, selecing land, region crossing and more will all fail.
+			// They are essentially disconnected from the region even though some things may still work.
+			// Since things won't get better until they relog we force a disconnect now.
+			LLAppViewer::instance()->forceDisconnect("You have been disconnected from the region you were in.");
+		}
 	}
 
 	//virtual
@@ -159,10 +236,13 @@ namespace
 		
 		if (mDone) return;
 
+		mErrorCount = 0;
+
 		if (!content.get("events") ||
 			!content.get("id"))
 		{
 			llwarns << "received event poll with no events or id key" << llendl;
+			makeRequest();
 			return;
 		}
 		
@@ -192,10 +272,13 @@ namespace
 	}	
 }
 
-LLEventPoll::LLEventPoll(const std::string&	pollURL, const LLHost& sender)
-	: mImpl(LLEventPollResponder::start(pollURL, sender))
+LLEventPoll::LLEventPoll(const std::string&	poll_url, const LLHost& sender)
+	: mImpl(LLEventPollResponder::start(poll_url, sender))
 	{ }
 
 LLEventPoll::~LLEventPoll()
 {
+	LLHTTPClient::Responder* responderp = mImpl.get();
+	LLEventPollResponder* event_poll_responder = dynamic_cast<LLEventPollResponder*>(responderp);
+	if (event_poll_responder) event_poll_responder->stop();
 }