SH-4061 WIP - moved retry policy to newview so it can work with either llmessage or CoreHttp libraries. Updated tests.

5 files changed, 492 insertions, 1 deletions

diff --git a/indra/newview/CMakeLists.txt b/indra/newview/CMakeLists.txt
index 05736f6360..27dbe15005 100644..100755
--- a/indra/newview/CMakeLists.txt
+++ b/indra/newview/CMakeLists.txt
@@ -299,6 +299,7 @@ set(viewer_SOURCE_FILES
     llgroupmgr.cpp
     llhints.cpp
     llhomelocationresponder.cpp
+    llhttpretrypolicy.cpp
     llhudeffect.cpp
     llhudeffectbeam.cpp
     llhudeffectlookat.cpp
@@ -877,6 +878,7 @@ set(viewer_HEADER_FILES
     llgrouplist.h
     llgroupmgr.h
     llhints.h
+    llhttpretrypolicy.h
     llhomelocationresponder.h
     llhudeffect.h
     llhudeffectbeam.h
@@ -2152,6 +2154,7 @@ if (LL_TESTS)
 
   set(test_libs
     ${LLMESSAGE_LIBRARIES}
+    ${LLCOREHTTP_LIBRARIES}
     ${WINDOWS_LIBRARIES}
     ${LLVFS_LIBRARIES}
     ${LLMATH_LIBRARIES}
@@ -2197,6 +2200,8 @@ if (LL_TESTS)
     "${test_libs}"
     )
 
+  LL_ADD_INTEGRATION_TEST(llhttpretrypolicy "llhttpretrypolicy.cpp" "${test_libs}")
+
   #ADD_VIEWER_BUILD_TEST(llmemoryview viewer)
   #ADD_VIEWER_BUILD_TEST(llagentaccess viewer)
   #ADD_VIEWER_BUILD_TEST(lltextureinfo viewer)
diff --git a/indra/newview/llhttpretrypolicy.cpp b/indra/newview/llhttpretrypolicy.cpp
new file mode 100755
index 0000000000..5c6dabbe99
--- /dev/null
+++ b/indra/newview/llhttpretrypolicy.cpp
@@ -0,0 +1,117 @@
+/** 
+ * @file llhttpretrypolicy.h
+ * @brief Header for a retry policy class intended for use with http responders.
+ *
+ * $LicenseInfo:firstyear=2013&license=viewerlgpl$
+ * Second Life Viewer Source Code
+ * Copyright (C) 2013, Linden Research, Inc.
+ * 
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation;
+ * version 2.1 of the License only.
+ * 
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ * 
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ * 
+ * Linden Research, Inc., 945 Battery Street, San Francisco, CA  94111  USA
+ * $/LicenseInfo$
+ */
+
+#include "llviewerprecompiledheaders.h"
+
+#include "llhttpretrypolicy.h"
+
+bool LLAdaptiveRetryPolicy::getRetryAfter(const LLSD& headers, F32& retry_header_time)
+{
+	return (headers.has(HTTP_IN_HEADER_RETRY_AFTER)
+			&& getSecondsUntilRetryAfter(headers[HTTP_IN_HEADER_RETRY_AFTER].asStringRef(), retry_header_time));
+}
+
+bool LLAdaptiveRetryPolicy::getRetryAfter(const LLCore::HttpHeaders *headers, F32& retry_header_time)
+{
+	// Look for matching header. Hopefully it's correct enough to let
+	// us extract the field we are looking for. Does not purport to be
+	// in any way a viable general HTTP header parser.
+	if (headers)
+	{
+		for (std::vector<std::string>::const_iterator it = headers->mHeaders.begin();
+			 it != headers->mHeaders.end();
+			 ++it)
+		{
+			const std::string& str = *it;
+			const std::string match = HTTP_IN_HEADER_RETRY_AFTER + ":";
+			size_t pos = str.find(match);
+			if ((pos != std::string::npos) &&
+				(pos+match.length() <= str.length()))
+			{
+				retry_header_time = strtod(str.substr(pos+match.length()).c_str(), NULL);
+				return true;
+			}
+		}
+	}
+	return false;
+}
+
+void LLAdaptiveRetryPolicy::onFailure(S32 status, const LLSD& headers)
+{
+	F32 retry_header_time;
+	bool has_retry_header_time = getRetryAfter(headers,retry_header_time);
+	onFailureCommon(status, has_retry_header_time, retry_header_time);
+}
+  
+// TODO: replace this parsing junk once CoreHttp has its own header parsing capabilities.
+void LLAdaptiveRetryPolicy::onFailure(const LLCore::HttpResponse *response)
+{
+	F32 retry_header_time;
+	const LLCore::HttpHeaders *headers = response->getHeaders();
+	bool has_retry_header_time = getRetryAfter(headers,retry_header_time);
+	onFailureCommon(response->getStatus().mType, has_retry_header_time, retry_header_time);
+}
+
+void LLAdaptiveRetryPolicy::onFailureCommon(S32 status, bool has_retry_header_time, F32 retry_header_time)
+{
+	if (mRetryCount > 0)
+	{
+		mDelay = llclamp(mDelay*mBackoffFactor,mMinDelay,mMaxDelay);
+	}
+	// Honor server Retry-After header.
+	// Status 503 may ask us to wait for a certain amount of time before retrying.
+	F32 wait_time = mDelay;
+	if (has_retry_header_time)
+	{
+		wait_time = retry_header_time;
+	}
+
+	if (mRetryCount>=mMaxRetries)
+	{
+		llinfos << "Too many retries " << mRetryCount << ", will not retry" << llendl;
+		mShouldRetry = false;
+	}
+	if (!isHttpServerErrorStatus(status))
+	{
+		llinfos << "Non-server error " << status << ", will not retry" << llendl;
+		mShouldRetry = false;
+	}
+	if (mShouldRetry)
+	{
+		llinfos << "Retry count " << mRetryCount << " should retry after " << wait_time << llendl;
+		mRetryTimer.reset();
+		mRetryTimer.setTimerExpirySec(wait_time);
+	}
+	mRetryCount++;
+}
+	
+
+bool LLAdaptiveRetryPolicy::shouldRetry(F32& seconds_to_wait) const
+{
+	llassert(mRetryCount>0); // have to call onFailure() before shouldRetry()
+	seconds_to_wait = mShouldRetry ? mRetryTimer.getRemainingTimeF32() : F32_MAX;
+	return mShouldRetry;
+}
diff --git a/indra/newview/llhttpretrypolicy.h b/indra/newview/llhttpretrypolicy.h
new file mode 100755
index 0000000000..ca37e5f73c
--- /dev/null
+++ b/indra/newview/llhttpretrypolicy.h
@@ -0,0 +1,94 @@
+/** 
+ * @file file llhttpretrypolicy.h
+ * @brief declarations for http retry policy class.
+ *
+ * $LicenseInfo:firstyear=2013&license=viewerlgpl$
+ * Second Life Viewer Source Code
+ * Copyright (C) 2013, Linden Research, Inc.
+ * 
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation;
+ * version 2.1 of the License only.
+ * 
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ * 
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ * 
+ * Linden Research, Inc., 945 Battery Street, San Francisco, CA  94111  USA
+ * $/LicenseInfo$
+ */
+										
+#ifndef LL_RETRYPOLICY_H
+#define LL_RETRYPOLICY_H
+
+#include "lltimer.h"
+#include "llthread.h"
+
+#include "llhttpconstants.h"
+
+// For compatibility with new core http lib.
+#include "httpresponse.h"
+#include "httpheaders.h"
+
+// This is intended for use with HTTP Clients/Responders, but is not
+// specifically coupled with those classes.
+class LLHTTPRetryPolicy: public LLThreadSafeRefCount
+{
+public:
+	LLHTTPRetryPolicy() {}
+	virtual ~LLHTTPRetryPolicy() {}
+	// Call once after an HTTP failure to update state.
+	virtual void onFailure(S32 status, const LLSD& headers) = 0;
+
+	virtual void onFailure(const LLCore::HttpResponse *response) = 0;
+
+	virtual bool shouldRetry(F32& seconds_to_wait) const = 0;
+};
+
+// Very general policy with geometric back-off after failures,
+// up to a maximum delay, and maximum number of retries.
+class LLAdaptiveRetryPolicy: public LLHTTPRetryPolicy
+{
+public:
+	LLAdaptiveRetryPolicy(F32 min_delay, F32 max_delay, F32 backoff_factor, U32 max_retries):
+		mMinDelay(min_delay),
+		mMaxDelay(max_delay),
+		mBackoffFactor(backoff_factor),
+		mMaxRetries(max_retries),
+		mDelay(min_delay),
+		mRetryCount(0),
+		mShouldRetry(true)
+	{
+	}
+
+	// virtual
+	void onFailure(S32 status, const LLSD& headers);
+	// virtual
+	void onFailure(const LLCore::HttpResponse *response);
+	// virtual
+	bool shouldRetry(F32& seconds_to_wait) const;
+
+protected:
+	bool getRetryAfter(const LLSD& headers, F32& retry_header_time);
+	bool getRetryAfter(const LLCore::HttpHeaders *headers, F32& retry_header_time);
+	void onFailureCommon(S32 status, bool has_retry_header_time, F32 retry_header_time);
+
+private:
+
+	F32 mMinDelay; // delay never less than this value
+	F32 mMaxDelay; // delay never exceeds this value
+	F32 mBackoffFactor; // delay increases by this factor after each retry, up to mMaxDelay.
+	U32 mMaxRetries; // maximum number of times shouldRetry will return true.
+	F32 mDelay; // current default delay.
+	U32 mRetryCount; // number of times shouldRetry has been called.
+	LLTimer mRetryTimer; // time until next retry.
+	bool mShouldRetry; // Becomes false after too many retries, or the wrong sort of status received, etc.
+};
+
+#endif
diff --git a/indra/newview/lltexturefetch.h b/indra/newview/lltexturefetch.h
index c6bd342a7b..12226d51c8 100755
--- a/indra/newview/lltexturefetch.h
+++ b/indra/newview/lltexturefetch.h
@@ -397,7 +397,7 @@ private:
 	e_tex_source mOriginFetchSource;
 
 	// Retry logic
-	LLAdaptiveRetryPolicy mFetchRetryPolicy;
+	//LLAdaptiveRetryPolicy mFetchRetryPolicy;
 	
 public:
 	//debug use
diff --git a/indra/newview/tests/llhttpretrypolicy_test.cpp b/indra/newview/tests/llhttpretrypolicy_test.cpp
new file mode 100755
index 0000000000..39bd15d62f
--- /dev/null
+++ b/indra/newview/tests/llhttpretrypolicy_test.cpp
@@ -0,0 +1,275 @@
+/** 
+ * @file llhttpretrypolicy_test.cpp
+ * @brief Header tests to exercise the LLHTTPRetryPolicy classes.
+ *
+ * $LicenseInfo:firstyear=2013&license=viewerlgpl$
+ * Second Life Viewer Source Code
+ * Copyright (C) 2013, Linden Research, Inc.
+ * 
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation;
+ * version 2.1 of the License only.
+ * 
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ * 
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ * 
+ * Linden Research, Inc., 945 Battery Street, San Francisco, CA  94111  USA
+ * $/LicenseInfo$
+ */
+
+#include "../llviewerprecompiledheaders.h"
+#include "../llhttpretrypolicy.h"
+#include "lltut.h"
+
+namespace tut
+{
+struct TestData
+{
+};
+
+typedef test_group<TestData>	RetryPolicyTestGroup;
+typedef RetryPolicyTestGroup::object		RetryPolicyTestObject;
+RetryPolicyTestGroup retryPolicyTestGroup("retry_policy");
+
+template<> template<>
+void RetryPolicyTestObject::test<1>()
+{
+	LLAdaptiveRetryPolicy never_retry(1.0,1.0,1.0,0);
+	LLSD headers;
+	F32 wait_seconds;
+	
+	never_retry.onFailure(500,headers);
+	ensure("never retry", !never_retry.shouldRetry(wait_seconds)); 
+}
+
+template<> template<>
+void RetryPolicyTestObject::test<2>()
+{
+	LLAdaptiveRetryPolicy retry404(1.0,2.0,3.0,10);
+	LLSD headers;
+	F32 wait_seconds;
+	
+	retry404.onFailure(404,headers);
+	ensure("no retry on 404", !retry404.shouldRetry(wait_seconds)); 
+}
+
+template<> template<>
+void RetryPolicyTestObject::test<3>()
+{
+	// Should retry after 1.0, 2.0, 3.0, 3.0 seconds.
+	LLAdaptiveRetryPolicy basic_retry(1.0,3.0,2.0,4);
+	LLSD headers;
+	F32 wait_seconds;
+	bool should_retry;
+	U32 frac_bits = 6;
+
+	// Starting wait 1.0
+	basic_retry.onFailure(500,headers);
+	should_retry = basic_retry.shouldRetry(wait_seconds);
+	ensure("basic_retry 1", should_retry);
+	ensure_approximately_equals("basic_retry 1", wait_seconds, 1.0F, frac_bits);
+
+	// Double wait to 2.0
+	basic_retry.onFailure(500,headers);
+	should_retry = basic_retry.shouldRetry(wait_seconds);
+	ensure("basic_retry 2", should_retry);
+	ensure_approximately_equals("basic_retry 2", wait_seconds, 2.0F, frac_bits);
+
+	// Hit max wait of 3.0 (4.0 clamped to max 3)
+	basic_retry.onFailure(500,headers);
+	should_retry = basic_retry.shouldRetry(wait_seconds);
+	ensure("basic_retry 3", should_retry);
+	ensure_approximately_equals("basic_retry 3", wait_seconds, 3.0F, frac_bits);
+
+	// At max wait, should stay at 3.0
+	basic_retry.onFailure(500,headers);
+	should_retry = basic_retry.shouldRetry(wait_seconds);
+	ensure("basic_retry 4", should_retry);
+	ensure_approximately_equals("basic_retry 4", wait_seconds, 3.0F, frac_bits);
+
+	// Max retries, should fail now.
+	basic_retry.onFailure(500,headers);
+	should_retry = basic_retry.shouldRetry(wait_seconds);
+	ensure("basic_retry 5", !should_retry);
+}
+
+// Retries should stop as soon as a non-5xx error is received.
+template<> template<>
+void RetryPolicyTestObject::test<4>()
+{
+	// Should retry after 1.0, 2.0, 3.0, 3.0 seconds.
+	LLAdaptiveRetryPolicy killer404(1.0,3.0,2.0,4);
+	LLSD headers;
+	F32 wait_seconds;
+	bool should_retry;
+	U32 frac_bits = 6;
+
+	// Starting wait 1.0
+	killer404.onFailure(500,headers);
+	should_retry = killer404.shouldRetry(wait_seconds);
+	ensure("killer404 1", should_retry);
+	ensure_approximately_equals("killer404 1", wait_seconds, 1.0F, frac_bits);
+
+	// Double wait to 2.0
+	killer404.onFailure(500,headers);
+	should_retry = killer404.shouldRetry(wait_seconds);
+	ensure("killer404 2", should_retry);
+	ensure_approximately_equals("killer404 2", wait_seconds, 2.0F, frac_bits);
+
+	// Should fail on non-5xx
+	killer404.onFailure(404,headers);
+	should_retry = killer404.shouldRetry(wait_seconds);
+	ensure("killer404 3", !should_retry);
+
+	// After a non-5xx, should keep failing.
+	killer404.onFailure(500,headers);
+	should_retry = killer404.shouldRetry(wait_seconds);
+	ensure("killer404 4", !should_retry);
+}
+
+// Test handling of "retry-after" header. If present, this header
+// value overrides the computed delay, but does not affect the
+// progression of delay values.  For example, if the normal
+// progression of delays would be 1,2,4,8..., but the 2nd and 3rd calls
+// get a retry header of 33, the pattern would become 1,33,33,8...
+template<> template<>
+void RetryPolicyTestObject::test<5>()
+{
+	LLAdaptiveRetryPolicy policy(1.0,25.0,2.0,6);
+	LLSD headers_with_retry;
+	headers_with_retry[HTTP_IN_HEADER_RETRY_AFTER] = "666";
+	LLSD headers_without_retry;
+	F32 wait_seconds;
+	bool should_retry;
+	U32 frac_bits = 6;
+
+	policy.onFailure(500,headers_without_retry);
+	should_retry = policy.shouldRetry(wait_seconds);
+	ensure("retry header 1", should_retry);
+	ensure_approximately_equals("retry header 1", wait_seconds, 1.0F, frac_bits);
+
+	policy.onFailure(500,headers_without_retry);
+	should_retry = policy.shouldRetry(wait_seconds);
+	ensure("retry header 2", should_retry);
+	ensure_approximately_equals("retry header 2", wait_seconds, 2.0F, frac_bits);
+
+	policy.onFailure(500,headers_with_retry);
+	should_retry = policy.shouldRetry(wait_seconds);
+	ensure("retry header 3", should_retry);
+	// 4.0 overrides by header -> 666.0
+	ensure_approximately_equals("retry header 3", wait_seconds, 666.0F, frac_bits);
+
+	policy.onFailure(500,headers_with_retry);
+	should_retry = policy.shouldRetry(wait_seconds);
+	ensure("retry header 4", should_retry);
+	// 8.0 overrides by header -> 666.0
+	ensure_approximately_equals("retry header 4", wait_seconds, 666.0F, frac_bits);
+
+	policy.onFailure(500,headers_without_retry);
+	should_retry = policy.shouldRetry(wait_seconds);
+	ensure("retry header 5", should_retry);
+	ensure_approximately_equals("retry header 5", wait_seconds, 16.0F, frac_bits);
+
+	policy.onFailure(500,headers_without_retry);
+	should_retry = policy.shouldRetry(wait_seconds);
+	ensure("retry header 6", should_retry);
+	ensure_approximately_equals("retry header 6", wait_seconds, 25.0F, frac_bits);
+
+	policy.onFailure(500,headers_with_retry);
+	should_retry = policy.shouldRetry(wait_seconds);
+	ensure("retry header 7", !should_retry);
+}
+
+// Test getSecondsUntilRetryAfter(const std::string& retry_after, F32& seconds_to_wait),
+// used by header parsing of the retry policy.
+template<> template<>
+void RetryPolicyTestObject::test<6>()
+{
+	F32 seconds_to_wait;
+	bool success;
+
+	std::string str1("0");
+	seconds_to_wait = F32_MAX;
+	success = getSecondsUntilRetryAfter(str1, seconds_to_wait);
+	ensure("parse 1", success);
+	ensure_equals("parse 1", seconds_to_wait, 0.0);
+
+	std::string str2("999.9");
+	seconds_to_wait = F32_MAX;
+	success = getSecondsUntilRetryAfter(str2, seconds_to_wait);
+	ensure("parse 2", success);
+	ensure_approximately_equals("parse 2", seconds_to_wait, 999.9F, 8);
+
+	time_t nowseconds;
+	time(&nowseconds);
+	std::string str3 = LLDate((F64)nowseconds).asRFC1123();
+	seconds_to_wait = F32_MAX;
+	success = getSecondsUntilRetryAfter(str3, seconds_to_wait);
+	ensure("parse 3", success);
+	ensure_approximately_equals("parse 3", seconds_to_wait, 0.0F, 6);
+}
+
+// Test retry-after field in both llmessage and CoreHttp headers.
+template<> template<>
+void RetryPolicyTestObject::test<7>()
+{
+	LLSD sd_headers;
+	time_t nowseconds;
+	time(&nowseconds);
+	sd_headers[HTTP_IN_HEADER_RETRY_AFTER] = LLDate((F64)nowseconds).asRFC1123();
+	LLAdaptiveRetryPolicy policy(17.0,644.0,3.0,10);
+	F32 seconds_to_wait;
+	bool should_retry;
+
+	// no retry header, use default.
+	policy.onFailure(500,LLSD());
+	should_retry = policy.shouldRetry(seconds_to_wait);
+	ensure("header 1", should_retry);
+	ensure_approximately_equals("header 1", seconds_to_wait, 17.0F, 6);
+
+	// retry header should override, give delay of 0
+	policy.onFailure(503,sd_headers);
+	should_retry = policy.shouldRetry(seconds_to_wait);
+	ensure("header 2", should_retry);
+	ensure_approximately_equals("header 2", seconds_to_wait, 0.0F, 6);
+
+	// retry header in LLCore::HttpHeaders
+	{
+		LLCore::HttpResponse *response = new LLCore::HttpResponse();
+		LLCore::HttpHeaders *headers = new LLCore::HttpHeaders();
+		response->setStatus(503);
+		response->setHeaders(headers);
+		headers->mHeaders.push_back("retry-after: 600");
+		policy.onFailure(response);
+		should_retry = policy.shouldRetry(seconds_to_wait);
+		ensure("header 3",should_retry);
+		ensure_approximately_equals("header 3", seconds_to_wait, 600.0F, 6);
+		response->release();
+	}
+
+	// retry header in LLCore::HttpHeaders
+	{
+		LLCore::HttpResponse *response = new LLCore::HttpResponse();
+		LLCore::HttpHeaders *headers = new LLCore::HttpHeaders();
+		response->setStatus(503);
+		response->setHeaders(headers);
+		LLSD sd_headers;
+		time(&nowseconds);
+		headers->mHeaders.push_back("retry-after: " + LLDate((F64)nowseconds).asRFC1123());
+		policy.onFailure(response);
+		should_retry = policy.shouldRetry(seconds_to_wait);
+		ensure("header 3",should_retry);
+		ensure_approximately_equals("header 3", seconds_to_wait, 0.0F, 6);
+		response->release();
+	}
+}
+
+}
+