Implemented HTTP retry for requests. Went in rather easily which

surprised me.  Added a retry queue similar to ready queue to the
policy object which is sorted by retry time.  Currently do five
retries (after the initial try) delayed by .25, .5, 1, 2 and 5
seconds.  Removed the retry logic from the lltexturefetch module.
Upped the waiting time in the unit test for the retries.  People
won't like this but tough, need tests.

1 files changed, 73 insertions, 25 deletions

diff --git a/indra/llcorehttp/_httppolicy.cpp b/indra/llcorehttp/_httppolicy.cpp
index 51f5e487dc..1f4cd34a4b 100644
--- a/indra/llcorehttp/_httppolicy.cpp
+++ b/indra/llcorehttp/_httppolicy.cpp
@@ -24,39 +24,46 @@
  * $/LicenseInfo$
  */
 
+#include "linden_common.h"
+
 #include "_httppolicy.h"
 
 #include "_httpoprequest.h"
 #include "_httpservice.h"
 #include "_httplibcurl.h"
 
+#include "lltimer.h"
+
 
 namespace LLCore
 {
 
 HttpPolicy::HttpPolicy(HttpService * service)
 	: mService(service)
-{
-	for (int policy_class(0); policy_class < HttpRequest::POLICY_CLASS_LIMIT; ++policy_class)
-	{
-		mReadyInClass[policy_class] = 0;
-	}
-}
+{}
 
 
 HttpPolicy::~HttpPolicy()
 {
-	for (int policy_class(0); policy_class < HttpRequest::POLICY_CLASS_LIMIT; ++policy_class)
+	for (int policy_class(0); policy_class < LL_ARRAY_SIZE(mState); ++policy_class)
 	{
-		HttpReadyQueue & readyq(mReadyQueue[policy_class]);
+		HttpRetryQueue & retryq(mState[policy_class].mRetryQueue);
+		while (! retryq.empty())
+		{
+			HttpOpRequest * op(retryq.top());
 		
+			op->cancel();
+			op->release();
+			retryq.pop();
+		}
+
+		HttpReadyQueue & readyq(mState[policy_class].mReadyQueue);
 		while (! readyq.empty())
 		{
 			HttpOpRequest * op(readyq.top());
 		
 			op->cancel();
 			op->release();
-			mReadyInClass[policy_class]--;
 			readyq.pop();
 		}
 	}
@@ -68,27 +75,69 @@ void HttpPolicy::addOp(HttpOpRequest * op)
 {
 	const int policy_class(op->mReqPolicy);
 	
-	mReadyQueue[policy_class].push(op);
-	++mReadyInClass[policy_class];
+	op->mPolicyRetries = 0;
+	mState[policy_class].mReadyQueue.push(op);
+}
+
+
+void HttpPolicy::retryOp(HttpOpRequest * op)
+{
+	static const HttpTime retry_deltas[] =
+		{
+			 250000,			// 1st retry in 0.25 S, etc...
+			 500000,
+			1000000,
+			2000000,
+			5000000				// ... to every 5.0 S.
+		};
+	static const int delta_max(int(LL_ARRAY_SIZE(retry_deltas)) - 1);
+	
+	const HttpTime now(totalTime());
+	const int policy_class(op->mReqPolicy);
+	
+	const HttpTime delta(retry_deltas[llclamp(op->mPolicyRetries, 0, delta_max)]);
+	op->mPolicyRetryAt = now + delta;
+	++op->mPolicyRetries;
+	LL_WARNS("CoreHttp") << "URL op retry #" << op->mPolicyRetries
+						 << " being scheduled for " << delta << " uSecs from now."
+						 << LL_ENDL;
+	mState[policy_class].mRetryQueue.push(op);
 }
 
 
 HttpService::ELoopSpeed HttpPolicy::processReadyQueue()
 {
+	const HttpTime now(totalTime());
 	HttpService::ELoopSpeed result(HttpService::REQUEST_SLEEP);
 	HttpLibcurl & transport(mService->getTransport());
 	
-	for (int policy_class(0); policy_class < HttpRequest::POLICY_CLASS_LIMIT; ++policy_class)
+	for (int policy_class(0); policy_class < LL_ARRAY_SIZE(mState); ++policy_class)
 	{
-		HttpReadyQueue & readyq(mReadyQueue[policy_class]);
 		int active(transport.getActiveCountInClass(policy_class));
 		int needed(8 - active);
 
-		if (needed > 0 && mReadyInClass[policy_class] > 0)
+		HttpRetryQueue & retryq(mState[policy_class].mRetryQueue);
+		HttpReadyQueue & readyq(mState[policy_class].mReadyQueue);
+		
+		if (needed > 0)
 		{
-			// Scan ready queue for requests that match policy
-
-			while (! readyq.empty() && needed > 0 && mReadyInClass[policy_class] > 0)
+			// First see if we have any retries...
+			while (needed > 0 && ! retryq.empty())
+			{
+				HttpOpRequest * op(retryq.top());
+				if (op->mPolicyRetryAt > now)
+					break;
+			
+				retryq.pop();
+				
+				op->stageFromReady(mService);
+				op->release();
+					
+				--needed;
+			}
+		
+			// Now go on to the new requests...
+			while (needed > 0 && ! readyq.empty())
 			{
 				HttpOpRequest * op(readyq.top());
 				readyq.pop();
@@ -96,17 +145,16 @@ HttpService::ELoopSpeed HttpPolicy::processReadyQueue()
 				op->stageFromReady(mService);
 				op->release();
 					
-				--mReadyInClass[policy_class];
 				--needed;
 			}
 		}
-
-		if (! readyq.empty())
+				
+		if (! readyq.empty() || ! retryq.empty())
 		{
 			// If anything is ready, continue looping...
 			result = (std::min)(result, HttpService::NORMAL);
 		}
-	}
+	} // end foreach policy_class
 
 	return result;
 }
@@ -114,9 +162,9 @@ HttpService::ELoopSpeed HttpPolicy::processReadyQueue()
 
 bool HttpPolicy::changePriority(HttpHandle handle, HttpRequest::priority_t priority)
 {
-	for (int policy_class(0); policy_class < HttpRequest::POLICY_CLASS_LIMIT; ++policy_class)
+	for (int policy_class(0); policy_class < LL_ARRAY_SIZE(mState); ++policy_class)
 	{
-		HttpReadyQueue::container_type & c(mReadyQueue[policy_class].get_container());
+		HttpReadyQueue::container_type & c(mState[policy_class].mReadyQueue.get_container());
 	
 		// Scan ready queue for requests that match policy
 		for (HttpReadyQueue::container_type::iterator iter(c.begin()); c.end() != iter;)
@@ -126,9 +174,9 @@ bool HttpPolicy::changePriority(HttpHandle handle, HttpRequest::priority_t prior
 			if (static_cast<HttpHandle>(*cur) == handle)
 			{
 				HttpOpRequest * op(*cur);
-				c.erase(cur);							// All iterators are now invalidated
+				c.erase(cur);									// All iterators are now invalidated
 				op->mReqPriority = priority;
-				mReadyQueue[policy_class].push(op);		// Re-insert using adapter class
+				mState[policy_class].mReadyQueue.push(op);		// Re-insert using adapter class
 				return true;
 			}
 		}