SH-4061 FIX - texture fetch failures added retry logic and fault injection for testing

5 files changed, 55 insertions, 16 deletions

diff --git a/indra/newview/app_settings/settings.xml b/indra/newview/app_settings/settings.xml
index f66d8fca5b..0b7f99ebc1 100755
--- a/indra/newview/app_settings/settings.xml
+++ b/indra/newview/app_settings/settings.xml
@@ -11130,6 +11130,17 @@
       <key>Value</key>
       <integer>0</integer>
     </map>
+  <key>TextureFetchFakeFailures</key>
+  <map>
+    <key>Comment</key>
+    <string>Simulate HTTP fetch failures for some server bake textures.</string>
+    <key>Persist</key>
+    <integer>1</integer>
+    <key>Type</key>
+    <string>Boolean</string>
+    <key>Value</key>
+    <integer>0</integer>
+  </map>
     <key>TextureFetchSource</key>
     <map>
       <key>Comment</key>
diff --git a/indra/newview/lltexturefetch.cpp b/indra/newview/lltexturefetch.cpp
index cc6dc64626..2e6fb160d5 100755
--- a/indra/newview/lltexturefetch.cpp
+++ b/indra/newview/lltexturefetch.cpp
@@ -388,10 +388,12 @@ public:
 	// Threads:  Ttf
 	virtual void onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * response);
 	
+	void setFakeFailure(bool fake_failure) { mFakeFailure = fake_failure; }
+
 protected:
 	LLTextureFetchWorker(LLTextureFetch* fetcher, FTType f_type,
 						 const std::string& url, const LLUUID& id, const LLHost& host,
-						 F32 priority, S32 discard, S32 size);
+						 F32 priority, S32 discard, S32 size, bool fake_failure);
 
 private:
 
@@ -547,6 +549,7 @@ private:
 	S32 mActiveCount;
 	LLCore::HttpStatus mGetStatus;
 	std::string mGetReason;
+	bool mFakeFailure;
 	
 	// Work Data
 	LLMutex mWorkMutex;
@@ -836,7 +839,8 @@ LLTextureFetchWorker::LLTextureFetchWorker(LLTextureFetch* fetcher,
 										   const LLHost& host,	// Simulator host
 										   F32 priority,		// Priority
 										   S32 discard,			// Desired discard
-										   S32 size)			// Desired size
+										   S32 size,			// Desired size
+										   bool fake_failure)   // For testing, simulate http failure if true.
 	: LLWorkerClass(fetcher, "TextureFetch"),
 	  LLCore::HttpHandler(),
 	  mState(INIT),
@@ -889,7 +893,8 @@ LLTextureFetchWorker::LLTextureFetchWorker(LLTextureFetch* fetcher,
 	  mHttpHasResource(false),
 	  mCacheReadCount(0U),
 	  mCacheWriteCount(0U),
-	  mResourceWaitCount(0U)
+	  mResourceWaitCount(0U),
+	  mFakeFailure(fake_failure)
 {
 	mCanUseNET = mUrl.empty() ;
 	
@@ -1519,15 +1524,16 @@ bool LLTextureFetchWorker::doWork(S32 param)
 			{
 				if (http_not_found == mGetStatus)
 				{
+					llwarns << "Texture missing from server (404): " << mUrl << llendl;
+
 					if(mWriteToCacheState == NOT_WRITE) //map tiles
 					{
 						setState(DONE);
 						releaseHttpSemaphore();
-						LL_DEBUGS("Texture") << mID << " abort: WAIT_HTTP_REQ not found" << llendl;
+						LL_WARNS("Texture") << mID << " abort: WAIT_HTTP_REQ not found" << llendl;
 						return true; // failed, means no map tile on the empty region.
 					}
 
-					llwarns << "Texture missing from server (404): " << mUrl << llendl;
 
 					// roll back to try UDP
 					if (mCanUseNET)
@@ -1891,6 +1897,11 @@ void LLTextureFetchWorker::onCompleted(LLCore::HttpHandle handle, LLCore::HttpRe
 		mFetcher->mTextureInfo.setRequestCompleteTimeAndLog(mID, timeNow);
 	}
 
+	if (mFakeFailure)
+	{
+		llwarns << "For debugging, setting fake failure status for texture " << mID << llendl;
+		response->setStatus(LLCore::HttpStatus(404));
+	}
 	bool success = true;
 	bool partial = false;
 	LLCore::HttpStatus status(response->getStatus());
@@ -2455,7 +2466,7 @@ LLTextureFetch::~LLTextureFetch()
 }
 
 bool LLTextureFetch::createRequest(FTType f_type, const std::string& url, const LLUUID& id, const LLHost& host, F32 priority,
-								   S32 w, S32 h, S32 c, S32 desired_discard, bool needs_aux, bool can_use_http)
+								   S32 w, S32 h, S32 c, S32 desired_discard, bool needs_aux, bool can_use_http, bool fake_failure)
 {
 	if(mFetcherLocked)
 	{
@@ -2523,7 +2534,8 @@ bool LLTextureFetch::createRequest(FTType f_type, const std::string& url, const
 		worker->mNeedsAux = needs_aux;
 		worker->setImagePriority(priority);
 		worker->setDesiredDiscard(desired_discard, desired_size);
-		worker->setCanUseHTTP(can_use_http) ;
+		worker->setCanUseHTTP(can_use_http);
+		worker->setFakeFailure(fake_failure);
 		if (!worker->haveWork())
 		{
 			worker->setState(LLTextureFetchWorker::INIT);
@@ -2538,7 +2550,7 @@ bool LLTextureFetch::createRequest(FTType f_type, const std::string& url, const
 	}
 	else
 	{
-		worker = new LLTextureFetchWorker(this, f_type, url, id, host, priority, desired_discard, desired_size);
+		worker = new LLTextureFetchWorker(this, f_type, url, id, host, priority, desired_discard, desired_size, fake_failure);
 		lockQueue();													// +Mfq
 		mRequestMap[id] = worker;
 		unlockQueue();													// -Mfq
diff --git a/indra/newview/lltexturefetch.h b/indra/newview/lltexturefetch.h
index 902a3d7a25..b99480d18e 100644..100755
--- a/indra/newview/lltexturefetch.h
+++ b/indra/newview/lltexturefetch.h
@@ -78,7 +78,7 @@ public:
 
 	// Threads:  T* (but Tmain mostly)
 	bool createRequest(FTType f_type, const std::string& url, const LLUUID& id, const LLHost& host, F32 priority,
-					   S32 w, S32 h, S32 c, S32 discard, bool needs_aux, bool can_use_http);
+					   S32 w, S32 h, S32 c, S32 discard, bool needs_aux, bool can_use_http, bool fake_failure);
 
 	// Requests that a fetch operation be deleted from the queue.
 	// If @cancel is true, also stops any I/O operations pending.
diff --git a/indra/newview/llviewertexture.cpp b/indra/newview/llviewertexture.cpp
index 9cca8a244e..a157600cc9 100755
--- a/indra/newview/llviewertexture.cpp
+++ b/indra/newview/llviewertexture.cpp
@@ -962,6 +962,8 @@ void LLViewerFetchedTexture::init(bool firstinit)
 	// does not contain this image.
 	mIsMissingAsset = FALSE;
 
+	mFetchFailureCount = 0;
+
 	mLoadedCallbackDesiredDiscardLevel = S8_MAX;
 	mPauseLoadedCallBacks = FALSE ;
 
@@ -1823,12 +1825,18 @@ bool LLViewerFetchedTexture::updateFetch()
 				// We finished but received no data
 				if (current_discard < 0)
 				{
-					llwarns << "!mIsFetching, setting as missing, decode_priority " << decode_priority
-							<< " mRawDiscardLevel " << mRawDiscardLevel
-							<< " current_discard " << current_discard
-							<< llendl;
-					setIsMissingAsset();
-					desired_discard = -1;
+					const S32 MAX_FETCH_FAILURE = 1;
+					mFetchFailureCount++;
+					llwarns << "Fetch failure for " << mID << " failure count " << mFetchFailureCount << llendl;
+					if (getFTType() != FTT_SERVER_BAKE || mFetchFailureCount >= MAX_FETCH_FAILURE)
+					{
+						llwarns << "!mIsFetching, setting as missing, decode_priority " << decode_priority
+								<< " mRawDiscardLevel " << mRawDiscardLevel
+								<< " current_discard " << current_discard
+								<< llendl;
+						setIsMissingAsset();
+						desired_discard = -1;
+					}
 				}
 				else
 				{
@@ -1942,8 +1950,14 @@ bool LLViewerFetchedTexture::updateFetch()
 		
 		// bypass texturefetch directly by pulling from LLTextureCache
 		bool fetch_request_created = false;
+		bool fake_failure = false;
+		const bool debug_setting_fake_failures = gSavedSettings.getBOOL("TextureFetchFakeFailures");
+		if (getFTType() == FTT_SERVER_BAKE && mFetchFailureCount == 0 && debug_setting_fake_failures)
+		{
+			fake_failure = true;
+		}
 		fetch_request_created = LLAppViewer::getTextureFetch()->createRequest(mFTType, mUrl, getID(), getTargetHost(), decode_priority,
-																			  w, h, c, desired_discard, needsAux(), mCanUseHTTP);
+																			  w, h, c, desired_discard, needsAux(), mCanUseHTTP, fake_failure);
 		
 		if (fetch_request_created)
 		{
@@ -2040,6 +2054,7 @@ void LLViewerFetchedTexture::setIsMissingAsset(BOOL is_missing)
 	else
 	{
 		llinfos << mID << ": un-flagging missing asset" << llendl;
+		mFetchFailureCount = 0;
 	}
 	mIsMissingAsset = is_missing;
 }
diff --git a/indra/newview/llviewertexture.h b/indra/newview/llviewertexture.h
index 9465fe180b..320e6f8630 100755
--- a/indra/newview/llviewertexture.h
+++ b/indra/newview/llviewertexture.h
@@ -450,6 +450,7 @@ protected:
 
 	FTType mFTType; // What category of image is this - map tile, server bake, etc?
 	mutable BOOL mIsMissingAsset;		// True if we know that there is no image asset with this image id in the database.		
+	S32 mFetchFailureCount; // How many times has a fetch failed in a way that suggests the asset is missing?
 
 	typedef std::list<LLLoadedCallbackEntry*> callback_list_t;
 	S8              mLoadedCallbackDesiredDiscardLevel;