FIX STORM-1325 Retry bake uploads in case of upload failure

4 files changed, 66 insertions, 30 deletions

diff --git a/indra/newview/llassetuploadresponders.cpp b/indra/newview/llassetuploadresponders.cpp
index c08771c5e7..e1d3de8409 100644
--- a/indra/newview/llassetuploadresponders.cpp
+++ b/indra/newview/llassetuploadresponders.cpp
@@ -449,7 +449,7 @@ void LLSendTexLayerResponder::uploadComplete(const LLSD& content)
 	std::string result = content["state"];
 	LLUUID new_id = content["new_asset"];
 
-	llinfos << "result: " << result << "new_id:" << new_id << llendl;
+	llinfos << "result: " << result << " new_id: " << new_id << llendl;
 	if (result == "complete"
 		&& mBakedUploadData != NULL)
 	{	// Invoke 
diff --git a/indra/newview/llassetuploadresponders.h b/indra/newview/llassetuploadresponders.h
index 70871b62e2..381b919c4a 100644
--- a/indra/newview/llassetuploadresponders.h
+++ b/indra/newview/llassetuploadresponders.h
@@ -112,6 +112,7 @@ private:
 struct LLBakedUploadData;
 class LLSendTexLayerResponder : public LLAssetUploadResponder
 {
+	LOG_CLASS(LLSendTexLayerResponder);
 public:
 	LLSendTexLayerResponder(const LLSD& post_data,
 							const LLUUID& vfile_id,
diff --git a/indra/newview/lltexlayer.cpp b/indra/newview/lltexlayer.cpp
index 500c2a7b86..bd41aa64f0 100644
--- a/indra/newview/lltexlayer.cpp
+++ b/indra/newview/lltexlayer.cpp
@@ -51,6 +51,9 @@
 
 using namespace LLVOAvatarDefines;
 
+static const S32 BAKE_UPLOAD_ATTEMPTS = 7;
+static const F32 BAKE_UPLOAD_RETRY_DELAY = 2.f; // actual delay grows by power of 2 each attempt
+
 class LLTexLayerInfo
 {
 	friend class LLTexLayer;
@@ -93,11 +96,13 @@ private:
 //-----------------------------------------------------------------------------
 LLBakedUploadData::LLBakedUploadData(const LLVOAvatarSelf* avatar,
 									 LLTexLayerSet* layerset,
-									 const LLUUID& id) : 
+									 const LLUUID& id,
+									 bool highest_res) :
 	mAvatar(avatar),
 	mTexLayerSet(layerset),
 	mID(id),
-	mStartTime(LLFrameTimer::getTotalTime())		// Record starting time
+	mStartTime(LLFrameTimer::getTotalTime()),		// Record starting time
+	mIsHighestRes(highest_res)
 { 
 }
 
@@ -116,6 +121,7 @@ LLTexLayerSetBuffer::LLTexLayerSetBuffer(LLTexLayerSet* const owner,
 	mUploadPending(FALSE), // Not used for any logic here, just to sync sending of updates
 	mNeedsUpload(FALSE),
 	mNumLowresUploads(0),
+	mUploadFailCount(0),
 	mNeedsUpdate(TRUE),
 	mNumLowresUpdates(0),
 	mTexLayerSet(owner)
@@ -204,6 +210,7 @@ void LLTexLayerSetBuffer::cancelUpload()
 	mNeedsUpload = FALSE;
 	mUploadPending = FALSE;
 	mNeedsUploadTimer.pause();
+	mUploadRetryTimer.reset();
 }
 
 void LLTexLayerSetBuffer::pushProjection() const
@@ -356,25 +363,38 @@ BOOL LLTexLayerSetBuffer::isReadyToUpload() const
 	if (!gAgentQueryManager.hasNoPendingQueries()) return FALSE; // Can't upload if there are pending queries.
 	if (isAgentAvatarValid() && !gAgentAvatarp->isUsingBakedTextures()) return FALSE; // Don't upload if avatar is using composites.
 
-	// If we requested an upload and have the final LOD ready, then upload.
-	if (mTexLayerSet->isLocalTextureDataFinal()) return TRUE;
-
-	// Upload if we've hit a timeout.  Upload is a pretty expensive process so we need to make sure
-	// we aren't doing uploads too frequently.
-	const U32 texture_timeout = gSavedSettings.getU32("AvatarBakedTextureUploadTimeout");
-	if (texture_timeout != 0)
+	BOOL ready = FALSE;
+	if (mTexLayerSet->isLocalTextureDataFinal())
+	{
+		// If we requested an upload and have the final LOD ready, upload (or wait a while if this is a retry)
+		if (mUploadFailCount == 0)
+		{
+			ready = TRUE;
+		}
+		else
+		{
+			ready = mUploadRetryTimer.getElapsedTimeF32() >= BAKE_UPLOAD_RETRY_DELAY * (1 << (mUploadFailCount - 1));
+		}
+	}
+	else
 	{
-		// The timeout period increases exponentially between every lowres upload in order to prevent
-		// spamming the server with frequent uploads.
-		const U32 texture_timeout_threshold = texture_timeout*(1 << mNumLowresUploads);
+		// Upload if we've hit a timeout.  Upload is a pretty expensive process so we need to make sure
+		// we aren't doing uploads too frequently.
+		const U32 texture_timeout = gSavedSettings.getU32("AvatarBakedTextureUploadTimeout");
+		if (texture_timeout != 0)
+		{
+			// The timeout period increases exponentially between every lowres upload in order to prevent
+			// spamming the server with frequent uploads.
+			const U32 texture_timeout_threshold = texture_timeout*(1 << mNumLowresUploads);
 
-		// If we hit our timeout and have textures available at even lower resolution, then upload.
-		const BOOL is_upload_textures_timeout = mNeedsUploadTimer.getElapsedTimeF32() >= texture_timeout_threshold;
-		const BOOL has_lower_lod = mTexLayerSet->isLocalTextureDataAvailable();
-		if (has_lower_lod && is_upload_textures_timeout) return TRUE; 
+			// If we hit our timeout and have textures available at even lower resolution, then upload.
+			const BOOL is_upload_textures_timeout = mNeedsUploadTimer.getElapsedTimeF32() >= texture_timeout_threshold;
+			const BOOL has_lower_lod = mTexLayerSet->isLocalTextureDataAvailable();
+			ready = has_lower_lod && is_upload_textures_timeout;
+		}
 	}
 
-	return FALSE;
+	return ready;
 }
 
 BOOL LLTexLayerSetBuffer::isReadyToUpdate() const
@@ -482,17 +502,20 @@ void LLTexLayerSetBuffer::doUpload()
 			
 			if (valid)
 			{
+				const bool highest_lod = mTexLayerSet->isLocalTextureDataFinal();
 				// Baked_upload_data is owned by the responder and deleted after the request completes.
 				LLBakedUploadData* baked_upload_data = new LLBakedUploadData(gAgentAvatarp, 
 																			 this->mTexLayerSet, 
-																			 asset_id);
+																			 asset_id,
+																			 highest_lod);
 				// upload ID is used to avoid overlaps, e.g. when the user rapidly makes two changes outside of Face Edit.
 				mUploadID = asset_id;
 
 				// Upload the image
 				const std::string url = gAgent.getRegion()->getCapability("UploadBakedTexture");
 				if(!url.empty()
-					&& !LLPipeline::sForceOldBakedUpload) // toggle debug setting UploadBakedTexOld to change between the new caps method and old method
+					&& !LLPipeline::sForceOldBakedUpload // toggle debug setting UploadBakedTexOld to change between the new caps method and old method
+					&& (mUploadFailCount < (BAKE_UPLOAD_ATTEMPTS - 1))) // Try last ditch attempt via asset store if cap upload is failing.
 				{
 					LLSD body = LLSD::emptyMap();
 					// The responder will call LLTexLayerSetBuffer::onTextureUploadComplete()
@@ -511,7 +534,6 @@ void LLTexLayerSetBuffer::doUpload()
 					llinfos << "Baked texture upload via Asset Store." <<  llendl;
 				}
 
-				const BOOL highest_lod = mTexLayerSet->isLocalTextureDataFinal();	
 				if (highest_lod)
 				{
 					// Sending the final LOD for the baked texture.  All done, pause 
@@ -603,14 +625,15 @@ void LLTexLayerSetBuffer::onTextureUploadComplete(const LLUUID& uuid,
 {
 	LLBakedUploadData* baked_upload_data = (LLBakedUploadData*)userdata;
 
-	if ((result == 0) &&
-		isAgentAvatarValid() &&
+	if (isAgentAvatarValid() &&
 		!gAgentAvatarp->isDead() &&
 		(baked_upload_data->mAvatar == gAgentAvatarp) && // Sanity check: only the user's avatar should be uploading textures.
 		(baked_upload_data->mTexLayerSet->hasComposite()))
 	{
 		LLTexLayerSetBuffer* layerset_buffer = baked_upload_data->mTexLayerSet->getComposite();
-			
+		S32 failures = layerset_buffer->mUploadFailCount;
+		layerset_buffer->mUploadFailCount = 0;
+
 		if (layerset_buffer->mUploadID.isNull())
 		{
 			// The upload got canceled, we should be in the
@@ -626,20 +649,28 @@ void LLTexLayerSetBuffer::onTextureUploadComplete(const LLUUID& uuid,
 		{
 			// This is the upload we're currently waiting for.
 			layerset_buffer->mUploadID.setNull();
+			const std::string name(baked_upload_data->mTexLayerSet->getBodyRegionName());
+			const std::string resolution = baked_upload_data->mIsHighestRes ? " full res " : " low res ";
 			if (result >= 0)
 			{
-				layerset_buffer->mUploadPending = FALSE;
+				layerset_buffer->mUploadPending = FALSE; // Allows sending of AgentSetAppearance later
 				LLVOAvatarDefines::ETextureIndex baked_te = gAgentAvatarp->getBakedTE(layerset_buffer->mTexLayerSet);
 				// Update baked texture info with the new UUID
 				U64 now = LLFrameTimer::getTotalTime();		// Record starting time
-				llinfos << "Baked texture upload took " << (S32)((now - baked_upload_data->mStartTime) / 1000) << " ms" << llendl;
+				llinfos << "Baked" << resolution << "texture upload for " << name << " took " << (S32)((now - baked_upload_data->mStartTime) / 1000) << " ms" << llendl;
 				gAgentAvatarp->setNewBakedTexture(baked_te, uuid);
 			}
 			else
 			{	
-				// Avatar appearance is changing, ignore the upload results
-				llinfos << "Baked upload failed. Reason: " << result << llendl;
-				// *FIX: retry upload after n seconds, asset server could be busy
+				++failures;
+				S32 max_attempts = baked_upload_data->mIsHighestRes ? BAKE_UPLOAD_ATTEMPTS : 1; // only retry final bakes
+				llwarns << "Baked" << resolution << "texture upload for " << name << " failed (attempt " << failures << "/" << max_attempts << ")" << llendl;
+				if (failures < max_attempts)
+				{
+					layerset_buffer->mUploadFailCount = failures;
+					layerset_buffer->mUploadRetryTimer.start();
+					layerset_buffer->requestUpload();
+				}
 			}
 		}
 		else
diff --git a/indra/newview/lltexlayer.h b/indra/newview/lltexlayer.h
index 2d710d2dce..85dadb213c 100644
--- a/indra/newview/lltexlayer.h
+++ b/indra/newview/lltexlayer.h
@@ -312,6 +312,8 @@ private:
 	BOOL					mUploadPending; 				// Whether we have received back the new baked textures
 	LLUUID					mUploadID; 						// The current upload process (null if none).
 	LLFrameTimer    		mNeedsUploadTimer; 				// Tracks time since upload was requested and performed.
+	S32						mUploadFailCount;				// Number of consecutive upload failures
+	LLFrameTimer    		mUploadRetryTimer; 				// Tracks time since last upload failure.
 
 	//--------------------------------------------------------------------
 	// Updates
@@ -363,12 +365,14 @@ struct LLBakedUploadData
 {
 	LLBakedUploadData(const LLVOAvatarSelf* avatar, 
 					  LLTexLayerSet* layerset, 
-					  const LLUUID& id);
+					  const LLUUID& id,
+					  bool highest_res);
 	~LLBakedUploadData() {}
 	const LLUUID				mID;
 	const LLVOAvatarSelf*		mAvatar; // note: backlink only; don't LLPointer 
 	LLTexLayerSet*				mTexLayerSet;
    	const U64					mStartTime;	// for measuring baked texture upload time
+   	const bool					mIsHighestRes; // whether this is a "final" bake, or intermediate low res
 };
 
 #endif  // LL_LLTEXLAYER_H