MAINT-1958 Fix for crash on OSX when resizing window with deferred rendering enabled.

Reviewed by VoidPointer

4 files changed, 60 insertions, 12 deletions

diff --git a/indra/newview/app_settings/settings.xml b/indra/newview/app_settings/settings.xml
index 93362e2a55..2e91d10cd3 100644
--- a/indra/newview/app_settings/settings.xml
+++ b/indra/newview/app_settings/settings.xml
@@ -14082,5 +14082,17 @@
       <key>Value</key>
       <integer>0</integer>
     </map>
+
+  <key>SimulateFBOFailure</key>
+  <map>
+    <key>Comment</key>
+    <string>[DEBUG] Make allocateScreenBuffer return false.  Used to test error handling.</string>
+    <key>Persist</key>
+    <integer>0</integer>
+    <key>Type</key>
+    <string>Boolean</string>
+    <key>Value</key>
+    <integer>0</integer>
+  </map>
 </map>
 </llsd>
diff --git a/indra/newview/llfloaterpreference.cpp b/indra/newview/llfloaterpreference.cpp
index 5752f839ce..542e96cf16 100755
--- a/indra/newview/llfloaterpreference.cpp
+++ b/indra/newview/llfloaterpreference.cpp
@@ -750,7 +750,10 @@ void LLFloaterPreference::onClose(bool app_quitting)
 {
 	gSavedSettings.setS32("LastPrefTab", getChild<LLTabContainer>("pref core")->getCurrentPanelIndex());
 	LLPanelLogin::setAlwaysRefresh(false);
-	cancel();
+	if (!app_quitting)
+	{
+		cancel();
+	}
 }
 
 void LLFloaterPreference::onOpenHardwareSettings()
diff --git a/indra/newview/pipeline.cpp b/indra/newview/pipeline.cpp
index 9685e45348..2bcbc0b083 100644
--- a/indra/newview/pipeline.cpp
+++ b/indra/newview/pipeline.cpp
@@ -761,7 +761,16 @@ void LLPipeline::resizeScreenTexture()
 		GLuint resX = gViewerWindow->getWorldViewWidthRaw();
 		GLuint resY = gViewerWindow->getWorldViewHeightRaw();
 	
-		allocateScreenBuffer(resX,resY);
+		if (!allocateScreenBuffer(resX,resY))
+		{ //FAILSAFE: screen buffer allocation failed, disable deferred rendering if it's enabled
+			//NOTE: if the session closes successfully after this call, deferred rendering will be 
+			// disabled on future sessions
+			if (LLPipeline::sRenderDeferred)
+			{
+				gSavedSettings.setBOOL("RenderDeferred", FALSE);
+				LLPipeline::refreshCachedSettings();
+			}
+		}
 	}
 }
 
@@ -779,15 +788,38 @@ void LLPipeline::allocatePhysicsBuffer()
 bool LLPipeline::allocateScreenBuffer(U32 resX, U32 resY)
 {
 	refreshCachedSettings();
-	U32 samples = RenderFSAASamples;
+	
+	bool save_settings = sRenderDeferred;
+	if (save_settings)
+	{
+		// Set this flag in case we crash while resizing window or allocating space for deferred rendering targets
+		gSavedSettings.setBOOL("RenderInitError", TRUE);
+		gSavedSettings.saveToFile( gSavedSettings.getString("ClientSettingsFile"), TRUE );
+	}
+
+	bool ret = doAllocateScreenBuffer(resX, resY);
+
+	if (save_settings)
+	{
+		// don't disable shaders on next session
+		gSavedSettings.setBOOL("RenderInitError", FALSE);
+		gSavedSettings.saveToFile( gSavedSettings.getString("ClientSettingsFile"), TRUE );
+	}
+	
+	return ret;
+}
 
+
+bool LLPipeline::doAllocateScreenBuffer(U32 resX, U32 resY)
+{
 	//try to allocate screen buffers at requested resolution and samples
 	// - on failure, shrink number of samples and try again
 	// - if not multisampled, shrink resolution and try again (favor X resolution over Y)
 	// Make sure to call "releaseScreenBuffers" after each failure to cleanup the partially loaded state
 
-	bool ret = true;
+	U32 samples = RenderFSAASamples;
 
+	bool ret = true;
 	if (!allocateScreenBuffer(resX, resY, samples))
 	{
 		//failed to allocate at requested specification, return false
@@ -831,7 +863,6 @@ bool LLPipeline::allocateScreenBuffer(U32 resX, U32 resY)
 	return ret;
 }
 
-
 bool LLPipeline::allocateScreenBuffer(U32 resX, U32 resY, U32 samples)
 {
 	refreshCachedSettings();
@@ -858,10 +889,6 @@ bool LLPipeline::allocateScreenBuffer(U32 resX, U32 resY, U32 samples)
 
 	if (LLPipeline::sRenderDeferred)
 	{
-		// Set this flag in case we crash while resizing window or allocating space for deferred rendering targets
-		gSavedSettings.setBOOL("RenderInitError", TRUE);
-		gSavedSettings.saveToFile( gSavedSettings.getString("ClientSettingsFile"), TRUE );
-
 		S32 shadow_detail = RenderShadowDetail;
 		BOOL ssao = RenderDeferredSSAO;
 		
@@ -926,9 +953,11 @@ bool LLPipeline::allocateScreenBuffer(U32 resX, U32 resY, U32 samples)
 			}
 		}
 
-		// don't disable shaders on next session
-		gSavedSettings.setBOOL("RenderInitError", FALSE);
-		gSavedSettings.saveToFile( gSavedSettings.getString("ClientSettingsFile"), TRUE );
+		//HACK make screenbuffer allocations start failing after 30 seconds
+		if (gSavedSettings.getBOOL("SimulateFBOFailure"))
+		{
+			return false;
+		}
 	}
 	else
 	{
diff --git a/indra/newview/pipeline.h b/indra/newview/pipeline.h
index c38e7fbdc1..e5a11d5fc6 100644
--- a/indra/newview/pipeline.h
+++ b/indra/newview/pipeline.h
@@ -122,6 +122,10 @@ public:
 	//allocate the largest screen buffer possible up to resX, resY
 	//returns true if full size buffer allocated, false if some other size is allocated
 	bool allocateScreenBuffer(U32 resX, U32 resY);
+private:
+	//implementation of above, wrapped for easy error handling
+	bool doAllocateScreenBuffer(U32 resX, U32 resY);
+public:
 
 	//attempt to allocate screen buffers at resX, resY
 	//returns true if allocation successful, false otherwise