SH-1838 Add error handling for allocation of off screen render targets.

Reviewed by Leslie

4 files changed, 135 insertions, 50 deletions

diff --git a/indra/llrender/llrendertarget.cpp b/indra/llrender/llrendertarget.cpp
index b6463309e1..8c0d3592df 100644
--- a/indra/llrender/llrendertarget.cpp
+++ b/indra/llrender/llrendertarget.cpp
@@ -72,11 +72,11 @@ LLRenderTarget::~LLRenderTarget()
 	release();
 }
 
-void LLRenderTarget::allocate(U32 resx, U32 resy, U32 color_fmt, bool depth, bool stencil, LLTexUnit::eTextureType usage, bool use_fbo, S32 samples)
+bool LLRenderTarget::allocate(U32 resx, U32 resy, U32 color_fmt, bool depth, bool stencil, LLTexUnit::eTextureType usage, bool use_fbo, S32 samples)
 {
 	stop_glerror();
-	
 	release();
+	stop_glerror();
 
 	mResX = resx;
 	mResY = resy;
@@ -103,9 +103,11 @@ void LLRenderTarget::allocate(U32 resx, U32 resy, U32 color_fmt, bool depth, boo
 	{
 		if (depth)
 		{
-			stop_glerror();
-			allocateDepth();
-			stop_glerror();
+			if (!allocateDepth())
+			{
+				llwarns << "Failed to allocate depth buffer for render target." << llendl;
+				return false;
+			}
 		}
 
 		glGenFramebuffers(1, (GLuint *) &mFBO);
@@ -131,14 +133,14 @@ void LLRenderTarget::allocate(U32 resx, U32 resy, U32 color_fmt, bool depth, boo
 		stop_glerror();
 	}
 
-	addColorAttachment(color_fmt);
+	return addColorAttachment(color_fmt);
 }
 
-void LLRenderTarget::addColorAttachment(U32 color_fmt)
+bool LLRenderTarget::addColorAttachment(U32 color_fmt)
 {
 	if (color_fmt == 0)
 	{
-		return;
+		return true;
 	}
 
 	U32 offset = mTex.size();
@@ -158,14 +160,26 @@ void LLRenderTarget::addColorAttachment(U32 color_fmt)
 #ifdef GL_ARB_texture_multisample
 	if (mSamples > 1)
 	{
+		clear_glerror();
 		glTexImage2DMultisample(LLTexUnit::getInternalType(mUsage), mSamples, color_fmt, mResX, mResY, GL_TRUE);
+		if (glGetError() != GL_NO_ERROR)
+		{
+			llwarns << "Could not allocate multisample color buffer for render target." << llendl;
+			return false;
+		}
 	}
 	else
 #else
 	llassert_always(mSamples <= 1);
 #endif
 	{
+		clear_glerror();
 		LLImageGL::setManualImage(LLTexUnit::getInternalType(mUsage), 0, color_fmt, mResX, mResY, GL_RGBA, GL_UNSIGNED_BYTE, NULL);
+		if (glGetError() != GL_NO_ERROR)
+		{
+			llwarns << "Could not allocate color buffer for render target." << llendl;
+			return false;
+		}
 	}
 	
 	stop_glerror();
@@ -217,15 +231,18 @@ void LLRenderTarget::addColorAttachment(U32 color_fmt)
 		flush();
 	}
 
+	return true;
 }
 
-void LLRenderTarget::allocateDepth()
+bool LLRenderTarget::allocateDepth()
 {
 	if (mStencil)
 	{
 		//use render buffers where stencil buffers are in play
 		glGenRenderbuffers(1, (GLuint *) &mDepth);
 		glBindRenderbuffer(GL_RENDERBUFFER, mDepth);
+		stop_glerror();
+		clear_glerror();
 		glRenderbufferStorage(GL_RENDERBUFFER, GL_DEPTH24_STENCIL8, mResX, mResY);
 		glBindRenderbuffer(GL_RENDERBUFFER, 0);
 	}
@@ -237,17 +254,29 @@ void LLRenderTarget::allocateDepth()
 		{
 			U32 internal_type = LLTexUnit::getInternalType(mUsage);
 			gGL.getTexUnit(0)->setTextureFilteringOption(LLTexUnit::TFO_POINT);
+			stop_glerror();
+			clear_glerror();
 			LLImageGL::setManualImage(internal_type, 0, GL_DEPTH_COMPONENT32, mResX, mResY, GL_DEPTH_COMPONENT, GL_UNSIGNED_INT, NULL);
 		}
 #ifdef GL_ARB_texture_multisample
 		else
 		{
+			stop_glerror();
+			clear_glerror();
 			glTexImage2DMultisample(LLTexUnit::getInternalType(mUsage), mSamples, GL_DEPTH_COMPONENT32, mResX, mResY, GL_TRUE);
 		}
 #else
 		llassert_always(mSamples <= 1);
 #endif
 	}
+
+	if (glGetError() != GL_NO_ERROR)
+	{
+		llwarns << "Unable to allocate depth buffer for render target." << llendl;
+		return false;
+	}
+
+	return true;
 }
 
 void LLRenderTarget::shareDepthBuffer(LLRenderTarget& target)
diff --git a/indra/llrender/llrendertarget.h b/indra/llrender/llrendertarget.h
index 094b58b562..dea1de12d8 100644
--- a/indra/llrender/llrendertarget.h
+++ b/indra/llrender/llrendertarget.h
@@ -66,30 +66,30 @@ public:
 	static bool sUseFBO; 
 
 	LLRenderTarget();
-	virtual ~LLRenderTarget();
+	~LLRenderTarget();
 
 	//allocate resources for rendering
 	//must be called before use
 	//multiple calls will release previously allocated resources
-	void allocate(U32 resx, U32 resy, U32 color_fmt, bool depth, bool stencil, LLTexUnit::eTextureType usage = LLTexUnit::TT_TEXTURE, bool use_fbo = false, S32 samples = 0);
+	bool allocate(U32 resx, U32 resy, U32 color_fmt, bool depth, bool stencil, LLTexUnit::eTextureType usage = LLTexUnit::TT_TEXTURE, bool use_fbo = false, S32 samples = 0);
 
 	//add color buffer attachment
 	//limit of 4 color attachments per render target
-	virtual void addColorAttachment(U32 color_fmt);
+	bool addColorAttachment(U32 color_fmt);
 
 	//allocate a depth texture
-	virtual void allocateDepth();
+	bool allocateDepth();
 
 	//share depth buffer with provided render target
-	virtual void shareDepthBuffer(LLRenderTarget& target);
+	void shareDepthBuffer(LLRenderTarget& target);
 
 	//free any allocated resources
 	//safe to call redundantly
-	virtual void release();
+	void release();
 
 	//bind target for rendering
 	//applies appropriate viewport
-	virtual void bindTarget();
+	void bindTarget();
 
 	//unbind target for rendering
 	static void unbindTarget();
diff --git a/indra/newview/pipeline.cpp b/indra/newview/pipeline.cpp
index 3e35e0e41a..99f8a87b16 100644
--- a/indra/newview/pipeline.cpp
+++ b/indra/newview/pipeline.cpp
@@ -336,10 +336,10 @@ static const U32 gl_cube_face[] =
 void validate_framebuffer_object();
 
 
-void addDeferredAttachments(LLRenderTarget& target)
+bool addDeferredAttachments(LLRenderTarget& target)
 {
-	target.addColorAttachment(GL_RGBA); //specular
-	target.addColorAttachment(GL_RGBA); //normal+z	
+	return target.addColorAttachment(GL_RGBA) && //specular
+			target.addColorAttachment(GL_RGBA); //normal+z	
 }
 
 LLPipeline::LLPipeline() :
@@ -586,18 +586,61 @@ void LLPipeline::allocatePhysicsBuffer()
 
 void LLPipeline::allocateScreenBuffer(U32 resX, U32 resY)
 {
-	// remember these dimensions
-	mScreenWidth = resX;
-	mScreenHeight = resY;
-	
-	//cap samples at 4 for render targets to avoid out of memory errors
 	U32 samples = gGLManager.getNumFBOFSAASamples(gSavedSettings.getU32("RenderFSAASamples"));
 
 	if (gGLManager.mIsATI)
-	{ //disable multisampling of render targets where ATI is involved
+	{ //ATI doesn't like the way we use multisample texture
 		samples = 0;
 	}
 
+	//try to allocate screen buffers at requested resolution and samples
+	// - on failure, shrink number of samples and try again
+	// - if not multisampled, shrink resolution and try again (favor X resolution over Y)
+	// Make sure to call "releaseScreenBuffers" after each failure to cleanup the partially loaded state
+
+	if (!allocateScreenBuffer(resX, resY, samples))
+	{
+		releaseScreenBuffers();
+		//reduce number of samples 
+		while (samples > 0)
+		{
+			samples /= 2;
+			if (allocateScreenBuffer(resX, resY, samples))
+			{ //success
+				return;
+			}
+			releaseScreenBuffers();
+		}
+
+		//reduce resolution
+		while (resY > 0 && resX > 0)
+		{
+			resY /= 2;
+			if (allocateScreenBuffer(resX, resY, samples))
+			{
+				return;
+			}
+			releaseScreenBuffers();
+
+			resX /= 2;
+			if (allocateScreenBuffer(resX, resY, samples))
+			{
+				return;
+			}
+			releaseScreenBuffers();
+		}
+
+		llwarns << "Unable to allocate screen buffer at any resolution!" << llendl;
+	}
+}
+
+
+bool LLPipeline::allocateScreenBuffer(U32 resX, U32 resY, U32 samples)
+{
+	// remember these dimensions
+	mScreenWidth = resX;
+	mScreenHeight = resY;
+	
 	U32 res_mod = gSavedSettings.getU32("RenderResolutionDivisor");
 
 	if (res_mod > 1 && res_mod < resX && res_mod < resY)
@@ -608,7 +651,10 @@ void LLPipeline::allocateScreenBuffer(U32 resX, U32 resY)
 
 	if (gSavedSettings.getBOOL("RenderUIBuffer"))
 	{
-		mUIScreen.allocate(resX,resY, GL_RGBA, FALSE, FALSE, LLTexUnit::TT_RECT_TEXTURE, FALSE);
+		if (!mUIScreen.allocate(resX,resY, GL_RGBA, FALSE, FALSE, LLTexUnit::TT_RECT_TEXTURE, FALSE))
+		{
+			return false;
+		}
 	}	
 
 	if (LLPipeline::sRenderDeferred)
@@ -618,22 +664,22 @@ void LLPipeline::allocateScreenBuffer(U32 resX, U32 resY)
 		bool gi = LLViewerShaderMgr::instance()->getVertexShaderLevel(LLViewerShaderMgr::SHADER_DEFERRED);
 
 		//allocate deferred rendering color buffers
-		mDeferredScreen.allocate(resX, resY, GL_RGBA, TRUE, TRUE, LLTexUnit::TT_RECT_TEXTURE, FALSE, samples);
-		mDeferredDepth.allocate(resX, resY, 0, TRUE, FALSE, LLTexUnit::TT_RECT_TEXTURE, FALSE, samples);
-		addDeferredAttachments(mDeferredScreen);
+		if (!mDeferredScreen.allocate(resX, resY, GL_RGBA, TRUE, TRUE, LLTexUnit::TT_RECT_TEXTURE, FALSE, samples)) return false;
+		if (!mDeferredDepth.allocate(resX, resY, 0, TRUE, FALSE, LLTexUnit::TT_RECT_TEXTURE, FALSE, samples)) return false;
+		if (!addDeferredAttachments(mDeferredScreen)) return false;
 	
-		mScreen.allocate(resX, resY, GL_RGBA, FALSE, FALSE, LLTexUnit::TT_RECT_TEXTURE, FALSE, samples);		
+		if (!mScreen.allocate(resX, resY, GL_RGBA, FALSE, FALSE, LLTexUnit::TT_RECT_TEXTURE, FALSE, samples)) return false;
 		
 #if LL_DARWIN
 		// As of OS X 10.6.7, Apple doesn't support multiple color formats in a single FBO
-		mEdgeMap.allocate(resX, resY, GL_RGBA, FALSE, FALSE, LLTexUnit::TT_RECT_TEXTURE, FALSE);
+		if (!mEdgeMap.allocate(resX, resY, GL_RGBA, FALSE, FALSE, LLTexUnit::TT_RECT_TEXTURE, FALSE)) return false;
 #else
-		mEdgeMap.allocate(resX, resY, GL_ALPHA, FALSE, FALSE, LLTexUnit::TT_RECT_TEXTURE, FALSE);
+		if (!mEdgeMap.allocate(resX, resY, GL_ALPHA, FALSE, FALSE, LLTexUnit::TT_RECT_TEXTURE, FALSE)) return false;
 #endif
 
 		if (shadow_detail > 0 || ssao)
 		{ //only need mDeferredLight[0] for shadows OR ssao
-			mDeferredLight[0].allocate(resX, resY, GL_RGBA, FALSE, FALSE, LLTexUnit::TT_RECT_TEXTURE, FALSE);
+			if (!mDeferredLight[0].allocate(resX, resY, GL_RGBA, FALSE, FALSE, LLTexUnit::TT_RECT_TEXTURE, FALSE)) return false;
 		}
 		else
 		{
@@ -642,7 +688,7 @@ void LLPipeline::allocateScreenBuffer(U32 resX, U32 resY)
 
 		if (ssao)
 		{ //only need mDeferredLight[1] for ssao
-			mDeferredLight[1].allocate(resX, resY, GL_RGBA, FALSE, FALSE, LLTexUnit::TT_RECT_TEXTURE, false);
+			if (!mDeferredLight[1].allocate(resX, resY, GL_RGBA, FALSE, FALSE, LLTexUnit::TT_RECT_TEXTURE, false)) return false;
 		}
 		else
 		{
@@ -651,14 +697,14 @@ void LLPipeline::allocateScreenBuffer(U32 resX, U32 resY)
 
 		if (gi)
 		{ //only need mDeferredLight[2] and mGIMapPost for gi
-			mDeferredLight[2].allocate(resX, resY, GL_RGBA, FALSE, FALSE, LLTexUnit::TT_RECT_TEXTURE, false);
+			if (!mDeferredLight[2].allocate(resX, resY, GL_RGBA, FALSE, FALSE, LLTexUnit::TT_RECT_TEXTURE, false)) return false;
 			for (U32 i = 0; i < 2; i++)
 			{
 #if LL_DARWIN
 				// As of OS X 10.6.7, Apple doesn't support multiple color formats in a single FBO
-				mGIMapPost[i].allocate(resX,resY, GL_RGBA, FALSE, FALSE, LLTexUnit::TT_RECT_TEXTURE);
+				if (!mGIMapPost[i].allocate(resX,resY, GL_RGBA, FALSE, FALSE, LLTexUnit::TT_RECT_TEXTURE)) return false;
 #else
-				mGIMapPost[i].allocate(resX,resY, GL_RGB, FALSE, FALSE, LLTexUnit::TT_RECT_TEXTURE);
+				if (!mGIMapPost[i].allocate(resX,resY, GL_RGB, FALSE, FALSE, LLTexUnit::TT_RECT_TEXTURE)) return false;
 #endif
 			}
 		}
@@ -685,7 +731,7 @@ void LLPipeline::allocateScreenBuffer(U32 resX, U32 resY)
 		{ //allocate 4 sun shadow maps
 			for (U32 i = 0; i < 4; i++)
 			{
-				mShadow[i].allocate(U32(resX*scale),U32(resY*scale), shadow_fmt, TRUE, FALSE, LLTexUnit::TT_RECT_TEXTURE);
+				if (!mShadow[i].allocate(U32(resX*scale),U32(resY*scale), shadow_fmt, TRUE, FALSE, LLTexUnit::TT_RECT_TEXTURE)) return false;
 			}
 		}
 		else
@@ -703,7 +749,7 @@ void LLPipeline::allocateScreenBuffer(U32 resX, U32 resY)
 		{ //allocate two spot shadow maps
 			for (U32 i = 4; i < 6; i++)
 			{
-				mShadow[i].allocate(width, height, shadow_fmt, TRUE, FALSE);
+				if (!mShadow[i].allocate(width, height, shadow_fmt, TRUE, FALSE)) return false;
 			}
 		}
 		else
@@ -716,7 +762,7 @@ void LLPipeline::allocateScreenBuffer(U32 resX, U32 resY)
 
 		width = nhpo2(resX)/2;
 		height = nhpo2(resY)/2;
-		mLuminanceMap.allocate(width,height, GL_RGBA, FALSE, FALSE);
+		if (!mLuminanceMap.allocate(width,height, GL_RGBA, FALSE, FALSE)) return false;
 	}
 	else
 	{
@@ -738,7 +784,7 @@ void LLPipeline::allocateScreenBuffer(U32 resX, U32 resY)
 		mEdgeMap.release();
 		mLuminanceMap.release();
 		
-		mScreen.allocate(resX, resY, GL_RGBA, TRUE, TRUE, LLTexUnit::TT_RECT_TEXTURE, FALSE);		
+		if (!mScreen.allocate(resX, resY, GL_RGBA, TRUE, TRUE, LLTexUnit::TT_RECT_TEXTURE, FALSE)) return false;		
 	}
 	
 	if (LLPipeline::sRenderDeferred)
@@ -750,6 +796,7 @@ void LLPipeline::allocateScreenBuffer(U32 resX, U32 resY)
 
 	stop_glerror();
 
+	return true;
 }
 
 //static
@@ -800,9 +847,23 @@ void LLPipeline::releaseGLBuffers()
 
 	mWaterRef.release();
 	mWaterDis.release();
+	
+	for (U32 i = 0; i < 3; i++)
+	{
+		mGlow[i].release();
+	}
+
+	releaseScreenBuffers();
+
+	gBumpImageList.destroyGL();
+	LLVOAvatar::resetImpostors();
+}
+
+void LLPipeline::releaseScreenBuffers()
+{
+	mUIScreen.release();
 	mScreen.release();
 	mPhysicsDisplay.release();
-	mUIScreen.release();
 	mDeferredScreen.release();
 	mDeferredDepth.release();
 	for (U32 i = 0; i < 3; i++)
@@ -821,16 +882,9 @@ void LLPipeline::releaseGLBuffers()
 	{
 		mShadow[i].release();
 	}
-
-	for (U32 i = 0; i < 3; i++)
-	{
-		mGlow[i].release();
-	}
-
-	gBumpImageList.destroyGL();
-	LLVOAvatar::resetImpostors();
 }
 
+
 void LLPipeline::createGLBuffers()
 {
 	LLMemType mt_cb(LLMemType::MTYPE_PIPELINE_CREATE_BUFFERS);
diff --git a/indra/newview/pipeline.h b/indra/newview/pipeline.h
index e9da25e544..28e6526acd 100644
--- a/indra/newview/pipeline.h
+++ b/indra/newview/pipeline.h
@@ -113,9 +113,11 @@ public:
 	void resetVertexBuffers();
 	void resizeScreenTexture();
 	void releaseGLBuffers();
+	void releaseScreenBuffers();
 	void createGLBuffers();
 
 	void allocateScreenBuffer(U32 resX, U32 resY);
+	bool allocateScreenBuffer(U32 resX, U32 resY, U32 samples);
 	void allocatePhysicsBuffer();
 	
 	void resetVertexBuffers(LLDrawable* drawable);