SH-469 Don't use depth buffers and multisample buffers where not absolutely needed -- make sample counts consistent between shaders and render targets.

10 files changed, 88 insertions, 67 deletions

diff --git a/indra/newview/app_settings/shaders/class1/deferred/blurLightMSF.glsl b/indra/newview/app_settings/shaders/class1/deferred/blurLightMSF.glsl
index 4366376d99..edec83ea07 100644
--- a/indra/newview/app_settings/shaders/class1/deferred/blurLightMSF.glsl
+++ b/indra/newview/app_settings/shaders/class1/deferred/blurLightMSF.glsl
@@ -12,7 +12,7 @@
 
 uniform sampler2DMS depthMap;
 uniform sampler2DMS normalMap;
-uniform sampler2DMS lightMap;
+uniform sampler2DRect lightMap;
 
 uniform float dist_factor;
 uniform float blur_size;
@@ -25,12 +25,23 @@ varying vec2 vary_fragcoord;
 uniform mat4 inv_proj;
 uniform vec2 screen_res;
 
-vec4 texture2DMS(sampler2DMS tex, ivec2 tc)
+vec3 texture2DMS3(sampler2DMS tex, ivec2 tc)
 {
-	vec4 ret = vec4(0,0,0,0);
+	vec3 ret = vec3(0,0,0);
 	for (int i = 0; i < samples; i++)
 	{
-		ret += texelFetch(tex, tc, i);
+		ret += texelFetch(tex, tc, i).rgb;
+	}
+
+	return ret/samples;
+}
+
+float texture2DMS1(sampler2DMS tex, ivec2 tc)
+{
+	float ret = 0;
+	for (int i = 0; i < samples; i++)
+	{
+		ret += texelFetch(tex, tc, i).r;
 	}
 
 	return ret/samples;
@@ -38,7 +49,7 @@ vec4 texture2DMS(sampler2DMS tex, ivec2 tc)
 
 vec4 getPosition(ivec2 pos_screen)
 {
-	float depth = texture2DMS(depthMap, pos_screen.xy).r;
+	float depth = texture2DMS1(depthMap, pos_screen.xy);
 	vec2 sc = pos_screen.xy*2.0;
 	sc /= screen_res;
 	sc -= vec2(1.0,1.0);
@@ -54,10 +65,10 @@ void main()
     vec2 tc = vary_fragcoord.xy;
 	ivec2 itc = ivec2(tc);
 
-	vec3 norm = texture2DMS(normalMap, itc).xyz;
+	vec3 norm = texture2DMS3(normalMap, itc).xyz;
 	norm = vec3((norm.xy-0.5)*2.0,norm.z); // unpack norm
 	vec3 pos = getPosition(itc).xyz;
-	vec4 ccol = texture2DMS(lightMap, itc).rgba;
+	vec4 ccol = texture2DRect(lightMap, tc).rgba;
 	
 	vec2 dlt = kern_scale * delta / (1.0+norm.xy*norm.xy);
 	dlt /= max(-pos.z*dist_factor, 1.0);
@@ -73,23 +84,23 @@ void main()
 
 	for (int i = 1; i < 4; i++)
 	{
-		ivec2 samptc = ivec2(tc + kern[i].z*dlt);
-		vec3 samppos = getPosition(samptc).xyz; 
+		vec2 samptc = tc + kern[i].z*dlt;
+		vec3 samppos = getPosition(ivec2(samptc)).xyz; 
 		float d = dot(norm.xyz, samppos.xyz-pos.xyz);// dist from plane
 		if (d*d <= pointplanedist_tolerance_pow2)
 		{
-			col += texture2DMS(lightMap, samptc)*kern[i].xyxx;
+			col += texture2DRect(lightMap, samptc)*kern[i].xyxx;
 			defined_weight += kern[i].xy;
 		}
 	}
 	for (int i = 1; i < 4; i++)
 	{
-		ivec2 samptc = ivec2(tc - kern[i].z*dlt);
-		vec3 samppos = getPosition(samptc).xyz; 
+		vec2 samptc = vec2(tc - kern[i].z*dlt);
+		vec3 samppos = getPosition(ivec2(samptc)).xyz; 
 		float d = dot(norm.xyz, samppos.xyz-pos.xyz);// dist from plane
 		if (d*d <= pointplanedist_tolerance_pow2)
 		{
-			col += texture2DMS(lightMap, samptc)*kern[i].xyxx;
+			col += texture2DRect(lightMap, samptc)*kern[i].xyxx;
 			defined_weight += kern[i].xy;
 		}
 	}
diff --git a/indra/newview/app_settings/shaders/class1/deferred/multiPointLightMSF.glsl b/indra/newview/app_settings/shaders/class1/deferred/multiPointLightMSF.glsl
index 16e97109f9..66c7a5cb4a 100644
--- a/indra/newview/app_settings/shaders/class1/deferred/multiPointLightMSF.glsl
+++ b/indra/newview/app_settings/shaders/class1/deferred/multiPointLightMSF.glsl
@@ -130,7 +130,7 @@ void main()
 		discard;
 	}
 
-	gl_FragColor.rgb = fcol/wght;
+	gl_FragColor.rgb = fcol/samples;
 	gl_FragColor.a = 0.0;
 
 	
diff --git a/indra/newview/app_settings/shaders/class1/deferred/multiSpotLightMSF.glsl b/indra/newview/app_settings/shaders/class1/deferred/multiSpotLightMSF.glsl
index bdaa8e59c4..b450ff1ca2 100644
--- a/indra/newview/app_settings/shaders/class1/deferred/multiSpotLightMSF.glsl
+++ b/indra/newview/app_settings/shaders/class1/deferred/multiSpotLightMSF.glsl
@@ -34,7 +34,6 @@ uniform float far_clip;
 
 uniform vec3 proj_origin; //origin of projection to be used for angular attenuation
 uniform float sun_wash;
-uniform int proj_shadow_idx;
 uniform float shadow_fade;
 
 varying vec4 vary_light;
@@ -228,6 +227,6 @@ void main()
 		discard;
 	}
 
-	gl_FragColor.rgb = fcol/wght;	
+	gl_FragColor.rgb = fcol/samples;	
 	gl_FragColor.a = 0.0;
 }
diff --git a/indra/newview/app_settings/shaders/class1/deferred/pointLightMSF.glsl b/indra/newview/app_settings/shaders/class1/deferred/pointLightMSF.glsl
index 23b5e76735..7521c3310c 100644
--- a/indra/newview/app_settings/shaders/class1/deferred/pointLightMSF.glsl
+++ b/indra/newview/app_settings/shaders/class1/deferred/pointLightMSF.glsl
@@ -103,6 +103,6 @@ void main()
 		discard;
 	}
 		
-	gl_FragColor.rgb = fcol/wght;	
+	gl_FragColor.rgb = fcol/samples;	
 	gl_FragColor.a = 0.0;
 }
diff --git a/indra/newview/app_settings/shaders/class1/deferred/spotLightMSF.glsl b/indra/newview/app_settings/shaders/class1/deferred/spotLightMSF.glsl
index 3a9d9266bb..6702bd5014 100644
--- a/indra/newview/app_settings/shaders/class1/deferred/spotLightMSF.glsl
+++ b/indra/newview/app_settings/shaders/class1/deferred/spotLightMSF.glsl
@@ -229,6 +229,6 @@ void main()
 		discard;
 	}
 
-	gl_FragColor.rgb = fcol/wght;	
+	gl_FragColor.rgb = fcol/samples;	
 	gl_FragColor.a = 0.0;
 }
diff --git a/indra/newview/app_settings/shaders/class2/deferred/multiSpotLightMSF.glsl b/indra/newview/app_settings/shaders/class2/deferred/multiSpotLightMSF.glsl
index c6f0ea2a5d..d0f9101774 100644
--- a/indra/newview/app_settings/shaders/class2/deferred/multiSpotLightMSF.glsl
+++ b/indra/newview/app_settings/shaders/class2/deferred/multiSpotLightMSF.glsl
@@ -14,7 +14,7 @@ uniform sampler2DMS diffuseRect;
 uniform sampler2DMS specularRect;
 uniform sampler2DMS depthMap;
 uniform sampler2DMS normalMap;
-uniform sampler2DMS lightMap;
+uniform sampler2DRect lightMap;
 uniform sampler2D noiseMap;
 uniform sampler2D lightFunc;
 uniform sampler2D projectionMap;
@@ -112,6 +112,17 @@ void main()
 	
 	ivec2 itc = ivec2(frag.xy);
 
+	float shadow = 1.0;
+
+	if (proj_shadow_idx >= 0)
+	{
+		vec4 shd = texture2DRect(lightMap, frag);
+		float sh[2];
+		sh[0] = shd.b;
+		sh[1] = shd.a;
+		shadow = min(sh[proj_shadow_idx]+shadow_fade, 1.0);
+	}
+		
 	for (int i = 0; i < samples; i++)
 	{
 		vec3 pos = getPosition(itc, i).xyz;
@@ -120,17 +131,6 @@ void main()
 		dist2 /= vary_light.w;
 		if (dist2 <= 1.0)
 		{
-			float shadow = 1.0;
-	
-			if (proj_shadow_idx >= 0)
-			{
-				vec4 shd = texelFetch(lightMap, itc, i);
-				float sh[2];
-				sh[0] = shd.b;
-				sh[1] = shd.a;
-				shadow = min(sh[proj_shadow_idx]+shadow_fade, 1.0);
-			}
-	
 			vec3 norm = texelFetch(normalMap, itc, i).xyz;
 			norm = vec3((norm.xy-0.5)*2.0,norm.z); // unpack norm
 	
@@ -239,6 +239,6 @@ void main()
 		discard;
 	}
 
-	gl_FragColor.rgb = fcol/wght;	
+	gl_FragColor.rgb = fcol/samples;	
 	gl_FragColor.a = 0.0;
 }
diff --git a/indra/newview/app_settings/shaders/class2/deferred/softenLightMSF.glsl b/indra/newview/app_settings/shaders/class2/deferred/softenLightMSF.glsl
index 71482d0c7b..cb09fe9895 100644
--- a/indra/newview/app_settings/shaders/class2/deferred/softenLightMSF.glsl
+++ b/indra/newview/app_settings/shaders/class2/deferred/softenLightMSF.glsl
@@ -13,7 +13,7 @@
 uniform sampler2DMS diffuseRect;
 uniform sampler2DMS specularRect;
 uniform sampler2DMS normalMap;
-uniform sampler2DMS lightMap;
+uniform sampler2DRect lightMap;
 uniform sampler2DMS depthMap;
 uniform sampler2D	  noiseMap;
 uniform samplerCube environmentMap;
@@ -257,7 +257,8 @@ void main()
 
 	vec3 fcol = vec3(0,0,0);
 
-	float amb = 0;
+	vec2 scol_ambocc = texture2DRect(lightMap, tc).rg;
+	float ambocc = scol_ambocc.g;
 
 	for (int i = 0; i < samples; ++i)
 	{
@@ -271,9 +272,9 @@ void main()
 		vec4 diffuse = texelFetch(diffuseRect, itc, i);
 		vec4 spec = texelFetch(specularRect, itc, i);
 	
-		vec2 scol_ambocc = texelFetch(lightMap, itc, i).rg;
+		float amb = 0;
+
 		float scol = max(scol_ambocc.r, diffuse.a); 
-		float ambocc = scol_ambocc.g;
 		amb += ambocc;
 
 		calcAtmospherics(pos.xyz, ambocc);
diff --git a/indra/newview/app_settings/shaders/class2/deferred/spotLightMSF.glsl b/indra/newview/app_settings/shaders/class2/deferred/spotLightMSF.glsl
index 099a45718a..8d6ffd79c5 100644
--- a/indra/newview/app_settings/shaders/class2/deferred/spotLightMSF.glsl
+++ b/indra/newview/app_settings/shaders/class2/deferred/spotLightMSF.glsl
@@ -14,7 +14,7 @@ uniform sampler2DMS diffuseRect;
 uniform sampler2DMS specularRect;
 uniform sampler2DMS depthMap;
 uniform sampler2DMS normalMap;
-uniform sampler2DMS lightMap;
+uniform sampler2DRect lightMap;
 uniform sampler2D noiseMap;
 uniform sampler2D lightFunc;
 uniform sampler2D projectionMap;
@@ -113,6 +113,17 @@ void main()
 	vec3 fcol = vec3(0,0,0);
 	int wght = 0;
 
+	float shadow = 1.0;
+	
+	if (proj_shadow_idx >= 0)
+	{
+		vec4 shd = texture2DRect(lightMap, frag.xy);
+		float sh[2];
+		sh[0] = shd.b;
+		sh[1] = shd.a;
+		shadow = min(sh[proj_shadow_idx]+shadow_fade, 1.0);
+	}
+	
 	for (int i = 0; i < samples; i++)
 	{
 		vec3 pos = getPosition(itc, i).xyz;
@@ -121,17 +132,6 @@ void main()
 		dist2 /= vary_light.w;
 		if (dist2 <= 1.0)
 		{
-			float shadow = 1.0;
-	
-			if (proj_shadow_idx >= 0)
-			{
-				vec4 shd = texelFetch(lightMap, itc, i);
-				float sh[2];
-				sh[0] = shd.b;
-				sh[1] = shd.a;
-				shadow = min(sh[proj_shadow_idx]+shadow_fade, 1.0);
-			}
-	
 			vec3 norm = texelFetch(normalMap, itc, i).xyz;
 			norm = vec3((norm.xy-0.5)*2.0,norm.z); // unpack norm
 	
diff --git a/indra/newview/llviewershadermgr.cpp b/indra/newview/llviewershadermgr.cpp
index 911a8bb65f..b818da205e 100644
--- a/indra/newview/llviewershadermgr.cpp
+++ b/indra/newview/llviewershadermgr.cpp
@@ -353,7 +353,7 @@ void LLViewerShaderMgr::setShaders()
 	}
 
 	//setup preprocessor definitions
-	LLShaderMgr::instance()->mDefinitions["samples"] = llformat("%d", gSavedSettings.getU32("RenderFSAASamples"));
+	LLShaderMgr::instance()->mDefinitions["samples"] = llformat("%d", gGLManager.getNumFBOFSAASamples(gSavedSettings.getU32("RenderFSAASamples")));
 
 	reentrance = true;
 
@@ -841,7 +841,8 @@ BOOL LLViewerShaderMgr::loadShadersEffects()
 {
 	BOOL success = TRUE;
 
-	bool multisample = gSavedSettings.getU32("RenderFSAASamples") > 0 && gGLManager.mHasTextureMultisample;
+	U32 samples = gGLManager.getNumFBOFSAASamples(gSavedSettings.getU32("RenderFSAASamples"));
+	bool multisample = samples > 1 && LLPipeline::sRenderDeferred && gGLManager.mHasTextureMultisample;
 
 	if (mVertexShaderLevel[SHADER_EFFECT] == 0)
 	{
@@ -870,7 +871,7 @@ BOOL LLViewerShaderMgr::loadShadersEffects()
 	{
 		std::string fragment;
 
-		if (gSavedSettings.getU32("RenderFSAASamples") > 0 && LLRenderTarget::sUseFBO && gGLManager.mHasTextureMultisample)
+		if (multisample)
 		{
 			fragment = "effects/glowExtractMSF.glsl";
 		}
@@ -983,7 +984,8 @@ BOOL LLViewerShaderMgr::loadShadersDeferred()
 
 	BOOL success = TRUE;
 
-	bool multisample = gSavedSettings.getU32("RenderFSAASamples") > 0 && gGLManager.mHasTextureMultisample;
+	U32 samples = gGLManager.getNumFBOFSAASamples(gSavedSettings.getU32("RenderFSAASamples"));
+	bool multisample = samples > 1 && gGLManager.mHasTextureMultisample;
 
 	if (success)
 	{
diff --git a/indra/newview/pipeline.cpp b/indra/newview/pipeline.cpp
index ffe3b4b1a4..1214d1b545 100644
--- a/indra/newview/pipeline.cpp
+++ b/indra/newview/pipeline.cpp
@@ -589,7 +589,8 @@ void LLPipeline::allocateScreenBuffer(U32 resX, U32 resY)
 	mScreenWidth = resX;
 	mScreenHeight = resY;
 	
-	U32 samples = llmin(gSavedSettings.getU32("RenderFSAASamples"), (U32) 16);
+	//cap samples at 4 for render targets to avoid out of memory errors
+	U32 samples = gGLManager.getNumFBOFSAASamples(gSavedSettings.getU32("RenderFSAASamples"));
 
 	if (gGLManager.mIsATI)
 	{ //disable multisampling of render targets where ATI is involved
@@ -631,7 +632,7 @@ void LLPipeline::allocateScreenBuffer(U32 resX, U32 resY)
 
 		if (shadow_detail > 0 || ssao)
 		{ //only need mDeferredLight[0] for shadows OR ssao
-			mDeferredLight[0].allocate(resX, resY, GL_RGBA, FALSE, FALSE, LLTexUnit::TT_RECT_TEXTURE, FALSE, samples);
+			mDeferredLight[0].allocate(resX, resY, GL_RGBA, FALSE, FALSE, LLTexUnit::TT_RECT_TEXTURE, FALSE);
 		}
 		else
 		{
@@ -640,7 +641,7 @@ void LLPipeline::allocateScreenBuffer(U32 resX, U32 resY)
 
 		if (ssao)
 		{ //only need mDeferredLight[1] for ssao
-			mDeferredLight[1].allocate(resX, resY, GL_RGBA, FALSE, FALSE, LLTexUnit::TT_RECT_TEXTURE, false, samples);
+			mDeferredLight[1].allocate(resX, resY, GL_RGBA, FALSE, FALSE, LLTexUnit::TT_RECT_TEXTURE, false);
 		}
 		else
 		{
@@ -742,13 +743,6 @@ void LLPipeline::allocateScreenBuffer(U32 resX, U32 resY)
 	if (LLPipeline::sRenderDeferred)
 	{ //share depth buffer between deferred targets
 		mDeferredScreen.shareDepthBuffer(mScreen);
-		for (U32 i = 0; i < 3; i++)
-		{ //share stencil buffer with screen space lightmap to stencil out sky
-			if (mDeferredLight[i].getTexture(0))
-			{
-				mDeferredScreen.shareDepthBuffer(mDeferredLight[i]);
-			}
-		}
 	}
 
 	gGL.getTexUnit(0)->disable();
@@ -7751,18 +7745,27 @@ void LLPipeline::setupSpotLight(LLGLSLShader& shader, LLDrawable* drawablep)
 
 	LLViewerTexture* img = volume->getLightTexture();
 
+	if (img == NULL)
+	{
+		img = LLViewerFetchedTexture::sWhiteImagep;
+	}
+
 	S32 channel = shader.enableTexture(LLViewerShaderMgr::DEFERRED_PROJECTION);
 
-	if (channel > -1 && img)
+	if (channel > -1)
 	{
-		gGL.getTexUnit(channel)->bind(img);
+		if (img)
+		{
+			gGL.getTexUnit(channel)->bind(img);
 
-		F32 lod_range = logf(img->getWidth())/logf(2.f);
+			F32 lod_range = logf(img->getWidth())/logf(2.f);
 
-		shader.uniform1f("proj_focus", focus);
-		shader.uniform1f("proj_lod", lod_range);
-		shader.uniform1f("proj_ambient_lod", llclamp((proj_range-focus)/proj_range*lod_range, 0.f, 1.f));
+			shader.uniform1f("proj_focus", focus);
+			shader.uniform1f("proj_lod", lod_range);
+			shader.uniform1f("proj_ambient_lod", llclamp((proj_range-focus)/proj_range*lod_range, 0.f, 1.f));
+		}
 	}
+		
 }
 
 void LLPipeline::unbindDeferredShader(LLGLSLShader &shader)
@@ -9372,6 +9375,11 @@ void LLPipeline::generateSunShadow(LLCamera& camera)
 			mShadow[i+4].flush();
  		}
 	}
+	else
+	{ //no spotlight shadows
+		mShadowSpotLight[0] = mShadowSpotLight[1] = NULL;
+	}
+
 
 	if (!gSavedSettings.getBOOL("CameraOffset"))
 	{