SL-11109

Make shadowAlphaMask shader ignore vertex alpha before discards iff we're a fullbright object.

Keep llappearance from stuffing dangling pointers into the alpha cache (crash iff you're using nSight debugging and disable readbacks).

6 files changed, 48 insertions, 5 deletions

diff --git a/indra/newview/app_settings/shaders/class1/deferred/shadowAlphaMaskF.glsl b/indra/newview/app_settings/shaders/class1/deferred/shadowAlphaMaskF.glsl
index 0d028d2729..9b8df0a5a4 100644
--- a/indra/newview/app_settings/shaders/class1/deferred/shadowAlphaMaskF.glsl
+++ b/indra/newview/app_settings/shaders/class1/deferred/shadowAlphaMaskF.glsl
@@ -49,6 +49,10 @@ void main()
         discard;
     }
 
+#if !defined(IS_FULLBRIGHT)
+    alpha *= vertex_color.a;
+#endif
+
 	if (alpha < 0.05) // treat as totally transparent
 	{
 		discard;
@@ -62,8 +66,6 @@ void main()
 	  }
 	}
 
-    alpha *= vertex_color.a;
-
 	frag_color = vec4(1,1,1,1);
 	
 #if !defined(DEPTH_CLAMP)
diff --git a/indra/newview/llviewershadermgr.cpp b/indra/newview/llviewershadermgr.cpp
index bd8064999f..5297e59e18 100644
--- a/indra/newview/llviewershadermgr.cpp
+++ b/indra/newview/llviewershadermgr.cpp
@@ -215,6 +215,7 @@ LLGLSLShader			gDeferredSoftenWaterProgram;
 LLGLSLShader			gDeferredShadowProgram;
 LLGLSLShader			gDeferredShadowCubeProgram;
 LLGLSLShader			gDeferredShadowAlphaMaskProgram;
+LLGLSLShader			gDeferredShadowFullbrightAlphaMaskProgram;
 LLGLSLShader			gDeferredAvatarShadowProgram;
 LLGLSLShader			gDeferredAvatarAlphaShadowProgram;
 LLGLSLShader			gDeferredAvatarAlphaMaskShadowProgram;
@@ -1293,6 +1294,7 @@ BOOL LLViewerShaderMgr::loadShadersDeferred()
 		gDeferredShadowProgram.unload();
 		gDeferredShadowCubeProgram.unload();
         gDeferredShadowAlphaMaskProgram.unload();
+        gDeferredShadowFullbrightAlphaMaskProgram.unload();
 		gDeferredAvatarShadowProgram.unload();
         gDeferredAvatarAlphaShadowProgram.unload();
         gDeferredAvatarAlphaMaskShadowProgram.unload();
@@ -2402,6 +2404,26 @@ BOOL LLViewerShaderMgr::loadShadersDeferred()
 
 	if (success)
 	{
+		gDeferredShadowFullbrightAlphaMaskProgram.mName = "Deferred Shadow Fullbright Alpha Mask Shader";
+		gDeferredShadowFullbrightAlphaMaskProgram.mFeatures.mIndexedTextureChannels = LLGLSLShader::sIndexedTextureChannels;
+
+		gDeferredShadowFullbrightAlphaMaskProgram.mShaderFiles.clear();
+		gDeferredShadowFullbrightAlphaMaskProgram.mShaderFiles.push_back(make_pair("deferred/shadowAlphaMaskV.glsl", GL_VERTEX_SHADER_ARB));
+		gDeferredShadowFullbrightAlphaMaskProgram.mShaderFiles.push_back(make_pair("deferred/shadowAlphaMaskF.glsl", GL_FRAGMENT_SHADER_ARB));
+
+        gDeferredShadowFullbrightAlphaMaskProgram.clearPermutations();
+		if (gGLManager.mHasDepthClamp)
+		{
+			gDeferredShadowFullbrightAlphaMaskProgram.addPermutation("DEPTH_CLAMP", "1");
+		}
+        gDeferredShadowFullbrightAlphaMaskProgram.addPermutation("IS_FULLBRIGHT", "1");
+		gDeferredShadowFullbrightAlphaMaskProgram.mShaderLevel = mShaderLevel[SHADER_DEFERRED];
+		success = gDeferredShadowFullbrightAlphaMaskProgram.createShader(NULL, NULL);
+		llassert(success);
+	}
+    
+    if (success)
+	{
 		gDeferredShadowAlphaMaskProgram.mName = "Deferred Shadow Alpha Mask Shader";
 		gDeferredShadowAlphaMaskProgram.mFeatures.mIndexedTextureChannels = LLGLSLShader::sIndexedTextureChannels;
 
diff --git a/indra/newview/llviewershadermgr.h b/indra/newview/llviewershadermgr.h
index 18be3c22aa..081221f15b 100644
--- a/indra/newview/llviewershadermgr.h
+++ b/indra/newview/llviewershadermgr.h
@@ -301,6 +301,7 @@ extern LLGLSLShader			gDeferredSoftenWaterProgram;
 extern LLGLSLShader			gDeferredShadowProgram;
 extern LLGLSLShader			gDeferredShadowCubeProgram;
 extern LLGLSLShader			gDeferredShadowAlphaMaskProgram;
+extern LLGLSLShader			gDeferredShadowFullbrightAlphaMaskProgram;
 extern LLGLSLShader			gDeferredPostProgram;
 extern LLGLSLShader			gDeferredCoFProgram;
 extern LLGLSLShader			gDeferredDoFCombineProgram;
diff --git a/indra/newview/llvovolume.cpp b/indra/newview/llvovolume.cpp
index 3740caa53f..d149a61d89 100644
--- a/indra/newview/llvovolume.cpp
+++ b/indra/newview/llvovolume.cpp
@@ -6680,7 +6680,7 @@ U32 LLVolumeGeometryManager::genDrawInfo(LLSpatialGroup* group, U32 mask, LLFace
 						{
 							registerFace(group, facep, LLRenderPass::PASS_SIMPLE);
 						}
-				}
+				    }
 				}
 				
 				
diff --git a/indra/newview/pipeline.cpp b/indra/newview/pipeline.cpp
index 6f3e6819c4..863785889f 100644
--- a/indra/newview/pipeline.cpp
+++ b/indra/newview/pipeline.cpp
@@ -7452,6 +7452,15 @@ void LLPipeline::renderMaskedObjects(U32 type, U32 mask, bool texture, bool batc
 	gGLLastMatrix = NULL;		
 }
 
+void LLPipeline::renderFullbrightMaskedObjects(U32 type, U32 mask, bool texture, bool batch_texture)
+{
+	assertInitialized();
+	gGL.loadMatrix(gGLModelView);
+	gGLLastMatrix = NULL;
+	mFullbrightAlphaMaskPool->pushMaskBatches(type, mask, texture, batch_texture);
+	gGL.loadMatrix(gGLModelView);
+	gGLLastMatrix = NULL;		
+}
 
 void apply_cube_face_rotation(U32 face)
 {
@@ -9608,6 +9617,7 @@ static LLTrace::BlockTimerStatHandle FTM_SHADOW_ALPHA_MASKED("Alpha Masked");
 static LLTrace::BlockTimerStatHandle FTM_SHADOW_ALPHA_BLEND("Alpha Blend");
 static LLTrace::BlockTimerStatHandle FTM_SHADOW_ALPHA_TREE("Alpha Tree");
 static LLTrace::BlockTimerStatHandle FTM_SHADOW_ALPHA_GRASS("Alpha Grass");
+static LLTrace::BlockTimerStatHandle FTM_SHADOW_FULLBRIGHT_ALPHA_MASKED("Fullbright Alpha Masked");
 
 void LLPipeline::renderShadow(glh::matrix4f& view, glh::matrix4f& proj, LLCamera& shadow_cam, LLCullResult &result, bool use_shader, bool use_occlusion, U32 target_width)
 {
@@ -9740,8 +9750,7 @@ void LLPipeline::renderShadow(glh::matrix4f& view, glh::matrix4f& proj, LLCamera
 
         {
             LL_RECORD_BLOCK_TIME(FTM_SHADOW_ALPHA_MASKED);
-            renderMaskedObjects(LLRenderPass::PASS_ALPHA_MASK, mask, TRUE, TRUE);
-            renderMaskedObjects(LLRenderPass::PASS_FULLBRIGHT_ALPHA_MASK, mask, TRUE, TRUE);
+            renderMaskedObjects(LLRenderPass::PASS_ALPHA_MASK, mask, TRUE, TRUE);            
         }
 
         {
@@ -9750,6 +9759,14 @@ void LLPipeline::renderShadow(glh::matrix4f& view, glh::matrix4f& proj, LLCamera
             renderObjects(LLRenderPass::PASS_ALPHA, mask, TRUE, TRUE);
         }
 
+        {
+            LL_RECORD_BLOCK_TIME(FTM_SHADOW_FULLBRIGHT_ALPHA_MASKED);
+            gDeferredShadowFullbrightAlphaMaskProgram.bind();
+            gDeferredShadowFullbrightAlphaMaskProgram.uniform1f(LLShaderMgr::DEFERRED_SHADOW_TARGET_WIDTH, (float)target_width);
+            gDeferredShadowFullbrightAlphaMaskProgram.uniform1i(LLShaderMgr::SUN_UP_FACTOR, environment.getIsSunUp() ? 1 : 0);
+            renderFullbrightMaskedObjects(LLRenderPass::PASS_FULLBRIGHT_ALPHA_MASK, mask, TRUE, TRUE);
+        }
+
         mask = mask & ~LLVertexBuffer::MAP_TEXTURE_INDEX;
 
         {
diff --git a/indra/newview/pipeline.h b/indra/newview/pipeline.h
index fe81ae91e8..b0c40d145b 100644
--- a/indra/newview/pipeline.h
+++ b/indra/newview/pipeline.h
@@ -261,6 +261,7 @@ public:
 
 	void renderObjects(U32 type, U32 mask, bool texture = true, bool batch_texture = false);
 	void renderMaskedObjects(U32 type, U32 mask, bool texture = true, bool batch_texture = false);
+    void renderFullbrightMaskedObjects(U32 type, U32 mask, bool texture = true, bool batch_texture = false);
 
 	void renderGroups(LLRenderPass* pass, U32 type, U32 mask, bool texture);