SL-11109

Make shadowAlphaMask shader ignore vertex alpha before discards iff we're a fullbright object.

Keep llappearance from stuffing dangling pointers into the alpha cache (crash iff you're using nSight debugging and disable readbacks).

8 files changed, 64 insertions, 21 deletions

diff --git a/indra/llappearance/lltexlayer.cpp b/indra/llappearance/lltexlayer.cpp
index ca0d102b74..19a4f3e12d 100644
--- a/indra/llappearance/lltexlayer.cpp
+++ b/indra/llappearance/lltexlayer.cpp
@@ -1586,8 +1586,6 @@ void LLTexLayer::renderMorphMasks(S32 x, S32 y, S32 width, S32 height, const LLC
 
             alpha_data = (U8*)ll_aligned_malloc_32(mem_size);
 
-			mAlphaCache[cache_index] = alpha_data;
-
             bool skip_readback = LLRender::sNsightDebugSupport || gGLManager.mIsIntel; // nSight doesn't support use of glReadPixels
 
 			if (!skip_readback)
@@ -1598,7 +1596,9 @@ void LLTexLayer::renderMorphMasks(S32 x, S32 y, S32 width, S32 height, const LLC
             {
                 ll_aligned_free_32(alpha_data);
                 alpha_data = nullptr;
-            }            
+            }
+
+            mAlphaCache[cache_index] = alpha_data;
 		}
 		
 		getTexLayerSet()->getAvatarAppearance()->dirtyMesh();
diff --git a/indra/llrender/llvertexbuffer.cpp b/indra/llrender/llvertexbuffer.cpp
index 1312f6afda..0c53d48c08 100644
--- a/indra/llrender/llvertexbuffer.cpp
+++ b/indra/llrender/llvertexbuffer.cpp
@@ -942,15 +942,15 @@ S32 LLVertexBuffer::determineUsage(S32 usage)
 	{ //only stream_draw and dynamic_draw are supported when using VBOs, dynamic draw is the default
 		if (ret_usage != GL_DYNAMIC_COPY_ARB)
 		{
-		if (sDisableVBOMapping)
-		{ //always use stream draw if VBO mapping is disabled
-			ret_usage = GL_STREAM_DRAW_ARB;
-		}
-		else
-		{
-			ret_usage = GL_DYNAMIC_DRAW_ARB;
-		}
-	}
+		    if (sDisableVBOMapping)
+		    { //always use stream draw if VBO mapping is disabled
+			    ret_usage = GL_STREAM_DRAW_ARB;
+		    }
+		    else
+		    {
+			    ret_usage = GL_DYNAMIC_DRAW_ARB;
+		    }
+	    }
 	}
 	
 	return ret_usage;
@@ -1506,10 +1506,10 @@ bool LLVertexBuffer::resizeBuffer(S32 newnverts, S32 newnindices)
 	llassert(newnverts >= 0);
 	llassert(newnindices >= 0);
 
-	bool sucsess = true;
+	bool success = true;
 
-	sucsess &= updateNumVerts(newnverts);		
-	sucsess &= updateNumIndices(newnindices);
+	success &= updateNumVerts(newnverts);		
+	success &= updateNumIndices(newnindices);
 	
 	if (useVBOs())
 	{
@@ -1521,7 +1521,7 @@ bool LLVertexBuffer::resizeBuffer(S32 newnverts, S32 newnindices)
 		}
 	}
 
-	return sucsess;
+	return success;
 }
 
 bool LLVertexBuffer::useVBOs() const
diff --git a/indra/newview/app_settings/shaders/class1/deferred/shadowAlphaMaskF.glsl b/indra/newview/app_settings/shaders/class1/deferred/shadowAlphaMaskF.glsl
index 0d028d2729..9b8df0a5a4 100644
--- a/indra/newview/app_settings/shaders/class1/deferred/shadowAlphaMaskF.glsl
+++ b/indra/newview/app_settings/shaders/class1/deferred/shadowAlphaMaskF.glsl
@@ -49,6 +49,10 @@ void main()
         discard;
     }
 
+#if !defined(IS_FULLBRIGHT)
+    alpha *= vertex_color.a;
+#endif
+
 	if (alpha < 0.05) // treat as totally transparent
 	{
 		discard;
@@ -62,8 +66,6 @@ void main()
 	  }
 	}
 
-    alpha *= vertex_color.a;
-
 	frag_color = vec4(1,1,1,1);
 	
 #if !defined(DEPTH_CLAMP)
diff --git a/indra/newview/llviewershadermgr.cpp b/indra/newview/llviewershadermgr.cpp
index bd8064999f..5297e59e18 100644
--- a/indra/newview/llviewershadermgr.cpp
+++ b/indra/newview/llviewershadermgr.cpp
@@ -215,6 +215,7 @@ LLGLSLShader			gDeferredSoftenWaterProgram;
 LLGLSLShader			gDeferredShadowProgram;
 LLGLSLShader			gDeferredShadowCubeProgram;
 LLGLSLShader			gDeferredShadowAlphaMaskProgram;
+LLGLSLShader			gDeferredShadowFullbrightAlphaMaskProgram;
 LLGLSLShader			gDeferredAvatarShadowProgram;
 LLGLSLShader			gDeferredAvatarAlphaShadowProgram;
 LLGLSLShader			gDeferredAvatarAlphaMaskShadowProgram;
@@ -1293,6 +1294,7 @@ BOOL LLViewerShaderMgr::loadShadersDeferred()
 		gDeferredShadowProgram.unload();
 		gDeferredShadowCubeProgram.unload();
         gDeferredShadowAlphaMaskProgram.unload();
+        gDeferredShadowFullbrightAlphaMaskProgram.unload();
 		gDeferredAvatarShadowProgram.unload();
         gDeferredAvatarAlphaShadowProgram.unload();
         gDeferredAvatarAlphaMaskShadowProgram.unload();
@@ -2402,6 +2404,26 @@ BOOL LLViewerShaderMgr::loadShadersDeferred()
 
 	if (success)
 	{
+		gDeferredShadowFullbrightAlphaMaskProgram.mName = "Deferred Shadow Fullbright Alpha Mask Shader";
+		gDeferredShadowFullbrightAlphaMaskProgram.mFeatures.mIndexedTextureChannels = LLGLSLShader::sIndexedTextureChannels;
+
+		gDeferredShadowFullbrightAlphaMaskProgram.mShaderFiles.clear();
+		gDeferredShadowFullbrightAlphaMaskProgram.mShaderFiles.push_back(make_pair("deferred/shadowAlphaMaskV.glsl", GL_VERTEX_SHADER_ARB));
+		gDeferredShadowFullbrightAlphaMaskProgram.mShaderFiles.push_back(make_pair("deferred/shadowAlphaMaskF.glsl", GL_FRAGMENT_SHADER_ARB));
+
+        gDeferredShadowFullbrightAlphaMaskProgram.clearPermutations();
+		if (gGLManager.mHasDepthClamp)
+		{
+			gDeferredShadowFullbrightAlphaMaskProgram.addPermutation("DEPTH_CLAMP", "1");
+		}
+        gDeferredShadowFullbrightAlphaMaskProgram.addPermutation("IS_FULLBRIGHT", "1");
+		gDeferredShadowFullbrightAlphaMaskProgram.mShaderLevel = mShaderLevel[SHADER_DEFERRED];
+		success = gDeferredShadowFullbrightAlphaMaskProgram.createShader(NULL, NULL);
+		llassert(success);
+	}
+    
+    if (success)
+	{
 		gDeferredShadowAlphaMaskProgram.mName = "Deferred Shadow Alpha Mask Shader";
 		gDeferredShadowAlphaMaskProgram.mFeatures.mIndexedTextureChannels = LLGLSLShader::sIndexedTextureChannels;
 
diff --git a/indra/newview/llviewershadermgr.h b/indra/newview/llviewershadermgr.h
index 18be3c22aa..081221f15b 100644
--- a/indra/newview/llviewershadermgr.h
+++ b/indra/newview/llviewershadermgr.h
@@ -301,6 +301,7 @@ extern LLGLSLShader			gDeferredSoftenWaterProgram;
 extern LLGLSLShader			gDeferredShadowProgram;
 extern LLGLSLShader			gDeferredShadowCubeProgram;
 extern LLGLSLShader			gDeferredShadowAlphaMaskProgram;
+extern LLGLSLShader			gDeferredShadowFullbrightAlphaMaskProgram;
 extern LLGLSLShader			gDeferredPostProgram;
 extern LLGLSLShader			gDeferredCoFProgram;
 extern LLGLSLShader			gDeferredDoFCombineProgram;
diff --git a/indra/newview/llvovolume.cpp b/indra/newview/llvovolume.cpp
index 3740caa53f..d149a61d89 100644
--- a/indra/newview/llvovolume.cpp
+++ b/indra/newview/llvovolume.cpp
@@ -6680,7 +6680,7 @@ U32 LLVolumeGeometryManager::genDrawInfo(LLSpatialGroup* group, U32 mask, LLFace
 						{
 							registerFace(group, facep, LLRenderPass::PASS_SIMPLE);
 						}
-				}
+				    }
 				}
 				
 				
diff --git a/indra/newview/pipeline.cpp b/indra/newview/pipeline.cpp
index 6f3e6819c4..863785889f 100644
--- a/indra/newview/pipeline.cpp
+++ b/indra/newview/pipeline.cpp
@@ -7452,6 +7452,15 @@ void LLPipeline::renderMaskedObjects(U32 type, U32 mask, bool texture, bool batc
 	gGLLastMatrix = NULL;		
 }
 
+void LLPipeline::renderFullbrightMaskedObjects(U32 type, U32 mask, bool texture, bool batch_texture)
+{
+	assertInitialized();
+	gGL.loadMatrix(gGLModelView);
+	gGLLastMatrix = NULL;
+	mFullbrightAlphaMaskPool->pushMaskBatches(type, mask, texture, batch_texture);
+	gGL.loadMatrix(gGLModelView);
+	gGLLastMatrix = NULL;		
+}
 
 void apply_cube_face_rotation(U32 face)
 {
@@ -9608,6 +9617,7 @@ static LLTrace::BlockTimerStatHandle FTM_SHADOW_ALPHA_MASKED("Alpha Masked");
 static LLTrace::BlockTimerStatHandle FTM_SHADOW_ALPHA_BLEND("Alpha Blend");
 static LLTrace::BlockTimerStatHandle FTM_SHADOW_ALPHA_TREE("Alpha Tree");
 static LLTrace::BlockTimerStatHandle FTM_SHADOW_ALPHA_GRASS("Alpha Grass");
+static LLTrace::BlockTimerStatHandle FTM_SHADOW_FULLBRIGHT_ALPHA_MASKED("Fullbright Alpha Masked");
 
 void LLPipeline::renderShadow(glh::matrix4f& view, glh::matrix4f& proj, LLCamera& shadow_cam, LLCullResult &result, bool use_shader, bool use_occlusion, U32 target_width)
 {
@@ -9740,8 +9750,7 @@ void LLPipeline::renderShadow(glh::matrix4f& view, glh::matrix4f& proj, LLCamera
 
         {
             LL_RECORD_BLOCK_TIME(FTM_SHADOW_ALPHA_MASKED);
-            renderMaskedObjects(LLRenderPass::PASS_ALPHA_MASK, mask, TRUE, TRUE);
-            renderMaskedObjects(LLRenderPass::PASS_FULLBRIGHT_ALPHA_MASK, mask, TRUE, TRUE);
+            renderMaskedObjects(LLRenderPass::PASS_ALPHA_MASK, mask, TRUE, TRUE);            
         }
 
         {
@@ -9750,6 +9759,14 @@ void LLPipeline::renderShadow(glh::matrix4f& view, glh::matrix4f& proj, LLCamera
             renderObjects(LLRenderPass::PASS_ALPHA, mask, TRUE, TRUE);
         }
 
+        {
+            LL_RECORD_BLOCK_TIME(FTM_SHADOW_FULLBRIGHT_ALPHA_MASKED);
+            gDeferredShadowFullbrightAlphaMaskProgram.bind();
+            gDeferredShadowFullbrightAlphaMaskProgram.uniform1f(LLShaderMgr::DEFERRED_SHADOW_TARGET_WIDTH, (float)target_width);
+            gDeferredShadowFullbrightAlphaMaskProgram.uniform1i(LLShaderMgr::SUN_UP_FACTOR, environment.getIsSunUp() ? 1 : 0);
+            renderFullbrightMaskedObjects(LLRenderPass::PASS_FULLBRIGHT_ALPHA_MASK, mask, TRUE, TRUE);
+        }
+
         mask = mask & ~LLVertexBuffer::MAP_TEXTURE_INDEX;
 
         {
diff --git a/indra/newview/pipeline.h b/indra/newview/pipeline.h
index fe81ae91e8..b0c40d145b 100644
--- a/indra/newview/pipeline.h
+++ b/indra/newview/pipeline.h
@@ -261,6 +261,7 @@ public:
 
 	void renderObjects(U32 type, U32 mask, bool texture = true, bool batch_texture = false);
 	void renderMaskedObjects(U32 type, U32 mask, bool texture = true, bool batch_texture = false);
+    void renderFullbrightMaskedObjects(U32 type, U32 mask, bool texture = true, bool batch_texture = false);
 
 	void renderGroups(LLRenderPass* pass, U32 type, U32 mask, bool texture);