SL-19838 Fix for transparent PBR materials always casting shadows.

5 files changed, 75 insertions, 4 deletions

diff --git a/indra/newview/app_settings/shaders/class1/deferred/pbrShadowAlphaBlendF.glsl b/indra/newview/app_settings/shaders/class1/deferred/pbrShadowAlphaBlendF.glsl
new file mode 100644
index 0000000000..79425ad716
--- /dev/null
+++ b/indra/newview/app_settings/shaders/class1/deferred/pbrShadowAlphaBlendF.glsl
@@ -0,0 +1,56 @@
+/** 
+ * @file pbrShadowAlphaBlendF.glsl
+ *
+ * $LicenseInfo:firstyear=2023&license=viewerlgpl$
+ * Second Life Viewer Source Code
+ * Copyright (C) 2023, Linden Research, Inc.
+ * 
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation;
+ * version 2.1 of the License only.
+ * 
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ * 
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ * 
+ * Linden Research, Inc., 945 Battery Street, San Francisco, CA  94111  USA
+ * $/LicenseInfo$
+ */
+
+out vec4 frag_color;
+
+uniform sampler2D diffuseMap;
+
+in vec4 post_pos;
+in float target_pos_x;
+in vec4 vertex_color;
+in vec2 vary_texcoord0;
+uniform float minimum_alpha;
+
+void main() 
+{
+    float alpha = texture(diffuseMap,vary_texcoord0.xy).a;
+
+    alpha *= vertex_color.a;
+
+    if (alpha < 0.05) // treat as totally transparent
+    {
+        discard;
+    }
+
+    if (alpha < 0.88) // treat as semi-transparent
+    {
+        if (fract(0.5*floor(target_pos_x / post_pos.w )) < 0.25)
+        {
+            discard;
+        }
+    }
+
+	frag_color = vec4(1,1,1,1);
+}
diff --git a/indra/newview/app_settings/shaders/class1/deferred/pbrShadowAlphaMaskF.glsl b/indra/newview/app_settings/shaders/class1/deferred/pbrShadowAlphaMaskF.glsl
index e06f3a5f44..1d1545be7e 100644
--- a/indra/newview/app_settings/shaders/class1/deferred/pbrShadowAlphaMaskF.glsl
+++ b/indra/newview/app_settings/shaders/class1/deferred/pbrShadowAlphaMaskF.glsl
@@ -35,7 +35,7 @@ uniform float minimum_alpha;
 
 void main() 
 {
-    float alpha = diffuseLookup(vary_texcoord0.xy).a;
+    float alpha = texture(diffuseMap,vary_texcoord0.xy).a;
 
     if (alpha < minimum_alpha)
     {
diff --git a/indra/newview/llviewershadermgr.cpp b/indra/newview/llviewershadermgr.cpp
index 5ec78df6c4..5eae1dfb18 100644
--- a/indra/newview/llviewershadermgr.cpp
+++ b/indra/newview/llviewershadermgr.cpp
@@ -158,6 +158,8 @@ LLGLSLShader			gDeferredShadowAlphaMaskProgram;
 LLGLSLShader            gDeferredSkinnedShadowAlphaMaskProgram;
 LLGLSLShader			gDeferredShadowGLTFAlphaMaskProgram;
 LLGLSLShader			gDeferredSkinnedShadowGLTFAlphaMaskProgram;
+LLGLSLShader            gDeferredShadowGLTFAlphaBlendProgram;
+LLGLSLShader            gDeferredSkinnedShadowGLTFAlphaBlendProgram;
 LLGLSLShader			gDeferredShadowFullbrightAlphaMaskProgram;
 LLGLSLShader            gDeferredSkinnedShadowFullbrightAlphaMaskProgram;
 LLGLSLShader			gDeferredAvatarShadowProgram;
@@ -2364,7 +2366,6 @@ BOOL LLViewerShaderMgr::loadShadersDeferred()
     if (success)
     {
         gDeferredShadowGLTFAlphaMaskProgram.mName = "Deferred GLTF Shadow Alpha Mask Shader";
-        gDeferredShadowGLTFAlphaMaskProgram.mFeatures.mIndexedTextureChannels = LLGLSLShader::sIndexedTextureChannels;
         gDeferredShadowGLTFAlphaMaskProgram.mShaderFiles.clear();
         gDeferredShadowGLTFAlphaMaskProgram.mShaderFiles.push_back(make_pair("deferred/pbrShadowAlphaMaskV.glsl", GL_VERTEX_SHADER));
         gDeferredShadowGLTFAlphaMaskProgram.mShaderFiles.push_back(make_pair("deferred/pbrShadowAlphaMaskF.glsl", GL_FRAGMENT_SHADER));
@@ -2375,6 +2376,19 @@ BOOL LLViewerShaderMgr::loadShadersDeferred()
         llassert(success);
     }
 
+    if (success)
+    {
+        gDeferredShadowGLTFAlphaBlendProgram.mName = "Deferred GLTF Shadow Alpha Blend Shader";
+        gDeferredShadowGLTFAlphaBlendProgram.mShaderFiles.clear();
+        gDeferredShadowGLTFAlphaBlendProgram.mShaderFiles.push_back(make_pair("deferred/pbrShadowAlphaMaskV.glsl", GL_VERTEX_SHADER));
+        gDeferredShadowGLTFAlphaBlendProgram.mShaderFiles.push_back(make_pair("deferred/pbrShadowAlphaBlendF.glsl", GL_FRAGMENT_SHADER));
+        gDeferredShadowGLTFAlphaBlendProgram.mShaderLevel = mShaderLevel[SHADER_DEFERRED];
+        gDeferredShadowGLTFAlphaBlendProgram.clearPermutations();
+        success = make_rigged_variant(gDeferredShadowGLTFAlphaBlendProgram, gDeferredSkinnedShadowGLTFAlphaBlendProgram);
+        success = success && gDeferredShadowGLTFAlphaBlendProgram.createShader(NULL, NULL);
+        llassert(success);
+    }
+
 	if (success)
 	{
 		gDeferredAvatarShadowProgram.mName = "Deferred Avatar Shadow Shader";
diff --git a/indra/newview/llviewershadermgr.h b/indra/newview/llviewershadermgr.h
index 76e81e17b7..b0b9719d76 100644
--- a/indra/newview/llviewershadermgr.h
+++ b/indra/newview/llviewershadermgr.h
@@ -227,6 +227,7 @@ extern LLGLSLShader			gDeferredShadowProgram;
 extern LLGLSLShader			gDeferredShadowCubeProgram;
 extern LLGLSLShader			gDeferredShadowAlphaMaskProgram;
 extern LLGLSLShader         gDeferredShadowGLTFAlphaMaskProgram;
+extern LLGLSLShader         gDeferredShadowGLTFAlphaBlendProgram;
 extern LLGLSLShader			gDeferredShadowFullbrightAlphaMaskProgram;
 extern LLGLSLShader			gDeferredPostProgram;
 extern LLGLSLShader			gDeferredCoFProgram;
diff --git a/indra/newview/pipeline.cpp b/indra/newview/pipeline.cpp
index a41bed9186..c88b8b42ff 100644
--- a/indra/newview/pipeline.cpp
+++ b/indra/newview/pipeline.cpp
@@ -6514,7 +6514,7 @@ void LLPipeline::renderAlphaObjects(bool rigged)
         {
             if (pparams->mGLTFMaterial)
             {
-                gDeferredShadowGLTFAlphaMaskProgram.bind(rigged);
+                gDeferredShadowGLTFAlphaBlendProgram.bind(rigged);
                 LLGLSLShader::sCurBoundShaderPtr->uniform1i(LLShaderMgr::SUN_UP_FACTOR, sun_up);
                 LLGLSLShader::sCurBoundShaderPtr->uniform1f(LLShaderMgr::DEFERRED_SHADOW_TARGET_WIDTH, (float)target_width);
                 LLGLSLShader::sCurBoundShaderPtr->setMinimumAlpha(ALPHA_BLEND_CUTOFF);
@@ -6540,7 +6540,7 @@ void LLPipeline::renderAlphaObjects(bool rigged)
         {
             if (pparams->mGLTFMaterial)
             {
-                gDeferredShadowGLTFAlphaMaskProgram.bind(rigged);
+                gDeferredShadowGLTFAlphaBlendProgram.bind(rigged);
                 LLGLSLShader::sCurBoundShaderPtr->uniform1i(LLShaderMgr::SUN_UP_FACTOR, sun_up);
                 LLGLSLShader::sCurBoundShaderPtr->uniform1f(LLShaderMgr::DEFERRED_SHADOW_TARGET_WIDTH, (float)target_width);
                 LLGLSLShader::sCurBoundShaderPtr->setMinimumAlpha(ALPHA_BLEND_CUTOFF);