SL-18669 Fix for broken lighting on transparent faces. Optimize handling of "bindDeferredShader" and shadow map setup.

3 files changed, 180 insertions, 101 deletions

diff --git a/indra/newview/lldrawpoolalpha.cpp b/indra/newview/lldrawpoolalpha.cpp
index 5ed6d3cc2a..437a0c2ea1 100644
--- a/indra/newview/lldrawpoolalpha.cpp
+++ b/indra/newview/lldrawpoolalpha.cpp
@@ -183,10 +183,10 @@ void LLDrawPoolAlpha::renderPostDeferred(S32 pass)
     LLGLSLShader* materialShader = LLPipeline::sUnderWaterRender ? gDeferredMaterialWaterProgram : gDeferredMaterialProgram;
     for (int i = 0; i < LLMaterial::SHADER_COUNT*2; ++i)
     {
-        prepare_alpha_shader(&materialShader[i], false, false, water_sign);
+        prepare_alpha_shader(&materialShader[i], false, true, water_sign);
     }
 
-    prepare_alpha_shader(&gDeferredPBRAlphaProgram, false, false, water_sign);
+    prepare_alpha_shader(&gDeferredPBRAlphaProgram, false, true, water_sign);
 
     // first pass, render rigged objects only and render to depth buffer
     forwardRender(true);
@@ -674,10 +674,10 @@ void LLDrawPoolAlpha::renderAlpha(U32 mask, bool depth_only, bool rigged)
                         target_shader = target_shader->mRiggedVariant;
                     }
 
+                    // shader must be bound before LLGLTFMaterial::bind
                     if (current_shader != target_shader)
                     {
-                        target_shader->bind();
-                        //gPipeline.bindDeferredShader(*target_shader);
+                        gPipeline.bindDeferredShaderFast(*target_shader);
                     }
 
                     params.mGLTFMaterial->bind(target_shader);
@@ -725,12 +725,6 @@ void LLDrawPoolAlpha::renderAlpha(U32 mask, bool depth_only, bool rigged)
                             llassert(target_shader->mRiggedVariant != nullptr);
                             target_shader = target_shader->mRiggedVariant;
                         }
-
-                        if (current_shader != target_shader)
-                        {
-                            //gPipeline.bindDeferredShader(*target_shader);
-                            target_shader->bind();
-                        }
                     }
                     else if (!params.mFullbright)
                     {
@@ -750,8 +744,7 @@ void LLDrawPoolAlpha::renderAlpha(U32 mask, bool depth_only, bool rigged)
                     if (current_shader != target_shader)
                     {// If we need shaders, and we're not ALREADY using the proper shader, then bind it
                     // (this way we won't rebind shaders unnecessarily).
-                        //gPipeline.bindDeferredShader(*target_shader);
-                        target_shader->bind();
+                        gPipeline.bindDeferredShaderFast(*target_shader);
                     }
 
                     LLVector4 spec_color(1, 1, 1, 1);
diff --git a/indra/newview/pipeline.cpp b/indra/newview/pipeline.cpp
index a2e01eec3f..b59da7ad19 100644
--- a/indra/newview/pipeline.cpp
+++ b/indra/newview/pipeline.cpp
@@ -904,26 +904,53 @@ inline U32 BlurHappySize(U32 x, F32 scale) { return U32( x * scale + 16.0f) & ~0
 bool LLPipeline::allocateShadowBuffer(U32 resX, U32 resY)
 {
     LL_PROFILE_ZONE_SCOPED_CATEGORY_DISPLAY;
-    if (LLPipeline::sRenderDeferred)
-    {
-        S32 shadow_detail = RenderShadowDetail;
+    S32 shadow_detail = RenderShadowDetail;
+
+    const U32 occlusion_divisor = 3;
 
-        const U32 occlusion_divisor = 3;
+    F32 scale = llmax(0.f, RenderShadowResolutionScale);
+    U32 sun_shadow_map_width = BlurHappySize(resX, scale);
+    U32 sun_shadow_map_height = BlurHappySize(resY, scale);
+
+    if (shadow_detail > 0)
+    { //allocate 4 sun shadow maps
+        for (U32 i = 0; i < 4; i++)
+        {
+            if (!mRT->shadow[i].allocate(sun_shadow_map_width, sun_shadow_map_height, 0, TRUE, FALSE, LLTexUnit::TT_TEXTURE))
+            {
+                return false;
+            }
+
+            if (!mRT->shadowOcclusion[i].allocate(sun_shadow_map_width / occlusion_divisor, sun_shadow_map_height / occlusion_divisor, 0, TRUE, FALSE, LLTexUnit::TT_TEXTURE))
+            {
+                return false;
+            }
+        }
+    }
+    else
+    {
+        for (U32 i = 0; i < 4; i++)
+        {
+            releaseSunShadowTarget(i);
+        }
+    }
 
-        F32 scale = llmax(0.f, RenderShadowResolutionScale);
-        U32 sun_shadow_map_width = BlurHappySize(resX, scale);
-        U32 sun_shadow_map_height = BlurHappySize(resY, scale);
+    if (!gCubeSnapshot) // hack to not allocate spot shadow maps during ReflectionMapManager init
+    {
+        U32 width = (U32)(resX * scale);
+        U32 height = width;
 
-        if (shadow_detail > 0)
-        { //allocate 4 sun shadow maps
-            for (U32 i = 0; i < 4; i++)
+        if (shadow_detail > 1)
+        { //allocate two spot shadow maps
+            U32 spot_shadow_map_width = width;
+            U32 spot_shadow_map_height = height;
+            for (U32 i = 0; i < 2; i++)
             {
-                if (!mRT->shadow[i].allocate(sun_shadow_map_width, sun_shadow_map_height, 0, TRUE, FALSE, LLTexUnit::TT_TEXTURE))
+                if (!mSpotShadow[i].allocate(spot_shadow_map_width, spot_shadow_map_height, 0, TRUE, FALSE))
                 {
                     return false;
                 }
-
-                if (!mRT->shadowOcclusion[i].allocate(sun_shadow_map_width / occlusion_divisor, sun_shadow_map_height / occlusion_divisor, 0, TRUE, FALSE, LLTexUnit::TT_TEXTURE))
+                if (!mSpotShadowOcclusion[i].allocate(spot_shadow_map_width / occlusion_divisor, height / occlusion_divisor, 0, TRUE, FALSE))
                 {
                     return false;
                 }
@@ -931,36 +958,42 @@ bool LLPipeline::allocateShadowBuffer(U32 resX, U32 resY)
         }
         else
         {
-            for (U32 i = 0; i < 4; i++)
-            {
-                releaseSunShadowTarget(i);
-            }
+            releaseSpotShadowTargets();
         }
+    }
 
-        if (!gCubeSnapshot) // hack to not allocate spot shadow maps during ReflectionMapManager init
+
+    // set up shadow map filtering and compare modes
+    if (shadow_detail > 0)
+    { 
+        for (U32 i = 0; i < 4; i++)
         {
-            U32 width = (U32)(resX * scale);
-            U32 height = width;
+            LLRenderTarget* shadow_target = getSunShadowTarget(i);
+            if (shadow_target)
+            {
+                gGL.getTexUnit(0)->bind(getSunShadowTarget(i), TRUE);
+                gGL.getTexUnit(0)->setTextureFilteringOption(LLTexUnit::TFO_ANISOTROPIC);
+                gGL.getTexUnit(0)->setTextureAddressMode(LLTexUnit::TAM_CLAMP);
 
-            if (shadow_detail > 1)
-            { //allocate two spot shadow maps
-                U32 spot_shadow_map_width = width;
-                U32 spot_shadow_map_height = height;
-                for (U32 i = 0; i < 2; i++)
-                {
-                    if (!mSpotShadow[i].allocate(spot_shadow_map_width, spot_shadow_map_height, 0, TRUE, FALSE))
-                    {
-                        return false;
-                    }
-                    if (!mSpotShadowOcclusion[i].allocate(spot_shadow_map_width / occlusion_divisor, height / occlusion_divisor, 0, TRUE, FALSE))
-                    {
-                        return false;
-                    }
-                }
+                glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_COMPARE_MODE, GL_COMPARE_R_TO_TEXTURE);
+                glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_COMPARE_FUNC, GL_LEQUAL);
             }
-            else
+        }
+    }
+
+    if (shadow_detail > 1)
+    {
+        for (U32 i = 0; i < 2; i++)
+        {
+            LLRenderTarget* shadow_target = getSpotShadowTarget(i);
+            if (shadow_target)
             {
-                releaseSpotShadowTargets();
+                gGL.getTexUnit(0)->bind(shadow_target, TRUE);
+                gGL.getTexUnit(0)->setTextureFilteringOption(LLTexUnit::TFO_ANISOTROPIC);
+                gGL.getTexUnit(0)->setTextureAddressMode(LLTexUnit::TAM_CLAMP);
+
+                glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_COMPARE_MODE, GL_COMPARE_R_TO_TEXTURE);
+                glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_COMPARE_FUNC, GL_LEQUAL);
             }
         }
     }
@@ -8248,6 +8281,102 @@ void LLPipeline::renderFinalize()
     recordTrianglesDrawn();
 }
 
+void LLPipeline::bindLightFunc(LLGLSLShader& shader)
+{
+    S32 channel = shader.enableTexture(LLShaderMgr::DEFERRED_LIGHTFUNC);
+    if (channel > -1)
+    {
+        gGL.getTexUnit(channel)->bindManual(LLTexUnit::TT_TEXTURE, mLightFunc);
+    }
+}
+
+void LLPipeline::bindShadowMaps(LLGLSLShader& shader)
+{
+    for (U32 i = 0; i < 4; i++)
+    {
+        LLRenderTarget* shadow_target = getSunShadowTarget(i);
+        if (shadow_target)
+        {
+            S32 channel = shader.enableTexture(LLShaderMgr::DEFERRED_SHADOW0 + i, LLTexUnit::TT_TEXTURE);
+            if (channel > -1)
+            {
+                gGL.getTexUnit(channel)->bind(getSunShadowTarget(i), TRUE);
+            }
+        }
+    }
+
+    for (U32 i = 4; i < 6; i++)
+    {
+        S32 channel = shader.enableTexture(LLShaderMgr::DEFERRED_SHADOW0 + i);
+        if (channel > -1)
+        {
+            LLRenderTarget* shadow_target = getSpotShadowTarget(i - 4);
+            if (shadow_target)
+            {
+                gGL.getTexUnit(channel)->bind(shadow_target, TRUE);
+            }
+        }
+    }
+}
+
+void LLPipeline::bindDeferredShaderFast(LLGLSLShader& shader)
+{
+    shader.bind();
+    bindLightFunc(shader);
+    bindShadowMaps(shader);
+    bindReflectionProbes(shader);
+
+#if 0
+    shader.uniform1f(LLShaderMgr::DEFERRED_SUN_WASH, RenderDeferredSunWash);
+    shader.uniform1f(LLShaderMgr::DEFERRED_SHADOW_NOISE, RenderShadowNoise);
+    shader.uniform1f(LLShaderMgr::DEFERRED_BLUR_SIZE, RenderShadowBlurSize);
+
+    shader.uniform1f(LLShaderMgr::DEFERRED_SSAO_RADIUS, RenderSSAOScale);
+    shader.uniform1f(LLShaderMgr::DEFERRED_SSAO_MAX_RADIUS, RenderSSAOMaxScale);
+
+    F32 ssao_factor = RenderSSAOFactor;
+    shader.uniform1f(LLShaderMgr::DEFERRED_SSAO_FACTOR, ssao_factor);
+    shader.uniform1f(LLShaderMgr::DEFERRED_SSAO_FACTOR_INV, 1.0 / ssao_factor);
+
+    LLVector3 ssao_effect = RenderSSAOEffect;
+    F32 matrix_diag = (ssao_effect[0] + 2.0 * ssao_effect[1]) / 3.0;
+    F32 matrix_nondiag = (ssao_effect[0] - ssao_effect[1]) / 3.0;
+    // This matrix scales (proj of color onto <1/rt(3),1/rt(3),1/rt(3)>) by
+    // value factor, and scales remainder by saturation factor
+    F32 ssao_effect_mat[] = { matrix_diag, matrix_nondiag, matrix_nondiag,
+                                matrix_nondiag, matrix_diag, matrix_nondiag,
+                                matrix_nondiag, matrix_nondiag, matrix_diag };
+    shader.uniformMatrix3fv(LLShaderMgr::DEFERRED_SSAO_EFFECT_MAT, 1, GL_FALSE, ssao_effect_mat);
+
+    //F32 shadow_offset_error = 1.f + RenderShadowOffsetError * fabsf(LLViewerCamera::getInstance()->getOrigin().mV[2]);
+    F32 shadow_bias_error = RenderShadowBiasError * fabsf(LLViewerCamera::getInstance()->getOrigin().mV[2]) / 3000.f;
+    F32 shadow_bias = RenderShadowBias + shadow_bias_error;
+
+    //shader.uniform2f(LLShaderMgr::DEFERRED_SCREEN_RES, deferred_target->getWidth(), deferred_target->getHeight());
+    shader.uniform1f(LLShaderMgr::DEFERRED_NEAR_CLIP, LLViewerCamera::getInstance()->getNear() * 2.f);
+    shader.uniform1f(LLShaderMgr::DEFERRED_SHADOW_OFFSET, RenderShadowOffset); //*shadow_offset_error);
+    shader.uniform1f(LLShaderMgr::DEFERRED_SHADOW_BIAS, shadow_bias);
+    shader.uniform1f(LLShaderMgr::DEFERRED_SPOT_SHADOW_OFFSET, RenderSpotShadowOffset);
+    shader.uniform1f(LLShaderMgr::DEFERRED_SPOT_SHADOW_BIAS, RenderSpotShadowBias);
+
+    shader.uniform3fv(LLShaderMgr::DEFERRED_SUN_DIR, 1, mTransformedSunDir.mV);
+    shader.uniform3fv(LLShaderMgr::DEFERRED_MOON_DIR, 1, mTransformedMoonDir.mV);
+    shader.uniform2f(LLShaderMgr::DEFERRED_SHADOW_RES, mRT->shadow[0].getWidth(), mRT->shadow[0].getHeight());
+    shader.uniform2f(LLShaderMgr::DEFERRED_PROJ_SHADOW_RES, mSpotShadow[0].getWidth(), mSpotShadow[0].getHeight());
+    shader.uniform1f(LLShaderMgr::DEFERRED_DEPTH_CUTOFF, RenderEdgeDepthCutoff);
+    shader.uniform1f(LLShaderMgr::DEFERRED_NORM_CUTOFF, RenderEdgeNormCutoff);
+
+    if (shader.getUniformLocation(LLShaderMgr::DEFERRED_NORM_MATRIX) >= 0)
+    {
+        glh::matrix4f norm_mat = get_current_modelview().inverse().transpose();
+        shader.uniformMatrix4fv(LLShaderMgr::DEFERRED_NORM_MATRIX, 1, FALSE, norm_mat.m);
+    }
+
+    shader.uniform3fv(LLShaderMgr::SUNLIGHT_COLOR, 1, mSunDiffuse.mV);
+    shader.uniform3fv(LLShaderMgr::MOONLIGHT_COLOR, 1, mMoonDiffuse.mV);
+#endif
+}
+
 void LLPipeline::bindDeferredShader(LLGLSLShader& shader, LLRenderTarget* light_target)
 {
     LL_PROFILE_ZONE_SCOPED_CATEGORY_PIPELINE;
@@ -8292,7 +8421,6 @@ void LLPipeline::bindDeferredShader(LLGLSLShader& shader, LLRenderTarget* light_
         mPbrBrdfLut.bindTexture(0, channel);
     }
 
-
 #if 0
     channel = shader.enableTexture(LLShaderMgr::DEFERRED_DEPTH, deferred_depth_target->getUsage());
 	if (channel > -1)
@@ -8329,11 +8457,7 @@ void LLPipeline::bindDeferredShader(LLGLSLShader& shader, LLRenderTarget* light_
 		gGL.getTexUnit(channel)->setTextureFilteringOption(LLTexUnit::TFO_POINT);
 	}
 
-	channel = shader.enableTexture(LLShaderMgr::DEFERRED_LIGHTFUNC);
-	if (channel > -1)
-	{
-		gGL.getTexUnit(channel)->bindManual(LLTexUnit::TT_TEXTURE, mLightFunc);
-	}
+    bindLightFunc(shader);
 
 	stop_glerror();
 
@@ -8352,49 +8476,7 @@ void LLPipeline::bindDeferredShader(LLGLSLShader& shader, LLRenderTarget* light_
 
 	stop_glerror();
 
-	for (U32 i = 0; i < 4; i++)
-	{
-        LLRenderTarget* shadow_target = getSunShadowTarget(i);
-        if (shadow_target)
-        {
-		channel = shader.enableTexture(LLShaderMgr::DEFERRED_SHADOW0+i, LLTexUnit::TT_TEXTURE);
-		stop_glerror();
-		if (channel > -1)
-		{
-			stop_glerror();
-                gGL.getTexUnit(channel)->bind(getSunShadowTarget(i), TRUE);
-                gGL.getTexUnit(channel)->setTextureFilteringOption(LLTexUnit::TFO_ANISOTROPIC);
-			gGL.getTexUnit(channel)->setTextureAddressMode(LLTexUnit::TAM_CLAMP);
-			stop_glerror();
-			
-			glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_COMPARE_MODE, GL_COMPARE_R_TO_TEXTURE);
-			glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_COMPARE_FUNC, GL_LEQUAL);
-			stop_glerror();
-		}
-	}
-    }
-
-    for (U32 i = 4; i < 6; i++)
-    {
-        channel = shader.enableTexture(LLShaderMgr::DEFERRED_SHADOW0 + i);
-        stop_glerror();
-        if (channel > -1)
-        {
-            stop_glerror();
-            LLRenderTarget* shadow_target = getSpotShadowTarget(i-4);
-            if (shadow_target)
-            {
-                gGL.getTexUnit(channel)->bind(shadow_target, TRUE);
-                gGL.getTexUnit(channel)->setTextureFilteringOption(LLTexUnit::TFO_ANISOTROPIC);
-                gGL.getTexUnit(channel)->setTextureAddressMode(LLTexUnit::TAM_CLAMP);
-                stop_glerror();
-
-                glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_COMPARE_MODE, GL_COMPARE_R_TO_TEXTURE);
-                glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_COMPARE_FUNC, GL_LEQUAL);
-                stop_glerror();
-            }
-        }
-    }
+    bindShadowMaps(shader);
 
 	stop_glerror();
 
@@ -8517,11 +8599,9 @@ void LLPipeline::bindDeferredShader(LLGLSLShader& shader, LLRenderTarget* light_
 
     shader.uniform3fv(LLShaderMgr::SUNLIGHT_COLOR, 1, mSunDiffuse.mV);
     shader.uniform3fv(LLShaderMgr::MOONLIGHT_COLOR, 1, mMoonDiffuse.mV);
-
-    LLEnvironment& environment = LLEnvironment::instance();
-    LLSettingsSky::ptr_t sky = environment.getCurrentSky();
 }
 
+
 LLColor3 pow3f(LLColor3 v, F32 f)
 {
 	v.mV[0] = powf(v.mV[0], f);
diff --git a/indra/newview/pipeline.h b/indra/newview/pipeline.h
index 5b068fc386..0fbc5cf960 100644
--- a/indra/newview/pipeline.h
+++ b/indra/newview/pipeline.h
@@ -293,6 +293,12 @@ public:
 	void renderGeomDeferred(LLCamera& camera, bool do_occlusion = false);
 	void renderGeomPostDeferred(LLCamera& camera);
 	void renderGeomShadow(LLCamera& camera);
+    void bindLightFunc(LLGLSLShader& shader);
+
+    // bind shadow maps
+    // if setup is true, wil lset texture compare mode function and filtering options
+    void bindShadowMaps(LLGLSLShader& shader);
+    void bindDeferredShaderFast(LLGLSLShader& shader);
 	void bindDeferredShader(LLGLSLShader& shader, LLRenderTarget* light_target = nullptr);
 	void setupSpotLight(LLGLSLShader& shader, LLDrawable* drawablep);