Fix sampling normals from gbuffer and cleanup redundant gbuffer samples (#1294)

* Fix sampling normals from gbuffer and cleanup redundant gbuffer samples

* Allow negative normals in gbuffer to fix broken material rendering

14 files changed, 43 insertions, 62 deletions

diff --git a/indra/newview/app_settings/shaders/class1/deferred/blurLightF.glsl b/indra/newview/app_settings/shaders/class1/deferred/blurLightF.glsl
index 08baf98686..8627ab1852 100644
--- a/indra/newview/app_settings/shaders/class1/deferred/blurLightF.glsl
+++ b/indra/newview/app_settings/shaders/class1/deferred/blurLightF.glsl
@@ -40,12 +40,12 @@ uniform float kern_scale;
 in vec2 vary_fragcoord;
 
 vec4 getPosition(vec2 pos_screen);
-vec3 getNorm(vec2 pos_screen);
+vec4 getNorm(vec2 pos_screen);
 
 void main() 
 {
     vec2 tc = vary_fragcoord.xy;
-    vec3 norm = getNorm(tc);
+    vec4 norm = getNorm(tc);
     vec3 pos = getPosition(tc).xyz;
     vec4 ccol = texture(lightMap, tc).rgba;
   
diff --git a/indra/newview/app_settings/shaders/class1/deferred/deferredUtil.glsl b/indra/newview/app_settings/shaders/class1/deferred/deferredUtil.glsl
index 065fcec45a..f6696e270c 100644
--- a/indra/newview/app_settings/shaders/class1/deferred/deferredUtil.glsl
+++ b/indra/newview/app_settings/shaders/class1/deferred/deferredUtil.glsl
@@ -141,31 +141,20 @@ vec2 getScreenCoordinate(vec2 screenpos)
     return sc - vec2(1.0, 1.0);
 }
 
-vec3 getNorm(vec2 screenpos)
+vec4 getNorm(vec2 screenpos)
 {
-    return texture(normalMap, screenpos.xy).rgb;
-}
-
-vec3 getNormalFromPacked(vec4 packedNormalEnvIntensityFlags)
-{
-   vec2 enc = packedNormalEnvIntensityFlags.xy;
-   vec2 fenc = enc*4-2;
-   float f = dot(fenc,fenc);
-   float g = sqrt(1-f/4);
-   vec3 n;
-   n.xy = fenc*g;
-   n.z = 1-f/2;
-   return normalize(n); // TODO: Is this normalize redundant?
+    return texture(normalMap, screenpos.xy);
 }
 
 // return packedNormalEnvIntensityFlags since GBUFFER_FLAG_HAS_PBR needs .w
 // See: C++: addDeferredAttachments(), GLSL: softenLightF
 vec4 getNormalEnvIntensityFlags(vec2 screenpos, out vec3 n, out float envIntensity)
 {
-    n = texture(normalMap, screenpos.xy).rgb;
+    vec4 norm = texture(normalMap, screenpos.xy);
+    n = norm.xyz;
     envIntensity = texture(emissiveRect, screenpos.xy).r;
 
-    return vec4(n, envIntensity);
+    return norm;
 }
 
 // get linear depth value given a depth buffer sample d and znear and zfar values
diff --git a/indra/newview/app_settings/shaders/class1/deferred/pbropaqueF.glsl b/indra/newview/app_settings/shaders/class1/deferred/pbropaqueF.glsl
index 55fc94b7f5..1d5f810cf3 100644
--- a/indra/newview/app_settings/shaders/class1/deferred/pbropaqueF.glsl
+++ b/indra/newview/app_settings/shaders/class1/deferred/pbropaqueF.glsl
@@ -109,7 +109,7 @@ void main()
     // See: C++: addDeferredAttachments(), GLSL: softenLightF
     frag_data[0] = max(vec4(col, 0.0), vec4(0));                                                   // Diffuse
     frag_data[1] = max(vec4(spec.rgb,vertex_color.a), vec4(0));                                    // PBR linear packed Occlusion, Roughness, Metal.
-    frag_data[2] = max(vec4(tnorm, GBUFFER_FLAG_HAS_PBR), vec4(0)); // normal, environment intensity, flags
+    frag_data[2] = vec4(tnorm, GBUFFER_FLAG_HAS_PBR); // normal, environment intensity, flags
     frag_data[3] = max(vec4(emissive,0), vec4(0));                                                // PBR sRGB Emissive
 }
 
diff --git a/indra/newview/app_settings/shaders/class1/deferred/pbrterrainF.glsl b/indra/newview/app_settings/shaders/class1/deferred/pbrterrainF.glsl
index 025ecb64df..de4745c1c4 100644
--- a/indra/newview/app_settings/shaders/class1/deferred/pbrterrainF.glsl
+++ b/indra/newview/app_settings/shaders/class1/deferred/pbrterrainF.glsl
@@ -341,7 +341,7 @@ void main()
 #endif
     frag_data[0] = max(vec4(mix.col.xyz, 0.0), vec4(0));                                                   // Diffuse
     frag_data[1] = max(vec4(orm.rgb, base_color_factor_alpha), vec4(0));                                    // PBR linear packed Occlusion, Roughness, Metal.
-    frag_data[2] = max(vec4(tnorm, GBUFFER_FLAG_HAS_PBR), vec4(0)); // normal, flags
+    frag_data[2] = vec4(tnorm, GBUFFER_FLAG_HAS_PBR); // normal, flags
     frag_data[3] = max(vec4(emissive,0), vec4(0));                                                // PBR sRGB Emissive
 }
 
diff --git a/indra/newview/app_settings/shaders/class2/deferred/sunLightF.glsl b/indra/newview/app_settings/shaders/class2/deferred/sunLightF.glsl
index 1ea57516a4..1bd5f5a718 100644
--- a/indra/newview/app_settings/shaders/class2/deferred/sunLightF.glsl
+++ b/indra/newview/app_settings/shaders/class2/deferred/sunLightF.glsl
@@ -35,7 +35,7 @@ in vec2 vary_fragcoord;
 uniform vec3 sun_dir;
 uniform float shadow_bias;
 
-vec3 getNorm(vec2 pos_screen);
+vec4 getNorm(vec2 pos_screen);
 vec4 getPosition(vec2 pos_screen);
 
 float sampleDirectionalShadow(vec3 pos, vec3 norm, vec2 pos_screen);
@@ -45,13 +45,13 @@ void main()
 {
     vec2 pos_screen = vary_fragcoord.xy;
     vec4 pos        = getPosition(pos_screen);
-    vec3 norm       = getNorm(pos_screen);
+    vec4 norm       = getNorm(pos_screen);
 
     vec4 col;
-    col.r = sampleDirectionalShadow(pos.xyz, norm, pos_screen);
+    col.r = sampleDirectionalShadow(pos.xyz, norm.xyz, pos_screen);
     col.g = 1.0f;
-    col.b = sampleSpotShadow(pos.xyz, norm, 0, pos_screen); 
-    col.a = sampleSpotShadow(pos.xyz, norm, 1, pos_screen);
+    col.b = sampleSpotShadow(pos.xyz, norm.xyz, 0, pos_screen); 
+    col.a = sampleSpotShadow(pos.xyz, norm.xyz, 1, pos_screen);
 
     frag_color = clamp(col, vec4(0), vec4(1));
 }
diff --git a/indra/newview/app_settings/shaders/class2/deferred/sunLightSSAOF.glsl b/indra/newview/app_settings/shaders/class2/deferred/sunLightSSAOF.glsl
index 0126e09d4c..e0333b6044 100644
--- a/indra/newview/app_settings/shaders/class2/deferred/sunLightSSAOF.glsl
+++ b/indra/newview/app_settings/shaders/class2/deferred/sunLightSSAOF.glsl
@@ -32,7 +32,7 @@ out vec4 frag_color;
 in vec2 vary_fragcoord;
 
 vec4 getPosition(vec2 pos_screen);
-vec3 getNorm(vec2 pos_screen);
+vec4 getNorm(vec2 pos_screen);
 
 float sampleDirectionalShadow(vec3 shadow_pos, vec3 norm, vec2 pos_screen);
 float sampleSpotShadow(vec3 shadow_pos, vec3 norm, int index, vec2 pos_screen);
@@ -42,13 +42,13 @@ void main()
 {
     vec2 pos_screen = vary_fragcoord.xy;
     vec4 pos  = getPosition(pos_screen);
-    vec3 norm = getNorm(pos_screen);
+    vec4 norm = getNorm(pos_screen);
 
     vec4 col;
-    col.r = sampleDirectionalShadow(pos.xyz, norm, pos_screen);
-    col.g = calcAmbientOcclusion(pos, norm, pos_screen);
-    col.b = sampleSpotShadow(pos.xyz, norm, 0, pos_screen);
-    col.a = sampleSpotShadow(pos.xyz, norm, 1, pos_screen);
+    col.r = sampleDirectionalShadow(pos.xyz, norm.xyz, pos_screen);
+    col.g = calcAmbientOcclusion(pos, norm.xyz, pos_screen);
+    col.b = sampleSpotShadow(pos.xyz, norm.xyz, 0, pos_screen);
+    col.a = sampleSpotShadow(pos.xyz, norm.xyz, 1, pos_screen);
 
     frag_color = clamp(col, vec4(0), vec4(1));
 }
diff --git a/indra/newview/app_settings/shaders/class3/deferred/hazeF.glsl b/indra/newview/app_settings/shaders/class3/deferred/hazeF.glsl
index 0b154e82ad..4af57e3b80 100644
--- a/indra/newview/app_settings/shaders/class3/deferred/hazeF.glsl
+++ b/indra/newview/app_settings/shaders/class3/deferred/hazeF.glsl
@@ -33,7 +33,7 @@ uniform vec3 moon_dir;
 uniform int  sun_up_factor;
 in vec2 vary_fragcoord;
 
-vec3 getNorm(vec2 pos_screen);
+vec4 getNorm(vec2 pos_screen);
 vec4 getPositionWithDepth(vec2 pos_screen, float depth);
 void calcAtmosphericVarsLinear(vec3 inPositionEye, vec3 norm, vec3 light_dir, out vec3 sunlit, out vec3 amblit, out vec3 atten, out vec3 additive);
 
@@ -53,8 +53,7 @@ void main()
     vec2  tc           = vary_fragcoord.xy;
     float depth        = getDepth(tc.xy);
     vec4  pos          = getPositionWithDepth(tc, depth);
-    vec4  norm         = texture(normalMap, tc);
-    norm.xyz           = getNorm(tc);
+    vec4  norm         = getNorm(tc);
     vec3  light_dir   = (sun_up_factor == 1) ? sun_dir : moon_dir;
 
     vec3  color = vec3(0);
diff --git a/indra/newview/app_settings/shaders/class3/deferred/materialF.glsl b/indra/newview/app_settings/shaders/class3/deferred/materialF.glsl
index d49db4b74d..2a20eb79d2 100644
--- a/indra/newview/app_settings/shaders/class3/deferred/materialF.glsl
+++ b/indra/newview/app_settings/shaders/class3/deferred/materialF.glsl
@@ -415,7 +415,7 @@ void main()
 
     frag_data[0] = max(vec4(diffcol.rgb, emissive), vec4(0));        // gbuffer is sRGB for legacy materials
     frag_data[1] = max(vec4(spec.rgb, glossiness), vec4(0));           // XYZ = Specular color. W = Specular exponent.
-    frag_data[2] = max(vec4(norm, flag), vec4(0));   // XY = Normal.  Z = Env. intensity. W = 1 skip atmos (mask off fog)
+    frag_data[2] = vec4(norm, flag);   // XY = Normal.  Z = Env. intensity. W = 1 skip atmos (mask off fog)
     frag_data[3] = vec4(env, 0, 0, 0);
 
 #endif
diff --git a/indra/newview/app_settings/shaders/class3/deferred/multiPointLightF.glsl b/indra/newview/app_settings/shaders/class3/deferred/multiPointLightF.glsl
index ec8168465e..edfd6cbced 100644
--- a/indra/newview/app_settings/shaders/class3/deferred/multiPointLightF.glsl
+++ b/indra/newview/app_settings/shaders/class3/deferred/multiPointLightF.glsl
@@ -48,7 +48,7 @@ in vec4 vary_fragcoord;
 void calcHalfVectors(vec3 lv, vec3 n, vec3 v, out vec3 h, out vec3 l, out float nh, out float nl, out float nv, out float vh, out float lightDist);
 float calcLegacyDistanceAttenuation(float distance, float falloff);
 vec4 getPosition(vec2 pos_screen);
-vec4 getNormalEnvIntensityFlags(vec2 screenpos, out vec3 n, out float envIntensity);
+vec4 getNorm(vec2 screenpos);
 vec2 getScreenXY(vec4 clip);
 vec2 getScreenCoord(vec4 clip);
 vec3 srgb_to_linear(vec3 c);
@@ -74,9 +74,8 @@ void main()
         discard;
     }
 
-    float envIntensity; // not used for this shader
-    vec3 n;
-    vec4 norm = getNormalEnvIntensityFlags(tc, n, envIntensity); // need `norm.w` for GET_GBUFFER_FLAG()
+    vec4 norm = getNorm(tc); // need `norm.w` for GET_GBUFFER_FLAG()
+    vec3 n = norm.xyz;
 
     vec4 spec    = texture(specularRect, tc);
     vec3 diffuse = texture(diffuseRect, tc).rgb;
diff --git a/indra/newview/app_settings/shaders/class3/deferred/pointLightF.glsl b/indra/newview/app_settings/shaders/class3/deferred/pointLightF.glsl
index 31af1208bd..60be9f4407 100644
--- a/indra/newview/app_settings/shaders/class3/deferred/pointLightF.glsl
+++ b/indra/newview/app_settings/shaders/class3/deferred/pointLightF.glsl
@@ -52,7 +52,7 @@ uniform vec4 viewport;
 
 void calcHalfVectors(vec3 lv, vec3 n, vec3 v, out vec3 h, out vec3 l, out float nh, out float nl, out float nv, out float vh, out float lightDist);
 float calcLegacyDistanceAttenuation(float distance, float falloff);
-vec4 getNormalEnvIntensityFlags(vec2 screenpos, out vec3 n, out float envIntensity);
+vec4 getNorm(vec2 screenpos);
 vec4 getPosition(vec2 pos_screen);
 vec2 getScreenXY(vec4 clip);
 vec2 getScreenCoord(vec4 clip);
@@ -72,9 +72,8 @@ void main()
     vec2 tc          = getScreenCoord(vary_fragcoord);
     vec3 pos         = getPosition(tc).xyz;
 
-    float envIntensity;
-    vec3 n;
-    vec4 norm = getNormalEnvIntensityFlags(tc, n, envIntensity); // need `norm.w` for GET_GBUFFER_FLAG()
+    vec4 norm = getNorm(tc); // need `norm.w` for GET_GBUFFER_FLAG()
+    vec3 n = norm.xyz;
 
     vec3 diffuse = texture(diffuseRect, tc).rgb;
     vec4 spec    = texture(specularRect, tc);
diff --git a/indra/newview/app_settings/shaders/class3/deferred/screenSpaceReflPostF.glsl b/indra/newview/app_settings/shaders/class3/deferred/screenSpaceReflPostF.glsl
index dbf20fe2d8..deb276ef9d 100644
--- a/indra/newview/app_settings/shaders/class3/deferred/screenSpaceReflPostF.glsl
+++ b/indra/newview/app_settings/shaders/class3/deferred/screenSpaceReflPostF.glsl
@@ -40,14 +40,13 @@ uniform sampler2D specularRect;
 uniform sampler2D diffuseRect;
 uniform sampler2D diffuseMap;
 
-vec3 getNorm(vec2 screenpos);
+vec4 getNorm(vec2 screenpos);
 float getDepth(vec2 pos_screen);
 float linearDepth(float d, float znear, float zfar);
 float linearDepth01(float d, float znear, float zfar);
 
 vec4 getPositionWithDepth(vec2 pos_screen, float depth);
 vec4 getPosition(vec2 pos_screen);
-vec4 getNormalEnvIntensityFlags(vec2 screenpos, out vec3 n, out float envIntensity);
 
 float random (vec2 uv);
 
@@ -57,9 +56,7 @@ void main()
 {
     vec2  tc = vary_fragcoord.xy;
     float depth = linearDepth01(getDepth(tc), zNear, zFar);
-    float envIntensity;
-    vec3 n;
-    vec4 norm = getNormalEnvIntensityFlags(tc, n, envIntensity); // need `norm.w` for GET_GBUFFER_FLAG()
+    vec4 norm = getNorm(tc); // need `norm.w` for GET_GBUFFER_FLAG()
     vec3 pos = getPositionWithDepth(tc, getDepth(tc)).xyz;
     vec4 spec    = texture(specularRect, tc);
     vec2 hitpixel;
@@ -84,7 +81,7 @@ void main()
 
     vec4 collectedColor = vec4(0);
 
-    float w = tapScreenSpaceReflection(4, tc, pos, n, collectedColor, diffuseMap, 0);
+    float w = tapScreenSpaceReflection(4, tc, pos, norm.xyz, collectedColor, diffuseMap, 0);
 
     collectedColor.rgb *= specCol.rgb;
 
diff --git a/indra/newview/app_settings/shaders/class3/deferred/softenLightF.glsl b/indra/newview/app_settings/shaders/class3/deferred/softenLightF.glsl
index f71f8a028f..96c32734e4 100644
--- a/indra/newview/app_settings/shaders/class3/deferred/softenLightF.glsl
+++ b/indra/newview/app_settings/shaders/class3/deferred/softenLightF.glsl
@@ -61,7 +61,7 @@ in vec2 vary_fragcoord;
 uniform mat4 inv_proj;
 uniform vec2 screen_res;
 
-vec3 getNorm(vec2 pos_screen);
+vec4 getNorm(vec2 pos_screen);
 vec4 getPositionWithDepth(vec2 pos_screen, float depth);
 
 void calcAtmosphericVarsLinear(vec3 inPositionEye, vec3 norm, vec3 light_dir, out vec3 sunlit, out vec3 amblit, out vec3 atten, out vec3 additive);
@@ -128,14 +128,13 @@ void main()
     vec2  tc           = vary_fragcoord.xy;
     float depth        = getDepth(tc.xy);
     vec4  pos          = getPositionWithDepth(tc, depth);
-    vec4  norm         = texture(normalMap, tc);
+    vec4  norm         = getNorm(tc);
     vec3 colorEmissive = texture(emissiveRect, tc).rgb;
     float envIntensity = colorEmissive.r;
-    norm.xyz           = getNorm(tc);
     vec3  light_dir   = (sun_up_factor == 1) ? sun_dir : moon_dir;
 
     vec4 baseColor     = texture(diffuseRect, tc);
-    vec4 spec        = texture(specularRect, vary_fragcoord.xy); // NOTE: PBR linear Emissive
+    vec4 spec        = texture(specularRect, tc); // NOTE: PBR linear Emissive
 
 #if defined(HAS_SUN_SHADOW) || defined(HAS_SSAO)
     vec2 scol_ambocc = texture(lightMap, vary_fragcoord.xy).rg;
@@ -170,7 +169,7 @@ void main()
 
     if (GET_GBUFFER_FLAG(GBUFFER_FLAG_HAS_PBR))
     {
-        vec3 orm = texture(specularRect, tc).rgb; 
+        vec3 orm = spec.rgb; 
         float perceptualRoughness = orm.g;
         float metallic = orm.b;
         float ao = orm.r;
@@ -193,12 +192,12 @@ void main()
     else if (GET_GBUFFER_FLAG(GBUFFER_FLAG_HAS_HDRI))
     {
         // actual HDRI sky, just copy color value
-        color = texture(emissiveRect, tc).rgb;
+        color = colorEmissive.rgb;
     }
     else if (GET_GBUFFER_FLAG(GBUFFER_FLAG_SKIP_ATMOS))
     {
         //should only be true of WL sky, port over base color value and scale for fake HDR
-        color = texture(emissiveRect, tc).rgb;
+        color = colorEmissive.rgb;
         color = srgb_to_linear(color);
         color *= sky_hdr_scale;
     }
diff --git a/indra/newview/app_settings/shaders/class3/deferred/spotLightF.glsl b/indra/newview/app_settings/shaders/class3/deferred/spotLightF.glsl
index d31b37fb60..319fa86148 100644
--- a/indra/newview/app_settings/shaders/class3/deferred/spotLightF.glsl
+++ b/indra/newview/app_settings/shaders/class3/deferred/spotLightF.glsl
@@ -72,7 +72,7 @@ uniform mat4 inv_proj;
 void calcHalfVectors(vec3 lv, vec3 n, vec3 v, out vec3 h, out vec3 l, out float nh, out float nl, out float nv, out float vh, out float lightDist);
 float calcLegacyDistanceAttenuation(float distance, float falloff);
 bool clipProjectedLightVars(vec3 center, vec3 pos, out float dist, out float l_dist, out vec3 lv, out vec4 proj_tc );
-vec4 getNormalEnvIntensityFlags(vec2 screenpos, out vec3 n, out float envIntensity);
+vec4 getNorm(vec2 screenpos);
 vec3 getProjectedLightAmbiance(float amb_da, float attenuation, float lit, float nl, float noise, vec2 projected_uv);
 vec3 getProjectedLightDiffuseColor(float light_distance, vec2 projected_uv );
 vec2 getScreenCoord(vec4 clip);
@@ -121,9 +121,8 @@ void main()
         shadow = clamp(shadow, 0.0, 1.0);        
     }
 
-    float envIntensity;
-    vec3 n;
-    vec4 norm = getNormalEnvIntensityFlags(tc, n, envIntensity);
+    vec4 norm = getNorm(tc);
+    vec3 n = norm.xyz;
 
     float dist_atten = calcLegacyDistanceAttenuation(dist, falloff);
     if (dist_atten <= 0.0)
@@ -145,7 +144,6 @@ void main()
 
     if (GET_GBUFFER_FLAG(GBUFFER_FLAG_HAS_PBR))
     {
-        vec3 colorEmissive = texture(emissiveRect, tc).rgb; 
         vec3 orm = spec.rgb;
         float perceptualRoughness = orm.g;
         float metallic = orm.b;
@@ -182,6 +180,8 @@ void main()
     }
     else
     {
+        float envIntensity = texture(emissiveRect, tc).r;
+
         diffuse = srgb_to_linear(diffuse);
         spec.rgb = srgb_to_linear(spec.rgb);
 
diff --git a/indra/newview/app_settings/shaders/class3/deferred/waterHazeF.glsl b/indra/newview/app_settings/shaders/class3/deferred/waterHazeF.glsl
index f6b8299f91..f6bef1e498 100644
--- a/indra/newview/app_settings/shaders/class3/deferred/waterHazeF.glsl
+++ b/indra/newview/app_settings/shaders/class3/deferred/waterHazeF.glsl
@@ -56,7 +56,6 @@ void main()
     }
 
     vec4  pos          = getPositionWithDepth(tc, depth);
-    vec4  norm         = texture(normalMap, tc);
 
     vec4 fogged = getWaterFogView(pos.xyz);