#1139 Fix for world going black/white in some scenes (NaNs) (#1273)

* #1139 Fix for world going black/white in some scenes (NaNs)

NaNs were coming from normal encode/decode.  Take advantage of available gbuffer space to send full normal instead of encoding.

* #1139 Cleanup

remove now unused encodeNormF.glsl

* #1139 Remove final reference to encodeNormF

* #1139 Fix for getNormalEnvIntensityFlags referencing wrong sampler for env intensity

* Mac build fix

14 files changed, 23 insertions, 81 deletions

diff --git a/indra/newview/app_settings/shaders/class1/deferred/avatarF.glsl b/indra/newview/app_settings/shaders/class1/deferred/avatarF.glsl
index c0607d0149..db7597dd74 100644
--- a/indra/newview/app_settings/shaders/class1/deferred/avatarF.glsl
+++ b/indra/newview/app_settings/shaders/class1/deferred/avatarF.glsl
@@ -25,7 +25,7 @@
 
 /*[EXTRA_CODE_HERE]*/
 
-out vec4 frag_data[3];
+out vec4 frag_data[4];
 
 uniform sampler2D diffuseMap;
 
@@ -35,7 +35,6 @@ in vec3 vary_normal;
 in vec2 vary_texcoord0;
 in vec3 vary_position;
 
-vec2 encode_normal(vec3 n);
 void mirrorClip(vec3 pos);
 
 void main() 
@@ -52,6 +51,7 @@ void main()
 	frag_data[0] = vec4(diff.rgb, 0.0);
 	frag_data[1] = vec4(0,0,0,0);
 	vec3 nvn = normalize(vary_normal);
-	frag_data[2] = vec4(encode_normal(nvn.xyz), 0.0, GBUFFER_FLAG_HAS_ATMOS);
+	frag_data[2] = vec4(nvn.xyz, GBUFFER_FLAG_HAS_ATMOS);
+    frag_data[3] = vec4(0);
 }
 
diff --git a/indra/newview/app_settings/shaders/class1/deferred/bumpF.glsl b/indra/newview/app_settings/shaders/class1/deferred/bumpF.glsl
index a22c174349..8483f257fa 100644
--- a/indra/newview/app_settings/shaders/class1/deferred/bumpF.glsl
+++ b/indra/newview/app_settings/shaders/class1/deferred/bumpF.glsl
@@ -39,8 +39,6 @@ in vec4 vertex_color;
 in vec2 vary_texcoord0;
 in vec3 vary_position;
 
-vec2 encode_normal(vec3 n);
-
 void mirrorClip(vec3 pos);
 void main() 
 {
@@ -64,6 +62,6 @@ void main()
 		frag_data[1] = vertex_color.aaaa; // spec
 		//frag_data[1] = vec4(vec3(vertex_color.a), vertex_color.a+(1.0-vertex_color.a)*vertex_color.a); // spec - from former class3 - maybe better, but not so well tested
 		vec3 nvn = normalize(tnorm);
-		frag_data[2] = vec4(encode_normal(nvn), vertex_color.a, GBUFFER_FLAG_HAS_ATMOS);
-        frag_data[3] = vec4(0);
+		frag_data[2] = vec4(nvn, GBUFFER_FLAG_HAS_ATMOS);
+        frag_data[3] = vec4(vertex_color.a, 0, 0, 0);
 }
diff --git a/indra/newview/app_settings/shaders/class1/deferred/deferredUtil.glsl b/indra/newview/app_settings/shaders/class1/deferred/deferredUtil.glsl
index f9ebf33b4a..065fcec45a 100644
--- a/indra/newview/app_settings/shaders/class1/deferred/deferredUtil.glsl
+++ b/indra/newview/app_settings/shaders/class1/deferred/deferredUtil.glsl
@@ -50,6 +50,7 @@ SOFTWARE.
 
 uniform sampler2D   normalMap;
 uniform sampler2D   depthMap;
+uniform sampler2D emissiveRect;
 uniform sampler2D projectionMap; // rgba
 uniform sampler2D brdfLut;
 
@@ -140,18 +141,9 @@ vec2 getScreenCoordinate(vec2 screenpos)
     return sc - vec2(1.0, 1.0);
 }
 
-// See: https://aras-p.info/texts/CompactNormalStorage.html
-//      Method #4: Spheremap Transform, Lambert Azimuthal Equal-Area projection
 vec3 getNorm(vec2 screenpos)
 {
-   vec2 enc = texture(normalMap, screenpos.xy).xy;
-   vec2 fenc = enc*4-2;
-   float f = dot(fenc,fenc);
-   float g = sqrt(1-f/4);
-   vec3 n;
-   n.xy = fenc*g;
-   n.z = 1-f/2;
-   return n;
+    return texture(normalMap, screenpos.xy).rgb;
 }
 
 vec3 getNormalFromPacked(vec4 packedNormalEnvIntensityFlags)
@@ -170,10 +162,10 @@ vec3 getNormalFromPacked(vec4 packedNormalEnvIntensityFlags)
 // See: C++: addDeferredAttachments(), GLSL: softenLightF
 vec4 getNormalEnvIntensityFlags(vec2 screenpos, out vec3 n, out float envIntensity)
 {
-    vec4 packedNormalEnvIntensityFlags = texture(normalMap, screenpos.xy);
-    n = getNormalFromPacked( packedNormalEnvIntensityFlags );
-    envIntensity = packedNormalEnvIntensityFlags.z;
-    return packedNormalEnvIntensityFlags;
+    n = texture(normalMap, screenpos.xy).rgb;
+    envIntensity = texture(emissiveRect, screenpos.xy).r;
+
+    return vec4(n, envIntensity);
 }
 
 // get linear depth value given a depth buffer sample d and znear and zfar values
diff --git a/indra/newview/app_settings/shaders/class1/deferred/diffuseAlphaMaskF.glsl b/indra/newview/app_settings/shaders/class1/deferred/diffuseAlphaMaskF.glsl
index e8ead91504..82d5d363d2 100644
--- a/indra/newview/app_settings/shaders/class1/deferred/diffuseAlphaMaskF.glsl
+++ b/indra/newview/app_settings/shaders/class1/deferred/diffuseAlphaMaskF.glsl
@@ -37,8 +37,6 @@ in vec3 vary_normal;
 in vec4 vertex_color;
 in vec2 vary_texcoord0;
 
-vec2 encode_normal(vec3 n);
-
 void mirrorClip(vec3 pos);
 
 void main() 
@@ -55,7 +53,7 @@ void main()
 	frag_data[0] = vec4(col.rgb, 0.0);
 	frag_data[1] = vec4(0,0,0,0); // spec
 	vec3 nvn = normalize(vary_normal);
-	frag_data[2] = vec4(encode_normal(nvn.xyz), 0.0, GBUFFER_FLAG_HAS_ATMOS);
+	frag_data[2] = vec4(nvn.xyz, GBUFFER_FLAG_HAS_ATMOS);
     frag_data[3] = vec4(0);
 }
 
diff --git a/indra/newview/app_settings/shaders/class1/deferred/diffuseAlphaMaskIndexedF.glsl b/indra/newview/app_settings/shaders/class1/deferred/diffuseAlphaMaskIndexedF.glsl
index a07c892d8e..788ea633fc 100644
--- a/indra/newview/app_settings/shaders/class1/deferred/diffuseAlphaMaskIndexedF.glsl
+++ b/indra/newview/app_settings/shaders/class1/deferred/diffuseAlphaMaskIndexedF.glsl
@@ -35,8 +35,6 @@ uniform float minimum_alpha;
 in vec4 vertex_color;
 in vec2 vary_texcoord0;
 
-vec2 encode_normal(vec3 n);
-
 void mirrorClip(vec3 pos);
 
 void main() 
@@ -53,6 +51,6 @@ void main()
 	frag_data[0] = vec4(col.rgb, 0.0);
 	frag_data[1] = vec4(0,0,0,0);
 	vec3 nvn = normalize(vary_normal);
-	frag_data[2] = vec4(encode_normal(nvn.xyz), 0.0, GBUFFER_FLAG_HAS_ATMOS);
+	frag_data[2] = vec4(nvn.xyz, GBUFFER_FLAG_HAS_ATMOS);
     frag_data[3] = vec4(0);
 }
diff --git a/indra/newview/app_settings/shaders/class1/deferred/diffuseAlphaMaskNoColorF.glsl b/indra/newview/app_settings/shaders/class1/deferred/diffuseAlphaMaskNoColorF.glsl
index 234f096ed5..22e6d60419 100644
--- a/indra/newview/app_settings/shaders/class1/deferred/diffuseAlphaMaskNoColorF.glsl
+++ b/indra/newview/app_settings/shaders/class1/deferred/diffuseAlphaMaskNoColorF.glsl
@@ -34,8 +34,6 @@ uniform sampler2D diffuseMap;
 in vec3 vary_normal;
 in vec2 vary_texcoord0;
 
-vec2 encode_normal(vec3 n);
-
 void main() 
 {
 	vec4 col = texture(diffuseMap, vary_texcoord0.xy);
@@ -48,7 +46,7 @@ void main()
 	frag_data[0] = vec4(col.rgb, 0.0);
 	frag_data[1] = vec4(0,0,0,0); // spec
 	vec3 nvn = normalize(vary_normal);
-	frag_data[2] = vec4(encode_normal(nvn.xyz), 0.0, GBUFFER_FLAG_HAS_ATMOS);
+	frag_data[2] = vec4(nvn.xyz, GBUFFER_FLAG_HAS_ATMOS);
     frag_data[3] = vec4(0);
 }
 
diff --git a/indra/newview/app_settings/shaders/class1/deferred/diffuseF.glsl b/indra/newview/app_settings/shaders/class1/deferred/diffuseF.glsl
index 76776ede2c..799fc62a42 100644
--- a/indra/newview/app_settings/shaders/class1/deferred/diffuseF.glsl
+++ b/indra/newview/app_settings/shaders/class1/deferred/diffuseF.glsl
@@ -34,7 +34,6 @@ in vec4 vertex_color;
 in vec2 vary_texcoord0;
 in vec3 vary_position;
 
-vec2 encode_normal(vec3 n);
 void mirrorClip(vec3 pos);
 
 void main() 
@@ -45,7 +44,7 @@ void main()
 	frag_data[1] = vertex_color.aaaa; // spec
 	//frag_data[1] = vec4(vec3(vertex_color.a), vertex_color.a+(1.0-vertex_color.a)*vertex_color.a); // spec - from former class3 - maybe better, but not so well tested
 	vec3 nvn = normalize(vary_normal);
-	frag_data[2] = vec4(encode_normal(nvn.xyz), vertex_color.a, GBUFFER_FLAG_HAS_ATMOS);
-    frag_data[3] = vec4(0);
+	frag_data[2] = vec4(nvn.xyz, GBUFFER_FLAG_HAS_ATMOS);
+    frag_data[3] = vec4(vertex_color.a, 0, 0, 0);
 }
 
diff --git a/indra/newview/app_settings/shaders/class1/deferred/diffuseIndexedF.glsl b/indra/newview/app_settings/shaders/class1/deferred/diffuseIndexedF.glsl
index b983acf657..3362a180c6 100644
--- a/indra/newview/app_settings/shaders/class1/deferred/diffuseIndexedF.glsl
+++ b/indra/newview/app_settings/shaders/class1/deferred/diffuseIndexedF.glsl
@@ -33,7 +33,6 @@ in vec2 vary_texcoord0;
 in vec3 vary_position;
 
 void mirrorClip(vec3 pos);
-vec2 encode_normal(vec3 n);
 vec3 linear_to_srgb(vec3 c);
 
 void main() 
@@ -47,6 +46,6 @@ void main()
 	frag_data[0] = vec4(col, 0.0);
 	frag_data[1] = vec4(spec, vertex_color.a); // spec
 	vec3 nvn = normalize(vary_normal);
-	frag_data[2] = vec4(encode_normal(nvn.xyz), vertex_color.a, GBUFFER_FLAG_HAS_ATMOS);
-    frag_data[3] = vec4(0);
+	frag_data[2] = vec4(nvn.xyz, GBUFFER_FLAG_HAS_ATMOS);
+    frag_data[3] = vec4(vertex_color.a, 0, 0, 0);
 }
diff --git a/indra/newview/app_settings/shaders/class1/deferred/impostorF.glsl b/indra/newview/app_settings/shaders/class1/deferred/impostorF.glsl
index aae595f619..c64f65e32c 100644
--- a/indra/newview/app_settings/shaders/class1/deferred/impostorF.glsl
+++ b/indra/newview/app_settings/shaders/class1/deferred/impostorF.glsl
@@ -37,7 +37,6 @@ uniform sampler2D specularMap;
 in vec2 vary_texcoord0;
 
 vec3 linear_to_srgb(vec3 c);
-vec2 encode_normal (vec3 n);
 
 void main() 
 {
@@ -53,6 +52,6 @@ void main()
 
 	frag_data[0] = vec4(col.rgb, 0.0);
 	frag_data[1] = spec;
-	frag_data[2] = vec4(encode_normal(norm.xyz),0,GBUFFER_FLAG_HAS_ATMOS);
+	frag_data[2] = vec4(norm.xyz, GBUFFER_FLAG_HAS_ATMOS);
     frag_data[3] = vec4(0);
 }
diff --git a/indra/newview/app_settings/shaders/class1/deferred/pbropaqueF.glsl b/indra/newview/app_settings/shaders/class1/deferred/pbropaqueF.glsl
index 0683236460..55fc94b7f5 100644
--- a/indra/newview/app_settings/shaders/class1/deferred/pbropaqueF.glsl
+++ b/indra/newview/app_settings/shaders/class1/deferred/pbropaqueF.glsl
@@ -54,7 +54,6 @@ in vec2 emissive_texcoord;
 
 uniform float minimum_alpha; // PBR alphaMode: MASK, See: mAlphaCutoff, setAlphaCutoff()
 
-vec2 encode_normal(vec3 n);
 vec3 linear_to_srgb(vec3 c);
 vec3 srgb_to_linear(vec3 c);
 
@@ -110,7 +109,7 @@ void main()
     // See: C++: addDeferredAttachments(), GLSL: softenLightF
     frag_data[0] = max(vec4(col, 0.0), vec4(0));                                                   // Diffuse
     frag_data[1] = max(vec4(spec.rgb,vertex_color.a), vec4(0));                                    // PBR linear packed Occlusion, Roughness, Metal.
-    frag_data[2] = max(vec4(encode_normal(tnorm), vertex_color.a, GBUFFER_FLAG_HAS_PBR), vec4(0)); // normal, environment intensity, flags
+    frag_data[2] = max(vec4(tnorm, GBUFFER_FLAG_HAS_PBR), vec4(0)); // normal, environment intensity, flags
     frag_data[3] = max(vec4(emissive,0), vec4(0));                                                // PBR sRGB Emissive
 }
 
diff --git a/indra/newview/app_settings/shaders/class1/deferred/pbrterrainF.glsl b/indra/newview/app_settings/shaders/class1/deferred/pbrterrainF.glsl
index 57c0a6024f..025ecb64df 100644
--- a/indra/newview/app_settings/shaders/class1/deferred/pbrterrainF.glsl
+++ b/indra/newview/app_settings/shaders/class1/deferred/pbrterrainF.glsl
@@ -140,7 +140,6 @@ flat in float vary_sign;
 in vec4 vary_texcoord0;
 in vec4 vary_texcoord1;
 
-vec2 encode_normal(vec3 n);
 void mirrorClip(vec3 position);
 
 float terrain_mix(TerrainMix tm, vec4 tms4);
@@ -342,7 +341,7 @@ void main()
 #endif
     frag_data[0] = max(vec4(mix.col.xyz, 0.0), vec4(0));                                                   // Diffuse
     frag_data[1] = max(vec4(orm.rgb, base_color_factor_alpha), vec4(0));                                    // PBR linear packed Occlusion, Roughness, Metal.
-    frag_data[2] = max(vec4(encode_normal(tnorm), base_color_factor_alpha, GBUFFER_FLAG_HAS_PBR), vec4(0)); // normal, environment intensity, flags
+    frag_data[2] = max(vec4(tnorm, GBUFFER_FLAG_HAS_PBR), vec4(0)); // normal, flags
     frag_data[3] = max(vec4(emissive,0), vec4(0));                                                // PBR sRGB Emissive
 }
 
diff --git a/indra/newview/app_settings/shaders/class1/deferred/terrainF.glsl b/indra/newview/app_settings/shaders/class1/deferred/terrainF.glsl
index 2dba7cb9d9..5f598f84a7 100644
--- a/indra/newview/app_settings/shaders/class1/deferred/terrainF.glsl
+++ b/indra/newview/app_settings/shaders/class1/deferred/terrainF.glsl
@@ -38,7 +38,6 @@ in vec3 vary_normal;
 in vec4 vary_texcoord0;
 in vec4 vary_texcoord1;
 
-vec2 encode_normal(vec3 n);
 void mirrorClip(vec3 position);
 
 void main()
@@ -61,7 +60,7 @@ void main()
     frag_data[0] = outColor;
     frag_data[1] = vec4(0.0,0.0,0.0,-1.0);
     vec3 nvn = normalize(vary_normal);
-    frag_data[2] = vec4(encode_normal(nvn.xyz), 0.0, GBUFFER_FLAG_HAS_ATMOS);
+    frag_data[2] = vec4(nvn.xyz, GBUFFER_FLAG_HAS_ATMOS);
     frag_data[3] = vec4(0);
 }
 
diff --git a/indra/newview/app_settings/shaders/class1/deferred/treeF.glsl b/indra/newview/app_settings/shaders/class1/deferred/treeF.glsl
index f108faf283..708acd0194 100644
--- a/indra/newview/app_settings/shaders/class1/deferred/treeF.glsl
+++ b/indra/newview/app_settings/shaders/class1/deferred/treeF.glsl
@@ -36,8 +36,6 @@ in vec3 vary_position;
 
 uniform float minimum_alpha;
 
-vec2 encode_normal(vec3 n);
-
 void mirrorClip(vec3 pos);
 void main() 
 {
@@ -51,6 +49,6 @@ void main()
 	frag_data[0] = vec4(vertex_color.rgb*col.rgb, 0.0);
 	frag_data[1] = vec4(0,0,0,0);
 	vec3 nvn = normalize(vary_normal);
-	frag_data[2] = vec4(encode_normal(nvn.xyz), 0.0, GBUFFER_FLAG_HAS_ATMOS);
+	frag_data[2] = vec4(nvn.xyz, GBUFFER_FLAG_HAS_ATMOS);
     frag_data[3] = vec4(0);
 }
diff --git a/indra/newview/app_settings/shaders/class1/environment/encodeNormF.glsl b/indra/newview/app_settings/shaders/class1/environment/encodeNormF.glsl
deleted file mode 100644
index 6cd2445522..0000000000
--- a/indra/newview/app_settings/shaders/class1/environment/encodeNormF.glsl
+++ /dev/null
@@ -1,34 +0,0 @@
-/**
- * @file encodeNormF.glsl
- *
- * $LicenseInfo:firstyear=2018&license=viewerlgpl$
- * Second Life Viewer Source Code
- * Copyright (C) 2018, Linden Research, Inc.
- * 
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation;
- * version 2.1 of the License only.
- * 
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Lesser General Public License for more details.
- * 
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
- * 
- * Linden Research, Inc., 945 Battery Street, San Francisco, CA  94111  USA
- * $/LicenseInfo$
- */
-
-// Lambert Azimuthal Equal-Area projection
-// See: https://aras-p.info/texts/CompactNormalStorage.html
-// Also see: A_bit_more_deferred_-_CryEngine3.ppt
-vec2 encode_normal(vec3 n)
-{
-	float f = sqrt(8 * n.z + 8);
-	return n.xy / f + 0.5;
-}
-