#1139 Fix for world going black/white in some scenes (NaNs) (#1273)

* #1139 Fix for world going black/white in some scenes (NaNs)

NaNs were coming from normal encode/decode.  Take advantage of available gbuffer space to send full normal instead of encoding.

* #1139 Cleanup

remove now unused encodeNormF.glsl

* #1139 Remove final reference to encodeNormF

* #1139 Fix for getNormalEnvIntensityFlags referencing wrong sampler for env intensity

* Mac build fix

2 files changed, 9 insertions, 10 deletions

diff --git a/indra/newview/app_settings/shaders/class3/deferred/materialF.glsl b/indra/newview/app_settings/shaders/class3/deferred/materialF.glsl
index 0476b98e10..d49db4b74d 100644
--- a/indra/newview/app_settings/shaders/class3/deferred/materialF.glsl
+++ b/indra/newview/app_settings/shaders/class3/deferred/materialF.glsl
@@ -216,8 +216,6 @@ in vec3 vary_normal;
 in vec4 vertex_color;
 in vec2 vary_texcoord0;
 
-vec2 encode_normal(vec3 n);
-
 // get the transformed normal and apply glossiness component from normal map
 vec3 getNormal(inout float glossiness)
 {
@@ -306,8 +304,6 @@ void main()
     float glossiness = specular_color.a;
     vec3 norm = getNormal(glossiness);
 
-    vec2 abnormal = encode_normal(norm.xyz);
-
     float emissive = getEmissive(diffcol);
 
 #if (DIFFUSE_ALPHA_MODE == DIFFUSE_ALPHA_MODE_BLEND)
@@ -417,10 +413,12 @@ void main()
 
     float flag = GBUFFER_FLAG_HAS_ATMOS;
 
-    frag_data[0] = vec4(diffcol.rgb, emissive);        // gbuffer is sRGB for legacy materials
-    frag_data[1] = vec4(spec.rgb, glossiness);           // XYZ = Specular color. W = Specular exponent.
-    frag_data[2] = vec4(encode_normal(norm), env, flag);;   // XY = Normal.  Z = Env. intensity. W = 1 skip atmos (mask off fog)
-    frag_data[3] = vec4(0);
+    frag_data[0] = max(vec4(diffcol.rgb, emissive), vec4(0));        // gbuffer is sRGB for legacy materials
+    frag_data[1] = max(vec4(spec.rgb, glossiness), vec4(0));           // XYZ = Specular color. W = Specular exponent.
+    frag_data[2] = max(vec4(norm, flag), vec4(0));   // XY = Normal.  Z = Env. intensity. W = 1 skip atmos (mask off fog)
+    frag_data[3] = vec4(env, 0, 0, 0);
+
 #endif
 }
 
+
diff --git a/indra/newview/app_settings/shaders/class3/deferred/softenLightF.glsl b/indra/newview/app_settings/shaders/class3/deferred/softenLightF.glsl
index 9b9739ba22..f71f8a028f 100644
--- a/indra/newview/app_settings/shaders/class3/deferred/softenLightF.glsl
+++ b/indra/newview/app_settings/shaders/class3/deferred/softenLightF.glsl
@@ -129,7 +129,8 @@ void main()
     float depth        = getDepth(tc.xy);
     vec4  pos          = getPositionWithDepth(tc, depth);
     vec4  norm         = texture(normalMap, tc);
-    float envIntensity = norm.z;
+    vec3 colorEmissive = texture(emissiveRect, tc).rgb;
+    float envIntensity = colorEmissive.r;
     norm.xyz           = getNorm(tc);
     vec3  light_dir   = (sun_up_factor == 1) ? sun_dir : moon_dir;
 
@@ -174,7 +175,7 @@ void main()
         float metallic = orm.b;
         float ao = orm.r;
 
-        vec3 colorEmissive = texture(emissiveRect, tc).rgb;
+        
         // PBR IBL
         float gloss      = 1.0 - perceptualRoughness;