SL-12592, avoid light modulation of fullbright and preserve diffuse alpha component

1 files changed, 96 insertions, 95 deletions

diff --git a/indra/newview/app_settings/shaders/class1/deferred/materialF.glsl b/indra/newview/app_settings/shaders/class1/deferred/materialF.glsl
index d07de529ca..21a462e2db 100644
--- a/indra/newview/app_settings/shaders/class1/deferred/materialF.glsl
+++ b/indra/newview/app_settings/shaders/class1/deferred/materialF.glsl
@@ -1,28 +1,28 @@
-/** 
- * @file materialF.glsl
- *
- * $LicenseInfo:firstyear=2007&license=viewerlgpl$
- * Second Life Viewer Source Code
- * Copyright (C) 2007, Linden Research, Inc.
- * 
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation;
- * version 2.1 of the License only.
- * 
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Lesser General Public License for more details.
- * 
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
- * 
- * Linden Research, Inc., 945 Battery Street, San Francisco, CA  94111  USA
- * $/LicenseInfo$
- */
- 
+/**
+* @file materialF.glsl
+*
+* $LicenseInfo:firstyear=2007&license=viewerlgpl$
+* Second Life Viewer Source Code
+* Copyright (C) 2007, Linden Research, Inc.
+*
+* This library is free software; you can redistribute it and/or
+* modify it under the terms of the GNU Lesser General Public
+* License as published by the Free Software Foundation;
+* version 2.1 of the License only.
+*
+* This library is distributed in the hope that it will be useful,
+* but WITHOUT ANY WARRANTY; without even the implied warranty of
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+* Lesser General Public License for more details.
+*
+* You should have received a copy of the GNU Lesser General Public
+* License along with this library; if not, write to the Free Software
+* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+*
+* Linden Research, Inc., 945 Battery Street, San Francisco, CA  94111  USA
+* $/LicenseInfo$
+*/
+
 /*[EXTRA_CODE_HERE]*/
 
 //class1/deferred/materialF.glsl
@@ -34,7 +34,7 @@
 #define DIFFUSE_ALPHA_MODE_MASK     2
 #define DIFFUSE_ALPHA_MODE_EMISSIVE 3
 
-uniform float emissive_brightness;
+uniform float emissive_brightness;  // fullbright flag, 1.0 == fullbright, 0.0 otherwise
 uniform int sun_up_factor;
 
 #ifdef WATER_FOG
@@ -52,13 +52,13 @@ vec3 linear_to_srgb(vec3 cs);
 #if (DIFFUSE_ALPHA_MODE == DIFFUSE_ALPHA_MODE_BLEND)
 
 #ifdef DEFINE_GL_FRAGCOLOR
-    out vec4 frag_color;
+out vec4 frag_color;
 #else
-    #define frag_color gl_FragColor
+#define frag_color gl_FragColor
 #endif
 
 #ifdef HAS_SUN_SHADOW
-    float sampleDirectionalShadow(vec3 pos, vec3 norm, vec2 pos_screen);
+float sampleDirectionalShadow(vec3 pos, vec3 norm, vec2 pos_screen);
 #endif
 
 uniform samplerCube environmentMap;
@@ -81,7 +81,7 @@ uniform vec2 screen_res;
 
 uniform vec4 light_position[8];
 uniform vec3 light_direction[8];
-uniform vec4 light_attenuation[8]; 
+uniform vec4 light_attenuation[8];
 uniform vec3 light_diffuse[8];
 
 float getAmbientClamp();
@@ -91,7 +91,7 @@ vec3 calcPointLightOrSpotLight(vec3 light_col, vec3 npos, vec3 diffuse, vec4 spe
     vec3 col = vec3(0);
 
     //get light vector
-    vec3 lv = lp.xyz-v;
+    vec3 lv = lp.xyz - v;
 
     //get distance
     float dist = length(lv);
@@ -103,21 +103,21 @@ vec3 calcPointLightOrSpotLight(vec3 light_col, vec3 npos, vec3 diffuse, vec4 spe
     {
         //normalize light vector
         lv = normalize(lv);
-    
+
         //distance attenuation
-        float dist_atten = clamp(1.0-(dist-1.0*(1.0-fa))/fa, 0.0, 1.0);
+        float dist_atten = clamp(1.0 - (dist - 1.0*(1.0 - fa)) / fa, 0.0, 1.0);
         dist_atten *= dist_atten;
         dist_atten *= 2.0f;
 
         if (dist_atten <= 0.0)
         {
-           return col;
+            return col;
         }
 
         // spotlight coefficient.
         float spot = max(dot(-ln, lv), is_pointlight);
         da *= spot*spot; // GL_SPOT_EXPONENT=2
-    
+
         //angular attenuation
         da *= dot(n, lv);
 
@@ -126,9 +126,9 @@ vec3 calcPointLightOrSpotLight(vec3 light_col, vec3 npos, vec3 diffuse, vec4 spe
         float amb_da = ambiance;
         if (da >= 0)
         {
-            lit = max(da * dist_atten,0.0);
+            lit = max(da * dist_atten, 0.0);
             col = lit * light_col * diffuse;
-            amb_da += (da*0.5+0.5) * ambiance;
+            amb_da += (da*0.5 + 0.5) * ambiance;
         }
         amb_da += (da*da*0.5 + 0.5) * ambiance;
         amb_da *= dist_atten;
@@ -140,19 +140,19 @@ vec3 calcPointLightOrSpotLight(vec3 light_col, vec3 npos, vec3 diffuse, vec4 spe
         if (spec.a > 0.0)
         {
             //vec3 ref = dot(pos+lv, norm);
-            vec3 h = normalize(lv+npos);
+            vec3 h = normalize(lv + npos);
             float nh = dot(n, h);
             float nv = dot(n, npos);
             float vh = dot(npos, h);
             float sa = nh;
-            float fres = pow(1 - dot(h, npos), 5)*0.4+0.5;
+            float fres = pow(1 - dot(h, npos), 5)*0.4 + 0.5;
 
             float gtdenom = 2 * nh;
             float gt = max(0, min(gtdenom * nv / vh, gtdenom * da / vh));
-                                
+
             if (nh > 0.0)
             {
-                float scol = fres*texture2D(lightFunc, vec2(nh, spec.a)).r*gt/(nh*da);
+                float scol = fres*texture2D(lightFunc, vec2(nh, spec.a)).r*gt / (nh*da);
                 vec3 speccol = lit*scol*light_col.rgb*spec.rgb;
                 speccol = clamp(speccol, vec3(0), vec3(1));
                 col += speccol;
@@ -165,7 +165,7 @@ vec3 calcPointLightOrSpotLight(vec3 light_col, vec3 npos, vec3 diffuse, vec4 spe
         }
     }
 
-    return max(col, vec3(0.0,0.0,0.0));
+    return max(col, vec3(0.0, 0.0, 0.0));
 }
 
 #else
@@ -213,16 +213,9 @@ void main()
 {
     vec2 pos_screen = vary_texcoord0.xy;
 
-    vec4 diffuse_tap = texture2D(diffuseMap, vary_texcoord0.xy);
-    diffuse_tap.rgb *= vertex_color.rgb;
-    
-//#if (DIFFUSE_ALPHA_MODE == DIFFUSE_ALPHA_MODE_BLEND)
-    vec4 diffuse_srgb = diffuse_tap;
+    vec4 diffuse_srgb = texture2D(diffuseMap, vary_texcoord0.xy);
+    diffuse_srgb.rgb *= vertex_color.rgb;
     vec4 diffuse_linear = vec4(srgb_to_linear(diffuse_srgb.rgb), diffuse_srgb.a);
-/*#else
-    vec4 diffuse_linear = diffuse_tap;
-    vec4 diffuse_srgb = vec4(linear_to_srgb(diffuse_linear.rgb), diffuse_linear.a);
-#endif*/
 
 #if (DIFFUSE_ALPHA_MODE == DIFFUSE_ALPHA_MODE_MASK)
     if (diffuse_linear.a < minimum_alpha)
@@ -248,15 +241,15 @@ void main()
 
     // convert sampled normal to tangent space normal
     norm = vec3(dot(norm, vary_mat0),
-                dot(norm, vary_mat1),
-                dot(norm, vary_mat2));
+        dot(norm, vary_mat1),
+        dot(norm, vary_mat2));
 #else
     norm = vary_normal;
 #endif
 
     norm = normalize(norm);
 
-    vec2 abnormal   = encode_normal(norm);
+    vec2 abnormal = encode_normal(norm);
 
     vec4 final_color = vec4(diffuse_linear.rgb, 0.0);
 
@@ -286,7 +279,7 @@ void main()
     float al = 0;
 
 #ifdef HAS_SPECULAR_MAP
-    if (emissive_brightness >= 1.0)
+    if (emissive_brightness >= 1.0) // ie, if fullbright
     {
         float ei = env_intensity*0.5 + 0.5;
         final_normal = vec4(abnormal, ei, 0.0);
@@ -304,34 +297,43 @@ void main()
 
 
 #if (DIFFUSE_ALPHA_MODE == DIFFUSE_ALPHA_MODE_BLEND)
-    {
-        //forward rendering, output just lit sRGBA
-        vec3 pos = vary_position;
 
-        float shadow = 1.0f;
+    //forward rendering, output just lit sRGBA
+    vec3 pos = vary_position;
+
+    float shadow = 1.0f;
 
 #ifdef HAS_SUN_SHADOW
-        shadow = sampleDirectionalShadow(pos.xyz, norm, pos_screen);
+    shadow = sampleDirectionalShadow(pos.xyz, norm, pos_screen);
 #endif
 
-        spec = final_specular;
+    spec = final_specular;
 
-        float envIntensity = final_normal.z;
+    float envIntensity = final_normal.z;
 
-        vec3 light_dir = (sun_up_factor == 1) ? sun_dir : moon_dir;
+    vec3 light_dir = (sun_up_factor == 1) ? sun_dir : moon_dir;
 
-        float bloom = 0.0;
-        vec3 sunlit;
-        vec3 amblit;
-        vec3 additive;
-        vec3 atten;
+    float bloom = 0.0;
+    vec3 sunlit;
+    vec3 amblit;
+    vec3 additive;
+    vec3 atten;
 
-        calcAtmosphericVars(pos.xyz, light_dir, 1.0, sunlit, amblit, additive, atten, false);
+    calcAtmosphericVars(pos.xyz, light_dir, 1.0, sunlit, amblit, additive, atten, false);
 
+    if (emissive_brightness >= 1.0)	// fullbright, skip lighting calculations
+    {
+        // just do atmos attenuation (ad hoc 60% factor to match release viewer)
+        color = atmosFragLighting(diffuse_srgb.rgb, additive, atten*0.6);
+        color = scaleSoftClipFrag(color);
+        al = diffuse_srgb.a;
+    }
+    else // not fullbright, calculate lighting
+    {
         vec3 refnormpersp = normalize(reflect(pos.xyz, norm));
 
         float da = clamp(dot(normalize(norm.xyz), light_dir.xyz), 0.0, 1.0);
-        da = pow(da, 1.0/1.3);
+        da = pow(da, 1.0 / 1.3);
 
         float ambient = min(abs(dot(norm.xyz, sun_dir.xyz)), 1.0);
         ambient *= 0.5;
@@ -341,16 +343,15 @@ void main()
         vec3 sun_contrib = min(da, shadow) * sunlit;
 
 #if !defined(AMBIENT_KILL)
-        color.rgb = amblit;
-        color.rgb *= ambient;
+        color = amblit;
+        color *= ambient;
 #endif
 
 #if !defined(SUNLIGHT_KILL)
-        color.rgb += sun_contrib;
+        color += sun_contrib;
 #endif
+        color *= diffuse_srgb.rgb;
 
-        color.rgb *= diffuse_srgb.rgb;
-        
         float glare = 0.0;
 
         if (spec.a > 0.0) // specular reflection
@@ -358,19 +359,19 @@ void main()
             vec3 npos = -normalize(pos.xyz);
 
             //vec3 ref = dot(pos+lv, norm);
-            vec3 h = normalize(light_dir.xyz+npos);
+            vec3 h = normalize(light_dir.xyz + npos);
             float nh = dot(norm, h);
             float nv = dot(norm, npos);
             float vh = dot(npos, h);
             float sa = nh;
-            float fres = pow(1 - dot(h, npos), 5)*0.4+0.5;
+            float fres = pow(1 - dot(h, npos), 5)*0.4 + 0.5;
 
             float gtdenom = 2 * nh;
             float gt = max(0, min(gtdenom * nv / vh, gtdenom * da / vh));
 
             if (nh > 0.0)
             {
-                float scol = fres*texture2D(lightFunc, vec2(nh, spec.a)).r*gt/(nh*da);
+                float scol = fres*texture2D(lightFunc, vec2(nh, spec.a)).r*gt / (nh*da);
                 vec3 sp = sun_contrib*scol / 6.0f;
                 sp = clamp(sp, vec3(0), vec3(1));
                 bloom = dot(sp, sp) / 4.0;
@@ -384,11 +385,11 @@ void main()
         {
             //add environmentmap
             vec3 env_vec = env_mat * refnormpersp;
-            
+
             vec3 reflected_color = textureCube(environmentMap, env_vec).rgb;
 
 #if !defined(SUNLIGHT_KILL)
-            color = mix(color.rgb, reflected_color, envIntensity);
+            color = mix(color, reflected_color, envIntensity);
 #endif
             float cur_glare = max(reflected_color.r, reflected_color.g);
             cur_glare = max(cur_glare, reflected_color.b);
@@ -401,14 +402,14 @@ void main()
 
         vec3 npos = normalize(-pos.xyz);
 
-        vec3 light = vec3(0,0,0);
+        vec3 light = vec3(0, 0, 0);
 
         //convert to linear before adding local lights
-        color.rgb = srgb_to_linear(color.rgb);
+        color = srgb_to_linear(color);
 
 #define LIGHT_LOOP(i) light.rgb += calcPointLightOrSpotLight(light_diffuse[i].rgb, npos, diffuse_linear.rgb, final_specular, pos.xyz, norm, light_position[i], light_direction[i].xyz, light_attenuation[i].x, light_attenuation[i].y, light_attenuation[i].z, glare, light_attenuation[i].w );
 
-            LIGHT_LOOP(1)
+        LIGHT_LOOP(1)
             LIGHT_LOOP(2)
             LIGHT_LOOP(3)
             LIGHT_LOOP(4)
@@ -416,25 +417,25 @@ void main()
             LIGHT_LOOP(6)
             LIGHT_LOOP(7)
 
-        glare = min(glare, 1.0);
-        al = max(diffuse_linear.a,glare)*vertex_color.a;
+            glare = min(glare, 1.0);
+        al = max(diffuse_linear.a, glare)*vertex_color.a;
 
 #if !defined(LOCAL_LIGHT_KILL)
-        color.rgb += light.rgb;
+        color += light;
 #endif
 
-//convert to srgb as this color is being written post gamma correction
-    color.rgb = linear_to_srgb(color.rgb);
-   
+        //convert to srgb as this color is being written post gamma correction
+        color = linear_to_srgb(color);
+    }
+
 #ifdef WATER_FOG
-        vec4 temp = applyWaterFogView(pos, vec4(color.rgb, al));
-        color.rgb = temp.rgb;
-        al = temp.a;
+    vec4 temp = applyWaterFogView(pos, vec4(color, al));
+    color = temp.rgb;
+    al = temp.a;
 #endif
-    }
 
-    
-    frag_color = vec4(color, al);
+    // Don't allow alpha to exceed input value - SL-12592
+    frag_color = vec4(color, min(al, diffuse_srgb.a));
 
 #else // mode is not DIFFUSE_ALPHA_MODE_BLEND, encode to gbuffer