SL-11109

Fix alpha mask shadows to mult vert alpha prior to discard tests and obey min alpha settings.

Make shadow/shadowAlphaMask use consistent varying and output.

Also fixes bug with 0% and 1% transparency providing varying different visual results.

4 files changed, 11 insertions, 25 deletions

diff --git a/indra/newview/app_settings/shaders/class1/deferred/shadowAlphaMaskF.glsl b/indra/newview/app_settings/shaders/class1/deferred/shadowAlphaMaskF.glsl
index dd762958c4..d79d2423f1 100644
--- a/indra/newview/app_settings/shaders/class1/deferred/shadowAlphaMaskF.glsl
+++ b/indra/newview/app_settings/shaders/class1/deferred/shadowAlphaMaskF.glsl
@@ -33,38 +33,34 @@ out vec4 frag_color;
 
 uniform sampler2D diffuseMap;
 
-#if !defined(DEPTH_CLAMP)
-VARYING float pos_zd2;
-#endif
-
-VARYING float pos_w;
-
+VARYING vec4 post_pos;
 VARYING float target_pos_x;
 VARYING vec4 vertex_color;
 VARYING vec2 vary_texcoord0;
+uniform float minimum_alpha;
 
 void main() 
 {
 	float alpha = diffuseLookup(vary_texcoord0.xy).a;
 
+    alpha *= vertex_color.a;
+
 	if (alpha < 0.05) // treat as totally transparent
 	{
 		discard;
 	}
 
-	if (alpha < 0.88) // treat as semi-transparent
+	if (alpha < minimum_alpha) // treat as semi-transparent
 	{
-	  if (fract(0.5*floor(target_pos_x / pos_w )) < 0.25)
+	  if (fract(0.5*floor(target_pos_x / post_pos.w )) < 0.25)
 	  {
 	    discard;
 	  }
 	}
 
-    alpha *= vertex_color.a;
-
 	frag_color = vec4(1,1,1,1);
 	
 #if !defined(DEPTH_CLAMP)
-	gl_FragDepth = max(pos_zd2/pos_w+0.5, 0.0);
+	gl_FragDepth = max(post_pos.z/post_pos.w*0.5+0.5, 0.0);
 #endif
 }
diff --git a/indra/newview/app_settings/shaders/class1/deferred/shadowAlphaMaskV.glsl b/indra/newview/app_settings/shaders/class1/deferred/shadowAlphaMaskV.glsl
index f45c343066..b6a0f0b165 100644
--- a/indra/newview/app_settings/shaders/class1/deferred/shadowAlphaMaskV.glsl
+++ b/indra/newview/app_settings/shaders/class1/deferred/shadowAlphaMaskV.glsl
@@ -31,12 +31,7 @@ ATTRIBUTE vec3 position;
 ATTRIBUTE vec4 diffuse_color;
 ATTRIBUTE vec2 texcoord0;
 
-#if !defined(DEPTH_CLAMP)
-VARYING float pos_zd2;
-#endif
-
-VARYING float pos_w;
-
+VARYING vec4 post_pos;
 VARYING float target_pos_x;
 VARYING vec4 vertex_color;
 VARYING vec2 vary_texcoord0;
@@ -50,11 +45,9 @@ void main()
 	vec4 pos = modelview_projection_matrix * pre_pos;
 	target_pos_x = 0.5 * (shadow_target_width - 1.0) * pos.x;
 
-	pos_w = pos.w;
+	post_pos = pos;
 
 #if !defined(DEPTH_CLAMP)
-	pos_zd2 = pos.z * 0.5;
-	
 	gl_Position = vec4(pos.x, pos.y, pos.w*0.5, pos.w);
 #else
 	gl_Position = pos;
diff --git a/indra/newview/app_settings/shaders/class1/deferred/shadowF.glsl b/indra/newview/app_settings/shaders/class1/deferred/shadowF.glsl
index 22d42d38c1..1ea96918bb 100644
--- a/indra/newview/app_settings/shaders/class1/deferred/shadowF.glsl
+++ b/indra/newview/app_settings/shaders/class1/deferred/shadowF.glsl
@@ -31,9 +31,7 @@ out vec4 frag_color;
 #define frag_color gl_FragColor
 #endif
 
-#if !defined(DEPTH_CLAMP)
 VARYING vec4 post_pos;
-#endif
 
 void main() 
 {
diff --git a/indra/newview/app_settings/shaders/class1/deferred/shadowV.glsl b/indra/newview/app_settings/shaders/class1/deferred/shadowV.glsl
index 41a89fb8b4..72bd0f0f34 100644
--- a/indra/newview/app_settings/shaders/class1/deferred/shadowV.glsl
+++ b/indra/newview/app_settings/shaders/class1/deferred/shadowV.glsl
@@ -27,20 +27,19 @@ uniform mat4 modelview_projection_matrix;
 
 ATTRIBUTE vec3 position;
 
-#if !defined(DEPTH_CLAMP)
 VARYING vec4 post_pos;
-#endif
 
 void main()
 {
 	//transform vertex
 	vec4 pos = modelview_projection_matrix*vec4(position.xyz, 1.0);
 	
-#if !defined(DEPTH_CLAMP)
 	post_pos = pos;
 
+#if !defined(DEPTH_CLAMP)
 	gl_Position = vec4(pos.x, pos.y, pos.w*0.5, pos.w);
 #else
 	gl_Position = pos;
 #endif
+
 }