Back out changes causing broken shadows and other render shenanigans.

6 files changed, 172 insertions, 19 deletions

diff --git a/indra/newview/app_settings/shaders/class2/deferred/multiSpotLightF.glsl b/indra/newview/app_settings/shaders/class2/deferred/multiSpotLightF.glsl
index 0848adba73..864ba4859d 100644
--- a/indra/newview/app_settings/shaders/class2/deferred/multiSpotLightF.glsl
+++ b/indra/newview/app_settings/shaders/class2/deferred/multiSpotLightF.glsl
@@ -133,7 +133,18 @@ vec4 texture2DLodAmbient(sampler2D projectionMap, vec2 tc, float lod)
 }
 
 
-vec4 getPosition(vec2 pos_screen);
+vec4 getPosition(vec2 pos_screen)
+{
+	float depth = texture2DRect(depthMap, pos_screen.xy).r;
+	vec2 sc = pos_screen.xy*2.0;
+	sc /= screen_res;
+	sc -= vec2(1.0,1.0);
+	vec4 ndc = vec4(sc.x, sc.y, 2.0*depth-1.0, 1.0);
+	vec4 pos = inv_proj * ndc;
+	pos /= pos.w;
+	pos.w = 1.0;
+	return pos;
+}
 
 void main() 
 {
diff --git a/indra/newview/app_settings/shaders/class2/deferred/softenLightF.glsl b/indra/newview/app_settings/shaders/class2/deferred/softenLightF.glsl
index 33f3eb46c0..c92dbda185 100644
--- a/indra/newview/app_settings/shaders/class2/deferred/softenLightF.glsl
+++ b/indra/newview/app_settings/shaders/class2/deferred/softenLightF.glsl
@@ -83,12 +83,51 @@ vec3 atmosFragLighting(vec3 l, vec3 additive, vec3 atten);
 vec3 fullbrightScaleSoftClipFrag(vec3 l);
 vec3 scaleSoftClipFrag(vec3 l);
 
-vec3 atmosTransportFrag(vec3 light, vec3 additive, vec3 atten);
-vec3 fullbrightAtmosTransportFrag(vec3 light, vec3 additive, vec3 atten);
-vec3 fullbrightShinyAtmosTransportFrag(vec3 light, vec3 additive, vec3 atten);
+vec3 atmosTransportFrag(vec3 light, vec3 additive, vec3 atten)
+{
+    if (no_atmo == 1)
+    {
+        return light;
+    }
+	return (light + additive) * atten.r * 2.0;
+}
+
+vec3 fullbrightAtmosTransportFrag(vec3 light, vec3 additive, vec3 atten) {
+    if (no_atmo == 1)
+    {
+        return light;
+    }
+	float brightness = dot(light.rgb, vec3(0.33333));
+	return mix(atmosTransportFrag(light.rgb, additive, atten), light.rgb + additive.rgb, brightness * brightness);
+}
+
+vec3 fullbrightShinyAtmosTransportFrag(vec3 light, vec3 additive, vec3 atten) {
+    if (no_atmo == 1)
+    {
+        return light;
+    }
+	float brightness = dot(light.rgb, vec3(0.33333));
+	return mix(atmosTransportFrag(light.rgb, additive, atten), (light.rgb + additive.rgb) * (2.0 - brightness), brightness * brightness);
+}
+
+vec4 getPosition_d(vec2 pos_screen, float depth)
+{
+	vec2 sc = pos_screen.xy*2.0;
+	sc /= screen_res;
+	sc -= vec2(1.0,1.0);
+	vec4 ndc = vec4(sc.x, sc.y, 2.0*depth-1.0, 1.0);
+	vec4 pos = inv_proj * ndc;
+	pos /= pos.w;
+	pos.w = 1.0;
+	return pos;
+}
+
+vec4 getPosition(vec2 pos_screen)
+{ //get position in screen space (world units) given window coordinate and depth map
+	float depth = texture2DRect(depthMap, pos_screen.xy).r;
+	return getPosition_d(pos_screen, depth);
+}
 
-vec4 getPositionWithDepth(vec2 pos_screen, float depth);
-vec4 getPosition(vec2 pos_screen);
 
 #ifdef WATER_FOG
 vec4 applyWaterFogView(vec3 pos, vec4 color);
@@ -98,7 +137,7 @@ void main()
 {
 	vec2 tc = vary_fragcoord.xy;
 	float depth = texture2DRect(depthMap, tc.xy).r;
-	vec4 pos = getPositionWithDepth(tc, depth);
+	vec3 pos = getPosition_d(tc, depth).xyz;
 	vec4 norm = texture2DRect(normalMap, tc);
 	float envIntensity = norm.z;
 	norm.xyz = decode_normal(norm.xy); // unpack norm
diff --git a/indra/newview/app_settings/shaders/class2/deferred/softenLightV.glsl b/indra/newview/app_settings/shaders/class2/deferred/softenLightV.glsl
index 8b8b338f68..c840d72784 100644
--- a/indra/newview/app_settings/shaders/class2/deferred/softenLightV.glsl
+++ b/indra/newview/app_settings/shaders/class2/deferred/softenLightV.glsl
@@ -36,5 +36,7 @@ void main()
 	//transform vertex
 	vec4 pos = modelview_projection_matrix * vec4(position.xyz, 1.0);
 	gl_Position = pos; 
+	
+	
 	vary_fragcoord = (pos.xy*0.5+0.5)*screen_res;
 }
diff --git a/indra/newview/app_settings/shaders/class2/deferred/spotLightF.glsl b/indra/newview/app_settings/shaders/class2/deferred/spotLightF.glsl
index 80e2a9dd55..a7da140b31 100644
--- a/indra/newview/app_settings/shaders/class2/deferred/spotLightF.glsl
+++ b/indra/newview/app_settings/shaders/class2/deferred/spotLightF.glsl
@@ -133,7 +133,18 @@ vec4 texture2DLodAmbient(sampler2D projectionMap, vec2 tc, float lod)
 }
 
 
-vec4 getPosition(vec2 pos_screen);
+vec4 getPosition(vec2 pos_screen)
+{
+	float depth = texture2DRect(depthMap, pos_screen.xy).r;
+	vec2 sc = pos_screen.xy*2.0;
+	sc /= screen_res;
+	sc -= vec2(1.0,1.0);
+	vec4 ndc = vec4(sc.x, sc.y, 2.0*depth-1.0, 1.0);
+	vec4 pos = inv_proj * ndc;
+	pos /= pos.w;
+	pos.w = 1.0;
+	return pos;
+}
 
 void main() 
 {
diff --git a/indra/newview/app_settings/shaders/class2/deferred/sunLightF.glsl b/indra/newview/app_settings/shaders/class2/deferred/sunLightF.glsl
index c4018ca61a..aa5e99a2f7 100644
--- a/indra/newview/app_settings/shaders/class2/deferred/sunLightF.glsl
+++ b/indra/newview/app_settings/shaders/class2/deferred/sunLightF.glsl
@@ -67,9 +67,20 @@ uniform float shadow_offset;
 uniform float spot_shadow_bias;
 uniform float spot_shadow_offset;
 
-vec3 getNorm(vec2 pos_screen);
 vec3 decode_normal (vec2 enc);
-vec4 getPosition(vec2 pos_screen);
+
+vec4 getPosition(vec2 pos_screen)
+{
+	float depth = texture2DRect(depthMap, pos_screen.xy).r;
+	vec2 sc = pos_screen.xy*2.0;
+	sc /= screen_res;
+	sc -= vec2(1.0,1.0);
+	vec4 ndc = vec4(sc.x, sc.y, 2.0*depth-1.0, 1.0);
+	vec4 pos = inv_proj * ndc;
+	pos /= pos.w;
+	pos.w = 1.0;
+	return pos;
+}
 
 float pcfShadow(sampler2DShadow shadowMap, vec4 stc, float scl, vec2 pos_screen)
 {
@@ -112,9 +123,14 @@ float pcfSpotShadow(sampler2DShadow shadowMap, vec4 stc, float scl, vec2 pos_scr
 
 void main() 
 {
-	vec2 pos_screen = vary_fragcoord.xy;	
-	vec4 pos        = getPosition(pos_screen);
-	vec3 norm       = getNorm(pos_screen);
+	vec2 pos_screen = vary_fragcoord.xy;
+	
+	//try doing an unproject here
+	
+	vec4 pos = getPosition(pos_screen);
+	
+	vec3 norm = texture2DRect(normalMap, pos_screen).xyz;
+	norm = decode_normal(norm.xy); // unpack norm
 		
 	/*if (pos.z == 0.0) // do nothing for sky *FIX: REMOVE THIS IF/WHEN THE POSITION MAP IS BEING USED AS A STENCIL
 	{
diff --git a/indra/newview/app_settings/shaders/class2/deferred/sunLightSSAOF.glsl b/indra/newview/app_settings/shaders/class2/deferred/sunLightSSAOF.glsl
index bd468306ee..58f3f2f91e 100644
--- a/indra/newview/app_settings/shaders/class2/deferred/sunLightSSAOF.glsl
+++ b/indra/newview/app_settings/shaders/class2/deferred/sunLightSSAOF.glsl
@@ -70,10 +70,79 @@ uniform float spot_shadow_offset;
 
 vec3 decode_normal (vec2 enc);
 
-vec4 getPosition(vec2 pos_screen);
-vec3 getNorm(vec2 pos_screen);
+vec4 getPosition(vec2 pos_screen)
+{
+	float depth = texture2DRect(depthMap, pos_screen.xy).r;
+	vec2 sc = pos_screen.xy*2.0;
+	sc /= screen_res;
+	sc -= vec2(1.0,1.0);
+	vec4 ndc = vec4(sc.x, sc.y, 2.0*depth-1.0, 1.0);
+	vec4 pos = inv_proj * ndc;
+	pos /= pos.w;
+	pos.w = 1.0;
+	return pos;
+}
+
+vec2 getKern(int i)
+{
+	vec2 kern[8];
+	// exponentially (^2) distant occlusion samples spread around origin
+	kern[0] = vec2(-1.0, 0.0) * 0.125*0.125;
+	kern[1] = vec2(1.0, 0.0) * 0.250*0.250;
+	kern[2] = vec2(0.0, 1.0) * 0.375*0.375;
+	kern[3] = vec2(0.0, -1.0) * 0.500*0.500;
+	kern[4] = vec2(0.7071, 0.7071) * 0.625*0.625;
+	kern[5] = vec2(-0.7071, -0.7071) * 0.750*0.750;
+	kern[6] = vec2(-0.7071, 0.7071) * 0.875*0.875;
+	kern[7] = vec2(0.7071, -0.7071) * 1.000*1.000;
+       
+	return kern[i];
+}
+
+//calculate decreases in ambient lighting when crowded out (SSAO)
+float calcAmbientOcclusion(vec4 pos, vec3 norm)
+{
+	float ret = 1.0;
+
+	vec2 pos_screen = vary_fragcoord.xy;
+	vec3 pos_world = pos.xyz;
+	vec2 noise_reflect = texture2D(noiseMap, vary_fragcoord.xy/128.0).xy;
+		
+	float angle_hidden = 0.0;
+	float points = 0;
+		
+	float scale = min(ssao_radius / -pos_world.z, ssao_max_radius);
+	
+	// it was found that keeping # of samples a constant was the fastest, probably due to compiler optimizations (unrolling?)
+	for (int i = 0; i < 8; i++)
+	{
+		vec2 samppos_screen = pos_screen + scale * reflect(getKern(i), noise_reflect);
+		vec3 samppos_world = getPosition(samppos_screen).xyz; 
+		
+		vec3 diff = pos_world - samppos_world;
+		float dist2 = dot(diff, diff);
+			
+		// assume each sample corresponds to an occluding sphere with constant radius, constant x-sectional area
+		// --> solid angle shrinking by the square of distance
+		//radius is somewhat arbitrary, can approx with just some constant k * 1 / dist^2
+		//(k should vary inversely with # of samples, but this is taken care of later)
+		
+		float funky_val = (dot((samppos_world - 0.05*norm - pos_world), norm) > 0.0) ? 1.0 : 0.0;
+		angle_hidden = angle_hidden + funky_val * min(1.0/dist2, ssao_factor_inv);
+			
+		// 'blocked' samples (significantly closer to camera relative to pos_world) are "no data", not "no occlusion" 
+		float diffz_val = (diff.z > -1.0) ? 1.0 : 0.0;
+		points = points + diffz_val;
+	}
+		
+	angle_hidden = min(ssao_factor*angle_hidden/points, 1.0);
+	
+	float points_val = (points > 0.0) ? 1.0 : 0.0;
+	ret = (1.0 - (points_val * angle_hidden));
 
-float calcAmbientOcclusion(vec4 pos, vec3 norm, vec2 pos_screen);
+	ret = max(ret, 0.0);
+	return min(ret, 1.0);
+}
 
 float pcfShadow(sampler2DShadow shadowMap, vec4 stc, float scl, vec2 pos_screen)
 {
@@ -116,8 +185,13 @@ float pcfSpotShadow(sampler2DShadow shadowMap, vec4 stc, float scl, vec2 pos_scr
 void main() 
 {
 	vec2 pos_screen = vary_fragcoord.xy;
-	vec4 pos  = getPosition(pos_screen);
-	vec3 norm = getNorm(pos_screen);
+	
+	//try doing an unproject here
+	
+	vec4 pos = getPosition(pos_screen);
+	
+	vec3 norm = texture2DRect(normalMap, pos_screen).xyz;
+	norm = decode_normal(norm.xy); // unpack norm
 		
 	/*if (pos.z == 0.0) // do nothing for sky *FIX: REMOVE THIS IF/WHEN THE POSITION MAP IS BEING USED AS A STENCIL
 	{
@@ -217,7 +291,7 @@ void main()
 	}
 	
 	frag_color[0] = shadow;
-	frag_color[1] = calcAmbientOcclusion(pos, norm, pos_screen);
+	frag_color[1] = calcAmbientOcclusion(pos, norm);
 	
 	spos = vec4(shadow_pos+norm*spot_shadow_offset, 1.0);