CTS-54 Fix for SSAO artifacts far away.

2 files changed, 94 insertions, 74 deletions

diff --git a/indra/newview/app_settings/shaders/class1/deferred/sunLightF.glsl b/indra/newview/app_settings/shaders/class1/deferred/sunLightF.glsl
index 22bdd2c7f3..fafc2ae3cc 100644
--- a/indra/newview/app_settings/shaders/class1/deferred/sunLightF.glsl
+++ b/indra/newview/app_settings/shaders/class1/deferred/sunLightF.glsl
@@ -53,51 +53,61 @@ vec4 getPosition(vec2 pos_screen)
 //calculate decreases in ambient lighting when crowded out (SSAO)
 float calcAmbientOcclusion(vec4 pos, vec3 norm)
 {
-	vec2 kern[8];
-	// exponentially (^2) distant occlusion samples spread around origin
-	kern[0] = vec2(-1.0, 0.0) * 0.125*0.125;
-	kern[1] = vec2(1.0, 0.0) * 0.250*0.250;
-	kern[2] = vec2(0.0, 1.0) * 0.375*0.375;
-	kern[3] = vec2(0.0, -1.0) * 0.500*0.500;
-	kern[4] = vec2(0.7071, 0.7071) * 0.625*0.625;
-	kern[5] = vec2(-0.7071, -0.7071) * 0.750*0.750;
-	kern[6] = vec2(-0.7071, 0.7071) * 0.875*0.875;
-	kern[7] = vec2(0.7071, -0.7071) * 1.000*1.000;
-
-	vec2 pos_screen = vary_fragcoord.xy;
-	vec3 pos_world = pos.xyz;
-	vec2 noise_reflect = texture2D(noiseMap, vary_fragcoord.xy/128.0).xy;
+	float ret = 1.0;
 	
-	float angle_hidden = 0.0;
-	int points = 0;
+	float dist = dot(pos.xyz,pos.xyz);
 	
-	float scale = min(ssao_radius / -pos_world.z, ssao_max_radius);
-	
-	// it was found that keeping # of samples a constant was the fastest, probably due to compiler optimizations (unrolling?)
-	for (int i = 0; i < 8; i++)
+	if (dist < 64.0*64.0)
 	{
-		vec2 samppos_screen = pos_screen + scale * reflect(kern[i], noise_reflect);
-		vec3 samppos_world = getPosition(samppos_screen).xyz; 
+		vec2 kern[8];
+		// exponentially (^2) distant occlusion samples spread around origin
+		kern[0] = vec2(-1.0, 0.0) * 0.125*0.125;
+		kern[1] = vec2(1.0, 0.0) * 0.250*0.250;
+		kern[2] = vec2(0.0, 1.0) * 0.375*0.375;
+		kern[3] = vec2(0.0, -1.0) * 0.500*0.500;
+		kern[4] = vec2(0.7071, 0.7071) * 0.625*0.625;
+		kern[5] = vec2(-0.7071, -0.7071) * 0.750*0.750;
+		kern[6] = vec2(-0.7071, 0.7071) * 0.875*0.875;
+		kern[7] = vec2(0.7071, -0.7071) * 1.000*1.000;
+
+		vec2 pos_screen = vary_fragcoord.xy;
+		vec3 pos_world = pos.xyz;
+		vec2 noise_reflect = texture2D(noiseMap, vary_fragcoord.xy/128.0).xy;
 		
-		vec3 diff = pos_world - samppos_world;
-		float dist2 = dot(diff, diff);
+		float angle_hidden = 0.0;
+		int points = 0;
 		
-		// assume each sample corresponds to an occluding sphere with constant radius, constant x-sectional area
-		// --> solid angle shrinking by the square of distance
-		//radius is somewhat arbitrary, can approx with just some constant k * 1 / dist^2
-		//(k should vary inversely with # of samples, but this is taken care of later)
+		float scale = min(ssao_radius / -pos_world.z, ssao_max_radius);
 		
-		//if (dot((samppos_world - 0.05*norm - pos_world), norm) > 0.0)  // -0.05*norm to shift sample point back slightly for flat surfaces
-		//	angle_hidden += min(1.0/dist2, ssao_factor_inv); // dist != 0 follows from conditional.  max of 1.0 (= ssao_factor_inv * ssao_factor)
-		angle_hidden = angle_hidden + float(dot((samppos_world - 0.05*norm - pos_world), norm) > 0.0) * min(1.0/dist2, ssao_factor_inv);
+		// it was found that keeping # of samples a constant was the fastest, probably due to compiler optimizations (unrolling?)
+		for (int i = 0; i < 8; i++)
+		{
+			vec2 samppos_screen = pos_screen + scale * reflect(kern[i], noise_reflect);
+			vec3 samppos_world = getPosition(samppos_screen).xyz; 
+			
+			vec3 diff = pos_world - samppos_world;
+			float dist2 = dot(diff, diff);
+			
+			// assume each sample corresponds to an occluding sphere with constant radius, constant x-sectional area
+			// --> solid angle shrinking by the square of distance
+			//radius is somewhat arbitrary, can approx with just some constant k * 1 / dist^2
+			//(k should vary inversely with # of samples, but this is taken care of later)
+			
+			//if (dot((samppos_world - 0.05*norm - pos_world), norm) > 0.0)  // -0.05*norm to shift sample point back slightly for flat surfaces
+			//	angle_hidden += min(1.0/dist2, ssao_factor_inv); // dist != 0 follows from conditional.  max of 1.0 (= ssao_factor_inv * ssao_factor)
+			angle_hidden = angle_hidden + float(dot((samppos_world - 0.05*norm - pos_world), norm) > 0.0) * min(1.0/dist2, ssao_factor_inv);
+			
+			// 'blocked' samples (significantly closer to camera relative to pos_world) are "no data", not "no occlusion" 
+			points = points + int(diff.z > -1.0);
+		}
 		
-		// 'blocked' samples (significantly closer to camera relative to pos_world) are "no data", not "no occlusion" 
-		points = points + int(diff.z > -1.0);
+		angle_hidden = min(ssao_factor*angle_hidden/float(points), 1.0);
+		
+		ret = (1.0 - (float(points != 0) * angle_hidden));
+		ret += max((dist-32.0*32.0)/(32.0*32.0), 0.0);
 	}
 	
-	angle_hidden = min(ssao_factor*angle_hidden/float(points), 1.0);
-	
-	return (1.0 - (float(points != 0) * angle_hidden));
+	return min(ret, 1.0);
 }
 
 void main() 
diff --git a/indra/newview/app_settings/shaders/class2/deferred/sunLightF.glsl b/indra/newview/app_settings/shaders/class2/deferred/sunLightF.glsl
index a0026edcd2..74a948f584 100644
--- a/indra/newview/app_settings/shaders/class2/deferred/sunLightF.glsl
+++ b/indra/newview/app_settings/shaders/class2/deferred/sunLightF.glsl
@@ -55,51 +55,61 @@ vec4 getPosition(vec2 pos_screen)
 //calculate decreases in ambient lighting when crowded out (SSAO)
 float calcAmbientOcclusion(vec4 pos, vec3 norm)
 {
-	vec2 kern[8];
-	// exponentially (^2) distant occlusion samples spread around origin
-	kern[0] = vec2(-1.0, 0.0) * 0.125*0.125;
-	kern[1] = vec2(1.0, 0.0) * 0.250*0.250;
-	kern[2] = vec2(0.0, 1.0) * 0.375*0.375;
-	kern[3] = vec2(0.0, -1.0) * 0.500*0.500;
-	kern[4] = vec2(0.7071, 0.7071) * 0.625*0.625;
-	kern[5] = vec2(-0.7071, -0.7071) * 0.750*0.750;
-	kern[6] = vec2(-0.7071, 0.7071) * 0.875*0.875;
-	kern[7] = vec2(0.7071, -0.7071) * 1.000*1.000;
-
-	vec2 pos_screen = vary_fragcoord.xy;
-	vec3 pos_world = pos.xyz;
-	vec2 noise_reflect = texture2D(noiseMap, vary_fragcoord.xy/128.0).xy;
-	
-	float angle_hidden = 0.0;
-	int points = 0;
+	float ret = 1.0;
 	
-	float scale = min(ssao_radius / -pos_world.z, ssao_max_radius);
+	float dist = dot(pos.xyz,pos.xyz);
 	
-	// it was found that keeping # of samples a constant was the fastest, probably due to compiler optimizations (unrolling?)
-	for (int i = 0; i < 8; i++)
+	if (dist < 64.0*64.0)
 	{
-		vec2 samppos_screen = pos_screen + scale * reflect(kern[i], noise_reflect);
-		vec3 samppos_world = getPosition(samppos_screen).xyz; 
+		vec2 kern[8];
+		// exponentially (^2) distant occlusion samples spread around origin
+		kern[0] = vec2(-1.0, 0.0) * 0.125*0.125;
+		kern[1] = vec2(1.0, 0.0) * 0.250*0.250;
+		kern[2] = vec2(0.0, 1.0) * 0.375*0.375;
+		kern[3] = vec2(0.0, -1.0) * 0.500*0.500;
+		kern[4] = vec2(0.7071, 0.7071) * 0.625*0.625;
+		kern[5] = vec2(-0.7071, -0.7071) * 0.750*0.750;
+		kern[6] = vec2(-0.7071, 0.7071) * 0.875*0.875;
+		kern[7] = vec2(0.7071, -0.7071) * 1.000*1.000;
+
+		vec2 pos_screen = vary_fragcoord.xy;
+		vec3 pos_world = pos.xyz;
+		vec2 noise_reflect = texture2D(noiseMap, vary_fragcoord.xy/128.0).xy;
 		
-		vec3 diff = pos_world - samppos_world;
-		float dist2 = dot(diff, diff);
+		float angle_hidden = 0.0;
+		int points = 0;
 		
-		// assume each sample corresponds to an occluding sphere with constant radius, constant x-sectional area
-		// --> solid angle shrinking by the square of distance
-		//radius is somewhat arbitrary, can approx with just some constant k * 1 / dist^2
-		//(k should vary inversely with # of samples, but this is taken care of later)
+		float scale = min(ssao_radius / -pos_world.z, ssao_max_radius);
 		
-		//if (dot((samppos_world - 0.05*norm - pos_world), norm) > 0.0)  // -0.05*norm to shift sample point back slightly for flat surfaces
-		//	angle_hidden += min(1.0/dist2, ssao_factor_inv); // dist != 0 follows from conditional.  max of 1.0 (= ssao_factor_inv * ssao_factor)
-		angle_hidden = angle_hidden + float(dot((samppos_world - 0.05*norm - pos_world), norm) > 0.0) * min(1.0/dist2, ssao_factor_inv);
+		// it was found that keeping # of samples a constant was the fastest, probably due to compiler optimizations (unrolling?)
+		for (int i = 0; i < 8; i++)
+		{
+			vec2 samppos_screen = pos_screen + scale * reflect(kern[i], noise_reflect);
+			vec3 samppos_world = getPosition(samppos_screen).xyz; 
+			
+			vec3 diff = pos_world - samppos_world;
+			float dist2 = dot(diff, diff);
+			
+			// assume each sample corresponds to an occluding sphere with constant radius, constant x-sectional area
+			// --> solid angle shrinking by the square of distance
+			//radius is somewhat arbitrary, can approx with just some constant k * 1 / dist^2
+			//(k should vary inversely with # of samples, but this is taken care of later)
+			
+			//if (dot((samppos_world - 0.05*norm - pos_world), norm) > 0.0)  // -0.05*norm to shift sample point back slightly for flat surfaces
+			//	angle_hidden += min(1.0/dist2, ssao_factor_inv); // dist != 0 follows from conditional.  max of 1.0 (= ssao_factor_inv * ssao_factor)
+			angle_hidden = angle_hidden + float(dot((samppos_world - 0.05*norm - pos_world), norm) > 0.0) * min(1.0/dist2, ssao_factor_inv);
+			
+			// 'blocked' samples (significantly closer to camera relative to pos_world) are "no data", not "no occlusion" 
+			points = points + int(diff.z > -1.0);
+		}
 		
-		// 'blocked' samples (significantly closer to camera relative to pos_world) are "no data", not "no occlusion" 
-		points = points + int(diff.z > -1.0);
+		angle_hidden = min(ssao_factor*angle_hidden/float(points), 1.0);
+		
+		ret = (1.0 - (float(points != 0) * angle_hidden));
+		ret += max((dist-32.0*32.0)/(32.0*32.0), 0.0);
 	}
 	
-	angle_hidden = min(ssao_factor*angle_hidden/float(points), 1.0);
-	
-	return (1.0 - (float(points != 0) * angle_hidden));
+	return min(ret, 1.0);
 }
 
 float pcfShadow(sampler2DRectShadow shadowMap, vec4 stc, float scl)
@@ -224,11 +234,11 @@ void main()
 	
 	//spotlight shadow 1
 	vec4 lpos = shadow_matrix[4]*spos;
-	gl_FragColor[2] = pcfShadow(shadowMap4, lpos, 0.1).x; 
+	gl_FragColor[2] = pcfShadow(shadowMap4, lpos, 0.8).x; 
 	
 	//spotlight shadow 2
 	lpos = shadow_matrix[5]*spos;
-	gl_FragColor[3] = pcfShadow(shadowMap5, lpos, 0.1).x; 
+	gl_FragColor[3] = pcfShadow(shadowMap5, lpos, 0.8).x; 
 
 	//gl_FragColor.rgb = pos.xyz;
 	//gl_FragColor.b = shadow;