SH-1618 FIX -- Lighting and shadows crash ATI macs

* Fixed ATI mac "lighting and shadows" related crash
* Fixed up numerous GL errors on macs related to multiple color formats,
  the use of glEnable/glDisable on textures above the texture unit count
  and old ATI-specific code that was not appropriate for Mac.
* Disabled SSAO for ATI macs due to it not working with shadows
* Ongoing work to properly get shadows and SSAO functioning on ATI macs is required.

Reviewed by davep

4 files changed, 54 insertions, 29 deletions

diff --git a/indra/newview/app_settings/shaders/class1/deferred/multiPointLightF.glsl b/indra/newview/app_settings/shaders/class1/deferred/multiPointLightF.glsl
index c5ddf31ac0..609fc4f14f 100644
--- a/indra/newview/app_settings/shaders/class1/deferred/multiPointLightF.glsl
+++ b/indra/newview/app_settings/shaders/class1/deferred/multiPointLightF.glsl
@@ -23,8 +23,9 @@ uniform float sun_wash;
 
 uniform int light_count;
 
-uniform vec4 light[16];
-uniform vec4 light_col[16];
+#define MAX_LIGHT_COUNT		16
+uniform vec4 light[MAX_LIGHT_COUNT];
+uniform vec4 light_col[MAX_LIGHT_COUNT];
 
 varying vec4 vary_fragcoord;
 uniform vec2 screen_res;
@@ -63,50 +64,56 @@ void main()
 	float noise = texture2D(noiseMap, frag.xy/128.0).b;
 	vec3 out_col = vec3(0,0,0);
 	vec3 npos = normalize(-pos);
-	
-	for (int i = 0; i < light_count; ++i)
+
+	// As of OSX 10.6.7 ATI Apple's crash when using a variable size loop
+	for (int i = 0; i < MAX_LIGHT_COUNT; ++i)
 	{
+		bool light_contrib = (i < light_count);
+		
 		vec3 lv = light[i].xyz-pos;
 		float dist2 = dot(lv,lv);
 		dist2 /= light[i].w;
 		if (dist2 > 1.0)
 		{
-			continue;
+			light_contrib = false;
 		}
 		
 		float da = dot(norm, lv);
 		if (da < 0.0)
 		{
-			continue;
+			light_contrib = false;
 		}
-				
-		lv = normalize(lv);
-		da = dot(norm, lv);
-				
-		float fa = light_col[i].a+1.0;
-		float dist_atten = clamp(1.0-(dist2-1.0*(1.0-fa))/fa, 0.0, 1.0);
-		dist_atten *= noise;
-
-		float lit = da * dist_atten;
 		
-		vec3 col = light_col[i].rgb*lit*diff;
-		//vec3 col = vec3(dist2, light_col[i].a, lit);
-		
-		if (spec.a > 0.0)
+		if (light_contrib)
 		{
-			//vec3 ref = dot(pos+lv, norm);
+			lv = normalize(lv);
+			da = dot(norm, lv);
+					
+			float fa = light_col[i].a+1.0;
+			float dist_atten = clamp(1.0-(dist2-1.0*(1.0-fa))/fa, 0.0, 1.0);
+			dist_atten *= noise;
+
+			float lit = da * dist_atten;
 			
-			float sa = dot(normalize(lv+npos),norm);
+			vec3 col = light_col[i].rgb*lit*diff;
+			//vec3 col = vec3(dist2, light_col[i].a, lit);
 			
-			if (sa > 0.0)
+			if (spec.a > 0.0)
 			{
-				sa = texture2D(lightFunc,vec2(sa, spec.a)).a * min(dist_atten*4.0, 1.0);
-				sa *= noise;
-				col += da*sa*light_col[i].rgb*spec.rgb;
+				//vec3 ref = dot(pos+lv, norm);
+				
+				float sa = dot(normalize(lv+npos),norm);
+				
+				if (sa > 0.0)
+				{
+					sa = texture2D(lightFunc,vec2(sa, spec.a)).a * min(dist_atten*4.0, 1.0);
+					sa *= noise;
+					col += da*sa*light_col[i].rgb*spec.rgb;
+				}
 			}
+			
+			out_col += col;
 		}
-		
-		out_col += col;	
 	}
 	
 	if (dot(out_col, out_col) <= 0.0)
diff --git a/indra/newview/featuretable_mac.txt b/indra/newview/featuretable_mac.txt
index c075c660f3..e2b979d9e9 100644
--- a/indra/newview/featuretable_mac.txt
+++ b/indra/newview/featuretable_mac.txt
@@ -281,6 +281,9 @@ RenderVBOEnable				1	0
 list TexUnit8orLess
 RenderDeferredSSAO			0	0
 
+list ATI
+RenderDeferredSSAO			0	0
+
 list Intel
 RenderAnisotropic			1	0
 RenderLocalLights			1	0
diff --git a/indra/newview/llfeaturemanager.cpp b/indra/newview/llfeaturemanager.cpp
index 524d2d74ef..b7aabe2aeb 100644
--- a/indra/newview/llfeaturemanager.cpp
+++ b/indra/newview/llfeaturemanager.cpp
@@ -753,7 +753,7 @@ void LLFeatureManager::applyBaseMasks()
 	{
 		maskFeatures("OpenGLPre30");
 	}
-	if (gGLManager.mNumTextureUnits <= 8)
+	if (gGLManager.mNumTextureImageUnits <= 8)
 	{
 		maskFeatures("TexUnit8orLess");
 	}
diff --git a/indra/newview/pipeline.cpp b/indra/newview/pipeline.cpp
index f64eb89866..845a87b8cf 100644
--- a/indra/newview/pipeline.cpp
+++ b/indra/newview/pipeline.cpp
@@ -625,8 +625,14 @@ void LLPipeline::allocateScreenBuffer(U32 resX, U32 resY)
 		mDeferredDepth.allocate(resX, resY, 0, TRUE, FALSE, LLTexUnit::TT_RECT_TEXTURE, FALSE);
 		addDeferredAttachments(mDeferredScreen);
 	
-		mScreen.allocate(resX, resY, GL_RGBA, FALSE, FALSE, LLTexUnit::TT_RECT_TEXTURE, FALSE);		
+		mScreen.allocate(resX, resY, GL_RGBA, FALSE, FALSE, LLTexUnit::TT_RECT_TEXTURE, FALSE);
+		
+#if LL_DARWIN
+		// As of OS X 10.6.7, Apple doesn't support multiple color formats in a single FBO
+		mEdgeMap.allocate(resX, resY, GL_RGBA, FALSE, FALSE, LLTexUnit::TT_RECT_TEXTURE, FALSE);
+#else
 		mEdgeMap.allocate(resX, resY, GL_ALPHA, FALSE, FALSE, LLTexUnit::TT_RECT_TEXTURE, FALSE);
+#endif
 
 		if (shadow_detail > 0 || ssao)
 		{ //only need mDeferredLight[0] for shadows OR ssao
@@ -651,7 +657,12 @@ void LLPipeline::allocateScreenBuffer(U32 resX, U32 resY)
 			mDeferredLight[2].allocate(resX, resY, GL_RGBA, FALSE, FALSE, LLTexUnit::TT_RECT_TEXTURE);
 			for (U32 i = 0; i < 2; i++)
 			{
+#if LL_DARWIN
+				// As of OS X 10.6.7, Apple doesn't support multiple color formats in a single FBO
+				mGIMapPost[i].allocate(resX,resY, GL_RGBA, FALSE, FALSE, LLTexUnit::TT_RECT_TEXTURE);
+#else
 				mGIMapPost[i].allocate(resX,resY, GL_RGB, FALSE, FALSE, LLTexUnit::TT_RECT_TEXTURE);
+#endif
 			}
 		}
 		else
@@ -666,8 +677,12 @@ void LLPipeline::allocateScreenBuffer(U32 resX, U32 resY)
 
 		F32 scale = gSavedSettings.getF32("RenderShadowResolutionScale");
 
+#if LL_DARWIN
+		U32 shadow_fmt = 0;
+#else
 		//HACK: make alpha masking work on ATI depth shadows (work around for ATI driver bug)
 		U32 shadow_fmt = gGLManager.mIsATI ? GL_ALPHA : 0;
+#endif
 
 		if (shadow_detail > 0)
 		{ //allocate 4 sun shadow maps