SH-2181 Fix for alpha flickering when basic shaders enabled (don't use glAlphaFunc when shaders are available).

Reviewed by Leslie

5 files changed, 147 insertions, 17 deletions

diff --git a/indra/llrender/llgl.cpp b/indra/llrender/llgl.cpp
index 8937726209..2f6ef2b663 100644
--- a/indra/llrender/llgl.cpp
+++ b/indra/llrender/llgl.cpp
@@ -48,6 +48,7 @@
 #include "llstacktrace.h"
 
 #include "llglheaders.h"
+#include "llglslshader.h"
 
 #ifdef _DEBUG
 //#define GL_STATE_VERIFY
@@ -1781,6 +1782,16 @@ void LLGLState::checkClientArrays(const std::string& msg, U32 data_mask)
 LLGLState::LLGLState(LLGLenum state, S32 enabled) :
 	mState(state), mWasEnabled(FALSE), mIsEnabled(FALSE)
 {
+	if (LLGLSLShader::sNoFixedFunction)
+	{ //always disable state that's deprecated post GL 3.0
+		switch (state)
+		{
+			case GL_ALPHA_TEST:
+				enabled = 0;
+				break;
+		}
+	}
+
 	stop_glerror();
 	if (state)
 	{
diff --git a/indra/llrender/llglslshader.cpp b/indra/llrender/llglslshader.cpp
index 80c93bb0d2..f51d83abe4 100644
--- a/indra/llrender/llglslshader.cpp
+++ b/indra/llrender/llglslshader.cpp
@@ -49,6 +49,7 @@ using std::make_pair;
 using std::string;
 
 GLhandleARB LLGLSLShader::sCurBoundShader = 0;
+LLGLSLShader* LLGLSLShader::sCurBoundShaderPtr = NULL;
 bool LLGLSLShader::sNoFixedFunction = false;
 
 //UI shader -- declared here so llui_libtest will link properly
@@ -63,7 +64,8 @@ BOOL shouldChange(const LLVector4& v1, const LLVector4& v2)
 LLShaderFeatures::LLShaderFeatures()
 : calculatesLighting(false), isShiny(false), isFullbright(false), hasWaterFog(false),
 hasTransport(false), hasSkinning(false), hasObjectSkinning(false), hasAtmospherics(false), isSpecular(false),
-hasGamma(false), hasLighting(false), calculatesAtmospherics(false), mIndexedTextureChannels(0), disableTextureIndex(false)
+hasGamma(false), hasLighting(false), calculatesAtmospherics(false), mIndexedTextureChannels(0), disableTextureIndex(false),
+hasAlphaMask(false)
 {
 }
 
@@ -386,6 +388,7 @@ void LLGLSLShader::bind()
 	{
 		glUseProgramObjectARB(mProgramObject);
 		sCurBoundShader = mProgramObject;
+		sCurBoundShaderPtr = this;
 		if (mUniformsDirty)
 		{
 			LLShaderMgr::instance()->updateShaderUniforms(this);
@@ -410,6 +413,7 @@ void LLGLSLShader::unbind()
 		}
 		glUseProgramObjectARB(0);
 		sCurBoundShader = 0;
+		sCurBoundShaderPtr = NULL;
 		stop_glerror();
 	}
 }
@@ -418,6 +422,7 @@ void LLGLSLShader::bindNoShader(void)
 {
 	glUseProgramObjectARB(0);
 	sCurBoundShader = 0;
+	sCurBoundShaderPtr = NULL;
 }
 
 S32 LLGLSLShader::enableTexture(S32 uniform, LLTexUnit::eTextureType mode)
@@ -979,3 +984,9 @@ void LLGLSLShader::vertexAttrib4fv(U32 index, GLfloat* v)
 		glVertexAttrib4fvARB(mAttribute[index], v);
 	}
 }
+
+void LLGLSLShader::setAlphaRange(F32 minimum, F32 maximum)
+{
+	uniform1f("minimum_alpha", minimum);
+	uniform1f("maximum_alpha", maximum);
+}
diff --git a/indra/llrender/llglslshader.h b/indra/llrender/llglslshader.h
index 621e0b82ee..558ea66b50 100644
--- a/indra/llrender/llglslshader.h
+++ b/indra/llrender/llglslshader.h
@@ -47,6 +47,7 @@ public:
 	bool hasGamma;
 	S32 mIndexedTextureChannels;
 	bool disableTextureIndex;
+	bool hasAlphaMask;
 
 	// char numLights;
 	
@@ -67,6 +68,8 @@ public:
 	LLGLSLShader();
 
 	static GLhandleARB sCurBoundShader;
+	static LLGLSLShader* sCurBoundShaderPtr;
+
 	static bool sNoFixedFunction;
 
 	void unload();
@@ -105,6 +108,8 @@ public:
 	void uniformMatrix3fv(const std::string& uniform, U32 count, GLboolean transpose, const GLfloat *v);
 	void uniformMatrix4fv(const std::string& uniform, U32 count, GLboolean transpose, const GLfloat *v);
 
+	void setAlphaRange(F32 minimum, F32 maximum);
+
 	void vertexAttrib4f(U32 index, GLfloat x, GLfloat y, GLfloat z, GLfloat w);
 	void vertexAttrib4fv(U32 index, GLfloat* v);
 	
diff --git a/indra/llrender/llrender.cpp b/indra/llrender/llrender.cpp
index 70df1dd1d1..d72918b15d 100644
--- a/indra/llrender/llrender.cpp
+++ b/indra/llrender/llrender.cpp
@@ -1168,6 +1168,11 @@ void LLRender::setAlphaRejectSettings(eCompareFunc func, F32 value)
 {
 	flush();
 
+	if (LLGLSLShader::sNoFixedFunction)
+	{ //glAlphaFunc is deprecated in OpenGL 3.3
+		return;
+	}
+
 	if (mCurrAlphaFunc != func ||
 		mCurrAlphaFuncVal != value)
 	{
@@ -1182,6 +1187,30 @@ void LLRender::setAlphaRejectSettings(eCompareFunc func, F32 value)
 			glAlphaFunc(sGLCompareFunc[func], value);
 		}
 	}
+
+	if (gDebugGL)
+	{ //make sure cached state is correct
+		GLint cur_func = 0;
+		glGetIntegerv(GL_ALPHA_TEST_FUNC, &cur_func);
+
+		if (func == CF_DEFAULT)
+		{
+			func = CF_GREATER;
+		}
+
+		if (cur_func != sGLCompareFunc[func])
+		{
+			llerrs << "Alpha test function corrupted!" << llendl;
+		}
+
+		F32 ref = 0.f;
+		glGetFloatv(GL_ALPHA_TEST_REF, &ref);
+
+		if (ref != value)
+		{
+			llerrs << "Alpha test value corrupted!" << llendl;
+		}
+	}
 }
 
 void LLRender::blendFunc(eBlendFactor sfactor, eBlendFactor dfactor)
diff --git a/indra/llrender/llshadermgr.cpp b/indra/llrender/llshadermgr.cpp
index 751b250d96..986c1f2774 100644
--- a/indra/llrender/llshadermgr.cpp
+++ b/indra/llrender/llshadermgr.cpp
@@ -206,21 +206,40 @@ BOOL LLShaderMgr::attachShaderFeatures(LLGLSLShader * shader)
 	
 	if (features->hasLighting)
 	{
-	
 		if (features->hasWaterFog)
 		{
 			if (features->disableTextureIndex)
 			{
-				if (!shader->attachObject("lighting/lightWaterNonIndexedF.glsl"))
+				if (features->hasAlphaMask)
 				{
-					return FALSE;
+					if (!shader->attachObject("lighting/lightWaterAlphaMaskNonIndexedF.glsl"))
+					{
+						return FALSE;
+					}
+				}
+				else
+				{
+					if (!shader->attachObject("lighting/lightWaterNonIndexedF.glsl"))
+					{
+						return FALSE;
+					}
 				}
 			}
 			else 
 			{
-				if (!shader->attachObject("lighting/lightWaterF.glsl"))
+				if (features->hasAlphaMask)
 				{
-					return FALSE;
+					if (!shader->attachObject("lighting/lightWaterAlphaMaskF.glsl"))
+					{
+						return FALSE;
+					}
+				}
+				else
+				{
+					if (!shader->attachObject("lighting/lightWaterF.glsl"))
+					{
+						return FALSE;
+					}
 				}
 				shader->mFeatures.mIndexedTextureChannels = gGLManager.mNumTextureImageUnits-1;
 			}
@@ -230,16 +249,36 @@ BOOL LLShaderMgr::attachShaderFeatures(LLGLSLShader * shader)
 		{
 			if (features->disableTextureIndex)
 			{
-				if (!shader->attachObject("lighting/lightNonIndexedF.glsl"))
+				if (features->hasAlphaMask)
 				{
-					return FALSE;
+					if (!shader->attachObject("lighting/lightAlphaMaskNonIndexedF.glsl"))
+					{
+						return FALSE;
+					}
+				}
+				else
+				{
+					if (!shader->attachObject("lighting/lightNonIndexedF.glsl"))
+					{
+						return FALSE;
+					}
 				}
 			}
 			else 
 			{
-				if (!shader->attachObject("lighting/lightF.glsl"))
+				if (features->hasAlphaMask)
 				{
-					return FALSE;
+					if (!shader->attachObject("lighting/lightAlphaMaskF.glsl"))
+					{
+						return FALSE;
+					}
+				}
+				else
+				{
+					if (!shader->attachObject("lighting/lightF.glsl"))
+					{
+						return FALSE;
+					}
 				}
 				shader->mFeatures.mIndexedTextureChannels = gGLManager.mNumTextureImageUnits-1;
 			}
@@ -272,14 +311,28 @@ BOOL LLShaderMgr::attachShaderFeatures(LLGLSLShader * shader)
 		{
 			if (features->disableTextureIndex)
 			{
-				if (!shader->attachObject("lighting/lightFullbrightWaterNonIndexedF.glsl"))
+				if (features->hasAlphaMask)
+				{
+					if (!shader->attachObject("lighting/lightFullbrightWaterNonIndexedAlphaMaskF.glsl"))
+					{
+						return FALSE;
+					}
+				}
+				else if (!shader->attachObject("lighting/lightFullbrightWaterNonIndexedF.glsl"))
 				{
 					return FALSE;
 				}
 			}
 			else 
 			{
-				if (!shader->attachObject("lighting/lightFullbrightWaterF.glsl"))
+				if (features->hasAlphaMask)
+				{
+					if (!shader->attachObject("lighting/lightFullbrightWaterAlphaMaskF.glsl"))
+					{
+						return FALSE;
+					}
+				}
+				else if (!shader->attachObject("lighting/lightFullbrightWaterF.glsl"))
 				{
 					return FALSE;
 				}
@@ -310,16 +363,37 @@ BOOL LLShaderMgr::attachShaderFeatures(LLGLSLShader * shader)
 		{
 			if (features->disableTextureIndex)
 			{
-				if (!shader->attachObject("lighting/lightFullbrightNonIndexedF.glsl"))
+
+				if (features->hasAlphaMask)
 				{
-					return FALSE;
+					if (!shader->attachObject("lighting/lightFullbrightNonIndexedAlphaMaskF.glsl"))
+					{
+						return FALSE;
+					}
+				}
+				else
+				{
+					if (!shader->attachObject("lighting/lightFullbrightNonIndexedF.glsl"))
+					{
+						return FALSE;
+					}
 				}
 			}
 			else 
 			{
-				if (!shader->attachObject("lighting/lightFullbrightF.glsl"))
+				if (features->hasAlphaMask)
 				{
-					return FALSE;
+					if (!shader->attachObject("lighting/lightFullbrightAlphaMaskF.glsl"))
+					{
+						return FALSE;
+					}
+				}
+				else
+				{
+					if (!shader->attachObject("lighting/lightFullbrightF.glsl"))
+					{
+						return FALSE;
+					}
 				}
 				shader->mFeatures.mIndexedTextureChannels = gGLManager.mNumTextureImageUnits-1;
 			}
@@ -406,7 +480,7 @@ void LLShaderMgr::dumpObjectLog(GLhandleARB ret, BOOL warns)
 			LL_DEBUGS("ShaderLoading") << log << LL_ENDL;
 		}
 	}
-}
+ }
 
 GLhandleARB LLShaderMgr::loadShaderFile(const std::string& filename, S32 & shader_level, GLenum type, S32 texture_index_channels)
 {