NORSPEC-342 WIP MATBUG-347 make impostor rendering not contribute to depth for mostly-transparent buffer pixels, aka hair in ALM only

7 files changed, 74 insertions, 9 deletions

diff --git a/indra/newview/app_settings/shaders/class1/deferred/alphaF.glsl b/indra/newview/app_settings/shaders/class1/deferred/alphaF.glsl
index d0c7cc9dde..f35ff25dcb 100755
--- a/indra/newview/app_settings/shaders/class1/deferred/alphaF.glsl
+++ b/indra/newview/app_settings/shaders/class1/deferred/alphaF.glsl
@@ -531,6 +531,19 @@ void main()
 	vec4 diff = texture2D(diffuseMap,vary_texcoord0.xy);
 #endif
 
+#if FOR_IMPOSTOR
+	vec4 color;
+	color.rgb = diff.rgb;
+	color.a   = diff.a;
+
+	// Insure we don't pollute depth with invis pixels in impostor rendering
+	//
+	if (color.a < 0.01)
+	{
+		discard;
+	}
+#else
+
 #ifdef USE_VERTEX_COLOR
 	float final_alpha = diff.a * vertex_color.a;
 	diff.rgb *= vertex_color.rgb;
@@ -538,6 +551,7 @@ void main()
 	float final_alpha = diff.a;
 #endif
 
+
 	vec4 gamma_diff = diff;	
 	diff.rgb = srgb_to_linear(diff.rgb);
 
@@ -567,11 +581,10 @@ void main()
 	ambient = (1.0-ambient);
 
 	color.rgb *= ambient;
-
 	color.rgb += atmosAffectDirectionalLight(final_da);
 	color.rgb *= gamma_diff.rgb;
 
-	color.rgb = mix(diff.rgb, color.rgb, final_alpha);
+	//color.rgb = mix(diff.rgb, color.rgb, final_alpha);
 
 	color.rgb = atmosLighting(color.rgb);
 	color.rgb = scaleSoftClip(color.rgb);
@@ -602,6 +615,8 @@ void main()
 	color = applyWaterFogDeferred(pos.xyz, color);
 #endif
 
+#endif
+
 	frag_color = color;
 }
 
diff --git a/indra/newview/app_settings/shaders/class1/deferred/impostorF.glsl b/indra/newview/app_settings/shaders/class1/deferred/impostorF.glsl
index d1eaabf901..506323fdce 100755
--- a/indra/newview/app_settings/shaders/class1/deferred/impostorF.glsl
+++ b/indra/newview/app_settings/shaders/class1/deferred/impostorF.glsl
@@ -86,6 +86,8 @@ void main()
 	vec4 norm = texture2D(normalMap,   vary_texcoord0.xy);
 	vec4 spec = texture2D(specularMap, vary_texcoord0.xy);
 
+	col.rgb = linear_to_srgb(col.rgb);
+
 	frag_data[0] = vec4(col.rgb, col.a);
 	frag_data[1] = spec;
 	frag_data[2] = vec4(norm.xy,0,0);
diff --git a/indra/newview/lldrawpoolalpha.cpp b/indra/newview/lldrawpoolalpha.cpp
index ba83c95518..a8010d17d6 100755
--- a/indra/newview/lldrawpoolalpha.cpp
+++ b/indra/newview/lldrawpoolalpha.cpp
@@ -93,7 +93,12 @@ void LLDrawPoolAlpha::beginPostDeferredPass(S32 pass)
 
 	if (pass == 0)
 	{
-		if (LLPipeline::sUnderWaterRender)
+		if (LLPipeline::sImpostorRender)
+		{
+			simple_shader = &gDeferredAlphaImpostorProgram;
+			fullbright_shader = &gDeferredFullbrightProgram;
+		}
+		else if (LLPipeline::sUnderWaterRender)
 		{
 			simple_shader = &gDeferredAlphaWaterProgram;
 			fullbright_shader = &gDeferredFullbrightWaterProgram;
@@ -116,7 +121,7 @@ void LLDrawPoolAlpha::beginPostDeferredPass(S32 pass)
 
 		simple_shader->uniform1f(LLShaderMgr::DISPLAY_GAMMA, (gamma > 0.1f) ? 1.0f / gamma : (1.0f/2.2f));
 	}
-	else
+	else if (!LLPipeline::sImpostorRender)
 	{
 		//update depth buffer sampler
 		gPipeline.mScreen.flush();
@@ -156,7 +161,7 @@ void LLDrawPoolAlpha::beginPostDeferredPass(S32 pass)
 
 void LLDrawPoolAlpha::endPostDeferredPass(S32 pass) 
 { 
-	if (pass == 1)
+	if (pass == 1 && !LLPipeline::sImpostorRender)
 	{
 		gPipeline.mDeferredDepth.flush();
 		gPipeline.mScreen.bindTarget();
diff --git a/indra/newview/lldrawpoolavatar.cpp b/indra/newview/lldrawpoolavatar.cpp
index 9dbc85d121..8b1e934224 100755
--- a/indra/newview/lldrawpoolavatar.cpp
+++ b/indra/newview/lldrawpoolavatar.cpp
@@ -691,7 +691,7 @@ void LLDrawPoolAvatar::beginDeferredImpostor()
 	}
 
 	sVertexProgram = &gDeferredImpostorProgram;
-    specular_channel = sVertexProgram->enableTexture(LLViewerShaderMgr::SPECULAR_MAP);
+	specular_channel = sVertexProgram->enableTexture(LLViewerShaderMgr::SPECULAR_MAP);
 	normal_channel = sVertexProgram->enableTexture(LLViewerShaderMgr::DEFERRED_NORMAL);
 	sDiffuseChannel = sVertexProgram->enableTexture(LLViewerShaderMgr::DIFFUSE_MAP);
 	gPipeline.bindDeferredShader(*sVertexProgram);
@@ -701,12 +701,12 @@ void LLDrawPoolAvatar::beginDeferredImpostor()
 void LLDrawPoolAvatar::endDeferredImpostor()
 {
 	sShaderLevel = mVertexShaderLevel;
-    sVertexProgram->disableTexture(LLViewerShaderMgr::DEFERRED_NORMAL);
+   sVertexProgram->disableTexture(LLViewerShaderMgr::DEFERRED_NORMAL);
 	sVertexProgram->disableTexture(LLViewerShaderMgr::SPECULAR_MAP);
 	sVertexProgram->disableTexture(LLViewerShaderMgr::DIFFUSE_MAP);
 	gPipeline.unbindDeferredShader(*sVertexProgram);
-    sVertexProgram = NULL;
-    sDiffuseChannel = 0;
+   sVertexProgram = NULL;
+   sDiffuseChannel = 0;
 }
 
 void LLDrawPoolAvatar::beginDeferredRigid()
diff --git a/indra/newview/llviewershadermgr.cpp b/indra/newview/llviewershadermgr.cpp
index 264655029a..b2f7949388 100755
--- a/indra/newview/llviewershadermgr.cpp
+++ b/indra/newview/llviewershadermgr.cpp
@@ -197,6 +197,7 @@ LLGLSLShader			gDeferredShadowAlphaMaskProgram;
 LLGLSLShader			gDeferredAvatarShadowProgram;
 LLGLSLShader			gDeferredAttachmentShadowProgram;
 LLGLSLShader			gDeferredAlphaProgram;
+LLGLSLShader			gDeferredAlphaImpostorProgram;
 LLGLSLShader			gDeferredAlphaWaterProgram;
 LLGLSLShader			gDeferredAvatarEyesProgram;
 LLGLSLShader			gDeferredFullbrightProgram;
@@ -304,6 +305,8 @@ LLViewerShaderMgr::LLViewerShaderMgr() :
 	mShaderList.push_back(&gDeferredMaterialWaterProgram[9+LLMaterial::SHADER_COUNT]);
 	mShaderList.push_back(&gDeferredMaterialWaterProgram[13+LLMaterial::SHADER_COUNT]);	
 	mShaderList.push_back(&gDeferredAlphaProgram);
+	mShaderList.push_back(&gDeferredAlphaProgram);
+	mShaderList.push_back(&gDeferredAlphaImpostorProgram);
 	mShaderList.push_back(&gDeferredAlphaWaterProgram);
 	mShaderList.push_back(&gDeferredSkinnedAlphaProgram);
 	mShaderList.push_back(&gDeferredFullbrightProgram);
@@ -1533,6 +1536,40 @@ BOOL LLViewerShaderMgr::loadShadersDeferred()
 
 	if (success)
 	{
+		gDeferredAlphaImpostorProgram.mName = "Deferred Alpha Shader";
+
+		gDeferredAlphaImpostorProgram.mFeatures.calculatesLighting = false;
+		gDeferredAlphaImpostorProgram.mFeatures.hasLighting = false;
+		gDeferredAlphaImpostorProgram.mFeatures.isAlphaLighting = true;
+		gDeferredAlphaImpostorProgram.mFeatures.disableTextureIndex = true; //hack to disable auto-setup of texture channels
+		if (mVertexShaderLevel[SHADER_DEFERRED] < 1)
+		{
+			gDeferredAlphaImpostorProgram.mFeatures.mIndexedTextureChannels = LLGLSLShader::sIndexedTextureChannels;
+		}
+		else
+		{ //shave off some texture units for shadow maps
+			gDeferredAlphaImpostorProgram.mFeatures.mIndexedTextureChannels = llmax(LLGLSLShader::sIndexedTextureChannels - 6, 1);
+		}
+
+		gDeferredAlphaImpostorProgram.mShaderFiles.clear();
+		gDeferredAlphaImpostorProgram.mShaderFiles.push_back(make_pair("deferred/alphaV.glsl", GL_VERTEX_SHADER_ARB));
+		gDeferredAlphaImpostorProgram.mShaderFiles.push_back(make_pair("deferred/alphaF.glsl", GL_FRAGMENT_SHADER_ARB));
+		gDeferredAlphaImpostorProgram.addPermutation("USE_INDEXED_TEX", "1");
+		gDeferredAlphaImpostorProgram.addPermutation("HAS_SHADOW", mVertexShaderLevel[SHADER_DEFERRED] > 1 ? "1" : "0");
+		gDeferredAlphaImpostorProgram.addPermutation("USE_VERTEX_COLOR", "1");
+		gDeferredAlphaImpostorProgram.addPermutation("FOR_IMPOSTOR", "1");
+
+		gDeferredAlphaImpostorProgram.mShaderLevel = mVertexShaderLevel[SHADER_DEFERRED];
+
+		success = gDeferredAlphaImpostorProgram.createShader(NULL, NULL);
+
+		// Hack
+		gDeferredAlphaImpostorProgram.mFeatures.calculatesLighting = true;
+		gDeferredAlphaImpostorProgram.mFeatures.hasLighting = true;
+	}
+
+	if (success)
+	{
 		gDeferredAlphaWaterProgram.mName = "Deferred Alpha Underwater Shader";
 		gDeferredAlphaWaterProgram.mFeatures.calculatesLighting = false;
 		gDeferredAlphaWaterProgram.mFeatures.hasLighting = false;
diff --git a/indra/newview/llviewershadermgr.h b/indra/newview/llviewershadermgr.h
index 5fa9b82170..08c1ce6911 100755
--- a/indra/newview/llviewershadermgr.h
+++ b/indra/newview/llviewershadermgr.h
@@ -364,6 +364,7 @@ extern LLGLSLShader			gDeferredPostGammaCorrectProgram;
 extern LLGLSLShader			gDeferredAvatarShadowProgram;
 extern LLGLSLShader			gDeferredAttachmentShadowProgram;
 extern LLGLSLShader			gDeferredAlphaProgram;
+extern LLGLSLShader			gDeferredAlphaImpostorProgram;
 extern LLGLSLShader			gDeferredFullbrightProgram;
 extern LLGLSLShader			gDeferredFullbrightAlphaMaskProgram;
 extern LLGLSLShader			gDeferredAlphaWaterProgram;
diff --git a/indra/newview/pipeline.cpp b/indra/newview/pipeline.cpp
index 9378c89eb7..31baf4188b 100755
--- a/indra/newview/pipeline.cpp
+++ b/indra/newview/pipeline.cpp
@@ -11402,7 +11402,9 @@ void LLPipeline::generateImpostor(LLVOAvatar* avatar)
 		//
 		gGL.setColorMask(false,false);
 		renderGeomPostDeferred(camera);
+
 		sImpostorRenderAlphaDepthPass = false;
+
 	}
 	else
 	{
@@ -11418,10 +11420,13 @@ void LLPipeline::generateImpostor(LLVOAvatar* avatar)
 		// from the previous pass
 		//
 		sImpostorRenderAlphaDepthPass = true;
+
 		// depth-only here...
 		//
 		gGL.setColorMask(false,false);
 		renderGeom(camera);
+
+		sImpostorRenderAlphaDepthPass = false;
 	}
 
 	{ //create alpha mask based on depth buffer (grey out if muted)