fixes VWR-4214 Glow effect passes through alpha textures where alpha is actually "solid"

and its many many dupes.

In summary this does the following:
* When laying down alpha, uses the new separated color-and-alpha blend func support to suppress existing glow in proportion to the opacity of the incoming fragments
* Moves glowing-alpha support from the glow pool to the alpha pool, ensures that the glow pass for a vertex buffer happens after the alpha pass for the vertex buffer so that it doesn't suppress its *own* glow.

Tested with all shader modes, FBO on/off, and deferred rendering.  Tested with every permutation of alpha/non-alpha/glow/non-glow/fullbright/waterglow/particle interaction I could contrive so far. :)

2 files changed, 41 insertions, 12 deletions

diff --git a/indra/newview/lldrawpoolalpha.cpp b/indra/newview/lldrawpoolalpha.cpp
index 75973cfa54..9a778a063d 100644
--- a/indra/newview/lldrawpoolalpha.cpp
+++ b/indra/newview/lldrawpoolalpha.cpp
@@ -204,7 +204,12 @@ void LLDrawPoolAlpha::render(S32 pass)
 	}
 
 	LLGLDepthTest depth(GL_TRUE, LLDrawPoolWater::sSkipScreenCopy ? GL_TRUE : GL_FALSE);
+
+	gGL.setColorMask(true, true);
+	gGL.blendFunc(LLRender::BF_SOURCE_ALPHA, LLRender::BF_ONE_MINUS_SOURCE_ALPHA,
+		      LLRender::BF_ZERO, LLRender::BF_ONE_MINUS_SOURCE_ALPHA);
 	renderAlpha(getVertexDataMask());
+	gGL.setColorMask(true, false);
 
 	if (deferred_render && current_shader != NULL)
 	{
@@ -283,9 +288,18 @@ void LLDrawPoolAlpha::renderAlpha(U32 mask)
 	for (LLCullResult::sg_list_t::iterator i = gPipeline.beginAlphaGroups(); i != gPipeline.endAlphaGroups(); ++i)
 	{
 		LLSpatialGroup* group = *i;
+		llassert(group);
+		llassert(group->mSpatialPartition);
+
 		if (group->mSpatialPartition->mRenderByGroup &&
-			!group->isDead())
+		    !group->isDead())
 		{
+			bool draw_glow_for_this_partition = mVertexShaderLevel > 0 && // no shaders = no glow.
+				// All particle systems seem to come off the wire with texture entries which claim that they glow.  This is probably a bug in the data.  Suppress.
+				group->mSpatialPartition->mPartitionType != LLViewerRegion::PARTITION_PARTICLE &&
+				group->mSpatialPartition->mPartitionType != LLViewerRegion::PARTITION_CLOUD &&
+				group->mSpatialPartition->mPartitionType != LLViewerRegion::PARTITION_HUD_PARTICLE;
+
 			LLSpatialGroup::drawmap_elem_t& draw_info = group->mDrawMap[LLRenderPass::PASS_ALPHA];
 
 			for (LLSpatialGroup::drawmap_elem_t::iterator k = draw_info.begin(); k != draw_info.end(); ++k)	
@@ -385,6 +399,27 @@ void LLDrawPoolAlpha::renderAlpha(U32 mask)
 				params.mVertexBuffer->setBuffer(mask);
 				params.mVertexBuffer->drawRange(params.mDrawMode, params.mStart, params.mEnd, params.mCount, params.mOffset);
 				gPipeline.addTrianglesDrawn(params.mCount, params.mDrawMode);
+				
+				// If this alpha mesh has glow, then draw it a second time to add the destination-alpha (=glow).  Interleaving these state-changing calls could be expensive, but glow must be drawn Z-sorted with alpha.
+				if (draw_glow_for_this_partition &&
+				    params.mGlowColor.mV[3] > 0)
+				{
+					// install glow-accumulating blend mode
+					gGL.blendFunc(LLRender::BF_ZERO, LLRender::BF_ONE, // don't touch color
+						      LLRender::BF_ONE, LLRender::BF_ONE); // add to alpha (glow)
+
+					// glow doesn't use vertex colors from the mesh data
+					params.mVertexBuffer->setBuffer(mask & ~LLVertexBuffer::MAP_COLOR);
+					glColor4ubv(params.mGlowColor.mV);
+
+					// do the actual drawing, again
+					params.mVertexBuffer->drawRange(params.mDrawMode, params.mStart, params.mEnd, params.mCount, params.mOffset);
+					gPipeline.addTrianglesDrawn(params.mCount, params.mDrawMode);
+
+					// restore our alpha blend mode
+					gGL.blendFunc(LLRender::BF_SOURCE_ALPHA, LLRender::BF_ONE_MINUS_SOURCE_ALPHA,
+						      LLRender::BF_ZERO, LLRender::BF_ONE_MINUS_SOURCE_ALPHA);
+				}
 			
 				if (params.mTextureMatrix && params.mTexture.notNull())
 				{
diff --git a/indra/newview/llvovolume.cpp b/indra/newview/llvovolume.cpp
index 1d3d943b06..91aa994eab 100644
--- a/indra/newview/llvovolume.cpp
+++ b/indra/newview/llvovolume.cpp
@@ -3317,11 +3317,10 @@ void LLVolumeGeometryManager::registerFace(LLSpatialGroup* group, LLFace* facep,
 
 	S32 idx = draw_vec.size()-1;
 
-
 	BOOL fullbright = (type == LLRenderPass::PASS_FULLBRIGHT) ||
-					  (type == LLRenderPass::PASS_INVISIBLE) ||
-					  (type == LLRenderPass::PASS_ALPHA ? facep->isState(LLFace::FULLBRIGHT) : FALSE);
-
+		(type == LLRenderPass::PASS_INVISIBLE) ||
+		(type == LLRenderPass::PASS_ALPHA ? facep->isState(LLFace::FULLBRIGHT) : FALSE);
+	
 	if (!fullbright && type != LLRenderPass::PASS_GLOW && !facep->mVertexBuffer->hasDataType(LLVertexBuffer::TYPE_NORMAL))
 	{
 		llwarns << "Non fullbright face has no normals!" << llendl;
@@ -3350,12 +3349,7 @@ void LLVolumeGeometryManager::registerFace(LLSpatialGroup* group, LLFace* facep,
 	
 	LLViewerTexture* tex = facep->getTexture();
 
-	U8 glow = 0;
-		
-	if (type == LLRenderPass::PASS_GLOW)
-	{
-		glow = (U8) (facep->getTextureEntry()->getGlow() * 255);
-	}
+	U8 glow = (U8) (facep->getTextureEntry()->getGlow() * 255);
 
 	if (facep->mVertexBuffer.isNull())
 	{
@@ -4001,7 +3995,7 @@ void LLVolumeGeometryManager::genDrawInfo(LLSpatialGroup* group, U32 mask, std::
 				}
 			}
 
-			if (LLPipeline::sRenderGlow && te->getGlow() > 0.f)
+			if (!is_alpha && LLPipeline::sRenderGlow && te->getGlow() > 0.f)
 			{
 				registerFace(group, facep, LLRenderPass::PASS_GLOW);
 			}