EXT-2818 not using IMG_INVISIBLE in baked textures

this patch allows IMG_INVISIBLE in an alpha mask to set the baked texture to
also be IMG_INVISIBLE for the appropriate baked texture.

Will be post-reviewed with other alpha mask fixes.

--HG--
branch : avatar-pipeline

5 files changed, 96 insertions, 26 deletions

diff --git a/indra/newview/lltexlayer.cpp b/indra/newview/lltexlayer.cpp
index 25e0ca46e4..a90f3ee181 100644
--- a/indra/newview/lltexlayer.cpp
+++ b/indra/newview/lltexlayer.cpp
@@ -223,7 +223,16 @@ BOOL LLTexLayerSetBuffer::render()
 		}
 		else
 		{
-			readBackAndUpload();
+			if (mTexLayerSet->isVisible())
+			{
+				readBackAndUpload();
+			}
+			else
+			{
+				mUploadPending = FALSE;
+				mNeedsUpload = FALSE;
+				mTexLayerSet->getAvatar()->setNewBakedTexture(mTexLayerSet->getBakedTexIndex(),IMG_INVISIBLE);
+			}
 		}
 	}
 
@@ -551,6 +560,7 @@ LLTexLayerSet::LLTexLayerSet(LLVOAvatarSelf* const avatar) :
 	mComposite( NULL ),
 	mAvatar( avatar ),
 	mUpdatesEnabled( FALSE ),
+	mIsVisible( TRUE ),
 	mInfo( NULL )
 {
 }
@@ -665,38 +675,54 @@ BOOL LLTexLayerSet::isLocalTextureDataFinal() const
 BOOL LLTexLayerSet::render( S32 x, S32 y, S32 width, S32 height )
 {
 	BOOL success = TRUE;
+	mIsVisible = TRUE;
 
-	LLGLSUIDefault gls_ui;
-	LLGLDepthTest gls_depth(GL_FALSE, GL_FALSE);
-	gGL.setColorMask(true, true);
-
-	// clear buffer area to ensure we don't pick up UI elements
+	if (mMaskLayerList.size() > 0)
 	{
-		gGL.flush();
-		LLGLDisable no_alpha(GL_ALPHA_TEST);
-		gGL.getTexUnit(0)->unbind(LLTexUnit::TT_TEXTURE);
-		gGL.color4f( 0.f, 0.f, 0.f, 1.f );
-		
-		gl_rect_2d_simple( width, height );
-		
-		gGL.flush();
+		for (layer_list_t::iterator iter = mMaskLayerList.begin(); iter != mMaskLayerList.end(); iter++)
+		{
+			LLTexLayerInterface* layer = *iter;
+			if (layer->isInvisibleAlphaMask())
+			{
+				mIsVisible = FALSE;
+			}
+		}
 	}
 
-	// composite color layers
-	for( layer_list_t::iterator iter = mLayerList.begin(); iter != mLayerList.end(); iter++ )
+	if (mIsVisible)
 	{
-		LLTexLayerInterface* layer = *iter;
-		if (layer->getRenderPass() == LLTexLayer::RP_COLOR)
+		LLGLSUIDefault gls_ui;
+		LLGLDepthTest gls_depth(GL_FALSE, GL_FALSE);
+		gGL.setColorMask(true, true);
+	
+		// clear buffer area to ensure we don't pick up UI elements
 		{
 			gGL.flush();
-			success &= layer->render(x, y, width, height);
+			LLGLDisable no_alpha(GL_ALPHA_TEST);
+			gGL.getTexUnit(0)->unbind(LLTexUnit::TT_TEXTURE);
+			gGL.color4f( 0.f, 0.f, 0.f, 1.f );
+			
+			gl_rect_2d_simple( width, height );
+			
 			gGL.flush();
 		}
-	}
 	
-	renderAlphaMaskTextures(x, y, width, height, false);
-
-	stop_glerror();
+		// composite color layers
+		for( layer_list_t::iterator iter = mLayerList.begin(); iter != mLayerList.end(); iter++ )
+		{
+			LLTexLayerInterface* layer = *iter;
+			if (layer->getRenderPass() == LLTexLayer::RP_COLOR)
+			{
+				gGL.flush();
+				success &= layer->render(x, y, width, height);
+				gGL.flush();
+			}
+		}
+		
+		renderAlphaMaskTextures(x, y, width, height, false);
+	
+		stop_glerror();
+	}
 
 	return success;
 }
@@ -1709,6 +1735,19 @@ void LLTexLayer::addAlphaMask(U8 *data, S32 originX, S32 originY, S32 width, S32
 	}
 }
 
+/*virtual*/ BOOL LLTexLayer::isInvisibleAlphaMask()
+{
+	if (mLocalTextureObject)
+	{
+		if (mLocalTextureObject->getID() == IMG_INVISIBLE)
+		{
+			return TRUE;
+		}
+	}
+
+	return FALSE;
+}
+
 // private helper function
 LLUUID LLTexLayer::getUUID()
 {
@@ -1898,6 +1937,23 @@ LLTexLayer* LLTexLayerTemplate::getLayer(U32 i)
 	}
 }
 
+/*virtual*/ BOOL LLTexLayerTemplate::isInvisibleAlphaMask()
+{
+	U32 num_wearables = updateWearableCache();
+	for (U32 i = 0; i < num_wearables; i++)
+	{
+		LLTexLayer *layer = getLayer(i);
+		if (layer)
+		{
+			 if (layer->isInvisibleAlphaMask())
+			 {
+				 return TRUE;
+			 }
+		}
+	}
+
+	return FALSE;
+}
 
 
 //-----------------------------------------------------------------------------
diff --git a/indra/newview/lltexlayer.h b/indra/newview/lltexlayer.h
index cd8f27a96b..5be58f64a9 100644
--- a/indra/newview/lltexlayer.h
+++ b/indra/newview/lltexlayer.h
@@ -99,6 +99,7 @@ public:
 	virtual void			gatherAlphaMasks(U8 *data, S32 originX, S32 originY, S32 width, S32 height) = 0;
 	BOOL					hasAlphaParams() const { return !mParamAlphaList.empty(); }
 	BOOL					isVisibilityMask() const;
+	virtual BOOL			isInvisibleAlphaMask() = 0;
 
 	LLTexLayerSet*			getLayerSet() {return mTexLayerSet;}
 
@@ -141,6 +142,8 @@ public:
 	/*virtual*/ void		gatherAlphaMasks(U8 *data, S32 originX, S32 originY, S32 width, S32 height);
 	/*virtual*/ void		setHasMorph(BOOL newval);
 	/*virtual*/ void		deleteCaches();
+	/*virtual*/ BOOL		isInvisibleAlphaMask();
+
 private:
 	U32 	updateWearableCache();
 	LLTexLayer* getLayer(U32 i);
@@ -173,6 +176,7 @@ public:
 	/*virtual*/ void		gatherAlphaMasks(U8 *data, S32 originX, S32 originY, S32 width, S32 height);
 	BOOL					renderMorphMasks(S32 x, S32 y, S32 width, S32 height, const LLColor4 &layer_color);
 	void					addAlphaMask(U8 *data, S32 originX, S32 originY, S32 width, S32 height);
+	/*virtual*/ BOOL		isInvisibleAlphaMask();
 
 	void					setLTO(LLLocalTextureObject *lto) { mLocalTextureObject = lto; }
 	LLLocalTextureObject* 	getLTO() { return mLocalTextureObject; }
@@ -272,6 +276,7 @@ public:
 	BOOL					hasComposite() const { return (mComposite.notNull()); }
 	LLVOAvatarDefines::EBakedTextureIndex getBakedTexIndex() { return mBakedTexIndex; }
 	void					setBakedTexIndex( LLVOAvatarDefines::EBakedTextureIndex index) { mBakedTexIndex = index; }
+	BOOL					isVisible() const { return mIsVisible; }
 
 public:
 	static BOOL		sHasCaches;
@@ -284,6 +289,7 @@ private:
 	LLPointer<LLTexLayerSetBuffer>	mComposite;
 	LLVOAvatarSelf*	const	mAvatar; // Backlink only; don't make this an LLPointer.
 	BOOL					mUpdatesEnabled;
+	BOOL					mIsVisible;
 
 	LLVOAvatarDefines::EBakedTextureIndex mBakedTexIndex;
 
diff --git a/indra/newview/llvoavatar.cpp b/indra/newview/llvoavatar.cpp
index 9e75d7853d..3bf8038a66 100644
--- a/indra/newview/llvoavatar.cpp
+++ b/indra/newview/llvoavatar.cpp
@@ -3832,7 +3832,7 @@ U32 LLVOAvatar::renderTransparent(BOOL first_pass)
 		}
 		// Can't test for baked hair being defined, since that won't always be the case (not all viewers send baked hair)
 		// TODO: 1.25 will be able to switch this logic back to calling isTextureVisible();
-		if (getImage(TEX_HAIR_BAKED, 0)->getID() != IMG_INVISIBLE || LLDrawPoolAlpha::sShowDebugAlpha)
+		if ((getImage(TEX_HAIR_ALPHA, 0)->getID() != IMG_INVISIBLE && getImage(TEX_HAIR_BAKED, 0)->getID() != IMG_INVISIBLE) || LLDrawPoolAlpha::sShowDebugAlpha)
 		{
 			num_indices += mMeshLOD[MESH_ID_HAIR]->render(mAdjustedPixelArea, first_pass, mIsDummy);
 			first_pass = FALSE;
@@ -3868,7 +3868,7 @@ U32 LLVOAvatar::renderRigid()
 		return 0;
 	}
 
-	if (isTextureVisible(TEX_EYES_BAKED) || mIsDummy)
+	if ((isTextureVisible(TEX_EYES_ALPHA) && isTextureVisible(TEX_EYES_BAKED))  || mIsDummy)
 	{
 		num_indices += mMeshLOD[MESH_ID_EYEBALL_LEFT]->render(mAdjustedPixelArea, TRUE, mIsDummy);
 		num_indices += mMeshLOD[MESH_ID_EYEBALL_RIGHT]->render(mAdjustedPixelArea, TRUE, mIsDummy);
@@ -3898,7 +3898,7 @@ U32 LLVOAvatar::renderFootShadows()
 	
 	// Don't render foot shadows if your lower body is completely invisible.
 	// (non-humanoid avatars rule!)
-	if (!isTextureVisible(TEX_LOWER_BAKED))
+	if (!isTextureVisible(TEX_LOWER_BAKED) || ! isTextureVisible(TEX_LOWER_ALPHA))
 	{
 		return 0;
 	}
diff --git a/indra/newview/llvoavatarself.cpp b/indra/newview/llvoavatarself.cpp
index 843cba7860..f3e787ae7e 100644
--- a/indra/newview/llvoavatarself.cpp
+++ b/indra/newview/llvoavatarself.cpp
@@ -1839,6 +1839,13 @@ ETextureIndex LLVOAvatarSelf::getBakedTE( const LLTexLayerSet* layerset ) const
 }
 
 
+void LLVOAvatarSelf::setNewBakedTexture(LLVOAvatarDefines::EBakedTextureIndex i, const LLUUID &uuid)
+{
+	ETextureIndex index = LLVOAvatarDictionary::bakedToLocalTextureIndex(i);
+	setNewBakedTexture(index, uuid);
+}
+
+
 //-----------------------------------------------------------------------------
 // setNewBakedTexture()
 // A new baked texture has been successfully uploaded and we can start using it now.
diff --git a/indra/newview/llvoavatarself.h b/indra/newview/llvoavatarself.h
index a1cad82eff..e376e5e9ef 100644
--- a/indra/newview/llvoavatarself.h
+++ b/indra/newview/llvoavatarself.h
@@ -209,6 +209,7 @@ private:
 	//--------------------------------------------------------------------
 public:
 	LLVOAvatarDefines::ETextureIndex getBakedTE(const LLTexLayerSet* layerset ) const;
+	void				setNewBakedTexture(LLVOAvatarDefines::EBakedTextureIndex i, const LLUUID &uuid);
 	void				setNewBakedTexture(LLVOAvatarDefines::ETextureIndex i, const LLUUID& uuid);
 	void				setCachedBakedTexture(LLVOAvatarDefines::ETextureIndex i, const LLUUID& uuid);
 	void				forceBakeAllTextures(bool slam_for_debug = false);