EXT-1823 avatar shape corrupted with huge legs/feet

As suspected a morph mask issue. Apparently we were setting success to false
if the user specifies a pant or shrit texture without an alpha channel.
Result: bad morph masks.

Patch begins to clean up the isMorphMaskValid issue, as well as fixing this
error.

Code reviewed by Vir

5 files changed, 86 insertions, 89 deletions

diff --git a/indra/newview/lltexlayer.cpp b/indra/newview/lltexlayer.cpp
index 5d9046ac90..73340cbc03 100644
--- a/indra/newview/lltexlayer.cpp
+++ b/indra/newview/lltexlayer.cpp
@@ -670,8 +670,6 @@ BOOL LLTexLayerSet::render( S32 x, S32 y, S32 width, S32 height )
 	LLGLDepthTest gls_depth(GL_FALSE, GL_FALSE);
 	gGL.setColorMask(true, true);
 
-	BOOL render_morph = mAvatar->morphMaskNeedsUpdate(mBakedTexIndex);
-
 	// clear buffer area to ensure we don't pick up UI elements
 	{
 		gGL.flush();
@@ -691,12 +689,8 @@ BOOL LLTexLayerSet::render( S32 x, S32 y, S32 width, S32 height )
 		if (layer->getRenderPass() == LLTexLayer::RP_COLOR)
 		{
 			gGL.flush();
-			success &= layer->render(x, y, width, height, render_morph);
+			success &= layer->render(x, y, width, height);
 			gGL.flush();
-			if (layer->isMorphValid())
-			{
-				mAvatar->setMorphMasksValid(TRUE, mBakedTexIndex);
-			}
 		}
 	}
 	
@@ -786,12 +780,10 @@ void LLTexLayerSet::gatherMorphMaskAlpha(U8 *data, S32 width, S32 height)
 {
 	memset(data, 255, width * height);
 
-	BOOL render_morph = mAvatar->morphMaskNeedsUpdate(mBakedTexIndex);
-
 	for( layer_list_t::iterator iter = mLayerList.begin(); iter != mLayerList.end(); iter++ )
 	{
 		LLTexLayerInterface* layer = *iter;
-		layer->gatherAlphaMasks(data, mComposite->getOriginX(),mComposite->getOriginY(), width, height, render_morph);
+		layer->gatherAlphaMasks(data, mComposite->getOriginX(),mComposite->getOriginY(), width, height);
 	}
 	
 	// Set alpha back to that of our alpha masks.
@@ -863,6 +855,31 @@ void LLTexLayerSet::applyMorphMask(U8* tex_data, S32 width, S32 height, S32 num_
 	mAvatar->applyMorphMask(tex_data, width, height, num_components, mBakedTexIndex);
 }
 
+BOOL LLTexLayerSet::isMorphValid()
+{
+	for( layer_list_t::iterator iter = mLayerList.begin(); iter != mLayerList.end(); iter++ )
+	{
+		LLTexLayerInterface* layer = *iter;
+		if (layer && !layer->isMorphValid())
+		{
+			return FALSE;
+		}
+	}
+	return TRUE;
+}
+
+void LLTexLayerSet::invalidateMorphMasks()
+{
+	for( layer_list_t::iterator iter = mLayerList.begin(); iter != mLayerList.end(); iter++ )
+	{
+		LLTexLayerInterface* layer = *iter;
+		if (layer)
+		{
+			layer->invalidateMorphMasks();
+		}
+	}
+}
+
 
 //-----------------------------------------------------------------------------
 // LLTexLayerInfo
@@ -1282,7 +1299,7 @@ void LLTexLayer::calculateTexLayerColor(const param_color_list_t &param_list, LL
 	}
 }
 
-BOOL LLTexLayer::render(S32 x, S32 y, S32 width, S32 height, BOOL render_morph)
+BOOL LLTexLayer::render(S32 x, S32 y, S32 width, S32 height)
 {
 	LLGLEnable color_mat(GL_COLOR_MATERIAL);
 	gPipeline.disableLights();
@@ -1333,7 +1350,7 @@ BOOL LLTexLayer::render(S32 x, S32 y, S32 width, S32 height, BOOL render_morph)
 			}
 		}//*/
 
-		renderMorphMasks(x, y, width, height, net_color, render_morph);
+		renderMorphMasks(x, y, width, height, net_color);
 		alpha_mask_specified = TRUE;
 		gGL.flush();
 		gGL.blendFunc(LLRender::BF_DEST_ALPHA, LLRender::BF_ONE_MINUS_DEST_ALPHA);
@@ -1534,12 +1551,12 @@ BOOL LLTexLayer::blendAlphaTexture(S32 x, S32 y, S32 width, S32 height)
 	return success;
 }
 
-/*virtual*/ void LLTexLayer::gatherAlphaMasks(U8 *data, S32 originX, S32 originY, S32 width, S32 height, BOOL render_morph)
+/*virtual*/ void LLTexLayer::gatherAlphaMasks(U8 *data, S32 originX, S32 originY, S32 width, S32 height)
 {
-	addAlphaMask(data, originX, originY, width, height, render_morph);
+	addAlphaMask(data, originX, originY, width, height);
 }
 
-BOOL LLTexLayer::renderMorphMasks(S32 x, S32 y, S32 width, S32 height, const LLColor4 &layer_color, BOOL render_morph)
+BOOL LLTexLayer::renderMorphMasks(S32 x, S32 y, S32 width, S32 height, const LLColor4 &layer_color)
 {
 	BOOL success = TRUE;
 
@@ -1578,46 +1595,38 @@ BOOL LLTexLayer::renderMorphMasks(S32 x, S32 y, S32 width, S32 height, const LLC
 	// Accumulate the alpha component of the texture
 	if( getInfo()->mLocalTexture != -1 )
 	{
-			LLViewerTexture* tex = mLocalTextureObject->getImage();
-			if( tex && (tex->getComponents() == 4) )
-			{
-				LLGLSNoAlphaTest gls_no_alpha_test;
+		LLViewerTexture* tex = mLocalTextureObject->getImage();
+		if( tex && (tex->getComponents() == 4) )
+		{
+			LLGLSNoAlphaTest gls_no_alpha_test;
 
-				LLTexUnit::eTextureAddressMode old_mode = tex->getAddressMode();
-				
-				gGL.getTexUnit(0)->bind(tex);
-				gGL.getTexUnit(0)->setTextureAddressMode(LLTexUnit::TAM_CLAMP);
+			LLTexUnit::eTextureAddressMode old_mode = tex->getAddressMode();
+			
+			gGL.getTexUnit(0)->bind(tex);
+			gGL.getTexUnit(0)->setTextureAddressMode(LLTexUnit::TAM_CLAMP);
 
-				gl_rect_2d_simple_tex( width, height );
+			gl_rect_2d_simple_tex( width, height );
 
-				gGL.getTexUnit(0)->setTextureAddressMode(old_mode);
-				gGL.getTexUnit(0)->unbind(LLTexUnit::TT_TEXTURE);
-			}
-			else
-			{
-				success = FALSE;
-			}
+			gGL.getTexUnit(0)->setTextureAddressMode(old_mode);
+			gGL.getTexUnit(0)->unbind(LLTexUnit::TT_TEXTURE);
 		}
+	}
 
 	if( !getInfo()->mStaticImageFileName.empty() )
 	{
-			LLViewerTexture* tex = LLTexLayerStaticImageList::getInstance()->getTexture(getInfo()->mStaticImageFileName, getInfo()->mStaticImageIsMask);
-			if( tex )
-			{
-				if(	(tex->getComponents() == 4) ||
-					( (tex->getComponents() == 1) && getInfo()->mStaticImageIsMask ) )
-				{
-					LLGLSNoAlphaTest gls_no_alpha_test;
-					gGL.getTexUnit(0)->bind(tex);
-					gl_rect_2d_simple_tex( width, height );
-					gGL.getTexUnit(0)->unbind(LLTexUnit::TT_TEXTURE);
-				}
-			}
-			else
+		LLViewerTexture* tex = LLTexLayerStaticImageList::getInstance()->getTexture(getInfo()->mStaticImageFileName, getInfo()->mStaticImageIsMask);
+		if( tex )
+		{
+			if(	(tex->getComponents() == 4) ||
+				( (tex->getComponents() == 1) && getInfo()->mStaticImageIsMask ) )
 			{
-				success = FALSE;
+				LLGLSNoAlphaTest gls_no_alpha_test;
+				gGL.getTexUnit(0)->bind(tex);
+				gl_rect_2d_simple_tex( width, height );
+				gGL.getTexUnit(0)->unbind(LLTexUnit::TT_TEXTURE);
 			}
 		}
+	}
 
 	// Draw a rectangle with the layer color to multiply the alpha by that color's alpha.
 	// Note: we're still using gGL.blendFunc( GL_DST_ALPHA, GL_ZERO );
@@ -1634,7 +1643,7 @@ BOOL LLTexLayer::renderMorphMasks(S32 x, S32 y, S32 width, S32 height, const LLC
 
 	gGL.setColorMask(true, true);
 	
-	if (render_morph && mHasMorph && success)
+	if (hasMorph() && success)
 	{
 		LLCRC alpha_mask_crc;
 		const LLUUID& uuid = getUUID();
@@ -1674,7 +1683,7 @@ BOOL LLTexLayer::renderMorphMasks(S32 x, S32 y, S32 width, S32 height, const LLC
 	return success;
 }
 
-void LLTexLayer::addAlphaMask(U8 *data, S32 originX, S32 originY, S32 width, S32 height, BOOL render_morph)
+void LLTexLayer::addAlphaMask(U8 *data, S32 originX, S32 originY, S32 width, S32 height)
 {
 	S32 size = width * height;
 	U8* alphaData = getAlphaData();
@@ -1684,7 +1693,7 @@ void LLTexLayer::addAlphaMask(U8 *data, S32 originX, S32 originY, S32 width, S32
 		findNetColor( &net_color );
 		// TODO: eliminate need for layer morph mask valid flag
 		invalidateMorphMasks();
-		renderMorphMasks(originX, originY, width, height, net_color, render_morph);
+		renderMorphMasks(originX, originY, width, height, net_color);
 		alphaData = getAlphaData();
 	}
 	if (alphaData)
@@ -1805,7 +1814,7 @@ LLTexLayer* LLTexLayerTemplate::getLayer(U32 i)
 	return layer;
 }
 
-/*virtual*/ BOOL LLTexLayerTemplate::render(S32 x, S32 y, S32 width, S32 height, BOOL render_morph)
+/*virtual*/ BOOL LLTexLayerTemplate::render(S32 x, S32 y, S32 width, S32 height)
 {
 	BOOL success = TRUE;
 	updateWearableCache();
@@ -1827,7 +1836,7 @@ LLTexLayer* LLTexLayerTemplate::getLayer(U32 i)
 		{
 			wearable->writeToAvatar(FALSE, FALSE);
 			layer->setLTO(lto);
-			success &= layer->render(x,y,width,height,render_morph);
+			success &= layer->render(x,y,width,height);
 		}
 	}
 
@@ -1849,7 +1858,7 @@ LLTexLayer* LLTexLayerTemplate::getLayer(U32 i)
 	return success;
 }
 
-/*virtual*/ void LLTexLayerTemplate::gatherAlphaMasks(U8 *data, S32 originX, S32 originY, S32 width, S32 height, BOOL render_morph)
+/*virtual*/ void LLTexLayerTemplate::gatherAlphaMasks(U8 *data, S32 originX, S32 originY, S32 width, S32 height)
 {
 	U32 num_wearables = updateWearableCache();
 	for (U32 i = 0; i < num_wearables; i++)
@@ -1857,7 +1866,7 @@ LLTexLayer* LLTexLayerTemplate::getLayer(U32 i)
 		LLTexLayer *layer = getLayer(i);
 		if (layer)
 		{
-			layer->addAlphaMask(data, originX, originY, width, height, render_morph);
+			layer->addAlphaMask(data, originX, originY, width, height);
 		}
 	}
 }
diff --git a/indra/newview/lltexlayer.h b/indra/newview/lltexlayer.h
index e4a6e82ba5..cd8f27a96b 100644
--- a/indra/newview/lltexlayer.h
+++ b/indra/newview/lltexlayer.h
@@ -81,13 +81,14 @@ public:
 
 	const LLTexLayerInfo* 	getInfo() const { return mInfo; }
 	virtual BOOL			setInfo(const LLTexLayerInfo *info, LLWearable* wearable ); // This sets mInfo and calls initialization functions
-	virtual BOOL			render(S32 x, S32 y, S32 width, S32 height, BOOL render_morph) = 0;
+	virtual BOOL			render(S32 x, S32 y, S32 width, S32 height) = 0;
 	void					requestUpdate();
 	LLTexLayerSet*			const getTexLayerSet() const { return mTexLayerSet; }
 
 	virtual void			deleteCaches() = 0;
 	void					invalidateMorphMasks();
 	virtual void			setHasMorph(BOOL newval) { mHasMorph = newval; }
+	BOOL					hasMorph()				 { return mHasMorph; }
 	BOOL					isMorphValid()			 { return mMorphMasksValid; }
 
 	const std::string&		getName() const;
@@ -95,7 +96,7 @@ public:
 	const std::string&		getGlobalColor() const;
 
 	virtual BOOL			blendAlphaTexture( S32 x, S32 y, S32 width, S32 height) = 0;
-	virtual void			gatherAlphaMasks(U8 *data, S32 originX, S32 originY, S32 width, S32 height, BOOL render_morph) = 0;
+	virtual void			gatherAlphaMasks(U8 *data, S32 originX, S32 originY, S32 width, S32 height) = 0;
 	BOOL					hasAlphaParams() const { return !mParamAlphaList.empty(); }
 	BOOL					isVisibilityMask() const;
 
@@ -134,10 +135,10 @@ public:
 	LLTexLayerTemplate(const LLTexLayerTemplate &layer);
 	/*virtual*/ ~LLTexLayerTemplate();
 
-	/*virtual*/ BOOL		render(S32 x, S32 y, S32 width, S32 height, BOOL render_morph);
+	/*virtual*/ BOOL		render(S32 x, S32 y, S32 width, S32 height);
 	/*virtual*/ BOOL		setInfo(const LLTexLayerInfo *info, LLWearable* wearable ); // This sets mInfo and calls initialization functions
 	/*virtual*/ BOOL		blendAlphaTexture( S32 x, S32 y, S32 width, S32 height); // Multiplies a single alpha texture against the frame buffer
-	/*virtual*/ void		gatherAlphaMasks(U8 *data, S32 originX, S32 originY, S32 width, S32 height, BOOL render_morph);
+	/*virtual*/ void		gatherAlphaMasks(U8 *data, S32 originX, S32 originY, S32 width, S32 height);
 	/*virtual*/ void		setHasMorph(BOOL newval);
 	/*virtual*/ void		deleteCaches();
 private:
@@ -162,16 +163,16 @@ public:
 	/*virtual*/ ~LLTexLayer();
 
 	/*virtual*/ BOOL		setInfo(const LLTexLayerInfo *info, LLWearable* wearable ); // This sets mInfo and calls initialization functions
-	/*virtual*/ BOOL		render(S32 x, S32 y, S32 width, S32 height, BOOL render_morph);
+	/*virtual*/ BOOL		render(S32 x, S32 y, S32 width, S32 height);
 
 	/*virtual*/ void		deleteCaches();
 	U8*						getAlphaData();
 
 	BOOL					findNetColor(LLColor4* color) const;
 	/*virtual*/ BOOL		blendAlphaTexture( S32 x, S32 y, S32 width, S32 height); // Multiplies a single alpha texture against the frame buffer
-	/*virtual*/ void		gatherAlphaMasks(U8 *data, S32 originX, S32 originY, S32 width, S32 height, BOOL render_morph);
-	BOOL					renderMorphMasks(S32 x, S32 y, S32 width, S32 height, const LLColor4 &layer_color, BOOL render_morph);
-	void					addAlphaMask(U8 *data, S32 originX, S32 originY, S32 width, S32 height, BOOL render_morph);
+	/*virtual*/ void		gatherAlphaMasks(U8 *data, S32 originX, S32 originY, S32 width, S32 height);
+	BOOL					renderMorphMasks(S32 x, S32 y, S32 width, S32 height, const LLColor4 &layer_color);
+	void					addAlphaMask(U8 *data, S32 originX, S32 originY, S32 width, S32 height);
 
 	void					setLTO(LLLocalTextureObject *lto) { mLocalTextureObject = lto; }
 	LLLocalTextureObject* 	getLTO() { return mLocalTextureObject; }
@@ -261,6 +262,8 @@ public:
 	void					deleteCaches();
 	void					gatherMorphMaskAlpha(U8 *data, S32 width, S32 height);
 	void					applyMorphMask(U8* tex_data, S32 width, S32 height, S32 num_components);
+	BOOL					isMorphValid();
+	void					invalidateMorphMasks();
 	LLTexLayerInterface*	findLayerByName(const std::string& name);
 	void					cloneTemplates(LLLocalTextureObject *lto, LLVOAvatarDefines::ETextureIndex tex_index, LLWearable* wearable);
 	
diff --git a/indra/newview/llvoavatar.cpp b/indra/newview/llvoavatar.cpp
index 60ab6f8075..acf1222455 100644
--- a/indra/newview/llvoavatar.cpp
+++ b/indra/newview/llvoavatar.cpp
@@ -689,7 +689,6 @@ LLVOAvatar::LLVOAvatar(const LLUUID& id,
 		mBakedTextureDatas[i].mIsUsed = false;
 		mBakedTextureDatas[i].mMaskTexName = 0;
 		mBakedTextureDatas[i].mTextureIndex = LLVOAvatarDictionary::bakedToLocalTextureIndex((EBakedTextureIndex)i);
-		mBakedTextureDatas[i].mMorphMasksValid = FALSE;
 	}
 
 	mDirtyMesh = TRUE;	// Dirty geometry, need to regenerate.
@@ -6091,28 +6090,6 @@ void LLVOAvatar::addMaskedMorph(EBakedTextureIndex index, LLPolyMorphTarget* mor
 	}
 }
 
-// invalidates morph masks for a given layer. Don't pass a parameter to invalidate all morph masks.
-void LLVOAvatar::invalidateMorphMasks(LLVOAvatarDefines::EBakedTextureIndex index)
-{
-	setMorphMasksValid(FALSE, index);
-}
-
-// updates morph masks to be a value for a given layer. Don't pass an argument to set value for all morph masks
-void LLVOAvatar::setMorphMasksValid(BOOL new_status, LLVOAvatarDefines::EBakedTextureIndex index)
-{
-	if (index == BAKED_NUM_INDICES)
-	{
-		for (U8 tex = 0; tex < (U8)BAKED_NUM_INDICES; tex++)
-		{
-			mBakedTextureDatas[tex].mMorphMasksValid = new_status;
-		}
-	} 
-	else if (index < BAKED_NUM_INDICES) 
-	{
-		mBakedTextureDatas[index].mMorphMasksValid = new_status;
-	}
-}
-
 // returns TRUE if morph masks are present and not valid for a given baked texture, FALSE otherwise
 BOOL LLVOAvatar::morphMaskNeedsUpdate(LLVOAvatarDefines::EBakedTextureIndex index)
 {
@@ -6121,9 +6098,20 @@ BOOL LLVOAvatar::morphMaskNeedsUpdate(LLVOAvatarDefines::EBakedTextureIndex inde
 		return FALSE;
 	}
 
-	if (!mBakedTextureDatas[index].mMaskedMorphs.empty() && !mBakedTextureDatas[index].mMorphMasksValid)
+	if (!mBakedTextureDatas[index].mMaskedMorphs.empty())
 	{
-		return TRUE;
+		if (isSelf())
+		{
+			LLTexLayerSet *layer_set = mBakedTextureDatas[index].mTexLayerSet;
+			if (layer_set)
+			{
+				return !layer_set->isMorphValid();
+			}
+		}
+		else
+		{
+			return FALSE;
+		}
 	}
 
 	return FALSE;
diff --git a/indra/newview/llvoavatar.h b/indra/newview/llvoavatar.h
index f7c794defe..cf86612ce6 100644
--- a/indra/newview/llvoavatar.h
+++ b/indra/newview/llvoavatar.h
@@ -355,10 +355,8 @@ private:
 	// Morph masks
 	//--------------------------------------------------------------------
 public:
-	void 		invalidateMorphMasks(LLVOAvatarDefines::EBakedTextureIndex index = LLVOAvatarDefines::BAKED_NUM_INDICES);
 	BOOL 		morphMaskNeedsUpdate(LLVOAvatarDefines::EBakedTextureIndex index = LLVOAvatarDefines::BAKED_NUM_INDICES);
 	void 		addMaskedMorph(LLVOAvatarDefines::EBakedTextureIndex index, LLPolyMorphTarget* morph_target, BOOL invert, std::string layer);
-	void 		setMorphMasksValid(BOOL new_status, LLVOAvatarDefines::EBakedTextureIndex index = LLVOAvatarDefines::BAKED_NUM_INDICES);
 	void 		applyMorphMask(U8* tex_data, S32 width, S32 height, S32 num_components, LLVOAvatarDefines::EBakedTextureIndex index = LLVOAvatarDefines::BAKED_NUM_INDICES);
 
 	//--------------------------------------------------------------------
@@ -489,7 +487,6 @@ protected:
 		// Stores pointers to the joint meshes that this baked texture deals with
 		std::vector< LLViewerJointMesh * > 	mMeshes;  // std::vector<LLViewerJointMesh> mJoints[i]->mMeshParts
 		morph_list_t						mMaskedMorphs;
-		BOOL								mMorphMasksValid;
 	};
 	typedef std::vector<BakedTextureData> 	bakedtexturedata_vec_t;
 	bakedtexturedata_vec_t 					mBakedTextureDatas;
diff --git a/indra/newview/llvoavatarself.cpp b/indra/newview/llvoavatarself.cpp
index 85a68fdd3f..6c3348b247 100644
--- a/indra/newview/llvoavatarself.cpp
+++ b/indra/newview/llvoavatarself.cpp
@@ -1387,8 +1387,8 @@ void LLVOAvatarSelf::invalidateComposite( LLTexLayerSet* layerset, BOOL set_by_u
 	}
 	// llinfos << "LLVOAvatar::invalidComposite() " << layerset->getBodyRegion() << llendl;
 
-	invalidateMorphMasks(layerset->getBakedTexIndex());
 	layerset->requestUpdate();
+	layerset->invalidateMorphMasks();
 
 	if( set_by_user )
 	{