SL-12069 Issues with bakes-on-mesh and alpha modes

4 files changed, 94 insertions, 211 deletions

diff --git a/indra/newview/lldrawpoolavatar.cpp b/indra/newview/lldrawpoolavatar.cpp
index 789a254389..edb896a1d2 100644
--- a/indra/newview/lldrawpoolavatar.cpp
+++ b/indra/newview/lldrawpoolavatar.cpp
@@ -2125,7 +2125,7 @@ void LLDrawPoolAvatar::renderRigged(LLVOAvatar* avatar, U32 type, bool glow)
 
             if (mat)
             {                
-                switch (LLMaterial::eDiffuseAlphaMode(mat->getDiffuseAlphaMode()))
+                switch (LLMaterial::eDiffuseAlphaMode(mat->getDiffuseAlphaModeRender()))
                 {
                     case LLMaterial::DIFFUSE_ALPHA_MODE_MASK:
                     {
@@ -2263,7 +2263,7 @@ void LLDrawPoolAvatar::renderRigged(LLVOAvatar* avatar, U32 type, bool glow)
 				sVertexProgram->uniform4f(LLShaderMgr::SPECULAR_COLOR, col.mV[0], col.mV[1], col.mV[2], spec);
 				sVertexProgram->uniform1f(LLShaderMgr::ENVIRONMENT_INTENSITY, env);
 
-				if (mat->getDiffuseAlphaMode() == LLMaterial::DIFFUSE_ALPHA_MODE_MASK)
+				if (mat->getDiffuseAlphaModeRender() == LLMaterial::DIFFUSE_ALPHA_MODE_MASK)
 				{
                     F32 cutoff = mat->getAlphaMaskCutoff()/255.f;
 					sVertexProgram->setMinimumAlpha(cutoff);
diff --git a/indra/newview/llface.cpp b/indra/newview/llface.cpp
index 18ea184da6..651c49e699 100644
--- a/indra/newview/llface.cpp
+++ b/indra/newview/llface.cpp
@@ -1165,7 +1165,7 @@ bool LLFace::canRenderAsMask()
 	}
 	
 	LLMaterial* mat = te->getMaterialParams();
-	if (mat && mat->getDiffuseAlphaMode() == LLMaterial::DIFFUSE_ALPHA_MODE_BLEND)
+	if (mat && mat->getDiffuseAlphaModeRender() == LLMaterial::DIFFUSE_ALPHA_MODE_BLEND)
 	{
 		return false;
 	}
@@ -1412,7 +1412,7 @@ BOOL LLFace::getGeometryVolume(const LLVolume& volume,
 			}
 			else
 			{
-				if (!mat || mat->getDiffuseAlphaMode() != LLMaterial::DIFFUSE_ALPHA_MODE_MASK)
+				if (!mat || mat->getDiffuseAlphaModeRender() != LLMaterial::DIFFUSE_ALPHA_MODE_MASK)
 				{
 					shiny_in_alpha = true;
 				}
diff --git a/indra/newview/llvovolume.cpp b/indra/newview/llvovolume.cpp
index 34b801a097..819933fa72 100644
--- a/indra/newview/llvovolume.cpp
+++ b/indra/newview/llvovolume.cpp
@@ -2283,51 +2283,34 @@ bool LLVOVolume::notifyAboutCreatingTexture(LLViewerTexture *texture)
 
 	std::pair<mmap_UUID_MAP_t::iterator, mmap_UUID_MAP_t::iterator> range = mWaitingTextureInfo.equal_range(texture->getID());
 
-	typedef std::map<U8, LLMaterialPtr> map_te_material;
-	map_te_material new_material;
+    bool refresh_materials = false;
 
-	for(mmap_UUID_MAP_t::iterator range_it = range.first; range_it != range.second; ++range_it)
-	{
-		LLMaterialPtr cur_material = getTEMaterialParams(range_it->second.te);
-
-		//here we just interesting in DIFFUSE_MAP only!
-		if(NULL != cur_material.get() && LLRender::DIFFUSE_MAP == range_it->second.map && GL_RGBA != texture->getPrimaryFormat())
-		{ //ok let's check the diffuse mode
-			switch(cur_material->getDiffuseAlphaMode())
-			{
-			case LLMaterial::DIFFUSE_ALPHA_MODE_BLEND:
-			case LLMaterial::DIFFUSE_ALPHA_MODE_EMISSIVE:
-			case LLMaterial::DIFFUSE_ALPHA_MODE_MASK:
-				{ //uups... we have non 32 bit texture with LLMaterial::DIFFUSE_ALPHA_MODE_* => LLMaterial::DIFFUSE_ALPHA_MODE_NONE
-
-					LLMaterialPtr mat = NULL;
-					map_te_material::iterator it = new_material.find(range_it->second.te);
-					if(new_material.end() == it) {
-						mat = new LLMaterial(cur_material->asLLSD());
-						new_material.insert(map_te_material::value_type(range_it->second.te, mat));
-					} else {
-						mat = it->second;
-					}
-
-					mat->setDiffuseAlphaMode(LLMaterial::DIFFUSE_ALPHA_MODE_NONE);
-
-				} break;
-			} //switch
-		} //if
-	} //for
+    // RGB textures without alpha channels won't work right with alpha,
+    // we provide format to material for material to decide when to drop alpha
+    for (mmap_UUID_MAP_t::iterator range_it = range.first; range_it != range.second; ++range_it)
+    {
+        LLMaterialPtr cur_material = getTEMaterialParams(range_it->second.te);
+        if (cur_material.notNull()
+            && LLRender::DIFFUSE_MAP == range_it->second.map)
+        {
+            U32 format = texture->getPrimaryFormat();
+            if (format != cur_material->getDiffuseFormatPrimary())
+            {
+                cur_material->setDiffuseFormatPrimary(format);
+                refresh_materials = true;
+            }
+        }
+    } //for
 
-	//setup new materials
-	for(map_te_material::const_iterator it = new_material.begin(), end = new_material.end(); it != end; ++it)
-	{
-		// These are placeholder materials, they shouldn't be sent to server
-		LLMaterialMgr::getInstance()->setLocalMaterial(getRegion()->getRegionID(), it->second);
-		LLViewerObject::setTEMaterialParams(it->first, it->second);
-	}
+    if (refresh_materials)
+    {
+        LLViewerObject::refreshMaterials();
+    }
 
 	//clear wait-list
-	mWaitingTextureInfo.erase(range.first, range.second);
+    mWaitingTextureInfo.erase(range.first, range.second);
 
-	return 0 != new_material.size();
+	return refresh_materials;
 }
 
 bool LLVOVolume::notifyAboutMissingAsset(LLViewerTexture *texture)
@@ -2337,184 +2320,84 @@ bool LLVOVolume::notifyAboutMissingAsset(LLViewerTexture *texture)
 	std::pair<mmap_UUID_MAP_t::iterator, mmap_UUID_MAP_t::iterator> range = mWaitingTextureInfo.equal_range(texture->getID());
 	if(range.first == range.second) return false;
 
-	typedef std::map<U8, LLMaterialPtr> map_te_material;
-	map_te_material new_material;
-	
+    bool refresh_materials = false;
+
 	for(mmap_UUID_MAP_t::iterator range_it = range.first; range_it != range.second; ++range_it)
 	{
 		LLMaterialPtr cur_material = getTEMaterialParams(range_it->second.te);
 		if (cur_material.isNull())
 			continue;
 
-		switch(range_it->second.map)
-		{
-		case LLRender::DIFFUSE_MAP:
-			{
-				if(LLMaterial::DIFFUSE_ALPHA_MODE_NONE != cur_material->getDiffuseAlphaMode())
-				{ //missing texture + !LLMaterial::DIFFUSE_ALPHA_MODE_NONE => LLMaterial::DIFFUSE_ALPHA_MODE_NONE
-					LLMaterialPtr mat = NULL;
-					map_te_material::iterator it = new_material.find(range_it->second.te);
-					if(new_material.end() == it) {
-						mat = new LLMaterial(cur_material->asLLSD());
-						new_material.insert(map_te_material::value_type(range_it->second.te, mat));
-					} else {
-						mat = it->second;
-					}
-
-					mat->setDiffuseAlphaMode(LLMaterial::DIFFUSE_ALPHA_MODE_NONE);
-				}
-			} break;
-		case LLRender::NORMAL_MAP:
-			{ //missing texture => reset material texture id
-				LLMaterialPtr mat = NULL;
-				map_te_material::iterator it = new_material.find(range_it->second.te);
-				if(new_material.end() == it) {
-					mat = new LLMaterial(cur_material->asLLSD());
-					new_material.insert(map_te_material::value_type(range_it->second.te, mat));
-				} else {
-					mat = it->second;
-				}
-
-				mat->setNormalID(LLUUID::null);
-			} break;
-		case LLRender::SPECULAR_MAP:
-			{ //missing texture => reset material texture id
-				LLMaterialPtr mat = NULL;
-				map_te_material::iterator it = new_material.find(range_it->second.te);
-				if(new_material.end() == it) {
-					mat = new LLMaterial(cur_material->asLLSD());
-					new_material.insert(map_te_material::value_type(range_it->second.te, mat));
-				} else {
-					mat = it->second;
-				}
-
-				mat->setSpecularID(LLUUID::null);
-			} break;
-		case LLRender::NUM_TEXTURE_CHANNELS:
-				//nothing to do, make compiler happy
-			break;
-		} //switch
-	} //for
+        if (range_it->second.map == LLRender::DIFFUSE_MAP)
+        {
+            LLMaterialPtr cur_material = getTEMaterialParams(range_it->second.te);
+            if (cur_material.notNull()
+                && LLRender::DIFFUSE_MAP == range_it->second.map)
+            {
+                if (0 != cur_material->getDiffuseFormatPrimary())
+                {
+                    cur_material->setDiffuseFormatPrimary(0);
+                    refresh_materials = true;
+                }
+            }
+        }
+    } //for
 
-	//setup new materials
-	for(map_te_material::const_iterator it = new_material.begin(), end = new_material.end(); it != end; ++it)
-	{
-		LLMaterialMgr::getInstance()->setLocalMaterial(getRegion()->getRegionID(), it->second);
-		LLViewerObject::setTEMaterialParams(it->first, it->second);
-	}
+    if (refresh_materials)
+    {
+        LLViewerObject::refreshMaterials();
+    }
 
 	//clear wait-list
 	mWaitingTextureInfo.erase(range.first, range.second);
 
-	return 0 != new_material.size();
+    return refresh_materials;
 }
 
 S32 LLVOVolume::setTEMaterialParams(const U8 te, const LLMaterialPtr pMaterialParams)
 {
-	LLMaterialPtr pMaterial = const_cast<LLMaterialPtr&>(pMaterialParams);
-
-	if(pMaterialParams)
-	{ //check all of them according to material settings
-
-		LLViewerTexture *img_diffuse = getTEImage(te);
-		LLViewerTexture *img_normal = getTENormalMap(te);
-		LLViewerTexture *img_specular = getTESpecularMap(te);
-
-		llassert(NULL != img_diffuse);
-
-		LLMaterialPtr new_material = NULL;
-
-		//diffuse
-		if(NULL != img_diffuse)
-		{ //guard
-			if(0 == img_diffuse->getPrimaryFormat() && !img_diffuse->isMissingAsset())
-			{ //ok here we don't have information about texture, let's belief and leave material settings
-			  //but we remember this case
-				mWaitingTextureInfo.insert(mmap_UUID_MAP_t::value_type(img_diffuse->getID(), material_info(LLRender::DIFFUSE_MAP, te)));
-			}
-			else
-			{
-				bool bSetDiffuseNone = false;
-				if(img_diffuse->isMissingAsset())
-				{
-					bSetDiffuseNone = true;
-				}
-				else
-				{
-					switch(pMaterialParams->getDiffuseAlphaMode())
-					{
-					case LLMaterial::DIFFUSE_ALPHA_MODE_BLEND:
-					case LLMaterial::DIFFUSE_ALPHA_MODE_EMISSIVE:
-					case LLMaterial::DIFFUSE_ALPHA_MODE_MASK:
-						{ //all of them modes available only for 32 bit textures
-							LLTextureEntry* tex_entry = getTE(te);
-							bool bIsBakedImageId = false;
-							if (tex_entry && LLAvatarAppearanceDefines::LLAvatarAppearanceDictionary::isBakedImageId(tex_entry->getID()))
-							{
-								bIsBakedImageId = true;
-							}
-							if (GL_RGBA != img_diffuse->getPrimaryFormat() && !bIsBakedImageId)
-							{
-								bSetDiffuseNone = true;
-							}
-						} break;
-					}
-				} //else
-
-
-				if(bSetDiffuseNone)
-				{ //upps... we should substitute this material with LLMaterial::DIFFUSE_ALPHA_MODE_NONE
-					new_material = new LLMaterial(pMaterialParams->asLLSD());
-					new_material->setDiffuseAlphaMode(LLMaterial::DIFFUSE_ALPHA_MODE_NONE);
-				}
-			}
-		}
-
-		//normal
-		if(LLUUID::null != pMaterialParams->getNormalID())
-		{
-			if(img_normal && img_normal->isMissingAsset() && img_normal->getID() == pMaterialParams->getNormalID())
-			{
-				if(!new_material) {
-					new_material = new LLMaterial(pMaterialParams->asLLSD());
-				}
-				new_material->setNormalID(LLUUID::null);
-			}
-			else if(NULL == img_normal || 0 == img_normal->getPrimaryFormat())
-			{ //ok here we don't have information about texture, let's belief and leave material settings
-				//but we remember this case
-				mWaitingTextureInfo.insert(mmap_UUID_MAP_t::value_type(pMaterialParams->getNormalID(), material_info(LLRender::NORMAL_MAP,te)));
-			}
-
-		}
+    LLMaterialPtr material = pMaterialParams;
 
+    if (material.notNull())
+    {
+        LLTextureEntry* tex_entry = getTE(te);
+        bool is_baked = tex_entry && LLAvatarAppearanceDefines::LLAvatarAppearanceDictionary::isBakedImageId(tex_entry->getID());
+        material->setDiffuseBaked(is_baked);
 
-		//specular
-		if(LLUUID::null != pMaterialParams->getSpecularID())
-		{
-			if(img_specular && img_specular->isMissingAsset() && img_specular->getID() == pMaterialParams->getSpecularID())
-			{
-				if(!new_material) {
-					new_material = new LLMaterial(pMaterialParams->asLLSD());
-				}
-				new_material->setSpecularID(LLUUID::null);
-			}
-			else if(NULL == img_specular || 0 == img_specular->getPrimaryFormat())
-			{ //ok here we don't have information about texture, let's belief and leave material settings
-				//but we remember this case
-				mWaitingTextureInfo.insert(mmap_UUID_MAP_t::value_type(pMaterialParams->getSpecularID(), material_info(LLRender::SPECULAR_MAP, te)));
-			}
-		}
+        LLViewerTexture *img_diffuse = getTEImage(te);
+        llassert(NULL != img_diffuse);
 
-		if(new_material) {
-			pMaterial = new_material;
-			LLMaterialMgr::getInstance()->setLocalMaterial(getRegion()->getRegionID(), pMaterial);
-		}
+        //diffuse
+        if (!is_baked && NULL != img_diffuse)
+        {
+            if (0 == img_diffuse->getPrimaryFormat() && !img_diffuse->isMissingAsset())
+            {
+                // we don't have information about this texture, wait for it
+                mWaitingTextureInfo.insert(mmap_UUID_MAP_t::value_type(img_diffuse->getID(), material_info(LLRender::DIFFUSE_MAP, te)));
+                // Temporary assume RGBA image
+                material->setDiffuseFormatPrimary(GL_RGBA);
+            }
+            else
+            {
+                if (img_diffuse->isMissingAsset())
+                {
+                    material->setDiffuseFormatPrimary(0);
+                }
+                else
+                {
+                    material->setDiffuseFormatPrimary(img_diffuse->getPrimaryFormat());
+                }
+            }
+        }
+        else
+        {
+            material->setDiffuseFormatPrimary(0);
+        }
 	}
 
-	S32 res = LLViewerObject::setTEMaterialParams(te, pMaterial);
+    S32 res = LLViewerObject::setTEMaterialParams(te, material);
 
-	LL_DEBUGS("MaterialTEs") << "te " << (S32)te << " material " << ((pMaterial) ? pMaterial->asLLSD() : LLSD("null")) << " res " << res
+    LL_DEBUGS("MaterialTEs") << "te " << (S32)te << " material " << ((material) ? material->asLLSD() : LLSD("null")) << " res " << res
 							 << ( LLSelectMgr::getInstance()->getSelection()->contains(const_cast<LLVOVolume*>(this), te) ? " selected" : " not selected" )
 							 << LL_ENDL;
 	setChanged(ALL_CHANGED);
@@ -4581,7 +4464,7 @@ BOOL LLVOVolume::lineSegmentIntersect(const LLVector4a& start, const LLVector4a&
 					LLMaterial* mat = te->getMaterialParams();
 					if (mat)
 					{
-						U8 mode = mat->getDiffuseAlphaMode();
+						U8 mode = mat->getDiffuseAlphaModeRender();
 
 						if (mode == LLMaterial::DIFFUSE_ALPHA_MODE_EMISSIVE ||
 							mode == LLMaterial::DIFFUSE_ALPHA_MODE_NONE)
@@ -5204,7 +5087,7 @@ void LLVolumeGeometryManager::registerFace(LLSpatialGroup* group, LLFace* facep,
 			}
 
 			draw_info->mAlphaMaskCutoff = mat->getAlphaMaskCutoff() * (1.f / 255.f);
-			draw_info->mDiffuseAlphaMode = mat->getDiffuseAlphaMode();
+			draw_info->mDiffuseAlphaMode = mat->getDiffuseAlphaModeRender();
 			draw_info->mNormalMap = facep->getViewerObject()->getTENormalMap(facep->getTEOffset());
 		}
 		else 
@@ -5564,7 +5447,7 @@ void LLVolumeGeometryManager::rebuildGeom(LLSpatialGroup* group)
 
 						if (mat && LLPipeline::sRenderDeferred)
 						{
-							U8 alpha_mode = mat->getDiffuseAlphaMode();
+							U8 alpha_mode = mat->getDiffuseAlphaModeRender();
 
 							bool is_alpha = type == LLDrawPool::POOL_ALPHA &&
 								(alpha_mode == LLMaterial::DIFFUSE_ALPHA_MODE_BLEND ||
@@ -5593,7 +5476,7 @@ void LLVolumeGeometryManager::rebuildGeom(LLSpatialGroup* group)
 						else if (mat)
 						{							
 							bool is_alpha = type == LLDrawPool::POOL_ALPHA;
-							U8 mode = mat->getDiffuseAlphaMode();
+							U8 mode = mat->getDiffuseAlphaModeRender();
 							bool can_be_shiny = mode == LLMaterial::DIFFUSE_ALPHA_MODE_NONE ||
 												mode == LLMaterial::DIFFUSE_ALPHA_MODE_EMISSIVE;
 							
@@ -6491,7 +6374,7 @@ U32 LLVolumeGeometryManager::genDrawInfo(LLSpatialGroup* group, U32 mask, LLFace
 			bool can_be_shiny = true;
 			if (mat)
 			{
-				U8 mode = mat->getDiffuseAlphaMode();
+				U8 mode = mat->getDiffuseAlphaModeRender();
 				can_be_shiny = mode == LLMaterial::DIFFUSE_ALPHA_MODE_NONE ||
 								mode == LLMaterial::DIFFUSE_ALPHA_MODE_EMISSIVE;
 			}
@@ -6513,7 +6396,7 @@ U32 LLVolumeGeometryManager::genDrawInfo(LLSpatialGroup* group, U32 mask, LLFace
 				//
 				if (te->getFullbright())
 				{
-					if (mat->getDiffuseAlphaMode() == LLMaterial::DIFFUSE_ALPHA_MODE_MASK)
+					if (mat->getDiffuseAlphaModeRender() == LLMaterial::DIFFUSE_ALPHA_MODE_MASK)
 					{
 						if (opaque)
 						{
@@ -6598,7 +6481,7 @@ U32 LLVolumeGeometryManager::genDrawInfo(LLSpatialGroup* group, U32 mask, LLFace
 			}
 			else if (mat)
 			{
-				U8 mode = mat->getDiffuseAlphaMode();
+				U8 mode = mat->getDiffuseAlphaModeRender();
 
                 is_alpha = (is_alpha || (mode == LLMaterial::DIFFUSE_ALPHA_MODE_BLEND));
 
@@ -6697,7 +6580,7 @@ U32 LLVolumeGeometryManager::genDrawInfo(LLSpatialGroup* group, U32 mask, LLFace
 				}
 				else if (fullbright || bake_sunlight)
 				{ //fullbright
-					if (mat && mat->getDiffuseAlphaMode() == LLMaterial::DIFFUSE_ALPHA_MODE_MASK)
+					if (mat && mat->getDiffuseAlphaModeRender() == LLMaterial::DIFFUSE_ALPHA_MODE_MASK)
 					{
 						registerFace(group, facep, LLRenderPass::PASS_FULLBRIGHT_ALPHA_MASK);
 					}
@@ -6719,7 +6602,7 @@ U32 LLVolumeGeometryManager::genDrawInfo(LLSpatialGroup* group, U32 mask, LLFace
 					else
 					{ //all around simple
 						llassert(mask & LLVertexBuffer::MAP_NORMAL);
-						if (mat && mat->getDiffuseAlphaMode() == LLMaterial::DIFFUSE_ALPHA_MODE_MASK)
+						if (mat && mat->getDiffuseAlphaModeRender() == LLMaterial::DIFFUSE_ALPHA_MODE_MASK)
 						{ //material alpha mask can be respected in non-deferred
 							registerFace(group, facep, LLRenderPass::PASS_ALPHA_MASK);
 						}
diff --git a/indra/newview/pipeline.cpp b/indra/newview/pipeline.cpp
index 01438bfb9f..0deb41541b 100644
--- a/indra/newview/pipeline.cpp
+++ b/indra/newview/pipeline.cpp
@@ -1678,7 +1678,7 @@ U32 LLPipeline::getPoolTypeFromTE(const LLTextureEntry* te, LLViewerTexture* ima
 
 	if (alpha && mat)
 	{
-		switch (mat->getDiffuseAlphaMode())
+		switch (mat->getDiffuseAlphaModeRender())
 		{
 			case 1:
 				alpha = true; // Material's alpha mode is set to blend.  Toss it into the alpha draw pool.