extend alpha mask calculation to GL_UNSIGNED_INT_8_8_8_8 and GL_UNSIGNED_INT_8_8_8_8_REV. also fix EXT-389: clean up warnings: Cannot analyze alpha for image with format type 8035.

2 files changed, 93 insertions, 16 deletions

diff --git a/indra/llrender/llimagegl.cpp b/indra/llrender/llimagegl.cpp
index c3d1d9e894..20c45aa2d3 100644
--- a/indra/llrender/llimagegl.cpp
+++ b/indra/llrender/llimagegl.cpp
@@ -132,6 +132,13 @@ void LLImageGL::checkTexSize() const
 //**************************************************************************************
 
 //----------------------------------------------------------------------------
+BOOL is_little_endian()
+{
+	S32 a = 0x12345678;
+    U8 *c = (U8*)(&a);
+    
+	return (*c == 0x78) ;
+}
 
 //static
 S32 LLImageGL::dataFormatBits(S32 dataformat)
@@ -363,6 +370,8 @@ void LLImageGL::init(BOOL usemipmaps)
 	mGLTextureCreated = FALSE ;
 	mIsMask = FALSE;
 	mNeedsAlphaAndPickMask = TRUE ;
+	mAlphaStride = 0 ;
+	mAlphaOffset = 0 ;
 }
 
 void LLImageGL::cleanup()
@@ -501,6 +510,8 @@ void LLImageGL::setExplicitFormat( LLGLint internal_format, LLGLenum primary_for
 	else
 		mFormatType = type_format;
 	mFormatSwapBytes = swap_bytes;
+
+	calcAlphaChannelOffsetAndStride() ;
 }
 
 //----------------------------------------------------------------------------
@@ -951,6 +962,8 @@ BOOL LLImageGL::createGLTexture(S32 discard_level, const LLImageRaw* imageraw, S
 		  default:
 			llerrs << "Bad number of components for texture: " << (U32)getComponents() << llendl;
 		}
+
+		calcAlphaChannelOffsetAndStride() ;
 	}
 
  	const U8* rawdata = imageraw->getData();
@@ -1355,44 +1368,105 @@ void LLImageGL::setTarget(const LLGLenum target, const LLTexUnit::eTextureType b
 	mBindTarget = bind_target;
 }
 
-void LLImageGL::analyzeAlpha(const void* data_in, S32 w, S32 h)
+const S8 INVALID_OFFSET = -99 ;
+void LLImageGL::setNeedsAlphaAndPickMask(BOOL need_mask) 
 {
-	if(!mNeedsAlphaAndPickMask)
+	if(mNeedsAlphaAndPickMask != need_mask)
+	{
+		mNeedsAlphaAndPickMask = need_mask;
+
+		if(mNeedsAlphaAndPickMask)
+		{
+			mAlphaOffset = 0 ;
+		}
+		else //do not need alpha mask
+		{
+			mAlphaOffset = INVALID_OFFSET ;
+			mIsMask = FALSE;
+		}
+	}
+}
+
+void LLImageGL::calcAlphaChannelOffsetAndStride()
+{
+	if(mAlphaOffset == INVALID_OFFSET)//do not need alpha mask
 	{
 		return ;
 	}
 
-	if (mFormatType != GL_UNSIGNED_BYTE)
+	mAlphaOffset = -1 ;
+
+	if (mFormatType == GL_UNSIGNED_BYTE)
 	{
-		llwarns << "Cannot analyze alpha for image with format type " << std::hex << mFormatType << std::dec << llendl;
+		mAlphaOffset = 3 ;
+	}
+	else if(is_little_endian())
+	{
+		if (mFormatType == GL_UNSIGNED_INT_8_8_8_8)
+		{
+			mAlphaOffset = 0 ;
+		}
+		else if (mFormatType == GL_UNSIGNED_INT_8_8_8_8_REV)
+		{
+			mAlphaOffset = 3 ;
+		}
+	}
+	else //big endian
+	{
+		if (mFormatType == GL_UNSIGNED_INT_8_8_8_8)
+		{
+			mAlphaOffset = 3 ;
+		}
+		else if (mFormatType == GL_UNSIGNED_INT_8_8_8_8_REV)
+		{
+			mAlphaOffset = 0 ;
+		}
 	}
 
-	U32 stride = 0;
+	mAlphaStride = -1 ;
 	switch (mFormatPrimary)
 	{
 	case GL_LUMINANCE:
 	case GL_ALPHA:
-		stride = 1;
+		mAlphaStride = 1;
 		break;
 	case GL_LUMINANCE_ALPHA:
-		stride = 2;
+		mAlphaStride = 2;
 		break;
 	case GL_RGB:
-		//no alpha
+		mNeedsAlphaAndPickMask = FALSE ;
 		mIsMask = FALSE;
-		return;
+		return ; //no alpha channel.
 	case GL_RGBA:
-		stride = 4;
+		mAlphaStride = 4;
 		break;
 	case GL_BGRA_EXT:
-		stride = 4;
+		mAlphaStride = 4;
 		break;
 	default:
-		return;
+		break;
+	}
+
+	if( mAlphaStride < 1 || //unsupported format
+		mAlphaOffset < 0 || //unsupported type
+		(mFormatPrimary == GL_BGRA_EXT && mFormatType != GL_UNSIGNED_BYTE)) //unknown situation
+	{
+		llwarns << "Cannot analyze alpha for image with format type " << std::hex << mFormatType << std::dec << llendl;
+
+		mNeedsAlphaAndPickMask = FALSE ;
+		mIsMask = FALSE;
+	}
+}
+
+void LLImageGL::analyzeAlpha(const void* data_in, S32 w, S32 h)
+{
+	if(!mNeedsAlphaAndPickMask)
+	{
+		return ;
 	}
 
 	U32 length = w * h;
-	const GLubyte* current = ((const GLubyte*) data_in)+stride-1;
+	const GLubyte* current = ((const GLubyte*) data_in) + mAlphaOffset ;
 	
 	S32 sample[16];
 	memset(sample, 0, sizeof(S32)*16);
@@ -1400,7 +1474,7 @@ void LLImageGL::analyzeAlpha(const void* data_in, S32 w, S32 h)
 	for (U32 i = 0; i < length; i++)
 	{
 		++sample[*current/16];
-		current += stride;
+		current += mAlphaStride ;
 	}
 
 	U32 total = 0;
diff --git a/indra/llrender/llimagegl.h b/indra/llrender/llimagegl.h
index 5f32b23356..bcb57d3a21 100644
--- a/indra/llrender/llimagegl.h
+++ b/indra/llrender/llimagegl.h
@@ -88,6 +88,7 @@ protected:
 	virtual ~LLImageGL();
 
 	void analyzeAlpha(const void* data_in, S32 w, S32 h);
+	void calcAlphaChannelOffsetAndStride();
 
 public:
 	virtual void dump();	// debugging info to llinfos
@@ -167,7 +168,7 @@ public:
 	void init(BOOL usemipmaps);
 	virtual void cleanup(); // Clean up the LLImageGL so it can be reinitialized.  Be careful when using this in derived class destructors
 
-	void setNeedsAlphaAndPickMask(BOOL need_mask) {mNeedsAlphaAndPickMask = need_mask;}
+	void setNeedsAlphaAndPickMask(BOOL need_mask);
 public:
 	// Various GL/Rendering options
 	S32 mTextureMemory;
@@ -182,7 +183,9 @@ private:
 
 	BOOL mIsMask;
 	BOOL mNeedsAlphaAndPickMask;
-	
+	S8   mAlphaStride ;
+	S8   mAlphaOffset ;
+
 	bool     mGLTextureCreated ;
 	LLGLuint mTexName;
 	U16      mWidth;