MAINT-8923 Better allocation failure handling, createGLTexture crashes

10 files changed, 104 insertions, 22 deletions

diff --git a/indra/llappearance/llavatarappearance.cpp b/indra/llappearance/llavatarappearance.cpp
index e5089f028f..60359ca304 100644
--- a/indra/llappearance/llavatarappearance.cpp
+++ b/indra/llappearance/llavatarappearance.cpp
@@ -1566,9 +1566,10 @@ BOOL LLAvatarAppearance::allocateCollisionVolumes( U32 num )
         delete_and_clear_array(mCollisionVolumes);
         mNumCollisionVolumes = 0;
 
-        mCollisionVolumes = new LLAvatarJointCollisionVolume[num];
+        mCollisionVolumes = new(std::nothrow) LLAvatarJointCollisionVolume[num];
         if (!mCollisionVolumes)
         {
+            LL_WARNS() << "Failed to allocate collision volumes" << LL_ENDL;
             return FALSE;
         }
         
diff --git a/indra/llcharacter/llkeyframemotion.cpp b/indra/llcharacter/llkeyframemotion.cpp
index f12c64023a..330d812985 100644
--- a/indra/llcharacter/llkeyframemotion.cpp
+++ b/indra/llcharacter/llkeyframemotion.cpp
@@ -579,8 +579,15 @@ LLMotion::LLMotionInitStatus LLKeyframeMotion::onInitialize(LLCharacter *charact
 	else
 	{
 		anim_file_size = anim_file->getSize();
-		anim_data = new U8[anim_file_size];
-		success = anim_file->read(anim_data, anim_file_size);	/*Flawfinder: ignore*/
+		anim_data = new(std::nothrow) U8[anim_file_size];
+		if (anim_data)
+		{
+			success = anim_file->read(anim_data, anim_file_size);	/*Flawfinder: ignore*/
+		}
+		else
+		{
+			LL_WARNS() << "Failed to allocate buffer: " << anim_file_size << mID << LL_ENDL;
+		}
 		delete anim_file;
 		anim_file = NULL;
 	}
diff --git a/indra/llimage/llimage.cpp b/indra/llimage/llimage.cpp
index dca03cfe04..1a4dd2ca99 100644
--- a/indra/llimage/llimage.cpp
+++ b/indra/llimage/llimage.cpp
@@ -889,7 +889,7 @@ void LLImageRaw::setDataAndSize(U8 *data, S32 width, S32 height, S8 components)
 
 bool LLImageRaw::resize(U16 width, U16 height, S8 components)
 {
-	if ((getWidth() == width) && (getHeight() == height) && (getComponents() == components))
+	if ((getWidth() == width) && (getHeight() == height) && (getComponents() == components) && !isBufferInvalid())
 	{
 		return true;
 	}
@@ -898,7 +898,7 @@ bool LLImageRaw::resize(U16 width, U16 height, S8 components)
 
 	allocateDataSize(width,height,components);
 
-	return true;
+	return !isBufferInvalid();
 }
 
 bool LLImageRaw::setSubImage(U32 x_pos, U32 y_pos, U32 width, U32 height,
diff --git a/indra/llimage/llimagebmp.cpp b/indra/llimage/llimagebmp.cpp
index 2cdd26c22b..867b2bb47b 100644
--- a/indra/llimage/llimagebmp.cpp
+++ b/indra/llimage/llimagebmp.cpp
@@ -318,7 +318,7 @@ bool LLImageBMP::updateData()
 
 	if( 0 != mColorPaletteColors )
 	{
-		mColorPalette = new U8[color_palette_size];
+		mColorPalette = new(std::nothrow) U8[color_palette_size];
 		if (!mColorPalette)
 		{
 			LL_ERRS() << "Out of memory in LLImageBMP::updateData()" << LL_ENDL;
@@ -344,7 +344,11 @@ bool LLImageBMP::decode(LLImageRaw* raw_image, F32 decode_time)
 		return false;
 	}
 	
-	raw_image->resize(getWidth(), getHeight(), 3);
+	if (!raw_image->resize(getWidth(), getHeight(), 3))
+	{
+		setLastError("llimagebmp failed to resize image!");
+		return false;
+	}
 
 	U8* src = mdata + mBitmapOffset;
 	U8* dst = raw_image->getData();
diff --git a/indra/llimage/llimagedxt.cpp b/indra/llimage/llimagedxt.cpp
index 3a7319d765..36317a5ba8 100644
--- a/indra/llimage/llimagedxt.cpp
+++ b/indra/llimage/llimagedxt.cpp
@@ -289,7 +289,11 @@ bool LLImageDXT::decode(LLImageRaw* raw_image, F32 time)
 		return false;
 	}
 
-	raw_image->resize(width, height, ncomponents);
+	if (!raw_image->resize(width, height, ncomponents))
+	{
+		setLastError("llImageDXT failed to resize image!");
+		return false;
+	}
 	memcpy(raw_image->getData(), data, image_size);	/* Flawfinder: ignore */
 
 	return true;
diff --git a/indra/llimage/llimagejpeg.cpp b/indra/llimage/llimagejpeg.cpp
index 60b2d0faa5..5c8b63a493 100644
--- a/indra/llimage/llimagejpeg.cpp
+++ b/indra/llimage/llimagejpeg.cpp
@@ -256,7 +256,10 @@ bool LLImageJPEG::decode(LLImageRaw* raw_image, F32 decode_time)
 
 		setSize(cinfo.image_width, cinfo.image_height, 3); // Force to 3 components (RGB)
 
-		raw_image->resize(getWidth(), getHeight(), getComponents());
+		if (!raw_image->resize(getWidth(), getHeight(), getComponents()))
+		{
+			throw std::bad_alloc();
+		}
 		raw_image_data = raw_image->getData();
 		
 		
@@ -311,6 +314,13 @@ bool LLImageJPEG::decode(LLImageRaw* raw_image, F32 decode_time)
 		jpeg_destroy_decompress(&cinfo);
 	}
 
+	catch (std::bad_alloc)
+	{
+		setLastError( "Out of memory");
+		jpeg_destroy_decompress(&cinfo);
+		return true; // done
+	}
+
 	catch (int)
 	{
 		jpeg_destroy_decompress(&cinfo);
@@ -370,7 +380,7 @@ boolean LLImageJPEG::encodeEmptyOutputBuffer( j_compress_ptr cinfo )
   
   // Double the buffer size;
   S32 new_buffer_size = self->mOutputBufferSize * 2;
-  U8* new_buffer = new U8[ new_buffer_size ];
+  U8* new_buffer = new(std::nothrow) U8[ new_buffer_size ];
   if (!new_buffer)
   {
   	LL_ERRS() << "Out of memory in LLImageJPEG::encodeEmptyOutputBuffer( j_compress_ptr cinfo )" << LL_ENDL;
@@ -493,7 +503,14 @@ bool LLImageJPEG::encode( const LLImageRaw* raw_image, F32 encode_time )
 	disclaimMem(mOutputBufferSize);
 	mOutputBufferSize = getWidth() * getHeight() * getComponents() + 1024;
 	claimMem(mOutputBufferSize);
-	mOutputBuffer = new U8[ mOutputBufferSize ];
+	mOutputBuffer = new(std::nothrow) U8[ mOutputBufferSize ];
+	if (mOutputBuffer == NULL)
+	{
+		disclaimMem(mOutputBufferSize);
+		mOutputBufferSize = 0;
+		setLastError("Failed to allocate output buffer");
+		return false;
+	}
 
 	const U8* raw_image_data = NULL;
 	S32 row_stride = 0;
diff --git a/indra/llimage/llimagepng.cpp b/indra/llimage/llimagepng.cpp
index a4823ed859..c4b98d8260 100644
--- a/indra/llimage/llimagepng.cpp
+++ b/indra/llimage/llimagepng.cpp
@@ -125,7 +125,12 @@ bool LLImagePNG::encode(const LLImageRaw* raw_image, F32 encode_time)
 	// Temporary buffer to hold the encoded image. Note: the final image
 	// size should be much smaller due to compression.
 	U32 bufferSize = getWidth() * getHeight() * getComponents() + 8192;
-    U8* tmpWriteBuffer = new U8[ bufferSize ];
+	U8* tmpWriteBuffer = new(std::nothrow) U8[ bufferSize ];
+	if (!tmpWriteBuffer)
+	{
+		setLastError("LLImagePNG::out of memory");
+		return false;
+	}
 
 	// Delegate actual encoding work to wrapper
 	LLPngWrapper pngWrapper;
diff --git a/indra/llimage/llimagetga.cpp b/indra/llimage/llimagetga.cpp
index 7c75aa1e2a..88bdae9b80 100644
--- a/indra/llimage/llimagetga.cpp
+++ b/indra/llimage/llimagetga.cpp
@@ -263,7 +263,7 @@ bool LLImageTGA::updateData()
 		// only allocate memory for one if _we_ intend to use it.
 		if ( (1 == mImageType) || (9 == mImageType)  )
 		{
-			mColorMap = new U8[ color_map_bytes ];  
+			mColorMap = new(std::nothrow) U8[ color_map_bytes ];  
 			if (!mColorMap)
 			{
 				LL_ERRS() << "Out of Memory in bool LLImageTGA::updateData()" << LL_ENDL;
@@ -336,7 +336,11 @@ bool LLImageTGA::decode(LLImageRaw* raw_image, F32 decode_time)
 
 	// Copy everything after the header.
 
-	raw_image->resize(getWidth(), getHeight(), getComponents());
+	if( !raw_image->resize(getWidth(), getHeight(), getComponents()))
+	{
+		setLastError("LLImageTGA::out of memory");
+		return false;
+	}
 
 	if( (getComponents() != 1) &&
 		(getComponents() != 3) &&
@@ -346,6 +350,11 @@ bool LLImageTGA::decode(LLImageRaw* raw_image, F32 decode_time)
 		return false;
 	}
 
+	if( raw_image->isBufferInvalid())
+	{
+		setLastError("LLImageTGA::out of memory");
+		return false;
+	}
 
 	if( mOriginRightBit )
 	{
@@ -395,6 +404,11 @@ bool LLImageTGA::decodeTruecolor( LLImageRaw* raw_image, bool rle, bool flipped
 				// alpha was entirely opaque
 				// convert to 24 bit image
 				LLPointer<LLImageRaw> compacted_image = new LLImageRaw(raw_image->getWidth(), raw_image->getHeight(), 3);
+				if (compacted_image->isBufferInvalid())
+				{
+					success = false;
+					break;
+				}
 				compacted_image->copy(raw_image);
 				raw_image->resize(raw_image->getWidth(), raw_image->getHeight(), 3);
 				raw_image->copy(compacted_image);
@@ -411,9 +425,16 @@ bool LLImageTGA::decodeTruecolor( LLImageRaw* raw_image, bool rle, bool flipped
 			// alpha was entirely opaque
 			// convert to 24 bit image
 			LLPointer<LLImageRaw> compacted_image = new LLImageRaw(raw_image->getWidth(), raw_image->getHeight(), 3);
-			compacted_image->copy(raw_image);
-			raw_image->resize(raw_image->getWidth(), raw_image->getHeight(), 3);
-			raw_image->copy(compacted_image);
+			if (compacted_image->isBufferInvalid())
+			{
+				success = false;
+			}
+			else
+			{
+				compacted_image->copy(raw_image);
+				raw_image->resize(raw_image->getWidth(), raw_image->getHeight(), 3);
+				raw_image->copy(compacted_image);
+			}
 		}
 	}
 	
@@ -1053,7 +1074,11 @@ bool LLImageTGA::decodeAndProcess( LLImageRaw* raw_image, F32 domain, F32 weight
 		return false;
 	}
 
-	raw_image->resize(getWidth(), getHeight(), getComponents());
+	if( !raw_image->resize(getWidth(), getHeight(), getComponents()) )
+	{
+		LL_ERRS() << "LLImageTGA: Failed to resize image" << LL_ENDL;
+		return false;
+	}
 
 	U8* dst = raw_image->getData();
 	U8* src = getData() + mDataOffset;
diff --git a/indra/llimage/llpngwrapper.cpp b/indra/llimage/llpngwrapper.cpp
index eb70b78a36..f298764cc0 100644
--- a/indra/llimage/llpngwrapper.cpp
+++ b/indra/llimage/llpngwrapper.cpp
@@ -173,8 +173,11 @@ BOOL LLPngWrapper::readPng(U8* src, S32 dataSize, LLImageRaw* rawImage, ImageInf
 		// data space
 		if (rawImage != NULL)
 		{
-			rawImage->resize(static_cast<U16>(mWidth),
-				static_cast<U16>(mHeight), mChannels);
+			if (!rawImage->resize(static_cast<U16>(mWidth),
+				static_cast<U16>(mHeight), mChannels))
+			{
+				LLTHROW(PngError("Failed to resize image"));
+			}
 			U8 *dest = rawImage->getData();
 			int offset = mWidth * mChannels;
 
@@ -207,6 +210,12 @@ BOOL LLPngWrapper::readPng(U8* src, S32 dataSize, LLImageRaw* rawImage, ImageInf
 		releaseResources();
 		return (FALSE);
 	}
+	catch (std::bad_alloc)
+	{
+		mErrorMessage = "LLPngWrapper";
+		releaseResources();
+		return (FALSE);
+	}
 
 	// Clean up and return
 	releaseResources();
diff --git a/indra/llrender/llimagegl.cpp b/indra/llrender/llimagegl.cpp
index 89500dcc04..40217b2e80 100644
--- a/indra/llrender/llimagegl.cpp
+++ b/indra/llrender/llimagegl.cpp
@@ -1262,7 +1262,8 @@ BOOL LLImageGL::createGLTexture()
 	stop_glerror();
 	if (!mTexName)
 	{
-		LL_ERRS() << "LLImageGL::createGLTexture failed to make an empty texture" << LL_ENDL;
+		LL_WARNS() << "LLImageGL::createGLTexture failed to make an empty texture" << LL_ENDL;
+		return FALSE;
 	}
 
 	return TRUE ;
@@ -1395,7 +1396,16 @@ BOOL LLImageGL::createGLTexture(S32 discard_level, const U8* data_in, BOOL data_
 	}
 	if (!mTexName)
 	{
-		LL_ERRS() << "LLImageGL::createGLTexture failed to make texture" << LL_ENDL;
+		if (old_name)
+		{
+			sGlobalTextureMemory -= mTextureMemory;
+			LLImageGL::deleteTextures(1, &old_name);
+			disclaimMem(mTextureMemory);
+			stop_glerror();
+		}
+
+		LL_WARNS() << "LLImageGL::createGLTexture failed to make texture" << LL_ENDL;
+		return FALSE;
 	}
 
 	if (mUseMipMaps)