MAINT-8043 Fix for bad_alloc crash in LLImageGL::setImage()

2 files changed, 30 insertions, 12 deletions

diff --git a/indra/llrender/llimagegl.cpp b/indra/llrender/llimagegl.cpp
index 20cba68f84..89500dcc04 100644
--- a/indra/llrender/llimagegl.cpp
+++ b/indra/llrender/llimagegl.cpp
@@ -622,7 +622,7 @@ void LLImageGL::setImage(const LLImageRaw* imageraw)
 }
 
 static LLTrace::BlockTimerStatHandle FTM_SET_IMAGE("setImage");
-void LLImageGL::setImage(const U8* data_in, BOOL data_hasmips)
+BOOL LLImageGL::setImage(const U8* data_in, BOOL data_hasmips)
 {
 	LL_RECORD_BLOCK_TIME(FTM_SET_IMAGE);
 	bool is_compressed = false;
@@ -787,19 +787,33 @@ void LLImageGL::setImage(const U8* data_in, BOOL data_hasmips)
 						llassert(prev_mip_data);
 						llassert(cur_mip_size == bytes*4);
 #endif
-						U8* new_data = new U8[bytes];
+						U8* new_data = new(std::nothrow) U8[bytes];
+						if (!new_data)
+						{
+							stop_glerror();
+
+							if (prev_mip_data)
+								delete[] prev_mip_data;
+							if (cur_mip_data)
+								delete[] cur_mip_data;
+							
+							mGLTextureCreated = false;
+							return FALSE;
+						}
+						else
+						{
 
 #ifdef SHOW_ASSERT
-						llassert(prev_mip_data);
-						llassert(cur_mip_size == bytes*4);
-						llassert_always(new_data);
+							llassert(prev_mip_data);
+							llassert(cur_mip_size == bytes * 4);
 #endif
 
-						LLImageBase::generateMip(prev_mip_data, new_data, w, h, mComponents);
-						cur_mip_data = new_data;
+							LLImageBase::generateMip(prev_mip_data, new_data, w, h, mComponents);
+							cur_mip_data = new_data;
 #ifdef SHOW_ASSERT
-						cur_mip_size = bytes; 
+							cur_mip_size = bytes;
 #endif
+						}
 
 					}
 					llassert(w > 0 && h > 0 && cur_mip_data);
@@ -886,6 +900,7 @@ void LLImageGL::setImage(const U8* data_in, BOOL data_hasmips)
 	}
 	stop_glerror();
 	mGLTextureCreated = true;
+	return TRUE;
 }
 
 BOOL LLImageGL::preAddToAtlas(S32 discard_level, const LLImageRaw* raw_image)
@@ -1355,8 +1370,7 @@ BOOL LLImageGL::createGLTexture(S32 discard_level, const U8* data_in, BOOL data_
 	if (mTexName != 0 && discard_level == mCurrentDiscardLevel)
 	{
 		// This will only be true if the size has not changed
-		setImage(data_in, data_hasmips);
-		return TRUE;
+		return setImage(data_in, data_hasmips);
 	}
 	
 	U32 old_name = mTexName;
@@ -1398,7 +1412,11 @@ BOOL LLImageGL::createGLTexture(S32 discard_level, const U8* data_in, BOOL data_
 
 	mCurrentDiscardLevel = discard_level;	
 
-	setImage(data_in, data_hasmips);
+	if (!setImage(data_in, data_hasmips))
+	{
+		stop_glerror();
+		return FALSE;
+	}
 
 	// Set texture options to our defaults.
 	gGL.getTexUnit(0)->setHasMipMaps(mHasMipMaps);
diff --git a/indra/llrender/llimagegl.h b/indra/llrender/llimagegl.h
index ad2aea9067..2be54be062 100644
--- a/indra/llrender/llimagegl.h
+++ b/indra/llrender/llimagegl.h
@@ -105,7 +105,7 @@ public:
 		S32 category = sMaxCategories-1);
 	BOOL createGLTexture(S32 discard_level, const U8* data, BOOL data_hasmips = FALSE, S32 usename = 0);
 	void setImage(const LLImageRaw* imageraw);
-	void setImage(const U8* data_in, BOOL data_hasmips = FALSE);
+	BOOL setImage(const U8* data_in, BOOL data_hasmips = FALSE);
 	BOOL setSubImage(const LLImageRaw* imageraw, S32 x_pos, S32 y_pos, S32 width, S32 height, BOOL force_fast_update = FALSE);
 	BOOL setSubImage(const U8* datap, S32 data_width, S32 data_height, S32 x_pos, S32 y_pos, S32 width, S32 height, BOOL force_fast_update = FALSE);
 	BOOL setSubImageFromFrameBuffer(S32 fb_x, S32 fb_y, S32 x_pos, S32 y_pos, S32 width, S32 height);