From 7d156389e46cfb3c0bb135e1b3eb873b24beaebe Mon Sep 17 00:00:00 2001 From: AndreyL ProductEngine Date: Mon, 18 Dec 2017 20:29:50 +0200 Subject: MAINT-8043 Fix for bad_alloc crash in LLImageGL::setImage() --- indra/llrender/llimagegl.cpp | 40 +++++++++++++++++++++++++++++----------- indra/llrender/llimagegl.h | 2 +- 2 files changed, 30 insertions(+), 12 deletions(-) (limited to 'indra') diff --git a/indra/llrender/llimagegl.cpp b/indra/llrender/llimagegl.cpp index 20cba68f84..89500dcc04 100644 --- a/indra/llrender/llimagegl.cpp +++ b/indra/llrender/llimagegl.cpp @@ -622,7 +622,7 @@ void LLImageGL::setImage(const LLImageRaw* imageraw) } static LLTrace::BlockTimerStatHandle FTM_SET_IMAGE("setImage"); -void LLImageGL::setImage(const U8* data_in, BOOL data_hasmips) +BOOL LLImageGL::setImage(const U8* data_in, BOOL data_hasmips) { LL_RECORD_BLOCK_TIME(FTM_SET_IMAGE); bool is_compressed = false; @@ -787,19 +787,33 @@ void LLImageGL::setImage(const U8* data_in, BOOL data_hasmips) llassert(prev_mip_data); llassert(cur_mip_size == bytes*4); #endif - U8* new_data = new U8[bytes]; + U8* new_data = new(std::nothrow) U8[bytes]; + if (!new_data) + { + stop_glerror(); + + if (prev_mip_data) + delete[] prev_mip_data; + if (cur_mip_data) + delete[] cur_mip_data; + + mGLTextureCreated = false; + return FALSE; + } + else + { #ifdef SHOW_ASSERT - llassert(prev_mip_data); - llassert(cur_mip_size == bytes*4); - llassert_always(new_data); + llassert(prev_mip_data); + llassert(cur_mip_size == bytes * 4); #endif - LLImageBase::generateMip(prev_mip_data, new_data, w, h, mComponents); - cur_mip_data = new_data; + LLImageBase::generateMip(prev_mip_data, new_data, w, h, mComponents); + cur_mip_data = new_data; #ifdef SHOW_ASSERT - cur_mip_size = bytes; + cur_mip_size = bytes; #endif + } } llassert(w > 0 && h > 0 && cur_mip_data); @@ -886,6 +900,7 @@ void LLImageGL::setImage(const U8* data_in, BOOL data_hasmips) } stop_glerror(); mGLTextureCreated = true; + return TRUE; } BOOL LLImageGL::preAddToAtlas(S32 discard_level, const LLImageRaw* raw_image) @@ -1355,8 +1370,7 @@ BOOL LLImageGL::createGLTexture(S32 discard_level, const U8* data_in, BOOL data_ if (mTexName != 0 && discard_level == mCurrentDiscardLevel) { // This will only be true if the size has not changed - setImage(data_in, data_hasmips); - return TRUE; + return setImage(data_in, data_hasmips); } U32 old_name = mTexName; @@ -1398,7 +1412,11 @@ BOOL LLImageGL::createGLTexture(S32 discard_level, const U8* data_in, BOOL data_ mCurrentDiscardLevel = discard_level; - setImage(data_in, data_hasmips); + if (!setImage(data_in, data_hasmips)) + { + stop_glerror(); + return FALSE; + } // Set texture options to our defaults. gGL.getTexUnit(0)->setHasMipMaps(mHasMipMaps); diff --git a/indra/llrender/llimagegl.h b/indra/llrender/llimagegl.h index ad2aea9067..2be54be062 100644 --- a/indra/llrender/llimagegl.h +++ b/indra/llrender/llimagegl.h @@ -105,7 +105,7 @@ public: S32 category = sMaxCategories-1); BOOL createGLTexture(S32 discard_level, const U8* data, BOOL data_hasmips = FALSE, S32 usename = 0); void setImage(const LLImageRaw* imageraw); - void setImage(const U8* data_in, BOOL data_hasmips = FALSE); + BOOL setImage(const U8* data_in, BOOL data_hasmips = FALSE); BOOL setSubImage(const LLImageRaw* imageraw, S32 x_pos, S32 y_pos, S32 width, S32 height, BOOL force_fast_update = FALSE); BOOL setSubImage(const U8* datap, S32 data_width, S32 data_height, S32 x_pos, S32 y_pos, S32 width, S32 height, BOOL force_fast_update = FALSE); BOOL setSubImageFromFrameBuffer(S32 fb_x, S32 fb_y, S32 x_pos, S32 y_pos, S32 width, S32 height); -- cgit v1.2.3