From e24d4c9f4d2f37ee80685c6ab276633b94b366b8 Mon Sep 17 00:00:00 2001 From: andreykproductengine Date: Thu, 26 Jul 2018 20:06:26 +0300 Subject: MAINT-8923 Better allocation failure handling, createGLTexture crashes --- indra/llappearance/llavatarappearance.cpp | 3 ++- indra/llcharacter/llkeyframemotion.cpp | 11 +++++++-- indra/llimage/llimage.cpp | 4 ++-- indra/llimage/llimagebmp.cpp | 8 +++++-- indra/llimage/llimagedxt.cpp | 6 ++++- indra/llimage/llimagejpeg.cpp | 23 ++++++++++++++++--- indra/llimage/llimagepng.cpp | 7 +++++- indra/llimage/llimagetga.cpp | 37 ++++++++++++++++++++++++++----- indra/llimage/llpngwrapper.cpp | 13 +++++++++-- indra/llrender/llimagegl.cpp | 14 ++++++++++-- 10 files changed, 104 insertions(+), 22 deletions(-) diff --git a/indra/llappearance/llavatarappearance.cpp b/indra/llappearance/llavatarappearance.cpp index e5089f028f..60359ca304 100644 --- a/indra/llappearance/llavatarappearance.cpp +++ b/indra/llappearance/llavatarappearance.cpp @@ -1566,9 +1566,10 @@ BOOL LLAvatarAppearance::allocateCollisionVolumes( U32 num ) delete_and_clear_array(mCollisionVolumes); mNumCollisionVolumes = 0; - mCollisionVolumes = new LLAvatarJointCollisionVolume[num]; + mCollisionVolumes = new(std::nothrow) LLAvatarJointCollisionVolume[num]; if (!mCollisionVolumes) { + LL_WARNS() << "Failed to allocate collision volumes" << LL_ENDL; return FALSE; } diff --git a/indra/llcharacter/llkeyframemotion.cpp b/indra/llcharacter/llkeyframemotion.cpp index f12c64023a..330d812985 100644 --- a/indra/llcharacter/llkeyframemotion.cpp +++ b/indra/llcharacter/llkeyframemotion.cpp @@ -579,8 +579,15 @@ LLMotion::LLMotionInitStatus LLKeyframeMotion::onInitialize(LLCharacter *charact else { anim_file_size = anim_file->getSize(); - anim_data = new U8[anim_file_size]; - success = anim_file->read(anim_data, anim_file_size); /*Flawfinder: ignore*/ + anim_data = new(std::nothrow) U8[anim_file_size]; + if (anim_data) + { + success = anim_file->read(anim_data, anim_file_size); /*Flawfinder: ignore*/ + } + else + { + LL_WARNS() << "Failed to allocate buffer: " << anim_file_size << mID << LL_ENDL; + } delete anim_file; anim_file = NULL; } diff --git a/indra/llimage/llimage.cpp b/indra/llimage/llimage.cpp index dca03cfe04..1a4dd2ca99 100644 --- a/indra/llimage/llimage.cpp +++ b/indra/llimage/llimage.cpp @@ -889,7 +889,7 @@ void LLImageRaw::setDataAndSize(U8 *data, S32 width, S32 height, S8 components) bool LLImageRaw::resize(U16 width, U16 height, S8 components) { - if ((getWidth() == width) && (getHeight() == height) && (getComponents() == components)) + if ((getWidth() == width) && (getHeight() == height) && (getComponents() == components) && !isBufferInvalid()) { return true; } @@ -898,7 +898,7 @@ bool LLImageRaw::resize(U16 width, U16 height, S8 components) allocateDataSize(width,height,components); - return true; + return !isBufferInvalid(); } bool LLImageRaw::setSubImage(U32 x_pos, U32 y_pos, U32 width, U32 height, diff --git a/indra/llimage/llimagebmp.cpp b/indra/llimage/llimagebmp.cpp index 2cdd26c22b..867b2bb47b 100644 --- a/indra/llimage/llimagebmp.cpp +++ b/indra/llimage/llimagebmp.cpp @@ -318,7 +318,7 @@ bool LLImageBMP::updateData() if( 0 != mColorPaletteColors ) { - mColorPalette = new U8[color_palette_size]; + mColorPalette = new(std::nothrow) U8[color_palette_size]; if (!mColorPalette) { LL_ERRS() << "Out of memory in LLImageBMP::updateData()" << LL_ENDL; @@ -344,7 +344,11 @@ bool LLImageBMP::decode(LLImageRaw* raw_image, F32 decode_time) return false; } - raw_image->resize(getWidth(), getHeight(), 3); + if (!raw_image->resize(getWidth(), getHeight(), 3)) + { + setLastError("llimagebmp failed to resize image!"); + return false; + } U8* src = mdata + mBitmapOffset; U8* dst = raw_image->getData(); diff --git a/indra/llimage/llimagedxt.cpp b/indra/llimage/llimagedxt.cpp index 3a7319d765..36317a5ba8 100644 --- a/indra/llimage/llimagedxt.cpp +++ b/indra/llimage/llimagedxt.cpp @@ -289,7 +289,11 @@ bool LLImageDXT::decode(LLImageRaw* raw_image, F32 time) return false; } - raw_image->resize(width, height, ncomponents); + if (!raw_image->resize(width, height, ncomponents)) + { + setLastError("llImageDXT failed to resize image!"); + return false; + } memcpy(raw_image->getData(), data, image_size); /* Flawfinder: ignore */ return true; diff --git a/indra/llimage/llimagejpeg.cpp b/indra/llimage/llimagejpeg.cpp index 60b2d0faa5..5c8b63a493 100644 --- a/indra/llimage/llimagejpeg.cpp +++ b/indra/llimage/llimagejpeg.cpp @@ -256,7 +256,10 @@ bool LLImageJPEG::decode(LLImageRaw* raw_image, F32 decode_time) setSize(cinfo.image_width, cinfo.image_height, 3); // Force to 3 components (RGB) - raw_image->resize(getWidth(), getHeight(), getComponents()); + if (!raw_image->resize(getWidth(), getHeight(), getComponents())) + { + throw std::bad_alloc(); + } raw_image_data = raw_image->getData(); @@ -311,6 +314,13 @@ bool LLImageJPEG::decode(LLImageRaw* raw_image, F32 decode_time) jpeg_destroy_decompress(&cinfo); } + catch (std::bad_alloc) + { + setLastError( "Out of memory"); + jpeg_destroy_decompress(&cinfo); + return true; // done + } + catch (int) { jpeg_destroy_decompress(&cinfo); @@ -370,7 +380,7 @@ boolean LLImageJPEG::encodeEmptyOutputBuffer( j_compress_ptr cinfo ) // Double the buffer size; S32 new_buffer_size = self->mOutputBufferSize * 2; - U8* new_buffer = new U8[ new_buffer_size ]; + U8* new_buffer = new(std::nothrow) U8[ new_buffer_size ]; if (!new_buffer) { LL_ERRS() << "Out of memory in LLImageJPEG::encodeEmptyOutputBuffer( j_compress_ptr cinfo )" << LL_ENDL; @@ -493,7 +503,14 @@ bool LLImageJPEG::encode( const LLImageRaw* raw_image, F32 encode_time ) disclaimMem(mOutputBufferSize); mOutputBufferSize = getWidth() * getHeight() * getComponents() + 1024; claimMem(mOutputBufferSize); - mOutputBuffer = new U8[ mOutputBufferSize ]; + mOutputBuffer = new(std::nothrow) U8[ mOutputBufferSize ]; + if (mOutputBuffer == NULL) + { + disclaimMem(mOutputBufferSize); + mOutputBufferSize = 0; + setLastError("Failed to allocate output buffer"); + return false; + } const U8* raw_image_data = NULL; S32 row_stride = 0; diff --git a/indra/llimage/llimagepng.cpp b/indra/llimage/llimagepng.cpp index a4823ed859..c4b98d8260 100644 --- a/indra/llimage/llimagepng.cpp +++ b/indra/llimage/llimagepng.cpp @@ -125,7 +125,12 @@ bool LLImagePNG::encode(const LLImageRaw* raw_image, F32 encode_time) // Temporary buffer to hold the encoded image. Note: the final image // size should be much smaller due to compression. U32 bufferSize = getWidth() * getHeight() * getComponents() + 8192; - U8* tmpWriteBuffer = new U8[ bufferSize ]; + U8* tmpWriteBuffer = new(std::nothrow) U8[ bufferSize ]; + if (!tmpWriteBuffer) + { + setLastError("LLImagePNG::out of memory"); + return false; + } // Delegate actual encoding work to wrapper LLPngWrapper pngWrapper; diff --git a/indra/llimage/llimagetga.cpp b/indra/llimage/llimagetga.cpp index 7c75aa1e2a..88bdae9b80 100644 --- a/indra/llimage/llimagetga.cpp +++ b/indra/llimage/llimagetga.cpp @@ -263,7 +263,7 @@ bool LLImageTGA::updateData() // only allocate memory for one if _we_ intend to use it. if ( (1 == mImageType) || (9 == mImageType) ) { - mColorMap = new U8[ color_map_bytes ]; + mColorMap = new(std::nothrow) U8[ color_map_bytes ]; if (!mColorMap) { LL_ERRS() << "Out of Memory in bool LLImageTGA::updateData()" << LL_ENDL; @@ -336,7 +336,11 @@ bool LLImageTGA::decode(LLImageRaw* raw_image, F32 decode_time) // Copy everything after the header. - raw_image->resize(getWidth(), getHeight(), getComponents()); + if( !raw_image->resize(getWidth(), getHeight(), getComponents())) + { + setLastError("LLImageTGA::out of memory"); + return false; + } if( (getComponents() != 1) && (getComponents() != 3) && @@ -346,6 +350,11 @@ bool LLImageTGA::decode(LLImageRaw* raw_image, F32 decode_time) return false; } + if( raw_image->isBufferInvalid()) + { + setLastError("LLImageTGA::out of memory"); + return false; + } if( mOriginRightBit ) { @@ -395,6 +404,11 @@ bool LLImageTGA::decodeTruecolor( LLImageRaw* raw_image, bool rle, bool flipped // alpha was entirely opaque // convert to 24 bit image LLPointer compacted_image = new LLImageRaw(raw_image->getWidth(), raw_image->getHeight(), 3); + if (compacted_image->isBufferInvalid()) + { + success = false; + break; + } compacted_image->copy(raw_image); raw_image->resize(raw_image->getWidth(), raw_image->getHeight(), 3); raw_image->copy(compacted_image); @@ -411,9 +425,16 @@ bool LLImageTGA::decodeTruecolor( LLImageRaw* raw_image, bool rle, bool flipped // alpha was entirely opaque // convert to 24 bit image LLPointer compacted_image = new LLImageRaw(raw_image->getWidth(), raw_image->getHeight(), 3); - compacted_image->copy(raw_image); - raw_image->resize(raw_image->getWidth(), raw_image->getHeight(), 3); - raw_image->copy(compacted_image); + if (compacted_image->isBufferInvalid()) + { + success = false; + } + else + { + compacted_image->copy(raw_image); + raw_image->resize(raw_image->getWidth(), raw_image->getHeight(), 3); + raw_image->copy(compacted_image); + } } } @@ -1053,7 +1074,11 @@ bool LLImageTGA::decodeAndProcess( LLImageRaw* raw_image, F32 domain, F32 weight return false; } - raw_image->resize(getWidth(), getHeight(), getComponents()); + if( !raw_image->resize(getWidth(), getHeight(), getComponents()) ) + { + LL_ERRS() << "LLImageTGA: Failed to resize image" << LL_ENDL; + return false; + } U8* dst = raw_image->getData(); U8* src = getData() + mDataOffset; diff --git a/indra/llimage/llpngwrapper.cpp b/indra/llimage/llpngwrapper.cpp index eb70b78a36..f298764cc0 100644 --- a/indra/llimage/llpngwrapper.cpp +++ b/indra/llimage/llpngwrapper.cpp @@ -173,8 +173,11 @@ BOOL LLPngWrapper::readPng(U8* src, S32 dataSize, LLImageRaw* rawImage, ImageInf // data space if (rawImage != NULL) { - rawImage->resize(static_cast(mWidth), - static_cast(mHeight), mChannels); + if (!rawImage->resize(static_cast(mWidth), + static_cast(mHeight), mChannels)) + { + LLTHROW(PngError("Failed to resize image")); + } U8 *dest = rawImage->getData(); int offset = mWidth * mChannels; @@ -207,6 +210,12 @@ BOOL LLPngWrapper::readPng(U8* src, S32 dataSize, LLImageRaw* rawImage, ImageInf releaseResources(); return (FALSE); } + catch (std::bad_alloc) + { + mErrorMessage = "LLPngWrapper"; + releaseResources(); + return (FALSE); + } // Clean up and return releaseResources(); diff --git a/indra/llrender/llimagegl.cpp b/indra/llrender/llimagegl.cpp index 89500dcc04..40217b2e80 100644 --- a/indra/llrender/llimagegl.cpp +++ b/indra/llrender/llimagegl.cpp @@ -1262,7 +1262,8 @@ BOOL LLImageGL::createGLTexture() stop_glerror(); if (!mTexName) { - LL_ERRS() << "LLImageGL::createGLTexture failed to make an empty texture" << LL_ENDL; + LL_WARNS() << "LLImageGL::createGLTexture failed to make an empty texture" << LL_ENDL; + return FALSE; } return TRUE ; @@ -1395,7 +1396,16 @@ BOOL LLImageGL::createGLTexture(S32 discard_level, const U8* data_in, BOOL data_ } if (!mTexName) { - LL_ERRS() << "LLImageGL::createGLTexture failed to make texture" << LL_ENDL; + if (old_name) + { + sGlobalTextureMemory -= mTextureMemory; + LLImageGL::deleteTextures(1, &old_name); + disclaimMem(mTextureMemory); + stop_glerror(); + } + + LL_WARNS() << "LLImageGL::createGLTexture failed to make texture" << LL_ENDL; + return FALSE; } if (mUseMipMaps) -- cgit v1.2.3