From b00813ae3ca14a7531d93d35d795ce7c23119886 Mon Sep 17 00:00:00 2001 From: Tofu Linden Date: Mon, 25 Jan 2010 16:35:50 -0800 Subject: EXT-4388 Crash in octree line segment intersection code a more complete fix and some sanity to prevent recurrance of a similar problem. reviewed by bao! --- indra/llrender/llimagegl.cpp | 43 +++++++++++++++++++++++++------------------ indra/llrender/llimagegl.h | 2 ++ 2 files changed, 27 insertions(+), 18 deletions(-) (limited to 'indra/llrender') diff --git a/indra/llrender/llimagegl.cpp b/indra/llrender/llimagegl.cpp index cd493481d5..46478ba3c9 100644 --- a/indra/llrender/llimagegl.cpp +++ b/indra/llrender/llimagegl.cpp @@ -436,6 +436,8 @@ void LLImageGL::init(BOOL usemipmaps) mLastBindTime = 0.f; mPickMask = NULL; + mPickMaskWidth = 0; + mPickMaskHeight = 0; mUseMipMaps = usemipmaps; mHasExplicitFormat = FALSE; mAutoGenMips = FALSE; @@ -527,7 +529,12 @@ void LLImageGL::setSize(S32 width, S32 height, S32 ncomponents) // llwarns << "Setting Size of LLImageGL with existing mTexName = " << mTexName << llendl; destroyGLTexture(); } - + + // pickmask validity depends on old image size, delete it + delete [] mPickMask; + mPickMask = NULL; + mPickMaskWidth = mPickMaskHeight = 0; + mWidth = width; mHeight = height; mComponents = ncomponents; @@ -1675,12 +1682,14 @@ void LLImageGL::updatePickMask(S32 width, S32 height, const U8* data_in) return ; } + delete [] mPickMask; + mPickMask = NULL; + mPickMaskWidth = mPickMaskHeight = 0; + if (mFormatType != GL_UNSIGNED_BYTE || mFormatPrimary != GL_RGBA) { //cannot generate a pick mask for this texture - delete [] mPickMask; - mPickMask = NULL; return; } @@ -1688,11 +1697,10 @@ void LLImageGL::updatePickMask(S32 width, S32 height, const U8* data_in) U32 pick_height = height/2; U32 size = llmax(pick_width, (U32) 1) * llmax(pick_height, (U32) 1); - size = size/8 + 1; - - delete[] mPickMask; mPickMask = new U8[size]; + mPickMaskWidth = pick_width; + mPickMaskHeight = pick_height; memset(mPickMask, 0, sizeof(U8) * size); @@ -1727,35 +1735,34 @@ BOOL LLImageGL::getMask(const LLVector2 &tc) if (mPickMask) { - S32 width = getWidth()/2; - S32 height = getHeight()/2; - F32 u = tc.mV[0] - floorf(tc.mV[0]); F32 v = tc.mV[1] - floorf(tc.mV[1]); - if (u < 0.f || u > 1.f || - v < 0.f || v > 1.f) + if (LL_UNLIKELY(u < 0.f || u > 1.f || + v < 0.f || v > 1.f)) { LL_WARNS_ONCE("render") << "Ugh, u/v out of range in image mask pick" << LL_ENDL; u = v = 0.f; llassert(false); } + + llassert(mPickMaskWidth > 0 && mPickMaskHeight > 0); - S32 x = (S32)(u * width); - S32 y = (S32)(v * height); + S32 x = (S32)(u * mPickMaskWidth); + S32 y = (S32)(v * mPickMaskHeight); - if (x >= width) + if (LL_UNLIKELY(x >= mPickMaskWidth)) { LL_WARNS_ONCE("render") << "Ooh, width overrun on pick mask read, that coulda been bad." << LL_ENDL; - x = llmax(0, width-1); + x = llmax(0, mPickMaskWidth-1); } - if (y >= height) + if (LL_UNLIKELY(y >= mPickMaskHeight)) { LL_WARNS_ONCE("render") << "Ooh, height overrun on pick mask read, that woulda been bad." << LL_ENDL; - y = llmax(0, height-1); + y = llmax(0, mPickMaskHeight-1); } - S32 idx = y*width+x; + S32 idx = y*mPickMaskWidth+x; S32 offset = idx%8; res = mPickMask[idx/8] & (1 << offset) ? TRUE : FALSE; diff --git a/indra/llrender/llimagegl.h b/indra/llrender/llimagegl.h index facfb7bd62..f0870c3fc4 100644 --- a/indra/llrender/llimagegl.h +++ b/indra/llrender/llimagegl.h @@ -193,6 +193,8 @@ public: private: LLPointer mSaveData; // used for destroyGL/restoreGL U8* mPickMask; //downsampled bitmap approximation of alpha channel. NULL if no alpha channel + U16 mPickMaskWidth; + U16 mPickMaskHeight; S8 mUseMipMaps; S8 mHasExplicitFormat; // If false (default), GL format is f(mComponents) S8 mAutoGenMips; -- cgit v1.3 From 132cb6dbfe256cbf1a7b7558b091a63f99a364be Mon Sep 17 00:00:00 2001 From: Tofu Linden Date: Wed, 27 Jan 2010 13:58:04 -0800 Subject: CID-126 Checker: NO_EFFECT Function: LLFontBitmapCache::getImageGL(unsigned int) const File: /indra/llrender/llfontbitmapcache.cpp --- indra/llrender/llfontbitmapcache.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'indra/llrender') diff --git a/indra/llrender/llfontbitmapcache.cpp b/indra/llrender/llfontbitmapcache.cpp index f01878642a..cc9a012f98 100644 --- a/indra/llrender/llfontbitmapcache.cpp +++ b/indra/llrender/llfontbitmapcache.cpp @@ -72,7 +72,7 @@ LLImageRaw *LLFontBitmapCache::getImageRaw(U32 bitmap_num) const LLImageGL *LLFontBitmapCache::getImageGL(U32 bitmap_num) const { - if ((bitmap_num < 0) || (bitmap_num >= mImageGLVec.size())) + if (bitmap_num >= mImageGLVec.size()) return NULL; return mImageGLVec[bitmap_num]; -- cgit v1.3 From ccacd7d7acb1a24b754008cd35e9552fed93a1d1 Mon Sep 17 00:00:00 2001 From: Tofu Linden Date: Wed, 27 Jan 2010 13:58:48 -0800 Subject: CID-125 Checker: NO_EFFECT Function: LLFontBitmapCache::getImageRaw(unsigned int) const File: /indra/llrender/llfontbitmapcache.cpp --- indra/llrender/llfontbitmapcache.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'indra/llrender') diff --git a/indra/llrender/llfontbitmapcache.cpp b/indra/llrender/llfontbitmapcache.cpp index cc9a012f98..fa231c9e6a 100644 --- a/indra/llrender/llfontbitmapcache.cpp +++ b/indra/llrender/llfontbitmapcache.cpp @@ -64,7 +64,7 @@ void LLFontBitmapCache::init(S32 num_components, LLImageRaw *LLFontBitmapCache::getImageRaw(U32 bitmap_num) const { - if ((bitmap_num < 0) || (bitmap_num >= mImageRawVec.size())) + if (bitmap_num >= mImageRawVec.size()) return NULL; return mImageRawVec[bitmap_num]; -- cgit v1.3 From e70d7fca4b55d15547082a67101d5f8935d3a40a Mon Sep 17 00:00:00 2001 From: Tofu Linden Date: Wed, 27 Jan 2010 14:04:02 -0800 Subject: CID-155 Checker: OVERRUN_DYNAMIC Function: LLImageGL::updatePickMask(int, int, const unsigned char *) File: /indra/llrender/llimagegl.cpp not a bug, AFAICT. --- indra/llrender/llimagegl.cpp | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) (limited to 'indra/llrender') diff --git a/indra/llrender/llimagegl.cpp b/indra/llrender/llimagegl.cpp index 46478ba3c9..8bcc4723ae 100644 --- a/indra/llrender/llimagegl.cpp +++ b/indra/llrender/llimagegl.cpp @@ -1716,10 +1716,7 @@ void LLImageGL::updatePickMask(S32 width, S32 height, const U8* data_in) { U32 pick_idx = pick_bit/8; U32 pick_offset = pick_bit%8; - if (pick_idx >= size) - { - llerrs << "WTF?" << llendl; - } + llassert(pick_idx < size); mPickMask[pick_idx] |= 1 << pick_offset; } -- cgit v1.3 From 68422d9505ecf78e793bb5fb29eb8db58e6b6f89 Mon Sep 17 00:00:00 2001 From: Tofu Linden Date: Wed, 27 Jan 2010 14:15:02 -0800 Subject: CID 159 CID 158 CID 157 Checker: OVERRUN_STATIC Function: LLVertexBuffer::drawRange(unsigned int, unsigned int, unsigned int, unsigned int, unsigned int) const File: /indra/llrender/llvertexbuffer.cpp --- indra/llrender/llvertexbuffer.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'indra/llrender') diff --git a/indra/llrender/llvertexbuffer.cpp b/indra/llrender/llvertexbuffer.cpp index 572ae13909..ecfe845b34 100644 --- a/indra/llrender/llvertexbuffer.cpp +++ b/indra/llrender/llvertexbuffer.cpp @@ -237,7 +237,7 @@ void LLVertexBuffer::drawRange(U32 mode, U32 start, U32 end, U32 count, U32 indi llerrs << "Wrong vertex buffer bound." << llendl; } - if (mode > LLRender::NUM_MODES) + if (mode >= LLRender::NUM_MODES) { llerrs << "Invalid draw mode: " << mode << llendl; return; @@ -267,7 +267,7 @@ void LLVertexBuffer::draw(U32 mode, U32 count, U32 indices_offset) const llerrs << "Wrong vertex buffer bound." << llendl; } - if (mode > LLRender::NUM_MODES) + if (mode >= LLRender::NUM_MODES) { llerrs << "Invalid draw mode: " << mode << llendl; return; @@ -292,7 +292,7 @@ void LLVertexBuffer::drawArrays(U32 mode, U32 first, U32 count) const llerrs << "Wrong vertex buffer bound." << llendl; } - if (mode > LLRender::NUM_MODES) + if (mode >= LLRender::NUM_MODES) { llerrs << "Invalid draw mode: " << mode << llendl; return; -- cgit v1.3