diff options
author | Tofu Linden <tofu.linden@lindenlab.com> | 2010-01-25 16:35:50 -0800 |
---|---|---|
committer | Tofu Linden <tofu.linden@lindenlab.com> | 2010-01-25 16:35:50 -0800 |
commit | b00813ae3ca14a7531d93d35d795ce7c23119886 (patch) | |
tree | f7f8b911b8a48682ee47f953e0b91f82a541bfec /indra | |
parent | dbcb6feb132a502598ebd299beb9656455bf4650 (diff) |
EXT-4388 Crash in octree line segment intersection code
a more complete fix and some sanity to prevent recurrance of a similar problem.
reviewed by bao!
Diffstat (limited to 'indra')
-rw-r--r-- | indra/llrender/llimagegl.cpp | 43 | ||||
-rw-r--r-- | indra/llrender/llimagegl.h | 2 |
2 files changed, 27 insertions, 18 deletions
diff --git a/indra/llrender/llimagegl.cpp b/indra/llrender/llimagegl.cpp index cd493481d5..46478ba3c9 100644 --- a/indra/llrender/llimagegl.cpp +++ b/indra/llrender/llimagegl.cpp @@ -436,6 +436,8 @@ void LLImageGL::init(BOOL usemipmaps) mLastBindTime = 0.f; mPickMask = NULL; + mPickMaskWidth = 0; + mPickMaskHeight = 0; mUseMipMaps = usemipmaps; mHasExplicitFormat = FALSE; mAutoGenMips = FALSE; @@ -527,7 +529,12 @@ void LLImageGL::setSize(S32 width, S32 height, S32 ncomponents) // llwarns << "Setting Size of LLImageGL with existing mTexName = " << mTexName << llendl; destroyGLTexture(); } - + + // pickmask validity depends on old image size, delete it + delete [] mPickMask; + mPickMask = NULL; + mPickMaskWidth = mPickMaskHeight = 0; + mWidth = width; mHeight = height; mComponents = ncomponents; @@ -1675,12 +1682,14 @@ void LLImageGL::updatePickMask(S32 width, S32 height, const U8* data_in) return ; } + delete [] mPickMask; + mPickMask = NULL; + mPickMaskWidth = mPickMaskHeight = 0; + if (mFormatType != GL_UNSIGNED_BYTE || mFormatPrimary != GL_RGBA) { //cannot generate a pick mask for this texture - delete [] mPickMask; - mPickMask = NULL; return; } @@ -1688,11 +1697,10 @@ void LLImageGL::updatePickMask(S32 width, S32 height, const U8* data_in) U32 pick_height = height/2; U32 size = llmax(pick_width, (U32) 1) * llmax(pick_height, (U32) 1); - size = size/8 + 1; - - delete[] mPickMask; mPickMask = new U8[size]; + mPickMaskWidth = pick_width; + mPickMaskHeight = pick_height; memset(mPickMask, 0, sizeof(U8) * size); @@ -1727,35 +1735,34 @@ BOOL LLImageGL::getMask(const LLVector2 &tc) if (mPickMask) { - S32 width = getWidth()/2; - S32 height = getHeight()/2; - F32 u = tc.mV[0] - floorf(tc.mV[0]); F32 v = tc.mV[1] - floorf(tc.mV[1]); - if (u < 0.f || u > 1.f || - v < 0.f || v > 1.f) + if (LL_UNLIKELY(u < 0.f || u > 1.f || + v < 0.f || v > 1.f)) { LL_WARNS_ONCE("render") << "Ugh, u/v out of range in image mask pick" << LL_ENDL; u = v = 0.f; llassert(false); } + + llassert(mPickMaskWidth > 0 && mPickMaskHeight > 0); - S32 x = (S32)(u * width); - S32 y = (S32)(v * height); + S32 x = (S32)(u * mPickMaskWidth); + S32 y = (S32)(v * mPickMaskHeight); - if (x >= width) + if (LL_UNLIKELY(x >= mPickMaskWidth)) { LL_WARNS_ONCE("render") << "Ooh, width overrun on pick mask read, that coulda been bad." << LL_ENDL; - x = llmax(0, width-1); + x = llmax(0, mPickMaskWidth-1); } - if (y >= height) + if (LL_UNLIKELY(y >= mPickMaskHeight)) { LL_WARNS_ONCE("render") << "Ooh, height overrun on pick mask read, that woulda been bad." << LL_ENDL; - y = llmax(0, height-1); + y = llmax(0, mPickMaskHeight-1); } - S32 idx = y*width+x; + S32 idx = y*mPickMaskWidth+x; S32 offset = idx%8; res = mPickMask[idx/8] & (1 << offset) ? TRUE : FALSE; diff --git a/indra/llrender/llimagegl.h b/indra/llrender/llimagegl.h index facfb7bd62..f0870c3fc4 100644 --- a/indra/llrender/llimagegl.h +++ b/indra/llrender/llimagegl.h @@ -193,6 +193,8 @@ public: private: LLPointer<LLImageRaw> mSaveData; // used for destroyGL/restoreGL U8* mPickMask; //downsampled bitmap approximation of alpha channel. NULL if no alpha channel + U16 mPickMaskWidth; + U16 mPickMaskHeight; S8 mUseMipMaps; S8 mHasExplicitFormat; // If false (default), GL format is f(mComponents) S8 mAutoGenMips; |