summaryrefslogtreecommitdiff
path: root/indra
diff options
context:
space:
mode:
authorTofu Linden <tofu.linden@lindenlab.com>2010-01-25 16:35:50 -0800
committerTofu Linden <tofu.linden@lindenlab.com>2010-01-25 16:35:50 -0800
commitb00813ae3ca14a7531d93d35d795ce7c23119886 (patch)
treef7f8b911b8a48682ee47f953e0b91f82a541bfec /indra
parentdbcb6feb132a502598ebd299beb9656455bf4650 (diff)
EXT-4388 Crash in octree line segment intersection code
a more complete fix and some sanity to prevent recurrance of a similar problem. reviewed by bao!
Diffstat (limited to 'indra')
-rw-r--r--indra/llrender/llimagegl.cpp43
-rw-r--r--indra/llrender/llimagegl.h2
2 files changed, 27 insertions, 18 deletions
diff --git a/indra/llrender/llimagegl.cpp b/indra/llrender/llimagegl.cpp
index cd493481d5..46478ba3c9 100644
--- a/indra/llrender/llimagegl.cpp
+++ b/indra/llrender/llimagegl.cpp
@@ -436,6 +436,8 @@ void LLImageGL::init(BOOL usemipmaps)
mLastBindTime = 0.f;
mPickMask = NULL;
+ mPickMaskWidth = 0;
+ mPickMaskHeight = 0;
mUseMipMaps = usemipmaps;
mHasExplicitFormat = FALSE;
mAutoGenMips = FALSE;
@@ -527,7 +529,12 @@ void LLImageGL::setSize(S32 width, S32 height, S32 ncomponents)
// llwarns << "Setting Size of LLImageGL with existing mTexName = " << mTexName << llendl;
destroyGLTexture();
}
-
+
+ // pickmask validity depends on old image size, delete it
+ delete [] mPickMask;
+ mPickMask = NULL;
+ mPickMaskWidth = mPickMaskHeight = 0;
+
mWidth = width;
mHeight = height;
mComponents = ncomponents;
@@ -1675,12 +1682,14 @@ void LLImageGL::updatePickMask(S32 width, S32 height, const U8* data_in)
return ;
}
+ delete [] mPickMask;
+ mPickMask = NULL;
+ mPickMaskWidth = mPickMaskHeight = 0;
+
if (mFormatType != GL_UNSIGNED_BYTE ||
mFormatPrimary != GL_RGBA)
{
//cannot generate a pick mask for this texture
- delete [] mPickMask;
- mPickMask = NULL;
return;
}
@@ -1688,11 +1697,10 @@ void LLImageGL::updatePickMask(S32 width, S32 height, const U8* data_in)
U32 pick_height = height/2;
U32 size = llmax(pick_width, (U32) 1) * llmax(pick_height, (U32) 1);
-
size = size/8 + 1;
-
- delete[] mPickMask;
mPickMask = new U8[size];
+ mPickMaskWidth = pick_width;
+ mPickMaskHeight = pick_height;
memset(mPickMask, 0, sizeof(U8) * size);
@@ -1727,35 +1735,34 @@ BOOL LLImageGL::getMask(const LLVector2 &tc)
if (mPickMask)
{
- S32 width = getWidth()/2;
- S32 height = getHeight()/2;
-
F32 u = tc.mV[0] - floorf(tc.mV[0]);
F32 v = tc.mV[1] - floorf(tc.mV[1]);
- if (u < 0.f || u > 1.f ||
- v < 0.f || v > 1.f)
+ if (LL_UNLIKELY(u < 0.f || u > 1.f ||
+ v < 0.f || v > 1.f))
{
LL_WARNS_ONCE("render") << "Ugh, u/v out of range in image mask pick" << LL_ENDL;
u = v = 0.f;
llassert(false);
}
+
+ llassert(mPickMaskWidth > 0 && mPickMaskHeight > 0);
- S32 x = (S32)(u * width);
- S32 y = (S32)(v * height);
+ S32 x = (S32)(u * mPickMaskWidth);
+ S32 y = (S32)(v * mPickMaskHeight);
- if (x >= width)
+ if (LL_UNLIKELY(x >= mPickMaskWidth))
{
LL_WARNS_ONCE("render") << "Ooh, width overrun on pick mask read, that coulda been bad." << LL_ENDL;
- x = llmax(0, width-1);
+ x = llmax(0, mPickMaskWidth-1);
}
- if (y >= height)
+ if (LL_UNLIKELY(y >= mPickMaskHeight))
{
LL_WARNS_ONCE("render") << "Ooh, height overrun on pick mask read, that woulda been bad." << LL_ENDL;
- y = llmax(0, height-1);
+ y = llmax(0, mPickMaskHeight-1);
}
- S32 idx = y*width+x;
+ S32 idx = y*mPickMaskWidth+x;
S32 offset = idx%8;
res = mPickMask[idx/8] & (1 << offset) ? TRUE : FALSE;
diff --git a/indra/llrender/llimagegl.h b/indra/llrender/llimagegl.h
index facfb7bd62..f0870c3fc4 100644
--- a/indra/llrender/llimagegl.h
+++ b/indra/llrender/llimagegl.h
@@ -193,6 +193,8 @@ public:
private:
LLPointer<LLImageRaw> mSaveData; // used for destroyGL/restoreGL
U8* mPickMask; //downsampled bitmap approximation of alpha channel. NULL if no alpha channel
+ U16 mPickMaskWidth;
+ U16 mPickMaskHeight;
S8 mUseMipMaps;
S8 mHasExplicitFormat; // If false (default), GL format is f(mComponents)
S8 mAutoGenMips;