From 6db14e3988a1248141d085e5005baa70e48f8172 Mon Sep 17 00:00:00 2001 From: simon Date: Tue, 30 Jul 2013 15:12:02 -0700 Subject: MAINT-2913: crash in LLCamera::AABBInFrustum. Tried to make array indexes bullet-proof. Reviewed by Kelly --- indra/llmath/llcamera.cpp | 86 ++++++++++++++++++++++++---------------------- indra/llmath/llcamera.h | 37 +++++++++++++------- indra/newview/pipeline.cpp | 18 +++++----- 3 files changed, 78 insertions(+), 63 deletions(-) (limited to 'indra') diff --git a/indra/llmath/llcamera.cpp b/indra/llmath/llcamera.cpp index c14c117da9..33cf185196 100755 --- a/indra/llmath/llcamera.cpp +++ b/indra/llmath/llcamera.cpp @@ -42,6 +42,11 @@ LLCamera::LLCamera() : mPlaneCount(6), mFrustumCornerDist(0.f) { + for (U32 i = 0; i < PLANE_MASK_NUM; i++) + { + mPlaneMask[i] = PLANE_MASK_NONE; + } + calculateFrustumPlanes(); } @@ -52,6 +57,11 @@ LLCamera::LLCamera(F32 vertical_fov_rads, F32 aspect_ratio, S32 view_height_in_p mPlaneCount(6), mFrustumCornerDist(0.f) { + for (U32 i = 0; i < PLANE_MASK_NUM; i++) + { + mPlaneMask[i] = PLANE_MASK_NONE; + } + mAspect = llclamp(aspect_ratio, MIN_ASPECT_RATIO, MAX_ASPECT_RATIO); mNearPlane = llclamp(near_plane, MIN_NEAR_PLANE, MAX_NEAR_PLANE); if(far_plane < 0) far_plane = DEFAULT_FAR_PLANE; @@ -87,14 +97,14 @@ F32 LLCamera::getMaxView() const void LLCamera::setUserClipPlane(LLPlane& plane) { - mPlaneCount = 7; - mAgentPlanes[6] = plane; - mPlaneMask[6] = plane.calcPlaneMask(); + mPlaneCount = AGENT_PLANE_USER_CLIP_NUM; + mAgentPlanes[AGENT_PLANE_USER_CLIP] = plane; + mPlaneMask[AGENT_PLANE_USER_CLIP] = plane.calcPlaneMask(); } void LLCamera::disableUserClipPlane() { - mPlaneCount = 6; + mPlaneCount = AGENT_PLANE_NO_USER_CLIP_NUM; } void LLCamera::setView(F32 vertical_fov_rads) @@ -161,31 +171,33 @@ size_t LLCamera::readFrustumFromBuffer(const char *buffer) // ---------------- test methods ---------------- -S32 LLCamera::AABBInFrustum(const LLVector4a ¢er, const LLVector4a& radius) +static const LLVector4a sFrustumScaler[] = { - const LLVector4a scaler[] = { - LLVector4a(-1,-1,-1), - LLVector4a( 1,-1,-1), - LLVector4a(-1, 1,-1), - LLVector4a( 1, 1,-1), - LLVector4a(-1,-1, 1), - LLVector4a( 1,-1, 1), - LLVector4a(-1, 1, 1), - LLVector4a( 1, 1, 1) - }; + LLVector4a(-1,-1,-1), + LLVector4a( 1,-1,-1), + LLVector4a(-1, 1,-1), + LLVector4a( 1, 1,-1), + LLVector4a(-1,-1, 1), + LLVector4a( 1,-1, 1), + LLVector4a(-1, 1, 1), + LLVector4a( 1, 1, 1) // 8 entries +}; +S32 LLCamera::AABBInFrustum(const LLVector4a ¢er, const LLVector4a& radius) +{ U8 mask = 0; bool result = false; LLVector4a rscale, maxp, minp; LLSimdScalar d; - for (U32 i = 0; i < mPlaneCount; i++) + U32 max_planes = llmin(mPlaneCount, (U32) AGENT_PLANE_USER_CLIP_NUM); // mAgentPlanes[] size is 7 + for (U32 i = 0; i < max_planes; i++) { mask = mPlaneMask[i]; - if (mask != 0xff) + if (mask < PLANE_MASK_NUM) { const LLPlane& p(mAgentPlanes[i]); p.getAt<3>(d); - rscale.setMul(radius, scaler[mask]); + rscale.setMul(radius, sFrustumScaler[mask]); minp.setSub(center, rscale); d = -d; if (p.dot3(minp).getF32() > d) @@ -207,29 +219,19 @@ S32 LLCamera::AABBInFrustum(const LLVector4a ¢er, const LLVector4a& radius) S32 LLCamera::AABBInFrustumNoFarClip(const LLVector4a& center, const LLVector4a& radius) { - const LLVector4a scaler[] = { - LLVector4a(-1,-1,-1), - LLVector4a( 1,-1,-1), - LLVector4a(-1, 1,-1), - LLVector4a( 1, 1,-1), - LLVector4a(-1,-1, 1), - LLVector4a( 1,-1, 1), - LLVector4a(-1, 1, 1), - LLVector4a( 1, 1, 1) - }; - U8 mask = 0; bool result = false; LLVector4a rscale, maxp, minp; LLSimdScalar d; - for (U32 i = 0; i < mPlaneCount; i++) + U32 max_planes = llmin(mPlaneCount, (U32) AGENT_PLANE_USER_CLIP_NUM); // mAgentPlanes[] size is 7 + for (U32 i = 0; i < max_planes; i++) { mask = mPlaneMask[i]; - if ((i != 5) && (mask != 0xff)) + if ((i != 5) && (mask < PLANE_MASK_NUM)) { const LLPlane& p(mAgentPlanes[i]); p.getAt<3>(d); - rscale.setMul(radius, scaler[mask]); + rscale.setMul(radius, sFrustumScaler[mask]); minp.setSub(center, rscale); d = -d; if (p.dot3(minp).getF32() > d) @@ -369,7 +371,7 @@ int LLCamera::sphereInFrustum(const LLVector3 &sphere_center, const F32 radius) bool res = false; for (int i = 0; i < 6; i++) { - if (mPlaneMask[i] != 0xff) + if (mPlaneMask[i] != PLANE_MASK_NONE) { float d = mAgentPlanes[i].dist(sphere_center); @@ -541,14 +543,14 @@ void LLCamera::ignoreAgentFrustumPlane(S32 idx) return; } - mPlaneMask[idx] = 0xff; + mPlaneMask[idx] = PLANE_MASK_NONE; mAgentPlanes[idx].clear(); } void LLCamera::calcAgentFrustumPlanes(LLVector3* frust) { - for (int i = 0; i < 8; i++) + for (int i = 0; i < AGENT_FRUSTRUM_NUM; i++) { mAgentFrustum[i] = frust[i]; } @@ -560,22 +562,22 @@ void LLCamera::calcAgentFrustumPlanes(LLVector3* frust) //order of planes is important, keep most likely to fail in the front of the list //near - frust[0], frust[1], frust[2] - mAgentPlanes[2] = planeFromPoints(frust[0], frust[1], frust[2]); + mAgentPlanes[AGENT_PLANE_NEAR] = planeFromPoints(frust[0], frust[1], frust[2]); //far - mAgentPlanes[5] = planeFromPoints(frust[5], frust[4], frust[6]); + mAgentPlanes[AGENT_PLANE_FAR] = planeFromPoints(frust[5], frust[4], frust[6]); //left - mAgentPlanes[0] = planeFromPoints(frust[4], frust[0], frust[7]); + mAgentPlanes[AGENT_PLANE_LEFT] = planeFromPoints(frust[4], frust[0], frust[7]); //right - mAgentPlanes[1] = planeFromPoints(frust[1], frust[5], frust[6]); + mAgentPlanes[AGENT_PLANE_RIGHT] = planeFromPoints(frust[1], frust[5], frust[6]); //top - mAgentPlanes[4] = planeFromPoints(frust[3], frust[2], frust[6]); + mAgentPlanes[AGENT_PLANE_TOP] = planeFromPoints(frust[3], frust[2], frust[6]); //bottom - mAgentPlanes[3] = planeFromPoints(frust[1], frust[0], frust[4]); + mAgentPlanes[AGENT_PLANE_BOTTOM] = planeFromPoints(frust[1], frust[0], frust[4]); //cache plane octant facing mask for use in AABBInFrustum for (U32 i = 0; i < mPlaneCount; i++) @@ -635,7 +637,7 @@ void LLCamera::calculateWorldFrustumPlanes() LLVector3 center = mOrigin - mXAxis*mNearPlane; mWorldPlanePos = center; LLVector3 pnorm; - for (int p=0; p<4; p++) + for (int p = 0; p < PLANE_NUM; p++) { mLocalPlanes[p].getVector3(pnorm); LLVector3 norm = rotateToAbsolute(pnorm); diff --git a/indra/llmath/llcamera.h b/indra/llmath/llcamera.h index 0b591be622..1283cfb16b 100755 --- a/indra/llmath/llcamera.h +++ b/indra/llmath/llcamera.h @@ -76,26 +76,39 @@ public: PLANE_RIGHT = 1, PLANE_BOTTOM = 2, PLANE_TOP = 3, - PLANE_NUM = 4 + PLANE_NUM = 4, + PLANE_MASK_NONE = 0xff // Disable this plane }; enum { PLANE_LEFT_MASK = (1< 3) { //render shadow frusta as volumes if (mShadowFrustPoints[i-4].empty()) - { + { continue; } @@ -9651,7 +9651,7 @@ BOOL LLPipeline::getVisiblePointCloud(LLCamera& camera, LLVector3& min, LLVector pp.push_back(LLVector3(max.mV[0], max.mV[1], max.mV[2])); //add corners of camera frustum - for (U32 i = 0; i < 8; i++) + for (U32 i = 0; i < LLCamera::AGENT_FRUSTRUM_NUM; i++) { pp.push_back(camera.mAgentFrustum[i]); } @@ -9678,7 +9678,7 @@ BOOL LLPipeline::getVisiblePointCloud(LLCamera& camera, LLVector3& min, LLVector for (U32 i = 0; i < 12; i++) { //for each line segment in bounding box - for (U32 j = 0; j < 6; j++) + for (U32 j = 0; j < LLCamera::AGENT_PLANE_NO_USER_CLIP_NUM; j++) { //for each plane in camera frustum const LLPlane& cp = camera.getAgentPlane(j); const LLVector3& v1 = pp[bs[i*2+0]]; @@ -9764,19 +9764,19 @@ BOOL LLPipeline::getVisiblePointCloud(LLCamera& camera, LLVector3& min, LLVector } } - for (U32 j = 0; j < 6; ++j) + for (U32 j = 0; j < LLCamera::AGENT_PLANE_NO_USER_CLIP_NUM; ++j) { const LLPlane& cp = camera.getAgentPlane(j); F32 dist = cp.dist(pp[i]); if (dist > 0.05f) //point is above some plane, not contained - { + { found = false; break; - } - } + } + } - if (found) - { + if (found) + { fp.push_back(pp[i]); } } -- cgit v1.2.3