MAINT-2913: crash in LLCamera::AABBInFrustum. Tried to make array indexes

bullet-proof.  Reviewed by Kelly

1 files changed, 44 insertions, 42 deletions

diff --git a/indra/llmath/llcamera.cpp b/indra/llmath/llcamera.cpp
index c14c117da9..33cf185196 100755
--- a/indra/llmath/llcamera.cpp
+++ b/indra/llmath/llcamera.cpp
@@ -42,6 +42,11 @@ LLCamera::LLCamera() :
 	mPlaneCount(6),
 	mFrustumCornerDist(0.f)
 {
+	for (U32 i = 0; i < PLANE_MASK_NUM; i++)
+	{
+		mPlaneMask[i] = PLANE_MASK_NONE;
+	}
+
 	calculateFrustumPlanes();
 } 
 
@@ -52,6 +57,11 @@ LLCamera::LLCamera(F32 vertical_fov_rads, F32 aspect_ratio, S32 view_height_in_p
 	mPlaneCount(6),
 	mFrustumCornerDist(0.f)
 {
+	for (U32 i = 0; i < PLANE_MASK_NUM; i++)
+	{
+		mPlaneMask[i] = PLANE_MASK_NONE;
+	}
+
 	mAspect = llclamp(aspect_ratio, MIN_ASPECT_RATIO, MAX_ASPECT_RATIO);
 	mNearPlane = llclamp(near_plane, MIN_NEAR_PLANE, MAX_NEAR_PLANE);
 	if(far_plane < 0) far_plane = DEFAULT_FAR_PLANE;
@@ -87,14 +97,14 @@ F32 LLCamera::getMaxView() const
 
 void LLCamera::setUserClipPlane(LLPlane& plane)
 {
-	mPlaneCount = 7;
-	mAgentPlanes[6] = plane;
-	mPlaneMask[6] = plane.calcPlaneMask();
+	mPlaneCount = AGENT_PLANE_USER_CLIP_NUM;
+	mAgentPlanes[AGENT_PLANE_USER_CLIP] = plane;
+	mPlaneMask[AGENT_PLANE_USER_CLIP] = plane.calcPlaneMask();
 }
 
 void LLCamera::disableUserClipPlane()
 {
-	mPlaneCount = 6;
+	mPlaneCount = AGENT_PLANE_NO_USER_CLIP_NUM;
 }
 
 void LLCamera::setView(F32 vertical_fov_rads) 
@@ -161,31 +171,33 @@ size_t LLCamera::readFrustumFromBuffer(const char *buffer)
 
 // ---------------- test methods  ---------------- 
 
-S32 LLCamera::AABBInFrustum(const LLVector4a &center, const LLVector4a& radius) 
+static	const LLVector4a sFrustumScaler[] = 
 {
-	const LLVector4a scaler[] = {
-		LLVector4a(-1,-1,-1),
-		LLVector4a( 1,-1,-1),
-		LLVector4a(-1, 1,-1),
-		LLVector4a( 1, 1,-1),
-		LLVector4a(-1,-1, 1),
-		LLVector4a( 1,-1, 1),
-		LLVector4a(-1, 1, 1),
-		LLVector4a( 1, 1, 1)
-	};
+	LLVector4a(-1,-1,-1),
+	LLVector4a( 1,-1,-1),
+	LLVector4a(-1, 1,-1),
+	LLVector4a( 1, 1,-1),
+	LLVector4a(-1,-1, 1),
+	LLVector4a( 1,-1, 1),
+	LLVector4a(-1, 1, 1),
+	LLVector4a( 1, 1, 1)		// 8 entries
+};
 
+S32 LLCamera::AABBInFrustum(const LLVector4a &center, const LLVector4a& radius) 
+{
 	U8 mask = 0;
 	bool result = false;
 	LLVector4a rscale, maxp, minp;
 	LLSimdScalar d;
-	for (U32 i = 0; i < mPlaneCount; i++)
+	U32 max_planes = llmin(mPlaneCount, (U32) AGENT_PLANE_USER_CLIP_NUM);		// mAgentPlanes[] size is 7
+	for (U32 i = 0; i < max_planes; i++)
 	{
 		mask = mPlaneMask[i];
-		if (mask != 0xff)
+		if (mask < PLANE_MASK_NUM)
 		{
 			const LLPlane& p(mAgentPlanes[i]);
 			p.getAt<3>(d);
-			rscale.setMul(radius, scaler[mask]);
+			rscale.setMul(radius, sFrustumScaler[mask]);
 			minp.setSub(center, rscale);
 			d = -d;
 			if (p.dot3(minp).getF32() > d) 
@@ -207,29 +219,19 @@ S32 LLCamera::AABBInFrustum(const LLVector4a &center, const LLVector4a& radius)
 
 S32 LLCamera::AABBInFrustumNoFarClip(const LLVector4a& center, const LLVector4a& radius) 
 {
-	const LLVector4a scaler[] = {
-		LLVector4a(-1,-1,-1),
-		LLVector4a( 1,-1,-1),
-		LLVector4a(-1, 1,-1),
-		LLVector4a( 1, 1,-1),
-		LLVector4a(-1,-1, 1),
-		LLVector4a( 1,-1, 1),
-		LLVector4a(-1, 1, 1),
-		LLVector4a( 1, 1, 1)
-	};
-
 	U8 mask = 0;
 	bool result = false;
 	LLVector4a rscale, maxp, minp;
 	LLSimdScalar d;
-	for (U32 i = 0; i < mPlaneCount; i++)
+	U32 max_planes = llmin(mPlaneCount, (U32) AGENT_PLANE_USER_CLIP_NUM);		// mAgentPlanes[] size is 7
+	for (U32 i = 0; i < max_planes; i++)
 	{
 		mask = mPlaneMask[i];
-		if ((i != 5) && (mask != 0xff))
+		if ((i != 5) && (mask < PLANE_MASK_NUM))
 		{
 			const LLPlane& p(mAgentPlanes[i]);
 			p.getAt<3>(d);
-			rscale.setMul(radius, scaler[mask]);
+			rscale.setMul(radius, sFrustumScaler[mask]);
 			minp.setSub(center, rscale);
 			d = -d;
 			if (p.dot3(minp).getF32() > d) 
@@ -369,7 +371,7 @@ int LLCamera::sphereInFrustum(const LLVector3 &sphere_center, const F32 radius)
 	bool res = false;
 	for (int i = 0; i < 6; i++)
 	{
-		if (mPlaneMask[i] != 0xff)
+		if (mPlaneMask[i] != PLANE_MASK_NONE)
 		{
 			float d = mAgentPlanes[i].dist(sphere_center);
 
@@ -541,14 +543,14 @@ void LLCamera::ignoreAgentFrustumPlane(S32 idx)
 		return;
 	}
 
-	mPlaneMask[idx] = 0xff;
+	mPlaneMask[idx] = PLANE_MASK_NONE;
 	mAgentPlanes[idx].clear();
 }
 
 void LLCamera::calcAgentFrustumPlanes(LLVector3* frust)
 {
 	
-	for (int i = 0; i < 8; i++)
+	for (int i = 0; i < AGENT_FRUSTRUM_NUM; i++)
 	{
 		mAgentFrustum[i] = frust[i];
 	}
@@ -560,22 +562,22 @@ void LLCamera::calcAgentFrustumPlanes(LLVector3* frust)
 	//order of planes is important, keep most likely to fail in the front of the list
 
 	//near - frust[0], frust[1], frust[2]
-	mAgentPlanes[2] = planeFromPoints(frust[0], frust[1], frust[2]);
+	mAgentPlanes[AGENT_PLANE_NEAR] = planeFromPoints(frust[0], frust[1], frust[2]);
 
 	//far  
-	mAgentPlanes[5] = planeFromPoints(frust[5], frust[4], frust[6]);
+	mAgentPlanes[AGENT_PLANE_FAR] = planeFromPoints(frust[5], frust[4], frust[6]);
 
 	//left  
-	mAgentPlanes[0] = planeFromPoints(frust[4], frust[0], frust[7]);
+	mAgentPlanes[AGENT_PLANE_LEFT] = planeFromPoints(frust[4], frust[0], frust[7]);
 
 	//right  
-	mAgentPlanes[1] = planeFromPoints(frust[1], frust[5], frust[6]);
+	mAgentPlanes[AGENT_PLANE_RIGHT] = planeFromPoints(frust[1], frust[5], frust[6]);
 
 	//top  
-	mAgentPlanes[4] = planeFromPoints(frust[3], frust[2], frust[6]);
+	mAgentPlanes[AGENT_PLANE_TOP] = planeFromPoints(frust[3], frust[2], frust[6]);
 
 	//bottom  
-	mAgentPlanes[3] = planeFromPoints(frust[1], frust[0], frust[4]);
+	mAgentPlanes[AGENT_PLANE_BOTTOM] = planeFromPoints(frust[1], frust[0], frust[4]);
 
 	//cache plane octant facing mask for use in AABBInFrustum
 	for (U32 i = 0; i < mPlaneCount; i++)
@@ -635,7 +637,7 @@ void LLCamera::calculateWorldFrustumPlanes()
 	LLVector3 center = mOrigin - mXAxis*mNearPlane;
 	mWorldPlanePos = center;
 	LLVector3 pnorm;	
-	for (int p=0; p<4; p++)
+	for (int p = 0; p < PLANE_NUM; p++)
 	{
 		mLocalPlanes[p].getVector3(pnorm);
 		LLVector3 norm = rotateToAbsolute(pnorm);