Changes to protect against use of normalize3fast on degenerate vectors

5 files changed, 44 insertions, 9 deletions

diff --git a/indra/newview/llface.cpp b/indra/newview/llface.cpp
index 3e503cb750..b34370fa87 100755
--- a/indra/newview/llface.cpp
+++ b/indra/newview/llface.cpp
@@ -817,6 +817,12 @@ BOOL LLFace::genVolumeBBoxes(const LLVolume &volume, S32 f,
 			size.mul(scale);
 		}
 
+		// Catch potential badness from normalization before it happens
+		//
+		llassert(mat_normal.mMatrix[0].isFinite3() && (mat_normal.mMatrix[0].dot3(mat_normal.mMatrix[0]).getF32() > F_APPROXIMATELY_ZERO));
+		llassert(mat_normal.mMatrix[1].isFinite3() && (mat_normal.mMatrix[1].dot3(mat_normal.mMatrix[1]).getF32() > F_APPROXIMATELY_ZERO));
+		llassert(mat_normal.mMatrix[2].isFinite3() && (mat_normal.mMatrix[2].dot3(mat_normal.mMatrix[2]).getF32() > F_APPROXIMATELY_ZERO));
+
 		mat_normal.mMatrix[0].normalize3fast();
 		mat_normal.mMatrix[1].normalize3fast();
 		mat_normal.mMatrix[2].normalize3fast();
@@ -936,7 +942,9 @@ LLVector2 LLFace::surfaceToTexture(LLVector2 surface_coord, const LLVector4a& po
 		LLVector4a volume_normal;
 		LLVector3 v_normal(normal.getF32ptr());
 		volume_normal.load3(mDrawablep->getVOVolume()->agentDirectionToVolume(v_normal).mV);
-		volume_normal.normalize3fast();
+		LLVector4a default_norm;
+		default_norm.set(0,1,0,1);
+		volume_normal.normalize3fast_checked(&default_norm);
 		
 		if (texgen == LLTextureEntry::TEX_GEN_PLANAR)
 		{
@@ -1909,7 +1917,10 @@ BOOL LLFace::getGeometryVolume(const LLVolume& volume,
 							binormal.load3(t.mV);
 						}
 
-						binormal.normalize3fast();
+						LLVector4a default_binorm;
+						default_binorm.set(1,0,0,1);
+						binormal.normalize3fast_checked(&default_binorm);
+
 						LLVector2 tc = bump_tc[i];
 						tc += LLVector2( bump_s_primary_light_ray.dot3(tangent).getF32(), bump_t_primary_light_ray.dot3(binormal).getF32() );
 					
@@ -1996,12 +2007,13 @@ BOOL LLFace::getGeometryVolume(const LLVolume& volume,
 			LLFastTimer t(FTM_FACE_GEOM_NORMAL);
 			mVertexBuffer->getNormalStrider(norm, mGeomIndex, mGeomCount, map_range);
 			F32* normals = (F32*) norm.get();
-	
+			LLVector4a default_norm;
+			default_norm.set(0,1,0,1);
 			for (S32 i = 0; i < num_vertices; i++)
 			{	
 				LLVector4a normal;
 				mat_normal.rotate(vf.mNormals[i], normal);
-				normal.normalize3fast();
+				normal.normalize3fast_checked(&default_norm);
 				normal.store4a(normals);
 				normals += 4;
 			}
@@ -2024,12 +2036,14 @@ BOOL LLFace::getGeometryVolume(const LLVolume& volume,
 			mask.clear();
 			mask.setElement<3>();
 
+			LLVector4a default_tangent;
+			default_tangent.set(0,0,1,1);
+
 			for (S32 i = 0; i < num_vertices; i++)
 			{
 				LLVector4a tangent_out;
 				mat_normal.rotate(vf.mTangents[i], tangent_out);
-				tangent_out.normalize3fast();
-				
+				tangent_out.normalize3fast_checked(&default_tangent);
 				tangent_out.setSelectWithMask(mask, vf.mTangents[i], tangent_out);
 				tangent_out.store4a(tangents);
 				
@@ -2244,7 +2258,10 @@ BOOL LLFace::calcPixelArea(F32& cos_angle_to_view_dir, F32& radius)
 		dist *= 16.f;
 	}
 
-	lookAt.normalize3fast() ;	
+	LLVector4a default_lookat;
+	default_lookat.set(0,0,1,1);
+
+	lookAt.normalize3fast_checked(&default_lookat);	
 
 	//get area of circle around node
 	F32 app_angle = atanf((F32) sqrt(size_squared) / dist);
diff --git a/indra/newview/llspatialpartition.cpp b/indra/newview/llspatialpartition.cpp
index 78401020a6..dc99fd469b 100755
--- a/indra/newview/llspatialpartition.cpp
+++ b/indra/newview/llspatialpartition.cpp
@@ -1259,12 +1259,15 @@ F32 LLSpatialPartition::calcDistance(LLSpatialGroup* group, LLCamera& camera)
 
 	F32 dist = 0.f;
 
+	LLVector4a default_eyevec;
+	default_eyevec.set(0,0,1,1);
+
 	if (group->mDrawMap.find(LLRenderPass::PASS_ALPHA) != group->mDrawMap.end())
 	{
 		LLVector4a v = eye;
 
 		dist = eye.getLength3().getF32();
-		eye.normalize3fast();
+		eye.normalize3fast_checked(&default_eyevec);
 
 		if (!group->isState(LLSpatialGroup::ALPHA_DIRTY))
 		{
diff --git a/indra/newview/llvopartgroup.cpp b/indra/newview/llvopartgroup.cpp
index 6a7f26bdb5..b25213d85f 100755
--- a/indra/newview/llvopartgroup.cpp
+++ b/indra/newview/llvopartgroup.cpp
@@ -411,14 +411,21 @@ void LLVOPartGroup::getGeometry(S32 idx,
 	LLVector4a right;
 
 	right.setCross3(at, up);
+	// guard against NaNs in normalize below
+	llassert(right.dot3(right).getF32() > F_APPROXIMATELY_ZERO);
 	right.normalize3fast();
+
 	up.setCross3(right, at);
+	// guard against NaNs in normalize below
+	llassert(up.dot3(up).getF32() > F_APPROXIMATELY_ZERO);
 	up.normalize3fast();
 
 	if (part.mFlags & LLPartData::LL_PART_FOLLOW_VELOCITY_MASK)
 	{
 		LLVector4a normvel;
 		normvel.load3(part.mVelocity.mV);
+		// guard against NaNs in normalize below
+		llassert(normvel.dot3(normvel).getF32() > F_APPROXIMATELY_ZERO);
 		normvel.normalize3fast();
 		LLVector2 up_fracs;
 		up_fracs.mV[0] = normvel.dot3(right).getF32();
@@ -443,6 +450,9 @@ void LLVOPartGroup::getGeometry(S32 idx,
 
 		up = new_up;
 		right = t;
+		// guard against NaNs in normalize below
+		llassert(up.dot3(up).getF32() > F_APPROXIMATELY_ZERO);
+		llassert(right.dot3(right).getF32() > F_APPROXIMATELY_ZERO);
 		up.normalize3fast();
 		right.normalize3fast();
 	}
diff --git a/indra/newview/llvovolume.cpp b/indra/newview/llvovolume.cpp
index 0aa56fcc0f..8962d7cadf 100755
--- a/indra/newview/llvovolume.cpp
+++ b/indra/newview/llvovolume.cpp
@@ -3751,7 +3751,8 @@ BOOL LLVOVolume::lineSegmentIntersect(const LLVector4a& start, const LLVector4a&
 						{
 							*normal = n;
 						}
-
+						// guard against NaNs in normalize below
+						llassert(normal->dot3(*normal).getF32() > F_APPROXIMATELY_ZERO);
 						(*normal).normalize3fast();
 					}
 
@@ -3774,6 +3775,8 @@ BOOL LLVOVolume::lineSegmentIntersect(const LLVector4a& start, const LLVector4a&
 						{
 							*tangent = tn;
 						}
+						// guard against NaNs in normalize below
+						llassert(tangent->dot3(*tangent).getF32() > F_APPROXIMATELY_ZERO);
 						(*tangent).normalize3fast();
 					}
 
diff --git a/indra/newview/pipeline.cpp b/indra/newview/pipeline.cpp
index 05ef8060d4..72912db041 100755
--- a/indra/newview/pipeline.cpp
+++ b/indra/newview/pipeline.cpp
@@ -10569,11 +10569,13 @@ void LLPipeline::generateImpostor(LLVOAvatar* avatar)
 		LLVector4a left;
 		left.load3(camera.getLeftAxis().mV);
 		left.mul(left);
+		llassert(left.dot3(left).getF32() > F_APPROXIMATELY_ZERO);
 		left.normalize3fast();
 
 		LLVector4a up;
 		up.load3(camera.getUpAxis().mV);
 		up.mul(up);
+		llassert(up.dot3(up).getF32() > F_APPROXIMATELY_ZERO);
 		up.normalize3fast();
 
 		tdim.mV[0] = fabsf(half_height.dot3(left).getF32());