Changes to protect against use of normalize3fast on degenerate vectors

1 files changed, 24 insertions, 7 deletions

diff --git a/indra/newview/llface.cpp b/indra/newview/llface.cpp
index 3e503cb750..b34370fa87 100755
--- a/indra/newview/llface.cpp
+++ b/indra/newview/llface.cpp
@@ -817,6 +817,12 @@ BOOL LLFace::genVolumeBBoxes(const LLVolume &volume, S32 f,
 			size.mul(scale);
 		}
 
+		// Catch potential badness from normalization before it happens
+		//
+		llassert(mat_normal.mMatrix[0].isFinite3() && (mat_normal.mMatrix[0].dot3(mat_normal.mMatrix[0]).getF32() > F_APPROXIMATELY_ZERO));
+		llassert(mat_normal.mMatrix[1].isFinite3() && (mat_normal.mMatrix[1].dot3(mat_normal.mMatrix[1]).getF32() > F_APPROXIMATELY_ZERO));
+		llassert(mat_normal.mMatrix[2].isFinite3() && (mat_normal.mMatrix[2].dot3(mat_normal.mMatrix[2]).getF32() > F_APPROXIMATELY_ZERO));
+
 		mat_normal.mMatrix[0].normalize3fast();
 		mat_normal.mMatrix[1].normalize3fast();
 		mat_normal.mMatrix[2].normalize3fast();
@@ -936,7 +942,9 @@ LLVector2 LLFace::surfaceToTexture(LLVector2 surface_coord, const LLVector4a& po
 		LLVector4a volume_normal;
 		LLVector3 v_normal(normal.getF32ptr());
 		volume_normal.load3(mDrawablep->getVOVolume()->agentDirectionToVolume(v_normal).mV);
-		volume_normal.normalize3fast();
+		LLVector4a default_norm;
+		default_norm.set(0,1,0,1);
+		volume_normal.normalize3fast_checked(&default_norm);
 		
 		if (texgen == LLTextureEntry::TEX_GEN_PLANAR)
 		{
@@ -1909,7 +1917,10 @@ BOOL LLFace::getGeometryVolume(const LLVolume& volume,
 							binormal.load3(t.mV);
 						}
 
-						binormal.normalize3fast();
+						LLVector4a default_binorm;
+						default_binorm.set(1,0,0,1);
+						binormal.normalize3fast_checked(&default_binorm);
+
 						LLVector2 tc = bump_tc[i];
 						tc += LLVector2( bump_s_primary_light_ray.dot3(tangent).getF32(), bump_t_primary_light_ray.dot3(binormal).getF32() );
 					
@@ -1996,12 +2007,13 @@ BOOL LLFace::getGeometryVolume(const LLVolume& volume,
 			LLFastTimer t(FTM_FACE_GEOM_NORMAL);
 			mVertexBuffer->getNormalStrider(norm, mGeomIndex, mGeomCount, map_range);
 			F32* normals = (F32*) norm.get();
-	
+			LLVector4a default_norm;
+			default_norm.set(0,1,0,1);
 			for (S32 i = 0; i < num_vertices; i++)
 			{	
 				LLVector4a normal;
 				mat_normal.rotate(vf.mNormals[i], normal);
-				normal.normalize3fast();
+				normal.normalize3fast_checked(&default_norm);
 				normal.store4a(normals);
 				normals += 4;
 			}
@@ -2024,12 +2036,14 @@ BOOL LLFace::getGeometryVolume(const LLVolume& volume,
 			mask.clear();
 			mask.setElement<3>();
 
+			LLVector4a default_tangent;
+			default_tangent.set(0,0,1,1);
+
 			for (S32 i = 0; i < num_vertices; i++)
 			{
 				LLVector4a tangent_out;
 				mat_normal.rotate(vf.mTangents[i], tangent_out);
-				tangent_out.normalize3fast();
-				
+				tangent_out.normalize3fast_checked(&default_tangent);
 				tangent_out.setSelectWithMask(mask, vf.mTangents[i], tangent_out);
 				tangent_out.store4a(tangents);
 				
@@ -2244,7 +2258,10 @@ BOOL LLFace::calcPixelArea(F32& cos_angle_to_view_dir, F32& radius)
 		dist *= 16.f;
 	}
 
-	lookAt.normalize3fast() ;	
+	LLVector4a default_lookat;
+	default_lookat.set(0,0,1,1);
+
+	lookAt.normalize3fast_checked(&default_lookat);	
 
 	//get area of circle around node
 	F32 app_angle = atanf((F32) sqrt(size_squared) / dist);