Changes to protect against use of normalize3fast on degenerate vectors

9 files changed, 139 insertions, 83 deletions

diff --git a/indra/llappearance/llpolymorph.cpp b/indra/llappearance/llpolymorph.cpp
index 8a17819083..5e5813b9ac 100644
--- a/indra/llappearance/llpolymorph.cpp
+++ b/indra/llappearance/llpolymorph.cpp
@@ -568,6 +568,12 @@ void LLPolyMorphTarget::apply( ESex avatar_sex )
 
 		F32 *maskWeightArray = (mVertMask) ? mVertMask->getMorphMaskWeights() : NULL;
 
+		LLVector4a default_norm;
+		LLVector4a default_binorm;
+
+		default_norm.set(0,1,0,1);
+		default_binorm.set(1,0,0,1);
+
 		for(U32 vert_index_morph = 0; vert_index_morph < mMorphData->mNumIndices; vert_index_morph++)
 		{
 			S32 vert_index_mesh = mMorphData->mVertexIndices[vert_index_morph];
@@ -597,19 +603,31 @@ void LLPolyMorphTarget::apply( ESex avatar_sex )
 			norm.mul(delta_weight*maskWeight*NORMAL_SOFTEN_FACTOR);
 			scaled_normals[vert_index_mesh].add(norm);
 			norm = scaled_normals[vert_index_mesh];
-			norm.normalize3fast();
+
+			// guard against degenerate input data before we create NaNs below!
+			//
+			norm.normalize3fast_checked(&default_norm);
 			normals[vert_index_mesh] = norm;
 
 			// calculate new binormals
 			LLVector4a binorm = mMorphData->mBinormals[vert_index_morph];
+
+			// guard against degenerate input data before we create NaNs below!
+			//
+			if (!binorm.isFinite3() || (binorm.dot3(binorm).getF32() <= F_APPROXIMATELY_ZERO))
+			{
+				binorm.set(1,0,0,1);
+			}
+
 			binorm.mul(delta_weight*maskWeight*NORMAL_SOFTEN_FACTOR);
 			scaled_binormals[vert_index_mesh].add(binorm);
 			LLVector4a tangent;
 			tangent.setCross3(scaled_binormals[vert_index_mesh], norm);
 			LLVector4a& normalized_binormal = binormals[vert_index_mesh];
-			normalized_binormal.setCross3(norm, tangent); 
-			normalized_binormal.normalize3fast();
-			
+
+			normalized_binormal.setCross3(norm, tangent); 			
+			normalized_binormal.normalize3fast_checked(&default_binorm);
+
 			tex_coords[vert_index_mesh] += mMorphData->mTexCoords[vert_index_morph] * delta_weight * maskWeight;
 		}
 
diff --git a/indra/llmath/llvector4a.h b/indra/llmath/llvector4a.h
index 0526793d3a..94a61f2b1d 100755
--- a/indra/llmath/llvector4a.h
+++ b/indra/llmath/llvector4a.h
@@ -236,6 +236,11 @@ public:
 	// Note that this does not consider zero length vectors!
 	inline void normalize3fast();
 
+	// Normalize this vector with respect to the x, y, and z components only. Accurate only to 10-12 bits of precision. W component is destroyed
+	// Same as above except substitutes default vector contents if the vector is non-finite or degenerate due to zero length.
+	//
+	inline void normalize3fast_checked(LLVector4a* default = NULL);
+
 	// Return true if this vector is normalized with respect to x,y,z up to tolerance
 	inline LLBool32 isNormalized3( F32 tolerance = 1e-3 ) const;
 
diff --git a/indra/llmath/llvector4a.inl b/indra/llmath/llvector4a.inl
index 4589bac9fb..6860252a75 100755
--- a/indra/llmath/llvector4a.inl
+++ b/indra/llmath/llvector4a.inl
@@ -410,8 +410,26 @@ inline LLSimdScalar LLVector4a::normalize3withLength()
 // Note that this does not consider zero length vectors!
 inline void LLVector4a::normalize3fast()
 {
-	// find out about bad math before it takes two man-days to track down
-	llassert(isFinite3() && !equals3(getZero()));
+	LLVector4a lenSqrd; lenSqrd.setAllDot3( *this, *this );
+	const LLQuad approxRsqrt = _mm_rsqrt_ps(lenSqrd.mQ);
+	mQ = _mm_mul_ps( mQ, approxRsqrt );
+}
+
+// Normalize this vector with respect to the x, y, and z components only. Accurate only to 10-12 bits of precision. W component is destroyed
+// Note that this does not consider zero length vectors!
+inline void LLVector4a::normalize3fast_checked(LLVector4a* default)
+{
+	// handle bogus inputs before NaNs are generated below
+	//
+	if (!isFinite3() || (dot3(*this).getF32() < F_APPROXIMATELY_ZERO))
+	{
+		if (default)
+			*this = *default;
+		else
+			set(0,1,0,1);
+
+		return;
+	}
 
 	LLVector4a lenSqrd; lenSqrd.setAllDot3( *this, *this );
 	const LLQuad approxRsqrt = _mm_rsqrt_ps(lenSqrd.mQ);
diff --git a/indra/llmath/llvolume.cpp b/indra/llmath/llvolume.cpp
index bc2572375a..15621c2625 100755
--- a/indra/llmath/llvolume.cpp
+++ b/indra/llmath/llvolume.cpp
@@ -4472,6 +4472,9 @@ void LLVolume::generateSilhouetteVertices(std::vector<LLVector3> &vertices,
 					continue; //skip degenerate face
 				}
 
+				LLVector4a default_norm;
+				default_norm.set(0,1,0,1);
+
 				//for each edge
 				for (S32 k = 0; k < 3; k++) {
 					S32 index = face.mEdge[j*3+k];
@@ -4493,14 +4496,14 @@ void LLVolume::generateSilhouetteVertices(std::vector<LLVector3> &vertices,
 
 						norm_mat.rotate(n[v1], t);
 
-						t.normalize3fast();
+						t.normalize3fast_checked(&default_norm);
 						normals.push_back(LLVector3(t[0], t[1], t[2]));
 
 						mat.affineTransform(v[v2], t);
 						vertices.push_back(LLVector3(t[0], t[1], t[2]));
 						
 						norm_mat.rotate(n[v2], t);
-						t.normalize3fast();
+						t.normalize3fast_checked(&default_norm);
 						normals.push_back(LLVector3(t[0], t[1], t[2]));
 					}
 				}		
@@ -6096,6 +6099,9 @@ BOOL LLVolumeFace::createUnCutCubeCap(LLVolume* volume, BOOL partial_build)
 	{
 		VertexData	corners[4];
 		VertexData baseVert;
+		LLVector4a default_norm;
+		default_norm.set(0,1,0,1);
+
 		for(S32 t = 0; t < 4; t++)
 		{
 			corners[t].getPosition().load3( mesh[offset + (grid_size*t)].mPos.mV);
@@ -6108,8 +6114,8 @@ BOOL LLVolumeFace::createUnCutCubeCap(LLVolume* volume, BOOL partial_build)
 			lhs.setSub(corners[1].getPosition(), corners[0].getPosition());
 			LLVector4a rhs;
 			rhs.setSub(corners[2].getPosition(), corners[1].getPosition());
-			baseVert.getNormal().setCross3(lhs, rhs); 
-			baseVert.getNormal().normalize3fast();
+			baseVert.getNormal().setCross3(lhs, rhs);
+			baseVert.getNormal().normalize3fast_checked(&default_norm);
 		}
 
 		if(!(mTypeMask & TOP_MASK))
@@ -6559,17 +6565,12 @@ BOOL LLVolumeFace::createCap(LLVolume* volume, BOOL partial_build)
 	d1.setSub(mPositions[mIndices[2]], mPositions[mIndices[0]]);
 
 	LLVector4a normal;
-	normal.setCross3(d0,d1);
-
-	if (normal.dot3(normal).getF32() > F_APPROXIMATELY_ZERO)
-	{
-		normal.normalize3fast();
-	}
-	else
-	{ //degenerate, make up a value
-		normal.set(0,0,1);
-	}
+	LLVector4a default_norm;
+	default_norm.set(0,1,0,1);
 
+	normal.setCross3(d0,d1);
+	normal.normalize3fast_checked(&default_norm);
+	
 	llassert(llfinite(normal.getF32ptr()[0]));
 	llassert(llfinite(normal.getF32ptr()[1]));
 	llassert(llfinite(normal.getF32ptr()[2]));
@@ -6611,11 +6612,13 @@ void LLVolumeFace::createTangents()
 		CalculateTangentArray(mNumVertices, mPositions, mNormals, mTexCoords, mNumIndices/3, mIndices, mTangents);
 
 		//normalize tangents
+		LLVector4a default_norm;
+		default_norm.set(0,1,0,1);
 		for (U32 i = 0; i < mNumVertices; i++) 
 		{
 			//binorm[i].normalize3fast();
 			//bump map/planar projection code requires normals to be normalized
-			mNormals[i].normalize3fast();
+			mNormals[i].normalize3fast_checked(&default_norm);
 		}
 	}
 }
@@ -6793,6 +6796,9 @@ void LLVolumeFace::appendFace(const LLVolumeFace& face, LLMatrix4& mat_in, LLMat
 	mat.loadu(mat_in);
 	norm_mat.loadu(norm_mat_in);
 
+	LLVector4a default_norm;
+	default_norm.set(0,1,0,1);
+
 	for (U32 i = 0; i < face.mNumVertices; ++i)
 	{
 		//transform appended face position and store
@@ -6800,7 +6806,7 @@ void LLVolumeFace::appendFace(const LLVolumeFace& face, LLMatrix4& mat_in, LLMat
 
 		//transform appended face normal and store
 		norm_mat.rotate(src_norm[i], dst_norm[i]);
-		dst_norm[i].normalize3fast();
+		dst_norm[i].normalize3fast_checked(&default_norm);
 
 		//copy appended face texture coordinate
 		dst_tc[i] = src_tc[i];
@@ -7209,7 +7215,7 @@ BOOL LLVolumeFace::createSide(LLVolume* volume, BOOL partial_build)
 	return TRUE;
 }
 
-#define TANGENTIAL_PARANOIA_ASSERTS 1
+#define TANGENTIAL_PARANOIA_ASSERTS 0
 
 #if TANGENTIAL_PARANOIA_ASSERTS
 	#define tangential_paranoia(a) llassert(a)
@@ -7289,47 +7295,28 @@ void CalculateTangentArray(U32 vertexCount, const LLVector4a *vertex, const LLVe
 
 	// These appear to come out of the summing above distinctly non-unit-length
 	//
+	LLVector4a default_norm;
+	default_norm.set(0,1,0,1);
+
 	for (U32 a = 0; a < vertexCount; a++)
 	{
-		// Conditioning required by assets which don't necessarily reference every vert index
-		// (i.e. some of the tangents can end up uninitialized and therefore indeterminate/INF)
-		// and protection against zero length vectors which are not handled by normalize3fast.
-		//
-		if (!tan1[a].isFinite3() || tan1[a].equals3(LLVector4a::getZero()))
-		{
-			tan1[a].set(0,0,1,1);
-		}
-		else
-		{
-			tan1[a].normalize3fast();	
-		}
+		tan1[a].normalize3fast_checked(&default_norm);
+		tan2[a].normalize3fast_checked(&default_norm);
 
-		if (!tan2[a].isFinite3() || tan2[a].equals3(LLVector4a::getZero()))
-		{
-			tan2[a].set(0,0,1,1);
-		}
-		else
-		{
-			tan2[a].normalize3fast();
-		}		
-
-		const F32 cefgw = 0.03f;
 		tangential_paranoia(tan1[a].isFinite3());
 		tangential_paranoia(tan2[a].isFinite3());		
-		tangential_paranoia(tan1[a].isNormalized3(cefgw));
-		tangential_paranoia(tan2[a].isNormalized3(cefgw));	
+		tangential_paranoia(tan1[a].isNormalized3(0.03f));
+		tangential_paranoia(tan2[a].isNormalized3(0.03f));	
 	}
 
+	LLVector4a default_tangent;
+	default_tangent.set(0,0,1,1);
+
    for (U32 a = 0; a < vertexCount; a++)
 	{
 		LLVector4a n = normal[a];
-
-		if (!n.isFinite3() || n.equals3(LLVector4a::getZero()))
-		{
-			n.set(0,1,0,1);
-		}
-
-		n.normalize3fast();
+		
+		n.normalize3fast_checked(&default_norm);
 
 		const LLVector4a& t = tan1[a];
 
@@ -7353,34 +7340,27 @@ void CalculateTangentArray(U32 vertexCount, const LLVector4a *vertex, const LLVe
 
 		tangential_paranoia(tsubn.isFinite3());
 
-		if (tsubn.dot3(tsubn).getF32() > F_APPROXIMATELY_ZERO)
-		{
-			tsubn.normalize3fast();
+		tsubn.normalize3fast_checked(&default_tangent);
 		
-			// Calculate handedness
-			F32 handedness = ncrosst.dot3(tan2[a]).getF32() < 0.f ? -1.f : 1.f;
+		// Calculate handedness
+		F32 handedness = ncrosst.dot3(tan2[a]).getF32() < 0.f ? -1.f : 1.f;
 		
-			tsubn.getF32ptr()[3] = handedness;
+		tsubn.getF32ptr()[3] = handedness;
 
-			tangent[a] = tsubn;
+		tangent[a] = tsubn;
 
-			tangential_paranoia(tangent[a].isNormalized3(0.1f));
+		tangential_paranoia(tangent[a].isNormalized3(0.1f));
 
-			llassert(llfinite(tangent[a].getF32ptr()[0]));
-			llassert(llfinite(tangent[a].getF32ptr()[1]));
-			llassert(llfinite(tangent[a].getF32ptr()[2]));
+		llassert(llfinite(tangent[a].getF32ptr()[0]));
+		llassert(llfinite(tangent[a].getF32ptr()[1]));
+		llassert(llfinite(tangent[a].getF32ptr()[2]));
 
-			llassert(!llisnan(tangent[a].getF32ptr()[0]));
-			llassert(!llisnan(tangent[a].getF32ptr()[1]));
-			llassert(!llisnan(tangent[a].getF32ptr()[2]));
-		}
-		else
-		{ //degenerate, make up a value
-			tangent[a].set(0,0,1,1);
-		}
+		llassert(!llisnan(tangent[a].getF32ptr()[0]));
+		llassert(!llisnan(tangent[a].getF32ptr()[1]));
+		llassert(!llisnan(tangent[a].getF32ptr()[2]));
     }
     
-	ll_aligned_free_16(tan1);
+	 ll_aligned_free_16(tan1);
 }
 
 
diff --git a/indra/newview/llface.cpp b/indra/newview/llface.cpp
index 3e503cb750..b34370fa87 100755
--- a/indra/newview/llface.cpp
+++ b/indra/newview/llface.cpp
@@ -817,6 +817,12 @@ BOOL LLFace::genVolumeBBoxes(const LLVolume &volume, S32 f,
 			size.mul(scale);
 		}
 
+		// Catch potential badness from normalization before it happens
+		//
+		llassert(mat_normal.mMatrix[0].isFinite3() && (mat_normal.mMatrix[0].dot3(mat_normal.mMatrix[0]).getF32() > F_APPROXIMATELY_ZERO));
+		llassert(mat_normal.mMatrix[1].isFinite3() && (mat_normal.mMatrix[1].dot3(mat_normal.mMatrix[1]).getF32() > F_APPROXIMATELY_ZERO));
+		llassert(mat_normal.mMatrix[2].isFinite3() && (mat_normal.mMatrix[2].dot3(mat_normal.mMatrix[2]).getF32() > F_APPROXIMATELY_ZERO));
+
 		mat_normal.mMatrix[0].normalize3fast();
 		mat_normal.mMatrix[1].normalize3fast();
 		mat_normal.mMatrix[2].normalize3fast();
@@ -936,7 +942,9 @@ LLVector2 LLFace::surfaceToTexture(LLVector2 surface_coord, const LLVector4a& po
 		LLVector4a volume_normal;
 		LLVector3 v_normal(normal.getF32ptr());
 		volume_normal.load3(mDrawablep->getVOVolume()->agentDirectionToVolume(v_normal).mV);
-		volume_normal.normalize3fast();
+		LLVector4a default_norm;
+		default_norm.set(0,1,0,1);
+		volume_normal.normalize3fast_checked(&default_norm);
 		
 		if (texgen == LLTextureEntry::TEX_GEN_PLANAR)
 		{
@@ -1909,7 +1917,10 @@ BOOL LLFace::getGeometryVolume(const LLVolume& volume,
 							binormal.load3(t.mV);
 						}
 
-						binormal.normalize3fast();
+						LLVector4a default_binorm;
+						default_binorm.set(1,0,0,1);
+						binormal.normalize3fast_checked(&default_binorm);
+
 						LLVector2 tc = bump_tc[i];
 						tc += LLVector2( bump_s_primary_light_ray.dot3(tangent).getF32(), bump_t_primary_light_ray.dot3(binormal).getF32() );
 					
@@ -1996,12 +2007,13 @@ BOOL LLFace::getGeometryVolume(const LLVolume& volume,
 			LLFastTimer t(FTM_FACE_GEOM_NORMAL);
 			mVertexBuffer->getNormalStrider(norm, mGeomIndex, mGeomCount, map_range);
 			F32* normals = (F32*) norm.get();
-	
+			LLVector4a default_norm;
+			default_norm.set(0,1,0,1);
 			for (S32 i = 0; i < num_vertices; i++)
 			{	
 				LLVector4a normal;
 				mat_normal.rotate(vf.mNormals[i], normal);
-				normal.normalize3fast();
+				normal.normalize3fast_checked(&default_norm);
 				normal.store4a(normals);
 				normals += 4;
 			}
@@ -2024,12 +2036,14 @@ BOOL LLFace::getGeometryVolume(const LLVolume& volume,
 			mask.clear();
 			mask.setElement<3>();
 
+			LLVector4a default_tangent;
+			default_tangent.set(0,0,1,1);
+
 			for (S32 i = 0; i < num_vertices; i++)
 			{
 				LLVector4a tangent_out;
 				mat_normal.rotate(vf.mTangents[i], tangent_out);
-				tangent_out.normalize3fast();
-				
+				tangent_out.normalize3fast_checked(&default_tangent);
 				tangent_out.setSelectWithMask(mask, vf.mTangents[i], tangent_out);
 				tangent_out.store4a(tangents);
 				
@@ -2244,7 +2258,10 @@ BOOL LLFace::calcPixelArea(F32& cos_angle_to_view_dir, F32& radius)
 		dist *= 16.f;
 	}
 
-	lookAt.normalize3fast() ;	
+	LLVector4a default_lookat;
+	default_lookat.set(0,0,1,1);
+
+	lookAt.normalize3fast_checked(&default_lookat);	
 
 	//get area of circle around node
 	F32 app_angle = atanf((F32) sqrt(size_squared) / dist);
diff --git a/indra/newview/llspatialpartition.cpp b/indra/newview/llspatialpartition.cpp
index 78401020a6..dc99fd469b 100755
--- a/indra/newview/llspatialpartition.cpp
+++ b/indra/newview/llspatialpartition.cpp
@@ -1259,12 +1259,15 @@ F32 LLSpatialPartition::calcDistance(LLSpatialGroup* group, LLCamera& camera)
 
 	F32 dist = 0.f;
 
+	LLVector4a default_eyevec;
+	default_eyevec.set(0,0,1,1);
+
 	if (group->mDrawMap.find(LLRenderPass::PASS_ALPHA) != group->mDrawMap.end())
 	{
 		LLVector4a v = eye;
 
 		dist = eye.getLength3().getF32();
-		eye.normalize3fast();
+		eye.normalize3fast_checked(&default_eyevec);
 
 		if (!group->isState(LLSpatialGroup::ALPHA_DIRTY))
 		{
diff --git a/indra/newview/llvopartgroup.cpp b/indra/newview/llvopartgroup.cpp
index 6a7f26bdb5..b25213d85f 100755
--- a/indra/newview/llvopartgroup.cpp
+++ b/indra/newview/llvopartgroup.cpp
@@ -411,14 +411,21 @@ void LLVOPartGroup::getGeometry(S32 idx,
 	LLVector4a right;
 
 	right.setCross3(at, up);
+	// guard against NaNs in normalize below
+	llassert(right.dot3(right).getF32() > F_APPROXIMATELY_ZERO);
 	right.normalize3fast();
+
 	up.setCross3(right, at);
+	// guard against NaNs in normalize below
+	llassert(up.dot3(up).getF32() > F_APPROXIMATELY_ZERO);
 	up.normalize3fast();
 
 	if (part.mFlags & LLPartData::LL_PART_FOLLOW_VELOCITY_MASK)
 	{
 		LLVector4a normvel;
 		normvel.load3(part.mVelocity.mV);
+		// guard against NaNs in normalize below
+		llassert(normvel.dot3(normvel).getF32() > F_APPROXIMATELY_ZERO);
 		normvel.normalize3fast();
 		LLVector2 up_fracs;
 		up_fracs.mV[0] = normvel.dot3(right).getF32();
@@ -443,6 +450,9 @@ void LLVOPartGroup::getGeometry(S32 idx,
 
 		up = new_up;
 		right = t;
+		// guard against NaNs in normalize below
+		llassert(up.dot3(up).getF32() > F_APPROXIMATELY_ZERO);
+		llassert(right.dot3(right).getF32() > F_APPROXIMATELY_ZERO);
 		up.normalize3fast();
 		right.normalize3fast();
 	}
diff --git a/indra/newview/llvovolume.cpp b/indra/newview/llvovolume.cpp
index 0aa56fcc0f..8962d7cadf 100755
--- a/indra/newview/llvovolume.cpp
+++ b/indra/newview/llvovolume.cpp
@@ -3751,7 +3751,8 @@ BOOL LLVOVolume::lineSegmentIntersect(const LLVector4a& start, const LLVector4a&
 						{
 							*normal = n;
 						}
-
+						// guard against NaNs in normalize below
+						llassert(normal->dot3(*normal).getF32() > F_APPROXIMATELY_ZERO);
 						(*normal).normalize3fast();
 					}
 
@@ -3774,6 +3775,8 @@ BOOL LLVOVolume::lineSegmentIntersect(const LLVector4a& start, const LLVector4a&
 						{
 							*tangent = tn;
 						}
+						// guard against NaNs in normalize below
+						llassert(tangent->dot3(*tangent).getF32() > F_APPROXIMATELY_ZERO);
 						(*tangent).normalize3fast();
 					}
 
diff --git a/indra/newview/pipeline.cpp b/indra/newview/pipeline.cpp
index 05ef8060d4..72912db041 100755
--- a/indra/newview/pipeline.cpp
+++ b/indra/newview/pipeline.cpp
@@ -10569,11 +10569,13 @@ void LLPipeline::generateImpostor(LLVOAvatar* avatar)
 		LLVector4a left;
 		left.load3(camera.getLeftAxis().mV);
 		left.mul(left);
+		llassert(left.dot3(left).getF32() > F_APPROXIMATELY_ZERO);
 		left.normalize3fast();
 
 		LLVector4a up;
 		up.load3(camera.getUpAxis().mV);
 		up.mul(up);
+		llassert(up.dot3(up).getF32() > F_APPROXIMATELY_ZERO);
 		up.normalize3fast();
 
 		tdim.mV[0] = fabsf(half_height.dot3(left).getF32());