From bba063c0c19d121cbecaecee92c0894c5b83eb2b Mon Sep 17 00:00:00 2001
From: Tofu Linden <tofu.linden@lindenlab.com>
Date: Thu, 15 Apr 2010 20:15:40 +0100
Subject: EXT-4388 Crash in octree line segment intersection code (getMask) Add
 yet more paranoia-and-return-or-assert, this time for +-INF in input vector.
 To be reviewed by Bao.

---
 indra/llmath/v2math.h        |  9 +++++++++
 indra/llrender/llimagegl.cpp | 14 ++++++++++++--
 2 files changed, 21 insertions(+), 2 deletions(-)

(limited to 'indra')

diff --git a/indra/llmath/v2math.h b/indra/llmath/v2math.h
index 9fef8851cc..65f3714313 100644
--- a/indra/llmath/v2math.h
+++ b/indra/llmath/v2math.h
@@ -70,6 +70,8 @@ class LLVector2
 		void	setVec(const LLVector2 &vec);	// deprecated
 		void	setVec(const F32 *vec);			// deprecated
 
+		inline bool isFinite() const; // checks to see if all values of LLVector2 are finite
+
 		F32		length() const;				// Returns magnitude of LLVector2
 		F32		lengthSquared() const;		// Returns magnitude squared of LLVector2
 		F32		normalize();					// Normalizes and returns the magnitude of LLVector2
@@ -215,6 +217,7 @@ inline void	LLVector2::setVec(const F32 *vec)
 	mV[VY] = vec[VY];
 }
 
+
 // LLVector2 Magnitude and Normalization Functions
 
 inline F32 LLVector2::length(void) const
@@ -247,6 +250,12 @@ inline F32		LLVector2::normalize(void)
 	return (mag);
 }
 
+// checker
+inline bool LLVector2::isFinite() const
+{
+	return (llfinite(mV[VX]) && llfinite(mV[VY]));
+}
+
 // deprecated
 inline F32		LLVector2::magVec(void) const
 {
diff --git a/indra/llrender/llimagegl.cpp b/indra/llrender/llimagegl.cpp
index 8addee606b..ff47c57c70 100644
--- a/indra/llrender/llimagegl.cpp
+++ b/indra/llrender/llimagegl.cpp
@@ -1738,8 +1738,18 @@ BOOL LLImageGL::getMask(const LLVector2 &tc)
 
 	if (mPickMask)
 	{
-		F32 u = tc.mV[0] - floorf(tc.mV[0]);
-		F32 v = tc.mV[1] - floorf(tc.mV[1]);
+		F32 u,v;
+		if (LL_LIKELY(tc.isFinite()))
+		{
+			u = tc.mV[0] - floorf(tc.mV[0]);
+			v = tc.mV[1] - floorf(tc.mV[1]);
+		}
+		else
+		{
+			LL_WARNS_ONCE("render") << "Ugh, non-finite u/v in mask pick" << LL_ENDL;
+			u = v = 0.f;
+			llassert(false);
+		}
 
 		if (LL_UNLIKELY(u < 0.f || u > 1.f ||
 				v < 0.f || v > 1.f))
-- 
cgit v1.2.3