MAINT-8269 Crahes in cacheOptimize()

2 files changed, 51 insertions, 14 deletions

diff --git a/indra/llmath/llvolume.cpp b/indra/llmath/llvolume.cpp
index 014de6b888..24f46d720b 100644
--- a/indra/llmath/llvolume.cpp
+++ b/indra/llmath/llvolume.cpp
@@ -2677,11 +2677,17 @@ bool LLVolume::unpackVolumeFaces(std::istream& is, S32 size)
 			}
 		}
 	}
+
+	if (!cacheOptimize())
+	{
+		// Out of memory?
+		LL_WARNS() << "Failed to optimize!" << LL_ENDL;
+		mVolumeFaces.clear();
+		return false;
+	}
 	
 	mSculptLevel = 0;  // success!
 
-	cacheOptimize();
-
 	return true;
 }
 
@@ -2713,12 +2719,16 @@ void LLVolume::copyVolumeFaces(const LLVolume* volume)
 	mSculptLevel = 0;
 }
 
-void LLVolume::cacheOptimize()
+bool LLVolume::cacheOptimize()
 {
 	for (S32 i = 0; i < mVolumeFaces.size(); ++i)
 	{
-		mVolumeFaces[i].cacheOptimize();
+		if (!mVolumeFaces[i].cacheOptimize())
+		{
+			return false;
+		}
 	}
+	return true;
 }
 
 
@@ -5174,7 +5184,7 @@ public:
 };
 
 
-void LLVolumeFace::cacheOptimize()
+bool LLVolumeFace::cacheOptimize()
 { //optimize for vertex cache according to Forsyth method: 
   // http://home.comcast.net/~tom_forsyth/papers/fast_vert_cache_opt.html
 	
@@ -5185,7 +5195,7 @@ void LLVolumeFace::cacheOptimize()
 	
 	if (mNumVertices < 3)
 	{ //nothing to do
-		return;
+		return true;
 	}
 
 	//mapping of vertices to triangles and indices
@@ -5194,8 +5204,16 @@ void LLVolumeFace::cacheOptimize()
 	//mapping of triangles do vertices
 	std::vector<LLVCacheTriangleData> triangle_data;
 
-	triangle_data.resize(mNumIndices/3);
-	vertex_data.resize(mNumVertices);
+	try
+	{
+		triangle_data.resize(mNumIndices / 3);
+		vertex_data.resize(mNumVertices);
+	}
+	catch (std::bad_alloc)
+	{
+		LL_WARNS("LLVOLUME") << "Resize failed" << LL_ENDL;
+		return false;
+	}
 
 	for (U32 i = 0; i < mNumIndices; i++)
 	{ //populate vertex data and triangle data arrays
@@ -5307,7 +5325,8 @@ void LLVolumeFace::cacheOptimize()
 	LLVector4a* pos = (LLVector4a*) ll_aligned_malloc<64>(sizeof(LLVector4a)*2*num_verts+size);
 	if (pos == NULL)
 	{
-		LL_ERRS("LLVOLUME") << "Allocation of positions vector[" << sizeof(LLVector4a) * 2 * num_verts + size  << "] failed. " << LL_ENDL;
+		LL_WARNS("LLVOLUME") << "Allocation of positions vector[" << sizeof(LLVector4a) * 2 * num_verts + size  << "] failed. " << LL_ENDL;
+		return false;
 	}
 	LLVector4a* norm = pos + num_verts;
 	LLVector2* tc = (LLVector2*) (norm + num_verts);
@@ -5318,7 +5337,9 @@ void LLVolumeFace::cacheOptimize()
 		wght = (LLVector4a*)ll_aligned_malloc_16(sizeof(LLVector4a)*num_verts);
 		if (wght == NULL)
 		{
-			LL_ERRS("LLVOLUME") << "Allocation of weights[" << sizeof(LLVector4a) * num_verts << "] failed" << LL_ENDL;
+			ll_aligned_free<64>(pos);
+			LL_WARNS("LLVOLUME") << "Allocation of weights[" << sizeof(LLVector4a) * num_verts << "] failed" << LL_ENDL;
+			return false;
 		}
 	}
 
@@ -5328,13 +5349,28 @@ void LLVolumeFace::cacheOptimize()
 		binorm = (LLVector4a*) ll_aligned_malloc_16(sizeof(LLVector4a)*num_verts);
 		if (binorm == NULL)
 		{
-			LL_ERRS("LLVOLUME") << "Allocation of binormals[" << sizeof(LLVector4a)*num_verts << "] failed" << LL_ENDL;
+			ll_aligned_free<64>(pos);
+			ll_aligned_free_16(wght);
+			LL_WARNS("LLVOLUME") << "Allocation of binormals[" << sizeof(LLVector4a)*num_verts << "] failed" << LL_ENDL;
+			return false;
 		}
 	}
 
 	//allocate mapping of old indices to new indices
 	std::vector<S32> new_idx;
-	new_idx.resize(mNumVertices, -1);
+
+	try
+	{
+		new_idx.resize(mNumVertices, -1);
+	}
+	catch (std::bad_alloc)
+	{
+		ll_aligned_free<64>(pos);
+		ll_aligned_free_16(wght);
+		ll_aligned_free_16(binorm);
+		LL_WARNS("LLVOLUME") << "Resize failed: " << mNumVertices << LL_ENDL;
+		return false;
+	}
 
 	S32 cur_idx = 0;
 	for (U32 i = 0; i < mNumIndices; ++i)
@@ -5380,6 +5416,7 @@ void LLVolumeFace::cacheOptimize()
 	//std::string result = llformat("ACMR pre/post: %.3f/%.3f  --  %d triangles %d breaks", pre_acmr, post_acmr, mNumIndices/3, breaks);
 	//LL_INFOS() << result << LL_ENDL;
 
+	return true;
 }
 
 void LLVolumeFace::createOctree(F32 scaler, const LLVector4a& center, const LLVector4a& size)
diff --git a/indra/llmath/llvolume.h b/indra/llmath/llvolume.h
index bba691d243..a8089f3709 100644
--- a/indra/llmath/llvolume.h
+++ b/indra/llmath/llvolume.h
@@ -903,7 +903,7 @@ public:
 	};
 
 	void optimize(F32 angle_cutoff = 2.f);
-	void cacheOptimize();
+	bool cacheOptimize();
 
 	void createOctree(F32 scaler = 0.25f, const LLVector4a& center = LLVector4a(0,0,0), const LLVector4a& size = LLVector4a(0.5f,0.5f,0.5f));
 
@@ -1063,7 +1063,7 @@ public:
 	void copyVolumeFaces(const LLVolume* volume);
 	void copyFacesTo(std::vector<LLVolumeFace> &faces) const;
 	void copyFacesFrom(const std::vector<LLVolumeFace> &faces);
-	void cacheOptimize();
+	bool cacheOptimize();
 
 private:
 	void sculptGenerateMapVertices(U16 sculpt_width, U16 sculpt_height, S8 sculpt_components, const U8* sculpt_data, U8 sculpt_type);