fix for SH-1808: uploading warthog model triggers LLPhysicsDecomp::doDecompositionSingleHull warnings

SH-1333: [PUBLIC] Simple mesh causing the viewer to crash

2 files changed, 61 insertions, 23 deletions

diff --git a/indra/newview/llmeshrepository.cpp b/indra/newview/llmeshrepository.cpp
index 4da5da9493..f3ceac9dbe 100755
--- a/indra/newview/llmeshrepository.cpp
+++ b/indra/newview/llmeshrepository.cpp
@@ -3321,24 +3321,27 @@ S32 LLPhysicsDecomp::llcdCallback(const char* status, S32 p1, S32 p2)
 	return 1;
 }
 
-void LLPhysicsDecomp::setMeshData(LLCDMeshData& mesh)
+void LLPhysicsDecomp::setMeshData(LLCDMeshData& mesh, bool vertex_based)
 {
 	mesh.mVertexBase = mCurRequest->mPositions[0].mV;
 	mesh.mVertexStrideBytes = 12;
 	mesh.mNumVertices = mCurRequest->mPositions.size();
 
-	mesh.mIndexType = LLCDMeshData::INT_16;
-	mesh.mIndexBase = &(mCurRequest->mIndices[0]);
-	mesh.mIndexStrideBytes = 6;
+	if(!vertex_based)
+	{
+		mesh.mIndexType = LLCDMeshData::INT_16;
+		mesh.mIndexBase = &(mCurRequest->mIndices[0]);
+		mesh.mIndexStrideBytes = 6;
 	
-	mesh.mNumTriangles = mCurRequest->mIndices.size()/3;
+		mesh.mNumTriangles = mCurRequest->mIndices.size()/3;
+	}
 
-	if (mesh.mNumTriangles > 0 && mesh.mNumVertices > 2)
+	if ((vertex_based || mesh.mNumTriangles > 0) && mesh.mNumVertices > 2)
 	{
 		LLCDResult ret = LLCD_OK;
 		if (LLConvexDecomposition::getInstance() != NULL)
 		{
-			ret  = LLConvexDecomposition::getInstance()->setMeshData(&mesh);
+			ret  = LLConvexDecomposition::getInstance()->setMeshData(&mesh, vertex_based);
 		}
 
 		if (ret)
@@ -3362,7 +3365,7 @@ void LLPhysicsDecomp::doDecomposition()
 	//load data intoLLCD
 	if (stage == 0)
 	{
-		setMeshData(mesh);
+		setMeshData(mesh, false);
 	}
 		
 	//build parameter map
@@ -3536,11 +3539,54 @@ void make_box(LLPhysicsDecomp::Request * request)
 
 void LLPhysicsDecomp::doDecompositionSingleHull()
 {
-	LLCDMeshData mesh;
+	LLConvexDecomposition* decomp = LLConvexDecomposition::getInstance();
+
+	if (decomp == NULL)
+	{
+		//stub. do nothing.
+		return;
+	}
 	
-	setMeshData(mesh);
+	LLCDMeshData mesh;	
+
+#if 1
+	setMeshData(mesh, true);
+
+	LLCDResult ret = decomp->buildSingleHull() ;
+	if(ret)
+	{
+		llwarns << "Could not execute decomposition stage when attempting to create single hull." << llendl;
+		make_box(mCurRequest);
+	}
+
+	mMutex->lock();
+	mCurRequest->mHull.clear();
+	mCurRequest->mHull.resize(1);
+	mCurRequest->mHullMesh.clear();
+	mMutex->unlock();
+
+	std::vector<LLVector3> p;
+	LLCDHull hull;
+		
+	// if LLConvexDecomposition is a stub, num_hulls should have been set to 0 above, and we should not reach this code
+	decomp->getSingleHull(&hull);
+
+	const F32* v = hull.mVertexBase;
+
+	for (S32 j = 0; j < hull.mNumVertices; ++j)
+	{
+		LLVector3 vert(v[0], v[1], v[2]); 
+		p.push_back(vert);
+		v = (F32*) (((U8*) v) + hull.mVertexStrideBytes);
+	}
+						
+	mMutex->lock();
+	mCurRequest->mHull[0] = p;
+	mMutex->unlock();	
 			
-	
+#else
+	setMeshData(mesh, false);
+
 	//set all parameters to default
 	std::map<std::string, const LLCDParam*> param_map;
 
@@ -3549,23 +3595,15 @@ void LLPhysicsDecomp::doDecompositionSingleHull()
 
 	if (!params)
 	{
-		param_count = LLConvexDecomposition::getInstance()->getParameters(&params);
+		param_count = decomp->getParameters(&params);
 	}
 	
-	LLConvexDecomposition* decomp = LLConvexDecomposition::getInstance();
-
-	if (decomp == NULL)
-	{
-		//stub. do nothing.
-		return;
-	}
-
 	for (S32 i = 0; i < param_count; ++i)
 	{
 		decomp->setParam(params[i].mName, params[i].mDefault.mIntOrEnumValue);
 	}
 
-	const S32 STAGE_DECOMPOSE = mStageID["Decompose"];
+	const S32 STAGE_DECOMPOSE = mStageID["Decompose"];	
 	const S32 STAGE_SIMPLIFY = mStageID["Simplify"];
 	const S32 DECOMP_PREVIEW = 0;
 	const S32 SIMPLIFY_RETAIN = 0;
@@ -3627,7 +3665,7 @@ void LLPhysicsDecomp::doDecompositionSingleHull()
 			}
 		}
 	}
-
+#endif
 
 	{
 		completeCurrent();
diff --git a/indra/newview/llmeshrepository.h b/indra/newview/llmeshrepository.h
index f56734a7de..26cef1650a 100644
--- a/indra/newview/llmeshrepository.h
+++ b/indra/newview/llmeshrepository.h
@@ -189,7 +189,7 @@ public:
 	static S32 llcdCallback(const char*, S32, S32);
 	void cancel();
 
-	void setMeshData(LLCDMeshData& mesh);
+	void setMeshData(LLCDMeshData& mesh, bool vertex_based);
 	void doDecomposition();
 	void doDecompositionSingleHull();