From 149b2d88dd75bddf1cb3e9927c4e8fcc84e263e1 Mon Sep 17 00:00:00 2001
From: andreykproductengine <andreykproductengine@lindenlab.com>
Date: Wed, 1 Nov 2017 19:36:13 +0200
Subject: MAINT-7228 Vertex buffer allocation failure handling

---
 indra/llrender/llvertexbuffer.cpp | 64 +++++++++++++++++++++++++++++----------
 indra/llrender/llvertexbuffer.h   | 12 ++++----
 2 files changed, 54 insertions(+), 22 deletions(-)

(limited to 'indra/llrender')

diff --git a/indra/llrender/llvertexbuffer.cpp b/indra/llrender/llvertexbuffer.cpp
index 607bbf3b3b..d4dfb373f0 100644
--- a/indra/llrender/llvertexbuffer.cpp
+++ b/indra/llrender/llvertexbuffer.cpp
@@ -1167,7 +1167,7 @@ void LLVertexBuffer::releaseIndices()
 	sGLCount--;
 }
 
-void LLVertexBuffer::createGLBuffer(U32 size)
+bool LLVertexBuffer::createGLBuffer(U32 size)
 {
 	if (mGLBuffer)
 	{
@@ -1176,9 +1176,11 @@ void LLVertexBuffer::createGLBuffer(U32 size)
 
 	if (size == 0)
 	{
-		return;
+		return true;
 	}
 
+	bool sucsess = true;
+
 	mEmpty = true;
 
 	mMappedDataUsingVBOs = useVBOs();
@@ -1196,9 +1198,15 @@ void LLVertexBuffer::createGLBuffer(U32 size)
 		mSize = size;
 		claimMem(mSize);
 	}
+
+	if (!mMappedData)
+	{
+		sucsess = false;
+	}
+	return sucsess;
 }
 
-void LLVertexBuffer::createGLIndices(U32 size)
+bool LLVertexBuffer::createGLIndices(U32 size)
 {
 	if (mGLIndices)
 	{
@@ -1207,9 +1215,11 @@ void LLVertexBuffer::createGLIndices(U32 size)
 	
 	if (size == 0)
 	{
-		return;
+		return true;
 	}
 
+	bool sucsess = true;
+
 	mEmpty = true;
 
 	//pad by 16 bytes for aligned copies
@@ -1230,6 +1240,12 @@ void LLVertexBuffer::createGLIndices(U32 size)
 		mGLIndices = ++gl_buffer_idx;
 		mIndicesSize = size;
 	}
+
+	if (!mMappedIndexData)
+	{
+		sucsess = false;
+	}
+	return sucsess;
 }
 
 void LLVertexBuffer::destroyGLBuffer()
@@ -1272,10 +1288,12 @@ void LLVertexBuffer::destroyGLIndices()
 	//unbind();
 }
 
-void LLVertexBuffer::updateNumVerts(S32 nverts)
+bool LLVertexBuffer::updateNumVerts(S32 nverts)
 {
 	llassert(nverts >= 0);
 
+	bool sucsess = true;
+
 	if (nverts > 65536)
 	{
 		LL_WARNS() << "Vertex buffer overflow!" << LL_ENDL;
@@ -1286,31 +1304,37 @@ void LLVertexBuffer::updateNumVerts(S32 nverts)
 
 	if (needed_size > mSize || needed_size <= mSize/2)
 	{
-		createGLBuffer(needed_size);
+		sucsess &= createGLBuffer(needed_size);
 	}
 
 	sVertexCount -= mNumVerts;
 	mNumVerts = nverts;
 	sVertexCount += mNumVerts;
+
+	return sucsess;
 }
 
-void LLVertexBuffer::updateNumIndices(S32 nindices)
+bool LLVertexBuffer::updateNumIndices(S32 nindices)
 {
 	llassert(nindices >= 0);
 
+	bool sucsess = true;
+
 	U32 needed_size = sizeof(U16) * nindices;
 
 	if (needed_size > mIndicesSize || needed_size <= mIndicesSize/2)
 	{
-		createGLIndices(needed_size);
+		sucsess &= createGLIndices(needed_size);
 	}
 
 	sIndexCount -= mNumIndices;
 	mNumIndices = nindices;
 	sIndexCount += mNumIndices;
+
+	return sucsess;
 }
 
-void LLVertexBuffer::allocateBuffer(S32 nverts, S32 nindices, bool create)
+bool LLVertexBuffer::allocateBuffer(S32 nverts, S32 nindices, bool create)
 {
 	stop_glerror();
 
@@ -1320,10 +1344,12 @@ void LLVertexBuffer::allocateBuffer(S32 nverts, S32 nindices, bool create)
 		LL_ERRS() << "Bad vertex buffer allocation: " << nverts << " : " << nindices << LL_ENDL;
 	}
 
-	updateNumVerts(nverts);
-	updateNumIndices(nindices);
+	bool sucsess = true;
+
+	sucsess &= updateNumVerts(nverts);
+	sucsess &= updateNumIndices(nindices);
 	
-	if (create && (nverts || nindices))
+	if (sucsess && create && (nverts || nindices))
 	{
 		//actually allocate space for the vertex buffer if using VBO mapping
 		flush();
@@ -1336,6 +1362,8 @@ void LLVertexBuffer::allocateBuffer(S32 nverts, S32 nindices, bool create)
 			setupVertexArray();
 		}
 	}
+
+	return sucsess;
 }
 
 static LLTrace::BlockTimerStatHandle FTM_SETUP_VERTEX_ARRAY("Setup VAO");
@@ -1457,15 +1485,17 @@ void LLVertexBuffer::setupVertexArray()
 	unbind();
 }
 
-void LLVertexBuffer::resizeBuffer(S32 newnverts, S32 newnindices)
+bool LLVertexBuffer::resizeBuffer(S32 newnverts, S32 newnindices)
 {
 	llassert(newnverts >= 0);
 	llassert(newnindices >= 0);
 
-	updateNumVerts(newnverts);		
-	updateNumIndices(newnindices);
+	bool sucsess = true;
+
+	sucsess &= updateNumVerts(newnverts);		
+	sucsess &= updateNumIndices(newnindices);
 	
-	if (useVBOs())
+	if (sucsess && useVBOs())
 	{
 		flush();
 
@@ -1474,6 +1504,8 @@ void LLVertexBuffer::resizeBuffer(S32 newnverts, S32 newnindices)
 			setupVertexArray();
 		}
 	}
+
+	return sucsess;
 }
 
 bool LLVertexBuffer::useVBOs() const
diff --git a/indra/llrender/llvertexbuffer.h b/indra/llrender/llvertexbuffer.h
index c05fd01595..bd27296eb6 100644
--- a/indra/llrender/llvertexbuffer.h
+++ b/indra/llrender/llvertexbuffer.h
@@ -214,12 +214,12 @@ protected:
 	bool	bindGLArray();
 	void	releaseBuffer();
 	void	releaseIndices();
-	void	createGLBuffer(U32 size);
-	void	createGLIndices(U32 size);
+	bool	createGLBuffer(U32 size);
+	bool	createGLIndices(U32 size);
 	void 	destroyGLBuffer();
 	void 	destroyGLIndices();
-	void	updateNumVerts(S32 nverts);
-	void	updateNumIndices(S32 nindices); 
+	bool	updateNumVerts(S32 nverts);
+	bool	updateNumIndices(S32 nindices); 
 	void	unmapBuffer();
 		
 public:
@@ -235,8 +235,8 @@ public:
 	virtual void	setBuffer(U32 data_mask); 	// calls  setupVertexBuffer() if data_mask is not 0
 	void flush(); //flush pending data to GL memory
 	// allocate buffer
-	void	allocateBuffer(S32 nverts, S32 nindices, bool create);
-	virtual void resizeBuffer(S32 newnverts, S32 newnindices);
+	bool	allocateBuffer(S32 nverts, S32 nindices, bool create);
+	virtual bool resizeBuffer(S32 newnverts, S32 newnindices);
 			
 	// Only call each getVertexPointer, etc, once before calling unmapBuffer()
 	// call unmapBuffer() after calls to getXXXStrider() before any cals to setBuffer()
-- 
cgit v1.2.3