Introduces a LLThreadLocalData class that can be

accessed through the static LLThread::tldata().
Currently this object contains two (public) thread-local
objects: a LLAPRRootPool and a LLVolatileAPRPool.

The first is the general memory pool used by this thread
(and this thread alone), while the second is intended
for short lived memory allocations (needed for APR).
The advantages of not mixing those two is that the latter
is used most frequently, and as a result of it's nature
can be destroyed and reconstructed on a "regular" basis.

This patch adds LLAPRPool (completely replacing the old one),
which is a wrapper around apr_pool_t* and has complete
thread-safity checking.

Whenever an apr call requires memory for some resource,
a memory pool in the form of an LLAPRPool object can
be created with the same life-time as this resource;
assuring clean up of the memory no sooner, but also
not much later than the life-time of the resource
that needs the memory.

Many, many function calls and constructors had the
pool parameter simply removed (it is no longer the
concern of the developer, if you don't write code
that actually does an libapr call then you are no
longer bothered with memory pools at all).

However, I kept the notion of short-lived and
long-lived allocations alive (see my remark in
the jira here: https://jira.secondlife.com/browse/STORM-864?focusedCommentId=235356&page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel#comment-235356
which requires that the LLAPRFile API needs
to allow the user to specify how long they
think a file will stay open. By choosing
'short_lived' as default for the constructor
that immediately opens a file, the number of
instances where this needs to be specified is
drastically reduced however (obviously, any
automatic LLAPRFile is short lived).

***

Addressed Boroondas remarks in https://codereview.secondlife.com/r/99/
regarding (doxygen) comments. This patch effectively only changes comments.

Includes some 'merge' stuff that ended up in llvocache.cpp
(while starting as a bug fix, now only resulting in a cleanup).

***

Added comment 'The use of apr_pool_t is OK here'.

Added this comment on every line where apr_pool_t
is correctly being used.

This should make it easier to spot (future) errors
where someone started to use apr_pool_t; you can
just grep all sources for 'apr_pool_t' and immediately
see where it's being used while LLAPRPool should
have been used.

Note that merging this patch is very easy:
If there are no other uses of apr_pool_t in the code
(one grep) and it compiles, then it will work.

***

Second Merge (needed to remove 'delete mCreationMutex'
from LLImageDecodeThread::~LLImageDecodeThread).

***

Added back #include <apr_pools.h>.

Apparently that is needed on libapr version 1.2.8.,
the version used by Linden Lab, for calls to
apr_queue_*. This is a bug in libapr (we also
include <apr_queue.h>, that is fixed in (at least) 1.3.7.

Note that 1.2.8 is VERY old. Even 1.3.x is old.

***

License fixes (GPL -> LGPL). And typo in comments.
Addresses merov's comments on the review board.

***

Added Merov's compile fixes for windows.

9 files changed, 40 insertions, 47 deletions

diff --git a/indra/llplugin/llplugininstance.cpp b/indra/llplugin/llplugininstance.cpp
index c326961db4..9c9909a017 100644
--- a/indra/llplugin/llplugininstance.cpp
+++ b/indra/llplugin/llplugininstance.cpp
@@ -29,8 +29,7 @@
 #include "linden_common.h"
 
 #include "llplugininstance.h"
-
-#include "llapr.h"
+#include "llthread.h"			// Needed for LLThread::tldata().mRootPool
 
 /** Virtual destructor. */
 LLPluginInstanceMessageListener::~LLPluginInstanceMessageListener()
@@ -48,6 +47,7 @@ const char *LLPluginInstance::PLUGIN_INIT_FUNCTION_NAME = "LLPluginInitEntryPoin
  * @param[in] owner Plugin instance. TODO:DOC is this a good description of what "owner" is?
  */
 LLPluginInstance::LLPluginInstance(LLPluginInstanceMessageListener *owner) :
+	mDSOHandlePool(LLThread::tldata().mRootPool),
 	mDSOHandle(NULL),
 	mPluginUserData(NULL),
 	mPluginSendMessageFunction(NULL)
@@ -79,7 +79,7 @@ int LLPluginInstance::load(std::string &plugin_file)
 	
 	int result = apr_dso_load(&mDSOHandle,
 					  plugin_file.c_str(),
-					  gAPRPoolp);
+					  mDSOHandlePool());
 	if(result != APR_SUCCESS)
 	{
 		char buf[1024];
diff --git a/indra/llplugin/llplugininstance.h b/indra/llplugin/llplugininstance.h
index 50531ca77f..1c3898e2e7 100644
--- a/indra/llplugin/llplugininstance.h
+++ b/indra/llplugin/llplugininstance.h
@@ -30,6 +30,7 @@
 
 #include "llstring.h"
 #include "llapr.h"
+#include "llaprpool.h"
 
 #include "apr_dso.h"
 
@@ -88,6 +89,7 @@ private:
 	static void staticReceiveMessage(const char *message_string, void **user_data);
 	void receiveMessage(const char *message_string);
 
+	LLAPRPool mDSOHandlePool;
 	apr_dso_handle_t *mDSOHandle;
 	
 	void *mPluginUserData;
diff --git a/indra/llplugin/llpluginmessagepipe.cpp b/indra/llplugin/llpluginmessagepipe.cpp
index 8d13e38ad5..dd47300b9c 100644
--- a/indra/llplugin/llpluginmessagepipe.cpp
+++ b/indra/llplugin/llpluginmessagepipe.cpp
@@ -92,8 +92,6 @@ void LLPluginMessagePipeOwner::killMessagePipe(void)
 }
 
 LLPluginMessagePipe::LLPluginMessagePipe(LLPluginMessagePipeOwner *owner, LLSocket::ptr_t socket):
-	mInputMutex(gAPRPoolp),
-	mOutputMutex(gAPRPoolp),
 	mOwner(owner),
 	mSocket(socket)
 {
diff --git a/indra/llplugin/llpluginprocesschild.cpp b/indra/llplugin/llpluginprocesschild.cpp
index 45a86476ac..2fa5dcdd01 100644
--- a/indra/llplugin/llpluginprocesschild.cpp
+++ b/indra/llplugin/llpluginprocesschild.cpp
@@ -40,7 +40,7 @@ LLPluginProcessChild::LLPluginProcessChild()
 {
 	mState = STATE_UNINITIALIZED;
 	mInstance = NULL;
-	mSocket = LLSocket::create(gAPRPoolp, LLSocket::STREAM_TCP);
+	mSocket = LLSocket::create(LLSocket::STREAM_TCP);
 	mSleepTime = PLUGIN_IDLE_SECONDS;	// default: send idle messages at 100Hz
 	mCPUElapsed = 0.0f;
 	mBlockingRequest = false;
diff --git a/indra/llplugin/llpluginprocessparent.cpp b/indra/llplugin/llpluginprocessparent.cpp
index c002de0462..eaf7ec4bf3 100644
--- a/indra/llplugin/llpluginprocessparent.cpp
+++ b/indra/llplugin/llpluginprocessparent.cpp
@@ -33,6 +33,7 @@
 #include "llpluginmessageclasses.h"
 
 #include "llapr.h"
+#include "llscopedvolatileaprpool.h"
 
 //virtual 
 LLPluginProcessParentOwner::~LLPluginProcessParentOwner()
@@ -42,6 +43,7 @@ LLPluginProcessParentOwner::~LLPluginProcessParentOwner()
 
 bool LLPluginProcessParent::sUseReadThread = false;
 apr_pollset_t *LLPluginProcessParent::sPollSet = NULL;
+LLAPRPool LLPluginProcessParent::sPollSetPool;
 bool LLPluginProcessParent::sPollsetNeedsRebuild = false;
 LLMutex *LLPluginProcessParent::sInstancesMutex;
 std::list<LLPluginProcessParent*> LLPluginProcessParent::sInstances;
@@ -52,7 +54,7 @@ class LLPluginProcessParentPollThread: public LLThread
 {
 public:
 	LLPluginProcessParentPollThread() :
-		LLThread("LLPluginProcessParentPollThread", gAPRPoolp)
+		LLThread("LLPluginProcessParentPollThread")
 	{
 	}
 protected:
@@ -77,12 +79,11 @@ protected:
 
 };
 
-LLPluginProcessParent::LLPluginProcessParent(LLPluginProcessParentOwner *owner):
-	mIncomingQueueMutex(gAPRPoolp)
+LLPluginProcessParent::LLPluginProcessParent(LLPluginProcessParentOwner* owner)
 {
 	if(!sInstancesMutex)
 	{
-		sInstancesMutex = new LLMutex(gAPRPoolp);
+		sInstancesMutex = new LLMutex;
 	}
 	
 	mOwner = owner;
@@ -95,6 +96,7 @@ LLPluginProcessParent::LLPluginProcessParent(LLPluginProcessParentOwner *owner):
 	mBlocked = false;
 	mPolledInput = false;
 	mPollFD.client_data = NULL;
+	mPollFDPool.create();
 
 	mPluginLaunchTimeout = 60.0f;
 	mPluginLockupTimeout = 15.0f;
@@ -169,44 +171,28 @@ void LLPluginProcessParent::init(const std::string &launcher_filename, const std
 bool LLPluginProcessParent::accept()
 {
 	bool result = false;
-	
 	apr_status_t status = APR_EGENERAL;
-	apr_socket_t *new_socket = NULL;
-	
-	status = apr_socket_accept(
-		&new_socket,
-		mListenSocket->getSocket(),
-		gAPRPoolp);
 
+	mSocket = LLSocket::create(status, mListenSocket);
 	
 	if(status == APR_SUCCESS)
 	{
 //		llinfos << "SUCCESS" << llendl;
 		// Success.  Create a message pipe on the new socket
-
-		// we MUST create a new pool for the LLSocket, since it will take ownership of it and delete it in its destructor!
-		apr_pool_t* new_pool = NULL;
-		status = apr_pool_create(&new_pool, gAPRPoolp);
-
-		mSocket = LLSocket::create(new_socket, new_pool);
 		new LLPluginMessagePipe(this, mSocket);
 
 		result = true;
 	}
-	else if(APR_STATUS_IS_EAGAIN(status))
-	{
-//		llinfos << "EAGAIN" << llendl;
-
-		// No incoming connections.  This is not an error.
-		status = APR_SUCCESS;
-	}
 	else
 	{
-//		llinfos << "Error:" << llendl;
-		ll_apr_warn_status(status);
-		
-		// Some other error.
-		errorState();
+		mSocket.reset();
+		// EAGAIN means "No incoming connections". This is not an error.
+		if (!APR_STATUS_IS_EAGAIN(status))
+		{
+			// Some other error.
+			ll_apr_warn_status(status);
+			errorState();
+		}
 	}
 	
 	return result;	
@@ -272,10 +258,10 @@ void LLPluginProcessParent::idle(void)
 
 			case STATE_INITIALIZED:
 			{
-	
 				apr_status_t status = APR_SUCCESS;
+				LLScopedVolatileAPRPool addr_pool;
 				apr_sockaddr_t* addr = NULL;
-				mListenSocket = LLSocket::create(gAPRPoolp, LLSocket::STREAM_TCP);
+				mListenSocket = LLSocket::create(LLSocket::STREAM_TCP);
 				mBoundPort = 0;
 				
 				// This code is based on parts of LLSocket::create() in lliosocket.cpp.
@@ -286,7 +272,7 @@ void LLPluginProcessParent::idle(void)
 					APR_INET,
 					0,	// port 0 = ephemeral ("find me a port")
 					0,
-					gAPRPoolp);
+					addr_pool);
 					
 				if(ll_apr_warn_status(status))
 				{
@@ -598,7 +584,7 @@ void LLPluginProcessParent::setMessagePipe(LLPluginMessagePipe *message_pipe)
 	if(message_pipe != NULL)
 	{
 		// Set up the apr_pollfd_t
-		mPollFD.p = gAPRPoolp;
+		mPollFD.p = mPollFDPool();
 		mPollFD.desc_type = APR_POLL_SOCKET;
 		mPollFD.reqevents = APR_POLLIN|APR_POLLERR|APR_POLLHUP;
 		mPollFD.rtnevents = 0;
@@ -645,6 +631,7 @@ void LLPluginProcessParent::updatePollset()
 		// delete the existing pollset.
 		apr_pollset_destroy(sPollSet);
 		sPollSet = NULL;
+		sPollSetPool.destroy();
 	}
 	
 	std::list<LLPluginProcessParent*>::iterator iter;
@@ -667,12 +654,14 @@ void LLPluginProcessParent::updatePollset()
 		{
 #ifdef APR_POLLSET_NOCOPY
 			// The pollset doesn't exist yet.  Create it now.
-			apr_status_t status = apr_pollset_create(&sPollSet, count, gAPRPoolp, APR_POLLSET_NOCOPY);
+			sPollSetPool.create();
+			apr_status_t status = apr_pollset_create(&sPollSet, count, sPollSetPool(), APR_POLLSET_NOCOPY);
 			if(status != APR_SUCCESS)
 			{
 #endif // APR_POLLSET_NOCOPY
 				LL_WARNS("PluginPoll") << "Couldn't create pollset.  Falling back to non-pollset mode." << LL_ENDL;
 				sPollSet = NULL;
+				sPollSetPool.destroy();
 #ifdef APR_POLLSET_NOCOPY
 			}
 			else
diff --git a/indra/llplugin/llpluginprocessparent.h b/indra/llplugin/llpluginprocessparent.h
index 32394809ef..6beeb64c7e 100644
--- a/indra/llplugin/llpluginprocessparent.h
+++ b/indra/llplugin/llpluginprocessparent.h
@@ -176,7 +176,9 @@ private:
 
 	static bool sUseReadThread;
 	apr_pollfd_t mPollFD;
+	LLAPRPool mPollFDPool;
 	static apr_pollset_t *sPollSet;
+	static LLAPRPool sPollSetPool;
 	static bool sPollsetNeedsRebuild;
 	static LLMutex *sInstancesMutex;
 	static std::list<LLPluginProcessParent*> sInstances;
diff --git a/indra/llplugin/llpluginsharedmemory.cpp b/indra/llplugin/llpluginsharedmemory.cpp
index 63ff5085c6..e2ff645a9c 100644
--- a/indra/llplugin/llpluginsharedmemory.cpp
+++ b/indra/llplugin/llpluginsharedmemory.cpp
@@ -187,7 +187,8 @@ bool LLPluginSharedMemory::create(size_t size)
 	mName += createName();
 	mSize = size;
 	
-	apr_status_t status = apr_shm_create( &(mImpl->mAprSharedMemory), mSize, mName.c_str(), gAPRPoolp );
+	mPool.create();
+	apr_status_t status = apr_shm_create( &(mImpl->mAprSharedMemory), mSize, mName.c_str(), mPool());
 	
 	if(ll_apr_warn_status(status))
 	{
@@ -210,7 +211,7 @@ bool LLPluginSharedMemory::destroy(void)
 		}
 		mImpl->mAprSharedMemory = NULL;
 	}
-	
+	mPool.destroy();
 	return true;
 }
 
@@ -219,7 +220,8 @@ bool LLPluginSharedMemory::attach(const std::string &name, size_t size)
 	mName = name;
 	mSize = size;
 	
-	apr_status_t status = apr_shm_attach( &(mImpl->mAprSharedMemory), mName.c_str(), gAPRPoolp );
+	mPool.create();
+	apr_status_t status = apr_shm_attach( &(mImpl->mAprSharedMemory), mName.c_str(), mPool() );
 	
 	if(ll_apr_warn_status(status))
 	{
@@ -241,6 +243,7 @@ bool LLPluginSharedMemory::detach(void)
 		}
 		mImpl->mAprSharedMemory = NULL;
 	}
+	mPool.destroy();
 	
 	return true;
 }
diff --git a/indra/llplugin/llpluginsharedmemory.h b/indra/llplugin/llpluginsharedmemory.h
index c6cd49cabb..84b7a58c32 100644
--- a/indra/llplugin/llpluginsharedmemory.h
+++ b/indra/llplugin/llpluginsharedmemory.h
@@ -28,6 +28,8 @@
 #ifndef LL_LLPLUGINSHAREDMEMORY_H
 #define LL_LLPLUGINSHAREDMEMORY_H
 
+#include "llaprpool.h"
+
 class LLPluginSharedMemoryPlatformImpl;
 
 /**
@@ -108,6 +110,7 @@ private:
 	bool close(void);
 	bool unlink(void);
 	
+	LLAPRPool mPool;
 	std::string mName;
 	size_t mSize;
 	void *mMappedAddress;
diff --git a/indra/llplugin/slplugin/slplugin.cpp b/indra/llplugin/slplugin/slplugin.cpp
index 516a58db88..ff86e4e135 100644
--- a/indra/llplugin/slplugin/slplugin.cpp
+++ b/indra/llplugin/slplugin/slplugin.cpp
@@ -176,8 +176,6 @@ int APIENTRY WinMain( HINSTANCE hInstance, HINSTANCE hPrevInstance, LPSTR lpCmdL
 int main(int argc, char **argv)
 #endif
 {
-	ll_init_apr();
-
 	// Set up llerror logging
 	{
 		LLError::initForApplication(".");
@@ -393,8 +391,6 @@ int main(int argc, char **argv)
 
 	delete plugin;
 
-	ll_cleanup_apr();
-
 	return 0;
 }