Introduces a LLThreadLocalData class that can be

accessed through the static LLThread::tldata().
Currently this object contains two (public) thread-local
objects: a LLAPRRootPool and a LLVolatileAPRPool.

The first is the general memory pool used by this thread
(and this thread alone), while the second is intended
for short lived memory allocations (needed for APR).
The advantages of not mixing those two is that the latter
is used most frequently, and as a result of it's nature
can be destroyed and reconstructed on a "regular" basis.

This patch adds LLAPRPool (completely replacing the old one),
which is a wrapper around apr_pool_t* and has complete
thread-safity checking.

Whenever an apr call requires memory for some resource,
a memory pool in the form of an LLAPRPool object can
be created with the same life-time as this resource;
assuring clean up of the memory no sooner, but also
not much later than the life-time of the resource
that needs the memory.

Many, many function calls and constructors had the
pool parameter simply removed (it is no longer the
concern of the developer, if you don't write code
that actually does an libapr call then you are no
longer bothered with memory pools at all).

However, I kept the notion of short-lived and
long-lived allocations alive (see my remark in
the jira here: https://jira.secondlife.com/browse/STORM-864?focusedCommentId=235356&page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel#comment-235356
which requires that the LLAPRFile API needs
to allow the user to specify how long they
think a file will stay open. By choosing
'short_lived' as default for the constructor
that immediately opens a file, the number of
instances where this needs to be specified is
drastically reduced however (obviously, any
automatic LLAPRFile is short lived).

***

Addressed Boroondas remarks in https://codereview.secondlife.com/r/99/
regarding (doxygen) comments. This patch effectively only changes comments.

Includes some 'merge' stuff that ended up in llvocache.cpp
(while starting as a bug fix, now only resulting in a cleanup).

***

Added comment 'The use of apr_pool_t is OK here'.

Added this comment on every line where apr_pool_t
is correctly being used.

This should make it easier to spot (future) errors
where someone started to use apr_pool_t; you can
just grep all sources for 'apr_pool_t' and immediately
see where it's being used while LLAPRPool should
have been used.

Note that merging this patch is very easy:
If there are no other uses of apr_pool_t in the code
(one grep) and it compiles, then it will work.

***

Second Merge (needed to remove 'delete mCreationMutex'
from LLImageDecodeThread::~LLImageDecodeThread).

***

Added back #include <apr_pools.h>.

Apparently that is needed on libapr version 1.2.8.,
the version used by Linden Lab, for calls to
apr_queue_*. This is a bug in libapr (we also
include <apr_queue.h>, that is fixed in (at least) 1.3.7.

Note that 1.2.8 is VERY old. Even 1.3.x is old.

***

License fixes (GPL -> LGPL). And typo in comments.
Addresses merov's comments on the review board.

***

Added Merov's compile fixes for windows.

1 files changed, 96 insertions, 28 deletions

diff --git a/indra/llcommon/llthread.h b/indra/llcommon/llthread.h
index f1c6cd75af..757832b8ca 100644
--- a/indra/llcommon/llthread.h
+++ b/indra/llcommon/llthread.h
@@ -29,12 +29,34 @@
 
 #include "llapp.h"
 #include "llapr.h"
+#include "llmemory.h"
 #include "apr_thread_cond.h"
+#include "llaprpool.h"
+
+#ifdef SHOW_ASSERT
+extern LL_COMMON_API bool is_main_thread(void);
+#endif
 
 class LLThread;
 class LLMutex;
 class LLCondition;
 
+class LL_COMMON_API LLThreadLocalData
+{
+private:
+	static apr_threadkey_t* sThreadLocalDataKey;
+
+public:
+	// Thread-local memory pools.
+	LLAPRRootPool mRootPool;
+	LLVolatileAPRPool mVolatileAPRPool;
+
+	static void init(void);
+	static void destroy(void* thread_local_data);
+	static void create(LLThread* pthread);
+	static LLThreadLocalData& tldata(void);
+};
+
 class LL_COMMON_API LLThread
 {
 public:
@@ -45,7 +67,7 @@ public:
 		QUITTING= 2 	// Someone wants this thread to quit
 	} EThreadStatus;
 
-	LLThread(const std::string& name, apr_pool_t *poolp = NULL);
+	LLThread(std::string const& name);
 	virtual ~LLThread(); // Warning!  You almost NEVER want to destroy a thread unless it's in the STOPPED state.
 	virtual void shutdown(); // stops the thread
 	
@@ -60,7 +82,7 @@ public:
 	// Called from MAIN THREAD.
 	void pause();
 	void unpause();
-	bool isPaused() { return isStopped() || mPaused == TRUE; }
+	bool isPaused() { return isStopped() || mPaused; }
 	
 	// Cause the thread to wake up and check its condition
 	void wake();
@@ -74,11 +96,11 @@ public:
 	// this kicks off the apr thread
 	void start(void);
 
-	apr_pool_t *getAPRPool() { return mAPRPoolp; }
-	LLVolatileAPRPool* getLocalAPRFilePool() { return mLocalAPRFilePoolp ; }
+	// Return thread-local data for the current thread.
+	static LLThreadLocalData& tldata(void) { return LLThreadLocalData::tldata(); }
 
 private:
-	BOOL				mPaused;
+	bool				mPaused;
 	
 	// static function passed to APR thread creation routine
 	static void *APR_THREAD_FUNC staticRun(apr_thread_t *apr_threadp, void *datap);
@@ -88,14 +110,10 @@ protected:
 	LLCondition*		mRunCondition;
 
 	apr_thread_t		*mAPRThreadp;
-	apr_pool_t			*mAPRPoolp;
-	BOOL				mIsLocalPool;
 	EThreadStatus		mStatus;
 
-	//a local apr_pool for APRFile operations in this thread. If it exists, LLAPRFile::sAPRFilePoolp should not be used.
-	//Note: this pool is used by APRFile ONLY, do NOT use it for any other purposes.
-	//      otherwise it will cause severe memory leaking!!! --bao
-	LLVolatileAPRPool  *mLocalAPRFilePoolp ; 
+	friend void LLThreadLocalData::create(LLThread* threadp);
+	LLThreadLocalData*	mThreadLocalData;
 
 	void setQuitting();
 	
@@ -125,30 +143,80 @@ protected:
 
 #define MUTEX_DEBUG (LL_DEBUG || LL_RELEASE_WITH_DEBUG_INFO)
 
-class LL_COMMON_API LLMutex
+#ifdef MUTEX_DEBUG
+// We really shouldn't be using recursive locks. Make sure of that in debug mode.
+#define MUTEX_FLAG APR_THREAD_MUTEX_UNNESTED
+#else
+// Use the fastest platform-optimal lock behavior (can be recursive or non-recursive).
+#define MUTEX_FLAG APR_THREAD_MUTEX_DEFAULT
+#endif
+
+class LL_COMMON_API LLMutexBase
 {
 public:
-	LLMutex(apr_pool_t *apr_poolp); // NULL pool constructs a new pool for the mutex
-	virtual ~LLMutex();
-	
-	void lock();		// blocks
-	void unlock();
-	bool isLocked(); 	// non-blocking, but does do a lock/unlock so not free
-	
+	void lock() { apr_thread_mutex_lock(mAPRMutexp); }
+	void unlock() { apr_thread_mutex_unlock(mAPRMutexp); }
+	// Returns true if lock was obtained successfully.
+	bool trylock() { return !APR_STATUS_IS_EBUSY(apr_thread_mutex_trylock(mAPRMutexp)); }
+
+	// non-blocking, but does do a lock/unlock so not free
+	bool isLocked() { bool is_not_locked = trylock(); if (is_not_locked) unlock(); return !is_not_locked; }
+
+protected:
+	// mAPRMutexp is initialized and uninitialized in the derived class.
+	apr_thread_mutex_t* mAPRMutexp;
+};
+
+class LL_COMMON_API LLMutex : public LLMutexBase
+{
+public:
+	LLMutex(LLAPRPool& parent = LLThread::tldata().mRootPool) : mPool(parent)
+	{
+		apr_thread_mutex_create(&mAPRMutexp, MUTEX_FLAG, mPool());
+	}
+	~LLMutex()
+	{
+		llassert(!isLocked()); // better not be locked!
+		apr_thread_mutex_destroy(mAPRMutexp);
+		mAPRMutexp = NULL;
+	}
+
 protected:
-	apr_thread_mutex_t *mAPRMutexp;
-	apr_pool_t			*mAPRPoolp;
-	BOOL				mIsLocalPool;
-#if MUTEX_DEBUG
-	std::map<U32, BOOL> mIsLocked;
+	LLAPRPool mPool;
+};
+
+#if APR_HAS_THREADS
+// No need to use a root pool in this case.
+typedef LLMutex LLMutexRootPool;
+#else // APR_HAS_THREADS
+class LL_COMMON_API LLMutexRootPool : public LLMutexBase
+{
+public:
+	LLMutexRootPool(void)
+	{
+		apr_thread_mutex_create(&mAPRMutexp, MUTEX_FLAG, mRootPool());
+	}
+	~LLMutexRootPool()
+	{
+#if APR_POOL_DEBUG
+		// It is allowed to destruct root pools from a different thread.
+		mRootPool.grab_ownership();
 #endif
+		llassert(!isLocked());
+		apr_thread_mutex_destroy(mAPRMutexp);
+		mAPRMutexp = NULL;
+	}
+
+protected:
+	LLAPRRootPool mRootPool;
 };
+#endif // APR_HAS_THREADS
 
 // Actually a condition/mutex pair (since each condition needs to be associated with a mutex).
 class LL_COMMON_API LLCondition : public LLMutex
 {
 public:
-	LLCondition(apr_pool_t *apr_poolp); // Defaults to global pool, could use the thread pool as well.
+	LLCondition(LLAPRPool& parent = LLThread::tldata().mRootPool);
 	~LLCondition();
 	
 	void wait();		// blocks
@@ -159,10 +227,10 @@ protected:
 	apr_thread_cond_t *mAPRCondp;
 };
 
-class LLMutexLock
+class LL_COMMON_API LLMutexLock
 {
 public:
-	LLMutexLock(LLMutex* mutex)
+	LLMutexLock(LLMutexBase* mutex)
 	{
 		mMutex = mutex;
 		mMutex->lock();
@@ -172,7 +240,7 @@ public:
 		mMutex->unlock();
 	}
 private:
-	LLMutex* mMutex;
+	LLMutexBase* mMutex;
 };
 
 //============================================================================