Introduces a LLThreadLocalData class that can be

accessed through the static LLThread::tldata().
Currently this object contains two (public) thread-local
objects: a LLAPRRootPool and a LLVolatileAPRPool.

The first is the general memory pool used by this thread
(and this thread alone), while the second is intended
for short lived memory allocations (needed for APR).
The advantages of not mixing those two is that the latter
is used most frequently, and as a result of it's nature
can be destroyed and reconstructed on a "regular" basis.

This patch adds LLAPRPool (completely replacing the old one),
which is a wrapper around apr_pool_t* and has complete
thread-safity checking.

Whenever an apr call requires memory for some resource,
a memory pool in the form of an LLAPRPool object can
be created with the same life-time as this resource;
assuring clean up of the memory no sooner, but also
not much later than the life-time of the resource
that needs the memory.

Many, many function calls and constructors had the
pool parameter simply removed (it is no longer the
concern of the developer, if you don't write code
that actually does an libapr call then you are no
longer bothered with memory pools at all).

However, I kept the notion of short-lived and
long-lived allocations alive (see my remark in
the jira here: https://jira.secondlife.com/browse/STORM-864?focusedCommentId=235356&page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel#comment-235356
which requires that the LLAPRFile API needs
to allow the user to specify how long they
think a file will stay open. By choosing
'short_lived' as default for the constructor
that immediately opens a file, the number of
instances where this needs to be specified is
drastically reduced however (obviously, any
automatic LLAPRFile is short lived).

***

Addressed Boroondas remarks in https://codereview.secondlife.com/r/99/
regarding (doxygen) comments. This patch effectively only changes comments.

Includes some 'merge' stuff that ended up in llvocache.cpp
(while starting as a bug fix, now only resulting in a cleanup).

***

Added comment 'The use of apr_pool_t is OK here'.

Added this comment on every line where apr_pool_t
is correctly being used.

This should make it easier to spot (future) errors
where someone started to use apr_pool_t; you can
just grep all sources for 'apr_pool_t' and immediately
see where it's being used while LLAPRPool should
have been used.

Note that merging this patch is very easy:
If there are no other uses of apr_pool_t in the code
(one grep) and it compiles, then it will work.

***

Second Merge (needed to remove 'delete mCreationMutex'
from LLImageDecodeThread::~LLImageDecodeThread).

***

Added back #include <apr_pools.h>.

Apparently that is needed on libapr version 1.2.8.,
the version used by Linden Lab, for calls to
apr_queue_*. This is a bug in libapr (we also
include <apr_queue.h>, that is fixed in (at least) 1.3.7.

Note that 1.2.8 is VERY old. Even 1.3.x is old.

***

License fixes (GPL -> LGPL). And typo in comments.
Addresses merov's comments on the review board.

***

Added Merov's compile fixes for windows.

1 files changed, 60 insertions, 91 deletions

diff --git a/indra/llcommon/llthread.cpp b/indra/llcommon/llthread.cpp
index 49d05ef411..4917e3b935 100644
--- a/indra/llcommon/llthread.cpp
+++ b/indra/llcommon/llthread.cpp
@@ -63,6 +63,9 @@ void *APR_THREAD_FUNC LLThread::staticRun(apr_thread_t *apr_threadp, void *datap
 {
 	LLThread *threadp = (LLThread *)datap;
 
+	// Create a thread local data.
+	LLThreadLocalData::create(threadp);
+
 	// Run the user supplied function
 	threadp->run();
 
@@ -75,38 +78,20 @@ void *APR_THREAD_FUNC LLThread::staticRun(apr_thread_t *apr_threadp, void *datap
 }
 
 
-LLThread::LLThread(const std::string& name, apr_pool_t *poolp) :
-	mPaused(FALSE),
+LLThread::LLThread(std::string const& name) :
+	mPaused(false),
 	mName(name),
 	mAPRThreadp(NULL),
-	mStatus(STOPPED)
+	mStatus(STOPPED),
+	mThreadLocalData(NULL)
 {
-	// Thread creation probably CAN be paranoid about APR being initialized, if necessary
-	if (poolp)
-	{
-		mIsLocalPool = FALSE;
-		mAPRPoolp = poolp;
-	}
-	else
-	{
-		mIsLocalPool = TRUE;
-		apr_pool_create(&mAPRPoolp, NULL); // Create a subpool for this thread
-	}
-	mRunCondition = new LLCondition(mAPRPoolp);
-
-	mLocalAPRFilePoolp = NULL ;
+	mRunCondition = new LLCondition;
 }
 
 
 LLThread::~LLThread()
 {
 	shutdown();
-
-	if(mLocalAPRFilePoolp)
-	{
-		delete mLocalAPRFilePoolp ;
-		mLocalAPRFilePoolp = NULL ;
-	}
 }
 
 void LLThread::shutdown()
@@ -143,7 +128,7 @@ void LLThread::shutdown()
 		if (!isStopped())
 		{
 			// This thread just wouldn't stop, even though we gave it time
-			llwarns << "LLThread::~LLThread() exiting thread before clean exit!" << llendl;
+			llwarns << "LLThread::shutdown() exiting thread before clean exit!" << llendl;
 			// Put a stake in its heart.
 			apr_thread_exit(mAPRThreadp, -1);
 			return;
@@ -153,15 +138,8 @@ void LLThread::shutdown()
 
 	delete mRunCondition;
 	mRunCondition = 0;
-	
-	if (mIsLocalPool && mAPRPoolp)
-	{
-		apr_pool_destroy(mAPRPoolp);
-		mAPRPoolp = 0;
-	}
 }
 
-
 void LLThread::start()
 {
 	llassert(isStopped());
@@ -170,7 +148,7 @@ void LLThread::start()
 	mStatus = RUNNING;
 
 	apr_status_t status =
-		apr_thread_create(&mAPRThreadp, NULL, staticRun, (void *)this, mAPRPoolp);
+		apr_thread_create(&mAPRThreadp, NULL, staticRun, (void *)this, tldata().mRootPool());
 	
 	if(status == APR_SUCCESS)
 	{	
@@ -195,7 +173,7 @@ void LLThread::pause()
 	if (!mPaused)
 	{
 		// this will cause the thread to stop execution as soon as checkPause() is called
-		mPaused = 1;		// Does not need to be atomic since this is only set/unset from the main thread
+		mPaused = true;		// Does not need to be atomic since this is only set/unset from the main thread
 	}	
 }
 
@@ -203,7 +181,7 @@ void LLThread::unpause()
 {
 	if (mPaused)
 	{
-		mPaused = 0;
+		mPaused = false;
 	}
 
 	wake(); // wake up the thread if necessary
@@ -280,85 +258,76 @@ void LLThread::wakeLocked()
 	}
 }
 
-//============================================================================
-
-LLMutex::LLMutex(apr_pool_t *poolp) :
-	mAPRMutexp(NULL)
-{
-	//if (poolp)
-	//{
-	//	mIsLocalPool = FALSE;
-	//	mAPRPoolp = poolp;
-	//}
-	//else
-	{
-		mIsLocalPool = TRUE;
-		apr_pool_create(&mAPRPoolp, NULL); // Create a subpool for this thread
-	}
-	apr_thread_mutex_create(&mAPRMutexp, APR_THREAD_MUTEX_UNNESTED, mAPRPoolp);
-}
+#ifdef SHOW_ASSERT
+// This allows the use of llassert(is_main_thread()) to assure the current thread is the main thread.
+static apr_os_thread_t main_thread_id;
+LL_COMMON_API bool is_main_thread(void) { return apr_os_thread_equal(main_thread_id, apr_os_thread_current()); }
+#endif
 
+// The thread private handle to access the LLThreadLocalData instance.
+apr_threadkey_t* LLThreadLocalData::sThreadLocalDataKey;
 
-LLMutex::~LLMutex()
+//static
+void LLThreadLocalData::init(void)
 {
-#if MUTEX_DEBUG
-	llassert_always(!isLocked()); // better not be locked!
-#endif
-	apr_thread_mutex_destroy(mAPRMutexp);
-	mAPRMutexp = NULL;
-	if (mIsLocalPool)
+	// Only do this once.
+	if (sThreadLocalDataKey)
 	{
-		apr_pool_destroy(mAPRPoolp);
+		return;
 	}
-}
 
+	apr_status_t status = apr_threadkey_private_create(&sThreadLocalDataKey, &LLThreadLocalData::destroy, LLAPRRootPool::get()());
+	ll_apr_assert_status(status);   // Or out of memory, or system-imposed limit on the
+									// total number of keys per process {PTHREAD_KEYS_MAX}
+									// has been exceeded.
 
-void LLMutex::lock()
-{
-	apr_thread_mutex_lock(mAPRMutexp);
-#if MUTEX_DEBUG
-	// Have to have the lock before we can access the debug info
-	U32 id = LLThread::currentID();
-	if (mIsLocked[id] != FALSE)
-		llerrs << "Already locked in Thread: " << id << llendl;
-	mIsLocked[id] = TRUE;
+	// Create the thread-local data for the main thread (this function is called by the main thread).
+	LLThreadLocalData::create(NULL);
+
+#ifdef SHOW_ASSERT
+	// This function is called by the main thread.
+	main_thread_id = apr_os_thread_current();
 #endif
 }
 
-void LLMutex::unlock()
+// This is called once for every thread when the thread is destructed.
+//static
+void LLThreadLocalData::destroy(void* thread_local_data)
 {
-#if MUTEX_DEBUG
-	// Access the debug info while we have the lock
-	U32 id = LLThread::currentID();
-	if (mIsLocked[id] != TRUE)
-		llerrs << "Not locked in Thread: " << id << llendl;	
-	mIsLocked[id] = FALSE;
-#endif
-	apr_thread_mutex_unlock(mAPRMutexp);
+	delete static_cast<LLThreadLocalData*>(thread_local_data);
 }
 
-bool LLMutex::isLocked()
+//static
+void LLThreadLocalData::create(LLThread* threadp)
 {
-	apr_status_t status = apr_thread_mutex_trylock(mAPRMutexp);
-	if (APR_STATUS_IS_EBUSY(status))
+	LLThreadLocalData* new_tld = new LLThreadLocalData;
+	if (threadp)
 	{
-		return true;
+		threadp->mThreadLocalData = new_tld;
 	}
-	else
+	apr_status_t status = apr_threadkey_private_set(new_tld, sThreadLocalDataKey);
+	llassert_always(status == APR_SUCCESS);
+}
+
+//static
+LLThreadLocalData& LLThreadLocalData::tldata(void)
+{
+	if (!sThreadLocalDataKey)
 	{
-		apr_thread_mutex_unlock(mAPRMutexp);
-		return false;
+		LLThreadLocalData::init();
 	}
+
+	void* data;
+	apr_status_t status = apr_threadkey_private_get(&data, sThreadLocalDataKey);
+	llassert_always(status == APR_SUCCESS);
+	return *static_cast<LLThreadLocalData*>(data);
 }
 
 //============================================================================
 
-LLCondition::LLCondition(apr_pool_t *poolp) :
-	LLMutex(poolp)
+LLCondition::LLCondition(LLAPRPool& parent) : LLMutex(parent)
 {
-	// base class (LLMutex) has already ensured that mAPRPoolp is set up.
-
-	apr_thread_cond_create(&mAPRCondp, mAPRPoolp);
+	apr_thread_cond_create(&mAPRCondp, mPool());
 }
 
 
@@ -396,7 +365,7 @@ void LLThreadSafeRefCount::initThreadSafeRefCount()
 {
 	if (!sMutex)
 	{
-		sMutex = new LLMutex(0);
+		sMutex = new LLMutex;
 	}
 }