SH-3275 WIP Run viewer metrics for object update messages

added LLThreadLocalPtr
broke llmutex out into llmutex.h
got primary sampling buffer under thread local storage

9 files changed, 826 insertions, 466 deletions

diff --git a/indra/llcommon/CMakeLists.txt b/indra/llcommon/CMakeLists.txt
index 1812e39b36..eec2695dde 100644
--- a/indra/llcommon/CMakeLists.txt
+++ b/indra/llcommon/CMakeLists.txt
@@ -74,6 +74,7 @@ set(llcommon_SOURCE_FILES
     llmetrics.cpp
     llmetricperformancetester.cpp
     llmortician.cpp
+    llmutex.cpp
     lloptioninterface.cpp
     llptrto.cpp 
     llprocess.cpp
@@ -99,6 +100,7 @@ set(llcommon_SOURCE_FILES
     llthread.cpp
     llthreadsafequeue.cpp
     lltimer.cpp
+    lltrace.cpp
     lluri.cpp
     lluuid.cpp
     llworkerthread.cpp
@@ -197,6 +199,7 @@ set(llcommon_HEADER_FILES
     llmetrics.h
     llmetricperformancetester.h
     llmortician.h
+    llmutex.h
     llnametable.h
     lloptioninterface.h
     llpointer.h
@@ -237,6 +240,7 @@ set(llcommon_HEADER_FILES
     llstringtable.h
     llsys.h
     llthread.h
+    llthreadlocalptr.h
     llthreadsafequeue.h
     lltimer.h
     lltrace.h
diff --git a/indra/llcommon/llmemory.h b/indra/llcommon/llmemory.h
index 6a2323e7d8..4480e381e8 100644
--- a/indra/llcommon/llmemory.h
+++ b/indra/llcommon/llmemory.h
@@ -30,6 +30,11 @@
 
 class LLMutex ;
 
+#ifdef LL_WINDOWS
+#define LL_ALIGNED(x) __declspec(align(x))
+#else
+#define LL_ALIGNED(x) __attribute__ ((aligned (16)))
+#endif
 inline void* ll_aligned_malloc( size_t size, int align )
 {
 	void* mem = malloc( size + (align - 1) + sizeof(void*) );
diff --git a/indra/llcommon/llmutex.cpp b/indra/llcommon/llmutex.cpp
new file mode 100644
index 0000000000..2ce14b3a2e
--- /dev/null
+++ b/indra/llcommon/llmutex.cpp
@@ -0,0 +1,229 @@
+/** 
+ * @file llmutex.cpp
+ *
+ * $LicenseInfo:firstyear=2004&license=viewerlgpl$
+ * Second Life Viewer Source Code
+ * Copyright (C) 2010, Linden Research, Inc.
+ * 
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation;
+ * version 2.1 of the License only.
+ * 
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ * 
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ * 
+ * Linden Research, Inc., 945 Battery Street, San Francisco, CA  94111  USA
+ * $/LicenseInfo$
+ */
+
+#include "linden_common.h"
+#include "llapr.h"
+
+#include "apr_portable.h"
+
+#include "llmutex.h"
+#include "llthread.h"
+
+//============================================================================
+
+LLMutex::LLMutex(apr_pool_t *poolp) :
+	mAPRMutexp(NULL), mCount(0), mLockingThread(NO_THREAD)
+{
+	//if (poolp)
+	//{
+	//	mIsLocalPool = FALSE;
+	//	mAPRPoolp = poolp;
+	//}
+	//else
+	{
+		mIsLocalPool = TRUE;
+		apr_pool_create(&mAPRPoolp, NULL); // Create a subpool for this thread
+	}
+	apr_thread_mutex_create(&mAPRMutexp, APR_THREAD_MUTEX_UNNESTED, mAPRPoolp);
+}
+
+
+LLMutex::~LLMutex()
+{
+#if MUTEX_DEBUG
+	//bad assertion, the subclass LLSignal might be "locked", and that's OK
+	//llassert_always(!isLocked()); // better not be locked!
+#endif
+	apr_thread_mutex_destroy(mAPRMutexp);
+	mAPRMutexp = NULL;
+	if (mIsLocalPool)
+	{
+		apr_pool_destroy(mAPRPoolp);
+	}
+}
+
+
+void LLMutex::lock()
+{
+	if(isSelfLocked())
+	{ //redundant lock
+		mCount++;
+		return;
+	}
+	
+	apr_thread_mutex_lock(mAPRMutexp);
+	
+#if MUTEX_DEBUG
+	// Have to have the lock before we can access the debug info
+	U32 id = LLThread::currentID();
+	if (mIsLocked[id] != FALSE)
+		llerrs << "Already locked in Thread: " << id << llendl;
+	mIsLocked[id] = TRUE;
+#endif
+
+#if LL_DARWIN
+	mLockingThread = LLThread::currentID();
+#else
+	mLockingThread = LLThread::sThreadIndex;
+#endif
+}
+
+void LLMutex::unlock()
+{
+	if (mCount > 0)
+	{ //not the root unlock
+		mCount--;
+		return;
+	}
+	
+#if MUTEX_DEBUG
+	// Access the debug info while we have the lock
+	U32 id = LLThread::currentID();
+	if (mIsLocked[id] != TRUE)
+		llerrs << "Not locked in Thread: " << id << llendl;	
+	mIsLocked[id] = FALSE;
+#endif
+
+	mLockingThread = NO_THREAD;
+	apr_thread_mutex_unlock(mAPRMutexp);
+}
+
+bool LLMutex::isLocked()
+{
+	apr_status_t status = apr_thread_mutex_trylock(mAPRMutexp);
+	if (APR_STATUS_IS_EBUSY(status))
+	{
+		return true;
+	}
+	else
+	{
+		apr_thread_mutex_unlock(mAPRMutexp);
+		return false;
+	}
+}
+
+bool LLMutex::isSelfLocked()
+{
+#if LL_DARWIN
+	return mLockingThread == LLThread::currentID();
+#else
+	return mLockingThread == LLThread::sThreadIndex;
+#endif
+}
+
+U32 LLMutex::lockingThread() const
+{
+	return mLockingThread;
+}
+
+//============================================================================
+
+LLCondition::LLCondition(apr_pool_t *poolp) :
+	LLMutex(poolp)
+{
+	// base class (LLMutex) has already ensured that mAPRPoolp is set up.
+
+	apr_thread_cond_create(&mAPRCondp, mAPRPoolp);
+}
+
+
+LLCondition::~LLCondition()
+{
+	apr_thread_cond_destroy(mAPRCondp);
+	mAPRCondp = NULL;
+}
+
+
+void LLCondition::wait()
+{
+	if (!isLocked())
+	{ //mAPRMutexp MUST be locked before calling apr_thread_cond_wait
+		apr_thread_mutex_lock(mAPRMutexp);
+#if MUTEX_DEBUG
+		// avoid asserts on destruction in non-release builds
+		U32 id = LLThread::currentID();
+		mIsLocked[id] = TRUE;
+#endif
+	}
+	apr_thread_cond_wait(mAPRCondp, mAPRMutexp);
+}
+
+void LLCondition::signal()
+{
+	apr_thread_cond_signal(mAPRCondp);
+}
+
+void LLCondition::broadcast()
+{
+	apr_thread_cond_broadcast(mAPRCondp);
+}
+
+
+//============================================================================
+
+//----------------------------------------------------------------------------
+
+//static
+LLMutex* LLThreadSafeRefCount::sMutex = 0;
+
+//static
+void LLThreadSafeRefCount::initThreadSafeRefCount()
+{
+	if (!sMutex)
+	{
+		sMutex = new LLMutex(0);
+	}
+}
+
+//static
+void LLThreadSafeRefCount::cleanupThreadSafeRefCount()
+{
+	delete sMutex;
+	sMutex = NULL;
+}
+
+
+//----------------------------------------------------------------------------
+
+LLThreadSafeRefCount::LLThreadSafeRefCount() :
+mRef(0)
+{
+}
+
+LLThreadSafeRefCount::~LLThreadSafeRefCount()
+{ 
+	if (mRef != 0)
+	{
+		llerrs << "deleting non-zero reference" << llendl;
+	}
+}
+
+//============================================================================
+
+LLResponder::~LLResponder()
+{
+}
+
+//============================================================================
diff --git a/indra/llcommon/llmutex.h b/indra/llcommon/llmutex.h
new file mode 100644
index 0000000000..bd0a59b577
--- /dev/null
+++ b/indra/llcommon/llmutex.h
@@ -0,0 +1,168 @@
+/** 
+ * @file llmutex.h
+ * @brief Base classes for mutex and condition handling.
+ *
+ * $LicenseInfo:firstyear=2004&license=viewerlgpl$
+ * Second Life Viewer Source Code
+ * Copyright (C) 2012, Linden Research, Inc.
+ * 
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation;
+ * version 2.1 of the License only.
+ * 
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ * 
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ * 
+ * Linden Research, Inc., 945 Battery Street, San Francisco, CA  94111  USA
+ * $/LicenseInfo$
+ */
+
+#ifndef LL_LLMUTEX_H
+#define LL_LLMUTEX_H
+
+#include "llapr.h"
+#include "apr_thread_cond.h"
+
+//============================================================================
+
+#define MUTEX_DEBUG (LL_DEBUG || LL_RELEASE_WITH_DEBUG_INFO)
+
+class LL_COMMON_API LLMutex
+{
+public:
+	typedef enum
+	{
+		NO_THREAD = 0xFFFFFFFF
+	} e_locking_thread;
+
+	LLMutex(apr_pool_t *apr_poolp = NULL); // NULL pool constructs a new pool for the mutex
+	virtual ~LLMutex();
+	
+	void lock();		// blocks
+	void unlock();
+	bool isLocked(); 	// non-blocking, but does do a lock/unlock so not free
+	bool isSelfLocked(); //return true if locked in a same thread
+	U32 lockingThread() const; //get ID of locking thread
+	
+protected:
+	apr_thread_mutex_t *mAPRMutexp;
+	mutable U32			mCount;
+	mutable U32			mLockingThread;
+	
+	apr_pool_t			*mAPRPoolp;
+	BOOL				mIsLocalPool;
+	
+#if MUTEX_DEBUG
+	std::map<U32, BOOL> mIsLocked;
+#endif
+};
+
+// Actually a condition/mutex pair (since each condition needs to be associated with a mutex).
+class LL_COMMON_API LLCondition : public LLMutex
+{
+public:
+	LLCondition(apr_pool_t *apr_poolp); // Defaults to global pool, could use the thread pool as well.
+	~LLCondition();
+	
+	void wait();		// blocks
+	void signal();
+	void broadcast();
+	
+protected:
+	apr_thread_cond_t *mAPRCondp;
+};
+
+class LLMutexLock
+{
+public:
+	LLMutexLock(LLMutex* mutex)
+	{
+		mMutex = mutex;
+		
+		if(mMutex)
+			mMutex->lock();
+	}
+	~LLMutexLock()
+	{
+		if(mMutex)
+			mMutex->unlock();
+	}
+private:
+	LLMutex* mMutex;
+};
+
+
+//============================================================================
+
+// see llmemory.h for LLPointer<> definition
+
+class LL_COMMON_API LLThreadSafeRefCount
+{
+public:
+	static void initThreadSafeRefCount(); // creates sMutex
+	static void cleanupThreadSafeRefCount(); // destroys sMutex
+
+private:
+	static LLMutex* sMutex;
+
+private:
+	LLThreadSafeRefCount(const LLThreadSafeRefCount&); // not implemented
+	LLThreadSafeRefCount&operator=(const LLThreadSafeRefCount&); // not implemented
+
+protected:
+	virtual ~LLThreadSafeRefCount(); // use unref()
+
+public:
+	LLThreadSafeRefCount();
+
+	void ref()
+	{
+		if (sMutex) sMutex->lock();
+		mRef++; 
+		if (sMutex) sMutex->unlock();
+	} 
+
+	S32 unref()
+	{
+		llassert(mRef >= 1);
+		if (sMutex) sMutex->lock();
+		S32 res = --mRef;
+		if (sMutex) sMutex->unlock();
+		if (0 == res) 
+		{
+			delete this; 
+			return 0;
+		}
+		return res;
+	}	
+	S32 getNumRefs() const
+	{
+		return mRef;
+	}
+
+private: 
+	S32	mRef; 
+};
+
+
+//============================================================================
+
+// Simple responder for self destructing callbacks
+// Pure virtual class
+class LL_COMMON_API LLResponder : public LLThreadSafeRefCount
+{
+protected:
+	virtual ~LLResponder();
+public:
+	virtual void completed(bool success) = 0;
+};
+
+
+#endif // LL_LLTHREAD_H
diff --git a/indra/llcommon/llthread.cpp b/indra/llcommon/llthread.cpp
index a6ad6b125c..f3ab8aa40c 100644
--- a/indra/llcommon/llthread.cpp
+++ b/indra/llcommon/llthread.cpp
@@ -29,6 +29,7 @@
 #include "apr_portable.h"
 
 #include "llthread.h"
+#include "llmutex.h"
 
 #include "lltimer.h"
 
@@ -56,12 +57,20 @@
 // 
 //----------------------------------------------------------------------------
 
-#if !LL_DARWIN
-U32 ll_thread_local sThreadID = 0;
-#endif 
+#if LL_DARWIN
+// statically allocated thread local storage not supported in Darwin executable formats
+#elif LL_WINDOWS
+U32 __declspec(thread) LLThread::sThreadIndex = 0;
+#elif LL_LINUX
+U32 __thread LLThread::sThreadID = 0;
+#endif
 
 U32 LLThread::sIDIter = 0;
 
+LLTrace::MasterThreadTrace gMasterThreadTrace;
+LLThreadLocalPtr<LLTrace::ThreadTraceData> LLThread::sTraceData(&gMasterThreadTrace);
+
+
 LL_COMMON_API void assert_main_thread()
 {
 	static U32 s_thread_id = LLThread::currentID();
@@ -78,8 +87,10 @@ void *APR_THREAD_FUNC LLThread::staticRun(apr_thread_t *apr_threadp, void *datap
 {
 	LLThread *threadp = (LLThread *)datap;
 
+	sTraceData = new LLTrace::SlaveThreadTrace(gMasterThreadTrace);
+
 #if !LL_DARWIN
-	sThreadID = threadp->mID;
+	sThreadIndex = threadp->mID;
 #endif
 
 	// Run the user supplied function
@@ -93,7 +104,6 @@ void *APR_THREAD_FUNC LLThread::staticRun(apr_thread_t *apr_threadp, void *datap
 	return NULL;
 }
 
-
 LLThread::LLThread(const std::string& name, apr_pool_t *poolp) :
 	mPaused(FALSE),
 	mName(name),
@@ -301,198 +311,12 @@ void LLThread::wakeLocked()
 	}
 }
 
-//============================================================================
-
-LLMutex::LLMutex(apr_pool_t *poolp) :
-	mAPRMutexp(NULL), mCount(0), mLockingThread(NO_THREAD)
-{
-	//if (poolp)
-	//{
-	//	mIsLocalPool = FALSE;
-	//	mAPRPoolp = poolp;
-	//}
-	//else
-	{
-		mIsLocalPool = TRUE;
-		apr_pool_create(&mAPRPoolp, NULL); // Create a subpool for this thread
-	}
-	apr_thread_mutex_create(&mAPRMutexp, APR_THREAD_MUTEX_UNNESTED, mAPRPoolp);
-}
-
-
-LLMutex::~LLMutex()
-{
-#if MUTEX_DEBUG
-	//bad assertion, the subclass LLSignal might be "locked", and that's OK
-	//llassert_always(!isLocked()); // better not be locked!
-#endif
-	apr_thread_mutex_destroy(mAPRMutexp);
-	mAPRMutexp = NULL;
-	if (mIsLocalPool)
-	{
-		apr_pool_destroy(mAPRPoolp);
-	}
-}
-
-
-void LLMutex::lock()
-{
-	if(isSelfLocked())
-	{ //redundant lock
-		mCount++;
-		return;
-	}
-	
-	apr_thread_mutex_lock(mAPRMutexp);
-	
-#if MUTEX_DEBUG
-	// Have to have the lock before we can access the debug info
-	U32 id = LLThread::currentID();
-	if (mIsLocked[id] != FALSE)
-		llerrs << "Already locked in Thread: " << id << llendl;
-	mIsLocked[id] = TRUE;
-#endif
-
-#if LL_DARWIN
-	mLockingThread = LLThread::currentID();
-#else
-	mLockingThread = sThreadID;
-#endif
-}
-
-void LLMutex::unlock()
-{
-	if (mCount > 0)
-	{ //not the root unlock
-		mCount--;
-		return;
-	}
-	
-#if MUTEX_DEBUG
-	// Access the debug info while we have the lock
-	U32 id = LLThread::currentID();
-	if (mIsLocked[id] != TRUE)
-		llerrs << "Not locked in Thread: " << id << llendl;	
-	mIsLocked[id] = FALSE;
-#endif
-
-	mLockingThread = NO_THREAD;
-	apr_thread_mutex_unlock(mAPRMutexp);
-}
-
-bool LLMutex::isLocked()
-{
-	apr_status_t status = apr_thread_mutex_trylock(mAPRMutexp);
-	if (APR_STATUS_IS_EBUSY(status))
-	{
-		return true;
-	}
-	else
-	{
-		apr_thread_mutex_unlock(mAPRMutexp);
-		return false;
-	}
-}
-
-bool LLMutex::isSelfLocked()
-{
-#if LL_DARWIN
-	return mLockingThread == LLThread::currentID();
-#else
-	return mLockingThread == sThreadID;
-#endif
-}
-
-U32 LLMutex::lockingThread() const
-{
-	return mLockingThread;
-}
-
-//============================================================================
-
-LLCondition::LLCondition(apr_pool_t *poolp) :
-	LLMutex(poolp)
-{
-	// base class (LLMutex) has already ensured that mAPRPoolp is set up.
-
-	apr_thread_cond_create(&mAPRCondp, mAPRPoolp);
-}
-
-
-LLCondition::~LLCondition()
-{
-	apr_thread_cond_destroy(mAPRCondp);
-	mAPRCondp = NULL;
-}
-
-
-void LLCondition::wait()
-{
-	if (!isLocked())
-	{ //mAPRMutexp MUST be locked before calling apr_thread_cond_wait
-		apr_thread_mutex_lock(mAPRMutexp);
-#if MUTEX_DEBUG
-		// avoid asserts on destruction in non-release builds
-		U32 id = LLThread::currentID();
-		mIsLocked[id] = TRUE;
-#endif
-	}
-	apr_thread_cond_wait(mAPRCondp, mAPRMutexp);
-}
-
-void LLCondition::signal()
-{
-	apr_thread_cond_signal(mAPRCondp);
-}
-
-void LLCondition::broadcast()
-{
-	apr_thread_cond_broadcast(mAPRCondp);
-}
-
-//============================================================================
-
-//----------------------------------------------------------------------------
-
-//static
-LLMutex* LLThreadSafeRefCount::sMutex = 0;
-
-//static
-void LLThreadSafeRefCount::initThreadSafeRefCount()
-{
-	if (!sMutex)
-	{
-		sMutex = new LLMutex(0);
-	}
-}
-
-//static
-void LLThreadSafeRefCount::cleanupThreadSafeRefCount()
+void LLThread::lockData()
 {
-	delete sMutex;
-	sMutex = NULL;
-}
-	
-
-//----------------------------------------------------------------------------
-
-LLThreadSafeRefCount::LLThreadSafeRefCount() :
-	mRef(0)
-{
-}
-
-LLThreadSafeRefCount::~LLThreadSafeRefCount()
-{ 
-	if (mRef != 0)
-	{
-		llerrs << "deleting non-zero reference" << llendl;
-	}
+	mRunCondition->lock();
 }
 
-//============================================================================
-
-LLResponder::~LLResponder()
+void LLThread::unlockData()
 {
+	mRunCondition->unlock();
 }
-
-//============================================================================
diff --git a/indra/llcommon/llthread.h b/indra/llcommon/llthread.h
index b52e70ab2e..e2de4c8b85 100644
--- a/indra/llcommon/llthread.h
+++ b/indra/llcommon/llthread.h
@@ -30,21 +30,21 @@
 #include "llapp.h"
 #include "llapr.h"
 #include "apr_thread_cond.h"
-
-class LLThread;
-class LLMutex;
-class LLCondition;
-
-#if LL_WINDOWS
-#define ll_thread_local __declspec(thread)
-#else
-#define ll_thread_local __thread
-#endif
+#include "lltrace.h"
+#include "llthreadlocalptr.h"
 
 class LL_COMMON_API LLThread
 {
 private:
+	friend class LLMutex;
 	static U32 sIDIter;
+#if LL_DARWIN
+	// statically allocated thread local storage not supported in Darwin executable formats
+#elif LL_WINDOWS
+	static U32 __declspec(thread) LLThread::sThreadIndex;
+#elif LL_LINUX
+	static U32 __thread LLThread::sThreadID ;
+#endif
 
 public:
 	typedef enum e_thread_status
@@ -88,6 +88,8 @@ public:
 
 	U32 getID() const { return mID; }
 
+	static LLTrace::ThreadTraceData* getTraceData() { return sTraceData.get(); }
+
 private:
 	BOOL				mPaused;
 	
@@ -96,7 +98,7 @@ private:
 
 protected:
 	std::string			mName;
-	LLCondition*		mRunCondition;
+	class LLCondition*	mRunCondition;
 
 	apr_thread_t		*mAPRThreadp;
 	apr_pool_t			*mAPRPoolp;
@@ -104,6 +106,8 @@ protected:
 	EThreadStatus		mStatus;
 	U32					mID;
 
+	static LLThreadLocalPtr<LLTrace::ThreadTraceData> sTraceData;
+
 	//a local apr_pool for APRFile operations in this thread. If it exists, LLAPRFile::sAPRFilePoolp should not be used.
 	//Note: this pool is used by APRFile ONLY, do NOT use it for any other purposes.
 	//      otherwise it will cause severe memory leaking!!! --bao
@@ -135,149 +139,4 @@ protected:
 
 //============================================================================
 
-#define MUTEX_DEBUG (LL_DEBUG || LL_RELEASE_WITH_DEBUG_INFO)
-
-class LL_COMMON_API LLMutex
-{
-public:
-	typedef enum
-	{
-		NO_THREAD = 0xFFFFFFFF
-	} e_locking_thread;
-
-	LLMutex(apr_pool_t *apr_poolp); // NULL pool constructs a new pool for the mutex
-	virtual ~LLMutex();
-	
-	void lock();		// blocks
-	void unlock();
-	bool isLocked(); 	// non-blocking, but does do a lock/unlock so not free
-	bool isSelfLocked(); //return true if locked in a same thread
-	U32 lockingThread() const; //get ID of locking thread
-	
-protected:
-	apr_thread_mutex_t *mAPRMutexp;
-	mutable U32			mCount;
-	mutable U32			mLockingThread;
-	
-	apr_pool_t			*mAPRPoolp;
-	BOOL				mIsLocalPool;
-	
-#if MUTEX_DEBUG
-	std::map<U32, BOOL> mIsLocked;
-#endif
-};
-
-// Actually a condition/mutex pair (since each condition needs to be associated with a mutex).
-class LL_COMMON_API LLCondition : public LLMutex
-{
-public:
-	LLCondition(apr_pool_t *apr_poolp); // Defaults to global pool, could use the thread pool as well.
-	~LLCondition();
-	
-	void wait();		// blocks
-	void signal();
-	void broadcast();
-	
-protected:
-	apr_thread_cond_t *mAPRCondp;
-};
-
-class LLMutexLock
-{
-public:
-	LLMutexLock(LLMutex* mutex)
-	{
-		mMutex = mutex;
-		
-		if(mMutex)
-			mMutex->lock();
-	}
-	~LLMutexLock()
-	{
-		if(mMutex)
-			mMutex->unlock();
-	}
-private:
-	LLMutex* mMutex;
-};
-
-//============================================================================
-
-void LLThread::lockData()
-{
-	mRunCondition->lock();
-}
-
-void LLThread::unlockData()
-{
-	mRunCondition->unlock();
-}
-
-
-//============================================================================
-
-// see llmemory.h for LLPointer<> definition
-
-class LL_COMMON_API LLThreadSafeRefCount
-{
-public:
-	static void initThreadSafeRefCount(); // creates sMutex
-	static void cleanupThreadSafeRefCount(); // destroys sMutex
-	
-private:
-	static LLMutex* sMutex;
-
-private:
-	LLThreadSafeRefCount(const LLThreadSafeRefCount&); // not implemented
-	LLThreadSafeRefCount&operator=(const LLThreadSafeRefCount&); // not implemented
-
-protected:
-	virtual ~LLThreadSafeRefCount(); // use unref()
-	
-public:
-	LLThreadSafeRefCount();
-	
-	void ref()
-	{
-		if (sMutex) sMutex->lock();
-		mRef++; 
-		if (sMutex) sMutex->unlock();
-	} 
-
-	S32 unref()
-	{
-		llassert(mRef >= 1);
-		if (sMutex) sMutex->lock();
-		S32 res = --mRef;
-		if (sMutex) sMutex->unlock();
-		if (0 == res) 
-		{
-			delete this; 
-			return 0;
-		}
-		return res;
-	}	
-	S32 getNumRefs() const
-	{
-		return mRef;
-	}
-
-private: 
-	S32	mRef; 
-};
-
-//============================================================================
-
-// Simple responder for self destructing callbacks
-// Pure virtual class
-class LL_COMMON_API LLResponder : public LLThreadSafeRefCount
-{
-protected:
-	virtual ~LLResponder();
-public:
-	virtual void completed(bool success) = 0;
-};
-
-//============================================================================
-
 #endif // LL_LLTHREAD_H
diff --git a/indra/llcommon/llthreadlocalptr.h b/indra/llcommon/llthreadlocalptr.h
new file mode 100644
index 0000000000..f02f4849ca
--- /dev/null
+++ b/indra/llcommon/llthreadlocalptr.h
@@ -0,0 +1,141 @@
+/** 
+ * @file llthreadlocalptr.h
+ * @brief manage thread local storage through non-copyable pointer
+ *
+ * $LicenseInfo:firstyear=2004&license=viewerlgpl$
+ * Second Life Viewer Source Code
+ * Copyright (C) 2012, Linden Research, Inc.
+ * 
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation;
+ * version 2.1 of the License only.
+ * 
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ * 
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ * 
+ * Linden Research, Inc., 945 Battery Street, San Francisco, CA  94111  USA
+ * $/LicenseInfo$
+ */
+
+#ifndef LL_LLTHREAD_LOCAL_PTR_H
+#define LL_LLTHREAD_LOCAL_PTR_H
+
+#include "llapr.h"
+
+template <typename T>
+class LLThreadLocalPtr
+{
+public:
+	LLThreadLocalPtr(T* value = NULL, apr_pool_t* pool = NULL)
+	{
+		apr_status_t result = apr_threadkey_private_create(&mThreadKey, cleanup, pool);
+		if (result != APR_SUCCESS)
+		{
+			ll_apr_warn_status(result);
+			llerrs << "Failed to allocate thread local data" << llendl;
+		}
+		set(value);
+	}
+
+
+	~LLThreadLocalPtr()
+	{
+		apr_status_t result = apr_threadkey_private_delete(mThreadKey);
+		if (result != APR_SUCCESS)
+		{
+			ll_apr_warn_status(result);
+			llerrs << "Failed to delete thread local data" << llendl;
+		}
+	}
+
+	T* operator -> ()
+	{
+		return get();
+	}
+
+	const T* operator -> () const
+	{
+		return get();
+	}
+
+	T& operator*()
+	{
+		return *get();
+	}
+
+	const T& operator*() const
+	{
+		return *get();
+	}
+
+	LLThreadLocalPtr<T>& operator = (T* value)
+	{
+		set(value);
+		return *this;
+	}
+
+	void copyFrom(const LLThreadLocalPtr<T>& other)
+	{
+		set(other.get());
+	}
+
+	LL_FORCE_INLINE void set(T* value)
+	{
+		apr_status_t result = apr_threadkey_private_set((void*)value, mThreadKey);
+		if (result != APR_SUCCESS)
+		{
+			ll_apr_warn_status(result);
+			llerrs << "Failed to set thread local data" << llendl;
+		}
+	}
+
+	LL_FORCE_INLINE T* get()
+	{
+		T* ptr;
+		//apr_status_t result =
+		apr_threadkey_private_get((void**)&ptr, mThreadKey);
+		//if (result != APR_SUCCESS)
+		//{
+		//	ll_apr_warn_status(s);
+		//	llerrs << "Failed to get thread local data" << llendl;
+		//}
+		return ptr;
+	}
+
+	LL_FORCE_INLINE const T* get() const
+	{
+		T* ptr;
+		//apr_status_t result =
+		apr_threadkey_private_get((void**)&ptr, mThreadKey);
+		//if (result != APR_SUCCESS)
+		//{
+		//	ll_apr_warn_status(s);
+		//	llerrs << "Failed to get thread local data" << llendl;
+		//}
+		return ptr;
+	}
+
+
+private:
+	static void cleanup(void* ptr)
+	{
+		delete reinterpret_cast<T*>(ptr);
+	}
+
+	LLThreadLocalPtr(const LLThreadLocalPtr<T>& other)
+	{
+		// do not copy construct
+		llassert(false);
+	}
+
+	apr_threadkey_t* mThreadKey;
+};
+
+#endif // LL_LLTHREAD_LOCAL_PTR_H
diff --git a/indra/llcommon/lltrace.cpp b/indra/llcommon/lltrace.cpp
new file mode 100644
index 0000000000..037c52f8c1
--- /dev/null
+++ b/indra/llcommon/lltrace.cpp
@@ -0,0 +1,95 @@
+/** 
+ * @file lltrace.cpp
+ *
+ * $LicenseInfo:firstyear=2001&license=viewerlgpl$
+ * Second Life Viewer Source Code
+ * Copyright (C) 2012, Linden Research, Inc.
+ * 
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation;
+ * version 2.1 of the License only.
+ * 
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ * 
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ * 
+ * Linden Research, Inc., 945 Battery Street, San Francisco, CA  94111  USA
+ * $/LicenseInfo$
+ */
+
+#include "linden_common.h"
+
+#include "lltrace.h"
+#include "llthread.h"
+
+namespace LLTrace
+{
+
+BlockTimer::Recorder::StackEntry BlockTimer::sCurRecorder;
+LLThreadLocalPtr<ThreadTraceData> ThreadTraceData::sCurThreadTrace;
+
+///////////////////////////////////////////////////////////////////////
+// Sampler
+///////////////////////////////////////////////////////////////////////
+
+void Sampler::stop()
+{
+	getThreadTrace()->deactivate(this);
+}
+
+void Sampler::resume()
+{
+	getThreadTrace()->activate(this);
+}
+
+class ThreadTraceData* Sampler::getThreadTrace()
+{
+	return LLThread::getTraceData();
+}
+
+///////////////////////////////////////////////////////////////////////
+// MasterThreadTrace
+///////////////////////////////////////////////////////////////////////
+
+void MasterThreadTrace::pullFromWorkerThreads()
+{
+	LLMutexLock lock(&mSlaveListMutex);
+
+	for (worker_thread_trace_list_t::iterator it = mSlaveThreadTraces.begin(), end_it = mSlaveThreadTraces.end();
+		it != end_it;
+		++it)
+	{
+		it->mWorkerTrace->mSharedData.copyTo(it->mSamplerStorage);
+	}
+}
+
+void MasterThreadTrace::addSlaveThread( class SlaveThreadTrace* child )
+{
+	LLMutexLock lock(&mSlaveListMutex);
+
+	mSlaveThreadTraces.push_back(WorkerThreadTraceProxy(child));
+}
+
+void MasterThreadTrace::removeSlaveThread( class SlaveThreadTrace* child )
+{
+	LLMutexLock lock(&mSlaveListMutex);
+
+	for (worker_thread_trace_list_t::iterator it = mSlaveThreadTraces.begin(), end_it = mSlaveThreadTraces.end();
+		it != end_it;
+		++it)
+	{
+		if (it->mWorkerTrace == child)
+		{
+			mSlaveThreadTraces.erase(it);
+			break;
+		}
+	}
+}
+
+}
diff --git a/indra/llcommon/lltrace.h b/indra/llcommon/lltrace.h
index 7da182df1e..401ddfd6f3 100644
--- a/indra/llcommon/lltrace.h
+++ b/indra/llcommon/lltrace.h
@@ -30,25 +30,27 @@
 #include "stdtypes.h"
 #include "llpreprocessor.h"
 
-#include "llthread.h"
+#include "llmutex.h"
+#include "llmemory.h"
+#include "llthreadlocalptr.h"
 
 #include <list>
 
+#define TOKEN_PASTE_ACTUAL(x, y) x##y
+#define TOKEN_PASTE(x, y) TOKEN_PASTE_ACTUAL(x, y)
+#define RECORD_BLOCK_TIME(block_timer) LLTrace::BlockTimer::Recorder TOKEN_PASTE(block_time_recorder, __COUNTER__)(block_timer);
+
 namespace LLTrace
 {
-
 	// one per thread per type
 	template<typename ACCUMULATOR>
-	struct AccumulatorBuffer : public AccumulatorBufferBase
+	class AccumulatorBuffer
 	{
-		ACCUMULATOR*				mStorage;
-		size_t						mStorageSize;
-		size_t						mNextStorageSlot;
-		static S32					sStorageKey; // key used to access thread local storage pointer to accumulator values
-		
+		static const U32 DEFAULT_ACCUMULATOR_BUFFER_SIZE = 64;
+	public:
 		AccumulatorBuffer()
 		:	mStorageSize(64),
-			mStorage(new ACCUMULATOR[64]),
+			mStorage(new ACCUMULATOR[DEFAULT_ACCUMULATOR_BUFFER_SIZE]),
 			mNextStorageSlot(0)
 		{}
 
@@ -56,12 +58,13 @@ namespace LLTrace
 		:	mStorageSize(other.mStorageSize),
 			mStorage(new ACCUMULATOR[other.mStorageSize]),
 			mNextStorageSlot(other.mNextStorageSlot)
-		{
+		{}
 
+		LL_FORCE_INLINE ACCUMULATOR& operator[](size_t index) 
+		{ 
+			return mStorage[index]; 
 		}
 
-		LL_FORCE_INLINE ACCUMULATOR& operator[](size_t index) { return (*mStorage)[index]; }
-
 		void mergeFrom(const AccumulatorBuffer<ACCUMULATOR>& other)
 		{
 			llassert(mNextStorageSlot == other.mNextStorageSlot);
@@ -72,7 +75,7 @@ namespace LLTrace
 			}
 		}
 
-		void copyFrom(const AccumulatorBuffer<Accumulator>& other)
+		void copyFrom(const AccumulatorBuffer<ACCUMULATOR>& other)
 		{
 			for (size_t i = 0; i < mNextStorageSlot; i++)
 			{
@@ -90,7 +93,12 @@ namespace LLTrace
 
 		void makePrimary()
 		{
-			//TODO: use sStorageKey to set mStorage as active buffer
+			sPrimaryStorage = mStorage;
+		}
+
+		LL_FORCE_INLINE static ACCUMULATOR* getPrimaryStorage() 
+		{ 
+			return sPrimaryStorage.get(); 
 		}
 
 		// NOTE: this is not thread-safe.  We assume that slots are reserved in the main thread before any child threads are spawned
@@ -101,14 +109,29 @@ namespace LLTrace
 			{
 				size_t new_size = mStorageSize + (mStorageSize >> 2);
 				delete [] mStorage;
-				mStorage = new mStorage(new_size);
+				mStorage = new ACCUMULATOR[new_size];
 				mStorageSize = new_size;
 			}
 			llassert(next_slot < mStorageSize);
 			return next_slot;
 		}
+
+	private:
+		ACCUMULATOR*							mStorage;
+		size_t									mStorageSize;
+		size_t									mNextStorageSlot;
+		static LLThreadLocalPtr<ACCUMULATOR>	sPrimaryStorage;
+	};
+	template<typename ACCUMULATOR> LLThreadLocalPtr<ACCUMULATOR> AccumulatorBuffer<ACCUMULATOR>::sPrimaryStorage;
+
+	template<typename ACCUMULATOR>
+	class PrimaryAccumulatorBuffer : public AccumulatorBuffer<ACCUMULATOR
+	{
+		PrimaryAccumulatorBuffer()
+		{
+			makePrimary();
+		}
 	};
-	template<typename ACCUMULATOR> S32 AccumulatorBuffer<ACCUMULATOR>::sStorageKey;
 
 	template<typename ACCUMULATOR>
 	class Trace
@@ -117,32 +140,34 @@ namespace LLTrace
 		Trace(const std::string& name)
 		:	mName(name)
 		{
-			mAccumulatorIndex = sAccumulatorBuffer.reserveSlot();
+			mAccumulatorIndex = getPrimaryBuffer().reserveSlot();
 		}
 
 		LL_FORCE_INLINE ACCUMULATOR& getAccumulator()
 		{
-			return sAccumulatorBuffer[mAccumulatorIndex];
+			return AccumulatorBuffer<ACCUMULATOR>::getPrimaryStorage()[mAccumulatorIndex];
+		}
+
+		static PrimaryAccumulatorBuffer& getPrimaryBuffer()
+		{
+			static PrimaryAccumulatorBuffer sBuffer;
+			return sBuffer;
 		}
 
 	private:
 		std::string	mName;
 		size_t		mAccumulatorIndex;
-
-		// this needs to be thread local
-		static AccumulatorBuffer<ACCUMULATOR>	sAccumulatorBuffer;
 	};
 
-	template<typename ACCUMULATOR> std::vector<ACCUMULATOR> Trace<ACCUMULATOR>::sAccumulatorBuffer;
 
 	template<typename T>
 	class StatAccumulator
 	{
 	public:
 		StatAccumulator()
-		:	mSum(),
-			mMin(),
-			mMax(),
+		:	mSum(0),
+			mMin(0),
+			mMax(0),
 			mNumSamples(0)
 		{}
 
@@ -160,7 +185,7 @@ namespace LLTrace
 			}
 		}
 
-		void mergeFrom(const Stat<T>& other)
+		void mergeFrom(const StatAccumulator<T>& other)
 		{
 			mSum += other.mSum;
 			if (other.mMin < mMin)
@@ -318,21 +343,13 @@ namespace LLTrace
 		static Recorder::StackEntry sCurRecorder;
 	};
 
-	BlockTimer::Recorder::StackEntry BlockTimer::sCurRecorder;
-
 	class Sampler
 	{
 	public:
-		Sampler(const Sampler& other)
-		:	mF32Stats(other.mF32Stats),
-			mS32Stats(other.mS32Stats),
-			mTimers(other.mTimers)
-		{}
+		Sampler() {}
+		Sampler(const Sampler& other);
 
-		~Sampler()
-		{
-			stop();
-		}
+		~Sampler();
 
 		void makePrimary()
 		{
@@ -347,17 +364,8 @@ namespace LLTrace
 			resume();
 		}
 
-		void stop()
-		{
-			getThreadTracer()->deactivate(this);
-		}
-
-		void resume()
-		{
-			ThreadTracer* thread_data = getThreadTracer();
-			thread_data->flushData();
-			thread_data->activate(this);
-		}
+		void stop();
+		void resume();
 
 		void mergeFrom(const Sampler& other)
 		{
@@ -375,7 +383,7 @@ namespace LLTrace
 
 	private:
 		// returns data for current thread
-		struct ThreadTracer* getThreadTracer() { return NULL; } 
+		class ThreadTraceData* getThreadTrace(); 
 
 		AccumulatorBuffer<StatAccumulator<F32> >	mF32Stats;
 		AccumulatorBuffer<StatAccumulator<S32> >	mS32Stats;
@@ -383,39 +391,39 @@ namespace LLTrace
 		AccumulatorBuffer<TimerAccumulator>			mTimers;
 	};
 
-	struct ThreadTracer
+	class ThreadTraceData
 	{
-		ThreadTracer(LLThread& this_thread, ThreadTracer& parent_data)
-		:	mPrimarySampler(parent_data.mPrimarySampler),
-			mSharedSampler(parent_data.mSharedSampler),
-			mSharedSamplerMutex(this_thread.getAPRPool()),
-			mParent(parent_data)
+	public:
+		ThreadTraceData()
 		{
 			mPrimarySampler.makePrimary();
-			parent_data.addChildThread(this);
 		}
 
-		~ThreadTracer()
+		ThreadTraceData(const ThreadTraceData& other)
+		:	mPrimarySampler(other.mPrimarySampler)
 		{
-			mParent.removeChildThread(this);
+			mPrimarySampler.makePrimary();
 		}
 
-		void addChildThread(ThreadTracer* child)
+		void activate(Sampler* sampler)
 		{
-			mChildThreadTracers.push_back(child);
+			flushPrimary();
+			mActiveSamplers.push_back(sampler);
 		}
 
-		void removeChildThread(ThreadTracer* child)
+		void deactivate(Sampler* sampler)
 		{
+			sampler->mergeFrom(mPrimarySampler);
+
 			// TODO: replace with intrusive list
-			std::list<ThreadTracer*>::iterator found_it = std::find(mChildThreadTracers.begin(), mChildThreadTracers.end(), child);
-			if (found_it != mChildThreadTracers.end())
+			std::list<Sampler*>::iterator found_it = std::find(mActiveSamplers.begin(), mActiveSamplers.end(), sampler);
+			if (found_it != mActiveSamplers.end())
 			{
-				mChildThreadTracers.erase(found_it);
+				mActiveSamplers.erase(found_it);
 			}
 		}
 
-		void flushData()
+		void flushPrimary()
 		{
 			for (std::list<Sampler*>::iterator it = mActiveSamplers.begin(), end_it = mActiveSamplers.end();
 				it != end_it;
@@ -426,56 +434,96 @@ namespace LLTrace
 			mPrimarySampler.reset();
 		}
 
-		void activate(Sampler* sampler)
+		Sampler* getPrimarySampler() { return &mPrimarySampler; }
+	protected:
+		Sampler					mPrimarySampler;
+		std::list<Sampler*>		mActiveSamplers;
+		static LLThreadLocalPtr<ThreadTraceData> sCurThreadTrace;
+	};
+
+	class MasterThreadTrace : public ThreadTraceData
+	{
+	public:
+		MasterThreadTrace()
+		{}
+
+		void addSlaveThread(class SlaveThreadTrace* child);
+		void removeSlaveThread(class SlaveThreadTrace* child);
+
+		// call this periodically to gather stats data from worker threads
+		void pullFromWorkerThreads();
+
+	private:
+		struct WorkerThreadTraceProxy
 		{
-			mActiveSamplers.push_back(sampler);
+			WorkerThreadTraceProxy(class SlaveThreadTrace* trace)
+				:	mWorkerTrace(trace)
+			{}
+			class SlaveThreadTrace*	mWorkerTrace;
+			Sampler				mSamplerStorage;
+		};
+		typedef std::list<WorkerThreadTraceProxy> worker_thread_trace_list_t;
+
+		worker_thread_trace_list_t		mSlaveThreadTraces;
+		LLMutex							mSlaveListMutex;
+	};
+
+	class SlaveThreadTrace : public ThreadTraceData
+	{
+	public:
+		explicit 
+		SlaveThreadTrace(MasterThreadTrace& master_trace)
+		:	mMaster(master_trace),
+			ThreadTraceData(master_trace),
+			mSharedData(mPrimarySampler)
+		{
+			master_trace.addSlaveThread(this);
 		}
 
-		void deactivate(Sampler* sampler)
+		~SlaveThreadTrace()
 		{
-			// TODO: replace with intrusive list
-			std::list<Sampler*>::iterator found_it = std::find(mActiveSamplers.begin(), mActiveSamplers.end(), sampler);
-			if (found_it != mActiveSamplers.end())
-			{
-				mActiveSamplers.erase(found_it);
-			}
+			mMaster.removeSlaveThread(this);
 		}
-		
-		// call this periodically to gather stats data in parent thread
-		void publishToParent()
+
+		// call this periodically to gather stats data for master thread to consume
+		void pushToParent()
 		{
-			mSharedSamplerMutex.lock();
-			{	
-				mSharedSampler.mergeFrom(mPrimarySampler);
-			}
-			mSharedSamplerMutex.unlock();
+			mSharedData.copyFrom(mPrimarySampler);
 		}
 
-		// call this periodically to gather stats data from children
-		void gatherChildData()
+		MasterThreadTrace& 	mMaster;
+
+		// this data is accessed by other threads, so give it a 64 byte alignment
+		// to avoid false sharing on most x86 processors
+		LL_ALIGNED(64) class SharedData
 		{
-			for (std::list<ThreadTracer*>::iterator child_it = mChildThreadTracers.begin(), end_it = mChildThreadTracers.end();
-				child_it != end_it;
-				++child_it)
+		public:
+			explicit 
+			SharedData(const Sampler& other_sampler) 
+			:	mSampler(other_sampler)
+			{}
+
+			void copyFrom(Sampler& source)
 			{
-				(*child_it)->mSharedSamplerMutex.lock();
-				{
-					//TODO for now, just aggregate, later keep track of thread data independently
-					mPrimarySampler.mergeFrom((*child_it)->mSharedSampler);
+				LLMutexLock lock(&mSamplerMutex);
+				{	
+					mSampler.mergeFrom(source);
 				}
-				(*child_it)->mSharedSamplerMutex.unlock();
 			}
-		}
 
-		Sampler	mPrimarySampler;
-
-		ThreadTracer&				mParent;
-		std::list<Sampler*>			mActiveSamplers;
-		std::list<ThreadTracer*>	mChildThreadTracers;
-
-		// TODO:  add unused space here to avoid false sharing?
-		LLMutex	mSharedSamplerMutex;
-		Sampler mSharedSampler;
+			void copyTo(Sampler& sink)
+			{
+				LLMutexLock lock(&mSamplerMutex);
+				{
+					sink.mergeFrom(mSampler);
+				}
+			}
+		private:
+			// add a cache line's worth of unused space to avoid any potential of false sharing
+			LLMutex					mSamplerMutex;
+			Sampler					mSampler;
+		};
+		SharedData					mSharedData;
 	};
 
 
@@ -486,19 +534,6 @@ namespace LLTrace
 		void stop() {}
 		void resume() {}
 	};
-
-	class Sampler
-	{
-	public:
-		void start() {}
-		void stop() {}
-		void resume() {}
-	};
-
 }
 
-#define TOKEN_PASTE_ACTUAL(x, y) x##y
-#define TOKEN_PASTE(x, y) TOKEN_PASTE_ACTUAL(x, y)
-#define RECORD_BLOCK_TIME(block_timer) LLTrace::BlockTimer::Recorder TOKEN_PASTE(block_time_recorder, __COUNTER__)(block_timer);
-
 #endif // LL_LLTRACE_H