SH-3468 WIP add memory tracking base class

improvements on lifetime of lltrace core data structures
tweaks to thread local pointer handling so that static constructors/destructors
can safely call functions that use lltrace

11 files changed, 326 insertions, 184 deletions

diff --git a/indra/llcommon/llapr.cpp b/indra/llcommon/llapr.cpp
index 1db3aa9e89..6affdad61b 100644
--- a/indra/llcommon/llapr.cpp
+++ b/indra/llcommon/llapr.cpp
@@ -542,12 +542,12 @@ void LLThreadLocalPointerBase::destroyAllThreadLocalStorage()
 {
 	if (sInitialized)
 	{
-		for (LLInstanceTracker<LLThreadLocalPointerBase>::instance_iter it = beginInstances(), end_it = endInstances();
-			it != end_it;
-			++it)
-		{
-			(*it).destroyStorage();
-		}
+		//for (LLInstanceTracker<LLThreadLocalPointerBase>::instance_iter it = beginInstances(), end_it = endInstances();
+		//	it != end_it;
+		//	++it)
+		//{
+		//	(*it).destroyStorage();
+		//}
 		sInitialized = false;
 	}
 }
diff --git a/indra/llcommon/llapr.h b/indra/llcommon/llapr.h
index f8f94263e4..2faf4aad2d 100644
--- a/indra/llcommon/llapr.h
+++ b/indra/llcommon/llapr.h
@@ -296,7 +296,7 @@ protected:
 
 	LL_FORCE_INLINE void* get()
 	{
-		llassert(sInitialized);
+		// llassert(sInitialized);
 		void* ptr;
 		apr_status_t result =
 		apr_threadkey_private_get(&ptr, mThreadKey);
diff --git a/indra/llcommon/llfasttimer.cpp b/indra/llcommon/llfasttimer.cpp
index f4fa8f1e61..7c90b946af 100644
--- a/indra/llcommon/llfasttimer.cpp
+++ b/indra/llcommon/llfasttimer.cpp
@@ -35,6 +35,7 @@
 #include "llunit.h"
 #include "llsd.h"
 #include "lltracerecording.h"
+#include "lltracethreadrecorder.h"
 
 #include <boost/bind.hpp>
 #include <queue>
@@ -167,52 +168,16 @@ U64 TimeBlock::countsPerSecond() // counts per second for the *64-bit* timer
 
 TimeBlock::TimeBlock(const char* name, bool open, TimeBlock* parent)
 :	TraceType(name),
-	mCollapsed(true),
-	mParent(NULL),
-	mNeedsSorting(false)
+	mCollapsed(true)
 {
 	setCollapsed(!open);
-
-	if (parent)
-	{
-		setParent(parent);
-	}
-	else
-	{
-		mParent = this;
-	}
 }
 
-void TimeBlock::setParent(TimeBlock* parent)
+TimeBlockTreeNode& TimeBlock::getTreeNode() const
 {
-	llassert_always(parent != this);
-	llassert_always(parent != NULL);
-
-	if (mParent)
-	{
-		//// subtract our accumulated from previous parent
-		//for (S32 i = 0; i < HISTORY_NUM; i++)
-		//{
-		//	mParent->mCountHistory[i] -= mCountHistory[i];
-		//}
-
-		//// subtract average timing from previous parent
-		//mParent->mCountAverage -= mCountAverage;
-
-		std::vector<TimeBlock*>& children = mParent->getChildren();
-		std::vector<TimeBlock*>::iterator found_it = std::find(children.begin(), children.end(), this);
-		if (found_it != children.end())
-		{
-			children.erase(found_it);
-		}
-	}
-
-	mParent = parent;
-	if (parent)
-	{
-		parent->getChildren().push_back(this);
-		parent->mNeedsSorting = true;
-	}
+	TimeBlockTreeNode* nodep = LLTrace::get_thread_recorder()->getTimeBlockTreeNode(getIndex());
+	llassert(nodep);
+	return *nodep;
 }
 
 // static
@@ -232,17 +197,17 @@ void TimeBlock::processTimes()
 
 		// bootstrap tree construction by attaching to last timer to be on stack
 		// when this timer was called
-		if (timer.mParent == &TimeBlock::getRootTimer())
+		if (timer.getParent() == &TimeBlock::getRootTimer())
 		{
-			TimeBlockAccumulator& accumulator = timer.getPrimaryAccumulator();
+			TimeBlockAccumulator* accumulator = timer.getPrimaryAccumulator();
 
-			if (accumulator.mLastCaller)
+			if (accumulator->mLastCaller)
 			{
-				timer.setParent(accumulator.mLastCaller);
-				accumulator.mParent = accumulator.mLastCaller;
+				timer.setParent(accumulator->mLastCaller);
+				accumulator->mParent = accumulator->mLastCaller;
 			}
 			// no need to push up tree on first use, flag can be set spuriously
-			accumulator.mMoveUpTree = false;
+			accumulator->mMoveUpTree = false;
 		}
 	}
 
@@ -256,25 +221,25 @@ void TimeBlock::processTimes()
 		TimeBlock* timerp = *it;
 
 		// sort timers by time last called, so call graph makes sense
-		if (timerp->mNeedsSorting)
+		if (timerp->getTreeNode().mNeedsSorting)
 		{
-			std::sort(timerp->getChildren().begin(), timerp->getChildren().end(), SortTimerByName());
+			std::sort(timerp->beginChildren(), timerp->endChildren(), SortTimerByName());
 		}
 
 		// skip root timer
 		if (timerp != &TimeBlock::getRootTimer())
 		{
-			TimeBlockAccumulator& accumulator = timerp->getPrimaryAccumulator();
+			TimeBlockAccumulator* accumulator = timerp->getPrimaryAccumulator();
 
-			if (accumulator.mMoveUpTree)
+			if (accumulator->mMoveUpTree)
 			{
 				// since ancestors have already been visited, re-parenting won't affect tree traversal
 				//step up tree, bringing our descendants with us
 				LL_DEBUGS("FastTimers") << "Moving " << timerp->getName() << " from child of " << timerp->getParent()->getName() <<
 					" to child of " << timerp->getParent()->getParent()->getName() << LL_ENDL;
 				timerp->setParent(timerp->getParent()->getParent());
-				accumulator.mParent = timerp->mParent;
-				accumulator.mMoveUpTree = false;
+				accumulator->mParent = timerp->getParent();
+				accumulator->mMoveUpTree = false;
 
 				// don't bubble up any ancestors until descendants are done bubbling up
 				// as ancestors may call this timer only on certain paths, so we want to resolve
@@ -286,15 +251,15 @@ void TimeBlock::processTimes()
 
 	// walk up stack of active timers and accumulate current time while leaving timing structures active
 	BlockTimer* cur_timer = cur_data->mCurTimer;
-	TimeBlockAccumulator& accumulator = cur_data->mTimerData->getPrimaryAccumulator();
+	TimeBlockAccumulator* accumulator = cur_data->mTimerData->getPrimaryAccumulator();
 	// root defined by parent pointing to self
 	while(cur_timer && cur_timer->mLastTimerData.mCurTimer != cur_timer)
 	{
 		U64 cumulative_time_delta = cur_time - cur_timer->mStartTime;
 		U64 self_time_delta = cumulative_time_delta - cur_data->mChildTime;
 		cur_data->mChildTime = 0;
-		accumulator.mSelfTimeCounter += self_time_delta;
-		accumulator.mTotalTimeCounter += cumulative_time_delta;
+		accumulator->mSelfTimeCounter += self_time_delta;
+		accumulator->mTotalTimeCounter += cumulative_time_delta;
 
 		cur_timer->mStartTime = cur_time;
 
@@ -316,10 +281,10 @@ void TimeBlock::processTimes()
 		++it)
 	{
 		TimeBlock& timer = *it;
-		TimeBlockAccumulator& accumulator = timer.getPrimaryAccumulator();
+		TimeBlockAccumulator* accumulator = timer.getPrimaryAccumulator();
 
-		accumulator.mLastCaller = NULL;
-		accumulator.mMoveUpTree = false;
+		accumulator->mLastCaller = NULL;
+		accumulator->mMoveUpTree = false;
 	}
 
 	// traverse tree in DFS post order, or bottom up
@@ -338,19 +303,19 @@ void TimeBlock::processTimes()
 }
 
 
-std::vector<TimeBlock*>::const_iterator TimeBlock::beginChildren()
+std::vector<TimeBlock*>::iterator TimeBlock::beginChildren()
 { 
-	return mChildren.begin(); 
+	return getTreeNode().mChildren.begin(); 
 }
 
-std::vector<TimeBlock*>::const_iterator TimeBlock::endChildren()
+std::vector<TimeBlock*>::iterator TimeBlock::endChildren()
 {
-	return mChildren.end();
+	return getTreeNode().mChildren.end();
 }
 
 std::vector<TimeBlock*>& TimeBlock::getChildren()
 {
-	return mChildren;
+	return getTreeNode().mChildren;
 }
 
 //static
diff --git a/indra/llcommon/llfasttimer.h b/indra/llcommon/llfasttimer.h
index 1e2e4b590f..4d820d0664 100644
--- a/indra/llcommon/llfasttimer.h
+++ b/indra/llcommon/llfasttimer.h
@@ -69,12 +69,14 @@ class TimeBlock
 public:
 	TimeBlock(const char* name, bool open = false, TimeBlock* parent = &getRootTimer());
 
-	TimeBlock* getParent() const { return mParent; }
-	void setParent(TimeBlock* parent);
+	TimeBlockTreeNode& getTreeNode() const;
+	TimeBlock* getParent() const { return getTreeNode().getParent(); }
+	void setParent(TimeBlock* parent) { getTreeNode().setParent(parent); }
 
+	typedef std::vector<TimeBlock*>::iterator child_iter;
 	typedef std::vector<TimeBlock*>::const_iterator child_const_iter;
-	child_const_iter beginChildren();
-	child_const_iter endChildren();
+	child_iter beginChildren();
+	child_iter endChildren();
 	std::vector<TimeBlock*>& getChildren();
 
 	void setCollapsed(bool collapsed)	{ mCollapsed = collapsed; }
@@ -244,11 +246,7 @@ public:
 	// call this once a frame to periodically log timers
 	static void logStats();
 
-	// tree structure, only updated from master trace thread
-	TimeBlock*					mParent;				// TimeBlock of caller(parent)
-	std::vector<TimeBlock*>		mChildren;				// TimeBlock of callees
-	bool						mCollapsed,				// don't show children
-								mNeedsSorting;			// sort children whenever child added
+	bool						mCollapsed;				// don't show children
 
 	// statics
 	static std::string							sLogName;
@@ -264,10 +262,10 @@ LL_FORCE_INLINE BlockTimer::BlockTimer(TimeBlock& timer)
 	mStartTime = TimeBlock::getCPUClockCount64();
 
 	CurTimerData* cur_timer_data = TimeBlock::sCurTimerData.get();
-	TimeBlockAccumulator& accumulator = cur_timer_data->mTimerData->getPrimaryAccumulator();
-	accumulator.mActiveCount++;
+	TimeBlockAccumulator* accumulator = cur_timer_data->mTimerData->getPrimaryAccumulator();
+	accumulator->mActiveCount++;
 	// keep current parent as long as it is active when we are
-	accumulator.mMoveUpTree |= (accumulator.mParent->getPrimaryAccumulator().mActiveCount == 0);
+	accumulator->mMoveUpTree |= (accumulator->mParent->getPrimaryAccumulator()->mActiveCount == 0);
 
 	// store top of stack
 	mLastTimerData = *cur_timer_data;
@@ -283,16 +281,16 @@ LL_FORCE_INLINE BlockTimer::~BlockTimer()
 #if FAST_TIMER_ON
 	U64 total_time = TimeBlock::getCPUClockCount64() - mStartTime;
 	CurTimerData* cur_timer_data = TimeBlock::sCurTimerData.get();
-	TimeBlockAccumulator& accumulator = cur_timer_data->mTimerData->getPrimaryAccumulator();
+	TimeBlockAccumulator* accumulator = cur_timer_data->mTimerData->getPrimaryAccumulator();
 
-	accumulator.mCalls++;
-	accumulator.mSelfTimeCounter += total_time - cur_timer_data->mChildTime;
-	accumulator.mTotalTimeCounter += total_time;
-	accumulator.mActiveCount--;
+	accumulator->mCalls++;
+	accumulator->mSelfTimeCounter += total_time - cur_timer_data->mChildTime;
+	accumulator->mTotalTimeCounter += total_time;
+	accumulator->mActiveCount--;
 
 	// store last caller to bootstrap tree creation
 	// do this in the destructor in case of recursion to get topmost caller
-	accumulator.mLastCaller = mLastTimerData.mTimerData;
+	accumulator->mLastCaller = mLastTimerData.mTimerData;
 
 	// we are only tracking self time, so subtract our total time delta from parents
 	mLastTimerData.mChildTime += total_time;
diff --git a/indra/llcommon/lltrace.cpp b/indra/llcommon/lltrace.cpp
index e11e39a1a2..9d0c93b352 100644
--- a/indra/llcommon/lltrace.cpp
+++ b/indra/llcommon/lltrace.cpp
@@ -66,5 +66,34 @@ LLThreadLocalPointer<ThreadRecorder>& get_thread_recorder()
 	return s_thread_recorder;
 }
 
+TimeBlockTreeNode::TimeBlockTreeNode() 
+:	mBlock(NULL),
+	mParent(NULL),
+	mNeedsSorting(false)
+{}
+
+void TimeBlockTreeNode::setParent( TimeBlock* parent )
+{
+	llassert_always(parent != mBlock);
+	llassert_always(parent != NULL);
+
+	TimeBlockTreeNode* parent_tree_node = get_thread_recorder()->getTimeBlockTreeNode(parent->getIndex());
+	if (!parent_tree_node) return;
+
+	if (mParent)
+	{
+		std::vector<TimeBlock*>& children = mParent->getChildren();
+		std::vector<TimeBlock*>::iterator found_it = std::find(children.begin(), children.end(), mBlock);
+		if (found_it != children.end())
+		{
+			children.erase(found_it);
+		}
+	}
+
+	mParent = parent;
+	mBlock->getPrimaryAccumulator()->mParent = parent;
+	parent_tree_node->mChildren.push_back(mBlock);
+	parent_tree_node->mNeedsSorting = true;
 }
 
+}
diff --git a/indra/llcommon/lltrace.h b/indra/llcommon/lltrace.h
index 5b5e2b7879..4bb19f6070 100644
--- a/indra/llcommon/lltrace.h
+++ b/indra/llcommon/lltrace.h
@@ -77,21 +77,21 @@ namespace LLTrace
 	template<typename ACCUMULATOR>
 	class AccumulatorBuffer : public LLRefCount
 	{
+		typedef AccumulatorBuffer<ACCUMULATOR> self_t;
 		static const U32 DEFAULT_ACCUMULATOR_BUFFER_SIZE = 64;
 	private:
-		enum StaticAllocationMarker { STATIC_ALLOC };
+		struct StaticAllocationMarker { };
 
 		AccumulatorBuffer(StaticAllocationMarker m)
 		:	mStorageSize(0),
 			mStorage(NULL),
 			mNextStorageSlot(0)
 		{
-			resize(DEFAULT_ACCUMULATOR_BUFFER_SIZE);
 		}
 
 	public:
 
-		AccumulatorBuffer(const AccumulatorBuffer& other = getDefaultBuffer())
+		AccumulatorBuffer(const AccumulatorBuffer& other = *getDefaultBuffer())
 		:	mStorageSize(0),
 			mStorage(NULL),
 			mNextStorageSlot(other.mNextStorageSlot)
@@ -107,8 +107,7 @@ namespace LLTrace
 		{
 			if (sPrimaryStorage == mStorage)
 			{
-				//TODO pick another primary?
-				sPrimaryStorage = NULL;
+				sPrimaryStorage = getDefaultBuffer()->mStorage;
 			}
 			delete[] mStorage;
 		}
@@ -182,6 +181,8 @@ namespace LLTrace
 
 		void resize(size_t new_size)
 		{
+			if (new_size <= mStorageSize) return;
+
 			ACCUMULATOR* old_storage = mStorage;
 			mStorage = new ACCUMULATOR[new_size];
 			if (old_storage)
@@ -193,16 +194,33 @@ namespace LLTrace
 			}
 			mStorageSize = new_size;
 			delete[] old_storage;
+
+			self_t* default_buffer = getDefaultBuffer();
+			if (this != default_buffer
+				&& new_size > default_buffer->size())
+			{
+				//NB: this is not thread safe, but we assume that all resizing occurs during static initialization
+				default_buffer->resize(new_size);
+			}
 		}
 
-		size_t size()
+		size_t size() const
 		{
 			return mNextStorageSlot;
 		}
 
-		static AccumulatorBuffer<ACCUMULATOR>& getDefaultBuffer()
+		static self_t* getDefaultBuffer()
 		{
-			static AccumulatorBuffer sBuffer(STATIC_ALLOC);
+			// this buffer is allowed to leak so that trace calls from global destructors have somewhere to put their data
+			// so as not to trigger an access violation
+			//TODO: make this thread local but need to either demand-init apr or remove apr dependency
+			static self_t* sBuffer = new AccumulatorBuffer(StaticAllocationMarker());
+			static bool sInitialized = false;
+			if (!sInitialized)
+			{
+				sBuffer->resize(DEFAULT_ACCUMULATOR_BUFFER_SIZE);
+				sInitialized = true;
+			}
 			return sBuffer;
 		}
 
@@ -233,13 +251,13 @@ namespace LLTrace
 			mName(name),
 			mDescription(description ? description : "")	
 		{
-			mAccumulatorIndex = AccumulatorBuffer<ACCUMULATOR>::getDefaultBuffer().reserveSlot();
+			mAccumulatorIndex = AccumulatorBuffer<ACCUMULATOR>::getDefaultBuffer()->reserveSlot();
 		}
 
-		LL_FORCE_INLINE ACCUMULATOR& getPrimaryAccumulator() const
+		LL_FORCE_INLINE ACCUMULATOR* getPrimaryAccumulator() const
 		{
 			ACCUMULATOR* accumulator_storage = AccumulatorBuffer<ACCUMULATOR>::getPrimaryStorage();
-			return accumulator_storage[mAccumulatorIndex];
+			return &accumulator_storage[mAccumulatorIndex];
 		}
 
 		size_t getIndex() const { return mAccumulatorIndex; }
@@ -472,6 +490,22 @@ namespace LLTrace
 		{}
 	};
 
+	class TimeBlock;
+	class TimeBlockTreeNode
+	{
+	public:
+		TimeBlockTreeNode();
+
+		void setParent(TimeBlock* parent);
+		TimeBlock* getParent() { return mParent; }
+
+		TimeBlock*					mBlock;
+		TimeBlock*					mParent;	
+		std::vector<TimeBlock*>		mChildren;
+		bool						mNeedsSorting;
+	};
+
+
 	template <typename T = F64>
 	class Measurement
 	:	public TraceType<MeasurementAccumulator<typename LLUnits::HighestPrecisionType<T>::type_t> >
@@ -488,7 +522,7 @@ namespace LLTrace
 		void sample(UNIT_T value)
 		{
 			T converted_value(value);
-			trace_t::getPrimaryAccumulator().sample(LLUnits::rawValue(converted_value));
+			trace_t::getPrimaryAccumulator()->sample(LLUnits::rawValue(converted_value));
 		}
 	};
 
@@ -508,7 +542,7 @@ namespace LLTrace
 		void add(UNIT_T value)
 		{
 			T converted_value(value);
-			trace_t::getPrimaryAccumulator().add(LLUnits::rawValue(converted_value));
+			trace_t::getPrimaryAccumulator()->add(LLUnits::rawValue(converted_value));
 		}
 	};
 
@@ -650,19 +684,25 @@ public:
 		// reserve 8 bytes for allocation size (and preserving 8 byte alignment of structs)
 		void* allocation = ::operator new(allocation_size + 8);
 		*(size_t*)allocation = allocation_size;
-		MemStatAccumulator& accumulator = sStat.getPrimaryAccumulator();
-		accumulator.mSize += allocation_size;
-		accumulator.mAllocatedCount++;
+		MemStatAccumulator* accumulator = sStat.getPrimaryAccumulator();
+		if (accumulator)
+		{
+			accumulator->mSize += allocation_size;
+			accumulator->mAllocatedCount++;
+		}
 		return (void*)((char*)allocation + 8);
 	}
 
 	void operator delete(void* ptr)
 	{
 		size_t* allocation_size = (size_t*)((char*)ptr - 8);
-		MemStatAccumulator& accumulator = sStat.getPrimaryAccumulator();
-		accumulator.mSize -= *allocation_size;
-		accumulator.mAllocatedCount--;
-		accumulator.mDeallocatedCount++;
+		MemStatAccumulator* accumulator = sStat.getPrimaryAccumulator();
+		if (accumulator)
+		{
+			accumulator->mSize -= *allocation_size;
+			accumulator->mAllocatedCount--;
+			accumulator->mDeallocatedCount++;
+		}
 		::delete((char*)ptr - 8);
 	}
 
@@ -670,19 +710,25 @@ public:
 	{
 		size_t* result = (size_t*)malloc(size + 8);
 		*result = size;
-		MemStatAccumulator& accumulator = sStat.getPrimaryAccumulator();
-		accumulator.mSize += size;
-		accumulator.mAllocatedCount++;
+		MemStatAccumulator* accumulator = sStat.getPrimaryAccumulator();
+		if (accumulator)
+		{
+			accumulator->mSize += size;
+			accumulator->mAllocatedCount++;
+		}
 		return (void*)((char*)result + 8);
 	}
 
 	void operator delete[](void* ptr)
 	{
 		size_t* allocation_size = (size_t*)((char*)ptr - 8);
-		MemStatAccumulator& accumulator = sStat.getPrimaryAccumulator();
-		accumulator.mSize -= *allocation_size;
-		accumulator.mAllocatedCount--;
-		accumulator.mDeallocatedCount++;
+		MemStatAccumulator* accumulator = sStat.getPrimaryAccumulator();
+		if (accumulator)
+		{
+			accumulator->mSize -= *allocation_size;
+			accumulator->mAllocatedCount--;
+			accumulator->mDeallocatedCount++;
+		}
 		::delete[]((char*)ptr - 8);
 	}
 
@@ -704,9 +750,12 @@ public:
 
 	void memClaim(size_t size)
 	{
-		MemStatAccumulator& accumulator = sStat.getPrimaryAccumulator();
+		MemStatAccumulator* accumulator = sStat.getPrimaryAccumulator();
 		mMemFootprint += size;
-		accumulator.mSize += size;
+		if (accumulator)
+		{
+			accumulator->mSize += size;
+		}
 	}
 
 	// remove memory we had claimed from our calculated footprint
@@ -726,8 +775,11 @@ public:
 
 	void memDisclaim(size_t size)
 	{
-		MemStatAccumulator& accumulator = sStat.getPrimaryAccumulator();
-		accumulator.mSize -= size;
+		MemStatAccumulator* accumulator = sStat.getPrimaryAccumulator();
+		if (accumulator)
+		{
+			accumulator->mSize -= size;
+		}
 		mMemFootprint -= size;
 	}
 
@@ -739,18 +791,24 @@ private:
 	{
 		static void claim(mem_trackable_t& tracker, const TRACKED& tracked)
 		{
-			MemStatAccumulator& accumulator = sStat.getPrimaryAccumulator();
-			size_t footprint = MemFootprint<TRACKED>::measure(tracked);
-			accumulator.mSize += footprint;
-			tracker.mMemFootprint += footprint;
+			MemStatAccumulator* accumulator = sStat.getPrimaryAccumulator();
+			if (accumulator)
+			{
+				size_t footprint = MemFootprint<TRACKED>::measure(tracked);
+				accumulator->mSize += footprint;
+				tracker.mMemFootprint += footprint;
+			}
 		}
 
 		static void disclaim(mem_trackable_t& tracker, const TRACKED& tracked)
 		{
-			MemStatAccumulator& accumulator = sStat.getPrimaryAccumulator();
-			size_t footprint = MemFootprint<TRACKED>::measure(tracked);
-			accumulator.mSize -= footprint;
-			tracker.mMemFootprint -= footprint;
+			MemStatAccumulator* accumulator = sStat.getPrimaryAccumulator();
+			if (accumulator)
+			{
+				size_t footprint = MemFootprint<TRACKED>::measure(tracked);
+				accumulator->mSize -= footprint;
+				tracker.mMemFootprint -= footprint;
+			}
 		}
 	};
 
@@ -759,14 +817,20 @@ private:
 	{
 		static void claim(mem_trackable_t& tracker, TRACKED& tracked)
 		{
-			MemStatAccumulator& accumulator = sStat.getPrimaryAccumulator();
-			accumulator.mChildSize += MemFootprint<TRACKED>::measure(tracked);
+			MemStatAccumulator* accumulator = sStat.getPrimaryAccumulator();
+			if (accumulator)
+			{
+				accumulator->mChildSize += MemFootprint<TRACKED>::measure(tracked);
+			}
 		}
 
 		static void disclaim(mem_trackable_t& tracker, TRACKED& tracked)
 		{
-			MemStatAccumulator& accumulator = sStat.getPrimaryAccumulator();
-			accumulator.mChildSize -= MemFootprint<TRACKED>::measure(tracked);
+			MemStatAccumulator* accumulator = sStat.getPrimaryAccumulator();
+			if (accumulator)
+			{
+				accumulator->mChildSize -= MemFootprint<TRACKED>::measure(tracked);
+			}
 		}
 	};
 	static MemStat sStat;
diff --git a/indra/llcommon/lltracerecording.cpp b/indra/llcommon/lltracerecording.cpp
index 40079f40f1..9dbafc4e82 100644
--- a/indra/llcommon/lltracerecording.cpp
+++ b/indra/llcommon/lltracerecording.cpp
@@ -122,6 +122,18 @@ void Recording::makePrimary()
 	mMeasurements.write()->makePrimary();
 	mStackTimers.write()->makePrimary();
 	mMemStats.write()->makePrimary();
+
+	ThreadRecorder* thread_recorder = get_thread_recorder().get();
+	AccumulatorBuffer<TimeBlockAccumulator>& timer_accumulator_buffer = *mStackTimers.write();
+	// update stacktimer parent pointers
+	for (S32 i = 0, end_i = mStackTimers->size(); i < end_i; i++)
+	{
+		TimeBlockTreeNode* tree_node = thread_recorder->getTimeBlockTreeNode(i);
+		if (tree_node)
+		{
+			timer_accumulator_buffer[i].mParent = tree_node->mParent;
+		}
+	}
 }
 
 bool Recording::isPrimary() const
@@ -145,11 +157,21 @@ void Recording::appendRecording( const Recording& other )
 	mMeasurementsFloat.write()->addSamples(*other.mMeasurementsFloat);
 	mCounts.write()->addSamples(*other.mCounts);
 	mMeasurements.write()->addSamples(*other.mMeasurements);
-	mStackTimers.write()->addSamples(*other.mStackTimers);
 	mMemStats.write()->addSamples(*other.mMemStats);
+
+	mStackTimers.write()->addSamples(*other.mStackTimers);
 	mElapsedSeconds += other.mElapsedSeconds;
 }
 
+void Recording::mergeRecording( const Recording& other)
+{
+	mCountsFloat.write()->addSamples(*other.mCountsFloat);
+	mMeasurementsFloat.write()->addSamples(*other.mMeasurementsFloat);
+	mCounts.write()->addSamples(*other.mCounts);
+	mMeasurements.write()->addSamples(*other.mMeasurements);
+	mMemStats.write()->addSamples(*other.mMemStats);
+}
+
 LLUnit<LLUnits::Seconds, F64> Recording::getSum(const TraceType<TimeBlockAccumulator>& stat) const
 {
 	return (F64)(*mStackTimers)[stat.getIndex()].mTotalTimeCounter / (F64)LLTrace::TimeBlock::countsPerSecond();
diff --git a/indra/llcommon/lltracerecording.h b/indra/llcommon/lltracerecording.h
index 8e5f1125ec..f2d7d42211 100644
--- a/indra/llcommon/lltracerecording.h
+++ b/indra/llcommon/lltracerecording.h
@@ -114,8 +114,12 @@ namespace LLTrace
 
 		void makeUnique();
 
+		// accumulate data from subsequent, non-overlapping recording
 		void appendRecording(const Recording& other);
 
+		// gather data from recording, ignoring time relationship (for example, pulling data from slave threads)
+		void mergeRecording(const Recording& other);
+
 		void update();
 
 		// Timer accessors
diff --git a/indra/llcommon/lltracethreadrecorder.cpp b/indra/llcommon/lltracethreadrecorder.cpp
index c4144b4999..156b0ef26b 100644
--- a/indra/llcommon/lltracethreadrecorder.cpp
+++ b/indra/llcommon/lltracethreadrecorder.cpp
@@ -38,25 +38,37 @@ namespace LLTrace
 
 ThreadRecorder::ThreadRecorder()
 {
+	//NB: the ordering of initialization in this function is very fragile due to a large number of implicit dependencies
 	get_thread_recorder() = this;
-	mFullRecording.start();
 
 	mRootTimerData = new CurTimerData();
 	mRootTimerData->mTimerData = &TimeBlock::getRootTimer();
-
 	TimeBlock::sCurTimerData = mRootTimerData;
-	TimeBlock::getRootTimer().getPrimaryAccumulator().mActiveCount = 1;
 
-	// initialize parent pointers in time blocks
+	mNumTimeBlockTreeNodes = AccumulatorBuffer<TimeBlockAccumulator>::getDefaultBuffer()->size();
+	mTimeBlockTreeNodes = new TimeBlockTreeNode[mNumTimeBlockTreeNodes];
+
+	mFullRecording.start();
+
+	TimeBlock& root_timer = TimeBlock::getRootTimer();
+
+	// initialize time block parent pointers
 	for (LLInstanceTracker<TimeBlock>::instance_iter it = LLInstanceTracker<TimeBlock>::beginInstances(), end_it = LLInstanceTracker<TimeBlock>::endInstances(); 
 		it != end_it; 
 		++it)
 	{
-		it->getPrimaryAccumulator().mParent = it->mParent;
+		TimeBlock& time_block = *it;
+		TimeBlockTreeNode& tree_node = mTimeBlockTreeNodes[it->getIndex()];
+		tree_node.mBlock = &time_block;
+		tree_node.mParent = &root_timer;
+
+		it->getPrimaryAccumulator()->mParent = &root_timer;
 	}
 
-	mRootTimer = new BlockTimer(TimeBlock::getRootTimer());
+	mRootTimer = new BlockTimer(root_timer);
 	mRootTimerData->mCurTimer = mRootTimer;
+
+	TimeBlock::getRootTimer().getPrimaryAccumulator()->mActiveCount = 1;
 }
 
 ThreadRecorder::~ThreadRecorder()
@@ -70,8 +82,19 @@ ThreadRecorder::~ThreadRecorder()
 	get_thread_recorder() = NULL;
 	TimeBlock::sCurTimerData = NULL;
 	delete mRootTimerData;
+	delete[] mTimeBlockTreeNodes;
 }
 
+TimeBlockTreeNode* ThreadRecorder::getTimeBlockTreeNode(S32 index)
+{
+	if (0 <= index && index < mNumTimeBlockTreeNodes)
+	{
+		return &mTimeBlockTreeNodes[index];
+	}
+	return NULL;
+}
+
+
 void ThreadRecorder::activate( Recording* recording )
 {
 	mActiveRecordings.push_front(ActiveRecording(recording));
@@ -113,6 +136,13 @@ std::list<ThreadRecorder::ActiveRecording>::iterator ThreadRecorder::update( Rec
 	return it;
 }
 
+AccumulatorBuffer<CountAccumulator<F64> > 		gCountsFloat;
+AccumulatorBuffer<MeasurementAccumulator<F64> >	gMeasurementsFloat;
+AccumulatorBuffer<CountAccumulator<S64> >		gCounts;
+AccumulatorBuffer<MeasurementAccumulator<S64> >	gMeasurements;
+AccumulatorBuffer<TimeBlockAccumulator>			gStackTimers;
+AccumulatorBuffer<MemStatAccumulator>			gMemStats;
+
 void ThreadRecorder::deactivate( Recording* recording )
 {
 	std::list<ActiveRecording>::iterator it = update(recording);
@@ -169,23 +199,42 @@ void SlaveThreadRecorder::pushToMaster()
 	mFullRecording.stop();
 	{
 		LLMutexLock(getMasterThreadRecorder().getSlaveListMutex());
-		mSharedData.copyFrom(mFullRecording);
+		mSharedData.appendFrom(mFullRecording);
 	}
 	mFullRecording.start();
 }
 
-void SlaveThreadRecorder::SharedData::copyFrom( const Recording& source )
+void SlaveThreadRecorder::SharedData::appendFrom( const Recording& source )
 {
 	LLMutexLock lock(&mRecordingMutex);
 	mRecording.appendRecording(source);
 }
 
-void SlaveThreadRecorder::SharedData::copyTo( Recording& sink )
+void SlaveThreadRecorder::SharedData::appendTo( Recording& sink )
 {
 	LLMutexLock lock(&mRecordingMutex);
 	sink.appendRecording(mRecording);
 }
 
+void SlaveThreadRecorder::SharedData::mergeFrom( const Recording& source )
+{
+	LLMutexLock lock(&mRecordingMutex);
+	mRecording.mergeRecording(source);
+}
+
+void SlaveThreadRecorder::SharedData::mergeTo( Recording& sink )
+{
+	LLMutexLock lock(&mRecordingMutex);
+	sink.mergeRecording(mRecording);
+}
+
+void SlaveThreadRecorder::SharedData::reset()
+{
+	LLMutexLock lock(&mRecordingMutex);
+	mRecording.reset();
+}
+
+
 ///////////////////////////////////////////////////////////////////////
 // MasterThreadRecorder
 ///////////////////////////////////////////////////////////////////////
@@ -203,7 +252,9 @@ void MasterThreadRecorder::pullFromSlaveThreads()
 		it != end_it;
 		++it)
 	{
-		(*it)->mSharedData.copyTo(target_recording);
+		// ignore block timing info for now
+		(*it)->mSharedData.mergeTo(target_recording);
+		(*it)->mSharedData.reset();
 	}
 }
 
diff --git a/indra/llcommon/lltracethreadrecorder.h b/indra/llcommon/lltracethreadrecorder.h
index 102b980e44..d09527eced 100644
--- a/indra/llcommon/lltracethreadrecorder.h
+++ b/indra/llcommon/lltracethreadrecorder.h
@@ -50,6 +50,8 @@ namespace LLTrace
 
 		virtual void pushToMaster() = 0;
 
+		TimeBlockTreeNode* getTimeBlockTreeNode(S32 index);
+
 	protected:
 		struct ActiveRecording
 		{
@@ -64,7 +66,9 @@ namespace LLTrace
 		std::list<ActiveRecording>	mActiveRecordings;
 
 		struct CurTimerData*	mRootTimerData;
-		class BlockTimer*				mRootTimer;
+		class BlockTimer*		mRootTimer;
+		TimeBlockTreeNode*		mTimeBlockTreeNodes;
+		size_t					mNumTimeBlockTreeNodes;
 	};
 
 	class LL_COMMON_API MasterThreadRecorder : public ThreadRecorder
@@ -104,8 +108,11 @@ namespace LLTrace
 		class SharedData
 		{
 		public:
-			void copyFrom(const Recording& source);
-			void copyTo(Recording& sink);
+			void appendFrom(const Recording& source);
+			void appendTo(Recording& sink);
+			void mergeFrom(const Recording& source);
+			void mergeTo(Recording& sink);
+			void reset();
 		private:
 			LLMutex		mRecordingMutex;
 			Recording	mRecording;
diff --git a/indra/newview/llfasttimerview.cpp b/indra/newview/llfasttimerview.cpp
index 2cf7939e5c..56e5710726 100644
--- a/indra/newview/llfasttimerview.cpp
+++ b/indra/newview/llfasttimerview.cpp
@@ -54,23 +54,25 @@
 
 //////////////////////////////////////////////////////////////////////////////
 
+using namespace LLTrace;
+
 static const S32 MAX_VISIBLE_HISTORY = 10;
 static const S32 LINE_GRAPH_HEIGHT = 240;
 
 const S32 FTV_MAX_DEPTH = 8;
 const S32 HISTORY_NUM = 300;
 
-std::vector<LLTrace::TimeBlock*> ft_display_idx; // line of table entry for display purposes (for collapse)
+std::vector<TimeBlock*> ft_display_idx; // line of table entry for display purposes (for collapse)
 
-typedef LLTreeDFSIter<LLTrace::TimeBlock, LLTrace::TimeBlock::child_const_iter> timer_tree_iterator_t;
+typedef LLTreeDFSIter<TimeBlock, TimeBlock::child_const_iter> timer_tree_iterator_t;
 
 BOOL LLFastTimerView::sAnalyzePerformance = FALSE;
 
-static timer_tree_iterator_t begin_timer_tree(LLTrace::TimeBlock& id) 
+static timer_tree_iterator_t begin_timer_tree(TimeBlock& id) 
 { 
 	return timer_tree_iterator_t(&id, 
-							boost::bind(boost::mem_fn(&LLTrace::TimeBlock::beginChildren), _1), 
-							boost::bind(boost::mem_fn(&LLTrace::TimeBlock::endChildren), _1));
+							boost::bind(boost::mem_fn(&TimeBlock::beginChildren), _1), 
+							boost::bind(boost::mem_fn(&TimeBlock::endChildren), _1));
 }
 
 static timer_tree_iterator_t end_timer_tree() 
@@ -81,12 +83,12 @@ static timer_tree_iterator_t end_timer_tree()
 S32 get_depth(const TimeBlock* blockp)
 {
 	S32 depth = 0;
-	TimeBlock* timerp = blockp->mParent;
+	TimeBlock* timerp = blockp->getParent();
 	while(timerp)
 	{
 		depth++;
 		if (timerp->getParent() == timerp) break;
-		timerp = timerp->mParent;
+		timerp = timerp->getParent();
 	}
 	return depth;
 }
@@ -102,13 +104,13 @@ LLFastTimerView::LLFastTimerView(const LLSD& key)
 	mHoverID(NULL),
 	mHoverBarIndex(-1),
 	mPrintStats(-1),
-	mRecording(&LLTrace::get_frame_recording()),
+	mRecording(&get_frame_recording()),
 	mPauseHistory(false)
 {}
 
 LLFastTimerView::~LLFastTimerView()
 {
-	if (mRecording != &LLTrace::get_frame_recording())
+	if (mRecording != &get_frame_recording())
 	{
 		delete mRecording;
 	}
@@ -121,17 +123,17 @@ void LLFastTimerView::onPause()
 	// reset scroll to bottom when unpausing
 	if (!mPauseHistory)
 	{
-		mRecording = new LLTrace::PeriodicRecording(LLTrace::get_frame_recording());
+		mRecording = new PeriodicRecording(get_frame_recording());
 		mScrollIndex = 0;
 		getChild<LLButton>("pause_btn")->setLabel(getString("pause"));
 	}
 	else
 	{
-		if (mRecording != &LLTrace::get_frame_recording())
+		if (mRecording != &get_frame_recording())
 		{
 			delete mRecording;
 		}
-		mRecording = &LLTrace::get_frame_recording();
+		mRecording = &get_frame_recording();
 
 		getChild<LLButton>("pause_btn")->setLabel(getString("run"));
 	}
@@ -170,7 +172,7 @@ BOOL LLFastTimerView::handleRightMouseDown(S32 x, S32 y, MASK mask)
 	return LLFloater::handleRightMouseDown(x, y, mask);
 }
 
-LLTrace::TimeBlock* LLFastTimerView::getLegendID(S32 y)
+TimeBlock* LLFastTimerView::getLegendID(S32 y)
 {
 	S32 idx = (getRect().getHeight() - y) / (LLFontGL::getFontMonospace()->getLineHeight()+2) - 5;
 
@@ -197,7 +199,7 @@ BOOL LLFastTimerView::handleMouseDown(S32 x, S32 y, MASK mask)
 {
 	if (x < mBarRect.mLeft) 
 	{
-		LLTrace::TimeBlock* idp = getLegendID(y);
+		TimeBlock* idp = getLegendID(y);
 		if (idp)
 		{
 			idp->setCollapsed(!idp->getCollapsed());
@@ -248,7 +250,7 @@ BOOL LLFastTimerView::handleMouseUp(S32 x, S32 y, MASK mask)
 
 BOOL LLFastTimerView::handleHover(S32 x, S32 y, MASK mask)
 {
-	LLTrace::PeriodicRecording& frame_recording = *mRecording;
+	PeriodicRecording& frame_recording = *mRecording;
 
 	if (hasMouseCapture())
 	{
@@ -306,7 +308,7 @@ BOOL LLFastTimerView::handleHover(S32 x, S32 y, MASK mask)
 	}
 	else if (x < mBarRect.mLeft) 
 	{
-		LLTrace::TimeBlock* timer_id = getLegendID(y);
+		TimeBlock* timer_id = getLegendID(y);
 		if (timer_id)
 		{
 			mHoverID = timer_id;
@@ -317,9 +319,9 @@ BOOL LLFastTimerView::handleHover(S32 x, S32 y, MASK mask)
 }
 
 
-static std::string get_tooltip(LLTrace::TimeBlock& timer, S32 history_index, LLTrace::PeriodicRecording& frame_recording)
+static std::string get_tooltip(TimeBlock& timer, S32 history_index, PeriodicRecording& frame_recording)
 {
-	F64 ms_multiplier = 1000.0 / (F64)LLTrace::TimeBlock::countsPerSecond();
+	F64 ms_multiplier = 1000.0 / (F64)TimeBlock::countsPerSecond();
 
 	std::string tooltip;
 	if (history_index < 0)
@@ -359,7 +361,7 @@ BOOL LLFastTimerView::handleToolTip(S32 x, S32 y, MASK mask)
 		// tooltips for timer legend
 		if (x < mBarRect.mLeft) 
 		{
-			LLTrace::TimeBlock* idp = getLegendID(y);
+			TimeBlock* idp = getLegendID(y);
 			if (idp)
 			{
 				LLToolTipMgr::instance().show(get_tooltip(*idp, -1,  *mRecording));
@@ -374,7 +376,7 @@ BOOL LLFastTimerView::handleToolTip(S32 x, S32 y, MASK mask)
 
 BOOL LLFastTimerView::handleScrollWheel(S32 x, S32 y, S32 clicks)
 {
-	LLTrace::PeriodicRecording& frame_recording = *mRecording;
+	PeriodicRecording& frame_recording = *mRecording;
 
 	mPauseHistory = true;
 	mScrollIndex = llclamp(	mScrollIndex + clicks,
@@ -383,15 +385,15 @@ BOOL LLFastTimerView::handleScrollWheel(S32 x, S32 y, S32 clicks)
 	return TRUE;
 }
 
-static LLTrace::TimeBlock FTM_RENDER_TIMER("Timers", true);
+static TimeBlock FTM_RENDER_TIMER("Timers", true);
 
-static std::map<LLTrace::TimeBlock*, LLColor4> sTimerColors;
+static std::map<TimeBlock*, LLColor4> sTimerColors;
 
 void LLFastTimerView::draw()
 {
 	LLFastTimer t(FTM_RENDER_TIMER);
 
-	LLTrace::PeriodicRecording& frame_recording = *mRecording;
+	PeriodicRecording& frame_recording = *mRecording;
 
 	std::string tdesc;
 
@@ -457,7 +459,7 @@ void LLFastTimerView::draw()
 			it != timer_tree_iterator_t();
 			++it)
 		{
-			LLTrace::TimeBlock* idp = (*it);
+			TimeBlock* idp = (*it);
 
 			const F32 HUE_INCREMENT = 0.23f;
 			hue = fmodf(hue + HUE_INCREMENT, 1.f);
@@ -480,12 +482,12 @@ void LLFastTimerView::draw()
 		LLLocalClipRect clip(LLRect(margin, y, LEGEND_WIDTH, margin));
 		S32 cur_line = 0;
 		ft_display_idx.clear();
-		std::map<LLTrace::TimeBlock*, S32> display_line;
+		std::map<TimeBlock*, S32> display_line;
 		for (timer_tree_iterator_t it = begin_timer_tree(getFrameTimer());
 			it != timer_tree_iterator_t();
 			++it)
 		{
-			LLTrace::TimeBlock* idp = (*it);
+			TimeBlock* idp = (*it);
 			display_line[idp] = cur_line;
 			ft_display_idx.push_back(idp);
 			cur_line++;
@@ -541,7 +543,7 @@ void LLFastTimerView::draw()
 
 			x += dx;
 			BOOL is_child_of_hover_item = (idp == mHoverID);
-			LLTrace::TimeBlock* next_parent = idp->getParent();
+			TimeBlock* next_parent = idp->getParent();
 			while(!is_child_of_hover_item && next_parent)
 			{
 				is_child_of_hover_item = (mHoverID == next_parent);
@@ -694,14 +696,14 @@ void LLFastTimerView::draw()
 			std::vector<S32> deltax;
 			xpos.push_back(x_start);
 			
-			LLTrace::TimeBlock* prev_id = NULL;
+			TimeBlock* prev_id = NULL;
 
 			S32 i = 0;
 			for(timer_tree_iterator_t it = begin_timer_tree(getFrameTimer());
 				it != end_timer_tree();
 				++it, ++i)
 			{
-				LLTrace::TimeBlock* idp = (*it);
+				TimeBlock* idp = (*it);
 				F32 frac = tidx == -1
 					? (frame_recording.getPeriodMean(*idp) / total_time) 
 					: (frame_recording.getPrevRecordingPeriod(tidx).getSum(*idp).value() / total_time.value());
@@ -724,11 +726,11 @@ void LLFastTimerView::draw()
 					sublevel_dx[level] = dx;
 					sublevel_right[level] = sublevel_left[level] + sublevel_dx[level];
 				}
-				else if (prev_id && prev_id->getDepth() < get_depth(idp))
+				else if (prev_id && get_depth(prev_id) < get_depth(idp))
 				{
 					U64 sublevelticks = 0;
 
-					for (LLTrace::TimeBlock::child_const_iter it = prev_id->beginChildren();
+					for (TimeBlock::child_const_iter it = prev_id->beginChildren();
 						it != prev_id->endChildren();
 						++it)
 					{
@@ -770,7 +772,7 @@ void LLFastTimerView::draw()
 					S32 scale_offset = 0;
 
 					BOOL is_child_of_hover_item = (idp == mHoverID);
-					LLTrace::TimeBlock* next_parent = idp->getParent();
+					TimeBlock* next_parent = idp->getParent();
 					while(!is_child_of_hover_item && next_parent)
 					{
 						is_child_of_hover_item = (mHoverID == next_parent);
@@ -866,7 +868,7 @@ void LLFastTimerView::draw()
 			it != end_timer_tree();
 			++it)
 		{
-			LLTrace::TimeBlock* idp = (*it);
+			TimeBlock* idp = (*it);
 				
 			//fatten highlighted timer
 			if (mHoverID == idp)
@@ -968,7 +970,7 @@ void LLFastTimerView::draw()
 			it != end_timer_tree();
 			++it)
 		{
-			LLTrace::TimeBlock* idp = (*it);
+			TimeBlock* idp = (*it);
 
 			if (!first)
 			{
@@ -990,7 +992,7 @@ void LLFastTimerView::draw()
 			it != end_timer_tree();
 			++it)
 		{
-			LLTrace::TimeBlock* idp = (*it);
+			TimeBlock* idp = (*it);
 
 			if (!first)
 			{
@@ -1527,13 +1529,13 @@ void LLFastTimerView::outputAllMetrics()
 //static
 void LLFastTimerView::doAnalysis(std::string baseline, std::string target, std::string output)
 {
-	if(LLTrace::TimeBlock::sLog)
+	if(TimeBlock::sLog)
 	{
 		doAnalysisDefault(baseline, target, output) ;
 		return ;
 	}
 
-	if(LLTrace::TimeBlock::sMetricLog)
+	if(TimeBlock::sMetricLog)
 	{
 		LLMetricPerformanceTesterBasic::doAnalysisMetrics(baseline, target, output) ;
 		return ;
@@ -1544,7 +1546,7 @@ void	LLFastTimerView::onClickCloseBtn()
 	setVisible(false);
 }
 
-LLTrace::TimeBlock& LLFastTimerView::getFrameTimer()
+TimeBlock& LLFastTimerView::getFrameTimer()
 {
 	return FTM_FRAME;
 }