SH-3931 WIP Interesting: Add graphs to visualize scene load metrics

fixed copy construction behavior of Recordings to not zero out data
split measurement into event and sample, with sample representing
a continuous function

8 files changed, 615 insertions, 162 deletions

diff --git a/indra/llcommon/llthread.cpp b/indra/llcommon/llthread.cpp
index 6374b5398b..118568d5ef 100644
--- a/indra/llcommon/llthread.cpp
+++ b/indra/llcommon/llthread.cpp
@@ -93,7 +93,7 @@ void *APR_THREAD_FUNC LLThread::staticRun(apr_thread_t *apr_threadp, void *datap
 {
 	LLThread *threadp = (LLThread *)datap;
 
-	LLTrace::ThreadRecorder* thread_recorder = new LLTrace::SlaveThreadRecorder();
+	LLTrace::ThreadRecorder* thread_recorder = new LLTrace::SlaveThreadRecorder(LLTrace::getUIThreadRecorder());
 
 #if !LL_DARWIN
 	sThreadID = threadp->mID;
diff --git a/indra/llcommon/llthreadlocalstorage.h b/indra/llcommon/llthreadlocalstorage.h
index 4873b2740d..471784749b 100644
--- a/indra/llcommon/llthreadlocalstorage.h
+++ b/indra/llcommon/llthreadlocalstorage.h
@@ -131,6 +131,10 @@ public:
 		if (!sInitialized) return false;
 		return get() == other;
 	}
+
+	bool isNull() const { return !sInitialized || get() == NULL; }
+
+	bool notNull() const { return sInitialized && get() != NULL; }
 };
 
 template<typename DERIVED_TYPE>
diff --git a/indra/llcommon/lltrace.cpp b/indra/llcommon/lltrace.cpp
index 463048008f..c831a1548d 100644
--- a/indra/llcommon/lltrace.cpp
+++ b/indra/llcommon/lltrace.cpp
@@ -35,13 +35,13 @@ static S32 sInitializationCount = 0;
 namespace LLTrace
 {
 
-static MasterThreadRecorder* gMasterThreadRecorder = NULL;
+static MasterThreadRecorder* gUIThreadRecorder = NULL;
 
 void init()
 {
 	if (sInitializationCount++ == 0)
 	{
-		gMasterThreadRecorder = new MasterThreadRecorder();
+		gUIThreadRecorder = new MasterThreadRecorder();
 	}
 }
 
@@ -54,15 +54,15 @@ void cleanup()
 {
 	if (--sInitializationCount == 0)
 	{
-		delete gMasterThreadRecorder;
-		gMasterThreadRecorder = NULL;
+		delete gUIThreadRecorder;
+		gUIThreadRecorder = NULL;
 	}
 }
 
-MasterThreadRecorder& getMasterThreadRecorder()
+MasterThreadRecorder& getUIThreadRecorder()
 {
-	llassert(gMasterThreadRecorder != NULL);
-	return *gMasterThreadRecorder;
+	llassert(gUIThreadRecorder != NULL);
+	return *gUIThreadRecorder;
 }
 
 LLThreadLocalPointer<ThreadRecorder>& get_thread_recorder_ptr()
diff --git a/indra/llcommon/lltrace.h b/indra/llcommon/lltrace.h
index d1edaf969b..f94576de45 100644
--- a/indra/llcommon/lltrace.h
+++ b/indra/llcommon/lltrace.h
@@ -73,7 +73,7 @@ bool isInitialized();
 const LLThreadLocalPointer<class ThreadRecorder>& get_thread_recorder();
 void set_thread_recorder(class ThreadRecorder*);
 
-class MasterThreadRecorder& getMasterThreadRecorder();
+class MasterThreadRecorder& getUIThreadRecorder();
 
 // one per thread per type
 template<typename ACCUMULATOR>
@@ -148,6 +148,15 @@ public:
 		}
 	}
 
+	void flush()
+	{
+		llassert(mStorageSize >= sNextStorageSlot);
+		for (size_t i = 0; i < sNextStorageSlot; i++)
+		{
+			mStorage[i].flush();
+		}
+	}
+
 	void makePrimary()
 	{
 		LLThreadLocalSingletonPointer<ACCUMULATOR>::setInstance(mStorage);
@@ -260,14 +269,14 @@ protected:
 };
 
 template<typename T>
-class MeasurementAccumulator
+class EventAccumulator
 {
 public:
 	typedef T value_t;
 	typedef F64 mean_t;
-	typedef MeasurementAccumulator<T> self_t;
+	typedef EventAccumulator<T> self_t;
 
-	MeasurementAccumulator()
+	EventAccumulator()
 	:	mSum(0),
 		mMin((std::numeric_limits<T>::max)()),
 		mMax((std::numeric_limits<T>::min)()),
@@ -277,7 +286,7 @@ public:
 		mLastValue(0)
 	{}
 
-	void sample(T value)
+	void record(T value)
 	{
 		mNumSamples++;
 		mSum += value;
@@ -301,17 +310,10 @@ public:
 		if (other.mNumSamples)
 		{
 			mSum += other.mSum;
-			if (other.mMin < mMin)
-			{
-				mMin = other.mMin;
-			}
-			if (other.mMax > mMax)
-			{
-				mMax = other.mMax;
-			}
-			F64 weight = (F64)mNumSamples / (F64)(mNumSamples + other.mNumSamples);
-			mNumSamples += other.mNumSamples;
-			mMean = mMean * weight + other.mMean * (1.f - weight);
+
+			// NOTE: both conditions will hold first time through
+			if (other.mMin < mMin) { mMin = other.mMin; }
+			if (other.mMax > mMax) { mMax = other.mMax; }
 
 			// combine variance (and hence standard deviation) of 2 different sized sample groups using
 			// the following formula: http://www.mrc-bsu.cam.ac.uk/cochrane/handbook/chapter_7/7_7_3_8_combining_groups.htm
@@ -333,12 +335,16 @@ public:
 			else
 			{
 				mVarianceSum = (F64)mNumSamples
-					* ((((n_1 - 1.f) * v_1)
-					+ ((n_2 - 1.f) * v_2)
-					+ (((n_1 * n_2) / (n_1 + n_2))
-					* ((m_1 * m_1) + (m_2 * m_2) - (2.f * m_1 * m_2))))
-					/ (n_1 + n_2 - 1.f));
+								* ((((n_1 - 1.f) * v_1)
+									+ ((n_2 - 1.f) * v_2)
+									+ (((n_1 * n_2) / (n_1 + n_2))
+										* ((m_1 * m_1) + (m_2 * m_2) - (2.f * m_1 * m_2))))
+									/ (n_1 + n_2 - 1.f));
 			}
+
+			F64 weight = (F64)mNumSamples / (F64)(mNumSamples + other.mNumSamples);
+			mNumSamples += other.mNumSamples;
+			mMean = mMean * weight + other.mMean * (1.f - weight);
 			mLastValue = other.mLastValue;
 		}
 	}
@@ -347,13 +353,15 @@ public:
 	{
 		mNumSamples = 0;
 		mSum = 0;
-		mMin = 0;
-		mMax = 0;
+		mMin = std::numeric_limits<T>::max();
+		mMax = std::numeric_limits<T>::min();
 		mMean = 0;
 		mVarianceSum = 0;
 		mLastValue = other ? other->mLastValue : 0;
 	}
 
+	void flush() {}
+
 	T	getSum() const { return (T)mSum; }
 	T	getMin() const { return (T)mMin; }
 	T	getMax() const { return (T)mMax; }
@@ -376,6 +384,150 @@ private:
 
 
 template<typename T>
+class SampleAccumulator
+{
+public:
+	typedef T value_t;
+	typedef F64 mean_t;
+	typedef SampleAccumulator<T> self_t;
+
+	SampleAccumulator()
+	:	mSum(0),
+		mMin((std::numeric_limits<T>::max)()),
+		mMax((std::numeric_limits<T>::min)()),
+		mMean(0),
+		mVarianceSum(0),
+		mLastSampleTimeStamp(LLTimer::getTotalSeconds()),
+		mTotalSamplingTime(0),
+		mNumSamples(0),
+		mLastValue(0),
+		mHasValue(false)
+	{}
+
+	void sample(T value)
+	{
+		LLUnitImplicit<LLUnits::Seconds, F64> time_stamp = LLTimer::getTotalSeconds();
+		LLUnitImplicit<LLUnits::Seconds, F64> delta_time = time_stamp - mLastSampleTimeStamp;
+		mLastSampleTimeStamp = time_stamp;
+
+		if (mHasValue)
+		{
+			mTotalSamplingTime += delta_time;
+			mSum += (F64)mLastValue * delta_time;
+
+			// NOTE: both conditions will hold first time through
+			if (value < mMin) { mMin = value; }
+			if (value > mMax) { mMax = value; }
+
+			F64 old_mean = mMean;
+			mMean += (delta_time / mTotalSamplingTime) * ((F64)mLastValue - old_mean);
+			mVarianceSum += delta_time * ((F64)mLastValue - old_mean) * ((F64)mLastValue - mMean);
+		}
+
+		mLastValue = value;
+		mNumSamples++;
+		mHasValue = true;
+	}
+
+	void addSamples(const self_t& other)
+	{
+		if (other.mTotalSamplingTime)
+		{
+			mSum += other.mSum;
+
+			// NOTE: both conditions will hold first time through
+			if (other.mMin < mMin) { mMin = other.mMin; }
+			if (other.mMax > mMax) { mMax = other.mMax; }
+
+			// combine variance (and hence standard deviation) of 2 different sized sample groups using
+			// the following formula: http://www.mrc-bsu.cam.ac.uk/cochrane/handbook/chapter_7/7_7_3_8_combining_groups.htm
+			F64 n_1 = mTotalSamplingTime,
+				n_2 = other.mTotalSamplingTime;
+			F64 m_1 = mMean,
+				m_2 = other.mMean;
+			F64 v_1 = mVarianceSum / mTotalSamplingTime,
+				v_2 = other.mVarianceSum / other.mTotalSamplingTime;
+			if (n_1 == 0)
+			{
+				mVarianceSum = other.mVarianceSum;
+			}
+			else if (n_2 == 0)
+			{
+				// variance is unchanged
+				// mVarianceSum = mVarianceSum;
+			}
+			else
+			{
+				mVarianceSum =	mTotalSamplingTime
+								* ((((n_1 - 1.f) * v_1)
+									+ ((n_2 - 1.f) * v_2)
+									+ (((n_1 * n_2) / (n_1 + n_2))
+										* ((m_1 * m_1) + (m_2 * m_2) - (2.f * m_1 * m_2))))
+									/ (n_1 + n_2 - 1.f));
+			}
+
+			llassert(other.mTotalSamplingTime > 0);
+			F64 weight = mTotalSamplingTime / (mTotalSamplingTime + other.mTotalSamplingTime);
+			mNumSamples += other.mNumSamples;
+			mTotalSamplingTime += other.mTotalSamplingTime;
+			mMean = (mMean * weight) + (other.mMean * (1.0 - weight));
+			mLastValue = other.mLastValue;
+			mLastSampleTimeStamp = other.mLastSampleTimeStamp;
+			mHasValue |= other.mHasValue;
+		}
+	}
+
+	void reset(const self_t* other)
+	{
+		mNumSamples = 0;
+		mSum = 0;
+		mMin = std::numeric_limits<T>::max();
+		mMax = std::numeric_limits<T>::min();
+		mMean = other ? other->mLastValue : 0;
+		mVarianceSum = 0;
+		mLastSampleTimeStamp = LLTimer::getTotalSeconds();
+		mTotalSamplingTime = 0;
+		mLastValue = other ? other->mLastValue : 0;
+		mHasValue = other ? other->mHasValue : false;
+	}
+
+	void flush()
+	{
+		LLUnitImplicit<LLUnits::Seconds, F64> time_stamp = LLTimer::getTotalSeconds();
+		LLUnitImplicit<LLUnits::Seconds, F64> delta_time = time_stamp - mLastSampleTimeStamp;
+
+		mSum += (F64)mLastValue * delta_time;
+
+		mTotalSamplingTime += delta_time;
+		mLastSampleTimeStamp = time_stamp;
+	}
+
+	T	getSum() const { return (T)mSum; }
+	T	getMin() const { return (T)mMin; }
+	T	getMax() const { return (T)mMax; }
+	T	getLastValue() const { return (T)mLastValue; }
+	F64	getMean() const { return mMean; }
+	F64 getStandardDeviation() const { return sqrtf(mVarianceSum / mTotalSamplingTime); }
+	U32 getSampleCount() const { return mNumSamples; }
+
+private:
+	T	mSum,
+		mMin,
+		mMax,
+		mLastValue;
+
+	bool mHasValue;
+
+	F64	mMean,
+		mVarianceSum;
+
+	LLUnitImplicit<LLUnits::Seconds, F64>	mLastSampleTimeStamp,
+											mTotalSamplingTime;
+
+	U32	mNumSamples;
+};
+
+template<typename T>
 class CountAccumulator
 {
 public:
@@ -406,6 +558,8 @@ public:
 		mSum = 0;
 	}
 
+	void flush() {}
+
 	T	getSum() const { return (T)mSum; }
 
 	U32 getSampleCount() const { return mNumSamples; }
@@ -439,6 +593,7 @@ public:
 	TimeBlockAccumulator();
 	void addSamples(const self_t& other);
 	void reset(const self_t* other);
+	void flush() {}
 
 	//
 	// members
@@ -493,25 +648,44 @@ public:
 
 
 template <typename T = F64>
-class MeasurementStatHandle
-:	public TraceType<MeasurementAccumulator<typename LLUnits::HighestPrecisionType<T>::type_t> >
+class EventStatHandle
+:	public TraceType<EventAccumulator<typename LLUnits::HighestPrecisionType<T>::type_t> >
 {
 public:
 	typedef typename LLUnits::HighestPrecisionType<T>::type_t storage_t;
-	typedef TraceType<MeasurementAccumulator<typename LLUnits::HighestPrecisionType<T>::type_t> > trace_t;
+	typedef TraceType<EventAccumulator<typename LLUnits::HighestPrecisionType<T>::type_t> > trace_t;
 
-	MeasurementStatHandle(const char* name, const char* description = NULL) 
+	EventStatHandle(const char* name, const char* description = NULL)
 	:	trace_t(name, description)
 	{}
 };
 
 template<typename T, typename VALUE_T>
-void sample(MeasurementStatHandle<T>& measurement, VALUE_T value)
+void record(EventStatHandle<T>& measurement, VALUE_T value)
 {
 	T converted_value(value);
-	measurement.getPrimaryAccumulator()->sample(LLUnits::rawValue(converted_value));
+	measurement.getPrimaryAccumulator()->record(LLUnits::rawValue(converted_value));
 }
 
+template <typename T = F64>
+class SampleStatHandle
+:	public TraceType<SampleAccumulator<typename LLUnits::HighestPrecisionType<T>::type_t> >
+{
+public:
+	typedef typename LLUnits::HighestPrecisionType<T>::type_t storage_t;
+	typedef TraceType<SampleAccumulator<typename LLUnits::HighestPrecisionType<T>::type_t> > trace_t;
+
+	SampleStatHandle(const char* name, const char* description = NULL)
+	:	trace_t(name, description)
+	{}
+};
+
+template<typename T, typename VALUE_T>
+void sample(SampleStatHandle<T>& measurement, VALUE_T value)
+{
+	T converted_value(value);
+	measurement.getPrimaryAccumulator()->sample(LLUnits::rawValue(converted_value));
+}
 
 template <typename T = F64>
 class CountStatHandle
@@ -560,6 +734,8 @@ struct MemStatAccumulator
 		mDeallocatedCount = 0;
 	}
 
+	void flush() {}
+
 	size_t		mSize,
 				mChildSize;
 	int			mAllocatedCount,
diff --git a/indra/llcommon/lltracerecording.cpp b/indra/llcommon/lltracerecording.cpp
index cced6546ba..5b0b74524f 100644
--- a/indra/llcommon/lltracerecording.cpp
+++ b/indra/llcommon/lltracerecording.cpp
@@ -45,9 +45,11 @@ RecordingBuffers::RecordingBuffers()
 void RecordingBuffers::handOffTo(RecordingBuffers& other)
 {
 	other.mCountsFloat.reset(&mCountsFloat);
-	other.mMeasurementsFloat.reset(&mMeasurementsFloat);
 	other.mCounts.reset(&mCounts);
-	other.mMeasurements.reset(&mMeasurements);
+	other.mSamplesFloat.reset(&mSamplesFloat);
+	other.mSamples.reset(&mSamples);
+	other.mEventsFloat.reset(&mEventsFloat);
+	other.mEvents.reset(&mEvents);
 	other.mStackTimers.reset(&mStackTimers);
 	other.mMemStats.reset(&mMemStats);
 }
@@ -55,9 +57,11 @@ void RecordingBuffers::handOffTo(RecordingBuffers& other)
 void RecordingBuffers::makePrimary()
 {
 	mCountsFloat.makePrimary();
-	mMeasurementsFloat.makePrimary();
 	mCounts.makePrimary();
-	mMeasurements.makePrimary();
+	mSamplesFloat.makePrimary();
+	mSamples.makePrimary();
+	mEventsFloat.makePrimary();
+	mEvents.makePrimary();
 	mStackTimers.makePrimary();
 	mMemStats.makePrimary();
 
@@ -82,9 +86,11 @@ bool RecordingBuffers::isPrimary() const
 void RecordingBuffers::append( const RecordingBuffers& other )
 {
 	mCountsFloat.addSamples(other.mCountsFloat);
-	mMeasurementsFloat.addSamples(other.mMeasurementsFloat);
 	mCounts.addSamples(other.mCounts);
-	mMeasurements.addSamples(other.mMeasurements);
+	mSamplesFloat.addSamples(other.mSamplesFloat);
+	mSamples.addSamples(other.mSamples);
+	mEventsFloat.addSamples(other.mEventsFloat);
+	mEvents.addSamples(other.mEvents);
 	mMemStats.addSamples(other.mMemStats);
 	mStackTimers.addSamples(other.mStackTimers);
 }
@@ -92,22 +98,32 @@ void RecordingBuffers::append( const RecordingBuffers& other )
 void RecordingBuffers::merge( const RecordingBuffers& other)
 {
 	mCountsFloat.addSamples(other.mCountsFloat);
-	mMeasurementsFloat.addSamples(other.mMeasurementsFloat);
 	mCounts.addSamples(other.mCounts);
-	mMeasurements.addSamples(other.mMeasurements);
+	mSamplesFloat.addSamples(other.mSamplesFloat);
+	mSamples.addSamples(other.mSamples);
+	mEventsFloat.addSamples(other.mEventsFloat);
+	mEvents.addSamples(other.mEvents);
 	mMemStats.addSamples(other.mMemStats);
 }
 
 void RecordingBuffers::reset(RecordingBuffers* other)
 {
 	mCountsFloat.reset(other ? &other->mCountsFloat : NULL);
-	mMeasurementsFloat.reset(other ? &other->mMeasurementsFloat : NULL);
 	mCounts.reset(other ? &other->mCounts : NULL);
-	mMeasurements.reset(other ? &other->mMeasurements : NULL);
+	mSamplesFloat.reset(other ? &other->mSamplesFloat : NULL);
+	mSamples.reset(other ? &other->mSamples : NULL);
+	mEventsFloat.reset(other ? &other->mEventsFloat : NULL);
+	mEvents.reset(other ? &other->mEvents : NULL);
 	mStackTimers.reset(other ? &other->mStackTimers : NULL);
 	mMemStats.reset(other ? &other->mMemStats : NULL);
 }
 
+void RecordingBuffers::flush()
+{
+	mSamplesFloat.flush();
+	mSamples.flush();
+}
+
 ///////////////////////////////////////////////////////////////////////
 // Recording
 ///////////////////////////////////////////////////////////////////////
@@ -120,6 +136,9 @@ Recording::Recording()
 
 Recording::Recording( const Recording& other )
 {
+	// this will allow us to seamlessly start without affecting any data we've acquired from other
+	setPlayState(PAUSED);
+
 	Recording& mutable_other = const_cast<Recording&>(other);
 	EPlayState other_play_state = other.getPlayState();
 	mutable_other.pause();
@@ -137,15 +156,18 @@ Recording::Recording( const Recording& other )
 
 Recording::~Recording()
 {
-	stop();
-	llassert(isStopped());
+	if (isStarted() && LLTrace::get_thread_recorder().notNull())
+	{
+		LLTrace::get_thread_recorder()->deactivate(this);
+	}
 }
 
 void Recording::update()
 {
 	if (isStarted())
 	{
-		LLTrace::get_thread_recorder()->update(this);
+		mBuffers.write()->flush();
+		LLTrace::get_thread_recorder()->bringUpToDate(this);
 		mSamplingTimer.reset();
 	}
 }
@@ -167,6 +189,7 @@ void Recording::handleStart()
 void Recording::handleStop()
 {
 	mElapsedSeconds += mSamplingTimer.getElapsedTimeF64();
+	mBuffers.write()->flush();
 	LLTrace::TimeBlock::processTimes();
 	LLTrace::get_thread_recorder()->deactivate(this);
 }
@@ -178,13 +201,23 @@ void Recording::handleSplitTo(Recording& other)
 
 void Recording::appendRecording( const Recording& other )
 {
-	mBuffers.write()->append(*other.mBuffers);
-	mElapsedSeconds += other.mElapsedSeconds;
+	EPlayState play_state = getPlayState();
+	{
+		pause();
+		mBuffers.write()->append(*other.mBuffers);
+		mElapsedSeconds += other.mElapsedSeconds;
+	}
+	setPlayState(play_state);
 }
 
 void Recording::mergeRecording( const Recording& other)
 {
-	mBuffers.write()->merge(*other.mBuffers);
+	EPlayState play_state = getPlayState();
+	{
+		pause();
+		mBuffers.write()->merge(*other.mBuffers);
+	}
+	setPlayState(play_state);
 }
 
 LLUnit<LLUnits::Seconds, F64> Recording::getSum(const TraceType<TimeBlockAccumulator>& stat) const
@@ -248,14 +281,14 @@ S64 Recording::getSum( const TraceType<CountAccumulator<S64> >& stat ) const
 	return mBuffers->mCounts[stat.getIndex()].getSum();
 }
 
-F64 Recording::getSum( const TraceType<MeasurementAccumulator<F64> >& stat ) const
+F64 Recording::getSum( const TraceType<EventAccumulator<F64> >& stat ) const
 {
-	return (F64)mBuffers->mMeasurementsFloat[stat.getIndex()].getSum();
+	return (F64)mBuffers->mEventsFloat[stat.getIndex()].getSum();
 }
 
-S64 Recording::getSum( const TraceType<MeasurementAccumulator<S64> >& stat ) const
+S64 Recording::getSum( const TraceType<EventAccumulator<S64> >& stat ) const
 {
-	return (S64)mBuffers->mMeasurements[stat.getIndex()].getSum();
+	return (S64)mBuffers->mEvents[stat.getIndex()].getSum();
 }
 
 
@@ -283,67 +316,127 @@ U32 Recording::getSampleCount( const TraceType<CountAccumulator<F64> >& stat ) c
 
 U32 Recording::getSampleCount( const TraceType<CountAccumulator<S64> >& stat ) const
 {
-	return mBuffers->mMeasurementsFloat[stat.getIndex()].getSampleCount();
+	return mBuffers->mCounts[stat.getIndex()].getSampleCount();
+}
+
+F64 Recording::getMin( const TraceType<SampleAccumulator<F64> >& stat ) const
+{
+	return mBuffers->mSamplesFloat[stat.getIndex()].getMin();
+}
+
+S64 Recording::getMin( const TraceType<SampleAccumulator<S64> >& stat ) const
+{
+	return mBuffers->mSamples[stat.getIndex()].getMin();
+}
+
+F64 Recording::getMax( const TraceType<SampleAccumulator<F64> >& stat ) const
+{
+	return mBuffers->mSamplesFloat[stat.getIndex()].getMax();
+}
+
+S64 Recording::getMax( const TraceType<SampleAccumulator<S64> >& stat ) const
+{
+	return mBuffers->mSamples[stat.getIndex()].getMax();
+}
+
+F64 Recording::getMean( const TraceType<SampleAccumulator<F64> >& stat ) const
+{
+	return mBuffers->mSamplesFloat[stat.getIndex()].getMean();
+}
+
+F64 Recording::getMean( const TraceType<SampleAccumulator<S64> >& stat ) const
+{
+	return mBuffers->mSamples[stat.getIndex()].getMean();
 }
 
-F64 Recording::getMin( const TraceType<MeasurementAccumulator<F64> >& stat ) const
+F64 Recording::getStandardDeviation( const TraceType<SampleAccumulator<F64> >& stat ) const
 {
-	return mBuffers->mMeasurementsFloat[stat.getIndex()].getMin();
+	return mBuffers->mSamplesFloat[stat.getIndex()].getStandardDeviation();
 }
 
-S64 Recording::getMin( const TraceType<MeasurementAccumulator<S64> >& stat ) const
+F64 Recording::getStandardDeviation( const TraceType<SampleAccumulator<S64> >& stat ) const
 {
-	return mBuffers->mMeasurements[stat.getIndex()].getMin();
+	return mBuffers->mSamples[stat.getIndex()].getStandardDeviation();
 }
 
-F64 Recording::getMax( const TraceType<MeasurementAccumulator<F64> >& stat ) const
+F64 Recording::getLastValue( const TraceType<SampleAccumulator<F64> >& stat ) const
 {
-	return mBuffers->mMeasurementsFloat[stat.getIndex()].getMax();
+	return mBuffers->mSamplesFloat[stat.getIndex()].getLastValue();
 }
 
-S64 Recording::getMax( const TraceType<MeasurementAccumulator<S64> >& stat ) const
+S64 Recording::getLastValue( const TraceType<SampleAccumulator<S64> >& stat ) const
 {
-	return mBuffers->mMeasurements[stat.getIndex()].getMax();
+	return mBuffers->mSamples[stat.getIndex()].getLastValue();
 }
 
-F64 Recording::getMean( const TraceType<MeasurementAccumulator<F64> >& stat ) const
+U32 Recording::getSampleCount( const TraceType<SampleAccumulator<F64> >& stat ) const
 {
-	return mBuffers->mMeasurementsFloat[stat.getIndex()].getMean();
+	return mBuffers->mSamplesFloat[stat.getIndex()].getSampleCount();
 }
 
-F64 Recording::getMean( const TraceType<MeasurementAccumulator<S64> >& stat ) const
+U32 Recording::getSampleCount( const TraceType<SampleAccumulator<S64> >& stat ) const
 {
-	return mBuffers->mMeasurements[stat.getIndex()].getMean();
+	return mBuffers->mSamples[stat.getIndex()].getSampleCount();
 }
 
-F64 Recording::getStandardDeviation( const TraceType<MeasurementAccumulator<F64> >& stat ) const
+F64 Recording::getMin( const TraceType<EventAccumulator<F64> >& stat ) const
 {
-	return mBuffers->mMeasurementsFloat[stat.getIndex()].getStandardDeviation();
+	return mBuffers->mEventsFloat[stat.getIndex()].getMin();
 }
 
-F64 Recording::getStandardDeviation( const TraceType<MeasurementAccumulator<S64> >& stat ) const
+S64 Recording::getMin( const TraceType<EventAccumulator<S64> >& stat ) const
 {
-	return mBuffers->mMeasurements[stat.getIndex()].getStandardDeviation();
+	return mBuffers->mEvents[stat.getIndex()].getMin();
 }
 
-F64 Recording::getLastValue( const TraceType<MeasurementAccumulator<F64> >& stat ) const
+F64 Recording::getMax( const TraceType<EventAccumulator<F64> >& stat ) const
 {
-	return mBuffers->mMeasurementsFloat[stat.getIndex()].getLastValue();
+	return mBuffers->mEventsFloat[stat.getIndex()].getMax();
 }
 
-S64 Recording::getLastValue( const TraceType<MeasurementAccumulator<S64> >& stat ) const
+S64 Recording::getMax( const TraceType<EventAccumulator<S64> >& stat ) const
 {
-	return mBuffers->mMeasurements[stat.getIndex()].getLastValue();
+	return mBuffers->mEvents[stat.getIndex()].getMax();
 }
 
-U32 Recording::getSampleCount( const TraceType<MeasurementAccumulator<F64> >& stat ) const
+F64 Recording::getMean( const TraceType<EventAccumulator<F64> >& stat ) const
 {
-	return mBuffers->mMeasurementsFloat[stat.getIndex()].getSampleCount();
+	return mBuffers->mEventsFloat[stat.getIndex()].getMean();
 }
 
-U32 Recording::getSampleCount( const TraceType<MeasurementAccumulator<S64> >& stat ) const
+F64 Recording::getMean( const TraceType<EventAccumulator<S64> >& stat ) const
 {
-	return mBuffers->mMeasurements[stat.getIndex()].getSampleCount();
+	return mBuffers->mEvents[stat.getIndex()].getMean();
+}
+
+F64 Recording::getStandardDeviation( const TraceType<EventAccumulator<F64> >& stat ) const
+{
+	return mBuffers->mEventsFloat[stat.getIndex()].getStandardDeviation();
+}
+
+F64 Recording::getStandardDeviation( const TraceType<EventAccumulator<S64> >& stat ) const
+{
+	return mBuffers->mEvents[stat.getIndex()].getStandardDeviation();
+}
+
+F64 Recording::getLastValue( const TraceType<EventAccumulator<F64> >& stat ) const
+{
+	return mBuffers->mEventsFloat[stat.getIndex()].getLastValue();
+}
+
+S64 Recording::getLastValue( const TraceType<EventAccumulator<S64> >& stat ) const
+{
+	return mBuffers->mEvents[stat.getIndex()].getLastValue();
+}
+
+U32 Recording::getSampleCount( const TraceType<EventAccumulator<F64> >& stat ) const
+{
+	return mBuffers->mEventsFloat[stat.getIndex()].getSampleCount();
+}
+
+U32 Recording::getSampleCount( const TraceType<EventAccumulator<S64> >& stat ) const
+{
+	return mBuffers->mEvents[stat.getIndex()].getSampleCount();
 }
 
 ///////////////////////////////////////////////////////////////////////
@@ -377,7 +470,7 @@ void PeriodicRecording::appendPeriodicRecording( PeriodicRecording& other )
 	if (other.mRecordingPeriods.empty()) return;
 
 	EPlayState play_state = getPlayState();
-	stop();
+	pause();
 
 	EPlayState other_play_state = other.getPlayState();
 	other.pause();
@@ -466,8 +559,7 @@ LLTrace::Recording PeriodicRecording::snapshotCurRecording() const
 
 Recording& PeriodicRecording::getLastRecording()
 {
-	U32 num_periods = mRecordingPeriods.size();
-	return mRecordingPeriods[(mCurPeriod + num_periods - 1) % num_periods];
+	return getPrevRecording(1);
 }
 
 const Recording& PeriodicRecording::getLastRecording() const
diff --git a/indra/llcommon/lltracerecording.h b/indra/llcommon/lltracerecording.h
index b339e72e5c..19a4fae737 100644
--- a/indra/llcommon/lltracerecording.h
+++ b/indra/llcommon/lltracerecording.h
@@ -117,11 +117,14 @@ namespace LLTrace
 		void append(const RecordingBuffers& other);
 		void merge(const RecordingBuffers& other);
 		void reset(RecordingBuffers* other = NULL);
+		void flush();
 
 		AccumulatorBuffer<CountAccumulator<F64> > 		mCountsFloat;
-		AccumulatorBuffer<MeasurementAccumulator<F64> > mMeasurementsFloat;
 		AccumulatorBuffer<CountAccumulator<S64> > 		mCounts;
-		AccumulatorBuffer<MeasurementAccumulator<S64> > mMeasurements;
+		AccumulatorBuffer<SampleAccumulator<F64> >		mSamplesFloat;
+		AccumulatorBuffer<SampleAccumulator<S64> >		mSamples;
+		AccumulatorBuffer<EventAccumulator<F64> >		mEventsFloat;
+		AccumulatorBuffer<EventAccumulator<S64> >		mEvents;
 		AccumulatorBuffer<TimeBlockAccumulator> 		mStackTimers;
 		AccumulatorBuffer<MemStatAccumulator> 			mMemStats;
 	};
@@ -181,57 +184,101 @@ namespace LLTrace
 		U32 getSampleCount(const TraceType<CountAccumulator<S64> >& stat) const;
 
 
-		// MeasurementStatHandle accessors
-		F64 getSum(const TraceType<MeasurementAccumulator<F64> >& stat) const;
-		S64 getSum(const TraceType<MeasurementAccumulator<S64> >& stat) const;
+		// SampleStatHandle accessors
+		F64 getMin(const TraceType<SampleAccumulator<F64> >& stat) const;
+		S64 getMin(const TraceType<SampleAccumulator<S64> >& stat) const;
 		template <typename T>
-		T getSum(const MeasurementStatHandle<T>& stat) const
+		T getMin(const SampleStatHandle<T>& stat) const
 		{
-			return (T)getSum(static_cast<const TraceType<MeasurementAccumulator<typename LLUnits::HighestPrecisionType<T>::type_t> >&> (stat));
+			return (T)getMin(static_cast<const TraceType<SampleAccumulator<typename LLUnits::HighestPrecisionType<T>::type_t> >&> (stat));
 		}
 
-		F64 getMin(const TraceType<MeasurementAccumulator<F64> >& stat) const;
-		S64 getMin(const TraceType<MeasurementAccumulator<S64> >& stat) const;
+		F64 getMax(const TraceType<SampleAccumulator<F64> >& stat) const;
+		S64 getMax(const TraceType<SampleAccumulator<S64> >& stat) const;
 		template <typename T>
-		T getMin(const MeasurementStatHandle<T>& stat) const
+		T getMax(const SampleStatHandle<T>& stat) const
 		{
-			return (T)getMin(static_cast<const TraceType<MeasurementAccumulator<typename LLUnits::HighestPrecisionType<T>::type_t> >&> (stat));
+			return (T)getMax(static_cast<const TraceType<SampleAccumulator<typename LLUnits::HighestPrecisionType<T>::type_t> >&> (stat));
 		}
 
-		F64 getMax(const TraceType<MeasurementAccumulator<F64> >& stat) const;
-		S64 getMax(const TraceType<MeasurementAccumulator<S64> >& stat) const;
+		F64 getMean(const TraceType<SampleAccumulator<F64> >& stat) const;
+		F64 getMean(const TraceType<SampleAccumulator<S64> >& stat) const;
 		template <typename T>
-		T getMax(const MeasurementStatHandle<T>& stat) const
+		T getMean(SampleStatHandle<T>& stat) const
 		{
-			return (T)getMax(static_cast<const TraceType<MeasurementAccumulator<typename LLUnits::HighestPrecisionType<T>::type_t> >&> (stat));
+			return (T)getMean(static_cast<const TraceType<SampleAccumulator<typename LLUnits::HighestPrecisionType<T>::type_t> >&> (stat));
 		}
 
-		F64 getMean(const TraceType<MeasurementAccumulator<F64> >& stat) const;
-		F64 getMean(const TraceType<MeasurementAccumulator<S64> >& stat) const;
+		F64 getStandardDeviation(const TraceType<SampleAccumulator<F64> >& stat) const;
+		F64 getStandardDeviation(const TraceType<SampleAccumulator<S64> >& stat) const;
 		template <typename T>
-		T getMean(MeasurementStatHandle<T>& stat) const
+		T getStandardDeviation(const SampleStatHandle<T>& stat) const
 		{
-			return (T)getMean(static_cast<const TraceType<MeasurementAccumulator<typename LLUnits::HighestPrecisionType<T>::type_t> >&> (stat));
+			return (T)getStandardDeviation(static_cast<const TraceType<SampleAccumulator<typename LLUnits::HighestPrecisionType<T>::type_t> >&> (stat));
 		}
 
-		F64 getStandardDeviation(const TraceType<MeasurementAccumulator<F64> >& stat) const;
-		F64 getStandardDeviation(const TraceType<MeasurementAccumulator<S64> >& stat) const;
+		F64 getLastValue(const TraceType<SampleAccumulator<F64> >& stat) const;
+		S64 getLastValue(const TraceType<SampleAccumulator<S64> >& stat) const;
 		template <typename T>
-		T getStandardDeviation(const MeasurementStatHandle<T>& stat) const
+		T getLastValue(const SampleStatHandle<T>& stat) const
 		{
-			return (T)getMean(static_cast<const TraceType<MeasurementAccumulator<typename LLUnits::HighestPrecisionType<T>::type_t> >&> (stat));
+			return (T)getLastValue(static_cast<const TraceType<SampleAccumulator<typename LLUnits::HighestPrecisionType<T>::type_t> >&> (stat));
 		}
 
-		F64 getLastValue(const TraceType<MeasurementAccumulator<F64> >& stat) const;
-		S64 getLastValue(const TraceType<MeasurementAccumulator<S64> >& stat) const;
+		U32 getSampleCount(const TraceType<SampleAccumulator<F64> >& stat) const;
+		U32 getSampleCount(const TraceType<SampleAccumulator<S64> >& stat) const;
+
+		// EventStatHandle accessors
+		F64 getSum(const TraceType<EventAccumulator<F64> >& stat) const;
+		S64 getSum(const TraceType<EventAccumulator<S64> >& stat) const;
+		template <typename T>
+		T getSum(const EventStatHandle<T>& stat) const
+		{
+			return (T)getSum(static_cast<const TraceType<EventAccumulator<typename LLUnits::HighestPrecisionType<T>::type_t> >&> (stat));
+		}
+
+		F64 getMin(const TraceType<EventAccumulator<F64> >& stat) const;
+		S64 getMin(const TraceType<EventAccumulator<S64> >& stat) const;
+		template <typename T>
+		T getMin(const EventStatHandle<T>& stat) const
+		{
+			return (T)getMin(static_cast<const TraceType<EventAccumulator<typename LLUnits::HighestPrecisionType<T>::type_t> >&> (stat));
+		}
+
+		F64 getMax(const TraceType<EventAccumulator<F64> >& stat) const;
+		S64 getMax(const TraceType<EventAccumulator<S64> >& stat) const;
+		template <typename T>
+		T getMax(const EventStatHandle<T>& stat) const
+		{
+			return (T)getMax(static_cast<const TraceType<EventAccumulator<typename LLUnits::HighestPrecisionType<T>::type_t> >&> (stat));
+		}
+
+		F64 getMean(const TraceType<EventAccumulator<F64> >& stat) const;
+		F64 getMean(const TraceType<EventAccumulator<S64> >& stat) const;
+		template <typename T>
+		T getMean(EventStatHandle<T>& stat) const
+		{
+			return (T)getMean(static_cast<const TraceType<EventAccumulator<typename LLUnits::HighestPrecisionType<T>::type_t> >&> (stat));
+		}
+
+		F64 getStandardDeviation(const TraceType<EventAccumulator<F64> >& stat) const;
+		F64 getStandardDeviation(const TraceType<EventAccumulator<S64> >& stat) const;
+		template <typename T>
+		T getStandardDeviation(const EventStatHandle<T>& stat) const
+		{
+			return (T)getStandardDeviation(static_cast<const TraceType<EventAccumulator<typename LLUnits::HighestPrecisionType<T>::type_t> >&> (stat));
+		}
+
+		F64 getLastValue(const TraceType<EventAccumulator<F64> >& stat) const;
+		S64 getLastValue(const TraceType<EventAccumulator<S64> >& stat) const;
 		template <typename T>
-		T getLastValue(const MeasurementStatHandle<T>& stat) const
+		T getLastValue(const EventStatHandle<T>& stat) const
 		{
-			return (T)getLastValue(static_cast<const TraceType<MeasurementAccumulator<typename LLUnits::HighestPrecisionType<T>::type_t> >&> (stat));
+			return (T)getLastValue(static_cast<const TraceType<EventAccumulator<typename LLUnits::HighestPrecisionType<T>::type_t> >&> (stat));
 		}
 
-		U32 getSampleCount(const TraceType<MeasurementAccumulator<F64> >& stat) const;
-		U32 getSampleCount(const TraceType<MeasurementAccumulator<S64> >& stat) const;
+		U32 getSampleCount(const TraceType<EventAccumulator<F64> >& stat) const;
+		U32 getSampleCount(const TraceType<EventAccumulator<S64> >& stat) const;
 
 		LLUnit<LLUnits::Seconds, F64> getDuration() const { return LLUnit<LLUnits::Seconds, F64>(mElapsedSeconds); }
 
@@ -272,13 +319,14 @@ namespace LLTrace
 		const Recording& getPrevRecording(U32 offset) const;
 		Recording snapshotCurRecording() const;
 
+		// catch all for stats that have a defined sum
 		template <typename T>
 		typename T::value_t getPeriodMin(const TraceType<T>& stat, size_t num_periods = U32_MAX) const
 		{
 			size_t total_periods = mRecordingPeriods.size();
 			num_periods = llmin(num_periods, total_periods);
 
-			typename T::value_t min_val = (std::numeric_limits<typename T::value_t>::max)();
+			typename T::value_t min_val = std::numeric_limits<typename T::value_t>::max();
 			for (S32 i = 1; i <= num_periods; i++)
 			{
 				S32 index = (mCurPeriod + total_periods - i) % total_periods;
@@ -288,12 +336,42 @@ namespace LLTrace
 		}
 
 		template <typename T>
+		typename T getPeriodMin(const TraceType<SampleAccumulator<T> >& stat, size_t num_periods = U32_MAX) const
+		{
+			size_t total_periods = mRecordingPeriods.size();
+			num_periods = llmin(num_periods, total_periods);
+
+			typename T min_val = std::numeric_limits<T>::max();
+			for (S32 i = 1; i <= num_periods; i++)
+			{
+				S32 index = (mCurPeriod + total_periods - i) % total_periods;
+				min_val = llmin(min_val, mRecordingPeriods[index].getMin(stat));
+			}
+			return min_val;
+		}
+		
+		template <typename T>
+		typename T getPeriodMin(const TraceType<EventAccumulator<T> >& stat, size_t num_periods = U32_MAX) const
+		{
+			size_t total_periods = mRecordingPeriods.size();
+			num_periods = llmin(num_periods, total_periods);
+
+			typename T min_val = std::numeric_limits<T>::max();
+			for (S32 i = 1; i <= num_periods; i++)
+			{
+				S32 index = (mCurPeriod + total_periods - i) % total_periods;
+				min_val = llmin(min_val, mRecordingPeriods[index].getMin(stat));
+			}
+			return min_val;
+		}
+
+		template <typename T>
 		F64 getPeriodMinPerSec(const TraceType<T>& stat, size_t num_periods = U32_MAX) const
 		{
 			size_t total_periods = mRecordingPeriods.size();
 			num_periods = llmin(num_periods, total_periods);
 
-			F64 min_val = (std::numeric_limits<F64>::max)();
+			F64 min_val = std::numeric_limits<F64>::max();
 			for (S32 i = 1; i <= num_periods; i++)
 			{
 				S32 index = (mCurPeriod + total_periods - i) % total_periods;
@@ -302,13 +380,14 @@ namespace LLTrace
 			return min_val;
 		}
 
+		// catch all for stats that have a defined sum
 		template <typename T>
 		typename T::value_t getPeriodMax(const TraceType<T>& stat, size_t num_periods = U32_MAX) const
 		{
 			size_t total_periods = mRecordingPeriods.size();
 			num_periods = llmin(num_periods, total_periods);
 
-			typename T::value_t max_val = (std::numeric_limits<typename T::value_t>::min)();
+			typename T::value_t max_val = std::numeric_limits<typename T::value_t>::min();
 			for (S32 i = 1; i <= num_periods; i++)
 			{
 				S32 index = (mCurPeriod + total_periods - i) % total_periods;
@@ -318,12 +397,42 @@ namespace LLTrace
 		}
 
 		template <typename T>
+		typename T getPeriodMax(const TraceType<SampleAccumulator<T> >& stat, size_t num_periods = U32_MAX) const
+		{
+			size_t total_periods = mRecordingPeriods.size();
+			num_periods = llmin(num_periods, total_periods);
+
+			typename T max_val = std::numeric_limits<T>::min();
+			for (S32 i = 1; i <= num_periods; i++)
+			{
+				S32 index = (mCurPeriod + total_periods - i) % total_periods;
+				max_val = llmax(max_val, mRecordingPeriods[index].getMax(stat));
+			}
+			return max_val;
+		}
+
+		template <typename T>
+		typename T getPeriodMax(const TraceType<EventAccumulator<T> >& stat, size_t num_periods = U32_MAX) const
+		{
+			size_t total_periods = mRecordingPeriods.size();
+			num_periods = llmin(num_periods, total_periods);
+
+			typename T max_val = std::numeric_limits<T>::min();
+			for (S32 i = 1; i <= num_periods; i++)
+			{
+				S32 index = (mCurPeriod + total_periods - i) % total_periods;
+				max_val = llmax(max_val, mRecordingPeriods[index].getMax(stat));
+			}
+			return max_val;
+		}
+
+		template <typename T>
 		F64 getPeriodMaxPerSec(const TraceType<T>& stat, size_t num_periods = U32_MAX) const
 		{
 			size_t total_periods = mRecordingPeriods.size();
 			num_periods = llmin(num_periods, total_periods);
 
-			F64 max_val = (std::numeric_limits<F64>::min)();
+			F64 max_val = std::numeric_limits<F64>::min();
 			for (S32 i = 1; i <= num_periods; i++)
 			{
 				S32 index = (mCurPeriod + total_periods - i) % total_periods;
@@ -332,13 +441,14 @@ namespace LLTrace
 			return max_val;
 		}
 
+		// catch all for stats that have a defined sum
 		template <typename T>
-		typename T::mean_t getPeriodMean(const TraceType<T>& stat, size_t num_periods = U32_MAX) const
+		typename T::mean_t getPeriodMean(const TraceType<T >& stat, size_t num_periods = U32_MAX) const
 		{
 			size_t total_periods = mRecordingPeriods.size();
 			num_periods = llmin(num_periods, total_periods);
 
-			typename T::mean_t mean = typename T::mean_t();
+			typename T::mean_t mean = 0;
 			if (num_periods <= 0) { return mean; }
 
 			for (S32 i = 1; i <= num_periods; i++)
@@ -349,7 +459,65 @@ namespace LLTrace
 					mean += mRecordingPeriods[index].getSum(stat);
 				}
 			}
-			mean /= num_periods;
+			mean = mean / num_periods;
+			return mean;
+		}
+
+		template <typename T>
+		typename SampleAccumulator<T>::mean_t getPeriodMean(const TraceType<SampleAccumulator<T> >& stat, size_t num_periods = U32_MAX) const
+		{
+			size_t total_periods = mRecordingPeriods.size();
+			num_periods = llmin(num_periods, total_periods);
+
+			LLUnit<LLUnits::Seconds, F64> total_duration = 0.f;
+
+			typename SampleAccumulator<T>::mean_t mean = 0;
+			if (num_periods <= 0) { return mean; }
+
+			for (S32 i = 1; i <= num_periods; i++)
+			{
+				S32 index = (mCurPeriod + total_periods - i) % total_periods;
+				if (mRecordingPeriods[index].getDuration() > 0.f)
+				{
+					LLUnit<LLUnits::Seconds, F64> recording_duration = mRecordingPeriods[index].getDuration();
+					mean += mRecordingPeriods[index].getMean(stat) * recording_duration.value();
+					total_duration += recording_duration;
+				}
+			}
+
+			if (total_duration.value())
+			{
+				mean = mean / total_duration;
+			}
+			return mean;
+		}
+
+		template <typename T>
+		typename EventAccumulator<T>::mean_t getPeriodMean(const TraceType<EventAccumulator<T> >& stat, size_t num_periods = U32_MAX) const
+		{
+			size_t total_periods = mRecordingPeriods.size();
+			num_periods = llmin(num_periods, total_periods);
+
+			typename EventAccumulator<T>::mean_t mean = 0;
+			if (num_periods <= 0) { return mean; }
+
+			S32 total_sample_count = 0;
+
+			for (S32 i = 1; i <= num_periods; i++)
+			{
+				S32 index = (mCurPeriod + total_periods - i) % total_periods;
+				if (mRecordingPeriods[index].getDuration() > 0.f)
+				{
+					S32 period_sample_count = mRecordingPeriods[index].getSampleCount(stat);
+					mean += mRecordingPeriods[index].getMean(stat) * period_sample_count;
+					total_sample_count += period_sample_count;
+				}
+			}
+
+			if (total_sample_count)
+			{
+				mean = mean / total_sample_count;
+			}
 			return mean;
 		}
 
@@ -359,7 +527,7 @@ namespace LLTrace
 			size_t total_periods = mRecordingPeriods.size();
 			num_periods = llmin(num_periods, total_periods);
 
-			typename T::mean_t mean = typename T::mean_t();
+			typename T::mean_t mean = 0;
 			if (num_periods <= 0) { return mean; }
 
 			for (S32 i = 1; i <= num_periods; i++)
@@ -370,7 +538,7 @@ namespace LLTrace
 					mean += mRecordingPeriods[index].getPerSec(stat);
 				}
 			}
-			mean /= num_periods;
+			mean = mean / num_periods;
 			return mean;
 		}
 
diff --git a/indra/llcommon/lltracethreadrecorder.cpp b/indra/llcommon/lltracethreadrecorder.cpp
index 2001b9cd7f..75c7cb2ff1 100644
--- a/indra/llcommon/lltracethreadrecorder.cpp
+++ b/indra/llcommon/lltracethreadrecorder.cpp
@@ -74,10 +74,12 @@ ThreadRecorder::~ThreadRecorder()
 {
 	delete mRootTimer;
 
-	while(mActiveRecordings.size())
+	if (!mActiveRecordings.empty())
 	{
-		mActiveRecordings.front()->mTargetRecording->stop();
+		std::for_each(mActiveRecordings.begin(), mActiveRecordings.end(), DeletePointer());
+		mActiveRecordings.clear();
 	}
+
 	set_thread_recorder(NULL);
 	delete[] mTimeBlockTreeNodes;
 }
@@ -97,34 +99,40 @@ void ThreadRecorder::activate( Recording* recording )
 	ActiveRecording* active_recording = new ActiveRecording(recording);
 	if (!mActiveRecordings.empty())
 	{
-		mActiveRecordings.front()->mPartialRecording.handOffTo(active_recording->mPartialRecording);
+		mActiveRecordings.back()->mPartialRecording.handOffTo(active_recording->mPartialRecording);
 	}
-	mActiveRecordings.push_front(active_recording);
+	mActiveRecordings.push_back(active_recording);
 
-	mActiveRecordings.front()->mPartialRecording.makePrimary();
+	mActiveRecordings.back()->mPartialRecording.makePrimary();
 }
 
-ThreadRecorder::active_recording_list_t::iterator ThreadRecorder::update( Recording* recording )
+ThreadRecorder::active_recording_list_t::reverse_iterator ThreadRecorder::bringUpToDate( Recording* recording )
 {
-	active_recording_list_t::iterator it, end_it;
-	for (it = mActiveRecordings.begin(), end_it = mActiveRecordings.end();
+	if (mActiveRecordings.empty()) return mActiveRecordings.rend();
+
+	mActiveRecordings.back()->mPartialRecording.flush();
+
+	active_recording_list_t::reverse_iterator it, end_it;
+	for (it = mActiveRecordings.rbegin(), end_it = mActiveRecordings.rend();
 		it != end_it;
 		++it)
 	{
-		active_recording_list_t::iterator next_it = it;
+		ActiveRecording* cur_recording = *it;
+
+		active_recording_list_t::reverse_iterator next_it = it;
 		++next_it;
 
 		// if we have another recording further down in the stack...
-		if (next_it != mActiveRecordings.end())
+		if (next_it != mActiveRecordings.rend())
 		{
 			// ...push our gathered data down to it
-			(*next_it)->mPartialRecording.append((*it)->mPartialRecording);
+			(*next_it)->mPartialRecording.append(cur_recording->mPartialRecording);
 		}
 
 		// copy accumulated measurements into result buffer and clear accumulator (mPartialRecording)
-		(*it)->moveBaselineToTarget();
+		cur_recording->movePartialToTarget();
 
-		if ((*it)->mTargetRecording == recording)
+		if (cur_recording->mTargetRecording == recording)
 		{
 			// found the recording, so return it
 			break;
@@ -139,28 +147,30 @@ ThreadRecorder::active_recording_list_t::iterator ThreadRecorder::update( Record
 	return it;
 }
 
-AccumulatorBuffer<CountAccumulator<F64> > 		gCountsFloat;
-AccumulatorBuffer<MeasurementAccumulator<F64> >	gMeasurementsFloat;
-AccumulatorBuffer<CountAccumulator<S64> >		gCounts;
-AccumulatorBuffer<MeasurementAccumulator<S64> >	gMeasurements;
-AccumulatorBuffer<TimeBlockAccumulator>			gStackTimers;
-AccumulatorBuffer<MemStatAccumulator>			gMemStats;
+AccumulatorBuffer<CountAccumulator<F64> > 	gCountsFloat;
+AccumulatorBuffer<EventAccumulator<F64> >	gMeasurementsFloat;
+AccumulatorBuffer<CountAccumulator<S64> >	gCounts;
+AccumulatorBuffer<EventAccumulator<S64> >	gMeasurements;
+AccumulatorBuffer<TimeBlockAccumulator>		gStackTimers;
+AccumulatorBuffer<MemStatAccumulator>		gMemStats;
 
 void ThreadRecorder::deactivate( Recording* recording )
 {
-	active_recording_list_t::iterator it = update(recording);
-	if (it != mActiveRecordings.end())
+	active_recording_list_t::reverse_iterator it = bringUpToDate(recording);
+	if (it != mActiveRecordings.rend())
 	{
 		// and if we've found the recording we wanted to update
-		active_recording_list_t::iterator next_it = it;
+		active_recording_list_t::reverse_iterator next_it = it;
 		++next_it;
-		if (next_it != mActiveRecordings.end())
+		if (next_it != mActiveRecordings.rend())
 		{
-			(*next_it)->mTargetRecording->mBuffers.write()->makePrimary();
+			(*next_it)->mPartialRecording.makePrimary();
 		}
 
-		delete *it;
-		mActiveRecordings.erase(it);
+		active_recording_list_t::iterator recording_to_remove = (++it).base();
+		llassert((*recording_to_remove)->mTargetRecording == recording);
+		delete *recording_to_remove;
+		mActiveRecordings.erase(recording_to_remove);
 	}
 }
 
@@ -169,10 +179,11 @@ ThreadRecorder::ActiveRecording::ActiveRecording( Recording* target )
 {
 }
 
-void ThreadRecorder::ActiveRecording::moveBaselineToTarget()
+void ThreadRecorder::ActiveRecording::movePartialToTarget()
 {
 	mTargetRecording->mBuffers.write()->append(mPartialRecording);
-	mPartialRecording.reset();
+	// reset based on self to keep history
+	mPartialRecording.reset(&mPartialRecording);
 }
 
 
@@ -180,21 +191,22 @@ void ThreadRecorder::ActiveRecording::moveBaselineToTarget()
 // SlaveThreadRecorder
 ///////////////////////////////////////////////////////////////////////
 
-SlaveThreadRecorder::SlaveThreadRecorder()
+SlaveThreadRecorder::SlaveThreadRecorder(MasterThreadRecorder& master)
+:	mMasterRecorder(master)
 {
-	getMasterThreadRecorder().addSlaveThread(this);
+	mMasterRecorder.addSlaveThread(this);
 }
 
 SlaveThreadRecorder::~SlaveThreadRecorder()
 {
-	getMasterThreadRecorder().removeSlaveThread(this);
+	mMasterRecorder.removeSlaveThread(this);
 }
 
 void SlaveThreadRecorder::pushToMaster()
 {
 	mThreadRecording.stop();
 	{
-		LLMutexLock(getMasterThreadRecorder().getSlaveListMutex());
+		LLMutexLock(mMasterRecorder.getSlaveListMutex());
 		mSharedData.appendFrom(mThreadRecording);
 	}
 	mThreadRecording.start();
@@ -243,7 +255,7 @@ void MasterThreadRecorder::pullFromSlaveThreads()
 
 	LLMutexLock lock(&mSlaveListMutex);
 
-	RecordingBuffers& target_recording_buffers = mActiveRecordings.front()->mPartialRecording;
+	RecordingBuffers& target_recording_buffers = mActiveRecordings.back()->mPartialRecording;
 	for (slave_thread_recorder_list_t::iterator it = mSlaveThreadRecorders.begin(), end_it = mSlaveThreadRecorders.end();
 		it != end_it;
 		++it)
diff --git a/indra/llcommon/lltracethreadrecorder.h b/indra/llcommon/lltracethreadrecorder.h
index c44bcbd12d..17a2d4a9a9 100644
--- a/indra/llcommon/lltracethreadrecorder.h
+++ b/indra/llcommon/lltracethreadrecorder.h
@@ -39,15 +39,15 @@ namespace LLTrace
 	{
 	protected:
 		struct ActiveRecording;
-		typedef std::list<ActiveRecording*> active_recording_list_t;
+		typedef std::vector<ActiveRecording*> active_recording_list_t;
 	public:
 		ThreadRecorder();
 
 		virtual ~ThreadRecorder();
 
 		void activate(Recording* recording);
-		active_recording_list_t::iterator update(Recording* recording);
 		void deactivate(Recording* recording);
+		active_recording_list_t::reverse_iterator bringUpToDate(Recording* recording);
 
 		virtual void pushToMaster() = 0;
 
@@ -58,10 +58,10 @@ namespace LLTrace
 		{
 			ActiveRecording(Recording* target);
 
-			Recording*	mTargetRecording;
+			Recording*			mTargetRecording;
 			RecordingBuffers	mPartialRecording;
 
-			void moveBaselineToTarget();
+			void movePartialToTarget();
 		};
 		Recording					mThreadRecording;
 
@@ -98,7 +98,7 @@ namespace LLTrace
 	class LL_COMMON_API SlaveThreadRecorder : public ThreadRecorder
 	{
 	public:
-		SlaveThreadRecorder();
+		SlaveThreadRecorder(MasterThreadRecorder& master);
 		~SlaveThreadRecorder();
 
 		// call this periodically to gather stats data for master thread to consume
@@ -117,7 +117,8 @@ namespace LLTrace
 		private:
 			LLMutex		mRecordingMutex;
 		};
-		SharedData		mSharedData;
+		SharedData				mSharedData;
+		MasterThreadRecorder&	mMasterRecorder;
 	};
 }