ESC-155 Multi-threaded umbrella collector for stats aggregation

Code complete with the intelligence to merge counts, mins,
maxes and means with reasonable defences.  Added QAMode
controls to the viewer so that we can QA this more quickly
by reducing the timing interval and sending the metrics
body to local logging as well as to the caps service.

5 files changed, 443 insertions, 87 deletions

diff --git a/indra/newview/llappviewer.cpp b/indra/newview/llappviewer.cpp
index 2e056238e4..e696e1af84 100644
--- a/indra/newview/llappviewer.cpp
+++ b/indra/newview/llappviewer.cpp
@@ -336,10 +336,9 @@ LLAppViewer::LLUpdaterInfo *LLAppViewer::sUpdaterInfo = NULL ;
 //----------------------------------------------------------------------------
 // Metrics logging control constants
 //----------------------------------------------------------------------------
-static const F32 METRICS_INTERVAL_MIN = 300.0;
-static const F32 METRICS_INTERVAL_MAX = 3600.0;
 static const F32 METRICS_INTERVAL_DEFAULT = 600.0;
-
+static const F32 METRICS_INTERVAL_QA = 30.0;
+static F32 app_metrics_interval = METRICS_INTERVAL_DEFAULT;
 
 void idle_afk_check()
 {
@@ -664,8 +663,15 @@ bool LLAppViewer::init()
     // Called before threads are created.
     LLCurl::initClass();
     LLMachineID::init();
-
-	LLViewerAssetStatsFF::init();
+	
+	{
+		// Viewer metrics initialization
+		if (gSavedSettings.getBOOL("QAMode") && gSavedSettings.getBOOL("QAModeMetricsSubmode"))
+		{
+			app_metrics_interval = METRICS_INTERVAL_QA;
+		}
+		LLViewerAssetStatsFF::init();
+	}
 
     initThreads();
     writeSystemInfo();
@@ -3701,7 +3707,7 @@ void LLAppViewer::idle()
 		static LLTimer report_interval;
 
 		// *TODO:  Add configuration controls for this
-		if (report_interval.getElapsedTimeF32() >= METRICS_INTERVAL_DEFAULT)
+		if (report_interval.getElapsedTimeF32() >= app_metrics_interval)
 		{
 			metricsIdle(! gDisconnected);
 			report_interval.reset();
@@ -4595,7 +4601,6 @@ void LLAppViewer::metricsIdle(bool enable_reporting)
 	if (regionp)
 	{
 		caps_url = regionp->getCapability("ViewerMetrics");
-		caps_url = "http://localhost:80/putz/";
 	}
 	
 	if (enable_reporting && regionp && ! caps_url.empty())
@@ -4608,9 +4613,9 @@ void LLAppViewer::metricsIdle(bool enable_reporting)
 		
 		LLSD * envelope = new LLSD(LLSD::emptyMap());
 		{
+			(*envelope) = gViewerAssetStatsMain->asLLSD();
 			(*envelope)["session_id"] = gAgentSessionID;
 			(*envelope)["agent_id"] = gAgentID;
-			(*envelope)["regions"] = gViewerAssetStatsMain->asLLSD();
 		}
 		
 		if (LLAppViewer::sTextureFetch)
diff --git a/indra/newview/lltexturefetch.cpp b/indra/newview/lltexturefetch.cpp
index df99818ee9..d303d425c8 100644
--- a/indra/newview/lltexturefetch.cpp
+++ b/indra/newview/lltexturefetch.cpp
@@ -2764,68 +2764,77 @@ TFReqSendMetrics::doWork(LLTextureFetchWorker * fetch_worker)
 	class lcl_responder : public LLCurl::Responder
 	{
 	public:
-		lcl_responder(volatile bool & post_failed,
-					  volatile bool & post_succeeded)
+		lcl_responder(volatile bool & reporting_break,
+					  volatile bool & reporting_started)
 			: LLHTTPClient::Responder(),
-			  mPostFailedStatus(post_failed),
-			  mPostSucceededStatus(post_succeeded)
+			  mReportingBreak(reporting_break),
+			  mReportingStarted(reporting_started)
 			{}
 
 		// virtual
 		void error(U32 status_num, const std::string & reason)
 			{
-				mPostFailedStatus = true;
+				mReportingBreak = true;
 			}
 
 		// virtual
 		void result(const LLSD & content)
 			{
-				mPostSucceededStatus = true;
+				mReportingBreak = false;
+				mReportingStarted = true;
 			}
 
 	private:
-		volatile bool & mPostFailedStatus;
-		volatile bool & mPostSucceededStatus;
+		volatile bool & mReportingBreak;
+		volatile bool & mReportingStarted;
 	};
 	
 	if (! gViewerAssetStatsThread1)
 		return true;
 
-	if (! mCapsURL.empty())
-	{
-		static volatile bool not_initial_report(false);
-		static S32 report_sequence(0);
+	static volatile bool reporting_started(false);
+	static S32 report_sequence(0);
 
-		// We've already taken over ownership of the LLSD at this point
-		// and can do normal LLSD sharing operations at this point.  But
-		// still being careful, regardless.
-		LLSD & envelope = *mReportMain;
-		{
-			envelope["sequence"] = report_sequence;
-			envelope["regions_alt"] = gViewerAssetStatsThread1->asLLSD();
-			envelope["initial"] = ! not_initial_report;					// Initial data from viewer
-			envelope["break"] = LLTextureFetch::svMetricsDataBreak;		// Break in data prior to this report
+	// We've already taken over ownership of the LLSD at this point
+	// and can do normal LLSD sharing operations at this point.  But
+	// still being careful, regardless.
+	LLSD & main_stats = *mReportMain;
 
-			// *FIXME:  Need to merge the two metrics streams here....
-		}
+	LLSD thread1_stats = gViewerAssetStatsThread1->asLLSD();			// 'duration' & 'regions' from here
+	thread1_stats["message"] = "ViewerAssetMetrics";
+	thread1_stats["sequence"] = report_sequence;
+	thread1_stats["initial"] = ! reporting_started;						// Initial data from viewer
+	thread1_stats["break"] = LLTextureFetch::svMetricsDataBreak;		// Break in data prior to this report
+		
+	// Update sequence number
+	if (S32_MAX == ++report_sequence)
+		report_sequence = 0;
+
+	// Merge the two LLSDs into a single report
+	LLViewerAssetStatsFF::merge_stats(main_stats, thread1_stats);
 
-		// Update sequence number and other metadata for next attempt.
-		if (S32_MAX == ++report_sequence)
-			report_sequence = 0;
-		LLTextureFetch::svMetricsDataBreak = false;
+	// *TODO:  Consider putting a report size limiter here.
 
+	if (! mCapsURL.empty())
+	{
 		LLCurlRequest::headers_t headers;
 		fetch_worker->getFetcher().getCurlRequest().post(mCapsURL,
 														 headers,
-														 envelope,
+														 thread1_stats,
 														 new lcl_responder(LLTextureFetch::svMetricsDataBreak,
-																		   not_initial_report));
+																		   reporting_started));
 	}
 	else
 	{
 		LLTextureFetch::svMetricsDataBreak = true;
 	}
 
+	// In QA mode, Metrics submode, log the result for ease of testing
+	if (gSavedSettings.getBOOL("QAMode") && gSavedSettings.getBOOL("QAModeMetricsSubmode"))
+	{
+		LL_INFOS("QAViewerMetrics") << thread1_stats << LL_ENDL;
+	}
+
 	gViewerAssetStatsThread1->reset();
 
 	return true;
diff --git a/indra/newview/llviewerassetstats.cpp b/indra/newview/llviewerassetstats.cpp
index 09c0364f09..c0287863f6 100644
--- a/indra/newview/llviewerassetstats.cpp
+++ b/indra/newview/llviewerassetstats.cpp
@@ -113,6 +113,16 @@ LLViewerAssetStats::PerRegionStats::reset()
 		mRequests[i].mDequeued.reset();
 		mRequests[i].mResponse.reset();
 	}
+
+	mTotalTime = 0;
+	mStartTimestamp = LLViewerAssetStatsFF::get_timestamp();
+}
+
+void
+LLViewerAssetStats::PerRegionStats::accumulateTime(duration_t now)
+{
+	mTotalTime += (now - mStartTimestamp);
+	mStartTimestamp = now;
 }
 
 
@@ -144,6 +154,9 @@ LLViewerAssetStats::reset()
 
 	// And add reference to map
 	mRegionStats[mRegionID] = mCurRegionStats;
+
+	// Start timestamp consistent with per-region collector
+	mResetTimestamp = mCurRegionStats->mStartTimestamp;
 }
 
 
@@ -155,7 +168,12 @@ LLViewerAssetStats::setRegionID(const LLUUID & region_id)
 		// Already active, ignore.
 		return;
 	}
-	
+
+	// Get duration for current set
+	const duration_t now = LLViewerAssetStatsFF::get_timestamp();
+	mCurRegionStats->accumulateTime(now);
+
+	// Prepare new set
 	PerRegionContainer::iterator new_stats = mRegionStats.find(region_id);
 	if (mRegionStats.end() == new_stats)
 	{
@@ -167,6 +185,7 @@ LLViewerAssetStats::setRegionID(const LLUUID & region_id)
 	{
 		mCurRegionStats = new_stats->second;
 	}
+	mCurRegionStats->mStartTimestamp = now;
 	mRegionID = region_id;
 }
 
@@ -195,8 +214,8 @@ LLViewerAssetStats::recordGetServiced(LLViewerAssetType::EType at, bool with_htt
 	mCurRegionStats->mRequests[int(eac)].mResponse.record(duration);
 }
 
-const LLSD
-LLViewerAssetStats::asLLSD() const
+LLSD
+LLViewerAssetStats::asLLSD()
 {
 	// Top-level tags
 	static const LLSD::String tags[EVACCount] = 
@@ -211,17 +230,18 @@ LLViewerAssetStats::asLLSD() const
 			LLSD::String("get_other")
 		};
 
-	// Sub-tags
+	// Sub-tags.  If you add or delete from this list, mergeLLSD() must be updated.
 	static const LLSD::String enq_tag("enqueued");
 	static const LLSD::String deq_tag("dequeued");
 	static const LLSD::String rcnt_tag("resp_count");
 	static const LLSD::String rmin_tag("resp_min");
 	static const LLSD::String rmax_tag("resp_max");
 	static const LLSD::String rmean_tag("resp_mean");
-	
-	LLSD ret = LLSD::emptyMap();
 
-	for (PerRegionContainer::const_iterator it = mRegionStats.begin();
+	const duration_t now = LLViewerAssetStatsFF::get_timestamp();
+	LLSD regions = LLSD::emptyMap();
+
+	for (PerRegionContainer::iterator it = mRegionStats.begin();
 		 mRegionStats.end() != it;
 		 ++it)
 	{
@@ -231,7 +251,8 @@ LLViewerAssetStats::asLLSD() const
 			continue;
 		}
 
-		const PerRegionStats & stats = *it->second;
+		PerRegionStats & stats = *it->second;
+		stats.accumulateTime(now);
 		
 		LLSD reg_stat = LLSD::emptyMap();
 		
@@ -247,12 +268,185 @@ LLViewerAssetStats::asLLSD() const
 			slot[rmean_tag] = LLSD(F64(stats.mRequests[i].mResponse.getMean()));
 		}
 
-		ret[it->first.asString()] = reg_stat;
+		reg_stat["duration"] = LLSD::Integer(stats.mTotalTime / 1000000);
+		
+		regions[it->first.asString()] = reg_stat;
 	}
 
+	LLSD ret = LLSD::emptyMap();
+	ret["regions"] = regions;
+	ret["duration"] = LLSD::Integer((now - mResetTimestamp) / 1000000);
+	
 	return ret;
 }
 
+/* static */ void
+LLViewerAssetStats::mergeLLSD(const LLSD & src, LLSD & dst)
+{
+	// Merge operator definitions
+	static const int MOP_ADD_INT(0);
+	static const int MOP_MIN_REAL(1);
+	static const int MOP_MAX_REAL(2);
+	static const int MOP_MEAN_REAL(3);	// Requires a 'mMergeOpArg' to weight the input terms
+
+	static const LLSD::String regions_key("regions");
+	static const LLSD::String root_key_list[] =
+		{
+			"duration",
+			regions_key
+		};
+	
+	static const struct
+		{
+			LLSD::String		mName;
+			int					mMergeOp;
+			LLSD::String		mMergeOpArg;
+		}
+	key_list[] =
+		{
+			// Order is important below.  We modify the data in-place and
+			// so operations like MOP_MEAN_REAL which need the "resp_count"
+			// value for weighting must be performed before "resp_count"
+			// is modified or the weight will be wrong.  Key list is
+			// defined in asLLSD() and must track it.
+
+			{ "resp_mean", MOP_MEAN_REAL, "resp_count" },
+			{ "enqueued", MOP_ADD_INT, "" },
+			{ "dequeued", MOP_ADD_INT, "" },
+			{ "resp_count", MOP_ADD_INT, "" },
+			{ "resp_min", MOP_MIN_REAL, "" },
+			{ "resp_max", MOP_MAX_REAL, "" }
+		};
+
+	// First normalized the root keys but remember if we need to do full merge
+	const bool needs_deep_merge(src.has(regions_key) && dst.has(regions_key));
+	
+	for (int root_index(0); root_index < LL_ARRAY_SIZE(root_key_list); ++root_index)
+	{
+		const LLSD::String & key_name(root_key_list[root_index]);
+						
+		if ((! src.has(key_name)) || dst.has(key_name))
+			continue;
+
+		// key present in source, not in dst here
+		dst[key_name] = src[key_name];
+	}
+
+	if (! needs_deep_merge)
+		return;
+
+	// Okay, had both src and dst 'regions' section, do the deep merge
+
+	const LLSD & root_src(src[regions_key]);
+	LLSD & root_dst(dst[regions_key]);
+	
+	const LLSD::map_const_iterator it_end(root_src.endMap());
+	for (LLSD::map_const_iterator it(root_src.beginMap()); it_end != it; ++it)
+	{
+		if (! root_dst.has(it->first))
+		{
+			// src[<region>] without matching dst[<region>]
+			root_dst[it->first] = it->second;
+		}
+		else
+		{
+			// src[<region>] with matching dst[<region>]
+			// We have matching source and destination regions.
+			// Now iterate over each asset bin in the region status.  Could iterate over
+			// an explicit list but this will do as well.
+			LLSD & reg_dst(root_dst[it->first]);
+			const LLSD & reg_src(root_src[it->first]);
+
+			const LLSD::map_const_iterator it_src_bin_end(reg_src.endMap());
+			for (LLSD::map_const_iterator it_src_bin(reg_src.beginMap()); it_src_bin_end != it_src_bin; ++it_src_bin)
+			{
+				static const LLSD::String no_touch_1("duration");
+
+				if (no_touch_1 == it_src_bin->first)
+				{
+					continue;
+				}
+				else if (! reg_dst.has(it_src_bin->first))
+				{
+					// src[<region>][<asset>] without matching dst[<region>][<asset>]
+					reg_dst[it_src_bin->first] = it_src_bin->second;
+				}
+				else
+				{
+					// src[<region>][<asset>] with matching dst[<region>][<asset>]
+					// Matching stats bin in both source and destination regions.
+					// Iterate over those bin keys we know how to merge, leave the remainder untouched.
+					LLSD & bin_dst(reg_dst[it_src_bin->first]);
+					const LLSD & bin_src(reg_src[it_src_bin->first]);
+
+					for (int key_index(0); key_index < LL_ARRAY_SIZE(key_list); ++key_index)
+					{
+						const LLSD::String & key_name(key_list[key_index].mName);
+						
+						if (! bin_src.has(key_name))
+						{
+							// Missing src[<region>][<asset>][<field>]
+							continue;
+						}
+
+						const LLSD & src_value(bin_src[key_name]);
+				
+						if (! bin_dst.has(key_name))
+						{
+							// src[<region>][<asset>][<field>] without matching dst[<region>][<asset>][<field>]
+							bin_dst[key_name] = src_value;
+						}
+						else
+						{
+							// src[<region>][<asset>][<field>] with matching dst[<region>][<asset>][<field>]
+							LLSD & dst_value(bin_dst[key_name]);
+					
+							switch (key_list[key_index].mMergeOp)
+							{
+							case MOP_ADD_INT:
+								// Simple counts, just add
+								dst_value = dst_value.asInteger() + src_value.asInteger();
+						
+								break;
+						
+							case MOP_MIN_REAL:
+								// Minimum
+								dst_value = llmin(dst_value.asReal(), src_value.asReal());
+								break;
+
+							case MOP_MAX_REAL:
+								// Maximum
+								dst_value = llmax(dst_value.asReal(), src_value.asReal());
+								break;
+
+							case MOP_MEAN_REAL:
+							    {
+									// Mean
+									const LLSD::String & weight_key(key_list[key_index].mMergeOpArg);
+									F64 src_weight(bin_src[weight_key].asReal());
+									F64 dst_weight(bin_dst[weight_key].asReal());
+									F64 tot_weight(src_weight + dst_weight);
+									if (tot_weight >= F64(0.5))
+									{
+										dst_value = (((dst_value.asReal() * dst_weight)
+													  + (src_value.asReal() * src_weight))
+													 / tot_weight);
+									}
+								}
+								break;
+						
+							default:
+								break;
+							}
+						}
+					}
+				}
+			}
+		}
+	}
+}
+
+
 // ------------------------------------------------------
 // Global free-function definitions (LLViewerAssetStatsFF namespace)
 // ------------------------------------------------------
@@ -377,7 +571,43 @@ cleanup()
 	delete gViewerAssetStatsThread1;
 	gViewerAssetStatsThread1 = 0;
 }
+
+
+void
+merge_stats(const LLSD & src, LLSD & dst)
+{
+	static const LLSD::String regions_key("regions");
+	static const LLSD::String dur_key("duration");
+
+	// Trivial cases first
+	if (! src.isMap())
+	{
+		return;
+	}
+
+	if (! dst.isMap())
+	{
+		dst = src;
+		return;
+	}
 	
+	// Okay, both src and dst are maps at this point.
+	// Collector class know how to merge it's part
+	LLViewerAssetStats::mergeLLSD(src, dst);
+
+	// Now merge non-collector bits manually.
+	const LLSD::map_const_iterator it_end(src.endMap());
+	for (LLSD::map_const_iterator it(src.beginMap()); it_end != it; ++it)
+	{
+		if (regions_key == it->first || dur_key == it->first)
+			continue;
+
+		if (dst.has(it->first))
+			continue;
+
+		dst[it->first] = it->second;
+	}
+}
 
 } // namespace LLViewerAssetStatsFF
 
diff --git a/indra/newview/llviewerassetstats.h b/indra/newview/llviewerassetstats.h
index efd0897bb8..65ecdca4a0 100644
--- a/indra/newview/llviewerassetstats.h
+++ b/indra/newview/llviewerassetstats.h
@@ -83,10 +83,10 @@ class LLViewerAssetStats
 public:
 	enum EViewerAssetCategories
 	{
-		EVACTextureTempHTTPGet,			//< Texture GETs
-		EVACTextureTempUDPGet,			//< Texture GETs
-		EVACTextureNonTempHTTPGet,		//< Texture GETs
-		EVACTextureNonTempUDPGet,		//< Texture GETs
+		EVACTextureTempHTTPGet,			//< Texture GETs - temp/baked, HTTP
+		EVACTextureTempUDPGet,			//< Texture GETs - temp/baked, UDP
+		EVACTextureNonTempHTTPGet,		//< Texture GETs - perm, HTTP
+		EVACTextureNonTempUDPGet,		//< Texture GETs - perm, UDP
 		EVACWearableUDPGet,				//< Wearable GETs
 		EVACSoundUDPGet,				//< Sound GETs
 		EVACGestureUDPGet,				//< Gesture GETs
@@ -103,7 +103,11 @@ public:
 	typedef U64 duration_t;
 	
 	/**
-	 * Collected data for a single region visited by the avatar.
+	 * @brief Collected data for a single region visited by the avatar.
+	 *
+	 * Fairly simple, for each asset bin enumerated above a count
+	 * of enqueue and dequeue operations and simple stats on response
+	 * times for completed requests.
 	 */
 	class PerRegionStats : public LLRefCount
 	{
@@ -118,8 +122,15 @@ public:
 		
 		void reset();
 
+		// Apply current running time to total and reset start point.
+		// Return current timestamp as a convenience.
+		void accumulateTime(duration_t now);
+		
 	public:
 		LLUUID mRegionID;
+		duration_t mTotalTime;
+		duration_t mStartTimestamp;
+		
 		struct
 		{
 			LLSimpleStatCounter			mEnqueued;
@@ -142,13 +153,17 @@ public:
 	// collection calls.
 	void setRegionID(const LLUUID & region_id);
 
-	// Non-Cached GET Requests
+	// Asset GET Requests
 	void recordGetEnqueued(LLViewerAssetType::EType at, bool with_http, bool is_temp);
 	void recordGetDequeued(LLViewerAssetType::EType at, bool with_http, bool is_temp);
 	void recordGetServiced(LLViewerAssetType::EType at, bool with_http, bool is_temp, duration_t duration);
 
-	// Retrieve current metrics for all visited regions.
-	const LLSD asLLSD() const;
+	// Retrieve current metrics for all visited regions (NULL region UUID excluded)
+	LLSD asLLSD();
+
+	// Merge two LLSD's structured as per asLLSD().  If inputs are not
+	// correctly formed, result is undefined (little defensive action).
+	static void mergeLLSD(const LLSD & src, LLSD & dst);
 	
 protected:
 	typedef std::map<LLUUID, LLPointer<PerRegionStats> > PerRegionContainer;
@@ -165,6 +180,9 @@ protected:
 
 	// Metrics data for all regions during one collection cycle
 	PerRegionContainer mRegionStats;
+
+	// Time of last reset
+	duration_t mResetTimestamp;
 };
 
 
@@ -189,6 +207,17 @@ extern LLViewerAssetStats * gViewerAssetStatsThread1;
 namespace LLViewerAssetStatsFF
 {
 /**
+ * @brief Allocation and deallocation of globals.
+ *
+ * init() should be called before threads are started that will access it though
+ * you'll likely get away with calling it afterwards.  cleanup() should only be
+ * called after threads are shutdown to prevent races on the global pointers.
+ */
+void init();
+
+void cleanup();
+
+/**
  * We have many timers, clocks etc. in the runtime.  This is the
  * canonical timestamp for these metrics which is compatible with
  * the pre-existing timestamping in the texture fetcher.
@@ -224,15 +253,16 @@ void record_response_thread1(LLViewerAssetType::EType at, bool with_http, bool i
 						  LLViewerAssetStats::duration_t duration);
 
 /**
- * @brief Allocation and deallocation of globals.
+ * @brief Merge two LLSD reports from different collector instances
  *
- * init() should be called before threads are started that will access it though
- * you'll likely get away with calling it afterwards.  cleanup() should only be
- * called after threads are shutdown to prevent races on the global pointers.
+ * Use this to merge the LLSD's from two threads.  For top-level,
+ * non-region data the destination (dst) is considered authoritative
+ * if the key is present in both source and destination.  For
+ * regions, a numerical merge is performed when data are present in
+ * both source and destination and the 'right thing' is done for
+ * counts, minimums, maximums and averages.
  */
-void init();
-
-void cleanup();
+void merge_stats(const LLSD & src, LLSD & dst);
 
 } // namespace LLViewerAssetStatsFF
 
diff --git a/indra/newview/tests/llviewerassetstats_test.cpp b/indra/newview/tests/llviewerassetstats_test.cpp
index c3c38ef925..e8cde5fc5d 100644
--- a/indra/newview/tests/llviewerassetstats_test.cpp
+++ b/indra/newview/tests/llviewerassetstats_test.cpp
@@ -39,9 +39,11 @@
 #include "lltut.h"
 #include "../llviewerassetstats.h"
 #include "lluuid.h"
+#include "llsdutil.h"
 
 static const char * all_keys[] = 
 {
+	"duration",
 	"get_other",
 	"get_texture_temp_http",
 	"get_texture_temp_udp",
@@ -77,11 +79,13 @@ static const char * sub_keys[] =
 static const LLUUID region1("4e2d81a3-6263-6ffe-ad5c-8ce04bee07e8");
 static const LLUUID region2("68762cc8-b68b-4e45-854b-e830734f2d4a");
 
+#if 0
 static bool
 is_empty_map(const LLSD & sd)
 {
 	return sd.isMap() && 0 == sd.size();
 }
+#endif
 
 static bool
 is_single_key_map(const LLSD & sd, const std::string & key)
@@ -95,6 +99,12 @@ is_double_key_map(const LLSD & sd, const std::string & key1, const std::string &
 	return sd.isMap() && 2 == sd.size() && sd.has(key1) && sd.has(key2);
 }
 
+static bool
+is_no_stats_map(const LLSD & sd)
+{
+	return is_double_key_map(sd, "duration", "regions");
+}
+
 namespace tut
 {
 	struct tst_viewerassetstats_index
@@ -131,14 +141,15 @@ namespace tut
 
 		// Default (NULL) region ID doesn't produce LLSD results so should
 		// get an empty map back from output
-		ensure("Null LLSD initially", is_empty_map(sd_full));
+		ensure("Stat-less LLSD initially", is_no_stats_map(sd_full));
 
 		// Once the region is set, we will get a response even with no data collection
 		it->setRegionID(region1);
 		sd_full = it->asLLSD();
-		ensure("Correct single-key LLSD map", is_single_key_map(sd_full, region1.asString()));
-
-		LLSD sd = sd_full[region1.asString()];
+		ensure("Correct single-key LLSD map root", is_double_key_map(sd_full, "duration", "regions"));
+		ensure("Correct single-key LLSD map regions", is_single_key_map(sd_full["regions"], region1.asString()));
+		
+		LLSD sd = sd_full["regions"][region1.asString()];
 
 		delete it;
 			
@@ -167,7 +178,8 @@ namespace tut
 		it->setRegionID(region1);
 		
 		LLSD sd = it->asLLSD();
-		ensure("Correct single-key LLSD map", is_single_key_map(sd, region1.asString()));
+		ensure("Correct single-key LLSD map root", is_double_key_map(sd, "regions", "duration"));
+		ensure("Correct single-key LLSD map regions", is_single_key_map(sd["regions"], region1.asString()));
 		sd = sd[region1.asString()];
 		
 		delete it;
@@ -191,8 +203,9 @@ namespace tut
 		LLViewerAssetStatsFF::record_dequeue_main(LLViewerAssetType::AT_BODYPART, false, false);
 
 		LLSD sd = gViewerAssetStatsMain->asLLSD();
-		ensure("Correct single-key LLSD map", is_single_key_map(sd, region1.asString()));
-		sd = sd[region1.asString()];
+		ensure("Correct single-key LLSD map root", is_double_key_map(sd, "regions", "duration"));
+		ensure("Correct single-key LLSD map regions", is_single_key_map(sd["regions"], region1.asString()));
+		sd = sd["regions"][region1.asString()];
 		
 		// Check a few points on the tree for content
 		ensure("sd[get_texture_non_temp_udp][enqueued] is 1", (1 == sd["get_texture_non_temp_udp"]["enqueued"].asInteger()));
@@ -204,7 +217,7 @@ namespace tut
 		// Reset and check zeros...
 		// Reset leaves current region in place
 		gViewerAssetStatsMain->reset();
-		sd = gViewerAssetStatsMain->asLLSD()[region1.asString()];
+		sd = gViewerAssetStatsMain->asLLSD()["regions"][region1.asString()];
 		
 		delete gViewerAssetStatsMain;
 		gViewerAssetStatsMain = NULL;
@@ -228,10 +241,11 @@ namespace tut
 		LLViewerAssetStatsFF::record_dequeue_main(LLViewerAssetType::AT_BODYPART, false, false);
 
 		LLSD sd = gViewerAssetStatsThread1->asLLSD();
-		ensure("Other collector is empty", is_empty_map(sd));
+		ensure("Other collector is empty", is_no_stats_map(sd));
 		sd = gViewerAssetStatsMain->asLLSD();
-		ensure("Correct single-key LLSD map", is_single_key_map(sd, region1.asString()));
-		sd = sd[region1.asString()];
+		ensure("Correct single-key LLSD map root", is_double_key_map(sd, "regions", "duration"));
+		ensure("Correct single-key LLSD map regions", is_single_key_map(sd["regions"], region1.asString()));
+		sd = sd["regions"][region1.asString()];
 		
 		// Check a few points on the tree for content
 		ensure("sd[get_texture_non_temp_udp][enqueued] is 1", (1 == sd["get_texture_non_temp_udp"]["enqueued"].asInteger()));
@@ -243,7 +257,7 @@ namespace tut
 		// Reset and check zeros...
 		// Reset leaves current region in place
 		gViewerAssetStatsMain->reset();
-		sd = gViewerAssetStatsMain->asLLSD()[region1.asString()];
+		sd = gViewerAssetStatsMain->asLLSD()["regions"][region1.asString()];
 		
 		delete gViewerAssetStatsMain;
 		gViewerAssetStatsMain = NULL;
@@ -277,9 +291,12 @@ namespace tut
 
 		LLSD sd = gViewerAssetStatsMain->asLLSD();
 
-		ensure("Correct double-key LLSD map", is_double_key_map(sd, region1.asString(), region2.asString()));
-		LLSD sd1 = sd[region1.asString()];
-		LLSD sd2 = sd[region2.asString()];
+		// std::cout << sd << std::endl;
+		
+		ensure("Correct double-key LLSD map root", is_double_key_map(sd, "duration", "regions"));
+		ensure("Correct double-key LLSD map regions", is_double_key_map(sd["regions"], region1.asString(), region2.asString()));
+		LLSD sd1 = sd["regions"][region1.asString()];
+		LLSD sd2 = sd["regions"][region2.asString()];
 		
 		// Check a few points on the tree for content
 		ensure("sd1[get_texture_non_temp_udp][enqueued] is 1", (1 == sd1["get_texture_non_temp_udp"]["enqueued"].asInteger()));
@@ -297,8 +314,9 @@ namespace tut
 		// Reset leaves current region in place
 		gViewerAssetStatsMain->reset();
 		sd = gViewerAssetStatsMain->asLLSD();
-		ensure("Correct single-key LLSD map", is_single_key_map(sd, region2.asString()));
-		sd2 = sd[region2.asString()];
+		ensure("Correct single-key LLSD map root", is_double_key_map(sd, "regions", "duration"));
+		ensure("Correct single-key LLSD map regions", is_single_key_map(sd["regions"], region2.asString()));
+		sd2 = sd["regions"][region2.asString()];
 		
 		delete gViewerAssetStatsMain;
 		gViewerAssetStatsMain = NULL;
@@ -345,9 +363,10 @@ namespace tut
 
 		LLSD sd = gViewerAssetStatsMain->asLLSD();
 
-		ensure("Correct double-key LLSD map", is_double_key_map(sd, region1.asString(), region2.asString()));
-		LLSD sd1 = sd[region1.asString()];
-		LLSD sd2 = sd[region2.asString()];
+		ensure("Correct double-key LLSD map root", is_double_key_map(sd, "duration", "regions"));
+		ensure("Correct double-key LLSD map regions", is_double_key_map(sd["regions"], region1.asString(), region2.asString()));
+		LLSD sd1 = sd["regions"][region1.asString()];
+		LLSD sd2 = sd["regions"][region2.asString()];
 		
 		// Check a few points on the tree for content
 		ensure("sd1[get_texture_non_temp_udp][enqueued] is 1", (1 == sd1["get_texture_non_temp_udp"]["enqueued"].asInteger()));
@@ -365,8 +384,9 @@ namespace tut
 		// Reset leaves current region in place
 		gViewerAssetStatsMain->reset();
 		sd = gViewerAssetStatsMain->asLLSD();
-		ensure("Correct single-key LLSD map", is_single_key_map(sd, region2.asString()));
-		sd2 = sd[region2.asString()];
+		ensure("Correct single-key LLSD map root", is_double_key_map(sd, "duration", "regions"));
+		ensure("Correct single-key LLSD map regions", is_single_key_map(sd["regions"], region2.asString()));
+		sd2 = sd["regions"][region2.asString()];
 		
 		delete gViewerAssetStatsMain;
 		gViewerAssetStatsMain = NULL;
@@ -407,10 +427,11 @@ namespace tut
 		LLViewerAssetStatsFF::record_enqueue_main(LLViewerAssetType::AT_LSL_BYTECODE, true, true);
 
 		LLSD sd = gViewerAssetStatsThread1->asLLSD();
-		ensure("Other collector is empty", is_empty_map(sd));
+		ensure("Other collector is empty", is_no_stats_map(sd));
 		sd = gViewerAssetStatsMain->asLLSD();
-		ensure("Correct single-key LLSD map", is_single_key_map(sd, region1.asString()));
-		sd = sd[region1.asString()];
+		ensure("Correct single-key LLSD map root", is_double_key_map(sd, "regions", "duration"));
+		ensure("Correct single-key LLSD map regions", is_single_key_map(sd["regions"], region1.asString()));
+		sd = sd["regions"][region1.asString()];
 		
 		// Check a few points on the tree for content
 		ensure("sd[get_gesture_udp][enqueued] is 0", (0 == sd["get_gesture_udp"]["enqueued"].asInteger()));
@@ -425,7 +446,7 @@ namespace tut
 		// Reset and check zeros...
 		// Reset leaves current region in place
 		gViewerAssetStatsMain->reset();
-		sd = gViewerAssetStatsMain->asLLSD()[region1.asString()];
+		sd = gViewerAssetStatsMain->asLLSD()["regions"][region1.asString()];
 		
 		delete gViewerAssetStatsMain;
 		gViewerAssetStatsMain = NULL;
@@ -436,4 +457,65 @@ namespace tut
 		ensure("sd[get_gesture_udp][dequeued] is reset", (0 == sd["get_gesture_udp"]["dequeued"].asInteger()));
 	}
 
+	// Check that the LLSD merger knows what it's doing (basic test)
+	template<> template<>
+	void tst_viewerassetstats_index_object_t::test<9>()
+	{
+		LLSD::String reg1_name = region1.asString();
+		LLSD::String reg2_name = region2.asString();
+
+		LLSD reg1_stats = LLSD::emptyMap();
+		LLSD reg2_stats = LLSD::emptyMap();
+
+		LLSD & tmp_other1 = reg1_stats["get_other"];
+		tmp_other1["enqueued"] = 4;
+		tmp_other1["dequeued"] = 4;
+		tmp_other1["resp_count"] = 8;
+		tmp_other1["resp_max"] = F64(23.2892);
+		tmp_other1["resp_min"] = F64(0.2829);
+		tmp_other1["resp_mean"] = F64(2.298928);
+
+		LLSD & tmp_other2 = reg2_stats["get_other"];
+		tmp_other2["enqueued"] = 8;
+		tmp_other2["dequeued"] = 7;
+		tmp_other2["resp_count"] = 3;
+		tmp_other2["resp_max"] = F64(6.5);
+		tmp_other2["resp_min"] = F64(0.01);
+		tmp_other2["resp_mean"] = F64(4.1);
+		
+		{
+			LLSD src = LLSD::emptyMap();
+			LLSD dst = LLSD::emptyMap();
+
+			src["regions"][reg1_name] = reg1_stats;
+			src["duration"] = 24;
+			dst["regions"][reg2_name] = reg2_stats;
+			dst["duration"] = 36;
+
+			LLViewerAssetStats::mergeLLSD(src, dst);
+		
+			ensure("region 1 in merged stats", llsd_equals(reg1_stats, dst["regions"][reg1_name]));
+			ensure("region 2 still in merged stats", llsd_equals(reg2_stats, dst["regions"][reg2_name]));
+		}
+
+		{
+			LLSD src = LLSD::emptyMap();
+			LLSD dst = LLSD::emptyMap();
+
+			src["regions"][reg1_name] = reg1_stats;
+			src["duration"] = 24;
+			dst["regions"][reg1_name] = reg2_stats;
+			dst["duration"] = 36;
+
+			LLViewerAssetStats::mergeLLSD(src, dst);
+
+			ensure("src not ruined", llsd_equals(reg1_stats, src["regions"][reg1_name]));
+			ensure_equals("added enqueued counts", dst["regions"][reg1_name]["get_other"]["enqueued"].asInteger(), 12);
+			ensure_equals("added dequeued counts", dst["regions"][reg1_name]["get_other"]["dequeued"].asInteger(), 11);
+			ensure_equals("added response counts", dst["regions"][reg1_name]["get_other"]["resp_count"].asInteger(), 11);
+			ensure_approximately_equals("min'd minimum response times", dst["regions"][reg1_name]["get_other"]["resp_min"].asReal(), 0.01, 20);
+			ensure_approximately_equals("max'd maximum response times", dst["regions"][reg1_name]["get_other"]["resp_max"].asReal(), 23.2892, 20);
+			ensure_approximately_equals("weighted mean of means", dst["regions"][reg1_name]["get_other"]["resp_mean"].asReal(), 2.7901295, 20);
+		}
+	}
 }