Merge branch 'master' v6.4.6 into DRTVWR-497

1 files changed, 314 insertions, 272 deletions

diff --git a/indra/llcommon/llsingleton.h b/indra/llcommon/llsingleton.h
index 7def9b019c..30a5b21cf8 100644
--- a/indra/llcommon/llsingleton.h
+++ b/indra/llcommon/llsingleton.h
@@ -30,18 +30,10 @@
 #include <list>
 #include <vector>
 #include <typeinfo>
-
-#if LL_WINDOWS
-#pragma warning (push)
-#pragma warning (disable:4265)
-#endif
-// warning C4265: 'std::_Pad' : class has virtual functions, but destructor is not virtual
-
-#include <mutex>
-
-#if LL_WINDOWS
-#pragma warning (pop)
-#endif
+#include "mutex.h"
+#include "lockstatic.h"
+#include "llthread.h"               // on_main_thread()
+#include "llmainthreadtask.h"
 
 class LLSingletonBase: private boost::noncopyable
 {
@@ -51,15 +43,13 @@ public:
 private:
     // All existing LLSingleton instances are tracked in this master list.
     typedef std::list<LLSingletonBase*> list_t;
-    static list_t& get_master();
-    // This, on the other hand, is a stack whose top indicates the LLSingleton
-    // currently being initialized.
-    static list_t& get_initializing();
+    // Size of stack whose top indicates the LLSingleton currently being
+    // initialized.
+    static list_t::size_type get_initializing_size();
     // Produce a vector<LLSingletonBase*> of master list, in dependency order.
     typedef std::vector<LLSingletonBase*> vec_t;
     static vec_t dep_sort();
 
-    bool mCleaned;                  // cleanupSingleton() has been called
     // we directly depend on these other LLSingletons
     typedef boost::unordered_set<LLSingletonBase*> set_t;
     set_t mDepends;
@@ -68,8 +58,8 @@ protected:
     typedef enum e_init_state
     {
         UNINITIALIZED = 0,          // must be default-initialized state
+        QUEUED,                     // construction queued, not yet executing
         CONSTRUCTING,               // within DERIVED_TYPE constructor
-        CONSTRUCTED,                // finished DERIVED_TYPE constructor
         INITIALIZING,               // within DERIVED_TYPE::initSingleton()
         INITIALIZED,                // normal case
         DELETED                     // deleteSingleton() or deleteAll() called
@@ -115,21 +105,23 @@ protected:
     // Remove 'this' from the init stack in case of exception in the
     // LLSingleton subclass constructor.
     static void reset_initializing(list_t::size_type size);
-private:
-    // logging
-    static void log_initializing(const char* verb, const char* name);
 protected:
     // If a given call to B::getInstance() happens during either A::A() or
     // A::initSingleton(), record that A directly depends on B.
-    void capture_dependency(list_t& initializing, EInitState);
+    void capture_dependency();
 
-    // delegate LL_ERRS() logging to llsingleton.cpp
+    // delegate logging calls to llsingleton.cpp
     static void logerrs(const char* p1, const char* p2="",
                         const char* p3="", const char* p4="");
-    // delegate LL_WARNS() logging to llsingleton.cpp
     static void logwarns(const char* p1, const char* p2="",
                          const char* p3="", const char* p4="");
+    static void loginfos(const char* p1, const char* p2="",
+                         const char* p3="", const char* p4="");
+    static void logdebugs(const char* p1, const char* p2="",
+                          const char* p3="", const char* p4="");
     static std::string demangle(const char* mangled);
+    // these classname() declarations restate template functions declared in
+    // llerror.h because we avoid #including that here
     template <typename T>
     static std::string classname()       { return demangle(typeid(T).name()); }
     template <typename T>
@@ -139,6 +131,9 @@ protected:
     virtual void initSingleton() {}
     virtual void cleanupSingleton() {}
 
+    // internal wrapper around calls to cleanupSingleton()
+    void cleanup_();
+
     // deleteSingleton() isn't -- and shouldn't be -- a virtual method. It's a
     // class static. However, given only Foo*, deleteAll() does need to be
     // able to reach Foo::deleteSingleton(). Make LLSingleton (which declares
@@ -148,32 +143,15 @@ protected:
 
 public:
     /**
-     * Call this to call the cleanupSingleton() method for every LLSingleton
-     * constructed since the start of the last cleanupAll() call. (Any
-     * LLSingleton constructed DURING a cleanupAll() call won't be cleaned up
-     * until the next cleanupAll() call.) cleanupSingleton() neither deletes
-     * nor destroys its LLSingleton; therefore it's safe to include logic that
-     * might take significant realtime or even throw an exception.
-     *
-     * The most important property of cleanupAll() is that cleanupSingleton()
-     * methods are called in dependency order, leaf classes last. Thus, given
-     * two LLSingleton subclasses A and B, if A's dependency on B is properly
-     * expressed as a B::getInstance() or B::instance() call during either
-     * A::A() or A::initSingleton(), B will be cleaned up after A.
-     *
-     * If a cleanupSingleton() method throws an exception, the exception is
-     * logged, but cleanupAll() attempts to continue calling the rest of the
-     * cleanupSingleton() methods.
-     */
-    static void cleanupAll();
-    /**
-     * Call this to call the deleteSingleton() method for every LLSingleton
-     * constructed since the start of the last deleteAll() call. (Any
-     * LLSingleton constructed DURING a deleteAll() call won't be cleaned up
-     * until the next deleteAll() call.) deleteSingleton() deletes and
-     * destroys its LLSingleton. Any cleanup logic that might take significant
-     * realtime -- or throw an exception -- must not be placed in your
-     * LLSingleton's destructor, but rather in its cleanupSingleton() method.
+     * deleteAll() calls the cleanupSingleton() and deleteSingleton() methods
+     * for every LLSingleton constructed since the start of the last
+     * deleteAll() call. (Any LLSingleton constructed DURING a deleteAll()
+     * call won't be cleaned up until the next deleteAll() call.)
+     * deleteSingleton() deletes and destroys its LLSingleton. Any cleanup
+     * logic that might take significant realtime -- or throw an exception --
+     * must not be placed in your LLSingleton's destructor, but rather in its
+     * cleanupSingleton() method, which is called implicitly by
+     * deleteSingleton().
      *
      * The most important property of deleteAll() is that deleteSingleton()
      * methods are called in dependency order, leaf classes last. Thus, given
@@ -181,9 +159,9 @@ public:
      * expressed as a B::getInstance() or B::instance() call during either
      * A::A() or A::initSingleton(), B will be cleaned up after A.
      *
-     * If a deleteSingleton() method throws an exception, the exception is
-     * logged, but deleteAll() attempts to continue calling the rest of the
-     * deleteSingleton() methods.
+     * If a cleanupSingleton() or deleteSingleton() method throws an
+     * exception, the exception is logged, but deleteAll() attempts to
+     * continue calling the rest of the deleteSingleton() methods.
      */
     static void deleteAll();
 };
@@ -203,9 +181,16 @@ struct LLSingleton_manage_master
     {
         LLSingletonBase::reset_initializing(size);
     }
-    // For any LLSingleton subclass except the MasterList, obtain the init
-    // stack from the MasterList singleton instance.
-    LLSingletonBase::list_t& get_initializing() { return LLSingletonBase::get_initializing(); }
+    // For any LLSingleton subclass except the MasterList, obtain the size of
+    // the init stack from the MasterList singleton instance.
+    LLSingletonBase::list_t::size_type get_initializing_size()
+    {
+        return LLSingletonBase::get_initializing_size();
+    }
+    void capture_dependency(LLSingletonBase* sb)
+    {
+        sb->capture_dependency();
+    }
 };
 
 // But for the specific case of LLSingletonBase::MasterList, don't.
@@ -218,20 +203,14 @@ struct LLSingleton_manage_master<LLSingletonBase::MasterList>
     void pop_initializing (LLSingletonBase*) {}
     // since we never pushed, no need to clean up
     void reset_initializing(LLSingletonBase::list_t::size_type size) {}
-    LLSingletonBase::list_t& get_initializing()
-    {
-        // The MasterList shouldn't depend on any other LLSingletons. We'd
-        // get into trouble if we tried to recursively engage that machinery.
-        static LLSingletonBase::list_t sDummyList;
-        return sDummyList;
-    }
+    LLSingletonBase::list_t::size_type get_initializing_size() { return 0; }
+    void capture_dependency(LLSingletonBase*) {}
 };
 
 // Now we can implement LLSingletonBase's template constructor.
 template <typename DERIVED_TYPE>
 LLSingletonBase::LLSingletonBase(tag<DERIVED_TYPE>):
-    mCleaned(false),
-    mDeleteSingleton(NULL)
+    mDeleteSingleton(nullptr)
 {
     // This is the earliest possible point at which we can push this new
     // instance onto the init stack. LLSingleton::constructSingleton() can't
@@ -273,10 +252,19 @@ class LLParamSingleton;
  * leading back to yours, move the instance reference from your constructor to
  * your initSingleton() method.
  *
- * If you override LLSingleton<T>::cleanupSingleton(), your method will be
- * called if someone calls LLSingletonBase::cleanupAll(). The significant part
- * of this promise is that cleanupAll() will call individual
- * cleanupSingleton() methods in reverse dependency order.
+ * If you override LLSingleton<T>::cleanupSingleton(), your method will
+ * implicitly be called by LLSingleton<T>::deleteSingleton() just before the
+ * instance is destroyed. We introduce a special cleanupSingleton() method
+ * because cleanupSingleton() operations can involve nontrivial realtime, or
+ * throw an exception. A destructor should do neither!
+ *
+ * If your cleanupSingleton() method throws an exception, we log that
+ * exception but carry on.
+ *
+ * If at some point you call LLSingletonBase::deleteAll(), all remaining
+ * LLSingleton<T> instances will be destroyed in reverse dependency order. (Or
+ * call MySubclass::deleteSingleton() to specifically destroy the canonical
+ * MySubclass instance.)
  *
  * That is, consider LLSingleton subclasses C, B and A. A depends on B, which
  * in turn depends on C. These dependencies are expressed as calls to
@@ -284,33 +272,34 @@ class LLParamSingleton;
  * It shouldn't matter whether these calls appear in A::A() or
  * A::initSingleton(), likewise B::B() or B::initSingleton().
  *
- * We promise that if you later call LLSingletonBase::cleanupAll():
- * 1. A::cleanupSingleton() will be called before
- * 2. B::cleanupSingleton(), which will be called before
- * 3. C::cleanupSingleton().
+ * We promise that if you later call LLSingletonBase::deleteAll():
+ * 1. A::deleteSingleton() will be called before
+ * 2. B::deleteSingleton(), which will be called before
+ * 3. C::deleteSingleton().
  * Put differently, if your LLSingleton subclass constructor or
  * initSingleton() method explicitly depends on some other LLSingleton
  * subclass, you may continue to rely on that other subclass in your
  * cleanupSingleton() method.
- *
- * We introduce a special cleanupSingleton() method because cleanupSingleton()
- * operations can involve nontrivial realtime, or might throw an exception. A
- * destructor should do neither!
- *
- * If your cleanupSingleton() method throws an exception, we log that
- * exception but proceed with the remaining cleanupSingleton() calls.
- *
- * Similarly, if at some point you call LLSingletonBase::deleteAll(), all
- * remaining LLSingleton instances will be destroyed in dependency order. (Or
- * call MySubclass::deleteSingleton() to specifically destroy the canonical
- * MySubclass instance.)
- *
- * As currently written, LLSingleton is not thread-safe.
  */
 template <typename DERIVED_TYPE>
 class LLSingleton : public LLSingletonBase
 {
 private:
+    // LLSingleton<DERIVED_TYPE> must have a distinct instance of
+    // SingletonData for every distinct DERIVED_TYPE. It's tempting to
+    // consider hoisting SingletonData up into LLSingletonBase. Don't do it.
+    struct SingletonData
+    {
+        // Use a recursive_mutex in case of constructor circularity. With a
+        // non-recursive mutex, that would result in deadlock.
+        typedef std::recursive_mutex mutex_t;
+        mutex_t mMutex;             // LockStatic looks for mMutex
+
+        EInitState      mInitState{UNINITIALIZED};
+        DERIVED_TYPE*   mInstance{nullptr};
+    };
+    typedef llthread::LockStatic<SingletonData> LockStatic;
+
     // Allow LLParamSingleton subclass -- but NOT DERIVED_TYPE itself -- to
     // access our private members.
     friend class LLParamSingleton<DERIVED_TYPE>;
@@ -319,17 +308,17 @@ private:
     // purpose for its subclass LLParamSingleton is to support Singletons
     // requiring constructor arguments. constructSingleton() supports both use
     // cases.
+    // Accepting LockStatic& requires that the caller has already locked our
+    // static data before calling.
     template <typename... Args>
-    static void constructSingleton(Args&&... args)
+    static void constructSingleton(LockStatic& lk, Args&&... args)
     {
-        auto prev_size = LLSingleton_manage_master<DERIVED_TYPE>().get_initializing().size();
-        // getInstance() calls are from within constructor
-        sData.mInitState = CONSTRUCTING;
+        auto prev_size = LLSingleton_manage_master<DERIVED_TYPE>().get_initializing_size();
+        // Any getInstance() calls after this point are from within constructor
+        lk->mInitState = CONSTRUCTING;
         try
         {
-            sData.mInstance = new DERIVED_TYPE(std::forward<Args>(args)...);
-            // we have called constructor, have not yet called initSingleton()
-            sData.mInitState = CONSTRUCTED;
+            lk->mInstance = new DERIVED_TYPE(std::forward<Args>(args)...);
         }
         catch (const std::exception& err)
         {
@@ -343,62 +332,56 @@ private:
             // There isn't a separate EInitState value meaning "we attempted
             // to construct this LLSingleton subclass but could not," so use
             // DELETED. That seems slightly more appropriate than UNINITIALIZED.
-            sData.mInitState = DELETED;
+            lk->mInitState = DELETED;
             // propagate the exception
             throw;
         }
-    }
 
-    static void finishInitializing()
-    {
-        // getInstance() calls are from within initSingleton()
-        sData.mInitState = INITIALIZING;
+        // Any getInstance() calls after this point are from within initSingleton()
+        lk->mInitState = INITIALIZING;
         try
         {
             // initialize singleton after constructing it so that it can
             // reference other singletons which in turn depend on it, thus
             // breaking cyclic dependencies
-            sData.mInstance->initSingleton();
-            sData.mInitState = INITIALIZED;
+            lk->mInstance->initSingleton();
+            lk->mInitState = INITIALIZED;
 
             // pop this off stack of initializing singletons
-            pop_initializing();
+            pop_initializing(lk->mInstance);
         }
         catch (const std::exception& err)
         {
             // pop this off stack of initializing singletons here, too --
             // BEFORE logging, so log-machinery LLSingletons don't record a
             // dependency on DERIVED_TYPE!
-            pop_initializing();
+            pop_initializing(lk->mInstance);
             logwarns("Error in ", classname<DERIVED_TYPE>().c_str(),
                      "::initSingleton(): ", err.what());
-            // and get rid of the instance entirely
+            // Get rid of the instance entirely. This call depends on our
+            // recursive_mutex. We could have a deleteSingleton(LockStatic&)
+            // overload and pass lk, but we don't strictly need it.
             deleteSingleton();
             // propagate the exception
             throw;
         }
     }
 
-    static void pop_initializing()
+    static void pop_initializing(LLSingletonBase* sb)
     {
         // route through LLSingleton_manage_master so we Do The Right Thing
         // (namely, nothing) for MasterList
-        LLSingleton_manage_master<DERIVED_TYPE>().pop_initializing(sData.mInstance);
+        LLSingleton_manage_master<DERIVED_TYPE>().pop_initializing(sb);
     }
 
-    // Without this 'using' declaration, the static method we're declaring
-    // here would hide the base-class method we want it to call.
-    using LLSingletonBase::capture_dependency;
-    static void capture_dependency()
+    static void capture_dependency(LLSingletonBase* sb)
     {
         // By this point, if DERIVED_TYPE was pushed onto the initializing
         // stack, it has been popped off. So the top of that stack, if any, is
         // an LLSingleton that directly depends on DERIVED_TYPE. If
         // getInstance() was called by another LLSingleton, rather than from
         // vanilla application code, record the dependency.
-        sData.mInstance->capture_dependency(
-            LLSingleton_manage_master<DERIVED_TYPE>().get_initializing(),
-            sData.mInitState);
+        LLSingleton_manage_master<DERIVED_TYPE>().capture_dependency(sb);
     }
 
     // We know of no way to instruct the compiler that every subclass
@@ -411,20 +394,6 @@ private:
     // subclass body.
     virtual void you_must_use_LLSINGLETON_macro() = 0;
 
-    // The purpose of this struct is to engage the C++11 guarantee that static
-    // variables declared in function scope are initialized exactly once, even
-    // if multiple threads concurrently reach the same declaration.
-    // https://en.cppreference.com/w/cpp/language/storage_duration#Static_local_variables
-    // Since getInstance() declares a static instance of SingletonInitializer,
-    // only the first call to getInstance() calls constructSingleton().
-    struct SingletonInitializer
-    {
-        SingletonInitializer()
-        {
-            constructSingleton();
-        }
-    };
-
 protected:
     // Pass DERIVED_TYPE explicitly to LLSingletonBase's constructor because,
     // until our subclass constructor completes, *this isn't yet a
@@ -439,97 +408,176 @@ protected:
         LLSingleton_manage_master<DERIVED_TYPE>().add(this);
     }
 
-public:
+protected:
     virtual ~LLSingleton()
     {
-        // remove this instance from the master list
+        // This phase of cleanup is performed in the destructor rather than in
+        // deleteSingleton() to defend against manual deletion. When we moved
+        // cleanup to deleteSingleton(), we hit crashes due to dangling
+        // pointers in the MasterList.
+        LockStatic lk;
+        lk->mInstance  = nullptr;
+        lk->mInitState = DELETED;
+
+        // Remove this instance from the master list.
         LLSingleton_manage_master<DERIVED_TYPE>().remove(this);
-        sData.mInstance = NULL;
-        sData.mInitState = DELETED;
     }
 
+public:
     /**
-     * @brief Immediately delete the singleton.
+     * @brief Cleanup and destroy the singleton instance.
      *
-     * A subsequent call to LLProxy::getInstance() will construct a new
-     * instance of the class.
+     * deleteSingleton() calls this instance's cleanupSingleton() method and
+     * then destroys the instance.
      *
-     * Without an explicit call to LLSingletonBase::deleteAll(), LLSingletons
-     * are implicitly destroyed after main() has exited and the C++ runtime is
-     * cleaning up statically-constructed objects. Some classes derived from
-     * LLSingleton have objects that are part of a runtime system that is
-     * terminated before main() exits. Calling the destructor of those objects
-     * after the termination of their respective systems can cause crashes and
-     * other problems during termination of the project. Using this method to
-     * destroy the singleton early can prevent these crashes.
+     * A subsequent call to LLSingleton<T>::getInstance() will construct a new
+     * instance of the class.
      *
-     * An example where this is needed is for a LLSingleton that has an APR
-     * object as a member that makes APR calls on destruction. The APR system is
-     * shut down explicitly before main() exits. This causes a crash on exit.
-     * Using this method before the call to apr_terminate() and NOT calling
-     * getInstance() again will prevent the crash.
+     * Without an explicit call to LLSingletonBase::deleteAll(), or
+     * LLSingleton<T>::deleteSingleton(), LLSingleton instances are simply
+     * leaked. (Allowing implicit destruction at shutdown caused too many
+     * problems.)
      */
     static void deleteSingleton()
     {
-        delete sData.mInstance;
-        // SingletonData state handled by destructor, above
+        // Hold the lock while we call cleanupSingleton() and the destructor.
+        // Our destructor also instantiates LockStatic, requiring a recursive
+        // mutex.
+        LockStatic lk;
+        // of course, only cleanup and delete if there's something there
+        if (lk->mInstance)
+        {
+            lk->mInstance->cleanup_();
+            delete lk->mInstance;
+            // destructor clears mInstance (and mInitState)
+        }
     }
 
     static DERIVED_TYPE* getInstance()
     {
-        // call constructSingleton() only the first time we get here
-        static SingletonInitializer sInitializer;
-
-        switch (sData.mInitState)
-        {
-        case UNINITIALIZED:
-            // should never be uninitialized at this point
-            logerrs("Uninitialized singleton ",
-                    classname<DERIVED_TYPE>().c_str());
-            return NULL;
-
-        case CONSTRUCTING:
-            // here if DERIVED_TYPE's constructor (directly or indirectly)
-            // calls DERIVED_TYPE::getInstance()
-            logerrs("Tried to access singleton ",
-                    classname<DERIVED_TYPE>().c_str(),
-                    " from singleton constructor!");
-            return NULL;
-
-        case CONSTRUCTED:
-            // first time through: set to CONSTRUCTED by
-            // constructSingleton(), called by sInitializer's constructor;
-            // still have to call initSingleton()
-            finishInitializing();
-            break;
-
-        case INITIALIZING:
-            // here if DERIVED_TYPE::initSingleton() (directly or indirectly)
-            // calls DERIVED_TYPE::getInstance(): go ahead and allow it
-        case INITIALIZED:
-            // normal subsequent calls
-            break;
-
-        case DELETED:
-            // called after deleteSingleton()
-            logwarns("Trying to access deleted singleton ",
-                     classname<DERIVED_TYPE>().c_str(),
-                     " -- creating new instance");
-            // This recovery sequence is NOT thread-safe! We would need a
-            // recursive_mutex a la LLParamSingleton.
-            constructSingleton();
-            finishInitializing();
-            break;
-        }
-
-        // record the dependency, if any: check if we got here from another
-        // LLSingleton's constructor or initSingleton() method
-        capture_dependency();
-        return sData.mInstance;
+        // We know the viewer has LLSingleton dependency circularities. If you
+        // feel strongly motivated to eliminate them, cheers and good luck.
+        // (At that point we could consider a much simpler locking mechanism.)
+
+        // If A and B depend on each other, and thread T1 requests A at the
+        // same moment thread T2 requests B, you could get a sequence like this:
+        // - T1 locks A
+        // - T2 locks B
+        // - T1, having constructed A, calls A::initSingleton(), which calls
+        //   B::getInstance() and blocks on B's lock
+        // - T2, having constructed B, calls B::initSingleton(), which calls
+        //   A::getInstance() and blocks on A's lock
+        // In other words, classic deadlock.
+
+        // Avoid that by constructing and initializing every LLSingleton on
+        // the main thread. In that scenario:
+        // - T1 locks A
+        // - T2 locks B
+        // - T1 discovers A is UNINITIALIZED, so it queues a task for the main
+        //   thread, unlocks A and blocks on the std::future.
+        // - T2 discovers B is UNINITIALIZED, so it queues a task for the main
+        //   thread, unlocks B and blocks on the std::future.
+        // - The main thread executes T1's request for A. It locks A and
+        //   starts to construct it.
+        // - A::initSingleton() calls B::getInstance(). Fine: nobody's holding
+        //   B's lock.
+        // - The main thread locks B, constructs B, calls B::initSingleton(),
+        //   which calls A::getInstance(), which returns A.
+        // - B::getInstance() returns B to A::initSingleton(), unlocking B.
+        // - A::getInstance() returns A to the task wrapper, unlocking A.
+        // - The task wrapper passes A to T1 via the future. T1 resumes.
+        // - The main thread executes T2's request for B. Oh look, B already
+        //   exists. The task wrapper passes B to T2 via the future. T2
+        //   resumes.
+        // This still works even if one of T1 or T2 *is* the main thread.
+        // This still works even if thread T3 requests B at the same moment as
+        // T2. Finding B still UNINITIALIZED, T3 also queues a task for the
+        // main thread, unlocks B and blocks on a (distinct) std::future. By
+        // the time the main thread executes T3's request for B, B already
+        // exists, and is simply delivered via the future.
+
+        { // nested scope for 'lk'
+            // In case racing threads call getInstance() at the same moment,
+            // serialize the calls.
+            LockStatic lk;
+
+            switch (lk->mInitState)
+            {
+            case CONSTRUCTING:
+                // here if DERIVED_TYPE's constructor (directly or indirectly)
+                // calls DERIVED_TYPE::getInstance()
+                logerrs("Tried to access singleton ",
+                        classname<DERIVED_TYPE>().c_str(),
+                        " from singleton constructor!");
+                return nullptr;
+
+            case INITIALIZING:
+                // here if DERIVED_TYPE::initSingleton() (directly or indirectly)
+                // calls DERIVED_TYPE::getInstance(): go ahead and allow it
+            case INITIALIZED:
+                // normal subsequent calls
+                // record the dependency, if any: check if we got here from another
+                // LLSingleton's constructor or initSingleton() method
+                capture_dependency(lk->mInstance);
+                return lk->mInstance;
+
+            case DELETED:
+                // called after deleteSingleton()
+                logwarns("Trying to access deleted singleton ",
+                         classname<DERIVED_TYPE>().c_str(),
+                         " -- creating new instance");
+                // fall through
+            case UNINITIALIZED:
+            case QUEUED:
+                // QUEUED means some secondary thread has already requested an
+                // instance, but for present purposes that's semantically
+                // identical to UNINITIALIZED: either way, we must ourselves
+                // request an instance.
+                break;
+            }
+
+            // Here we need to construct a new instance.
+            if (on_main_thread())
+            {
+                // On the main thread, directly construct the instance while
+                // holding the lock.
+                constructSingleton(lk);
+                capture_dependency(lk->mInstance);
+                return lk->mInstance;
+            }
+
+            // Here we need to construct a new instance, but we're on a secondary
+            // thread.
+            lk->mInitState = QUEUED;
+        } // unlock 'lk'
+
+        // Per the comment block above, dispatch to the main thread.
+        loginfos(classname<DERIVED_TYPE>().c_str(),
+                 "::getInstance() dispatching to main thread");
+        auto instance = LLMainThreadTask::dispatch(
+            [](){
+                // VERY IMPORTANT to call getInstance() on the main thread,
+                // rather than going straight to constructSingleton()!
+                // During the time window before mInitState is INITIALIZED,
+                // multiple requests might be queued. It's essential that, as
+                // the main thread processes them, only the FIRST such request
+                // actually constructs the instance -- every subsequent one
+                // simply returns the existing instance.
+                loginfos(classname<DERIVED_TYPE>().c_str(),
+                         "::getInstance() on main thread");
+                return getInstance();
+            });
+        // record the dependency chain tracked on THIS thread, not the main
+        // thread (consider a getInstance() overload with a tag param that
+        // suppresses dep tracking when dispatched to the main thread)
+        capture_dependency(instance);
+        loginfos(classname<DERIVED_TYPE>().c_str(),
+                 "::getInstance() returning on requesting thread");
+        return instance;
     }
 
     // Reference version of getInstance()
-    // Preferred over getInstance() as it disallows checking for NULL
+    // Preferred over getInstance() as it disallows checking for nullptr
     static DERIVED_TYPE& instance()
     {
         return *getInstance();
@@ -539,7 +587,9 @@ public:
     // Use this to avoid accessing singletons before they can safely be constructed.
     static bool instanceExists()
     {
-        return sData.mInitState == INITIALIZED;
+        // defend any access to sData from racing threads
+        LockStatic lk;
+        return lk->mInitState == INITIALIZED;
     }
 
     // Has this singleton been deleted? This can be useful during shutdown
@@ -547,23 +597,12 @@ public:
     // cleaned up.
     static bool wasDeleted()
     {
-        return sData.mInitState == DELETED;
+        // defend any access to sData from racing threads
+        LockStatic lk;
+        return lk->mInitState == DELETED;
     }
-
-private:
-    struct SingletonData
-    {
-        // explicitly has a default constructor so that member variables are zero initialized in BSS
-        // and only changed by singleton logic, not constructor running during startup
-        EInitState      mInitState;
-        DERIVED_TYPE*   mInstance;
-    };
-    static SingletonData sData;
 };
 
-template<typename T>
-typename LLSingleton<T>::SingletonData LLSingleton<T>::sData;
-
 
 /**
  * LLParamSingleton<T> is like LLSingleton<T>, except in the following ways:
@@ -588,47 +627,86 @@ class LLParamSingleton : public LLSingleton<DERIVED_TYPE>
 {
 private:
     typedef LLSingleton<DERIVED_TYPE> super;
-    // Use a recursive_mutex in case of constructor circularity. With a
-    // non-recursive mutex, that would result in deadlock rather than the
-    // logerrs() call in getInstance().
-    typedef std::recursive_mutex mutex_t;
+    using typename super::LockStatic;
 
-public:
-    using super::deleteSingleton;
-    using super::instanceExists;
-    using super::wasDeleted;
-
-    // Passes arguments to DERIVED_TYPE's constructor and sets appropriate states
+    // Passes arguments to DERIVED_TYPE's constructor and sets appropriate
+    // states, returning a pointer to the new instance.
     template <typename... Args>
-    static void initParamSingleton(Args&&... args)
+    static DERIVED_TYPE* initParamSingleton_(Args&&... args)
     {
         // In case racing threads both call initParamSingleton() at the same
         // time, serialize them. One should initialize; the other should see
         // mInitState already set.
-        std::unique_lock<mutex_t> lk(getMutex());
+        LockStatic lk;
         // For organizational purposes this function shouldn't be called twice
-        if (super::sData.mInitState != super::UNINITIALIZED)
+        if (lk->mInitState != super::UNINITIALIZED)
         {
             super::logerrs("Tried to initialize singleton ",
                            super::template classname<DERIVED_TYPE>().c_str(),
                            " twice!");
+            return nullptr;
+        }
+        else if (on_main_thread())
+        {
+            // on the main thread, simply construct instance while holding lock
+            super::logdebugs(super::template classname<DERIVED_TYPE>().c_str(),
+                             "::initParamSingleton()");
+            super::constructSingleton(lk, std::forward<Args>(args)...);
+            return lk->mInstance;
         }
         else
         {
-            super::constructSingleton(std::forward<Args>(args)...);
-            super::finishInitializing();
+            // on secondary thread, dispatch to main thread --
+            // set state so we catch any other calls before the main thread
+            // picks up the task
+            lk->mInitState = super::QUEUED;
+            // very important to unlock here so main thread can actually process
+            lk.unlock();
+            super::loginfos(super::template classname<DERIVED_TYPE>().c_str(),
+                            "::initParamSingleton() dispatching to main thread");
+            // Normally it would be the height of folly to reference-bind
+            // 'args' into a lambda to be executed on some other thread! By
+            // the time that thread executed the lambda, the references would
+            // all be dangling, and Bad Things would result. But
+            // LLMainThreadTask::dispatch() promises to block until the passed
+            // task has completed. So in this case we know the references will
+            // remain valid until the lambda has run, so we dare to bind
+            // references.
+            auto instance = LLMainThreadTask::dispatch(
+                [&](){
+                    super::loginfos(super::template classname<DERIVED_TYPE>().c_str(),
+                                    "::initParamSingleton() on main thread");
+                    return initParamSingleton_(std::forward<Args>(args)...);
+                });
+            super::loginfos(super::template classname<DERIVED_TYPE>().c_str(),
+                            "::initParamSingleton() returning on requesting thread");
+            return instance;
         }
     }
 
+public:
+    using super::deleteSingleton;
+    using super::instanceExists;
+    using super::wasDeleted;
+
+    /// initParamSingleton() constructs the instance, returning a reference.
+    /// Pass whatever arguments are required to construct DERIVED_TYPE.
+    template <typename... Args>
+    static DERIVED_TYPE& initParamSingleton(Args&&... args)
+    {
+        return *initParamSingleton_(std::forward<Args>(args)...);
+    }
+
     static DERIVED_TYPE* getInstance()
     {
         // In case racing threads call getInstance() at the same moment as
         // initParamSingleton(), serialize the calls.
-        std::unique_lock<mutex_t> lk(getMutex());
+        LockStatic lk;
 
-        switch (super::sData.mInitState)
+        switch (lk->mInitState)
         {
         case super::UNINITIALIZED:
+        case super::QUEUED:
             super::logerrs("Uninitialized param singleton ",
                            super::template classname<DERIVED_TYPE>().c_str());
             break;
@@ -639,25 +717,13 @@ public:
                            " from singleton constructor!");
             break;
 
-        case super::CONSTRUCTED:
-            // Should never happen!? The CONSTRUCTED state is specifically to
-            // navigate through LLSingleton::SingletonInitializer getting
-            // constructed (once) before LLSingleton::getInstance()'s switch
-            // on mInitState. But our initParamSingleton() method calls
-            // constructSingleton() and then calls finishInitializing(), which
-            // immediately sets INITIALIZING. Why are we here?
-            super::logerrs("Param singleton ",
-                           super::template classname<DERIVED_TYPE>().c_str(),
-                           "::initSingleton() not yet called");
-            break;
-
         case super::INITIALIZING:
             // As with LLSingleton, explicitly permit circular calls from
             // within initSingleton()
         case super::INITIALIZED:
             // for any valid call, capture dependencies
-            super::capture_dependency();
-            return super::sData.mInstance;
+            super::capture_dependency(lk->mInstance);
+            return lk->mInstance;
 
         case super::DELETED:
             super::logerrs("Trying to access deleted param singleton ",
@@ -677,30 +743,6 @@ public:
     {
         return *getInstance();
     }
-
-private:
-    // sMutex must be a function-local static rather than a static member. One
-    // of the essential features of LLSingleton and friends is that they must
-    // support getInstance() even when the containing module's static
-    // variables have not yet been runtime-initialized. A mutex requires
-    // construction. A static class member might not yet have been
-    // constructed.
-    //
-    // We could store a dumb mutex_t*, notice when it's NULL and allocate a
-    // heap mutex -- but that's vulnerable to race conditions. And we can't
-    // defend the dumb pointer with another mutex.
-    //
-    // We could store a std::atomic<mutex_t*> -- but a default-constructed
-    // std::atomic<T> does not contain a valid T, even a default-constructed
-    // T! Which means std::atomic, too, requires runtime initialization.
-    //
-    // But a function-local static is guaranteed to be initialized exactly
-    // once, the first time control reaches that declaration.
-    static mutex_t& getMutex()
-    {
-        static mutex_t sMutex;
-        return sMutex;
-    }
 };
 
 /**
@@ -725,9 +767,9 @@ public:
     using super::instanceExists;
     using super::wasDeleted;
 
-    static void construct()
+    static DT* construct()
     {
-        super::initParamSingleton();
+        return super::initParamSingleton();
     }
 };