SL-16220: Add LL::ThreadPool class and a "General" instance.

ThreadPool bundles a WorkQueue with the specified number of worker threads to
service it. Each ThreadPool has a name that can be used to locate its
WorkQueue.

Each worker thread calls WorkQueue::runUntilClose().

ThreadPool listens on the "LLApp" LLEventPump for shutdown notification. On
receiving that, it closes its WorkQueue and then join()s each of its worker
threads for orderly shutdown.

Add a settings.xml entry "ThreadPoolSizes", the first LLSD-valued settings
entry to expect a map: pool name->size. The expectation is that usually code
instantiating a particular ThreadPool will have a default size in mind, but it
should check "ThreadPoolSizes" for a user override.

Make idle_startup()'s STATE_SEED_CAP_GRANTED state instantiate a "General"
ThreadPool. This is function-static for lazy initialization.

Eliminate LLMainLoopRepeater, which is completely unreferenced. Any potential
future use cases are better addressed by posting to the main loop's WorkQueue.

Eliminate llappviewer.cpp's private LLDeferredTaskList class, which
implemented LLAppViewer::addOnIdleCallback(). Make addOnIdleCallback() post
work to the main loop's WorkQueue instead.

12 files changed, 171 insertions, 226 deletions

diff --git a/indra/llcommon/CMakeLists.txt b/indra/llcommon/CMakeLists.txt
index fda43dd24c..c374f1135c 100644
--- a/indra/llcommon/CMakeLists.txt
+++ b/indra/llcommon/CMakeLists.txt
@@ -121,8 +121,8 @@ set(llcommon_SOURCE_FILES
     lluriparser.cpp
     lluuid.cpp
     llworkerthread.cpp
-    timing.cpp
     u64.cpp
+    threadpool.cpp
     workqueue.cpp
     StackWalker.cpp
     )
@@ -258,6 +258,7 @@ set(llcommon_HEADER_FILES
     lockstatic.h
     stdtypes.h
     stringize.h
+    threadpool.h
     threadsafeschedule.h
     timer.h
     tuple.h
diff --git a/indra/llcommon/threadpool.cpp b/indra/llcommon/threadpool.cpp
new file mode 100644
index 0000000000..aa7d4179a2
--- /dev/null
+++ b/indra/llcommon/threadpool.cpp
@@ -0,0 +1,75 @@
+/**
+ * @file   threadpool.cpp
+ * @author Nat Goodspeed
+ * @date   2021-10-21
+ * @brief  Implementation for threadpool.
+ * 
+ * $LicenseInfo:firstyear=2021&license=viewerlgpl$
+ * Copyright (c) 2021, Linden Research, Inc.
+ * $/LicenseInfo$
+ */
+
+// Precompiled header
+#include "linden_common.h"
+// associated header
+#include "threadpool.h"
+// STL headers
+// std headers
+// external library headers
+// other Linden headers
+#include "llerror.h"
+#include "llevents.h"
+#include "stringize.h"
+
+LL::ThreadPool::ThreadPool(const std::string& name, size_t threads):
+    mQueue(name),
+    mName("ThreadPool:" + name)
+{
+    for (size_t i = 0; i < threads; ++i)
+    {
+        std::string tname{ STRINGIZE(mName << ':' << (i+i) << '/' << threads) };
+        mThreads.emplace_back(tname, [this, tname](){ run(tname); });
+    }
+    // Listen on "LLApp", and when the app is shutting down, close the queue
+    // and join the workers.
+    LLEventPumps::instance().obtain("LLApp").listen(
+        mName,
+        [this](const LLSD& stat)
+        {
+            std::string status(stat["status"]);
+            if (status != "running")
+            {
+                // viewer is starting shutdown -- proclaim the end is nigh!
+                LL_DEBUGS("ThreadPool") << mName << " saw " << status << LL_ENDL;
+                close();
+            }
+            return false;
+        });
+}
+
+LL::ThreadPool::~ThreadPool()
+{
+    close();
+}
+
+void LL::ThreadPool::close()
+{
+    if (! mQueue.isClosed())
+    {
+        LL_DEBUGS("ThreadPool") << mName << " closing queue and joining threads" << LL_ENDL;
+        mQueue.close();
+        for (auto& pair: mThreads)
+        {
+            LL_DEBUGS("ThreadPool") << mName << " waiting on thread " << pair.first << LL_ENDL;
+            pair.second.join();
+        }
+        LL_DEBUGS("ThreadPool") << mName << " shutdown complete" << LL_ENDL;
+    }
+}
+
+void LL::ThreadPool::run(const std::string& name)
+{
+    LL_DEBUGS("ThreadPool") << name << " starting" << LL_ENDL;
+    mQueue.runUntilClose();
+    LL_DEBUGS("ThreadPool") << name << " stopping" << LL_ENDL;
+}
diff --git a/indra/llcommon/threadpool.h b/indra/llcommon/threadpool.h
new file mode 100644
index 0000000000..8f3c8514b5
--- /dev/null
+++ b/indra/llcommon/threadpool.h
@@ -0,0 +1,46 @@
+/**
+ * @file   threadpool.h
+ * @author Nat Goodspeed
+ * @date   2021-10-21
+ * @brief  ThreadPool configures a WorkQueue along with a pool of threads to
+ *         service it.
+ * 
+ * $LicenseInfo:firstyear=2021&license=viewerlgpl$
+ * Copyright (c) 2021, Linden Research, Inc.
+ * $/LicenseInfo$
+ */
+
+#if ! defined(LL_THREADPOOL_H)
+#define LL_THREADPOOL_H
+
+#include "workqueue.h"
+#include <string>
+#include <thread>
+#include <utility>                  // std::pair
+#include <vector>
+
+namespace LL
+{
+
+    class ThreadPool
+    {
+    public:
+        /**
+         * Pass ThreadPool a string name. This can be used to look up the
+         * relevant WorkQueue.
+         */
+        ThreadPool(const std::string& name, size_t threads=1);
+        ~ThreadPool();
+        void close();
+
+    private:
+        void run(const std::string& name);
+
+        WorkQueue mQueue;
+        std::string mName;
+        std::vector<std::pair<std::string, std::thread>> mThreads;
+    };
+
+} // namespace LL
+
+#endif /* ! defined(LL_THREADPOOL_H) */
diff --git a/indra/llcommon/timing.cpp b/indra/llcommon/timing.cpp
deleted file mode 100644
index c2dc695ef3..0000000000
--- a/indra/llcommon/timing.cpp
+++ /dev/null
@@ -1,25 +0,0 @@
-/** 
- * @file timing.cpp
- * @brief This file will be deprecated in the future.
- *
- * $LicenseInfo:firstyear=2000&license=viewerlgpl$
- * Second Life Viewer Source Code
- * Copyright (C) 2010, Linden Research, Inc.
- * 
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation;
- * version 2.1 of the License only.
- * 
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Lesser General Public License for more details.
- * 
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
- * 
- * Linden Research, Inc., 945 Battery Street, San Francisco, CA  94111  USA
- * $/LicenseInfo$
- */
diff --git a/indra/llcommon/workqueue.cpp b/indra/llcommon/workqueue.cpp
index ffc9a97dc0..114aeea1f3 100644
--- a/indra/llcommon/workqueue.cpp
+++ b/indra/llcommon/workqueue.cpp
@@ -38,6 +38,16 @@ void LL::WorkQueue::close()
     mQueue.close();
 }
 
+bool LL::WorkQueue::isClosed()
+{
+    return mQueue.isClosed();
+}
+
+bool LL::WorkQueue::done()
+{
+    return mQueue.done();
+}
+
 void LL::WorkQueue::runUntilClose()
 {
     try
diff --git a/indra/llcommon/workqueue.h b/indra/llcommon/workqueue.h
index b88aef989a..cfae2019dc 100644
--- a/indra/llcommon/workqueue.h
+++ b/indra/llcommon/workqueue.h
@@ -59,6 +59,11 @@ namespace LL
          */
         void close();
 
+        /// producer end: are we prevented from pushing any additional items?
+        bool isClosed();
+        /// consumer end: are we done, is the queue entirely drained?
+        bool done();
+
         /*---------------------- fire and forget API -----------------------*/
 
         /// fire-and-forget, but at a particular (future?) time
diff --git a/indra/newview/CMakeLists.txt b/indra/newview/CMakeLists.txt
index fbe75af712..bad36505d1 100644
--- a/indra/newview/CMakeLists.txt
+++ b/indra/newview/CMakeLists.txt
@@ -393,7 +393,6 @@ set(viewer_SOURCE_FILES
     llloginhandler.cpp
     lllogininstance.cpp
     llmachineid.cpp
-    llmainlooprepeater.cpp
     llmanip.cpp
     llmaniprotate.cpp
     llmanipscale.cpp
@@ -1032,7 +1031,6 @@ set(viewer_HEADER_FILES
     llloginhandler.h
     lllogininstance.h
     llmachineid.h
-    llmainlooprepeater.h
     llmanip.h
     llmaniprotate.h
     llmanipscale.h
diff --git a/indra/newview/app_settings/settings.xml b/indra/newview/app_settings/settings.xml
index 802453d508..3c7fe174fd 100644
--- a/indra/newview/app_settings/settings.xml
+++ b/indra/newview/app_settings/settings.xml
@@ -12663,6 +12663,20 @@
       <key>Value</key>
       <integer>50</integer>
     </map>
+    <key>ThreadPoolSizes</key>
+    <map>
+      <key>Comment</key>
+      <string>Map of size overrides for specific thread pools.</string>
+      <key>Persist</key>
+      <integer>1</integer>
+      <key>Type</key>
+      <string>LLSD</string>
+      <key>Value</key>
+      <map>
+        <key>General</key>
+        <integer>4</integer>
+      </map>
+    </map>
     <key>ThrottleBandwidthKBPS</key>
     <map>
       <key>Comment</key>
diff --git a/indra/newview/llappviewer.cpp b/indra/newview/llappviewer.cpp
index 7c932a3959..7c363eea5e 100644
--- a/indra/newview/llappviewer.cpp
+++ b/indra/newview/llappviewer.cpp
@@ -239,7 +239,6 @@ using namespace LL;
 // Include for security api initialization
 #include "llsecapi.h"
 #include "llmachineid.h"
-#include "llmainlooprepeater.h"
 #include "llcleanup.h"
 
 #include "llcoproceduremanager.h"
@@ -385,42 +384,6 @@ static std::string gLaunchFileOnQuit;
 // Used on Win32 for other apps to identify our window (eg, win_setup)
 const char* const VIEWER_WINDOW_CLASSNAME = "Second Life";
 
-//-- LLDeferredTaskList ------------------------------------------------------
-
-/**
- * A list of deferred tasks.
- *
- * We sometimes need to defer execution of some code until the viewer gets idle,
- * e.g. removing an inventory item from within notifyObservers() may not work out.
- *
- * Tasks added to this list will be executed in the next LLAppViewer::idle() iteration.
- * All tasks are executed only once.
- */
-class LLDeferredTaskList: public LLSingleton<LLDeferredTaskList>
-{
-	LLSINGLETON_EMPTY_CTOR(LLDeferredTaskList);
-	LOG_CLASS(LLDeferredTaskList);
-
-	friend class LLAppViewer;
-	typedef boost::signals2::signal<void()> signal_t;
-
-	void addTask(const signal_t::slot_type& cb)
-	{
-		mSignal.connect(cb);
-	}
-
-	void run()
-	{
-		if (!mSignal.empty())
-		{
-			mSignal();
-			mSignal.disconnect_all_slots();
-		}
-	}
-
-	signal_t mSignal;
-};
-
 //----------------------------------------------------------------------------
 
 // List of entries from strings.xml to always replace
@@ -980,9 +943,6 @@ bool LLAppViewer::init()
 	}
 	LL_INFOS("InitInfo") << "Cache initialization is done." << LL_ENDL ;
 
-	// Initialize the repeater service.
-	LLMainLoopRepeater::instance().start();
-
 	//
 	// Initialize the window
 	//
@@ -2171,8 +2131,6 @@ bool LLAppViewer::cleanup()
 	SUBSYSTEM_CLEANUP(LLProxy);
     LLCore::LLHttp::cleanup();
 
-	LLMainLoopRepeater::instance().stop();
-
 	ll_close_fail_log();
 
 	LLError::LLCallStacks::cleanup();
@@ -4437,7 +4395,7 @@ bool LLAppViewer::initCache()
 
 void LLAppViewer::addOnIdleCallback(const boost::function<void()>& cb)
 {
-	LLDeferredTaskList::instance().addTask(cb);
+	gMainloopWork.post(cb);
 }
 
 void LLAppViewer::loadKeyBindings()
@@ -5211,9 +5169,6 @@ void LLAppViewer::idle()
 		}
 	}
 
-	// Execute deferred tasks.
-	LLDeferredTaskList::instance().run();
-
 	// Service the WorkQueue we use for replies from worker threads.
 	// Use function statics for the timeslice setting so we only have to fetch
 	// and convert MainWorkTime once.
diff --git a/indra/newview/llmainlooprepeater.cpp b/indra/newview/llmainlooprepeater.cpp
deleted file mode 100644
index 6736e9a950..0000000000
--- a/indra/newview/llmainlooprepeater.cpp
+++ /dev/null
@@ -1,88 +0,0 @@
-/** 
- * @file llmachineid.cpp
- * @brief retrieves unique machine ids
- *
- * $LicenseInfo:firstyear=2009&license=viewerlgpl$
- * Second Life Viewer Source Code
- * Copyright (C) 2010, Linden Research, Inc.
- * 
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation;
- * version 2.1 of the License only.
- * 
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Lesser General Public License for more details.
- * 
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
- * 
- * Linden Research, Inc., 945 Battery Street, San Francisco, CA  94111  USA
- * $/LicenseInfo$
- */
-
-#include "llviewerprecompiledheaders.h"
-#include "llapr.h"
-#include "llevents.h"
-#include "llmainlooprepeater.h"
-
-
-
-// LLMainLoopRepeater
-//-----------------------------------------------------------------------------
-
-
-LLMainLoopRepeater::LLMainLoopRepeater(void):
-	mQueue(0)
-{
-	; // No op.
-}
-
-
-void LLMainLoopRepeater::start(void)
-{
-	if(mQueue != 0) return;
-
-	mQueue = new LLThreadSafeQueue<LLSD>(1024);
-	mMainLoopConnection = LLEventPumps::instance().
-		obtain("mainloop").listen(LLEventPump::inventName(), boost::bind(&LLMainLoopRepeater::onMainLoop, this, _1));
-	mRepeaterConnection = LLEventPumps::instance().
-		obtain("mainlooprepeater").listen(LLEventPump::inventName(), boost::bind(&LLMainLoopRepeater::onMessage, this, _1));
-}
-
-
-void LLMainLoopRepeater::stop(void)
-{
-	mMainLoopConnection.release();
-	mRepeaterConnection.release();
-
-	delete mQueue;
-	mQueue = 0;
-}
-
-
-bool LLMainLoopRepeater::onMainLoop(LLSD const &)
-{
-	LLSD message;
-	while(mQueue->tryPopBack(message)) {
-		std::string pump = message["pump"].asString();
-		if(pump.length() == 0 ) continue; // No pump.
-		LLEventPumps::instance().obtain(pump).post(message["payload"]);
-	}
-	return false;
-}
-
-
-bool LLMainLoopRepeater::onMessage(LLSD const & event)
-{
-	try {
-		mQueue->pushFront(event);
-	} catch(LLThreadSafeQueueError & e) {
-		LL_WARNS() << "could not repeat message (" << e.what() << ")" << 
-			event.asString() << LL_ENDL;
-	}
-	return false;
-}
diff --git a/indra/newview/llmainlooprepeater.h b/indra/newview/llmainlooprepeater.h
deleted file mode 100644
index 2ec3a74e4a..0000000000
--- a/indra/newview/llmainlooprepeater.h
+++ /dev/null
@@ -1,64 +0,0 @@
-/** 
- * @file llmainlooprepeater.h
- * @brief a service for repeating messages on the main loop.
- *
- * $LicenseInfo:firstyear=2010&license=viewerlgpl$
- * Second Life Viewer Source Code
- * Copyright (C) 2010, Linden Research, Inc.
- * 
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation;
- * version 2.1 of the License only.
- * 
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Lesser General Public License for more details.
- * 
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
- * 
- * Linden Research, Inc., 945 Battery Street, San Francisco, CA  94111  USA
- * $/LicenseInfo$
- */
-
-#ifndef LL_LLMAINLOOPREPEATER_H
-#define LL_LLMAINLOOPREPEATER_H
-
-
-#include "llsd.h"
-#include "llthreadsafequeue.h"
-
-
-//
-// A service which creates the pump 'mainlooprepeater' to which any thread can
-// post a message that will be re-posted on the main loop.
-//
-// The posted message should contain two map elements: pump and payload.  The
-// pump value is a string naming the pump to which the message should be
-// re-posted.  The payload value is what will be posted to the designated pump.
-//
-class LLMainLoopRepeater:
-	public LLSingleton<LLMainLoopRepeater>
-{
-	LLSINGLETON(LLMainLoopRepeater);
-public:
-	// Start the repeater service.
-	void start(void);
-	
-	// Stop the repeater service.
-	void stop(void);
-	
-private:
-	LLTempBoundListener mMainLoopConnection;
-	LLTempBoundListener mRepeaterConnection;
-	LLThreadSafeQueue<LLSD> * mQueue;
-	
-	bool onMainLoop(LLSD const &);
-	bool onMessage(LLSD const & event);
-};
-
-
-#endif
diff --git a/indra/newview/llstartup.cpp b/indra/newview/llstartup.cpp
index 57c5074804..13e7fcb6e4 100644
--- a/indra/newview/llstartup.cpp
+++ b/indra/newview/llstartup.cpp
@@ -205,6 +205,9 @@
 
 #include "llstacktrace.h"
 
+#include "threadpool.h"
+
+
 #if LL_WINDOWS
 #include "lldxhardware.h"
 #endif
@@ -301,6 +304,18 @@ void callback_cache_name(const LLUUID& id, const std::string& full_name, bool is
 // local classes
 //
 
+void launchThreadPool()
+{
+    LLSD poolSizes{ gSavedSettings.getLLSD("ThreadPoolSizes") };
+    LLSD sizeSpec{ poolSizes["General"] };
+    LLSD::Integer size{ sizeSpec.isInteger()? sizeSpec.asInteger() : 3 };
+    LL_DEBUGS("ThreadPool") << "Instantiating General pool with "
+                            << size << " threads" << LL_ENDL;
+    // Use a function-static ThreadPool: static duration, but instantiated
+    // only on demand.
+    static LL::ThreadPool pool("General", size);
+}
+
 void update_texture_fetch()
 {
 	LLAppViewer::getTextureCache()->update(1); // unpauses the texture cache thread
@@ -1489,6 +1504,9 @@ bool idle_startup()
 		gAgentCamera.resetCamera();
 		display_startup();
 
+		// start up the ThreadPool we'll use for textures et al.
+		launchThreadPool();
+
 		// Initialize global class data needed for surfaces (i.e. textures)
 		LL_DEBUGS("AppInit") << "Initializing sky..." << LL_ENDL;
 		// Initialize all of the viewer object classes for the first time (doing things like texture fetches.