SL-16220: Add LL::ThreadPool class and a "General" instance.

ThreadPool bundles a WorkQueue with the specified number of worker threads to
service it. Each ThreadPool has a name that can be used to locate its
WorkQueue.

Each worker thread calls WorkQueue::runUntilClose().

ThreadPool listens on the "LLApp" LLEventPump for shutdown notification. On
receiving that, it closes its WorkQueue and then join()s each of its worker
threads for orderly shutdown.

Add a settings.xml entry "ThreadPoolSizes", the first LLSD-valued settings
entry to expect a map: pool name->size. The expectation is that usually code
instantiating a particular ThreadPool will have a default size in mind, but it
should check "ThreadPoolSizes" for a user override.

Make idle_startup()'s STATE_SEED_CAP_GRANTED state instantiate a "General"
ThreadPool. This is function-static for lazy initialization.

Eliminate LLMainLoopRepeater, which is completely unreferenced. Any potential
future use cases are better addressed by posting to the main loop's WorkQueue.

Eliminate llappviewer.cpp's private LLDeferredTaskList class, which
implemented LLAppViewer::addOnIdleCallback(). Make addOnIdleCallback() post
work to the main loop's WorkQueue instead.

6 files changed, 138 insertions, 26 deletions

diff --git a/indra/llcommon/CMakeLists.txt b/indra/llcommon/CMakeLists.txt
index fda43dd24c..c374f1135c 100644
--- a/indra/llcommon/CMakeLists.txt
+++ b/indra/llcommon/CMakeLists.txt
@@ -121,8 +121,8 @@ set(llcommon_SOURCE_FILES
     lluriparser.cpp
     lluuid.cpp
     llworkerthread.cpp
-    timing.cpp
     u64.cpp
+    threadpool.cpp
     workqueue.cpp
     StackWalker.cpp
     )
@@ -258,6 +258,7 @@ set(llcommon_HEADER_FILES
     lockstatic.h
     stdtypes.h
     stringize.h
+    threadpool.h
     threadsafeschedule.h
     timer.h
     tuple.h
diff --git a/indra/llcommon/threadpool.cpp b/indra/llcommon/threadpool.cpp
new file mode 100644
index 0000000000..aa7d4179a2
--- /dev/null
+++ b/indra/llcommon/threadpool.cpp
@@ -0,0 +1,75 @@
+/**
+ * @file   threadpool.cpp
+ * @author Nat Goodspeed
+ * @date   2021-10-21
+ * @brief  Implementation for threadpool.
+ * 
+ * $LicenseInfo:firstyear=2021&license=viewerlgpl$
+ * Copyright (c) 2021, Linden Research, Inc.
+ * $/LicenseInfo$
+ */
+
+// Precompiled header
+#include "linden_common.h"
+// associated header
+#include "threadpool.h"
+// STL headers
+// std headers
+// external library headers
+// other Linden headers
+#include "llerror.h"
+#include "llevents.h"
+#include "stringize.h"
+
+LL::ThreadPool::ThreadPool(const std::string& name, size_t threads):
+    mQueue(name),
+    mName("ThreadPool:" + name)
+{
+    for (size_t i = 0; i < threads; ++i)
+    {
+        std::string tname{ STRINGIZE(mName << ':' << (i+i) << '/' << threads) };
+        mThreads.emplace_back(tname, [this, tname](){ run(tname); });
+    }
+    // Listen on "LLApp", and when the app is shutting down, close the queue
+    // and join the workers.
+    LLEventPumps::instance().obtain("LLApp").listen(
+        mName,
+        [this](const LLSD& stat)
+        {
+            std::string status(stat["status"]);
+            if (status != "running")
+            {
+                // viewer is starting shutdown -- proclaim the end is nigh!
+                LL_DEBUGS("ThreadPool") << mName << " saw " << status << LL_ENDL;
+                close();
+            }
+            return false;
+        });
+}
+
+LL::ThreadPool::~ThreadPool()
+{
+    close();
+}
+
+void LL::ThreadPool::close()
+{
+    if (! mQueue.isClosed())
+    {
+        LL_DEBUGS("ThreadPool") << mName << " closing queue and joining threads" << LL_ENDL;
+        mQueue.close();
+        for (auto& pair: mThreads)
+        {
+            LL_DEBUGS("ThreadPool") << mName << " waiting on thread " << pair.first << LL_ENDL;
+            pair.second.join();
+        }
+        LL_DEBUGS("ThreadPool") << mName << " shutdown complete" << LL_ENDL;
+    }
+}
+
+void LL::ThreadPool::run(const std::string& name)
+{
+    LL_DEBUGS("ThreadPool") << name << " starting" << LL_ENDL;
+    mQueue.runUntilClose();
+    LL_DEBUGS("ThreadPool") << name << " stopping" << LL_ENDL;
+}
diff --git a/indra/llcommon/threadpool.h b/indra/llcommon/threadpool.h
new file mode 100644
index 0000000000..8f3c8514b5
--- /dev/null
+++ b/indra/llcommon/threadpool.h
@@ -0,0 +1,46 @@
+/**
+ * @file   threadpool.h
+ * @author Nat Goodspeed
+ * @date   2021-10-21
+ * @brief  ThreadPool configures a WorkQueue along with a pool of threads to
+ *         service it.
+ * 
+ * $LicenseInfo:firstyear=2021&license=viewerlgpl$
+ * Copyright (c) 2021, Linden Research, Inc.
+ * $/LicenseInfo$
+ */
+
+#if ! defined(LL_THREADPOOL_H)
+#define LL_THREADPOOL_H
+
+#include "workqueue.h"
+#include <string>
+#include <thread>
+#include <utility>                  // std::pair
+#include <vector>
+
+namespace LL
+{
+
+    class ThreadPool
+    {
+    public:
+        /**
+         * Pass ThreadPool a string name. This can be used to look up the
+         * relevant WorkQueue.
+         */
+        ThreadPool(const std::string& name, size_t threads=1);
+        ~ThreadPool();
+        void close();
+
+    private:
+        void run(const std::string& name);
+
+        WorkQueue mQueue;
+        std::string mName;
+        std::vector<std::pair<std::string, std::thread>> mThreads;
+    };
+
+} // namespace LL
+
+#endif /* ! defined(LL_THREADPOOL_H) */
diff --git a/indra/llcommon/timing.cpp b/indra/llcommon/timing.cpp
deleted file mode 100644
index c2dc695ef3..0000000000
--- a/indra/llcommon/timing.cpp
+++ /dev/null
@@ -1,25 +0,0 @@
-/** 
- * @file timing.cpp
- * @brief This file will be deprecated in the future.
- *
- * $LicenseInfo:firstyear=2000&license=viewerlgpl$
- * Second Life Viewer Source Code
- * Copyright (C) 2010, Linden Research, Inc.
- * 
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation;
- * version 2.1 of the License only.
- * 
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Lesser General Public License for more details.
- * 
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
- * 
- * Linden Research, Inc., 945 Battery Street, San Francisco, CA  94111  USA
- * $/LicenseInfo$
- */
diff --git a/indra/llcommon/workqueue.cpp b/indra/llcommon/workqueue.cpp
index ffc9a97dc0..114aeea1f3 100644
--- a/indra/llcommon/workqueue.cpp
+++ b/indra/llcommon/workqueue.cpp
@@ -38,6 +38,16 @@ void LL::WorkQueue::close()
     mQueue.close();
 }
 
+bool LL::WorkQueue::isClosed()
+{
+    return mQueue.isClosed();
+}
+
+bool LL::WorkQueue::done()
+{
+    return mQueue.done();
+}
+
 void LL::WorkQueue::runUntilClose()
 {
     try
diff --git a/indra/llcommon/workqueue.h b/indra/llcommon/workqueue.h
index b88aef989a..cfae2019dc 100644
--- a/indra/llcommon/workqueue.h
+++ b/indra/llcommon/workqueue.h
@@ -59,6 +59,11 @@ namespace LL
          */
         void close();
 
+        /// producer end: are we prevented from pushing any additional items?
+        bool isClosed();
+        /// consumer end: are we done, is the queue entirely drained?
+        bool done();
+
         /*---------------------- fire and forget API -----------------------*/
 
         /// fire-and-forget, but at a particular (future?) time