SL-10190: Introduce LLCoros::saveException() and rethrow().

This mechanism uses a queue of std::exception_ptrs to transport an (otherwise)
uncaught exception from a terminated coroutine to the thread's main fiber. The
main loop calls LLCoros::rethrow() just after giving some cycles to ready
coroutines that frame.

# Conflicts:
#	indra/llcommon/llcoros.cpp
#	indra/llcommon/llcoros.h
#	indra/newview/llappviewer.cpp

3 files changed, 53 insertions, 3 deletions

diff --git a/indra/llcommon/llcoros.cpp b/indra/llcommon/llcoros.cpp
index 6a534951ff..a182d305e8 100644
--- a/indra/llcommon/llcoros.cpp
+++ b/indra/llcommon/llcoros.cpp
@@ -35,6 +35,7 @@
 // STL headers
 // std headers
 #include <atomic>
+#include <stdexcept>
 // external library headers
 #include <boost/bind.hpp>
 #include <boost/fiber/fiber.hpp>
@@ -214,6 +215,22 @@ std::string LLCoros::logname()
     return data.mName.empty()? data.getKey() : data.mName;
 }
 
+void LLCoros::saveException(const std::string& name, std::exception_ptr exc)
+{
+    mExceptionQueue.emplace(name, exc);
+}
+
+void LLCoros::rethrow()
+{
+    if (! mExceptionQueue.empty())
+    {
+        ExceptionData front = mExceptionQueue.front();
+        mExceptionQueue.pop();
+        LL_WARNS("LLCoros") << "Rethrowing exception from coroutine " << front.name << LL_ENDL;
+        std::rethrow_exception(front.exception);
+    }
+}
+
 void LLCoros::setStackSize(S32 stacksize)
 {
     LL_DEBUGS("LLCoros") << "Setting coroutine stack size to " << stacksize << LL_ENDL;
@@ -330,9 +347,11 @@ void LLCoros::toplevel(std::string name, callable_t callable)
     }
     catch (...)
     {
-        // Any OTHER kind of uncaught exception will cause the viewer to
-        // crash, hopefully informatively.
-        CRASH_ON_UNHANDLED_EXCEPTION(STRINGIZE("coroutine " << name));
+        // Stash any OTHER kind of uncaught exception in the rethrow() queue
+        // to be rethrown by the main fiber.
+        LL_WARNS("LLCoros") << "Capturing uncaught exception in coroutine "
+                            << name << LL_ENDL;
+        LLCoros::instance().saveException(name, std::current_exception());
     }
 }
 
diff --git a/indra/llcommon/llcoros.h b/indra/llcommon/llcoros.h
index 51f7380def..59b2b91f96 100644
--- a/indra/llcommon/llcoros.h
+++ b/indra/llcommon/llcoros.h
@@ -38,6 +38,8 @@
 #include "llinstancetracker.h"
 #include <boost/function.hpp>
 #include <string>
+#include <exception>
+#include <queue>
 
 // e.g. #include LLCOROS_MUTEX_HEADER
 #define LLCOROS_MUTEX_HEADER   <boost/fiber/mutex.hpp>
@@ -156,6 +158,19 @@ public:
      * LLCoros::launch()).
      */
     static std::string getName();
+    
+    /**
+     * rethrow() is called by the thread's main fiber to propagate an
+     * exception from any coroutine into the main fiber, where it can engage
+     * the normal unhandled-exception machinery, up to and including crash
+     * reporting.
+     *
+     * LLCoros maintains a queue of otherwise-uncaught exceptions from
+     * terminated coroutines. Each call to rethrow() pops the first of those
+     * and rethrows it. When the queue is empty (normal case), rethrow() is a
+     * no-op.
+     */
+    void rethrow();
 
     /**
      * This variation returns a name suitable for log messages: the explicit
@@ -298,6 +313,20 @@ private:
     static void winlevel(const callable_t& callable);
 #endif
     static CoroData& get_CoroData(const std::string& caller);
+    void saveException(const std::string& name, std::exception_ptr exc);
+
+    struct ExceptionData
+    {
+        ExceptionData(const std::string& nm, std::exception_ptr exc):
+            name(nm),
+            exception(exc)
+        {}
+        // name of coroutine that originally threw this exception
+        std::string name;
+        // the thrown exception
+        std::exception_ptr exception;
+    };
+    std::queue<ExceptionData> mExceptionQueue;
 
     S32 mStackSize;
 
diff --git a/indra/newview/llappviewer.cpp b/indra/newview/llappviewer.cpp
index d17498a6ed..0d80ab543e 100644
--- a/indra/newview/llappviewer.cpp
+++ b/indra/newview/llappviewer.cpp
@@ -1471,6 +1471,8 @@ bool LLAppViewer::doFrame()
 		mainloop.post(newFrame);
 		// give listeners a chance to run
 		llcoro::suspend();
+		// if one of our coroutines threw an uncaught exception, rethrow it now
+		LLCoros::instance().rethrow();
 
 		if (!LLApp::isExiting())
 		{