MAINT-7634: Logging and instrumentation canges to narrow down viewer crashes.

1 files changed, 256 insertions, 209 deletions

diff --git a/indra/llcommon/llthread.cpp b/indra/llcommon/llthread.cpp
index 52255bfaeb..b96b2ce4bc 100644
--- a/indra/llcommon/llthread.cpp
+++ b/indra/llcommon/llthread.cpp
@@ -34,6 +34,7 @@
 #include "lltimer.h"
 #include "lltrace.h"
 #include "lltracethreadrecorder.h"
+#include "llexception.h"
 
 #if LL_LINUX || LL_SOLARIS
 #include <sched.h>
@@ -46,28 +47,28 @@ const DWORD MS_VC_EXCEPTION=0x406D1388;
 #pragma pack(push,8)
 typedef struct tagTHREADNAME_INFO
 {
-	DWORD dwType; // Must be 0x1000.
-	LPCSTR szName; // Pointer to name (in user addr space).
-	DWORD dwThreadID; // Thread ID (-1=caller thread).
-	DWORD dwFlags; // Reserved for future use, must be zero.
+    DWORD dwType; // Must be 0x1000.
+    LPCSTR szName; // Pointer to name (in user addr space).
+    DWORD dwThreadID; // Thread ID (-1=caller thread).
+    DWORD dwFlags; // Reserved for future use, must be zero.
 } THREADNAME_INFO;
 #pragma pack(pop)
 
 void set_thread_name( DWORD dwThreadID, const char* threadName)
 {
-	THREADNAME_INFO info;
-	info.dwType = 0x1000;
-	info.szName = threadName;
-	info.dwThreadID = dwThreadID;
-	info.dwFlags = 0;
-
-	__try
-	{
-		::RaiseException( MS_VC_EXCEPTION, 0, sizeof(info)/sizeof(DWORD), (ULONG_PTR*)&info );
-	}
-	__except(EXCEPTION_CONTINUE_EXECUTION)
-	{
-	}
+    THREADNAME_INFO info;
+    info.dwType = 0x1000;
+    info.szName = threadName;
+    info.dwThreadID = dwThreadID;
+    info.dwFlags = 0;
+
+    __try
+    {
+        ::RaiseException( MS_VC_EXCEPTION, 0, sizeof(info)/sizeof(DWORD), (ULONG_PTR*)&info );
+    }
+    __except(EXCEPTION_CONTINUE_EXECUTION)
+    {
+    }
 }
 #endif
 
@@ -99,17 +100,17 @@ U32 LLThread::sIDIter = 0;
 
 LL_COMMON_API void assert_main_thread()
 {
-	static U32 s_thread_id = LLThread::currentID();
-	if (LLThread::currentID() != s_thread_id)
-	{
-		LL_WARNS() << "Illegal execution from thread id " << (S32) LLThread::currentID()
-			<< " outside main thread " << (S32) s_thread_id << LL_ENDL;
-	}
+    static U32 s_thread_id = LLThread::currentID();
+    if (LLThread::currentID() != s_thread_id)
+    {
+        LL_WARNS() << "Illegal execution from thread id " << (S32) LLThread::currentID()
+            << " outside main thread " << (S32) s_thread_id << LL_ENDL;
+    }
 }
 
 void LLThread::registerThreadID()
 {
-	sThreadID = ++sIDIter;
+    sThreadID = ++sIDIter;
 }
 
 //
@@ -117,157 +118,203 @@ void LLThread::registerThreadID()
 //
 void *APR_THREAD_FUNC LLThread::staticRun(apr_thread_t *apr_threadp, void *datap)
 {
-	LLThread *threadp = (LLThread *)datap;
+    LLThread *threadp = (LLThread *)datap;
 
 #ifdef LL_WINDOWS
-	set_thread_name(-1, threadp->mName.c_str());
+    set_thread_name(-1, threadp->mName.c_str());
 #endif
 
-	// for now, hard code all LLThreads to report to single master thread recorder, which is known to be running on main thread
-	threadp->mRecorder = new LLTrace::ThreadRecorder(*LLTrace::get_master_thread_recorder());
-
-	sThreadID = threadp->mID;
-
-	// Run the user supplied function
-	threadp->run();
-
-	//LL_INFOS() << "LLThread::staticRun() Exiting: " << threadp->mName << LL_ENDL;
-	
-	delete threadp->mRecorder;
-	threadp->mRecorder = NULL;
-	
-	// We're done with the run function, this thread is done executing now.
-	//NB: we are using this flag to sync across threads...we really need memory barriers here
-	threadp->mStatus = STOPPED;
-
-	return NULL;
+    // for now, hard code all LLThreads to report to single master thread recorder, which is known to be running on main thread
+    threadp->mRecorder = new LLTrace::ThreadRecorder(*LLTrace::get_master_thread_recorder());
+
+    sThreadID = threadp->mID;
+
+    try
+    {
+        // Run the user supplied function
+        do 
+        {
+            try
+            {
+                threadp->run();
+            }
+            catch (const LLContinueError &e)
+            {
+                LL_WARNS("THREAD") << "ContinueException on thread '" << threadp->mName <<
+                    "' reentering run(). Error what is: '" << e.what() << "'" << LL_ENDL;
+                //output possible call stacks to log file.
+                LLError::LLCallStacks::print();
+
+                LOG_UNHANDLED_EXCEPTION("LLThread");
+                continue;
+            }
+            break;
+
+        } while (true);
+
+        //LL_INFOS() << "LLThread::staticRun() Exiting: " << threadp->mName << LL_ENDL;
+
+        // We're done with the run function, this thread is done executing now.
+        //NB: we are using this flag to sync across threads...we really need memory barriers here
+        threadp->mStatus = STOPPED;
+    }
+    catch (std::bad_alloc)
+    {
+        threadp->mStatus = CRASHED;
+        LLMemory::logMemoryInfo(TRUE);
+
+        //output possible call stacks to log file.
+        LLError::LLCallStacks::print();
+
+        LL_ERRS("THREAD") << "Bad memory allocation in LLThread::staticRun() named '" << threadp->mName << "'!" << LL_ENDL;
+    }
+    catch (...)
+    {
+        threadp->mStatus = CRASHED;
+        CRASH_ON_UNHANDLED_EXCEPTION("LLThread");
+    }
+
+    delete threadp->mRecorder;
+    threadp->mRecorder = NULL;
+
+    return NULL;
 }
 
 LLThread::LLThread(const std::string& name, apr_pool_t *poolp) :
-	mPaused(FALSE),
-	mName(name),
-	mAPRThreadp(NULL),
-	mStatus(STOPPED),
-	mRecorder(NULL)
+    mPaused(FALSE),
+    mName(name),
+    mAPRThreadp(NULL),
+    mStatus(STOPPED),
+    mRecorder(NULL)
 {
 
-	mID = ++sIDIter;
-
-	// Thread creation probably CAN be paranoid about APR being initialized, if necessary
-	if (poolp)
-	{
-		mIsLocalPool = FALSE;
-		mAPRPoolp = poolp;
-	}
-	else
-	{
-		mIsLocalPool = TRUE;
-		apr_pool_create(&mAPRPoolp, NULL); // Create a subpool for this thread
-	}
-	mRunCondition = new LLCondition(mAPRPoolp);
-	mDataLock = new LLMutex(mAPRPoolp);
-	mLocalAPRFilePoolp = NULL ;
+    mID = ++sIDIter;
+
+    // Thread creation probably CAN be paranoid about APR being initialized, if necessary
+    if (poolp)
+    {
+        mIsLocalPool = FALSE;
+        mAPRPoolp = poolp;
+    }
+    else
+    {
+        mIsLocalPool = TRUE;
+        apr_pool_create(&mAPRPoolp, NULL); // Create a subpool for this thread
+    }
+    mRunCondition = new LLCondition(mAPRPoolp);
+    mDataLock = new LLMutex(mAPRPoolp);
+    mLocalAPRFilePoolp = NULL ;
 }
 
 
 LLThread::~LLThread()
 {
-	shutdown();
-
-	if(mLocalAPRFilePoolp)
-	{
-		delete mLocalAPRFilePoolp ;
-		mLocalAPRFilePoolp = NULL ;
-	}
+    shutdown();
+
+    if (isCrashed())
+    {
+        LL_WARNS("THREAD") << "Destroying crashed thread named '" << mName << "'" << LL_ENDL;
+    }
+
+    if(mLocalAPRFilePoolp)
+    {
+        delete mLocalAPRFilePoolp ;
+        mLocalAPRFilePoolp = NULL ;
+    }
 }
 
 void LLThread::shutdown()
 {
-	// Warning!  If you somehow call the thread destructor from itself,
-	// the thread will die in an unclean fashion!
-	if (mAPRThreadp)
-	{
-		if (!isStopped())
-		{
-			// The thread isn't already stopped
-			// First, set the flag that indicates that we're ready to die
-			setQuitting();
-
-			//LL_INFOS() << "LLThread::~LLThread() Killing thread " << mName << " Status: " << mStatus << LL_ENDL;
-			// Now wait a bit for the thread to exit
-			// It's unclear whether I should even bother doing this - this destructor
-			// should never get called unless we're already stopped, really...
-			S32 counter = 0;
-			const S32 MAX_WAIT = 600;
-			while (counter < MAX_WAIT)
-			{
-				if (isStopped())
-				{
-					break;
-				}
-				// Sleep for a tenth of a second
-				ms_sleep(100);
-				yield();
-				counter++;
-			}
-		}
-
-		if (!isStopped())
-		{
-			// This thread just wouldn't stop, even though we gave it time
-			//LL_WARNS() << "LLThread::~LLThread() exiting thread before clean exit!" << LL_ENDL;
-			// Put a stake in its heart.
-			delete mRecorder;
-
-			apr_thread_exit(mAPRThreadp, -1);
-			return;
-		}
-		mAPRThreadp = NULL;
-	}
-
-	delete mRunCondition;
-	mRunCondition = NULL;
-
-	delete mDataLock;
-	mDataLock = NULL;
-	
-	if (mIsLocalPool && mAPRPoolp)
-	{
-		apr_pool_destroy(mAPRPoolp);
-		mAPRPoolp = 0;
-	}
-
-	if (mRecorder)
-	{
-		// missed chance to properly shut down recorder (needs to be done in thread context)
-		// probably due to abnormal thread termination
-		// so just leak it and remove it from parent
-		LLTrace::get_master_thread_recorder()->removeChildRecorder(mRecorder);
-	}
+    if (isCrashed())
+    {
+        LL_WARNS("THREAD") << "Shutting down crashed thread named '" << mName << "'" << LL_ENDL;
+    }
+
+    // Warning!  If you somehow call the thread destructor from itself,
+    // the thread will die in an unclean fashion!
+    if (mAPRThreadp)
+    {
+        if (!isStopped())
+        {
+            // The thread isn't already stopped
+            // First, set the flag that indicates that we're ready to die
+            setQuitting();
+
+            //LL_INFOS() << "LLThread::~LLThread() Killing thread " << mName << " Status: " << mStatus << LL_ENDL;
+            // Now wait a bit for the thread to exit
+            // It's unclear whether I should even bother doing this - this destructor
+            // should never get called unless we're already stopped, really...
+            S32 counter = 0;
+            const S32 MAX_WAIT = 600;
+            while (counter < MAX_WAIT)
+            {
+                if (isStopped())
+                {
+                    break;
+                }
+                // Sleep for a tenth of a second
+                ms_sleep(100);
+                yield();
+                counter++;
+            }
+        }
+
+        if (!isStopped())
+        {
+            // This thread just wouldn't stop, even though we gave it time
+            //LL_WARNS() << "LLThread::~LLThread() exiting thread before clean exit!" << LL_ENDL;
+            // Put a stake in its heart.
+            delete mRecorder;
+
+            apr_thread_exit(mAPRThreadp, -1);
+            return;
+        }
+        mAPRThreadp = NULL;
+    }
+
+    delete mRunCondition;
+    mRunCondition = NULL;
+
+    delete mDataLock;
+    mDataLock = NULL;
+    
+    if (mIsLocalPool && mAPRPoolp)
+    {
+        apr_pool_destroy(mAPRPoolp);
+        mAPRPoolp = 0;
+    }
+
+    if (mRecorder)
+    {
+        // missed chance to properly shut down recorder (needs to be done in thread context)
+        // probably due to abnormal thread termination
+        // so just leak it and remove it from parent
+        LLTrace::get_master_thread_recorder()->removeChildRecorder(mRecorder);
+    }
 }
 
 
 void LLThread::start()
 {
-	llassert(isStopped());
-	
-	// Set thread state to running
-	mStatus = RUNNING;
-
-	apr_status_t status =
-		apr_thread_create(&mAPRThreadp, NULL, staticRun, (void *)this, mAPRPoolp);
-	
-	if(status == APR_SUCCESS)
-	{	
-		// We won't bother joining
-		apr_thread_detach(mAPRThreadp);
-	}
-	else
-	{
-		mStatus = STOPPED;
-		LL_WARNS() << "failed to start thread " << mName << LL_ENDL;
-		ll_apr_warn_status(status);
-	}
+    llassert(isStopped());
+    
+    // Set thread state to running
+    mStatus = RUNNING;
+
+    apr_status_t status =
+        apr_thread_create(&mAPRThreadp, NULL, staticRun, (void *)this, mAPRPoolp);
+    
+    if(status == APR_SUCCESS)
+    {   
+        // We won't bother joining
+        apr_thread_detach(mAPRThreadp);
+    }
+    else
+    {
+        mStatus = STOPPED;
+        LL_WARNS() << "failed to start thread " << mName << LL_ENDL;
+        ll_apr_warn_status(status);
+    }
 
 }
 
@@ -278,28 +325,28 @@ void LLThread::start()
 // The thread will pause when (and if) it calls checkPause()
 void LLThread::pause()
 {
-	if (!mPaused)
-	{
-		// this will cause the thread to stop execution as soon as checkPause() is called
-		mPaused = 1;		// Does not need to be atomic since this is only set/unset from the main thread
-	}	
+    if (!mPaused)
+    {
+        // this will cause the thread to stop execution as soon as checkPause() is called
+        mPaused = 1;        // Does not need to be atomic since this is only set/unset from the main thread
+    }   
 }
 
 void LLThread::unpause()
 {
-	if (mPaused)
-	{
-		mPaused = 0;
-	}
+    if (mPaused)
+    {
+        mPaused = 0;
+    }
 
-	wake(); // wake up the thread if necessary
+    wake(); // wake up the thread if necessary
 }
 
 // virtual predicate function -- returns true if the thread should wake up, false if it should sleep.
 bool LLThread::runCondition(void)
 {
-	// by default, always run.  Handling of pause/unpause is done regardless of this function's result.
-	return true;
+    // by default, always run.  Handling of pause/unpause is done regardless of this function's result.
+    return true;
 }
 
 //============================================================================
@@ -307,65 +354,65 @@ bool LLThread::runCondition(void)
 // Stop thread execution if requested until unpaused.
 void LLThread::checkPause()
 {
-	mDataLock->lock();
-
-	// This is in a while loop because the pthread API allows for spurious wakeups.
-	while(shouldSleep())
-	{
-		mDataLock->unlock();
-		mRunCondition->wait(); // unlocks mRunCondition
-		mDataLock->lock();
-		// mRunCondition is locked when the thread wakes up
-	}
-	
- 	mDataLock->unlock();
+    mDataLock->lock();
+
+    // This is in a while loop because the pthread API allows for spurious wakeups.
+    while(shouldSleep())
+    {
+        mDataLock->unlock();
+        mRunCondition->wait(); // unlocks mRunCondition
+        mDataLock->lock();
+        // mRunCondition is locked when the thread wakes up
+    }
+    
+    mDataLock->unlock();
 }
 
 //============================================================================
 
 void LLThread::setQuitting()
 {
-	mDataLock->lock();
-	if (mStatus == RUNNING)
-	{
-		mStatus = QUITTING;
-	}
-	mDataLock->unlock();
-	wake();
+    mDataLock->lock();
+    if (mStatus == RUNNING)
+    {
+        mStatus = QUITTING;
+    }
+    mDataLock->unlock();
+    wake();
 }
 
 // static
 U32 LLThread::currentID()
 {
-	return sThreadID;
+    return sThreadID;
 }
 
 // static
 void LLThread::yield()
 {
 #if LL_LINUX || LL_SOLARIS
-	sched_yield(); // annoyingly, apr_thread_yield  is a noop on linux...
+    sched_yield(); // annoyingly, apr_thread_yield  is a noop on linux...
 #else
-	apr_thread_yield();
+    apr_thread_yield();
 #endif
 }
 
 void LLThread::wake()
 {
-	mDataLock->lock();
-	if(!shouldSleep())
-	{
-		mRunCondition->signal();
-	}
-	mDataLock->unlock();
+    mDataLock->lock();
+    if(!shouldSleep())
+    {
+        mRunCondition->signal();
+    }
+    mDataLock->unlock();
 }
 
 void LLThread::wakeLocked()
 {
-	if(!shouldSleep())
-	{
-		mRunCondition->signal();
-	}
+    if(!shouldSleep())
+    {
+        mRunCondition->signal();
+    }
 }
 
 //============================================================================
@@ -378,38 +425,38 @@ LLMutex* LLThreadSafeRefCount::sMutex = 0;
 //static
 void LLThreadSafeRefCount::initThreadSafeRefCount()
 {
-	if (!sMutex)
-	{
-		sMutex = new LLMutex(0);
-	}
+    if (!sMutex)
+    {
+        sMutex = new LLMutex(0);
+    }
 }
 
 //static
 void LLThreadSafeRefCount::cleanupThreadSafeRefCount()
 {
-	delete sMutex;
-	sMutex = NULL;
+    delete sMutex;
+    sMutex = NULL;
 }
-	
+    
 
 //----------------------------------------------------------------------------
 
 LLThreadSafeRefCount::LLThreadSafeRefCount() :
-	mRef(0)
+    mRef(0)
 {
 }
 
 LLThreadSafeRefCount::LLThreadSafeRefCount(const LLThreadSafeRefCount& src)
 {
-	mRef = 0;
+    mRef = 0;
 }
 
 LLThreadSafeRefCount::~LLThreadSafeRefCount()
 { 
-	if (mRef != 0)
-	{
-		LL_ERRS() << "deleting non-zero reference" << LL_ENDL;
-	}
+    if (mRef != 0)
+    {
+        LL_ERRS() << "deleting non-zero reference" << LL_ENDL;
+    }
 }
 
 //============================================================================