From 1038633526330cf931ba097dbafdd270b5bb56e3 Mon Sep 17 00:00:00 2001 From: Rider Linden Date: Tue, 8 Aug 2017 09:04:32 -0700 Subject: MAINT-7634: Logging and instrumentation canges to narrow down viewer crashes. --- indra/llcommon/llthread.cpp | 465 ++++++++++++++++++++++++-------------------- 1 file changed, 256 insertions(+), 209 deletions(-) (limited to 'indra/llcommon/llthread.cpp') diff --git a/indra/llcommon/llthread.cpp b/indra/llcommon/llthread.cpp index 52255bfaeb..b96b2ce4bc 100644 --- a/indra/llcommon/llthread.cpp +++ b/indra/llcommon/llthread.cpp @@ -34,6 +34,7 @@ #include "lltimer.h" #include "lltrace.h" #include "lltracethreadrecorder.h" +#include "llexception.h" #if LL_LINUX || LL_SOLARIS #include @@ -46,28 +47,28 @@ const DWORD MS_VC_EXCEPTION=0x406D1388; #pragma pack(push,8) typedef struct tagTHREADNAME_INFO { - DWORD dwType; // Must be 0x1000. - LPCSTR szName; // Pointer to name (in user addr space). - DWORD dwThreadID; // Thread ID (-1=caller thread). - DWORD dwFlags; // Reserved for future use, must be zero. + DWORD dwType; // Must be 0x1000. + LPCSTR szName; // Pointer to name (in user addr space). + DWORD dwThreadID; // Thread ID (-1=caller thread). + DWORD dwFlags; // Reserved for future use, must be zero. } THREADNAME_INFO; #pragma pack(pop) void set_thread_name( DWORD dwThreadID, const char* threadName) { - THREADNAME_INFO info; - info.dwType = 0x1000; - info.szName = threadName; - info.dwThreadID = dwThreadID; - info.dwFlags = 0; - - __try - { - ::RaiseException( MS_VC_EXCEPTION, 0, sizeof(info)/sizeof(DWORD), (ULONG_PTR*)&info ); - } - __except(EXCEPTION_CONTINUE_EXECUTION) - { - } + THREADNAME_INFO info; + info.dwType = 0x1000; + info.szName = threadName; + info.dwThreadID = dwThreadID; + info.dwFlags = 0; + + __try + { + ::RaiseException( MS_VC_EXCEPTION, 0, sizeof(info)/sizeof(DWORD), (ULONG_PTR*)&info ); + } + __except(EXCEPTION_CONTINUE_EXECUTION) + { + } } #endif @@ -99,17 +100,17 @@ U32 LLThread::sIDIter = 0; LL_COMMON_API void assert_main_thread() { - static U32 s_thread_id = LLThread::currentID(); - if (LLThread::currentID() != s_thread_id) - { - LL_WARNS() << "Illegal execution from thread id " << (S32) LLThread::currentID() - << " outside main thread " << (S32) s_thread_id << LL_ENDL; - } + static U32 s_thread_id = LLThread::currentID(); + if (LLThread::currentID() != s_thread_id) + { + LL_WARNS() << "Illegal execution from thread id " << (S32) LLThread::currentID() + << " outside main thread " << (S32) s_thread_id << LL_ENDL; + } } void LLThread::registerThreadID() { - sThreadID = ++sIDIter; + sThreadID = ++sIDIter; } // @@ -117,157 +118,203 @@ void LLThread::registerThreadID() // void *APR_THREAD_FUNC LLThread::staticRun(apr_thread_t *apr_threadp, void *datap) { - LLThread *threadp = (LLThread *)datap; + LLThread *threadp = (LLThread *)datap; #ifdef LL_WINDOWS - set_thread_name(-1, threadp->mName.c_str()); + set_thread_name(-1, threadp->mName.c_str()); #endif - // for now, hard code all LLThreads to report to single master thread recorder, which is known to be running on main thread - threadp->mRecorder = new LLTrace::ThreadRecorder(*LLTrace::get_master_thread_recorder()); - - sThreadID = threadp->mID; - - // Run the user supplied function - threadp->run(); - - //LL_INFOS() << "LLThread::staticRun() Exiting: " << threadp->mName << LL_ENDL; - - delete threadp->mRecorder; - threadp->mRecorder = NULL; - - // We're done with the run function, this thread is done executing now. - //NB: we are using this flag to sync across threads...we really need memory barriers here - threadp->mStatus = STOPPED; - - return NULL; + // for now, hard code all LLThreads to report to single master thread recorder, which is known to be running on main thread + threadp->mRecorder = new LLTrace::ThreadRecorder(*LLTrace::get_master_thread_recorder()); + + sThreadID = threadp->mID; + + try + { + // Run the user supplied function + do + { + try + { + threadp->run(); + } + catch (const LLContinueError &e) + { + LL_WARNS("THREAD") << "ContinueException on thread '" << threadp->mName << + "' reentering run(). Error what is: '" << e.what() << "'" << LL_ENDL; + //output possible call stacks to log file. + LLError::LLCallStacks::print(); + + LOG_UNHANDLED_EXCEPTION("LLThread"); + continue; + } + break; + + } while (true); + + //LL_INFOS() << "LLThread::staticRun() Exiting: " << threadp->mName << LL_ENDL; + + // We're done with the run function, this thread is done executing now. + //NB: we are using this flag to sync across threads...we really need memory barriers here + threadp->mStatus = STOPPED; + } + catch (std::bad_alloc) + { + threadp->mStatus = CRASHED; + LLMemory::logMemoryInfo(TRUE); + + //output possible call stacks to log file. + LLError::LLCallStacks::print(); + + LL_ERRS("THREAD") << "Bad memory allocation in LLThread::staticRun() named '" << threadp->mName << "'!" << LL_ENDL; + } + catch (...) + { + threadp->mStatus = CRASHED; + CRASH_ON_UNHANDLED_EXCEPTION("LLThread"); + } + + delete threadp->mRecorder; + threadp->mRecorder = NULL; + + return NULL; } LLThread::LLThread(const std::string& name, apr_pool_t *poolp) : - mPaused(FALSE), - mName(name), - mAPRThreadp(NULL), - mStatus(STOPPED), - mRecorder(NULL) + mPaused(FALSE), + mName(name), + mAPRThreadp(NULL), + mStatus(STOPPED), + mRecorder(NULL) { - mID = ++sIDIter; - - // Thread creation probably CAN be paranoid about APR being initialized, if necessary - if (poolp) - { - mIsLocalPool = FALSE; - mAPRPoolp = poolp; - } - else - { - mIsLocalPool = TRUE; - apr_pool_create(&mAPRPoolp, NULL); // Create a subpool for this thread - } - mRunCondition = new LLCondition(mAPRPoolp); - mDataLock = new LLMutex(mAPRPoolp); - mLocalAPRFilePoolp = NULL ; + mID = ++sIDIter; + + // Thread creation probably CAN be paranoid about APR being initialized, if necessary + if (poolp) + { + mIsLocalPool = FALSE; + mAPRPoolp = poolp; + } + else + { + mIsLocalPool = TRUE; + apr_pool_create(&mAPRPoolp, NULL); // Create a subpool for this thread + } + mRunCondition = new LLCondition(mAPRPoolp); + mDataLock = new LLMutex(mAPRPoolp); + mLocalAPRFilePoolp = NULL ; } LLThread::~LLThread() { - shutdown(); - - if(mLocalAPRFilePoolp) - { - delete mLocalAPRFilePoolp ; - mLocalAPRFilePoolp = NULL ; - } + shutdown(); + + if (isCrashed()) + { + LL_WARNS("THREAD") << "Destroying crashed thread named '" << mName << "'" << LL_ENDL; + } + + if(mLocalAPRFilePoolp) + { + delete mLocalAPRFilePoolp ; + mLocalAPRFilePoolp = NULL ; + } } void LLThread::shutdown() { - // Warning! If you somehow call the thread destructor from itself, - // the thread will die in an unclean fashion! - if (mAPRThreadp) - { - if (!isStopped()) - { - // The thread isn't already stopped - // First, set the flag that indicates that we're ready to die - setQuitting(); - - //LL_INFOS() << "LLThread::~LLThread() Killing thread " << mName << " Status: " << mStatus << LL_ENDL; - // Now wait a bit for the thread to exit - // It's unclear whether I should even bother doing this - this destructor - // should never get called unless we're already stopped, really... - S32 counter = 0; - const S32 MAX_WAIT = 600; - while (counter < MAX_WAIT) - { - if (isStopped()) - { - break; - } - // Sleep for a tenth of a second - ms_sleep(100); - yield(); - counter++; - } - } - - if (!isStopped()) - { - // This thread just wouldn't stop, even though we gave it time - //LL_WARNS() << "LLThread::~LLThread() exiting thread before clean exit!" << LL_ENDL; - // Put a stake in its heart. - delete mRecorder; - - apr_thread_exit(mAPRThreadp, -1); - return; - } - mAPRThreadp = NULL; - } - - delete mRunCondition; - mRunCondition = NULL; - - delete mDataLock; - mDataLock = NULL; - - if (mIsLocalPool && mAPRPoolp) - { - apr_pool_destroy(mAPRPoolp); - mAPRPoolp = 0; - } - - if (mRecorder) - { - // missed chance to properly shut down recorder (needs to be done in thread context) - // probably due to abnormal thread termination - // so just leak it and remove it from parent - LLTrace::get_master_thread_recorder()->removeChildRecorder(mRecorder); - } + if (isCrashed()) + { + LL_WARNS("THREAD") << "Shutting down crashed thread named '" << mName << "'" << LL_ENDL; + } + + // Warning! If you somehow call the thread destructor from itself, + // the thread will die in an unclean fashion! + if (mAPRThreadp) + { + if (!isStopped()) + { + // The thread isn't already stopped + // First, set the flag that indicates that we're ready to die + setQuitting(); + + //LL_INFOS() << "LLThread::~LLThread() Killing thread " << mName << " Status: " << mStatus << LL_ENDL; + // Now wait a bit for the thread to exit + // It's unclear whether I should even bother doing this - this destructor + // should never get called unless we're already stopped, really... + S32 counter = 0; + const S32 MAX_WAIT = 600; + while (counter < MAX_WAIT) + { + if (isStopped()) + { + break; + } + // Sleep for a tenth of a second + ms_sleep(100); + yield(); + counter++; + } + } + + if (!isStopped()) + { + // This thread just wouldn't stop, even though we gave it time + //LL_WARNS() << "LLThread::~LLThread() exiting thread before clean exit!" << LL_ENDL; + // Put a stake in its heart. + delete mRecorder; + + apr_thread_exit(mAPRThreadp, -1); + return; + } + mAPRThreadp = NULL; + } + + delete mRunCondition; + mRunCondition = NULL; + + delete mDataLock; + mDataLock = NULL; + + if (mIsLocalPool && mAPRPoolp) + { + apr_pool_destroy(mAPRPoolp); + mAPRPoolp = 0; + } + + if (mRecorder) + { + // missed chance to properly shut down recorder (needs to be done in thread context) + // probably due to abnormal thread termination + // so just leak it and remove it from parent + LLTrace::get_master_thread_recorder()->removeChildRecorder(mRecorder); + } } void LLThread::start() { - llassert(isStopped()); - - // Set thread state to running - mStatus = RUNNING; - - apr_status_t status = - apr_thread_create(&mAPRThreadp, NULL, staticRun, (void *)this, mAPRPoolp); - - if(status == APR_SUCCESS) - { - // We won't bother joining - apr_thread_detach(mAPRThreadp); - } - else - { - mStatus = STOPPED; - LL_WARNS() << "failed to start thread " << mName << LL_ENDL; - ll_apr_warn_status(status); - } + llassert(isStopped()); + + // Set thread state to running + mStatus = RUNNING; + + apr_status_t status = + apr_thread_create(&mAPRThreadp, NULL, staticRun, (void *)this, mAPRPoolp); + + if(status == APR_SUCCESS) + { + // We won't bother joining + apr_thread_detach(mAPRThreadp); + } + else + { + mStatus = STOPPED; + LL_WARNS() << "failed to start thread " << mName << LL_ENDL; + ll_apr_warn_status(status); + } } @@ -278,28 +325,28 @@ void LLThread::start() // The thread will pause when (and if) it calls checkPause() void LLThread::pause() { - if (!mPaused) - { - // this will cause the thread to stop execution as soon as checkPause() is called - mPaused = 1; // Does not need to be atomic since this is only set/unset from the main thread - } + if (!mPaused) + { + // this will cause the thread to stop execution as soon as checkPause() is called + mPaused = 1; // Does not need to be atomic since this is only set/unset from the main thread + } } void LLThread::unpause() { - if (mPaused) - { - mPaused = 0; - } + if (mPaused) + { + mPaused = 0; + } - wake(); // wake up the thread if necessary + wake(); // wake up the thread if necessary } // virtual predicate function -- returns true if the thread should wake up, false if it should sleep. bool LLThread::runCondition(void) { - // by default, always run. Handling of pause/unpause is done regardless of this function's result. - return true; + // by default, always run. Handling of pause/unpause is done regardless of this function's result. + return true; } //============================================================================ @@ -307,65 +354,65 @@ bool LLThread::runCondition(void) // Stop thread execution if requested until unpaused. void LLThread::checkPause() { - mDataLock->lock(); - - // This is in a while loop because the pthread API allows for spurious wakeups. - while(shouldSleep()) - { - mDataLock->unlock(); - mRunCondition->wait(); // unlocks mRunCondition - mDataLock->lock(); - // mRunCondition is locked when the thread wakes up - } - - mDataLock->unlock(); + mDataLock->lock(); + + // This is in a while loop because the pthread API allows for spurious wakeups. + while(shouldSleep()) + { + mDataLock->unlock(); + mRunCondition->wait(); // unlocks mRunCondition + mDataLock->lock(); + // mRunCondition is locked when the thread wakes up + } + + mDataLock->unlock(); } //============================================================================ void LLThread::setQuitting() { - mDataLock->lock(); - if (mStatus == RUNNING) - { - mStatus = QUITTING; - } - mDataLock->unlock(); - wake(); + mDataLock->lock(); + if (mStatus == RUNNING) + { + mStatus = QUITTING; + } + mDataLock->unlock(); + wake(); } // static U32 LLThread::currentID() { - return sThreadID; + return sThreadID; } // static void LLThread::yield() { #if LL_LINUX || LL_SOLARIS - sched_yield(); // annoyingly, apr_thread_yield is a noop on linux... + sched_yield(); // annoyingly, apr_thread_yield is a noop on linux... #else - apr_thread_yield(); + apr_thread_yield(); #endif } void LLThread::wake() { - mDataLock->lock(); - if(!shouldSleep()) - { - mRunCondition->signal(); - } - mDataLock->unlock(); + mDataLock->lock(); + if(!shouldSleep()) + { + mRunCondition->signal(); + } + mDataLock->unlock(); } void LLThread::wakeLocked() { - if(!shouldSleep()) - { - mRunCondition->signal(); - } + if(!shouldSleep()) + { + mRunCondition->signal(); + } } //============================================================================ @@ -378,38 +425,38 @@ LLMutex* LLThreadSafeRefCount::sMutex = 0; //static void LLThreadSafeRefCount::initThreadSafeRefCount() { - if (!sMutex) - { - sMutex = new LLMutex(0); - } + if (!sMutex) + { + sMutex = new LLMutex(0); + } } //static void LLThreadSafeRefCount::cleanupThreadSafeRefCount() { - delete sMutex; - sMutex = NULL; + delete sMutex; + sMutex = NULL; } - + //---------------------------------------------------------------------------- LLThreadSafeRefCount::LLThreadSafeRefCount() : - mRef(0) + mRef(0) { } LLThreadSafeRefCount::LLThreadSafeRefCount(const LLThreadSafeRefCount& src) { - mRef = 0; + mRef = 0; } LLThreadSafeRefCount::~LLThreadSafeRefCount() { - if (mRef != 0) - { - LL_ERRS() << "deleting non-zero reference" << LL_ENDL; - } + if (mRef != 0) + { + LL_ERRS() << "deleting non-zero reference" << LL_ENDL; + } } //============================================================================ -- cgit v1.2.3