MAINT-7634: Logging and instrumentation canges to narrow down viewer crashes.

2 files changed, 352 insertions, 303 deletions

diff --git a/indra/llcommon/llthread.cpp b/indra/llcommon/llthread.cpp
index 52255bfaeb..b96b2ce4bc 100644
--- a/indra/llcommon/llthread.cpp
+++ b/indra/llcommon/llthread.cpp
@@ -34,6 +34,7 @@
 #include "lltimer.h"
 #include "lltrace.h"
 #include "lltracethreadrecorder.h"
+#include "llexception.h"
 
 #if LL_LINUX || LL_SOLARIS
 #include <sched.h>
@@ -46,28 +47,28 @@ const DWORD MS_VC_EXCEPTION=0x406D1388;
 #pragma pack(push,8)
 typedef struct tagTHREADNAME_INFO
 {
-	DWORD dwType; // Must be 0x1000.
-	LPCSTR szName; // Pointer to name (in user addr space).
-	DWORD dwThreadID; // Thread ID (-1=caller thread).
-	DWORD dwFlags; // Reserved for future use, must be zero.
+    DWORD dwType; // Must be 0x1000.
+    LPCSTR szName; // Pointer to name (in user addr space).
+    DWORD dwThreadID; // Thread ID (-1=caller thread).
+    DWORD dwFlags; // Reserved for future use, must be zero.
 } THREADNAME_INFO;
 #pragma pack(pop)
 
 void set_thread_name( DWORD dwThreadID, const char* threadName)
 {
-	THREADNAME_INFO info;
-	info.dwType = 0x1000;
-	info.szName = threadName;
-	info.dwThreadID = dwThreadID;
-	info.dwFlags = 0;
-
-	__try
-	{
-		::RaiseException( MS_VC_EXCEPTION, 0, sizeof(info)/sizeof(DWORD), (ULONG_PTR*)&info );
-	}
-	__except(EXCEPTION_CONTINUE_EXECUTION)
-	{
-	}
+    THREADNAME_INFO info;
+    info.dwType = 0x1000;
+    info.szName = threadName;
+    info.dwThreadID = dwThreadID;
+    info.dwFlags = 0;
+
+    __try
+    {
+        ::RaiseException( MS_VC_EXCEPTION, 0, sizeof(info)/sizeof(DWORD), (ULONG_PTR*)&info );
+    }
+    __except(EXCEPTION_CONTINUE_EXECUTION)
+    {
+    }
 }
 #endif
 
@@ -99,17 +100,17 @@ U32 LLThread::sIDIter = 0;
 
 LL_COMMON_API void assert_main_thread()
 {
-	static U32 s_thread_id = LLThread::currentID();
-	if (LLThread::currentID() != s_thread_id)
-	{
-		LL_WARNS() << "Illegal execution from thread id " << (S32) LLThread::currentID()
-			<< " outside main thread " << (S32) s_thread_id << LL_ENDL;
-	}
+    static U32 s_thread_id = LLThread::currentID();
+    if (LLThread::currentID() != s_thread_id)
+    {
+        LL_WARNS() << "Illegal execution from thread id " << (S32) LLThread::currentID()
+            << " outside main thread " << (S32) s_thread_id << LL_ENDL;
+    }
 }
 
 void LLThread::registerThreadID()
 {
-	sThreadID = ++sIDIter;
+    sThreadID = ++sIDIter;
 }
 
 //
@@ -117,157 +118,203 @@ void LLThread::registerThreadID()
 //
 void *APR_THREAD_FUNC LLThread::staticRun(apr_thread_t *apr_threadp, void *datap)
 {
-	LLThread *threadp = (LLThread *)datap;
+    LLThread *threadp = (LLThread *)datap;
 
 #ifdef LL_WINDOWS
-	set_thread_name(-1, threadp->mName.c_str());
+    set_thread_name(-1, threadp->mName.c_str());
 #endif
 
-	// for now, hard code all LLThreads to report to single master thread recorder, which is known to be running on main thread
-	threadp->mRecorder = new LLTrace::ThreadRecorder(*LLTrace::get_master_thread_recorder());
-
-	sThreadID = threadp->mID;
-
-	// Run the user supplied function
-	threadp->run();
-
-	//LL_INFOS() << "LLThread::staticRun() Exiting: " << threadp->mName << LL_ENDL;
-	
-	delete threadp->mRecorder;
-	threadp->mRecorder = NULL;
-	
-	// We're done with the run function, this thread is done executing now.
-	//NB: we are using this flag to sync across threads...we really need memory barriers here
-	threadp->mStatus = STOPPED;
-
-	return NULL;
+    // for now, hard code all LLThreads to report to single master thread recorder, which is known to be running on main thread
+    threadp->mRecorder = new LLTrace::ThreadRecorder(*LLTrace::get_master_thread_recorder());
+
+    sThreadID = threadp->mID;
+
+    try
+    {
+        // Run the user supplied function
+        do 
+        {
+            try
+            {
+                threadp->run();
+            }
+            catch (const LLContinueError &e)
+            {
+                LL_WARNS("THREAD") << "ContinueException on thread '" << threadp->mName <<
+                    "' reentering run(). Error what is: '" << e.what() << "'" << LL_ENDL;
+                //output possible call stacks to log file.
+                LLError::LLCallStacks::print();
+
+                LOG_UNHANDLED_EXCEPTION("LLThread");
+                continue;
+            }
+            break;
+
+        } while (true);
+
+        //LL_INFOS() << "LLThread::staticRun() Exiting: " << threadp->mName << LL_ENDL;
+
+        // We're done with the run function, this thread is done executing now.
+        //NB: we are using this flag to sync across threads...we really need memory barriers here
+        threadp->mStatus = STOPPED;
+    }
+    catch (std::bad_alloc)
+    {
+        threadp->mStatus = CRASHED;
+        LLMemory::logMemoryInfo(TRUE);
+
+        //output possible call stacks to log file.
+        LLError::LLCallStacks::print();
+
+        LL_ERRS("THREAD") << "Bad memory allocation in LLThread::staticRun() named '" << threadp->mName << "'!" << LL_ENDL;
+    }
+    catch (...)
+    {
+        threadp->mStatus = CRASHED;
+        CRASH_ON_UNHANDLED_EXCEPTION("LLThread");
+    }
+
+    delete threadp->mRecorder;
+    threadp->mRecorder = NULL;
+
+    return NULL;
 }
 
 LLThread::LLThread(const std::string& name, apr_pool_t *poolp) :
-	mPaused(FALSE),
-	mName(name),
-	mAPRThreadp(NULL),
-	mStatus(STOPPED),
-	mRecorder(NULL)
+    mPaused(FALSE),
+    mName(name),
+    mAPRThreadp(NULL),
+    mStatus(STOPPED),
+    mRecorder(NULL)
 {
 
-	mID = ++sIDIter;
-
-	// Thread creation probably CAN be paranoid about APR being initialized, if necessary
-	if (poolp)
-	{
-		mIsLocalPool = FALSE;
-		mAPRPoolp = poolp;
-	}
-	else
-	{
-		mIsLocalPool = TRUE;
-		apr_pool_create(&mAPRPoolp, NULL); // Create a subpool for this thread
-	}
-	mRunCondition = new LLCondition(mAPRPoolp);
-	mDataLock = new LLMutex(mAPRPoolp);
-	mLocalAPRFilePoolp = NULL ;
+    mID = ++sIDIter;
+
+    // Thread creation probably CAN be paranoid about APR being initialized, if necessary
+    if (poolp)
+    {
+        mIsLocalPool = FALSE;
+        mAPRPoolp = poolp;
+    }
+    else
+    {
+        mIsLocalPool = TRUE;
+        apr_pool_create(&mAPRPoolp, NULL); // Create a subpool for this thread
+    }
+    mRunCondition = new LLCondition(mAPRPoolp);
+    mDataLock = new LLMutex(mAPRPoolp);
+    mLocalAPRFilePoolp = NULL ;
 }
 
 
 LLThread::~LLThread()
 {
-	shutdown();
-
-	if(mLocalAPRFilePoolp)
-	{
-		delete mLocalAPRFilePoolp ;
-		mLocalAPRFilePoolp = NULL ;
-	}
+    shutdown();
+
+    if (isCrashed())
+    {
+        LL_WARNS("THREAD") << "Destroying crashed thread named '" << mName << "'" << LL_ENDL;
+    }
+
+    if(mLocalAPRFilePoolp)
+    {
+        delete mLocalAPRFilePoolp ;
+        mLocalAPRFilePoolp = NULL ;
+    }
 }
 
 void LLThread::shutdown()
 {
-	// Warning!  If you somehow call the thread destructor from itself,
-	// the thread will die in an unclean fashion!
-	if (mAPRThreadp)
-	{
-		if (!isStopped())
-		{
-			// The thread isn't already stopped
-			// First, set the flag that indicates that we're ready to die
-			setQuitting();
-
-			//LL_INFOS() << "LLThread::~LLThread() Killing thread " << mName << " Status: " << mStatus << LL_ENDL;
-			// Now wait a bit for the thread to exit
-			// It's unclear whether I should even bother doing this - this destructor
-			// should never get called unless we're already stopped, really...
-			S32 counter = 0;
-			const S32 MAX_WAIT = 600;
-			while (counter < MAX_WAIT)
-			{
-				if (isStopped())
-				{
-					break;
-				}
-				// Sleep for a tenth of a second
-				ms_sleep(100);
-				yield();
-				counter++;
-			}
-		}
-
-		if (!isStopped())
-		{
-			// This thread just wouldn't stop, even though we gave it time
-			//LL_WARNS() << "LLThread::~LLThread() exiting thread before clean exit!" << LL_ENDL;
-			// Put a stake in its heart.
-			delete mRecorder;
-
-			apr_thread_exit(mAPRThreadp, -1);
-			return;
-		}
-		mAPRThreadp = NULL;
-	}
-
-	delete mRunCondition;
-	mRunCondition = NULL;
-
-	delete mDataLock;
-	mDataLock = NULL;
-	
-	if (mIsLocalPool && mAPRPoolp)
-	{
-		apr_pool_destroy(mAPRPoolp);
-		mAPRPoolp = 0;
-	}
-
-	if (mRecorder)
-	{
-		// missed chance to properly shut down recorder (needs to be done in thread context)
-		// probably due to abnormal thread termination
-		// so just leak it and remove it from parent
-		LLTrace::get_master_thread_recorder()->removeChildRecorder(mRecorder);
-	}
+    if (isCrashed())
+    {
+        LL_WARNS("THREAD") << "Shutting down crashed thread named '" << mName << "'" << LL_ENDL;
+    }
+
+    // Warning!  If you somehow call the thread destructor from itself,
+    // the thread will die in an unclean fashion!
+    if (mAPRThreadp)
+    {
+        if (!isStopped())
+        {
+            // The thread isn't already stopped
+            // First, set the flag that indicates that we're ready to die
+            setQuitting();
+
+            //LL_INFOS() << "LLThread::~LLThread() Killing thread " << mName << " Status: " << mStatus << LL_ENDL;
+            // Now wait a bit for the thread to exit
+            // It's unclear whether I should even bother doing this - this destructor
+            // should never get called unless we're already stopped, really...
+            S32 counter = 0;
+            const S32 MAX_WAIT = 600;
+            while (counter < MAX_WAIT)
+            {
+                if (isStopped())
+                {
+                    break;
+                }
+                // Sleep for a tenth of a second
+                ms_sleep(100);
+                yield();
+                counter++;
+            }
+        }
+
+        if (!isStopped())
+        {
+            // This thread just wouldn't stop, even though we gave it time
+            //LL_WARNS() << "LLThread::~LLThread() exiting thread before clean exit!" << LL_ENDL;
+            // Put a stake in its heart.
+            delete mRecorder;
+
+            apr_thread_exit(mAPRThreadp, -1);
+            return;
+        }
+        mAPRThreadp = NULL;
+    }
+
+    delete mRunCondition;
+    mRunCondition = NULL;
+
+    delete mDataLock;
+    mDataLock = NULL;
+    
+    if (mIsLocalPool && mAPRPoolp)
+    {
+        apr_pool_destroy(mAPRPoolp);
+        mAPRPoolp = 0;
+    }
+
+    if (mRecorder)
+    {
+        // missed chance to properly shut down recorder (needs to be done in thread context)
+        // probably due to abnormal thread termination
+        // so just leak it and remove it from parent
+        LLTrace::get_master_thread_recorder()->removeChildRecorder(mRecorder);
+    }
 }
 
 
 void LLThread::start()
 {
-	llassert(isStopped());
-	
-	// Set thread state to running
-	mStatus = RUNNING;
-
-	apr_status_t status =
-		apr_thread_create(&mAPRThreadp, NULL, staticRun, (void *)this, mAPRPoolp);
-	
-	if(status == APR_SUCCESS)
-	{	
-		// We won't bother joining
-		apr_thread_detach(mAPRThreadp);
-	}
-	else
-	{
-		mStatus = STOPPED;
-		LL_WARNS() << "failed to start thread " << mName << LL_ENDL;
-		ll_apr_warn_status(status);
-	}
+    llassert(isStopped());
+    
+    // Set thread state to running
+    mStatus = RUNNING;
+
+    apr_status_t status =
+        apr_thread_create(&mAPRThreadp, NULL, staticRun, (void *)this, mAPRPoolp);
+    
+    if(status == APR_SUCCESS)
+    {   
+        // We won't bother joining
+        apr_thread_detach(mAPRThreadp);
+    }
+    else
+    {
+        mStatus = STOPPED;
+        LL_WARNS() << "failed to start thread " << mName << LL_ENDL;
+        ll_apr_warn_status(status);
+    }
 
 }
 
@@ -278,28 +325,28 @@ void LLThread::start()
 // The thread will pause when (and if) it calls checkPause()
 void LLThread::pause()
 {
-	if (!mPaused)
-	{
-		// this will cause the thread to stop execution as soon as checkPause() is called
-		mPaused = 1;		// Does not need to be atomic since this is only set/unset from the main thread
-	}	
+    if (!mPaused)
+    {
+        // this will cause the thread to stop execution as soon as checkPause() is called
+        mPaused = 1;        // Does not need to be atomic since this is only set/unset from the main thread
+    }   
 }
 
 void LLThread::unpause()
 {
-	if (mPaused)
-	{
-		mPaused = 0;
-	}
+    if (mPaused)
+    {
+        mPaused = 0;
+    }
 
-	wake(); // wake up the thread if necessary
+    wake(); // wake up the thread if necessary
 }
 
 // virtual predicate function -- returns true if the thread should wake up, false if it should sleep.
 bool LLThread::runCondition(void)
 {
-	// by default, always run.  Handling of pause/unpause is done regardless of this function's result.
-	return true;
+    // by default, always run.  Handling of pause/unpause is done regardless of this function's result.
+    return true;
 }
 
 //============================================================================
@@ -307,65 +354,65 @@ bool LLThread::runCondition(void)
 // Stop thread execution if requested until unpaused.
 void LLThread::checkPause()
 {
-	mDataLock->lock();
-
-	// This is in a while loop because the pthread API allows for spurious wakeups.
-	while(shouldSleep())
-	{
-		mDataLock->unlock();
-		mRunCondition->wait(); // unlocks mRunCondition
-		mDataLock->lock();
-		// mRunCondition is locked when the thread wakes up
-	}
-	
- 	mDataLock->unlock();
+    mDataLock->lock();
+
+    // This is in a while loop because the pthread API allows for spurious wakeups.
+    while(shouldSleep())
+    {
+        mDataLock->unlock();
+        mRunCondition->wait(); // unlocks mRunCondition
+        mDataLock->lock();
+        // mRunCondition is locked when the thread wakes up
+    }
+    
+    mDataLock->unlock();
 }
 
 //============================================================================
 
 void LLThread::setQuitting()
 {
-	mDataLock->lock();
-	if (mStatus == RUNNING)
-	{
-		mStatus = QUITTING;
-	}
-	mDataLock->unlock();
-	wake();
+    mDataLock->lock();
+    if (mStatus == RUNNING)
+    {
+        mStatus = QUITTING;
+    }
+    mDataLock->unlock();
+    wake();
 }
 
 // static
 U32 LLThread::currentID()
 {
-	return sThreadID;
+    return sThreadID;
 }
 
 // static
 void LLThread::yield()
 {
 #if LL_LINUX || LL_SOLARIS
-	sched_yield(); // annoyingly, apr_thread_yield  is a noop on linux...
+    sched_yield(); // annoyingly, apr_thread_yield  is a noop on linux...
 #else
-	apr_thread_yield();
+    apr_thread_yield();
 #endif
 }
 
 void LLThread::wake()
 {
-	mDataLock->lock();
-	if(!shouldSleep())
-	{
-		mRunCondition->signal();
-	}
-	mDataLock->unlock();
+    mDataLock->lock();
+    if(!shouldSleep())
+    {
+        mRunCondition->signal();
+    }
+    mDataLock->unlock();
 }
 
 void LLThread::wakeLocked()
 {
-	if(!shouldSleep())
-	{
-		mRunCondition->signal();
-	}
+    if(!shouldSleep())
+    {
+        mRunCondition->signal();
+    }
 }
 
 //============================================================================
@@ -378,38 +425,38 @@ LLMutex* LLThreadSafeRefCount::sMutex = 0;
 //static
 void LLThreadSafeRefCount::initThreadSafeRefCount()
 {
-	if (!sMutex)
-	{
-		sMutex = new LLMutex(0);
-	}
+    if (!sMutex)
+    {
+        sMutex = new LLMutex(0);
+    }
 }
 
 //static
 void LLThreadSafeRefCount::cleanupThreadSafeRefCount()
 {
-	delete sMutex;
-	sMutex = NULL;
+    delete sMutex;
+    sMutex = NULL;
 }
-	
+    
 
 //----------------------------------------------------------------------------
 
 LLThreadSafeRefCount::LLThreadSafeRefCount() :
-	mRef(0)
+    mRef(0)
 {
 }
 
 LLThreadSafeRefCount::LLThreadSafeRefCount(const LLThreadSafeRefCount& src)
 {
-	mRef = 0;
+    mRef = 0;
 }
 
 LLThreadSafeRefCount::~LLThreadSafeRefCount()
 { 
-	if (mRef != 0)
-	{
-		LL_ERRS() << "deleting non-zero reference" << LL_ENDL;
-	}
+    if (mRef != 0)
+    {
+        LL_ERRS() << "deleting non-zero reference" << LL_ENDL;
+    }
 }
 
 //============================================================================
diff --git a/indra/llcommon/llthread.h b/indra/llcommon/llthread.h
index 6f9ec10fd3..dda7fa8ffb 100644
--- a/indra/llcommon/llthread.h
+++ b/indra/llcommon/llthread.h
@@ -38,118 +38,120 @@ LL_COMMON_API void assert_main_thread();
 
 namespace LLTrace
 {
-	class ThreadRecorder;
+    class ThreadRecorder;
 }
 
 class LL_COMMON_API LLThread
 {
 private:
-	friend class LLMutex;
-	static U32 sIDIter;
+    friend class LLMutex;
+    static U32 sIDIter;
 
 public:
-	typedef enum e_thread_status
-	{
-		STOPPED = 0,	// The thread is not running.  Not started, or has exited its run function
-		RUNNING = 1,	// The thread is currently running
-		QUITTING= 2 	// Someone wants this thread to quit
-	} EThreadStatus;
-
-	LLThread(const std::string& name, apr_pool_t *poolp = NULL);
-	virtual ~LLThread(); // Warning!  You almost NEVER want to destroy a thread unless it's in the STOPPED state.
-	virtual void shutdown(); // stops the thread
-	
-	bool isQuitting() const { return (QUITTING == mStatus); }
-	bool isStopped() const { return (STOPPED == mStatus); }
-	
-	static U32 currentID(); // Return ID of current thread
-	static void yield(); // Static because it can be called by the main thread, which doesn't have an LLThread data structure.
-	
+    typedef enum e_thread_status
+    {
+        STOPPED = 0,    // The thread is not running.  Not started, or has exited its run function
+        RUNNING = 1,    // The thread is currently running
+        QUITTING= 2,    // Someone wants this thread to quit
+        CRASHED = -1    // An uncaught exception was thrown by the thread
+    } EThreadStatus;
+
+    LLThread(const std::string& name, apr_pool_t *poolp = NULL);
+    virtual ~LLThread(); // Warning!  You almost NEVER want to destroy a thread unless it's in the STOPPED state.
+    virtual void shutdown(); // stops the thread
+    
+    bool isQuitting() const { return (QUITTING == mStatus); }
+    bool isStopped() const { return (STOPPED == mStatus) || (CRASHED == mStatus); }
+    bool isCrashed() const { return (CRASHED == mStatus); } 
+    
+    static U32 currentID(); // Return ID of current thread
+    static void yield(); // Static because it can be called by the main thread, which doesn't have an LLThread data structure.
+    
 public:
-	// PAUSE / RESUME functionality. See source code for important usage notes.
-	// Called from MAIN THREAD.
-	void pause();
-	void unpause();
-	bool isPaused() { return isStopped() || mPaused == TRUE; }
-	
-	// Cause the thread to wake up and check its condition
-	void wake();
-
-	// Same as above, but to be used when the condition is already locked.
-	void wakeLocked();
-
-	// Called from run() (CHILD THREAD). Pause the thread if requested until unpaused.
-	void checkPause();
-
-	// this kicks off the apr thread
-	void start(void);
-
-	apr_pool_t *getAPRPool() { return mAPRPoolp; }
-	LLVolatileAPRPool* getLocalAPRFilePool() { return mLocalAPRFilePoolp ; }
-
-	U32 getID() const { return mID; }
-
-	// Called by threads *not* created via LLThread to register some
-	// internal state used by LLMutex.  You must call this once early
-	// in the running thread to prevent collisions with the main thread.
-	static void registerThreadID();
-	
+    // PAUSE / RESUME functionality. See source code for important usage notes.
+    // Called from MAIN THREAD.
+    void pause();
+    void unpause();
+    bool isPaused() { return isStopped() || mPaused == TRUE; }
+    
+    // Cause the thread to wake up and check its condition
+    void wake();
+
+    // Same as above, but to be used when the condition is already locked.
+    void wakeLocked();
+
+    // Called from run() (CHILD THREAD). Pause the thread if requested until unpaused.
+    void checkPause();
+
+    // this kicks off the apr thread
+    void start(void);
+
+    apr_pool_t *getAPRPool() { return mAPRPoolp; }
+    LLVolatileAPRPool* getLocalAPRFilePool() { return mLocalAPRFilePoolp ; }
+
+    U32 getID() const { return mID; }
+
+    // Called by threads *not* created via LLThread to register some
+    // internal state used by LLMutex.  You must call this once early
+    // in the running thread to prevent collisions with the main thread.
+    static void registerThreadID();
+    
 private:
-	BOOL				mPaused;
-	
-	// static function passed to APR thread creation routine
-	static void *APR_THREAD_FUNC staticRun(struct apr_thread_t *apr_threadp, void *datap);
+    BOOL                mPaused;
+    
+    // static function passed to APR thread creation routine
+    static void *APR_THREAD_FUNC staticRun(struct apr_thread_t *apr_threadp, void *datap);
 
 protected:
-	std::string			mName;
-	class LLCondition*	mRunCondition;
-	LLMutex*			mDataLock;
-
-	apr_thread_t		*mAPRThreadp;
-	apr_pool_t			*mAPRPoolp;
-	BOOL				mIsLocalPool;
-	EThreadStatus		mStatus;
-	U32					mID;
-	LLTrace::ThreadRecorder* mRecorder;
-
-	//a local apr_pool for APRFile operations in this thread. If it exists, LLAPRFile::sAPRFilePoolp should not be used.
-	//Note: this pool is used by APRFile ONLY, do NOT use it for any other purposes.
-	//      otherwise it will cause severe memory leaking!!! --bao
-	LLVolatileAPRPool  *mLocalAPRFilePoolp ; 
-
-	void setQuitting();
-	
-	// virtual function overridden by subclass -- this will be called when the thread runs
-	virtual void run(void) = 0; 
-	
-	// virtual predicate function -- returns true if the thread should wake up, false if it should sleep.
-	virtual bool runCondition(void);
-
-	// Lock/Unlock Run Condition -- use around modification of any variable used in runCondition()
-	inline void lockData();
-	inline void unlockData();
-	
-	// This is the predicate that decides whether the thread should sleep.  
-	// It should only be called with mDataLock locked, since the virtual runCondition() function may need to access
-	// data structures that are thread-unsafe.
-	bool shouldSleep(void) { return (mStatus == RUNNING) && (isPaused() || (!runCondition())); }
-
-	// To avoid spurious signals (and the associated context switches) when the condition may or may not have changed, you can do the following:
-	// mDataLock->lock();
-	// if(!shouldSleep())
-	//     mRunCondition->signal();
-	// mDataLock->unlock();
+    std::string         mName;
+    class LLCondition*  mRunCondition;
+    LLMutex*            mDataLock;
+
+    apr_thread_t        *mAPRThreadp;
+    apr_pool_t          *mAPRPoolp;
+    BOOL                mIsLocalPool;
+    EThreadStatus       mStatus;
+    U32                 mID;
+    LLTrace::ThreadRecorder* mRecorder;
+
+    //a local apr_pool for APRFile operations in this thread. If it exists, LLAPRFile::sAPRFilePoolp should not be used.
+    //Note: this pool is used by APRFile ONLY, do NOT use it for any other purposes.
+    //      otherwise it will cause severe memory leaking!!! --bao
+    LLVolatileAPRPool  *mLocalAPRFilePoolp ; 
+
+    void setQuitting();
+    
+    // virtual function overridden by subclass -- this will be called when the thread runs
+    virtual void run(void) = 0; 
+    
+    // virtual predicate function -- returns true if the thread should wake up, false if it should sleep.
+    virtual bool runCondition(void);
+
+    // Lock/Unlock Run Condition -- use around modification of any variable used in runCondition()
+    inline void lockData();
+    inline void unlockData();
+    
+    // This is the predicate that decides whether the thread should sleep.  
+    // It should only be called with mDataLock locked, since the virtual runCondition() function may need to access
+    // data structures that are thread-unsafe.
+    bool shouldSleep(void) { return (mStatus == RUNNING) && (isPaused() || (!runCondition())); }
+
+    // To avoid spurious signals (and the associated context switches) when the condition may or may not have changed, you can do the following:
+    // mDataLock->lock();
+    // if(!shouldSleep())
+    //     mRunCondition->signal();
+    // mDataLock->unlock();
 };
 
 
 void LLThread::lockData()
 {
-	mDataLock->lock();
+    mDataLock->lock();
 }
 
 void LLThread::unlockData()
 {
-	mDataLock->unlock();
+    mDataLock->unlock();
 }
 
 
@@ -160,9 +162,9 @@ void LLThread::unlockData()
 class LL_COMMON_API LLResponder : public LLThreadSafeRefCount
 {
 protected:
-	virtual ~LLResponder();
+    virtual ~LLResponder();
 public:
-	virtual void completed(bool success) = 0;
+    virtual void completed(bool success) = 0;
 };
 
 //============================================================================