MAINT-7634: Logging and instrumentation canges to narrow down viewer crashes.

10 files changed, 426 insertions, 342 deletions

diff --git a/indra/llcommon/llthread.cpp b/indra/llcommon/llthread.cpp
index 52255bfaeb..b96b2ce4bc 100644
--- a/indra/llcommon/llthread.cpp
+++ b/indra/llcommon/llthread.cpp
@@ -34,6 +34,7 @@
 #include "lltimer.h"
 #include "lltrace.h"
 #include "lltracethreadrecorder.h"
+#include "llexception.h"
 
 #if LL_LINUX || LL_SOLARIS
 #include <sched.h>
@@ -46,28 +47,28 @@ const DWORD MS_VC_EXCEPTION=0x406D1388;
 #pragma pack(push,8)
 typedef struct tagTHREADNAME_INFO
 {
-	DWORD dwType; // Must be 0x1000.
-	LPCSTR szName; // Pointer to name (in user addr space).
-	DWORD dwThreadID; // Thread ID (-1=caller thread).
-	DWORD dwFlags; // Reserved for future use, must be zero.
+    DWORD dwType; // Must be 0x1000.
+    LPCSTR szName; // Pointer to name (in user addr space).
+    DWORD dwThreadID; // Thread ID (-1=caller thread).
+    DWORD dwFlags; // Reserved for future use, must be zero.
 } THREADNAME_INFO;
 #pragma pack(pop)
 
 void set_thread_name( DWORD dwThreadID, const char* threadName)
 {
-	THREADNAME_INFO info;
-	info.dwType = 0x1000;
-	info.szName = threadName;
-	info.dwThreadID = dwThreadID;
-	info.dwFlags = 0;
-
-	__try
-	{
-		::RaiseException( MS_VC_EXCEPTION, 0, sizeof(info)/sizeof(DWORD), (ULONG_PTR*)&info );
-	}
-	__except(EXCEPTION_CONTINUE_EXECUTION)
-	{
-	}
+    THREADNAME_INFO info;
+    info.dwType = 0x1000;
+    info.szName = threadName;
+    info.dwThreadID = dwThreadID;
+    info.dwFlags = 0;
+
+    __try
+    {
+        ::RaiseException( MS_VC_EXCEPTION, 0, sizeof(info)/sizeof(DWORD), (ULONG_PTR*)&info );
+    }
+    __except(EXCEPTION_CONTINUE_EXECUTION)
+    {
+    }
 }
 #endif
 
@@ -99,17 +100,17 @@ U32 LLThread::sIDIter = 0;
 
 LL_COMMON_API void assert_main_thread()
 {
-	static U32 s_thread_id = LLThread::currentID();
-	if (LLThread::currentID() != s_thread_id)
-	{
-		LL_WARNS() << "Illegal execution from thread id " << (S32) LLThread::currentID()
-			<< " outside main thread " << (S32) s_thread_id << LL_ENDL;
-	}
+    static U32 s_thread_id = LLThread::currentID();
+    if (LLThread::currentID() != s_thread_id)
+    {
+        LL_WARNS() << "Illegal execution from thread id " << (S32) LLThread::currentID()
+            << " outside main thread " << (S32) s_thread_id << LL_ENDL;
+    }
 }
 
 void LLThread::registerThreadID()
 {
-	sThreadID = ++sIDIter;
+    sThreadID = ++sIDIter;
 }
 
 //
@@ -117,157 +118,203 @@ void LLThread::registerThreadID()
 //
 void *APR_THREAD_FUNC LLThread::staticRun(apr_thread_t *apr_threadp, void *datap)
 {
-	LLThread *threadp = (LLThread *)datap;
+    LLThread *threadp = (LLThread *)datap;
 
 #ifdef LL_WINDOWS
-	set_thread_name(-1, threadp->mName.c_str());
+    set_thread_name(-1, threadp->mName.c_str());
 #endif
 
-	// for now, hard code all LLThreads to report to single master thread recorder, which is known to be running on main thread
-	threadp->mRecorder = new LLTrace::ThreadRecorder(*LLTrace::get_master_thread_recorder());
-
-	sThreadID = threadp->mID;
-
-	// Run the user supplied function
-	threadp->run();
-
-	//LL_INFOS() << "LLThread::staticRun() Exiting: " << threadp->mName << LL_ENDL;
-	
-	delete threadp->mRecorder;
-	threadp->mRecorder = NULL;
-	
-	// We're done with the run function, this thread is done executing now.
-	//NB: we are using this flag to sync across threads...we really need memory barriers here
-	threadp->mStatus = STOPPED;
-
-	return NULL;
+    // for now, hard code all LLThreads to report to single master thread recorder, which is known to be running on main thread
+    threadp->mRecorder = new LLTrace::ThreadRecorder(*LLTrace::get_master_thread_recorder());
+
+    sThreadID = threadp->mID;
+
+    try
+    {
+        // Run the user supplied function
+        do 
+        {
+            try
+            {
+                threadp->run();
+            }
+            catch (const LLContinueError &e)
+            {
+                LL_WARNS("THREAD") << "ContinueException on thread '" << threadp->mName <<
+                    "' reentering run(). Error what is: '" << e.what() << "'" << LL_ENDL;
+                //output possible call stacks to log file.
+                LLError::LLCallStacks::print();
+
+                LOG_UNHANDLED_EXCEPTION("LLThread");
+                continue;
+            }
+            break;
+
+        } while (true);
+
+        //LL_INFOS() << "LLThread::staticRun() Exiting: " << threadp->mName << LL_ENDL;
+
+        // We're done with the run function, this thread is done executing now.
+        //NB: we are using this flag to sync across threads...we really need memory barriers here
+        threadp->mStatus = STOPPED;
+    }
+    catch (std::bad_alloc)
+    {
+        threadp->mStatus = CRASHED;
+        LLMemory::logMemoryInfo(TRUE);
+
+        //output possible call stacks to log file.
+        LLError::LLCallStacks::print();
+
+        LL_ERRS("THREAD") << "Bad memory allocation in LLThread::staticRun() named '" << threadp->mName << "'!" << LL_ENDL;
+    }
+    catch (...)
+    {
+        threadp->mStatus = CRASHED;
+        CRASH_ON_UNHANDLED_EXCEPTION("LLThread");
+    }
+
+    delete threadp->mRecorder;
+    threadp->mRecorder = NULL;
+
+    return NULL;
 }
 
 LLThread::LLThread(const std::string& name, apr_pool_t *poolp) :
-	mPaused(FALSE),
-	mName(name),
-	mAPRThreadp(NULL),
-	mStatus(STOPPED),
-	mRecorder(NULL)
+    mPaused(FALSE),
+    mName(name),
+    mAPRThreadp(NULL),
+    mStatus(STOPPED),
+    mRecorder(NULL)
 {
 
-	mID = ++sIDIter;
-
-	// Thread creation probably CAN be paranoid about APR being initialized, if necessary
-	if (poolp)
-	{
-		mIsLocalPool = FALSE;
-		mAPRPoolp = poolp;
-	}
-	else
-	{
-		mIsLocalPool = TRUE;
-		apr_pool_create(&mAPRPoolp, NULL); // Create a subpool for this thread
-	}
-	mRunCondition = new LLCondition(mAPRPoolp);
-	mDataLock = new LLMutex(mAPRPoolp);
-	mLocalAPRFilePoolp = NULL ;
+    mID = ++sIDIter;
+
+    // Thread creation probably CAN be paranoid about APR being initialized, if necessary
+    if (poolp)
+    {
+        mIsLocalPool = FALSE;
+        mAPRPoolp = poolp;
+    }
+    else
+    {
+        mIsLocalPool = TRUE;
+        apr_pool_create(&mAPRPoolp, NULL); // Create a subpool for this thread
+    }
+    mRunCondition = new LLCondition(mAPRPoolp);
+    mDataLock = new LLMutex(mAPRPoolp);
+    mLocalAPRFilePoolp = NULL ;
 }
 
 
 LLThread::~LLThread()
 {
-	shutdown();
-
-	if(mLocalAPRFilePoolp)
-	{
-		delete mLocalAPRFilePoolp ;
-		mLocalAPRFilePoolp = NULL ;
-	}
+    shutdown();
+
+    if (isCrashed())
+    {
+        LL_WARNS("THREAD") << "Destroying crashed thread named '" << mName << "'" << LL_ENDL;
+    }
+
+    if(mLocalAPRFilePoolp)
+    {
+        delete mLocalAPRFilePoolp ;
+        mLocalAPRFilePoolp = NULL ;
+    }
 }
 
 void LLThread::shutdown()
 {
-	// Warning!  If you somehow call the thread destructor from itself,
-	// the thread will die in an unclean fashion!
-	if (mAPRThreadp)
-	{
-		if (!isStopped())
-		{
-			// The thread isn't already stopped
-			// First, set the flag that indicates that we're ready to die
-			setQuitting();
-
-			//LL_INFOS() << "LLThread::~LLThread() Killing thread " << mName << " Status: " << mStatus << LL_ENDL;
-			// Now wait a bit for the thread to exit
-			// It's unclear whether I should even bother doing this - this destructor
-			// should never get called unless we're already stopped, really...
-			S32 counter = 0;
-			const S32 MAX_WAIT = 600;
-			while (counter < MAX_WAIT)
-			{
-				if (isStopped())
-				{
-					break;
-				}
-				// Sleep for a tenth of a second
-				ms_sleep(100);
-				yield();
-				counter++;
-			}
-		}
-
-		if (!isStopped())
-		{
-			// This thread just wouldn't stop, even though we gave it time
-			//LL_WARNS() << "LLThread::~LLThread() exiting thread before clean exit!" << LL_ENDL;
-			// Put a stake in its heart.
-			delete mRecorder;
-
-			apr_thread_exit(mAPRThreadp, -1);
-			return;
-		}
-		mAPRThreadp = NULL;
-	}
-
-	delete mRunCondition;
-	mRunCondition = NULL;
-
-	delete mDataLock;
-	mDataLock = NULL;
-	
-	if (mIsLocalPool && mAPRPoolp)
-	{
-		apr_pool_destroy(mAPRPoolp);
-		mAPRPoolp = 0;
-	}
-
-	if (mRecorder)
-	{
-		// missed chance to properly shut down recorder (needs to be done in thread context)
-		// probably due to abnormal thread termination
-		// so just leak it and remove it from parent
-		LLTrace::get_master_thread_recorder()->removeChildRecorder(mRecorder);
-	}
+    if (isCrashed())
+    {
+        LL_WARNS("THREAD") << "Shutting down crashed thread named '" << mName << "'" << LL_ENDL;
+    }
+
+    // Warning!  If you somehow call the thread destructor from itself,
+    // the thread will die in an unclean fashion!
+    if (mAPRThreadp)
+    {
+        if (!isStopped())
+        {
+            // The thread isn't already stopped
+            // First, set the flag that indicates that we're ready to die
+            setQuitting();
+
+            //LL_INFOS() << "LLThread::~LLThread() Killing thread " << mName << " Status: " << mStatus << LL_ENDL;
+            // Now wait a bit for the thread to exit
+            // It's unclear whether I should even bother doing this - this destructor
+            // should never get called unless we're already stopped, really...
+            S32 counter = 0;
+            const S32 MAX_WAIT = 600;
+            while (counter < MAX_WAIT)
+            {
+                if (isStopped())
+                {
+                    break;
+                }
+                // Sleep for a tenth of a second
+                ms_sleep(100);
+                yield();
+                counter++;
+            }
+        }
+
+        if (!isStopped())
+        {
+            // This thread just wouldn't stop, even though we gave it time
+            //LL_WARNS() << "LLThread::~LLThread() exiting thread before clean exit!" << LL_ENDL;
+            // Put a stake in its heart.
+            delete mRecorder;
+
+            apr_thread_exit(mAPRThreadp, -1);
+            return;
+        }
+        mAPRThreadp = NULL;
+    }
+
+    delete mRunCondition;
+    mRunCondition = NULL;
+
+    delete mDataLock;
+    mDataLock = NULL;
+    
+    if (mIsLocalPool && mAPRPoolp)
+    {
+        apr_pool_destroy(mAPRPoolp);
+        mAPRPoolp = 0;
+    }
+
+    if (mRecorder)
+    {
+        // missed chance to properly shut down recorder (needs to be done in thread context)
+        // probably due to abnormal thread termination
+        // so just leak it and remove it from parent
+        LLTrace::get_master_thread_recorder()->removeChildRecorder(mRecorder);
+    }
 }
 
 
 void LLThread::start()
 {
-	llassert(isStopped());
-	
-	// Set thread state to running
-	mStatus = RUNNING;
-
-	apr_status_t status =
-		apr_thread_create(&mAPRThreadp, NULL, staticRun, (void *)this, mAPRPoolp);
-	
-	if(status == APR_SUCCESS)
-	{	
-		// We won't bother joining
-		apr_thread_detach(mAPRThreadp);
-	}
-	else
-	{
-		mStatus = STOPPED;
-		LL_WARNS() << "failed to start thread " << mName << LL_ENDL;
-		ll_apr_warn_status(status);
-	}
+    llassert(isStopped());
+    
+    // Set thread state to running
+    mStatus = RUNNING;
+
+    apr_status_t status =
+        apr_thread_create(&mAPRThreadp, NULL, staticRun, (void *)this, mAPRPoolp);
+    
+    if(status == APR_SUCCESS)
+    {   
+        // We won't bother joining
+        apr_thread_detach(mAPRThreadp);
+    }
+    else
+    {
+        mStatus = STOPPED;
+        LL_WARNS() << "failed to start thread " << mName << LL_ENDL;
+        ll_apr_warn_status(status);
+    }
 
 }
 
@@ -278,28 +325,28 @@ void LLThread::start()
 // The thread will pause when (and if) it calls checkPause()
 void LLThread::pause()
 {
-	if (!mPaused)
-	{
-		// this will cause the thread to stop execution as soon as checkPause() is called
-		mPaused = 1;		// Does not need to be atomic since this is only set/unset from the main thread
-	}	
+    if (!mPaused)
+    {
+        // this will cause the thread to stop execution as soon as checkPause() is called
+        mPaused = 1;        // Does not need to be atomic since this is only set/unset from the main thread
+    }   
 }
 
 void LLThread::unpause()
 {
-	if (mPaused)
-	{
-		mPaused = 0;
-	}
+    if (mPaused)
+    {
+        mPaused = 0;
+    }
 
-	wake(); // wake up the thread if necessary
+    wake(); // wake up the thread if necessary
 }
 
 // virtual predicate function -- returns true if the thread should wake up, false if it should sleep.
 bool LLThread::runCondition(void)
 {
-	// by default, always run.  Handling of pause/unpause is done regardless of this function's result.
-	return true;
+    // by default, always run.  Handling of pause/unpause is done regardless of this function's result.
+    return true;
 }
 
 //============================================================================
@@ -307,65 +354,65 @@ bool LLThread::runCondition(void)
 // Stop thread execution if requested until unpaused.
 void LLThread::checkPause()
 {
-	mDataLock->lock();
-
-	// This is in a while loop because the pthread API allows for spurious wakeups.
-	while(shouldSleep())
-	{
-		mDataLock->unlock();
-		mRunCondition->wait(); // unlocks mRunCondition
-		mDataLock->lock();
-		// mRunCondition is locked when the thread wakes up
-	}
-	
- 	mDataLock->unlock();
+    mDataLock->lock();
+
+    // This is in a while loop because the pthread API allows for spurious wakeups.
+    while(shouldSleep())
+    {
+        mDataLock->unlock();
+        mRunCondition->wait(); // unlocks mRunCondition
+        mDataLock->lock();
+        // mRunCondition is locked when the thread wakes up
+    }
+    
+    mDataLock->unlock();
 }
 
 //============================================================================
 
 void LLThread::setQuitting()
 {
-	mDataLock->lock();
-	if (mStatus == RUNNING)
-	{
-		mStatus = QUITTING;
-	}
-	mDataLock->unlock();
-	wake();
+    mDataLock->lock();
+    if (mStatus == RUNNING)
+    {
+        mStatus = QUITTING;
+    }
+    mDataLock->unlock();
+    wake();
 }
 
 // static
 U32 LLThread::currentID()
 {
-	return sThreadID;
+    return sThreadID;
 }
 
 // static
 void LLThread::yield()
 {
 #if LL_LINUX || LL_SOLARIS
-	sched_yield(); // annoyingly, apr_thread_yield  is a noop on linux...
+    sched_yield(); // annoyingly, apr_thread_yield  is a noop on linux...
 #else
-	apr_thread_yield();
+    apr_thread_yield();
 #endif
 }
 
 void LLThread::wake()
 {
-	mDataLock->lock();
-	if(!shouldSleep())
-	{
-		mRunCondition->signal();
-	}
-	mDataLock->unlock();
+    mDataLock->lock();
+    if(!shouldSleep())
+    {
+        mRunCondition->signal();
+    }
+    mDataLock->unlock();
 }
 
 void LLThread::wakeLocked()
 {
-	if(!shouldSleep())
-	{
-		mRunCondition->signal();
-	}
+    if(!shouldSleep())
+    {
+        mRunCondition->signal();
+    }
 }
 
 //============================================================================
@@ -378,38 +425,38 @@ LLMutex* LLThreadSafeRefCount::sMutex = 0;
 //static
 void LLThreadSafeRefCount::initThreadSafeRefCount()
 {
-	if (!sMutex)
-	{
-		sMutex = new LLMutex(0);
-	}
+    if (!sMutex)
+    {
+        sMutex = new LLMutex(0);
+    }
 }
 
 //static
 void LLThreadSafeRefCount::cleanupThreadSafeRefCount()
 {
-	delete sMutex;
-	sMutex = NULL;
+    delete sMutex;
+    sMutex = NULL;
 }
-	
+    
 
 //----------------------------------------------------------------------------
 
 LLThreadSafeRefCount::LLThreadSafeRefCount() :
-	mRef(0)
+    mRef(0)
 {
 }
 
 LLThreadSafeRefCount::LLThreadSafeRefCount(const LLThreadSafeRefCount& src)
 {
-	mRef = 0;
+    mRef = 0;
 }
 
 LLThreadSafeRefCount::~LLThreadSafeRefCount()
 { 
-	if (mRef != 0)
-	{
-		LL_ERRS() << "deleting non-zero reference" << LL_ENDL;
-	}
+    if (mRef != 0)
+    {
+        LL_ERRS() << "deleting non-zero reference" << LL_ENDL;
+    }
 }
 
 //============================================================================
diff --git a/indra/llcommon/llthread.h b/indra/llcommon/llthread.h
index 6f9ec10fd3..dda7fa8ffb 100644
--- a/indra/llcommon/llthread.h
+++ b/indra/llcommon/llthread.h
@@ -38,118 +38,120 @@ LL_COMMON_API void assert_main_thread();
 
 namespace LLTrace
 {
-	class ThreadRecorder;
+    class ThreadRecorder;
 }
 
 class LL_COMMON_API LLThread
 {
 private:
-	friend class LLMutex;
-	static U32 sIDIter;
+    friend class LLMutex;
+    static U32 sIDIter;
 
 public:
-	typedef enum e_thread_status
-	{
-		STOPPED = 0,	// The thread is not running.  Not started, or has exited its run function
-		RUNNING = 1,	// The thread is currently running
-		QUITTING= 2 	// Someone wants this thread to quit
-	} EThreadStatus;
-
-	LLThread(const std::string& name, apr_pool_t *poolp = NULL);
-	virtual ~LLThread(); // Warning!  You almost NEVER want to destroy a thread unless it's in the STOPPED state.
-	virtual void shutdown(); // stops the thread
-	
-	bool isQuitting() const { return (QUITTING == mStatus); }
-	bool isStopped() const { return (STOPPED == mStatus); }
-	
-	static U32 currentID(); // Return ID of current thread
-	static void yield(); // Static because it can be called by the main thread, which doesn't have an LLThread data structure.
-	
+    typedef enum e_thread_status
+    {
+        STOPPED = 0,    // The thread is not running.  Not started, or has exited its run function
+        RUNNING = 1,    // The thread is currently running
+        QUITTING= 2,    // Someone wants this thread to quit
+        CRASHED = -1    // An uncaught exception was thrown by the thread
+    } EThreadStatus;
+
+    LLThread(const std::string& name, apr_pool_t *poolp = NULL);
+    virtual ~LLThread(); // Warning!  You almost NEVER want to destroy a thread unless it's in the STOPPED state.
+    virtual void shutdown(); // stops the thread
+    
+    bool isQuitting() const { return (QUITTING == mStatus); }
+    bool isStopped() const { return (STOPPED == mStatus) || (CRASHED == mStatus); }
+    bool isCrashed() const { return (CRASHED == mStatus); } 
+    
+    static U32 currentID(); // Return ID of current thread
+    static void yield(); // Static because it can be called by the main thread, which doesn't have an LLThread data structure.
+    
 public:
-	// PAUSE / RESUME functionality. See source code for important usage notes.
-	// Called from MAIN THREAD.
-	void pause();
-	void unpause();
-	bool isPaused() { return isStopped() || mPaused == TRUE; }
-	
-	// Cause the thread to wake up and check its condition
-	void wake();
-
-	// Same as above, but to be used when the condition is already locked.
-	void wakeLocked();
-
-	// Called from run() (CHILD THREAD). Pause the thread if requested until unpaused.
-	void checkPause();
-
-	// this kicks off the apr thread
-	void start(void);
-
-	apr_pool_t *getAPRPool() { return mAPRPoolp; }
-	LLVolatileAPRPool* getLocalAPRFilePool() { return mLocalAPRFilePoolp ; }
-
-	U32 getID() const { return mID; }
-
-	// Called by threads *not* created via LLThread to register some
-	// internal state used by LLMutex.  You must call this once early
-	// in the running thread to prevent collisions with the main thread.
-	static void registerThreadID();
-	
+    // PAUSE / RESUME functionality. See source code for important usage notes.
+    // Called from MAIN THREAD.
+    void pause();
+    void unpause();
+    bool isPaused() { return isStopped() || mPaused == TRUE; }
+    
+    // Cause the thread to wake up and check its condition
+    void wake();
+
+    // Same as above, but to be used when the condition is already locked.
+    void wakeLocked();
+
+    // Called from run() (CHILD THREAD). Pause the thread if requested until unpaused.
+    void checkPause();
+
+    // this kicks off the apr thread
+    void start(void);
+
+    apr_pool_t *getAPRPool() { return mAPRPoolp; }
+    LLVolatileAPRPool* getLocalAPRFilePool() { return mLocalAPRFilePoolp ; }
+
+    U32 getID() const { return mID; }
+
+    // Called by threads *not* created via LLThread to register some
+    // internal state used by LLMutex.  You must call this once early
+    // in the running thread to prevent collisions with the main thread.
+    static void registerThreadID();
+    
 private:
-	BOOL				mPaused;
-	
-	// static function passed to APR thread creation routine
-	static void *APR_THREAD_FUNC staticRun(struct apr_thread_t *apr_threadp, void *datap);
+    BOOL                mPaused;
+    
+    // static function passed to APR thread creation routine
+    static void *APR_THREAD_FUNC staticRun(struct apr_thread_t *apr_threadp, void *datap);
 
 protected:
-	std::string			mName;
-	class LLCondition*	mRunCondition;
-	LLMutex*			mDataLock;
-
-	apr_thread_t		*mAPRThreadp;
-	apr_pool_t			*mAPRPoolp;
-	BOOL				mIsLocalPool;
-	EThreadStatus		mStatus;
-	U32					mID;
-	LLTrace::ThreadRecorder* mRecorder;
-
-	//a local apr_pool for APRFile operations in this thread. If it exists, LLAPRFile::sAPRFilePoolp should not be used.
-	//Note: this pool is used by APRFile ONLY, do NOT use it for any other purposes.
-	//      otherwise it will cause severe memory leaking!!! --bao
-	LLVolatileAPRPool  *mLocalAPRFilePoolp ; 
-
-	void setQuitting();
-	
-	// virtual function overridden by subclass -- this will be called when the thread runs
-	virtual void run(void) = 0; 
-	
-	// virtual predicate function -- returns true if the thread should wake up, false if it should sleep.
-	virtual bool runCondition(void);
-
-	// Lock/Unlock Run Condition -- use around modification of any variable used in runCondition()
-	inline void lockData();
-	inline void unlockData();
-	
-	// This is the predicate that decides whether the thread should sleep.  
-	// It should only be called with mDataLock locked, since the virtual runCondition() function may need to access
-	// data structures that are thread-unsafe.
-	bool shouldSleep(void) { return (mStatus == RUNNING) && (isPaused() || (!runCondition())); }
-
-	// To avoid spurious signals (and the associated context switches) when the condition may or may not have changed, you can do the following:
-	// mDataLock->lock();
-	// if(!shouldSleep())
-	//     mRunCondition->signal();
-	// mDataLock->unlock();
+    std::string         mName;
+    class LLCondition*  mRunCondition;
+    LLMutex*            mDataLock;
+
+    apr_thread_t        *mAPRThreadp;
+    apr_pool_t          *mAPRPoolp;
+    BOOL                mIsLocalPool;
+    EThreadStatus       mStatus;
+    U32                 mID;
+    LLTrace::ThreadRecorder* mRecorder;
+
+    //a local apr_pool for APRFile operations in this thread. If it exists, LLAPRFile::sAPRFilePoolp should not be used.
+    //Note: this pool is used by APRFile ONLY, do NOT use it for any other purposes.
+    //      otherwise it will cause severe memory leaking!!! --bao
+    LLVolatileAPRPool  *mLocalAPRFilePoolp ; 
+
+    void setQuitting();
+    
+    // virtual function overridden by subclass -- this will be called when the thread runs
+    virtual void run(void) = 0; 
+    
+    // virtual predicate function -- returns true if the thread should wake up, false if it should sleep.
+    virtual bool runCondition(void);
+
+    // Lock/Unlock Run Condition -- use around modification of any variable used in runCondition()
+    inline void lockData();
+    inline void unlockData();
+    
+    // This is the predicate that decides whether the thread should sleep.  
+    // It should only be called with mDataLock locked, since the virtual runCondition() function may need to access
+    // data structures that are thread-unsafe.
+    bool shouldSleep(void) { return (mStatus == RUNNING) && (isPaused() || (!runCondition())); }
+
+    // To avoid spurious signals (and the associated context switches) when the condition may or may not have changed, you can do the following:
+    // mDataLock->lock();
+    // if(!shouldSleep())
+    //     mRunCondition->signal();
+    // mDataLock->unlock();
 };
 
 
 void LLThread::lockData()
 {
-	mDataLock->lock();
+    mDataLock->lock();
 }
 
 void LLThread::unlockData()
 {
-	mDataLock->unlock();
+    mDataLock->unlock();
 }
 
 
@@ -160,9 +162,9 @@ void LLThread::unlockData()
 class LL_COMMON_API LLResponder : public LLThreadSafeRefCount
 {
 protected:
-	virtual ~LLResponder();
+    virtual ~LLResponder();
 public:
-	virtual void completed(bool success) = 0;
+    virtual void completed(bool success) = 0;
 };
 
 //============================================================================
diff --git a/indra/llcorehttp/_httpoprequest.cpp b/indra/llcorehttp/_httpoprequest.cpp
index f35f9848ff..36955f3797 100644
--- a/indra/llcorehttp/_httpoprequest.cpp
+++ b/indra/llcorehttp/_httpoprequest.cpp
@@ -247,6 +247,25 @@ void HttpOpRequest::visitNotifier(HttpRequest * request)
 		response->setHeaders(mReplyHeaders);
         response->setRequestURL(mReqURL);
 
+        std::string method("UNKNOWN");
+
+        if (mReqMethod == HOR_COPY)
+            method = "COPY";
+        else if (mReqMethod == HOR_DELETE)
+            method = "DELETE";
+        else if (mReqMethod == HOR_GET)
+            method = "GET";
+        else if (mReqMethod == HOR_MOVE)
+            method = "MOVE";
+        else if (mReqMethod == HOR_PATCH)
+            method = "PATCH";
+        else if (mReqMethod == HOR_POST)
+            method = "POST";
+        else if (mReqMethod == HOR_PUT)
+            method = "PUT";
+
+        response->setRequestMethod(method);
+
         if (mReplyOffset || mReplyLength)
 		{
 			// Got an explicit offset/length in response
diff --git a/indra/llcorehttp/_httppolicy.cpp b/indra/llcorehttp/_httppolicy.cpp
index 628a5c79e1..a302db8b46 100644
--- a/indra/llcorehttp/_httppolicy.cpp
+++ b/indra/llcorehttp/_httppolicy.cpp
@@ -450,7 +450,7 @@ bool HttpPolicy::stageAfterCompletion(const HttpOpRequest::ptr_t &op)
 
 	op->stageFromActive(mService);
 
-    HTTPStats::instance().recordResultCode(op->mStatus.getStatus());
+    HTTPStats::instance().recordResultCode(op->mStatus.getType());
 	return false;						// not active
 }
 
diff --git a/indra/llcorehttp/httprequest.cpp b/indra/llcorehttp/httprequest.cpp
index d9662c1232..2687f77217 100644
--- a/indra/llcorehttp/httprequest.cpp
+++ b/indra/llcorehttp/httprequest.cpp
@@ -52,7 +52,6 @@ namespace LLCore
 // ====================================
 // HttpRequest Implementation
 // ====================================
-HttpRequest::Statistics HttpRequest::mStatistics;
 
 
 HttpRequest::HttpRequest()
@@ -64,11 +63,7 @@ HttpRequest::HttpRequest()
 
 	mReplyQueue.reset( new HttpReplyQueue() );
 
-    ++mStatistics.mCurrentRequests;
-    ++mStatistics.mTotalRequests;
-
-    
-    LL_WARNS("HTTPRequest") << "New HttpRequest created (outstanding: " << mStatistics.mCurrentRequests << " total: " << mStatistics.mTotalRequests << ")" << LL_ENDL;
+    HTTPStats::instance().recordHTTPRequest();
 }
 
 
@@ -81,9 +76,6 @@ HttpRequest::~HttpRequest()
 	}
 
     mReplyQueue.reset();
-    --mStatistics.mCurrentRequests;
-
-    LL_WARNS("HTTPRequest") << "HttpRequest destroyed (outstanding: " << mStatistics.mCurrentRequests << " total: " << mStatistics.mTotalRequests << ")" << LL_ENDL;
 }
 
 
diff --git a/indra/llcorehttp/httprequest.h b/indra/llcorehttp/httprequest.h
index c958132ae2..a418eb6a7a 100644
--- a/indra/llcorehttp/httprequest.h
+++ b/indra/llcorehttp/httprequest.h
@@ -681,19 +681,6 @@ private:
 	// End Global State
 	// ====================================
 
-    struct Statistics
-    {
-        Statistics():
-            mTotalRequests(0),
-            mCurrentRequests(0)
-        {}
-
-        S32 mTotalRequests;
-        S32 mCurrentRequests;
-    };
-
-    static Statistics mStatistics;
-
 };  // end class HttpRequest
 
 
diff --git a/indra/llcorehttp/httpresponse.h b/indra/llcorehttp/httpresponse.h
index 0bfa4585c7..b834085e5c 100644
--- a/indra/llcorehttp/httpresponse.h
+++ b/indra/llcorehttp/httpresponse.h
@@ -204,6 +204,15 @@ public:
             return mRequestUrl;
         }
 
+    void setRequestMethod(const std::string &method)
+        {
+            mRequestMethod = method;
+        }
+
+    const std::string &getRequestMethod() const
+        {
+            return mRequestMethod;
+        }
 
 protected:
 	// Response data here
@@ -217,6 +226,7 @@ protected:
 	unsigned int		mRetries;
 	unsigned int		m503Retries;
     std::string         mRequestUrl;
+    std::string         mRequestMethod;
 
 	TransferStats::ptr_t	mStats;
 };
diff --git a/indra/llcorehttp/httpstats.cpp b/indra/llcorehttp/httpstats.cpp
index 467d364885..b2de7f51ff 100644
--- a/indra/llcorehttp/httpstats.cpp
+++ b/indra/llcorehttp/httpstats.cpp
@@ -44,6 +44,7 @@ void HTTPStats::resetStats()
     mResutCodes.clear();
     mDataDown.reset();
     mDataUp.reset();
+    mRequests = 0;
 }
 
 
@@ -60,16 +61,41 @@ void HTTPStats::recordResultCode(S32 code)
 
 }
 
+namespace
+{
+    std::string byte_count_converter(F32 bytes)
+    {
+        static const char unit_suffix[] = { 'B', 'K', 'M', 'G' };
+
+        F32 value = bytes;
+        int suffix = 0;
+
+        while ((value > 1024.0) && (suffix < 3))
+        {
+            value /= 1024.0;
+            ++suffix;
+        }
+
+        std::stringstream out;
+
+        out << std::setprecision(4) << value << unit_suffix[suffix];
+
+        return out.str();
+    }
+}
+
 void HTTPStats::dumpStats()
 {
     std::stringstream out;
 
-    out << "HTTPCore Stats" << std::endl;
-    out << "Bytes Sent: " << mDataUp.getSum() << std::endl;
-    out << "Bytes Recv: " << mDataDown.getSum() << std::endl;
+    out << "HTTP DATA SUMMARY" << std::endl;
+    out << "HTTP Transfer counts:" << std::endl;
+    out << "Data Sent: " << byte_count_converter(mDataUp.getSum()) << "   (" << mDataUp.getSum() << ")" << std::endl;
+    out << "Data Recv: " << byte_count_converter(mDataDown.getSum()) << "   (" << mDataDown.getSum() << ")" << std::endl;
+    out << "Total requests: " << mRequests << "(request objects created)" << std::endl;
+    out << std::endl;
     out << "Result Codes:" << std::endl << "--- -----" << std::endl;
 
-
     for (std::map<S32, S32>::iterator it = mResutCodes.begin(); it != mResutCodes.end(); ++it)
     { 
         out << (*it).first << " " << (*it).second << std::endl;
diff --git a/indra/llcorehttp/httpstats.h b/indra/llcorehttp/httpstats.h
index d4460bcfae..2c713cb548 100644
--- a/indra/llcorehttp/httpstats.h
+++ b/indra/llcorehttp/httpstats.h
@@ -55,6 +55,8 @@ namespace LLCore
             mDataUp.push(bytes);
         }
 
+        void    recordHTTPRequest() { ++mRequests; }
+
         void    recordResultCode(S32 code);
 
         void    dumpStats();
@@ -62,6 +64,8 @@ namespace LLCore
         StatsAccumulator mDataDown;
         StatsAccumulator mDataUp;
 
+        S32              mRequests;
+
         std::map<S32, S32> mResutCodes;
     };
 
diff --git a/indra/llmessage/llcorehttputil.cpp b/indra/llmessage/llcorehttputil.cpp
index 9bf38fb336..ac4b1310a4 100644
--- a/indra/llmessage/llcorehttputil.cpp
+++ b/indra/llmessage/llcorehttputil.cpp
@@ -84,7 +84,7 @@ void logMessageSuccess(std::string logAuth, std::string url, std::string message
 
 void logMessageFail(std::string logAuth, std::string url, std::string message)
 {
-    LL_WARNS() << logAuth << " Failure '" << message << "' for " << url << LL_ENDL;
+    LL_WARNS("CoreHTTP") << logAuth << " Failure '" << message << "' for " << url << LL_ENDL;
 }
 
 //=========================================================================
@@ -279,12 +279,10 @@ void HttpCoroHandler::onCompleted(LLCore::HttpHandle handle, LLCore::HttpRespons
         result = LLSD::emptyMap();
         LLCore::HttpStatus::type_enum_t errType = status.getType();
 
-        LL_WARNS()
-            << "\n--------------------------------------------------------------------------\n"
-            << " Error[" << status.toTerseString() << "] cannot access url '" << response->getRequestURL()
-            << "' because " << status.toString()
-            << "\n--------------------------------------------------------------------------"
-            << LL_ENDL;
+        LL_WARNS("CoreHTTP")
+            << " Error[" << status.toTerseString() << "] cannot "<< response->getRequestMethod() 
+            << " to url '" << response->getRequestURL()
+            << "' because " << status.toString() << LL_ENDL;
         if ((errType >= 400) && (errType < 500))
         {
             LLSD body = this->parseBody(response, parseSuccess);
@@ -299,7 +297,6 @@ void HttpCoroHandler::onCompleted(LLCore::HttpHandle handle, LLCore::HttpRespons
                     result = body;
                 }
             }
-
         }
     }
     else
@@ -323,7 +320,7 @@ void HttpCoroHandler::onCompleted(LLCore::HttpHandle handle, LLCore::HttpRespons
         if (getBoolSetting(HTTP_LOGBODY_KEY))
         {
             // commenting out, but keeping since this can be useful for debugging
-            LL_WARNS() << "Returned body=" << std::endl << httpStatus["error_body"].asString() << LL_ENDL;
+            LL_WARNS("CoreHTTP") << "Returned body=" << std::endl << httpStatus["error_body"].asString() << LL_ENDL;
         }
     }
 
@@ -423,7 +420,7 @@ LLSD HttpCoroLLSDHandler::handleSuccess(LLCore::HttpResponse * response, LLCore:
         if (contentType && (HTTP_CONTENT_LLSD_XML == *contentType))
         {
             std::string thebody = LLCoreHttpUtil::responseToString(response);
-            LL_WARNS() << "Failed to deserialize . " << response->getRequestURL() << " [status:" << response->getStatus().toString() << "] "
+            LL_WARNS("CoreHTTP") << "Failed to deserialize . " << response->getRequestURL() << " [status:" << response->getStatus().toString() << "] "
                 << " body: " << thebody << LL_ENDL;
 
             // Replace the status with a new one indicating the failure.
@@ -442,7 +439,7 @@ LLSD HttpCoroLLSDHandler::handleSuccess(LLCore::HttpResponse * response, LLCore:
         if (contentType && (HTTP_CONTENT_LLSD_XML == *contentType))
         {
             std::string thebody = LLCoreHttpUtil::responseToString(response);
-            LL_WARNS() << "Failed to deserialize . " << response->getRequestURL() << " [status:" << response->getStatus().toString() << "] "
+            LL_WARNS("CoreHTTP") << "Failed to deserialize . " << response->getRequestURL() << " [status:" << response->getStatus().toString() << "] "
                 << " body: " << thebody << LL_ENDL;
 
             // Replace the status with a new one indicating the failure.
@@ -1210,7 +1207,7 @@ LLSD HttpCoroutineAdapter::buildImmediateErrorResult(const LLCore::HttpRequest::
     const std::string &url) 
 {
     LLCore::HttpStatus status = request->getStatus();
-    LL_WARNS() << "Error posting to " << url << " Status=" << status.getStatus() <<
+    LL_WARNS("CoreHTTP") << "Error posting to " << url << " Status=" << status.getStatus() <<
         " message = " << status.getMessage() << LL_ENDL;
 
     // Mimic the status results returned from an http error that we had