fix for SH-2845, SH-2846, SH-2847, SH-2851: curl crashes and out-of-memory crashes.

11 files changed, 352 insertions, 78 deletions

diff --git a/indra/llmessage/llcurl.cpp b/indra/llmessage/llcurl.cpp
index e17380fdf5..eab2874596 100644
--- a/indra/llmessage/llcurl.cpp
+++ b/indra/llmessage/llcurl.cpp
@@ -72,10 +72,12 @@
 
 static const U32 EASY_HANDLE_POOL_SIZE		= 5;
 static const S32 MULTI_PERFORM_CALL_REPEAT	= 5;
-static const S32 CURL_REQUEST_TIMEOUT = 30; // seconds
+static const S32 CURL_REQUEST_TIMEOUT = 30; // seconds per operation
 static const S32 MAX_ACTIVE_REQUEST_COUNT = 100;
 
-static 
+static const F32 DEFAULT_MULTI_IDLE_TIME = 120.0f ; //seconds
+static const S32 MAX_NUM_OF_HANDLES = 256 ; //max number of handles, (multi handles and easy handles combined).
+
 // DEBUG //
 S32 gCurlEasyCount = 0;
 S32 gCurlMultiCount = 0;
@@ -87,6 +89,8 @@ std::vector<LLMutex*> LLCurl::sSSLMutex;
 std::string LLCurl::sCAPath;
 std::string LLCurl::sCAFile;
 LLCurlThread* LLCurl::sCurlThread = NULL ;
+LLMutex* LLCurl::sHandleMutexp = NULL ;
+S32      LLCurl::sTotalHandles = 0 ;
 
 void check_curl_code(CURLcode code)
 {
@@ -230,7 +234,7 @@ CURL* LLCurl::Easy::allocEasyHandle()
 
 	if (sFreeHandles.empty())
 	{
-		ret = curl_easy_init();
+		ret = LLCurl::newEasyHandle();
 	}
 	else
 	{
@@ -250,16 +254,27 @@ CURL* LLCurl::Easy::allocEasyHandle()
 //static
 void LLCurl::Easy::releaseEasyHandle(CURL* handle)
 {
+	static const S32 MAX_NUM_FREE_HANDLES = 32 ;
+
 	if (!handle)
 	{
-		llerrs << "handle cannot be NULL!" << llendl;
+		return ; //handle allocation failed.
+		//llerrs << "handle cannot be NULL!" << llendl;
 	}
 
 	LLMutexLock lock(sHandleMutexp) ;
 	if (sActiveHandles.find(handle) != sActiveHandles.end())
 	{
 		sActiveHandles.erase(handle);
-		sFreeHandles.insert(handle);
+
+		if(sFreeHandles.size() < MAX_NUM_FREE_HANDLES)
+		{
+			sFreeHandles.insert(handle);
+		}
+		else
+		{
+			LLCurl::deleteEasyHandle(handle) ;
+		}
 	}
 	else
 	{
@@ -517,8 +532,7 @@ void LLCurl::Easy::prepRequest(const std::string& url,
 }
 
 ////////////////////////////////////////////////////////////////////////////
-LLMutex* LLCurl::Multi::sMultiInitMutexp = NULL ;
-LLCurl::Multi::Multi()
+LLCurl::Multi::Multi(F32 idle_time_out)
 	: mQueued(0),
 	  mErrorCount(0),
 	  mState(STATE_READY),
@@ -527,28 +541,47 @@ LLCurl::Multi::Multi()
 	  mDeletionMutexp(NULL),
 	  mEasyMutexp(NULL)
 {
-	mCurlMultiHandle = initMulti();
+	mCurlMultiHandle = LLCurl::newMultiHandle();
 	if (!mCurlMultiHandle)
 	{
 		llwarns << "curl_multi_init() returned NULL! Easy handles: " << gCurlEasyCount << " Multi handles: " << gCurlMultiCount << llendl;
-		mCurlMultiHandle = initMulti();
+		mCurlMultiHandle = LLCurl::newMultiHandle();
 	}
 	
-	llassert_always(mCurlMultiHandle);	
-
-	if(LLCurl::getCurlThread()->getThreaded())
+	//llassert_always(mCurlMultiHandle);	
+	
+	if(mCurlMultiHandle)
 	{
-		mMutexp = new LLMutex(NULL) ;
-		mDeletionMutexp = new LLMutex(NULL) ;
-		mEasyMutexp = new LLMutex(NULL) ;
-	}
-	LLCurl::getCurlThread()->addMulti(this) ;
+		if(LLCurl::getCurlThread()->getThreaded())
+		{
+			mMutexp = new LLMutex(NULL) ;
+			mDeletionMutexp = new LLMutex(NULL) ;
+			mEasyMutexp = new LLMutex(NULL) ;
+		}
+		LLCurl::getCurlThread()->addMulti(this) ;
+
+		mIdleTimeOut = idle_time_out ;
+		if(mIdleTimeOut < DEFAULT_MULTI_IDLE_TIME)
+		{
+			mIdleTimeOut = DEFAULT_MULTI_IDLE_TIME ;
+		}
 
-	++gCurlMultiCount;
+		++gCurlMultiCount;
+	}
 }
 
 LLCurl::Multi::~Multi()
 {
+	cleanup() ;	
+}
+
+void LLCurl::Multi::cleanup()
+{
+	if(!mCurlMultiHandle)
+	{
+		return ; //nothing to clean.
+	}
+
 	// Clean up active
 	for(easy_active_list_t::iterator iter = mEasyActiveList.begin();
 		iter != mEasyActiveList.end(); ++iter)
@@ -564,7 +597,8 @@ LLCurl::Multi::~Multi()
 	for_each(mEasyFreeList.begin(), mEasyFreeList.end(), DeletePointer());	
 	mEasyFreeList.clear();
 
-	check_curl_multi_code(curl_multi_cleanup(mCurlMultiHandle));
+	check_curl_multi_code(LLCurl::deleteMultiHandle(mCurlMultiHandle));
+	mCurlMultiHandle = NULL ;
 
 	delete mMutexp ;
 	mMutexp = NULL ;
@@ -572,15 +606,13 @@ LLCurl::Multi::~Multi()
 	mDeletionMutexp = NULL ;
 	delete mEasyMutexp ;
 	mEasyMutexp = NULL ;
-
+	
+	mQueued = 0 ;
+	mState = STATE_COMPLETED;
+	
 	--gCurlMultiCount;
-}
-
-CURLM* LLCurl::Multi::initMulti()
-{
-	LLMutexLock lock(sMultiInitMutexp) ;
 
-	return curl_multi_init() ;
+	return ;
 }
 
 void LLCurl::Multi::lock()
@@ -604,6 +636,7 @@ void LLCurl::Multi::markDead()
 	LLMutexLock lock(mDeletionMutexp) ;
 	
 	mDead = TRUE ;
+	LLCurl::getCurlThread()->setPriority(mHandle, LLQueuedThread::PRIORITY_URGENT) ; 
 }
 
 void LLCurl::Multi::setState(LLCurl::Multi::ePerformState state)
@@ -630,6 +663,11 @@ bool LLCurl::Multi::isCompleted()
 
 bool LLCurl::Multi::waitToComplete()
 {
+	if(!isValid())
+	{
+		return true ;
+	}
+
 	if(!mMutexp) //not threaded
 	{
 		doPerform() ;
@@ -639,7 +677,7 @@ bool LLCurl::Multi::waitToComplete()
 	bool completed = (STATE_COMPLETED == mState) ;
 	if(!completed)
 	{
-		LLCurl::getCurlThread()->setPriority(mHandle, LLQueuedThread::PRIORITY_URGENT) ;
+		LLCurl::getCurlThread()->setPriority(mHandle, LLQueuedThread::PRIORITY_HIGH) ;
 	}
 	
 	return completed;
@@ -689,7 +727,12 @@ bool LLCurl::Multi::doPerform()
 		}
 
 		mQueued = q;	
-		setState(STATE_COMPLETED) ;
+		setState(STATE_COMPLETED) ;		
+		mIdleTimer.reset() ;
+	}
+	else if(mIdleTimer.getElapsedTimeF32() > mIdleTimeOut) //idle for too long, remove it.
+	{
+		dead = true ;
 	}
 
 	return dead ;
@@ -697,6 +740,11 @@ bool LLCurl::Multi::doPerform()
 
 S32 LLCurl::Multi::process()
 {
+	if(!isValid())
+	{
+		return 0 ;
+	}
+
 	waitToComplete() ;
 
 	if (getState() != STATE_COMPLETED)
@@ -849,7 +897,11 @@ bool LLCurlThread::CurlRequest::processRequest()
 	if(mMulti)
 	{
 		completed = mCurlThread->doMultiPerform(mMulti) ;
-		setPriority(LLQueuedThread::PRIORITY_LOW) ;
+
+		if(!completed)
+		{
+			setPriority(LLQueuedThread::PRIORITY_LOW) ;
+		}
 	}
 
 	return completed ;
@@ -857,24 +909,26 @@ bool LLCurlThread::CurlRequest::processRequest()
 
 void LLCurlThread::CurlRequest::finishRequest(bool completed)
 {
-	mCurlThread->deleteMulti(mMulti) ;
+	if(mMulti->isDead())
+	{
+		mCurlThread->deleteMulti(mMulti) ;
+	}
+	else
+	{
+		mMulti->cleanup() ; //being idle too long, remove the request.
+	}
+
 	mMulti = NULL ;
 }
 	
 LLCurlThread::LLCurlThread(bool threaded) :
 	LLQueuedThread("curlthread", threaded)
 {
-	if(!LLCurl::Multi::sMultiInitMutexp)
-	{
-		LLCurl::Multi::sMultiInitMutexp = new LLMutex(NULL) ;
-	}
 }
 	
 //virtual 
 LLCurlThread::~LLCurlThread() 
 {
-	delete LLCurl::Multi::sMultiInitMutexp ;
-	LLCurl::Multi::sMultiInitMutexp = NULL ;
 }
 
 S32 LLCurlThread::update(F32 max_time_ms)
@@ -896,7 +950,19 @@ void LLCurlThread::addMulti(LLCurl::Multi* multi)
 	
 void LLCurlThread::killMulti(LLCurl::Multi* multi)
 {
-	multi->markDead() ;
+	if(!multi)
+	{
+		return ;
+	}
+
+	if(multi->isValid())
+	{
+		multi->markDead() ;
+	}
+	else
+	{
+		deleteMulti(multi) ;
+	}
 }
 
 //private
@@ -942,7 +1008,14 @@ LLCurlRequest::~LLCurlRequest()
 void LLCurlRequest::addMulti()
 {
 	LLCurl::Multi* multi = new LLCurl::Multi();
-	
+	if(!multi->isValid())
+	{
+		LLCurl::getCurlThread()->killMulti(multi) ;
+		mActiveMulti = NULL ;
+		mActiveRequestCount = 0 ;
+		return;
+	}
+
 	mMultiSet.insert(multi);
 	mActiveMulti = multi;
 	mActiveRequestCount = 0;
@@ -1066,6 +1139,19 @@ S32 LLCurlRequest::process()
 	{
 		curlmulti_set_t::iterator curiter = iter++;
 		LLCurl::Multi* multi = *curiter;
+
+		if(!multi->isValid())
+		{
+			if(multi == mActiveMulti)
+			{				
+				mActiveMulti = NULL ;
+				mActiveRequestCount = 0 ;
+			}
+			mMultiSet.erase(curiter) ;
+			LLCurl::getCurlThread()->killMulti(multi) ;
+			continue ;
+		}
+
 		S32 tres = multi->process();
 		res += tres;
 		if (multi != mActiveMulti && tres == 0 && multi->mQueued == 0)
@@ -1086,6 +1172,19 @@ S32 LLCurlRequest::getQueued()
 	{
 		curlmulti_set_t::iterator curiter = iter++;
 		LLCurl::Multi* multi = *curiter;
+		
+		if(!multi->isValid())
+		{
+			if(multi == mActiveMulti)
+			{				
+				mActiveMulti = NULL ;
+				mActiveRequestCount = 0 ;
+			}
+			LLCurl::getCurlThread()->killMulti(multi);
+			mMultiSet.erase(curiter) ;
+			continue ;
+		}
+
 		queued += multi->mQueued;
 		if (multi->getState() != LLCurl::Multi::STATE_READY)
 		{
@@ -1105,13 +1204,22 @@ LLCurlEasyRequest::LLCurlEasyRequest()
 {
 	mMulti = new LLCurl::Multi();
 	
-	mEasy = mMulti->allocEasy();
-	if (mEasy)
+	if(mMulti->isValid())
 	{
-		mEasy->setErrorBuffer();
-		mEasy->setCA();
-		// Set proxy settings if configured to do so.
-		LLProxy::getInstance()->applyProxySettings(mEasy);
+		mEasy = mMulti->allocEasy();
+		if (mEasy)
+		{
+			mEasy->setErrorBuffer();
+			mEasy->setCA();
+			// Set proxy settings if configured to do so.
+			LLProxy::getInstance()->applyProxySettings(mEasy);
+		}
+	}
+	else
+	{
+		LLCurl::getCurlThread()->killMulti(mMulti) ;
+		mEasy = NULL ;
+		mMulti = NULL ;
 	}
 }
 
@@ -1122,7 +1230,7 @@ LLCurlEasyRequest::~LLCurlEasyRequest()
 	
 void LLCurlEasyRequest::setopt(CURLoption option, S32 value)
 {
-	if (mEasy)
+	if (isValid() && mEasy)
 	{
 		mEasy->setopt(option, value);
 	}
@@ -1130,7 +1238,7 @@ void LLCurlEasyRequest::setopt(CURLoption option, S32 value)
 
 void LLCurlEasyRequest::setoptString(CURLoption option, const std::string& value)
 {
-	if (mEasy)
+	if (isValid() && mEasy)
 	{
 		mEasy->setoptString(option, value);
 	}
@@ -1138,7 +1246,7 @@ void LLCurlEasyRequest::setoptString(CURLoption option, const std::string& value
 
 void LLCurlEasyRequest::setPost(char* postdata, S32 size)
 {
-	if (mEasy)
+	if (isValid() && mEasy)
 	{
 		mEasy->setopt(CURLOPT_POST, 1);
 		mEasy->setopt(CURLOPT_POSTFIELDS, postdata);
@@ -1148,7 +1256,7 @@ void LLCurlEasyRequest::setPost(char* postdata, S32 size)
 
 void LLCurlEasyRequest::setHeaderCallback(curl_header_callback callback, void* userdata)
 {
-	if (mEasy)
+	if (isValid() && mEasy)
 	{
 		mEasy->setopt(CURLOPT_HEADERFUNCTION, (void*)callback);
 		mEasy->setopt(CURLOPT_HEADERDATA, userdata); // aka CURLOPT_WRITEHEADER
@@ -1157,7 +1265,7 @@ void LLCurlEasyRequest::setHeaderCallback(curl_header_callback callback, void* u
 
 void LLCurlEasyRequest::setWriteCallback(curl_write_callback callback, void* userdata)
 {
-	if (mEasy)
+	if (isValid() && mEasy)
 	{
 		mEasy->setopt(CURLOPT_WRITEFUNCTION, (void*)callback);
 		mEasy->setopt(CURLOPT_WRITEDATA, userdata);
@@ -1166,7 +1274,7 @@ void LLCurlEasyRequest::setWriteCallback(curl_write_callback callback, void* use
 
 void LLCurlEasyRequest::setReadCallback(curl_read_callback callback, void* userdata)
 {
-	if (mEasy)
+	if (isValid() && mEasy)
 	{
 		mEasy->setopt(CURLOPT_READFUNCTION, (void*)callback);
 		mEasy->setopt(CURLOPT_READDATA, userdata);
@@ -1175,7 +1283,7 @@ void LLCurlEasyRequest::setReadCallback(curl_read_callback callback, void* userd
 
 void LLCurlEasyRequest::setSSLCtxCallback(curl_ssl_ctx_callback callback, void* userdata)
 {
-	if (mEasy)
+	if (isValid() && mEasy)
 	{
 		mEasy->setopt(CURLOPT_SSL_CTX_FUNCTION, (void*)callback);
 		mEasy->setopt(CURLOPT_SSL_CTX_DATA, userdata);
@@ -1184,7 +1292,7 @@ void LLCurlEasyRequest::setSSLCtxCallback(curl_ssl_ctx_callback callback, void*
 
 void LLCurlEasyRequest::slist_append(const char* str)
 {
-	if (mEasy)
+	if (isValid() && mEasy)
 	{
 		mEasy->slist_append(str);
 	}
@@ -1195,7 +1303,7 @@ void LLCurlEasyRequest::sendRequest(const std::string& url)
 	llassert_always(!mRequestSent);
 	mRequestSent = true;
 	lldebugs << url << llendl;
-	if (mEasy)
+	if (isValid() && mEasy)
 	{
 		mEasy->setHeaders();
 		mEasy->setoptString(CURLOPT_URL, url);
@@ -1207,7 +1315,7 @@ void LLCurlEasyRequest::requestComplete()
 {
 	llassert_always(mRequestSent);
 	mRequestSent = false;
-	if (mEasy)
+	if (isValid() && mEasy)
 	{
 		mMulti->removeEasy(mEasy);
 	}
@@ -1216,6 +1324,10 @@ void LLCurlEasyRequest::requestComplete()
 // Usage: Call getRestult until it returns false (no more messages)
 bool LLCurlEasyRequest::getResult(CURLcode* result, LLCurl::TransferInfo* info)
 {
+	if(!isValid())
+	{
+		return false ;
+	}
 	if (!mMulti->isCompleted())
 	{ //we're busy, try again later
 		return false;
@@ -1280,7 +1392,7 @@ CURLMsg* LLCurlEasyRequest::info_read(S32* q, LLCurl::TransferInfo* info)
 
 std::string LLCurlEasyRequest::getErrorString()
 {
-	return mEasy ? std::string(mEasy->getErrorBuffer()) : std::string();
+	return isValid() &&  mEasy ? std::string(mEasy->getErrorBuffer()) : std::string();
 }
 
 ////////////////////////////////////////////////////////////////////////////
@@ -1328,6 +1440,7 @@ void LLCurl::initClass(bool multi_threaded)
 	sCurlThread = new LLCurlThread(multi_threaded) ;
 	if(multi_threaded)
 	{
+		sHandleMutexp = new LLMutex(NULL) ;
 		Easy::sHandleMutexp = new LLMutex(NULL) ;
 	}
 }
@@ -1354,7 +1467,7 @@ void LLCurl::cleanupClass()
 	for (std::set<CURL*>::iterator iter = Easy::sFreeHandles.begin(); iter != Easy::sFreeHandles.end(); ++iter)
 	{
 		CURL* curl = *iter;
-		curl_easy_cleanup(curl);
+		LLCurl::deleteEasyHandle(curl);
 	}
 
 	Easy::sFreeHandles.clear();
@@ -1362,9 +1475,77 @@ void LLCurl::cleanupClass()
 	delete Easy::sHandleMutexp ;
 	Easy::sHandleMutexp = NULL ;
 
+	delete sHandleMutexp ;
+	sHandleMutexp = NULL ;
+
 	llassert(Easy::sActiveHandles.empty());
 }
 
+//static 
+CURLM* LLCurl::newMultiHandle()
+{
+	LLMutexLock lock(sHandleMutexp) ;
+
+	if(sTotalHandles + 1 > MAX_NUM_OF_HANDLES)
+	{
+		llwarns << "no more handles available." << llendl ;
+		return NULL ; //failed
+	}
+	sTotalHandles++;
+
+	CURLM* ret = curl_multi_init() ;
+	if(!ret)
+	{
+		llwarns << "curl_multi_init failed." << llendl ;
+	}
+
+	return ret ;
+}
+
+//static 
+CURLMcode  LLCurl::deleteMultiHandle(CURLM* handle)
+{
+	if(handle)
+	{
+		LLMutexLock lock(sHandleMutexp) ;		
+		sTotalHandles-- ;
+		return curl_multi_cleanup(handle) ;
+	}
+	return CURLM_OK ;
+}
+
+//static 
+CURL*  LLCurl::newEasyHandle()
+{
+	LLMutexLock lock(sHandleMutexp) ;
+
+	if(sTotalHandles + 1 > MAX_NUM_OF_HANDLES)
+	{
+		llwarns << "no more handles available." << llendl ;
+		return NULL ; //failed
+	}
+	sTotalHandles++;
+
+	CURL* ret = curl_easy_init() ;
+	if(!ret)
+	{
+		llwarns << "curl_easy_init failed." << llendl ;
+	}
+
+	return ret ;
+}
+
+//static 
+void  LLCurl::deleteEasyHandle(CURL* handle)
+{
+	if(handle)
+	{
+		LLMutexLock lock(sHandleMutexp) ;
+		curl_easy_cleanup(handle) ;
+		sTotalHandles-- ;
+	}
+}
+
 const unsigned int LLCurl::MAX_REDIRECTS = 5;
 
 // Provide access to LLCurl free functions outside of llcurl.cpp without polluting the global namespace.
diff --git a/indra/llmessage/llcurl.h b/indra/llmessage/llcurl.h
index 9c2c215c7a..32da911cbf 100644
--- a/indra/llmessage/llcurl.h
+++ b/indra/llmessage/llcurl.h
@@ -43,6 +43,7 @@
 #include "llsd.h"
 #include "llthread.h"
 #include "llqueuedthread.h"
+#include "llframetimer.h"
 
 class LLMutex;
 class LLCurlThread;
@@ -182,11 +183,20 @@ public:
 	static unsigned long ssl_thread_id(void);
 
 	static LLCurlThread* getCurlThread() { return sCurlThread ;}
+
+	static CURLM* newMultiHandle() ;
+	static CURLMcode deleteMultiHandle(CURLM* handle) ;
+	static CURL*  newEasyHandle() ;
+	static void   deleteEasyHandle(CURL* handle) ;
+
 private:
 	static std::string sCAPath;
 	static std::string sCAFile;
 	static const unsigned int MAX_REDIRECTS;
 	static LLCurlThread* sCurlThread;
+
+	static LLMutex* sHandleMutexp ;
+	static S32      sTotalHandles ;
 };
 
 class LLCurl::Easy
@@ -277,7 +287,7 @@ public:
 		STATE_COMPLETED=2
 	} ePerformState;
 
-	Multi();	
+	Multi(F32 idle_time_out = 0.f);	
 
 	LLCurl::Easy* allocEasy();
 	bool addEasy(LLCurl::Easy* easy);	
@@ -288,7 +298,10 @@ public:
 
 	void setState(ePerformState state) ;
 	ePerformState getState() ;
+	
 	bool isCompleted() ;
+	bool isValid() {return mCurlMultiHandle != NULL ;}
+	bool isDead() {return mDead;}
 
 	bool waitToComplete() ;
 
@@ -299,9 +312,9 @@ public:
 	S32 mQueued;
 	S32 mErrorCount;
 	
-	static CURLM* initMulti() ;
 private:
 	void easyFree(LLCurl::Easy*);
+	void cleanup() ;
 	
 	CURLM* mCurlMultiHandle;
 
@@ -319,8 +332,8 @@ private:
 	LLMutex* mMutexp ;
 	LLMutex* mDeletionMutexp ;
 	LLMutex* mEasyMutexp ;
-
-	static LLMutex* sMultiInitMutexp ;
+	LLFrameTimer mIdleTimer ;
+	F32 mIdleTimeOut;
 };
 
 class LLCurlThread : public LLQueuedThread
@@ -414,6 +427,7 @@ public:
 	std::string getErrorString();
 	bool isCompleted() {return mMulti->isCompleted() ;}
 	bool wait() { return mMulti->waitToComplete(); }
+	bool isValid() {return mMulti && mMulti->isValid(); }
 
 	LLCurl::Easy* getEasy() const { return mEasy; }
 
diff --git a/indra/llmessage/llhttpassetstorage.cpp b/indra/llmessage/llhttpassetstorage.cpp
index 2bca517e97..612d765969 100644
--- a/indra/llmessage/llhttpassetstorage.cpp
+++ b/indra/llmessage/llhttpassetstorage.cpp
@@ -232,7 +232,8 @@ LLSD LLHTTPAssetRequest::getFullDetails() const
 void LLHTTPAssetRequest::setupCurlHandle()
 {
 	// *NOTE: Similar code exists in mapserver/llcurlutil.cpp  JC
-	mCurlHandle = curl_easy_init();
+	mCurlHandle = LLCurl::newEasyHandle();
+	llassert_always(mCurlHandle != NULL) ;
 
 	// Apply proxy settings if configured to do so
 	LLProxy::getInstance()->applyProxySettings(mCurlHandle);
@@ -278,7 +279,7 @@ void LLHTTPAssetRequest::setupCurlHandle()
 
 void LLHTTPAssetRequest::cleanupCurlHandle()
 {
-	curl_easy_cleanup(mCurlHandle);
+	LLCurl::deleteEasyHandle(mCurlHandle);
 	if (mAssetStoragep)
 	{
 		// Terminating a request.  Thus upload or download is no longer pending.
@@ -429,12 +430,13 @@ void LLHTTPAssetStorage::_init(const std::string& web_host, const std::string& l
 
 	// curl_global_init moved to LLCurl::initClass()
 	
-	mCurlMultiHandle = curl_multi_init();
+	mCurlMultiHandle = LLCurl::newMultiHandle() ;
+	llassert_always(mCurlMultiHandle != NULL) ;
 }
 
 LLHTTPAssetStorage::~LLHTTPAssetStorage()
 {
-	curl_multi_cleanup(mCurlMultiHandle);
+	LLCurl::deleteMultiHandle(mCurlMultiHandle);
 	mCurlMultiHandle = NULL;
 	
 	// curl_global_cleanup moved to LLCurl::initClass()
diff --git a/indra/llmessage/llhttpclient.cpp b/indra/llmessage/llhttpclient.cpp
index dd4e3a6300..231cb7ca8f 100644
--- a/indra/llmessage/llhttpclient.cpp
+++ b/indra/llmessage/llhttpclient.cpp
@@ -228,6 +228,12 @@ static void request(
 	LLPumpIO::chain_t chain;
 
 	LLURLRequest* req = new LLURLRequest(method, url);
+	if(!req->isValid())//failed
+	{
+		delete req ;
+		return ;
+	}
+
 	req->setSSLVerifyCallback(LLHTTPClient::getCertVerifyCallback(), (void *)req);
 
 	
@@ -423,7 +429,9 @@ static LLSD blocking_request(
 {
 	lldebugs << "blockingRequest of " << url << llendl;
 	char curl_error_buffer[CURL_ERROR_SIZE] = "\0";
-	CURL* curlp = curl_easy_init();
+	CURL* curlp = LLCurl::newEasyHandle();
+	llassert_always(curlp != NULL) ;
+
 	LLHTTPBuffer http_buffer;
 	std::string body_str;
 	
@@ -517,7 +525,7 @@ static LLSD blocking_request(
 	}
 
 	// * Cleanup
-	curl_easy_cleanup(curlp);
+	LLCurl::deleteEasyHandle(curlp);
 	return response;
 }
 
diff --git a/indra/llmessage/lliopipe.cpp b/indra/llmessage/lliopipe.cpp
index 6e4eec74a6..8f827f7a30 100644
--- a/indra/llmessage/lliopipe.cpp
+++ b/indra/llmessage/lliopipe.cpp
@@ -75,6 +75,12 @@ LLIOPipe::~LLIOPipe()
 	//lldebugs << "destroying LLIOPipe" << llendl;
 }
 
+//virtual 
+bool LLIOPipe::isValid() 
+{
+	return true ;
+}
+
 // static
 std::string LLIOPipe::lookupStatusString(EStatus status)
 {
diff --git a/indra/llmessage/lliopipe.h b/indra/llmessage/lliopipe.h
index 8e656b6da1..cbd17b5a3d 100644
--- a/indra/llmessage/lliopipe.h
+++ b/indra/llmessage/lliopipe.h
@@ -231,6 +231,8 @@ public:
 	 */
 	virtual ~LLIOPipe();
 
+	virtual bool isValid() ;
+
 protected:
 	/**
 	 * @brief Base Constructor.
diff --git a/indra/llmessage/llpumpio.cpp b/indra/llmessage/llpumpio.cpp
index a8d2a0a224..0ff300efd0 100644
--- a/indra/llmessage/llpumpio.cpp
+++ b/indra/llmessage/llpumpio.cpp
@@ -195,7 +195,7 @@ bool LLPumpIO::prime(apr_pool_t* pool)
 	return ((pool == NULL) ? false : true);
 }
 
-bool LLPumpIO::addChain(const chain_t& chain, F32 timeout)
+bool LLPumpIO::addChain(const chain_t& chain, F32 timeout, bool has_curl_request)
 {
 	LLMemType m1(LLMemType::MTYPE_IO_PUMP);
 	if(chain.empty()) return false;
@@ -204,6 +204,7 @@ bool LLPumpIO::addChain(const chain_t& chain, F32 timeout)
 	LLScopedLock lock(mChainsMutex);
 #endif
 	LLChainInfo info;
+	info.mHasCurlRequest = has_curl_request;
 	info.setTimeoutSeconds(timeout);
 	info.mData = LLIOPipe::buffer_ptr_t(new LLBufferArray);
 	LLLinkInfo link;
@@ -440,6 +441,15 @@ void LLPumpIO::pump()
 
 static LLFastTimer::DeclareTimer FTM_PUMP_IO("Pump IO");
 
+LLPumpIO::current_chain_t LLPumpIO::removeRunningChain(LLPumpIO::current_chain_t& run_chain) 
+{
+	std::for_each(
+				(*run_chain).mDescriptors.begin(),
+				(*run_chain).mDescriptors.end(),
+				ll_delete_apr_pollset_fd_client_data());
+	return mRunningChains.erase(run_chain);
+}
+
 //timeout is in microseconds
 void LLPumpIO::pump(const S32& poll_timeout)
 {
@@ -585,10 +595,16 @@ void LLPumpIO::pump(const S32& poll_timeout)
 //						<< (*run_chain).mChainLinks[0].mPipe
 //						<< " because we reached the end." << llendl;
 #endif
-				run_chain = mRunningChains.erase(run_chain);
+				run_chain = removeRunningChain(run_chain);
 				continue;
 			}
 		}
+		else if(isChainExpired(*run_chain))
+		{
+			run_chain = removeRunningChain(run_chain);
+			continue;
+		}
+
 		PUMP_DEBUG;
 		if((*run_chain).mLock)
 		{
@@ -696,11 +712,7 @@ void LLPumpIO::pump(const S32& poll_timeout)
 			PUMP_DEBUG;
 			// This chain is done. Clean up any allocated memory and
 			// erase the chain info.
-			std::for_each(
-				(*run_chain).mDescriptors.begin(),
-				(*run_chain).mDescriptors.end(),
-				ll_delete_apr_pollset_fd_client_data());
-			run_chain = mRunningChains.erase(run_chain);
+			run_chain = removeRunningChain(run_chain);
 
 			// *NOTE: may not always need to rebuild the pollset.
 			mRebuildPollset = true;
@@ -1095,6 +1107,24 @@ void LLPumpIO::processChain(LLChainInfo& chain)
 	PUMP_DEBUG;
 }
 
+bool LLPumpIO::isChainExpired(LLChainInfo& chain)
+{
+	if(!chain.mHasCurlRequest)
+	{
+		return false ;
+	}
+
+	for(links_t::iterator iter = chain.mChainLinks.begin(); iter != chain.mChainLinks.end(); ++iter)
+	{
+		if(!(*iter).mPipe->isValid())
+		{
+			return true ;
+		}
+	}
+
+	return false ;
+}
+
 bool LLPumpIO::handleChainError(
 	LLChainInfo& chain,
 	LLIOPipe::EStatus error)
@@ -1136,6 +1166,9 @@ bool LLPumpIO::handleChainError(
 #endif
 			keep_going = false;
 			break;
+		case LLIOPipe::STATUS_EXPIRED:
+			keep_going = false;
+			break ;
 		default:
 			if(LLIOPipe::isSuccess(error))
 			{
@@ -1157,7 +1190,8 @@ bool LLPumpIO::handleChainError(
 LLPumpIO::LLChainInfo::LLChainInfo() :
 	mInit(false),
 	mLock(0),
-	mEOS(false)
+	mEOS(false),
+	mHasCurlRequest(false)
 {
 	LLMemType m1(LLMemType::MTYPE_IO_PUMP);
 	mTimer.setTimerExpirySec(DEFAULT_CHAIN_EXPIRY_SECS);
diff --git a/indra/llmessage/llpumpio.h b/indra/llmessage/llpumpio.h
index 9303c9d7fc..e405124403 100644
--- a/indra/llmessage/llpumpio.h
+++ b/indra/llmessage/llpumpio.h
@@ -113,7 +113,7 @@ public:
 	 * expire. Pass in 0.0f to never expire.
 	 * @return Returns true if anything was added to the pump.
 	 */
-	bool addChain(const chain_t& chain, F32 timeout);
+	bool addChain(const chain_t& chain, F32 timeout, bool has_curl_request = false);
 	
 	/** 
 	 * @brief Struct to associate a pipe with it's buffer io indexes.
@@ -356,12 +356,13 @@ protected:
 
 		// basic member data
 		bool mInit;
+		bool mEOS;
+		bool mHasCurlRequest;
 		S32 mLock;
 		LLFrameTimer mTimer;
 		links_t::iterator mHead;
 		links_t mChainLinks;
-		LLIOPipe::buffer_ptr_t mData;
-		bool mEOS;
+		LLIOPipe::buffer_ptr_t mData;		
 		LLSD mContext;
 
 		// tracking inside the pump
@@ -402,7 +403,7 @@ protected:
 protected:
 	void initialize(apr_pool_t* pool);
 	void cleanup();
-
+	current_chain_t removeRunningChain(current_chain_t& chain) ;
 	/** 
 	 * @brief Given the internal state of the chains, rebuild the pollset
 	 * @see setConditional()
@@ -429,6 +430,9 @@ protected:
 	 */
 	bool handleChainError(LLChainInfo& chain, LLIOPipe::EStatus error);
 
+	//if the chain is expired, remove it
+	bool isChainExpired(LLChainInfo& chain) ;
+
 public:
 	/** 
 	 * @brief Return number of running chains.
diff --git a/indra/llmessage/llsdrpcclient.h b/indra/llmessage/llsdrpcclient.h
index 9fb49a5c33..235077179e 100644
--- a/indra/llmessage/llsdrpcclient.h
+++ b/indra/llmessage/llsdrpcclient.h
@@ -239,6 +239,8 @@ public:
 	LLSDRPCClientFactory(const std::string& fixed_url) : mURL(fixed_url) {}
 	virtual bool build(LLPumpIO::chain_t& chain, LLSD context) const
 	{
+		llerrs << "Can not call this." << llendl ;
+
 		lldebugs << "LLSDRPCClientFactory::build" << llendl;
 		LLIOPipe::ptr_t service(new Client);
 		chain.push_back(service);
@@ -282,6 +284,7 @@ public:
 	LLXMLSDRPCClientFactory(const std::string& fixed_url) : mURL(fixed_url) {}
 	virtual bool build(LLPumpIO::chain_t& chain, LLSD context) const
 	{
+		llerrs << "who calls this?" << llendl ;
 		lldebugs << "LLXMLSDRPCClientFactory::build" << llendl;
 		LLIOPipe::ptr_t service(new Client);
 		chain.push_back(service);
diff --git a/indra/llmessage/llurlrequest.cpp b/indra/llmessage/llurlrequest.cpp
index 261e57e79e..f02c636838 100644
--- a/indra/llmessage/llurlrequest.cpp
+++ b/indra/llmessage/llurlrequest.cpp
@@ -83,6 +83,12 @@ LLURLRequestDetail::LLURLRequestDetail() :
 {
 	LLMemType m1(LLMemType::MTYPE_IO_URL_REQUEST);
 	mCurlRequest = new LLCurlEasyRequest();
+	
+	if(!mCurlRequest->isValid()) //failed.
+	{
+		delete mCurlRequest ;
+		mCurlRequest = NULL ;
+	}
 }
 
 LLURLRequestDetail::~LLURLRequestDetail()
@@ -250,12 +256,24 @@ void LLURLRequest::allowCookies()
 	mDetail->mCurlRequest->setoptString(CURLOPT_COOKIEFILE, "");
 }
 
+//virtual 
+bool LLURLRequest::isValid() 
+{
+	return mDetail->mCurlRequest && mDetail->mCurlRequest->isValid(); 
+}
+
 // virtual
 LLIOPipe::EStatus LLURLRequest::handleError(
 	LLIOPipe::EStatus status,
 	LLPumpIO* pump)
 {
 	LLMemType m1(LLMemType::MTYPE_IO_URL_REQUEST);
+	
+	if(!isValid())
+	{
+		return STATUS_EXPIRED ;
+	}
+
 	if(mCompletionCallback && pump)
 	{
 		LLURLRequestComplete* complete = NULL;
diff --git a/indra/llmessage/llurlrequest.h b/indra/llmessage/llurlrequest.h
index ec5c2c1941..44d358d906 100644
--- a/indra/llmessage/llurlrequest.h
+++ b/indra/llmessage/llurlrequest.h
@@ -188,6 +188,8 @@ public:
 	 */
 	void allowCookies();
 
+	/*virtual*/ bool isValid() ;
+
 public:
 	/** 
 	 * @brief Give this pipe a chance to handle a generated error