summaryrefslogtreecommitdiff
path: root/indra/llmessage/llcurl.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'indra/llmessage/llcurl.cpp')
-rw-r--r--indra/llmessage/llcurl.cpp331
1 files changed, 272 insertions, 59 deletions
diff --git a/indra/llmessage/llcurl.cpp b/indra/llmessage/llcurl.cpp
index f569630766..3bcaffc275 100644
--- a/indra/llmessage/llcurl.cpp
+++ b/indra/llmessage/llcurl.cpp
@@ -72,10 +72,9 @@
static const U32 EASY_HANDLE_POOL_SIZE = 5;
static const S32 MULTI_PERFORM_CALL_REPEAT = 5;
-static const S32 CURL_REQUEST_TIMEOUT = 30; // seconds
+static const S32 CURL_REQUEST_TIMEOUT = 30; // seconds per operation
static const S32 MAX_ACTIVE_REQUEST_COUNT = 100;
-static
// DEBUG //
S32 gCurlEasyCount = 0;
S32 gCurlMultiCount = 0;
@@ -87,6 +86,11 @@ std::vector<LLMutex*> LLCurl::sSSLMutex;
std::string LLCurl::sCAPath;
std::string LLCurl::sCAFile;
LLCurlThread* LLCurl::sCurlThread = NULL ;
+LLMutex* LLCurl::sHandleMutexp = NULL ;
+S32 LLCurl::sTotalHandles = 0 ;
+bool LLCurl::sNotQuitting = true;
+F32 LLCurl::sCurlRequestTimeOut = 120.f; //seonds
+S32 LLCurl::sMaxHandles = 256; //max number of handles, (multi handles and easy handles combined).
void check_curl_code(CURLcode code)
{
@@ -224,13 +228,15 @@ LLMutex* LLCurl::Easy::sHandleMutexp = NULL ;
//static
CURL* LLCurl::Easy::allocEasyHandle()
{
+ llassert(LLCurl::getCurlThread()) ;
+
CURL* ret = NULL;
LLMutexLock lock(sHandleMutexp) ;
if (sFreeHandles.empty())
{
- ret = curl_easy_init();
+ ret = LLCurl::newEasyHandle();
}
else
{
@@ -250,16 +256,27 @@ CURL* LLCurl::Easy::allocEasyHandle()
//static
void LLCurl::Easy::releaseEasyHandle(CURL* handle)
{
+ static const S32 MAX_NUM_FREE_HANDLES = 32 ;
+
if (!handle)
{
- llerrs << "handle cannot be NULL!" << llendl;
+ return ; //handle allocation failed.
+ //llerrs << "handle cannot be NULL!" << llendl;
}
LLMutexLock lock(sHandleMutexp) ;
if (sActiveHandles.find(handle) != sActiveHandles.end())
{
sActiveHandles.erase(handle);
- sFreeHandles.insert(handle);
+
+ if(sFreeHandles.size() < MAX_NUM_FREE_HANDLES)
+ {
+ sFreeHandles.insert(handle);
+ }
+ else
+ {
+ LLCurl::deleteEasyHandle(handle) ;
+ }
}
else
{
@@ -302,6 +319,14 @@ LLCurl::Easy::~Easy()
--gCurlEasyCount;
curl_slist_free_all(mHeaders);
for_each(mStrings.begin(), mStrings.end(), DeletePointerArray());
+
+ if (mResponder && LLCurl::sNotQuitting) //aborted
+ {
+ std::string reason("Request timeout, aborted.") ;
+ mResponder->completedRaw(408, //HTTP_REQUEST_TIME_OUT, timeout, abort
+ reason, mChannels, mOutput);
+ }
+ mResponder = NULL;
}
void LLCurl::Easy::resetState()
@@ -474,6 +499,7 @@ void LLCurl::Easy::prepRequest(const std::string& url,
LLProxy::getInstance()->applyProxySettings(this);
mOutput.reset(new LLBufferArray);
+ mOutput->setThreaded(true);
setopt(CURLOPT_WRITEFUNCTION, (void*)&curlWriteCallback);
setopt(CURLOPT_WRITEDATA, (void*)this);
@@ -517,8 +543,7 @@ void LLCurl::Easy::prepRequest(const std::string& url,
}
////////////////////////////////////////////////////////////////////////////
-LLMutex* LLCurl::Multi::sMultiInitMutexp = NULL ;
-LLCurl::Multi::Multi()
+LLCurl::Multi::Multi(F32 idle_time_out)
: mQueued(0),
mErrorCount(0),
mState(STATE_READY),
@@ -527,28 +552,47 @@ LLCurl::Multi::Multi()
mDeletionMutexp(NULL),
mEasyMutexp(NULL)
{
- mCurlMultiHandle = initMulti();
+ mCurlMultiHandle = LLCurl::newMultiHandle();
if (!mCurlMultiHandle)
{
llwarns << "curl_multi_init() returned NULL! Easy handles: " << gCurlEasyCount << " Multi handles: " << gCurlMultiCount << llendl;
- mCurlMultiHandle = initMulti();
+ mCurlMultiHandle = LLCurl::newMultiHandle();
}
- llassert_always(mCurlMultiHandle);
-
- if(LLCurl::getCurlThread()->getThreaded())
+ //llassert_always(mCurlMultiHandle);
+
+ if(mCurlMultiHandle)
{
- mMutexp = new LLMutex(NULL) ;
- mDeletionMutexp = new LLMutex(NULL) ;
- mEasyMutexp = new LLMutex(NULL) ;
- }
- LLCurl::getCurlThread()->addMulti(this) ;
+ if(LLCurl::getCurlThread()->getThreaded())
+ {
+ mMutexp = new LLMutex(NULL) ;
+ mDeletionMutexp = new LLMutex(NULL) ;
+ mEasyMutexp = new LLMutex(NULL) ;
+ }
+ LLCurl::getCurlThread()->addMulti(this) ;
- ++gCurlMultiCount;
+ mIdleTimeOut = idle_time_out ;
+ if(mIdleTimeOut < LLCurl::sCurlRequestTimeOut)
+ {
+ mIdleTimeOut = LLCurl::sCurlRequestTimeOut ;
+ }
+
+ ++gCurlMultiCount;
+ }
}
LLCurl::Multi::~Multi()
{
+ cleanup() ;
+}
+
+void LLCurl::Multi::cleanup()
+{
+ if(!mCurlMultiHandle)
+ {
+ return ; //nothing to clean.
+ }
+
// Clean up active
for(easy_active_list_t::iterator iter = mEasyActiveList.begin();
iter != mEasyActiveList.end(); ++iter)
@@ -564,7 +608,8 @@ LLCurl::Multi::~Multi()
for_each(mEasyFreeList.begin(), mEasyFreeList.end(), DeletePointer());
mEasyFreeList.clear();
- check_curl_multi_code(curl_multi_cleanup(mCurlMultiHandle));
+ check_curl_multi_code(LLCurl::deleteMultiHandle(mCurlMultiHandle));
+ mCurlMultiHandle = NULL ;
delete mMutexp ;
mMutexp = NULL ;
@@ -572,15 +617,13 @@ LLCurl::Multi::~Multi()
mDeletionMutexp = NULL ;
delete mEasyMutexp ;
mEasyMutexp = NULL ;
-
+
+ mQueued = 0 ;
+ mState = STATE_COMPLETED;
+
--gCurlMultiCount;
-}
-
-CURLM* LLCurl::Multi::initMulti()
-{
- LLMutexLock lock(sMultiInitMutexp) ;
- return curl_multi_init() ;
+ return ;
}
void LLCurl::Multi::lock()
@@ -604,6 +647,7 @@ void LLCurl::Multi::markDead()
LLMutexLock lock(mDeletionMutexp) ;
mDead = TRUE ;
+ LLCurl::getCurlThread()->setPriority(mHandle, LLQueuedThread::PRIORITY_URGENT) ;
}
void LLCurl::Multi::setState(LLCurl::Multi::ePerformState state)
@@ -630,6 +674,11 @@ bool LLCurl::Multi::isCompleted()
bool LLCurl::Multi::waitToComplete()
{
+ if(!isValid())
+ {
+ return true ;
+ }
+
if(!mMutexp) //not threaded
{
doPerform() ;
@@ -639,7 +688,7 @@ bool LLCurl::Multi::waitToComplete()
bool completed = (STATE_COMPLETED == mState) ;
if(!completed)
{
- LLCurl::getCurlThread()->setPriority(mHandle, LLQueuedThread::PRIORITY_URGENT) ;
+ LLCurl::getCurlThread()->setPriority(mHandle, LLQueuedThread::PRIORITY_HIGH) ;
}
return completed;
@@ -675,6 +724,10 @@ bool LLCurl::Multi::doPerform()
call_count++)
{
LLMutexLock lock(mMutexp) ;
+
+ //WARNING: curl_multi_perform will block for many hundreds of milliseconds
+ // NEVER call this from the main thread, and NEVER allow the main thread to
+ // wait on a mutex held by this thread while curl_multi_perform is executing
CURLMcode code = curl_multi_perform(mCurlMultiHandle, &q);
if (CURLM_CALL_MULTI_PERFORM != code || q == 0)
{
@@ -685,7 +738,12 @@ bool LLCurl::Multi::doPerform()
}
mQueued = q;
- setState(STATE_COMPLETED) ;
+ setState(STATE_COMPLETED) ;
+ mIdleTimer.reset() ;
+ }
+ else if(mIdleTimer.getElapsedTimeF32() > mIdleTimeOut) //idle for too long, remove it.
+ {
+ dead = true ;
}
return dead ;
@@ -693,6 +751,11 @@ bool LLCurl::Multi::doPerform()
S32 LLCurl::Multi::process()
{
+ if(!isValid())
+ {
+ return 0 ;
+ }
+
waitToComplete() ;
if (getState() != STATE_COMPLETED)
@@ -845,7 +908,11 @@ bool LLCurlThread::CurlRequest::processRequest()
if(mMulti)
{
completed = mCurlThread->doMultiPerform(mMulti) ;
- setPriority(LLQueuedThread::PRIORITY_LOW) ;
+
+ if(!completed)
+ {
+ setPriority(LLQueuedThread::PRIORITY_LOW) ;
+ }
}
return completed ;
@@ -853,27 +920,29 @@ bool LLCurlThread::CurlRequest::processRequest()
void LLCurlThread::CurlRequest::finishRequest(bool completed)
{
- mCurlThread->deleteMulti(mMulti) ;
+ if(mMulti->isDead())
+ {
+ mCurlThread->deleteMulti(mMulti) ;
+ }
+ else
+ {
+ mCurlThread->cleanupMulti(mMulti) ; //being idle too long, remove the request.
+ }
+
mMulti = NULL ;
}
LLCurlThread::LLCurlThread(bool threaded) :
LLQueuedThread("curlthread", threaded)
{
- if(!LLCurl::Multi::sMultiInitMutexp)
- {
- LLCurl::Multi::sMultiInitMutexp = new LLMutex(NULL) ;
- }
}
//virtual
LLCurlThread::~LLCurlThread()
{
- delete LLCurl::Multi::sMultiInitMutexp ;
- LLCurl::Multi::sMultiInitMutexp = NULL ;
}
-S32 LLCurlThread::update(U32 max_time_ms)
+S32 LLCurlThread::update(F32 max_time_ms)
{
return LLQueuedThread::update(max_time_ms);
}
@@ -892,7 +961,19 @@ void LLCurlThread::addMulti(LLCurl::Multi* multi)
void LLCurlThread::killMulti(LLCurl::Multi* multi)
{
- multi->markDead() ;
+ if(!multi)
+ {
+ return ;
+ }
+
+ if(multi->isValid())
+ {
+ multi->markDead() ;
+ }
+ else
+ {
+ deleteMulti(multi) ;
+ }
}
//private
@@ -906,6 +987,13 @@ void LLCurlThread::deleteMulti(LLCurl::Multi* multi)
{
delete multi ;
}
+
+//private
+void LLCurlThread::cleanupMulti(LLCurl::Multi* multi)
+{
+ multi->cleanup() ;
+}
+
//------------------------------------------------------------
//static
@@ -938,7 +1026,14 @@ LLCurlRequest::~LLCurlRequest()
void LLCurlRequest::addMulti()
{
LLCurl::Multi* multi = new LLCurl::Multi();
-
+ if(!multi->isValid())
+ {
+ LLCurl::getCurlThread()->killMulti(multi) ;
+ mActiveMulti = NULL ;
+ mActiveRequestCount = 0 ;
+ return;
+ }
+
mMultiSet.insert(multi);
mActiveMulti = multi;
mActiveRequestCount = 0;
@@ -952,7 +1047,12 @@ LLCurl::Easy* LLCurlRequest::allocEasy()
{
addMulti();
}
- llassert_always(mActiveMulti);
+ if(!mActiveMulti)
+ {
+ return NULL ;
+ }
+
+ //llassert_always(mActiveMulti);
++mActiveRequestCount;
LLCurl::Easy* easy = mActiveMulti->allocEasy();
return easy;
@@ -1062,6 +1162,19 @@ S32 LLCurlRequest::process()
{
curlmulti_set_t::iterator curiter = iter++;
LLCurl::Multi* multi = *curiter;
+
+ if(!multi->isValid())
+ {
+ if(multi == mActiveMulti)
+ {
+ mActiveMulti = NULL ;
+ mActiveRequestCount = 0 ;
+ }
+ mMultiSet.erase(curiter) ;
+ LLCurl::getCurlThread()->killMulti(multi) ;
+ continue ;
+ }
+
S32 tres = multi->process();
res += tres;
if (multi != mActiveMulti && tres == 0 && multi->mQueued == 0)
@@ -1082,6 +1195,19 @@ S32 LLCurlRequest::getQueued()
{
curlmulti_set_t::iterator curiter = iter++;
LLCurl::Multi* multi = *curiter;
+
+ if(!multi->isValid())
+ {
+ if(multi == mActiveMulti)
+ {
+ mActiveMulti = NULL ;
+ mActiveRequestCount = 0 ;
+ }
+ LLCurl::getCurlThread()->killMulti(multi);
+ mMultiSet.erase(curiter) ;
+ continue ;
+ }
+
queued += multi->mQueued;
if (multi->getState() != LLCurl::Multi::STATE_READY)
{
@@ -1101,13 +1227,22 @@ LLCurlEasyRequest::LLCurlEasyRequest()
{
mMulti = new LLCurl::Multi();
- mEasy = mMulti->allocEasy();
- if (mEasy)
+ if(mMulti->isValid())
{
- mEasy->setErrorBuffer();
- mEasy->setCA();
- // Set proxy settings if configured to do so.
- LLProxy::getInstance()->applyProxySettings(mEasy);
+ mEasy = mMulti->allocEasy();
+ if (mEasy)
+ {
+ mEasy->setErrorBuffer();
+ mEasy->setCA();
+ // Set proxy settings if configured to do so.
+ LLProxy::getInstance()->applyProxySettings(mEasy);
+ }
+ }
+ else
+ {
+ LLCurl::getCurlThread()->killMulti(mMulti) ;
+ mEasy = NULL ;
+ mMulti = NULL ;
}
}
@@ -1118,7 +1253,7 @@ LLCurlEasyRequest::~LLCurlEasyRequest()
void LLCurlEasyRequest::setopt(CURLoption option, S32 value)
{
- if (mEasy)
+ if (isValid() && mEasy)
{
mEasy->setopt(option, value);
}
@@ -1126,7 +1261,7 @@ void LLCurlEasyRequest::setopt(CURLoption option, S32 value)
void LLCurlEasyRequest::setoptString(CURLoption option, const std::string& value)
{
- if (mEasy)
+ if (isValid() && mEasy)
{
mEasy->setoptString(option, value);
}
@@ -1134,7 +1269,7 @@ void LLCurlEasyRequest::setoptString(CURLoption option, const std::string& value
void LLCurlEasyRequest::setPost(char* postdata, S32 size)
{
- if (mEasy)
+ if (isValid() && mEasy)
{
mEasy->setopt(CURLOPT_POST, 1);
mEasy->setopt(CURLOPT_POSTFIELDS, postdata);
@@ -1144,7 +1279,7 @@ void LLCurlEasyRequest::setPost(char* postdata, S32 size)
void LLCurlEasyRequest::setHeaderCallback(curl_header_callback callback, void* userdata)
{
- if (mEasy)
+ if (isValid() && mEasy)
{
mEasy->setopt(CURLOPT_HEADERFUNCTION, (void*)callback);
mEasy->setopt(CURLOPT_HEADERDATA, userdata); // aka CURLOPT_WRITEHEADER
@@ -1153,7 +1288,7 @@ void LLCurlEasyRequest::setHeaderCallback(curl_header_callback callback, void* u
void LLCurlEasyRequest::setWriteCallback(curl_write_callback callback, void* userdata)
{
- if (mEasy)
+ if (isValid() && mEasy)
{
mEasy->setopt(CURLOPT_WRITEFUNCTION, (void*)callback);
mEasy->setopt(CURLOPT_WRITEDATA, userdata);
@@ -1162,7 +1297,7 @@ void LLCurlEasyRequest::setWriteCallback(curl_write_callback callback, void* use
void LLCurlEasyRequest::setReadCallback(curl_read_callback callback, void* userdata)
{
- if (mEasy)
+ if (isValid() && mEasy)
{
mEasy->setopt(CURLOPT_READFUNCTION, (void*)callback);
mEasy->setopt(CURLOPT_READDATA, userdata);
@@ -1171,7 +1306,7 @@ void LLCurlEasyRequest::setReadCallback(curl_read_callback callback, void* userd
void LLCurlEasyRequest::setSSLCtxCallback(curl_ssl_ctx_callback callback, void* userdata)
{
- if (mEasy)
+ if (isValid() && mEasy)
{
mEasy->setopt(CURLOPT_SSL_CTX_FUNCTION, (void*)callback);
mEasy->setopt(CURLOPT_SSL_CTX_DATA, userdata);
@@ -1180,7 +1315,7 @@ void LLCurlEasyRequest::setSSLCtxCallback(curl_ssl_ctx_callback callback, void*
void LLCurlEasyRequest::slist_append(const char* str)
{
- if (mEasy)
+ if (isValid() && mEasy)
{
mEasy->slist_append(str);
}
@@ -1191,7 +1326,7 @@ void LLCurlEasyRequest::sendRequest(const std::string& url)
llassert_always(!mRequestSent);
mRequestSent = true;
lldebugs << url << llendl;
- if (mEasy)
+ if (isValid() && mEasy)
{
mEasy->setHeaders();
mEasy->setoptString(CURLOPT_URL, url);
@@ -1203,7 +1338,7 @@ void LLCurlEasyRequest::requestComplete()
{
llassert_always(mRequestSent);
mRequestSent = false;
- if (mEasy)
+ if (isValid() && mEasy)
{
mMulti->removeEasy(mEasy);
}
@@ -1212,6 +1347,10 @@ void LLCurlEasyRequest::requestComplete()
// Usage: Call getRestult until it returns false (no more messages)
bool LLCurlEasyRequest::getResult(CURLcode* result, LLCurl::TransferInfo* info)
{
+ if(!isValid())
+ {
+ return false ;
+ }
if (!mMulti->isCompleted())
{ //we're busy, try again later
return false;
@@ -1276,7 +1415,7 @@ CURLMsg* LLCurlEasyRequest::info_read(S32* q, LLCurl::TransferInfo* info)
std::string LLCurlEasyRequest::getErrorString()
{
- return mEasy ? std::string(mEasy->getErrorBuffer()) : std::string();
+ return isValid() && mEasy ? std::string(mEasy->getErrorBuffer()) : std::string();
}
////////////////////////////////////////////////////////////////////////////
@@ -1302,8 +1441,11 @@ unsigned long LLCurl::ssl_thread_id(void)
}
#endif
-void LLCurl::initClass(bool multi_threaded)
+void LLCurl::initClass(F32 curl_reuest_timeout, S32 max_number_handles, bool multi_threaded)
{
+ sCurlRequestTimeOut = curl_reuest_timeout ; //seconds
+ sMaxHandles = max_number_handles ; //max number of handles, (multi handles and easy handles combined).
+
// Do not change this "unless you are familiar with and mean to control
// internal operations of libcurl"
// - http://curl.haxx.se/libcurl/c/curl_global_init.html
@@ -1324,12 +1466,15 @@ void LLCurl::initClass(bool multi_threaded)
sCurlThread = new LLCurlThread(multi_threaded) ;
if(multi_threaded)
{
+ sHandleMutexp = new LLMutex(NULL) ;
Easy::sHandleMutexp = new LLMutex(NULL) ;
}
}
void LLCurl::cleanupClass()
{
+ sNotQuitting = false; //set quitting
+
//shut down curl thread
while(1)
{
@@ -1350,7 +1495,7 @@ void LLCurl::cleanupClass()
for (std::set<CURL*>::iterator iter = Easy::sFreeHandles.begin(); iter != Easy::sFreeHandles.end(); ++iter)
{
CURL* curl = *iter;
- curl_easy_cleanup(curl);
+ LLCurl::deleteEasyHandle(curl);
}
Easy::sFreeHandles.clear();
@@ -1358,9 +1503,77 @@ void LLCurl::cleanupClass()
delete Easy::sHandleMutexp ;
Easy::sHandleMutexp = NULL ;
+ delete sHandleMutexp ;
+ sHandleMutexp = NULL ;
+
llassert(Easy::sActiveHandles.empty());
}
+//static
+CURLM* LLCurl::newMultiHandle()
+{
+ LLMutexLock lock(sHandleMutexp) ;
+
+ if(sTotalHandles + 1 > sMaxHandles)
+ {
+ llwarns << "no more handles available." << llendl ;
+ return NULL ; //failed
+ }
+ sTotalHandles++;
+
+ CURLM* ret = curl_multi_init() ;
+ if(!ret)
+ {
+ llwarns << "curl_multi_init failed." << llendl ;
+ }
+
+ return ret ;
+}
+
+//static
+CURLMcode LLCurl::deleteMultiHandle(CURLM* handle)
+{
+ if(handle)
+ {
+ LLMutexLock lock(sHandleMutexp) ;
+ sTotalHandles-- ;
+ return curl_multi_cleanup(handle) ;
+ }
+ return CURLM_OK ;
+}
+
+//static
+CURL* LLCurl::newEasyHandle()
+{
+ LLMutexLock lock(sHandleMutexp) ;
+
+ if(sTotalHandles + 1 > sMaxHandles)
+ {
+ llwarns << "no more handles available." << llendl ;
+ return NULL ; //failed
+ }
+ sTotalHandles++;
+
+ CURL* ret = curl_easy_init() ;
+ if(!ret)
+ {
+ llwarns << "curl_easy_init failed." << llendl ;
+ }
+
+ return ret ;
+}
+
+//static
+void LLCurl::deleteEasyHandle(CURL* handle)
+{
+ if(handle)
+ {
+ LLMutexLock lock(sHandleMutexp) ;
+ curl_easy_cleanup(handle) ;
+ sTotalHandles-- ;
+ }
+}
+
const unsigned int LLCurl::MAX_REDIRECTS = 5;
// Provide access to LLCurl free functions outside of llcurl.cpp without polluting the global namespace.