diff options
Diffstat (limited to 'indra/llmessage/llcurl.cpp')
| -rw-r--r-- | indra/llmessage/llcurl.cpp | 287 |
1 files changed, 248 insertions, 39 deletions
diff --git a/indra/llmessage/llcurl.cpp b/indra/llmessage/llcurl.cpp index e17380fdf5..b93d429feb 100644 --- a/indra/llmessage/llcurl.cpp +++ b/indra/llmessage/llcurl.cpp @@ -72,10 +72,9 @@ static const U32 EASY_HANDLE_POOL_SIZE = 5; static const S32 MULTI_PERFORM_CALL_REPEAT = 5; -static const S32 CURL_REQUEST_TIMEOUT = 30; // seconds +static const S32 CURL_REQUEST_TIMEOUT = 30; // seconds per operation static const S32 MAX_ACTIVE_REQUEST_COUNT = 100; -static // DEBUG // S32 gCurlEasyCount = 0; S32 gCurlMultiCount = 0; @@ -87,6 +86,11 @@ std::vector<LLMutex*> LLCurl::sSSLMutex; std::string LLCurl::sCAPath; std::string LLCurl::sCAFile; LLCurlThread* LLCurl::sCurlThread = NULL ; +LLMutex* LLCurl::sHandleMutexp = NULL ; +S32 LLCurl::sTotalHandles = 0 ; +bool LLCurl::sNotQuitting = true; +F32 LLCurl::sCurlRequestTimeOut = 120.f; //seonds +S32 LLCurl::sMaxHandles = 256; //max number of handles, (multi handles and easy handles combined). void check_curl_code(CURLcode code) { @@ -224,13 +228,15 @@ LLMutex* LLCurl::Easy::sHandleMutexp = NULL ; //static CURL* LLCurl::Easy::allocEasyHandle() { + llassert(LLCurl::getCurlThread()) ; + CURL* ret = NULL; LLMutexLock lock(sHandleMutexp) ; if (sFreeHandles.empty()) { - ret = curl_easy_init(); + ret = LLCurl::newEasyHandle(); } else { @@ -250,19 +256,30 @@ CURL* LLCurl::Easy::allocEasyHandle() //static void LLCurl::Easy::releaseEasyHandle(CURL* handle) { + static const S32 MAX_NUM_FREE_HANDLES = 32 ; + if (!handle) { - llerrs << "handle cannot be NULL!" << llendl; + return ; //handle allocation failed. + //llerrs << "handle cannot be NULL!" << llendl; } LLMutexLock lock(sHandleMutexp) ; if (sActiveHandles.find(handle) != sActiveHandles.end()) { sActiveHandles.erase(handle); + + if(sFreeHandles.size() < MAX_NUM_FREE_HANDLES) + { sFreeHandles.insert(handle); } else { + LLCurl::deleteEasyHandle(handle) ; + } + } + else + { llerrs << "Invalid handle." << llendl; } } @@ -302,6 +319,14 @@ LLCurl::Easy::~Easy() --gCurlEasyCount; curl_slist_free_all(mHeaders); for_each(mStrings.begin(), mStrings.end(), DeletePointerArray()); + + if (mResponder && LLCurl::sNotQuitting) //aborted + { + std::string reason("Request timeout, aborted.") ; + mResponder->completedRaw(408, //HTTP_REQUEST_TIME_OUT, timeout, abort + reason, mChannels, mOutput); + } + mResponder = NULL; } void LLCurl::Easy::resetState() @@ -428,9 +453,9 @@ size_t curlReadCallback(char* data, size_t size, size_t nmemb, void* user_data) LLCurl::Easy* easy = (LLCurl::Easy*)user_data; S32 n = size * nmemb; - S32 startpos = easy->getInput().tellg(); + S32 startpos = (S32)easy->getInput().tellg(); easy->getInput().seekg(0, std::ios::end); - S32 endpos = easy->getInput().tellg(); + S32 endpos = (S32)easy->getInput().tellg(); easy->getInput().seekg(startpos, std::ios::beg); S32 maxn = endpos - startpos; n = llmin(n, maxn); @@ -474,6 +499,7 @@ void LLCurl::Easy::prepRequest(const std::string& url, LLProxy::getInstance()->applyProxySettings(this); mOutput.reset(new LLBufferArray); + mOutput->setThreaded(true); setopt(CURLOPT_WRITEFUNCTION, (void*)&curlWriteCallback); setopt(CURLOPT_WRITEDATA, (void*)this); @@ -517,8 +543,7 @@ void LLCurl::Easy::prepRequest(const std::string& url, } //////////////////////////////////////////////////////////////////////////// -LLMutex* LLCurl::Multi::sMultiInitMutexp = NULL ; -LLCurl::Multi::Multi() +LLCurl::Multi::Multi(F32 idle_time_out) : mQueued(0), mErrorCount(0), mState(STATE_READY), @@ -527,15 +552,17 @@ LLCurl::Multi::Multi() mDeletionMutexp(NULL), mEasyMutexp(NULL) { - mCurlMultiHandle = initMulti(); + mCurlMultiHandle = LLCurl::newMultiHandle(); if (!mCurlMultiHandle) { llwarns << "curl_multi_init() returned NULL! Easy handles: " << gCurlEasyCount << " Multi handles: " << gCurlMultiCount << llendl; - mCurlMultiHandle = initMulti(); + mCurlMultiHandle = LLCurl::newMultiHandle(); } - llassert_always(mCurlMultiHandle); + //llassert_always(mCurlMultiHandle); + if(mCurlMultiHandle) + { if(LLCurl::getCurlThread()->getThreaded()) { mMutexp = new LLMutex(NULL) ; @@ -544,11 +571,28 @@ LLCurl::Multi::Multi() } LLCurl::getCurlThread()->addMulti(this) ; + mIdleTimeOut = idle_time_out ; + if(mIdleTimeOut < LLCurl::sCurlRequestTimeOut) + { + mIdleTimeOut = LLCurl::sCurlRequestTimeOut ; + } + ++gCurlMultiCount; } +} LLCurl::Multi::~Multi() { + cleanup() ; +} + +void LLCurl::Multi::cleanup() +{ + if(!mCurlMultiHandle) + { + return ; //nothing to clean. + } + // Clean up active for(easy_active_list_t::iterator iter = mEasyActiveList.begin(); iter != mEasyActiveList.end(); ++iter) @@ -564,7 +608,8 @@ LLCurl::Multi::~Multi() for_each(mEasyFreeList.begin(), mEasyFreeList.end(), DeletePointer()); mEasyFreeList.clear(); - check_curl_multi_code(curl_multi_cleanup(mCurlMultiHandle)); + check_curl_multi_code(LLCurl::deleteMultiHandle(mCurlMultiHandle)); + mCurlMultiHandle = NULL ; delete mMutexp ; mMutexp = NULL ; @@ -573,14 +618,12 @@ LLCurl::Multi::~Multi() delete mEasyMutexp ; mEasyMutexp = NULL ; + mQueued = 0 ; + mState = STATE_COMPLETED; + --gCurlMultiCount; -} - -CURLM* LLCurl::Multi::initMulti() -{ - LLMutexLock lock(sMultiInitMutexp) ; - return curl_multi_init() ; + return ; } void LLCurl::Multi::lock() @@ -604,6 +647,7 @@ void LLCurl::Multi::markDead() LLMutexLock lock(mDeletionMutexp) ; mDead = TRUE ; + LLCurl::getCurlThread()->setPriority(mHandle, LLQueuedThread::PRIORITY_URGENT) ; } void LLCurl::Multi::setState(LLCurl::Multi::ePerformState state) @@ -630,6 +674,11 @@ bool LLCurl::Multi::isCompleted() bool LLCurl::Multi::waitToComplete() { + if(!isValid()) + { + return true ; + } + if(!mMutexp) //not threaded { doPerform() ; @@ -639,7 +688,7 @@ bool LLCurl::Multi::waitToComplete() bool completed = (STATE_COMPLETED == mState) ; if(!completed) { - LLCurl::getCurlThread()->setPriority(mHandle, LLQueuedThread::PRIORITY_URGENT) ; + LLCurl::getCurlThread()->setPriority(mHandle, LLQueuedThread::PRIORITY_HIGH) ; } return completed; @@ -690,6 +739,11 @@ bool LLCurl::Multi::doPerform() mQueued = q; setState(STATE_COMPLETED) ; + mIdleTimer.reset() ; + } + else if(mIdleTimer.getElapsedTimeF32() > mIdleTimeOut) //idle for too long, remove it. + { + dead = true ; } return dead ; @@ -697,6 +751,11 @@ bool LLCurl::Multi::doPerform() S32 LLCurl::Multi::process() { + if(!isValid()) + { + return 0 ; + } + waitToComplete() ; if (getState() != STATE_COMPLETED) @@ -849,32 +908,38 @@ bool LLCurlThread::CurlRequest::processRequest() if(mMulti) { completed = mCurlThread->doMultiPerform(mMulti) ; + + if(!completed) + { setPriority(LLQueuedThread::PRIORITY_LOW) ; } + } return completed ; } void LLCurlThread::CurlRequest::finishRequest(bool completed) { + if(mMulti->isDead()) + { mCurlThread->deleteMulti(mMulti) ; + } + else + { + mCurlThread->cleanupMulti(mMulti) ; //being idle too long, remove the request. + } + mMulti = NULL ; } LLCurlThread::LLCurlThread(bool threaded) : LLQueuedThread("curlthread", threaded) { - if(!LLCurl::Multi::sMultiInitMutexp) - { - LLCurl::Multi::sMultiInitMutexp = new LLMutex(NULL) ; - } } //virtual LLCurlThread::~LLCurlThread() { - delete LLCurl::Multi::sMultiInitMutexp ; - LLCurl::Multi::sMultiInitMutexp = NULL ; } S32 LLCurlThread::update(F32 max_time_ms) @@ -896,8 +961,20 @@ void LLCurlThread::addMulti(LLCurl::Multi* multi) void LLCurlThread::killMulti(LLCurl::Multi* multi) { + if(!multi) + { + return ; + } + + if(multi->isValid()) + { multi->markDead() ; } + else + { + deleteMulti(multi) ; + } +} //private bool LLCurlThread::doMultiPerform(LLCurl::Multi* multi) @@ -910,6 +987,13 @@ void LLCurlThread::deleteMulti(LLCurl::Multi* multi) { delete multi ; } + +//private +void LLCurlThread::cleanupMulti(LLCurl::Multi* multi) +{ + multi->cleanup() ; +} + //------------------------------------------------------------ //static @@ -942,6 +1026,13 @@ LLCurlRequest::~LLCurlRequest() void LLCurlRequest::addMulti() { LLCurl::Multi* multi = new LLCurl::Multi(); + if(!multi->isValid()) + { + LLCurl::getCurlThread()->killMulti(multi) ; + mActiveMulti = NULL ; + mActiveRequestCount = 0 ; + return; + } mMultiSet.insert(multi); mActiveMulti = multi; @@ -956,7 +1047,12 @@ LLCurl::Easy* LLCurlRequest::allocEasy() { addMulti(); } - llassert_always(mActiveMulti); + if(!mActiveMulti) + { + return NULL ; + } + + //llassert_always(mActiveMulti); ++mActiveRequestCount; LLCurl::Easy* easy = mActiveMulti->allocEasy(); return easy; @@ -1066,6 +1162,19 @@ S32 LLCurlRequest::process() { curlmulti_set_t::iterator curiter = iter++; LLCurl::Multi* multi = *curiter; + + if(!multi->isValid()) + { + if(multi == mActiveMulti) + { + mActiveMulti = NULL ; + mActiveRequestCount = 0 ; + } + mMultiSet.erase(curiter) ; + LLCurl::getCurlThread()->killMulti(multi) ; + continue ; + } + S32 tres = multi->process(); res += tres; if (multi != mActiveMulti && tres == 0 && multi->mQueued == 0) @@ -1086,6 +1195,19 @@ S32 LLCurlRequest::getQueued() { curlmulti_set_t::iterator curiter = iter++; LLCurl::Multi* multi = *curiter; + + if(!multi->isValid()) + { + if(multi == mActiveMulti) + { + mActiveMulti = NULL ; + mActiveRequestCount = 0 ; + } + LLCurl::getCurlThread()->killMulti(multi); + mMultiSet.erase(curiter) ; + continue ; + } + queued += multi->mQueued; if (multi->getState() != LLCurl::Multi::STATE_READY) { @@ -1105,6 +1227,8 @@ LLCurlEasyRequest::LLCurlEasyRequest() { mMulti = new LLCurl::Multi(); + if(mMulti->isValid()) + { mEasy = mMulti->allocEasy(); if (mEasy) { @@ -1114,6 +1238,13 @@ LLCurlEasyRequest::LLCurlEasyRequest() LLProxy::getInstance()->applyProxySettings(mEasy); } } + else + { + LLCurl::getCurlThread()->killMulti(mMulti) ; + mEasy = NULL ; + mMulti = NULL ; + } +} LLCurlEasyRequest::~LLCurlEasyRequest() { @@ -1122,7 +1253,7 @@ LLCurlEasyRequest::~LLCurlEasyRequest() void LLCurlEasyRequest::setopt(CURLoption option, S32 value) { - if (mEasy) + if (isValid() && mEasy) { mEasy->setopt(option, value); } @@ -1130,7 +1261,7 @@ void LLCurlEasyRequest::setopt(CURLoption option, S32 value) void LLCurlEasyRequest::setoptString(CURLoption option, const std::string& value) { - if (mEasy) + if (isValid() && mEasy) { mEasy->setoptString(option, value); } @@ -1138,7 +1269,7 @@ void LLCurlEasyRequest::setoptString(CURLoption option, const std::string& value void LLCurlEasyRequest::setPost(char* postdata, S32 size) { - if (mEasy) + if (isValid() && mEasy) { mEasy->setopt(CURLOPT_POST, 1); mEasy->setopt(CURLOPT_POSTFIELDS, postdata); @@ -1148,7 +1279,7 @@ void LLCurlEasyRequest::setPost(char* postdata, S32 size) void LLCurlEasyRequest::setHeaderCallback(curl_header_callback callback, void* userdata) { - if (mEasy) + if (isValid() && mEasy) { mEasy->setopt(CURLOPT_HEADERFUNCTION, (void*)callback); mEasy->setopt(CURLOPT_HEADERDATA, userdata); // aka CURLOPT_WRITEHEADER @@ -1157,7 +1288,7 @@ void LLCurlEasyRequest::setHeaderCallback(curl_header_callback callback, void* u void LLCurlEasyRequest::setWriteCallback(curl_write_callback callback, void* userdata) { - if (mEasy) + if (isValid() && mEasy) { mEasy->setopt(CURLOPT_WRITEFUNCTION, (void*)callback); mEasy->setopt(CURLOPT_WRITEDATA, userdata); @@ -1166,7 +1297,7 @@ void LLCurlEasyRequest::setWriteCallback(curl_write_callback callback, void* use void LLCurlEasyRequest::setReadCallback(curl_read_callback callback, void* userdata) { - if (mEasy) + if (isValid() && mEasy) { mEasy->setopt(CURLOPT_READFUNCTION, (void*)callback); mEasy->setopt(CURLOPT_READDATA, userdata); @@ -1175,7 +1306,7 @@ void LLCurlEasyRequest::setReadCallback(curl_read_callback callback, void* userd void LLCurlEasyRequest::setSSLCtxCallback(curl_ssl_ctx_callback callback, void* userdata) { - if (mEasy) + if (isValid() && mEasy) { mEasy->setopt(CURLOPT_SSL_CTX_FUNCTION, (void*)callback); mEasy->setopt(CURLOPT_SSL_CTX_DATA, userdata); @@ -1184,7 +1315,7 @@ void LLCurlEasyRequest::setSSLCtxCallback(curl_ssl_ctx_callback callback, void* void LLCurlEasyRequest::slist_append(const char* str) { - if (mEasy) + if (isValid() && mEasy) { mEasy->slist_append(str); } @@ -1195,7 +1326,7 @@ void LLCurlEasyRequest::sendRequest(const std::string& url) llassert_always(!mRequestSent); mRequestSent = true; lldebugs << url << llendl; - if (mEasy) + if (isValid() && mEasy) { mEasy->setHeaders(); mEasy->setoptString(CURLOPT_URL, url); @@ -1207,7 +1338,7 @@ void LLCurlEasyRequest::requestComplete() { llassert_always(mRequestSent); mRequestSent = false; - if (mEasy) + if (isValid() && mEasy) { mMulti->removeEasy(mEasy); } @@ -1216,6 +1347,10 @@ void LLCurlEasyRequest::requestComplete() // Usage: Call getRestult until it returns false (no more messages) bool LLCurlEasyRequest::getResult(CURLcode* result, LLCurl::TransferInfo* info) { + if(!isValid()) + { + return false ; + } if (!mMulti->isCompleted()) { //we're busy, try again later return false; @@ -1280,7 +1415,7 @@ CURLMsg* LLCurlEasyRequest::info_read(S32* q, LLCurl::TransferInfo* info) std::string LLCurlEasyRequest::getErrorString() { - return mEasy ? std::string(mEasy->getErrorBuffer()) : std::string(); + return isValid() && mEasy ? std::string(mEasy->getErrorBuffer()) : std::string(); } //////////////////////////////////////////////////////////////////////////// @@ -1306,8 +1441,11 @@ unsigned long LLCurl::ssl_thread_id(void) } #endif -void LLCurl::initClass(bool multi_threaded) +void LLCurl::initClass(F32 curl_reuest_timeout, S32 max_number_handles, bool multi_threaded) { + sCurlRequestTimeOut = curl_reuest_timeout ; //seconds + sMaxHandles = max_number_handles ; //max number of handles, (multi handles and easy handles combined). + // Do not change this "unless you are familiar with and mean to control // internal operations of libcurl" // - http://curl.haxx.se/libcurl/c/curl_global_init.html @@ -1328,12 +1466,15 @@ void LLCurl::initClass(bool multi_threaded) sCurlThread = new LLCurlThread(multi_threaded) ; if(multi_threaded) { + sHandleMutexp = new LLMutex(NULL) ; Easy::sHandleMutexp = new LLMutex(NULL) ; } } void LLCurl::cleanupClass() { + sNotQuitting = false; //set quitting + //shut down curl thread while(1) { @@ -1354,7 +1495,7 @@ void LLCurl::cleanupClass() for (std::set<CURL*>::iterator iter = Easy::sFreeHandles.begin(); iter != Easy::sFreeHandles.end(); ++iter) { CURL* curl = *iter; - curl_easy_cleanup(curl); + LLCurl::deleteEasyHandle(curl); } Easy::sFreeHandles.clear(); @@ -1362,9 +1503,77 @@ void LLCurl::cleanupClass() delete Easy::sHandleMutexp ; Easy::sHandleMutexp = NULL ; + delete sHandleMutexp ; + sHandleMutexp = NULL ; + llassert(Easy::sActiveHandles.empty()); } +//static +CURLM* LLCurl::newMultiHandle() +{ + LLMutexLock lock(sHandleMutexp) ; + + if(sTotalHandles + 1 > sMaxHandles) + { + llwarns << "no more handles available." << llendl ; + return NULL ; //failed + } + sTotalHandles++; + + CURLM* ret = curl_multi_init() ; + if(!ret) + { + llwarns << "curl_multi_init failed." << llendl ; + } + + return ret ; +} + +//static +CURLMcode LLCurl::deleteMultiHandle(CURLM* handle) +{ + if(handle) + { + LLMutexLock lock(sHandleMutexp) ; + sTotalHandles-- ; + return curl_multi_cleanup(handle) ; + } + return CURLM_OK ; +} + +//static +CURL* LLCurl::newEasyHandle() +{ + LLMutexLock lock(sHandleMutexp) ; + + if(sTotalHandles + 1 > sMaxHandles) + { + llwarns << "no more handles available." << llendl ; + return NULL ; //failed + } + sTotalHandles++; + + CURL* ret = curl_easy_init() ; + if(!ret) + { + llwarns << "curl_easy_init failed." << llendl ; + } + + return ret ; +} + +//static +void LLCurl::deleteEasyHandle(CURL* handle) +{ + if(handle) + { + LLMutexLock lock(sHandleMutexp) ; + curl_easy_cleanup(handle) ; + sTotalHandles-- ; + } +} + const unsigned int LLCurl::MAX_REDIRECTS = 5; // Provide access to LLCurl free functions outside of llcurl.cpp without polluting the global namespace. |