diff options
Diffstat (limited to 'indra/llmessage')
41 files changed, 2036 insertions, 442 deletions
diff --git a/indra/llmessage/CMakeLists.txt b/indra/llmessage/CMakeLists.txt index c5f82cf052..0f40a670fa 100644 --- a/indra/llmessage/CMakeLists.txt +++ b/indra/llmessage/CMakeLists.txt @@ -65,6 +65,7 @@ set(llmessage_SOURCE_FILES llpacketbuffer.cpp llpacketring.cpp llpartdata.cpp + llproxy.cpp llpumpio.cpp llregionpresenceverifier.cpp llsdappservices.cpp @@ -161,6 +162,7 @@ set(llmessage_HEADER_FILES llpacketring.h llpartdata.h llpumpio.h + llproxy.h llqueryflags.h llregionflags.h llregionhandle.h diff --git a/indra/llmessage/llares.cpp b/indra/llmessage/llares.cpp index 5a67035ed1..fab9858b69 100644 --- a/indra/llmessage/llares.cpp +++ b/indra/llmessage/llares.cpp @@ -28,6 +28,7 @@ #include "linden_common.h" #include "llares.h" +#include "llscopedvolatileaprpool.h" #include <ares_dns.h> #include <ares_version.h> @@ -464,11 +465,6 @@ void LLAres::search(const std::string &query, LLResType type, bool LLAres::process(U64 timeout) { - if (!gAPRPoolp) - { - ll_init_apr(); - } - ares_socket_t socks[ARES_GETSOCK_MAXNUM]; apr_pollfd_t aprFds[ARES_GETSOCK_MAXNUM]; apr_int32_t nsds = 0; @@ -482,10 +478,7 @@ bool LLAres::process(U64 timeout) return nsds > 0; } - apr_status_t status; - LLAPRPool pool; - status = pool.getStatus() ; - ll_apr_assert_status(status); + LLScopedVolatileAPRPool scoped_pool; for (int i = 0; i < ARES_GETSOCK_MAXNUM; i++) { @@ -502,7 +495,7 @@ bool LLAres::process(U64 timeout) apr_socket_t *aprSock = NULL; - status = apr_os_sock_put(&aprSock, (apr_os_sock_t *) &socks[i], pool.getAPRPool()); + apr_status_t status = apr_os_sock_put(&aprSock, (apr_os_sock_t *) &socks[i], scoped_pool); if (status != APR_SUCCESS) { ll_apr_warn_status(status); @@ -511,7 +504,7 @@ bool LLAres::process(U64 timeout) aprFds[nactive].desc.s = aprSock; aprFds[nactive].desc_type = APR_POLL_SOCKET; - aprFds[nactive].p = pool.getAPRPool(); + aprFds[nactive].p = scoped_pool; aprFds[nactive].rtnevents = 0; aprFds[nactive].client_data = &socks[i]; @@ -520,7 +513,7 @@ bool LLAres::process(U64 timeout) if (nactive > 0) { - status = apr_poll(aprFds, nactive, &nsds, timeout); + apr_status_t status = apr_poll(aprFds, nactive, &nsds, timeout); if (status != APR_SUCCESS && status != APR_TIMEUP) { diff --git a/indra/llmessage/llassetstorage.cpp b/indra/llmessage/llassetstorage.cpp index 27a368df3d..31cdb1219b 100644 --- a/indra/llmessage/llassetstorage.cpp +++ b/indra/llmessage/llassetstorage.cpp @@ -398,6 +398,12 @@ BOOL LLAssetStorage::hasLocalAsset(const LLUUID &uuid, const LLAssetType::EType bool LLAssetStorage::findInStaticVFSAndInvokeCallback(const LLUUID& uuid, LLAssetType::EType type, LLGetAssetCallback callback, void *user_data) { + if (user_data) + { + // The *user_data should not be passed without a callback to clean it up. + llassert(callback != NULL) + } + BOOL exists = mStaticVFS->getExists(uuid, type); if (exists) { @@ -432,15 +438,26 @@ void LLAssetStorage::getAssetData(const LLUUID uuid, LLAssetType::EType type, LL llinfos << "ASSET_TRACE requesting " << uuid << " type " << LLAssetType::lookup(type) << llendl; + if (user_data) + { + // The *user_data should not be passed without a callback to clean it up. + llassert(callback != NULL) + } + if (mShutDown) { llinfos << "ASSET_TRACE cancelled " << uuid << " type " << LLAssetType::lookup(type) << " shutting down" << llendl; - return; // don't get the asset or do any callbacks, we are shutting down + + if (callback) + { + callback(mVFS, uuid, type, user_data, LL_ERR_ASSET_REQUEST_FAILED, LL_EXSTAT_NONE); + } + return; } - + if (uuid.isNull()) { - // Special case early out for NULL uuid + // Special case early out for NULL uuid and for shutting down if (callback) { callback(mVFS, uuid, type, user_data, LL_ERR_ASSET_REQUEST_NOT_IN_DATABASE, LL_EXSTAT_NULL_UUID); @@ -544,7 +561,7 @@ void LLAssetStorage::_queueDataRequest(const LLUUID& uuid, LLAssetType::EType at tpvf.setAsset(uuid, atype); tpvf.setCallback(downloadCompleteCallback, req); - llinfos << "Starting transfer for " << uuid << llendl; + //llinfos << "Starting transfer for " << uuid << llendl; LLTransferTargetChannel *ttcp = gTransferManager.getTargetChannel(mUpstreamHost, LLTCT_ASSET); ttcp->requestTransfer(spa, tpvf, 100.f + (is_priority ? 1.f : 0.f)); } diff --git a/indra/llmessage/llavatarnamecache.cpp b/indra/llmessage/llavatarnamecache.cpp index 767001b633..97f2792686 100644 --- a/indra/llmessage/llavatarnamecache.cpp +++ b/indra/llmessage/llavatarnamecache.cpp @@ -553,12 +553,10 @@ void LLAvatarNameCache::eraseUnrefreshed() if (!sLastExpireCheck || sLastExpireCheck < max_unrefreshed) { sLastExpireCheck = now; - cache_t::iterator it = sCache.begin(); - while (it != sCache.end()) + + for (cache_t::iterator it = sCache.begin(); it != sCache.end();) { - cache_t::iterator cur = it; - ++it; - const LLAvatarName& av_name = cur->second; + const LLAvatarName& av_name = it->second; if (av_name.mExpires < max_unrefreshed) { const LLUUID& agent_id = it->first; @@ -566,8 +564,12 @@ void LLAvatarNameCache::eraseUnrefreshed() << " user '" << av_name.mUsername << "' " << "expired " << now - av_name.mExpires << " secs ago" << LL_ENDL; - sCache.erase(cur); + sCache.erase(it++); } + else + { + ++it; + } } LL_INFOS("AvNameCache") << sCache.size() << " cached avatar names" << LL_ENDL; } diff --git a/indra/llmessage/llcurl.cpp b/indra/llmessage/llcurl.cpp index a485fa0160..14e169c6b1 100644 --- a/indra/llmessage/llcurl.cpp +++ b/indra/llmessage/llcurl.cpp @@ -1,5 +1,5 @@ /** - * @file llcurl.h + * @file llcurl.cpp * @author Zero / Donovan * @date 2006-10-15 * @brief Implementation of wrapper around libcurl. @@ -46,9 +46,11 @@ #endif #include "llbufferstream.h" -#include "llstl.h" +#include "llproxy.h" #include "llsdserialize.h" +#include "llstl.h" #include "llthread.h" +#include "lltimer.h" ////////////////////////////////////////////////////////////////////////////// /* @@ -73,6 +75,7 @@ static const S32 MULTI_PERFORM_CALL_REPEAT = 5; static const S32 CURL_REQUEST_TIMEOUT = 30; // seconds static const S32 MAX_ACTIVE_REQUEST_COUNT = 100; +static // DEBUG // S32 gCurlEasyCount = 0; S32 gCurlMultiCount = 0; @@ -84,6 +87,29 @@ std::vector<LLMutex*> LLCurl::sSSLMutex; std::string LLCurl::sCAPath; std::string LLCurl::sCAFile; +bool LLCurl::sMultiThreaded = false; +static U32 sMainThreadID = 0; + +void check_curl_code(CURLcode code) +{ + if (code != CURLE_OK) + { + // linux appears to throw a curl error once per session for a bad initialization + // at a pretty random time (when enabling cookies). + llinfos << "curl error detected: " << curl_easy_strerror(code) << llendl; + } +} + +void check_curl_multi_code(CURLMcode code) +{ + if (code != CURLM_OK) + { + // linux appears to throw a curl error once per session for a bad initialization + // at a pretty random time (when enabling cookies). + llinfos << "curl multi error detected: " << curl_multi_strerror(code) << llendl; + } +} + //static void LLCurl::setCAPath(const std::string& path) { @@ -183,7 +209,7 @@ namespace boost void intrusive_ptr_release(LLCurl::Responder* p) { - if(p && 0 == --p->mReferenceCount) + if (p && 0 == --p->mReferenceCount) { delete p; } @@ -193,63 +219,55 @@ namespace boost ////////////////////////////////////////////////////////////////////////////// +std::set<CURL*> LLCurl::Easy::sFreeHandles; +std::set<CURL*> LLCurl::Easy::sActiveHandles; +LLMutex* LLCurl::Easy::sHandleMutex = NULL; -class LLCurl::Easy -{ - LOG_CLASS(Easy); - -private: - Easy(); - -public: - static Easy* getEasy(); - ~Easy(); - CURL* getCurlHandle() const { return mCurlEasyHandle; } +//static +CURL* LLCurl::Easy::allocEasyHandle() +{ + CURL* ret = NULL; + LLMutexLock lock(sHandleMutex); + if (sFreeHandles.empty()) + { + ret = curl_easy_init(); + } + else + { + ret = *(sFreeHandles.begin()); + sFreeHandles.erase(ret); + curl_easy_reset(ret); + } - void setErrorBuffer(); - void setCA(); - - void setopt(CURLoption option, S32 value); - // These assume the setter does not free value! - void setopt(CURLoption option, void* value); - void setopt(CURLoption option, char* value); - // Copies the string so that it is gauranteed to stick around - void setoptString(CURLoption option, const std::string& value); - - void slist_append(const char* str); - void setHeaders(); - - U32 report(CURLcode); - void getTransferInfo(LLCurl::TransferInfo* info); + if (ret) + { + sActiveHandles.insert(ret); + } - void prepRequest(const std::string& url, const std::vector<std::string>& headers, ResponderPtr, bool post = false); - - const char* getErrorBuffer(); + return ret; +} - std::stringstream& getInput() { return mInput; } - std::stringstream& getHeaderOutput() { return mHeaderOutput; } - LLIOPipe::buffer_ptr_t& getOutput() { return mOutput; } - const LLChannelDescriptors& getChannels() { return mChannels; } - - void resetState(); +//static +void LLCurl::Easy::releaseEasyHandle(CURL* handle) +{ + if (!handle) + { + llerrs << "handle cannot be NULL!" << llendl; + } -private: - CURL* mCurlEasyHandle; - struct curl_slist* mHeaders; + LLMutexLock lock(sHandleMutex); - std::stringstream mRequest; - LLChannelDescriptors mChannels; - LLIOPipe::buffer_ptr_t mOutput; - std::stringstream mInput; - std::stringstream mHeaderOutput; - char mErrorBuffer[CURL_ERROR_SIZE]; - - // Note: char*'s not strings since we pass pointers to curl - std::vector<char*> mStrings; - - ResponderPtr mResponder; -}; + if (sActiveHandles.find(handle) != sActiveHandles.end()) + { + sActiveHandles.erase(handle); + sFreeHandles.insert(handle); + } + else + { + llerrs << "Invalid handle." << llendl; + } +} LLCurl::Easy::Easy() : mHeaders(NULL), @@ -261,25 +279,28 @@ LLCurl::Easy::Easy() LLCurl::Easy* LLCurl::Easy::getEasy() { Easy* easy = new Easy(); - easy->mCurlEasyHandle = curl_easy_init(); + easy->mCurlEasyHandle = allocEasyHandle(); + if (!easy->mCurlEasyHandle) { // this can happen if we have too many open files (fails in c-ares/ares_init.c) - llwarns << "curl_multi_init() returned NULL! Easy handles: " << gCurlEasyCount << " Multi handles: " << gCurlMultiCount << llendl; + llwarns << "allocEasyHandle() returned NULL! Easy handles: " << gCurlEasyCount << " Multi handles: " << gCurlMultiCount << llendl; delete easy; return NULL; } - // set no DMS caching as default for all easy handles. This prevents them adopting a + // set no DNS caching as default for all easy handles. This prevents them adopting a // multi handles cache if they are added to one. - curl_easy_setopt(easy->mCurlEasyHandle, CURLOPT_DNS_CACHE_TIMEOUT, 0); + CURLcode result = curl_easy_setopt(easy->mCurlEasyHandle, CURLOPT_DNS_CACHE_TIMEOUT, 0); + check_curl_code(result); + ++gCurlEasyCount; return easy; } LLCurl::Easy::~Easy() { - curl_easy_cleanup(mCurlEasyHandle); + releaseEasyHandle(mCurlEasyHandle); --gCurlEasyCount; curl_slist_free_all(mHeaders); for_each(mStrings.begin(), mStrings.end(), DeletePointerArray()); @@ -321,11 +342,11 @@ const char* LLCurl::Easy::getErrorBuffer() void LLCurl::Easy::setCA() { - if(!sCAPath.empty()) + if (!sCAPath.empty()) { setoptString(CURLOPT_CAPATH, sCAPath); } - if(!sCAFile.empty()) + if (!sCAFile.empty()) { setoptString(CURLOPT_CAINFO, sCAFile); } @@ -338,9 +359,9 @@ void LLCurl::Easy::setHeaders() void LLCurl::Easy::getTransferInfo(LLCurl::TransferInfo* info) { - curl_easy_getinfo(mCurlEasyHandle, CURLINFO_SIZE_DOWNLOAD, &info->mSizeDownload); - curl_easy_getinfo(mCurlEasyHandle, CURLINFO_TOTAL_TIME, &info->mTotalTime); - curl_easy_getinfo(mCurlEasyHandle, CURLINFO_SPEED_DOWNLOAD, &info->mSpeedDownload); + check_curl_code(curl_easy_getinfo(mCurlEasyHandle, CURLINFO_SIZE_DOWNLOAD, &info->mSizeDownload)); + check_curl_code(curl_easy_getinfo(mCurlEasyHandle, CURLINFO_TOTAL_TIME, &info->mTotalTime)); + check_curl_code(curl_easy_getinfo(mCurlEasyHandle, CURLINFO_SPEED_DOWNLOAD, &info->mSpeedDownload)); } U32 LLCurl::Easy::report(CURLcode code) @@ -350,13 +371,14 @@ U32 LLCurl::Easy::report(CURLcode code) if (code == CURLE_OK) { - curl_easy_getinfo(mCurlEasyHandle, CURLINFO_RESPONSE_CODE, &responseCode); + check_curl_code(curl_easy_getinfo(mCurlEasyHandle, CURLINFO_RESPONSE_CODE, &responseCode)); //*TODO: get reason from first line of mHeaderOutput } else { responseCode = 499; responseReason = strerror(code) + " : " + mErrorBuffer; + setopt(CURLOPT_FRESH_CONNECT, TRUE); } if (mResponder) @@ -372,17 +394,20 @@ U32 LLCurl::Easy::report(CURLcode code) // Note: these all assume the caller tracks the value (i.e. keeps it persistant) void LLCurl::Easy::setopt(CURLoption option, S32 value) { - curl_easy_setopt(mCurlEasyHandle, option, value); + CURLcode result = curl_easy_setopt(mCurlEasyHandle, option, value); + check_curl_code(result); } void LLCurl::Easy::setopt(CURLoption option, void* value) { - curl_easy_setopt(mCurlEasyHandle, option, value); + CURLcode result = curl_easy_setopt(mCurlEasyHandle, option, value); + check_curl_code(result); } void LLCurl::Easy::setopt(CURLoption option, char* value) { - curl_easy_setopt(mCurlEasyHandle, option, value); + CURLcode result = curl_easy_setopt(mCurlEasyHandle, option, value); + check_curl_code(result); } // Note: this copies the string so that the caller does not have to keep it around @@ -391,7 +416,8 @@ void LLCurl::Easy::setoptString(CURLoption option, const std::string& value) char* tstring = new char[value.length()+1]; strcpy(tstring, value.c_str()); mStrings.push_back(tstring); - curl_easy_setopt(mCurlEasyHandle, option, tstring); + CURLcode result = curl_easy_setopt(mCurlEasyHandle, option, tstring); + check_curl_code(result); } void LLCurl::Easy::slist_append(const char* str) @@ -437,15 +463,18 @@ size_t curlHeaderCallback(void* data, size_t size, size_t nmemb, void* user_data void LLCurl::Easy::prepRequest(const std::string& url, const std::vector<std::string>& headers, - ResponderPtr responder, bool post) + ResponderPtr responder, S32 time_out, bool post) { resetState(); if (post) setoptString(CURLOPT_ENCODING, ""); -// setopt(CURLOPT_VERBOSE, 1); // usefull for debugging + //setopt(CURLOPT_VERBOSE, 1); // useful for debugging setopt(CURLOPT_NOSIGNAL, 1); + // Set the CURL options for either Socks or HTTP proxy + LLProxy::getInstance()->applyProxySettings(this); + mOutput.reset(new LLBufferArray); setopt(CURLOPT_WRITEFUNCTION, (void*)&curlWriteCallback); setopt(CURLOPT_WRITEDATA, (void*)this); @@ -457,7 +486,7 @@ void LLCurl::Easy::prepRequest(const std::string& url, setopt(CURLOPT_HEADERDATA, (void*)this); // Allow up to five redirects - if(responder && responder->followRedir()) + if (responder && responder->followRedir()) { setopt(CURLOPT_FOLLOWLOCATION, 1); setopt(CURLOPT_MAXREDIRS, MAX_REDIRECTS); @@ -467,7 +496,10 @@ void LLCurl::Easy::prepRequest(const std::string& url, setCA(); setopt(CURLOPT_SSL_VERIFYPEER, true); - setopt(CURLOPT_TIMEOUT, CURL_REQUEST_TIMEOUT); + + //don't verify host name so urls with scrubbed host names will work (improves DNS performance) + setopt(CURLOPT_SSL_VERIFYHOST, 0); + setopt(CURLOPT_TIMEOUT, llmax(time_out, CURL_REQUEST_TIMEOUT)); setoptString(CURLOPT_URL, url); @@ -488,62 +520,48 @@ void LLCurl::Easy::prepRequest(const std::string& url, //////////////////////////////////////////////////////////////////////////// -class LLCurl::Multi +LLCurl::Multi::Multi() + : LLThread("Curl Multi"), + mQueued(0), + mErrorCount(0), + mPerformState(PERFORM_STATE_READY) { - LOG_CLASS(Multi); -public: - - Multi(); - ~Multi(); + mQuitting = false; - Easy* allocEasy(); - bool addEasy(Easy* easy); - - void removeEasy(Easy* easy); - - S32 process(); - S32 perform(); - - CURLMsg* info_read(S32* msgs_in_queue); - - S32 mQueued; - S32 mErrorCount; - -private: - void easyFree(Easy*); - - CURLM* mCurlMultiHandle; - - typedef std::set<Easy*> easy_active_list_t; - easy_active_list_t mEasyActiveList; - typedef std::map<CURL*, Easy*> easy_active_map_t; - easy_active_map_t mEasyActiveMap; - typedef std::set<Easy*> easy_free_list_t; - easy_free_list_t mEasyFreeList; -}; + mThreaded = LLCurl::sMultiThreaded && LLThread::currentID() == sMainThreadID; + if (mThreaded) + { + mSignal = new LLCondition(); + } + else + { + mSignal = NULL; + } -LLCurl::Multi::Multi() - : mQueued(0), - mErrorCount(0) -{ mCurlMultiHandle = curl_multi_init(); if (!mCurlMultiHandle) { llwarns << "curl_multi_init() returned NULL! Easy handles: " << gCurlEasyCount << " Multi handles: " << gCurlMultiCount << llendl; mCurlMultiHandle = curl_multi_init(); } + llassert_always(mCurlMultiHandle); ++gCurlMultiCount; } LLCurl::Multi::~Multi() { + llassert(isStopped()); + + delete mSignal; + mSignal = NULL; + // Clean up active for(easy_active_list_t::iterator iter = mEasyActiveList.begin(); iter != mEasyActiveList.end(); ++iter) { Easy* easy = *iter; - curl_multi_remove_handle(mCurlMultiHandle, easy->getCurlHandle()); + check_curl_multi_code(curl_multi_remove_handle(mCurlMultiHandle, easy->getCurlHandle())); delete easy; } mEasyActiveList.clear(); @@ -553,7 +571,7 @@ LLCurl::Multi::~Multi() for_each(mEasyFreeList.begin(), mEasyFreeList.end(), DeletePointer()); mEasyFreeList.clear(); - curl_multi_cleanup(mCurlMultiHandle); + check_curl_multi_code(curl_multi_cleanup(mCurlMultiHandle)); --gCurlMultiCount; } @@ -563,28 +581,64 @@ CURLMsg* LLCurl::Multi::info_read(S32* msgs_in_queue) return curlmsg; } +void LLCurl::Multi::perform() +{ + if (mThreaded) + { + if (mPerformState == PERFORM_STATE_READY) + { + mSignal->signal(); + } + } + else + { + doPerform(); + } +} -S32 LLCurl::Multi::perform() +void LLCurl::Multi::run() +{ + llassert(mThreaded); + + while (!mQuitting) + { + mSignal->wait(); + mPerformState = PERFORM_STATE_PERFORMING; + if (!mQuitting) + { + doPerform(); + } + } +} + +void LLCurl::Multi::doPerform() { S32 q = 0; for (S32 call_count = 0; - call_count < MULTI_PERFORM_CALL_REPEAT; - call_count += 1) + call_count < MULTI_PERFORM_CALL_REPEAT; + call_count += 1) { CURLMcode code = curl_multi_perform(mCurlMultiHandle, &q); if (CURLM_CALL_MULTI_PERFORM != code || q == 0) { + check_curl_multi_code(code); break; } + } mQueued = q; - return q; + mPerformState = PERFORM_STATE_COMPLETED; } S32 LLCurl::Multi::process() { perform(); - + + if (mPerformState != PERFORM_STATE_COMPLETED) + { + return 0; + } + CURLMsg* msg; int msgs_in_queue; @@ -615,6 +669,8 @@ S32 LLCurl::Multi::process() } } } + + mPerformState = PERFORM_STATE_READY; return processed; } @@ -642,11 +698,12 @@ LLCurl::Easy* LLCurl::Multi::allocEasy() bool LLCurl::Multi::addEasy(Easy* easy) { CURLMcode mcode = curl_multi_add_handle(mCurlMultiHandle, easy->getCurlHandle()); - if (mcode != CURLM_OK) - { - llwarns << "Curl Error: " << curl_multi_strerror(mcode) << llendl; - return false; - } + check_curl_multi_code(mcode); + //if (mcode != CURLM_OK) + //{ + // llwarns << "Curl Error: " << curl_multi_strerror(mcode) << llendl; + // return false; + //} return true; } @@ -667,22 +724,14 @@ void LLCurl::Multi::easyFree(Easy* easy) void LLCurl::Multi::removeEasy(Easy* easy) { - curl_multi_remove_handle(mCurlMultiHandle, easy->getCurlHandle()); + check_curl_multi_code(curl_multi_remove_handle(mCurlMultiHandle, easy->getCurlHandle())); easyFree(easy); } //static std::string LLCurl::strerror(CURLcode errorcode) { -#if LL_DARWIN - // curl_easy_strerror was added in libcurl 7.12.0. Unfortunately, the version in the Mac OS X 10.3.9 SDK is 7.10.2... - // There's a problem with the custom curl headers in our build that keeps me from #ifdefing this on the libcurl version number - // (the correct check would be #if LIBCURL_VERSION_NUM >= 0x070c00). We'll fix the header problem soon, but for now - // just punt and print the numeric error code on the Mac. - return llformat("%d", errorcode); -#else // LL_DARWIN return std::string(curl_easy_strerror(errorcode)); -#endif // LL_DARWIN } //////////////////////////////////////////////////////////////////////////// @@ -694,11 +743,27 @@ LLCurlRequest::LLCurlRequest() : mActiveRequestCount(0) { mThreadID = LLThread::currentID(); + mProcessing = FALSE; } LLCurlRequest::~LLCurlRequest() { llassert_always(mThreadID == LLThread::currentID()); + + //stop all Multi handle background threads + for (curlmulti_set_t::iterator iter = mMultiSet.begin(); iter != mMultiSet.end(); ++iter) + { + LLCurl::Multi* multi = *iter; + multi->mQuitting = true; + if (multi->mThreaded) + { + while (!multi->isStopped()) + { + multi->mSignal->signal(); + apr_sleep(1000); + } + } + } for_each(mMultiSet.begin(), mMultiSet.end(), DeletePointer()); } @@ -706,6 +771,10 @@ void LLCurlRequest::addMulti() { llassert_always(mThreadID == LLThread::currentID()); LLCurl::Multi* multi = new LLCurl::Multi(); + if (multi->mThreaded) + { + multi->start(); + } mMultiSet.insert(multi); mActiveMulti = multi; mActiveRequestCount = 0; @@ -728,6 +797,11 @@ LLCurl::Easy* LLCurlRequest::allocEasy() bool LLCurlRequest::addEasy(LLCurl::Easy* easy) { llassert_always(mActiveMulti); + + if (mProcessing) + { + llerrs << "Posting to a LLCurlRequest instance from within a responder is not allowed (causes DNS timeouts)." << llendl; + } bool res = mActiveMulti->addEasy(easy); return res; } @@ -762,14 +836,14 @@ bool LLCurlRequest::getByteRange(const std::string& url, bool LLCurlRequest::post(const std::string& url, const headers_t& headers, const LLSD& data, - LLCurl::ResponderPtr responder) + LLCurl::ResponderPtr responder, S32 time_out) { LLCurl::Easy* easy = allocEasy(); if (!easy) { return false; } - easy->prepRequest(url, headers, responder); + easy->prepRequest(url, headers, responder, time_out); LLSDSerialize::toXML(data, easy->getInput()); S32 bytes = easy->getInput().str().length(); @@ -785,12 +859,41 @@ bool LLCurlRequest::post(const std::string& url, bool res = addEasy(easy); return res; } + +bool LLCurlRequest::post(const std::string& url, + const headers_t& headers, + const std::string& data, + LLCurl::ResponderPtr responder, S32 time_out) +{ + LLCurl::Easy* easy = allocEasy(); + if (!easy) + { + return false; + } + easy->prepRequest(url, headers, responder, time_out); + + easy->getInput().write(data.data(), data.size()); + S32 bytes = easy->getInput().str().length(); + easy->setopt(CURLOPT_POST, 1); + easy->setopt(CURLOPT_POSTFIELDS, (void*)NULL); + easy->setopt(CURLOPT_POSTFIELDSIZE, bytes); + + easy->slist_append("Content-Type: application/octet-stream"); + easy->setHeaders(); + + lldebugs << "POSTING: " << bytes << " bytes." << llendl; + bool res = addEasy(easy); + return res; +} + // Note: call once per frame S32 LLCurlRequest::process() { llassert_always(mThreadID == LLThread::currentID()); S32 res = 0; + + mProcessing = TRUE; for (curlmulti_set_t::iterator iter = mMultiSet.begin(); iter != mMultiSet.end(); ) { @@ -801,9 +904,20 @@ S32 LLCurlRequest::process() if (multi != mActiveMulti && tres == 0 && multi->mQueued == 0) { mMultiSet.erase(curiter); + multi->mQuitting = true; + if (multi->mThreaded) + { + while (!multi->isStopped()) + { + multi->mSignal->signal(); + apr_sleep(1000); + } + } + delete multi; } } + mProcessing = FALSE; return res; } @@ -817,6 +931,10 @@ S32 LLCurlRequest::getQueued() curlmulti_set_t::iterator curiter = iter++; LLCurl::Multi* multi = *curiter; queued += multi->mQueued; + if (multi->mPerformState != LLCurl::Multi::PERFORM_STATE_READY) + { + ++queued; + } } return queued; } @@ -830,16 +948,31 @@ LLCurlEasyRequest::LLCurlEasyRequest() mResultReturned(false) { mMulti = new LLCurl::Multi(); + if (mMulti->mThreaded) + { + mMulti->start(); + } mEasy = mMulti->allocEasy(); if (mEasy) { mEasy->setErrorBuffer(); mEasy->setCA(); + // Set proxy settings if configured to do so. + LLProxy::getInstance()->applyProxySettings(mEasy); } } LLCurlEasyRequest::~LLCurlEasyRequest() { + mMulti->mQuitting = true; + if (mMulti->mThreaded) + { + while (!mMulti->isStopped()) + { + mMulti->mSignal->signal(); + apr_sleep(1000); + } + } delete mMulti; } @@ -936,14 +1069,20 @@ void LLCurlEasyRequest::requestComplete() } } -S32 LLCurlEasyRequest::perform() +void LLCurlEasyRequest::perform() { - return mMulti->perform(); + mMulti->perform(); } // Usage: Call getRestult until it returns false (no more messages) bool LLCurlEasyRequest::getResult(CURLcode* result, LLCurl::TransferInfo* info) { + if (mMulti->mPerformState != LLCurl::Multi::PERFORM_STATE_COMPLETED) + { //we're busy, try again later + return false; + } + mMulti->mPerformState = LLCurl::Multi::PERFORM_STATE_READY; + if (!mEasy) { // Special case - we failed to initialize a curl_easy (can happen if too many open files) @@ -1028,18 +1167,24 @@ unsigned long LLCurl::ssl_thread_id(void) } #endif -void LLCurl::initClass() +void LLCurl::initClass(bool multi_threaded) { + sMainThreadID = LLThread::currentID(); + sMultiThreaded = multi_threaded; // Do not change this "unless you are familiar with and mean to control // internal operations of libcurl" // - http://curl.haxx.se/libcurl/c/curl_global_init.html - curl_global_init(CURL_GLOBAL_ALL); + CURLcode code = curl_global_init(CURL_GLOBAL_ALL); + + check_curl_code(code); + Easy::sHandleMutex = new LLMutex(); + #if SAFE_SSL S32 mutex_count = CRYPTO_num_locks(); for (S32 i=0; i<mutex_count; i++) { - sSSLMutex.push_back(new LLMutex(NULL)); + sSSLMutex.push_back(new LLMutex); } CRYPTO_set_id_callback(&LLCurl::ssl_thread_id); CRYPTO_set_locking_callback(&LLCurl::ssl_locking_callback); @@ -1052,7 +1197,29 @@ void LLCurl::cleanupClass() CRYPTO_set_locking_callback(NULL); for_each(sSSLMutex.begin(), sSSLMutex.end(), DeletePointer()); #endif - curl_global_cleanup(); + + delete Easy::sHandleMutex; + Easy::sHandleMutex = NULL; + + for (std::set<CURL*>::iterator iter = Easy::sFreeHandles.begin(); iter != Easy::sFreeHandles.end(); ++iter) + { + CURL* curl = *iter; + curl_easy_cleanup(curl); + } + + Easy::sFreeHandles.clear(); + + llassert(Easy::sActiveHandles.empty()); } const unsigned int LLCurl::MAX_REDIRECTS = 5; + +// Provide access to LLCurl free functions outside of llcurl.cpp without polluting the global namespace. +void LLCurlFF::check_easy_code(CURLcode code) +{ + check_curl_code(code); +} +void LLCurlFF::check_multi_code(CURLMcode code) +{ + check_curl_multi_code(code); +} diff --git a/indra/llmessage/llcurl.h b/indra/llmessage/llcurl.h index 64dadd6640..213b281e72 100644 --- a/indra/llmessage/llcurl.h +++ b/indra/llmessage/llcurl.h @@ -41,6 +41,7 @@ #include "llbuffer.h" #include "lliopipe.h" #include "llsd.h" +#include "llthread.h" class LLMutex; @@ -55,6 +56,8 @@ public: class Easy; class Multi; + static bool sMultiThreaded; + struct TransferInfo { TransferInfo() : mSizeDownload(0.0), mTotalTime(0.0), mSpeedDownload(0.0) {} @@ -159,7 +162,7 @@ public: /** * @ brief Initialize LLCurl class */ - static void initClass(); + static void initClass(bool multi_threaded = false); /** * @ brief Cleanup LLCurl class @@ -184,6 +187,122 @@ private: static const unsigned int MAX_REDIRECTS; }; +class LLCurl::Easy +{ + LOG_CLASS(Easy); + +private: + Easy(); + +public: + static Easy* getEasy(); + ~Easy(); + + CURL* getCurlHandle() const { return mCurlEasyHandle; } + + void setErrorBuffer(); + void setCA(); + + void setopt(CURLoption option, S32 value); + // These assume the setter does not free value! + void setopt(CURLoption option, void* value); + void setopt(CURLoption option, char* value); + // Copies the string so that it is guaranteed to stick around + void setoptString(CURLoption option, const std::string& value); + + void slist_append(const char* str); + void setHeaders(); + + U32 report(CURLcode); + void getTransferInfo(LLCurl::TransferInfo* info); + + void prepRequest(const std::string& url, const std::vector<std::string>& headers, ResponderPtr, S32 time_out = 0, bool post = false); + + const char* getErrorBuffer(); + + std::stringstream& getInput() { return mInput; } + std::stringstream& getHeaderOutput() { return mHeaderOutput; } + LLIOPipe::buffer_ptr_t& getOutput() { return mOutput; } + const LLChannelDescriptors& getChannels() { return mChannels; } + + void resetState(); + + static CURL* allocEasyHandle(); + static void releaseEasyHandle(CURL* handle); + +private: + friend class LLCurl; + + CURL* mCurlEasyHandle; + struct curl_slist* mHeaders; + + std::stringstream mRequest; + LLChannelDescriptors mChannels; + LLIOPipe::buffer_ptr_t mOutput; + std::stringstream mInput; + std::stringstream mHeaderOutput; + char mErrorBuffer[CURL_ERROR_SIZE]; + + // Note: char*'s not strings since we pass pointers to curl + std::vector<char*> mStrings; + + ResponderPtr mResponder; + + static std::set<CURL*> sFreeHandles; + static std::set<CURL*> sActiveHandles; + static LLMutex* sHandleMutex; +}; + +class LLCurl::Multi : public LLThread +{ + LOG_CLASS(Multi); +public: + + typedef enum + { + PERFORM_STATE_READY=0, + PERFORM_STATE_PERFORMING=1, + PERFORM_STATE_COMPLETED=2 + } ePerformState; + + Multi(); + ~Multi(); + + Easy* allocEasy(); + bool addEasy(Easy* easy); + + void removeEasy(Easy* easy); + + S32 process(); + void perform(); + void doPerform(); + + virtual void run(); + + CURLMsg* info_read(S32* msgs_in_queue); + + S32 mQueued; + S32 mErrorCount; + + S32 mPerformState; + + LLCondition* mSignal; + bool mQuitting; + bool mThreaded; + +private: + void easyFree(Easy*); + + CURLM* mCurlMultiHandle; + + typedef std::set<Easy*> easy_active_list_t; + easy_active_list_t mEasyActiveList; + typedef std::map<CURL*, Easy*> easy_active_map_t; + easy_active_map_t mEasyActiveMap; + typedef std::set<Easy*> easy_free_list_t; + easy_free_list_t mEasyFreeList; +}; + namespace boost { void intrusive_ptr_add_ref(LLCurl::Responder* p); @@ -201,7 +320,9 @@ public: void get(const std::string& url, LLCurl::ResponderPtr responder); bool getByteRange(const std::string& url, const headers_t& headers, S32 offset, S32 length, LLCurl::ResponderPtr responder); - bool post(const std::string& url, const headers_t& headers, const LLSD& data, LLCurl::ResponderPtr responder); + bool post(const std::string& url, const headers_t& headers, const LLSD& data, LLCurl::ResponderPtr responder, S32 time_out = 0); + bool post(const std::string& url, const headers_t& headers, const std::string& data, LLCurl::ResponderPtr responder, S32 time_out = 0); + S32 process(); S32 getQueued(); @@ -215,6 +336,7 @@ private: curlmulti_set_t mMultiSet; LLCurl::Multi* mActiveMulti; S32 mActiveRequestCount; + BOOL mProcessing; U32 mThreadID; // debug }; @@ -233,10 +355,12 @@ public: void slist_append(const char* str); void sendRequest(const std::string& url); void requestComplete(); - S32 perform(); + void perform(); bool getResult(CURLcode* result, LLCurl::TransferInfo* info = NULL); std::string getErrorString(); + LLCurl::Easy* getEasy() const { return mEasy; } + private: CURLMsg* info_read(S32* queue, LLCurl::TransferInfo* info); @@ -247,4 +371,11 @@ private: bool mResultReturned; }; +// Provide access to LLCurl free functions outside of llcurl.cpp without polluting the global namespace. +namespace LLCurlFF +{ + void check_easy_code(CURLcode code); + void check_multi_code(CURLMcode code); +} + #endif // LL_LLCURL_H diff --git a/indra/llmessage/lldatapacker.h b/indra/llmessage/lldatapacker.h index dd9c4eaa38..b0a638c16e 100644 --- a/indra/llmessage/lldatapacker.h +++ b/indra/llmessage/lldatapacker.h @@ -168,10 +168,15 @@ public: S32 getCurrentSize() const { return (S32)(mCurBufferp - mBufferp); } S32 getBufferSize() const { return mBufferSize; } + const U8* getBuffer() const { return mBufferp; } void reset() { mCurBufferp = mBufferp; mWriteEnabled = (mCurBufferp != NULL); } void freeBuffer() { delete [] mBufferp; mBufferp = mCurBufferp = NULL; mBufferSize = 0; mWriteEnabled = FALSE; } void assignBuffer(U8 *bufferp, S32 size) { + if(mBufferp && mBufferp != bufferp) + { + freeBuffer() ; + } mBufferp = bufferp; mCurBufferp = bufferp; mBufferSize = size; diff --git a/indra/llmessage/llfiltersd2xmlrpc.cpp b/indra/llmessage/llfiltersd2xmlrpc.cpp index 812ef7c151..e0ca056a5f 100644 --- a/indra/llmessage/llfiltersd2xmlrpc.cpp +++ b/indra/llmessage/llfiltersd2xmlrpc.cpp @@ -308,6 +308,7 @@ LLFilterSD2XMLRPCResponse::~LLFilterSD2XMLRPCResponse() } +static LLFastTimer::DeclareTimer FTM_PROCESS_SD2XMLRPC_RESPONSE("SD2XMLRPC Response"); // virtual LLIOPipe::EStatus LLFilterSD2XMLRPCResponse::process_impl( const LLChannelDescriptors& channels, @@ -316,6 +317,8 @@ LLIOPipe::EStatus LLFilterSD2XMLRPCResponse::process_impl( LLSD& context, LLPumpIO* pump) { + LLFastTimer t(FTM_PROCESS_SD2XMLRPC_RESPONSE); + PUMP_DEBUG; // This pipe does not work if it does not have everyting. This // could be addressed by making a stream parser for llsd which @@ -382,6 +385,8 @@ LLFilterSD2XMLRPCRequest::~LLFilterSD2XMLRPCRequest() { } +static LLFastTimer::DeclareTimer FTM_PROCESS_SD2XMLRPC_REQUEST("S22XMLRPC Request"); + // virtual LLIOPipe::EStatus LLFilterSD2XMLRPCRequest::process_impl( const LLChannelDescriptors& channels, @@ -390,6 +395,7 @@ LLIOPipe::EStatus LLFilterSD2XMLRPCRequest::process_impl( LLSD& context, LLPumpIO* pump) { + LLFastTimer t(FTM_PROCESS_SD2XMLRPC_REQUEST); // This pipe does not work if it does not have everyting. This // could be addressed by making a stream parser for llsd which // handled partial information. @@ -586,6 +592,8 @@ LLFilterXMLRPCResponse2LLSD::~LLFilterXMLRPCResponse2LLSD() { } +static LLFastTimer::DeclareTimer FTM_PROCESS_XMLRPC2LLSD_RESPONSE("XMLRPC2LLSD Response"); + LLIOPipe::EStatus LLFilterXMLRPCResponse2LLSD::process_impl( const LLChannelDescriptors& channels, buffer_ptr_t& buffer, @@ -593,6 +601,8 @@ LLIOPipe::EStatus LLFilterXMLRPCResponse2LLSD::process_impl( LLSD& context, LLPumpIO* pump) { + LLFastTimer t(FTM_PROCESS_XMLRPC2LLSD_RESPONSE); + PUMP_DEBUG; if(!eos) return STATUS_BREAK; if(!buffer) return STATUS_ERROR; @@ -668,6 +678,7 @@ LLFilterXMLRPCRequest2LLSD::~LLFilterXMLRPCRequest2LLSD() { } +static LLFastTimer::DeclareTimer FTM_PROCESS_XMLRPC2LLSD_REQUEST("XMLRPC2LLSD Request"); LLIOPipe::EStatus LLFilterXMLRPCRequest2LLSD::process_impl( const LLChannelDescriptors& channels, buffer_ptr_t& buffer, @@ -675,6 +686,7 @@ LLIOPipe::EStatus LLFilterXMLRPCRequest2LLSD::process_impl( LLSD& context, LLPumpIO* pump) { + LLFastTimer t(FTM_PROCESS_XMLRPC2LLSD_REQUEST); PUMP_DEBUG; if(!eos) return STATUS_BREAK; if(!buffer) return STATUS_ERROR; diff --git a/indra/llmessage/llhttpassetstorage.cpp b/indra/llmessage/llhttpassetstorage.cpp index 9ea2ff4153..2bca517e97 100644 --- a/indra/llmessage/llhttpassetstorage.cpp +++ b/indra/llmessage/llhttpassetstorage.cpp @@ -33,6 +33,7 @@ #include "indra_constants.h" #include "message.h" +#include "llproxy.h" #include "llvfile.h" #include "llvfs.h" @@ -174,8 +175,8 @@ LLSD LLHTTPAssetRequest::getFullDetails() const double curl_total_time = -1.0f; double curl_size_upload = -1.0f; double curl_size_download = -1.0f; - long curl_content_length_upload = -1; - long curl_content_length_download = -1; + double curl_content_length_upload = -1.0f; + double curl_content_length_download = -1.0f; long curl_request_size = -1; const char* curl_content_type = NULL; @@ -194,8 +195,8 @@ LLSD LLHTTPAssetRequest::getFullDetails() const sd["curl_total_time"] = curl_total_time; sd["curl_size_upload"] = curl_size_upload; sd["curl_size_download"] = curl_size_download; - sd["curl_content_length_upload"] = (int) curl_content_length_upload; - sd["curl_content_length_download"] = (int) curl_content_length_download; + sd["curl_content_length_upload"] = curl_content_length_upload; + sd["curl_content_length_download"] = curl_content_length_download; sd["curl_request_size"] = (int) curl_request_size; if (curl_content_type) { @@ -232,6 +233,10 @@ void LLHTTPAssetRequest::setupCurlHandle() { // *NOTE: Similar code exists in mapserver/llcurlutil.cpp JC mCurlHandle = curl_easy_init(); + + // Apply proxy settings if configured to do so + LLProxy::getInstance()->applyProxySettings(mCurlHandle); + curl_easy_setopt(mCurlHandle, CURLOPT_NOSIGNAL, 1); curl_easy_setopt(mCurlHandle, CURLOPT_NOPROGRESS, 1); curl_easy_setopt(mCurlHandle, CURLOPT_URL, mURLBuffer.c_str()); diff --git a/indra/llmessage/llhttpclient.cpp b/indra/llmessage/llhttpclient.cpp index 0e5206a520..dd4e3a6300 100644 --- a/indra/llmessage/llhttpclient.cpp +++ b/indra/llmessage/llhttpclient.cpp @@ -428,6 +428,9 @@ static LLSD blocking_request( std::string body_str; // other request method checks root cert first, we skip? + + // Apply configured proxy settings + LLProxy::getInstance()->applyProxySettings(curlp); // * Set curl handle options curl_easy_setopt(curlp, CURLOPT_NOSIGNAL, 1); // don't use SIGALRM for timeouts @@ -436,7 +439,7 @@ static LLSD blocking_request( curl_easy_setopt(curlp, CURLOPT_WRITEDATA, &http_buffer); curl_easy_setopt(curlp, CURLOPT_URL, url.c_str()); curl_easy_setopt(curlp, CURLOPT_ERRORBUFFER, curl_error_buffer); - + // * Setup headers (don't forget to free them after the call!) curl_slist* headers_list = NULL; if (headers.isMap()) diff --git a/indra/llmessage/lliohttpserver.cpp b/indra/llmessage/lliohttpserver.cpp index 3b18a9177c..920a57ab55 100644 --- a/indra/llmessage/lliohttpserver.cpp +++ b/indra/llmessage/lliohttpserver.cpp @@ -140,6 +140,7 @@ private: LLSD mHeaders; }; +static LLFastTimer::DeclareTimer FTM_PROCESS_HTTP_PIPE("HTTP Pipe"); LLIOPipe::EStatus LLHTTPPipe::process_impl( const LLChannelDescriptors& channels, buffer_ptr_t& buffer, @@ -147,6 +148,7 @@ LLIOPipe::EStatus LLHTTPPipe::process_impl( LLSD& context, LLPumpIO* pump) { + LLFastTimer t(FTM_PROCESS_HTTP_PIPE); PUMP_DEBUG; lldebugs << "LLSDHTTPServer::process_impl" << llendl; @@ -428,6 +430,9 @@ protected: /** * LLHTTPResponseHeader */ + +static LLFastTimer::DeclareTimer FTM_PROCESS_HTTP_HEADER("HTTP Header"); + // virtual LLIOPipe::EStatus LLHTTPResponseHeader::process_impl( const LLChannelDescriptors& channels, @@ -436,6 +441,7 @@ LLIOPipe::EStatus LLHTTPResponseHeader::process_impl( LLSD& context, LLPumpIO* pump) { + LLFastTimer t(FTM_PROCESS_HTTP_HEADER); PUMP_DEBUG; LLMemType m1(LLMemType::MTYPE_IO_HTTP_SERVER); if(eos) @@ -630,6 +636,8 @@ void LLHTTPResponder::markBad( << "</body>\n</html>\n"; } +static LLFastTimer::DeclareTimer FTM_PROCESS_HTTP_RESPONDER("HTTP Responder"); + // virtual LLIOPipe::EStatus LLHTTPResponder::process_impl( const LLChannelDescriptors& channels, @@ -638,6 +646,7 @@ LLIOPipe::EStatus LLHTTPResponder::process_impl( LLSD& context, LLPumpIO* pump) { + LLFastTimer t(FTM_PROCESS_HTTP_RESPONDER); PUMP_DEBUG; LLMemType m1(LLMemType::MTYPE_IO_HTTP_SERVER); LLIOPipe::EStatus status = STATUS_OK; @@ -954,13 +963,9 @@ private: // static -LLHTTPNode& LLIOHTTPServer::create( - apr_pool_t* pool, LLPumpIO& pump, U16 port) +LLHTTPNode& LLIOHTTPServer::create(LLPumpIO& pump, U16 port) { - LLSocket::ptr_t socket = LLSocket::create( - pool, - LLSocket::STREAM_TCP, - port); + LLSocket::ptr_t socket = LLSocket::create(LLSocket::STREAM_TCP, port); if(!socket) { llerrs << "Unable to initialize socket" << llendl; @@ -969,7 +974,7 @@ LLHTTPNode& LLIOHTTPServer::create( LLHTTPResponseFactory* factory = new LLHTTPResponseFactory; boost::shared_ptr<LLChainIOFactory> factory_ptr(factory); - LLIOServerSocket* server = new LLIOServerSocket(pool, socket, factory_ptr); + LLIOServerSocket* server = new LLIOServerSocket(socket, factory_ptr); LLPumpIO::chain_t chain; chain.push_back(LLIOPipe::ptr_t(server)); diff --git a/indra/llmessage/lliohttpserver.h b/indra/llmessage/lliohttpserver.h index 5c1b0531ff..2294e4b8ae 100644 --- a/indra/llmessage/lliohttpserver.h +++ b/indra/llmessage/lliohttpserver.h @@ -50,7 +50,7 @@ class LLIOHTTPServer public: typedef void (*timing_callback_t)(const char* hashed_name, F32 time, void* data); - static LLHTTPNode& create(apr_pool_t* pool, LLPumpIO& pump, U16 port); + static LLHTTPNode& create(LLPumpIO& pump, U16 port); /**< Creates an HTTP wire server on the pump for the given TCP port. * * Returns the root node of the new server. Add LLHTTPNode instances diff --git a/indra/llmessage/lliosocket.cpp b/indra/llmessage/lliosocket.cpp index ca84fa8bb8..a885ba8ee1 100644 --- a/indra/llmessage/lliosocket.cpp +++ b/indra/llmessage/lliosocket.cpp @@ -35,6 +35,7 @@ #include "llhost.h" #include "llmemtype.h" #include "llpumpio.h" +#include "llthread.h" // // constants @@ -98,51 +99,31 @@ void ll_debug_socket(const char* msg, apr_socket_t* apr_sock) /// // static -LLSocket::ptr_t LLSocket::create(apr_pool_t* pool, EType type, U16 port) +LLSocket::ptr_t LLSocket::create(EType type, U16 port) { LLMemType m1(LLMemType::MTYPE_IO_TCP); - LLSocket::ptr_t rv; - apr_socket_t* socket = NULL; - apr_pool_t* new_pool = NULL; apr_status_t status = APR_EGENERAL; - - // create a pool for the socket - status = apr_pool_create(&new_pool, pool); - if(ll_apr_warn_status(status)) - { - if(new_pool) apr_pool_destroy(new_pool); - return rv; - } + LLSocket::ptr_t rv(new LLSocket); if(STREAM_TCP == type) { - status = apr_socket_create( - &socket, - APR_INET, - SOCK_STREAM, - APR_PROTO_TCP, - new_pool); + status = apr_socket_create(&rv->mSocket, APR_INET, SOCK_STREAM, APR_PROTO_TCP, rv->mPool()); } else if(DATAGRAM_UDP == type) { - status = apr_socket_create( - &socket, - APR_INET, - SOCK_DGRAM, - APR_PROTO_UDP, - new_pool); + status = apr_socket_create(&rv->mSocket, APR_INET, SOCK_DGRAM, APR_PROTO_UDP, rv->mPool()); } else { - if(new_pool) apr_pool_destroy(new_pool); + rv.reset(); return rv; } if(ll_apr_warn_status(status)) { - if(new_pool) apr_pool_destroy(new_pool); + rv->mSocket = NULL; + rv.reset(); return rv; } - rv = ptr_t(new LLSocket(socket, new_pool)); if(port > 0) { apr_sockaddr_t* sa = NULL; @@ -152,7 +133,7 @@ LLSocket::ptr_t LLSocket::create(apr_pool_t* pool, EType type, U16 port) APR_UNSPEC, port, 0, - new_pool); + rv->mPool()); if(ll_apr_warn_status(status)) { rv.reset(); @@ -160,8 +141,8 @@ LLSocket::ptr_t LLSocket::create(apr_pool_t* pool, EType type, U16 port) } // This allows us to reuse the address on quick down/up. This // is unlikely to create problems. - ll_apr_warn_status(apr_socket_opt_set(socket, APR_SO_REUSEADDR, 1)); - status = apr_socket_bind(socket, sa); + ll_apr_warn_status(apr_socket_opt_set(rv->mSocket, APR_SO_REUSEADDR, 1)); + status = apr_socket_bind(rv->mSocket, sa); if(ll_apr_warn_status(status)) { rv.reset(); @@ -175,7 +156,7 @@ LLSocket::ptr_t LLSocket::create(apr_pool_t* pool, EType type, U16 port) // to keep a queue of incoming connections for ACCEPT. lldebugs << "Setting listen state for socket." << llendl; status = apr_socket_listen( - socket, + rv->mSocket, LL_DEFAULT_LISTEN_BACKLOG); if(ll_apr_warn_status(status)) { @@ -191,26 +172,33 @@ LLSocket::ptr_t LLSocket::create(apr_pool_t* pool, EType type, U16 port) port = PORT_EPHEMERAL; } rv->mPort = port; - rv->setOptions(); + rv->setNonBlocking(); return rv; } // static -LLSocket::ptr_t LLSocket::create(apr_socket_t* socket, apr_pool_t* pool) +LLSocket::ptr_t LLSocket::create(apr_status_t& status, LLSocket::ptr_t& listen_socket) { LLMemType m1(LLMemType::MTYPE_IO_TCP); - LLSocket::ptr_t rv; - if(!socket) + if (!listen_socket->getSocket()) { + status = APR_ENOSOCKET; + return LLSocket::ptr_t(); + } + LLSocket::ptr_t rv(new LLSocket); + lldebugs << "accepting socket" << llendl; + status = apr_socket_accept(&rv->mSocket, listen_socket->getSocket(), rv->mPool()); + if (status != APR_SUCCESS) + { + rv->mSocket = NULL; + rv.reset(); return rv; } - rv = ptr_t(new LLSocket(socket, pool)); rv->mPort = PORT_EPHEMERAL; - rv->setOptions(); + rv->setNonBlocking(); return rv; } - bool LLSocket::blockingConnect(const LLHost& host) { if(!mSocket) return false; @@ -223,24 +211,22 @@ bool LLSocket::blockingConnect(const LLHost& host) APR_UNSPEC, host.getPort(), 0, - mPool))) + mPool()))) { return false; } - apr_socket_timeout_set(mSocket, 1000); + setBlocking(1000); ll_debug_socket("Blocking connect", mSocket); if(ll_apr_warn_status(apr_socket_connect(mSocket, sa))) return false; - setOptions(); + setNonBlocking(); return true; } -LLSocket::LLSocket(apr_socket_t* socket, apr_pool_t* pool) : - mSocket(socket), - mPool(pool), +LLSocket::LLSocket() : + mSocket(NULL), + mPool(LLThread::tldata().mRootPool), mPort(PORT_INVALID) { - ll_debug_socket("Constructing wholely formed socket", mSocket); - LLMemType m1(LLMemType::MTYPE_IO_TCP); } LLSocket::~LLSocket() @@ -251,18 +237,31 @@ LLSocket::~LLSocket() { ll_debug_socket("Destroying socket", mSocket); apr_socket_close(mSocket); - } - if(mPool) - { - apr_pool_destroy(mPool); + mSocket = NULL; } } -void LLSocket::setOptions() +// See http://dev.ariel-networks.com/apr/apr-tutorial/html/apr-tutorial-13.html#ss13.4 +// for an explanation of how to get non-blocking sockets and timeouts with +// consistent behavior across platforms. + +void LLSocket::setBlocking(S32 timeout) +{ + LLMemType m1(LLMemType::MTYPE_IO_TCP); + // set up the socket options + ll_apr_warn_status(apr_socket_timeout_set(mSocket, timeout)); + ll_apr_warn_status(apr_socket_opt_set(mSocket, APR_SO_NONBLOCK, 0)); + ll_apr_warn_status(apr_socket_opt_set(mSocket, APR_SO_SNDBUF, LL_SEND_BUFFER_SIZE)); + ll_apr_warn_status(apr_socket_opt_set(mSocket, APR_SO_RCVBUF, LL_RECV_BUFFER_SIZE)); + +} + +void LLSocket::setNonBlocking() { LLMemType m1(LLMemType::MTYPE_IO_TCP); // set up the socket options ll_apr_warn_status(apr_socket_timeout_set(mSocket, 0)); + ll_apr_warn_status(apr_socket_opt_set(mSocket, APR_SO_NONBLOCK, 1)); ll_apr_warn_status(apr_socket_opt_set(mSocket, APR_SO_SNDBUF, LL_SEND_BUFFER_SIZE)); ll_apr_warn_status(apr_socket_opt_set(mSocket, APR_SO_RCVBUF, LL_RECV_BUFFER_SIZE)); @@ -285,6 +284,8 @@ LLIOSocketReader::~LLIOSocketReader() //lldebugs << "Destroying LLIOSocketReader" << llendl; } +static LLFastTimer::DeclareTimer FTM_PROCESS_SOCKET_READER("Socket Reader"); + // virtual LLIOPipe::EStatus LLIOSocketReader::process_impl( const LLChannelDescriptors& channels, @@ -293,6 +294,7 @@ LLIOPipe::EStatus LLIOSocketReader::process_impl( LLSD& context, LLPumpIO* pump) { + LLFastTimer t(FTM_PROCESS_SOCKET_READER); PUMP_DEBUG; LLMemType m1(LLMemType::MTYPE_IO_TCP); if(!mSource) return STATUS_PRECONDITION_NOT_MET; @@ -385,6 +387,7 @@ LLIOSocketWriter::~LLIOSocketWriter() //lldebugs << "Destroying LLIOSocketWriter" << llendl; } +static LLFastTimer::DeclareTimer FTM_PROCESS_SOCKET_WRITER("Socket Writer"); // virtual LLIOPipe::EStatus LLIOSocketWriter::process_impl( const LLChannelDescriptors& channels, @@ -393,6 +396,7 @@ LLIOPipe::EStatus LLIOSocketWriter::process_impl( LLSD& context, LLPumpIO* pump) { + LLFastTimer t(FTM_PROCESS_SOCKET_WRITER); PUMP_DEBUG; LLMemType m1(LLMemType::MTYPE_IO_TCP); if(!mDestination) return STATUS_PRECONDITION_NOT_MET; @@ -516,10 +520,8 @@ LLIOPipe::EStatus LLIOSocketWriter::process_impl( /// LLIOServerSocket::LLIOServerSocket( - apr_pool_t* pool, LLIOServerSocket::socket_t listener, factory_t factory) : - mPool(pool), mListenSocket(listener), mReactor(factory), mInitialized(false), @@ -539,6 +541,7 @@ void LLIOServerSocket::setResponseTimeout(F32 timeout_secs) mResponseTimeout = timeout_secs; } +static LLFastTimer::DeclareTimer FTM_PROCESS_SERVER_SOCKET("Server Socket"); // virtual LLIOPipe::EStatus LLIOServerSocket::process_impl( const LLChannelDescriptors& channels, @@ -547,6 +550,7 @@ LLIOPipe::EStatus LLIOServerSocket::process_impl( LLSD& context, LLPumpIO* pump) { + LLFastTimer t(FTM_PROCESS_SERVER_SOCKET); PUMP_DEBUG; LLMemType m1(LLMemType::MTYPE_IO_TCP); if(!pump) @@ -579,21 +583,15 @@ LLIOPipe::EStatus LLIOServerSocket::process_impl( lldebugs << "accepting socket" << llendl; PUMP_DEBUG; - apr_pool_t* new_pool = NULL; - apr_status_t status = apr_pool_create(&new_pool, mPool); - apr_socket_t* socket = NULL; - status = apr_socket_accept( - &socket, - mListenSocket->getSocket(), - new_pool); - LLSocket::ptr_t llsocket(LLSocket::create(socket, new_pool)); + apr_status_t status; + LLSocket::ptr_t llsocket(LLSocket::create(status, mListenSocket)); //EStatus rv = STATUS_ERROR; - if(llsocket) + if(llsocket && status == APR_SUCCESS) { PUMP_DEBUG; apr_sockaddr_t* remote_addr; - apr_socket_addr_get(&remote_addr, APR_REMOTE, socket); + apr_socket_addr_get(&remote_addr, APR_REMOTE, llsocket->getSocket()); char* remote_host_string; apr_sockaddr_ip_get(&remote_host_string, remote_addr); @@ -608,7 +606,6 @@ LLIOPipe::EStatus LLIOServerSocket::process_impl( { chain.push_back(LLIOPipe::ptr_t(new LLIOSocketWriter(llsocket))); pump->addChain(chain, mResponseTimeout); - status = STATUS_OK; } else { @@ -617,7 +614,8 @@ LLIOPipe::EStatus LLIOServerSocket::process_impl( } else { - llwarns << "Unable to create linden socket." << llendl; + char buf[256]; + llwarns << "Unable to accept linden socket: " << apr_strerror(status, buf, sizeof(buf)) << llendl; } PUMP_DEBUG; @@ -630,11 +628,10 @@ LLIOPipe::EStatus LLIOServerSocket::process_impl( #if 0 LLIODataSocket::LLIODataSocket( U16 suggested_port, - U16 start_discovery_port, - apr_pool_t* pool) : + U16 start_discovery_port) : mSocket(NULL) { - if(!pool || (PORT_INVALID == suggested_port)) return; + if(PORT_INVALID == suggested_port) return; if(ll_apr_warn_status(apr_socket_create(&mSocket, APR_INET, SOCK_DGRAM, APR_PROTO_UDP, pool))) return; apr_sockaddr_t* sa = NULL; if(ll_apr_warn_status(apr_sockaddr_info_get(&sa, APR_ANYADDR, APR_UNSPEC, suggested_port, 0, pool))) return; diff --git a/indra/llmessage/lliosocket.h b/indra/llmessage/lliosocket.h index 6806e5084a..f0a6f25657 100644 --- a/indra/llmessage/lliosocket.h +++ b/indra/llmessage/lliosocket.h @@ -38,7 +38,6 @@ */ #include "lliopipe.h" -#include "apr_pools.h" #include "apr_network_io.h" #include "llchainio.h" @@ -88,34 +87,22 @@ public: * socket. If you intend the socket to be known to external * clients without prior port notification, do not use * PORT_EPHEMERAL. - * @param pool The apr pool to use. A child pool will be created - * and associated with the socket. * @param type The type of socket to create * @param port The port for the socket * @return A valid socket shared pointer if the call worked. */ static ptr_t create( - apr_pool_t* pool, EType type, U16 port = PORT_EPHEMERAL); /** - * @brief Create a LLSocket when you already have an apr socket. + * @brief Create a LLSocket by accepting a connection from a listen socket. * - * This method assumes an ephemeral port. This is typically used - * by calls which spawn a socket such as a call to - * <code>accept()</code> as in the server socket. This call should - * not fail if you have a valid apr socket. - * Because of the nature of how accept() works, you are expected - * to create a new pool for the socket, use that pool for the - * accept, and pass it in here where it will be bound with the - * socket and destroyed at the same time. - * @param socket The apr socket to use - * @param pool The pool used to create the socket. *NOTE: The pool - * passed in will be DESTROYED. + * @param status Output. Status of the accept if a valid listen socket was passed. + * @param listen_socket The listen socket to use. * @return A valid socket shared pointer if the call worked. */ - static ptr_t create(apr_socket_t* socket, apr_pool_t* pool); + static ptr_t create(apr_status_t& status, ptr_t& listen_socket); /** * @brief Perform a blocking connect to a host. Do not use in production. @@ -145,17 +132,30 @@ public: */ apr_socket_t* getSocket() const { return mSocket; } -protected: /** * @brief Protected constructor since should only make sockets * with one of the two <code>create()</code> calls. */ - LLSocket(apr_socket_t* socket, apr_pool_t* pool); + LLSocket(void); /** - * @brief Set default socket options. + * @brief Set default socket options, with SO_NONBLOCK = 0 and a timeout in us. + * @param timeout Number of microseconds to wait on this socket. Any + * negative number means block-forever. TIMEOUT OF 0 IS NON-PORTABLE. + */ + void setBlocking(S32 timeout); + + /** + * @brief Set default socket options, with SO_NONBLOCK = 1 and timeout = 0. + */ + void setNonBlocking(); + +protected: + /** + * @brief Protected constructor since should only make sockets + * with one of the two <code>create()</code> calls. */ - void setOptions(); + LLSocket(apr_socket_t* socket, apr_pool_t* pool); public: /** @@ -167,8 +167,8 @@ protected: // The apr socket. apr_socket_t* mSocket; - // our memory pool - apr_pool_t* mPool; + // Our memory pool. + LLAPRPool mPool; // The port if we know it. U16 mPort; @@ -293,7 +293,7 @@ class LLIOServerSocket : public LLIOPipe public: typedef LLSocket::ptr_t socket_t; typedef boost::shared_ptr<LLChainIOFactory> factory_t; - LLIOServerSocket(apr_pool_t* pool, socket_t listener, factory_t reactor); + LLIOServerSocket(socket_t listener, factory_t reactor); virtual ~LLIOServerSocket(); /** @@ -325,7 +325,6 @@ protected: //@} protected: - apr_pool_t* mPool; socket_t mListenSocket; factory_t mReactor; bool mInitialized; @@ -359,8 +358,7 @@ public: */ LLIODataSocket( U16 suggested_port, - U16 start_discovery_port, - apr_pool_t* pool); + U16 start_discovery_port); virtual ~LLIODataSocket(); protected: diff --git a/indra/llmessage/llioutil.cpp b/indra/llmessage/llioutil.cpp index 2e6ee59ff2..8c50fd5069 100644 --- a/indra/llmessage/llioutil.cpp +++ b/indra/llmessage/llioutil.cpp @@ -43,6 +43,8 @@ LLIOPipe::EStatus LLIOFlush::process_impl( return STATUS_OK; } + +static LLFastTimer::DeclareTimer FTM_PROCESS_SLEEP("IO Sleep"); /** * @class LLIOSleep */ @@ -53,6 +55,7 @@ LLIOPipe::EStatus LLIOSleep::process_impl( LLSD& context, LLPumpIO* pump) { + LLFastTimer t(FTM_PROCESS_SLEEP); if(mSeconds > 0.0) { if(pump) pump->sleepChain(mSeconds); @@ -62,6 +65,7 @@ LLIOPipe::EStatus LLIOSleep::process_impl( return STATUS_DONE; } +static LLFastTimer::DeclareTimer FTM_PROCESS_ADD_CHAIN("Add Chain"); /** * @class LLIOAddChain */ @@ -72,6 +76,7 @@ LLIOPipe::EStatus LLIOAddChain::process_impl( LLSD& context, LLPumpIO* pump) { + LLFastTimer t(FTM_PROCESS_ADD_CHAIN); pump->addChain(mChain, mTimeout); return STATUS_DONE; } diff --git a/indra/llmessage/llmail.cpp b/indra/llmessage/llmail.cpp index 08b31e9c7a..8a898ab1b0 100644 --- a/indra/llmessage/llmail.cpp +++ b/indra/llmessage/llmail.cpp @@ -50,6 +50,7 @@ #include "llstring.h" #include "lluuid.h" #include "net.h" +#include "llaprpool.h" // // constants @@ -57,7 +58,7 @@ const size_t LL_MAX_KNOWN_GOOD_MAIL_SIZE = 4096; static bool gMailEnabled = true; -static apr_pool_t* gMailPool; +static LLAPRPool gMailPool; static apr_sockaddr_t* gSockAddr; static apr_socket_t* gMailSocket; @@ -82,7 +83,7 @@ bool connect_smtp() gSockAddr->sa.sin.sin_family, SOCK_STREAM, APR_PROTO_TCP, - gMailPool); + gMailPool()); if(ll_apr_warn_status(status)) return false; status = apr_socket_connect(gMailSocket, gSockAddr); if(ll_apr_warn_status(status)) @@ -139,19 +140,19 @@ BOOL LLMail::send( } // static -void LLMail::init(const std::string& hostname, apr_pool_t* pool) +void LLMail::init(const std::string& hostname) { gMailSocket = NULL; - if(hostname.empty() || !pool) + if (hostname.empty()) { - gMailPool = NULL; gSockAddr = NULL; + gMailPool.destroy(); } else { - gMailPool = pool; + gMailPool.create(); - // collect all the information into a socaddr sturcture. the + // Collect all the information into a sockaddr structure. the // documentation is a bit unclear, but I either have to // specify APR_UNSPEC or not specify any flags. I am not sure // which option is better. @@ -161,7 +162,7 @@ void LLMail::init(const std::string& hostname, apr_pool_t* pool) APR_UNSPEC, 25, APR_IPV4_ADDR_OK, - gMailPool); + gMailPool()); ll_apr_warn_status(status); } } diff --git a/indra/llmessage/llmail.h b/indra/llmessage/llmail.h index 3791714363..0a5c532088 100644 --- a/indra/llmessage/llmail.h +++ b/indra/llmessage/llmail.h @@ -27,15 +27,13 @@ #ifndef LL_LLMAIL_H #define LL_LLMAIL_H -typedef struct apr_pool_t apr_pool_t; - #include "llsd.h" class LLMail { public: // if hostname is NULL, then the host is resolved as 'mail' - static void init(const std::string& hostname, apr_pool_t* pool); + static void init(const std::string& hostname); // Allow all email transmission to be disabled/enabled. static void enable(bool mail_enabled); diff --git a/indra/llmessage/llpacketring.cpp b/indra/llmessage/llpacketring.cpp index 8999dec64a..fc6e9c5193 100644 --- a/indra/llmessage/llpacketring.cpp +++ b/indra/llmessage/llpacketring.cpp @@ -28,11 +28,20 @@ #include "llpacketring.h" +#if LL_WINDOWS + #include <winsock2.h> +#else + #include <sys/socket.h> + #include <netinet/in.h> +#endif + // linden library includes #include "llerror.h" #include "lltimer.h" -#include "timing.h" +#include "llproxy.h" #include "llrand.h" +#include "message.h" +#include "timing.h" #include "u64.h" /////////////////////////////////////////////////////////// @@ -216,8 +225,32 @@ S32 LLPacketRing::receivePacket (S32 socket, char *datap) else { // no delay, pull straight from net - packet_size = receive_packet(socket, datap); - mLastSender = ::get_sender(); + if (LLProxy::isSOCKSProxyEnabled()) + { + U8 buffer[NET_BUFFER_SIZE + SOCKS_HEADER_SIZE]; + packet_size = receive_packet(socket, static_cast<char*>(static_cast<void*>(buffer))); + + if (packet_size > SOCKS_HEADER_SIZE) + { + // *FIX We are assuming ATYP is 0x01 (IPv4), not 0x03 (hostname) or 0x04 (IPv6) + memcpy(datap, buffer + SOCKS_HEADER_SIZE, packet_size - SOCKS_HEADER_SIZE); + proxywrap_t * header = static_cast<proxywrap_t*>(static_cast<void*>(buffer)); + mLastSender.setAddress(header->addr); + mLastSender.setPort(ntohs(header->port)); + + packet_size -= SOCKS_HEADER_SIZE; // The unwrapped packet size + } + else + { + packet_size = 0; + } + } + else + { + packet_size = receive_packet(socket, datap); + mLastSender = ::get_sender(); + } + mLastReceivingIF = ::get_receiving_interface(); if (packet_size) // did we actually get a packet? @@ -243,7 +276,7 @@ BOOL LLPacketRing::sendPacket(int h_socket, char * send_buffer, S32 buf_size, LL BOOL status = TRUE; if (!mUseOutThrottle) { - return send_packet(h_socket, send_buffer, buf_size, host.getAddress(), host.getPort() ); + return sendPacketImpl(h_socket, send_buffer, buf_size, host ); } else { @@ -264,7 +297,7 @@ BOOL LLPacketRing::sendPacket(int h_socket, char * send_buffer, S32 buf_size, LL mOutBufferLength -= packetp->getSize(); packet_size = packetp->getSize(); - status = send_packet(h_socket, packetp->getData(), packet_size, packetp->getHost().getAddress(), packetp->getHost().getPort()); + status = sendPacketImpl(h_socket, packetp->getData(), packet_size, packetp->getHost()); delete packetp; // Update the throttle @@ -273,7 +306,7 @@ BOOL LLPacketRing::sendPacket(int h_socket, char * send_buffer, S32 buf_size, LL else { // If the queue's empty, we can just send this packet right away. - status = send_packet(h_socket, send_buffer, buf_size, host.getAddress(), host.getPort() ); + status = sendPacketImpl(h_socket, send_buffer, buf_size, host ); packet_size = buf_size; // Update the throttle @@ -311,3 +344,29 @@ BOOL LLPacketRing::sendPacket(int h_socket, char * send_buffer, S32 buf_size, LL return status; } + +BOOL LLPacketRing::sendPacketImpl(int h_socket, const char * send_buffer, S32 buf_size, LLHost host) +{ + + if (!LLProxy::isSOCKSProxyEnabled()) + { + return send_packet(h_socket, send_buffer, buf_size, host.getAddress(), host.getPort()); + } + + char headered_send_buffer[NET_BUFFER_SIZE + SOCKS_HEADER_SIZE]; + + proxywrap_t *socks_header = static_cast<proxywrap_t*>(static_cast<void*>(&headered_send_buffer)); + socks_header->rsv = 0; + socks_header->addr = host.getAddress(); + socks_header->port = htons(host.getPort()); + socks_header->atype = ADDRESS_IPV4; + socks_header->frag = 0; + + memcpy(headered_send_buffer + SOCKS_HEADER_SIZE, send_buffer, buf_size); + + return send_packet( h_socket, + headered_send_buffer, + buf_size + SOCKS_HEADER_SIZE, + LLProxy::getInstance()->getUDPProxy().getAddress(), + LLProxy::getInstance()->getUDPProxy().getPort()); +} diff --git a/indra/llmessage/llpacketring.h b/indra/llmessage/llpacketring.h index e6409d2048..b214271e78 100644 --- a/indra/llmessage/llpacketring.h +++ b/indra/llmessage/llpacketring.h @@ -30,11 +30,11 @@ #include <queue> -#include "llpacketbuffer.h" #include "llhost.h" -#include "net.h" +#include "llpacketbuffer.h" +#include "llproxy.h" #include "llthrottle.h" - +#include "net.h" class LLPacketRing { @@ -82,6 +82,9 @@ protected: LLHost mLastSender; LLHost mLastReceivingIF; + +private: + BOOL sendPacketImpl(int h_socket, const char * send_buffer, S32 buf_size, LLHost host); }; diff --git a/indra/llmessage/llproxy.cpp b/indra/llmessage/llproxy.cpp new file mode 100644 index 0000000000..4a7d326c0e --- /dev/null +++ b/indra/llmessage/llproxy.cpp @@ -0,0 +1,546 @@ +/** + * @file llproxy.cpp + * @brief UDP and HTTP proxy communications + * + * $LicenseInfo:firstyear=2011&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2011, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include "linden_common.h" + +#include "llproxy.h" + +#include <string> +#include <curl/curl.h> + +#include "llapr.h" +#include "llcurl.h" +#include "llhost.h" + +// Static class variable instances + +// We want this to be static to avoid excessive indirection on every +// incoming packet just to do a simple bool test. The getter for this +// member is also static +bool LLProxy::sUDPProxyEnabled = false; + +// Some helpful TCP static functions. +static apr_status_t tcp_blocking_handshake(LLSocket::ptr_t handle, char * dataout, apr_size_t outlen, char * datain, apr_size_t maxinlen); // Do a TCP data handshake +static LLSocket::ptr_t tcp_open_channel(LLHost host); // Open a TCP channel to a given host +static void tcp_close_channel(LLSocket::ptr_t* handle_ptr); // Close an open TCP channel + +LLProxy::LLProxy(): + mHTTPProxyEnabled(false), + mProxyMutex(), + mUDPProxy(), + mTCPProxy(), + mHTTPProxy(), + mProxyType(LLPROXY_SOCKS), + mAuthMethodSelected(METHOD_NOAUTH), + mSocksUsername(), + mSocksPassword() +{ +} + +LLProxy::~LLProxy() +{ + stopSOCKSProxy(); + disableHTTPProxy(); +} + +/** + * @brief Open the SOCKS 5 TCP control channel. + * + * Perform a SOCKS 5 authentication and UDP association with the proxy server. + * + * @param proxy The SOCKS 5 server to connect to. + * @return SOCKS_OK if successful, otherwise a socks error code from llproxy.h. + */ +S32 LLProxy::proxyHandshake(LLHost proxy) +{ + S32 result; + + /* SOCKS 5 Auth request */ + socks_auth_request_t socks_auth_request; + socks_auth_response_t socks_auth_response; + + socks_auth_request.version = SOCKS_VERSION; // SOCKS version 5 + socks_auth_request.num_methods = 1; // Sending 1 method. + socks_auth_request.methods = getSelectedAuthMethod(); // Send only the selected method. + + result = tcp_blocking_handshake(mProxyControlChannel, + static_cast<char*>(static_cast<void*>(&socks_auth_request)), + sizeof(socks_auth_request), + static_cast<char*>(static_cast<void*>(&socks_auth_response)), + sizeof(socks_auth_response)); + if (result != APR_SUCCESS) + { + LL_WARNS("Proxy") << "SOCKS authentication request failed, error on TCP control channel : " << result << LL_ENDL; + stopSOCKSProxy(); + return SOCKS_CONNECT_ERROR; + } + + if (socks_auth_response.method == AUTH_NOT_ACCEPTABLE) + { + LL_WARNS("Proxy") << "SOCKS 5 server refused all our authentication methods." << LL_ENDL; + stopSOCKSProxy(); + return SOCKS_NOT_ACCEPTABLE; + } + + /* SOCKS 5 USERNAME/PASSWORD authentication */ + if (socks_auth_response.method == METHOD_PASSWORD) + { + // The server has requested a username/password combination + std::string socks_username(getSocksUser()); + std::string socks_password(getSocksPwd()); + U32 request_size = socks_username.size() + socks_password.size() + 3; + char * password_auth = new char[request_size]; + password_auth[0] = 0x01; + password_auth[1] = socks_username.size(); + memcpy(&password_auth[2], socks_username.c_str(), socks_username.size()); + password_auth[socks_username.size() + 2] = socks_password.size(); + memcpy(&password_auth[socks_username.size() + 3], socks_password.c_str(), socks_password.size()); + + authmethod_password_reply_t password_reply; + + result = tcp_blocking_handshake(mProxyControlChannel, + password_auth, + request_size, + static_cast<char*>(static_cast<void*>(&password_reply)), + sizeof(password_reply)); + delete[] password_auth; + + if (result != APR_SUCCESS) + { + LL_WARNS("Proxy") << "SOCKS authentication failed, error on TCP control channel : " << result << LL_ENDL; + stopSOCKSProxy(); + return SOCKS_CONNECT_ERROR; + } + + if (password_reply.status != AUTH_SUCCESS) + { + LL_WARNS("Proxy") << "SOCKS authentication failed" << LL_ENDL; + stopSOCKSProxy(); + return SOCKS_AUTH_FAIL; + } + } + + /* SOCKS5 connect request */ + + socks_command_request_t connect_request; + socks_command_response_t connect_reply; + + connect_request.version = SOCKS_VERSION; // SOCKS V5 + connect_request.command = COMMAND_UDP_ASSOCIATE; // Associate UDP + connect_request.reserved = FIELD_RESERVED; + connect_request.atype = ADDRESS_IPV4; + connect_request.address = htonl(0); // 0.0.0.0 + connect_request.port = htons(0); // 0 + // "If the client is not in possession of the information at the time of the UDP ASSOCIATE, + // the client MUST use a port number and address of all zeros. RFC 1928" + + result = tcp_blocking_handshake(mProxyControlChannel, + static_cast<char*>(static_cast<void*>(&connect_request)), + sizeof(connect_request), + static_cast<char*>(static_cast<void*>(&connect_reply)), + sizeof(connect_reply)); + if (result != APR_SUCCESS) + { + LL_WARNS("Proxy") << "SOCKS connect request failed, error on TCP control channel : " << result << LL_ENDL; + stopSOCKSProxy(); + return SOCKS_CONNECT_ERROR; + } + + if (connect_reply.reply != REPLY_REQUEST_GRANTED) + { + LL_WARNS("Proxy") << "Connection to SOCKS 5 server failed, UDP forward request not granted" << LL_ENDL; + stopSOCKSProxy(); + return SOCKS_UDP_FWD_NOT_GRANTED; + } + + mUDPProxy.setPort(ntohs(connect_reply.port)); // reply port is in network byte order + mUDPProxy.setAddress(proxy.getAddress()); + // The connection was successful. We now have the UDP port to send requests that need forwarding to. + LL_INFOS("Proxy") << "SOCKS 5 UDP proxy connected on " << mUDPProxy << LL_ENDL; + + return SOCKS_OK; +} + +/** + * @brief Initiates a SOCKS 5 proxy session. + * + * Performs basic checks on host to verify that it is a valid address. Opens the control channel + * and then negotiates the proxy connection with the server. Closes any existing SOCKS + * connection before proceeding. Also disables an HTTP proxy if it is using SOCKS as the proxy. + * + * + * @param host Socks server to connect to. + * @return SOCKS_OK if successful, otherwise a SOCKS error code defined in llproxy.h. + */ +S32 LLProxy::startSOCKSProxy(LLHost host) +{ + if (host.isOk()) + { + mTCPProxy = host; + } + else + { + return SOCKS_INVALID_HOST; + } + + // Close any running SOCKS connection. + stopSOCKSProxy(); + + mProxyControlChannel = tcp_open_channel(mTCPProxy); + if (!mProxyControlChannel) + { + return SOCKS_HOST_CONNECT_FAILED; + } + + S32 status = proxyHandshake(mTCPProxy); + + if (status != SOCKS_OK) + { + // Shut down the proxy if any of the above steps failed. + stopSOCKSProxy(); + } + else + { + // Connection was successful. + sUDPProxyEnabled = true; + } + + return status; +} + +/** + * @brief Stop using the SOCKS 5 proxy. + * + * This will stop sending UDP packets through the SOCKS 5 proxy + * and will also stop the HTTP proxy if it is configured to use SOCKS. + * The proxy control channel will also be disconnected. + */ +void LLProxy::stopSOCKSProxy() +{ + sUDPProxyEnabled = false; + + // If the SOCKS proxy is requested to stop and we are using that for HTTP as well + // then we must shut down any HTTP proxy operations. But it is allowable if web + // proxy is being used to continue proxying HTTP. + + if (LLPROXY_SOCKS == getHTTPProxyType()) + { + disableHTTPProxy(); + } + + if (mProxyControlChannel) + { + tcp_close_channel(&mProxyControlChannel); + } +} + +/** + * @brief Set the proxy's SOCKS authentication method to none. + */ +void LLProxy::setAuthNone() +{ + LLMutexLock lock(&mProxyMutex); + + mAuthMethodSelected = METHOD_NOAUTH; +} + +/** + * @brief Set the proxy's SOCKS authentication method to password. + * + * Check whether the lengths of the supplied username + * and password conform to the lengths allowed by the + * SOCKS protocol. + * + * @param username The SOCKS username to send. + * @param password The SOCKS password to send. + * @return Return true if applying the settings was successful. No changes are made if false. + * + */ +bool LLProxy::setAuthPassword(const std::string &username, const std::string &password) +{ + if (username.length() > SOCKSMAXUSERNAMELEN || password.length() > SOCKSMAXPASSWORDLEN || + username.length() < SOCKSMINUSERNAMELEN || password.length() < SOCKSMINPASSWORDLEN) + { + LL_WARNS("Proxy") << "Invalid SOCKS 5 password or username length." << LL_ENDL; + return false; + } + + LLMutexLock lock(&mProxyMutex); + + mAuthMethodSelected = METHOD_PASSWORD; + mSocksUsername = username; + mSocksPassword = password; + + return true; +} + +/** + * @brief Enable the HTTP proxy for either SOCKS or HTTP. + * + * Check the supplied host to see if it is a valid IP and port. + * + * @param httpHost Proxy server to connect to. + * @param type Is the host a SOCKS or HTTP proxy. + * @return Return true if applying the setting was successful. No changes are made if false. + */ +bool LLProxy::enableHTTPProxy(LLHost httpHost, LLHttpProxyType type) +{ + if (!httpHost.isOk()) + { + LL_WARNS("Proxy") << "Invalid SOCKS 5 Server" << LL_ENDL; + return false; + } + + LLMutexLock lock(&mProxyMutex); + + mHTTPProxy = httpHost; + mProxyType = type; + + mHTTPProxyEnabled = true; + + return true; +} + +/** + * @brief Enable the HTTP proxy without changing the proxy settings. + * + * This should not be called unless the proxy has already been set up. + * + * @return Return true only if the current settings are valid and the proxy was enabled. + */ +bool LLProxy::enableHTTPProxy() +{ + bool ok; + + LLMutexLock lock(&mProxyMutex); + + ok = (mHTTPProxy.isOk()); + if (ok) + { + mHTTPProxyEnabled = true; + } + + return ok; +} + +/** + * @brief Disable the HTTP proxy. + */ +void LLProxy::disableHTTPProxy() +{ + LLMutexLock lock(&mProxyMutex); + + mHTTPProxyEnabled = false; +} + +/** + * @brief Get the currently selected HTTP proxy type + */ +LLHttpProxyType LLProxy::getHTTPProxyType() const +{ + LLMutexLock lock(&mProxyMutex); + return mProxyType; +} + +/** + * @brief Get the SOCKS 5 password. + */ +std::string LLProxy::getSocksPwd() const +{ + LLMutexLock lock(&mProxyMutex); + return mSocksPassword; +} + +/** + * @brief Get the SOCKS 5 username. + */ +std::string LLProxy::getSocksUser() const +{ + LLMutexLock lock(&mProxyMutex); + return mSocksUsername; +} + +/** + * @brief Get the currently selected SOCKS 5 authentication method. + * + * @return Returns either none or password. + */ +LLSocks5AuthType LLProxy::getSelectedAuthMethod() const +{ + LLMutexLock lock(&mProxyMutex); + return mAuthMethodSelected; +} + +/** + * @brief Stop the LLProxy and make certain that any APR pools and classes are deleted before terminating APR. + * + * Deletes the LLProxy singleton, destroying the APR pool used by the control channel as well as . + */ +//static +void LLProxy::cleanupClass() +{ + getInstance()->stopSOCKSProxy(); + deleteSingleton(); +} + +void LLProxy::applyProxySettings(LLCurlEasyRequest* handle) +{ + applyProxySettings(handle->getEasy()); +} + +void LLProxy::applyProxySettings(LLCurl::Easy* handle) +{ + applyProxySettings(handle->getCurlHandle()); +} + +/** + * @brief Apply proxy settings to a CuRL request if an HTTP proxy is enabled. + * + * This method has been designed to be safe to call from + * any thread in the viewer. This allows requests in the + * texture fetch thread to be aware of the proxy settings. + * When the HTTP proxy is enabled, the proxy mutex will + * be locked every time this method is called. + * + * @param handle A pointer to a valid CURL request, before it has been performed. + */ +void LLProxy::applyProxySettings(CURL* handle) +{ + // Do a faster unlocked check to see if we are supposed to proxy. + if (mHTTPProxyEnabled) + { + // We think we should proxy, lock the proxy mutex. + LLMutexLock lock(&mProxyMutex); + // Now test again to verify that the proxy wasn't disabled between the first check and the lock. + if (mHTTPProxyEnabled) + { + LLCurlFF::check_easy_code(curl_easy_setopt(handle, CURLOPT_PROXY, mHTTPProxy.getIPString().c_str())); + LLCurlFF::check_easy_code(curl_easy_setopt(handle, CURLOPT_PROXYPORT, mHTTPProxy.getPort())); + + if (mProxyType == LLPROXY_SOCKS) + { + LLCurlFF::check_easy_code(curl_easy_setopt(handle, CURLOPT_PROXYTYPE, CURLPROXY_SOCKS5)); + if (mAuthMethodSelected == METHOD_PASSWORD) + { + std::string auth_string = mSocksUsername + ":" + mSocksPassword; + LLCurlFF::check_easy_code(curl_easy_setopt(handle, CURLOPT_PROXYUSERPWD, auth_string.c_str())); + } + } + else + { + LLCurlFF::check_easy_code(curl_easy_setopt(handle, CURLOPT_PROXYTYPE, CURLPROXY_HTTP)); + } + } + } +} + +/** + * @brief Send one TCP packet and receive one in return. + * + * This operation is done synchronously with a 1000ms timeout. Therefore, it should not be used when a blocking + * operation would impact the operation of the viewer. + * + * @param handle_ptr Pointer to a connected LLSocket of type STREAM_TCP. + * @param dataout Data to send. + * @param outlen Length of dataout. + * @param datain Buffer for received data. Undefined if return value is not APR_SUCCESS. + * @param maxinlen Maximum possible length of received data. Short reads are allowed. + * @return Indicates APR status code of exchange. APR_SUCCESS if exchange was successful, -1 if invalid data length was received. + */ +static apr_status_t tcp_blocking_handshake(LLSocket::ptr_t handle, char * dataout, apr_size_t outlen, char * datain, apr_size_t maxinlen) +{ + apr_socket_t* apr_socket = handle->getSocket(); + apr_status_t rv = APR_SUCCESS; + + apr_size_t expected_len = outlen; + + handle->setBlocking(1000); + + rv = apr_socket_send(apr_socket, dataout, &outlen); + if (APR_SUCCESS != rv) + { + LL_WARNS("Proxy") << "Error sending data to proxy control channel, status: " << rv << LL_ENDL; + ll_apr_warn_status(rv); + } + else if (expected_len != outlen) + { + LL_WARNS("Proxy") << "Incorrect data length sent. Expected: " << expected_len << + " Sent: " << outlen << LL_ENDL; + rv = -1; + } + + if (APR_SUCCESS == rv) + { + expected_len = maxinlen; + rv = apr_socket_recv(apr_socket, datain, &maxinlen); + if (rv != APR_SUCCESS) + { + LL_WARNS("Proxy") << "Error receiving data from proxy control channel, status: " << rv << LL_ENDL; + ll_apr_warn_status(rv); + } + else if (expected_len < maxinlen) + { + LL_WARNS("Proxy") << "Incorrect data length received. Expected: " << expected_len << + " Received: " << maxinlen << LL_ENDL; + rv = -1; + } + } + + handle->setNonBlocking(); + + return rv; +} + +/** + * @brief Open a LLSocket and do a blocking connect to the chosen host. + * + * Checks for a successful connection, and makes sure the connection is closed if it fails. + * + * @param host The host to open the connection to. + * @return The created socket. Will evaluate as NULL if the connection is unsuccessful. + */ +static LLSocket::ptr_t tcp_open_channel(LLHost host) +{ + LLSocket::ptr_t socket = LLSocket::create(LLSocket::STREAM_TCP); + bool connected = socket->blockingConnect(host); + if (!connected) + { + tcp_close_channel(&socket); + } + + return socket; +} + +/** + * @brief Close the socket. + * + * @param handle_ptr The handle of the socket being closed. A pointer-to-pointer to avoid increasing the use count. + */ +static void tcp_close_channel(LLSocket::ptr_t* handle_ptr) +{ + LL_DEBUGS("Proxy") << "Resetting proxy LLSocket handle, use_count == " << handle_ptr->use_count() << LL_ENDL; + handle_ptr->reset(); +} diff --git a/indra/llmessage/llproxy.h b/indra/llmessage/llproxy.h new file mode 100644 index 0000000000..a919370540 --- /dev/null +++ b/indra/llmessage/llproxy.h @@ -0,0 +1,352 @@ +/** + * @file llproxy.h + * @brief UDP and HTTP proxy communications + * + * $LicenseInfo:firstyear=2011&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2011, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef LL_PROXY_H +#define LL_PROXY_H + +#include "llcurl.h" +#include "llhost.h" +#include "lliosocket.h" +#include "llmemory.h" +#include "llsingleton.h" +#include "llthread.h" +#include <string> + +// SOCKS error codes returned from the StartProxy method +#define SOCKS_OK 0 +#define SOCKS_CONNECT_ERROR (-1) +#define SOCKS_NOT_PERMITTED (-2) +#define SOCKS_NOT_ACCEPTABLE (-3) +#define SOCKS_AUTH_FAIL (-4) +#define SOCKS_UDP_FWD_NOT_GRANTED (-5) +#define SOCKS_HOST_CONNECT_FAILED (-6) +#define SOCKS_INVALID_HOST (-7) + +#ifndef MAXHOSTNAMELEN +#define MAXHOSTNAMELEN (255 + 1) /* socks5: 255, +1 for len. */ +#endif + +#define SOCKSMAXUSERNAMELEN 255 +#define SOCKSMAXPASSWORDLEN 255 + +#define SOCKSMINUSERNAMELEN 1 +#define SOCKSMINPASSWORDLEN 1 + +#define SOCKS_VERSION 0x05 // we are using SOCKS 5 + +#define SOCKS_HEADER_SIZE 10 + +// SOCKS 5 address/hostname types +#define ADDRESS_IPV4 0x01 +#define ADDRESS_HOSTNAME 0x03 +#define ADDRESS_IPV6 0x04 + +// Lets just use our own ipv4 struct rather than dragging in system +// specific headers +union ipv4_address_t { + U8 octets[4]; + U32 addr32; +}; + +// SOCKS 5 control channel commands +#define COMMAND_TCP_STREAM 0x01 +#define COMMAND_TCP_BIND 0x02 +#define COMMAND_UDP_ASSOCIATE 0x03 + +// SOCKS 5 command replies +#define REPLY_REQUEST_GRANTED 0x00 +#define REPLY_GENERAL_FAIL 0x01 +#define REPLY_RULESET_FAIL 0x02 +#define REPLY_NETWORK_UNREACHABLE 0x03 +#define REPLY_HOST_UNREACHABLE 0x04 +#define REPLY_CONNECTION_REFUSED 0x05 +#define REPLY_TTL_EXPIRED 0x06 +#define REPLY_PROTOCOL_ERROR 0x07 +#define REPLY_TYPE_NOT_SUPPORTED 0x08 + +#define FIELD_RESERVED 0x00 + +// The standard SOCKS 5 request packet +// Push current alignment to stack and set alignment to 1 byte boundary +// This enabled us to use structs directly to set up and receive network packets +// into the correct fields, without fear of boundary alignment causing issues +#pragma pack(push,1) + +// SOCKS 5 command packet +struct socks_command_request_t { + U8 version; + U8 command; + U8 reserved; + U8 atype; + U32 address; + U16 port; +}; + +// Standard SOCKS 5 reply packet +struct socks_command_response_t { + U8 version; + U8 reply; + U8 reserved; + U8 atype; + U8 add_bytes[4]; + U16 port; +}; + +#define AUTH_NOT_ACCEPTABLE 0xFF // reply if preferred methods are not available +#define AUTH_SUCCESS 0x00 // reply if authentication successful + +// SOCKS 5 authentication request, stating which methods the client supports +struct socks_auth_request_t { + U8 version; + U8 num_methods; + U8 methods; // We are only using a single method currently +}; + +// SOCKS 5 authentication response packet, stating server preferred method +struct socks_auth_response_t { + U8 version; + U8 method; +}; + +// SOCKS 5 password reply packet +struct authmethod_password_reply_t { + U8 version; + U8 status; +}; + +// SOCKS 5 UDP packet header +struct proxywrap_t { + U16 rsv; + U8 frag; + U8 atype; + U32 addr; + U16 port; +}; + +#pragma pack(pop) /* restore original alignment from stack */ + + +// Currently selected HTTP proxy type +enum LLHttpProxyType +{ + LLPROXY_SOCKS = 0, + LLPROXY_HTTP = 1 +}; + +// Auth types +enum LLSocks5AuthType +{ + METHOD_NOAUTH = 0x00, // Client supports no auth + METHOD_GSSAPI = 0x01, // Client supports GSSAPI (Not currently supported) + METHOD_PASSWORD = 0x02 // Client supports username/password +}; + +/** + * @brief Manage SOCKS 5 UDP proxy and HTTP proxy. + * + * This class is responsible for managing two interconnected tasks, + * connecting to a SOCKS 5 proxy for use by LLPacketRing to send UDP + * packets and managing proxy settings for HTTP requests. + * + * <h1>Threading:</h1> + * Because HTTP requests can be generated in threads outside the + * main thread, it is necessary for some of the information stored + * by this class to be available to other threads. The members that + * need to be read across threads are in a labeled section below. + * To protect those members, a mutex, mProxyMutex should be locked + * before reading or writing those members. Methods that can lock + * mProxyMutex are in a labeled section below. Those methods should + * not be called while the mutex is already locked. + * + * There is also a LLAtomic type flag (mHTTPProxyEnabled) that is used + * to track whether the HTTP proxy is currently enabled. This allows + * for faster unlocked checks to see if the proxy is enabled. This + * allows us to cut down on the performance hit when the proxy is + * disabled compared to before this class was introduced. + * + * <h1>UDP Proxying:</h1> + * UDP datagrams are proxied via a SOCKS 5 proxy with the UDP associate + * command. To initiate the proxy, a TCP socket connection is opened + * to the SOCKS 5 host, and after a handshake exchange, the server + * returns a port and address to send the UDP traffic that is to be + * proxied to. The LLProxy class tracks this address and port after the + * exchange and provides it to LLPacketRing when required to. All UDP + * proxy management occurs in the main thread. + * + * <h1>HTTP Proxying:</h1> + * This class allows all viewer HTTP packets to be sent through a proxy. + * The user can select whether to send HTTP packets through a standard + * "web" HTTP proxy, through a SOCKS 5 proxy, or to not proxy HTTP + * communication. This class does not manage the integrated web browser + * proxy, which is handled in llviewermedia.cpp. + * + * The implementation of HTTP proxying is handled by libcurl. LLProxy + * is responsible for managing the HTTP proxy options and provides a + * thread-safe method to apply those options to a curl request + * (LLProxy::applyProxySettings()). This method is overloaded + * to accommodate the various abstraction libcurl layers that exist + * throughout the viewer (LLCurlEasyRequest, LLCurl::Easy, and CURL). + * + * If you are working with LLCurl or LLCurlEasyRequest objects, + * the configured proxy settings will be applied in the constructors + * of those request handles. If you are working with CURL objects + * directly, you will need to pass the handle of the request to + * applyProxySettings() before issuing the request. + * + * To ensure thread safety, all LLProxy members that relate to the HTTP + * proxy require the LLProxyMutex to be locked before accessing. + */ +class LLProxy: public LLSingleton<LLProxy> +{ + LOG_CLASS(LLProxy); +public: + /*########################################################################################### + METHODS THAT DO NOT LOCK mProxyMutex! + ###########################################################################################*/ + // Constructor, cannot have parameters due to LLSingleton parent class. Call from main thread only. + LLProxy(); + + // Static check for enabled status for UDP packets. Call from main thread only. + static bool isSOCKSProxyEnabled() { return sUDPProxyEnabled; } + + // Get the UDP proxy address and port. Call from main thread only. + LLHost getUDPProxy() const { return mUDPProxy; } + + /*########################################################################################### + END OF NON-LOCKING METHODS + ###########################################################################################*/ + + /*########################################################################################### + METHODS THAT LOCK mProxyMutex! DO NOT CALL WHILE mProxyMutex IS LOCKED! + ###########################################################################################*/ + // Destructor, closes open connections. Do not call directly, use cleanupClass(). + ~LLProxy(); + + // Delete LLProxy singleton. Allows the apr_socket used in the SOCKS 5 control channel to be + // destroyed before the call to apr_terminate. Call from main thread only. + static void cleanupClass(); + + // Apply the current proxy settings to a curl request. Doesn't do anything if mHTTPProxyEnabled is false. + // Safe to call from any thread. + void applyProxySettings(CURL* handle); + void applyProxySettings(LLCurl::Easy* handle); + void applyProxySettings(LLCurlEasyRequest* handle); + + // Start a connection to the SOCKS 5 proxy. Call from main thread only. + S32 startSOCKSProxy(LLHost host); + + // Disconnect and clean up any connection to the SOCKS 5 proxy. Call from main thread only. + void stopSOCKSProxy(); + + // Use Password auth when connecting to the SOCKS proxy. Call from main thread only. + bool setAuthPassword(const std::string &username, const std::string &password); + + // Disable authentication when connecting to the SOCKS proxy. Call from main thread only. + void setAuthNone(); + + // Proxy HTTP packets via httpHost, which can be a SOCKS 5 or a HTTP proxy. + // as specified in type. Call from main thread only. + bool enableHTTPProxy(LLHost httpHost, LLHttpProxyType type); + bool enableHTTPProxy(); + + // Stop proxying HTTP packets. Call from main thread only. + void disableHTTPProxy(); + + /*########################################################################################### + END OF LOCKING METHODS + ###########################################################################################*/ +private: + /*########################################################################################### + METHODS THAT LOCK mProxyMutex! DO NOT CALL WHILE mProxyMutex IS LOCKED! + ###########################################################################################*/ + + // Perform a SOCKS 5 authentication and UDP association with the proxy server. + S32 proxyHandshake(LLHost proxy); + + // Get the currently selected auth method. + LLSocks5AuthType getSelectedAuthMethod() const; + + // Get the currently selected HTTP proxy type + LLHttpProxyType getHTTPProxyType() const; + + std::string getSocksPwd() const; + std::string getSocksUser() const; + + /*########################################################################################### + END OF LOCKING METHODS + ###########################################################################################*/ + +private: + // Is the HTTP proxy enabled? Safe to read in any thread, but do not write directly. + // Instead use enableHTTPProxy() and disableHTTPProxy() instead. + mutable LLAtomic32<bool> mHTTPProxyEnabled; + + // Mutex to protect shared members in non-main thread calls to applyProxySettings(). + mutable LLMutex mProxyMutex; + + /*########################################################################################### + MEMBERS READ AND WRITTEN ONLY IN THE MAIN THREAD. DO NOT SHARE! + ###########################################################################################*/ + + // Is the UDP proxy enabled? + static bool sUDPProxyEnabled; + + // UDP proxy address and port + LLHost mUDPProxy; + // TCP proxy control channel address and port + LLHost mTCPProxy; + + // socket handle to proxy TCP control channel + LLSocket::ptr_t mProxyControlChannel; + + /*########################################################################################### + END OF UNSHARED MEMBERS + ###########################################################################################*/ + + /*########################################################################################### + MEMBERS WRITTEN IN MAIN THREAD AND READ IN ANY THREAD. ONLY READ OR WRITE AFTER LOCKING mProxyMutex! + ###########################################################################################*/ + + // HTTP proxy address and port + LLHost mHTTPProxy; + + // Currently selected HTTP proxy type. Can be web or socks. + LLHttpProxyType mProxyType; + + // SOCKS 5 selected authentication method. + LLSocks5AuthType mAuthMethodSelected; + + // SOCKS 5 username + std::string mSocksUsername; + // SOCKS 5 password + std::string mSocksPassword; + + /*########################################################################################### + END OF SHARED MEMBERS + ###########################################################################################*/ +}; + +#endif diff --git a/indra/llmessage/llpumpio.cpp b/indra/llmessage/llpumpio.cpp index a8d2a0a224..89cfd66e1b 100644 --- a/indra/llmessage/llpumpio.cpp +++ b/indra/llmessage/llpumpio.cpp @@ -37,6 +37,7 @@ #include "llmemtype.h" #include "llstl.h" #include "llstat.h" +#include "llthread.h" // These should not be enabled in production, but they can be // intensely useful during development for finding certain kinds of @@ -162,14 +163,12 @@ struct ll_delete_apr_pollset_fd_client_data /** * LLPumpIO */ -LLPumpIO::LLPumpIO(apr_pool_t* pool) : +LLPumpIO::LLPumpIO(void) : mState(LLPumpIO::NORMAL), mRebuildPollset(false), mPollset(NULL), mPollsetClientID(0), mNextLock(0), - mPool(NULL), - mCurrentPool(NULL), mCurrentPoolReallocCount(0), mChainsMutex(NULL), mCallbackMutex(NULL), @@ -178,21 +177,24 @@ LLPumpIO::LLPumpIO(apr_pool_t* pool) : mCurrentChain = mRunningChains.end(); LLMemType m1(LLMemType::MTYPE_IO_PUMP); - initialize(pool); + initialize(); } LLPumpIO::~LLPumpIO() { LLMemType m1(LLMemType::MTYPE_IO_PUMP); - cleanup(); -} - -bool LLPumpIO::prime(apr_pool_t* pool) -{ - LLMemType m1(LLMemType::MTYPE_IO_PUMP); - cleanup(); - initialize(pool); - return ((pool == NULL) ? false : true); +#if LL_THREADS_APR + if (mChainsMutex) apr_thread_mutex_destroy(mChainsMutex); + if (mCallbackMutex) apr_thread_mutex_destroy(mCallbackMutex); +#endif + mChainsMutex = NULL; + mCallbackMutex = NULL; + if(mPollset) + { +// lldebugs << "cleaning up pollset" << llendl; + apr_pollset_destroy(mPollset); + mPollset = NULL; + } } bool LLPumpIO::addChain(const chain_t& chain, F32 timeout) @@ -352,8 +354,7 @@ bool LLPumpIO::setConditional(LLIOPipe* pipe, const apr_pollfd_t* poll) { // each fd needs a pool to work with, so if one was // not specified, use this pool. - // *FIX: Should it always be this pool? - value.second.p = mPool; + value.second.p = (*mCurrentChain).mDescriptorsPool->operator()(); } value.second.client_data = new S32(++mPollsetClientID); (*mCurrentChain).mDescriptors.push_back(value); @@ -825,39 +826,15 @@ void LLPumpIO::control(LLPumpIO::EControl op) } } -void LLPumpIO::initialize(apr_pool_t* pool) +void LLPumpIO::initialize(void) { LLMemType m1(LLMemType::MTYPE_IO_PUMP); - if(!pool) return; + mPool.create(); #if LL_THREADS_APR // SJB: Windows defaults to NESTED and OSX defaults to UNNESTED, so use UNNESTED explicitly. - apr_thread_mutex_create(&mChainsMutex, APR_THREAD_MUTEX_UNNESTED, pool); - apr_thread_mutex_create(&mCallbackMutex, APR_THREAD_MUTEX_UNNESTED, pool); -#endif - mPool = pool; -} - -void LLPumpIO::cleanup() -{ - LLMemType m1(LLMemType::MTYPE_IO_PUMP); -#if LL_THREADS_APR - if(mChainsMutex) apr_thread_mutex_destroy(mChainsMutex); - if(mCallbackMutex) apr_thread_mutex_destroy(mCallbackMutex); + apr_thread_mutex_create(&mChainsMutex, APR_THREAD_MUTEX_UNNESTED, mPool()); + apr_thread_mutex_create(&mCallbackMutex, APR_THREAD_MUTEX_UNNESTED, mPool()); #endif - mChainsMutex = NULL; - mCallbackMutex = NULL; - if(mPollset) - { -// lldebugs << "cleaning up pollset" << llendl; - apr_pollset_destroy(mPollset); - mPollset = NULL; - } - if(mCurrentPool) - { - apr_pool_destroy(mCurrentPool); - mCurrentPool = NULL; - } - mPool = NULL; } void LLPumpIO::rebuildPollset() @@ -885,21 +862,19 @@ void LLPumpIO::rebuildPollset() if(mCurrentPool && (0 == (++mCurrentPoolReallocCount % POLLSET_POOL_RECYCLE_COUNT))) { - apr_pool_destroy(mCurrentPool); - mCurrentPool = NULL; + mCurrentPool.destroy(); mCurrentPoolReallocCount = 0; } if(!mCurrentPool) { - apr_status_t status = apr_pool_create(&mCurrentPool, mPool); - (void)ll_apr_warn_status(status); + mCurrentPool.create(mPool); } // add all of the file descriptors run_it = mRunningChains.begin(); LLChainInfo::conditionals_t::iterator fd_it; LLChainInfo::conditionals_t::iterator fd_end; - apr_pollset_create(&mPollset, size, mCurrentPool, 0); + apr_pollset_create(&mPollset, size, mCurrentPool(), 0); for(; run_it != run_end; ++run_it) { fd_it = (*run_it).mDescriptors.begin(); @@ -1157,7 +1132,8 @@ bool LLPumpIO::handleChainError( LLPumpIO::LLChainInfo::LLChainInfo() : mInit(false), mLock(0), - mEOS(false) + mEOS(false), + mDescriptorsPool(new LLAPRPool(LLThread::tldata().mRootPool)) { LLMemType m1(LLMemType::MTYPE_IO_PUMP); mTimer.setTimerExpirySec(DEFAULT_CHAIN_EXPIRY_SECS); diff --git a/indra/llmessage/llpumpio.h b/indra/llmessage/llpumpio.h index 9303c9d7fc..75c35ae7ab 100644 --- a/indra/llmessage/llpumpio.h +++ b/indra/llmessage/llpumpio.h @@ -30,11 +30,12 @@ #define LL_LLPUMPIO_H #include <set> +#include <boost/shared_ptr.hpp> #if LL_LINUX // needed for PATH_MAX in APR. #include <sys/param.h> #endif -#include "apr_pools.h" +#include "llaprpool.h" #include "llbuffer.h" #include "llframetimer.h" #include "lliopipe.h" @@ -58,9 +59,8 @@ extern const F32 NEVER_CHAIN_EXPIRY_SECS; * <code>pump()</code> on a thread used for IO and call * <code>respond()</code> on a thread that is expected to do higher * level processing. You can call almost any other method from any - * thread - see notes for each method for details. In order for the - * threading abstraction to work, you need to call <code>prime()</code> - * with a valid apr pool. + * thread - see notes for each method for details. + * * A pump instance manages much of the state for the pipe, including * the list of pipes in the chain, the channel for each element in the * chain, the buffer, and if any pipe has marked the stream or process @@ -79,7 +79,7 @@ public: /** * @brief Constructor. */ - LLPumpIO(apr_pool_t* pool); + LLPumpIO(void); /** * @brief Destructor. @@ -87,17 +87,6 @@ public: ~LLPumpIO(); /** - * @brief Prepare this pump for usage. - * - * If you fail to call this method prior to use, the pump will - * try to work, but will not come with any thread locking - * mechanisms. - * @param pool The apr pool to use. - * @return Returns true if the pump is primed. - */ - bool prime(apr_pool_t* pool); - - /** * @brief Typedef for having a chain of pipes. */ typedef std::vector<LLIOPipe::ptr_t> chain_t; @@ -368,6 +357,7 @@ protected: typedef std::pair<LLIOPipe::ptr_t, apr_pollfd_t> pipe_conditional_t; typedef std::vector<pipe_conditional_t> conditionals_t; conditionals_t mDescriptors; + boost::shared_ptr<LLAPRPool> mDescriptorsPool; }; // All the running chains & info @@ -386,9 +376,9 @@ protected: callbacks_t mPendingCallbacks; callbacks_t mCallbacks; - // memory allocator for pollsets & mutexes. - apr_pool_t* mPool; - apr_pool_t* mCurrentPool; + // Memory pool for pollsets & mutexes. + LLAPRPool mPool; + LLAPRPool mCurrentPool; S32 mCurrentPoolReallocCount; #if LL_THREADS_APR @@ -400,8 +390,7 @@ protected: #endif protected: - void initialize(apr_pool_t* pool); - void cleanup(); + void initialize(); /** * @brief Given the internal state of the chains, rebuild the pollset diff --git a/indra/llmessage/llsdmessagebuilder.cpp b/indra/llmessage/llsdmessagebuilder.cpp index 42c179782f..2698a271ee 100644 --- a/indra/llmessage/llsdmessagebuilder.cpp +++ b/indra/llmessage/llsdmessagebuilder.cpp @@ -29,6 +29,7 @@ #include "llsdmessagebuilder.h" #include "llmessagetemplate.h" +#include "llmath.h" #include "llquaternion.h" #include "llsdutil.h" #include "llsdutil_math.h" diff --git a/indra/llmessage/llsdrpcclient.cpp b/indra/llmessage/llsdrpcclient.cpp index 86fe5c7912..91fd070f07 100644 --- a/indra/llmessage/llsdrpcclient.cpp +++ b/indra/llmessage/llsdrpcclient.cpp @@ -82,6 +82,8 @@ bool LLSDRPCResponse::extractResponse(const LLSD& sd) return rv; } +static LLFastTimer::DeclareTimer FTM_SDRPC_RESPONSE("SDRPC Response"); + // virtual LLIOPipe::EStatus LLSDRPCResponse::process_impl( const LLChannelDescriptors& channels, @@ -90,6 +92,7 @@ LLIOPipe::EStatus LLSDRPCResponse::process_impl( LLSD& context, LLPumpIO* pump) { + LLFastTimer t(FTM_SDRPC_RESPONSE); PUMP_DEBUG; LLMemType m1(LLMemType::MTYPE_IO_SD_CLIENT); if(mIsError) @@ -178,6 +181,8 @@ bool LLSDRPCClient::call( return true; } +static LLFastTimer::DeclareTimer FTM_PROCESS_SDRPC_CLIENT("SDRPC Client"); + // virtual LLIOPipe::EStatus LLSDRPCClient::process_impl( const LLChannelDescriptors& channels, @@ -186,6 +191,7 @@ LLIOPipe::EStatus LLSDRPCClient::process_impl( LLSD& context, LLPumpIO* pump) { + LLFastTimer t(FTM_PROCESS_SDRPC_CLIENT); PUMP_DEBUG; LLMemType m1(LLMemType::MTYPE_IO_SD_CLIENT); if((STATE_NONE == mState) || (!pump)) diff --git a/indra/llmessage/llsdrpcserver.cpp b/indra/llmessage/llsdrpcserver.cpp index f87c418fb1..9f776aca72 100644 --- a/indra/llmessage/llsdrpcserver.cpp +++ b/indra/llmessage/llsdrpcserver.cpp @@ -97,6 +97,8 @@ void LLSDRPCServer::clearLock() } } +static LLFastTimer::DeclareTimer FTM_PROCESS_SDRPC_SERVER("SDRPC Server"); + // virtual LLIOPipe::EStatus LLSDRPCServer::process_impl( const LLChannelDescriptors& channels, @@ -105,6 +107,7 @@ LLIOPipe::EStatus LLSDRPCServer::process_impl( LLSD& context, LLPumpIO* pump) { + LLFastTimer t(FTM_PROCESS_SDRPC_SERVER); PUMP_DEBUG; LLMemType m1(LLMemType::MTYPE_IO_SD_SERVER); // lldebugs << "LLSDRPCServer::process_impl" << llendl; diff --git a/indra/llmessage/lltemplatemessagebuilder.cpp b/indra/llmessage/lltemplatemessagebuilder.cpp index 6611d704e6..9e8eb48460 100644 --- a/indra/llmessage/lltemplatemessagebuilder.cpp +++ b/indra/llmessage/lltemplatemessagebuilder.cpp @@ -29,6 +29,7 @@ #include "lltemplatemessagebuilder.h" #include "llmessagetemplate.h" +#include "llmath.h" #include "llquaternion.h" #include "u64.h" #include "v3dmath.h" diff --git a/indra/llmessage/lltemplatemessagereader.cpp b/indra/llmessage/lltemplatemessagereader.cpp index 3bfcd58c69..ab91f74abe 100644 --- a/indra/llmessage/lltemplatemessagereader.cpp +++ b/indra/llmessage/lltemplatemessagereader.cpp @@ -30,6 +30,7 @@ #include "llfasttimer.h" #include "llmessagebuilder.h" #include "llmessagetemplate.h" +#include "llmath.h" #include "llquaternion.h" #include "message.h" #include "u64.h" @@ -794,7 +795,7 @@ const char* LLTemplateMessageReader::getMessageName() const { if (!mCurrentRMessageTemplate) { - llwarns << "no mCurrentRMessageTemplate" << llendl; + // no message currently being read return ""; } return mCurrentRMessageTemplate->mName; diff --git a/indra/llmessage/lltransfermanager.cpp b/indra/llmessage/lltransfermanager.cpp index 754eb99cbd..034680caf8 100644 --- a/indra/llmessage/lltransfermanager.cpp +++ b/indra/llmessage/lltransfermanager.cpp @@ -338,7 +338,7 @@ void LLTransferManager::processTransferInfo(LLMessageSystem *msgp, void **) } } - llinfos << "Receiving " << transfer_id << ", size " << size << " bytes" << llendl; + //llinfos << "Receiving " << transfer_id << ", size " << size << " bytes" << llendl; ttp->setSize(size); ttp->setGotInfo(TRUE); diff --git a/indra/llmessage/lltransfersourceasset.cpp b/indra/llmessage/lltransfersourceasset.cpp index 7e57841580..8537773a3f 100644 --- a/indra/llmessage/lltransfersourceasset.cpp +++ b/indra/llmessage/lltransfersourceasset.cpp @@ -251,3 +251,4 @@ BOOL LLTransferSourceParamsAsset::unpackParams(LLDataPacker &dp) return TRUE; } + diff --git a/indra/llmessage/llurlrequest.cpp b/indra/llmessage/llurlrequest.cpp index 28bd09fc4c..91a5a8ce2c 100644 --- a/indra/llmessage/llurlrequest.cpp +++ b/indra/llmessage/llurlrequest.cpp @@ -35,11 +35,13 @@ #include "llcurl.h" #include "llioutil.h" #include "llmemtype.h" +#include "llproxy.h" #include "llpumpio.h" #include "llsd.h" #include "llstring.h" #include "apr_env.h" #include "llapr.h" +#include "llscopedvolatileaprpool.h" static const U32 HTTP_STATUS_PIPE_ERROR = 499; /** @@ -210,27 +212,31 @@ void LLURLRequest::setCallback(LLURLRequestComplete* callback) // is called with use_proxy = FALSE void LLURLRequest::useProxy(bool use_proxy) { - static char *env_proxy; + static std::string env_proxy; - if (use_proxy && (env_proxy == NULL)) + if (use_proxy && env_proxy.empty()) { - apr_status_t status; - LLAPRPool pool; - status = apr_env_get(&env_proxy, "ALL_PROXY", pool.getAPRPool()); + char* env_proxy_str; + LLScopedVolatileAPRPool scoped_pool; + apr_status_t status = apr_env_get(&env_proxy_str, "ALL_PROXY", scoped_pool); if (status != APR_SUCCESS) { - status = apr_env_get(&env_proxy, "http_proxy", pool.getAPRPool()); + status = apr_env_get(&env_proxy_str, "http_proxy", scoped_pool); } if (status != APR_SUCCESS) { - use_proxy = FALSE; + use_proxy = false; } + else + { + // env_proxy_str is stored in the scoped_pool, so we have to make a copy. + env_proxy = env_proxy_str; + } } + LL_DEBUGS("Proxy") << "use_proxy = " << (use_proxy?'Y':'N') << ", env_proxy = " << (!env_proxy.empty() ? env_proxy : "(null)") << LL_ENDL; - lldebugs << "use_proxy = " << (use_proxy?'Y':'N') << ", env_proxy = " << (env_proxy ? env_proxy : "(null)") << llendl; - - if (env_proxy && use_proxy) + if (use_proxy && !env_proxy.empty()) { mDetail->mCurlRequest->setoptString(CURLOPT_PROXY, env_proxy); } @@ -270,6 +276,8 @@ LLIOPipe::EStatus LLURLRequest::handleError( return status; } +static LLFastTimer::DeclareTimer FTM_PROCESS_URL_REQUEST("URL Request"); + // virtual LLIOPipe::EStatus LLURLRequest::process_impl( const LLChannelDescriptors& channels, @@ -278,6 +286,7 @@ LLIOPipe::EStatus LLURLRequest::process_impl( LLSD& context, LLPumpIO* pump) { + LLFastTimer t(FTM_PROCESS_URL_REQUEST); PUMP_DEBUG; LLMemType m1(LLMemType::MTYPE_IO_URL_REQUEST); //llinfos << "LLURLRequest::process_impl()" << llendl; @@ -288,6 +297,8 @@ LLIOPipe::EStatus LLURLRequest::process_impl( const S32 MIN_ACCUMULATION = 100000; if(pump && (mDetail->mByteAccumulator > MIN_ACCUMULATION)) { + static LLFastTimer::DeclareTimer FTM_URL_ADJUST_TIMEOUT("Adjust Timeout"); + LLFastTimer t(FTM_URL_ADJUST_TIMEOUT); // This is a pretty sloppy calculation, but this // tries to make the gross assumption that if data // is coming in at 56kb/s, then this transfer will @@ -335,16 +346,30 @@ LLIOPipe::EStatus LLURLRequest::process_impl( { PUMP_DEBUG; LLIOPipe::EStatus status = STATUS_BREAK; - mDetail->mCurlRequest->perform(); + static LLFastTimer::DeclareTimer FTM_URL_PERFORM("Perform"); + { + LLFastTimer t(FTM_URL_PERFORM); + mDetail->mCurlRequest->perform(); + } + while(1) { CURLcode result; - bool newmsg = mDetail->mCurlRequest->getResult(&result); + + static LLFastTimer::DeclareTimer FTM_PROCESS_URL_REQUEST_GET_RESULT("Get Result"); + + bool newmsg = false; + { + LLFastTimer t(FTM_PROCESS_URL_REQUEST_GET_RESULT); + newmsg = mDetail->mCurlRequest->getResult(&result); + } + if(!newmsg) { // keep processing break; } + mState = STATE_HAVE_RESPONSE; context[CONTEXT_REQUEST][CONTEXT_TRANSFERED_BYTES] = mRequestTransferedBytes; @@ -370,7 +395,11 @@ LLIOPipe::EStatus LLURLRequest::process_impl( link.mChannels = LLBufferArray::makeChannelConsumer( channels); chain.push_back(link); - pump->respond(chain, buffer, context); + static LLFastTimer::DeclareTimer FTM_PROCESS_URL_PUMP_RESPOND("Pump Respond"); + { + LLFastTimer t(FTM_PROCESS_URL_PUMP_RESPOND); + pump->respond(chain, buffer, context); + } mCompletionCallback = NULL; } break; @@ -422,8 +451,11 @@ void LLURLRequest::initialize() mResponseTransferedBytes = 0; } +static LLFastTimer::DeclareTimer FTM_URL_REQUEST_CONFIGURE("URL Configure"); bool LLURLRequest::configure() { + LLFastTimer t(FTM_URL_REQUEST_CONFIGURE); + LLMemType m1(LLMemType::MTYPE_IO_URL_REQUEST); bool rv = false; S32 bytes = mDetail->mResponseBuffer->countAfter( @@ -624,6 +656,7 @@ static size_t headerCallback(void* data, size_t size, size_t nmemb, void* user) return header_len; } +static LLFastTimer::DeclareTimer FTM_PROCESS_URL_EXTRACTOR("URL Extractor"); /** * LLContextURLExtractor */ @@ -635,6 +668,7 @@ LLIOPipe::EStatus LLContextURLExtractor::process_impl( LLSD& context, LLPumpIO* pump) { + LLFastTimer t(FTM_PROCESS_URL_EXTRACTOR); PUMP_DEBUG; LLMemType m1(LLMemType::MTYPE_IO_URL_REQUEST); // The destination host is in the context. @@ -713,6 +747,7 @@ void LLURLRequestComplete::responseStatus(LLIOPipe::EStatus status) mRequestStatus = status; } +static LLFastTimer::DeclareTimer FTM_PROCESS_URL_COMPLETE("URL Complete"); // virtual LLIOPipe::EStatus LLURLRequestComplete::process_impl( const LLChannelDescriptors& channels, @@ -721,6 +756,7 @@ LLIOPipe::EStatus LLURLRequestComplete::process_impl( LLSD& context, LLPumpIO* pump) { + LLFastTimer t(FTM_PROCESS_URL_COMPLETE); PUMP_DEBUG; complete(channels, buffer); return STATUS_OK; diff --git a/indra/llmessage/message.cpp b/indra/llmessage/message.cpp index d0b0e178b8..7d21e35f96 100644 --- a/indra/llmessage/message.cpp +++ b/indra/llmessage/message.cpp @@ -97,8 +97,10 @@ std::string get_shared_secret(); class LLMessagePollInfo { public: + LLMessagePollInfo(void) : mPool(LLThread::tldata().mRootPool) { } apr_socket_t *mAPRSocketp; apr_pollfd_t mPollFD; + LLAPRPool mPool; }; namespace @@ -287,20 +289,13 @@ LLMessageSystem::LLMessageSystem(const std::string& filename, U32 port, } // LL_DEBUGS("Messaging") << << "*** port: " << mPort << llendl; - // - // Create the data structure that we can poll on - // - if (!gAPRPoolp) - { - LL_ERRS("Messaging") << "No APR pool before message system initialization!" << llendl; - ll_init_apr(); - } + mPollInfop = new LLMessagePollInfo; + apr_socket_t *aprSocketp = NULL; - apr_os_sock_put(&aprSocketp, (apr_os_sock_t*)&mSocket, gAPRPoolp); + apr_os_sock_put(&aprSocketp, (apr_os_sock_t*)&mSocket, mPollInfop->mPool()); - mPollInfop = new LLMessagePollInfo; mPollInfop->mAPRSocketp = aprSocketp; - mPollInfop->mPollFD.p = gAPRPoolp; + mPollInfop->mPollFD.p = mPollInfop->mPool(); mPollInfop->mPollFD.desc_type = APR_POLL_SOCKET; mPollInfop->mPollFD.reqevents = APR_POLLIN; mPollInfop->mPollFD.rtnevents = 0; diff --git a/indra/llmessage/message_prehash.cpp b/indra/llmessage/message_prehash.cpp index 5d03615e53..e71fb96540 100644 --- a/indra/llmessage/message_prehash.cpp +++ b/indra/llmessage/message_prehash.cpp @@ -742,6 +742,7 @@ char const* const _PREHASH_MoneyData = LLMessageStringTable::getInstance()->getS char const* const _PREHASH_ObjectDeselect = LLMessageStringTable::getInstance()->getString("ObjectDeselect"); char const* const _PREHASH_NewAssetID = LLMessageStringTable::getInstance()->getString("NewAssetID"); char const* const _PREHASH_ObjectAdd = LLMessageStringTable::getInstance()->getString("ObjectAdd"); +char const* const _PREHASH_SimulatorFeatures = LLMessageStringTable::getInstance()->getString("SimulatorFeatures"); char const* const _PREHASH_RayEndIsIntersection = LLMessageStringTable::getInstance()->getString("RayEndIsIntersection"); char const* const _PREHASH_CompleteAuction = LLMessageStringTable::getInstance()->getString("CompleteAuction"); char const* const _PREHASH_CircuitCode = LLMessageStringTable::getInstance()->getString("CircuitCode"); @@ -1375,3 +1376,6 @@ char const* const _PREHASH_VCoord = LLMessageStringTable::getInstance()->getStri char const* const _PREHASH_FaceIndex = LLMessageStringTable::getInstance()->getString("FaceIndex"); char const* const _PREHASH_StatusData = LLMessageStringTable::getInstance()->getString("StatusData"); char const* const _PREHASH_ProductSKU = LLMessageStringTable::getInstance()->getString("ProductSKU"); +char const* const _PREHASH_SeeAVs = LLMessageStringTable::getInstance()->getString("SeeAVs"); +char const* const _PREHASH_AnyAVSounds = LLMessageStringTable::getInstance()->getString("AnyAVSounds"); +char const* const _PREHASH_GroupAVSounds = LLMessageStringTable::getInstance()->getString("GroupAVSounds"); diff --git a/indra/llmessage/message_prehash.h b/indra/llmessage/message_prehash.h index 8dc86601e6..dd2c2dbd64 100644 --- a/indra/llmessage/message_prehash.h +++ b/indra/llmessage/message_prehash.h @@ -742,6 +742,7 @@ extern char const* const _PREHASH_MoneyData; extern char const* const _PREHASH_ObjectDeselect; extern char const* const _PREHASH_NewAssetID; extern char const* const _PREHASH_ObjectAdd; +extern char const* const _PREHASH_SimulatorFeatures; extern char const* const _PREHASH_RayEndIsIntersection; extern char const* const _PREHASH_CompleteAuction; extern char const* const _PREHASH_CircuitCode; @@ -1375,4 +1376,7 @@ extern char const* const _PREHASH_VCoord; extern char const* const _PREHASH_FaceIndex; extern char const* const _PREHASH_StatusData; extern char const* const _PREHASH_ProductSKU; +extern char const* const _PREHASH_SeeAVs; +extern char const* const _PREHASH_AnyAVSounds; +extern char const* const _PREHASH_GroupAVSounds; #endif diff --git a/indra/llmessage/net.cpp b/indra/llmessage/net.cpp index 97611c3b51..85aef5da00 100644 --- a/indra/llmessage/net.cpp +++ b/indra/llmessage/net.cpp @@ -50,7 +50,6 @@ #include "lltimer.h" #include "indra_constants.h" - // Globals #if LL_WINDOWS @@ -174,7 +173,7 @@ U32 ip_string_to_u32(const char* ip_string) // use wildcard addresses. -Ambroff U32 ip = inet_addr(ip_string); if (ip == INADDR_NONE - && strncmp(ip_string, BROADCAST_ADDRESS_STRING, MAXADDRSTR) != 0) + && strncmp(ip_string, BROADCAST_ADDRESS_STRING, MAXADDRSTR) != 0) { llwarns << "ip_string_to_u32() failed, Error: Invalid IP string '" << ip_string << "'" << llendl; return INVALID_HOST_IP_ADDRESS; @@ -188,11 +187,11 @@ U32 ip_string_to_u32(const char* ip_string) ////////////////////////////////////////////////////////////////////////////////////////// #if LL_WINDOWS - + S32 start_net(S32& socket_out, int& nPort) { // Create socket, make non-blocking - // Init WinSock + // Init WinSock int nRet; int hSocket; @@ -201,7 +200,7 @@ S32 start_net(S32& socket_out, int& nPort) int buff_size = 4; // Initialize windows specific stuff - if(WSAStartup(0x0202, &stWSAData)) + if (WSAStartup(0x0202, &stWSAData)) { S32 err = WSAGetLastError(); WSACleanup(); @@ -210,8 +209,8 @@ S32 start_net(S32& socket_out, int& nPort) } // Get a datagram socket - hSocket = (int)socket(AF_INET, SOCK_DGRAM, 0); - if (hSocket == INVALID_SOCKET) + hSocket = (int)socket(AF_INET, SOCK_DGRAM, 0); + if (hSocket == INVALID_SOCKET) { S32 err = WSAGetLastError(); WSACleanup(); @@ -304,7 +303,7 @@ S32 start_net(S32& socket_out, int& nPort) // Setup a destination address stDstAddr.sin_family = AF_INET; stDstAddr.sin_addr.s_addr = INVALID_HOST_IP_ADDRESS; - stDstAddr.sin_port = htons(nPort); + stDstAddr.sin_port = htons(nPort); socket_out = hSocket; return 0; @@ -393,10 +392,10 @@ S32 start_net(S32& socket_out, int& nPort) int rec_size = RECEIVE_BUFFER_SIZE; socklen_t buff_size = 4; - + // Create socket - hSocket = socket(AF_INET, SOCK_DGRAM, 0); - if (hSocket < 0) + hSocket = socket(AF_INET, SOCK_DGRAM, 0); + if (hSocket < 0) { llwarns << "socket() failed" << llendl; return 1; @@ -429,7 +428,7 @@ S32 start_net(S32& socket_out, int& nPort) } else { - // Name the socket (assign the local port number to receive on) + // Name the socket (assign the local port number to receive on) stLclAddr.sin_family = AF_INET; stLclAddr.sin_addr.s_addr = htonl(INADDR_ANY); stLclAddr.sin_port = htons(nPort); @@ -474,7 +473,7 @@ S32 start_net(S32& socket_out, int& nPort) nPort = attempt_port; } // Set socket to be non-blocking - fcntl(hSocket, F_SETFL, O_NONBLOCK); + fcntl(hSocket, F_SETFL, O_NONBLOCK); // set a large receive buffer nRet = setsockopt(hSocket, SOL_SOCKET, SO_RCVBUF, (char *)&rec_size, buff_size); if (nRet) @@ -510,8 +509,8 @@ S32 start_net(S32& socket_out, int& nPort) // Setup a destination address char achMCAddr[MAXADDRSTR] = "127.0.0.1"; /* Flawfinder: ignore */ stDstAddr.sin_family = AF_INET; - stDstAddr.sin_addr.s_addr = ip_string_to_u32(achMCAddr); - stDstAddr.sin_port = htons(nPort); + stDstAddr.sin_addr.s_addr = ip_string_to_u32(achMCAddr); + stDstAddr.sin_port = htons(nPort); socket_out = hSocket; return 0; @@ -537,7 +536,7 @@ static int recvfrom_destip( int socket, void *buf, int len, struct sockaddr *fro iov[0].iov_base = buf; iov[0].iov_len = len; - memset( &msg, 0, sizeof msg ); + memset(&msg, 0, sizeof msg); msg.msg_name = from; msg.msg_namelen = *fromlen; msg.msg_iov = iov; @@ -545,14 +544,14 @@ static int recvfrom_destip( int socket, void *buf, int len, struct sockaddr *fro msg.msg_control = &cmsg; msg.msg_controllen = sizeof(cmsg); - size = recvmsg( socket, &msg, 0 ); + size = recvmsg(socket, &msg, 0); - if( size == -1 ) + if (size == -1) { return -1; } - for( cmsgptr = CMSG_FIRSTHDR(&msg); cmsgptr != NULL; cmsgptr = CMSG_NXTHDR( &msg, cmsgptr ) ) + for (cmsgptr = CMSG_FIRSTHDR(&msg); cmsgptr != NULL; cmsgptr = CMSG_NXTHDR( &msg, cmsgptr)) { if( cmsgptr->cmsg_level == SOL_IP && cmsgptr->cmsg_type == IP_PKTINFO ) { @@ -650,7 +649,7 @@ BOOL send_packet(int hSocket, const char * sendBuffer, int size, U32 recipient, } } } - while ( resend && send_attempts < 3); + while (resend && send_attempts < 3); if (send_attempts >= 3) { diff --git a/indra/llmessage/net.h b/indra/llmessage/net.h index 9f4f5c5821..0f2437479d 100644 --- a/indra/llmessage/net.h +++ b/indra/llmessage/net.h @@ -46,10 +46,10 @@ S32 receive_packet(int hSocket, char * receiveBuffer); BOOL send_packet(int hSocket, const char *sendBuffer, int size, U32 recipient, int nPort); // Returns TRUE on success. //void get_sender(char * tmp); -LLHost get_sender(); +LLHost get_sender(); U32 get_sender_port(); U32 get_sender_ip(void); -LLHost get_receiving_interface(); +LLHost get_receiving_interface(); U32 get_receiving_interface_ip(void); const char* u32_to_ip_string(U32 ip); // Returns pointer to internal string buffer, "(bad IP addr)" on failure, cannot nest calls diff --git a/indra/llmessage/tests/commtest.h b/indra/llmessage/tests/commtest.h index 32035783e2..0d149b5258 100644 --- a/indra/llmessage/tests/commtest.h +++ b/indra/llmessage/tests/commtest.h @@ -34,7 +34,67 @@ #include "llsd.h" #include "llhost.h" #include "stringize.h" +#include <map> #include <string> +#include <stdexcept> +#include <boost/lexical_cast.hpp> + +struct CommtestError: public std::runtime_error +{ + CommtestError(const std::string& what): std::runtime_error(what) {} +}; + +static bool query_verbose() +{ + const char* cbose = getenv("INTEGRATION_TEST_VERBOSE"); + if (! cbose) + { + cbose = "1"; + } + std::string strbose(cbose); + return (! (strbose == "0" || strbose == "off" || + strbose == "false" || strbose == "quiet")); +} + +bool verbose() +{ + // This should only be initialized once. + static bool vflag = query_verbose(); + return vflag; +} + +static int query_port(const std::string& var) +{ + const char* cport = getenv(var.c_str()); + if (! cport) + { + throw CommtestError(STRINGIZE("missing environment variable" << var)); + } + // This will throw, too, if the value of PORT isn't numeric. + int port(boost::lexical_cast<int>(cport)); + if (verbose()) + { + std::cout << "getport('" << var << "') = " << port << std::endl; + } + return port; +} + +static int getport(const std::string& var) +{ + typedef std::map<std::string, int> portsmap; + static portsmap ports; + // We can do this with a single map lookup with map::insert(). Either it + // returns an existing entry and 'false' (not newly inserted), or it + // inserts the specified value and 'true'. + std::pair<portsmap::iterator, bool> inserted(ports.insert(portsmap::value_type(var, 0))); + if (inserted.second) + { + // We haven't yet seen this var. Remember its value. + inserted.first->second = query_port(var); + } + // Return the (existing or new) iterator's value. + return inserted.first->second; +} /** * This struct is shared by a couple of standalone comm tests (ADD_COMM_BUILD_TEST). @@ -55,13 +115,21 @@ struct commtest_data replyPump("reply"), errorPump("error"), success(false), - host("127.0.0.1", 8000), + host("127.0.0.1", getport("PORT")), server(STRINGIZE("http://" << host.getString() << "/")) { replyPump.listen("self", boost::bind(&commtest_data::outcome, this, _1, true)); errorPump.listen("self", boost::bind(&commtest_data::outcome, this, _1, false)); } + static int getport(const std::string& var) + { + // We have a couple consumers of commtest_data::getport(). But we've + // since moved it out to the global namespace. So this is just a + // facade. + return ::getport(var); + } + bool outcome(const LLSD& _result, bool _success) { // std::cout << "commtest_data::outcome(" << _result << ", " << _success << ")\n"; diff --git a/indra/llmessage/tests/llsdmessage_test.cpp b/indra/llmessage/tests/llsdmessage_test.cpp index 9998a1b8bb..0f2c069303 100644 --- a/indra/llmessage/tests/llsdmessage_test.cpp +++ b/indra/llmessage/tests/llsdmessage_test.cpp @@ -61,6 +61,7 @@ namespace tut llsdmessage_data(): httpPump(pumps.obtain("LLHTTPClient")) { + LLCurl::initClass(); LLSDMessage::link(); } }; diff --git a/indra/llmessage/tests/networkio.h b/indra/llmessage/tests/networkio.h index 2aff90ca1e..23e1c791f4 100644 --- a/indra/llmessage/tests/networkio.h +++ b/indra/llmessage/tests/networkio.h @@ -30,7 +30,6 @@ #define LL_NETWORKIO_H #include "llmemory.h" // LLSingleton -#include "llapr.h" #include "llares.h" #include "llpumpio.h" #include "llhttpclient.h" @@ -48,14 +47,8 @@ public: mServicePump(NULL), mDone(false) { - ll_init_apr(); - if (! gAPRPoolp) - { - throw std::runtime_error("Can't initialize APR"); - } - // Create IO Pump to use for HTTP Requests. - mServicePump = new LLPumpIO(gAPRPoolp); + mServicePump = new LLPumpIO; LLHTTPClient::setPump(*mServicePump); if (ll_init_ares() == NULL || !gAres->isInitialized()) { diff --git a/indra/llmessage/tests/test_llsdmessage_peer.py b/indra/llmessage/tests/test_llsdmessage_peer.py index 580ee7f8b4..22edd9dad8 100644 --- a/indra/llmessage/tests/test_llsdmessage_peer.py +++ b/indra/llmessage/tests/test_llsdmessage_peer.py @@ -38,7 +38,7 @@ mydir = os.path.dirname(__file__) # expected to be .../indra/llmessage/tes sys.path.insert(0, os.path.join(mydir, os.pardir, os.pardir, "lib", "python")) from indra.util.fastest_elementtree import parse as xml_parse from indra.base import llsd -from testrunner import run, debug +from testrunner import freeport, run, debug, VERBOSE class TestHTTPRequestHandler(BaseHTTPRequestHandler): """This subclass of BaseHTTPRequestHandler is to receive and echo @@ -72,10 +72,10 @@ class TestHTTPRequestHandler(BaseHTTPRequestHandler): ## # assuming that the underlying XML parser reads its input file ## # incrementally. Unfortunately I haven't been able to make it work. ## tree = xml_parse(self.rfile) -## debug("Finished raw parse\n") -## debug("parsed XML tree %s\n" % tree) -## debug("parsed root node %s\n" % tree.getroot()) -## debug("root node tag %s\n" % tree.getroot().tag) +## debug("Finished raw parse") +## debug("parsed XML tree %s", tree) +## debug("parsed root node %s", tree.getroot()) +## debug("root node tag %s", tree.getroot().tag) ## return llsd.to_python(tree.getroot()) def do_GET(self): @@ -88,8 +88,10 @@ class TestHTTPRequestHandler(BaseHTTPRequestHandler): self.answer(self.read_xml()) def answer(self, data): + debug("%s.answer(%s): self.path = %r", self.__class__.__name__, data, self.path) if "fail" not in self.path: response = llsd.format_xml(data.get("reply", llsd.LLSD("success"))) + debug("success: %s", response) self.send_response(200) self.send_header("Content-type", "application/llsd+xml") self.send_header("Content-Length", str(len(response))) @@ -97,27 +99,48 @@ class TestHTTPRequestHandler(BaseHTTPRequestHandler): self.wfile.write(response) else: # fail requested status = data.get("status", 500) + # self.responses maps an int status to a (short, long) pair of + # strings. We want the longer string. That's why we pass a string + # pair to get(): the [1] will select the second string, whether it + # came from self.responses or from our default pair. reason = data.get("reason", self.responses.get(status, ("fail requested", "Your request specified failure status %s " "without providing a reason" % status))[1]) + debug("fail requested: %s: %r", status, reason) self.send_error(status, reason) - def log_request(self, code, size=None): - # For present purposes, we don't want the request splattered onto - # stderr, as it would upset devs watching the test run - pass + if not VERBOSE: + # When VERBOSE is set, skip both these overrides because they exist to + # suppress output. - def log_error(self, format, *args): - # Suppress error output as well - pass + def log_request(self, code, size=None): + # For present purposes, we don't want the request splattered onto + # stderr, as it would upset devs watching the test run + pass -class TestHTTPServer(Thread): - def run(self): - httpd = HTTPServer(('127.0.0.1', 8000), TestHTTPRequestHandler) - debug("Starting HTTP server...\n") - httpd.serve_forever() + def log_error(self, format, *args): + # Suppress error output as well + pass + +class Server(HTTPServer): + # This pernicious flag is on by default in HTTPServer. But proper + # operation of freeport() absolutely depends on it being off. + allow_reuse_address = False if __name__ == "__main__": - sys.exit(run(server=TestHTTPServer(name="httpd"), *sys.argv[1:])) + # Instantiate a Server(TestHTTPRequestHandler) on the first free port + # in the specified port range. Doing this inline is better than in a + # daemon thread: if it blows up here, we'll get a traceback. If it blew up + # in some other thread, the traceback would get eaten and we'd run the + # subject test program anyway. + httpd, port = freeport(xrange(8000, 8020), + lambda port: Server(('127.0.0.1', port), TestHTTPRequestHandler)) + # Pass the selected port number to the subject test program via the + # environment. We don't want to impose requirements on the test program's + # command-line parsing -- and anyway, for C++ integration tests, that's + # performed in TUT code rather than our own. + os.environ["PORT"] = str(port) + debug("$PORT = %s", port) + sys.exit(run(server=Thread(name="httpd", target=httpd.serve_forever), *sys.argv[1:])) diff --git a/indra/llmessage/tests/testrunner.py b/indra/llmessage/tests/testrunner.py index b70ce91ee7..f2c841532a 100644 --- a/indra/llmessage/tests/testrunner.py +++ b/indra/llmessage/tests/testrunner.py @@ -27,14 +27,118 @@ Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA $/LicenseInfo$ """ +from __future__ import with_statement + import os import sys +import re +import errno +import socket + +VERBOSE = os.environ.get("INTEGRATION_TEST_VERBOSE", "1") # default to verbose +# Support usage such as INTEGRATION_TEST_VERBOSE=off -- distressing to user if +# that construct actually turns on verbosity... +VERBOSE = not re.match(r"(0|off|false|quiet)$", VERBOSE, re.IGNORECASE) + +if VERBOSE: + def debug(fmt, *args): + print fmt % args + sys.stdout.flush() +else: + debug = lambda *args: None + +def freeport(portlist, expr): + """ + Find a free server port to use. Specifically, evaluate 'expr' (a + callable(port)) until it stops raising EADDRINUSE exception. + + Pass: + + portlist: an iterable (e.g. xrange()) of ports to try. If you exhaust the + range, freeport() lets the socket.error exception propagate. If you want + unbounded, you could pass itertools.count(baseport), though of course in + practice the ceiling is 2^16-1 anyway. But it seems prudent to constrain + the range much more sharply: if we're iterating an absurd number of times, + probably something else is wrong. + + expr: a callable accepting a port number, specifically one of the items + from portlist. If calling that callable raises socket.error with + EADDRINUSE, freeport() retrieves the next item from portlist and retries. + + Returns: (expr(port), port) + + port: the value from portlist for which expr(port) succeeded + + Raises: + + Any exception raised by expr(port) other than EADDRINUSE. + + socket.error if, for every item from portlist, expr(port) raises + socket.error. The exception you see is the one from the last item in + portlist. + + StopIteration if portlist is completely empty. + + Example: + + class Server(HTTPServer): + # If you use BaseHTTPServer.HTTPServer, turning off this flag is + # essential for proper operation of freeport()! + allow_reuse_address = False + # ... + server, port = freeport(xrange(8000, 8010), + lambda port: Server(("localhost", port), + MyRequestHandler)) + # pass 'port' to client code + # call server.serve_forever() + """ + try: + # If portlist is completely empty, let StopIteration propagate: that's an + # error because we can't return meaningful values. We have no 'port', + # therefore no 'expr(port)'. + portiter = iter(portlist) + port = portiter.next() + + while True: + try: + # If this value of port works, return as promised. + value = expr(port) + + except socket.error, err: + # Anything other than 'Address already in use', propagate + if err.args[0] != errno.EADDRINUSE: + raise + + # Here we want the next port from portiter. But on StopIteration, + # we want to raise the original exception rather than + # StopIteration. So save the original exc_info(). + type, value, tb = sys.exc_info() + try: + try: + port = portiter.next() + except StopIteration: + raise type, value, tb + finally: + # Clean up local traceback, see docs for sys.exc_info() + del tb + + else: + debug("freeport() returning %s on port %s", value, port) + return value, port -def debug(*args): - sys.stdout.writelines(args) - sys.stdout.flush() -# comment out the line below to enable debug output -debug = lambda *args: None + # Recap of the control flow above: + # If expr(port) doesn't raise, return as promised. + # If expr(port) raises anything but EADDRINUSE, propagate that + # exception. + # If portiter.next() raises StopIteration -- that is, if the port + # value we just passed to expr(port) was the last available -- reraise + # the EADDRINUSE exception. + # If we've actually arrived at this point, portiter.next() delivered a + # new port value. Loop back to pass that to expr(port). + + except Exception, err: + debug("*** freeport() raising %s: %s", err.__class__.__name__, err) + raise def run(*args, **kwds): """All positional arguments collectively form a command line, executed as @@ -63,8 +167,96 @@ def run(*args, **kwds): # - [no p] don't use the PATH because we specifically want to invoke the # executable passed as our first arg, # - [no e] child should inherit this process's environment. - debug("Running %s...\n" % (" ".join(args))) - sys.stdout.flush() + debug("Running %s...", " ".join(args)) rc = os.spawnv(os.P_WAIT, args[0], args) - debug("%s returned %s\n" % (args[0], rc)) + debug("%s returned %s", args[0], rc) return rc + +# **************************************************************************** +# test code -- manual at this point, see SWAT-564 +# **************************************************************************** +def test_freeport(): + # ------------------------------- Helpers -------------------------------- + from contextlib import contextmanager + # helper Context Manager for expecting an exception + # with exc(SomeError): + # raise SomeError() + # raises AssertionError otherwise. + @contextmanager + def exc(exception_class, *args): + try: + yield + except exception_class, err: + for i, expected_arg in enumerate(args): + assert expected_arg == err.args[i], \ + "Raised %s, but args[%s] is %r instead of %r" % \ + (err.__class__.__name__, i, err.args[i], expected_arg) + print "Caught expected exception %s(%s)" % \ + (err.__class__.__name__, ', '.join(repr(arg) for arg in err.args)) + else: + assert False, "Failed to raise " + exception_class.__class__.__name__ + + # helper to raise specified exception + def raiser(exception): + raise exception + + # the usual + def assert_equals(a, b): + assert a == b, "%r != %r" % (a, b) + + # ------------------------ Sanity check the above ------------------------ + class SomeError(Exception): pass + # Without extra args, accept any err.args value + with exc(SomeError): + raiser(SomeError("abc")) + # With extra args, accept only the specified value + with exc(SomeError, "abc"): + raiser(SomeError("abc")) + with exc(AssertionError): + with exc(SomeError, "abc"): + raiser(SomeError("def")) + with exc(AssertionError): + with exc(socket.error, errno.EADDRINUSE): + raiser(socket.error(errno.ECONNREFUSED, 'Connection refused')) + + # ----------- freeport() without engaging socket functionality ----------- + # If portlist is empty, freeport() raises StopIteration. + with exc(StopIteration): + freeport([], None) + + assert_equals(freeport([17], str), ("17", 17)) + + # This is the magic exception that should prompt us to retry + inuse = socket.error(errno.EADDRINUSE, 'Address already in use') + # Get the iterator to our ports list so we can check later if we've used all + ports = iter(xrange(5)) + with exc(socket.error, errno.EADDRINUSE): + freeport(ports, lambda port: raiser(inuse)) + # did we entirely exhaust 'ports'? + with exc(StopIteration): + ports.next() + + ports = iter(xrange(2)) + # Any exception but EADDRINUSE should quit immediately + with exc(SomeError): + freeport(ports, lambda port: raiser(SomeError())) + assert_equals(ports.next(), 1) + + # ----------- freeport() with platform-dependent socket stuff ------------ + # This is what we should've had unit tests to begin with (see CHOP-661). + def newbind(port): + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.bind(('127.0.0.1', port)) + return sock + + bound0, port0 = freeport(xrange(7777, 7780), newbind) + assert_equals(port0, 7777) + bound1, port1 = freeport(xrange(7777, 7780), newbind) + assert_equals(port1, 7778) + bound2, port2 = freeport(xrange(7777, 7780), newbind) + assert_equals(port2, 7779) + with exc(socket.error, errno.EADDRINUSE): + bound3, port3 = freeport(xrange(7777, 7780), newbind) + +if __name__ == "__main__": + test_freeport() |