diff options
Diffstat (limited to 'indra/llmessage')
36 files changed, 2150 insertions, 288 deletions
diff --git a/indra/llmessage/CMakeLists.txt b/indra/llmessage/CMakeLists.txt index c5f82cf052..0f40a670fa 100644 --- a/indra/llmessage/CMakeLists.txt +++ b/indra/llmessage/CMakeLists.txt @@ -65,6 +65,7 @@ set(llmessage_SOURCE_FILES llpacketbuffer.cpp llpacketring.cpp llpartdata.cpp + llproxy.cpp llpumpio.cpp llregionpresenceverifier.cpp llsdappservices.cpp @@ -161,6 +162,7 @@ set(llmessage_HEADER_FILES llpacketring.h llpartdata.h llpumpio.h + llproxy.h llqueryflags.h llregionflags.h llregionhandle.h diff --git a/indra/llmessage/llassetstorage.cpp b/indra/llmessage/llassetstorage.cpp index 31cdb1219b..9b86daebe5 100644 --- a/indra/llmessage/llassetstorage.cpp +++ b/indra/llmessage/llassetstorage.cpp @@ -149,8 +149,8 @@ void LLAssetInfo::setFromNameValue( const LLNameValue& nv ) setName( buf ); buf.assign( str, pos2, std::string::npos ); setDescription( buf ); - llinfos << "uuid: " << mUuid << llendl; - llinfos << "creator: " << mCreatorID << llendl; + LL_DEBUGS("AssetStorage") << "uuid: " << mUuid << llendl; + LL_DEBUGS("AssetStorage") << "creator: " << mCreatorID << llendl; } ///---------------------------------------------------------------------------- @@ -434,9 +434,9 @@ bool LLAssetStorage::findInStaticVFSAndInvokeCallback(const LLUUID& uuid, LLAsse // IW - uuid is passed by value to avoid side effects, please don't re-add & void LLAssetStorage::getAssetData(const LLUUID uuid, LLAssetType::EType type, LLGetAssetCallback callback, void *user_data, BOOL is_priority) { - lldebugs << "LLAssetStorage::getAssetData() - " << uuid << "," << LLAssetType::lookup(type) << llendl; + LL_DEBUGS("AssetStorage") << "LLAssetStorage::getAssetData() - " << uuid << "," << LLAssetType::lookup(type) << llendl; - llinfos << "ASSET_TRACE requesting " << uuid << " type " << LLAssetType::lookup(type) << llendl; + LL_DEBUGS("AssetStorage") << "ASSET_TRACE requesting " << uuid << " type " << LLAssetType::lookup(type) << llendl; if (user_data) { @@ -446,7 +446,7 @@ void LLAssetStorage::getAssetData(const LLUUID uuid, LLAssetType::EType type, LL if (mShutDown) { - llinfos << "ASSET_TRACE cancelled " << uuid << " type " << LLAssetType::lookup(type) << " shutting down" << llendl; + LL_DEBUGS("AssetStorage") << "ASSET_TRACE cancelled " << uuid << " type " << LLAssetType::lookup(type) << " shutting down" << llendl; if (callback) { @@ -468,7 +468,7 @@ void LLAssetStorage::getAssetData(const LLUUID uuid, LLAssetType::EType type, LL // Try static VFS first. if (findInStaticVFSAndInvokeCallback(uuid,type,callback,user_data)) { - llinfos << "ASSET_TRACE asset " << uuid << " found in static VFS" << llendl; + LL_DEBUGS("AssetStorage") << "ASSET_TRACE asset " << uuid << " found in static VFS" << llendl; return; } @@ -486,7 +486,7 @@ void LLAssetStorage::getAssetData(const LLUUID uuid, LLAssetType::EType type, LL callback(mVFS, uuid, type, user_data, LL_ERR_NOERR, LL_EXSTAT_VFS_CACHED); } - llinfos << "ASSET_TRACE asset " << uuid << " found in VFS" << llendl; + LL_DEBUGS("AssetStorage") << "ASSET_TRACE asset " << uuid << " found in VFS" << llendl; } else { @@ -520,7 +520,7 @@ void LLAssetStorage::getAssetData(const LLUUID uuid, LLAssetType::EType type, LL } if (duplicate) { - llinfos << "Adding additional non-duplicate request for asset " << uuid + LL_DEBUGS("AssetStorage") << "Adding additional non-duplicate request for asset " << uuid << "." << LLAssetType::lookup(type) << llendl; } @@ -584,9 +584,9 @@ void LLAssetStorage::downloadCompleteCallback( LLAssetType::EType file_type, void* user_data, LLExtStat ext_status) { - llinfos << "ASSET_TRACE asset " << file_id << " downloadCompleteCallback" << llendl; + LL_DEBUGS("AssetStorage") << "ASSET_TRACE asset " << file_id << " downloadCompleteCallback" << llendl; - lldebugs << "LLAssetStorage::downloadCompleteCallback() for " << file_id + LL_DEBUGS("AssetStorage") << "LLAssetStorage::downloadCompleteCallback() for " << file_id << "," << LLAssetType::lookup(file_type) << llendl; LLAssetRequest* req = (LLAssetRequest*)user_data; if(!req) @@ -731,7 +731,7 @@ void LLAssetStorage::getEstateAsset(const LLHost &object_sim, const LLUUID &agen tpvf.setAsset(asset_id, atype); tpvf.setCallback(downloadEstateAssetCompleteCallback, req); - llinfos << "Starting transfer for " << asset_id << llendl; + LL_DEBUGS("AssetStorage") << "Starting transfer for " << asset_id << llendl; LLTransferTargetChannel *ttcp = gTransferManager.getTargetChannel(source_host, LLTCT_ASSET); ttcp->requestTransfer(spe, tpvf, 100.f + (is_priority ? 1.f : 0.f)); } @@ -871,7 +871,7 @@ void LLAssetStorage::getInvItemAsset(const LLHost &object_sim, const LLUUID &age tpvf.setAsset(asset_id, atype); tpvf.setCallback(downloadInvItemCompleteCallback, req); - llinfos << "Starting transfer for inventory asset " + LL_DEBUGS("AssetStorage") << "Starting transfer for inventory asset " << item_id << " owned by " << owner_id << "," << task_id << llendl; LLTransferTargetChannel *ttcp = gTransferManager.getTargetChannel(source_host, LLTCT_ASSET); @@ -1211,7 +1211,7 @@ bool LLAssetStorage::deletePendingRequest(LLAssetStorage::ERequestType rt, request_list_t* requests = getRequestList(rt); if (deletePendingRequestImpl(requests, asset_type, asset_id)) { - llinfos << "Asset " << getRequestName(rt) << " request for " + LL_DEBUGS("AssetStorage") << "Asset " << getRequestName(rt) << " request for " << asset_id << "." << LLAssetType::lookup(asset_type) << " removed from pending queue." << llendl; return true; @@ -1307,7 +1307,7 @@ void LLAssetStorage::getAssetData(const LLUUID uuid, LLAssetType::EType type, vo user_data == ((LLLegacyAssetRequest *)tmp->mUserData)->mUserData) { // this is a duplicate from the same subsystem - throw it away - llinfos << "Discarding duplicate request for UUID " << uuid << llendl; + LL_DEBUGS("AssetStorage") << "Discarding duplicate request for UUID " << uuid << llendl; return; } } @@ -1490,7 +1490,7 @@ void LLAssetStorage::reportMetric( const LLUUID& asset_id, const LLAssetType::ET { if( !metric_recipient ) { - llinfos << "Couldn't store LLAssetStoreage::reportMetric - no metrics_recipient" << llendl; + LL_DEBUGS("AssetStorage") << "Couldn't store LLAssetStoreage::reportMetric - no metrics_recipient" << llendl; return; } diff --git a/indra/llmessage/llbuffer.cpp b/indra/llmessage/llbuffer.cpp index 0316797f00..250cace6e9 100644 --- a/indra/llmessage/llbuffer.cpp +++ b/indra/llmessage/llbuffer.cpp @@ -32,6 +32,9 @@ #include "llmath.h" #include "llmemtype.h" #include "llstl.h" +#include "llthread.h" + +#define ASSERT_LLBUFFERARRAY_MUTEX_LOCKED llassert(!mMutexp || mMutexp->isSelfLocked()); /** * LLSegment @@ -224,7 +227,8 @@ void LLHeapBuffer::allocate(S32 size) * LLBufferArray */ LLBufferArray::LLBufferArray() : - mNextBaseChannel(0) + mNextBaseChannel(0), + mMutexp(NULL) { LLMemType m1(LLMemType::MTYPE_IO_BUFFER); } @@ -233,6 +237,8 @@ LLBufferArray::~LLBufferArray() { LLMemType m1(LLMemType::MTYPE_IO_BUFFER); std::for_each(mBuffers.begin(), mBuffers.end(), DeletePointer()); + + delete mMutexp; } // static @@ -243,14 +249,57 @@ LLChannelDescriptors LLBufferArray::makeChannelConsumer( return rv; } +void LLBufferArray::lock() +{ + if(mMutexp) + { + mMutexp->lock() ; + } +} + +void LLBufferArray::unlock() +{ + if(mMutexp) + { + mMutexp->unlock() ; + } +} + +LLMutex* LLBufferArray::getMutex() +{ + return mMutexp ; +} + +void LLBufferArray::setThreaded(bool threaded) +{ + if(threaded) + { + if(!mMutexp) + { + mMutexp = new LLMutex(NULL); + } + } + else + { + if(mMutexp) + { + delete mMutexp ; + mMutexp = NULL ; + } + } +} + LLChannelDescriptors LLBufferArray::nextChannel() { LLChannelDescriptors rv(mNextBaseChannel++); return rv; } +//mMutexp should be locked before calling this. S32 LLBufferArray::capacity() const { + ASSERT_LLBUFFERARRAY_MUTEX_LOCKED + S32 total = 0; const_buffer_iterator_t iter = mBuffers.begin(); const_buffer_iterator_t end = mBuffers.end(); @@ -263,6 +312,8 @@ S32 LLBufferArray::capacity() const bool LLBufferArray::append(S32 channel, const U8* src, S32 len) { + LLMutexLock lock(mMutexp) ; + LLMemType m1(LLMemType::MTYPE_IO_BUFFER); std::vector<LLSegment> segments; if(copyIntoBuffers(channel, src, len, segments)) @@ -273,8 +324,11 @@ bool LLBufferArray::append(S32 channel, const U8* src, S32 len) return false; } +//mMutexp should be locked before calling this. bool LLBufferArray::prepend(S32 channel, const U8* src, S32 len) { + ASSERT_LLBUFFERARRAY_MUTEX_LOCKED + LLMemType m1(LLMemType::MTYPE_IO_BUFFER); std::vector<LLSegment> segments; if(copyIntoBuffers(channel, src, len, segments)) @@ -293,6 +347,8 @@ bool LLBufferArray::insertAfter( { LLMemType m1(LLMemType::MTYPE_IO_BUFFER); std::vector<LLSegment> segments; + + LLMutexLock lock(mMutexp) ; if(mSegments.end() != segment) { ++segment; @@ -305,8 +361,11 @@ bool LLBufferArray::insertAfter( return false; } +//mMutexp should be locked before calling this. LLBufferArray::segment_iterator_t LLBufferArray::splitAfter(U8* address) { + ASSERT_LLBUFFERARRAY_MUTEX_LOCKED + LLMemType m1(LLMemType::MTYPE_IO_BUFFER); segment_iterator_t end = mSegments.end(); segment_iterator_t it = getSegment(address); @@ -335,20 +394,26 @@ LLBufferArray::segment_iterator_t LLBufferArray::splitAfter(U8* address) return rv; } +//mMutexp should be locked before calling this. LLBufferArray::segment_iterator_t LLBufferArray::beginSegment() { + ASSERT_LLBUFFERARRAY_MUTEX_LOCKED return mSegments.begin(); } +//mMutexp should be locked before calling this. LLBufferArray::segment_iterator_t LLBufferArray::endSegment() { + ASSERT_LLBUFFERARRAY_MUTEX_LOCKED return mSegments.end(); } +//mMutexp should be locked before calling this. LLBufferArray::segment_iterator_t LLBufferArray::constructSegmentAfter( U8* address, LLSegment& segment) { + ASSERT_LLBUFFERARRAY_MUTEX_LOCKED LLMemType m1(LLMemType::MTYPE_IO_BUFFER); segment_iterator_t rv = mSegments.begin(); segment_iterator_t end = mSegments.end(); @@ -395,8 +460,10 @@ LLBufferArray::segment_iterator_t LLBufferArray::constructSegmentAfter( return rv; } +//mMutexp should be locked before calling this. LLBufferArray::segment_iterator_t LLBufferArray::getSegment(U8* address) { + ASSERT_LLBUFFERARRAY_MUTEX_LOCKED segment_iterator_t end = mSegments.end(); if(!address) { @@ -414,9 +481,11 @@ LLBufferArray::segment_iterator_t LLBufferArray::getSegment(U8* address) return end; } +//mMutexp should be locked before calling this. LLBufferArray::const_segment_iterator_t LLBufferArray::getSegment( U8* address) const { + ASSERT_LLBUFFERARRAY_MUTEX_LOCKED const_segment_iterator_t end = mSegments.end(); if(!address) { @@ -466,6 +535,8 @@ S32 LLBufferArray::countAfter(S32 channel, U8* start) const S32 count = 0; S32 offset = 0; const_segment_iterator_t it; + + LLMutexLock lock(mMutexp) ; const_segment_iterator_t end = mSegments.end(); if(start) { @@ -517,6 +588,8 @@ U8* LLBufferArray::readAfter( len = 0; S32 bytes_to_copy = 0; const_segment_iterator_t it; + + LLMutexLock lock(mMutexp) ; const_segment_iterator_t end = mSegments.end(); if(start) { @@ -568,6 +641,7 @@ U8* LLBufferArray::seek( U8* start, S32 delta) const { + ASSERT_LLBUFFERARRAY_MUTEX_LOCKED LLMemType m1(LLMemType::MTYPE_IO_BUFFER); const_segment_iterator_t it; const_segment_iterator_t end = mSegments.end(); @@ -709,9 +783,14 @@ U8* LLBufferArray::seek( return rv; } +//test use only bool LLBufferArray::takeContents(LLBufferArray& source) { LLMemType m1(LLMemType::MTYPE_IO_BUFFER); + + LLMutexLock lock(mMutexp); + source.lock(); + std::copy( source.mBuffers.begin(), source.mBuffers.end(), @@ -723,13 +802,17 @@ bool LLBufferArray::takeContents(LLBufferArray& source) std::back_insert_iterator<segment_list_t>(mSegments)); source.mSegments.clear(); source.mNextBaseChannel = 0; + source.unlock(); + return true; } +//mMutexp should be locked before calling this. LLBufferArray::segment_iterator_t LLBufferArray::makeSegment( S32 channel, S32 len) { + ASSERT_LLBUFFERARRAY_MUTEX_LOCKED LLMemType m1(LLMemType::MTYPE_IO_BUFFER); // start at the end of the buffers, because it is the most likely // to have free space. @@ -765,8 +848,10 @@ LLBufferArray::segment_iterator_t LLBufferArray::makeSegment( return send; } +//mMutexp should be locked before calling this. bool LLBufferArray::eraseSegment(const segment_iterator_t& erase_iter) { + ASSERT_LLBUFFERARRAY_MUTEX_LOCKED LLMemType m1(LLMemType::MTYPE_IO_BUFFER); // Find out which buffer contains the segment, and if it is found, @@ -792,13 +877,14 @@ bool LLBufferArray::eraseSegment(const segment_iterator_t& erase_iter) return rv; } - +//mMutexp should be locked before calling this. bool LLBufferArray::copyIntoBuffers( S32 channel, const U8* src, S32 len, std::vector<LLSegment>& segments) { + ASSERT_LLBUFFERARRAY_MUTEX_LOCKED LLMemType m1(LLMemType::MTYPE_IO_BUFFER); if(!src || !len) return false; S32 copied = 0; diff --git a/indra/llmessage/llbuffer.h b/indra/llmessage/llbuffer.h index 1c42b6fbc6..ccdb9fa7ee 100644 --- a/indra/llmessage/llbuffer.h +++ b/indra/llmessage/llbuffer.h @@ -39,6 +39,7 @@ #include <list> #include <vector> +class LLMutex; /** * @class LLChannelDescriptors * @brief A way simple interface to accesss channels inside a buffer @@ -564,6 +565,29 @@ public: * @return Returns true on success. */ bool eraseSegment(const segment_iterator_t& iter); + + /** + * @brief Lock the mutex if it exists + * This method locks mMutexp to make accessing LLBufferArray thread-safe + */ + void lock(); + + /** + * @brief Unlock the mutex if it exists + */ + void unlock(); + + /** + * @brief Return mMutexp + */ + LLMutex* getMutex(); + + /** + * @brief Set LLBufferArray to be shared across threads or not + * This method is to create mMutexp if is threaded. + * @param threaded Indicates this LLBufferArray instance is shared across threads if true. + */ + void setThreaded(bool threaded); //@} protected: @@ -595,6 +619,7 @@ protected: S32 mNextBaseChannel; buffer_list_t mBuffers; segment_list_t mSegments; + LLMutex* mMutexp; }; #endif // LL_LLBUFFER_H diff --git a/indra/llmessage/llbufferstream.cpp b/indra/llmessage/llbufferstream.cpp index 6257983c43..8d8ad05ad5 100644 --- a/indra/llmessage/llbufferstream.cpp +++ b/indra/llmessage/llbufferstream.cpp @@ -31,6 +31,7 @@ #include "llbuffer.h" #include "llmemtype.h" +#include "llthread.h" static const S32 DEFAULT_OUTPUT_SEGMENT_SIZE = 1024 * 4; @@ -62,6 +63,7 @@ int LLBufferStreamBuf::underflow() return EOF; } + LLMutexLock lock(mBuffer->getMutex()); LLBufferArray::segment_iterator_t iter; LLBufferArray::segment_iterator_t end = mBuffer->endSegment(); U8* last_pos = (U8*)gptr(); @@ -149,6 +151,7 @@ int LLBufferStreamBuf::overflow(int c) // since we got here, we have a buffer, and we have a character to // put on it. LLBufferArray::segment_iterator_t it; + LLMutexLock lock(mBuffer->getMutex()); it = mBuffer->makeSegment(mChannels.out(), DEFAULT_OUTPUT_SEGMENT_SIZE); if(it != mBuffer->endSegment()) { @@ -210,6 +213,7 @@ int LLBufferStreamBuf::sync() // *NOTE: I bet we could just --address if address is not NULL. // Need to think about that. + LLMutexLock lock(mBuffer->getMutex()); address = mBuffer->seek(mChannels.out(), address, -1); if(address) { @@ -273,6 +277,8 @@ streampos LLBufferStreamBuf::seekoff( // NULL is fine break; } + + LLMutexLock lock(mBuffer->getMutex()); address = mBuffer->seek(mChannels.in(), base_addr, off); if(address) { @@ -304,6 +310,8 @@ streampos LLBufferStreamBuf::seekoff( // NULL is fine break; } + + LLMutexLock lock(mBuffer->getMutex()); address = mBuffer->seek(mChannels.out(), base_addr, off); if(address) { diff --git a/indra/llmessage/llcurl.cpp b/indra/llmessage/llcurl.cpp index 7c8b7e3584..b93d429feb 100644 --- a/indra/llmessage/llcurl.cpp +++ b/indra/llmessage/llcurl.cpp @@ -1,5 +1,5 @@ /** - * @file llcurl.h + * @file llcurl.cpp * @author Zero / Donovan * @date 2006-10-15 * @brief Implementation of wrapper around libcurl. @@ -46,8 +46,9 @@ #endif #include "llbufferstream.h" -#include "llstl.h" +#include "llproxy.h" #include "llsdserialize.h" +#include "llstl.h" #include "llthread.h" #include "lltimer.h" @@ -71,7 +72,7 @@ static const U32 EASY_HANDLE_POOL_SIZE = 5; static const S32 MULTI_PERFORM_CALL_REPEAT = 5; -static const S32 CURL_REQUEST_TIMEOUT = 30; // seconds +static const S32 CURL_REQUEST_TIMEOUT = 30; // seconds per operation static const S32 MAX_ACTIVE_REQUEST_COUNT = 100; // DEBUG // @@ -84,6 +85,12 @@ S32 gCurlMultiCount = 0; std::vector<LLMutex*> LLCurl::sSSLMutex; std::string LLCurl::sCAPath; std::string LLCurl::sCAFile; +LLCurlThread* LLCurl::sCurlThread = NULL ; +LLMutex* LLCurl::sHandleMutexp = NULL ; +S32 LLCurl::sTotalHandles = 0 ; +bool LLCurl::sNotQuitting = true; +F32 LLCurl::sCurlRequestTimeOut = 120.f; //seonds +S32 LLCurl::sMaxHandles = 256; //max number of handles, (multi handles and easy handles combined). void check_curl_code(CURLcode code) { @@ -204,7 +211,7 @@ namespace boost void intrusive_ptr_release(LLCurl::Responder* p) { - if(p && 0 == --p->mReferenceCount) + if (p && 0 == --p->mReferenceCount) { delete p; } @@ -214,86 +221,22 @@ namespace boost ////////////////////////////////////////////////////////////////////////////// - -class LLCurl::Easy -{ - LOG_CLASS(Easy); - -private: - Easy(); - -public: - static Easy* getEasy(); - ~Easy(); - - CURL* getCurlHandle() const { return mCurlEasyHandle; } - - void setErrorBuffer(); - void setCA(); - - void setopt(CURLoption option, S32 value); - // These assume the setter does not free value! - void setopt(CURLoption option, void* value); - void setopt(CURLoption option, char* value); - // Copies the string so that it is gauranteed to stick around - void setoptString(CURLoption option, const std::string& value); - - void slist_append(const char* str); - void setHeaders(); - - U32 report(CURLcode); - void getTransferInfo(LLCurl::TransferInfo* info); - - void prepRequest(const std::string& url, const std::vector<std::string>& headers, ResponderPtr, bool post = false); - - const char* getErrorBuffer(); - - std::stringstream& getInput() { return mInput; } - std::stringstream& getHeaderOutput() { return mHeaderOutput; } - LLIOPipe::buffer_ptr_t& getOutput() { return mOutput; } - const LLChannelDescriptors& getChannels() { return mChannels; } - - void resetState(); - - static CURL* allocEasyHandle(); - static void releaseEasyHandle(CURL* handle); - -private: - friend class LLCurl; - - CURL* mCurlEasyHandle; - struct curl_slist* mHeaders; - - std::stringstream mRequest; - LLChannelDescriptors mChannels; - LLIOPipe::buffer_ptr_t mOutput; - std::stringstream mInput; - std::stringstream mHeaderOutput; - char mErrorBuffer[CURL_ERROR_SIZE]; - - // Note: char*'s not strings since we pass pointers to curl - std::vector<char*> mStrings; - - ResponderPtr mResponder; - - static std::set<CURL*> sFreeHandles; - static std::set<CURL*> sActiveHandles; - static LLMutex* sHandleMutex; -}; - std::set<CURL*> LLCurl::Easy::sFreeHandles; std::set<CURL*> LLCurl::Easy::sActiveHandles; -LLMutex* LLCurl::Easy::sHandleMutex = NULL; - +LLMutex* LLCurl::Easy::sHandleMutexp = NULL ; //static CURL* LLCurl::Easy::allocEasyHandle() { + llassert(LLCurl::getCurlThread()) ; + CURL* ret = NULL; - LLMutexLock lock(sHandleMutex); + + LLMutexLock lock(sHandleMutexp) ; + if (sFreeHandles.empty()) { - ret = curl_easy_init(); + ret = LLCurl::newEasyHandle(); } else { @@ -313,20 +256,30 @@ CURL* LLCurl::Easy::allocEasyHandle() //static void LLCurl::Easy::releaseEasyHandle(CURL* handle) { + static const S32 MAX_NUM_FREE_HANDLES = 32 ; + if (!handle) { - llerrs << "handle cannot be NULL!" << llendl; + return ; //handle allocation failed. + //llerrs << "handle cannot be NULL!" << llendl; } - LLMutexLock lock(sHandleMutex); - + LLMutexLock lock(sHandleMutexp) ; if (sActiveHandles.find(handle) != sActiveHandles.end()) { sActiveHandles.erase(handle); + + if(sFreeHandles.size() < MAX_NUM_FREE_HANDLES) + { sFreeHandles.insert(handle); } else { + LLCurl::deleteEasyHandle(handle) ; + } + } + else + { llerrs << "Invalid handle." << llendl; } } @@ -366,6 +319,14 @@ LLCurl::Easy::~Easy() --gCurlEasyCount; curl_slist_free_all(mHeaders); for_each(mStrings.begin(), mStrings.end(), DeletePointerArray()); + + if (mResponder && LLCurl::sNotQuitting) //aborted + { + std::string reason("Request timeout, aborted.") ; + mResponder->completedRaw(408, //HTTP_REQUEST_TIME_OUT, timeout, abort + reason, mChannels, mOutput); + } + mResponder = NULL; } void LLCurl::Easy::resetState() @@ -404,11 +365,11 @@ const char* LLCurl::Easy::getErrorBuffer() void LLCurl::Easy::setCA() { - if(!sCAPath.empty()) + if (!sCAPath.empty()) { setoptString(CURLOPT_CAPATH, sCAPath); } - if(!sCAFile.empty()) + if (!sCAFile.empty()) { setoptString(CURLOPT_CAINFO, sCAFile); } @@ -492,9 +453,9 @@ size_t curlReadCallback(char* data, size_t size, size_t nmemb, void* user_data) LLCurl::Easy* easy = (LLCurl::Easy*)user_data; S32 n = size * nmemb; - S32 startpos = easy->getInput().tellg(); + S32 startpos = (S32)easy->getInput().tellg(); easy->getInput().seekg(0, std::ios::end); - S32 endpos = easy->getInput().tellg(); + S32 endpos = (S32)easy->getInput().tellg(); easy->getInput().seekg(startpos, std::ios::beg); S32 maxn = endpos - startpos; n = llmin(n, maxn); @@ -525,16 +486,20 @@ size_t curlHeaderCallback(void* data, size_t size, size_t nmemb, void* user_data void LLCurl::Easy::prepRequest(const std::string& url, const std::vector<std::string>& headers, - ResponderPtr responder, bool post) + ResponderPtr responder, S32 time_out, bool post) { resetState(); if (post) setoptString(CURLOPT_ENCODING, ""); - //setopt(CURLOPT_VERBOSE, 1); // usefull for debugging + //setopt(CURLOPT_VERBOSE, 1); // useful for debugging setopt(CURLOPT_NOSIGNAL, 1); + // Set the CURL options for either Socks or HTTP proxy + LLProxy::getInstance()->applyProxySettings(this); + mOutput.reset(new LLBufferArray); + mOutput->setThreaded(true); setopt(CURLOPT_WRITEFUNCTION, (void*)&curlWriteCallback); setopt(CURLOPT_WRITEDATA, (void*)this); @@ -545,7 +510,7 @@ void LLCurl::Easy::prepRequest(const std::string& url, setopt(CURLOPT_HEADERDATA, (void*)this); // Allow up to five redirects - if(responder && responder->followRedir()) + if (responder && responder->followRedir()) { setopt(CURLOPT_FOLLOWLOCATION, 1); setopt(CURLOPT_MAXREDIRS, MAX_REDIRECTS); @@ -558,7 +523,7 @@ void LLCurl::Easy::prepRequest(const std::string& url, //don't verify host name so urls with scrubbed host names will work (improves DNS performance) setopt(CURLOPT_SSL_VERIFYHOST, 0); - setopt(CURLOPT_TIMEOUT, CURL_REQUEST_TIMEOUT); + setopt(CURLOPT_TIMEOUT, llmax(time_out, CURL_REQUEST_TIMEOUT)); setoptString(CURLOPT_URL, url); @@ -578,58 +543,56 @@ void LLCurl::Easy::prepRequest(const std::string& url, } //////////////////////////////////////////////////////////////////////////// - -class LLCurl::Multi -{ - LOG_CLASS(Multi); -public: - - Multi(); - ~Multi(); - - Easy* allocEasy(); - bool addEasy(Easy* easy); - - void removeEasy(Easy* easy); - - S32 process(); - S32 perform(); - - CURLMsg* info_read(S32* msgs_in_queue); - - S32 mQueued; - S32 mErrorCount; - -private: - void easyFree(Easy*); - - CURLM* mCurlMultiHandle; - - typedef std::set<Easy*> easy_active_list_t; - easy_active_list_t mEasyActiveList; - typedef std::map<CURL*, Easy*> easy_active_map_t; - easy_active_map_t mEasyActiveMap; - typedef std::set<Easy*> easy_free_list_t; - easy_free_list_t mEasyFreeList; -}; - -LLCurl::Multi::Multi() +LLCurl::Multi::Multi(F32 idle_time_out) : mQueued(0), - mErrorCount(0) -{ - mCurlMultiHandle = curl_multi_init(); + mErrorCount(0), + mState(STATE_READY), + mDead(FALSE), + mMutexp(NULL), + mDeletionMutexp(NULL), + mEasyMutexp(NULL) +{ + mCurlMultiHandle = LLCurl::newMultiHandle(); if (!mCurlMultiHandle) { llwarns << "curl_multi_init() returned NULL! Easy handles: " << gCurlEasyCount << " Multi handles: " << gCurlMultiCount << llendl; - mCurlMultiHandle = curl_multi_init(); + mCurlMultiHandle = LLCurl::newMultiHandle(); } - llassert_always(mCurlMultiHandle); + //llassert_always(mCurlMultiHandle); + + if(mCurlMultiHandle) + { + if(LLCurl::getCurlThread()->getThreaded()) + { + mMutexp = new LLMutex(NULL) ; + mDeletionMutexp = new LLMutex(NULL) ; + mEasyMutexp = new LLMutex(NULL) ; + } + LLCurl::getCurlThread()->addMulti(this) ; + + mIdleTimeOut = idle_time_out ; + if(mIdleTimeOut < LLCurl::sCurlRequestTimeOut) + { + mIdleTimeOut = LLCurl::sCurlRequestTimeOut ; + } + ++gCurlMultiCount; } +} LLCurl::Multi::~Multi() { + cleanup() ; +} + +void LLCurl::Multi::cleanup() +{ + if(!mCurlMultiHandle) + { + return ; //nothing to clean. + } + // Clean up active for(easy_active_list_t::iterator iter = mEasyActiveList.begin(); iter != mEasyActiveList.end(); ++iter) @@ -645,40 +608,161 @@ LLCurl::Multi::~Multi() for_each(mEasyFreeList.begin(), mEasyFreeList.end(), DeletePointer()); mEasyFreeList.clear(); - check_curl_multi_code(curl_multi_cleanup(mCurlMultiHandle)); + check_curl_multi_code(LLCurl::deleteMultiHandle(mCurlMultiHandle)); + mCurlMultiHandle = NULL ; + + delete mMutexp ; + mMutexp = NULL ; + delete mDeletionMutexp ; + mDeletionMutexp = NULL ; + delete mEasyMutexp ; + mEasyMutexp = NULL ; + + mQueued = 0 ; + mState = STATE_COMPLETED; + --gCurlMultiCount; + + return ; +} + +void LLCurl::Multi::lock() +{ + if(mMutexp) + { + mMutexp->lock() ; + } +} + +void LLCurl::Multi::unlock() +{ + if(mMutexp) + { + mMutexp->unlock() ; + } +} + +void LLCurl::Multi::markDead() +{ + LLMutexLock lock(mDeletionMutexp) ; + + mDead = TRUE ; + LLCurl::getCurlThread()->setPriority(mHandle, LLQueuedThread::PRIORITY_URGENT) ; +} + +void LLCurl::Multi::setState(LLCurl::Multi::ePerformState state) +{ + lock() ; + mState = state ; + unlock() ; + + if(mState == STATE_READY) + { + LLCurl::getCurlThread()->setPriority(mHandle, LLQueuedThread::PRIORITY_NORMAL) ; + } +} + +LLCurl::Multi::ePerformState LLCurl::Multi::getState() +{ + return mState; +} + +bool LLCurl::Multi::isCompleted() +{ + return STATE_COMPLETED == getState() ; +} + +bool LLCurl::Multi::waitToComplete() +{ + if(!isValid()) + { + return true ; + } + + if(!mMutexp) //not threaded + { + doPerform() ; + return true ; + } + + bool completed = (STATE_COMPLETED == mState) ; + if(!completed) + { + LLCurl::getCurlThread()->setPriority(mHandle, LLQueuedThread::PRIORITY_HIGH) ; + } + + return completed; } CURLMsg* LLCurl::Multi::info_read(S32* msgs_in_queue) { + LLMutexLock lock(mMutexp) ; + CURLMsg* curlmsg = curl_multi_info_read(mCurlMultiHandle, msgs_in_queue); return curlmsg; } - -S32 LLCurl::Multi::perform() +//return true if dead +bool LLCurl::Multi::doPerform() { - S32 q = 0; - for (S32 call_count = 0; - call_count < MULTI_PERFORM_CALL_REPEAT; - call_count += 1) + LLMutexLock lock(mDeletionMutexp) ; + + bool dead = mDead ; + + if(mDead) { - CURLMcode code = curl_multi_perform(mCurlMultiHandle, &q); - if (CURLM_CALL_MULTI_PERFORM != code || q == 0) + setState(STATE_COMPLETED); + mQueued = 0 ; + } + else if(getState() != STATE_COMPLETED) + { + setState(STATE_PERFORMING); + + S32 q = 0; + for (S32 call_count = 0; + call_count < MULTI_PERFORM_CALL_REPEAT; + call_count++) { - check_curl_multi_code(code); - break; + LLMutexLock lock(mMutexp) ; + + //WARNING: curl_multi_perform will block for many hundreds of milliseconds + // NEVER call this from the main thread, and NEVER allow the main thread to + // wait on a mutex held by this thread while curl_multi_perform is executing + CURLMcode code = curl_multi_perform(mCurlMultiHandle, &q); + if (CURLM_CALL_MULTI_PERFORM != code || q == 0) + { + check_curl_multi_code(code); + + break; + } } - + + mQueued = q; + setState(STATE_COMPLETED) ; + mIdleTimer.reset() ; } - mQueued = q; - return q; + else if(mIdleTimer.getElapsedTimeF32() > mIdleTimeOut) //idle for too long, remove it. + { + dead = true ; + } + + return dead ; } S32 LLCurl::Multi::process() { - perform(); - + if(!isValid()) + { + return 0 ; + } + + waitToComplete() ; + + if (getState() != STATE_COMPLETED) + { + return 0; + } + CURLMsg* msg; int msgs_in_queue; @@ -689,10 +773,19 @@ S32 LLCurl::Multi::process() if (msg->msg == CURLMSG_DONE) { U32 response = 0; - easy_active_map_t::iterator iter = mEasyActiveMap.find(msg->easy_handle); - if (iter != mEasyActiveMap.end()) + Easy* easy = NULL ; + + { + LLMutexLock lock(mEasyMutexp) ; + easy_active_map_t::iterator iter = mEasyActiveMap.find(msg->easy_handle); + if (iter != mEasyActiveMap.end()) + { + easy = iter->second; + } + } + + if(easy) { - Easy* easy = iter->second; response = easy->report(msg->data.result); removeEasy(easy); } @@ -709,24 +802,29 @@ S32 LLCurl::Multi::process() } } } + + setState(STATE_READY); + return processed; } LLCurl::Easy* LLCurl::Multi::allocEasy() { - Easy* easy = 0; + Easy* easy = 0; if (mEasyFreeList.empty()) - { + { easy = Easy::getEasy(); } else { + LLMutexLock lock(mEasyMutexp) ; easy = *(mEasyFreeList.begin()); mEasyFreeList.erase(easy); } if (easy) { + LLMutexLock lock(mEasyMutexp) ; mEasyActiveList.insert(easy); mEasyActiveMap[easy->getCurlHandle()] = easy; } @@ -735,6 +833,7 @@ LLCurl::Easy* LLCurl::Multi::allocEasy() bool LLCurl::Multi::addEasy(Easy* easy) { + LLMutexLock lock(mMutexp) ; CURLMcode mcode = curl_multi_add_handle(mCurlMultiHandle, easy->getCurlHandle()); check_curl_multi_code(mcode); //if (mcode != CURLM_OK) @@ -747,25 +846,156 @@ bool LLCurl::Multi::addEasy(Easy* easy) void LLCurl::Multi::easyFree(Easy* easy) { + if(mEasyMutexp) + { + mEasyMutexp->lock() ; + } + mEasyActiveList.erase(easy); mEasyActiveMap.erase(easy->getCurlHandle()); + if (mEasyFreeList.size() < EASY_HANDLE_POOL_SIZE) - { - easy->resetState(); + { mEasyFreeList.insert(easy); + + if(mEasyMutexp) + { + mEasyMutexp->unlock() ; + } + + easy->resetState(); } else { + if(mEasyMutexp) + { + mEasyMutexp->unlock() ; + } delete easy; } } void LLCurl::Multi::removeEasy(Easy* easy) { - check_curl_multi_code(curl_multi_remove_handle(mCurlMultiHandle, easy->getCurlHandle())); + { + LLMutexLock lock(mMutexp) ; + check_curl_multi_code(curl_multi_remove_handle(mCurlMultiHandle, easy->getCurlHandle())); + } easyFree(easy); } +//------------------------------------------------------------ +//LLCurlThread +LLCurlThread::CurlRequest::CurlRequest(handle_t handle, LLCurl::Multi* multi, LLCurlThread* curl_thread) : + LLQueuedThread::QueuedRequest(handle, LLQueuedThread::PRIORITY_NORMAL, FLAG_AUTO_COMPLETE), + mMulti(multi), + mCurlThread(curl_thread) +{ +} + +LLCurlThread::CurlRequest::~CurlRequest() +{ + if(mMulti) + { + mCurlThread->deleteMulti(mMulti) ; + mMulti = NULL ; + } +} + +bool LLCurlThread::CurlRequest::processRequest() +{ + bool completed = true ; + if(mMulti) + { + completed = mCurlThread->doMultiPerform(mMulti) ; + + if(!completed) + { + setPriority(LLQueuedThread::PRIORITY_LOW) ; + } + } + + return completed ; +} + +void LLCurlThread::CurlRequest::finishRequest(bool completed) +{ + if(mMulti->isDead()) + { + mCurlThread->deleteMulti(mMulti) ; + } + else + { + mCurlThread->cleanupMulti(mMulti) ; //being idle too long, remove the request. + } + + mMulti = NULL ; +} + +LLCurlThread::LLCurlThread(bool threaded) : + LLQueuedThread("curlthread", threaded) +{ +} + +//virtual +LLCurlThread::~LLCurlThread() +{ +} + +S32 LLCurlThread::update(F32 max_time_ms) +{ + return LLQueuedThread::update(max_time_ms); +} + +void LLCurlThread::addMulti(LLCurl::Multi* multi) +{ + multi->mHandle = generateHandle() ; + + CurlRequest* req = new CurlRequest(multi->mHandle, multi, this) ; + + if (!addRequest(req)) + { + llwarns << "curl request added when the thread is quitted" << llendl; + } +} + +void LLCurlThread::killMulti(LLCurl::Multi* multi) +{ + if(!multi) + { + return ; + } + + if(multi->isValid()) + { + multi->markDead() ; +} + else + { + deleteMulti(multi) ; + } +} + +//private +bool LLCurlThread::doMultiPerform(LLCurl::Multi* multi) +{ + return multi->doPerform() ; +} + +//private +void LLCurlThread::deleteMulti(LLCurl::Multi* multi) +{ + delete multi ; +} + +//private +void LLCurlThread::cleanupMulti(LLCurl::Multi* multi) +{ + multi->cleanup() ; +} + +//------------------------------------------------------------ + //static std::string LLCurl::strerror(CURLcode errorcode) { @@ -780,20 +1010,30 @@ LLCurlRequest::LLCurlRequest() : mActiveMulti(NULL), mActiveRequestCount(0) { - mThreadID = LLThread::currentID(); mProcessing = FALSE; } LLCurlRequest::~LLCurlRequest() { - llassert_always(mThreadID == LLThread::currentID()); - for_each(mMultiSet.begin(), mMultiSet.end(), DeletePointer()); + //stop all Multi handle background threads + for (curlmulti_set_t::iterator iter = mMultiSet.begin(); iter != mMultiSet.end(); ++iter) + { + LLCurl::getCurlThread()->killMulti(*iter) ; + } + mMultiSet.clear() ; } void LLCurlRequest::addMulti() { - llassert_always(mThreadID == LLThread::currentID()); LLCurl::Multi* multi = new LLCurl::Multi(); + if(!multi->isValid()) + { + LLCurl::getCurlThread()->killMulti(multi) ; + mActiveMulti = NULL ; + mActiveRequestCount = 0 ; + return; + } + mMultiSet.insert(multi); mActiveMulti = multi; mActiveRequestCount = 0; @@ -807,7 +1047,12 @@ LLCurl::Easy* LLCurlRequest::allocEasy() { addMulti(); } - llassert_always(mActiveMulti); + if(!mActiveMulti) + { + return NULL ; + } + + //llassert_always(mActiveMulti); ++mActiveRequestCount; LLCurl::Easy* easy = mActiveMulti->allocEasy(); return easy; @@ -855,14 +1100,14 @@ bool LLCurlRequest::getByteRange(const std::string& url, bool LLCurlRequest::post(const std::string& url, const headers_t& headers, const LLSD& data, - LLCurl::ResponderPtr responder) + LLCurl::ResponderPtr responder, S32 time_out) { LLCurl::Easy* easy = allocEasy(); if (!easy) { return false; } - easy->prepRequest(url, headers, responder); + easy->prepRequest(url, headers, responder, time_out); LLSDSerialize::toXML(data, easy->getInput()); S32 bytes = easy->getInput().str().length(); @@ -882,14 +1127,14 @@ bool LLCurlRequest::post(const std::string& url, bool LLCurlRequest::post(const std::string& url, const headers_t& headers, const std::string& data, - LLCurl::ResponderPtr responder) + LLCurl::ResponderPtr responder, S32 time_out) { LLCurl::Easy* easy = allocEasy(); if (!easy) { return false; } - easy->prepRequest(url, headers, responder); + easy->prepRequest(url, headers, responder, time_out); easy->getInput().write(data.data(), data.size()); S32 bytes = easy->getInput().str().length(); @@ -909,7 +1154,6 @@ bool LLCurlRequest::post(const std::string& url, // Note: call once per frame S32 LLCurlRequest::process() { - llassert_always(mThreadID == LLThread::currentID()); S32 res = 0; mProcessing = TRUE; @@ -918,12 +1162,25 @@ S32 LLCurlRequest::process() { curlmulti_set_t::iterator curiter = iter++; LLCurl::Multi* multi = *curiter; + + if(!multi->isValid()) + { + if(multi == mActiveMulti) + { + mActiveMulti = NULL ; + mActiveRequestCount = 0 ; + } + mMultiSet.erase(curiter) ; + LLCurl::getCurlThread()->killMulti(multi) ; + continue ; + } + S32 tres = multi->process(); res += tres; if (multi != mActiveMulti && tres == 0 && multi->mQueued == 0) { mMultiSet.erase(curiter); - delete multi; + LLCurl::getCurlThread()->killMulti(multi); } } mProcessing = FALSE; @@ -932,14 +1189,30 @@ S32 LLCurlRequest::process() S32 LLCurlRequest::getQueued() { - llassert_always(mThreadID == LLThread::currentID()); S32 queued = 0; for (curlmulti_set_t::iterator iter = mMultiSet.begin(); iter != mMultiSet.end(); ) { curlmulti_set_t::iterator curiter = iter++; LLCurl::Multi* multi = *curiter; + + if(!multi->isValid()) + { + if(multi == mActiveMulti) + { + mActiveMulti = NULL ; + mActiveRequestCount = 0 ; + } + LLCurl::getCurlThread()->killMulti(multi); + mMultiSet.erase(curiter) ; + continue ; + } + queued += multi->mQueued; + if (multi->getState() != LLCurl::Multi::STATE_READY) + { + ++queued; + } } return queued; } @@ -953,22 +1226,34 @@ LLCurlEasyRequest::LLCurlEasyRequest() mResultReturned(false) { mMulti = new LLCurl::Multi(); + + if(mMulti->isValid()) + { mEasy = mMulti->allocEasy(); if (mEasy) { mEasy->setErrorBuffer(); mEasy->setCA(); + // Set proxy settings if configured to do so. + LLProxy::getInstance()->applyProxySettings(mEasy); + } +} + else + { + LLCurl::getCurlThread()->killMulti(mMulti) ; + mEasy = NULL ; + mMulti = NULL ; } } LLCurlEasyRequest::~LLCurlEasyRequest() { - delete mMulti; + LLCurl::getCurlThread()->killMulti(mMulti) ; } void LLCurlEasyRequest::setopt(CURLoption option, S32 value) { - if (mEasy) + if (isValid() && mEasy) { mEasy->setopt(option, value); } @@ -976,7 +1261,7 @@ void LLCurlEasyRequest::setopt(CURLoption option, S32 value) void LLCurlEasyRequest::setoptString(CURLoption option, const std::string& value) { - if (mEasy) + if (isValid() && mEasy) { mEasy->setoptString(option, value); } @@ -984,7 +1269,7 @@ void LLCurlEasyRequest::setoptString(CURLoption option, const std::string& value void LLCurlEasyRequest::setPost(char* postdata, S32 size) { - if (mEasy) + if (isValid() && mEasy) { mEasy->setopt(CURLOPT_POST, 1); mEasy->setopt(CURLOPT_POSTFIELDS, postdata); @@ -994,7 +1279,7 @@ void LLCurlEasyRequest::setPost(char* postdata, S32 size) void LLCurlEasyRequest::setHeaderCallback(curl_header_callback callback, void* userdata) { - if (mEasy) + if (isValid() && mEasy) { mEasy->setopt(CURLOPT_HEADERFUNCTION, (void*)callback); mEasy->setopt(CURLOPT_HEADERDATA, userdata); // aka CURLOPT_WRITEHEADER @@ -1003,7 +1288,7 @@ void LLCurlEasyRequest::setHeaderCallback(curl_header_callback callback, void* u void LLCurlEasyRequest::setWriteCallback(curl_write_callback callback, void* userdata) { - if (mEasy) + if (isValid() && mEasy) { mEasy->setopt(CURLOPT_WRITEFUNCTION, (void*)callback); mEasy->setopt(CURLOPT_WRITEDATA, userdata); @@ -1012,7 +1297,7 @@ void LLCurlEasyRequest::setWriteCallback(curl_write_callback callback, void* use void LLCurlEasyRequest::setReadCallback(curl_read_callback callback, void* userdata) { - if (mEasy) + if (isValid() && mEasy) { mEasy->setopt(CURLOPT_READFUNCTION, (void*)callback); mEasy->setopt(CURLOPT_READDATA, userdata); @@ -1021,7 +1306,7 @@ void LLCurlEasyRequest::setReadCallback(curl_read_callback callback, void* userd void LLCurlEasyRequest::setSSLCtxCallback(curl_ssl_ctx_callback callback, void* userdata) { - if (mEasy) + if (isValid() && mEasy) { mEasy->setopt(CURLOPT_SSL_CTX_FUNCTION, (void*)callback); mEasy->setopt(CURLOPT_SSL_CTX_DATA, userdata); @@ -1030,7 +1315,7 @@ void LLCurlEasyRequest::setSSLCtxCallback(curl_ssl_ctx_callback callback, void* void LLCurlEasyRequest::slist_append(const char* str) { - if (mEasy) + if (isValid() && mEasy) { mEasy->slist_append(str); } @@ -1041,7 +1326,7 @@ void LLCurlEasyRequest::sendRequest(const std::string& url) llassert_always(!mRequestSent); mRequestSent = true; lldebugs << url << llendl; - if (mEasy) + if (isValid() && mEasy) { mEasy->setHeaders(); mEasy->setoptString(CURLOPT_URL, url); @@ -1053,20 +1338,25 @@ void LLCurlEasyRequest::requestComplete() { llassert_always(mRequestSent); mRequestSent = false; - if (mEasy) + if (isValid() && mEasy) { mMulti->removeEasy(mEasy); } } -S32 LLCurlEasyRequest::perform() -{ - return mMulti->perform(); -} - // Usage: Call getRestult until it returns false (no more messages) bool LLCurlEasyRequest::getResult(CURLcode* result, LLCurl::TransferInfo* info) { + if(!isValid()) + { + return false ; + } + if (!mMulti->isCompleted()) + { //we're busy, try again later + return false; + } + mMulti->setState(LLCurl::Multi::STATE_READY) ; + if (!mEasy) { // Special case - we failed to initialize a curl_easy (can happen if too many open files) @@ -1125,7 +1415,7 @@ CURLMsg* LLCurlEasyRequest::info_read(S32* q, LLCurl::TransferInfo* info) std::string LLCurlEasyRequest::getErrorString() { - return mEasy ? std::string(mEasy->getErrorBuffer()) : std::string(); + return isValid() && mEasy ? std::string(mEasy->getErrorBuffer()) : std::string(); } //////////////////////////////////////////////////////////////////////////// @@ -1151,8 +1441,11 @@ unsigned long LLCurl::ssl_thread_id(void) } #endif -void LLCurl::initClass() +void LLCurl::initClass(F32 curl_reuest_timeout, S32 max_number_handles, bool multi_threaded) { + sCurlRequestTimeOut = curl_reuest_timeout ; //seconds + sMaxHandles = max_number_handles ; //max number of handles, (multi handles and easy handles combined). + // Do not change this "unless you are familiar with and mean to control // internal operations of libcurl" // - http://curl.haxx.se/libcurl/c/curl_global_init.html @@ -1160,8 +1453,6 @@ void LLCurl::initClass() check_curl_code(code); - Easy::sHandleMutex = new LLMutex(NULL); - #if SAFE_SSL S32 mutex_count = CRYPTO_num_locks(); for (S32 i=0; i<mutex_count; i++) @@ -1171,27 +1462,126 @@ void LLCurl::initClass() CRYPTO_set_id_callback(&LLCurl::ssl_thread_id); CRYPTO_set_locking_callback(&LLCurl::ssl_locking_callback); #endif + + sCurlThread = new LLCurlThread(multi_threaded) ; + if(multi_threaded) + { + sHandleMutexp = new LLMutex(NULL) ; + Easy::sHandleMutexp = new LLMutex(NULL) ; + } } void LLCurl::cleanupClass() { + sNotQuitting = false; //set quitting + + //shut down curl thread + while(1) + { + if(!sCurlThread->update(1)) //finish all tasks + { + break ; + } + } + sCurlThread->shutdown() ; + delete sCurlThread ; + sCurlThread = NULL ; + #if SAFE_SSL CRYPTO_set_locking_callback(NULL); for_each(sSSLMutex.begin(), sSSLMutex.end(), DeletePointer()); #endif - delete Easy::sHandleMutex; - Easy::sHandleMutex = NULL; - for (std::set<CURL*>::iterator iter = Easy::sFreeHandles.begin(); iter != Easy::sFreeHandles.end(); ++iter) { CURL* curl = *iter; - curl_easy_cleanup(curl); + LLCurl::deleteEasyHandle(curl); } Easy::sFreeHandles.clear(); + delete Easy::sHandleMutexp ; + Easy::sHandleMutexp = NULL ; + + delete sHandleMutexp ; + sHandleMutexp = NULL ; + llassert(Easy::sActiveHandles.empty()); } +//static +CURLM* LLCurl::newMultiHandle() +{ + LLMutexLock lock(sHandleMutexp) ; + + if(sTotalHandles + 1 > sMaxHandles) + { + llwarns << "no more handles available." << llendl ; + return NULL ; //failed + } + sTotalHandles++; + + CURLM* ret = curl_multi_init() ; + if(!ret) + { + llwarns << "curl_multi_init failed." << llendl ; + } + + return ret ; +} + +//static +CURLMcode LLCurl::deleteMultiHandle(CURLM* handle) +{ + if(handle) + { + LLMutexLock lock(sHandleMutexp) ; + sTotalHandles-- ; + return curl_multi_cleanup(handle) ; + } + return CURLM_OK ; +} + +//static +CURL* LLCurl::newEasyHandle() +{ + LLMutexLock lock(sHandleMutexp) ; + + if(sTotalHandles + 1 > sMaxHandles) + { + llwarns << "no more handles available." << llendl ; + return NULL ; //failed + } + sTotalHandles++; + + CURL* ret = curl_easy_init() ; + if(!ret) + { + llwarns << "curl_easy_init failed." << llendl ; + } + + return ret ; +} + +//static +void LLCurl::deleteEasyHandle(CURL* handle) +{ + if(handle) + { + LLMutexLock lock(sHandleMutexp) ; + curl_easy_cleanup(handle) ; + sTotalHandles-- ; + } +} + const unsigned int LLCurl::MAX_REDIRECTS = 5; + +// Provide access to LLCurl free functions outside of llcurl.cpp without polluting the global namespace. +void LLCurlFF::check_easy_code(CURLcode code) +{ + check_curl_code(code); +} +void LLCurlFF::check_multi_code(CURLMcode code) +{ + check_curl_multi_code(code); +} diff --git a/indra/llmessage/llcurl.h b/indra/llmessage/llcurl.h index 4ce3fa1078..fd664c0fa1 100644 --- a/indra/llmessage/llcurl.h +++ b/indra/llmessage/llcurl.h @@ -41,8 +41,12 @@ #include "llbuffer.h" #include "lliopipe.h" #include "llsd.h" +#include "llthread.h" +#include "llqueuedthread.h" +#include "llframetimer.h" class LLMutex; +class LLCurlThread; // For whatever reason, this is not typedef'd in curl.h typedef size_t (*curl_header_callback)(void *ptr, size_t size, size_t nmemb, void *stream); @@ -159,7 +163,7 @@ public: /** * @ brief Initialize LLCurl class */ - static void initClass(); + static void initClass(F32 curl_reuest_timeout = 120.f, S32 max_number_handles = 256, bool multi_threaded = false); /** * @ brief Cleanup LLCurl class @@ -178,12 +182,201 @@ public: static void ssl_locking_callback(int mode, int type, const char *file, int line); static unsigned long ssl_thread_id(void); + static LLCurlThread* getCurlThread() { return sCurlThread ;} + + static CURLM* newMultiHandle() ; + static CURLMcode deleteMultiHandle(CURLM* handle) ; + static CURL* newEasyHandle() ; + static void deleteEasyHandle(CURL* handle) ; + private: static std::string sCAPath; static std::string sCAFile; static const unsigned int MAX_REDIRECTS; + static LLCurlThread* sCurlThread; + + static LLMutex* sHandleMutexp ; + static S32 sTotalHandles ; + static S32 sMaxHandles; +public: + static bool sNotQuitting; + static F32 sCurlRequestTimeOut; +}; + +class LLCurl::Easy +{ + LOG_CLASS(Easy); + +private: + Easy(); + +public: + static Easy* getEasy(); + ~Easy(); + + CURL* getCurlHandle() const { return mCurlEasyHandle; } + + void setErrorBuffer(); + void setCA(); + + void setopt(CURLoption option, S32 value); + // These assume the setter does not free value! + void setopt(CURLoption option, void* value); + void setopt(CURLoption option, char* value); + // Copies the string so that it is guaranteed to stick around + void setoptString(CURLoption option, const std::string& value); + + void slist_append(const char* str); + void setHeaders(); + + U32 report(CURLcode); + void getTransferInfo(LLCurl::TransferInfo* info); + + void prepRequest(const std::string& url, const std::vector<std::string>& headers, LLCurl::ResponderPtr, S32 time_out = 0, bool post = false); + + const char* getErrorBuffer(); + + std::stringstream& getInput() { return mInput; } + std::stringstream& getHeaderOutput() { return mHeaderOutput; } + LLIOPipe::buffer_ptr_t& getOutput() { return mOutput; } + const LLChannelDescriptors& getChannels() { return mChannels; } + + void resetState(); + + static CURL* allocEasyHandle(); + static void releaseEasyHandle(CURL* handle); + +private: + friend class LLCurl; + friend class LLCurl::Multi; + + CURL* mCurlEasyHandle; + struct curl_slist* mHeaders; + + std::stringstream mRequest; + LLChannelDescriptors mChannels; + LLIOPipe::buffer_ptr_t mOutput; + std::stringstream mInput; + std::stringstream mHeaderOutput; + char mErrorBuffer[CURL_ERROR_SIZE]; + + // Note: char*'s not strings since we pass pointers to curl + std::vector<char*> mStrings; + + LLCurl::ResponderPtr mResponder; + + static std::set<CURL*> sFreeHandles; + static std::set<CURL*> sActiveHandles; + static LLMutex* sHandleMutexp ; }; +class LLCurl::Multi +{ + LOG_CLASS(Multi); + + friend class LLCurlThread ; + +private: + ~Multi(); + + void markDead() ; + bool doPerform(); + +public: + + typedef enum + { + STATE_READY=0, + STATE_PERFORMING=1, + STATE_COMPLETED=2 + } ePerformState; + + Multi(F32 idle_time_out = 0.f); + + LLCurl::Easy* allocEasy(); + bool addEasy(LLCurl::Easy* easy); + void removeEasy(LLCurl::Easy* easy); + + void lock() ; + void unlock() ; + + void setState(ePerformState state) ; + ePerformState getState() ; + + bool isCompleted() ; + bool isValid() {return mCurlMultiHandle != NULL ;} + bool isDead() {return mDead;} + + bool waitToComplete() ; + + S32 process(); + + CURLMsg* info_read(S32* msgs_in_queue); + + S32 mQueued; + S32 mErrorCount; + +private: + void easyFree(LLCurl::Easy*); + void cleanup() ; + + CURLM* mCurlMultiHandle; + + typedef std::set<LLCurl::Easy*> easy_active_list_t; + easy_active_list_t mEasyActiveList; + typedef std::map<CURL*, LLCurl::Easy*> easy_active_map_t; + easy_active_map_t mEasyActiveMap; + typedef std::set<LLCurl::Easy*> easy_free_list_t; + easy_free_list_t mEasyFreeList; + + LLQueuedThread::handle_t mHandle ; + ePerformState mState; + + BOOL mDead ; + LLMutex* mMutexp ; + LLMutex* mDeletionMutexp ; + LLMutex* mEasyMutexp ; + LLFrameTimer mIdleTimer ; + F32 mIdleTimeOut; +}; + +class LLCurlThread : public LLQueuedThread +{ +public: + + class CurlRequest : public LLQueuedThread::QueuedRequest + { + protected: + virtual ~CurlRequest(); // use deleteRequest() + + public: + CurlRequest(handle_t handle, LLCurl::Multi* multi, LLCurlThread* curl_thread); + + /*virtual*/ bool processRequest(); + /*virtual*/ void finishRequest(bool completed); + + private: + // input + LLCurl::Multi* mMulti; + LLCurlThread* mCurlThread; + }; + friend class CurlRequest; + +public: + LLCurlThread(bool threaded = true) ; + virtual ~LLCurlThread() ; + + S32 update(F32 max_time_ms); + + void addMulti(LLCurl::Multi* multi) ; + void killMulti(LLCurl::Multi* multi) ; + +private: + bool doMultiPerform(LLCurl::Multi* multi) ; + void deleteMulti(LLCurl::Multi* multi) ; + void cleanupMulti(LLCurl::Multi* multi) ; +} ; + namespace boost { void intrusive_ptr_add_ref(LLCurl::Responder* p); @@ -201,8 +394,8 @@ public: void get(const std::string& url, LLCurl::ResponderPtr responder); bool getByteRange(const std::string& url, const headers_t& headers, S32 offset, S32 length, LLCurl::ResponderPtr responder); - bool post(const std::string& url, const headers_t& headers, const LLSD& data, LLCurl::ResponderPtr responder); - bool post(const std::string& url, const headers_t& headers, const std::string& data, LLCurl::ResponderPtr responder); + bool post(const std::string& url, const headers_t& headers, const LLSD& data, LLCurl::ResponderPtr responder, S32 time_out = 0); + bool post(const std::string& url, const headers_t& headers, const std::string& data, LLCurl::ResponderPtr responder, S32 time_out = 0); S32 process(); S32 getQueued(); @@ -218,7 +411,6 @@ private: LLCurl::Multi* mActiveMulti; S32 mActiveRequestCount; BOOL mProcessing; - U32 mThreadID; // debug }; class LLCurlEasyRequest @@ -236,9 +428,13 @@ public: void slist_append(const char* str); void sendRequest(const std::string& url); void requestComplete(); - S32 perform(); bool getResult(CURLcode* result, LLCurl::TransferInfo* info = NULL); std::string getErrorString(); + bool isCompleted() {return mMulti->isCompleted() ;} + bool wait() { return mMulti->waitToComplete(); } + bool isValid() {return mMulti && mMulti->isValid(); } + + LLCurl::Easy* getEasy() const { return mEasy; } private: CURLMsg* info_read(S32* queue, LLCurl::TransferInfo* info); @@ -250,4 +446,11 @@ private: bool mResultReturned; }; +// Provide access to LLCurl free functions outside of llcurl.cpp without polluting the global namespace. +namespace LLCurlFF +{ + void check_easy_code(CURLcode code); + void check_multi_code(CURLMcode code); +} + #endif // LL_LLCURL_H diff --git a/indra/llmessage/llfiltersd2xmlrpc.cpp b/indra/llmessage/llfiltersd2xmlrpc.cpp index 812ef7c151..e0ca056a5f 100644 --- a/indra/llmessage/llfiltersd2xmlrpc.cpp +++ b/indra/llmessage/llfiltersd2xmlrpc.cpp @@ -308,6 +308,7 @@ LLFilterSD2XMLRPCResponse::~LLFilterSD2XMLRPCResponse() } +static LLFastTimer::DeclareTimer FTM_PROCESS_SD2XMLRPC_RESPONSE("SD2XMLRPC Response"); // virtual LLIOPipe::EStatus LLFilterSD2XMLRPCResponse::process_impl( const LLChannelDescriptors& channels, @@ -316,6 +317,8 @@ LLIOPipe::EStatus LLFilterSD2XMLRPCResponse::process_impl( LLSD& context, LLPumpIO* pump) { + LLFastTimer t(FTM_PROCESS_SD2XMLRPC_RESPONSE); + PUMP_DEBUG; // This pipe does not work if it does not have everyting. This // could be addressed by making a stream parser for llsd which @@ -382,6 +385,8 @@ LLFilterSD2XMLRPCRequest::~LLFilterSD2XMLRPCRequest() { } +static LLFastTimer::DeclareTimer FTM_PROCESS_SD2XMLRPC_REQUEST("S22XMLRPC Request"); + // virtual LLIOPipe::EStatus LLFilterSD2XMLRPCRequest::process_impl( const LLChannelDescriptors& channels, @@ -390,6 +395,7 @@ LLIOPipe::EStatus LLFilterSD2XMLRPCRequest::process_impl( LLSD& context, LLPumpIO* pump) { + LLFastTimer t(FTM_PROCESS_SD2XMLRPC_REQUEST); // This pipe does not work if it does not have everyting. This // could be addressed by making a stream parser for llsd which // handled partial information. @@ -586,6 +592,8 @@ LLFilterXMLRPCResponse2LLSD::~LLFilterXMLRPCResponse2LLSD() { } +static LLFastTimer::DeclareTimer FTM_PROCESS_XMLRPC2LLSD_RESPONSE("XMLRPC2LLSD Response"); + LLIOPipe::EStatus LLFilterXMLRPCResponse2LLSD::process_impl( const LLChannelDescriptors& channels, buffer_ptr_t& buffer, @@ -593,6 +601,8 @@ LLIOPipe::EStatus LLFilterXMLRPCResponse2LLSD::process_impl( LLSD& context, LLPumpIO* pump) { + LLFastTimer t(FTM_PROCESS_XMLRPC2LLSD_RESPONSE); + PUMP_DEBUG; if(!eos) return STATUS_BREAK; if(!buffer) return STATUS_ERROR; @@ -668,6 +678,7 @@ LLFilterXMLRPCRequest2LLSD::~LLFilterXMLRPCRequest2LLSD() { } +static LLFastTimer::DeclareTimer FTM_PROCESS_XMLRPC2LLSD_REQUEST("XMLRPC2LLSD Request"); LLIOPipe::EStatus LLFilterXMLRPCRequest2LLSD::process_impl( const LLChannelDescriptors& channels, buffer_ptr_t& buffer, @@ -675,6 +686,7 @@ LLIOPipe::EStatus LLFilterXMLRPCRequest2LLSD::process_impl( LLSD& context, LLPumpIO* pump) { + LLFastTimer t(FTM_PROCESS_XMLRPC2LLSD_REQUEST); PUMP_DEBUG; if(!eos) return STATUS_BREAK; if(!buffer) return STATUS_ERROR; diff --git a/indra/llmessage/llhttpassetstorage.cpp b/indra/llmessage/llhttpassetstorage.cpp index 5a38b7fd9f..612d765969 100644 --- a/indra/llmessage/llhttpassetstorage.cpp +++ b/indra/llmessage/llhttpassetstorage.cpp @@ -33,6 +33,7 @@ #include "indra_constants.h" #include "message.h" +#include "llproxy.h" #include "llvfile.h" #include "llvfs.h" @@ -231,7 +232,12 @@ LLSD LLHTTPAssetRequest::getFullDetails() const void LLHTTPAssetRequest::setupCurlHandle() { // *NOTE: Similar code exists in mapserver/llcurlutil.cpp JC - mCurlHandle = curl_easy_init(); + mCurlHandle = LLCurl::newEasyHandle(); + llassert_always(mCurlHandle != NULL) ; + + // Apply proxy settings if configured to do so + LLProxy::getInstance()->applyProxySettings(mCurlHandle); + curl_easy_setopt(mCurlHandle, CURLOPT_NOSIGNAL, 1); curl_easy_setopt(mCurlHandle, CURLOPT_NOPROGRESS, 1); curl_easy_setopt(mCurlHandle, CURLOPT_URL, mURLBuffer.c_str()); @@ -273,7 +279,7 @@ void LLHTTPAssetRequest::setupCurlHandle() void LLHTTPAssetRequest::cleanupCurlHandle() { - curl_easy_cleanup(mCurlHandle); + LLCurl::deleteEasyHandle(mCurlHandle); if (mAssetStoragep) { // Terminating a request. Thus upload or download is no longer pending. @@ -424,12 +430,13 @@ void LLHTTPAssetStorage::_init(const std::string& web_host, const std::string& l // curl_global_init moved to LLCurl::initClass() - mCurlMultiHandle = curl_multi_init(); + mCurlMultiHandle = LLCurl::newMultiHandle() ; + llassert_always(mCurlMultiHandle != NULL) ; } LLHTTPAssetStorage::~LLHTTPAssetStorage() { - curl_multi_cleanup(mCurlMultiHandle); + LLCurl::deleteMultiHandle(mCurlMultiHandle); mCurlMultiHandle = NULL; // curl_global_cleanup moved to LLCurl::initClass() diff --git a/indra/llmessage/llhttpclient.cpp b/indra/llmessage/llhttpclient.cpp index 0e5206a520..0c325a68aa 100644 --- a/indra/llmessage/llhttpclient.cpp +++ b/indra/llmessage/llhttpclient.cpp @@ -158,7 +158,7 @@ namespace if(fstream.is_open()) { fstream.seekg(0, std::ios::end); - U32 fileSize = fstream.tellg(); + U32 fileSize = (U32)fstream.tellg(); fstream.seekg(0, std::ios::beg); std::vector<char> fileBuffer(fileSize); fstream.read(&fileBuffer[0], fileSize); @@ -228,6 +228,12 @@ static void request( LLPumpIO::chain_t chain; LLURLRequest* req = new LLURLRequest(method, url); + if(!req->isValid())//failed + { + delete req ; + return ; + } + req->setSSLVerifyCallback(LLHTTPClient::getCertVerifyCallback(), (void *)req); @@ -423,11 +429,16 @@ static LLSD blocking_request( { lldebugs << "blockingRequest of " << url << llendl; char curl_error_buffer[CURL_ERROR_SIZE] = "\0"; - CURL* curlp = curl_easy_init(); + CURL* curlp = LLCurl::newEasyHandle(); + llassert_always(curlp != NULL) ; + LLHTTPBuffer http_buffer; std::string body_str; // other request method checks root cert first, we skip? + + // Apply configured proxy settings + LLProxy::getInstance()->applyProxySettings(curlp); // * Set curl handle options curl_easy_setopt(curlp, CURLOPT_NOSIGNAL, 1); // don't use SIGALRM for timeouts @@ -436,7 +447,7 @@ static LLSD blocking_request( curl_easy_setopt(curlp, CURLOPT_WRITEDATA, &http_buffer); curl_easy_setopt(curlp, CURLOPT_URL, url.c_str()); curl_easy_setopt(curlp, CURLOPT_ERRORBUFFER, curl_error_buffer); - + // * Setup headers (don't forget to free them after the call!) curl_slist* headers_list = NULL; if (headers.isMap()) @@ -514,7 +525,7 @@ static LLSD blocking_request( } // * Cleanup - curl_easy_cleanup(curlp); + LLCurl::deleteEasyHandle(curlp); return response; } diff --git a/indra/llmessage/lliohttpserver.cpp b/indra/llmessage/lliohttpserver.cpp index 3b18a9177c..987f386aa3 100644 --- a/indra/llmessage/lliohttpserver.cpp +++ b/indra/llmessage/lliohttpserver.cpp @@ -140,6 +140,7 @@ private: LLSD mHeaders; }; +static LLFastTimer::DeclareTimer FTM_PROCESS_HTTP_PIPE("HTTP Pipe"); LLIOPipe::EStatus LLHTTPPipe::process_impl( const LLChannelDescriptors& channels, buffer_ptr_t& buffer, @@ -147,6 +148,7 @@ LLIOPipe::EStatus LLHTTPPipe::process_impl( LLSD& context, LLPumpIO* pump) { + LLFastTimer t(FTM_PROCESS_HTTP_PIPE); PUMP_DEBUG; lldebugs << "LLSDHTTPServer::process_impl" << llendl; @@ -428,6 +430,9 @@ protected: /** * LLHTTPResponseHeader */ + +static LLFastTimer::DeclareTimer FTM_PROCESS_HTTP_HEADER("HTTP Header"); + // virtual LLIOPipe::EStatus LLHTTPResponseHeader::process_impl( const LLChannelDescriptors& channels, @@ -436,6 +441,7 @@ LLIOPipe::EStatus LLHTTPResponseHeader::process_impl( LLSD& context, LLPumpIO* pump) { + LLFastTimer t(FTM_PROCESS_HTTP_HEADER); PUMP_DEBUG; LLMemType m1(LLMemType::MTYPE_IO_HTTP_SERVER); if(eos) @@ -630,6 +636,8 @@ void LLHTTPResponder::markBad( << "</body>\n</html>\n"; } +static LLFastTimer::DeclareTimer FTM_PROCESS_HTTP_RESPONDER("HTTP Responder"); + // virtual LLIOPipe::EStatus LLHTTPResponder::process_impl( const LLChannelDescriptors& channels, @@ -638,6 +646,7 @@ LLIOPipe::EStatus LLHTTPResponder::process_impl( LLSD& context, LLPumpIO* pump) { + LLFastTimer t(FTM_PROCESS_HTTP_RESPONDER); PUMP_DEBUG; LLMemType m1(LLMemType::MTYPE_IO_HTTP_SERVER); LLIOPipe::EStatus status = STATUS_OK; @@ -809,6 +818,8 @@ LLIOPipe::EStatus LLHTTPResponder::process_impl( // Copy everything after mLast read to the out. LLBufferArray::segment_iterator_t seg_iter; + + buffer->lock(); seg_iter = buffer->splitAfter(mLastRead); if(seg_iter != buffer->endSegment()) { @@ -829,7 +840,7 @@ LLIOPipe::EStatus LLHTTPResponder::process_impl( } #endif } - + buffer->unlock(); // // *FIX: get rid of extra bytes off the end // diff --git a/indra/llmessage/lliopipe.cpp b/indra/llmessage/lliopipe.cpp index 6e4eec74a6..8f827f7a30 100644 --- a/indra/llmessage/lliopipe.cpp +++ b/indra/llmessage/lliopipe.cpp @@ -75,6 +75,12 @@ LLIOPipe::~LLIOPipe() //lldebugs << "destroying LLIOPipe" << llendl; } +//virtual +bool LLIOPipe::isValid() +{ + return true ; +} + // static std::string LLIOPipe::lookupStatusString(EStatus status) { diff --git a/indra/llmessage/lliopipe.h b/indra/llmessage/lliopipe.h index 8e656b6da1..cbd17b5a3d 100644 --- a/indra/llmessage/lliopipe.h +++ b/indra/llmessage/lliopipe.h @@ -231,6 +231,8 @@ public: */ virtual ~LLIOPipe(); + virtual bool isValid() ; + protected: /** * @brief Base Constructor. diff --git a/indra/llmessage/lliosocket.cpp b/indra/llmessage/lliosocket.cpp index 8c752fbe30..d5b4d45821 100644 --- a/indra/llmessage/lliosocket.cpp +++ b/indra/llmessage/lliosocket.cpp @@ -251,6 +251,7 @@ LLSocket::~LLSocket() { ll_debug_socket("Destroying socket", mSocket); apr_socket_close(mSocket); + mSocket = NULL; } if(mPool) { @@ -301,6 +302,8 @@ LLIOSocketReader::~LLIOSocketReader() //lldebugs << "Destroying LLIOSocketReader" << llendl; } +static LLFastTimer::DeclareTimer FTM_PROCESS_SOCKET_READER("Socket Reader"); + // virtual LLIOPipe::EStatus LLIOSocketReader::process_impl( const LLChannelDescriptors& channels, @@ -309,6 +312,7 @@ LLIOPipe::EStatus LLIOSocketReader::process_impl( LLSD& context, LLPumpIO* pump) { + LLFastTimer t(FTM_PROCESS_SOCKET_READER); PUMP_DEBUG; LLMemType m1(LLMemType::MTYPE_IO_TCP); if(!mSource) return STATUS_PRECONDITION_NOT_MET; @@ -401,6 +405,7 @@ LLIOSocketWriter::~LLIOSocketWriter() //lldebugs << "Destroying LLIOSocketWriter" << llendl; } +static LLFastTimer::DeclareTimer FTM_PROCESS_SOCKET_WRITER("Socket Writer"); // virtual LLIOPipe::EStatus LLIOSocketWriter::process_impl( const LLChannelDescriptors& channels, @@ -409,6 +414,7 @@ LLIOPipe::EStatus LLIOSocketWriter::process_impl( LLSD& context, LLPumpIO* pump) { + LLFastTimer t(FTM_PROCESS_SOCKET_WRITER); PUMP_DEBUG; LLMemType m1(LLMemType::MTYPE_IO_TCP); if(!mDestination) return STATUS_PRECONDITION_NOT_MET; @@ -439,6 +445,7 @@ LLIOPipe::EStatus LLIOSocketWriter::process_impl( // efficient - not only because writev() is better, but also // because we won't have to do as much work to find the start // address. + buffer->lock(); LLBufferArray::segment_iterator_t it; LLBufferArray::segment_iterator_t end = buffer->endSegment(); LLSegment segment; @@ -518,6 +525,8 @@ LLIOPipe::EStatus LLIOSocketWriter::process_impl( } } + buffer->unlock(); + PUMP_DEBUG; if(done && eos) { @@ -555,6 +564,7 @@ void LLIOServerSocket::setResponseTimeout(F32 timeout_secs) mResponseTimeout = timeout_secs; } +static LLFastTimer::DeclareTimer FTM_PROCESS_SERVER_SOCKET("Server Socket"); // virtual LLIOPipe::EStatus LLIOServerSocket::process_impl( const LLChannelDescriptors& channels, @@ -563,6 +573,7 @@ LLIOPipe::EStatus LLIOServerSocket::process_impl( LLSD& context, LLPumpIO* pump) { + LLFastTimer t(FTM_PROCESS_SERVER_SOCKET); PUMP_DEBUG; LLMemType m1(LLMemType::MTYPE_IO_TCP); if(!pump) diff --git a/indra/llmessage/lliosocket.h b/indra/llmessage/lliosocket.h index e0f6c1e34d..be0f7dfcc6 100644 --- a/indra/llmessage/lliosocket.h +++ b/indra/llmessage/lliosocket.h @@ -145,13 +145,6 @@ public: */ apr_socket_t* getSocket() const { return mSocket; } -protected: - /** - * @brief Protected constructor since should only make sockets - * with one of the two <code>create()</code> calls. - */ - LLSocket(apr_socket_t* socket, apr_pool_t* pool); - /** * @brief Set default socket options, with SO_NONBLOCK = 0 and a timeout in us. * @param timeout Number of microseconds to wait on this socket. Any @@ -164,6 +157,13 @@ protected: */ void setNonBlocking(); +protected: + /** + * @brief Protected constructor since should only make sockets + * with one of the two <code>create()</code> calls. + */ + LLSocket(apr_socket_t* socket, apr_pool_t* pool); + public: /** * @brief Do not call this directly. diff --git a/indra/llmessage/llioutil.cpp b/indra/llmessage/llioutil.cpp index 2e6ee59ff2..8c50fd5069 100644 --- a/indra/llmessage/llioutil.cpp +++ b/indra/llmessage/llioutil.cpp @@ -43,6 +43,8 @@ LLIOPipe::EStatus LLIOFlush::process_impl( return STATUS_OK; } + +static LLFastTimer::DeclareTimer FTM_PROCESS_SLEEP("IO Sleep"); /** * @class LLIOSleep */ @@ -53,6 +55,7 @@ LLIOPipe::EStatus LLIOSleep::process_impl( LLSD& context, LLPumpIO* pump) { + LLFastTimer t(FTM_PROCESS_SLEEP); if(mSeconds > 0.0) { if(pump) pump->sleepChain(mSeconds); @@ -62,6 +65,7 @@ LLIOPipe::EStatus LLIOSleep::process_impl( return STATUS_DONE; } +static LLFastTimer::DeclareTimer FTM_PROCESS_ADD_CHAIN("Add Chain"); /** * @class LLIOAddChain */ @@ -72,6 +76,7 @@ LLIOPipe::EStatus LLIOAddChain::process_impl( LLSD& context, LLPumpIO* pump) { + LLFastTimer t(FTM_PROCESS_ADD_CHAIN); pump->addChain(mChain, mTimeout); return STATUS_DONE; } diff --git a/indra/llmessage/llmime.cpp b/indra/llmessage/llmime.cpp index 943a734927..9d9c4ebd68 100644 --- a/indra/llmessage/llmime.cpp +++ b/indra/llmessage/llmime.cpp @@ -388,7 +388,7 @@ bool LLMimeParser::Impl::parseHeaders( // not to read past limit when we get() the newline. S32 max_get = llmin((S32)LINE_BUFFER_LENGTH, limit - mScanCount - 1); istr.getline(mBuffer, max_get, '\r'); - mScanCount += istr.gcount(); + mScanCount += (S32)istr.gcount(); int c = istr.get(); if(EOF == c) { @@ -496,7 +496,7 @@ void LLMimeParser::Impl::scanPastSeparator( // past limit when we get() the newline. S32 max_get = llmin((S32)LINE_BUFFER_LENGTH, limit - mScanCount - 1); istr.getline(mBuffer, max_get, '\r'); - mScanCount += istr.gcount(); + mScanCount += (S32)istr.gcount(); if(istr.gcount() >= LINE_BUFFER_LENGTH - 1) { // that's way too long to be a separator, so ignore it. diff --git a/indra/llmessage/llpacketring.cpp b/indra/llmessage/llpacketring.cpp index 8999dec64a..fc6e9c5193 100644 --- a/indra/llmessage/llpacketring.cpp +++ b/indra/llmessage/llpacketring.cpp @@ -28,11 +28,20 @@ #include "llpacketring.h" +#if LL_WINDOWS + #include <winsock2.h> +#else + #include <sys/socket.h> + #include <netinet/in.h> +#endif + // linden library includes #include "llerror.h" #include "lltimer.h" -#include "timing.h" +#include "llproxy.h" #include "llrand.h" +#include "message.h" +#include "timing.h" #include "u64.h" /////////////////////////////////////////////////////////// @@ -216,8 +225,32 @@ S32 LLPacketRing::receivePacket (S32 socket, char *datap) else { // no delay, pull straight from net - packet_size = receive_packet(socket, datap); - mLastSender = ::get_sender(); + if (LLProxy::isSOCKSProxyEnabled()) + { + U8 buffer[NET_BUFFER_SIZE + SOCKS_HEADER_SIZE]; + packet_size = receive_packet(socket, static_cast<char*>(static_cast<void*>(buffer))); + + if (packet_size > SOCKS_HEADER_SIZE) + { + // *FIX We are assuming ATYP is 0x01 (IPv4), not 0x03 (hostname) or 0x04 (IPv6) + memcpy(datap, buffer + SOCKS_HEADER_SIZE, packet_size - SOCKS_HEADER_SIZE); + proxywrap_t * header = static_cast<proxywrap_t*>(static_cast<void*>(buffer)); + mLastSender.setAddress(header->addr); + mLastSender.setPort(ntohs(header->port)); + + packet_size -= SOCKS_HEADER_SIZE; // The unwrapped packet size + } + else + { + packet_size = 0; + } + } + else + { + packet_size = receive_packet(socket, datap); + mLastSender = ::get_sender(); + } + mLastReceivingIF = ::get_receiving_interface(); if (packet_size) // did we actually get a packet? @@ -243,7 +276,7 @@ BOOL LLPacketRing::sendPacket(int h_socket, char * send_buffer, S32 buf_size, LL BOOL status = TRUE; if (!mUseOutThrottle) { - return send_packet(h_socket, send_buffer, buf_size, host.getAddress(), host.getPort() ); + return sendPacketImpl(h_socket, send_buffer, buf_size, host ); } else { @@ -264,7 +297,7 @@ BOOL LLPacketRing::sendPacket(int h_socket, char * send_buffer, S32 buf_size, LL mOutBufferLength -= packetp->getSize(); packet_size = packetp->getSize(); - status = send_packet(h_socket, packetp->getData(), packet_size, packetp->getHost().getAddress(), packetp->getHost().getPort()); + status = sendPacketImpl(h_socket, packetp->getData(), packet_size, packetp->getHost()); delete packetp; // Update the throttle @@ -273,7 +306,7 @@ BOOL LLPacketRing::sendPacket(int h_socket, char * send_buffer, S32 buf_size, LL else { // If the queue's empty, we can just send this packet right away. - status = send_packet(h_socket, send_buffer, buf_size, host.getAddress(), host.getPort() ); + status = sendPacketImpl(h_socket, send_buffer, buf_size, host ); packet_size = buf_size; // Update the throttle @@ -311,3 +344,29 @@ BOOL LLPacketRing::sendPacket(int h_socket, char * send_buffer, S32 buf_size, LL return status; } + +BOOL LLPacketRing::sendPacketImpl(int h_socket, const char * send_buffer, S32 buf_size, LLHost host) +{ + + if (!LLProxy::isSOCKSProxyEnabled()) + { + return send_packet(h_socket, send_buffer, buf_size, host.getAddress(), host.getPort()); + } + + char headered_send_buffer[NET_BUFFER_SIZE + SOCKS_HEADER_SIZE]; + + proxywrap_t *socks_header = static_cast<proxywrap_t*>(static_cast<void*>(&headered_send_buffer)); + socks_header->rsv = 0; + socks_header->addr = host.getAddress(); + socks_header->port = htons(host.getPort()); + socks_header->atype = ADDRESS_IPV4; + socks_header->frag = 0; + + memcpy(headered_send_buffer + SOCKS_HEADER_SIZE, send_buffer, buf_size); + + return send_packet( h_socket, + headered_send_buffer, + buf_size + SOCKS_HEADER_SIZE, + LLProxy::getInstance()->getUDPProxy().getAddress(), + LLProxy::getInstance()->getUDPProxy().getPort()); +} diff --git a/indra/llmessage/llpacketring.h b/indra/llmessage/llpacketring.h index e6409d2048..b214271e78 100644 --- a/indra/llmessage/llpacketring.h +++ b/indra/llmessage/llpacketring.h @@ -30,11 +30,11 @@ #include <queue> -#include "llpacketbuffer.h" #include "llhost.h" -#include "net.h" +#include "llpacketbuffer.h" +#include "llproxy.h" #include "llthrottle.h" - +#include "net.h" class LLPacketRing { @@ -82,6 +82,9 @@ protected: LLHost mLastSender; LLHost mLastReceivingIF; + +private: + BOOL sendPacketImpl(int h_socket, const char * send_buffer, S32 buf_size, LLHost host); }; diff --git a/indra/llmessage/llproxy.cpp b/indra/llmessage/llproxy.cpp new file mode 100644 index 0000000000..9988fcd9c0 --- /dev/null +++ b/indra/llmessage/llproxy.cpp @@ -0,0 +1,546 @@ +/** + * @file llproxy.cpp + * @brief UDP and HTTP proxy communications + * + * $LicenseInfo:firstyear=2011&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2011, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include "linden_common.h" + +#include "llproxy.h" + +#include <string> +#include <curl/curl.h> + +#include "llapr.h" +#include "llcurl.h" +#include "llhost.h" + +// Static class variable instances + +// We want this to be static to avoid excessive indirection on every +// incoming packet just to do a simple bool test. The getter for this +// member is also static +bool LLProxy::sUDPProxyEnabled = false; + +// Some helpful TCP static functions. +static apr_status_t tcp_blocking_handshake(LLSocket::ptr_t handle, char * dataout, apr_size_t outlen, char * datain, apr_size_t maxinlen); // Do a TCP data handshake +static LLSocket::ptr_t tcp_open_channel(LLHost host); // Open a TCP channel to a given host +static void tcp_close_channel(LLSocket::ptr_t* handle_ptr); // Close an open TCP channel + +LLProxy::LLProxy(): + mHTTPProxyEnabled(false), + mProxyMutex(NULL), + mUDPProxy(), + mTCPProxy(), + mHTTPProxy(), + mProxyType(LLPROXY_SOCKS), + mAuthMethodSelected(METHOD_NOAUTH), + mSocksUsername(), + mSocksPassword() +{ +} + +LLProxy::~LLProxy() +{ + stopSOCKSProxy(); + disableHTTPProxy(); +} + +/** + * @brief Open the SOCKS 5 TCP control channel. + * + * Perform a SOCKS 5 authentication and UDP association with the proxy server. + * + * @param proxy The SOCKS 5 server to connect to. + * @return SOCKS_OK if successful, otherwise a socks error code from llproxy.h. + */ +S32 LLProxy::proxyHandshake(LLHost proxy) +{ + S32 result; + + /* SOCKS 5 Auth request */ + socks_auth_request_t socks_auth_request; + socks_auth_response_t socks_auth_response; + + socks_auth_request.version = SOCKS_VERSION; // SOCKS version 5 + socks_auth_request.num_methods = 1; // Sending 1 method. + socks_auth_request.methods = getSelectedAuthMethod(); // Send only the selected method. + + result = tcp_blocking_handshake(mProxyControlChannel, + static_cast<char*>(static_cast<void*>(&socks_auth_request)), + sizeof(socks_auth_request), + static_cast<char*>(static_cast<void*>(&socks_auth_response)), + sizeof(socks_auth_response)); + if (result != APR_SUCCESS) + { + LL_WARNS("Proxy") << "SOCKS authentication request failed, error on TCP control channel : " << result << LL_ENDL; + stopSOCKSProxy(); + return SOCKS_CONNECT_ERROR; + } + + if (socks_auth_response.method == AUTH_NOT_ACCEPTABLE) + { + LL_WARNS("Proxy") << "SOCKS 5 server refused all our authentication methods." << LL_ENDL; + stopSOCKSProxy(); + return SOCKS_NOT_ACCEPTABLE; + } + + /* SOCKS 5 USERNAME/PASSWORD authentication */ + if (socks_auth_response.method == METHOD_PASSWORD) + { + // The server has requested a username/password combination + std::string socks_username(getSocksUser()); + std::string socks_password(getSocksPwd()); + U32 request_size = socks_username.size() + socks_password.size() + 3; + char * password_auth = new char[request_size]; + password_auth[0] = 0x01; + password_auth[1] = socks_username.size(); + memcpy(&password_auth[2], socks_username.c_str(), socks_username.size()); + password_auth[socks_username.size() + 2] = socks_password.size(); + memcpy(&password_auth[socks_username.size() + 3], socks_password.c_str(), socks_password.size()); + + authmethod_password_reply_t password_reply; + + result = tcp_blocking_handshake(mProxyControlChannel, + password_auth, + request_size, + static_cast<char*>(static_cast<void*>(&password_reply)), + sizeof(password_reply)); + delete[] password_auth; + + if (result != APR_SUCCESS) + { + LL_WARNS("Proxy") << "SOCKS authentication failed, error on TCP control channel : " << result << LL_ENDL; + stopSOCKSProxy(); + return SOCKS_CONNECT_ERROR; + } + + if (password_reply.status != AUTH_SUCCESS) + { + LL_WARNS("Proxy") << "SOCKS authentication failed" << LL_ENDL; + stopSOCKSProxy(); + return SOCKS_AUTH_FAIL; + } + } + + /* SOCKS5 connect request */ + + socks_command_request_t connect_request; + socks_command_response_t connect_reply; + + connect_request.version = SOCKS_VERSION; // SOCKS V5 + connect_request.command = COMMAND_UDP_ASSOCIATE; // Associate UDP + connect_request.reserved = FIELD_RESERVED; + connect_request.atype = ADDRESS_IPV4; + connect_request.address = htonl(0); // 0.0.0.0 + connect_request.port = htons(0); // 0 + // "If the client is not in possession of the information at the time of the UDP ASSOCIATE, + // the client MUST use a port number and address of all zeros. RFC 1928" + + result = tcp_blocking_handshake(mProxyControlChannel, + static_cast<char*>(static_cast<void*>(&connect_request)), + sizeof(connect_request), + static_cast<char*>(static_cast<void*>(&connect_reply)), + sizeof(connect_reply)); + if (result != APR_SUCCESS) + { + LL_WARNS("Proxy") << "SOCKS connect request failed, error on TCP control channel : " << result << LL_ENDL; + stopSOCKSProxy(); + return SOCKS_CONNECT_ERROR; + } + + if (connect_reply.reply != REPLY_REQUEST_GRANTED) + { + LL_WARNS("Proxy") << "Connection to SOCKS 5 server failed, UDP forward request not granted" << LL_ENDL; + stopSOCKSProxy(); + return SOCKS_UDP_FWD_NOT_GRANTED; + } + + mUDPProxy.setPort(ntohs(connect_reply.port)); // reply port is in network byte order + mUDPProxy.setAddress(proxy.getAddress()); + // The connection was successful. We now have the UDP port to send requests that need forwarding to. + LL_INFOS("Proxy") << "SOCKS 5 UDP proxy connected on " << mUDPProxy << LL_ENDL; + + return SOCKS_OK; +} + +/** + * @brief Initiates a SOCKS 5 proxy session. + * + * Performs basic checks on host to verify that it is a valid address. Opens the control channel + * and then negotiates the proxy connection with the server. Closes any existing SOCKS + * connection before proceeding. Also disables an HTTP proxy if it is using SOCKS as the proxy. + * + * + * @param host Socks server to connect to. + * @return SOCKS_OK if successful, otherwise a SOCKS error code defined in llproxy.h. + */ +S32 LLProxy::startSOCKSProxy(LLHost host) +{ + if (host.isOk()) + { + mTCPProxy = host; + } + else + { + return SOCKS_INVALID_HOST; + } + + // Close any running SOCKS connection. + stopSOCKSProxy(); + + mProxyControlChannel = tcp_open_channel(mTCPProxy); + if (!mProxyControlChannel) + { + return SOCKS_HOST_CONNECT_FAILED; + } + + S32 status = proxyHandshake(mTCPProxy); + + if (status != SOCKS_OK) + { + // Shut down the proxy if any of the above steps failed. + stopSOCKSProxy(); + } + else + { + // Connection was successful. + sUDPProxyEnabled = true; + } + + return status; +} + +/** + * @brief Stop using the SOCKS 5 proxy. + * + * This will stop sending UDP packets through the SOCKS 5 proxy + * and will also stop the HTTP proxy if it is configured to use SOCKS. + * The proxy control channel will also be disconnected. + */ +void LLProxy::stopSOCKSProxy() +{ + sUDPProxyEnabled = false; + + // If the SOCKS proxy is requested to stop and we are using that for HTTP as well + // then we must shut down any HTTP proxy operations. But it is allowable if web + // proxy is being used to continue proxying HTTP. + + if (LLPROXY_SOCKS == getHTTPProxyType()) + { + disableHTTPProxy(); + } + + if (mProxyControlChannel) + { + tcp_close_channel(&mProxyControlChannel); + } +} + +/** + * @brief Set the proxy's SOCKS authentication method to none. + */ +void LLProxy::setAuthNone() +{ + LLMutexLock lock(&mProxyMutex); + + mAuthMethodSelected = METHOD_NOAUTH; +} + +/** + * @brief Set the proxy's SOCKS authentication method to password. + * + * Check whether the lengths of the supplied username + * and password conform to the lengths allowed by the + * SOCKS protocol. + * + * @param username The SOCKS username to send. + * @param password The SOCKS password to send. + * @return Return true if applying the settings was successful. No changes are made if false. + * + */ +bool LLProxy::setAuthPassword(const std::string &username, const std::string &password) +{ + if (username.length() > SOCKSMAXUSERNAMELEN || password.length() > SOCKSMAXPASSWORDLEN || + username.length() < SOCKSMINUSERNAMELEN || password.length() < SOCKSMINPASSWORDLEN) + { + LL_WARNS("Proxy") << "Invalid SOCKS 5 password or username length." << LL_ENDL; + return false; + } + + LLMutexLock lock(&mProxyMutex); + + mAuthMethodSelected = METHOD_PASSWORD; + mSocksUsername = username; + mSocksPassword = password; + + return true; +} + +/** + * @brief Enable the HTTP proxy for either SOCKS or HTTP. + * + * Check the supplied host to see if it is a valid IP and port. + * + * @param httpHost Proxy server to connect to. + * @param type Is the host a SOCKS or HTTP proxy. + * @return Return true if applying the setting was successful. No changes are made if false. + */ +bool LLProxy::enableHTTPProxy(LLHost httpHost, LLHttpProxyType type) +{ + if (!httpHost.isOk()) + { + LL_WARNS("Proxy") << "Invalid SOCKS 5 Server" << LL_ENDL; + return false; + } + + LLMutexLock lock(&mProxyMutex); + + mHTTPProxy = httpHost; + mProxyType = type; + + mHTTPProxyEnabled = true; + + return true; +} + +/** + * @brief Enable the HTTP proxy without changing the proxy settings. + * + * This should not be called unless the proxy has already been set up. + * + * @return Return true only if the current settings are valid and the proxy was enabled. + */ +bool LLProxy::enableHTTPProxy() +{ + bool ok; + + LLMutexLock lock(&mProxyMutex); + + ok = (mHTTPProxy.isOk()); + if (ok) + { + mHTTPProxyEnabled = true; + } + + return ok; +} + +/** + * @brief Disable the HTTP proxy. + */ +void LLProxy::disableHTTPProxy() +{ + LLMutexLock lock(&mProxyMutex); + + mHTTPProxyEnabled = false; +} + +/** + * @brief Get the currently selected HTTP proxy type + */ +LLHttpProxyType LLProxy::getHTTPProxyType() const +{ + LLMutexLock lock(&mProxyMutex); + return mProxyType; +} + +/** + * @brief Get the SOCKS 5 password. + */ +std::string LLProxy::getSocksPwd() const +{ + LLMutexLock lock(&mProxyMutex); + return mSocksPassword; +} + +/** + * @brief Get the SOCKS 5 username. + */ +std::string LLProxy::getSocksUser() const +{ + LLMutexLock lock(&mProxyMutex); + return mSocksUsername; +} + +/** + * @brief Get the currently selected SOCKS 5 authentication method. + * + * @return Returns either none or password. + */ +LLSocks5AuthType LLProxy::getSelectedAuthMethod() const +{ + LLMutexLock lock(&mProxyMutex); + return mAuthMethodSelected; +} + +/** + * @brief Stop the LLProxy and make certain that any APR pools and classes are deleted before terminating APR. + * + * Deletes the LLProxy singleton, destroying the APR pool used by the control channel as well as . + */ +//static +void LLProxy::cleanupClass() +{ + getInstance()->stopSOCKSProxy(); + deleteSingleton(); +} + +void LLProxy::applyProxySettings(LLCurlEasyRequest* handle) +{ + applyProxySettings(handle->getEasy()); +} + +void LLProxy::applyProxySettings(LLCurl::Easy* handle) +{ + applyProxySettings(handle->getCurlHandle()); +} + +/** + * @brief Apply proxy settings to a CuRL request if an HTTP proxy is enabled. + * + * This method has been designed to be safe to call from + * any thread in the viewer. This allows requests in the + * texture fetch thread to be aware of the proxy settings. + * When the HTTP proxy is enabled, the proxy mutex will + * be locked every time this method is called. + * + * @param handle A pointer to a valid CURL request, before it has been performed. + */ +void LLProxy::applyProxySettings(CURL* handle) +{ + // Do a faster unlocked check to see if we are supposed to proxy. + if (mHTTPProxyEnabled) + { + // We think we should proxy, lock the proxy mutex. + LLMutexLock lock(&mProxyMutex); + // Now test again to verify that the proxy wasn't disabled between the first check and the lock. + if (mHTTPProxyEnabled) + { + LLCurlFF::check_easy_code(curl_easy_setopt(handle, CURLOPT_PROXY, mHTTPProxy.getIPString().c_str())); + LLCurlFF::check_easy_code(curl_easy_setopt(handle, CURLOPT_PROXYPORT, mHTTPProxy.getPort())); + + if (mProxyType == LLPROXY_SOCKS) + { + LLCurlFF::check_easy_code(curl_easy_setopt(handle, CURLOPT_PROXYTYPE, CURLPROXY_SOCKS5)); + if (mAuthMethodSelected == METHOD_PASSWORD) + { + std::string auth_string = mSocksUsername + ":" + mSocksPassword; + LLCurlFF::check_easy_code(curl_easy_setopt(handle, CURLOPT_PROXYUSERPWD, auth_string.c_str())); + } + } + else + { + LLCurlFF::check_easy_code(curl_easy_setopt(handle, CURLOPT_PROXYTYPE, CURLPROXY_HTTP)); + } + } + } +} + +/** + * @brief Send one TCP packet and receive one in return. + * + * This operation is done synchronously with a 1000ms timeout. Therefore, it should not be used when a blocking + * operation would impact the operation of the viewer. + * + * @param handle_ptr Pointer to a connected LLSocket of type STREAM_TCP. + * @param dataout Data to send. + * @param outlen Length of dataout. + * @param datain Buffer for received data. Undefined if return value is not APR_SUCCESS. + * @param maxinlen Maximum possible length of received data. Short reads are allowed. + * @return Indicates APR status code of exchange. APR_SUCCESS if exchange was successful, -1 if invalid data length was received. + */ +static apr_status_t tcp_blocking_handshake(LLSocket::ptr_t handle, char * dataout, apr_size_t outlen, char * datain, apr_size_t maxinlen) +{ + apr_socket_t* apr_socket = handle->getSocket(); + apr_status_t rv = APR_SUCCESS; + + apr_size_t expected_len = outlen; + + handle->setBlocking(1000); + + rv = apr_socket_send(apr_socket, dataout, &outlen); + if (APR_SUCCESS != rv) + { + LL_WARNS("Proxy") << "Error sending data to proxy control channel, status: " << rv << LL_ENDL; + ll_apr_warn_status(rv); + } + else if (expected_len != outlen) + { + LL_WARNS("Proxy") << "Incorrect data length sent. Expected: " << expected_len << + " Sent: " << outlen << LL_ENDL; + rv = -1; + } + + if (APR_SUCCESS == rv) + { + expected_len = maxinlen; + rv = apr_socket_recv(apr_socket, datain, &maxinlen); + if (rv != APR_SUCCESS) + { + LL_WARNS("Proxy") << "Error receiving data from proxy control channel, status: " << rv << LL_ENDL; + ll_apr_warn_status(rv); + } + else if (expected_len < maxinlen) + { + LL_WARNS("Proxy") << "Incorrect data length received. Expected: " << expected_len << + " Received: " << maxinlen << LL_ENDL; + rv = -1; + } + } + + handle->setNonBlocking(); + + return rv; +} + +/** + * @brief Open a LLSocket and do a blocking connect to the chosen host. + * + * Checks for a successful connection, and makes sure the connection is closed if it fails. + * + * @param host The host to open the connection to. + * @return The created socket. Will evaluate as NULL if the connection is unsuccessful. + */ +static LLSocket::ptr_t tcp_open_channel(LLHost host) +{ + LLSocket::ptr_t socket = LLSocket::create(NULL, LLSocket::STREAM_TCP); + bool connected = socket->blockingConnect(host); + if (!connected) + { + tcp_close_channel(&socket); + } + + return socket; +} + +/** + * @brief Close the socket. + * + * @param handle_ptr The handle of the socket being closed. A pointer-to-pointer to avoid increasing the use count. + */ +static void tcp_close_channel(LLSocket::ptr_t* handle_ptr) +{ + LL_DEBUGS("Proxy") << "Resetting proxy LLSocket handle, use_count == " << handle_ptr->use_count() << LL_ENDL; + handle_ptr->reset(); +} diff --git a/indra/llmessage/llproxy.h b/indra/llmessage/llproxy.h new file mode 100644 index 0000000000..a919370540 --- /dev/null +++ b/indra/llmessage/llproxy.h @@ -0,0 +1,352 @@ +/** + * @file llproxy.h + * @brief UDP and HTTP proxy communications + * + * $LicenseInfo:firstyear=2011&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2011, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef LL_PROXY_H +#define LL_PROXY_H + +#include "llcurl.h" +#include "llhost.h" +#include "lliosocket.h" +#include "llmemory.h" +#include "llsingleton.h" +#include "llthread.h" +#include <string> + +// SOCKS error codes returned from the StartProxy method +#define SOCKS_OK 0 +#define SOCKS_CONNECT_ERROR (-1) +#define SOCKS_NOT_PERMITTED (-2) +#define SOCKS_NOT_ACCEPTABLE (-3) +#define SOCKS_AUTH_FAIL (-4) +#define SOCKS_UDP_FWD_NOT_GRANTED (-5) +#define SOCKS_HOST_CONNECT_FAILED (-6) +#define SOCKS_INVALID_HOST (-7) + +#ifndef MAXHOSTNAMELEN +#define MAXHOSTNAMELEN (255 + 1) /* socks5: 255, +1 for len. */ +#endif + +#define SOCKSMAXUSERNAMELEN 255 +#define SOCKSMAXPASSWORDLEN 255 + +#define SOCKSMINUSERNAMELEN 1 +#define SOCKSMINPASSWORDLEN 1 + +#define SOCKS_VERSION 0x05 // we are using SOCKS 5 + +#define SOCKS_HEADER_SIZE 10 + +// SOCKS 5 address/hostname types +#define ADDRESS_IPV4 0x01 +#define ADDRESS_HOSTNAME 0x03 +#define ADDRESS_IPV6 0x04 + +// Lets just use our own ipv4 struct rather than dragging in system +// specific headers +union ipv4_address_t { + U8 octets[4]; + U32 addr32; +}; + +// SOCKS 5 control channel commands +#define COMMAND_TCP_STREAM 0x01 +#define COMMAND_TCP_BIND 0x02 +#define COMMAND_UDP_ASSOCIATE 0x03 + +// SOCKS 5 command replies +#define REPLY_REQUEST_GRANTED 0x00 +#define REPLY_GENERAL_FAIL 0x01 +#define REPLY_RULESET_FAIL 0x02 +#define REPLY_NETWORK_UNREACHABLE 0x03 +#define REPLY_HOST_UNREACHABLE 0x04 +#define REPLY_CONNECTION_REFUSED 0x05 +#define REPLY_TTL_EXPIRED 0x06 +#define REPLY_PROTOCOL_ERROR 0x07 +#define REPLY_TYPE_NOT_SUPPORTED 0x08 + +#define FIELD_RESERVED 0x00 + +// The standard SOCKS 5 request packet +// Push current alignment to stack and set alignment to 1 byte boundary +// This enabled us to use structs directly to set up and receive network packets +// into the correct fields, without fear of boundary alignment causing issues +#pragma pack(push,1) + +// SOCKS 5 command packet +struct socks_command_request_t { + U8 version; + U8 command; + U8 reserved; + U8 atype; + U32 address; + U16 port; +}; + +// Standard SOCKS 5 reply packet +struct socks_command_response_t { + U8 version; + U8 reply; + U8 reserved; + U8 atype; + U8 add_bytes[4]; + U16 port; +}; + +#define AUTH_NOT_ACCEPTABLE 0xFF // reply if preferred methods are not available +#define AUTH_SUCCESS 0x00 // reply if authentication successful + +// SOCKS 5 authentication request, stating which methods the client supports +struct socks_auth_request_t { + U8 version; + U8 num_methods; + U8 methods; // We are only using a single method currently +}; + +// SOCKS 5 authentication response packet, stating server preferred method +struct socks_auth_response_t { + U8 version; + U8 method; +}; + +// SOCKS 5 password reply packet +struct authmethod_password_reply_t { + U8 version; + U8 status; +}; + +// SOCKS 5 UDP packet header +struct proxywrap_t { + U16 rsv; + U8 frag; + U8 atype; + U32 addr; + U16 port; +}; + +#pragma pack(pop) /* restore original alignment from stack */ + + +// Currently selected HTTP proxy type +enum LLHttpProxyType +{ + LLPROXY_SOCKS = 0, + LLPROXY_HTTP = 1 +}; + +// Auth types +enum LLSocks5AuthType +{ + METHOD_NOAUTH = 0x00, // Client supports no auth + METHOD_GSSAPI = 0x01, // Client supports GSSAPI (Not currently supported) + METHOD_PASSWORD = 0x02 // Client supports username/password +}; + +/** + * @brief Manage SOCKS 5 UDP proxy and HTTP proxy. + * + * This class is responsible for managing two interconnected tasks, + * connecting to a SOCKS 5 proxy for use by LLPacketRing to send UDP + * packets and managing proxy settings for HTTP requests. + * + * <h1>Threading:</h1> + * Because HTTP requests can be generated in threads outside the + * main thread, it is necessary for some of the information stored + * by this class to be available to other threads. The members that + * need to be read across threads are in a labeled section below. + * To protect those members, a mutex, mProxyMutex should be locked + * before reading or writing those members. Methods that can lock + * mProxyMutex are in a labeled section below. Those methods should + * not be called while the mutex is already locked. + * + * There is also a LLAtomic type flag (mHTTPProxyEnabled) that is used + * to track whether the HTTP proxy is currently enabled. This allows + * for faster unlocked checks to see if the proxy is enabled. This + * allows us to cut down on the performance hit when the proxy is + * disabled compared to before this class was introduced. + * + * <h1>UDP Proxying:</h1> + * UDP datagrams are proxied via a SOCKS 5 proxy with the UDP associate + * command. To initiate the proxy, a TCP socket connection is opened + * to the SOCKS 5 host, and after a handshake exchange, the server + * returns a port and address to send the UDP traffic that is to be + * proxied to. The LLProxy class tracks this address and port after the + * exchange and provides it to LLPacketRing when required to. All UDP + * proxy management occurs in the main thread. + * + * <h1>HTTP Proxying:</h1> + * This class allows all viewer HTTP packets to be sent through a proxy. + * The user can select whether to send HTTP packets through a standard + * "web" HTTP proxy, through a SOCKS 5 proxy, or to not proxy HTTP + * communication. This class does not manage the integrated web browser + * proxy, which is handled in llviewermedia.cpp. + * + * The implementation of HTTP proxying is handled by libcurl. LLProxy + * is responsible for managing the HTTP proxy options and provides a + * thread-safe method to apply those options to a curl request + * (LLProxy::applyProxySettings()). This method is overloaded + * to accommodate the various abstraction libcurl layers that exist + * throughout the viewer (LLCurlEasyRequest, LLCurl::Easy, and CURL). + * + * If you are working with LLCurl or LLCurlEasyRequest objects, + * the configured proxy settings will be applied in the constructors + * of those request handles. If you are working with CURL objects + * directly, you will need to pass the handle of the request to + * applyProxySettings() before issuing the request. + * + * To ensure thread safety, all LLProxy members that relate to the HTTP + * proxy require the LLProxyMutex to be locked before accessing. + */ +class LLProxy: public LLSingleton<LLProxy> +{ + LOG_CLASS(LLProxy); +public: + /*########################################################################################### + METHODS THAT DO NOT LOCK mProxyMutex! + ###########################################################################################*/ + // Constructor, cannot have parameters due to LLSingleton parent class. Call from main thread only. + LLProxy(); + + // Static check for enabled status for UDP packets. Call from main thread only. + static bool isSOCKSProxyEnabled() { return sUDPProxyEnabled; } + + // Get the UDP proxy address and port. Call from main thread only. + LLHost getUDPProxy() const { return mUDPProxy; } + + /*########################################################################################### + END OF NON-LOCKING METHODS + ###########################################################################################*/ + + /*########################################################################################### + METHODS THAT LOCK mProxyMutex! DO NOT CALL WHILE mProxyMutex IS LOCKED! + ###########################################################################################*/ + // Destructor, closes open connections. Do not call directly, use cleanupClass(). + ~LLProxy(); + + // Delete LLProxy singleton. Allows the apr_socket used in the SOCKS 5 control channel to be + // destroyed before the call to apr_terminate. Call from main thread only. + static void cleanupClass(); + + // Apply the current proxy settings to a curl request. Doesn't do anything if mHTTPProxyEnabled is false. + // Safe to call from any thread. + void applyProxySettings(CURL* handle); + void applyProxySettings(LLCurl::Easy* handle); + void applyProxySettings(LLCurlEasyRequest* handle); + + // Start a connection to the SOCKS 5 proxy. Call from main thread only. + S32 startSOCKSProxy(LLHost host); + + // Disconnect and clean up any connection to the SOCKS 5 proxy. Call from main thread only. + void stopSOCKSProxy(); + + // Use Password auth when connecting to the SOCKS proxy. Call from main thread only. + bool setAuthPassword(const std::string &username, const std::string &password); + + // Disable authentication when connecting to the SOCKS proxy. Call from main thread only. + void setAuthNone(); + + // Proxy HTTP packets via httpHost, which can be a SOCKS 5 or a HTTP proxy. + // as specified in type. Call from main thread only. + bool enableHTTPProxy(LLHost httpHost, LLHttpProxyType type); + bool enableHTTPProxy(); + + // Stop proxying HTTP packets. Call from main thread only. + void disableHTTPProxy(); + + /*########################################################################################### + END OF LOCKING METHODS + ###########################################################################################*/ +private: + /*########################################################################################### + METHODS THAT LOCK mProxyMutex! DO NOT CALL WHILE mProxyMutex IS LOCKED! + ###########################################################################################*/ + + // Perform a SOCKS 5 authentication and UDP association with the proxy server. + S32 proxyHandshake(LLHost proxy); + + // Get the currently selected auth method. + LLSocks5AuthType getSelectedAuthMethod() const; + + // Get the currently selected HTTP proxy type + LLHttpProxyType getHTTPProxyType() const; + + std::string getSocksPwd() const; + std::string getSocksUser() const; + + /*########################################################################################### + END OF LOCKING METHODS + ###########################################################################################*/ + +private: + // Is the HTTP proxy enabled? Safe to read in any thread, but do not write directly. + // Instead use enableHTTPProxy() and disableHTTPProxy() instead. + mutable LLAtomic32<bool> mHTTPProxyEnabled; + + // Mutex to protect shared members in non-main thread calls to applyProxySettings(). + mutable LLMutex mProxyMutex; + + /*########################################################################################### + MEMBERS READ AND WRITTEN ONLY IN THE MAIN THREAD. DO NOT SHARE! + ###########################################################################################*/ + + // Is the UDP proxy enabled? + static bool sUDPProxyEnabled; + + // UDP proxy address and port + LLHost mUDPProxy; + // TCP proxy control channel address and port + LLHost mTCPProxy; + + // socket handle to proxy TCP control channel + LLSocket::ptr_t mProxyControlChannel; + + /*########################################################################################### + END OF UNSHARED MEMBERS + ###########################################################################################*/ + + /*########################################################################################### + MEMBERS WRITTEN IN MAIN THREAD AND READ IN ANY THREAD. ONLY READ OR WRITE AFTER LOCKING mProxyMutex! + ###########################################################################################*/ + + // HTTP proxy address and port + LLHost mHTTPProxy; + + // Currently selected HTTP proxy type. Can be web or socks. + LLHttpProxyType mProxyType; + + // SOCKS 5 selected authentication method. + LLSocks5AuthType mAuthMethodSelected; + + // SOCKS 5 username + std::string mSocksUsername; + // SOCKS 5 password + std::string mSocksPassword; + + /*########################################################################################### + END OF SHARED MEMBERS + ###########################################################################################*/ +}; + +#endif diff --git a/indra/llmessage/llpumpio.cpp b/indra/llmessage/llpumpio.cpp index a8d2a0a224..f3ef4f2684 100644 --- a/indra/llmessage/llpumpio.cpp +++ b/indra/llmessage/llpumpio.cpp @@ -195,7 +195,7 @@ bool LLPumpIO::prime(apr_pool_t* pool) return ((pool == NULL) ? false : true); } -bool LLPumpIO::addChain(const chain_t& chain, F32 timeout) +bool LLPumpIO::addChain(const chain_t& chain, F32 timeout, bool has_curl_request) { LLMemType m1(LLMemType::MTYPE_IO_PUMP); if(chain.empty()) return false; @@ -204,8 +204,10 @@ bool LLPumpIO::addChain(const chain_t& chain, F32 timeout) LLScopedLock lock(mChainsMutex); #endif LLChainInfo info; + info.mHasCurlRequest = has_curl_request; info.setTimeoutSeconds(timeout); info.mData = LLIOPipe::buffer_ptr_t(new LLBufferArray); + info.mData->setThreaded(has_curl_request); LLLinkInfo link; #if LL_DEBUG_PIPE_TYPE_IN_PUMP lldebugs << "LLPumpIO::addChain() " << chain[0] << " '" @@ -440,6 +442,15 @@ void LLPumpIO::pump() static LLFastTimer::DeclareTimer FTM_PUMP_IO("Pump IO"); +LLPumpIO::current_chain_t LLPumpIO::removeRunningChain(LLPumpIO::current_chain_t& run_chain) +{ + std::for_each( + (*run_chain).mDescriptors.begin(), + (*run_chain).mDescriptors.end(), + ll_delete_apr_pollset_fd_client_data()); + return mRunningChains.erase(run_chain); +} + //timeout is in microseconds void LLPumpIO::pump(const S32& poll_timeout) { @@ -585,10 +596,16 @@ void LLPumpIO::pump(const S32& poll_timeout) // << (*run_chain).mChainLinks[0].mPipe // << " because we reached the end." << llendl; #endif - run_chain = mRunningChains.erase(run_chain); + run_chain = removeRunningChain(run_chain); continue; } } + else if(isChainExpired(*run_chain)) + { + run_chain = removeRunningChain(run_chain); + continue; + } + PUMP_DEBUG; if((*run_chain).mLock) { @@ -696,11 +713,7 @@ void LLPumpIO::pump(const S32& poll_timeout) PUMP_DEBUG; // This chain is done. Clean up any allocated memory and // erase the chain info. - std::for_each( - (*run_chain).mDescriptors.begin(), - (*run_chain).mDescriptors.end(), - ll_delete_apr_pollset_fd_client_data()); - run_chain = mRunningChains.erase(run_chain); + run_chain = removeRunningChain(run_chain); // *NOTE: may not always need to rebuild the pollset. mRebuildPollset = true; @@ -1095,6 +1108,24 @@ void LLPumpIO::processChain(LLChainInfo& chain) PUMP_DEBUG; } +bool LLPumpIO::isChainExpired(LLChainInfo& chain) +{ + if(!chain.mHasCurlRequest) + { + return false ; + } + + for(links_t::iterator iter = chain.mChainLinks.begin(); iter != chain.mChainLinks.end(); ++iter) + { + if(!(*iter).mPipe->isValid()) + { + return true ; + } + } + + return false ; +} + bool LLPumpIO::handleChainError( LLChainInfo& chain, LLIOPipe::EStatus error) @@ -1136,6 +1167,9 @@ bool LLPumpIO::handleChainError( #endif keep_going = false; break; + case LLIOPipe::STATUS_EXPIRED: + keep_going = false; + break ; default: if(LLIOPipe::isSuccess(error)) { @@ -1157,7 +1191,8 @@ bool LLPumpIO::handleChainError( LLPumpIO::LLChainInfo::LLChainInfo() : mInit(false), mLock(0), - mEOS(false) + mEOS(false), + mHasCurlRequest(false) { LLMemType m1(LLMemType::MTYPE_IO_PUMP); mTimer.setTimerExpirySec(DEFAULT_CHAIN_EXPIRY_SECS); diff --git a/indra/llmessage/llpumpio.h b/indra/llmessage/llpumpio.h index 9303c9d7fc..d2c5d37571 100644 --- a/indra/llmessage/llpumpio.h +++ b/indra/llmessage/llpumpio.h @@ -111,9 +111,10 @@ public: * @param chain The pipes for the chain * @param timeout The number of seconds in the future to * expire. Pass in 0.0f to never expire. + * @param has_curl_request The chain contains LLURLRequest if true. * @return Returns true if anything was added to the pump. */ - bool addChain(const chain_t& chain, F32 timeout); + bool addChain(const chain_t& chain, F32 timeout, bool has_curl_request = false); /** * @brief Struct to associate a pipe with it's buffer io indexes. @@ -356,12 +357,13 @@ protected: // basic member data bool mInit; + bool mEOS; + bool mHasCurlRequest; S32 mLock; LLFrameTimer mTimer; links_t::iterator mHead; links_t mChainLinks; - LLIOPipe::buffer_ptr_t mData; - bool mEOS; + LLIOPipe::buffer_ptr_t mData; LLSD mContext; // tracking inside the pump @@ -402,7 +404,7 @@ protected: protected: void initialize(apr_pool_t* pool); void cleanup(); - + current_chain_t removeRunningChain(current_chain_t& chain) ; /** * @brief Given the internal state of the chains, rebuild the pollset * @see setConditional() @@ -429,6 +431,9 @@ protected: */ bool handleChainError(LLChainInfo& chain, LLIOPipe::EStatus error); + //if the chain is expired, remove it + bool isChainExpired(LLChainInfo& chain) ; + public: /** * @brief Return number of running chains. diff --git a/indra/llmessage/llsdmessage.cpp b/indra/llmessage/llsdmessage.cpp index 9148c9dd15..1c93c12d99 100644 --- a/indra/llmessage/llsdmessage.cpp +++ b/indra/llmessage/llsdmessage.cpp @@ -88,7 +88,7 @@ bool LLSDMessage::httpListener(const LLSD& request) request, url, "POST", reply, error), LLSD(), // headers - timeout); + (F32)timeout); return false; } diff --git a/indra/llmessage/llsdmessagebuilder.cpp b/indra/llmessage/llsdmessagebuilder.cpp index 2698a271ee..615221e0ad 100644 --- a/indra/llmessage/llsdmessagebuilder.cpp +++ b/indra/llmessage/llsdmessagebuilder.cpp @@ -317,7 +317,7 @@ void LLSDMessageBuilder::copyFromMessageData(const LLMsgData& data) // S64 not supported in LLSD so we just truncate it case MVT_S64: - addS32(varname, *(S64*)mvci.getData()); + addS32(varname, (S32)*(S64*)mvci.getData()); break; case MVT_F32: diff --git a/indra/llmessage/llsdmessagereader.cpp b/indra/llmessage/llsdmessagereader.cpp index 304a692cdf..3d8ca2ad9f 100644 --- a/indra/llmessage/llsdmessagereader.cpp +++ b/indra/llmessage/llsdmessagereader.cpp @@ -291,9 +291,10 @@ S32 getElementSize(const LLSD& llsd) case LLSD::TypeMap: case LLSD::TypeArray: case LLSD::TypeUndefined: + default: // TypeLLSDTypeEnd, TypeLLSDNumTypes, etc. return 0; } - return 0; + //return 0; } //virtual diff --git a/indra/llmessage/llsdrpcclient.cpp b/indra/llmessage/llsdrpcclient.cpp index 86fe5c7912..91fd070f07 100644 --- a/indra/llmessage/llsdrpcclient.cpp +++ b/indra/llmessage/llsdrpcclient.cpp @@ -82,6 +82,8 @@ bool LLSDRPCResponse::extractResponse(const LLSD& sd) return rv; } +static LLFastTimer::DeclareTimer FTM_SDRPC_RESPONSE("SDRPC Response"); + // virtual LLIOPipe::EStatus LLSDRPCResponse::process_impl( const LLChannelDescriptors& channels, @@ -90,6 +92,7 @@ LLIOPipe::EStatus LLSDRPCResponse::process_impl( LLSD& context, LLPumpIO* pump) { + LLFastTimer t(FTM_SDRPC_RESPONSE); PUMP_DEBUG; LLMemType m1(LLMemType::MTYPE_IO_SD_CLIENT); if(mIsError) @@ -178,6 +181,8 @@ bool LLSDRPCClient::call( return true; } +static LLFastTimer::DeclareTimer FTM_PROCESS_SDRPC_CLIENT("SDRPC Client"); + // virtual LLIOPipe::EStatus LLSDRPCClient::process_impl( const LLChannelDescriptors& channels, @@ -186,6 +191,7 @@ LLIOPipe::EStatus LLSDRPCClient::process_impl( LLSD& context, LLPumpIO* pump) { + LLFastTimer t(FTM_PROCESS_SDRPC_CLIENT); PUMP_DEBUG; LLMemType m1(LLMemType::MTYPE_IO_SD_CLIENT); if((STATE_NONE == mState) || (!pump)) diff --git a/indra/llmessage/llsdrpcclient.h b/indra/llmessage/llsdrpcclient.h index 9fb49a5c33..0cecf4f688 100644 --- a/indra/llmessage/llsdrpcclient.h +++ b/indra/llmessage/llsdrpcclient.h @@ -240,9 +240,16 @@ public: virtual bool build(LLPumpIO::chain_t& chain, LLSD context) const { lldebugs << "LLSDRPCClientFactory::build" << llendl; - LLIOPipe::ptr_t service(new Client); - chain.push_back(service); LLURLRequest* http(new LLURLRequest(LLURLRequest::HTTP_POST)); + if(!http->isValid()) + { + llwarns << "Creating LLURLRequest failed." << llendl ; + delete http; + return false; + } + + LLIOPipe::ptr_t service(new Client); + chain.push_back(service); LLIOPipe::ptr_t http_pipe(http); http->addHeader("Content-Type: text/llsd"); if(mURL.empty()) @@ -283,9 +290,16 @@ public: virtual bool build(LLPumpIO::chain_t& chain, LLSD context) const { lldebugs << "LLXMLSDRPCClientFactory::build" << llendl; - LLIOPipe::ptr_t service(new Client); - chain.push_back(service); + LLURLRequest* http(new LLURLRequest(LLURLRequest::HTTP_POST)); + if(!http->isValid()) + { + llwarns << "Creating LLURLRequest failed." << llendl ; + delete http; + return false ; + } + LLIOPipe::ptr_t service(new Client); + chain.push_back(service); LLIOPipe::ptr_t http_pipe(http); http->addHeader("Content-Type: text/xml"); if(mURL.empty()) diff --git a/indra/llmessage/llsdrpcserver.cpp b/indra/llmessage/llsdrpcserver.cpp index f87c418fb1..9f776aca72 100644 --- a/indra/llmessage/llsdrpcserver.cpp +++ b/indra/llmessage/llsdrpcserver.cpp @@ -97,6 +97,8 @@ void LLSDRPCServer::clearLock() } } +static LLFastTimer::DeclareTimer FTM_PROCESS_SDRPC_SERVER("SDRPC Server"); + // virtual LLIOPipe::EStatus LLSDRPCServer::process_impl( const LLChannelDescriptors& channels, @@ -105,6 +107,7 @@ LLIOPipe::EStatus LLSDRPCServer::process_impl( LLSD& context, LLPumpIO* pump) { + LLFastTimer t(FTM_PROCESS_SDRPC_SERVER); PUMP_DEBUG; LLMemType m1(LLMemType::MTYPE_IO_SD_SERVER); // lldebugs << "LLSDRPCServer::process_impl" << llendl; diff --git a/indra/llmessage/lltemplatemessagereader.cpp b/indra/llmessage/lltemplatemessagereader.cpp index f470e1b2a5..ab91f74abe 100644 --- a/indra/llmessage/lltemplatemessagereader.cpp +++ b/indra/llmessage/lltemplatemessagereader.cpp @@ -795,7 +795,7 @@ const char* LLTemplateMessageReader::getMessageName() const { if (!mCurrentRMessageTemplate) { - llwarns << "no mCurrentRMessageTemplate" << llendl; + // no message currently being read return ""; } return mCurrentRMessageTemplate->mName; diff --git a/indra/llmessage/llurlrequest.cpp b/indra/llmessage/llurlrequest.cpp index 28bd09fc4c..a16f5c7bf0 100644 --- a/indra/llmessage/llurlrequest.cpp +++ b/indra/llmessage/llurlrequest.cpp @@ -35,6 +35,7 @@ #include "llcurl.h" #include "llioutil.h" #include "llmemtype.h" +#include "llproxy.h" #include "llpumpio.h" #include "llsd.h" #include "llstring.h" @@ -63,7 +64,7 @@ public: ~LLURLRequestDetail(); std::string mURL; LLCurlEasyRequest* mCurlRequest; - LLBufferArray* mResponseBuffer; + LLIOPipe::buffer_ptr_t mResponseBuffer; LLChannelDescriptors mChannels; U8* mLastRead; U32 mBodyLimit; @@ -74,7 +75,6 @@ public: LLURLRequestDetail::LLURLRequestDetail() : mCurlRequest(NULL), - mResponseBuffer(NULL), mLastRead(NULL), mBodyLimit(0), mByteAccumulator(0), @@ -83,13 +83,18 @@ LLURLRequestDetail::LLURLRequestDetail() : { LLMemType m1(LLMemType::MTYPE_IO_URL_REQUEST); mCurlRequest = new LLCurlEasyRequest(); + + if(!mCurlRequest->isValid()) //failed. + { + delete mCurlRequest ; + mCurlRequest = NULL ; + } } LLURLRequestDetail::~LLURLRequestDetail() { LLMemType m1(LLMemType::MTYPE_IO_URL_REQUEST); delete mCurlRequest; - mResponseBuffer = NULL; mLastRead = NULL; } @@ -169,6 +174,7 @@ LLURLRequest::~LLURLRequest() { LLMemType m1(LLMemType::MTYPE_IO_URL_REQUEST); delete mDetail; + mDetail = NULL ; } void LLURLRequest::setURL(const std::string& url) @@ -250,12 +256,24 @@ void LLURLRequest::allowCookies() mDetail->mCurlRequest->setoptString(CURLOPT_COOKIEFILE, ""); } +//virtual +bool LLURLRequest::isValid() +{ + return mDetail->mCurlRequest && mDetail->mCurlRequest->isValid(); +} + // virtual LLIOPipe::EStatus LLURLRequest::handleError( LLIOPipe::EStatus status, LLPumpIO* pump) { LLMemType m1(LLMemType::MTYPE_IO_URL_REQUEST); + + if(!isValid()) + { + return STATUS_EXPIRED ; + } + if(mCompletionCallback && pump) { LLURLRequestComplete* complete = NULL; @@ -270,6 +288,8 @@ LLIOPipe::EStatus LLURLRequest::handleError( return status; } +static LLFastTimer::DeclareTimer FTM_PROCESS_URL_REQUEST("URL Request"); + // virtual LLIOPipe::EStatus LLURLRequest::process_impl( const LLChannelDescriptors& channels, @@ -278,6 +298,7 @@ LLIOPipe::EStatus LLURLRequest::process_impl( LLSD& context, LLPumpIO* pump) { + LLFastTimer t(FTM_PROCESS_URL_REQUEST); PUMP_DEBUG; LLMemType m1(LLMemType::MTYPE_IO_URL_REQUEST); //llinfos << "LLURLRequest::process_impl()" << llendl; @@ -288,6 +309,8 @@ LLIOPipe::EStatus LLURLRequest::process_impl( const S32 MIN_ACCUMULATION = 100000; if(pump && (mDetail->mByteAccumulator > MIN_ACCUMULATION)) { + static LLFastTimer::DeclareTimer FTM_URL_ADJUST_TIMEOUT("Adjust Timeout"); + LLFastTimer t(FTM_URL_ADJUST_TIMEOUT); // This is a pretty sloppy calculation, but this // tries to make the gross assumption that if data // is coming in at 56kb/s, then this transfer will @@ -319,7 +342,7 @@ LLIOPipe::EStatus LLURLRequest::process_impl( // *FIX: bit of a hack, but it should work. The configure and // callback method expect this information to be ready. - mDetail->mResponseBuffer = buffer.get(); + mDetail->mResponseBuffer = buffer; mDetail->mChannels = channels; if(!configure()) { @@ -335,16 +358,33 @@ LLIOPipe::EStatus LLURLRequest::process_impl( { PUMP_DEBUG; LLIOPipe::EStatus status = STATUS_BREAK; - mDetail->mCurlRequest->perform(); + static LLFastTimer::DeclareTimer FTM_URL_PERFORM("Perform"); + { + LLFastTimer t(FTM_URL_PERFORM); + if(!mDetail->mCurlRequest->wait()) + { + return status ; + } + } + while(1) { CURLcode result; - bool newmsg = mDetail->mCurlRequest->getResult(&result); + + static LLFastTimer::DeclareTimer FTM_PROCESS_URL_REQUEST_GET_RESULT("Get Result"); + + bool newmsg = false; + { + LLFastTimer t(FTM_PROCESS_URL_REQUEST_GET_RESULT); + newmsg = mDetail->mCurlRequest->getResult(&result); + } + if(!newmsg) { // keep processing break; } + mState = STATE_HAVE_RESPONSE; context[CONTEXT_REQUEST][CONTEXT_TRANSFERED_BYTES] = mRequestTransferedBytes; @@ -370,7 +410,11 @@ LLIOPipe::EStatus LLURLRequest::process_impl( link.mChannels = LLBufferArray::makeChannelConsumer( channels); chain.push_back(link); - pump->respond(chain, buffer, context); + static LLFastTimer::DeclareTimer FTM_PROCESS_URL_PUMP_RESPOND("Pump Respond"); + { + LLFastTimer t(FTM_PROCESS_URL_PUMP_RESPOND); + pump->respond(chain, buffer, context); + } mCompletionCallback = NULL; } break; @@ -415,6 +459,12 @@ void LLURLRequest::initialize() LLMemType m1(LLMemType::MTYPE_IO_URL_REQUEST); mState = STATE_INITIALIZED; mDetail = new LLURLRequestDetail; + + if(!isValid()) + { + return ; + } + mDetail->mCurlRequest->setopt(CURLOPT_NOSIGNAL, 1); mDetail->mCurlRequest->setWriteCallback(&downCallback, (void*)this); mDetail->mCurlRequest->setReadCallback(&upCallback, (void*)this); @@ -422,8 +472,11 @@ void LLURLRequest::initialize() mResponseTransferedBytes = 0; } +static LLFastTimer::DeclareTimer FTM_URL_REQUEST_CONFIGURE("URL Configure"); bool LLURLRequest::configure() { + LLFastTimer t(FTM_URL_REQUEST_CONFIGURE); + LLMemType m1(LLMemType::MTYPE_IO_URL_REQUEST); bool rv = false; S32 bytes = mDetail->mResponseBuffer->countAfter( @@ -624,6 +677,7 @@ static size_t headerCallback(void* data, size_t size, size_t nmemb, void* user) return header_len; } +static LLFastTimer::DeclareTimer FTM_PROCESS_URL_EXTRACTOR("URL Extractor"); /** * LLContextURLExtractor */ @@ -635,6 +689,7 @@ LLIOPipe::EStatus LLContextURLExtractor::process_impl( LLSD& context, LLPumpIO* pump) { + LLFastTimer t(FTM_PROCESS_URL_EXTRACTOR); PUMP_DEBUG; LLMemType m1(LLMemType::MTYPE_IO_URL_REQUEST); // The destination host is in the context. @@ -713,6 +768,7 @@ void LLURLRequestComplete::responseStatus(LLIOPipe::EStatus status) mRequestStatus = status; } +static LLFastTimer::DeclareTimer FTM_PROCESS_URL_COMPLETE("URL Complete"); // virtual LLIOPipe::EStatus LLURLRequestComplete::process_impl( const LLChannelDescriptors& channels, @@ -721,6 +777,7 @@ LLIOPipe::EStatus LLURLRequestComplete::process_impl( LLSD& context, LLPumpIO* pump) { + LLFastTimer t(FTM_PROCESS_URL_COMPLETE); PUMP_DEBUG; complete(channels, buffer); return STATUS_OK; diff --git a/indra/llmessage/llurlrequest.h b/indra/llmessage/llurlrequest.h index ec5c2c1941..44d358d906 100644 --- a/indra/llmessage/llurlrequest.h +++ b/indra/llmessage/llurlrequest.h @@ -188,6 +188,8 @@ public: */ void allowCookies(); + /*virtual*/ bool isValid() ; + public: /** * @brief Give this pipe a chance to handle a generated error diff --git a/indra/llmessage/llxfer.h b/indra/llmessage/llxfer.h index 989e8b2cab..f9348eb11f 100644 --- a/indra/llmessage/llxfer.h +++ b/indra/llmessage/llxfer.h @@ -29,6 +29,7 @@ #include "message.h" #include "lltimer.h" +#include "llextendedstatus.h" const S32 LL_XFER_LARGE_PAYLOAD = 7680; diff --git a/indra/llmessage/message.cpp b/indra/llmessage/message.cpp index d0b0e178b8..6a425cfe98 100644 --- a/indra/llmessage/message.cpp +++ b/indra/llmessage/message.cpp @@ -3147,7 +3147,7 @@ bool LLMessageSystem::generateDigestForWindowAndUUIDs(char* digest, const S32 wi LL_ERRS("Messaging") << "Trying to generate complex digest on a machine without a shared secret!" << llendl; } - U32 now = time(NULL); + U32 now = (U32)time(NULL); now /= window; @@ -3167,7 +3167,7 @@ bool LLMessageSystem::isMatchingDigestForWindowAndUUIDs(const char* digest, cons } char our_digest[MD5HEX_STR_SIZE]; /* Flawfinder: ignore */ - U32 now = time(NULL); + U32 now = (U32)time(NULL); now /= window; @@ -3213,7 +3213,7 @@ bool LLMessageSystem::generateDigestForWindow(char* digest, const S32 window) co LL_ERRS("Messaging") << "Trying to generate simple digest on a machine without a shared secret!" << llendl; } - U32 now = time(NULL); + U32 now = (U32)time(NULL); now /= window; diff --git a/indra/llmessage/net.cpp b/indra/llmessage/net.cpp index 97611c3b51..85aef5da00 100644 --- a/indra/llmessage/net.cpp +++ b/indra/llmessage/net.cpp @@ -50,7 +50,6 @@ #include "lltimer.h" #include "indra_constants.h" - // Globals #if LL_WINDOWS @@ -174,7 +173,7 @@ U32 ip_string_to_u32(const char* ip_string) // use wildcard addresses. -Ambroff U32 ip = inet_addr(ip_string); if (ip == INADDR_NONE - && strncmp(ip_string, BROADCAST_ADDRESS_STRING, MAXADDRSTR) != 0) + && strncmp(ip_string, BROADCAST_ADDRESS_STRING, MAXADDRSTR) != 0) { llwarns << "ip_string_to_u32() failed, Error: Invalid IP string '" << ip_string << "'" << llendl; return INVALID_HOST_IP_ADDRESS; @@ -188,11 +187,11 @@ U32 ip_string_to_u32(const char* ip_string) ////////////////////////////////////////////////////////////////////////////////////////// #if LL_WINDOWS - + S32 start_net(S32& socket_out, int& nPort) { // Create socket, make non-blocking - // Init WinSock + // Init WinSock int nRet; int hSocket; @@ -201,7 +200,7 @@ S32 start_net(S32& socket_out, int& nPort) int buff_size = 4; // Initialize windows specific stuff - if(WSAStartup(0x0202, &stWSAData)) + if (WSAStartup(0x0202, &stWSAData)) { S32 err = WSAGetLastError(); WSACleanup(); @@ -210,8 +209,8 @@ S32 start_net(S32& socket_out, int& nPort) } // Get a datagram socket - hSocket = (int)socket(AF_INET, SOCK_DGRAM, 0); - if (hSocket == INVALID_SOCKET) + hSocket = (int)socket(AF_INET, SOCK_DGRAM, 0); + if (hSocket == INVALID_SOCKET) { S32 err = WSAGetLastError(); WSACleanup(); @@ -304,7 +303,7 @@ S32 start_net(S32& socket_out, int& nPort) // Setup a destination address stDstAddr.sin_family = AF_INET; stDstAddr.sin_addr.s_addr = INVALID_HOST_IP_ADDRESS; - stDstAddr.sin_port = htons(nPort); + stDstAddr.sin_port = htons(nPort); socket_out = hSocket; return 0; @@ -393,10 +392,10 @@ S32 start_net(S32& socket_out, int& nPort) int rec_size = RECEIVE_BUFFER_SIZE; socklen_t buff_size = 4; - + // Create socket - hSocket = socket(AF_INET, SOCK_DGRAM, 0); - if (hSocket < 0) + hSocket = socket(AF_INET, SOCK_DGRAM, 0); + if (hSocket < 0) { llwarns << "socket() failed" << llendl; return 1; @@ -429,7 +428,7 @@ S32 start_net(S32& socket_out, int& nPort) } else { - // Name the socket (assign the local port number to receive on) + // Name the socket (assign the local port number to receive on) stLclAddr.sin_family = AF_INET; stLclAddr.sin_addr.s_addr = htonl(INADDR_ANY); stLclAddr.sin_port = htons(nPort); @@ -474,7 +473,7 @@ S32 start_net(S32& socket_out, int& nPort) nPort = attempt_port; } // Set socket to be non-blocking - fcntl(hSocket, F_SETFL, O_NONBLOCK); + fcntl(hSocket, F_SETFL, O_NONBLOCK); // set a large receive buffer nRet = setsockopt(hSocket, SOL_SOCKET, SO_RCVBUF, (char *)&rec_size, buff_size); if (nRet) @@ -510,8 +509,8 @@ S32 start_net(S32& socket_out, int& nPort) // Setup a destination address char achMCAddr[MAXADDRSTR] = "127.0.0.1"; /* Flawfinder: ignore */ stDstAddr.sin_family = AF_INET; - stDstAddr.sin_addr.s_addr = ip_string_to_u32(achMCAddr); - stDstAddr.sin_port = htons(nPort); + stDstAddr.sin_addr.s_addr = ip_string_to_u32(achMCAddr); + stDstAddr.sin_port = htons(nPort); socket_out = hSocket; return 0; @@ -537,7 +536,7 @@ static int recvfrom_destip( int socket, void *buf, int len, struct sockaddr *fro iov[0].iov_base = buf; iov[0].iov_len = len; - memset( &msg, 0, sizeof msg ); + memset(&msg, 0, sizeof msg); msg.msg_name = from; msg.msg_namelen = *fromlen; msg.msg_iov = iov; @@ -545,14 +544,14 @@ static int recvfrom_destip( int socket, void *buf, int len, struct sockaddr *fro msg.msg_control = &cmsg; msg.msg_controllen = sizeof(cmsg); - size = recvmsg( socket, &msg, 0 ); + size = recvmsg(socket, &msg, 0); - if( size == -1 ) + if (size == -1) { return -1; } - for( cmsgptr = CMSG_FIRSTHDR(&msg); cmsgptr != NULL; cmsgptr = CMSG_NXTHDR( &msg, cmsgptr ) ) + for (cmsgptr = CMSG_FIRSTHDR(&msg); cmsgptr != NULL; cmsgptr = CMSG_NXTHDR( &msg, cmsgptr)) { if( cmsgptr->cmsg_level == SOL_IP && cmsgptr->cmsg_type == IP_PKTINFO ) { @@ -650,7 +649,7 @@ BOOL send_packet(int hSocket, const char * sendBuffer, int size, U32 recipient, } } } - while ( resend && send_attempts < 3); + while (resend && send_attempts < 3); if (send_attempts >= 3) { diff --git a/indra/llmessage/net.h b/indra/llmessage/net.h index 9f4f5c5821..0f2437479d 100644 --- a/indra/llmessage/net.h +++ b/indra/llmessage/net.h @@ -46,10 +46,10 @@ S32 receive_packet(int hSocket, char * receiveBuffer); BOOL send_packet(int hSocket, const char *sendBuffer, int size, U32 recipient, int nPort); // Returns TRUE on success. //void get_sender(char * tmp); -LLHost get_sender(); +LLHost get_sender(); U32 get_sender_port(); U32 get_sender_ip(void); -LLHost get_receiving_interface(); +LLHost get_receiving_interface(); U32 get_receiving_interface_ip(void); const char* u32_to_ip_string(U32 ip); // Returns pointer to internal string buffer, "(bad IP addr)" on failure, cannot nest calls |