diff options
26 files changed, 1625 insertions, 747 deletions
diff --git a/etc/message.xml b/etc/message.xml index c9b9220ba2..bfbf1b15be 100644 --- a/etc/message.xml +++ b/etc/message.xml @@ -439,6 +439,14 @@ <key>trusted-sender</key> <boolean>false</boolean> </map> + + <key>FetchInventoryDescendents</key> + <map> + <key>flavor</key> + <string>llsd</string> + <key>trusted-sender</key> + <boolean>false</boolean> + </map> <key>GroupProposalBallot</key> <map> diff --git a/indra/llcommon/llsdserialize.cpp b/indra/llcommon/llsdserialize.cpp index 02152cfe79..6f4a49180d 100644 --- a/indra/llcommon/llsdserialize.cpp +++ b/indra/llcommon/llsdserialize.cpp @@ -227,7 +227,8 @@ F64 ll_ntohd(F64 netdouble) * * @param istr The stream to read from. * @param value [out] The string which was found. - * @param max_bytes The maximum possible length of the string. + * @param max_bytes The maximum possible length of the string. Passing in + * a negative value will skip this check. * @return Returns number of bytes read off of the stream. Returns * PARSE_FAILURE (-1) on failure. */ @@ -251,7 +252,8 @@ int deserialize_string_delim(std::istream& istr, std::string& value, char d); * leading the stream. * @param value [out] The string which was found. * @param d The delimiter to use. - * @param max_bytes The maximum possible length of the string. + * @param max_bytes The maximum possible length of the string. Passing in + * a negative value will skip this check. * @return Returns number of bytes read off of the stream. Returns * PARSE_FAILURE (-1) on failure. */ @@ -768,7 +770,7 @@ bool LLSDNotationParser::parseBinary(std::istream& istr, LLSD& data) const // We probably have a valid raw binary stream. determine // the size, and read it. S32 len = strtol(buf + 2, NULL, 0); - if(len > mMaxBytesLeft) return false; + if(mCheckLimits && (len > mMaxBytesLeft)) return false; std::vector<U8> value; if(len) { @@ -1043,7 +1045,7 @@ S32 LLSDBinaryParser::doParse(std::istream& istr, LLSD& data) const U32 size_nbo = 0; read(istr, (char*)&size_nbo, sizeof(U32)); /*Flawfinder: ignore*/ S32 size = (S32)ntohl(size_nbo); - if(size > mMaxBytesLeft) + if(mCheckLimits && (size > mMaxBytesLeft)) { parse_count = PARSE_FAILURE; } @@ -1179,7 +1181,7 @@ bool LLSDBinaryParser::parseString( U32 value_nbo = 0; read(istr, (char*)&value_nbo, sizeof(U32)); /*Flawfinder: ignore*/ S32 size = (S32)ntohl(value_nbo); - if(size > mMaxBytesLeft) return false; + if(mCheckLimits && (size > mMaxBytesLeft)) return false; std::vector<char> buf; if(size) { @@ -1635,7 +1637,7 @@ int deserialize_string_raw( // the size, and read it. // *FIX: This is memory inefficient. S32 len = strtol(buf + 1, NULL, 0); - if(len > max_bytes) return LLSDParser::PARSE_FAILURE; + if((max_bytes>0)&&(len>max_bytes)) return LLSDParser::PARSE_FAILURE; std::vector<char> buf; if(len) { diff --git a/indra/llcommon/llthread.cpp b/indra/llcommon/llthread.cpp index ea877bda84..02ed0dcfc6 100644 --- a/indra/llcommon/llthread.cpp +++ b/indra/llcommon/llthread.cpp @@ -31,6 +31,8 @@ #include "linden_common.h" #include "llapr.h" +#include "apr-1/apr_portable.h" + #include "llthread.h" #include "lltimer.h" @@ -225,6 +227,11 @@ void LLThread::setQuitting() wake(); } +// static +U32 LLThread::currentID() +{ + return (U32)apr_os_thread_current(); +} // static void LLThread::yield() diff --git a/indra/llcommon/llthread.h b/indra/llcommon/llthread.h index 8106c08835..a07c64b8fc 100644 --- a/indra/llcommon/llthread.h +++ b/indra/llcommon/llthread.h @@ -56,14 +56,14 @@ public: virtual ~LLThread(); // Warning! You almost NEVER want to destroy a thread unless it's in the STOPPED state. virtual void shutdown(); // stops the thread - static void yield(); // Static because it can be called by the main thread, which doesn't have an LLThread data structure. - - bool isQuitting() const { return (QUITTING == mStatus); } bool isStopped() const { return (STOPPED == mStatus); } - // PAUSE / RESUME functionality. See source code for important usage notes. + static U32 currentID(); // Return ID of current thread + static void yield(); // Static because it can be called by the main thread, which doesn't have an LLThread data structure. + public: + // PAUSE / RESUME functionality. See source code for important usage notes. // Called from MAIN THREAD. void pause(); void unpause(); @@ -127,7 +127,7 @@ protected: class LLMutex { public: - LLMutex(apr_pool_t *apr_poolp); // Defaults to global pool, could use the thread pool as well. + LLMutex(apr_pool_t *apr_poolp); // NULL pool constructs a new pool for the mutex ~LLMutex(); void lock(); // blocks diff --git a/indra/llcommon/llversionserver.h b/indra/llcommon/llversionserver.h index b7c525be98..b9aa71b4dc 100644 --- a/indra/llcommon/llversionserver.h +++ b/indra/llcommon/llversionserver.h @@ -35,7 +35,7 @@ const S32 LL_VERSION_MAJOR = 1; const S32 LL_VERSION_MINOR = 19; const S32 LL_VERSION_PATCH = 1; -const S32 LL_VERSION_BUILD = 80264; +const S32 LL_VERSION_BUILD = 80913; const char * const LL_CHANNEL = "Second Life Server"; diff --git a/indra/llcrashlogger/llcrashlogger.cpp b/indra/llcrashlogger/llcrashlogger.cpp index 9ea0cef7be..c8b5f06b49 100755 --- a/indra/llcrashlogger/llcrashlogger.cpp +++ b/indra/llcrashlogger/llcrashlogger.cpp @@ -130,14 +130,14 @@ void LLCrashLogger::gatherFiles() LLSDSerialize::fromXML(mDebugLog, debug_log_file); mFileMap["SecondLifeLog"] = mDebugLog["SLLog"].asString(); mFileMap["SettingsXml"] = mDebugLog["SettingsFilename"].asString(); - LLHTTPClient::setCABundle(mDebugLog["CAFilename"].asString()); + LLCurl::setCAFile(mDebugLog["CAFilename"].asString()); llinfos << "Using log file from debug log " << mFileMap["SecondLifeLog"] << llendl; llinfos << "Using settings file from debug log " << mFileMap["SettingsXml"] << llendl; } else { // Figure out the filename of the second life log - LLHTTPClient::setCABundle(gDirUtilp->getCAFile()); + LLCurl::setCAFile(gDirUtilp->getCAFile()); mFileMap["SecondLifeLog"] = gDirUtilp->getExpandedFilename(LL_PATH_LOGS,"SecondLife.log"); mFileMap["SettingsXml"] = gDirUtilp->getExpandedFilename(LL_PATH_USER_SETTINGS,"settings.xml"); } diff --git a/indra/llinventory/llinventory.cpp b/indra/llinventory/llinventory.cpp index 4d411e6b8d..272e8ffba2 100644 --- a/indra/llinventory/llinventory.cpp +++ b/indra/llinventory/llinventory.cpp @@ -54,6 +54,7 @@ static const std::string INV_INVENTORY_TYPE_LABEL("inv_type"); static const std::string INV_NAME_LABEL("name"); static const std::string INV_DESC_LABEL("desc"); static const std::string INV_PERMISSIONS_LABEL("permissions"); +static const std::string INV_SHADOW_ID_LABEL("shadow_id"); static const std::string INV_ASSET_ID_LABEL("asset_id"); static const std::string INV_SALE_INFO_LABEL("sale_info"); static const std::string INV_FLAGS_LABEL("flags"); @@ -927,34 +928,34 @@ BOOL LLInventoryItem::exportLegacyStream(std::ostream& output_stream, BOOL inclu LLSD LLInventoryItem::asLLSD() const { LLSD sd = LLSD(); - sd["item_id"] = mUUID; - sd["parent_id"] = mParentUUID; - sd["permissions"] = ll_create_sd_from_permissions(mPermissions); + sd[INV_ITEM_ID_LABEL] = mUUID; + sd[INV_PARENT_ID_LABEL] = mParentUUID; + sd[INV_PERMISSIONS_LABEL] = ll_create_sd_from_permissions(mPermissions); U32 mask = mPermissions.getMaskBase(); if(((mask & PERM_ITEM_UNRESTRICTED) == PERM_ITEM_UNRESTRICTED) || (mAssetUUID.isNull())) { - sd["asset_id"] = mAssetUUID; + sd[INV_ASSET_ID_LABEL] = mAssetUUID; } else { LLUUID shadow_id(mAssetUUID); LLXORCipher cipher(MAGIC_ID.mData, UUID_BYTES); cipher.encrypt(shadow_id.mData, UUID_BYTES); - sd["shadow_id"] = shadow_id; + sd[INV_SHADOW_ID_LABEL] = shadow_id; } - sd["type"] = LLAssetType::lookup(mType); + sd[INV_ASSET_TYPE_LABEL] = LLAssetType::lookup(mType); const char* inv_type_str = LLInventoryType::lookup(mInventoryType); if(inv_type_str) { - sd["inv_type"] = inv_type_str; + sd[INV_INVENTORY_TYPE_LABEL] = inv_type_str; } - sd["flags"] = ll_sd_from_U32(mFlags); - sd["sale_info"] = mSaleInfo; - sd["name"] = mName; - sd["desc"] = mDescription; - sd["creation_date"] = mCreationDate; + sd[INV_FLAGS_LABEL] = ll_sd_from_U32(mFlags); + sd[INV_SALE_INFO_LABEL] = mSaleInfo; + sd[INV_NAME_LABEL] = mName; + sd[INV_DESC_LABEL] = mDescription; + sd[INV_CREATION_DATE_LABEL] = mCreationDate; return sd; } @@ -1007,7 +1008,7 @@ bool LLInventoryItem::fromLLSD(LLSD& sd) mPermissions.setMaskNext(perm_mask); } } - w = "shadow_id"; + w = INV_SHADOW_ID_LABEL; if (sd.has(w)) { mAssetUUID = sd[w]; @@ -1357,6 +1358,19 @@ void LLInventoryCategory::setPreferredType(LLAssetType::EType type) mPreferredType = type; } +LLSD LLInventoryCategory::asLLSD() const +{ + LLSD sd = LLSD(); + sd["item_id"] = mUUID; + sd["parent_id"] = mParentUUID; + S8 type = static_cast<S8>(mPreferredType); + sd["type"] = type; + sd["name"] = mName; + + return sd; +} + + // virtual void LLInventoryCategory::packMessage(LLMessageSystem* msg) const { @@ -1367,6 +1381,36 @@ void LLInventoryCategory::packMessage(LLMessageSystem* msg) const msg->addStringFast(_PREHASH_Name, mName); } +bool LLInventoryCategory::fromLLSD(LLSD& sd) +{ + std::string w; + + w = INV_ITEM_ID_LABEL; + if (sd.has(w)) + { + mUUID = sd[w]; + } + w = INV_PARENT_ID_LABEL; + if (sd.has(w)) + { + mParentUUID = sd[w]; + } + w = INV_ASSET_TYPE_LABEL; + if (sd.has(w)) + { + S8 type = (U8)sd[w].asInteger(); + mPreferredType = static_cast<LLAssetType::EType>(type); + } + w = INV_NAME_LABEL; + if (sd.has(w)) + { + mName = sd[w].asString(); + LLString::replaceNonstandardASCII(mName, ' '); + LLString::replaceChar(mName, '|', ' '); + } + return true; +} + // virtual void LLInventoryCategory::unpackMessage(LLMessageSystem* msg, const char* block, diff --git a/indra/llinventory/llinventory.h b/indra/llinventory/llinventory.h index 5ff7a1e72b..db4843d8b5 100644 --- a/indra/llinventory/llinventory.h +++ b/indra/llinventory/llinventory.h @@ -230,7 +230,6 @@ public: // network ok. It uses a simple crc check which is defeatable, but // we want to detect network mangling somehow. virtual BOOL unpackMessage(LLMessageSystem* msg, const char* block, S32 block_num = 0); - // file support virtual BOOL importFile(FILE* fp); virtual BOOL exportFile(FILE* fp, BOOL include_asset_key = TRUE) const; @@ -288,6 +287,9 @@ public: virtual void packMessage(LLMessageSystem* msg) const; virtual void unpackMessage(LLMessageSystem* msg, const char* block, S32 block_num = 0); + LLSD asLLSD() const; + bool fromLLSD(LLSD& sd); + // file support virtual BOOL importFile(FILE* fp); virtual BOOL exportFile(FILE* fp, BOOL include_asset_key = TRUE) const; diff --git a/indra/llinventory/llpermissions.cpp b/indra/llinventory/llpermissions.cpp index 8f9f73d0bd..f816d54181 100644 --- a/indra/llinventory/llpermissions.cpp +++ b/indra/llinventory/llpermissions.cpp @@ -37,6 +37,7 @@ // library includes #include "message.h" #include "metapropertyt.h" +#include "llsd.h" ///---------------------------------------------------------------------------- /// Class LLPermissions @@ -473,6 +474,25 @@ BOOL LLPermissions::allowOperationBy(PermissionBit op, const LLUUID& requester, } // +// LLSD support for HTTP messages. +// +LLSD LLPermissions::packMessage() const +{ + LLSD result; + result["creator-id"] = mCreator; + result["owner-id"] = mOwner; + result["group-id"] = mGroup; + + result["base-mask"] = (S32)mMaskBase; + result["owner-mask"] = (S32)mMaskOwner; + result["group-mask"] = (S32)mMaskGroup; + result["everyone-mask"] = (S32)mMaskEveryone; + result["next-owner-mask"]= (S32)mMaskNextOwner; + result["group-owned"] = (BOOL)mIsGroupOwned; + return result; +} + +// // Messaging support // void LLPermissions::packMessage(LLMessageSystem* msg) const @@ -489,6 +509,19 @@ void LLPermissions::packMessage(LLMessageSystem* msg) const msg->addBOOLFast(_PREHASH_GroupOwned, (BOOL)mIsGroupOwned); } +void LLPermissions::unpackMessage(LLSD perms) +{ + mCreator = perms["creator-id"]; + mOwner = perms["owner-id"]; + mGroup = perms["group-id"]; + + mMaskBase = (U32)perms["base-mask"].asInteger(); + mMaskOwner = (U32)perms["owner-mask"].asInteger(); + mMaskGroup = (U32)perms["group-mask"].asInteger(); + mMaskEveryone = (U32)perms["everyone-mask"].asInteger(); + mMaskNextOwner = (U32)perms["next-owner-mask"].asInteger(); + mIsGroupOwned = perms["group-owned"].asBoolean(); +} void LLPermissions::unpackMessage(LLMessageSystem* msg, const char* block, S32 block_num) { diff --git a/indra/llinventory/llpermissions.h b/indra/llinventory/llpermissions.h index 9370d6480b..36acc438be 100644 --- a/indra/llinventory/llpermissions.h +++ b/indra/llinventory/llpermissions.h @@ -294,6 +294,9 @@ public: // MISC METHODS and OPERATORS // + LLSD packMessage() const; + void unpackMessage(LLSD perms); + // For messaging system support void packMessage(LLMessageSystem* msg) const; void unpackMessage(LLMessageSystem* msg, const char* block, S32 block_num = 0); diff --git a/indra/llinventory/llsaleinfo.cpp b/indra/llinventory/llsaleinfo.cpp index d0c7c728f3..c268544955 100644 --- a/indra/llinventory/llsaleinfo.cpp +++ b/indra/llinventory/llsaleinfo.cpp @@ -280,6 +280,17 @@ void LLSaleInfo::setSalePrice(S32 price) mSalePrice = llclamp(mSalePrice, 0, S32_MAX); } +LLSD LLSaleInfo::packMessage() const +{ + LLSD result; + + U8 sale_type = static_cast<U8>(mSaleType); + result["sale-type"] = (U8)sale_type; + result["sale-price"] = (S32)mSalePrice; + //result[_PREHASH_NextOwnerMask] = mNextOwnerPermMask; + return result; +} + void LLSaleInfo::packMessage(LLMessageSystem* msg) const { U8 sale_type = static_cast<U8>(mSaleType); @@ -288,6 +299,16 @@ void LLSaleInfo::packMessage(LLMessageSystem* msg) const //msg->addU32Fast(_PREHASH_NextOwnerMask, mNextOwnerPermMask); } +void LLSaleInfo::unpackMessage(LLSD sales) +{ + U8 sale_type = (U8)sales["sale-type"].asInteger(); + mSaleType = static_cast<EForSale>(sale_type); + + mSalePrice = (S32)sales["sale-price"].asInteger(); + mSalePrice = llclamp(mSalePrice, 0, S32_MAX); + //msg->getU32Fast(block, _PREHASH_NextOwnerMask, mNextOwnerPermMask); +} + void LLSaleInfo::unpackMessage(LLMessageSystem* msg, const char* block) { U8 sale_type; diff --git a/indra/llinventory/llsaleinfo.h b/indra/llinventory/llsaleinfo.h index e22466bfa8..1c9db6e346 100644 --- a/indra/llinventory/llsaleinfo.h +++ b/indra/llinventory/llsaleinfo.h @@ -103,6 +103,9 @@ public: LLXMLNode *exportFileXML() const; BOOL importXML(LLXMLNode* node); + LLSD packMessage() const; + void unpackMessage(LLSD sales); + // message serialization void packMessage(LLMessageSystem* msg) const; void unpackMessage(LLMessageSystem* msg, const char* block); diff --git a/indra/llmessage/llcurl.cpp b/indra/llmessage/llcurl.cpp index 9b5e6cd4e6..8b9a45ff3f 100644 --- a/indra/llmessage/llcurl.cpp +++ b/indra/llmessage/llcurl.cpp @@ -2,7 +2,7 @@ * @file llcurl.h * @author Zero / Donovan * @date 2006-10-15 - * @brief Curl wrapper + * @brief Implementation of wrapper around libcurl. * * $LicenseInfo:firstyear=2006&license=viewergpl$ * @@ -31,13 +31,29 @@ * $/LicenseInfo$ */ +#if LL_WINDOWS +#define SAFE_SSL 1 +#elif LL_DARWIN +#define SAFE_SSL 1 +#else +#define SAFE_SSL 1 +#endif + #include "linden_common.h" #include "llcurl.h" +#include <algorithm> #include <iomanip> +#include <curl/curl.h> +#if SAFE_SSL +#include <openssl/crypto.h> +#endif +#include "llbufferstream.h" +#include "llstl.h" #include "llsdserialize.h" +#include "llthread.h" ////////////////////////////////////////////////////////////////////////////// /* @@ -55,40 +71,112 @@ do this. */ -using namespace std; - +////////////////////////////////////////////////////////////////////////////// + +static const S32 EASY_HANDLE_POOL_SIZE = 5; +static const S32 MULTI_PERFORM_CALL_REPEAT = 5; +static const S32 CURL_REQUEST_TIMEOUT = 30; // seconds +static const S32 MAX_ACTIVE_REQUEST_COUNT = 100; + +// DEBUG // +S32 gCurlEasyCount = 0; +S32 gCurlMultiCount = 0; + +////////////////////////////////////////////////////////////////////////////// + +//static +std::vector<LLMutex*> LLCurl::sSSLMutex; +std::string LLCurl::sCAPath; +std::string LLCurl::sCAFile; + +//static +void LLCurl::setCAPath(const std::string& path) +{ + sCAPath = path; +} + +//static +void LLCurl::setCAFile(const std::string& file) +{ + sCAFile = file; +} + +////////////////////////////////////////////////////////////////////////////// + LLCurl::Responder::Responder() : mReferenceCount(0) { } + LLCurl::Responder::~Responder() { } // virtual -void LLCurl::Responder::error(U32 status, const std::stringstream& content) +void LLCurl::Responder::error(U32 status, const std::string& reason) { - llinfos << "LLCurl::Responder::error " << status << ": " << content.str() << llendl; + llinfos << status << ": " << reason << llendl; } // virtual -void LLCurl::Responder::result(const std::stringstream& content) +void LLCurl::Responder::result(const LLSD& content) { + llwarns << "Virtual Function not implemented" << llendl; } // virtual -void LLCurl::Responder::completed(U32 status, const std::stringstream& content) +void LLCurl::Responder::completedRaw(U32 status, const std::string& reason, + const LLChannelDescriptors& channels, + const LLIOPipe::buffer_ptr_t& buffer) { - if (200 <= status && status < 300) + if (isGoodStatus(status)) + { + LLSD content; + LLBufferStream istr(channels, buffer.get()); + LLSDSerialize::fromXML(content, istr); +/* + const S32 parseError = -1; + if(LLSDSerialize::fromXML(content, istr) == parseError) + { + mStatus = 498; + mReason = "Client Parse Error"; + } +*/ + completed(status, reason, content); + } + else if (status == 400) + { + // Get reason from buffer + char tbuf[4096]; + S32 len = 4096; + buffer->readAfter(channels.in(), NULL, (U8*)tbuf, len); + tbuf[len] = 0; + completed(status, std::string(tbuf), LLSD()); + } + else + { + completed(status, reason, LLSD()); + } +} + +// virtual +void LLCurl::Responder::completed(U32 status, const std::string& reason, const LLSD& content) +{ + if (isGoodStatus(status)) { result(content); } else { - error(status, content); + error(status, reason); } } +//virtual +void LLCurl::Responder::completedHeader(U32 status, const std::string& reason, const LLSD& content) +{ + +} namespace boost { @@ -106,226 +194,456 @@ namespace boost } }; + ////////////////////////////////////////////////////////////////////////////// -size_t -curlOutputCallback(void* data, size_t size, size_t nmemb, void* user_data) + +class LLCurl::Easy { - stringstream& output = *(stringstream*)user_data; + LOG_CLASS(Easy); + +private: + Easy(); - size_t n = size * nmemb; - output.write((const char*)data, n); - if (!((istream&)output).good()) { - std::cerr << "WHAT!?!?!? istream side bad" << std::endl; - } - if (!((ostream&)output).good()) { - std::cerr << "WHAT!?!?!? ostream side bad" << std::endl; - } +public: + static Easy* getEasy(); + ~Easy(); - return n; -} + CURL* getCurlHandle() const { return mCurlEasyHandle; } -// Only used if request contained a body (post or put), Not currently implemented. -// size_t -// curlRequestCallback(void* data, size_t size, size_t nmemb, void* user_data) -// { -// stringstream& request = *(stringstream*)user_data; + void setErrorBuffer(); + void setCA(); -// size_t n = size * nmemb; -// request.read((char*)data, n); -// return request.gcount(); -// } - + void setopt(CURLoption option, S32 value); + // These assume the setter does not free value! + void setopt(CURLoption option, void* value); + void setopt(CURLoption option, char* value); + // Copies the string so that it is gauranteed to stick around + void setoptString(CURLoption option, const std::string& value); + + void slist_append(const char* str); + void setHeaders(); + + U32 report(CURLcode); + void getTransferInfo(LLCurl::TransferInfo* info); + void prepRequest(const std::string& url, ResponderPtr, bool post = false); + + const char* getErrorBuffer(); + std::stringstream& getInput() { return mInput; } + std::stringstream& getHeaderOutput() { return mHeaderOutput; } + LLIOPipe::buffer_ptr_t& getOutput() { return mOutput; } + const LLChannelDescriptors& getChannels() { return mChannels; } + + void resetState(); +private: + CURL* mCurlEasyHandle; + struct curl_slist* mHeaders; + + std::stringstream mRequest; + LLChannelDescriptors mChannels; + LLIOPipe::buffer_ptr_t mOutput; + std::stringstream mInput; + std::stringstream mHeaderOutput; + char mErrorBuffer[CURL_ERROR_SIZE]; + + // Note: char*'s not strings since we pass pointers to curl + std::vector<char*> mStrings; + + ResponderPtr mResponder; +}; LLCurl::Easy::Easy() + : mHeaders(NULL), + mCurlEasyHandle(NULL) { - mHeaders = 0; - mHeaders = curl_slist_append(mHeaders, "Connection: keep-alive"); - mHeaders = curl_slist_append(mHeaders, "Keep-alive: 300"); - mHeaders = curl_slist_append(mHeaders, "Content-Type: application/xml"); - // FIXME: shouldn't be there for GET/DELETE - // FIXME: should have ACCEPT headers - - mHandle = curl_easy_init(); + mErrorBuffer[0] = 0; +} + +LLCurl::Easy* LLCurl::Easy::getEasy() +{ + Easy* easy = new Easy(); + easy->mCurlEasyHandle = curl_easy_init(); + if (!easy->mCurlEasyHandle) + { + // this can happen if we have too many open files (fails in c-ares/ares_init.c) + llwarns << "curl_multi_init() returned NULL! Easy handles: " << gCurlEasyCount << " Multi handles: " << gCurlMultiCount << llendl; + delete easy; + return NULL; + } + ++gCurlEasyCount; + return easy; } LLCurl::Easy::~Easy() { - curl_easy_cleanup(mHandle); + curl_easy_cleanup(mCurlEasyHandle); + --gCurlEasyCount; curl_slist_free_all(mHeaders); + for_each(mStrings.begin(), mStrings.end(), DeletePointer()); } -void -LLCurl::Easy::get(const string& url, ResponderPtr responder) +void LLCurl::Easy::resetState() { - prep(url, responder); - curl_easy_setopt(mHandle, CURLOPT_HTTPGET, 1); + curl_easy_reset(mCurlEasyHandle); + + if (mHeaders) + { + curl_slist_free_all(mHeaders); + mHeaders = NULL; + } + + mRequest.str(""); + mRequest.clear(); + + mOutput.reset(); + + mInput.str(""); + mInput.clear(); + + mErrorBuffer[0] = 0; + + mHeaderOutput.str(""); + mHeaderOutput.clear(); } -void -LLCurl::Easy::getByteRange(const string& url, S32 offset, S32 length, ResponderPtr responder) +void LLCurl::Easy::setErrorBuffer() { - mRange = llformat("Range: bytes=%d-%d", offset,offset+length-1); - mHeaders = curl_slist_append(mHeaders, mRange.c_str()); - prep(url, responder); - curl_easy_setopt(mHandle, CURLOPT_HTTPGET, 1); + setopt(CURLOPT_ERRORBUFFER, &mErrorBuffer); } -void -LLCurl::Easy::perform() +const char* LLCurl::Easy::getErrorBuffer() { - report(curl_easy_perform(mHandle)); + return mErrorBuffer; } -void -LLCurl::Easy::prep(const std::string& url, ResponderPtr responder) +void LLCurl::Easy::setCA() { -#if !LL_DARWIN - curl_easy_reset(mHandle); // SJB: doesn't exisit on OSX 10.3.9 -#else - // SJB: equivalent? fast? - curl_easy_cleanup(mHandle); - mHandle = curl_easy_init(); -#endif - - curl_easy_setopt(mHandle, CURLOPT_PRIVATE, this); - -// curl_easy_setopt(mHandle, CURLOPT_VERBOSE, 1); // usefull for debugging - curl_easy_setopt(mHandle, CURLOPT_NOSIGNAL, 1); - curl_easy_setopt(mHandle, CURLOPT_WRITEFUNCTION, &curlOutputCallback); - curl_easy_setopt(mHandle, CURLOPT_WRITEDATA, &mOutput); -#if 1 // For debug - curl_easy_setopt(mHandle, CURLOPT_HEADERFUNCTION, &curlOutputCallback); - curl_easy_setopt(mHandle, CURLOPT_HEADERDATA, &mHeaderOutput); -#endif - curl_easy_setopt(mHandle, CURLOPT_ERRORBUFFER, &mErrorBuffer); - curl_easy_setopt(mHandle, CURLOPT_ENCODING, ""); - curl_easy_setopt(mHandle, CURLOPT_SSL_VERIFYPEER, false); - curl_easy_setopt(mHandle, CURLOPT_HTTPHEADER, mHeaders); - - mOutput.str(""); - if (!((istream&)mOutput).good()) { - std::cerr << "WHAT!?!?!? istream side bad" << std::endl; + if(!sCAPath.empty()) + { + setoptString(CURLOPT_CAPATH, sCAPath); } - if (!((ostream&)mOutput).good()) { - std::cerr << "WHAT!?!?!? ostream side bad" << std::endl; + if(!sCAFile.empty()) + { + setoptString(CURLOPT_CAINFO, sCAFile); } +} - mURL = url; - curl_easy_setopt(mHandle, CURLOPT_URL, mURL.c_str()); +void LLCurl::Easy::setHeaders() +{ + setopt(CURLOPT_HTTPHEADER, mHeaders); +} - mResponder = responder; +void LLCurl::Easy::getTransferInfo(LLCurl::TransferInfo* info) +{ + curl_easy_getinfo(mCurlEasyHandle, CURLINFO_SIZE_DOWNLOAD, &info->mSizeDownload); + curl_easy_getinfo(mCurlEasyHandle, CURLINFO_TOTAL_TIME, &info->mTotalTime); + curl_easy_getinfo(mCurlEasyHandle, CURLINFO_SPEED_DOWNLOAD, &info->mSpeedDownload); } -void -LLCurl::Easy::report(CURLcode code) +U32 LLCurl::Easy::report(CURLcode code) { - if (!mResponder) return; - - long responseCode; + U32 responseCode = 0; + std::string responseReason; if (code == CURLE_OK) { - curl_easy_getinfo(mHandle, CURLINFO_RESPONSE_CODE, &responseCode); + curl_easy_getinfo(mCurlEasyHandle, CURLINFO_RESPONSE_CODE, &responseCode); + //*TODO: get reason from first line of mHeaderOutput } else { responseCode = 499; + responseReason = strerror(code) + " : " + mErrorBuffer; + } + + if (mResponder) + { + mResponder->completedRaw(responseCode, responseReason, mChannels, mOutput); + mResponder = NULL; } - mResponder->completed(responseCode, mOutput); - mResponder = NULL; + resetState(); + return responseCode; } +// Note: these all assume the caller tracks the value (i.e. keeps it persistant) +void LLCurl::Easy::setopt(CURLoption option, S32 value) +{ + curl_easy_setopt(mCurlEasyHandle, option, value); +} +void LLCurl::Easy::setopt(CURLoption option, void* value) +{ + curl_easy_setopt(mCurlEasyHandle, option, value); +} +void LLCurl::Easy::setopt(CURLoption option, char* value) +{ + curl_easy_setopt(mCurlEasyHandle, option, value); +} +// Note: this copies the string so that the caller does not have to keep it around +void LLCurl::Easy::setoptString(CURLoption option, const std::string& value) +{ + char* tstring = new char[value.length()+1]; + strcpy(tstring, value.c_str()); + mStrings.push_back(tstring); + curl_easy_setopt(mCurlEasyHandle, option, tstring); +} +void LLCurl::Easy::slist_append(const char* str) +{ + mHeaders = curl_slist_append(mHeaders, str); +} -LLCurl::Multi::Multi() +size_t curlReadCallback(char* data, size_t size, size_t nmemb, void* user_data) { - mHandle = curl_multi_init(); + LLCurl::Easy* easy = (LLCurl::Easy*)user_data; + + S32 n = size * nmemb; + S32 startpos = easy->getInput().tellg(); + easy->getInput().seekg(0, std::ios::end); + S32 endpos = easy->getInput().tellg(); + easy->getInput().seekg(startpos, std::ios::beg); + S32 maxn = endpos - startpos; + n = llmin(n, maxn); + easy->getInput().read((char*)data, n); + + return n; } -LLCurl::Multi::~Multi() +size_t curlWriteCallback(char* data, size_t size, size_t nmemb, void* user_data) { - // FIXME: should clean up excess handles in mFreeEasy - curl_multi_cleanup(mHandle); + LLCurl::Easy* easy = (LLCurl::Easy*)user_data; + + S32 n = size * nmemb; + easy->getOutput()->append(easy->getChannels().in(), (const U8*)data, n); + + return n; } +size_t curlHeaderCallback(void* data, size_t size, size_t nmemb, void* user_data) +{ + LLCurl::Easy* easy = (LLCurl::Easy*)user_data; + + size_t n = size * nmemb; + easy->getHeaderOutput().write((const char*)data, n); -void -LLCurl::Multi::get(const std::string& url, ResponderPtr responder) + return n; +} + +void LLCurl::Easy::prepRequest(const std::string& url, ResponderPtr responder, bool post) { - LLCurl::Easy* easy = easyAlloc(); - easy->get(url, responder); - curl_multi_add_handle(mHandle, easy->mHandle); + resetState(); + + if (post) setoptString(CURLOPT_ENCODING, ""); + +// setopt(CURLOPT_VERBOSE, 1); // usefull for debugging + setopt(CURLOPT_NOSIGNAL, 1); + + mOutput.reset(new LLBufferArray); + setopt(CURLOPT_WRITEFUNCTION, (void*)&curlWriteCallback); + setopt(CURLOPT_WRITEDATA, (void*)this); + + setopt(CURLOPT_READFUNCTION, (void*)&curlReadCallback); + setopt(CURLOPT_READDATA, (void*)this); + + setopt(CURLOPT_HEADERFUNCTION, (void*)&curlHeaderCallback); + setopt(CURLOPT_HEADERDATA, (void*)this); + + setErrorBuffer(); + setCA(); + + setopt(CURLOPT_SSL_VERIFYPEER, true); + setopt(CURLOPT_TIMEOUT, CURL_REQUEST_TIMEOUT); + + setoptString(CURLOPT_URL, url); + + mResponder = responder; + + if (!post) + { + slist_append("Connection: keep-alive"); + slist_append("Keep-alive: 300"); + } + // *FIX: should have ACCEPT headers } + +//////////////////////////////////////////////////////////////////////////// + +class LLCurl::Multi +{ + LOG_CLASS(Multi); +public: + + Multi(); + ~Multi(); + + Easy* allocEasy(); + bool addEasy(Easy* easy); -void -LLCurl::Multi::getByteRange(const std::string& url, S32 offset, S32 length, ResponderPtr responder) + void removeEasy(Easy* easy); + + S32 process(); + S32 perform(); + + CURLMsg* info_read(S32* msgs_in_queue); + + S32 mQueued; + S32 mErrorCount; + +private: + void easyFree(Easy*); + + CURLM* mCurlMultiHandle; + + typedef std::set<Easy*> easy_active_list_t; + easy_active_list_t mEasyActiveList; + typedef std::map<CURL*, Easy*> easy_active_map_t; + easy_active_map_t mEasyActiveMap; + typedef std::set<Easy*> easy_free_list_t; + easy_free_list_t mEasyFreeList; +}; + +LLCurl::Multi::Multi() + : mQueued(0), + mErrorCount(0) { - LLCurl::Easy* easy = easyAlloc(); - easy->getByteRange(url, offset, length, responder); - curl_multi_add_handle(mHandle, easy->mHandle); + mCurlMultiHandle = curl_multi_init(); + if (!mCurlMultiHandle) + { + llwarns << "curl_multi_init() returned NULL! Easy handles: " << gCurlEasyCount << " Multi handles: " << gCurlMultiCount << llendl; + mCurlMultiHandle = curl_multi_init(); + } + llassert_always(mCurlMultiHandle); + ++gCurlMultiCount; } + +LLCurl::Multi::~Multi() +{ + // Clean up active + for(easy_active_list_t::iterator iter = mEasyActiveList.begin(); + iter != mEasyActiveList.end(); ++iter) + { + Easy* easy = *iter; + curl_multi_remove_handle(mCurlMultiHandle, easy->getCurlHandle()); + delete easy; + } + mEasyActiveList.clear(); + mEasyActiveMap.clear(); -void -LLCurl::Multi::process() + // Clean up freed + for_each(mEasyFreeList.begin(), mEasyFreeList.end(), DeletePointer()); + mEasyFreeList.clear(); + + curl_multi_cleanup(mCurlMultiHandle); + --gCurlMultiCount; +} + +CURLMsg* LLCurl::Multi::info_read(S32* msgs_in_queue) { - int count; - for (int call_count = 0; call_count < 5; call_count += 1) + CURLMsg* curlmsg = curl_multi_info_read(mCurlMultiHandle, msgs_in_queue); + return curlmsg; +} + + +S32 LLCurl::Multi::perform() +{ + S32 q = 0; + for (S32 call_count = 0; + call_count < MULTI_PERFORM_CALL_REPEAT; + call_count += 1) { - if (CURLM_CALL_MULTI_PERFORM != curl_multi_perform(mHandle, &count)) + CURLMcode code = curl_multi_perform(mCurlMultiHandle, &q); + if (CURLM_CALL_MULTI_PERFORM != code || q == 0) { break; } } - + mQueued = q; + return q; +} + +S32 LLCurl::Multi::process() +{ + perform(); + CURLMsg* msg; int msgs_in_queue; - while ((msg = curl_multi_info_read(mHandle, &msgs_in_queue))) + + S32 processed = 0; + while ((msg = info_read(&msgs_in_queue))) { - if (msg->msg != CURLMSG_DONE) continue; - Easy* easy = 0; - curl_easy_getinfo(msg->easy_handle, CURLINFO_PRIVATE, &easy); - if (!easy) continue; - easy->report(msg->data.result); - - curl_multi_remove_handle(mHandle, easy->mHandle); - easyFree(easy); + ++processed; + if (msg->msg == CURLMSG_DONE) + { + U32 response = 0; + easy_active_map_t::iterator iter = mEasyActiveMap.find(msg->easy_handle); + if (iter != mEasyActiveMap.end()) + { + Easy* easy = iter->second; + response = easy->report(msg->data.result); + removeEasy(easy); + } + else + { + response = 499; + //*TODO: change to llwarns + llerrs << "cleaned up curl request completed!" << llendl; + } + if (response >= 400) + { + // failure of some sort, inc mErrorCount for debugging and flagging multi for destruction + ++mErrorCount; + } + } } + return processed; } - - -LLCurl::Easy* -LLCurl::Multi::easyAlloc() +LLCurl::Easy* LLCurl::Multi::allocEasy() { Easy* easy = 0; - - if (mFreeEasy.empty()) + + if (mEasyFreeList.empty()) { - easy = new Easy(); + easy = Easy::getEasy(); } else { - easy = mFreeEasy.back(); - mFreeEasy.pop_back(); + easy = *(mEasyFreeList.begin()); + mEasyFreeList.erase(easy); + } + if (easy) + { + mEasyActiveList.insert(easy); + mEasyActiveMap[easy->getCurlHandle()] = easy; } - return easy; } -void -LLCurl::Multi::easyFree(Easy* easy) +bool LLCurl::Multi::addEasy(Easy* easy) { - if (mFreeEasy.size() < 5) + CURLMcode mcode = curl_multi_add_handle(mCurlMultiHandle, easy->getCurlHandle()); + if (mcode != CURLM_OK) { - mFreeEasy.push_back(easy); + llwarns << "Curl Error: " << curl_multi_strerror(mcode) << llendl; + return false; + } + return true; +} + +void LLCurl::Multi::easyFree(Easy* easy) +{ + mEasyActiveList.erase(easy); + mEasyActiveMap.erase(easy->getCurlHandle()); + if (mEasyFreeList.size() < EASY_HANDLE_POOL_SIZE) + { + easy->resetState(); + mEasyFreeList.insert(easy); } else { @@ -333,53 +651,371 @@ LLCurl::Multi::easyFree(Easy* easy) } } +void LLCurl::Multi::removeEasy(Easy* easy) +{ + curl_multi_remove_handle(mCurlMultiHandle, easy->getCurlHandle()); + easyFree(easy); +} + +//static +std::string LLCurl::strerror(CURLcode errorcode) +{ +#if LL_DARWIN + // curl_easy_strerror was added in libcurl 7.12.0. Unfortunately, the version in the Mac OS X 10.3.9 SDK is 7.10.2... + // There's a problem with the custom curl headers in our build that keeps me from #ifdefing this on the libcurl version number + // (the correct check would be #if LIBCURL_VERSION_NUM >= 0x070c00). We'll fix the header problem soon, but for now + // just punt and print the numeric error code on the Mac. + return llformat("%d", errorcode); +#else // LL_DARWIN + return std::string(curl_easy_strerror(errorcode)); +#endif // LL_DARWIN +} + +//////////////////////////////////////////////////////////////////////////// +// For generating a simple request for data +// using one multi and one easy per request + +LLCurlRequest::LLCurlRequest() + : mActiveMulti(NULL) +{ +} + +LLCurlRequest::~LLCurlRequest() +{ + for_each(mMultiSet.begin(), mMultiSet.end(), DeletePointer()); +} + +void LLCurlRequest::addMulti() +{ + LLCurl::Multi* multi = new LLCurl::Multi(); + mMultiSet.insert(multi); + mActiveMulti = multi; + mActiveRequestCount = 0; +} + +LLCurl::Easy* LLCurlRequest::allocEasy() +{ + if (!mActiveMulti || + mActiveRequestCount >= MAX_ACTIVE_REQUEST_COUNT || + mActiveMulti->mErrorCount > 0) + { + addMulti(); + } + llassert_always(mActiveMulti); + ++mActiveRequestCount; + LLCurl::Easy* easy = mActiveMulti->allocEasy(); + return easy; +} + +bool LLCurlRequest::addEasy(LLCurl::Easy* easy) +{ + llassert_always(mActiveMulti); + bool res = mActiveMulti->addEasy(easy); + return res; +} +void LLCurlRequest::get(const std::string& url, LLCurl::ResponderPtr responder) +{ + getByteRange(url, 0, -1, responder); +} + +bool LLCurlRequest::getByteRange(const std::string& url, S32 offset, S32 length, LLCurl::ResponderPtr responder) +{ + LLCurl::Easy* easy = allocEasy(); + if (!easy) + { + return false; + } + easy->prepRequest(url, responder); + easy->setopt(CURLOPT_HTTPGET, 1); + if (length > 0) + { + std::string range = llformat("Range: bytes=%d-%d", offset,offset+length-1); + easy->slist_append(range.c_str()); + } + easy->setHeaders(); + bool res = addEasy(easy); + return res; +} -namespace +bool LLCurlRequest::post(const std::string& url, const LLSD& data, LLCurl::ResponderPtr responder) { - static LLCurl::Multi* sMainMulti = 0; + LLCurl::Easy* easy = allocEasy(); + if (!easy) + { + return false; + } + easy->prepRequest(url, responder); + + LLSDSerialize::toXML(data, easy->getInput()); + S32 bytes = easy->getInput().str().length(); + + easy->setopt(CURLOPT_POST, 1); + easy->setopt(CURLOPT_POSTFIELDS, (void*)NULL); + easy->setopt(CURLOPT_POSTFIELDSIZE, bytes); + + easy->slist_append("Content-Type: application/xml"); + easy->setHeaders(); + + lldebugs << "POSTING: " << bytes << " bytes." << llendl; + bool res = addEasy(easy); + return res; +} - LLCurl::Multi* - mainMulti() +// Note: call once per frame +S32 LLCurlRequest::process() +{ + S32 res = 0; + for (curlmulti_set_t::iterator iter = mMultiSet.begin(); + iter != mMultiSet.end(); ) { - if (!sMainMulti) { - sMainMulti = new LLCurl::Multi(); + curlmulti_set_t::iterator curiter = iter++; + LLCurl::Multi* multi = *curiter; + S32 tres = multi->process(); + res += tres; + if (multi != mActiveMulti && tres == 0 && multi->mQueued == 0) + { + mMultiSet.erase(curiter); + delete multi; } - return sMainMulti; } + return res; +} - void freeMulti() +S32 LLCurlRequest::getQueued() +{ + S32 queued = 0; + for (curlmulti_set_t::iterator iter = mMultiSet.begin(); + iter != mMultiSet.end(); ) { - delete sMainMulti; - sMainMulti = NULL; + curlmulti_set_t::iterator curiter = iter++; + LLCurl::Multi* multi = *curiter; + queued += multi->mQueued; } + return queued; } -void -LLCurl::get(const std::string& url, ResponderPtr responder) +//////////////////////////////////////////////////////////////////////////// +// For generating one easy request +// associated with a single multi request + +LLCurlEasyRequest::LLCurlEasyRequest() + : mRequestSent(false), + mResultReturned(false) { - mainMulti()->get(url, responder); + mMulti = new LLCurl::Multi(); + mEasy = mMulti->allocEasy(); + if (mEasy) + { + mEasy->setErrorBuffer(); + mEasy->setCA(); + } } - -void -LLCurl::getByteRange(const std::string& url, S32 offset, S32 length, ResponderPtr responder) + +LLCurlEasyRequest::~LLCurlEasyRequest() { - mainMulti()->getByteRange(url, offset, length, responder); + delete mMulti; } -void LLCurl::initClass() +void LLCurlEasyRequest::setopt(CURLoption option, S32 value) { - curl_global_init(CURL_GLOBAL_ALL); + if (mEasy) + { + mEasy->setopt(option, value); + } } -void -LLCurl::process() +void LLCurlEasyRequest::setoptString(CURLoption option, const std::string& value) { - mainMulti()->process(); + if (mEasy) + { + mEasy->setoptString(option, value); + } } -void LLCurl::cleanup() +void LLCurlEasyRequest::setPost(char* postdata, S32 size) { - freeMulti(); + if (mEasy) + { + mEasy->setopt(CURLOPT_POST, 1); + mEasy->setopt(CURLOPT_POSTFIELDS, postdata); + mEasy->setopt(CURLOPT_POSTFIELDSIZE, size); + } +} + +void LLCurlEasyRequest::setHeaderCallback(curl_header_callback callback, void* userdata) +{ + if (mEasy) + { + mEasy->setopt(CURLOPT_HEADERFUNCTION, (void*)callback); + mEasy->setopt(CURLOPT_HEADERDATA, userdata); // aka CURLOPT_WRITEHEADER + } +} + +void LLCurlEasyRequest::setWriteCallback(curl_write_callback callback, void* userdata) +{ + if (mEasy) + { + mEasy->setopt(CURLOPT_WRITEFUNCTION, (void*)callback); + mEasy->setopt(CURLOPT_WRITEDATA, userdata); + } +} + +void LLCurlEasyRequest::setReadCallback(curl_read_callback callback, void* userdata) +{ + if (mEasy) + { + mEasy->setopt(CURLOPT_READFUNCTION, (void*)callback); + mEasy->setopt(CURLOPT_READDATA, userdata); + } +} + +void LLCurlEasyRequest::slist_append(const char* str) +{ + if (mEasy) + { + mEasy->slist_append(str); + } +} + +void LLCurlEasyRequest::sendRequest(const std::string& url) +{ + llassert_always(!mRequestSent); + mRequestSent = true; + if (mEasy) + { + mEasy->setHeaders(); + mEasy->setoptString(CURLOPT_URL, url); + mMulti->addEasy(mEasy); + } +} + +void LLCurlEasyRequest::requestComplete() +{ + llassert_always(mRequestSent); + mRequestSent = false; + if (mEasy) + { + mMulti->removeEasy(mEasy); + } +} + +S32 LLCurlEasyRequest::perform() +{ + return mMulti->perform(); +} + +// Usage: Call getRestult until it returns false (no more messages) +bool LLCurlEasyRequest::getResult(CURLcode* result, LLCurl::TransferInfo* info) +{ + if (!mEasy) + { + // Special case - we failed to initialize a curl_easy (can happen if too many open files) + // Act as though the request failed to connect + if (mResultReturned) + { + return false; + } + else + { + *result = CURLE_FAILED_INIT; + mResultReturned = true; + return true; + } + } + // In theory, info_read might return a message with a status other than CURLMSG_DONE + // In practice for all messages returned, msg == CURLMSG_DONE + // Ignore other messages just in case + while(1) + { + S32 q; + CURLMsg* curlmsg = info_read(&q, info); + if (curlmsg) + { + if (curlmsg->msg == CURLMSG_DONE) + { + *result = curlmsg->data.result; + return true; + } + // else continue + } + else + { + return false; + } + } +} + +// private +CURLMsg* LLCurlEasyRequest::info_read(S32* q, LLCurl::TransferInfo* info) +{ + if (mEasy) + { + CURLMsg* curlmsg = mMulti->info_read(q); + if (curlmsg && curlmsg->msg == CURLMSG_DONE) + { + if (info) + { + mEasy->getTransferInfo(info); + } + } + return curlmsg; + } + return NULL; +} + +std::string LLCurlEasyRequest::getErrorString() +{ + return mEasy ? std::string(mEasy->getErrorBuffer()) : std::string(); +} + +//////////////////////////////////////////////////////////////////////////// + +#if SAFE_SSL +//static +void LLCurl::ssl_locking_callback(int mode, int type, const char *file, int line) +{ + if (mode & CRYPTO_LOCK) + { + LLCurl::sSSLMutex[type]->lock(); + } + else + { + LLCurl::sSSLMutex[type]->unlock(); + } +} + +//static +unsigned long LLCurl::ssl_thread_id(void) +{ + return LLThread::currentID(); +} +#endif + +void LLCurl::initClass() +{ + // Do not change this "unless you are familiar with and mean to control + // internal operations of libcurl" + // - http://curl.haxx.se/libcurl/c/curl_global_init.html + curl_global_init(CURL_GLOBAL_ALL); + +#if SAFE_SSL + S32 mutex_count = CRYPTO_num_locks(); + for (S32 i=0; i<mutex_count; i++) + { + sSSLMutex.push_back(new LLMutex(gAPRPoolp)); + } + CRYPTO_set_id_callback(&LLCurl::ssl_thread_id); + CRYPTO_set_locking_callback(&LLCurl::ssl_locking_callback); +#endif +} + +void LLCurl::cleanupClass() +{ +#if SAFE_SSL + CRYPTO_set_locking_callback(NULL); + for_each(sSSLMutex.begin(), sSSLMutex.end(), DeletePointer()); +#endif curl_global_cleanup(); } + diff --git a/indra/llmessage/llcurl.h b/indra/llmessage/llcurl.h index 53287c2988..48c14d9460 100644 --- a/indra/llmessage/llcurl.h +++ b/indra/llmessage/llcurl.h @@ -1,8 +1,8 @@ -/** +/** * @file llcurl.h * @author Zero / Donovan * @date 2006-10-15 - * @brief Curl wrapper + * @brief A wrapper around libcurl. * * $LicenseInfo:firstyear=2006&license=viewergpl$ * @@ -41,104 +41,183 @@ #include <vector> #include <boost/intrusive_ptr.hpp> -#include <curl/curl.h> +#include <curl/curl.h> // TODO: remove dependency -// #include "llhttpclient.h" +#include "llbuffer.h" +#include "lliopipe.h" +#include "llsd.h" + +class LLMutex; + +// For whatever reason, this is not typedef'd in curl.h +typedef size_t (*curl_header_callback)(void *ptr, size_t size, size_t nmemb, void *stream); class LLCurl { + LOG_CLASS(LLCurl); + public: + class Easy; class Multi; + struct TransferInfo + { + TransferInfo() : mSizeDownload(0.0), mTotalTime(0.0), mSpeedDownload(0.0) {} + F64 mSizeDownload; + F64 mTotalTime; + F64 mSpeedDownload; + }; + class Responder { + //LOG_CLASS(Responder); public: + Responder(); virtual ~Responder(); - virtual void error(U32 status, const std::stringstream& content); // called with bad status codes + /** + * @brief return true if the status code indicates success. + */ + static bool isGoodStatus(U32 status) + { + return((200 <= status) && (status < 300)); + } + + virtual void error(U32 status, const std::string& reason); + // called with non-200 status codes - virtual void result(const std::stringstream& content); + virtual void result(const LLSD& content); - virtual void completed(U32 status, const std::stringstream& content); + // Override point for clients that may want to use this class when the response is some other format besides LLSD + virtual void completedRaw(U32 status, const std::string& reason, + const LLChannelDescriptors& channels, + const LLIOPipe::buffer_ptr_t& buffer); + + virtual void completed(U32 status, const std::string& reason, const LLSD& content); /**< The default implemetnation calls either: * result(), or * error() */ + // Override to handle parsing of the header only. Note: this is the only place where the contents + // of the header can be parsed. In the ::completed call above only the body is contained in the LLSD. + virtual void completedHeader(U32 status, const std::string& reason, const LLSD& content); + public: /* but not really -- don't touch this */ U32 mReferenceCount; }; typedef boost::intrusive_ptr<Responder> ResponderPtr; - - class Easy - { - public: - Easy(); - ~Easy(); - - void get(const std::string& url, ResponderPtr); - void getByteRange(const std::string& url, S32 offset, S32 length, ResponderPtr); - - void perform(); - private: - void prep(const std::string& url, ResponderPtr); - void report(CURLcode); - - CURL* mHandle; - struct curl_slist* mHeaders; - - std::string mURL; - std::string mRange; - std::stringstream mRequest; - std::stringstream mOutput; - char mErrorBuffer[CURL_ERROR_SIZE]; - - std::stringstream mHeaderOutput; // Debug - - ResponderPtr mResponder; - - friend class Multi; - }; + /** + * @ brief Set certificate authority file used to verify HTTPS certs. + */ + static void setCAFile(const std::string& file); + /** + * @ brief Set certificate authority path used to verify HTTPS certs. + */ + static void setCAPath(const std::string& path); + + /** + * @ brief Get certificate authority file used to verify HTTPS certs. + */ + static const std::string& getCAFile() { return sCAFile; } + + /** + * @ brief Get certificate authority path used to verify HTTPS certs. + */ + static const std::string& getCAPath() { return sCAPath; } + + /** + * @ brief Initialize LLCurl class + */ + static void initClass(); + + /** + * @ brief Cleanup LLCurl class + */ + static void cleanupClass(); + + /** + * @ brief curl error code -> string + */ + static std::string strerror(CURLcode errorcode); + + // For OpenSSL callbacks + static std::vector<LLMutex*> sSSLMutex; - class Multi - { - public: - Multi(); - ~Multi(); + // OpenSSL callbacks + static void LLCurl::ssl_locking_callback(int mode, int type, const char *file, int line); + static unsigned long LLCurl::ssl_thread_id(void); + + + +private: - void get(const std::string& url, ResponderPtr); - void getByteRange(const std::string& url, S32 offset, S32 length, ResponderPtr); + static std::string sCAPath; + static std::string sCAFile; +}; - void process(); - - private: - Easy* easyAlloc(); - void easyFree(Easy*); - - CURLM* mHandle; - - typedef std::vector<Easy*> EasyList; - EasyList mFreeEasy; - }; +namespace boost +{ + void intrusive_ptr_add_ref(LLCurl::Responder* p); + void intrusive_ptr_release(LLCurl::Responder* p); +}; - static void get(const std::string& url, ResponderPtr); - static void getByteRange(const std::string& url, S32 offset, S32 length, ResponderPtr responder); +class LLCurlRequest +{ +public: + LLCurlRequest(); + ~LLCurlRequest(); + + void get(const std::string& url, LLCurl::ResponderPtr responder); + bool getByteRange(const std::string& url, S32 offset, S32 length, LLCurl::ResponderPtr responder); + bool post(const std::string& url, const LLSD& data, LLCurl::ResponderPtr responder); + S32 process(); + S32 getQueued(); + +private: + void addMulti(); + LLCurl::Easy* allocEasy(); + bool addEasy(LLCurl::Easy* easy); - static void initClass(); // *NOTE:Mani - not thread safe! - static void process(); - static void cleanup(); // *NOTE:Mani - not thread safe! +private: + typedef std::set<LLCurl::Multi*> curlmulti_set_t; + curlmulti_set_t mMultiSet; + LLCurl::Multi* mActiveMulti; + S32 mActiveRequestCount; }; -namespace boost +class LLCurlEasyRequest { - void intrusive_ptr_add_ref(LLCurl::Responder* p); - void intrusive_ptr_release(LLCurl::Responder* p); +public: + LLCurlEasyRequest(); + ~LLCurlEasyRequest(); + void setopt(CURLoption option, S32 value); + void setoptString(CURLoption option, const std::string& value); + void setPost(char* postdata, S32 size); + void setHeaderCallback(curl_header_callback callback, void* userdata); + void setWriteCallback(curl_write_callback callback, void* userdata); + void setReadCallback(curl_read_callback callback, void* userdata); + void slist_append(const char* str); + void sendRequest(const std::string& url); + void requestComplete(); + S32 perform(); + bool getResult(CURLcode* result, LLCurl::TransferInfo* info = NULL); + std::string getErrorString(); + +private: + CURLMsg* info_read(S32* queue, LLCurl::TransferInfo* info); + +private: + LLCurl::Multi* mMulti; + LLCurl::Easy* mEasy; + bool mRequestSent; + bool mResultReturned; }; #endif // LL_LLCURL_H diff --git a/indra/llmessage/llhttpassetstorage.cpp b/indra/llmessage/llhttpassetstorage.cpp index cf9bde6fec..2179064807 100644 --- a/indra/llmessage/llhttpassetstorage.cpp +++ b/indra/llmessage/llhttpassetstorage.cpp @@ -422,11 +422,8 @@ void LLHTTPAssetStorage::_init(const char *web_host, const char *local_web_host, mLocalBaseURL = local_web_host; mHostName = host_name; - // Do not change this "unless you are familiar with and mean to control - // internal operations of libcurl" - // - http://curl.haxx.se/libcurl/c/curl_global_init.html - curl_global_init(CURL_GLOBAL_ALL); - + // curl_global_init moved to LLCurl::initClass() + mCurlMultiHandle = curl_multi_init(); } @@ -435,7 +432,7 @@ LLHTTPAssetStorage::~LLHTTPAssetStorage() curl_multi_cleanup(mCurlMultiHandle); mCurlMultiHandle = NULL; - curl_global_cleanup(); + // curl_global_cleanup moved to LLCurl::initClass() } // storing data is simpler than getting it, so we just overload the whole method diff --git a/indra/llmessage/llhttpclient.cpp b/indra/llmessage/llhttpclient.cpp index 23295476ff..3b892aec50 100644 --- a/indra/llmessage/llhttpclient.cpp +++ b/indra/llmessage/llhttpclient.cpp @@ -1,4 +1,4 @@ -/** + /** * @file llhttpclient.cpp * @brief Implementation of classes for making HTTP requests. * @@ -38,7 +38,6 @@ #include "llurlrequest.h" #include "llbufferstream.h" #include "llsdserialize.h" -#include "llsdutil.h" #include "llvfile.h" #include "llvfs.h" #include "lluri.h" @@ -47,85 +46,18 @@ #include <curl/curl.h> const F32 HTTP_REQUEST_EXPIRY_SECS = 60.0f; -static std::string gCABundle; +//////////////////////////////////////////////////////////////////////////// +// Responder class moved to LLCurl -LLHTTPClient::Responder::Responder() - : mReferenceCount(0) -{ -} - -LLHTTPClient::Responder::~Responder() -{ -} - -// virtual -void LLHTTPClient::Responder::error(U32 status, const std::string& reason) -{ - llinfos << "LLHTTPClient::Responder::error " - << status << ": " << reason << llendl; -} - -// virtual -void LLHTTPClient::Responder::result(const LLSD& content) -{ -} - -// virtual -void LLHTTPClient::Responder::completedRaw( - U32 status, - const std::string& reason, - const LLChannelDescriptors& channels, - const LLIOPipe::buffer_ptr_t& buffer) -{ - LLBufferStream istr(channels, buffer.get()); - LLSD content; - - if (isGoodStatus(status)) - { - LLSDSerialize::fromXML(content, istr); -/* - const S32 parseError = -1; - if(LLSDSerialize::fromXML(content, istr) == parseError) - { - mStatus = 498; - mReason = "Client Parse Error"; - } -*/ - } - - completed(status, reason, content); -} - -// virtual -void LLHTTPClient::Responder::completed( - U32 status, - const std::string& reason, - const LLSD& content) -{ - if(isGoodStatus(status)) - { - result(content); - } - else - { - error(status, reason); - } -} - -// virtual -void LLHTTPClient::Responder::completedHeader(U32 status, const std::string& reason, const LLSD& content) -{ - -} namespace { class LLHTTPClientURLAdaptor : public LLURLRequestComplete { public: - LLHTTPClientURLAdaptor(LLHTTPClient::ResponderPtr responder) - : mResponder(responder), - mStatus(499), mReason("LLURLRequest complete w/no status") + LLHTTPClientURLAdaptor(LLCurl::ResponderPtr responder) + : mResponder(responder), mStatus(499), + mReason("LLURLRequest complete w/no status") { } @@ -140,7 +72,7 @@ namespace } virtual void complete(const LLChannelDescriptors& channels, - const buffer_ptr_t& buffer) + const buffer_ptr_t& buffer) { if (mResponder.get()) { @@ -154,7 +86,7 @@ namespace } private: - LLHTTPClient::ResponderPtr mResponder; + LLCurl::ResponderPtr mResponder; U32 mStatus; std::string mReason; LLSD mHeaderOutput; @@ -267,13 +199,14 @@ namespace LLPumpIO* theClientPump = NULL; } -static void request( - const std::string& url, - LLURLRequest::ERequestAction method, - Injector* body_injector, - LLHTTPClient::ResponderPtr responder, - const LLSD& headers, - const F32 timeout=HTTP_REQUEST_EXPIRY_SECS) +static void request(const std::string& url, + LLURLRequest::ERequestAction method, + Injector* body_injector, + LLCurl::ResponderPtr responder, + const LLSD& headers = LLSD(), + const F32 timeout = HTTP_REQUEST_EXPIRY_SECS, + S32 offset = 0, + S32 bytes = 0) { if (!LLHTTPClient::hasPump()) { @@ -283,7 +216,7 @@ static void request( LLPumpIO::chain_t chain; LLURLRequest *req = new LLURLRequest(method, url); - req->requestEncoding(""); + req->checkRootCertificate(true); // Insert custom headers is the caller sent any if (headers.isMap()) @@ -308,10 +241,6 @@ static void request( req->addHeader(header.str().c_str()); } } - if (!gCABundle.empty()) - { - req->checkRootCertificate(true, gCABundle.c_str()); - } req->setCallback(new LLHTTPClientURLAdaptor(responder)); if (method == LLURLRequest::HTTP_POST && gMessageSystem) @@ -327,19 +256,26 @@ static void request( chain.push_back(LLIOPipe::ptr_t(body_injector)); } + + if (method == LLURLRequest::HTTP_GET && (offset > 0 || bytes > 0)) + { + std::string range = llformat("Range: bytes=%d-%d", offset,offset+bytes-1); + req->addHeader(range.c_str()); + } + chain.push_back(LLIOPipe::ptr_t(req)); theClientPump->addChain(chain, timeout); } -static void request( - const std::string& url, - LLURLRequest::ERequestAction method, - Injector* body_injector, - LLHTTPClient::ResponderPtr responder, - const F32 timeout=HTTP_REQUEST_EXPIRY_SECS) + +void LLHTTPClient::getByteRange(const std::string& url, + S32 offset, S32 bytes, + ResponderPtr responder, + const LLSD& headers, + const F32 timeout) { - request(url, method, body_injector, responder, LLSD(), timeout); + request(url, LLURLRequest::HTTP_GET, NULL, responder, LLSD(), timeout, offset, bytes); } void LLHTTPClient::head(const std::string& url, ResponderPtr responder, const F32 timeout) @@ -355,10 +291,6 @@ void LLHTTPClient::getHeaderOnly(const std::string& url, ResponderPtr responder, { request(url, LLURLRequest::HTTP_HEAD, NULL, responder, headers, timeout); } -void LLHTTPClient::get(const std::string& url, ResponderPtr responder, const F32 timeout) -{ - get(url, responder, LLSD(), timeout); -} void LLHTTPClient::getHeaderOnly(const std::string& url, ResponderPtr responder, const F32 timeout) { getHeaderOnly(url, responder, LLSD(), timeout); @@ -372,11 +304,6 @@ void LLHTTPClient::get(const std::string& url, const LLSD& query, ResponderPtr r get(uri.asString(), responder, headers, timeout); } -void LLHTTPClient::get(const std::string& url, const LLSD& query, ResponderPtr responder, const F32 timeout) -{ - get(url, query, responder, LLSD(), timeout); -} - // A simple class for managing data returned from a curl http request. class LLHTTPBuffer { @@ -412,6 +339,7 @@ private: std::string mBuffer; }; +// *TODO: Deprecate (only used by dataserver) // This call is blocking! This is probably usually bad. :( LLSD LLHTTPClient::blockingGet(const std::string& url) { @@ -505,24 +433,3 @@ bool LLHTTPClient::hasPump() { return theClientPump != NULL; } - -void LLHTTPClient::setCABundle(const std::string& caBundle) -{ - gCABundle = caBundle; -} - -namespace boost -{ - void intrusive_ptr_add_ref(LLHTTPClient::Responder* p) - { - ++p->mReferenceCount; - } - - void intrusive_ptr_release(LLHTTPClient::Responder* p) - { - if(p && 0 == --p->mReferenceCount) - { - delete p; - } - } -}; diff --git a/indra/llmessage/llhttpclient.h b/indra/llmessage/llhttpclient.h index 1fbf0c36dc..b011761f5f 100644 --- a/indra/llmessage/llhttpclient.h +++ b/indra/llmessage/llhttpclient.h @@ -41,7 +41,7 @@ #include <boost/intrusive_ptr.hpp> #include "llassettype.h" -#include "llbuffer.h" +#include "llcurl.h" #include "lliopipe.h" extern const F32 HTTP_REQUEST_EXPIRY_SECS; @@ -54,57 +54,18 @@ class LLSD; class LLHTTPClient { public: - class Responder - { - public: - Responder(); - virtual ~Responder(); - - /** - * @brief return true if the status code indicates success. - */ - static bool isGoodStatus(U32 status) - { - return((200 <= status) && (status < 300)); - } - - virtual void error(U32 status, const std::string& reason); // called with bad status codes - - virtual void result(const LLSD& content); - - // Override point for clients that may want to use this class - // when the response is some other format besides LLSD - virtual void completedRaw( - U32 status, - const std::string& reason, - const LLChannelDescriptors& channels, - const LLIOPipe::buffer_ptr_t& buffer); - - virtual void completed( - U32 status, - const std::string& reason, - const LLSD& content); - /**< The default implemetnation calls - either: - * result(), or - * error() - */ - - // Override to handle parsing of the header only. Note: this is the only place where the contents - // of the header can be parsed. In the ::completed call above only the body is contained in the LLSD. - virtual void completedHeader(U32 status, const std::string& reason, const LLSD& content); - - public: /* but not really -- don't touch this */ - U32 mReferenceCount; - }; - - typedef boost::intrusive_ptr<Responder> ResponderPtr; - + // class Responder moved to LLCurl + + // For convenience + typedef LLCurl::Responder Responder; + typedef LLCurl::ResponderPtr ResponderPtr; + + // non-blocking static void head(const std::string& url, ResponderPtr, const F32 timeout=HTTP_REQUEST_EXPIRY_SECS); - static void get(const std::string& url, ResponderPtr, const F32 timeout=HTTP_REQUEST_EXPIRY_SECS); - static void get(const std::string& url, ResponderPtr, const LLSD& headers, const F32 timeout=HTTP_REQUEST_EXPIRY_SECS); - static void get(const std::string& url, const LLSD& query, ResponderPtr, const F32 timeout=HTTP_REQUEST_EXPIRY_SECS); - static void get(const std::string& url, const LLSD& query, ResponderPtr, const LLSD& headers, const F32 timeout=HTTP_REQUEST_EXPIRY_SECS); + static void getByteRange(const std::string& url, S32 offset, S32 bytes, ResponderPtr, const LLSD& headers=LLSD(), const F32 timeout=HTTP_REQUEST_EXPIRY_SECS); + static void get(const std::string& url, ResponderPtr, const LLSD& headers = LLSD(), const F32 timeout=HTTP_REQUEST_EXPIRY_SECS); + static void get(const std::string& url, const LLSD& query, ResponderPtr, const LLSD& headers = LLSD(), const F32 timeout=HTTP_REQUEST_EXPIRY_SECS); + static void put(const std::string& url, const LLSD& body, ResponderPtr, const F32 timeout=HTTP_REQUEST_EXPIRY_SECS); static void getHeaderOnly(const std::string& url, ResponderPtr, const F32 timeout=HTTP_REQUEST_EXPIRY_SECS); static void getHeaderOnly(const std::string& url, ResponderPtr, const LLSD& headers, const F32 timeout=HTTP_REQUEST_EXPIRY_SECS); @@ -127,20 +88,6 @@ public: ///< must be called before any of the above calls are made static bool hasPump(); ///< for testing - - static void setCABundle(const std::string& caBundle); - ///< use this root CA bundle when checking SSL connections - ///< defaults to the standard system root CA bundle - ///< @see LLURLRequest::checkRootCertificate() }; - - -namespace boost -{ - void intrusive_ptr_add_ref(LLHTTPClient::Responder* p); - void intrusive_ptr_release(LLHTTPClient::Responder* p); -}; - - #endif // LL_LLHTTPCLIENT_H diff --git a/indra/llmessage/llurlrequest.cpp b/indra/llmessage/llurlrequest.cpp index 631eea3e88..f850656785 100644 --- a/indra/llmessage/llurlrequest.cpp +++ b/indra/llmessage/llurlrequest.cpp @@ -37,6 +37,7 @@ #include <curl/curl.h> #include <algorithm> +#include "llcurl.h" #include "llioutil.h" #include "llmemtype.h" #include "llpumpio.h" @@ -52,8 +53,7 @@ static const U32 HTTP_STATUS_PIPE_ERROR = 499; const std::string CONTEXT_DEST_URI_SD_LABEL("dest_uri"); -static -size_t headerCallback(void* data, size_t size, size_t nmemb, void* user); +static size_t headerCallback(void* data, size_t size, size_t nmemb, void* user); /** * class LLURLRequestDetail @@ -63,12 +63,8 @@ class LLURLRequestDetail public: LLURLRequestDetail(); ~LLURLRequestDetail(); - CURLM* mCurlMulti; - CURL* mCurl; - struct curl_slist* mHeaders; - char* mURL; - char mCurlErrorBuf[CURL_ERROR_SIZE + 1]; /* Flawfinder: ignore */ - bool mNeedToRemoveEasyHandle; + std::string mURL; + LLCurlEasyRequest* mCurlRequest; LLBufferArray* mResponseBuffer; LLChannelDescriptors mChannels; U8* mLastRead; @@ -77,11 +73,7 @@ public: }; LLURLRequestDetail::LLURLRequestDetail() : - mCurlMulti(NULL), - mCurl(NULL), - mHeaders(NULL), - mURL(NULL), - mNeedToRemoveEasyHandle(false), + mCurlRequest(NULL), mResponseBuffer(NULL), mLastRead(NULL), mBodyLimit(0), @@ -89,34 +81,13 @@ LLURLRequestDetail::LLURLRequestDetail() : { LLMemType m1(LLMemType::MTYPE_IO_URL_REQUEST); - mCurlErrorBuf[0] = '\0'; + mCurlRequest = new LLCurlEasyRequest(); } LLURLRequestDetail::~LLURLRequestDetail() { LLMemType m1(LLMemType::MTYPE_IO_URL_REQUEST); - if(mCurl) - { - if(mNeedToRemoveEasyHandle && mCurlMulti) - { - curl_multi_remove_handle(mCurlMulti, mCurl); - mNeedToRemoveEasyHandle = false; - } - curl_easy_cleanup(mCurl); - mCurl = NULL; - } - if(mCurlMulti) - { - curl_multi_cleanup(mCurlMulti); - mCurlMulti = NULL; - } - if(mHeaders) - { - curl_slist_free_all(mHeaders); - mHeaders = NULL; - } - delete[] mURL; - mURL = NULL; + delete mCurlRequest; mResponseBuffer = NULL; mLastRead = NULL; } @@ -126,9 +97,6 @@ LLURLRequestDetail::~LLURLRequestDetail() * class LLURLRequest */ -static std::string sCAFile(""); -static std::string sCAPath(""); - LLURLRequest::LLURLRequest(LLURLRequest::ERequestAction action) : mAction(action) { @@ -155,31 +123,13 @@ LLURLRequest::~LLURLRequest() void LLURLRequest::setURL(const std::string& url) { LLMemType m1(LLMemType::MTYPE_IO_URL_REQUEST); - if(mDetail->mURL) - { - // *NOTE: if any calls to set the url have been made to curl, - // this will probably lead to a crash. - delete[] mDetail->mURL; - mDetail->mURL = NULL; - } - if(!url.empty()) - { - mDetail->mURL = new char[url.size() + 1]; - url.copy(mDetail->mURL, url.size()); - mDetail->mURL[url.size()] = '\0'; - } + mDetail->mURL = url; } void LLURLRequest::addHeader(const char* header) { LLMemType m1(LLMemType::MTYPE_IO_URL_REQUEST); - mDetail->mHeaders = curl_slist_append(mDetail->mHeaders, header); -} - -void LLURLRequest::requestEncoding(const char* encoding) -{ - LLMemType m1(LLMemType::MTYPE_IO_URL_REQUEST); - curl_easy_setopt(mDetail->mCurl, CURLOPT_ENCODING, encoding); + mDetail->mCurlRequest->slist_append(header); } void LLURLRequest::setBodyLimit(U32 size) @@ -188,22 +138,17 @@ void LLURLRequest::setBodyLimit(U32 size) mDetail->mIsBodyLimitSet = true; } -void LLURLRequest::checkRootCertificate(bool check, const char* caBundle) +void LLURLRequest::checkRootCertificate(bool check) { - curl_easy_setopt(mDetail->mCurl, CURLOPT_SSL_VERIFYPEER, (check? TRUE : FALSE)); - if (caBundle) - { - curl_easy_setopt(mDetail->mCurl, CURLOPT_CAINFO, caBundle); - } + mDetail->mCurlRequest->setopt(CURLOPT_SSL_VERIFYPEER, (check? TRUE : FALSE)); + mDetail->mCurlRequest->setoptString(CURLOPT_ENCODING, ""); } void LLURLRequest::setCallback(LLURLRequestComplete* callback) { LLMemType m1(LLMemType::MTYPE_IO_URL_REQUEST); mCompletionCallback = callback; - - curl_easy_setopt(mDetail->mCurl, CURLOPT_HEADERFUNCTION, &headerCallback); - curl_easy_setopt(mDetail->mCurl, CURLOPT_WRITEHEADER, callback); + mDetail->mCurlRequest->setHeaderCallback(&headerCallback, (void*)callback); } // Added to mitigate the effect of libcurl looking @@ -239,11 +184,11 @@ void LLURLRequest::useProxy(bool use_proxy) if (env_proxy && use_proxy) { - curl_easy_setopt(mDetail->mCurl, CURLOPT_PROXY, env_proxy); + mDetail->mCurlRequest->setoptString(CURLOPT_PROXY, env_proxy); } else { - curl_easy_setopt(mDetail->mCurl, CURLOPT_PROXY, ""); + mDetail->mCurlRequest->setoptString(CURLOPT_PROXY, ""); } } @@ -309,27 +254,20 @@ LLIOPipe::EStatus LLURLRequest::process_impl( case STATE_PROCESSING_RESPONSE: { PUMP_DEBUG; - const S32 MAX_CALLS = 5; - S32 count = MAX_CALLS; - CURLMcode code; LLIOPipe::EStatus status = STATUS_BREAK; - S32 queue; - do - { - LLFastTimer t2(LLFastTimer::FTM_CURL); - code = curl_multi_perform(mDetail->mCurlMulti, &queue); - }while((CURLM_CALL_MULTI_PERFORM == code) && (queue > 0) && count--); - CURLMsg* curl_msg; - do + mDetail->mCurlRequest->perform(); + while(1) { - curl_msg = curl_multi_info_read(mDetail->mCurlMulti, &queue); - if(curl_msg && (curl_msg->msg == CURLMSG_DONE)) + CURLcode result; + bool newmsg = mDetail->mCurlRequest->getResult(&result); + if (!newmsg) { - mState = STATE_HAVE_RESPONSE; + break; + } - CURLcode result = curl_msg->data.result; - switch(result) - { + mState = STATE_HAVE_RESPONSE; + switch(result) + { case CURLE_OK: case CURLE_WRITE_ERROR: // NB: The error indication means that we stopped the @@ -352,31 +290,21 @@ LLIOPipe::EStatus LLURLRequest::process_impl( mCompletionCallback = NULL; } break; + case CURLE_FAILED_INIT: case CURLE_COULDNT_CONNECT: status = STATUS_NO_CONNECTION; break; default: - llwarns << "URLRequest Error: " << curl_msg->data.result + llwarns << "URLRequest Error: " << result << ", " -#if LL_DARWIN - // curl_easy_strerror was added in libcurl 7.12.0. Unfortunately, the version in the Mac OS X 10.3.9 SDK is 7.10.2... - // There's a problem with the custom curl headers in our build that keeps me from #ifdefing this on the libcurl version number - // (the correct check would be #if LIBCURL_VERSION_NUM >= 0x070c00). We'll fix the header problem soon, but for now - // just punt and print the numeric error code on the Mac. - << curl_msg->data.result -#else // LL_DARWIN - << curl_easy_strerror(curl_msg->data.result) -#endif // LL_DARWIN + << LLCurl::strerror(result) << ", " - << (mDetail->mURL ? mDetail->mURL : "<EMPTY URL>") + << (mDetail->mURL.empty() ? "<EMPTY URL>" : mDetail->mURL) << llendl; status = STATUS_ERROR; break; - } - curl_multi_remove_handle(mDetail->mCurlMulti, mDetail->mCurl); - mDetail->mNeedToRemoveEasyHandle = false; } - }while(curl_msg && (queue > 0)); + } return status; } case STATE_HAVE_RESPONSE: @@ -397,26 +325,9 @@ void LLURLRequest::initialize() LLMemType m1(LLMemType::MTYPE_IO_URL_REQUEST); mState = STATE_INITIALIZED; mDetail = new LLURLRequestDetail; - mDetail->mCurl = curl_easy_init(); - mDetail->mCurlMulti = curl_multi_init(); - curl_easy_setopt(mDetail->mCurl, CURLOPT_NOSIGNAL, 1); - curl_easy_setopt(mDetail->mCurl, CURLOPT_WRITEFUNCTION, &downCallback); - curl_easy_setopt(mDetail->mCurl, CURLOPT_WRITEDATA, this); - curl_easy_setopt(mDetail->mCurl, CURLOPT_READFUNCTION, &upCallback); - curl_easy_setopt(mDetail->mCurl, CURLOPT_READDATA, this); - curl_easy_setopt( - mDetail->mCurl, - CURLOPT_ERRORBUFFER, - mDetail->mCurlErrorBuf); - - if(sCAPath != std::string("")) - { - curl_easy_setopt(mDetail->mCurl, CURLOPT_CAPATH, sCAPath.c_str()); - } - if(sCAFile != std::string("")) - { - curl_easy_setopt(mDetail->mCurl, CURLOPT_CAINFO, sCAFile.c_str()); - } + mDetail->mCurlRequest->setopt(CURLOPT_NOSIGNAL, 1); + mDetail->mCurlRequest->setWriteCallback(&downCallback, (void*)this); + mDetail->mCurlRequest->setReadCallback(&upCallback, (void*)this); } bool LLURLRequest::configure() @@ -429,13 +340,14 @@ bool LLURLRequest::configure() switch(mAction) { case HTTP_HEAD: - // These are in addition to the HTTP_GET options. - curl_easy_setopt(mDetail->mCurl, CURLOPT_HEADER, 1); - curl_easy_setopt(mDetail->mCurl, CURLOPT_NOBODY, 1); - + mDetail->mCurlRequest->setopt(CURLOPT_HEADER, 1); + mDetail->mCurlRequest->setopt(CURLOPT_NOBODY, 1); + mDetail->mCurlRequest->setopt(CURLOPT_FOLLOWLOCATION, 1); + rv = true; + break; case HTTP_GET: - curl_easy_setopt(mDetail->mCurl, CURLOPT_HTTPGET, 1); - curl_easy_setopt(mDetail->mCurl, CURLOPT_FOLLOWLOCATION, 1); + mDetail->mCurlRequest->setopt(CURLOPT_HTTPGET, 1); + mDetail->mCurlRequest->setopt(CURLOPT_FOLLOWLOCATION, 1); rv = true; break; @@ -444,8 +356,8 @@ bool LLURLRequest::configure() // to turning this on, and I am not too sure what it means. addHeader("Expect:"); - curl_easy_setopt(mDetail->mCurl, CURLOPT_UPLOAD, 1); - curl_easy_setopt(mDetail->mCurl, CURLOPT_INFILESIZE, bytes); + mDetail->mCurlRequest->setopt(CURLOPT_UPLOAD, 1); + mDetail->mCurlRequest->setopt(CURLOPT_INFILESIZE, bytes); rv = true; break; @@ -459,15 +371,13 @@ bool LLURLRequest::configure() addHeader("Content-Type:"); // Set the handle for an http post - curl_easy_setopt(mDetail->mCurl, CURLOPT_POST, 1); - curl_easy_setopt(mDetail->mCurl, CURLOPT_POSTFIELDS, NULL); - curl_easy_setopt(mDetail->mCurl, CURLOPT_POSTFIELDSIZE, bytes); + mDetail->mCurlRequest->setPost(NULL, bytes); rv = true; break; case HTTP_DELETE: // Set the handle for an http post - curl_easy_setopt(mDetail->mCurl, CURLOPT_CUSTOMREQUEST, "DELETE"); + mDetail->mCurlRequest->setoptString(CURLOPT_CUSTOMREQUEST, "DELETE"); rv = true; break; @@ -477,24 +387,14 @@ bool LLURLRequest::configure() } if(rv) { - if(mDetail->mHeaders) - { - curl_easy_setopt( - mDetail->mCurl, - CURLOPT_HTTPHEADER, - mDetail->mHeaders); - } - curl_easy_setopt(mDetail->mCurl, CURLOPT_URL, mDetail->mURL); - lldebugs << "URL: " << mDetail->mURL << llendl; - curl_multi_add_handle(mDetail->mCurlMulti, mDetail->mCurl); - mDetail->mNeedToRemoveEasyHandle = true; + mDetail->mCurlRequest->sendRequest(mDetail->mURL); } return rv; } // static size_t LLURLRequest::downCallback( - void* data, + char* data, size_t size, size_t nmemb, void* user) @@ -528,7 +428,7 @@ size_t LLURLRequest::downCallback( // static size_t LLURLRequest::upCallback( - void* data, + char* data, size_t size, size_t nmemb, void* user) @@ -548,8 +448,7 @@ size_t LLURLRequest::upCallback( return bytes; } -static -size_t headerCallback(void* data, size_t size, size_t nmemb, void* user) +static size_t headerCallback(void* data, size_t size, size_t nmemb, void* user) { const char* headerLine = (const char*)data; size_t headerLen = size * nmemb; @@ -605,18 +504,6 @@ size_t headerCallback(void* data, size_t size, size_t nmemb, void* user) return headerLen; } -//static -void LLURLRequest::setCertificateAuthorityFile(const std::string& file_name) -{ - sCAFile = file_name; -} - -//static -void LLURLRequest::setCertificateAuthorityPath(const std::string& path) -{ - sCAPath = path; -} - /** * LLContextURLExtractor */ diff --git a/indra/llmessage/llurlrequest.h b/indra/llmessage/llurlrequest.h index 5bdb6a1e69..b154794ff1 100644 --- a/indra/llmessage/llurlrequest.h +++ b/indra/llmessage/llurlrequest.h @@ -129,18 +129,8 @@ public: * * Set whether request will check that remote server * certificates are signed by a known root CA when using HTTPS. - * Use the supplied root certificate bundle if supplied, else use - * the standard bundle as found by libcurl and openssl. */ - void checkRootCertificate(bool check, const char* caBundle = NULL); - - /** - * @brief Request a particular response encoding if available. - * - * This call is a shortcut for requesting a particular encoding - * from the server, eg, 'gzip'. - */ - void requestEncoding(const char* encoding); + void checkRootCertificate(bool check); /** * @brief Return at most size bytes of body. @@ -168,16 +158,6 @@ public: void setCallback(LLURLRequestComplete* callback); //@} - /** - * @ brief Set certificate authority file used to verify HTTPS certs. - */ - static void setCertificateAuthorityFile(const std::string& file_name); - - /** - * @ brief Set certificate authority path used to verify HTTPS certs. - */ - static void setCertificateAuthorityPath(const std::string& path); - /* @name LLIOPipe virtual implementations */ @@ -234,7 +214,7 @@ private: * @brief Download callback method. */ static size_t downCallback( - void* data, + char* data, size_t size, size_t nmemb, void* user); @@ -243,7 +223,7 @@ private: * @brief Upload callback method. */ static size_t upCallback( - void* data, + char* data, size_t size, size_t nmemb, void* user); diff --git a/indra/newview/llappviewer.cpp b/indra/newview/llappviewer.cpp index 666fcd1301..61699d21c8 100644 --- a/indra/newview/llappviewer.cpp +++ b/indra/newview/llappviewer.cpp @@ -51,6 +51,7 @@ #include "llstartup.h" #include "llfocusmgr.h" #include "llviewerjoystick.h" +#include "llares.h" #include "llcurl.h" #include "llfloatersnapshot.h" #include "llviewerwindow.h" @@ -1338,7 +1339,7 @@ bool LLAppViewer::mainLoop() // Create IO Pump to use for HTTP Requests. gServicePump = new LLPumpIO(gAPRPoolp); LLHTTPClient::setPump(*gServicePump); - LLHTTPClient::setCABundle(gDirUtilp->getCAFile()); + LLCurl::setCAFile(gDirUtilp->getCAFile()); // initialize voice stuff here gLocalSpeakerMgr = new LLLocalSpeakerMgr(); @@ -1398,10 +1399,14 @@ bool LLAppViewer::mainLoop() { LLFastTimer t3(LLFastTimer::FTM_IDLE); idle(); - LLCurl::process(); - // this pump is necessary to make the login screen show up - gServicePump->pump(); - gServicePump->callback(); + + { + LLFastTimer t4(LLFastTimer::FTM_PUMP); + gAres->process(); + // this pump is necessary to make the login screen show up + gServicePump->pump(); + gServicePump->callback(); + } } if (gDoDisconnect && (LLStartUp::getStartupState() == STATE_STARTED)) @@ -1811,7 +1816,7 @@ bool LLAppViewer::cleanup() end_messaging_system(); // *NOTE:Mani - The following call is not thread safe. - LLCurl::cleanup(); + LLCurl::cleanupClass(); // If we're exiting to launch an URL, do that here so the screen // is at the right resolution before we launch IE. diff --git a/indra/newview/llinventorymodel.cpp b/indra/newview/llinventorymodel.cpp index 7b27a830c4..17a8b84472 100644 --- a/indra/newview/llinventorymodel.cpp +++ b/indra/newview/llinventorymodel.cpp @@ -47,6 +47,7 @@ #include "llviewerinventory.h" #include "llviewermessage.h" #include "llviewerwindow.h" +#include "llviewerregion.h" #include "llappviewer.h" #include "lldbstrings.h" #include "llviewerstats.h" @@ -54,6 +55,8 @@ #include "llnotify.h" #include "llcallbacklist.h" #include "llpreview.h" +#include "llviewercontrol.h" +#include "llsdutil.h" #include <deque> //#define DIFF_INVENTORY_FILES @@ -69,6 +72,8 @@ F32 LLInventoryModel::sMinTimeBetweenFetches = 0.3f; F32 LLInventoryModel::sMaxTimeBetweenFetches = 10.f; BOOL LLInventoryModel::sTimelyFetchPending = FALSE; LLFrameTimer LLInventoryModel::sFetchTimer; +LLInventoryModel::cat_map_t LLInventoryModel::sBulkFetchMap; +S16 LLInventoryModel::sBulkFetchCount = 0; // RN: for some reason, using std::queue in the header file confuses the compiler which things it's an xmlrpc_queue static std::deque<LLUUID> sFetchQueue; @@ -1002,6 +1007,286 @@ void LLInventoryModel::fetchDescendentsOf(const LLUUID& folder_id) } } +//Initialize statics. +LLAlertDialog* LLInventoryModel::fetchDescendentsResponder::sRetryDialog=NULL; +LLSD LLInventoryModel::fetchDescendentsResponder::sRetrySD; + +bool LLInventoryModel::isBulkFetchProcessingComplete() +{ + return ( (sFetchQueue.empty() + && sBulkFetchMap.empty() + && sBulkFetchCount==0) ? TRUE : FALSE ) ; +} + +//If we get back a normal response, handle it here +void LLInventoryModel::fetchDescendentsResponder::result(const LLSD& content) +{ + if (content.has("folders")) + { + for(LLSD::array_const_iterator folder_it = content["folders"].beginArray(); + folder_it != content["folders"].endArray(); + ++folder_it) + { + LLSD folder_sd = *folder_it; + + + LLUUID agent_id = folder_sd["agent-id"]; + + if(agent_id != gAgent.getID()) //This should never happen. + { + llwarns << "Got a UpdateInventoryItem for the wrong agent." + << llendl; + break; + } + LLUUID parent_id = folder_sd["folder-id"]; + LLUUID owner_id = folder_sd["owner-id"]; + S32 version = (S32)folder_sd["version"].asInteger(); + S32 descendents = (S32)folder_sd["descendents"].asInteger(); + LLPointer<LLViewerInventoryCategory> tcategory = new LLViewerInventoryCategory(owner_id); + for(LLSD::array_const_iterator category_it = folder_sd["categories"].beginArray(); + category_it != folder_sd["categories"].endArray(); + ++category_it) + { + LLSD category = *category_it; + tcategory->fromLLSD(category); + + if (sFullFetchStarted) + { + sFetchQueue.push_back(tcategory->getUUID()); + } + else if ( !gInventory.isCategoryComplete(tcategory->getUUID()) ) + { + gInventory.updateCategory(tcategory); + } + + } + LLPointer<LLViewerInventoryItem> titem = new LLViewerInventoryItem; + for(LLSD::array_const_iterator item_it = folder_sd["items"].beginArray(); + item_it != folder_sd["items"].endArray(); + ++item_it) + { + LLSD item = *item_it; + titem->unpackMessage(item); + + gInventory.updateItem(titem); + } + + // set version and descendentcount according to message. + LLViewerInventoryCategory* cat = gInventory.getCategory(parent_id); + if(cat) + { + cat->setVersion(version); + cat->setDescendentCount(descendents); + } + + } + } + + if (content.has("bad-folders")) + { + for(LLSD::array_const_iterator folder_it = content["bad-folders"].beginArray(); + folder_it != content["bad-folders"].endArray(); + ++folder_it) + { + LLSD folder_sd = *folder_it; + + //These folders failed on the dataserver. We probably don't want to retry them. + llinfos << "Folder " << folder_sd["folder-id"].asString() + << "Error: " << folder_sd["error"].asString() << llendl; + } + } + + LLInventoryModel::incrBulkFetch(-1); + + if (isBulkFetchProcessingComplete()) + { + llinfos << "Inventory fetch completed" << llendl; + if (sFullFetchStarted) + { + sAllFoldersFetched = TRUE; + } + stopBackgroundFetch(); + } + + gInventory.notifyObservers(); +} + +//If we get back an error (not found, etc...), handle it here +void LLInventoryModel::fetchDescendentsResponder::error(U32 status, const std::string& reason) +{ + llinfos << "fetchDescendentsResponder::error " + << status << ": " << reason << llendl; + + LLInventoryModel::incrBulkFetch(-1); + + if (status==499) //timed out. Let's be awesome! + { + for(LLSD::array_const_iterator folder_it = mRequestSD["folders"].beginArray(); + folder_it != mRequestSD["folders"].endArray(); + ++folder_it) + { + LLSD folder_sd = *folder_it; + sRetrySD["folders"].append(folder_sd); + } + sMinTimeBetweenFetches = 10.0f; //Add 10 seconds for every time out in this sequence. + + if (!sRetryDialog) //The dialog isn't up. Prompt the resident. + { + sRetryDialog = gViewerWindow->alertXml("RetryFetchInventoryDescendents", onClickRetry, this); + } + } + else + { + if (isBulkFetchProcessingComplete()) + { + if (sFullFetchStarted) + { + sAllFoldersFetched = TRUE; + } + stopBackgroundFetch(); + } + } + gInventory.notifyObservers(); +} + +void LLInventoryModel::fetchDescendentsResponder::onClickRetry(S32 option, void* userdata) +{ + if (option == 0) + { + std::string url = gAgent.getRegion()->getCapability("FetchInventoryDescendents"); + + if (!url.empty()) //Capability found. Build up LLSD and use it. + { + LLSD body = sRetrySD; + LLInventoryModel::incrBulkFetch(1); + LLHTTPClient::post(url, body, new LLInventoryModel::fetchDescendentsResponder(body),300); + } + } + else + { + if (isBulkFetchProcessingComplete()) + { + if (sFullFetchStarted) + { + sAllFoldersFetched = TRUE; + } + stopBackgroundFetch(); + } + } + sRetryDialog=NULL; + sRetrySD.clear(); +} + +//static Bundle up a bunch of requests to send all at once. +void LLInventoryModel::bulkFetch(std::string url) +{ + //Background fetch is called from gIdleCallbacks in a loop until background fetch is stopped. + //If there are items in sFetchQueue, we want to check the time since the last bulkFetch was + //sent. If it exceeds our retry time, go ahead and fire off another batch. + //Stopbackgroundfetch will be run from the Responder instead of here. + + S16 max_concurrent_fetches=8; + F32 new_min_time = 0.5f; //HACK! Clean this up when old code goes away entirely. + if (sMinTimeBetweenFetches <= new_min_time) sMinTimeBetweenFetches=new_min_time; //HACK! See above. + + if(gDisconnected + || sBulkFetchCount > max_concurrent_fetches + || sFetchTimer.getElapsedTimeF32() < sMinTimeBetweenFetches) + { + return; // just bail if we are disconnected. + } + + //HACK. This is inelegant. We're shuffling a dequeue to a map to get rid of + //redundant requests. When we get rid of the old code entirely, we can change + //the dequeue to a map. In the new model, there is no benefit to queue order. + U32 folder_count=0; + U32 max_batch_size=10; + while( !(sFetchQueue.empty() ) ) + { + LLViewerInventoryCategory* cat = gInventory.getCategory(sFetchQueue.front()); + + if (cat) + { + if ( !gInventory.isCategoryComplete(cat->getUUID()) ) //grab this folder. + { + sBulkFetchMap[(cat->getUUID())] = cat; + } + else if (sFullFetchStarted) + { //Already have this folder but append child folders to list. + // add all children to queue + parent_cat_map_t::iterator cat_it = gInventory.mParentChildCategoryTree.find(cat->getUUID()); + if (cat_it != gInventory.mParentChildCategoryTree.end()) + { + cat_array_t* child_categories = cat_it->second; + + for (S32 child_num = 0; child_num < child_categories->count(); child_num++) + { + sFetchQueue.push_back(child_categories->get(child_num)->getUUID()); + } + } + + } + } + sFetchQueue.pop_front(); + } + + + if (!sBulkFetchMap.empty()) //There's stuff to fetch. + { + U32 sort_order = gSavedSettings.getU32("InventorySortOrder") & 0x1; + + LLSD body; + + cat_map_t::iterator iter=sBulkFetchMap.begin(); + while( iter!=sBulkFetchMap.end() && (folder_count < max_batch_size) ) + { + LLViewerInventoryCategory* cat = iter->second; + + if (cat && !gInventory.isCategoryComplete(cat->getUUID()) ) //Category exists + { + BOOL fetchItems=TRUE; + if ( sFullFetchStarted + && gInventory.isCategoryComplete(cat->getUUID()) ) + { + fetchItems=FALSE; + } + + LLSD folder_sd; + folder_sd["folder-id"] = cat->getUUID(); + folder_sd["owner-id"] = cat->getOwnerID(); + folder_sd["sort-order"] = (LLSD::Integer)sort_order; + folder_sd["fetch-folders"] = (LLSD::Boolean)sFullFetchStarted; + folder_sd["fetch-items"] = (LLSD::Boolean)fetchItems; + body["folders"].append(folder_sd); + + folder_count++; + } + sBulkFetchMap.erase(iter); + iter=sBulkFetchMap.begin(); + } + + if (iter == sBulkFetchMap.end()) sBulkFetchMap.clear(); + + if (folder_count > 0) + { + sBulkFetchCount++; + + LLHTTPClient::post(url, body, new LLInventoryModel::fetchDescendentsResponder(body)); + sFetchTimer.reset(); + } + + } + + if (isBulkFetchProcessingComplete()) + { + if (sFullFetchStarted) + { + sAllFoldersFetched = TRUE; + } + stopBackgroundFetch(); + } +} + // static bool LLInventoryModel::isEverythingFetched() { @@ -1049,6 +1334,9 @@ void LLInventoryModel::stopBackgroundFetch() { sBackgroundFetchActive = FALSE; gIdleCallbacks.deleteFunction(&LLInventoryModel::backgroundFetch, NULL); + sBulkFetchCount=0; + sMinTimeBetweenFetches=0.0f; +// sFullFetchStarted=FALSE; } } @@ -1057,6 +1345,15 @@ void LLInventoryModel::backgroundFetch(void*) { if (sBackgroundFetchActive) { + //If we'll be using the capability, we'll be sending batches and the background thing isn't as important. + std::string url = gAgent.getRegion()->getCapability("FetchInventoryDescendents"); + if (!url.empty()) + { + bulkFetch(url); + return; + } + + //DEPRECATED OLD CODE FOLLOWS. // no more categories to fetch, stop fetch process if (sFetchQueue.empty()) { @@ -3063,8 +3360,8 @@ void LLInventoryFetchDescendentsObserver::fetchDescendents( if(!cat) continue; if(!isComplete(cat)) { - cat->fetchDescendents(); - mIncompleteFolders.push_back(*it); + cat->fetchDescendents(); //blindly fetch it without seeing if anything else is fetching it. + mIncompleteFolders.push_back(*it); //Add to list of things being downloaded for this observer. } else { diff --git a/indra/newview/llinventorymodel.h b/indra/newview/llinventorymodel.h index b560aea8b2..79a35f78ea 100644 --- a/indra/newview/llinventorymodel.h +++ b/indra/newview/llinventorymodel.h @@ -91,6 +91,7 @@ class LLViewerInventoryItem; class LLViewerInventoryCategory; class LLMessageSystem; class LLInventoryCollectFunctor; +class LLAlertDialog; class LLInventoryModel { @@ -105,11 +106,26 @@ public: // These are used a lot... typedef LLDynamicArray<LLPointer<LLViewerInventoryCategory> > cat_array_t; typedef LLDynamicArray<LLPointer<LLViewerInventoryItem> > item_array_t; - // construction & destruction LLInventoryModel(); ~LLInventoryModel(); + class fetchDescendentsResponder: public LLHTTPClient::Responder + { + public: + fetchDescendentsResponder(const LLSD& request_sd) : mRequestSD(request_sd) {}; + void result(const LLSD& content); + void error(U32 status, const std::string& reason); + static void onClickRetry(S32 option, void* userdata); + static void appendRetryList(LLSD retry_sd); + public: + typedef std::vector<LLViewerInventoryCategory*> folder_ref_t; + protected: + LLSD mRequestSD; + static LLSD sRetrySD; + static LLAlertDialog *sRetryDialog; + }; + // // Accessors // @@ -263,6 +279,9 @@ public: // make sure we have the descendents in the structure. void fetchDescendentsOf(const LLUUID& folder_id); + + // Add categories to a list to be fetched in bulk. + static void bulkFetch(std::string url); // call this method to request the inventory. //void requestFromServer(const LLUUID& agent_id); @@ -348,7 +367,7 @@ public: static BOOL backgroundFetchActive(); static bool isEverythingFetched(); static void backgroundFetch(void*); // background fetch idle function - + static void incrBulkFetch(S16 fetching) { sBulkFetchCount+=fetching; if (sBulkFetchCount<0) sBulkFetchCount=0; } protected: // Internal methods which add inventory and make sure that all of @@ -395,7 +414,8 @@ protected: static void processInventoryDescendents(LLMessageSystem* msg, void**); static void processMoveInventoryItem(LLMessageSystem* msg, void**); static void processFetchInventoryReply(LLMessageSystem* msg, void**); - + static bool isBulkFetchProcessingComplete(); + bool messageUpdateCore(LLMessageSystem* msg, bool do_accounting); protected: @@ -430,6 +450,7 @@ protected: observer_list_t mObservers; // completing the fetch once per session should be sufficient + static cat_map_t sBulkFetchMap; static BOOL sBackgroundFetchActive; static BOOL sTimelyFetchPending; static BOOL sAllFoldersFetched; @@ -438,6 +459,7 @@ protected: static LLFrameTimer sFetchTimer; static F32 sMinTimeBetweenFetches; static F32 sMaxTimeBetweenFetches; + static S16 sBulkFetchCount; // This flag is used to handle an invalid inventory state. bool mIsAgentInvUsable; diff --git a/indra/newview/llviewerinventory.cpp b/indra/newview/llviewerinventory.cpp index 981605d1fa..01feff9b3c 100644 --- a/indra/newview/llviewerinventory.cpp +++ b/indra/newview/llviewerinventory.cpp @@ -48,6 +48,8 @@ #include "llviewerregion.h" #include "llviewerobjectlist.h" #include "llpreviewgesture.h" +#include "llviewerwindow.h" + ///---------------------------------------------------------------------------- /// Local function declarations, constants, enums, and typedefs ///---------------------------------------------------------------------------- @@ -213,6 +215,14 @@ void LLViewerInventoryItem::fetchFromServer(void) const } // virtual +BOOL LLViewerInventoryItem::unpackMessage(LLSD item) +{ + BOOL rv = LLInventoryItem::fromLLSD(item); + mIsComplete = TRUE; + return rv; +} + +// virtual BOOL LLViewerInventoryItem::unpackMessage( LLMessageSystem* msg, const char* block, S32 block_num) { @@ -420,30 +430,42 @@ void LLViewerInventoryCategory::removeFromServer( void ) bool LLViewerInventoryCategory::fetchDescendents() { if((VERSION_UNKNOWN == mVersion) - && mDescendentsRequested.hasExpired()) + && mDescendentsRequested.hasExpired()) //Expired check prevents multiple downloads. { const F32 FETCH_TIMER_EXPIRY = 10.0f; mDescendentsRequested.reset(); mDescendentsRequested.setTimerExpirySec(FETCH_TIMER_EXPIRY); - LLMessageSystem* msg = gMessageSystem; - msg->newMessage("FetchInventoryDescendents"); - msg->nextBlock("AgentData"); - msg->addUUID("AgentID", gAgent.getID()); - msg->addUUID("SessionID", gAgent.getSessionID()); - msg->nextBlock("InventoryData"); - msg->addUUID("FolderID", mUUID); - msg->addUUID("OwnerID", mOwnerID); // bitfield // 1 = by date // 2 = folders by date // Need to mask off anything but the first bit. // This comes from LLInventoryFilter from llfolderview.h U32 sort_order = gSavedSettings.getU32("InventorySortOrder") & 0x1; - msg->addS32("SortOrder", sort_order); - msg->addBOOL("FetchFolders", FALSE); - msg->addBOOL("FetchItems", TRUE); - gAgent.sendReliableMessage(); + + std::string url = gAgent.getRegion()->getCapability("FetchInventoryDescendents"); + + if (!url.empty()) //Capability found. Build up LLSD and use it. + { + LLInventoryModel::startBackgroundFetch(mUUID); + } + else + { //Deprecated, but if we don't have a capability, use the old system. + llinfos << "FetchInventoryDescendents capability not found. Using deprecated UDP message." << llendl; + LLMessageSystem* msg = gMessageSystem; + msg->newMessage("FetchInventoryDescendents"); + msg->nextBlock("AgentData"); + msg->addUUID("AgentID", gAgent.getID()); + msg->addUUID("SessionID", gAgent.getSessionID()); + msg->nextBlock("InventoryData"); + msg->addUUID("FolderID", mUUID); + msg->addUUID("OwnerID", mOwnerID); + + msg->addS32("SortOrder", sort_order); + msg->addBOOL("FetchFolders", FALSE); + msg->addBOOL("FetchItems", TRUE); + gAgent.sendReliableMessage(); + } return true; } return false; diff --git a/indra/newview/llviewerinventory.h b/indra/newview/llviewerinventory.h index fd6928243b..bf49a1604f 100644 --- a/indra/newview/llviewerinventory.h +++ b/indra/newview/llviewerinventory.h @@ -99,6 +99,7 @@ public: //virtual void packMessage(LLMessageSystem* msg) const; virtual BOOL unpackMessage(LLMessageSystem* msg, const char* block, S32 block_num = 0); + virtual BOOL unpackMessage(LLSD item); virtual BOOL importFile(FILE* fp); virtual BOOL importLegacyStream(std::istream& input_stream); diff --git a/indra/newview/llviewerregion.cpp b/indra/newview/llviewerregion.cpp index 3f0f5bee98..42654e250b 100644 --- a/indra/newview/llviewerregion.cpp +++ b/indra/newview/llviewerregion.cpp @@ -1378,6 +1378,7 @@ void LLViewerRegion::setSeedCapability(const std::string& url) capabilityNames.append("DispatchRegionInfo"); capabilityNames.append("EstateChangeInfo"); capabilityNames.append("EventQueueGet"); + capabilityNames.append("FetchInventoryDescendents"); capabilityNames.append("GroupProposalBallot"); capabilityNames.append("MapLayer"); capabilityNames.append("MapLayerGod"); diff --git a/indra/newview/llxmlrpctransaction.cpp b/indra/newview/llxmlrpctransaction.cpp index 3df2073c6a..9dc92efa81 100644 --- a/indra/newview/llxmlrpctransaction.cpp +++ b/indra/newview/llxmlrpctransaction.cpp @@ -33,10 +33,10 @@ #include "llxmlrpctransaction.h" +#include "llcurl.h" #include "llviewercontrol.h" // Have to include these last to avoid queue redefinition! -#include <curl/curl.h> #include <xmlrpc-epi/xmlrpc.h> #include "llappviewer.h" @@ -150,51 +150,48 @@ class LLXMLRPCTransaction::Impl { public: typedef LLXMLRPCTransaction::Status Status; - - CURL* mCurl; - CURLM* mCurlMulti; + + LLCurlEasyRequest* mCurlRequest; Status mStatus; CURLcode mCurlCode; std::string mStatusMessage; std::string mStatusURI; + LLCurl::TransferInfo mTransferInfo; - char mCurlErrorBuffer[CURL_ERROR_SIZE]; /* Flawfinder: ignore */ - std::string mURI; char* mRequestText; int mRequestTextSize; std::string mProxyAddress; - struct curl_slist* mHeaders; std::string mResponseText; XMLRPC_REQUEST mResponse; Impl(const std::string& uri, XMLRPC_REQUEST request, bool useGzip); Impl(const std::string& uri, - const std::string& method, LLXMLRPCValue params, bool useGzip); + const std::string& method, LLXMLRPCValue params, bool useGzip); ~Impl(); bool process(); void setStatus(Status code, - const std::string& message = "", const std::string& uri = ""); + const std::string& message = "", const std::string& uri = ""); void setCurlStatus(CURLcode); private: void init(XMLRPC_REQUEST request, bool useGzip); static size_t curlDownloadCallback( - void* data, size_t size, size_t nmemb, void* user_data); + char* data, size_t size, size_t nmemb, void* user_data); }; LLXMLRPCTransaction::Impl::Impl(const std::string& uri, XMLRPC_REQUEST request, bool useGzip) - : mCurl(0), mCurlMulti(0), + : mCurlRequest(0), mStatus(LLXMLRPCTransaction::StatusNotStarted), mURI(uri), - mRequestText(0), mHeaders(0), + mRequestText(0), mResponse(0) { init(request, useGzip); @@ -203,10 +200,10 @@ LLXMLRPCTransaction::Impl::Impl(const std::string& uri, LLXMLRPCTransaction::Impl::Impl(const std::string& uri, const std::string& method, LLXMLRPCValue params, bool useGzip) - : mCurl(0), mCurlMulti(0), + : mCurlRequest(0), mStatus(LLXMLRPCTransaction::StatusNotStarted), mURI(uri), - mRequestText(0), mHeaders(0), + mRequestText(0), mResponse(0) { XMLRPC_REQUEST request = XMLRPC_RequestNew(); @@ -222,55 +219,53 @@ LLXMLRPCTransaction::Impl::Impl(const std::string& uri, void LLXMLRPCTransaction::Impl::init(XMLRPC_REQUEST request, bool useGzip) { - mCurl = curl_easy_init(); - + if (!mCurlRequest) + { + mCurlRequest = new LLCurlEasyRequest(); + } + if (gSavedSettings.getBOOL("BrowserProxyEnabled")) { mProxyAddress = gSavedSettings.getString("BrowserProxyAddress"); S32 port = gSavedSettings.getS32 ( "BrowserProxyPort" ); // tell curl about the settings - curl_easy_setopt(mCurl, CURLOPT_PROXY, mProxyAddress.c_str()); - curl_easy_setopt(mCurl, CURLOPT_PROXYPORT, (long) port); - curl_easy_setopt(mCurl, CURLOPT_PROXYTYPE, (long) CURLPROXY_HTTP); - }; - -// curl_easy_setopt(mCurl, CURLOPT_VERBOSE, 1L); // usefull for debugging - curl_easy_setopt(mCurl, CURLOPT_NOSIGNAL, 1L); - curl_easy_setopt(mCurl, CURLOPT_WRITEFUNCTION, &curlDownloadCallback); - curl_easy_setopt(mCurl, CURLOPT_WRITEDATA, this); - curl_easy_setopt(mCurl, CURLOPT_ERRORBUFFER, &mCurlErrorBuffer); - curl_easy_setopt(mCurl, CURLOPT_CAINFO, gDirUtilp->getCAFile().c_str()); - curl_easy_setopt(mCurl, CURLOPT_SSL_VERIFYPEER, (long) gVerifySSLCert); - curl_easy_setopt(mCurl, CURLOPT_SSL_VERIFYHOST, gVerifySSLCert? 2L : 0L); + mCurlRequest->setoptString(CURLOPT_PROXY, mProxyAddress); + mCurlRequest->setopt(CURLOPT_PROXYPORT, port); + mCurlRequest->setopt(CURLOPT_PROXYTYPE, CURLPROXY_HTTP); + } + +// mCurlRequest->setopt(CURLOPT_VERBOSE, 1); // usefull for debugging + mCurlRequest->setopt(CURLOPT_NOSIGNAL, 1); + mCurlRequest->setWriteCallback(&curlDownloadCallback, (void*)this); + mCurlRequest->setopt(CURLOPT_SSL_VERIFYPEER, gVerifySSLCert); + mCurlRequest->setopt(CURLOPT_SSL_VERIFYHOST, gVerifySSLCert? 2 : 0); // Be a little impatient about establishing connections. - curl_easy_setopt(mCurl, CURLOPT_CONNECTTIMEOUT, 40L); + mCurlRequest->setopt(CURLOPT_CONNECTTIMEOUT, 40L); /* Setting the DNS cache timeout to -1 disables it completely. This might help with bug #503 */ - curl_easy_setopt(mCurl, CURLOPT_DNS_CACHE_TIMEOUT, -1L); + mCurlRequest->setopt(CURLOPT_DNS_CACHE_TIMEOUT, -1); + + mCurlRequest->slist_append("Content-Type: text/xml"); - mHeaders = curl_slist_append(mHeaders, "Content-Type: text/xml"); - curl_easy_setopt(mCurl, CURLOPT_URL, mURI.c_str()); - curl_easy_setopt(mCurl, CURLOPT_HTTPHEADER, mHeaders); if (useGzip) { - curl_easy_setopt(mCurl, CURLOPT_ENCODING, ""); + mCurlRequest->setoptString(CURLOPT_ENCODING, ""); } mRequestText = XMLRPC_REQUEST_ToXML(request, &mRequestTextSize); if (mRequestText) { - curl_easy_setopt(mCurl, CURLOPT_POSTFIELDS, mRequestText); - curl_easy_setopt(mCurl, CURLOPT_POSTFIELDSIZE, (long) mRequestTextSize); + mCurlRequest->setoptString(CURLOPT_POSTFIELDS, mRequestText); + mCurlRequest->setopt(CURLOPT_POSTFIELDSIZE, mRequestTextSize); } else { setStatus(StatusOtherError); } - - mCurlMulti = curl_multi_init(); - curl_multi_add_handle(mCurlMulti, mCurl); + + mCurlRequest->sendRequest(mURI); } @@ -281,30 +276,12 @@ LLXMLRPCTransaction::Impl::~Impl() XMLRPC_RequestFree(mResponse, 1); } - if (mHeaders) - { - curl_slist_free_all(mHeaders); - } - if (mRequestText) { XMLRPC_Free(mRequestText); } - if (mCurl) - { - if (mCurlMulti) - { - curl_multi_remove_handle(mCurlMulti, mCurl); - } - curl_easy_cleanup(mCurl); - } - - if (mCurlMulti) - { - curl_multi_cleanup(mCurlMulti); - } - + delete mCurlRequest; } bool LLXMLRPCTransaction::Impl::process() @@ -333,27 +310,28 @@ bool LLXMLRPCTransaction::Impl::process() const F32 MAX_PROCESSING_TIME = 0.05f; LLTimer timer; - int count; - - while (CURLM_CALL_MULTI_PERFORM == curl_multi_perform(mCurlMulti, &count)) + + while (mCurlRequest->perform() > 0) { if (timer.getElapsedTimeF32() >= MAX_PROCESSING_TIME) { return false; } } - - while(CURLMsg* curl_msg = curl_multi_info_read(mCurlMulti, &count)) + + while(1) { - if (CURLMSG_DONE == curl_msg->msg) + CURLcode result; + bool newmsg = mCurlRequest->getResult(&result, &mTransferInfo); + if (newmsg) { - if (curl_msg->data.result != CURLE_OK) + if (result != CURLE_OK) { - setCurlStatus(curl_msg->data.result); + setCurlStatus(result); llwarns << "LLXMLRPCTransaction CURL error " - << mCurlCode << ": " << mCurlErrorBuffer << llendl; + << mCurlCode << ": " << mCurlRequest->getErrorString() << llendl; llwarns << "LLXMLRPCTransaction request URI: " - << mURI << llendl; + << mURI << llendl; return true; } @@ -361,7 +339,7 @@ bool LLXMLRPCTransaction::Impl::process() setStatus(LLXMLRPCTransaction::StatusComplete); mResponse = XMLRPC_REQUEST_FromXML( - mResponseText.data(), mResponseText.size(), NULL); + mResponseText.data(), mResponseText.size(), NULL); bool hasError = false; bool hasFault = false; @@ -387,15 +365,19 @@ bool LLXMLRPCTransaction::Impl::process() setStatus(LLXMLRPCTransaction::StatusXMLRPCError); llwarns << "LLXMLRPCTransaction XMLRPC " - << (hasError ? "error " : "fault ") - << faultCode << ": " - << faultString << llendl; + << (hasError ? "error " : "fault ") + << faultCode << ": " + << faultString << llendl; llwarns << "LLXMLRPCTransaction request URI: " - << mURI << llendl; + << mURI << llendl; } return true; } + else + { + break; // done + } } return false; @@ -504,13 +486,13 @@ void LLXMLRPCTransaction::Impl::setCurlStatus(CURLcode code) } size_t LLXMLRPCTransaction::Impl::curlDownloadCallback( - void* data, size_t size, size_t nmemb, void* user_data) + char* data, size_t size, size_t nmemb, void* user_data) { Impl& impl(*(Impl*)user_data); size_t n = size * nmemb; - impl.mResponseText.append((const char*)data, n); + impl.mResponseText.append(data, n); if (impl.mStatus == LLXMLRPCTransaction::StatusStarted) { @@ -579,25 +561,17 @@ LLXMLRPCValue LLXMLRPCTransaction::responseValue() F64 LLXMLRPCTransaction::transferRate() { - if (!impl.mCurl || impl.mStatus != StatusComplete) + if (impl.mStatus != StatusComplete) { return 0.0L; } - double size_bytes = 0.0; - double time_seconds = 0.0; - double rate_bytes_per_sec = 0.0; - - curl_easy_getinfo(impl.mCurl, CURLINFO_SIZE_DOWNLOAD, &size_bytes); - curl_easy_getinfo(impl.mCurl, CURLINFO_TOTAL_TIME, &time_seconds); - curl_easy_getinfo(impl.mCurl, CURLINFO_SPEED_DOWNLOAD, &rate_bytes_per_sec); - - double rate_bits_per_sec = rate_bytes_per_sec * 8.0; + double rate_bits_per_sec = impl.mTransferInfo.mSpeedDownload * 8.0; llinfos << "Buffer size: " << impl.mResponseText.size() << " B" << llendl; - llinfos << "Transfer size: " << size_bytes << " B" << llendl; - llinfos << "Transfer time: " << time_seconds << " s" << llendl; - llinfos << "Transfer rate: " << rate_bits_per_sec/1000.0 << " Kb/s" << llendl; + llinfos << "Transfer size: " << impl.mTransferInfo.mSizeDownload << " B" << llendl; + llinfos << "Transfer time: " << impl.mTransferInfo.mTotalTime << " s" << llendl; + llinfos << "Transfer rate: " << rate_bits_per_sec / 1000.0 << " Kb/s" << llendl; return rate_bits_per_sec; } |