diff options
Diffstat (limited to 'indra/llmessage/llcurl.cpp')
-rwxr-xr-x[-rw-r--r--] | indra/llmessage/llcurl.cpp | 346 |
1 files changed, 269 insertions, 77 deletions
diff --git a/indra/llmessage/llcurl.cpp b/indra/llmessage/llcurl.cpp index 8ffa8e4271..73df47b933 100644..100755 --- a/indra/llmessage/llcurl.cpp +++ b/indra/llmessage/llcurl.cpp @@ -6,7 +6,7 @@ * * $LicenseInfo:firstyear=2006&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2010, Linden Research, Inc. + * Copyright (C) 2010-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -49,6 +49,7 @@ #include "llproxy.h" #include "llsdserialize.h" #include "llstl.h" +#include "llstring.h" #include "llthread.h" #include "lltimer.h" @@ -72,7 +73,8 @@ static const U32 EASY_HANDLE_POOL_SIZE = 5; static const S32 MULTI_PERFORM_CALL_REPEAT = 5; -static const S32 CURL_REQUEST_TIMEOUT = 30; // seconds per operation +static const S32 CURL_REQUEST_TIMEOUT = 120; // seconds per operation +static const S32 CURL_CONNECT_TIMEOUT = 30; //seconds to wait for a connection static const S32 MAX_ACTIVE_REQUEST_COUNT = 100; // DEBUG // @@ -91,6 +93,7 @@ S32 LLCurl::sTotalHandles = 0 ; bool LLCurl::sNotQuitting = true; F32 LLCurl::sCurlRequestTimeOut = 120.f; //seonds S32 LLCurl::sMaxHandles = 256; //max number of handles, (multi handles and easy handles combined). +CURL* LLCurl::sCurlTemplateStandardHandle = NULL; void check_curl_code(CURLcode code) { @@ -98,7 +101,7 @@ void check_curl_code(CURLcode code) { // linux appears to throw a curl error once per session for a bad initialization // at a pretty random time (when enabling cookies). - llinfos << "curl error detected: " << curl_easy_strerror(code) << llendl; + LL_WARNS("curl") << "curl error detected: " << curl_easy_strerror(code) << LL_ENDL; } } @@ -108,7 +111,7 @@ void check_curl_multi_code(CURLMcode code) { // linux appears to throw a curl error once per session for a bad initialization // at a pretty random time (when enabling cookies). - llinfos << "curl multi error detected: " << curl_multi_strerror(code) << llendl; + LL_WARNS("curl") << "curl multi error detected: " << curl_multi_strerror(code) << LL_ENDL; } } @@ -133,6 +136,7 @@ std::string LLCurl::getVersionString() ////////////////////////////////////////////////////////////////////////////// LLCurl::Responder::Responder() + : mHTTPMethod(HTTP_INVALID), mStatus(HTTP_INTERNAL_ERROR) { } @@ -142,22 +146,30 @@ LLCurl::Responder::~Responder() } // virtual -void LLCurl::Responder::errorWithContent( - U32 status, - const std::string& reason, - const LLSD&) +void LLCurl::Responder::httpFailure() { - error(status, reason); + LL_WARNS("curl") << dumpResponse() << LL_ENDL; } -// virtual -void LLCurl::Responder::error(U32 status, const std::string& reason) +std::string LLCurl::Responder::dumpResponse() const { - llinfos << mURL << " [" << status << "]: " << reason << llendl; + std::ostringstream s; + s << "[" << httpMethodAsVerb(mHTTPMethod) << ":" << mURL << "] " + << "[status:" << mStatus << "] " + << "[reason:" << mReason << "] "; + + if (mResponseHeaders.has(HTTP_IN_HEADER_CONTENT_TYPE)) + { + s << "[content-type:" << mResponseHeaders[HTTP_IN_HEADER_CONTENT_TYPE] << "] "; + } + + s << "[content:" << mContent << "]"; + + return s.str(); } // virtual -void LLCurl::Responder::result(const LLSD& content) +void LLCurl::Responder::httpSuccess() { } @@ -166,42 +178,109 @@ void LLCurl::Responder::setURL(const std::string& url) mURL = url; } +void LLCurl::Responder::successResult(const LLSD& content) +{ + setResult(HTTP_OK, "", content); + httpSuccess(); +} + +void LLCurl::Responder::failureResult(S32 status, const std::string& reason, const LLSD& content /* = LLSD() */) +{ + setResult(status, reason, content); + httpFailure(); +} + +void LLCurl::Responder::completeResult(S32 status, const std::string& reason, const LLSD& content /* = LLSD() */) +{ + setResult(status, reason, content); + httpCompleted(); +} + +void LLCurl::Responder::setResult(S32 status, const std::string& reason, const LLSD& content /* = LLSD() */) +{ + mStatus = status; + mReason = reason; + mContent = content; +} + +void LLCurl::Responder::setHTTPMethod(EHTTPMethod method) +{ + mHTTPMethod = method; +} + +void LLCurl::Responder::setResponseHeader(const std::string& header, const std::string& value) +{ + mResponseHeaders[header] = value; +} + +const std::string& LLCurl::Responder::getResponseHeader(const std::string& header) const +{ + if (mResponseHeaders.has(header)) + { + return mResponseHeaders[header].asStringRef(); + } + static const std::string empty; + return empty; +} + +bool LLCurl::Responder::hasResponseHeader(const std::string& header) const +{ + if (mResponseHeaders.has(header)) return true; + return false; +} + // virtual void LLCurl::Responder::completedRaw( - U32 status, - const std::string& reason, const LLChannelDescriptors& channels, const LLIOPipe::buffer_ptr_t& buffer) { - LLSD content; LLBufferStream istr(channels, buffer.get()); - if (!LLSDSerialize::fromXML(content, istr)) + const bool emit_parse_errors = false; + + std::string debug_body("(empty)"); + bool parsed=true; + if (EOF == istr.peek()) { - llinfos << "Failed to deserialize LLSD. " << mURL << " [" << status << "]: " << reason << llendl; + parsed=false; + } + // Try to parse body as llsd, no matter what 'content-type' says. + else if (LLSDParser::PARSE_FAILURE == LLSDSerialize::fromXML(mContent, istr, emit_parse_errors)) + { + parsed=false; + char body[1025]; + body[1024] = '\0'; + istr.seekg(0, std::ios::beg); + istr.get(body,1024); + if (strlen(body) > 0) + { + mContent = body; + debug_body = body; + } } - completed(status, reason, content); + // Only emit a warning if we failed to parse when 'content-type' == 'application/llsd+xml' + if (!parsed && (HTTP_CONTENT_LLSD_XML == getResponseHeader(HTTP_IN_HEADER_CONTENT_TYPE))) + { + LL_WARNS() << "Failed to deserialize . " << mURL << " [status:" << mStatus << "] " + << "(" << mReason << ") body: " << debug_body << LL_ENDL; + } + + httpCompleted(); } // virtual -void LLCurl::Responder::completed(U32 status, const std::string& reason, const LLSD& content) +void LLCurl::Responder::httpCompleted() { - if (isGoodStatus(status)) + if (isGoodStatus()) { - result(content); + httpSuccess(); } else { - errorWithContent(status, reason, content); + httpFailure(); } } -//virtual -void LLCurl::Responder::completedHeader(U32 status, const std::string& reason, const LLSD& content) -{ - -} - ////////////////////////////////////////////////////////////////////////////// std::set<CURL*> LLCurl::Easy::sFreeHandles; @@ -244,7 +323,7 @@ void LLCurl::Easy::releaseEasyHandle(CURL* handle) if (!handle) { return ; //handle allocation failed. - //llerrs << "handle cannot be NULL!" << llendl; + //LL_ERRS() << "handle cannot be NULL!" << LL_ENDL; } LLMutexLock lock(sHandleMutexp) ; @@ -266,10 +345,40 @@ void LLCurl::Easy::releaseEasyHandle(CURL* handle) } else { - llerrs << "Invalid handle." << llendl; + LL_ERRS() << "Invalid handle." << LL_ENDL; } } +//static +void LLCurl::Easy::deleteAllActiveHandles() +{ + LLMutexLock lock(sHandleMutexp) ; + LL_CHECK_MEMORY + for (std::set<CURL*>::iterator activeHandle = sActiveHandles.begin(); activeHandle != sActiveHandles.end(); ++activeHandle) + { + CURL* curlHandle = *activeHandle; + LLCurl::deleteEasyHandle(curlHandle); + LL_CHECK_MEMORY + } + + sFreeHandles.clear(); +} + +//static +void LLCurl::Easy::deleteAllFreeHandles() +{ + LLMutexLock lock(sHandleMutexp) ; + LL_CHECK_MEMORY + for (std::set<CURL*>::iterator freeHandle = sFreeHandles.begin(); freeHandle != sFreeHandles.end(); ++freeHandle) + { + CURL* curlHandle = *freeHandle; + LLCurl::deleteEasyHandle(curlHandle); + LL_CHECK_MEMORY + } + + sFreeHandles.clear(); +} + LLCurl::Easy::Easy() : mHeaders(NULL), mCurlEasyHandle(NULL) @@ -285,14 +394,18 @@ LLCurl::Easy* LLCurl::Easy::getEasy() if (!easy->mCurlEasyHandle) { // this can happen if we have too many open files (fails in c-ares/ares_init.c) - llwarns << "allocEasyHandle() returned NULL! Easy handles: " << gCurlEasyCount << " Multi handles: " << gCurlMultiCount << llendl; + LL_WARNS("curl") << "allocEasyHandle() returned NULL! Easy handles: " + << gCurlEasyCount << " Multi handles: " << gCurlMultiCount << LL_ENDL; delete easy; return NULL; } - // set no DNS caching as default for all easy handles. This prevents them adopting a - // multi handles cache if they are added to one. - CURLcode result = curl_easy_setopt(easy->mCurlEasyHandle, CURLOPT_DNS_CACHE_TIMEOUT, 0); + // Enable a brief cache period for now. This was zero for the longest time + // which caused some routers grief and generated unneeded traffic. For the + // threaded resolver, we're using system resolution libraries and non-zero values + // are preferred. The c-ares resolver is another matter and it might not + // track server changes as well. + CURLcode result = curl_easy_setopt(easy->mCurlEasyHandle, CURLOPT_DNS_CACHE_TIMEOUT, 15); check_curl_code(result); result = curl_easy_setopt(easy->mCurlEasyHandle, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4); check_curl_code(result); @@ -310,10 +423,14 @@ LLCurl::Easy::~Easy() for_each(mStrings.begin(), mStrings.end(), DeletePointerArray()); LL_CHECK_MEMORY if (mResponder && LLCurl::sNotQuitting) //aborted - { - std::string reason("Request timeout, aborted.") ; - mResponder->completedRaw(408, //HTTP_REQUEST_TIME_OUT, timeout, abort - reason, mChannels, mOutput); + { + // HTTP_REQUEST_TIME_OUT, timeout, abort + // *TODO: This looks like improper use of the 408 status code. + // See: http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.4.9 + // This status code should be returned by the *server* when: + // "The client did not produce a request within the time that the server was prepared to wait." + mResponder->setResult(HTTP_REQUEST_TIME_OUT, "Request timeout, aborted."); + mResponder->completedRaw(mChannels, mOutput); LL_CHECK_MEMORY } mResponder = NULL; @@ -377,9 +494,9 @@ void LLCurl::Easy::getTransferInfo(LLCurl::TransferInfo* info) check_curl_code(curl_easy_getinfo(mCurlEasyHandle, CURLINFO_SPEED_DOWNLOAD, &info->mSpeedDownload)); } -U32 LLCurl::Easy::report(CURLcode code) +S32 LLCurl::Easy::report(CURLcode code) { - U32 responseCode = 0; + S32 responseCode = 0; std::string responseReason; if (code == CURLE_OK) @@ -389,14 +506,15 @@ U32 LLCurl::Easy::report(CURLcode code) } else { - responseCode = 499; + responseCode = HTTP_INTERNAL_ERROR; responseReason = strerror(code) + " : " + mErrorBuffer; setopt(CURLOPT_FRESH_CONNECT, TRUE); } if (mResponder) { - mResponder->completedRaw(responseCode, responseReason, mChannels, mOutput); + mResponder->setResult(responseCode, responseReason); + mResponder->completedRaw(mChannels, mOutput); mResponder = NULL; } @@ -433,9 +551,31 @@ void LLCurl::Easy::setoptString(CURLoption option, const std::string& value) check_curl_code(result); } +void LLCurl::Easy::slist_append(const std::string& header, const std::string& value) +{ + std::string pair(header); + if (value.empty()) + { + pair += ":"; + } + else + { + pair += ": "; + pair += value; + } + slist_append(pair.c_str()); +} + void LLCurl::Easy::slist_append(const char* str) { - mHeaders = curl_slist_append(mHeaders, str); + if (str) + { + mHeaders = curl_slist_append(mHeaders, str); + if (!mHeaders) + { + LL_WARNS() << "curl_slist_append() call returned NULL appending " << str << LL_ENDL; + } + } } size_t curlReadCallback(char* data, size_t size, size_t nmemb, void* user_data) @@ -515,6 +655,7 @@ void LLCurl::Easy::prepRequest(const std::string& url, //don't verify host name so urls with scrubbed host names will work (improves DNS performance) setopt(CURLOPT_SSL_VERIFYHOST, 0); setopt(CURLOPT_TIMEOUT, llmax(time_out, CURL_REQUEST_TIMEOUT)); + setopt(CURLOPT_CONNECTTIMEOUT, CURL_CONNECT_TIMEOUT); setoptString(CURLOPT_URL, url); @@ -522,8 +663,9 @@ void LLCurl::Easy::prepRequest(const std::string& url, if (!post) { - slist_append("Connection: keep-alive"); - slist_append("Keep-alive: 300"); + // *TODO: Should this be set to 'Keep-Alive' ? + slist_append(HTTP_OUT_HEADER_CONNECTION, "keep-alive"); + slist_append(HTTP_OUT_HEADER_KEEP_ALIVE, "300"); // Accept and other headers for (std::vector<std::string>::const_iterator iter = headers.begin(); iter != headers.end(); ++iter) @@ -547,7 +689,7 @@ LLCurl::Multi::Multi(F32 idle_time_out) mCurlMultiHandle = LLCurl::newMultiHandle(); if (!mCurlMultiHandle) { - llwarns << "curl_multi_init() returned NULL! Easy handles: " << gCurlEasyCount << " Multi handles: " << gCurlMultiCount << llendl; + LL_WARNS() << "curl_multi_init() returned NULL! Easy handles: " << gCurlEasyCount << " Multi handles: " << gCurlMultiCount << LL_ENDL; mCurlMultiHandle = LLCurl::newMultiHandle(); } @@ -802,7 +944,7 @@ S32 LLCurl::Multi::process() ++processed; if (msg->msg == CURLMSG_DONE) { - U32 response = 0; + S32 response = 0; Easy* easy = NULL ; { @@ -821,9 +963,9 @@ S32 LLCurl::Multi::process() } else { - response = 499; - //*TODO: change to llwarns - llerrs << "cleaned up curl request completed!" << llendl; + response = HTTP_INTERNAL_ERROR; + //*TODO: change to LL_WARNS() + LL_ERRS() << "cleaned up curl request completed!" << LL_ENDL; } if (response >= 400) { @@ -868,7 +1010,7 @@ bool LLCurl::Multi::addEasy(Easy* easy) check_curl_multi_code(mcode); //if (mcode != CURLM_OK) //{ - // llwarns << "Curl Error: " << curl_multi_strerror(mcode) << llendl; + // LL_WARNS() << "Curl Error: " << curl_multi_strerror(mcode) << LL_ENDL; // return false; //} return true; @@ -985,7 +1127,7 @@ void LLCurlThread::addMulti(LLCurl::Multi* multi) if (!addRequest(req)) { - llwarns << "curl request added when the thread is quitted" << llendl; + LL_WARNS() << "curl request added when the thread is quitted" << LL_ENDL; } } @@ -1092,7 +1234,7 @@ bool LLCurlRequest::addEasy(LLCurl::Easy* easy) if (mProcessing) { - llerrs << "Posting to a LLCurlRequest instance from within a responder is not allowed (causes DNS timeouts)." << llendl; + LL_ERRS() << "Posting to a LLCurlRequest instance from within a responder is not allowed (causes DNS timeouts)." << LL_ENDL; } bool res = mActiveMulti->addEasy(easy); return res; @@ -1120,13 +1262,13 @@ bool LLCurlRequest::getByteRange(const std::string& url, easy->setopt(CURLOPT_HTTPGET, 1); if (length > 0) { - std::string range = llformat("Range: bytes=%d-%d", offset,offset+length-1); - easy->slist_append(range.c_str()); + std::string range = llformat("bytes=%d-%d", offset,offset+length-1); + easy->slist_append(HTTP_OUT_HEADER_RANGE, range); } else if (offset > 0) { - std::string range = llformat("Range: bytes=%d-", offset); - easy->slist_append(range.c_str()); + std::string range = llformat("bytes=%d-", offset); + easy->slist_append(HTTP_OUT_HEADER_RANGE, range); } easy->setHeaders(); bool res = addEasy(easy); @@ -1153,10 +1295,10 @@ bool LLCurlRequest::post(const std::string& url, easy->setopt(CURLOPT_POSTFIELDS, (void*)NULL); easy->setopt(CURLOPT_POSTFIELDSIZE, bytes); - easy->slist_append("Content-Type: application/llsd+xml"); + easy->slist_append(HTTP_OUT_HEADER_CONTENT_TYPE, HTTP_CONTENT_LLSD_XML); easy->setHeaders(); - lldebugs << "POSTING: " << bytes << " bytes." << llendl; + LL_DEBUGS() << "POSTING: " << bytes << " bytes." << LL_ENDL; bool res = addEasy(easy); return res; } @@ -1181,10 +1323,10 @@ bool LLCurlRequest::post(const std::string& url, easy->setopt(CURLOPT_POSTFIELDS, (void*)NULL); easy->setopt(CURLOPT_POSTFIELDSIZE, bytes); - easy->slist_append("Content-Type: application/octet-stream"); + easy->slist_append(HTTP_OUT_HEADER_CONTENT_TYPE, HTTP_CONTENT_OCTET_STREAM); easy->setHeaders(); - lldebugs << "POSTING: " << bytes << " bytes." << llendl; + LL_DEBUGS() << "POSTING: " << bytes << " bytes." << LL_ENDL; bool res = addEasy(easy); return res; } @@ -1553,6 +1695,14 @@ void LLCurlEasyRequest::setSSLCtxCallback(curl_ssl_ctx_callback callback, void* } } +void LLCurlEasyRequest::slist_append(const std::string& header, const std::string& value) +{ + if (isValid() && mEasy) + { + mEasy->slist_append(header, value); + } +} + void LLCurlEasyRequest::slist_append(const char* str) { if (isValid() && mEasy) @@ -1565,7 +1715,7 @@ void LLCurlEasyRequest::sendRequest(const std::string& url) { llassert_always(!mRequestSent); mRequestSent = true; - lldebugs << url << llendl; + LL_DEBUGS() << url << LL_ENDL; if (isValid() && mEasy) { mEasy->setHeaders(); @@ -1733,20 +1883,18 @@ void LLCurl::cleanupClass() #if SAFE_SSL CRYPTO_set_locking_callback(NULL); for_each(sSSLMutex.begin(), sSSLMutex.end(), DeletePointer()); + sSSLMutex.clear(); #endif LL_CHECK_MEMORY - - for (std::set<CURL*>::iterator iter = Easy::sFreeHandles.begin(); iter != Easy::sFreeHandles.end(); ++iter) - { - CURL* curl = *iter; - LLCurl::deleteEasyHandle(curl); - } - + Easy::deleteAllFreeHandles(); + LL_CHECK_MEMORY + Easy::deleteAllActiveHandles(); LL_CHECK_MEMORY - Easy::sFreeHandles.clear(); - + // Free the template easy handle + curl_easy_cleanup(sCurlTemplateStandardHandle); + sCurlTemplateStandardHandle = NULL; LL_CHECK_MEMORY delete Easy::sHandleMutexp ; @@ -1772,7 +1920,7 @@ CURLM* LLCurl::newMultiHandle() if(sTotalHandles + 1 > sMaxHandles) { - llwarns << "no more handles available." << llendl ; + LL_WARNS() << "no more handles available." << LL_ENDL ; return NULL ; //failed } sTotalHandles++; @@ -1780,7 +1928,7 @@ CURLM* LLCurl::newMultiHandle() CURLM* ret = curl_multi_init() ; if(!ret) { - llwarns << "curl_multi_init failed." << llendl ; + LL_WARNS() << "curl_multi_init failed." << LL_ENDL ; } return ret ; @@ -1806,15 +1954,15 @@ CURL* LLCurl::newEasyHandle() if(sTotalHandles + 1 > sMaxHandles) { - llwarns << "no more handles available." << llendl ; + LL_WARNS() << "no more handles available." << LL_ENDL ; return NULL ; //failed } sTotalHandles++; - CURL* ret = curl_easy_init() ; + CURL* ret = createStandardCurlHandle(); if(!ret) { - llwarns << "curl_easy_init failed." << llendl ; + LL_WARNS() << "failed to create curl handle." << LL_ENDL ; } return ret ; @@ -1844,3 +1992,47 @@ void LLCurlFF::check_multi_code(CURLMcode code) { check_curl_multi_code(code); } + + +// Static +CURL* LLCurl::createStandardCurlHandle() +{ + if (sCurlTemplateStandardHandle == NULL) + { // Late creation of the template curl handle + sCurlTemplateStandardHandle = curl_easy_init(); + if (sCurlTemplateStandardHandle == NULL) + { + LL_WARNS() << "curl error calling curl_easy_init()" << LL_ENDL; + } + else + { + CURLcode result = curl_easy_setopt(sCurlTemplateStandardHandle, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4); + check_curl_code(result); + result = curl_easy_setopt(sCurlTemplateStandardHandle, CURLOPT_NOSIGNAL, 1); + check_curl_code(result); + result = curl_easy_setopt(sCurlTemplateStandardHandle, CURLOPT_NOPROGRESS, 1); + check_curl_code(result); + result = curl_easy_setopt(sCurlTemplateStandardHandle, CURLOPT_ENCODING, ""); + check_curl_code(result); + result = curl_easy_setopt(sCurlTemplateStandardHandle, CURLOPT_AUTOREFERER, 1); + check_curl_code(result); + result = curl_easy_setopt(sCurlTemplateStandardHandle, CURLOPT_FOLLOWLOCATION, 1); + check_curl_code(result); + result = curl_easy_setopt(sCurlTemplateStandardHandle, CURLOPT_SSL_VERIFYPEER, 1); + check_curl_code(result); + result = curl_easy_setopt(sCurlTemplateStandardHandle, CURLOPT_SSL_VERIFYHOST, 0); + check_curl_code(result); + + // The Linksys WRT54G V5 router has an issue with frequent + // DNS lookups from LAN machines. If they happen too often, + // like for every HTTP request, the router gets annoyed after + // about 700 or so requests and starts issuing TCP RSTs to + // new connections. Reuse the DNS lookups for even a few + // seconds and no RSTs. + result = curl_easy_setopt(sCurlTemplateStandardHandle, CURLOPT_DNS_CACHE_TIMEOUT, 15); + check_curl_code(result); + } + } + + return curl_easy_duphandle(sCurlTemplateStandardHandle); +} |