diff options
-rw-r--r-- | indra/llcommon/llthread.cpp | 7 | ||||
-rw-r--r-- | indra/llcommon/llthread.h | 5 | ||||
-rw-r--r-- | indra/llcorehttp/_httplibcurl.cpp | 24 | ||||
-rw-r--r-- | indra/llcorehttp/_httpoperation.cpp | 15 | ||||
-rw-r--r-- | indra/llcorehttp/_httpoperation.h | 4 | ||||
-rw-r--r-- | indra/llcorehttp/_httpoprequest.cpp | 271 | ||||
-rw-r--r-- | indra/llcorehttp/_httpoprequest.h | 16 | ||||
-rw-r--r-- | indra/llcorehttp/_httpopsetget.cpp | 4 | ||||
-rw-r--r-- | indra/llcorehttp/_httppolicy.cpp | 21 | ||||
-rw-r--r-- | indra/llcorehttp/_httppolicy.h | 5 | ||||
-rw-r--r-- | indra/llcorehttp/_httppolicyglobal.cpp | 52 | ||||
-rw-r--r-- | indra/llcorehttp/_httppolicyglobal.h | 6 | ||||
-rw-r--r-- | indra/llcorehttp/_httpservice.cpp | 18 | ||||
-rw-r--r-- | indra/llcorehttp/httpoptions.cpp | 18 | ||||
-rw-r--r-- | indra/llcorehttp/httpoptions.h | 14 | ||||
-rw-r--r-- | indra/llcorehttp/httprequest.cpp | 27 | ||||
-rw-r--r-- | indra/llcorehttp/httprequest.h | 75 | ||||
-rw-r--r-- | indra/llcorehttp/tests/test_httprequest.hpp | 228 | ||||
-rw-r--r-- | indra/newview/llappviewer.cpp | 20 | ||||
-rw-r--r-- | indra/newview/lltexturefetch.cpp | 120 | ||||
-rw-r--r-- | indra/newview/lltexturefetch.h | 9 | ||||
-rw-r--r-- | indra/newview/lltextureview.cpp | 5 |
22 files changed, 803 insertions, 161 deletions
diff --git a/indra/llcommon/llthread.cpp b/indra/llcommon/llthread.cpp index a6ad6b125c..b27b64b26f 100644 --- a/indra/llcommon/llthread.cpp +++ b/indra/llcommon/llthread.cpp @@ -71,6 +71,13 @@ LL_COMMON_API void assert_main_thread() } } +void LLThread::registerThreadID() +{ +#if !LL_DARWIN + sThreadID = ++sIDIter; +#endif +} + // // Handed to the APR thread creation function // diff --git a/indra/llcommon/llthread.h b/indra/llcommon/llthread.h index b52e70ab2e..54af41ec59 100644 --- a/indra/llcommon/llthread.h +++ b/indra/llcommon/llthread.h @@ -88,6 +88,11 @@ public: U32 getID() const { return mID; } + // Called by threads *not* created via LLThread to register some + // internal state used by LLMutex. You must call this once early + // in the running thread to prevent collisions with the main thread. + static void registerThreadID(); + private: BOOL mPaused; diff --git a/indra/llcorehttp/_httplibcurl.cpp b/indra/llcorehttp/_httplibcurl.cpp index e134a28401..a176dd5b2a 100644 --- a/indra/llcorehttp/_httplibcurl.cpp +++ b/indra/llcorehttp/_httplibcurl.cpp @@ -159,6 +159,17 @@ void HttpLibcurl::addOp(HttpOpRequest * op) curl_multi_add_handle(mMultiHandles[op->mReqPolicy], op->mCurlHandle); op->mCurlActive = true; + if (op->mTracing > 0) + { + HttpPolicy & policy(mService->getPolicy()); + + LL_INFOS("CoreHttp") << "TRACE, ToActiveQueue, Handle: " + << static_cast<HttpHandle>(op) + << ", Actives: " << mActiveOps.size() + << ", Readies: " << policy.getReadyCount(op->mReqPolicy) + << LL_ENDL; + } + // On success, make operation active mActiveOps.insert(op); } @@ -190,10 +201,9 @@ bool HttpLibcurl::completeRequest(CURLM * multi_handle, CURL * handle, CURLcode // Deactivate request op->mCurlActive = false; - // Set final status of request + // Set final status of request if it hasn't failed by other mechanisms yet if (op->mStatus) { - // Only set if it hasn't failed by other mechanisms yet op->mStatus = HttpStatus(HttpStatus::EXT_CURL_EASY, status); } if (op->mStatus) @@ -209,6 +219,16 @@ bool HttpLibcurl::completeRequest(CURLM * multi_handle, CURL * handle, CURLcode curl_easy_cleanup(handle); op->mCurlHandle = NULL; + // Tracing + if (op->mTracing > 0) + { + LL_INFOS("CoreHttp") << "TRACE, RequestComplete, Handle: " + << static_cast<HttpHandle>(op) + << ", Status: " << op->mStatus.toHex() + << LL_ENDL; + } + + // Dispatch to next stage HttpPolicy & policy(mService->getPolicy()); bool still_active(policy.stageAfterCompletion(op)); diff --git a/indra/llcorehttp/_httpoperation.cpp b/indra/llcorehttp/_httpoperation.cpp index b5c58013d4..5a31bf90e7 100644 --- a/indra/llcorehttp/_httpoperation.cpp +++ b/indra/llcorehttp/_httpoperation.cpp @@ -34,6 +34,8 @@ #include "_httpreplyqueue.h" #include "_httpservice.h" +#include "lltimer.h" + namespace LLCore { @@ -49,8 +51,10 @@ HttpOperation::HttpOperation() mReplyQueue(NULL), mUserHandler(NULL), mReqPolicy(HttpRequest::DEFAULT_POLICY_ID), - mReqPriority(0U) + mReqPriority(0U), + mTracing(0) { + mMetricCreated = totalTime(); } @@ -113,7 +117,7 @@ void HttpOperation::stageFromActive(HttpService *) llassert_always(false); } - + void HttpOperation::visitNotifier(HttpRequest *) { if (mUserHandler) @@ -138,6 +142,13 @@ HttpStatus HttpOperation::cancel() void HttpOperation::addAsReply() { + if (mTracing > 0) + { + LL_INFOS("CoreHttp") << "TRACE, ToReplyQueue, Handle: " + << static_cast<HttpHandle>(this) + << LL_ENDL; + } + if (mReplyQueue) { addRef(); diff --git a/indra/llcorehttp/_httpoperation.h b/indra/llcorehttp/_httpoperation.h index c93aa2def9..de4939a0ac 100644 --- a/indra/llcorehttp/_httpoperation.h +++ b/indra/llcorehttp/_httpoperation.h @@ -110,6 +110,10 @@ public: // Reply Data HttpStatus mStatus; + + // Tracing, debug and metrics + HttpTime mMetricCreated; + int mTracing; }; // end class HttpOperation diff --git a/indra/llcorehttp/_httpoprequest.cpp b/indra/llcorehttp/_httpoprequest.cpp index e2550d057e..f78971d8f2 100644 --- a/indra/llcorehttp/_httpoprequest.cpp +++ b/indra/llcorehttp/_httpoprequest.cpp @@ -63,6 +63,16 @@ int parse_content_range_header(char * buffer, unsigned int * last, unsigned int * length); + +// Take data from libcurl's CURLOPT_DEBUGFUNCTION callback and +// escape and format it for a tracing line in logging. Absolutely +// anything including NULs can be in the data. If @scrub is true, +// non-printing or non-ascii characters are replaced with spaces +// otherwise a %XX form of escaping is used. +void escape_libcurl_debug_data(char * buffer, size_t len, bool scrub, + std::string & safe_line); + + #if defined(WIN32) // Not available on windows where the legacy strtok interface @@ -78,11 +88,6 @@ namespace LLCore { -// ================================== -// HttpOpRequest -// ================================== - - HttpOpRequest::HttpOpRequest() : HttpOperation(), mProcFlags(0U), @@ -237,6 +242,19 @@ HttpStatus HttpOpRequest::cancel() } +HttpStatus HttpOpRequest::setupGet(HttpRequest::policy_t policy_id, + HttpRequest::priority_t priority, + const std::string & url, + HttpOptions * options, + HttpHeaders * headers) +{ + setupCommon(policy_id, priority, url, NULL, options, headers); + mReqMethod = HOR_GET; + + return HttpStatus(); +} + + HttpStatus HttpOpRequest::setupGetByteRange(HttpRequest::policy_t policy_id, HttpRequest::priority_t priority, const std::string & url, @@ -245,30 +263,16 @@ HttpStatus HttpOpRequest::setupGetByteRange(HttpRequest::policy_t policy_id, HttpOptions * options, HttpHeaders * headers) { - HttpStatus status; - - mProcFlags = 0; - mReqPolicy = policy_id; - mReqPriority = priority; + setupCommon(policy_id, priority, url, NULL, options, headers); mReqMethod = HOR_GET; - mReqURL = url; mReqOffset = offset; mReqLength = len; if (offset || len) { mProcFlags |= PF_SCAN_RANGE_HEADER; } - if (headers && ! mReqHeaders) - { - headers->addRef(); - mReqHeaders = headers; - } - if (options && ! mReqOptions) - { - mReqOptions = new HttpOptions(*options); - } - return status; + return HttpStatus(); } @@ -279,29 +283,10 @@ HttpStatus HttpOpRequest::setupPost(HttpRequest::policy_t policy_id, HttpOptions * options, HttpHeaders * headers) { - HttpStatus status; - - mProcFlags = 0; - mReqPolicy = policy_id; - mReqPriority = priority; + setupCommon(policy_id, priority, url, body, options, headers); mReqMethod = HOR_POST; - mReqURL = url; - if (body) - { - body->addRef(); - mReqBody = body; - } - if (headers && ! mReqHeaders) - { - headers->addRef(); - mReqHeaders = headers; - } - if (options && ! mReqOptions) - { - mReqOptions = new HttpOptions(*options); - } - return status; + return HttpStatus(); } @@ -312,12 +297,23 @@ HttpStatus HttpOpRequest::setupPut(HttpRequest::policy_t policy_id, HttpOptions * options, HttpHeaders * headers) { - HttpStatus status; + setupCommon(policy_id, priority, url, body, options, headers); + mReqMethod = HOR_PUT; + + return HttpStatus(); +} + +void HttpOpRequest::setupCommon(HttpRequest::policy_t policy_id, + HttpRequest::priority_t priority, + const std::string & url, + BufferArray * body, + HttpOptions * options, + HttpHeaders * headers) +{ mProcFlags = 0; mReqPolicy = policy_id; mReqPriority = priority; - mReqMethod = HOR_PUT; mReqURL = url; if (body) { @@ -331,10 +327,14 @@ HttpStatus HttpOpRequest::setupPut(HttpRequest::policy_t policy_id, } if (options && ! mReqOptions) { - mReqOptions = new HttpOptions(*options); + options->addRef(); + mReqOptions = options; + if (options->getWantHeaders()) + { + mProcFlags |= PF_SAVE_HEADERS; + } + mTracing = (std::max)(mTracing, llclamp(options->getTrace(), 0, 3)); } - - return status; } @@ -394,30 +394,29 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service) curl_easy_setopt(mCurlHandle, CURLOPT_SSL_VERIFYHOST, 0); const std::string * opt_value(NULL); - if (policy.get(HttpRequest::GP_CA_PATH, opt_value)) + long opt_long(0L); + policy.get(HttpRequest::GP_LLPROXY, &opt_long); + if (opt_long) { - curl_easy_setopt(mCurlHandle, CURLOPT_CAPATH, opt_value->c_str()); + // Use the viewer-based thread-safe API which has a + // fast/safe check for proxy enable. Would like to + // encapsulate this someway... + LLProxy::getInstance()->applyProxySettings(mCurlHandle); } - if (policy.get(HttpRequest::GP_CA_FILE, opt_value)) + else if (policy.get(HttpRequest::GP_HTTP_PROXY, &opt_value)) { - curl_easy_setopt(mCurlHandle, CURLOPT_CAINFO, opt_value->c_str()); + // *TODO: This is fine for now but get fuller socks/ + // authentication thing going later.... + curl_easy_setopt(mCurlHandle, CURLOPT_PROXY, opt_value->c_str()); + curl_easy_setopt(mCurlHandle, CURLOPT_PROXYTYPE, CURLPROXY_HTTP); } - if (policy.get(HttpRequest::GP_HTTP_PROXY, opt_value)) + if (policy.get(HttpRequest::GP_CA_PATH, &opt_value)) { - if (*opt_value == "LLProxy") - { - // Use the viewer-based thread-safe API which has a - // fast/safe check for proxy enable. Would like to - // encapsulate this someway... - LLProxy::getInstance()->applyProxySettings(mCurlHandle); - } - else - { - // *TODO: This is fine for now but get fuller socks/ - // authentication thing going later.... - curl_easy_setopt(mCurlHandle, CURLOPT_PROXY, opt_value->c_str()); - curl_easy_setopt(mCurlHandle, CURLOPT_PROXYTYPE, CURLPROXY_HTTP); - } + curl_easy_setopt(mCurlHandle, CURLOPT_CAPATH, opt_value->c_str()); + } + if (policy.get(HttpRequest::GP_CA_FILE, &opt_value)) + { + curl_easy_setopt(mCurlHandle, CURLOPT_CAINFO, opt_value->c_str()); } switch (mReqMethod) @@ -463,6 +462,14 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service) // *FIXME: fail out here break; } + + // Tracing + if (mTracing > 1) + { + curl_easy_setopt(mCurlHandle, CURLOPT_VERBOSE, 1); + curl_easy_setopt(mCurlHandle, CURLOPT_DEBUGDATA, mCurlHandle); + curl_easy_setopt(mCurlHandle, CURLOPT_DEBUGFUNCTION, debugCallback); + } // There's a CURLOPT for this now... if ((mReqOffset || mReqLength) && HOR_GET == mReqMethod) @@ -621,6 +628,101 @@ size_t HttpOpRequest::headerCallback(void * data, size_t size, size_t nmemb, voi return hdr_size; } + +int HttpOpRequest::debugCallback(CURL * handle, curl_infotype info, char * buffer, size_t len, void * userdata) +{ + HttpOpRequest * op(NULL); + curl_easy_getinfo(handle, CURLINFO_PRIVATE, &op); + // *FIXME: check the pointer + + std::string safe_line; + std::string tag; + bool logit(false); + len = (std::min)(len, size_t(256)); // Keep things reasonable in all cases + + switch (info) + { + case CURLINFO_TEXT: + if (op->mTracing > 1) + { + tag = "TEXT"; + escape_libcurl_debug_data(buffer, len, true, safe_line); + logit = true; + } + break; + + case CURLINFO_HEADER_IN: + if (op->mTracing > 1) + { + tag = "HEADERIN"; + escape_libcurl_debug_data(buffer, len, true, safe_line); + logit = true; + } + break; + + case CURLINFO_HEADER_OUT: + if (op->mTracing > 1) + { + tag = "HEADEROUT"; + escape_libcurl_debug_data(buffer, len, true, safe_line); + logit = true; + } + break; + + case CURLINFO_DATA_IN: + if (op->mTracing > 1) + { + tag = "DATAIN"; + logit = true; + if (op->mTracing > 2) + { + escape_libcurl_debug_data(buffer, len, false, safe_line); + } + else + { + std::ostringstream out; + out << len << " Bytes"; + safe_line = out.str(); + } + } + break; + + case CURLINFO_DATA_OUT: + if (op->mTracing > 1) + { + tag = "DATAOUT"; + logit = true; + if (op->mTracing > 2) + { + escape_libcurl_debug_data(buffer, len, false, safe_line); + } + else + { + std::ostringstream out; + out << len << " Bytes"; + safe_line = out.str(); + } + } + break; + + default: + logit = false; + break; + } + + if (logit) + { + LL_INFOS("CoreHttp") << "TRACE, LibcurlDebug, Handle: " + << static_cast<HttpHandle>(op) + << ", Type: " << tag + << ", Data: " << safe_line + << LL_ENDL; + } + + return 0; +} + + } // end namespace LLCore @@ -694,6 +796,43 @@ char *strtok_r(char *str, const char *delim, char ** savestate) #endif + +void escape_libcurl_debug_data(char * buffer, size_t len, bool scrub, std::string & safe_line) +{ + std::string out; + len = (std::min)(len, size_t(200)); + out.reserve(3 * len); + for (int i(0); i < len; ++i) + { + unsigned char uc(static_cast<unsigned char>(buffer[i])); + + if (uc < 32 || uc > 126) + { + if (scrub) + { + out.append(1, ' '); + } + else + { + static const char hex[] = "0123456789ABCDEF"; + char convert[4]; + + convert[0] = '%'; + convert[1] = hex[(uc >> 4) % 16]; + convert[2] = hex[uc % 16]; + convert[3] = '\0'; + out.append(convert); + } + } + else + { + out.append(1, buffer[i]); + } + } + safe_line.swap(out); +} + + } // end anonymous namespace diff --git a/indra/llcorehttp/_httpoprequest.h b/indra/llcorehttp/_httpoprequest.h index 80893beb40..fc2301057c 100644 --- a/indra/llcorehttp/_httpoprequest.h +++ b/indra/llcorehttp/_httpoprequest.h @@ -76,6 +76,12 @@ public: public: // Setup Methods + HttpStatus setupGet(HttpRequest::policy_t policy_id, + HttpRequest::priority_t priority, + const std::string & url, + HttpOptions * options, + HttpHeaders * headers); + HttpStatus setupGetByteRange(HttpRequest::policy_t policy_id, HttpRequest::priority_t priority, const std::string & url, @@ -103,15 +109,23 @@ public: virtual HttpStatus cancel(); protected: + void setupCommon(HttpRequest::policy_t policy_id, + HttpRequest::priority_t priority, + const std::string & url, + BufferArray * body, + HttpOptions * options, + HttpHeaders * headers); + static size_t writeCallback(void * data, size_t size, size_t nmemb, void * userdata); static size_t readCallback(void * data, size_t size, size_t nmemb, void * userdata); static size_t headerCallback(void * data, size_t size, size_t nmemb, void * userdata); + static int debugCallback(CURL *, curl_infotype info, char * buffer, size_t len, void * userdata); protected: unsigned int mProcFlags; static const unsigned int PF_SCAN_RANGE_HEADER = 0x00000001U; static const unsigned int PF_SAVE_HEADERS = 0x00000002U; - + public: // Request data EMethod mReqMethod; diff --git a/indra/llcorehttp/_httpopsetget.cpp b/indra/llcorehttp/_httpopsetget.cpp index 21e058b2be..c1357f9ae5 100644 --- a/indra/llcorehttp/_httpopsetget.cpp +++ b/indra/llcorehttp/_httpopsetget.cpp @@ -89,8 +89,8 @@ void HttpOpSetGet::stageFromRequest(HttpService * service) } if (mStatus) { - const std::string * value; - if ((mStatus = pol_opt.get(setting, value))) + const std::string * value(NULL); + if ((mStatus = pol_opt.get(setting, &value))) { mStrValue = *value; } diff --git a/indra/llcorehttp/_httppolicy.cpp b/indra/llcorehttp/_httppolicy.cpp index 8ee3f88658..0e08d88276 100644 --- a/indra/llcorehttp/_httppolicy.cpp +++ b/indra/llcorehttp/_httppolicy.cpp @@ -107,6 +107,12 @@ void HttpPolicy::retryOp(HttpOpRequest * op) LL_WARNS("CoreHttp") << "URL op retry #" << op->mPolicyRetries << " being scheduled for " << delta << " uSecs from now." << LL_ENDL; + if (op->mTracing > 0) + { + LL_INFOS("CoreHttp") << "TRACE, ToRetryQueue, Handle: " + << static_cast<HttpHandle>(op) + << LL_ENDL; + } mState[policy_class].mRetryQueue.push(op); } @@ -224,8 +230,8 @@ bool HttpPolicy::stageAfterCompletion(HttpOpRequest * op) } else if (op->mPolicyRetries) { - LL_WARNS("CoreHttp") << "URL op succeeded after " << op->mPolicyRetries << " retries." - << LL_ENDL; + LL_DEBUGS("CoreHttp") << "URL op succeeded after " << op->mPolicyRetries << " retries." + << LL_ENDL; } op->stageFromActive(mService); @@ -234,4 +240,15 @@ bool HttpPolicy::stageAfterCompletion(HttpOpRequest * op) } +int HttpPolicy::getReadyCount(HttpRequest::policy_t policy_class) +{ + if (policy_class < HttpRequest::POLICY_CLASS_LIMIT) + { + return (mState[policy_class].mReadyQueue.size() + + mState[policy_class].mRetryQueue.size()); + } + return 0; +} + + } // end namespace LLCore diff --git a/indra/llcorehttp/_httppolicy.h b/indra/llcorehttp/_httppolicy.h index 73c22bab78..4114f64848 100644 --- a/indra/llcorehttp/_httppolicy.h +++ b/indra/llcorehttp/_httppolicy.h @@ -97,7 +97,10 @@ public: } void setPolicies(const HttpPolicyGlobal & global); - + + // Get ready counts for a particular class + int getReadyCount(HttpRequest::policy_t policy_class); + protected: struct State { diff --git a/indra/llcorehttp/_httppolicyglobal.cpp b/indra/llcorehttp/_httppolicyglobal.cpp index d95d73cfba..6b1de38fd6 100644 --- a/indra/llcorehttp/_httppolicyglobal.cpp +++ b/indra/llcorehttp/_httppolicyglobal.cpp @@ -33,7 +33,9 @@ namespace LLCore HttpPolicyGlobal::HttpPolicyGlobal() : mSetMask(0UL), - mConnectionLimit(32L) + mConnectionLimit(32L), + mTrace(0), + mUseLLProxy(0) {} @@ -50,6 +52,8 @@ HttpPolicyGlobal & HttpPolicyGlobal::operator=(const HttpPolicyGlobal & other) mCAPath = other.mCAPath; mCAFile = other.mCAFile; mHttpProxy = other.mHttpProxy; + mTrace = other.mTrace; + mUseLLProxy = other.mUseLLProxy; } return *this; } @@ -63,6 +67,14 @@ HttpStatus HttpPolicyGlobal::set(HttpRequest::EGlobalPolicy opt, long value) mConnectionLimit = value; break; + case HttpRequest::GP_TRACE: + mTrace = llclamp(value, 0L, 3L); + break; + + case HttpRequest::GP_LLPROXY: + mUseLLProxy = llclamp(value, 0L, 1L); + break; + default: return HttpStatus(HttpStatus::LLCORE, HE_INVALID_ARG); } @@ -97,54 +109,64 @@ HttpStatus HttpPolicyGlobal::set(HttpRequest::EGlobalPolicy opt, const std::stri } -HttpStatus HttpPolicyGlobal::get(HttpRequest::EGlobalPolicy opt, long & value) +HttpStatus HttpPolicyGlobal::get(HttpRequest::EGlobalPolicy opt, long * value) { static const HttpStatus not_set(HttpStatus::LLCORE, HE_OPT_NOT_SET); + long * src(NULL); switch (opt) { case HttpRequest::GP_CONNECTION_LIMIT: - if (! (mSetMask & (1UL << int(opt)))) - return not_set; - value = mConnectionLimit; + src = &mConnectionLimit; + break; + + case HttpRequest::GP_TRACE: + src = &mTrace; + break; + + case HttpRequest::GP_LLPROXY: + src = &mUseLLProxy; break; default: return HttpStatus(HttpStatus::LLCORE, HE_INVALID_ARG); } + if (! (mSetMask & (1UL << int(opt)))) + return not_set; + + *value = *src; return HttpStatus(); } -HttpStatus HttpPolicyGlobal::get(HttpRequest::EGlobalPolicy opt, const std::string *& value) +HttpStatus HttpPolicyGlobal::get(HttpRequest::EGlobalPolicy opt, const std::string ** value) { static const HttpStatus not_set(HttpStatus::LLCORE, HE_OPT_NOT_SET); + const std::string * src(NULL); switch (opt) { case HttpRequest::GP_CA_PATH: - if (! (mSetMask & (1UL << int(opt)))) - return not_set; - value = &mCAPath; + src = &mCAPath; break; case HttpRequest::GP_CA_FILE: - if (! (mSetMask & (1UL << int(opt)))) - return not_set; - value = &mCAFile; + src = &mCAFile; break; case HttpRequest::GP_HTTP_PROXY: - if (! (mSetMask & (1UL << int(opt)))) - return not_set; - value = &mHttpProxy; + src = &mHttpProxy; break; default: return HttpStatus(HttpStatus::LLCORE, HE_INVALID_ARG); } + if (! (mSetMask & (1UL << int(opt)))) + return not_set; + + *value = src; return HttpStatus(); } diff --git a/indra/llcorehttp/_httppolicyglobal.h b/indra/llcorehttp/_httppolicyglobal.h index f4bb4d4b25..a50d0e4188 100644 --- a/indra/llcorehttp/_httppolicyglobal.h +++ b/indra/llcorehttp/_httppolicyglobal.h @@ -48,8 +48,8 @@ private: public: HttpStatus set(HttpRequest::EGlobalPolicy opt, long value); HttpStatus set(HttpRequest::EGlobalPolicy opt, const std::string & value); - HttpStatus get(HttpRequest::EGlobalPolicy opt, long & value); - HttpStatus get(HttpRequest::EGlobalPolicy opt, const std::string *& value); + HttpStatus get(HttpRequest::EGlobalPolicy opt, long * value); + HttpStatus get(HttpRequest::EGlobalPolicy opt, const std::string ** value); public: unsigned long mSetMask; @@ -57,6 +57,8 @@ public: std::string mCAPath; std::string mCAFile; std::string mHttpProxy; + long mTrace; + long mUseLLProxy; }; // end class HttpPolicyGlobal } // end namespace LLCore diff --git a/indra/llcorehttp/_httpservice.cpp b/indra/llcorehttp/_httpservice.cpp index 920a3f3b6d..beba8f08f4 100644 --- a/indra/llcorehttp/_httpservice.cpp +++ b/indra/llcorehttp/_httpservice.cpp @@ -36,6 +36,7 @@ #include "_thread.h" #include "lltimer.h" +#include "llthread.h" // Tuning parameters @@ -186,8 +187,10 @@ void HttpService::shutdown() void HttpService::threadRun(LLCoreInt::HttpThread * thread) { boost::this_thread::disable_interruption di; - ELoopSpeed loop(REQUEST_SLEEP); + + LLThread::registerThreadID(); + ELoopSpeed loop(REQUEST_SLEEP); while (! mExitRequested) { loop = processRequestQueue(loop); @@ -226,6 +229,19 @@ HttpService::ELoopSpeed HttpService::processRequestQueue(ELoopSpeed loop) // Process operation if (! mExitRequested) { + // Setup for subsequent tracing + long tracing(0); + mPolicy->getGlobalOptions().get(HttpRequest::GP_TRACE, &tracing); + op->mTracing = (std::max)(op->mTracing, int(tracing)); + + if (op->mTracing > 0) + { + LL_INFOS("CoreHttp") << "TRACE, FromRequestQueue, Handle: " + << static_cast<HttpHandle>(op) + << LL_ENDL; + } + + // Stage op->stageFromRequest(this); } diff --git a/indra/llcorehttp/httpoptions.cpp b/indra/llcorehttp/httpoptions.cpp index 15b505f5bf..155fbda7f1 100644 --- a/indra/llcorehttp/httpoptions.cpp +++ b/indra/llcorehttp/httpoptions.cpp @@ -33,13 +33,15 @@ namespace LLCore HttpOptions::HttpOptions() : RefCounted(true), - mWantHeaders(false) + mWantHeaders(false), + mTracing(0) {} HttpOptions::HttpOptions(const HttpOptions & rhs) : RefCounted(true), - mWantHeaders(rhs.mWantHeaders) + mWantHeaders(rhs.mWantHeaders), + mTracing(rhs.mTracing) {} @@ -47,4 +49,16 @@ HttpOptions::~HttpOptions() {} +void HttpOptions::setWantHeaders() +{ + mWantHeaders = true; +} + + +void HttpOptions::setTrace(long level) +{ + mTracing = int(level); +} + + } // end namespace LLCore diff --git a/indra/llcorehttp/httpoptions.h b/indra/llcorehttp/httpoptions.h index 267a982dd5..0b9dfdc1de 100644 --- a/indra/llcorehttp/httpoptions.h +++ b/indra/llcorehttp/httpoptions.h @@ -67,11 +67,21 @@ protected: void operator=(const HttpOptions &); // Not defined public: + void setWantHeaders(); + bool getWantHeaders() const + { + return mWantHeaders; + } + + void setTrace(long level); + int getTrace() const + { + return mTracing; + } protected: - // *TODO: add some options bool mWantHeaders; - + long int mTracing; }; // end class HttpOptions diff --git a/indra/llcorehttp/httprequest.cpp b/indra/llcorehttp/httprequest.cpp index 089eee76f3..2036ecfd1c 100644 --- a/indra/llcorehttp/httprequest.cpp +++ b/indra/llcorehttp/httprequest.cpp @@ -135,6 +135,33 @@ HttpStatus HttpRequest::getStatus() const } +HttpHandle HttpRequest::requestGet(policy_t policy_id, + priority_t priority, + const std::string & url, + HttpOptions * options, + HttpHeaders * headers, + HttpHandler * user_handler) +{ + HttpStatus status; + HttpHandle handle(LLCORE_HTTP_HANDLE_INVALID); + + HttpOpRequest * op = new HttpOpRequest(); + if (! (status = op->setupGet(policy_id, priority, url, options, headers))) + { + op->release(); + mLastReqStatus = status; + return handle; + } + op->setReplyPath(mReplyQueue, user_handler); + mRequestQueue->addOp(op); // transfers refcount + + mLastReqStatus = status; + handle = static_cast<HttpHandle>(op); + + return handle; +} + + HttpHandle HttpRequest::requestGetByteRange(policy_t policy_id, priority_t priority, const std::string & url, diff --git a/indra/llcorehttp/httprequest.h b/indra/llcorehttp/httprequest.h index a953aa28d0..4e78ed3719 100644 --- a/indra/llcorehttp/httprequest.h +++ b/indra/llcorehttp/httprequest.h @@ -111,10 +111,43 @@ public: /// Maximum number of connections the library will use to /// perform operations. This is somewhat soft as the underlying /// transport will cache some connections (up to 5). - GP_CONNECTION_LIMIT, ///< Takes long giving number of connections - GP_CA_PATH, ///< System path/directory where SSL certs are stored. - GP_CA_FILE, ///< System path/file containing certs. - GP_HTTP_PROXY ///< String giving host/port to use for HTTP proxy + + /// A long value setting the maximum number of connections + /// allowed over all policy classes. Note that this will be + /// a somewhat soft value. There may be an additional five + /// connections per policy class depending upon runtime + /// behavior. + GP_CONNECTION_LIMIT, + + /// String containing a system-appropriate directory name + /// where SSL certs are stored. + GP_CA_PATH, + + /// String giving a full path to a file containing SSL certs. + GP_CA_FILE, + + /// String of host/port to use as simple HTTP proxy. This is + /// going to change in the future into something more elaborate + /// that may support richer schemes. + GP_HTTP_PROXY, + + /// Long value that if non-zero enables the use of the + /// traditional LLProxy code for http/socks5 support. If + /// enabled, has priority over GP_HTTP_PROXY. + GP_LLPROXY, + + /// Long value setting the logging trace level for the + /// library. Possible values are: + /// 0 - No tracing (default) + /// 1 - Basic tracing of request start, stop and major events. + /// 2 - Connection, header and payload size information from + /// HTTP transactions. + /// 3 - Partial logging of payload itself. + /// + /// These values are also used in the trace modes for + /// individual requests in HttpOptions. Also be aware that + /// tracing tends to impact performance of the viewer. + GP_TRACE }; /// Set a parameter on a global policy option. Calls @@ -133,7 +166,7 @@ public: /// the class in other methods. If -1, an error /// occurred and @see getStatus() may provide more /// detail on the reason. - policy_t createPolicyClass(); + static policy_t createPolicyClass(); enum EClassPolicy { @@ -157,7 +190,7 @@ public: /// @param opt Enum of option to be set. /// @param value Desired value of option. /// @return Standard status code. - HttpStatus setPolicyClassOption(policy_t policy_id, EClassPolicy opt, long value); + static HttpStatus setPolicyClassOption(policy_t policy_id, EClassPolicy opt, long value); /// @} @@ -176,6 +209,36 @@ public: /// HttpStatus getStatus() const; + /// Queue a full HTTP GET request to be issued for entire entity. + /// The request is queued and serviced by the working thread and + /// notification of completion delivered to the optional HttpHandler + /// argument during @see update() calls. + /// + /// With a valid handle returned, it can be used to reference the + /// request in other requests (like cancellation) and will be an + /// argument when any HttpHandler object is invoked. + /// + /// @param policy_id Default or user-defined policy class under + /// which this request is to be serviced. + /// @param priority Standard priority scheme inherited from + /// Indra code base (U32-type scheme). + /// @param url + /// @param options (optional) + /// @param headers (optional) + /// @param handler (optional) + /// @return The handle of the request if successfully + /// queued or LLCORE_HTTP_HANDLE_INVALID if the + /// request could not be queued. In the latter + /// case, @see getStatus() will return more info. + /// + HttpHandle requestGet(policy_t policy_id, + priority_t priority, + const std::string & url, + HttpOptions * options, + HttpHeaders * headers, + HttpHandler * handler); + + /// Queue a full HTTP GET request to be issued with a 'Range' header. /// The request is queued and serviced by the working thread and /// notification of completion delivered to the optional HttpHandler diff --git a/indra/llcorehttp/tests/test_httprequest.hpp b/indra/llcorehttp/tests/test_httprequest.hpp index 61698f34d8..5b04796c8a 100644 --- a/indra/llcorehttp/tests/test_httprequest.hpp +++ b/indra/llcorehttp/tests/test_httprequest.hpp @@ -395,7 +395,7 @@ void HttpRequestTestObjectType::test<5>() { ScopedCurlInit ready; - set_test_name("HttpRequest GET + Stop execution"); + set_test_name("HttpRequest GET to dead port + Stop execution"); // Handler can be stack-allocated *if* there are no dangling // references to it after completion of this method. @@ -496,6 +496,7 @@ void HttpRequestTestObjectType::test<5>() } } + template <> template <> void HttpRequestTestObjectType::test<6>() { @@ -532,6 +533,114 @@ void HttpRequestTestObjectType::test<6>() // Issue a GET that *can* connect mStatus = HttpStatus(200); + HttpHandle handle = req->requestGet(HttpRequest::DEFAULT_POLICY_ID, + 0U, + url_base, + NULL, + NULL, + &handler); + ensure("Valid handle returned for ranged request", handle != LLCORE_HTTP_HANDLE_INVALID); + + // Run the notification pump. + int count(0); + int limit(10); + while (count++ < limit && mHandlerCalls < 1) + { + req->update(1000); + usleep(100000); + } + ensure("Request executed in reasonable time", count < limit); + ensure("One handler invocation for request", mHandlerCalls == 1); + + // Okay, request a shutdown of the servicing thread + mStatus = HttpStatus(); + handle = req->requestStopThread(&handler); + ensure("Valid handle returned for second request", handle != LLCORE_HTTP_HANDLE_INVALID); + + // Run the notification pump again + count = 0; + limit = 10; + while (count++ < limit && mHandlerCalls < 2) + { + req->update(1000); + usleep(100000); + } + ensure("Second request executed in reasonable time", count < limit); + ensure("Second handler invocation", mHandlerCalls == 2); + + // See that we actually shutdown the thread + count = 0; + limit = 10; + while (count++ < limit && ! HttpService::isStopped()) + { + usleep(100000); + } + ensure("Thread actually stopped running", HttpService::isStopped()); + + // release the request object + delete req; + req = NULL; + + // Shut down service + HttpRequest::destroyService(); + + ensure("Two handler calls on the way out", 2 == mHandlerCalls); + +#if defined(WIN32) + // Can only do this memory test on Windows. On other platforms, + // the LL logging system holds on to memory and produces what looks + // like memory leaks... + + // printf("Old mem: %d, New mem: %d\n", mMemTotal, GetMemTotal()); + ensure("Memory usage back to that at entry", mMemTotal == GetMemTotal()); +#endif + } + catch (...) + { + stop_thread(req); + delete req; + HttpRequest::destroyService(); + throw; + } +} + + +template <> template <> +void HttpRequestTestObjectType::test<7>() +{ + ScopedCurlInit ready; + + std::string url_base(get_base_url()); + std::cerr << "Base: " << url_base << std::endl; + + set_test_name("HttpRequest GET with Range: header to real service"); + + // Handler can be stack-allocated *if* there are no dangling + // references to it after completion of this method. + // Create before memory record as the string copy will bump numbers. + TestHandler2 handler(this, "handler"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + mHandlerCalls = 0; + + HttpRequest * req = NULL; + + try + { + // Get singletons created + HttpRequest::createService(); + + // Start threading early so that thread memory is invariant + // over the test. + HttpRequest::startThread(); + + // create a new ref counted object with an implicit reference + req = new HttpRequest(); + ensure("Memory allocated on construction", mMemTotal < GetMemTotal()); + + // Issue a GET that *can* connect + mStatus = HttpStatus(200); HttpHandle handle = req->requestGetByteRange(HttpRequest::DEFAULT_POLICY_ID, 0U, url_base, @@ -605,8 +714,9 @@ void HttpRequestTestObjectType::test<6>() } } + template <> template <> -void HttpRequestTestObjectType::test<7>() +void HttpRequestTestObjectType::test<8>() { ScopedCurlInit ready; @@ -725,7 +835,7 @@ void HttpRequestTestObjectType::test<7>() } template <> template <> -void HttpRequestTestObjectType::test<8>() +void HttpRequestTestObjectType::test<9>() { ScopedCurlInit ready; @@ -843,6 +953,118 @@ void HttpRequestTestObjectType::test<8>() } } +template <> template <> +void HttpRequestTestObjectType::test<10>() +{ + ScopedCurlInit ready; + + std::string url_base(get_base_url()); + std::cerr << "Base: " << url_base << std::endl; + + set_test_name("HttpRequest GET with some tracing"); + + // Handler can be stack-allocated *if* there are no dangling + // references to it after completion of this method. + // Create before memory record as the string copy will bump numbers. + TestHandler2 handler(this, "handler"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + mHandlerCalls = 0; + + HttpRequest * req = NULL; + + try + { + // Get singletons created + HttpRequest::createService(); + + // Enable tracing + HttpRequest::setPolicyGlobalOption(LLCore::HttpRequest::GP_TRACE, 2); + + // Start threading early so that thread memory is invariant + // over the test. + HttpRequest::startThread(); + + // create a new ref counted object with an implicit reference + req = new HttpRequest(); + ensure("Memory allocated on construction", mMemTotal < GetMemTotal()); + + // Issue a GET that *can* connect + mStatus = HttpStatus(200); + HttpHandle handle = req->requestGetByteRange(HttpRequest::DEFAULT_POLICY_ID, + 0U, + url_base, + 0, + 0, + NULL, + NULL, + &handler); + ensure("Valid handle returned for ranged request", handle != LLCORE_HTTP_HANDLE_INVALID); + + // Run the notification pump. + int count(0); + int limit(10); + while (count++ < limit && mHandlerCalls < 1) + { + req->update(1000); + usleep(100000); + } + ensure("Request executed in reasonable time", count < limit); + ensure("One handler invocation for request", mHandlerCalls == 1); + + // Okay, request a shutdown of the servicing thread + mStatus = HttpStatus(); + handle = req->requestStopThread(&handler); + ensure("Valid handle returned for second request", handle != LLCORE_HTTP_HANDLE_INVALID); + + // Run the notification pump again + count = 0; + limit = 10; + while (count++ < limit && mHandlerCalls < 2) + { + req->update(1000); + usleep(100000); + } + ensure("Second request executed in reasonable time", count < limit); + ensure("Second handler invocation", mHandlerCalls == 2); + + // See that we actually shutdown the thread + count = 0; + limit = 10; + while (count++ < limit && ! HttpService::isStopped()) + { + usleep(100000); + } + ensure("Thread actually stopped running", HttpService::isStopped()); + + // release the request object + delete req; + req = NULL; + + // Shut down service + HttpRequest::destroyService(); + + ensure("Two handler calls on the way out", 2 == mHandlerCalls); + +#if defined(WIN32) + // Can only do this memory test on Windows. On other platforms, + // the LL logging system holds on to memory and produces what looks + // like memory leaks... + + // printf("Old mem: %d, New mem: %d\n", mMemTotal, GetMemTotal()); + ensure("Memory usage back to that at entry", mMemTotal == GetMemTotal()); +#endif + } + catch (...) + { + stop_thread(req); + delete req; + HttpRequest::destroyService(); + throw; + } +} + } // end namespace tut namespace diff --git a/indra/newview/llappviewer.cpp b/indra/newview/llappviewer.cpp index e2c13e77e3..430dd89c3e 100644 --- a/indra/newview/llappviewer.cpp +++ b/indra/newview/llappviewer.cpp @@ -5356,11 +5356,9 @@ void CoreHttp::init() << LL_ENDL; } - mRequest = new LLCore::HttpRequest; - // Point to our certs or SSH/https: will fail on connect - status = mRequest->setPolicyGlobalOption(LLCore::HttpRequest::GP_CA_FILE, - gDirUtilp->getCAFile()); + status = LLCore::HttpRequest::setPolicyGlobalOption(LLCore::HttpRequest::GP_CA_FILE, + gDirUtilp->getCAFile()); if (! status) { LL_ERRS("Init") << "Failed to set CA File for HTTP services. Reason: " @@ -5371,15 +5369,22 @@ void CoreHttp::init() // Establish HTTP Proxy. "LLProxy" is a special string which directs // the code to use LLProxy::applyProxySettings() to establish any // HTTP or SOCKS proxy for http operations. - status = mRequest->setPolicyGlobalOption(LLCore::HttpRequest::GP_HTTP_PROXY, - std::string("LLProxy")); + status = LLCore::HttpRequest::setPolicyGlobalOption(LLCore::HttpRequest::GP_LLPROXY, 1); if (! status) { LL_ERRS("Init") << "Failed to set HTTP proxy for HTTP services. Reason: " << status.toString() << LL_ENDL; } - + + // Tracing levels for library & libcurl (note that 2 & 3 are beyond spammy): + // 0 - None + // 1 - Basic start, stop simple transitions + // 2 - libcurl CURLOPT_VERBOSE mode with brief lines + // 3 - with partial data content + status = LLCore::HttpRequest::setPolicyGlobalOption(LLCore::HttpRequest::GP_TRACE, 0); + + // Kick the thread status = LLCore::HttpRequest::startThread(); if (! status) { @@ -5388,6 +5393,7 @@ void CoreHttp::init() << LL_ENDL; } + mRequest = new LLCore::HttpRequest; } diff --git a/indra/newview/lltexturefetch.cpp b/indra/newview/lltexturefetch.cpp index f5e7540e85..664af02f78 100644 --- a/indra/newview/lltexturefetch.cpp +++ b/indra/newview/lltexturefetch.cpp @@ -431,6 +431,22 @@ private: void lockWorkMutex() { mWorkMutex.lock(); } void unlockWorkMutex() { mWorkMutex.unlock(); } + // Locks: Mw + void acquireHttpSemaphore() + { + llassert(! mHttpHasResource); + mHttpHasResource = true; + --mFetcher->mHttpSemaphore; + } + + // Locks: Mw + void releaseHttpSemaphore() + { + llassert(mHttpHasResource); + mHttpHasResource = false; + ++mFetcher->mHttpSemaphore; + } + private: enum e_state // mState { @@ -444,8 +460,9 @@ private: CACHE_POST, LOAD_FROM_NETWORK, LOAD_FROM_SIMULATOR, - SEND_HTTP_REQ, // Commit to sending as HTTP WAIT_HTTP_RESOURCE, // Waiting for HTTP resources + WAIT_HTTP_RESOURCE2, // Waiting for HTTP resources + SEND_HTTP_REQ, // Commit to sending as HTTP WAIT_HTTP_REQ, // Request sent, wait for completion DECODE_IMAGE, DECODE_IMAGE_UPDATE, @@ -532,7 +549,7 @@ private: bool mHttpActive; // Active request to http library unsigned int mHttpReplySize; unsigned int mHttpReplyOffset; - bool mHttpReleased; // Has been released from resource wait once + bool mHttpHasResource; // Counts against Fetcher's mHttpSemaphore }; ////////////////////////////////////////////////////////////////////////////// @@ -768,8 +785,9 @@ const char* LLTextureFetchWorker::sStateDescs[] = { "CACHE_POST", "LOAD_FROM_NETWORK", "LOAD_FROM_SIMULATOR", - "SEND_HTTP_REQ", "WAIT_HTTP_RESOURCE", + "WAIT_HTTP_RESOURCE2", + "SEND_HTTP_REQ", "WAIT_HTTP_REQ", "DECODE_IMAGE", "DECODE_IMAGE_UPDATE", @@ -836,7 +854,7 @@ LLTextureFetchWorker::LLTextureFetchWorker(LLTextureFetch* fetcher, mHttpActive(false), mHttpReplySize(0U), mHttpReplyOffset(0U), - mHttpReleased(true) + mHttpHasResource(false) { mCanUseNET = mUrl.empty() ; @@ -860,6 +878,10 @@ LLTextureFetchWorker::~LLTextureFetchWorker() llassert_always(!haveWork()); lockWorkMutex(); // +Mw (should be useless) + if (mHttpHasResource) + { + releaseHttpSemaphore(); + } if (mHttpActive) { // Issue a cancel on a live request... @@ -1126,7 +1148,7 @@ bool LLTextureFetchWorker::doWork(S32 param) llwarns << "Unknown URL Type: " << mUrl << llendl; } setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); - mState = SEND_HTTP_REQ; + mState = WAIT_HTTP_RESOURCE; } else { @@ -1223,7 +1245,7 @@ bool LLTextureFetchWorker::doWork(S32 param) } if (mCanUseHTTP && !mUrl.empty()) { - mState = SEND_HTTP_REQ; + mState = WAIT_HTTP_RESOURCE; setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); if(mWriteToCacheState != NOT_WRITE) { @@ -1287,31 +1309,38 @@ bool LLTextureFetchWorker::doWork(S32 param) return false; } - if (mState == SEND_HTTP_REQ) + if (mState == WAIT_HTTP_RESOURCE) { - if (! mCanUseHTTP) - { - return true; // abort - } - // NOTE: // control the number of the http requests issued for: // 1, not openning too many file descriptors at the same time; // 2, control the traffic of http so udp gets bandwidth. // - if (! mHttpReleased) + // If it looks like we're busy, keep this request here. + // Otherwise, advance into the HTTP states. + if (mFetcher->mHttpSemaphore <= 0 || mFetcher->getHttpWaitersCount()) { - // If this request hasn't been released before and it looks like - // we're busy, put this request into resource wait and allow something - // else to come to the front. - if (mFetcher->getNumHTTPRequests() >= HTTP_REQUESTS_IN_QUEUE_HIGH_WATER || - mFetcher->getHttpWaitersCount()) - { - mState = WAIT_HTTP_RESOURCE; - setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); - mFetcher->addHttpWaiter(this->mID); - return false; - } + mState = WAIT_HTTP_RESOURCE2; + setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); + mFetcher->addHttpWaiter(this->mID); + return false; + } + mState = SEND_HTTP_REQ; + acquireHttpSemaphore(); + } + + if (mState == WAIT_HTTP_RESOURCE2) + { + // Just idle it if we make it to the head... + return false; + } + + if (mState == SEND_HTTP_REQ) + { + if (! mCanUseHTTP) + { + releaseHttpSemaphore(); + return true; // abort } mFetcher->removeFromNetworkQueue(this, false); @@ -1327,10 +1356,12 @@ bool LLTextureFetchWorker::doWork(S32 param) // We already have all the data, just decode it mLoadedDiscard = mFormattedImage->getDiscardLevel(); mState = DECODE_IMAGE; + releaseHttpSemaphore(); return false; } else { + releaseHttpSemaphore(); return true; // abort. } } @@ -1365,6 +1396,7 @@ bool LLTextureFetchWorker::doWork(S32 param) { llwarns << "HTTP GET request failed for " << mID << llendl; resetFormattedData(); + releaseHttpSemaphore(); return true; // failed } @@ -1377,13 +1409,6 @@ bool LLTextureFetchWorker::doWork(S32 param) // fall through } - if (mState == WAIT_HTTP_RESOURCE) - { - // Nothing to do until releaseHttpWaiters() puts us back - // into the flow... - return false; - } - if (mState == WAIT_HTTP_REQ) { if (mLoaded) @@ -1401,6 +1426,7 @@ bool LLTextureFetchWorker::doWork(S32 param) mState = INIT; mCanUseHTTP = false; setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); + releaseHttpSemaphore(); return false; } } @@ -1423,12 +1449,14 @@ bool LLTextureFetchWorker::doWork(S32 param) // Use available data mLoadedDiscard = mFormattedImage->getDiscardLevel(); mState = DECODE_IMAGE; + releaseHttpSemaphore(); return false; } // Fail harder resetFormattedData(); mState = DONE; + releaseHttpSemaphore(); return true; // failed } @@ -1443,6 +1471,7 @@ bool LLTextureFetchWorker::doWork(S32 param) // abort. mState = DONE; + releaseHttpSemaphore(); return true; } @@ -1491,6 +1520,7 @@ bool LLTextureFetchWorker::doWork(S32 param) mWriteToCacheState = SHOULD_WRITE ; } setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); + releaseHttpSemaphore(); return false; } else @@ -2137,7 +2167,8 @@ LLTextureFetch::LLTextureFetch(LLTextureCache* cache, LLImageDecodeThread* image mQAMode(qa_mode), mHttpRequest(NULL), mHttpOptions(NULL), - mHttpHeaders(NULL) + mHttpHeaders(NULL), + mHttpSemaphore(HTTP_REQUESTS_IN_QUEUE_HIGH_WATER) { mMaxBandwidth = gSavedSettings.getF32("ThrottleBandwidthKBPS"); mTextureInfo.setUpLogging(gSavedSettings.getBOOL("LogTextureDownloadsToViewerLog"), gSavedSettings.getBOOL("LogTextureDownloadsToSimulator"), gSavedSettings.getU32("TextureLoggingThreshold")); @@ -3155,12 +3186,11 @@ void LLTextureFetch::removeHttpWaiter(const LLUUID & tid) // Locks: -Mw (must not hold any worker when called) void LLTextureFetch::releaseHttpWaiters() { - if (HTTP_REQUESTS_IN_QUEUE_LOW_WATER < getNumHTTPRequests()) + if (mHttpSemaphore < HTTP_REQUESTS_IN_QUEUE_LOW_WATER) return; // Quickly make a copy of all the LLUIDs. Get off the // mutex as early as possible. - typedef std::vector<LLUUID> uuid_vec_t; uuid_vec_t tids; @@ -3171,13 +3201,12 @@ void LLTextureFetch::releaseHttpWaiters() return; const size_t limit(mHttpWaitResource.size()); - tids.resize(limit); - wait_http_res_queue_t::iterator iter(mHttpWaitResource.begin()); - for (int i(0); - i < limit && mHttpWaitResource.end() != iter; - ++i, ++iter) + tids.reserve(limit); + for (wait_http_res_queue_t::iterator iter(mHttpWaitResource.begin()); + mHttpWaitResource.end() != iter; + ++iter) { - tids[i] = *iter; + tids.push_back(*iter); } } // -Mfnq @@ -3196,28 +3225,29 @@ void LLTextureFetch::releaseHttpWaiters() tids2.insert(worker); } } + tids.clear(); // Release workers up to the high water mark. Since we aren't // holding any locks at this point, we can be in competition // with other callers. Do defensive things like getting // refreshed counts of requests and checking if someone else // has moved any worker state around.... - tids.clear(); for (worker_set_t::iterator iter2(tids2.begin()); - tids2.end() != iter2 && 0 < (HTTP_REQUESTS_IN_QUEUE_HIGH_WATER - getNumHTTPRequests()); + tids2.end() != iter2 && mHttpSemaphore > 0; ++iter2) { LLTextureFetchWorker * worker(* iter2); worker->lockWorkMutex(); // +Mw - if (LLTextureFetchWorker::WAIT_HTTP_RESOURCE != worker->mState) + if (LLTextureFetchWorker::WAIT_HTTP_RESOURCE2 != worker->mState) { worker->unlockWorkMutex(); // -Mw continue; } - worker->mHttpReleased = true; + worker->mState = LLTextureFetchWorker::SEND_HTTP_REQ; worker->setPriority(LLWorkerThread::PRIORITY_HIGH | worker->mWorkPriority); + worker->acquireHttpSemaphore(); worker->unlockWorkMutex(); // -Mw removeHttpWaiter(worker->mID); @@ -3456,7 +3486,7 @@ TFReqSendMetrics::doWork(LLTextureFetch * fetcher) } // In QA mode, Metrics submode, log the result for ease of testing - if (fetcher->isQAMode() || true) + if (fetcher->isQAMode()) { LL_INFOS("Textures") << merged_llsd << LL_ENDL; } diff --git a/indra/newview/lltexturefetch.h b/indra/newview/lltexturefetch.h index 4ee13d171e..50e3181623 100644 --- a/indra/newview/lltexturefetch.h +++ b/indra/newview/lltexturefetch.h @@ -312,6 +312,15 @@ private: LLCore::HttpOptions * mHttpOptions; // Ttf LLCore::HttpHeaders * mHttpHeaders; // Ttf + // We use a resource semaphore to keep HTTP requests in + // WAIT_HTTP_RESOURCE2 if there aren't sufficient slots in the + // transport. This keeps them near where they can be cheaply + // reprioritized rather than dumping them all across a thread + // where it's more expensive to get at them. Requests in either + // SEND_HTTP_REQ or WAIT_HTTP_REQ charge against the semaphore + // and tracking state transitions is critical to liveness. + int mHttpSemaphore; // Ttf + typedef std::set<LLUUID> wait_http_res_queue_t; wait_http_res_queue_t mHttpWaitResource; // Mfnq diff --git a/indra/newview/lltextureview.cpp b/indra/newview/lltextureview.cpp index 5f1d7829ed..bb1535d23d 100644 --- a/indra/newview/lltextureview.cpp +++ b/indra/newview/lltextureview.cpp @@ -234,15 +234,16 @@ void LLTextureBar::draw() { "DSK", LLColor4::blue }, // CACHE_POST { "NET", LLColor4::green }, // LOAD_FROM_NETWORK { "SIM", LLColor4::green }, // LOAD_FROM_SIMULATOR + { "HTW", LLColor4::green }, // WAIT_HTTP_RESOURCE + { "HTW", LLColor4::green }, // WAIT_HTTP_RESOURCE2 { "REQ", LLColor4::yellow },// SEND_HTTP_REQ - { "HTW", LLColor4::green }, // WAIT_HTTP_RES { "HTP", LLColor4::green }, // WAIT_HTTP_REQ { "DEC", LLColor4::yellow },// DECODE_IMAGE { "DEC", LLColor4::green }, // DECODE_IMAGE_UPDATE { "WRT", LLColor4::purple },// WRITE_TO_CACHE { "WRT", LLColor4::orange },// WAIT_ON_WRITE { "END", LLColor4::red }, // DONE -#define LAST_STATE 13 +#define LAST_STATE 14 { "CRE", LLColor4::magenta }, // LAST_STATE+1 { "FUL", LLColor4::green }, // LAST_STATE+2 { "BAD", LLColor4::red }, // LAST_STATE+3 |