From 5cca78e718f15522cc3db9aec76aa910dd696aa8 Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Mon, 23 Jun 2014 14:23:33 -0400 Subject: First HTTP pipelining viewer. Enable pipelining for GetTexture and GetMesh2 at a pipeline depth of 5. Create global debug option, HttpPipelining, to enable and disable HTTP pipelining (defaults to true). Tweak texture and mesh low- and high-water request levels based on pipelining status and depth. Fixup texture console which was damaged in a recent release. Split logging of the no-request HTTP error case into two cases: one for missing URL in HTTP request, one for HTTP request not created. A refactor in llcorehttp is coming: I will be moving all libcurl- using code into libcurl-specific modules. --- indra/llcorehttp/_httpinternal.h | 5 ++- indra/llcorehttp/_httplibcurl.cpp | 69 ++++++++++++++++++++++++++++++++++- indra/llcorehttp/_httppolicy.cpp | 11 ++++-- indra/llcorehttp/_httppolicyclass.cpp | 8 ++-- indra/llcorehttp/httprequest.h | 35 ++++++++++++++++-- 5 files changed, 114 insertions(+), 14 deletions(-) (limited to 'indra/llcorehttp') diff --git a/indra/llcorehttp/_httpinternal.h b/indra/llcorehttp/_httpinternal.h index f80d7f60f5..a2a60ca056 100755 --- a/indra/llcorehttp/_httpinternal.h +++ b/indra/llcorehttp/_httpinternal.h @@ -145,8 +145,11 @@ const int HTTP_CONNECTION_LIMIT_DEFAULT = 8; const int HTTP_CONNECTION_LIMIT_MIN = 1; const int HTTP_CONNECTION_LIMIT_MAX = 256; -// Miscellaneous defaults +// Pipelining limits const long HTTP_PIPELINING_DEFAULT = 0L; +const long HTTP_PIPELINING_MAX = 20L; + +// Miscellaneous defaults const bool HTTP_USE_RETRY_AFTER_DEFAULT = true; const long HTTP_THROTTLE_RATE_DEFAULT = 0L; diff --git a/indra/llcorehttp/_httplibcurl.cpp b/indra/llcorehttp/_httplibcurl.cpp index e56bc84174..fb907f6318 100755 --- a/indra/llcorehttp/_httplibcurl.cpp +++ b/indra/llcorehttp/_httplibcurl.cpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012-2013, Linden Research, Inc. + * Copyright (C) 2012-2014, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -33,6 +33,15 @@ #include "llhttpconstants.h" +namespace +{ + +// Error testing and reporting for libcurl status codes +void check_curl_multi_code(CURLMcode code); +void check_curl_multi_code(CURLMcode code, int curl_setopt_option); + +} // end anonymous namespace + namespace LLCore { @@ -92,14 +101,44 @@ void HttpLibcurl::start(int policy_count) llassert_always(policy_count <= HTTP_POLICY_CLASS_LIMIT); llassert_always(! mMultiHandles); // One-time call only + HttpPolicy & policy(mService->getPolicy()); mPolicyCount = policy_count; mMultiHandles = new CURLM * [mPolicyCount]; mActiveHandles = new int [mPolicyCount]; for (int policy_class(0); policy_class < mPolicyCount; ++policy_class) { - mMultiHandles[policy_class] = curl_multi_init(); + HttpPolicyClass & options(policy.getClassOptions(policy_class)); + mActiveHandles[policy_class] = 0; + if (NULL == (mMultiHandles[policy_class] = curl_multi_init())) + { + LL_ERRS("CoreHttp") << "Failed to allocate multi handle in libcurl." + << LL_ENDL; + } + + if (options.mPipelining > 1) + { + CURLMcode code; + + // We'll try to do pipelining on this multihandle + code = curl_multi_setopt(mMultiHandles[policy_class], + CURLMOPT_PIPELINING, + 1L); + check_curl_multi_code(code, CURLMOPT_PIPELINING); + code = curl_multi_setopt(mMultiHandles[policy_class], + CURLMOPT_MAX_PIPELINE_LENGTH, + long(options.mPipelining)); + check_curl_multi_code(code, CURLMOPT_MAX_PIPELINE_LENGTH); + code = curl_multi_setopt(mMultiHandles[policy_class], + CURLMOPT_MAX_HOST_CONNECTIONS, + long(options.mPerHostConnectionLimit)); + check_curl_multi_code(code, CURLMOPT_MAX_HOST_CONNECTIONS); + code = curl_multi_setopt(mMultiHandles[policy_class], + CURLMOPT_MAX_TOTAL_CONNECTIONS, + long(options.mConnectionLimit)); + check_curl_multi_code(code, CURLMOPT_MAX_TOTAL_CONNECTIONS); + } } } @@ -376,3 +415,29 @@ struct curl_slist * append_headers_to_slist(const HttpHeaders * headers, struct } // end namespace LLCore + + +namespace +{ + +void check_curl_multi_code(CURLMcode code, int curl_setopt_option) +{ + if (CURLM_OK != code) + { + LL_WARNS("CoreHttp") << "libcurl multi error detected: " << curl_multi_strerror(code) + << ", curl_multi_setopt option: " << curl_setopt_option + << LL_ENDL; + } +} + + +void check_curl_multi_code(CURLMcode code) +{ + if (CURLM_OK != code) + { + LL_WARNS("CoreHttp") << "libcurl multi error detected: " << curl_multi_strerror(code) + << LL_ENDL; + } +} + +} // end anonymous namespace diff --git a/indra/llcorehttp/_httppolicy.cpp b/indra/llcorehttp/_httppolicy.cpp index fd5a93e192..bb7959b578 100755 --- a/indra/llcorehttp/_httppolicy.cpp +++ b/indra/llcorehttp/_httppolicy.cpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012-2013, Linden Research, Inc. + * Copyright (C) 2012-2014, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -128,7 +128,8 @@ void HttpPolicy::shutdown() void HttpPolicy::start() -{} +{ +} void HttpPolicy::addOp(HttpOpRequest * op) @@ -234,7 +235,11 @@ HttpService::ELoopSpeed HttpPolicy::processReadyQueue() } int active(transport.getActiveCountInClass(policy_class)); - int needed(state.mOptions.mConnectionLimit - active); // Expect negatives here + int active_limit(state.mOptions.mPipelining > 1L + ? (state.mOptions.mPerHostConnectionLimit + * state.mOptions.mPipelining) + : state.mOptions.mConnectionLimit); + int needed(active_limit - active); // Expect negatives here if (needed > 0) { diff --git a/indra/llcorehttp/_httppolicyclass.cpp b/indra/llcorehttp/_httppolicyclass.cpp index f34a8e9f1e..2c0f650155 100755 --- a/indra/llcorehttp/_httppolicyclass.cpp +++ b/indra/llcorehttp/_httppolicyclass.cpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012-2013, Linden Research, Inc. + * Copyright (C) 2012-2014, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -78,8 +78,8 @@ HttpStatus HttpPolicyClass::set(HttpRequest::EPolicyOption opt, long value) mPerHostConnectionLimit = llclamp(value, long(HTTP_CONNECTION_LIMIT_MIN), mConnectionLimit); break; - case HttpRequest::PO_ENABLE_PIPELINING: - mPipelining = llclamp(value, 0L, 1L); + case HttpRequest::PO_PIPELINING_DEPTH: + mPipelining = llclamp(value, 0L, HTTP_PIPELINING_MAX); break; case HttpRequest::PO_THROTTLE_RATE: @@ -106,7 +106,7 @@ HttpStatus HttpPolicyClass::get(HttpRequest::EPolicyOption opt, long * value) co *value = mPerHostConnectionLimit; break; - case HttpRequest::PO_ENABLE_PIPELINING: + case HttpRequest::PO_PIPELINING_DEPTH: *value = mPipelining; break; diff --git a/indra/llcorehttp/httprequest.h b/indra/llcorehttp/httprequest.h index 651654844a..7f23723b0b 100755 --- a/indra/llcorehttp/httprequest.h +++ b/indra/llcorehttp/httprequest.h @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012-2013, Linden Research, Inc. + * Copyright (C) 2012-2014, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -183,11 +183,38 @@ public: /// Global only PO_TRACE, - /// Suitable requests are allowed to pipeline on their - /// connections when they ask for it. + /// If greater than 1, suitable requests are allowed to + /// pipeline on their connections when they ask for it. + /// Value gives the maximum number of outstanding requests + /// on a connection. + /// + /// There is some interaction between PO_CONNECTION_LIMIT, + /// PO_PER_HOST_CONNECTION_LIMIT, and PO_PIPELINING_DEPTH. + /// When PIPELINING_DEPTH is 0 or 1 (no pipelining), this + /// library manages connection lifecycle and honors the + /// PO_CONNECTION_LIMIT setting as the maximum in-flight + /// request limit. Libcurl itself may be caching additional + /// connections under its connection cache policy. + /// + /// When PIPELINING_DEPTH is 2 or more, libcurl performs + /// connection management and both PO_CONNECTION_LIMIT and + /// PO_PER_HOST_CONNECTION_LIMIT should be set and non-zero. + /// In this case (as of libcurl 7.37.0), libcurl will + /// open new connections in preference to pipelining, up + /// to the above limits at which time pipelining begins. + /// And as usual, an additional cache of open but inactive + /// connections may still be maintained within libcurl. + /// For SL, a good rule-of-thumb is to set + /// PO_PER_HOST_CONNECTION_LIMIT to the user-visible + /// concurrency value and PO_CONNECTION_LIMIT to twice + /// that for baked texture loads and region crossings where + /// additional connection load will be tolerated. If + /// either limit is 0, libcurl will prefer pipelining + /// over connection creation, which is still interesting, + /// but won't be pursued at this time. /// /// Per-class only - PO_ENABLE_PIPELINING, + PO_PIPELINING_DEPTH, /// Controls whether client-side throttling should be /// performed on this policy class. Positive values -- cgit v1.3 From 9fb96b416f0612882ddcaa658945a5b3f73ff331 Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Thu, 3 Jul 2014 19:44:02 -0400 Subject: Add pipelining and tracing command line options to the test program. --- indra/llcorehttp/examples/http_texture_load.cpp | 64 +++++++++++++++++++++++-- 1 file changed, 59 insertions(+), 5 deletions(-) (limited to 'indra/llcorehttp') diff --git a/indra/llcorehttp/examples/http_texture_load.cpp b/indra/llcorehttp/examples/http_texture_load.cpp index 73c49687d7..88692c3f69 100755 --- a/indra/llcorehttp/examples/http_texture_load.cpp +++ b/indra/llcorehttp/examples/http_texture_load.cpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012-2013, Linden Research, Inc. + * Copyright (C) 2012-2014, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -59,11 +59,13 @@ void usage(std::ostream & out); // Default command line settings static int concurrency_limit(40); static int highwater(100); +static int pipeline_depth(0); +static int tracing(0); static char url_format[1024] = "http://example.com/some/path?texture_id=%s.texture"; #if defined(WIN32) -#define strncpy(_a, _b, _c) strncpy_s(_a, _b, _c) +#define strncpy(_a, _b, _c) strncpy_s(_a, _b, _c) #define strtok_r(_a, _b, _c) strtok_s(_a, _b, _c) int getopt(int argc, char * const argv[], const char *optstring); @@ -163,7 +165,7 @@ int main(int argc, char** argv) bool do_verbose(false); int option(-1); - while (-1 != (option = getopt(argc, argv, "u:c:h?RvH:"))) + while (-1 != (option = getopt(argc, argv, "u:c:h?RvH:p:t:"))) { switch (option) { @@ -193,7 +195,7 @@ int main(int argc, char** argv) char * end; value = strtoul(optarg, &end, 10); - if (value < 1 || value > 100 || *end != '\0') + if (value < 1 || value > 200 || *end != '\0') { usage(std::cerr); return 1; @@ -202,6 +204,36 @@ int main(int argc, char** argv) } break; + case 'p': + { + unsigned long value; + char * end; + + value = strtoul(optarg, &end, 10); + if (value < 0 || value > 100 || *end != '\0') + { + usage(std::cerr); + return 1; + } + pipeline_depth = value; + } + break; + + case '5': + { + unsigned long value; + char * end; + + value = strtoul(optarg, &end, 10); + if (value < 0 || value > 3 || *end != '\0') + { + usage(std::cerr); + return 1; + } + tracing = value; + } + break; + case 'R': do_random = true; break; @@ -240,6 +272,24 @@ int main(int argc, char** argv) LLCore::HttpRequest::DEFAULT_POLICY_ID, concurrency_limit, NULL); + LLCore::HttpRequest::setStaticPolicyOption(LLCore::HttpRequest::PO_PER_HOST_CONNECTION_LIMIT, + LLCore::HttpRequest::DEFAULT_POLICY_ID, + concurrency_limit, + NULL); + if (pipeline_depth) + { + LLCore::HttpRequest::setStaticPolicyOption(LLCore::HttpRequest::PO_PIPELINING_DEPTH, + LLCore::HttpRequest::DEFAULT_POLICY_ID, + pipeline_depth, + NULL); + } + if (tracing) + { + LLCore::HttpRequest::setStaticPolicyOption(LLCore::HttpRequest::PO_TRACE, + LLCore::HttpRequest::DEFAULT_POLICY_ID, + tracing, + NULL); + } LLCore::HttpRequest::startThread(); // Get service point @@ -334,7 +384,11 @@ void usage(std::ostream & out) " -c Maximum connection concurrency. Range: [1..100]\n" " Default: " << concurrency_limit << "\n" " -H HTTP request highwater (requests fed to llcorehttp).\n" - " Range: [1..100] Default: " << highwater << "\n" + " Range: [1..200] Default: " << highwater << "\n" + " -p If is positive, enables and sets pipelineing\n" + " depth on HTTP requests. Default: " << pipeline_depth << "\n" + " -t If is positive ([1..3]), enables and sets HTTP\n" + " tracing on HTTP requests. Default: " << tracing << "\n" " -v Verbose mode. Issue some chatter while running\n" " -h print this help\n" "\n" -- cgit v1.3 From e79a88c8ccfadcd260892000d4dec2ae921b26de Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Tue, 12 Aug 2014 18:21:26 -0400 Subject: Better support for dynamic option changes in llcorehttp. Libcurl has some problems disabling pipelining on a multi handle with outstanding requests so build a more conservative system that allows requests to drain before setting curl multi options. Would rather not have this but it is significantly safer. "HttpPipelining" debug setting is now fully dynamic. Connection limits can also be made dynamic in the near future. Upped the default connection count back to 8 for now but will revisit this in the tuning phase. It might be time to combine mesh and textures into a single asset class. For normal server operations that would be a clear path, but for server under load, the current scheme may be better. Minor cleanup in logging to elminate some redundant strings. Might add some more tracing to the stall logic 'just in case'. --- indra/llcorehttp/_httplibcurl.cpp | 198 ++++++++++++++++++++++---------- indra/llcorehttp/_httplibcurl.h | 12 +- indra/llcorehttp/_httpoperation.cpp | 28 +++-- indra/llcorehttp/_httpoprequest.cpp | 46 ++++---- indra/llcorehttp/_httppolicy.cpp | 83 ++++++++----- indra/llcorehttp/_httppolicy.h | 8 ++ indra/llcorehttp/_httpservice.cpp | 30 +++-- indra/newview/app_settings/settings.xml | 4 +- indra/newview/llappcorehttp.cpp | 174 ++++++++++++++++------------ indra/newview/llappcorehttp.h | 1 + indra/newview/lltexturefetch.cpp | 55 +++++---- indra/newview/lltexturefetch.h | 10 +- 12 files changed, 421 insertions(+), 228 deletions(-) (limited to 'indra/llcorehttp') diff --git a/indra/llcorehttp/_httplibcurl.cpp b/indra/llcorehttp/_httplibcurl.cpp index fb907f6318..b46833a1f3 100755 --- a/indra/llcorehttp/_httplibcurl.cpp +++ b/indra/llcorehttp/_httplibcurl.cpp @@ -40,6 +40,8 @@ namespace void check_curl_multi_code(CURLMcode code); void check_curl_multi_code(CURLMcode code, int curl_setopt_option); +static const char * const LOG_CORE("CoreHttp"); + } // end anonymous namespace @@ -51,7 +53,8 @@ HttpLibcurl::HttpLibcurl(HttpService * service) : mService(service), mPolicyCount(0), mMultiHandles(NULL), - mActiveHandles(NULL) + mActiveHandles(NULL), + mDirtyPolicy(NULL) {} @@ -90,6 +93,9 @@ void HttpLibcurl::shutdown() delete [] mActiveHandles; mActiveHandles = NULL; + + delete [] mDirtyPolicy; + mDirtyPolicy = NULL; } mPolicyCount = 0; @@ -101,44 +107,21 @@ void HttpLibcurl::start(int policy_count) llassert_always(policy_count <= HTTP_POLICY_CLASS_LIMIT); llassert_always(! mMultiHandles); // One-time call only - HttpPolicy & policy(mService->getPolicy()); mPolicyCount = policy_count; mMultiHandles = new CURLM * [mPolicyCount]; mActiveHandles = new int [mPolicyCount]; + mDirtyPolicy = new bool [mPolicyCount]; for (int policy_class(0); policy_class < mPolicyCount; ++policy_class) { - HttpPolicyClass & options(policy.getClassOptions(policy_class)); - - mActiveHandles[policy_class] = 0; if (NULL == (mMultiHandles[policy_class] = curl_multi_init())) { - LL_ERRS("CoreHttp") << "Failed to allocate multi handle in libcurl." - << LL_ENDL; - } - - if (options.mPipelining > 1) - { - CURLMcode code; - - // We'll try to do pipelining on this multihandle - code = curl_multi_setopt(mMultiHandles[policy_class], - CURLMOPT_PIPELINING, - 1L); - check_curl_multi_code(code, CURLMOPT_PIPELINING); - code = curl_multi_setopt(mMultiHandles[policy_class], - CURLMOPT_MAX_PIPELINE_LENGTH, - long(options.mPipelining)); - check_curl_multi_code(code, CURLMOPT_MAX_PIPELINE_LENGTH); - code = curl_multi_setopt(mMultiHandles[policy_class], - CURLMOPT_MAX_HOST_CONNECTIONS, - long(options.mPerHostConnectionLimit)); - check_curl_multi_code(code, CURLMOPT_MAX_HOST_CONNECTIONS); - code = curl_multi_setopt(mMultiHandles[policy_class], - CURLMOPT_MAX_TOTAL_CONNECTIONS, - long(options.mConnectionLimit)); - check_curl_multi_code(code, CURLMOPT_MAX_TOTAL_CONNECTIONS); + LL_ERRS(LOG_CORE) << "Failed to allocate multi handle in libcurl." + << LL_ENDL; } + mActiveHandles[policy_class] = 0; + mDirtyPolicy[policy_class] = false; + policyUpdated(policy_class); } } @@ -156,8 +139,19 @@ HttpService::ELoopSpeed HttpLibcurl::processTransport() // Give libcurl some cycles to do I/O & callbacks for (int policy_class(0); policy_class < mPolicyCount; ++policy_class) { - if (! mActiveHandles[policy_class] || ! mMultiHandles[policy_class]) + if (! mMultiHandles[policy_class]) { + // No handle, nothing to do. + continue; + } + if (! mActiveHandles[policy_class]) + { + // If we've gone quiet and there's a dirty update, apply it, + // otherwise we're done. + if (mDirtyPolicy[policy_class]) + { + policyUpdated(policy_class); + } continue; } @@ -192,9 +186,9 @@ HttpService::ELoopSpeed HttpLibcurl::processTransport() } else { - LL_WARNS_ONCE("CoreHttp") << "Unexpected message from libcurl. Msg code: " - << msg->msg - << LL_ENDL; + LL_WARNS_ONCE(LOG_CORE) << "Unexpected message from libcurl. Msg code: " + << msg->msg + << LL_ENDL; } msgs_in_queue = 0; } @@ -230,11 +224,11 @@ void HttpLibcurl::addOp(HttpOpRequest * op) { HttpPolicy & policy(mService->getPolicy()); - LL_INFOS("CoreHttp") << "TRACE, ToActiveQueue, Handle: " - << static_cast(op) - << ", Actives: " << mActiveOps.size() - << ", Readies: " << policy.getReadyCount(op->mReqPolicy) - << LL_ENDL; + LL_INFOS(LOG_CORE) << "TRACE, ToActiveQueue, Handle: " + << static_cast(op) + << ", Actives: " << mActiveOps.size() + << ", Readies: " << policy.getReadyCount(op->mReqPolicy) + << LL_ENDL; } // On success, make operation active @@ -286,10 +280,10 @@ void HttpLibcurl::cancelRequest(HttpOpRequest * op) // Tracing if (op->mTracing > HTTP_TRACE_OFF) { - LL_INFOS("CoreHttp") << "TRACE, RequestCanceled, Handle: " - << static_cast(op) - << ", Status: " << op->mStatus.toTerseString() - << LL_ENDL; + LL_INFOS(LOG_CORE) << "TRACE, RequestCanceled, Handle: " + << static_cast(op) + << ", Status: " << op->mStatus.toTerseString() + << LL_ENDL; } // Cancel op and deliver for notification @@ -306,18 +300,18 @@ bool HttpLibcurl::completeRequest(CURLM * multi_handle, CURL * handle, CURLcode if (handle != op->mCurlHandle || ! op->mCurlActive) { - LL_WARNS("CoreHttp") << "libcurl handle and HttpOpRequest handle in disagreement or inactive request." - << " Handle: " << static_cast(handle) - << LL_ENDL; + LL_WARNS(LOG_CORE) << "libcurl handle and HttpOpRequest handle in disagreement or inactive request." + << " Handle: " << static_cast(handle) + << LL_ENDL; return false; } active_set_t::iterator it(mActiveOps.find(op)); if (mActiveOps.end() == it) { - LL_WARNS("CoreHttp") << "libcurl completion for request not on active list. Continuing." - << " Handle: " << static_cast(handle) - << LL_ENDL; + LL_WARNS(LOG_CORE) << "libcurl completion for request not on active list. Continuing." + << " Handle: " << static_cast(handle) + << LL_ENDL; return false; } @@ -348,9 +342,9 @@ bool HttpLibcurl::completeRequest(CURLM * multi_handle, CURL * handle, CURLcode } else { - LL_WARNS("CoreHttp") << "Invalid HTTP response code (" - << http_status << ") received from server." - << LL_ENDL; + LL_WARNS(LOG_CORE) << "Invalid HTTP response code (" + << http_status << ") received from server." + << LL_ENDL; op->mStatus = HttpStatus(HttpStatus::LLCORE, HE_INVALID_HTTP_STATUS); } } @@ -363,10 +357,10 @@ bool HttpLibcurl::completeRequest(CURLM * multi_handle, CURL * handle, CURLcode // Tracing if (op->mTracing > HTTP_TRACE_OFF) { - LL_INFOS("CoreHttp") << "TRACE, RequestComplete, Handle: " - << static_cast(op) - << ", Status: " << op->mStatus.toTerseString() - << LL_ENDL; + LL_INFOS(LOG_CORE) << "TRACE, RequestComplete, Handle: " + << static_cast(op) + << ", Status: " << op->mStatus.toTerseString() + << LL_ENDL; } // Dispatch to next stage @@ -390,6 +384,88 @@ int HttpLibcurl::getActiveCountInClass(int policy_class) const return mActiveHandles ? mActiveHandles[policy_class] : 0; } +void HttpLibcurl::policyUpdated(int policy_class) +{ + if (policy_class < 0 || policy_class >= mPolicyCount || ! mMultiHandles) + { + return; + } + + HttpPolicy & policy(mService->getPolicy()); + + if (! mActiveHandles[policy_class]) + { + // Clear to set options. As of libcurl 7.37.0, if a pipelining + // multi handle has active requests and you try to set the + // multi handle to non-pipelining, the library gets very angry + // and goes off the rails corrupting memory. A clue that you're + // about to crash is that you'll get a missing server response + // error (curl code 9). So, if options are to be set, we let + // the multi handle run out of requests, then set options, and + // re-enable request processing. + // + // All of this stall mechanism exists for this reason. If + // libcurl becomes more resilient later, it should be possible + // to remove all of this. The connection limit settings are fine, + // it's just that pipelined-to-non-pipelined transition that + // is fatal at the moment. + + HttpPolicyClass & options(policy.getClassOptions(policy_class)); + CURLM * multi_handle(mMultiHandles[policy_class]); + CURLMcode code; + + // Enable policy if stalled + policy.stallPolicy(policy_class, false); + mDirtyPolicy[policy_class] = false; + + if (options.mPipelining > 1) + { + // We'll try to do pipelining on this multihandle + code = curl_multi_setopt(multi_handle, + CURLMOPT_PIPELINING, + 1L); + check_curl_multi_code(code, CURLMOPT_PIPELINING); + code = curl_multi_setopt(multi_handle, + CURLMOPT_MAX_PIPELINE_LENGTH, + long(options.mPipelining)); + check_curl_multi_code(code, CURLMOPT_MAX_PIPELINE_LENGTH); + code = curl_multi_setopt(multi_handle, + CURLMOPT_MAX_HOST_CONNECTIONS, + long(options.mPerHostConnectionLimit)); + check_curl_multi_code(code, CURLMOPT_MAX_HOST_CONNECTIONS); + code = curl_multi_setopt(multi_handle, + CURLMOPT_MAX_TOTAL_CONNECTIONS, + long(options.mConnectionLimit)); + check_curl_multi_code(code, CURLMOPT_MAX_TOTAL_CONNECTIONS); + } + else + { + code = curl_multi_setopt(multi_handle, + CURLMOPT_PIPELINING, + 0L); + check_curl_multi_code(code, CURLMOPT_PIPELINING); + code = curl_multi_setopt(multi_handle, + CURLMOPT_MAX_HOST_CONNECTIONS, + 0L); + check_curl_multi_code(code, CURLMOPT_MAX_HOST_CONNECTIONS); + code = curl_multi_setopt(multi_handle, + CURLMOPT_MAX_TOTAL_CONNECTIONS, + long(options.mConnectionLimit)); + check_curl_multi_code(code, CURLMOPT_MAX_TOTAL_CONNECTIONS); + } + } + else if (! mDirtyPolicy[policy_class]) + { + // Mark policy dirty and request a stall in the policy. + // When policy goes idle, we'll re-invoke this method + // and perform the change. Don't allow this thread to + // sleep while we're waiting for quiescence, we'll just + // stop processing. + mDirtyPolicy[policy_class] = true; + policy.stallPolicy(policy_class, true); + } +} + // --------------------------------------- // Free functions @@ -424,9 +500,9 @@ void check_curl_multi_code(CURLMcode code, int curl_setopt_option) { if (CURLM_OK != code) { - LL_WARNS("CoreHttp") << "libcurl multi error detected: " << curl_multi_strerror(code) - << ", curl_multi_setopt option: " << curl_setopt_option - << LL_ENDL; + LL_WARNS(LOG_CORE) << "libcurl multi error detected: " << curl_multi_strerror(code) + << ", curl_multi_setopt option: " << curl_setopt_option + << LL_ENDL; } } @@ -435,8 +511,8 @@ void check_curl_multi_code(CURLMcode code) { if (CURLM_OK != code) { - LL_WARNS("CoreHttp") << "libcurl multi error detected: " << curl_multi_strerror(code) - << LL_ENDL; + LL_WARNS(LOG_CORE) << "libcurl multi error detected: " << curl_multi_strerror(code) + << LL_ENDL; } } diff --git a/indra/llcorehttp/_httplibcurl.h b/indra/llcorehttp/_httplibcurl.h index 67f98dd4f0..2c7ad1fa8e 100755 --- a/indra/llcorehttp/_httplibcurl.h +++ b/indra/llcorehttp/_httplibcurl.h @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012-2013, Linden Research, Inc. + * Copyright (C) 2012-2014, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -116,6 +116,14 @@ public: /// Threading: called by worker thread. bool cancel(HttpHandle handle); + /// Informs transport that a particular policy class has had + /// options changed and so should effect any transport state + /// change necessary to effect those changes. Used mainly for + /// initialization and dynamic option setting. + /// + /// Threading: called by worker thread. + void policyUpdated(int policy_class); + protected: /// Invoked when libcurl has indicated a request has been processed /// to completion and we need to move the request to a new state. @@ -134,6 +142,8 @@ protected: int mPolicyCount; CURLM ** mMultiHandles; // One handle per policy class int * mActiveHandles; // Active count per policy class + bool * mDirtyPolicy; // Dirty policy update waiting for stall (per pc) + }; // end class HttpLibcurl } // end namespace LLCore diff --git a/indra/llcorehttp/_httpoperation.cpp b/indra/llcorehttp/_httpoperation.cpp index 5bb0654652..fefe561f80 100755 --- a/indra/llcorehttp/_httpoperation.cpp +++ b/indra/llcorehttp/_httpoperation.cpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012-2013, Linden Research, Inc. + * Copyright (C) 2012-2014, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -38,6 +38,14 @@ #include "lltimer.h" +namespace +{ + +static const char * const LOG_CORE("CoreHttp"); + +} // end anonymous namespace + + namespace LLCore { @@ -94,8 +102,8 @@ void HttpOperation::stageFromRequest(HttpService *) // Default implementation should never be called. This // indicates an operation making a transition that isn't // defined. - LL_ERRS("CoreHttp") << "Default stageFromRequest method may not be called." - << LL_ENDL; + LL_ERRS(LOG_CORE) << "Default stageFromRequest method may not be called." + << LL_ENDL; } @@ -104,8 +112,8 @@ void HttpOperation::stageFromReady(HttpService *) // Default implementation should never be called. This // indicates an operation making a transition that isn't // defined. - LL_ERRS("CoreHttp") << "Default stageFromReady method may not be called." - << LL_ENDL; + LL_ERRS(LOG_CORE) << "Default stageFromReady method may not be called." + << LL_ENDL; } @@ -114,8 +122,8 @@ void HttpOperation::stageFromActive(HttpService *) // Default implementation should never be called. This // indicates an operation making a transition that isn't // defined. - LL_ERRS("CoreHttp") << "Default stageFromActive method may not be called." - << LL_ENDL; + LL_ERRS(LOG_CORE) << "Default stageFromActive method may not be called." + << LL_ENDL; } @@ -145,9 +153,9 @@ void HttpOperation::addAsReply() { if (mTracing > HTTP_TRACE_OFF) { - LL_INFOS("CoreHttp") << "TRACE, ToReplyQueue, Handle: " - << static_cast(this) - << LL_ENDL; + LL_INFOS(LOG_CORE) << "TRACE, ToReplyQueue, Handle: " + << static_cast(this) + << LL_ENDL; } if (mReplyQueue) diff --git a/indra/llcorehttp/_httpoprequest.cpp b/indra/llcorehttp/_httpoprequest.cpp index 43dd069bc6..eb664fdced 100755 --- a/indra/llcorehttp/_httpoprequest.cpp +++ b/indra/llcorehttp/_httpoprequest.cpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012-2013, Linden Research, Inc. + * Copyright (C) 2012-2014, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -94,6 +94,8 @@ void os_strlower(char * str); void check_curl_easy_code(CURLcode code); void check_curl_easy_code(CURLcode code, int curl_setopt_option); +static const char * const LOG_CORE("CoreHttp"); + } // end anonymous namespace @@ -416,8 +418,8 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service) if (! mCurlHandle) { // We're in trouble. We'll continue but it won't go well. - LL_WARNS("CoreHttp") << "Failed to allocate libcurl easy handle. Continuing." - << LL_ENDL; + LL_WARNS(LOG_CORE) << "Failed to allocate libcurl easy handle. Continuing." + << LL_ENDL; return HttpStatus(HttpStatus::LLCORE, HE_BAD_ALLOC); } code = curl_easy_setopt(mCurlHandle, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4); @@ -538,9 +540,9 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service) break; default: - LL_ERRS("CoreHttp") << "Invalid HTTP method in request: " - << int(mReqMethod) << ". Can't recover." - << LL_ENDL; + LL_ERRS(LOG_CORE) << "Invalid HTTP method in request: " + << int(mReqMethod) << ". Can't recover." + << LL_ENDL; break; } @@ -652,8 +654,8 @@ size_t HttpOpRequest::readCallback(void * data, size_t size, size_t nmemb, void { // Warn but continue if the read position moves beyond end-of-body // for some reason. - LL_WARNS("CoreHttp") << "Request body position beyond body size. Truncating request body." - << LL_ENDL; + LL_WARNS(LOG_CORE) << "Request body position beyond body size. Truncating request body." + << LL_ENDL; } return 0; } @@ -790,10 +792,10 @@ size_t HttpOpRequest::headerCallback(void * data, size_t size, size_t nmemb, voi else { // Ignore the unparsable. - LL_INFOS_ONCE("CoreHttp") << "Problem parsing odd Content-Range header: '" - << std::string(hdr_data, wanted_hdr_size) - << "'. Ignoring." - << LL_ENDL; + LL_INFOS_ONCE(LOG_CORE) << "Problem parsing odd Content-Range header: '" + << std::string(hdr_data, wanted_hdr_size) + << "'. Ignoring." + << LL_ENDL; } } @@ -895,11 +897,11 @@ int HttpOpRequest::debugCallback(CURL * handle, curl_infotype info, char * buffe if (logit) { - LL_INFOS("CoreHttp") << "TRACE, LibcurlDebug, Handle: " - << static_cast(op) - << ", Type: " << tag - << ", Data: " << safe_line - << LL_ENDL; + LL_INFOS(LOG_CORE) << "TRACE, LibcurlDebug, Handle: " + << static_cast(op) + << ", Type: " << tag + << ", Data: " << safe_line + << LL_ENDL; } return 0; @@ -1094,9 +1096,9 @@ void check_curl_easy_code(CURLcode code, int curl_setopt_option) // // linux appears to throw a curl error once per session for a bad initialization // at a pretty random time (when enabling cookies). - LL_WARNS("CoreHttp") << "libcurl error detected: " << curl_easy_strerror(code) - << ", curl_easy_setopt option: " << curl_setopt_option - << LL_ENDL; + LL_WARNS(LOG_CORE) << "libcurl error detected: " << curl_easy_strerror(code) + << ", curl_easy_setopt option: " << curl_setopt_option + << LL_ENDL; } } @@ -1109,8 +1111,8 @@ void check_curl_easy_code(CURLcode code) // // linux appears to throw a curl error once per session for a bad initialization // at a pretty random time (when enabling cookies). - LL_WARNS("CoreHttp") << "libcurl error detected: " << curl_easy_strerror(code) - << LL_ENDL; + LL_WARNS(LOG_CORE) << "libcurl error detected: " << curl_easy_strerror(code) + << LL_ENDL; } } diff --git a/indra/llcorehttp/_httppolicy.cpp b/indra/llcorehttp/_httppolicy.cpp index bb7959b578..09b9206f63 100755 --- a/indra/llcorehttp/_httppolicy.cpp +++ b/indra/llcorehttp/_httppolicy.cpp @@ -35,6 +35,13 @@ #include "lltimer.h" +namespace +{ + +static const char * const LOG_CORE("CoreHttp"); + +} // end anonymous namespace + namespace LLCore { @@ -51,7 +58,8 @@ public: ClassState() : mThrottleEnd(0), mThrottleLeft(0L), - mRequestCount(0L) + mRequestCount(0L), + mStallStaging(false) {} HttpReadyQueue mReadyQueue; @@ -61,6 +69,7 @@ public: HttpTime mThrottleEnd; long mThrottleLeft; long mRequestCount; + bool mStallStaging; }; @@ -171,19 +180,19 @@ void HttpPolicy::retryOp(HttpOpRequest * op) { ++op->mPolicy503Retries; } - LL_DEBUGS("CoreHttp") << "HTTP request " << static_cast(op) - << " retry " << op->mPolicyRetries - << " scheduled in " << (delta / HttpTime(1000)) - << " mS (" << (external_delta ? "external" : "internal") - << "). Status: " << op->mStatus.toTerseString() - << LL_ENDL; + LL_DEBUGS(LOG_CORE) << "HTTP request " << static_cast(op) + << " retry " << op->mPolicyRetries + << " scheduled in " << (delta / HttpTime(1000)) + << " mS (" << (external_delta ? "external" : "internal") + << "). Status: " << op->mStatus.toTerseString() + << LL_ENDL; if (op->mTracing > HTTP_TRACE_OFF) { - LL_INFOS("CoreHttp") << "TRACE, ToRetryQueue, Handle: " - << static_cast(op) - << ", Delta: " << (delta / HttpTime(1000)) - << ", Retries: " << op->mPolicyRetries - << LL_ENDL; + LL_INFOS(LOG_CORE) << "TRACE, ToRetryQueue, Handle: " + << static_cast(op) + << ", Delta: " << (delta / HttpTime(1000)) + << ", Retries: " << op->mPolicyRetries + << LL_ENDL; } mClasses[policy_class]->mRetryQueue.push(op); } @@ -219,6 +228,15 @@ HttpService::ELoopSpeed HttpPolicy::processReadyQueue() HttpRetryQueue & retryq(state.mRetryQueue); HttpReadyQueue & readyq(state.mReadyQueue); + if (state.mStallStaging) + { + // Stalling but don't sleep. Need to complete operations + // and get back to servicing queues. Do this test before + // the retryq/readyq test or you'll get stalls until you + // click a setting or an asset request comes in. + result = HttpService::NORMAL; + continue; + } if (retryq.empty() && readyq.empty()) { continue; @@ -262,9 +280,9 @@ HttpService::ELoopSpeed HttpPolicy::processReadyQueue() if (now >= state.mThrottleEnd) { // Throttle expired, move to next window - LL_DEBUGS("CoreHttp") << "Throttle expired with " << state.mThrottleLeft - << " requests to go and " << state.mRequestCount - << " requests issued." << LL_ENDL; + LL_DEBUGS(LOG_CORE) << "Throttle expired with " << state.mThrottleLeft + << " requests to go and " << state.mRequestCount + << " requests issued." << LL_ENDL; state.mThrottleLeft = state.mOptions.mThrottleRate; state.mThrottleEnd = now + HttpTime(1000000); } @@ -291,9 +309,9 @@ HttpService::ELoopSpeed HttpPolicy::processReadyQueue() if (now >= state.mThrottleEnd) { // Throttle expired, move to next window - LL_DEBUGS("CoreHttp") << "Throttle expired with " << state.mThrottleLeft - << " requests to go and " << state.mRequestCount - << " requests issued." << LL_ENDL; + LL_DEBUGS(LOG_CORE) << "Throttle expired with " << state.mThrottleLeft + << " requests to go and " << state.mRequestCount + << " requests issued." << LL_ENDL; state.mThrottleLeft = state.mOptions.mThrottleRate; state.mThrottleEnd = now + HttpTime(1000000); } @@ -408,17 +426,17 @@ bool HttpPolicy::stageAfterCompletion(HttpOpRequest * op) // This op is done, finalize it delivering it to the reply queue... if (! op->mStatus) { - LL_WARNS("CoreHttp") << "HTTP request " << static_cast(op) - << " failed after " << op->mPolicyRetries - << " retries. Reason: " << op->mStatus.toString() - << " (" << op->mStatus.toTerseString() << ")" - << LL_ENDL; + LL_WARNS(LOG_CORE) << "HTTP request " << static_cast(op) + << " failed after " << op->mPolicyRetries + << " retries. Reason: " << op->mStatus.toString() + << " (" << op->mStatus.toTerseString() << ")" + << LL_ENDL; } else if (op->mPolicyRetries) { - LL_DEBUGS("CoreHttp") << "HTTP request " << static_cast(op) - << " succeeded on retry " << op->mPolicyRetries << "." - << LL_ENDL; + LL_DEBUGS(LOG_CORE) << "HTTP request " << static_cast(op) + << " succeeded on retry " << op->mPolicyRetries << "." + << LL_ENDL; } op->stageFromActive(mService); @@ -446,4 +464,17 @@ int HttpPolicy::getReadyCount(HttpRequest::policy_t policy_class) const } +bool HttpPolicy::stallPolicy(HttpRequest::policy_t policy_class, bool stall) +{ + bool ret(false); + + if (policy_class < mClasses.size()) + { + ret = mClasses[policy_class]->mStallStaging; + mClasses[policy_class]->mStallStaging = stall; + } + return ret; +} + + } // end namespace LLCore diff --git a/indra/llcorehttp/_httppolicy.h b/indra/llcorehttp/_httppolicy.h index bf1aa74267..11cd89bbd1 100755 --- a/indra/llcorehttp/_httppolicy.h +++ b/indra/llcorehttp/_httppolicy.h @@ -158,6 +158,14 @@ public: /// Threading: called by worker thread int getReadyCount(HttpRequest::policy_t policy_class) const; + /// Stall (or unstall) a policy class preventing requests from + /// transitioning to an active state. Used to allow an HTTP + /// request policy to empty prior to changing settings or state + /// that isn't tolerant of changes when work is outstanding. + /// + /// Threading: called by worker thread + bool stallPolicy(HttpRequest::policy_t policy_class, bool stall); + protected: struct ClassState; typedef std::vector class_list_t; diff --git a/indra/llcorehttp/_httpservice.cpp b/indra/llcorehttp/_httpservice.cpp index c94249dc2d..c673e1be1d 100755 --- a/indra/llcorehttp/_httpservice.cpp +++ b/indra/llcorehttp/_httpservice.cpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012-2013, Linden Research, Inc. + * Copyright (C) 2012-2014, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -40,6 +40,14 @@ #include "llthread.h" +namespace +{ + +static const char * const LOG_CORE("CoreHttp"); + +} // end anonymous namespace + + namespace LLCore { @@ -87,8 +95,8 @@ HttpService::~HttpService() // Failed to join, expect problems ahead so do a hard termination. mThread->cancel(); - LL_WARNS("CoreHttp") << "Destroying HttpService with running thread. Expect problems." - << LL_ENDL; + LL_WARNS(LOG_CORE) << "Destroying HttpService with running thread. Expect problems." + << LL_ENDL; } } } @@ -328,9 +336,9 @@ HttpService::ELoopSpeed HttpService::processRequestQueue(ELoopSpeed loop) if (op->mTracing > HTTP_TRACE_OFF) { - LL_INFOS("CoreHttp") << "TRACE, FromRequestQueue, Handle: " - << static_cast(op) - << LL_ENDL; + LL_INFOS(LOG_CORE) << "TRACE, FromRequestQueue, Handle: " + << static_cast(op) + << LL_ENDL; } // Stage @@ -437,9 +445,13 @@ HttpStatus HttpService::setPolicyOption(HttpRequest::EPolicyOption opt, HttpRequ HttpPolicyClass & opts(mPolicy->getClassOptions(pclass)); status = opts.set(opt, value); - if (status && ret_value) + if (status) { - status = opts.get(opt, ret_value); + mTransport->policyUpdated(pclass); + if (ret_value) + { + status = opts.get(opt, ret_value); + } } } @@ -463,7 +475,7 @@ HttpStatus HttpService::setPolicyOption(HttpRequest::EPolicyOption opt, HttpRequ return status; } - // Only string values are global at this time + // String values are always global (at this time). if (pclass == HttpRequest::GLOBAL_POLICY_ID) { HttpPolicyGlobal & opts(mPolicy->getGlobalOptions()); diff --git a/indra/newview/app_settings/settings.xml b/indra/newview/app_settings/settings.xml index 50f56cf6ff..0607579a08 100755 --- a/indra/newview/app_settings/settings.xml +++ b/indra/newview/app_settings/settings.xml @@ -4459,7 +4459,7 @@ HttpPipelining Comment - If true, viewer will pipeline HTTP requests to servers. Static. + If true, viewer will pipeline HTTP requests to servers. Persist 1 Type @@ -4470,7 +4470,7 @@ HttpRangeRequestsDisable Comment - If true, viewer will not issued range GET requests for meshes and textures. + If true, viewer will not issued range GET requests for meshes and textures. May resolve problems with certain ISPs and networking gear. Persist 1 Type diff --git a/indra/newview/llappcorehttp.cpp b/indra/newview/llappcorehttp.cpp index d097f18d61..464e60948a 100755 --- a/indra/newview/llappcorehttp.cpp +++ b/indra/newview/llappcorehttp.cpp @@ -60,7 +60,7 @@ static const struct "other" }, { // AP_TEXTURE - 4, 1, 12, 0, true, + 8, 1, 12, 0, true, "TextureFetchConcurrency", "texture fetch" }, @@ -70,7 +70,7 @@ static const struct "mesh fetch" }, { // AP_MESH2 - 4, 1, 32, 100, true, + 8, 1, 32, 100, true, "Mesh2MaxConcurrentRequests", "mesh2 fetch" }, @@ -126,14 +126,6 @@ void LLAppCoreHttp::init() << LL_ENDL; } - // Global pipelining preference from settings - static const std::string http_pipelining("HttpPipelining"); - if (gSavedSettings.controlExists(http_pipelining)) - { - // Default to true if absent. - mPipelined = gSavedSettings.getBOOL(http_pipelining); - } - // Point to our certs or SSH/https: will fail on connect status = LLCore::HttpRequest::setStaticPolicyOption(LLCore::HttpRequest::PO_CA_FILE, LLCore::HttpRequest::GLOBAL_POLICY_ID, @@ -210,12 +202,27 @@ void LLAppCoreHttp::init() << LL_ENDL; } - // *NOTE: Pipelining isn't dynamic yet. When it is, add a global - // signal for the setting here. - + // Signal for global pipelining preference from settings + static const std::string http_pipelining("HttpPipelining"); + if (gSavedSettings.controlExists(http_pipelining)) + { + LLPointer cntrl_ptr = gSavedSettings.getControl(http_pipelining); + if (cntrl_ptr.isNull()) + { + LL_WARNS("Init") << "Unable to set signal on global setting '" << http_pipelining + << "'" << LL_ENDL; + } + else + { + mPipelinedSignal = cntrl_ptr->getCommitSignal()->connect(boost::bind(&setting_changed)); + } + } + // Register signals for settings and state changes for (int i(0); i < LL_ARRAY_SIZE(init_data); ++i) { + const EAppPolicy app_policy(static_cast(i)); + if (! init_data[i].mKey.empty() && gSavedSettings.controlExists(init_data[i].mKey)) { LLPointer cntrl_ptr = gSavedSettings.getControl(init_data[i].mKey); @@ -226,7 +233,7 @@ void LLAppCoreHttp::init() } else { - mHttpClasses[i].mSettingsSignal = cntrl_ptr->getCommitSignal()->connect(boost::bind(&setting_changed)); + mHttpClasses[app_policy].mSettingsSignal = cntrl_ptr->getCommitSignal()->connect(boost::bind(&setting_changed)); } } } @@ -282,6 +289,7 @@ void LLAppCoreHttp::cleanup() { mHttpClasses[i].mSettingsSignal.disconnect(); } + mPipelinedSignal.disconnect(); delete mRequest; mRequest = NULL; @@ -299,6 +307,20 @@ void LLAppCoreHttp::cleanup() void LLAppCoreHttp::refreshSettings(bool initial) { LLCore::HttpStatus status; + + // Global pipelining setting + bool pipeline_changed(false); + static const std::string http_pipelining("HttpPipelining"); + if (gSavedSettings.controlExists(http_pipelining)) + { + // Default to true (in ctor) if absent. + bool pipelined(gSavedSettings.getBOOL(http_pipelining)); + if (pipelined != mPipelined) + { + mPipelined = pipelined; + pipeline_changed = true; + } + } for (int i(0); i < LL_ARRAY_SIZE(init_data); ++i) { @@ -323,33 +345,42 @@ void LLAppCoreHttp::refreshSettings(bool initial) } } - mHttpClasses[app_policy].mPipelined = false; - if (mPipelined && init_data[i].mPipelined) + } + + // Init- or run-time settings. Must use the queued request API. + + // Pipelining changes + if (initial || pipeline_changed) + { + const bool to_pipeline(mPipelined && init_data[i].mPipelined); + if (to_pipeline != mHttpClasses[app_policy].mPipelined) { - // Pipelining election is currently static (init-time). - // Making it dynamic isn't too hard in the SL code but verifying - // that libcurl handles the on-to-off transition while holding - // outstanding requests is something that should be tested. + // Pipeline election changing, set dynamic option via request + + LLCore::HttpHandle handle; + const long new_depth(to_pipeline ? PIPELINING_DEPTH : 0); - status = LLCore::HttpRequest::setStaticPolicyOption(LLCore::HttpRequest::PO_PIPELINING_DEPTH, - mHttpClasses[app_policy].mPolicy, - PIPELINING_DEPTH, - NULL); - if (! status) + handle = mRequest->setPolicyOption(LLCore::HttpRequest::PO_PIPELINING_DEPTH, + mHttpClasses[app_policy].mPolicy, + new_depth, + NULL); + if (LLCORE_HTTP_HANDLE_INVALID == handle) { + status = mRequest->getStatus(); LL_WARNS("Init") << "Unable to set " << init_data[i].mUsage - << " to pipelined mode. Reason: " << status.toString() + << " pipelining. Reason: " << status.toString() << LL_ENDL; } else { - mHttpClasses[app_policy].mPipelined = true; + LL_DEBUGS("Init") << "Changed " << init_data[i].mUsage + << " pipelining. New value: " << new_depth + << LL_ENDL; + mHttpClasses[app_policy].mPipelined = to_pipeline; } } } - - // Init- or run-time settings - + // Get target connection concurrency value U32 setting(init_data[i].mDefault); if (! init_data[i].mKey.empty() && gSavedSettings.controlExists(init_data[i].mKey)) @@ -362,63 +393,60 @@ void LLAppCoreHttp::refreshSettings(bool initial) } } - if (! initial && setting == mHttpClasses[app_policy].mConnLimit) + if (initial || setting != mHttpClasses[app_policy].mConnLimit || pipeline_changed) { - // Unchanged, try next setting - continue; - } - - // Set it and report. Strategies depend on pipelining: - // - // No Pipelining. Llcorehttp manages connections itself based - // on the PO_CONNECTION_LIMIT setting. Set both limits to the - // same value for logical consistency. In the future, may - // hand over connection management to libcurl after the - // connection cache has been better vetted. - // - // Pipelining. Libcurl is allowed to manage connections to a - // great degree. Steady state will connection limit based on - // the per-host setting. Transitions (region crossings, new - // avatars, etc.) can request additional outbound connections - // to other servers via 2X total connection limit. - // - LLCore::HttpHandle handle; - handle = mRequest->setPolicyOption(LLCore::HttpRequest::PO_CONNECTION_LIMIT, - mHttpClasses[app_policy].mPolicy, - (mHttpClasses[app_policy].mPipelined ? 2 * setting : setting), - NULL); - if (LLCORE_HTTP_HANDLE_INVALID == handle) - { - status = mRequest->getStatus(); - LL_WARNS("Init") << "Unable to set " << init_data[i].mUsage - << " concurrency. Reason: " << status.toString() - << LL_ENDL; - } - else - { - handle = mRequest->setPolicyOption(LLCore::HttpRequest::PO_PER_HOST_CONNECTION_LIMIT, + // Set it and report. Strategies depend on pipelining: + // + // No Pipelining. Llcorehttp manages connections itself based + // on the PO_CONNECTION_LIMIT setting. Set both limits to the + // same value for logical consistency. In the future, may + // hand over connection management to libcurl after the + // connection cache has been better vetted. + // + // Pipelining. Libcurl is allowed to manage connections to a + // great degree. Steady state will connection limit based on + // the per-host setting. Transitions (region crossings, new + // avatars, etc.) can request additional outbound connections + // to other servers via 2X total connection limit. + // + LLCore::HttpHandle handle; + handle = mRequest->setPolicyOption(LLCore::HttpRequest::PO_CONNECTION_LIMIT, mHttpClasses[app_policy].mPolicy, - setting, + (mHttpClasses[app_policy].mPipelined ? 2 * setting : setting), NULL); if (LLCORE_HTTP_HANDLE_INVALID == handle) { status = mRequest->getStatus(); LL_WARNS("Init") << "Unable to set " << init_data[i].mUsage - << " per-host concurrency. Reason: " << status.toString() + << " concurrency. Reason: " << status.toString() << LL_ENDL; } else { - LL_DEBUGS("Init") << "Changed " << init_data[i].mUsage - << " concurrency. New value: " << setting - << LL_ENDL; - mHttpClasses[app_policy].mConnLimit = setting; - if (initial && setting != init_data[i].mDefault) + handle = mRequest->setPolicyOption(LLCore::HttpRequest::PO_PER_HOST_CONNECTION_LIMIT, + mHttpClasses[app_policy].mPolicy, + setting, + NULL); + if (LLCORE_HTTP_HANDLE_INVALID == handle) { - LL_INFOS("Init") << "Application settings overriding default " << init_data[i].mUsage - << " concurrency. New value: " << setting + status = mRequest->getStatus(); + LL_WARNS("Init") << "Unable to set " << init_data[i].mUsage + << " per-host concurrency. Reason: " << status.toString() << LL_ENDL; } + else + { + LL_DEBUGS("Init") << "Changed " << init_data[i].mUsage + << " concurrency. New value: " << setting + << LL_ENDL; + mHttpClasses[app_policy].mConnLimit = setting; + if (initial && setting != init_data[i].mDefault) + { + LL_INFOS("Init") << "Application settings overriding default " << init_data[i].mUsage + << " concurrency. New value: " << setting + << LL_ENDL; + } + } } } } diff --git a/indra/newview/llappcorehttp.h b/indra/newview/llappcorehttp.h index 63c8a11180..9ad4eb4b30 100755 --- a/indra/newview/llappcorehttp.h +++ b/indra/newview/llappcorehttp.h @@ -218,6 +218,7 @@ private: bool mStopped; HttpClass mHttpClasses[AP_COUNT]; bool mPipelined; // Global setting + boost::signals2::connection mPipelinedSignal; // Signal for 'HttpPipelining' setting }; diff --git a/indra/newview/lltexturefetch.cpp b/indra/newview/lltexturefetch.cpp index 4008a6948d..097a7b374f 100755 --- a/indra/newview/lltexturefetch.cpp +++ b/indra/newview/lltexturefetch.cpp @@ -483,12 +483,12 @@ private: bool acquireHttpSemaphore() { llassert(! mHttpHasResource); - if (mFetcher->mHttpSemaphore <= 0) + if (mFetcher->mHttpSemaphore >= mFetcher->mHttpHighWater) { return false; } mHttpHasResource = true; - mFetcher->mHttpSemaphore--; + mFetcher->mHttpSemaphore++; return true; } @@ -498,7 +498,8 @@ private: { llassert(mHttpHasResource); mHttpHasResource = false; - mFetcher->mHttpSemaphore++; + mFetcher->mHttpSemaphore--; + llassert_always(mFetcher->mHttpSemaphore >= 0); } private: @@ -2523,6 +2524,8 @@ LLTextureFetch::LLTextureFetch(LLTextureCache* cache, LLImageDecodeThread* image mMaxBandwidth = gSavedSettings.getF32("ThrottleBandwidthKBPS"); mTextureInfo.setUpLogging(gSavedSettings.getBOOL("LogTextureDownloadsToViewerLog"), gSavedSettings.getBOOL("LogTextureDownloadsToSimulator"), U32Bytes(gSavedSettings.getU32("TextureLoggingThreshold"))); + LLAppCoreHttp & app_core_http(LLAppViewer::instance()->getAppCoreHttp()); + mHttpPolicyClass = app_core_http.getPolicy(LLAppCoreHttp::AP_TEXTURE); mHttpRequest = new LLCore::HttpRequest; mHttpOptions = new LLCore::HttpOptions; mHttpOptionsWithHeaders = new LLCore::HttpOptions; @@ -2531,21 +2534,9 @@ LLTextureFetch::LLTextureFetch(LLTextureCache* cache, LLImageDecodeThread* image mHttpHeaders->append("Accept", "image/x-j2c"); mHttpMetricsHeaders = new LLCore::HttpHeaders; mHttpMetricsHeaders->append("Content-Type", "application/llsd+xml"); - LLAppCoreHttp & app_core_http(LLAppViewer::instance()->getAppCoreHttp()); - mHttpPolicyClass = app_core_http.getPolicy(LLAppCoreHttp::AP_TEXTURE); - if (app_core_http.isPipelined(LLAppCoreHttp::AP_TEXTURE)) - { - // Init-time election that will have to change for - // support of dynamic changes to the pipelining enable flag. - mHttpHighWater = HTTP_PIPE_REQUESTS_HIGH_WATER; - mHttpLowWater = HTTP_PIPE_REQUESTS_LOW_WATER; - } - else - { - mHttpHighWater = HTTP_NONPIPE_REQUESTS_HIGH_WATER; - mHttpLowWater = HTTP_NONPIPE_REQUESTS_LOW_WATER; - } - mHttpSemaphore = mHttpHighWater; + mHttpHighWater = HTTP_NONPIPE_REQUESTS_HIGH_WATER; + mHttpLowWater = HTTP_NONPIPE_REQUESTS_LOW_WATER; + mHttpSemaphore = 0; // Conditionally construct debugger object after 'this' is // fully initialized. @@ -3032,6 +3023,20 @@ bool LLTextureFetch::runCondition() // Threads: Ttf void LLTextureFetch::commonUpdate() { + // Update low/high water levels based on pipelining. We pick + // up setting eventually, so the semaphore/request level can + // fall outside the [0..HIGH_WATER] range. Expect that. + if (LLAppViewer::instance()->getAppCoreHttp().isPipelined(LLAppCoreHttp::AP_TEXTURE)) + { + mHttpHighWater = HTTP_PIPE_REQUESTS_HIGH_WATER; + mHttpLowWater = HTTP_PIPE_REQUESTS_LOW_WATER; + } + else + { + mHttpHighWater = HTTP_NONPIPE_REQUESTS_HIGH_WATER; + mHttpLowWater = HTTP_NONPIPE_REQUESTS_LOW_WATER; + } + // Release waiters releaseHttpWaiters(); @@ -3693,8 +3698,16 @@ void LLTextureFetch::releaseHttpWaiters() { // Use mHttpSemaphore rather than mHTTPTextureQueue.size() // to avoid a lock. - if (mHttpSemaphore < (mHttpHighWater - mHttpLowWater)) + if (mHttpSemaphore >= mHttpLowWater) return; + S32 needed(mHttpHighWater - mHttpSemaphore); + if (needed <= 0) + { + // Would only happen if High/LowWater were changed behind + // our back. In that case, defer fill until usage falls within + // limits. + return; + } // Quickly make a copy of all the LLUIDs. Get off the // mutex as early as possible. @@ -3743,10 +3756,10 @@ void LLTextureFetch::releaseHttpWaiters() tids.clear(); // Sort into priority order, if necessary and only as much as needed - if (tids2.size() > mHttpSemaphore) + if (tids2.size() > needed) { LLTextureFetchWorker::Compare compare; - std::partial_sort(tids2.begin(), tids2.begin() + mHttpSemaphore, tids2.end(), compare); + std::partial_sort(tids2.begin(), tids2.begin() + needed, tids2.end(), compare); } // Release workers up to the high water mark. Since we aren't diff --git a/indra/newview/lltexturefetch.h b/indra/newview/lltexturefetch.h index d13736997f..89d18e2c67 100755 --- a/indra/newview/lltexturefetch.h +++ b/indra/newview/lltexturefetch.h @@ -356,8 +356,8 @@ private: LLCore::HttpHeaders * mHttpHeaders; // Ttf LLCore::HttpHeaders * mHttpMetricsHeaders; // Ttf LLCore::HttpRequest::policy_t mHttpPolicyClass; // T* - S32 mHttpHighWater; // T* (ro) - S32 mHttpLowWater; // T* (ro) + S32 mHttpHighWater; // Ttf + S32 mHttpLowWater; // Ttf // We use a resource semaphore to keep HTTP requests in // WAIT_HTTP_RESOURCE2 if there aren't sufficient slots in the @@ -366,7 +366,11 @@ private: // where it's more expensive to get at them. Requests in either // SEND_HTTP_REQ or WAIT_HTTP_REQ charge against the semaphore // and tracking state transitions is critical to liveness. - LLAtomicS32 mHttpSemaphore; // Ttf + Tmain + // + // Originally implemented as a traditional semaphore (heading towards + // zero), it now is an outstanding request count that is allowed to + // exceed the high water level (but not go below zero). + LLAtomicS32 mHttpSemaphore; // Ttf typedef std::set wait_http_res_queue_t; wait_http_res_queue_t mHttpWaitResource; // Mfnq -- cgit v1.3 From 0c20beda6800149ee71a307ca4e943b5bba56908 Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Thu, 4 Sep 2014 16:57:44 -0400 Subject: Pipelining work. Extend transfer timeout by the pipeline depth as transfers can appear delayed with deep pipelining and more requests in the pool. Added bad HTTP status error (typically getting a 0 back as HTTP status from libcurl) to the list of retryable errors. There's a response stream problem with libcurl and pipelining that induces this problem. Retrying helps but may not be entirely safe. Watch bug 1420 on the libcurl sourceforge bug tracker. Extend options of test/example program to include un-ranged requests. Document the excessive data transfer induced when ranged requests are disabled. This is an abnormal mode for very rare users so we'll just eat that for now. --- indra/llcorehttp/_httplibcurl.cpp | 15 +++++++---- indra/llcorehttp/_httpoprequest.cpp | 34 ++++++++++++++++++------- indra/llcorehttp/examples/http_texture_load.cpp | 21 ++++++++++++--- indra/llcorehttp/httpcommon.cpp | 13 ++++++++-- indra/newview/app_settings/settings.xml | 4 +-- indra/newview/llmeshrepository.cpp | 13 +++++++++- 6 files changed, 78 insertions(+), 22 deletions(-) (limited to 'indra/llcorehttp') diff --git a/indra/llcorehttp/_httplibcurl.cpp b/indra/llcorehttp/_httplibcurl.cpp index b46833a1f3..cfbe0fd2bb 100755 --- a/indra/llcorehttp/_httplibcurl.cpp +++ b/indra/llcorehttp/_httplibcurl.cpp @@ -217,8 +217,17 @@ void HttpLibcurl::addOp(HttpOpRequest * op) } // Make the request live - curl_multi_add_handle(mMultiHandles[op->mReqPolicy], op->mCurlHandle); + CURLMcode code; + code = curl_multi_add_handle(mMultiHandles[op->mReqPolicy], op->mCurlHandle); + if (CURLM_OK != code) + { + // *TODO: Better cleanup and recovery but not much we can do here. + check_curl_multi_code(code); + return; + } op->mCurlActive = true; + mActiveOps.insert(op); + ++mActiveHandles[op->mReqPolicy]; if (op->mTracing > HTTP_TRACE_OFF) { @@ -230,10 +239,6 @@ void HttpLibcurl::addOp(HttpOpRequest * op) << ", Readies: " << policy.getReadyCount(op->mReqPolicy) << LL_ENDL; } - - // On success, make operation active - mActiveOps.insert(op); - ++mActiveHandles[op->mReqPolicy]; } diff --git a/indra/llcorehttp/_httpoprequest.cpp b/indra/llcorehttp/_httpoprequest.cpp index eb664fdced..38c1f1e78a 100755 --- a/indra/llcorehttp/_httpoprequest.cpp +++ b/indra/llcorehttp/_httpoprequest.cpp @@ -378,6 +378,7 @@ void HttpOpRequest::setupCommon(HttpRequest::policy_t policy_id, // Junk may be left around from a failed request and that // needs to be cleaned out. // +// *TODO: Move this to _httplibcurl where it belongs. HttpStatus HttpOpRequest::prepareRequest(HttpService * service) { CURLcode code; @@ -411,8 +412,9 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service) // *FIXME: better error handling later HttpStatus status; - // Get global policy options - HttpPolicyGlobal & policy(service->getPolicy().getGlobalOptions()); + // Get global and class policy options + HttpPolicyGlobal & gpolicy(service->getPolicy().getGlobalOptions()); + HttpPolicyClass & cpolicy(service->getPolicy().getClassOptions(mReqPolicy)); mCurlHandle = LLCurl::createStandardCurlHandle(); if (! mCurlHandle) @@ -462,30 +464,30 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service) code = curl_easy_setopt(mCurlHandle, CURLOPT_SSL_VERIFYHOST, 0); check_curl_easy_code(code, CURLOPT_SSL_VERIFYHOST); - if (policy.mUseLLProxy) + if (gpolicy.mUseLLProxy) { // Use the viewer-based thread-safe API which has a // fast/safe check for proxy enable. Would like to // encapsulate this someway... LLProxy::getInstance()->applyProxySettings(mCurlHandle); } - else if (policy.mHttpProxy.size()) + else if (gpolicy.mHttpProxy.size()) { // *TODO: This is fine for now but get fuller socks5/ // authentication thing going later.... - code = curl_easy_setopt(mCurlHandle, CURLOPT_PROXY, policy.mHttpProxy.c_str()); + code = curl_easy_setopt(mCurlHandle, CURLOPT_PROXY, gpolicy.mHttpProxy.c_str()); check_curl_easy_code(code, CURLOPT_PROXY); code = curl_easy_setopt(mCurlHandle, CURLOPT_PROXYTYPE, CURLPROXY_HTTP); check_curl_easy_code(code, CURLOPT_PROXYTYPE); } - if (policy.mCAPath.size()) + if (gpolicy.mCAPath.size()) { - code = curl_easy_setopt(mCurlHandle, CURLOPT_CAPATH, policy.mCAPath.c_str()); + code = curl_easy_setopt(mCurlHandle, CURLOPT_CAPATH, gpolicy.mCAPath.c_str()); check_curl_easy_code(code, CURLOPT_CAPATH); } - if (policy.mCAFile.size()) + if (gpolicy.mCAFile.size()) { - code = curl_easy_setopt(mCurlHandle, CURLOPT_CAINFO, policy.mCAFile.c_str()); + code = curl_easy_setopt(mCurlHandle, CURLOPT_CAINFO, gpolicy.mCAFile.c_str()); check_curl_easy_code(code, CURLOPT_CAINFO); } @@ -594,6 +596,20 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service) { xfer_timeout = timeout; } + if (cpolicy.mPipelining > 1L) + { + // Pipelining affects both connection and transfer timeout values. + // Requests that are added to a pipeling immediately have completed + // their connection so the connection delay tends to be less than + // the non-pipelined value. Transfers are the opposite. Transfer + // timeout starts once the connection is established and completion + // can be delayed due to the pipelined requests ahead. So, it's + // a handwave but bump the transfer timeout up by the pipelining + // depth to give some room. + // + // *TODO: Find a better scheme than timeouts to guarantee liveness. + xfer_timeout *= cpolicy.mPipelining; + } code = curl_easy_setopt(mCurlHandle, CURLOPT_TIMEOUT, xfer_timeout); check_curl_easy_code(code, CURLOPT_TIMEOUT); code = curl_easy_setopt(mCurlHandle, CURLOPT_CONNECTTIMEOUT, timeout); diff --git a/indra/llcorehttp/examples/http_texture_load.cpp b/indra/llcorehttp/examples/http_texture_load.cpp index 88692c3f69..b76c874557 100755 --- a/indra/llcorehttp/examples/http_texture_load.cpp +++ b/indra/llcorehttp/examples/http_texture_load.cpp @@ -102,6 +102,7 @@ public: public: bool mVerbose; bool mRandomRange; + bool mNoRange; int mRequestLowWater; int mRequestHighWater; handle_set_t mHandles; @@ -162,10 +163,11 @@ int main(int argc, char** argv) { LLCore::HttpStatus status; bool do_random(false); + bool do_whole(false); bool do_verbose(false); int option(-1); - while (-1 != (option = getopt(argc, argv, "u:c:h?RvH:p:t:"))) + while (-1 != (option = getopt(argc, argv, "u:c:h?RwvH:p:t:"))) { switch (option) { @@ -236,6 +238,12 @@ int main(int argc, char** argv) case 'R': do_random = true; + do_whole = false; + break; + + case 'w': + do_whole = true; + do_random = false; break; case 'v': @@ -307,6 +315,7 @@ int main(int argc, char** argv) ws.mUrl = url_format; ws.loadAssetUuids(uuids); ws.mRandomRange = do_random; + ws.mNoRange = do_whole; ws.mVerbose = do_verbose; ws.mRequestHighWater = highwater; ws.mRequestLowWater = ws.mRequestHighWater / 2; @@ -381,6 +390,7 @@ void usage(std::ostream & out) " -u printf-style format string for URL generation\n" " Default: " << url_format << "\n" " -R Issue GETs with random Range: headers\n" + " -w Issue GETs without Range: headers to get whole object\n" " -c Maximum connection concurrency. Range: [1..100]\n" " Default: " << concurrency_limit << "\n" " -H HTTP request highwater (requests fed to llcorehttp).\n" @@ -400,6 +410,7 @@ WorkingSet::WorkingSet() : LLCore::HttpHandler(), mVerbose(false), mRandomRange(false), + mNoRange(false), mRemaining(200), mLimit(200), mAt(0), @@ -449,8 +460,12 @@ bool WorkingSet::reload(LLCore::HttpRequest * hr, LLCore::HttpOptions * opt) #else snprintf(buffer, sizeof(buffer), mUrl.c_str(), mAssets[mAt].mUuid.c_str()); #endif - int offset(mRandomRange ? ((unsigned long) rand()) % 1000000UL : mAssets[mAt].mOffset); - int length(mRandomRange ? ((unsigned long) rand()) % 1000000UL : mAssets[mAt].mLength); + int offset(mNoRange + ? 0 + : (mRandomRange ? ((unsigned long) rand()) % 1000000UL : mAssets[mAt].mOffset)); + int length(mNoRange + ? 0 + : (mRandomRange ? ((unsigned long) rand()) % 1000000UL : mAssets[mAt].mLength)); LLCore::HttpHandle handle; if (offset || length) diff --git a/indra/llcorehttp/httpcommon.cpp b/indra/llcorehttp/httpcommon.cpp index c2f15155ac..9bcf7ac5e3 100755 --- a/indra/llcorehttp/httpcommon.cpp +++ b/indra/llcorehttp/httpcommon.cpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012-2013, Linden Research, Inc. + * Copyright (C) 2012-2014, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -219,6 +219,13 @@ std::string HttpStatus::toTerseString() const // Pass true on statuses that might actually be cleared by a // retry. Library failures, calling problems, etc. aren't // going to be fixed by squirting bits all over the Net. +// +// HE_INVALID_HTTP_STATUS is special. As of 7.37.0, there are +// some scenarios where response processing in libcurl appear +// to go wrong and response data is corrupted. A side-effect +// of this is that the HTTP status is read as 0 from the library. +// See libcurl bug report 1420 (https://sourceforge.net/p/curl/bugs/1420/) +// for details. bool HttpStatus::isRetryable() const { static const HttpStatus cant_connect(HttpStatus::EXT_CURL_EASY, CURLE_COULDNT_CONNECT); @@ -231,6 +238,7 @@ bool HttpStatus::isRetryable() const static const HttpStatus post_error(HttpStatus::EXT_CURL_EASY, CURLE_HTTP_POST_ERROR); static const HttpStatus partial_file(HttpStatus::EXT_CURL_EASY, CURLE_PARTIAL_FILE); static const HttpStatus inv_cont_range(HttpStatus::LLCORE, HE_INV_CONTENT_RANGE_HDR); + static const HttpStatus inv_status(HttpStatus::LLCORE, HE_INVALID_HTTP_STATUS); return ((isHttpStatus() && mType >= 499 && mType <= 599) || // Include special 499 in retryables *this == cant_connect || // Connection reset/endpoint problems @@ -242,7 +250,8 @@ bool HttpStatus::isRetryable() const *this == op_timedout || // Timer expired *this == post_error || // Transport problem *this == partial_file || // Data inconsistency in response - *this == inv_cont_range); // Short data read disagrees with content-range + *this == inv_cont_range || // Short data read disagrees with content-range + *this == inv_status); // Inv status can reflect internal state problem in libcurl } } // end namespace LLCore diff --git a/indra/newview/app_settings/settings.xml b/indra/newview/app_settings/settings.xml index 0607579a08..ab15a03229 100755 --- a/indra/newview/app_settings/settings.xml +++ b/indra/newview/app_settings/settings.xml @@ -4459,7 +4459,7 @@ HttpPipelining Comment - If true, viewer will pipeline HTTP requests to servers. + If true, viewer will attempt to pipeline HTTP requests. Persist 1 Type @@ -4470,7 +4470,7 @@ HttpRangeRequestsDisable Comment - If true, viewer will not issued range GET requests for meshes and textures. May resolve problems with certain ISPs and networking gear. + If true, viewer will not issue GET requests with 'Range:' headers for meshes and textures. May resolve problems with certain ISPs and networking gear. Persist 1 Type diff --git a/indra/newview/llmeshrepository.cpp b/indra/newview/llmeshrepository.cpp index 6477389d4c..a6707392fe 100755 --- a/indra/newview/llmeshrepository.cpp +++ b/indra/newview/llmeshrepository.cpp @@ -2644,6 +2644,17 @@ void LLMeshRepository::cacheOutgoingMesh(LLMeshUploadData& data, LLSD& header) } +// Handle failed or successful requests for mesh assets. +// +// Support for 200 responses was added for several reasons. One, +// a service or cache can ignore range headers and give us a +// 200 with full asset should it elect to. We also support +// a debug flag which disables range requests for those very +// few users that have some sort of problem with their networking +// services. But the 200 response handling is suboptimal: rather +// than cache the whole asset, we just extract the part that would +// have been sent in a 206 and process that. Inefficient but these +// are cases far off the norm. void LLMeshHandlerBase::onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * response) { mProcessed = true; @@ -2685,7 +2696,7 @@ void LLMeshHandlerBase::onCompleted(LLCore::HttpHandle handle, LLCore::HttpRespo if (par_status == status) { - // 216 case + // 206 case response->getRange(&offset, &length, &full_length); if (! offset && ! length) { -- cgit v1.3 From 3057d246f0c63be1b3015313b9ad9824a66f4e0f Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Thu, 18 Sep 2014 18:42:30 -0400 Subject: Documentation. Describe curl bug 1420 testing and how to reproduce data corruption via timeouts. --- indra/llcorehttp/_httpoprequest.cpp | 15 +++++++++++++++ indra/llcorehttp/httpcommon.cpp | 4 ++++ 2 files changed, 19 insertions(+) (limited to 'indra/llcorehttp') diff --git a/indra/llcorehttp/_httpoprequest.cpp b/indra/llcorehttp/_httpoprequest.cpp index 38c1f1e78a..4453bf2922 100755 --- a/indra/llcorehttp/_httpoprequest.cpp +++ b/indra/llcorehttp/_httpoprequest.cpp @@ -47,6 +47,19 @@ #include "llhttpconstants.h" #include "llproxy.h" +// *DEBUG: "[curl:bugs] #1420" problem and testing. +// +// A pipelining problem, https://sourceforge.net/p/curl/bugs/1420/, +// was a source of Core_9 failures. Code related to this can be +// identified and tested by: +// * Looking for '[curl:bugs]' strings in source and following +// instructions there. +// * Set 'QAModeHttpTrace' to 2 or 3 in settings.xml and look for +// 'timed out' events in the log. +// * Enable the HttpRangeRequestsDisable debug setting which causes +// full asset fetches. These slow the pipelines down a bit. +// + namespace { @@ -610,6 +623,8 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service) // *TODO: Find a better scheme than timeouts to guarantee liveness. xfer_timeout *= cpolicy.mPipelining; } + // *DEBUG: Useful for timeout handling and "[curl:bugs] #1420" tests + // xfer_timeout = 3L; code = curl_easy_setopt(mCurlHandle, CURLOPT_TIMEOUT, xfer_timeout); check_curl_easy_code(code, CURLOPT_TIMEOUT); code = curl_easy_setopt(mCurlHandle, CURLOPT_CONNECTTIMEOUT, timeout); diff --git a/indra/llcorehttp/httpcommon.cpp b/indra/llcorehttp/httpcommon.cpp index 9bcf7ac5e3..8714915fa2 100755 --- a/indra/llcorehttp/httpcommon.cpp +++ b/indra/llcorehttp/httpcommon.cpp @@ -240,6 +240,10 @@ bool HttpStatus::isRetryable() const static const HttpStatus inv_cont_range(HttpStatus::LLCORE, HE_INV_CONTENT_RANGE_HDR); static const HttpStatus inv_status(HttpStatus::LLCORE, HE_INVALID_HTTP_STATUS); + // *DEBUG: For "[curl:bugs] #1420" tests. + // Disable the '*this == inv_status' test and look for 'Core_9' + // failures in log files. + return ((isHttpStatus() && mType >= 499 && mType <= 599) || // Include special 499 in retryables *this == cant_connect || // Connection reset/endpoint problems *this == cant_res_proxy || // DNS problems -- cgit v1.3 From 79ab7c20703c092a4416a4f9a885e0246fc17ee0 Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Fri, 19 Sep 2014 15:34:09 -0400 Subject: Introduce libcurl handle cache. Create a private cache of used handles and a fast handle factory that's thread- correct. --- indra/llcorehttp/_httplibcurl.cpp | 83 +++++++++++++++++++++++++++++++++++-- indra/llcorehttp/_httplibcurl.h | 78 ++++++++++++++++++++++++++++++++-- indra/llcorehttp/_httpoprequest.cpp | 5 ++- 3 files changed, 158 insertions(+), 8 deletions(-) (limited to 'indra/llcorehttp') diff --git a/indra/llcorehttp/_httplibcurl.cpp b/indra/llcorehttp/_httplibcurl.cpp index cfbe0fd2bb..81b44ab90b 100755 --- a/indra/llcorehttp/_httplibcurl.cpp +++ b/indra/llcorehttp/_httplibcurl.cpp @@ -51,6 +51,7 @@ namespace LLCore HttpLibcurl::HttpLibcurl(HttpService * service) : mService(service), + mHandleCache(), mPolicyCount(0), mMultiHandles(NULL), mActiveHandles(NULL), @@ -61,7 +62,7 @@ HttpLibcurl::HttpLibcurl(HttpService * service) HttpLibcurl::~HttpLibcurl() { shutdown(); - + mService = NULL; } @@ -279,7 +280,7 @@ void HttpLibcurl::cancelRequest(HttpOpRequest * op) // Detach from multi and recycle handle curl_multi_remove_handle(mMultiHandles[op->mReqPolicy], op->mCurlHandle); - curl_easy_cleanup(op->mCurlHandle); + mHandleCache.freeHandle(op->mCurlHandle); op->mCurlHandle = NULL; // Tracing @@ -356,7 +357,7 @@ bool HttpLibcurl::completeRequest(CURLM * multi_handle, CURL * handle, CURLcode // Detach from multi and recycle handle curl_multi_remove_handle(multi_handle, handle); - curl_easy_cleanup(handle); + mHandleCache.freeHandle(op->mCurlHandle); op->mCurlHandle = NULL; // Tracing @@ -471,6 +472,82 @@ void HttpLibcurl::policyUpdated(int policy_class) } } +// --------------------------------------- +// HttpLibcurl::HandleCache +// --------------------------------------- + +HttpLibcurl::HandleCache::HandleCache() + : mHandleTemplate(NULL) +{ + mCache.reserve(50); +} + + +HttpLibcurl::HandleCache::~HandleCache() +{ + if (mHandleTemplate) + { + curl_easy_cleanup(mHandleTemplate); + mHandleTemplate = NULL; + } + + for (handle_cache_t::iterator it(mCache.begin()); mCache.end() != it; ++it) + { + curl_easy_cleanup(*it); + } + mCache.clear(); +} + + +CURL * HttpLibcurl::HandleCache::getHandle() +{ + CURL * ret(NULL); + + if (! mCache.empty()) + { + // Fastest path to handle + ret = mCache.back(); + mCache.pop_back(); + } + else if (mHandleTemplate) + { + // Still fast path + ret = curl_easy_duphandle(mHandleTemplate); + } + else + { + // When all else fails + ret = curl_easy_init(); + } + + return ret; +} + + +void HttpLibcurl::HandleCache::freeHandle(CURL * handle) +{ + if (! handle) + { + return; + } + + curl_easy_reset(handle); + if (! mHandleTemplate) + { + // Save the first freed handle as a template. + mHandleTemplate = handle; + } + else + { + // Otherwise add it to the cache + if (mCache.size() >= mCache.capacity()) + { + mCache.reserve(mCache.capacity() + 50); + } + mCache.push_back(handle); + } +} + // --------------------------------------- // Free functions diff --git a/indra/llcorehttp/_httplibcurl.h b/indra/llcorehttp/_httplibcurl.h index 2c7ad1fa8e..ffc24c63a8 100755 --- a/indra/llcorehttp/_httplibcurl.h +++ b/indra/llcorehttp/_httplibcurl.h @@ -124,6 +124,23 @@ public: /// Threading: called by worker thread. void policyUpdated(int policy_class); + /// Allocate a curl handle for caller. May be freed using + /// either the freeHandle() method or calling curl_easy_cleanup() + /// directly. + /// + /// @return Libcurl handle (CURL *) or NULL on allocation + /// problem. Handle will be in curl_easy_reset() + /// condition. + /// + /// Threading: callable by worker thread. + /// + /// Deprecation: Expect this to go away after _httpoprequest is + /// refactored bringing code into this class. + CURL * getHandle() + { + return mHandleCache.getHandle(); + } + protected: /// Invoked when libcurl has indicated a request has been processed /// to completion and we need to move the request to a new state. @@ -135,14 +152,67 @@ protected: protected: typedef std::set active_set_t; + + /// Simple request handle cache for libcurl. + /// + /// Handle creation is somewhat slow and chunky in libcurl and there's + /// a pretty good speedup to be had from handle re-use. So, a simple + /// vector is kept of 'freed' handles to be reused as needed. When + /// that is empty, the first freed handle is kept as a template for + /// handle duplication. This is still faster than creation from nothing. + /// And when that fails, we init fresh from curl_easy_init(). + /// + /// Handles allocated with getHandle() may be freed with either + /// freeHandle() or curl_easy_cleanup(). Choice may be dictated + /// by thread constraints. + /// + /// Threading: Single-threaded. May only be used by a single thread, + /// typically the worker thread. If freeing requests' handles in an + /// unknown threading context, use curl_easy_cleanup() for safety. + + class HandleCache + { + public: + HandleCache(); + ~HandleCache(); + + private: + HandleCache(const HandleCache &); // Not defined + void operator=(const HandleCache &); // Not defined + + public: + /// Allocate a curl handle for caller. May be freed using + /// either the freeHandle() method or calling curl_easy_cleanup() + /// directly. + /// + /// @return Libcurl handle (CURL *) or NULL on allocation + /// problem. + /// + /// Threading: Single-thread (worker) only. + CURL * getHandle(); + + /// Free a libcurl handle acquired by whatever means. Thread + /// safety is left to the caller. + /// + /// Threading: Single-thread (worker) only. + void freeHandle(CURL * handle); + + protected: + typedef std::vector handle_cache_t; + + protected: + CURL * mHandleTemplate; // Template for duplicating new handles + handle_cache_t mCache; // Cache of old handles + }; // end class HandleCache protected: - HttpService * mService; // Simple reference, not owner + HttpService * mService; // Simple reference, not owner + HandleCache mHandleCache; // Handle allocator, owner active_set_t mActiveOps; int mPolicyCount; - CURLM ** mMultiHandles; // One handle per policy class - int * mActiveHandles; // Active count per policy class - bool * mDirtyPolicy; // Dirty policy update waiting for stall (per pc) + CURLM ** mMultiHandles; // One handle per policy class + int * mActiveHandles; // Active count per policy class + bool * mDirtyPolicy; // Dirty policy update waiting for stall (per pc) }; // end class HttpLibcurl diff --git a/indra/llcorehttp/_httpoprequest.cpp b/indra/llcorehttp/_httpoprequest.cpp index 4453bf2922..bbda0b82fd 100755 --- a/indra/llcorehttp/_httpoprequest.cpp +++ b/indra/llcorehttp/_httpoprequest.cpp @@ -170,6 +170,8 @@ HttpOpRequest::~HttpOpRequest() if (mCurlHandle) { + // Uncertain of thread context so free using + // safest method. curl_easy_cleanup(mCurlHandle); mCurlHandle = NULL; } @@ -429,7 +431,7 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service) HttpPolicyGlobal & gpolicy(service->getPolicy().getGlobalOptions()); HttpPolicyClass & cpolicy(service->getPolicy().getClassOptions(mReqPolicy)); - mCurlHandle = LLCurl::createStandardCurlHandle(); + mCurlHandle = service->getTransport().getHandle(); if (! mCurlHandle) { // We're in trouble. We'll continue but it won't go well. @@ -437,6 +439,7 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service) << LL_ENDL; return HttpStatus(HttpStatus::LLCORE, HE_BAD_ALLOC); } + code = curl_easy_setopt(mCurlHandle, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4); check_curl_easy_code(code, CURLOPT_IPRESOLVE); code = curl_easy_setopt(mCurlHandle, CURLOPT_NOSIGNAL, 1); -- cgit v1.3 From 329608d24668b044e16b54ff7a7d0ac592b2b88d Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Mon, 22 Sep 2014 18:49:45 -0400 Subject: Tuning and documentation. Use a fast poll frequency (0.05S) on the HTTP requests for inventory. We'll benchmark with that and see how it goes. Document some of the history of the background fetcher for future devs. Suggest some future projects to make things faster. Pointers on using LLSD with the llcorehttp library in the readme. And restructured the LLSD onCompleted() processing phases using do{}while(false) which produced a code flow that is fairly attractive. --- indra/llcorehttp/README.Linden | 8 ++ indra/newview/llinventorymodel.cpp | 92 +++++++------- indra/newview/llinventorymodelbackgroundfetch.cpp | 140 +++++++++++++++------- 3 files changed, 147 insertions(+), 93 deletions(-) (limited to 'indra/llcorehttp') diff --git a/indra/llcorehttp/README.Linden b/indra/llcorehttp/README.Linden index eb6ccab3bc..c3aaa9158d 100644 --- a/indra/llcorehttp/README.Linden +++ b/indra/llcorehttp/README.Linden @@ -529,6 +529,14 @@ HttpOperation::addAsReply: TRACE, ToReplyQueue, Handle: 086D3148 data = NULL; + There are now helper functions in llmessage/llcorehttputil.h to + assist with LLSD usage. requestPostWithLLSD(...) provides a + requestPost()-like interface that takes an LLSD object rather than + a BufferArray. And responseToLLSD(...) attempts to convert a + BufferArray received from a server into an LLSD object. You can + find examples in llmeshrepository.cpp, llinventorymodel.cpp, + llinventorymodelbackgroundfetch.cpp and lltexturefetch.cpp. + LLSD will often go hand-in-hand with BufferArray and data transport. But you can also do all the streaming I/O you'd expect of a std::iostream object: diff --git a/indra/newview/llinventorymodel.cpp b/indra/newview/llinventorymodel.cpp index ee28cef640..dab3a4c06d 100755 --- a/indra/newview/llinventorymodel.cpp +++ b/indra/newview/llinventorymodel.cpp @@ -3974,65 +3974,63 @@ LLInventoryModel::FetchItemHttpHandler::~FetchItemHttpHandler() void LLInventoryModel::FetchItemHttpHandler::onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * response) { - // Single-pass do-while used for common exit handling - do + do // Single-pass do-while used for common exit handling { LLCore::HttpStatus status(response->getStatus()); // status = LLCore::HttpStatus(404); // Dev tool to force error handling if (! status) { processFailure(status, response); + break; // Goto common exit } - else - { - LLCore::BufferArray * body(response->getBody()); - // body = NULL; // Dev tool to force error handling - if (! body || ! body->size()) - { - LL_WARNS(LOG_INV) << "Missing data in inventory item query." << LL_ENDL; - processFailure("HTTP response for inventory item query missing body", response); - break; // Goto common exit - } - // body->write(0, "Garbage Response", 16); // Dev tool to force error handling - LLSD body_llsd; - if (! LLCoreHttpUtil::responseToLLSD(response, true, body_llsd)) - { - // INFOS-level logging will occur on the parsed failure - processFailure("HTTP response for inventory item query has malformed LLSD", response); - break; // Goto common exit - } + LLCore::BufferArray * body(response->getBody()); + // body = NULL; // Dev tool to force error handling + if (! body || ! body->size()) + { + LL_WARNS(LOG_INV) << "Missing data in inventory item query." << LL_ENDL; + processFailure("HTTP response for inventory item query missing body", response); + break; // Goto common exit + } - // Expect top-level structure to be a map - // body_llsd = LLSD::emptyArray(); // Dev tool to force error handling - if (! body_llsd.isMap()) - { - processFailure("LLSD response for inventory item not a map", response); - break; // Goto common exit - } + // body->write(0, "Garbage Response", 16); // Dev tool to force error handling + LLSD body_llsd; + if (! LLCoreHttpUtil::responseToLLSD(response, true, body_llsd)) + { + // INFOS-level logging will occur on the parsed failure + processFailure("HTTP response for inventory item query has malformed LLSD", response); + break; // Goto common exit + } - // Check for 200-with-error failures - // - // Original Responder-based serivce model didn't check for these errors. - // It may be more robust to ignore this condition. With aggregated requests, - // an error in one inventory item might take down the entire request. - // So if this instead broke up the aggregated items into single requests, - // maybe that would make progress. Or perhaps there's structured information - // that can tell us what went wrong. Need to dig into this and firm up - // the API. - // - // body_llsd["error"] = LLSD::emptyMap(); // Dev tool to force error handling - // body_llsd["error"]["identifier"] = "Development"; - // body_llsd["error"]["message"] = "You left development code in the viewer"; - if (body_llsd.has("error")) - { - processFailure("Inventory application error (200-with-error)", response); - break; // Goto common exit - } + // Expect top-level structure to be a map + // body_llsd = LLSD::emptyArray(); // Dev tool to force error handling + if (! body_llsd.isMap()) + { + processFailure("LLSD response for inventory item not a map", response); + break; // Goto common exit + } - // Okay, process data if possible - processData(body_llsd, response); + // Check for 200-with-error failures + // + // Original Responder-based serivce model didn't check for these errors. + // It may be more robust to ignore this condition. With aggregated requests, + // an error in one inventory item might take down the entire request. + // So if this instead broke up the aggregated items into single requests, + // maybe that would make progress. Or perhaps there's structured information + // that can tell us what went wrong. Need to dig into this and firm up + // the API. + // + // body_llsd["error"] = LLSD::emptyMap(); // Dev tool to force error handling + // body_llsd["error"]["identifier"] = "Development"; + // body_llsd["error"]["message"] = "You left development code in the viewer"; + if (body_llsd.has("error")) + { + processFailure("Inventory application error (200-with-error)", response); + break; // Goto common exit } + + // Okay, process data if possible + processData(body_llsd, response); } while (false); diff --git a/indra/newview/llinventorymodelbackgroundfetch.cpp b/indra/newview/llinventorymodelbackgroundfetch.cpp index de1d123fe5..f18832fe95 100755 --- a/indra/newview/llinventorymodelbackgroundfetch.cpp +++ b/indra/newview/llinventorymodelbackgroundfetch.cpp @@ -42,6 +42,56 @@ #include "bufferstream.h" #include "llcorehttputil.h" +// History (may be apocryphal) +// +// Around V2, an HTTP inventory download mechanism was added +// along with inventory LINK items referencing other inventory +// items. As part of this, at login, the entire inventory +// structure is downloaded 'in the background' using the +// backgroundFetch()/bulkFetch() methods. The UDP path can +// still be used and is found in the 'DEPRECATED OLD CODE' +// section. +// +// The old UDP path implemented a throttle that adapted +// itself during running. The mechanism survived info HTTP +// somewhat but was pinned to poll the HTTP plumbing at +// 0.5S intervals. The reasons for this particular value +// have been lost. It's possible to switch between UDP +// and HTTP while this is happening but there may be +// surprises in what happens in that case. +// +// Conversion to llcorehttp reduced the number of connections +// used but batches more data and queues more requests (but +// doesn't due pipelining due to libcurl restrictions). The +// poll interval above was re-examined and reduced to get +// inventory into the viewer more quickly. +// +// Possible future work: +// +// * Don't download the entire heirarchy in one go (which +// might have been how V1 worked). Implications for +// links (which may not have a valid target) and search +// which would then be missing data. +// +// * Review the download rate throttling. Slow then fast? +// Detect bandwidth usage and speed up when it drops? +// +// * A lot of calls to notifyObservers(). It looks like +// these could be collapsed by maintaining a 'dirty' +// bit and there appears to be an attempt to do this. +// But it isn't used or is used in a limited fashion. +// Are there semanic issues requiring a call after certain +// updateItem() calls? +// +// * An error on a fetch could be due to one item in the batch. +// If the batch were broken up, perhaps more of the inventory +// would download. (Handwave here, not certain this is an +// issue in practice.) +// +// * Conversion to AISv3. +// + + namespace { @@ -488,11 +538,11 @@ void LLInventoryModelBackgroundFetch::bulkFetch() // a fast/slow fetch throttle. Once login is complete and the scene // is mostly loaded, we could turn up the throttle and fill missing // inventory more quickly. + static const U32 max_batch_size(10); static const S32 max_concurrent_fetches(12); // Outstanding requests, not connections static const F32 new_min_time(0.05f); // *HACK: Clean this up when old code goes away entirely. - static const U32 max_batch_size(10); - mMinTimeBetweenFetches = 0.01f; + mMinTimeBetweenFetches = new_min_time; if (mMinTimeBetweenFetches < new_min_time) { mMinTimeBetweenFetches = new_min_time; // *HACK: See above. @@ -702,63 +752,61 @@ namespace void BGFolderHttpHandler::onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * response) { - // Single-pass do-while used for common exit handling - do + do // Single-pass do-while used for common exit handling { LLCore::HttpStatus status(response->getStatus()); // status = LLCore::HttpStatus(404); // Dev tool to force error handling if (! status) { processFailure(status, response); + break; // Goto common exit } - else - { - // Response body should be present. - LLCore::BufferArray * body(response->getBody()); - // body = NULL; // Dev tool to force error handling - if (! body || ! body->size()) - { - LL_WARNS(LOG_INV) << "Missing data in inventory folder query." << LL_ENDL; - processFailure("HTTP response missing expected body", response); - break; // Goto common exit - } - // Could test 'Content-Type' header but probably unreliable. + // Response body should be present. + LLCore::BufferArray * body(response->getBody()); + // body = NULL; // Dev tool to force error handling + if (! body || ! body->size()) + { + LL_WARNS(LOG_INV) << "Missing data in inventory folder query." << LL_ENDL; + processFailure("HTTP response missing expected body", response); + break; // Goto common exit + } - // Convert response to LLSD - // body->write(0, "Garbage Response", 16); // Dev tool to force error handling - LLSD body_llsd; - if (! LLCoreHttpUtil::responseToLLSD(response, true, body_llsd)) - { - // INFOS-level logging will occur on the parsed failure - processFailure("HTTP response contained malformed LLSD", response); - break; // goto common exit - } + // Could test 'Content-Type' header but probably unreliable. - // Expect top-level structure to be a map - // body_llsd = LLSD::emptyArray(); // Dev tool to force error handling - if (! body_llsd.isMap()) - { - processFailure("LLSD response not a map", response); - break; // goto common exit - } + // Convert response to LLSD + // body->write(0, "Garbage Response", 16); // Dev tool to force error handling + LLSD body_llsd; + if (! LLCoreHttpUtil::responseToLLSD(response, true, body_llsd)) + { + // INFOS-level logging will occur on the parsed failure + processFailure("HTTP response contained malformed LLSD", response); + break; // goto common exit + } - // Check for 200-with-error failures - // - // See comments in llinventorymodel.cpp about this mode of error. - // - // body_llsd["error"] = LLSD::emptyMap(); // Dev tool to force error handling - // body_llsd["error"]["identifier"] = "Development"; - // body_llsd["error"]["message"] = "You left development code in the viewer"; - if (body_llsd.has("error")) - { - processFailure("Inventory application error (200-with-error)", response); - break; // goto common exit - } + // Expect top-level structure to be a map + // body_llsd = LLSD::emptyArray(); // Dev tool to force error handling + if (! body_llsd.isMap()) + { + processFailure("LLSD response not a map", response); + break; // goto common exit + } - // Okay, process data if possible - processData(body_llsd, response); + // Check for 200-with-error failures + // + // See comments in llinventorymodel.cpp about this mode of error. + // + // body_llsd["error"] = LLSD::emptyMap(); // Dev tool to force error handling + // body_llsd["error"]["identifier"] = "Development"; + // body_llsd["error"]["message"] = "You left development code in the viewer"; + if (body_llsd.has("error")) + { + processFailure("Inventory application error (200-with-error)", response); + break; // goto common exit } + + // Okay, process data if possible + processData(body_llsd, response); } while (false); -- cgit v1.3 From ec4fd2f0e226bb2cae5982760317e1d6ea2d2d69 Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Fri, 10 Oct 2014 16:43:04 -0400 Subject: MAINT-4564 HTTP Pipelining is not happening in Drano HTTP Phase 4 Incorporate the new libcurl 7.38.0 build with curl bug 1420 workaround. Add developer-centric testing code to evaluate the workaround or a future fix for 1420. --- autobuild.xml | 12 ++++++------ indra/llcorehttp/_httpoprequest.cpp | 4 ++-- indra/llcorehttp/_httppolicy.cpp | 12 ++++++++++++ indra/llcorehttp/httpcommon.cpp | 5 +++-- 4 files changed, 23 insertions(+), 10 deletions(-) (limited to 'indra/llcorehttp') diff --git a/autobuild.xml b/autobuild.xml index b9db04c28b..1045e11b6d 100755 --- a/autobuild.xml +++ b/autobuild.xml @@ -282,9 +282,9 @@ archive hash - ed283b163e8f74d2c9d6ea5874fcca54 + 40b1c6b3727ebedafc2f1a172797ccd1 url - http://automated-builds-secondlife-com.s3.amazonaws.com/hg/repo/3pl_3p-curl-update/rev/294562/arch/Darwin/installer/curl-7.38.0-darwin-20140923.tar.bz2 + http://automated-builds-secondlife-com.s3.amazonaws.com/hg/repo/3pl_3p-curl-update/rev/295367/arch/Darwin/installer/curl-7.38.0-darwin-20141010.tar.bz2 name darwin @@ -294,9 +294,9 @@ archive hash - 6a0a62b6c026fa0b33c0978f4afd152e + 06149da3d7a34adf40853f813ae55328 url - http://automated-builds-secondlife-com.s3.amazonaws.com/hg/repo/3pl_3p-curl-update/rev/294562/arch/Linux/installer/curl-7.38.0-linux-20140923.tar.bz2 + http://automated-builds-secondlife-com.s3.amazonaws.com/hg/repo/3pl_3p-curl-update/rev/295367/arch/Linux/installer/curl-7.38.0-linux-20141010.tar.bz2 name linux @@ -306,9 +306,9 @@ archive hash - 56ff4698d5b39a37994f4cc8acba19f0 + e4280eae792a5f13bc9d01d8cfb7c557 url - http://automated-builds-secondlife-com.s3.amazonaws.com/hg/repo/3pl_3p-curl-update/rev/294562/arch/CYGWIN/installer/curl-7.38.0-windows-20140923.tar.bz2 + http://automated-builds-secondlife-com.s3.amazonaws.com/hg/repo/3pl_3p-curl-update/rev/295367/arch/CYGWIN/installer/curl-7.38.0-windows-20141010.tar.bz2 name windows diff --git a/indra/llcorehttp/_httpoprequest.cpp b/indra/llcorehttp/_httpoprequest.cpp index bbda0b82fd..fbbb1614fb 100755 --- a/indra/llcorehttp/_httpoprequest.cpp +++ b/indra/llcorehttp/_httpoprequest.cpp @@ -626,8 +626,8 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service) // *TODO: Find a better scheme than timeouts to guarantee liveness. xfer_timeout *= cpolicy.mPipelining; } - // *DEBUG: Useful for timeout handling and "[curl:bugs] #1420" tests - // xfer_timeout = 3L; + // *DEBUG: Enable following override for timeout handling and "[curl:bugs] #1420" tests + // xfer_timeout = 1L; code = curl_easy_setopt(mCurlHandle, CURLOPT_TIMEOUT, xfer_timeout); check_curl_easy_code(code, CURLOPT_TIMEOUT); code = curl_easy_setopt(mCurlHandle, CURLOPT_CONNECTTIMEOUT, timeout); diff --git a/indra/llcorehttp/_httppolicy.cpp b/indra/llcorehttp/_httppolicy.cpp index 09b9206f63..e5d6321401 100755 --- a/indra/llcorehttp/_httppolicy.cpp +++ b/indra/llcorehttp/_httppolicy.cpp @@ -414,6 +414,18 @@ bool HttpPolicy::stageAfterCompletion(HttpOpRequest * op) // Retry or finalize if (! op->mStatus) { + // *DEBUG: For "[curl:bugs] #1420" tests. This will interfere + // with unit tests due to allocation retention by logging code. + // But you won't be checking this in enabled. +#if 0 + if (op->mStatus == HttpStatus(HttpStatus::EXT_CURL_EASY, CURLE_OPERATION_TIMEDOUT)) + { + LL_WARNS(LOG_CORE) << "HTTP request " << static_cast(op) + << " timed out." + << LL_ENDL; + } +#endif + // If this failed, we might want to retry. if (op->mPolicyRetries < op->mPolicyRetryLimit && op->mStatus.isRetryable()) { diff --git a/indra/llcorehttp/httpcommon.cpp b/indra/llcorehttp/httpcommon.cpp index 8714915fa2..7907e958a4 100755 --- a/indra/llcorehttp/httpcommon.cpp +++ b/indra/llcorehttp/httpcommon.cpp @@ -254,8 +254,9 @@ bool HttpStatus::isRetryable() const *this == op_timedout || // Timer expired *this == post_error || // Transport problem *this == partial_file || // Data inconsistency in response - *this == inv_cont_range || // Short data read disagrees with content-range - *this == inv_status); // Inv status can reflect internal state problem in libcurl + // *DEBUG: Comment out 'inv_status' test for [curl:bugs] #1420 testing. + *this == inv_status || // Inv status can reflect internal state problem in libcurl + *this == inv_cont_range); // Short data read disagrees with content-range } } // end namespace LLCore -- cgit v1.3