From 5611cb6d476540e6a1c654c1f9acdce2787b3505 Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Mon, 23 Apr 2012 16:19:39 -0400 Subject: Okay, imported the core-http library and got it compiling suspiciously easily. The unit/integration tests don't work yet as I'm still battling cmake/autobuild as usual but first milestone passed. --- indra/llcorehttp/_httppolicy.cpp | 76 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 indra/llcorehttp/_httppolicy.cpp (limited to 'indra/llcorehttp/_httppolicy.cpp') diff --git a/indra/llcorehttp/_httppolicy.cpp b/indra/llcorehttp/_httppolicy.cpp new file mode 100644 index 0000000000..d965a6cf3a --- /dev/null +++ b/indra/llcorehttp/_httppolicy.cpp @@ -0,0 +1,76 @@ +/** + * @file _httppolicy.cpp + * @brief Internal definitions of the Http policy thread + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include "_httppolicy.h" + +#include "_httpoprequest.h" +#include "_httpservice.h" + + +namespace LLCore +{ + + +HttpPolicy::HttpPolicy(HttpService * service) + : mService(service) +{} + + +HttpPolicy::~HttpPolicy() +{ + for (ready_queue_t::reverse_iterator i(mReadyQueue.rbegin()); + mReadyQueue.rend() != i;) + { + ready_queue_t::reverse_iterator cur(i++); + + (*cur)->cancel(); + (*cur)->release(); + } + + mService = NULL; +} + + +void HttpPolicy::addOp(HttpOpRequest * op) +{ + mReadyQueue.push_back(op); +} + + +void HttpPolicy::processReadyQueue() +{ + while (! mReadyQueue.empty()) + { + HttpOpRequest * op(mReadyQueue.front()); + mReadyQueue.erase(mReadyQueue.begin()); + + op->stageFromReady(mService); + op->release(); + } +} + + +} // end namespace LLCore -- cgit v1.2.3 From b8edacd0bb4feacc3ac1d61421e600c75ab87f7c Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Fri, 1 Jun 2012 14:07:34 -0400 Subject: Major steps towards implementing the policy component. Identified and reacted to the priority inversion problem we have in texturefetch. Includes the introduction of a priority_queue for the requests that are ready. Start some parameterization in anticipation of having policy_class everywhere. Removed _assert.h which isn't really needed in indra codebase. Implemented async setPriority request (which I hope I can get rid of eventually along with all priorities in this library). Converted to using unsigned int for priority rather than float. Implemented POST and did groundwork for PUT. --- indra/llcorehttp/_httppolicy.cpp | 97 +++++++++++++++++++++++++++++++++------- 1 file changed, 81 insertions(+), 16 deletions(-) (limited to 'indra/llcorehttp/_httppolicy.cpp') diff --git a/indra/llcorehttp/_httppolicy.cpp b/indra/llcorehttp/_httppolicy.cpp index d965a6cf3a..873b519c51 100644 --- a/indra/llcorehttp/_httppolicy.cpp +++ b/indra/llcorehttp/_httppolicy.cpp @@ -28,48 +28,113 @@ #include "_httpoprequest.h" #include "_httpservice.h" +#include "_httplibcurl.h" namespace LLCore { - HttpPolicy::HttpPolicy(HttpService * service) : mService(service) -{} +{ + for (int policy_class(0); policy_class < HttpRequest::POLICY_CLASS_LIMIT; ++policy_class) + { + mReadyInClass[policy_class] = 0; + } +} HttpPolicy::~HttpPolicy() { - for (ready_queue_t::reverse_iterator i(mReadyQueue.rbegin()); - mReadyQueue.rend() != i;) + for (int policy_class(0); policy_class < HttpRequest::POLICY_CLASS_LIMIT; ++policy_class) { - ready_queue_t::reverse_iterator cur(i++); - - (*cur)->cancel(); - (*cur)->release(); + HttpReadyQueue & readyq(mReadyQueue[policy_class]); + + while (! readyq.empty()) + { + HttpOpRequest * op(readyq.top()); + + op->cancel(); + op->release(); + mReadyInClass[policy_class]--; + readyq.pop(); + } } - mService = NULL; } void HttpPolicy::addOp(HttpOpRequest * op) { - mReadyQueue.push_back(op); + const int policy_class(op->mReqPolicy); + + mReadyQueue[policy_class].push(op); + ++mReadyInClass[policy_class]; } -void HttpPolicy::processReadyQueue() +HttpService::ELoopSpeed HttpPolicy::processReadyQueue() { - while (! mReadyQueue.empty()) + HttpService::ELoopSpeed result(HttpService::REQUEST_SLEEP); + HttpLibcurl * pTransport(mService->getTransport()); + + for (int policy_class(0); policy_class < HttpRequest::POLICY_CLASS_LIMIT; ++policy_class) { - HttpOpRequest * op(mReadyQueue.front()); - mReadyQueue.erase(mReadyQueue.begin()); + HttpReadyQueue & readyq(mReadyQueue[policy_class]); + int active(pTransport->getActiveCountInClass(policy_class)); + int needed(8 - active); + + if (needed > 0 && mReadyInClass[policy_class] > 0) + { + // Scan ready queue for requests that match policy + + while (! readyq.empty() && needed > 0 && mReadyInClass[policy_class] > 0) + { + HttpOpRequest * op(readyq.top()); + readyq.pop(); + + op->stageFromReady(mService); + op->release(); + + --mReadyInClass[policy_class]; + --needed; + } + } + + if (! readyq.empty()) + { + // If anything is ready, continue looping... + result = (std::min)(result, HttpService::NORMAL); + } + } + + return result; +} - op->stageFromReady(mService); - op->release(); + +bool HttpPolicy::changePriority(HttpHandle handle, unsigned int priority) +{ + for (int policy_class(0); policy_class < HttpRequest::POLICY_CLASS_LIMIT; ++policy_class) + { + HttpReadyQueue::container_type & c(mReadyQueue[policy_class].get_container()); + + // Scan ready queue for requests that match policy + for (HttpReadyQueue::container_type::iterator iter(c.begin()); c.end() != iter;) + { + HttpReadyQueue::container_type::iterator cur(iter++); + + if (static_cast(*cur) == handle) + { + HttpOpRequest * op(*cur); + c.erase(cur); // All iterators are now invalidated + op->mReqPriority = priority; + mReadyQueue[policy_class].push(op); // Re-insert using adapter class + return true; + } + } } + + return false; } -- cgit v1.2.3 From 7b9da4eeda7505162f37cbfa52591f7adff032e7 Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Fri, 1 Jun 2012 17:23:51 -0400 Subject: Missed two instances of priority typed as 'float'. Became an excuse to go through an use a typedef for priority and policy class id. --- indra/llcorehttp/_httppolicy.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'indra/llcorehttp/_httppolicy.cpp') diff --git a/indra/llcorehttp/_httppolicy.cpp b/indra/llcorehttp/_httppolicy.cpp index 873b519c51..1d28f23d56 100644 --- a/indra/llcorehttp/_httppolicy.cpp +++ b/indra/llcorehttp/_httppolicy.cpp @@ -112,7 +112,7 @@ HttpService::ELoopSpeed HttpPolicy::processReadyQueue() } -bool HttpPolicy::changePriority(HttpHandle handle, unsigned int priority) +bool HttpPolicy::changePriority(HttpHandle handle, HttpRequest::priority_t priority) { for (int policy_class(0); policy_class < HttpRequest::POLICY_CLASS_LIMIT; ++policy_class) { -- cgit v1.2.3 From 05af16a23abe37210e0b880aa27387d8994419dd Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Wed, 6 Jun 2012 13:52:38 -0400 Subject: Policy + caching fixes + https support + POST working Implemented first global policy definitions to support SSL CA certificate configuration to support https: operations. Fixed HTTP 206 status handling to match what is currently being done by grid services and to lay a foundation for fixes that will be a response to ER-1824. More libcurl CURLOPT options set on easy handles to do peer verification in the traditional way. HTTP POST working and now reporting asset metrics back to grid for the viewer's asset system. This uses LLSD so that is also showing as compatible with the new library. --- indra/llcorehttp/_httppolicy.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'indra/llcorehttp/_httppolicy.cpp') diff --git a/indra/llcorehttp/_httppolicy.cpp b/indra/llcorehttp/_httppolicy.cpp index 1d28f23d56..51f5e487dc 100644 --- a/indra/llcorehttp/_httppolicy.cpp +++ b/indra/llcorehttp/_httppolicy.cpp @@ -76,12 +76,12 @@ void HttpPolicy::addOp(HttpOpRequest * op) HttpService::ELoopSpeed HttpPolicy::processReadyQueue() { HttpService::ELoopSpeed result(HttpService::REQUEST_SLEEP); - HttpLibcurl * pTransport(mService->getTransport()); + HttpLibcurl & transport(mService->getTransport()); for (int policy_class(0); policy_class < HttpRequest::POLICY_CLASS_LIMIT; ++policy_class) { HttpReadyQueue & readyq(mReadyQueue[policy_class]); - int active(pTransport->getActiveCountInClass(policy_class)); + int active(transport.getActiveCountInClass(policy_class)); int needed(8 - active); if (needed > 0 && mReadyInClass[policy_class] > 0) -- cgit v1.2.3 From 28a04400b4160dd34166483ddcf0c12637bcc363 Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Fri, 8 Jun 2012 20:21:54 -0400 Subject: Implemented HTTP retry for requests. Went in rather easily which surprised me. Added a retry queue similar to ready queue to the policy object which is sorted by retry time. Currently do five retries (after the initial try) delayed by .25, .5, 1, 2 and 5 seconds. Removed the retry logic from the lltexturefetch module. Upped the waiting time in the unit test for the retries. People won't like this but tough, need tests. --- indra/llcorehttp/_httppolicy.cpp | 98 ++++++++++++++++++++++++++++++---------- 1 file changed, 73 insertions(+), 25 deletions(-) (limited to 'indra/llcorehttp/_httppolicy.cpp') diff --git a/indra/llcorehttp/_httppolicy.cpp b/indra/llcorehttp/_httppolicy.cpp index 51f5e487dc..1f4cd34a4b 100644 --- a/indra/llcorehttp/_httppolicy.cpp +++ b/indra/llcorehttp/_httppolicy.cpp @@ -24,39 +24,46 @@ * $/LicenseInfo$ */ +#include "linden_common.h" + #include "_httppolicy.h" #include "_httpoprequest.h" #include "_httpservice.h" #include "_httplibcurl.h" +#include "lltimer.h" + namespace LLCore { HttpPolicy::HttpPolicy(HttpService * service) : mService(service) -{ - for (int policy_class(0); policy_class < HttpRequest::POLICY_CLASS_LIMIT; ++policy_class) - { - mReadyInClass[policy_class] = 0; - } -} +{} HttpPolicy::~HttpPolicy() { - for (int policy_class(0); policy_class < HttpRequest::POLICY_CLASS_LIMIT; ++policy_class) + for (int policy_class(0); policy_class < LL_ARRAY_SIZE(mState); ++policy_class) { - HttpReadyQueue & readyq(mReadyQueue[policy_class]); + HttpRetryQueue & retryq(mState[policy_class].mRetryQueue); + while (! retryq.empty()) + { + HttpOpRequest * op(retryq.top()); + op->cancel(); + op->release(); + retryq.pop(); + } + + HttpReadyQueue & readyq(mState[policy_class].mReadyQueue); while (! readyq.empty()) { HttpOpRequest * op(readyq.top()); op->cancel(); op->release(); - mReadyInClass[policy_class]--; readyq.pop(); } } @@ -68,27 +75,69 @@ void HttpPolicy::addOp(HttpOpRequest * op) { const int policy_class(op->mReqPolicy); - mReadyQueue[policy_class].push(op); - ++mReadyInClass[policy_class]; + op->mPolicyRetries = 0; + mState[policy_class].mReadyQueue.push(op); +} + + +void HttpPolicy::retryOp(HttpOpRequest * op) +{ + static const HttpTime retry_deltas[] = + { + 250000, // 1st retry in 0.25 S, etc... + 500000, + 1000000, + 2000000, + 5000000 // ... to every 5.0 S. + }; + static const int delta_max(int(LL_ARRAY_SIZE(retry_deltas)) - 1); + + const HttpTime now(totalTime()); + const int policy_class(op->mReqPolicy); + + const HttpTime delta(retry_deltas[llclamp(op->mPolicyRetries, 0, delta_max)]); + op->mPolicyRetryAt = now + delta; + ++op->mPolicyRetries; + LL_WARNS("CoreHttp") << "URL op retry #" << op->mPolicyRetries + << " being scheduled for " << delta << " uSecs from now." + << LL_ENDL; + mState[policy_class].mRetryQueue.push(op); } HttpService::ELoopSpeed HttpPolicy::processReadyQueue() { + const HttpTime now(totalTime()); HttpService::ELoopSpeed result(HttpService::REQUEST_SLEEP); HttpLibcurl & transport(mService->getTransport()); - for (int policy_class(0); policy_class < HttpRequest::POLICY_CLASS_LIMIT; ++policy_class) + for (int policy_class(0); policy_class < LL_ARRAY_SIZE(mState); ++policy_class) { - HttpReadyQueue & readyq(mReadyQueue[policy_class]); int active(transport.getActiveCountInClass(policy_class)); int needed(8 - active); - if (needed > 0 && mReadyInClass[policy_class] > 0) + HttpRetryQueue & retryq(mState[policy_class].mRetryQueue); + HttpReadyQueue & readyq(mState[policy_class].mReadyQueue); + + if (needed > 0) { - // Scan ready queue for requests that match policy - - while (! readyq.empty() && needed > 0 && mReadyInClass[policy_class] > 0) + // First see if we have any retries... + while (needed > 0 && ! retryq.empty()) + { + HttpOpRequest * op(retryq.top()); + if (op->mPolicyRetryAt > now) + break; + + retryq.pop(); + + op->stageFromReady(mService); + op->release(); + + --needed; + } + + // Now go on to the new requests... + while (needed > 0 && ! readyq.empty()) { HttpOpRequest * op(readyq.top()); readyq.pop(); @@ -96,17 +145,16 @@ HttpService::ELoopSpeed HttpPolicy::processReadyQueue() op->stageFromReady(mService); op->release(); - --mReadyInClass[policy_class]; --needed; } } - - if (! readyq.empty()) + + if (! readyq.empty() || ! retryq.empty()) { // If anything is ready, continue looping... result = (std::min)(result, HttpService::NORMAL); } - } + } // end foreach policy_class return result; } @@ -114,9 +162,9 @@ HttpService::ELoopSpeed HttpPolicy::processReadyQueue() bool HttpPolicy::changePriority(HttpHandle handle, HttpRequest::priority_t priority) { - for (int policy_class(0); policy_class < HttpRequest::POLICY_CLASS_LIMIT; ++policy_class) + for (int policy_class(0); policy_class < LL_ARRAY_SIZE(mState); ++policy_class) { - HttpReadyQueue::container_type & c(mReadyQueue[policy_class].get_container()); + HttpReadyQueue::container_type & c(mState[policy_class].mReadyQueue.get_container()); // Scan ready queue for requests that match policy for (HttpReadyQueue::container_type::iterator iter(c.begin()); c.end() != iter;) @@ -126,9 +174,9 @@ bool HttpPolicy::changePriority(HttpHandle handle, HttpRequest::priority_t prior if (static_cast(*cur) == handle) { HttpOpRequest * op(*cur); - c.erase(cur); // All iterators are now invalidated + c.erase(cur); // All iterators are now invalidated op->mReqPriority = priority; - mReadyQueue[policy_class].push(op); // Re-insert using adapter class + mState[policy_class].mReadyQueue.push(op); // Re-insert using adapter class return true; } } -- cgit v1.2.3 From 89187229dd630845177ecd7a16e2b9cb01dc47ce Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Mon, 11 Jun 2012 15:28:06 -0400 Subject: Refactoring of the request completion thread and removal of 206/content-range hack in xport. Retry/response handling is decided in policy so moved that there. Removed special case 206-without-content-range response in transport. Have this sitation recognizable in the API and let callers deal with it as needed. --- indra/llcorehttp/_httppolicy.cpp | 42 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) (limited to 'indra/llcorehttp/_httppolicy.cpp') diff --git a/indra/llcorehttp/_httppolicy.cpp b/indra/llcorehttp/_httppolicy.cpp index 1f4cd34a4b..72bb6f14e4 100644 --- a/indra/llcorehttp/_httppolicy.cpp +++ b/indra/llcorehttp/_httppolicy.cpp @@ -185,5 +185,47 @@ bool HttpPolicy::changePriority(HttpHandle handle, HttpRequest::priority_t prior return false; } +bool HttpPolicy::stageAfterCompletion(HttpOpRequest * op) +{ + static const HttpStatus cant_connect(HttpStatus::EXT_CURL_EASY, CURLE_COULDNT_CONNECT); + static const HttpStatus cant_res_proxy(HttpStatus::EXT_CURL_EASY, CURLE_COULDNT_RESOLVE_PROXY); + static const HttpStatus cant_res_host(HttpStatus::EXT_CURL_EASY, CURLE_COULDNT_RESOLVE_HOST); + + // Retry or finalize + if (! op->mStatus) + { + // If this failed, we might want to retry. Have to inspect + // the status a little more deeply for those reasons worth retrying... + if (op->mPolicyRetries < op->mPolicyRetryLimit && + ((op->mStatus.isHttpStatus() && op->mStatus.mType >= 499 && op->mStatus.mType <= 599) || + cant_connect == op->mStatus || + cant_res_proxy == op->mStatus || + cant_res_host == op->mStatus)) + { + // Okay, worth a retry. We include 499 in this test as + // it's the old 'who knows?' error from many grid services... + retryOp(op); + return true; // still active/ready + } + } + + // This op is done, finalize it delivering it to the reply queue... + if (! op->mStatus) + { + LL_WARNS("CoreHttp") << "URL op failed after " << op->mPolicyRetries + << " retries. Reason: " << op->mStatus.toString() + << LL_ENDL; + } + else if (op->mPolicyRetries) + { + LL_WARNS("CoreHttp") << "URL op succeeded after " << op->mPolicyRetries << " retries." + << LL_ENDL; + } + + op->stageFromActive(mService); + op->release(); + return false; // not active +} + } // end namespace LLCore -- cgit v1.2.3 From 7adeb3923728ca84a309a6af141c148ce38066fc Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Tue, 12 Jun 2012 17:42:33 -0400 Subject: HTTP Proxy, PUT & POST, unit tests and refactoring. Implemented/modified PUT & POST to not used chunked encoding for the request. Made the unit test much happier and probably a better thing for the pipeline. Have a cheesy static & dynamic proxy capability using both local options and a way to wire into LLProxy in llmessages. Not a clean thing but it will get the proxy path working with both socks5 & http proxies. Refactoring to get rid of unneeded library handler and unified an HttpStatus return for all requests. Big batch of code removed as a result of that and more is possible as well as some syscall avoidance with a bit more work. Boosted the unit tests for simple PUT & POST test which revealed the test harness does *not* like chunked encoding so we'll avoid it for now (and don't really need it in any of our schemes). --- indra/llcorehttp/_httppolicy.cpp | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'indra/llcorehttp/_httppolicy.cpp') diff --git a/indra/llcorehttp/_httppolicy.cpp b/indra/llcorehttp/_httppolicy.cpp index 72bb6f14e4..8ee3f88658 100644 --- a/indra/llcorehttp/_httppolicy.cpp +++ b/indra/llcorehttp/_httppolicy.cpp @@ -71,6 +71,12 @@ HttpPolicy::~HttpPolicy() } +void HttpPolicy::setPolicies(const HttpPolicyGlobal & global) +{ + mGlobalOptions = global; +} + + void HttpPolicy::addOp(HttpOpRequest * op) { const int policy_class(op->mReqPolicy); -- cgit v1.2.3 From b08125a5874a89ce5210f8fb2c961ae17fb80fde Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Thu, 14 Jun 2012 16:31:48 -0400 Subject: LLMutex recursive lock, global & per-request tracing, simple GET request, LLProxy support, HttpOptions starting to work, HTTP resource waiting fixed. Non-LLThread-based threads need to do some registration or LLMutex locks taken out in these threads will not work as expected (SH-3154). We'll get a better solution later, this fixes some things for now. Tracing of operations now supported. Global and per-request (via HttpOptions) tracing levels of [0..3]. The 2 and 3 levels use libcurl's VERBOSE mode combined with CURLOPT_DEBUGFUNCTION to stream high levels of detail into the log. *Very* laggy but useful. Simple GET request supported (no Range: header). Really just a degenrate case of a ranged get but supplied an API anyway. Global option to use the LLProxy interface to setup CURL handles for either socks5 or http proxy usage. This isn't really the most encapsulated way to do this but a better solution will have to come later. The wantHeaders and tracing options are now supported in HttpOptions giving per-request controls. Big refactoring of the HTTP resource waiter in lltexturefetch. What I was doing before wasn't correct. Instead, I'm implementing the resource wait after the Semaphore model (though not using system semaphores). So instead of having a sequence like: SEND_HTTP_REQ -> WAIT_HTTP_RESOURCE -> SEND_HTTP_REQ, we now do WAIT_HTTP_RESOURCE -> WAIT_HTTP_RESOURCE2 (actual wait) -> SEND_HTTP_REQ. Works well but the prioritized filling of the corehttp library needs some performance work later. --- indra/llcorehttp/_httppolicy.cpp | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) (limited to 'indra/llcorehttp/_httppolicy.cpp') diff --git a/indra/llcorehttp/_httppolicy.cpp b/indra/llcorehttp/_httppolicy.cpp index 8ee3f88658..0e08d88276 100644 --- a/indra/llcorehttp/_httppolicy.cpp +++ b/indra/llcorehttp/_httppolicy.cpp @@ -107,6 +107,12 @@ void HttpPolicy::retryOp(HttpOpRequest * op) LL_WARNS("CoreHttp") << "URL op retry #" << op->mPolicyRetries << " being scheduled for " << delta << " uSecs from now." << LL_ENDL; + if (op->mTracing > 0) + { + LL_INFOS("CoreHttp") << "TRACE, ToRetryQueue, Handle: " + << static_cast(op) + << LL_ENDL; + } mState[policy_class].mRetryQueue.push(op); } @@ -224,8 +230,8 @@ bool HttpPolicy::stageAfterCompletion(HttpOpRequest * op) } else if (op->mPolicyRetries) { - LL_WARNS("CoreHttp") << "URL op succeeded after " << op->mPolicyRetries << " retries." - << LL_ENDL; + LL_DEBUGS("CoreHttp") << "URL op succeeded after " << op->mPolicyRetries << " retries." + << LL_ENDL; } op->stageFromActive(mService); @@ -234,4 +240,15 @@ bool HttpPolicy::stageAfterCompletion(HttpOpRequest * op) } +int HttpPolicy::getReadyCount(HttpRequest::policy_t policy_class) +{ + if (policy_class < HttpRequest::POLICY_CLASS_LIMIT) + { + return (mState[policy_class].mReadyQueue.size() + + mState[policy_class].mRetryQueue.size()); + } + return 0; +} + + } // end namespace LLCore -- cgit v1.2.3 From a50944e078b98435685f04eda0ba93e95d4c61f2 Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Tue, 19 Jun 2012 17:01:02 -0400 Subject: Cleanup: move magic nubmers to new _httpinternal.h header file. --- indra/llcorehttp/_httppolicy.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'indra/llcorehttp/_httppolicy.cpp') diff --git a/indra/llcorehttp/_httppolicy.cpp b/indra/llcorehttp/_httppolicy.cpp index 0e08d88276..4be9f1d45f 100644 --- a/indra/llcorehttp/_httppolicy.cpp +++ b/indra/llcorehttp/_httppolicy.cpp @@ -126,7 +126,7 @@ HttpService::ELoopSpeed HttpPolicy::processReadyQueue() for (int policy_class(0); policy_class < LL_ARRAY_SIZE(mState); ++policy_class) { int active(transport.getActiveCountInClass(policy_class)); - int needed(8 - active); + int needed(DEFAULT_CONNECTIONS - active); // *FIXME: move to policy class HttpRetryQueue & retryq(mState[policy_class].mRetryQueue); HttpReadyQueue & readyq(mState[policy_class].mReadyQueue); @@ -242,7 +242,7 @@ bool HttpPolicy::stageAfterCompletion(HttpOpRequest * op) int HttpPolicy::getReadyCount(HttpRequest::policy_t policy_class) { - if (policy_class < HttpRequest::POLICY_CLASS_LIMIT) + if (policy_class < POLICY_CLASS_LIMIT) // *FIXME: use actual active class count { return (mState[policy_class].mReadyQueue.size() + mState[policy_class].mRetryQueue.size()); -- cgit v1.2.3 From bc7d5b24d16963a2715e880c518a4706a99f02fa Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Fri, 22 Jun 2012 19:13:50 -0400 Subject: This sets down the groundwork for dynamic policy classes. Groundwork is used for the default class which currently represents texture fetching. Class options implemented from API user into HttpLibcurl. Policy layer is going to start doing some traffic shaping like work to solve problems with consumer-grade gear. Need to have dynamic aspects to policies and that starts now... --- indra/llcorehttp/_httppolicy.cpp | 69 +++++++++++++++++++++++++++++++++------- 1 file changed, 58 insertions(+), 11 deletions(-) (limited to 'indra/llcorehttp/_httppolicy.cpp') diff --git a/indra/llcorehttp/_httppolicy.cpp b/indra/llcorehttp/_httppolicy.cpp index 4be9f1d45f..1dae20add6 100644 --- a/indra/llcorehttp/_httppolicy.cpp +++ b/indra/llcorehttp/_httppolicy.cpp @@ -31,6 +31,7 @@ #include "_httpoprequest.h" #include "_httpservice.h" #include "_httplibcurl.h" +#include "_httppolicyclass.h" #include "lltimer.h" @@ -38,14 +39,44 @@ namespace LLCore { + +struct HttpPolicy::State +{ +public: + State() + : mConnMax(DEFAULT_CONNECTIONS), + mConnAt(DEFAULT_CONNECTIONS), + mConnMin(2), + mNextSample(0), + mErrorCount(0), + mErrorFactor(0) + {} + + HttpReadyQueue mReadyQueue; + HttpRetryQueue mRetryQueue; + + HttpPolicyClass mOptions; + + long mConnMax; + long mConnAt; + long mConnMin; + + HttpTime mNextSample; + unsigned long mErrorCount; + unsigned long mErrorFactor; +}; + + HttpPolicy::HttpPolicy(HttpService * service) - : mService(service) + : mActiveClasses(0), + mState(NULL), + mService(service) {} HttpPolicy::~HttpPolicy() { - for (int policy_class(0); policy_class < LL_ARRAY_SIZE(mState); ++policy_class) + for (int policy_class(0); policy_class < mActiveClasses; ++policy_class) { HttpRetryQueue & retryq(mState[policy_class].mRetryQueue); while (! retryq.empty()) @@ -67,13 +98,27 @@ HttpPolicy::~HttpPolicy() readyq.pop(); } } + delete [] mState; + mState = NULL; mService = NULL; } -void HttpPolicy::setPolicies(const HttpPolicyGlobal & global) +void HttpPolicy::setPolicies(const HttpPolicyGlobal & global, + const std::vector & classes) { + llassert_always(! mState); + mGlobalOptions = global; + mActiveClasses = classes.size(); + mState = new State [mActiveClasses]; + for (int i(0); i < mActiveClasses; ++i) + { + mState[i].mOptions = classes[i]; + mState[i].mConnMax = classes[i].mConnectionLimit; + mState[i].mConnAt = mState[i].mConnMax; + mState[i].mConnMin = 2; + } } @@ -123,13 +168,14 @@ HttpService::ELoopSpeed HttpPolicy::processReadyQueue() HttpService::ELoopSpeed result(HttpService::REQUEST_SLEEP); HttpLibcurl & transport(mService->getTransport()); - for (int policy_class(0); policy_class < LL_ARRAY_SIZE(mState); ++policy_class) + for (int policy_class(0); policy_class < mActiveClasses; ++policy_class) { + State & state(mState[policy_class]); int active(transport.getActiveCountInClass(policy_class)); - int needed(DEFAULT_CONNECTIONS - active); // *FIXME: move to policy class + int needed(state.mConnAt - active); // Expect negatives here - HttpRetryQueue & retryq(mState[policy_class].mRetryQueue); - HttpReadyQueue & readyq(mState[policy_class].mReadyQueue); + HttpRetryQueue & retryq(state.mRetryQueue); + HttpReadyQueue & readyq(state.mReadyQueue); if (needed > 0) { @@ -174,9 +220,10 @@ HttpService::ELoopSpeed HttpPolicy::processReadyQueue() bool HttpPolicy::changePriority(HttpHandle handle, HttpRequest::priority_t priority) { - for (int policy_class(0); policy_class < LL_ARRAY_SIZE(mState); ++policy_class) + for (int policy_class(0); policy_class < mActiveClasses; ++policy_class) { - HttpReadyQueue::container_type & c(mState[policy_class].mReadyQueue.get_container()); + State & state(mState[policy_class]); + HttpReadyQueue::container_type & c(state.mReadyQueue.get_container()); // Scan ready queue for requests that match policy for (HttpReadyQueue::container_type::iterator iter(c.begin()); c.end() != iter;) @@ -188,7 +235,7 @@ bool HttpPolicy::changePriority(HttpHandle handle, HttpRequest::priority_t prior HttpOpRequest * op(*cur); c.erase(cur); // All iterators are now invalidated op->mReqPriority = priority; - mState[policy_class].mReadyQueue.push(op); // Re-insert using adapter class + state.mReadyQueue.push(op); // Re-insert using adapter class return true; } } @@ -242,7 +289,7 @@ bool HttpPolicy::stageAfterCompletion(HttpOpRequest * op) int HttpPolicy::getReadyCount(HttpRequest::policy_t policy_class) { - if (policy_class < POLICY_CLASS_LIMIT) // *FIXME: use actual active class count + if (policy_class < mActiveClasses) { return (mState[policy_class].mReadyQueue.size() + mState[policy_class].mRetryQueue.size()); -- cgit v1.2.3 From e172ec84fa217aae8d1e51c1e0673322c30891fe Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Sat, 23 Jun 2012 23:33:50 -0400 Subject: SH-3184/SH-3221 Improve cleanup, destructor, thread termination, etc. logic in library. With this commit, the cleanup paths should be production quality. Unit tests have been expanded to include cases requiring thread termination and cleanup by the worker thread. Special operation/request added to support the unit tests. Thread interface expanded to include a very aggressive cancel() method that does not do cleanup but prevents the thread from accessing objects that will be destroyed. --- indra/llcorehttp/_httppolicy.cpp | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) (limited to 'indra/llcorehttp/_httppolicy.cpp') diff --git a/indra/llcorehttp/_httppolicy.cpp b/indra/llcorehttp/_httppolicy.cpp index 1dae20add6..93e295537c 100644 --- a/indra/llcorehttp/_httppolicy.cpp +++ b/indra/llcorehttp/_httppolicy.cpp @@ -75,6 +75,14 @@ HttpPolicy::HttpPolicy(HttpService * service) HttpPolicy::~HttpPolicy() +{ + shutdown(); + + mService = NULL; +} + + +void HttpPolicy::shutdown() { for (int policy_class(0); policy_class < mActiveClasses; ++policy_class) { @@ -100,12 +108,12 @@ HttpPolicy::~HttpPolicy() } delete [] mState; mState = NULL; - mService = NULL; + mActiveClasses = 0; } -void HttpPolicy::setPolicies(const HttpPolicyGlobal & global, - const std::vector & classes) +void HttpPolicy::start(const HttpPolicyGlobal & global, + const std::vector & classes) { llassert_always(! mState); @@ -244,6 +252,7 @@ bool HttpPolicy::changePriority(HttpHandle handle, HttpRequest::priority_t prior return false; } + bool HttpPolicy::stageAfterCompletion(HttpOpRequest * op) { static const HttpStatus cant_connect(HttpStatus::EXT_CURL_EASY, CURLE_COULDNT_CONNECT); -- cgit v1.2.3 From e8b0088d1a0c02bfa1f9768dc91fc3df4322adae Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Tue, 26 Jun 2012 12:28:58 -0400 Subject: SH-3184/SH-3221 More work on cleanup with better unit test work and more aggressive shutdown of a thread. Some additional work let me enable a memory check for the clean shutdown case and generally do a better job on other interfaces. Request queue waiters now awake on shutdown and don't sleep once the queue is turned off. Much better semantically for how this will be used. --- indra/llcorehttp/_httppolicy.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'indra/llcorehttp/_httppolicy.cpp') diff --git a/indra/llcorehttp/_httppolicy.cpp b/indra/llcorehttp/_httppolicy.cpp index 93e295537c..4350ff617b 100644 --- a/indra/llcorehttp/_httppolicy.cpp +++ b/indra/llcorehttp/_httppolicy.cpp @@ -90,20 +90,20 @@ void HttpPolicy::shutdown() while (! retryq.empty()) { HttpOpRequest * op(retryq.top()); + retryq.pop(); op->cancel(); op->release(); - retryq.pop(); } HttpReadyQueue & readyq(mState[policy_class].mReadyQueue); while (! readyq.empty()) { HttpOpRequest * op(readyq.top()); + readyq.pop(); op->cancel(); op->release(); - readyq.pop(); } } delete [] mState; @@ -218,7 +218,7 @@ HttpService::ELoopSpeed HttpPolicy::processReadyQueue() if (! readyq.empty() || ! retryq.empty()) { // If anything is ready, continue looping... - result = (std::min)(result, HttpService::NORMAL); + result = HttpService::NORMAL; } } // end foreach policy_class -- cgit v1.2.3 From 2d7b7de20327a40be12a620debaae9917af16cd6 Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Tue, 3 Jul 2012 13:06:46 -0400 Subject: More integration work for texture fetch timeouts. The fetch state machine received a new timeout during the WAIT_HTTP_REQ state. For the integration, rather than jump the state to done, we issue a request cancel and let the notification plumbing do the rest without any race conditions or special-case logic. --- indra/llcorehttp/_httppolicy.cpp | 49 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 47 insertions(+), 2 deletions(-) (limited to 'indra/llcorehttp/_httppolicy.cpp') diff --git a/indra/llcorehttp/_httppolicy.cpp b/indra/llcorehttp/_httppolicy.cpp index 4350ff617b..1b10805b72 100644 --- a/indra/llcorehttp/_httppolicy.cpp +++ b/indra/llcorehttp/_httppolicy.cpp @@ -231,9 +231,12 @@ bool HttpPolicy::changePriority(HttpHandle handle, HttpRequest::priority_t prior for (int policy_class(0); policy_class < mActiveClasses; ++policy_class) { State & state(mState[policy_class]); - HttpReadyQueue::container_type & c(state.mReadyQueue.get_container()); - + // We don't scan retry queue because a priority change there + // is meaningless. The request will be issued based on retry + // intervals not priority value, which is now moot. + // Scan ready queue for requests that match policy + HttpReadyQueue::container_type & c(state.mReadyQueue.get_container()); for (HttpReadyQueue::container_type::iterator iter(c.begin()); c.end() != iter;) { HttpReadyQueue::container_type::iterator cur(iter++); @@ -253,6 +256,48 @@ bool HttpPolicy::changePriority(HttpHandle handle, HttpRequest::priority_t prior } +bool HttpPolicy::cancel(HttpHandle handle) +{ + for (int policy_class(0); policy_class < mActiveClasses; ++policy_class) + { + State & state(mState[policy_class]); + + // Scan retry queue + HttpRetryQueue::container_type & c1(state.mRetryQueue.get_container()); + for (HttpRetryQueue::container_type::iterator iter(c1.begin()); c1.end() != iter;) + { + HttpRetryQueue::container_type::iterator cur(iter++); + + if (static_cast(*cur) == handle) + { + HttpOpRequest * op(*cur); + c1.erase(cur); // All iterators are now invalidated + op->cancel(); + op->release(); + return true; + } + } + + // Scan ready queue + HttpReadyQueue::container_type & c2(state.mReadyQueue.get_container()); + for (HttpReadyQueue::container_type::iterator iter(c2.begin()); c2.end() != iter;) + { + HttpReadyQueue::container_type::iterator cur(iter++); + + if (static_cast(*cur) == handle) + { + HttpOpRequest * op(*cur); + c2.erase(cur); // All iterators are now invalidated + op->cancel(); + op->release(); + return true; + } + } + } + + return false; +} + bool HttpPolicy::stageAfterCompletion(HttpOpRequest * op) { static const HttpStatus cant_connect(HttpStatus::EXT_CURL_EASY, CURLE_COULDNT_CONNECT); -- cgit v1.2.3 From e38a676c087d0adce9bd35cf3bdf8ff0e898f201 Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Tue, 3 Jul 2012 19:22:24 -0400 Subject: Add CURLE_SEND_ERROR and CURLE_RECV_ERROR to the set of retryable errors. Data problems after connections are established should be retried as well. Extend to appropriate libcurl codes. Also allow our connectivity to drop to as low as a single connection when trying to recover. --- indra/llcorehttp/_httppolicy.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'indra/llcorehttp/_httppolicy.cpp') diff --git a/indra/llcorehttp/_httppolicy.cpp b/indra/llcorehttp/_httppolicy.cpp index 1b10805b72..1e64924198 100644 --- a/indra/llcorehttp/_httppolicy.cpp +++ b/indra/llcorehttp/_httppolicy.cpp @@ -46,7 +46,7 @@ public: State() : mConnMax(DEFAULT_CONNECTIONS), mConnAt(DEFAULT_CONNECTIONS), - mConnMin(2), + mConnMin(1), mNextSample(0), mErrorCount(0), mErrorFactor(0) @@ -303,6 +303,8 @@ bool HttpPolicy::stageAfterCompletion(HttpOpRequest * op) static const HttpStatus cant_connect(HttpStatus::EXT_CURL_EASY, CURLE_COULDNT_CONNECT); static const HttpStatus cant_res_proxy(HttpStatus::EXT_CURL_EASY, CURLE_COULDNT_RESOLVE_PROXY); static const HttpStatus cant_res_host(HttpStatus::EXT_CURL_EASY, CURLE_COULDNT_RESOLVE_HOST); + static const HttpStatus send_error(HttpStatus::EXT_CURL_EASY, CURLE_SEND_ERROR); + static const HttpStatus recv_error(HttpStatus::EXT_CURL_EASY, CURLE_RECV_ERROR); // Retry or finalize if (! op->mStatus) @@ -313,7 +315,9 @@ bool HttpPolicy::stageAfterCompletion(HttpOpRequest * op) ((op->mStatus.isHttpStatus() && op->mStatus.mType >= 499 && op->mStatus.mType <= 599) || cant_connect == op->mStatus || cant_res_proxy == op->mStatus || - cant_res_host == op->mStatus)) + cant_res_host == op->mStatus || + send_error == op->mStatus || + recv_error == op->mStatus)) { // Okay, worth a retry. We include 499 in this test as // it's the old 'who knows?' error from many grid services... -- cgit v1.2.3 From 85e69b043b098dbe5a09f2eac6ff541123089f13 Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Mon, 23 Jul 2012 23:40:07 +0000 Subject: Big comment and naming cleanup. Ready for prime-time. Add to-do list to _httpinternal.h to guide anyone who wants to pitch in and help. --- indra/llcorehttp/_httppolicy.cpp | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) (limited to 'indra/llcorehttp/_httppolicy.cpp') diff --git a/indra/llcorehttp/_httppolicy.cpp b/indra/llcorehttp/_httppolicy.cpp index 1e64924198..c7a69ad133 100644 --- a/indra/llcorehttp/_httppolicy.cpp +++ b/indra/llcorehttp/_httppolicy.cpp @@ -40,12 +40,17 @@ namespace LLCore { +// Per-policy-class data for a running system. +// Collection of queues, parameters, history, metrics, etc. +// for a single policy class. +// +// Threading: accessed only by worker thread struct HttpPolicy::State { public: State() - : mConnMax(DEFAULT_CONNECTIONS), - mConnAt(DEFAULT_CONNECTIONS), + : mConnMax(HTTP_CONNECTION_LIMIT_DEFAULT), + mConnAt(HTTP_CONNECTION_LIMIT_DEFAULT), mConnMin(1), mNextSample(0), mErrorCount(0), @@ -298,6 +303,7 @@ bool HttpPolicy::cancel(HttpHandle handle) return false; } + bool HttpPolicy::stageAfterCompletion(HttpOpRequest * op) { static const HttpStatus cant_connect(HttpStatus::EXT_CURL_EASY, CURLE_COULDNT_CONNECT); @@ -345,7 +351,7 @@ bool HttpPolicy::stageAfterCompletion(HttpOpRequest * op) } -int HttpPolicy::getReadyCount(HttpRequest::policy_t policy_class) +int HttpPolicy::getReadyCount(HttpRequest::policy_t policy_class) const { if (policy_class < mActiveClasses) { -- cgit v1.2.3 From bf004be1023347bcabaae6baa1624b2ed78d69fd Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Wed, 1 Aug 2012 12:38:28 -0400 Subject: SH-3308 Beef up retry messaging. Reformatted messages around request retry. Successfully retried requests also message so you can see the cycle closed. Added additional retryable error codes (timeout, other libcurl failures). Commenting and removed some unnecessary std::min logic. --- indra/llcorehttp/_httppolicy.cpp | 34 ++++++++++++++++++++++++++++------ 1 file changed, 28 insertions(+), 6 deletions(-) (limited to 'indra/llcorehttp/_httppolicy.cpp') diff --git a/indra/llcorehttp/_httppolicy.cpp b/indra/llcorehttp/_httppolicy.cpp index c7a69ad133..76c1e22431 100644 --- a/indra/llcorehttp/_httppolicy.cpp +++ b/indra/llcorehttp/_httppolicy.cpp @@ -162,8 +162,10 @@ void HttpPolicy::retryOp(HttpOpRequest * op) const HttpTime delta(retry_deltas[llclamp(op->mPolicyRetries, 0, delta_max)]); op->mPolicyRetryAt = now + delta; ++op->mPolicyRetries; - LL_WARNS("CoreHttp") << "URL op retry #" << op->mPolicyRetries - << " being scheduled for " << delta << " uSecs from now." + LL_WARNS("CoreHttp") << "HTTP request " << static_cast(op) + << " retry " << op->mPolicyRetries + << " scheduled for +" << (delta / HttpTime(1000)) + << " mS. Status: " << op->mStatus.toHex() << LL_ENDL; if (op->mTracing > 0) { @@ -175,6 +177,17 @@ void HttpPolicy::retryOp(HttpOpRequest * op) } +// Attempt to deliver requests to the transport layer. +// +// Tries to find HTTP requests for each policy class with +// available capacity. Starts with the retry queue first +// looking for requests that have waited long enough then +// moves on to the ready queue. +// +// If all queues are empty, will return an indication that +// the worker thread may sleep hard otherwise will ask for +// normal polling frequency. +// HttpService::ELoopSpeed HttpPolicy::processReadyQueue() { const HttpTime now(totalTime()); @@ -311,6 +324,9 @@ bool HttpPolicy::stageAfterCompletion(HttpOpRequest * op) static const HttpStatus cant_res_host(HttpStatus::EXT_CURL_EASY, CURLE_COULDNT_RESOLVE_HOST); static const HttpStatus send_error(HttpStatus::EXT_CURL_EASY, CURLE_SEND_ERROR); static const HttpStatus recv_error(HttpStatus::EXT_CURL_EASY, CURLE_RECV_ERROR); + static const HttpStatus upload_failed(HttpStatus::EXT_CURL_EASY, CURLE_UPLOAD_FAILED); + static const HttpStatus op_timedout(HttpStatus::EXT_CURL_EASY, CURLE_OPERATION_TIMEDOUT); + static const HttpStatus post_error(HttpStatus::EXT_CURL_EASY, CURLE_HTTP_POST_ERROR); // Retry or finalize if (! op->mStatus) @@ -323,7 +339,10 @@ bool HttpPolicy::stageAfterCompletion(HttpOpRequest * op) cant_res_proxy == op->mStatus || cant_res_host == op->mStatus || send_error == op->mStatus || - recv_error == op->mStatus)) + recv_error == op->mStatus || + upload_failed == op->mStatus || + op_timedout == op->mStatus || + post_error == op->mStatus)) { // Okay, worth a retry. We include 499 in this test as // it's the old 'who knows?' error from many grid services... @@ -335,14 +354,17 @@ bool HttpPolicy::stageAfterCompletion(HttpOpRequest * op) // This op is done, finalize it delivering it to the reply queue... if (! op->mStatus) { - LL_WARNS("CoreHttp") << "URL op failed after " << op->mPolicyRetries + LL_WARNS("CoreHttp") << "HTTP request " << static_cast(op) + << " failed after " << op->mPolicyRetries << " retries. Reason: " << op->mStatus.toString() + << " (" << op->mStatus.toHex() << ")" << LL_ENDL; } else if (op->mPolicyRetries) { - LL_DEBUGS("CoreHttp") << "URL op succeeded after " << op->mPolicyRetries << " retries." - << LL_ENDL; + LL_WARNS("CoreHttp") << "HTTP request " << static_cast(op) + << " succeeded on retry " << op->mPolicyRetries << "." + << LL_ENDL; } op->stageFromActive(mService); -- cgit v1.2.3