diff options
Diffstat (limited to 'indra/llcorehttp')
30 files changed, 2415 insertions, 656 deletions
diff --git a/indra/llcorehttp/_httpinternal.h b/indra/llcorehttp/_httpinternal.h index 14f744a9f1..effc6a42c5 100755 --- a/indra/llcorehttp/_httpinternal.h +++ b/indra/llcorehttp/_httpinternal.h @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -36,7 +36,8 @@ // General library to-do list // // - Implement policy classes. Structure is mostly there just didn't -// need it for the first consumer. +// need it for the first consumer. [Classes are there. More +// advanced features, like borrowing, aren't there yet.] // - Consider Removing 'priority' from the request interface. Its use // in an always active class can lead to starvation of low-priority // requests. Requires coodination of priority values across all @@ -46,6 +47,7 @@ // may not really need it. // - Set/get for global policy and policy classes is clumsy. Rework // it heading in a direction that allows for more dynamic behavior. +// [Mostly fixed] // - Move HttpOpRequest::prepareRequest() to HttpLibcurl for the // pedantic. // - Update downloader and other long-duration services are going to @@ -73,7 +75,7 @@ // the main source file. // - Expand areas of usage eventually leading to the removal of LLCurl. // Rough order of expansion: -// . Mesh fetch +// . Mesh fetch [Underway] // . Avatar names // . Group membership lists // . Caps access in general @@ -97,8 +99,8 @@ namespace LLCore { // Maxium number of policy classes that can be defined. -// *TODO: Currently limited to the default class, extend. -const int HTTP_POLICY_CLASS_LIMIT = 1; +// *TODO: Currently limited to the default class + 1, extend. +const int HTTP_POLICY_CLASS_LIMIT = 8; // Debug/informational tracing. Used both // as a global option and in per-request traces. @@ -129,6 +131,7 @@ const int HTTP_REDIRECTS_DEFAULT = 10; // Retries and time-on-queue are not included and aren't // accounted for. const long HTTP_REQUEST_TIMEOUT_DEFAULT = 30L; +const long HTTP_REQUEST_XFER_TIMEOUT_DEFAULT = 0L; const long HTTP_REQUEST_TIMEOUT_MIN = 0L; const long HTTP_REQUEST_TIMEOUT_MAX = 3600L; @@ -137,6 +140,11 @@ const int HTTP_CONNECTION_LIMIT_DEFAULT = 8; const int HTTP_CONNECTION_LIMIT_MIN = 1; const int HTTP_CONNECTION_LIMIT_MAX = 256; +// Miscellaneous defaults +const long HTTP_PIPELINING_DEFAULT = 0L; +const bool HTTP_USE_RETRY_AFTER_DEFAULT = true; +const long HTTP_THROTTLE_RATE_DEFAULT = 0L; + // Tuning parameters // Time worker thread sleeps after a pass through the diff --git a/indra/llcorehttp/_httplibcurl.cpp b/indra/llcorehttp/_httplibcurl.cpp index 6fe0bfc7d1..0cb4e9d8b7 100755 --- a/indra/llcorehttp/_httplibcurl.cpp +++ b/indra/llcorehttp/_httplibcurl.cpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -41,7 +41,8 @@ namespace LLCore HttpLibcurl::HttpLibcurl(HttpService * service) : mService(service), mPolicyCount(0), - mMultiHandles(NULL) + mMultiHandles(NULL), + mActiveHandles(NULL) {} @@ -77,6 +78,9 @@ void HttpLibcurl::shutdown() delete [] mMultiHandles; mMultiHandles = NULL; + + delete [] mActiveHandles; + mActiveHandles = NULL; } mPolicyCount = 0; @@ -90,9 +94,12 @@ void HttpLibcurl::start(int policy_count) mPolicyCount = policy_count; mMultiHandles = new CURLM * [mPolicyCount]; + mActiveHandles = new int [mPolicyCount]; + for (int policy_class(0); policy_class < mPolicyCount; ++policy_class) { mMultiHandles[policy_class] = curl_multi_init(); + mActiveHandles[policy_class] = 0; } } @@ -110,8 +117,10 @@ HttpService::ELoopSpeed HttpLibcurl::processTransport() // Give libcurl some cycles to do I/O & callbacks for (int policy_class(0); policy_class < mPolicyCount; ++policy_class) { - if (! mMultiHandles[policy_class]) + if (! mActiveHandles[policy_class] || ! mMultiHandles[policy_class]) + { continue; + } int running(0); CURLMcode status(CURLM_CALL_MULTI_PERFORM); @@ -132,12 +141,10 @@ HttpService::ELoopSpeed HttpLibcurl::processTransport() CURL * handle(msg->easy_handle); CURLcode result(msg->data.result); - if (completeRequest(mMultiHandles[policy_class], handle, result)) - { - // Request is still active, don't get too sleepy - ret = HttpService::NORMAL; - } - handle = NULL; // No longer valid on return + completeRequest(mMultiHandles[policy_class], handle, result); + handle = NULL; // No longer valid on return + ret = HttpService::NORMAL; // If anything completes, we may have a free slot. + // Turning around quickly reduces connection gap by 7-10mS. } else if (CURLMSG_NONE == msg->msg) { @@ -193,6 +200,7 @@ void HttpLibcurl::addOp(HttpOpRequest * op) // On success, make operation active mActiveOps.insert(op); + ++mActiveHandles[op->mReqPolicy]; } @@ -214,6 +222,7 @@ bool HttpLibcurl::cancel(HttpHandle handle) // Drop references mActiveOps.erase(it); + --mActiveHandles[op->mReqPolicy]; op->release(); return true; @@ -275,6 +284,7 @@ bool HttpLibcurl::completeRequest(CURLM * multi_handle, CURL * handle, CURLcode // Deactivate request mActiveOps.erase(it); + --mActiveHandles[op->mReqPolicy]; op->mCurlActive = false; // Set final status of request if it hasn't failed by other mechanisms yet @@ -336,19 +346,9 @@ int HttpLibcurl::getActiveCount() const int HttpLibcurl::getActiveCountInClass(int policy_class) const { - int count(0); - - for (active_set_t::const_iterator iter(mActiveOps.begin()); - mActiveOps.end() != iter; - ++iter) - { - if ((*iter)->mReqPolicy == policy_class) - { - ++count; - } - } - - return count; + llassert_always(policy_class < mPolicyCount); + + return mActiveHandles ? mActiveHandles[policy_class] : 0; } @@ -359,12 +359,17 @@ int HttpLibcurl::getActiveCountInClass(int policy_class) const struct curl_slist * append_headers_to_slist(const HttpHeaders * headers, struct curl_slist * slist) { - for (HttpHeaders::container_t::const_iterator it(headers->mHeaders.begin()); - - headers->mHeaders.end() != it; - ++it) + const HttpHeaders::const_iterator end(headers->end()); + for (HttpHeaders::const_iterator it(headers->begin()); end != it; ++it) { - slist = curl_slist_append(slist, (*it).c_str()); + static const char sep[] = ": "; + std::string header; + header.reserve((*it).first.size() + (*it).second.size() + sizeof(sep)); + header.append((*it).first); + header.append(sep); + header.append((*it).second); + + slist = curl_slist_append(slist, header.c_str()); } return slist; } diff --git a/indra/llcorehttp/_httplibcurl.h b/indra/llcorehttp/_httplibcurl.h index 611f029ef5..67f98dd4f0 100755 --- a/indra/llcorehttp/_httplibcurl.h +++ b/indra/llcorehttp/_httplibcurl.h @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -71,16 +71,22 @@ public: /// /// @return Indication of how long this method is /// willing to wait for next service call. + /// + /// Threading: called by worker thread. HttpService::ELoopSpeed processTransport(); /// Add request to the active list. Caller is expected to have /// provided us with a reference count on the op to hold the /// request. (No additional references will be added.) + /// + /// Threading: called by worker thread. void addOp(HttpOpRequest * op); /// One-time call to set the number of policy classes to be /// serviced and to create the resources for each. Value /// must agree with HttpPolicy::setPolicies() call. + /// + /// Threading: called by init thread. void start(int policy_count); /// Synchronously stop libcurl operations. All active requests @@ -91,9 +97,13 @@ public: /// respective reply queues. /// /// Can be restarted with a start() call. + /// + /// Threading: called by worker thread. void shutdown(); /// Return global and per-class counts of active requests. + /// + /// Threading: called by worker thread. int getActiveCount() const; int getActiveCountInClass(int policy_class) const; @@ -103,6 +113,7 @@ public: /// /// @return True if handle was found and operation canceled. /// + /// Threading: called by worker thread. bool cancel(HttpHandle handle); protected: @@ -121,7 +132,8 @@ protected: HttpService * mService; // Simple reference, not owner active_set_t mActiveOps; int mPolicyCount; - CURLM ** mMultiHandles; + CURLM ** mMultiHandles; // One handle per policy class + int * mActiveHandles; // Active count per policy class }; // end class HttpLibcurl } // end namespace LLCore diff --git a/indra/llcorehttp/_httpoperation.cpp b/indra/llcorehttp/_httpoperation.cpp index 5cf5bc5930..7acd728bbd 100755 --- a/indra/llcorehttp/_httpoperation.cpp +++ b/indra/llcorehttp/_httpoperation.cpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -53,7 +53,7 @@ HttpOperation::HttpOperation() mUserHandler(NULL), mReqPolicy(HttpRequest::DEFAULT_POLICY_ID), mReqPriority(0U), - mTracing(0) + mTracing(HTTP_TRACE_OFF) { mMetricCreated = totalTime(); } diff --git a/indra/llcorehttp/_httpoperation.h b/indra/llcorehttp/_httpoperation.h index 914627fad0..937a61187d 100755 --- a/indra/llcorehttp/_httpoperation.h +++ b/indra/llcorehttp/_httpoperation.h @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -72,7 +72,7 @@ class HttpService; class HttpOperation : public LLCoreInt::RefCounted { public: - /// Threading: called by a consumer/application thread. + /// Threading: called by consumer thread. HttpOperation(); protected: @@ -108,7 +108,7 @@ public: /// by the worker thread. This is passible data /// until notification is performed. /// - /// Threading: called by application thread. + /// Threading: called by consumer thread. /// void setReplyPath(HttpReplyQueue * reply_queue, HttpHandler * handler); @@ -141,7 +141,7 @@ public: /// call to HttpRequest::update(). This method does the necessary /// dispatching. /// - /// Threading: called by application thread. + /// Threading: called by consumer thread. /// virtual void visitNotifier(HttpRequest *); diff --git a/indra/llcorehttp/_httpoprequest.cpp b/indra/llcorehttp/_httpoprequest.cpp index 51a8eaf998..d72f8f6119 100755 --- a/indra/llcorehttp/_httpoprequest.cpp +++ b/indra/llcorehttp/_httpoprequest.cpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -64,6 +64,15 @@ int parse_content_range_header(char * buffer, unsigned int * last, unsigned int * length); +// Similar for Retry-After headers. Only parses the delta form +// of the header, HTTP time formats aren't interesting for client +// purposes. +// +// @return 0 if successfully parsed and seconds time delta +// returned in time argument. +// +int parse_retry_after_header(char * buffer, int * time); + // Take data from libcurl's CURLOPT_DEBUGFUNCTION callback and // escape and format it for a tracing line in logging. Absolutely @@ -74,14 +83,12 @@ void escape_libcurl_debug_data(char * buffer, size_t len, bool scrub, std::string & safe_line); -// OS-neutral string comparisons of various types -int os_strncasecmp(const char *s1, const char *s2, size_t n); -int os_strcasecmp(const char *s1, const char *s2); -char * os_strtok_r(char *str, const char *delim, char **saveptr); - - -static const char * const hdr_whitespace(" \t"); -static const char * const hdr_separator(": \t"); +// OS-neutral string comparisons of various types. +int os_strcasecmp(const char * s1, const char * s2); +char * os_strtok_r(char * str, const char * delim, char ** saveptr); +char * os_strtrim(char * str); +char * os_strltrim(char * str); +void os_strlower(char * str); } // end anonymous namespace @@ -104,12 +111,15 @@ HttpOpRequest::HttpOpRequest() mCurlService(NULL), mCurlHeaders(NULL), mCurlBodyPos(0), + mCurlTemp(NULL), + mCurlTempLen(0), mReplyBody(NULL), mReplyOffset(0), mReplyLength(0), mReplyFullLength(0), mReplyHeaders(NULL), mPolicyRetries(0), + mPolicy503Retries(0), mPolicyRetryAt(HttpTime(0)), mPolicyRetryLimit(HTTP_RETRY_COUNT_DEFAULT) { @@ -153,6 +163,10 @@ HttpOpRequest::~HttpOpRequest() mCurlHeaders = NULL; } + delete [] mCurlTemp; + mCurlTemp = NULL; + mCurlTempLen = 0; + if (mReplyBody) { mReplyBody->release(); @@ -206,6 +220,11 @@ void HttpOpRequest::stageFromActive(HttpService * service) mCurlHeaders = NULL; } + // Also not needed on the other side + delete [] mCurlTemp; + mCurlTemp = NULL; + mCurlTempLen = 0; + addAsReply(); } @@ -224,6 +243,7 @@ void HttpOpRequest::visitNotifier(HttpRequest * request) response->setRange(mReplyOffset, mReplyLength, mReplyFullLength); } response->setContentType(mReplyConType); + response->setRetries(mPolicyRetries, mPolicy503Retries); mUserHandler->onCompleted(static_cast<HttpHandle>(this), response); @@ -333,6 +353,10 @@ void HttpOpRequest::setupCommon(HttpRequest::policy_t policy_id, { mProcFlags |= PF_SAVE_HEADERS; } + if (options->getUseRetryAfter()) + { + mProcFlags |= PF_USE_RETRY_AFTER; + } mPolicyRetryLimit = options->getRetries(); mPolicyRetryLimit = llclamp(mPolicyRetryLimit, HTTP_RETRY_COUNT_MIN, HTTP_RETRY_COUNT_MAX); mTracing = (std::max)(mTracing, llclamp(options->getTrace(), HTTP_TRACE_MIN, HTTP_TRACE_MAX)); @@ -378,7 +402,7 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service) // *FIXME: better error handling later HttpStatus status; - // Get policy options + // Get global policy options HttpPolicyGlobal & policy(service->getPolicy().getGlobalOptions()); mCurlHandle = curl_easy_init(); @@ -417,30 +441,27 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service) curl_easy_setopt(mCurlHandle, CURLOPT_SSL_VERIFYPEER, 1); curl_easy_setopt(mCurlHandle, CURLOPT_SSL_VERIFYHOST, 0); - const std::string * opt_value(NULL); - long opt_long(0L); - policy.get(HttpRequest::GP_LLPROXY, &opt_long); - if (opt_long) + if (policy.mUseLLProxy) { // Use the viewer-based thread-safe API which has a // fast/safe check for proxy enable. Would like to // encapsulate this someway... LLProxy::getInstance()->applyProxySettings(mCurlHandle); } - else if (policy.get(HttpRequest::GP_HTTP_PROXY, &opt_value)) + else if (policy.mHttpProxy.size()) { // *TODO: This is fine for now but get fuller socks5/ // authentication thing going later.... - curl_easy_setopt(mCurlHandle, CURLOPT_PROXY, opt_value->c_str()); + curl_easy_setopt(mCurlHandle, CURLOPT_PROXY, policy.mHttpProxy.c_str()); curl_easy_setopt(mCurlHandle, CURLOPT_PROXYTYPE, CURLPROXY_HTTP); } - if (policy.get(HttpRequest::GP_CA_PATH, &opt_value)) + if (policy.mCAPath.size()) { - curl_easy_setopt(mCurlHandle, CURLOPT_CAPATH, opt_value->c_str()); + curl_easy_setopt(mCurlHandle, CURLOPT_CAPATH, policy.mCAPath.c_str()); } - if (policy.get(HttpRequest::GP_CA_FILE, &opt_value)) + if (policy.mCAFile.size()) { - curl_easy_setopt(mCurlHandle, CURLOPT_CAINFO, opt_value->c_str()); + curl_easy_setopt(mCurlHandle, CURLOPT_CAINFO, policy.mCAFile.c_str()); } switch (mReqMethod) @@ -524,12 +545,19 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service) // Request options long timeout(HTTP_REQUEST_TIMEOUT_DEFAULT); + long xfer_timeout(HTTP_REQUEST_XFER_TIMEOUT_DEFAULT); if (mReqOptions) - { + { timeout = mReqOptions->getTimeout(); timeout = llclamp(timeout, HTTP_REQUEST_TIMEOUT_MIN, HTTP_REQUEST_TIMEOUT_MAX); + xfer_timeout = mReqOptions->getTransferTimeout(); + xfer_timeout = llclamp(xfer_timeout, HTTP_REQUEST_TIMEOUT_MIN, HTTP_REQUEST_TIMEOUT_MAX); + } + if (xfer_timeout == 0L) + { + xfer_timeout = timeout; } - curl_easy_setopt(mCurlHandle, CURLOPT_TIMEOUT, timeout); + curl_easy_setopt(mCurlHandle, CURLOPT_TIMEOUT, xfer_timeout); curl_easy_setopt(mCurlHandle, CURLOPT_CONNECTTIMEOUT, timeout); // Request headers @@ -540,7 +568,7 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service) } curl_easy_setopt(mCurlHandle, CURLOPT_HTTPHEADER, mCurlHeaders); - if (mProcFlags & (PF_SCAN_RANGE_HEADER | PF_SAVE_HEADERS)) + if (mProcFlags & (PF_SCAN_RANGE_HEADER | PF_SAVE_HEADERS | PF_USE_RETRY_AFTER)) { curl_easy_setopt(mCurlHandle, CURLOPT_HEADERFUNCTION, headerCallback); curl_easy_setopt(mCurlHandle, CURLOPT_HEADERDATA, this); @@ -601,15 +629,15 @@ size_t HttpOpRequest::headerCallback(void * data, size_t size, size_t nmemb, voi { static const char status_line[] = "HTTP/"; static const size_t status_line_len = sizeof(status_line) - 1; - - static const char con_ran_line[] = "content-range:"; - static const size_t con_ran_line_len = sizeof(con_ran_line) - 1; - + static const char con_ran_line[] = "content-range"; + static const char con_retry_line[] = "retry-after"; + HttpOpRequest * op(static_cast<HttpOpRequest *>(userdata)); const size_t hdr_size(size * nmemb); const char * hdr_data(static_cast<const char *>(data)); // Not null terminated - + bool is_header(true); + if (hdr_size >= status_line_len && ! strncmp(status_line, hdr_data, status_line_len)) { // One of possibly several status lines. Reset what we know and start over @@ -617,11 +645,13 @@ size_t HttpOpRequest::headerCallback(void * data, size_t size, size_t nmemb, voi op->mReplyOffset = 0; op->mReplyLength = 0; op->mReplyFullLength = 0; + op->mReplyRetryAfter = 0; op->mStatus = HttpStatus(); if (op->mReplyHeaders) { - op->mReplyHeaders->mHeaders.clear(); + op->mReplyHeaders->clear(); } + is_header = false; } // Nothing in here wants a final CR/LF combination. Remove @@ -634,52 +664,109 @@ size_t HttpOpRequest::headerCallback(void * data, size_t size, size_t nmemb, voi --wanted_hdr_size; } } + + // Copy and normalize header fragments for the following + // stages. Would like to modify the data in-place but that + // may not be allowed and we need one byte extra for NUL. + // At the end of this we will have: + // + // If ':' present in header: + // 1. name points to text to left of colon which + // will be ascii lower-cased and left and right + // trimmed of whitespace. + // 2. value points to text to right of colon which + // will be left trimmed of whitespace. + // Otherwise: + // 1. name points to header which will be left + // trimmed of whitespace. + // 2. value is NULL + // Any non-NULL pointer may point to a zero-length string. + // + if (wanted_hdr_size >= op->mCurlTempLen) + { + delete [] op->mCurlTemp; + op->mCurlTempLen = 2 * wanted_hdr_size + 1; + op->mCurlTemp = new char [op->mCurlTempLen]; + } + memcpy(op->mCurlTemp, hdr_data, wanted_hdr_size); + op->mCurlTemp[wanted_hdr_size] = '\0'; + char * name(op->mCurlTemp); + char * value(strchr(name, ':')); + if (value) + { + *value++ = '\0'; + os_strlower(name); + name = os_strtrim(name); + value = os_strltrim(value); + } + else + { + // Doesn't look well-formed, do minimal normalization on it + name = os_strltrim(name); + } + + // Normalized, now reject headers with empty names. + if (! *name) + { + // No use continuing + return hdr_size; + } // Save header if caller wants them in the response - if (op->mProcFlags & PF_SAVE_HEADERS) + if (is_header && op->mProcFlags & PF_SAVE_HEADERS) { // Save headers in response if (! op->mReplyHeaders) { op->mReplyHeaders = new HttpHeaders; } - op->mReplyHeaders->mHeaders.push_back(std::string(hdr_data, wanted_hdr_size)); + op->mReplyHeaders->append(name, value ? value : ""); } + // From this point, header-specific processors are free to + // modify the header value. + // Detect and parse 'Content-Range' headers - if (op->mProcFlags & PF_SCAN_RANGE_HEADER) + if (is_header + && op->mProcFlags & PF_SCAN_RANGE_HEADER + && value && *value + && ! strcmp(name, con_ran_line)) { - char hdr_buffer[128]; // Enough for a reasonable header - size_t frag_size((std::min)(wanted_hdr_size, sizeof(hdr_buffer) - 1)); - - memcpy(hdr_buffer, hdr_data, frag_size); - hdr_buffer[frag_size] = '\0'; - if (frag_size > con_ran_line_len && - ! os_strncasecmp(hdr_buffer, con_ran_line, con_ran_line_len)) + unsigned int first(0), last(0), length(0); + int status; + + if (! (status = parse_content_range_header(value, &first, &last, &length))) + { + // Success, record the fragment position + op->mReplyOffset = first; + op->mReplyLength = last - first + 1; + op->mReplyFullLength = length; + } + else if (-1 == status) + { + // Response is badly formed and shouldn't be accepted + op->mStatus = HttpStatus(HttpStatus::LLCORE, HE_INV_CONTENT_RANGE_HDR); + } + else { - unsigned int first(0), last(0), length(0); - int status; + // Ignore the unparsable. + LL_INFOS_ONCE("CoreHttp") << "Problem parsing odd Content-Range header: '" + << std::string(hdr_data, wanted_hdr_size) + << "'. Ignoring." + << LL_ENDL; + } + } - if (! (status = parse_content_range_header(hdr_buffer, &first, &last, &length))) - { - // Success, record the fragment position - op->mReplyOffset = first; - op->mReplyLength = last - first + 1; - op->mReplyFullLength = length; - } - else if (-1 == status) - { - // Response is badly formed and shouldn't be accepted - op->mStatus = HttpStatus(HttpStatus::LLCORE, HE_INV_CONTENT_RANGE_HDR); - } - else - { - // Ignore the unparsable. - LL_INFOS_ONCE("CoreHttp") << "Problem parsing odd Content-Range header: '" - << std::string(hdr_data, frag_size) - << "'. Ignoring." - << LL_ENDL; - } + // Detect and parse 'Retry-After' headers + if (is_header + && op->mProcFlags & PF_USE_RETRY_AFTER + && value && *value + && ! strcmp(name, con_retry_line)) + { + int time(0); + if (! parse_retry_after_header(value, &time)) + { + op->mReplyRetryAfter = time; } } @@ -794,14 +881,16 @@ int parse_content_range_header(char * buffer, unsigned int * last, unsigned int * length) { + static const char * const hdr_whitespace(" \t"); + char * tok_state(NULL), * tok(NULL); bool match(true); - if (! os_strtok_r(buffer, hdr_separator, &tok_state)) + if (! (tok = os_strtok_r(buffer, hdr_whitespace, &tok_state))) match = false; - if (match && (tok = os_strtok_r(NULL, hdr_whitespace, &tok_state))) - match = 0 == os_strcasecmp("bytes", tok); - if (match && ! (tok = os_strtok_r(NULL, " \t", &tok_state))) + else + match = (0 == os_strcasecmp("bytes", tok)); + if (match && ! (tok = os_strtok_r(NULL, hdr_whitespace, &tok_state))) match = false; if (match) { @@ -840,6 +929,25 @@ int parse_content_range_header(char * buffer, } +int parse_retry_after_header(char * buffer, int * time) +{ + char * endptr(buffer); + long lcl_time(strtol(buffer, &endptr, 10)); + if (*endptr == '\0' && endptr != buffer && lcl_time > 0) + { + *time = lcl_time; + return 0; + } + + // Could attempt to parse HTTP time here but we're not really + // interested in it. Scheduling based on wallclock time on + // user hardware will lead to tears. + + // Header is there but badly/unexpectedly formed, try to ignore it. + return 1; +} + + void escape_libcurl_debug_data(char * buffer, size_t len, bool scrub, std::string & safe_line) { std::string out; @@ -876,15 +984,6 @@ void escape_libcurl_debug_data(char * buffer, size_t len, bool scrub, std::strin } -int os_strncasecmp(const char *s1, const char *s2, size_t n) -{ -#if LL_WINDOWS - return _strnicmp(s1, s2, n); -#else - return strncasecmp(s1, s2, n); -#endif // LL_WINDOWS -} - int os_strcasecmp(const char *s1, const char *s2) { @@ -906,6 +1005,47 @@ char * os_strtok_r(char *str, const char *delim, char ** savestate) } +void os_strlower(char * str) +{ + for (char c(0); (c = *str); ++str) + { + *str = tolower(c); + } +} + + +char * os_strtrim(char * lstr) +{ + while (' ' == *lstr || '\t' == *lstr) + { + ++lstr; + } + if (*lstr) + { + char * rstr(lstr + strlen(lstr)); + while (lstr < rstr && *--rstr) + { + if (' ' == *rstr || '\t' == *rstr) + { + *rstr = '\0'; + } + } + llassert(lstr <= rstr); + } + return lstr; +} + + +char * os_strltrim(char * lstr) +{ + while (' ' == *lstr || '\t' == *lstr) + { + ++lstr; + } + return lstr; +} + + } // end anonymous namespace diff --git a/indra/llcorehttp/_httpoprequest.h b/indra/llcorehttp/_httpoprequest.h index 7b65d17783..2e737cf1cc 100755 --- a/indra/llcorehttp/_httpoprequest.h +++ b/indra/llcorehttp/_httpoprequest.h @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -158,6 +158,7 @@ protected: unsigned int mProcFlags; static const unsigned int PF_SCAN_RANGE_HEADER = 0x00000001U; static const unsigned int PF_SAVE_HEADERS = 0x00000002U; + static const unsigned int PF_USE_RETRY_AFTER = 0x00000004U; public: // Request data @@ -175,6 +176,8 @@ public: HttpService * mCurlService; curl_slist * mCurlHeaders; size_t mCurlBodyPos; + char * mCurlTemp; // Scratch buffer for header processing + size_t mCurlTempLen; // Result data HttpStatus mStatus; @@ -184,9 +187,11 @@ public: size_t mReplyFullLength; HttpHeaders * mReplyHeaders; std::string mReplyConType; + int mReplyRetryAfter; // Policy data int mPolicyRetries; + int mPolicy503Retries; HttpTime mPolicyRetryAt; int mPolicyRetryLimit; }; // end class HttpOpRequest diff --git a/indra/llcorehttp/_httpopsetget.cpp b/indra/llcorehttp/_httpopsetget.cpp index 8198528a9b..a5363f9170 100755 --- a/indra/llcorehttp/_httpopsetget.cpp +++ b/indra/llcorehttp/_httpopsetget.cpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -27,6 +27,7 @@ #include "_httpopsetget.h" #include "httpcommon.h" +#include "httprequest.h" #include "_httpservice.h" #include "_httppolicy.h" @@ -43,10 +44,11 @@ namespace LLCore HttpOpSetGet::HttpOpSetGet() : HttpOperation(), - mIsGlobal(false), - mDoSet(false), - mSetting(-1), // Nothing requested - mLongValue(0L) + mReqOption(HttpRequest::PO_CONNECTION_LIMIT), + mReqClass(HttpRequest::INVALID_POLICY_ID), + mReqDoSet(false), + mReqLongValue(0L), + mReplyLongValue(0L) {} @@ -54,37 +56,84 @@ HttpOpSetGet::~HttpOpSetGet() {} -void HttpOpSetGet::setupGet(HttpRequest::EGlobalPolicy setting) +HttpStatus HttpOpSetGet::setupGet(HttpRequest::EPolicyOption opt, HttpRequest::policy_t pclass) { - mIsGlobal = true; - mSetting = setting; + HttpStatus status; + + mReqOption = opt; + mReqClass = pclass; + return status; } -void HttpOpSetGet::setupSet(HttpRequest::EGlobalPolicy setting, const std::string & value) +HttpStatus HttpOpSetGet::setupSet(HttpRequest::EPolicyOption opt, HttpRequest::policy_t pclass, long value) { - mIsGlobal = true; - mDoSet = true; - mSetting = setting; - mStrValue = value; + HttpStatus status; + + if (! HttpService::sOptionDesc[opt].mIsLong) + { + return HttpStatus(HttpStatus::LLCORE, HE_INVALID_ARG); + } + if (! HttpService::sOptionDesc[opt].mIsDynamic) + { + return HttpStatus(HttpStatus::LLCORE, HE_OPT_NOT_DYNAMIC); + } + + mReqOption = opt; + mReqClass = pclass; + mReqDoSet = true; + mReqLongValue = value; + + return status; } -void HttpOpSetGet::stageFromRequest(HttpService * service) +HttpStatus HttpOpSetGet::setupSet(HttpRequest::EPolicyOption opt, HttpRequest::policy_t pclass, const std::string & value) { - HttpPolicyGlobal & pol_opt(service->getPolicy().getGlobalOptions()); - HttpRequest::EGlobalPolicy setting(static_cast<HttpRequest::EGlobalPolicy>(mSetting)); + HttpStatus status; + + if (HttpService::sOptionDesc[opt].mIsLong) + { + return HttpStatus(HttpStatus::LLCORE, HE_INVALID_ARG); + } + if (! HttpService::sOptionDesc[opt].mIsDynamic) + { + return HttpStatus(HttpStatus::LLCORE, HE_OPT_NOT_DYNAMIC); + } + + mReqOption = opt; + mReqClass = pclass; + mReqDoSet = true; + mReqStrValue = value; - if (mDoSet) + return status; +} + + +void HttpOpSetGet::stageFromRequest(HttpService * service) +{ + if (mReqDoSet) { - mStatus = pol_opt.set(setting, mStrValue); + if (HttpService::sOptionDesc[mReqOption].mIsLong) + { + mStatus = service->setPolicyOption(mReqOption, mReqClass, + mReqLongValue, &mReplyLongValue); + } + else + { + mStatus = service->setPolicyOption(mReqOption, mReqClass, + mReqStrValue, &mReplyStrValue); + } } - if (mStatus) + else { - const std::string * value(NULL); - if ((mStatus = pol_opt.get(setting, &value))) + if (HttpService::sOptionDesc[mReqOption].mIsLong) + { + mStatus = service->getPolicyOption(mReqOption, mReqClass, &mReplyLongValue); + } + else { - mStrValue = *value; + mStatus = service->getPolicyOption(mReqOption, mReqClass, &mReplyStrValue); } } diff --git a/indra/llcorehttp/_httpopsetget.h b/indra/llcorehttp/_httpopsetget.h index 6966b9d94e..a1e76dd429 100755 --- a/indra/llcorehttp/_httpopsetget.h +++ b/indra/llcorehttp/_httpopsetget.h @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -46,7 +46,10 @@ namespace LLCore /// configuration settings. /// /// *NOTE: Expect this to change. Don't really like it yet. - +/// +/// *TODO: Can't return values to caller yet. Need to do +/// something better with HttpResponse and visitNotifier(). +/// class HttpOpSetGet : public HttpOperation { public: @@ -61,19 +64,23 @@ private: public: /// Threading: called by application thread - void setupGet(HttpRequest::EGlobalPolicy setting); - void setupSet(HttpRequest::EGlobalPolicy setting, const std::string & value); + HttpStatus setupGet(HttpRequest::EPolicyOption opt, HttpRequest::policy_t pclass); + HttpStatus setupSet(HttpRequest::EPolicyOption opt, HttpRequest::policy_t pclass, long value); + HttpStatus setupSet(HttpRequest::EPolicyOption opt, HttpRequest::policy_t pclass, const std::string & value); virtual void stageFromRequest(HttpService *); public: // Request data - bool mIsGlobal; - bool mDoSet; - int mSetting; - long mLongValue; - std::string mStrValue; - + HttpRequest::EPolicyOption mReqOption; + HttpRequest::policy_t mReqClass; + bool mReqDoSet; + long mReqLongValue; + std::string mReqStrValue; + + // Reply Data + long mReplyLongValue; + std::string mReplyStrValue; }; // end class HttpOpSetGet diff --git a/indra/llcorehttp/_httppolicy.cpp b/indra/llcorehttp/_httppolicy.cpp index 76c1e22431..ac79a77659 100755 --- a/indra/llcorehttp/_httppolicy.cpp +++ b/indra/llcorehttp/_httppolicy.cpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -41,57 +41,70 @@ namespace LLCore // Per-policy-class data for a running system. -// Collection of queues, parameters, history, metrics, etc. +// Collection of queues, options and other data // for a single policy class. // // Threading: accessed only by worker thread -struct HttpPolicy::State +struct HttpPolicy::ClassState { public: - State() - : mConnMax(HTTP_CONNECTION_LIMIT_DEFAULT), - mConnAt(HTTP_CONNECTION_LIMIT_DEFAULT), - mConnMin(1), - mNextSample(0), - mErrorCount(0), - mErrorFactor(0) + ClassState() + : mThrottleEnd(0), + mThrottleLeft(0L), + mRequestCount(0L) {} HttpReadyQueue mReadyQueue; HttpRetryQueue mRetryQueue; HttpPolicyClass mOptions; - - long mConnMax; - long mConnAt; - long mConnMin; - - HttpTime mNextSample; - unsigned long mErrorCount; - unsigned long mErrorFactor; + HttpTime mThrottleEnd; + long mThrottleLeft; + long mRequestCount; }; HttpPolicy::HttpPolicy(HttpService * service) - : mActiveClasses(0), - mState(NULL), - mService(service) -{} + : mService(service) +{ + // Create default class + mClasses.push_back(new ClassState()); +} HttpPolicy::~HttpPolicy() { shutdown(); + + for (class_list_t::iterator it(mClasses.begin()); it != mClasses.end(); ++it) + { + delete (*it); + } + mClasses.clear(); mService = NULL; } +HttpRequest::policy_t HttpPolicy::createPolicyClass() +{ + const HttpRequest::policy_t policy_class(mClasses.size()); + if (policy_class >= HTTP_POLICY_CLASS_LIMIT) + { + return HttpRequest::INVALID_POLICY_ID; + } + mClasses.push_back(new ClassState()); + return policy_class; +} + + void HttpPolicy::shutdown() { - for (int policy_class(0); policy_class < mActiveClasses; ++policy_class) + for (int policy_class(0); policy_class < mClasses.size(); ++policy_class) { - HttpRetryQueue & retryq(mState[policy_class].mRetryQueue); + ClassState & state(*mClasses[policy_class]); + + HttpRetryQueue & retryq(state.mRetryQueue); while (! retryq.empty()) { HttpOpRequest * op(retryq.top()); @@ -101,7 +114,7 @@ void HttpPolicy::shutdown() op->release(); } - HttpReadyQueue & readyq(mState[policy_class].mReadyQueue); + HttpReadyQueue & readyq(state.mReadyQueue); while (! readyq.empty()) { HttpOpRequest * op(readyq.top()); @@ -111,28 +124,11 @@ void HttpPolicy::shutdown() op->release(); } } - delete [] mState; - mState = NULL; - mActiveClasses = 0; } -void HttpPolicy::start(const HttpPolicyGlobal & global, - const std::vector<HttpPolicyClass> & classes) -{ - llassert_always(! mState); - - mGlobalOptions = global; - mActiveClasses = classes.size(); - mState = new State [mActiveClasses]; - for (int i(0); i < mActiveClasses; ++i) - { - mState[i].mOptions = classes[i]; - mState[i].mConnMax = classes[i].mConnectionLimit; - mState[i].mConnAt = mState[i].mConnMax; - mState[i].mConnMin = 2; - } -} +void HttpPolicy::start() +{} void HttpPolicy::addOp(HttpOpRequest * op) @@ -140,7 +136,8 @@ void HttpPolicy::addOp(HttpOpRequest * op) const int policy_class(op->mReqPolicy); op->mPolicyRetries = 0; - mState[policy_class].mReadyQueue.push(op); + op->mPolicy503Retries = 0; + mClasses[policy_class]->mReadyQueue.push(op); } @@ -155,25 +152,39 @@ void HttpPolicy::retryOp(HttpOpRequest * op) 5000000 // ... to every 5.0 S. }; static const int delta_max(int(LL_ARRAY_SIZE(retry_deltas)) - 1); - + static const HttpStatus error_503(503); + const HttpTime now(totalTime()); const int policy_class(op->mReqPolicy); - - const HttpTime delta(retry_deltas[llclamp(op->mPolicyRetries, 0, delta_max)]); + HttpTime delta(retry_deltas[llclamp(op->mPolicyRetries, 0, delta_max)]); + bool external_delta(false); + + if (op->mReplyRetryAfter > 0 && op->mReplyRetryAfter < 30) + { + delta = op->mReplyRetryAfter * U64L(1000000); + external_delta = true; + } op->mPolicyRetryAt = now + delta; ++op->mPolicyRetries; - LL_WARNS("CoreHttp") << "HTTP request " << static_cast<HttpHandle>(op) - << " retry " << op->mPolicyRetries - << " scheduled for +" << (delta / HttpTime(1000)) - << " mS. Status: " << op->mStatus.toHex() - << LL_ENDL; - if (op->mTracing > 0) + if (error_503 == op->mStatus) + { + ++op->mPolicy503Retries; + } + LL_DEBUGS("CoreHttp") << "HTTP request " << static_cast<HttpHandle>(op) + << " retry " << op->mPolicyRetries + << " scheduled in " << (delta / HttpTime(1000)) + << " mS (" << (external_delta ? "external" : "internal") + << "). Status: " << op->mStatus.toHex() + << LL_ENDL; + if (op->mTracing > HTTP_TRACE_OFF) { LL_INFOS("CoreHttp") << "TRACE, ToRetryQueue, Handle: " << static_cast<HttpHandle>(op) + << ", Delta: " << (delta / HttpTime(1000)) + << ", Retries: " << op->mPolicyRetries << LL_ENDL; } - mState[policy_class].mRetryQueue.push(op); + mClasses[policy_class]->mRetryQueue.push(op); } @@ -188,21 +199,43 @@ void HttpPolicy::retryOp(HttpOpRequest * op) // the worker thread may sleep hard otherwise will ask for // normal polling frequency. // +// Implements a client-side request rate throttle as well. +// This is intended to mimic and predict throttling behavior +// of grid services but that is difficult to do with different +// time bases. This also represents a rigid coupling between +// viewer and server that makes it hard to change parameters +// and I hope we can make this go away with pipelining. +// HttpService::ELoopSpeed HttpPolicy::processReadyQueue() { const HttpTime now(totalTime()); HttpService::ELoopSpeed result(HttpService::REQUEST_SLEEP); HttpLibcurl & transport(mService->getTransport()); - for (int policy_class(0); policy_class < mActiveClasses; ++policy_class) + for (int policy_class(0); policy_class < mClasses.size(); ++policy_class) { - State & state(mState[policy_class]); - int active(transport.getActiveCountInClass(policy_class)); - int needed(state.mConnAt - active); // Expect negatives here - + ClassState & state(*mClasses[policy_class]); HttpRetryQueue & retryq(state.mRetryQueue); HttpReadyQueue & readyq(state.mReadyQueue); + + if (retryq.empty() && readyq.empty()) + { + continue; + } + const bool throttle_enabled(state.mOptions.mThrottleRate > 0L); + const bool throttle_current(throttle_enabled && now < state.mThrottleEnd); + + if (throttle_current && state.mThrottleLeft <= 0) + { + // Throttled condition, don't serve this class but don't sleep hard. + result = HttpService::NORMAL; + continue; + } + + int active(transport.getActiveCountInClass(policy_class)); + int needed(state.mOptions.mConnectionLimit - active); // Expect negatives here + if (needed > 0) { // First see if we have any retries... @@ -216,10 +249,27 @@ HttpService::ELoopSpeed HttpPolicy::processReadyQueue() op->stageFromReady(mService); op->release(); - + + ++state.mRequestCount; --needed; + if (throttle_enabled) + { + if (now >= state.mThrottleEnd) + { + // Throttle expired, move to next window + LL_DEBUGS("CoreHttp") << "Throttle expired with " << state.mThrottleLeft + << " requests to go and " << state.mRequestCount + << " requests issued." << LL_ENDL; + state.mThrottleLeft = state.mOptions.mThrottleRate; + state.mThrottleEnd = now + HttpTime(1000000); + } + if (--state.mThrottleLeft <= 0) + { + goto throttle_on; + } + } } - + // Now go on to the new requests... while (needed > 0 && ! readyq.empty()) { @@ -229,10 +279,29 @@ HttpService::ELoopSpeed HttpPolicy::processReadyQueue() op->stageFromReady(mService); op->release(); + ++state.mRequestCount; --needed; + if (throttle_enabled) + { + if (now >= state.mThrottleEnd) + { + // Throttle expired, move to next window + LL_DEBUGS("CoreHttp") << "Throttle expired with " << state.mThrottleLeft + << " requests to go and " << state.mRequestCount + << " requests issued." << LL_ENDL; + state.mThrottleLeft = state.mOptions.mThrottleRate; + state.mThrottleEnd = now + HttpTime(1000000); + } + if (--state.mThrottleLeft <= 0) + { + goto throttle_on; + } + } } } - + + throttle_on: + if (! readyq.empty() || ! retryq.empty()) { // If anything is ready, continue looping... @@ -246,9 +315,9 @@ HttpService::ELoopSpeed HttpPolicy::processReadyQueue() bool HttpPolicy::changePriority(HttpHandle handle, HttpRequest::priority_t priority) { - for (int policy_class(0); policy_class < mActiveClasses; ++policy_class) + for (int policy_class(0); policy_class < mClasses.size(); ++policy_class) { - State & state(mState[policy_class]); + ClassState & state(*mClasses[policy_class]); // We don't scan retry queue because a priority change there // is meaningless. The request will be issued based on retry // intervals not priority value, which is now moot. @@ -276,9 +345,9 @@ bool HttpPolicy::changePriority(HttpHandle handle, HttpRequest::priority_t prior bool HttpPolicy::cancel(HttpHandle handle) { - for (int policy_class(0); policy_class < mActiveClasses; ++policy_class) + for (int policy_class(0); policy_class < mClasses.size(); ++policy_class) { - State & state(mState[policy_class]); + ClassState & state(*mClasses[policy_class]); // Scan retry queue HttpRetryQueue::container_type & c1(state.mRetryQueue.get_container()); @@ -362,9 +431,9 @@ bool HttpPolicy::stageAfterCompletion(HttpOpRequest * op) } else if (op->mPolicyRetries) { - LL_WARNS("CoreHttp") << "HTTP request " << static_cast<HttpHandle>(op) - << " succeeded on retry " << op->mPolicyRetries << "." - << LL_ENDL; + LL_DEBUGS("CoreHttp") << "HTTP request " << static_cast<HttpHandle>(op) + << " succeeded on retry " << op->mPolicyRetries << "." + << LL_ENDL; } op->stageFromActive(mService); @@ -372,13 +441,21 @@ bool HttpPolicy::stageAfterCompletion(HttpOpRequest * op) return false; // not active } + +HttpPolicyClass & HttpPolicy::getClassOptions(HttpRequest::policy_t pclass) +{ + llassert_always(pclass >= 0 && pclass < mClasses.size()); + + return mClasses[pclass]->mOptions; +} + int HttpPolicy::getReadyCount(HttpRequest::policy_t policy_class) const { - if (policy_class < mActiveClasses) + if (policy_class < mClasses.size()) { - return (mState[policy_class].mReadyQueue.size() - + mState[policy_class].mRetryQueue.size()); + return (mClasses[policy_class]->mReadyQueue.size() + + mClasses[policy_class]->mRetryQueue.size()); } return 0; } diff --git a/indra/llcorehttp/_httppolicy.h b/indra/llcorehttp/_httppolicy.h index 03d92c0b8e..bf1aa74267 100755 --- a/indra/llcorehttp/_httppolicy.h +++ b/indra/llcorehttp/_httppolicy.h @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -60,6 +60,9 @@ private: void operator=(const HttpPolicy &); // Not defined public: + /// Threading: called by init thread. + HttpRequest::policy_t createPolicyClass(); + /// Cancel all ready and retry requests sending them to /// their notification queues. Release state resources /// making further request handling impossible. @@ -71,9 +74,8 @@ public: /// requests. One-time call invoked before starting /// the worker thread. /// - /// Threading: called by application thread - void start(const HttpPolicyGlobal & global, - const std::vector<HttpPolicyClass> & classes); + /// Threading: called by init thread + void start(); /// Give the policy layer some cycles to scan the ready /// queue promoting higher-priority requests to active @@ -93,7 +95,7 @@ public: /// and should not be modified by anyone until retrieved /// from queue. /// - /// Threading: called by any thread + /// Threading: called by worker thread void addOp(HttpOpRequest *); /// Similar to addOp, used when a caller wants to retry a @@ -130,30 +132,39 @@ public: /// Threading: called by worker thread bool stageAfterCompletion(HttpOpRequest * op); - // Get pointer to global policy options. Caller is expected - // to do context checks like no setting once running. + /// Get a reference to global policy options. Caller is expected + /// to do context checks like no setting once running. These + /// are done, for example, in @see HttpService interfaces. /// /// Threading: called by any thread *but* the object may /// only be modified by the worker thread once running. - /// HttpPolicyGlobal & getGlobalOptions() { return mGlobalOptions; } + /// Get a reference to class policy options. Caller is expected + /// to do context checks like no setting once running. These + /// are done, for example, in @see HttpService interfaces. + /// + /// Threading: called by any thread *but* the object may + /// only be modified by the worker thread once running and + /// read accesses by other threads are exposed to races at + /// that point. + HttpPolicyClass & getClassOptions(HttpRequest::policy_t pclass); + /// Get ready counts for a particular policy class /// /// Threading: called by worker thread int getReadyCount(HttpRequest::policy_t policy_class) const; protected: - struct State; - - int mActiveClasses; - State * mState; - HttpService * mService; // Naked pointer, not refcounted, not owner - HttpPolicyGlobal mGlobalOptions; + struct ClassState; + typedef std::vector<ClassState *> class_list_t; + HttpPolicyGlobal mGlobalOptions; + class_list_t mClasses; + HttpService * mService; // Naked pointer, not refcounted, not owner }; // end class HttpPolicy } // end namespace LLCore diff --git a/indra/llcorehttp/_httppolicyclass.cpp b/indra/llcorehttp/_httppolicyclass.cpp index a23b81322c..f34a8e9f1e 100755 --- a/indra/llcorehttp/_httppolicyclass.cpp +++ b/indra/llcorehttp/_httppolicyclass.cpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -34,10 +34,10 @@ namespace LLCore HttpPolicyClass::HttpPolicyClass() - : mSetMask(0UL), - mConnectionLimit(HTTP_CONNECTION_LIMIT_DEFAULT), + : mConnectionLimit(HTTP_CONNECTION_LIMIT_DEFAULT), mPerHostConnectionLimit(HTTP_CONNECTION_LIMIT_DEFAULT), - mPipelining(0) + mPipelining(HTTP_PIPELINING_DEFAULT), + mThrottleRate(HTTP_THROTTLE_RATE_DEFAULT) {} @@ -49,75 +49,75 @@ HttpPolicyClass & HttpPolicyClass::operator=(const HttpPolicyClass & other) { if (this != &other) { - mSetMask = other.mSetMask; mConnectionLimit = other.mConnectionLimit; mPerHostConnectionLimit = other.mPerHostConnectionLimit; mPipelining = other.mPipelining; + mThrottleRate = other.mThrottleRate; } return *this; } HttpPolicyClass::HttpPolicyClass(const HttpPolicyClass & other) - : mSetMask(other.mSetMask), - mConnectionLimit(other.mConnectionLimit), + : mConnectionLimit(other.mConnectionLimit), mPerHostConnectionLimit(other.mPerHostConnectionLimit), - mPipelining(other.mPipelining) + mPipelining(other.mPipelining), + mThrottleRate(other.mThrottleRate) {} -HttpStatus HttpPolicyClass::set(HttpRequest::EClassPolicy opt, long value) +HttpStatus HttpPolicyClass::set(HttpRequest::EPolicyOption opt, long value) { switch (opt) { - case HttpRequest::CP_CONNECTION_LIMIT: + case HttpRequest::PO_CONNECTION_LIMIT: mConnectionLimit = llclamp(value, long(HTTP_CONNECTION_LIMIT_MIN), long(HTTP_CONNECTION_LIMIT_MAX)); break; - case HttpRequest::CP_PER_HOST_CONNECTION_LIMIT: + case HttpRequest::PO_PER_HOST_CONNECTION_LIMIT: mPerHostConnectionLimit = llclamp(value, long(HTTP_CONNECTION_LIMIT_MIN), mConnectionLimit); break; - case HttpRequest::CP_ENABLE_PIPELINING: + case HttpRequest::PO_ENABLE_PIPELINING: mPipelining = llclamp(value, 0L, 1L); break; + case HttpRequest::PO_THROTTLE_RATE: + mThrottleRate = llclamp(value, 0L, 1000000L); + break; + default: return HttpStatus(HttpStatus::LLCORE, HE_INVALID_ARG); } - mSetMask |= 1UL << int(opt); return HttpStatus(); } -HttpStatus HttpPolicyClass::get(HttpRequest::EClassPolicy opt, long * value) +HttpStatus HttpPolicyClass::get(HttpRequest::EPolicyOption opt, long * value) const { - static const HttpStatus not_set(HttpStatus::LLCORE, HE_OPT_NOT_SET); - long * src(NULL); - switch (opt) { - case HttpRequest::CP_CONNECTION_LIMIT: - src = &mConnectionLimit; + case HttpRequest::PO_CONNECTION_LIMIT: + *value = mConnectionLimit; break; - case HttpRequest::CP_PER_HOST_CONNECTION_LIMIT: - src = &mPerHostConnectionLimit; + case HttpRequest::PO_PER_HOST_CONNECTION_LIMIT: + *value = mPerHostConnectionLimit; break; - case HttpRequest::CP_ENABLE_PIPELINING: - src = &mPipelining; + case HttpRequest::PO_ENABLE_PIPELINING: + *value = mPipelining; + break; + + case HttpRequest::PO_THROTTLE_RATE: + *value = mThrottleRate; break; default: return HttpStatus(HttpStatus::LLCORE, HE_INVALID_ARG); } - if (! (mSetMask & (1UL << int(opt)))) - return not_set; - - *value = *src; return HttpStatus(); } diff --git a/indra/llcorehttp/_httppolicyclass.h b/indra/llcorehttp/_httppolicyclass.h index d175413cbd..38f1194ded 100755 --- a/indra/llcorehttp/_httppolicyclass.h +++ b/indra/llcorehttp/_httppolicyclass.h @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -34,6 +34,18 @@ namespace LLCore { +/// Options struct for per-class policy options. +/// +/// Combines both raw blob data access with semantics-enforcing +/// set/get interfaces. For internal operations by the worker +/// thread, just grab the setting directly from instance and test/use +/// as needed. When attached to external APIs (the public API +/// options interfaces) the set/get methods are available to +/// enforce correct ranges, data types, contexts, etc. and suitable +/// status values are returned. +/// +/// Threading: Single-threaded. In practice, init thread before +/// worker starts, worker thread after. class HttpPolicyClass { public: @@ -44,14 +56,14 @@ public: HttpPolicyClass(const HttpPolicyClass &); // Not defined public: - HttpStatus set(HttpRequest::EClassPolicy opt, long value); - HttpStatus get(HttpRequest::EClassPolicy opt, long * value); + HttpStatus set(HttpRequest::EPolicyOption opt, long value); + HttpStatus get(HttpRequest::EPolicyOption opt, long * value) const; public: - unsigned long mSetMask; long mConnectionLimit; long mPerHostConnectionLimit; long mPipelining; + long mThrottleRate; }; // end class HttpPolicyClass } // end namespace LLCore diff --git a/indra/llcorehttp/_httppolicyglobal.cpp b/indra/llcorehttp/_httppolicyglobal.cpp index 72f409d3b1..1dc95f3dce 100755 --- a/indra/llcorehttp/_httppolicyglobal.cpp +++ b/indra/llcorehttp/_httppolicyglobal.cpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -34,8 +34,7 @@ namespace LLCore HttpPolicyGlobal::HttpPolicyGlobal() - : mSetMask(0UL), - mConnectionLimit(HTTP_CONNECTION_LIMIT_DEFAULT), + : mConnectionLimit(HTTP_CONNECTION_LIMIT_DEFAULT), mTrace(HTTP_TRACE_OFF), mUseLLProxy(0) {} @@ -49,7 +48,6 @@ HttpPolicyGlobal & HttpPolicyGlobal::operator=(const HttpPolicyGlobal & other) { if (this != &other) { - mSetMask = other.mSetMask; mConnectionLimit = other.mConnectionLimit; mCAPath = other.mCAPath; mCAFile = other.mCAFile; @@ -61,19 +59,19 @@ HttpPolicyGlobal & HttpPolicyGlobal::operator=(const HttpPolicyGlobal & other) } -HttpStatus HttpPolicyGlobal::set(HttpRequest::EGlobalPolicy opt, long value) +HttpStatus HttpPolicyGlobal::set(HttpRequest::EPolicyOption opt, long value) { switch (opt) { - case HttpRequest::GP_CONNECTION_LIMIT: + case HttpRequest::PO_CONNECTION_LIMIT: mConnectionLimit = llclamp(value, long(HTTP_CONNECTION_LIMIT_MIN), long(HTTP_CONNECTION_LIMIT_MAX)); break; - case HttpRequest::GP_TRACE: + case HttpRequest::PO_TRACE: mTrace = llclamp(value, long(HTTP_TRACE_MIN), long(HTTP_TRACE_MAX)); break; - case HttpRequest::GP_LLPROXY: + case HttpRequest::PO_LLPROXY: mUseLLProxy = llclamp(value, 0L, 1L); break; @@ -81,24 +79,23 @@ HttpStatus HttpPolicyGlobal::set(HttpRequest::EGlobalPolicy opt, long value) return HttpStatus(HttpStatus::LLCORE, HE_INVALID_ARG); } - mSetMask |= 1UL << int(opt); return HttpStatus(); } -HttpStatus HttpPolicyGlobal::set(HttpRequest::EGlobalPolicy opt, const std::string & value) +HttpStatus HttpPolicyGlobal::set(HttpRequest::EPolicyOption opt, const std::string & value) { switch (opt) { - case HttpRequest::GP_CA_PATH: + case HttpRequest::PO_CA_PATH: mCAPath = value; break; - case HttpRequest::GP_CA_FILE: + case HttpRequest::PO_CA_FILE: mCAFile = value; break; - case HttpRequest::GP_HTTP_PROXY: + case HttpRequest::PO_HTTP_PROXY: mCAFile = value; break; @@ -106,69 +103,54 @@ HttpStatus HttpPolicyGlobal::set(HttpRequest::EGlobalPolicy opt, const std::stri return HttpStatus(HttpStatus::LLCORE, HE_INVALID_ARG); } - mSetMask |= 1UL << int(opt); return HttpStatus(); } -HttpStatus HttpPolicyGlobal::get(HttpRequest::EGlobalPolicy opt, long * value) +HttpStatus HttpPolicyGlobal::get(HttpRequest::EPolicyOption opt, long * value) const { - static const HttpStatus not_set(HttpStatus::LLCORE, HE_OPT_NOT_SET); - long * src(NULL); - switch (opt) { - case HttpRequest::GP_CONNECTION_LIMIT: - src = &mConnectionLimit; + case HttpRequest::PO_CONNECTION_LIMIT: + *value = mConnectionLimit; break; - case HttpRequest::GP_TRACE: - src = &mTrace; + case HttpRequest::PO_TRACE: + *value = mTrace; break; - case HttpRequest::GP_LLPROXY: - src = &mUseLLProxy; + case HttpRequest::PO_LLPROXY: + *value = mUseLLProxy; break; default: return HttpStatus(HttpStatus::LLCORE, HE_INVALID_ARG); } - if (! (mSetMask & (1UL << int(opt)))) - return not_set; - - *value = *src; return HttpStatus(); } -HttpStatus HttpPolicyGlobal::get(HttpRequest::EGlobalPolicy opt, const std::string ** value) +HttpStatus HttpPolicyGlobal::get(HttpRequest::EPolicyOption opt, std::string * value) const { - static const HttpStatus not_set(HttpStatus::LLCORE, HE_OPT_NOT_SET); - const std::string * src(NULL); - switch (opt) { - case HttpRequest::GP_CA_PATH: - src = &mCAPath; + case HttpRequest::PO_CA_PATH: + *value = mCAPath; break; - case HttpRequest::GP_CA_FILE: - src = &mCAFile; + case HttpRequest::PO_CA_FILE: + *value = mCAFile; break; - case HttpRequest::GP_HTTP_PROXY: - src = &mHttpProxy; + case HttpRequest::PO_HTTP_PROXY: + *value = mHttpProxy; break; default: return HttpStatus(HttpStatus::LLCORE, HE_INVALID_ARG); } - if (! (mSetMask & (1UL << int(opt)))) - return not_set; - - *value = src; return HttpStatus(); } diff --git a/indra/llcorehttp/_httppolicyglobal.h b/indra/llcorehttp/_httppolicyglobal.h index a50d0e4188..67c4ba9481 100755 --- a/indra/llcorehttp/_httppolicyglobal.h +++ b/indra/llcorehttp/_httppolicyglobal.h @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -34,6 +34,18 @@ namespace LLCore { +/// Options struct for global policy options. +/// +/// Combines both raw blob data access with semantics-enforcing +/// set/get interfaces. For internal operations by the worker +/// thread, just grab the setting directly from instance and test/use +/// as needed. When attached to external APIs (the public API +/// options interfaces) the set/get methods are available to +/// enforce correct ranges, data types, contexts, etc. and suitable +/// status values are returned. +/// +/// Threading: Single-threaded. In practice, init thread before +/// worker starts, worker thread after. class HttpPolicyGlobal { public: @@ -46,13 +58,12 @@ private: HttpPolicyGlobal(const HttpPolicyGlobal &); // Not defined public: - HttpStatus set(HttpRequest::EGlobalPolicy opt, long value); - HttpStatus set(HttpRequest::EGlobalPolicy opt, const std::string & value); - HttpStatus get(HttpRequest::EGlobalPolicy opt, long * value); - HttpStatus get(HttpRequest::EGlobalPolicy opt, const std::string ** value); + HttpStatus set(HttpRequest::EPolicyOption opt, long value); + HttpStatus set(HttpRequest::EPolicyOption opt, const std::string & value); + HttpStatus get(HttpRequest::EPolicyOption opt, long * value) const; + HttpStatus get(HttpRequest::EPolicyOption opt, std::string * value) const; public: - unsigned long mSetMask; long mConnectionLimit; std::string mCAPath; std::string mCAFile; diff --git a/indra/llcorehttp/_httpservice.cpp b/indra/llcorehttp/_httpservice.cpp index 0825888d0f..c94249dc2d 100755 --- a/indra/llcorehttp/_httpservice.cpp +++ b/indra/llcorehttp/_httpservice.cpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -43,6 +43,18 @@ namespace LLCore { +const HttpService::OptionDescriptor HttpService::sOptionDesc[] = +{ // isLong isDynamic isGlobal isClass + { true, true, true, true }, // PO_CONNECTION_LIMIT + { true, true, false, true }, // PO_PER_HOST_CONNECTION_LIMIT + { false, false, true, false }, // PO_CA_PATH + { false, false, true, false }, // PO_CA_FILE + { false, true, true, false }, // PO_HTTP_PROXY + { true, true, true, false }, // PO_LLPROXY + { true, true, true, false }, // PO_TRACE + { true, true, false, true }, // PO_ENABLE_PIPELINING + { true, true, false, true } // PO_THROTTLE_RATE +}; HttpService * HttpService::sInstance(NULL); volatile HttpService::EState HttpService::sState(NOT_INITIALIZED); @@ -51,15 +63,9 @@ HttpService::HttpService() mExitRequested(0U), mThread(NULL), mPolicy(NULL), - mTransport(NULL) -{ - // Create the default policy class - HttpPolicyClass pol_class; - pol_class.set(HttpRequest::CP_CONNECTION_LIMIT, HTTP_CONNECTION_LIMIT_DEFAULT); - pol_class.set(HttpRequest::CP_PER_HOST_CONNECTION_LIMIT, HTTP_CONNECTION_LIMIT_DEFAULT); - pol_class.set(HttpRequest::CP_ENABLE_PIPELINING, 0L); - mPolicyClasses.push_back(pol_class); -} + mTransport(NULL), + mLastPolicy(0) +{} HttpService::~HttpService() @@ -149,13 +155,8 @@ void HttpService::term() HttpRequest::policy_t HttpService::createPolicyClass() { - const HttpRequest::policy_t policy_class(mPolicyClasses.size()); - if (policy_class >= HTTP_POLICY_CLASS_LIMIT) - { - return 0; - } - mPolicyClasses.push_back(HttpPolicyClass()); - return policy_class; + mLastPolicy = mPolicy->createPolicyClass(); + return mLastPolicy; } @@ -188,8 +189,8 @@ void HttpService::startThread() } // Push current policy definitions, enable policy & transport components - mPolicy->start(mPolicyGlobal, mPolicyClasses); - mTransport->start(mPolicyClasses.size()); + mPolicy->start(); + mTransport->start(mLastPolicy + 1); mThread = new LLCoreInt::HttpThread(boost::bind(&HttpService::threadRun, this, _1)); sState = RUNNING; @@ -322,7 +323,7 @@ HttpService::ELoopSpeed HttpService::processRequestQueue(ELoopSpeed loop) { // Setup for subsequent tracing long tracing(HTTP_TRACE_OFF); - mPolicy->getGlobalOptions().get(HttpRequest::GP_TRACE, &tracing); + mPolicy->getGlobalOptions().get(HttpRequest::PO_TRACE, &tracing); op->mTracing = (std::max)(op->mTracing, int(tracing)); if (op->mTracing > HTTP_TRACE_OFF) @@ -345,4 +346,137 @@ HttpService::ELoopSpeed HttpService::processRequestQueue(ELoopSpeed loop) } +HttpStatus HttpService::getPolicyOption(HttpRequest::EPolicyOption opt, HttpRequest::policy_t pclass, + long * ret_value) +{ + if (opt < HttpRequest::PO_CONNECTION_LIMIT // option must be in range + || opt >= HttpRequest::PO_LAST // ditto + || (! sOptionDesc[opt].mIsLong) // datatype is long + || (pclass != HttpRequest::GLOBAL_POLICY_ID && pclass > mLastPolicy) // pclass in valid range + || (pclass == HttpRequest::GLOBAL_POLICY_ID && ! sOptionDesc[opt].mIsGlobal) // global setting permitted + || (pclass != HttpRequest::GLOBAL_POLICY_ID && ! sOptionDesc[opt].mIsClass)) // class setting permitted + // can always get, no dynamic check + { + return HttpStatus(HttpStatus::LLCORE, LLCore::HE_INVALID_ARG); + } + + HttpStatus status; + if (pclass == HttpRequest::GLOBAL_POLICY_ID) + { + HttpPolicyGlobal & opts(mPolicy->getGlobalOptions()); + + status = opts.get(opt, ret_value); + } + else + { + HttpPolicyClass & opts(mPolicy->getClassOptions(pclass)); + + status = opts.get(opt, ret_value); + } + + return status; +} + + +HttpStatus HttpService::getPolicyOption(HttpRequest::EPolicyOption opt, HttpRequest::policy_t pclass, + std::string * ret_value) +{ + HttpStatus status(HttpStatus::LLCORE, LLCore::HE_INVALID_ARG); + + if (opt < HttpRequest::PO_CONNECTION_LIMIT // option must be in range + || opt >= HttpRequest::PO_LAST // ditto + || (sOptionDesc[opt].mIsLong) // datatype is string + || (pclass != HttpRequest::GLOBAL_POLICY_ID && pclass > mLastPolicy) // pclass in valid range + || (pclass == HttpRequest::GLOBAL_POLICY_ID && ! sOptionDesc[opt].mIsGlobal) // global setting permitted + || (pclass != HttpRequest::GLOBAL_POLICY_ID && ! sOptionDesc[opt].mIsClass)) // class setting permitted + // can always get, no dynamic check + { + return status; + } + + // Only global has string values + if (pclass == HttpRequest::GLOBAL_POLICY_ID) + { + HttpPolicyGlobal & opts(mPolicy->getGlobalOptions()); + + status = opts.get(opt, ret_value); + } + + return status; +} + + +HttpStatus HttpService::setPolicyOption(HttpRequest::EPolicyOption opt, HttpRequest::policy_t pclass, + long value, long * ret_value) +{ + HttpStatus status(HttpStatus::LLCORE, LLCore::HE_INVALID_ARG); + + if (opt < HttpRequest::PO_CONNECTION_LIMIT // option must be in range + || opt >= HttpRequest::PO_LAST // ditto + || (! sOptionDesc[opt].mIsLong) // datatype is long + || (pclass != HttpRequest::GLOBAL_POLICY_ID && pclass > mLastPolicy) // pclass in valid range + || (pclass == HttpRequest::GLOBAL_POLICY_ID && ! sOptionDesc[opt].mIsGlobal) // global setting permitted + || (pclass != HttpRequest::GLOBAL_POLICY_ID && ! sOptionDesc[opt].mIsClass) // class setting permitted + || (RUNNING == sState && ! sOptionDesc[opt].mIsDynamic)) // dynamic setting permitted + { + return status; + } + + if (pclass == HttpRequest::GLOBAL_POLICY_ID) + { + HttpPolicyGlobal & opts(mPolicy->getGlobalOptions()); + + status = opts.set(opt, value); + if (status && ret_value) + { + status = opts.get(opt, ret_value); + } + } + else + { + HttpPolicyClass & opts(mPolicy->getClassOptions(pclass)); + + status = opts.set(opt, value); + if (status && ret_value) + { + status = opts.get(opt, ret_value); + } + } + + return status; +} + + +HttpStatus HttpService::setPolicyOption(HttpRequest::EPolicyOption opt, HttpRequest::policy_t pclass, + const std::string & value, std::string * ret_value) +{ + HttpStatus status(HttpStatus::LLCORE, LLCore::HE_INVALID_ARG); + + if (opt < HttpRequest::PO_CONNECTION_LIMIT // option must be in range + || opt >= HttpRequest::PO_LAST // ditto + || (sOptionDesc[opt].mIsLong) // datatype is string + || (pclass != HttpRequest::GLOBAL_POLICY_ID && pclass > mLastPolicy) // pclass in valid range + || (pclass == HttpRequest::GLOBAL_POLICY_ID && ! sOptionDesc[opt].mIsGlobal) // global setting permitted + || (pclass != HttpRequest::GLOBAL_POLICY_ID && ! sOptionDesc[opt].mIsClass) // class setting permitted + || (RUNNING == sState && ! sOptionDesc[opt].mIsDynamic)) // dynamic setting permitted + { + return status; + } + + // Only string values are global at this time + if (pclass == HttpRequest::GLOBAL_POLICY_ID) + { + HttpPolicyGlobal & opts(mPolicy->getGlobalOptions()); + + status = opts.set(opt, value); + if (status && ret_value) + { + status = opts.get(opt, ret_value); + } + } + + return status; +} + + } // end namespace LLCore diff --git a/indra/llcorehttp/_httpservice.h b/indra/llcorehttp/_httpservice.h index ffe0349d4d..cf23f3ab61 100755 --- a/indra/llcorehttp/_httpservice.h +++ b/indra/llcorehttp/_httpservice.h @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -53,6 +53,7 @@ namespace LLCore class HttpRequestQueue; class HttpPolicy; class HttpLibcurl; +class HttpOpSetGet; /// The HttpService class does the work behind the request queue. It @@ -106,7 +107,7 @@ public: NORMAL, ///< continuous polling of request, ready, active queues REQUEST_SLEEP ///< can sleep indefinitely waiting for request queue write }; - + static void init(HttpRequestQueue *); static void term(); @@ -136,7 +137,7 @@ public: /// acquires its weaknesses. static bool isStopped(); - /// Threading: callable by consumer thread *once*. + /// Threading: callable by init thread *once*. void startThread(); /// Threading: callable by worker thread. @@ -181,27 +182,38 @@ public: } /// Threading: callable by consumer thread. - HttpPolicyGlobal & getGlobalOptions() - { - return mPolicyGlobal; - } - - /// Threading: callable by consumer thread. HttpRequest::policy_t createPolicyClass(); - /// Threading: callable by consumer thread. - HttpPolicyClass & getClassOptions(HttpRequest::policy_t policy_class) - { - llassert(policy_class >= 0 && policy_class < mPolicyClasses.size()); - return mPolicyClasses[policy_class]; - } - protected: void threadRun(LLCoreInt::HttpThread * thread); ELoopSpeed processRequestQueue(ELoopSpeed loop); + +protected: + friend class HttpOpSetGet; + friend class HttpRequest; + + // Used internally to describe what operations are allowed + // on each policy option. + struct OptionDescriptor + { + bool mIsLong; + bool mIsDynamic; + bool mIsGlobal; + bool mIsClass; + }; + + HttpStatus getPolicyOption(HttpRequest::EPolicyOption opt, HttpRequest::policy_t, + long * ret_value); + HttpStatus getPolicyOption(HttpRequest::EPolicyOption opt, HttpRequest::policy_t, + std::string * ret_value); + HttpStatus setPolicyOption(HttpRequest::EPolicyOption opt, HttpRequest::policy_t, + long value, long * ret_value); + HttpStatus setPolicyOption(HttpRequest::EPolicyOption opt, HttpRequest::policy_t, + const std::string & value, std::string * ret_value); protected: + static const OptionDescriptor sOptionDesc[HttpRequest::PO_LAST]; static HttpService * sInstance; // === shared data === @@ -210,13 +222,13 @@ protected: LLAtomicU32 mExitRequested; LLCoreInt::HttpThread * mThread; - // === consumer-thread-only data === - HttpPolicyGlobal mPolicyGlobal; - std::vector<HttpPolicyClass> mPolicyClasses; - // === working-thread-only data === HttpPolicy * mPolicy; // Simple pointer, has ownership HttpLibcurl * mTransport; // Simple pointer, has ownership + + // === main-thread-only data === + HttpRequest::policy_t mLastPolicy; + }; // end class HttpService } // end namespace LLCore diff --git a/indra/llcorehttp/examples/http_texture_load.cpp b/indra/llcorehttp/examples/http_texture_load.cpp index 40ad4f047d..73c49687d7 100755 --- a/indra/llcorehttp/examples/http_texture_load.cpp +++ b/indra/llcorehttp/examples/http_texture_load.cpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -39,6 +39,7 @@ #include "httprequest.h" #include "httphandler.h" #include "httpresponse.h" +#include "httpoptions.h" #include "httpheaders.h" #include "bufferarray.h" #include "_mutex.h" @@ -57,6 +58,7 @@ void usage(std::ostream & out); // Default command line settings static int concurrency_limit(40); +static int highwater(100); static char url_format[1024] = "http://example.com/some/path?texture_id=%s.texture"; #if defined(WIN32) @@ -79,11 +81,11 @@ public: WorkingSet(); ~WorkingSet(); - bool reload(LLCore::HttpRequest *); + bool reload(LLCore::HttpRequest *, LLCore::HttpOptions *); virtual void onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * response); - void loadTextureUuids(FILE * in); + void loadAssetUuids(FILE * in); public: struct Spec @@ -93,24 +95,27 @@ public: int mLength; }; typedef std::set<LLCore::HttpHandle> handle_set_t; - typedef std::vector<Spec> texture_list_t; + typedef std::vector<Spec> asset_list_t; public: bool mVerbose; bool mRandomRange; - int mMaxConcurrency; + int mRequestLowWater; + int mRequestHighWater; handle_set_t mHandles; int mRemaining; int mLimit; int mAt; std::string mUrl; - texture_list_t mTextures; + asset_list_t mAssets; int mErrorsApi; int mErrorsHttp; int mErrorsHttp404; int mErrorsHttp416; int mErrorsHttp500; int mErrorsHttp503; + int mRetries; + int mRetriesHttp503; int mSuccesses; long mByteCount; LLCore::HttpHeaders * mHeaders; @@ -158,7 +163,7 @@ int main(int argc, char** argv) bool do_verbose(false); int option(-1); - while (-1 != (option = getopt(argc, argv, "u:c:h?Rv"))) + while (-1 != (option = getopt(argc, argv, "u:c:h?RvH:"))) { switch (option) { @@ -182,6 +187,21 @@ int main(int argc, char** argv) } break; + case 'H': + { + unsigned long value; + char * end; + + value = strtoul(optarg, &end, 10); + if (value < 1 || value > 100 || *end != '\0') + { + usage(std::cerr); + return 1; + } + highwater = value; + } + break; + case 'R': do_random = true; break; @@ -216,25 +236,32 @@ int main(int argc, char** argv) // Initialization init_curl(); LLCore::HttpRequest::createService(); - LLCore::HttpRequest::setPolicyClassOption(LLCore::HttpRequest::DEFAULT_POLICY_ID, - LLCore::HttpRequest::CP_CONNECTION_LIMIT, - concurrency_limit); + LLCore::HttpRequest::setStaticPolicyOption(LLCore::HttpRequest::PO_CONNECTION_LIMIT, + LLCore::HttpRequest::DEFAULT_POLICY_ID, + concurrency_limit, + NULL); LLCore::HttpRequest::startThread(); // Get service point LLCore::HttpRequest * hr = new LLCore::HttpRequest(); + // Get request options + LLCore::HttpOptions * opt = new LLCore::HttpOptions(); + opt->setRetries(12); + opt->setUseRetryAfter(true); + // Get a handler/working set WorkingSet ws; // Fill the working set with work ws.mUrl = url_format; - ws.loadTextureUuids(uuids); + ws.loadAssetUuids(uuids); ws.mRandomRange = do_random; ws.mVerbose = do_verbose; - ws.mMaxConcurrency = 100; + ws.mRequestHighWater = highwater; + ws.mRequestLowWater = ws.mRequestHighWater / 2; - if (! ws.mTextures.size()) + if (! ws.mAssets.size()) { std::cerr << "No UUIDs found in file '" << argv[optind] << "'." << std::endl; return 1; @@ -246,9 +273,9 @@ int main(int argc, char** argv) // Run it int passes(0); - while (! ws.reload(hr)) + while (! ws.reload(hr, opt)) { - hr->update(5000000); + hr->update(0); ms_sleep(2); if (0 == (++passes % 200)) { @@ -265,6 +292,8 @@ int main(int argc, char** argv) std::cout << "HTTP 404 errors: " << ws.mErrorsHttp404 << " HTTP 416 errors: " << ws.mErrorsHttp416 << " HTTP 500 errors: " << ws.mErrorsHttp500 << " HTTP 503 errors: " << ws.mErrorsHttp503 << std::endl; + std::cout << "Retries: " << ws.mRetries << " Retries on 503: " << ws.mRetriesHttp503 + << std::endl; std::cout << "User CPU: " << (metrics.mEndUTime - metrics.mStartUTime) << " uS System CPU: " << (metrics.mEndSTime - metrics.mStartSTime) << " uS Wall Time: " << (metrics.mEndWallTime - metrics.mStartWallTime) @@ -275,6 +304,8 @@ int main(int argc, char** argv) // Clean up hr->requestStopThread(NULL); ms_sleep(1000); + opt->release(); + opt = NULL; delete hr; LLCore::HttpRequest::destroyService(); term_curl(); @@ -300,8 +331,10 @@ void usage(std::ostream & out) " -u <url_format> printf-style format string for URL generation\n" " Default: " << url_format << "\n" " -R Issue GETs with random Range: headers\n" - " -c <limit> Maximum request concurrency. Range: [1..100]\n" + " -c <limit> Maximum connection concurrency. Range: [1..100]\n" " Default: " << concurrency_limit << "\n" + " -H <limit> HTTP request highwater (requests fed to llcorehttp).\n" + " Range: [1..100] Default: " << highwater << "\n" " -v Verbose mode. Issue some chatter while running\n" " -h print this help\n" "\n" @@ -322,13 +355,15 @@ WorkingSet::WorkingSet() mErrorsHttp416(0), mErrorsHttp500(0), mErrorsHttp503(0), + mRetries(0), + mRetriesHttp503(0), mSuccesses(0), mByteCount(0L) { - mTextures.reserve(30000); + mAssets.reserve(30000); mHeaders = new LLCore::HttpHeaders; - mHeaders->mHeaders.push_back("Accept: image/x-j2c"); + mHeaders->append("Accept", "image/x-j2c"); } @@ -342,29 +377,35 @@ WorkingSet::~WorkingSet() } -bool WorkingSet::reload(LLCore::HttpRequest * hr) +bool WorkingSet::reload(LLCore::HttpRequest * hr, LLCore::HttpOptions * opt) { - int to_do((std::min)(mRemaining, mMaxConcurrency - int(mHandles.size()))); + if (mRequestLowWater <= mHandles.size()) + { + // Haven't fallen below low-water level yet. + return false; + } + + int to_do((std::min)(mRemaining, mRequestHighWater - int(mHandles.size()))); for (int i(0); i < to_do; ++i) { char buffer[1024]; #if defined(WIN32) - _snprintf_s(buffer, sizeof(buffer), sizeof(buffer) - 1, mUrl.c_str(), mTextures[mAt].mUuid.c_str()); + _snprintf_s(buffer, sizeof(buffer), sizeof(buffer) - 1, mUrl.c_str(), mAssets[mAt].mUuid.c_str()); #else - snprintf(buffer, sizeof(buffer), mUrl.c_str(), mTextures[mAt].mUuid.c_str()); + snprintf(buffer, sizeof(buffer), mUrl.c_str(), mAssets[mAt].mUuid.c_str()); #endif - int offset(mRandomRange ? ((unsigned long) rand()) % 1000000UL : mTextures[mAt].mOffset); - int length(mRandomRange ? ((unsigned long) rand()) % 1000000UL : mTextures[mAt].mLength); + int offset(mRandomRange ? ((unsigned long) rand()) % 1000000UL : mAssets[mAt].mOffset); + int length(mRandomRange ? ((unsigned long) rand()) % 1000000UL : mAssets[mAt].mLength); LLCore::HttpHandle handle; if (offset || length) { - handle = hr->requestGetByteRange(0, 0, buffer, offset, length, NULL, mHeaders, this); + handle = hr->requestGetByteRange(0, 0, buffer, offset, length, opt, mHeaders, this); } else { - handle = hr->requestGet(0, 0, buffer, NULL, mHeaders, this); + handle = hr->requestGet(0, 0, buffer, opt, mHeaders, this); } if (! handle) { @@ -410,7 +451,7 @@ void WorkingSet::onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * r { // More success LLCore::BufferArray * data(response->getBody()); - mByteCount += data->size(); + mByteCount += data ? data->size() : 0; ++mSuccesses; } else @@ -446,6 +487,10 @@ void WorkingSet::onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * r ++mErrorsApi; } } + unsigned int retry(0U), retry_503(0U); + response->getRetries(&retry, &retry_503); + mRetries += int(retry); + mRetriesHttp503 += int(retry_503); mHandles.erase(it); } @@ -459,21 +504,21 @@ void WorkingSet::onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * r } -void WorkingSet::loadTextureUuids(FILE * in) +void WorkingSet::loadAssetUuids(FILE * in) { char buffer[1024]; while (fgets(buffer, sizeof(buffer), in)) { - WorkingSet::Spec texture; + WorkingSet::Spec asset; char * state(NULL); char * token = strtok_r(buffer, " \t\n,", &state); if (token && 36 == strlen(token)) { // Close enough for this function - texture.mUuid = token; - texture.mOffset = 0; - texture.mLength = 0; + asset.mUuid = token; + asset.mOffset = 0; + asset.mLength = 0; token = strtok_r(buffer, " \t\n,", &state); if (token) { @@ -482,14 +527,14 @@ void WorkingSet::loadTextureUuids(FILE * in) if (token) { int length(atoi(token)); - texture.mOffset = offset; - texture.mLength = length; + asset.mOffset = offset; + asset.mLength = length; } } - mTextures.push_back(texture); + mAssets.push_back(asset); } } - mRemaining = mLimit = mTextures.size(); + mRemaining = mLimit = mAssets.size(); } diff --git a/indra/llcorehttp/httpcommon.h b/indra/llcorehttp/httpcommon.h index c0d4ec5aad..9db884057f 100755 --- a/indra/llcorehttp/httpcommon.h +++ b/indra/llcorehttp/httpcommon.h @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -29,9 +29,9 @@ /// @package LLCore::HTTP /// -/// This library implements a high-level, Indra-code-free client interface to -/// HTTP services based on actual patterns found in the viewer and simulator. -/// Interfaces are similar to those supplied by the legacy classes +/// This library implements a high-level, Indra-code-free (somewhat) client +/// interface to HTTP services based on actual patterns found in the viewer +/// and simulator. Interfaces are similar to those supplied by the legacy classes /// LLCurlRequest and LLHTTPClient. To that is added a policy scheme that /// allows an application to specify connection behaviors: limits on /// connections, HTTP keepalive, HTTP pipelining, retry-on-error limits, etc. @@ -52,7 +52,7 @@ /// - "llcorehttp/httprequest.h" /// - "llcorehttp/httpresponse.h" /// -/// The library is still under early development and particular users +/// The library is still under development and particular users /// may need access to internal implementation details that are found /// in the _*.h header files. But this is a crutch to be avoided if at /// all possible and probably indicates some interface work is neeeded. @@ -66,6 +66,8 @@ /// . CRYPTO_set_id_callback(...) /// - HttpRequest::createService() called to instantiate singletons /// and support objects. +/// - HttpRequest::startThread() to kick off the worker thread and +/// begin servicing requests. /// /// An HTTP consumer in an application, and an application may have many /// consumers, does a few things: @@ -91,10 +93,12 @@ /// objects. /// - Do completion processing in your onCompletion() method. /// -/// Code fragments: -/// Rather than a poorly-maintained example in comments, look in the -/// example subdirectory which is a minimal yet functional tool to do -/// GET request performance testing. With four calls: +/// Code fragments. +/// +/// Initialization. Rather than a poorly-maintained example in +/// comments, look in the example subdirectory which is a minimal +/// yet functional tool to do GET request performance testing. +/// With four calls: /// /// init_curl(); /// LLCore::HttpRequest::createService(); @@ -103,7 +107,85 @@ /// /// the program is basically ready to issue requests. /// - +/// HttpHandler. Having started life as a non-indra library, +/// this code broke away from the classic Responder model and +/// introduced a handler class to represent an interface for +/// request responses. This is a non-reference-counted entity +/// which can be used as a base class or a mixin. An instance +/// of a handler can be used for each request or can be shared +/// among any number of requests. Your choice but expect to +/// code something like the following: +/// +/// class AppHandler : public LLCore::HttpHandler +/// { +/// public: +/// virtual void onCompleted(HttpHandle handle, +/// HttpResponse * response) +/// { +/// ... +/// } +/// ... +/// }; +/// ... +/// handler = new handler(...); +/// +/// +/// Issuing requests. Using 'hr' above, +/// +/// hr->requestGet(HttpRequest::DEFAULT_POLICY_ID, +/// 0, // Priority, not used yet +/// url, +/// NULL, // options +/// NULL, // additional headers +/// handler); +/// +/// If that returns a value other than LLCORE_HTTP_HANDLE_INVALID, +/// the request was successfully issued and there will eventally +/// be a status delivered to the handler. If invalid is returnedd, +/// the actual status can be retrieved by calling hr->getStatus(). +/// +/// Completing requests and delivering notifications. Operations +/// are all performed by the worker thread and will be driven to +/// completion regardless of caller actions. Notification of +/// completion (success or failure) is done by calls to +/// HttpRequest::update() which will invoke handlers for completed +/// requests: +/// +/// hr->update(0); +/// // Callbacks into handler->onCompleted() +/// +/// +/// Threads. +/// +/// Threads are supported and used by this library. The various +/// classes, methods and members are documented with thread +/// constraints which programmers must follow and which are +/// defined as follows: +/// +/// consumer Any thread that has instanced HttpRequest and is +/// issuing requests. A particular instance can only +/// be used by one consumer thread but a consumer may +/// have many instances available to it. +/// init Special consumer thread, usually the main thread, +/// involved in setting up the library at startup. +/// worker Thread used internally by the library to perform +/// HTTP operations. Consumers will not have to deal +/// with this thread directly but some APIs are reserved +/// to it. +/// any Consumer or worker thread. +/// +/// For the most part, API users will not have to do much in the +/// way of ensuring thread safely. However, there is a tremendous +/// amount of sharing between threads of read-only data. So when +/// documentation declares that an option or header instance +/// becomes shared between consumer and worker, the consumer must +/// not modify the shared object. +/// +/// Internally, there is almost no thread synchronization. During +/// normal operations (non-init, non-term), only the request queue +/// and the multiple reply queues are shared between threads and +/// only here are mutexes used. +/// #include "linden_common.h" // Modifies curl/curl.h interfaces @@ -239,9 +321,10 @@ struct HttpStatus return *this; } - static const type_enum_t EXT_CURL_EASY = 0; - static const type_enum_t EXT_CURL_MULTI = 1; - static const type_enum_t LLCORE = 2; + static const type_enum_t EXT_CURL_EASY = 0; ///< mStatus is an error from a curl_easy_*() call + static const type_enum_t EXT_CURL_MULTI = 1; ///< mStatus is an error from a curl_multi_*() call + static const type_enum_t LLCORE = 2; ///< mStatus is an HE_* error code + ///< 100-999 directly represent HTTP status codes type_enum_t mType; short mStatus; diff --git a/indra/llcorehttp/httpheaders.cpp b/indra/llcorehttp/httpheaders.cpp index 2832696271..23ebea361c 100755 --- a/indra/llcorehttp/httpheaders.cpp +++ b/indra/llcorehttp/httpheaders.cpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -26,6 +26,8 @@ #include "httpheaders.h" +#include "llstring.h" + namespace LLCore { @@ -40,5 +42,142 @@ HttpHeaders::~HttpHeaders() {} +void +HttpHeaders::clear() +{ + mHeaders.clear(); +} + + +void HttpHeaders::append(const std::string & name, const std::string & value) +{ + mHeaders.push_back(value_type(name, value)); +} + + +void HttpHeaders::append(const char * name, const char * value) +{ + mHeaders.push_back(value_type(name, value)); +} + + +void HttpHeaders::appendNormal(const char * header, size_t size) +{ + std::string name; + std::string value; + + int col_pos(0); + for (; col_pos < size; ++col_pos) + { + if (':' == header[col_pos]) + break; + } + + if (col_pos < size) + { + // Looks like a header, split it and normalize. + // Name is everything before the colon, may be zero-length. + name.assign(header, col_pos); + + // Value is everything after the colon, may also be zero-length. + const size_t val_len(size - col_pos - 1); + if (val_len) + { + value.assign(header + col_pos + 1, val_len); + } + + // Clean the strings + LLStringUtil::toLower(name); + LLStringUtil::trim(name); + LLStringUtil::trimHead(value); + } + else + { + // Uncertain what this is, we'll pack it as + // a name without a value. Won't clean as we don't + // know what it is... + name.assign(header, size); + } + + mHeaders.push_back(value_type(name, value)); +} + + +// Find from end to simulate a tradition of using single-valued +// std::map for this in the past. +const std::string * HttpHeaders::find(const char * name) const +{ + const_reverse_iterator iend(rend()); + for (const_reverse_iterator iter(rbegin()); iend != iter; ++iter) + { + if ((*iter).first == name) + { + return &(*iter).second; + } + } + return NULL; +} + + +// Standard Iterators +HttpHeaders::iterator HttpHeaders::begin() +{ + return mHeaders.begin(); +} + + +HttpHeaders::const_iterator HttpHeaders::begin() const +{ + return mHeaders.begin(); +} + + +HttpHeaders::iterator HttpHeaders::end() +{ + return mHeaders.end(); +} + + +HttpHeaders::const_iterator HttpHeaders::end() const +{ + return mHeaders.end(); +} + + +// Standard Reverse Iterators +HttpHeaders::reverse_iterator HttpHeaders::rbegin() +{ + return mHeaders.rbegin(); +} + + +HttpHeaders::const_reverse_iterator HttpHeaders::rbegin() const +{ + return mHeaders.rbegin(); +} + + +HttpHeaders::reverse_iterator HttpHeaders::rend() +{ + return mHeaders.rend(); +} + + +HttpHeaders::const_reverse_iterator HttpHeaders::rend() const +{ + return mHeaders.rend(); +} + + +// Return the raw container to the caller. +// +// To be used FOR UNIT TESTS ONLY. +// +HttpHeaders::container_t & HttpHeaders::getContainerTESTONLY() +{ + return mHeaders; +} + + } // end namespace LLCore diff --git a/indra/llcorehttp/httpheaders.h b/indra/llcorehttp/httpheaders.h index 3449daa3a1..f70cd898f3 100755 --- a/indra/llcorehttp/httpheaders.h +++ b/indra/llcorehttp/httpheaders.h @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -43,13 +43,26 @@ namespace LLCore /// caller has asked that headers be returned (not the default /// option). /// -/// @note -/// This is a minimally-functional placeholder at the moment -/// to fill out the class hierarchy. The final class will be -/// something else, probably more pair-oriented. It's also -/// an area where shared values are desirable so refcounting is -/// already specced and a copy-on-write scheme imagined. -/// Expect changes here. +/// Class is mostly a thin wrapper around a vector of pairs +/// of strings. Methods provided are few and intended to +/// reflect actual use patterns. These include: +/// - Clearing the list +/// - Appending a name/value pair to the vector +/// - Processing a raw byte string into a normalized name/value +/// pair and appending the result. +/// - Simple case-sensitive find-last-by-name search +/// - Forward and reverse iterators over all pairs +/// +/// Container is ordered and multi-valued. Headers are +/// written in the order in which they are appended and +/// are stored in the order in which they're received from +/// the wire. The same header may appear two or more times +/// in any container. Searches using the simple find() +/// interface will find only the last occurrence (somewhat +/// simulates the use of std::map). Fuller searches require +/// the use of an iterator. Headers received from the wire +/// are only returned from the last request when redirections +/// are involved. /// /// Threading: Not intrinsically thread-safe. It *is* expected /// that callers will build these objects and then share them @@ -64,6 +77,16 @@ namespace LLCore class HttpHeaders : public LLCoreInt::RefCounted { public: + typedef std::pair<std::string, std::string> header_t; + typedef std::vector<header_t> container_t; + typedef container_t::iterator iterator; + typedef container_t::const_iterator const_iterator; + typedef container_t::reverse_iterator reverse_iterator; + typedef container_t::const_reverse_iterator const_reverse_iterator; + typedef container_t::value_type value_type; + typedef container_t::size_type size_type; + +public: /// @post In addition to the instance, caller has a refcount /// to the instance. A call to @see release() will destroy /// the instance. @@ -76,7 +99,78 @@ protected: void operator=(const HttpHeaders &); // Not defined public: - typedef std::vector<std::string> container_t; + // Empty the list of headers. + void clear(); + + // Append a name/value pair supplied as either std::strings + // or NUL-terminated char * to the header list. No normalization + // is performed on the strings. No conformance test is + // performed (names may contain spaces, colons, etc.). + // + void append(const std::string & name, const std::string & value); + void append(const char * name, const char * value); + + // Extract a name/value pair from a raw byte array using + // the first colon character as a separator. Input string + // does not need to be NUL-terminated. Resulting name/value + // pair is appended to the header list. + // + // Normalization is performed on the name/value pair as + // follows: + // - name is lower-cased according to mostly ASCII rules + // - name is left- and right-trimmed of spaces and tabs + // - value is left-trimmed of spaces and tabs + // - either or both of name and value may be zero-length + // + // By convention, headers read from the wire will be normalized + // in this fashion prior to delivery to any HttpHandler code. + // Headers to be written to the wire are left as appended to + // the list. + void appendNormal(const char * header, size_t size); + + // Perform a simple, case-sensitive search of the header list + // returning a pointer to the value of the last matching header + // in the header list. If none is found, a NULL pointer is returned. + // + // Any pointer returned references objects in the container itself + // and will have the same lifetime as this class. If you want + // the value beyond the lifetime of this instance, make a copy. + // + // @arg name C-style string giving the name of a header + // to search. The comparison is case-sensitive + // though list entries may have been normalized + // to lower-case. + // + // @return NULL if the header wasn't found otherwise + // a pointer to a std::string in the container. + // Pointer is valid only for the lifetime of + // the container or until container is modifed. + // + const std::string * find(const char * name) const; + + // Count of headers currently in the list. + size_type size() const + { + return mHeaders.size(); + } + + // Standard std::vector-based forward iterators. + iterator begin(); + const_iterator begin() const; + iterator end(); + const_iterator end() const; + + // Standard std::vector-based reverse iterators. + reverse_iterator rbegin(); + const_reverse_iterator rbegin() const; + reverse_iterator rend(); + const_reverse_iterator rend() const; + +public: + // For unit tests only - not a public API + container_t & getContainerTESTONLY(); + +protected: container_t mHeaders; }; // end class HttpHeaders diff --git a/indra/llcorehttp/httpoptions.cpp b/indra/llcorehttp/httpoptions.cpp index 1699d19f8d..5bf1ecb4a5 100755 --- a/indra/llcorehttp/httpoptions.cpp +++ b/indra/llcorehttp/httpoptions.cpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -38,7 +38,9 @@ HttpOptions::HttpOptions() mWantHeaders(false), mTracing(HTTP_TRACE_OFF), mTimeout(HTTP_REQUEST_TIMEOUT_DEFAULT), - mRetries(HTTP_RETRY_COUNT_DEFAULT) + mTransferTimeout(HTTP_REQUEST_XFER_TIMEOUT_DEFAULT), + mRetries(HTTP_RETRY_COUNT_DEFAULT), + mUseRetryAfter(HTTP_USE_RETRY_AFTER_DEFAULT) {} @@ -64,10 +66,21 @@ void HttpOptions::setTimeout(unsigned int timeout) } +void HttpOptions::setTransferTimeout(unsigned int timeout) +{ + mTransferTimeout = timeout; +} + + void HttpOptions::setRetries(unsigned int retries) { mRetries = retries; } +void HttpOptions::setUseRetryAfter(bool use_retry) +{ + mUseRetryAfter = use_retry; +} + } // end namespace LLCore diff --git a/indra/llcorehttp/httpoptions.h b/indra/llcorehttp/httpoptions.h index 97e46a8cd3..f49a3555aa 100755 --- a/indra/llcorehttp/httpoptions.h +++ b/indra/llcorehttp/httpoptions.h @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -68,36 +68,55 @@ protected: void operator=(const HttpOptions &); // Not defined public: + // Default: false void setWantHeaders(bool wanted); bool getWantHeaders() const { return mWantHeaders; } - + + // Default: 0 void setTrace(int long); int getTrace() const { return mTracing; } + // Default: 30 void setTimeout(unsigned int timeout); unsigned int getTimeout() const { return mTimeout; } + // Default: 0 + void setTransferTimeout(unsigned int timeout); + unsigned int getTransferTimeout() const + { + return mTransferTimeout; + } + + // Default: 8 void setRetries(unsigned int retries); unsigned int getRetries() const { return mRetries; } + + // Default: false + void setUseRetryAfter(bool use_retry); + bool getUseRetryAfter() const + { + return mUseRetryAfter; + } protected: bool mWantHeaders; int mTracing; unsigned int mTimeout; + unsigned int mTransferTimeout; unsigned int mRetries; - + bool mUseRetryAfter; }; // end class HttpOptions diff --git a/indra/llcorehttp/httprequest.cpp b/indra/llcorehttp/httprequest.cpp index 9b739a8825..7b1888e3eb 100755 --- a/indra/llcorehttp/httprequest.cpp +++ b/indra/llcorehttp/httprequest.cpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -54,12 +54,8 @@ namespace LLCore // ==================================== -HttpRequest::policy_t HttpRequest::sNextPolicyID(1); - - HttpRequest::HttpRequest() - : //HttpHandler(), - mReplyQueue(NULL), + : mReplyQueue(NULL), mRequestQueue(NULL) { mRequestQueue = HttpRequestQueue::instanceOf(); @@ -90,45 +86,91 @@ HttpRequest::~HttpRequest() // ==================================== -HttpStatus HttpRequest::setPolicyGlobalOption(EGlobalPolicy opt, long value) +HttpRequest::policy_t HttpRequest::createPolicyClass() { if (HttpService::RUNNING == HttpService::instanceOf()->getState()) { - return HttpStatus(HttpStatus::LLCORE, HE_OPT_NOT_DYNAMIC); + return 0; } - return HttpService::instanceOf()->getGlobalOptions().set(opt, value); + return HttpService::instanceOf()->createPolicyClass(); } -HttpStatus HttpRequest::setPolicyGlobalOption(EGlobalPolicy opt, const std::string & value) +HttpStatus HttpRequest::setStaticPolicyOption(EPolicyOption opt, policy_t pclass, + long value, long * ret_value) { if (HttpService::RUNNING == HttpService::instanceOf()->getState()) { return HttpStatus(HttpStatus::LLCORE, HE_OPT_NOT_DYNAMIC); } - return HttpService::instanceOf()->getGlobalOptions().set(opt, value); + return HttpService::instanceOf()->setPolicyOption(opt, pclass, value, ret_value); } -HttpRequest::policy_t HttpRequest::createPolicyClass() +HttpStatus HttpRequest::setStaticPolicyOption(EPolicyOption opt, policy_t pclass, + const std::string & value, std::string * ret_value) { if (HttpService::RUNNING == HttpService::instanceOf()->getState()) { - return 0; + return HttpStatus(HttpStatus::LLCORE, HE_OPT_NOT_DYNAMIC); } - return HttpService::instanceOf()->createPolicyClass(); + return HttpService::instanceOf()->setPolicyOption(opt, pclass, value, ret_value); } -HttpStatus HttpRequest::setPolicyClassOption(policy_t policy_id, - EClassPolicy opt, - long value) +HttpHandle HttpRequest::setPolicyOption(EPolicyOption opt, policy_t pclass, + long value, HttpHandler * handler) { - if (HttpService::RUNNING == HttpService::instanceOf()->getState()) + HttpStatus status; + HttpHandle handle(LLCORE_HTTP_HANDLE_INVALID); + + HttpOpSetGet * op = new HttpOpSetGet(); + if (! (status = op->setupSet(opt, pclass, value))) { - return HttpStatus(HttpStatus::LLCORE, HE_OPT_NOT_DYNAMIC); + op->release(); + mLastReqStatus = status; + return handle; + } + op->setReplyPath(mReplyQueue, handler); + if (! (status = mRequestQueue->addOp(op))) // transfers refcount + { + op->release(); + mLastReqStatus = status; + return handle; + } + + mLastReqStatus = status; + handle = static_cast<HttpHandle>(op); + + return handle; +} + + +HttpHandle HttpRequest::setPolicyOption(EPolicyOption opt, policy_t pclass, + const std::string & value, HttpHandler * handler) +{ + HttpStatus status; + HttpHandle handle(LLCORE_HTTP_HANDLE_INVALID); + + HttpOpSetGet * op = new HttpOpSetGet(); + if (! (status = op->setupSet(opt, pclass, value))) + { + op->release(); + mLastReqStatus = status; + return handle; + } + op->setReplyPath(mReplyQueue, handler); + if (! (status = mRequestQueue->addOp(op))) // transfers refcount + { + op->release(); + mLastReqStatus = status; + return handle; } - return HttpService::instanceOf()->getClassOptions(policy_id).set(opt, value); + + mLastReqStatus = status; + handle = static_cast<HttpHandle>(op); + + return handle; } @@ -474,31 +516,6 @@ HttpHandle HttpRequest::requestSpin(int mode) return handle; } -// ==================================== -// Dynamic Policy Methods -// ==================================== - -HttpHandle HttpRequest::requestSetHttpProxy(const std::string & proxy, HttpHandler * handler) -{ - HttpStatus status; - HttpHandle handle(LLCORE_HTTP_HANDLE_INVALID); - - HttpOpSetGet * op = new HttpOpSetGet(); - op->setupSet(GP_HTTP_PROXY, proxy); - op->setReplyPath(mReplyQueue, handler); - if (! (status = mRequestQueue->addOp(op))) // transfers refcount - { - op->release(); - mLastReqStatus = status; - return handle; - } - - mLastReqStatus = status; - handle = static_cast<HttpHandle>(op); - - return handle; -} - } // end namespace LLCore diff --git a/indra/llcorehttp/httprequest.h b/indra/llcorehttp/httprequest.h index ab2f302d34..651654844a 100755 --- a/indra/llcorehttp/httprequest.h +++ b/indra/llcorehttp/httprequest.h @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -56,6 +56,9 @@ class BufferArray; /// The class supports the current HTTP request operations: /// /// - requestGetByteRange: GET with Range header for a single range of bytes +/// - requestGet: +/// - requestPost: +/// - requestPut: /// /// Policy Classes /// @@ -100,9 +103,26 @@ public: /// Represents a default, catch-all policy class that guarantees /// eventual service for any HTTP request. - static const int DEFAULT_POLICY_ID = 0; + static const policy_t DEFAULT_POLICY_ID = 0; + static const policy_t INVALID_POLICY_ID = 0xFFFFFFFFU; + static const policy_t GLOBAL_POLICY_ID = 0xFFFFFFFEU; - enum EGlobalPolicy + /// Create a new policy class into which requests can be made. + /// + /// All class creation must occur before threads are started and + /// transport begins. Policy classes are limited to a small value. + /// Currently that limit is the default class + 1. + /// + /// @return If positive, the policy_id used to reference + /// the class in other methods. If 0, requests + /// for classes have exceeded internal limits + /// or caller has tried to create a class after + /// threads have been started. Caller must fallback + /// and recover. + /// + static policy_t createPolicyClass(); + + enum EPolicyOption { /// Maximum number of connections the library will use to /// perform operations. This is somewhat soft as the underlying @@ -113,24 +133,40 @@ public: /// a somewhat soft value. There may be an additional five /// connections per policy class depending upon runtime /// behavior. - GP_CONNECTION_LIMIT, + /// + /// Both global and per-class + PO_CONNECTION_LIMIT, + + /// Limits the number of connections used for a single + /// literal address/port pair within the class. + /// + /// Per-class only + PO_PER_HOST_CONNECTION_LIMIT, /// String containing a system-appropriate directory name /// where SSL certs are stored. - GP_CA_PATH, + /// + /// Global only + PO_CA_PATH, /// String giving a full path to a file containing SSL certs. - GP_CA_FILE, + /// + /// Global only + PO_CA_FILE, /// String of host/port to use as simple HTTP proxy. This is /// going to change in the future into something more elaborate /// that may support richer schemes. - GP_HTTP_PROXY, + /// + /// Global only + PO_HTTP_PROXY, /// Long value that if non-zero enables the use of the /// traditional LLProxy code for http/socks5 support. If - /// enabled, has priority over GP_HTTP_PROXY. - GP_LLPROXY, + // enabled, has priority over GP_HTTP_PROXY. + /// + /// Global only + PO_LLPROXY, /// Long value setting the logging trace level for the /// library. Possible values are: @@ -143,50 +179,59 @@ public: /// These values are also used in the trace modes for /// individual requests in HttpOptions. Also be aware that /// tracing tends to impact performance of the viewer. - GP_TRACE - }; - - /// Set a parameter on a global policy option. Calls - /// made after the start of the servicing thread are - /// not honored and return an error status. - /// - /// @param opt Enum of option to be set. - /// @param value Desired value of option. - /// @return Standard status code. - static HttpStatus setPolicyGlobalOption(EGlobalPolicy opt, long value); - static HttpStatus setPolicyGlobalOption(EGlobalPolicy opt, const std::string & value); - - /// Create a new policy class into which requests can be made. - /// - /// @return If positive, the policy_id used to reference - /// the class in other methods. If 0, an error - /// occurred and @see getStatus() may provide more - /// detail on the reason. - static policy_t createPolicyClass(); - - enum EClassPolicy - { - /// Limits the number of connections used for the class. - CP_CONNECTION_LIMIT, - - /// Limits the number of connections used for a single - /// literal address/port pair within the class. - CP_PER_HOST_CONNECTION_LIMIT, + /// + /// Global only + PO_TRACE, /// Suitable requests are allowed to pipeline on their /// connections when they ask for it. - CP_ENABLE_PIPELINING + /// + /// Per-class only + PO_ENABLE_PIPELINING, + + /// Controls whether client-side throttling should be + /// performed on this policy class. Positive values + /// enable throttling and specify the request rate + /// (requests per second) that should be targetted. + /// A value of zero, the default, specifies no throttling. + /// + /// Per-class only + PO_THROTTLE_RATE, + + PO_LAST // Always at end }; - + + /// Set a policy option for a global or class parameter at + /// startup time (prior to thread start). + /// + /// @param opt Enum of option to be set. + /// @param pclass For class-based options, the policy class ID to + /// be changed. For globals, specify GLOBAL_POLICY_ID. + /// @param value Desired value of option. + /// @param ret_value Pointer to receive effective set value + /// if successful. May be NULL if effective + /// value not wanted. + /// @return Standard status code. + static HttpStatus setStaticPolicyOption(EPolicyOption opt, policy_t pclass, + long value, long * ret_value); + static HttpStatus setStaticPolicyOption(EPolicyOption opt, policy_t pclass, + const std::string & value, std::string * ret_value); + /// Set a parameter on a class-based policy option. Calls /// made after the start of the servicing thread are /// not honored and return an error status. /// - /// @param policy_id ID of class as returned by @see createPolicyClass(). - /// @param opt Enum of option to be set. - /// @param value Desired value of option. - /// @return Standard status code. - static HttpStatus setPolicyClassOption(policy_t policy_id, EClassPolicy opt, long value); + /// @param opt Enum of option to be set. + /// @param pclass For class-based options, the policy class ID to + /// be changed. Ignored for globals but recommend + /// using INVALID_POLICY_ID in this case. + /// @param value Desired value of option. + /// @return Handle of dynamic request. Use @see getStatus() if + /// the returned handle is invalid. + HttpHandle setPolicyOption(EPolicyOption opt, policy_t pclass, long value, + HttpHandler * handler); + HttpHandle setPolicyOption(EPolicyOption opt, policy_t pclass, const std::string & value, + HttpHandler * handler); /// @} @@ -488,16 +533,6 @@ public: /// @} - /// @name DynamicPolicyMethods - /// - /// @{ - - /// Request that a running transport pick up a new proxy setting. - /// An empty string will indicate no proxy is to be used. - HttpHandle requestSetHttpProxy(const std::string & proxy, HttpHandler * handler); - - /// @} - protected: void generateNotification(HttpOperation * op); @@ -519,7 +554,6 @@ private: /// Must be established before any threading is allowed to /// start. /// - static policy_t sNextPolicyID; /// @} // End Global State diff --git a/indra/llcorehttp/httpresponse.cpp b/indra/llcorehttp/httpresponse.cpp index a552e48a1b..c974395b0a 100755 --- a/indra/llcorehttp/httpresponse.cpp +++ b/indra/llcorehttp/httpresponse.cpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -39,7 +39,9 @@ HttpResponse::HttpResponse() mReplyLength(0U), mReplyFullLength(0U), mBufferArray(NULL), - mHeaders(NULL) + mHeaders(NULL), + mRetries(0U), + m503Retries(0U) {} diff --git a/indra/llcorehttp/httpresponse.h b/indra/llcorehttp/httpresponse.h index 4a481db6ac..bb4b59c16e 100755 --- a/indra/llcorehttp/httpresponse.h +++ b/indra/llcorehttp/httpresponse.h @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -144,6 +144,25 @@ public: mContentType = con_type; } + /// Get and set retry attempt information on the request. + void getRetries(unsigned int * retries, unsigned int * retries_503) const + { + if (retries) + { + *retries = mRetries; + } + if (retries_503) + { + *retries_503 = m503Retries; + } + } + + void setRetries(unsigned int retries, unsigned int retries_503) + { + mRetries = retries; + m503Retries = retries_503; + } + protected: // Response data here HttpStatus mStatus; @@ -153,6 +172,8 @@ protected: BufferArray * mBufferArray; HttpHeaders * mHeaders; std::string mContentType; + unsigned int mRetries; + unsigned int m503Retries; }; diff --git a/indra/llcorehttp/tests/test_httpheaders.hpp b/indra/llcorehttp/tests/test_httpheaders.hpp index ce0d19b058..668c36dc66 100755 --- a/indra/llcorehttp/tests/test_httpheaders.hpp +++ b/indra/llcorehttp/tests/test_httpheaders.hpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -36,7 +36,6 @@ using namespace LLCoreInt; - namespace tut { @@ -63,7 +62,7 @@ void HttpHeadersTestObjectType::test<1>() HttpHeaders * headers = new HttpHeaders(); ensure("One ref on construction of HttpHeaders", headers->getRefCount() == 1); ensure("Memory being used", mMemTotal < GetMemTotal()); - ensure("Nothing in headers", 0 == headers->mHeaders.size()); + ensure("Nothing in headers", 0 == headers->size()); // release the implicit reference, causing the object to be released headers->release(); @@ -85,14 +84,340 @@ void HttpHeadersTestObjectType::test<2>() { // Append a few strings - std::string str1("Pragma:"); - headers->mHeaders.push_back(str1); - std::string str2("Accept: application/json"); - headers->mHeaders.push_back(str2); + std::string str1n("Pragma"); + std::string str1v(""); + headers->append(str1n, str1v); + std::string str2n("Accept"); + std::string str2v("application/json"); + headers->append(str2n, str2v); + + ensure("Headers retained", 2 == headers->size()); + HttpHeaders::container_t & c(headers->getContainerTESTONLY()); + + ensure("First name is first name", c[0].first == str1n); + ensure("First value is first value", c[0].second == str1v); + ensure("Second name is second name", c[1].first == str2n); + ensure("Second value is second value", c[1].second == str2v); + } + + // release the implicit reference, causing the object to be released + headers->release(); + + // make sure we didn't leak any memory + ensure(mMemTotal == GetMemTotal()); +} + +template <> template <> +void HttpHeadersTestObjectType::test<3>() +{ + set_test_name("HttpHeaders basic find"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + + // create a new ref counted object with an implicit reference + HttpHeaders * headers = new HttpHeaders(); + + { + // Append a few strings + std::string str1n("Uno"); + std::string str1v("1"); + headers->append(str1n, str1v); + std::string str2n("doS"); + std::string str2v("2-2-2-2"); + headers->append(str2n, str2v); + std::string str3n("TRES"); + std::string str3v("trois gymnopedie"); + headers->append(str3n, str3v); + + ensure("Headers retained", 3 == headers->size()); + + const std::string * result(NULL); + + // Find a header + result = headers->find("TRES"); + ensure("Found the last item", result != NULL); + ensure("Last item is a nice", result != NULL && str3v == *result); + + // appends above are raw and find is case sensitive + result = headers->find("TReS"); + ensure("Last item not found due to case", result == NULL); + + result = headers->find("TRE"); + ensure("Last item not found due to prefixing (1)", result == NULL); + + result = headers->find("TRESS"); + ensure("Last item not found due to prefixing (2)", result == NULL); + } + + // release the implicit reference, causing the object to be released + headers->release(); + + // make sure we didn't leak any memory + ensure(mMemTotal == GetMemTotal()); +} + +template <> template <> +void HttpHeadersTestObjectType::test<4>() +{ + set_test_name("HttpHeaders normalized header entry"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + + // create a new ref counted object with an implicit reference + HttpHeaders * headers = new HttpHeaders(); + + { + static char line1[] = " AcCePT : image/yourfacehere"; + static char line1v[] = "image/yourfacehere"; + headers->appendNormal(line1, sizeof(line1) - 1); + + ensure("First append worked in some fashion", 1 == headers->size()); + + const std::string * result(NULL); + + // Find a header + result = headers->find("accept"); + ensure("Found 'accept'", result != NULL); + ensure("accept value has face", result != NULL && *result == line1v); + + // Left-clean on value + static char line2[] = " next : \t\tlinejunk \t"; + headers->appendNormal(line2, sizeof(line2) - 1); + ensure("Second append worked", 2 == headers->size()); + result = headers->find("next"); + ensure("Found 'next'", result != NULL); + ensure("next value is left-clean", result != NULL && + *result == "linejunk \t"); + + // First value unmolested + result = headers->find("accept"); + ensure("Found 'accept' again", result != NULL); + ensure("accept value has face", result != NULL && *result == line1v); + + // Colons in value are okay + static char line3[] = "FancY-PANTs::plop:-neuf-=vleem="; + static char line3v[] = ":plop:-neuf-=vleem="; + headers->appendNormal(line3, sizeof(line3) - 1); + ensure("Third append worked", 3 == headers->size()); + result = headers->find("fancy-pants"); + ensure("Found 'fancy-pants'", result != NULL); + ensure("fancy-pants value has colons", result != NULL && *result == line3v); + + // Zero-length value + static char line4[] = "all-talk-no-walk:"; + headers->appendNormal(line4, sizeof(line4) - 1); + ensure("Fourth append worked", 4 == headers->size()); + result = headers->find("all-talk-no-walk"); + ensure("Found 'all-talk'", result != NULL); + ensure("al-talk value is zero-length", result != NULL && result->size() == 0); + + // Zero-length name + static char line5[] = ":all-talk-no-walk"; + static char line5v[] = "all-talk-no-walk"; + headers->appendNormal(line5, sizeof(line5) - 1); + ensure("Fifth append worked", 5 == headers->size()); + result = headers->find(""); + ensure("Found no-name", result != NULL); + ensure("no-name value is something", result != NULL && *result == line5v); + + // Lone colon is still something + headers->clear(); + static char line6[] = " :"; + headers->appendNormal(line6, sizeof(line6) - 1); + ensure("Sixth append worked", 1 == headers->size()); + result = headers->find(""); + ensure("Found 2nd no-name", result != NULL); + ensure("2nd no-name value is nothing", result != NULL && result->size() == 0); + + // Line without colons is taken as-is and unstripped in name + static char line7[] = " \toskdgioasdghaosdghoowg28342908tg8902hg0hwedfhqew890v7qh0wdebv78q0wdevbhq>?M>BNM<ZV>?NZ? \t"; + headers->appendNormal(line7, sizeof(line7) - 1); + ensure("Seventh append worked", 2 == headers->size()); + result = headers->find(line7); + ensure("Found whatsit line", result != NULL); + ensure("Whatsit line has no value", result != NULL && result->size() == 0); + + // Normaling interface heeds the byte count, doesn't look for NUL-terminator + static char line8[] = "binary:ignorestuffontheendofthis"; + headers->appendNormal(line8, 13); + ensure("Eighth append worked", 3 == headers->size()); + result = headers->find("binary"); + ensure("Found 'binary'", result != NULL); + ensure("binary value was limited to 'ignore'", result != NULL && + *result == "ignore"); + + } - ensure("Headers retained", 2 == headers->mHeaders.size()); - ensure("First is first", headers->mHeaders[0] == str1); - ensure("Second is second", headers->mHeaders[1] == str2); + // release the implicit reference, causing the object to be released + headers->release(); + + // make sure we didn't leak any memory + ensure(mMemTotal == GetMemTotal()); +} + +// Verify forward iterator finds everything as expected +template <> template <> +void HttpHeadersTestObjectType::test<5>() +{ + set_test_name("HttpHeaders iterator tests"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + + // create a new ref counted object with an implicit reference + HttpHeaders * headers = new HttpHeaders(); + + HttpHeaders::iterator end(headers->end()), begin(headers->begin()); + ensure("Empty container has equal begin/end const iterators", end == begin); + HttpHeaders::const_iterator cend(headers->end()), cbegin(headers->begin()); + ensure("Empty container has equal rbegin/rend const iterators", cend == cbegin); + + ensure("Empty container has equal begin/end iterators", headers->end() == headers->begin()); + + { + static char line1[] = " AcCePT : image/yourfacehere"; + static char line1v[] = "image/yourfacehere"; + headers->appendNormal(line1, sizeof(line1) - 1); + + static char line2[] = " next : \t\tlinejunk \t"; + static char line2v[] = "linejunk \t"; + headers->appendNormal(line2, sizeof(line2) - 1); + + static char line3[] = "FancY-PANTs::plop:-neuf-=vleem="; + static char line3v[] = ":plop:-neuf-=vleem="; + headers->appendNormal(line3, sizeof(line3) - 1); + + static char line4[] = "all-talk-no-walk:"; + static char line4v[] = ""; + headers->appendNormal(line4, sizeof(line4) - 1); + + static char line5[] = ":all-talk-no-walk"; + static char line5v[] = "all-talk-no-walk"; + headers->appendNormal(line5, sizeof(line5) - 1); + + static char line6[] = " :"; + static char line6v[] = ""; + headers->appendNormal(line6, sizeof(line6) - 1); + + ensure("All entries accounted for", 6 == headers->size()); + + static char * values[] = { + line1v, + line2v, + line3v, + line4v, + line5v, + line6v + }; + + int i(0); + HttpHeaders::const_iterator cend(headers->end()); + for (HttpHeaders::const_iterator it(headers->begin()); + cend != it; + ++it, ++i) + { + std::ostringstream str; + str << "Const Iterator value # " << i << " was " << values[i]; + ensure(str.str(), (*it).second == values[i]); + } + + // Rewind, do non-consts + i = 0; + HttpHeaders::iterator end(headers->end()); + for (HttpHeaders::iterator it(headers->begin()); + end != it; + ++it, ++i) + { + std::ostringstream str; + str << "Const Iterator value # " << i << " was " << values[i]; + ensure(str.str(), (*it).second == values[i]); + } + } + + // release the implicit reference, causing the object to be released + headers->release(); + + // make sure we didn't leak any memory + ensure(mMemTotal == GetMemTotal()); +} + +// Reverse iterators find everything as expected +template <> template <> +void HttpHeadersTestObjectType::test<6>() +{ + set_test_name("HttpHeaders reverse iterator tests"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + + // create a new ref counted object with an implicit reference + HttpHeaders * headers = new HttpHeaders(); + + HttpHeaders::reverse_iterator rend(headers->rend()), rbegin(headers->rbegin()); + ensure("Empty container has equal rbegin/rend const iterators", rend == rbegin); + HttpHeaders::const_reverse_iterator crend(headers->rend()), crbegin(headers->rbegin()); + ensure("Empty container has equal rbegin/rend const iterators", crend == crbegin); + + { + static char line1[] = " AcCePT : image/yourfacehere"; + static char line1v[] = "image/yourfacehere"; + headers->appendNormal(line1, sizeof(line1) - 1); + + static char line2[] = " next : \t\tlinejunk \t"; + static char line2v[] = "linejunk \t"; + headers->appendNormal(line2, sizeof(line2) - 1); + + static char line3[] = "FancY-PANTs::plop:-neuf-=vleem="; + static char line3v[] = ":plop:-neuf-=vleem="; + headers->appendNormal(line3, sizeof(line3) - 1); + + static char line4[] = "all-talk-no-walk:"; + static char line4v[] = ""; + headers->appendNormal(line4, sizeof(line4) - 1); + + static char line5[] = ":all-talk-no-walk"; + static char line5v[] = "all-talk-no-walk"; + headers->appendNormal(line5, sizeof(line5) - 1); + + static char line6[] = " :"; + static char line6v[] = ""; + headers->appendNormal(line6, sizeof(line6) - 1); + + ensure("All entries accounted for", 6 == headers->size()); + + static char * values[] = { + line6v, + line5v, + line4v, + line3v, + line2v, + line1v + }; + + int i(0); + HttpHeaders::const_reverse_iterator cend(headers->rend()); + for (HttpHeaders::const_reverse_iterator it(headers->rbegin()); + cend != it; + ++it, ++i) + { + std::ostringstream str; + str << "Const Iterator value # " << i << " was " << values[i]; + ensure(str.str(), (*it).second == values[i]); + } + + // Rewind, do non-consts + i = 0; + HttpHeaders::reverse_iterator end(headers->rend()); + for (HttpHeaders::reverse_iterator it(headers->rbegin()); + end != it; + ++it, ++i) + { + std::ostringstream str; + str << "Iterator value # " << i << " was " << values[i]; + ensure(str.str(), (*it).second == values[i]); + } } // release the implicit reference, causing the object to be released diff --git a/indra/llcorehttp/tests/test_httprequest.hpp b/indra/llcorehttp/tests/test_httprequest.hpp index e5488cf941..f1b9c02393 100755 --- a/indra/llcorehttp/tests/test_httprequest.hpp +++ b/indra/llcorehttp/tests/test_httprequest.hpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -60,6 +60,8 @@ void usleep(unsigned long usec); namespace tut { +typedef std::vector<std::pair<boost::regex, boost::regex> > regex_container_t; + struct HttpRequestTestData { // the test objects inherit from this so the member functions and variables @@ -109,11 +111,17 @@ public: for (int i(0); i < mHeadersRequired.size(); ++i) { bool found = false; - for (HttpHeaders::container_t::const_iterator iter(header->mHeaders.begin()); - header->mHeaders.end() != iter; + for (HttpHeaders::const_iterator iter(header->begin()); + header->end() != iter; ++iter) { - if (boost::regex_match(*iter, mHeadersRequired[i])) + // std::cerr << "Header: " << (*iter).first + // << ": " << (*iter).second << std::endl; + + if (boost::regex_match((*iter).first, + mHeadersRequired[i].first) && + boost::regex_match((*iter).second, + mHeadersRequired[i].second)) { found = true; break; @@ -129,11 +137,14 @@ public: { for (int i(0); i < mHeadersDisallowed.size(); ++i) { - for (HttpHeaders::container_t::const_iterator iter(header->mHeaders.begin()); - header->mHeaders.end() != iter; + for (HttpHeaders::const_iterator iter(header->begin()); + header->end() != iter; ++iter) { - if (boost::regex_match(*iter, mHeadersDisallowed[i])) + if (boost::regex_match((*iter).first, + mHeadersDisallowed[i].first) && + boost::regex_match((*iter).second, + mHeadersDisallowed[i].second)) { std::ostringstream str; str << "Disallowed header # " << i << " not found in response"; @@ -159,8 +170,8 @@ public: std::string mName; HttpHandle mExpectHandle; std::string mCheckContentType; - std::vector<boost::regex> mHeadersRequired; - std::vector<boost::regex> mHeadersDisallowed; + regex_container_t mHeadersRequired; + regex_container_t mHeadersDisallowed; }; typedef test_group<HttpRequestTestData> HttpRequestTestGroupType; @@ -1202,7 +1213,7 @@ void HttpRequestTestObjectType::test<12>() HttpRequest::createService(); // Enable tracing - HttpRequest::setPolicyGlobalOption(LLCore::HttpRequest::GP_TRACE, 2); + HttpRequest::setStaticPolicyOption(HttpRequest::PO_TRACE, HttpRequest::DEFAULT_POLICY_ID, 2, NULL); // Start threading early so that thread memory is invariant // over the test. @@ -1320,7 +1331,7 @@ void HttpRequestTestObjectType::test<13>() HttpRequest::createService(); // Enable tracing - HttpRequest::setPolicyGlobalOption(LLCore::HttpRequest::GP_TRACE, 2); + HttpRequest::setStaticPolicyOption(HttpRequest::PO_TRACE, HttpRequest::DEFAULT_POLICY_ID, 2, NULL); // Start threading early so that thread memory is invariant // over the test. @@ -1335,7 +1346,9 @@ void HttpRequestTestObjectType::test<13>() // Issue a GET that succeeds mStatus = HttpStatus(200); - handler.mHeadersRequired.push_back(boost::regex("\\W*X-LL-Special:.*", boost::regex::icase)); + handler.mHeadersRequired.push_back( + regex_container_t::value_type(boost::regex("X-LL-Special", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); HttpHandle handle = req->requestGetByteRange(HttpRequest::DEFAULT_POLICY_ID, 0U, url_base, @@ -1702,18 +1715,54 @@ void HttpRequestTestObjectType::test<16>() // Issue a GET that *can* connect mStatus = HttpStatus(200); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-connection:\\s*keep-alive", boost::regex::icase)); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-accept:\\s*\\*/\\*", boost::regex::icase)); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-accept-encoding:\\s*((gzip|deflate),\\s*)+(gzip|deflate)", boost::regex::icase)); // close enough - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-keep-alive:\\s*\\d+", boost::regex::icase)); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-host:\\s*.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-cache-control:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-pragma:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-range:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-transfer-encoding:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-referer:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-content-type:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-content-encoding:.*", boost::regex::icase)); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-connection", boost::regex::icase), + boost::regex("keep-alive", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-accept", boost::regex::icase), + boost::regex("\\*/\\*", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-accept-encoding", boost::regex::icase), + boost::regex("((gzip|deflate),\\s*)+(gzip|deflate)", boost::regex::icase))); // close enough + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-keep-alive", boost::regex::icase), + boost::regex("\\d+", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-host", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-cache-control", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-pragma", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-range", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-transfer-encoding", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-referer", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-content-type", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-content-encoding", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); HttpHandle handle = req->requestGet(HttpRequest::DEFAULT_POLICY_ID, 0U, url_base + "reflect/", @@ -1735,23 +1784,60 @@ void HttpRequestTestObjectType::test<16>() // Do a texture-style fetch headers = new HttpHeaders; - headers->mHeaders.push_back("Accept: image/x-j2c"); + headers->append("Accept", "image/x-j2c"); mStatus = HttpStatus(200); handler.mHeadersRequired.clear(); handler.mHeadersDisallowed.clear(); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-connection:\\s*keep-alive", boost::regex::icase)); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-accept:\\s*image/x-j2c", boost::regex::icase)); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-accept-encoding:\\s*((gzip|deflate),\\s*)+(gzip|deflate)", boost::regex::icase)); // close enough - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-keep-alive:\\s*\\d+", boost::regex::icase)); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-host:\\s*.*", boost::regex::icase)); - handler.mHeadersRequired.push_back(boost::regex("\\W*X-Reflect-range:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-cache-control:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-pragma:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-transfer-encoding:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-referer:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-content-type:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-content-encoding:.*", boost::regex::icase)); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-connection", boost::regex::icase), + boost::regex("keep-alive", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-accept", boost::regex::icase), + boost::regex("image/x-j2c", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-accept-encoding", boost::regex::icase), + boost::regex("((gzip|deflate),\\s*)+(gzip|deflate)", boost::regex::icase))); // close enough + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-keep-alive", boost::regex::icase), + boost::regex("\\d+", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-host", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("\\W*X-Reflect-range", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-cache-control", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-pragma", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-transfer-encoding", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-referer", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-content-type", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-content-encoding", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); handle = req->requestGetByteRange(HttpRequest::DEFAULT_POLICY_ID, 0U, url_base + "reflect/", @@ -1892,20 +1978,63 @@ void HttpRequestTestObjectType::test<17>() // Issue a default POST mStatus = HttpStatus(200); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-connection:\\s*keep-alive", boost::regex::icase)); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-accept:\\s*\\*/\\*", boost::regex::icase)); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-accept-encoding:\\s*((gzip|deflate),\\s*)+(gzip|deflate)", boost::regex::icase)); // close enough - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-keep-alive:\\s*\\d+", boost::regex::icase)); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-host:\\s*.*", boost::regex::icase)); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-content-length:\\s*\\d+", boost::regex::icase)); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-content-type:\\s*application/x-www-form-urlencoded", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-cache-control:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-pragma:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-range:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-referer:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-content-encoding:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-expect:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-transfer-encoding:\\s*.*chunked.*", boost::regex::icase)); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-connection", boost::regex::icase), + boost::regex("keep-alive", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-accept", boost::regex::icase), + boost::regex("\\*/\\*", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-accept-encoding", boost::regex::icase), + boost::regex("((gzip|deflate),\\s*)+(gzip|deflate)", boost::regex::icase))); // close enough + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-keep-alive", boost::regex::icase), + boost::regex("\\d+", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-host", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-content-length", boost::regex::icase), + boost::regex("\\d+", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-content-type", boost::regex::icase), + boost::regex("application/x-www-form-urlencoded", boost::regex::icase))); + + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-cache-control", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-pragma", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-range", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-referer", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-content-encoding", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-expect", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-transfer_encoding", boost::regex::icase), + boost::regex(".*chunked.*", boost::regex::icase))); HttpHandle handle = req->requestPost(HttpRequest::DEFAULT_POLICY_ID, 0U, url_base + "reflect/", @@ -2052,20 +2181,64 @@ void HttpRequestTestObjectType::test<18>() // Issue a default PUT mStatus = HttpStatus(200); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-connection:\\s*keep-alive", boost::regex::icase)); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-accept:\\s*\\*/\\*", boost::regex::icase)); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-accept-encoding:\\s*((gzip|deflate),\\s*)+(gzip|deflate)", boost::regex::icase)); // close enough - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-keep-alive:\\s*\\d+", boost::regex::icase)); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-host:\\s*.*", boost::regex::icase)); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-content-length:\\s*\\d+", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-cache-control:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-pragma:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-range:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-referer:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-content-encoding:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-expect:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-transfer-encoding:\\s*.*chunked.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("X-Reflect-content-type:.*", boost::regex::icase)); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-connection", boost::regex::icase), + boost::regex("keep-alive", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-accept", boost::regex::icase), + boost::regex("\\*/\\*", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-accept-encoding", boost::regex::icase), + boost::regex("((gzip|deflate),\\s*)+(gzip|deflate)", boost::regex::icase))); // close enough + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-keep-alive", boost::regex::icase), + boost::regex("\\d+", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-host", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-content-length", boost::regex::icase), + boost::regex("\\d+", boost::regex::icase))); + + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-cache-control", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-pragma", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-range", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-referer", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-content-encoding", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-expect", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-transfer-encoding", boost::regex::icase), + boost::regex(".*chunked.*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-content-type", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + HttpHandle handle = req->requestPut(HttpRequest::DEFAULT_POLICY_ID, 0U, url_base + "reflect/", @@ -2206,27 +2379,73 @@ void HttpRequestTestObjectType::test<19>() // headers headers = new HttpHeaders; - headers->mHeaders.push_back("Keep-Alive: 120"); - headers->mHeaders.push_back("Accept-encoding: deflate"); - headers->mHeaders.push_back("Accept: text/plain"); + headers->append("Keep-Alive", "120"); + headers->append("Accept-encoding", "deflate"); + headers->append("Accept", "text/plain"); // Issue a GET with modified headers mStatus = HttpStatus(200); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-connection:\\s*keep-alive", boost::regex::icase)); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-accept:\\s*text/plain", boost::regex::icase)); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-accept-encoding:\\s*deflate", boost::regex::icase)); // close enough - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-keep-alive:\\s*120", boost::regex::icase)); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-host:\\s*.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("X-Reflect-accept-encoding:\\s*((gzip|deflate),\\s*)+(gzip|deflate)", boost::regex::icase)); // close enough - handler.mHeadersDisallowed.push_back(boost::regex("X-Reflect-keep-alive:\\s*300", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("X-Reflect-accept:\\s*\\*/\\*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-cache-control:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-pragma:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-range:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-transfer-encoding:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-referer:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-content-type:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-content-encoding:.*", boost::regex::icase)); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-connection", boost::regex::icase), + boost::regex("keep-alive", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-accept", boost::regex::icase), + boost::regex("text/plain", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-accept-encoding", boost::regex::icase), + boost::regex("deflate", boost::regex::icase))); // close enough + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-keep-alive", boost::regex::icase), + boost::regex("120", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-host", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-accept-encoding", boost::regex::icase), + boost::regex("((gzip|deflate),\\s*)+(gzip|deflate)", boost::regex::icase))); // close enough + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-keep-alive", boost::regex::icase), + boost::regex("300", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-accept", boost::regex::icase), + boost::regex("\\*/\\*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-cache-control", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-pragma", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-range", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-transfer-encoding", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-referer", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-content-type", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-content-encoding", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); HttpHandle handle = req->requestGet(HttpRequest::DEFAULT_POLICY_ID, 0U, url_base + "reflect/", @@ -2359,10 +2578,10 @@ void HttpRequestTestObjectType::test<20>() // headers headers = new HttpHeaders(); - headers->mHeaders.push_back("keep-Alive: 120"); - headers->mHeaders.push_back("Accept: text/html"); - headers->mHeaders.push_back("content-type: application/llsd+xml"); - headers->mHeaders.push_back("cache-control: no-store"); + headers->append("keep-Alive", "120"); + headers->append("Accept", "text/html"); + headers->append("content-type", "application/llsd+xml"); + headers->append("cache-control", "no-store"); // And a buffer array const char * msg("<xml><llsd><string>It was the best of times, it was the worst of times.</string></llsd></xml>"); @@ -2371,23 +2590,76 @@ void HttpRequestTestObjectType::test<20>() // Issue a default POST mStatus = HttpStatus(200); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-connection:\\s*keep-alive", boost::regex::icase)); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-accept:\\s*text/html", boost::regex::icase)); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-accept-encoding:\\s*((gzip|deflate),\\s*)+(gzip|deflate)", boost::regex::icase)); // close enough - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-keep-alive:\\s*120", boost::regex::icase)); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-host:\\s*.*", boost::regex::icase)); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-content-length:\\s*\\d+", boost::regex::icase)); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-content-type:\\s*application/llsd\\+xml", boost::regex::icase)); - handler.mHeadersRequired.push_back(boost::regex("\\s*X-Reflect-cache-control:\\s*no-store", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("X-Reflect-content-type:\\s*application/x-www-form-urlencoded", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("X-Reflect-accept:\\s*\\*/\\*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("X-Reflect-keep-alive:\\s*300", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-pragma:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-range:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-referer:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-content-encoding:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-expect:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-transfer-encoding:\\s*.*chunked.*", boost::regex::icase)); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-connection", boost::regex::icase), + boost::regex("keep-alive", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-accept", boost::regex::icase), + boost::regex("text/html", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-accept-encoding", boost::regex::icase), + boost::regex("((gzip|deflate),\\s*)+(gzip|deflate)", boost::regex::icase))); // close enough + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-keep-alive", boost::regex::icase), + boost::regex("120", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-host", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-content-length", boost::regex::icase), + boost::regex("\\d+", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-content-type", boost::regex::icase), + boost::regex("application/llsd\\+xml", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-cache-control", boost::regex::icase), + boost::regex("no-store", boost::regex::icase))); + + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-content-type", boost::regex::icase), + boost::regex("application/x-www-form-urlencoded", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-accept", boost::regex::icase), + boost::regex("\\*/\\*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-keep-alive", boost::regex::icase), + boost::regex("300", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-pragma", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-range", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-referer", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-content-encoding", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-expect", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-transfer-encoding", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + HttpHandle handle = req->requestPost(HttpRequest::DEFAULT_POLICY_ID, 0U, url_base + "reflect/", @@ -2529,9 +2801,9 @@ void HttpRequestTestObjectType::test<21>() // headers headers = new HttpHeaders; - headers->mHeaders.push_back("content-type: text/plain"); - headers->mHeaders.push_back("content-type: text/html"); - headers->mHeaders.push_back("content-type: application/llsd+xml"); + headers->append("content-type", "text/plain"); + headers->append("content-type", "text/html"); + headers->append("content-type", "application/llsd+xml"); // And a buffer array const char * msg("<xml><llsd><string>It was the best of times, it was the worst of times.</string></llsd></xml>"); @@ -2540,22 +2812,71 @@ void HttpRequestTestObjectType::test<21>() // Issue a default PUT mStatus = HttpStatus(200); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-connection:\\s*keep-alive", boost::regex::icase)); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-accept:\\s*\\*/\\*", boost::regex::icase)); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-accept-encoding:\\s*((gzip|deflate),\\s*)+(gzip|deflate)", boost::regex::icase)); // close enough - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-keep-alive:\\s*\\d+", boost::regex::icase)); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-host:\\s*.*", boost::regex::icase)); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-content-length:\\s*\\d+", boost::regex::icase)); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-content-type:\\s*application/llsd\\+xml", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-cache-control:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-pragma:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-range:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-referer:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-content-encoding:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-expect:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-transfer-encoding:\\s*.*chunked.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("X-Reflect-content-type:\\s*text/plain", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("X-Reflect-content-type:\\s*text/html", boost::regex::icase)); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-connection", boost::regex::icase), + boost::regex("keep-alive", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-accept", boost::regex::icase), + boost::regex("\\*/\\*", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-accept-encoding", boost::regex::icase), + boost::regex("((gzip|deflate),\\s*)+(gzip|deflate)", boost::regex::icase))); // close enough + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-keep-alive", boost::regex::icase), + boost::regex("\\d+", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-host", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-content-length", boost::regex::icase), + boost::regex("\\d+", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-content-type", boost::regex::icase), + boost::regex("application/llsd\\+xml", boost::regex::icase))); + + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-cache-control", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-pragma", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-range", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-referer", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-content-encoding", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-expect", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-transfer-encoding", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-content-type", boost::regex::icase), + boost::regex("text/plain", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-content-type", boost::regex::icase), + boost::regex("text/html", boost::regex::icase))); HttpHandle handle = req->requestPut(HttpRequest::DEFAULT_POLICY_ID, 0U, url_base + "reflect/", @@ -2651,6 +2972,142 @@ void HttpRequestTestObjectType::test<21>() } +template <> template <> +void HttpRequestTestObjectType::test<22>() +{ + ScopedCurlInit ready; + + set_test_name("HttpRequest GET 503s with 'Retry-After'"); + + // This tests mainly that the code doesn't fall over if + // various well- and mis-formed Retry-After headers are + // sent along with the response. Direct inspection of + // the parsing result isn't supported. + + // Handler can be stack-allocated *if* there are no dangling + // references to it after completion of this method. + // Create before memory record as the string copy will bump numbers. + TestHandler2 handler(this, "handler"); + std::string url_base(get_base_url() + "/503/"); // path to 503 generators + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + mHandlerCalls = 0; + + HttpRequest * req = NULL; + HttpOptions * opts = NULL; + + try + { + // Get singletons created + HttpRequest::createService(); + + // Start threading early so that thread memory is invariant + // over the test. + HttpRequest::startThread(); + + // create a new ref counted object with an implicit reference + req = new HttpRequest(); + ensure("Memory allocated on construction", mMemTotal < GetMemTotal()); + + opts = new HttpOptions(); + opts->setRetries(1); // Retry once only + opts->setUseRetryAfter(true); // Try to parse the retry-after header + + // Issue a GET that 503s with valid retry-after + mStatus = HttpStatus(503); + int url_limit(6); + for (int i(0); i < url_limit; ++i) + { + std::ostringstream url; + url << url_base << i << "/"; + HttpHandle handle = req->requestGetByteRange(HttpRequest::DEFAULT_POLICY_ID, + 0U, + url.str(), + 0, + 0, + opts, + NULL, + &handler); + + std::ostringstream testtag; + testtag << "Valid handle returned for 503 request #" << i; + ensure(testtag.str(), handle != LLCORE_HTTP_HANDLE_INVALID); + } + + + // Run the notification pump. + int count(0); + int limit(300); // One retry but several seconds needed + while (count++ < limit && mHandlerCalls < url_limit) + { + req->update(0); + usleep(100000); + } + ensure("Request executed in reasonable time", count < limit); + ensure("One handler invocation for request", mHandlerCalls == url_limit); + + // Okay, request a shutdown of the servicing thread + mStatus = HttpStatus(); + mHandlerCalls = 0; + HttpHandle handle = req->requestStopThread(&handler); + ensure("Valid handle returned for second request", handle != LLCORE_HTTP_HANDLE_INVALID); + + // Run the notification pump again + count = 0; + limit = 100; + while (count++ < limit && mHandlerCalls < 1) + { + req->update(1000000); + usleep(100000); + } + ensure("Second request executed in reasonable time", count < limit); + ensure("Second handler invocation", mHandlerCalls == 1); + + // See that we actually shutdown the thread + count = 0; + limit = 10; + while (count++ < limit && ! HttpService::isStopped()) + { + usleep(100000); + } + ensure("Thread actually stopped running", HttpService::isStopped()); + + // release options + opts->release(); + opts = NULL; + + // release the request object + delete req; + req = NULL; + + // Shut down service + HttpRequest::destroyService(); + +#if defined(WIN32) + // Can only do this memory test on Windows. On other platforms, + // the LL logging system holds on to memory and produces what looks + // like memory leaks... + + // printf("Old mem: %d, New mem: %d\n", mMemTotal, GetMemTotal()); + ensure("Memory usage back to that at entry", mMemTotal == GetMemTotal()); +#endif + } + catch (...) + { + stop_thread(req); + if (opts) + { + opts->release(); + opts = NULL; + } + delete req; + HttpRequest::destroyService(); + throw; + } +} + + } // end namespace tut namespace diff --git a/indra/llcorehttp/tests/test_llcorehttp_peer.py b/indra/llcorehttp/tests/test_llcorehttp_peer.py index 75a3c39ef2..f6c4d1a820 100755 --- a/indra/llcorehttp/tests/test_llcorehttp_peer.py +++ b/indra/llcorehttp/tests/test_llcorehttp_peer.py @@ -9,7 +9,7 @@ $LicenseInfo:firstyear=2008&license=viewerlgpl$ Second Life Viewer Source Code -Copyright (C) 2012, Linden Research, Inc. +Copyright (C) 2012-2013, Linden Research, Inc. This library is free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General Public @@ -47,6 +47,17 @@ from testrunner import freeport, run, debug, VERBOSE class TestHTTPRequestHandler(BaseHTTPRequestHandler): """This subclass of BaseHTTPRequestHandler is to receive and echo LLSD-flavored messages sent by the C++ LLHTTPClient. + + [Merge with viewer-cat later] + - '/503/' Generate 503 responses with various kinds + of 'retry-after' headers + -- '/503/0/' "Retry-After: 2" + -- '/503/1/' "Retry-After: Thu, 31 Dec 2043 23:59:59 GMT" + -- '/503/2/' "Retry-After: Fri, 31 Dec 1999 23:59:59 GMT" + -- '/503/3/' "Retry-After: " + -- '/503/4/' "Retry-After: (*#*(@*(@(")" + -- '/503/5/' "Retry-After: aklsjflajfaklsfaklfasfklasdfklasdgahsdhgasdiogaioshdgo" + -- '/503/6/' "Retry-After: 1 2 3 4 5 6 7 8 9 10" """ def read(self): # The following logic is adapted from the library module @@ -107,7 +118,41 @@ class TestHTTPRequestHandler(BaseHTTPRequestHandler): if "/sleep/" in self.path: time.sleep(30) - if "fail" not in self.path: + if "/503/" in self.path: + # Tests for various kinds of 'Retry-After' header parsing + body = None + if "/503/0/" in self.path: + self.send_response(503) + self.send_header("retry-after", "2") + elif "/503/1/" in self.path: + self.send_response(503) + self.send_header("retry-after", "Thu, 31 Dec 2043 23:59:59 GMT") + elif "/503/2/" in self.path: + self.send_response(503) + self.send_header("retry-after", "Fri, 31 Dec 1999 23:59:59 GMT") + elif "/503/3/" in self.path: + self.send_response(503) + self.send_header("retry-after", "") + elif "/503/4/" in self.path: + self.send_response(503) + self.send_header("retry-after", "(*#*(@*(@(") + elif "/503/5/" in self.path: + self.send_response(503) + self.send_header("retry-after", "aklsjflajfaklsfaklfasfklasdfklasdgahsdhgasdiogaioshdgo") + elif "/503/6/" in self.path: + self.send_response(503) + self.send_header("retry-after", "1 2 3 4 5 6 7 8 9 10") + else: + # Unknown request + self.send_response(400) + body = "Unknown /503/ path in server" + if "/reflect/" in self.path: + self.reflect_headers() + self.send_header("Content-type", "text/plain") + self.end_headers() + if body: + self.wfile.write(body) + elif "fail" not in self.path: data = data.copy() # we're going to modify # Ensure there's a "reply" key in data, even if there wasn't before data["reply"] = data.get("reply", llsd.LLSD("success")) |