diff options
Diffstat (limited to 'indra/llcorehttp')
| -rwxr-xr-x | indra/llcorehttp/_httpinternal.h | 6 | ||||
| -rwxr-xr-x | indra/llcorehttp/_httpoperation.cpp | 4 | ||||
| -rwxr-xr-x | indra/llcorehttp/_httpoprequest.cpp | 244 | ||||
| -rwxr-xr-x | indra/llcorehttp/_httpoprequest.h | 4 | ||||
| -rwxr-xr-x | indra/llcorehttp/_httppolicy.cpp | 12 | ||||
| -rwxr-xr-x | indra/llcorehttp/_httppolicyclass.cpp | 4 | ||||
| -rwxr-xr-x | indra/llcorehttp/_httpservice.cpp | 3 | ||||
| -rwxr-xr-x | indra/llcorehttp/httpoptions.cpp | 8 | ||||
| -rwxr-xr-x | indra/llcorehttp/httpoptions.h | 8 |
9 files changed, 222 insertions, 71 deletions
diff --git a/indra/llcorehttp/_httpinternal.h b/indra/llcorehttp/_httpinternal.h index d60996756f..f085ca3b91 100755 --- a/indra/llcorehttp/_httpinternal.h +++ b/indra/llcorehttp/_httpinternal.h @@ -98,7 +98,7 @@ namespace LLCore // Maxium number of policy classes that can be defined. // *TODO: Currently limited to the default class + 1, extend. -const int HTTP_POLICY_CLASS_LIMIT = 4; +const int HTTP_POLICY_CLASS_LIMIT = 8; // Debug/informational tracing. Used both // as a global option and in per-request traces. @@ -138,6 +138,10 @@ const int HTTP_CONNECTION_LIMIT_DEFAULT = 8; const int HTTP_CONNECTION_LIMIT_MIN = 1; const int HTTP_CONNECTION_LIMIT_MAX = 256; +// Miscellaneous defaults +const long HTTP_PIPELINING_DEFAULT = 0L; +const bool HTTP_USE_RETRY_AFTER_DEFAULT = true; + // Tuning parameters // Time worker thread sleeps after a pass through the diff --git a/indra/llcorehttp/_httpoperation.cpp b/indra/llcorehttp/_httpoperation.cpp index 5cf5bc5930..7acd728bbd 100755 --- a/indra/llcorehttp/_httpoperation.cpp +++ b/indra/llcorehttp/_httpoperation.cpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -53,7 +53,7 @@ HttpOperation::HttpOperation() mUserHandler(NULL), mReqPolicy(HttpRequest::DEFAULT_POLICY_ID), mReqPriority(0U), - mTracing(0) + mTracing(HTTP_TRACE_OFF) { mMetricCreated = totalTime(); } diff --git a/indra/llcorehttp/_httpoprequest.cpp b/indra/llcorehttp/_httpoprequest.cpp index a4c0a12fdc..8cb7fee701 100755 --- a/indra/llcorehttp/_httpoprequest.cpp +++ b/indra/llcorehttp/_httpoprequest.cpp @@ -64,6 +64,15 @@ int parse_content_range_header(char * buffer, unsigned int * last, unsigned int * length); +// Similar for Retry-After headers. Only parses the delta form +// of the header, HTTP time formats aren't interesting for client +// purposes. +// +// @return 0 if successfully parsed and seconds time delta +// returned in time argument. +// +int parse_retry_after_header(char * buffer, int * time); + // Take data from libcurl's CURLOPT_DEBUGFUNCTION callback and // escape and format it for a tracing line in logging. Absolutely @@ -74,14 +83,12 @@ void escape_libcurl_debug_data(char * buffer, size_t len, bool scrub, std::string & safe_line); -// OS-neutral string comparisons of various types -int os_strncasecmp(const char *s1, const char *s2, size_t n); -int os_strcasecmp(const char *s1, const char *s2); -char * os_strtok_r(char *str, const char *delim, char **saveptr); - - -static const char * const hdr_whitespace(" \t"); -static const char * const hdr_separator(": \t"); +// OS-neutral string comparisons of various types. +int os_strcasecmp(const char * s1, const char * s2); +char * os_strtok_r(char * str, const char * delim, char ** saveptr); +char * os_strtrim(char * str); +char * os_strltrim(char * str); +void os_strlower(char * str); } // end anonymous namespace @@ -104,6 +111,8 @@ HttpOpRequest::HttpOpRequest() mCurlService(NULL), mCurlHeaders(NULL), mCurlBodyPos(0), + mCurlTemp(NULL), + mCurlTempLen(0), mReplyBody(NULL), mReplyOffset(0), mReplyLength(0), @@ -154,6 +163,10 @@ HttpOpRequest::~HttpOpRequest() mCurlHeaders = NULL; } + delete [] mCurlTemp; + mCurlTemp = NULL; + mCurlTempLen = 0; + if (mReplyBody) { mReplyBody->release(); @@ -207,6 +220,11 @@ void HttpOpRequest::stageFromActive(HttpService * service) mCurlHeaders = NULL; } + // Also not needed on the other side + delete [] mCurlTemp; + mCurlTemp = NULL; + mCurlTempLen = 0; + addAsReply(); } @@ -335,6 +353,10 @@ void HttpOpRequest::setupCommon(HttpRequest::policy_t policy_id, { mProcFlags |= PF_SAVE_HEADERS; } + if (options->getUseRetryAfter()) + { + mProcFlags |= PF_USE_RETRY_AFTER; + } mPolicyRetryLimit = options->getRetries(); mPolicyRetryLimit = llclamp(mPolicyRetryLimit, HTTP_RETRY_COUNT_MIN, HTTP_RETRY_COUNT_MAX); mTracing = (std::max)(mTracing, llclamp(options->getTrace(), HTTP_TRACE_MIN, HTTP_TRACE_MAX)); @@ -549,7 +571,7 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service) } curl_easy_setopt(mCurlHandle, CURLOPT_HTTPHEADER, mCurlHeaders); - if (mProcFlags & (PF_SCAN_RANGE_HEADER | PF_SAVE_HEADERS)) + if (mProcFlags & (PF_SCAN_RANGE_HEADER | PF_SAVE_HEADERS | PF_USE_RETRY_AFTER)) { curl_easy_setopt(mCurlHandle, CURLOPT_HEADERFUNCTION, headerCallback); curl_easy_setopt(mCurlHandle, CURLOPT_HEADERDATA, this); @@ -610,10 +632,9 @@ size_t HttpOpRequest::headerCallback(void * data, size_t size, size_t nmemb, voi { static const char status_line[] = "HTTP/"; static const size_t status_line_len = sizeof(status_line) - 1; - - static const char con_ran_line[] = "content-range:"; - static const size_t con_ran_line_len = sizeof(con_ran_line) - 1; - + static const char con_ran_line[] = "content-range"; + static const char con_retry_line[] = "retry-after"; + HttpOpRequest * op(static_cast<HttpOpRequest *>(userdata)); const size_t hdr_size(size * nmemb); @@ -627,6 +648,7 @@ size_t HttpOpRequest::headerCallback(void * data, size_t size, size_t nmemb, voi op->mReplyOffset = 0; op->mReplyLength = 0; op->mReplyFullLength = 0; + op->mReplyRetryAfter = 0; op->mStatus = HttpStatus(); if (op->mReplyHeaders) { @@ -645,6 +667,53 @@ size_t HttpOpRequest::headerCallback(void * data, size_t size, size_t nmemb, voi --wanted_hdr_size; } } + + // Copy and normalize header fragments for the following + // stages. Would like to modify the data in-place but that + // may not be allowed and we need one byte extra for NUL. + // At the end of this we will have: + // + // If ':' present in header: + // 1. name points to text to left of colon which + // will be ascii lower-cased and left and right + // trimmed of whitespace. + // 2. value points to text to right of colon which + // will be left trimmed of whitespace. + // Otherwise: + // 1. name points to header which will be left + // trimmed of whitespace. + // 2. value is NULL + // Any non-NULL pointer may point to a zero-length string. + // + if (wanted_hdr_size >= op->mCurlTempLen) + { + delete [] op->mCurlTemp; + op->mCurlTempLen = 2 * wanted_hdr_size + 1; + op->mCurlTemp = new char [op->mCurlTempLen]; + } + memcpy(op->mCurlTemp, hdr_data, wanted_hdr_size); + op->mCurlTemp[wanted_hdr_size] = '\0'; + char * name(op->mCurlTemp); + char * value(strchr(name, ':')); + if (value) + { + *value++ = '\0'; + os_strlower(name); + name = os_strtrim(name); + value = os_strltrim(value); + } + else + { + // Doesn't look well-formed, do minimal normalization on it + name = os_strltrim(name); + } + + // Normalized, now reject headers with empty names. + if (! *name) + { + // No use continuing + return hdr_size; + } // Save header if caller wants them in the response if (is_header && op->mProcFlags & PF_SAVE_HEADERS) @@ -654,43 +723,53 @@ size_t HttpOpRequest::headerCallback(void * data, size_t size, size_t nmemb, voi { op->mReplyHeaders = new HttpHeaders; } - op->mReplyHeaders->appendNormal(hdr_data, wanted_hdr_size); + op->mReplyHeaders->append(name, value ? value : ""); } + // From this point, header-specific processors are free to + // modify the header value. + // Detect and parse 'Content-Range' headers - if (is_header && op->mProcFlags & PF_SCAN_RANGE_HEADER) + if (is_header + && op->mProcFlags & PF_SCAN_RANGE_HEADER + && value && *value + && ! strcmp(name, con_ran_line)) { - char hdr_buffer[128]; // Enough for a reasonable header - size_t frag_size((std::min)(wanted_hdr_size, sizeof(hdr_buffer) - 1)); - - memcpy(hdr_buffer, hdr_data, frag_size); - hdr_buffer[frag_size] = '\0'; - if (frag_size > con_ran_line_len && - ! os_strncasecmp(hdr_buffer, con_ran_line, con_ran_line_len)) + unsigned int first(0), last(0), length(0); + int status; + + if (! (status = parse_content_range_header(value, &first, &last, &length))) + { + // Success, record the fragment position + op->mReplyOffset = first; + op->mReplyLength = last - first + 1; + op->mReplyFullLength = length; + } + else if (-1 == status) { - unsigned int first(0), last(0), length(0); - int status; + // Response is badly formed and shouldn't be accepted + op->mStatus = HttpStatus(HttpStatus::LLCORE, HE_INV_CONTENT_RANGE_HDR); + } + else + { + // Ignore the unparsable. + LL_INFOS_ONCE("CoreHttp") << "Problem parsing odd Content-Range header: '" + << std::string(hdr_data, wanted_hdr_size) + << "'. Ignoring." + << LL_ENDL; + } + } - if (! (status = parse_content_range_header(hdr_buffer, &first, &last, &length))) - { - // Success, record the fragment position - op->mReplyOffset = first; - op->mReplyLength = last - first + 1; - op->mReplyFullLength = length; - } - else if (-1 == status) - { - // Response is badly formed and shouldn't be accepted - op->mStatus = HttpStatus(HttpStatus::LLCORE, HE_INV_CONTENT_RANGE_HDR); - } - else - { - // Ignore the unparsable. - LL_INFOS_ONCE("CoreHttp") << "Problem parsing odd Content-Range header: '" - << std::string(hdr_data, frag_size) - << "'. Ignoring." - << LL_ENDL; - } + // Detect and parse 'Retry-After' headers + if (is_header + && op->mProcFlags & PF_USE_RETRY_AFTER + && value && *value + && ! strcmp(name, con_retry_line)) + { + int time(0); + if (! parse_retry_after_header(value, &time)) + { + op->mReplyRetryAfter = time; } } @@ -805,14 +884,16 @@ int parse_content_range_header(char * buffer, unsigned int * last, unsigned int * length) { + static const char * const hdr_whitespace(" \t"); + char * tok_state(NULL), * tok(NULL); bool match(true); - if (! os_strtok_r(buffer, hdr_separator, &tok_state)) + if (! (tok = os_strtok_r(buffer, hdr_whitespace, &tok_state))) match = false; - if (match && (tok = os_strtok_r(NULL, hdr_whitespace, &tok_state))) - match = 0 == os_strcasecmp("bytes", tok); - if (match && ! (tok = os_strtok_r(NULL, " \t", &tok_state))) + else + match = (0 == os_strcasecmp("bytes", tok)); + if (match && ! (tok = os_strtok_r(NULL, hdr_whitespace, &tok_state))) match = false; if (match) { @@ -851,6 +932,25 @@ int parse_content_range_header(char * buffer, } +int parse_retry_after_header(char * buffer, int * time) +{ + char * endptr(buffer); + long lcl_time(strtol(buffer, &endptr, 10)); + if (*endptr == '\0' && endptr != buffer && lcl_time > 0) + { + *time = lcl_time; + return 0; + } + + // Could attempt to parse HTTP time here but we're not really + // interested in it. Scheduling based on wallclock time on + // user hardware will lead to tears. + + // Header is there but badly/unexpectedly formed, try to ignore it. + return 1; +} + + void escape_libcurl_debug_data(char * buffer, size_t len, bool scrub, std::string & safe_line) { std::string out; @@ -887,15 +987,6 @@ void escape_libcurl_debug_data(char * buffer, size_t len, bool scrub, std::strin } -int os_strncasecmp(const char *s1, const char *s2, size_t n) -{ -#if LL_WINDOWS - return _strnicmp(s1, s2, n); -#else - return strncasecmp(s1, s2, n); -#endif // LL_WINDOWS -} - int os_strcasecmp(const char *s1, const char *s2) { @@ -917,6 +1008,45 @@ char * os_strtok_r(char *str, const char *delim, char ** savestate) } +void os_strlower(char * str) +{ + for (char c(0); (c = *str); ++str) + { + *str = tolower(c); + } +} + + +char * os_strtrim(char * lstr) +{ + while (' ' == *lstr || '\t' == *lstr) + { + ++lstr; + } + if (*lstr) + { + for (char * rstr(lstr + strlen(lstr)); *--rstr;) + { + if (' ' == *rstr || '\t' == *rstr) + { + *rstr = '\0'; + } + } + } + return lstr; +} + + +char * os_strltrim(char * lstr) +{ + while (' ' == *lstr || '\t' == *lstr) + { + ++lstr; + } + return lstr; +} + + } // end anonymous namespace diff --git a/indra/llcorehttp/_httpoprequest.h b/indra/llcorehttp/_httpoprequest.h index 831e5bebf7..2e737cf1cc 100755 --- a/indra/llcorehttp/_httpoprequest.h +++ b/indra/llcorehttp/_httpoprequest.h @@ -158,6 +158,7 @@ protected: unsigned int mProcFlags; static const unsigned int PF_SCAN_RANGE_HEADER = 0x00000001U; static const unsigned int PF_SAVE_HEADERS = 0x00000002U; + static const unsigned int PF_USE_RETRY_AFTER = 0x00000004U; public: // Request data @@ -175,6 +176,8 @@ public: HttpService * mCurlService; curl_slist * mCurlHeaders; size_t mCurlBodyPos; + char * mCurlTemp; // Scratch buffer for header processing + size_t mCurlTempLen; // Result data HttpStatus mStatus; @@ -184,6 +187,7 @@ public: size_t mReplyFullLength; HttpHeaders * mReplyHeaders; std::string mReplyConType; + int mReplyRetryAfter; // Policy data int mPolicyRetries; diff --git a/indra/llcorehttp/_httppolicy.cpp b/indra/llcorehttp/_httppolicy.cpp index 54c9c6bb1b..5f303dd0fe 100755 --- a/indra/llcorehttp/_httppolicy.cpp +++ b/indra/llcorehttp/_httppolicy.cpp @@ -160,8 +160,12 @@ void HttpPolicy::retryOp(HttpOpRequest * op) const HttpTime now(totalTime()); const int policy_class(op->mReqPolicy); - - const HttpTime delta(retry_deltas[llclamp(op->mPolicyRetries, 0, delta_max)]); + HttpTime delta(retry_deltas[llclamp(op->mPolicyRetries, 0, delta_max)]); + + if (op->mReplyRetryAfter > 0 && op->mReplyRetryAfter < 30) + { + delta = op->mReplyRetryAfter * U64L(1000000); + } op->mPolicyRetryAt = now + delta; ++op->mPolicyRetries; if (error_503 == op->mStatus) @@ -170,10 +174,10 @@ void HttpPolicy::retryOp(HttpOpRequest * op) } LL_WARNS("CoreHttp") << "HTTP request " << static_cast<HttpHandle>(op) << " retry " << op->mPolicyRetries - << " scheduled for +" << (delta / HttpTime(1000)) + << " scheduled in " << (delta / HttpTime(1000)) << " mS. Status: " << op->mStatus.toHex() << LL_ENDL; - if (op->mTracing > 0) + if (op->mTracing > HTTP_TRACE_OFF) { LL_INFOS("CoreHttp") << "TRACE, ToRetryQueue, Handle: " << static_cast<HttpHandle>(op) diff --git a/indra/llcorehttp/_httppolicyclass.cpp b/indra/llcorehttp/_httppolicyclass.cpp index a23b81322c..1a55ab1ac6 100755 --- a/indra/llcorehttp/_httppolicyclass.cpp +++ b/indra/llcorehttp/_httppolicyclass.cpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -37,7 +37,7 @@ HttpPolicyClass::HttpPolicyClass() : mSetMask(0UL), mConnectionLimit(HTTP_CONNECTION_LIMIT_DEFAULT), mPerHostConnectionLimit(HTTP_CONNECTION_LIMIT_DEFAULT), - mPipelining(0) + mPipelining(HTTP_PIPELINING_DEFAULT) {} diff --git a/indra/llcorehttp/_httpservice.cpp b/indra/llcorehttp/_httpservice.cpp index 0825888d0f..0821401289 100755 --- a/indra/llcorehttp/_httpservice.cpp +++ b/indra/llcorehttp/_httpservice.cpp @@ -55,9 +55,6 @@ HttpService::HttpService() { // Create the default policy class HttpPolicyClass pol_class; - pol_class.set(HttpRequest::CP_CONNECTION_LIMIT, HTTP_CONNECTION_LIMIT_DEFAULT); - pol_class.set(HttpRequest::CP_PER_HOST_CONNECTION_LIMIT, HTTP_CONNECTION_LIMIT_DEFAULT); - pol_class.set(HttpRequest::CP_ENABLE_PIPELINING, 0L); mPolicyClasses.push_back(pol_class); } diff --git a/indra/llcorehttp/httpoptions.cpp b/indra/llcorehttp/httpoptions.cpp index 4dcd862ca4..5bf1ecb4a5 100755 --- a/indra/llcorehttp/httpoptions.cpp +++ b/indra/llcorehttp/httpoptions.cpp @@ -39,7 +39,8 @@ HttpOptions::HttpOptions() mTracing(HTTP_TRACE_OFF), mTimeout(HTTP_REQUEST_TIMEOUT_DEFAULT), mTransferTimeout(HTTP_REQUEST_XFER_TIMEOUT_DEFAULT), - mRetries(HTTP_RETRY_COUNT_DEFAULT) + mRetries(HTTP_RETRY_COUNT_DEFAULT), + mUseRetryAfter(HTTP_USE_RETRY_AFTER_DEFAULT) {} @@ -76,5 +77,10 @@ void HttpOptions::setRetries(unsigned int retries) mRetries = retries; } +void HttpOptions::setUseRetryAfter(bool use_retry) +{ + mUseRetryAfter = use_retry; +} + } // end namespace LLCore diff --git a/indra/llcorehttp/httpoptions.h b/indra/llcorehttp/httpoptions.h index 623d71d3e6..04531425d8 100755 --- a/indra/llcorehttp/httpoptions.h +++ b/indra/llcorehttp/httpoptions.h @@ -98,13 +98,19 @@ public: return mRetries; } + void setUseRetryAfter(bool use_retry); + bool getUseRetryAfter() const + { + return mUseRetryAfter; + } + protected: bool mWantHeaders; int mTracing; unsigned int mTimeout; unsigned int mTransferTimeout; unsigned int mRetries; - + bool mUseRetryAfter; }; // end class HttpOptions |