From 5611cb6d476540e6a1c654c1f9acdce2787b3505 Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Mon, 23 Apr 2012 16:19:39 -0400 Subject: Okay, imported the core-http library and got it compiling suspiciously easily. The unit/integration tests don't work yet as I'm still battling cmake/autobuild as usual but first milestone passed. --- indra/llcorehttp/httpcommon.h | 204 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 204 insertions(+) create mode 100644 indra/llcorehttp/httpcommon.h (limited to 'indra/llcorehttp/httpcommon.h') diff --git a/indra/llcorehttp/httpcommon.h b/indra/llcorehttp/httpcommon.h new file mode 100644 index 0000000000..617286fb38 --- /dev/null +++ b/indra/llcorehttp/httpcommon.h @@ -0,0 +1,204 @@ +/** + * @file httpcommon.h + * @brief Public-facing declarations and definitions of common types + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef _LLCORE_HTTP_COMMON_H_ +#define _LLCORE_HTTP_COMMON_H_ + +/// @package LLCore::HTTP +/// +/// This library implements a high-level, Indra-code-free client interface to +/// HTTP services based on actual patterns found in the viewer and simulator. +/// Interfaces are similar to those supplied by the legacy classes +/// LLCurlRequest and LLHTTPClient. To that is added a policy scheme that +/// allows an application to specify connection behaviors: limits on +/// connections, HTTP keepalive, HTTP pipelining, retry-on-error limits, etc. +/// +/// Features of the library include: +/// - Single, private working thread where all transport and processing occurs. +/// - Support for multiple consumers running in multiple threads. +/// - Scatter/gather (a.k.a. buffer array) model for bulk data movement. +/// - Reference counting used for many object instance lifetimes. +/// - Minimal data sharing across threads for correctness and low latency. +/// +/// The public interface is declared in a few key header files: +/// - +/// - +/// - +/// - +/// - +/// - +/// - +/// +/// The library is still under early development and particular users +/// may need access to internal implementation details that are found +/// in the _*.h header files. But this is a crutch to be avoided if at +/// all possible and probably indicates some interface work is neeeded. +/// +/// Using the library is fairly easy. Global setup needs a few +/// steps: +/// +/// - libcurl initialization with thread-safely callbacks for c-ares +/// DNS lookups. +/// - HttpRequest::createService() called to instantiate singletons +/// and support objects. +/// +/// An HTTP consumer in an application, and an application may have many +/// consumers, does a few things: +/// +/// - Instantiate and retain an object based on HttpRequest. This +/// object becomes the portal into runtime services for the consumer. +/// - Derive or mixin the HttpHandler class if you want notification +/// when requests succeed or fail. This object's onCompleted() +/// method is invoked and an instance can be shared across +/// requests. +/// +/// Issuing a request is straightforward: +/// - Construct a suitable URL. +/// - Configure HTTP options for the request. (optional) +/// - Build a list of additional headers. (optional) +/// - Invoke one of the requestXXXX() methods (requestGetByteRange, +/// requestPost, etc.) on the HttpRequest instance supplying the +/// above along with a policy class, a priority and an optional +/// pointer to an HttpHandler instance. Work is then queued to +/// the worker thread and occurs asynchronously. +/// - Periodically invoke the update() method on the HttpRequest +/// instance which performs completion notification to HttpHandler +/// objects. +/// - Do completion processing in your onCompletion() method. +/// +/// Code fragments: +/// +/// + +#include + + +namespace LLCore +{ + + +/// All queued requests are represented by an HttpHandle value. +/// The invalid value is returned when a request failed to queue. +/// The actual status for these failures is then fetched with +/// HttpRequest::getStatus(). +/// +/// The handle is valid only for the life of a request. On +/// return from any HttpHandler notification, the handle immediately +/// becomes invalid and may be recycled for other queued requests. + +typedef void * HttpHandle; +#define LLCORE_HTTP_HANDLE_INVALID (NULL) + + +/// Error codes defined by the library itself as distinct from +/// libcurl (or any other transport provider). +enum HttpError +{ + // Successful value compatible with the libcurl codes. + HE_SUCCESS = 0, + + // Service is shutting down and requested operation will + // not be queued or performed. + HE_SHUTTING_DOWN = 1, + + // Operation was canceled by request. + HE_OP_CANCELED = 2, + + // Invalid content range header received. + HE_INV_CONTENT_RANGE_HDR = 3 + +}; // end enum HttpError + + +/// HttpStatus encapsulates errors from libcurl (easy, multi) as well as +/// internal errors. The encapsulation isn't expected to completely +/// isolate the caller from libcurl but basic operational tests (success +/// or failure) are provided. +struct HttpStatus +{ + typedef unsigned short type_enum_t; + + HttpStatus() + : mType(LLCORE), + mStatus(HE_SUCCESS) + {} + + HttpStatus(type_enum_t type, short status) + : mType(type), + mStatus(status) + {} + + HttpStatus(const HttpStatus & rhs) + : mType(rhs.mType), + mStatus(rhs.mStatus) + {} + + HttpStatus & operator=(const HttpStatus & rhs) + { + // Don't care if lhs & rhs are the same object + + mType = rhs.mType; + mStatus = rhs.mStatus; + return *this; + } + + static const type_enum_t EXT_CURL_EASY = 0; + static const type_enum_t EXT_CURL_MULTI = 1; + static const type_enum_t LLCORE = 2; + + type_enum_t mType; + short mStatus; + + /// Test for successful status in the code regardless + /// of error source (internal, libcurl). + /// + /// @return 'true' when status is successful. + /// + operator bool() const + { + return 0 == mStatus; + } + + /// Inverse of previous operator. + /// + /// @return 'true' on any error condition + bool operator !() const + { + return 0 != mStatus; + } + + /// Convert status to a string representation. For + /// success, returns an empty string. For failure + /// statuses, a string as appropriate for the source of + /// the error code (libcurl easy, libcurl multi, or + /// LLCore itself). + std::string toString() const; + +}; // end struct HttpStatus + +} // end namespace LLCore + +#endif // _LLCORE_HTTP_COMMON_H_ -- cgit v1.2.3 From 438a6431e418eac5a3a4e00f7adfe379994869d7 Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Thu, 26 Apr 2012 11:35:07 -0400 Subject: Bring llcorehttp into the compile and link phases. Windows looks okay though it's a dead library so far. --- indra/llcorehttp/httpcommon.h | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'indra/llcorehttp/httpcommon.h') diff --git a/indra/llcorehttp/httpcommon.h b/indra/llcorehttp/httpcommon.h index 617286fb38..cd7c09f097 100644 --- a/indra/llcorehttp/httpcommon.h +++ b/indra/llcorehttp/httpcommon.h @@ -44,13 +44,13 @@ /// - Minimal data sharing across threads for correctness and low latency. /// /// The public interface is declared in a few key header files: -/// - -/// - -/// - -/// - -/// - -/// - -/// - +/// - "llcorehttp/bufferarray.h" +/// - "llcorehttp/httpcommon.h" +/// - "llcorehttp/httphandler.h" +/// - "llcorehttp/httpheaders.h" +/// - "llcorehttp/httpoptions.h" +/// - "llcorehttp/httprequest.h" +/// - "llcorehttp/httpresponse.h" /// /// The library is still under early development and particular users /// may need access to internal implementation details that are found -- cgit v1.2.3 From 74d59e7128bb02a4b49af99e44f437a736a3f62b Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Mon, 7 May 2012 15:16:31 -0400 Subject: Build llcorehttp as part of a viewer dependency with unit tests. This required boost::thread and the easiest path to that was to go with the 1.48 Boost release in the 3P tree (eliminating a fork for a modified 1.45 packaging). One unit test, the most important one, is failing in test_httprequest but that can be attended to later. This test issues a GET to http://localhost:2/ and that is hitting the wire but the libcurl plumbing isn't delivering the failure, only the eventual timeout. An unexpected change in behavior. --- indra/llcorehttp/httpcommon.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'indra/llcorehttp/httpcommon.h') diff --git a/indra/llcorehttp/httpcommon.h b/indra/llcorehttp/httpcommon.h index cd7c09f097..5fc497e720 100644 --- a/indra/llcorehttp/httpcommon.h +++ b/indra/llcorehttp/httpcommon.h @@ -93,6 +93,8 @@ /// /// +#include "linden_common.h" + #include -- cgit v1.2.3 From 239e072bfcf97b8a12c18ff9974fd0a2929c9ee4 Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Tue, 8 May 2012 12:27:24 -0400 Subject: Unit test still giving me issues on the local windows system. Seems to be a hard stall while allocating the first easy handle in a descent of the global initiailization code but that doesn't seem to be a problem on TC machines. Perhaps the static linking is creating multiple data copies. More work needed. --- indra/llcorehttp/httpcommon.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'indra/llcorehttp/httpcommon.h') diff --git a/indra/llcorehttp/httpcommon.h b/indra/llcorehttp/httpcommon.h index 5fc497e720..9de5769d57 100644 --- a/indra/llcorehttp/httpcommon.h +++ b/indra/llcorehttp/httpcommon.h @@ -93,7 +93,7 @@ /// /// -#include "linden_common.h" +#include "linden_common.h" // Modifies curl/curl.h interfaces #include -- cgit v1.2.3 From 8fc350125c671baeae6b7f8b1814251009f4f50a Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Wed, 23 May 2012 19:12:09 -0400 Subject: Integrate llcorehttp library into lltexturefetch design. This is the first functional viewer pass with the HTTP work of the texture fetch code performed by the llcorehttp library. Not exactly a 'drop-in' replacement but a work-alike with some changes (e.g. handler notification in consumer thread versus responder notification in worker thread). This also includes some temporary changes in the priority scheme to prevent the kind of priority inversion found in VWR-28996. Scheme used here does provide liveness if not optimal responsiveness or order-of-operation. The llcorehttp library at this point is far from optimally performing. Its worker thread is making relatively poor use of cycles it gets and it doesn't idle or sleep intelligently yet. This early integration step helps shake out the interfaces, implementation niceties will be covered soon. --- indra/llcorehttp/httpcommon.h | 59 ++++++++++++++++++++++++++++++++++++++----- 1 file changed, 52 insertions(+), 7 deletions(-) (limited to 'indra/llcorehttp/httpcommon.h') diff --git a/indra/llcorehttp/httpcommon.h b/indra/llcorehttp/httpcommon.h index 9de5769d57..f81be7103e 100644 --- a/indra/llcorehttp/httpcommon.h +++ b/indra/llcorehttp/httpcommon.h @@ -122,23 +122,38 @@ enum HttpError // Successful value compatible with the libcurl codes. HE_SUCCESS = 0, + // Intended for HTTP reply codes 100-999, indicates that + // the reply should be considered an error by the application. + HE_REPLY_ERROR = 1, + // Service is shutting down and requested operation will // not be queued or performed. - HE_SHUTTING_DOWN = 1, + HE_SHUTTING_DOWN = 2, // Operation was canceled by request. - HE_OP_CANCELED = 2, + HE_OP_CANCELED = 3, // Invalid content range header received. - HE_INV_CONTENT_RANGE_HDR = 3 + HE_INV_CONTENT_RANGE_HDR = 4 }; // end enum HttpError -/// HttpStatus encapsulates errors from libcurl (easy, multi) as well as -/// internal errors. The encapsulation isn't expected to completely -/// isolate the caller from libcurl but basic operational tests (success -/// or failure) are provided. +/// HttpStatus encapsulates errors from libcurl (easy, multi), HTTP +/// reply status codes and internal errors as well. The encapsulation +/// isn't expected to completely isolate the caller from libcurl but +/// basic operational tests (success or failure) are provided. +/// +/// Non-HTTP status are encoded as (type, status) with type being +/// one of: EXT_CURL_EASY, EXT_CURL_MULTI or LLCORE and status +/// being the success/error code from that domain. HTTP status +/// is encoded as (status, error_flag). Status should be in the +/// range [100, 999] and error_flag is either HE_SUCCESS or +/// HE_REPLY_ERROR to indicate whether this should be treated as +/// a successful status or an error. The application is responsible +/// for making that determination and a range like [200, 299] isn't +/// automatically assumed to be definitive. + struct HttpStatus { typedef unsigned short type_enum_t; @@ -192,12 +207,42 @@ struct HttpStatus return 0 != mStatus; } + /// Equality and inequality tests to bypass bool conversion + /// which will do the wrong thing in conditional expressions. + bool operator==(const HttpStatus & rhs) const + { + return mType == rhs.mType && mStatus == rhs.mStatus; + } + + bool operator!=(const HttpStatus & rhs) const + { + return ! operator==(rhs); + } + + /// Convert to single numeric representation. Mainly + /// for logging or other informal purposes. Also + /// creates an ambiguous second path to integer conversion + /// which tends to find programming errors such as formatting + /// the status to a stream (operator<<). + operator unsigned long() const; + unsigned long toULong() const + { + return operator unsigned long(); + } + /// Convert status to a string representation. For /// success, returns an empty string. For failure /// statuses, a string as appropriate for the source of /// the error code (libcurl easy, libcurl multi, or /// LLCore itself). std::string toString() const; + + /// Returns true if the status value represents an + /// HTTP response status (100 - 999). + bool isHttpStatus() const + { + return mType >= type_enum_t(100) && mType <= type_enum_t(999); + } }; // end struct HttpStatus -- cgit v1.2.3 From b8edacd0bb4feacc3ac1d61421e600c75ab87f7c Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Fri, 1 Jun 2012 14:07:34 -0400 Subject: Major steps towards implementing the policy component. Identified and reacted to the priority inversion problem we have in texturefetch. Includes the introduction of a priority_queue for the requests that are ready. Start some parameterization in anticipation of having policy_class everywhere. Removed _assert.h which isn't really needed in indra codebase. Implemented async setPriority request (which I hope I can get rid of eventually along with all priorities in this library). Converted to using unsigned int for priority rather than float. Implemented POST and did groundwork for PUT. --- indra/llcorehttp/httpcommon.h | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'indra/llcorehttp/httpcommon.h') diff --git a/indra/llcorehttp/httpcommon.h b/indra/llcorehttp/httpcommon.h index f81be7103e..c01a5f85d3 100644 --- a/indra/llcorehttp/httpcommon.h +++ b/indra/llcorehttp/httpcommon.h @@ -134,7 +134,10 @@ enum HttpError HE_OP_CANCELED = 3, // Invalid content range header received. - HE_INV_CONTENT_RANGE_HDR = 4 + HE_INV_CONTENT_RANGE_HDR = 4, + + // Request handle not found + HE_HANDLE_NOT_FOUND = 5 }; // end enum HttpError @@ -229,6 +232,9 @@ struct HttpStatus { return operator unsigned long(); } + + /// And to convert to a hex string. + std::string toHex() const; /// Convert status to a string representation. For /// success, returns an empty string. For failure -- cgit v1.2.3 From 05af16a23abe37210e0b880aa27387d8994419dd Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Wed, 6 Jun 2012 13:52:38 -0400 Subject: Policy + caching fixes + https support + POST working Implemented first global policy definitions to support SSL CA certificate configuration to support https: operations. Fixed HTTP 206 status handling to match what is currently being done by grid services and to lay a foundation for fixes that will be a response to ER-1824. More libcurl CURLOPT options set on easy handles to do peer verification in the traditional way. HTTP POST working and now reporting asset metrics back to grid for the viewer's asset system. This uses LLSD so that is also showing as compatible with the new library. --- indra/llcorehttp/httpcommon.h | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) (limited to 'indra/llcorehttp/httpcommon.h') diff --git a/indra/llcorehttp/httpcommon.h b/indra/llcorehttp/httpcommon.h index c01a5f85d3..fd2661b700 100644 --- a/indra/llcorehttp/httpcommon.h +++ b/indra/llcorehttp/httpcommon.h @@ -137,7 +137,16 @@ enum HttpError HE_INV_CONTENT_RANGE_HDR = 4, // Request handle not found - HE_HANDLE_NOT_FOUND = 5 + HE_HANDLE_NOT_FOUND = 5, + + // Invalid datatype for option/setting + HE_INVALID_ARG = 6, + + // Option hasn't been explicitly set + HE_OPT_NOT_SET = 7, + + // Option not dynamic, must be set during init phase + HE_OPT_NOT_DYNAMIC = 8 }; // end enum HttpError -- cgit v1.2.3 From 28a04400b4160dd34166483ddcf0c12637bcc363 Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Fri, 8 Jun 2012 20:21:54 -0400 Subject: Implemented HTTP retry for requests. Went in rather easily which surprised me. Added a retry queue similar to ready queue to the policy object which is sorted by retry time. Currently do five retries (after the initial try) delayed by .25, .5, 1, 2 and 5 seconds. Removed the retry logic from the lltexturefetch module. Upped the waiting time in the unit test for the retries. People won't like this but tough, need tests. --- indra/llcorehttp/httpcommon.h | 12 ++++++++++++ 1 file changed, 12 insertions(+) (limited to 'indra/llcorehttp/httpcommon.h') diff --git a/indra/llcorehttp/httpcommon.h b/indra/llcorehttp/httpcommon.h index fd2661b700..42b75edb41 100644 --- a/indra/llcorehttp/httpcommon.h +++ b/indra/llcorehttp/httpcommon.h @@ -114,6 +114,9 @@ namespace LLCore typedef void * HttpHandle; #define LLCORE_HTTP_HANDLE_INVALID (NULL) +/// For internal scheduling and metrics, we use a microsecond +/// timebase compatible with the environment. +typedef U64 HttpTime; /// Error codes defined by the library itself as distinct from /// libcurl (or any other transport provider). @@ -180,6 +183,15 @@ struct HttpStatus mStatus(status) {} + HttpStatus(int http_status) + : mType(http_status), + mStatus(http_status >= 200 && http_status <= 299 + ? HE_SUCCESS + : HE_REPLY_ERROR) + { + llassert(http_status >= 100 && http_status <= 999); + } + HttpStatus(const HttpStatus & rhs) : mType(rhs.mType), mStatus(rhs.mStatus) -- cgit v1.2.3 From f5f51d3cda8861b3b3a380cc96aaca98e572c377 Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Sat, 7 Jul 2012 19:35:32 -0400 Subject: SH-3185 Fill in some FIXME/TODO cases Also added some comments and changed the callback userdata argument to be an HttpOpRequest rather than a libcurl handle. Less code, less clutter. --- indra/llcorehttp/httpcommon.h | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) (limited to 'indra/llcorehttp/httpcommon.h') diff --git a/indra/llcorehttp/httpcommon.h b/indra/llcorehttp/httpcommon.h index 42b75edb41..576a113e54 100644 --- a/indra/llcorehttp/httpcommon.h +++ b/indra/llcorehttp/httpcommon.h @@ -168,6 +168,24 @@ enum HttpError /// a successful status or an error. The application is responsible /// for making that determination and a range like [200, 299] isn't /// automatically assumed to be definitive. +/// +/// Examples: +/// +/// 1. Construct a default, successful status code: +/// HttpStatus(); +/// +/// 2. Construct a successful, HTTP 200 status code: +/// HttpStatus(200); +/// +/// 3. Construct a failed, HTTP 404 not-found status code: +/// HttpStatus(404); +/// +/// 4. Construct a failed libcurl couldn't connect status code: +/// HttpStatus(HttpStatus::EXT_CURL_EASY, CURLE_COULDNT_CONNECT); +/// +/// 5. Construct an HTTP 301 status code to be treated as success: +/// HttpStatus(301, HE_SUCCESS); +/// struct HttpStatus { -- cgit v1.2.3 From d238341afaecedfe227141126c4c35dcde4a0671 Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Mon, 16 Jul 2012 11:53:04 -0400 Subject: SH-3189 Remove/improve naive data structures When releasing HTTP waiters, avoid unnecessary sort activity. For Content-Type in responses, let libcurl do the work and removed my parsing of headers. Drop Content-Encoding as libcurl will deal with that. If anyone is interested, they can parse. --- indra/llcorehttp/httpcommon.h | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'indra/llcorehttp/httpcommon.h') diff --git a/indra/llcorehttp/httpcommon.h b/indra/llcorehttp/httpcommon.h index 576a113e54..dd5798edf9 100644 --- a/indra/llcorehttp/httpcommon.h +++ b/indra/llcorehttp/httpcommon.h @@ -149,7 +149,10 @@ enum HttpError HE_OPT_NOT_SET = 7, // Option not dynamic, must be set during init phase - HE_OPT_NOT_DYNAMIC = 8 + HE_OPT_NOT_DYNAMIC = 8, + + // Invalid HTTP status code returned by server + HE_INVALID_HTTP_STATUS = 9 }; // end enum HttpError -- cgit v1.2.3 From 85e69b043b098dbe5a09f2eac6ff541123089f13 Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Mon, 23 Jul 2012 23:40:07 +0000 Subject: Big comment and naming cleanup. Ready for prime-time. Add to-do list to _httpinternal.h to guide anyone who wants to pitch in and help. --- indra/llcorehttp/httpcommon.h | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) (limited to 'indra/llcorehttp/httpcommon.h') diff --git a/indra/llcorehttp/httpcommon.h b/indra/llcorehttp/httpcommon.h index dd5798edf9..c0d4ec5aad 100644 --- a/indra/llcorehttp/httpcommon.h +++ b/indra/llcorehttp/httpcommon.h @@ -60,8 +60,10 @@ /// Using the library is fairly easy. Global setup needs a few /// steps: /// -/// - libcurl initialization with thread-safely callbacks for c-ares -/// DNS lookups. +/// - libcurl initialization including thread-safely callbacks for SSL: +/// . curl_global_init(...) +/// . CRYPTO_set_locking_callback(...) +/// . CRYPTO_set_id_callback(...) /// - HttpRequest::createService() called to instantiate singletons /// and support objects. /// @@ -90,8 +92,18 @@ /// - Do completion processing in your onCompletion() method. /// /// Code fragments: -/// +/// Rather than a poorly-maintained example in comments, look in the +/// example subdirectory which is a minimal yet functional tool to do +/// GET request performance testing. With four calls: /// +/// init_curl(); +/// LLCore::HttpRequest::createService(); +/// LLCore::HttpRequest::startThread(); +/// LLCore::HttpRequest * hr = new LLCore::HttpRequest(); +/// +/// the program is basically ready to issue requests. +/// + #include "linden_common.h" // Modifies curl/curl.h interfaces -- cgit v1.2.3