diff options
Diffstat (limited to 'indra/llcorehttp')
33 files changed, 3331 insertions, 701 deletions
diff --git a/indra/llcorehttp/README.Linden b/indra/llcorehttp/README.Linden new file mode 100644 index 0000000000..eb6ccab3bc --- /dev/null +++ b/indra/llcorehttp/README.Linden @@ -0,0 +1,671 @@ + + + +1. HTTP Fetching in 15 Minutes + + Let's start with a trivial working example. You'll need a throwaway + build of the viewer. And we'll use indra/newview/llappviewer.cpp as + the host module for these hacks. + + First, add some headers: + + + #include "httpcommon.h" + #include "httprequest.h" + #include "httphandler.h" + + + You'll need to derive a class from HttpHandler (not HttpHandle). + This is used to deliver notifications of HTTP completion to your + code. Place it near the top, before LLDeferredTaskList, say: + + + class MyHandler : public LLCore::HttpHandler + { + public: + MyHandler() + : LLCore::HttpHandler() + {} + + virtual void onCompleted(LLCore::HttpHandle /* handle */, + LLCore::HttpResponse * /* response */) + { + LL_INFOS("Hack") << "It is happening again." << LL_ENDL; + + delete this; // Last statement + } + }; + + + Add some statics up there as well: + + + // Our request object. Allocate during initialiation. + static LLCore::HttpRequest * my_request(NULL); + + // The policy class for HTTP traffic. + // Use HttpRequest::DEFAULT_POLICY_ID, but DO NOT SHIP WITH THIS VALUE!! + static LLCore::HttpRequest::policy_t my_policy(LLCore::HttpRequest::DEFAULT_POLICY_ID); + + // Priority for HTTP requests. Use 0U. + static LLCore::HttpRequest::priority_t my_priority(0U); + + + In LLAppViewer::init() after mAppCoreHttp.init(), create a request object: + + + my_request = new LLCore::HttpRequest(); + + + In LLAppViewer::mainLoop(), just before entering the while loop, + we'll kick off one HTTP request: + + + // Construct a handler object (we'll use the heap this time): + MyHandler * my_handler = new MyHandler; + + // Issue a GET request to 'http://www.example.com/' kicking off + // all the I/O, retry logic, etc. + LLCore::HttpHandle handle; + handle = my_request->requestGet(my_policy, + my_priority, + "http://www.example.com/", + NULL, + NULL, + my_handler); + if (LLCORE_HTTP_HANDLE_INVALID == handle) + { + LL_WARNS("Hack") << "Failed to launch HTTP request. Try again." + << LL_ENDL; + } + + + Finally, arrange to periodically call update() on the request object + to find out when the request completes. This will be done by + calling the onCompleted() method with status information and + response data from the HTTP operation. Add this to the + LLAppViewer::idle() method after the ping: + + + my_request->update(0); + + + That's it. Build it, run it and watch the log file. You should get + the "It is happening again." message indicating that the HTTP + operation completed in some manner. + + +2. What Does All That Mean + + MyHandler/HttpHandler. This class replaces the Responder-style in + legacy code. One method is currently defined. It is used for all + request completions, successful or failed: + + + void onCompleted(LLCore::HttpHandle /* handle */, + LLCore::HttpResponse * /* response */); + + + The onCompleted() method is invoked as a callback during calls to + HttpRequest::update(). All I/O is completed asynchronously in + another thread. But notifications are polled by calling update() + and invoking a handler for completed requests. + + In this example, the invocation also deletes the handler (which is + never referenced by the llcorehttp code again). But other + allocation models are possible including handlers shared by many + requests, stack-based handlers and handlers mixed in with other, + unrelated classes. + + LLCore::HttpRequest(). Instances of this class are used to request + all major functions of the library. Initialization, starting + requests, delivering final notification of completion and various + utility operations are all done via instances. There is one very + important rule for instances: + + Request objects may NOT be shared between threads. + + my_priority. The APIs support the idea of priority ordering of + requests but it hasn't been implemented and the hope is that this + will become useless and removed from the interface. Use 0U except + as noted. + + my_policy. This is an important one. This library attempts to + manage TCP connection usage more rigorously than in the past. This + is done by issuing requests to a queue that has various settable + properties. These establish connection usage for the queue as well + as how queues compete with one another. (This is patterned after + class-based queueing used in various networking stacks.) Several + classes are pre-defined. Deciding when to use an existing class and + when to create a new one will determine what kind of experience + users have. We'll pick up this question in detail below. + + requestGet(). Issues an ordinary HTTP GET request to a given URL + and associating the request with a policy class, a priority and an + response handler. Two additional arguments, not used here, allow + for additional headers on the request and for per-request options. + If successful, the call returns a handle whose value is other than + LLCORE_HTTP_HANDLE_INVALID. The HTTP operation is then performed + asynchronously by another thread without any additional work by the + caller. If the handle returned is invalid, you can get the status + code by calling my_request->getStatus(). + + update(). To get notification that the request has completed, a + call to update() will invoke onCompleted() methods. + + +3. Refinements, Necessary and Otherwise + + MyHandler::onCompleted(). You'll want to do something useful with + your response. Distinguish errors from successes and getting the + response body back in some form. + + Add a new header: + + + #include "bufferarray.h" + + + Replace the existing MyHandler::onCompleted() definition with: + + + virtual void onCompleted(LLCore::HttpHandle /* handle */, + LLCore::HttpResponse * response) + { + LLCore::HttpStatus status = response->getStatus(); + if (status) + { + // Successful request. Try to fetch the data + LLCore::BufferArray * data = response->getBody(); + + if (data && data->size()) + { + // There's some data. A BufferArray is a linked list + // of buckets. We'll create a linear buffer and copy + // the data into it. + size_t data_len = data->size(); + char * data_blob = new char [data_len + 1]; + data->read(0, data_blob, data_len); + data_blob[data_len] = '\0'; + + // Process the data now in NUL-terminated string. + // Needs more scrubbing but this will do. + LL_INFOS("Hack") << "Received: " << data_blob << LL_ENDL; + + // Free the temporary data + delete [] data_blob; + } + } + else + { + // Something went wrong. Translate the status to + // a meaningful message. + LL_WARNS("Hack") << "HTTP GET failed. Status: " + << status.toTerseString() + << ", Reason: " << status.toString() + << LL_ENDL; + } + + delete this; // Last statement + } + + + HttpHeaders. The header file "httprequest.h" documents the expected + important headers that will go out with the request. You can add to + these by including an HttpHeaders object with the requestGet() call. + These are typically setup once as part of init rather than + dynamically created. + + Add another header: + + + #include "httpheaders.h" + + + In LLAppViewer::mainLoop(), add this alongside the allocation of + my_handler: + + + // Additional headers for all requests + LLCore::HttpHeaders * my_headers = new LLCore::HttpHeaders(); + my_headers->append("Accept", "text/html, application/llsd+xml"); + + + HttpOptions. Options are similar and include a mix of value types. + One interesting per-request option is the trace setting. This + enables various debug-type messages in the log file that show the + progress of the request through the library. It takes values from + zero to three with higher values giving more verbose logging. We'll + use '2' and this will also give us a chance to verify that + HttpHeaders works as expected. + + Same as above, a new header: + + + #include "httpoptions.h" + + + And in LLAppView::mainLoop(): + + + // Special options for requests + LLCore::HttpOptions * my_options = new LLCore::HttpOptions(); + my_options->setTrace(2); + + + Now let's put that all together into a more complete requesting + sequence. Replace the existing invocation of requestGet() with this + slightly more elaborate block: + + + LLCore::HttpHandle handle; + handle = my_request->requestGet(my_policy, + my_priority, + "http://www.example.com/", + my_options, + my_headers, + my_handler); + if (LLCORE_HTTP_HANDLE_INVALID == handle) + { + LLCore::HttpStatus status = my_request->getStatus(); + + LL_WARNS("Hack") << "Failed to request HTTP GET. Status: " + << status.toTerseString() + << ", Reason: " << status.toString() + << LL_ENDL; + + delete my_handler; // No longer needed. + my_handler = NULL; + } + + + Build, run and examine the log file. You'll get some new data with + this run. First, you should get the www.example.com home page + content: + + +---------------------------------------------------------------------------- +2013-09-17T20:26:51Z INFO: MyHandler::onCompleted: Received: <!doctype html> +<html> +<head> + <title>Example Domain</title> + + <meta charset="utf-8" /> + <meta http-equiv="Content-type" content="text/html; charset=utf-8" /> + <meta name="viewport" content="width=device-width, initial-scale=1" /> + <style type="text/css"> + body { + background-color: #f0f0f2; + margin: 0; + padding: 0; + font-family: "Open Sans", "Helvetica Neue", Helvetica, Arial, sans-serif; + + } + div { + width: 600px; + margin: 5em auto; + padding: 50px; + background-color: #fff; + border-radius: 1em; + } + a:link, a:visited { + color: #38488f; + text-decoration: none; + } + @media (max-width: 700px) { + body { + background-color: #fff; + } + div { + width: auto; + margin: 0 auto; + border-radius: 0; + padding: 1em; + } + } + </style> +</head> + +<body> +<div> + <h1>Example Domain</h1> + <p>This domain is established to be used for illustrative examples in documents. You may use this + domain in examples without prior coordination or asking for permission.</p> + <p><a href="http://www.iana.org/domains/example">More information...</a></p> +</div> +</body> +</html> +---------------------------------------------------------------------------- + + + You'll also get a detailed trace of the HTTP operation itself. Note + the HEADEROUT line which shows the additional header added to the + request. + + +---------------------------------------------------------------------------- +HttpService::processRequestQueue: TRACE, FromRequestQueue, Handle: 086D3148 +HttpLibcurl::addOp: TRACE, ToActiveQueue, Handle: 086D3148, Actives: 0, Readies: 0 +HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle: 086D3148, Type: TEXT, Data: About to connect() to www.example.com port 80 (#0) +HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle: 086D3148, Type: TEXT, Data: Trying 93.184.216.119... +HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle: 086D3148, Type: TEXT, Data: Connected to www.example.com (93.184.216.119) port 80 (#0) +HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle: 086D3148, Type: TEXT, Data: Connected to www.example.com (93.184.216.119) port 80 (#0) +HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle: 086D3148, Type: HEADEROUT, Data: GET / HTTP/1.1 Host: www.example.com Accept-Encoding: deflate, gzip Connection: keep-alive Keep-alive: 300 Accept: text/html, application/llsd+xml +HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle: 086D3148, Type: HEADERIN, Data: HTTP/1.1 200 OK +HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle: 086D3148, Type: HEADERIN, Data: Accept-Ranges: bytes +HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle: 086D3148, Type: HEADERIN, Data: Cache-Control: max-age=604800 +HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle: 086D3148, Type: HEADERIN, Data: Content-Type: text/html +HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle: 086D3148, Type: HEADERIN, Data: Date: Tue, 17 Sep 2013 20:26:56 GMT +HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle: 086D3148, Type: HEADERIN, Data: Etag: "3012602696" +HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle: 086D3148, Type: HEADERIN, Data: Expires: Tue, 24 Sep 2013 20:26:56 GMT +HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle: 086D3148, Type: HEADERIN, Data: Last-Modified: Fri, 09 Aug 2013 23:54:35 GMT +HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle: 086D3148, Type: HEADERIN, Data: Server: ECS (ewr/1590) +HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle: 086D3148, Type: HEADERIN, Data: X-Cache: HIT +HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle: 086D3148, Type: HEADERIN, Data: x-ec-custom-error: 1 +HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle: 086D3148, Type: HEADERIN, Data: Content-Length: 1270 +HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle: 086D3148, Type: HEADERIN, Data: +HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle: 086D3148, Type: DATAIN, Data: 256 Bytes +HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle: 086D3148, Type: TEXT, Data: Connection #0 to host www.example.com left intact +HttpLibcurl::completeRequest: TRACE, RequestComplete, Handle: 086D3148, Status: Http_200 +HttpOperation::addAsReply: TRACE, ToReplyQueue, Handle: 086D3148 +---------------------------------------------------------------------------- + + +4. What Does All That Mean, Part 2 + + HttpStatus. The HttpStatus object encodes errors from libcurl, the + library itself and HTTP status values. It does this to avoid + collapsing all non-HTTP error into a single '499' HTTP status and to + make errors distinct. + + To aid programming, the usual bool conversions are available so that + you can write 'if (status)' and the expected thing will happen + whether it's an HTTP, libcurl or library error. There's also + provision to override the treatment of HTTP errors (making 404 a + success, say). + + Share data, don't copy it. The library was started with the goal of + avoiding data copies as much as possible. Instead, read-only data + sharing across threads with atomic reference counts is used for a + number of data types. These currently are: + + * BufferArray. Linked list of data blocks/HTTP bodies. + * HttpHeaders. Shared headers for both requests and responses. + * HttpOptions. Request-only data modifying HTTP behavior. + * HttpResponse. HTTP response description given to onCompleted. + + Using objects of these types requires a few rules: + + * Constructor always gives a reference to caller. + * References are dropped with release() not delete. + * Additional references may be taken out with addRef(). + * Unless otherwise stated, once an object is shared with another + thread it should be treated as read-only. There's no + synchronization on the objects themselves. + + HttpResponse. You'll encounter this mainly in onCompleted() methods. + Commonly-used interfaces on this object: + + * getStatus() to return the final status of the request. + * getBody() to retrieve the response body which may be NULL or + zero-length. + * getContentType() to return the value of the 'Content-Type' + header or an empty string if none was sent. + + This is a reference-counted object so you can call addRef() on it + and hold onto the response for an arbitrary time. But you'll + usually just call a few methods and return from onCompleted() whose + caller will release the object. + + BufferArray. The core data representation for request and response + bodies. In HTTP responses, it's fetched with the getBody() method + and may be NULL or non-NULL with zero length. All successful data + handling should check both conditions before attempting to fetch + data from the object. Data access model uses simple read/write + semantics: + + * append() + * size() + * read() + * write() + + (There is a more sophisticated stream adapter that extends these + methods and will be covered below.) So, one way to retrieve data + from a request is as follows: + + + LLCore::BufferArray * data = response->getBody(); + if (data && data->size()) + { + size_t data_len = data->size(); + char * data_blob = new char [data_len + 1]; + data->read(0, data_blob, data_len); + + + HttpOptions and HttpResponse. Really just simple containers of POD + and std::string pairs. But reference counted and the rule about not + modifying after sharing must be followed. You'll have the urge to + change options dynamically at some point. And you'll try to do that + by just writing new values to the shared object. And in tests + everything will appear to work. Then you ship and people in the + real world start hitting read/write races in strings and then crash. + Don't be lazy. + + HttpHandle. Uniquely identifies a request and can be used to + identify it in an onCompleted() method or cancel it if it's still + queued. But as soon as a request's onCompleted() invocation + returns, the handle becomes invalid and may be reused immediately + for new requests. Don't hold on to handles after notification. + + +5. And Still More Refinements + + (Note: The following refinements are just code fragments. They + don't directly fit into the working example above. But they + demonstrate several idioms you'll want to copy.) + + LLSD, std::streambuf, std::iostream. The read(), write() and + append() methods may be adequate for your purposes. But we use a + lot of LLSD. Its interfaces aren't particularly compatible with + BufferArray. And so two adapters are available to give + stream-like behaviors: BufferArrayStreamBuf and BufferArrayStream, + which implement the std::streambuf and std::iostream interfaces, + respectively. + + A std::streambuf interface isn't something you'll want to use + directly. Instead, you'll use the much friendlier std::iostream + interface found in BufferArrayStream. This adapter gives you all + the '>>' and '<<' operators you'll want as well as working + directly with the LLSD conversion operators. + + Some new headers: + + + #include "bufferstream.h" + #include "llsdserialize.h" + + + And an updated fragment based on onCompleted() above: + + + // Successful request. Try to fetch the data + LLCore::BufferArray * data = response->getBody(); + LLSD resp_llsd; + + if (data && data->size()) + { + // There's some data and we expect this to be + // LLSD. Checking of content type and validation + // during parsing would be admirable additions. + // But we'll forgo that now. + LLCore::BufferArrayStream data_stream(data); + LLSDSerialize::fromXML(resp_llsd, data_stream); + } + LL_INFOS("Hack") << "LLSD Received: " << resp_llsd << LL_ENDL; + } + else + { + + + Converting an LLSD object into an XML stream stored in a + BufferArray is just the reverse of the above: + + + BufferArray * data = new BufferArray(); + LLCore::BufferArrayStream data_stream(data); + + LLSD src_llsd; + src_llsd["foo"] = "bar"; + + LLSDSerialize::toXML(src_llsd, data_stream); + + // 'data' now contains an XML payload and can be sent + // to a web service using the requestPut() or requestPost() + // methods. + ... requestPost(...); + + // And don't forget to release the BufferArray. + data->release(); + data = NULL; + + + LLSD will often go hand-in-hand with BufferArray and data + transport. But you can also do all the streaming I/O you'd expect + of a std::iostream object: + + + BufferArray * data = new BufferArray(); + LLCore::BufferArrayStream data_stream(data); + + data_stream << "Hello, World!" << 29.4 << '\n'; + std::string str; + data_stream >> str; + std::cout << str << std::endl; + + data->release(); + // Actual delete will occur when 'data_stream' + // falls out of scope and is destructed. + + + Scoping objects and cleaning up. The examples haven't bothered + with cleanup of objects that are no longer needed. Instead, most + objects have been allocated as if they were global and eternal. + You'll put the objects in more appropriate feature objects and + clean them up as a group. Here's a checklist for actions you may + need to take on cleanup: + + * Call delete on: + o HttpHandlers created on the heap + o HttpRequest objects + * Call release() on: + o BufferArray objects + o HttpHeaders objects + o HttpOptions objects + o HttpResponse objects + + On program exit, as threads wind down, the library continues to + operate safely. Threads don't interact via the library and even + dangling references to HttpHandler objects are safe. If you don't + call HttpRequest::update(), handler references are never + dereferenced. + + You can take a more thorough approach to wind-down. Keep a list + of HttpHandles (not HttpHandlers) of outstanding requests. For + each of these, call HttpRequest::requestCancel() to cancel the + operation. (Don't add the cancel requests' handled to the list.) + This will cancel the outstanding requests that haven't completed. + Canceled or completed, all requests will queue notifications. You + can now cycle calling update() discarding responses. Continue + until all requests notify or a few seconds have passed. + + Global startup and shutdown is handled in the viewer. But you can + learn about it in the code or in the documentation in the headers. + + +6. Choosing a Policy Class + + Now it's time to get rid of the default policy class. Take a look + at the policy class definitions in newview/llappcorehttp.h. + Ideally, you'll find one that's compatible with what you're doing. + Some of the compatibility guidelines are: + + * Destination: Pair of host and port. Mixing requests with + different destinations may cause more connection setup and tear + down. + + * Method: http or https. Usually moot given destination. But + mixing these may also cause connection churn. + + * Transfer size: If you're moving 100MB at a time and you make your + requests to the same policy class as a lot of small, fast event + information that fast traffic is going to get stuck behind you + and someone's experience is going to be miserable. + + * Long poll requests: These are long-lived, must- do operations. + They have a special home called AP_LONG_POLL. + + * Concurrency: High concurrency (5 or more) and large transfer + sizes are incompatible. Another head-of-the-line problem. High + concurrency is tolerated when it's desired to get maximal + throughput. Mesh and texture downloads, for example. + + * Pipelined: If your requests are not idempotent, stay away from + anything marked 'soon' or 'yes'. Hidden retries may be a + problem for you. For now, would also recommend keeping PUT and + POST requests out of classes that may be pipelined. Support for + that is still a bit new. + + If you haven't found a compatible match, you can either create a + new class (llappcorehttp.*) or just use AP_DEFAULT, the catchall + class when all else fails. Inventory query operations might be a + candidate for a new class that supported pipelining on https:. + Same with display name lookups and other bursty-at-login + operations. For other things, AP_DEFAULT will do what it can and + will, in some way or another, tolerate any usage. Whether the + users' experiences are good are for you to determine. + + +7. FAQ + + Q1. What do these policy classes achieve? + + A1. Previously, HTTP-using code in the viewer was written as if + it were some isolated, local operation that didn't have to + consider resources, contention or impact on services and the + larger environment. The result was an application with on the + order of 100 HTTP launch points in its codebase that could create + dozens or even 100's of TCP connections zeroing in on grid + services and disrupting networking equipment, web services and + innocent users. The use of policy classes (modeled on + http://en.wikipedia.org/wiki/Class-based_queueing) is a means to + restrict connection concurrency, good and necessary in itself. In + turn, that reduces demands on an expensive resource (connection + setup and concurrency) which relieves strain on network points. + That enables connection keepalive and opportunites for true + improvements in throughput and user experience. + + Another aspect of the classes is that they give some control over + how competing demands for the network will be apportioned. If + mesh fetches, texture fetches and inventory queries are all being + made at once, the relative weights of their classes' concurrency + limits established that apportioning. We now have an opportunity + to balance the entire viewer system. + + Q2. How's that data sharing with refcounts working for you? + + A2. Meh. It does reduce memory churn and the frequency at which + free blocks must be moved between threads. But it's also a design + for static configuration and dynamic reconfiguration (not + requiring a restart) is favored. Creating new options for every + request isn't too bad, it a sequence of "new, fill, request, + release" for each requested operation. That in contrast to doing + the "new, fill, release" at startup. The bad comes in getting at + the source data. One rule in this work was "no new thread + problems." And one source for those is pulling setting values out + of gSettings in threads. None of that is thread safe though we + tend to get away with it. + + Q3. What needs to be done? + + A3. There's a To-Do list in _httpinternal.h. It has both large + and small projects here if someone would like to try changes. diff --git a/indra/llcorehttp/_httpinternal.h b/indra/llcorehttp/_httpinternal.h index 008e4fd95c..f80d7f60f5 100755 --- a/indra/llcorehttp/_httpinternal.h +++ b/indra/llcorehttp/_httpinternal.h @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -36,7 +36,8 @@ // General library to-do list // // - Implement policy classes. Structure is mostly there just didn't -// need it for the first consumer. +// need it for the first consumer. [Classes are there. More +// advanced features, like borrowing, aren't there yet.] // - Consider Removing 'priority' from the request interface. Its use // in an always active class can lead to starvation of low-priority // requests. Requires coodination of priority values across all @@ -46,6 +47,7 @@ // may not really need it. // - Set/get for global policy and policy classes is clumsy. Rework // it heading in a direction that allows for more dynamic behavior. +// [Mostly fixed] // - Move HttpOpRequest::prepareRequest() to HttpLibcurl for the // pedantic. // - Update downloader and other long-duration services are going to @@ -64,6 +66,12 @@ // This won't help in the face of the router problems we've looked // at, however. Detect starvation due to UDP activity and provide // feedback to it. +// - Change the transfer timeout scheme. We're less interested in +// absolute time, in most cases, than in continuous progress. +// - Many of the policy class settings are currently applied to the +// entire class. Some, like connection limits, would be better +// applied to each destination target making multiple targets +// independent. // // Integration to-do list // - LLTextureFetch still needs a major refactor. The use of @@ -73,7 +81,6 @@ // the main source file. // - Expand areas of usage eventually leading to the removal of LLCurl. // Rough order of expansion: -// . Mesh fetch // . Avatar names // . Group membership lists // . Caps access in general @@ -97,8 +104,8 @@ namespace LLCore { // Maxium number of policy classes that can be defined. -// *TODO: Currently limited to the default class, extend. -const int HTTP_POLICY_CLASS_LIMIT = 1; +// *TODO: Currently limited to the default class + 1, extend. +const int HTTP_POLICY_CLASS_LIMIT = 8; // Debug/informational tracing. Used both // as a global option and in per-request traces. @@ -129,6 +136,7 @@ const int HTTP_REDIRECTS_DEFAULT = 10; // Retries and time-on-queue are not included and aren't // accounted for. const long HTTP_REQUEST_TIMEOUT_DEFAULT = 30L; +const long HTTP_REQUEST_XFER_TIMEOUT_DEFAULT = 0L; const long HTTP_REQUEST_TIMEOUT_MIN = 0L; const long HTTP_REQUEST_TIMEOUT_MAX = 3600L; @@ -137,6 +145,11 @@ const int HTTP_CONNECTION_LIMIT_DEFAULT = 8; const int HTTP_CONNECTION_LIMIT_MIN = 1; const int HTTP_CONNECTION_LIMIT_MAX = 256; +// Miscellaneous defaults +const long HTTP_PIPELINING_DEFAULT = 0L; +const bool HTTP_USE_RETRY_AFTER_DEFAULT = true; +const long HTTP_THROTTLE_RATE_DEFAULT = 0L; + // Tuning parameters // Time worker thread sleeps after a pass through the diff --git a/indra/llcorehttp/_httplibcurl.cpp b/indra/llcorehttp/_httplibcurl.cpp index 6fe0bfc7d1..fc257fb0c1 100755 --- a/indra/llcorehttp/_httplibcurl.cpp +++ b/indra/llcorehttp/_httplibcurl.cpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -41,7 +41,8 @@ namespace LLCore HttpLibcurl::HttpLibcurl(HttpService * service) : mService(service), mPolicyCount(0), - mMultiHandles(NULL) + mMultiHandles(NULL), + mActiveHandles(NULL) {} @@ -77,6 +78,9 @@ void HttpLibcurl::shutdown() delete [] mMultiHandles; mMultiHandles = NULL; + + delete [] mActiveHandles; + mActiveHandles = NULL; } mPolicyCount = 0; @@ -90,9 +94,12 @@ void HttpLibcurl::start(int policy_count) mPolicyCount = policy_count; mMultiHandles = new CURLM * [mPolicyCount]; + mActiveHandles = new int [mPolicyCount]; + for (int policy_class(0); policy_class < mPolicyCount; ++policy_class) { mMultiHandles[policy_class] = curl_multi_init(); + mActiveHandles[policy_class] = 0; } } @@ -110,8 +117,10 @@ HttpService::ELoopSpeed HttpLibcurl::processTransport() // Give libcurl some cycles to do I/O & callbacks for (int policy_class(0); policy_class < mPolicyCount; ++policy_class) { - if (! mMultiHandles[policy_class]) + if (! mActiveHandles[policy_class] || ! mMultiHandles[policy_class]) + { continue; + } int running(0); CURLMcode status(CURLM_CALL_MULTI_PERFORM); @@ -132,12 +141,10 @@ HttpService::ELoopSpeed HttpLibcurl::processTransport() CURL * handle(msg->easy_handle); CURLcode result(msg->data.result); - if (completeRequest(mMultiHandles[policy_class], handle, result)) - { - // Request is still active, don't get too sleepy - ret = HttpService::NORMAL; - } - handle = NULL; // No longer valid on return + completeRequest(mMultiHandles[policy_class], handle, result); + handle = NULL; // No longer valid on return + ret = HttpService::NORMAL; // If anything completes, we may have a free slot. + // Turning around quickly reduces connection gap by 7-10mS. } else if (CURLMSG_NONE == msg->msg) { @@ -193,6 +200,7 @@ void HttpLibcurl::addOp(HttpOpRequest * op) // On success, make operation active mActiveOps.insert(op); + ++mActiveHandles[op->mReqPolicy]; } @@ -214,6 +222,7 @@ bool HttpLibcurl::cancel(HttpHandle handle) // Drop references mActiveOps.erase(it); + --mActiveHandles[op->mReqPolicy]; op->release(); return true; @@ -240,7 +249,7 @@ void HttpLibcurl::cancelRequest(HttpOpRequest * op) { LL_INFOS("CoreHttp") << "TRACE, RequestCanceled, Handle: " << static_cast<HttpHandle>(op) - << ", Status: " << op->mStatus.toHex() + << ", Status: " << op->mStatus.toTerseString() << LL_ENDL; } @@ -275,6 +284,7 @@ bool HttpLibcurl::completeRequest(CURLM * multi_handle, CURL * handle, CURLcode // Deactivate request mActiveOps.erase(it); + --mActiveHandles[op->mReqPolicy]; op->mCurlActive = false; // Set final status of request if it hasn't failed by other mechanisms yet @@ -316,7 +326,7 @@ bool HttpLibcurl::completeRequest(CURLM * multi_handle, CURL * handle, CURLcode { LL_INFOS("CoreHttp") << "TRACE, RequestComplete, Handle: " << static_cast<HttpHandle>(op) - << ", Status: " << op->mStatus.toHex() + << ", Status: " << op->mStatus.toTerseString() << LL_ENDL; } @@ -336,19 +346,9 @@ int HttpLibcurl::getActiveCount() const int HttpLibcurl::getActiveCountInClass(int policy_class) const { - int count(0); - - for (active_set_t::const_iterator iter(mActiveOps.begin()); - mActiveOps.end() != iter; - ++iter) - { - if ((*iter)->mReqPolicy == policy_class) - { - ++count; - } - } - - return count; + llassert_always(policy_class < mPolicyCount); + + return mActiveHandles ? mActiveHandles[policy_class] : 0; } @@ -359,12 +359,17 @@ int HttpLibcurl::getActiveCountInClass(int policy_class) const struct curl_slist * append_headers_to_slist(const HttpHeaders * headers, struct curl_slist * slist) { - for (HttpHeaders::container_t::const_iterator it(headers->mHeaders.begin()); - - headers->mHeaders.end() != it; - ++it) + const HttpHeaders::const_iterator end(headers->end()); + for (HttpHeaders::const_iterator it(headers->begin()); end != it; ++it) { - slist = curl_slist_append(slist, (*it).c_str()); + static const char sep[] = ": "; + std::string header; + header.reserve((*it).first.size() + (*it).second.size() + sizeof(sep)); + header.append((*it).first); + header.append(sep); + header.append((*it).second); + + slist = curl_slist_append(slist, header.c_str()); } return slist; } diff --git a/indra/llcorehttp/_httplibcurl.h b/indra/llcorehttp/_httplibcurl.h index 611f029ef5..67f98dd4f0 100755 --- a/indra/llcorehttp/_httplibcurl.h +++ b/indra/llcorehttp/_httplibcurl.h @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -71,16 +71,22 @@ public: /// /// @return Indication of how long this method is /// willing to wait for next service call. + /// + /// Threading: called by worker thread. HttpService::ELoopSpeed processTransport(); /// Add request to the active list. Caller is expected to have /// provided us with a reference count on the op to hold the /// request. (No additional references will be added.) + /// + /// Threading: called by worker thread. void addOp(HttpOpRequest * op); /// One-time call to set the number of policy classes to be /// serviced and to create the resources for each. Value /// must agree with HttpPolicy::setPolicies() call. + /// + /// Threading: called by init thread. void start(int policy_count); /// Synchronously stop libcurl operations. All active requests @@ -91,9 +97,13 @@ public: /// respective reply queues. /// /// Can be restarted with a start() call. + /// + /// Threading: called by worker thread. void shutdown(); /// Return global and per-class counts of active requests. + /// + /// Threading: called by worker thread. int getActiveCount() const; int getActiveCountInClass(int policy_class) const; @@ -103,6 +113,7 @@ public: /// /// @return True if handle was found and operation canceled. /// + /// Threading: called by worker thread. bool cancel(HttpHandle handle); protected: @@ -121,7 +132,8 @@ protected: HttpService * mService; // Simple reference, not owner active_set_t mActiveOps; int mPolicyCount; - CURLM ** mMultiHandles; + CURLM ** mMultiHandles; // One handle per policy class + int * mActiveHandles; // Active count per policy class }; // end class HttpLibcurl } // end namespace LLCore diff --git a/indra/llcorehttp/_httpoperation.cpp b/indra/llcorehttp/_httpoperation.cpp index 5cf5bc5930..5bb0654652 100755 --- a/indra/llcorehttp/_httpoperation.cpp +++ b/indra/llcorehttp/_httpoperation.cpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -53,7 +53,7 @@ HttpOperation::HttpOperation() mUserHandler(NULL), mReqPolicy(HttpRequest::DEFAULT_POLICY_ID), mReqPriority(0U), - mTracing(0) + mTracing(HTTP_TRACE_OFF) { mMetricCreated = totalTime(); } @@ -94,7 +94,7 @@ void HttpOperation::stageFromRequest(HttpService *) // Default implementation should never be called. This // indicates an operation making a transition that isn't // defined. - LL_ERRS("HttpCore") << "Default stageFromRequest method may not be called." + LL_ERRS("CoreHttp") << "Default stageFromRequest method may not be called." << LL_ENDL; } @@ -104,7 +104,7 @@ void HttpOperation::stageFromReady(HttpService *) // Default implementation should never be called. This // indicates an operation making a transition that isn't // defined. - LL_ERRS("HttpCore") << "Default stageFromReady method may not be called." + LL_ERRS("CoreHttp") << "Default stageFromReady method may not be called." << LL_ENDL; } @@ -114,7 +114,7 @@ void HttpOperation::stageFromActive(HttpService *) // Default implementation should never be called. This // indicates an operation making a transition that isn't // defined. - LL_ERRS("HttpCore") << "Default stageFromActive method may not be called." + LL_ERRS("CoreHttp") << "Default stageFromActive method may not be called." << LL_ENDL; } diff --git a/indra/llcorehttp/_httpoperation.h b/indra/llcorehttp/_httpoperation.h index 914627fad0..937a61187d 100755 --- a/indra/llcorehttp/_httpoperation.h +++ b/indra/llcorehttp/_httpoperation.h @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -72,7 +72,7 @@ class HttpService; class HttpOperation : public LLCoreInt::RefCounted { public: - /// Threading: called by a consumer/application thread. + /// Threading: called by consumer thread. HttpOperation(); protected: @@ -108,7 +108,7 @@ public: /// by the worker thread. This is passible data /// until notification is performed. /// - /// Threading: called by application thread. + /// Threading: called by consumer thread. /// void setReplyPath(HttpReplyQueue * reply_queue, HttpHandler * handler); @@ -141,7 +141,7 @@ public: /// call to HttpRequest::update(). This method does the necessary /// dispatching. /// - /// Threading: called by application thread. + /// Threading: called by consumer thread. /// virtual void visitNotifier(HttpRequest *); diff --git a/indra/llcorehttp/_httpoprequest.cpp b/indra/llcorehttp/_httpoprequest.cpp index 207ed8e1e4..926031501e 100755 --- a/indra/llcorehttp/_httpoprequest.cpp +++ b/indra/llcorehttp/_httpoprequest.cpp @@ -64,6 +64,15 @@ int parse_content_range_header(char * buffer, unsigned int * last, unsigned int * length); +// Similar for Retry-After headers. Only parses the delta form +// of the header, HTTP time formats aren't interesting for client +// purposes. +// +// @return 0 if successfully parsed and seconds time delta +// returned in time argument. +// +int parse_retry_after_header(char * buffer, int * time); + // Take data from libcurl's CURLOPT_DEBUGFUNCTION callback and // escape and format it for a tracing line in logging. Absolutely @@ -74,14 +83,16 @@ void escape_libcurl_debug_data(char * buffer, size_t len, bool scrub, std::string & safe_line); -// OS-neutral string comparisons of various types -int os_strncasecmp(const char *s1, const char *s2, size_t n); -int os_strcasecmp(const char *s1, const char *s2); -char * os_strtok_r(char *str, const char *delim, char **saveptr); - +// OS-neutral string comparisons of various types. +int os_strcasecmp(const char * s1, const char * s2); +char * os_strtok_r(char * str, const char * delim, char ** saveptr); +char * os_strtrim(char * str); +char * os_strltrim(char * str); +void os_strlower(char * str); -static const char * const hdr_whitespace(" \t"); -static const char * const hdr_separator(": \t"); +// Error testing and reporting for libcurl status codes +void check_curl_easy_code(CURLcode code); +void check_curl_easy_code(CURLcode code, int curl_setopt_option); } // end anonymous namespace @@ -104,12 +115,15 @@ HttpOpRequest::HttpOpRequest() mCurlService(NULL), mCurlHeaders(NULL), mCurlBodyPos(0), + mCurlTemp(NULL), + mCurlTempLen(0), mReplyBody(NULL), mReplyOffset(0), mReplyLength(0), mReplyFullLength(0), mReplyHeaders(NULL), mPolicyRetries(0), + mPolicy503Retries(0), mPolicyRetryAt(HttpTime(0)), mPolicyRetryLimit(HTTP_RETRY_COUNT_DEFAULT) { @@ -153,6 +167,10 @@ HttpOpRequest::~HttpOpRequest() mCurlHeaders = NULL; } + delete [] mCurlTemp; + mCurlTemp = NULL; + mCurlTempLen = 0; + if (mReplyBody) { mReplyBody->release(); @@ -208,6 +226,11 @@ void HttpOpRequest::stageFromActive(HttpService * service) mCurlHeaders = NULL; } + // Also not needed on the other side + delete [] mCurlTemp; + mCurlTemp = NULL; + mCurlTempLen = 0; + addAsReply(); } @@ -226,6 +249,7 @@ void HttpOpRequest::visitNotifier(HttpRequest * request) response->setRange(mReplyOffset, mReplyLength, mReplyFullLength); } response->setContentType(mReplyConType); + response->setRetries(mPolicyRetries, mPolicy503Retries); mUserHandler->onCompleted(static_cast<HttpHandle>(this), response); @@ -335,6 +359,10 @@ void HttpOpRequest::setupCommon(HttpRequest::policy_t policy_id, { mProcFlags |= PF_SAVE_HEADERS; } + if (options->getUseRetryAfter()) + { + mProcFlags |= PF_USE_RETRY_AFTER; + } mPolicyRetryLimit = options->getRetries(); mPolicyRetryLimit = llclamp(mPolicyRetryLimit, HTTP_RETRY_COUNT_MIN, HTTP_RETRY_COUNT_MAX); mTracing = (std::max)(mTracing, llclamp(options->getTrace(), HTTP_TRACE_MIN, HTTP_TRACE_MAX)); @@ -350,6 +378,8 @@ void HttpOpRequest::setupCommon(HttpRequest::policy_t policy_id, // HttpStatus HttpOpRequest::prepareRequest(HttpService * service) { + CURLcode code; + // Scrub transport and result data for retried op case mCurlActive = false; mCurlHandle = NULL; @@ -379,64 +409,108 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service) // *FIXME: better error handling later HttpStatus status; - // Get policy options + // Get global policy options HttpPolicyGlobal & policy(service->getPolicy().getGlobalOptions()); mCurlHandle = LLCurl::createStandardCurlHandle(); - - curl_easy_setopt(mCurlHandle, CURLOPT_WRITEFUNCTION, writeCallback); - curl_easy_setopt(mCurlHandle, CURLOPT_READFUNCTION, readCallback); - curl_easy_setopt(mCurlHandle, CURLOPT_READDATA, this); - curl_easy_setopt(mCurlHandle, CURLOPT_WRITEDATA, this); - curl_easy_setopt(mCurlHandle, CURLOPT_URL, mReqURL.c_str()); - curl_easy_setopt(mCurlHandle, CURLOPT_PRIVATE, this); - curl_easy_setopt(mCurlHandle, CURLOPT_MAXREDIRS, HTTP_REDIRECTS_DEFAULT); - - const std::string * opt_value(NULL); - long opt_long(0L); - policy.get(HttpRequest::GP_LLPROXY, &opt_long); - if (opt_long) + if (! mCurlHandle) + { + // We're in trouble. We'll continue but it won't go well. + LL_WARNS("CoreHttp") << "Failed to allocate libcurl easy handle. Continuing." + << LL_ENDL; + return HttpStatus(HttpStatus::LLCORE, HE_BAD_ALLOC); + } + code = curl_easy_setopt(mCurlHandle, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4); + check_curl_easy_code(code, CURLOPT_IPRESOLVE); + code = curl_easy_setopt(mCurlHandle, CURLOPT_NOSIGNAL, 1); + check_curl_easy_code(code, CURLOPT_NOSIGNAL); + code = curl_easy_setopt(mCurlHandle, CURLOPT_NOPROGRESS, 1); + check_curl_easy_code(code, CURLOPT_NOPROGRESS); + code = curl_easy_setopt(mCurlHandle, CURLOPT_URL, mReqURL.c_str()); + check_curl_easy_code(code, CURLOPT_URL); + code = curl_easy_setopt(mCurlHandle, CURLOPT_PRIVATE, this); + check_curl_easy_code(code, CURLOPT_PRIVATE); + code = curl_easy_setopt(mCurlHandle, CURLOPT_ENCODING, ""); + check_curl_easy_code(code, CURLOPT_ENCODING); + + // The Linksys WRT54G V5 router has an issue with frequent + // DNS lookups from LAN machines. If they happen too often, + // like for every HTTP request, the router gets annoyed after + // about 700 or so requests and starts issuing TCP RSTs to + // new connections. Reuse the DNS lookups for even a few + // seconds and no RSTs. + code = curl_easy_setopt(mCurlHandle, CURLOPT_DNS_CACHE_TIMEOUT, 15); + check_curl_easy_code(code, CURLOPT_DNS_CACHE_TIMEOUT); + code = curl_easy_setopt(mCurlHandle, CURLOPT_AUTOREFERER, 1); + check_curl_easy_code(code, CURLOPT_AUTOREFERER); + code = curl_easy_setopt(mCurlHandle, CURLOPT_FOLLOWLOCATION, 1); + check_curl_easy_code(code, CURLOPT_FOLLOWLOCATION); + code = curl_easy_setopt(mCurlHandle, CURLOPT_MAXREDIRS, HTTP_REDIRECTS_DEFAULT); + check_curl_easy_code(code, CURLOPT_MAXREDIRS); + code = curl_easy_setopt(mCurlHandle, CURLOPT_WRITEFUNCTION, writeCallback); + check_curl_easy_code(code, CURLOPT_WRITEFUNCTION); + code = curl_easy_setopt(mCurlHandle, CURLOPT_WRITEDATA, this); + check_curl_easy_code(code, CURLOPT_WRITEDATA); + code = curl_easy_setopt(mCurlHandle, CURLOPT_READFUNCTION, readCallback); + check_curl_easy_code(code, CURLOPT_READFUNCTION); + code = curl_easy_setopt(mCurlHandle, CURLOPT_READDATA, this); + check_curl_easy_code(code, CURLOPT_READDATA); + code = curl_easy_setopt(mCurlHandle, CURLOPT_SSL_VERIFYPEER, 1); + check_curl_easy_code(code, CURLOPT_SSL_VERIFYPEER); + code = curl_easy_setopt(mCurlHandle, CURLOPT_SSL_VERIFYHOST, 0); + check_curl_easy_code(code, CURLOPT_SSL_VERIFYHOST); + + if (policy.mUseLLProxy) { // Use the viewer-based thread-safe API which has a // fast/safe check for proxy enable. Would like to // encapsulate this someway... LLProxy::getInstance()->applyProxySettings(mCurlHandle); } - else if (policy.get(HttpRequest::GP_HTTP_PROXY, &opt_value)) + else if (policy.mHttpProxy.size()) { // *TODO: This is fine for now but get fuller socks5/ // authentication thing going later.... - curl_easy_setopt(mCurlHandle, CURLOPT_PROXY, opt_value->c_str()); - curl_easy_setopt(mCurlHandle, CURLOPT_PROXYTYPE, CURLPROXY_HTTP); + code = curl_easy_setopt(mCurlHandle, CURLOPT_PROXY, policy.mHttpProxy.c_str()); + check_curl_easy_code(code, CURLOPT_PROXY); + code = curl_easy_setopt(mCurlHandle, CURLOPT_PROXYTYPE, CURLPROXY_HTTP); + check_curl_easy_code(code, CURLOPT_PROXYTYPE); } - if (policy.get(HttpRequest::GP_CA_PATH, &opt_value)) + if (policy.mCAPath.size()) { - curl_easy_setopt(mCurlHandle, CURLOPT_CAPATH, opt_value->c_str()); + code = curl_easy_setopt(mCurlHandle, CURLOPT_CAPATH, policy.mCAPath.c_str()); + check_curl_easy_code(code, CURLOPT_CAPATH); } - if (policy.get(HttpRequest::GP_CA_FILE, &opt_value)) + if (policy.mCAFile.size()) { - curl_easy_setopt(mCurlHandle, CURLOPT_CAINFO, opt_value->c_str()); + code = curl_easy_setopt(mCurlHandle, CURLOPT_CAINFO, policy.mCAFile.c_str()); + check_curl_easy_code(code, CURLOPT_CAINFO); } switch (mReqMethod) { case HOR_GET: - curl_easy_setopt(mCurlHandle, CURLOPT_HTTPGET, 1); + code = curl_easy_setopt(mCurlHandle, CURLOPT_HTTPGET, 1); + check_curl_easy_code(code, CURLOPT_HTTPGET); mCurlHeaders = curl_slist_append(mCurlHeaders, "Connection: keep-alive"); mCurlHeaders = curl_slist_append(mCurlHeaders, "Keep-alive: 300"); break; case HOR_POST: { - curl_easy_setopt(mCurlHandle, CURLOPT_POST, 1); - curl_easy_setopt(mCurlHandle, CURLOPT_ENCODING, ""); + code = curl_easy_setopt(mCurlHandle, CURLOPT_POST, 1); + check_curl_easy_code(code, CURLOPT_POST); + code = curl_easy_setopt(mCurlHandle, CURLOPT_ENCODING, ""); + check_curl_easy_code(code, CURLOPT_ENCODING); long data_size(0); if (mReqBody) { data_size = mReqBody->size(); } - curl_easy_setopt(mCurlHandle, CURLOPT_POSTFIELDS, static_cast<void *>(NULL)); - curl_easy_setopt(mCurlHandle, CURLOPT_POSTFIELDSIZE, data_size); + code = curl_easy_setopt(mCurlHandle, CURLOPT_POSTFIELDS, static_cast<void *>(NULL)); + check_curl_easy_code(code, CURLOPT_POSTFIELDS); + code = curl_easy_setopt(mCurlHandle, CURLOPT_POSTFIELDSIZE, data_size); + check_curl_easy_code(code, CURLOPT_POSTFIELDSIZE); mCurlHeaders = curl_slist_append(mCurlHeaders, "Expect:"); mCurlHeaders = curl_slist_append(mCurlHeaders, "Connection: keep-alive"); mCurlHeaders = curl_slist_append(mCurlHeaders, "Keep-alive: 300"); @@ -445,14 +519,17 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service) case HOR_PUT: { - curl_easy_setopt(mCurlHandle, CURLOPT_UPLOAD, 1); + code = curl_easy_setopt(mCurlHandle, CURLOPT_UPLOAD, 1); + check_curl_easy_code(code, CURLOPT_UPLOAD); long data_size(0); if (mReqBody) { data_size = mReqBody->size(); } - curl_easy_setopt(mCurlHandle, CURLOPT_INFILESIZE, data_size); - curl_easy_setopt(mCurlHandle, CURLOPT_POSTFIELDS, (void *) NULL); + code = curl_easy_setopt(mCurlHandle, CURLOPT_INFILESIZE, data_size); + check_curl_easy_code(code, CURLOPT_INFILESIZE); + code = curl_easy_setopt(mCurlHandle, CURLOPT_POSTFIELDS, (void *) NULL); + check_curl_easy_code(code, CURLOPT_POSTFIELDS); mCurlHeaders = curl_slist_append(mCurlHeaders, "Expect:"); mCurlHeaders = curl_slist_append(mCurlHeaders, "Connection: keep-alive"); mCurlHeaders = curl_slist_append(mCurlHeaders, "Keep-alive: 300"); @@ -469,9 +546,12 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service) // Tracing if (mTracing >= HTTP_TRACE_CURL_HEADERS) { - curl_easy_setopt(mCurlHandle, CURLOPT_VERBOSE, 1); - curl_easy_setopt(mCurlHandle, CURLOPT_DEBUGDATA, this); - curl_easy_setopt(mCurlHandle, CURLOPT_DEBUGFUNCTION, debugCallback); + code = curl_easy_setopt(mCurlHandle, CURLOPT_VERBOSE, 1); + check_curl_easy_code(code, CURLOPT_VERBOSE); + code = curl_easy_setopt(mCurlHandle, CURLOPT_DEBUGDATA, this); + check_curl_easy_code(code, CURLOPT_DEBUGDATA); + code = curl_easy_setopt(mCurlHandle, CURLOPT_DEBUGFUNCTION, debugCallback); + check_curl_easy_code(code, CURLOPT_DEBUGFUNCTION); } // There's a CURLOPT for this now... @@ -499,13 +579,22 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service) // Request options long timeout(HTTP_REQUEST_TIMEOUT_DEFAULT); + long xfer_timeout(HTTP_REQUEST_XFER_TIMEOUT_DEFAULT); if (mReqOptions) - { + { timeout = mReqOptions->getTimeout(); timeout = llclamp(timeout, HTTP_REQUEST_TIMEOUT_MIN, HTTP_REQUEST_TIMEOUT_MAX); + xfer_timeout = mReqOptions->getTransferTimeout(); + xfer_timeout = llclamp(xfer_timeout, HTTP_REQUEST_TIMEOUT_MIN, HTTP_REQUEST_TIMEOUT_MAX); + } + if (xfer_timeout == 0L) + { + xfer_timeout = timeout; } - curl_easy_setopt(mCurlHandle, CURLOPT_TIMEOUT, timeout); - curl_easy_setopt(mCurlHandle, CURLOPT_CONNECTTIMEOUT, timeout); + code = curl_easy_setopt(mCurlHandle, CURLOPT_TIMEOUT, xfer_timeout); + check_curl_easy_code(code, CURLOPT_TIMEOUT); + code = curl_easy_setopt(mCurlHandle, CURLOPT_CONNECTTIMEOUT, timeout); + check_curl_easy_code(code, CURLOPT_CONNECTTIMEOUT); // Request headers if (mReqHeaders) @@ -513,12 +602,15 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service) // Caller's headers last to override mCurlHeaders = append_headers_to_slist(mReqHeaders, mCurlHeaders); } - curl_easy_setopt(mCurlHandle, CURLOPT_HTTPHEADER, mCurlHeaders); + code = curl_easy_setopt(mCurlHandle, CURLOPT_HTTPHEADER, mCurlHeaders); + check_curl_easy_code(code, CURLOPT_HTTPHEADER); - if (mProcFlags & (PF_SCAN_RANGE_HEADER | PF_SAVE_HEADERS)) + if (mProcFlags & (PF_SCAN_RANGE_HEADER | PF_SAVE_HEADERS | PF_USE_RETRY_AFTER)) { - curl_easy_setopt(mCurlHandle, CURLOPT_HEADERFUNCTION, headerCallback); - curl_easy_setopt(mCurlHandle, CURLOPT_HEADERDATA, this); + code = curl_easy_setopt(mCurlHandle, CURLOPT_HEADERFUNCTION, headerCallback); + check_curl_easy_code(code, CURLOPT_HEADERFUNCTION); + code = curl_easy_setopt(mCurlHandle, CURLOPT_HEADERDATA, this); + check_curl_easy_code(code, CURLOPT_HEADERDATA); } if (status) @@ -559,7 +651,7 @@ size_t HttpOpRequest::readCallback(void * data, size_t size, size_t nmemb, void { // Warn but continue if the read position moves beyond end-of-body // for some reason. - LL_WARNS("HttpCore") << "Request body position beyond body size. Truncating request body." + LL_WARNS("CoreHttp") << "Request body position beyond body size. Truncating request body." << LL_ENDL; } return 0; @@ -576,15 +668,15 @@ size_t HttpOpRequest::headerCallback(void * data, size_t size, size_t nmemb, voi { static const char status_line[] = "HTTP/"; static const size_t status_line_len = sizeof(status_line) - 1; - - static const char con_ran_line[] = "content-range:"; - static const size_t con_ran_line_len = sizeof(con_ran_line) - 1; - + static const char con_ran_line[] = "content-range"; + static const char con_retry_line[] = "retry-after"; + HttpOpRequest * op(static_cast<HttpOpRequest *>(userdata)); const size_t hdr_size(size * nmemb); const char * hdr_data(static_cast<const char *>(data)); // Not null terminated - + bool is_header(true); + if (hdr_size >= status_line_len && ! strncmp(status_line, hdr_data, status_line_len)) { // One of possibly several status lines. Reset what we know and start over @@ -592,11 +684,13 @@ size_t HttpOpRequest::headerCallback(void * data, size_t size, size_t nmemb, voi op->mReplyOffset = 0; op->mReplyLength = 0; op->mReplyFullLength = 0; + op->mReplyRetryAfter = 0; op->mStatus = HttpStatus(); if (op->mReplyHeaders) { - op->mReplyHeaders->mHeaders.clear(); + op->mReplyHeaders->clear(); } + is_header = false; } // Nothing in here wants a final CR/LF combination. Remove @@ -609,52 +703,109 @@ size_t HttpOpRequest::headerCallback(void * data, size_t size, size_t nmemb, voi --wanted_hdr_size; } } + + // Copy and normalize header fragments for the following + // stages. Would like to modify the data in-place but that + // may not be allowed and we need one byte extra for NUL. + // At the end of this we will have: + // + // If ':' present in header: + // 1. name points to text to left of colon which + // will be ascii lower-cased and left and right + // trimmed of whitespace. + // 2. value points to text to right of colon which + // will be left trimmed of whitespace. + // Otherwise: + // 1. name points to header which will be left + // trimmed of whitespace. + // 2. value is NULL + // Any non-NULL pointer may point to a zero-length string. + // + if (wanted_hdr_size >= op->mCurlTempLen) + { + delete [] op->mCurlTemp; + op->mCurlTempLen = 2 * wanted_hdr_size + 1; + op->mCurlTemp = new char [op->mCurlTempLen]; + } + memcpy(op->mCurlTemp, hdr_data, wanted_hdr_size); + op->mCurlTemp[wanted_hdr_size] = '\0'; + char * name(op->mCurlTemp); + char * value(strchr(name, ':')); + if (value) + { + *value++ = '\0'; + os_strlower(name); + name = os_strtrim(name); + value = os_strltrim(value); + } + else + { + // Doesn't look well-formed, do minimal normalization on it + name = os_strltrim(name); + } + + // Normalized, now reject headers with empty names. + if (! *name) + { + // No use continuing + return hdr_size; + } // Save header if caller wants them in the response - if (op->mProcFlags & PF_SAVE_HEADERS) + if (is_header && op->mProcFlags & PF_SAVE_HEADERS) { // Save headers in response if (! op->mReplyHeaders) { op->mReplyHeaders = new HttpHeaders; } - op->mReplyHeaders->mHeaders.push_back(std::string(hdr_data, wanted_hdr_size)); + op->mReplyHeaders->append(name, value ? value : ""); } + // From this point, header-specific processors are free to + // modify the header value. + // Detect and parse 'Content-Range' headers - if (op->mProcFlags & PF_SCAN_RANGE_HEADER) + if (is_header + && op->mProcFlags & PF_SCAN_RANGE_HEADER + && value && *value + && ! strcmp(name, con_ran_line)) { - char hdr_buffer[128]; // Enough for a reasonable header - size_t frag_size((std::min)(wanted_hdr_size, sizeof(hdr_buffer) - 1)); - - memcpy(hdr_buffer, hdr_data, frag_size); - hdr_buffer[frag_size] = '\0'; - if (frag_size > con_ran_line_len && - ! os_strncasecmp(hdr_buffer, con_ran_line, con_ran_line_len)) + unsigned int first(0), last(0), length(0); + int status; + + if (! (status = parse_content_range_header(value, &first, &last, &length))) + { + // Success, record the fragment position + op->mReplyOffset = first; + op->mReplyLength = last - first + 1; + op->mReplyFullLength = length; + } + else if (-1 == status) { - unsigned int first(0), last(0), length(0); - int status; + // Response is badly formed and shouldn't be accepted + op->mStatus = HttpStatus(HttpStatus::LLCORE, HE_INV_CONTENT_RANGE_HDR); + } + else + { + // Ignore the unparsable. + LL_INFOS_ONCE("CoreHttp") << "Problem parsing odd Content-Range header: '" + << std::string(hdr_data, wanted_hdr_size) + << "'. Ignoring." + << LL_ENDL; + } + } - if (! (status = parse_content_range_header(hdr_buffer, &first, &last, &length))) - { - // Success, record the fragment position - op->mReplyOffset = first; - op->mReplyLength = last - first + 1; - op->mReplyFullLength = length; - } - else if (-1 == status) - { - // Response is badly formed and shouldn't be accepted - op->mStatus = HttpStatus(HttpStatus::LLCORE, HE_INV_CONTENT_RANGE_HDR); - } - else - { - // Ignore the unparsable. - LL_INFOS_ONCE("CoreHttp") << "Problem parsing odd Content-Range header: '" - << std::string(hdr_data, frag_size) - << "'. Ignoring." - << LL_ENDL; - } + // Detect and parse 'Retry-After' headers + if (is_header + && op->mProcFlags & PF_USE_RETRY_AFTER + && value && *value + && ! strcmp(name, con_retry_line)) + { + int time(0); + if (! parse_retry_after_header(value, &time)) + { + op->mReplyRetryAfter = time; } } @@ -769,14 +920,16 @@ int parse_content_range_header(char * buffer, unsigned int * last, unsigned int * length) { + static const char * const hdr_whitespace(" \t"); + char * tok_state(NULL), * tok(NULL); bool match(true); - if (! os_strtok_r(buffer, hdr_separator, &tok_state)) + if (! (tok = os_strtok_r(buffer, hdr_whitespace, &tok_state))) match = false; - if (match && (tok = os_strtok_r(NULL, hdr_whitespace, &tok_state))) - match = 0 == os_strcasecmp("bytes", tok); - if (match && ! (tok = os_strtok_r(NULL, " \t", &tok_state))) + else + match = (0 == os_strcasecmp("bytes", tok)); + if (match && ! (tok = os_strtok_r(NULL, hdr_whitespace, &tok_state))) match = false; if (match) { @@ -815,6 +968,25 @@ int parse_content_range_header(char * buffer, } +int parse_retry_after_header(char * buffer, int * time) +{ + char * endptr(buffer); + long lcl_time(strtol(buffer, &endptr, 10)); + if (*endptr == '\0' && endptr != buffer && lcl_time > 0) + { + *time = lcl_time; + return 0; + } + + // Could attempt to parse HTTP time here but we're not really + // interested in it. Scheduling based on wallclock time on + // user hardware will lead to tears. + + // Header is there but badly/unexpectedly formed, try to ignore it. + return 1; +} + + void escape_libcurl_debug_data(char * buffer, size_t len, bool scrub, std::string & safe_line) { std::string out; @@ -851,15 +1023,6 @@ void escape_libcurl_debug_data(char * buffer, size_t len, bool scrub, std::strin } -int os_strncasecmp(const char *s1, const char *s2, size_t n) -{ -#if LL_WINDOWS - return _strnicmp(s1, s2, n); -#else - return strncasecmp(s1, s2, n); -#endif // LL_WINDOWS -} - int os_strcasecmp(const char *s1, const char *s2) { @@ -881,6 +1044,73 @@ char * os_strtok_r(char *str, const char *delim, char ** savestate) } -} // end anonymous namespace +void os_strlower(char * str) +{ + for (char c(0); (c = *str); ++str) + { + *str = tolower(c); + } +} - + +char * os_strtrim(char * lstr) +{ + while (' ' == *lstr || '\t' == *lstr) + { + ++lstr; + } + if (*lstr) + { + char * rstr(lstr + strlen(lstr)); + while (lstr < rstr && *--rstr) + { + if (' ' == *rstr || '\t' == *rstr) + { + *rstr = '\0'; + } + } + llassert(lstr <= rstr); + } + return lstr; +} + + +char * os_strltrim(char * lstr) +{ + while (' ' == *lstr || '\t' == *lstr) + { + ++lstr; + } + return lstr; +} + + +void check_curl_easy_code(CURLcode code, int curl_setopt_option) +{ + if (CURLE_OK != code) + { + // Comment from old llcurl code which may no longer apply: + // + // linux appears to throw a curl error once per session for a bad initialization + // at a pretty random time (when enabling cookies). + LL_WARNS("CoreHttp") << "libcurl error detected: " << curl_easy_strerror(code) + << ", curl_easy_setopt option: " << curl_setopt_option + << LL_ENDL; + } +} + + +void check_curl_easy_code(CURLcode code) +{ + if (CURLE_OK != code) + { + // Comment from old llcurl code which may no longer apply: + // + // linux appears to throw a curl error once per session for a bad initialization + // at a pretty random time (when enabling cookies). + LL_WARNS("CoreHttp") << "libcurl error detected: " << curl_easy_strerror(code) + << LL_ENDL; + } +} + +} // end anonymous namespace diff --git a/indra/llcorehttp/_httpoprequest.h b/indra/llcorehttp/_httpoprequest.h index 74a349b0bf..2f628b5aba 100755 --- a/indra/llcorehttp/_httpoprequest.h +++ b/indra/llcorehttp/_httpoprequest.h @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -157,6 +157,7 @@ protected: unsigned int mProcFlags; static const unsigned int PF_SCAN_RANGE_HEADER = 0x00000001U; static const unsigned int PF_SAVE_HEADERS = 0x00000002U; + static const unsigned int PF_USE_RETRY_AFTER = 0x00000004U; public: // Request data @@ -174,6 +175,8 @@ public: HttpService * mCurlService; curl_slist * mCurlHeaders; size_t mCurlBodyPos; + char * mCurlTemp; // Scratch buffer for header processing + size_t mCurlTempLen; // Result data HttpStatus mStatus; @@ -183,9 +186,11 @@ public: size_t mReplyFullLength; HttpHeaders * mReplyHeaders; std::string mReplyConType; + int mReplyRetryAfter; // Policy data int mPolicyRetries; + int mPolicy503Retries; HttpTime mPolicyRetryAt; int mPolicyRetryLimit; }; // end class HttpOpRequest diff --git a/indra/llcorehttp/_httpopsetget.cpp b/indra/llcorehttp/_httpopsetget.cpp index 8198528a9b..a5363f9170 100755 --- a/indra/llcorehttp/_httpopsetget.cpp +++ b/indra/llcorehttp/_httpopsetget.cpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -27,6 +27,7 @@ #include "_httpopsetget.h" #include "httpcommon.h" +#include "httprequest.h" #include "_httpservice.h" #include "_httppolicy.h" @@ -43,10 +44,11 @@ namespace LLCore HttpOpSetGet::HttpOpSetGet() : HttpOperation(), - mIsGlobal(false), - mDoSet(false), - mSetting(-1), // Nothing requested - mLongValue(0L) + mReqOption(HttpRequest::PO_CONNECTION_LIMIT), + mReqClass(HttpRequest::INVALID_POLICY_ID), + mReqDoSet(false), + mReqLongValue(0L), + mReplyLongValue(0L) {} @@ -54,37 +56,84 @@ HttpOpSetGet::~HttpOpSetGet() {} -void HttpOpSetGet::setupGet(HttpRequest::EGlobalPolicy setting) +HttpStatus HttpOpSetGet::setupGet(HttpRequest::EPolicyOption opt, HttpRequest::policy_t pclass) { - mIsGlobal = true; - mSetting = setting; + HttpStatus status; + + mReqOption = opt; + mReqClass = pclass; + return status; } -void HttpOpSetGet::setupSet(HttpRequest::EGlobalPolicy setting, const std::string & value) +HttpStatus HttpOpSetGet::setupSet(HttpRequest::EPolicyOption opt, HttpRequest::policy_t pclass, long value) { - mIsGlobal = true; - mDoSet = true; - mSetting = setting; - mStrValue = value; + HttpStatus status; + + if (! HttpService::sOptionDesc[opt].mIsLong) + { + return HttpStatus(HttpStatus::LLCORE, HE_INVALID_ARG); + } + if (! HttpService::sOptionDesc[opt].mIsDynamic) + { + return HttpStatus(HttpStatus::LLCORE, HE_OPT_NOT_DYNAMIC); + } + + mReqOption = opt; + mReqClass = pclass; + mReqDoSet = true; + mReqLongValue = value; + + return status; } -void HttpOpSetGet::stageFromRequest(HttpService * service) +HttpStatus HttpOpSetGet::setupSet(HttpRequest::EPolicyOption opt, HttpRequest::policy_t pclass, const std::string & value) { - HttpPolicyGlobal & pol_opt(service->getPolicy().getGlobalOptions()); - HttpRequest::EGlobalPolicy setting(static_cast<HttpRequest::EGlobalPolicy>(mSetting)); + HttpStatus status; + + if (HttpService::sOptionDesc[opt].mIsLong) + { + return HttpStatus(HttpStatus::LLCORE, HE_INVALID_ARG); + } + if (! HttpService::sOptionDesc[opt].mIsDynamic) + { + return HttpStatus(HttpStatus::LLCORE, HE_OPT_NOT_DYNAMIC); + } + + mReqOption = opt; + mReqClass = pclass; + mReqDoSet = true; + mReqStrValue = value; - if (mDoSet) + return status; +} + + +void HttpOpSetGet::stageFromRequest(HttpService * service) +{ + if (mReqDoSet) { - mStatus = pol_opt.set(setting, mStrValue); + if (HttpService::sOptionDesc[mReqOption].mIsLong) + { + mStatus = service->setPolicyOption(mReqOption, mReqClass, + mReqLongValue, &mReplyLongValue); + } + else + { + mStatus = service->setPolicyOption(mReqOption, mReqClass, + mReqStrValue, &mReplyStrValue); + } } - if (mStatus) + else { - const std::string * value(NULL); - if ((mStatus = pol_opt.get(setting, &value))) + if (HttpService::sOptionDesc[mReqOption].mIsLong) + { + mStatus = service->getPolicyOption(mReqOption, mReqClass, &mReplyLongValue); + } + else { - mStrValue = *value; + mStatus = service->getPolicyOption(mReqOption, mReqClass, &mReplyStrValue); } } diff --git a/indra/llcorehttp/_httpopsetget.h b/indra/llcorehttp/_httpopsetget.h index 6966b9d94e..a1e76dd429 100755 --- a/indra/llcorehttp/_httpopsetget.h +++ b/indra/llcorehttp/_httpopsetget.h @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -46,7 +46,10 @@ namespace LLCore /// configuration settings. /// /// *NOTE: Expect this to change. Don't really like it yet. - +/// +/// *TODO: Can't return values to caller yet. Need to do +/// something better with HttpResponse and visitNotifier(). +/// class HttpOpSetGet : public HttpOperation { public: @@ -61,19 +64,23 @@ private: public: /// Threading: called by application thread - void setupGet(HttpRequest::EGlobalPolicy setting); - void setupSet(HttpRequest::EGlobalPolicy setting, const std::string & value); + HttpStatus setupGet(HttpRequest::EPolicyOption opt, HttpRequest::policy_t pclass); + HttpStatus setupSet(HttpRequest::EPolicyOption opt, HttpRequest::policy_t pclass, long value); + HttpStatus setupSet(HttpRequest::EPolicyOption opt, HttpRequest::policy_t pclass, const std::string & value); virtual void stageFromRequest(HttpService *); public: // Request data - bool mIsGlobal; - bool mDoSet; - int mSetting; - long mLongValue; - std::string mStrValue; - + HttpRequest::EPolicyOption mReqOption; + HttpRequest::policy_t mReqClass; + bool mReqDoSet; + long mReqLongValue; + std::string mReqStrValue; + + // Reply Data + long mReplyLongValue; + std::string mReplyStrValue; }; // end class HttpOpSetGet diff --git a/indra/llcorehttp/_httppolicy.cpp b/indra/llcorehttp/_httppolicy.cpp index 014bd37e2e..fd5a93e192 100755 --- a/indra/llcorehttp/_httppolicy.cpp +++ b/indra/llcorehttp/_httppolicy.cpp @@ -41,57 +41,70 @@ namespace LLCore // Per-policy-class data for a running system. -// Collection of queues, parameters, history, metrics, etc. +// Collection of queues, options and other data // for a single policy class. // // Threading: accessed only by worker thread -struct HttpPolicy::State +struct HttpPolicy::ClassState { public: - State() - : mConnMax(HTTP_CONNECTION_LIMIT_DEFAULT), - mConnAt(HTTP_CONNECTION_LIMIT_DEFAULT), - mConnMin(1), - mNextSample(0), - mErrorCount(0), - mErrorFactor(0) + ClassState() + : mThrottleEnd(0), + mThrottleLeft(0L), + mRequestCount(0L) {} HttpReadyQueue mReadyQueue; HttpRetryQueue mRetryQueue; HttpPolicyClass mOptions; - - long mConnMax; - long mConnAt; - long mConnMin; - - HttpTime mNextSample; - unsigned long mErrorCount; - unsigned long mErrorFactor; + HttpTime mThrottleEnd; + long mThrottleLeft; + long mRequestCount; }; HttpPolicy::HttpPolicy(HttpService * service) - : mActiveClasses(0), - mState(NULL), - mService(service) -{} + : mService(service) +{ + // Create default class + mClasses.push_back(new ClassState()); +} HttpPolicy::~HttpPolicy() { shutdown(); + + for (class_list_t::iterator it(mClasses.begin()); it != mClasses.end(); ++it) + { + delete (*it); + } + mClasses.clear(); mService = NULL; } +HttpRequest::policy_t HttpPolicy::createPolicyClass() +{ + const HttpRequest::policy_t policy_class(mClasses.size()); + if (policy_class >= HTTP_POLICY_CLASS_LIMIT) + { + return HttpRequest::INVALID_POLICY_ID; + } + mClasses.push_back(new ClassState()); + return policy_class; +} + + void HttpPolicy::shutdown() { - for (int policy_class(0); policy_class < mActiveClasses; ++policy_class) + for (int policy_class(0); policy_class < mClasses.size(); ++policy_class) { - HttpRetryQueue & retryq(mState[policy_class].mRetryQueue); + ClassState & state(*mClasses[policy_class]); + + HttpRetryQueue & retryq(state.mRetryQueue); while (! retryq.empty()) { HttpOpRequest * op(retryq.top()); @@ -101,7 +114,7 @@ void HttpPolicy::shutdown() op->release(); } - HttpReadyQueue & readyq(mState[policy_class].mReadyQueue); + HttpReadyQueue & readyq(state.mReadyQueue); while (! readyq.empty()) { HttpOpRequest * op(readyq.top()); @@ -111,28 +124,11 @@ void HttpPolicy::shutdown() op->release(); } } - delete [] mState; - mState = NULL; - mActiveClasses = 0; } -void HttpPolicy::start(const HttpPolicyGlobal & global, - const std::vector<HttpPolicyClass> & classes) -{ - llassert_always(! mState); - - mGlobalOptions = global; - mActiveClasses = classes.size(); - mState = new State [mActiveClasses]; - for (int i(0); i < mActiveClasses; ++i) - { - mState[i].mOptions = classes[i]; - mState[i].mConnMax = classes[i].mConnectionLimit; - mState[i].mConnAt = mState[i].mConnMax; - mState[i].mConnMin = 2; - } -} +void HttpPolicy::start() +{} void HttpPolicy::addOp(HttpOpRequest * op) @@ -140,7 +136,8 @@ void HttpPolicy::addOp(HttpOpRequest * op) const int policy_class(op->mReqPolicy); op->mPolicyRetries = 0; - mState[policy_class].mReadyQueue.push(op); + op->mPolicy503Retries = 0; + mClasses[policy_class]->mReadyQueue.push(op); } @@ -155,25 +152,39 @@ void HttpPolicy::retryOp(HttpOpRequest * op) 5000000 // ... to every 5.0 S. }; static const int delta_max(int(LL_ARRAY_SIZE(retry_deltas)) - 1); - + static const HttpStatus error_503(503); + const HttpTime now(totalTime()); const int policy_class(op->mReqPolicy); - - const HttpTime delta(retry_deltas[llclamp(op->mPolicyRetries, 0, delta_max)]); + HttpTime delta(retry_deltas[llclamp(op->mPolicyRetries, 0, delta_max)]); + bool external_delta(false); + + if (op->mReplyRetryAfter > 0 && op->mReplyRetryAfter < 30) + { + delta = op->mReplyRetryAfter * U64L(1000000); + external_delta = true; + } op->mPolicyRetryAt = now + delta; ++op->mPolicyRetries; - LL_WARNS("CoreHttp") << "HTTP request " << static_cast<HttpHandle>(op) - << " retry " << op->mPolicyRetries - << " scheduled for +" << (delta / HttpTime(1000)) - << " mS. Status: " << op->mStatus.toHex() - << LL_ENDL; - if (op->mTracing > 0) + if (error_503 == op->mStatus) + { + ++op->mPolicy503Retries; + } + LL_DEBUGS("CoreHttp") << "HTTP request " << static_cast<HttpHandle>(op) + << " retry " << op->mPolicyRetries + << " scheduled in " << (delta / HttpTime(1000)) + << " mS (" << (external_delta ? "external" : "internal") + << "). Status: " << op->mStatus.toTerseString() + << LL_ENDL; + if (op->mTracing > HTTP_TRACE_OFF) { LL_INFOS("CoreHttp") << "TRACE, ToRetryQueue, Handle: " << static_cast<HttpHandle>(op) + << ", Delta: " << (delta / HttpTime(1000)) + << ", Retries: " << op->mPolicyRetries << LL_ENDL; } - mState[policy_class].mRetryQueue.push(op); + mClasses[policy_class]->mRetryQueue.push(op); } @@ -188,21 +199,43 @@ void HttpPolicy::retryOp(HttpOpRequest * op) // the worker thread may sleep hard otherwise will ask for // normal polling frequency. // +// Implements a client-side request rate throttle as well. +// This is intended to mimic and predict throttling behavior +// of grid services but that is difficult to do with different +// time bases. This also represents a rigid coupling between +// viewer and server that makes it hard to change parameters +// and I hope we can make this go away with pipelining. +// HttpService::ELoopSpeed HttpPolicy::processReadyQueue() { const HttpTime now(totalTime()); HttpService::ELoopSpeed result(HttpService::REQUEST_SLEEP); HttpLibcurl & transport(mService->getTransport()); - for (int policy_class(0); policy_class < mActiveClasses; ++policy_class) + for (int policy_class(0); policy_class < mClasses.size(); ++policy_class) { - State & state(mState[policy_class]); - int active(transport.getActiveCountInClass(policy_class)); - int needed(state.mConnAt - active); // Expect negatives here - + ClassState & state(*mClasses[policy_class]); HttpRetryQueue & retryq(state.mRetryQueue); HttpReadyQueue & readyq(state.mReadyQueue); + + if (retryq.empty() && readyq.empty()) + { + continue; + } + const bool throttle_enabled(state.mOptions.mThrottleRate > 0L); + const bool throttle_current(throttle_enabled && now < state.mThrottleEnd); + + if (throttle_current && state.mThrottleLeft <= 0) + { + // Throttled condition, don't serve this class but don't sleep hard. + result = HttpService::NORMAL; + continue; + } + + int active(transport.getActiveCountInClass(policy_class)); + int needed(state.mOptions.mConnectionLimit - active); // Expect negatives here + if (needed > 0) { // First see if we have any retries... @@ -216,10 +249,27 @@ HttpService::ELoopSpeed HttpPolicy::processReadyQueue() op->stageFromReady(mService); op->release(); - + + ++state.mRequestCount; --needed; + if (throttle_enabled) + { + if (now >= state.mThrottleEnd) + { + // Throttle expired, move to next window + LL_DEBUGS("CoreHttp") << "Throttle expired with " << state.mThrottleLeft + << " requests to go and " << state.mRequestCount + << " requests issued." << LL_ENDL; + state.mThrottleLeft = state.mOptions.mThrottleRate; + state.mThrottleEnd = now + HttpTime(1000000); + } + if (--state.mThrottleLeft <= 0) + { + goto throttle_on; + } + } } - + // Now go on to the new requests... while (needed > 0 && ! readyq.empty()) { @@ -229,10 +279,29 @@ HttpService::ELoopSpeed HttpPolicy::processReadyQueue() op->stageFromReady(mService); op->release(); + ++state.mRequestCount; --needed; + if (throttle_enabled) + { + if (now >= state.mThrottleEnd) + { + // Throttle expired, move to next window + LL_DEBUGS("CoreHttp") << "Throttle expired with " << state.mThrottleLeft + << " requests to go and " << state.mRequestCount + << " requests issued." << LL_ENDL; + state.mThrottleLeft = state.mOptions.mThrottleRate; + state.mThrottleEnd = now + HttpTime(1000000); + } + if (--state.mThrottleLeft <= 0) + { + goto throttle_on; + } + } } } - + + throttle_on: + if (! readyq.empty() || ! retryq.empty()) { // If anything is ready, continue looping... @@ -246,9 +315,9 @@ HttpService::ELoopSpeed HttpPolicy::processReadyQueue() bool HttpPolicy::changePriority(HttpHandle handle, HttpRequest::priority_t priority) { - for (int policy_class(0); policy_class < mActiveClasses; ++policy_class) + for (int policy_class(0); policy_class < mClasses.size(); ++policy_class) { - State & state(mState[policy_class]); + ClassState & state(*mClasses[policy_class]); // We don't scan retry queue because a priority change there // is meaningless. The request will be issued based on retry // intervals not priority value, which is now moot. @@ -276,9 +345,9 @@ bool HttpPolicy::changePriority(HttpHandle handle, HttpRequest::priority_t prior bool HttpPolicy::cancel(HttpHandle handle) { - for (int policy_class(0); policy_class < mActiveClasses; ++policy_class) + for (int policy_class(0); policy_class < mClasses.size(); ++policy_class) { - State & state(mState[policy_class]); + ClassState & state(*mClasses[policy_class]); // Scan retry queue HttpRetryQueue::container_type & c1(state.mRetryQueue.get_container()); @@ -337,14 +406,14 @@ bool HttpPolicy::stageAfterCompletion(HttpOpRequest * op) LL_WARNS("CoreHttp") << "HTTP request " << static_cast<HttpHandle>(op) << " failed after " << op->mPolicyRetries << " retries. Reason: " << op->mStatus.toString() - << " (" << op->mStatus.toHex() << ")" + << " (" << op->mStatus.toTerseString() << ")" << LL_ENDL; } else if (op->mPolicyRetries) { - LL_WARNS("CoreHttp") << "HTTP request " << static_cast<HttpHandle>(op) - << " succeeded on retry " << op->mPolicyRetries << "." - << LL_ENDL; + LL_DEBUGS("CoreHttp") << "HTTP request " << static_cast<HttpHandle>(op) + << " succeeded on retry " << op->mPolicyRetries << "." + << LL_ENDL; } op->stageFromActive(mService); @@ -352,13 +421,21 @@ bool HttpPolicy::stageAfterCompletion(HttpOpRequest * op) return false; // not active } + +HttpPolicyClass & HttpPolicy::getClassOptions(HttpRequest::policy_t pclass) +{ + llassert_always(pclass >= 0 && pclass < mClasses.size()); + + return mClasses[pclass]->mOptions; +} + int HttpPolicy::getReadyCount(HttpRequest::policy_t policy_class) const { - if (policy_class < mActiveClasses) + if (policy_class < mClasses.size()) { - return (mState[policy_class].mReadyQueue.size() - + mState[policy_class].mRetryQueue.size()); + return (mClasses[policy_class]->mReadyQueue.size() + + mClasses[policy_class]->mRetryQueue.size()); } return 0; } diff --git a/indra/llcorehttp/_httppolicy.h b/indra/llcorehttp/_httppolicy.h index 03d92c0b8e..bf1aa74267 100755 --- a/indra/llcorehttp/_httppolicy.h +++ b/indra/llcorehttp/_httppolicy.h @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -60,6 +60,9 @@ private: void operator=(const HttpPolicy &); // Not defined public: + /// Threading: called by init thread. + HttpRequest::policy_t createPolicyClass(); + /// Cancel all ready and retry requests sending them to /// their notification queues. Release state resources /// making further request handling impossible. @@ -71,9 +74,8 @@ public: /// requests. One-time call invoked before starting /// the worker thread. /// - /// Threading: called by application thread - void start(const HttpPolicyGlobal & global, - const std::vector<HttpPolicyClass> & classes); + /// Threading: called by init thread + void start(); /// Give the policy layer some cycles to scan the ready /// queue promoting higher-priority requests to active @@ -93,7 +95,7 @@ public: /// and should not be modified by anyone until retrieved /// from queue. /// - /// Threading: called by any thread + /// Threading: called by worker thread void addOp(HttpOpRequest *); /// Similar to addOp, used when a caller wants to retry a @@ -130,30 +132,39 @@ public: /// Threading: called by worker thread bool stageAfterCompletion(HttpOpRequest * op); - // Get pointer to global policy options. Caller is expected - // to do context checks like no setting once running. + /// Get a reference to global policy options. Caller is expected + /// to do context checks like no setting once running. These + /// are done, for example, in @see HttpService interfaces. /// /// Threading: called by any thread *but* the object may /// only be modified by the worker thread once running. - /// HttpPolicyGlobal & getGlobalOptions() { return mGlobalOptions; } + /// Get a reference to class policy options. Caller is expected + /// to do context checks like no setting once running. These + /// are done, for example, in @see HttpService interfaces. + /// + /// Threading: called by any thread *but* the object may + /// only be modified by the worker thread once running and + /// read accesses by other threads are exposed to races at + /// that point. + HttpPolicyClass & getClassOptions(HttpRequest::policy_t pclass); + /// Get ready counts for a particular policy class /// /// Threading: called by worker thread int getReadyCount(HttpRequest::policy_t policy_class) const; protected: - struct State; - - int mActiveClasses; - State * mState; - HttpService * mService; // Naked pointer, not refcounted, not owner - HttpPolicyGlobal mGlobalOptions; + struct ClassState; + typedef std::vector<ClassState *> class_list_t; + HttpPolicyGlobal mGlobalOptions; + class_list_t mClasses; + HttpService * mService; // Naked pointer, not refcounted, not owner }; // end class HttpPolicy } // end namespace LLCore diff --git a/indra/llcorehttp/_httppolicyclass.cpp b/indra/llcorehttp/_httppolicyclass.cpp index a23b81322c..f34a8e9f1e 100755 --- a/indra/llcorehttp/_httppolicyclass.cpp +++ b/indra/llcorehttp/_httppolicyclass.cpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -34,10 +34,10 @@ namespace LLCore HttpPolicyClass::HttpPolicyClass() - : mSetMask(0UL), - mConnectionLimit(HTTP_CONNECTION_LIMIT_DEFAULT), + : mConnectionLimit(HTTP_CONNECTION_LIMIT_DEFAULT), mPerHostConnectionLimit(HTTP_CONNECTION_LIMIT_DEFAULT), - mPipelining(0) + mPipelining(HTTP_PIPELINING_DEFAULT), + mThrottleRate(HTTP_THROTTLE_RATE_DEFAULT) {} @@ -49,75 +49,75 @@ HttpPolicyClass & HttpPolicyClass::operator=(const HttpPolicyClass & other) { if (this != &other) { - mSetMask = other.mSetMask; mConnectionLimit = other.mConnectionLimit; mPerHostConnectionLimit = other.mPerHostConnectionLimit; mPipelining = other.mPipelining; + mThrottleRate = other.mThrottleRate; } return *this; } HttpPolicyClass::HttpPolicyClass(const HttpPolicyClass & other) - : mSetMask(other.mSetMask), - mConnectionLimit(other.mConnectionLimit), + : mConnectionLimit(other.mConnectionLimit), mPerHostConnectionLimit(other.mPerHostConnectionLimit), - mPipelining(other.mPipelining) + mPipelining(other.mPipelining), + mThrottleRate(other.mThrottleRate) {} -HttpStatus HttpPolicyClass::set(HttpRequest::EClassPolicy opt, long value) +HttpStatus HttpPolicyClass::set(HttpRequest::EPolicyOption opt, long value) { switch (opt) { - case HttpRequest::CP_CONNECTION_LIMIT: + case HttpRequest::PO_CONNECTION_LIMIT: mConnectionLimit = llclamp(value, long(HTTP_CONNECTION_LIMIT_MIN), long(HTTP_CONNECTION_LIMIT_MAX)); break; - case HttpRequest::CP_PER_HOST_CONNECTION_LIMIT: + case HttpRequest::PO_PER_HOST_CONNECTION_LIMIT: mPerHostConnectionLimit = llclamp(value, long(HTTP_CONNECTION_LIMIT_MIN), mConnectionLimit); break; - case HttpRequest::CP_ENABLE_PIPELINING: + case HttpRequest::PO_ENABLE_PIPELINING: mPipelining = llclamp(value, 0L, 1L); break; + case HttpRequest::PO_THROTTLE_RATE: + mThrottleRate = llclamp(value, 0L, 1000000L); + break; + default: return HttpStatus(HttpStatus::LLCORE, HE_INVALID_ARG); } - mSetMask |= 1UL << int(opt); return HttpStatus(); } -HttpStatus HttpPolicyClass::get(HttpRequest::EClassPolicy opt, long * value) +HttpStatus HttpPolicyClass::get(HttpRequest::EPolicyOption opt, long * value) const { - static const HttpStatus not_set(HttpStatus::LLCORE, HE_OPT_NOT_SET); - long * src(NULL); - switch (opt) { - case HttpRequest::CP_CONNECTION_LIMIT: - src = &mConnectionLimit; + case HttpRequest::PO_CONNECTION_LIMIT: + *value = mConnectionLimit; break; - case HttpRequest::CP_PER_HOST_CONNECTION_LIMIT: - src = &mPerHostConnectionLimit; + case HttpRequest::PO_PER_HOST_CONNECTION_LIMIT: + *value = mPerHostConnectionLimit; break; - case HttpRequest::CP_ENABLE_PIPELINING: - src = &mPipelining; + case HttpRequest::PO_ENABLE_PIPELINING: + *value = mPipelining; + break; + + case HttpRequest::PO_THROTTLE_RATE: + *value = mThrottleRate; break; default: return HttpStatus(HttpStatus::LLCORE, HE_INVALID_ARG); } - if (! (mSetMask & (1UL << int(opt)))) - return not_set; - - *value = *src; return HttpStatus(); } diff --git a/indra/llcorehttp/_httppolicyclass.h b/indra/llcorehttp/_httppolicyclass.h index d175413cbd..38f1194ded 100755 --- a/indra/llcorehttp/_httppolicyclass.h +++ b/indra/llcorehttp/_httppolicyclass.h @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -34,6 +34,18 @@ namespace LLCore { +/// Options struct for per-class policy options. +/// +/// Combines both raw blob data access with semantics-enforcing +/// set/get interfaces. For internal operations by the worker +/// thread, just grab the setting directly from instance and test/use +/// as needed. When attached to external APIs (the public API +/// options interfaces) the set/get methods are available to +/// enforce correct ranges, data types, contexts, etc. and suitable +/// status values are returned. +/// +/// Threading: Single-threaded. In practice, init thread before +/// worker starts, worker thread after. class HttpPolicyClass { public: @@ -44,14 +56,14 @@ public: HttpPolicyClass(const HttpPolicyClass &); // Not defined public: - HttpStatus set(HttpRequest::EClassPolicy opt, long value); - HttpStatus get(HttpRequest::EClassPolicy opt, long * value); + HttpStatus set(HttpRequest::EPolicyOption opt, long value); + HttpStatus get(HttpRequest::EPolicyOption opt, long * value) const; public: - unsigned long mSetMask; long mConnectionLimit; long mPerHostConnectionLimit; long mPipelining; + long mThrottleRate; }; // end class HttpPolicyClass } // end namespace LLCore diff --git a/indra/llcorehttp/_httppolicyglobal.cpp b/indra/llcorehttp/_httppolicyglobal.cpp index 72f409d3b1..1dc95f3dce 100755 --- a/indra/llcorehttp/_httppolicyglobal.cpp +++ b/indra/llcorehttp/_httppolicyglobal.cpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -34,8 +34,7 @@ namespace LLCore HttpPolicyGlobal::HttpPolicyGlobal() - : mSetMask(0UL), - mConnectionLimit(HTTP_CONNECTION_LIMIT_DEFAULT), + : mConnectionLimit(HTTP_CONNECTION_LIMIT_DEFAULT), mTrace(HTTP_TRACE_OFF), mUseLLProxy(0) {} @@ -49,7 +48,6 @@ HttpPolicyGlobal & HttpPolicyGlobal::operator=(const HttpPolicyGlobal & other) { if (this != &other) { - mSetMask = other.mSetMask; mConnectionLimit = other.mConnectionLimit; mCAPath = other.mCAPath; mCAFile = other.mCAFile; @@ -61,19 +59,19 @@ HttpPolicyGlobal & HttpPolicyGlobal::operator=(const HttpPolicyGlobal & other) } -HttpStatus HttpPolicyGlobal::set(HttpRequest::EGlobalPolicy opt, long value) +HttpStatus HttpPolicyGlobal::set(HttpRequest::EPolicyOption opt, long value) { switch (opt) { - case HttpRequest::GP_CONNECTION_LIMIT: + case HttpRequest::PO_CONNECTION_LIMIT: mConnectionLimit = llclamp(value, long(HTTP_CONNECTION_LIMIT_MIN), long(HTTP_CONNECTION_LIMIT_MAX)); break; - case HttpRequest::GP_TRACE: + case HttpRequest::PO_TRACE: mTrace = llclamp(value, long(HTTP_TRACE_MIN), long(HTTP_TRACE_MAX)); break; - case HttpRequest::GP_LLPROXY: + case HttpRequest::PO_LLPROXY: mUseLLProxy = llclamp(value, 0L, 1L); break; @@ -81,24 +79,23 @@ HttpStatus HttpPolicyGlobal::set(HttpRequest::EGlobalPolicy opt, long value) return HttpStatus(HttpStatus::LLCORE, HE_INVALID_ARG); } - mSetMask |= 1UL << int(opt); return HttpStatus(); } -HttpStatus HttpPolicyGlobal::set(HttpRequest::EGlobalPolicy opt, const std::string & value) +HttpStatus HttpPolicyGlobal::set(HttpRequest::EPolicyOption opt, const std::string & value) { switch (opt) { - case HttpRequest::GP_CA_PATH: + case HttpRequest::PO_CA_PATH: mCAPath = value; break; - case HttpRequest::GP_CA_FILE: + case HttpRequest::PO_CA_FILE: mCAFile = value; break; - case HttpRequest::GP_HTTP_PROXY: + case HttpRequest::PO_HTTP_PROXY: mCAFile = value; break; @@ -106,69 +103,54 @@ HttpStatus HttpPolicyGlobal::set(HttpRequest::EGlobalPolicy opt, const std::stri return HttpStatus(HttpStatus::LLCORE, HE_INVALID_ARG); } - mSetMask |= 1UL << int(opt); return HttpStatus(); } -HttpStatus HttpPolicyGlobal::get(HttpRequest::EGlobalPolicy opt, long * value) +HttpStatus HttpPolicyGlobal::get(HttpRequest::EPolicyOption opt, long * value) const { - static const HttpStatus not_set(HttpStatus::LLCORE, HE_OPT_NOT_SET); - long * src(NULL); - switch (opt) { - case HttpRequest::GP_CONNECTION_LIMIT: - src = &mConnectionLimit; + case HttpRequest::PO_CONNECTION_LIMIT: + *value = mConnectionLimit; break; - case HttpRequest::GP_TRACE: - src = &mTrace; + case HttpRequest::PO_TRACE: + *value = mTrace; break; - case HttpRequest::GP_LLPROXY: - src = &mUseLLProxy; + case HttpRequest::PO_LLPROXY: + *value = mUseLLProxy; break; default: return HttpStatus(HttpStatus::LLCORE, HE_INVALID_ARG); } - if (! (mSetMask & (1UL << int(opt)))) - return not_set; - - *value = *src; return HttpStatus(); } -HttpStatus HttpPolicyGlobal::get(HttpRequest::EGlobalPolicy opt, const std::string ** value) +HttpStatus HttpPolicyGlobal::get(HttpRequest::EPolicyOption opt, std::string * value) const { - static const HttpStatus not_set(HttpStatus::LLCORE, HE_OPT_NOT_SET); - const std::string * src(NULL); - switch (opt) { - case HttpRequest::GP_CA_PATH: - src = &mCAPath; + case HttpRequest::PO_CA_PATH: + *value = mCAPath; break; - case HttpRequest::GP_CA_FILE: - src = &mCAFile; + case HttpRequest::PO_CA_FILE: + *value = mCAFile; break; - case HttpRequest::GP_HTTP_PROXY: - src = &mHttpProxy; + case HttpRequest::PO_HTTP_PROXY: + *value = mHttpProxy; break; default: return HttpStatus(HttpStatus::LLCORE, HE_INVALID_ARG); } - if (! (mSetMask & (1UL << int(opt)))) - return not_set; - - *value = src; return HttpStatus(); } diff --git a/indra/llcorehttp/_httppolicyglobal.h b/indra/llcorehttp/_httppolicyglobal.h index a50d0e4188..67c4ba9481 100755 --- a/indra/llcorehttp/_httppolicyglobal.h +++ b/indra/llcorehttp/_httppolicyglobal.h @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -34,6 +34,18 @@ namespace LLCore { +/// Options struct for global policy options. +/// +/// Combines both raw blob data access with semantics-enforcing +/// set/get interfaces. For internal operations by the worker +/// thread, just grab the setting directly from instance and test/use +/// as needed. When attached to external APIs (the public API +/// options interfaces) the set/get methods are available to +/// enforce correct ranges, data types, contexts, etc. and suitable +/// status values are returned. +/// +/// Threading: Single-threaded. In practice, init thread before +/// worker starts, worker thread after. class HttpPolicyGlobal { public: @@ -46,13 +58,12 @@ private: HttpPolicyGlobal(const HttpPolicyGlobal &); // Not defined public: - HttpStatus set(HttpRequest::EGlobalPolicy opt, long value); - HttpStatus set(HttpRequest::EGlobalPolicy opt, const std::string & value); - HttpStatus get(HttpRequest::EGlobalPolicy opt, long * value); - HttpStatus get(HttpRequest::EGlobalPolicy opt, const std::string ** value); + HttpStatus set(HttpRequest::EPolicyOption opt, long value); + HttpStatus set(HttpRequest::EPolicyOption opt, const std::string & value); + HttpStatus get(HttpRequest::EPolicyOption opt, long * value) const; + HttpStatus get(HttpRequest::EPolicyOption opt, std::string * value) const; public: - unsigned long mSetMask; long mConnectionLimit; std::string mCAPath; std::string mCAFile; diff --git a/indra/llcorehttp/_httpservice.cpp b/indra/llcorehttp/_httpservice.cpp index 0825888d0f..c94249dc2d 100755 --- a/indra/llcorehttp/_httpservice.cpp +++ b/indra/llcorehttp/_httpservice.cpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -43,6 +43,18 @@ namespace LLCore { +const HttpService::OptionDescriptor HttpService::sOptionDesc[] = +{ // isLong isDynamic isGlobal isClass + { true, true, true, true }, // PO_CONNECTION_LIMIT + { true, true, false, true }, // PO_PER_HOST_CONNECTION_LIMIT + { false, false, true, false }, // PO_CA_PATH + { false, false, true, false }, // PO_CA_FILE + { false, true, true, false }, // PO_HTTP_PROXY + { true, true, true, false }, // PO_LLPROXY + { true, true, true, false }, // PO_TRACE + { true, true, false, true }, // PO_ENABLE_PIPELINING + { true, true, false, true } // PO_THROTTLE_RATE +}; HttpService * HttpService::sInstance(NULL); volatile HttpService::EState HttpService::sState(NOT_INITIALIZED); @@ -51,15 +63,9 @@ HttpService::HttpService() mExitRequested(0U), mThread(NULL), mPolicy(NULL), - mTransport(NULL) -{ - // Create the default policy class - HttpPolicyClass pol_class; - pol_class.set(HttpRequest::CP_CONNECTION_LIMIT, HTTP_CONNECTION_LIMIT_DEFAULT); - pol_class.set(HttpRequest::CP_PER_HOST_CONNECTION_LIMIT, HTTP_CONNECTION_LIMIT_DEFAULT); - pol_class.set(HttpRequest::CP_ENABLE_PIPELINING, 0L); - mPolicyClasses.push_back(pol_class); -} + mTransport(NULL), + mLastPolicy(0) +{} HttpService::~HttpService() @@ -149,13 +155,8 @@ void HttpService::term() HttpRequest::policy_t HttpService::createPolicyClass() { - const HttpRequest::policy_t policy_class(mPolicyClasses.size()); - if (policy_class >= HTTP_POLICY_CLASS_LIMIT) - { - return 0; - } - mPolicyClasses.push_back(HttpPolicyClass()); - return policy_class; + mLastPolicy = mPolicy->createPolicyClass(); + return mLastPolicy; } @@ -188,8 +189,8 @@ void HttpService::startThread() } // Push current policy definitions, enable policy & transport components - mPolicy->start(mPolicyGlobal, mPolicyClasses); - mTransport->start(mPolicyClasses.size()); + mPolicy->start(); + mTransport->start(mLastPolicy + 1); mThread = new LLCoreInt::HttpThread(boost::bind(&HttpService::threadRun, this, _1)); sState = RUNNING; @@ -322,7 +323,7 @@ HttpService::ELoopSpeed HttpService::processRequestQueue(ELoopSpeed loop) { // Setup for subsequent tracing long tracing(HTTP_TRACE_OFF); - mPolicy->getGlobalOptions().get(HttpRequest::GP_TRACE, &tracing); + mPolicy->getGlobalOptions().get(HttpRequest::PO_TRACE, &tracing); op->mTracing = (std::max)(op->mTracing, int(tracing)); if (op->mTracing > HTTP_TRACE_OFF) @@ -345,4 +346,137 @@ HttpService::ELoopSpeed HttpService::processRequestQueue(ELoopSpeed loop) } +HttpStatus HttpService::getPolicyOption(HttpRequest::EPolicyOption opt, HttpRequest::policy_t pclass, + long * ret_value) +{ + if (opt < HttpRequest::PO_CONNECTION_LIMIT // option must be in range + || opt >= HttpRequest::PO_LAST // ditto + || (! sOptionDesc[opt].mIsLong) // datatype is long + || (pclass != HttpRequest::GLOBAL_POLICY_ID && pclass > mLastPolicy) // pclass in valid range + || (pclass == HttpRequest::GLOBAL_POLICY_ID && ! sOptionDesc[opt].mIsGlobal) // global setting permitted + || (pclass != HttpRequest::GLOBAL_POLICY_ID && ! sOptionDesc[opt].mIsClass)) // class setting permitted + // can always get, no dynamic check + { + return HttpStatus(HttpStatus::LLCORE, LLCore::HE_INVALID_ARG); + } + + HttpStatus status; + if (pclass == HttpRequest::GLOBAL_POLICY_ID) + { + HttpPolicyGlobal & opts(mPolicy->getGlobalOptions()); + + status = opts.get(opt, ret_value); + } + else + { + HttpPolicyClass & opts(mPolicy->getClassOptions(pclass)); + + status = opts.get(opt, ret_value); + } + + return status; +} + + +HttpStatus HttpService::getPolicyOption(HttpRequest::EPolicyOption opt, HttpRequest::policy_t pclass, + std::string * ret_value) +{ + HttpStatus status(HttpStatus::LLCORE, LLCore::HE_INVALID_ARG); + + if (opt < HttpRequest::PO_CONNECTION_LIMIT // option must be in range + || opt >= HttpRequest::PO_LAST // ditto + || (sOptionDesc[opt].mIsLong) // datatype is string + || (pclass != HttpRequest::GLOBAL_POLICY_ID && pclass > mLastPolicy) // pclass in valid range + || (pclass == HttpRequest::GLOBAL_POLICY_ID && ! sOptionDesc[opt].mIsGlobal) // global setting permitted + || (pclass != HttpRequest::GLOBAL_POLICY_ID && ! sOptionDesc[opt].mIsClass)) // class setting permitted + // can always get, no dynamic check + { + return status; + } + + // Only global has string values + if (pclass == HttpRequest::GLOBAL_POLICY_ID) + { + HttpPolicyGlobal & opts(mPolicy->getGlobalOptions()); + + status = opts.get(opt, ret_value); + } + + return status; +} + + +HttpStatus HttpService::setPolicyOption(HttpRequest::EPolicyOption opt, HttpRequest::policy_t pclass, + long value, long * ret_value) +{ + HttpStatus status(HttpStatus::LLCORE, LLCore::HE_INVALID_ARG); + + if (opt < HttpRequest::PO_CONNECTION_LIMIT // option must be in range + || opt >= HttpRequest::PO_LAST // ditto + || (! sOptionDesc[opt].mIsLong) // datatype is long + || (pclass != HttpRequest::GLOBAL_POLICY_ID && pclass > mLastPolicy) // pclass in valid range + || (pclass == HttpRequest::GLOBAL_POLICY_ID && ! sOptionDesc[opt].mIsGlobal) // global setting permitted + || (pclass != HttpRequest::GLOBAL_POLICY_ID && ! sOptionDesc[opt].mIsClass) // class setting permitted + || (RUNNING == sState && ! sOptionDesc[opt].mIsDynamic)) // dynamic setting permitted + { + return status; + } + + if (pclass == HttpRequest::GLOBAL_POLICY_ID) + { + HttpPolicyGlobal & opts(mPolicy->getGlobalOptions()); + + status = opts.set(opt, value); + if (status && ret_value) + { + status = opts.get(opt, ret_value); + } + } + else + { + HttpPolicyClass & opts(mPolicy->getClassOptions(pclass)); + + status = opts.set(opt, value); + if (status && ret_value) + { + status = opts.get(opt, ret_value); + } + } + + return status; +} + + +HttpStatus HttpService::setPolicyOption(HttpRequest::EPolicyOption opt, HttpRequest::policy_t pclass, + const std::string & value, std::string * ret_value) +{ + HttpStatus status(HttpStatus::LLCORE, LLCore::HE_INVALID_ARG); + + if (opt < HttpRequest::PO_CONNECTION_LIMIT // option must be in range + || opt >= HttpRequest::PO_LAST // ditto + || (sOptionDesc[opt].mIsLong) // datatype is string + || (pclass != HttpRequest::GLOBAL_POLICY_ID && pclass > mLastPolicy) // pclass in valid range + || (pclass == HttpRequest::GLOBAL_POLICY_ID && ! sOptionDesc[opt].mIsGlobal) // global setting permitted + || (pclass != HttpRequest::GLOBAL_POLICY_ID && ! sOptionDesc[opt].mIsClass) // class setting permitted + || (RUNNING == sState && ! sOptionDesc[opt].mIsDynamic)) // dynamic setting permitted + { + return status; + } + + // Only string values are global at this time + if (pclass == HttpRequest::GLOBAL_POLICY_ID) + { + HttpPolicyGlobal & opts(mPolicy->getGlobalOptions()); + + status = opts.set(opt, value); + if (status && ret_value) + { + status = opts.get(opt, ret_value); + } + } + + return status; +} + + } // end namespace LLCore diff --git a/indra/llcorehttp/_httpservice.h b/indra/llcorehttp/_httpservice.h index ffe0349d4d..cf23f3ab61 100755 --- a/indra/llcorehttp/_httpservice.h +++ b/indra/llcorehttp/_httpservice.h @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -53,6 +53,7 @@ namespace LLCore class HttpRequestQueue; class HttpPolicy; class HttpLibcurl; +class HttpOpSetGet; /// The HttpService class does the work behind the request queue. It @@ -106,7 +107,7 @@ public: NORMAL, ///< continuous polling of request, ready, active queues REQUEST_SLEEP ///< can sleep indefinitely waiting for request queue write }; - + static void init(HttpRequestQueue *); static void term(); @@ -136,7 +137,7 @@ public: /// acquires its weaknesses. static bool isStopped(); - /// Threading: callable by consumer thread *once*. + /// Threading: callable by init thread *once*. void startThread(); /// Threading: callable by worker thread. @@ -181,27 +182,38 @@ public: } /// Threading: callable by consumer thread. - HttpPolicyGlobal & getGlobalOptions() - { - return mPolicyGlobal; - } - - /// Threading: callable by consumer thread. HttpRequest::policy_t createPolicyClass(); - /// Threading: callable by consumer thread. - HttpPolicyClass & getClassOptions(HttpRequest::policy_t policy_class) - { - llassert(policy_class >= 0 && policy_class < mPolicyClasses.size()); - return mPolicyClasses[policy_class]; - } - protected: void threadRun(LLCoreInt::HttpThread * thread); ELoopSpeed processRequestQueue(ELoopSpeed loop); + +protected: + friend class HttpOpSetGet; + friend class HttpRequest; + + // Used internally to describe what operations are allowed + // on each policy option. + struct OptionDescriptor + { + bool mIsLong; + bool mIsDynamic; + bool mIsGlobal; + bool mIsClass; + }; + + HttpStatus getPolicyOption(HttpRequest::EPolicyOption opt, HttpRequest::policy_t, + long * ret_value); + HttpStatus getPolicyOption(HttpRequest::EPolicyOption opt, HttpRequest::policy_t, + std::string * ret_value); + HttpStatus setPolicyOption(HttpRequest::EPolicyOption opt, HttpRequest::policy_t, + long value, long * ret_value); + HttpStatus setPolicyOption(HttpRequest::EPolicyOption opt, HttpRequest::policy_t, + const std::string & value, std::string * ret_value); protected: + static const OptionDescriptor sOptionDesc[HttpRequest::PO_LAST]; static HttpService * sInstance; // === shared data === @@ -210,13 +222,13 @@ protected: LLAtomicU32 mExitRequested; LLCoreInt::HttpThread * mThread; - // === consumer-thread-only data === - HttpPolicyGlobal mPolicyGlobal; - std::vector<HttpPolicyClass> mPolicyClasses; - // === working-thread-only data === HttpPolicy * mPolicy; // Simple pointer, has ownership HttpLibcurl * mTransport; // Simple pointer, has ownership + + // === main-thread-only data === + HttpRequest::policy_t mLastPolicy; + }; // end class HttpService } // end namespace LLCore diff --git a/indra/llcorehttp/examples/http_texture_load.cpp b/indra/llcorehttp/examples/http_texture_load.cpp index 40ad4f047d..73c49687d7 100755 --- a/indra/llcorehttp/examples/http_texture_load.cpp +++ b/indra/llcorehttp/examples/http_texture_load.cpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -39,6 +39,7 @@ #include "httprequest.h" #include "httphandler.h" #include "httpresponse.h" +#include "httpoptions.h" #include "httpheaders.h" #include "bufferarray.h" #include "_mutex.h" @@ -57,6 +58,7 @@ void usage(std::ostream & out); // Default command line settings static int concurrency_limit(40); +static int highwater(100); static char url_format[1024] = "http://example.com/some/path?texture_id=%s.texture"; #if defined(WIN32) @@ -79,11 +81,11 @@ public: WorkingSet(); ~WorkingSet(); - bool reload(LLCore::HttpRequest *); + bool reload(LLCore::HttpRequest *, LLCore::HttpOptions *); virtual void onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * response); - void loadTextureUuids(FILE * in); + void loadAssetUuids(FILE * in); public: struct Spec @@ -93,24 +95,27 @@ public: int mLength; }; typedef std::set<LLCore::HttpHandle> handle_set_t; - typedef std::vector<Spec> texture_list_t; + typedef std::vector<Spec> asset_list_t; public: bool mVerbose; bool mRandomRange; - int mMaxConcurrency; + int mRequestLowWater; + int mRequestHighWater; handle_set_t mHandles; int mRemaining; int mLimit; int mAt; std::string mUrl; - texture_list_t mTextures; + asset_list_t mAssets; int mErrorsApi; int mErrorsHttp; int mErrorsHttp404; int mErrorsHttp416; int mErrorsHttp500; int mErrorsHttp503; + int mRetries; + int mRetriesHttp503; int mSuccesses; long mByteCount; LLCore::HttpHeaders * mHeaders; @@ -158,7 +163,7 @@ int main(int argc, char** argv) bool do_verbose(false); int option(-1); - while (-1 != (option = getopt(argc, argv, "u:c:h?Rv"))) + while (-1 != (option = getopt(argc, argv, "u:c:h?RvH:"))) { switch (option) { @@ -182,6 +187,21 @@ int main(int argc, char** argv) } break; + case 'H': + { + unsigned long value; + char * end; + + value = strtoul(optarg, &end, 10); + if (value < 1 || value > 100 || *end != '\0') + { + usage(std::cerr); + return 1; + } + highwater = value; + } + break; + case 'R': do_random = true; break; @@ -216,25 +236,32 @@ int main(int argc, char** argv) // Initialization init_curl(); LLCore::HttpRequest::createService(); - LLCore::HttpRequest::setPolicyClassOption(LLCore::HttpRequest::DEFAULT_POLICY_ID, - LLCore::HttpRequest::CP_CONNECTION_LIMIT, - concurrency_limit); + LLCore::HttpRequest::setStaticPolicyOption(LLCore::HttpRequest::PO_CONNECTION_LIMIT, + LLCore::HttpRequest::DEFAULT_POLICY_ID, + concurrency_limit, + NULL); LLCore::HttpRequest::startThread(); // Get service point LLCore::HttpRequest * hr = new LLCore::HttpRequest(); + // Get request options + LLCore::HttpOptions * opt = new LLCore::HttpOptions(); + opt->setRetries(12); + opt->setUseRetryAfter(true); + // Get a handler/working set WorkingSet ws; // Fill the working set with work ws.mUrl = url_format; - ws.loadTextureUuids(uuids); + ws.loadAssetUuids(uuids); ws.mRandomRange = do_random; ws.mVerbose = do_verbose; - ws.mMaxConcurrency = 100; + ws.mRequestHighWater = highwater; + ws.mRequestLowWater = ws.mRequestHighWater / 2; - if (! ws.mTextures.size()) + if (! ws.mAssets.size()) { std::cerr << "No UUIDs found in file '" << argv[optind] << "'." << std::endl; return 1; @@ -246,9 +273,9 @@ int main(int argc, char** argv) // Run it int passes(0); - while (! ws.reload(hr)) + while (! ws.reload(hr, opt)) { - hr->update(5000000); + hr->update(0); ms_sleep(2); if (0 == (++passes % 200)) { @@ -265,6 +292,8 @@ int main(int argc, char** argv) std::cout << "HTTP 404 errors: " << ws.mErrorsHttp404 << " HTTP 416 errors: " << ws.mErrorsHttp416 << " HTTP 500 errors: " << ws.mErrorsHttp500 << " HTTP 503 errors: " << ws.mErrorsHttp503 << std::endl; + std::cout << "Retries: " << ws.mRetries << " Retries on 503: " << ws.mRetriesHttp503 + << std::endl; std::cout << "User CPU: " << (metrics.mEndUTime - metrics.mStartUTime) << " uS System CPU: " << (metrics.mEndSTime - metrics.mStartSTime) << " uS Wall Time: " << (metrics.mEndWallTime - metrics.mStartWallTime) @@ -275,6 +304,8 @@ int main(int argc, char** argv) // Clean up hr->requestStopThread(NULL); ms_sleep(1000); + opt->release(); + opt = NULL; delete hr; LLCore::HttpRequest::destroyService(); term_curl(); @@ -300,8 +331,10 @@ void usage(std::ostream & out) " -u <url_format> printf-style format string for URL generation\n" " Default: " << url_format << "\n" " -R Issue GETs with random Range: headers\n" - " -c <limit> Maximum request concurrency. Range: [1..100]\n" + " -c <limit> Maximum connection concurrency. Range: [1..100]\n" " Default: " << concurrency_limit << "\n" + " -H <limit> HTTP request highwater (requests fed to llcorehttp).\n" + " Range: [1..100] Default: " << highwater << "\n" " -v Verbose mode. Issue some chatter while running\n" " -h print this help\n" "\n" @@ -322,13 +355,15 @@ WorkingSet::WorkingSet() mErrorsHttp416(0), mErrorsHttp500(0), mErrorsHttp503(0), + mRetries(0), + mRetriesHttp503(0), mSuccesses(0), mByteCount(0L) { - mTextures.reserve(30000); + mAssets.reserve(30000); mHeaders = new LLCore::HttpHeaders; - mHeaders->mHeaders.push_back("Accept: image/x-j2c"); + mHeaders->append("Accept", "image/x-j2c"); } @@ -342,29 +377,35 @@ WorkingSet::~WorkingSet() } -bool WorkingSet::reload(LLCore::HttpRequest * hr) +bool WorkingSet::reload(LLCore::HttpRequest * hr, LLCore::HttpOptions * opt) { - int to_do((std::min)(mRemaining, mMaxConcurrency - int(mHandles.size()))); + if (mRequestLowWater <= mHandles.size()) + { + // Haven't fallen below low-water level yet. + return false; + } + + int to_do((std::min)(mRemaining, mRequestHighWater - int(mHandles.size()))); for (int i(0); i < to_do; ++i) { char buffer[1024]; #if defined(WIN32) - _snprintf_s(buffer, sizeof(buffer), sizeof(buffer) - 1, mUrl.c_str(), mTextures[mAt].mUuid.c_str()); + _snprintf_s(buffer, sizeof(buffer), sizeof(buffer) - 1, mUrl.c_str(), mAssets[mAt].mUuid.c_str()); #else - snprintf(buffer, sizeof(buffer), mUrl.c_str(), mTextures[mAt].mUuid.c_str()); + snprintf(buffer, sizeof(buffer), mUrl.c_str(), mAssets[mAt].mUuid.c_str()); #endif - int offset(mRandomRange ? ((unsigned long) rand()) % 1000000UL : mTextures[mAt].mOffset); - int length(mRandomRange ? ((unsigned long) rand()) % 1000000UL : mTextures[mAt].mLength); + int offset(mRandomRange ? ((unsigned long) rand()) % 1000000UL : mAssets[mAt].mOffset); + int length(mRandomRange ? ((unsigned long) rand()) % 1000000UL : mAssets[mAt].mLength); LLCore::HttpHandle handle; if (offset || length) { - handle = hr->requestGetByteRange(0, 0, buffer, offset, length, NULL, mHeaders, this); + handle = hr->requestGetByteRange(0, 0, buffer, offset, length, opt, mHeaders, this); } else { - handle = hr->requestGet(0, 0, buffer, NULL, mHeaders, this); + handle = hr->requestGet(0, 0, buffer, opt, mHeaders, this); } if (! handle) { @@ -410,7 +451,7 @@ void WorkingSet::onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * r { // More success LLCore::BufferArray * data(response->getBody()); - mByteCount += data->size(); + mByteCount += data ? data->size() : 0; ++mSuccesses; } else @@ -446,6 +487,10 @@ void WorkingSet::onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * r ++mErrorsApi; } } + unsigned int retry(0U), retry_503(0U); + response->getRetries(&retry, &retry_503); + mRetries += int(retry); + mRetriesHttp503 += int(retry_503); mHandles.erase(it); } @@ -459,21 +504,21 @@ void WorkingSet::onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * r } -void WorkingSet::loadTextureUuids(FILE * in) +void WorkingSet::loadAssetUuids(FILE * in) { char buffer[1024]; while (fgets(buffer, sizeof(buffer), in)) { - WorkingSet::Spec texture; + WorkingSet::Spec asset; char * state(NULL); char * token = strtok_r(buffer, " \t\n,", &state); if (token && 36 == strlen(token)) { // Close enough for this function - texture.mUuid = token; - texture.mOffset = 0; - texture.mLength = 0; + asset.mUuid = token; + asset.mOffset = 0; + asset.mLength = 0; token = strtok_r(buffer, " \t\n,", &state); if (token) { @@ -482,14 +527,14 @@ void WorkingSet::loadTextureUuids(FILE * in) if (token) { int length(atoi(token)); - texture.mOffset = offset; - texture.mLength = length; + asset.mOffset = offset; + asset.mLength = length; } } - mTextures.push_back(texture); + mAssets.push_back(asset); } } - mRemaining = mLimit = mTextures.size(); + mRemaining = mLimit = mAssets.size(); } diff --git a/indra/llcorehttp/httpcommon.cpp b/indra/llcorehttp/httpcommon.cpp index 0738760763..c2f15155ac 100755 --- a/indra/llcorehttp/httpcommon.cpp +++ b/indra/llcorehttp/httpcommon.cpp @@ -70,7 +70,8 @@ std::string HttpStatus::toString() const "Invalid datatype for argument or option", "Option has not been explicitly set", "Option is not dynamic and must be set early", - "Invalid HTTP status code received from server" + "Invalid HTTP status code received from server", + "Could not allocate required resource" }; static const int llcore_errors_count(sizeof(llcore_errors) / sizeof(llcore_errors[0])); @@ -177,6 +178,44 @@ std::string HttpStatus::toString() const } +std::string HttpStatus::toTerseString() const +{ + std::ostringstream result; + + unsigned int error_value((unsigned short) mStatus); + + switch (mType) + { + case EXT_CURL_EASY: + result << "Easy_"; + break; + + case EXT_CURL_MULTI: + result << "Multi_"; + break; + + case LLCORE: + result << "Core_"; + break; + + default: + if (isHttpStatus()) + { + result << "Http_"; + error_value = mType; + } + else + { + result << "Unknown_"; + } + break; + } + + result << error_value; + return result.str(); +} + + // Pass true on statuses that might actually be cleared by a // retry. Library failures, calling problems, etc. aren't // going to be fixed by squirting bits all over the Net. @@ -206,6 +245,5 @@ bool HttpStatus::isRetryable() const *this == inv_cont_range); // Short data read disagrees with content-range } - } // end namespace LLCore diff --git a/indra/llcorehttp/httpcommon.h b/indra/llcorehttp/httpcommon.h index 41fb5164cf..9601f94125 100755 --- a/indra/llcorehttp/httpcommon.h +++ b/indra/llcorehttp/httpcommon.h @@ -29,9 +29,9 @@ /// @package LLCore::HTTP /// -/// This library implements a high-level, Indra-code-free client interface to -/// HTTP services based on actual patterns found in the viewer and simulator. -/// Interfaces are similar to those supplied by the legacy classes +/// This library implements a high-level, Indra-code-free (somewhat) client +/// interface to HTTP services based on actual patterns found in the viewer +/// and simulator. Interfaces are similar to those supplied by the legacy classes /// LLCurlRequest and LLHTTPClient. To that is added a policy scheme that /// allows an application to specify connection behaviors: limits on /// connections, HTTP keepalive, HTTP pipelining, retry-on-error limits, etc. @@ -52,7 +52,7 @@ /// - "llcorehttp/httprequest.h" /// - "llcorehttp/httpresponse.h" /// -/// The library is still under early development and particular users +/// The library is still under development and particular users /// may need access to internal implementation details that are found /// in the _*.h header files. But this is a crutch to be avoided if at /// all possible and probably indicates some interface work is neeeded. @@ -66,6 +66,8 @@ /// . CRYPTO_set_id_callback(...) /// - HttpRequest::createService() called to instantiate singletons /// and support objects. +/// - HttpRequest::startThread() to kick off the worker thread and +/// begin servicing requests. /// /// An HTTP consumer in an application, and an application may have many /// consumers, does a few things: @@ -91,10 +93,12 @@ /// objects. /// - Do completion processing in your onCompletion() method. /// -/// Code fragments: -/// Rather than a poorly-maintained example in comments, look in the -/// example subdirectory which is a minimal yet functional tool to do -/// GET request performance testing. With four calls: +/// Code fragments. +/// +/// Initialization. Rather than a poorly-maintained example in +/// comments, look in the example subdirectory which is a minimal +/// yet functional tool to do GET request performance testing. +/// With four calls: /// /// init_curl(); /// LLCore::HttpRequest::createService(); @@ -103,7 +107,85 @@ /// /// the program is basically ready to issue requests. /// - +/// HttpHandler. Having started life as a non-indra library, +/// this code broke away from the classic Responder model and +/// introduced a handler class to represent an interface for +/// request responses. This is a non-reference-counted entity +/// which can be used as a base class or a mixin. An instance +/// of a handler can be used for each request or can be shared +/// among any number of requests. Your choice but expect to +/// code something like the following: +/// +/// class AppHandler : public LLCore::HttpHandler +/// { +/// public: +/// virtual void onCompleted(HttpHandle handle, +/// HttpResponse * response) +/// { +/// ... +/// } +/// ... +/// }; +/// ... +/// handler = new handler(...); +/// +/// +/// Issuing requests. Using 'hr' above, +/// +/// hr->requestGet(HttpRequest::DEFAULT_POLICY_ID, +/// 0, // Priority, not used yet +/// url, +/// NULL, // options +/// NULL, // additional headers +/// handler); +/// +/// If that returns a value other than LLCORE_HTTP_HANDLE_INVALID, +/// the request was successfully issued and there will eventally +/// be a status delivered to the handler. If invalid is returnedd, +/// the actual status can be retrieved by calling hr->getStatus(). +/// +/// Completing requests and delivering notifications. Operations +/// are all performed by the worker thread and will be driven to +/// completion regardless of caller actions. Notification of +/// completion (success or failure) is done by calls to +/// HttpRequest::update() which will invoke handlers for completed +/// requests: +/// +/// hr->update(0); +/// // Callbacks into handler->onCompleted() +/// +/// +/// Threads. +/// +/// Threads are supported and used by this library. The various +/// classes, methods and members are documented with thread +/// constraints which programmers must follow and which are +/// defined as follows: +/// +/// consumer Any thread that has instanced HttpRequest and is +/// issuing requests. A particular instance can only +/// be used by one consumer thread but a consumer may +/// have many instances available to it. +/// init Special consumer thread, usually the main thread, +/// involved in setting up the library at startup. +/// worker Thread used internally by the library to perform +/// HTTP operations. Consumers will not have to deal +/// with this thread directly but some APIs are reserved +/// to it. +/// any Consumer or worker thread. +/// +/// For the most part, API users will not have to do much in the +/// way of ensuring thread safely. However, there is a tremendous +/// amount of sharing between threads of read-only data. So when +/// documentation declares that an option or header instance +/// becomes shared between consumer and worker, the consumer must +/// not modify the shared object. +/// +/// Internally, there is almost no thread synchronization. During +/// normal operations (non-init, non-term), only the request queue +/// and the multiple reply queues are shared between threads and +/// only here are mutexes used. +/// #include "linden_common.h" // Modifies curl/curl.h interfaces @@ -164,7 +246,10 @@ enum HttpError HE_OPT_NOT_DYNAMIC = 8, // Invalid HTTP status code returned by server - HE_INVALID_HTTP_STATUS = 9 + HE_INVALID_HTTP_STATUS = 9, + + // Couldn't allocate resource, typically libcurl handle + HE_BAD_ALLOC = 10 }; // end enum HttpError @@ -239,9 +324,10 @@ struct HttpStatus return *this; } - static const type_enum_t EXT_CURL_EASY = 0; - static const type_enum_t EXT_CURL_MULTI = 1; - static const type_enum_t LLCORE = 2; + static const type_enum_t EXT_CURL_EASY = 0; ///< mStatus is an error from a curl_easy_*() call + static const type_enum_t EXT_CURL_MULTI = 1; ///< mStatus is an error from a curl_multi_*() call + static const type_enum_t LLCORE = 2; ///< mStatus is an HE_* error code + ///< 100-999 directly represent HTTP status codes type_enum_t mType; short mStatus; @@ -297,6 +383,14 @@ struct HttpStatus /// LLCore itself). std::string toString() const; + /// Convert status to a compact string representation + /// of the form: "<type>_<value>". The <type> will be + /// one of: Core, Http, Easy, Multi, Unknown. And + /// <value> will be an unsigned integer. More easily + /// interpreted than the hex representation, it's still + /// compact and easily searched. + std::string toTerseString() const; + /// Returns true if the status value represents an /// HTTP response status (100 - 999). bool isHttpStatus() const diff --git a/indra/llcorehttp/httpheaders.cpp b/indra/llcorehttp/httpheaders.cpp index 2832696271..23ebea361c 100755 --- a/indra/llcorehttp/httpheaders.cpp +++ b/indra/llcorehttp/httpheaders.cpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -26,6 +26,8 @@ #include "httpheaders.h" +#include "llstring.h" + namespace LLCore { @@ -40,5 +42,142 @@ HttpHeaders::~HttpHeaders() {} +void +HttpHeaders::clear() +{ + mHeaders.clear(); +} + + +void HttpHeaders::append(const std::string & name, const std::string & value) +{ + mHeaders.push_back(value_type(name, value)); +} + + +void HttpHeaders::append(const char * name, const char * value) +{ + mHeaders.push_back(value_type(name, value)); +} + + +void HttpHeaders::appendNormal(const char * header, size_t size) +{ + std::string name; + std::string value; + + int col_pos(0); + for (; col_pos < size; ++col_pos) + { + if (':' == header[col_pos]) + break; + } + + if (col_pos < size) + { + // Looks like a header, split it and normalize. + // Name is everything before the colon, may be zero-length. + name.assign(header, col_pos); + + // Value is everything after the colon, may also be zero-length. + const size_t val_len(size - col_pos - 1); + if (val_len) + { + value.assign(header + col_pos + 1, val_len); + } + + // Clean the strings + LLStringUtil::toLower(name); + LLStringUtil::trim(name); + LLStringUtil::trimHead(value); + } + else + { + // Uncertain what this is, we'll pack it as + // a name without a value. Won't clean as we don't + // know what it is... + name.assign(header, size); + } + + mHeaders.push_back(value_type(name, value)); +} + + +// Find from end to simulate a tradition of using single-valued +// std::map for this in the past. +const std::string * HttpHeaders::find(const char * name) const +{ + const_reverse_iterator iend(rend()); + for (const_reverse_iterator iter(rbegin()); iend != iter; ++iter) + { + if ((*iter).first == name) + { + return &(*iter).second; + } + } + return NULL; +} + + +// Standard Iterators +HttpHeaders::iterator HttpHeaders::begin() +{ + return mHeaders.begin(); +} + + +HttpHeaders::const_iterator HttpHeaders::begin() const +{ + return mHeaders.begin(); +} + + +HttpHeaders::iterator HttpHeaders::end() +{ + return mHeaders.end(); +} + + +HttpHeaders::const_iterator HttpHeaders::end() const +{ + return mHeaders.end(); +} + + +// Standard Reverse Iterators +HttpHeaders::reverse_iterator HttpHeaders::rbegin() +{ + return mHeaders.rbegin(); +} + + +HttpHeaders::const_reverse_iterator HttpHeaders::rbegin() const +{ + return mHeaders.rbegin(); +} + + +HttpHeaders::reverse_iterator HttpHeaders::rend() +{ + return mHeaders.rend(); +} + + +HttpHeaders::const_reverse_iterator HttpHeaders::rend() const +{ + return mHeaders.rend(); +} + + +// Return the raw container to the caller. +// +// To be used FOR UNIT TESTS ONLY. +// +HttpHeaders::container_t & HttpHeaders::getContainerTESTONLY() +{ + return mHeaders; +} + + } // end namespace LLCore diff --git a/indra/llcorehttp/httpheaders.h b/indra/llcorehttp/httpheaders.h index 3449daa3a1..f70cd898f3 100755 --- a/indra/llcorehttp/httpheaders.h +++ b/indra/llcorehttp/httpheaders.h @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -43,13 +43,26 @@ namespace LLCore /// caller has asked that headers be returned (not the default /// option). /// -/// @note -/// This is a minimally-functional placeholder at the moment -/// to fill out the class hierarchy. The final class will be -/// something else, probably more pair-oriented. It's also -/// an area where shared values are desirable so refcounting is -/// already specced and a copy-on-write scheme imagined. -/// Expect changes here. +/// Class is mostly a thin wrapper around a vector of pairs +/// of strings. Methods provided are few and intended to +/// reflect actual use patterns. These include: +/// - Clearing the list +/// - Appending a name/value pair to the vector +/// - Processing a raw byte string into a normalized name/value +/// pair and appending the result. +/// - Simple case-sensitive find-last-by-name search +/// - Forward and reverse iterators over all pairs +/// +/// Container is ordered and multi-valued. Headers are +/// written in the order in which they are appended and +/// are stored in the order in which they're received from +/// the wire. The same header may appear two or more times +/// in any container. Searches using the simple find() +/// interface will find only the last occurrence (somewhat +/// simulates the use of std::map). Fuller searches require +/// the use of an iterator. Headers received from the wire +/// are only returned from the last request when redirections +/// are involved. /// /// Threading: Not intrinsically thread-safe. It *is* expected /// that callers will build these objects and then share them @@ -64,6 +77,16 @@ namespace LLCore class HttpHeaders : public LLCoreInt::RefCounted { public: + typedef std::pair<std::string, std::string> header_t; + typedef std::vector<header_t> container_t; + typedef container_t::iterator iterator; + typedef container_t::const_iterator const_iterator; + typedef container_t::reverse_iterator reverse_iterator; + typedef container_t::const_reverse_iterator const_reverse_iterator; + typedef container_t::value_type value_type; + typedef container_t::size_type size_type; + +public: /// @post In addition to the instance, caller has a refcount /// to the instance. A call to @see release() will destroy /// the instance. @@ -76,7 +99,78 @@ protected: void operator=(const HttpHeaders &); // Not defined public: - typedef std::vector<std::string> container_t; + // Empty the list of headers. + void clear(); + + // Append a name/value pair supplied as either std::strings + // or NUL-terminated char * to the header list. No normalization + // is performed on the strings. No conformance test is + // performed (names may contain spaces, colons, etc.). + // + void append(const std::string & name, const std::string & value); + void append(const char * name, const char * value); + + // Extract a name/value pair from a raw byte array using + // the first colon character as a separator. Input string + // does not need to be NUL-terminated. Resulting name/value + // pair is appended to the header list. + // + // Normalization is performed on the name/value pair as + // follows: + // - name is lower-cased according to mostly ASCII rules + // - name is left- and right-trimmed of spaces and tabs + // - value is left-trimmed of spaces and tabs + // - either or both of name and value may be zero-length + // + // By convention, headers read from the wire will be normalized + // in this fashion prior to delivery to any HttpHandler code. + // Headers to be written to the wire are left as appended to + // the list. + void appendNormal(const char * header, size_t size); + + // Perform a simple, case-sensitive search of the header list + // returning a pointer to the value of the last matching header + // in the header list. If none is found, a NULL pointer is returned. + // + // Any pointer returned references objects in the container itself + // and will have the same lifetime as this class. If you want + // the value beyond the lifetime of this instance, make a copy. + // + // @arg name C-style string giving the name of a header + // to search. The comparison is case-sensitive + // though list entries may have been normalized + // to lower-case. + // + // @return NULL if the header wasn't found otherwise + // a pointer to a std::string in the container. + // Pointer is valid only for the lifetime of + // the container or until container is modifed. + // + const std::string * find(const char * name) const; + + // Count of headers currently in the list. + size_type size() const + { + return mHeaders.size(); + } + + // Standard std::vector-based forward iterators. + iterator begin(); + const_iterator begin() const; + iterator end(); + const_iterator end() const; + + // Standard std::vector-based reverse iterators. + reverse_iterator rbegin(); + const_reverse_iterator rbegin() const; + reverse_iterator rend(); + const_reverse_iterator rend() const; + +public: + // For unit tests only - not a public API + container_t & getContainerTESTONLY(); + +protected: container_t mHeaders; }; // end class HttpHeaders diff --git a/indra/llcorehttp/httpoptions.cpp b/indra/llcorehttp/httpoptions.cpp index 1699d19f8d..5bf1ecb4a5 100755 --- a/indra/llcorehttp/httpoptions.cpp +++ b/indra/llcorehttp/httpoptions.cpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -38,7 +38,9 @@ HttpOptions::HttpOptions() mWantHeaders(false), mTracing(HTTP_TRACE_OFF), mTimeout(HTTP_REQUEST_TIMEOUT_DEFAULT), - mRetries(HTTP_RETRY_COUNT_DEFAULT) + mTransferTimeout(HTTP_REQUEST_XFER_TIMEOUT_DEFAULT), + mRetries(HTTP_RETRY_COUNT_DEFAULT), + mUseRetryAfter(HTTP_USE_RETRY_AFTER_DEFAULT) {} @@ -64,10 +66,21 @@ void HttpOptions::setTimeout(unsigned int timeout) } +void HttpOptions::setTransferTimeout(unsigned int timeout) +{ + mTransferTimeout = timeout; +} + + void HttpOptions::setRetries(unsigned int retries) { mRetries = retries; } +void HttpOptions::setUseRetryAfter(bool use_retry) +{ + mUseRetryAfter = use_retry; +} + } // end namespace LLCore diff --git a/indra/llcorehttp/httpoptions.h b/indra/llcorehttp/httpoptions.h index 97e46a8cd3..4ab5ff18c4 100755 --- a/indra/llcorehttp/httpoptions.h +++ b/indra/llcorehttp/httpoptions.h @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -68,36 +68,55 @@ protected: void operator=(const HttpOptions &); // Not defined public: + // Default: false void setWantHeaders(bool wanted); bool getWantHeaders() const { return mWantHeaders; } - + + // Default: 0 void setTrace(int long); int getTrace() const { return mTracing; } + // Default: 30 void setTimeout(unsigned int timeout); unsigned int getTimeout() const { return mTimeout; } + // Default: 0 + void setTransferTimeout(unsigned int timeout); + unsigned int getTransferTimeout() const + { + return mTransferTimeout; + } + + // Default: 8 void setRetries(unsigned int retries); unsigned int getRetries() const { return mRetries; } + + // Default: true + void setUseRetryAfter(bool use_retry); + bool getUseRetryAfter() const + { + return mUseRetryAfter; + } protected: bool mWantHeaders; int mTracing; unsigned int mTimeout; + unsigned int mTransferTimeout; unsigned int mRetries; - + bool mUseRetryAfter; }; // end class HttpOptions diff --git a/indra/llcorehttp/httprequest.cpp b/indra/llcorehttp/httprequest.cpp index 9b739a8825..7b1888e3eb 100755 --- a/indra/llcorehttp/httprequest.cpp +++ b/indra/llcorehttp/httprequest.cpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -54,12 +54,8 @@ namespace LLCore // ==================================== -HttpRequest::policy_t HttpRequest::sNextPolicyID(1); - - HttpRequest::HttpRequest() - : //HttpHandler(), - mReplyQueue(NULL), + : mReplyQueue(NULL), mRequestQueue(NULL) { mRequestQueue = HttpRequestQueue::instanceOf(); @@ -90,45 +86,91 @@ HttpRequest::~HttpRequest() // ==================================== -HttpStatus HttpRequest::setPolicyGlobalOption(EGlobalPolicy opt, long value) +HttpRequest::policy_t HttpRequest::createPolicyClass() { if (HttpService::RUNNING == HttpService::instanceOf()->getState()) { - return HttpStatus(HttpStatus::LLCORE, HE_OPT_NOT_DYNAMIC); + return 0; } - return HttpService::instanceOf()->getGlobalOptions().set(opt, value); + return HttpService::instanceOf()->createPolicyClass(); } -HttpStatus HttpRequest::setPolicyGlobalOption(EGlobalPolicy opt, const std::string & value) +HttpStatus HttpRequest::setStaticPolicyOption(EPolicyOption opt, policy_t pclass, + long value, long * ret_value) { if (HttpService::RUNNING == HttpService::instanceOf()->getState()) { return HttpStatus(HttpStatus::LLCORE, HE_OPT_NOT_DYNAMIC); } - return HttpService::instanceOf()->getGlobalOptions().set(opt, value); + return HttpService::instanceOf()->setPolicyOption(opt, pclass, value, ret_value); } -HttpRequest::policy_t HttpRequest::createPolicyClass() +HttpStatus HttpRequest::setStaticPolicyOption(EPolicyOption opt, policy_t pclass, + const std::string & value, std::string * ret_value) { if (HttpService::RUNNING == HttpService::instanceOf()->getState()) { - return 0; + return HttpStatus(HttpStatus::LLCORE, HE_OPT_NOT_DYNAMIC); } - return HttpService::instanceOf()->createPolicyClass(); + return HttpService::instanceOf()->setPolicyOption(opt, pclass, value, ret_value); } -HttpStatus HttpRequest::setPolicyClassOption(policy_t policy_id, - EClassPolicy opt, - long value) +HttpHandle HttpRequest::setPolicyOption(EPolicyOption opt, policy_t pclass, + long value, HttpHandler * handler) { - if (HttpService::RUNNING == HttpService::instanceOf()->getState()) + HttpStatus status; + HttpHandle handle(LLCORE_HTTP_HANDLE_INVALID); + + HttpOpSetGet * op = new HttpOpSetGet(); + if (! (status = op->setupSet(opt, pclass, value))) { - return HttpStatus(HttpStatus::LLCORE, HE_OPT_NOT_DYNAMIC); + op->release(); + mLastReqStatus = status; + return handle; + } + op->setReplyPath(mReplyQueue, handler); + if (! (status = mRequestQueue->addOp(op))) // transfers refcount + { + op->release(); + mLastReqStatus = status; + return handle; + } + + mLastReqStatus = status; + handle = static_cast<HttpHandle>(op); + + return handle; +} + + +HttpHandle HttpRequest::setPolicyOption(EPolicyOption opt, policy_t pclass, + const std::string & value, HttpHandler * handler) +{ + HttpStatus status; + HttpHandle handle(LLCORE_HTTP_HANDLE_INVALID); + + HttpOpSetGet * op = new HttpOpSetGet(); + if (! (status = op->setupSet(opt, pclass, value))) + { + op->release(); + mLastReqStatus = status; + return handle; + } + op->setReplyPath(mReplyQueue, handler); + if (! (status = mRequestQueue->addOp(op))) // transfers refcount + { + op->release(); + mLastReqStatus = status; + return handle; } - return HttpService::instanceOf()->getClassOptions(policy_id).set(opt, value); + + mLastReqStatus = status; + handle = static_cast<HttpHandle>(op); + + return handle; } @@ -474,31 +516,6 @@ HttpHandle HttpRequest::requestSpin(int mode) return handle; } -// ==================================== -// Dynamic Policy Methods -// ==================================== - -HttpHandle HttpRequest::requestSetHttpProxy(const std::string & proxy, HttpHandler * handler) -{ - HttpStatus status; - HttpHandle handle(LLCORE_HTTP_HANDLE_INVALID); - - HttpOpSetGet * op = new HttpOpSetGet(); - op->setupSet(GP_HTTP_PROXY, proxy); - op->setReplyPath(mReplyQueue, handler); - if (! (status = mRequestQueue->addOp(op))) // transfers refcount - { - op->release(); - mLastReqStatus = status; - return handle; - } - - mLastReqStatus = status; - handle = static_cast<HttpHandle>(op); - - return handle; -} - } // end namespace LLCore diff --git a/indra/llcorehttp/httprequest.h b/indra/llcorehttp/httprequest.h index ab2f302d34..651654844a 100755 --- a/indra/llcorehttp/httprequest.h +++ b/indra/llcorehttp/httprequest.h @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -56,6 +56,9 @@ class BufferArray; /// The class supports the current HTTP request operations: /// /// - requestGetByteRange: GET with Range header for a single range of bytes +/// - requestGet: +/// - requestPost: +/// - requestPut: /// /// Policy Classes /// @@ -100,9 +103,26 @@ public: /// Represents a default, catch-all policy class that guarantees /// eventual service for any HTTP request. - static const int DEFAULT_POLICY_ID = 0; + static const policy_t DEFAULT_POLICY_ID = 0; + static const policy_t INVALID_POLICY_ID = 0xFFFFFFFFU; + static const policy_t GLOBAL_POLICY_ID = 0xFFFFFFFEU; - enum EGlobalPolicy + /// Create a new policy class into which requests can be made. + /// + /// All class creation must occur before threads are started and + /// transport begins. Policy classes are limited to a small value. + /// Currently that limit is the default class + 1. + /// + /// @return If positive, the policy_id used to reference + /// the class in other methods. If 0, requests + /// for classes have exceeded internal limits + /// or caller has tried to create a class after + /// threads have been started. Caller must fallback + /// and recover. + /// + static policy_t createPolicyClass(); + + enum EPolicyOption { /// Maximum number of connections the library will use to /// perform operations. This is somewhat soft as the underlying @@ -113,24 +133,40 @@ public: /// a somewhat soft value. There may be an additional five /// connections per policy class depending upon runtime /// behavior. - GP_CONNECTION_LIMIT, + /// + /// Both global and per-class + PO_CONNECTION_LIMIT, + + /// Limits the number of connections used for a single + /// literal address/port pair within the class. + /// + /// Per-class only + PO_PER_HOST_CONNECTION_LIMIT, /// String containing a system-appropriate directory name /// where SSL certs are stored. - GP_CA_PATH, + /// + /// Global only + PO_CA_PATH, /// String giving a full path to a file containing SSL certs. - GP_CA_FILE, + /// + /// Global only + PO_CA_FILE, /// String of host/port to use as simple HTTP proxy. This is /// going to change in the future into something more elaborate /// that may support richer schemes. - GP_HTTP_PROXY, + /// + /// Global only + PO_HTTP_PROXY, /// Long value that if non-zero enables the use of the /// traditional LLProxy code for http/socks5 support. If - /// enabled, has priority over GP_HTTP_PROXY. - GP_LLPROXY, + // enabled, has priority over GP_HTTP_PROXY. + /// + /// Global only + PO_LLPROXY, /// Long value setting the logging trace level for the /// library. Possible values are: @@ -143,50 +179,59 @@ public: /// These values are also used in the trace modes for /// individual requests in HttpOptions. Also be aware that /// tracing tends to impact performance of the viewer. - GP_TRACE - }; - - /// Set a parameter on a global policy option. Calls - /// made after the start of the servicing thread are - /// not honored and return an error status. - /// - /// @param opt Enum of option to be set. - /// @param value Desired value of option. - /// @return Standard status code. - static HttpStatus setPolicyGlobalOption(EGlobalPolicy opt, long value); - static HttpStatus setPolicyGlobalOption(EGlobalPolicy opt, const std::string & value); - - /// Create a new policy class into which requests can be made. - /// - /// @return If positive, the policy_id used to reference - /// the class in other methods. If 0, an error - /// occurred and @see getStatus() may provide more - /// detail on the reason. - static policy_t createPolicyClass(); - - enum EClassPolicy - { - /// Limits the number of connections used for the class. - CP_CONNECTION_LIMIT, - - /// Limits the number of connections used for a single - /// literal address/port pair within the class. - CP_PER_HOST_CONNECTION_LIMIT, + /// + /// Global only + PO_TRACE, /// Suitable requests are allowed to pipeline on their /// connections when they ask for it. - CP_ENABLE_PIPELINING + /// + /// Per-class only + PO_ENABLE_PIPELINING, + + /// Controls whether client-side throttling should be + /// performed on this policy class. Positive values + /// enable throttling and specify the request rate + /// (requests per second) that should be targetted. + /// A value of zero, the default, specifies no throttling. + /// + /// Per-class only + PO_THROTTLE_RATE, + + PO_LAST // Always at end }; - + + /// Set a policy option for a global or class parameter at + /// startup time (prior to thread start). + /// + /// @param opt Enum of option to be set. + /// @param pclass For class-based options, the policy class ID to + /// be changed. For globals, specify GLOBAL_POLICY_ID. + /// @param value Desired value of option. + /// @param ret_value Pointer to receive effective set value + /// if successful. May be NULL if effective + /// value not wanted. + /// @return Standard status code. + static HttpStatus setStaticPolicyOption(EPolicyOption opt, policy_t pclass, + long value, long * ret_value); + static HttpStatus setStaticPolicyOption(EPolicyOption opt, policy_t pclass, + const std::string & value, std::string * ret_value); + /// Set a parameter on a class-based policy option. Calls /// made after the start of the servicing thread are /// not honored and return an error status. /// - /// @param policy_id ID of class as returned by @see createPolicyClass(). - /// @param opt Enum of option to be set. - /// @param value Desired value of option. - /// @return Standard status code. - static HttpStatus setPolicyClassOption(policy_t policy_id, EClassPolicy opt, long value); + /// @param opt Enum of option to be set. + /// @param pclass For class-based options, the policy class ID to + /// be changed. Ignored for globals but recommend + /// using INVALID_POLICY_ID in this case. + /// @param value Desired value of option. + /// @return Handle of dynamic request. Use @see getStatus() if + /// the returned handle is invalid. + HttpHandle setPolicyOption(EPolicyOption opt, policy_t pclass, long value, + HttpHandler * handler); + HttpHandle setPolicyOption(EPolicyOption opt, policy_t pclass, const std::string & value, + HttpHandler * handler); /// @} @@ -488,16 +533,6 @@ public: /// @} - /// @name DynamicPolicyMethods - /// - /// @{ - - /// Request that a running transport pick up a new proxy setting. - /// An empty string will indicate no proxy is to be used. - HttpHandle requestSetHttpProxy(const std::string & proxy, HttpHandler * handler); - - /// @} - protected: void generateNotification(HttpOperation * op); @@ -519,7 +554,6 @@ private: /// Must be established before any threading is allowed to /// start. /// - static policy_t sNextPolicyID; /// @} // End Global State diff --git a/indra/llcorehttp/httpresponse.cpp b/indra/llcorehttp/httpresponse.cpp index a552e48a1b..c974395b0a 100755 --- a/indra/llcorehttp/httpresponse.cpp +++ b/indra/llcorehttp/httpresponse.cpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -39,7 +39,9 @@ HttpResponse::HttpResponse() mReplyLength(0U), mReplyFullLength(0U), mBufferArray(NULL), - mHeaders(NULL) + mHeaders(NULL), + mRetries(0U), + m503Retries(0U) {} diff --git a/indra/llcorehttp/httpresponse.h b/indra/llcorehttp/httpresponse.h index f19b521fbf..aee64e2878 100755 --- a/indra/llcorehttp/httpresponse.h +++ b/indra/llcorehttp/httpresponse.h @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -149,6 +149,25 @@ public: mContentType = con_type; } + /// Get and set retry attempt information on the request. + void getRetries(unsigned int * retries, unsigned int * retries_503) const + { + if (retries) + { + *retries = mRetries; + } + if (retries_503) + { + *retries_503 = m503Retries; + } + } + + void setRetries(unsigned int retries, unsigned int retries_503) + { + mRetries = retries; + m503Retries = retries_503; + } + protected: // Response data here HttpStatus mStatus; @@ -158,6 +177,8 @@ protected: BufferArray * mBufferArray; HttpHeaders * mHeaders; std::string mContentType; + unsigned int mRetries; + unsigned int m503Retries; }; diff --git a/indra/llcorehttp/tests/test_httpheaders.hpp b/indra/llcorehttp/tests/test_httpheaders.hpp index ce0d19b058..668c36dc66 100755 --- a/indra/llcorehttp/tests/test_httpheaders.hpp +++ b/indra/llcorehttp/tests/test_httpheaders.hpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -36,7 +36,6 @@ using namespace LLCoreInt; - namespace tut { @@ -63,7 +62,7 @@ void HttpHeadersTestObjectType::test<1>() HttpHeaders * headers = new HttpHeaders(); ensure("One ref on construction of HttpHeaders", headers->getRefCount() == 1); ensure("Memory being used", mMemTotal < GetMemTotal()); - ensure("Nothing in headers", 0 == headers->mHeaders.size()); + ensure("Nothing in headers", 0 == headers->size()); // release the implicit reference, causing the object to be released headers->release(); @@ -85,14 +84,340 @@ void HttpHeadersTestObjectType::test<2>() { // Append a few strings - std::string str1("Pragma:"); - headers->mHeaders.push_back(str1); - std::string str2("Accept: application/json"); - headers->mHeaders.push_back(str2); + std::string str1n("Pragma"); + std::string str1v(""); + headers->append(str1n, str1v); + std::string str2n("Accept"); + std::string str2v("application/json"); + headers->append(str2n, str2v); + + ensure("Headers retained", 2 == headers->size()); + HttpHeaders::container_t & c(headers->getContainerTESTONLY()); + + ensure("First name is first name", c[0].first == str1n); + ensure("First value is first value", c[0].second == str1v); + ensure("Second name is second name", c[1].first == str2n); + ensure("Second value is second value", c[1].second == str2v); + } + + // release the implicit reference, causing the object to be released + headers->release(); + + // make sure we didn't leak any memory + ensure(mMemTotal == GetMemTotal()); +} + +template <> template <> +void HttpHeadersTestObjectType::test<3>() +{ + set_test_name("HttpHeaders basic find"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + + // create a new ref counted object with an implicit reference + HttpHeaders * headers = new HttpHeaders(); + + { + // Append a few strings + std::string str1n("Uno"); + std::string str1v("1"); + headers->append(str1n, str1v); + std::string str2n("doS"); + std::string str2v("2-2-2-2"); + headers->append(str2n, str2v); + std::string str3n("TRES"); + std::string str3v("trois gymnopedie"); + headers->append(str3n, str3v); + + ensure("Headers retained", 3 == headers->size()); + + const std::string * result(NULL); + + // Find a header + result = headers->find("TRES"); + ensure("Found the last item", result != NULL); + ensure("Last item is a nice", result != NULL && str3v == *result); + + // appends above are raw and find is case sensitive + result = headers->find("TReS"); + ensure("Last item not found due to case", result == NULL); + + result = headers->find("TRE"); + ensure("Last item not found due to prefixing (1)", result == NULL); + + result = headers->find("TRESS"); + ensure("Last item not found due to prefixing (2)", result == NULL); + } + + // release the implicit reference, causing the object to be released + headers->release(); + + // make sure we didn't leak any memory + ensure(mMemTotal == GetMemTotal()); +} + +template <> template <> +void HttpHeadersTestObjectType::test<4>() +{ + set_test_name("HttpHeaders normalized header entry"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + + // create a new ref counted object with an implicit reference + HttpHeaders * headers = new HttpHeaders(); + + { + static char line1[] = " AcCePT : image/yourfacehere"; + static char line1v[] = "image/yourfacehere"; + headers->appendNormal(line1, sizeof(line1) - 1); + + ensure("First append worked in some fashion", 1 == headers->size()); + + const std::string * result(NULL); + + // Find a header + result = headers->find("accept"); + ensure("Found 'accept'", result != NULL); + ensure("accept value has face", result != NULL && *result == line1v); + + // Left-clean on value + static char line2[] = " next : \t\tlinejunk \t"; + headers->appendNormal(line2, sizeof(line2) - 1); + ensure("Second append worked", 2 == headers->size()); + result = headers->find("next"); + ensure("Found 'next'", result != NULL); + ensure("next value is left-clean", result != NULL && + *result == "linejunk \t"); + + // First value unmolested + result = headers->find("accept"); + ensure("Found 'accept' again", result != NULL); + ensure("accept value has face", result != NULL && *result == line1v); + + // Colons in value are okay + static char line3[] = "FancY-PANTs::plop:-neuf-=vleem="; + static char line3v[] = ":plop:-neuf-=vleem="; + headers->appendNormal(line3, sizeof(line3) - 1); + ensure("Third append worked", 3 == headers->size()); + result = headers->find("fancy-pants"); + ensure("Found 'fancy-pants'", result != NULL); + ensure("fancy-pants value has colons", result != NULL && *result == line3v); + + // Zero-length value + static char line4[] = "all-talk-no-walk:"; + headers->appendNormal(line4, sizeof(line4) - 1); + ensure("Fourth append worked", 4 == headers->size()); + result = headers->find("all-talk-no-walk"); + ensure("Found 'all-talk'", result != NULL); + ensure("al-talk value is zero-length", result != NULL && result->size() == 0); + + // Zero-length name + static char line5[] = ":all-talk-no-walk"; + static char line5v[] = "all-talk-no-walk"; + headers->appendNormal(line5, sizeof(line5) - 1); + ensure("Fifth append worked", 5 == headers->size()); + result = headers->find(""); + ensure("Found no-name", result != NULL); + ensure("no-name value is something", result != NULL && *result == line5v); + + // Lone colon is still something + headers->clear(); + static char line6[] = " :"; + headers->appendNormal(line6, sizeof(line6) - 1); + ensure("Sixth append worked", 1 == headers->size()); + result = headers->find(""); + ensure("Found 2nd no-name", result != NULL); + ensure("2nd no-name value is nothing", result != NULL && result->size() == 0); + + // Line without colons is taken as-is and unstripped in name + static char line7[] = " \toskdgioasdghaosdghoowg28342908tg8902hg0hwedfhqew890v7qh0wdebv78q0wdevbhq>?M>BNM<ZV>?NZ? \t"; + headers->appendNormal(line7, sizeof(line7) - 1); + ensure("Seventh append worked", 2 == headers->size()); + result = headers->find(line7); + ensure("Found whatsit line", result != NULL); + ensure("Whatsit line has no value", result != NULL && result->size() == 0); + + // Normaling interface heeds the byte count, doesn't look for NUL-terminator + static char line8[] = "binary:ignorestuffontheendofthis"; + headers->appendNormal(line8, 13); + ensure("Eighth append worked", 3 == headers->size()); + result = headers->find("binary"); + ensure("Found 'binary'", result != NULL); + ensure("binary value was limited to 'ignore'", result != NULL && + *result == "ignore"); + + } - ensure("Headers retained", 2 == headers->mHeaders.size()); - ensure("First is first", headers->mHeaders[0] == str1); - ensure("Second is second", headers->mHeaders[1] == str2); + // release the implicit reference, causing the object to be released + headers->release(); + + // make sure we didn't leak any memory + ensure(mMemTotal == GetMemTotal()); +} + +// Verify forward iterator finds everything as expected +template <> template <> +void HttpHeadersTestObjectType::test<5>() +{ + set_test_name("HttpHeaders iterator tests"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + + // create a new ref counted object with an implicit reference + HttpHeaders * headers = new HttpHeaders(); + + HttpHeaders::iterator end(headers->end()), begin(headers->begin()); + ensure("Empty container has equal begin/end const iterators", end == begin); + HttpHeaders::const_iterator cend(headers->end()), cbegin(headers->begin()); + ensure("Empty container has equal rbegin/rend const iterators", cend == cbegin); + + ensure("Empty container has equal begin/end iterators", headers->end() == headers->begin()); + + { + static char line1[] = " AcCePT : image/yourfacehere"; + static char line1v[] = "image/yourfacehere"; + headers->appendNormal(line1, sizeof(line1) - 1); + + static char line2[] = " next : \t\tlinejunk \t"; + static char line2v[] = "linejunk \t"; + headers->appendNormal(line2, sizeof(line2) - 1); + + static char line3[] = "FancY-PANTs::plop:-neuf-=vleem="; + static char line3v[] = ":plop:-neuf-=vleem="; + headers->appendNormal(line3, sizeof(line3) - 1); + + static char line4[] = "all-talk-no-walk:"; + static char line4v[] = ""; + headers->appendNormal(line4, sizeof(line4) - 1); + + static char line5[] = ":all-talk-no-walk"; + static char line5v[] = "all-talk-no-walk"; + headers->appendNormal(line5, sizeof(line5) - 1); + + static char line6[] = " :"; + static char line6v[] = ""; + headers->appendNormal(line6, sizeof(line6) - 1); + + ensure("All entries accounted for", 6 == headers->size()); + + static char * values[] = { + line1v, + line2v, + line3v, + line4v, + line5v, + line6v + }; + + int i(0); + HttpHeaders::const_iterator cend(headers->end()); + for (HttpHeaders::const_iterator it(headers->begin()); + cend != it; + ++it, ++i) + { + std::ostringstream str; + str << "Const Iterator value # " << i << " was " << values[i]; + ensure(str.str(), (*it).second == values[i]); + } + + // Rewind, do non-consts + i = 0; + HttpHeaders::iterator end(headers->end()); + for (HttpHeaders::iterator it(headers->begin()); + end != it; + ++it, ++i) + { + std::ostringstream str; + str << "Const Iterator value # " << i << " was " << values[i]; + ensure(str.str(), (*it).second == values[i]); + } + } + + // release the implicit reference, causing the object to be released + headers->release(); + + // make sure we didn't leak any memory + ensure(mMemTotal == GetMemTotal()); +} + +// Reverse iterators find everything as expected +template <> template <> +void HttpHeadersTestObjectType::test<6>() +{ + set_test_name("HttpHeaders reverse iterator tests"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + + // create a new ref counted object with an implicit reference + HttpHeaders * headers = new HttpHeaders(); + + HttpHeaders::reverse_iterator rend(headers->rend()), rbegin(headers->rbegin()); + ensure("Empty container has equal rbegin/rend const iterators", rend == rbegin); + HttpHeaders::const_reverse_iterator crend(headers->rend()), crbegin(headers->rbegin()); + ensure("Empty container has equal rbegin/rend const iterators", crend == crbegin); + + { + static char line1[] = " AcCePT : image/yourfacehere"; + static char line1v[] = "image/yourfacehere"; + headers->appendNormal(line1, sizeof(line1) - 1); + + static char line2[] = " next : \t\tlinejunk \t"; + static char line2v[] = "linejunk \t"; + headers->appendNormal(line2, sizeof(line2) - 1); + + static char line3[] = "FancY-PANTs::plop:-neuf-=vleem="; + static char line3v[] = ":plop:-neuf-=vleem="; + headers->appendNormal(line3, sizeof(line3) - 1); + + static char line4[] = "all-talk-no-walk:"; + static char line4v[] = ""; + headers->appendNormal(line4, sizeof(line4) - 1); + + static char line5[] = ":all-talk-no-walk"; + static char line5v[] = "all-talk-no-walk"; + headers->appendNormal(line5, sizeof(line5) - 1); + + static char line6[] = " :"; + static char line6v[] = ""; + headers->appendNormal(line6, sizeof(line6) - 1); + + ensure("All entries accounted for", 6 == headers->size()); + + static char * values[] = { + line6v, + line5v, + line4v, + line3v, + line2v, + line1v + }; + + int i(0); + HttpHeaders::const_reverse_iterator cend(headers->rend()); + for (HttpHeaders::const_reverse_iterator it(headers->rbegin()); + cend != it; + ++it, ++i) + { + std::ostringstream str; + str << "Const Iterator value # " << i << " was " << values[i]; + ensure(str.str(), (*it).second == values[i]); + } + + // Rewind, do non-consts + i = 0; + HttpHeaders::reverse_iterator end(headers->rend()); + for (HttpHeaders::reverse_iterator it(headers->rbegin()); + end != it; + ++it, ++i) + { + std::ostringstream str; + str << "Iterator value # " << i << " was " << values[i]; + ensure(str.str(), (*it).second == values[i]); + } } // release the implicit reference, causing the object to be released diff --git a/indra/llcorehttp/tests/test_httprequest.hpp b/indra/llcorehttp/tests/test_httprequest.hpp index 900a699887..43f7e36da5 100755 --- a/indra/llcorehttp/tests/test_httprequest.hpp +++ b/indra/llcorehttp/tests/test_httprequest.hpp @@ -69,6 +69,8 @@ void usleep(unsigned long usec); namespace tut { +typedef std::vector<std::pair<boost::regex, boost::regex> > regex_container_t; + struct HttpRequestTestData { // the test objects inherit from this so the member functions and variables @@ -118,11 +120,17 @@ public: for (int i(0); i < mHeadersRequired.size(); ++i) { bool found = false; - for (HttpHeaders::container_t::const_iterator iter(header->mHeaders.begin()); - header->mHeaders.end() != iter; + for (HttpHeaders::const_iterator iter(header->begin()); + header->end() != iter; ++iter) { - if (boost::regex_match(*iter, mHeadersRequired[i])) + // std::cerr << "Header: " << (*iter).first + // << ": " << (*iter).second << std::endl; + + if (boost::regex_match((*iter).first, + mHeadersRequired[i].first) && + boost::regex_match((*iter).second, + mHeadersRequired[i].second)) { found = true; break; @@ -138,11 +146,14 @@ public: { for (int i(0); i < mHeadersDisallowed.size(); ++i) { - for (HttpHeaders::container_t::const_iterator iter(header->mHeaders.begin()); - header->mHeaders.end() != iter; + for (HttpHeaders::const_iterator iter(header->begin()); + header->end() != iter; ++iter) { - if (boost::regex_match(*iter, mHeadersDisallowed[i])) + if (boost::regex_match((*iter).first, + mHeadersDisallowed[i].first) && + boost::regex_match((*iter).second, + mHeadersDisallowed[i].second)) { std::ostringstream str; str << "Disallowed header # " << i << " not found in response"; @@ -168,8 +179,8 @@ public: std::string mName; HttpHandle mExpectHandle; std::string mCheckContentType; - std::vector<boost::regex> mHeadersRequired; - std::vector<boost::regex> mHeadersDisallowed; + regex_container_t mHeadersRequired; + regex_container_t mHeadersDisallowed; }; typedef test_group<HttpRequestTestData> HttpRequestTestGroupType; @@ -1211,7 +1222,7 @@ void HttpRequestTestObjectType::test<12>() HttpRequest::createService(); // Enable tracing - HttpRequest::setPolicyGlobalOption(LLCore::HttpRequest::GP_TRACE, 2); + HttpRequest::setStaticPolicyOption(HttpRequest::PO_TRACE, HttpRequest::DEFAULT_POLICY_ID, 2, NULL); // Start threading early so that thread memory is invariant // over the test. @@ -1329,7 +1340,7 @@ void HttpRequestTestObjectType::test<13>() HttpRequest::createService(); // Enable tracing - HttpRequest::setPolicyGlobalOption(LLCore::HttpRequest::GP_TRACE, 2); + HttpRequest::setStaticPolicyOption(HttpRequest::PO_TRACE, HttpRequest::DEFAULT_POLICY_ID, 2, NULL); // Start threading early so that thread memory is invariant // over the test. @@ -1344,7 +1355,9 @@ void HttpRequestTestObjectType::test<13>() // Issue a GET that succeeds mStatus = HttpStatus(200); - handler.mHeadersRequired.push_back(boost::regex("\\W*X-LL-Special:.*", boost::regex::icase)); + handler.mHeadersRequired.push_back( + regex_container_t::value_type(boost::regex("X-LL-Special", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); HttpHandle handle = req->requestGetByteRange(HttpRequest::DEFAULT_POLICY_ID, 0U, url_base, @@ -1711,18 +1724,54 @@ void HttpRequestTestObjectType::test<16>() // Issue a GET that *can* connect mStatus = HttpStatus(200); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-connection:\\s*keep-alive", boost::regex::icase)); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-accept:\\s*\\*/\\*", boost::regex::icase)); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-accept-encoding:\\s*((gzip|deflate),\\s*)+(gzip|deflate)", boost::regex::icase)); // close enough - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-keep-alive:\\s*\\d+", boost::regex::icase)); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-host:\\s*.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-cache-control:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-pragma:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-range:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-transfer-encoding:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-referer:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-content-type:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-content-encoding:.*", boost::regex::icase)); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-connection", boost::regex::icase), + boost::regex("keep-alive", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-accept", boost::regex::icase), + boost::regex("\\*/\\*", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-accept-encoding", boost::regex::icase), + boost::regex("((gzip|deflate),\\s*)+(gzip|deflate)", boost::regex::icase))); // close enough + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-keep-alive", boost::regex::icase), + boost::regex("\\d+", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-host", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-cache-control", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-pragma", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-range", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-transfer-encoding", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-referer", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-content-type", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-content-encoding", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); HttpHandle handle = req->requestGet(HttpRequest::DEFAULT_POLICY_ID, 0U, url_base + "reflect/", @@ -1744,23 +1793,60 @@ void HttpRequestTestObjectType::test<16>() // Do a texture-style fetch headers = new HttpHeaders; - headers->mHeaders.push_back("Accept: image/x-j2c"); + headers->append("Accept", "image/x-j2c"); mStatus = HttpStatus(200); handler.mHeadersRequired.clear(); handler.mHeadersDisallowed.clear(); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-connection:\\s*keep-alive", boost::regex::icase)); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-accept:\\s*image/x-j2c", boost::regex::icase)); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-accept-encoding:\\s*((gzip|deflate),\\s*)+(gzip|deflate)", boost::regex::icase)); // close enough - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-keep-alive:\\s*\\d+", boost::regex::icase)); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-host:\\s*.*", boost::regex::icase)); - handler.mHeadersRequired.push_back(boost::regex("\\W*X-Reflect-range:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-cache-control:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-pragma:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-transfer-encoding:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-referer:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-content-type:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-content-encoding:.*", boost::regex::icase)); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-connection", boost::regex::icase), + boost::regex("keep-alive", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-accept", boost::regex::icase), + boost::regex("image/x-j2c", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-accept-encoding", boost::regex::icase), + boost::regex("((gzip|deflate),\\s*)+(gzip|deflate)", boost::regex::icase))); // close enough + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-keep-alive", boost::regex::icase), + boost::regex("\\d+", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-host", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("\\W*X-Reflect-range", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-cache-control", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-pragma", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-transfer-encoding", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-referer", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-content-type", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-content-encoding", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); handle = req->requestGetByteRange(HttpRequest::DEFAULT_POLICY_ID, 0U, url_base + "reflect/", @@ -1901,20 +1987,63 @@ void HttpRequestTestObjectType::test<17>() // Issue a default POST mStatus = HttpStatus(200); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-connection:\\s*keep-alive", boost::regex::icase)); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-accept:\\s*\\*/\\*", boost::regex::icase)); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-accept-encoding:\\s*((gzip|deflate),\\s*)+(gzip|deflate)", boost::regex::icase)); // close enough - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-keep-alive:\\s*\\d+", boost::regex::icase)); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-host:\\s*.*", boost::regex::icase)); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-content-length:\\s*\\d+", boost::regex::icase)); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-content-type:\\s*application/x-www-form-urlencoded", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-cache-control:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-pragma:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-range:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-referer:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-content-encoding:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-expect:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-transfer-encoding:\\s*.*chunked.*", boost::regex::icase)); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-connection", boost::regex::icase), + boost::regex("keep-alive", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-accept", boost::regex::icase), + boost::regex("\\*/\\*", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-accept-encoding", boost::regex::icase), + boost::regex("((gzip|deflate),\\s*)+(gzip|deflate)", boost::regex::icase))); // close enough + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-keep-alive", boost::regex::icase), + boost::regex("\\d+", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-host", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-content-length", boost::regex::icase), + boost::regex("\\d+", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-content-type", boost::regex::icase), + boost::regex("application/x-www-form-urlencoded", boost::regex::icase))); + + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-cache-control", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-pragma", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-range", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-referer", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-content-encoding", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-expect", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-transfer_encoding", boost::regex::icase), + boost::regex(".*chunked.*", boost::regex::icase))); HttpHandle handle = req->requestPost(HttpRequest::DEFAULT_POLICY_ID, 0U, url_base + "reflect/", @@ -2061,20 +2190,64 @@ void HttpRequestTestObjectType::test<18>() // Issue a default PUT mStatus = HttpStatus(200); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-connection:\\s*keep-alive", boost::regex::icase)); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-accept:\\s*\\*/\\*", boost::regex::icase)); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-accept-encoding:\\s*((gzip|deflate),\\s*)+(gzip|deflate)", boost::regex::icase)); // close enough - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-keep-alive:\\s*\\d+", boost::regex::icase)); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-host:\\s*.*", boost::regex::icase)); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-content-length:\\s*\\d+", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-cache-control:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-pragma:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-range:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-referer:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-content-encoding:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-expect:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-transfer-encoding:\\s*.*chunked.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("X-Reflect-content-type:.*", boost::regex::icase)); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-connection", boost::regex::icase), + boost::regex("keep-alive", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-accept", boost::regex::icase), + boost::regex("\\*/\\*", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-accept-encoding", boost::regex::icase), + boost::regex("((gzip|deflate),\\s*)+(gzip|deflate)", boost::regex::icase))); // close enough + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-keep-alive", boost::regex::icase), + boost::regex("\\d+", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-host", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-content-length", boost::regex::icase), + boost::regex("\\d+", boost::regex::icase))); + + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-cache-control", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-pragma", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-range", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-referer", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-content-encoding", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-expect", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-transfer-encoding", boost::regex::icase), + boost::regex(".*chunked.*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-content-type", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + HttpHandle handle = req->requestPut(HttpRequest::DEFAULT_POLICY_ID, 0U, url_base + "reflect/", @@ -2215,27 +2388,73 @@ void HttpRequestTestObjectType::test<19>() // headers headers = new HttpHeaders; - headers->mHeaders.push_back("Keep-Alive: 120"); - headers->mHeaders.push_back("Accept-encoding: deflate"); - headers->mHeaders.push_back("Accept: text/plain"); + headers->append("Keep-Alive", "120"); + headers->append("Accept-encoding", "deflate"); + headers->append("Accept", "text/plain"); // Issue a GET with modified headers mStatus = HttpStatus(200); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-connection:\\s*keep-alive", boost::regex::icase)); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-accept:\\s*text/plain", boost::regex::icase)); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-accept-encoding:\\s*deflate", boost::regex::icase)); // close enough - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-keep-alive:\\s*120", boost::regex::icase)); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-host:\\s*.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("X-Reflect-accept-encoding:\\s*((gzip|deflate),\\s*)+(gzip|deflate)", boost::regex::icase)); // close enough - handler.mHeadersDisallowed.push_back(boost::regex("X-Reflect-keep-alive:\\s*300", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("X-Reflect-accept:\\s*\\*/\\*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-cache-control:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-pragma:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-range:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-transfer-encoding:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-referer:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-content-type:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-content-encoding:.*", boost::regex::icase)); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-connection", boost::regex::icase), + boost::regex("keep-alive", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-accept", boost::regex::icase), + boost::regex("text/plain", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-accept-encoding", boost::regex::icase), + boost::regex("deflate", boost::regex::icase))); // close enough + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-keep-alive", boost::regex::icase), + boost::regex("120", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-host", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-accept-encoding", boost::regex::icase), + boost::regex("((gzip|deflate),\\s*)+(gzip|deflate)", boost::regex::icase))); // close enough + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-keep-alive", boost::regex::icase), + boost::regex("300", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-accept", boost::regex::icase), + boost::regex("\\*/\\*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-cache-control", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-pragma", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-range", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-transfer-encoding", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-referer", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-content-type", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-content-encoding", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); HttpHandle handle = req->requestGet(HttpRequest::DEFAULT_POLICY_ID, 0U, url_base + "reflect/", @@ -2368,10 +2587,10 @@ void HttpRequestTestObjectType::test<20>() // headers headers = new HttpHeaders(); - headers->mHeaders.push_back("keep-Alive: 120"); - headers->mHeaders.push_back("Accept: text/html"); - headers->mHeaders.push_back("content-type: application/llsd+xml"); - headers->mHeaders.push_back("cache-control: no-store"); + headers->append("keep-Alive", "120"); + headers->append("Accept", "text/html"); + headers->append("content-type", "application/llsd+xml"); + headers->append("cache-control", "no-store"); // And a buffer array const char * msg("<xml><llsd><string>It was the best of times, it was the worst of times.</string></llsd></xml>"); @@ -2380,23 +2599,76 @@ void HttpRequestTestObjectType::test<20>() // Issue a default POST mStatus = HttpStatus(200); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-connection:\\s*keep-alive", boost::regex::icase)); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-accept:\\s*text/html", boost::regex::icase)); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-accept-encoding:\\s*((gzip|deflate),\\s*)+(gzip|deflate)", boost::regex::icase)); // close enough - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-keep-alive:\\s*120", boost::regex::icase)); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-host:\\s*.*", boost::regex::icase)); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-content-length:\\s*\\d+", boost::regex::icase)); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-content-type:\\s*application/llsd\\+xml", boost::regex::icase)); - handler.mHeadersRequired.push_back(boost::regex("\\s*X-Reflect-cache-control:\\s*no-store", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("X-Reflect-content-type:\\s*application/x-www-form-urlencoded", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("X-Reflect-accept:\\s*\\*/\\*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("X-Reflect-keep-alive:\\s*300", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-pragma:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-range:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-referer:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-content-encoding:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-expect:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-transfer-encoding:\\s*.*chunked.*", boost::regex::icase)); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-connection", boost::regex::icase), + boost::regex("keep-alive", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-accept", boost::regex::icase), + boost::regex("text/html", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-accept-encoding", boost::regex::icase), + boost::regex("((gzip|deflate),\\s*)+(gzip|deflate)", boost::regex::icase))); // close enough + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-keep-alive", boost::regex::icase), + boost::regex("120", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-host", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-content-length", boost::regex::icase), + boost::regex("\\d+", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-content-type", boost::regex::icase), + boost::regex("application/llsd\\+xml", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-cache-control", boost::regex::icase), + boost::regex("no-store", boost::regex::icase))); + + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-content-type", boost::regex::icase), + boost::regex("application/x-www-form-urlencoded", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-accept", boost::regex::icase), + boost::regex("\\*/\\*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-keep-alive", boost::regex::icase), + boost::regex("300", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-pragma", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-range", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-referer", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-content-encoding", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-expect", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-transfer-encoding", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + HttpHandle handle = req->requestPost(HttpRequest::DEFAULT_POLICY_ID, 0U, url_base + "reflect/", @@ -2538,9 +2810,9 @@ void HttpRequestTestObjectType::test<21>() // headers headers = new HttpHeaders; - headers->mHeaders.push_back("content-type: text/plain"); - headers->mHeaders.push_back("content-type: text/html"); - headers->mHeaders.push_back("content-type: application/llsd+xml"); + headers->append("content-type", "text/plain"); + headers->append("content-type", "text/html"); + headers->append("content-type", "application/llsd+xml"); // And a buffer array const char * msg("<xml><llsd><string>It was the best of times, it was the worst of times.</string></llsd></xml>"); @@ -2549,22 +2821,71 @@ void HttpRequestTestObjectType::test<21>() // Issue a default PUT mStatus = HttpStatus(200); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-connection:\\s*keep-alive", boost::regex::icase)); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-accept:\\s*\\*/\\*", boost::regex::icase)); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-accept-encoding:\\s*((gzip|deflate),\\s*)+(gzip|deflate)", boost::regex::icase)); // close enough - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-keep-alive:\\s*\\d+", boost::regex::icase)); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-host:\\s*.*", boost::regex::icase)); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-content-length:\\s*\\d+", boost::regex::icase)); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-content-type:\\s*application/llsd\\+xml", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-cache-control:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-pragma:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-range:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-referer:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-content-encoding:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-expect:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-transfer-encoding:\\s*.*chunked.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("X-Reflect-content-type:\\s*text/plain", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("X-Reflect-content-type:\\s*text/html", boost::regex::icase)); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-connection", boost::regex::icase), + boost::regex("keep-alive", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-accept", boost::regex::icase), + boost::regex("\\*/\\*", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-accept-encoding", boost::regex::icase), + boost::regex("((gzip|deflate),\\s*)+(gzip|deflate)", boost::regex::icase))); // close enough + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-keep-alive", boost::regex::icase), + boost::regex("\\d+", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-host", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-content-length", boost::regex::icase), + boost::regex("\\d+", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-content-type", boost::regex::icase), + boost::regex("application/llsd\\+xml", boost::regex::icase))); + + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-cache-control", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-pragma", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-range", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-referer", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-content-encoding", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-expect", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-transfer-encoding", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-content-type", boost::regex::icase), + boost::regex("text/plain", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-content-type", boost::regex::icase), + boost::regex("text/html", boost::regex::icase))); HttpHandle handle = req->requestPut(HttpRequest::DEFAULT_POLICY_ID, 0U, url_base + "reflect/", @@ -2854,6 +3175,142 @@ void HttpRequestTestObjectType::test<22>() } } +template <> template <> +void HttpRequestTestObjectType::test<23>() +{ + ScopedCurlInit ready; + + set_test_name("HttpRequest GET 503s with 'Retry-After'"); + + // This tests mainly that the code doesn't fall over if + // various well- and mis-formed Retry-After headers are + // sent along with the response. Direct inspection of + // the parsing result isn't supported. + + // Handler can be stack-allocated *if* there are no dangling + // references to it after completion of this method. + // Create before memory record as the string copy will bump numbers. + TestHandler2 handler(this, "handler"); + std::string url_base(get_base_url() + "/503/"); // path to 503 generators + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + mHandlerCalls = 0; + + HttpRequest * req = NULL; + HttpOptions * opts = NULL; + + try + { + // Get singletons created + HttpRequest::createService(); + + // Start threading early so that thread memory is invariant + // over the test. + HttpRequest::startThread(); + + // create a new ref counted object with an implicit reference + req = new HttpRequest(); + ensure("Memory allocated on construction", mMemTotal < GetMemTotal()); + + opts = new HttpOptions(); + opts->setRetries(1); // Retry once only + opts->setUseRetryAfter(true); // Try to parse the retry-after header + + // Issue a GET that 503s with valid retry-after + mStatus = HttpStatus(503); + int url_limit(6); + for (int i(0); i < url_limit; ++i) + { + std::ostringstream url; + url << url_base << i << "/"; + HttpHandle handle = req->requestGetByteRange(HttpRequest::DEFAULT_POLICY_ID, + 0U, + url.str(), + 0, + 0, + opts, + NULL, + &handler); + + std::ostringstream testtag; + testtag << "Valid handle returned for 503 request #" << i; + ensure(testtag.str(), handle != LLCORE_HTTP_HANDLE_INVALID); + } + + + // Run the notification pump. + int count(0); + int limit(LOOP_COUNT_LONG); + while (count++ < limit && mHandlerCalls < url_limit) + { + req->update(0); + usleep(LOOP_SLEEP_INTERVAL); + } + ensure("Request executed in reasonable time", count < limit); + ensure("One handler invocation for request", mHandlerCalls == url_limit); + + // Okay, request a shutdown of the servicing thread + mStatus = HttpStatus(); + mHandlerCalls = 0; + HttpHandle handle = req->requestStopThread(&handler); + ensure("Valid handle returned for second request", handle != LLCORE_HTTP_HANDLE_INVALID); + + // Run the notification pump again + count = 0; + limit = LOOP_COUNT_LONG; + while (count++ < limit && mHandlerCalls < 1) + { + req->update(1000000); + usleep(LOOP_SLEEP_INTERVAL); + } + ensure("Second request executed in reasonable time", count < limit); + ensure("Second handler invocation", mHandlerCalls == 1); + + // See that we actually shutdown the thread + count = 0; + limit = LOOP_COUNT_SHORT; + while (count++ < limit && ! HttpService::isStopped()) + { + usleep(LOOP_SLEEP_INTERVAL); + } + ensure("Thread actually stopped running", HttpService::isStopped()); + + // release options + opts->release(); + opts = NULL; + + // release the request object + delete req; + req = NULL; + + // Shut down service + HttpRequest::destroyService(); + +#if defined(WIN32) + // Can only do this memory test on Windows. On other platforms, + // the LL logging system holds on to memory and produces what looks + // like memory leaks... + + // printf("Old mem: %d, New mem: %d\n", mMemTotal, GetMemTotal()); + ensure("Memory usage back to that at entry", mMemTotal == GetMemTotal()); +#endif + } + catch (...) + { + stop_thread(req); + if (opts) + { + opts->release(); + opts = NULL; + } + delete req; + HttpRequest::destroyService(); + throw; + } +} + + } // end namespace tut namespace diff --git a/indra/llcorehttp/tests/test_httpstatus.hpp b/indra/llcorehttp/tests/test_httpstatus.hpp index b5538528c5..0b379836c9 100755 --- a/indra/llcorehttp/tests/test_httpstatus.hpp +++ b/indra/llcorehttp/tests/test_httpstatus.hpp @@ -259,6 +259,65 @@ void HttpStatusTestObjectType::test<7>() ensure(msg == "Unknown error"); } + +template <> template <> +void HttpStatusTestObjectType::test<8>() +{ + set_test_name("HttpStatus toHex() nominal function"); + + HttpStatus status(404); + std::string msg = status.toHex(); + // std::cout << "Result: " << msg << std::endl; + ensure(msg == "01940001"); +} + + +template <> template <> +void HttpStatusTestObjectType::test<9>() +{ + set_test_name("HttpStatus toTerseString() nominal function"); + + HttpStatus status(404); + std::string msg = status.toTerseString(); + // std::cout << "Result: " << msg << std::endl; + ensure("Normal HTTP 404", msg == "Http_404"); + + status = HttpStatus(200); + msg = status.toTerseString(); + // std::cout << "Result: " << msg << std::endl; + ensure("Normal HTTP 200", msg == "Http_200"); + + status = HttpStatus(200, HE_REPLY_ERROR); + msg = status.toTerseString(); + // std::cout << "Result: " << msg << std::endl; + ensure("Unsuccessful HTTP 200", msg == "Http_200"); // No distinction for error + + status = HttpStatus(HttpStatus::EXT_CURL_EASY, CURLE_COULDNT_CONNECT); + msg = status.toTerseString(); + // std::cout << "Result: " << msg << std::endl; + ensure("Easy couldn't connect error", msg == "Easy_7"); + + status = HttpStatus(HttpStatus::EXT_CURL_MULTI, CURLM_OUT_OF_MEMORY); + msg = status.toTerseString(); + // std::cout << "Result: " << msg << std::endl; + ensure("Multi out-of-memory error", msg == "Multi_3"); + + status = HttpStatus(HttpStatus::LLCORE, HE_OPT_NOT_SET); + msg = status.toTerseString(); + // std::cout << "Result: " << msg << std::endl; + ensure("Core option not set error", msg == "Core_7"); + + status = HttpStatus(22000, 1); + msg = status.toTerseString(); + // std::cout << "Result: " << msg << std::endl; + ensure("Undecodable error", msg == "Unknown_1"); + + status = HttpStatus(22000, -1); + msg = status.toTerseString(); + // std::cout << "Result: " << msg << std::endl; + ensure("Undecodable error 65535", msg == "Unknown_65535"); +} + } // end namespace tut #endif // TEST_HTTP_STATUS_H diff --git a/indra/llcorehttp/tests/test_llcorehttp_peer.py b/indra/llcorehttp/tests/test_llcorehttp_peer.py index 3c3af8dc75..04cde651c4 100755 --- a/indra/llcorehttp/tests/test_llcorehttp_peer.py +++ b/indra/llcorehttp/tests/test_llcorehttp_peer.py @@ -69,6 +69,15 @@ class TestHTTPRequestHandler(BaseHTTPRequestHandler): "Content-Range: bytes 0-75/2983", "Content-Length: 76" -- '/bug2295/inv_cont_range/0/' Generates HE_INVALID_CONTENT_RANGE error in llcorehttp. + - '/503/' Generate 503 responses with various kinds + of 'retry-after' headers + -- '/503/0/' "Retry-After: 2" + -- '/503/1/' "Retry-After: Thu, 31 Dec 2043 23:59:59 GMT" + -- '/503/2/' "Retry-After: Fri, 31 Dec 1999 23:59:59 GMT" + -- '/503/3/' "Retry-After: " + -- '/503/4/' "Retry-After: (*#*(@*(@(")" + -- '/503/5/' "Retry-After: aklsjflajfaklsfaklfasfklasdfklasdgahsdhgasdiogaioshdgo" + -- '/503/6/' "Retry-After: 1 2 3 4 5 6 7 8 9 10" Some combinations make no sense, there's no effort to protect you from that. @@ -143,22 +152,40 @@ class TestHTTPRequestHandler(BaseHTTPRequestHandler): if "/sleep/" in self.path: time.sleep(30) - if "fail" in self.path: - status = data.get("status", 500) - # self.responses maps an int status to a (short, long) pair of - # strings. We want the longer string. That's why we pass a string - # pair to get(): the [1] will select the second string, whether it - # came from self.responses or from our default pair. - reason = data.get("reason", - self.responses.get(status, - ("fail requested", - "Your request specified failure status %s " - "without providing a reason" % status))[1]) - debug("fail requested: %s: %r", status, reason) - self.send_error(status, reason) + if "/503/" in self.path: + # Tests for various kinds of 'Retry-After' header parsing + body = None + if "/503/0/" in self.path: + self.send_response(503) + self.send_header("retry-after", "2") + elif "/503/1/" in self.path: + self.send_response(503) + self.send_header("retry-after", "Thu, 31 Dec 2043 23:59:59 GMT") + elif "/503/2/" in self.path: + self.send_response(503) + self.send_header("retry-after", "Fri, 31 Dec 1999 23:59:59 GMT") + elif "/503/3/" in self.path: + self.send_response(503) + self.send_header("retry-after", "") + elif "/503/4/" in self.path: + self.send_response(503) + self.send_header("retry-after", "(*#*(@*(@(") + elif "/503/5/" in self.path: + self.send_response(503) + self.send_header("retry-after", "aklsjflajfaklsfaklfasfklasdfklasdgahsdhgasdiogaioshdgo") + elif "/503/6/" in self.path: + self.send_response(503) + self.send_header("retry-after", "1 2 3 4 5 6 7 8 9 10") + else: + # Unknown request + self.send_response(400) + body = "Unknown /503/ path in server" if "/reflect/" in self.path: self.reflect_headers() + self.send_header("Content-type", "text/plain") self.end_headers() + if body: + self.wfile.write(body) elif "/bug2295/" in self.path: # Test for https://jira.secondlife.com/browse/BUG-2295 # @@ -194,8 +221,7 @@ class TestHTTPRequestHandler(BaseHTTPRequestHandler): self.end_headers() if body: self.wfile.write(body) - else: - # Normal response path + elif "fail" not in self.path: data = data.copy() # we're going to modify # Ensure there's a "reply" key in data, even if there wasn't before data["reply"] = data.get("reply", llsd.LLSD("success")) @@ -210,6 +236,22 @@ class TestHTTPRequestHandler(BaseHTTPRequestHandler): self.end_headers() if withdata: self.wfile.write(response) + else: # fail requested + status = data.get("status", 500) + # self.responses maps an int status to a (short, long) pair of + # strings. We want the longer string. That's why we pass a string + # pair to get(): the [1] will select the second string, whether it + # came from self.responses or from our default pair. + reason = data.get("reason", + self.responses.get(status, + ("fail requested", + "Your request specified failure status %s " + "without providing a reason" % status))[1]) + debug("fail requested: %s: %r", status, reason) + self.send_error(status, reason) + if "/reflect/" in self.path: + self.reflect_headers() + self.end_headers() def reflect_headers(self): for name in self.headers.keys(): |