summaryrefslogtreecommitdiff
path: root/indra/llcorehttp
diff options
context:
space:
mode:
authorsimon <none@none>2014-02-25 16:59:46 -0800
committersimon <none@none>2014-02-25 16:59:46 -0800
commit43e36b7ee328f49149292f27cc5621238d03ac26 (patch)
tree2d46556a050699dee527e0ebbf82260e27bae793 /indra/llcorehttp
parent5705a2c0181a9376db5eadf6ccf97e0675f6afb5 (diff)
parentdd8b32473e652c2d3193c78db21b06ba25e28b67 (diff)
Merge downstream code
Diffstat (limited to 'indra/llcorehttp')
-rw-r--r--indra/llcorehttp/README.Linden671
-rwxr-xr-xindra/llcorehttp/_httpinternal.h23
-rwxr-xr-xindra/llcorehttp/_httplibcurl.cpp63
-rwxr-xr-xindra/llcorehttp/_httplibcurl.h16
-rwxr-xr-xindra/llcorehttp/_httpoperation.cpp10
-rwxr-xr-xindra/llcorehttp/_httpoperation.h8
-rwxr-xr-xindra/llcorehttp/_httpoprequest.cpp430
-rwxr-xr-xindra/llcorehttp/_httpoprequest.h7
-rwxr-xr-xindra/llcorehttp/_httpopsetget.cpp93
-rwxr-xr-xindra/llcorehttp/_httpopsetget.h27
-rwxr-xr-xindra/llcorehttp/_httppolicy.cpp223
-rwxr-xr-xindra/llcorehttp/_httppolicy.h39
-rwxr-xr-xindra/llcorehttp/_httppolicyclass.cpp54
-rwxr-xr-xindra/llcorehttp/_httppolicyclass.h20
-rwxr-xr-xindra/llcorehttp/_httppolicyglobal.cpp66
-rwxr-xr-xindra/llcorehttp/_httppolicyglobal.h23
-rwxr-xr-xindra/llcorehttp/_httpservice.cpp174
-rwxr-xr-xindra/llcorehttp/_httpservice.h52
-rwxr-xr-xindra/llcorehttp/examples/http_texture_load.cpp117
-rwxr-xr-xindra/llcorehttp/httpcommon.cpp42
-rwxr-xr-xindra/llcorehttp/httpcommon.h120
-rwxr-xr-xindra/llcorehttp/httpheaders.cpp141
-rwxr-xr-xindra/llcorehttp/httpheaders.h112
-rwxr-xr-xindra/llcorehttp/httpoptions.cpp17
-rwxr-xr-xindra/llcorehttp/httpoptions.h25
-rwxr-xr-xindra/llcorehttp/httprequest.cpp107
-rwxr-xr-xindra/llcorehttp/httprequest.h146
-rwxr-xr-xindra/llcorehttp/httpresponse.cpp6
-rwxr-xr-xindra/llcorehttp/httpresponse.h23
-rwxr-xr-xindra/llcorehttp/tests/test_httpheaders.hpp345
-rwxr-xr-xindra/llcorehttp/tests/test_httprequest.hpp701
-rwxr-xr-xindra/llcorehttp/tests/test_httpstatus.hpp59
-rwxr-xr-xindra/llcorehttp/tests/test_llcorehttp_peer.py72
33 files changed, 3331 insertions, 701 deletions
diff --git a/indra/llcorehttp/README.Linden b/indra/llcorehttp/README.Linden
new file mode 100644
index 0000000000..eb6ccab3bc
--- /dev/null
+++ b/indra/llcorehttp/README.Linden
@@ -0,0 +1,671 @@
+
+
+
+1. HTTP Fetching in 15 Minutes
+
+ Let's start with a trivial working example. You'll need a throwaway
+ build of the viewer. And we'll use indra/newview/llappviewer.cpp as
+ the host module for these hacks.
+
+ First, add some headers:
+
+
+ #include "httpcommon.h"
+ #include "httprequest.h"
+ #include "httphandler.h"
+
+
+ You'll need to derive a class from HttpHandler (not HttpHandle).
+ This is used to deliver notifications of HTTP completion to your
+ code. Place it near the top, before LLDeferredTaskList, say:
+
+
+ class MyHandler : public LLCore::HttpHandler
+ {
+ public:
+ MyHandler()
+ : LLCore::HttpHandler()
+ {}
+
+ virtual void onCompleted(LLCore::HttpHandle /* handle */,
+ LLCore::HttpResponse * /* response */)
+ {
+ LL_INFOS("Hack") << "It is happening again." << LL_ENDL;
+
+ delete this; // Last statement
+ }
+ };
+
+
+ Add some statics up there as well:
+
+
+ // Our request object. Allocate during initialiation.
+ static LLCore::HttpRequest * my_request(NULL);
+
+ // The policy class for HTTP traffic.
+ // Use HttpRequest::DEFAULT_POLICY_ID, but DO NOT SHIP WITH THIS VALUE!!
+ static LLCore::HttpRequest::policy_t my_policy(LLCore::HttpRequest::DEFAULT_POLICY_ID);
+
+ // Priority for HTTP requests. Use 0U.
+ static LLCore::HttpRequest::priority_t my_priority(0U);
+
+
+ In LLAppViewer::init() after mAppCoreHttp.init(), create a request object:
+
+
+ my_request = new LLCore::HttpRequest();
+
+
+ In LLAppViewer::mainLoop(), just before entering the while loop,
+ we'll kick off one HTTP request:
+
+
+ // Construct a handler object (we'll use the heap this time):
+ MyHandler * my_handler = new MyHandler;
+
+ // Issue a GET request to 'http://www.example.com/' kicking off
+ // all the I/O, retry logic, etc.
+ LLCore::HttpHandle handle;
+ handle = my_request->requestGet(my_policy,
+ my_priority,
+ "http://www.example.com/",
+ NULL,
+ NULL,
+ my_handler);
+ if (LLCORE_HTTP_HANDLE_INVALID == handle)
+ {
+ LL_WARNS("Hack") << "Failed to launch HTTP request. Try again."
+ << LL_ENDL;
+ }
+
+
+ Finally, arrange to periodically call update() on the request object
+ to find out when the request completes. This will be done by
+ calling the onCompleted() method with status information and
+ response data from the HTTP operation. Add this to the
+ LLAppViewer::idle() method after the ping:
+
+
+ my_request->update(0);
+
+
+ That's it. Build it, run it and watch the log file. You should get
+ the "It is happening again." message indicating that the HTTP
+ operation completed in some manner.
+
+
+2. What Does All That Mean
+
+ MyHandler/HttpHandler. This class replaces the Responder-style in
+ legacy code. One method is currently defined. It is used for all
+ request completions, successful or failed:
+
+
+ void onCompleted(LLCore::HttpHandle /* handle */,
+ LLCore::HttpResponse * /* response */);
+
+
+ The onCompleted() method is invoked as a callback during calls to
+ HttpRequest::update(). All I/O is completed asynchronously in
+ another thread. But notifications are polled by calling update()
+ and invoking a handler for completed requests.
+
+ In this example, the invocation also deletes the handler (which is
+ never referenced by the llcorehttp code again). But other
+ allocation models are possible including handlers shared by many
+ requests, stack-based handlers and handlers mixed in with other,
+ unrelated classes.
+
+ LLCore::HttpRequest(). Instances of this class are used to request
+ all major functions of the library. Initialization, starting
+ requests, delivering final notification of completion and various
+ utility operations are all done via instances. There is one very
+ important rule for instances:
+
+ Request objects may NOT be shared between threads.
+
+ my_priority. The APIs support the idea of priority ordering of
+ requests but it hasn't been implemented and the hope is that this
+ will become useless and removed from the interface. Use 0U except
+ as noted.
+
+ my_policy. This is an important one. This library attempts to
+ manage TCP connection usage more rigorously than in the past. This
+ is done by issuing requests to a queue that has various settable
+ properties. These establish connection usage for the queue as well
+ as how queues compete with one another. (This is patterned after
+ class-based queueing used in various networking stacks.) Several
+ classes are pre-defined. Deciding when to use an existing class and
+ when to create a new one will determine what kind of experience
+ users have. We'll pick up this question in detail below.
+
+ requestGet(). Issues an ordinary HTTP GET request to a given URL
+ and associating the request with a policy class, a priority and an
+ response handler. Two additional arguments, not used here, allow
+ for additional headers on the request and for per-request options.
+ If successful, the call returns a handle whose value is other than
+ LLCORE_HTTP_HANDLE_INVALID. The HTTP operation is then performed
+ asynchronously by another thread without any additional work by the
+ caller. If the handle returned is invalid, you can get the status
+ code by calling my_request->getStatus().
+
+ update(). To get notification that the request has completed, a
+ call to update() will invoke onCompleted() methods.
+
+
+3. Refinements, Necessary and Otherwise
+
+ MyHandler::onCompleted(). You'll want to do something useful with
+ your response. Distinguish errors from successes and getting the
+ response body back in some form.
+
+ Add a new header:
+
+
+ #include "bufferarray.h"
+
+
+ Replace the existing MyHandler::onCompleted() definition with:
+
+
+ virtual void onCompleted(LLCore::HttpHandle /* handle */,
+ LLCore::HttpResponse * response)
+ {
+ LLCore::HttpStatus status = response->getStatus();
+ if (status)
+ {
+ // Successful request. Try to fetch the data
+ LLCore::BufferArray * data = response->getBody();
+
+ if (data && data->size())
+ {
+ // There's some data. A BufferArray is a linked list
+ // of buckets. We'll create a linear buffer and copy
+ // the data into it.
+ size_t data_len = data->size();
+ char * data_blob = new char [data_len + 1];
+ data->read(0, data_blob, data_len);
+ data_blob[data_len] = '\0';
+
+ // Process the data now in NUL-terminated string.
+ // Needs more scrubbing but this will do.
+ LL_INFOS("Hack") << "Received: " << data_blob << LL_ENDL;
+
+ // Free the temporary data
+ delete [] data_blob;
+ }
+ }
+ else
+ {
+ // Something went wrong. Translate the status to
+ // a meaningful message.
+ LL_WARNS("Hack") << "HTTP GET failed. Status: "
+ << status.toTerseString()
+ << ", Reason: " << status.toString()
+ << LL_ENDL;
+ }
+
+ delete this; // Last statement
+ }
+
+
+ HttpHeaders. The header file "httprequest.h" documents the expected
+ important headers that will go out with the request. You can add to
+ these by including an HttpHeaders object with the requestGet() call.
+ These are typically setup once as part of init rather than
+ dynamically created.
+
+ Add another header:
+
+
+ #include "httpheaders.h"
+
+
+ In LLAppViewer::mainLoop(), add this alongside the allocation of
+ my_handler:
+
+
+ // Additional headers for all requests
+ LLCore::HttpHeaders * my_headers = new LLCore::HttpHeaders();
+ my_headers->append("Accept", "text/html, application/llsd+xml");
+
+
+ HttpOptions. Options are similar and include a mix of value types.
+ One interesting per-request option is the trace setting. This
+ enables various debug-type messages in the log file that show the
+ progress of the request through the library. It takes values from
+ zero to three with higher values giving more verbose logging. We'll
+ use '2' and this will also give us a chance to verify that
+ HttpHeaders works as expected.
+
+ Same as above, a new header:
+
+
+ #include "httpoptions.h"
+
+
+ And in LLAppView::mainLoop():
+
+
+ // Special options for requests
+ LLCore::HttpOptions * my_options = new LLCore::HttpOptions();
+ my_options->setTrace(2);
+
+
+ Now let's put that all together into a more complete requesting
+ sequence. Replace the existing invocation of requestGet() with this
+ slightly more elaborate block:
+
+
+ LLCore::HttpHandle handle;
+ handle = my_request->requestGet(my_policy,
+ my_priority,
+ "http://www.example.com/",
+ my_options,
+ my_headers,
+ my_handler);
+ if (LLCORE_HTTP_HANDLE_INVALID == handle)
+ {
+ LLCore::HttpStatus status = my_request->getStatus();
+
+ LL_WARNS("Hack") << "Failed to request HTTP GET. Status: "
+ << status.toTerseString()
+ << ", Reason: " << status.toString()
+ << LL_ENDL;
+
+ delete my_handler; // No longer needed.
+ my_handler = NULL;
+ }
+
+
+ Build, run and examine the log file. You'll get some new data with
+ this run. First, you should get the www.example.com home page
+ content:
+
+
+----------------------------------------------------------------------------
+2013-09-17T20:26:51Z INFO: MyHandler::onCompleted: Received: <!doctype html>
+<html>
+<head>
+ <title>Example Domain</title>
+
+ <meta charset="utf-8" />
+ <meta http-equiv="Content-type" content="text/html; charset=utf-8" />
+ <meta name="viewport" content="width=device-width, initial-scale=1" />
+ <style type="text/css">
+ body {
+ background-color: #f0f0f2;
+ margin: 0;
+ padding: 0;
+ font-family: "Open Sans", "Helvetica Neue", Helvetica, Arial, sans-serif;
+
+ }
+ div {
+ width: 600px;
+ margin: 5em auto;
+ padding: 50px;
+ background-color: #fff;
+ border-radius: 1em;
+ }
+ a:link, a:visited {
+ color: #38488f;
+ text-decoration: none;
+ }
+ @media (max-width: 700px) {
+ body {
+ background-color: #fff;
+ }
+ div {
+ width: auto;
+ margin: 0 auto;
+ border-radius: 0;
+ padding: 1em;
+ }
+ }
+ </style>
+</head>
+
+<body>
+<div>
+ <h1>Example Domain</h1>
+ <p>This domain is established to be used for illustrative examples in documents. You may use this
+ domain in examples without prior coordination or asking for permission.</p>
+ <p><a href="http://www.iana.org/domains/example">More information...</a></p>
+</div>
+</body>
+</html>
+----------------------------------------------------------------------------
+
+
+ You'll also get a detailed trace of the HTTP operation itself. Note
+ the HEADEROUT line which shows the additional header added to the
+ request.
+
+
+----------------------------------------------------------------------------
+HttpService::processRequestQueue: TRACE, FromRequestQueue, Handle: 086D3148
+HttpLibcurl::addOp: TRACE, ToActiveQueue, Handle: 086D3148, Actives: 0, Readies: 0
+HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle: 086D3148, Type: TEXT, Data: About to connect() to www.example.com port 80 (#0)
+HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle: 086D3148, Type: TEXT, Data: Trying 93.184.216.119...
+HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle: 086D3148, Type: TEXT, Data: Connected to www.example.com (93.184.216.119) port 80 (#0)
+HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle: 086D3148, Type: TEXT, Data: Connected to www.example.com (93.184.216.119) port 80 (#0)
+HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle: 086D3148, Type: HEADEROUT, Data: GET / HTTP/1.1 Host: www.example.com Accept-Encoding: deflate, gzip Connection: keep-alive Keep-alive: 300 Accept: text/html, application/llsd+xml
+HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle: 086D3148, Type: HEADERIN, Data: HTTP/1.1 200 OK
+HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle: 086D3148, Type: HEADERIN, Data: Accept-Ranges: bytes
+HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle: 086D3148, Type: HEADERIN, Data: Cache-Control: max-age=604800
+HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle: 086D3148, Type: HEADERIN, Data: Content-Type: text/html
+HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle: 086D3148, Type: HEADERIN, Data: Date: Tue, 17 Sep 2013 20:26:56 GMT
+HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle: 086D3148, Type: HEADERIN, Data: Etag: "3012602696"
+HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle: 086D3148, Type: HEADERIN, Data: Expires: Tue, 24 Sep 2013 20:26:56 GMT
+HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle: 086D3148, Type: HEADERIN, Data: Last-Modified: Fri, 09 Aug 2013 23:54:35 GMT
+HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle: 086D3148, Type: HEADERIN, Data: Server: ECS (ewr/1590)
+HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle: 086D3148, Type: HEADERIN, Data: X-Cache: HIT
+HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle: 086D3148, Type: HEADERIN, Data: x-ec-custom-error: 1
+HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle: 086D3148, Type: HEADERIN, Data: Content-Length: 1270
+HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle: 086D3148, Type: HEADERIN, Data:
+HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle: 086D3148, Type: DATAIN, Data: 256 Bytes
+HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle: 086D3148, Type: TEXT, Data: Connection #0 to host www.example.com left intact
+HttpLibcurl::completeRequest: TRACE, RequestComplete, Handle: 086D3148, Status: Http_200
+HttpOperation::addAsReply: TRACE, ToReplyQueue, Handle: 086D3148
+----------------------------------------------------------------------------
+
+
+4. What Does All That Mean, Part 2
+
+ HttpStatus. The HttpStatus object encodes errors from libcurl, the
+ library itself and HTTP status values. It does this to avoid
+ collapsing all non-HTTP error into a single '499' HTTP status and to
+ make errors distinct.
+
+ To aid programming, the usual bool conversions are available so that
+ you can write 'if (status)' and the expected thing will happen
+ whether it's an HTTP, libcurl or library error. There's also
+ provision to override the treatment of HTTP errors (making 404 a
+ success, say).
+
+ Share data, don't copy it. The library was started with the goal of
+ avoiding data copies as much as possible. Instead, read-only data
+ sharing across threads with atomic reference counts is used for a
+ number of data types. These currently are:
+
+ * BufferArray. Linked list of data blocks/HTTP bodies.
+ * HttpHeaders. Shared headers for both requests and responses.
+ * HttpOptions. Request-only data modifying HTTP behavior.
+ * HttpResponse. HTTP response description given to onCompleted.
+
+ Using objects of these types requires a few rules:
+
+ * Constructor always gives a reference to caller.
+ * References are dropped with release() not delete.
+ * Additional references may be taken out with addRef().
+ * Unless otherwise stated, once an object is shared with another
+ thread it should be treated as read-only. There's no
+ synchronization on the objects themselves.
+
+ HttpResponse. You'll encounter this mainly in onCompleted() methods.
+ Commonly-used interfaces on this object:
+
+ * getStatus() to return the final status of the request.
+ * getBody() to retrieve the response body which may be NULL or
+ zero-length.
+ * getContentType() to return the value of the 'Content-Type'
+ header or an empty string if none was sent.
+
+ This is a reference-counted object so you can call addRef() on it
+ and hold onto the response for an arbitrary time. But you'll
+ usually just call a few methods and return from onCompleted() whose
+ caller will release the object.
+
+ BufferArray. The core data representation for request and response
+ bodies. In HTTP responses, it's fetched with the getBody() method
+ and may be NULL or non-NULL with zero length. All successful data
+ handling should check both conditions before attempting to fetch
+ data from the object. Data access model uses simple read/write
+ semantics:
+
+ * append()
+ * size()
+ * read()
+ * write()
+
+ (There is a more sophisticated stream adapter that extends these
+ methods and will be covered below.) So, one way to retrieve data
+ from a request is as follows:
+
+
+ LLCore::BufferArray * data = response->getBody();
+ if (data && data->size())
+ {
+ size_t data_len = data->size();
+ char * data_blob = new char [data_len + 1];
+ data->read(0, data_blob, data_len);
+
+
+ HttpOptions and HttpResponse. Really just simple containers of POD
+ and std::string pairs. But reference counted and the rule about not
+ modifying after sharing must be followed. You'll have the urge to
+ change options dynamically at some point. And you'll try to do that
+ by just writing new values to the shared object. And in tests
+ everything will appear to work. Then you ship and people in the
+ real world start hitting read/write races in strings and then crash.
+ Don't be lazy.
+
+ HttpHandle. Uniquely identifies a request and can be used to
+ identify it in an onCompleted() method or cancel it if it's still
+ queued. But as soon as a request's onCompleted() invocation
+ returns, the handle becomes invalid and may be reused immediately
+ for new requests. Don't hold on to handles after notification.
+
+
+5. And Still More Refinements
+
+ (Note: The following refinements are just code fragments. They
+ don't directly fit into the working example above. But they
+ demonstrate several idioms you'll want to copy.)
+
+ LLSD, std::streambuf, std::iostream. The read(), write() and
+ append() methods may be adequate for your purposes. But we use a
+ lot of LLSD. Its interfaces aren't particularly compatible with
+ BufferArray. And so two adapters are available to give
+ stream-like behaviors: BufferArrayStreamBuf and BufferArrayStream,
+ which implement the std::streambuf and std::iostream interfaces,
+ respectively.
+
+ A std::streambuf interface isn't something you'll want to use
+ directly. Instead, you'll use the much friendlier std::iostream
+ interface found in BufferArrayStream. This adapter gives you all
+ the '>>' and '<<' operators you'll want as well as working
+ directly with the LLSD conversion operators.
+
+ Some new headers:
+
+
+ #include "bufferstream.h"
+ #include "llsdserialize.h"
+
+
+ And an updated fragment based on onCompleted() above:
+
+
+ // Successful request. Try to fetch the data
+ LLCore::BufferArray * data = response->getBody();
+ LLSD resp_llsd;
+
+ if (data && data->size())
+ {
+ // There's some data and we expect this to be
+ // LLSD. Checking of content type and validation
+ // during parsing would be admirable additions.
+ // But we'll forgo that now.
+ LLCore::BufferArrayStream data_stream(data);
+ LLSDSerialize::fromXML(resp_llsd, data_stream);
+ }
+ LL_INFOS("Hack") << "LLSD Received: " << resp_llsd << LL_ENDL;
+ }
+ else
+ {
+
+
+ Converting an LLSD object into an XML stream stored in a
+ BufferArray is just the reverse of the above:
+
+
+ BufferArray * data = new BufferArray();
+ LLCore::BufferArrayStream data_stream(data);
+
+ LLSD src_llsd;
+ src_llsd["foo"] = "bar";
+
+ LLSDSerialize::toXML(src_llsd, data_stream);
+
+ // 'data' now contains an XML payload and can be sent
+ // to a web service using the requestPut() or requestPost()
+ // methods.
+ ... requestPost(...);
+
+ // And don't forget to release the BufferArray.
+ data->release();
+ data = NULL;
+
+
+ LLSD will often go hand-in-hand with BufferArray and data
+ transport. But you can also do all the streaming I/O you'd expect
+ of a std::iostream object:
+
+
+ BufferArray * data = new BufferArray();
+ LLCore::BufferArrayStream data_stream(data);
+
+ data_stream << "Hello, World!" << 29.4 << '\n';
+ std::string str;
+ data_stream >> str;
+ std::cout << str << std::endl;
+
+ data->release();
+ // Actual delete will occur when 'data_stream'
+ // falls out of scope and is destructed.
+
+
+ Scoping objects and cleaning up. The examples haven't bothered
+ with cleanup of objects that are no longer needed. Instead, most
+ objects have been allocated as if they were global and eternal.
+ You'll put the objects in more appropriate feature objects and
+ clean them up as a group. Here's a checklist for actions you may
+ need to take on cleanup:
+
+ * Call delete on:
+ o HttpHandlers created on the heap
+ o HttpRequest objects
+ * Call release() on:
+ o BufferArray objects
+ o HttpHeaders objects
+ o HttpOptions objects
+ o HttpResponse objects
+
+ On program exit, as threads wind down, the library continues to
+ operate safely. Threads don't interact via the library and even
+ dangling references to HttpHandler objects are safe. If you don't
+ call HttpRequest::update(), handler references are never
+ dereferenced.
+
+ You can take a more thorough approach to wind-down. Keep a list
+ of HttpHandles (not HttpHandlers) of outstanding requests. For
+ each of these, call HttpRequest::requestCancel() to cancel the
+ operation. (Don't add the cancel requests' handled to the list.)
+ This will cancel the outstanding requests that haven't completed.
+ Canceled or completed, all requests will queue notifications. You
+ can now cycle calling update() discarding responses. Continue
+ until all requests notify or a few seconds have passed.
+
+ Global startup and shutdown is handled in the viewer. But you can
+ learn about it in the code or in the documentation in the headers.
+
+
+6. Choosing a Policy Class
+
+ Now it's time to get rid of the default policy class. Take a look
+ at the policy class definitions in newview/llappcorehttp.h.
+ Ideally, you'll find one that's compatible with what you're doing.
+ Some of the compatibility guidelines are:
+
+ * Destination: Pair of host and port. Mixing requests with
+ different destinations may cause more connection setup and tear
+ down.
+
+ * Method: http or https. Usually moot given destination. But
+ mixing these may also cause connection churn.
+
+ * Transfer size: If you're moving 100MB at a time and you make your
+ requests to the same policy class as a lot of small, fast event
+ information that fast traffic is going to get stuck behind you
+ and someone's experience is going to be miserable.
+
+ * Long poll requests: These are long-lived, must- do operations.
+ They have a special home called AP_LONG_POLL.
+
+ * Concurrency: High concurrency (5 or more) and large transfer
+ sizes are incompatible. Another head-of-the-line problem. High
+ concurrency is tolerated when it's desired to get maximal
+ throughput. Mesh and texture downloads, for example.
+
+ * Pipelined: If your requests are not idempotent, stay away from
+ anything marked 'soon' or 'yes'. Hidden retries may be a
+ problem for you. For now, would also recommend keeping PUT and
+ POST requests out of classes that may be pipelined. Support for
+ that is still a bit new.
+
+ If you haven't found a compatible match, you can either create a
+ new class (llappcorehttp.*) or just use AP_DEFAULT, the catchall
+ class when all else fails. Inventory query operations might be a
+ candidate for a new class that supported pipelining on https:.
+ Same with display name lookups and other bursty-at-login
+ operations. For other things, AP_DEFAULT will do what it can and
+ will, in some way or another, tolerate any usage. Whether the
+ users' experiences are good are for you to determine.
+
+
+7. FAQ
+
+ Q1. What do these policy classes achieve?
+
+ A1. Previously, HTTP-using code in the viewer was written as if
+ it were some isolated, local operation that didn't have to
+ consider resources, contention or impact on services and the
+ larger environment. The result was an application with on the
+ order of 100 HTTP launch points in its codebase that could create
+ dozens or even 100's of TCP connections zeroing in on grid
+ services and disrupting networking equipment, web services and
+ innocent users. The use of policy classes (modeled on
+ http://en.wikipedia.org/wiki/Class-based_queueing) is a means to
+ restrict connection concurrency, good and necessary in itself. In
+ turn, that reduces demands on an expensive resource (connection
+ setup and concurrency) which relieves strain on network points.
+ That enables connection keepalive and opportunites for true
+ improvements in throughput and user experience.
+
+ Another aspect of the classes is that they give some control over
+ how competing demands for the network will be apportioned. If
+ mesh fetches, texture fetches and inventory queries are all being
+ made at once, the relative weights of their classes' concurrency
+ limits established that apportioning. We now have an opportunity
+ to balance the entire viewer system.
+
+ Q2. How's that data sharing with refcounts working for you?
+
+ A2. Meh. It does reduce memory churn and the frequency at which
+ free blocks must be moved between threads. But it's also a design
+ for static configuration and dynamic reconfiguration (not
+ requiring a restart) is favored. Creating new options for every
+ request isn't too bad, it a sequence of "new, fill, request,
+ release" for each requested operation. That in contrast to doing
+ the "new, fill, release" at startup. The bad comes in getting at
+ the source data. One rule in this work was "no new thread
+ problems." And one source for those is pulling setting values out
+ of gSettings in threads. None of that is thread safe though we
+ tend to get away with it.
+
+ Q3. What needs to be done?
+
+ A3. There's a To-Do list in _httpinternal.h. It has both large
+ and small projects here if someone would like to try changes.
diff --git a/indra/llcorehttp/_httpinternal.h b/indra/llcorehttp/_httpinternal.h
index 008e4fd95c..f80d7f60f5 100755
--- a/indra/llcorehttp/_httpinternal.h
+++ b/indra/llcorehttp/_httpinternal.h
@@ -4,7 +4,7 @@
*
* $LicenseInfo:firstyear=2012&license=viewerlgpl$
* Second Life Viewer Source Code
- * Copyright (C) 2012, Linden Research, Inc.
+ * Copyright (C) 2012-2013, Linden Research, Inc.
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
@@ -36,7 +36,8 @@
// General library to-do list
//
// - Implement policy classes. Structure is mostly there just didn't
-// need it for the first consumer.
+// need it for the first consumer. [Classes are there. More
+// advanced features, like borrowing, aren't there yet.]
// - Consider Removing 'priority' from the request interface. Its use
// in an always active class can lead to starvation of low-priority
// requests. Requires coodination of priority values across all
@@ -46,6 +47,7 @@
// may not really need it.
// - Set/get for global policy and policy classes is clumsy. Rework
// it heading in a direction that allows for more dynamic behavior.
+// [Mostly fixed]
// - Move HttpOpRequest::prepareRequest() to HttpLibcurl for the
// pedantic.
// - Update downloader and other long-duration services are going to
@@ -64,6 +66,12 @@
// This won't help in the face of the router problems we've looked
// at, however. Detect starvation due to UDP activity and provide
// feedback to it.
+// - Change the transfer timeout scheme. We're less interested in
+// absolute time, in most cases, than in continuous progress.
+// - Many of the policy class settings are currently applied to the
+// entire class. Some, like connection limits, would be better
+// applied to each destination target making multiple targets
+// independent.
//
// Integration to-do list
// - LLTextureFetch still needs a major refactor. The use of
@@ -73,7 +81,6 @@
// the main source file.
// - Expand areas of usage eventually leading to the removal of LLCurl.
// Rough order of expansion:
-// . Mesh fetch
// . Avatar names
// . Group membership lists
// . Caps access in general
@@ -97,8 +104,8 @@ namespace LLCore
{
// Maxium number of policy classes that can be defined.
-// *TODO: Currently limited to the default class, extend.
-const int HTTP_POLICY_CLASS_LIMIT = 1;
+// *TODO: Currently limited to the default class + 1, extend.
+const int HTTP_POLICY_CLASS_LIMIT = 8;
// Debug/informational tracing. Used both
// as a global option and in per-request traces.
@@ -129,6 +136,7 @@ const int HTTP_REDIRECTS_DEFAULT = 10;
// Retries and time-on-queue are not included and aren't
// accounted for.
const long HTTP_REQUEST_TIMEOUT_DEFAULT = 30L;
+const long HTTP_REQUEST_XFER_TIMEOUT_DEFAULT = 0L;
const long HTTP_REQUEST_TIMEOUT_MIN = 0L;
const long HTTP_REQUEST_TIMEOUT_MAX = 3600L;
@@ -137,6 +145,11 @@ const int HTTP_CONNECTION_LIMIT_DEFAULT = 8;
const int HTTP_CONNECTION_LIMIT_MIN = 1;
const int HTTP_CONNECTION_LIMIT_MAX = 256;
+// Miscellaneous defaults
+const long HTTP_PIPELINING_DEFAULT = 0L;
+const bool HTTP_USE_RETRY_AFTER_DEFAULT = true;
+const long HTTP_THROTTLE_RATE_DEFAULT = 0L;
+
// Tuning parameters
// Time worker thread sleeps after a pass through the
diff --git a/indra/llcorehttp/_httplibcurl.cpp b/indra/llcorehttp/_httplibcurl.cpp
index 6fe0bfc7d1..fc257fb0c1 100755
--- a/indra/llcorehttp/_httplibcurl.cpp
+++ b/indra/llcorehttp/_httplibcurl.cpp
@@ -4,7 +4,7 @@
*
* $LicenseInfo:firstyear=2012&license=viewerlgpl$
* Second Life Viewer Source Code
- * Copyright (C) 2012, Linden Research, Inc.
+ * Copyright (C) 2012-2013, Linden Research, Inc.
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
@@ -41,7 +41,8 @@ namespace LLCore
HttpLibcurl::HttpLibcurl(HttpService * service)
: mService(service),
mPolicyCount(0),
- mMultiHandles(NULL)
+ mMultiHandles(NULL),
+ mActiveHandles(NULL)
{}
@@ -77,6 +78,9 @@ void HttpLibcurl::shutdown()
delete [] mMultiHandles;
mMultiHandles = NULL;
+
+ delete [] mActiveHandles;
+ mActiveHandles = NULL;
}
mPolicyCount = 0;
@@ -90,9 +94,12 @@ void HttpLibcurl::start(int policy_count)
mPolicyCount = policy_count;
mMultiHandles = new CURLM * [mPolicyCount];
+ mActiveHandles = new int [mPolicyCount];
+
for (int policy_class(0); policy_class < mPolicyCount; ++policy_class)
{
mMultiHandles[policy_class] = curl_multi_init();
+ mActiveHandles[policy_class] = 0;
}
}
@@ -110,8 +117,10 @@ HttpService::ELoopSpeed HttpLibcurl::processTransport()
// Give libcurl some cycles to do I/O & callbacks
for (int policy_class(0); policy_class < mPolicyCount; ++policy_class)
{
- if (! mMultiHandles[policy_class])
+ if (! mActiveHandles[policy_class] || ! mMultiHandles[policy_class])
+ {
continue;
+ }
int running(0);
CURLMcode status(CURLM_CALL_MULTI_PERFORM);
@@ -132,12 +141,10 @@ HttpService::ELoopSpeed HttpLibcurl::processTransport()
CURL * handle(msg->easy_handle);
CURLcode result(msg->data.result);
- if (completeRequest(mMultiHandles[policy_class], handle, result))
- {
- // Request is still active, don't get too sleepy
- ret = HttpService::NORMAL;
- }
- handle = NULL; // No longer valid on return
+ completeRequest(mMultiHandles[policy_class], handle, result);
+ handle = NULL; // No longer valid on return
+ ret = HttpService::NORMAL; // If anything completes, we may have a free slot.
+ // Turning around quickly reduces connection gap by 7-10mS.
}
else if (CURLMSG_NONE == msg->msg)
{
@@ -193,6 +200,7 @@ void HttpLibcurl::addOp(HttpOpRequest * op)
// On success, make operation active
mActiveOps.insert(op);
+ ++mActiveHandles[op->mReqPolicy];
}
@@ -214,6 +222,7 @@ bool HttpLibcurl::cancel(HttpHandle handle)
// Drop references
mActiveOps.erase(it);
+ --mActiveHandles[op->mReqPolicy];
op->release();
return true;
@@ -240,7 +249,7 @@ void HttpLibcurl::cancelRequest(HttpOpRequest * op)
{
LL_INFOS("CoreHttp") << "TRACE, RequestCanceled, Handle: "
<< static_cast<HttpHandle>(op)
- << ", Status: " << op->mStatus.toHex()
+ << ", Status: " << op->mStatus.toTerseString()
<< LL_ENDL;
}
@@ -275,6 +284,7 @@ bool HttpLibcurl::completeRequest(CURLM * multi_handle, CURL * handle, CURLcode
// Deactivate request
mActiveOps.erase(it);
+ --mActiveHandles[op->mReqPolicy];
op->mCurlActive = false;
// Set final status of request if it hasn't failed by other mechanisms yet
@@ -316,7 +326,7 @@ bool HttpLibcurl::completeRequest(CURLM * multi_handle, CURL * handle, CURLcode
{
LL_INFOS("CoreHttp") << "TRACE, RequestComplete, Handle: "
<< static_cast<HttpHandle>(op)
- << ", Status: " << op->mStatus.toHex()
+ << ", Status: " << op->mStatus.toTerseString()
<< LL_ENDL;
}
@@ -336,19 +346,9 @@ int HttpLibcurl::getActiveCount() const
int HttpLibcurl::getActiveCountInClass(int policy_class) const
{
- int count(0);
-
- for (active_set_t::const_iterator iter(mActiveOps.begin());
- mActiveOps.end() != iter;
- ++iter)
- {
- if ((*iter)->mReqPolicy == policy_class)
- {
- ++count;
- }
- }
-
- return count;
+ llassert_always(policy_class < mPolicyCount);
+
+ return mActiveHandles ? mActiveHandles[policy_class] : 0;
}
@@ -359,12 +359,17 @@ int HttpLibcurl::getActiveCountInClass(int policy_class) const
struct curl_slist * append_headers_to_slist(const HttpHeaders * headers, struct curl_slist * slist)
{
- for (HttpHeaders::container_t::const_iterator it(headers->mHeaders.begin());
-
- headers->mHeaders.end() != it;
- ++it)
+ const HttpHeaders::const_iterator end(headers->end());
+ for (HttpHeaders::const_iterator it(headers->begin()); end != it; ++it)
{
- slist = curl_slist_append(slist, (*it).c_str());
+ static const char sep[] = ": ";
+ std::string header;
+ header.reserve((*it).first.size() + (*it).second.size() + sizeof(sep));
+ header.append((*it).first);
+ header.append(sep);
+ header.append((*it).second);
+
+ slist = curl_slist_append(slist, header.c_str());
}
return slist;
}
diff --git a/indra/llcorehttp/_httplibcurl.h b/indra/llcorehttp/_httplibcurl.h
index 611f029ef5..67f98dd4f0 100755
--- a/indra/llcorehttp/_httplibcurl.h
+++ b/indra/llcorehttp/_httplibcurl.h
@@ -4,7 +4,7 @@
*
* $LicenseInfo:firstyear=2012&license=viewerlgpl$
* Second Life Viewer Source Code
- * Copyright (C) 2012, Linden Research, Inc.
+ * Copyright (C) 2012-2013, Linden Research, Inc.
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
@@ -71,16 +71,22 @@ public:
///
/// @return Indication of how long this method is
/// willing to wait for next service call.
+ ///
+ /// Threading: called by worker thread.
HttpService::ELoopSpeed processTransport();
/// Add request to the active list. Caller is expected to have
/// provided us with a reference count on the op to hold the
/// request. (No additional references will be added.)
+ ///
+ /// Threading: called by worker thread.
void addOp(HttpOpRequest * op);
/// One-time call to set the number of policy classes to be
/// serviced and to create the resources for each. Value
/// must agree with HttpPolicy::setPolicies() call.
+ ///
+ /// Threading: called by init thread.
void start(int policy_count);
/// Synchronously stop libcurl operations. All active requests
@@ -91,9 +97,13 @@ public:
/// respective reply queues.
///
/// Can be restarted with a start() call.
+ ///
+ /// Threading: called by worker thread.
void shutdown();
/// Return global and per-class counts of active requests.
+ ///
+ /// Threading: called by worker thread.
int getActiveCount() const;
int getActiveCountInClass(int policy_class) const;
@@ -103,6 +113,7 @@ public:
///
/// @return True if handle was found and operation canceled.
///
+ /// Threading: called by worker thread.
bool cancel(HttpHandle handle);
protected:
@@ -121,7 +132,8 @@ protected:
HttpService * mService; // Simple reference, not owner
active_set_t mActiveOps;
int mPolicyCount;
- CURLM ** mMultiHandles;
+ CURLM ** mMultiHandles; // One handle per policy class
+ int * mActiveHandles; // Active count per policy class
}; // end class HttpLibcurl
} // end namespace LLCore
diff --git a/indra/llcorehttp/_httpoperation.cpp b/indra/llcorehttp/_httpoperation.cpp
index 5cf5bc5930..5bb0654652 100755
--- a/indra/llcorehttp/_httpoperation.cpp
+++ b/indra/llcorehttp/_httpoperation.cpp
@@ -4,7 +4,7 @@
*
* $LicenseInfo:firstyear=2012&license=viewerlgpl$
* Second Life Viewer Source Code
- * Copyright (C) 2012, Linden Research, Inc.
+ * Copyright (C) 2012-2013, Linden Research, Inc.
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
@@ -53,7 +53,7 @@ HttpOperation::HttpOperation()
mUserHandler(NULL),
mReqPolicy(HttpRequest::DEFAULT_POLICY_ID),
mReqPriority(0U),
- mTracing(0)
+ mTracing(HTTP_TRACE_OFF)
{
mMetricCreated = totalTime();
}
@@ -94,7 +94,7 @@ void HttpOperation::stageFromRequest(HttpService *)
// Default implementation should never be called. This
// indicates an operation making a transition that isn't
// defined.
- LL_ERRS("HttpCore") << "Default stageFromRequest method may not be called."
+ LL_ERRS("CoreHttp") << "Default stageFromRequest method may not be called."
<< LL_ENDL;
}
@@ -104,7 +104,7 @@ void HttpOperation::stageFromReady(HttpService *)
// Default implementation should never be called. This
// indicates an operation making a transition that isn't
// defined.
- LL_ERRS("HttpCore") << "Default stageFromReady method may not be called."
+ LL_ERRS("CoreHttp") << "Default stageFromReady method may not be called."
<< LL_ENDL;
}
@@ -114,7 +114,7 @@ void HttpOperation::stageFromActive(HttpService *)
// Default implementation should never be called. This
// indicates an operation making a transition that isn't
// defined.
- LL_ERRS("HttpCore") << "Default stageFromActive method may not be called."
+ LL_ERRS("CoreHttp") << "Default stageFromActive method may not be called."
<< LL_ENDL;
}
diff --git a/indra/llcorehttp/_httpoperation.h b/indra/llcorehttp/_httpoperation.h
index 914627fad0..937a61187d 100755
--- a/indra/llcorehttp/_httpoperation.h
+++ b/indra/llcorehttp/_httpoperation.h
@@ -4,7 +4,7 @@
*
* $LicenseInfo:firstyear=2012&license=viewerlgpl$
* Second Life Viewer Source Code
- * Copyright (C) 2012, Linden Research, Inc.
+ * Copyright (C) 2012-2013, Linden Research, Inc.
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
@@ -72,7 +72,7 @@ class HttpService;
class HttpOperation : public LLCoreInt::RefCounted
{
public:
- /// Threading: called by a consumer/application thread.
+ /// Threading: called by consumer thread.
HttpOperation();
protected:
@@ -108,7 +108,7 @@ public:
/// by the worker thread. This is passible data
/// until notification is performed.
///
- /// Threading: called by application thread.
+ /// Threading: called by consumer thread.
///
void setReplyPath(HttpReplyQueue * reply_queue,
HttpHandler * handler);
@@ -141,7 +141,7 @@ public:
/// call to HttpRequest::update(). This method does the necessary
/// dispatching.
///
- /// Threading: called by application thread.
+ /// Threading: called by consumer thread.
///
virtual void visitNotifier(HttpRequest *);
diff --git a/indra/llcorehttp/_httpoprequest.cpp b/indra/llcorehttp/_httpoprequest.cpp
index 207ed8e1e4..926031501e 100755
--- a/indra/llcorehttp/_httpoprequest.cpp
+++ b/indra/llcorehttp/_httpoprequest.cpp
@@ -64,6 +64,15 @@ int parse_content_range_header(char * buffer,
unsigned int * last,
unsigned int * length);
+// Similar for Retry-After headers. Only parses the delta form
+// of the header, HTTP time formats aren't interesting for client
+// purposes.
+//
+// @return 0 if successfully parsed and seconds time delta
+// returned in time argument.
+//
+int parse_retry_after_header(char * buffer, int * time);
+
// Take data from libcurl's CURLOPT_DEBUGFUNCTION callback and
// escape and format it for a tracing line in logging. Absolutely
@@ -74,14 +83,16 @@ void escape_libcurl_debug_data(char * buffer, size_t len, bool scrub,
std::string & safe_line);
-// OS-neutral string comparisons of various types
-int os_strncasecmp(const char *s1, const char *s2, size_t n);
-int os_strcasecmp(const char *s1, const char *s2);
-char * os_strtok_r(char *str, const char *delim, char **saveptr);
-
+// OS-neutral string comparisons of various types.
+int os_strcasecmp(const char * s1, const char * s2);
+char * os_strtok_r(char * str, const char * delim, char ** saveptr);
+char * os_strtrim(char * str);
+char * os_strltrim(char * str);
+void os_strlower(char * str);
-static const char * const hdr_whitespace(" \t");
-static const char * const hdr_separator(": \t");
+// Error testing and reporting for libcurl status codes
+void check_curl_easy_code(CURLcode code);
+void check_curl_easy_code(CURLcode code, int curl_setopt_option);
} // end anonymous namespace
@@ -104,12 +115,15 @@ HttpOpRequest::HttpOpRequest()
mCurlService(NULL),
mCurlHeaders(NULL),
mCurlBodyPos(0),
+ mCurlTemp(NULL),
+ mCurlTempLen(0),
mReplyBody(NULL),
mReplyOffset(0),
mReplyLength(0),
mReplyFullLength(0),
mReplyHeaders(NULL),
mPolicyRetries(0),
+ mPolicy503Retries(0),
mPolicyRetryAt(HttpTime(0)),
mPolicyRetryLimit(HTTP_RETRY_COUNT_DEFAULT)
{
@@ -153,6 +167,10 @@ HttpOpRequest::~HttpOpRequest()
mCurlHeaders = NULL;
}
+ delete [] mCurlTemp;
+ mCurlTemp = NULL;
+ mCurlTempLen = 0;
+
if (mReplyBody)
{
mReplyBody->release();
@@ -208,6 +226,11 @@ void HttpOpRequest::stageFromActive(HttpService * service)
mCurlHeaders = NULL;
}
+ // Also not needed on the other side
+ delete [] mCurlTemp;
+ mCurlTemp = NULL;
+ mCurlTempLen = 0;
+
addAsReply();
}
@@ -226,6 +249,7 @@ void HttpOpRequest::visitNotifier(HttpRequest * request)
response->setRange(mReplyOffset, mReplyLength, mReplyFullLength);
}
response->setContentType(mReplyConType);
+ response->setRetries(mPolicyRetries, mPolicy503Retries);
mUserHandler->onCompleted(static_cast<HttpHandle>(this), response);
@@ -335,6 +359,10 @@ void HttpOpRequest::setupCommon(HttpRequest::policy_t policy_id,
{
mProcFlags |= PF_SAVE_HEADERS;
}
+ if (options->getUseRetryAfter())
+ {
+ mProcFlags |= PF_USE_RETRY_AFTER;
+ }
mPolicyRetryLimit = options->getRetries();
mPolicyRetryLimit = llclamp(mPolicyRetryLimit, HTTP_RETRY_COUNT_MIN, HTTP_RETRY_COUNT_MAX);
mTracing = (std::max)(mTracing, llclamp(options->getTrace(), HTTP_TRACE_MIN, HTTP_TRACE_MAX));
@@ -350,6 +378,8 @@ void HttpOpRequest::setupCommon(HttpRequest::policy_t policy_id,
//
HttpStatus HttpOpRequest::prepareRequest(HttpService * service)
{
+ CURLcode code;
+
// Scrub transport and result data for retried op case
mCurlActive = false;
mCurlHandle = NULL;
@@ -379,64 +409,108 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service)
// *FIXME: better error handling later
HttpStatus status;
- // Get policy options
+ // Get global policy options
HttpPolicyGlobal & policy(service->getPolicy().getGlobalOptions());
mCurlHandle = LLCurl::createStandardCurlHandle();
-
- curl_easy_setopt(mCurlHandle, CURLOPT_WRITEFUNCTION, writeCallback);
- curl_easy_setopt(mCurlHandle, CURLOPT_READFUNCTION, readCallback);
- curl_easy_setopt(mCurlHandle, CURLOPT_READDATA, this);
- curl_easy_setopt(mCurlHandle, CURLOPT_WRITEDATA, this);
- curl_easy_setopt(mCurlHandle, CURLOPT_URL, mReqURL.c_str());
- curl_easy_setopt(mCurlHandle, CURLOPT_PRIVATE, this);
- curl_easy_setopt(mCurlHandle, CURLOPT_MAXREDIRS, HTTP_REDIRECTS_DEFAULT);
-
- const std::string * opt_value(NULL);
- long opt_long(0L);
- policy.get(HttpRequest::GP_LLPROXY, &opt_long);
- if (opt_long)
+ if (! mCurlHandle)
+ {
+ // We're in trouble. We'll continue but it won't go well.
+ LL_WARNS("CoreHttp") << "Failed to allocate libcurl easy handle. Continuing."
+ << LL_ENDL;
+ return HttpStatus(HttpStatus::LLCORE, HE_BAD_ALLOC);
+ }
+ code = curl_easy_setopt(mCurlHandle, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4);
+ check_curl_easy_code(code, CURLOPT_IPRESOLVE);
+ code = curl_easy_setopt(mCurlHandle, CURLOPT_NOSIGNAL, 1);
+ check_curl_easy_code(code, CURLOPT_NOSIGNAL);
+ code = curl_easy_setopt(mCurlHandle, CURLOPT_NOPROGRESS, 1);
+ check_curl_easy_code(code, CURLOPT_NOPROGRESS);
+ code = curl_easy_setopt(mCurlHandle, CURLOPT_URL, mReqURL.c_str());
+ check_curl_easy_code(code, CURLOPT_URL);
+ code = curl_easy_setopt(mCurlHandle, CURLOPT_PRIVATE, this);
+ check_curl_easy_code(code, CURLOPT_PRIVATE);
+ code = curl_easy_setopt(mCurlHandle, CURLOPT_ENCODING, "");
+ check_curl_easy_code(code, CURLOPT_ENCODING);
+
+ // The Linksys WRT54G V5 router has an issue with frequent
+ // DNS lookups from LAN machines. If they happen too often,
+ // like for every HTTP request, the router gets annoyed after
+ // about 700 or so requests and starts issuing TCP RSTs to
+ // new connections. Reuse the DNS lookups for even a few
+ // seconds and no RSTs.
+ code = curl_easy_setopt(mCurlHandle, CURLOPT_DNS_CACHE_TIMEOUT, 15);
+ check_curl_easy_code(code, CURLOPT_DNS_CACHE_TIMEOUT);
+ code = curl_easy_setopt(mCurlHandle, CURLOPT_AUTOREFERER, 1);
+ check_curl_easy_code(code, CURLOPT_AUTOREFERER);
+ code = curl_easy_setopt(mCurlHandle, CURLOPT_FOLLOWLOCATION, 1);
+ check_curl_easy_code(code, CURLOPT_FOLLOWLOCATION);
+ code = curl_easy_setopt(mCurlHandle, CURLOPT_MAXREDIRS, HTTP_REDIRECTS_DEFAULT);
+ check_curl_easy_code(code, CURLOPT_MAXREDIRS);
+ code = curl_easy_setopt(mCurlHandle, CURLOPT_WRITEFUNCTION, writeCallback);
+ check_curl_easy_code(code, CURLOPT_WRITEFUNCTION);
+ code = curl_easy_setopt(mCurlHandle, CURLOPT_WRITEDATA, this);
+ check_curl_easy_code(code, CURLOPT_WRITEDATA);
+ code = curl_easy_setopt(mCurlHandle, CURLOPT_READFUNCTION, readCallback);
+ check_curl_easy_code(code, CURLOPT_READFUNCTION);
+ code = curl_easy_setopt(mCurlHandle, CURLOPT_READDATA, this);
+ check_curl_easy_code(code, CURLOPT_READDATA);
+ code = curl_easy_setopt(mCurlHandle, CURLOPT_SSL_VERIFYPEER, 1);
+ check_curl_easy_code(code, CURLOPT_SSL_VERIFYPEER);
+ code = curl_easy_setopt(mCurlHandle, CURLOPT_SSL_VERIFYHOST, 0);
+ check_curl_easy_code(code, CURLOPT_SSL_VERIFYHOST);
+
+ if (policy.mUseLLProxy)
{
// Use the viewer-based thread-safe API which has a
// fast/safe check for proxy enable. Would like to
// encapsulate this someway...
LLProxy::getInstance()->applyProxySettings(mCurlHandle);
}
- else if (policy.get(HttpRequest::GP_HTTP_PROXY, &opt_value))
+ else if (policy.mHttpProxy.size())
{
// *TODO: This is fine for now but get fuller socks5/
// authentication thing going later....
- curl_easy_setopt(mCurlHandle, CURLOPT_PROXY, opt_value->c_str());
- curl_easy_setopt(mCurlHandle, CURLOPT_PROXYTYPE, CURLPROXY_HTTP);
+ code = curl_easy_setopt(mCurlHandle, CURLOPT_PROXY, policy.mHttpProxy.c_str());
+ check_curl_easy_code(code, CURLOPT_PROXY);
+ code = curl_easy_setopt(mCurlHandle, CURLOPT_PROXYTYPE, CURLPROXY_HTTP);
+ check_curl_easy_code(code, CURLOPT_PROXYTYPE);
}
- if (policy.get(HttpRequest::GP_CA_PATH, &opt_value))
+ if (policy.mCAPath.size())
{
- curl_easy_setopt(mCurlHandle, CURLOPT_CAPATH, opt_value->c_str());
+ code = curl_easy_setopt(mCurlHandle, CURLOPT_CAPATH, policy.mCAPath.c_str());
+ check_curl_easy_code(code, CURLOPT_CAPATH);
}
- if (policy.get(HttpRequest::GP_CA_FILE, &opt_value))
+ if (policy.mCAFile.size())
{
- curl_easy_setopt(mCurlHandle, CURLOPT_CAINFO, opt_value->c_str());
+ code = curl_easy_setopt(mCurlHandle, CURLOPT_CAINFO, policy.mCAFile.c_str());
+ check_curl_easy_code(code, CURLOPT_CAINFO);
}
switch (mReqMethod)
{
case HOR_GET:
- curl_easy_setopt(mCurlHandle, CURLOPT_HTTPGET, 1);
+ code = curl_easy_setopt(mCurlHandle, CURLOPT_HTTPGET, 1);
+ check_curl_easy_code(code, CURLOPT_HTTPGET);
mCurlHeaders = curl_slist_append(mCurlHeaders, "Connection: keep-alive");
mCurlHeaders = curl_slist_append(mCurlHeaders, "Keep-alive: 300");
break;
case HOR_POST:
{
- curl_easy_setopt(mCurlHandle, CURLOPT_POST, 1);
- curl_easy_setopt(mCurlHandle, CURLOPT_ENCODING, "");
+ code = curl_easy_setopt(mCurlHandle, CURLOPT_POST, 1);
+ check_curl_easy_code(code, CURLOPT_POST);
+ code = curl_easy_setopt(mCurlHandle, CURLOPT_ENCODING, "");
+ check_curl_easy_code(code, CURLOPT_ENCODING);
long data_size(0);
if (mReqBody)
{
data_size = mReqBody->size();
}
- curl_easy_setopt(mCurlHandle, CURLOPT_POSTFIELDS, static_cast<void *>(NULL));
- curl_easy_setopt(mCurlHandle, CURLOPT_POSTFIELDSIZE, data_size);
+ code = curl_easy_setopt(mCurlHandle, CURLOPT_POSTFIELDS, static_cast<void *>(NULL));
+ check_curl_easy_code(code, CURLOPT_POSTFIELDS);
+ code = curl_easy_setopt(mCurlHandle, CURLOPT_POSTFIELDSIZE, data_size);
+ check_curl_easy_code(code, CURLOPT_POSTFIELDSIZE);
mCurlHeaders = curl_slist_append(mCurlHeaders, "Expect:");
mCurlHeaders = curl_slist_append(mCurlHeaders, "Connection: keep-alive");
mCurlHeaders = curl_slist_append(mCurlHeaders, "Keep-alive: 300");
@@ -445,14 +519,17 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service)
case HOR_PUT:
{
- curl_easy_setopt(mCurlHandle, CURLOPT_UPLOAD, 1);
+ code = curl_easy_setopt(mCurlHandle, CURLOPT_UPLOAD, 1);
+ check_curl_easy_code(code, CURLOPT_UPLOAD);
long data_size(0);
if (mReqBody)
{
data_size = mReqBody->size();
}
- curl_easy_setopt(mCurlHandle, CURLOPT_INFILESIZE, data_size);
- curl_easy_setopt(mCurlHandle, CURLOPT_POSTFIELDS, (void *) NULL);
+ code = curl_easy_setopt(mCurlHandle, CURLOPT_INFILESIZE, data_size);
+ check_curl_easy_code(code, CURLOPT_INFILESIZE);
+ code = curl_easy_setopt(mCurlHandle, CURLOPT_POSTFIELDS, (void *) NULL);
+ check_curl_easy_code(code, CURLOPT_POSTFIELDS);
mCurlHeaders = curl_slist_append(mCurlHeaders, "Expect:");
mCurlHeaders = curl_slist_append(mCurlHeaders, "Connection: keep-alive");
mCurlHeaders = curl_slist_append(mCurlHeaders, "Keep-alive: 300");
@@ -469,9 +546,12 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service)
// Tracing
if (mTracing >= HTTP_TRACE_CURL_HEADERS)
{
- curl_easy_setopt(mCurlHandle, CURLOPT_VERBOSE, 1);
- curl_easy_setopt(mCurlHandle, CURLOPT_DEBUGDATA, this);
- curl_easy_setopt(mCurlHandle, CURLOPT_DEBUGFUNCTION, debugCallback);
+ code = curl_easy_setopt(mCurlHandle, CURLOPT_VERBOSE, 1);
+ check_curl_easy_code(code, CURLOPT_VERBOSE);
+ code = curl_easy_setopt(mCurlHandle, CURLOPT_DEBUGDATA, this);
+ check_curl_easy_code(code, CURLOPT_DEBUGDATA);
+ code = curl_easy_setopt(mCurlHandle, CURLOPT_DEBUGFUNCTION, debugCallback);
+ check_curl_easy_code(code, CURLOPT_DEBUGFUNCTION);
}
// There's a CURLOPT for this now...
@@ -499,13 +579,22 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service)
// Request options
long timeout(HTTP_REQUEST_TIMEOUT_DEFAULT);
+ long xfer_timeout(HTTP_REQUEST_XFER_TIMEOUT_DEFAULT);
if (mReqOptions)
- {
+ {
timeout = mReqOptions->getTimeout();
timeout = llclamp(timeout, HTTP_REQUEST_TIMEOUT_MIN, HTTP_REQUEST_TIMEOUT_MAX);
+ xfer_timeout = mReqOptions->getTransferTimeout();
+ xfer_timeout = llclamp(xfer_timeout, HTTP_REQUEST_TIMEOUT_MIN, HTTP_REQUEST_TIMEOUT_MAX);
+ }
+ if (xfer_timeout == 0L)
+ {
+ xfer_timeout = timeout;
}
- curl_easy_setopt(mCurlHandle, CURLOPT_TIMEOUT, timeout);
- curl_easy_setopt(mCurlHandle, CURLOPT_CONNECTTIMEOUT, timeout);
+ code = curl_easy_setopt(mCurlHandle, CURLOPT_TIMEOUT, xfer_timeout);
+ check_curl_easy_code(code, CURLOPT_TIMEOUT);
+ code = curl_easy_setopt(mCurlHandle, CURLOPT_CONNECTTIMEOUT, timeout);
+ check_curl_easy_code(code, CURLOPT_CONNECTTIMEOUT);
// Request headers
if (mReqHeaders)
@@ -513,12 +602,15 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service)
// Caller's headers last to override
mCurlHeaders = append_headers_to_slist(mReqHeaders, mCurlHeaders);
}
- curl_easy_setopt(mCurlHandle, CURLOPT_HTTPHEADER, mCurlHeaders);
+ code = curl_easy_setopt(mCurlHandle, CURLOPT_HTTPHEADER, mCurlHeaders);
+ check_curl_easy_code(code, CURLOPT_HTTPHEADER);
- if (mProcFlags & (PF_SCAN_RANGE_HEADER | PF_SAVE_HEADERS))
+ if (mProcFlags & (PF_SCAN_RANGE_HEADER | PF_SAVE_HEADERS | PF_USE_RETRY_AFTER))
{
- curl_easy_setopt(mCurlHandle, CURLOPT_HEADERFUNCTION, headerCallback);
- curl_easy_setopt(mCurlHandle, CURLOPT_HEADERDATA, this);
+ code = curl_easy_setopt(mCurlHandle, CURLOPT_HEADERFUNCTION, headerCallback);
+ check_curl_easy_code(code, CURLOPT_HEADERFUNCTION);
+ code = curl_easy_setopt(mCurlHandle, CURLOPT_HEADERDATA, this);
+ check_curl_easy_code(code, CURLOPT_HEADERDATA);
}
if (status)
@@ -559,7 +651,7 @@ size_t HttpOpRequest::readCallback(void * data, size_t size, size_t nmemb, void
{
// Warn but continue if the read position moves beyond end-of-body
// for some reason.
- LL_WARNS("HttpCore") << "Request body position beyond body size. Truncating request body."
+ LL_WARNS("CoreHttp") << "Request body position beyond body size. Truncating request body."
<< LL_ENDL;
}
return 0;
@@ -576,15 +668,15 @@ size_t HttpOpRequest::headerCallback(void * data, size_t size, size_t nmemb, voi
{
static const char status_line[] = "HTTP/";
static const size_t status_line_len = sizeof(status_line) - 1;
-
- static const char con_ran_line[] = "content-range:";
- static const size_t con_ran_line_len = sizeof(con_ran_line) - 1;
-
+ static const char con_ran_line[] = "content-range";
+ static const char con_retry_line[] = "retry-after";
+
HttpOpRequest * op(static_cast<HttpOpRequest *>(userdata));
const size_t hdr_size(size * nmemb);
const char * hdr_data(static_cast<const char *>(data)); // Not null terminated
-
+ bool is_header(true);
+
if (hdr_size >= status_line_len && ! strncmp(status_line, hdr_data, status_line_len))
{
// One of possibly several status lines. Reset what we know and start over
@@ -592,11 +684,13 @@ size_t HttpOpRequest::headerCallback(void * data, size_t size, size_t nmemb, voi
op->mReplyOffset = 0;
op->mReplyLength = 0;
op->mReplyFullLength = 0;
+ op->mReplyRetryAfter = 0;
op->mStatus = HttpStatus();
if (op->mReplyHeaders)
{
- op->mReplyHeaders->mHeaders.clear();
+ op->mReplyHeaders->clear();
}
+ is_header = false;
}
// Nothing in here wants a final CR/LF combination. Remove
@@ -609,52 +703,109 @@ size_t HttpOpRequest::headerCallback(void * data, size_t size, size_t nmemb, voi
--wanted_hdr_size;
}
}
+
+ // Copy and normalize header fragments for the following
+ // stages. Would like to modify the data in-place but that
+ // may not be allowed and we need one byte extra for NUL.
+ // At the end of this we will have:
+ //
+ // If ':' present in header:
+ // 1. name points to text to left of colon which
+ // will be ascii lower-cased and left and right
+ // trimmed of whitespace.
+ // 2. value points to text to right of colon which
+ // will be left trimmed of whitespace.
+ // Otherwise:
+ // 1. name points to header which will be left
+ // trimmed of whitespace.
+ // 2. value is NULL
+ // Any non-NULL pointer may point to a zero-length string.
+ //
+ if (wanted_hdr_size >= op->mCurlTempLen)
+ {
+ delete [] op->mCurlTemp;
+ op->mCurlTempLen = 2 * wanted_hdr_size + 1;
+ op->mCurlTemp = new char [op->mCurlTempLen];
+ }
+ memcpy(op->mCurlTemp, hdr_data, wanted_hdr_size);
+ op->mCurlTemp[wanted_hdr_size] = '\0';
+ char * name(op->mCurlTemp);
+ char * value(strchr(name, ':'));
+ if (value)
+ {
+ *value++ = '\0';
+ os_strlower(name);
+ name = os_strtrim(name);
+ value = os_strltrim(value);
+ }
+ else
+ {
+ // Doesn't look well-formed, do minimal normalization on it
+ name = os_strltrim(name);
+ }
+
+ // Normalized, now reject headers with empty names.
+ if (! *name)
+ {
+ // No use continuing
+ return hdr_size;
+ }
// Save header if caller wants them in the response
- if (op->mProcFlags & PF_SAVE_HEADERS)
+ if (is_header && op->mProcFlags & PF_SAVE_HEADERS)
{
// Save headers in response
if (! op->mReplyHeaders)
{
op->mReplyHeaders = new HttpHeaders;
}
- op->mReplyHeaders->mHeaders.push_back(std::string(hdr_data, wanted_hdr_size));
+ op->mReplyHeaders->append(name, value ? value : "");
}
+ // From this point, header-specific processors are free to
+ // modify the header value.
+
// Detect and parse 'Content-Range' headers
- if (op->mProcFlags & PF_SCAN_RANGE_HEADER)
+ if (is_header
+ && op->mProcFlags & PF_SCAN_RANGE_HEADER
+ && value && *value
+ && ! strcmp(name, con_ran_line))
{
- char hdr_buffer[128]; // Enough for a reasonable header
- size_t frag_size((std::min)(wanted_hdr_size, sizeof(hdr_buffer) - 1));
-
- memcpy(hdr_buffer, hdr_data, frag_size);
- hdr_buffer[frag_size] = '\0';
- if (frag_size > con_ran_line_len &&
- ! os_strncasecmp(hdr_buffer, con_ran_line, con_ran_line_len))
+ unsigned int first(0), last(0), length(0);
+ int status;
+
+ if (! (status = parse_content_range_header(value, &first, &last, &length)))
+ {
+ // Success, record the fragment position
+ op->mReplyOffset = first;
+ op->mReplyLength = last - first + 1;
+ op->mReplyFullLength = length;
+ }
+ else if (-1 == status)
{
- unsigned int first(0), last(0), length(0);
- int status;
+ // Response is badly formed and shouldn't be accepted
+ op->mStatus = HttpStatus(HttpStatus::LLCORE, HE_INV_CONTENT_RANGE_HDR);
+ }
+ else
+ {
+ // Ignore the unparsable.
+ LL_INFOS_ONCE("CoreHttp") << "Problem parsing odd Content-Range header: '"
+ << std::string(hdr_data, wanted_hdr_size)
+ << "'. Ignoring."
+ << LL_ENDL;
+ }
+ }
- if (! (status = parse_content_range_header(hdr_buffer, &first, &last, &length)))
- {
- // Success, record the fragment position
- op->mReplyOffset = first;
- op->mReplyLength = last - first + 1;
- op->mReplyFullLength = length;
- }
- else if (-1 == status)
- {
- // Response is badly formed and shouldn't be accepted
- op->mStatus = HttpStatus(HttpStatus::LLCORE, HE_INV_CONTENT_RANGE_HDR);
- }
- else
- {
- // Ignore the unparsable.
- LL_INFOS_ONCE("CoreHttp") << "Problem parsing odd Content-Range header: '"
- << std::string(hdr_data, frag_size)
- << "'. Ignoring."
- << LL_ENDL;
- }
+ // Detect and parse 'Retry-After' headers
+ if (is_header
+ && op->mProcFlags & PF_USE_RETRY_AFTER
+ && value && *value
+ && ! strcmp(name, con_retry_line))
+ {
+ int time(0);
+ if (! parse_retry_after_header(value, &time))
+ {
+ op->mReplyRetryAfter = time;
}
}
@@ -769,14 +920,16 @@ int parse_content_range_header(char * buffer,
unsigned int * last,
unsigned int * length)
{
+ static const char * const hdr_whitespace(" \t");
+
char * tok_state(NULL), * tok(NULL);
bool match(true);
- if (! os_strtok_r(buffer, hdr_separator, &tok_state))
+ if (! (tok = os_strtok_r(buffer, hdr_whitespace, &tok_state)))
match = false;
- if (match && (tok = os_strtok_r(NULL, hdr_whitespace, &tok_state)))
- match = 0 == os_strcasecmp("bytes", tok);
- if (match && ! (tok = os_strtok_r(NULL, " \t", &tok_state)))
+ else
+ match = (0 == os_strcasecmp("bytes", tok));
+ if (match && ! (tok = os_strtok_r(NULL, hdr_whitespace, &tok_state)))
match = false;
if (match)
{
@@ -815,6 +968,25 @@ int parse_content_range_header(char * buffer,
}
+int parse_retry_after_header(char * buffer, int * time)
+{
+ char * endptr(buffer);
+ long lcl_time(strtol(buffer, &endptr, 10));
+ if (*endptr == '\0' && endptr != buffer && lcl_time > 0)
+ {
+ *time = lcl_time;
+ return 0;
+ }
+
+ // Could attempt to parse HTTP time here but we're not really
+ // interested in it. Scheduling based on wallclock time on
+ // user hardware will lead to tears.
+
+ // Header is there but badly/unexpectedly formed, try to ignore it.
+ return 1;
+}
+
+
void escape_libcurl_debug_data(char * buffer, size_t len, bool scrub, std::string & safe_line)
{
std::string out;
@@ -851,15 +1023,6 @@ void escape_libcurl_debug_data(char * buffer, size_t len, bool scrub, std::strin
}
-int os_strncasecmp(const char *s1, const char *s2, size_t n)
-{
-#if LL_WINDOWS
- return _strnicmp(s1, s2, n);
-#else
- return strncasecmp(s1, s2, n);
-#endif // LL_WINDOWS
-}
-
int os_strcasecmp(const char *s1, const char *s2)
{
@@ -881,6 +1044,73 @@ char * os_strtok_r(char *str, const char *delim, char ** savestate)
}
-} // end anonymous namespace
+void os_strlower(char * str)
+{
+ for (char c(0); (c = *str); ++str)
+ {
+ *str = tolower(c);
+ }
+}
-
+
+char * os_strtrim(char * lstr)
+{
+ while (' ' == *lstr || '\t' == *lstr)
+ {
+ ++lstr;
+ }
+ if (*lstr)
+ {
+ char * rstr(lstr + strlen(lstr));
+ while (lstr < rstr && *--rstr)
+ {
+ if (' ' == *rstr || '\t' == *rstr)
+ {
+ *rstr = '\0';
+ }
+ }
+ llassert(lstr <= rstr);
+ }
+ return lstr;
+}
+
+
+char * os_strltrim(char * lstr)
+{
+ while (' ' == *lstr || '\t' == *lstr)
+ {
+ ++lstr;
+ }
+ return lstr;
+}
+
+
+void check_curl_easy_code(CURLcode code, int curl_setopt_option)
+{
+ if (CURLE_OK != code)
+ {
+ // Comment from old llcurl code which may no longer apply:
+ //
+ // linux appears to throw a curl error once per session for a bad initialization
+ // at a pretty random time (when enabling cookies).
+ LL_WARNS("CoreHttp") << "libcurl error detected: " << curl_easy_strerror(code)
+ << ", curl_easy_setopt option: " << curl_setopt_option
+ << LL_ENDL;
+ }
+}
+
+
+void check_curl_easy_code(CURLcode code)
+{
+ if (CURLE_OK != code)
+ {
+ // Comment from old llcurl code which may no longer apply:
+ //
+ // linux appears to throw a curl error once per session for a bad initialization
+ // at a pretty random time (when enabling cookies).
+ LL_WARNS("CoreHttp") << "libcurl error detected: " << curl_easy_strerror(code)
+ << LL_ENDL;
+ }
+}
+
+} // end anonymous namespace
diff --git a/indra/llcorehttp/_httpoprequest.h b/indra/llcorehttp/_httpoprequest.h
index 74a349b0bf..2f628b5aba 100755
--- a/indra/llcorehttp/_httpoprequest.h
+++ b/indra/llcorehttp/_httpoprequest.h
@@ -4,7 +4,7 @@
*
* $LicenseInfo:firstyear=2012&license=viewerlgpl$
* Second Life Viewer Source Code
- * Copyright (C) 2012, Linden Research, Inc.
+ * Copyright (C) 2012-2013, Linden Research, Inc.
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
@@ -157,6 +157,7 @@ protected:
unsigned int mProcFlags;
static const unsigned int PF_SCAN_RANGE_HEADER = 0x00000001U;
static const unsigned int PF_SAVE_HEADERS = 0x00000002U;
+ static const unsigned int PF_USE_RETRY_AFTER = 0x00000004U;
public:
// Request data
@@ -174,6 +175,8 @@ public:
HttpService * mCurlService;
curl_slist * mCurlHeaders;
size_t mCurlBodyPos;
+ char * mCurlTemp; // Scratch buffer for header processing
+ size_t mCurlTempLen;
// Result data
HttpStatus mStatus;
@@ -183,9 +186,11 @@ public:
size_t mReplyFullLength;
HttpHeaders * mReplyHeaders;
std::string mReplyConType;
+ int mReplyRetryAfter;
// Policy data
int mPolicyRetries;
+ int mPolicy503Retries;
HttpTime mPolicyRetryAt;
int mPolicyRetryLimit;
}; // end class HttpOpRequest
diff --git a/indra/llcorehttp/_httpopsetget.cpp b/indra/llcorehttp/_httpopsetget.cpp
index 8198528a9b..a5363f9170 100755
--- a/indra/llcorehttp/_httpopsetget.cpp
+++ b/indra/llcorehttp/_httpopsetget.cpp
@@ -4,7 +4,7 @@
*
* $LicenseInfo:firstyear=2012&license=viewerlgpl$
* Second Life Viewer Source Code
- * Copyright (C) 2012, Linden Research, Inc.
+ * Copyright (C) 2012-2013, Linden Research, Inc.
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
@@ -27,6 +27,7 @@
#include "_httpopsetget.h"
#include "httpcommon.h"
+#include "httprequest.h"
#include "_httpservice.h"
#include "_httppolicy.h"
@@ -43,10 +44,11 @@ namespace LLCore
HttpOpSetGet::HttpOpSetGet()
: HttpOperation(),
- mIsGlobal(false),
- mDoSet(false),
- mSetting(-1), // Nothing requested
- mLongValue(0L)
+ mReqOption(HttpRequest::PO_CONNECTION_LIMIT),
+ mReqClass(HttpRequest::INVALID_POLICY_ID),
+ mReqDoSet(false),
+ mReqLongValue(0L),
+ mReplyLongValue(0L)
{}
@@ -54,37 +56,84 @@ HttpOpSetGet::~HttpOpSetGet()
{}
-void HttpOpSetGet::setupGet(HttpRequest::EGlobalPolicy setting)
+HttpStatus HttpOpSetGet::setupGet(HttpRequest::EPolicyOption opt, HttpRequest::policy_t pclass)
{
- mIsGlobal = true;
- mSetting = setting;
+ HttpStatus status;
+
+ mReqOption = opt;
+ mReqClass = pclass;
+ return status;
}
-void HttpOpSetGet::setupSet(HttpRequest::EGlobalPolicy setting, const std::string & value)
+HttpStatus HttpOpSetGet::setupSet(HttpRequest::EPolicyOption opt, HttpRequest::policy_t pclass, long value)
{
- mIsGlobal = true;
- mDoSet = true;
- mSetting = setting;
- mStrValue = value;
+ HttpStatus status;
+
+ if (! HttpService::sOptionDesc[opt].mIsLong)
+ {
+ return HttpStatus(HttpStatus::LLCORE, HE_INVALID_ARG);
+ }
+ if (! HttpService::sOptionDesc[opt].mIsDynamic)
+ {
+ return HttpStatus(HttpStatus::LLCORE, HE_OPT_NOT_DYNAMIC);
+ }
+
+ mReqOption = opt;
+ mReqClass = pclass;
+ mReqDoSet = true;
+ mReqLongValue = value;
+
+ return status;
}
-void HttpOpSetGet::stageFromRequest(HttpService * service)
+HttpStatus HttpOpSetGet::setupSet(HttpRequest::EPolicyOption opt, HttpRequest::policy_t pclass, const std::string & value)
{
- HttpPolicyGlobal & pol_opt(service->getPolicy().getGlobalOptions());
- HttpRequest::EGlobalPolicy setting(static_cast<HttpRequest::EGlobalPolicy>(mSetting));
+ HttpStatus status;
+
+ if (HttpService::sOptionDesc[opt].mIsLong)
+ {
+ return HttpStatus(HttpStatus::LLCORE, HE_INVALID_ARG);
+ }
+ if (! HttpService::sOptionDesc[opt].mIsDynamic)
+ {
+ return HttpStatus(HttpStatus::LLCORE, HE_OPT_NOT_DYNAMIC);
+ }
+
+ mReqOption = opt;
+ mReqClass = pclass;
+ mReqDoSet = true;
+ mReqStrValue = value;
- if (mDoSet)
+ return status;
+}
+
+
+void HttpOpSetGet::stageFromRequest(HttpService * service)
+{
+ if (mReqDoSet)
{
- mStatus = pol_opt.set(setting, mStrValue);
+ if (HttpService::sOptionDesc[mReqOption].mIsLong)
+ {
+ mStatus = service->setPolicyOption(mReqOption, mReqClass,
+ mReqLongValue, &mReplyLongValue);
+ }
+ else
+ {
+ mStatus = service->setPolicyOption(mReqOption, mReqClass,
+ mReqStrValue, &mReplyStrValue);
+ }
}
- if (mStatus)
+ else
{
- const std::string * value(NULL);
- if ((mStatus = pol_opt.get(setting, &value)))
+ if (HttpService::sOptionDesc[mReqOption].mIsLong)
+ {
+ mStatus = service->getPolicyOption(mReqOption, mReqClass, &mReplyLongValue);
+ }
+ else
{
- mStrValue = *value;
+ mStatus = service->getPolicyOption(mReqOption, mReqClass, &mReplyStrValue);
}
}
diff --git a/indra/llcorehttp/_httpopsetget.h b/indra/llcorehttp/_httpopsetget.h
index 6966b9d94e..a1e76dd429 100755
--- a/indra/llcorehttp/_httpopsetget.h
+++ b/indra/llcorehttp/_httpopsetget.h
@@ -4,7 +4,7 @@
*
* $LicenseInfo:firstyear=2012&license=viewerlgpl$
* Second Life Viewer Source Code
- * Copyright (C) 2012, Linden Research, Inc.
+ * Copyright (C) 2012-2013, Linden Research, Inc.
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
@@ -46,7 +46,10 @@ namespace LLCore
/// configuration settings.
///
/// *NOTE: Expect this to change. Don't really like it yet.
-
+///
+/// *TODO: Can't return values to caller yet. Need to do
+/// something better with HttpResponse and visitNotifier().
+///
class HttpOpSetGet : public HttpOperation
{
public:
@@ -61,19 +64,23 @@ private:
public:
/// Threading: called by application thread
- void setupGet(HttpRequest::EGlobalPolicy setting);
- void setupSet(HttpRequest::EGlobalPolicy setting, const std::string & value);
+ HttpStatus setupGet(HttpRequest::EPolicyOption opt, HttpRequest::policy_t pclass);
+ HttpStatus setupSet(HttpRequest::EPolicyOption opt, HttpRequest::policy_t pclass, long value);
+ HttpStatus setupSet(HttpRequest::EPolicyOption opt, HttpRequest::policy_t pclass, const std::string & value);
virtual void stageFromRequest(HttpService *);
public:
// Request data
- bool mIsGlobal;
- bool mDoSet;
- int mSetting;
- long mLongValue;
- std::string mStrValue;
-
+ HttpRequest::EPolicyOption mReqOption;
+ HttpRequest::policy_t mReqClass;
+ bool mReqDoSet;
+ long mReqLongValue;
+ std::string mReqStrValue;
+
+ // Reply Data
+ long mReplyLongValue;
+ std::string mReplyStrValue;
}; // end class HttpOpSetGet
diff --git a/indra/llcorehttp/_httppolicy.cpp b/indra/llcorehttp/_httppolicy.cpp
index 014bd37e2e..fd5a93e192 100755
--- a/indra/llcorehttp/_httppolicy.cpp
+++ b/indra/llcorehttp/_httppolicy.cpp
@@ -41,57 +41,70 @@ namespace LLCore
// Per-policy-class data for a running system.
-// Collection of queues, parameters, history, metrics, etc.
+// Collection of queues, options and other data
// for a single policy class.
//
// Threading: accessed only by worker thread
-struct HttpPolicy::State
+struct HttpPolicy::ClassState
{
public:
- State()
- : mConnMax(HTTP_CONNECTION_LIMIT_DEFAULT),
- mConnAt(HTTP_CONNECTION_LIMIT_DEFAULT),
- mConnMin(1),
- mNextSample(0),
- mErrorCount(0),
- mErrorFactor(0)
+ ClassState()
+ : mThrottleEnd(0),
+ mThrottleLeft(0L),
+ mRequestCount(0L)
{}
HttpReadyQueue mReadyQueue;
HttpRetryQueue mRetryQueue;
HttpPolicyClass mOptions;
-
- long mConnMax;
- long mConnAt;
- long mConnMin;
-
- HttpTime mNextSample;
- unsigned long mErrorCount;
- unsigned long mErrorFactor;
+ HttpTime mThrottleEnd;
+ long mThrottleLeft;
+ long mRequestCount;
};
HttpPolicy::HttpPolicy(HttpService * service)
- : mActiveClasses(0),
- mState(NULL),
- mService(service)
-{}
+ : mService(service)
+{
+ // Create default class
+ mClasses.push_back(new ClassState());
+}
HttpPolicy::~HttpPolicy()
{
shutdown();
+
+ for (class_list_t::iterator it(mClasses.begin()); it != mClasses.end(); ++it)
+ {
+ delete (*it);
+ }
+ mClasses.clear();
mService = NULL;
}
+HttpRequest::policy_t HttpPolicy::createPolicyClass()
+{
+ const HttpRequest::policy_t policy_class(mClasses.size());
+ if (policy_class >= HTTP_POLICY_CLASS_LIMIT)
+ {
+ return HttpRequest::INVALID_POLICY_ID;
+ }
+ mClasses.push_back(new ClassState());
+ return policy_class;
+}
+
+
void HttpPolicy::shutdown()
{
- for (int policy_class(0); policy_class < mActiveClasses; ++policy_class)
+ for (int policy_class(0); policy_class < mClasses.size(); ++policy_class)
{
- HttpRetryQueue & retryq(mState[policy_class].mRetryQueue);
+ ClassState & state(*mClasses[policy_class]);
+
+ HttpRetryQueue & retryq(state.mRetryQueue);
while (! retryq.empty())
{
HttpOpRequest * op(retryq.top());
@@ -101,7 +114,7 @@ void HttpPolicy::shutdown()
op->release();
}
- HttpReadyQueue & readyq(mState[policy_class].mReadyQueue);
+ HttpReadyQueue & readyq(state.mReadyQueue);
while (! readyq.empty())
{
HttpOpRequest * op(readyq.top());
@@ -111,28 +124,11 @@ void HttpPolicy::shutdown()
op->release();
}
}
- delete [] mState;
- mState = NULL;
- mActiveClasses = 0;
}
-void HttpPolicy::start(const HttpPolicyGlobal & global,
- const std::vector<HttpPolicyClass> & classes)
-{
- llassert_always(! mState);
-
- mGlobalOptions = global;
- mActiveClasses = classes.size();
- mState = new State [mActiveClasses];
- for (int i(0); i < mActiveClasses; ++i)
- {
- mState[i].mOptions = classes[i];
- mState[i].mConnMax = classes[i].mConnectionLimit;
- mState[i].mConnAt = mState[i].mConnMax;
- mState[i].mConnMin = 2;
- }
-}
+void HttpPolicy::start()
+{}
void HttpPolicy::addOp(HttpOpRequest * op)
@@ -140,7 +136,8 @@ void HttpPolicy::addOp(HttpOpRequest * op)
const int policy_class(op->mReqPolicy);
op->mPolicyRetries = 0;
- mState[policy_class].mReadyQueue.push(op);
+ op->mPolicy503Retries = 0;
+ mClasses[policy_class]->mReadyQueue.push(op);
}
@@ -155,25 +152,39 @@ void HttpPolicy::retryOp(HttpOpRequest * op)
5000000 // ... to every 5.0 S.
};
static const int delta_max(int(LL_ARRAY_SIZE(retry_deltas)) - 1);
-
+ static const HttpStatus error_503(503);
+
const HttpTime now(totalTime());
const int policy_class(op->mReqPolicy);
-
- const HttpTime delta(retry_deltas[llclamp(op->mPolicyRetries, 0, delta_max)]);
+ HttpTime delta(retry_deltas[llclamp(op->mPolicyRetries, 0, delta_max)]);
+ bool external_delta(false);
+
+ if (op->mReplyRetryAfter > 0 && op->mReplyRetryAfter < 30)
+ {
+ delta = op->mReplyRetryAfter * U64L(1000000);
+ external_delta = true;
+ }
op->mPolicyRetryAt = now + delta;
++op->mPolicyRetries;
- LL_WARNS("CoreHttp") << "HTTP request " << static_cast<HttpHandle>(op)
- << " retry " << op->mPolicyRetries
- << " scheduled for +" << (delta / HttpTime(1000))
- << " mS. Status: " << op->mStatus.toHex()
- << LL_ENDL;
- if (op->mTracing > 0)
+ if (error_503 == op->mStatus)
+ {
+ ++op->mPolicy503Retries;
+ }
+ LL_DEBUGS("CoreHttp") << "HTTP request " << static_cast<HttpHandle>(op)
+ << " retry " << op->mPolicyRetries
+ << " scheduled in " << (delta / HttpTime(1000))
+ << " mS (" << (external_delta ? "external" : "internal")
+ << "). Status: " << op->mStatus.toTerseString()
+ << LL_ENDL;
+ if (op->mTracing > HTTP_TRACE_OFF)
{
LL_INFOS("CoreHttp") << "TRACE, ToRetryQueue, Handle: "
<< static_cast<HttpHandle>(op)
+ << ", Delta: " << (delta / HttpTime(1000))
+ << ", Retries: " << op->mPolicyRetries
<< LL_ENDL;
}
- mState[policy_class].mRetryQueue.push(op);
+ mClasses[policy_class]->mRetryQueue.push(op);
}
@@ -188,21 +199,43 @@ void HttpPolicy::retryOp(HttpOpRequest * op)
// the worker thread may sleep hard otherwise will ask for
// normal polling frequency.
//
+// Implements a client-side request rate throttle as well.
+// This is intended to mimic and predict throttling behavior
+// of grid services but that is difficult to do with different
+// time bases. This also represents a rigid coupling between
+// viewer and server that makes it hard to change parameters
+// and I hope we can make this go away with pipelining.
+//
HttpService::ELoopSpeed HttpPolicy::processReadyQueue()
{
const HttpTime now(totalTime());
HttpService::ELoopSpeed result(HttpService::REQUEST_SLEEP);
HttpLibcurl & transport(mService->getTransport());
- for (int policy_class(0); policy_class < mActiveClasses; ++policy_class)
+ for (int policy_class(0); policy_class < mClasses.size(); ++policy_class)
{
- State & state(mState[policy_class]);
- int active(transport.getActiveCountInClass(policy_class));
- int needed(state.mConnAt - active); // Expect negatives here
-
+ ClassState & state(*mClasses[policy_class]);
HttpRetryQueue & retryq(state.mRetryQueue);
HttpReadyQueue & readyq(state.mReadyQueue);
+
+ if (retryq.empty() && readyq.empty())
+ {
+ continue;
+ }
+ const bool throttle_enabled(state.mOptions.mThrottleRate > 0L);
+ const bool throttle_current(throttle_enabled && now < state.mThrottleEnd);
+
+ if (throttle_current && state.mThrottleLeft <= 0)
+ {
+ // Throttled condition, don't serve this class but don't sleep hard.
+ result = HttpService::NORMAL;
+ continue;
+ }
+
+ int active(transport.getActiveCountInClass(policy_class));
+ int needed(state.mOptions.mConnectionLimit - active); // Expect negatives here
+
if (needed > 0)
{
// First see if we have any retries...
@@ -216,10 +249,27 @@ HttpService::ELoopSpeed HttpPolicy::processReadyQueue()
op->stageFromReady(mService);
op->release();
-
+
+ ++state.mRequestCount;
--needed;
+ if (throttle_enabled)
+ {
+ if (now >= state.mThrottleEnd)
+ {
+ // Throttle expired, move to next window
+ LL_DEBUGS("CoreHttp") << "Throttle expired with " << state.mThrottleLeft
+ << " requests to go and " << state.mRequestCount
+ << " requests issued." << LL_ENDL;
+ state.mThrottleLeft = state.mOptions.mThrottleRate;
+ state.mThrottleEnd = now + HttpTime(1000000);
+ }
+ if (--state.mThrottleLeft <= 0)
+ {
+ goto throttle_on;
+ }
+ }
}
-
+
// Now go on to the new requests...
while (needed > 0 && ! readyq.empty())
{
@@ -229,10 +279,29 @@ HttpService::ELoopSpeed HttpPolicy::processReadyQueue()
op->stageFromReady(mService);
op->release();
+ ++state.mRequestCount;
--needed;
+ if (throttle_enabled)
+ {
+ if (now >= state.mThrottleEnd)
+ {
+ // Throttle expired, move to next window
+ LL_DEBUGS("CoreHttp") << "Throttle expired with " << state.mThrottleLeft
+ << " requests to go and " << state.mRequestCount
+ << " requests issued." << LL_ENDL;
+ state.mThrottleLeft = state.mOptions.mThrottleRate;
+ state.mThrottleEnd = now + HttpTime(1000000);
+ }
+ if (--state.mThrottleLeft <= 0)
+ {
+ goto throttle_on;
+ }
+ }
}
}
-
+
+ throttle_on:
+
if (! readyq.empty() || ! retryq.empty())
{
// If anything is ready, continue looping...
@@ -246,9 +315,9 @@ HttpService::ELoopSpeed HttpPolicy::processReadyQueue()
bool HttpPolicy::changePriority(HttpHandle handle, HttpRequest::priority_t priority)
{
- for (int policy_class(0); policy_class < mActiveClasses; ++policy_class)
+ for (int policy_class(0); policy_class < mClasses.size(); ++policy_class)
{
- State & state(mState[policy_class]);
+ ClassState & state(*mClasses[policy_class]);
// We don't scan retry queue because a priority change there
// is meaningless. The request will be issued based on retry
// intervals not priority value, which is now moot.
@@ -276,9 +345,9 @@ bool HttpPolicy::changePriority(HttpHandle handle, HttpRequest::priority_t prior
bool HttpPolicy::cancel(HttpHandle handle)
{
- for (int policy_class(0); policy_class < mActiveClasses; ++policy_class)
+ for (int policy_class(0); policy_class < mClasses.size(); ++policy_class)
{
- State & state(mState[policy_class]);
+ ClassState & state(*mClasses[policy_class]);
// Scan retry queue
HttpRetryQueue::container_type & c1(state.mRetryQueue.get_container());
@@ -337,14 +406,14 @@ bool HttpPolicy::stageAfterCompletion(HttpOpRequest * op)
LL_WARNS("CoreHttp") << "HTTP request " << static_cast<HttpHandle>(op)
<< " failed after " << op->mPolicyRetries
<< " retries. Reason: " << op->mStatus.toString()
- << " (" << op->mStatus.toHex() << ")"
+ << " (" << op->mStatus.toTerseString() << ")"
<< LL_ENDL;
}
else if (op->mPolicyRetries)
{
- LL_WARNS("CoreHttp") << "HTTP request " << static_cast<HttpHandle>(op)
- << " succeeded on retry " << op->mPolicyRetries << "."
- << LL_ENDL;
+ LL_DEBUGS("CoreHttp") << "HTTP request " << static_cast<HttpHandle>(op)
+ << " succeeded on retry " << op->mPolicyRetries << "."
+ << LL_ENDL;
}
op->stageFromActive(mService);
@@ -352,13 +421,21 @@ bool HttpPolicy::stageAfterCompletion(HttpOpRequest * op)
return false; // not active
}
+
+HttpPolicyClass & HttpPolicy::getClassOptions(HttpRequest::policy_t pclass)
+{
+ llassert_always(pclass >= 0 && pclass < mClasses.size());
+
+ return mClasses[pclass]->mOptions;
+}
+
int HttpPolicy::getReadyCount(HttpRequest::policy_t policy_class) const
{
- if (policy_class < mActiveClasses)
+ if (policy_class < mClasses.size())
{
- return (mState[policy_class].mReadyQueue.size()
- + mState[policy_class].mRetryQueue.size());
+ return (mClasses[policy_class]->mReadyQueue.size()
+ + mClasses[policy_class]->mRetryQueue.size());
}
return 0;
}
diff --git a/indra/llcorehttp/_httppolicy.h b/indra/llcorehttp/_httppolicy.h
index 03d92c0b8e..bf1aa74267 100755
--- a/indra/llcorehttp/_httppolicy.h
+++ b/indra/llcorehttp/_httppolicy.h
@@ -4,7 +4,7 @@
*
* $LicenseInfo:firstyear=2012&license=viewerlgpl$
* Second Life Viewer Source Code
- * Copyright (C) 2012, Linden Research, Inc.
+ * Copyright (C) 2012-2013, Linden Research, Inc.
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
@@ -60,6 +60,9 @@ private:
void operator=(const HttpPolicy &); // Not defined
public:
+ /// Threading: called by init thread.
+ HttpRequest::policy_t createPolicyClass();
+
/// Cancel all ready and retry requests sending them to
/// their notification queues. Release state resources
/// making further request handling impossible.
@@ -71,9 +74,8 @@ public:
/// requests. One-time call invoked before starting
/// the worker thread.
///
- /// Threading: called by application thread
- void start(const HttpPolicyGlobal & global,
- const std::vector<HttpPolicyClass> & classes);
+ /// Threading: called by init thread
+ void start();
/// Give the policy layer some cycles to scan the ready
/// queue promoting higher-priority requests to active
@@ -93,7 +95,7 @@ public:
/// and should not be modified by anyone until retrieved
/// from queue.
///
- /// Threading: called by any thread
+ /// Threading: called by worker thread
void addOp(HttpOpRequest *);
/// Similar to addOp, used when a caller wants to retry a
@@ -130,30 +132,39 @@ public:
/// Threading: called by worker thread
bool stageAfterCompletion(HttpOpRequest * op);
- // Get pointer to global policy options. Caller is expected
- // to do context checks like no setting once running.
+ /// Get a reference to global policy options. Caller is expected
+ /// to do context checks like no setting once running. These
+ /// are done, for example, in @see HttpService interfaces.
///
/// Threading: called by any thread *but* the object may
/// only be modified by the worker thread once running.
- ///
HttpPolicyGlobal & getGlobalOptions()
{
return mGlobalOptions;
}
+ /// Get a reference to class policy options. Caller is expected
+ /// to do context checks like no setting once running. These
+ /// are done, for example, in @see HttpService interfaces.
+ ///
+ /// Threading: called by any thread *but* the object may
+ /// only be modified by the worker thread once running and
+ /// read accesses by other threads are exposed to races at
+ /// that point.
+ HttpPolicyClass & getClassOptions(HttpRequest::policy_t pclass);
+
/// Get ready counts for a particular policy class
///
/// Threading: called by worker thread
int getReadyCount(HttpRequest::policy_t policy_class) const;
protected:
- struct State;
-
- int mActiveClasses;
- State * mState;
- HttpService * mService; // Naked pointer, not refcounted, not owner
- HttpPolicyGlobal mGlobalOptions;
+ struct ClassState;
+ typedef std::vector<ClassState *> class_list_t;
+ HttpPolicyGlobal mGlobalOptions;
+ class_list_t mClasses;
+ HttpService * mService; // Naked pointer, not refcounted, not owner
}; // end class HttpPolicy
} // end namespace LLCore
diff --git a/indra/llcorehttp/_httppolicyclass.cpp b/indra/llcorehttp/_httppolicyclass.cpp
index a23b81322c..f34a8e9f1e 100755
--- a/indra/llcorehttp/_httppolicyclass.cpp
+++ b/indra/llcorehttp/_httppolicyclass.cpp
@@ -4,7 +4,7 @@
*
* $LicenseInfo:firstyear=2012&license=viewerlgpl$
* Second Life Viewer Source Code
- * Copyright (C) 2012, Linden Research, Inc.
+ * Copyright (C) 2012-2013, Linden Research, Inc.
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
@@ -34,10 +34,10 @@ namespace LLCore
HttpPolicyClass::HttpPolicyClass()
- : mSetMask(0UL),
- mConnectionLimit(HTTP_CONNECTION_LIMIT_DEFAULT),
+ : mConnectionLimit(HTTP_CONNECTION_LIMIT_DEFAULT),
mPerHostConnectionLimit(HTTP_CONNECTION_LIMIT_DEFAULT),
- mPipelining(0)
+ mPipelining(HTTP_PIPELINING_DEFAULT),
+ mThrottleRate(HTTP_THROTTLE_RATE_DEFAULT)
{}
@@ -49,75 +49,75 @@ HttpPolicyClass & HttpPolicyClass::operator=(const HttpPolicyClass & other)
{
if (this != &other)
{
- mSetMask = other.mSetMask;
mConnectionLimit = other.mConnectionLimit;
mPerHostConnectionLimit = other.mPerHostConnectionLimit;
mPipelining = other.mPipelining;
+ mThrottleRate = other.mThrottleRate;
}
return *this;
}
HttpPolicyClass::HttpPolicyClass(const HttpPolicyClass & other)
- : mSetMask(other.mSetMask),
- mConnectionLimit(other.mConnectionLimit),
+ : mConnectionLimit(other.mConnectionLimit),
mPerHostConnectionLimit(other.mPerHostConnectionLimit),
- mPipelining(other.mPipelining)
+ mPipelining(other.mPipelining),
+ mThrottleRate(other.mThrottleRate)
{}
-HttpStatus HttpPolicyClass::set(HttpRequest::EClassPolicy opt, long value)
+HttpStatus HttpPolicyClass::set(HttpRequest::EPolicyOption opt, long value)
{
switch (opt)
{
- case HttpRequest::CP_CONNECTION_LIMIT:
+ case HttpRequest::PO_CONNECTION_LIMIT:
mConnectionLimit = llclamp(value, long(HTTP_CONNECTION_LIMIT_MIN), long(HTTP_CONNECTION_LIMIT_MAX));
break;
- case HttpRequest::CP_PER_HOST_CONNECTION_LIMIT:
+ case HttpRequest::PO_PER_HOST_CONNECTION_LIMIT:
mPerHostConnectionLimit = llclamp(value, long(HTTP_CONNECTION_LIMIT_MIN), mConnectionLimit);
break;
- case HttpRequest::CP_ENABLE_PIPELINING:
+ case HttpRequest::PO_ENABLE_PIPELINING:
mPipelining = llclamp(value, 0L, 1L);
break;
+ case HttpRequest::PO_THROTTLE_RATE:
+ mThrottleRate = llclamp(value, 0L, 1000000L);
+ break;
+
default:
return HttpStatus(HttpStatus::LLCORE, HE_INVALID_ARG);
}
- mSetMask |= 1UL << int(opt);
return HttpStatus();
}
-HttpStatus HttpPolicyClass::get(HttpRequest::EClassPolicy opt, long * value)
+HttpStatus HttpPolicyClass::get(HttpRequest::EPolicyOption opt, long * value) const
{
- static const HttpStatus not_set(HttpStatus::LLCORE, HE_OPT_NOT_SET);
- long * src(NULL);
-
switch (opt)
{
- case HttpRequest::CP_CONNECTION_LIMIT:
- src = &mConnectionLimit;
+ case HttpRequest::PO_CONNECTION_LIMIT:
+ *value = mConnectionLimit;
break;
- case HttpRequest::CP_PER_HOST_CONNECTION_LIMIT:
- src = &mPerHostConnectionLimit;
+ case HttpRequest::PO_PER_HOST_CONNECTION_LIMIT:
+ *value = mPerHostConnectionLimit;
break;
- case HttpRequest::CP_ENABLE_PIPELINING:
- src = &mPipelining;
+ case HttpRequest::PO_ENABLE_PIPELINING:
+ *value = mPipelining;
+ break;
+
+ case HttpRequest::PO_THROTTLE_RATE:
+ *value = mThrottleRate;
break;
default:
return HttpStatus(HttpStatus::LLCORE, HE_INVALID_ARG);
}
- if (! (mSetMask & (1UL << int(opt))))
- return not_set;
-
- *value = *src;
return HttpStatus();
}
diff --git a/indra/llcorehttp/_httppolicyclass.h b/indra/llcorehttp/_httppolicyclass.h
index d175413cbd..38f1194ded 100755
--- a/indra/llcorehttp/_httppolicyclass.h
+++ b/indra/llcorehttp/_httppolicyclass.h
@@ -4,7 +4,7 @@
*
* $LicenseInfo:firstyear=2012&license=viewerlgpl$
* Second Life Viewer Source Code
- * Copyright (C) 2012, Linden Research, Inc.
+ * Copyright (C) 2012-2013, Linden Research, Inc.
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
@@ -34,6 +34,18 @@
namespace LLCore
{
+/// Options struct for per-class policy options.
+///
+/// Combines both raw blob data access with semantics-enforcing
+/// set/get interfaces. For internal operations by the worker
+/// thread, just grab the setting directly from instance and test/use
+/// as needed. When attached to external APIs (the public API
+/// options interfaces) the set/get methods are available to
+/// enforce correct ranges, data types, contexts, etc. and suitable
+/// status values are returned.
+///
+/// Threading: Single-threaded. In practice, init thread before
+/// worker starts, worker thread after.
class HttpPolicyClass
{
public:
@@ -44,14 +56,14 @@ public:
HttpPolicyClass(const HttpPolicyClass &); // Not defined
public:
- HttpStatus set(HttpRequest::EClassPolicy opt, long value);
- HttpStatus get(HttpRequest::EClassPolicy opt, long * value);
+ HttpStatus set(HttpRequest::EPolicyOption opt, long value);
+ HttpStatus get(HttpRequest::EPolicyOption opt, long * value) const;
public:
- unsigned long mSetMask;
long mConnectionLimit;
long mPerHostConnectionLimit;
long mPipelining;
+ long mThrottleRate;
}; // end class HttpPolicyClass
} // end namespace LLCore
diff --git a/indra/llcorehttp/_httppolicyglobal.cpp b/indra/llcorehttp/_httppolicyglobal.cpp
index 72f409d3b1..1dc95f3dce 100755
--- a/indra/llcorehttp/_httppolicyglobal.cpp
+++ b/indra/llcorehttp/_httppolicyglobal.cpp
@@ -4,7 +4,7 @@
*
* $LicenseInfo:firstyear=2012&license=viewerlgpl$
* Second Life Viewer Source Code
- * Copyright (C) 2012, Linden Research, Inc.
+ * Copyright (C) 2012-2013, Linden Research, Inc.
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
@@ -34,8 +34,7 @@ namespace LLCore
HttpPolicyGlobal::HttpPolicyGlobal()
- : mSetMask(0UL),
- mConnectionLimit(HTTP_CONNECTION_LIMIT_DEFAULT),
+ : mConnectionLimit(HTTP_CONNECTION_LIMIT_DEFAULT),
mTrace(HTTP_TRACE_OFF),
mUseLLProxy(0)
{}
@@ -49,7 +48,6 @@ HttpPolicyGlobal & HttpPolicyGlobal::operator=(const HttpPolicyGlobal & other)
{
if (this != &other)
{
- mSetMask = other.mSetMask;
mConnectionLimit = other.mConnectionLimit;
mCAPath = other.mCAPath;
mCAFile = other.mCAFile;
@@ -61,19 +59,19 @@ HttpPolicyGlobal & HttpPolicyGlobal::operator=(const HttpPolicyGlobal & other)
}
-HttpStatus HttpPolicyGlobal::set(HttpRequest::EGlobalPolicy opt, long value)
+HttpStatus HttpPolicyGlobal::set(HttpRequest::EPolicyOption opt, long value)
{
switch (opt)
{
- case HttpRequest::GP_CONNECTION_LIMIT:
+ case HttpRequest::PO_CONNECTION_LIMIT:
mConnectionLimit = llclamp(value, long(HTTP_CONNECTION_LIMIT_MIN), long(HTTP_CONNECTION_LIMIT_MAX));
break;
- case HttpRequest::GP_TRACE:
+ case HttpRequest::PO_TRACE:
mTrace = llclamp(value, long(HTTP_TRACE_MIN), long(HTTP_TRACE_MAX));
break;
- case HttpRequest::GP_LLPROXY:
+ case HttpRequest::PO_LLPROXY:
mUseLLProxy = llclamp(value, 0L, 1L);
break;
@@ -81,24 +79,23 @@ HttpStatus HttpPolicyGlobal::set(HttpRequest::EGlobalPolicy opt, long value)
return HttpStatus(HttpStatus::LLCORE, HE_INVALID_ARG);
}
- mSetMask |= 1UL << int(opt);
return HttpStatus();
}
-HttpStatus HttpPolicyGlobal::set(HttpRequest::EGlobalPolicy opt, const std::string & value)
+HttpStatus HttpPolicyGlobal::set(HttpRequest::EPolicyOption opt, const std::string & value)
{
switch (opt)
{
- case HttpRequest::GP_CA_PATH:
+ case HttpRequest::PO_CA_PATH:
mCAPath = value;
break;
- case HttpRequest::GP_CA_FILE:
+ case HttpRequest::PO_CA_FILE:
mCAFile = value;
break;
- case HttpRequest::GP_HTTP_PROXY:
+ case HttpRequest::PO_HTTP_PROXY:
mCAFile = value;
break;
@@ -106,69 +103,54 @@ HttpStatus HttpPolicyGlobal::set(HttpRequest::EGlobalPolicy opt, const std::stri
return HttpStatus(HttpStatus::LLCORE, HE_INVALID_ARG);
}
- mSetMask |= 1UL << int(opt);
return HttpStatus();
}
-HttpStatus HttpPolicyGlobal::get(HttpRequest::EGlobalPolicy opt, long * value)
+HttpStatus HttpPolicyGlobal::get(HttpRequest::EPolicyOption opt, long * value) const
{
- static const HttpStatus not_set(HttpStatus::LLCORE, HE_OPT_NOT_SET);
- long * src(NULL);
-
switch (opt)
{
- case HttpRequest::GP_CONNECTION_LIMIT:
- src = &mConnectionLimit;
+ case HttpRequest::PO_CONNECTION_LIMIT:
+ *value = mConnectionLimit;
break;
- case HttpRequest::GP_TRACE:
- src = &mTrace;
+ case HttpRequest::PO_TRACE:
+ *value = mTrace;
break;
- case HttpRequest::GP_LLPROXY:
- src = &mUseLLProxy;
+ case HttpRequest::PO_LLPROXY:
+ *value = mUseLLProxy;
break;
default:
return HttpStatus(HttpStatus::LLCORE, HE_INVALID_ARG);
}
- if (! (mSetMask & (1UL << int(opt))))
- return not_set;
-
- *value = *src;
return HttpStatus();
}
-HttpStatus HttpPolicyGlobal::get(HttpRequest::EGlobalPolicy opt, const std::string ** value)
+HttpStatus HttpPolicyGlobal::get(HttpRequest::EPolicyOption opt, std::string * value) const
{
- static const HttpStatus not_set(HttpStatus::LLCORE, HE_OPT_NOT_SET);
- const std::string * src(NULL);
-
switch (opt)
{
- case HttpRequest::GP_CA_PATH:
- src = &mCAPath;
+ case HttpRequest::PO_CA_PATH:
+ *value = mCAPath;
break;
- case HttpRequest::GP_CA_FILE:
- src = &mCAFile;
+ case HttpRequest::PO_CA_FILE:
+ *value = mCAFile;
break;
- case HttpRequest::GP_HTTP_PROXY:
- src = &mHttpProxy;
+ case HttpRequest::PO_HTTP_PROXY:
+ *value = mHttpProxy;
break;
default:
return HttpStatus(HttpStatus::LLCORE, HE_INVALID_ARG);
}
- if (! (mSetMask & (1UL << int(opt))))
- return not_set;
-
- *value = src;
return HttpStatus();
}
diff --git a/indra/llcorehttp/_httppolicyglobal.h b/indra/llcorehttp/_httppolicyglobal.h
index a50d0e4188..67c4ba9481 100755
--- a/indra/llcorehttp/_httppolicyglobal.h
+++ b/indra/llcorehttp/_httppolicyglobal.h
@@ -4,7 +4,7 @@
*
* $LicenseInfo:firstyear=2012&license=viewerlgpl$
* Second Life Viewer Source Code
- * Copyright (C) 2012, Linden Research, Inc.
+ * Copyright (C) 2012-2013, Linden Research, Inc.
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
@@ -34,6 +34,18 @@
namespace LLCore
{
+/// Options struct for global policy options.
+///
+/// Combines both raw blob data access with semantics-enforcing
+/// set/get interfaces. For internal operations by the worker
+/// thread, just grab the setting directly from instance and test/use
+/// as needed. When attached to external APIs (the public API
+/// options interfaces) the set/get methods are available to
+/// enforce correct ranges, data types, contexts, etc. and suitable
+/// status values are returned.
+///
+/// Threading: Single-threaded. In practice, init thread before
+/// worker starts, worker thread after.
class HttpPolicyGlobal
{
public:
@@ -46,13 +58,12 @@ private:
HttpPolicyGlobal(const HttpPolicyGlobal &); // Not defined
public:
- HttpStatus set(HttpRequest::EGlobalPolicy opt, long value);
- HttpStatus set(HttpRequest::EGlobalPolicy opt, const std::string & value);
- HttpStatus get(HttpRequest::EGlobalPolicy opt, long * value);
- HttpStatus get(HttpRequest::EGlobalPolicy opt, const std::string ** value);
+ HttpStatus set(HttpRequest::EPolicyOption opt, long value);
+ HttpStatus set(HttpRequest::EPolicyOption opt, const std::string & value);
+ HttpStatus get(HttpRequest::EPolicyOption opt, long * value) const;
+ HttpStatus get(HttpRequest::EPolicyOption opt, std::string * value) const;
public:
- unsigned long mSetMask;
long mConnectionLimit;
std::string mCAPath;
std::string mCAFile;
diff --git a/indra/llcorehttp/_httpservice.cpp b/indra/llcorehttp/_httpservice.cpp
index 0825888d0f..c94249dc2d 100755
--- a/indra/llcorehttp/_httpservice.cpp
+++ b/indra/llcorehttp/_httpservice.cpp
@@ -4,7 +4,7 @@
*
* $LicenseInfo:firstyear=2012&license=viewerlgpl$
* Second Life Viewer Source Code
- * Copyright (C) 2012, Linden Research, Inc.
+ * Copyright (C) 2012-2013, Linden Research, Inc.
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
@@ -43,6 +43,18 @@
namespace LLCore
{
+const HttpService::OptionDescriptor HttpService::sOptionDesc[] =
+{ // isLong isDynamic isGlobal isClass
+ { true, true, true, true }, // PO_CONNECTION_LIMIT
+ { true, true, false, true }, // PO_PER_HOST_CONNECTION_LIMIT
+ { false, false, true, false }, // PO_CA_PATH
+ { false, false, true, false }, // PO_CA_FILE
+ { false, true, true, false }, // PO_HTTP_PROXY
+ { true, true, true, false }, // PO_LLPROXY
+ { true, true, true, false }, // PO_TRACE
+ { true, true, false, true }, // PO_ENABLE_PIPELINING
+ { true, true, false, true } // PO_THROTTLE_RATE
+};
HttpService * HttpService::sInstance(NULL);
volatile HttpService::EState HttpService::sState(NOT_INITIALIZED);
@@ -51,15 +63,9 @@ HttpService::HttpService()
mExitRequested(0U),
mThread(NULL),
mPolicy(NULL),
- mTransport(NULL)
-{
- // Create the default policy class
- HttpPolicyClass pol_class;
- pol_class.set(HttpRequest::CP_CONNECTION_LIMIT, HTTP_CONNECTION_LIMIT_DEFAULT);
- pol_class.set(HttpRequest::CP_PER_HOST_CONNECTION_LIMIT, HTTP_CONNECTION_LIMIT_DEFAULT);
- pol_class.set(HttpRequest::CP_ENABLE_PIPELINING, 0L);
- mPolicyClasses.push_back(pol_class);
-}
+ mTransport(NULL),
+ mLastPolicy(0)
+{}
HttpService::~HttpService()
@@ -149,13 +155,8 @@ void HttpService::term()
HttpRequest::policy_t HttpService::createPolicyClass()
{
- const HttpRequest::policy_t policy_class(mPolicyClasses.size());
- if (policy_class >= HTTP_POLICY_CLASS_LIMIT)
- {
- return 0;
- }
- mPolicyClasses.push_back(HttpPolicyClass());
- return policy_class;
+ mLastPolicy = mPolicy->createPolicyClass();
+ return mLastPolicy;
}
@@ -188,8 +189,8 @@ void HttpService::startThread()
}
// Push current policy definitions, enable policy & transport components
- mPolicy->start(mPolicyGlobal, mPolicyClasses);
- mTransport->start(mPolicyClasses.size());
+ mPolicy->start();
+ mTransport->start(mLastPolicy + 1);
mThread = new LLCoreInt::HttpThread(boost::bind(&HttpService::threadRun, this, _1));
sState = RUNNING;
@@ -322,7 +323,7 @@ HttpService::ELoopSpeed HttpService::processRequestQueue(ELoopSpeed loop)
{
// Setup for subsequent tracing
long tracing(HTTP_TRACE_OFF);
- mPolicy->getGlobalOptions().get(HttpRequest::GP_TRACE, &tracing);
+ mPolicy->getGlobalOptions().get(HttpRequest::PO_TRACE, &tracing);
op->mTracing = (std::max)(op->mTracing, int(tracing));
if (op->mTracing > HTTP_TRACE_OFF)
@@ -345,4 +346,137 @@ HttpService::ELoopSpeed HttpService::processRequestQueue(ELoopSpeed loop)
}
+HttpStatus HttpService::getPolicyOption(HttpRequest::EPolicyOption opt, HttpRequest::policy_t pclass,
+ long * ret_value)
+{
+ if (opt < HttpRequest::PO_CONNECTION_LIMIT // option must be in range
+ || opt >= HttpRequest::PO_LAST // ditto
+ || (! sOptionDesc[opt].mIsLong) // datatype is long
+ || (pclass != HttpRequest::GLOBAL_POLICY_ID && pclass > mLastPolicy) // pclass in valid range
+ || (pclass == HttpRequest::GLOBAL_POLICY_ID && ! sOptionDesc[opt].mIsGlobal) // global setting permitted
+ || (pclass != HttpRequest::GLOBAL_POLICY_ID && ! sOptionDesc[opt].mIsClass)) // class setting permitted
+ // can always get, no dynamic check
+ {
+ return HttpStatus(HttpStatus::LLCORE, LLCore::HE_INVALID_ARG);
+ }
+
+ HttpStatus status;
+ if (pclass == HttpRequest::GLOBAL_POLICY_ID)
+ {
+ HttpPolicyGlobal & opts(mPolicy->getGlobalOptions());
+
+ status = opts.get(opt, ret_value);
+ }
+ else
+ {
+ HttpPolicyClass & opts(mPolicy->getClassOptions(pclass));
+
+ status = opts.get(opt, ret_value);
+ }
+
+ return status;
+}
+
+
+HttpStatus HttpService::getPolicyOption(HttpRequest::EPolicyOption opt, HttpRequest::policy_t pclass,
+ std::string * ret_value)
+{
+ HttpStatus status(HttpStatus::LLCORE, LLCore::HE_INVALID_ARG);
+
+ if (opt < HttpRequest::PO_CONNECTION_LIMIT // option must be in range
+ || opt >= HttpRequest::PO_LAST // ditto
+ || (sOptionDesc[opt].mIsLong) // datatype is string
+ || (pclass != HttpRequest::GLOBAL_POLICY_ID && pclass > mLastPolicy) // pclass in valid range
+ || (pclass == HttpRequest::GLOBAL_POLICY_ID && ! sOptionDesc[opt].mIsGlobal) // global setting permitted
+ || (pclass != HttpRequest::GLOBAL_POLICY_ID && ! sOptionDesc[opt].mIsClass)) // class setting permitted
+ // can always get, no dynamic check
+ {
+ return status;
+ }
+
+ // Only global has string values
+ if (pclass == HttpRequest::GLOBAL_POLICY_ID)
+ {
+ HttpPolicyGlobal & opts(mPolicy->getGlobalOptions());
+
+ status = opts.get(opt, ret_value);
+ }
+
+ return status;
+}
+
+
+HttpStatus HttpService::setPolicyOption(HttpRequest::EPolicyOption opt, HttpRequest::policy_t pclass,
+ long value, long * ret_value)
+{
+ HttpStatus status(HttpStatus::LLCORE, LLCore::HE_INVALID_ARG);
+
+ if (opt < HttpRequest::PO_CONNECTION_LIMIT // option must be in range
+ || opt >= HttpRequest::PO_LAST // ditto
+ || (! sOptionDesc[opt].mIsLong) // datatype is long
+ || (pclass != HttpRequest::GLOBAL_POLICY_ID && pclass > mLastPolicy) // pclass in valid range
+ || (pclass == HttpRequest::GLOBAL_POLICY_ID && ! sOptionDesc[opt].mIsGlobal) // global setting permitted
+ || (pclass != HttpRequest::GLOBAL_POLICY_ID && ! sOptionDesc[opt].mIsClass) // class setting permitted
+ || (RUNNING == sState && ! sOptionDesc[opt].mIsDynamic)) // dynamic setting permitted
+ {
+ return status;
+ }
+
+ if (pclass == HttpRequest::GLOBAL_POLICY_ID)
+ {
+ HttpPolicyGlobal & opts(mPolicy->getGlobalOptions());
+
+ status = opts.set(opt, value);
+ if (status && ret_value)
+ {
+ status = opts.get(opt, ret_value);
+ }
+ }
+ else
+ {
+ HttpPolicyClass & opts(mPolicy->getClassOptions(pclass));
+
+ status = opts.set(opt, value);
+ if (status && ret_value)
+ {
+ status = opts.get(opt, ret_value);
+ }
+ }
+
+ return status;
+}
+
+
+HttpStatus HttpService::setPolicyOption(HttpRequest::EPolicyOption opt, HttpRequest::policy_t pclass,
+ const std::string & value, std::string * ret_value)
+{
+ HttpStatus status(HttpStatus::LLCORE, LLCore::HE_INVALID_ARG);
+
+ if (opt < HttpRequest::PO_CONNECTION_LIMIT // option must be in range
+ || opt >= HttpRequest::PO_LAST // ditto
+ || (sOptionDesc[opt].mIsLong) // datatype is string
+ || (pclass != HttpRequest::GLOBAL_POLICY_ID && pclass > mLastPolicy) // pclass in valid range
+ || (pclass == HttpRequest::GLOBAL_POLICY_ID && ! sOptionDesc[opt].mIsGlobal) // global setting permitted
+ || (pclass != HttpRequest::GLOBAL_POLICY_ID && ! sOptionDesc[opt].mIsClass) // class setting permitted
+ || (RUNNING == sState && ! sOptionDesc[opt].mIsDynamic)) // dynamic setting permitted
+ {
+ return status;
+ }
+
+ // Only string values are global at this time
+ if (pclass == HttpRequest::GLOBAL_POLICY_ID)
+ {
+ HttpPolicyGlobal & opts(mPolicy->getGlobalOptions());
+
+ status = opts.set(opt, value);
+ if (status && ret_value)
+ {
+ status = opts.get(opt, ret_value);
+ }
+ }
+
+ return status;
+}
+
+
} // end namespace LLCore
diff --git a/indra/llcorehttp/_httpservice.h b/indra/llcorehttp/_httpservice.h
index ffe0349d4d..cf23f3ab61 100755
--- a/indra/llcorehttp/_httpservice.h
+++ b/indra/llcorehttp/_httpservice.h
@@ -4,7 +4,7 @@
*
* $LicenseInfo:firstyear=2012&license=viewerlgpl$
* Second Life Viewer Source Code
- * Copyright (C) 2012, Linden Research, Inc.
+ * Copyright (C) 2012-2013, Linden Research, Inc.
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
@@ -53,6 +53,7 @@ namespace LLCore
class HttpRequestQueue;
class HttpPolicy;
class HttpLibcurl;
+class HttpOpSetGet;
/// The HttpService class does the work behind the request queue. It
@@ -106,7 +107,7 @@ public:
NORMAL, ///< continuous polling of request, ready, active queues
REQUEST_SLEEP ///< can sleep indefinitely waiting for request queue write
};
-
+
static void init(HttpRequestQueue *);
static void term();
@@ -136,7 +137,7 @@ public:
/// acquires its weaknesses.
static bool isStopped();
- /// Threading: callable by consumer thread *once*.
+ /// Threading: callable by init thread *once*.
void startThread();
/// Threading: callable by worker thread.
@@ -181,27 +182,38 @@ public:
}
/// Threading: callable by consumer thread.
- HttpPolicyGlobal & getGlobalOptions()
- {
- return mPolicyGlobal;
- }
-
- /// Threading: callable by consumer thread.
HttpRequest::policy_t createPolicyClass();
- /// Threading: callable by consumer thread.
- HttpPolicyClass & getClassOptions(HttpRequest::policy_t policy_class)
- {
- llassert(policy_class >= 0 && policy_class < mPolicyClasses.size());
- return mPolicyClasses[policy_class];
- }
-
protected:
void threadRun(LLCoreInt::HttpThread * thread);
ELoopSpeed processRequestQueue(ELoopSpeed loop);
+
+protected:
+ friend class HttpOpSetGet;
+ friend class HttpRequest;
+
+ // Used internally to describe what operations are allowed
+ // on each policy option.
+ struct OptionDescriptor
+ {
+ bool mIsLong;
+ bool mIsDynamic;
+ bool mIsGlobal;
+ bool mIsClass;
+ };
+
+ HttpStatus getPolicyOption(HttpRequest::EPolicyOption opt, HttpRequest::policy_t,
+ long * ret_value);
+ HttpStatus getPolicyOption(HttpRequest::EPolicyOption opt, HttpRequest::policy_t,
+ std::string * ret_value);
+ HttpStatus setPolicyOption(HttpRequest::EPolicyOption opt, HttpRequest::policy_t,
+ long value, long * ret_value);
+ HttpStatus setPolicyOption(HttpRequest::EPolicyOption opt, HttpRequest::policy_t,
+ const std::string & value, std::string * ret_value);
protected:
+ static const OptionDescriptor sOptionDesc[HttpRequest::PO_LAST];
static HttpService * sInstance;
// === shared data ===
@@ -210,13 +222,13 @@ protected:
LLAtomicU32 mExitRequested;
LLCoreInt::HttpThread * mThread;
- // === consumer-thread-only data ===
- HttpPolicyGlobal mPolicyGlobal;
- std::vector<HttpPolicyClass> mPolicyClasses;
-
// === working-thread-only data ===
HttpPolicy * mPolicy; // Simple pointer, has ownership
HttpLibcurl * mTransport; // Simple pointer, has ownership
+
+ // === main-thread-only data ===
+ HttpRequest::policy_t mLastPolicy;
+
}; // end class HttpService
} // end namespace LLCore
diff --git a/indra/llcorehttp/examples/http_texture_load.cpp b/indra/llcorehttp/examples/http_texture_load.cpp
index 40ad4f047d..73c49687d7 100755
--- a/indra/llcorehttp/examples/http_texture_load.cpp
+++ b/indra/llcorehttp/examples/http_texture_load.cpp
@@ -4,7 +4,7 @@
*
* $LicenseInfo:firstyear=2012&license=viewerlgpl$
* Second Life Viewer Source Code
- * Copyright (C) 2012, Linden Research, Inc.
+ * Copyright (C) 2012-2013, Linden Research, Inc.
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
@@ -39,6 +39,7 @@
#include "httprequest.h"
#include "httphandler.h"
#include "httpresponse.h"
+#include "httpoptions.h"
#include "httpheaders.h"
#include "bufferarray.h"
#include "_mutex.h"
@@ -57,6 +58,7 @@ void usage(std::ostream & out);
// Default command line settings
static int concurrency_limit(40);
+static int highwater(100);
static char url_format[1024] = "http://example.com/some/path?texture_id=%s.texture";
#if defined(WIN32)
@@ -79,11 +81,11 @@ public:
WorkingSet();
~WorkingSet();
- bool reload(LLCore::HttpRequest *);
+ bool reload(LLCore::HttpRequest *, LLCore::HttpOptions *);
virtual void onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * response);
- void loadTextureUuids(FILE * in);
+ void loadAssetUuids(FILE * in);
public:
struct Spec
@@ -93,24 +95,27 @@ public:
int mLength;
};
typedef std::set<LLCore::HttpHandle> handle_set_t;
- typedef std::vector<Spec> texture_list_t;
+ typedef std::vector<Spec> asset_list_t;
public:
bool mVerbose;
bool mRandomRange;
- int mMaxConcurrency;
+ int mRequestLowWater;
+ int mRequestHighWater;
handle_set_t mHandles;
int mRemaining;
int mLimit;
int mAt;
std::string mUrl;
- texture_list_t mTextures;
+ asset_list_t mAssets;
int mErrorsApi;
int mErrorsHttp;
int mErrorsHttp404;
int mErrorsHttp416;
int mErrorsHttp500;
int mErrorsHttp503;
+ int mRetries;
+ int mRetriesHttp503;
int mSuccesses;
long mByteCount;
LLCore::HttpHeaders * mHeaders;
@@ -158,7 +163,7 @@ int main(int argc, char** argv)
bool do_verbose(false);
int option(-1);
- while (-1 != (option = getopt(argc, argv, "u:c:h?Rv")))
+ while (-1 != (option = getopt(argc, argv, "u:c:h?RvH:")))
{
switch (option)
{
@@ -182,6 +187,21 @@ int main(int argc, char** argv)
}
break;
+ case 'H':
+ {
+ unsigned long value;
+ char * end;
+
+ value = strtoul(optarg, &end, 10);
+ if (value < 1 || value > 100 || *end != '\0')
+ {
+ usage(std::cerr);
+ return 1;
+ }
+ highwater = value;
+ }
+ break;
+
case 'R':
do_random = true;
break;
@@ -216,25 +236,32 @@ int main(int argc, char** argv)
// Initialization
init_curl();
LLCore::HttpRequest::createService();
- LLCore::HttpRequest::setPolicyClassOption(LLCore::HttpRequest::DEFAULT_POLICY_ID,
- LLCore::HttpRequest::CP_CONNECTION_LIMIT,
- concurrency_limit);
+ LLCore::HttpRequest::setStaticPolicyOption(LLCore::HttpRequest::PO_CONNECTION_LIMIT,
+ LLCore::HttpRequest::DEFAULT_POLICY_ID,
+ concurrency_limit,
+ NULL);
LLCore::HttpRequest::startThread();
// Get service point
LLCore::HttpRequest * hr = new LLCore::HttpRequest();
+ // Get request options
+ LLCore::HttpOptions * opt = new LLCore::HttpOptions();
+ opt->setRetries(12);
+ opt->setUseRetryAfter(true);
+
// Get a handler/working set
WorkingSet ws;
// Fill the working set with work
ws.mUrl = url_format;
- ws.loadTextureUuids(uuids);
+ ws.loadAssetUuids(uuids);
ws.mRandomRange = do_random;
ws.mVerbose = do_verbose;
- ws.mMaxConcurrency = 100;
+ ws.mRequestHighWater = highwater;
+ ws.mRequestLowWater = ws.mRequestHighWater / 2;
- if (! ws.mTextures.size())
+ if (! ws.mAssets.size())
{
std::cerr << "No UUIDs found in file '" << argv[optind] << "'." << std::endl;
return 1;
@@ -246,9 +273,9 @@ int main(int argc, char** argv)
// Run it
int passes(0);
- while (! ws.reload(hr))
+ while (! ws.reload(hr, opt))
{
- hr->update(5000000);
+ hr->update(0);
ms_sleep(2);
if (0 == (++passes % 200))
{
@@ -265,6 +292,8 @@ int main(int argc, char** argv)
std::cout << "HTTP 404 errors: " << ws.mErrorsHttp404 << " HTTP 416 errors: " << ws.mErrorsHttp416
<< " HTTP 500 errors: " << ws.mErrorsHttp500 << " HTTP 503 errors: " << ws.mErrorsHttp503
<< std::endl;
+ std::cout << "Retries: " << ws.mRetries << " Retries on 503: " << ws.mRetriesHttp503
+ << std::endl;
std::cout << "User CPU: " << (metrics.mEndUTime - metrics.mStartUTime)
<< " uS System CPU: " << (metrics.mEndSTime - metrics.mStartSTime)
<< " uS Wall Time: " << (metrics.mEndWallTime - metrics.mStartWallTime)
@@ -275,6 +304,8 @@ int main(int argc, char** argv)
// Clean up
hr->requestStopThread(NULL);
ms_sleep(1000);
+ opt->release();
+ opt = NULL;
delete hr;
LLCore::HttpRequest::destroyService();
term_curl();
@@ -300,8 +331,10 @@ void usage(std::ostream & out)
" -u <url_format> printf-style format string for URL generation\n"
" Default: " << url_format << "\n"
" -R Issue GETs with random Range: headers\n"
- " -c <limit> Maximum request concurrency. Range: [1..100]\n"
+ " -c <limit> Maximum connection concurrency. Range: [1..100]\n"
" Default: " << concurrency_limit << "\n"
+ " -H <limit> HTTP request highwater (requests fed to llcorehttp).\n"
+ " Range: [1..100] Default: " << highwater << "\n"
" -v Verbose mode. Issue some chatter while running\n"
" -h print this help\n"
"\n"
@@ -322,13 +355,15 @@ WorkingSet::WorkingSet()
mErrorsHttp416(0),
mErrorsHttp500(0),
mErrorsHttp503(0),
+ mRetries(0),
+ mRetriesHttp503(0),
mSuccesses(0),
mByteCount(0L)
{
- mTextures.reserve(30000);
+ mAssets.reserve(30000);
mHeaders = new LLCore::HttpHeaders;
- mHeaders->mHeaders.push_back("Accept: image/x-j2c");
+ mHeaders->append("Accept", "image/x-j2c");
}
@@ -342,29 +377,35 @@ WorkingSet::~WorkingSet()
}
-bool WorkingSet::reload(LLCore::HttpRequest * hr)
+bool WorkingSet::reload(LLCore::HttpRequest * hr, LLCore::HttpOptions * opt)
{
- int to_do((std::min)(mRemaining, mMaxConcurrency - int(mHandles.size())));
+ if (mRequestLowWater <= mHandles.size())
+ {
+ // Haven't fallen below low-water level yet.
+ return false;
+ }
+
+ int to_do((std::min)(mRemaining, mRequestHighWater - int(mHandles.size())));
for (int i(0); i < to_do; ++i)
{
char buffer[1024];
#if defined(WIN32)
- _snprintf_s(buffer, sizeof(buffer), sizeof(buffer) - 1, mUrl.c_str(), mTextures[mAt].mUuid.c_str());
+ _snprintf_s(buffer, sizeof(buffer), sizeof(buffer) - 1, mUrl.c_str(), mAssets[mAt].mUuid.c_str());
#else
- snprintf(buffer, sizeof(buffer), mUrl.c_str(), mTextures[mAt].mUuid.c_str());
+ snprintf(buffer, sizeof(buffer), mUrl.c_str(), mAssets[mAt].mUuid.c_str());
#endif
- int offset(mRandomRange ? ((unsigned long) rand()) % 1000000UL : mTextures[mAt].mOffset);
- int length(mRandomRange ? ((unsigned long) rand()) % 1000000UL : mTextures[mAt].mLength);
+ int offset(mRandomRange ? ((unsigned long) rand()) % 1000000UL : mAssets[mAt].mOffset);
+ int length(mRandomRange ? ((unsigned long) rand()) % 1000000UL : mAssets[mAt].mLength);
LLCore::HttpHandle handle;
if (offset || length)
{
- handle = hr->requestGetByteRange(0, 0, buffer, offset, length, NULL, mHeaders, this);
+ handle = hr->requestGetByteRange(0, 0, buffer, offset, length, opt, mHeaders, this);
}
else
{
- handle = hr->requestGet(0, 0, buffer, NULL, mHeaders, this);
+ handle = hr->requestGet(0, 0, buffer, opt, mHeaders, this);
}
if (! handle)
{
@@ -410,7 +451,7 @@ void WorkingSet::onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * r
{
// More success
LLCore::BufferArray * data(response->getBody());
- mByteCount += data->size();
+ mByteCount += data ? data->size() : 0;
++mSuccesses;
}
else
@@ -446,6 +487,10 @@ void WorkingSet::onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * r
++mErrorsApi;
}
}
+ unsigned int retry(0U), retry_503(0U);
+ response->getRetries(&retry, &retry_503);
+ mRetries += int(retry);
+ mRetriesHttp503 += int(retry_503);
mHandles.erase(it);
}
@@ -459,21 +504,21 @@ void WorkingSet::onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * r
}
-void WorkingSet::loadTextureUuids(FILE * in)
+void WorkingSet::loadAssetUuids(FILE * in)
{
char buffer[1024];
while (fgets(buffer, sizeof(buffer), in))
{
- WorkingSet::Spec texture;
+ WorkingSet::Spec asset;
char * state(NULL);
char * token = strtok_r(buffer, " \t\n,", &state);
if (token && 36 == strlen(token))
{
// Close enough for this function
- texture.mUuid = token;
- texture.mOffset = 0;
- texture.mLength = 0;
+ asset.mUuid = token;
+ asset.mOffset = 0;
+ asset.mLength = 0;
token = strtok_r(buffer, " \t\n,", &state);
if (token)
{
@@ -482,14 +527,14 @@ void WorkingSet::loadTextureUuids(FILE * in)
if (token)
{
int length(atoi(token));
- texture.mOffset = offset;
- texture.mLength = length;
+ asset.mOffset = offset;
+ asset.mLength = length;
}
}
- mTextures.push_back(texture);
+ mAssets.push_back(asset);
}
}
- mRemaining = mLimit = mTextures.size();
+ mRemaining = mLimit = mAssets.size();
}
diff --git a/indra/llcorehttp/httpcommon.cpp b/indra/llcorehttp/httpcommon.cpp
index 0738760763..c2f15155ac 100755
--- a/indra/llcorehttp/httpcommon.cpp
+++ b/indra/llcorehttp/httpcommon.cpp
@@ -70,7 +70,8 @@ std::string HttpStatus::toString() const
"Invalid datatype for argument or option",
"Option has not been explicitly set",
"Option is not dynamic and must be set early",
- "Invalid HTTP status code received from server"
+ "Invalid HTTP status code received from server",
+ "Could not allocate required resource"
};
static const int llcore_errors_count(sizeof(llcore_errors) / sizeof(llcore_errors[0]));
@@ -177,6 +178,44 @@ std::string HttpStatus::toString() const
}
+std::string HttpStatus::toTerseString() const
+{
+ std::ostringstream result;
+
+ unsigned int error_value((unsigned short) mStatus);
+
+ switch (mType)
+ {
+ case EXT_CURL_EASY:
+ result << "Easy_";
+ break;
+
+ case EXT_CURL_MULTI:
+ result << "Multi_";
+ break;
+
+ case LLCORE:
+ result << "Core_";
+ break;
+
+ default:
+ if (isHttpStatus())
+ {
+ result << "Http_";
+ error_value = mType;
+ }
+ else
+ {
+ result << "Unknown_";
+ }
+ break;
+ }
+
+ result << error_value;
+ return result.str();
+}
+
+
// Pass true on statuses that might actually be cleared by a
// retry. Library failures, calling problems, etc. aren't
// going to be fixed by squirting bits all over the Net.
@@ -206,6 +245,5 @@ bool HttpStatus::isRetryable() const
*this == inv_cont_range); // Short data read disagrees with content-range
}
-
} // end namespace LLCore
diff --git a/indra/llcorehttp/httpcommon.h b/indra/llcorehttp/httpcommon.h
index 41fb5164cf..9601f94125 100755
--- a/indra/llcorehttp/httpcommon.h
+++ b/indra/llcorehttp/httpcommon.h
@@ -29,9 +29,9 @@
/// @package LLCore::HTTP
///
-/// This library implements a high-level, Indra-code-free client interface to
-/// HTTP services based on actual patterns found in the viewer and simulator.
-/// Interfaces are similar to those supplied by the legacy classes
+/// This library implements a high-level, Indra-code-free (somewhat) client
+/// interface to HTTP services based on actual patterns found in the viewer
+/// and simulator. Interfaces are similar to those supplied by the legacy classes
/// LLCurlRequest and LLHTTPClient. To that is added a policy scheme that
/// allows an application to specify connection behaviors: limits on
/// connections, HTTP keepalive, HTTP pipelining, retry-on-error limits, etc.
@@ -52,7 +52,7 @@
/// - "llcorehttp/httprequest.h"
/// - "llcorehttp/httpresponse.h"
///
-/// The library is still under early development and particular users
+/// The library is still under development and particular users
/// may need access to internal implementation details that are found
/// in the _*.h header files. But this is a crutch to be avoided if at
/// all possible and probably indicates some interface work is neeeded.
@@ -66,6 +66,8 @@
/// . CRYPTO_set_id_callback(...)
/// - HttpRequest::createService() called to instantiate singletons
/// and support objects.
+/// - HttpRequest::startThread() to kick off the worker thread and
+/// begin servicing requests.
///
/// An HTTP consumer in an application, and an application may have many
/// consumers, does a few things:
@@ -91,10 +93,12 @@
/// objects.
/// - Do completion processing in your onCompletion() method.
///
-/// Code fragments:
-/// Rather than a poorly-maintained example in comments, look in the
-/// example subdirectory which is a minimal yet functional tool to do
-/// GET request performance testing. With four calls:
+/// Code fragments.
+///
+/// Initialization. Rather than a poorly-maintained example in
+/// comments, look in the example subdirectory which is a minimal
+/// yet functional tool to do GET request performance testing.
+/// With four calls:
///
/// init_curl();
/// LLCore::HttpRequest::createService();
@@ -103,7 +107,85 @@
///
/// the program is basically ready to issue requests.
///
-
+/// HttpHandler. Having started life as a non-indra library,
+/// this code broke away from the classic Responder model and
+/// introduced a handler class to represent an interface for
+/// request responses. This is a non-reference-counted entity
+/// which can be used as a base class or a mixin. An instance
+/// of a handler can be used for each request or can be shared
+/// among any number of requests. Your choice but expect to
+/// code something like the following:
+///
+/// class AppHandler : public LLCore::HttpHandler
+/// {
+/// public:
+/// virtual void onCompleted(HttpHandle handle,
+/// HttpResponse * response)
+/// {
+/// ...
+/// }
+/// ...
+/// };
+/// ...
+/// handler = new handler(...);
+///
+///
+/// Issuing requests. Using 'hr' above,
+///
+/// hr->requestGet(HttpRequest::DEFAULT_POLICY_ID,
+/// 0, // Priority, not used yet
+/// url,
+/// NULL, // options
+/// NULL, // additional headers
+/// handler);
+///
+/// If that returns a value other than LLCORE_HTTP_HANDLE_INVALID,
+/// the request was successfully issued and there will eventally
+/// be a status delivered to the handler. If invalid is returnedd,
+/// the actual status can be retrieved by calling hr->getStatus().
+///
+/// Completing requests and delivering notifications. Operations
+/// are all performed by the worker thread and will be driven to
+/// completion regardless of caller actions. Notification of
+/// completion (success or failure) is done by calls to
+/// HttpRequest::update() which will invoke handlers for completed
+/// requests:
+///
+/// hr->update(0);
+/// // Callbacks into handler->onCompleted()
+///
+///
+/// Threads.
+///
+/// Threads are supported and used by this library. The various
+/// classes, methods and members are documented with thread
+/// constraints which programmers must follow and which are
+/// defined as follows:
+///
+/// consumer Any thread that has instanced HttpRequest and is
+/// issuing requests. A particular instance can only
+/// be used by one consumer thread but a consumer may
+/// have many instances available to it.
+/// init Special consumer thread, usually the main thread,
+/// involved in setting up the library at startup.
+/// worker Thread used internally by the library to perform
+/// HTTP operations. Consumers will not have to deal
+/// with this thread directly but some APIs are reserved
+/// to it.
+/// any Consumer or worker thread.
+///
+/// For the most part, API users will not have to do much in the
+/// way of ensuring thread safely. However, there is a tremendous
+/// amount of sharing between threads of read-only data. So when
+/// documentation declares that an option or header instance
+/// becomes shared between consumer and worker, the consumer must
+/// not modify the shared object.
+///
+/// Internally, there is almost no thread synchronization. During
+/// normal operations (non-init, non-term), only the request queue
+/// and the multiple reply queues are shared between threads and
+/// only here are mutexes used.
+///
#include "linden_common.h" // Modifies curl/curl.h interfaces
@@ -164,7 +246,10 @@ enum HttpError
HE_OPT_NOT_DYNAMIC = 8,
// Invalid HTTP status code returned by server
- HE_INVALID_HTTP_STATUS = 9
+ HE_INVALID_HTTP_STATUS = 9,
+
+ // Couldn't allocate resource, typically libcurl handle
+ HE_BAD_ALLOC = 10
}; // end enum HttpError
@@ -239,9 +324,10 @@ struct HttpStatus
return *this;
}
- static const type_enum_t EXT_CURL_EASY = 0;
- static const type_enum_t EXT_CURL_MULTI = 1;
- static const type_enum_t LLCORE = 2;
+ static const type_enum_t EXT_CURL_EASY = 0; ///< mStatus is an error from a curl_easy_*() call
+ static const type_enum_t EXT_CURL_MULTI = 1; ///< mStatus is an error from a curl_multi_*() call
+ static const type_enum_t LLCORE = 2; ///< mStatus is an HE_* error code
+ ///< 100-999 directly represent HTTP status codes
type_enum_t mType;
short mStatus;
@@ -297,6 +383,14 @@ struct HttpStatus
/// LLCore itself).
std::string toString() const;
+ /// Convert status to a compact string representation
+ /// of the form: "<type>_<value>". The <type> will be
+ /// one of: Core, Http, Easy, Multi, Unknown. And
+ /// <value> will be an unsigned integer. More easily
+ /// interpreted than the hex representation, it's still
+ /// compact and easily searched.
+ std::string toTerseString() const;
+
/// Returns true if the status value represents an
/// HTTP response status (100 - 999).
bool isHttpStatus() const
diff --git a/indra/llcorehttp/httpheaders.cpp b/indra/llcorehttp/httpheaders.cpp
index 2832696271..23ebea361c 100755
--- a/indra/llcorehttp/httpheaders.cpp
+++ b/indra/llcorehttp/httpheaders.cpp
@@ -4,7 +4,7 @@
*
* $LicenseInfo:firstyear=2012&license=viewerlgpl$
* Second Life Viewer Source Code
- * Copyright (C) 2012, Linden Research, Inc.
+ * Copyright (C) 2012-2013, Linden Research, Inc.
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
@@ -26,6 +26,8 @@
#include "httpheaders.h"
+#include "llstring.h"
+
namespace LLCore
{
@@ -40,5 +42,142 @@ HttpHeaders::~HttpHeaders()
{}
+void
+HttpHeaders::clear()
+{
+ mHeaders.clear();
+}
+
+
+void HttpHeaders::append(const std::string & name, const std::string & value)
+{
+ mHeaders.push_back(value_type(name, value));
+}
+
+
+void HttpHeaders::append(const char * name, const char * value)
+{
+ mHeaders.push_back(value_type(name, value));
+}
+
+
+void HttpHeaders::appendNormal(const char * header, size_t size)
+{
+ std::string name;
+ std::string value;
+
+ int col_pos(0);
+ for (; col_pos < size; ++col_pos)
+ {
+ if (':' == header[col_pos])
+ break;
+ }
+
+ if (col_pos < size)
+ {
+ // Looks like a header, split it and normalize.
+ // Name is everything before the colon, may be zero-length.
+ name.assign(header, col_pos);
+
+ // Value is everything after the colon, may also be zero-length.
+ const size_t val_len(size - col_pos - 1);
+ if (val_len)
+ {
+ value.assign(header + col_pos + 1, val_len);
+ }
+
+ // Clean the strings
+ LLStringUtil::toLower(name);
+ LLStringUtil::trim(name);
+ LLStringUtil::trimHead(value);
+ }
+ else
+ {
+ // Uncertain what this is, we'll pack it as
+ // a name without a value. Won't clean as we don't
+ // know what it is...
+ name.assign(header, size);
+ }
+
+ mHeaders.push_back(value_type(name, value));
+}
+
+
+// Find from end to simulate a tradition of using single-valued
+// std::map for this in the past.
+const std::string * HttpHeaders::find(const char * name) const
+{
+ const_reverse_iterator iend(rend());
+ for (const_reverse_iterator iter(rbegin()); iend != iter; ++iter)
+ {
+ if ((*iter).first == name)
+ {
+ return &(*iter).second;
+ }
+ }
+ return NULL;
+}
+
+
+// Standard Iterators
+HttpHeaders::iterator HttpHeaders::begin()
+{
+ return mHeaders.begin();
+}
+
+
+HttpHeaders::const_iterator HttpHeaders::begin() const
+{
+ return mHeaders.begin();
+}
+
+
+HttpHeaders::iterator HttpHeaders::end()
+{
+ return mHeaders.end();
+}
+
+
+HttpHeaders::const_iterator HttpHeaders::end() const
+{
+ return mHeaders.end();
+}
+
+
+// Standard Reverse Iterators
+HttpHeaders::reverse_iterator HttpHeaders::rbegin()
+{
+ return mHeaders.rbegin();
+}
+
+
+HttpHeaders::const_reverse_iterator HttpHeaders::rbegin() const
+{
+ return mHeaders.rbegin();
+}
+
+
+HttpHeaders::reverse_iterator HttpHeaders::rend()
+{
+ return mHeaders.rend();
+}
+
+
+HttpHeaders::const_reverse_iterator HttpHeaders::rend() const
+{
+ return mHeaders.rend();
+}
+
+
+// Return the raw container to the caller.
+//
+// To be used FOR UNIT TESTS ONLY.
+//
+HttpHeaders::container_t & HttpHeaders::getContainerTESTONLY()
+{
+ return mHeaders;
+}
+
+
} // end namespace LLCore
diff --git a/indra/llcorehttp/httpheaders.h b/indra/llcorehttp/httpheaders.h
index 3449daa3a1..f70cd898f3 100755
--- a/indra/llcorehttp/httpheaders.h
+++ b/indra/llcorehttp/httpheaders.h
@@ -4,7 +4,7 @@
*
* $LicenseInfo:firstyear=2012&license=viewerlgpl$
* Second Life Viewer Source Code
- * Copyright (C) 2012, Linden Research, Inc.
+ * Copyright (C) 2012-2013, Linden Research, Inc.
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
@@ -43,13 +43,26 @@ namespace LLCore
/// caller has asked that headers be returned (not the default
/// option).
///
-/// @note
-/// This is a minimally-functional placeholder at the moment
-/// to fill out the class hierarchy. The final class will be
-/// something else, probably more pair-oriented. It's also
-/// an area where shared values are desirable so refcounting is
-/// already specced and a copy-on-write scheme imagined.
-/// Expect changes here.
+/// Class is mostly a thin wrapper around a vector of pairs
+/// of strings. Methods provided are few and intended to
+/// reflect actual use patterns. These include:
+/// - Clearing the list
+/// - Appending a name/value pair to the vector
+/// - Processing a raw byte string into a normalized name/value
+/// pair and appending the result.
+/// - Simple case-sensitive find-last-by-name search
+/// - Forward and reverse iterators over all pairs
+///
+/// Container is ordered and multi-valued. Headers are
+/// written in the order in which they are appended and
+/// are stored in the order in which they're received from
+/// the wire. The same header may appear two or more times
+/// in any container. Searches using the simple find()
+/// interface will find only the last occurrence (somewhat
+/// simulates the use of std::map). Fuller searches require
+/// the use of an iterator. Headers received from the wire
+/// are only returned from the last request when redirections
+/// are involved.
///
/// Threading: Not intrinsically thread-safe. It *is* expected
/// that callers will build these objects and then share them
@@ -64,6 +77,16 @@ namespace LLCore
class HttpHeaders : public LLCoreInt::RefCounted
{
public:
+ typedef std::pair<std::string, std::string> header_t;
+ typedef std::vector<header_t> container_t;
+ typedef container_t::iterator iterator;
+ typedef container_t::const_iterator const_iterator;
+ typedef container_t::reverse_iterator reverse_iterator;
+ typedef container_t::const_reverse_iterator const_reverse_iterator;
+ typedef container_t::value_type value_type;
+ typedef container_t::size_type size_type;
+
+public:
/// @post In addition to the instance, caller has a refcount
/// to the instance. A call to @see release() will destroy
/// the instance.
@@ -76,7 +99,78 @@ protected:
void operator=(const HttpHeaders &); // Not defined
public:
- typedef std::vector<std::string> container_t;
+ // Empty the list of headers.
+ void clear();
+
+ // Append a name/value pair supplied as either std::strings
+ // or NUL-terminated char * to the header list. No normalization
+ // is performed on the strings. No conformance test is
+ // performed (names may contain spaces, colons, etc.).
+ //
+ void append(const std::string & name, const std::string & value);
+ void append(const char * name, const char * value);
+
+ // Extract a name/value pair from a raw byte array using
+ // the first colon character as a separator. Input string
+ // does not need to be NUL-terminated. Resulting name/value
+ // pair is appended to the header list.
+ //
+ // Normalization is performed on the name/value pair as
+ // follows:
+ // - name is lower-cased according to mostly ASCII rules
+ // - name is left- and right-trimmed of spaces and tabs
+ // - value is left-trimmed of spaces and tabs
+ // - either or both of name and value may be zero-length
+ //
+ // By convention, headers read from the wire will be normalized
+ // in this fashion prior to delivery to any HttpHandler code.
+ // Headers to be written to the wire are left as appended to
+ // the list.
+ void appendNormal(const char * header, size_t size);
+
+ // Perform a simple, case-sensitive search of the header list
+ // returning a pointer to the value of the last matching header
+ // in the header list. If none is found, a NULL pointer is returned.
+ //
+ // Any pointer returned references objects in the container itself
+ // and will have the same lifetime as this class. If you want
+ // the value beyond the lifetime of this instance, make a copy.
+ //
+ // @arg name C-style string giving the name of a header
+ // to search. The comparison is case-sensitive
+ // though list entries may have been normalized
+ // to lower-case.
+ //
+ // @return NULL if the header wasn't found otherwise
+ // a pointer to a std::string in the container.
+ // Pointer is valid only for the lifetime of
+ // the container or until container is modifed.
+ //
+ const std::string * find(const char * name) const;
+
+ // Count of headers currently in the list.
+ size_type size() const
+ {
+ return mHeaders.size();
+ }
+
+ // Standard std::vector-based forward iterators.
+ iterator begin();
+ const_iterator begin() const;
+ iterator end();
+ const_iterator end() const;
+
+ // Standard std::vector-based reverse iterators.
+ reverse_iterator rbegin();
+ const_reverse_iterator rbegin() const;
+ reverse_iterator rend();
+ const_reverse_iterator rend() const;
+
+public:
+ // For unit tests only - not a public API
+ container_t & getContainerTESTONLY();
+
+protected:
container_t mHeaders;
}; // end class HttpHeaders
diff --git a/indra/llcorehttp/httpoptions.cpp b/indra/llcorehttp/httpoptions.cpp
index 1699d19f8d..5bf1ecb4a5 100755
--- a/indra/llcorehttp/httpoptions.cpp
+++ b/indra/llcorehttp/httpoptions.cpp
@@ -4,7 +4,7 @@
*
* $LicenseInfo:firstyear=2012&license=viewerlgpl$
* Second Life Viewer Source Code
- * Copyright (C) 2012, Linden Research, Inc.
+ * Copyright (C) 2012-2013, Linden Research, Inc.
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
@@ -38,7 +38,9 @@ HttpOptions::HttpOptions()
mWantHeaders(false),
mTracing(HTTP_TRACE_OFF),
mTimeout(HTTP_REQUEST_TIMEOUT_DEFAULT),
- mRetries(HTTP_RETRY_COUNT_DEFAULT)
+ mTransferTimeout(HTTP_REQUEST_XFER_TIMEOUT_DEFAULT),
+ mRetries(HTTP_RETRY_COUNT_DEFAULT),
+ mUseRetryAfter(HTTP_USE_RETRY_AFTER_DEFAULT)
{}
@@ -64,10 +66,21 @@ void HttpOptions::setTimeout(unsigned int timeout)
}
+void HttpOptions::setTransferTimeout(unsigned int timeout)
+{
+ mTransferTimeout = timeout;
+}
+
+
void HttpOptions::setRetries(unsigned int retries)
{
mRetries = retries;
}
+void HttpOptions::setUseRetryAfter(bool use_retry)
+{
+ mUseRetryAfter = use_retry;
+}
+
} // end namespace LLCore
diff --git a/indra/llcorehttp/httpoptions.h b/indra/llcorehttp/httpoptions.h
index 97e46a8cd3..4ab5ff18c4 100755
--- a/indra/llcorehttp/httpoptions.h
+++ b/indra/llcorehttp/httpoptions.h
@@ -4,7 +4,7 @@
*
* $LicenseInfo:firstyear=2012&license=viewerlgpl$
* Second Life Viewer Source Code
- * Copyright (C) 2012, Linden Research, Inc.
+ * Copyright (C) 2012-2013, Linden Research, Inc.
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
@@ -68,36 +68,55 @@ protected:
void operator=(const HttpOptions &); // Not defined
public:
+ // Default: false
void setWantHeaders(bool wanted);
bool getWantHeaders() const
{
return mWantHeaders;
}
-
+
+ // Default: 0
void setTrace(int long);
int getTrace() const
{
return mTracing;
}
+ // Default: 30
void setTimeout(unsigned int timeout);
unsigned int getTimeout() const
{
return mTimeout;
}
+ // Default: 0
+ void setTransferTimeout(unsigned int timeout);
+ unsigned int getTransferTimeout() const
+ {
+ return mTransferTimeout;
+ }
+
+ // Default: 8
void setRetries(unsigned int retries);
unsigned int getRetries() const
{
return mRetries;
}
+
+ // Default: true
+ void setUseRetryAfter(bool use_retry);
+ bool getUseRetryAfter() const
+ {
+ return mUseRetryAfter;
+ }
protected:
bool mWantHeaders;
int mTracing;
unsigned int mTimeout;
+ unsigned int mTransferTimeout;
unsigned int mRetries;
-
+ bool mUseRetryAfter;
}; // end class HttpOptions
diff --git a/indra/llcorehttp/httprequest.cpp b/indra/llcorehttp/httprequest.cpp
index 9b739a8825..7b1888e3eb 100755
--- a/indra/llcorehttp/httprequest.cpp
+++ b/indra/llcorehttp/httprequest.cpp
@@ -4,7 +4,7 @@
*
* $LicenseInfo:firstyear=2012&license=viewerlgpl$
* Second Life Viewer Source Code
- * Copyright (C) 2012, Linden Research, Inc.
+ * Copyright (C) 2012-2013, Linden Research, Inc.
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
@@ -54,12 +54,8 @@ namespace LLCore
// ====================================
-HttpRequest::policy_t HttpRequest::sNextPolicyID(1);
-
-
HttpRequest::HttpRequest()
- : //HttpHandler(),
- mReplyQueue(NULL),
+ : mReplyQueue(NULL),
mRequestQueue(NULL)
{
mRequestQueue = HttpRequestQueue::instanceOf();
@@ -90,45 +86,91 @@ HttpRequest::~HttpRequest()
// ====================================
-HttpStatus HttpRequest::setPolicyGlobalOption(EGlobalPolicy opt, long value)
+HttpRequest::policy_t HttpRequest::createPolicyClass()
{
if (HttpService::RUNNING == HttpService::instanceOf()->getState())
{
- return HttpStatus(HttpStatus::LLCORE, HE_OPT_NOT_DYNAMIC);
+ return 0;
}
- return HttpService::instanceOf()->getGlobalOptions().set(opt, value);
+ return HttpService::instanceOf()->createPolicyClass();
}
-HttpStatus HttpRequest::setPolicyGlobalOption(EGlobalPolicy opt, const std::string & value)
+HttpStatus HttpRequest::setStaticPolicyOption(EPolicyOption opt, policy_t pclass,
+ long value, long * ret_value)
{
if (HttpService::RUNNING == HttpService::instanceOf()->getState())
{
return HttpStatus(HttpStatus::LLCORE, HE_OPT_NOT_DYNAMIC);
}
- return HttpService::instanceOf()->getGlobalOptions().set(opt, value);
+ return HttpService::instanceOf()->setPolicyOption(opt, pclass, value, ret_value);
}
-HttpRequest::policy_t HttpRequest::createPolicyClass()
+HttpStatus HttpRequest::setStaticPolicyOption(EPolicyOption opt, policy_t pclass,
+ const std::string & value, std::string * ret_value)
{
if (HttpService::RUNNING == HttpService::instanceOf()->getState())
{
- return 0;
+ return HttpStatus(HttpStatus::LLCORE, HE_OPT_NOT_DYNAMIC);
}
- return HttpService::instanceOf()->createPolicyClass();
+ return HttpService::instanceOf()->setPolicyOption(opt, pclass, value, ret_value);
}
-HttpStatus HttpRequest::setPolicyClassOption(policy_t policy_id,
- EClassPolicy opt,
- long value)
+HttpHandle HttpRequest::setPolicyOption(EPolicyOption opt, policy_t pclass,
+ long value, HttpHandler * handler)
{
- if (HttpService::RUNNING == HttpService::instanceOf()->getState())
+ HttpStatus status;
+ HttpHandle handle(LLCORE_HTTP_HANDLE_INVALID);
+
+ HttpOpSetGet * op = new HttpOpSetGet();
+ if (! (status = op->setupSet(opt, pclass, value)))
{
- return HttpStatus(HttpStatus::LLCORE, HE_OPT_NOT_DYNAMIC);
+ op->release();
+ mLastReqStatus = status;
+ return handle;
+ }
+ op->setReplyPath(mReplyQueue, handler);
+ if (! (status = mRequestQueue->addOp(op))) // transfers refcount
+ {
+ op->release();
+ mLastReqStatus = status;
+ return handle;
+ }
+
+ mLastReqStatus = status;
+ handle = static_cast<HttpHandle>(op);
+
+ return handle;
+}
+
+
+HttpHandle HttpRequest::setPolicyOption(EPolicyOption opt, policy_t pclass,
+ const std::string & value, HttpHandler * handler)
+{
+ HttpStatus status;
+ HttpHandle handle(LLCORE_HTTP_HANDLE_INVALID);
+
+ HttpOpSetGet * op = new HttpOpSetGet();
+ if (! (status = op->setupSet(opt, pclass, value)))
+ {
+ op->release();
+ mLastReqStatus = status;
+ return handle;
+ }
+ op->setReplyPath(mReplyQueue, handler);
+ if (! (status = mRequestQueue->addOp(op))) // transfers refcount
+ {
+ op->release();
+ mLastReqStatus = status;
+ return handle;
}
- return HttpService::instanceOf()->getClassOptions(policy_id).set(opt, value);
+
+ mLastReqStatus = status;
+ handle = static_cast<HttpHandle>(op);
+
+ return handle;
}
@@ -474,31 +516,6 @@ HttpHandle HttpRequest::requestSpin(int mode)
return handle;
}
-// ====================================
-// Dynamic Policy Methods
-// ====================================
-
-HttpHandle HttpRequest::requestSetHttpProxy(const std::string & proxy, HttpHandler * handler)
-{
- HttpStatus status;
- HttpHandle handle(LLCORE_HTTP_HANDLE_INVALID);
-
- HttpOpSetGet * op = new HttpOpSetGet();
- op->setupSet(GP_HTTP_PROXY, proxy);
- op->setReplyPath(mReplyQueue, handler);
- if (! (status = mRequestQueue->addOp(op))) // transfers refcount
- {
- op->release();
- mLastReqStatus = status;
- return handle;
- }
-
- mLastReqStatus = status;
- handle = static_cast<HttpHandle>(op);
-
- return handle;
-}
-
} // end namespace LLCore
diff --git a/indra/llcorehttp/httprequest.h b/indra/llcorehttp/httprequest.h
index ab2f302d34..651654844a 100755
--- a/indra/llcorehttp/httprequest.h
+++ b/indra/llcorehttp/httprequest.h
@@ -4,7 +4,7 @@
*
* $LicenseInfo:firstyear=2012&license=viewerlgpl$
* Second Life Viewer Source Code
- * Copyright (C) 2012, Linden Research, Inc.
+ * Copyright (C) 2012-2013, Linden Research, Inc.
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
@@ -56,6 +56,9 @@ class BufferArray;
/// The class supports the current HTTP request operations:
///
/// - requestGetByteRange: GET with Range header for a single range of bytes
+/// - requestGet:
+/// - requestPost:
+/// - requestPut:
///
/// Policy Classes
///
@@ -100,9 +103,26 @@ public:
/// Represents a default, catch-all policy class that guarantees
/// eventual service for any HTTP request.
- static const int DEFAULT_POLICY_ID = 0;
+ static const policy_t DEFAULT_POLICY_ID = 0;
+ static const policy_t INVALID_POLICY_ID = 0xFFFFFFFFU;
+ static const policy_t GLOBAL_POLICY_ID = 0xFFFFFFFEU;
- enum EGlobalPolicy
+ /// Create a new policy class into which requests can be made.
+ ///
+ /// All class creation must occur before threads are started and
+ /// transport begins. Policy classes are limited to a small value.
+ /// Currently that limit is the default class + 1.
+ ///
+ /// @return If positive, the policy_id used to reference
+ /// the class in other methods. If 0, requests
+ /// for classes have exceeded internal limits
+ /// or caller has tried to create a class after
+ /// threads have been started. Caller must fallback
+ /// and recover.
+ ///
+ static policy_t createPolicyClass();
+
+ enum EPolicyOption
{
/// Maximum number of connections the library will use to
/// perform operations. This is somewhat soft as the underlying
@@ -113,24 +133,40 @@ public:
/// a somewhat soft value. There may be an additional five
/// connections per policy class depending upon runtime
/// behavior.
- GP_CONNECTION_LIMIT,
+ ///
+ /// Both global and per-class
+ PO_CONNECTION_LIMIT,
+
+ /// Limits the number of connections used for a single
+ /// literal address/port pair within the class.
+ ///
+ /// Per-class only
+ PO_PER_HOST_CONNECTION_LIMIT,
/// String containing a system-appropriate directory name
/// where SSL certs are stored.
- GP_CA_PATH,
+ ///
+ /// Global only
+ PO_CA_PATH,
/// String giving a full path to a file containing SSL certs.
- GP_CA_FILE,
+ ///
+ /// Global only
+ PO_CA_FILE,
/// String of host/port to use as simple HTTP proxy. This is
/// going to change in the future into something more elaborate
/// that may support richer schemes.
- GP_HTTP_PROXY,
+ ///
+ /// Global only
+ PO_HTTP_PROXY,
/// Long value that if non-zero enables the use of the
/// traditional LLProxy code for http/socks5 support. If
- /// enabled, has priority over GP_HTTP_PROXY.
- GP_LLPROXY,
+ // enabled, has priority over GP_HTTP_PROXY.
+ ///
+ /// Global only
+ PO_LLPROXY,
/// Long value setting the logging trace level for the
/// library. Possible values are:
@@ -143,50 +179,59 @@ public:
/// These values are also used in the trace modes for
/// individual requests in HttpOptions. Also be aware that
/// tracing tends to impact performance of the viewer.
- GP_TRACE
- };
-
- /// Set a parameter on a global policy option. Calls
- /// made after the start of the servicing thread are
- /// not honored and return an error status.
- ///
- /// @param opt Enum of option to be set.
- /// @param value Desired value of option.
- /// @return Standard status code.
- static HttpStatus setPolicyGlobalOption(EGlobalPolicy opt, long value);
- static HttpStatus setPolicyGlobalOption(EGlobalPolicy opt, const std::string & value);
-
- /// Create a new policy class into which requests can be made.
- ///
- /// @return If positive, the policy_id used to reference
- /// the class in other methods. If 0, an error
- /// occurred and @see getStatus() may provide more
- /// detail on the reason.
- static policy_t createPolicyClass();
-
- enum EClassPolicy
- {
- /// Limits the number of connections used for the class.
- CP_CONNECTION_LIMIT,
-
- /// Limits the number of connections used for a single
- /// literal address/port pair within the class.
- CP_PER_HOST_CONNECTION_LIMIT,
+ ///
+ /// Global only
+ PO_TRACE,
/// Suitable requests are allowed to pipeline on their
/// connections when they ask for it.
- CP_ENABLE_PIPELINING
+ ///
+ /// Per-class only
+ PO_ENABLE_PIPELINING,
+
+ /// Controls whether client-side throttling should be
+ /// performed on this policy class. Positive values
+ /// enable throttling and specify the request rate
+ /// (requests per second) that should be targetted.
+ /// A value of zero, the default, specifies no throttling.
+ ///
+ /// Per-class only
+ PO_THROTTLE_RATE,
+
+ PO_LAST // Always at end
};
-
+
+ /// Set a policy option for a global or class parameter at
+ /// startup time (prior to thread start).
+ ///
+ /// @param opt Enum of option to be set.
+ /// @param pclass For class-based options, the policy class ID to
+ /// be changed. For globals, specify GLOBAL_POLICY_ID.
+ /// @param value Desired value of option.
+ /// @param ret_value Pointer to receive effective set value
+ /// if successful. May be NULL if effective
+ /// value not wanted.
+ /// @return Standard status code.
+ static HttpStatus setStaticPolicyOption(EPolicyOption opt, policy_t pclass,
+ long value, long * ret_value);
+ static HttpStatus setStaticPolicyOption(EPolicyOption opt, policy_t pclass,
+ const std::string & value, std::string * ret_value);
+
/// Set a parameter on a class-based policy option. Calls
/// made after the start of the servicing thread are
/// not honored and return an error status.
///
- /// @param policy_id ID of class as returned by @see createPolicyClass().
- /// @param opt Enum of option to be set.
- /// @param value Desired value of option.
- /// @return Standard status code.
- static HttpStatus setPolicyClassOption(policy_t policy_id, EClassPolicy opt, long value);
+ /// @param opt Enum of option to be set.
+ /// @param pclass For class-based options, the policy class ID to
+ /// be changed. Ignored for globals but recommend
+ /// using INVALID_POLICY_ID in this case.
+ /// @param value Desired value of option.
+ /// @return Handle of dynamic request. Use @see getStatus() if
+ /// the returned handle is invalid.
+ HttpHandle setPolicyOption(EPolicyOption opt, policy_t pclass, long value,
+ HttpHandler * handler);
+ HttpHandle setPolicyOption(EPolicyOption opt, policy_t pclass, const std::string & value,
+ HttpHandler * handler);
/// @}
@@ -488,16 +533,6 @@ public:
/// @}
- /// @name DynamicPolicyMethods
- ///
- /// @{
-
- /// Request that a running transport pick up a new proxy setting.
- /// An empty string will indicate no proxy is to be used.
- HttpHandle requestSetHttpProxy(const std::string & proxy, HttpHandler * handler);
-
- /// @}
-
protected:
void generateNotification(HttpOperation * op);
@@ -519,7 +554,6 @@ private:
/// Must be established before any threading is allowed to
/// start.
///
- static policy_t sNextPolicyID;
/// @}
// End Global State
diff --git a/indra/llcorehttp/httpresponse.cpp b/indra/llcorehttp/httpresponse.cpp
index a552e48a1b..c974395b0a 100755
--- a/indra/llcorehttp/httpresponse.cpp
+++ b/indra/llcorehttp/httpresponse.cpp
@@ -4,7 +4,7 @@
*
* $LicenseInfo:firstyear=2012&license=viewerlgpl$
* Second Life Viewer Source Code
- * Copyright (C) 2012, Linden Research, Inc.
+ * Copyright (C) 2012-2013, Linden Research, Inc.
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
@@ -39,7 +39,9 @@ HttpResponse::HttpResponse()
mReplyLength(0U),
mReplyFullLength(0U),
mBufferArray(NULL),
- mHeaders(NULL)
+ mHeaders(NULL),
+ mRetries(0U),
+ m503Retries(0U)
{}
diff --git a/indra/llcorehttp/httpresponse.h b/indra/llcorehttp/httpresponse.h
index f19b521fbf..aee64e2878 100755
--- a/indra/llcorehttp/httpresponse.h
+++ b/indra/llcorehttp/httpresponse.h
@@ -4,7 +4,7 @@
*
* $LicenseInfo:firstyear=2012&license=viewerlgpl$
* Second Life Viewer Source Code
- * Copyright (C) 2012, Linden Research, Inc.
+ * Copyright (C) 2012-2013, Linden Research, Inc.
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
@@ -149,6 +149,25 @@ public:
mContentType = con_type;
}
+ /// Get and set retry attempt information on the request.
+ void getRetries(unsigned int * retries, unsigned int * retries_503) const
+ {
+ if (retries)
+ {
+ *retries = mRetries;
+ }
+ if (retries_503)
+ {
+ *retries_503 = m503Retries;
+ }
+ }
+
+ void setRetries(unsigned int retries, unsigned int retries_503)
+ {
+ mRetries = retries;
+ m503Retries = retries_503;
+ }
+
protected:
// Response data here
HttpStatus mStatus;
@@ -158,6 +177,8 @@ protected:
BufferArray * mBufferArray;
HttpHeaders * mHeaders;
std::string mContentType;
+ unsigned int mRetries;
+ unsigned int m503Retries;
};
diff --git a/indra/llcorehttp/tests/test_httpheaders.hpp b/indra/llcorehttp/tests/test_httpheaders.hpp
index ce0d19b058..668c36dc66 100755
--- a/indra/llcorehttp/tests/test_httpheaders.hpp
+++ b/indra/llcorehttp/tests/test_httpheaders.hpp
@@ -4,7 +4,7 @@
*
* $LicenseInfo:firstyear=2012&license=viewerlgpl$
* Second Life Viewer Source Code
- * Copyright (C) 2012, Linden Research, Inc.
+ * Copyright (C) 2012-2013, Linden Research, Inc.
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
@@ -36,7 +36,6 @@
using namespace LLCoreInt;
-
namespace tut
{
@@ -63,7 +62,7 @@ void HttpHeadersTestObjectType::test<1>()
HttpHeaders * headers = new HttpHeaders();
ensure("One ref on construction of HttpHeaders", headers->getRefCount() == 1);
ensure("Memory being used", mMemTotal < GetMemTotal());
- ensure("Nothing in headers", 0 == headers->mHeaders.size());
+ ensure("Nothing in headers", 0 == headers->size());
// release the implicit reference, causing the object to be released
headers->release();
@@ -85,14 +84,340 @@ void HttpHeadersTestObjectType::test<2>()
{
// Append a few strings
- std::string str1("Pragma:");
- headers->mHeaders.push_back(str1);
- std::string str2("Accept: application/json");
- headers->mHeaders.push_back(str2);
+ std::string str1n("Pragma");
+ std::string str1v("");
+ headers->append(str1n, str1v);
+ std::string str2n("Accept");
+ std::string str2v("application/json");
+ headers->append(str2n, str2v);
+
+ ensure("Headers retained", 2 == headers->size());
+ HttpHeaders::container_t & c(headers->getContainerTESTONLY());
+
+ ensure("First name is first name", c[0].first == str1n);
+ ensure("First value is first value", c[0].second == str1v);
+ ensure("Second name is second name", c[1].first == str2n);
+ ensure("Second value is second value", c[1].second == str2v);
+ }
+
+ // release the implicit reference, causing the object to be released
+ headers->release();
+
+ // make sure we didn't leak any memory
+ ensure(mMemTotal == GetMemTotal());
+}
+
+template <> template <>
+void HttpHeadersTestObjectType::test<3>()
+{
+ set_test_name("HttpHeaders basic find");
+
+ // record the total amount of dynamically allocated memory
+ mMemTotal = GetMemTotal();
+
+ // create a new ref counted object with an implicit reference
+ HttpHeaders * headers = new HttpHeaders();
+
+ {
+ // Append a few strings
+ std::string str1n("Uno");
+ std::string str1v("1");
+ headers->append(str1n, str1v);
+ std::string str2n("doS");
+ std::string str2v("2-2-2-2");
+ headers->append(str2n, str2v);
+ std::string str3n("TRES");
+ std::string str3v("trois gymnopedie");
+ headers->append(str3n, str3v);
+
+ ensure("Headers retained", 3 == headers->size());
+
+ const std::string * result(NULL);
+
+ // Find a header
+ result = headers->find("TRES");
+ ensure("Found the last item", result != NULL);
+ ensure("Last item is a nice", result != NULL && str3v == *result);
+
+ // appends above are raw and find is case sensitive
+ result = headers->find("TReS");
+ ensure("Last item not found due to case", result == NULL);
+
+ result = headers->find("TRE");
+ ensure("Last item not found due to prefixing (1)", result == NULL);
+
+ result = headers->find("TRESS");
+ ensure("Last item not found due to prefixing (2)", result == NULL);
+ }
+
+ // release the implicit reference, causing the object to be released
+ headers->release();
+
+ // make sure we didn't leak any memory
+ ensure(mMemTotal == GetMemTotal());
+}
+
+template <> template <>
+void HttpHeadersTestObjectType::test<4>()
+{
+ set_test_name("HttpHeaders normalized header entry");
+
+ // record the total amount of dynamically allocated memory
+ mMemTotal = GetMemTotal();
+
+ // create a new ref counted object with an implicit reference
+ HttpHeaders * headers = new HttpHeaders();
+
+ {
+ static char line1[] = " AcCePT : image/yourfacehere";
+ static char line1v[] = "image/yourfacehere";
+ headers->appendNormal(line1, sizeof(line1) - 1);
+
+ ensure("First append worked in some fashion", 1 == headers->size());
+
+ const std::string * result(NULL);
+
+ // Find a header
+ result = headers->find("accept");
+ ensure("Found 'accept'", result != NULL);
+ ensure("accept value has face", result != NULL && *result == line1v);
+
+ // Left-clean on value
+ static char line2[] = " next : \t\tlinejunk \t";
+ headers->appendNormal(line2, sizeof(line2) - 1);
+ ensure("Second append worked", 2 == headers->size());
+ result = headers->find("next");
+ ensure("Found 'next'", result != NULL);
+ ensure("next value is left-clean", result != NULL &&
+ *result == "linejunk \t");
+
+ // First value unmolested
+ result = headers->find("accept");
+ ensure("Found 'accept' again", result != NULL);
+ ensure("accept value has face", result != NULL && *result == line1v);
+
+ // Colons in value are okay
+ static char line3[] = "FancY-PANTs::plop:-neuf-=vleem=";
+ static char line3v[] = ":plop:-neuf-=vleem=";
+ headers->appendNormal(line3, sizeof(line3) - 1);
+ ensure("Third append worked", 3 == headers->size());
+ result = headers->find("fancy-pants");
+ ensure("Found 'fancy-pants'", result != NULL);
+ ensure("fancy-pants value has colons", result != NULL && *result == line3v);
+
+ // Zero-length value
+ static char line4[] = "all-talk-no-walk:";
+ headers->appendNormal(line4, sizeof(line4) - 1);
+ ensure("Fourth append worked", 4 == headers->size());
+ result = headers->find("all-talk-no-walk");
+ ensure("Found 'all-talk'", result != NULL);
+ ensure("al-talk value is zero-length", result != NULL && result->size() == 0);
+
+ // Zero-length name
+ static char line5[] = ":all-talk-no-walk";
+ static char line5v[] = "all-talk-no-walk";
+ headers->appendNormal(line5, sizeof(line5) - 1);
+ ensure("Fifth append worked", 5 == headers->size());
+ result = headers->find("");
+ ensure("Found no-name", result != NULL);
+ ensure("no-name value is something", result != NULL && *result == line5v);
+
+ // Lone colon is still something
+ headers->clear();
+ static char line6[] = " :";
+ headers->appendNormal(line6, sizeof(line6) - 1);
+ ensure("Sixth append worked", 1 == headers->size());
+ result = headers->find("");
+ ensure("Found 2nd no-name", result != NULL);
+ ensure("2nd no-name value is nothing", result != NULL && result->size() == 0);
+
+ // Line without colons is taken as-is and unstripped in name
+ static char line7[] = " \toskdgioasdghaosdghoowg28342908tg8902hg0hwedfhqew890v7qh0wdebv78q0wdevbhq>?M>BNM<ZV>?NZ? \t";
+ headers->appendNormal(line7, sizeof(line7) - 1);
+ ensure("Seventh append worked", 2 == headers->size());
+ result = headers->find(line7);
+ ensure("Found whatsit line", result != NULL);
+ ensure("Whatsit line has no value", result != NULL && result->size() == 0);
+
+ // Normaling interface heeds the byte count, doesn't look for NUL-terminator
+ static char line8[] = "binary:ignorestuffontheendofthis";
+ headers->appendNormal(line8, 13);
+ ensure("Eighth append worked", 3 == headers->size());
+ result = headers->find("binary");
+ ensure("Found 'binary'", result != NULL);
+ ensure("binary value was limited to 'ignore'", result != NULL &&
+ *result == "ignore");
+
+ }
- ensure("Headers retained", 2 == headers->mHeaders.size());
- ensure("First is first", headers->mHeaders[0] == str1);
- ensure("Second is second", headers->mHeaders[1] == str2);
+ // release the implicit reference, causing the object to be released
+ headers->release();
+
+ // make sure we didn't leak any memory
+ ensure(mMemTotal == GetMemTotal());
+}
+
+// Verify forward iterator finds everything as expected
+template <> template <>
+void HttpHeadersTestObjectType::test<5>()
+{
+ set_test_name("HttpHeaders iterator tests");
+
+ // record the total amount of dynamically allocated memory
+ mMemTotal = GetMemTotal();
+
+ // create a new ref counted object with an implicit reference
+ HttpHeaders * headers = new HttpHeaders();
+
+ HttpHeaders::iterator end(headers->end()), begin(headers->begin());
+ ensure("Empty container has equal begin/end const iterators", end == begin);
+ HttpHeaders::const_iterator cend(headers->end()), cbegin(headers->begin());
+ ensure("Empty container has equal rbegin/rend const iterators", cend == cbegin);
+
+ ensure("Empty container has equal begin/end iterators", headers->end() == headers->begin());
+
+ {
+ static char line1[] = " AcCePT : image/yourfacehere";
+ static char line1v[] = "image/yourfacehere";
+ headers->appendNormal(line1, sizeof(line1) - 1);
+
+ static char line2[] = " next : \t\tlinejunk \t";
+ static char line2v[] = "linejunk \t";
+ headers->appendNormal(line2, sizeof(line2) - 1);
+
+ static char line3[] = "FancY-PANTs::plop:-neuf-=vleem=";
+ static char line3v[] = ":plop:-neuf-=vleem=";
+ headers->appendNormal(line3, sizeof(line3) - 1);
+
+ static char line4[] = "all-talk-no-walk:";
+ static char line4v[] = "";
+ headers->appendNormal(line4, sizeof(line4) - 1);
+
+ static char line5[] = ":all-talk-no-walk";
+ static char line5v[] = "all-talk-no-walk";
+ headers->appendNormal(line5, sizeof(line5) - 1);
+
+ static char line6[] = " :";
+ static char line6v[] = "";
+ headers->appendNormal(line6, sizeof(line6) - 1);
+
+ ensure("All entries accounted for", 6 == headers->size());
+
+ static char * values[] = {
+ line1v,
+ line2v,
+ line3v,
+ line4v,
+ line5v,
+ line6v
+ };
+
+ int i(0);
+ HttpHeaders::const_iterator cend(headers->end());
+ for (HttpHeaders::const_iterator it(headers->begin());
+ cend != it;
+ ++it, ++i)
+ {
+ std::ostringstream str;
+ str << "Const Iterator value # " << i << " was " << values[i];
+ ensure(str.str(), (*it).second == values[i]);
+ }
+
+ // Rewind, do non-consts
+ i = 0;
+ HttpHeaders::iterator end(headers->end());
+ for (HttpHeaders::iterator it(headers->begin());
+ end != it;
+ ++it, ++i)
+ {
+ std::ostringstream str;
+ str << "Const Iterator value # " << i << " was " << values[i];
+ ensure(str.str(), (*it).second == values[i]);
+ }
+ }
+
+ // release the implicit reference, causing the object to be released
+ headers->release();
+
+ // make sure we didn't leak any memory
+ ensure(mMemTotal == GetMemTotal());
+}
+
+// Reverse iterators find everything as expected
+template <> template <>
+void HttpHeadersTestObjectType::test<6>()
+{
+ set_test_name("HttpHeaders reverse iterator tests");
+
+ // record the total amount of dynamically allocated memory
+ mMemTotal = GetMemTotal();
+
+ // create a new ref counted object with an implicit reference
+ HttpHeaders * headers = new HttpHeaders();
+
+ HttpHeaders::reverse_iterator rend(headers->rend()), rbegin(headers->rbegin());
+ ensure("Empty container has equal rbegin/rend const iterators", rend == rbegin);
+ HttpHeaders::const_reverse_iterator crend(headers->rend()), crbegin(headers->rbegin());
+ ensure("Empty container has equal rbegin/rend const iterators", crend == crbegin);
+
+ {
+ static char line1[] = " AcCePT : image/yourfacehere";
+ static char line1v[] = "image/yourfacehere";
+ headers->appendNormal(line1, sizeof(line1) - 1);
+
+ static char line2[] = " next : \t\tlinejunk \t";
+ static char line2v[] = "linejunk \t";
+ headers->appendNormal(line2, sizeof(line2) - 1);
+
+ static char line3[] = "FancY-PANTs::plop:-neuf-=vleem=";
+ static char line3v[] = ":plop:-neuf-=vleem=";
+ headers->appendNormal(line3, sizeof(line3) - 1);
+
+ static char line4[] = "all-talk-no-walk:";
+ static char line4v[] = "";
+ headers->appendNormal(line4, sizeof(line4) - 1);
+
+ static char line5[] = ":all-talk-no-walk";
+ static char line5v[] = "all-talk-no-walk";
+ headers->appendNormal(line5, sizeof(line5) - 1);
+
+ static char line6[] = " :";
+ static char line6v[] = "";
+ headers->appendNormal(line6, sizeof(line6) - 1);
+
+ ensure("All entries accounted for", 6 == headers->size());
+
+ static char * values[] = {
+ line6v,
+ line5v,
+ line4v,
+ line3v,
+ line2v,
+ line1v
+ };
+
+ int i(0);
+ HttpHeaders::const_reverse_iterator cend(headers->rend());
+ for (HttpHeaders::const_reverse_iterator it(headers->rbegin());
+ cend != it;
+ ++it, ++i)
+ {
+ std::ostringstream str;
+ str << "Const Iterator value # " << i << " was " << values[i];
+ ensure(str.str(), (*it).second == values[i]);
+ }
+
+ // Rewind, do non-consts
+ i = 0;
+ HttpHeaders::reverse_iterator end(headers->rend());
+ for (HttpHeaders::reverse_iterator it(headers->rbegin());
+ end != it;
+ ++it, ++i)
+ {
+ std::ostringstream str;
+ str << "Iterator value # " << i << " was " << values[i];
+ ensure(str.str(), (*it).second == values[i]);
+ }
}
// release the implicit reference, causing the object to be released
diff --git a/indra/llcorehttp/tests/test_httprequest.hpp b/indra/llcorehttp/tests/test_httprequest.hpp
index 900a699887..43f7e36da5 100755
--- a/indra/llcorehttp/tests/test_httprequest.hpp
+++ b/indra/llcorehttp/tests/test_httprequest.hpp
@@ -69,6 +69,8 @@ void usleep(unsigned long usec);
namespace tut
{
+typedef std::vector<std::pair<boost::regex, boost::regex> > regex_container_t;
+
struct HttpRequestTestData
{
// the test objects inherit from this so the member functions and variables
@@ -118,11 +120,17 @@ public:
for (int i(0); i < mHeadersRequired.size(); ++i)
{
bool found = false;
- for (HttpHeaders::container_t::const_iterator iter(header->mHeaders.begin());
- header->mHeaders.end() != iter;
+ for (HttpHeaders::const_iterator iter(header->begin());
+ header->end() != iter;
++iter)
{
- if (boost::regex_match(*iter, mHeadersRequired[i]))
+ // std::cerr << "Header: " << (*iter).first
+ // << ": " << (*iter).second << std::endl;
+
+ if (boost::regex_match((*iter).first,
+ mHeadersRequired[i].first) &&
+ boost::regex_match((*iter).second,
+ mHeadersRequired[i].second))
{
found = true;
break;
@@ -138,11 +146,14 @@ public:
{
for (int i(0); i < mHeadersDisallowed.size(); ++i)
{
- for (HttpHeaders::container_t::const_iterator iter(header->mHeaders.begin());
- header->mHeaders.end() != iter;
+ for (HttpHeaders::const_iterator iter(header->begin());
+ header->end() != iter;
++iter)
{
- if (boost::regex_match(*iter, mHeadersDisallowed[i]))
+ if (boost::regex_match((*iter).first,
+ mHeadersDisallowed[i].first) &&
+ boost::regex_match((*iter).second,
+ mHeadersDisallowed[i].second))
{
std::ostringstream str;
str << "Disallowed header # " << i << " not found in response";
@@ -168,8 +179,8 @@ public:
std::string mName;
HttpHandle mExpectHandle;
std::string mCheckContentType;
- std::vector<boost::regex> mHeadersRequired;
- std::vector<boost::regex> mHeadersDisallowed;
+ regex_container_t mHeadersRequired;
+ regex_container_t mHeadersDisallowed;
};
typedef test_group<HttpRequestTestData> HttpRequestTestGroupType;
@@ -1211,7 +1222,7 @@ void HttpRequestTestObjectType::test<12>()
HttpRequest::createService();
// Enable tracing
- HttpRequest::setPolicyGlobalOption(LLCore::HttpRequest::GP_TRACE, 2);
+ HttpRequest::setStaticPolicyOption(HttpRequest::PO_TRACE, HttpRequest::DEFAULT_POLICY_ID, 2, NULL);
// Start threading early so that thread memory is invariant
// over the test.
@@ -1329,7 +1340,7 @@ void HttpRequestTestObjectType::test<13>()
HttpRequest::createService();
// Enable tracing
- HttpRequest::setPolicyGlobalOption(LLCore::HttpRequest::GP_TRACE, 2);
+ HttpRequest::setStaticPolicyOption(HttpRequest::PO_TRACE, HttpRequest::DEFAULT_POLICY_ID, 2, NULL);
// Start threading early so that thread memory is invariant
// over the test.
@@ -1344,7 +1355,9 @@ void HttpRequestTestObjectType::test<13>()
// Issue a GET that succeeds
mStatus = HttpStatus(200);
- handler.mHeadersRequired.push_back(boost::regex("\\W*X-LL-Special:.*", boost::regex::icase));
+ handler.mHeadersRequired.push_back(
+ regex_container_t::value_type(boost::regex("X-LL-Special", boost::regex::icase),
+ boost::regex(".*", boost::regex::icase)));
HttpHandle handle = req->requestGetByteRange(HttpRequest::DEFAULT_POLICY_ID,
0U,
url_base,
@@ -1711,18 +1724,54 @@ void HttpRequestTestObjectType::test<16>()
// Issue a GET that *can* connect
mStatus = HttpStatus(200);
- handler.mHeadersRequired.push_back(boost::regex("X-Reflect-connection:\\s*keep-alive", boost::regex::icase));
- handler.mHeadersRequired.push_back(boost::regex("X-Reflect-accept:\\s*\\*/\\*", boost::regex::icase));
- handler.mHeadersRequired.push_back(boost::regex("X-Reflect-accept-encoding:\\s*((gzip|deflate),\\s*)+(gzip|deflate)", boost::regex::icase)); // close enough
- handler.mHeadersRequired.push_back(boost::regex("X-Reflect-keep-alive:\\s*\\d+", boost::regex::icase));
- handler.mHeadersRequired.push_back(boost::regex("X-Reflect-host:\\s*.*", boost::regex::icase));
- handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-cache-control:.*", boost::regex::icase));
- handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-pragma:.*", boost::regex::icase));
- handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-range:.*", boost::regex::icase));
- handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-transfer-encoding:.*", boost::regex::icase));
- handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-referer:.*", boost::regex::icase));
- handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-content-type:.*", boost::regex::icase));
- handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-content-encoding:.*", boost::regex::icase));
+ handler.mHeadersRequired.push_back(
+ regex_container_t::value_type(
+ boost::regex("X-Reflect-connection", boost::regex::icase),
+ boost::regex("keep-alive", boost::regex::icase)));
+ handler.mHeadersRequired.push_back(
+ regex_container_t::value_type(
+ boost::regex("X-Reflect-accept", boost::regex::icase),
+ boost::regex("\\*/\\*", boost::regex::icase)));
+ handler.mHeadersRequired.push_back(
+ regex_container_t::value_type(
+ boost::regex("X-Reflect-accept-encoding", boost::regex::icase),
+ boost::regex("((gzip|deflate),\\s*)+(gzip|deflate)", boost::regex::icase))); // close enough
+ handler.mHeadersRequired.push_back(
+ regex_container_t::value_type(
+ boost::regex("X-Reflect-keep-alive", boost::regex::icase),
+ boost::regex("\\d+", boost::regex::icase)));
+ handler.mHeadersRequired.push_back(
+ regex_container_t::value_type(
+ boost::regex("X-Reflect-host", boost::regex::icase),
+ boost::regex(".*", boost::regex::icase)));
+ handler.mHeadersDisallowed.push_back(
+ regex_container_t::value_type(
+ boost::regex("X-Reflect-cache-control", boost::regex::icase),
+ boost::regex(".*", boost::regex::icase)));
+ handler.mHeadersDisallowed.push_back(
+ regex_container_t::value_type(
+ boost::regex("X-Reflect-pragma", boost::regex::icase),
+ boost::regex(".*", boost::regex::icase)));
+ handler.mHeadersDisallowed.push_back(
+ regex_container_t::value_type(
+ boost::regex("X-Reflect-range", boost::regex::icase),
+ boost::regex(".*", boost::regex::icase)));
+ handler.mHeadersDisallowed.push_back(
+ regex_container_t::value_type(
+ boost::regex("X-Reflect-transfer-encoding", boost::regex::icase),
+ boost::regex(".*", boost::regex::icase)));
+ handler.mHeadersDisallowed.push_back(
+ regex_container_t::value_type(
+ boost::regex("X-Reflect-referer", boost::regex::icase),
+ boost::regex(".*", boost::regex::icase)));
+ handler.mHeadersDisallowed.push_back(
+ regex_container_t::value_type(
+ boost::regex("X-Reflect-content-type", boost::regex::icase),
+ boost::regex(".*", boost::regex::icase)));
+ handler.mHeadersDisallowed.push_back(
+ regex_container_t::value_type(
+ boost::regex("X-Reflect-content-encoding", boost::regex::icase),
+ boost::regex(".*", boost::regex::icase)));
HttpHandle handle = req->requestGet(HttpRequest::DEFAULT_POLICY_ID,
0U,
url_base + "reflect/",
@@ -1744,23 +1793,60 @@ void HttpRequestTestObjectType::test<16>()
// Do a texture-style fetch
headers = new HttpHeaders;
- headers->mHeaders.push_back("Accept: image/x-j2c");
+ headers->append("Accept", "image/x-j2c");
mStatus = HttpStatus(200);
handler.mHeadersRequired.clear();
handler.mHeadersDisallowed.clear();
- handler.mHeadersRequired.push_back(boost::regex("X-Reflect-connection:\\s*keep-alive", boost::regex::icase));
- handler.mHeadersRequired.push_back(boost::regex("X-Reflect-accept:\\s*image/x-j2c", boost::regex::icase));
- handler.mHeadersRequired.push_back(boost::regex("X-Reflect-accept-encoding:\\s*((gzip|deflate),\\s*)+(gzip|deflate)", boost::regex::icase)); // close enough
- handler.mHeadersRequired.push_back(boost::regex("X-Reflect-keep-alive:\\s*\\d+", boost::regex::icase));
- handler.mHeadersRequired.push_back(boost::regex("X-Reflect-host:\\s*.*", boost::regex::icase));
- handler.mHeadersRequired.push_back(boost::regex("\\W*X-Reflect-range:.*", boost::regex::icase));
- handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-cache-control:.*", boost::regex::icase));
- handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-pragma:.*", boost::regex::icase));
- handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-transfer-encoding:.*", boost::regex::icase));
- handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-referer:.*", boost::regex::icase));
- handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-content-type:.*", boost::regex::icase));
- handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-content-encoding:.*", boost::regex::icase));
+ handler.mHeadersRequired.push_back(
+ regex_container_t::value_type(
+ boost::regex("X-Reflect-connection", boost::regex::icase),
+ boost::regex("keep-alive", boost::regex::icase)));
+ handler.mHeadersRequired.push_back(
+ regex_container_t::value_type(
+ boost::regex("X-Reflect-accept", boost::regex::icase),
+ boost::regex("image/x-j2c", boost::regex::icase)));
+ handler.mHeadersRequired.push_back(
+ regex_container_t::value_type(
+ boost::regex("X-Reflect-accept-encoding", boost::regex::icase),
+ boost::regex("((gzip|deflate),\\s*)+(gzip|deflate)", boost::regex::icase))); // close enough
+ handler.mHeadersRequired.push_back(
+ regex_container_t::value_type(
+ boost::regex("X-Reflect-keep-alive", boost::regex::icase),
+ boost::regex("\\d+", boost::regex::icase)));
+ handler.mHeadersRequired.push_back(
+ regex_container_t::value_type(
+ boost::regex("X-Reflect-host", boost::regex::icase),
+ boost::regex(".*", boost::regex::icase)));
+ handler.mHeadersRequired.push_back(
+ regex_container_t::value_type(
+ boost::regex("\\W*X-Reflect-range", boost::regex::icase),
+ boost::regex(".*", boost::regex::icase)));
+
+ handler.mHeadersDisallowed.push_back(
+ regex_container_t::value_type(
+ boost::regex("X-Reflect-cache-control", boost::regex::icase),
+ boost::regex(".*", boost::regex::icase)));
+ handler.mHeadersDisallowed.push_back(
+ regex_container_t::value_type(
+ boost::regex("X-Reflect-pragma", boost::regex::icase),
+ boost::regex(".*", boost::regex::icase)));
+ handler.mHeadersDisallowed.push_back(
+ regex_container_t::value_type(
+ boost::regex("X-Reflect-transfer-encoding", boost::regex::icase),
+ boost::regex(".*", boost::regex::icase)));
+ handler.mHeadersDisallowed.push_back(
+ regex_container_t::value_type(
+ boost::regex("X-Reflect-referer", boost::regex::icase),
+ boost::regex(".*", boost::regex::icase)));
+ handler.mHeadersDisallowed.push_back(
+ regex_container_t::value_type(
+ boost::regex("X-Reflect-content-type", boost::regex::icase),
+ boost::regex(".*", boost::regex::icase)));
+ handler.mHeadersDisallowed.push_back(
+ regex_container_t::value_type(
+ boost::regex("X-Reflect-content-encoding", boost::regex::icase),
+ boost::regex(".*", boost::regex::icase)));
handle = req->requestGetByteRange(HttpRequest::DEFAULT_POLICY_ID,
0U,
url_base + "reflect/",
@@ -1901,20 +1987,63 @@ void HttpRequestTestObjectType::test<17>()
// Issue a default POST
mStatus = HttpStatus(200);
- handler.mHeadersRequired.push_back(boost::regex("X-Reflect-connection:\\s*keep-alive", boost::regex::icase));
- handler.mHeadersRequired.push_back(boost::regex("X-Reflect-accept:\\s*\\*/\\*", boost::regex::icase));
- handler.mHeadersRequired.push_back(boost::regex("X-Reflect-accept-encoding:\\s*((gzip|deflate),\\s*)+(gzip|deflate)", boost::regex::icase)); // close enough
- handler.mHeadersRequired.push_back(boost::regex("X-Reflect-keep-alive:\\s*\\d+", boost::regex::icase));
- handler.mHeadersRequired.push_back(boost::regex("X-Reflect-host:\\s*.*", boost::regex::icase));
- handler.mHeadersRequired.push_back(boost::regex("X-Reflect-content-length:\\s*\\d+", boost::regex::icase));
- handler.mHeadersRequired.push_back(boost::regex("X-Reflect-content-type:\\s*application/x-www-form-urlencoded", boost::regex::icase));
- handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-cache-control:.*", boost::regex::icase));
- handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-pragma:.*", boost::regex::icase));
- handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-range:.*", boost::regex::icase));
- handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-referer:.*", boost::regex::icase));
- handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-content-encoding:.*", boost::regex::icase));
- handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-expect:.*", boost::regex::icase));
- handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-transfer-encoding:\\s*.*chunked.*", boost::regex::icase));
+ handler.mHeadersRequired.push_back(
+ regex_container_t::value_type(
+ boost::regex("X-Reflect-connection", boost::regex::icase),
+ boost::regex("keep-alive", boost::regex::icase)));
+ handler.mHeadersRequired.push_back(
+ regex_container_t::value_type(
+ boost::regex("X-Reflect-accept", boost::regex::icase),
+ boost::regex("\\*/\\*", boost::regex::icase)));
+ handler.mHeadersRequired.push_back(
+ regex_container_t::value_type(
+ boost::regex("X-Reflect-accept-encoding", boost::regex::icase),
+ boost::regex("((gzip|deflate),\\s*)+(gzip|deflate)", boost::regex::icase))); // close enough
+ handler.mHeadersRequired.push_back(
+ regex_container_t::value_type(
+ boost::regex("X-Reflect-keep-alive", boost::regex::icase),
+ boost::regex("\\d+", boost::regex::icase)));
+ handler.mHeadersRequired.push_back(
+ regex_container_t::value_type(
+ boost::regex("X-Reflect-host", boost::regex::icase),
+ boost::regex(".*", boost::regex::icase)));
+ handler.mHeadersRequired.push_back(
+ regex_container_t::value_type(
+ boost::regex("X-Reflect-content-length", boost::regex::icase),
+ boost::regex("\\d+", boost::regex::icase)));
+ handler.mHeadersRequired.push_back(
+ regex_container_t::value_type(
+ boost::regex("X-Reflect-content-type", boost::regex::icase),
+ boost::regex("application/x-www-form-urlencoded", boost::regex::icase)));
+
+ handler.mHeadersDisallowed.push_back(
+ regex_container_t::value_type(
+ boost::regex("X-Reflect-cache-control", boost::regex::icase),
+ boost::regex(".*", boost::regex::icase)));
+ handler.mHeadersDisallowed.push_back(
+ regex_container_t::value_type(
+ boost::regex("X-Reflect-pragma", boost::regex::icase),
+ boost::regex(".*", boost::regex::icase)));
+ handler.mHeadersDisallowed.push_back(
+ regex_container_t::value_type(
+ boost::regex("X-Reflect-range", boost::regex::icase),
+ boost::regex(".*", boost::regex::icase)));
+ handler.mHeadersDisallowed.push_back(
+ regex_container_t::value_type(
+ boost::regex("X-Reflect-referer", boost::regex::icase),
+ boost::regex(".*", boost::regex::icase)));
+ handler.mHeadersDisallowed.push_back(
+ regex_container_t::value_type(
+ boost::regex("X-Reflect-content-encoding", boost::regex::icase),
+ boost::regex(".*", boost::regex::icase)));
+ handler.mHeadersDisallowed.push_back(
+ regex_container_t::value_type(
+ boost::regex("X-Reflect-expect", boost::regex::icase),
+ boost::regex(".*", boost::regex::icase)));
+ handler.mHeadersDisallowed.push_back(
+ regex_container_t::value_type(
+ boost::regex("X-Reflect-transfer_encoding", boost::regex::icase),
+ boost::regex(".*chunked.*", boost::regex::icase)));
HttpHandle handle = req->requestPost(HttpRequest::DEFAULT_POLICY_ID,
0U,
url_base + "reflect/",
@@ -2061,20 +2190,64 @@ void HttpRequestTestObjectType::test<18>()
// Issue a default PUT
mStatus = HttpStatus(200);
- handler.mHeadersRequired.push_back(boost::regex("X-Reflect-connection:\\s*keep-alive", boost::regex::icase));
- handler.mHeadersRequired.push_back(boost::regex("X-Reflect-accept:\\s*\\*/\\*", boost::regex::icase));
- handler.mHeadersRequired.push_back(boost::regex("X-Reflect-accept-encoding:\\s*((gzip|deflate),\\s*)+(gzip|deflate)", boost::regex::icase)); // close enough
- handler.mHeadersRequired.push_back(boost::regex("X-Reflect-keep-alive:\\s*\\d+", boost::regex::icase));
- handler.mHeadersRequired.push_back(boost::regex("X-Reflect-host:\\s*.*", boost::regex::icase));
- handler.mHeadersRequired.push_back(boost::regex("X-Reflect-content-length:\\s*\\d+", boost::regex::icase));
- handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-cache-control:.*", boost::regex::icase));
- handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-pragma:.*", boost::regex::icase));
- handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-range:.*", boost::regex::icase));
- handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-referer:.*", boost::regex::icase));
- handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-content-encoding:.*", boost::regex::icase));
- handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-expect:.*", boost::regex::icase));
- handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-transfer-encoding:\\s*.*chunked.*", boost::regex::icase));
- handler.mHeadersDisallowed.push_back(boost::regex("X-Reflect-content-type:.*", boost::regex::icase));
+ handler.mHeadersRequired.push_back(
+ regex_container_t::value_type(
+ boost::regex("X-Reflect-connection", boost::regex::icase),
+ boost::regex("keep-alive", boost::regex::icase)));
+ handler.mHeadersRequired.push_back(
+ regex_container_t::value_type(
+ boost::regex("X-Reflect-accept", boost::regex::icase),
+ boost::regex("\\*/\\*", boost::regex::icase)));
+ handler.mHeadersRequired.push_back(
+ regex_container_t::value_type(
+ boost::regex("X-Reflect-accept-encoding", boost::regex::icase),
+ boost::regex("((gzip|deflate),\\s*)+(gzip|deflate)", boost::regex::icase))); // close enough
+ handler.mHeadersRequired.push_back(
+ regex_container_t::value_type(
+ boost::regex("X-Reflect-keep-alive", boost::regex::icase),
+ boost::regex("\\d+", boost::regex::icase)));
+ handler.mHeadersRequired.push_back(
+ regex_container_t::value_type(
+ boost::regex("X-Reflect-host", boost::regex::icase),
+ boost::regex(".*", boost::regex::icase)));
+ handler.mHeadersRequired.push_back(
+ regex_container_t::value_type(
+ boost::regex("X-Reflect-content-length", boost::regex::icase),
+ boost::regex("\\d+", boost::regex::icase)));
+
+ handler.mHeadersDisallowed.push_back(
+ regex_container_t::value_type(
+ boost::regex("X-Reflect-cache-control", boost::regex::icase),
+ boost::regex(".*", boost::regex::icase)));
+ handler.mHeadersDisallowed.push_back(
+ regex_container_t::value_type(
+ boost::regex("X-Reflect-pragma", boost::regex::icase),
+ boost::regex(".*", boost::regex::icase)));
+ handler.mHeadersDisallowed.push_back(
+ regex_container_t::value_type(
+ boost::regex("X-Reflect-range", boost::regex::icase),
+ boost::regex(".*", boost::regex::icase)));
+ handler.mHeadersDisallowed.push_back(
+ regex_container_t::value_type(
+ boost::regex("X-Reflect-referer", boost::regex::icase),
+ boost::regex(".*", boost::regex::icase)));
+ handler.mHeadersDisallowed.push_back(
+ regex_container_t::value_type(
+ boost::regex("X-Reflect-content-encoding", boost::regex::icase),
+ boost::regex(".*", boost::regex::icase)));
+ handler.mHeadersDisallowed.push_back(
+ regex_container_t::value_type(
+ boost::regex("X-Reflect-expect", boost::regex::icase),
+ boost::regex(".*", boost::regex::icase)));
+ handler.mHeadersDisallowed.push_back(
+ regex_container_t::value_type(
+ boost::regex("X-Reflect-transfer-encoding", boost::regex::icase),
+ boost::regex(".*chunked.*", boost::regex::icase)));
+ handler.mHeadersDisallowed.push_back(
+ regex_container_t::value_type(
+ boost::regex("X-Reflect-content-type", boost::regex::icase),
+ boost::regex(".*", boost::regex::icase)));
+
HttpHandle handle = req->requestPut(HttpRequest::DEFAULT_POLICY_ID,
0U,
url_base + "reflect/",
@@ -2215,27 +2388,73 @@ void HttpRequestTestObjectType::test<19>()
// headers
headers = new HttpHeaders;
- headers->mHeaders.push_back("Keep-Alive: 120");
- headers->mHeaders.push_back("Accept-encoding: deflate");
- headers->mHeaders.push_back("Accept: text/plain");
+ headers->append("Keep-Alive", "120");
+ headers->append("Accept-encoding", "deflate");
+ headers->append("Accept", "text/plain");
// Issue a GET with modified headers
mStatus = HttpStatus(200);
- handler.mHeadersRequired.push_back(boost::regex("X-Reflect-connection:\\s*keep-alive", boost::regex::icase));
- handler.mHeadersRequired.push_back(boost::regex("X-Reflect-accept:\\s*text/plain", boost::regex::icase));
- handler.mHeadersRequired.push_back(boost::regex("X-Reflect-accept-encoding:\\s*deflate", boost::regex::icase)); // close enough
- handler.mHeadersRequired.push_back(boost::regex("X-Reflect-keep-alive:\\s*120", boost::regex::icase));
- handler.mHeadersRequired.push_back(boost::regex("X-Reflect-host:\\s*.*", boost::regex::icase));
- handler.mHeadersDisallowed.push_back(boost::regex("X-Reflect-accept-encoding:\\s*((gzip|deflate),\\s*)+(gzip|deflate)", boost::regex::icase)); // close enough
- handler.mHeadersDisallowed.push_back(boost::regex("X-Reflect-keep-alive:\\s*300", boost::regex::icase));
- handler.mHeadersDisallowed.push_back(boost::regex("X-Reflect-accept:\\s*\\*/\\*", boost::regex::icase));
- handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-cache-control:.*", boost::regex::icase));
- handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-pragma:.*", boost::regex::icase));
- handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-range:.*", boost::regex::icase));
- handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-transfer-encoding:.*", boost::regex::icase));
- handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-referer:.*", boost::regex::icase));
- handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-content-type:.*", boost::regex::icase));
- handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-content-encoding:.*", boost::regex::icase));
+ handler.mHeadersRequired.push_back(
+ regex_container_t::value_type(
+ boost::regex("X-Reflect-connection", boost::regex::icase),
+ boost::regex("keep-alive", boost::regex::icase)));
+ handler.mHeadersRequired.push_back(
+ regex_container_t::value_type(
+ boost::regex("X-Reflect-accept", boost::regex::icase),
+ boost::regex("text/plain", boost::regex::icase)));
+ handler.mHeadersRequired.push_back(
+ regex_container_t::value_type(
+ boost::regex("X-Reflect-accept-encoding", boost::regex::icase),
+ boost::regex("deflate", boost::regex::icase))); // close enough
+ handler.mHeadersRequired.push_back(
+ regex_container_t::value_type(
+ boost::regex("X-Reflect-keep-alive", boost::regex::icase),
+ boost::regex("120", boost::regex::icase)));
+ handler.mHeadersRequired.push_back(
+ regex_container_t::value_type(
+ boost::regex("X-Reflect-host", boost::regex::icase),
+ boost::regex(".*", boost::regex::icase)));
+
+ handler.mHeadersDisallowed.push_back(
+ regex_container_t::value_type(
+ boost::regex("X-Reflect-accept-encoding", boost::regex::icase),
+ boost::regex("((gzip|deflate),\\s*)+(gzip|deflate)", boost::regex::icase))); // close enough
+ handler.mHeadersDisallowed.push_back(
+ regex_container_t::value_type(
+ boost::regex("X-Reflect-keep-alive", boost::regex::icase),
+ boost::regex("300", boost::regex::icase)));
+ handler.mHeadersDisallowed.push_back(
+ regex_container_t::value_type(
+ boost::regex("X-Reflect-accept", boost::regex::icase),
+ boost::regex("\\*/\\*", boost::regex::icase)));
+ handler.mHeadersDisallowed.push_back(
+ regex_container_t::value_type(
+ boost::regex("X-Reflect-cache-control", boost::regex::icase),
+ boost::regex(".*", boost::regex::icase)));
+ handler.mHeadersDisallowed.push_back(
+ regex_container_t::value_type(
+ boost::regex("X-Reflect-pragma", boost::regex::icase),
+ boost::regex(".*", boost::regex::icase)));
+ handler.mHeadersDisallowed.push_back(
+ regex_container_t::value_type(
+ boost::regex("X-Reflect-range", boost::regex::icase),
+ boost::regex(".*", boost::regex::icase)));
+ handler.mHeadersDisallowed.push_back(
+ regex_container_t::value_type(
+ boost::regex("X-Reflect-transfer-encoding", boost::regex::icase),
+ boost::regex(".*", boost::regex::icase)));
+ handler.mHeadersDisallowed.push_back(
+ regex_container_t::value_type(
+ boost::regex("X-Reflect-referer", boost::regex::icase),
+ boost::regex(".*", boost::regex::icase)));
+ handler.mHeadersDisallowed.push_back(
+ regex_container_t::value_type(
+ boost::regex("X-Reflect-content-type", boost::regex::icase),
+ boost::regex(".*", boost::regex::icase)));
+ handler.mHeadersDisallowed.push_back(
+ regex_container_t::value_type(
+ boost::regex("X-Reflect-content-encoding", boost::regex::icase),
+ boost::regex(".*", boost::regex::icase)));
HttpHandle handle = req->requestGet(HttpRequest::DEFAULT_POLICY_ID,
0U,
url_base + "reflect/",
@@ -2368,10 +2587,10 @@ void HttpRequestTestObjectType::test<20>()
// headers
headers = new HttpHeaders();
- headers->mHeaders.push_back("keep-Alive: 120");
- headers->mHeaders.push_back("Accept: text/html");
- headers->mHeaders.push_back("content-type: application/llsd+xml");
- headers->mHeaders.push_back("cache-control: no-store");
+ headers->append("keep-Alive", "120");
+ headers->append("Accept", "text/html");
+ headers->append("content-type", "application/llsd+xml");
+ headers->append("cache-control", "no-store");
// And a buffer array
const char * msg("<xml><llsd><string>It was the best of times, it was the worst of times.</string></llsd></xml>");
@@ -2380,23 +2599,76 @@ void HttpRequestTestObjectType::test<20>()
// Issue a default POST
mStatus = HttpStatus(200);
- handler.mHeadersRequired.push_back(boost::regex("X-Reflect-connection:\\s*keep-alive", boost::regex::icase));
- handler.mHeadersRequired.push_back(boost::regex("X-Reflect-accept:\\s*text/html", boost::regex::icase));
- handler.mHeadersRequired.push_back(boost::regex("X-Reflect-accept-encoding:\\s*((gzip|deflate),\\s*)+(gzip|deflate)", boost::regex::icase)); // close enough
- handler.mHeadersRequired.push_back(boost::regex("X-Reflect-keep-alive:\\s*120", boost::regex::icase));
- handler.mHeadersRequired.push_back(boost::regex("X-Reflect-host:\\s*.*", boost::regex::icase));
- handler.mHeadersRequired.push_back(boost::regex("X-Reflect-content-length:\\s*\\d+", boost::regex::icase));
- handler.mHeadersRequired.push_back(boost::regex("X-Reflect-content-type:\\s*application/llsd\\+xml", boost::regex::icase));
- handler.mHeadersRequired.push_back(boost::regex("\\s*X-Reflect-cache-control:\\s*no-store", boost::regex::icase));
- handler.mHeadersDisallowed.push_back(boost::regex("X-Reflect-content-type:\\s*application/x-www-form-urlencoded", boost::regex::icase));
- handler.mHeadersDisallowed.push_back(boost::regex("X-Reflect-accept:\\s*\\*/\\*", boost::regex::icase));
- handler.mHeadersDisallowed.push_back(boost::regex("X-Reflect-keep-alive:\\s*300", boost::regex::icase));
- handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-pragma:.*", boost::regex::icase));
- handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-range:.*", boost::regex::icase));
- handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-referer:.*", boost::regex::icase));
- handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-content-encoding:.*", boost::regex::icase));
- handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-expect:.*", boost::regex::icase));
- handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-transfer-encoding:\\s*.*chunked.*", boost::regex::icase));
+ handler.mHeadersRequired.push_back(
+ regex_container_t::value_type(
+ boost::regex("X-Reflect-connection", boost::regex::icase),
+ boost::regex("keep-alive", boost::regex::icase)));
+ handler.mHeadersRequired.push_back(
+ regex_container_t::value_type(
+ boost::regex("X-Reflect-accept", boost::regex::icase),
+ boost::regex("text/html", boost::regex::icase)));
+ handler.mHeadersRequired.push_back(
+ regex_container_t::value_type(
+ boost::regex("X-Reflect-accept-encoding", boost::regex::icase),
+ boost::regex("((gzip|deflate),\\s*)+(gzip|deflate)", boost::regex::icase))); // close enough
+ handler.mHeadersRequired.push_back(
+ regex_container_t::value_type(
+ boost::regex("X-Reflect-keep-alive", boost::regex::icase),
+ boost::regex("120", boost::regex::icase)));
+ handler.mHeadersRequired.push_back(
+ regex_container_t::value_type(
+ boost::regex("X-Reflect-host", boost::regex::icase),
+ boost::regex(".*", boost::regex::icase)));
+ handler.mHeadersRequired.push_back(
+ regex_container_t::value_type(
+ boost::regex("X-Reflect-content-length", boost::regex::icase),
+ boost::regex("\\d+", boost::regex::icase)));
+ handler.mHeadersRequired.push_back(
+ regex_container_t::value_type(
+ boost::regex("X-Reflect-content-type", boost::regex::icase),
+ boost::regex("application/llsd\\+xml", boost::regex::icase)));
+ handler.mHeadersRequired.push_back(
+ regex_container_t::value_type(
+ boost::regex("X-Reflect-cache-control", boost::regex::icase),
+ boost::regex("no-store", boost::regex::icase)));
+
+ handler.mHeadersDisallowed.push_back(
+ regex_container_t::value_type(
+ boost::regex("X-Reflect-content-type", boost::regex::icase),
+ boost::regex("application/x-www-form-urlencoded", boost::regex::icase)));
+ handler.mHeadersDisallowed.push_back(
+ regex_container_t::value_type(
+ boost::regex("X-Reflect-accept", boost::regex::icase),
+ boost::regex("\\*/\\*", boost::regex::icase)));
+ handler.mHeadersDisallowed.push_back(
+ regex_container_t::value_type(
+ boost::regex("X-Reflect-keep-alive", boost::regex::icase),
+ boost::regex("300", boost::regex::icase)));
+ handler.mHeadersDisallowed.push_back(
+ regex_container_t::value_type(
+ boost::regex("X-Reflect-pragma", boost::regex::icase),
+ boost::regex(".*", boost::regex::icase)));
+ handler.mHeadersDisallowed.push_back(
+ regex_container_t::value_type(
+ boost::regex("X-Reflect-range", boost::regex::icase),
+ boost::regex(".*", boost::regex::icase)));
+ handler.mHeadersDisallowed.push_back(
+ regex_container_t::value_type(
+ boost::regex("X-Reflect-referer", boost::regex::icase),
+ boost::regex(".*", boost::regex::icase)));
+ handler.mHeadersDisallowed.push_back(
+ regex_container_t::value_type(
+ boost::regex("X-Reflect-content-encoding", boost::regex::icase),
+ boost::regex(".*", boost::regex::icase)));
+ handler.mHeadersDisallowed.push_back(
+ regex_container_t::value_type(
+ boost::regex("X-Reflect-expect", boost::regex::icase),
+ boost::regex(".*", boost::regex::icase)));
+ handler.mHeadersDisallowed.push_back(
+ regex_container_t::value_type(
+ boost::regex("X-Reflect-transfer-encoding", boost::regex::icase),
+ boost::regex(".*", boost::regex::icase)));
+
HttpHandle handle = req->requestPost(HttpRequest::DEFAULT_POLICY_ID,
0U,
url_base + "reflect/",
@@ -2538,9 +2810,9 @@ void HttpRequestTestObjectType::test<21>()
// headers
headers = new HttpHeaders;
- headers->mHeaders.push_back("content-type: text/plain");
- headers->mHeaders.push_back("content-type: text/html");
- headers->mHeaders.push_back("content-type: application/llsd+xml");
+ headers->append("content-type", "text/plain");
+ headers->append("content-type", "text/html");
+ headers->append("content-type", "application/llsd+xml");
// And a buffer array
const char * msg("<xml><llsd><string>It was the best of times, it was the worst of times.</string></llsd></xml>");
@@ -2549,22 +2821,71 @@ void HttpRequestTestObjectType::test<21>()
// Issue a default PUT
mStatus = HttpStatus(200);
- handler.mHeadersRequired.push_back(boost::regex("X-Reflect-connection:\\s*keep-alive", boost::regex::icase));
- handler.mHeadersRequired.push_back(boost::regex("X-Reflect-accept:\\s*\\*/\\*", boost::regex::icase));
- handler.mHeadersRequired.push_back(boost::regex("X-Reflect-accept-encoding:\\s*((gzip|deflate),\\s*)+(gzip|deflate)", boost::regex::icase)); // close enough
- handler.mHeadersRequired.push_back(boost::regex("X-Reflect-keep-alive:\\s*\\d+", boost::regex::icase));
- handler.mHeadersRequired.push_back(boost::regex("X-Reflect-host:\\s*.*", boost::regex::icase));
- handler.mHeadersRequired.push_back(boost::regex("X-Reflect-content-length:\\s*\\d+", boost::regex::icase));
- handler.mHeadersRequired.push_back(boost::regex("X-Reflect-content-type:\\s*application/llsd\\+xml", boost::regex::icase));
- handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-cache-control:.*", boost::regex::icase));
- handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-pragma:.*", boost::regex::icase));
- handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-range:.*", boost::regex::icase));
- handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-referer:.*", boost::regex::icase));
- handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-content-encoding:.*", boost::regex::icase));
- handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-expect:.*", boost::regex::icase));
- handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-transfer-encoding:\\s*.*chunked.*", boost::regex::icase));
- handler.mHeadersDisallowed.push_back(boost::regex("X-Reflect-content-type:\\s*text/plain", boost::regex::icase));
- handler.mHeadersDisallowed.push_back(boost::regex("X-Reflect-content-type:\\s*text/html", boost::regex::icase));
+ handler.mHeadersRequired.push_back(
+ regex_container_t::value_type(
+ boost::regex("X-Reflect-connection", boost::regex::icase),
+ boost::regex("keep-alive", boost::regex::icase)));
+ handler.mHeadersRequired.push_back(
+ regex_container_t::value_type(
+ boost::regex("X-Reflect-accept", boost::regex::icase),
+ boost::regex("\\*/\\*", boost::regex::icase)));
+ handler.mHeadersRequired.push_back(
+ regex_container_t::value_type(
+ boost::regex("X-Reflect-accept-encoding", boost::regex::icase),
+ boost::regex("((gzip|deflate),\\s*)+(gzip|deflate)", boost::regex::icase))); // close enough
+ handler.mHeadersRequired.push_back(
+ regex_container_t::value_type(
+ boost::regex("X-Reflect-keep-alive", boost::regex::icase),
+ boost::regex("\\d+", boost::regex::icase)));
+ handler.mHeadersRequired.push_back(
+ regex_container_t::value_type(
+ boost::regex("X-Reflect-host", boost::regex::icase),
+ boost::regex(".*", boost::regex::icase)));
+ handler.mHeadersRequired.push_back(
+ regex_container_t::value_type(
+ boost::regex("X-Reflect-content-length", boost::regex::icase),
+ boost::regex("\\d+", boost::regex::icase)));
+ handler.mHeadersRequired.push_back(
+ regex_container_t::value_type(
+ boost::regex("X-Reflect-content-type", boost::regex::icase),
+ boost::regex("application/llsd\\+xml", boost::regex::icase)));
+
+ handler.mHeadersDisallowed.push_back(
+ regex_container_t::value_type(
+ boost::regex("X-Reflect-cache-control", boost::regex::icase),
+ boost::regex(".*", boost::regex::icase)));
+ handler.mHeadersDisallowed.push_back(
+ regex_container_t::value_type(
+ boost::regex("X-Reflect-pragma", boost::regex::icase),
+ boost::regex(".*", boost::regex::icase)));
+ handler.mHeadersDisallowed.push_back(
+ regex_container_t::value_type(
+ boost::regex("X-Reflect-range", boost::regex::icase),
+ boost::regex(".*", boost::regex::icase)));
+ handler.mHeadersDisallowed.push_back(
+ regex_container_t::value_type(
+ boost::regex("X-Reflect-referer", boost::regex::icase),
+ boost::regex(".*", boost::regex::icase)));
+ handler.mHeadersDisallowed.push_back(
+ regex_container_t::value_type(
+ boost::regex("X-Reflect-content-encoding", boost::regex::icase),
+ boost::regex(".*", boost::regex::icase)));
+ handler.mHeadersDisallowed.push_back(
+ regex_container_t::value_type(
+ boost::regex("X-Reflect-expect", boost::regex::icase),
+ boost::regex(".*", boost::regex::icase)));
+ handler.mHeadersDisallowed.push_back(
+ regex_container_t::value_type(
+ boost::regex("X-Reflect-transfer-encoding", boost::regex::icase),
+ boost::regex(".*", boost::regex::icase)));
+ handler.mHeadersDisallowed.push_back(
+ regex_container_t::value_type(
+ boost::regex("X-Reflect-content-type", boost::regex::icase),
+ boost::regex("text/plain", boost::regex::icase)));
+ handler.mHeadersDisallowed.push_back(
+ regex_container_t::value_type(
+ boost::regex("X-Reflect-content-type", boost::regex::icase),
+ boost::regex("text/html", boost::regex::icase)));
HttpHandle handle = req->requestPut(HttpRequest::DEFAULT_POLICY_ID,
0U,
url_base + "reflect/",
@@ -2854,6 +3175,142 @@ void HttpRequestTestObjectType::test<22>()
}
}
+template <> template <>
+void HttpRequestTestObjectType::test<23>()
+{
+ ScopedCurlInit ready;
+
+ set_test_name("HttpRequest GET 503s with 'Retry-After'");
+
+ // This tests mainly that the code doesn't fall over if
+ // various well- and mis-formed Retry-After headers are
+ // sent along with the response. Direct inspection of
+ // the parsing result isn't supported.
+
+ // Handler can be stack-allocated *if* there are no dangling
+ // references to it after completion of this method.
+ // Create before memory record as the string copy will bump numbers.
+ TestHandler2 handler(this, "handler");
+ std::string url_base(get_base_url() + "/503/"); // path to 503 generators
+
+ // record the total amount of dynamically allocated memory
+ mMemTotal = GetMemTotal();
+ mHandlerCalls = 0;
+
+ HttpRequest * req = NULL;
+ HttpOptions * opts = NULL;
+
+ try
+ {
+ // Get singletons created
+ HttpRequest::createService();
+
+ // Start threading early so that thread memory is invariant
+ // over the test.
+ HttpRequest::startThread();
+
+ // create a new ref counted object with an implicit reference
+ req = new HttpRequest();
+ ensure("Memory allocated on construction", mMemTotal < GetMemTotal());
+
+ opts = new HttpOptions();
+ opts->setRetries(1); // Retry once only
+ opts->setUseRetryAfter(true); // Try to parse the retry-after header
+
+ // Issue a GET that 503s with valid retry-after
+ mStatus = HttpStatus(503);
+ int url_limit(6);
+ for (int i(0); i < url_limit; ++i)
+ {
+ std::ostringstream url;
+ url << url_base << i << "/";
+ HttpHandle handle = req->requestGetByteRange(HttpRequest::DEFAULT_POLICY_ID,
+ 0U,
+ url.str(),
+ 0,
+ 0,
+ opts,
+ NULL,
+ &handler);
+
+ std::ostringstream testtag;
+ testtag << "Valid handle returned for 503 request #" << i;
+ ensure(testtag.str(), handle != LLCORE_HTTP_HANDLE_INVALID);
+ }
+
+
+ // Run the notification pump.
+ int count(0);
+ int limit(LOOP_COUNT_LONG);
+ while (count++ < limit && mHandlerCalls < url_limit)
+ {
+ req->update(0);
+ usleep(LOOP_SLEEP_INTERVAL);
+ }
+ ensure("Request executed in reasonable time", count < limit);
+ ensure("One handler invocation for request", mHandlerCalls == url_limit);
+
+ // Okay, request a shutdown of the servicing thread
+ mStatus = HttpStatus();
+ mHandlerCalls = 0;
+ HttpHandle handle = req->requestStopThread(&handler);
+ ensure("Valid handle returned for second request", handle != LLCORE_HTTP_HANDLE_INVALID);
+
+ // Run the notification pump again
+ count = 0;
+ limit = LOOP_COUNT_LONG;
+ while (count++ < limit && mHandlerCalls < 1)
+ {
+ req->update(1000000);
+ usleep(LOOP_SLEEP_INTERVAL);
+ }
+ ensure("Second request executed in reasonable time", count < limit);
+ ensure("Second handler invocation", mHandlerCalls == 1);
+
+ // See that we actually shutdown the thread
+ count = 0;
+ limit = LOOP_COUNT_SHORT;
+ while (count++ < limit && ! HttpService::isStopped())
+ {
+ usleep(LOOP_SLEEP_INTERVAL);
+ }
+ ensure("Thread actually stopped running", HttpService::isStopped());
+
+ // release options
+ opts->release();
+ opts = NULL;
+
+ // release the request object
+ delete req;
+ req = NULL;
+
+ // Shut down service
+ HttpRequest::destroyService();
+
+#if defined(WIN32)
+ // Can only do this memory test on Windows. On other platforms,
+ // the LL logging system holds on to memory and produces what looks
+ // like memory leaks...
+
+ // printf("Old mem: %d, New mem: %d\n", mMemTotal, GetMemTotal());
+ ensure("Memory usage back to that at entry", mMemTotal == GetMemTotal());
+#endif
+ }
+ catch (...)
+ {
+ stop_thread(req);
+ if (opts)
+ {
+ opts->release();
+ opts = NULL;
+ }
+ delete req;
+ HttpRequest::destroyService();
+ throw;
+ }
+}
+
+
} // end namespace tut
namespace
diff --git a/indra/llcorehttp/tests/test_httpstatus.hpp b/indra/llcorehttp/tests/test_httpstatus.hpp
index b5538528c5..0b379836c9 100755
--- a/indra/llcorehttp/tests/test_httpstatus.hpp
+++ b/indra/llcorehttp/tests/test_httpstatus.hpp
@@ -259,6 +259,65 @@ void HttpStatusTestObjectType::test<7>()
ensure(msg == "Unknown error");
}
+
+template <> template <>
+void HttpStatusTestObjectType::test<8>()
+{
+ set_test_name("HttpStatus toHex() nominal function");
+
+ HttpStatus status(404);
+ std::string msg = status.toHex();
+ // std::cout << "Result: " << msg << std::endl;
+ ensure(msg == "01940001");
+}
+
+
+template <> template <>
+void HttpStatusTestObjectType::test<9>()
+{
+ set_test_name("HttpStatus toTerseString() nominal function");
+
+ HttpStatus status(404);
+ std::string msg = status.toTerseString();
+ // std::cout << "Result: " << msg << std::endl;
+ ensure("Normal HTTP 404", msg == "Http_404");
+
+ status = HttpStatus(200);
+ msg = status.toTerseString();
+ // std::cout << "Result: " << msg << std::endl;
+ ensure("Normal HTTP 200", msg == "Http_200");
+
+ status = HttpStatus(200, HE_REPLY_ERROR);
+ msg = status.toTerseString();
+ // std::cout << "Result: " << msg << std::endl;
+ ensure("Unsuccessful HTTP 200", msg == "Http_200"); // No distinction for error
+
+ status = HttpStatus(HttpStatus::EXT_CURL_EASY, CURLE_COULDNT_CONNECT);
+ msg = status.toTerseString();
+ // std::cout << "Result: " << msg << std::endl;
+ ensure("Easy couldn't connect error", msg == "Easy_7");
+
+ status = HttpStatus(HttpStatus::EXT_CURL_MULTI, CURLM_OUT_OF_MEMORY);
+ msg = status.toTerseString();
+ // std::cout << "Result: " << msg << std::endl;
+ ensure("Multi out-of-memory error", msg == "Multi_3");
+
+ status = HttpStatus(HttpStatus::LLCORE, HE_OPT_NOT_SET);
+ msg = status.toTerseString();
+ // std::cout << "Result: " << msg << std::endl;
+ ensure("Core option not set error", msg == "Core_7");
+
+ status = HttpStatus(22000, 1);
+ msg = status.toTerseString();
+ // std::cout << "Result: " << msg << std::endl;
+ ensure("Undecodable error", msg == "Unknown_1");
+
+ status = HttpStatus(22000, -1);
+ msg = status.toTerseString();
+ // std::cout << "Result: " << msg << std::endl;
+ ensure("Undecodable error 65535", msg == "Unknown_65535");
+}
+
} // end namespace tut
#endif // TEST_HTTP_STATUS_H
diff --git a/indra/llcorehttp/tests/test_llcorehttp_peer.py b/indra/llcorehttp/tests/test_llcorehttp_peer.py
index 3c3af8dc75..04cde651c4 100755
--- a/indra/llcorehttp/tests/test_llcorehttp_peer.py
+++ b/indra/llcorehttp/tests/test_llcorehttp_peer.py
@@ -69,6 +69,15 @@ class TestHTTPRequestHandler(BaseHTTPRequestHandler):
"Content-Range: bytes 0-75/2983",
"Content-Length: 76"
-- '/bug2295/inv_cont_range/0/' Generates HE_INVALID_CONTENT_RANGE error in llcorehttp.
+ - '/503/' Generate 503 responses with various kinds
+ of 'retry-after' headers
+ -- '/503/0/' "Retry-After: 2"
+ -- '/503/1/' "Retry-After: Thu, 31 Dec 2043 23:59:59 GMT"
+ -- '/503/2/' "Retry-After: Fri, 31 Dec 1999 23:59:59 GMT"
+ -- '/503/3/' "Retry-After: "
+ -- '/503/4/' "Retry-After: (*#*(@*(@(")"
+ -- '/503/5/' "Retry-After: aklsjflajfaklsfaklfasfklasdfklasdgahsdhgasdiogaioshdgo"
+ -- '/503/6/' "Retry-After: 1 2 3 4 5 6 7 8 9 10"
Some combinations make no sense, there's no effort to protect
you from that.
@@ -143,22 +152,40 @@ class TestHTTPRequestHandler(BaseHTTPRequestHandler):
if "/sleep/" in self.path:
time.sleep(30)
- if "fail" in self.path:
- status = data.get("status", 500)
- # self.responses maps an int status to a (short, long) pair of
- # strings. We want the longer string. That's why we pass a string
- # pair to get(): the [1] will select the second string, whether it
- # came from self.responses or from our default pair.
- reason = data.get("reason",
- self.responses.get(status,
- ("fail requested",
- "Your request specified failure status %s "
- "without providing a reason" % status))[1])
- debug("fail requested: %s: %r", status, reason)
- self.send_error(status, reason)
+ if "/503/" in self.path:
+ # Tests for various kinds of 'Retry-After' header parsing
+ body = None
+ if "/503/0/" in self.path:
+ self.send_response(503)
+ self.send_header("retry-after", "2")
+ elif "/503/1/" in self.path:
+ self.send_response(503)
+ self.send_header("retry-after", "Thu, 31 Dec 2043 23:59:59 GMT")
+ elif "/503/2/" in self.path:
+ self.send_response(503)
+ self.send_header("retry-after", "Fri, 31 Dec 1999 23:59:59 GMT")
+ elif "/503/3/" in self.path:
+ self.send_response(503)
+ self.send_header("retry-after", "")
+ elif "/503/4/" in self.path:
+ self.send_response(503)
+ self.send_header("retry-after", "(*#*(@*(@(")
+ elif "/503/5/" in self.path:
+ self.send_response(503)
+ self.send_header("retry-after", "aklsjflajfaklsfaklfasfklasdfklasdgahsdhgasdiogaioshdgo")
+ elif "/503/6/" in self.path:
+ self.send_response(503)
+ self.send_header("retry-after", "1 2 3 4 5 6 7 8 9 10")
+ else:
+ # Unknown request
+ self.send_response(400)
+ body = "Unknown /503/ path in server"
if "/reflect/" in self.path:
self.reflect_headers()
+ self.send_header("Content-type", "text/plain")
self.end_headers()
+ if body:
+ self.wfile.write(body)
elif "/bug2295/" in self.path:
# Test for https://jira.secondlife.com/browse/BUG-2295
#
@@ -194,8 +221,7 @@ class TestHTTPRequestHandler(BaseHTTPRequestHandler):
self.end_headers()
if body:
self.wfile.write(body)
- else:
- # Normal response path
+ elif "fail" not in self.path:
data = data.copy() # we're going to modify
# Ensure there's a "reply" key in data, even if there wasn't before
data["reply"] = data.get("reply", llsd.LLSD("success"))
@@ -210,6 +236,22 @@ class TestHTTPRequestHandler(BaseHTTPRequestHandler):
self.end_headers()
if withdata:
self.wfile.write(response)
+ else: # fail requested
+ status = data.get("status", 500)
+ # self.responses maps an int status to a (short, long) pair of
+ # strings. We want the longer string. That's why we pass a string
+ # pair to get(): the [1] will select the second string, whether it
+ # came from self.responses or from our default pair.
+ reason = data.get("reason",
+ self.responses.get(status,
+ ("fail requested",
+ "Your request specified failure status %s "
+ "without providing a reason" % status))[1])
+ debug("fail requested: %s: %r", status, reason)
+ self.send_error(status, reason)
+ if "/reflect/" in self.path:
+ self.reflect_headers()
+ self.end_headers()
def reflect_headers(self):
for name in self.headers.keys():