diff options
author | Monty Brandenberg <monty@lindenlab.com> | 2012-07-23 23:40:07 +0000 |
---|---|---|
committer | Monty Brandenberg <monty@lindenlab.com> | 2012-07-23 23:40:07 +0000 |
commit | 85e69b043b098dbe5a09f2eac6ff541123089f13 (patch) | |
tree | e0f5cc824c41563ea8b780cc4ba0cdb8c701c3f8 /indra | |
parent | 334ce2556f0d51c38a76d655084ae1d4671f6aec (diff) |
Big comment and naming cleanup. Ready for prime-time.
Add to-do list to _httpinternal.h to guide anyone who
wants to pitch in and help.
Diffstat (limited to 'indra')
-rw-r--r-- | indra/llcorehttp/_httpinternal.h | 93 | ||||
-rw-r--r-- | indra/llcorehttp/_httplibcurl.cpp | 13 | ||||
-rw-r--r-- | indra/llcorehttp/_httplibcurl.h | 27 | ||||
-rw-r--r-- | indra/llcorehttp/_httpopcancel.cpp | 7 | ||||
-rw-r--r-- | indra/llcorehttp/_httpopcancel.h | 5 | ||||
-rw-r--r-- | indra/llcorehttp/_httpoperation.cpp | 8 | ||||
-rw-r--r-- | indra/llcorehttp/_httpoperation.h | 77 | ||||
-rw-r--r-- | indra/llcorehttp/_httpoprequest.cpp | 47 | ||||
-rw-r--r-- | indra/llcorehttp/_httpoprequest.h | 26 | ||||
-rw-r--r-- | indra/llcorehttp/_httpopsetget.cpp | 8 | ||||
-rw-r--r-- | indra/llcorehttp/_httpopsetget.h | 3 | ||||
-rw-r--r-- | indra/llcorehttp/_httppolicy.cpp | 12 | ||||
-rw-r--r-- | indra/llcorehttp/_httppolicy.h | 39 | ||||
-rw-r--r-- | indra/llcorehttp/_httppolicyclass.cpp | 8 | ||||
-rw-r--r-- | indra/llcorehttp/_httppolicyglobal.cpp | 8 | ||||
-rw-r--r-- | indra/llcorehttp/_httpreadyqueue.h | 10 | ||||
-rw-r--r-- | indra/llcorehttp/_httpservice.cpp | 24 | ||||
-rw-r--r-- | indra/llcorehttp/_httpservice.h | 6 | ||||
-rw-r--r-- | indra/llcorehttp/httpcommon.h | 18 | ||||
-rw-r--r-- | indra/llcorehttp/httpoptions.cpp | 6 |
20 files changed, 313 insertions, 132 deletions
diff --git a/indra/llcorehttp/_httpinternal.h b/indra/llcorehttp/_httpinternal.h index 97ec5ee1d6..465e2036b3 100644 --- a/indra/llcorehttp/_httpinternal.h +++ b/indra/llcorehttp/_httpinternal.h @@ -32,12 +32,65 @@ // something wrong is probably happening. +// -------------------------------------------------------------------- +// General library to-do list +// +// - Implement policy classes. Structure is mostly there just didn't +// need it for the first consumer. +// - Consider Removing 'priority' from the request interface. Its use +// in an always active class can lead to starvation of low-priority +// requests. Requires coodination of priority values across all +// components that share a class. Changing priority across threads +// is slightly expensive (relative to gain) and hasn't been completely +// implemented. And the major user of priority, texture fetches, +// may not really need it. +// - Set/get for global policy and policy classes is clumsy. Rework +// it heading in a direction that allows for more dynamic behavior. +// - Move HttpOpRequest::prepareRequest() to HttpLibcurl for the +// pedantic. +// - Update downloader and other long-duration services are going to +// need a progress notification. Initial idea is to introduce a +// 'repeating request' which can piggyback on another request and +// persist until canceled or carrier completes. Current queue +// structures allow an HttpOperation object to be enqueued +// repeatedly, so... +// - Investigate making c-ares' re-implementation of a resolver library +// more resilient or more intelligent on Mac. Part of the DNS failure +// lies in here. The mechanism also looks a little less dynamic +// than needed in an environments where networking is changing. +// - Global optimizations: 'borrowing' connections from other classes, +// HTTP pipelining. +// - Dynamic/control system stuff: detect problems and self-adjust. +// This won't help in the face of the router problems we've looked +// at, however. Detect starvation due to UDP activity and provide +// feedback to it. +// +// Integration to-do list +// - LLTextureFetch still needs a major refactor. The use of +// LLQueuedThread makes it hard to inspect workers and do the +// resource waiting we're now doing. Rebuild along simpler lines +// some of which are suggested in new commentary at the top of +// the main source file. +// - Expand areas of usage eventually leading to the removal of LLCurl. +// Rough order of expansion: +// . Mesh fetch +// . Avatar names +// . Group membership lists +// . Caps access in general +// . 'The rest' +// - Adapt texture cache, image decode and other image consumers to +// the BufferArray model to reduce data copying. Alternatively, +// adapt this library to something else. +// +// -------------------------------------------------------------------- + + // If '1', internal ready queues will not order ready // requests by priority, instead it's first-come-first-served. // Reprioritization requests have the side-effect of then // putting the modified request at the back of the ready queue. -#define LLCORE_READY_QUEUE_IGNORES_PRIORITY 1 +#define LLCORE_HTTP_READY_QUEUE_IGNORES_PRIORITY 1 namespace LLCore @@ -45,48 +98,48 @@ namespace LLCore // Maxium number of policy classes that can be defined. // *TODO: Currently limited to the default class, extend. -const int POLICY_CLASS_LIMIT = 1; +const int HTTP_POLICY_CLASS_LIMIT = 1; // Debug/informational tracing. Used both // as a global option and in per-request traces. -const int TRACE_OFF = 0; -const int TRACE_LOW = 1; -const int TRACE_CURL_HEADERS = 2; -const int TRACE_CURL_BODIES = 3; +const int HTTP_TRACE_OFF = 0; +const int HTTP_TRACE_LOW = 1; +const int HTTP_TRACE_CURL_HEADERS = 2; +const int HTTP_TRACE_CURL_BODIES = 3; -const int TRACE_MIN = TRACE_OFF; -const int TRACE_MAX = TRACE_CURL_BODIES; +const int HTTP_TRACE_MIN = HTTP_TRACE_OFF; +const int HTTP_TRACE_MAX = HTTP_TRACE_CURL_BODIES; // Request retry limits -const int DEFAULT_RETRY_COUNT = 5; -const int LIMIT_RETRY_MIN = 0; -const int LIMIT_RETRY_MAX = 100; +const int HTTP_RETRY_COUNT_DEFAULT = 5; +const int HTTP_RETRY_COUNT_MIN = 0; +const int HTTP_RETRY_COUNT_MAX = 100; -const int DEFAULT_HTTP_REDIRECTS = 10; +const int HTTP_REDIRECTS_DEFAULT = 10; // Timeout value used for both connect and protocol exchange. // Retries and time-on-queue are not included and aren't // accounted for. -const long DEFAULT_TIMEOUT = 30L; -const long LIMIT_TIMEOUT_MIN = 0L; -const long LIMIT_TIMEOUT_MAX = 3600L; +const long HTTP_REQUEST_TIMEOUT_DEFAULT = 30L; +const long HTTP_REQUEST_TIMEOUT_MIN = 0L; +const long HTTP_REQUEST_TIMEOUT_MAX = 3600L; // Limits on connection counts -const int DEFAULT_CONNECTIONS = 8; -const int LIMIT_CONNECTIONS_MIN = 1; -const int LIMIT_CONNECTIONS_MAX = 256; +const int HTTP_CONNECTION_LIMIT_DEFAULT = 8; +const int HTTP_CONNECTION_LIMIT_MIN = 1; +const int HTTP_CONNECTION_LIMIT_MAX = 256; // Tuning parameters // Time worker thread sleeps after a pass through the // request, ready and active queues. -const int LOOP_SLEEP_NORMAL_MS = 2; +const int HTTP_SERVICE_LOOP_SLEEP_NORMAL_MS = 2; // Block allocation size (a tuning parameter) is found // in bufferarray.h. // Compatibility controls -const bool ENABLE_LINKSYS_WRT54G_V5_DNS_FIX = true; +const bool HTTP_ENABLE_LINKSYS_WRT54G_V5_DNS_FIX = true; } // end namespace LLCore diff --git a/indra/llcorehttp/_httplibcurl.cpp b/indra/llcorehttp/_httplibcurl.cpp index e031efbc91..4e2e3f0e0e 100644 --- a/indra/llcorehttp/_httplibcurl.cpp +++ b/indra/llcorehttp/_httplibcurl.cpp @@ -85,7 +85,7 @@ void HttpLibcurl::shutdown() void HttpLibcurl::start(int policy_count) { - llassert_always(policy_count <= POLICY_CLASS_LIMIT); + llassert_always(policy_count <= HTTP_POLICY_CLASS_LIMIT); llassert_always(! mMultiHandles); // One-time call only mPolicyCount = policy_count; @@ -156,6 +156,7 @@ HttpService::ELoopSpeed HttpLibcurl::processTransport() } +// Caller has provided us with a ref count on op. void HttpLibcurl::addOp(HttpOpRequest * op) { llassert_always(op->mReqPolicy < mPolicyCount); @@ -165,7 +166,7 @@ void HttpLibcurl::addOp(HttpOpRequest * op) if (! op->prepareRequest(mService)) { // Couldn't issue request, fail with notification - // *FIXME: Need failure path + // *TODO: Need failure path return; } @@ -173,7 +174,7 @@ void HttpLibcurl::addOp(HttpOpRequest * op) curl_multi_add_handle(mMultiHandles[op->mReqPolicy], op->mCurlHandle); op->mCurlActive = true; - if (op->mTracing > TRACE_OFF) + if (op->mTracing > HTTP_TRACE_OFF) { HttpPolicy & policy(mService->getPolicy()); @@ -215,7 +216,7 @@ bool HttpLibcurl::cancel(HttpHandle handle) // *NOTE: cancelRequest logic parallels completeRequest logic. // Keep them synchronized as necessary. Caller is expected to -// remove to op from the active list and release the op *after* +// remove the op from the active list and release the op *after* // calling this method. It must be called first to deliver the // op to the reply queue with refcount intact. void HttpLibcurl::cancelRequest(HttpOpRequest * op) @@ -229,7 +230,7 @@ void HttpLibcurl::cancelRequest(HttpOpRequest * op) op->mCurlHandle = NULL; // Tracing - if (op->mTracing > TRACE_OFF) + if (op->mTracing > HTTP_TRACE_OFF) { LL_INFOS("CoreHttp") << "TRACE, RequestCanceled, Handle: " << static_cast<HttpHandle>(op) @@ -305,7 +306,7 @@ bool HttpLibcurl::completeRequest(CURLM * multi_handle, CURL * handle, CURLcode op->mCurlHandle = NULL; // Tracing - if (op->mTracing > TRACE_OFF) + if (op->mTracing > HTTP_TRACE_OFF) { LL_INFOS("CoreHttp") << "TRACE, RequestComplete, Handle: " << static_cast<HttpHandle>(op) diff --git a/indra/llcorehttp/_httplibcurl.h b/indra/llcorehttp/_httplibcurl.h index 53972b1ffa..611f029ef5 100644 --- a/indra/llcorehttp/_httplibcurl.h +++ b/indra/llcorehttp/_httplibcurl.h @@ -68,24 +68,41 @@ public: /// Give cycles to libcurl to run active requests. Completed /// operations (successful or failed) will be retried or handed /// over to the reply queue as final responses. + /// + /// @return Indication of how long this method is + /// willing to wait for next service call. HttpService::ELoopSpeed processTransport(); /// Add request to the active list. Caller is expected to have - /// provided us with a reference count to hold the request. (No - /// additional references will be added.) + /// provided us with a reference count on the op to hold the + /// request. (No additional references will be added.) void addOp(HttpOpRequest * op); /// One-time call to set the number of policy classes to be /// serviced and to create the resources for each. Value /// must agree with HttpPolicy::setPolicies() call. void start(int policy_count); - + + /// Synchronously stop libcurl operations. All active requests + /// are canceled and removed from libcurl's handling. Easy + /// handles are detached from their multi handles and released. + /// Multi handles are also released. Canceled requests are + /// completed with canceled status and made available on their + /// respective reply queues. + /// + /// Can be restarted with a start() call. void shutdown(); - + + /// Return global and per-class counts of active requests. int getActiveCount() const; int getActiveCountInClass(int policy_class) const; - // Shadows HttpService's method + /// Attempt to cancel a request identified by handle. + /// + /// Interface shadows HttpService's method. + /// + /// @return True if handle was found and operation canceled. + /// bool cancel(HttpHandle handle); protected: diff --git a/indra/llcorehttp/_httpopcancel.cpp b/indra/llcorehttp/_httpopcancel.cpp index 8e1105dc81..c1912eb3db 100644 --- a/indra/llcorehttp/_httpopcancel.cpp +++ b/indra/llcorehttp/_httpopcancel.cpp @@ -26,18 +26,11 @@ #include "_httpopcancel.h" -#include <cstdio> -#include <algorithm> - #include "httpcommon.h" #include "httphandler.h" #include "httpresponse.h" -#include "_httprequestqueue.h" -#include "_httpreplyqueue.h" #include "_httpservice.h" -#include "_httppolicy.h" -#include "_httplibcurl.h" namespace LLCore diff --git a/indra/llcorehttp/_httpopcancel.h b/indra/llcorehttp/_httpopcancel.h index 659d28955f..336dfdc573 100644 --- a/indra/llcorehttp/_httpopcancel.h +++ b/indra/llcorehttp/_httpopcancel.h @@ -46,11 +46,14 @@ namespace LLCore /// be canceled, if possible. This includes active requests /// that may be in the middle of an HTTP transaction. Any /// completed request will not be canceled and will return -/// its final status unchanged. +/// its final status unchanged and *this* request will complete +/// with an HE_HANDLE_NOT_FOUND error status. class HttpOpCancel : public HttpOperation { public: + /// @param handle Handle of previously-issued request to + /// be canceled. HttpOpCancel(HttpHandle handle); protected: diff --git a/indra/llcorehttp/_httpoperation.cpp b/indra/llcorehttp/_httpoperation.cpp index 910dbf1f2f..5cf5bc5930 100644 --- a/indra/llcorehttp/_httpoperation.cpp +++ b/indra/llcorehttp/_httpoperation.cpp @@ -94,7 +94,7 @@ void HttpOperation::stageFromRequest(HttpService *) // Default implementation should never be called. This // indicates an operation making a transition that isn't // defined. - LL_ERRS("HttpCore") << "Default stateFromRequest method may not be called." + LL_ERRS("HttpCore") << "Default stageFromRequest method may not be called." << LL_ENDL; } @@ -104,7 +104,7 @@ void HttpOperation::stageFromReady(HttpService *) // Default implementation should never be called. This // indicates an operation making a transition that isn't // defined. - LL_ERRS("HttpCore") << "Default stateFromReady method may not be called." + LL_ERRS("HttpCore") << "Default stageFromReady method may not be called." << LL_ENDL; } @@ -114,7 +114,7 @@ void HttpOperation::stageFromActive(HttpService *) // Default implementation should never be called. This // indicates an operation making a transition that isn't // defined. - LL_ERRS("HttpCore") << "Default stateFromActive method may not be called." + LL_ERRS("HttpCore") << "Default stageFromActive method may not be called." << LL_ENDL; } @@ -143,7 +143,7 @@ HttpStatus HttpOperation::cancel() void HttpOperation::addAsReply() { - if (mTracing > TRACE_OFF) + if (mTracing > HTTP_TRACE_OFF) { LL_INFOS("CoreHttp") << "TRACE, ToReplyQueue, Handle: " << static_cast<HttpHandle>(this) diff --git a/indra/llcorehttp/_httpoperation.h b/indra/llcorehttp/_httpoperation.h index 717a9b0d72..914627fad0 100644 --- a/indra/llcorehttp/_httpoperation.h +++ b/indra/llcorehttp/_httpoperation.h @@ -72,9 +72,11 @@ class HttpService; class HttpOperation : public LLCoreInt::RefCounted { public: + /// Threading: called by a consumer/application thread. HttpOperation(); protected: + /// Threading: called by any thread. virtual ~HttpOperation(); // Use release() private: @@ -82,28 +84,87 @@ private: void operator=(const HttpOperation &); // Not defined public: + /// Register a reply queue and a handler for completion notifications. + /// + /// Invokers of operations that want to receive notification that an + /// operation has been completed do so by binding a reply queue and + /// a handler object to the request. + /// + /// @param reply_queue Pointer to the reply queue where completion + /// notifications are to be queued (typically + /// by addAsReply()). This will typically be + /// the reply queue referenced by the request + /// object. This method will increment the + /// refcount on the queue holding the queue + /// until delivery is complete. Using a reply_queue + /// even if the handler is NULL has some benefits + /// for memory deallocation by keeping it in the + /// originating thread. + /// + /// @param handler Possibly NULL pointer to a non-refcounted + //// handler object to be invoked (onCompleted) + /// when the operation is finished. Note that + /// the handler object is never dereferenced + /// by the worker thread. This is passible data + /// until notification is performed. + /// + /// Threading: called by application thread. + /// void setReplyPath(HttpReplyQueue * reply_queue, HttpHandler * handler); - HttpHandler * getUserHandler() const - { - return mUserHandler; - } - + /// The three possible staging steps in an operation's lifecycle. + /// Asynchronous requests like HTTP operations move from the + /// request queue to the ready queue via stageFromRequest. Then + /// from the ready queue to the active queue by stageFromReady. And + /// when complete, to the reply queue via stageFromActive and the + /// addAsReply utility. + /// + /// Immediate mode operations (everything else) move from the + /// request queue to the reply queue directly via stageFromRequest + /// and addAsReply with no existence on the ready or active queues. + /// + /// These methods will take out a reference count on the request, + /// caller only needs to dispose of its reference when done with + /// the request. + /// + /// Threading: called by worker thread. + /// virtual void stageFromRequest(HttpService *); virtual void stageFromReady(HttpService *); virtual void stageFromActive(HttpService *); + /// Delivers a notification to a handler object on completion. + /// + /// Once a request is complete and it has been removed from its + /// reply queue, a handler notification may be delivered by a + /// call to HttpRequest::update(). This method does the necessary + /// dispatching. + /// + /// Threading: called by application thread. + /// virtual void visitNotifier(HttpRequest *); - + + /// Cancels the operation whether queued or active. + /// Final status of the request becomes canceled (an error) and + /// that will be delivered to caller via notification scheme. + /// + /// Threading: called by worker thread. + /// virtual HttpStatus cancel(); protected: + /// Delivers request to reply queue on completion. After this + /// call, worker thread no longer accesses the object and it + /// is owned by the reply queue. + /// + /// Threading: called by worker thread. + /// void addAsReply(); protected: HttpReplyQueue * mReplyQueue; // Have refcount - HttpHandler * mUserHandler; + HttpHandler * mUserHandler; // Naked pointer public: // Request Data @@ -172,7 +233,7 @@ public: /// HttpOpSpin is a test-only request that puts the worker /// thread into a cpu spin. Used for unit tests and cleanup -/// evaluation. You do not want to use this. +/// evaluation. You do not want to use this in production. class HttpOpSpin : public HttpOperation { public: diff --git a/indra/llcorehttp/_httpoprequest.cpp b/indra/llcorehttp/_httpoprequest.cpp index a18a164f0d..7db19b1841 100644 --- a/indra/llcorehttp/_httpoprequest.cpp +++ b/indra/llcorehttp/_httpoprequest.cpp @@ -111,10 +111,10 @@ HttpOpRequest::HttpOpRequest() mReplyHeaders(NULL), mPolicyRetries(0), mPolicyRetryAt(HttpTime(0)), - mPolicyRetryLimit(DEFAULT_RETRY_COUNT) + mPolicyRetryLimit(HTTP_RETRY_COUNT_DEFAULT) { // *NOTE: As members are added, retry initialization/cleanup - // may need to be extended in @prepareRequest(). + // may need to be extended in @see prepareRequest(). } @@ -153,9 +153,6 @@ HttpOpRequest::~HttpOpRequest() mCurlHeaders = NULL; } - mReplyOffset = 0; - mReplyLength = 0; - mReplyFullLength = 0; if (mReplyBody) { mReplyBody->release(); @@ -215,8 +212,6 @@ void HttpOpRequest::stageFromActive(HttpService * service) void HttpOpRequest::visitNotifier(HttpRequest * request) { - static const HttpStatus partial_content(HTTP_PARTIAL_CONTENT, HE_SUCCESS); - if (mUserHandler) { HttpResponse * response = new HttpResponse(); @@ -339,8 +334,8 @@ void HttpOpRequest::setupCommon(HttpRequest::policy_t policy_id, mProcFlags |= PF_SAVE_HEADERS; } mPolicyRetryLimit = options->getRetries(); - mPolicyRetryLimit = llclamp(mPolicyRetryLimit, LIMIT_RETRY_MIN, LIMIT_RETRY_MAX); - mTracing = (std::max)(mTracing, llclamp(options->getTrace(), TRACE_MIN, TRACE_MAX)); + mPolicyRetryLimit = llclamp(mPolicyRetryLimit, HTTP_RETRY_COUNT_MIN, HTTP_RETRY_COUNT_MAX); + mTracing = (std::max)(mTracing, llclamp(options->getTrace(), HTTP_TRACE_MIN, HTTP_TRACE_MAX)); } } @@ -394,7 +389,7 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service) curl_easy_setopt(mCurlHandle, CURLOPT_PRIVATE, this); curl_easy_setopt(mCurlHandle, CURLOPT_ENCODING, ""); - if (ENABLE_LINKSYS_WRT54G_V5_DNS_FIX) + if (HTTP_ENABLE_LINKSYS_WRT54G_V5_DNS_FIX) { // The Linksys WRT54G V5 router has an issue with frequent // DNS lookups from LAN machines. If they happen too often, @@ -402,7 +397,7 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service) // about 700 or so requests and starts issuing TCP RSTs to // new connections. Reuse the DNS lookups for even a few // seconds and no RSTs. - curl_easy_setopt(mCurlHandle, CURLOPT_DNS_CACHE_TIMEOUT, 10); + curl_easy_setopt(mCurlHandle, CURLOPT_DNS_CACHE_TIMEOUT, 15); } else { @@ -414,7 +409,7 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service) } curl_easy_setopt(mCurlHandle, CURLOPT_AUTOREFERER, 1); curl_easy_setopt(mCurlHandle, CURLOPT_FOLLOWLOCATION, 1); - curl_easy_setopt(mCurlHandle, CURLOPT_MAXREDIRS, DEFAULT_HTTP_REDIRECTS); + curl_easy_setopt(mCurlHandle, CURLOPT_MAXREDIRS, HTTP_REDIRECTS_DEFAULT); curl_easy_setopt(mCurlHandle, CURLOPT_WRITEFUNCTION, writeCallback); curl_easy_setopt(mCurlHandle, CURLOPT_WRITEDATA, this); curl_easy_setopt(mCurlHandle, CURLOPT_READFUNCTION, readCallback); @@ -434,7 +429,7 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service) } else if (policy.get(HttpRequest::GP_HTTP_PROXY, &opt_value)) { - // *TODO: This is fine for now but get fuller socks/ + // *TODO: This is fine for now but get fuller socks5/ // authentication thing going later.... curl_easy_setopt(mCurlHandle, CURLOPT_PROXY, opt_value->c_str()); curl_easy_setopt(mCurlHandle, CURLOPT_PROXYTYPE, CURLPROXY_HTTP); @@ -497,7 +492,7 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service) } // Tracing - if (mTracing >= TRACE_CURL_HEADERS) + if (mTracing >= HTTP_TRACE_CURL_HEADERS) { curl_easy_setopt(mCurlHandle, CURLOPT_VERBOSE, 1); curl_easy_setopt(mCurlHandle, CURLOPT_DEBUGDATA, this); @@ -528,11 +523,11 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service) mCurlHeaders = curl_slist_append(mCurlHeaders, "Pragma:"); // Request options - long timeout(DEFAULT_TIMEOUT); + long timeout(HTTP_REQUEST_TIMEOUT_DEFAULT); if (mReqOptions) { timeout = mReqOptions->getTimeout(); - timeout = llclamp(timeout, LIMIT_TIMEOUT_MIN, LIMIT_TIMEOUT_MAX); + timeout = llclamp(timeout, HTTP_REQUEST_TIMEOUT_MIN, HTTP_REQUEST_TIMEOUT_MAX); } curl_easy_setopt(mCurlHandle, CURLOPT_TIMEOUT, timeout); curl_easy_setopt(mCurlHandle, CURLOPT_CONNECTTIMEOUT, timeout); @@ -605,12 +600,6 @@ size_t HttpOpRequest::headerCallback(void * data, size_t size, size_t nmemb, voi static const char con_ran_line[] = "content-range:"; static const size_t con_ran_line_len = sizeof(con_ran_line) - 1; - static const char con_type_line[] = "content-type:"; - static const size_t con_type_line_len = sizeof(con_type_line) - 1; - - static const char con_enc_line[] = "content-encoding:"; - static const size_t con_enc_line_len = sizeof(con_enc_line) - 1; - HttpOpRequest * op(static_cast<HttpOpRequest *>(userdata)); const size_t hdr_size(size * nmemb); @@ -705,7 +694,7 @@ int HttpOpRequest::debugCallback(CURL * handle, curl_infotype info, char * buffe switch (info) { case CURLINFO_TEXT: - if (op->mTracing >= TRACE_CURL_HEADERS) + if (op->mTracing >= HTTP_TRACE_CURL_HEADERS) { tag = "TEXT"; escape_libcurl_debug_data(buffer, len, true, safe_line); @@ -714,7 +703,7 @@ int HttpOpRequest::debugCallback(CURL * handle, curl_infotype info, char * buffe break; case CURLINFO_HEADER_IN: - if (op->mTracing >= TRACE_CURL_HEADERS) + if (op->mTracing >= HTTP_TRACE_CURL_HEADERS) { tag = "HEADERIN"; escape_libcurl_debug_data(buffer, len, true, safe_line); @@ -723,7 +712,7 @@ int HttpOpRequest::debugCallback(CURL * handle, curl_infotype info, char * buffe break; case CURLINFO_HEADER_OUT: - if (op->mTracing >= TRACE_CURL_HEADERS) + if (op->mTracing >= HTTP_TRACE_CURL_HEADERS) { tag = "HEADEROUT"; escape_libcurl_debug_data(buffer, 2 * len, true, safe_line); // Goes out as one line @@ -732,11 +721,11 @@ int HttpOpRequest::debugCallback(CURL * handle, curl_infotype info, char * buffe break; case CURLINFO_DATA_IN: - if (op->mTracing >= TRACE_CURL_HEADERS) + if (op->mTracing >= HTTP_TRACE_CURL_HEADERS) { tag = "DATAIN"; logit = true; - if (op->mTracing >= TRACE_CURL_BODIES) + if (op->mTracing >= HTTP_TRACE_CURL_BODIES) { escape_libcurl_debug_data(buffer, len, false, safe_line); } @@ -750,11 +739,11 @@ int HttpOpRequest::debugCallback(CURL * handle, curl_infotype info, char * buffe break; case CURLINFO_DATA_OUT: - if (op->mTracing >= TRACE_CURL_HEADERS) + if (op->mTracing >= HTTP_TRACE_CURL_HEADERS) { tag = "DATAOUT"; logit = true; - if (op->mTracing >= TRACE_CURL_BODIES) + if (op->mTracing >= HTTP_TRACE_CURL_BODIES) { escape_libcurl_debug_data(buffer, len, false, safe_line); } diff --git a/indra/llcorehttp/_httpoprequest.h b/indra/llcorehttp/_httpoprequest.h index 36dc5dc876..7b65d17783 100644 --- a/indra/llcorehttp/_httpoprequest.h +++ b/indra/llcorehttp/_httpoprequest.h @@ -88,7 +88,14 @@ public: virtual void visitNotifier(HttpRequest * request); public: - // Setup Methods + /// Setup Methods + /// + /// Basically an RPC setup for each type of HTTP method + /// invocation with one per method type. These are + /// generally invoked right after construction. + /// + /// Threading: called by application thread + /// HttpStatus setupGet(HttpRequest::policy_t policy_id, HttpRequest::priority_t priority, const std::string & url, @@ -116,19 +123,32 @@ public: BufferArray * body, HttpOptions * options, HttpHeaders * headers); - + + // Internal method used to setup the libcurl options for a request. + // Does all the libcurl handle setup in one place. + // + // Threading: called by worker thread + // HttpStatus prepareRequest(HttpService * service); virtual HttpStatus cancel(); protected: + // Common setup for all the request methods. + // + // Threading: called by application thread + // void setupCommon(HttpRequest::policy_t policy_id, HttpRequest::priority_t priority, const std::string & url, BufferArray * body, HttpOptions * options, HttpHeaders * headers); - + + // libcurl operational callbacks + // + // Threading: called by worker thread + // static size_t writeCallback(void * data, size_t size, size_t nmemb, void * userdata); static size_t readCallback(void * data, size_t size, size_t nmemb, void * userdata); static size_t headerCallback(void * data, size_t size, size_t nmemb, void * userdata); diff --git a/indra/llcorehttp/_httpopsetget.cpp b/indra/llcorehttp/_httpopsetget.cpp index c1357f9ae5..8198528a9b 100644 --- a/indra/llcorehttp/_httpopsetget.cpp +++ b/indra/llcorehttp/_httpopsetget.cpp @@ -26,18 +26,10 @@ #include "_httpopsetget.h" -#include <cstdio> -#include <algorithm> - #include "httpcommon.h" -#include "httphandler.h" -#include "httpresponse.h" -#include "_httprequestqueue.h" -#include "_httpreplyqueue.h" #include "_httpservice.h" #include "_httppolicy.h" -#include "_httplibcurl.h" namespace LLCore diff --git a/indra/llcorehttp/_httpopsetget.h b/indra/llcorehttp/_httpopsetget.h index efb24855c5..6966b9d94e 100644 --- a/indra/llcorehttp/_httpopsetget.h +++ b/indra/llcorehttp/_httpopsetget.h @@ -44,6 +44,8 @@ namespace LLCore /// HttpOpSetGet requests dynamic changes to policy and /// configuration settings. +/// +/// *NOTE: Expect this to change. Don't really like it yet. class HttpOpSetGet : public HttpOperation { @@ -58,6 +60,7 @@ private: void operator=(const HttpOpSetGet &); // Not defined public: + /// Threading: called by application thread void setupGet(HttpRequest::EGlobalPolicy setting); void setupSet(HttpRequest::EGlobalPolicy setting, const std::string & value); diff --git a/indra/llcorehttp/_httppolicy.cpp b/indra/llcorehttp/_httppolicy.cpp index 1e64924198..c7a69ad133 100644 --- a/indra/llcorehttp/_httppolicy.cpp +++ b/indra/llcorehttp/_httppolicy.cpp @@ -40,12 +40,17 @@ namespace LLCore { +// Per-policy-class data for a running system. +// Collection of queues, parameters, history, metrics, etc. +// for a single policy class. +// +// Threading: accessed only by worker thread struct HttpPolicy::State { public: State() - : mConnMax(DEFAULT_CONNECTIONS), - mConnAt(DEFAULT_CONNECTIONS), + : mConnMax(HTTP_CONNECTION_LIMIT_DEFAULT), + mConnAt(HTTP_CONNECTION_LIMIT_DEFAULT), mConnMin(1), mNextSample(0), mErrorCount(0), @@ -298,6 +303,7 @@ bool HttpPolicy::cancel(HttpHandle handle) return false; } + bool HttpPolicy::stageAfterCompletion(HttpOpRequest * op) { static const HttpStatus cant_connect(HttpStatus::EXT_CURL_EASY, CURLE_COULDNT_CONNECT); @@ -345,7 +351,7 @@ bool HttpPolicy::stageAfterCompletion(HttpOpRequest * op) } -int HttpPolicy::getReadyCount(HttpRequest::policy_t policy_class) +int HttpPolicy::getReadyCount(HttpRequest::policy_t policy_class) const { if (policy_class < mActiveClasses) { diff --git a/indra/llcorehttp/_httppolicy.h b/indra/llcorehttp/_httppolicy.h index a02bf084c1..03d92c0b8e 100644 --- a/indra/llcorehttp/_httppolicy.h +++ b/indra/llcorehttp/_httppolicy.h @@ -63,22 +63,37 @@ public: /// Cancel all ready and retry requests sending them to /// their notification queues. Release state resources /// making further request handling impossible. + /// + /// Threading: called by worker thread void shutdown(); /// Deliver policy definitions and enable handling of /// requests. One-time call invoked before starting /// the worker thread. + /// + /// Threading: called by application thread void start(const HttpPolicyGlobal & global, const std::vector<HttpPolicyClass> & classes); /// Give the policy layer some cycles to scan the ready /// queue promoting higher-priority requests to active /// as permited. + /// + /// @return Indication of how soon this method + /// should be called again. + /// + /// Threading: called by worker thread HttpService::ELoopSpeed processReadyQueue(); /// Add request to a ready queue. Caller is expected to have /// provided us with a reference count to hold the request. (No /// additional references will be added.) + /// + /// OpRequest is owned by the request queue after this call + /// and should not be modified by anyone until retrieved + /// from queue. + /// + /// Threading: called by any thread void addOp(HttpOpRequest *); /// Similar to addOp, used when a caller wants to retry a @@ -87,12 +102,20 @@ public: /// handling is the same and retried operations are considered /// before new ones but that doesn't guarantee completion /// order. + /// + /// Threading: called by worker thread void retryOp(HttpOpRequest *); - // Shadows HttpService's method + /// Attempt to change the priority of an earlier request. + /// Request that Shadows HttpService's method + /// + /// Threading: called by worker thread bool changePriority(HttpHandle handle, HttpRequest::priority_t priority); - // Shadows HttpService's method as well + /// Attempt to cancel a previous request. + /// Shadows HttpService's method as well + /// + /// Threading: called by worker thread bool cancel(HttpHandle handle); /// When transport is finished with an op and takes it off the @@ -103,17 +126,25 @@ public: /// @return Returns true of the request is still active /// or ready after staging, false if has been /// sent on to the reply queue. + /// + /// Threading: called by worker thread bool stageAfterCompletion(HttpOpRequest * op); // Get pointer to global policy options. Caller is expected // to do context checks like no setting once running. + /// + /// Threading: called by any thread *but* the object may + /// only be modified by the worker thread once running. + /// HttpPolicyGlobal & getGlobalOptions() { return mGlobalOptions; } - // Get ready counts for a particular class - int getReadyCount(HttpRequest::policy_t policy_class); + /// Get ready counts for a particular policy class + /// + /// Threading: called by worker thread + int getReadyCount(HttpRequest::policy_t policy_class) const; protected: struct State; diff --git a/indra/llcorehttp/_httppolicyclass.cpp b/indra/llcorehttp/_httppolicyclass.cpp index 8007468d3c..a23b81322c 100644 --- a/indra/llcorehttp/_httppolicyclass.cpp +++ b/indra/llcorehttp/_httppolicyclass.cpp @@ -35,8 +35,8 @@ namespace LLCore HttpPolicyClass::HttpPolicyClass() : mSetMask(0UL), - mConnectionLimit(DEFAULT_CONNECTIONS), - mPerHostConnectionLimit(DEFAULT_CONNECTIONS), + mConnectionLimit(HTTP_CONNECTION_LIMIT_DEFAULT), + mPerHostConnectionLimit(HTTP_CONNECTION_LIMIT_DEFAULT), mPipelining(0) {} @@ -71,11 +71,11 @@ HttpStatus HttpPolicyClass::set(HttpRequest::EClassPolicy opt, long value) switch (opt) { case HttpRequest::CP_CONNECTION_LIMIT: - mConnectionLimit = llclamp(value, long(LIMIT_CONNECTIONS_MIN), long(LIMIT_CONNECTIONS_MAX)); + mConnectionLimit = llclamp(value, long(HTTP_CONNECTION_LIMIT_MIN), long(HTTP_CONNECTION_LIMIT_MAX)); break; case HttpRequest::CP_PER_HOST_CONNECTION_LIMIT: - mPerHostConnectionLimit = llclamp(value, long(LIMIT_CONNECTIONS_MIN), mConnectionLimit); + mPerHostConnectionLimit = llclamp(value, long(HTTP_CONNECTION_LIMIT_MIN), mConnectionLimit); break; case HttpRequest::CP_ENABLE_PIPELINING: diff --git a/indra/llcorehttp/_httppolicyglobal.cpp b/indra/llcorehttp/_httppolicyglobal.cpp index ca04839eaf..72f409d3b1 100644 --- a/indra/llcorehttp/_httppolicyglobal.cpp +++ b/indra/llcorehttp/_httppolicyglobal.cpp @@ -35,8 +35,8 @@ namespace LLCore HttpPolicyGlobal::HttpPolicyGlobal() : mSetMask(0UL), - mConnectionLimit(DEFAULT_CONNECTIONS), - mTrace(TRACE_OFF), + mConnectionLimit(HTTP_CONNECTION_LIMIT_DEFAULT), + mTrace(HTTP_TRACE_OFF), mUseLLProxy(0) {} @@ -66,11 +66,11 @@ HttpStatus HttpPolicyGlobal::set(HttpRequest::EGlobalPolicy opt, long value) switch (opt) { case HttpRequest::GP_CONNECTION_LIMIT: - mConnectionLimit = llclamp(value, long(LIMIT_CONNECTIONS_MIN), long(LIMIT_CONNECTIONS_MAX)); + mConnectionLimit = llclamp(value, long(HTTP_CONNECTION_LIMIT_MIN), long(HTTP_CONNECTION_LIMIT_MAX)); break; case HttpRequest::GP_TRACE: - mTrace = llclamp(value, long(TRACE_MIN), long(TRACE_MAX)); + mTrace = llclamp(value, long(HTTP_TRACE_MIN), long(HTTP_TRACE_MAX)); break; case HttpRequest::GP_LLPROXY: diff --git a/indra/llcorehttp/_httpreadyqueue.h b/indra/llcorehttp/_httpreadyqueue.h index 9cf4b059a1..5f19a9c5f9 100644 --- a/indra/llcorehttp/_httpreadyqueue.h +++ b/indra/llcorehttp/_httpreadyqueue.h @@ -45,7 +45,7 @@ namespace LLCore /// important of those rules is that any iterator becomes invalid /// on element erasure. So pay attention. /// -/// If LLCORE_READY_QUEUE_IGNORES_PRIORITY tests true, the class +/// If LLCORE_HTTP_READY_QUEUE_IGNORES_PRIORITY tests true, the class /// implements a std::priority_queue interface but on std::deque /// behavior to eliminate sensitivity to priority. In the future, /// this will likely become the only behavior or it may become @@ -54,7 +54,7 @@ namespace LLCore /// Threading: not thread-safe. Expected to be used entirely by /// a single thread, typically a worker thread of some sort. -#if LLCORE_READY_QUEUE_IGNORES_PRIORITY +#if LLCORE_HTTP_READY_QUEUE_IGNORES_PRIORITY typedef std::deque<HttpOpRequest *> HttpReadyQueueBase; @@ -64,7 +64,7 @@ typedef std::priority_queue<HttpOpRequest *, std::deque<HttpOpRequest *>, LLCore::HttpOpRequestCompare> HttpReadyQueueBase; -#endif // LLCORE_READY_QUEUE_IGNORES_PRIORITY +#endif // LLCORE_HTTP_READY_QUEUE_IGNORES_PRIORITY class HttpReadyQueue : public HttpReadyQueueBase { @@ -82,7 +82,7 @@ protected: public: -#if LLCORE_READY_QUEUE_IGNORES_PRIORITY +#if LLCORE_HTTP_READY_QUEUE_IGNORES_PRIORITY // Types and methods needed to make a std::deque look // more like a std::priority_queue, at least for our // purposes. @@ -103,7 +103,7 @@ public: push_back(v); } -#endif // LLCORE_READY_QUEUE_IGNORES_PRIORITY +#endif // LLCORE_HTTP_READY_QUEUE_IGNORES_PRIORITY const container_type & get_container() const { diff --git a/indra/llcorehttp/_httpservice.cpp b/indra/llcorehttp/_httpservice.cpp index f7d9813db0..0825888d0f 100644 --- a/indra/llcorehttp/_httpservice.cpp +++ b/indra/llcorehttp/_httpservice.cpp @@ -55,8 +55,8 @@ HttpService::HttpService() { // Create the default policy class HttpPolicyClass pol_class; - pol_class.set(HttpRequest::CP_CONNECTION_LIMIT, DEFAULT_CONNECTIONS); - pol_class.set(HttpRequest::CP_PER_HOST_CONNECTION_LIMIT, DEFAULT_CONNECTIONS); + pol_class.set(HttpRequest::CP_CONNECTION_LIMIT, HTTP_CONNECTION_LIMIT_DEFAULT); + pol_class.set(HttpRequest::CP_PER_HOST_CONNECTION_LIMIT, HTTP_CONNECTION_LIMIT_DEFAULT); pol_class.set(HttpRequest::CP_ENABLE_PIPELINING, 0L); mPolicyClasses.push_back(pol_class); } @@ -150,7 +150,7 @@ void HttpService::term() HttpRequest::policy_t HttpService::createPolicyClass() { const HttpRequest::policy_t policy_class(mPolicyClasses.size()); - if (policy_class >= POLICY_CLASS_LIMIT) + if (policy_class >= HTTP_POLICY_CLASS_LIMIT) { return 0; } @@ -219,12 +219,12 @@ bool HttpService::changePriority(HttpHandle handle, HttpRequest::priority_t prio } - /// Try to find the given request handle on any of the request - /// queues and cancel the operation. - /// - /// @return True if the request was canceled. - /// - /// Threading: callable by worker thread. +/// Try to find the given request handle on any of the request +/// queues and cancel the operation. +/// +/// @return True if the request was canceled. +/// +/// Threading: callable by worker thread. bool HttpService::cancel(HttpHandle handle) { bool canceled(false); @@ -297,7 +297,7 @@ void HttpService::threadRun(LLCoreInt::HttpThread * thread) // Determine whether to spin, sleep briefly or sleep for next request if (REQUEST_SLEEP != loop) { - ms_sleep(LOOP_SLEEP_NORMAL_MS); + ms_sleep(HTTP_SERVICE_LOOP_SLEEP_NORMAL_MS); } } @@ -321,11 +321,11 @@ HttpService::ELoopSpeed HttpService::processRequestQueue(ELoopSpeed loop) if (! mExitRequested) { // Setup for subsequent tracing - long tracing(TRACE_OFF); + long tracing(HTTP_TRACE_OFF); mPolicy->getGlobalOptions().get(HttpRequest::GP_TRACE, &tracing); op->mTracing = (std::max)(op->mTracing, int(tracing)); - if (op->mTracing > TRACE_OFF) + if (op->mTracing > HTTP_TRACE_OFF) { LL_INFOS("CoreHttp") << "TRACE, FromRequestQueue, Handle: " << static_cast<HttpHandle>(op) diff --git a/indra/llcorehttp/_httpservice.h b/indra/llcorehttp/_httpservice.h index d24c497ca9..ffe0349d4d 100644 --- a/indra/llcorehttp/_httpservice.h +++ b/indra/llcorehttp/_httpservice.h @@ -55,7 +55,7 @@ class HttpPolicy; class HttpLibcurl; -/// The HttpService class does the work behind the request queue. It +/// The HttpService class does the work behind the request queue. It /// oversees the HTTP workflow carrying out a number of tasks: /// - Pulling requests from the global request queue /// - Executing 'immediate' requests directly @@ -76,7 +76,7 @@ class HttpLibcurl; /// HttpPolicy and HttpLibcurl (transport). These always exist in a /// 1:1:1 relationship with HttpService managing instances of the other /// two. So, these classes do not use reference counting to refer -/// to one-another, their lifecycles are always managed together. +/// to one another, their lifecycles are always managed together. class HttpService { @@ -206,7 +206,7 @@ protected: // === shared data === static volatile EState sState; - HttpRequestQueue * mRequestQueue; + HttpRequestQueue * mRequestQueue; // Refcounted LLAtomicU32 mExitRequested; LLCoreInt::HttpThread * mThread; diff --git a/indra/llcorehttp/httpcommon.h b/indra/llcorehttp/httpcommon.h index dd5798edf9..c0d4ec5aad 100644 --- a/indra/llcorehttp/httpcommon.h +++ b/indra/llcorehttp/httpcommon.h @@ -60,8 +60,10 @@ /// Using the library is fairly easy. Global setup needs a few /// steps: /// -/// - libcurl initialization with thread-safely callbacks for c-ares -/// DNS lookups. +/// - libcurl initialization including thread-safely callbacks for SSL: +/// . curl_global_init(...) +/// . CRYPTO_set_locking_callback(...) +/// . CRYPTO_set_id_callback(...) /// - HttpRequest::createService() called to instantiate singletons /// and support objects. /// @@ -90,8 +92,18 @@ /// - Do completion processing in your onCompletion() method. /// /// Code fragments: -/// <TBD> +/// Rather than a poorly-maintained example in comments, look in the +/// example subdirectory which is a minimal yet functional tool to do +/// GET request performance testing. With four calls: /// +/// init_curl(); +/// LLCore::HttpRequest::createService(); +/// LLCore::HttpRequest::startThread(); +/// LLCore::HttpRequest * hr = new LLCore::HttpRequest(); +/// +/// the program is basically ready to issue requests. +/// + #include "linden_common.h" // Modifies curl/curl.h interfaces diff --git a/indra/llcorehttp/httpoptions.cpp b/indra/llcorehttp/httpoptions.cpp index 68f7277ed3..1699d19f8d 100644 --- a/indra/llcorehttp/httpoptions.cpp +++ b/indra/llcorehttp/httpoptions.cpp @@ -36,9 +36,9 @@ namespace LLCore HttpOptions::HttpOptions() : RefCounted(true), mWantHeaders(false), - mTracing(TRACE_OFF), - mTimeout(DEFAULT_TIMEOUT), - mRetries(DEFAULT_RETRY_COUNT) + mTracing(HTTP_TRACE_OFF), + mTimeout(HTTP_REQUEST_TIMEOUT_DEFAULT), + mRetries(HTTP_RETRY_COUNT_DEFAULT) {} |