diff options
Diffstat (limited to 'indra/llcorehttp')
23 files changed, 769 insertions, 140 deletions
diff --git a/indra/llcorehttp/CMakeLists.txt b/indra/llcorehttp/CMakeLists.txt index ae92fb96fd..85df5364db 100644 --- a/indra/llcorehttp/CMakeLists.txt +++ b/indra/llcorehttp/CMakeLists.txt @@ -23,17 +23,18 @@ include_directories( set(llcorehttp_SOURCE_FILES bufferarray.cpp httpcommon.cpp + httpheaders.cpp + httpoptions.cpp httprequest.cpp httpresponse.cpp - httpoptions.cpp - httpheaders.cpp - _httprequestqueue.cpp + _httplibcurl.cpp + _httpopcancel.cpp _httpoperation.cpp _httpoprequest.cpp - _httpopcancel.cpp - _httpreplyqueue.cpp + _httpopsetpriority.cpp _httppolicy.cpp - _httplibcurl.cpp + _httpreplyqueue.cpp + _httprequestqueue.cpp _httpservice.cpp _refcounted.cpp ) @@ -48,17 +49,19 @@ set(llcorehttp_HEADER_FILES httpoptions.h httprequest.h httpresponse.h + _assert.h + _httplibcurl.h + _httpopcancel.h _httpoperation.h _httpoprequest.h - _httpopcancel.h - _httprequestqueue.h + _httpopsetpriority.h + _httppolicy.h + _httpreadyqueue.h _httpreplyqueue.h + _httprequestqueue.h _httpservice.h - _httppolicy.h - _httplibcurl.h - _assert.h - _refcounted.h _mutex.h + _refcounted.h _thread.h ) diff --git a/indra/llcorehttp/_assert.h b/indra/llcorehttp/_assert.h deleted file mode 100644 index 054f23ef32..0000000000 --- a/indra/llcorehttp/_assert.h +++ /dev/null @@ -1,39 +0,0 @@ -/** - * @file _assert - * @brief assert abstraction - * - * $LicenseInfo:firstyear=2012&license=viewerlgpl$ - * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; - * version 2.1 of the License only. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - * - * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA - * $/LicenseInfo$ - */ - -#ifndef LLCOREINT__ASSERT_H_ -#define LLCOREINT__ASSERT_H_ - -#ifdef DEBUG_ASSERT -#include <cassert> -#define LLINT_ASSERT(x) assert(x) -#else -#define LLINT_ASSERT(x) -#endif - -#endif // LLCOREINT__ASSERT_H_ - - diff --git a/indra/llcorehttp/_httplibcurl.cpp b/indra/llcorehttp/_httplibcurl.cpp index 1b951818e4..704f9baac9 100644 --- a/indra/llcorehttp/_httplibcurl.cpp +++ b/indra/llcorehttp/_httplibcurl.cpp @@ -28,7 +28,6 @@ #include "httpheaders.h" #include "_httpoprequest.h" -#include "_httpservice.h" namespace LLCore @@ -38,6 +37,12 @@ namespace LLCore HttpLibcurl::HttpLibcurl(HttpService * service) : mService(service) { + for (int policy_class(0); policy_class < HttpRequest::POLICY_CLASS_LIMIT; ++policy_class) + { + mMultiHandles[policy_class] = 0; + } + + // Create multi handle for default class mMultiHandles[0] = curl_multi_init(); } @@ -51,15 +56,18 @@ HttpLibcurl::~HttpLibcurl() (*item)->cancel(); (*item)->release(); + mActiveOps.erase(item); } - if (mMultiHandles[0]) + for (int policy_class(0); policy_class < HttpRequest::POLICY_CLASS_LIMIT; ++policy_class) { - // *FIXME: Do some multi cleanup here first - + if (mMultiHandles[policy_class]) + { + // *FIXME: Do some multi cleanup here first - curl_multi_cleanup(mMultiHandles[0]); - mMultiHandles[0] = NULL; + curl_multi_cleanup(mMultiHandles[policy_class]); + mMultiHandles[policy_class] = 0; + } } mService = NULL; @@ -74,31 +82,34 @@ void HttpLibcurl::term() {} -void HttpLibcurl::processTransport() +HttpService::ELoopSpeed HttpLibcurl::processTransport() { - if (mMultiHandles[0]) + // Give libcurl some cycles to do I/O & callbacks + for (int policy_class(0); policy_class < HttpRequest::POLICY_CLASS_LIMIT; ++policy_class) { - // Give libcurl some cycles to do I/O & callbacks + if (! mMultiHandles[policy_class]) + continue; + int running(0); CURLMcode status(CURLM_CALL_MULTI_PERFORM); do { running = 0; - status = curl_multi_perform(mMultiHandles[0], &running); + status = curl_multi_perform(mMultiHandles[policy_class], &running); } while (0 != running && CURLM_CALL_MULTI_PERFORM == status); // Run completion on anything done CURLMsg * msg(NULL); int msgs_in_queue(0); - while ((msg = curl_multi_info_read(mMultiHandles[0], &msgs_in_queue))) + while ((msg = curl_multi_info_read(mMultiHandles[policy_class], &msgs_in_queue))) { if (CURLMSG_DONE == msg->msg) { CURL * handle(msg->easy_handle); CURLcode result(msg->data.result); - completeRequest(mMultiHandles[0], handle, result); + completeRequest(mMultiHandles[policy_class], handle, result); handle = NULL; // No longer valid on return } else if (CURLMSG_NONE == msg->msg) @@ -114,13 +125,18 @@ void HttpLibcurl::processTransport() msgs_in_queue = 0; } } + + return mActiveOps.empty() ? HttpService::REQUEST_SLEEP : HttpService::NORMAL; } void HttpLibcurl::addOp(HttpOpRequest * op) { + llassert_always(op->mReqPolicy < HttpRequest::POLICY_CLASS_LIMIT); + llassert_always(mMultiHandles[op->mReqPolicy] != NULL); + // Create standard handle - if (! op->prepareForGet(mService)) + if (! op->prepareRequest(mService)) { // Couldn't issue request, fail with notification // *FIXME: Need failure path @@ -128,7 +144,7 @@ void HttpLibcurl::addOp(HttpOpRequest * op) } // Make the request live - curl_multi_add_handle(mMultiHandles[0], op->mCurlHandle); + curl_multi_add_handle(mMultiHandles[op->mReqPolicy], op->mCurlHandle); op->mCurlActive = true; // On success, make operation active @@ -190,12 +206,18 @@ void HttpLibcurl::completeRequest(CURLM * multi_handle, CURL * handle, CURLcode } -int HttpLibcurl::activeCount() const +int HttpLibcurl::getActiveCount() const { return mActiveOps.size(); } +int HttpLibcurl::getActiveCountInClass(int /* policy_class */) const +{ + return getActiveCount(); +} + + // --------------------------------------- // Free functions // --------------------------------------- diff --git a/indra/llcorehttp/_httplibcurl.h b/indra/llcorehttp/_httplibcurl.h index 807196628d..ec325c1946 100644 --- a/indra/llcorehttp/_httplibcurl.h +++ b/indra/llcorehttp/_httplibcurl.h @@ -34,18 +34,25 @@ #include <set> +#include "httprequest.h" +#include "_httpservice.h" + namespace LLCore { -class HttpService; class HttpPolicy; class HttpOpRequest; class HttpHeaders; /// Implements libcurl-based transport for an HttpService instance. +/// +/// Threading: Single-threaded. Other than for construction/destruction, +/// all methods are expected to be invoked in a single thread, typically +/// a worker thread of some sort. + class HttpLibcurl { public: @@ -60,12 +67,22 @@ public: static void init(); static void term(); - void processTransport(); - void addOp(HttpOpRequest * op); + /// Give cycles to libcurl to run active requests. Completed + /// operations (successful or failed) will be retried or handed + /// over to the reply queue as final responses. + HttpService::ELoopSpeed processTransport(); - int activeCount() const; + /// Add request to the active list. Caller is expected to have + /// provided us with a reference count to hold the request. (No + /// additional references will be added.) + void addOp(HttpOpRequest * op); + int getActiveCount() const; + int getActiveCountInClass(int policy_class) const; + protected: + /// Invoked when libcurl has indicated a request has been processed + /// to completion and we need to move the request to a new state. void completeRequest(CURLM * multi_handle, CURL * handle, CURLcode status); protected: @@ -74,7 +91,7 @@ protected: protected: HttpService * mService; // Simple reference, not owner active_set_t mActiveOps; - CURLM * mMultiHandles[1]; + CURLM * mMultiHandles[HttpRequest::POLICY_CLASS_LIMIT]; }; // end class HttpLibcurl diff --git a/indra/llcorehttp/_httpopcancel.h b/indra/llcorehttp/_httpopcancel.h index 38ccc585ed..fab6f1f362 100644 --- a/indra/llcorehttp/_httpopcancel.h +++ b/indra/llcorehttp/_httpopcancel.h @@ -45,7 +45,7 @@ namespace LLCore /// HttpOpCancel requests that a previously issued request /// be canceled, if possible. Requests that have been made /// active and are available for sending on the wire cannot -/// be canceled. +/// be canceled. class HttpOpCancel : public HttpOperation { diff --git a/indra/llcorehttp/_httpoperation.cpp b/indra/llcorehttp/_httpoperation.cpp index 17c65b0379..d966efd12b 100644 --- a/indra/llcorehttp/_httpoperation.cpp +++ b/indra/llcorehttp/_httpoperation.cpp @@ -50,7 +50,7 @@ HttpOperation::HttpOperation() mLibraryHandler(NULL), mUserHandler(NULL), mReqPolicy(HttpRequest::DEFAULT_POLICY_ID), - mReqPriority(0.0f) + mReqPriority(0U) { } @@ -95,7 +95,7 @@ void HttpOperation::stageFromRequest(HttpService *) // Default implementation should never be called. This // indicates an operation making a transition that isn't // defined. - LLINT_ASSERT(false); + llassert_always(false); } @@ -105,7 +105,7 @@ void HttpOperation::stageFromReady(HttpService *) // Default implementation should never be called. This // indicates an operation making a transition that isn't // defined. - LLINT_ASSERT(false); + llassert_always(false); } @@ -115,7 +115,7 @@ void HttpOperation::stageFromActive(HttpService *) // Default implementation should never be called. This // indicates an operation making a transition that isn't // defined. - LLINT_ASSERT(false); + llassert_always(false); } diff --git a/indra/llcorehttp/_httpoperation.h b/indra/llcorehttp/_httpoperation.h index 5d06a28586..6c0c3183b7 100644 --- a/indra/llcorehttp/_httpoperation.h +++ b/indra/llcorehttp/_httpoperation.h @@ -44,6 +44,32 @@ class HttpRequest; /// HttpOperation is the base class for all request/reply /// pairs. /// +/// Operations are expected to be of two types: immediate +/// and queued. Immediate requests go to the singleton +/// request queue and when picked up by the worker thread +/// are executed immediately and there results placed on +/// the supplied reply queue. Queued requests (namely for +/// HTTP operations), go to the request queue, are picked +/// up and moved to a ready queue where they're ordered by +/// priority and managed by the policy component, are +/// then activated issuing HTTP requests and moved to an +/// active list managed by the transport (libcurl) component +/// and eventually finalized when a response is available +/// and status and data return via reply queue. +/// +/// To manage these transitions, derived classes implement +/// three methods: stageFromRequest, stageFromReady and +/// stageFromActive. Immediate requests will only override +/// stageFromRequest which will perform the operation and +/// return the result by invoking addAsReply() to put the +/// request on a reply queue. Queued requests will involve +/// all three stage methods. +/// +/// Threading: not thread-safe. Base and derived classes +/// provide no locking. Instances move across threads +/// via queue-like interfaces that are thread compatible +/// and those interfaces establish the access rules. + class HttpOperation : public LLCoreInt::RefCounted { public: @@ -82,7 +108,7 @@ protected: public: unsigned int mReqPolicy; - float mReqPriority; + unsigned int mReqPriority; }; // end class HttpOperation @@ -133,6 +159,20 @@ public: }; // end class HttpOpNull + +/// HttpOpCompare isn't an operation but a uniform comparison +/// functor for STL containers that order by priority. Mainly +/// used for the ready queue container but defined here. +class HttpOpCompare +{ +public: + bool operator()(const HttpOperation * lhs, const HttpOperation * rhs) + { + return lhs->mReqPriority > rhs->mReqPriority; + } +}; // end class HttpOpCompare + + } // end namespace LLCore #endif // _LLCORE_HTTP_OPERATION_H_ diff --git a/indra/llcorehttp/_httpoprequest.cpp b/indra/llcorehttp/_httpoprequest.cpp index 521bd5b879..54b9990057 100644 --- a/indra/llcorehttp/_httpoprequest.cpp +++ b/indra/llcorehttp/_httpoprequest.cpp @@ -93,6 +93,7 @@ HttpOpRequest::HttpOpRequest() mCurlHandle(NULL), mCurlService(NULL), mCurlHeaders(NULL), + mCurlBodyPos(0), mReplyBody(NULL), mReplyOffset(0), mReplyLength(0), @@ -267,12 +268,46 @@ HttpStatus HttpOpRequest::setupGetByteRange(unsigned int policy_id, } -HttpStatus HttpOpRequest::prepareForGet(HttpService * service) +HttpStatus HttpOpRequest::setupPost(unsigned int policy_id, + float priority, + const std::string & url, + BufferArray * body, + HttpOptions * options, + HttpHeaders * headers) +{ + HttpStatus status; + + mProcFlags = 0; + mReqPolicy = policy_id; + mReqPriority = priority; + mReqMethod = HOR_POST; + mReqURL = url; + if (body) + { + body->addRef(); + mReqBody = body; + } + if (headers && ! mReqHeaders) + { + headers->addRef(); + mReqHeaders = headers; + } + if (options && ! mReqOptions) + { + mReqOptions = new HttpOptions(*options); + } + + return status; +} + + +HttpStatus HttpOpRequest::prepareRequest(HttpService * service) { // *FIXME: better error handling later HttpStatus status; mCurlHandle = curl_easy_init(); + // curl_easy_setopt(mCurlHandle, CURLOPT_VERBOSE, 1); curl_easy_setopt(mCurlHandle, CURLOPT_TIMEOUT, 30); curl_easy_setopt(mCurlHandle, CURLOPT_CONNECTTIMEOUT, 30); curl_easy_setopt(mCurlHandle, CURLOPT_NOSIGNAL, 1); @@ -280,20 +315,68 @@ HttpStatus HttpOpRequest::prepareForGet(HttpService * service) curl_easy_setopt(mCurlHandle, CURLOPT_URL, mReqURL.c_str()); curl_easy_setopt(mCurlHandle, CURLOPT_PRIVATE, this); curl_easy_setopt(mCurlHandle, CURLOPT_ENCODING, ""); + // *FIXME: Need to deal with proxy setup... // curl_easy_setopt(handle, CURLOPT_PROXY, ""); + // *FIXME: Revisit this old DNS timeout setting - may no longer be valid curl_easy_setopt(mCurlHandle, CURLOPT_DNS_CACHE_TIMEOUT, 0); + curl_easy_setopt(mCurlHandle, CURLOPT_AUTOREFERER, 1); curl_easy_setopt(mCurlHandle, CURLOPT_FOLLOWLOCATION, 1); + curl_easy_setopt(mCurlHandle, CURLOPT_MAXREDIRS, 10); curl_easy_setopt(mCurlHandle, CURLOPT_WRITEFUNCTION, writeCallback); curl_easy_setopt(mCurlHandle, CURLOPT_WRITEDATA, mCurlHandle); + curl_easy_setopt(mCurlHandle, CURLOPT_READFUNCTION, readCallback); + curl_easy_setopt(mCurlHandle, CURLOPT_READDATA, mCurlHandle); + switch (mReqMethod) + { + case HOR_GET: + curl_easy_setopt(mCurlHandle, CURLOPT_HTTPGET, 1); + break; + + case HOR_POST: + { + curl_easy_setopt(mCurlHandle, CURLOPT_POST, 1); + long data_size(0); + if (mReqBody) + { + mReqBody->seek(0); + data_size = mReqBody->size(); + } + curl_easy_setopt(mCurlHandle, CURLOPT_POSTFIELDS, static_cast<void *>(NULL)); + curl_easy_setopt(mCurlHandle, CURLOPT_POSTFIELDSIZE, data_size); + mCurlHeaders = curl_slist_append(mCurlHeaders, "Transfer-Encoding: chunked"); + mCurlHeaders = curl_slist_append(mCurlHeaders, "Expect:"); + } + break; + + case HOR_PUT: + { + curl_easy_setopt(mCurlHandle, CURLOPT_UPLOAD, 1); + long data_size(0); + if (mReqBody) + { + mReqBody->seek(0); + data_size = mReqBody->size(); + } + curl_easy_setopt(mCurlHandle, CURLOPT_INFILESIZE, data_size); + mCurlHeaders = curl_slist_append(mCurlHeaders, "Transfer-Encoding: chunked"); + mCurlHeaders = curl_slist_append(mCurlHeaders, "Expect:"); + } + break; + + default: + // *FIXME: fail out here + break; + } + if (mReqHeaders) { mCurlHeaders = append_headers_to_slist(mReqHeaders, mCurlHeaders); } mCurlHeaders = curl_slist_append(mCurlHeaders, "Pragma:"); - if (mReqOffset || mReqLength) + if ((mReqOffset || mReqLength) && HOR_GET == mReqMethod) { static const char * fmt1("Range: bytes=%d-%d"); static const char * fmt2("Range: bytes=%d-"); @@ -347,6 +430,32 @@ size_t HttpOpRequest::writeCallback(void * data, size_t size, size_t nmemb, void } +size_t HttpOpRequest::readCallback(void * data, size_t size, size_t nmemb, void * userdata) +{ + CURL * handle(static_cast<CURL *>(userdata)); + HttpOpRequest * op(NULL); + curl_easy_getinfo(handle, CURLINFO_PRIVATE, &op); + // *FIXME: check the pointer + + if (! op->mReqBody) + { + return 0; + } + const size_t req_size(size * nmemb); + const size_t body_size(op->mReqBody->size()); + if (body_size <= op->mCurlBodyPos) + { + // *FIXME: should probably log this event - unexplained + return 0; + } + + const size_t do_size((std::min)(req_size, body_size - op->mCurlBodyPos)); + op->mReqBody->read(static_cast<char *>(data), do_size); + op->mCurlBodyPos += do_size; + return do_size; +} + + size_t HttpOpRequest::headerCallback(void * data, size_t size, size_t nmemb, void * userdata) { static const char status_line[] = "HTTP/"; diff --git a/indra/llcorehttp/_httpoprequest.h b/indra/llcorehttp/_httpoprequest.h index 601937a943..7efed0b1d9 100644 --- a/indra/llcorehttp/_httpoprequest.h +++ b/indra/llcorehttp/_httpoprequest.h @@ -84,12 +84,20 @@ public: HttpOptions * options, HttpHeaders * headers); - HttpStatus prepareForGet(HttpService * service); + HttpStatus setupPost(unsigned int policy_id, + float priority, + const std::string & url, + BufferArray * body, + HttpOptions * options, + HttpHeaders * headers); + + HttpStatus prepareRequest(HttpService * service); virtual HttpStatus cancel(); protected: static size_t writeCallback(void * data, size_t size, size_t nmemb, void * userdata); + static size_t readCallback(void * data, size_t size, size_t nmemb, void * userdata); static size_t headerCallback(void * data, size_t size, size_t nmemb, void * userdata); protected: @@ -112,6 +120,7 @@ public: CURL * mCurlHandle; HttpService * mCurlService; curl_slist * mCurlHeaders; + size_t mCurlBodyPos; // Result data HttpStatus mStatus; diff --git a/indra/llcorehttp/_httpopsetpriority.cpp b/indra/llcorehttp/_httpopsetpriority.cpp new file mode 100644 index 0000000000..dc609421ed --- /dev/null +++ b/indra/llcorehttp/_httpopsetpriority.cpp @@ -0,0 +1,77 @@ +/** + * @file _httpopsetpriority.cpp + * @brief Definitions for internal classes based on HttpOpSetPriority + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include "_httpopsetpriority.h" + +#include "httpresponse.h" +#include "httphandler.h" +#include "_httpservice.h" + + +namespace LLCore +{ + + +HttpOpSetPriority::HttpOpSetPriority(HttpHandle handle, unsigned int priority) + : HttpOperation(), + mHandle(handle), + mPriority(priority) +{} + + +HttpOpSetPriority::~HttpOpSetPriority() +{} + + +void HttpOpSetPriority::stageFromRequest(HttpService * service) +{ + // Do operations + if (! service->changePriority(mHandle, mPriority)) + { + // Request not found, fail the final status + mStatus = HttpStatus(HttpStatus::LLCORE, HE_HANDLE_NOT_FOUND); + } + + // Move directly to response queue + addAsReply(); +} + + +void HttpOpSetPriority::visitNotifier(HttpRequest * request) +{ + if (mLibraryHandler) + { + HttpResponse * response = new HttpResponse(); + + response->setStatus(mStatus); + mLibraryHandler->onCompleted(static_cast<HttpHandle>(this), response); + + response->release(); + } +} + + +} // end namespace LLCore diff --git a/indra/llcorehttp/_httpopsetpriority.h b/indra/llcorehttp/_httpopsetpriority.h new file mode 100644 index 0000000000..e5d8e5fc1f --- /dev/null +++ b/indra/llcorehttp/_httpopsetpriority.h @@ -0,0 +1,70 @@ +/** + * @file _httpsetpriority.h + * @brief Internal declarations for HttpSetPriority + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef _LLCORE_HTTP_SETPRIORITY_H_ +#define _LLCORE_HTTP_SETPRIORITY_H_ + + +#include "httpcommon.h" + +#include "_httpoperation.h" +#include "_refcounted.h" + + +namespace LLCore +{ + + +/// HttpOpSetPriority is an immediate request that +/// searches the various queues looking for a given +/// request handle and changing it's priority if +/// found. + +class HttpOpSetPriority : public HttpOperation +{ +public: + HttpOpSetPriority(HttpHandle handle, unsigned int priority); + virtual ~HttpOpSetPriority(); + +private: + HttpOpSetPriority(const HttpOpSetPriority &); // Not defined + void operator=(const HttpOpSetPriority &); // Not defined + +public: + virtual void stageFromRequest(HttpService *); + + virtual void visitNotifier(HttpRequest * request); + +protected: + HttpStatus mStatus; + HttpHandle mHandle; + unsigned int mPriority; +}; // end class HttpOpSetPriority + +} // end namespace LLCore + +#endif // _LLCORE_HTTP_SETPRIORITY_H_ + diff --git a/indra/llcorehttp/_httppolicy.cpp b/indra/llcorehttp/_httppolicy.cpp index d965a6cf3a..873b519c51 100644 --- a/indra/llcorehttp/_httppolicy.cpp +++ b/indra/llcorehttp/_httppolicy.cpp @@ -28,48 +28,113 @@ #include "_httpoprequest.h" #include "_httpservice.h" +#include "_httplibcurl.h" namespace LLCore { - HttpPolicy::HttpPolicy(HttpService * service) : mService(service) -{} +{ + for (int policy_class(0); policy_class < HttpRequest::POLICY_CLASS_LIMIT; ++policy_class) + { + mReadyInClass[policy_class] = 0; + } +} HttpPolicy::~HttpPolicy() { - for (ready_queue_t::reverse_iterator i(mReadyQueue.rbegin()); - mReadyQueue.rend() != i;) + for (int policy_class(0); policy_class < HttpRequest::POLICY_CLASS_LIMIT; ++policy_class) { - ready_queue_t::reverse_iterator cur(i++); - - (*cur)->cancel(); - (*cur)->release(); + HttpReadyQueue & readyq(mReadyQueue[policy_class]); + + while (! readyq.empty()) + { + HttpOpRequest * op(readyq.top()); + + op->cancel(); + op->release(); + mReadyInClass[policy_class]--; + readyq.pop(); + } } - mService = NULL; } void HttpPolicy::addOp(HttpOpRequest * op) { - mReadyQueue.push_back(op); + const int policy_class(op->mReqPolicy); + + mReadyQueue[policy_class].push(op); + ++mReadyInClass[policy_class]; } -void HttpPolicy::processReadyQueue() +HttpService::ELoopSpeed HttpPolicy::processReadyQueue() { - while (! mReadyQueue.empty()) + HttpService::ELoopSpeed result(HttpService::REQUEST_SLEEP); + HttpLibcurl * pTransport(mService->getTransport()); + + for (int policy_class(0); policy_class < HttpRequest::POLICY_CLASS_LIMIT; ++policy_class) { - HttpOpRequest * op(mReadyQueue.front()); - mReadyQueue.erase(mReadyQueue.begin()); + HttpReadyQueue & readyq(mReadyQueue[policy_class]); + int active(pTransport->getActiveCountInClass(policy_class)); + int needed(8 - active); + + if (needed > 0 && mReadyInClass[policy_class] > 0) + { + // Scan ready queue for requests that match policy + + while (! readyq.empty() && needed > 0 && mReadyInClass[policy_class] > 0) + { + HttpOpRequest * op(readyq.top()); + readyq.pop(); + + op->stageFromReady(mService); + op->release(); + + --mReadyInClass[policy_class]; + --needed; + } + } + + if (! readyq.empty()) + { + // If anything is ready, continue looping... + result = (std::min)(result, HttpService::NORMAL); + } + } + + return result; +} - op->stageFromReady(mService); - op->release(); + +bool HttpPolicy::changePriority(HttpHandle handle, unsigned int priority) +{ + for (int policy_class(0); policy_class < HttpRequest::POLICY_CLASS_LIMIT; ++policy_class) + { + HttpReadyQueue::container_type & c(mReadyQueue[policy_class].get_container()); + + // Scan ready queue for requests that match policy + for (HttpReadyQueue::container_type::iterator iter(c.begin()); c.end() != iter;) + { + HttpReadyQueue::container_type::iterator cur(iter++); + + if (static_cast<HttpHandle>(*cur) == handle) + { + HttpOpRequest * op(*cur); + c.erase(cur); // All iterators are now invalidated + op->mReqPriority = priority; + mReadyQueue[policy_class].push(op); // Re-insert using adapter class + return true; + } + } } + + return false; } diff --git a/indra/llcorehttp/_httppolicy.h b/indra/llcorehttp/_httppolicy.h index 192bc73b31..c5e82d0a65 100644 --- a/indra/llcorehttp/_httppolicy.h +++ b/indra/llcorehttp/_httppolicy.h @@ -28,18 +28,23 @@ #define _LLCORE_HTTP_POLICY_H_ -#include <vector> +#include "httprequest.h" +#include "_httpservice.h" +#include "_httpreadyqueue.h" namespace LLCore { - -class HttpService; +class HttpReadyQueue; class HttpOpRequest; /// Implements class-based queuing policies for an HttpService instance. +/// +/// Threading: Single-threaded. Other than for construction/destruction, +/// all methods are expected to be invoked in a single thread, typically +/// a worker thread of some sort. class HttpPolicy { public: @@ -51,16 +56,23 @@ private: void operator=(const HttpPolicy &); // Not defined public: - void processReadyQueue(); + /// Give the policy layer some cycles to scan the ready + /// queue promoting higher-priority requests to active + /// as permited. + HttpService::ELoopSpeed processReadyQueue(); + /// Add request to a ready queue. Caller is expected to have + /// provided us with a reference count to hold the request. (No + /// additional references will be added.) void addOp(HttpOpRequest *); + + // Shadows HttpService's method + bool changePriority(HttpHandle handle, unsigned int priority); protected: - typedef std::vector<HttpOpRequest *> ready_queue_t; - -protected: + int mReadyInClass[HttpRequest::POLICY_CLASS_LIMIT]; + HttpReadyQueue mReadyQueue[HttpRequest::POLICY_CLASS_LIMIT]; HttpService * mService; // Naked pointer, not refcounted, not owner - ready_queue_t mReadyQueue; }; // end class HttpPolicy diff --git a/indra/llcorehttp/_httpreadyqueue.h b/indra/llcorehttp/_httpreadyqueue.h new file mode 100644 index 0000000000..283e868b4c --- /dev/null +++ b/indra/llcorehttp/_httpreadyqueue.h @@ -0,0 +1,85 @@ +/** + * @file _httpreadyqueue.h + * @brief Internal declaration for the operation ready queue + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef _LLCORE_HTTP_READY_QUEUE_H_ +#define _LLCORE_HTTP_READY_QUEUE_H_ + + +#include <queue> + +#include "_httpoperation.h" + + +namespace LLCore +{ + +class HttpOpRequest; + +/// HttpReadyQueue provides a simple priority queue for HttpOpRequest objects. +/// +/// This uses the priority_queue adaptor class to provide the queue +/// as well as the ordering scheme while allowing us access to the +/// raw container if we follow a few simple rules. One of the more +/// important of those rules is that any iterator becomes invalid +/// on element erasure. So pay attention. +/// +/// Threading: not thread-safe. Expected to be used entirely by +/// a single thread, typically a worker thread of some sort. + +class HttpReadyQueue : public std::priority_queue<HttpOpRequest *, + std::deque<HttpOpRequest *>, + LLCore::HttpOpCompare> +{ +public: + HttpReadyQueue() + : priority_queue() + {} + + ~HttpReadyQueue() + {} + +protected: + HttpReadyQueue(const HttpReadyQueue &); // Not defined + void operator=(const HttpReadyQueue &); // Not defined + +public: + const container_type & get_container() const + { + return c; + } + + container_type & get_container() + { + return c; + } + +}; // end class HttpReadyQueue + + +} // end namespace LLCore + + +#endif // _LLCORE_HTTP_READY_QUEUE_H_ diff --git a/indra/llcorehttp/_httprequestqueue.cpp b/indra/llcorehttp/_httprequestqueue.cpp index c36814aee3..92bb5ec5c1 100644 --- a/indra/llcorehttp/_httprequestqueue.cpp +++ b/indra/llcorehttp/_httprequestqueue.cpp @@ -57,7 +57,7 @@ HttpRequestQueue::~HttpRequestQueue() void HttpRequestQueue::init() { - LLINT_ASSERT(! sInstance); + llassert_always(! sInstance); sInstance = new HttpRequestQueue(); } diff --git a/indra/llcorehttp/_httpservice.cpp b/indra/llcorehttp/_httpservice.cpp index 6ebc0ec6cb..48884ca060 100644 --- a/indra/llcorehttp/_httpservice.cpp +++ b/indra/llcorehttp/_httpservice.cpp @@ -35,6 +35,8 @@ #include "_httplibcurl.h" #include "_thread.h" +#include "lltimer.h" + namespace LLCore { @@ -89,8 +91,8 @@ HttpService::~HttpService() void HttpService::init(HttpRequestQueue * queue) { - LLINT_ASSERT(! sInstance); - LLINT_ASSERT(NOT_INITIALIZED == sState); + llassert_always(! sInstance); + llassert_always(NOT_INITIALIZED == sState); sInstance = new HttpService(); queue->addRef(); @@ -103,7 +105,7 @@ void HttpService::init(HttpRequestQueue * queue) void HttpService::term() { - LLINT_ASSERT(RUNNING != sState); + llassert_always(RUNNING != sState); if (sInstance) { delete sInstance; @@ -132,8 +134,8 @@ bool HttpService::isStopped() void HttpService::startThread() { - LLINT_ASSERT(! mThread || STOPPED == sState); - LLINT_ASSERT(INITIALIZED == sState || STOPPED == sState); + llassert_always(! mThread || STOPPED == sState); + llassert_always(INITIALIZED == sState || STOPPED == sState); if (mThread) { @@ -150,6 +152,20 @@ void HttpService::stopRequested() mExitRequested = true; } +bool HttpService::changePriority(HttpHandle handle, unsigned int priority) +{ + bool found(false); + + // Skip the request queue as we currently don't leave earlier + // requests sitting there. Start with the ready queue... + found = mPolicy->changePriority(handle, priority); + + // If not there, we could try the transport/active queue but priority + // doesn't really have much effect there so we don't waste cycles. + + return found; +} + void HttpService::shutdown() { @@ -157,38 +173,46 @@ void HttpService::shutdown() } +// Working thread loop-forever method. Gives time to +// each of the request queue, policy layer and transport +// layer pieces and then either sleeps for a small time +// or waits for a request to come in. Repeats until +// requested to stop. void HttpService::threadRun(LLCoreInt::HttpThread * thread) { boost::this_thread::disable_interruption di; - + ELoopSpeed loop(REQUEST_SLEEP); + while (! mExitRequested) { - processRequestQueue(); + loop = processRequestQueue(loop); // Process ready queue issuing new requests as needed - mPolicy->processReadyQueue(); + ELoopSpeed new_loop = mPolicy->processReadyQueue(); + loop = (std::min)(loop, new_loop); // Give libcurl some cycles - mTransport->processTransport(); + new_loop = mTransport->processTransport(); + loop = (std::min)(loop, new_loop); // Determine whether to spin, sleep briefly or sleep for next request - // *FIXME: For now, do this -#if defined(WIN32) - Sleep(50); -#else - usleep(5000); -#endif + if (REQUEST_SLEEP != loop) + { + ms_sleep(50); + } } + shutdown(); sState = STOPPED; } -void HttpService::processRequestQueue() +HttpService::ELoopSpeed HttpService::processRequestQueue(ELoopSpeed loop) { HttpRequestQueue::OpContainer ops; - - mRequestQueue->fetchAll(false, ops); + const bool wait_for_req(REQUEST_SLEEP == loop); + + mRequestQueue->fetchAll(wait_for_req, ops); while (! ops.empty()) { HttpOperation * op(ops.front()); @@ -203,6 +227,9 @@ void HttpService::processRequestQueue() // Done with operation op->release(); } + + // Queue emptied, allow polling loop to sleep + return REQUEST_SLEEP; } diff --git a/indra/llcorehttp/_httpservice.h b/indra/llcorehttp/_httpservice.h index ba76e1eeca..3e5a5457d7 100644 --- a/indra/llcorehttp/_httpservice.h +++ b/indra/llcorehttp/_httpservice.h @@ -28,6 +28,9 @@ #define _LLCORE_HTTP_SERVICE_H_ +#include "httpcommon.h" + + namespace LLCoreInt { @@ -86,7 +89,17 @@ public: RUNNING, ///< thread created and running STOPPED ///< thread has committed to exiting }; - + + // Ordered enumeration of idling strategies available to + // threadRun's loop. Ordered so that std::min on values + // produces the most conservative result of multiple + // requests. + enum ELoopSpeed + { + NORMAL, ///< continuous polling of request, ready, active queues + REQUEST_SLEEP ///< can sleep indefinitely waiting for request queue write + }; + static void init(HttpRequestQueue *); static void term(); @@ -124,6 +137,15 @@ public: /// Threading: callable by worker thread. void shutdown(); + + /// Try to find the given request handle on any of the request + /// queues and reset the priority (and queue position) of the + /// request if found. + /// + /// @return True if the request was found somewhere. + /// + /// Threading: callable by worker thread. + bool changePriority(HttpHandle handle, unsigned int priority); HttpPolicy * getPolicy() { @@ -138,7 +160,7 @@ public: protected: void threadRun(LLCoreInt::HttpThread * thread); - void processRequestQueue(); + ELoopSpeed processRequestQueue(ELoopSpeed loop); protected: static HttpService * sInstance; diff --git a/indra/llcorehttp/_refcounted.h b/indra/llcorehttp/_refcounted.h index 4a6ce8420a..72cef6b342 100644 --- a/indra/llcorehttp/_refcounted.h +++ b/indra/llcorehttp/_refcounted.h @@ -30,7 +30,7 @@ #include <boost/thread.hpp> -#include "_assert.h" +#include "linden_common.h" namespace LLCoreInt @@ -71,7 +71,7 @@ private: inline void RefCounted::addRef() const { boost::mutex::scoped_lock lock(mRefLock); - LLINT_ASSERT(mRefCount >= 0); + llassert_always(mRefCount >= 0); ++mRefCount; } @@ -82,8 +82,8 @@ inline void RefCounted::release() const { // CRITICAL SECTION boost::mutex::scoped_lock lock(mRefLock); - LLINT_ASSERT(mRefCount != NOT_REF_COUNTED); - LLINT_ASSERT(mRefCount > 0); + llassert_always(mRefCount != NOT_REF_COUNTED); + llassert_always(mRefCount > 0); count = --mRefCount; // CRITICAL SECTION } @@ -104,8 +104,8 @@ inline bool RefCounted::isLastRef() const // CRITICAL SECTION boost::mutex::scoped_lock lock(mRefLock); - LLINT_ASSERT(mRefCount != NOT_REF_COUNTED); - LLINT_ASSERT(mRefCount >= 1); + llassert_always(mRefCount != NOT_REF_COUNTED); + llassert_always(mRefCount >= 1); count = mRefCount; // CRITICAL SECTION } @@ -125,7 +125,7 @@ inline int RefCounted::getRefCount() const inline void RefCounted::noRef() const { boost::mutex::scoped_lock lock(mRefLock); - LLINT_ASSERT(mRefCount <= 1); + llassert_always(mRefCount <= 1); mRefCount = NOT_REF_COUNTED; } diff --git a/indra/llcorehttp/httpcommon.cpp b/indra/llcorehttp/httpcommon.cpp index b5872606b8..273acae132 100644 --- a/indra/llcorehttp/httpcommon.cpp +++ b/indra/llcorehttp/httpcommon.cpp @@ -28,6 +28,7 @@ #include <curl/curl.h> #include <string> +#include <sstream> namespace LLCore @@ -46,6 +47,16 @@ HttpStatus::operator unsigned long() const } +std::string HttpStatus::toHex() const +{ + std::ostringstream result; + result.width(8); + result.fill('0'); + result << std::hex << operator unsigned long(); + return result.str(); + +} + std::string HttpStatus::toString() const { static const char * llcore_errors[] = @@ -54,7 +65,8 @@ std::string HttpStatus::toString() const "HTTP error reply status", "Services shutting down", "Operation canceled", - "Invalid Content-Range header encountered" + "Invalid Content-Range header encountered", + "Request handle not found" }; static const int llcore_errors_count(sizeof(llcore_errors) / sizeof(llcore_errors[0])); @@ -131,6 +143,7 @@ std::string HttpStatus::toString() const default: if (isHttpStatus()) { + // Binary search for the error code and string int bottom(0), top(http_errors_count); while (true) { diff --git a/indra/llcorehttp/httpcommon.h b/indra/llcorehttp/httpcommon.h index f81be7103e..c01a5f85d3 100644 --- a/indra/llcorehttp/httpcommon.h +++ b/indra/llcorehttp/httpcommon.h @@ -134,7 +134,10 @@ enum HttpError HE_OP_CANCELED = 3, // Invalid content range header received. - HE_INV_CONTENT_RANGE_HDR = 4 + HE_INV_CONTENT_RANGE_HDR = 4, + + // Request handle not found + HE_HANDLE_NOT_FOUND = 5 }; // end enum HttpError @@ -229,6 +232,9 @@ struct HttpStatus { return operator unsigned long(); } + + /// And to convert to a hex string. + std::string toHex() const; /// Convert status to a string representation. For /// success, returns an empty string. For failure diff --git a/indra/llcorehttp/httprequest.cpp b/indra/llcorehttp/httprequest.cpp index 6c62f931ff..a06b859a91 100644 --- a/indra/llcorehttp/httprequest.cpp +++ b/indra/llcorehttp/httprequest.cpp @@ -31,6 +31,7 @@ #include "_httpservice.h" #include "_httpoperation.h" #include "_httpoprequest.h" +#include "_httpopsetpriority.h" #include "_httpopcancel.h" @@ -162,7 +163,7 @@ HttpStatus HttpRequest::getStatus() const HttpHandle HttpRequest::requestGetByteRange(unsigned int policy_id, - float priority, + unsigned int priority, const std::string & url, size_t offset, size_t len, @@ -190,6 +191,34 @@ HttpHandle HttpRequest::requestGetByteRange(unsigned int policy_id, } +HttpHandle HttpRequest::requestPost(unsigned int policy_id, + unsigned int priority, + const std::string & url, + BufferArray * body, + HttpOptions * options, + HttpHeaders * headers, + HttpHandler * user_handler) +{ + HttpStatus status; + HttpHandle handle(LLCORE_HTTP_HANDLE_INVALID); + + HttpOpRequest * op = new HttpOpRequest(); + if (! (status = op->setupPost(policy_id, priority, url, body, options, headers))) + { + op->release(); + mLastReqStatus = status; + return handle; + } + op->setHandlers(mReplyQueue, mSelfHandler, user_handler); + mRequestQueue->addOp(op); // transfers refcount + + mLastReqStatus = status; + handle = static_cast<HttpHandle>(op); + + return handle; +} + + HttpHandle HttpRequest::requestCancel(HttpHandle handle, HttpHandler * user_handler) { HttpStatus status; @@ -222,6 +251,23 @@ HttpHandle HttpRequest::requestNoOp(HttpHandler * user_handler) } +HttpHandle HttpRequest::requestSetPriority(HttpHandle request, unsigned int priority, + HttpHandler * handler) +{ + HttpStatus status; + HttpHandle ret_handle(LLCORE_HTTP_HANDLE_INVALID); + + HttpOpSetPriority * op = new HttpOpSetPriority(request, priority); + op->setHandlers(mReplyQueue, mSelfHandler, handler); + mRequestQueue->addOp(op); // transfer refcount as well + + mLastReqStatus = status; + ret_handle = static_cast<HttpHandle>(op); + + return ret_handle; +} + + HttpStatus HttpRequest::update(long millis) { HttpStatus status; @@ -259,7 +305,7 @@ HttpStatus HttpRequest::createService() { HttpStatus status; - LLINT_ASSERT(! has_inited); + llassert_always(! has_inited); HttpRequestQueue::init(); HttpRequestQueue * rq = HttpRequestQueue::instanceOf(); HttpService::init(rq); @@ -273,7 +319,7 @@ HttpStatus HttpRequest::destroyService() { HttpStatus status; - LLINT_ASSERT(has_inited); + llassert_always(has_inited); HttpService::term(); HttpRequestQueue::term(); has_inited = false; diff --git a/indra/llcorehttp/httprequest.h b/indra/llcorehttp/httprequest.h index 4bbd13a13a..e2ab9be533 100644 --- a/indra/llcorehttp/httprequest.h +++ b/indra/llcorehttp/httprequest.h @@ -41,6 +41,7 @@ class HttpService; class HttpOptions; class HttpHeaders; class HttpOperation; +class BufferArray; /// HttpRequest supplies the entry into the HTTP transport /// services in the LLCore libraries. Services provided include: @@ -96,6 +97,10 @@ public: /// Represents a default, catch-all policy class that guarantees /// eventual service for any HTTP request. static const int DEFAULT_POLICY_ID = 0; + + /// Maximum number of policies that may be defined. No policy + /// ID will equal or exceed this value. + static const int POLICY_CLASS_LIMIT = 1; enum EGlobalPolicy { @@ -177,7 +182,7 @@ public: /// @param policy_id Default or user-defined policy class under /// which this request is to be serviced. /// @param priority Standard priority scheme inherited from - /// Indra code base. + /// Indra code base (U32-type scheme). /// @param url /// @param offset /// @param len @@ -190,7 +195,7 @@ public: /// case, @see getStatus() will return more info. /// HttpHandle requestGetByteRange(unsigned int policy_id, - float priority, + unsigned int priority, const std::string & url, size_t offset, size_t len, @@ -199,6 +204,32 @@ public: HttpHandler * handler); + /// + /// @param policy_id Default or user-defined policy class under + /// which this request is to be serviced. + /// @param priority Standard priority scheme inherited from + /// Indra code base. + /// @param url + /// @param body Byte stream to be sent as the body. No + /// further encoding or escaping will be done + /// to the content. + /// @param options (optional) + /// @param headers (optional) + /// @param handler (optional) + /// @return The handle of the request if successfully + /// queued or LLCORE_HTTP_HANDLE_INVALID if the + /// request could not be queued. In the latter + /// case, @see getStatus() will return more info. + /// + HttpHandle requestPost(unsigned int policy_id, + unsigned int priority, + const std::string & url, + BufferArray * body, + HttpOptions * options, + HttpHeaders * headers, + HttpHandler * handler); + + /// Queue a NoOp request. /// The request is queued and serviced by the working thread which /// immediately processes it and returns the request to the reply @@ -235,6 +266,20 @@ public: HttpHandle requestCancel(HttpHandle request, HttpHandler *); + /// Request that a previously-issued request be reprioritized. + /// The status of whether the change itself succeeded arrives + /// via notification. + /// + /// @param request Handle of previously-issued request to + /// be changed. + /// @param priority New priority value. + /// @param handler (optional) + /// @return The handle of the request if successfully + /// queued or LLCORE_HTTP_HANDLE_INVALID if the + /// request could not be queued. + /// + HttpHandle requestSetPriority(HttpHandle request, unsigned int priority, HttpHandler * handler); + /// @} /// @name UtilityMethods diff --git a/indra/llcorehttp/tests/test_httprequest.hpp b/indra/llcorehttp/tests/test_httprequest.hpp index a73d90957e..0e9d7d8979 100644 --- a/indra/llcorehttp/tests/test_httprequest.hpp +++ b/indra/llcorehttp/tests/test_httprequest.hpp @@ -370,7 +370,7 @@ void HttpRequestTestObjectType::test<5>() // Issue a GET that can't connect mStatus = HttpStatus(HttpStatus::EXT_CURL_EASY, CURLE_COULDNT_CONNECT); HttpHandle handle = req->requestGetByteRange(HttpRequest::DEFAULT_POLICY_ID, - 0.0f, + 0U, "http://127.0.0.1:2/nothing/here", 0, 0, |