From 5611cb6d476540e6a1c654c1f9acdce2787b3505 Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Mon, 23 Apr 2012 16:19:39 -0400 Subject: Okay, imported the core-http library and got it compiling suspiciously easily. The unit/integration tests don't work yet as I'm still battling cmake/autobuild as usual but first milestone passed. --- indra/llcorehttp/_httplibcurl.h | 89 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 indra/llcorehttp/_httplibcurl.h (limited to 'indra/llcorehttp/_httplibcurl.h') diff --git a/indra/llcorehttp/_httplibcurl.h b/indra/llcorehttp/_httplibcurl.h new file mode 100644 index 0000000000..5ba244cee4 --- /dev/null +++ b/indra/llcorehttp/_httplibcurl.h @@ -0,0 +1,89 @@ +/** + * @file _httplibcurl.h + * @brief Declarations for internal class providing libcurl transport. + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef _LLCORE_HTTP_LIBCURL_H_ +#define _LLCORE_HTTP_LIBCURL_H_ + + +#include +#include + +#include + + +namespace LLCore +{ + + +class HttpService; +class HttpPolicy; +class HttpOpRequest; +class HttpHeaders; + + +class HttpLibcurl +{ +public: + HttpLibcurl(HttpService * service); + virtual ~HttpLibcurl(); + +private: + HttpLibcurl(const HttpLibcurl &); // Not defined + void operator=(const HttpLibcurl &); // Not defined + +public: + static void init(); + static void term(); + + void processTransport(); + void addOp(HttpOpRequest * op); + + int activeCount() const; + +protected: + void completeRequest(CURLM * multi_handle, CURL * handle, CURLcode status); + +protected: + typedef std::set active_set_t; + +protected: + HttpService * mService; + active_set_t mActiveOps; + CURLM * mMultiHandles[1]; +}; // end class HttpLibcurl + + +// --------------------------------------- +// Free functions +// --------------------------------------- + + +curl_slist * append_headers_to_slist(const HttpHeaders *, curl_slist * slist); + + +} // end namespace LLCore + +#endif // _LLCORE_HTTP_LIBCURL_H_ -- cgit v1.2.3 From 239e072bfcf97b8a12c18ff9974fd0a2929c9ee4 Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Tue, 8 May 2012 12:27:24 -0400 Subject: Unit test still giving me issues on the local windows system. Seems to be a hard stall while allocating the first easy handle in a descent of the global initiailization code but that doesn't seem to be a problem on TC machines. Perhaps the static linking is creating multiple data copies. More work needed. --- indra/llcorehttp/_httplibcurl.h | 1 + 1 file changed, 1 insertion(+) (limited to 'indra/llcorehttp/_httplibcurl.h') diff --git a/indra/llcorehttp/_httplibcurl.h b/indra/llcorehttp/_httplibcurl.h index 5ba244cee4..01c68320af 100644 --- a/indra/llcorehttp/_httplibcurl.h +++ b/indra/llcorehttp/_httplibcurl.h @@ -27,6 +27,7 @@ #ifndef _LLCORE_HTTP_LIBCURL_H_ #define _LLCORE_HTTP_LIBCURL_H_ +#include "linden_common.h" // Modifies curl/curl.h interfaces #include #include -- cgit v1.2.3 From 8fc350125c671baeae6b7f8b1814251009f4f50a Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Wed, 23 May 2012 19:12:09 -0400 Subject: Integrate llcorehttp library into lltexturefetch design. This is the first functional viewer pass with the HTTP work of the texture fetch code performed by the llcorehttp library. Not exactly a 'drop-in' replacement but a work-alike with some changes (e.g. handler notification in consumer thread versus responder notification in worker thread). This also includes some temporary changes in the priority scheme to prevent the kind of priority inversion found in VWR-28996. Scheme used here does provide liveness if not optimal responsiveness or order-of-operation. The llcorehttp library at this point is far from optimally performing. Its worker thread is making relatively poor use of cycles it gets and it doesn't idle or sleep intelligently yet. This early integration step helps shake out the interfaces, implementation niceties will be covered soon. --- indra/llcorehttp/_httplibcurl.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'indra/llcorehttp/_httplibcurl.h') diff --git a/indra/llcorehttp/_httplibcurl.h b/indra/llcorehttp/_httplibcurl.h index 01c68320af..807196628d 100644 --- a/indra/llcorehttp/_httplibcurl.h +++ b/indra/llcorehttp/_httplibcurl.h @@ -45,6 +45,7 @@ class HttpOpRequest; class HttpHeaders; +/// Implements libcurl-based transport for an HttpService instance. class HttpLibcurl { public: @@ -71,7 +72,7 @@ protected: typedef std::set active_set_t; protected: - HttpService * mService; + HttpService * mService; // Simple reference, not owner active_set_t mActiveOps; CURLM * mMultiHandles[1]; }; // end class HttpLibcurl -- cgit v1.2.3 From b8edacd0bb4feacc3ac1d61421e600c75ab87f7c Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Fri, 1 Jun 2012 14:07:34 -0400 Subject: Major steps towards implementing the policy component. Identified and reacted to the priority inversion problem we have in texturefetch. Includes the introduction of a priority_queue for the requests that are ready. Start some parameterization in anticipation of having policy_class everywhere. Removed _assert.h which isn't really needed in indra codebase. Implemented async setPriority request (which I hope I can get rid of eventually along with all priorities in this library). Converted to using unsigned int for priority rather than float. Implemented POST and did groundwork for PUT. --- indra/llcorehttp/_httplibcurl.h | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) (limited to 'indra/llcorehttp/_httplibcurl.h') diff --git a/indra/llcorehttp/_httplibcurl.h b/indra/llcorehttp/_httplibcurl.h index 807196628d..ec325c1946 100644 --- a/indra/llcorehttp/_httplibcurl.h +++ b/indra/llcorehttp/_httplibcurl.h @@ -34,18 +34,25 @@ #include +#include "httprequest.h" +#include "_httpservice.h" + namespace LLCore { -class HttpService; class HttpPolicy; class HttpOpRequest; class HttpHeaders; /// Implements libcurl-based transport for an HttpService instance. +/// +/// Threading: Single-threaded. Other than for construction/destruction, +/// all methods are expected to be invoked in a single thread, typically +/// a worker thread of some sort. + class HttpLibcurl { public: @@ -60,12 +67,22 @@ public: static void init(); static void term(); - void processTransport(); - void addOp(HttpOpRequest * op); + /// Give cycles to libcurl to run active requests. Completed + /// operations (successful or failed) will be retried or handed + /// over to the reply queue as final responses. + HttpService::ELoopSpeed processTransport(); - int activeCount() const; + /// Add request to the active list. Caller is expected to have + /// provided us with a reference count to hold the request. (No + /// additional references will be added.) + void addOp(HttpOpRequest * op); + int getActiveCount() const; + int getActiveCountInClass(int policy_class) const; + protected: + /// Invoked when libcurl has indicated a request has been processed + /// to completion and we need to move the request to a new state. void completeRequest(CURLM * multi_handle, CURL * handle, CURLcode status); protected: @@ -74,7 +91,7 @@ protected: protected: HttpService * mService; // Simple reference, not owner active_set_t mActiveOps; - CURLM * mMultiHandles[1]; + CURLM * mMultiHandles[HttpRequest::POLICY_CLASS_LIMIT]; }; // end class HttpLibcurl -- cgit v1.2.3 From 28a04400b4160dd34166483ddcf0c12637bcc363 Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Fri, 8 Jun 2012 20:21:54 -0400 Subject: Implemented HTTP retry for requests. Went in rather easily which surprised me. Added a retry queue similar to ready queue to the policy object which is sorted by retry time. Currently do five retries (after the initial try) delayed by .25, .5, 1, 2 and 5 seconds. Removed the retry logic from the lltexturefetch module. Upped the waiting time in the unit test for the retries. People won't like this but tough, need tests. --- indra/llcorehttp/_httplibcurl.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'indra/llcorehttp/_httplibcurl.h') diff --git a/indra/llcorehttp/_httplibcurl.h b/indra/llcorehttp/_httplibcurl.h index ec325c1946..fe628b9ab0 100644 --- a/indra/llcorehttp/_httplibcurl.h +++ b/indra/llcorehttp/_httplibcurl.h @@ -83,7 +83,9 @@ public: protected: /// Invoked when libcurl has indicated a request has been processed /// to completion and we need to move the request to a new state. - void completeRequest(CURLM * multi_handle, CURL * handle, CURLcode status); + HttpService::ELoopSpeed completeRequest(CURLM * multi_handle, + CURL * handle, + CURLcode status); protected: typedef std::set active_set_t; -- cgit v1.2.3 From 89187229dd630845177ecd7a16e2b9cb01dc47ce Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Mon, 11 Jun 2012 15:28:06 -0400 Subject: Refactoring of the request completion thread and removal of 206/content-range hack in xport. Retry/response handling is decided in policy so moved that there. Removed special case 206-without-content-range response in transport. Have this sitation recognizable in the API and let callers deal with it as needed. --- indra/llcorehttp/_httplibcurl.h | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'indra/llcorehttp/_httplibcurl.h') diff --git a/indra/llcorehttp/_httplibcurl.h b/indra/llcorehttp/_httplibcurl.h index fe628b9ab0..16b68bde43 100644 --- a/indra/llcorehttp/_httplibcurl.h +++ b/indra/llcorehttp/_httplibcurl.h @@ -83,9 +83,7 @@ public: protected: /// Invoked when libcurl has indicated a request has been processed /// to completion and we need to move the request to a new state. - HttpService::ELoopSpeed completeRequest(CURLM * multi_handle, - CURL * handle, - CURLcode status); + bool completeRequest(CURLM * multi_handle, CURL * handle, CURLcode status); protected: typedef std::set active_set_t; -- cgit v1.2.3 From a50944e078b98435685f04eda0ba93e95d4c61f2 Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Tue, 19 Jun 2012 17:01:02 -0400 Subject: Cleanup: move magic nubmers to new _httpinternal.h header file. --- indra/llcorehttp/_httplibcurl.h | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) (limited to 'indra/llcorehttp/_httplibcurl.h') diff --git a/indra/llcorehttp/_httplibcurl.h b/indra/llcorehttp/_httplibcurl.h index 16b68bde43..0d0c4cad6d 100644 --- a/indra/llcorehttp/_httplibcurl.h +++ b/indra/llcorehttp/_httplibcurl.h @@ -36,6 +36,7 @@ #include "httprequest.h" #include "_httpservice.h" +#include "_httpinternal.h" namespace LLCore @@ -91,17 +92,8 @@ protected: protected: HttpService * mService; // Simple reference, not owner active_set_t mActiveOps; - CURLM * mMultiHandles[HttpRequest::POLICY_CLASS_LIMIT]; -}; // end class HttpLibcurl - - -// --------------------------------------- -// Free functions -// --------------------------------------- - - -curl_slist * append_headers_to_slist(const HttpHeaders *, curl_slist * slist); - + CURLM * mMultiHandles[POLICY_CLASS_LIMIT]; +}; // end class HttpLibcurl } // end namespace LLCore -- cgit v1.2.3 From bc7d5b24d16963a2715e880c518a4706a99f02fa Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Fri, 22 Jun 2012 19:13:50 -0400 Subject: This sets down the groundwork for dynamic policy classes. Groundwork is used for the default class which currently represents texture fetching. Class options implemented from API user into HttpLibcurl. Policy layer is going to start doing some traffic shaping like work to solve problems with consumer-grade gear. Need to have dynamic aspects to policies and that starts now... --- indra/llcorehttp/_httplibcurl.h | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'indra/llcorehttp/_httplibcurl.h') diff --git a/indra/llcorehttp/_httplibcurl.h b/indra/llcorehttp/_httplibcurl.h index 0d0c4cad6d..5e1dd1bfbf 100644 --- a/indra/llcorehttp/_httplibcurl.h +++ b/indra/llcorehttp/_httplibcurl.h @@ -78,6 +78,10 @@ public: /// additional references will be added.) void addOp(HttpOpRequest * op); + /// One-time call to set the number of policy classes to be + /// serviced and to create the resources for each. + void setPolicyCount(int policy_count); + int getActiveCount() const; int getActiveCountInClass(int policy_class) const; @@ -92,7 +96,8 @@ protected: protected: HttpService * mService; // Simple reference, not owner active_set_t mActiveOps; - CURLM * mMultiHandles[POLICY_CLASS_LIMIT]; + int mPolicyCount; + CURLM ** mMultiHandles; }; // end class HttpLibcurl } // end namespace LLCore -- cgit v1.2.3 From e172ec84fa217aae8d1e51c1e0673322c30891fe Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Sat, 23 Jun 2012 23:33:50 -0400 Subject: SH-3184/SH-3221 Improve cleanup, destructor, thread termination, etc. logic in library. With this commit, the cleanup paths should be production quality. Unit tests have been expanded to include cases requiring thread termination and cleanup by the worker thread. Special operation/request added to support the unit tests. Thread interface expanded to include a very aggressive cancel() method that does not do cleanup but prevents the thread from accessing objects that will be destroyed. --- indra/llcorehttp/_httplibcurl.h | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) (limited to 'indra/llcorehttp/_httplibcurl.h') diff --git a/indra/llcorehttp/_httplibcurl.h b/indra/llcorehttp/_httplibcurl.h index 5e1dd1bfbf..69f7bb2b6d 100644 --- a/indra/llcorehttp/_httplibcurl.h +++ b/indra/llcorehttp/_httplibcurl.h @@ -65,9 +65,6 @@ private: void operator=(const HttpLibcurl &); // Not defined public: - static void init(); - static void term(); - /// Give cycles to libcurl to run active requests. Completed /// operations (successful or failed) will be retried or handed /// over to the reply queue as final responses. @@ -79,16 +76,23 @@ public: void addOp(HttpOpRequest * op); /// One-time call to set the number of policy classes to be - /// serviced and to create the resources for each. - void setPolicyCount(int policy_count); + /// serviced and to create the resources for each. Value + /// must agree with HttpPolicy::setPolicies() call. + void start(int policy_count); + + void shutdown(); int getActiveCount() const; int getActiveCountInClass(int policy_class) const; - + protected: /// Invoked when libcurl has indicated a request has been processed /// to completion and we need to move the request to a new state. bool completeRequest(CURLM * multi_handle, CURL * handle, CURLcode status); + + /// Invoked to cancel an active request, mainly during shutdown + /// and destroy. + void cancelRequest(HttpOpRequest * op); protected: typedef std::set active_set_t; -- cgit v1.2.3 From 2d7b7de20327a40be12a620debaae9917af16cd6 Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Tue, 3 Jul 2012 13:06:46 -0400 Subject: More integration work for texture fetch timeouts. The fetch state machine received a new timeout during the WAIT_HTTP_REQ state. For the integration, rather than jump the state to done, we issue a request cancel and let the notification plumbing do the rest without any race conditions or special-case logic. --- indra/llcorehttp/_httplibcurl.h | 3 +++ 1 file changed, 3 insertions(+) (limited to 'indra/llcorehttp/_httplibcurl.h') diff --git a/indra/llcorehttp/_httplibcurl.h b/indra/llcorehttp/_httplibcurl.h index 69f7bb2b6d..53972b1ffa 100644 --- a/indra/llcorehttp/_httplibcurl.h +++ b/indra/llcorehttp/_httplibcurl.h @@ -85,6 +85,9 @@ public: int getActiveCount() const; int getActiveCountInClass(int policy_class) const; + // Shadows HttpService's method + bool cancel(HttpHandle handle); + protected: /// Invoked when libcurl has indicated a request has been processed /// to completion and we need to move the request to a new state. -- cgit v1.2.3 From 85e69b043b098dbe5a09f2eac6ff541123089f13 Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Mon, 23 Jul 2012 23:40:07 +0000 Subject: Big comment and naming cleanup. Ready for prime-time. Add to-do list to _httpinternal.h to guide anyone who wants to pitch in and help. --- indra/llcorehttp/_httplibcurl.h | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) (limited to 'indra/llcorehttp/_httplibcurl.h') diff --git a/indra/llcorehttp/_httplibcurl.h b/indra/llcorehttp/_httplibcurl.h index 53972b1ffa..611f029ef5 100644 --- a/indra/llcorehttp/_httplibcurl.h +++ b/indra/llcorehttp/_httplibcurl.h @@ -68,24 +68,41 @@ public: /// Give cycles to libcurl to run active requests. Completed /// operations (successful or failed) will be retried or handed /// over to the reply queue as final responses. + /// + /// @return Indication of how long this method is + /// willing to wait for next service call. HttpService::ELoopSpeed processTransport(); /// Add request to the active list. Caller is expected to have - /// provided us with a reference count to hold the request. (No - /// additional references will be added.) + /// provided us with a reference count on the op to hold the + /// request. (No additional references will be added.) void addOp(HttpOpRequest * op); /// One-time call to set the number of policy classes to be /// serviced and to create the resources for each. Value /// must agree with HttpPolicy::setPolicies() call. void start(int policy_count); - + + /// Synchronously stop libcurl operations. All active requests + /// are canceled and removed from libcurl's handling. Easy + /// handles are detached from their multi handles and released. + /// Multi handles are also released. Canceled requests are + /// completed with canceled status and made available on their + /// respective reply queues. + /// + /// Can be restarted with a start() call. void shutdown(); - + + /// Return global and per-class counts of active requests. int getActiveCount() const; int getActiveCountInClass(int policy_class) const; - // Shadows HttpService's method + /// Attempt to cancel a request identified by handle. + /// + /// Interface shadows HttpService's method. + /// + /// @return True if handle was found and operation canceled. + /// bool cancel(HttpHandle handle); protected: -- cgit v1.2.3