From 5611cb6d476540e6a1c654c1f9acdce2787b3505 Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Mon, 23 Apr 2012 16:19:39 -0400 Subject: Okay, imported the core-http library and got it compiling suspiciously easily. The unit/integration tests don't work yet as I'm still battling cmake/autobuild as usual but first milestone passed. --- indra/llcorehttp/_httpoperation.cpp | 203 ++++++++++++++++++++++++++++++++++++ 1 file changed, 203 insertions(+) create mode 100644 indra/llcorehttp/_httpoperation.cpp (limited to 'indra/llcorehttp/_httpoperation.cpp') diff --git a/indra/llcorehttp/_httpoperation.cpp b/indra/llcorehttp/_httpoperation.cpp new file mode 100644 index 0000000000..17c65b0379 --- /dev/null +++ b/indra/llcorehttp/_httpoperation.cpp @@ -0,0 +1,203 @@ +/** + * @file _httpoperation.cpp + * @brief Definitions for internal classes based on HttpOperation + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include "_httpoperation.h" + +#include "httphandler.h" +#include "httpresponse.h" +#include "httprequest.h" + +#include "_httprequestqueue.h" +#include "_httpreplyqueue.h" +#include "_httpservice.h" + + +namespace LLCore +{ + + +// ================================== +// HttpOperation +// ================================== + + +HttpOperation::HttpOperation() + : LLCoreInt::RefCounted(true), + mReplyQueue(NULL), + mLibraryHandler(NULL), + mUserHandler(NULL), + mReqPolicy(HttpRequest::DEFAULT_POLICY_ID), + mReqPriority(0.0f) +{ +} + + +HttpOperation::~HttpOperation() +{ + setHandlers(NULL, NULL, NULL); +} + + +void HttpOperation::setHandlers(HttpReplyQueue * reply_queue, + HttpHandler * lib_handler, + HttpHandler * user_handler) +{ + if (reply_queue != mReplyQueue) + { + if (mReplyQueue) + { + mReplyQueue->release(); + } + + if (reply_queue) + { + reply_queue->addRef(); + } + + mReplyQueue = reply_queue; + } + + // Not refcounted + mLibraryHandler = lib_handler; + + // Not refcounted + mUserHandler = user_handler; +} + + + +void HttpOperation::stageFromRequest(HttpService *) +{ + // *FIXME: Message this a better way later. + // Default implementation should never be called. This + // indicates an operation making a transition that isn't + // defined. + LLINT_ASSERT(false); +} + + +void HttpOperation::stageFromReady(HttpService *) +{ + // *FIXME: Message this a better way later. + // Default implementation should never be called. This + // indicates an operation making a transition that isn't + // defined. + LLINT_ASSERT(false); +} + + +void HttpOperation::stageFromActive(HttpService *) +{ + // *FIXME: Message this a better way later. + // Default implementation should never be called. This + // indicates an operation making a transition that isn't + // defined. + LLINT_ASSERT(false); +} + + +void HttpOperation::visitNotifier(HttpRequest *) +{ + if (mLibraryHandler) + { + HttpResponse * response = new HttpResponse(); + + mLibraryHandler->onCompleted(static_cast(this), response); + + response->release(); + } +} + + +HttpStatus HttpOperation::cancel() +{ + HttpStatus status; + + return status; +} + + +void HttpOperation::addAsReply() +{ + if (mReplyQueue && mLibraryHandler) + { + addRef(); + mReplyQueue->addOp(this); + } +} + + +// ================================== +// HttpOpStop +// ================================== + + +HttpOpStop::HttpOpStop() + : HttpOperation() +{} + + +HttpOpStop::~HttpOpStop() +{} + + +void HttpOpStop::stageFromRequest(HttpService * service) +{ + // Do operations + service->stopRequested(); + + // Prepare response if needed + addAsReply(); +} + + +// ================================== +// HttpOpNull +// ================================== + + +HttpOpNull::HttpOpNull() + : HttpOperation() +{} + + +HttpOpNull::~HttpOpNull() +{} + + +void HttpOpNull::stageFromRequest(HttpService * service) +{ + // Perform op + // Nothing to perform. This doesn't fall into the libcurl + // ready/active queues, it just bounces over to the reply + // queue directly. + + // Prepare response if needed + addAsReply(); +} + + +} // end namespace LLCore -- cgit v1.2.3 From b8edacd0bb4feacc3ac1d61421e600c75ab87f7c Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Fri, 1 Jun 2012 14:07:34 -0400 Subject: Major steps towards implementing the policy component. Identified and reacted to the priority inversion problem we have in texturefetch. Includes the introduction of a priority_queue for the requests that are ready. Start some parameterization in anticipation of having policy_class everywhere. Removed _assert.h which isn't really needed in indra codebase. Implemented async setPriority request (which I hope I can get rid of eventually along with all priorities in this library). Converted to using unsigned int for priority rather than float. Implemented POST and did groundwork for PUT. --- indra/llcorehttp/_httpoperation.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'indra/llcorehttp/_httpoperation.cpp') diff --git a/indra/llcorehttp/_httpoperation.cpp b/indra/llcorehttp/_httpoperation.cpp index 17c65b0379..d966efd12b 100644 --- a/indra/llcorehttp/_httpoperation.cpp +++ b/indra/llcorehttp/_httpoperation.cpp @@ -50,7 +50,7 @@ HttpOperation::HttpOperation() mLibraryHandler(NULL), mUserHandler(NULL), mReqPolicy(HttpRequest::DEFAULT_POLICY_ID), - mReqPriority(0.0f) + mReqPriority(0U) { } @@ -95,7 +95,7 @@ void HttpOperation::stageFromRequest(HttpService *) // Default implementation should never be called. This // indicates an operation making a transition that isn't // defined. - LLINT_ASSERT(false); + llassert_always(false); } @@ -105,7 +105,7 @@ void HttpOperation::stageFromReady(HttpService *) // Default implementation should never be called. This // indicates an operation making a transition that isn't // defined. - LLINT_ASSERT(false); + llassert_always(false); } @@ -115,7 +115,7 @@ void HttpOperation::stageFromActive(HttpService *) // Default implementation should never be called. This // indicates an operation making a transition that isn't // defined. - LLINT_ASSERT(false); + llassert_always(false); } -- cgit v1.2.3 From 7adeb3923728ca84a309a6af141c148ce38066fc Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Tue, 12 Jun 2012 17:42:33 -0400 Subject: HTTP Proxy, PUT & POST, unit tests and refactoring. Implemented/modified PUT & POST to not used chunked encoding for the request. Made the unit test much happier and probably a better thing for the pipeline. Have a cheesy static & dynamic proxy capability using both local options and a way to wire into LLProxy in llmessages. Not a clean thing but it will get the proxy path working with both socks5 & http proxies. Refactoring to get rid of unneeded library handler and unified an HttpStatus return for all requests. Big batch of code removed as a result of that and more is possible as well as some syscall avoidance with a bit more work. Boosted the unit tests for simple PUT & POST test which revealed the test harness does *not* like chunked encoding so we'll avoid it for now (and don't really need it in any of our schemes). --- indra/llcorehttp/_httpoperation.cpp | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) (limited to 'indra/llcorehttp/_httpoperation.cpp') diff --git a/indra/llcorehttp/_httpoperation.cpp b/indra/llcorehttp/_httpoperation.cpp index d966efd12b..b5c58013d4 100644 --- a/indra/llcorehttp/_httpoperation.cpp +++ b/indra/llcorehttp/_httpoperation.cpp @@ -47,7 +47,6 @@ namespace LLCore HttpOperation::HttpOperation() : LLCoreInt::RefCounted(true), mReplyQueue(NULL), - mLibraryHandler(NULL), mUserHandler(NULL), mReqPolicy(HttpRequest::DEFAULT_POLICY_ID), mReqPriority(0U) @@ -57,13 +56,12 @@ HttpOperation::HttpOperation() HttpOperation::~HttpOperation() { - setHandlers(NULL, NULL, NULL); + setReplyPath(NULL, NULL); } -void HttpOperation::setHandlers(HttpReplyQueue * reply_queue, - HttpHandler * lib_handler, - HttpHandler * user_handler) +void HttpOperation::setReplyPath(HttpReplyQueue * reply_queue, + HttpHandler * user_handler) { if (reply_queue != mReplyQueue) { @@ -80,9 +78,6 @@ void HttpOperation::setHandlers(HttpReplyQueue * reply_queue, mReplyQueue = reply_queue; } - // Not refcounted - mLibraryHandler = lib_handler; - // Not refcounted mUserHandler = user_handler; } @@ -121,11 +116,12 @@ void HttpOperation::stageFromActive(HttpService *) void HttpOperation::visitNotifier(HttpRequest *) { - if (mLibraryHandler) + if (mUserHandler) { HttpResponse * response = new HttpResponse(); - mLibraryHandler->onCompleted(static_cast(this), response); + response->setStatus(mStatus); + mUserHandler->onCompleted(static_cast(this), response); response->release(); } @@ -142,7 +138,7 @@ HttpStatus HttpOperation::cancel() void HttpOperation::addAsReply() { - if (mReplyQueue && mLibraryHandler) + if (mReplyQueue) { addRef(); mReplyQueue->addOp(this); -- cgit v1.2.3 From b08125a5874a89ce5210f8fb2c961ae17fb80fde Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Thu, 14 Jun 2012 16:31:48 -0400 Subject: LLMutex recursive lock, global & per-request tracing, simple GET request, LLProxy support, HttpOptions starting to work, HTTP resource waiting fixed. Non-LLThread-based threads need to do some registration or LLMutex locks taken out in these threads will not work as expected (SH-3154). We'll get a better solution later, this fixes some things for now. Tracing of operations now supported. Global and per-request (via HttpOptions) tracing levels of [0..3]. The 2 and 3 levels use libcurl's VERBOSE mode combined with CURLOPT_DEBUGFUNCTION to stream high levels of detail into the log. *Very* laggy but useful. Simple GET request supported (no Range: header). Really just a degenrate case of a ranged get but supplied an API anyway. Global option to use the LLProxy interface to setup CURL handles for either socks5 or http proxy usage. This isn't really the most encapsulated way to do this but a better solution will have to come later. The wantHeaders and tracing options are now supported in HttpOptions giving per-request controls. Big refactoring of the HTTP resource waiter in lltexturefetch. What I was doing before wasn't correct. Instead, I'm implementing the resource wait after the Semaphore model (though not using system semaphores). So instead of having a sequence like: SEND_HTTP_REQ -> WAIT_HTTP_RESOURCE -> SEND_HTTP_REQ, we now do WAIT_HTTP_RESOURCE -> WAIT_HTTP_RESOURCE2 (actual wait) -> SEND_HTTP_REQ. Works well but the prioritized filling of the corehttp library needs some performance work later. --- indra/llcorehttp/_httpoperation.cpp | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) (limited to 'indra/llcorehttp/_httpoperation.cpp') diff --git a/indra/llcorehttp/_httpoperation.cpp b/indra/llcorehttp/_httpoperation.cpp index b5c58013d4..5a31bf90e7 100644 --- a/indra/llcorehttp/_httpoperation.cpp +++ b/indra/llcorehttp/_httpoperation.cpp @@ -34,6 +34,8 @@ #include "_httpreplyqueue.h" #include "_httpservice.h" +#include "lltimer.h" + namespace LLCore { @@ -49,8 +51,10 @@ HttpOperation::HttpOperation() mReplyQueue(NULL), mUserHandler(NULL), mReqPolicy(HttpRequest::DEFAULT_POLICY_ID), - mReqPriority(0U) + mReqPriority(0U), + mTracing(0) { + mMetricCreated = totalTime(); } @@ -113,7 +117,7 @@ void HttpOperation::stageFromActive(HttpService *) llassert_always(false); } - + void HttpOperation::visitNotifier(HttpRequest *) { if (mUserHandler) @@ -138,6 +142,13 @@ HttpStatus HttpOperation::cancel() void HttpOperation::addAsReply() { + if (mTracing > 0) + { + LL_INFOS("CoreHttp") << "TRACE, ToReplyQueue, Handle: " + << static_cast(this) + << LL_ENDL; + } + if (mReplyQueue) { addRef(); -- cgit v1.2.3 From a50944e078b98435685f04eda0ba93e95d4c61f2 Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Tue, 19 Jun 2012 17:01:02 -0400 Subject: Cleanup: move magic nubmers to new _httpinternal.h header file. --- indra/llcorehttp/_httpoperation.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'indra/llcorehttp/_httpoperation.cpp') diff --git a/indra/llcorehttp/_httpoperation.cpp b/indra/llcorehttp/_httpoperation.cpp index 5a31bf90e7..0d9553434e 100644 --- a/indra/llcorehttp/_httpoperation.cpp +++ b/indra/llcorehttp/_httpoperation.cpp @@ -33,6 +33,7 @@ #include "_httprequestqueue.h" #include "_httpreplyqueue.h" #include "_httpservice.h" +#include "_httpinternal.h" #include "lltimer.h" @@ -142,7 +143,7 @@ HttpStatus HttpOperation::cancel() void HttpOperation::addAsReply() { - if (mTracing > 0) + if (mTracing > TRACE_OFF) { LL_INFOS("CoreHttp") << "TRACE, ToReplyQueue, Handle: " << static_cast(this) -- cgit v1.2.3 From e172ec84fa217aae8d1e51c1e0673322c30891fe Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Sat, 23 Jun 2012 23:33:50 -0400 Subject: SH-3184/SH-3221 Improve cleanup, destructor, thread termination, etc. logic in library. With this commit, the cleanup paths should be production quality. Unit tests have been expanded to include cases requiring thread termination and cleanup by the worker thread. Special operation/request added to support the unit tests. Thread interface expanded to include a very aggressive cancel() method that does not do cleanup but prevents the thread from accessing objects that will be destroyed. --- indra/llcorehttp/_httpoperation.cpp | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) (limited to 'indra/llcorehttp/_httpoperation.cpp') diff --git a/indra/llcorehttp/_httpoperation.cpp b/indra/llcorehttp/_httpoperation.cpp index 0d9553434e..b35ab79d65 100644 --- a/indra/llcorehttp/_httpoperation.cpp +++ b/indra/llcorehttp/_httpoperation.cpp @@ -208,4 +208,40 @@ void HttpOpNull::stageFromRequest(HttpService * service) } +// ================================== +// HttpOpSpin +// ================================== + + +HttpOpSpin::HttpOpSpin(int mode) + : HttpOperation(), + mMode(mode) +{} + + +HttpOpSpin::~HttpOpSpin() +{} + + +void HttpOpSpin::stageFromRequest(HttpService * service) +{ + if (0 == mMode) + { + // Spin forever + while (true) + { + ms_sleep(100); + } + } + else + { + this->addRef(); + if (! HttpRequestQueue::instanceOf()->addOp(this)) + { + this->release(); + } + } +} + + } // end namespace LLCore -- cgit v1.2.3 From e8b0088d1a0c02bfa1f9768dc91fc3df4322adae Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Tue, 26 Jun 2012 12:28:58 -0400 Subject: SH-3184/SH-3221 More work on cleanup with better unit test work and more aggressive shutdown of a thread. Some additional work let me enable a memory check for the clean shutdown case and generally do a better job on other interfaces. Request queue waiters now awake on shutdown and don't sleep once the queue is turned off. Much better semantically for how this will be used. --- indra/llcorehttp/_httpoperation.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'indra/llcorehttp/_httpoperation.cpp') diff --git a/indra/llcorehttp/_httpoperation.cpp b/indra/llcorehttp/_httpoperation.cpp index b35ab79d65..d80a8236e6 100644 --- a/indra/llcorehttp/_httpoperation.cpp +++ b/indra/llcorehttp/_httpoperation.cpp @@ -235,8 +235,9 @@ void HttpOpSpin::stageFromRequest(HttpService * service) } else { + ms_sleep(1); // backoff interlock plumbing a bit this->addRef(); - if (! HttpRequestQueue::instanceOf()->addOp(this)) + if (! service->getRequestQueue().addOp(this)) { this->release(); } -- cgit v1.2.3 From f5f51d3cda8861b3b3a380cc96aaca98e572c377 Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Sat, 7 Jul 2012 19:35:32 -0400 Subject: SH-3185 Fill in some FIXME/TODO cases Also added some comments and changed the callback userdata argument to be an HttpOpRequest rather than a libcurl handle. Less code, less clutter. --- indra/llcorehttp/_httpoperation.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'indra/llcorehttp/_httpoperation.cpp') diff --git a/indra/llcorehttp/_httpoperation.cpp b/indra/llcorehttp/_httpoperation.cpp index d80a8236e6..910dbf1f2f 100644 --- a/indra/llcorehttp/_httpoperation.cpp +++ b/indra/llcorehttp/_httpoperation.cpp @@ -91,31 +91,31 @@ void HttpOperation::setReplyPath(HttpReplyQueue * reply_queue, void HttpOperation::stageFromRequest(HttpService *) { - // *FIXME: Message this a better way later. // Default implementation should never be called. This // indicates an operation making a transition that isn't // defined. - llassert_always(false); + LL_ERRS("HttpCore") << "Default stateFromRequest method may not be called." + << LL_ENDL; } void HttpOperation::stageFromReady(HttpService *) { - // *FIXME: Message this a better way later. // Default implementation should never be called. This // indicates an operation making a transition that isn't // defined. - llassert_always(false); + LL_ERRS("HttpCore") << "Default stateFromReady method may not be called." + << LL_ENDL; } void HttpOperation::stageFromActive(HttpService *) { - // *FIXME: Message this a better way later. // Default implementation should never be called. This // indicates an operation making a transition that isn't // defined. - llassert_always(false); + LL_ERRS("HttpCore") << "Default stateFromActive method may not be called." + << LL_ENDL; } -- cgit v1.2.3 From 85e69b043b098dbe5a09f2eac6ff541123089f13 Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Mon, 23 Jul 2012 23:40:07 +0000 Subject: Big comment and naming cleanup. Ready for prime-time. Add to-do list to _httpinternal.h to guide anyone who wants to pitch in and help. --- indra/llcorehttp/_httpoperation.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'indra/llcorehttp/_httpoperation.cpp') diff --git a/indra/llcorehttp/_httpoperation.cpp b/indra/llcorehttp/_httpoperation.cpp index 910dbf1f2f..5cf5bc5930 100644 --- a/indra/llcorehttp/_httpoperation.cpp +++ b/indra/llcorehttp/_httpoperation.cpp @@ -94,7 +94,7 @@ void HttpOperation::stageFromRequest(HttpService *) // Default implementation should never be called. This // indicates an operation making a transition that isn't // defined. - LL_ERRS("HttpCore") << "Default stateFromRequest method may not be called." + LL_ERRS("HttpCore") << "Default stageFromRequest method may not be called." << LL_ENDL; } @@ -104,7 +104,7 @@ void HttpOperation::stageFromReady(HttpService *) // Default implementation should never be called. This // indicates an operation making a transition that isn't // defined. - LL_ERRS("HttpCore") << "Default stateFromReady method may not be called." + LL_ERRS("HttpCore") << "Default stageFromReady method may not be called." << LL_ENDL; } @@ -114,7 +114,7 @@ void HttpOperation::stageFromActive(HttpService *) // Default implementation should never be called. This // indicates an operation making a transition that isn't // defined. - LL_ERRS("HttpCore") << "Default stateFromActive method may not be called." + LL_ERRS("HttpCore") << "Default stageFromActive method may not be called." << LL_ENDL; } @@ -143,7 +143,7 @@ HttpStatus HttpOperation::cancel() void HttpOperation::addAsReply() { - if (mTracing > TRACE_OFF) + if (mTracing > HTTP_TRACE_OFF) { LL_INFOS("CoreHttp") << "TRACE, ToReplyQueue, Handle: " << static_cast(this) -- cgit v1.2.3