From 8fc350125c671baeae6b7f8b1814251009f4f50a Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Wed, 23 May 2012 19:12:09 -0400 Subject: Integrate llcorehttp library into lltexturefetch design. This is the first functional viewer pass with the HTTP work of the texture fetch code performed by the llcorehttp library. Not exactly a 'drop-in' replacement but a work-alike with some changes (e.g. handler notification in consumer thread versus responder notification in worker thread). This also includes some temporary changes in the priority scheme to prevent the kind of priority inversion found in VWR-28996. Scheme used here does provide liveness if not optimal responsiveness or order-of-operation. The llcorehttp library at this point is far from optimally performing. Its worker thread is making relatively poor use of cycles it gets and it doesn't idle or sleep intelligently yet. This early integration step helps shake out the interfaces, implementation niceties will be covered soon. --- indra/newview/llappviewer.cpp | 159 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 159 insertions(+) (limited to 'indra/newview/llappviewer.cpp') diff --git a/indra/newview/llappviewer.cpp b/indra/newview/llappviewer.cpp index 1174d108d2..8e6deb9cce 100644 --- a/indra/newview/llappviewer.cpp +++ b/indra/newview/llappviewer.cpp @@ -212,6 +212,11 @@ #include "llmachineid.h" #include "llmainlooprepeater.h" +// LLCore::HTTP +#include "httpcommon.h" +#include "httprequest.h" +#include "httphandler.h" + // *FIX: These extern globals should be cleaned up. // The globals either represent state/config/resource-storage of either // this app, or another 'component' of the viewer. App globals should be @@ -326,6 +331,53 @@ static std::string gLaunchFileOnQuit; // Used on Win32 for other apps to identify our window (eg, win_setup) const char* const VIEWER_WINDOW_CLASSNAME = "Second Life"; +namespace +{ + +// This class manages the lifecyle of the core http library. +// Slightly different style than traditional code but reflects +// the use of handler classes and light-weight interface +// object instances of the new libraries. To be used +// as a singleton and static construction is fine. +class CoreHttp : public LLCore::HttpHandler +{ +public: + CoreHttp(); + ~CoreHttp(); + + // Initialize the LLCore::HTTP library creating service classes + // and starting the servicing thread. Caller is expected to do + // other initializations (SSL mutex, thread hash function) appropriate + // for the application. + void init(); + + // Request that the servicing thread stop servicing requests, + // release resource references and stop. + void requestStop(); + + // Terminate LLCore::HTTP library services. Caller is expected + // to have made a best-effort to shutdown the servicing thread + // by issuing a requestThreadStop() and waiting for completion + // notification that the stop has completed. + void cleanup(); + + // Notification when the stop request is complete. + virtual void onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * response); + +private: + static const F64 MAX_THREAD_WAIT_TIME; + +private: + LLCore::HttpRequest * mRequest; + LLCore::HttpHandle mStopHandle; + F64 mStopRequested; + bool mStopped; +}; + +CoreHttp coreHttpLib; + +} // end anonymous namespace + //-- LLDeferredTaskList ------------------------------------------------------ /** @@ -720,6 +772,9 @@ bool LLAppViewer::init() LLViewerStatsRecorder::initClass(); #endif + // Initialize the non-LLCurl libcurl library + coreHttpLib.init(); + // *NOTE:Mani - LLCurl::initClass is not thread safe. // Called before threads are created. LLCurl::initClass(gSavedSettings.getF32("CurlRequestTimeOut"), @@ -1807,6 +1862,7 @@ bool LLAppViewer::cleanup() // Delete workers first // shotdown all worker threads before deleting them in case of co-dependencies + coreHttpLib.requestStop(); sTextureFetch->shutdown(); sTextureCache->shutdown(); sImageDecodeThread->shutdown(); @@ -1890,6 +1946,9 @@ bool LLAppViewer::cleanup() // *NOTE:Mani - The following call is not thread safe. LLCurl::cleanupClass(); + // Non-LLCurl libcurl library + coreHttpLib.cleanup(); + // If we're exiting to launch an URL, do that here so the screen // is at the right resolution before we launch IE. if (!gLaunchFileOnQuit.empty()) @@ -5267,3 +5326,103 @@ void LLAppViewer::metricsSend(bool enable_reporting) gViewerAssetStatsMain->reset(); } +namespace +{ + +const F64 CoreHttp::MAX_THREAD_WAIT_TIME(10.0); + +CoreHttp::CoreHttp() + : mRequest(NULL), + mStopHandle(LLCORE_HTTP_HANDLE_INVALID), + mStopRequested(0.0), + mStopped(false) +{} + + +CoreHttp::~CoreHttp() +{ + delete mRequest; + mRequest = NULL; +} + + +void CoreHttp::init() +{ + LLCore::HttpStatus status = LLCore::HttpRequest::createService(); + if (! status) + { + LL_ERRS("Init") << "Failed to initialize HTTP services. Reason: " + << status.toString() + << LL_ENDL; + } + + status = LLCore::HttpRequest::startThread(); + if (! status) + { + LL_ERRS("Init") << "Failed to start HTTP servicing thread. Reason: " + << status.toString() + << LL_ENDL; + } + + mRequest = new LLCore::HttpRequest; +} + + +void CoreHttp::requestStop() +{ + llassert_always(mRequest); + + mStopHandle = mRequest->requestStopThread(this); + if (LLCORE_HTTP_HANDLE_INVALID != mStopHandle) + { + mStopRequested = LLTimer::getTotalSeconds(); + } +} + + +void CoreHttp::cleanup() +{ + if (LLCORE_HTTP_HANDLE_INVALID == mStopHandle) + { + // Should have been started already... + requestStop(); + } + + if (LLCORE_HTTP_HANDLE_INVALID == mStopHandle) + { + LL_WARNS("Cleanup") << "Attempting to cleanup HTTP services without thread shutdown" + << LL_ENDL; + } + else + { + while (! mStopped && LLTimer::getTotalSeconds() < (mStopRequested + MAX_THREAD_WAIT_TIME)) + { + mRequest->update(200); + ms_sleep(50); + } + if (! mStopped) + { + LL_WARNS("Cleanup") << "Attempting to cleanup HTTP services with thread shutdown incomplete" + << LL_ENDL; + } + } + + delete mRequest; + mRequest = NULL; + + LLCore::HttpStatus status = LLCore::HttpRequest::destroyService(); + if (! status) + { + LL_WARNS("Cleanup") << "Failed to shutdown HTTP services, continuing. Reason: " + << status.toString() + << LL_ENDL; + } +} + + +void CoreHttp::onCompleted(LLCore::HttpHandle, LLCore::HttpResponse *) +{ + mStopped = true; +} + +} // end anonymous namespace -- cgit v1.2.3 From 05af16a23abe37210e0b880aa27387d8994419dd Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Wed, 6 Jun 2012 13:52:38 -0400 Subject: Policy + caching fixes + https support + POST working Implemented first global policy definitions to support SSL CA certificate configuration to support https: operations. Fixed HTTP 206 status handling to match what is currently being done by grid services and to lay a foundation for fixes that will be a response to ER-1824. More libcurl CURLOPT options set on easy handles to do peer verification in the traditional way. HTTP POST working and now reporting asset metrics back to grid for the viewer's asset system. This uses LLSD so that is also showing as compatible with the new library. --- indra/newview/llappviewer.cpp | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) (limited to 'indra/newview/llappviewer.cpp') diff --git a/indra/newview/llappviewer.cpp b/indra/newview/llappviewer.cpp index 8e6deb9cce..7a44415fba 100644 --- a/indra/newview/llappviewer.cpp +++ b/indra/newview/llappviewer.cpp @@ -5356,6 +5356,17 @@ void CoreHttp::init() << LL_ENDL; } + mRequest = new LLCore::HttpRequest; + + status = mRequest->setPolicyGlobalOption(LLCore::HttpRequest::GP_CA_FILE, + gDirUtilp->getCAFile()); + if (! status) + { + LL_ERRS("Init") << "Failed to set CA File for HTTP services. Reason: " + << status.toString() + << LL_ENDL; + } + status = LLCore::HttpRequest::startThread(); if (! status) { @@ -5364,7 +5375,6 @@ void CoreHttp::init() << LL_ENDL; } - mRequest = new LLCore::HttpRequest; } -- cgit v1.2.3 From 7adeb3923728ca84a309a6af141c148ce38066fc Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Tue, 12 Jun 2012 17:42:33 -0400 Subject: HTTP Proxy, PUT & POST, unit tests and refactoring. Implemented/modified PUT & POST to not used chunked encoding for the request. Made the unit test much happier and probably a better thing for the pipeline. Have a cheesy static & dynamic proxy capability using both local options and a way to wire into LLProxy in llmessages. Not a clean thing but it will get the proxy path working with both socks5 & http proxies. Refactoring to get rid of unneeded library handler and unified an HttpStatus return for all requests. Big batch of code removed as a result of that and more is possible as well as some syscall avoidance with a bit more work. Boosted the unit tests for simple PUT & POST test which revealed the test harness does *not* like chunked encoding so we'll avoid it for now (and don't really need it in any of our schemes). --- indra/newview/llappviewer.cpp | 13 +++++++++++++ 1 file changed, 13 insertions(+) (limited to 'indra/newview/llappviewer.cpp') diff --git a/indra/newview/llappviewer.cpp b/indra/newview/llappviewer.cpp index 7a44415fba..e2c13e77e3 100644 --- a/indra/newview/llappviewer.cpp +++ b/indra/newview/llappviewer.cpp @@ -5358,6 +5358,7 @@ void CoreHttp::init() mRequest = new LLCore::HttpRequest; + // Point to our certs or SSH/https: will fail on connect status = mRequest->setPolicyGlobalOption(LLCore::HttpRequest::GP_CA_FILE, gDirUtilp->getCAFile()); if (! status) @@ -5366,6 +5367,18 @@ void CoreHttp::init() << status.toString() << LL_ENDL; } + + // Establish HTTP Proxy. "LLProxy" is a special string which directs + // the code to use LLProxy::applyProxySettings() to establish any + // HTTP or SOCKS proxy for http operations. + status = mRequest->setPolicyGlobalOption(LLCore::HttpRequest::GP_HTTP_PROXY, + std::string("LLProxy")); + if (! status) + { + LL_ERRS("Init") << "Failed to set HTTP proxy for HTTP services. Reason: " + << status.toString() + << LL_ENDL; + } status = LLCore::HttpRequest::startThread(); if (! status) -- cgit v1.2.3 From b08125a5874a89ce5210f8fb2c961ae17fb80fde Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Thu, 14 Jun 2012 16:31:48 -0400 Subject: LLMutex recursive lock, global & per-request tracing, simple GET request, LLProxy support, HttpOptions starting to work, HTTP resource waiting fixed. Non-LLThread-based threads need to do some registration or LLMutex locks taken out in these threads will not work as expected (SH-3154). We'll get a better solution later, this fixes some things for now. Tracing of operations now supported. Global and per-request (via HttpOptions) tracing levels of [0..3]. The 2 and 3 levels use libcurl's VERBOSE mode combined with CURLOPT_DEBUGFUNCTION to stream high levels of detail into the log. *Very* laggy but useful. Simple GET request supported (no Range: header). Really just a degenrate case of a ranged get but supplied an API anyway. Global option to use the LLProxy interface to setup CURL handles for either socks5 or http proxy usage. This isn't really the most encapsulated way to do this but a better solution will have to come later. The wantHeaders and tracing options are now supported in HttpOptions giving per-request controls. Big refactoring of the HTTP resource waiter in lltexturefetch. What I was doing before wasn't correct. Instead, I'm implementing the resource wait after the Semaphore model (though not using system semaphores). So instead of having a sequence like: SEND_HTTP_REQ -> WAIT_HTTP_RESOURCE -> SEND_HTTP_REQ, we now do WAIT_HTTP_RESOURCE -> WAIT_HTTP_RESOURCE2 (actual wait) -> SEND_HTTP_REQ. Works well but the prioritized filling of the corehttp library needs some performance work later. --- indra/newview/llappviewer.cpp | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) (limited to 'indra/newview/llappviewer.cpp') diff --git a/indra/newview/llappviewer.cpp b/indra/newview/llappviewer.cpp index e2c13e77e3..430dd89c3e 100644 --- a/indra/newview/llappviewer.cpp +++ b/indra/newview/llappviewer.cpp @@ -5356,11 +5356,9 @@ void CoreHttp::init() << LL_ENDL; } - mRequest = new LLCore::HttpRequest; - // Point to our certs or SSH/https: will fail on connect - status = mRequest->setPolicyGlobalOption(LLCore::HttpRequest::GP_CA_FILE, - gDirUtilp->getCAFile()); + status = LLCore::HttpRequest::setPolicyGlobalOption(LLCore::HttpRequest::GP_CA_FILE, + gDirUtilp->getCAFile()); if (! status) { LL_ERRS("Init") << "Failed to set CA File for HTTP services. Reason: " @@ -5371,15 +5369,22 @@ void CoreHttp::init() // Establish HTTP Proxy. "LLProxy" is a special string which directs // the code to use LLProxy::applyProxySettings() to establish any // HTTP or SOCKS proxy for http operations. - status = mRequest->setPolicyGlobalOption(LLCore::HttpRequest::GP_HTTP_PROXY, - std::string("LLProxy")); + status = LLCore::HttpRequest::setPolicyGlobalOption(LLCore::HttpRequest::GP_LLPROXY, 1); if (! status) { LL_ERRS("Init") << "Failed to set HTTP proxy for HTTP services. Reason: " << status.toString() << LL_ENDL; } - + + // Tracing levels for library & libcurl (note that 2 & 3 are beyond spammy): + // 0 - None + // 1 - Basic start, stop simple transitions + // 2 - libcurl CURLOPT_VERBOSE mode with brief lines + // 3 - with partial data content + status = LLCore::HttpRequest::setPolicyGlobalOption(LLCore::HttpRequest::GP_TRACE, 0); + + // Kick the thread status = LLCore::HttpRequest::startThread(); if (! status) { @@ -5388,6 +5393,7 @@ void CoreHttp::init() << LL_ENDL; } + mRequest = new LLCore::HttpRequest; } -- cgit v1.2.3 From bc72acbfd2410e01946375bcfa29cf37a7c01c17 Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Tue, 10 Jul 2012 18:50:21 -0400 Subject: SH-3244 Syscall avoidance in HttpRequest::update() method Well, achieved that by doing work in bulk when needed. But turned into some additional things. Change timebase from mS to uS as, well, things are headed that way. Implement an HttpReplyQueue::fetchAll method (advertised one, hadn't implemented it). --- indra/newview/llappviewer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'indra/newview/llappviewer.cpp') diff --git a/indra/newview/llappviewer.cpp b/indra/newview/llappviewer.cpp index 8243d4f2f3..0549a972e1 100644 --- a/indra/newview/llappviewer.cpp +++ b/indra/newview/llappviewer.cpp @@ -5398,7 +5398,7 @@ void CoreHttp::cleanup() { while (! mStopped && LLTimer::getTotalSeconds() < (mStopRequested + MAX_THREAD_WAIT_TIME)) { - mRequest->update(200); + mRequest->update(200000); ms_sleep(50); } if (! mStopped) -- cgit v1.2.3 From 3f032e33f2b2f929b229cf4d358b9c9d297856b8 Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Thu, 19 Jul 2012 13:41:18 -0400 Subject: SH-3280 Better init/shutdown functionality for llcorehttp by llappviewer Isolate llcorehttp initialization into a utility class (LLAppCoreHttp) that provides glue between app and library (sets up policies, handles notifications). Introduce 'TextureFetchConcurrency' debug setting to provide some field control when absolutely necessary. --- indra/newview/llappviewer.cpp | 189 ++---------------------------------------- 1 file changed, 5 insertions(+), 184 deletions(-) (limited to 'indra/newview/llappviewer.cpp') diff --git a/indra/newview/llappviewer.cpp b/indra/newview/llappviewer.cpp index 0549a972e1..f8ee1a477f 100644 --- a/indra/newview/llappviewer.cpp +++ b/indra/newview/llappviewer.cpp @@ -215,10 +215,6 @@ #include "llmachineid.h" #include "llmainlooprepeater.h" -// LLCore::HTTP -#include "httpcommon.h" -#include "httprequest.h" -#include "httphandler.h" // *FIX: These extern globals should be cleaned up. // The globals either represent state/config/resource-storage of either @@ -334,53 +330,6 @@ static std::string gLaunchFileOnQuit; // Used on Win32 for other apps to identify our window (eg, win_setup) const char* const VIEWER_WINDOW_CLASSNAME = "Second Life"; -namespace -{ - -// This class manages the lifecyle of the core http library. -// Slightly different style than traditional code but reflects -// the use of handler classes and light-weight interface -// object instances of the new libraries. To be used -// as a singleton and static construction is fine. -class CoreHttp : public LLCore::HttpHandler -{ -public: - CoreHttp(); - ~CoreHttp(); - - // Initialize the LLCore::HTTP library creating service classes - // and starting the servicing thread. Caller is expected to do - // other initializations (SSL mutex, thread hash function) appropriate - // for the application. - void init(); - - // Request that the servicing thread stop servicing requests, - // release resource references and stop. - void requestStop(); - - // Terminate LLCore::HTTP library services. Caller is expected - // to have made a best-effort to shutdown the servicing thread - // by issuing a requestThreadStop() and waiting for completion - // notification that the stop has completed. - void cleanup(); - - // Notification when the stop request is complete. - virtual void onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * response); - -private: - static const F64 MAX_THREAD_WAIT_TIME; - -private: - LLCore::HttpRequest * mRequest; - LLCore::HttpHandle mStopHandle; - F64 mStopRequested; - bool mStopped; -}; - -CoreHttp coreHttpLib; - -} // end anonymous namespace - //-- LLDeferredTaskList ------------------------------------------------------ /** @@ -775,8 +724,9 @@ bool LLAppViewer::init() LLViewerStatsRecorder::initClass(); #endif - // Initialize the non-LLCurl libcurl library - coreHttpLib.init(); + // Initialize the non-LLCurl libcurl library. Should be called + // before consumers (LLTextureFetch). + mAppCoreHttp.init(); // *NOTE:Mani - LLCurl::initClass is not thread safe. // Called before threads are created. @@ -1915,7 +1865,7 @@ bool LLAppViewer::cleanup() // Delete workers first // shotdown all worker threads before deleting them in case of co-dependencies - coreHttpLib.requestStop(); + mAppCoreHttp.requestStop(); sTextureFetch->shutdown(); sTextureCache->shutdown(); sImageDecodeThread->shutdown(); @@ -2000,7 +1950,7 @@ bool LLAppViewer::cleanup() LLCurl::cleanupClass(); // Non-LLCurl libcurl library - coreHttpLib.cleanup(); + mAppCoreHttp.cleanup(); // If we're exiting to launch an URL, do that here so the screen // is at the right resolution before we launch IE. @@ -5298,132 +5248,3 @@ void LLAppViewer::metricsSend(bool enable_reporting) gViewerAssetStatsMain->reset(); } -namespace -{ - -const F64 CoreHttp::MAX_THREAD_WAIT_TIME(10.0); - -CoreHttp::CoreHttp() - : mRequest(NULL), - mStopHandle(LLCORE_HTTP_HANDLE_INVALID), - mStopRequested(0.0), - mStopped(false) -{} - - -CoreHttp::~CoreHttp() -{ - delete mRequest; - mRequest = NULL; -} - - -void CoreHttp::init() -{ - LLCore::HttpStatus status = LLCore::HttpRequest::createService(); - if (! status) - { - LL_ERRS("Init") << "Failed to initialize HTTP services. Reason: " - << status.toString() - << LL_ENDL; - } - - // Point to our certs or SSH/https: will fail on connect - status = LLCore::HttpRequest::setPolicyGlobalOption(LLCore::HttpRequest::GP_CA_FILE, - gDirUtilp->getCAFile()); - if (! status) - { - LL_ERRS("Init") << "Failed to set CA File for HTTP services. Reason: " - << status.toString() - << LL_ENDL; - } - - // Establish HTTP Proxy. "LLProxy" is a special string which directs - // the code to use LLProxy::applyProxySettings() to establish any - // HTTP or SOCKS proxy for http operations. - status = LLCore::HttpRequest::setPolicyGlobalOption(LLCore::HttpRequest::GP_LLPROXY, 1); - if (! status) - { - LL_ERRS("Init") << "Failed to set HTTP proxy for HTTP services. Reason: " - << status.toString() - << LL_ENDL; - } - - // Tracing levels for library & libcurl (note that 2 & 3 are beyond spammy): - // 0 - None - // 1 - Basic start, stop simple transitions - // 2 - libcurl CURLOPT_VERBOSE mode with brief lines - // 3 - with partial data content - status = LLCore::HttpRequest::setPolicyGlobalOption(LLCore::HttpRequest::GP_TRACE, 0); - - // Kick the thread - status = LLCore::HttpRequest::startThread(); - if (! status) - { - LL_ERRS("Init") << "Failed to start HTTP servicing thread. Reason: " - << status.toString() - << LL_ENDL; - } - - mRequest = new LLCore::HttpRequest; -} - - -void CoreHttp::requestStop() -{ - llassert_always(mRequest); - - mStopHandle = mRequest->requestStopThread(this); - if (LLCORE_HTTP_HANDLE_INVALID != mStopHandle) - { - mStopRequested = LLTimer::getTotalSeconds(); - } -} - - -void CoreHttp::cleanup() -{ - if (LLCORE_HTTP_HANDLE_INVALID == mStopHandle) - { - // Should have been started already... - requestStop(); - } - - if (LLCORE_HTTP_HANDLE_INVALID == mStopHandle) - { - LL_WARNS("Cleanup") << "Attempting to cleanup HTTP services without thread shutdown" - << LL_ENDL; - } - else - { - while (! mStopped && LLTimer::getTotalSeconds() < (mStopRequested + MAX_THREAD_WAIT_TIME)) - { - mRequest->update(200000); - ms_sleep(50); - } - if (! mStopped) - { - LL_WARNS("Cleanup") << "Attempting to cleanup HTTP services with thread shutdown incomplete" - << LL_ENDL; - } - } - - delete mRequest; - mRequest = NULL; - - LLCore::HttpStatus status = LLCore::HttpRequest::destroyService(); - if (! status) - { - LL_WARNS("Cleanup") << "Failed to shutdown HTTP services, continuing. Reason: " - << status.toString() - << LL_ENDL; - } -} - - -void CoreHttp::onCompleted(LLCore::HttpHandle, LLCore::HttpResponse *) -{ - mStopped = true; -} - -} // end anonymous namespace -- cgit v1.2.3