From 626752beab7c12a355ab707d70aba6f4fe096c10 Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Wed, 19 Jun 2013 13:55:54 -0400 Subject: SH-4252 Add second policy class for large mesh asset downloads Added second mesh class as well as an asset upload class. Refactored initialization to use less code and more data to cleanly get http started. Modified mesh to use the new http class for large requests (>2MB for now). Added additional timeout setting to llcorehttp to distinguish connection timeout from transport timeout and are now using transport timeout values for large asset downloads that may need more time. --- indra/llcorehttp/_httpinternal.h | 3 +- indra/llcorehttp/_httpoprequest.cpp | 15 ++- indra/llcorehttp/_httpoprequest.h | 3 +- indra/llcorehttp/_httppolicy.cpp | 6 ++ indra/llcorehttp/httpoptions.cpp | 9 +- indra/llcorehttp/httpoptions.h | 9 +- indra/llcorehttp/httpresponse.cpp | 6 +- indra/llcorehttp/httpresponse.h | 17 +++- indra/newview/llappcorehttp.cpp | 156 ++++++++++++++++------------ indra/newview/llappcorehttp.h | 36 +++---- indra/newview/llmeshrepository.cpp | 197 +++++++++++++++++++++--------------- indra/newview/llmeshrepository.h | 21 +++- indra/newview/lltexturefetch.cpp | 4 +- 13 files changed, 301 insertions(+), 181 deletions(-) diff --git a/indra/llcorehttp/_httpinternal.h b/indra/llcorehttp/_httpinternal.h index 30b0905c12..d60996756f 100755 --- a/indra/llcorehttp/_httpinternal.h +++ b/indra/llcorehttp/_httpinternal.h @@ -98,7 +98,7 @@ namespace LLCore // Maxium number of policy classes that can be defined. // *TODO: Currently limited to the default class + 1, extend. -const int HTTP_POLICY_CLASS_LIMIT = 2; +const int HTTP_POLICY_CLASS_LIMIT = 4; // Debug/informational tracing. Used both // as a global option and in per-request traces. @@ -129,6 +129,7 @@ const int HTTP_REDIRECTS_DEFAULT = 10; // Retries and time-on-queue are not included and aren't // accounted for. const long HTTP_REQUEST_TIMEOUT_DEFAULT = 30L; +const long HTTP_REQUEST_XFER_TIMEOUT_DEFAULT = 0L; const long HTTP_REQUEST_TIMEOUT_MIN = 0L; const long HTTP_REQUEST_TIMEOUT_MAX = 3600L; diff --git a/indra/llcorehttp/_httpoprequest.cpp b/indra/llcorehttp/_httpoprequest.cpp index 51a8eaf998..d403b2d249 100755 --- a/indra/llcorehttp/_httpoprequest.cpp +++ b/indra/llcorehttp/_httpoprequest.cpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -110,6 +110,7 @@ HttpOpRequest::HttpOpRequest() mReplyFullLength(0), mReplyHeaders(NULL), mPolicyRetries(0), + mPolicy503Retries(0), mPolicyRetryAt(HttpTime(0)), mPolicyRetryLimit(HTTP_RETRY_COUNT_DEFAULT) { @@ -224,6 +225,7 @@ void HttpOpRequest::visitNotifier(HttpRequest * request) response->setRange(mReplyOffset, mReplyLength, mReplyFullLength); } response->setContentType(mReplyConType); + response->setRetries(mPolicyRetries, mPolicy503Retries); mUserHandler->onCompleted(static_cast(this), response); @@ -524,12 +526,19 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service) // Request options long timeout(HTTP_REQUEST_TIMEOUT_DEFAULT); + long xfer_timeout(HTTP_REQUEST_XFER_TIMEOUT_DEFAULT); if (mReqOptions) - { + { timeout = mReqOptions->getTimeout(); timeout = llclamp(timeout, HTTP_REQUEST_TIMEOUT_MIN, HTTP_REQUEST_TIMEOUT_MAX); + xfer_timeout = mReqOptions->getTransferTimeout(); + xfer_timeout = llclamp(xfer_timeout, HTTP_REQUEST_TIMEOUT_MIN, HTTP_REQUEST_TIMEOUT_MAX); + } + if (xfer_timeout == 0L) + { + xfer_timeout = timeout; } - curl_easy_setopt(mCurlHandle, CURLOPT_TIMEOUT, timeout); + curl_easy_setopt(mCurlHandle, CURLOPT_TIMEOUT, xfer_timeout); curl_easy_setopt(mCurlHandle, CURLOPT_CONNECTTIMEOUT, timeout); // Request headers diff --git a/indra/llcorehttp/_httpoprequest.h b/indra/llcorehttp/_httpoprequest.h index 7b65d17783..831e5bebf7 100755 --- a/indra/llcorehttp/_httpoprequest.h +++ b/indra/llcorehttp/_httpoprequest.h @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -187,6 +187,7 @@ public: // Policy data int mPolicyRetries; + int mPolicy503Retries; HttpTime mPolicyRetryAt; int mPolicyRetryLimit; }; // end class HttpOpRequest diff --git a/indra/llcorehttp/_httppolicy.cpp b/indra/llcorehttp/_httppolicy.cpp index 76c1e22431..54c9c6bb1b 100755 --- a/indra/llcorehttp/_httppolicy.cpp +++ b/indra/llcorehttp/_httppolicy.cpp @@ -140,6 +140,7 @@ void HttpPolicy::addOp(HttpOpRequest * op) const int policy_class(op->mReqPolicy); op->mPolicyRetries = 0; + op->mPolicy503Retries = 0; mState[policy_class].mReadyQueue.push(op); } @@ -155,6 +156,7 @@ void HttpPolicy::retryOp(HttpOpRequest * op) 5000000 // ... to every 5.0 S. }; static const int delta_max(int(LL_ARRAY_SIZE(retry_deltas)) - 1); + static const HttpStatus error_503(503); const HttpTime now(totalTime()); const int policy_class(op->mReqPolicy); @@ -162,6 +164,10 @@ void HttpPolicy::retryOp(HttpOpRequest * op) const HttpTime delta(retry_deltas[llclamp(op->mPolicyRetries, 0, delta_max)]); op->mPolicyRetryAt = now + delta; ++op->mPolicyRetries; + if (error_503 == op->mStatus) + { + ++op->mPolicy503Retries; + } LL_WARNS("CoreHttp") << "HTTP request " << static_cast(op) << " retry " << op->mPolicyRetries << " scheduled for +" << (delta / HttpTime(1000)) diff --git a/indra/llcorehttp/httpoptions.cpp b/indra/llcorehttp/httpoptions.cpp index 1699d19f8d..4dcd862ca4 100755 --- a/indra/llcorehttp/httpoptions.cpp +++ b/indra/llcorehttp/httpoptions.cpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -38,6 +38,7 @@ HttpOptions::HttpOptions() mWantHeaders(false), mTracing(HTTP_TRACE_OFF), mTimeout(HTTP_REQUEST_TIMEOUT_DEFAULT), + mTransferTimeout(HTTP_REQUEST_XFER_TIMEOUT_DEFAULT), mRetries(HTTP_RETRY_COUNT_DEFAULT) {} @@ -64,6 +65,12 @@ void HttpOptions::setTimeout(unsigned int timeout) } +void HttpOptions::setTransferTimeout(unsigned int timeout) +{ + mTransferTimeout = timeout; +} + + void HttpOptions::setRetries(unsigned int retries) { mRetries = retries; diff --git a/indra/llcorehttp/httpoptions.h b/indra/llcorehttp/httpoptions.h index 97e46a8cd3..623d71d3e6 100755 --- a/indra/llcorehttp/httpoptions.h +++ b/indra/llcorehttp/httpoptions.h @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -86,6 +86,12 @@ public: return mTimeout; } + void setTransferTimeout(unsigned int timeout); + unsigned int getTransferTimeout() const + { + return mTransferTimeout; + } + void setRetries(unsigned int retries); unsigned int getRetries() const { @@ -96,6 +102,7 @@ protected: bool mWantHeaders; int mTracing; unsigned int mTimeout; + unsigned int mTransferTimeout; unsigned int mRetries; }; // end class HttpOptions diff --git a/indra/llcorehttp/httpresponse.cpp b/indra/llcorehttp/httpresponse.cpp index a552e48a1b..c974395b0a 100755 --- a/indra/llcorehttp/httpresponse.cpp +++ b/indra/llcorehttp/httpresponse.cpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -39,7 +39,9 @@ HttpResponse::HttpResponse() mReplyLength(0U), mReplyFullLength(0U), mBufferArray(NULL), - mHeaders(NULL) + mHeaders(NULL), + mRetries(0U), + m503Retries(0U) {} diff --git a/indra/llcorehttp/httpresponse.h b/indra/llcorehttp/httpresponse.h index 4a481db6ac..a7f296e03f 100755 --- a/indra/llcorehttp/httpresponse.h +++ b/indra/llcorehttp/httpresponse.h @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -144,6 +144,19 @@ public: mContentType = con_type; } + /// Get and set retry attempt information on the request. + void getRetries(unsigned int * retries, unsigned int * retries_503) const + { + *retries = mRetries; + *retries_503 = m503Retries; + } + + void setRetries(unsigned int retries, unsigned int retries_503) + { + mRetries = retries; + m503Retries = retries_503; + } + protected: // Response data here HttpStatus mStatus; @@ -153,6 +166,8 @@ protected: BufferArray * mBufferArray; HttpHeaders * mHeaders; std::string mContentType; + unsigned int mRetries; + unsigned int m503Retries; }; diff --git a/indra/newview/llappcorehttp.cpp b/indra/newview/llappcorehttp.cpp index 142344e277..b601b31d21 100755 --- a/indra/newview/llappcorehttp.cpp +++ b/indra/newview/llappcorehttp.cpp @@ -37,11 +37,13 @@ LLAppCoreHttp::LLAppCoreHttp() : mRequest(NULL), mStopHandle(LLCORE_HTTP_HANDLE_INVALID), mStopRequested(0.0), - mStopped(false), - mPolicyDefault(-1), - mPolicyTexture(-1), - mPolicyMesh(-1) -{} + mStopped(false) +{ + for (int i(0); i < LL_ARRAY_SIZE(mPolicies); ++i) + { + mPolicies[i] = LLCore::HttpRequest::DEFAULT_POLICY_ID; + } +} LLAppCoreHttp::~LLAppCoreHttp() @@ -53,11 +55,43 @@ LLAppCoreHttp::~LLAppCoreHttp() void LLAppCoreHttp::init() { + static const struct + { + EAppPolicy mPolicy; + U32 mDefault; + U32 mMin; + U32 mMax; + U32 mDivisor; + std::string mKey; + const char * mUsage; + } init_data[] = // Default and dynamic values for classes + { + { + AP_TEXTURE, 8, 1, 12, 1, + "TextureFetchConcurrency", + "texture fetch" + }, + { + AP_MESH, 8, 1, 32, 4, + "MeshMaxConcurrentRequests", + "mesh fetch" + }, + { + AP_LARGE_MESH, 2, 1, 8, 1, + "", + "large mesh fetch" + }, + { + AP_UPLOADS, 2, 1, 8, 1, + "", + "asset upload" + } + }; + LLCore::HttpStatus status = LLCore::HttpRequest::createService(); if (! status) { - LL_ERRS("Init") << "Failed to initialize HTTP services. Reason: " - << status.toString() + LL_ERRS("Init") << "Failed to initialize HTTP services. Reason: " << status.toString() << LL_ENDL; } @@ -66,8 +100,7 @@ void LLAppCoreHttp::init() gDirUtilp->getCAFile()); if (! status) { - LL_ERRS("Init") << "Failed to set CA File for HTTP services. Reason: " - << status.toString() + LL_ERRS("Init") << "Failed to set CA File for HTTP services. Reason: " << status.toString() << LL_ENDL; } @@ -77,8 +110,7 @@ void LLAppCoreHttp::init() status = LLCore::HttpRequest::setPolicyGlobalOption(LLCore::HttpRequest::GP_LLPROXY, 1); if (! status) { - LL_ERRS("Init") << "Failed to set HTTP proxy for HTTP services. Reason: " - << status.toString() + LL_ERRS("Init") << "Failed to set HTTP proxy for HTTP services. Reason: " << status.toString() << LL_ENDL; } @@ -96,80 +128,72 @@ void LLAppCoreHttp::init() } // Setup default policy and constrain if directed to - mPolicyDefault = LLCore::HttpRequest::DEFAULT_POLICY_ID; + mPolicies[AP_DEFAULT] = LLCore::HttpRequest::DEFAULT_POLICY_ID; - // Texture policy will use default for now. - mPolicyTexture = mPolicyDefault; - static const std::string texture_concur("TextureFetchConcurrency"); - if (gSavedSettings.controlExists(texture_concur)) + // Setup additional policies based on table and some special rules + // *TODO: Make these configurations dynamic later + for (int i(0); i < LL_ARRAY_SIZE(init_data); ++i) { - U32 concur(llmin(gSavedSettings.getU32(texture_concur), U32(12))); + const EAppPolicy policy(init_data[i].mPolicy); - if (concur > 0) + // Create a policy class but use default for texture for now. + // This also has the side-effect of initializing the default + // class to desired values. + if (AP_TEXTURE == policy) { - LLCore::HttpStatus status; - status = LLCore::HttpRequest::setPolicyClassOption(mPolicyTexture, - LLCore::HttpRequest::CP_CONNECTION_LIMIT, - concur); - if (! status) - { - LL_WARNS("Init") << "Unable to set texture fetch concurrency. Reason: " - << status.toString() - << LL_ENDL; - } - else + mPolicies[policy] = mPolicies[AP_DEFAULT]; + } + else + { + mPolicies[policy] = LLCore::HttpRequest::createPolicyClass(); + if (! mPolicies[policy]) { - LL_INFOS("Init") << "Application settings overriding default texture fetch concurrency. New value: " - << concur + // Use default policy (but don't accidentally modify default) + LL_WARNS("Init") << "Failed to create HTTP policy class for " << init_data[i].mUsage + << ". Using default policy." << LL_ENDL; + mPolicies[policy] = mPolicies[AP_DEFAULT]; + continue; } } - } - // Create the mesh class - mPolicyMesh = LLCore::HttpRequest::createPolicyClass(); - if (! mPolicyMesh) - { - LL_WARNS("Init") << "Failed to create HTTP policy class for Mesh. Using default policy." - << LL_ENDL; - mPolicyMesh = mPolicyDefault; - } - else - { - static const std::string mesh_concur("MeshMaxConcurrentRequests"); - if (gSavedSettings.controlExists(mesh_concur)) + // Get target connection concurrency value + U32 setting(init_data[i].mDefault); + if (! init_data[i].mKey.empty() && gSavedSettings.controlExists(init_data[i].mKey)) { - U32 setting(llmin(gSavedSettings.getU32(mesh_concur), 256U) / 4U); - setting = llmax(setting, 2U); - - if (setting > 0) + U32 new_setting(gSavedSettings.getU32(init_data[i].mKey)); + if (new_setting) { - LLCore::HttpStatus status; - status = LLCore::HttpRequest::setPolicyClassOption(mPolicyMesh, - LLCore::HttpRequest::CP_CONNECTION_LIMIT, - setting); - if (! status) - { - LL_WARNS("Init") << "Unable to set mesh fetch concurrency. Reason: " - << status.toString() - << LL_ENDL; - } - else - { - LL_INFOS("Init") << "Application settings overriding default mesh fetch concurrency. New value: " - << setting - << LL_ENDL; - } + // Treat zero settings as an ask for default + setting = new_setting / init_data[i].mDivisor; + setting = llclamp(setting, init_data[i].mMin, init_data[i].mMax); } } + + // Set it and report + LLCore::HttpStatus status; + status = LLCore::HttpRequest::setPolicyClassOption(mPolicies[policy], + LLCore::HttpRequest::CP_CONNECTION_LIMIT, + setting); + if (! status) + { + LL_WARNS("Init") << "Unable to set " << init_data[i].mUsage + << " concurrency. Reason: " << status.toString() + << LL_ENDL; + } + else if (setting != init_data[i].mDefault) + { + LL_INFOS("Init") << "Application settings overriding default " << init_data[i].mUsage + << " concurrency. New value: " << setting + << LL_ENDL; + } } // Kick the thread status = LLCore::HttpRequest::startThread(); if (! status) { - LL_ERRS("Init") << "Failed to start HTTP servicing thread. Reason: " - << status.toString() + LL_ERRS("Init") << "Failed to start HTTP servicing thread. Reason: " << status.toString() << LL_ENDL; } diff --git a/indra/newview/llappcorehttp.h b/indra/newview/llappcorehttp.h index d90af9e5ca..532e1f5cb0 100755 --- a/indra/newview/llappcorehttp.h +++ b/indra/newview/llappcorehttp.h @@ -40,6 +40,19 @@ // as a singleton and static construction is fine. class LLAppCoreHttp : public LLCore::HttpHandler { +public: + typedef LLCore::HttpRequest::policy_t policy_t; + + enum EAppPolicy + { + AP_DEFAULT, + AP_TEXTURE, + AP_MESH, + AP_LARGE_MESH, + AP_UPLOADS, + AP_COUNT // Must be last + }; + public: LLAppCoreHttp(); ~LLAppCoreHttp(); @@ -65,22 +78,11 @@ public: // Notification when the stop request is complete. virtual void onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * response); - // Retrieve the policy class for default operations. - int getPolicyDefault() const - { - return mPolicyDefault; - } - - // Get the texture fetch policy class. - int getPolicyTexture() const - { - return mPolicyTexture; - } - - // Get the mesh fetch policy class. - int getPolicyMesh() const + // Retrieve a policy class identifier for desired + // application function. + policy_t getPolicy(EAppPolicy policy) const { - return mPolicyMesh; + return mPolicies[policy]; } private: @@ -91,9 +93,7 @@ private: LLCore::HttpHandle mStopHandle; F64 mStopRequested; bool mStopped; - int mPolicyDefault; - int mPolicyTexture; - int mPolicyMesh; + policy_t mPolicies[AP_COUNT]; }; diff --git a/indra/newview/llmeshrepository.cpp b/indra/newview/llmeshrepository.cpp index 702e940983..f0ec97a34d 100755 --- a/indra/newview/llmeshrepository.cpp +++ b/indra/newview/llmeshrepository.cpp @@ -80,6 +80,10 @@ LLMeshRepository gMeshRepo; const S32 MESH_HEADER_SIZE = 4096; const U32 MAX_MESH_REQUESTS_PER_SECOND = 100; +const S32 REQUEST_HIGH_WATER_MIN = 32; +const S32 REQUEST_LOW_WATER_MIN = 16; +const U32 LARGE_MESH_FETCH_THRESHOLD = 1U << 21; // Size at which requests goes to narrow/slow queue +const long LARGE_MESH_XFER_TIMEOUT = 240L; // Seconds to complete xfer // Maximum mesh version to support. Three least significant digits are reserved for the minor version, // with major version changes indicating a format change that is not backwards compatible and should not @@ -210,6 +214,8 @@ void get_vertex_buffer_from_mesh(LLCDMeshData& mesh, LLModel::PhysicsMesh& res, S32 LLMeshRepoThread::sActiveHeaderRequests = 0; S32 LLMeshRepoThread::sActiveLODRequests = 0; U32 LLMeshRepoThread::sMaxConcurrentRequests = 1; +S32 LLMeshRepoThread::sRequestLowWater = REQUEST_LOW_WATER_MIN; +S32 LLMeshRepoThread::sRequestHighWater = REQUEST_HIGH_WATER_MIN; class LLMeshHandlerBase : public LLCore::HttpHandler { @@ -548,25 +554,37 @@ public: LLMeshRepoThread::LLMeshRepoThread() : LLThread("mesh repo"), - mCurlRequest(NULL), mWaiting(false), mHttpRequest(NULL), mHttpOptions(NULL), + mHttpLargeOptions(NULL), mHttpHeaders(NULL), - mHttpPolicyClass(LLCore::HttpRequest::DEFAULT_POLICY_ID) + mHttpPolicyClass(LLCore::HttpRequest::DEFAULT_POLICY_ID), + mHttpLargePolicyClass(LLCore::HttpRequest::DEFAULT_POLICY_ID), + mHttpPriority(0), + mHttpGetCount(0U), + mHttpLargeGetCount(0U) { mMutex = new LLMutex(NULL); mHeaderMutex = new LLMutex(NULL); mSignal = new LLCondition(NULL); mHttpRequest = new LLCore::HttpRequest; mHttpOptions = new LLCore::HttpOptions; + mHttpLargeOptions = new LLCore::HttpOptions; + mHttpLargeOptions->setTransferTimeout(LARGE_MESH_XFER_TIMEOUT); mHttpHeaders = new LLCore::HttpHeaders; mHttpHeaders->mHeaders.push_back("Accept: application/vnd.ll.mesh"); - mHttpPolicyClass = LLAppViewer::instance()->getAppCoreHttp().getPolicyMesh(); + mHttpPolicyClass = LLAppViewer::instance()->getAppCoreHttp().getPolicy(LLAppCoreHttp::AP_MESH); + mHttpPolicyClass = LLAppViewer::instance()->getAppCoreHttp().getPolicy(LLAppCoreHttp::AP_LARGE_MESH); } + LLMeshRepoThread::~LLMeshRepoThread() { + LL_INFOS("Mesh") << "Small GETs issued: " + << mHttpGetCount << ", Large GETs issued: " + << mHttpLargeGetCount << LL_ENDL; + for (http_request_set::iterator iter(mHttpRequestSet.begin()); iter != mHttpRequestSet.end(); ++iter) @@ -584,6 +602,11 @@ LLMeshRepoThread::~LLMeshRepoThread() mHttpOptions->release(); mHttpOptions = NULL; } + if (mHttpLargeOptions) + { + mHttpLargeOptions->release(); + mHttpLargeOptions = NULL; + } delete mHttpRequest; mHttpRequest = NULL; delete mMutex; @@ -596,7 +619,6 @@ LLMeshRepoThread::~LLMeshRepoThread() void LLMeshRepoThread::run() { - mCurlRequest = new LLCurlRequest(); LLCDResult res = LLConvexDecomposition::initThread(); if (res != LLCD_OK) { @@ -627,7 +649,7 @@ void LLMeshRepoThread::run() // NOTE: throttling intentionally favors LOD requests over header requests - while (!mLODReqQ.empty() && count < MAX_MESH_REQUESTS_PER_SECOND && sActiveLODRequests < sMaxConcurrentRequests) + while (!mLODReqQ.empty() && count < MAX_MESH_REQUESTS_PER_SECOND) { if (mMutex) { @@ -646,7 +668,7 @@ void LLMeshRepoThread::run() } } - while (!mHeaderReqQ.empty() && count < MAX_MESH_REQUESTS_PER_SECOND && sActiveHeaderRequests < sMaxConcurrentRequests) + while (!mHeaderReqQ.empty() && count < MAX_MESH_REQUESTS_PER_SECOND) { if (mMutex) { @@ -701,8 +723,6 @@ void LLMeshRepoThread::run() } mPhysicsShapeRequests = incomplete; } - - mCurlRequest->process(); } } @@ -716,9 +736,6 @@ void LLMeshRepoThread::run() { llwarns << "convex decomposition unable to be quit" << llendl; } - - delete mCurlRequest; - mCurlRequest = NULL; } void LLMeshRepoThread::loadMeshSkinInfo(const LLUUID& mesh_id) @@ -800,6 +817,42 @@ std::string LLMeshRepoThread::constructUrl(LLUUID mesh_id) return http_url; } +// May only be called by repo thread +LLCore::HttpHandle LLMeshRepoThread::getByteRange(const std::string & url, + size_t offset, + size_t len, + LLCore::HttpHandler * handler) +{ + LLCore::HttpHandle handle(LLCORE_HTTP_HANDLE_INVALID); + + if (len < LARGE_MESH_FETCH_THRESHOLD) + { + handle = mHttpRequest->requestGetByteRange(mHttpPolicyClass, + mHttpPriority, + url, + offset, + len, + mHttpOptions, + mHttpHeaders, + handler); + ++mHttpGetCount; + } + else + { + handle = mHttpRequest->requestGetByteRange(mHttpLargePolicyClass, + mHttpPriority, + url, + offset, + len, + mHttpLargeOptions, + mHttpHeaders, + handler); + ++mHttpLargeGetCount; + } + return handle; +} + + bool LLMeshRepoThread::fetchMeshSkinInfo(const LLUUID& mesh_id) { //protected by mMutex @@ -863,14 +916,7 @@ bool LLMeshRepoThread::fetchMeshSkinInfo(const LLUUID& mesh_id) { LLMeshSkinInfoHandler * handler = new LLMeshSkinInfoHandler(mesh_id, offset, size); // LL_WARNS("Mesh") << "MESH: Issuing Skin Info Request" << LL_ENDL; - LLCore::HttpHandle handle = mHttpRequest->requestGetByteRange(mHttpPolicyClass, - 0, // *TODO: Get better priority value - http_url, - offset, - size, - mHttpOptions, - mHttpHeaders, - handler); + LLCore::HttpHandle handle = getByteRange(http_url, offset, size, handler); if (LLCORE_HTTP_HANDLE_INVALID == handle) { // *TODO: Better error message @@ -959,14 +1005,7 @@ bool LLMeshRepoThread::fetchMeshDecomposition(const LLUUID& mesh_id) { LLMeshDecompositionHandler * handler = new LLMeshDecompositionHandler(mesh_id, offset, size); // LL_WARNS("Mesh") << "MESH: Issuing Decomp Request" << LL_ENDL; - LLCore::HttpHandle handle = mHttpRequest->requestGetByteRange(mHttpPolicyClass, - 0, // *TODO: Get better priority value - http_url, - offset, - size, - mHttpOptions, - mHttpHeaders, - handler); + LLCore::HttpHandle handle = getByteRange(http_url, offset, size, handler); if (LLCORE_HTTP_HANDLE_INVALID == handle) { // *TODO: Better error message @@ -1054,14 +1093,7 @@ bool LLMeshRepoThread::fetchMeshPhysicsShape(const LLUUID& mesh_id) { LLMeshPhysicsShapeHandler * handler = new LLMeshPhysicsShapeHandler(mesh_id, offset, size); // LL_WARNS("Mesh") << "MESH: Issuing Physics Shape Request" << LL_ENDL; - LLCore::HttpHandle handle = mHttpRequest->requestGetByteRange(mHttpPolicyClass, - 0, // *TODO: Get better priority value - http_url, - offset, - size, - mHttpOptions, - mHttpHeaders, - handler); + LLCore::HttpHandle handle = getByteRange(http_url, offset, size, handler); if (LLCORE_HTTP_HANDLE_INVALID == handle) { // *TODO: Better error message @@ -1154,14 +1186,7 @@ bool LLMeshRepoThread::fetchMeshHeader(const LLVolumeParams& mesh_params, U32& c LLMeshHeaderHandler * handler = new LLMeshHeaderHandler(mesh_params); // LL_WARNS("Mesh") << "MESH: Issuing Request" << LL_ENDL; - LLCore::HttpHandle handle = mHttpRequest->requestGetByteRange(mHttpPolicyClass, - 0, // *TODO: Get better priority value - http_url, - 0, - MESH_HEADER_SIZE, - mHttpOptions, - mHttpHeaders, - handler); + LLCore::HttpHandle handle = getByteRange(http_url, 0, MESH_HEADER_SIZE, handler); if (LLCORE_HTTP_HANDLE_INVALID == handle) { // *TODO: Better error message @@ -1241,14 +1266,7 @@ bool LLMeshRepoThread::fetchMeshLOD(const LLVolumeParams& mesh_params, S32 lod, { LLMeshLODHandler * handler = new LLMeshLODHandler(mesh_params, lod, offset, size); // LL_WARNS("Mesh") << "MESH: Issuing LOD Request" << LL_ENDL; - LLCore::HttpHandle handle = mHttpRequest->requestGetByteRange(mHttpPolicyClass, - 0, // *TODO: Get better priority value - http_url, - offset, - size, - mHttpOptions, - mHttpHeaders, - handler); + LLCore::HttpHandle handle = getByteRange(http_url, offset, size, handler); if (LLCORE_HTTP_HANDLE_INVALID == handle) { // *TODO: Better error message @@ -2537,9 +2555,14 @@ S32 LLMeshRepository::loadMesh(LLVOVolume* vobj, const LLVolumeParams& mesh_para void LLMeshRepository::notifyLoadedMeshes() { //called from main thread - - LLMeshRepoThread::sMaxConcurrentRequests = gSavedSettings.getU32("MeshMaxConcurrentRequests"); - + // *FIXME: Scaling down the setting by a factor of 4 for now to reflect + // target goal. May want to rename the setting before release. + LLMeshRepoThread::sMaxConcurrentRequests = gSavedSettings.getU32("MeshMaxConcurrentRequests") / 4; + LLMeshRepoThread::sRequestHighWater = llmax(50 * S32(LLMeshRepoThread::sMaxConcurrentRequests), + REQUEST_HIGH_WATER_MIN); + LLMeshRepoThread::sRequestLowWater = llmax(LLMeshRepoThread::sRequestLowWater / 2, + REQUEST_LOW_WATER_MIN); + //clean up completed upload threads for (std::vector::iterator iter = mUploads.begin(); iter != mUploads.end(); ) { @@ -2617,7 +2640,7 @@ void LLMeshRepository::notifyLoadedMeshes() //call completed callbacks on finished decompositions mDecompThread->notifyCompleted(); - if (!mThread->mWaiting) + if (!mThread->mWaiting && mPendingRequests.empty()) { //curl thread is churning, wait for it to go idle return; } @@ -2644,47 +2667,55 @@ void LLMeshRepository::notifyLoadedMeshes() mUploadErrorQ.pop(); } - S32 push_count = LLMeshRepoThread::sMaxConcurrentRequests-(LLMeshRepoThread::sActiveHeaderRequests+LLMeshRepoThread::sActiveLODRequests); - - if (push_count > 0) + S32 active_count = LLMeshRepoThread::sActiveHeaderRequests + LLMeshRepoThread::sActiveLODRequests; + if (active_count < LLMeshRepoThread::sRequestLowWater) { - //calculate "score" for pending requests - - //create score map - std::map score_map; + S32 push_count = LLMeshRepoThread::sRequestHighWater - active_count; - for (U32 i = 0; i < 4; ++i) + if (mPendingRequests.size() > push_count) { - for (mesh_load_map::iterator iter = mLoadingMeshes[i].begin(); iter != mLoadingMeshes[i].end(); ++iter) + // More requests than the high-water limit allows so + // sort and forward the most important. + + //calculate "score" for pending requests + + //create score map + std::map score_map; + + for (U32 i = 0; i < 4; ++i) { - F32 max_score = 0.f; - for (std::set::iterator obj_iter = iter->second.begin(); obj_iter != iter->second.end(); ++obj_iter) + for (mesh_load_map::iterator iter = mLoadingMeshes[i].begin(); iter != mLoadingMeshes[i].end(); ++iter) { - LLViewerObject* object = gObjectList.findObject(*obj_iter); - - if (object) + F32 max_score = 0.f; + for (std::set::iterator obj_iter = iter->second.begin(); obj_iter != iter->second.end(); ++obj_iter) { - LLDrawable* drawable = object->mDrawable; - if (drawable) + LLViewerObject* object = gObjectList.findObject(*obj_iter); + + if (object) { - F32 cur_score = drawable->getRadius()/llmax(drawable->mDistanceWRTCamera, 1.f); - max_score = llmax(max_score, cur_score); + LLDrawable* drawable = object->mDrawable; + if (drawable) + { + F32 cur_score = drawable->getRadius()/llmax(drawable->mDistanceWRTCamera, 1.f); + max_score = llmax(max_score, cur_score); + } } } - } - score_map[iter->first.getSculptID()] = max_score; + score_map[iter->first.getSculptID()] = max_score; + } } - } - //set "score" for pending requests - for (std::vector::iterator iter = mPendingRequests.begin(); iter != mPendingRequests.end(); ++iter) - { - iter->mScore = score_map[iter->mMeshParams.getSculptID()]; - } + //set "score" for pending requests + for (std::vector::iterator iter = mPendingRequests.begin(); iter != mPendingRequests.end(); ++iter) + { + iter->mScore = score_map[iter->mMeshParams.getSculptID()]; + } - //sort by "score" - std::sort(mPendingRequests.begin(), mPendingRequests.end(), LLMeshRepoThread::CompareScoreGreater()); + //sort by "score" + std::partial_sort(mPendingRequests.begin(), mPendingRequests.begin() + push_count, + mPendingRequests.end(), LLMeshRepoThread::CompareScoreGreater()); + } while (!mPendingRequests.empty() && push_count > 0) { diff --git a/indra/newview/llmeshrepository.h b/indra/newview/llmeshrepository.h index 62f81ce9e2..0dca29e7d4 100755 --- a/indra/newview/llmeshrepository.h +++ b/indra/newview/llmeshrepository.h @@ -224,13 +224,14 @@ public: static S32 sActiveHeaderRequests; static S32 sActiveLODRequests; static U32 sMaxConcurrentRequests; + static S32 sRequestLowWater; + static S32 sRequestHighWater; - LLCurlRequest* mCurlRequest; LLMutex* mMutex; LLMutex* mHeaderMutex; LLCondition* mSignal; - bool mWaiting; + volatile bool mWaiting; //map of known mesh headers typedef std::map mesh_header_map; @@ -324,8 +325,11 @@ public: // llcorehttp library interface objects. LLCore::HttpRequest * mHttpRequest; LLCore::HttpOptions * mHttpOptions; + LLCore::HttpOptions * mHttpLargeOptions; LLCore::HttpHeaders * mHttpHeaders; LLCore::HttpRequest::policy_t mHttpPolicyClass; + LLCore::HttpRequest::policy_t mHttpLargePolicyClass; + LLCore::HttpRequest::priority_t mHttpPriority; typedef std::set http_request_set; http_request_set mHttpRequestSet; // Outstanding HTTP requests @@ -373,6 +377,19 @@ public: static void incActiveHeaderRequests(); static void decActiveHeaderRequests(); +private: + // Issue a GET request to a URL with 'Range' header using + // the correct policy class and other attributes. If an invalid + // handle is returned, the request failed and caller must retry + // or dispose of handler. + // + // Threads: Repo thread only + LLCore::HttpHandle getByteRange(const std::string & url, size_t offset, size_t len, + LLCore::HttpHandler * handler); + +private: + U32 mHttpGetCount; + U32 mHttpLargeGetCount; }; class LLMeshUploadThread : public LLThread diff --git a/indra/newview/lltexturefetch.cpp b/indra/newview/lltexturefetch.cpp index be5fde9e2b..d934ef9dc4 100755 --- a/indra/newview/lltexturefetch.cpp +++ b/indra/newview/lltexturefetch.cpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2000&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -2410,7 +2410,7 @@ LLTextureFetch::LLTextureFetch(LLTextureCache* cache, LLImageDecodeThread* image mHttpHeaders->mHeaders.push_back("Accept: image/x-j2c"); mHttpMetricsHeaders = new LLCore::HttpHeaders; mHttpMetricsHeaders->mHeaders.push_back("Content-Type: application/llsd+xml"); - mHttpPolicyClass = LLAppViewer::instance()->getAppCoreHttp().getPolicyDefault(); + mHttpPolicyClass = LLAppViewer::instance()->getAppCoreHttp().getPolicy(LLAppCoreHttp::AP_TEXTURE); } LLTextureFetch::~LLTextureFetch() -- cgit v1.2.3