From 5cca78e718f15522cc3db9aec76aa910dd696aa8 Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Mon, 23 Jun 2014 14:23:33 -0400 Subject: First HTTP pipelining viewer. Enable pipelining for GetTexture and GetMesh2 at a pipeline depth of 5. Create global debug option, HttpPipelining, to enable and disable HTTP pipelining (defaults to true). Tweak texture and mesh low- and high-water request levels based on pipelining status and depth. Fixup texture console which was damaged in a recent release. Split logging of the no-request HTTP error case into two cases: one for missing URL in HTTP request, one for HTTP request not created. A refactor in llcorehttp is coming: I will be moving all libcurl- using code into libcurl-specific modules. --- indra/newview/app_settings/settings.xml | 11 +++ indra/newview/llappcorehttp.cpp | 169 ++++++++++++++++++++++---------- indra/newview/llappcorehttp.h | 39 ++++++-- indra/newview/llmeshrepository.cpp | 82 +++++++++------- indra/newview/lltexturefetch.cpp | 126 ++++++++++++++---------- indra/newview/lltexturefetch.h | 4 +- 6 files changed, 282 insertions(+), 149 deletions(-) (limited to 'indra/newview') diff --git a/indra/newview/app_settings/settings.xml b/indra/newview/app_settings/settings.xml index 8d3a7327b9..eb8a8bff51 100755 --- a/indra/newview/app_settings/settings.xml +++ b/indra/newview/app_settings/settings.xml @@ -4467,6 +4467,17 @@ Value + HttpPipelining + + Comment + If true, viewer will pipeline HTTP requests to servers. Static. + Persist + 1 + Type + Boolean + Value + 1 + IMShowTimestamps Comment diff --git a/indra/newview/llappcorehttp.cpp b/indra/newview/llappcorehttp.cpp index 70dcffefb2..deda0ccb41 100755 --- a/indra/newview/llappcorehttp.cpp +++ b/indra/newview/llappcorehttp.cpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012-2013, Linden Research, Inc. + * Copyright (C) 2012-2014, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -40,6 +40,8 @@ // be open at a time. const F64 LLAppCoreHttp::MAX_THREAD_WAIT_TIME(10.0); +const long LLAppCoreHttp::PIPELINING_DEPTH(5L); + static const struct { LLAppCoreHttp::EAppPolicy mPolicy; @@ -47,42 +49,43 @@ static const struct U32 mMin; U32 mMax; U32 mRate; + bool mPipelined; std::string mKey; const char * mUsage; } init_data[] = // Default and dynamic values for classes { { - LLAppCoreHttp::AP_DEFAULT, 8, 8, 8, 0, + LLAppCoreHttp::AP_DEFAULT, 8, 8, 8, 0, false, "", "other" }, { - LLAppCoreHttp::AP_TEXTURE, 8, 1, 12, 0, + LLAppCoreHttp::AP_TEXTURE, 8, 1, 12, 0, true, "TextureFetchConcurrency", "texture fetch" }, { - LLAppCoreHttp::AP_MESH1, 32, 1, 128, 100, + LLAppCoreHttp::AP_MESH1, 32, 1, 128, 100, false, "MeshMaxConcurrentRequests", "mesh fetch" }, { - LLAppCoreHttp::AP_MESH2, 8, 1, 32, 100, + LLAppCoreHttp::AP_MESH2, 8, 1, 32, 100, true, "Mesh2MaxConcurrentRequests", "mesh2 fetch" }, { - LLAppCoreHttp::AP_LARGE_MESH, 2, 1, 8, 0, + LLAppCoreHttp::AP_LARGE_MESH, 2, 1, 8, 0, false, "", "large mesh fetch" }, { - LLAppCoreHttp::AP_UPLOADS, 2, 1, 8, 0, + LLAppCoreHttp::AP_UPLOADS, 2, 1, 8, 0, false, "", "asset upload" }, { - LLAppCoreHttp::AP_LONG_POLL, 32, 32, 32, 0, + LLAppCoreHttp::AP_LONG_POLL, 32, 32, 32, 0, false, "", "long poll" } @@ -91,18 +94,20 @@ static const struct static void setting_changed(); +LLAppCoreHttp::HttpClass::HttpClass() + : mPolicy(LLCore::HttpRequest::DEFAULT_POLICY_ID), + mConnLimit(0U), + mPipelined(false) +{} + + LLAppCoreHttp::LLAppCoreHttp() : mRequest(NULL), mStopHandle(LLCORE_HTTP_HANDLE_INVALID), mStopRequested(0.0), - mStopped(false) -{ - for (int i(0); i < LL_ARRAY_SIZE(mPolicies); ++i) - { - mPolicies[i] = LLCore::HttpRequest::DEFAULT_POLICY_ID; - mSettings[i] = 0U; - } -} + mStopped(false), + mPipelined(true) +{} LLAppCoreHttp::~LLAppCoreHttp() @@ -121,6 +126,14 @@ void LLAppCoreHttp::init() << LL_ENDL; } + // Global pipelining preference from settings + static const std::string http_pipelining("HttpPipelining"); + if (gSavedSettings.controlExists(http_pipelining)) + { + // Default to true if absent. + mPipelined = gSavedSettings.getBOOL(http_pipelining); + } + // Point to our certs or SSH/https: will fail on connect status = LLCore::HttpRequest::setStaticPolicyOption(LLCore::HttpRequest::PO_CA_FILE, LLCore::HttpRequest::GLOBAL_POLICY_ID, @@ -157,27 +170,27 @@ void LLAppCoreHttp::init() } // Setup default policy and constrain if directed to - mPolicies[AP_DEFAULT] = LLCore::HttpRequest::DEFAULT_POLICY_ID; + mHttpClasses[AP_DEFAULT].mPolicy = LLCore::HttpRequest::DEFAULT_POLICY_ID; // Setup additional policies based on table and some special rules for (int i(0); i < LL_ARRAY_SIZE(init_data); ++i) { - const EAppPolicy policy(init_data[i].mPolicy); + const EAppPolicy app_policy(init_data[i].mPolicy); - if (AP_DEFAULT == policy) + if (AP_DEFAULT == app_policy) { // Pre-created continue; } - mPolicies[policy] = LLCore::HttpRequest::createPolicyClass(); - if (! mPolicies[policy]) + mHttpClasses[app_policy].mPolicy = LLCore::HttpRequest::createPolicyClass(); + if (! mHttpClasses[app_policy].mPolicy) { // Use default policy (but don't accidentally modify default) LL_WARNS("Init") << "Failed to create HTTP policy class for " << init_data[i].mUsage << ". Using default policy." << LL_ENDL; - mPolicies[policy] = mPolicies[AP_DEFAULT]; + mHttpClasses[app_policy].mPolicy = mHttpClasses[AP_DEFAULT].mPolicy; continue; } } @@ -196,6 +209,9 @@ void LLAppCoreHttp::init() << LL_ENDL; } + // *NOTE: Pipelining isn't dynamic yet. When it is, add a global + // signal for the setting here. + // Register signals for settings and state changes for (int i(0); i < LL_ARRAY_SIZE(init_data); ++i) { @@ -209,7 +225,7 @@ void LLAppCoreHttp::init() } else { - mSettingsSignal[i] = cntrl_ptr->getCommitSignal()->connect(boost::bind(&setting_changed)); + mHttpClasses[i].mSettingsSignal = cntrl_ptr->getCommitSignal()->connect(boost::bind(&setting_changed)); } } } @@ -261,9 +277,9 @@ void LLAppCoreHttp::cleanup() } } - for (int i(0); i < LL_ARRAY_SIZE(init_data); ++i) + for (int i(0); i < LL_ARRAY_SIZE(mHttpClasses); ++i) { - mSettingsSignal[i].disconnect(); + mHttpClasses[i].mSettingsSignal.disconnect(); } delete mRequest; @@ -278,30 +294,57 @@ void LLAppCoreHttp::cleanup() } } + void LLAppCoreHttp::refreshSettings(bool initial) { LLCore::HttpStatus status; for (int i(0); i < LL_ARRAY_SIZE(init_data); ++i) { - const EAppPolicy policy(init_data[i].mPolicy); + const EAppPolicy app_policy(init_data[i].mPolicy); - // Set any desired throttle - if (initial && init_data[i].mRate) + if (initial) { - // Init-time only, can use the static setters here - status = LLCore::HttpRequest::setStaticPolicyOption(LLCore::HttpRequest::PO_THROTTLE_RATE, - mPolicies[policy], - init_data[i].mRate, - NULL); - if (! status) + // Init-time only settings, can use the static setters here + + if (init_data[i].mRate) { - LL_WARNS("Init") << "Unable to set " << init_data[i].mUsage - << " throttle rate. Reason: " << status.toString() - << LL_ENDL; + // Set any desired throttle + status = LLCore::HttpRequest::setStaticPolicyOption(LLCore::HttpRequest::PO_THROTTLE_RATE, + mHttpClasses[app_policy].mPolicy, + init_data[i].mRate, + NULL); + if (! status) + { + LL_WARNS("Init") << "Unable to set " << init_data[i].mUsage + << " throttle rate. Reason: " << status.toString() + << LL_ENDL; + } + } + + mHttpClasses[app_policy].mPipelined = false; + if (mPipelined && init_data[i].mPipelined) + { + // Pipelining election is currently static (init-time). + status = LLCore::HttpRequest::setStaticPolicyOption(LLCore::HttpRequest::PO_PIPELINING_DEPTH, + mHttpClasses[app_policy].mPolicy, + PIPELINING_DEPTH, + NULL); + if (! status) + { + LL_WARNS("Init") << "Unable to set " << init_data[i].mUsage + << " to pipelined mode. Reason: " << status.toString() + << LL_ENDL; + } + else + { + mHttpClasses[app_policy].mPipelined = true; + } } } + // Init- or run-time settings + // Get target connection concurrency value U32 setting(init_data[i].mDefault); if (! init_data[i].mKey.empty() && gSavedSettings.controlExists(init_data[i].mKey)) @@ -314,19 +357,31 @@ void LLAppCoreHttp::refreshSettings(bool initial) } } - if (! initial && setting == mSettings[policy]) + if (! initial && setting == mHttpClasses[app_policy].mConnLimit) { // Unchanged, try next setting continue; } - // Set it and report - // *TODO: These are intended to be per-host limits when we can - // support that in llcorehttp/libcurl. + // Set it and report. Strategies depend on pipelining: + // + // No Pipelining. Llcorehttp manages connections itself based + // on the PO_CONNECTION_LIMIT setting. Set both limits to the + // same value for logical consistency. In the future, may + // hand over connection management to libcurl after the + // connection cache has been better vetted. + // + // Pipelining. Libcurl is allowed to manage connections to a + // great degree. Steady state will connection limit based on + // the per-host setting. Transitions (region crossings, new + // avatars, etc.) can request additional outbound connections + // to other servers via 2X total connection limit. + // LLCore::HttpHandle handle; handle = mRequest->setPolicyOption(LLCore::HttpRequest::PO_CONNECTION_LIMIT, - mPolicies[policy], - setting, NULL); + mHttpClasses[app_policy].mPolicy, + (mHttpClasses[app_policy].mPipelined ? 2 * setting : setting), + NULL); if (LLCORE_HTTP_HANDLE_INVALID == handle) { status = mRequest->getStatus(); @@ -336,16 +391,30 @@ void LLAppCoreHttp::refreshSettings(bool initial) } else { - LL_DEBUGS("Init") << "Changed " << init_data[i].mUsage - << " concurrency. New value: " << setting - << LL_ENDL; - mSettings[policy] = setting; - if (initial && setting != init_data[i].mDefault) + handle = mRequest->setPolicyOption(LLCore::HttpRequest::PO_PER_HOST_CONNECTION_LIMIT, + mHttpClasses[app_policy].mPolicy, + setting, + NULL); + if (LLCORE_HTTP_HANDLE_INVALID == handle) { - LL_INFOS("Init") << "Application settings overriding default " << init_data[i].mUsage - << " concurrency. New value: " << setting + status = mRequest->getStatus(); + LL_WARNS("Init") << "Unable to set " << init_data[i].mUsage + << " per-host concurrency. Reason: " << status.toString() << LL_ENDL; } + else + { + LL_DEBUGS("Init") << "Changed " << init_data[i].mUsage + << " concurrency. New value: " << setting + << LL_ENDL; + mHttpClasses[app_policy].mConnLimit = setting; + if (initial && setting != init_data[i].mDefault) + { + LL_INFOS("Init") << "Application settings overriding default " << init_data[i].mUsage + << " concurrency. New value: " << setting + << LL_ENDL; + } + } } } } diff --git a/indra/newview/llappcorehttp.h b/indra/newview/llappcorehttp.h index 40e3042b84..63c8a11180 100755 --- a/indra/newview/llappcorehttp.h +++ b/indra/newview/llappcorehttp.h @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012-2013, Linden Research, Inc. + * Copyright (C) 2012-2014, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -41,6 +41,8 @@ class LLAppCoreHttp : public LLCore::HttpHandler { public: + static const long PIPELINING_DEPTH; + typedef LLCore::HttpRequest::policy_t policy_t; enum EAppPolicy @@ -70,7 +72,7 @@ public: /// Long poll: no /// Concurrency: high /// Request rate: high - /// Pipelined: soon + /// Pipelined: yes AP_TEXTURE, /// Legacy mesh fetching policy class. Used to @@ -98,7 +100,7 @@ public: /// Long poll: no /// Concurrency: high /// Request rate: high - /// Pipelined: soon + /// Pipelined: yes AP_MESH2, /// Large mesh fetching policy class. Used to @@ -116,7 +118,7 @@ public: /// Long poll: no /// Concurrency: low /// Request rate: low - /// Pipelined: soon + /// Pipelined: no AP_LARGE_MESH, /// Asset upload policy class. Used to store @@ -180,7 +182,13 @@ public: // application function. policy_t getPolicy(EAppPolicy policy) const { - return mPolicies[policy]; + return mHttpClasses[policy].mPolicy; + } + + // Return whether a policy is using pipelined operations. + bool isPipelined(EAppPolicy policy) const + { + return mHttpClasses[policy].mPipelined; } // Apply initial or new settings from the environment. @@ -190,13 +198,26 @@ private: static const F64 MAX_THREAD_WAIT_TIME; private: - LLCore::HttpRequest * mRequest; // Request queue to issue shutdowns + + // PODish container for per-class settings and state. + struct HttpClass + { + public: + HttpClass(); + + public: + policy_t mPolicy; // Policy class id for the class + U32 mConnLimit; + bool mPipelined; + boost::signals2::connection mSettingsSignal; // Signal to global setting that affect this class (if any) + }; + + LLCore::HttpRequest * mRequest; // Request queue to issue shutdowns LLCore::HttpHandle mStopHandle; F64 mStopRequested; bool mStopped; - policy_t mPolicies[AP_COUNT]; // Policy class id for each connection set - U32 mSettings[AP_COUNT]; - boost::signals2::connection mSettingsSignal[AP_COUNT]; // Signals to global settings that affect us + HttpClass mHttpClasses[AP_COUNT]; + bool mPipelined; // Global setting }; diff --git a/indra/newview/llmeshrepository.cpp b/indra/newview/llmeshrepository.cpp index 80a427c0b8..c91ae975ea 100755 --- a/indra/newview/llmeshrepository.cpp +++ b/indra/newview/llmeshrepository.cpp @@ -343,9 +343,9 @@ const S32 REQUEST_HIGH_WATER_MAX = 150; // Should remain under 2X throttle const S32 REQUEST_LOW_WATER_MIN = 16; const S32 REQUEST_LOW_WATER_MAX = 75; const S32 REQUEST2_HIGH_WATER_MIN = 32; // Limits for GetMesh2 regions -const S32 REQUEST2_HIGH_WATER_MAX = 80; +const S32 REQUEST2_HIGH_WATER_MAX = 100; const S32 REQUEST2_LOW_WATER_MIN = 16; -const S32 REQUEST2_LOW_WATER_MAX = 40; +const S32 REQUEST2_LOW_WATER_MAX = 50; const U32 LARGE_MESH_FETCH_THRESHOLD = 1U << 21; // Size at which requests goes to narrow/slow queue const long SMALL_MESH_XFER_TIMEOUT = 120L; // Seconds to complete xfer, small mesh downloads const long LARGE_MESH_XFER_TIMEOUT = 600L; // Seconds to complete xfer, large downloads @@ -754,7 +754,9 @@ LLMeshRepoThread::LLMeshRepoThread() mHttpLargePolicyClass(LLCore::HttpRequest::DEFAULT_POLICY_ID), mHttpPriority(0), mGetMeshVersion(2) - { +{ + LLAppCoreHttp & app_core_http(LLAppViewer::instance()->getAppCoreHttp()); + mMutex = new LLMutex(NULL); mHeaderMutex = new LLMutex(NULL); mSignal = new LLCondition(NULL); @@ -767,10 +769,10 @@ LLMeshRepoThread::LLMeshRepoThread() mHttpLargeOptions->setUseRetryAfter(gSavedSettings.getBOOL("MeshUseHttpRetryAfter")); mHttpHeaders = new LLCore::HttpHeaders; mHttpHeaders->append("Accept", "application/vnd.ll.mesh"); - mHttpPolicyClass = LLAppViewer::instance()->getAppCoreHttp().getPolicy(LLAppCoreHttp::AP_MESH2); - mHttpLegacyPolicyClass = LLAppViewer::instance()->getAppCoreHttp().getPolicy(LLAppCoreHttp::AP_MESH1); - mHttpLargePolicyClass = LLAppViewer::instance()->getAppCoreHttp().getPolicy(LLAppCoreHttp::AP_LARGE_MESH); - } + mHttpPolicyClass = app_core_http.getPolicy(LLAppCoreHttp::AP_MESH2); + mHttpLegacyPolicyClass = app_core_http.getPolicy(LLAppCoreHttp::AP_MESH1); + mHttpLargePolicyClass = app_core_http.getPolicy(LLAppCoreHttp::AP_LARGE_MESH); +} LLMeshRepoThread::~LLMeshRepoThread() @@ -846,48 +848,49 @@ void LLMeshRepoThread::run() { // Dispatch all HttpHandler notifications mHttpRequest->update(0L); - } + } sRequestWaterLevel = mHttpRequestSet.size(); // Stats data update // NOTE: order of queue processing intentionally favors LOD requests over header requests while (!mLODReqQ.empty() && mHttpRequestSet.size() < sRequestHighWater) - { + { if (! mMutex) - { + { break; } - mMutex->lock(); - LODRequest req = mLODReqQ.front(); - mLODReqQ.pop(); - LLMeshRepository::sLODProcessing--; - mMutex->unlock(); + mMutex->lock(); + LODRequest req = mLODReqQ.front(); + mLODReqQ.pop(); + LLMeshRepository::sLODProcessing--; + mMutex->unlock(); + if (!fetchMeshLOD(req.mMeshParams, req.mLOD)) // failed, resubmit - { - mMutex->lock(); - mLODReqQ.push(req); + { + mMutex->lock(); + mLODReqQ.push(req); ++LLMeshRepository::sLODProcessing; - mMutex->unlock(); - } - } + mMutex->unlock(); + } + } while (!mHeaderReqQ.empty() && mHttpRequestSet.size() < sRequestHighWater) - { + { if (! mMutex) - { + { break; } - mMutex->lock(); - HeaderRequest req = mHeaderReqQ.front(); - mHeaderReqQ.pop(); - mMutex->unlock(); + mMutex->lock(); + HeaderRequest req = mHeaderReqQ.front(); + mHeaderReqQ.pop(); + mMutex->unlock(); if (!fetchMeshHeader(req.mMeshParams))//failed, resubmit - { - mMutex->lock(); - mHeaderReqQ.push(req) ; - mMutex->unlock(); - } - } + { + mMutex->lock(); + mHeaderReqQ.push(req) ; + mMutex->unlock(); + } + } // For the final three request lists, similar goal to above but // slightly different queue structures. Stay off the mutex when @@ -983,7 +986,7 @@ void LLMeshRepoThread::run() } } mMutex->unlock(); - } + } // For dev purposes only. A dynamic change could make this false // and that shouldn't assert. @@ -1250,7 +1253,6 @@ bool LLMeshRepoThread::fetchMeshSkinInfo(const LLUUID& mesh_id) << LL_ENDL; delete handler; ret = false; - } else { @@ -1860,7 +1862,7 @@ LLMeshUploadThread::LLMeshUploadThread(LLMeshUploadThread::instance_list& data, bool upload_skin, bool upload_joints, const std::string & upload_url, bool do_upload, LLHandle fee_observer, LLHandle upload_observer) -: LLThread("mesh upload"), + : LLThread("mesh upload"), LLCore::HttpHandler(), mDiscarded(false), mDoUpload(do_upload), @@ -3198,9 +3200,13 @@ void LLMeshRepository::notifyLoadedMeshes() else { // GetMesh2 operation with keepalives, etc. With pipelining, - // we'll increase this. + // we'll increase this. See llappcorehttp and llcorehttp for + // discussion on connection strategies. + LLAppCoreHttp & app_core_http(LLAppViewer::instance()->getAppCoreHttp()); + S32 scale(app_core_http.isPipelined(LLAppCoreHttp::AP_MESH2) ? 10 : 5); + LLMeshRepoThread::sMaxConcurrentRequests = gSavedSettings.getU32("Mesh2MaxConcurrentRequests"); - LLMeshRepoThread::sRequestHighWater = llclamp(5 * S32(LLMeshRepoThread::sMaxConcurrentRequests), + LLMeshRepoThread::sRequestHighWater = llclamp(scale * S32(LLMeshRepoThread::sMaxConcurrentRequests), REQUEST2_HIGH_WATER_MIN, REQUEST2_HIGH_WATER_MAX); LLMeshRepoThread::sRequestLowWater = llclamp(LLMeshRepoThread::sRequestHighWater / 2, diff --git a/indra/newview/lltexturefetch.cpp b/indra/newview/lltexturefetch.cpp index 548ed98f39..c33884e1e6 100755 --- a/indra/newview/lltexturefetch.cpp +++ b/indra/newview/lltexturefetch.cpp @@ -241,8 +241,10 @@ LLTrace::EventStatHandle LLTextureFetch::sCacheReadLatency("te // Tuning/Parameterization Constants -static const S32 HTTP_REQUESTS_IN_QUEUE_HIGH_WATER = 40; // Maximum requests to have active in HTTP -static const S32 HTTP_REQUESTS_IN_QUEUE_LOW_WATER = 20; // Active level at which to refill +static const S32 HTTP_PIPE_REQUESTS_HIGH_WATER = 100; // Maximum requests to have active in HTTP (pipelined) +static const S32 HTTP_PIPE_REQUESTS_LOW_WATER = 50; // Active level at which to refill +static const S32 HTTP_NONPIPE_REQUESTS_HIGH_WATER = 40; +static const S32 HTTP_NONPIPE_REQUESTS_LOW_WATER = 20; // BUG-3323/SH-4375 // *NOTE: This is a heuristic value. Texture fetches have a habit of using a @@ -608,16 +610,16 @@ private: LLCore::HttpHandle mHttpHandle; // Handle of any active request LLCore::BufferArray * mHttpBufferArray; // Refcounted pointer to response data - S32 mHttpPolicyClass; + S32 mHttpPolicyClass; bool mHttpActive; // Active request to http library - U32 mHttpReplySize, // Actual received data size - mHttpReplyOffset; // Actual received data offset + U32 mHttpReplySize, // Actual received data size + mHttpReplyOffset; // Actual received data offset bool mHttpHasResource; // Counts against Fetcher's mHttpSemaphore // State history - U32 mCacheReadCount, - mCacheWriteCount, - mResourceWaitCount; // Requests entering WAIT_HTTP_RESOURCE2 + U32 mCacheReadCount, + mCacheWriteCount, + mResourceWaitCount; // Requests entering WAIT_HTTP_RESOURCE2 }; ////////////////////////////////////////////////////////////////////////////// @@ -1525,36 +1527,49 @@ bool LLTextureFetchWorker::doWork(S32 param) mRequestedOffset -= 1; mRequestedSize += 1; } - mHttpHandle = LLCORE_HTTP_HANDLE_INVALID; - if (!mUrl.empty()) - { - mRequestedTimer.reset(); - mLoaded = FALSE; - mGetStatus = LLCore::HttpStatus(); - mGetReason.clear(); - LL_DEBUGS(LOG_TXT) << "HTTP GET: " << mID << " Offset: " << mRequestedOffset - << " Bytes: " << mRequestedSize - << " Bandwidth(kbps): " << mFetcher->getTextureBandwidth() << "/" << mFetcher->mMaxBandwidth - << LL_ENDL; - // Will call callbackHttpGet when curl request completes - // Only server bake images use the returned headers currently, for getting retry-after field. - LLCore::HttpOptions *options = (mFTType == FTT_SERVER_BAKE) ? mFetcher->mHttpOptionsWithHeaders: mFetcher->mHttpOptions; - mHttpHandle = mFetcher->mHttpRequest->requestGetByteRange(mHttpPolicyClass, - mWorkPriority, - mUrl, - mRequestedOffset, - (mRequestedOffset + mRequestedSize) > HTTP_REQUESTS_RANGE_END_MAX - ? 0 - : mRequestedSize, - options, - mFetcher->mHttpHeaders, - this); + if (mUrl.empty()) + { + // *FIXME: This should not be reachable except it has become + // so after some recent 'work'. Need to track this down + // and illuminate the unenlightened. + LL_WARNS(LOG_TXT) << "HTTP GET request failed for " << mID + << " on empty URL." << LL_ENDL; + resetFormattedData(); + releaseHttpSemaphore(); + return true; // failed } + + mRequestedTimer.reset(); + mLoaded = FALSE; + mGetStatus = LLCore::HttpStatus(); + mGetReason.clear(); + LL_DEBUGS(LOG_TXT) << "HTTP GET: " << mID << " Offset: " << mRequestedOffset + << " Bytes: " << mRequestedSize + << " Bandwidth(kbps): " << mFetcher->getTextureBandwidth() << "/" << mFetcher->mMaxBandwidth + << LL_ENDL; + + // Will call callbackHttpGet when curl request completes + // Only server bake images use the returned headers currently, for getting retry-after field. + LLCore::HttpOptions *options = (mFTType == FTT_SERVER_BAKE) ? mFetcher->mHttpOptionsWithHeaders: mFetcher->mHttpOptions; + mHttpHandle = mFetcher->mHttpRequest->requestGetByteRange(mHttpPolicyClass, + mWorkPriority, + mUrl, + mRequestedOffset, + (mRequestedOffset + mRequestedSize) > HTTP_REQUESTS_RANGE_END_MAX + ? 0 + : mRequestedSize, + options, + mFetcher->mHttpHeaders, + this); if (LLCORE_HTTP_HANDLE_INVALID == mHttpHandle) { - LL_WARNS(LOG_TXT) << "HTTP GET request failed for " << mID << LL_ENDL; + LLCore::HttpStatus status(mFetcher->mHttpRequest->getStatus()); + LL_WARNS(LOG_TXT) << "HTTP GET request failed for " << mID + << ", Status: " << status.toTerseString() + << " Reason: '" << status.toString() << "'" + << LL_ENDL; resetFormattedData(); releaseHttpSemaphore(); return true; // failed @@ -1610,10 +1625,6 @@ bool LLTextureFetchWorker::doWork(S32 param) else if (http_service_unavail == mGetStatus) { LL_INFOS_ONCE(LOG_TXT) << "Texture server busy (503): " << mUrl << LL_ENDL; - LL_INFOS(LOG_TXT) << "503: HTTP GET failed for: " << mUrl - << " Status: " << mGetStatus.toHex() - << " Reason: '" << mGetReason << "'" - << LL_ENDL; } else if (http_not_sat == mGetStatus) { @@ -2482,7 +2493,6 @@ LLTextureFetch::LLTextureFetch(LLTextureCache* cache, LLImageDecodeThread* image mHttpHeaders(NULL), mHttpMetricsHeaders(NULL), mHttpPolicyClass(LLCore::HttpRequest::DEFAULT_POLICY_ID), - mHttpSemaphore(HTTP_REQUESTS_IN_QUEUE_HIGH_WATER), mTotalCacheReadCount(0U), mTotalCacheWriteCount(0U), mTotalResourceWaitCount(0U), @@ -2494,6 +2504,30 @@ LLTextureFetch::LLTextureFetch(LLTextureCache* cache, LLImageDecodeThread* image mMaxBandwidth = gSavedSettings.getF32("ThrottleBandwidthKBPS"); mTextureInfo.setUpLogging(gSavedSettings.getBOOL("LogTextureDownloadsToViewerLog"), gSavedSettings.getBOOL("LogTextureDownloadsToSimulator"), U32Bytes(gSavedSettings.getU32("TextureLoggingThreshold"))); + mHttpRequest = new LLCore::HttpRequest; + mHttpOptions = new LLCore::HttpOptions; + mHttpOptionsWithHeaders = new LLCore::HttpOptions; + mHttpOptionsWithHeaders->setWantHeaders(true); + mHttpHeaders = new LLCore::HttpHeaders; + mHttpHeaders->append("Accept", "image/x-j2c"); + mHttpMetricsHeaders = new LLCore::HttpHeaders; + mHttpMetricsHeaders->append("Content-Type", "application/llsd+xml"); + LLAppCoreHttp & app_core_http(LLAppViewer::instance()->getAppCoreHttp()); + mHttpPolicyClass = app_core_http.getPolicy(LLAppCoreHttp::AP_TEXTURE); + if (app_core_http.isPipelined(LLAppCoreHttp::AP_TEXTURE)) + { + mHttpHighWater = HTTP_PIPE_REQUESTS_HIGH_WATER; + mHttpLowWater = HTTP_PIPE_REQUESTS_LOW_WATER; + } + else + { + mHttpHighWater = HTTP_NONPIPE_REQUESTS_HIGH_WATER; + mHttpLowWater = HTTP_NONPIPE_REQUESTS_LOW_WATER; + } + mHttpSemaphore = mHttpHighWater; + + // Conditionally construct debugger object after 'this' is + // fully initialized. LLTextureFetchDebugger::sDebuggerEnabled = gSavedSettings.getBOOL("TextureFetchDebuggerEnabled"); if(LLTextureFetchDebugger::isEnabled()) { @@ -2506,16 +2540,6 @@ LLTextureFetch::LLTextureFetch(LLTextureCache* cache, LLImageDecodeThread* image } mOriginFetchSource = mFetchSource; } - - mHttpRequest = new LLCore::HttpRequest; - mHttpOptions = new LLCore::HttpOptions; - mHttpOptionsWithHeaders = new LLCore::HttpOptions; - mHttpOptionsWithHeaders->setWantHeaders(true); - mHttpHeaders = new LLCore::HttpHeaders; - mHttpHeaders->append("Accept", "image/x-j2c"); - mHttpMetricsHeaders = new LLCore::HttpHeaders; - mHttpMetricsHeaders->append("Content-Type", "application/llsd+xml"); - mHttpPolicyClass = LLAppViewer::instance()->getAppCoreHttp().getPolicy(LLAppCoreHttp::AP_TEXTURE); } LLTextureFetch::~LLTextureFetch() @@ -3645,7 +3669,7 @@ void LLTextureFetch::releaseHttpWaiters() { // Use mHttpSemaphore rather than mHTTPTextureQueue.size() // to avoid a lock. - if (mHttpSemaphore < (HTTP_REQUESTS_IN_QUEUE_HIGH_WATER - HTTP_REQUESTS_IN_QUEUE_LOW_WATER)) + if (mHttpSemaphore < (mHttpHighWater - mHttpLowWater)) return; // Quickly make a copy of all the LLUIDs. Get off the @@ -4538,7 +4562,7 @@ S32 LLTextureFetchDebugger::fillCurlQueue() mNbCurlCompleted = mFetchingHistory.size(); return 0; } - if (mNbCurlRequests > HTTP_REQUESTS_IN_QUEUE_LOW_WATER) + if (mNbCurlRequests > HTTP_NONPIPE_REQUESTS_LOW_WATER) { return mNbCurlRequests; } @@ -4571,7 +4595,7 @@ S32 LLTextureFetchDebugger::fillCurlQueue() mFetchingHistory[i].mHttpHandle = handle; mFetchingHistory[i].mCurlState = FetchEntry::CURL_IN_PROGRESS; mNbCurlRequests++; - if (mNbCurlRequests >= HTTP_REQUESTS_IN_QUEUE_HIGH_WATER) // emulate normal pipeline + if (mNbCurlRequests >= HTTP_NONPIPE_REQUESTS_HIGH_WATER) // emulate normal pipeline { break; } diff --git a/indra/newview/lltexturefetch.h b/indra/newview/lltexturefetch.h index c4da2e8685..d13736997f 100755 --- a/indra/newview/lltexturefetch.h +++ b/indra/newview/lltexturefetch.h @@ -356,7 +356,9 @@ private: LLCore::HttpHeaders * mHttpHeaders; // Ttf LLCore::HttpHeaders * mHttpMetricsHeaders; // Ttf LLCore::HttpRequest::policy_t mHttpPolicyClass; // T* - + S32 mHttpHighWater; // T* (ro) + S32 mHttpLowWater; // T* (ro) + // We use a resource semaphore to keep HTTP requests in // WAIT_HTTP_RESOURCE2 if there aren't sufficient slots in the // transport. This keeps them near where they can be cheaply -- cgit v1.2.3 From 17da4cf57aadcf1987b48af298d8b2742089a35c Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Fri, 27 Jun 2014 17:25:39 -0400 Subject: Cleanup and tuning. Use a consistent index on some initialization data so their isn't an opportunity for gaps over overruns (init_data). Start some preliminary tweaking of policy class numbers. It looks like I can easily drop the default connection count to '4' and still hit the throttles. Did some experiments running pipeline deeper which was mostly fine for textures but tended to slow meshes. Reason uncertain but a depth of '5' seems generally healthy for mesh. I had one run of 52.6S with a theoretical minimum of 51.2S. That's as good as I've ever seen. --- indra/newview/llappcorehttp.cpp | 41 +++++++++++++++++++++----------------- indra/newview/llmeshrepository.cpp | 7 ++++++- indra/newview/lltexturefetch.cpp | 2 ++ 3 files changed, 31 insertions(+), 19 deletions(-) (limited to 'indra/newview') diff --git a/indra/newview/llappcorehttp.cpp b/indra/newview/llappcorehttp.cpp index deda0ccb41..d097f18d61 100755 --- a/indra/newview/llappcorehttp.cpp +++ b/indra/newview/llappcorehttp.cpp @@ -42,9 +42,9 @@ const F64 LLAppCoreHttp::MAX_THREAD_WAIT_TIME(10.0); const long LLAppCoreHttp::PIPELINING_DEPTH(5L); +// Default and dynamic values for classes static const struct { - LLAppCoreHttp::EAppPolicy mPolicy; U32 mDefault; U32 mMin; U32 mMax; @@ -52,40 +52,40 @@ static const struct bool mPipelined; std::string mKey; const char * mUsage; -} init_data[] = // Default and dynamic values for classes +} init_data[LLAppCoreHttp::AP_COUNT] = { - { - LLAppCoreHttp::AP_DEFAULT, 8, 8, 8, 0, false, + { // AP_DEFAULT + 8, 8, 8, 0, false, "", "other" }, - { - LLAppCoreHttp::AP_TEXTURE, 8, 1, 12, 0, true, + { // AP_TEXTURE + 4, 1, 12, 0, true, "TextureFetchConcurrency", "texture fetch" }, - { - LLAppCoreHttp::AP_MESH1, 32, 1, 128, 100, false, + { // AP_MESH1 + 32, 1, 128, 100, false, "MeshMaxConcurrentRequests", "mesh fetch" }, - { - LLAppCoreHttp::AP_MESH2, 8, 1, 32, 100, true, + { // AP_MESH2 + 4, 1, 32, 100, true, "Mesh2MaxConcurrentRequests", "mesh2 fetch" }, - { - LLAppCoreHttp::AP_LARGE_MESH, 2, 1, 8, 0, false, + { // AP_LARGE_MESH + 2, 1, 8, 0, false, "", "large mesh fetch" }, - { - LLAppCoreHttp::AP_UPLOADS, 2, 1, 8, 0, false, + { // AP_UPLOADS + 2, 1, 8, 0, false, "", "asset upload" }, - { - LLAppCoreHttp::AP_LONG_POLL, 32, 32, 32, 0, false, + { // AP_LONG_POLL + 32, 32, 32, 0, false, "", "long poll" } @@ -173,9 +173,10 @@ void LLAppCoreHttp::init() mHttpClasses[AP_DEFAULT].mPolicy = LLCore::HttpRequest::DEFAULT_POLICY_ID; // Setup additional policies based on table and some special rules + llassert(LL_ARRAY_SIZE(init_data) == AP_COUNT); for (int i(0); i < LL_ARRAY_SIZE(init_data); ++i) { - const EAppPolicy app_policy(init_data[i].mPolicy); + const EAppPolicy app_policy(static_cast(i)); if (AP_DEFAULT == app_policy) { @@ -301,7 +302,7 @@ void LLAppCoreHttp::refreshSettings(bool initial) for (int i(0); i < LL_ARRAY_SIZE(init_data); ++i) { - const EAppPolicy app_policy(init_data[i].mPolicy); + const EAppPolicy app_policy(static_cast(i)); if (initial) { @@ -326,6 +327,10 @@ void LLAppCoreHttp::refreshSettings(bool initial) if (mPipelined && init_data[i].mPipelined) { // Pipelining election is currently static (init-time). + // Making it dynamic isn't too hard in the SL code but verifying + // that libcurl handles the on-to-off transition while holding + // outstanding requests is something that should be tested. + status = LLCore::HttpRequest::setStaticPolicyOption(LLCore::HttpRequest::PO_PIPELINING_DEPTH, mHttpClasses[app_policy].mPolicy, PIPELINING_DEPTH, diff --git a/indra/newview/llmeshrepository.cpp b/indra/newview/llmeshrepository.cpp index c91ae975ea..74eb6015f2 100755 --- a/indra/newview/llmeshrepository.cpp +++ b/indra/newview/llmeshrepository.cpp @@ -338,14 +338,17 @@ static LLFastTimer::DeclareTimer FTM_MESH_FETCH("Mesh Fetch"); LLMeshRepository gMeshRepo; const S32 MESH_HEADER_SIZE = 4096; // Important: assumption is that headers fit in this space + const S32 REQUEST_HIGH_WATER_MIN = 32; // Limits for GetMesh regions const S32 REQUEST_HIGH_WATER_MAX = 150; // Should remain under 2X throttle const S32 REQUEST_LOW_WATER_MIN = 16; const S32 REQUEST_LOW_WATER_MAX = 75; + const S32 REQUEST2_HIGH_WATER_MIN = 32; // Limits for GetMesh2 regions const S32 REQUEST2_HIGH_WATER_MAX = 100; const S32 REQUEST2_LOW_WATER_MIN = 16; const S32 REQUEST2_LOW_WATER_MAX = 50; + const U32 LARGE_MESH_FETCH_THRESHOLD = 1U << 21; // Size at which requests goes to narrow/slow queue const long SMALL_MESH_XFER_TIMEOUT = 120L; // Seconds to complete xfer, small mesh downloads const long LARGE_MESH_XFER_TIMEOUT = 600L; // Seconds to complete xfer, large downloads @@ -3203,7 +3206,9 @@ void LLMeshRepository::notifyLoadedMeshes() // we'll increase this. See llappcorehttp and llcorehttp for // discussion on connection strategies. LLAppCoreHttp & app_core_http(LLAppViewer::instance()->getAppCoreHttp()); - S32 scale(app_core_http.isPipelined(LLAppCoreHttp::AP_MESH2) ? 10 : 5); + S32 scale(app_core_http.isPipelined(LLAppCoreHttp::AP_MESH2) + ? (2 * LLAppCoreHttp::PIPELINING_DEPTH) + : 5); LLMeshRepoThread::sMaxConcurrentRequests = gSavedSettings.getU32("Mesh2MaxConcurrentRequests"); LLMeshRepoThread::sRequestHighWater = llclamp(scale * S32(LLMeshRepoThread::sMaxConcurrentRequests), diff --git a/indra/newview/lltexturefetch.cpp b/indra/newview/lltexturefetch.cpp index ee11e7b47b..ecaf5f8d06 100755 --- a/indra/newview/lltexturefetch.cpp +++ b/indra/newview/lltexturefetch.cpp @@ -2516,6 +2516,8 @@ LLTextureFetch::LLTextureFetch(LLTextureCache* cache, LLImageDecodeThread* image mHttpPolicyClass = app_core_http.getPolicy(LLAppCoreHttp::AP_TEXTURE); if (app_core_http.isPipelined(LLAppCoreHttp::AP_TEXTURE)) { + // Init-time election that will have to change for + // support of dynamic changes to the pipelining enable flag. mHttpHighWater = HTTP_PIPE_REQUESTS_HIGH_WATER; mHttpLowWater = HTTP_PIPE_REQUESTS_LOW_WATER; } -- cgit v1.2.3 From 5dffe16aef7c6687a2ed47c5324f4365ff9d84df Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Mon, 11 Aug 2014 14:38:47 -0400 Subject: Add 'HttpRangeRequestsDisable' debug setting to inhibit use of 'Range:' header. Intended for users with bad networking gear or twitchy ISPs, if set to True, forces plain GET requests to asset servers for textures and meshes. This change kicked off a slight refactor in the mesh repository code which made it resilient against unexpected 200's and responses not covering the requested start range. There's still too much data copying in the Mesh code (always has been). Would love to fix that and get rid of the monolithic temp buffer. Cleaned up white space damage caused by unnamed linden who likes to drag his magical editor through code. --- indra/newview/app_settings/settings.xml | 11 + indra/newview/llmeshrepository.cpp | 364 ++++++++++++++++++-------------- indra/newview/lltexturefetch.cpp | 43 ++-- 3 files changed, 243 insertions(+), 175 deletions(-) (limited to 'indra/newview') diff --git a/indra/newview/app_settings/settings.xml b/indra/newview/app_settings/settings.xml index 410473c54f..50f56cf6ff 100755 --- a/indra/newview/app_settings/settings.xml +++ b/indra/newview/app_settings/settings.xml @@ -4467,6 +4467,17 @@ Value 1 + HttpRangeRequestsDisable + + Comment + If true, viewer will not issued range GET requests for meshes and textures. + Persist + 1 + Type + Boolean + Value + 0 + IMShowTimestamps Comment diff --git a/indra/newview/llmeshrepository.cpp b/indra/newview/llmeshrepository.cpp index fc69ecfae9..6477389d4c 100755 --- a/indra/newview/llmeshrepository.cpp +++ b/indra/newview/llmeshrepository.cpp @@ -1,4 +1,3 @@ - /** * @file llmeshrepository.cpp * @brief Mesh repository implementation. @@ -521,11 +520,13 @@ class LLMeshHandlerBase : public LLCore::HttpHandler { public: LOG_CLASS(LLMeshHandlerBase); - LLMeshHandlerBase() + LLMeshHandlerBase(U32 offset, U32 requested_bytes) : LLCore::HttpHandler(), mMeshParams(), mProcessed(false), - mHttpHandle(LLCORE_HTTP_HANDLE_INVALID) + mHttpHandle(LLCORE_HTTP_HANDLE_INVALID), + mOffset(offset), + mRequestedBytes(requested_bytes) {} virtual ~LLMeshHandlerBase() @@ -537,13 +538,15 @@ protected: public: virtual void onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * response); - virtual void processData(LLCore::BufferArray * body, U8 * data, S32 data_size) = 0; + virtual void processData(LLCore::BufferArray * body, S32 body_offset, U8 * data, S32 data_size) = 0; virtual void processFailure(LLCore::HttpStatus status) = 0; public: LLVolumeParams mMeshParams; bool mProcessed; - LLCore::HttpHandle mHttpHandle; + LLCore::HttpHandle mHttpHandle; + U32 mOffset; + U32 mRequestedBytes; }; @@ -554,8 +557,8 @@ class LLMeshHeaderHandler : public LLMeshHandlerBase { public: LOG_CLASS(LLMeshHeaderHandler); - LLMeshHeaderHandler(const LLVolumeParams & mesh_params) - : LLMeshHandlerBase() + LLMeshHeaderHandler(const LLVolumeParams & mesh_params, U32 offset, U32 requested_bytes) + : LLMeshHandlerBase(offset, requested_bytes) { mMeshParams = mesh_params; LLMeshRepoThread::incActiveHeaderRequests(); @@ -567,7 +570,7 @@ protected: void operator=(const LLMeshHeaderHandler &); // Not defined public: - virtual void processData(LLCore::BufferArray * body, U8 * data, S32 data_size); + virtual void processData(LLCore::BufferArray * body, S32 body_offset, U8 * data, S32 data_size); virtual void processFailure(LLCore::HttpStatus status); }; @@ -576,17 +579,16 @@ public: // // Thread: repo class LLMeshLODHandler : public LLMeshHandlerBase - { +{ public: + LOG_CLASS(LLMeshLODHandler); LLMeshLODHandler(const LLVolumeParams & mesh_params, S32 lod, U32 offset, U32 requested_bytes) - : LLMeshHandlerBase(), - mLOD(lod), - mRequestedBytes(requested_bytes), - mOffset(offset) + : LLMeshHandlerBase(offset, requested_bytes), + mLOD(lod) { - mMeshParams = mesh_params; - LLMeshRepoThread::incActiveLODRequests(); - } + mMeshParams = mesh_params; + LLMeshRepoThread::incActiveLODRequests(); + } virtual ~LLMeshLODHandler(); protected: @@ -594,13 +596,11 @@ protected: void operator=(const LLMeshLODHandler &); // Not defined public: - virtual void processData(LLCore::BufferArray * body, U8 * data, S32 data_size); + virtual void processData(LLCore::BufferArray * body, S32 body_offset, U8 * data, S32 data_size); virtual void processFailure(LLCore::HttpStatus status); public: S32 mLOD; - U32 mRequestedBytes; - U32 mOffset; }; @@ -608,14 +608,12 @@ public: // // Thread: repo class LLMeshSkinInfoHandler : public LLMeshHandlerBase - { +{ public: LOG_CLASS(LLMeshSkinInfoHandler); - LLMeshSkinInfoHandler(const LLUUID& id, U32 offset, U32 size) - : LLMeshHandlerBase(), - mMeshID(id), - mRequestedBytes(size), - mOffset(offset) + LLMeshSkinInfoHandler(const LLUUID& id, U32 offset, U32 requested_bytes) + : LLMeshHandlerBase(offset, requested_bytes), + mMeshID(id) {} virtual ~LLMeshSkinInfoHandler(); @@ -624,13 +622,11 @@ protected: void operator=(const LLMeshSkinInfoHandler &); // Not defined public: - virtual void processData(LLCore::BufferArray * body, U8 * data, S32 data_size); + virtual void processData(LLCore::BufferArray * body, S32 body_offset, U8 * data, S32 data_size); virtual void processFailure(LLCore::HttpStatus status); public: LLUUID mMeshID; - U32 mRequestedBytes; - U32 mOffset; }; @@ -638,14 +634,12 @@ public: // // Thread: repo class LLMeshDecompositionHandler : public LLMeshHandlerBase - { +{ public: LOG_CLASS(LLMeshDecompositionHandler); - LLMeshDecompositionHandler(const LLUUID& id, U32 offset, U32 size) - : LLMeshHandlerBase(), - mMeshID(id), - mRequestedBytes(size), - mOffset(offset) + LLMeshDecompositionHandler(const LLUUID& id, U32 offset, U32 requested_bytes) + : LLMeshHandlerBase(offset, requested_bytes), + mMeshID(id) {} virtual ~LLMeshDecompositionHandler(); @@ -654,13 +648,11 @@ protected: void operator=(const LLMeshDecompositionHandler &); // Not defined public: - virtual void processData(LLCore::BufferArray * body, U8 * data, S32 data_size); + virtual void processData(LLCore::BufferArray * body, S32 body_offset, U8 * data, S32 data_size); virtual void processFailure(LLCore::HttpStatus status); public: LLUUID mMeshID; - U32 mRequestedBytes; - U32 mOffset; }; @@ -668,14 +660,12 @@ public: // // Thread: repo class LLMeshPhysicsShapeHandler : public LLMeshHandlerBase - { +{ public: LOG_CLASS(LLMeshPhysicsShapeHandler); - LLMeshPhysicsShapeHandler(const LLUUID& id, U32 offset, U32 size) - : LLMeshHandlerBase(), - mMeshID(id), - mRequestedBytes(size), - mOffset(offset) + LLMeshPhysicsShapeHandler(const LLUUID& id, U32 offset, U32 requested_bytes) + : LLMeshHandlerBase(offset, requested_bytes), + mMeshID(id) {} virtual ~LLMeshPhysicsShapeHandler(); @@ -684,13 +674,11 @@ protected: void operator=(const LLMeshPhysicsShapeHandler &); // Not defined public: - virtual void processData(LLCore::BufferArray * body, U8 * data, S32 data_size); + virtual void processData(LLCore::BufferArray * body, S32 body_offset, U8 * data, S32 data_size); virtual void processFailure(LLCore::HttpStatus status); public: LLUUID mMeshID; - U32 mRequestedBytes; - U32 mOffset; }; @@ -716,8 +704,8 @@ void log_upload_error(LLCore::HttpStatus status, const LLSD& content, LL_WARNS(LOG_MESH) << "error: " << err << LL_ENDL; LL_WARNS(LOG_MESH) << " mesh upload failed, stage '" << stage << "', error '" << err["error"].asString() - << "', message '" << err["message"].asString() - << "', id '" << err["identifier"].asString() + << "', message '" << err["message"].asString() + << "', id '" << err["identifier"].asString() << "'" << LL_ENDL; if (err.has("errors")) { @@ -779,7 +767,7 @@ LLMeshRepoThread::LLMeshRepoThread() LLMeshRepoThread::~LLMeshRepoThread() - { +{ LL_INFOS(LOG_MESH) << "Small GETs issued: " << LLMeshRepository::sHTTPRequestCount << ", Large GETs issued: " << LLMeshRepository::sHTTPLargeRequestCount << ", Max Lock Holdoffs: " << LLMeshRepository::sMaxLockHoldoffs @@ -790,23 +778,23 @@ LLMeshRepoThread::~LLMeshRepoThread() ++iter) { delete *iter; - } + } mHttpRequestSet.clear(); if (mHttpHeaders) - { + { mHttpHeaders->release(); mHttpHeaders = NULL; - } + } if (mHttpOptions) - { + { mHttpOptions->release(); mHttpOptions = NULL; - } + } if (mHttpLargeOptions) -{ + { mHttpLargeOptions->release(); mHttpLargeOptions = NULL; -} + } delete mHttpRequest; mHttpRequest = NULL; delete mMutex; @@ -1137,6 +1125,9 @@ LLCore::HttpHandle LLMeshRepoThread::getByteRange(const std::string & url, int c size_t offset, size_t len, LLCore::HttpHandler * handler) { + // Also used in lltexturefetch.cpp + static LLCachedControl disable_range_req(gSavedSettings, "HttpRangeRequestsDisable", false); + LLCore::HttpHandle handle(LLCORE_HTTP_HANDLE_INVALID); if (len < LARGE_MESH_FETCH_THRESHOLD) @@ -1146,8 +1137,8 @@ LLCore::HttpHandle LLMeshRepoThread::getByteRange(const std::string & url, int c : mHttpLegacyPolicyClass), mHttpPriority, url, - offset, - len, + (disable_range_req ? size_t(0) : offset), + (disable_range_req ? size_t(0) : len), mHttpOptions, mHttpHeaders, handler); @@ -1161,8 +1152,8 @@ LLCore::HttpHandle LLMeshRepoThread::getByteRange(const std::string & url, int c handle = mHttpRequest->requestGetByteRange(mHttpLargePolicyClass, mHttpPriority, url, - offset, - len, + (disable_range_req ? size_t(0) : offset), + (disable_range_req ? size_t(0) : len), mHttpLargeOptions, mHttpHeaders, handler); @@ -1532,7 +1523,7 @@ bool LLMeshRepoThread::fetchMeshHeader(const LLVolumeParams& mesh_params) //within the first 4KB //NOTE -- this will break of headers ever exceed 4KB - LLMeshHeaderHandler * handler = new LLMeshHeaderHandler(mesh_params); + LLMeshHeaderHandler * handler = new LLMeshHeaderHandler(mesh_params, 0, MESH_HEADER_SIZE); LLCore::HttpHandle handle = getByteRange(http_url, cap_version, 0, MESH_HEADER_SIZE, handler); if (LLCORE_HTTP_HANDLE_INVALID == handle) { @@ -2276,7 +2267,7 @@ void LLMeshUploadThread::doWholeModelUpload() mHttpRequest->update(0); while (! LLApp::isQuitting() && ! finished() && ! isDiscarded()) - { + { ms_sleep(sleep_time); sleep_time = llmin(250U, sleep_time + sleep_time); mHttpRequest->update(0); @@ -2292,7 +2283,7 @@ void LLMeshUploadThread::doWholeModelUpload() } } } - } +} void LLMeshUploadThread::requestWholeModelFee() { @@ -2323,7 +2314,7 @@ void LLMeshUploadThread::requestWholeModelFee() LL_WARNS(LOG_MESH) << "Couldn't issue request for model fee. Reason: " << mHttpStatus.toString() << " (" << mHttpStatus.toTerseString() << ")" << LL_ENDL; - } + } else { U32 sleep_time(10); @@ -2340,7 +2331,7 @@ void LLMeshUploadThread::requestWholeModelFee() LL_DEBUGS(LOG_MESH) << "Mesh fee query operation discarded." << LL_ENDL; } } - } +} // Does completion duty for both fee queries and actual uploads. @@ -2393,12 +2384,12 @@ void LLMeshUploadThread::onCompleted(LLCore::HttpHandle handle, LLCore::HttpResp { LLCore::BufferArrayStream bas(ba); LLSDSerialize::fromXML(body, bas); -} + } } dump_llsd_to_file(body, make_dump_name("whole_model_upload_response_", dump_num)); if (body["state"].asString() == "complete") -{ + { // requested "mesh" asset type isn't actually the type // of the resultant object, fix it up here. mModelData["asset_type"] = "object"; @@ -2451,18 +2442,18 @@ void LLMeshUploadThread::onCompleted(LLCore::HttpHandle handle, LLCore::HttpResp body = llsd_from_file("fake_upload_error.xml"); } else - { + { LLCore::BufferArray * ba(response->getBody()); if (ba && ba->size()) - { + { LLCore::BufferArrayStream bas(ba); LLSDSerialize::fromXML(body, bas); - } - } + } + } dump_llsd_to_file(body, make_dump_name("whole_model_fee_response_", dump_num)); if (body["state"].asString() == "upload") - { + { mWholeModelUploadURL = body["uploader"].asString(); if (observer) @@ -2548,18 +2539,18 @@ void LLMeshRepoThread::notifyLoadedMeshes() skin_info_q.swap(mSkinInfoQ); } if (! mDecompositionQ.empty()) - { + { decomp_q.swap(mDecompositionQ); - } + } mMutex->unlock(); // Process the elements free of the lock while (! skin_info_q.empty()) - { + { gMeshRepo.notifySkinInfoReceived(skin_info_q.front()); skin_info_q.pop_front(); - } + } while (! decomp_q.empty()) { @@ -2681,35 +2672,78 @@ void LLMeshHandlerBase::onCompleted(LLCore::HttpHandle handle, LLCore::HttpRespo // rather than partial) and 416 (request completely unsatisfyable). // Always been exposed to these but are less likely here where // speculative loads aren't done. - static const LLCore::HttpStatus par_status(HTTP_PARTIAL_CONTENT); + LLCore::BufferArray * body(response->getBody()); + S32 body_offset(0); + U8 * data(NULL); + S32 data_size(body ? body->size() : 0); - if (par_status != status) + if (data_size > 0) { - LL_WARNS_ONCE(LOG_MESH) << "Non-206 successful status received for fetch: " - << status.toTerseString() << LL_ENDL; - } + static const LLCore::HttpStatus par_status(HTTP_PARTIAL_CONTENT); + + unsigned int offset(0), length(0), full_length(0); + + if (par_status == status) + { + // 216 case + response->getRange(&offset, &length, &full_length); + if (! offset && ! length) + { + // This is the case where we receive a 206 status but + // there wasn't a useful Content-Range header in the response. + // This could be because it was badly formatted but is more + // likely due to capabilities services which scrub headers + // from responses. Assume we got what we asked for...` + // length = data_size; + offset = mOffset; + } + } + else + { + // 200 case, typically + offset = 0; + } - LLCore::BufferArray * body(response->getBody()); - S32 data_size(body ? body->size() : 0); - U8 * data(NULL); + // *DEBUG: To test validation below + // offset += 1; - if (data_size > 0) - { + // Validate that what we think we received is consistent with + // what we've asked for. I.e. first byte we wanted lies somewhere + // in the response. + if (offset > mOffset + || (offset + data_size) <= mOffset + || (mOffset - offset) >= data_size) + { + // No overlap with requested range. Fail request with + // suitable error. Shouldn't happen unless server/cache/ISP + // is doing something awful. + LL_WARNS(LOG_MESH) << "Mesh response (bytes [" + << offset << ".." << (offset + length - 1) + << "]) didn't overlap with request's origin (bytes [" + << mOffset << ".." << (mOffset + mRequestedBytes - 1) + << "])." << LL_ENDL; + processFailure(LLCore::HttpStatus(LLCore::HttpStatus::LLCORE, LLCore::HE_INV_CONTENT_RANGE_HDR)); + ++LLMeshRepository::sHTTPErrorCount; + goto common_exit; + } + // *TODO: Try to get rid of data copying and add interfaces // that support BufferArray directly. Introduce a two-phase // handler, optional first that takes a body, fallback second // that requires a temporary allocation and data copy. - data = new U8[data_size]; - body->read(0, (char *) data, data_size); + body_offset = mOffset - offset; + data = new U8[data_size - body_offset]; + body->read(body_offset, (char *) data, data_size - body_offset); LLMeshRepository::sBytesReceived += data_size; } - processData(body, data, data_size); + processData(body, body_offset, data, data_size - body_offset); delete [] data; } // Release handler +common_exit: gMeshRepo.mThread->mHttpRequestSet.erase(this); delete this; // Must be last statement } @@ -2744,9 +2778,10 @@ void LLMeshHeaderHandler::processFailure(LLCore::HttpStatus status) { gMeshRepo.mThread->mUnavailableQ.push(LLMeshRepoThread::LODRequest(mMeshParams, i)); } - } +} -void LLMeshHeaderHandler::processData(LLCore::BufferArray * body, U8 * data, S32 data_size) +void LLMeshHeaderHandler::processData(LLCore::BufferArray * /* body */, S32 /* body_offset */, + U8 * data, S32 data_size) { LLUUID mesh_id = mMeshParams.getSculptID(); bool success = (! MESH_HEADER_PROCESS_FAILED) && gMeshRepo.mThread->headerReceived(mMeshParams, data, data_size); @@ -2761,12 +2796,12 @@ void LLMeshHeaderHandler::processData(LLCore::BufferArray * body, U8 * data, S32 // Can't get the header so none of the LODs will be available LLMutexLock lock(gMeshRepo.mThread->mMutex); for (int i(0); i < 4; ++i) - { + { gMeshRepo.mThread->mUnavailableQ.push(LLMeshRepoThread::LODRequest(mMeshParams, i)); - } } + } else if (data && data_size > 0) - { + { // header was successfully retrieved from sim, cache in vfs LLSD header = gMeshRepo.mThread->mMeshHeader[mesh_id]; @@ -2779,11 +2814,11 @@ void LLMeshHeaderHandler::processData(LLCore::BufferArray * body, U8 * data, S32 S32 lod_bytes = 0; for (U32 i = 0; i < LLModel::LOD_PHYSICS; ++i) - { + { // figure out how many bytes we'll need to reserve in the file const std::string & lod_name = header_lod[i]; lod_bytes = llmax(lod_bytes, header[lod_name]["offset"].asInteger()+header[lod_name]["size"].asInteger()); - } + } // just in case skin info or decomposition is at the end of the file (which it shouldn't be) lod_bytes = llmax(lod_bytes, header["skin"]["offset"].asInteger() + header["skin"]["size"].asInteger()); @@ -2799,7 +2834,7 @@ void LLMeshHeaderHandler::processData(LLCore::BufferArray * body, U8 * data, S32 LLVFile file(gVFS, mesh_id, LLAssetType::AT_MESH, LLVFile::WRITE); if (file.getMaxSize() >= bytes || file.setMaxSize(bytes)) - { + { LLMeshRepository::sCacheBytesWritten += data_size; ++LLMeshRepository::sCacheWrites; @@ -2810,19 +2845,19 @@ void LLMeshHeaderHandler::processData(LLCore::BufferArray * body, U8 * data, S32 memset(block, 0, sizeof(block)); while (bytes-file.tell() > sizeof(block)) - { + { file.write(block, sizeof(block)); - } + } S32 remaining = bytes-file.tell(); if (remaining > 0) - { + { file.write(block, remaining); } } } } - } +} LLMeshLODHandler::~LLMeshLODHandler() { @@ -2848,8 +2883,9 @@ void LLMeshLODHandler::processFailure(LLCore::HttpStatus status) gMeshRepo.mThread->mUnavailableQ.push(LLMeshRepoThread::LODRequest(mMeshParams, mLOD)); } -void LLMeshLODHandler::processData(LLCore::BufferArray * body, U8 * data, S32 data_size) - { +void LLMeshLODHandler::processData(LLCore::BufferArray * /* body */, S32 /* body_offset */, + U8 * data, S32 data_size) +{ if ((! MESH_LOD_PROCESS_FAILED) && gMeshRepo.mThread->lodReceived(mMeshParams, mLOD, data, data_size)) { //good fetch from sim, write to VFS for caching @@ -2865,7 +2901,7 @@ void LLMeshLODHandler::processData(LLCore::BufferArray * body, U8 * data, S32 da LLMeshRepository::sCacheBytesWritten += size; ++LLMeshRepository::sCacheWrites; } - } + } else { LL_WARNS(LOG_MESH) << "Error during mesh LOD processing. ID: " << mMeshParams.getSculptID() @@ -2877,12 +2913,12 @@ void LLMeshLODHandler::processData(LLCore::BufferArray * body, U8 * data, S32 da } LLMeshSkinInfoHandler::~LLMeshSkinInfoHandler() - { - llassert(mProcessed); - } +{ + llassert(mProcessed); +} void LLMeshSkinInfoHandler::processFailure(LLCore::HttpStatus status) - { +{ LL_WARNS(LOG_MESH) << "Error during mesh skin info handling. ID: " << mMeshID << ", Reason: " << status.toString() << " (" << status.toTerseString() << "). Not retrying." @@ -2890,10 +2926,11 @@ void LLMeshSkinInfoHandler::processFailure(LLCore::HttpStatus status) // *TODO: Mark mesh unavailable on error. For now, simply leave // request unfulfilled rather than retry forever. - } +} -void LLMeshSkinInfoHandler::processData(LLCore::BufferArray * body, U8 * data, S32 data_size) - { +void LLMeshSkinInfoHandler::processData(LLCore::BufferArray * /* body */, S32 /* body_offset */, + U8 * data, S32 data_size) +{ if ((! MESH_SKIN_INFO_PROCESS_FAILED) && gMeshRepo.mThread->skinInfoReceived(mMeshID, data, data_size)) { //good fetch from sim, write to VFS for caching @@ -2921,20 +2958,21 @@ void LLMeshSkinInfoHandler::processData(LLCore::BufferArray * body, U8 * data, S LLMeshDecompositionHandler::~LLMeshDecompositionHandler() { - llassert(mProcessed); + llassert(mProcessed); } void LLMeshDecompositionHandler::processFailure(LLCore::HttpStatus status) - { +{ LL_WARNS(LOG_MESH) << "Error during mesh decomposition handling. ID: " << mMeshID << ", Reason: " << status.toString() << " (" << status.toTerseString() << "). Not retrying." << LL_ENDL; // *TODO: Mark mesh unavailable on error. For now, simply leave // request unfulfilled rather than retry forever. - } +} -void LLMeshDecompositionHandler::processData(LLCore::BufferArray * body, U8 * data, S32 data_size) +void LLMeshDecompositionHandler::processData(LLCore::BufferArray * /* body */, S32 /* body_offset */, + U8 * data, S32 data_size) { if ((! MESH_DECOMP_PROCESS_FAILED) && gMeshRepo.mThread->decompositionReceived(mMeshID, data, data_size)) { @@ -2951,34 +2989,35 @@ void LLMeshDecompositionHandler::processData(LLCore::BufferArray * body, U8 * da file.seek(offset); file.write(data, size); } - } - else - { + } + else + { LL_WARNS(LOG_MESH) << "Error during mesh decomposition processing. ID: " << mMeshID << ", Unknown reason. Not retrying." << LL_ENDL; // *TODO: Mark mesh unavailable on error - } } +} LLMeshPhysicsShapeHandler::~LLMeshPhysicsShapeHandler() - { - llassert(mProcessed); - } +{ + llassert(mProcessed); +} void LLMeshPhysicsShapeHandler::processFailure(LLCore::HttpStatus status) - { +{ LL_WARNS(LOG_MESH) << "Error during mesh physics shape handling. ID: " << mMeshID << ", Reason: " << status.toString() << " (" << status.toTerseString() << "). Not retrying." << LL_ENDL; // *TODO: Mark mesh unavailable on error - } +} -void LLMeshPhysicsShapeHandler::processData(LLCore::BufferArray * body, U8 * data, S32 data_size) - { +void LLMeshPhysicsShapeHandler::processData(LLCore::BufferArray * /* body */, S32 /* body_offset */, + U8 * data, S32 data_size) +{ if ((! MESH_PHYS_SHAPE_PROCESS_FAILED) && gMeshRepo.mThread->physicsShapeReceived(mMeshID, data, data_size)) - { + { // good fetch from sim, write to VFS for caching LLVFile file(gVFS, mMeshID, LLAssetType::AT_MESH, LLVFile::WRITE); @@ -2986,13 +3025,13 @@ void LLMeshPhysicsShapeHandler::processData(LLCore::BufferArray * body, U8 * dat S32 size = mRequestedBytes; if (file.getSize() >= offset+size) - { + { LLMeshRepository::sCacheBytesWritten += size; ++LLMeshRepository::sCacheWrites; file.seek(offset); file.write(data, size); - } } + } else { LL_WARNS(LOG_MESH) << "Error during mesh physics shape processing. ID: " << mMeshID @@ -3192,7 +3231,7 @@ void LLMeshRepository::notifyLoadedMeshes() if (1 == mGetMeshVersion) { // Legacy GetMesh operation with high connection concurrency - LLMeshRepoThread::sMaxConcurrentRequests = gSavedSettings.getU32("MeshMaxConcurrentRequests"); + LLMeshRepoThread::sMaxConcurrentRequests = gSavedSettings.getU32("MeshMaxConcurrentRequests"); LLMeshRepoThread::sRequestHighWater = llclamp(2 * S32(LLMeshRepoThread::sMaxConcurrentRequests), REQUEST_HIGH_WATER_MIN, REQUEST_HIGH_WATER_MAX); @@ -3311,18 +3350,18 @@ void LLMeshRepository::notifyLoadedMeshes() // If we can't get the locks, skip and pick this up later. ++hold_offs; sMaxLockHoldoffs = llmax(sMaxLockHoldoffs, hold_offs); - return; - } + return; + } hold_offs = 0; if (gAgent.getRegion()) { // Update capability urls - static std::string region_name("never name a region this"); + static std::string region_name("never name a region this"); - if (gAgent.getRegion()->getName() != region_name && gAgent.getRegion()->capabilitiesReceived()) - { - region_name = gAgent.getRegion()->getName(); + if (gAgent.getRegion()->getName() != region_name && gAgent.getRegion()->capabilitiesReceived()) + { + region_name = gAgent.getRegion()->getName(); const bool use_v1(gSavedSettings.getBOOL("MeshUseGetMesh1")); const std::string mesh1(gAgent.getRegion()->getCapability("GetMesh")); const std::string mesh2(gAgent.getRegion()->getCapability("GetMesh2")); @@ -3333,8 +3372,8 @@ void LLMeshRepository::notifyLoadedMeshes() << ", GetMesh: " << mesh1 << ", using version: " << mGetMeshVersion << LL_ENDL; + } } - } //popup queued error messages from background threads while (!mUploadErrorQ.empty()) @@ -3349,46 +3388,46 @@ void LLMeshRepository::notifyLoadedMeshes() S32 push_count = LLMeshRepoThread::sRequestHighWater - active_count; if (mPendingRequests.size() > push_count) - { + { // More requests than the high-water limit allows so // sort and forward the most important. - //calculate "score" for pending requests + //calculate "score" for pending requests - //create score map - std::map score_map; + //create score map + std::map score_map; - for (U32 i = 0; i < 4; ++i) - { - for (mesh_load_map::iterator iter = mLoadingMeshes[i].begin(); iter != mLoadingMeshes[i].end(); ++iter) + for (U32 i = 0; i < 4; ++i) { - F32 max_score = 0.f; - for (std::set::iterator obj_iter = iter->second.begin(); obj_iter != iter->second.end(); ++obj_iter) + for (mesh_load_map::iterator iter = mLoadingMeshes[i].begin(); iter != mLoadingMeshes[i].end(); ++iter) { - LLViewerObject* object = gObjectList.findObject(*obj_iter); - - if (object) + F32 max_score = 0.f; + for (std::set::iterator obj_iter = iter->second.begin(); obj_iter != iter->second.end(); ++obj_iter) { - LLDrawable* drawable = object->mDrawable; - if (drawable) + LLViewerObject* object = gObjectList.findObject(*obj_iter); + + if (object) { - F32 cur_score = drawable->getRadius()/llmax(drawable->mDistanceWRTCamera, 1.f); - max_score = llmax(max_score, cur_score); + LLDrawable* drawable = object->mDrawable; + if (drawable) + { + F32 cur_score = drawable->getRadius()/llmax(drawable->mDistanceWRTCamera, 1.f); + max_score = llmax(max_score, cur_score); + } } } - } - score_map[iter->first.getSculptID()] = max_score; + score_map[iter->first.getSculptID()] = max_score; + } } - } - //set "score" for pending requests - for (std::vector::iterator iter = mPendingRequests.begin(); iter != mPendingRequests.end(); ++iter) - { - iter->mScore = score_map[iter->mMeshParams.getSculptID()]; - } + //set "score" for pending requests + for (std::vector::iterator iter = mPendingRequests.begin(); iter != mPendingRequests.end(); ++iter) + { + iter->mScore = score_map[iter->mMeshParams.getSculptID()]; + } - //sort by "score" + //sort by "score" std::partial_sort(mPendingRequests.begin(), mPendingRequests.begin() + push_count, mPendingRequests.end(), LLMeshRepoThread::CompareScoreGreater()); } @@ -3599,7 +3638,6 @@ void LLMeshRepository::fetchPhysicsShape(const LLUUID& mesh_id) } } } - } LLModel::Decomposition* LLMeshRepository::getDecomposition(const LLUUID& mesh_id) diff --git a/indra/newview/lltexturefetch.cpp b/indra/newview/lltexturefetch.cpp index ab7df02100..4008a6948d 100755 --- a/indra/newview/lltexturefetch.cpp +++ b/indra/newview/lltexturefetch.cpp @@ -1327,7 +1327,7 @@ bool LLTextureFetchWorker::doWork(S32 param) } } - static LLCachedControl use_http(gSavedSettings,"ImagePipelineUseHTTP", true); + static LLCachedControl use_http(gSavedSettings, "ImagePipelineUseHTTP", true); // if (mHost != LLHost::invalid) get_url = false; if ( use_http && mCanUseHTTP && mUrl.empty())//get http url. @@ -1472,6 +1472,9 @@ bool LLTextureFetchWorker::doWork(S32 param) if (mState == SEND_HTTP_REQ) { + // Also used in llmeshrepository + static LLCachedControl disable_range_req(gSavedSettings, "HttpRangeRequestsDisable", false); + if (! mCanUseHTTP) { releaseHttpSemaphore(); @@ -1553,16 +1556,32 @@ bool LLTextureFetchWorker::doWork(S32 param) // Will call callbackHttpGet when curl request completes // Only server bake images use the returned headers currently, for getting retry-after field. LLCore::HttpOptions *options = (mFTType == FTT_SERVER_BAKE) ? mFetcher->mHttpOptionsWithHeaders: mFetcher->mHttpOptions; - mHttpHandle = mFetcher->mHttpRequest->requestGetByteRange(mHttpPolicyClass, - mWorkPriority, - mUrl, - mRequestedOffset, - (mRequestedOffset + mRequestedSize) > HTTP_REQUESTS_RANGE_END_MAX - ? 0 - : mRequestedSize, - options, - mFetcher->mHttpHeaders, - this); + if (disable_range_req) + { + // 'Range:' requests may be disabled in which case all HTTP + // texture fetches result in full fetches. This can be used + // by people with questionable ISPs or networking gear that + // doesn't handle these well. + mHttpHandle = mFetcher->mHttpRequest->requestGet(mHttpPolicyClass, + mWorkPriority, + mUrl, + options, + mFetcher->mHttpHeaders, + this); + } + else + { + mHttpHandle = mFetcher->mHttpRequest->requestGetByteRange(mHttpPolicyClass, + mWorkPriority, + mUrl, + mRequestedOffset, + (mRequestedOffset + mRequestedSize) > HTTP_REQUESTS_RANGE_END_MAX + ? 0 + : mRequestedSize, + options, + mFetcher->mHttpHeaders, + this); + } if (LLCORE_HTTP_HANDLE_INVALID == mHttpHandle) { LLCore::HttpStatus status(mFetcher->mHttpRequest->getStatus()); @@ -1782,7 +1801,7 @@ bool LLTextureFetchWorker::doWork(S32 param) if (mState == DECODE_IMAGE) { - static LLCachedControl textures_decode_disabled(gSavedSettings,"TextureDecodeDisabled", false); + static LLCachedControl textures_decode_disabled(gSavedSettings, "TextureDecodeDisabled", false); setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); // Set priority first since Responder may change it if (textures_decode_disabled) -- cgit v1.2.3 From e79a88c8ccfadcd260892000d4dec2ae921b26de Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Tue, 12 Aug 2014 18:21:26 -0400 Subject: Better support for dynamic option changes in llcorehttp. Libcurl has some problems disabling pipelining on a multi handle with outstanding requests so build a more conservative system that allows requests to drain before setting curl multi options. Would rather not have this but it is significantly safer. "HttpPipelining" debug setting is now fully dynamic. Connection limits can also be made dynamic in the near future. Upped the default connection count back to 8 for now but will revisit this in the tuning phase. It might be time to combine mesh and textures into a single asset class. For normal server operations that would be a clear path, but for server under load, the current scheme may be better. Minor cleanup in logging to elminate some redundant strings. Might add some more tracing to the stall logic 'just in case'. --- indra/newview/app_settings/settings.xml | 4 +- indra/newview/llappcorehttp.cpp | 174 ++++++++++++++++++-------------- indra/newview/llappcorehttp.h | 1 + indra/newview/lltexturefetch.cpp | 55 ++++++---- indra/newview/lltexturefetch.h | 10 +- 5 files changed, 145 insertions(+), 99 deletions(-) (limited to 'indra/newview') diff --git a/indra/newview/app_settings/settings.xml b/indra/newview/app_settings/settings.xml index 50f56cf6ff..0607579a08 100755 --- a/indra/newview/app_settings/settings.xml +++ b/indra/newview/app_settings/settings.xml @@ -4459,7 +4459,7 @@ HttpPipelining Comment - If true, viewer will pipeline HTTP requests to servers. Static. + If true, viewer will pipeline HTTP requests to servers. Persist 1 Type @@ -4470,7 +4470,7 @@ HttpRangeRequestsDisable Comment - If true, viewer will not issued range GET requests for meshes and textures. + If true, viewer will not issued range GET requests for meshes and textures. May resolve problems with certain ISPs and networking gear. Persist 1 Type diff --git a/indra/newview/llappcorehttp.cpp b/indra/newview/llappcorehttp.cpp index d097f18d61..464e60948a 100755 --- a/indra/newview/llappcorehttp.cpp +++ b/indra/newview/llappcorehttp.cpp @@ -60,7 +60,7 @@ static const struct "other" }, { // AP_TEXTURE - 4, 1, 12, 0, true, + 8, 1, 12, 0, true, "TextureFetchConcurrency", "texture fetch" }, @@ -70,7 +70,7 @@ static const struct "mesh fetch" }, { // AP_MESH2 - 4, 1, 32, 100, true, + 8, 1, 32, 100, true, "Mesh2MaxConcurrentRequests", "mesh2 fetch" }, @@ -126,14 +126,6 @@ void LLAppCoreHttp::init() << LL_ENDL; } - // Global pipelining preference from settings - static const std::string http_pipelining("HttpPipelining"); - if (gSavedSettings.controlExists(http_pipelining)) - { - // Default to true if absent. - mPipelined = gSavedSettings.getBOOL(http_pipelining); - } - // Point to our certs or SSH/https: will fail on connect status = LLCore::HttpRequest::setStaticPolicyOption(LLCore::HttpRequest::PO_CA_FILE, LLCore::HttpRequest::GLOBAL_POLICY_ID, @@ -210,12 +202,27 @@ void LLAppCoreHttp::init() << LL_ENDL; } - // *NOTE: Pipelining isn't dynamic yet. When it is, add a global - // signal for the setting here. - + // Signal for global pipelining preference from settings + static const std::string http_pipelining("HttpPipelining"); + if (gSavedSettings.controlExists(http_pipelining)) + { + LLPointer cntrl_ptr = gSavedSettings.getControl(http_pipelining); + if (cntrl_ptr.isNull()) + { + LL_WARNS("Init") << "Unable to set signal on global setting '" << http_pipelining + << "'" << LL_ENDL; + } + else + { + mPipelinedSignal = cntrl_ptr->getCommitSignal()->connect(boost::bind(&setting_changed)); + } + } + // Register signals for settings and state changes for (int i(0); i < LL_ARRAY_SIZE(init_data); ++i) { + const EAppPolicy app_policy(static_cast(i)); + if (! init_data[i].mKey.empty() && gSavedSettings.controlExists(init_data[i].mKey)) { LLPointer cntrl_ptr = gSavedSettings.getControl(init_data[i].mKey); @@ -226,7 +233,7 @@ void LLAppCoreHttp::init() } else { - mHttpClasses[i].mSettingsSignal = cntrl_ptr->getCommitSignal()->connect(boost::bind(&setting_changed)); + mHttpClasses[app_policy].mSettingsSignal = cntrl_ptr->getCommitSignal()->connect(boost::bind(&setting_changed)); } } } @@ -282,6 +289,7 @@ void LLAppCoreHttp::cleanup() { mHttpClasses[i].mSettingsSignal.disconnect(); } + mPipelinedSignal.disconnect(); delete mRequest; mRequest = NULL; @@ -299,6 +307,20 @@ void LLAppCoreHttp::cleanup() void LLAppCoreHttp::refreshSettings(bool initial) { LLCore::HttpStatus status; + + // Global pipelining setting + bool pipeline_changed(false); + static const std::string http_pipelining("HttpPipelining"); + if (gSavedSettings.controlExists(http_pipelining)) + { + // Default to true (in ctor) if absent. + bool pipelined(gSavedSettings.getBOOL(http_pipelining)); + if (pipelined != mPipelined) + { + mPipelined = pipelined; + pipeline_changed = true; + } + } for (int i(0); i < LL_ARRAY_SIZE(init_data); ++i) { @@ -323,33 +345,42 @@ void LLAppCoreHttp::refreshSettings(bool initial) } } - mHttpClasses[app_policy].mPipelined = false; - if (mPipelined && init_data[i].mPipelined) + } + + // Init- or run-time settings. Must use the queued request API. + + // Pipelining changes + if (initial || pipeline_changed) + { + const bool to_pipeline(mPipelined && init_data[i].mPipelined); + if (to_pipeline != mHttpClasses[app_policy].mPipelined) { - // Pipelining election is currently static (init-time). - // Making it dynamic isn't too hard in the SL code but verifying - // that libcurl handles the on-to-off transition while holding - // outstanding requests is something that should be tested. + // Pipeline election changing, set dynamic option via request + + LLCore::HttpHandle handle; + const long new_depth(to_pipeline ? PIPELINING_DEPTH : 0); - status = LLCore::HttpRequest::setStaticPolicyOption(LLCore::HttpRequest::PO_PIPELINING_DEPTH, - mHttpClasses[app_policy].mPolicy, - PIPELINING_DEPTH, - NULL); - if (! status) + handle = mRequest->setPolicyOption(LLCore::HttpRequest::PO_PIPELINING_DEPTH, + mHttpClasses[app_policy].mPolicy, + new_depth, + NULL); + if (LLCORE_HTTP_HANDLE_INVALID == handle) { + status = mRequest->getStatus(); LL_WARNS("Init") << "Unable to set " << init_data[i].mUsage - << " to pipelined mode. Reason: " << status.toString() + << " pipelining. Reason: " << status.toString() << LL_ENDL; } else { - mHttpClasses[app_policy].mPipelined = true; + LL_DEBUGS("Init") << "Changed " << init_data[i].mUsage + << " pipelining. New value: " << new_depth + << LL_ENDL; + mHttpClasses[app_policy].mPipelined = to_pipeline; } } } - - // Init- or run-time settings - + // Get target connection concurrency value U32 setting(init_data[i].mDefault); if (! init_data[i].mKey.empty() && gSavedSettings.controlExists(init_data[i].mKey)) @@ -362,63 +393,60 @@ void LLAppCoreHttp::refreshSettings(bool initial) } } - if (! initial && setting == mHttpClasses[app_policy].mConnLimit) + if (initial || setting != mHttpClasses[app_policy].mConnLimit || pipeline_changed) { - // Unchanged, try next setting - continue; - } - - // Set it and report. Strategies depend on pipelining: - // - // No Pipelining. Llcorehttp manages connections itself based - // on the PO_CONNECTION_LIMIT setting. Set both limits to the - // same value for logical consistency. In the future, may - // hand over connection management to libcurl after the - // connection cache has been better vetted. - // - // Pipelining. Libcurl is allowed to manage connections to a - // great degree. Steady state will connection limit based on - // the per-host setting. Transitions (region crossings, new - // avatars, etc.) can request additional outbound connections - // to other servers via 2X total connection limit. - // - LLCore::HttpHandle handle; - handle = mRequest->setPolicyOption(LLCore::HttpRequest::PO_CONNECTION_LIMIT, - mHttpClasses[app_policy].mPolicy, - (mHttpClasses[app_policy].mPipelined ? 2 * setting : setting), - NULL); - if (LLCORE_HTTP_HANDLE_INVALID == handle) - { - status = mRequest->getStatus(); - LL_WARNS("Init") << "Unable to set " << init_data[i].mUsage - << " concurrency. Reason: " << status.toString() - << LL_ENDL; - } - else - { - handle = mRequest->setPolicyOption(LLCore::HttpRequest::PO_PER_HOST_CONNECTION_LIMIT, + // Set it and report. Strategies depend on pipelining: + // + // No Pipelining. Llcorehttp manages connections itself based + // on the PO_CONNECTION_LIMIT setting. Set both limits to the + // same value for logical consistency. In the future, may + // hand over connection management to libcurl after the + // connection cache has been better vetted. + // + // Pipelining. Libcurl is allowed to manage connections to a + // great degree. Steady state will connection limit based on + // the per-host setting. Transitions (region crossings, new + // avatars, etc.) can request additional outbound connections + // to other servers via 2X total connection limit. + // + LLCore::HttpHandle handle; + handle = mRequest->setPolicyOption(LLCore::HttpRequest::PO_CONNECTION_LIMIT, mHttpClasses[app_policy].mPolicy, - setting, + (mHttpClasses[app_policy].mPipelined ? 2 * setting : setting), NULL); if (LLCORE_HTTP_HANDLE_INVALID == handle) { status = mRequest->getStatus(); LL_WARNS("Init") << "Unable to set " << init_data[i].mUsage - << " per-host concurrency. Reason: " << status.toString() + << " concurrency. Reason: " << status.toString() << LL_ENDL; } else { - LL_DEBUGS("Init") << "Changed " << init_data[i].mUsage - << " concurrency. New value: " << setting - << LL_ENDL; - mHttpClasses[app_policy].mConnLimit = setting; - if (initial && setting != init_data[i].mDefault) + handle = mRequest->setPolicyOption(LLCore::HttpRequest::PO_PER_HOST_CONNECTION_LIMIT, + mHttpClasses[app_policy].mPolicy, + setting, + NULL); + if (LLCORE_HTTP_HANDLE_INVALID == handle) { - LL_INFOS("Init") << "Application settings overriding default " << init_data[i].mUsage - << " concurrency. New value: " << setting + status = mRequest->getStatus(); + LL_WARNS("Init") << "Unable to set " << init_data[i].mUsage + << " per-host concurrency. Reason: " << status.toString() << LL_ENDL; } + else + { + LL_DEBUGS("Init") << "Changed " << init_data[i].mUsage + << " concurrency. New value: " << setting + << LL_ENDL; + mHttpClasses[app_policy].mConnLimit = setting; + if (initial && setting != init_data[i].mDefault) + { + LL_INFOS("Init") << "Application settings overriding default " << init_data[i].mUsage + << " concurrency. New value: " << setting + << LL_ENDL; + } + } } } } diff --git a/indra/newview/llappcorehttp.h b/indra/newview/llappcorehttp.h index 63c8a11180..9ad4eb4b30 100755 --- a/indra/newview/llappcorehttp.h +++ b/indra/newview/llappcorehttp.h @@ -218,6 +218,7 @@ private: bool mStopped; HttpClass mHttpClasses[AP_COUNT]; bool mPipelined; // Global setting + boost::signals2::connection mPipelinedSignal; // Signal for 'HttpPipelining' setting }; diff --git a/indra/newview/lltexturefetch.cpp b/indra/newview/lltexturefetch.cpp index 4008a6948d..097a7b374f 100755 --- a/indra/newview/lltexturefetch.cpp +++ b/indra/newview/lltexturefetch.cpp @@ -483,12 +483,12 @@ private: bool acquireHttpSemaphore() { llassert(! mHttpHasResource); - if (mFetcher->mHttpSemaphore <= 0) + if (mFetcher->mHttpSemaphore >= mFetcher->mHttpHighWater) { return false; } mHttpHasResource = true; - mFetcher->mHttpSemaphore--; + mFetcher->mHttpSemaphore++; return true; } @@ -498,7 +498,8 @@ private: { llassert(mHttpHasResource); mHttpHasResource = false; - mFetcher->mHttpSemaphore++; + mFetcher->mHttpSemaphore--; + llassert_always(mFetcher->mHttpSemaphore >= 0); } private: @@ -2523,6 +2524,8 @@ LLTextureFetch::LLTextureFetch(LLTextureCache* cache, LLImageDecodeThread* image mMaxBandwidth = gSavedSettings.getF32("ThrottleBandwidthKBPS"); mTextureInfo.setUpLogging(gSavedSettings.getBOOL("LogTextureDownloadsToViewerLog"), gSavedSettings.getBOOL("LogTextureDownloadsToSimulator"), U32Bytes(gSavedSettings.getU32("TextureLoggingThreshold"))); + LLAppCoreHttp & app_core_http(LLAppViewer::instance()->getAppCoreHttp()); + mHttpPolicyClass = app_core_http.getPolicy(LLAppCoreHttp::AP_TEXTURE); mHttpRequest = new LLCore::HttpRequest; mHttpOptions = new LLCore::HttpOptions; mHttpOptionsWithHeaders = new LLCore::HttpOptions; @@ -2531,21 +2534,9 @@ LLTextureFetch::LLTextureFetch(LLTextureCache* cache, LLImageDecodeThread* image mHttpHeaders->append("Accept", "image/x-j2c"); mHttpMetricsHeaders = new LLCore::HttpHeaders; mHttpMetricsHeaders->append("Content-Type", "application/llsd+xml"); - LLAppCoreHttp & app_core_http(LLAppViewer::instance()->getAppCoreHttp()); - mHttpPolicyClass = app_core_http.getPolicy(LLAppCoreHttp::AP_TEXTURE); - if (app_core_http.isPipelined(LLAppCoreHttp::AP_TEXTURE)) - { - // Init-time election that will have to change for - // support of dynamic changes to the pipelining enable flag. - mHttpHighWater = HTTP_PIPE_REQUESTS_HIGH_WATER; - mHttpLowWater = HTTP_PIPE_REQUESTS_LOW_WATER; - } - else - { - mHttpHighWater = HTTP_NONPIPE_REQUESTS_HIGH_WATER; - mHttpLowWater = HTTP_NONPIPE_REQUESTS_LOW_WATER; - } - mHttpSemaphore = mHttpHighWater; + mHttpHighWater = HTTP_NONPIPE_REQUESTS_HIGH_WATER; + mHttpLowWater = HTTP_NONPIPE_REQUESTS_LOW_WATER; + mHttpSemaphore = 0; // Conditionally construct debugger object after 'this' is // fully initialized. @@ -3032,6 +3023,20 @@ bool LLTextureFetch::runCondition() // Threads: Ttf void LLTextureFetch::commonUpdate() { + // Update low/high water levels based on pipelining. We pick + // up setting eventually, so the semaphore/request level can + // fall outside the [0..HIGH_WATER] range. Expect that. + if (LLAppViewer::instance()->getAppCoreHttp().isPipelined(LLAppCoreHttp::AP_TEXTURE)) + { + mHttpHighWater = HTTP_PIPE_REQUESTS_HIGH_WATER; + mHttpLowWater = HTTP_PIPE_REQUESTS_LOW_WATER; + } + else + { + mHttpHighWater = HTTP_NONPIPE_REQUESTS_HIGH_WATER; + mHttpLowWater = HTTP_NONPIPE_REQUESTS_LOW_WATER; + } + // Release waiters releaseHttpWaiters(); @@ -3693,8 +3698,16 @@ void LLTextureFetch::releaseHttpWaiters() { // Use mHttpSemaphore rather than mHTTPTextureQueue.size() // to avoid a lock. - if (mHttpSemaphore < (mHttpHighWater - mHttpLowWater)) + if (mHttpSemaphore >= mHttpLowWater) return; + S32 needed(mHttpHighWater - mHttpSemaphore); + if (needed <= 0) + { + // Would only happen if High/LowWater were changed behind + // our back. In that case, defer fill until usage falls within + // limits. + return; + } // Quickly make a copy of all the LLUIDs. Get off the // mutex as early as possible. @@ -3743,10 +3756,10 @@ void LLTextureFetch::releaseHttpWaiters() tids.clear(); // Sort into priority order, if necessary and only as much as needed - if (tids2.size() > mHttpSemaphore) + if (tids2.size() > needed) { LLTextureFetchWorker::Compare compare; - std::partial_sort(tids2.begin(), tids2.begin() + mHttpSemaphore, tids2.end(), compare); + std::partial_sort(tids2.begin(), tids2.begin() + needed, tids2.end(), compare); } // Release workers up to the high water mark. Since we aren't diff --git a/indra/newview/lltexturefetch.h b/indra/newview/lltexturefetch.h index d13736997f..89d18e2c67 100755 --- a/indra/newview/lltexturefetch.h +++ b/indra/newview/lltexturefetch.h @@ -356,8 +356,8 @@ private: LLCore::HttpHeaders * mHttpHeaders; // Ttf LLCore::HttpHeaders * mHttpMetricsHeaders; // Ttf LLCore::HttpRequest::policy_t mHttpPolicyClass; // T* - S32 mHttpHighWater; // T* (ro) - S32 mHttpLowWater; // T* (ro) + S32 mHttpHighWater; // Ttf + S32 mHttpLowWater; // Ttf // We use a resource semaphore to keep HTTP requests in // WAIT_HTTP_RESOURCE2 if there aren't sufficient slots in the @@ -366,7 +366,11 @@ private: // where it's more expensive to get at them. Requests in either // SEND_HTTP_REQ or WAIT_HTTP_REQ charge against the semaphore // and tracking state transitions is critical to liveness. - LLAtomicS32 mHttpSemaphore; // Ttf + Tmain + // + // Originally implemented as a traditional semaphore (heading towards + // zero), it now is an outstanding request count that is allowed to + // exceed the high water level (but not go below zero). + LLAtomicS32 mHttpSemaphore; // Ttf typedef std::set wait_http_res_queue_t; wait_http_res_queue_t mHttpWaitResource; // Mfnq -- cgit v1.2.3 From 7fa382937679a9937fd7b09e33b6c2f39ec680ff Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Thu, 21 Aug 2014 15:20:31 -0400 Subject: Remove viewer-side throttles on mesh requests. --- indra/newview/llappcorehttp.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'indra/newview') diff --git a/indra/newview/llappcorehttp.cpp b/indra/newview/llappcorehttp.cpp index 464e60948a..e9274c5c1e 100755 --- a/indra/newview/llappcorehttp.cpp +++ b/indra/newview/llappcorehttp.cpp @@ -65,12 +65,12 @@ static const struct "texture fetch" }, { // AP_MESH1 - 32, 1, 128, 100, false, + 32, 1, 128, 0, false, "MeshMaxConcurrentRequests", "mesh fetch" }, { // AP_MESH2 - 8, 1, 32, 100, true, + 8, 1, 32, 0, true, "Mesh2MaxConcurrentRequests", "mesh2 fetch" }, -- cgit v1.2.3 From 85cba58ad473ed28efda7f645af20d56229e8637 Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Fri, 22 Aug 2014 18:04:27 -0400 Subject: Add an HTTP policy class for inventory operations using four (4) connections. Convert background and foreground fetches, both items and folders/inventory and library, to use new HTTP. Non-fetch inventory operations continue to use LLHTTPClient (at least for now). Error handling and retry on fetches wasn't 100% previously and that's still the case. I'll rip through this again to clean that up. Cleaned up logging in much of the inventory code with consistent labels on logging events and correct macros (removed deprecation warnings). This started as an attempt to get libcurl to do pipelining on POSTs and PUTs. Discovered that this is going to be very difficult to support in general in libcurl. May look at that again in the future. --- indra/newview/llappcorehttp.cpp | 9 +- indra/newview/llappcorehttp.h | 18 +- indra/newview/llinventorymodel.cpp | 601 ++++++++++++------ indra/newview/llinventorymodel.h | 82 ++- indra/newview/llinventorymodelbackgroundfetch.cpp | 740 +++++++++++++--------- indra/newview/llinventorymodelbackgroundfetch.h | 28 +- indra/newview/llinventoryobserver.cpp | 3 +- indra/newview/llsnapshotlivepreview.cpp | 16 +- indra/newview/llviewerinventory.cpp | 128 ++-- indra/newview/llviewermenu.cpp | 4 +- 10 files changed, 1034 insertions(+), 595 deletions(-) (limited to 'indra/newview') diff --git a/indra/newview/llappcorehttp.cpp b/indra/newview/llappcorehttp.cpp index 464e60948a..2b7d91b5ff 100755 --- a/indra/newview/llappcorehttp.cpp +++ b/indra/newview/llappcorehttp.cpp @@ -60,7 +60,7 @@ static const struct "other" }, { // AP_TEXTURE - 8, 1, 12, 0, true, + 8, 1, 12, 0, true, "TextureFetchConcurrency", "texture fetch" }, @@ -70,7 +70,7 @@ static const struct "mesh fetch" }, { // AP_MESH2 - 8, 1, 32, 100, true, + 8, 1, 32, 100, true, "Mesh2MaxConcurrentRequests", "mesh2 fetch" }, @@ -88,6 +88,11 @@ static const struct 32, 32, 32, 0, false, "", "long poll" + }, + { // AP_INVENTORY + 4, 1, 4, 0, false, + "", + "inventory" } }; diff --git a/indra/newview/llappcorehttp.h b/indra/newview/llappcorehttp.h index 9ad4eb4b30..e822a40300 100755 --- a/indra/newview/llappcorehttp.h +++ b/indra/newview/llappcorehttp.h @@ -64,7 +64,7 @@ public: /// Texture fetching policy class. Used to /// download textures via capability or SSA /// baking service. Deep queueing of requests. - /// Do not share. + /// Do not share. GET requests only. /// /// Destination: simhost:12046 & bake-texture:80 /// Protocol: http: @@ -92,7 +92,8 @@ public: /// download textures via 'GetMesh2' capability. /// Used when fetch request (typically one LOD) /// is 'small', currently defined as 2MB. - /// Very deeply queued. Do not share. + /// Very deeply queued. Do not share. GET + /// requests only. /// /// Destination: simhost:12046 /// Protocol: http: @@ -150,6 +151,19 @@ public: /// Pipelined: no AP_LONG_POLL, + /// Inventory operations (really Capabilities- + /// related operations). Mix of high-priority + /// and low-priority operations. + /// + /// Destination: simhost:12043 + /// Protocol: https: + /// Transfer size: KB-MB + /// Long poll: no + /// Concurrency: high + /// Request rate: high + /// Pipelined: no + AP_INVENTORY, + AP_COUNT // Must be last }; diff --git a/indra/newview/llinventorymodel.cpp b/indra/newview/llinventorymodel.cpp index 14ca0095ae..2dd31f047f 100755 --- a/indra/newview/llinventorymodel.cpp +++ b/indra/newview/llinventorymodel.cpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2002&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2010, Linden Research, Inc. + * Copyright (C) 2014, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -25,6 +25,9 @@ */ #include "llviewerprecompiledheaders.h" + +#include + #include "llinventorymodel.h" #include "llaisapi.h" @@ -50,7 +53,9 @@ #include "llvoavatarself.h" #include "llgesturemgr.h" #include "llsdutil.h" -#include +#include "bufferarray.h" +#include "bufferstream.h" +#include "llsdserialize.h" //#define DIFF_INVENTORY_FILES #ifdef DIFF_INVENTORY_FILES @@ -67,7 +72,14 @@ BOOL LLInventoryModel::sFirstTimeInViewer2 = TRUE; ///---------------------------------------------------------------------------- //BOOL decompress_file(const char* src_filename, const char* dst_filename); -const char CACHE_FORMAT_STRING[] = "%s.inv"; +static const char CACHE_FORMAT_STRING[] = "%s.inv"; +static const char * const LOG_INV("Inventory"); + +static std::string dumpResponse() +{ + return std::string("ADD SOMETHING MEANINGFUL HERE"); +} + struct InventoryIDPtrLess { @@ -125,24 +137,32 @@ LLInventoryModel gInventory; // Default constructor LLInventoryModel::LLInventoryModel() -: mModifyMask(LLInventoryObserver::ALL), - mChangedItemIDs(), +: // These are now ordered, keep them that way. mBacklinkMMap(), + mIsAgentInvUsable(false), + mRootFolderID(), + mLibraryRootFolderID(), + mLibraryOwnerID(), mCategoryMap(), mItemMap(), - mCategoryLock(), - mItemLock(), - mLastItem(NULL), mParentChildCategoryTree(), mParentChildItemTree(), - mObservers(), - mRootFolderID(), - mLibraryRootFolderID(), - mLibraryOwnerID(), + mLastItem(NULL), mIsNotifyObservers(FALSE), - mIsAgentInvUsable(false) -{ -} + mModifyMask(LLInventoryObserver::ALL), + mChangedItemIDs(), + mObservers(), + mHttpRequestFG(NULL), + mHttpRequestBG(NULL), + mHttpOptions(NULL), + mHttpHeaders(NULL), + mHttpPolicyClass(LLCore::HttpRequest::DEFAULT_POLICY_ID), + mHttpPriorityFG(0), + mHttpPriorityBG(0), + mCategoryLock(), + mItemLock() +{} + // Destroys the object LLInventoryModel::~LLInventoryModel() @@ -162,6 +182,22 @@ void LLInventoryModel::cleanupInventory() delete observer; } mObservers.clear(); + + // Run down HTTP transport + if (mHttpHeaders) + { + mHttpHeaders->release(); + mHttpHeaders = NULL; + } + if (mHttpOptions) + { + mHttpOptions->release(); + mHttpOptions = NULL; + } + delete mHttpRequestFG; + mHttpRequestFG = NULL; + delete mHttpRequestBG; + mHttpRequestBG = NULL; } // This is a convenience function to check if one object has a parent @@ -253,7 +289,7 @@ bool LLInventoryModel::getObjectTopmostAncestor(const LLUUID& object_id, LLUUID& LLInventoryObject *parent_object = getObject(object->getParentUUID()); if (!parent_object) { - LL_WARNS() << "unable to trace topmost ancestor, missing item for uuid " << object->getParentUUID() << LL_ENDL; + LL_WARNS(LOG_INV) << "unable to trace topmost ancestor, missing item for uuid " << object->getParentUUID() << LL_ENDL; return false; } object = parent_object; @@ -508,7 +544,7 @@ public: protected: virtual void httpFailure() { - LL_WARNS("InvAPI") << dumpResponse() << LL_ENDL; + LL_WARNS(LOG_INV) << dumpResponse() << LL_ENDL; } virtual void httpSuccess() @@ -522,7 +558,7 @@ protected: } LLUUID category_id = content["folder_id"].asUUID(); - LL_DEBUGS("Avatar") << ll_pretty_print_sd(content) << LL_ENDL; + LL_DEBUGS(LOG_INV) << ll_pretty_print_sd(content) << LL_ENDL; // Add the category to the internal representation LLPointer cat = new LLViewerInventoryCategory( category_id, @@ -560,13 +596,13 @@ LLUUID LLInventoryModel::createNewCategory(const LLUUID& parent_id, LLUUID id; if(!isInventoryUsable()) { - LL_WARNS() << "Inventory is broken." << LL_ENDL; + LL_WARNS(LOG_INV) << "Inventory is broken." << LL_ENDL; return id; } if(LLFolderType::lookup(preferred_type) == LLFolderType::badLookup()) { - LL_DEBUGS() << "Attempt to create undefined category." << LL_ENDL; + LL_DEBUGS(LOG_INV) << "Attempt to create undefined category." << LL_ENDL; return id; } @@ -599,7 +635,7 @@ LLUUID LLInventoryModel::createNewCategory(const LLUUID& parent_id, request["message"] = "CreateInventoryCategory"; request["payload"] = body; - LL_DEBUGS("Avatar") << "create category request: " << ll_pretty_print_sd(request) << LL_ENDL; + LL_DEBUGS(LOG_INV) << "create category request: " << ll_pretty_print_sd(request) << LL_ENDL; // viewer_region->getCapAPI().post(request); LLHTTPClient::post( url, @@ -820,7 +856,7 @@ U32 LLInventoryModel::updateItem(const LLViewerInventoryItem* item, U32 mask) if(!isInventoryUsable()) { - LL_WARNS() << "Inventory is broken." << LL_ENDL; + LL_WARNS(LOG_INV) << "Inventory is broken." << LL_ENDL; return mask; } @@ -883,7 +919,7 @@ U32 LLInventoryModel::updateItem(const LLViewerInventoryItem* item, U32 mask) } else { - LL_WARNS() << "Couldn't find parent-child item tree for " << new_item->getName() << LL_ENDL; + LL_WARNS(LOG_INV) << "Couldn't find parent-child item tree for " << new_item->getName() << LL_ENDL; } } else @@ -912,8 +948,8 @@ U32 LLInventoryModel::updateItem(const LLViewerInventoryItem* item, U32 mask) else { // Whoops! No such parent, make one. - LL_INFOS() << "Lost item: " << new_item->getUUID() << " - " - << new_item->getName() << LL_ENDL; + LL_INFOS(LOG_INV) << "Lost item: " << new_item->getUUID() << " - " + << new_item->getName() << LL_ENDL; parent_id = findCategoryUUIDForType(LLFolderType::FT_LOST_AND_FOUND); new_item->setParent(parent_id); item_array = get_ptr_in_map(mParentChildItemTree, parent_id); @@ -926,7 +962,7 @@ U32 LLInventoryModel::updateItem(const LLViewerInventoryItem* item, U32 mask) } else { - LL_WARNS() << "Lost and found Not there!!" << LL_ENDL; + LL_WARNS(LOG_INV) << "Lost and found Not there!!" << LL_ENDL; } } } @@ -1000,7 +1036,7 @@ void LLInventoryModel::updateCategory(const LLViewerInventoryCategory* cat, U32 if(!isInventoryUsable()) { - LL_WARNS() << "Inventory is broken." << LL_ENDL; + LL_WARNS(LOG_INV) << "Inventory is broken." << LL_ENDL; return; } @@ -1062,17 +1098,17 @@ void LLInventoryModel::updateCategory(const LLViewerInventoryCategory* cat, U32 void LLInventoryModel::moveObject(const LLUUID& object_id, const LLUUID& cat_id) { - LL_DEBUGS() << "LLInventoryModel::moveObject()" << LL_ENDL; + LL_DEBUGS(LOG_INV) << "LLInventoryModel::moveObject()" << LL_ENDL; if(!isInventoryUsable()) { - LL_WARNS() << "Inventory is broken." << LL_ENDL; + LL_WARNS(LOG_INV) << "Inventory is broken." << LL_ENDL; return; } if((object_id == cat_id) || !is_in_map(mCategoryMap, cat_id)) { - LL_WARNS() << "Could not move inventory object " << object_id << " to " - << cat_id << LL_ENDL; + LL_WARNS(LOG_INV) << "Could not move inventory object " << object_id << " to " + << cat_id << LL_ENDL; return; } LLPointer cat = getCategory(object_id); @@ -1108,14 +1144,14 @@ void LLInventoryModel::changeItemParent(LLViewerInventoryItem* item, { if (item->getParentUUID() == new_parent_id) { - LL_DEBUGS("Inventory") << "'" << item->getName() << "' (" << item->getUUID() - << ") is already in folder " << new_parent_id << LL_ENDL; + LL_DEBUGS(LOG_INV) << "'" << item->getName() << "' (" << item->getUUID() + << ") is already in folder " << new_parent_id << LL_ENDL; } else { - LL_INFOS("Inventory") << "Moving '" << item->getName() << "' (" << item->getUUID() - << ") from " << item->getParentUUID() << " to folder " - << new_parent_id << LL_ENDL; + LL_INFOS(LOG_INV) << "Moving '" << item->getName() << "' (" << item->getUUID() + << ") from " << item->getParentUUID() << " to folder " + << new_parent_id << LL_ENDL; LLInventoryModel::update_list_t update; LLInventoryModel::LLCategoryUpdate old_folder(item->getParentUUID(),-1); update.push_back(old_folder); @@ -1171,7 +1207,7 @@ void LLInventoryModel::onAISUpdateReceived(const std::string& context, const LLS AISUpdate ais_update(update); // parse update llsd into stuff to do. ais_update.doUpdate(); // execute the updates in the appropriate order. - LL_INFOS() << "elapsed: " << timer.getElapsedTimeF32() << LL_ENDL; + LL_INFOS(LOG_INV) << "elapsed: " << timer.getElapsedTimeF32() << LL_ENDL; } // Does not appear to be used currently. @@ -1180,7 +1216,7 @@ void LLInventoryModel::onItemUpdated(const LLUUID& item_id, const LLSD& updates, U32 mask = LLInventoryObserver::NONE; LLPointer item = gInventory.getItem(item_id); - LL_DEBUGS("Inventory") << "item_id: [" << item_id << "] name " << (item ? item->getName() : "(NOT FOUND)") << LL_ENDL; + LL_DEBUGS(LOG_INV) << "item_id: [" << item_id << "] name " << (item ? item->getName() : "(NOT FOUND)") << LL_ENDL; if(item) { for (LLSD::map_const_iterator it = updates.beginMap(); @@ -1188,19 +1224,19 @@ void LLInventoryModel::onItemUpdated(const LLUUID& item_id, const LLSD& updates, { if (it->first == "name") { - LL_INFOS() << "Updating name from " << item->getName() << " to " << it->second.asString() << LL_ENDL; + LL_INFOS(LOG_INV) << "Updating name from " << item->getName() << " to " << it->second.asString() << LL_ENDL; item->rename(it->second.asString()); mask |= LLInventoryObserver::LABEL; } else if (it->first == "desc") { - LL_INFOS() << "Updating description from " << item->getActualDescription() - << " to " << it->second.asString() << LL_ENDL; + LL_INFOS(LOG_INV) << "Updating description from " << item->getActualDescription() + << " to " << it->second.asString() << LL_ENDL; item->setDescription(it->second.asString()); } else { - LL_ERRS() << "unhandled updates for field: " << it->first << LL_ENDL; + LL_ERRS(LOG_INV) << "unhandled updates for field: " << it->first << LL_ENDL; } } mask |= LLInventoryObserver::INTERNAL; @@ -1211,7 +1247,7 @@ void LLInventoryModel::onItemUpdated(const LLUUID& item_id, const LLSD& updates, LLInventoryModel::LLCategoryUpdate up(item->getParentUUID(), 0); accountForUpdate(up); } - gInventory.notifyObservers(); // do we want to be able to make this optional? + notifyObservers(); // do we want to be able to make this optional? } } @@ -1221,7 +1257,7 @@ void LLInventoryModel::onCategoryUpdated(const LLUUID& cat_id, const LLSD& updat U32 mask = LLInventoryObserver::NONE; LLPointer cat = gInventory.getCategory(cat_id); - LL_DEBUGS("Inventory") << "cat_id: [" << cat_id << "] name " << (cat ? cat->getName() : "(NOT FOUND)") << LL_ENDL; + LL_DEBUGS(LOG_INV) << "cat_id: [" << cat_id << "] name " << (cat ? cat->getName() : "(NOT FOUND)") << LL_ENDL; if(cat) { for (LLSD::map_const_iterator it = updates.beginMap(); @@ -1229,18 +1265,18 @@ void LLInventoryModel::onCategoryUpdated(const LLUUID& cat_id, const LLSD& updat { if (it->first == "name") { - LL_INFOS() << "Updating name from " << cat->getName() << " to " << it->second.asString() << LL_ENDL; + LL_INFOS(LOG_INV) << "Updating name from " << cat->getName() << " to " << it->second.asString() << LL_ENDL; cat->rename(it->second.asString()); mask |= LLInventoryObserver::LABEL; } else { - LL_ERRS() << "unhandled updates for field: " << it->first << LL_ENDL; + LL_ERRS(LOG_INV) << "unhandled updates for field: " << it->first << LL_ENDL; } } mask |= LLInventoryObserver::INTERNAL; addChangedMask(mask, cat->getUUID()); - gInventory.notifyObservers(); // do we want to be able to make this optional? + notifyObservers(); // do we want to be able to make this optional? } } @@ -1317,8 +1353,8 @@ void LLInventoryModel::onDescendentsPurgedFromServer(const LLUUID& object_id, bo while (deleted_count > 0); if (total_deleted_count != count) { - LL_WARNS() << "Unexpected count of categories deleted, got " - << total_deleted_count << " expected " << count << LL_ENDL; + LL_WARNS(LOG_INV) << "Unexpected count of categories deleted, got " + << total_deleted_count << " expected " << count << LL_ENDL; } //gInventory.validate(); } @@ -1355,15 +1391,15 @@ void LLInventoryModel::onObjectDeletedFromServer(const LLUUID& object_id, bool f // Delete a particular inventory object by ID. void LLInventoryModel::deleteObject(const LLUUID& id, bool fix_broken_links, bool do_notify_observers) { - LL_DEBUGS() << "LLInventoryModel::deleteObject()" << LL_ENDL; + LL_DEBUGS(LOG_INV) << "LLInventoryModel::deleteObject()" << LL_ENDL; LLPointer obj = getObject(id); if (!obj) { - LL_WARNS() << "Deleting non-existent object [ id: " << id << " ] " << LL_ENDL; + LL_WARNS(LOG_INV) << "Deleting non-existent object [ id: " << id << " ] " << LL_ENDL; return; } - LL_DEBUGS() << "Deleting inventory object " << id << LL_ENDL; + LL_DEBUGS(LOG_INV) << "Deleting inventory object " << id << LL_ENDL; mLastItem = NULL; LLUUID parent_id = obj->getParentUUID(); mCategoryMap.erase(id); @@ -1386,7 +1422,7 @@ void LLInventoryModel::deleteObject(const LLUUID& id, bool fix_broken_links, boo { if (item_list->size()) { - LL_WARNS() << "Deleting cat " << id << " while it still has child items" << LL_ENDL; + LL_WARNS(LOG_INV) << "Deleting cat " << id << " while it still has child items" << LL_ENDL; } delete item_list; mParentChildItemTree.erase(id); @@ -1396,7 +1432,7 @@ void LLInventoryModel::deleteObject(const LLUUID& id, bool fix_broken_links, boo { if (cat_list->size()) { - LL_WARNS() << "Deleting cat " << id << " while it still has child cats" << LL_ENDL; + LL_WARNS(LOG_INV) << "Deleting cat " << id << " while it still has child cats" << LL_ENDL; } delete cat_list; mParentChildCategoryTree.erase(id); @@ -1431,7 +1467,7 @@ void LLInventoryModel::updateLinkedObjectsFromPurge(const LLUUID &baseobj_id) // everything else on the changelist will also get rebuilt. if (item_array.size() > 0) { - gInventory.notifyObservers(); + notifyObservers(); for (LLInventoryModel::item_array_t::const_iterator iter = item_array.begin(); iter != item_array.end(); iter++) @@ -1441,7 +1477,7 @@ void LLInventoryModel::updateLinkedObjectsFromPurge(const LLUUID &baseobj_id) if (item_id == baseobj_id) continue; addChangedMask(LLInventoryObserver::REBUILD, item_id); } - gInventory.notifyObservers(); + notifyObservers(); } } @@ -1464,6 +1500,9 @@ BOOL LLInventoryModel::containsObserver(LLInventoryObserver* observer) const void LLInventoryModel::idleNotifyObservers() { + // *FIX: Think I want this conditional or moved elsewhere... + handleResponses(true); + if (mModifyMask == LLInventoryObserver::NONE && (mChangedItemIDs.size() == 0)) { return; @@ -1479,7 +1518,7 @@ void LLInventoryModel::notifyObservers() // Within notifyObservers, something called notifyObservers // again. This type of recursion is unsafe because it causes items to be // processed twice, and this can easily lead to infinite loops. - LL_WARNS() << "Call was made to notifyObservers within notifyObservers!" << LL_ENDL; + LL_WARNS(LOG_INV) << "Call was made to notifyObservers within notifyObservers!" << LL_ENDL; return; } @@ -1509,18 +1548,18 @@ void LLInventoryModel::addChangedMask(U32 mask, const LLUUID& referent) // Something marked an item for change within a call to notifyObservers // (which is in the process of processing the list of items marked for change). // This means the change may fail to be processed. - LL_WARNS() << "Adding changed mask within notify observers! Change will likely be lost." << LL_ENDL; + LL_WARNS(LOG_INV) << "Adding changed mask within notify observers! Change will likely be lost." << LL_ENDL; LLViewerInventoryItem *item = getItem(referent); if (item) { - LL_WARNS() << "Item " << item->getName() << LL_ENDL; + LL_WARNS(LOG_INV) << "Item " << item->getName() << LL_ENDL; } else { LLViewerInventoryCategory *cat = getCategory(referent); if (cat) { - LL_WARNS() << "Category " << cat->getName() << LL_ENDL; + LL_WARNS(LOG_INV) << "Category " << cat->getName() << LL_ENDL; } } } @@ -1544,91 +1583,18 @@ void LLInventoryModel::addChangedMask(U32 mask, const LLUUID& referent) } } -// If we get back a normal response, handle it here -void LLInventoryModel::fetchInventoryResponder::httpSuccess() -{ - const LLSD& content = getContent(); - if (!content.isMap()) - { - failureResult(HTTP_INTERNAL_ERROR, "Malformed response contents", content); - return; - } - start_new_inventory_observer(); - - /*LLUUID agent_id; - agent_id = content["agent_id"].asUUID(); - if(agent_id != gAgent.getID()) - { - LL_WARNS() << "Got a inventory update for the wrong agent: " << agent_id - << LL_ENDL; - return; - }*/ - item_array_t items; - update_map_t update; - S32 count = content["items"].size(); - LLUUID folder_id; - // Does this loop ever execute more than once? - for(S32 i = 0; i < count; ++i) - { - LLPointer titem = new LLViewerInventoryItem; - titem->unpackMessage(content["items"][i]); - - LL_DEBUGS() << "LLInventoryModel::fetchInventoryResponder item id: " - << titem->getUUID() << LL_ENDL; - items.push_back(titem); - // examine update for changes. - LLViewerInventoryItem* itemp = gInventory.getItem(titem->getUUID()); - if(itemp) - { - if(titem->getParentUUID() == itemp->getParentUUID()) - { - update[titem->getParentUUID()]; - } - else - { - ++update[titem->getParentUUID()]; - --update[itemp->getParentUUID()]; - } - } - else - { - ++update[titem->getParentUUID()]; - } - if (folder_id.isNull()) - { - folder_id = titem->getParentUUID(); - } - } - - U32 changes = 0x0; - //as above, this loop never seems to loop more than once per call - for (item_array_t::iterator it = items.begin(); it != items.end(); ++it) - { - changes |= gInventory.updateItem(*it); - } - gInventory.notifyObservers(); - gViewerWindow->getWindow()->decBusyCount(); -} - -//If we get back an error (not found, etc...), handle it here -void LLInventoryModel::fetchInventoryResponder::httpFailure() -{ - LL_WARNS() << dumpResponse() << LL_ENDL; - gInventory.notifyObservers(); -} - bool LLInventoryModel::fetchDescendentsOf(const LLUUID& folder_id) const { if(folder_id.isNull()) { - LL_WARNS() << "Calling fetch descendents on NULL folder id!" << LL_ENDL; + LL_WARNS(LOG_INV) << "Calling fetch descendents on NULL folder id!" << LL_ENDL; return false; } LLViewerInventoryCategory* cat = getCategory(folder_id); if(!cat) { - LL_WARNS() << "Asked to fetch descendents of non-existent folder: " - << folder_id << LL_ENDL; + LL_WARNS(LOG_INV) << "Asked to fetch descendents of non-existent folder: " + << folder_id << LL_ENDL; return false; } //S32 known_descendents = 0; @@ -1649,8 +1615,8 @@ void LLInventoryModel::cache( const LLUUID& parent_folder_id, const LLUUID& agent_id) { - LL_DEBUGS() << "Caching " << parent_folder_id << " for " << agent_id - << LL_ENDL; + LL_DEBUGS(LOG_INV) << "Caching " << parent_folder_id << " for " << agent_id + << LL_ENDL; LLViewerInventoryCategory* root_cat = getCategory(parent_folder_id); if(!root_cat) return; cat_array_t categories; @@ -1675,19 +1641,19 @@ void LLInventoryModel::cache( gzip_filename.append(".gz"); if(gzip_file(inventory_filename, gzip_filename)) { - LL_DEBUGS() << "Successfully compressed " << inventory_filename << LL_ENDL; + LL_DEBUGS(LOG_INV) << "Successfully compressed " << inventory_filename << LL_ENDL; LLFile::remove(inventory_filename); } else { - LL_WARNS() << "Unable to compress " << inventory_filename << LL_ENDL; + LL_WARNS(LOG_INV) << "Unable to compress " << inventory_filename << LL_ENDL; } } void LLInventoryModel::addCategory(LLViewerInventoryCategory* category) { - //LL_INFOS() << "LLInventoryModel::addCategory()" << LL_ENDL; + //LL_INFOS(LOG_INV) << "LLInventoryModel::addCategory()" << LL_ENDL; if(category) { // We aren't displaying the Meshes folder @@ -1757,7 +1723,9 @@ void LLInventoryModel::addItem(LLViewerInventoryItem* item) if ((item->getType() == LLAssetType::AT_NONE) || LLAssetType::lookup(item->getType()) == LLAssetType::badLookup()) { - LL_WARNS() << "Got bad asset type for item [ name: " << item->getName() << " type: " << item->getType() << " inv-type: " << item->getInventoryType() << " ], ignoring." << LL_ENDL; + LL_WARNS(LOG_INV) << "Got bad asset type for item [ name: " << item->getName() + << " type: " << item->getType() + << " inv-type: " << item->getInventoryType() << " ], ignoring." << LL_ENDL; return; } @@ -1765,7 +1733,9 @@ void LLInventoryModel::addItem(LLViewerInventoryItem* item) // The item will show up as a broken link. if (item->getIsBrokenLink()) { - LL_INFOS() << "Adding broken link [ name: " << item->getName() << " itemID: " << item->getUUID() << " assetID: " << item->getAssetUUID() << " ) parent: " << item->getParentUUID() << LL_ENDL; + LL_INFOS(LOG_INV) << "Adding broken link [ name: " << item->getName() + << " itemID: " << item->getUUID() + << " assetID: " << item->getAssetUUID() << " ) parent: " << item->getParentUUID() << LL_ENDL; } if (item->getIsLinkType()) { @@ -1781,7 +1751,7 @@ void LLInventoryModel::addItem(LLViewerInventoryItem* item) // Empty the entire contents void LLInventoryModel::empty() { -// LL_INFOS() << "LLInventoryModel::empty()" << LL_ENDL; +// LL_INFOS(LOG_INV) << "LLInventoryModel::empty()" << LL_ENDL; std::for_each( mParentChildCategoryTree.begin(), mParentChildCategoryTree.end(), @@ -1814,29 +1784,29 @@ void LLInventoryModel::accountForUpdate(const LLCategoryUpdate& update) const descendents_actual += update.mDescendentDelta; cat->setDescendentCount(descendents_actual); cat->setVersion(++version); - LL_DEBUGS("Inventory") << "accounted: '" << cat->getName() << "' " - << version << " with " << descendents_actual - << " descendents." << LL_ENDL; + LL_DEBUGS(LOG_INV) << "accounted: '" << cat->getName() << "' " + << version << " with " << descendents_actual + << " descendents." << LL_ENDL; } else { // Error condition, this means that the category did not register that // it got new descendents (perhaps because it is still being loaded) // which means its descendent count will be wrong. - LL_WARNS() << "Accounting failed for '" << cat->getName() << "' version:" - << version << " due to mismatched descendent count: server == " - << descendents_server << ", viewer == " << descendents_actual << LL_ENDL; + LL_WARNS(LOG_INV) << "Accounting failed for '" << cat->getName() << "' version:" + << version << " due to mismatched descendent count: server == " + << descendents_server << ", viewer == " << descendents_actual << LL_ENDL; } } else { - LL_WARNS() << "Accounting failed for '" << cat->getName() << "' version: unknown (" - << version << ")" << LL_ENDL; + LL_WARNS(LOG_INV) << "Accounting failed for '" << cat->getName() << "' version: unknown (" + << version << ")" << LL_ENDL; } } else { - LL_WARNS() << "No category found for update " << update.mCategoryID << LL_ENDL; + LL_WARNS(LOG_INV) << "No category found for update " << update.mCategoryID << LL_ENDL; } } @@ -1918,7 +1888,7 @@ bool LLInventoryModel::loadSkeleton( const LLSD& options, const LLUUID& owner_id) { - LL_DEBUGS() << "importing inventory skeleton for " << owner_id << LL_ENDL; + LL_DEBUGS(LOG_INV) << "importing inventory skeleton for " << owner_id << LL_ENDL; typedef std::set, InventoryIDPtrLess> cat_set_t; cat_set_t temp_cats; @@ -1955,7 +1925,7 @@ bool LLInventoryModel::loadSkeleton( } else { - LL_WARNS() << "Unable to import near " << name.asString() << LL_ENDL; + LL_WARNS(LOG_INV) << "Unable to import near " << name.asString() << LL_ENDL; rv = false; } } @@ -1992,7 +1962,7 @@ bool LLInventoryModel::loadSkeleton( } else { - LL_INFOS() << "Unable to gunzip " << gzip_filename << LL_ENDL; + LL_INFOS(LOG_INV) << "Unable to gunzip " << gzip_filename << LL_ENDL; } } bool is_cache_obsolete = false; @@ -2073,10 +2043,10 @@ bool LLInventoryModel::loadSkeleton( if (item->getIsBrokenLink()) { //bad_link_count++; - LL_DEBUGS() << "Attempted to add cached link item without baseobj present ( name: " - << item->getName() << " itemID: " << item->getUUID() - << " assetID: " << item->getAssetUUID() - << " ). Ignoring and invalidating " << cat->getName() << " . " << LL_ENDL; + LL_DEBUGS(LOG_INV) << "Attempted to add cached link item without baseobj present ( name: " + << item->getName() << " itemID: " << item->getUUID() + << " assetID: " << item->getAssetUUID() + << " ). Ignoring and invalidating " << cat->getName() << " . " << LL_ENDL; possible_broken_links.push_back(item); continue; } @@ -2103,7 +2073,7 @@ bool LLInventoryModel::loadSkeleton( { bad_link_count++; invalid_categories.insert(cit->second); - //LL_INFOS() << "link still broken: " << item->getName() << " in folder " << cat->getName() << LL_ENDL; + //LL_INFOS(LOG_INV) << "link still broken: " << item->getName() << " in folder " << cat->getName() << LL_ENDL; } else { @@ -2115,11 +2085,11 @@ bool LLInventoryModel::loadSkeleton( } } - LL_INFOS() << "Attempted to add " << bad_link_count - << " cached link items without baseobj present. " - << good_link_count << " link items were successfully added. " - << recovered_link_count << " links added in recovery. " - << "The corresponding categories were invalidated." << LL_ENDL; + LL_INFOS(LOG_INV) << "Attempted to add " << bad_link_count + << " cached link items without baseobj present. " + << good_link_count << " link items were successfully added. " + << recovered_link_count << " links added in recovery. " + << "The corresponding categories were invalidated." << LL_ENDL; } } @@ -2143,9 +2113,9 @@ bool LLInventoryModel::loadSkeleton( { LLViewerInventoryCategory* cat = (*invalid_cat_it).get(); cat->setVersion(NO_VERSION); - LL_DEBUGS("Inventory") << "Invalidating category name: " << cat->getName() << " UUID: " << cat->getUUID() << " due to invalid descendents cache" << LL_ENDL; + LL_DEBUGS(LOG_INV) << "Invalidating category name: " << cat->getName() << " UUID: " << cat->getUUID() << " due to invalid descendents cache" << LL_ENDL; } - LL_INFOS("Inventory") << "Invalidated " << invalid_categories.size() << " categories due to invalid descendents cache" << LL_ENDL; + LL_INFOS(LOG_INV) << "Invalidated " << invalid_categories.size() << " categories due to invalid descendents cache" << LL_ENDL; // At this point, we need to set the known descendents for each // category which successfully cached so that we do not @@ -2177,15 +2147,15 @@ bool LLInventoryModel::loadSkeleton( if(is_cache_obsolete) { // If out of date, remove the gzipped file too. - LL_WARNS() << "Inv cache out of date, removing" << LL_ENDL; + LL_WARNS(LOG_INV) << "Inv cache out of date, removing" << LL_ENDL; LLFile::remove(gzip_filename); } categories.clear(); // will unref and delete entries } - LL_INFOS() << "Successfully loaded " << cached_category_count - << " categories and " << cached_item_count << " items from cache." - << LL_ENDL; + LL_INFOS(LOG_INV) << "Successfully loaded " << cached_category_count + << " categories and " << cached_item_count << " items from cache." + << LL_ENDL; return rv; } @@ -2195,7 +2165,7 @@ bool LLInventoryModel::loadSkeleton( // should be sufficient for our needs. void LLInventoryModel::buildParentChildMap() { - LL_INFOS() << "LLInventoryModel::buildParentChildMap()" << LL_ENDL; + LL_INFOS(LOG_INV) << "LLInventoryModel::buildParentChildMap()" << LL_ENDL; // *NOTE: I am skipping the logic around folder version // synchronization here because it seems if a folder is lost, we @@ -2264,15 +2234,15 @@ void LLInventoryModel::buildParentChildMap() // implement it, we would need a set or map of uuid pairs // which would be (folder_id, new_parent_id) to be sent up // to the server. - LL_INFOS() << "Lost category: " << cat->getUUID() << " - " - << cat->getName() << LL_ENDL; + LL_INFOS(LOG_INV) << "Lost category: " << cat->getUUID() << " - " + << cat->getName() << LL_ENDL; ++lost; lost_cats.push_back(cat); } } if(lost) { - LL_WARNS() << "Found " << lost << " lost categories." << LL_ENDL; + LL_WARNS(LOG_INV) << "Found " << lost << " lost categories." << LL_ENDL; } // Do moves in a separate pass to make sure we've properly filed @@ -2307,7 +2277,7 @@ void LLInventoryModel::buildParentChildMap() } else { - LL_WARNS() << "Lost and found Not there!!" << LL_ENDL; + LL_WARNS(LOG_INV) << "Lost and found Not there!!" << LL_ENDL; } } @@ -2342,8 +2312,8 @@ void LLInventoryModel::buildParentChildMap() } else { - LL_INFOS() << "Lost item: " << item->getUUID() << " - " - << item->getName() << LL_ENDL; + LL_INFOS(LOG_INV) << "Lost item: " << item->getUUID() << " - " + << item->getName() << LL_ENDL; ++lost; // plop it into the lost & found. // @@ -2359,13 +2329,13 @@ void LLInventoryModel::buildParentChildMap() } else { - LL_WARNS() << "Lost and found Not there!!" << LL_ENDL; + LL_WARNS(LOG_INV) << "Lost and found Not there!!" << LL_ENDL; } } } if(lost) { - LL_WARNS() << "Found " << lost << " lost items." << LL_ENDL; + LL_WARNS(LOG_INV) << "Found " << lost << " lost items." << LL_ENDL; LLMessageSystem* msg = gMessageSystem; BOOL start_new_message = TRUE; const LLUUID lnf = findCategoryUUIDForType(LLFolderType::FT_LOST_AND_FOUND); @@ -2443,10 +2413,85 @@ void LLInventoryModel::buildParentChildMap() if (!gInventory.validate()) { - LL_WARNS() << "model failed validity check!" << LL_ENDL; + LL_WARNS(LOG_INV) << "model failed validity check!" << LL_ENDL; + } +} + +// Would normally do this at construction but that's too early +// in the process for gInventory. Have the first requestPost() +// call set things up. +void LLInventoryModel::initHttpRequest() +{ + if (! mHttpRequestFG) + { + // Haven't initialized, get to it + LLAppCoreHttp & app_core_http(LLAppViewer::instance()->getAppCoreHttp()); + + mHttpRequestFG = new LLCore::HttpRequest; + mHttpRequestBG = new LLCore::HttpRequest; + mHttpOptions = new LLCore::HttpOptions; + mHttpOptions->setTransferTimeout(300); + mHttpOptions->setUseRetryAfter(true); + // mHttpOptions->setTrace(2); // Do tracing of requests + mHttpHeaders = new LLCore::HttpHeaders; + mHttpHeaders->append(HTTP_OUT_HEADER_CONTENT_TYPE, HTTP_CONTENT_LLSD_XML); + mHttpHeaders->append(HTTP_OUT_HEADER_ACCEPT, HTTP_CONTENT_LLSD_XML); + mHttpPolicyClass = app_core_http.getPolicy(LLAppCoreHttp::AP_INVENTORY); } } +void LLInventoryModel::handleResponses(bool foreground) +{ + if (foreground) + { + mHttpRequestFG->update(0); + } + else + { + mHttpRequestBG->update(50000L); + } +} + +LLCore::HttpHandle LLInventoryModel::requestPost(bool foreground, + const std::string & url, + const LLSD & body, + LLCore::HttpHandler * handler, + const char * const message) +{ + if (! mHttpRequestFG) + { + // We do the initialization late and lazily as this class is + // statically-constructed and not all the bits are ready at + // that time. + initHttpRequest(); + } + + LLCore::HttpRequest * request(foreground ? mHttpRequestFG : mHttpRequestBG); + LLCore::HttpHandle handle(LLCORE_HTTP_HANDLE_INVALID); + LLCore::BufferArray * ba = new LLCore::BufferArray; + LLCore::BufferArrayStream bas(ba); + LLSDSerialize::toXML(body, bas); + + handle = request->requestPost(mHttpPolicyClass, + (foreground ? mHttpPriorityFG : mHttpPriorityBG), + url, + ba, + mHttpOptions, + mHttpHeaders, + handler); + ba->release(); + if (LLCORE_HTTP_HANDLE_INVALID == handle) + { + LLCore::HttpStatus status(request->getStatus()); + LL_WARNS(LOG_INV) << "HTTP POST request failed for " << message + << ", Status: " << status.toTerseString() + << " Reason: '" << status.toString() << "'" + << LL_ENDL; + delete handler; + } + return handle; +} + void LLInventoryModel::createCommonSystemCategories() { gInventory.findCategoryUUIDForType(LLFolderType::FT_TRASH,true); @@ -2495,14 +2540,14 @@ bool LLInventoryModel::loadFromFile(const std::string& filename, { if(filename.empty()) { - LL_ERRS() << "Filename is Null!" << LL_ENDL; + LL_ERRS(LOG_INV) << "Filename is Null!" << LL_ENDL; return false; } - LL_INFOS() << "LLInventoryModel::loadFromFile(" << filename << ")" << LL_ENDL; + LL_INFOS(LOG_INV) << "LLInventoryModel::loadFromFile(" << filename << ")" << LL_ENDL; LLFILE* file = LLFile::fopen(filename, "rb"); /*Flawfinder: ignore*/ if(!file) { - LL_INFOS() << "unable to load inventory from: " << filename << LL_ENDL; + LL_INFOS(LOG_INV) << "unable to load inventory from: " << filename << LL_ENDL; return false; } // *NOTE: This buffer size is hard coded into scanf() below. @@ -2541,7 +2586,7 @@ bool LLInventoryModel::loadFromFile(const std::string& filename, } else { - LL_WARNS() << "loadInventoryFromFile(). Ignoring invalid inventory category: " << inv_cat->getName() << LL_ENDL; + LL_WARNS(LOG_INV) << "loadInventoryFromFile(). Ignoring invalid inventory category: " << inv_cat->getName() << LL_ENDL; //delete inv_cat; // automatic when inv_cat is reassigned or destroyed } } @@ -2559,8 +2604,8 @@ bool LLInventoryModel::loadFromFile(const std::string& filename, if(inv_item->getUUID().isNull()) { //delete inv_item; // automatic when inv_cat is reassigned or destroyed - LL_WARNS() << "Ignoring inventory with null item id: " - << inv_item->getName() << LL_ENDL; + LL_WARNS(LOG_INV) << "Ignoring inventory with null item id: " + << inv_item->getName() << LL_ENDL; } else @@ -2570,14 +2615,14 @@ bool LLInventoryModel::loadFromFile(const std::string& filename, } else { - LL_WARNS() << "loadInventoryFromFile(). Ignoring invalid inventory item: " << inv_item->getName() << LL_ENDL; + LL_WARNS(LOG_INV) << "loadInventoryFromFile(). Ignoring invalid inventory item: " << inv_item->getName() << LL_ENDL; //delete inv_item; // automatic when inv_cat is reassigned or destroyed } } else { - LL_WARNS() << "Unknown token in inventory file '" << keyword << "'" - << LL_ENDL; + LL_WARNS(LOG_INV) << "Unknown token in inventory file '" << keyword << "'" + << LL_ENDL; } } fclose(file); @@ -2593,14 +2638,14 @@ bool LLInventoryModel::saveToFile(const std::string& filename, { if(filename.empty()) { - LL_ERRS() << "Filename is Null!" << LL_ENDL; + LL_ERRS(LOG_INV) << "Filename is Null!" << LL_ENDL; return false; } - LL_INFOS() << "LLInventoryModel::saveToFile(" << filename << ")" << LL_ENDL; + LL_INFOS(LOG_INV) << "LLInventoryModel::saveToFile(" << filename << ")" << LL_ENDL; LLFILE* file = LLFile::fopen(filename, "wb"); /*Flawfinder: ignore*/ if(!file) { - LL_WARNS() << "unable to save inventory to: " << filename << LL_ENDL; + LL_WARNS(LOG_INV) << "unable to save inventory to: " << filename << LL_ENDL; return false; } @@ -2705,8 +2750,8 @@ bool LLInventoryModel::messageUpdateCore(LLMessageSystem* msg, bool account, U32 msg->getUUIDFast(_PREHASH_AgentData, _PREHASH_AgentID, agent_id); if(agent_id != gAgent.getID()) { - LL_WARNS() << "Got a inventory update for the wrong agent: " << agent_id - << LL_ENDL; + LL_WARNS(LOG_INV) << "Got a inventory update for the wrong agent: " << agent_id + << LL_ENDL; return false; } item_array_t items; @@ -2718,8 +2763,8 @@ bool LLInventoryModel::messageUpdateCore(LLMessageSystem* msg, bool account, U32 { LLPointer titem = new LLViewerInventoryItem; titem->unpackMessage(msg, _PREHASH_InventoryData, i); - LL_DEBUGS() << "LLInventoryModel::messageUpdateCore() item id:" - << titem->getUUID() << LL_ENDL; + LL_DEBUGS(LOG_INV) << "LLInventoryModel::messageUpdateCore() item id: " + << titem->getUUID() << LL_ENDL; items.push_back(titem); // examine update for changes. LLViewerInventoryItem* itemp = gInventory.getItem(titem->getUUID()); @@ -2770,17 +2815,17 @@ void LLInventoryModel::removeInventoryItem(LLUUID agent_id, LLMessageSystem* msg { LLUUID item_id; S32 count = msg->getNumberOfBlocksFast(msg_label); - LL_DEBUGS() << "Message has " << count << " item blocks" << LL_ENDL; + LL_DEBUGS(LOG_INV) << "Message has " << count << " item blocks" << LL_ENDL; uuid_vec_t item_ids; update_map_t update; for(S32 i = 0; i < count; ++i) { msg->getUUIDFast(msg_label, _PREHASH_ItemID, item_id, i); - LL_DEBUGS() << "Checking for item-to-be-removed " << item_id << LL_ENDL; + LL_DEBUGS(LOG_INV) << "Checking for item-to-be-removed " << item_id << LL_ENDL; LLViewerInventoryItem* itemp = gInventory.getItem(item_id); if(itemp) { - LL_DEBUGS() << "Item will be removed " << item_id << LL_ENDL; + LL_DEBUGS(LOG_INV) << "Item will be removed " << item_id << LL_ENDL; // we only bother with the delete and account if we found // the item - this is usually a back-up for permissions, // so frequently the item will already be gone. @@ -2791,7 +2836,7 @@ void LLInventoryModel::removeInventoryItem(LLUUID agent_id, LLMessageSystem* msg gInventory.accountForUpdate(update); for(uuid_vec_t::iterator it = item_ids.begin(); it != item_ids.end(); ++it) { - LL_DEBUGS() << "Calling deleteObject " << *it << LL_ENDL; + LL_DEBUGS(LOG_INV) << "Calling deleteObject " << *it << LL_ENDL; gInventory.deleteObject(*it); } } @@ -2799,13 +2844,13 @@ void LLInventoryModel::removeInventoryItem(LLUUID agent_id, LLMessageSystem* msg // static void LLInventoryModel::processRemoveInventoryItem(LLMessageSystem* msg, void**) { - LL_DEBUGS() << "LLInventoryModel::processRemoveInventoryItem()" << LL_ENDL; + LL_DEBUGS(LOG_INV) << "LLInventoryModel::processRemoveInventoryItem()" << LL_ENDL; LLUUID agent_id, item_id; msg->getUUIDFast(_PREHASH_AgentData, _PREHASH_AgentID, agent_id); if(agent_id != gAgent.getID()) { - LL_WARNS() << "Got a RemoveInventoryItem for the wrong agent." - << LL_ENDL; + LL_WARNS(LOG_INV) << "Got a RemoveInventoryItem for the wrong agent." + << LL_ENDL; return; } LLInventoryModel::removeInventoryItem(agent_id, msg, _PREHASH_InventoryData); @@ -2816,14 +2861,14 @@ void LLInventoryModel::processRemoveInventoryItem(LLMessageSystem* msg, void**) void LLInventoryModel::processUpdateInventoryFolder(LLMessageSystem* msg, void**) { - LL_DEBUGS() << "LLInventoryModel::processUpdateInventoryFolder()" << LL_ENDL; + LL_DEBUGS(LOG_INV) << "LLInventoryModel::processUpdateInventoryFolder()" << LL_ENDL; LLUUID agent_id, folder_id, parent_id; //char name[DB_INV_ITEM_NAME_BUF_SIZE]; msg->getUUIDFast(_PREHASH_FolderData, _PREHASH_AgentID, agent_id); if(agent_id != gAgent.getID()) { - LL_WARNS() << "Got an UpdateInventoryFolder for the wrong agent." - << LL_ENDL; + LL_WARNS(LOG_INV) << "Got an UpdateInventoryFolder for the wrong agent." + << LL_ENDL; return; } LLPointer lastfolder; // hack @@ -3881,7 +3926,7 @@ bool LLInventoryModel::validate() const ///---------------------------------------------------------------------------- -/* +#if 0 BOOL decompress_file(const char* src_filename, const char* dst_filename) { BOOL rv = FALSE; @@ -3920,4 +3965,140 @@ BOOL decompress_file(const char* src_filename, const char* dst_filename) if(dst != NULL) fclose(dst); return rv; } -*/ +#endif + +// ==== FetchItemHttpHandler ==== + +LLInventoryModel::FetchItemHttpHandler::FetchItemHttpHandler(const LLSD & request_sd) + : LLCore::HttpHandler(), + mRequestSD(request_sd) +{} + +LLInventoryModel::FetchItemHttpHandler::~FetchItemHttpHandler() +{} + +void LLInventoryModel::FetchItemHttpHandler::onCompleted(LLCore::HttpHandle handle, + LLCore::HttpResponse * response) +{ + LLCore::HttpStatus status(response->getStatus()); + if (! status) + { + processFailure(status, response); + } + else + { + LLCore::BufferArray * body(response->getBody()); + if (! body || ! body->size()) + { + LL_WARNS(LOG_INV) << "Missing data in inventory item query." << LL_ENDL; + processFailure("HTTP response for inventory item query missing body", response); + goto only_exit; + } + + LLCore::BufferArrayStream bas(body); + LLSD body_llsd; + S32 parse_status(LLSDSerialize::fromXML(body_llsd, bas)); + if (LLSDParser::PARSE_FAILURE == parse_status) + { + // INFOS-level logging will occur on the parsed failure + processFailure("HTTP response for inventory item query has malformed LLSD", response); + goto only_exit; + } + + // Okay, process data if possible + processData(body_llsd, response); + } + +only_exit: + // Must delete on completion. + delete this; +} + +void LLInventoryModel::FetchItemHttpHandler::processData(LLSD & content, LLCore::HttpResponse * response) +{ + if (! content.isMap()) + { + processFailure("LLSD response for inventory item not a map", response); + return; + } + + start_new_inventory_observer(); + +#if 0 + LLUUID agent_id; + agent_id = content["agent_id"].asUUID(); + if (agent_id != gAgent.getID()) + { + LL_WARNS(LOG_INV) << "Got a inventory update for the wrong agent: " << agent_id + << LL_ENDL; + return; + } +#endif + + LLInventoryModel::item_array_t items; + LLInventoryModel::update_map_t update; + LLUUID folder_id; + LLSD content_items(content["items"]); + const S32 count(content_items.size()); + + // Does this loop ever execute more than once? + for (S32 i(0); i < count; ++i) + { + LLPointer titem = new LLViewerInventoryItem; + titem->unpackMessage(content_items[i]); + + LL_DEBUGS(LOG_INV) << "ItemHttpHandler::httpSuccess item id: " + << titem->getUUID() << LL_ENDL; + items.push_back(titem); + + // examine update for changes. + LLViewerInventoryItem * itemp(gInventory.getItem(titem->getUUID())); + + if (itemp) + { + if (titem->getParentUUID() == itemp->getParentUUID()) + { + update[titem->getParentUUID()]; + } + else + { + ++update[titem->getParentUUID()]; + --update[itemp->getParentUUID()]; + } + } + else + { + ++update[titem->getParentUUID()]; + } + + if (folder_id.isNull()) + { + folder_id = titem->getParentUUID(); + } + } + + // as above, this loop never seems to loop more than once per call + U32 changes(0U); + for (LLInventoryModel::item_array_t::iterator it = items.begin(); it != items.end(); ++it) + { + changes |= gInventory.updateItem(*it); + } + // *HUH: Have computed changes, nothing uses it. + + gInventory.notifyObservers(); + gViewerWindow->getWindow()->decBusyCount(); +} + + +void LLInventoryModel::FetchItemHttpHandler::processFailure(LLCore::HttpStatus status, LLCore::HttpResponse * response) +{ + LL_WARNS(LOG_INV) << dumpResponse() << LL_ENDL; + gInventory.notifyObservers(); +} + +void LLInventoryModel::FetchItemHttpHandler::processFailure(const char * const reason, LLCore::HttpResponse * response) +{ + LL_WARNS(LOG_INV) << dumpResponse() << LL_ENDL; + gInventory.notifyObservers(); +} + diff --git a/indra/newview/llinventorymodel.h b/indra/newview/llinventorymodel.h index 2e957809be..ac336e347c 100755 --- a/indra/newview/llinventorymodel.h +++ b/indra/newview/llinventorymodel.h @@ -27,6 +27,11 @@ #ifndef LL_LLINVENTORYMODEL_H #define LL_LLINVENTORYMODEL_H +#include +#include +#include +#include + #include "llassettype.h" #include "llfoldertype.h" #include "llframetimer.h" @@ -36,10 +41,11 @@ #include "llviewerinventory.h" #include "llstring.h" #include "llmd5.h" -#include -#include -#include -#include +#include "httpcommon.h" +#include "httprequest.h" +#include "httpoptions.h" +#include "httpheaders.h" +#include "httphandler.h" class LLInventoryObserver; class LLInventoryObject; @@ -60,9 +66,8 @@ class LLInventoryCollectFunctor; class LLInventoryModel { LOG_CLASS(LLInventoryModel); -public: - friend class LLInventoryModelFetchDescendentsResponder; +public: enum EHasChildren { CHILDREN_NO, @@ -74,14 +79,31 @@ public: typedef std::vector > item_array_t; typedef std::set changed_items_t; - class fetchInventoryResponder : public LLCurl::Responder + // HTTP handler for individual item requests (inventory or library). + // Background item requests are derived from this in the background + // inventory system. All folder requests are also located there + // but have their own handler derived from HttpHandler. + class FetchItemHttpHandler : public LLCore::HttpHandler { - LOG_CLASS(fetchInventoryResponder); public: - fetchInventoryResponder(const LLSD& request_sd) : mRequestSD(request_sd) {}; + LOG_CLASS(FetchItemHttpHandler); + + FetchItemHttpHandler(const LLSD & request_sd); + virtual ~FetchItemHttpHandler(); + protected: - virtual void httpSuccess(); - virtual void httpFailure(); + FetchItemHttpHandler(const FetchItemHttpHandler &); // Not defined + void operator=(const FetchItemHttpHandler &); // Not defined + + public: + virtual void onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * response); + + private: + void processData(LLSD & body, LLCore::HttpResponse * response); + void processFailure(LLCore::HttpStatus status, LLCore::HttpResponse * response); + void processFailure(const char * const reason, LLCore::HttpResponse * response); + + private: LLSD mRequestSD; }; @@ -109,6 +131,9 @@ public: private: bool mIsAgentInvUsable; // used to handle an invalid inventory state + // One-time initialization of HTTP system. + void initHttpRequest(); + //-------------------------------------------------------------------- // Root Folders //-------------------------------------------------------------------- @@ -518,6 +543,41 @@ private: *******************************************************************************/ +/******************************************************************************** + ** ** + ** HTTP Transport + **/ +public: + // Invoke handler completion method (onCompleted) for all + // requests that are ready. + void handleResponses(bool foreground); + + // Request an inventory HTTP operation to either the + // foreground or background processor. These are actually + // the same service queue but the background requests are + // seviced more slowly effectively de-prioritizing new + // requests. + LLCore::HttpHandle requestPost(bool foreground, + const std::string & url, + const LLSD & body, + LLCore::HttpHandler * handler, + const char * const message); + +private: + // Usual plumbing for LLCore:: HTTP operations. + LLCore::HttpRequest * mHttpRequestFG; + LLCore::HttpRequest * mHttpRequestBG; + LLCore::HttpOptions * mHttpOptions; + LLCore::HttpHeaders * mHttpHeaders; + LLCore::HttpRequest::policy_t mHttpPolicyClass; + LLCore::HttpRequest::priority_t mHttpPriorityFG; + LLCore::HttpRequest::priority_t mHttpPriorityBG; + +/** HTTP Transport + ** ** + *******************************************************************************/ + + /******************************************************************************** ** ** ** MISCELLANEOUS diff --git a/indra/newview/llinventorymodelbackgroundfetch.cpp b/indra/newview/llinventorymodelbackgroundfetch.cpp index 2de37b0790..443c54df42 100755 --- a/indra/newview/llinventorymodelbackgroundfetch.cpp +++ b/indra/newview/llinventorymodelbackgroundfetch.cpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2002&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2010, Linden Research, Inc. + * Copyright (C) 2014, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -37,31 +37,113 @@ #include "llviewermessage.h" #include "llviewerregion.h" #include "llviewerwindow.h" +#include "llhttpconstants.h" +#include "bufferarray.h" +#include "bufferstream.h" +#include "llsdserialize.h" + +namespace +{ + +// Http request handler class for single inventory item requests. +// +// We'll use a handler-per-request pattern here rather than +// a shared handler. Mainly convenient as this was converted +// from a Responder class model. +// +class BGItemHttpHandler : public LLInventoryModel::FetchItemHttpHandler +{ + LOG_CLASS(BGItemHttpHandler); + +public: + BGItemHttpHandler(const LLSD & request_sd) + : LLInventoryModel::FetchItemHttpHandler(request_sd) + { + LLInventoryModelBackgroundFetch::instance().incrFetchCount(1); + } + + virtual ~BGItemHttpHandler() + { + LLInventoryModelBackgroundFetch::instance().incrFetchCount(-1); + } + +protected: + BGItemHttpHandler(const BGItemHttpHandler &); // Not defined + void operator=(const BGItemHttpHandler &); // Not defined +}; + + +// Http request handler class for folders. +class BGFolderHttpHandler : public LLCore::HttpHandler +{ + LOG_CLASS(BGFolderHttpHandler); + +public: + BGFolderHttpHandler(const LLSD & request_sd, const uuid_vec_t & recursive_cats) + : LLCore::HttpHandler(), + mRequestSD(request_sd), + mRecursiveCatUUIDs(recursive_cats) + { + LLInventoryModelBackgroundFetch::instance().incrFetchCount(1); + } + + virtual ~BGFolderHttpHandler() + { + LLInventoryModelBackgroundFetch::instance().incrFetchCount(-1); + } + +protected: + BGFolderHttpHandler(const BGFolderHttpHandler &); // Not defined + void operator=(const BGFolderHttpHandler &); // Not defined + +public: + virtual void onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * response); + + bool getIsRecursive(const LLUUID & cat_id) const; + +private: + void processData(LLSD & body, LLCore::HttpResponse * response); + void processFailure(LLCore::HttpStatus status, LLCore::HttpResponse * response); + void processFailure(const char * const reason, LLCore::HttpResponse * response); + +private: + LLSD mRequestSD; + const uuid_vec_t mRecursiveCatUUIDs; // hack for storing away which cat fetches are recursive +}; + -const F32 MAX_TIME_FOR_SINGLE_FETCH = 10.f; const S32 MAX_FETCH_RETRIES = 10; -LLInventoryModelBackgroundFetch::LLInventoryModelBackgroundFetch() : +const char * const LOG_INV("Inventory"); + +std::string dumpResponse() +{ + return std::string("ADD SOMETHING MEANINGFUL HERE"); +} + + +} // end of namespace anonymous + + +LLInventoryModelBackgroundFetch::LLInventoryModelBackgroundFetch(): mBackgroundFetchActive(FALSE), mFolderFetchActive(false), + mFetchCount(0), mAllFoldersFetched(FALSE), mRecursiveInventoryFetchStarted(FALSE), mRecursiveLibraryFetchStarted(FALSE), mNumFetchRetries(0), mMinTimeBetweenFetches(0.3f), mMaxTimeBetweenFetches(10.f), - mTimelyFetchPending(FALSE), - mFetchCount(0) -{ -} + mTimelyFetchPending(FALSE) +{} LLInventoryModelBackgroundFetch::~LLInventoryModelBackgroundFetch() -{ -} +{} bool LLInventoryModelBackgroundFetch::isBulkFetchProcessingComplete() const { - return mFetchQueue.empty() && mFetchCount<=0; + return mFetchQueue.empty() && mFetchCount <= 0; } bool LLInventoryModelBackgroundFetch::libraryFetchStarted() const @@ -91,7 +173,7 @@ bool LLInventoryModelBackgroundFetch::inventoryFetchCompleted() const bool LLInventoryModelBackgroundFetch::inventoryFetchInProgress() const { - return inventoryFetchStarted() && !inventoryFetchCompleted(); + return inventoryFetchStarted() && ! inventoryFetchCompleted(); } bool LLInventoryModelBackgroundFetch::isEverythingFetched() const @@ -104,24 +186,36 @@ BOOL LLInventoryModelBackgroundFetch::folderFetchActive() const return mFolderFetchActive; } +void LLInventoryModelBackgroundFetch::addRequestAtFront(const LLUUID & id, BOOL recursive, bool is_category) +{ + mFetchQueue.push_front(FetchQueueInfo(id, recursive, is_category)); +} + +void LLInventoryModelBackgroundFetch::addRequestAtBack(const LLUUID & id, BOOL recursive, bool is_category) +{ + mFetchQueue.push_back(FetchQueueInfo(id, recursive, is_category)); +} + void LLInventoryModelBackgroundFetch::start(const LLUUID& id, BOOL recursive) { - LLViewerInventoryCategory* cat = gInventory.getCategory(id); - if (cat || (id.isNull() && !isEverythingFetched())) - { // it's a folder, do a bulk fetch - LL_DEBUGS("InventoryFetch") << "Start fetching category: " << id << ", recursive: " << recursive << LL_ENDL; + LLViewerInventoryCategory * cat(gInventory.getCategory(id)); + + if (cat || (id.isNull() && ! isEverythingFetched())) + { + // it's a folder, do a bulk fetch + LL_DEBUGS(LOG_INV) << "Start fetching category: " << id << ", recursive: " << recursive << LL_ENDL; mBackgroundFetchActive = TRUE; mFolderFetchActive = true; if (id.isNull()) { - if (!mRecursiveInventoryFetchStarted) + if (! mRecursiveInventoryFetchStarted) { mRecursiveInventoryFetchStarted |= recursive; mFetchQueue.push_back(FetchQueueInfo(gInventory.getRootFolderID(), recursive)); gIdleCallbacks.addFunction(&LLInventoryModelBackgroundFetch::backgroundFetchCB, NULL); } - if (!mRecursiveLibraryFetchStarted) + if (! mRecursiveLibraryFetchStarted) { mRecursiveLibraryFetchStarted |= recursive; mFetchQueue.push_back(FetchQueueInfo(gInventory.getLibraryRootFolderID(), recursive)); @@ -146,9 +240,9 @@ void LLInventoryModelBackgroundFetch::start(const LLUUID& id, BOOL recursive) } } } - else if (LLViewerInventoryItem* itemp = gInventory.getItem(id)) + else if (LLViewerInventoryItem * itemp = gInventory.getItem(id)) { - if (!itemp->mIsComplete && (mFetchQueue.empty() || mFetchQueue.front().mUUID != id)) + if (! itemp->mIsComplete && (mFetchQueue.empty() || mFetchQueue.front().mUUID != id)) { mBackgroundFetchActive = TRUE; @@ -172,7 +266,7 @@ void LLInventoryModelBackgroundFetch::setAllFoldersFetched() mRecursiveLibraryFetchStarted) { mAllFoldersFetched = TRUE; - //LL_INFOS() << "All folders fetched, validating" << LL_ENDL; + //LL_INFOS(LOG_INV) << "All folders fetched, validating" << LL_ENDL; //gInventory.validate(); } mFolderFetchActive = false; @@ -203,7 +297,7 @@ void LLInventoryModelBackgroundFetch::backgroundFetch() // No more categories to fetch, stop fetch process. if (mFetchQueue.empty()) { - LL_INFOS() << "Inventory fetch completed" << LL_ENDL; + LL_INFOS(LOG_INV) << "Inventory fetch completed" << LL_ENDL; setAllFoldersFetched(); mBackgroundFetchActive = false; @@ -219,7 +313,7 @@ void LLInventoryModelBackgroundFetch::backgroundFetch() // Double timeouts on failure. mMinTimeBetweenFetches = llmin(mMinTimeBetweenFetches * 2.f, 10.f); mMaxTimeBetweenFetches = llmin(mMaxTimeBetweenFetches * 2.f, 120.f); - LL_DEBUGS() << "Inventory fetch times grown to (" << mMinTimeBetweenFetches << ", " << mMaxTimeBetweenFetches << ")" << LL_ENDL; + LL_DEBUGS(LOG_INV) << "Inventory fetch times grown to (" << mMinTimeBetweenFetches << ", " << mMaxTimeBetweenFetches << ")" << LL_ENDL; // fetch is no longer considered "timely" although we will wait for full time-out. mTimelyFetchPending = FALSE; } @@ -231,7 +325,7 @@ void LLInventoryModelBackgroundFetch::backgroundFetch() break; } - if(gDisconnected) + if (gDisconnected) { // Just bail if we are disconnected. break; @@ -292,7 +386,7 @@ void LLInventoryModelBackgroundFetch::backgroundFetch() // Shrink timeouts based on success. mMinTimeBetweenFetches = llmax(mMinTimeBetweenFetches * 0.8f, 0.3f); mMaxTimeBetweenFetches = llmax(mMaxTimeBetweenFetches * 0.8f, 10.f); - LL_DEBUGS() << "Inventory fetch times shrunk to (" << mMinTimeBetweenFetches << ", " << mMaxTimeBetweenFetches << ")" << LL_ENDL; + LL_DEBUGS(LOG_INV) << "Inventory fetch times shrunk to (" << mMinTimeBetweenFetches << ", " << mMaxTimeBetweenFetches << ")" << LL_ENDL; } mTimelyFetchPending = FALSE; @@ -355,257 +449,53 @@ void LLInventoryModelBackgroundFetch::backgroundFetch() } } -void LLInventoryModelBackgroundFetch::incrFetchCount(S16 fetching) +void LLInventoryModelBackgroundFetch::incrFetchCount(S32 fetching) { mFetchCount += fetching; if (mFetchCount < 0) { + LL_WARNS_ONCE(LOG_INV) << "Inventory fetch count fell below zero (0)." << LL_ENDL; mFetchCount = 0; } } -class LLInventoryModelFetchItemResponder : public LLInventoryModel::fetchInventoryResponder -{ - LOG_CLASS(LLInventoryModelFetchItemResponder); -public: - LLInventoryModelFetchItemResponder(const LLSD& request_sd) : - LLInventoryModel::fetchInventoryResponder(request_sd) - { - LLInventoryModelBackgroundFetch::instance().incrFetchCount(1); - } -private: - /* virtual */ void httpCompleted() - { - LLInventoryModelBackgroundFetch::instance().incrFetchCount(-1); - LLInventoryModel::fetchInventoryResponder::httpCompleted(); - } -}; - -class LLInventoryModelFetchDescendentsResponder: public LLHTTPClient::Responder -{ - LOG_CLASS(LLInventoryModelFetchDescendentsResponder); -public: - LLInventoryModelFetchDescendentsResponder(const LLSD& request_sd, uuid_vec_t recursive_cats) : - mRequestSD(request_sd), - mRecursiveCatUUIDs(recursive_cats) - { - LLInventoryModelBackgroundFetch::instance().incrFetchCount(1); - } - //LLInventoryModelFetchDescendentsResponder() {}; -private: - /* virtual */ void httpCompleted() - { - LLInventoryModelBackgroundFetch::instance().incrFetchCount(-1); - LLHTTPClient::Responder::httpCompleted(); - } - /* virtual */ void httpSuccess(); - /* virtual */ void httpFailure(); -protected: - BOOL getIsRecursive(const LLUUID& cat_id) const; -private: - LLSD mRequestSD; - uuid_vec_t mRecursiveCatUUIDs; // hack for storing away which cat fetches are recursive -}; - -// If we get back a normal response, handle it here. -void LLInventoryModelFetchDescendentsResponder::httpSuccess() +// Bundle up a bunch of requests to send all at once. +void LLInventoryModelBackgroundFetch::bulkFetch() { - const LLSD& content = getContent(); - if (!content.isMap()) + //Background fetch is called from gIdleCallbacks in a loop until background fetch is stopped. + //If there are items in mFetchQueue, we want to check the time since the last bulkFetch was + //sent. If it exceeds our retry time, go ahead and fire off another batch. + LLViewerRegion * region(gAgent.getRegion()); + if (! region || gDisconnected) { - failureResult(HTTP_INTERNAL_ERROR, "Malformed response contents", content); return; } - LLInventoryModelBackgroundFetch *fetcher = LLInventoryModelBackgroundFetch::getInstance(); - if (content.has("folders")) - { - - for(LLSD::array_const_iterator folder_it = content["folders"].beginArray(); - folder_it != content["folders"].endArray(); - ++folder_it) - { - LLSD folder_sd = *folder_it; - - - //LLUUID agent_id = folder_sd["agent_id"]; - - //if(agent_id != gAgent.getID()) //This should never happen. - //{ - // LL_WARNS() << "Got a UpdateInventoryItem for the wrong agent." - // << LL_ENDL; - // break; - //} - - LLUUID parent_id = folder_sd["folder_id"]; - LLUUID owner_id = folder_sd["owner_id"]; - S32 version = (S32)folder_sd["version"].asInteger(); - S32 descendents = (S32)folder_sd["descendents"].asInteger(); - LLPointer tcategory = new LLViewerInventoryCategory(owner_id); - if (parent_id.isNull()) - { - LLPointer titem = new LLViewerInventoryItem; - for(LLSD::array_const_iterator item_it = folder_sd["items"].beginArray(); - item_it != folder_sd["items"].endArray(); - ++item_it) - { - const LLUUID lost_uuid = gInventory.findCategoryUUIDForType(LLFolderType::FT_LOST_AND_FOUND); - if (lost_uuid.notNull()) - { - LLSD item = *item_it; - titem->unpackMessage(item); - - LLInventoryModel::update_list_t update; - LLInventoryModel::LLCategoryUpdate new_folder(lost_uuid, 1); - update.push_back(new_folder); - gInventory.accountForUpdate(update); - - titem->setParent(lost_uuid); - titem->updateParentOnServer(FALSE); - gInventory.updateItem(titem); - gInventory.notifyObservers(); - - } - } - } - - LLViewerInventoryCategory* pcat = gInventory.getCategory(parent_id); - if (!pcat) - { - continue; - } - - for(LLSD::array_const_iterator category_it = folder_sd["categories"].beginArray(); - category_it != folder_sd["categories"].endArray(); - ++category_it) - { - LLSD category = *category_it; - tcategory->fromLLSD(category); - - const BOOL recursive = getIsRecursive(tcategory->getUUID()); - - if (recursive) - { - fetcher->mFetchQueue.push_back(LLInventoryModelBackgroundFetch::FetchQueueInfo(tcategory->getUUID(), recursive)); - } - else if ( !gInventory.isCategoryComplete(tcategory->getUUID()) ) - { - gInventory.updateCategory(tcategory); - } - - } - LLPointer titem = new LLViewerInventoryItem; - for(LLSD::array_const_iterator item_it = folder_sd["items"].beginArray(); - item_it != folder_sd["items"].endArray(); - ++item_it) - { - LLSD item = *item_it; - titem->unpackMessage(item); - - gInventory.updateItem(titem); - } - - // Set version and descendentcount according to message. - LLViewerInventoryCategory* cat = gInventory.getCategory(parent_id); - if(cat) - { - cat->setVersion(version); - cat->setDescendentCount(descendents); - cat->determineFolderType(); - } - - } - } - - if (content.has("bad_folders")) - { - for(LLSD::array_const_iterator folder_it = content["bad_folders"].beginArray(); - folder_it != content["bad_folders"].endArray(); - ++folder_it) - { - // *TODO: Stop copying data - LLSD folder_sd = *folder_it; - - // These folders failed on the dataserver. We probably don't want to retry them. - LL_WARNS() << "Folder " << folder_sd["folder_id"].asString() - << "Error: " << folder_sd["error"].asString() << LL_ENDL; - } - } + static const S32 max_concurrent_fetches(12); // Outstanding requests, not connections + static const F32 new_min_time(0.5f); // *HACK: Clean this up when old code goes away entirely. + static const U32 max_batch_size(10); - if (fetcher->isBulkFetchProcessingComplete()) + if (mMinTimeBetweenFetches < new_min_time) { - LL_INFOS() << "Inventory fetch completed" << LL_ENDL; - fetcher->setAllFoldersFetched(); + mMinTimeBetweenFetches = new_min_time; // *HACK: See above. } - - gInventory.notifyObservers(); -} - -// If we get back an error (not found, etc...), handle it here. -void LLInventoryModelFetchDescendentsResponder::httpFailure() -{ - LL_WARNS() << dumpResponse() << LL_ENDL; - LLInventoryModelBackgroundFetch *fetcher = LLInventoryModelBackgroundFetch::getInstance(); - - LL_INFOS() << dumpResponse() << LL_ENDL; - - fetcher->incrFetchCount(-1); - if (getStatus()==HTTP_INTERNAL_ERROR) // timed out or curl failure - { - for(LLSD::array_const_iterator folder_it = mRequestSD["folders"].beginArray(); - folder_it != mRequestSD["folders"].endArray(); - ++folder_it) - { - LLSD folder_sd = *folder_it; - LLUUID folder_id = folder_sd["folder_id"]; - const BOOL recursive = getIsRecursive(folder_id); - fetcher->mFetchQueue.push_front(LLInventoryModelBackgroundFetch::FetchQueueInfo(folder_id, recursive)); - } - } - else + if (mFetchCount) { - if (fetcher->isBulkFetchProcessingComplete()) - { - fetcher->setAllFoldersFetched(); - } - } - gInventory.notifyObservers(); -} - -BOOL LLInventoryModelFetchDescendentsResponder::getIsRecursive(const LLUUID& cat_id) const -{ - return (std::find(mRecursiveCatUUIDs.begin(),mRecursiveCatUUIDs.end(), cat_id) != mRecursiveCatUUIDs.end()); -} -// Bundle up a bunch of requests to send all at once. -// static -void LLInventoryModelBackgroundFetch::bulkFetch() -{ - //Background fetch is called from gIdleCallbacks in a loop until background fetch is stopped. - //If there are items in mFetchQueue, we want to check the time since the last bulkFetch was - //sent. If it exceeds our retry time, go ahead and fire off another batch. - LLViewerRegion* region = gAgent.getRegion(); - if (!region) return; - - S16 max_concurrent_fetches=8; - F32 new_min_time = 0.5f; //HACK! Clean this up when old code goes away entirely. - if (mMinTimeBetweenFetches < new_min_time) - { - mMinTimeBetweenFetches=new_min_time; //HACK! See above. + // Process completed HTTP requests + gInventory.handleResponses(false); } - if (gDisconnected || - (mFetchCount > max_concurrent_fetches) || + if ((mFetchCount > max_concurrent_fetches) || (mFetchTimer.getElapsedTimeF32() < mMinTimeBetweenFetches)) { - return; // just bail if we are disconnected + return; } - U32 item_count=0; - U32 folder_count=0; - U32 max_batch_size=5; + U32 item_count(0); + U32 folder_count(0); - U32 sort_order = gSavedSettings.getU32(LLInventoryPanel::DEFAULT_SORT_ORDER) & 0x1; + const U32 sort_order(gSavedSettings.getU32(LLInventoryPanel::DEFAULT_SORT_ORDER) & 0x1); uuid_vec_t recursive_cats; @@ -614,27 +504,27 @@ void LLInventoryModelBackgroundFetch::bulkFetch() LLSD item_request_body; LLSD item_request_body_lib; - while (!mFetchQueue.empty() + while (! mFetchQueue.empty() && (item_count + folder_count) < max_batch_size) { - const FetchQueueInfo& fetch_info = mFetchQueue.front(); + const FetchQueueInfo & fetch_info(mFetchQueue.front()); if (fetch_info.mIsCategory) { - const LLUUID &cat_id = fetch_info.mUUID; + const LLUUID & cat_id(fetch_info.mUUID); if (cat_id.isNull()) //DEV-17797 { LLSD folder_sd; folder_sd["folder_id"] = LLUUID::null.asString(); folder_sd["owner_id"] = gAgent.getID(); - folder_sd["sort_order"] = (LLSD::Integer)sort_order; - folder_sd["fetch_folders"] = (LLSD::Boolean)FALSE; - folder_sd["fetch_items"] = (LLSD::Boolean)TRUE; + folder_sd["sort_order"] = LLSD::Integer(sort_order); + folder_sd["fetch_folders"] = LLSD::Boolean(FALSE); + folder_sd["fetch_items"] = LLSD::Boolean(TRUE); folder_request_body["folders"].append(folder_sd); folder_count++; } else { - const LLViewerInventoryCategory* cat = gInventory.getCategory(cat_id); + const LLViewerInventoryCategory * cat(gInventory.getCategory(cat_id)); if (cat) { @@ -643,21 +533,26 @@ void LLInventoryModelBackgroundFetch::bulkFetch() LLSD folder_sd; folder_sd["folder_id"] = cat->getUUID(); folder_sd["owner_id"] = cat->getOwnerID(); - folder_sd["sort_order"] = (LLSD::Integer)sort_order; - folder_sd["fetch_folders"] = TRUE; //(LLSD::Boolean)sFullFetchStarted; - folder_sd["fetch_items"] = (LLSD::Boolean)TRUE; + folder_sd["sort_order"] = LLSD::Integer(sort_order); + folder_sd["fetch_folders"] = LLSD::Boolean(TRUE); //(LLSD::Boolean)sFullFetchStarted; + folder_sd["fetch_items"] = LLSD::Boolean(TRUE); if (ALEXANDRIA_LINDEN_ID == cat->getOwnerID()) + { folder_request_body_lib["folders"].append(folder_sd); + } else + { folder_request_body["folders"].append(folder_sd); + } folder_count++; } + // May already have this folder, but append child folders to list. if (fetch_info.mRecursive) { - LLInventoryModel::cat_array_t* categories; - LLInventoryModel::item_array_t* items; + LLInventoryModel::cat_array_t * categories(NULL); + LLInventoryModel::item_array_t * items(NULL); gInventory.getDirectDescendentsOf(cat->getUUID(), categories, items); for (LLInventoryModel::cat_array_t::const_iterator it = categories->begin(); it != categories->end(); @@ -669,11 +564,14 @@ void LLInventoryModelBackgroundFetch::bulkFetch() } } if (fetch_info.mRecursive) + { recursive_cats.push_back(cat_id); + } } else { - LLViewerInventoryItem* itemp = gInventory.getItem(fetch_info.mUUID); + LLViewerInventoryItem * itemp(gInventory.getItem(fetch_info.mUUID)); + if (itemp) { LLSD item_sd; @@ -694,72 +592,80 @@ void LLInventoryModelBackgroundFetch::bulkFetch() mFetchQueue.pop_front(); } - + + // Issue HTTP POST requests to fetch folders and items + if (item_count + folder_count > 0) { if (folder_count) { - std::string url = region->getCapability("FetchInventoryDescendents2"); - if ( !url.empty() ) + if (folder_request_body["folders"].size()) { - if (folder_request_body["folders"].size()) + const std::string url(region->getCapability("FetchInventoryDescendents2")); + + if (! url.empty()) { - LLInventoryModelFetchDescendentsResponder *fetcher = new LLInventoryModelFetchDescendentsResponder(folder_request_body, recursive_cats); - LLHTTPClient::post(url, folder_request_body, fetcher, 300.0); + BGFolderHttpHandler * handler(new BGFolderHttpHandler(folder_request_body, recursive_cats)); + gInventory.requestPost(false, url, folder_request_body, handler, "Inventory Folder"); } - if (folder_request_body_lib["folders"].size()) - { - std::string url_lib = gAgent.getRegion()->getCapability("FetchLibDescendents2"); + } + + if (folder_request_body_lib["folders"].size()) + { + const std::string url(region->getCapability("FetchLibDescendents2")); - LLInventoryModelFetchDescendentsResponder *fetcher = new LLInventoryModelFetchDescendentsResponder(folder_request_body_lib, recursive_cats); - LLHTTPClient::post(url_lib, folder_request_body_lib, fetcher, 300.0); + if (! url.empty()) + { + BGFolderHttpHandler * handler(new BGFolderHttpHandler(folder_request_body_lib, recursive_cats)); + gInventory.requestPost(false, url, folder_request_body_lib, handler, "Library Folder"); } } - } + } // if (folder_count) + if (item_count) { - std::string url; - if (item_request_body.size()) { - url = region->getCapability("FetchInventory2"); - if (!url.empty()) + const std::string url(region->getCapability("FetchInventory2")); + + if (! url.empty()) { LLSD body; body["items"] = item_request_body; - - LLHTTPClient::post(url, body, new LLInventoryModelFetchItemResponder(body)); + BGItemHttpHandler * handler(new BGItemHttpHandler(body)); + gInventory.requestPost(false, url, body, handler, "Inventory Item"); } } if (item_request_body_lib.size()) { + const std::string url(region->getCapability("FetchLib2")); - url = region->getCapability("FetchLib2"); - if (!url.empty()) + if (! url.empty()) { LLSD body; body["items"] = item_request_body_lib; - - LLHTTPClient::post(url, body, new LLInventoryModelFetchItemResponder(body)); + BGItemHttpHandler * handler(new BGItemHttpHandler(body)); + gInventory.requestPost(false, url, body, handler, "Library Item"); } } - } + } // if (item_count) + mFetchTimer.reset(); } - else if (isBulkFetchProcessingComplete()) { setAllFoldersFetched(); } } -bool LLInventoryModelBackgroundFetch::fetchQueueContainsNoDescendentsOf(const LLUUID& cat_id) const +bool LLInventoryModelBackgroundFetch::fetchQueueContainsNoDescendentsOf(const LLUUID & cat_id) const { for (fetch_queue_t::const_iterator it = mFetchQueue.begin(); - it != mFetchQueue.end(); ++it) + it != mFetchQueue.end(); + ++it) { - const LLUUID& fetch_id = (*it).mUUID; + const LLUUID & fetch_id = (*it).mUUID; if (gInventory.isObjectDescendentOf(fetch_id, cat_id)) return false; } @@ -767,3 +673,257 @@ bool LLInventoryModelBackgroundFetch::fetchQueueContainsNoDescendentsOf(const LL } +// Anonymous Namespace Definitions + +namespace +{ + +// ==== BGFolderHttpHandler ==== + +void BGFolderHttpHandler::onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * response) +{ + LLCore::HttpStatus status(response->getStatus()); + if (! status) + { + processFailure(status, response); + } + else + { + LLCore::BufferArray * body(response->getBody()); + if (! body || ! body->size()) + { + LL_WARNS(LOG_INV) << "Missing data in inventory folder query." << LL_ENDL; + processFailure("HTTP response missing expected body", response); + goto only_exit; + } + + LLCore::BufferArrayStream bas(body); + LLSD body_llsd; + S32 parse_status(LLSDSerialize::fromXML(body_llsd, bas)); + if (LLSDParser::PARSE_FAILURE == parse_status) + { + // INFOS-level logging will occur on the parsed failure + processFailure("HTTP response contained malformed LLSD", response); + goto only_exit; + } + + // Okay, process data if possible + processData(body_llsd, response); + } + +only_exit: + // Must delete on completion. + delete this; +} + + +void BGFolderHttpHandler::processData(LLSD & content, LLCore::HttpResponse * response) +{ + if (! content.isMap()) + { + processFailure("LLSD response not a map", response); + return; + } + + LLInventoryModelBackgroundFetch * fetcher(LLInventoryModelBackgroundFetch::getInstance()); + if (content.has("folders")) + { + LLSD folders(content["folders"]); + + for (LLSD::array_const_iterator folder_it = folders.beginArray(); + folder_it != folders.endArray(); + ++folder_it) + { + LLSD folder_sd(*folder_it); + + //LLUUID agent_id = folder_sd["agent_id"]; + + //if(agent_id != gAgent.getID()) //This should never happen. + //{ + // LL_WARNS(LOG_INV) << "Got a UpdateInventoryItem for the wrong agent." + // << LL_ENDL; + // break; + //} + + LLUUID parent_id(folder_sd["folder_id"].asUUID()); + LLUUID owner_id(folder_sd["owner_id"].asUUID()); + S32 version(folder_sd["version"].asInteger()); + S32 descendents(folder_sd["descendents"].asInteger()); + LLPointer tcategory = new LLViewerInventoryCategory(owner_id); + + if (parent_id.isNull()) + { + LLSD items(folder_sd["items"]); + LLPointer titem = new LLViewerInventoryItem; + + for (LLSD::array_const_iterator item_it = items.beginArray(); + item_it != items.endArray(); + ++item_it) + { + const LLUUID lost_uuid(gInventory.findCategoryUUIDForType(LLFolderType::FT_LOST_AND_FOUND)); + + if (lost_uuid.notNull()) + { + LLSD item(*item_it); + + titem->unpackMessage(item); + + LLInventoryModel::update_list_t update; + LLInventoryModel::LLCategoryUpdate new_folder(lost_uuid, 1); + update.push_back(new_folder); + gInventory.accountForUpdate(update); + + titem->setParent(lost_uuid); + titem->updateParentOnServer(FALSE); + gInventory.updateItem(titem); + gInventory.notifyObservers(); + } + } + } + + LLViewerInventoryCategory * pcat(gInventory.getCategory(parent_id)); + if (! pcat) + { + continue; + } + + LLSD categories(folder_sd["categories"]); + for (LLSD::array_const_iterator category_it = categories.beginArray(); + category_it != categories.endArray(); + ++category_it) + { + LLSD category(*category_it); + tcategory->fromLLSD(category); + + const bool recursive(getIsRecursive(tcategory->getUUID())); + if (recursive) + { + fetcher->addRequestAtBack(tcategory->getUUID(), recursive, true); + } + else if (! gInventory.isCategoryComplete(tcategory->getUUID())) + { + gInventory.updateCategory(tcategory); + } + } + + LLSD items(folder_sd["items"]); + LLPointer titem = new LLViewerInventoryItem; + for (LLSD::array_const_iterator item_it = items.beginArray(); + item_it != items.endArray(); + ++item_it) + { + LLSD item(*item_it); + titem->unpackMessage(item); + + gInventory.updateItem(titem); + } + + // Set version and descendentcount according to message. + LLViewerInventoryCategory * cat(gInventory.getCategory(parent_id)); + if (cat) + { + cat->setVersion(version); + cat->setDescendentCount(descendents); + cat->determineFolderType(); + } + } + } + + if (content.has("bad_folders")) + { + LLSD bad_folders(content["bad_folders"]); + for (LLSD::array_const_iterator folder_it = bad_folders.beginArray(); + folder_it != bad_folders.endArray(); + ++folder_it) + { + // *TODO: Stop copying data + LLSD folder_sd(*folder_it); + + // These folders failed on the dataserver. We probably don't want to retry them. + LL_WARNS(LOG_INV) << "Folder " << folder_sd["folder_id"].asString() + << "Error: " << folder_sd["error"].asString() << LL_ENDL; + } + } + + if (fetcher->isBulkFetchProcessingComplete()) + { + LL_INFOS(LOG_INV) << "Inventory fetch completed" << LL_ENDL; + fetcher->setAllFoldersFetched(); + } + + gInventory.notifyObservers(); +} + + +void BGFolderHttpHandler::processFailure(LLCore::HttpStatus status, LLCore::HttpResponse * response) +{ + LL_WARNS(LOG_INV) << dumpResponse() << LL_ENDL; + LLInventoryModelBackgroundFetch *fetcher = LLInventoryModelBackgroundFetch::getInstance(); + + LL_INFOS(LOG_INV) << dumpResponse() << LL_ENDL; + + // *FIX: Not the correct test here... + if (status == LLCore::HttpStatus(408)) // timed out or curl failure + { + for (LLSD::array_const_iterator folder_it = mRequestSD["folders"].beginArray(); + folder_it != mRequestSD["folders"].endArray(); + ++folder_it) + { + LLSD folder_sd(*folder_it); + LLUUID folder_id(folder_sd["folder_id"]); + const BOOL recursive = getIsRecursive(folder_id); + fetcher->addRequestAtFront(folder_id, recursive, true); + } + } + else + { + if (fetcher->isBulkFetchProcessingComplete()) + { + fetcher->setAllFoldersFetched(); + } + } + gInventory.notifyObservers(); +} + + +void BGFolderHttpHandler::processFailure(const char * const reason, LLCore::HttpResponse * response) +{ + LL_WARNS(LOG_INV) << dumpResponse() << LL_ENDL; + LLInventoryModelBackgroundFetch *fetcher = LLInventoryModelBackgroundFetch::getInstance(); + + LL_INFOS(LOG_INV) << dumpResponse() << LL_ENDL; + + if (true /* status == LLCore::HttpStatus(408)*/) // timed out or curl failure + { + for (LLSD::array_const_iterator folder_it = mRequestSD["folders"].beginArray(); + folder_it != mRequestSD["folders"].endArray(); + ++folder_it) + { + LLSD folder_sd(*folder_it); + LLUUID folder_id(folder_sd["folder_id"]); + const BOOL recursive = getIsRecursive(folder_id); + fetcher->addRequestAtFront(folder_id, recursive, true); + } + } + else + { + if (fetcher->isBulkFetchProcessingComplete()) + { + fetcher->setAllFoldersFetched(); + } + } + gInventory.notifyObservers(); +} + + +bool BGFolderHttpHandler::getIsRecursive(const LLUUID & cat_id) const +{ + return std::find(mRecursiveCatUUIDs.begin(), mRecursiveCatUUIDs.end(), cat_id) != mRecursiveCatUUIDs.end(); +} + + +// ==== BGItemHttpHandler ==== + +// Nothing to implement here. All ctor/dtor changes. + +} // end namespace anonymous diff --git a/indra/newview/llinventorymodelbackgroundfetch.h b/indra/newview/llinventorymodelbackgroundfetch.h index 9dfedddd6d..2139f85519 100755 --- a/indra/newview/llinventorymodelbackgroundfetch.h +++ b/indra/newview/llinventorymodelbackgroundfetch.h @@ -29,6 +29,11 @@ #include "llsingleton.h" #include "lluuid.h" +#include "httpcommon.h" +#include "httprequest.h" +#include "httpoptions.h" +#include "httpheaders.h" +#include "httphandler.h" //~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ // Class LLInventoryModelBackgroundFetch @@ -38,8 +43,6 @@ //~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ class LLInventoryModelBackgroundFetch : public LLSingleton { - friend class LLInventoryModelFetchDescendentsResponder; - public: LLInventoryModelBackgroundFetch(); ~LLInventoryModelBackgroundFetch(); @@ -60,16 +63,22 @@ public: bool inventoryFetchInProgress() const; void findLostItems(); - void incrFetchCount(S16 fetching); -protected: + void incrFetchCount(S32 fetching); + bool isBulkFetchProcessingComplete() const; + void setAllFoldersFetched(); + + void addRequestAtFront(const LLUUID & id, BOOL recursive, bool is_category); + void addRequestAtBack(const LLUUID & id, BOOL recursive, bool is_category); + +protected: void bulkFetch(); void backgroundFetch(); static void backgroundFetchCB(void*); // background fetch idle function - void setAllFoldersFetched(); bool fetchQueueContainsNoDescendentsOf(const LLUUID& cat_id) const; + private: BOOL mRecursiveInventoryFetchStarted; BOOL mRecursiveLibraryFetchStarted; @@ -77,7 +86,7 @@ private: BOOL mBackgroundFetchActive; bool mFolderFetchActive; - S16 mFetchCount; + S32 mFetchCount; BOOL mTimelyFetchPending; S32 mNumFetchRetries; @@ -87,9 +96,12 @@ private: struct FetchQueueInfo { - FetchQueueInfo(const LLUUID& id, BOOL recursive, bool is_category = true) : - mUUID(id), mRecursive(recursive), mIsCategory(is_category) + FetchQueueInfo(const LLUUID& id, BOOL recursive, bool is_category = true) + : mUUID(id), + mIsCategory(is_category), + mRecursive(recursive) {} + LLUUID mUUID; bool mIsCategory; BOOL mRecursive; diff --git a/indra/newview/llinventoryobserver.cpp b/indra/newview/llinventoryobserver.cpp index 2dd8dce42f..d81401b59b 100755 --- a/indra/newview/llinventoryobserver.cpp +++ b/indra/newview/llinventoryobserver.cpp @@ -237,7 +237,8 @@ void fetch_items_from_llsd(const LLSD& items_llsd) if (!url.empty()) { body[i]["agent_id"] = gAgent.getID(); - LLHTTPClient::post(url, body[i], new LLInventoryModel::fetchInventoryResponder(body[i])); + LLInventoryModel::FetchItemHttpHandler * handler(new LLInventoryModel::FetchItemHttpHandler(body[i])); + gInventory.requestPost(true, url, body[i], handler, (i ? "Library Item" : "Inventory Item")); continue; } diff --git a/indra/newview/llsnapshotlivepreview.cpp b/indra/newview/llsnapshotlivepreview.cpp index f61db77169..942360b650 100644 --- a/indra/newview/llsnapshotlivepreview.cpp +++ b/indra/newview/llsnapshotlivepreview.cpp @@ -5,7 +5,7 @@ * * $LicenseInfo:firstyear=2013&license=viewerlgpl$ * Second Life Viewer Source Code -* Copyright (C) 2013, Linden Research, Inc. +* Copyright (C) 2014, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -151,7 +151,7 @@ F32 LLSnapshotLivePreview::getImageAspect() void LLSnapshotLivePreview::updateSnapshot(BOOL new_snapshot, BOOL new_thumbnail, F32 delay) { - lldebugs << "updateSnapshot: mSnapshotUpToDate = " << getSnapshotUpToDate() << llendl; + LL_DEBUGS() << "updateSnapshot: mSnapshotUpToDate = " << getSnapshotUpToDate() << LL_ENDL; // Update snapshot if requested. if (new_snapshot) @@ -594,7 +594,7 @@ void LLSnapshotLivePreview::generateThumbnailImage(BOOL force_update) } else { - llwarns << "Couldn't find a path to the following filter : " << getFilter() << llendl; + LL_WARNS() << "Couldn't find a path to the following filter : " << getFilter() << LL_ENDL; } } // Scale to a power of 2 so it can be mapped to a texture @@ -642,7 +642,7 @@ LLViewerTexture* LLSnapshotLivePreview::getBigThumbnailImage() } else { - llwarns << "Couldn't find a path to the following filter : " << getFilter() << llendl; + LL_WARNS() << "Couldn't find a path to the following filter : " << getFilter() << LL_ENDL; } } // Scale to a power of 2 so it can be mapped to a texture @@ -695,7 +695,7 @@ BOOL LLSnapshotLivePreview::onIdle( void* snapshot_preview ) // time to produce a snapshot if(!previewp->getSnapshotUpToDate()) { - lldebugs << "producing snapshot" << llendl; + LL_DEBUGS() << "producing snapshot" << LL_ENDL; if (!previewp->mPreviewImage) { previewp->mPreviewImage = new LLImageRaw; @@ -775,7 +775,7 @@ BOOL LLSnapshotLivePreview::onIdle( void* snapshot_preview ) previewp->setVisible(gSavedSettings.getBOOL("UseFreezeFrame") && previewp->mAllowFullScreenPreview); // only show fullscreen preview when in freeze frame mode previewp->mSnapshotDelayTimer.stop(); previewp->mSnapshotActive = FALSE; - lldebugs << "done creating snapshot" << llendl; + LL_DEBUGS() << "done creating snapshot" << LL_ENDL; } if (!previewp->getThumbnailUpToDate()) @@ -910,13 +910,13 @@ LLPointer LLSnapshotLivePreview::getFormattedImage() } else { - llwarns << "Couldn't find a path to the following filter : " << getFilter() << llendl; + LL_WARNS() << "Couldn't find a path to the following filter : " << getFilter() << LL_ENDL; } } // Create the new formatted image of the appropriate format. LLFloaterSnapshot::ESnapshotFormat format = getSnapshotFormat(); - lldebugs << "Encoding new image of format " << format << llendl; + LL_DEBUGS() << "Encoding new image of format " << format << LL_ENDL; switch (format) { diff --git a/indra/newview/llviewerinventory.cpp b/indra/newview/llviewerinventory.cpp index 4e4c3471be..d364fce45a 100755 --- a/indra/newview/llviewerinventory.cpp +++ b/indra/newview/llviewerinventory.cpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2002&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2010, Linden Research, Inc. + * Copyright (C) 2014, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -75,6 +75,10 @@ void no_op_inventory_func(const LLUUID&) {} void no_op_llsd_func(const LLSD&) {} void no_op() {} +static const char * const LOG_INV("Inventory"); +static const char * const LOG_LOCAL("InventoryLocalize"); +static const char * const LOG_NOTECARD("copy_inventory_from_notecard"); + ///---------------------------------------------------------------------------- /// Helper class to store special inventory item names and their localized values. ///---------------------------------------------------------------------------- @@ -189,7 +193,7 @@ public: */ bool localizeInventoryObjectName(std::string& object_name) { - LL_DEBUGS("InventoryLocalize") << "Searching for localization: " << object_name << LL_ENDL; + LL_DEBUGS(LOG_LOCAL) << "Searching for localization: " << object_name << LL_ENDL; std::map::const_iterator dictionary_iter = mInventoryItemsDict.find(object_name); @@ -197,7 +201,7 @@ public: if(found) { object_name = dictionary_iter->second; - LL_DEBUGS("InventoryLocalize") << "Found, new name is: " << object_name << LL_ENDL; + LL_DEBUGS(LOG_LOCAL) << "Found, new name is: " << object_name << LL_ENDL; } return found; } @@ -307,8 +311,8 @@ LLViewerInventoryItem::LLViewerInventoryItem(const LLViewerInventoryItem* other) copyViewerItem(other); if (!mIsComplete) { - LL_WARNS() << "LLViewerInventoryItem copy constructor for incomplete item" - << mUUID << LL_ENDL; + LL_WARNS(LOG_INV) << "LLViewerInventoryItem copy constructor for incomplete item" + << mUUID << LL_ENDL; } } @@ -355,16 +359,16 @@ void LLViewerInventoryItem::updateServer(BOOL is_new) const { // *FIX: deal with this better. // If we're crashing here then the UI is incorrectly enabled. - LL_ERRS() << "LLViewerInventoryItem::updateServer() - for incomplete item" - << LL_ENDL; + LL_ERRS(LOG_INV) << "LLViewerInventoryItem::updateServer() - for incomplete item" + << LL_ENDL; return; } if(gAgent.getID() != mPermissions.getOwner()) { // *FIX: deal with this better. - LL_WARNS() << "LLViewerInventoryItem::updateServer() - for unowned item " - << ll_pretty_print_sd(this->asLLSD()) - << LL_ENDL; + LL_WARNS(LOG_INV) << "LLViewerInventoryItem::updateServer() - for unowned item " + << ll_pretty_print_sd(this->asLLSD()) + << LL_ENDL; return; } LLInventoryModel::LLCategoryUpdate up(mParentUUID, is_new ? 1 : 0); @@ -392,18 +396,18 @@ void LLViewerInventoryItem::fetchFromServer(void) const // we have to check region. It can be null after region was destroyed. See EXT-245 if (region) { - if(gAgent.getID() != mPermissions.getOwner()) - { + if (gAgent.getID() != mPermissions.getOwner()) + { url = region->getCapability("FetchLib2"); - } + } else - { + { url = region->getCapability("FetchInventory2"); - } + } } else { - LL_WARNS() << "Agent Region is absent" << LL_ENDL; + LL_WARNS(LOG_INV) << "Agent Region is absent" << LL_ENDL; } if (!url.empty()) @@ -413,7 +417,8 @@ void LLViewerInventoryItem::fetchFromServer(void) const body["items"][0]["owner_id"] = mPermissions.getOwner(); body["items"][0]["item_id"] = mUUID; - LLHTTPClient::post(url, body, new LLInventoryModel::fetchInventoryResponder(body)); + LLInventoryModel::FetchItemHttpHandler * handler(new LLInventoryModel::FetchItemHttpHandler(body)); + gInventory.requestPost(true, url, body, handler, "Inventory Item"); } else { @@ -649,7 +654,7 @@ bool LLViewerInventoryCategory::fetch() if((VERSION_UNKNOWN == getVersion()) && mDescendentsRequested.hasExpired()) //Expired check prevents multiple downloads. { - LL_DEBUGS("InventoryFetch") << "Fetching category children: " << mName << ", UUID: " << mUUID << LL_ENDL; + LL_DEBUGS(LOG_INV) << "Fetching category children: " << mName << ", UUID: " << mUUID << LL_ENDL; const F32 FETCH_TIMER_EXPIRY = 10.0f; mDescendentsRequested.reset(); mDescendentsRequested.setTimerExpirySec(FETCH_TIMER_EXPIRY); @@ -674,7 +679,7 @@ bool LLViewerInventoryCategory::fetch() } else { - LL_WARNS() << "agent region is null" << LL_ENDL; + LL_WARNS(LOG_INV) << "agent region is null" << LL_ENDL; } if (!url.empty()) //Capability found. Build up LLSD and use it. { @@ -682,7 +687,8 @@ bool LLViewerInventoryCategory::fetch() } else { //Deprecated, but if we don't have a capability, use the old system. - LL_INFOS() << "FetchInventoryDescendents2 capability not found. Using deprecated UDP message." << LL_ENDL; + LL_INFOS(LOG_INV) << "FetchInventoryDescendents2 capability not found. Using deprecated UDP message." << LL_ENDL; + LLMessageSystem* msg = gMessageSystem; msg->newMessage("FetchInventoryDescendents"); msg->nextBlock("AgentData"); @@ -777,8 +783,8 @@ bool LLViewerInventoryCategory::importFileLocal(LLFILE* fp) } else { - LL_WARNS() << "unknown keyword '" << keyword - << "' in inventory import category " << mUUID << LL_ENDL; + LL_WARNS(LOG_INV) << "unknown keyword '" << keyword + << "' in inventory import category " << mUUID << LL_ENDL; } } return true; @@ -906,7 +912,7 @@ LLInventoryCallbackManager::LLInventoryCallbackManager() : { if( sInstance != NULL ) { - LL_WARNS() << "LLInventoryCallbackManager::LLInventoryCallbackManager: unexpected multiple instances" << LL_ENDL; + LL_WARNS(LOG_INV) << "LLInventoryCallbackManager::LLInventoryCallbackManager: unexpected multiple instances" << LL_ENDL; return; } sInstance = this; @@ -916,7 +922,7 @@ LLInventoryCallbackManager::~LLInventoryCallbackManager() { if( sInstance != this ) { - LL_WARNS() << "LLInventoryCallbackManager::~LLInventoryCallbackManager: unexpected multiple instances" << LL_ENDL; + LL_WARNS(LOG_INV) << "LLInventoryCallbackManager::~LLInventoryCallbackManager: unexpected multiple instances" << LL_ENDL; return; } sInstance = NULL; @@ -1144,7 +1150,7 @@ void link_inventory_object(const LLUUID& category, { if (!baseobj) { - LL_WARNS() << "Attempt to link to non-existent object" << LL_ENDL; + LL_WARNS(LOG_INV) << "Attempt to link to non-existent object" << LL_ENDL; return; } @@ -1178,7 +1184,7 @@ void link_inventory_array(const LLUUID& category, const LLInventoryObject* baseobj = *it; if (!baseobj) { - LL_WARNS() << "attempt to link to unknown object" << LL_ENDL; + LL_WARNS(LOG_INV) << "attempt to link to unknown object" << LL_ENDL; continue; } @@ -1187,7 +1193,7 @@ void link_inventory_array(const LLUUID& category, // Fail if item can be found but is of a type that can't be linked. // Arguably should fail if the item can't be found too, but that could // be a larger behavioral change. - LL_WARNS() << "attempt to link an unlinkable object, type = " << baseobj->getActualType() << LL_ENDL; + LL_WARNS(LOG_INV) << "attempt to link an unlinkable object, type = " << baseobj->getActualType() << LL_ENDL; continue; } @@ -1223,7 +1229,7 @@ void link_inventory_array(const LLUUID& category, } else { - LL_WARNS() << "could not convert object into an item or category: " << baseobj->getUUID() << LL_ENDL; + LL_WARNS(LOG_INV) << "could not convert object into an item or category: " << baseobj->getUUID() << LL_ENDL; continue; } } @@ -1237,10 +1243,10 @@ void link_inventory_array(const LLUUID& category, links.append(link); #ifndef LL_RELEASE_FOR_DOWNLOAD - LL_DEBUGS("Inventory") << "Linking Object [ name:" << baseobj->getName() - << " UUID:" << baseobj->getUUID() - << " ] into Category [ name:" << cat_name - << " UUID:" << category << " ] " << LL_ENDL; + LL_DEBUGS(LOG_INV) << "Linking Object [ name:" << baseobj->getName() + << " UUID:" << baseobj->getUUID() + << " ] into Category [ name:" << cat_name + << " UUID:" << category << " ] " << LL_ENDL; #endif } @@ -1331,7 +1337,7 @@ void update_inventory_item( if (!ais_ran) { LLPointer obj = gInventory.getItem(item_id); - LL_DEBUGS("Inventory") << "item_id: [" << item_id << "] name " << (update_item ? update_item->getName() : "(NOT FOUND)") << LL_ENDL; + LL_DEBUGS(LOG_INV) << "item_id: [" << item_id << "] name " << (update_item ? update_item->getName() : "(NOT FOUND)") << LL_ENDL; if(obj) { LLMessageSystem* msg = gMessageSystem; @@ -1373,7 +1379,7 @@ void update_inventory_item( if (!ais_ran) { LLPointer obj = gInventory.getItem(item_id); - LL_DEBUGS("Inventory") << "item_id: [" << item_id << "] name " << (obj ? obj->getName() : "(NOT FOUND)") << LL_ENDL; + LL_DEBUGS(LOG_INV) << "item_id: [" << item_id << "] name " << (obj ? obj->getName() : "(NOT FOUND)") << LL_ENDL; if(obj) { LLPointer new_item(new LLViewerInventoryItem); @@ -1408,7 +1414,7 @@ void update_inventory_category( LLPointer cb) { LLPointer obj = gInventory.getCategory(cat_id); - LL_DEBUGS("Inventory") << "cat_id: [" << cat_id << "] name " << (obj ? obj->getName() : "(NOT FOUND)") << LL_ENDL; + LL_DEBUGS(LOG_INV) << "cat_id: [" << cat_id << "] name " << (obj ? obj->getName() : "(NOT FOUND)") << LL_ENDL; if(obj) { if (LLFolderType::lookupIsProtectedType(obj->getPreferredType())) @@ -1471,7 +1477,7 @@ void remove_inventory_item( } else { - LL_DEBUGS("Inventory") << "item_id: [" << item_id << "] name " << "(NOT FOUND)" << LL_ENDL; + LL_DEBUGS(LOG_INV) << "item_id: [" << item_id << "] name " << "(NOT FOUND)" << LL_ENDL; } } @@ -1482,7 +1488,7 @@ void remove_inventory_item( if(obj) { const LLUUID item_id(obj->getUUID()); - LL_DEBUGS("Inventory") << "item_id: [" << item_id << "] name " << obj->getName() << LL_ENDL; + LL_DEBUGS(LOG_INV) << "item_id: [" << item_id << "] name " << obj->getName() << LL_ENDL; if (AISCommand::isAPIAvailable()) { LLPointer cmd_ptr = new RemoveItemCommand(item_id, cb); @@ -1511,7 +1517,7 @@ void remove_inventory_item( else { // *TODO: Clean up callback? - LL_WARNS() << "remove_inventory_item called for invalid or nonexistent item." << LL_ENDL; + LL_WARNS(LOG_INV) << "remove_inventory_item called for invalid or nonexistent item." << LL_ENDL; } } @@ -1529,7 +1535,7 @@ public: LLInventoryModel::EHasChildren children = gInventory.categoryHasChildren(mID); if(children != LLInventoryModel::CHILDREN_NO) { - LL_WARNS() << "remove descendents failed, cannot remove category " << LL_ENDL; + LL_WARNS(LOG_INV) << "remove descendents failed, cannot remove category " << LL_ENDL; } else { @@ -1545,7 +1551,7 @@ void remove_inventory_category( const LLUUID& cat_id, LLPointer cb) { - LL_DEBUGS("Inventory") << "cat_id: [" << cat_id << "] " << LL_ENDL; + LL_DEBUGS(LOG_INV) << "cat_id: [" << cat_id << "] " << LL_ENDL; LLPointer obj = gInventory.getCategory(cat_id); if(obj) { @@ -1566,7 +1572,7 @@ void remove_inventory_category( LLInventoryModel::EHasChildren children = gInventory.categoryHasChildren(cat_id); if(children != LLInventoryModel::CHILDREN_NO) { - LL_DEBUGS("Inventory") << "Will purge descendents first before deleting category " << cat_id << LL_ENDL; + LL_DEBUGS(LOG_INV) << "Will purge descendents first before deleting category " << cat_id << LL_ENDL; LLPointer wrap_cb = new LLRemoveCategoryOnDestroy(cat_id, cb); purge_descendents_of(cat_id, wrap_cb); return; @@ -1592,7 +1598,7 @@ void remove_inventory_category( } else { - LL_WARNS() << "remove_inventory_category called for invalid or nonexistent item " << cat_id << LL_ENDL; + LL_WARNS(LOG_INV) << "remove_inventory_category called for invalid or nonexistent item " << cat_id << LL_ENDL; } } @@ -1620,7 +1626,7 @@ void purge_descendents_of(const LLUUID& id, LLPointer cb) LLInventoryModel::EHasChildren children = gInventory.categoryHasChildren(id); if(children == LLInventoryModel::CHILDREN_NO) { - LL_DEBUGS("Inventory") << "No descendents to purge for " << id << LL_ENDL; + LL_DEBUGS(LOG_INV) << "No descendents to purge for " << id << LL_ENDL; return; } LLPointer cat = gInventory.getCategory(id); @@ -1629,8 +1635,8 @@ void purge_descendents_of(const LLUUID& id, LLPointer cb) if (LLClipboard::instance().hasContents() && LLClipboard::instance().isCutMode()) { // Something on the clipboard is in "cut mode" and needs to be preserved - LL_DEBUGS("Inventory") << "purge_descendents_of clipboard case " << cat->getName() - << " iterate and purge non hidden items" << LL_ENDL; + LL_DEBUGS(LOG_INV) << "purge_descendents_of clipboard case " << cat->getName() + << " iterate and purge non hidden items" << LL_ENDL; LLInventoryModel::cat_array_t* categories; LLInventoryModel::item_array_t* items; // Get the list of direct descendants in tha categoy passed as argument @@ -1665,7 +1671,7 @@ void purge_descendents_of(const LLUUID& id, LLPointer cb) else // no cap { // Fast purge - LL_DEBUGS("Inventory") << "purge_descendents_of fast case " << cat->getName() << LL_ENDL; + LL_DEBUGS(LOG_INV) << "purge_descendents_of fast case " << cat->getName() << LL_ENDL; // send it upstream LLMessageSystem* msg = gMessageSystem; @@ -1708,9 +1714,9 @@ void copy_inventory_from_notecard(const LLUUID& destination_id, { if (NULL == src) { - LL_WARNS("copy_inventory_from_notecard") << "Null pointer to item was passed for object_id " - << object_id << " and notecard_inv_id " - << notecard_inv_id << LL_ENDL; + LL_WARNS(LOG_NOTECARD) << "Null pointer to item was passed for object_id " + << object_id << " and notecard_inv_id " + << notecard_inv_id << LL_ENDL; return; } @@ -1730,9 +1736,9 @@ void copy_inventory_from_notecard(const LLUUID& destination_id, if (! viewer_region) { - LL_WARNS("copy_inventory_from_notecard") << "Can't find region from object_id " - << object_id << " or gAgent" - << LL_ENDL; + LL_WARNS(LOG_NOTECARD) << "Can't find region from object_id " + << object_id << " or gAgent" + << LL_ENDL; return; } @@ -1740,9 +1746,9 @@ void copy_inventory_from_notecard(const LLUUID& destination_id, std::string url = viewer_region->getCapability("CopyInventoryFromNotecard"); if (url.empty()) { - LL_WARNS("copy_inventory_from_notecard") << "There is no 'CopyInventoryFromNotecard' capability" - << " for region: " << viewer_region->getName() - << LL_ENDL; + LL_WARNS(LOG_NOTECARD) << "There is no 'CopyInventoryFromNotecard' capability" + << " for region: " << viewer_region->getName() + << LL_ENDL; return; } @@ -1816,15 +1822,15 @@ void slam_inventory_folder(const LLUUID& folder_id, { if (AISCommand::isAPIAvailable()) { - LL_DEBUGS("Avatar") << "using AISv3 to slam folder, id " << folder_id - << " new contents: " << ll_pretty_print_sd(contents) << LL_ENDL; + LL_DEBUGS(LOG_INV) << "using AISv3 to slam folder, id " << folder_id + << " new contents: " << ll_pretty_print_sd(contents) << LL_ENDL; LLPointer cmd_ptr = new SlamFolderCommand(folder_id, contents, cb); cmd_ptr->run_command(); } else // no cap { - LL_DEBUGS("Avatar") << "using item-by-item calls to slam folder, id " << folder_id - << " new contents: " << ll_pretty_print_sd(contents) << LL_ENDL; + LL_DEBUGS(LOG_INV) << "using item-by-item calls to slam folder, id " << folder_id + << " new contents: " << ll_pretty_print_sd(contents) << LL_ENDL; for (LLSD::array_const_iterator it = contents.beginArray(); it != contents.endArray(); ++it) @@ -1926,7 +1932,7 @@ void menu_create_inventory_item(LLInventoryPanel* panel, LLFolderBridge *bridge, } else { - LL_WARNS() << "Can't create unrecognized type " << type_name << LL_ENDL; + LL_WARNS(LOG_INV) << "Can't create unrecognized type " << type_name << LL_ENDL; } } panel->getRootFolder()->setNeedsAutoRename(TRUE); @@ -2161,7 +2167,7 @@ LLViewerInventoryItem *LLViewerInventoryItem::getLinkedItem() const LLViewerInventoryItem *linked_item = gInventory.getItem(mAssetUUID); if (linked_item && linked_item->getIsLinkType()) { - LL_WARNS() << "Warning: Accessing link to link" << LL_ENDL; + LL_WARNS(LOG_INV) << "Warning: Accessing link to link" << LL_ENDL; return NULL; } return linked_item; diff --git a/indra/newview/llviewermenu.cpp b/indra/newview/llviewermenu.cpp index 8c9429c05d..10c77da5f3 100755 --- a/indra/newview/llviewermenu.cpp +++ b/indra/newview/llviewermenu.cpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2002&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2010, Linden Research, Inc. + * Copyright (C) 2014, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -8258,7 +8258,7 @@ class LLWorldEnableEnvSettings : public view_listener_t } else { - llwarns << "Unknown item" << llendl; + LL_WARNS() << "Unknown time-of-day item: " << tod << LL_ENDL; } } return result; -- cgit v1.2.3 From b64ef2ecd4af5265483527f2ef030554133ab137 Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Fri, 22 Aug 2014 19:00:11 -0400 Subject: Fix ambiguous constructor due to LLSD access which broke *ix builds. --- indra/newview/llinventorymodelbackgroundfetch.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'indra/newview') diff --git a/indra/newview/llinventorymodelbackgroundfetch.cpp b/indra/newview/llinventorymodelbackgroundfetch.cpp index 443c54df42..7b944edf45 100755 --- a/indra/newview/llinventorymodelbackgroundfetch.cpp +++ b/indra/newview/llinventorymodelbackgroundfetch.cpp @@ -870,7 +870,7 @@ void BGFolderHttpHandler::processFailure(LLCore::HttpStatus status, LLCore::Http ++folder_it) { LLSD folder_sd(*folder_it); - LLUUID folder_id(folder_sd["folder_id"]); + LLUUID folder_id(folder_sd["folder_id"].asUUID()); const BOOL recursive = getIsRecursive(folder_id); fetcher->addRequestAtFront(folder_id, recursive, true); } @@ -900,7 +900,7 @@ void BGFolderHttpHandler::processFailure(const char * const reason, LLCore::Http ++folder_it) { LLSD folder_sd(*folder_it); - LLUUID folder_id(folder_sd["folder_id"]); + LLUUID folder_id(folder_sd["folder_id"].asUUID()); const BOOL recursive = getIsRecursive(folder_id); fetcher->addRequestAtFront(folder_id, recursive, true); } -- cgit v1.2.3 From bbf9de9c6717f38a77a39d42d8493d275d558db9 Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Tue, 26 Aug 2014 18:33:14 -0400 Subject: Bring better error handling to inventory item and folder fetching. First, introduced some LLSD-based interfaces to the llcorehttp code using utils classes (in llcorehttputil). I've kept LLSD out of the llcorehttp library up to now and will continue to do that. Functions provide a requestPost based on LLSD body and conversion utils for HttpResponse-to-LLSD and HttpResponse-to-string conversions. Inventory fetch operations now do more thorough error checking including 200-with-error status checking. Still do retry forever on folders though I don't like that. --- indra/newview/llinventorymodel.cpp | 76 ++++++++------ indra/newview/llinventorymodelbackgroundfetch.cpp | 115 ++++++++++++++++------ 2 files changed, 130 insertions(+), 61 deletions(-) (limited to 'indra/newview') diff --git a/indra/newview/llinventorymodel.cpp b/indra/newview/llinventorymodel.cpp index 2dd31f047f..5273fb6d96 100755 --- a/indra/newview/llinventorymodel.cpp +++ b/indra/newview/llinventorymodel.cpp @@ -55,7 +55,7 @@ #include "llsdutil.h" #include "bufferarray.h" #include "bufferstream.h" -#include "llsdserialize.h" +#include "llcorehttputil.h" //#define DIFF_INVENTORY_FILES #ifdef DIFF_INVENTORY_FILES @@ -75,12 +75,6 @@ BOOL LLInventoryModel::sFirstTimeInViewer2 = TRUE; static const char CACHE_FORMAT_STRING[] = "%s.inv"; static const char * const LOG_INV("Inventory"); -static std::string dumpResponse() -{ - return std::string("ADD SOMETHING MEANINGFUL HERE"); -} - - struct InventoryIDPtrLess { bool operator()(const LLViewerInventoryCategory* i1, const LLViewerInventoryCategory* i2) const @@ -2468,18 +2462,15 @@ LLCore::HttpHandle LLInventoryModel::requestPost(bool foreground, LLCore::HttpRequest * request(foreground ? mHttpRequestFG : mHttpRequestBG); LLCore::HttpHandle handle(LLCORE_HTTP_HANDLE_INVALID); - LLCore::BufferArray * ba = new LLCore::BufferArray; - LLCore::BufferArrayStream bas(ba); - LLSDSerialize::toXML(body, bas); - handle = request->requestPost(mHttpPolicyClass, - (foreground ? mHttpPriorityFG : mHttpPriorityBG), - url, - ba, - mHttpOptions, - mHttpHeaders, - handler); - ba->release(); + handle = LLCoreHttpUtil::requestPostWithLLSD(request, + mHttpPolicyClass, + (foreground ? mHttpPriorityFG : mHttpPriorityBG), + url, + body, + mHttpOptions, + mHttpHeaders, + handler); if (LLCORE_HTTP_HANDLE_INVALID == handle) { LLCore::HttpStatus status(request->getStatus()); @@ -3981,6 +3972,7 @@ void LLInventoryModel::FetchItemHttpHandler::onCompleted(LLCore::HttpHandle hand LLCore::HttpResponse * response) { LLCore::HttpStatus status(response->getStatus()); + // status = LLCore::HttpStatus(404); // Dev tool to force error handling if (! status) { processFailure(status, response); @@ -3988,23 +3980,41 @@ void LLInventoryModel::FetchItemHttpHandler::onCompleted(LLCore::HttpHandle hand else { LLCore::BufferArray * body(response->getBody()); + // body = NULL; // Dev tool to force error handling if (! body || ! body->size()) { LL_WARNS(LOG_INV) << "Missing data in inventory item query." << LL_ENDL; processFailure("HTTP response for inventory item query missing body", response); goto only_exit; } - - LLCore::BufferArrayStream bas(body); + + // body->write(0, "Garbage Response", 16); // Dev tool to force error handling LLSD body_llsd; - S32 parse_status(LLSDSerialize::fromXML(body_llsd, bas)); - if (LLSDParser::PARSE_FAILURE == parse_status) + if (! LLCoreHttpUtil::responseToLLSD(response, true, body_llsd)) { // INFOS-level logging will occur on the parsed failure processFailure("HTTP response for inventory item query has malformed LLSD", response); goto only_exit; } + // Expect top-level structure to be a map + // body_llsd = LLSD::emptyArray(); // Dev tool to force error handling + if (! body_llsd.isMap()) + { + processFailure("LLSD response for inventory item not a map", response); + goto only_exit; + } + + // Check for 200-with-error failures + // body_llsd["error"] = LLSD::emptyMap(); // Dev tool to force error handling + // body_llsd["error"]["identifier"] = "Development"; + // body_llsd["error"]["message"] = "You left development code in the viewer"; + if (body_llsd.has("error")) + { + processFailure("Inventory application error (200-with-error)", response); + goto only_exit; + } + // Okay, process data if possible processData(body_llsd, response); } @@ -4016,12 +4026,6 @@ only_exit: void LLInventoryModel::FetchItemHttpHandler::processData(LLSD & content, LLCore::HttpResponse * response) { - if (! content.isMap()) - { - processFailure("LLSD response for inventory item not a map", response); - return; - } - start_new_inventory_observer(); #if 0 @@ -4083,7 +4087,7 @@ void LLInventoryModel::FetchItemHttpHandler::processData(LLSD & content, LLCore: { changes |= gInventory.updateItem(*it); } - // *HUH: Have computed changes, nothing uses it. + // *HUH: Have computed 'changes', nothing uses it. gInventory.notifyObservers(); gViewerWindow->getWindow()->decBusyCount(); @@ -4092,13 +4096,23 @@ void LLInventoryModel::FetchItemHttpHandler::processData(LLSD & content, LLCore: void LLInventoryModel::FetchItemHttpHandler::processFailure(LLCore::HttpStatus status, LLCore::HttpResponse * response) { - LL_WARNS(LOG_INV) << dumpResponse() << LL_ENDL; + const std::string & ct(response->getContentType()); + LL_WARNS(LOG_INV) << "Inventory item fetch failure\n" + << "[Status: " << status.toTerseString() << "]\n" + << "[Reason: " << status.toString() << "]\n" + << "[Content-type: " << ct << "]\n" + << "[Content (abridged): " + << LLCoreHttpUtil::responseToString(response) << "]" << LL_ENDL; gInventory.notifyObservers(); } void LLInventoryModel::FetchItemHttpHandler::processFailure(const char * const reason, LLCore::HttpResponse * response) { - LL_WARNS(LOG_INV) << dumpResponse() << LL_ENDL; + LL_WARNS(LOG_INV) << "Inventory item fetch failure\n" + << "[Status: internal error]\n" + << "[Reason: " << reason << "]\n" + << "[Content (abridged): " + << LLCoreHttpUtil::responseToString(response) << "]" << LL_ENDL; gInventory.notifyObservers(); } diff --git a/indra/newview/llinventorymodelbackgroundfetch.cpp b/indra/newview/llinventorymodelbackgroundfetch.cpp index 7b944edf45..0c04a9c039 100755 --- a/indra/newview/llinventorymodelbackgroundfetch.cpp +++ b/indra/newview/llinventorymodelbackgroundfetch.cpp @@ -40,17 +40,24 @@ #include "llhttpconstants.h" #include "bufferarray.h" #include "bufferstream.h" -#include "llsdserialize.h" +#include "llcorehttputil.h" namespace { +// // Http request handler class for single inventory item requests. // // We'll use a handler-per-request pattern here rather than // a shared handler. Mainly convenient as this was converted // from a Responder class model. // +// Derives from and is identical to the normal FetchItemHttpHandler +// except that: 1) it uses the background request object which is +// updated more slowly than the foreground and 2) keeps a count of +// active requests on the LLInventoryModelBackgroundFetch object +// to indicate outstanding operations are in-flight. +// class BGItemHttpHandler : public LLInventoryModel::FetchItemHttpHandler { LOG_CLASS(BGItemHttpHandler); @@ -74,6 +81,10 @@ protected: // Http request handler class for folders. +// +// Handler for FetchInventoryDescendents2 and FetchLibDescendents2 +// caps requests for folders. +// class BGFolderHttpHandler : public LLCore::HttpHandler { LOG_CLASS(BGFolderHttpHandler); @@ -116,12 +127,6 @@ const S32 MAX_FETCH_RETRIES = 10; const char * const LOG_INV("Inventory"); -std::string dumpResponse() -{ - return std::string("ADD SOMETHING MEANINGFUL HERE"); -} - - } // end of namespace anonymous @@ -471,6 +476,10 @@ void LLInventoryModelBackgroundFetch::bulkFetch() return; } + // *TODO: These values could be tweaked at runtime to effect + // a fast/slow fetch throttle. Once login is complete and the scene + // is mostly loaded, we could turn up the throttle and fill missing + // inventory more quickly. static const S32 max_concurrent_fetches(12); // Outstanding requests, not connections static const F32 new_min_time(0.5f); // *HACK: Clean this up when old code goes away entirely. static const U32 max_batch_size(10); @@ -482,7 +491,7 @@ void LLInventoryModelBackgroundFetch::bulkFetch() if (mFetchCount) { - // Process completed HTTP requests + // Process completed background HTTP requests gInventory.handleResponses(false); } @@ -497,6 +506,8 @@ void LLInventoryModelBackgroundFetch::bulkFetch() const U32 sort_order(gSavedSettings.getU32(LLInventoryPanel::DEFAULT_SORT_ORDER) & 0x1); + // *TODO: Think I'd like to get a shared pointer to this and share it + // among all the folder requests. uuid_vec_t recursive_cats; LLSD folder_request_body; @@ -673,7 +684,9 @@ bool LLInventoryModelBackgroundFetch::fetchQueueContainsNoDescendentsOf(const LL } +// =============================== // Anonymous Namespace Definitions +// =============================== namespace { @@ -683,30 +696,53 @@ namespace void BGFolderHttpHandler::onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * response) { LLCore::HttpStatus status(response->getStatus()); + // status = LLCore::HttpStatus(404); // Dev tool to force error handling if (! status) { processFailure(status, response); } else { + // Response body should be present. LLCore::BufferArray * body(response->getBody()); + // body = NULL; // Dev tool to force error handling if (! body || ! body->size()) { LL_WARNS(LOG_INV) << "Missing data in inventory folder query." << LL_ENDL; processFailure("HTTP response missing expected body", response); goto only_exit; } - - LLCore::BufferArrayStream bas(body); + + // Could test 'Content-Type' header but probably unreliable. + + // Convert response to LLSD + // body->write(0, "Garbage Response", 16); // Dev tool to force error handling LLSD body_llsd; - S32 parse_status(LLSDSerialize::fromXML(body_llsd, bas)); - if (LLSDParser::PARSE_FAILURE == parse_status) + if (! LLCoreHttpUtil::responseToLLSD(response, true, body_llsd)) { // INFOS-level logging will occur on the parsed failure processFailure("HTTP response contained malformed LLSD", response); goto only_exit; } + // Expect top-level structure to be a map + // body_llsd = LLSD::emptyArray(); // Dev tool to force error handling + if (! body_llsd.isMap()) + { + processFailure("LLSD response not a map", response); + goto only_exit; + } + + // Check for 200-with-error failures + // body_llsd["error"] = LLSD::emptyMap(); // Dev tool to force error handling + // body_llsd["error"]["identifier"] = "Development"; + // body_llsd["error"]["message"] = "You left development code in the viewer"; + if (body_llsd.has("error")) + { + processFailure("Inventory application error (200-with-error)", response); + goto only_exit; + } + // Okay, process data if possible processData(body_llsd, response); } @@ -719,13 +755,12 @@ only_exit: void BGFolderHttpHandler::processData(LLSD & content, LLCore::HttpResponse * response) { - if (! content.isMap()) - { - processFailure("LLSD response not a map", response); - return; - } - LLInventoryModelBackgroundFetch * fetcher(LLInventoryModelBackgroundFetch::getInstance()); + + // API V2 and earlier should probably be testing for "error" map + // in response as an application-level error. + + // Instead, we assume success and attempt to extract information. if (content.has("folders")) { LLSD folders(content["folders"]); @@ -836,7 +871,7 @@ void BGFolderHttpHandler::processData(LLSD & content, LLCore::HttpResponse * res folder_it != bad_folders.endArray(); ++folder_it) { - // *TODO: Stop copying data + // *TODO: Stop copying data [ed: this isn't copying data] LLSD folder_sd(*folder_it); // These folders failed on the dataserver. We probably don't want to retry them. @@ -857,14 +892,26 @@ void BGFolderHttpHandler::processData(LLSD & content, LLCore::HttpResponse * res void BGFolderHttpHandler::processFailure(LLCore::HttpStatus status, LLCore::HttpResponse * response) { - LL_WARNS(LOG_INV) << dumpResponse() << LL_ENDL; + const std::string & ct(response->getContentType()); + LL_WARNS(LOG_INV) << "Inventory folder fetch failure\n" + << "[Status: " << status.toTerseString() << "]\n" + << "[Reason: " << status.toString() << "]\n" + << "[Content-type: " << ct << "]\n" + << "[Content (abridged): " + << LLCoreHttpUtil::responseToString(response) << "]" << LL_ENDL; + + // Could use a 404 test here to try to detect revoked caps... + + // This was originally the request retry logic for the inventory + // request which tested on HTTP_INTERNAL_ERROR status. This + // retry logic was unbounded and lacked discrimination as to the + // cause of the retry. The new http library should be doing + // adquately on retries but I want to keep the structure of a + // retry for reference. LLInventoryModelBackgroundFetch *fetcher = LLInventoryModelBackgroundFetch::getInstance(); - - LL_INFOS(LOG_INV) << dumpResponse() << LL_ENDL; - - // *FIX: Not the correct test here... - if (status == LLCore::HttpStatus(408)) // timed out or curl failure + if (false) { + // timed out or curl failure for (LLSD::array_const_iterator folder_it = mRequestSD["folders"].beginArray(); folder_it != mRequestSD["folders"].endArray(); ++folder_it) @@ -888,12 +935,20 @@ void BGFolderHttpHandler::processFailure(LLCore::HttpStatus status, LLCore::Http void BGFolderHttpHandler::processFailure(const char * const reason, LLCore::HttpResponse * response) { - LL_WARNS(LOG_INV) << dumpResponse() << LL_ENDL; + LL_WARNS(LOG_INV) << "Inventory folder fetch failure\n" + << "[Status: internal error]\n" + << "[Reason: " << reason << "]\n" + << "[Content (abridged): " + << LLCoreHttpUtil::responseToString(response) << "]" << LL_ENDL; + + // Reverse of previous processFailure() method, this is invoked + // when response structure is found to be invalid. Original + // always re-issued the request (without limit). This does + // the same but be aware that this may be a source of problems. + // Philosophy is that inventory folders are so essential to + // operation that this is a reasonable action. LLInventoryModelBackgroundFetch *fetcher = LLInventoryModelBackgroundFetch::getInstance(); - - LL_INFOS(LOG_INV) << dumpResponse() << LL_ENDL; - - if (true /* status == LLCore::HttpStatus(408)*/) // timed out or curl failure + if (true) { for (LLSD::array_const_iterator folder_it = mRequestSD["folders"].beginArray(); folder_it != mRequestSD["folders"].endArray(); -- cgit v1.2.3 From 0c20beda6800149ee71a307ca4e943b5bba56908 Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Thu, 4 Sep 2014 16:57:44 -0400 Subject: Pipelining work. Extend transfer timeout by the pipeline depth as transfers can appear delayed with deep pipelining and more requests in the pool. Added bad HTTP status error (typically getting a 0 back as HTTP status from libcurl) to the list of retryable errors. There's a response stream problem with libcurl and pipelining that induces this problem. Retrying helps but may not be entirely safe. Watch bug 1420 on the libcurl sourceforge bug tracker. Extend options of test/example program to include un-ranged requests. Document the excessive data transfer induced when ranged requests are disabled. This is an abnormal mode for very rare users so we'll just eat that for now. --- indra/newview/app_settings/settings.xml | 4 ++-- indra/newview/llmeshrepository.cpp | 13 ++++++++++++- 2 files changed, 14 insertions(+), 3 deletions(-) (limited to 'indra/newview') diff --git a/indra/newview/app_settings/settings.xml b/indra/newview/app_settings/settings.xml index 0607579a08..ab15a03229 100755 --- a/indra/newview/app_settings/settings.xml +++ b/indra/newview/app_settings/settings.xml @@ -4459,7 +4459,7 @@ HttpPipelining Comment - If true, viewer will pipeline HTTP requests to servers. + If true, viewer will attempt to pipeline HTTP requests. Persist 1 Type @@ -4470,7 +4470,7 @@ HttpRangeRequestsDisable Comment - If true, viewer will not issued range GET requests for meshes and textures. May resolve problems with certain ISPs and networking gear. + If true, viewer will not issue GET requests with 'Range:' headers for meshes and textures. May resolve problems with certain ISPs and networking gear. Persist 1 Type diff --git a/indra/newview/llmeshrepository.cpp b/indra/newview/llmeshrepository.cpp index 6477389d4c..a6707392fe 100755 --- a/indra/newview/llmeshrepository.cpp +++ b/indra/newview/llmeshrepository.cpp @@ -2644,6 +2644,17 @@ void LLMeshRepository::cacheOutgoingMesh(LLMeshUploadData& data, LLSD& header) } +// Handle failed or successful requests for mesh assets. +// +// Support for 200 responses was added for several reasons. One, +// a service or cache can ignore range headers and give us a +// 200 with full asset should it elect to. We also support +// a debug flag which disables range requests for those very +// few users that have some sort of problem with their networking +// services. But the 200 response handling is suboptimal: rather +// than cache the whole asset, we just extract the part that would +// have been sent in a 206 and process that. Inefficient but these +// are cases far off the norm. void LLMeshHandlerBase::onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * response) { mProcessed = true; @@ -2685,7 +2696,7 @@ void LLMeshHandlerBase::onCompleted(LLCore::HttpHandle handle, LLCore::HttpRespo if (par_status == status) { - // 216 case + // 206 case response->getRange(&offset, &length, &full_length); if (! offset && ! length) { -- cgit v1.2.3 From bd0a9b6b4c929f640164b623f5183d371404bb96 Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Tue, 9 Sep 2014 15:30:54 -0400 Subject: LLSlurl and LLViewerNetwork unit test fixes. Got tired of the TC builds failing on these tests. Appears to be a conflict/race condition on a temporary file. Changed the name of the file. Should use a tempname()-like API but this will do for a unit test. --- indra/newview/tests/llslurl_test.cpp | 23 ++++++++++++++++------- indra/newview/tests/llviewernetwork_test.cpp | 23 +++++++++++++++-------- 2 files changed, 31 insertions(+), 15 deletions(-) (limited to 'indra/newview') diff --git a/indra/newview/tests/llslurl_test.cpp b/indra/newview/tests/llslurl_test.cpp index 86229ad636..2bc0d5a086 100755 --- a/indra/newview/tests/llslurl_test.cpp +++ b/indra/newview/tests/llslurl_test.cpp @@ -6,7 +6,7 @@ * * $LicenseInfo:firstyear=2009&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2010, Linden Research, Inc. + * Copyright (C) 2014, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -31,6 +31,15 @@ #include "../llslurl.h" #include "../../llxml/llcontrol.h" #include "llsdserialize.h" + +namespace +{ + +// Should not collide with other test programs creating temp files. +static const char * const TEST_FILENAME("llslurl_test.xml"); + +} + //---------------------------------------------------------------------------- // Mock objects for the dependencies of the code we're testing @@ -143,11 +152,11 @@ namespace tut template<> template<> void slurlTestObject::test<1>() { - llofstream gridfile("grid_test.xml"); + llofstream gridfile(TEST_FILENAME); gridfile << gSampleGridFile; gridfile.close(); - LLGridManager::getInstance()->initialize("grid_test.xml"); + LLGridManager::getInstance()->initialize(TEST_FILENAME); LLGridManager::getInstance()->setGridChoice("util.agni.lindenlab.com"); @@ -260,11 +269,11 @@ namespace tut template<> template<> void slurlTestObject::test<2>() { - llofstream gridfile("grid_test.xml"); + llofstream gridfile(TEST_FILENAME); gridfile << gSampleGridFile; gridfile.close(); - LLGridManager::getInstance()->initialize("grid_test.xml"); + LLGridManager::getInstance()->initialize(TEST_FILENAME); LLSLURL slurl = LLSLURL("my.grid.com", "my region"); ensure_equals("grid/region - type", slurl.getType(), LLSLURL::LOCATION); @@ -293,11 +302,11 @@ namespace tut template<> template<> void slurlTestObject::test<3>() { - llofstream gridfile("grid_test.xml"); + llofstream gridfile(TEST_FILENAME); gridfile << gSampleGridFile; gridfile.close(); - LLGridManager::getInstance()->initialize("grid_test.xml"); + LLGridManager::getInstance()->initialize(TEST_FILENAME); LLGridManager::getInstance()->setGridChoice("my.grid.com"); LLSLURL slurl = LLSLURL("https://my.grid.com/region/my%20region/1/2/3"); diff --git a/indra/newview/tests/llviewernetwork_test.cpp b/indra/newview/tests/llviewernetwork_test.cpp index 7ad7947ca4..0eb0ab6500 100755 --- a/indra/newview/tests/llviewernetwork_test.cpp +++ b/indra/newview/tests/llviewernetwork_test.cpp @@ -6,7 +6,7 @@ * * $LicenseInfo:firstyear=2009&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2010, Linden Research, Inc. + * Copyright (C) 2014, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -31,6 +31,13 @@ #include "../../llxml/llcontrol.h" #include "llfile.h" +namespace +{ + +// Should not collide with other test programs creating temp files. +static const char * const TEST_FILENAME("llviewernetwork_test.xml"); + +} //---------------------------------------------------------------------------- // Mock objects for the dependencies of the code we're testing @@ -143,7 +150,7 @@ namespace tut { viewerNetworkTest() { - LLFile::remove("grid_test.xml"); + LLFile::remove(TEST_FILENAME); gCmdLineLoginURI.clear(); gCmdLineGridChoice.clear(); gCmdLineHelperURI.clear(); @@ -152,7 +159,7 @@ namespace tut } ~viewerNetworkTest() { - LLFile::remove("grid_test.xml"); + LLFile::remove(TEST_FILENAME); } }; @@ -170,7 +177,7 @@ namespace tut { LLGridManager *manager = LLGridManager::getInstance(); // grid file doesn't exist - manager->initialize("grid_test.xml"); + manager->initialize(TEST_FILENAME); // validate that some of the defaults are available. std::map known_grids = manager->getKnownGrids(); ensure_equals("Known grids is a string-string map of size 2", known_grids.size(), 2); @@ -238,11 +245,11 @@ namespace tut template<> template<> void viewerNetworkTestObject::test<2>() { - llofstream gridfile("grid_test.xml"); + llofstream gridfile(TEST_FILENAME); gridfile << gSampleGridFile; gridfile.close(); - LLGridManager::getInstance()->initialize("grid_test.xml"); + LLGridManager::getInstance()->initialize(TEST_FILENAME); std::map known_grids = LLGridManager::getInstance()->getKnownGrids(); ensure_equals("adding a grid via a grid file increases known grid size",4, known_grids.size()); @@ -369,11 +376,11 @@ namespace tut void viewerNetworkTestObject::test<7>() { // adding a grid with simply a name will populate the values. - llofstream gridfile("grid_test.xml"); + llofstream gridfile(TEST_FILENAME); gridfile << gSampleGridFile; gridfile.close(); - LLGridManager::getInstance()->initialize("grid_test.xml"); + LLGridManager::getInstance()->initialize(TEST_FILENAME); LLGridManager::getInstance()->setGridChoice("util.agni.lindenlab.com"); ensure_equals("getGridLabel", -- cgit v1.2.3 From f71c6c745bc390fadc571801a0d7c043249ade24 Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Tue, 9 Sep 2014 15:36:35 -0400 Subject: Cleanup pass. Documentation. Get older llcorehttp-using code to use utils for any LLSD interfaces. --- indra/newview/llinventorymodel.cpp | 14 ++++- indra/newview/llinventorymodelbackgroundfetch.cpp | 28 +++++++--- indra/newview/llmeshrepository.cpp | 64 +++++++++-------------- indra/newview/lltexturefetch.cpp | 34 ++++++------ 4 files changed, 75 insertions(+), 65 deletions(-) (limited to 'indra/newview') diff --git a/indra/newview/llinventorymodel.cpp b/indra/newview/llinventorymodel.cpp index 5273fb6d96..1625ae0d1b 100755 --- a/indra/newview/llinventorymodel.cpp +++ b/indra/newview/llinventorymodel.cpp @@ -3958,7 +3958,10 @@ BOOL decompress_file(const char* src_filename, const char* dst_filename) } #endif -// ==== FetchItemHttpHandler ==== + +///---------------------------------------------------------------------------- +/// Class LLInventoryModel::FetchItemHttpHandler +///---------------------------------------------------------------------------- LLInventoryModel::FetchItemHttpHandler::FetchItemHttpHandler(const LLSD & request_sd) : LLCore::HttpHandler(), @@ -4006,6 +4009,15 @@ void LLInventoryModel::FetchItemHttpHandler::onCompleted(LLCore::HttpHandle hand } // Check for 200-with-error failures + // + // Original Responder-based serivce model didn't check for these errors. + // It may be more robust to ignore this condition. With aggregated requests, + // an error in one inventory item might take down the entire request. + // So if this instead broke up the aggregated items into single requests, + // maybe that would make progress. Or perhaps there's structured information + // that can tell us what went wrong. Need to dig into this and firm up + // the API. + // // body_llsd["error"] = LLSD::emptyMap(); // Dev tool to force error handling // body_llsd["error"]["identifier"] = "Development"; // body_llsd["error"]["message"] = "You left development code in the viewer"; diff --git a/indra/newview/llinventorymodelbackgroundfetch.cpp b/indra/newview/llinventorymodelbackgroundfetch.cpp index 0c04a9c039..013134843a 100755 --- a/indra/newview/llinventorymodelbackgroundfetch.cpp +++ b/indra/newview/llinventorymodelbackgroundfetch.cpp @@ -45,6 +45,10 @@ namespace { +///---------------------------------------------------------------------------- +/// Class ::BGItemHttpHandler +///---------------------------------------------------------------------------- + // // Http request handler class for single inventory item requests. // @@ -80,6 +84,10 @@ protected: }; +///---------------------------------------------------------------------------- +/// Class ::BGFolderHttpHandler +///---------------------------------------------------------------------------- + // Http request handler class for folders. // // Handler for FetchInventoryDescendents2 and FetchLibDescendents2 @@ -130,6 +138,10 @@ const char * const LOG_INV("Inventory"); } // end of namespace anonymous +///---------------------------------------------------------------------------- +/// Class LLInventoryModelBackgroundFetch +///---------------------------------------------------------------------------- + LLInventoryModelBackgroundFetch::LLInventoryModelBackgroundFetch(): mBackgroundFetchActive(FALSE), mFolderFetchActive(false), @@ -684,14 +696,12 @@ bool LLInventoryModelBackgroundFetch::fetchQueueContainsNoDescendentsOf(const LL } -// =============================== -// Anonymous Namespace Definitions -// =============================== - namespace { -// ==== BGFolderHttpHandler ==== +///---------------------------------------------------------------------------- +/// Class ::BGFolderHttpHandler +///---------------------------------------------------------------------------- void BGFolderHttpHandler::onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * response) { @@ -734,6 +744,9 @@ void BGFolderHttpHandler::onCompleted(LLCore::HttpHandle handle, LLCore::HttpRes } // Check for 200-with-error failures + // + // See comments in llinventorymodel.cpp about this mode of error. + // // body_llsd["error"] = LLSD::emptyMap(); // Dev tool to force error handling // body_llsd["error"]["identifier"] = "Development"; // body_llsd["error"]["message"] = "You left development code in the viewer"; @@ -976,8 +989,9 @@ bool BGFolderHttpHandler::getIsRecursive(const LLUUID & cat_id) const return std::find(mRecursiveCatUUIDs.begin(), mRecursiveCatUUIDs.end(), cat_id) != mRecursiveCatUUIDs.end(); } - -// ==== BGItemHttpHandler ==== +///---------------------------------------------------------------------------- +/// Class ::BGItemHttpHandler +///---------------------------------------------------------------------------- // Nothing to implement here. All ctor/dtor changes. diff --git a/indra/newview/llmeshrepository.cpp b/indra/newview/llmeshrepository.cpp index a6707392fe..2b044c6916 100755 --- a/indra/newview/llmeshrepository.cpp +++ b/indra/newview/llmeshrepository.cpp @@ -71,6 +71,7 @@ #include "bufferarray.h" #include "bufferstream.h" #include "llfasttimer.h" +#include "llcorehttputil.h" #include "boost/lexical_cast.hpp" @@ -2236,21 +2237,17 @@ void LLMeshUploadThread::doWholeModelUpload() mModelData = LLSD::emptyMap(); wholeModelToLLSD(mModelData, true); LLSD body = mModelData["asset_resources"]; - dump_llsd_to_file(body,make_dump_name("whole_model_body_",dump_num)); - - LLCore::BufferArray * ba = new LLCore::BufferArray; - LLCore::BufferArrayStream bas(ba); - LLSDSerialize::toXML(body, bas); - // LLSDSerialize::toXML(mModelData, bas); // <- Enabling this will generate a convenient upload error - LLCore::HttpHandle handle = mHttpRequest->requestPost(mHttpPolicyClass, - mHttpPriority, - mWholeModelUploadURL, - ba, - mHttpOptions, - mHttpHeaders, - this); - ba->release(); - + + dump_llsd_to_file(body, make_dump_name("whole_model_body_", dump_num)); + + LLCore::HttpHandle handle = LLCoreHttpUtil::requestPostWithLLSD(mHttpRequest, + mHttpPolicyClass, + mHttpPriority, + mWholeModelUploadURL, + body, + mHttpOptions, + mHttpHeaders, + this); if (LLCORE_HTTP_HANDLE_INVALID == handle) { mHttpStatus = mHttpRequest->getStatus(); @@ -2294,19 +2291,14 @@ void LLMeshUploadThread::requestWholeModelFee() mModelData = LLSD::emptyMap(); wholeModelToLLSD(mModelData, false); dump_llsd_to_file(mModelData, make_dump_name("whole_model_fee_request_", dump_num)); - - LLCore::BufferArray * ba = new LLCore::BufferArray; - LLCore::BufferArrayStream bas(ba); - LLSDSerialize::toXML(mModelData, bas); - - LLCore::HttpHandle handle = mHttpRequest->requestPost(mHttpPolicyClass, - mHttpPriority, - mWholeModelFeeCapability, - ba, - mHttpOptions, - mHttpHeaders, - this); - ba->release(); + LLCore::HttpHandle handle = LLCoreHttpUtil::requestPostWithLLSD(mHttpRequest, + mHttpPolicyClass, + mHttpPriority, + mWholeModelFeeCapability, + mModelData, + mHttpOptions, + mHttpHeaders, + this); if (LLCORE_HTTP_HANDLE_INVALID == handle) { mHttpStatus = mHttpRequest->getStatus(); @@ -2379,12 +2371,8 @@ void LLMeshUploadThread::onCompleted(LLCore::HttpHandle handle, LLCore::HttpResp } else { - LLCore::BufferArray * ba(response->getBody()); - if (ba && ba->size()) - { - LLCore::BufferArrayStream bas(ba); - LLSDSerialize::fromXML(body, bas); - } + // *TODO: handle error in conversion process + LLCoreHttpUtil::responseToLLSD(response, true, body); } dump_llsd_to_file(body, make_dump_name("whole_model_upload_response_", dump_num)); @@ -2443,12 +2431,8 @@ void LLMeshUploadThread::onCompleted(LLCore::HttpHandle handle, LLCore::HttpResp } else { - LLCore::BufferArray * ba(response->getBody()); - if (ba && ba->size()) - { - LLCore::BufferArrayStream bas(ba); - LLSDSerialize::fromXML(body, bas); - } + // *TODO: handle error in conversion process + LLCoreHttpUtil::responseToLLSD(response, true, body); } dump_llsd_to_file(body, make_dump_name("whole_model_fee_response_", dump_num)); diff --git a/indra/newview/lltexturefetch.cpp b/indra/newview/lltexturefetch.cpp index a64a6ee091..a955b907d4 100755 --- a/indra/newview/lltexturefetch.cpp +++ b/indra/newview/lltexturefetch.cpp @@ -56,13 +56,13 @@ #include "llsdparam.h" #include "llsdutil.h" #include "llstartup.h" -#include "llsdserialize.h" #include "httprequest.h" #include "httphandler.h" #include "httpresponse.h" #include "bufferarray.h" #include "bufferstream.h" +#include "llcorehttputil.h" #include "llhttpretrypolicy.h" @@ -1349,20 +1349,20 @@ bool LLTextureFetchWorker::doWork(S32 param) LL_WARNS(LOG_TXT) << "trying to seek a non-default texture on the sim. Bad!" << LL_ENDL; } setUrl(http_url + "/?texture_id=" + mID.asString().c_str()); - LL_DEBUGS("Texture") << "Texture URL " << mUrl << LL_ENDL; + LL_DEBUGS(LOG_TXT) << "Texture URL: " << mUrl << LL_ENDL; mWriteToCacheState = CAN_WRITE ; //because this texture has a fixed texture id. } else { mCanUseHTTP = false ; - LL_DEBUGS("Texture") << "Texture not available via HTTP: no URL " << mUrl << LL_ENDL; + LL_DEBUGS(LOG_TXT) << "Texture not available via HTTP: empty URL." << LL_ENDL; } } else { // This will happen if not logged in or if a region deoes not have HTTP Texture enabled //LL_WARNS(LOG_TXT) << "Region not found for host: " << mHost << LL_ENDL; - LL_DEBUGS("Texture") << "Texture not available via HTTP: no region " << mUrl << LL_ENDL; + LL_DEBUGS(LOG_TXT) << "Texture not available via HTTP: no region " << mUrl << LL_ENDL; mCanUseHTTP = false; } } @@ -4026,7 +4026,9 @@ TFReqSendMetrics::doWork(LLTextureFetch * fetcher) // Update sequence number if (S32_MAX == ++report_sequence) + { report_sequence = 0; + } reporting_started = true; // Limit the size of the stats report if necessary. @@ -4035,18 +4037,16 @@ TFReqSendMetrics::doWork(LLTextureFetch * fetcher) if (! mCapsURL.empty()) { - LLCore::BufferArray * ba = new LLCore::BufferArray; - LLCore::BufferArrayStream bas(ba); - LLSDSerialize::toXML(sd, bas); - - fetcher->getHttpRequest().requestPost(fetcher->getPolicyClass(), - report_priority, - mCapsURL, - ba, - NULL, - fetcher->getMetricsHeaders(), - handler); - ba->release(); + // *TODO: Move this to a different class that expects POSTs sometime. + // Don't care about handle, this is a fire-and-forget operation. + LLCoreHttpUtil::requestPostWithLLSD(&fetcher->getHttpRequest(), + fetcher->getPolicyClass(), + report_priority, + mCapsURL, + sd, + NULL, + fetcher->getMetricsHeaders(), + handler); LLTextureFetch::svMetricsDataBreak = false; } else @@ -4057,7 +4057,7 @@ TFReqSendMetrics::doWork(LLTextureFetch * fetcher) // In QA mode, Metrics submode, log the result for ease of testing if (fetcher->isQAMode()) { - LL_INFOS("Textures") << ll_pretty_print_sd(sd) << LL_ENDL; + LL_INFOS(LOG_TXT) << ll_pretty_print_sd(sd) << LL_ENDL; } return true; -- cgit v1.2.3 From 11036d7bf471953ada9b877b8d9ce9de4b94dc5b Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Fri, 19 Sep 2014 19:43:25 -0400 Subject: Cleanup work. Use http constants for content-type and accept headers in mesh and textures. For texture metrics reporting, use the AP_INVENTORY policy class which is non-pipelined and pointing (usually) in the right direction. Use a do-while(false) structure to manage common exit path code in onCompleted() methods. Identical to a 'goto' but might amuse the pedantic. Tuning on background fetch to have it cycle faster. This is experimental. I suspect with HTTP balancing in llcorehttp, we can do away with the timers here. --- indra/newview/llappcorehttp.h | 9 +- indra/newview/llinventorymodel.cpp | 102 +++++++++++---------- indra/newview/llinventorymodelbackgroundfetch.cpp | 106 +++++++++++----------- indra/newview/llmeshrepository.cpp | 4 +- indra/newview/lltexturefetch.cpp | 15 +-- indra/newview/lltexturefetch.h | 6 +- 6 files changed, 126 insertions(+), 116 deletions(-) (limited to 'indra/newview') diff --git a/indra/newview/llappcorehttp.h b/indra/newview/llappcorehttp.h index e822a40300..37d7a737e7 100755 --- a/indra/newview/llappcorehttp.h +++ b/indra/newview/llappcorehttp.h @@ -66,7 +66,7 @@ public: /// baking service. Deep queueing of requests. /// Do not share. GET requests only. /// - /// Destination: simhost:12046 & bake-texture:80 + /// Destination: simhost:12046 & {bake-texture,cdn}:80 /// Protocol: http: /// Transfer size: KB-MB /// Long poll: no @@ -95,7 +95,7 @@ public: /// Very deeply queued. Do not share. GET /// requests only. /// - /// Destination: simhost:12046 + /// Destination: simhost:12046 & cdn:80 /// Protocol: http: /// Transfer size: KB-MB /// Long poll: no @@ -113,7 +113,7 @@ public: /// traffic that can wait for longish stalls /// (default timeout 600S). /// - /// Destination: simhost:12046 + /// Destination: simhost:12046 & cdn:80 /// Protocol: http: /// Transfer size: MB /// Long poll: no @@ -163,7 +163,8 @@ public: /// Request rate: high /// Pipelined: no AP_INVENTORY, - + AP_REPORTING = AP_INVENTORY, // Piggy-back on inventory + AP_COUNT // Must be last }; diff --git a/indra/newview/llinventorymodel.cpp b/indra/newview/llinventorymodel.cpp index 1625ae0d1b..ee28cef640 100755 --- a/indra/newview/llinventorymodel.cpp +++ b/indra/newview/llinventorymodel.cpp @@ -3974,64 +3974,68 @@ LLInventoryModel::FetchItemHttpHandler::~FetchItemHttpHandler() void LLInventoryModel::FetchItemHttpHandler::onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * response) { - LLCore::HttpStatus status(response->getStatus()); - // status = LLCore::HttpStatus(404); // Dev tool to force error handling - if (! status) - { - processFailure(status, response); - } - else + // Single-pass do-while used for common exit handling + do { - LLCore::BufferArray * body(response->getBody()); - // body = NULL; // Dev tool to force error handling - if (! body || ! body->size()) + LLCore::HttpStatus status(response->getStatus()); + // status = LLCore::HttpStatus(404); // Dev tool to force error handling + if (! status) { - LL_WARNS(LOG_INV) << "Missing data in inventory item query." << LL_ENDL; - processFailure("HTTP response for inventory item query missing body", response); - goto only_exit; + processFailure(status, response); } - - // body->write(0, "Garbage Response", 16); // Dev tool to force error handling - LLSD body_llsd; - if (! LLCoreHttpUtil::responseToLLSD(response, true, body_llsd)) + else { - // INFOS-level logging will occur on the parsed failure - processFailure("HTTP response for inventory item query has malformed LLSD", response); - goto only_exit; - } + LLCore::BufferArray * body(response->getBody()); + // body = NULL; // Dev tool to force error handling + if (! body || ! body->size()) + { + LL_WARNS(LOG_INV) << "Missing data in inventory item query." << LL_ENDL; + processFailure("HTTP response for inventory item query missing body", response); + break; // Goto common exit + } - // Expect top-level structure to be a map - // body_llsd = LLSD::emptyArray(); // Dev tool to force error handling - if (! body_llsd.isMap()) - { - processFailure("LLSD response for inventory item not a map", response); - goto only_exit; - } + // body->write(0, "Garbage Response", 16); // Dev tool to force error handling + LLSD body_llsd; + if (! LLCoreHttpUtil::responseToLLSD(response, true, body_llsd)) + { + // INFOS-level logging will occur on the parsed failure + processFailure("HTTP response for inventory item query has malformed LLSD", response); + break; // Goto common exit + } - // Check for 200-with-error failures - // - // Original Responder-based serivce model didn't check for these errors. - // It may be more robust to ignore this condition. With aggregated requests, - // an error in one inventory item might take down the entire request. - // So if this instead broke up the aggregated items into single requests, - // maybe that would make progress. Or perhaps there's structured information - // that can tell us what went wrong. Need to dig into this and firm up - // the API. - // - // body_llsd["error"] = LLSD::emptyMap(); // Dev tool to force error handling - // body_llsd["error"]["identifier"] = "Development"; - // body_llsd["error"]["message"] = "You left development code in the viewer"; - if (body_llsd.has("error")) - { - processFailure("Inventory application error (200-with-error)", response); - goto only_exit; - } + // Expect top-level structure to be a map + // body_llsd = LLSD::emptyArray(); // Dev tool to force error handling + if (! body_llsd.isMap()) + { + processFailure("LLSD response for inventory item not a map", response); + break; // Goto common exit + } - // Okay, process data if possible - processData(body_llsd, response); + // Check for 200-with-error failures + // + // Original Responder-based serivce model didn't check for these errors. + // It may be more robust to ignore this condition. With aggregated requests, + // an error in one inventory item might take down the entire request. + // So if this instead broke up the aggregated items into single requests, + // maybe that would make progress. Or perhaps there's structured information + // that can tell us what went wrong. Need to dig into this and firm up + // the API. + // + // body_llsd["error"] = LLSD::emptyMap(); // Dev tool to force error handling + // body_llsd["error"]["identifier"] = "Development"; + // body_llsd["error"]["message"] = "You left development code in the viewer"; + if (body_llsd.has("error")) + { + processFailure("Inventory application error (200-with-error)", response); + break; // Goto common exit + } + + // Okay, process data if possible + processData(body_llsd, response); + } } + while (false); -only_exit: // Must delete on completion. delete this; } diff --git a/indra/newview/llinventorymodelbackgroundfetch.cpp b/indra/newview/llinventorymodelbackgroundfetch.cpp index 013134843a..de1d123fe5 100755 --- a/indra/newview/llinventorymodelbackgroundfetch.cpp +++ b/indra/newview/llinventorymodelbackgroundfetch.cpp @@ -288,6 +288,7 @@ void LLInventoryModelBackgroundFetch::setAllFoldersFetched() } mFolderFetchActive = false; mBackgroundFetchActive = false; + LL_INFOS(LOG_INV) << "Inventory background fetch completed" << LL_ENDL; } void LLInventoryModelBackgroundFetch::backgroundFetchCB(void *) @@ -314,12 +315,7 @@ void LLInventoryModelBackgroundFetch::backgroundFetch() // No more categories to fetch, stop fetch process. if (mFetchQueue.empty()) { - LL_INFOS(LOG_INV) << "Inventory fetch completed" << LL_ENDL; - setAllFoldersFetched(); - mBackgroundFetchActive = false; - mFolderFetchActive = false; - return; } @@ -493,9 +489,10 @@ void LLInventoryModelBackgroundFetch::bulkFetch() // is mostly loaded, we could turn up the throttle and fill missing // inventory more quickly. static const S32 max_concurrent_fetches(12); // Outstanding requests, not connections - static const F32 new_min_time(0.5f); // *HACK: Clean this up when old code goes away entirely. + static const F32 new_min_time(0.05f); // *HACK: Clean this up when old code goes away entirely. static const U32 max_batch_size(10); + mMinTimeBetweenFetches = 0.01f; if (mMinTimeBetweenFetches < new_min_time) { mMinTimeBetweenFetches = new_min_time; // *HACK: See above. @@ -705,62 +702,66 @@ namespace void BGFolderHttpHandler::onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * response) { - LLCore::HttpStatus status(response->getStatus()); - // status = LLCore::HttpStatus(404); // Dev tool to force error handling - if (! status) - { - processFailure(status, response); - } - else + // Single-pass do-while used for common exit handling + do { - // Response body should be present. - LLCore::BufferArray * body(response->getBody()); - // body = NULL; // Dev tool to force error handling - if (! body || ! body->size()) + LLCore::HttpStatus status(response->getStatus()); + // status = LLCore::HttpStatus(404); // Dev tool to force error handling + if (! status) { - LL_WARNS(LOG_INV) << "Missing data in inventory folder query." << LL_ENDL; - processFailure("HTTP response missing expected body", response); - goto only_exit; + processFailure(status, response); } + else + { + // Response body should be present. + LLCore::BufferArray * body(response->getBody()); + // body = NULL; // Dev tool to force error handling + if (! body || ! body->size()) + { + LL_WARNS(LOG_INV) << "Missing data in inventory folder query." << LL_ENDL; + processFailure("HTTP response missing expected body", response); + break; // Goto common exit + } - // Could test 'Content-Type' header but probably unreliable. + // Could test 'Content-Type' header but probably unreliable. - // Convert response to LLSD - // body->write(0, "Garbage Response", 16); // Dev tool to force error handling - LLSD body_llsd; - if (! LLCoreHttpUtil::responseToLLSD(response, true, body_llsd)) - { - // INFOS-level logging will occur on the parsed failure - processFailure("HTTP response contained malformed LLSD", response); - goto only_exit; - } + // Convert response to LLSD + // body->write(0, "Garbage Response", 16); // Dev tool to force error handling + LLSD body_llsd; + if (! LLCoreHttpUtil::responseToLLSD(response, true, body_llsd)) + { + // INFOS-level logging will occur on the parsed failure + processFailure("HTTP response contained malformed LLSD", response); + break; // goto common exit + } - // Expect top-level structure to be a map - // body_llsd = LLSD::emptyArray(); // Dev tool to force error handling - if (! body_llsd.isMap()) - { - processFailure("LLSD response not a map", response); - goto only_exit; - } + // Expect top-level structure to be a map + // body_llsd = LLSD::emptyArray(); // Dev tool to force error handling + if (! body_llsd.isMap()) + { + processFailure("LLSD response not a map", response); + break; // goto common exit + } - // Check for 200-with-error failures - // - // See comments in llinventorymodel.cpp about this mode of error. - // - // body_llsd["error"] = LLSD::emptyMap(); // Dev tool to force error handling - // body_llsd["error"]["identifier"] = "Development"; - // body_llsd["error"]["message"] = "You left development code in the viewer"; - if (body_llsd.has("error")) - { - processFailure("Inventory application error (200-with-error)", response); - goto only_exit; - } + // Check for 200-with-error failures + // + // See comments in llinventorymodel.cpp about this mode of error. + // + // body_llsd["error"] = LLSD::emptyMap(); // Dev tool to force error handling + // body_llsd["error"]["identifier"] = "Development"; + // body_llsd["error"]["message"] = "You left development code in the viewer"; + if (body_llsd.has("error")) + { + processFailure("Inventory application error (200-with-error)", response); + break; // goto common exit + } - // Okay, process data if possible - processData(body_llsd, response); + // Okay, process data if possible + processData(body_llsd, response); + } } + while (false); -only_exit: // Must delete on completion. delete this; } @@ -895,7 +896,6 @@ void BGFolderHttpHandler::processData(LLSD & content, LLCore::HttpResponse * res if (fetcher->isBulkFetchProcessingComplete()) { - LL_INFOS(LOG_INV) << "Inventory fetch completed" << LL_ENDL; fetcher->setAllFoldersFetched(); } diff --git a/indra/newview/llmeshrepository.cpp b/indra/newview/llmeshrepository.cpp index 2b044c6916..648056484e 100755 --- a/indra/newview/llmeshrepository.cpp +++ b/indra/newview/llmeshrepository.cpp @@ -760,7 +760,7 @@ LLMeshRepoThread::LLMeshRepoThread() mHttpLargeOptions->setTransferTimeout(LARGE_MESH_XFER_TIMEOUT); mHttpLargeOptions->setUseRetryAfter(gSavedSettings.getBOOL("MeshUseHttpRetryAfter")); mHttpHeaders = new LLCore::HttpHeaders; - mHttpHeaders->append("Accept", "application/vnd.ll.mesh"); + mHttpHeaders->append(HTTP_OUT_HEADER_ACCEPT, HTTP_CONTENT_VND_LL_MESH); mHttpPolicyClass = app_core_http.getPolicy(LLAppCoreHttp::AP_MESH2); mHttpLegacyPolicyClass = app_core_http.getPolicy(LLAppCoreHttp::AP_MESH1); mHttpLargePolicyClass = app_core_http.getPolicy(LLAppCoreHttp::AP_LARGE_MESH); @@ -1887,7 +1887,7 @@ LLMeshUploadThread::LLMeshUploadThread(LLMeshUploadThread::instance_list& data, mHttpOptions->setUseRetryAfter(gSavedSettings.getBOOL("MeshUseHttpRetryAfter")); mHttpOptions->setRetries(UPLOAD_RETRY_LIMIT); mHttpHeaders = new LLCore::HttpHeaders; - mHttpHeaders->append("Content-Type", "application/llsd+xml"); + mHttpHeaders->append(HTTP_OUT_HEADER_CONTENT_TYPE, HTTP_CONTENT_LLSD_XML); mHttpPolicyClass = LLAppViewer::instance()->getAppCoreHttp().getPolicy(LLAppCoreHttp::AP_UPLOADS); mHttpPriority = 0; } diff --git a/indra/newview/lltexturefetch.cpp b/indra/newview/lltexturefetch.cpp index a955b907d4..acd4cf2d8d 100755 --- a/indra/newview/lltexturefetch.cpp +++ b/indra/newview/lltexturefetch.cpp @@ -2514,8 +2514,9 @@ LLTextureFetch::LLTextureFetch(LLTextureCache* cache, LLImageDecodeThread* image mHttpOptions(NULL), mHttpOptionsWithHeaders(NULL), mHttpHeaders(NULL), - mHttpMetricsHeaders(NULL), mHttpPolicyClass(LLCore::HttpRequest::DEFAULT_POLICY_ID), + mHttpMetricsHeaders(NULL), + mHttpMetricsPolicyClass(LLCore::HttpRequest::DEFAULT_POLICY_ID), mTotalCacheReadCount(0U), mTotalCacheWriteCount(0U), mTotalResourceWaitCount(0U), @@ -2528,15 +2529,16 @@ LLTextureFetch::LLTextureFetch(LLTextureCache* cache, LLImageDecodeThread* image mTextureInfo.setUpLogging(gSavedSettings.getBOOL("LogTextureDownloadsToViewerLog"), gSavedSettings.getBOOL("LogTextureDownloadsToSimulator"), U32Bytes(gSavedSettings.getU32("TextureLoggingThreshold"))); LLAppCoreHttp & app_core_http(LLAppViewer::instance()->getAppCoreHttp()); - mHttpPolicyClass = app_core_http.getPolicy(LLAppCoreHttp::AP_TEXTURE); mHttpRequest = new LLCore::HttpRequest; mHttpOptions = new LLCore::HttpOptions; mHttpOptionsWithHeaders = new LLCore::HttpOptions; mHttpOptionsWithHeaders->setWantHeaders(true); mHttpHeaders = new LLCore::HttpHeaders; - mHttpHeaders->append("Accept", "image/x-j2c"); + mHttpHeaders->append(HTTP_OUT_HEADER_ACCEPT, HTTP_CONTENT_IMAGE_X_J2C); + mHttpPolicyClass = app_core_http.getPolicy(LLAppCoreHttp::AP_TEXTURE); mHttpMetricsHeaders = new LLCore::HttpHeaders; - mHttpMetricsHeaders->append("Content-Type", "application/llsd+xml"); + mHttpMetricsHeaders->append(HTTP_OUT_HEADER_CONTENT_TYPE, HTTP_CONTENT_LLSD_XML); + mHttpMetricsPolicyClass = app_core_http.getPolicy(LLAppCoreHttp::AP_REPORTING); mHttpHighWater = HTTP_NONPIPE_REQUESTS_HIGH_WATER; mHttpLowWater = HTTP_NONPIPE_REQUESTS_LOW_WATER; mHttpSemaphore = 0; @@ -4037,10 +4039,9 @@ TFReqSendMetrics::doWork(LLTextureFetch * fetcher) if (! mCapsURL.empty()) { - // *TODO: Move this to a different class that expects POSTs sometime. // Don't care about handle, this is a fire-and-forget operation. LLCoreHttpUtil::requestPostWithLLSD(&fetcher->getHttpRequest(), - fetcher->getPolicyClass(), + fetcher->getMetricsPolicyClass(), report_priority, mCapsURL, sd, @@ -4227,7 +4228,7 @@ void LLTextureFetchDebugger::init() if (! mHttpHeaders) { mHttpHeaders = new LLCore::HttpHeaders; - mHttpHeaders->append("Accept", "image/x-j2c"); + mHttpHeaders->append(HTTP_OUT_HEADER_ACCEPT, HTTP_CONTENT_IMAGE_X_J2C); } } diff --git a/indra/newview/lltexturefetch.h b/indra/newview/lltexturefetch.h index 89d18e2c67..27779a31e0 100755 --- a/indra/newview/lltexturefetch.h +++ b/indra/newview/lltexturefetch.h @@ -179,6 +179,9 @@ public: // Threads: T* LLCore::HttpHeaders * getMetricsHeaders() const { return mHttpMetricsHeaders; } + // Threads: T* + LLCore::HttpRequest::policy_t getMetricsPolicyClass() const { return mHttpMetricsPolicyClass; } + bool isQAMode() const { return mQAMode; } // ---------------------------------- @@ -354,8 +357,9 @@ private: LLCore::HttpOptions * mHttpOptions; // Ttf LLCore::HttpOptions * mHttpOptionsWithHeaders; // Ttf LLCore::HttpHeaders * mHttpHeaders; // Ttf - LLCore::HttpHeaders * mHttpMetricsHeaders; // Ttf LLCore::HttpRequest::policy_t mHttpPolicyClass; // T* + LLCore::HttpHeaders * mHttpMetricsHeaders; // Ttf + LLCore::HttpRequest::policy_t mHttpMetricsPolicyClass; // T* S32 mHttpHighWater; // Ttf S32 mHttpLowWater; // Ttf -- cgit v1.2.3 From 329608d24668b044e16b54ff7a7d0ac592b2b88d Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Mon, 22 Sep 2014 18:49:45 -0400 Subject: Tuning and documentation. Use a fast poll frequency (0.05S) on the HTTP requests for inventory. We'll benchmark with that and see how it goes. Document some of the history of the background fetcher for future devs. Suggest some future projects to make things faster. Pointers on using LLSD with the llcorehttp library in the readme. And restructured the LLSD onCompleted() processing phases using do{}while(false) which produced a code flow that is fairly attractive. --- indra/newview/llinventorymodel.cpp | 92 +++++++------- indra/newview/llinventorymodelbackgroundfetch.cpp | 140 +++++++++++++++------- 2 files changed, 139 insertions(+), 93 deletions(-) (limited to 'indra/newview') diff --git a/indra/newview/llinventorymodel.cpp b/indra/newview/llinventorymodel.cpp index ee28cef640..dab3a4c06d 100755 --- a/indra/newview/llinventorymodel.cpp +++ b/indra/newview/llinventorymodel.cpp @@ -3974,65 +3974,63 @@ LLInventoryModel::FetchItemHttpHandler::~FetchItemHttpHandler() void LLInventoryModel::FetchItemHttpHandler::onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * response) { - // Single-pass do-while used for common exit handling - do + do // Single-pass do-while used for common exit handling { LLCore::HttpStatus status(response->getStatus()); // status = LLCore::HttpStatus(404); // Dev tool to force error handling if (! status) { processFailure(status, response); + break; // Goto common exit } - else - { - LLCore::BufferArray * body(response->getBody()); - // body = NULL; // Dev tool to force error handling - if (! body || ! body->size()) - { - LL_WARNS(LOG_INV) << "Missing data in inventory item query." << LL_ENDL; - processFailure("HTTP response for inventory item query missing body", response); - break; // Goto common exit - } - // body->write(0, "Garbage Response", 16); // Dev tool to force error handling - LLSD body_llsd; - if (! LLCoreHttpUtil::responseToLLSD(response, true, body_llsd)) - { - // INFOS-level logging will occur on the parsed failure - processFailure("HTTP response for inventory item query has malformed LLSD", response); - break; // Goto common exit - } + LLCore::BufferArray * body(response->getBody()); + // body = NULL; // Dev tool to force error handling + if (! body || ! body->size()) + { + LL_WARNS(LOG_INV) << "Missing data in inventory item query." << LL_ENDL; + processFailure("HTTP response for inventory item query missing body", response); + break; // Goto common exit + } - // Expect top-level structure to be a map - // body_llsd = LLSD::emptyArray(); // Dev tool to force error handling - if (! body_llsd.isMap()) - { - processFailure("LLSD response for inventory item not a map", response); - break; // Goto common exit - } + // body->write(0, "Garbage Response", 16); // Dev tool to force error handling + LLSD body_llsd; + if (! LLCoreHttpUtil::responseToLLSD(response, true, body_llsd)) + { + // INFOS-level logging will occur on the parsed failure + processFailure("HTTP response for inventory item query has malformed LLSD", response); + break; // Goto common exit + } - // Check for 200-with-error failures - // - // Original Responder-based serivce model didn't check for these errors. - // It may be more robust to ignore this condition. With aggregated requests, - // an error in one inventory item might take down the entire request. - // So if this instead broke up the aggregated items into single requests, - // maybe that would make progress. Or perhaps there's structured information - // that can tell us what went wrong. Need to dig into this and firm up - // the API. - // - // body_llsd["error"] = LLSD::emptyMap(); // Dev tool to force error handling - // body_llsd["error"]["identifier"] = "Development"; - // body_llsd["error"]["message"] = "You left development code in the viewer"; - if (body_llsd.has("error")) - { - processFailure("Inventory application error (200-with-error)", response); - break; // Goto common exit - } + // Expect top-level structure to be a map + // body_llsd = LLSD::emptyArray(); // Dev tool to force error handling + if (! body_llsd.isMap()) + { + processFailure("LLSD response for inventory item not a map", response); + break; // Goto common exit + } - // Okay, process data if possible - processData(body_llsd, response); + // Check for 200-with-error failures + // + // Original Responder-based serivce model didn't check for these errors. + // It may be more robust to ignore this condition. With aggregated requests, + // an error in one inventory item might take down the entire request. + // So if this instead broke up the aggregated items into single requests, + // maybe that would make progress. Or perhaps there's structured information + // that can tell us what went wrong. Need to dig into this and firm up + // the API. + // + // body_llsd["error"] = LLSD::emptyMap(); // Dev tool to force error handling + // body_llsd["error"]["identifier"] = "Development"; + // body_llsd["error"]["message"] = "You left development code in the viewer"; + if (body_llsd.has("error")) + { + processFailure("Inventory application error (200-with-error)", response); + break; // Goto common exit } + + // Okay, process data if possible + processData(body_llsd, response); } while (false); diff --git a/indra/newview/llinventorymodelbackgroundfetch.cpp b/indra/newview/llinventorymodelbackgroundfetch.cpp index de1d123fe5..f18832fe95 100755 --- a/indra/newview/llinventorymodelbackgroundfetch.cpp +++ b/indra/newview/llinventorymodelbackgroundfetch.cpp @@ -42,6 +42,56 @@ #include "bufferstream.h" #include "llcorehttputil.h" +// History (may be apocryphal) +// +// Around V2, an HTTP inventory download mechanism was added +// along with inventory LINK items referencing other inventory +// items. As part of this, at login, the entire inventory +// structure is downloaded 'in the background' using the +// backgroundFetch()/bulkFetch() methods. The UDP path can +// still be used and is found in the 'DEPRECATED OLD CODE' +// section. +// +// The old UDP path implemented a throttle that adapted +// itself during running. The mechanism survived info HTTP +// somewhat but was pinned to poll the HTTP plumbing at +// 0.5S intervals. The reasons for this particular value +// have been lost. It's possible to switch between UDP +// and HTTP while this is happening but there may be +// surprises in what happens in that case. +// +// Conversion to llcorehttp reduced the number of connections +// used but batches more data and queues more requests (but +// doesn't due pipelining due to libcurl restrictions). The +// poll interval above was re-examined and reduced to get +// inventory into the viewer more quickly. +// +// Possible future work: +// +// * Don't download the entire heirarchy in one go (which +// might have been how V1 worked). Implications for +// links (which may not have a valid target) and search +// which would then be missing data. +// +// * Review the download rate throttling. Slow then fast? +// Detect bandwidth usage and speed up when it drops? +// +// * A lot of calls to notifyObservers(). It looks like +// these could be collapsed by maintaining a 'dirty' +// bit and there appears to be an attempt to do this. +// But it isn't used or is used in a limited fashion. +// Are there semanic issues requiring a call after certain +// updateItem() calls? +// +// * An error on a fetch could be due to one item in the batch. +// If the batch were broken up, perhaps more of the inventory +// would download. (Handwave here, not certain this is an +// issue in practice.) +// +// * Conversion to AISv3. +// + + namespace { @@ -488,11 +538,11 @@ void LLInventoryModelBackgroundFetch::bulkFetch() // a fast/slow fetch throttle. Once login is complete and the scene // is mostly loaded, we could turn up the throttle and fill missing // inventory more quickly. + static const U32 max_batch_size(10); static const S32 max_concurrent_fetches(12); // Outstanding requests, not connections static const F32 new_min_time(0.05f); // *HACK: Clean this up when old code goes away entirely. - static const U32 max_batch_size(10); - mMinTimeBetweenFetches = 0.01f; + mMinTimeBetweenFetches = new_min_time; if (mMinTimeBetweenFetches < new_min_time) { mMinTimeBetweenFetches = new_min_time; // *HACK: See above. @@ -702,63 +752,61 @@ namespace void BGFolderHttpHandler::onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * response) { - // Single-pass do-while used for common exit handling - do + do // Single-pass do-while used for common exit handling { LLCore::HttpStatus status(response->getStatus()); // status = LLCore::HttpStatus(404); // Dev tool to force error handling if (! status) { processFailure(status, response); + break; // Goto common exit } - else - { - // Response body should be present. - LLCore::BufferArray * body(response->getBody()); - // body = NULL; // Dev tool to force error handling - if (! body || ! body->size()) - { - LL_WARNS(LOG_INV) << "Missing data in inventory folder query." << LL_ENDL; - processFailure("HTTP response missing expected body", response); - break; // Goto common exit - } - // Could test 'Content-Type' header but probably unreliable. + // Response body should be present. + LLCore::BufferArray * body(response->getBody()); + // body = NULL; // Dev tool to force error handling + if (! body || ! body->size()) + { + LL_WARNS(LOG_INV) << "Missing data in inventory folder query." << LL_ENDL; + processFailure("HTTP response missing expected body", response); + break; // Goto common exit + } - // Convert response to LLSD - // body->write(0, "Garbage Response", 16); // Dev tool to force error handling - LLSD body_llsd; - if (! LLCoreHttpUtil::responseToLLSD(response, true, body_llsd)) - { - // INFOS-level logging will occur on the parsed failure - processFailure("HTTP response contained malformed LLSD", response); - break; // goto common exit - } + // Could test 'Content-Type' header but probably unreliable. - // Expect top-level structure to be a map - // body_llsd = LLSD::emptyArray(); // Dev tool to force error handling - if (! body_llsd.isMap()) - { - processFailure("LLSD response not a map", response); - break; // goto common exit - } + // Convert response to LLSD + // body->write(0, "Garbage Response", 16); // Dev tool to force error handling + LLSD body_llsd; + if (! LLCoreHttpUtil::responseToLLSD(response, true, body_llsd)) + { + // INFOS-level logging will occur on the parsed failure + processFailure("HTTP response contained malformed LLSD", response); + break; // goto common exit + } - // Check for 200-with-error failures - // - // See comments in llinventorymodel.cpp about this mode of error. - // - // body_llsd["error"] = LLSD::emptyMap(); // Dev tool to force error handling - // body_llsd["error"]["identifier"] = "Development"; - // body_llsd["error"]["message"] = "You left development code in the viewer"; - if (body_llsd.has("error")) - { - processFailure("Inventory application error (200-with-error)", response); - break; // goto common exit - } + // Expect top-level structure to be a map + // body_llsd = LLSD::emptyArray(); // Dev tool to force error handling + if (! body_llsd.isMap()) + { + processFailure("LLSD response not a map", response); + break; // goto common exit + } - // Okay, process data if possible - processData(body_llsd, response); + // Check for 200-with-error failures + // + // See comments in llinventorymodel.cpp about this mode of error. + // + // body_llsd["error"] = LLSD::emptyMap(); // Dev tool to force error handling + // body_llsd["error"]["identifier"] = "Development"; + // body_llsd["error"]["message"] = "You left development code in the viewer"; + if (body_llsd.has("error")) + { + processFailure("Inventory application error (200-with-error)", response); + break; // goto common exit } + + // Okay, process data if possible + processData(body_llsd, response); } while (false); -- cgit v1.2.3 From f9c73f089a5a213e965aa59d6128ba5dd381186e Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Mon, 6 Oct 2014 05:44:54 +0000 Subject: BUG-7454 Viewer crashes while logging with a turned off "HTTP Inventory" The HttpRequest objects used in inventory ops are created lazily when needed. The update() operation didn't protect against their not being created. Added some guard tests around the update() operations and we're good. --- indra/newview/llinventorymodel.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'indra/newview') diff --git a/indra/newview/llinventorymodel.cpp b/indra/newview/llinventorymodel.cpp index dab3a4c06d..f92332dea5 100755 --- a/indra/newview/llinventorymodel.cpp +++ b/indra/newview/llinventorymodel.cpp @@ -2436,11 +2436,11 @@ void LLInventoryModel::initHttpRequest() void LLInventoryModel::handleResponses(bool foreground) { - if (foreground) + if (foreground && mHttpRequestFG) { mHttpRequestFG->update(0); } - else + else if (! foreground && mHttpRequestBG) { mHttpRequestBG->update(50000L); } -- cgit v1.2.3 From 02e2235277a90f2e291557a429ae4e5de3e0d3b6 Mon Sep 17 00:00:00 2001 From: Oz Linden Date: Tue, 28 Oct 2014 09:55:55 -0400 Subject: increment viewer version to 3.7.20 --- indra/newview/VIEWER_VERSION.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'indra/newview') diff --git a/indra/newview/VIEWER_VERSION.txt b/indra/newview/VIEWER_VERSION.txt index 7707ee0e29..82a60c0bb1 100644 --- a/indra/newview/VIEWER_VERSION.txt +++ b/indra/newview/VIEWER_VERSION.txt @@ -1 +1 @@ -3.7.19 +3.7.20 -- cgit v1.2.3