From 24f8745914f7cec320d707f36895ac393575b861 Mon Sep 17 00:00:00 2001 From: Dave Parks Date: Wed, 19 Mar 2014 17:57:00 -0500 Subject: MAINT-3131 Use benchmark to determine GPU class instead of GPU table. --- indra/llrender/llglslshader.cpp | 58 +++++++++++++++++++++++------------ indra/llrender/llglslshader.h | 4 ++- indra/llrender/llrendertarget.cpp | 1 + indra/llrender/llshadermgr.cpp | 6 +++- indra/newview/llfeaturemanager.cpp | 63 +++++++++++++++++++++++++++++++++++--- indra/newview/llglsandbox.cpp | 31 ++++++++++++++++--- indra/newview/llviewermenu.cpp | 2 +- 7 files changed, 133 insertions(+), 32 deletions(-) (limited to 'indra') diff --git a/indra/llrender/llglslshader.cpp b/indra/llrender/llglslshader.cpp index 1c50a51d02..b99435f39e 100755 --- a/indra/llrender/llglslshader.cpp +++ b/indra/llrender/llglslshader.cpp @@ -87,6 +87,7 @@ LLShaderFeatures::LLShaderFeatures() , mIndexedTextureChannels(0) , disableTextureIndex(false) , hasAlphaMask(false) + , attachNothing(false) { } @@ -119,28 +120,31 @@ struct LLGLSLShaderCompareTimeElapsed }; //static -void LLGLSLShader::finishProfile() +void LLGLSLShader::finishProfile(bool emit_report) { sProfileEnabled = false; - std::vector sorted; - - for (std::set::iterator iter = sInstances.begin(); iter != sInstances.end(); ++iter) + if (emit_report) { - sorted.push_back(*iter); - } + std::vector sorted; - std::sort(sorted.begin(), sorted.end(), LLGLSLShaderCompareTimeElapsed()); + for (std::set::iterator iter = sInstances.begin(); iter != sInstances.end(); ++iter) + { + sorted.push_back(*iter); + } - for (std::vector::iterator iter = sorted.begin(); iter != sorted.end(); ++iter) - { - (*iter)->dumpStats(); - } + std::sort(sorted.begin(), sorted.end(), LLGLSLShaderCompareTimeElapsed()); - llinfos << "-----------------------------------" << llendl; - llinfos << "Total rendering time: " << llformat("%.4f ms", sTotalTimeElapsed/1000000.f) << llendl; - llinfos << "Total samples drawn: " << llformat("%.4f million", sTotalSamplesDrawn/1000000.f) << llendl; - llinfos << "Total triangles drawn: " << llformat("%.3f million", sTotalTrianglesDrawn/1000000.f) << llendl; + for (std::vector::iterator iter = sorted.begin(); iter != sorted.end(); ++iter) + { + (*iter)->dumpStats(); + } + + llinfos << "-----------------------------------" << llendl; + llinfos << "Total rendering time: " << llformat("%.4f ms", sTotalTimeElapsed/1000000.f) << llendl; + llinfos << "Total samples drawn: " << llformat("%.4f million", sTotalSamplesDrawn/1000000.f) << llendl; + llinfos << "Total triangles drawn: " << llformat("%.3f million", sTotalTrianglesDrawn/1000000.f) << llendl; + } } void LLGLSLShader::clearStats() @@ -175,7 +179,7 @@ void LLGLSLShader::dumpStats() } } llinfos << "=============================================" << llendl; - + F32 ms = mTimeElapsed/1000000.f; F32 seconds = ms/1000.f; @@ -221,6 +225,7 @@ void LLGLSLShader::placeProfileQuery() #if !LL_DARWIN if (mTimerQuery == 0) { + glGenQueriesARB(1, &mSamplesQuery); glGenQueriesARB(1, &mTimerQuery); } @@ -257,7 +262,7 @@ void LLGLSLShader::placeProfileQuery() } - glBeginQueryARB(GL_SAMPLES_PASSED, 1); + glBeginQueryARB(GL_SAMPLES_PASSED, mSamplesQuery); glBeginQueryARB(GL_TIME_ELAPSED, mTimerQuery); #endif } @@ -272,7 +277,7 @@ void LLGLSLShader::readProfileQuery(U32 count, U32 mode) glGetQueryObjectui64v(mTimerQuery, GL_QUERY_RESULT, &time_elapsed); U64 samples_passed = 0; - glGetQueryObjectui64v(1, GL_QUERY_RESULT, &samples_passed); + glGetQueryObjectui64v(mSamplesQuery, GL_QUERY_RESULT, &samples_passed); sTotalTimeElapsed += time_elapsed; mTimeElapsed += time_elapsed; @@ -307,14 +312,15 @@ LLGLSLShader::LLGLSLShader() mShaderLevel(0), mShaderGroup(SG_DEFAULT), mUniformsDirty(FALSE), - mTimerQuery(0) + mTimerQuery(0), + mSamplesQuery(0) + { } LLGLSLShader::~LLGLSLShader() { - } void LLGLSLShader::unload() @@ -349,6 +355,18 @@ void LLGLSLShader::unload() mProgramObject = 0; } + if (mTimerQuery) + { + glDeleteQueriesARB(1, &mTimerQuery); + mTimerQuery = 0; + } + + if (mSamplesQuery) + { + glDeleteQueriesARB(1, &mSamplesQuery); + mSamplesQuery = 0; + } + //hack to make apple not complain glGetError(); diff --git a/indra/llrender/llglslshader.h b/indra/llrender/llglslshader.h index 7b2f5f04c2..5abddf274b 100755 --- a/indra/llrender/llglslshader.h +++ b/indra/llrender/llglslshader.h @@ -51,6 +51,7 @@ public: S32 mIndexedTextureChannels; bool disableTextureIndex; bool hasAlphaMask; + bool attachNothing; // char numLights; @@ -80,7 +81,7 @@ public: static bool sNoFixedFunction; static void initProfile(); - static void finishProfile(); + static void finishProfile(bool emit_report = true); static void startProfile(); static void stopProfile(U32 count, U32 mode); @@ -184,6 +185,7 @@ public: //statistcis for profiling shader performance U32 mTimerQuery; + U32 mSamplesQuery; U64 mTimeElapsed; static U64 sTotalTimeElapsed; U32 mTrianglesDrawn; diff --git a/indra/llrender/llrendertarget.cpp b/indra/llrender/llrendertarget.cpp index fe8110904d..33ef831cb4 100755 --- a/indra/llrender/llrendertarget.cpp +++ b/indra/llrender/llrendertarget.cpp @@ -388,6 +388,7 @@ void LLRenderTarget::release() // if (mFBO && (mTex.size() > 1)) { + glBindFramebuffer(GL_FRAMEBUFFER, mFBO); S32 z; for (z = mTex.size() - 1; z >= 1; z--) { diff --git a/indra/llrender/llshadermgr.cpp b/indra/llrender/llshadermgr.cpp index 6e04fc82df..b81ae9af09 100755 --- a/indra/llrender/llshadermgr.cpp +++ b/indra/llrender/llshadermgr.cpp @@ -73,7 +73,11 @@ BOOL LLShaderMgr::attachShaderFeatures(LLGLSLShader * shader) { llassert_always(shader != NULL); LLShaderFeatures *features = & shader->mFeatures; - + + if (features->attachNothing) + { + return TRUE; + } ////////////////////////////////////// // Attach Vertex Shader Features First ////////////////////////////////////// diff --git a/indra/newview/llfeaturemanager.cpp b/indra/newview/llfeaturemanager.cpp index ba6f26d3ef..938ea707f9 100755 --- a/indra/newview/llfeaturemanager.cpp +++ b/indra/newview/llfeaturemanager.cpp @@ -417,13 +417,67 @@ bool LLFeatureManager::parseFeatureTable(std::string filename) return parse_ok; } +F32 gpu_benchmark(); + bool LLFeatureManager::loadGPUClass() { + //get memory bandwidth from benchmark + F32 gbps = gpu_benchmark(); + + if (gbps < -1.f) + { //couldn't bench, use GLVersion + + if (gGLManager.mGLVersion < 2.f) + { + mGPUClass = GPU_CLASS_0; + } + else if (gGLManager.mGLVersion < 3.f) + { + mGPUClass = GPU_CLASS_1; + } + else if (gGLManager.mGLVersion < 3.3f) + { + mGPUClass = GPU_CLASS_2; + } + else if (gGLManager.mGLVersion < 4.f) + { + mGPUClass = GPU_CLASS_3; + } + else + { + mGPUClass = GPU_CLASS_4; + } + } + else if (gbps < 5.f) + { + mGPUClass = GPU_CLASS_0; + } + else if (gbps < 10.f) + { + mGPUClass = GPU_CLASS_1; + } + else if (gbps < 20.f) + { + mGPUClass = GPU_CLASS_2; + } + else if (gbps < 40.f) + { + mGPUClass = GPU_CLASS_3; + } + else if (gbps < 80.f) + { + mGPUClass = GPU_CLASS_4; + } + else + { + mGPUClass = GPU_CLASS_5; + } + // defaults - mGPUClass = GPU_CLASS_UNKNOWN; mGPUString = gGLManager.getRawGLString(); - mGPUSupported = FALSE; + mGPUSupported = TRUE; +#if 0 // first table is in the app dir std::string app_path = gDirUtilp->getAppRODataDir(); app_path += gDirUtilp->getDirDelimiter(); @@ -451,8 +505,8 @@ bool LLFeatureManager::loadGPUClass() { parse_ok = parseGPUTable(app_path); } - - return parse_ok; // indicates that the file parsed correctly, not that the gpu was recognized +#endif + return true; // indicates that the file parsed correctly, not that the gpu was recognized } @@ -719,6 +773,7 @@ void LLFeatureManager::init() void LLFeatureManager::applyRecommendedSettings() { + loadGPUClass(); // apply saved settings // cap the level at 2 (high) U32 level = llmax(GPU_CLASS_0, llmin(mGPUClass, GPU_CLASS_5)); diff --git a/indra/newview/llglsandbox.cpp b/indra/newview/llglsandbox.cpp index c4c1827266..92553e875d 100755 --- a/indra/newview/llglsandbox.cpp +++ b/indra/newview/llglsandbox.cpp @@ -878,13 +878,32 @@ void LLViewerObjectList::renderObjectBeacons() } -void gpu_benchmark() +F32 gpu_benchmark() { - if (!LLGLSLShader::sNoFixedFunction) - { //don't bother benchmarking the fixed function - return; + if (!gGLManager.mHasShaderObjects || !gGLManager.mHasTimerQuery) + { //don't bother benchmarking the fixed function or using CPU timers + return -1.f; } + + if (gBenchmarkProgram.mProgramObject == 0) + { + LLViewerShaderMgr::instance()->initAttribsAndUniforms(); + + gBenchmarkProgram.mName = "Benchmark Shader"; + gBenchmarkProgram.mFeatures.attachNothing = true; + gBenchmarkProgram.mShaderFiles.clear(); + gBenchmarkProgram.mShaderFiles.push_back(std::make_pair("interface/benchmarkV.glsl", GL_VERTEX_SHADER_ARB)); + gBenchmarkProgram.mShaderFiles.push_back(std::make_pair("interface/benchmarkF.glsl", GL_FRAGMENT_SHADER_ARB)); + gBenchmarkProgram.mShaderLevel = 1; + if (!gBenchmarkProgram.createShader(NULL, NULL)) + { + return -1.f; + } + } + + LLGLDisable blend(GL_BLEND); + //measure memory bandwidth by: // - allocating a batch of textures and render targets // - rendering those textures to those render targets @@ -981,7 +1000,7 @@ void gpu_benchmark() gBenchmarkProgram.unbind(); - LLGLSLShader::finishProfile(); + LLGLSLShader::finishProfile(false); LLImageGL::deleteTextures(count, source); @@ -1007,5 +1026,7 @@ void gpu_benchmark() { llinfos << "ARB_timer_query unavailable." << llendl; } + + return gbps; } diff --git a/indra/newview/llviewermenu.cpp b/indra/newview/llviewermenu.cpp index c7c8da27f3..d5acec7c9a 100755 --- a/indra/newview/llviewermenu.cpp +++ b/indra/newview/llviewermenu.cpp @@ -7197,7 +7197,7 @@ class LLAdvancedClickRenderProfile: public view_listener_t } }; -void gpu_benchmark(); +F32 gpu_benchmark(); class LLAdvancedClickRenderBenchmark: public view_listener_t { -- cgit v1.2.3 From 7defa95f6778778d850aa1aba3ea8cb16396d0dd Mon Sep 17 00:00:00 2001 From: Dave Parks Date: Fri, 21 Mar 2014 14:30:14 -0500 Subject: MAINT-3131 fallback to CPU timers for return value of benchmark if ARB_timer_query is unavailable. --- indra/newview/llglsandbox.cpp | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) (limited to 'indra') diff --git a/indra/newview/llglsandbox.cpp b/indra/newview/llglsandbox.cpp index 92553e875d..f620bf19fb 100755 --- a/indra/newview/llglsandbox.cpp +++ b/indra/newview/llglsandbox.cpp @@ -880,8 +880,8 @@ void LLViewerObjectList::renderObjectBeacons() F32 gpu_benchmark() { - if (!gGLManager.mHasShaderObjects || !gGLManager.mHasTimerQuery) - { //don't bother benchmarking the fixed function or using CPU timers + if (!gGLManager.mHasShaderObjects) + { //don't bother benchmarking the fixed function return -1.f; } @@ -1011,15 +1011,15 @@ F32 gpu_benchmark() llinfos << "Memory bandwidth is " << llformat("%.3f", gbps) << "GB/sec according to CPU timers" << llendl; - F32 ms = gBenchmarkProgram.mTimeElapsed/1000000.f; - F32 seconds = ms/1000.f; - - F64 samples_drawn = res*res*count*samples; - F32 samples_sec = (samples_drawn/1000000000.0)/seconds; - gbps = samples_sec*8; - if (gGLManager.mHasTimerQuery) { + F32 ms = gBenchmarkProgram.mTimeElapsed/1000000.f; + F32 seconds = ms/1000.f; + + F64 samples_drawn = res*res*count*samples; + F32 samples_sec = (samples_drawn/1000000000.0)/seconds; + gbps = samples_sec*8; + llinfos << "Memory bandwidth is " << llformat("%.3f", gbps) << "GB/sec according to ARB_timer_query" << llendl; } else -- cgit v1.2.3 From c142696f9dd18dcebee903444225026caf08e6e3 Mon Sep 17 00:00:00 2001 From: Dave Parks Date: Fri, 21 Mar 2014 14:33:46 -0500 Subject: MAINT-3131 Actually fallback to GL version guess instead of treating error value from gpu_benchmark as actual memory bandwidth. --- indra/newview/llfeaturemanager.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'indra') diff --git a/indra/newview/llfeaturemanager.cpp b/indra/newview/llfeaturemanager.cpp index 938ea707f9..5f37c0ea73 100755 --- a/indra/newview/llfeaturemanager.cpp +++ b/indra/newview/llfeaturemanager.cpp @@ -424,7 +424,7 @@ bool LLFeatureManager::loadGPUClass() //get memory bandwidth from benchmark F32 gbps = gpu_benchmark(); - if (gbps < -1.f) + if (gbps < 0.f) { //couldn't bench, use GLVersion if (gGLManager.mGLVersion < 2.f) -- cgit v1.2.3 From e43f43b4d9dbcaeb20884772b3fcba6df5e9a62c Mon Sep 17 00:00:00 2001 From: Dave Parks Date: Mon, 24 Mar 2014 11:56:02 -0500 Subject: MAINT-3131 Attempt to work around drivers that lack timer query and ignore glFinish calls. --- indra/newview/llglsandbox.cpp | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) (limited to 'indra') diff --git a/indra/newview/llglsandbox.cpp b/indra/newview/llglsandbox.cpp index f620bf19fb..0a423e6785 100755 --- a/indra/newview/llglsandbox.cpp +++ b/indra/newview/llglsandbox.cpp @@ -969,6 +969,8 @@ F32 gpu_benchmark() //wait for any previoius GL commands to finish glFinish(); + bool busted_finish = false; + for (S32 c = -1; c < samples; ++c) { LLTimer timer; @@ -983,7 +985,18 @@ F32 gpu_benchmark() } //wait for current batch of copies to finish - glFinish(); + if (busted_finish) + { + //read a pixel off the last target since some drivers seem to ignore glFinish + dest[count-1].bindTarget(); + U32 pixel = 0; + glReadPixels(0,0,1,1,GL_RGBA, GL_UNSIGNED_BYTE, &pixel); + dest[count-1].flush(); + } + else + { + glFinish(); + } F32 time = timer.getElapsedTimeF32(); @@ -994,7 +1007,14 @@ F32 gpu_benchmark() F32 gbps = gb/time; - results.push_back(gbps); + if (!gGLManager.mHasTimerQuery && !busted_finish && gbps > 128.f) + { //unrealistically high bandwidth for a card without timer queries, glFinish is probably ignored + busted_finish = true; + } + else + { + results.push_back(gbps); + } } } -- cgit v1.2.3 From 5cca78e718f15522cc3db9aec76aa910dd696aa8 Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Mon, 23 Jun 2014 14:23:33 -0400 Subject: First HTTP pipelining viewer. Enable pipelining for GetTexture and GetMesh2 at a pipeline depth of 5. Create global debug option, HttpPipelining, to enable and disable HTTP pipelining (defaults to true). Tweak texture and mesh low- and high-water request levels based on pipelining status and depth. Fixup texture console which was damaged in a recent release. Split logging of the no-request HTTP error case into two cases: one for missing URL in HTTP request, one for HTTP request not created. A refactor in llcorehttp is coming: I will be moving all libcurl- using code into libcurl-specific modules. --- indra/llcorehttp/_httpinternal.h | 5 +- indra/llcorehttp/_httplibcurl.cpp | 69 ++++++++++++- indra/llcorehttp/_httppolicy.cpp | 11 ++- indra/llcorehttp/_httppolicyclass.cpp | 8 +- indra/llcorehttp/httprequest.h | 35 ++++++- indra/newview/app_settings/settings.xml | 11 +++ indra/newview/llappcorehttp.cpp | 169 ++++++++++++++++++++++---------- indra/newview/llappcorehttp.h | 39 ++++++-- indra/newview/llmeshrepository.cpp | 82 +++++++++------- indra/newview/lltexturefetch.cpp | 126 ++++++++++++++---------- indra/newview/lltexturefetch.h | 4 +- 11 files changed, 396 insertions(+), 163 deletions(-) (limited to 'indra') diff --git a/indra/llcorehttp/_httpinternal.h b/indra/llcorehttp/_httpinternal.h index f80d7f60f5..a2a60ca056 100755 --- a/indra/llcorehttp/_httpinternal.h +++ b/indra/llcorehttp/_httpinternal.h @@ -145,8 +145,11 @@ const int HTTP_CONNECTION_LIMIT_DEFAULT = 8; const int HTTP_CONNECTION_LIMIT_MIN = 1; const int HTTP_CONNECTION_LIMIT_MAX = 256; -// Miscellaneous defaults +// Pipelining limits const long HTTP_PIPELINING_DEFAULT = 0L; +const long HTTP_PIPELINING_MAX = 20L; + +// Miscellaneous defaults const bool HTTP_USE_RETRY_AFTER_DEFAULT = true; const long HTTP_THROTTLE_RATE_DEFAULT = 0L; diff --git a/indra/llcorehttp/_httplibcurl.cpp b/indra/llcorehttp/_httplibcurl.cpp index e56bc84174..fb907f6318 100755 --- a/indra/llcorehttp/_httplibcurl.cpp +++ b/indra/llcorehttp/_httplibcurl.cpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012-2013, Linden Research, Inc. + * Copyright (C) 2012-2014, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -33,6 +33,15 @@ #include "llhttpconstants.h" +namespace +{ + +// Error testing and reporting for libcurl status codes +void check_curl_multi_code(CURLMcode code); +void check_curl_multi_code(CURLMcode code, int curl_setopt_option); + +} // end anonymous namespace + namespace LLCore { @@ -92,14 +101,44 @@ void HttpLibcurl::start(int policy_count) llassert_always(policy_count <= HTTP_POLICY_CLASS_LIMIT); llassert_always(! mMultiHandles); // One-time call only + HttpPolicy & policy(mService->getPolicy()); mPolicyCount = policy_count; mMultiHandles = new CURLM * [mPolicyCount]; mActiveHandles = new int [mPolicyCount]; for (int policy_class(0); policy_class < mPolicyCount; ++policy_class) { - mMultiHandles[policy_class] = curl_multi_init(); + HttpPolicyClass & options(policy.getClassOptions(policy_class)); + mActiveHandles[policy_class] = 0; + if (NULL == (mMultiHandles[policy_class] = curl_multi_init())) + { + LL_ERRS("CoreHttp") << "Failed to allocate multi handle in libcurl." + << LL_ENDL; + } + + if (options.mPipelining > 1) + { + CURLMcode code; + + // We'll try to do pipelining on this multihandle + code = curl_multi_setopt(mMultiHandles[policy_class], + CURLMOPT_PIPELINING, + 1L); + check_curl_multi_code(code, CURLMOPT_PIPELINING); + code = curl_multi_setopt(mMultiHandles[policy_class], + CURLMOPT_MAX_PIPELINE_LENGTH, + long(options.mPipelining)); + check_curl_multi_code(code, CURLMOPT_MAX_PIPELINE_LENGTH); + code = curl_multi_setopt(mMultiHandles[policy_class], + CURLMOPT_MAX_HOST_CONNECTIONS, + long(options.mPerHostConnectionLimit)); + check_curl_multi_code(code, CURLMOPT_MAX_HOST_CONNECTIONS); + code = curl_multi_setopt(mMultiHandles[policy_class], + CURLMOPT_MAX_TOTAL_CONNECTIONS, + long(options.mConnectionLimit)); + check_curl_multi_code(code, CURLMOPT_MAX_TOTAL_CONNECTIONS); + } } } @@ -376,3 +415,29 @@ struct curl_slist * append_headers_to_slist(const HttpHeaders * headers, struct } // end namespace LLCore + + +namespace +{ + +void check_curl_multi_code(CURLMcode code, int curl_setopt_option) +{ + if (CURLM_OK != code) + { + LL_WARNS("CoreHttp") << "libcurl multi error detected: " << curl_multi_strerror(code) + << ", curl_multi_setopt option: " << curl_setopt_option + << LL_ENDL; + } +} + + +void check_curl_multi_code(CURLMcode code) +{ + if (CURLM_OK != code) + { + LL_WARNS("CoreHttp") << "libcurl multi error detected: " << curl_multi_strerror(code) + << LL_ENDL; + } +} + +} // end anonymous namespace diff --git a/indra/llcorehttp/_httppolicy.cpp b/indra/llcorehttp/_httppolicy.cpp index fd5a93e192..bb7959b578 100755 --- a/indra/llcorehttp/_httppolicy.cpp +++ b/indra/llcorehttp/_httppolicy.cpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012-2013, Linden Research, Inc. + * Copyright (C) 2012-2014, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -128,7 +128,8 @@ void HttpPolicy::shutdown() void HttpPolicy::start() -{} +{ +} void HttpPolicy::addOp(HttpOpRequest * op) @@ -234,7 +235,11 @@ HttpService::ELoopSpeed HttpPolicy::processReadyQueue() } int active(transport.getActiveCountInClass(policy_class)); - int needed(state.mOptions.mConnectionLimit - active); // Expect negatives here + int active_limit(state.mOptions.mPipelining > 1L + ? (state.mOptions.mPerHostConnectionLimit + * state.mOptions.mPipelining) + : state.mOptions.mConnectionLimit); + int needed(active_limit - active); // Expect negatives here if (needed > 0) { diff --git a/indra/llcorehttp/_httppolicyclass.cpp b/indra/llcorehttp/_httppolicyclass.cpp index f34a8e9f1e..2c0f650155 100755 --- a/indra/llcorehttp/_httppolicyclass.cpp +++ b/indra/llcorehttp/_httppolicyclass.cpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012-2013, Linden Research, Inc. + * Copyright (C) 2012-2014, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -78,8 +78,8 @@ HttpStatus HttpPolicyClass::set(HttpRequest::EPolicyOption opt, long value) mPerHostConnectionLimit = llclamp(value, long(HTTP_CONNECTION_LIMIT_MIN), mConnectionLimit); break; - case HttpRequest::PO_ENABLE_PIPELINING: - mPipelining = llclamp(value, 0L, 1L); + case HttpRequest::PO_PIPELINING_DEPTH: + mPipelining = llclamp(value, 0L, HTTP_PIPELINING_MAX); break; case HttpRequest::PO_THROTTLE_RATE: @@ -106,7 +106,7 @@ HttpStatus HttpPolicyClass::get(HttpRequest::EPolicyOption opt, long * value) co *value = mPerHostConnectionLimit; break; - case HttpRequest::PO_ENABLE_PIPELINING: + case HttpRequest::PO_PIPELINING_DEPTH: *value = mPipelining; break; diff --git a/indra/llcorehttp/httprequest.h b/indra/llcorehttp/httprequest.h index 651654844a..7f23723b0b 100755 --- a/indra/llcorehttp/httprequest.h +++ b/indra/llcorehttp/httprequest.h @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012-2013, Linden Research, Inc. + * Copyright (C) 2012-2014, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -183,11 +183,38 @@ public: /// Global only PO_TRACE, - /// Suitable requests are allowed to pipeline on their - /// connections when they ask for it. + /// If greater than 1, suitable requests are allowed to + /// pipeline on their connections when they ask for it. + /// Value gives the maximum number of outstanding requests + /// on a connection. + /// + /// There is some interaction between PO_CONNECTION_LIMIT, + /// PO_PER_HOST_CONNECTION_LIMIT, and PO_PIPELINING_DEPTH. + /// When PIPELINING_DEPTH is 0 or 1 (no pipelining), this + /// library manages connection lifecycle and honors the + /// PO_CONNECTION_LIMIT setting as the maximum in-flight + /// request limit. Libcurl itself may be caching additional + /// connections under its connection cache policy. + /// + /// When PIPELINING_DEPTH is 2 or more, libcurl performs + /// connection management and both PO_CONNECTION_LIMIT and + /// PO_PER_HOST_CONNECTION_LIMIT should be set and non-zero. + /// In this case (as of libcurl 7.37.0), libcurl will + /// open new connections in preference to pipelining, up + /// to the above limits at which time pipelining begins. + /// And as usual, an additional cache of open but inactive + /// connections may still be maintained within libcurl. + /// For SL, a good rule-of-thumb is to set + /// PO_PER_HOST_CONNECTION_LIMIT to the user-visible + /// concurrency value and PO_CONNECTION_LIMIT to twice + /// that for baked texture loads and region crossings where + /// additional connection load will be tolerated. If + /// either limit is 0, libcurl will prefer pipelining + /// over connection creation, which is still interesting, + /// but won't be pursued at this time. /// /// Per-class only - PO_ENABLE_PIPELINING, + PO_PIPELINING_DEPTH, /// Controls whether client-side throttling should be /// performed on this policy class. Positive values diff --git a/indra/newview/app_settings/settings.xml b/indra/newview/app_settings/settings.xml index 8d3a7327b9..eb8a8bff51 100755 --- a/indra/newview/app_settings/settings.xml +++ b/indra/newview/app_settings/settings.xml @@ -4467,6 +4467,17 @@ Value + HttpPipelining + + Comment + If true, viewer will pipeline HTTP requests to servers. Static. + Persist + 1 + Type + Boolean + Value + 1 + IMShowTimestamps Comment diff --git a/indra/newview/llappcorehttp.cpp b/indra/newview/llappcorehttp.cpp index 70dcffefb2..deda0ccb41 100755 --- a/indra/newview/llappcorehttp.cpp +++ b/indra/newview/llappcorehttp.cpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012-2013, Linden Research, Inc. + * Copyright (C) 2012-2014, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -40,6 +40,8 @@ // be open at a time. const F64 LLAppCoreHttp::MAX_THREAD_WAIT_TIME(10.0); +const long LLAppCoreHttp::PIPELINING_DEPTH(5L); + static const struct { LLAppCoreHttp::EAppPolicy mPolicy; @@ -47,42 +49,43 @@ static const struct U32 mMin; U32 mMax; U32 mRate; + bool mPipelined; std::string mKey; const char * mUsage; } init_data[] = // Default and dynamic values for classes { { - LLAppCoreHttp::AP_DEFAULT, 8, 8, 8, 0, + LLAppCoreHttp::AP_DEFAULT, 8, 8, 8, 0, false, "", "other" }, { - LLAppCoreHttp::AP_TEXTURE, 8, 1, 12, 0, + LLAppCoreHttp::AP_TEXTURE, 8, 1, 12, 0, true, "TextureFetchConcurrency", "texture fetch" }, { - LLAppCoreHttp::AP_MESH1, 32, 1, 128, 100, + LLAppCoreHttp::AP_MESH1, 32, 1, 128, 100, false, "MeshMaxConcurrentRequests", "mesh fetch" }, { - LLAppCoreHttp::AP_MESH2, 8, 1, 32, 100, + LLAppCoreHttp::AP_MESH2, 8, 1, 32, 100, true, "Mesh2MaxConcurrentRequests", "mesh2 fetch" }, { - LLAppCoreHttp::AP_LARGE_MESH, 2, 1, 8, 0, + LLAppCoreHttp::AP_LARGE_MESH, 2, 1, 8, 0, false, "", "large mesh fetch" }, { - LLAppCoreHttp::AP_UPLOADS, 2, 1, 8, 0, + LLAppCoreHttp::AP_UPLOADS, 2, 1, 8, 0, false, "", "asset upload" }, { - LLAppCoreHttp::AP_LONG_POLL, 32, 32, 32, 0, + LLAppCoreHttp::AP_LONG_POLL, 32, 32, 32, 0, false, "", "long poll" } @@ -91,18 +94,20 @@ static const struct static void setting_changed(); +LLAppCoreHttp::HttpClass::HttpClass() + : mPolicy(LLCore::HttpRequest::DEFAULT_POLICY_ID), + mConnLimit(0U), + mPipelined(false) +{} + + LLAppCoreHttp::LLAppCoreHttp() : mRequest(NULL), mStopHandle(LLCORE_HTTP_HANDLE_INVALID), mStopRequested(0.0), - mStopped(false) -{ - for (int i(0); i < LL_ARRAY_SIZE(mPolicies); ++i) - { - mPolicies[i] = LLCore::HttpRequest::DEFAULT_POLICY_ID; - mSettings[i] = 0U; - } -} + mStopped(false), + mPipelined(true) +{} LLAppCoreHttp::~LLAppCoreHttp() @@ -121,6 +126,14 @@ void LLAppCoreHttp::init() << LL_ENDL; } + // Global pipelining preference from settings + static const std::string http_pipelining("HttpPipelining"); + if (gSavedSettings.controlExists(http_pipelining)) + { + // Default to true if absent. + mPipelined = gSavedSettings.getBOOL(http_pipelining); + } + // Point to our certs or SSH/https: will fail on connect status = LLCore::HttpRequest::setStaticPolicyOption(LLCore::HttpRequest::PO_CA_FILE, LLCore::HttpRequest::GLOBAL_POLICY_ID, @@ -157,27 +170,27 @@ void LLAppCoreHttp::init() } // Setup default policy and constrain if directed to - mPolicies[AP_DEFAULT] = LLCore::HttpRequest::DEFAULT_POLICY_ID; + mHttpClasses[AP_DEFAULT].mPolicy = LLCore::HttpRequest::DEFAULT_POLICY_ID; // Setup additional policies based on table and some special rules for (int i(0); i < LL_ARRAY_SIZE(init_data); ++i) { - const EAppPolicy policy(init_data[i].mPolicy); + const EAppPolicy app_policy(init_data[i].mPolicy); - if (AP_DEFAULT == policy) + if (AP_DEFAULT == app_policy) { // Pre-created continue; } - mPolicies[policy] = LLCore::HttpRequest::createPolicyClass(); - if (! mPolicies[policy]) + mHttpClasses[app_policy].mPolicy = LLCore::HttpRequest::createPolicyClass(); + if (! mHttpClasses[app_policy].mPolicy) { // Use default policy (but don't accidentally modify default) LL_WARNS("Init") << "Failed to create HTTP policy class for " << init_data[i].mUsage << ". Using default policy." << LL_ENDL; - mPolicies[policy] = mPolicies[AP_DEFAULT]; + mHttpClasses[app_policy].mPolicy = mHttpClasses[AP_DEFAULT].mPolicy; continue; } } @@ -196,6 +209,9 @@ void LLAppCoreHttp::init() << LL_ENDL; } + // *NOTE: Pipelining isn't dynamic yet. When it is, add a global + // signal for the setting here. + // Register signals for settings and state changes for (int i(0); i < LL_ARRAY_SIZE(init_data); ++i) { @@ -209,7 +225,7 @@ void LLAppCoreHttp::init() } else { - mSettingsSignal[i] = cntrl_ptr->getCommitSignal()->connect(boost::bind(&setting_changed)); + mHttpClasses[i].mSettingsSignal = cntrl_ptr->getCommitSignal()->connect(boost::bind(&setting_changed)); } } } @@ -261,9 +277,9 @@ void LLAppCoreHttp::cleanup() } } - for (int i(0); i < LL_ARRAY_SIZE(init_data); ++i) + for (int i(0); i < LL_ARRAY_SIZE(mHttpClasses); ++i) { - mSettingsSignal[i].disconnect(); + mHttpClasses[i].mSettingsSignal.disconnect(); } delete mRequest; @@ -278,30 +294,57 @@ void LLAppCoreHttp::cleanup() } } + void LLAppCoreHttp::refreshSettings(bool initial) { LLCore::HttpStatus status; for (int i(0); i < LL_ARRAY_SIZE(init_data); ++i) { - const EAppPolicy policy(init_data[i].mPolicy); + const EAppPolicy app_policy(init_data[i].mPolicy); - // Set any desired throttle - if (initial && init_data[i].mRate) + if (initial) { - // Init-time only, can use the static setters here - status = LLCore::HttpRequest::setStaticPolicyOption(LLCore::HttpRequest::PO_THROTTLE_RATE, - mPolicies[policy], - init_data[i].mRate, - NULL); - if (! status) + // Init-time only settings, can use the static setters here + + if (init_data[i].mRate) { - LL_WARNS("Init") << "Unable to set " << init_data[i].mUsage - << " throttle rate. Reason: " << status.toString() - << LL_ENDL; + // Set any desired throttle + status = LLCore::HttpRequest::setStaticPolicyOption(LLCore::HttpRequest::PO_THROTTLE_RATE, + mHttpClasses[app_policy].mPolicy, + init_data[i].mRate, + NULL); + if (! status) + { + LL_WARNS("Init") << "Unable to set " << init_data[i].mUsage + << " throttle rate. Reason: " << status.toString() + << LL_ENDL; + } + } + + mHttpClasses[app_policy].mPipelined = false; + if (mPipelined && init_data[i].mPipelined) + { + // Pipelining election is currently static (init-time). + status = LLCore::HttpRequest::setStaticPolicyOption(LLCore::HttpRequest::PO_PIPELINING_DEPTH, + mHttpClasses[app_policy].mPolicy, + PIPELINING_DEPTH, + NULL); + if (! status) + { + LL_WARNS("Init") << "Unable to set " << init_data[i].mUsage + << " to pipelined mode. Reason: " << status.toString() + << LL_ENDL; + } + else + { + mHttpClasses[app_policy].mPipelined = true; + } } } + // Init- or run-time settings + // Get target connection concurrency value U32 setting(init_data[i].mDefault); if (! init_data[i].mKey.empty() && gSavedSettings.controlExists(init_data[i].mKey)) @@ -314,19 +357,31 @@ void LLAppCoreHttp::refreshSettings(bool initial) } } - if (! initial && setting == mSettings[policy]) + if (! initial && setting == mHttpClasses[app_policy].mConnLimit) { // Unchanged, try next setting continue; } - // Set it and report - // *TODO: These are intended to be per-host limits when we can - // support that in llcorehttp/libcurl. + // Set it and report. Strategies depend on pipelining: + // + // No Pipelining. Llcorehttp manages connections itself based + // on the PO_CONNECTION_LIMIT setting. Set both limits to the + // same value for logical consistency. In the future, may + // hand over connection management to libcurl after the + // connection cache has been better vetted. + // + // Pipelining. Libcurl is allowed to manage connections to a + // great degree. Steady state will connection limit based on + // the per-host setting. Transitions (region crossings, new + // avatars, etc.) can request additional outbound connections + // to other servers via 2X total connection limit. + // LLCore::HttpHandle handle; handle = mRequest->setPolicyOption(LLCore::HttpRequest::PO_CONNECTION_LIMIT, - mPolicies[policy], - setting, NULL); + mHttpClasses[app_policy].mPolicy, + (mHttpClasses[app_policy].mPipelined ? 2 * setting : setting), + NULL); if (LLCORE_HTTP_HANDLE_INVALID == handle) { status = mRequest->getStatus(); @@ -336,16 +391,30 @@ void LLAppCoreHttp::refreshSettings(bool initial) } else { - LL_DEBUGS("Init") << "Changed " << init_data[i].mUsage - << " concurrency. New value: " << setting - << LL_ENDL; - mSettings[policy] = setting; - if (initial && setting != init_data[i].mDefault) + handle = mRequest->setPolicyOption(LLCore::HttpRequest::PO_PER_HOST_CONNECTION_LIMIT, + mHttpClasses[app_policy].mPolicy, + setting, + NULL); + if (LLCORE_HTTP_HANDLE_INVALID == handle) { - LL_INFOS("Init") << "Application settings overriding default " << init_data[i].mUsage - << " concurrency. New value: " << setting + status = mRequest->getStatus(); + LL_WARNS("Init") << "Unable to set " << init_data[i].mUsage + << " per-host concurrency. Reason: " << status.toString() << LL_ENDL; } + else + { + LL_DEBUGS("Init") << "Changed " << init_data[i].mUsage + << " concurrency. New value: " << setting + << LL_ENDL; + mHttpClasses[app_policy].mConnLimit = setting; + if (initial && setting != init_data[i].mDefault) + { + LL_INFOS("Init") << "Application settings overriding default " << init_data[i].mUsage + << " concurrency. New value: " << setting + << LL_ENDL; + } + } } } } diff --git a/indra/newview/llappcorehttp.h b/indra/newview/llappcorehttp.h index 40e3042b84..63c8a11180 100755 --- a/indra/newview/llappcorehttp.h +++ b/indra/newview/llappcorehttp.h @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012-2013, Linden Research, Inc. + * Copyright (C) 2012-2014, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -41,6 +41,8 @@ class LLAppCoreHttp : public LLCore::HttpHandler { public: + static const long PIPELINING_DEPTH; + typedef LLCore::HttpRequest::policy_t policy_t; enum EAppPolicy @@ -70,7 +72,7 @@ public: /// Long poll: no /// Concurrency: high /// Request rate: high - /// Pipelined: soon + /// Pipelined: yes AP_TEXTURE, /// Legacy mesh fetching policy class. Used to @@ -98,7 +100,7 @@ public: /// Long poll: no /// Concurrency: high /// Request rate: high - /// Pipelined: soon + /// Pipelined: yes AP_MESH2, /// Large mesh fetching policy class. Used to @@ -116,7 +118,7 @@ public: /// Long poll: no /// Concurrency: low /// Request rate: low - /// Pipelined: soon + /// Pipelined: no AP_LARGE_MESH, /// Asset upload policy class. Used to store @@ -180,7 +182,13 @@ public: // application function. policy_t getPolicy(EAppPolicy policy) const { - return mPolicies[policy]; + return mHttpClasses[policy].mPolicy; + } + + // Return whether a policy is using pipelined operations. + bool isPipelined(EAppPolicy policy) const + { + return mHttpClasses[policy].mPipelined; } // Apply initial or new settings from the environment. @@ -190,13 +198,26 @@ private: static const F64 MAX_THREAD_WAIT_TIME; private: - LLCore::HttpRequest * mRequest; // Request queue to issue shutdowns + + // PODish container for per-class settings and state. + struct HttpClass + { + public: + HttpClass(); + + public: + policy_t mPolicy; // Policy class id for the class + U32 mConnLimit; + bool mPipelined; + boost::signals2::connection mSettingsSignal; // Signal to global setting that affect this class (if any) + }; + + LLCore::HttpRequest * mRequest; // Request queue to issue shutdowns LLCore::HttpHandle mStopHandle; F64 mStopRequested; bool mStopped; - policy_t mPolicies[AP_COUNT]; // Policy class id for each connection set - U32 mSettings[AP_COUNT]; - boost::signals2::connection mSettingsSignal[AP_COUNT]; // Signals to global settings that affect us + HttpClass mHttpClasses[AP_COUNT]; + bool mPipelined; // Global setting }; diff --git a/indra/newview/llmeshrepository.cpp b/indra/newview/llmeshrepository.cpp index 80a427c0b8..c91ae975ea 100755 --- a/indra/newview/llmeshrepository.cpp +++ b/indra/newview/llmeshrepository.cpp @@ -343,9 +343,9 @@ const S32 REQUEST_HIGH_WATER_MAX = 150; // Should remain under 2X throttle const S32 REQUEST_LOW_WATER_MIN = 16; const S32 REQUEST_LOW_WATER_MAX = 75; const S32 REQUEST2_HIGH_WATER_MIN = 32; // Limits for GetMesh2 regions -const S32 REQUEST2_HIGH_WATER_MAX = 80; +const S32 REQUEST2_HIGH_WATER_MAX = 100; const S32 REQUEST2_LOW_WATER_MIN = 16; -const S32 REQUEST2_LOW_WATER_MAX = 40; +const S32 REQUEST2_LOW_WATER_MAX = 50; const U32 LARGE_MESH_FETCH_THRESHOLD = 1U << 21; // Size at which requests goes to narrow/slow queue const long SMALL_MESH_XFER_TIMEOUT = 120L; // Seconds to complete xfer, small mesh downloads const long LARGE_MESH_XFER_TIMEOUT = 600L; // Seconds to complete xfer, large downloads @@ -754,7 +754,9 @@ LLMeshRepoThread::LLMeshRepoThread() mHttpLargePolicyClass(LLCore::HttpRequest::DEFAULT_POLICY_ID), mHttpPriority(0), mGetMeshVersion(2) - { +{ + LLAppCoreHttp & app_core_http(LLAppViewer::instance()->getAppCoreHttp()); + mMutex = new LLMutex(NULL); mHeaderMutex = new LLMutex(NULL); mSignal = new LLCondition(NULL); @@ -767,10 +769,10 @@ LLMeshRepoThread::LLMeshRepoThread() mHttpLargeOptions->setUseRetryAfter(gSavedSettings.getBOOL("MeshUseHttpRetryAfter")); mHttpHeaders = new LLCore::HttpHeaders; mHttpHeaders->append("Accept", "application/vnd.ll.mesh"); - mHttpPolicyClass = LLAppViewer::instance()->getAppCoreHttp().getPolicy(LLAppCoreHttp::AP_MESH2); - mHttpLegacyPolicyClass = LLAppViewer::instance()->getAppCoreHttp().getPolicy(LLAppCoreHttp::AP_MESH1); - mHttpLargePolicyClass = LLAppViewer::instance()->getAppCoreHttp().getPolicy(LLAppCoreHttp::AP_LARGE_MESH); - } + mHttpPolicyClass = app_core_http.getPolicy(LLAppCoreHttp::AP_MESH2); + mHttpLegacyPolicyClass = app_core_http.getPolicy(LLAppCoreHttp::AP_MESH1); + mHttpLargePolicyClass = app_core_http.getPolicy(LLAppCoreHttp::AP_LARGE_MESH); +} LLMeshRepoThread::~LLMeshRepoThread() @@ -846,48 +848,49 @@ void LLMeshRepoThread::run() { // Dispatch all HttpHandler notifications mHttpRequest->update(0L); - } + } sRequestWaterLevel = mHttpRequestSet.size(); // Stats data update // NOTE: order of queue processing intentionally favors LOD requests over header requests while (!mLODReqQ.empty() && mHttpRequestSet.size() < sRequestHighWater) - { + { if (! mMutex) - { + { break; } - mMutex->lock(); - LODRequest req = mLODReqQ.front(); - mLODReqQ.pop(); - LLMeshRepository::sLODProcessing--; - mMutex->unlock(); + mMutex->lock(); + LODRequest req = mLODReqQ.front(); + mLODReqQ.pop(); + LLMeshRepository::sLODProcessing--; + mMutex->unlock(); + if (!fetchMeshLOD(req.mMeshParams, req.mLOD)) // failed, resubmit - { - mMutex->lock(); - mLODReqQ.push(req); + { + mMutex->lock(); + mLODReqQ.push(req); ++LLMeshRepository::sLODProcessing; - mMutex->unlock(); - } - } + mMutex->unlock(); + } + } while (!mHeaderReqQ.empty() && mHttpRequestSet.size() < sRequestHighWater) - { + { if (! mMutex) - { + { break; } - mMutex->lock(); - HeaderRequest req = mHeaderReqQ.front(); - mHeaderReqQ.pop(); - mMutex->unlock(); + mMutex->lock(); + HeaderRequest req = mHeaderReqQ.front(); + mHeaderReqQ.pop(); + mMutex->unlock(); if (!fetchMeshHeader(req.mMeshParams))//failed, resubmit - { - mMutex->lock(); - mHeaderReqQ.push(req) ; - mMutex->unlock(); - } - } + { + mMutex->lock(); + mHeaderReqQ.push(req) ; + mMutex->unlock(); + } + } // For the final three request lists, similar goal to above but // slightly different queue structures. Stay off the mutex when @@ -983,7 +986,7 @@ void LLMeshRepoThread::run() } } mMutex->unlock(); - } + } // For dev purposes only. A dynamic change could make this false // and that shouldn't assert. @@ -1250,7 +1253,6 @@ bool LLMeshRepoThread::fetchMeshSkinInfo(const LLUUID& mesh_id) << LL_ENDL; delete handler; ret = false; - } else { @@ -1860,7 +1862,7 @@ LLMeshUploadThread::LLMeshUploadThread(LLMeshUploadThread::instance_list& data, bool upload_skin, bool upload_joints, const std::string & upload_url, bool do_upload, LLHandle fee_observer, LLHandle upload_observer) -: LLThread("mesh upload"), + : LLThread("mesh upload"), LLCore::HttpHandler(), mDiscarded(false), mDoUpload(do_upload), @@ -3198,9 +3200,13 @@ void LLMeshRepository::notifyLoadedMeshes() else { // GetMesh2 operation with keepalives, etc. With pipelining, - // we'll increase this. + // we'll increase this. See llappcorehttp and llcorehttp for + // discussion on connection strategies. + LLAppCoreHttp & app_core_http(LLAppViewer::instance()->getAppCoreHttp()); + S32 scale(app_core_http.isPipelined(LLAppCoreHttp::AP_MESH2) ? 10 : 5); + LLMeshRepoThread::sMaxConcurrentRequests = gSavedSettings.getU32("Mesh2MaxConcurrentRequests"); - LLMeshRepoThread::sRequestHighWater = llclamp(5 * S32(LLMeshRepoThread::sMaxConcurrentRequests), + LLMeshRepoThread::sRequestHighWater = llclamp(scale * S32(LLMeshRepoThread::sMaxConcurrentRequests), REQUEST2_HIGH_WATER_MIN, REQUEST2_HIGH_WATER_MAX); LLMeshRepoThread::sRequestLowWater = llclamp(LLMeshRepoThread::sRequestHighWater / 2, diff --git a/indra/newview/lltexturefetch.cpp b/indra/newview/lltexturefetch.cpp index 548ed98f39..c33884e1e6 100755 --- a/indra/newview/lltexturefetch.cpp +++ b/indra/newview/lltexturefetch.cpp @@ -241,8 +241,10 @@ LLTrace::EventStatHandle LLTextureFetch::sCacheReadLatency("te // Tuning/Parameterization Constants -static const S32 HTTP_REQUESTS_IN_QUEUE_HIGH_WATER = 40; // Maximum requests to have active in HTTP -static const S32 HTTP_REQUESTS_IN_QUEUE_LOW_WATER = 20; // Active level at which to refill +static const S32 HTTP_PIPE_REQUESTS_HIGH_WATER = 100; // Maximum requests to have active in HTTP (pipelined) +static const S32 HTTP_PIPE_REQUESTS_LOW_WATER = 50; // Active level at which to refill +static const S32 HTTP_NONPIPE_REQUESTS_HIGH_WATER = 40; +static const S32 HTTP_NONPIPE_REQUESTS_LOW_WATER = 20; // BUG-3323/SH-4375 // *NOTE: This is a heuristic value. Texture fetches have a habit of using a @@ -608,16 +610,16 @@ private: LLCore::HttpHandle mHttpHandle; // Handle of any active request LLCore::BufferArray * mHttpBufferArray; // Refcounted pointer to response data - S32 mHttpPolicyClass; + S32 mHttpPolicyClass; bool mHttpActive; // Active request to http library - U32 mHttpReplySize, // Actual received data size - mHttpReplyOffset; // Actual received data offset + U32 mHttpReplySize, // Actual received data size + mHttpReplyOffset; // Actual received data offset bool mHttpHasResource; // Counts against Fetcher's mHttpSemaphore // State history - U32 mCacheReadCount, - mCacheWriteCount, - mResourceWaitCount; // Requests entering WAIT_HTTP_RESOURCE2 + U32 mCacheReadCount, + mCacheWriteCount, + mResourceWaitCount; // Requests entering WAIT_HTTP_RESOURCE2 }; ////////////////////////////////////////////////////////////////////////////// @@ -1525,36 +1527,49 @@ bool LLTextureFetchWorker::doWork(S32 param) mRequestedOffset -= 1; mRequestedSize += 1; } - mHttpHandle = LLCORE_HTTP_HANDLE_INVALID; - if (!mUrl.empty()) - { - mRequestedTimer.reset(); - mLoaded = FALSE; - mGetStatus = LLCore::HttpStatus(); - mGetReason.clear(); - LL_DEBUGS(LOG_TXT) << "HTTP GET: " << mID << " Offset: " << mRequestedOffset - << " Bytes: " << mRequestedSize - << " Bandwidth(kbps): " << mFetcher->getTextureBandwidth() << "/" << mFetcher->mMaxBandwidth - << LL_ENDL; - // Will call callbackHttpGet when curl request completes - // Only server bake images use the returned headers currently, for getting retry-after field. - LLCore::HttpOptions *options = (mFTType == FTT_SERVER_BAKE) ? mFetcher->mHttpOptionsWithHeaders: mFetcher->mHttpOptions; - mHttpHandle = mFetcher->mHttpRequest->requestGetByteRange(mHttpPolicyClass, - mWorkPriority, - mUrl, - mRequestedOffset, - (mRequestedOffset + mRequestedSize) > HTTP_REQUESTS_RANGE_END_MAX - ? 0 - : mRequestedSize, - options, - mFetcher->mHttpHeaders, - this); + if (mUrl.empty()) + { + // *FIXME: This should not be reachable except it has become + // so after some recent 'work'. Need to track this down + // and illuminate the unenlightened. + LL_WARNS(LOG_TXT) << "HTTP GET request failed for " << mID + << " on empty URL." << LL_ENDL; + resetFormattedData(); + releaseHttpSemaphore(); + return true; // failed } + + mRequestedTimer.reset(); + mLoaded = FALSE; + mGetStatus = LLCore::HttpStatus(); + mGetReason.clear(); + LL_DEBUGS(LOG_TXT) << "HTTP GET: " << mID << " Offset: " << mRequestedOffset + << " Bytes: " << mRequestedSize + << " Bandwidth(kbps): " << mFetcher->getTextureBandwidth() << "/" << mFetcher->mMaxBandwidth + << LL_ENDL; + + // Will call callbackHttpGet when curl request completes + // Only server bake images use the returned headers currently, for getting retry-after field. + LLCore::HttpOptions *options = (mFTType == FTT_SERVER_BAKE) ? mFetcher->mHttpOptionsWithHeaders: mFetcher->mHttpOptions; + mHttpHandle = mFetcher->mHttpRequest->requestGetByteRange(mHttpPolicyClass, + mWorkPriority, + mUrl, + mRequestedOffset, + (mRequestedOffset + mRequestedSize) > HTTP_REQUESTS_RANGE_END_MAX + ? 0 + : mRequestedSize, + options, + mFetcher->mHttpHeaders, + this); if (LLCORE_HTTP_HANDLE_INVALID == mHttpHandle) { - LL_WARNS(LOG_TXT) << "HTTP GET request failed for " << mID << LL_ENDL; + LLCore::HttpStatus status(mFetcher->mHttpRequest->getStatus()); + LL_WARNS(LOG_TXT) << "HTTP GET request failed for " << mID + << ", Status: " << status.toTerseString() + << " Reason: '" << status.toString() << "'" + << LL_ENDL; resetFormattedData(); releaseHttpSemaphore(); return true; // failed @@ -1610,10 +1625,6 @@ bool LLTextureFetchWorker::doWork(S32 param) else if (http_service_unavail == mGetStatus) { LL_INFOS_ONCE(LOG_TXT) << "Texture server busy (503): " << mUrl << LL_ENDL; - LL_INFOS(LOG_TXT) << "503: HTTP GET failed for: " << mUrl - << " Status: " << mGetStatus.toHex() - << " Reason: '" << mGetReason << "'" - << LL_ENDL; } else if (http_not_sat == mGetStatus) { @@ -2482,7 +2493,6 @@ LLTextureFetch::LLTextureFetch(LLTextureCache* cache, LLImageDecodeThread* image mHttpHeaders(NULL), mHttpMetricsHeaders(NULL), mHttpPolicyClass(LLCore::HttpRequest::DEFAULT_POLICY_ID), - mHttpSemaphore(HTTP_REQUESTS_IN_QUEUE_HIGH_WATER), mTotalCacheReadCount(0U), mTotalCacheWriteCount(0U), mTotalResourceWaitCount(0U), @@ -2494,6 +2504,30 @@ LLTextureFetch::LLTextureFetch(LLTextureCache* cache, LLImageDecodeThread* image mMaxBandwidth = gSavedSettings.getF32("ThrottleBandwidthKBPS"); mTextureInfo.setUpLogging(gSavedSettings.getBOOL("LogTextureDownloadsToViewerLog"), gSavedSettings.getBOOL("LogTextureDownloadsToSimulator"), U32Bytes(gSavedSettings.getU32("TextureLoggingThreshold"))); + mHttpRequest = new LLCore::HttpRequest; + mHttpOptions = new LLCore::HttpOptions; + mHttpOptionsWithHeaders = new LLCore::HttpOptions; + mHttpOptionsWithHeaders->setWantHeaders(true); + mHttpHeaders = new LLCore::HttpHeaders; + mHttpHeaders->append("Accept", "image/x-j2c"); + mHttpMetricsHeaders = new LLCore::HttpHeaders; + mHttpMetricsHeaders->append("Content-Type", "application/llsd+xml"); + LLAppCoreHttp & app_core_http(LLAppViewer::instance()->getAppCoreHttp()); + mHttpPolicyClass = app_core_http.getPolicy(LLAppCoreHttp::AP_TEXTURE); + if (app_core_http.isPipelined(LLAppCoreHttp::AP_TEXTURE)) + { + mHttpHighWater = HTTP_PIPE_REQUESTS_HIGH_WATER; + mHttpLowWater = HTTP_PIPE_REQUESTS_LOW_WATER; + } + else + { + mHttpHighWater = HTTP_NONPIPE_REQUESTS_HIGH_WATER; + mHttpLowWater = HTTP_NONPIPE_REQUESTS_LOW_WATER; + } + mHttpSemaphore = mHttpHighWater; + + // Conditionally construct debugger object after 'this' is + // fully initialized. LLTextureFetchDebugger::sDebuggerEnabled = gSavedSettings.getBOOL("TextureFetchDebuggerEnabled"); if(LLTextureFetchDebugger::isEnabled()) { @@ -2506,16 +2540,6 @@ LLTextureFetch::LLTextureFetch(LLTextureCache* cache, LLImageDecodeThread* image } mOriginFetchSource = mFetchSource; } - - mHttpRequest = new LLCore::HttpRequest; - mHttpOptions = new LLCore::HttpOptions; - mHttpOptionsWithHeaders = new LLCore::HttpOptions; - mHttpOptionsWithHeaders->setWantHeaders(true); - mHttpHeaders = new LLCore::HttpHeaders; - mHttpHeaders->append("Accept", "image/x-j2c"); - mHttpMetricsHeaders = new LLCore::HttpHeaders; - mHttpMetricsHeaders->append("Content-Type", "application/llsd+xml"); - mHttpPolicyClass = LLAppViewer::instance()->getAppCoreHttp().getPolicy(LLAppCoreHttp::AP_TEXTURE); } LLTextureFetch::~LLTextureFetch() @@ -3645,7 +3669,7 @@ void LLTextureFetch::releaseHttpWaiters() { // Use mHttpSemaphore rather than mHTTPTextureQueue.size() // to avoid a lock. - if (mHttpSemaphore < (HTTP_REQUESTS_IN_QUEUE_HIGH_WATER - HTTP_REQUESTS_IN_QUEUE_LOW_WATER)) + if (mHttpSemaphore < (mHttpHighWater - mHttpLowWater)) return; // Quickly make a copy of all the LLUIDs. Get off the @@ -4538,7 +4562,7 @@ S32 LLTextureFetchDebugger::fillCurlQueue() mNbCurlCompleted = mFetchingHistory.size(); return 0; } - if (mNbCurlRequests > HTTP_REQUESTS_IN_QUEUE_LOW_WATER) + if (mNbCurlRequests > HTTP_NONPIPE_REQUESTS_LOW_WATER) { return mNbCurlRequests; } @@ -4571,7 +4595,7 @@ S32 LLTextureFetchDebugger::fillCurlQueue() mFetchingHistory[i].mHttpHandle = handle; mFetchingHistory[i].mCurlState = FetchEntry::CURL_IN_PROGRESS; mNbCurlRequests++; - if (mNbCurlRequests >= HTTP_REQUESTS_IN_QUEUE_HIGH_WATER) // emulate normal pipeline + if (mNbCurlRequests >= HTTP_NONPIPE_REQUESTS_HIGH_WATER) // emulate normal pipeline { break; } diff --git a/indra/newview/lltexturefetch.h b/indra/newview/lltexturefetch.h index c4da2e8685..d13736997f 100755 --- a/indra/newview/lltexturefetch.h +++ b/indra/newview/lltexturefetch.h @@ -356,7 +356,9 @@ private: LLCore::HttpHeaders * mHttpHeaders; // Ttf LLCore::HttpHeaders * mHttpMetricsHeaders; // Ttf LLCore::HttpRequest::policy_t mHttpPolicyClass; // T* - + S32 mHttpHighWater; // T* (ro) + S32 mHttpLowWater; // T* (ro) + // We use a resource semaphore to keep HTTP requests in // WAIT_HTTP_RESOURCE2 if there aren't sufficient slots in the // transport. This keeps them near where they can be cheaply -- cgit v1.2.3 From 17da4cf57aadcf1987b48af298d8b2742089a35c Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Fri, 27 Jun 2014 17:25:39 -0400 Subject: Cleanup and tuning. Use a consistent index on some initialization data so their isn't an opportunity for gaps over overruns (init_data). Start some preliminary tweaking of policy class numbers. It looks like I can easily drop the default connection count to '4' and still hit the throttles. Did some experiments running pipeline deeper which was mostly fine for textures but tended to slow meshes. Reason uncertain but a depth of '5' seems generally healthy for mesh. I had one run of 52.6S with a theoretical minimum of 51.2S. That's as good as I've ever seen. --- indra/newview/llappcorehttp.cpp | 41 +++++++++++++++++++++----------------- indra/newview/llmeshrepository.cpp | 7 ++++++- indra/newview/lltexturefetch.cpp | 2 ++ 3 files changed, 31 insertions(+), 19 deletions(-) (limited to 'indra') diff --git a/indra/newview/llappcorehttp.cpp b/indra/newview/llappcorehttp.cpp index deda0ccb41..d097f18d61 100755 --- a/indra/newview/llappcorehttp.cpp +++ b/indra/newview/llappcorehttp.cpp @@ -42,9 +42,9 @@ const F64 LLAppCoreHttp::MAX_THREAD_WAIT_TIME(10.0); const long LLAppCoreHttp::PIPELINING_DEPTH(5L); +// Default and dynamic values for classes static const struct { - LLAppCoreHttp::EAppPolicy mPolicy; U32 mDefault; U32 mMin; U32 mMax; @@ -52,40 +52,40 @@ static const struct bool mPipelined; std::string mKey; const char * mUsage; -} init_data[] = // Default and dynamic values for classes +} init_data[LLAppCoreHttp::AP_COUNT] = { - { - LLAppCoreHttp::AP_DEFAULT, 8, 8, 8, 0, false, + { // AP_DEFAULT + 8, 8, 8, 0, false, "", "other" }, - { - LLAppCoreHttp::AP_TEXTURE, 8, 1, 12, 0, true, + { // AP_TEXTURE + 4, 1, 12, 0, true, "TextureFetchConcurrency", "texture fetch" }, - { - LLAppCoreHttp::AP_MESH1, 32, 1, 128, 100, false, + { // AP_MESH1 + 32, 1, 128, 100, false, "MeshMaxConcurrentRequests", "mesh fetch" }, - { - LLAppCoreHttp::AP_MESH2, 8, 1, 32, 100, true, + { // AP_MESH2 + 4, 1, 32, 100, true, "Mesh2MaxConcurrentRequests", "mesh2 fetch" }, - { - LLAppCoreHttp::AP_LARGE_MESH, 2, 1, 8, 0, false, + { // AP_LARGE_MESH + 2, 1, 8, 0, false, "", "large mesh fetch" }, - { - LLAppCoreHttp::AP_UPLOADS, 2, 1, 8, 0, false, + { // AP_UPLOADS + 2, 1, 8, 0, false, "", "asset upload" }, - { - LLAppCoreHttp::AP_LONG_POLL, 32, 32, 32, 0, false, + { // AP_LONG_POLL + 32, 32, 32, 0, false, "", "long poll" } @@ -173,9 +173,10 @@ void LLAppCoreHttp::init() mHttpClasses[AP_DEFAULT].mPolicy = LLCore::HttpRequest::DEFAULT_POLICY_ID; // Setup additional policies based on table and some special rules + llassert(LL_ARRAY_SIZE(init_data) == AP_COUNT); for (int i(0); i < LL_ARRAY_SIZE(init_data); ++i) { - const EAppPolicy app_policy(init_data[i].mPolicy); + const EAppPolicy app_policy(static_cast(i)); if (AP_DEFAULT == app_policy) { @@ -301,7 +302,7 @@ void LLAppCoreHttp::refreshSettings(bool initial) for (int i(0); i < LL_ARRAY_SIZE(init_data); ++i) { - const EAppPolicy app_policy(init_data[i].mPolicy); + const EAppPolicy app_policy(static_cast(i)); if (initial) { @@ -326,6 +327,10 @@ void LLAppCoreHttp::refreshSettings(bool initial) if (mPipelined && init_data[i].mPipelined) { // Pipelining election is currently static (init-time). + // Making it dynamic isn't too hard in the SL code but verifying + // that libcurl handles the on-to-off transition while holding + // outstanding requests is something that should be tested. + status = LLCore::HttpRequest::setStaticPolicyOption(LLCore::HttpRequest::PO_PIPELINING_DEPTH, mHttpClasses[app_policy].mPolicy, PIPELINING_DEPTH, diff --git a/indra/newview/llmeshrepository.cpp b/indra/newview/llmeshrepository.cpp index c91ae975ea..74eb6015f2 100755 --- a/indra/newview/llmeshrepository.cpp +++ b/indra/newview/llmeshrepository.cpp @@ -338,14 +338,17 @@ static LLFastTimer::DeclareTimer FTM_MESH_FETCH("Mesh Fetch"); LLMeshRepository gMeshRepo; const S32 MESH_HEADER_SIZE = 4096; // Important: assumption is that headers fit in this space + const S32 REQUEST_HIGH_WATER_MIN = 32; // Limits for GetMesh regions const S32 REQUEST_HIGH_WATER_MAX = 150; // Should remain under 2X throttle const S32 REQUEST_LOW_WATER_MIN = 16; const S32 REQUEST_LOW_WATER_MAX = 75; + const S32 REQUEST2_HIGH_WATER_MIN = 32; // Limits for GetMesh2 regions const S32 REQUEST2_HIGH_WATER_MAX = 100; const S32 REQUEST2_LOW_WATER_MIN = 16; const S32 REQUEST2_LOW_WATER_MAX = 50; + const U32 LARGE_MESH_FETCH_THRESHOLD = 1U << 21; // Size at which requests goes to narrow/slow queue const long SMALL_MESH_XFER_TIMEOUT = 120L; // Seconds to complete xfer, small mesh downloads const long LARGE_MESH_XFER_TIMEOUT = 600L; // Seconds to complete xfer, large downloads @@ -3203,7 +3206,9 @@ void LLMeshRepository::notifyLoadedMeshes() // we'll increase this. See llappcorehttp and llcorehttp for // discussion on connection strategies. LLAppCoreHttp & app_core_http(LLAppViewer::instance()->getAppCoreHttp()); - S32 scale(app_core_http.isPipelined(LLAppCoreHttp::AP_MESH2) ? 10 : 5); + S32 scale(app_core_http.isPipelined(LLAppCoreHttp::AP_MESH2) + ? (2 * LLAppCoreHttp::PIPELINING_DEPTH) + : 5); LLMeshRepoThread::sMaxConcurrentRequests = gSavedSettings.getU32("Mesh2MaxConcurrentRequests"); LLMeshRepoThread::sRequestHighWater = llclamp(scale * S32(LLMeshRepoThread::sMaxConcurrentRequests), diff --git a/indra/newview/lltexturefetch.cpp b/indra/newview/lltexturefetch.cpp index ee11e7b47b..ecaf5f8d06 100755 --- a/indra/newview/lltexturefetch.cpp +++ b/indra/newview/lltexturefetch.cpp @@ -2516,6 +2516,8 @@ LLTextureFetch::LLTextureFetch(LLTextureCache* cache, LLImageDecodeThread* image mHttpPolicyClass = app_core_http.getPolicy(LLAppCoreHttp::AP_TEXTURE); if (app_core_http.isPipelined(LLAppCoreHttp::AP_TEXTURE)) { + // Init-time election that will have to change for + // support of dynamic changes to the pipelining enable flag. mHttpHighWater = HTTP_PIPE_REQUESTS_HIGH_WATER; mHttpLowWater = HTTP_PIPE_REQUESTS_LOW_WATER; } -- cgit v1.2.3 From 9fb96b416f0612882ddcaa658945a5b3f73ff331 Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Thu, 3 Jul 2014 19:44:02 -0400 Subject: Add pipelining and tracing command line options to the test program. --- indra/llcorehttp/examples/http_texture_load.cpp | 64 +++++++++++++++++++++++-- 1 file changed, 59 insertions(+), 5 deletions(-) (limited to 'indra') diff --git a/indra/llcorehttp/examples/http_texture_load.cpp b/indra/llcorehttp/examples/http_texture_load.cpp index 73c49687d7..88692c3f69 100755 --- a/indra/llcorehttp/examples/http_texture_load.cpp +++ b/indra/llcorehttp/examples/http_texture_load.cpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012-2013, Linden Research, Inc. + * Copyright (C) 2012-2014, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -59,11 +59,13 @@ void usage(std::ostream & out); // Default command line settings static int concurrency_limit(40); static int highwater(100); +static int pipeline_depth(0); +static int tracing(0); static char url_format[1024] = "http://example.com/some/path?texture_id=%s.texture"; #if defined(WIN32) -#define strncpy(_a, _b, _c) strncpy_s(_a, _b, _c) +#define strncpy(_a, _b, _c) strncpy_s(_a, _b, _c) #define strtok_r(_a, _b, _c) strtok_s(_a, _b, _c) int getopt(int argc, char * const argv[], const char *optstring); @@ -163,7 +165,7 @@ int main(int argc, char** argv) bool do_verbose(false); int option(-1); - while (-1 != (option = getopt(argc, argv, "u:c:h?RvH:"))) + while (-1 != (option = getopt(argc, argv, "u:c:h?RvH:p:t:"))) { switch (option) { @@ -193,7 +195,7 @@ int main(int argc, char** argv) char * end; value = strtoul(optarg, &end, 10); - if (value < 1 || value > 100 || *end != '\0') + if (value < 1 || value > 200 || *end != '\0') { usage(std::cerr); return 1; @@ -202,6 +204,36 @@ int main(int argc, char** argv) } break; + case 'p': + { + unsigned long value; + char * end; + + value = strtoul(optarg, &end, 10); + if (value < 0 || value > 100 || *end != '\0') + { + usage(std::cerr); + return 1; + } + pipeline_depth = value; + } + break; + + case '5': + { + unsigned long value; + char * end; + + value = strtoul(optarg, &end, 10); + if (value < 0 || value > 3 || *end != '\0') + { + usage(std::cerr); + return 1; + } + tracing = value; + } + break; + case 'R': do_random = true; break; @@ -240,6 +272,24 @@ int main(int argc, char** argv) LLCore::HttpRequest::DEFAULT_POLICY_ID, concurrency_limit, NULL); + LLCore::HttpRequest::setStaticPolicyOption(LLCore::HttpRequest::PO_PER_HOST_CONNECTION_LIMIT, + LLCore::HttpRequest::DEFAULT_POLICY_ID, + concurrency_limit, + NULL); + if (pipeline_depth) + { + LLCore::HttpRequest::setStaticPolicyOption(LLCore::HttpRequest::PO_PIPELINING_DEPTH, + LLCore::HttpRequest::DEFAULT_POLICY_ID, + pipeline_depth, + NULL); + } + if (tracing) + { + LLCore::HttpRequest::setStaticPolicyOption(LLCore::HttpRequest::PO_TRACE, + LLCore::HttpRequest::DEFAULT_POLICY_ID, + tracing, + NULL); + } LLCore::HttpRequest::startThread(); // Get service point @@ -334,7 +384,11 @@ void usage(std::ostream & out) " -c Maximum connection concurrency. Range: [1..100]\n" " Default: " << concurrency_limit << "\n" " -H HTTP request highwater (requests fed to llcorehttp).\n" - " Range: [1..100] Default: " << highwater << "\n" + " Range: [1..200] Default: " << highwater << "\n" + " -p If is positive, enables and sets pipelineing\n" + " depth on HTTP requests. Default: " << pipeline_depth << "\n" + " -t If is positive ([1..3]), enables and sets HTTP\n" + " tracing on HTTP requests. Default: " << tracing << "\n" " -v Verbose mode. Issue some chatter while running\n" " -h print this help\n" "\n" -- cgit v1.2.3 From 5dffe16aef7c6687a2ed47c5324f4365ff9d84df Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Mon, 11 Aug 2014 14:38:47 -0400 Subject: Add 'HttpRangeRequestsDisable' debug setting to inhibit use of 'Range:' header. Intended for users with bad networking gear or twitchy ISPs, if set to True, forces plain GET requests to asset servers for textures and meshes. This change kicked off a slight refactor in the mesh repository code which made it resilient against unexpected 200's and responses not covering the requested start range. There's still too much data copying in the Mesh code (always has been). Would love to fix that and get rid of the monolithic temp buffer. Cleaned up white space damage caused by unnamed linden who likes to drag his magical editor through code. --- indra/newview/app_settings/settings.xml | 11 + indra/newview/llmeshrepository.cpp | 364 ++++++++++++++++++-------------- indra/newview/lltexturefetch.cpp | 43 ++-- 3 files changed, 243 insertions(+), 175 deletions(-) (limited to 'indra') diff --git a/indra/newview/app_settings/settings.xml b/indra/newview/app_settings/settings.xml index 410473c54f..50f56cf6ff 100755 --- a/indra/newview/app_settings/settings.xml +++ b/indra/newview/app_settings/settings.xml @@ -4467,6 +4467,17 @@ Value 1 + HttpRangeRequestsDisable + + Comment + If true, viewer will not issued range GET requests for meshes and textures. + Persist + 1 + Type + Boolean + Value + 0 + IMShowTimestamps Comment diff --git a/indra/newview/llmeshrepository.cpp b/indra/newview/llmeshrepository.cpp index fc69ecfae9..6477389d4c 100755 --- a/indra/newview/llmeshrepository.cpp +++ b/indra/newview/llmeshrepository.cpp @@ -1,4 +1,3 @@ - /** * @file llmeshrepository.cpp * @brief Mesh repository implementation. @@ -521,11 +520,13 @@ class LLMeshHandlerBase : public LLCore::HttpHandler { public: LOG_CLASS(LLMeshHandlerBase); - LLMeshHandlerBase() + LLMeshHandlerBase(U32 offset, U32 requested_bytes) : LLCore::HttpHandler(), mMeshParams(), mProcessed(false), - mHttpHandle(LLCORE_HTTP_HANDLE_INVALID) + mHttpHandle(LLCORE_HTTP_HANDLE_INVALID), + mOffset(offset), + mRequestedBytes(requested_bytes) {} virtual ~LLMeshHandlerBase() @@ -537,13 +538,15 @@ protected: public: virtual void onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * response); - virtual void processData(LLCore::BufferArray * body, U8 * data, S32 data_size) = 0; + virtual void processData(LLCore::BufferArray * body, S32 body_offset, U8 * data, S32 data_size) = 0; virtual void processFailure(LLCore::HttpStatus status) = 0; public: LLVolumeParams mMeshParams; bool mProcessed; - LLCore::HttpHandle mHttpHandle; + LLCore::HttpHandle mHttpHandle; + U32 mOffset; + U32 mRequestedBytes; }; @@ -554,8 +557,8 @@ class LLMeshHeaderHandler : public LLMeshHandlerBase { public: LOG_CLASS(LLMeshHeaderHandler); - LLMeshHeaderHandler(const LLVolumeParams & mesh_params) - : LLMeshHandlerBase() + LLMeshHeaderHandler(const LLVolumeParams & mesh_params, U32 offset, U32 requested_bytes) + : LLMeshHandlerBase(offset, requested_bytes) { mMeshParams = mesh_params; LLMeshRepoThread::incActiveHeaderRequests(); @@ -567,7 +570,7 @@ protected: void operator=(const LLMeshHeaderHandler &); // Not defined public: - virtual void processData(LLCore::BufferArray * body, U8 * data, S32 data_size); + virtual void processData(LLCore::BufferArray * body, S32 body_offset, U8 * data, S32 data_size); virtual void processFailure(LLCore::HttpStatus status); }; @@ -576,17 +579,16 @@ public: // // Thread: repo class LLMeshLODHandler : public LLMeshHandlerBase - { +{ public: + LOG_CLASS(LLMeshLODHandler); LLMeshLODHandler(const LLVolumeParams & mesh_params, S32 lod, U32 offset, U32 requested_bytes) - : LLMeshHandlerBase(), - mLOD(lod), - mRequestedBytes(requested_bytes), - mOffset(offset) + : LLMeshHandlerBase(offset, requested_bytes), + mLOD(lod) { - mMeshParams = mesh_params; - LLMeshRepoThread::incActiveLODRequests(); - } + mMeshParams = mesh_params; + LLMeshRepoThread::incActiveLODRequests(); + } virtual ~LLMeshLODHandler(); protected: @@ -594,13 +596,11 @@ protected: void operator=(const LLMeshLODHandler &); // Not defined public: - virtual void processData(LLCore::BufferArray * body, U8 * data, S32 data_size); + virtual void processData(LLCore::BufferArray * body, S32 body_offset, U8 * data, S32 data_size); virtual void processFailure(LLCore::HttpStatus status); public: S32 mLOD; - U32 mRequestedBytes; - U32 mOffset; }; @@ -608,14 +608,12 @@ public: // // Thread: repo class LLMeshSkinInfoHandler : public LLMeshHandlerBase - { +{ public: LOG_CLASS(LLMeshSkinInfoHandler); - LLMeshSkinInfoHandler(const LLUUID& id, U32 offset, U32 size) - : LLMeshHandlerBase(), - mMeshID(id), - mRequestedBytes(size), - mOffset(offset) + LLMeshSkinInfoHandler(const LLUUID& id, U32 offset, U32 requested_bytes) + : LLMeshHandlerBase(offset, requested_bytes), + mMeshID(id) {} virtual ~LLMeshSkinInfoHandler(); @@ -624,13 +622,11 @@ protected: void operator=(const LLMeshSkinInfoHandler &); // Not defined public: - virtual void processData(LLCore::BufferArray * body, U8 * data, S32 data_size); + virtual void processData(LLCore::BufferArray * body, S32 body_offset, U8 * data, S32 data_size); virtual void processFailure(LLCore::HttpStatus status); public: LLUUID mMeshID; - U32 mRequestedBytes; - U32 mOffset; }; @@ -638,14 +634,12 @@ public: // // Thread: repo class LLMeshDecompositionHandler : public LLMeshHandlerBase - { +{ public: LOG_CLASS(LLMeshDecompositionHandler); - LLMeshDecompositionHandler(const LLUUID& id, U32 offset, U32 size) - : LLMeshHandlerBase(), - mMeshID(id), - mRequestedBytes(size), - mOffset(offset) + LLMeshDecompositionHandler(const LLUUID& id, U32 offset, U32 requested_bytes) + : LLMeshHandlerBase(offset, requested_bytes), + mMeshID(id) {} virtual ~LLMeshDecompositionHandler(); @@ -654,13 +648,11 @@ protected: void operator=(const LLMeshDecompositionHandler &); // Not defined public: - virtual void processData(LLCore::BufferArray * body, U8 * data, S32 data_size); + virtual void processData(LLCore::BufferArray * body, S32 body_offset, U8 * data, S32 data_size); virtual void processFailure(LLCore::HttpStatus status); public: LLUUID mMeshID; - U32 mRequestedBytes; - U32 mOffset; }; @@ -668,14 +660,12 @@ public: // // Thread: repo class LLMeshPhysicsShapeHandler : public LLMeshHandlerBase - { +{ public: LOG_CLASS(LLMeshPhysicsShapeHandler); - LLMeshPhysicsShapeHandler(const LLUUID& id, U32 offset, U32 size) - : LLMeshHandlerBase(), - mMeshID(id), - mRequestedBytes(size), - mOffset(offset) + LLMeshPhysicsShapeHandler(const LLUUID& id, U32 offset, U32 requested_bytes) + : LLMeshHandlerBase(offset, requested_bytes), + mMeshID(id) {} virtual ~LLMeshPhysicsShapeHandler(); @@ -684,13 +674,11 @@ protected: void operator=(const LLMeshPhysicsShapeHandler &); // Not defined public: - virtual void processData(LLCore::BufferArray * body, U8 * data, S32 data_size); + virtual void processData(LLCore::BufferArray * body, S32 body_offset, U8 * data, S32 data_size); virtual void processFailure(LLCore::HttpStatus status); public: LLUUID mMeshID; - U32 mRequestedBytes; - U32 mOffset; }; @@ -716,8 +704,8 @@ void log_upload_error(LLCore::HttpStatus status, const LLSD& content, LL_WARNS(LOG_MESH) << "error: " << err << LL_ENDL; LL_WARNS(LOG_MESH) << " mesh upload failed, stage '" << stage << "', error '" << err["error"].asString() - << "', message '" << err["message"].asString() - << "', id '" << err["identifier"].asString() + << "', message '" << err["message"].asString() + << "', id '" << err["identifier"].asString() << "'" << LL_ENDL; if (err.has("errors")) { @@ -779,7 +767,7 @@ LLMeshRepoThread::LLMeshRepoThread() LLMeshRepoThread::~LLMeshRepoThread() - { +{ LL_INFOS(LOG_MESH) << "Small GETs issued: " << LLMeshRepository::sHTTPRequestCount << ", Large GETs issued: " << LLMeshRepository::sHTTPLargeRequestCount << ", Max Lock Holdoffs: " << LLMeshRepository::sMaxLockHoldoffs @@ -790,23 +778,23 @@ LLMeshRepoThread::~LLMeshRepoThread() ++iter) { delete *iter; - } + } mHttpRequestSet.clear(); if (mHttpHeaders) - { + { mHttpHeaders->release(); mHttpHeaders = NULL; - } + } if (mHttpOptions) - { + { mHttpOptions->release(); mHttpOptions = NULL; - } + } if (mHttpLargeOptions) -{ + { mHttpLargeOptions->release(); mHttpLargeOptions = NULL; -} + } delete mHttpRequest; mHttpRequest = NULL; delete mMutex; @@ -1137,6 +1125,9 @@ LLCore::HttpHandle LLMeshRepoThread::getByteRange(const std::string & url, int c size_t offset, size_t len, LLCore::HttpHandler * handler) { + // Also used in lltexturefetch.cpp + static LLCachedControl disable_range_req(gSavedSettings, "HttpRangeRequestsDisable", false); + LLCore::HttpHandle handle(LLCORE_HTTP_HANDLE_INVALID); if (len < LARGE_MESH_FETCH_THRESHOLD) @@ -1146,8 +1137,8 @@ LLCore::HttpHandle LLMeshRepoThread::getByteRange(const std::string & url, int c : mHttpLegacyPolicyClass), mHttpPriority, url, - offset, - len, + (disable_range_req ? size_t(0) : offset), + (disable_range_req ? size_t(0) : len), mHttpOptions, mHttpHeaders, handler); @@ -1161,8 +1152,8 @@ LLCore::HttpHandle LLMeshRepoThread::getByteRange(const std::string & url, int c handle = mHttpRequest->requestGetByteRange(mHttpLargePolicyClass, mHttpPriority, url, - offset, - len, + (disable_range_req ? size_t(0) : offset), + (disable_range_req ? size_t(0) : len), mHttpLargeOptions, mHttpHeaders, handler); @@ -1532,7 +1523,7 @@ bool LLMeshRepoThread::fetchMeshHeader(const LLVolumeParams& mesh_params) //within the first 4KB //NOTE -- this will break of headers ever exceed 4KB - LLMeshHeaderHandler * handler = new LLMeshHeaderHandler(mesh_params); + LLMeshHeaderHandler * handler = new LLMeshHeaderHandler(mesh_params, 0, MESH_HEADER_SIZE); LLCore::HttpHandle handle = getByteRange(http_url, cap_version, 0, MESH_HEADER_SIZE, handler); if (LLCORE_HTTP_HANDLE_INVALID == handle) { @@ -2276,7 +2267,7 @@ void LLMeshUploadThread::doWholeModelUpload() mHttpRequest->update(0); while (! LLApp::isQuitting() && ! finished() && ! isDiscarded()) - { + { ms_sleep(sleep_time); sleep_time = llmin(250U, sleep_time + sleep_time); mHttpRequest->update(0); @@ -2292,7 +2283,7 @@ void LLMeshUploadThread::doWholeModelUpload() } } } - } +} void LLMeshUploadThread::requestWholeModelFee() { @@ -2323,7 +2314,7 @@ void LLMeshUploadThread::requestWholeModelFee() LL_WARNS(LOG_MESH) << "Couldn't issue request for model fee. Reason: " << mHttpStatus.toString() << " (" << mHttpStatus.toTerseString() << ")" << LL_ENDL; - } + } else { U32 sleep_time(10); @@ -2340,7 +2331,7 @@ void LLMeshUploadThread::requestWholeModelFee() LL_DEBUGS(LOG_MESH) << "Mesh fee query operation discarded." << LL_ENDL; } } - } +} // Does completion duty for both fee queries and actual uploads. @@ -2393,12 +2384,12 @@ void LLMeshUploadThread::onCompleted(LLCore::HttpHandle handle, LLCore::HttpResp { LLCore::BufferArrayStream bas(ba); LLSDSerialize::fromXML(body, bas); -} + } } dump_llsd_to_file(body, make_dump_name("whole_model_upload_response_", dump_num)); if (body["state"].asString() == "complete") -{ + { // requested "mesh" asset type isn't actually the type // of the resultant object, fix it up here. mModelData["asset_type"] = "object"; @@ -2451,18 +2442,18 @@ void LLMeshUploadThread::onCompleted(LLCore::HttpHandle handle, LLCore::HttpResp body = llsd_from_file("fake_upload_error.xml"); } else - { + { LLCore::BufferArray * ba(response->getBody()); if (ba && ba->size()) - { + { LLCore::BufferArrayStream bas(ba); LLSDSerialize::fromXML(body, bas); - } - } + } + } dump_llsd_to_file(body, make_dump_name("whole_model_fee_response_", dump_num)); if (body["state"].asString() == "upload") - { + { mWholeModelUploadURL = body["uploader"].asString(); if (observer) @@ -2548,18 +2539,18 @@ void LLMeshRepoThread::notifyLoadedMeshes() skin_info_q.swap(mSkinInfoQ); } if (! mDecompositionQ.empty()) - { + { decomp_q.swap(mDecompositionQ); - } + } mMutex->unlock(); // Process the elements free of the lock while (! skin_info_q.empty()) - { + { gMeshRepo.notifySkinInfoReceived(skin_info_q.front()); skin_info_q.pop_front(); - } + } while (! decomp_q.empty()) { @@ -2681,35 +2672,78 @@ void LLMeshHandlerBase::onCompleted(LLCore::HttpHandle handle, LLCore::HttpRespo // rather than partial) and 416 (request completely unsatisfyable). // Always been exposed to these but are less likely here where // speculative loads aren't done. - static const LLCore::HttpStatus par_status(HTTP_PARTIAL_CONTENT); + LLCore::BufferArray * body(response->getBody()); + S32 body_offset(0); + U8 * data(NULL); + S32 data_size(body ? body->size() : 0); - if (par_status != status) + if (data_size > 0) { - LL_WARNS_ONCE(LOG_MESH) << "Non-206 successful status received for fetch: " - << status.toTerseString() << LL_ENDL; - } + static const LLCore::HttpStatus par_status(HTTP_PARTIAL_CONTENT); + + unsigned int offset(0), length(0), full_length(0); + + if (par_status == status) + { + // 216 case + response->getRange(&offset, &length, &full_length); + if (! offset && ! length) + { + // This is the case where we receive a 206 status but + // there wasn't a useful Content-Range header in the response. + // This could be because it was badly formatted but is more + // likely due to capabilities services which scrub headers + // from responses. Assume we got what we asked for...` + // length = data_size; + offset = mOffset; + } + } + else + { + // 200 case, typically + offset = 0; + } - LLCore::BufferArray * body(response->getBody()); - S32 data_size(body ? body->size() : 0); - U8 * data(NULL); + // *DEBUG: To test validation below + // offset += 1; - if (data_size > 0) - { + // Validate that what we think we received is consistent with + // what we've asked for. I.e. first byte we wanted lies somewhere + // in the response. + if (offset > mOffset + || (offset + data_size) <= mOffset + || (mOffset - offset) >= data_size) + { + // No overlap with requested range. Fail request with + // suitable error. Shouldn't happen unless server/cache/ISP + // is doing something awful. + LL_WARNS(LOG_MESH) << "Mesh response (bytes [" + << offset << ".." << (offset + length - 1) + << "]) didn't overlap with request's origin (bytes [" + << mOffset << ".." << (mOffset + mRequestedBytes - 1) + << "])." << LL_ENDL; + processFailure(LLCore::HttpStatus(LLCore::HttpStatus::LLCORE, LLCore::HE_INV_CONTENT_RANGE_HDR)); + ++LLMeshRepository::sHTTPErrorCount; + goto common_exit; + } + // *TODO: Try to get rid of data copying and add interfaces // that support BufferArray directly. Introduce a two-phase // handler, optional first that takes a body, fallback second // that requires a temporary allocation and data copy. - data = new U8[data_size]; - body->read(0, (char *) data, data_size); + body_offset = mOffset - offset; + data = new U8[data_size - body_offset]; + body->read(body_offset, (char *) data, data_size - body_offset); LLMeshRepository::sBytesReceived += data_size; } - processData(body, data, data_size); + processData(body, body_offset, data, data_size - body_offset); delete [] data; } // Release handler +common_exit: gMeshRepo.mThread->mHttpRequestSet.erase(this); delete this; // Must be last statement } @@ -2744,9 +2778,10 @@ void LLMeshHeaderHandler::processFailure(LLCore::HttpStatus status) { gMeshRepo.mThread->mUnavailableQ.push(LLMeshRepoThread::LODRequest(mMeshParams, i)); } - } +} -void LLMeshHeaderHandler::processData(LLCore::BufferArray * body, U8 * data, S32 data_size) +void LLMeshHeaderHandler::processData(LLCore::BufferArray * /* body */, S32 /* body_offset */, + U8 * data, S32 data_size) { LLUUID mesh_id = mMeshParams.getSculptID(); bool success = (! MESH_HEADER_PROCESS_FAILED) && gMeshRepo.mThread->headerReceived(mMeshParams, data, data_size); @@ -2761,12 +2796,12 @@ void LLMeshHeaderHandler::processData(LLCore::BufferArray * body, U8 * data, S32 // Can't get the header so none of the LODs will be available LLMutexLock lock(gMeshRepo.mThread->mMutex); for (int i(0); i < 4; ++i) - { + { gMeshRepo.mThread->mUnavailableQ.push(LLMeshRepoThread::LODRequest(mMeshParams, i)); - } } + } else if (data && data_size > 0) - { + { // header was successfully retrieved from sim, cache in vfs LLSD header = gMeshRepo.mThread->mMeshHeader[mesh_id]; @@ -2779,11 +2814,11 @@ void LLMeshHeaderHandler::processData(LLCore::BufferArray * body, U8 * data, S32 S32 lod_bytes = 0; for (U32 i = 0; i < LLModel::LOD_PHYSICS; ++i) - { + { // figure out how many bytes we'll need to reserve in the file const std::string & lod_name = header_lod[i]; lod_bytes = llmax(lod_bytes, header[lod_name]["offset"].asInteger()+header[lod_name]["size"].asInteger()); - } + } // just in case skin info or decomposition is at the end of the file (which it shouldn't be) lod_bytes = llmax(lod_bytes, header["skin"]["offset"].asInteger() + header["skin"]["size"].asInteger()); @@ -2799,7 +2834,7 @@ void LLMeshHeaderHandler::processData(LLCore::BufferArray * body, U8 * data, S32 LLVFile file(gVFS, mesh_id, LLAssetType::AT_MESH, LLVFile::WRITE); if (file.getMaxSize() >= bytes || file.setMaxSize(bytes)) - { + { LLMeshRepository::sCacheBytesWritten += data_size; ++LLMeshRepository::sCacheWrites; @@ -2810,19 +2845,19 @@ void LLMeshHeaderHandler::processData(LLCore::BufferArray * body, U8 * data, S32 memset(block, 0, sizeof(block)); while (bytes-file.tell() > sizeof(block)) - { + { file.write(block, sizeof(block)); - } + } S32 remaining = bytes-file.tell(); if (remaining > 0) - { + { file.write(block, remaining); } } } } - } +} LLMeshLODHandler::~LLMeshLODHandler() { @@ -2848,8 +2883,9 @@ void LLMeshLODHandler::processFailure(LLCore::HttpStatus status) gMeshRepo.mThread->mUnavailableQ.push(LLMeshRepoThread::LODRequest(mMeshParams, mLOD)); } -void LLMeshLODHandler::processData(LLCore::BufferArray * body, U8 * data, S32 data_size) - { +void LLMeshLODHandler::processData(LLCore::BufferArray * /* body */, S32 /* body_offset */, + U8 * data, S32 data_size) +{ if ((! MESH_LOD_PROCESS_FAILED) && gMeshRepo.mThread->lodReceived(mMeshParams, mLOD, data, data_size)) { //good fetch from sim, write to VFS for caching @@ -2865,7 +2901,7 @@ void LLMeshLODHandler::processData(LLCore::BufferArray * body, U8 * data, S32 da LLMeshRepository::sCacheBytesWritten += size; ++LLMeshRepository::sCacheWrites; } - } + } else { LL_WARNS(LOG_MESH) << "Error during mesh LOD processing. ID: " << mMeshParams.getSculptID() @@ -2877,12 +2913,12 @@ void LLMeshLODHandler::processData(LLCore::BufferArray * body, U8 * data, S32 da } LLMeshSkinInfoHandler::~LLMeshSkinInfoHandler() - { - llassert(mProcessed); - } +{ + llassert(mProcessed); +} void LLMeshSkinInfoHandler::processFailure(LLCore::HttpStatus status) - { +{ LL_WARNS(LOG_MESH) << "Error during mesh skin info handling. ID: " << mMeshID << ", Reason: " << status.toString() << " (" << status.toTerseString() << "). Not retrying." @@ -2890,10 +2926,11 @@ void LLMeshSkinInfoHandler::processFailure(LLCore::HttpStatus status) // *TODO: Mark mesh unavailable on error. For now, simply leave // request unfulfilled rather than retry forever. - } +} -void LLMeshSkinInfoHandler::processData(LLCore::BufferArray * body, U8 * data, S32 data_size) - { +void LLMeshSkinInfoHandler::processData(LLCore::BufferArray * /* body */, S32 /* body_offset */, + U8 * data, S32 data_size) +{ if ((! MESH_SKIN_INFO_PROCESS_FAILED) && gMeshRepo.mThread->skinInfoReceived(mMeshID, data, data_size)) { //good fetch from sim, write to VFS for caching @@ -2921,20 +2958,21 @@ void LLMeshSkinInfoHandler::processData(LLCore::BufferArray * body, U8 * data, S LLMeshDecompositionHandler::~LLMeshDecompositionHandler() { - llassert(mProcessed); + llassert(mProcessed); } void LLMeshDecompositionHandler::processFailure(LLCore::HttpStatus status) - { +{ LL_WARNS(LOG_MESH) << "Error during mesh decomposition handling. ID: " << mMeshID << ", Reason: " << status.toString() << " (" << status.toTerseString() << "). Not retrying." << LL_ENDL; // *TODO: Mark mesh unavailable on error. For now, simply leave // request unfulfilled rather than retry forever. - } +} -void LLMeshDecompositionHandler::processData(LLCore::BufferArray * body, U8 * data, S32 data_size) +void LLMeshDecompositionHandler::processData(LLCore::BufferArray * /* body */, S32 /* body_offset */, + U8 * data, S32 data_size) { if ((! MESH_DECOMP_PROCESS_FAILED) && gMeshRepo.mThread->decompositionReceived(mMeshID, data, data_size)) { @@ -2951,34 +2989,35 @@ void LLMeshDecompositionHandler::processData(LLCore::BufferArray * body, U8 * da file.seek(offset); file.write(data, size); } - } - else - { + } + else + { LL_WARNS(LOG_MESH) << "Error during mesh decomposition processing. ID: " << mMeshID << ", Unknown reason. Not retrying." << LL_ENDL; // *TODO: Mark mesh unavailable on error - } } +} LLMeshPhysicsShapeHandler::~LLMeshPhysicsShapeHandler() - { - llassert(mProcessed); - } +{ + llassert(mProcessed); +} void LLMeshPhysicsShapeHandler::processFailure(LLCore::HttpStatus status) - { +{ LL_WARNS(LOG_MESH) << "Error during mesh physics shape handling. ID: " << mMeshID << ", Reason: " << status.toString() << " (" << status.toTerseString() << "). Not retrying." << LL_ENDL; // *TODO: Mark mesh unavailable on error - } +} -void LLMeshPhysicsShapeHandler::processData(LLCore::BufferArray * body, U8 * data, S32 data_size) - { +void LLMeshPhysicsShapeHandler::processData(LLCore::BufferArray * /* body */, S32 /* body_offset */, + U8 * data, S32 data_size) +{ if ((! MESH_PHYS_SHAPE_PROCESS_FAILED) && gMeshRepo.mThread->physicsShapeReceived(mMeshID, data, data_size)) - { + { // good fetch from sim, write to VFS for caching LLVFile file(gVFS, mMeshID, LLAssetType::AT_MESH, LLVFile::WRITE); @@ -2986,13 +3025,13 @@ void LLMeshPhysicsShapeHandler::processData(LLCore::BufferArray * body, U8 * dat S32 size = mRequestedBytes; if (file.getSize() >= offset+size) - { + { LLMeshRepository::sCacheBytesWritten += size; ++LLMeshRepository::sCacheWrites; file.seek(offset); file.write(data, size); - } } + } else { LL_WARNS(LOG_MESH) << "Error during mesh physics shape processing. ID: " << mMeshID @@ -3192,7 +3231,7 @@ void LLMeshRepository::notifyLoadedMeshes() if (1 == mGetMeshVersion) { // Legacy GetMesh operation with high connection concurrency - LLMeshRepoThread::sMaxConcurrentRequests = gSavedSettings.getU32("MeshMaxConcurrentRequests"); + LLMeshRepoThread::sMaxConcurrentRequests = gSavedSettings.getU32("MeshMaxConcurrentRequests"); LLMeshRepoThread::sRequestHighWater = llclamp(2 * S32(LLMeshRepoThread::sMaxConcurrentRequests), REQUEST_HIGH_WATER_MIN, REQUEST_HIGH_WATER_MAX); @@ -3311,18 +3350,18 @@ void LLMeshRepository::notifyLoadedMeshes() // If we can't get the locks, skip and pick this up later. ++hold_offs; sMaxLockHoldoffs = llmax(sMaxLockHoldoffs, hold_offs); - return; - } + return; + } hold_offs = 0; if (gAgent.getRegion()) { // Update capability urls - static std::string region_name("never name a region this"); + static std::string region_name("never name a region this"); - if (gAgent.getRegion()->getName() != region_name && gAgent.getRegion()->capabilitiesReceived()) - { - region_name = gAgent.getRegion()->getName(); + if (gAgent.getRegion()->getName() != region_name && gAgent.getRegion()->capabilitiesReceived()) + { + region_name = gAgent.getRegion()->getName(); const bool use_v1(gSavedSettings.getBOOL("MeshUseGetMesh1")); const std::string mesh1(gAgent.getRegion()->getCapability("GetMesh")); const std::string mesh2(gAgent.getRegion()->getCapability("GetMesh2")); @@ -3333,8 +3372,8 @@ void LLMeshRepository::notifyLoadedMeshes() << ", GetMesh: " << mesh1 << ", using version: " << mGetMeshVersion << LL_ENDL; + } } - } //popup queued error messages from background threads while (!mUploadErrorQ.empty()) @@ -3349,46 +3388,46 @@ void LLMeshRepository::notifyLoadedMeshes() S32 push_count = LLMeshRepoThread::sRequestHighWater - active_count; if (mPendingRequests.size() > push_count) - { + { // More requests than the high-water limit allows so // sort and forward the most important. - //calculate "score" for pending requests + //calculate "score" for pending requests - //create score map - std::map score_map; + //create score map + std::map score_map; - for (U32 i = 0; i < 4; ++i) - { - for (mesh_load_map::iterator iter = mLoadingMeshes[i].begin(); iter != mLoadingMeshes[i].end(); ++iter) + for (U32 i = 0; i < 4; ++i) { - F32 max_score = 0.f; - for (std::set::iterator obj_iter = iter->second.begin(); obj_iter != iter->second.end(); ++obj_iter) + for (mesh_load_map::iterator iter = mLoadingMeshes[i].begin(); iter != mLoadingMeshes[i].end(); ++iter) { - LLViewerObject* object = gObjectList.findObject(*obj_iter); - - if (object) + F32 max_score = 0.f; + for (std::set::iterator obj_iter = iter->second.begin(); obj_iter != iter->second.end(); ++obj_iter) { - LLDrawable* drawable = object->mDrawable; - if (drawable) + LLViewerObject* object = gObjectList.findObject(*obj_iter); + + if (object) { - F32 cur_score = drawable->getRadius()/llmax(drawable->mDistanceWRTCamera, 1.f); - max_score = llmax(max_score, cur_score); + LLDrawable* drawable = object->mDrawable; + if (drawable) + { + F32 cur_score = drawable->getRadius()/llmax(drawable->mDistanceWRTCamera, 1.f); + max_score = llmax(max_score, cur_score); + } } } - } - score_map[iter->first.getSculptID()] = max_score; + score_map[iter->first.getSculptID()] = max_score; + } } - } - //set "score" for pending requests - for (std::vector::iterator iter = mPendingRequests.begin(); iter != mPendingRequests.end(); ++iter) - { - iter->mScore = score_map[iter->mMeshParams.getSculptID()]; - } + //set "score" for pending requests + for (std::vector::iterator iter = mPendingRequests.begin(); iter != mPendingRequests.end(); ++iter) + { + iter->mScore = score_map[iter->mMeshParams.getSculptID()]; + } - //sort by "score" + //sort by "score" std::partial_sort(mPendingRequests.begin(), mPendingRequests.begin() + push_count, mPendingRequests.end(), LLMeshRepoThread::CompareScoreGreater()); } @@ -3599,7 +3638,6 @@ void LLMeshRepository::fetchPhysicsShape(const LLUUID& mesh_id) } } } - } LLModel::Decomposition* LLMeshRepository::getDecomposition(const LLUUID& mesh_id) diff --git a/indra/newview/lltexturefetch.cpp b/indra/newview/lltexturefetch.cpp index ab7df02100..4008a6948d 100755 --- a/indra/newview/lltexturefetch.cpp +++ b/indra/newview/lltexturefetch.cpp @@ -1327,7 +1327,7 @@ bool LLTextureFetchWorker::doWork(S32 param) } } - static LLCachedControl use_http(gSavedSettings,"ImagePipelineUseHTTP", true); + static LLCachedControl use_http(gSavedSettings, "ImagePipelineUseHTTP", true); // if (mHost != LLHost::invalid) get_url = false; if ( use_http && mCanUseHTTP && mUrl.empty())//get http url. @@ -1472,6 +1472,9 @@ bool LLTextureFetchWorker::doWork(S32 param) if (mState == SEND_HTTP_REQ) { + // Also used in llmeshrepository + static LLCachedControl disable_range_req(gSavedSettings, "HttpRangeRequestsDisable", false); + if (! mCanUseHTTP) { releaseHttpSemaphore(); @@ -1553,16 +1556,32 @@ bool LLTextureFetchWorker::doWork(S32 param) // Will call callbackHttpGet when curl request completes // Only server bake images use the returned headers currently, for getting retry-after field. LLCore::HttpOptions *options = (mFTType == FTT_SERVER_BAKE) ? mFetcher->mHttpOptionsWithHeaders: mFetcher->mHttpOptions; - mHttpHandle = mFetcher->mHttpRequest->requestGetByteRange(mHttpPolicyClass, - mWorkPriority, - mUrl, - mRequestedOffset, - (mRequestedOffset + mRequestedSize) > HTTP_REQUESTS_RANGE_END_MAX - ? 0 - : mRequestedSize, - options, - mFetcher->mHttpHeaders, - this); + if (disable_range_req) + { + // 'Range:' requests may be disabled in which case all HTTP + // texture fetches result in full fetches. This can be used + // by people with questionable ISPs or networking gear that + // doesn't handle these well. + mHttpHandle = mFetcher->mHttpRequest->requestGet(mHttpPolicyClass, + mWorkPriority, + mUrl, + options, + mFetcher->mHttpHeaders, + this); + } + else + { + mHttpHandle = mFetcher->mHttpRequest->requestGetByteRange(mHttpPolicyClass, + mWorkPriority, + mUrl, + mRequestedOffset, + (mRequestedOffset + mRequestedSize) > HTTP_REQUESTS_RANGE_END_MAX + ? 0 + : mRequestedSize, + options, + mFetcher->mHttpHeaders, + this); + } if (LLCORE_HTTP_HANDLE_INVALID == mHttpHandle) { LLCore::HttpStatus status(mFetcher->mHttpRequest->getStatus()); @@ -1782,7 +1801,7 @@ bool LLTextureFetchWorker::doWork(S32 param) if (mState == DECODE_IMAGE) { - static LLCachedControl textures_decode_disabled(gSavedSettings,"TextureDecodeDisabled", false); + static LLCachedControl textures_decode_disabled(gSavedSettings, "TextureDecodeDisabled", false); setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); // Set priority first since Responder may change it if (textures_decode_disabled) -- cgit v1.2.3 From e201c9d274539f987ca1d41fd9fb21431838fe4d Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Mon, 11 Aug 2014 17:04:39 -0400 Subject: Doc cleanup. Unicode cruft, copy editing. --- indra/cmake/00-COMPILE-LINK-RUN.txt | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'indra') diff --git a/indra/cmake/00-COMPILE-LINK-RUN.txt b/indra/cmake/00-COMPILE-LINK-RUN.txt index d08cc2dc0c..49b899c50d 100644 --- a/indra/cmake/00-COMPILE-LINK-RUN.txt +++ b/indra/cmake/00-COMPILE-LINK-RUN.txt @@ -115,12 +115,12 @@ Compilation ---------------------------------------------------------------------------- Notes: - 1. We’re also building dylibs in a somewhat unusual way. They’re + 1. We're also building dylibs in a somewhat unusual way. They're currently being generated with a link path of - ‘@executable_path/../Resources/’. If we were to follow - the recommendations in dyld’s man page, we’d instead reference - ‘@loader_path/’, use -rpath on the executable link - (pointing to the ‘Resources’ subdir of the main executable), and + '@executable_path/../Resources/'. If we were to follow + the recommendations in dyld's man page, we’d instead reference + '@loader_path/', use -rpath on the executable link + (pointing to the 'Resources' subdir of the main executable), and be able to avoid some symlinking in the .app tree. 2. Use the -headerpad_max_install_names link option on all .dylibs. @@ -184,7 +184,7 @@ Linking second, incompatible version of the library. Switching colladadom to a static library ended the re-export problem. - * Preventing re-export is not sufficient. other libraries will + * Preventing re-export is not sufficient. Other libraries will still be shipped as shared and they can still have Singleton and Fragile Base Class issues. A DLL may be built with a static archive of a library that has global data. That same static -- cgit v1.2.3 From e79a88c8ccfadcd260892000d4dec2ae921b26de Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Tue, 12 Aug 2014 18:21:26 -0400 Subject: Better support for dynamic option changes in llcorehttp. Libcurl has some problems disabling pipelining on a multi handle with outstanding requests so build a more conservative system that allows requests to drain before setting curl multi options. Would rather not have this but it is significantly safer. "HttpPipelining" debug setting is now fully dynamic. Connection limits can also be made dynamic in the near future. Upped the default connection count back to 8 for now but will revisit this in the tuning phase. It might be time to combine mesh and textures into a single asset class. For normal server operations that would be a clear path, but for server under load, the current scheme may be better. Minor cleanup in logging to elminate some redundant strings. Might add some more tracing to the stall logic 'just in case'. --- indra/llcorehttp/_httplibcurl.cpp | 198 ++++++++++++++++++++++---------- indra/llcorehttp/_httplibcurl.h | 12 +- indra/llcorehttp/_httpoperation.cpp | 28 +++-- indra/llcorehttp/_httpoprequest.cpp | 46 ++++---- indra/llcorehttp/_httppolicy.cpp | 83 ++++++++----- indra/llcorehttp/_httppolicy.h | 8 ++ indra/llcorehttp/_httpservice.cpp | 30 +++-- indra/newview/app_settings/settings.xml | 4 +- indra/newview/llappcorehttp.cpp | 174 ++++++++++++++++------------ indra/newview/llappcorehttp.h | 1 + indra/newview/lltexturefetch.cpp | 55 +++++---- indra/newview/lltexturefetch.h | 10 +- 12 files changed, 421 insertions(+), 228 deletions(-) (limited to 'indra') diff --git a/indra/llcorehttp/_httplibcurl.cpp b/indra/llcorehttp/_httplibcurl.cpp index fb907f6318..b46833a1f3 100755 --- a/indra/llcorehttp/_httplibcurl.cpp +++ b/indra/llcorehttp/_httplibcurl.cpp @@ -40,6 +40,8 @@ namespace void check_curl_multi_code(CURLMcode code); void check_curl_multi_code(CURLMcode code, int curl_setopt_option); +static const char * const LOG_CORE("CoreHttp"); + } // end anonymous namespace @@ -51,7 +53,8 @@ HttpLibcurl::HttpLibcurl(HttpService * service) : mService(service), mPolicyCount(0), mMultiHandles(NULL), - mActiveHandles(NULL) + mActiveHandles(NULL), + mDirtyPolicy(NULL) {} @@ -90,6 +93,9 @@ void HttpLibcurl::shutdown() delete [] mActiveHandles; mActiveHandles = NULL; + + delete [] mDirtyPolicy; + mDirtyPolicy = NULL; } mPolicyCount = 0; @@ -101,44 +107,21 @@ void HttpLibcurl::start(int policy_count) llassert_always(policy_count <= HTTP_POLICY_CLASS_LIMIT); llassert_always(! mMultiHandles); // One-time call only - HttpPolicy & policy(mService->getPolicy()); mPolicyCount = policy_count; mMultiHandles = new CURLM * [mPolicyCount]; mActiveHandles = new int [mPolicyCount]; + mDirtyPolicy = new bool [mPolicyCount]; for (int policy_class(0); policy_class < mPolicyCount; ++policy_class) { - HttpPolicyClass & options(policy.getClassOptions(policy_class)); - - mActiveHandles[policy_class] = 0; if (NULL == (mMultiHandles[policy_class] = curl_multi_init())) { - LL_ERRS("CoreHttp") << "Failed to allocate multi handle in libcurl." - << LL_ENDL; - } - - if (options.mPipelining > 1) - { - CURLMcode code; - - // We'll try to do pipelining on this multihandle - code = curl_multi_setopt(mMultiHandles[policy_class], - CURLMOPT_PIPELINING, - 1L); - check_curl_multi_code(code, CURLMOPT_PIPELINING); - code = curl_multi_setopt(mMultiHandles[policy_class], - CURLMOPT_MAX_PIPELINE_LENGTH, - long(options.mPipelining)); - check_curl_multi_code(code, CURLMOPT_MAX_PIPELINE_LENGTH); - code = curl_multi_setopt(mMultiHandles[policy_class], - CURLMOPT_MAX_HOST_CONNECTIONS, - long(options.mPerHostConnectionLimit)); - check_curl_multi_code(code, CURLMOPT_MAX_HOST_CONNECTIONS); - code = curl_multi_setopt(mMultiHandles[policy_class], - CURLMOPT_MAX_TOTAL_CONNECTIONS, - long(options.mConnectionLimit)); - check_curl_multi_code(code, CURLMOPT_MAX_TOTAL_CONNECTIONS); + LL_ERRS(LOG_CORE) << "Failed to allocate multi handle in libcurl." + << LL_ENDL; } + mActiveHandles[policy_class] = 0; + mDirtyPolicy[policy_class] = false; + policyUpdated(policy_class); } } @@ -156,8 +139,19 @@ HttpService::ELoopSpeed HttpLibcurl::processTransport() // Give libcurl some cycles to do I/O & callbacks for (int policy_class(0); policy_class < mPolicyCount; ++policy_class) { - if (! mActiveHandles[policy_class] || ! mMultiHandles[policy_class]) + if (! mMultiHandles[policy_class]) { + // No handle, nothing to do. + continue; + } + if (! mActiveHandles[policy_class]) + { + // If we've gone quiet and there's a dirty update, apply it, + // otherwise we're done. + if (mDirtyPolicy[policy_class]) + { + policyUpdated(policy_class); + } continue; } @@ -192,9 +186,9 @@ HttpService::ELoopSpeed HttpLibcurl::processTransport() } else { - LL_WARNS_ONCE("CoreHttp") << "Unexpected message from libcurl. Msg code: " - << msg->msg - << LL_ENDL; + LL_WARNS_ONCE(LOG_CORE) << "Unexpected message from libcurl. Msg code: " + << msg->msg + << LL_ENDL; } msgs_in_queue = 0; } @@ -230,11 +224,11 @@ void HttpLibcurl::addOp(HttpOpRequest * op) { HttpPolicy & policy(mService->getPolicy()); - LL_INFOS("CoreHttp") << "TRACE, ToActiveQueue, Handle: " - << static_cast(op) - << ", Actives: " << mActiveOps.size() - << ", Readies: " << policy.getReadyCount(op->mReqPolicy) - << LL_ENDL; + LL_INFOS(LOG_CORE) << "TRACE, ToActiveQueue, Handle: " + << static_cast(op) + << ", Actives: " << mActiveOps.size() + << ", Readies: " << policy.getReadyCount(op->mReqPolicy) + << LL_ENDL; } // On success, make operation active @@ -286,10 +280,10 @@ void HttpLibcurl::cancelRequest(HttpOpRequest * op) // Tracing if (op->mTracing > HTTP_TRACE_OFF) { - LL_INFOS("CoreHttp") << "TRACE, RequestCanceled, Handle: " - << static_cast(op) - << ", Status: " << op->mStatus.toTerseString() - << LL_ENDL; + LL_INFOS(LOG_CORE) << "TRACE, RequestCanceled, Handle: " + << static_cast(op) + << ", Status: " << op->mStatus.toTerseString() + << LL_ENDL; } // Cancel op and deliver for notification @@ -306,18 +300,18 @@ bool HttpLibcurl::completeRequest(CURLM * multi_handle, CURL * handle, CURLcode if (handle != op->mCurlHandle || ! op->mCurlActive) { - LL_WARNS("CoreHttp") << "libcurl handle and HttpOpRequest handle in disagreement or inactive request." - << " Handle: " << static_cast(handle) - << LL_ENDL; + LL_WARNS(LOG_CORE) << "libcurl handle and HttpOpRequest handle in disagreement or inactive request." + << " Handle: " << static_cast(handle) + << LL_ENDL; return false; } active_set_t::iterator it(mActiveOps.find(op)); if (mActiveOps.end() == it) { - LL_WARNS("CoreHttp") << "libcurl completion for request not on active list. Continuing." - << " Handle: " << static_cast(handle) - << LL_ENDL; + LL_WARNS(LOG_CORE) << "libcurl completion for request not on active list. Continuing." + << " Handle: " << static_cast(handle) + << LL_ENDL; return false; } @@ -348,9 +342,9 @@ bool HttpLibcurl::completeRequest(CURLM * multi_handle, CURL * handle, CURLcode } else { - LL_WARNS("CoreHttp") << "Invalid HTTP response code (" - << http_status << ") received from server." - << LL_ENDL; + LL_WARNS(LOG_CORE) << "Invalid HTTP response code (" + << http_status << ") received from server." + << LL_ENDL; op->mStatus = HttpStatus(HttpStatus::LLCORE, HE_INVALID_HTTP_STATUS); } } @@ -363,10 +357,10 @@ bool HttpLibcurl::completeRequest(CURLM * multi_handle, CURL * handle, CURLcode // Tracing if (op->mTracing > HTTP_TRACE_OFF) { - LL_INFOS("CoreHttp") << "TRACE, RequestComplete, Handle: " - << static_cast(op) - << ", Status: " << op->mStatus.toTerseString() - << LL_ENDL; + LL_INFOS(LOG_CORE) << "TRACE, RequestComplete, Handle: " + << static_cast(op) + << ", Status: " << op->mStatus.toTerseString() + << LL_ENDL; } // Dispatch to next stage @@ -390,6 +384,88 @@ int HttpLibcurl::getActiveCountInClass(int policy_class) const return mActiveHandles ? mActiveHandles[policy_class] : 0; } +void HttpLibcurl::policyUpdated(int policy_class) +{ + if (policy_class < 0 || policy_class >= mPolicyCount || ! mMultiHandles) + { + return; + } + + HttpPolicy & policy(mService->getPolicy()); + + if (! mActiveHandles[policy_class]) + { + // Clear to set options. As of libcurl 7.37.0, if a pipelining + // multi handle has active requests and you try to set the + // multi handle to non-pipelining, the library gets very angry + // and goes off the rails corrupting memory. A clue that you're + // about to crash is that you'll get a missing server response + // error (curl code 9). So, if options are to be set, we let + // the multi handle run out of requests, then set options, and + // re-enable request processing. + // + // All of this stall mechanism exists for this reason. If + // libcurl becomes more resilient later, it should be possible + // to remove all of this. The connection limit settings are fine, + // it's just that pipelined-to-non-pipelined transition that + // is fatal at the moment. + + HttpPolicyClass & options(policy.getClassOptions(policy_class)); + CURLM * multi_handle(mMultiHandles[policy_class]); + CURLMcode code; + + // Enable policy if stalled + policy.stallPolicy(policy_class, false); + mDirtyPolicy[policy_class] = false; + + if (options.mPipelining > 1) + { + // We'll try to do pipelining on this multihandle + code = curl_multi_setopt(multi_handle, + CURLMOPT_PIPELINING, + 1L); + check_curl_multi_code(code, CURLMOPT_PIPELINING); + code = curl_multi_setopt(multi_handle, + CURLMOPT_MAX_PIPELINE_LENGTH, + long(options.mPipelining)); + check_curl_multi_code(code, CURLMOPT_MAX_PIPELINE_LENGTH); + code = curl_multi_setopt(multi_handle, + CURLMOPT_MAX_HOST_CONNECTIONS, + long(options.mPerHostConnectionLimit)); + check_curl_multi_code(code, CURLMOPT_MAX_HOST_CONNECTIONS); + code = curl_multi_setopt(multi_handle, + CURLMOPT_MAX_TOTAL_CONNECTIONS, + long(options.mConnectionLimit)); + check_curl_multi_code(code, CURLMOPT_MAX_TOTAL_CONNECTIONS); + } + else + { + code = curl_multi_setopt(multi_handle, + CURLMOPT_PIPELINING, + 0L); + check_curl_multi_code(code, CURLMOPT_PIPELINING); + code = curl_multi_setopt(multi_handle, + CURLMOPT_MAX_HOST_CONNECTIONS, + 0L); + check_curl_multi_code(code, CURLMOPT_MAX_HOST_CONNECTIONS); + code = curl_multi_setopt(multi_handle, + CURLMOPT_MAX_TOTAL_CONNECTIONS, + long(options.mConnectionLimit)); + check_curl_multi_code(code, CURLMOPT_MAX_TOTAL_CONNECTIONS); + } + } + else if (! mDirtyPolicy[policy_class]) + { + // Mark policy dirty and request a stall in the policy. + // When policy goes idle, we'll re-invoke this method + // and perform the change. Don't allow this thread to + // sleep while we're waiting for quiescence, we'll just + // stop processing. + mDirtyPolicy[policy_class] = true; + policy.stallPolicy(policy_class, true); + } +} + // --------------------------------------- // Free functions @@ -424,9 +500,9 @@ void check_curl_multi_code(CURLMcode code, int curl_setopt_option) { if (CURLM_OK != code) { - LL_WARNS("CoreHttp") << "libcurl multi error detected: " << curl_multi_strerror(code) - << ", curl_multi_setopt option: " << curl_setopt_option - << LL_ENDL; + LL_WARNS(LOG_CORE) << "libcurl multi error detected: " << curl_multi_strerror(code) + << ", curl_multi_setopt option: " << curl_setopt_option + << LL_ENDL; } } @@ -435,8 +511,8 @@ void check_curl_multi_code(CURLMcode code) { if (CURLM_OK != code) { - LL_WARNS("CoreHttp") << "libcurl multi error detected: " << curl_multi_strerror(code) - << LL_ENDL; + LL_WARNS(LOG_CORE) << "libcurl multi error detected: " << curl_multi_strerror(code) + << LL_ENDL; } } diff --git a/indra/llcorehttp/_httplibcurl.h b/indra/llcorehttp/_httplibcurl.h index 67f98dd4f0..2c7ad1fa8e 100755 --- a/indra/llcorehttp/_httplibcurl.h +++ b/indra/llcorehttp/_httplibcurl.h @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012-2013, Linden Research, Inc. + * Copyright (C) 2012-2014, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -116,6 +116,14 @@ public: /// Threading: called by worker thread. bool cancel(HttpHandle handle); + /// Informs transport that a particular policy class has had + /// options changed and so should effect any transport state + /// change necessary to effect those changes. Used mainly for + /// initialization and dynamic option setting. + /// + /// Threading: called by worker thread. + void policyUpdated(int policy_class); + protected: /// Invoked when libcurl has indicated a request has been processed /// to completion and we need to move the request to a new state. @@ -134,6 +142,8 @@ protected: int mPolicyCount; CURLM ** mMultiHandles; // One handle per policy class int * mActiveHandles; // Active count per policy class + bool * mDirtyPolicy; // Dirty policy update waiting for stall (per pc) + }; // end class HttpLibcurl } // end namespace LLCore diff --git a/indra/llcorehttp/_httpoperation.cpp b/indra/llcorehttp/_httpoperation.cpp index 5bb0654652..fefe561f80 100755 --- a/indra/llcorehttp/_httpoperation.cpp +++ b/indra/llcorehttp/_httpoperation.cpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012-2013, Linden Research, Inc. + * Copyright (C) 2012-2014, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -38,6 +38,14 @@ #include "lltimer.h" +namespace +{ + +static const char * const LOG_CORE("CoreHttp"); + +} // end anonymous namespace + + namespace LLCore { @@ -94,8 +102,8 @@ void HttpOperation::stageFromRequest(HttpService *) // Default implementation should never be called. This // indicates an operation making a transition that isn't // defined. - LL_ERRS("CoreHttp") << "Default stageFromRequest method may not be called." - << LL_ENDL; + LL_ERRS(LOG_CORE) << "Default stageFromRequest method may not be called." + << LL_ENDL; } @@ -104,8 +112,8 @@ void HttpOperation::stageFromReady(HttpService *) // Default implementation should never be called. This // indicates an operation making a transition that isn't // defined. - LL_ERRS("CoreHttp") << "Default stageFromReady method may not be called." - << LL_ENDL; + LL_ERRS(LOG_CORE) << "Default stageFromReady method may not be called." + << LL_ENDL; } @@ -114,8 +122,8 @@ void HttpOperation::stageFromActive(HttpService *) // Default implementation should never be called. This // indicates an operation making a transition that isn't // defined. - LL_ERRS("CoreHttp") << "Default stageFromActive method may not be called." - << LL_ENDL; + LL_ERRS(LOG_CORE) << "Default stageFromActive method may not be called." + << LL_ENDL; } @@ -145,9 +153,9 @@ void HttpOperation::addAsReply() { if (mTracing > HTTP_TRACE_OFF) { - LL_INFOS("CoreHttp") << "TRACE, ToReplyQueue, Handle: " - << static_cast(this) - << LL_ENDL; + LL_INFOS(LOG_CORE) << "TRACE, ToReplyQueue, Handle: " + << static_cast(this) + << LL_ENDL; } if (mReplyQueue) diff --git a/indra/llcorehttp/_httpoprequest.cpp b/indra/llcorehttp/_httpoprequest.cpp index 43dd069bc6..eb664fdced 100755 --- a/indra/llcorehttp/_httpoprequest.cpp +++ b/indra/llcorehttp/_httpoprequest.cpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012-2013, Linden Research, Inc. + * Copyright (C) 2012-2014, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -94,6 +94,8 @@ void os_strlower(char * str); void check_curl_easy_code(CURLcode code); void check_curl_easy_code(CURLcode code, int curl_setopt_option); +static const char * const LOG_CORE("CoreHttp"); + } // end anonymous namespace @@ -416,8 +418,8 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service) if (! mCurlHandle) { // We're in trouble. We'll continue but it won't go well. - LL_WARNS("CoreHttp") << "Failed to allocate libcurl easy handle. Continuing." - << LL_ENDL; + LL_WARNS(LOG_CORE) << "Failed to allocate libcurl easy handle. Continuing." + << LL_ENDL; return HttpStatus(HttpStatus::LLCORE, HE_BAD_ALLOC); } code = curl_easy_setopt(mCurlHandle, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4); @@ -538,9 +540,9 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service) break; default: - LL_ERRS("CoreHttp") << "Invalid HTTP method in request: " - << int(mReqMethod) << ". Can't recover." - << LL_ENDL; + LL_ERRS(LOG_CORE) << "Invalid HTTP method in request: " + << int(mReqMethod) << ". Can't recover." + << LL_ENDL; break; } @@ -652,8 +654,8 @@ size_t HttpOpRequest::readCallback(void * data, size_t size, size_t nmemb, void { // Warn but continue if the read position moves beyond end-of-body // for some reason. - LL_WARNS("CoreHttp") << "Request body position beyond body size. Truncating request body." - << LL_ENDL; + LL_WARNS(LOG_CORE) << "Request body position beyond body size. Truncating request body." + << LL_ENDL; } return 0; } @@ -790,10 +792,10 @@ size_t HttpOpRequest::headerCallback(void * data, size_t size, size_t nmemb, voi else { // Ignore the unparsable. - LL_INFOS_ONCE("CoreHttp") << "Problem parsing odd Content-Range header: '" - << std::string(hdr_data, wanted_hdr_size) - << "'. Ignoring." - << LL_ENDL; + LL_INFOS_ONCE(LOG_CORE) << "Problem parsing odd Content-Range header: '" + << std::string(hdr_data, wanted_hdr_size) + << "'. Ignoring." + << LL_ENDL; } } @@ -895,11 +897,11 @@ int HttpOpRequest::debugCallback(CURL * handle, curl_infotype info, char * buffe if (logit) { - LL_INFOS("CoreHttp") << "TRACE, LibcurlDebug, Handle: " - << static_cast(op) - << ", Type: " << tag - << ", Data: " << safe_line - << LL_ENDL; + LL_INFOS(LOG_CORE) << "TRACE, LibcurlDebug, Handle: " + << static_cast(op) + << ", Type: " << tag + << ", Data: " << safe_line + << LL_ENDL; } return 0; @@ -1094,9 +1096,9 @@ void check_curl_easy_code(CURLcode code, int curl_setopt_option) // // linux appears to throw a curl error once per session for a bad initialization // at a pretty random time (when enabling cookies). - LL_WARNS("CoreHttp") << "libcurl error detected: " << curl_easy_strerror(code) - << ", curl_easy_setopt option: " << curl_setopt_option - << LL_ENDL; + LL_WARNS(LOG_CORE) << "libcurl error detected: " << curl_easy_strerror(code) + << ", curl_easy_setopt option: " << curl_setopt_option + << LL_ENDL; } } @@ -1109,8 +1111,8 @@ void check_curl_easy_code(CURLcode code) // // linux appears to throw a curl error once per session for a bad initialization // at a pretty random time (when enabling cookies). - LL_WARNS("CoreHttp") << "libcurl error detected: " << curl_easy_strerror(code) - << LL_ENDL; + LL_WARNS(LOG_CORE) << "libcurl error detected: " << curl_easy_strerror(code) + << LL_ENDL; } } diff --git a/indra/llcorehttp/_httppolicy.cpp b/indra/llcorehttp/_httppolicy.cpp index bb7959b578..09b9206f63 100755 --- a/indra/llcorehttp/_httppolicy.cpp +++ b/indra/llcorehttp/_httppolicy.cpp @@ -35,6 +35,13 @@ #include "lltimer.h" +namespace +{ + +static const char * const LOG_CORE("CoreHttp"); + +} // end anonymous namespace + namespace LLCore { @@ -51,7 +58,8 @@ public: ClassState() : mThrottleEnd(0), mThrottleLeft(0L), - mRequestCount(0L) + mRequestCount(0L), + mStallStaging(false) {} HttpReadyQueue mReadyQueue; @@ -61,6 +69,7 @@ public: HttpTime mThrottleEnd; long mThrottleLeft; long mRequestCount; + bool mStallStaging; }; @@ -171,19 +180,19 @@ void HttpPolicy::retryOp(HttpOpRequest * op) { ++op->mPolicy503Retries; } - LL_DEBUGS("CoreHttp") << "HTTP request " << static_cast(op) - << " retry " << op->mPolicyRetries - << " scheduled in " << (delta / HttpTime(1000)) - << " mS (" << (external_delta ? "external" : "internal") - << "). Status: " << op->mStatus.toTerseString() - << LL_ENDL; + LL_DEBUGS(LOG_CORE) << "HTTP request " << static_cast(op) + << " retry " << op->mPolicyRetries + << " scheduled in " << (delta / HttpTime(1000)) + << " mS (" << (external_delta ? "external" : "internal") + << "). Status: " << op->mStatus.toTerseString() + << LL_ENDL; if (op->mTracing > HTTP_TRACE_OFF) { - LL_INFOS("CoreHttp") << "TRACE, ToRetryQueue, Handle: " - << static_cast(op) - << ", Delta: " << (delta / HttpTime(1000)) - << ", Retries: " << op->mPolicyRetries - << LL_ENDL; + LL_INFOS(LOG_CORE) << "TRACE, ToRetryQueue, Handle: " + << static_cast(op) + << ", Delta: " << (delta / HttpTime(1000)) + << ", Retries: " << op->mPolicyRetries + << LL_ENDL; } mClasses[policy_class]->mRetryQueue.push(op); } @@ -219,6 +228,15 @@ HttpService::ELoopSpeed HttpPolicy::processReadyQueue() HttpRetryQueue & retryq(state.mRetryQueue); HttpReadyQueue & readyq(state.mReadyQueue); + if (state.mStallStaging) + { + // Stalling but don't sleep. Need to complete operations + // and get back to servicing queues. Do this test before + // the retryq/readyq test or you'll get stalls until you + // click a setting or an asset request comes in. + result = HttpService::NORMAL; + continue; + } if (retryq.empty() && readyq.empty()) { continue; @@ -262,9 +280,9 @@ HttpService::ELoopSpeed HttpPolicy::processReadyQueue() if (now >= state.mThrottleEnd) { // Throttle expired, move to next window - LL_DEBUGS("CoreHttp") << "Throttle expired with " << state.mThrottleLeft - << " requests to go and " << state.mRequestCount - << " requests issued." << LL_ENDL; + LL_DEBUGS(LOG_CORE) << "Throttle expired with " << state.mThrottleLeft + << " requests to go and " << state.mRequestCount + << " requests issued." << LL_ENDL; state.mThrottleLeft = state.mOptions.mThrottleRate; state.mThrottleEnd = now + HttpTime(1000000); } @@ -291,9 +309,9 @@ HttpService::ELoopSpeed HttpPolicy::processReadyQueue() if (now >= state.mThrottleEnd) { // Throttle expired, move to next window - LL_DEBUGS("CoreHttp") << "Throttle expired with " << state.mThrottleLeft - << " requests to go and " << state.mRequestCount - << " requests issued." << LL_ENDL; + LL_DEBUGS(LOG_CORE) << "Throttle expired with " << state.mThrottleLeft + << " requests to go and " << state.mRequestCount + << " requests issued." << LL_ENDL; state.mThrottleLeft = state.mOptions.mThrottleRate; state.mThrottleEnd = now + HttpTime(1000000); } @@ -408,17 +426,17 @@ bool HttpPolicy::stageAfterCompletion(HttpOpRequest * op) // This op is done, finalize it delivering it to the reply queue... if (! op->mStatus) { - LL_WARNS("CoreHttp") << "HTTP request " << static_cast(op) - << " failed after " << op->mPolicyRetries - << " retries. Reason: " << op->mStatus.toString() - << " (" << op->mStatus.toTerseString() << ")" - << LL_ENDL; + LL_WARNS(LOG_CORE) << "HTTP request " << static_cast(op) + << " failed after " << op->mPolicyRetries + << " retries. Reason: " << op->mStatus.toString() + << " (" << op->mStatus.toTerseString() << ")" + << LL_ENDL; } else if (op->mPolicyRetries) { - LL_DEBUGS("CoreHttp") << "HTTP request " << static_cast(op) - << " succeeded on retry " << op->mPolicyRetries << "." - << LL_ENDL; + LL_DEBUGS(LOG_CORE) << "HTTP request " << static_cast(op) + << " succeeded on retry " << op->mPolicyRetries << "." + << LL_ENDL; } op->stageFromActive(mService); @@ -446,4 +464,17 @@ int HttpPolicy::getReadyCount(HttpRequest::policy_t policy_class) const } +bool HttpPolicy::stallPolicy(HttpRequest::policy_t policy_class, bool stall) +{ + bool ret(false); + + if (policy_class < mClasses.size()) + { + ret = mClasses[policy_class]->mStallStaging; + mClasses[policy_class]->mStallStaging = stall; + } + return ret; +} + + } // end namespace LLCore diff --git a/indra/llcorehttp/_httppolicy.h b/indra/llcorehttp/_httppolicy.h index bf1aa74267..11cd89bbd1 100755 --- a/indra/llcorehttp/_httppolicy.h +++ b/indra/llcorehttp/_httppolicy.h @@ -158,6 +158,14 @@ public: /// Threading: called by worker thread int getReadyCount(HttpRequest::policy_t policy_class) const; + /// Stall (or unstall) a policy class preventing requests from + /// transitioning to an active state. Used to allow an HTTP + /// request policy to empty prior to changing settings or state + /// that isn't tolerant of changes when work is outstanding. + /// + /// Threading: called by worker thread + bool stallPolicy(HttpRequest::policy_t policy_class, bool stall); + protected: struct ClassState; typedef std::vector class_list_t; diff --git a/indra/llcorehttp/_httpservice.cpp b/indra/llcorehttp/_httpservice.cpp index c94249dc2d..c673e1be1d 100755 --- a/indra/llcorehttp/_httpservice.cpp +++ b/indra/llcorehttp/_httpservice.cpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012-2013, Linden Research, Inc. + * Copyright (C) 2012-2014, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -40,6 +40,14 @@ #include "llthread.h" +namespace +{ + +static const char * const LOG_CORE("CoreHttp"); + +} // end anonymous namespace + + namespace LLCore { @@ -87,8 +95,8 @@ HttpService::~HttpService() // Failed to join, expect problems ahead so do a hard termination. mThread->cancel(); - LL_WARNS("CoreHttp") << "Destroying HttpService with running thread. Expect problems." - << LL_ENDL; + LL_WARNS(LOG_CORE) << "Destroying HttpService with running thread. Expect problems." + << LL_ENDL; } } } @@ -328,9 +336,9 @@ HttpService::ELoopSpeed HttpService::processRequestQueue(ELoopSpeed loop) if (op->mTracing > HTTP_TRACE_OFF) { - LL_INFOS("CoreHttp") << "TRACE, FromRequestQueue, Handle: " - << static_cast(op) - << LL_ENDL; + LL_INFOS(LOG_CORE) << "TRACE, FromRequestQueue, Handle: " + << static_cast(op) + << LL_ENDL; } // Stage @@ -437,9 +445,13 @@ HttpStatus HttpService::setPolicyOption(HttpRequest::EPolicyOption opt, HttpRequ HttpPolicyClass & opts(mPolicy->getClassOptions(pclass)); status = opts.set(opt, value); - if (status && ret_value) + if (status) { - status = opts.get(opt, ret_value); + mTransport->policyUpdated(pclass); + if (ret_value) + { + status = opts.get(opt, ret_value); + } } } @@ -463,7 +475,7 @@ HttpStatus HttpService::setPolicyOption(HttpRequest::EPolicyOption opt, HttpRequ return status; } - // Only string values are global at this time + // String values are always global (at this time). if (pclass == HttpRequest::GLOBAL_POLICY_ID) { HttpPolicyGlobal & opts(mPolicy->getGlobalOptions()); diff --git a/indra/newview/app_settings/settings.xml b/indra/newview/app_settings/settings.xml index 50f56cf6ff..0607579a08 100755 --- a/indra/newview/app_settings/settings.xml +++ b/indra/newview/app_settings/settings.xml @@ -4459,7 +4459,7 @@ HttpPipelining Comment - If true, viewer will pipeline HTTP requests to servers. Static. + If true, viewer will pipeline HTTP requests to servers. Persist 1 Type @@ -4470,7 +4470,7 @@ HttpRangeRequestsDisable Comment - If true, viewer will not issued range GET requests for meshes and textures. + If true, viewer will not issued range GET requests for meshes and textures. May resolve problems with certain ISPs and networking gear. Persist 1 Type diff --git a/indra/newview/llappcorehttp.cpp b/indra/newview/llappcorehttp.cpp index d097f18d61..464e60948a 100755 --- a/indra/newview/llappcorehttp.cpp +++ b/indra/newview/llappcorehttp.cpp @@ -60,7 +60,7 @@ static const struct "other" }, { // AP_TEXTURE - 4, 1, 12, 0, true, + 8, 1, 12, 0, true, "TextureFetchConcurrency", "texture fetch" }, @@ -70,7 +70,7 @@ static const struct "mesh fetch" }, { // AP_MESH2 - 4, 1, 32, 100, true, + 8, 1, 32, 100, true, "Mesh2MaxConcurrentRequests", "mesh2 fetch" }, @@ -126,14 +126,6 @@ void LLAppCoreHttp::init() << LL_ENDL; } - // Global pipelining preference from settings - static const std::string http_pipelining("HttpPipelining"); - if (gSavedSettings.controlExists(http_pipelining)) - { - // Default to true if absent. - mPipelined = gSavedSettings.getBOOL(http_pipelining); - } - // Point to our certs or SSH/https: will fail on connect status = LLCore::HttpRequest::setStaticPolicyOption(LLCore::HttpRequest::PO_CA_FILE, LLCore::HttpRequest::GLOBAL_POLICY_ID, @@ -210,12 +202,27 @@ void LLAppCoreHttp::init() << LL_ENDL; } - // *NOTE: Pipelining isn't dynamic yet. When it is, add a global - // signal for the setting here. - + // Signal for global pipelining preference from settings + static const std::string http_pipelining("HttpPipelining"); + if (gSavedSettings.controlExists(http_pipelining)) + { + LLPointer cntrl_ptr = gSavedSettings.getControl(http_pipelining); + if (cntrl_ptr.isNull()) + { + LL_WARNS("Init") << "Unable to set signal on global setting '" << http_pipelining + << "'" << LL_ENDL; + } + else + { + mPipelinedSignal = cntrl_ptr->getCommitSignal()->connect(boost::bind(&setting_changed)); + } + } + // Register signals for settings and state changes for (int i(0); i < LL_ARRAY_SIZE(init_data); ++i) { + const EAppPolicy app_policy(static_cast(i)); + if (! init_data[i].mKey.empty() && gSavedSettings.controlExists(init_data[i].mKey)) { LLPointer cntrl_ptr = gSavedSettings.getControl(init_data[i].mKey); @@ -226,7 +233,7 @@ void LLAppCoreHttp::init() } else { - mHttpClasses[i].mSettingsSignal = cntrl_ptr->getCommitSignal()->connect(boost::bind(&setting_changed)); + mHttpClasses[app_policy].mSettingsSignal = cntrl_ptr->getCommitSignal()->connect(boost::bind(&setting_changed)); } } } @@ -282,6 +289,7 @@ void LLAppCoreHttp::cleanup() { mHttpClasses[i].mSettingsSignal.disconnect(); } + mPipelinedSignal.disconnect(); delete mRequest; mRequest = NULL; @@ -299,6 +307,20 @@ void LLAppCoreHttp::cleanup() void LLAppCoreHttp::refreshSettings(bool initial) { LLCore::HttpStatus status; + + // Global pipelining setting + bool pipeline_changed(false); + static const std::string http_pipelining("HttpPipelining"); + if (gSavedSettings.controlExists(http_pipelining)) + { + // Default to true (in ctor) if absent. + bool pipelined(gSavedSettings.getBOOL(http_pipelining)); + if (pipelined != mPipelined) + { + mPipelined = pipelined; + pipeline_changed = true; + } + } for (int i(0); i < LL_ARRAY_SIZE(init_data); ++i) { @@ -323,33 +345,42 @@ void LLAppCoreHttp::refreshSettings(bool initial) } } - mHttpClasses[app_policy].mPipelined = false; - if (mPipelined && init_data[i].mPipelined) + } + + // Init- or run-time settings. Must use the queued request API. + + // Pipelining changes + if (initial || pipeline_changed) + { + const bool to_pipeline(mPipelined && init_data[i].mPipelined); + if (to_pipeline != mHttpClasses[app_policy].mPipelined) { - // Pipelining election is currently static (init-time). - // Making it dynamic isn't too hard in the SL code but verifying - // that libcurl handles the on-to-off transition while holding - // outstanding requests is something that should be tested. + // Pipeline election changing, set dynamic option via request + + LLCore::HttpHandle handle; + const long new_depth(to_pipeline ? PIPELINING_DEPTH : 0); - status = LLCore::HttpRequest::setStaticPolicyOption(LLCore::HttpRequest::PO_PIPELINING_DEPTH, - mHttpClasses[app_policy].mPolicy, - PIPELINING_DEPTH, - NULL); - if (! status) + handle = mRequest->setPolicyOption(LLCore::HttpRequest::PO_PIPELINING_DEPTH, + mHttpClasses[app_policy].mPolicy, + new_depth, + NULL); + if (LLCORE_HTTP_HANDLE_INVALID == handle) { + status = mRequest->getStatus(); LL_WARNS("Init") << "Unable to set " << init_data[i].mUsage - << " to pipelined mode. Reason: " << status.toString() + << " pipelining. Reason: " << status.toString() << LL_ENDL; } else { - mHttpClasses[app_policy].mPipelined = true; + LL_DEBUGS("Init") << "Changed " << init_data[i].mUsage + << " pipelining. New value: " << new_depth + << LL_ENDL; + mHttpClasses[app_policy].mPipelined = to_pipeline; } } } - - // Init- or run-time settings - + // Get target connection concurrency value U32 setting(init_data[i].mDefault); if (! init_data[i].mKey.empty() && gSavedSettings.controlExists(init_data[i].mKey)) @@ -362,63 +393,60 @@ void LLAppCoreHttp::refreshSettings(bool initial) } } - if (! initial && setting == mHttpClasses[app_policy].mConnLimit) + if (initial || setting != mHttpClasses[app_policy].mConnLimit || pipeline_changed) { - // Unchanged, try next setting - continue; - } - - // Set it and report. Strategies depend on pipelining: - // - // No Pipelining. Llcorehttp manages connections itself based - // on the PO_CONNECTION_LIMIT setting. Set both limits to the - // same value for logical consistency. In the future, may - // hand over connection management to libcurl after the - // connection cache has been better vetted. - // - // Pipelining. Libcurl is allowed to manage connections to a - // great degree. Steady state will connection limit based on - // the per-host setting. Transitions (region crossings, new - // avatars, etc.) can request additional outbound connections - // to other servers via 2X total connection limit. - // - LLCore::HttpHandle handle; - handle = mRequest->setPolicyOption(LLCore::HttpRequest::PO_CONNECTION_LIMIT, - mHttpClasses[app_policy].mPolicy, - (mHttpClasses[app_policy].mPipelined ? 2 * setting : setting), - NULL); - if (LLCORE_HTTP_HANDLE_INVALID == handle) - { - status = mRequest->getStatus(); - LL_WARNS("Init") << "Unable to set " << init_data[i].mUsage - << " concurrency. Reason: " << status.toString() - << LL_ENDL; - } - else - { - handle = mRequest->setPolicyOption(LLCore::HttpRequest::PO_PER_HOST_CONNECTION_LIMIT, + // Set it and report. Strategies depend on pipelining: + // + // No Pipelining. Llcorehttp manages connections itself based + // on the PO_CONNECTION_LIMIT setting. Set both limits to the + // same value for logical consistency. In the future, may + // hand over connection management to libcurl after the + // connection cache has been better vetted. + // + // Pipelining. Libcurl is allowed to manage connections to a + // great degree. Steady state will connection limit based on + // the per-host setting. Transitions (region crossings, new + // avatars, etc.) can request additional outbound connections + // to other servers via 2X total connection limit. + // + LLCore::HttpHandle handle; + handle = mRequest->setPolicyOption(LLCore::HttpRequest::PO_CONNECTION_LIMIT, mHttpClasses[app_policy].mPolicy, - setting, + (mHttpClasses[app_policy].mPipelined ? 2 * setting : setting), NULL); if (LLCORE_HTTP_HANDLE_INVALID == handle) { status = mRequest->getStatus(); LL_WARNS("Init") << "Unable to set " << init_data[i].mUsage - << " per-host concurrency. Reason: " << status.toString() + << " concurrency. Reason: " << status.toString() << LL_ENDL; } else { - LL_DEBUGS("Init") << "Changed " << init_data[i].mUsage - << " concurrency. New value: " << setting - << LL_ENDL; - mHttpClasses[app_policy].mConnLimit = setting; - if (initial && setting != init_data[i].mDefault) + handle = mRequest->setPolicyOption(LLCore::HttpRequest::PO_PER_HOST_CONNECTION_LIMIT, + mHttpClasses[app_policy].mPolicy, + setting, + NULL); + if (LLCORE_HTTP_HANDLE_INVALID == handle) { - LL_INFOS("Init") << "Application settings overriding default " << init_data[i].mUsage - << " concurrency. New value: " << setting + status = mRequest->getStatus(); + LL_WARNS("Init") << "Unable to set " << init_data[i].mUsage + << " per-host concurrency. Reason: " << status.toString() << LL_ENDL; } + else + { + LL_DEBUGS("Init") << "Changed " << init_data[i].mUsage + << " concurrency. New value: " << setting + << LL_ENDL; + mHttpClasses[app_policy].mConnLimit = setting; + if (initial && setting != init_data[i].mDefault) + { + LL_INFOS("Init") << "Application settings overriding default " << init_data[i].mUsage + << " concurrency. New value: " << setting + << LL_ENDL; + } + } } } } diff --git a/indra/newview/llappcorehttp.h b/indra/newview/llappcorehttp.h index 63c8a11180..9ad4eb4b30 100755 --- a/indra/newview/llappcorehttp.h +++ b/indra/newview/llappcorehttp.h @@ -218,6 +218,7 @@ private: bool mStopped; HttpClass mHttpClasses[AP_COUNT]; bool mPipelined; // Global setting + boost::signals2::connection mPipelinedSignal; // Signal for 'HttpPipelining' setting }; diff --git a/indra/newview/lltexturefetch.cpp b/indra/newview/lltexturefetch.cpp index 4008a6948d..097a7b374f 100755 --- a/indra/newview/lltexturefetch.cpp +++ b/indra/newview/lltexturefetch.cpp @@ -483,12 +483,12 @@ private: bool acquireHttpSemaphore() { llassert(! mHttpHasResource); - if (mFetcher->mHttpSemaphore <= 0) + if (mFetcher->mHttpSemaphore >= mFetcher->mHttpHighWater) { return false; } mHttpHasResource = true; - mFetcher->mHttpSemaphore--; + mFetcher->mHttpSemaphore++; return true; } @@ -498,7 +498,8 @@ private: { llassert(mHttpHasResource); mHttpHasResource = false; - mFetcher->mHttpSemaphore++; + mFetcher->mHttpSemaphore--; + llassert_always(mFetcher->mHttpSemaphore >= 0); } private: @@ -2523,6 +2524,8 @@ LLTextureFetch::LLTextureFetch(LLTextureCache* cache, LLImageDecodeThread* image mMaxBandwidth = gSavedSettings.getF32("ThrottleBandwidthKBPS"); mTextureInfo.setUpLogging(gSavedSettings.getBOOL("LogTextureDownloadsToViewerLog"), gSavedSettings.getBOOL("LogTextureDownloadsToSimulator"), U32Bytes(gSavedSettings.getU32("TextureLoggingThreshold"))); + LLAppCoreHttp & app_core_http(LLAppViewer::instance()->getAppCoreHttp()); + mHttpPolicyClass = app_core_http.getPolicy(LLAppCoreHttp::AP_TEXTURE); mHttpRequest = new LLCore::HttpRequest; mHttpOptions = new LLCore::HttpOptions; mHttpOptionsWithHeaders = new LLCore::HttpOptions; @@ -2531,21 +2534,9 @@ LLTextureFetch::LLTextureFetch(LLTextureCache* cache, LLImageDecodeThread* image mHttpHeaders->append("Accept", "image/x-j2c"); mHttpMetricsHeaders = new LLCore::HttpHeaders; mHttpMetricsHeaders->append("Content-Type", "application/llsd+xml"); - LLAppCoreHttp & app_core_http(LLAppViewer::instance()->getAppCoreHttp()); - mHttpPolicyClass = app_core_http.getPolicy(LLAppCoreHttp::AP_TEXTURE); - if (app_core_http.isPipelined(LLAppCoreHttp::AP_TEXTURE)) - { - // Init-time election that will have to change for - // support of dynamic changes to the pipelining enable flag. - mHttpHighWater = HTTP_PIPE_REQUESTS_HIGH_WATER; - mHttpLowWater = HTTP_PIPE_REQUESTS_LOW_WATER; - } - else - { - mHttpHighWater = HTTP_NONPIPE_REQUESTS_HIGH_WATER; - mHttpLowWater = HTTP_NONPIPE_REQUESTS_LOW_WATER; - } - mHttpSemaphore = mHttpHighWater; + mHttpHighWater = HTTP_NONPIPE_REQUESTS_HIGH_WATER; + mHttpLowWater = HTTP_NONPIPE_REQUESTS_LOW_WATER; + mHttpSemaphore = 0; // Conditionally construct debugger object after 'this' is // fully initialized. @@ -3032,6 +3023,20 @@ bool LLTextureFetch::runCondition() // Threads: Ttf void LLTextureFetch::commonUpdate() { + // Update low/high water levels based on pipelining. We pick + // up setting eventually, so the semaphore/request level can + // fall outside the [0..HIGH_WATER] range. Expect that. + if (LLAppViewer::instance()->getAppCoreHttp().isPipelined(LLAppCoreHttp::AP_TEXTURE)) + { + mHttpHighWater = HTTP_PIPE_REQUESTS_HIGH_WATER; + mHttpLowWater = HTTP_PIPE_REQUESTS_LOW_WATER; + } + else + { + mHttpHighWater = HTTP_NONPIPE_REQUESTS_HIGH_WATER; + mHttpLowWater = HTTP_NONPIPE_REQUESTS_LOW_WATER; + } + // Release waiters releaseHttpWaiters(); @@ -3693,8 +3698,16 @@ void LLTextureFetch::releaseHttpWaiters() { // Use mHttpSemaphore rather than mHTTPTextureQueue.size() // to avoid a lock. - if (mHttpSemaphore < (mHttpHighWater - mHttpLowWater)) + if (mHttpSemaphore >= mHttpLowWater) return; + S32 needed(mHttpHighWater - mHttpSemaphore); + if (needed <= 0) + { + // Would only happen if High/LowWater were changed behind + // our back. In that case, defer fill until usage falls within + // limits. + return; + } // Quickly make a copy of all the LLUIDs. Get off the // mutex as early as possible. @@ -3743,10 +3756,10 @@ void LLTextureFetch::releaseHttpWaiters() tids.clear(); // Sort into priority order, if necessary and only as much as needed - if (tids2.size() > mHttpSemaphore) + if (tids2.size() > needed) { LLTextureFetchWorker::Compare compare; - std::partial_sort(tids2.begin(), tids2.begin() + mHttpSemaphore, tids2.end(), compare); + std::partial_sort(tids2.begin(), tids2.begin() + needed, tids2.end(), compare); } // Release workers up to the high water mark. Since we aren't diff --git a/indra/newview/lltexturefetch.h b/indra/newview/lltexturefetch.h index d13736997f..89d18e2c67 100755 --- a/indra/newview/lltexturefetch.h +++ b/indra/newview/lltexturefetch.h @@ -356,8 +356,8 @@ private: LLCore::HttpHeaders * mHttpHeaders; // Ttf LLCore::HttpHeaders * mHttpMetricsHeaders; // Ttf LLCore::HttpRequest::policy_t mHttpPolicyClass; // T* - S32 mHttpHighWater; // T* (ro) - S32 mHttpLowWater; // T* (ro) + S32 mHttpHighWater; // Ttf + S32 mHttpLowWater; // Ttf // We use a resource semaphore to keep HTTP requests in // WAIT_HTTP_RESOURCE2 if there aren't sufficient slots in the @@ -366,7 +366,11 @@ private: // where it's more expensive to get at them. Requests in either // SEND_HTTP_REQ or WAIT_HTTP_REQ charge against the semaphore // and tracking state transitions is critical to liveness. - LLAtomicS32 mHttpSemaphore; // Ttf + Tmain + // + // Originally implemented as a traditional semaphore (heading towards + // zero), it now is an outstanding request count that is allowed to + // exceed the high water level (but not go below zero). + LLAtomicS32 mHttpSemaphore; // Ttf typedef std::set wait_http_res_queue_t; wait_http_res_queue_t mHttpWaitResource; // Mfnq -- cgit v1.2.3 From 7fa382937679a9937fd7b09e33b6c2f39ec680ff Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Thu, 21 Aug 2014 15:20:31 -0400 Subject: Remove viewer-side throttles on mesh requests. --- indra/newview/llappcorehttp.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'indra') diff --git a/indra/newview/llappcorehttp.cpp b/indra/newview/llappcorehttp.cpp index 464e60948a..e9274c5c1e 100755 --- a/indra/newview/llappcorehttp.cpp +++ b/indra/newview/llappcorehttp.cpp @@ -65,12 +65,12 @@ static const struct "texture fetch" }, { // AP_MESH1 - 32, 1, 128, 100, false, + 32, 1, 128, 0, false, "MeshMaxConcurrentRequests", "mesh fetch" }, { // AP_MESH2 - 8, 1, 32, 100, true, + 8, 1, 32, 0, true, "Mesh2MaxConcurrentRequests", "mesh2 fetch" }, -- cgit v1.2.3 From 85cba58ad473ed28efda7f645af20d56229e8637 Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Fri, 22 Aug 2014 18:04:27 -0400 Subject: Add an HTTP policy class for inventory operations using four (4) connections. Convert background and foreground fetches, both items and folders/inventory and library, to use new HTTP. Non-fetch inventory operations continue to use LLHTTPClient (at least for now). Error handling and retry on fetches wasn't 100% previously and that's still the case. I'll rip through this again to clean that up. Cleaned up logging in much of the inventory code with consistent labels on logging events and correct macros (removed deprecation warnings). This started as an attempt to get libcurl to do pipelining on POSTs and PUTs. Discovered that this is going to be very difficult to support in general in libcurl. May look at that again in the future. --- indra/newview/llappcorehttp.cpp | 9 +- indra/newview/llappcorehttp.h | 18 +- indra/newview/llinventorymodel.cpp | 601 ++++++++++++------ indra/newview/llinventorymodel.h | 82 ++- indra/newview/llinventorymodelbackgroundfetch.cpp | 740 +++++++++++++--------- indra/newview/llinventorymodelbackgroundfetch.h | 28 +- indra/newview/llinventoryobserver.cpp | 3 +- indra/newview/llsnapshotlivepreview.cpp | 16 +- indra/newview/llviewerinventory.cpp | 128 ++-- indra/newview/llviewermenu.cpp | 4 +- 10 files changed, 1034 insertions(+), 595 deletions(-) (limited to 'indra') diff --git a/indra/newview/llappcorehttp.cpp b/indra/newview/llappcorehttp.cpp index 464e60948a..2b7d91b5ff 100755 --- a/indra/newview/llappcorehttp.cpp +++ b/indra/newview/llappcorehttp.cpp @@ -60,7 +60,7 @@ static const struct "other" }, { // AP_TEXTURE - 8, 1, 12, 0, true, + 8, 1, 12, 0, true, "TextureFetchConcurrency", "texture fetch" }, @@ -70,7 +70,7 @@ static const struct "mesh fetch" }, { // AP_MESH2 - 8, 1, 32, 100, true, + 8, 1, 32, 100, true, "Mesh2MaxConcurrentRequests", "mesh2 fetch" }, @@ -88,6 +88,11 @@ static const struct 32, 32, 32, 0, false, "", "long poll" + }, + { // AP_INVENTORY + 4, 1, 4, 0, false, + "", + "inventory" } }; diff --git a/indra/newview/llappcorehttp.h b/indra/newview/llappcorehttp.h index 9ad4eb4b30..e822a40300 100755 --- a/indra/newview/llappcorehttp.h +++ b/indra/newview/llappcorehttp.h @@ -64,7 +64,7 @@ public: /// Texture fetching policy class. Used to /// download textures via capability or SSA /// baking service. Deep queueing of requests. - /// Do not share. + /// Do not share. GET requests only. /// /// Destination: simhost:12046 & bake-texture:80 /// Protocol: http: @@ -92,7 +92,8 @@ public: /// download textures via 'GetMesh2' capability. /// Used when fetch request (typically one LOD) /// is 'small', currently defined as 2MB. - /// Very deeply queued. Do not share. + /// Very deeply queued. Do not share. GET + /// requests only. /// /// Destination: simhost:12046 /// Protocol: http: @@ -150,6 +151,19 @@ public: /// Pipelined: no AP_LONG_POLL, + /// Inventory operations (really Capabilities- + /// related operations). Mix of high-priority + /// and low-priority operations. + /// + /// Destination: simhost:12043 + /// Protocol: https: + /// Transfer size: KB-MB + /// Long poll: no + /// Concurrency: high + /// Request rate: high + /// Pipelined: no + AP_INVENTORY, + AP_COUNT // Must be last }; diff --git a/indra/newview/llinventorymodel.cpp b/indra/newview/llinventorymodel.cpp index 14ca0095ae..2dd31f047f 100755 --- a/indra/newview/llinventorymodel.cpp +++ b/indra/newview/llinventorymodel.cpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2002&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2010, Linden Research, Inc. + * Copyright (C) 2014, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -25,6 +25,9 @@ */ #include "llviewerprecompiledheaders.h" + +#include + #include "llinventorymodel.h" #include "llaisapi.h" @@ -50,7 +53,9 @@ #include "llvoavatarself.h" #include "llgesturemgr.h" #include "llsdutil.h" -#include +#include "bufferarray.h" +#include "bufferstream.h" +#include "llsdserialize.h" //#define DIFF_INVENTORY_FILES #ifdef DIFF_INVENTORY_FILES @@ -67,7 +72,14 @@ BOOL LLInventoryModel::sFirstTimeInViewer2 = TRUE; ///---------------------------------------------------------------------------- //BOOL decompress_file(const char* src_filename, const char* dst_filename); -const char CACHE_FORMAT_STRING[] = "%s.inv"; +static const char CACHE_FORMAT_STRING[] = "%s.inv"; +static const char * const LOG_INV("Inventory"); + +static std::string dumpResponse() +{ + return std::string("ADD SOMETHING MEANINGFUL HERE"); +} + struct InventoryIDPtrLess { @@ -125,24 +137,32 @@ LLInventoryModel gInventory; // Default constructor LLInventoryModel::LLInventoryModel() -: mModifyMask(LLInventoryObserver::ALL), - mChangedItemIDs(), +: // These are now ordered, keep them that way. mBacklinkMMap(), + mIsAgentInvUsable(false), + mRootFolderID(), + mLibraryRootFolderID(), + mLibraryOwnerID(), mCategoryMap(), mItemMap(), - mCategoryLock(), - mItemLock(), - mLastItem(NULL), mParentChildCategoryTree(), mParentChildItemTree(), - mObservers(), - mRootFolderID(), - mLibraryRootFolderID(), - mLibraryOwnerID(), + mLastItem(NULL), mIsNotifyObservers(FALSE), - mIsAgentInvUsable(false) -{ -} + mModifyMask(LLInventoryObserver::ALL), + mChangedItemIDs(), + mObservers(), + mHttpRequestFG(NULL), + mHttpRequestBG(NULL), + mHttpOptions(NULL), + mHttpHeaders(NULL), + mHttpPolicyClass(LLCore::HttpRequest::DEFAULT_POLICY_ID), + mHttpPriorityFG(0), + mHttpPriorityBG(0), + mCategoryLock(), + mItemLock() +{} + // Destroys the object LLInventoryModel::~LLInventoryModel() @@ -162,6 +182,22 @@ void LLInventoryModel::cleanupInventory() delete observer; } mObservers.clear(); + + // Run down HTTP transport + if (mHttpHeaders) + { + mHttpHeaders->release(); + mHttpHeaders = NULL; + } + if (mHttpOptions) + { + mHttpOptions->release(); + mHttpOptions = NULL; + } + delete mHttpRequestFG; + mHttpRequestFG = NULL; + delete mHttpRequestBG; + mHttpRequestBG = NULL; } // This is a convenience function to check if one object has a parent @@ -253,7 +289,7 @@ bool LLInventoryModel::getObjectTopmostAncestor(const LLUUID& object_id, LLUUID& LLInventoryObject *parent_object = getObject(object->getParentUUID()); if (!parent_object) { - LL_WARNS() << "unable to trace topmost ancestor, missing item for uuid " << object->getParentUUID() << LL_ENDL; + LL_WARNS(LOG_INV) << "unable to trace topmost ancestor, missing item for uuid " << object->getParentUUID() << LL_ENDL; return false; } object = parent_object; @@ -508,7 +544,7 @@ public: protected: virtual void httpFailure() { - LL_WARNS("InvAPI") << dumpResponse() << LL_ENDL; + LL_WARNS(LOG_INV) << dumpResponse() << LL_ENDL; } virtual void httpSuccess() @@ -522,7 +558,7 @@ protected: } LLUUID category_id = content["folder_id"].asUUID(); - LL_DEBUGS("Avatar") << ll_pretty_print_sd(content) << LL_ENDL; + LL_DEBUGS(LOG_INV) << ll_pretty_print_sd(content) << LL_ENDL; // Add the category to the internal representation LLPointer cat = new LLViewerInventoryCategory( category_id, @@ -560,13 +596,13 @@ LLUUID LLInventoryModel::createNewCategory(const LLUUID& parent_id, LLUUID id; if(!isInventoryUsable()) { - LL_WARNS() << "Inventory is broken." << LL_ENDL; + LL_WARNS(LOG_INV) << "Inventory is broken." << LL_ENDL; return id; } if(LLFolderType::lookup(preferred_type) == LLFolderType::badLookup()) { - LL_DEBUGS() << "Attempt to create undefined category." << LL_ENDL; + LL_DEBUGS(LOG_INV) << "Attempt to create undefined category." << LL_ENDL; return id; } @@ -599,7 +635,7 @@ LLUUID LLInventoryModel::createNewCategory(const LLUUID& parent_id, request["message"] = "CreateInventoryCategory"; request["payload"] = body; - LL_DEBUGS("Avatar") << "create category request: " << ll_pretty_print_sd(request) << LL_ENDL; + LL_DEBUGS(LOG_INV) << "create category request: " << ll_pretty_print_sd(request) << LL_ENDL; // viewer_region->getCapAPI().post(request); LLHTTPClient::post( url, @@ -820,7 +856,7 @@ U32 LLInventoryModel::updateItem(const LLViewerInventoryItem* item, U32 mask) if(!isInventoryUsable()) { - LL_WARNS() << "Inventory is broken." << LL_ENDL; + LL_WARNS(LOG_INV) << "Inventory is broken." << LL_ENDL; return mask; } @@ -883,7 +919,7 @@ U32 LLInventoryModel::updateItem(const LLViewerInventoryItem* item, U32 mask) } else { - LL_WARNS() << "Couldn't find parent-child item tree for " << new_item->getName() << LL_ENDL; + LL_WARNS(LOG_INV) << "Couldn't find parent-child item tree for " << new_item->getName() << LL_ENDL; } } else @@ -912,8 +948,8 @@ U32 LLInventoryModel::updateItem(const LLViewerInventoryItem* item, U32 mask) else { // Whoops! No such parent, make one. - LL_INFOS() << "Lost item: " << new_item->getUUID() << " - " - << new_item->getName() << LL_ENDL; + LL_INFOS(LOG_INV) << "Lost item: " << new_item->getUUID() << " - " + << new_item->getName() << LL_ENDL; parent_id = findCategoryUUIDForType(LLFolderType::FT_LOST_AND_FOUND); new_item->setParent(parent_id); item_array = get_ptr_in_map(mParentChildItemTree, parent_id); @@ -926,7 +962,7 @@ U32 LLInventoryModel::updateItem(const LLViewerInventoryItem* item, U32 mask) } else { - LL_WARNS() << "Lost and found Not there!!" << LL_ENDL; + LL_WARNS(LOG_INV) << "Lost and found Not there!!" << LL_ENDL; } } } @@ -1000,7 +1036,7 @@ void LLInventoryModel::updateCategory(const LLViewerInventoryCategory* cat, U32 if(!isInventoryUsable()) { - LL_WARNS() << "Inventory is broken." << LL_ENDL; + LL_WARNS(LOG_INV) << "Inventory is broken." << LL_ENDL; return; } @@ -1062,17 +1098,17 @@ void LLInventoryModel::updateCategory(const LLViewerInventoryCategory* cat, U32 void LLInventoryModel::moveObject(const LLUUID& object_id, const LLUUID& cat_id) { - LL_DEBUGS() << "LLInventoryModel::moveObject()" << LL_ENDL; + LL_DEBUGS(LOG_INV) << "LLInventoryModel::moveObject()" << LL_ENDL; if(!isInventoryUsable()) { - LL_WARNS() << "Inventory is broken." << LL_ENDL; + LL_WARNS(LOG_INV) << "Inventory is broken." << LL_ENDL; return; } if((object_id == cat_id) || !is_in_map(mCategoryMap, cat_id)) { - LL_WARNS() << "Could not move inventory object " << object_id << " to " - << cat_id << LL_ENDL; + LL_WARNS(LOG_INV) << "Could not move inventory object " << object_id << " to " + << cat_id << LL_ENDL; return; } LLPointer cat = getCategory(object_id); @@ -1108,14 +1144,14 @@ void LLInventoryModel::changeItemParent(LLViewerInventoryItem* item, { if (item->getParentUUID() == new_parent_id) { - LL_DEBUGS("Inventory") << "'" << item->getName() << "' (" << item->getUUID() - << ") is already in folder " << new_parent_id << LL_ENDL; + LL_DEBUGS(LOG_INV) << "'" << item->getName() << "' (" << item->getUUID() + << ") is already in folder " << new_parent_id << LL_ENDL; } else { - LL_INFOS("Inventory") << "Moving '" << item->getName() << "' (" << item->getUUID() - << ") from " << item->getParentUUID() << " to folder " - << new_parent_id << LL_ENDL; + LL_INFOS(LOG_INV) << "Moving '" << item->getName() << "' (" << item->getUUID() + << ") from " << item->getParentUUID() << " to folder " + << new_parent_id << LL_ENDL; LLInventoryModel::update_list_t update; LLInventoryModel::LLCategoryUpdate old_folder(item->getParentUUID(),-1); update.push_back(old_folder); @@ -1171,7 +1207,7 @@ void LLInventoryModel::onAISUpdateReceived(const std::string& context, const LLS AISUpdate ais_update(update); // parse update llsd into stuff to do. ais_update.doUpdate(); // execute the updates in the appropriate order. - LL_INFOS() << "elapsed: " << timer.getElapsedTimeF32() << LL_ENDL; + LL_INFOS(LOG_INV) << "elapsed: " << timer.getElapsedTimeF32() << LL_ENDL; } // Does not appear to be used currently. @@ -1180,7 +1216,7 @@ void LLInventoryModel::onItemUpdated(const LLUUID& item_id, const LLSD& updates, U32 mask = LLInventoryObserver::NONE; LLPointer item = gInventory.getItem(item_id); - LL_DEBUGS("Inventory") << "item_id: [" << item_id << "] name " << (item ? item->getName() : "(NOT FOUND)") << LL_ENDL; + LL_DEBUGS(LOG_INV) << "item_id: [" << item_id << "] name " << (item ? item->getName() : "(NOT FOUND)") << LL_ENDL; if(item) { for (LLSD::map_const_iterator it = updates.beginMap(); @@ -1188,19 +1224,19 @@ void LLInventoryModel::onItemUpdated(const LLUUID& item_id, const LLSD& updates, { if (it->first == "name") { - LL_INFOS() << "Updating name from " << item->getName() << " to " << it->second.asString() << LL_ENDL; + LL_INFOS(LOG_INV) << "Updating name from " << item->getName() << " to " << it->second.asString() << LL_ENDL; item->rename(it->second.asString()); mask |= LLInventoryObserver::LABEL; } else if (it->first == "desc") { - LL_INFOS() << "Updating description from " << item->getActualDescription() - << " to " << it->second.asString() << LL_ENDL; + LL_INFOS(LOG_INV) << "Updating description from " << item->getActualDescription() + << " to " << it->second.asString() << LL_ENDL; item->setDescription(it->second.asString()); } else { - LL_ERRS() << "unhandled updates for field: " << it->first << LL_ENDL; + LL_ERRS(LOG_INV) << "unhandled updates for field: " << it->first << LL_ENDL; } } mask |= LLInventoryObserver::INTERNAL; @@ -1211,7 +1247,7 @@ void LLInventoryModel::onItemUpdated(const LLUUID& item_id, const LLSD& updates, LLInventoryModel::LLCategoryUpdate up(item->getParentUUID(), 0); accountForUpdate(up); } - gInventory.notifyObservers(); // do we want to be able to make this optional? + notifyObservers(); // do we want to be able to make this optional? } } @@ -1221,7 +1257,7 @@ void LLInventoryModel::onCategoryUpdated(const LLUUID& cat_id, const LLSD& updat U32 mask = LLInventoryObserver::NONE; LLPointer cat = gInventory.getCategory(cat_id); - LL_DEBUGS("Inventory") << "cat_id: [" << cat_id << "] name " << (cat ? cat->getName() : "(NOT FOUND)") << LL_ENDL; + LL_DEBUGS(LOG_INV) << "cat_id: [" << cat_id << "] name " << (cat ? cat->getName() : "(NOT FOUND)") << LL_ENDL; if(cat) { for (LLSD::map_const_iterator it = updates.beginMap(); @@ -1229,18 +1265,18 @@ void LLInventoryModel::onCategoryUpdated(const LLUUID& cat_id, const LLSD& updat { if (it->first == "name") { - LL_INFOS() << "Updating name from " << cat->getName() << " to " << it->second.asString() << LL_ENDL; + LL_INFOS(LOG_INV) << "Updating name from " << cat->getName() << " to " << it->second.asString() << LL_ENDL; cat->rename(it->second.asString()); mask |= LLInventoryObserver::LABEL; } else { - LL_ERRS() << "unhandled updates for field: " << it->first << LL_ENDL; + LL_ERRS(LOG_INV) << "unhandled updates for field: " << it->first << LL_ENDL; } } mask |= LLInventoryObserver::INTERNAL; addChangedMask(mask, cat->getUUID()); - gInventory.notifyObservers(); // do we want to be able to make this optional? + notifyObservers(); // do we want to be able to make this optional? } } @@ -1317,8 +1353,8 @@ void LLInventoryModel::onDescendentsPurgedFromServer(const LLUUID& object_id, bo while (deleted_count > 0); if (total_deleted_count != count) { - LL_WARNS() << "Unexpected count of categories deleted, got " - << total_deleted_count << " expected " << count << LL_ENDL; + LL_WARNS(LOG_INV) << "Unexpected count of categories deleted, got " + << total_deleted_count << " expected " << count << LL_ENDL; } //gInventory.validate(); } @@ -1355,15 +1391,15 @@ void LLInventoryModel::onObjectDeletedFromServer(const LLUUID& object_id, bool f // Delete a particular inventory object by ID. void LLInventoryModel::deleteObject(const LLUUID& id, bool fix_broken_links, bool do_notify_observers) { - LL_DEBUGS() << "LLInventoryModel::deleteObject()" << LL_ENDL; + LL_DEBUGS(LOG_INV) << "LLInventoryModel::deleteObject()" << LL_ENDL; LLPointer obj = getObject(id); if (!obj) { - LL_WARNS() << "Deleting non-existent object [ id: " << id << " ] " << LL_ENDL; + LL_WARNS(LOG_INV) << "Deleting non-existent object [ id: " << id << " ] " << LL_ENDL; return; } - LL_DEBUGS() << "Deleting inventory object " << id << LL_ENDL; + LL_DEBUGS(LOG_INV) << "Deleting inventory object " << id << LL_ENDL; mLastItem = NULL; LLUUID parent_id = obj->getParentUUID(); mCategoryMap.erase(id); @@ -1386,7 +1422,7 @@ void LLInventoryModel::deleteObject(const LLUUID& id, bool fix_broken_links, boo { if (item_list->size()) { - LL_WARNS() << "Deleting cat " << id << " while it still has child items" << LL_ENDL; + LL_WARNS(LOG_INV) << "Deleting cat " << id << " while it still has child items" << LL_ENDL; } delete item_list; mParentChildItemTree.erase(id); @@ -1396,7 +1432,7 @@ void LLInventoryModel::deleteObject(const LLUUID& id, bool fix_broken_links, boo { if (cat_list->size()) { - LL_WARNS() << "Deleting cat " << id << " while it still has child cats" << LL_ENDL; + LL_WARNS(LOG_INV) << "Deleting cat " << id << " while it still has child cats" << LL_ENDL; } delete cat_list; mParentChildCategoryTree.erase(id); @@ -1431,7 +1467,7 @@ void LLInventoryModel::updateLinkedObjectsFromPurge(const LLUUID &baseobj_id) // everything else on the changelist will also get rebuilt. if (item_array.size() > 0) { - gInventory.notifyObservers(); + notifyObservers(); for (LLInventoryModel::item_array_t::const_iterator iter = item_array.begin(); iter != item_array.end(); iter++) @@ -1441,7 +1477,7 @@ void LLInventoryModel::updateLinkedObjectsFromPurge(const LLUUID &baseobj_id) if (item_id == baseobj_id) continue; addChangedMask(LLInventoryObserver::REBUILD, item_id); } - gInventory.notifyObservers(); + notifyObservers(); } } @@ -1464,6 +1500,9 @@ BOOL LLInventoryModel::containsObserver(LLInventoryObserver* observer) const void LLInventoryModel::idleNotifyObservers() { + // *FIX: Think I want this conditional or moved elsewhere... + handleResponses(true); + if (mModifyMask == LLInventoryObserver::NONE && (mChangedItemIDs.size() == 0)) { return; @@ -1479,7 +1518,7 @@ void LLInventoryModel::notifyObservers() // Within notifyObservers, something called notifyObservers // again. This type of recursion is unsafe because it causes items to be // processed twice, and this can easily lead to infinite loops. - LL_WARNS() << "Call was made to notifyObservers within notifyObservers!" << LL_ENDL; + LL_WARNS(LOG_INV) << "Call was made to notifyObservers within notifyObservers!" << LL_ENDL; return; } @@ -1509,18 +1548,18 @@ void LLInventoryModel::addChangedMask(U32 mask, const LLUUID& referent) // Something marked an item for change within a call to notifyObservers // (which is in the process of processing the list of items marked for change). // This means the change may fail to be processed. - LL_WARNS() << "Adding changed mask within notify observers! Change will likely be lost." << LL_ENDL; + LL_WARNS(LOG_INV) << "Adding changed mask within notify observers! Change will likely be lost." << LL_ENDL; LLViewerInventoryItem *item = getItem(referent); if (item) { - LL_WARNS() << "Item " << item->getName() << LL_ENDL; + LL_WARNS(LOG_INV) << "Item " << item->getName() << LL_ENDL; } else { LLViewerInventoryCategory *cat = getCategory(referent); if (cat) { - LL_WARNS() << "Category " << cat->getName() << LL_ENDL; + LL_WARNS(LOG_INV) << "Category " << cat->getName() << LL_ENDL; } } } @@ -1544,91 +1583,18 @@ void LLInventoryModel::addChangedMask(U32 mask, const LLUUID& referent) } } -// If we get back a normal response, handle it here -void LLInventoryModel::fetchInventoryResponder::httpSuccess() -{ - const LLSD& content = getContent(); - if (!content.isMap()) - { - failureResult(HTTP_INTERNAL_ERROR, "Malformed response contents", content); - return; - } - start_new_inventory_observer(); - - /*LLUUID agent_id; - agent_id = content["agent_id"].asUUID(); - if(agent_id != gAgent.getID()) - { - LL_WARNS() << "Got a inventory update for the wrong agent: " << agent_id - << LL_ENDL; - return; - }*/ - item_array_t items; - update_map_t update; - S32 count = content["items"].size(); - LLUUID folder_id; - // Does this loop ever execute more than once? - for(S32 i = 0; i < count; ++i) - { - LLPointer titem = new LLViewerInventoryItem; - titem->unpackMessage(content["items"][i]); - - LL_DEBUGS() << "LLInventoryModel::fetchInventoryResponder item id: " - << titem->getUUID() << LL_ENDL; - items.push_back(titem); - // examine update for changes. - LLViewerInventoryItem* itemp = gInventory.getItem(titem->getUUID()); - if(itemp) - { - if(titem->getParentUUID() == itemp->getParentUUID()) - { - update[titem->getParentUUID()]; - } - else - { - ++update[titem->getParentUUID()]; - --update[itemp->getParentUUID()]; - } - } - else - { - ++update[titem->getParentUUID()]; - } - if (folder_id.isNull()) - { - folder_id = titem->getParentUUID(); - } - } - - U32 changes = 0x0; - //as above, this loop never seems to loop more than once per call - for (item_array_t::iterator it = items.begin(); it != items.end(); ++it) - { - changes |= gInventory.updateItem(*it); - } - gInventory.notifyObservers(); - gViewerWindow->getWindow()->decBusyCount(); -} - -//If we get back an error (not found, etc...), handle it here -void LLInventoryModel::fetchInventoryResponder::httpFailure() -{ - LL_WARNS() << dumpResponse() << LL_ENDL; - gInventory.notifyObservers(); -} - bool LLInventoryModel::fetchDescendentsOf(const LLUUID& folder_id) const { if(folder_id.isNull()) { - LL_WARNS() << "Calling fetch descendents on NULL folder id!" << LL_ENDL; + LL_WARNS(LOG_INV) << "Calling fetch descendents on NULL folder id!" << LL_ENDL; return false; } LLViewerInventoryCategory* cat = getCategory(folder_id); if(!cat) { - LL_WARNS() << "Asked to fetch descendents of non-existent folder: " - << folder_id << LL_ENDL; + LL_WARNS(LOG_INV) << "Asked to fetch descendents of non-existent folder: " + << folder_id << LL_ENDL; return false; } //S32 known_descendents = 0; @@ -1649,8 +1615,8 @@ void LLInventoryModel::cache( const LLUUID& parent_folder_id, const LLUUID& agent_id) { - LL_DEBUGS() << "Caching " << parent_folder_id << " for " << agent_id - << LL_ENDL; + LL_DEBUGS(LOG_INV) << "Caching " << parent_folder_id << " for " << agent_id + << LL_ENDL; LLViewerInventoryCategory* root_cat = getCategory(parent_folder_id); if(!root_cat) return; cat_array_t categories; @@ -1675,19 +1641,19 @@ void LLInventoryModel::cache( gzip_filename.append(".gz"); if(gzip_file(inventory_filename, gzip_filename)) { - LL_DEBUGS() << "Successfully compressed " << inventory_filename << LL_ENDL; + LL_DEBUGS(LOG_INV) << "Successfully compressed " << inventory_filename << LL_ENDL; LLFile::remove(inventory_filename); } else { - LL_WARNS() << "Unable to compress " << inventory_filename << LL_ENDL; + LL_WARNS(LOG_INV) << "Unable to compress " << inventory_filename << LL_ENDL; } } void LLInventoryModel::addCategory(LLViewerInventoryCategory* category) { - //LL_INFOS() << "LLInventoryModel::addCategory()" << LL_ENDL; + //LL_INFOS(LOG_INV) << "LLInventoryModel::addCategory()" << LL_ENDL; if(category) { // We aren't displaying the Meshes folder @@ -1757,7 +1723,9 @@ void LLInventoryModel::addItem(LLViewerInventoryItem* item) if ((item->getType() == LLAssetType::AT_NONE) || LLAssetType::lookup(item->getType()) == LLAssetType::badLookup()) { - LL_WARNS() << "Got bad asset type for item [ name: " << item->getName() << " type: " << item->getType() << " inv-type: " << item->getInventoryType() << " ], ignoring." << LL_ENDL; + LL_WARNS(LOG_INV) << "Got bad asset type for item [ name: " << item->getName() + << " type: " << item->getType() + << " inv-type: " << item->getInventoryType() << " ], ignoring." << LL_ENDL; return; } @@ -1765,7 +1733,9 @@ void LLInventoryModel::addItem(LLViewerInventoryItem* item) // The item will show up as a broken link. if (item->getIsBrokenLink()) { - LL_INFOS() << "Adding broken link [ name: " << item->getName() << " itemID: " << item->getUUID() << " assetID: " << item->getAssetUUID() << " ) parent: " << item->getParentUUID() << LL_ENDL; + LL_INFOS(LOG_INV) << "Adding broken link [ name: " << item->getName() + << " itemID: " << item->getUUID() + << " assetID: " << item->getAssetUUID() << " ) parent: " << item->getParentUUID() << LL_ENDL; } if (item->getIsLinkType()) { @@ -1781,7 +1751,7 @@ void LLInventoryModel::addItem(LLViewerInventoryItem* item) // Empty the entire contents void LLInventoryModel::empty() { -// LL_INFOS() << "LLInventoryModel::empty()" << LL_ENDL; +// LL_INFOS(LOG_INV) << "LLInventoryModel::empty()" << LL_ENDL; std::for_each( mParentChildCategoryTree.begin(), mParentChildCategoryTree.end(), @@ -1814,29 +1784,29 @@ void LLInventoryModel::accountForUpdate(const LLCategoryUpdate& update) const descendents_actual += update.mDescendentDelta; cat->setDescendentCount(descendents_actual); cat->setVersion(++version); - LL_DEBUGS("Inventory") << "accounted: '" << cat->getName() << "' " - << version << " with " << descendents_actual - << " descendents." << LL_ENDL; + LL_DEBUGS(LOG_INV) << "accounted: '" << cat->getName() << "' " + << version << " with " << descendents_actual + << " descendents." << LL_ENDL; } else { // Error condition, this means that the category did not register that // it got new descendents (perhaps because it is still being loaded) // which means its descendent count will be wrong. - LL_WARNS() << "Accounting failed for '" << cat->getName() << "' version:" - << version << " due to mismatched descendent count: server == " - << descendents_server << ", viewer == " << descendents_actual << LL_ENDL; + LL_WARNS(LOG_INV) << "Accounting failed for '" << cat->getName() << "' version:" + << version << " due to mismatched descendent count: server == " + << descendents_server << ", viewer == " << descendents_actual << LL_ENDL; } } else { - LL_WARNS() << "Accounting failed for '" << cat->getName() << "' version: unknown (" - << version << ")" << LL_ENDL; + LL_WARNS(LOG_INV) << "Accounting failed for '" << cat->getName() << "' version: unknown (" + << version << ")" << LL_ENDL; } } else { - LL_WARNS() << "No category found for update " << update.mCategoryID << LL_ENDL; + LL_WARNS(LOG_INV) << "No category found for update " << update.mCategoryID << LL_ENDL; } } @@ -1918,7 +1888,7 @@ bool LLInventoryModel::loadSkeleton( const LLSD& options, const LLUUID& owner_id) { - LL_DEBUGS() << "importing inventory skeleton for " << owner_id << LL_ENDL; + LL_DEBUGS(LOG_INV) << "importing inventory skeleton for " << owner_id << LL_ENDL; typedef std::set, InventoryIDPtrLess> cat_set_t; cat_set_t temp_cats; @@ -1955,7 +1925,7 @@ bool LLInventoryModel::loadSkeleton( } else { - LL_WARNS() << "Unable to import near " << name.asString() << LL_ENDL; + LL_WARNS(LOG_INV) << "Unable to import near " << name.asString() << LL_ENDL; rv = false; } } @@ -1992,7 +1962,7 @@ bool LLInventoryModel::loadSkeleton( } else { - LL_INFOS() << "Unable to gunzip " << gzip_filename << LL_ENDL; + LL_INFOS(LOG_INV) << "Unable to gunzip " << gzip_filename << LL_ENDL; } } bool is_cache_obsolete = false; @@ -2073,10 +2043,10 @@ bool LLInventoryModel::loadSkeleton( if (item->getIsBrokenLink()) { //bad_link_count++; - LL_DEBUGS() << "Attempted to add cached link item without baseobj present ( name: " - << item->getName() << " itemID: " << item->getUUID() - << " assetID: " << item->getAssetUUID() - << " ). Ignoring and invalidating " << cat->getName() << " . " << LL_ENDL; + LL_DEBUGS(LOG_INV) << "Attempted to add cached link item without baseobj present ( name: " + << item->getName() << " itemID: " << item->getUUID() + << " assetID: " << item->getAssetUUID() + << " ). Ignoring and invalidating " << cat->getName() << " . " << LL_ENDL; possible_broken_links.push_back(item); continue; } @@ -2103,7 +2073,7 @@ bool LLInventoryModel::loadSkeleton( { bad_link_count++; invalid_categories.insert(cit->second); - //LL_INFOS() << "link still broken: " << item->getName() << " in folder " << cat->getName() << LL_ENDL; + //LL_INFOS(LOG_INV) << "link still broken: " << item->getName() << " in folder " << cat->getName() << LL_ENDL; } else { @@ -2115,11 +2085,11 @@ bool LLInventoryModel::loadSkeleton( } } - LL_INFOS() << "Attempted to add " << bad_link_count - << " cached link items without baseobj present. " - << good_link_count << " link items were successfully added. " - << recovered_link_count << " links added in recovery. " - << "The corresponding categories were invalidated." << LL_ENDL; + LL_INFOS(LOG_INV) << "Attempted to add " << bad_link_count + << " cached link items without baseobj present. " + << good_link_count << " link items were successfully added. " + << recovered_link_count << " links added in recovery. " + << "The corresponding categories were invalidated." << LL_ENDL; } } @@ -2143,9 +2113,9 @@ bool LLInventoryModel::loadSkeleton( { LLViewerInventoryCategory* cat = (*invalid_cat_it).get(); cat->setVersion(NO_VERSION); - LL_DEBUGS("Inventory") << "Invalidating category name: " << cat->getName() << " UUID: " << cat->getUUID() << " due to invalid descendents cache" << LL_ENDL; + LL_DEBUGS(LOG_INV) << "Invalidating category name: " << cat->getName() << " UUID: " << cat->getUUID() << " due to invalid descendents cache" << LL_ENDL; } - LL_INFOS("Inventory") << "Invalidated " << invalid_categories.size() << " categories due to invalid descendents cache" << LL_ENDL; + LL_INFOS(LOG_INV) << "Invalidated " << invalid_categories.size() << " categories due to invalid descendents cache" << LL_ENDL; // At this point, we need to set the known descendents for each // category which successfully cached so that we do not @@ -2177,15 +2147,15 @@ bool LLInventoryModel::loadSkeleton( if(is_cache_obsolete) { // If out of date, remove the gzipped file too. - LL_WARNS() << "Inv cache out of date, removing" << LL_ENDL; + LL_WARNS(LOG_INV) << "Inv cache out of date, removing" << LL_ENDL; LLFile::remove(gzip_filename); } categories.clear(); // will unref and delete entries } - LL_INFOS() << "Successfully loaded " << cached_category_count - << " categories and " << cached_item_count << " items from cache." - << LL_ENDL; + LL_INFOS(LOG_INV) << "Successfully loaded " << cached_category_count + << " categories and " << cached_item_count << " items from cache." + << LL_ENDL; return rv; } @@ -2195,7 +2165,7 @@ bool LLInventoryModel::loadSkeleton( // should be sufficient for our needs. void LLInventoryModel::buildParentChildMap() { - LL_INFOS() << "LLInventoryModel::buildParentChildMap()" << LL_ENDL; + LL_INFOS(LOG_INV) << "LLInventoryModel::buildParentChildMap()" << LL_ENDL; // *NOTE: I am skipping the logic around folder version // synchronization here because it seems if a folder is lost, we @@ -2264,15 +2234,15 @@ void LLInventoryModel::buildParentChildMap() // implement it, we would need a set or map of uuid pairs // which would be (folder_id, new_parent_id) to be sent up // to the server. - LL_INFOS() << "Lost category: " << cat->getUUID() << " - " - << cat->getName() << LL_ENDL; + LL_INFOS(LOG_INV) << "Lost category: " << cat->getUUID() << " - " + << cat->getName() << LL_ENDL; ++lost; lost_cats.push_back(cat); } } if(lost) { - LL_WARNS() << "Found " << lost << " lost categories." << LL_ENDL; + LL_WARNS(LOG_INV) << "Found " << lost << " lost categories." << LL_ENDL; } // Do moves in a separate pass to make sure we've properly filed @@ -2307,7 +2277,7 @@ void LLInventoryModel::buildParentChildMap() } else { - LL_WARNS() << "Lost and found Not there!!" << LL_ENDL; + LL_WARNS(LOG_INV) << "Lost and found Not there!!" << LL_ENDL; } } @@ -2342,8 +2312,8 @@ void LLInventoryModel::buildParentChildMap() } else { - LL_INFOS() << "Lost item: " << item->getUUID() << " - " - << item->getName() << LL_ENDL; + LL_INFOS(LOG_INV) << "Lost item: " << item->getUUID() << " - " + << item->getName() << LL_ENDL; ++lost; // plop it into the lost & found. // @@ -2359,13 +2329,13 @@ void LLInventoryModel::buildParentChildMap() } else { - LL_WARNS() << "Lost and found Not there!!" << LL_ENDL; + LL_WARNS(LOG_INV) << "Lost and found Not there!!" << LL_ENDL; } } } if(lost) { - LL_WARNS() << "Found " << lost << " lost items." << LL_ENDL; + LL_WARNS(LOG_INV) << "Found " << lost << " lost items." << LL_ENDL; LLMessageSystem* msg = gMessageSystem; BOOL start_new_message = TRUE; const LLUUID lnf = findCategoryUUIDForType(LLFolderType::FT_LOST_AND_FOUND); @@ -2443,10 +2413,85 @@ void LLInventoryModel::buildParentChildMap() if (!gInventory.validate()) { - LL_WARNS() << "model failed validity check!" << LL_ENDL; + LL_WARNS(LOG_INV) << "model failed validity check!" << LL_ENDL; + } +} + +// Would normally do this at construction but that's too early +// in the process for gInventory. Have the first requestPost() +// call set things up. +void LLInventoryModel::initHttpRequest() +{ + if (! mHttpRequestFG) + { + // Haven't initialized, get to it + LLAppCoreHttp & app_core_http(LLAppViewer::instance()->getAppCoreHttp()); + + mHttpRequestFG = new LLCore::HttpRequest; + mHttpRequestBG = new LLCore::HttpRequest; + mHttpOptions = new LLCore::HttpOptions; + mHttpOptions->setTransferTimeout(300); + mHttpOptions->setUseRetryAfter(true); + // mHttpOptions->setTrace(2); // Do tracing of requests + mHttpHeaders = new LLCore::HttpHeaders; + mHttpHeaders->append(HTTP_OUT_HEADER_CONTENT_TYPE, HTTP_CONTENT_LLSD_XML); + mHttpHeaders->append(HTTP_OUT_HEADER_ACCEPT, HTTP_CONTENT_LLSD_XML); + mHttpPolicyClass = app_core_http.getPolicy(LLAppCoreHttp::AP_INVENTORY); } } +void LLInventoryModel::handleResponses(bool foreground) +{ + if (foreground) + { + mHttpRequestFG->update(0); + } + else + { + mHttpRequestBG->update(50000L); + } +} + +LLCore::HttpHandle LLInventoryModel::requestPost(bool foreground, + const std::string & url, + const LLSD & body, + LLCore::HttpHandler * handler, + const char * const message) +{ + if (! mHttpRequestFG) + { + // We do the initialization late and lazily as this class is + // statically-constructed and not all the bits are ready at + // that time. + initHttpRequest(); + } + + LLCore::HttpRequest * request(foreground ? mHttpRequestFG : mHttpRequestBG); + LLCore::HttpHandle handle(LLCORE_HTTP_HANDLE_INVALID); + LLCore::BufferArray * ba = new LLCore::BufferArray; + LLCore::BufferArrayStream bas(ba); + LLSDSerialize::toXML(body, bas); + + handle = request->requestPost(mHttpPolicyClass, + (foreground ? mHttpPriorityFG : mHttpPriorityBG), + url, + ba, + mHttpOptions, + mHttpHeaders, + handler); + ba->release(); + if (LLCORE_HTTP_HANDLE_INVALID == handle) + { + LLCore::HttpStatus status(request->getStatus()); + LL_WARNS(LOG_INV) << "HTTP POST request failed for " << message + << ", Status: " << status.toTerseString() + << " Reason: '" << status.toString() << "'" + << LL_ENDL; + delete handler; + } + return handle; +} + void LLInventoryModel::createCommonSystemCategories() { gInventory.findCategoryUUIDForType(LLFolderType::FT_TRASH,true); @@ -2495,14 +2540,14 @@ bool LLInventoryModel::loadFromFile(const std::string& filename, { if(filename.empty()) { - LL_ERRS() << "Filename is Null!" << LL_ENDL; + LL_ERRS(LOG_INV) << "Filename is Null!" << LL_ENDL; return false; } - LL_INFOS() << "LLInventoryModel::loadFromFile(" << filename << ")" << LL_ENDL; + LL_INFOS(LOG_INV) << "LLInventoryModel::loadFromFile(" << filename << ")" << LL_ENDL; LLFILE* file = LLFile::fopen(filename, "rb"); /*Flawfinder: ignore*/ if(!file) { - LL_INFOS() << "unable to load inventory from: " << filename << LL_ENDL; + LL_INFOS(LOG_INV) << "unable to load inventory from: " << filename << LL_ENDL; return false; } // *NOTE: This buffer size is hard coded into scanf() below. @@ -2541,7 +2586,7 @@ bool LLInventoryModel::loadFromFile(const std::string& filename, } else { - LL_WARNS() << "loadInventoryFromFile(). Ignoring invalid inventory category: " << inv_cat->getName() << LL_ENDL; + LL_WARNS(LOG_INV) << "loadInventoryFromFile(). Ignoring invalid inventory category: " << inv_cat->getName() << LL_ENDL; //delete inv_cat; // automatic when inv_cat is reassigned or destroyed } } @@ -2559,8 +2604,8 @@ bool LLInventoryModel::loadFromFile(const std::string& filename, if(inv_item->getUUID().isNull()) { //delete inv_item; // automatic when inv_cat is reassigned or destroyed - LL_WARNS() << "Ignoring inventory with null item id: " - << inv_item->getName() << LL_ENDL; + LL_WARNS(LOG_INV) << "Ignoring inventory with null item id: " + << inv_item->getName() << LL_ENDL; } else @@ -2570,14 +2615,14 @@ bool LLInventoryModel::loadFromFile(const std::string& filename, } else { - LL_WARNS() << "loadInventoryFromFile(). Ignoring invalid inventory item: " << inv_item->getName() << LL_ENDL; + LL_WARNS(LOG_INV) << "loadInventoryFromFile(). Ignoring invalid inventory item: " << inv_item->getName() << LL_ENDL; //delete inv_item; // automatic when inv_cat is reassigned or destroyed } } else { - LL_WARNS() << "Unknown token in inventory file '" << keyword << "'" - << LL_ENDL; + LL_WARNS(LOG_INV) << "Unknown token in inventory file '" << keyword << "'" + << LL_ENDL; } } fclose(file); @@ -2593,14 +2638,14 @@ bool LLInventoryModel::saveToFile(const std::string& filename, { if(filename.empty()) { - LL_ERRS() << "Filename is Null!" << LL_ENDL; + LL_ERRS(LOG_INV) << "Filename is Null!" << LL_ENDL; return false; } - LL_INFOS() << "LLInventoryModel::saveToFile(" << filename << ")" << LL_ENDL; + LL_INFOS(LOG_INV) << "LLInventoryModel::saveToFile(" << filename << ")" << LL_ENDL; LLFILE* file = LLFile::fopen(filename, "wb"); /*Flawfinder: ignore*/ if(!file) { - LL_WARNS() << "unable to save inventory to: " << filename << LL_ENDL; + LL_WARNS(LOG_INV) << "unable to save inventory to: " << filename << LL_ENDL; return false; } @@ -2705,8 +2750,8 @@ bool LLInventoryModel::messageUpdateCore(LLMessageSystem* msg, bool account, U32 msg->getUUIDFast(_PREHASH_AgentData, _PREHASH_AgentID, agent_id); if(agent_id != gAgent.getID()) { - LL_WARNS() << "Got a inventory update for the wrong agent: " << agent_id - << LL_ENDL; + LL_WARNS(LOG_INV) << "Got a inventory update for the wrong agent: " << agent_id + << LL_ENDL; return false; } item_array_t items; @@ -2718,8 +2763,8 @@ bool LLInventoryModel::messageUpdateCore(LLMessageSystem* msg, bool account, U32 { LLPointer titem = new LLViewerInventoryItem; titem->unpackMessage(msg, _PREHASH_InventoryData, i); - LL_DEBUGS() << "LLInventoryModel::messageUpdateCore() item id:" - << titem->getUUID() << LL_ENDL; + LL_DEBUGS(LOG_INV) << "LLInventoryModel::messageUpdateCore() item id: " + << titem->getUUID() << LL_ENDL; items.push_back(titem); // examine update for changes. LLViewerInventoryItem* itemp = gInventory.getItem(titem->getUUID()); @@ -2770,17 +2815,17 @@ void LLInventoryModel::removeInventoryItem(LLUUID agent_id, LLMessageSystem* msg { LLUUID item_id; S32 count = msg->getNumberOfBlocksFast(msg_label); - LL_DEBUGS() << "Message has " << count << " item blocks" << LL_ENDL; + LL_DEBUGS(LOG_INV) << "Message has " << count << " item blocks" << LL_ENDL; uuid_vec_t item_ids; update_map_t update; for(S32 i = 0; i < count; ++i) { msg->getUUIDFast(msg_label, _PREHASH_ItemID, item_id, i); - LL_DEBUGS() << "Checking for item-to-be-removed " << item_id << LL_ENDL; + LL_DEBUGS(LOG_INV) << "Checking for item-to-be-removed " << item_id << LL_ENDL; LLViewerInventoryItem* itemp = gInventory.getItem(item_id); if(itemp) { - LL_DEBUGS() << "Item will be removed " << item_id << LL_ENDL; + LL_DEBUGS(LOG_INV) << "Item will be removed " << item_id << LL_ENDL; // we only bother with the delete and account if we found // the item - this is usually a back-up for permissions, // so frequently the item will already be gone. @@ -2791,7 +2836,7 @@ void LLInventoryModel::removeInventoryItem(LLUUID agent_id, LLMessageSystem* msg gInventory.accountForUpdate(update); for(uuid_vec_t::iterator it = item_ids.begin(); it != item_ids.end(); ++it) { - LL_DEBUGS() << "Calling deleteObject " << *it << LL_ENDL; + LL_DEBUGS(LOG_INV) << "Calling deleteObject " << *it << LL_ENDL; gInventory.deleteObject(*it); } } @@ -2799,13 +2844,13 @@ void LLInventoryModel::removeInventoryItem(LLUUID agent_id, LLMessageSystem* msg // static void LLInventoryModel::processRemoveInventoryItem(LLMessageSystem* msg, void**) { - LL_DEBUGS() << "LLInventoryModel::processRemoveInventoryItem()" << LL_ENDL; + LL_DEBUGS(LOG_INV) << "LLInventoryModel::processRemoveInventoryItem()" << LL_ENDL; LLUUID agent_id, item_id; msg->getUUIDFast(_PREHASH_AgentData, _PREHASH_AgentID, agent_id); if(agent_id != gAgent.getID()) { - LL_WARNS() << "Got a RemoveInventoryItem for the wrong agent." - << LL_ENDL; + LL_WARNS(LOG_INV) << "Got a RemoveInventoryItem for the wrong agent." + << LL_ENDL; return; } LLInventoryModel::removeInventoryItem(agent_id, msg, _PREHASH_InventoryData); @@ -2816,14 +2861,14 @@ void LLInventoryModel::processRemoveInventoryItem(LLMessageSystem* msg, void**) void LLInventoryModel::processUpdateInventoryFolder(LLMessageSystem* msg, void**) { - LL_DEBUGS() << "LLInventoryModel::processUpdateInventoryFolder()" << LL_ENDL; + LL_DEBUGS(LOG_INV) << "LLInventoryModel::processUpdateInventoryFolder()" << LL_ENDL; LLUUID agent_id, folder_id, parent_id; //char name[DB_INV_ITEM_NAME_BUF_SIZE]; msg->getUUIDFast(_PREHASH_FolderData, _PREHASH_AgentID, agent_id); if(agent_id != gAgent.getID()) { - LL_WARNS() << "Got an UpdateInventoryFolder for the wrong agent." - << LL_ENDL; + LL_WARNS(LOG_INV) << "Got an UpdateInventoryFolder for the wrong agent." + << LL_ENDL; return; } LLPointer lastfolder; // hack @@ -3881,7 +3926,7 @@ bool LLInventoryModel::validate() const ///---------------------------------------------------------------------------- -/* +#if 0 BOOL decompress_file(const char* src_filename, const char* dst_filename) { BOOL rv = FALSE; @@ -3920,4 +3965,140 @@ BOOL decompress_file(const char* src_filename, const char* dst_filename) if(dst != NULL) fclose(dst); return rv; } -*/ +#endif + +// ==== FetchItemHttpHandler ==== + +LLInventoryModel::FetchItemHttpHandler::FetchItemHttpHandler(const LLSD & request_sd) + : LLCore::HttpHandler(), + mRequestSD(request_sd) +{} + +LLInventoryModel::FetchItemHttpHandler::~FetchItemHttpHandler() +{} + +void LLInventoryModel::FetchItemHttpHandler::onCompleted(LLCore::HttpHandle handle, + LLCore::HttpResponse * response) +{ + LLCore::HttpStatus status(response->getStatus()); + if (! status) + { + processFailure(status, response); + } + else + { + LLCore::BufferArray * body(response->getBody()); + if (! body || ! body->size()) + { + LL_WARNS(LOG_INV) << "Missing data in inventory item query." << LL_ENDL; + processFailure("HTTP response for inventory item query missing body", response); + goto only_exit; + } + + LLCore::BufferArrayStream bas(body); + LLSD body_llsd; + S32 parse_status(LLSDSerialize::fromXML(body_llsd, bas)); + if (LLSDParser::PARSE_FAILURE == parse_status) + { + // INFOS-level logging will occur on the parsed failure + processFailure("HTTP response for inventory item query has malformed LLSD", response); + goto only_exit; + } + + // Okay, process data if possible + processData(body_llsd, response); + } + +only_exit: + // Must delete on completion. + delete this; +} + +void LLInventoryModel::FetchItemHttpHandler::processData(LLSD & content, LLCore::HttpResponse * response) +{ + if (! content.isMap()) + { + processFailure("LLSD response for inventory item not a map", response); + return; + } + + start_new_inventory_observer(); + +#if 0 + LLUUID agent_id; + agent_id = content["agent_id"].asUUID(); + if (agent_id != gAgent.getID()) + { + LL_WARNS(LOG_INV) << "Got a inventory update for the wrong agent: " << agent_id + << LL_ENDL; + return; + } +#endif + + LLInventoryModel::item_array_t items; + LLInventoryModel::update_map_t update; + LLUUID folder_id; + LLSD content_items(content["items"]); + const S32 count(content_items.size()); + + // Does this loop ever execute more than once? + for (S32 i(0); i < count; ++i) + { + LLPointer titem = new LLViewerInventoryItem; + titem->unpackMessage(content_items[i]); + + LL_DEBUGS(LOG_INV) << "ItemHttpHandler::httpSuccess item id: " + << titem->getUUID() << LL_ENDL; + items.push_back(titem); + + // examine update for changes. + LLViewerInventoryItem * itemp(gInventory.getItem(titem->getUUID())); + + if (itemp) + { + if (titem->getParentUUID() == itemp->getParentUUID()) + { + update[titem->getParentUUID()]; + } + else + { + ++update[titem->getParentUUID()]; + --update[itemp->getParentUUID()]; + } + } + else + { + ++update[titem->getParentUUID()]; + } + + if (folder_id.isNull()) + { + folder_id = titem->getParentUUID(); + } + } + + // as above, this loop never seems to loop more than once per call + U32 changes(0U); + for (LLInventoryModel::item_array_t::iterator it = items.begin(); it != items.end(); ++it) + { + changes |= gInventory.updateItem(*it); + } + // *HUH: Have computed changes, nothing uses it. + + gInventory.notifyObservers(); + gViewerWindow->getWindow()->decBusyCount(); +} + + +void LLInventoryModel::FetchItemHttpHandler::processFailure(LLCore::HttpStatus status, LLCore::HttpResponse * response) +{ + LL_WARNS(LOG_INV) << dumpResponse() << LL_ENDL; + gInventory.notifyObservers(); +} + +void LLInventoryModel::FetchItemHttpHandler::processFailure(const char * const reason, LLCore::HttpResponse * response) +{ + LL_WARNS(LOG_INV) << dumpResponse() << LL_ENDL; + gInventory.notifyObservers(); +} + diff --git a/indra/newview/llinventorymodel.h b/indra/newview/llinventorymodel.h index 2e957809be..ac336e347c 100755 --- a/indra/newview/llinventorymodel.h +++ b/indra/newview/llinventorymodel.h @@ -27,6 +27,11 @@ #ifndef LL_LLINVENTORYMODEL_H #define LL_LLINVENTORYMODEL_H +#include +#include +#include +#include + #include "llassettype.h" #include "llfoldertype.h" #include "llframetimer.h" @@ -36,10 +41,11 @@ #include "llviewerinventory.h" #include "llstring.h" #include "llmd5.h" -#include -#include -#include -#include +#include "httpcommon.h" +#include "httprequest.h" +#include "httpoptions.h" +#include "httpheaders.h" +#include "httphandler.h" class LLInventoryObserver; class LLInventoryObject; @@ -60,9 +66,8 @@ class LLInventoryCollectFunctor; class LLInventoryModel { LOG_CLASS(LLInventoryModel); -public: - friend class LLInventoryModelFetchDescendentsResponder; +public: enum EHasChildren { CHILDREN_NO, @@ -74,14 +79,31 @@ public: typedef std::vector > item_array_t; typedef std::set changed_items_t; - class fetchInventoryResponder : public LLCurl::Responder + // HTTP handler for individual item requests (inventory or library). + // Background item requests are derived from this in the background + // inventory system. All folder requests are also located there + // but have their own handler derived from HttpHandler. + class FetchItemHttpHandler : public LLCore::HttpHandler { - LOG_CLASS(fetchInventoryResponder); public: - fetchInventoryResponder(const LLSD& request_sd) : mRequestSD(request_sd) {}; + LOG_CLASS(FetchItemHttpHandler); + + FetchItemHttpHandler(const LLSD & request_sd); + virtual ~FetchItemHttpHandler(); + protected: - virtual void httpSuccess(); - virtual void httpFailure(); + FetchItemHttpHandler(const FetchItemHttpHandler &); // Not defined + void operator=(const FetchItemHttpHandler &); // Not defined + + public: + virtual void onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * response); + + private: + void processData(LLSD & body, LLCore::HttpResponse * response); + void processFailure(LLCore::HttpStatus status, LLCore::HttpResponse * response); + void processFailure(const char * const reason, LLCore::HttpResponse * response); + + private: LLSD mRequestSD; }; @@ -109,6 +131,9 @@ public: private: bool mIsAgentInvUsable; // used to handle an invalid inventory state + // One-time initialization of HTTP system. + void initHttpRequest(); + //-------------------------------------------------------------------- // Root Folders //-------------------------------------------------------------------- @@ -518,6 +543,41 @@ private: *******************************************************************************/ +/******************************************************************************** + ** ** + ** HTTP Transport + **/ +public: + // Invoke handler completion method (onCompleted) for all + // requests that are ready. + void handleResponses(bool foreground); + + // Request an inventory HTTP operation to either the + // foreground or background processor. These are actually + // the same service queue but the background requests are + // seviced more slowly effectively de-prioritizing new + // requests. + LLCore::HttpHandle requestPost(bool foreground, + const std::string & url, + const LLSD & body, + LLCore::HttpHandler * handler, + const char * const message); + +private: + // Usual plumbing for LLCore:: HTTP operations. + LLCore::HttpRequest * mHttpRequestFG; + LLCore::HttpRequest * mHttpRequestBG; + LLCore::HttpOptions * mHttpOptions; + LLCore::HttpHeaders * mHttpHeaders; + LLCore::HttpRequest::policy_t mHttpPolicyClass; + LLCore::HttpRequest::priority_t mHttpPriorityFG; + LLCore::HttpRequest::priority_t mHttpPriorityBG; + +/** HTTP Transport + ** ** + *******************************************************************************/ + + /******************************************************************************** ** ** ** MISCELLANEOUS diff --git a/indra/newview/llinventorymodelbackgroundfetch.cpp b/indra/newview/llinventorymodelbackgroundfetch.cpp index 2de37b0790..443c54df42 100755 --- a/indra/newview/llinventorymodelbackgroundfetch.cpp +++ b/indra/newview/llinventorymodelbackgroundfetch.cpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2002&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2010, Linden Research, Inc. + * Copyright (C) 2014, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -37,31 +37,113 @@ #include "llviewermessage.h" #include "llviewerregion.h" #include "llviewerwindow.h" +#include "llhttpconstants.h" +#include "bufferarray.h" +#include "bufferstream.h" +#include "llsdserialize.h" + +namespace +{ + +// Http request handler class for single inventory item requests. +// +// We'll use a handler-per-request pattern here rather than +// a shared handler. Mainly convenient as this was converted +// from a Responder class model. +// +class BGItemHttpHandler : public LLInventoryModel::FetchItemHttpHandler +{ + LOG_CLASS(BGItemHttpHandler); + +public: + BGItemHttpHandler(const LLSD & request_sd) + : LLInventoryModel::FetchItemHttpHandler(request_sd) + { + LLInventoryModelBackgroundFetch::instance().incrFetchCount(1); + } + + virtual ~BGItemHttpHandler() + { + LLInventoryModelBackgroundFetch::instance().incrFetchCount(-1); + } + +protected: + BGItemHttpHandler(const BGItemHttpHandler &); // Not defined + void operator=(const BGItemHttpHandler &); // Not defined +}; + + +// Http request handler class for folders. +class BGFolderHttpHandler : public LLCore::HttpHandler +{ + LOG_CLASS(BGFolderHttpHandler); + +public: + BGFolderHttpHandler(const LLSD & request_sd, const uuid_vec_t & recursive_cats) + : LLCore::HttpHandler(), + mRequestSD(request_sd), + mRecursiveCatUUIDs(recursive_cats) + { + LLInventoryModelBackgroundFetch::instance().incrFetchCount(1); + } + + virtual ~BGFolderHttpHandler() + { + LLInventoryModelBackgroundFetch::instance().incrFetchCount(-1); + } + +protected: + BGFolderHttpHandler(const BGFolderHttpHandler &); // Not defined + void operator=(const BGFolderHttpHandler &); // Not defined + +public: + virtual void onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * response); + + bool getIsRecursive(const LLUUID & cat_id) const; + +private: + void processData(LLSD & body, LLCore::HttpResponse * response); + void processFailure(LLCore::HttpStatus status, LLCore::HttpResponse * response); + void processFailure(const char * const reason, LLCore::HttpResponse * response); + +private: + LLSD mRequestSD; + const uuid_vec_t mRecursiveCatUUIDs; // hack for storing away which cat fetches are recursive +}; + -const F32 MAX_TIME_FOR_SINGLE_FETCH = 10.f; const S32 MAX_FETCH_RETRIES = 10; -LLInventoryModelBackgroundFetch::LLInventoryModelBackgroundFetch() : +const char * const LOG_INV("Inventory"); + +std::string dumpResponse() +{ + return std::string("ADD SOMETHING MEANINGFUL HERE"); +} + + +} // end of namespace anonymous + + +LLInventoryModelBackgroundFetch::LLInventoryModelBackgroundFetch(): mBackgroundFetchActive(FALSE), mFolderFetchActive(false), + mFetchCount(0), mAllFoldersFetched(FALSE), mRecursiveInventoryFetchStarted(FALSE), mRecursiveLibraryFetchStarted(FALSE), mNumFetchRetries(0), mMinTimeBetweenFetches(0.3f), mMaxTimeBetweenFetches(10.f), - mTimelyFetchPending(FALSE), - mFetchCount(0) -{ -} + mTimelyFetchPending(FALSE) +{} LLInventoryModelBackgroundFetch::~LLInventoryModelBackgroundFetch() -{ -} +{} bool LLInventoryModelBackgroundFetch::isBulkFetchProcessingComplete() const { - return mFetchQueue.empty() && mFetchCount<=0; + return mFetchQueue.empty() && mFetchCount <= 0; } bool LLInventoryModelBackgroundFetch::libraryFetchStarted() const @@ -91,7 +173,7 @@ bool LLInventoryModelBackgroundFetch::inventoryFetchCompleted() const bool LLInventoryModelBackgroundFetch::inventoryFetchInProgress() const { - return inventoryFetchStarted() && !inventoryFetchCompleted(); + return inventoryFetchStarted() && ! inventoryFetchCompleted(); } bool LLInventoryModelBackgroundFetch::isEverythingFetched() const @@ -104,24 +186,36 @@ BOOL LLInventoryModelBackgroundFetch::folderFetchActive() const return mFolderFetchActive; } +void LLInventoryModelBackgroundFetch::addRequestAtFront(const LLUUID & id, BOOL recursive, bool is_category) +{ + mFetchQueue.push_front(FetchQueueInfo(id, recursive, is_category)); +} + +void LLInventoryModelBackgroundFetch::addRequestAtBack(const LLUUID & id, BOOL recursive, bool is_category) +{ + mFetchQueue.push_back(FetchQueueInfo(id, recursive, is_category)); +} + void LLInventoryModelBackgroundFetch::start(const LLUUID& id, BOOL recursive) { - LLViewerInventoryCategory* cat = gInventory.getCategory(id); - if (cat || (id.isNull() && !isEverythingFetched())) - { // it's a folder, do a bulk fetch - LL_DEBUGS("InventoryFetch") << "Start fetching category: " << id << ", recursive: " << recursive << LL_ENDL; + LLViewerInventoryCategory * cat(gInventory.getCategory(id)); + + if (cat || (id.isNull() && ! isEverythingFetched())) + { + // it's a folder, do a bulk fetch + LL_DEBUGS(LOG_INV) << "Start fetching category: " << id << ", recursive: " << recursive << LL_ENDL; mBackgroundFetchActive = TRUE; mFolderFetchActive = true; if (id.isNull()) { - if (!mRecursiveInventoryFetchStarted) + if (! mRecursiveInventoryFetchStarted) { mRecursiveInventoryFetchStarted |= recursive; mFetchQueue.push_back(FetchQueueInfo(gInventory.getRootFolderID(), recursive)); gIdleCallbacks.addFunction(&LLInventoryModelBackgroundFetch::backgroundFetchCB, NULL); } - if (!mRecursiveLibraryFetchStarted) + if (! mRecursiveLibraryFetchStarted) { mRecursiveLibraryFetchStarted |= recursive; mFetchQueue.push_back(FetchQueueInfo(gInventory.getLibraryRootFolderID(), recursive)); @@ -146,9 +240,9 @@ void LLInventoryModelBackgroundFetch::start(const LLUUID& id, BOOL recursive) } } } - else if (LLViewerInventoryItem* itemp = gInventory.getItem(id)) + else if (LLViewerInventoryItem * itemp = gInventory.getItem(id)) { - if (!itemp->mIsComplete && (mFetchQueue.empty() || mFetchQueue.front().mUUID != id)) + if (! itemp->mIsComplete && (mFetchQueue.empty() || mFetchQueue.front().mUUID != id)) { mBackgroundFetchActive = TRUE; @@ -172,7 +266,7 @@ void LLInventoryModelBackgroundFetch::setAllFoldersFetched() mRecursiveLibraryFetchStarted) { mAllFoldersFetched = TRUE; - //LL_INFOS() << "All folders fetched, validating" << LL_ENDL; + //LL_INFOS(LOG_INV) << "All folders fetched, validating" << LL_ENDL; //gInventory.validate(); } mFolderFetchActive = false; @@ -203,7 +297,7 @@ void LLInventoryModelBackgroundFetch::backgroundFetch() // No more categories to fetch, stop fetch process. if (mFetchQueue.empty()) { - LL_INFOS() << "Inventory fetch completed" << LL_ENDL; + LL_INFOS(LOG_INV) << "Inventory fetch completed" << LL_ENDL; setAllFoldersFetched(); mBackgroundFetchActive = false; @@ -219,7 +313,7 @@ void LLInventoryModelBackgroundFetch::backgroundFetch() // Double timeouts on failure. mMinTimeBetweenFetches = llmin(mMinTimeBetweenFetches * 2.f, 10.f); mMaxTimeBetweenFetches = llmin(mMaxTimeBetweenFetches * 2.f, 120.f); - LL_DEBUGS() << "Inventory fetch times grown to (" << mMinTimeBetweenFetches << ", " << mMaxTimeBetweenFetches << ")" << LL_ENDL; + LL_DEBUGS(LOG_INV) << "Inventory fetch times grown to (" << mMinTimeBetweenFetches << ", " << mMaxTimeBetweenFetches << ")" << LL_ENDL; // fetch is no longer considered "timely" although we will wait for full time-out. mTimelyFetchPending = FALSE; } @@ -231,7 +325,7 @@ void LLInventoryModelBackgroundFetch::backgroundFetch() break; } - if(gDisconnected) + if (gDisconnected) { // Just bail if we are disconnected. break; @@ -292,7 +386,7 @@ void LLInventoryModelBackgroundFetch::backgroundFetch() // Shrink timeouts based on success. mMinTimeBetweenFetches = llmax(mMinTimeBetweenFetches * 0.8f, 0.3f); mMaxTimeBetweenFetches = llmax(mMaxTimeBetweenFetches * 0.8f, 10.f); - LL_DEBUGS() << "Inventory fetch times shrunk to (" << mMinTimeBetweenFetches << ", " << mMaxTimeBetweenFetches << ")" << LL_ENDL; + LL_DEBUGS(LOG_INV) << "Inventory fetch times shrunk to (" << mMinTimeBetweenFetches << ", " << mMaxTimeBetweenFetches << ")" << LL_ENDL; } mTimelyFetchPending = FALSE; @@ -355,257 +449,53 @@ void LLInventoryModelBackgroundFetch::backgroundFetch() } } -void LLInventoryModelBackgroundFetch::incrFetchCount(S16 fetching) +void LLInventoryModelBackgroundFetch::incrFetchCount(S32 fetching) { mFetchCount += fetching; if (mFetchCount < 0) { + LL_WARNS_ONCE(LOG_INV) << "Inventory fetch count fell below zero (0)." << LL_ENDL; mFetchCount = 0; } } -class LLInventoryModelFetchItemResponder : public LLInventoryModel::fetchInventoryResponder -{ - LOG_CLASS(LLInventoryModelFetchItemResponder); -public: - LLInventoryModelFetchItemResponder(const LLSD& request_sd) : - LLInventoryModel::fetchInventoryResponder(request_sd) - { - LLInventoryModelBackgroundFetch::instance().incrFetchCount(1); - } -private: - /* virtual */ void httpCompleted() - { - LLInventoryModelBackgroundFetch::instance().incrFetchCount(-1); - LLInventoryModel::fetchInventoryResponder::httpCompleted(); - } -}; - -class LLInventoryModelFetchDescendentsResponder: public LLHTTPClient::Responder -{ - LOG_CLASS(LLInventoryModelFetchDescendentsResponder); -public: - LLInventoryModelFetchDescendentsResponder(const LLSD& request_sd, uuid_vec_t recursive_cats) : - mRequestSD(request_sd), - mRecursiveCatUUIDs(recursive_cats) - { - LLInventoryModelBackgroundFetch::instance().incrFetchCount(1); - } - //LLInventoryModelFetchDescendentsResponder() {}; -private: - /* virtual */ void httpCompleted() - { - LLInventoryModelBackgroundFetch::instance().incrFetchCount(-1); - LLHTTPClient::Responder::httpCompleted(); - } - /* virtual */ void httpSuccess(); - /* virtual */ void httpFailure(); -protected: - BOOL getIsRecursive(const LLUUID& cat_id) const; -private: - LLSD mRequestSD; - uuid_vec_t mRecursiveCatUUIDs; // hack for storing away which cat fetches are recursive -}; - -// If we get back a normal response, handle it here. -void LLInventoryModelFetchDescendentsResponder::httpSuccess() +// Bundle up a bunch of requests to send all at once. +void LLInventoryModelBackgroundFetch::bulkFetch() { - const LLSD& content = getContent(); - if (!content.isMap()) + //Background fetch is called from gIdleCallbacks in a loop until background fetch is stopped. + //If there are items in mFetchQueue, we want to check the time since the last bulkFetch was + //sent. If it exceeds our retry time, go ahead and fire off another batch. + LLViewerRegion * region(gAgent.getRegion()); + if (! region || gDisconnected) { - failureResult(HTTP_INTERNAL_ERROR, "Malformed response contents", content); return; } - LLInventoryModelBackgroundFetch *fetcher = LLInventoryModelBackgroundFetch::getInstance(); - if (content.has("folders")) - { - - for(LLSD::array_const_iterator folder_it = content["folders"].beginArray(); - folder_it != content["folders"].endArray(); - ++folder_it) - { - LLSD folder_sd = *folder_it; - - - //LLUUID agent_id = folder_sd["agent_id"]; - - //if(agent_id != gAgent.getID()) //This should never happen. - //{ - // LL_WARNS() << "Got a UpdateInventoryItem for the wrong agent." - // << LL_ENDL; - // break; - //} - - LLUUID parent_id = folder_sd["folder_id"]; - LLUUID owner_id = folder_sd["owner_id"]; - S32 version = (S32)folder_sd["version"].asInteger(); - S32 descendents = (S32)folder_sd["descendents"].asInteger(); - LLPointer tcategory = new LLViewerInventoryCategory(owner_id); - if (parent_id.isNull()) - { - LLPointer titem = new LLViewerInventoryItem; - for(LLSD::array_const_iterator item_it = folder_sd["items"].beginArray(); - item_it != folder_sd["items"].endArray(); - ++item_it) - { - const LLUUID lost_uuid = gInventory.findCategoryUUIDForType(LLFolderType::FT_LOST_AND_FOUND); - if (lost_uuid.notNull()) - { - LLSD item = *item_it; - titem->unpackMessage(item); - - LLInventoryModel::update_list_t update; - LLInventoryModel::LLCategoryUpdate new_folder(lost_uuid, 1); - update.push_back(new_folder); - gInventory.accountForUpdate(update); - - titem->setParent(lost_uuid); - titem->updateParentOnServer(FALSE); - gInventory.updateItem(titem); - gInventory.notifyObservers(); - - } - } - } - - LLViewerInventoryCategory* pcat = gInventory.getCategory(parent_id); - if (!pcat) - { - continue; - } - - for(LLSD::array_const_iterator category_it = folder_sd["categories"].beginArray(); - category_it != folder_sd["categories"].endArray(); - ++category_it) - { - LLSD category = *category_it; - tcategory->fromLLSD(category); - - const BOOL recursive = getIsRecursive(tcategory->getUUID()); - - if (recursive) - { - fetcher->mFetchQueue.push_back(LLInventoryModelBackgroundFetch::FetchQueueInfo(tcategory->getUUID(), recursive)); - } - else if ( !gInventory.isCategoryComplete(tcategory->getUUID()) ) - { - gInventory.updateCategory(tcategory); - } - - } - LLPointer titem = new LLViewerInventoryItem; - for(LLSD::array_const_iterator item_it = folder_sd["items"].beginArray(); - item_it != folder_sd["items"].endArray(); - ++item_it) - { - LLSD item = *item_it; - titem->unpackMessage(item); - - gInventory.updateItem(titem); - } - - // Set version and descendentcount according to message. - LLViewerInventoryCategory* cat = gInventory.getCategory(parent_id); - if(cat) - { - cat->setVersion(version); - cat->setDescendentCount(descendents); - cat->determineFolderType(); - } - - } - } - - if (content.has("bad_folders")) - { - for(LLSD::array_const_iterator folder_it = content["bad_folders"].beginArray(); - folder_it != content["bad_folders"].endArray(); - ++folder_it) - { - // *TODO: Stop copying data - LLSD folder_sd = *folder_it; - - // These folders failed on the dataserver. We probably don't want to retry them. - LL_WARNS() << "Folder " << folder_sd["folder_id"].asString() - << "Error: " << folder_sd["error"].asString() << LL_ENDL; - } - } + static const S32 max_concurrent_fetches(12); // Outstanding requests, not connections + static const F32 new_min_time(0.5f); // *HACK: Clean this up when old code goes away entirely. + static const U32 max_batch_size(10); - if (fetcher->isBulkFetchProcessingComplete()) + if (mMinTimeBetweenFetches < new_min_time) { - LL_INFOS() << "Inventory fetch completed" << LL_ENDL; - fetcher->setAllFoldersFetched(); + mMinTimeBetweenFetches = new_min_time; // *HACK: See above. } - - gInventory.notifyObservers(); -} - -// If we get back an error (not found, etc...), handle it here. -void LLInventoryModelFetchDescendentsResponder::httpFailure() -{ - LL_WARNS() << dumpResponse() << LL_ENDL; - LLInventoryModelBackgroundFetch *fetcher = LLInventoryModelBackgroundFetch::getInstance(); - - LL_INFOS() << dumpResponse() << LL_ENDL; - - fetcher->incrFetchCount(-1); - if (getStatus()==HTTP_INTERNAL_ERROR) // timed out or curl failure - { - for(LLSD::array_const_iterator folder_it = mRequestSD["folders"].beginArray(); - folder_it != mRequestSD["folders"].endArray(); - ++folder_it) - { - LLSD folder_sd = *folder_it; - LLUUID folder_id = folder_sd["folder_id"]; - const BOOL recursive = getIsRecursive(folder_id); - fetcher->mFetchQueue.push_front(LLInventoryModelBackgroundFetch::FetchQueueInfo(folder_id, recursive)); - } - } - else + if (mFetchCount) { - if (fetcher->isBulkFetchProcessingComplete()) - { - fetcher->setAllFoldersFetched(); - } - } - gInventory.notifyObservers(); -} - -BOOL LLInventoryModelFetchDescendentsResponder::getIsRecursive(const LLUUID& cat_id) const -{ - return (std::find(mRecursiveCatUUIDs.begin(),mRecursiveCatUUIDs.end(), cat_id) != mRecursiveCatUUIDs.end()); -} -// Bundle up a bunch of requests to send all at once. -// static -void LLInventoryModelBackgroundFetch::bulkFetch() -{ - //Background fetch is called from gIdleCallbacks in a loop until background fetch is stopped. - //If there are items in mFetchQueue, we want to check the time since the last bulkFetch was - //sent. If it exceeds our retry time, go ahead and fire off another batch. - LLViewerRegion* region = gAgent.getRegion(); - if (!region) return; - - S16 max_concurrent_fetches=8; - F32 new_min_time = 0.5f; //HACK! Clean this up when old code goes away entirely. - if (mMinTimeBetweenFetches < new_min_time) - { - mMinTimeBetweenFetches=new_min_time; //HACK! See above. + // Process completed HTTP requests + gInventory.handleResponses(false); } - if (gDisconnected || - (mFetchCount > max_concurrent_fetches) || + if ((mFetchCount > max_concurrent_fetches) || (mFetchTimer.getElapsedTimeF32() < mMinTimeBetweenFetches)) { - return; // just bail if we are disconnected + return; } - U32 item_count=0; - U32 folder_count=0; - U32 max_batch_size=5; + U32 item_count(0); + U32 folder_count(0); - U32 sort_order = gSavedSettings.getU32(LLInventoryPanel::DEFAULT_SORT_ORDER) & 0x1; + const U32 sort_order(gSavedSettings.getU32(LLInventoryPanel::DEFAULT_SORT_ORDER) & 0x1); uuid_vec_t recursive_cats; @@ -614,27 +504,27 @@ void LLInventoryModelBackgroundFetch::bulkFetch() LLSD item_request_body; LLSD item_request_body_lib; - while (!mFetchQueue.empty() + while (! mFetchQueue.empty() && (item_count + folder_count) < max_batch_size) { - const FetchQueueInfo& fetch_info = mFetchQueue.front(); + const FetchQueueInfo & fetch_info(mFetchQueue.front()); if (fetch_info.mIsCategory) { - const LLUUID &cat_id = fetch_info.mUUID; + const LLUUID & cat_id(fetch_info.mUUID); if (cat_id.isNull()) //DEV-17797 { LLSD folder_sd; folder_sd["folder_id"] = LLUUID::null.asString(); folder_sd["owner_id"] = gAgent.getID(); - folder_sd["sort_order"] = (LLSD::Integer)sort_order; - folder_sd["fetch_folders"] = (LLSD::Boolean)FALSE; - folder_sd["fetch_items"] = (LLSD::Boolean)TRUE; + folder_sd["sort_order"] = LLSD::Integer(sort_order); + folder_sd["fetch_folders"] = LLSD::Boolean(FALSE); + folder_sd["fetch_items"] = LLSD::Boolean(TRUE); folder_request_body["folders"].append(folder_sd); folder_count++; } else { - const LLViewerInventoryCategory* cat = gInventory.getCategory(cat_id); + const LLViewerInventoryCategory * cat(gInventory.getCategory(cat_id)); if (cat) { @@ -643,21 +533,26 @@ void LLInventoryModelBackgroundFetch::bulkFetch() LLSD folder_sd; folder_sd["folder_id"] = cat->getUUID(); folder_sd["owner_id"] = cat->getOwnerID(); - folder_sd["sort_order"] = (LLSD::Integer)sort_order; - folder_sd["fetch_folders"] = TRUE; //(LLSD::Boolean)sFullFetchStarted; - folder_sd["fetch_items"] = (LLSD::Boolean)TRUE; + folder_sd["sort_order"] = LLSD::Integer(sort_order); + folder_sd["fetch_folders"] = LLSD::Boolean(TRUE); //(LLSD::Boolean)sFullFetchStarted; + folder_sd["fetch_items"] = LLSD::Boolean(TRUE); if (ALEXANDRIA_LINDEN_ID == cat->getOwnerID()) + { folder_request_body_lib["folders"].append(folder_sd); + } else + { folder_request_body["folders"].append(folder_sd); + } folder_count++; } + // May already have this folder, but append child folders to list. if (fetch_info.mRecursive) { - LLInventoryModel::cat_array_t* categories; - LLInventoryModel::item_array_t* items; + LLInventoryModel::cat_array_t * categories(NULL); + LLInventoryModel::item_array_t * items(NULL); gInventory.getDirectDescendentsOf(cat->getUUID(), categories, items); for (LLInventoryModel::cat_array_t::const_iterator it = categories->begin(); it != categories->end(); @@ -669,11 +564,14 @@ void LLInventoryModelBackgroundFetch::bulkFetch() } } if (fetch_info.mRecursive) + { recursive_cats.push_back(cat_id); + } } else { - LLViewerInventoryItem* itemp = gInventory.getItem(fetch_info.mUUID); + LLViewerInventoryItem * itemp(gInventory.getItem(fetch_info.mUUID)); + if (itemp) { LLSD item_sd; @@ -694,72 +592,80 @@ void LLInventoryModelBackgroundFetch::bulkFetch() mFetchQueue.pop_front(); } - + + // Issue HTTP POST requests to fetch folders and items + if (item_count + folder_count > 0) { if (folder_count) { - std::string url = region->getCapability("FetchInventoryDescendents2"); - if ( !url.empty() ) + if (folder_request_body["folders"].size()) { - if (folder_request_body["folders"].size()) + const std::string url(region->getCapability("FetchInventoryDescendents2")); + + if (! url.empty()) { - LLInventoryModelFetchDescendentsResponder *fetcher = new LLInventoryModelFetchDescendentsResponder(folder_request_body, recursive_cats); - LLHTTPClient::post(url, folder_request_body, fetcher, 300.0); + BGFolderHttpHandler * handler(new BGFolderHttpHandler(folder_request_body, recursive_cats)); + gInventory.requestPost(false, url, folder_request_body, handler, "Inventory Folder"); } - if (folder_request_body_lib["folders"].size()) - { - std::string url_lib = gAgent.getRegion()->getCapability("FetchLibDescendents2"); + } + + if (folder_request_body_lib["folders"].size()) + { + const std::string url(region->getCapability("FetchLibDescendents2")); - LLInventoryModelFetchDescendentsResponder *fetcher = new LLInventoryModelFetchDescendentsResponder(folder_request_body_lib, recursive_cats); - LLHTTPClient::post(url_lib, folder_request_body_lib, fetcher, 300.0); + if (! url.empty()) + { + BGFolderHttpHandler * handler(new BGFolderHttpHandler(folder_request_body_lib, recursive_cats)); + gInventory.requestPost(false, url, folder_request_body_lib, handler, "Library Folder"); } } - } + } // if (folder_count) + if (item_count) { - std::string url; - if (item_request_body.size()) { - url = region->getCapability("FetchInventory2"); - if (!url.empty()) + const std::string url(region->getCapability("FetchInventory2")); + + if (! url.empty()) { LLSD body; body["items"] = item_request_body; - - LLHTTPClient::post(url, body, new LLInventoryModelFetchItemResponder(body)); + BGItemHttpHandler * handler(new BGItemHttpHandler(body)); + gInventory.requestPost(false, url, body, handler, "Inventory Item"); } } if (item_request_body_lib.size()) { + const std::string url(region->getCapability("FetchLib2")); - url = region->getCapability("FetchLib2"); - if (!url.empty()) + if (! url.empty()) { LLSD body; body["items"] = item_request_body_lib; - - LLHTTPClient::post(url, body, new LLInventoryModelFetchItemResponder(body)); + BGItemHttpHandler * handler(new BGItemHttpHandler(body)); + gInventory.requestPost(false, url, body, handler, "Library Item"); } } - } + } // if (item_count) + mFetchTimer.reset(); } - else if (isBulkFetchProcessingComplete()) { setAllFoldersFetched(); } } -bool LLInventoryModelBackgroundFetch::fetchQueueContainsNoDescendentsOf(const LLUUID& cat_id) const +bool LLInventoryModelBackgroundFetch::fetchQueueContainsNoDescendentsOf(const LLUUID & cat_id) const { for (fetch_queue_t::const_iterator it = mFetchQueue.begin(); - it != mFetchQueue.end(); ++it) + it != mFetchQueue.end(); + ++it) { - const LLUUID& fetch_id = (*it).mUUID; + const LLUUID & fetch_id = (*it).mUUID; if (gInventory.isObjectDescendentOf(fetch_id, cat_id)) return false; } @@ -767,3 +673,257 @@ bool LLInventoryModelBackgroundFetch::fetchQueueContainsNoDescendentsOf(const LL } +// Anonymous Namespace Definitions + +namespace +{ + +// ==== BGFolderHttpHandler ==== + +void BGFolderHttpHandler::onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * response) +{ + LLCore::HttpStatus status(response->getStatus()); + if (! status) + { + processFailure(status, response); + } + else + { + LLCore::BufferArray * body(response->getBody()); + if (! body || ! body->size()) + { + LL_WARNS(LOG_INV) << "Missing data in inventory folder query." << LL_ENDL; + processFailure("HTTP response missing expected body", response); + goto only_exit; + } + + LLCore::BufferArrayStream bas(body); + LLSD body_llsd; + S32 parse_status(LLSDSerialize::fromXML(body_llsd, bas)); + if (LLSDParser::PARSE_FAILURE == parse_status) + { + // INFOS-level logging will occur on the parsed failure + processFailure("HTTP response contained malformed LLSD", response); + goto only_exit; + } + + // Okay, process data if possible + processData(body_llsd, response); + } + +only_exit: + // Must delete on completion. + delete this; +} + + +void BGFolderHttpHandler::processData(LLSD & content, LLCore::HttpResponse * response) +{ + if (! content.isMap()) + { + processFailure("LLSD response not a map", response); + return; + } + + LLInventoryModelBackgroundFetch * fetcher(LLInventoryModelBackgroundFetch::getInstance()); + if (content.has("folders")) + { + LLSD folders(content["folders"]); + + for (LLSD::array_const_iterator folder_it = folders.beginArray(); + folder_it != folders.endArray(); + ++folder_it) + { + LLSD folder_sd(*folder_it); + + //LLUUID agent_id = folder_sd["agent_id"]; + + //if(agent_id != gAgent.getID()) //This should never happen. + //{ + // LL_WARNS(LOG_INV) << "Got a UpdateInventoryItem for the wrong agent." + // << LL_ENDL; + // break; + //} + + LLUUID parent_id(folder_sd["folder_id"].asUUID()); + LLUUID owner_id(folder_sd["owner_id"].asUUID()); + S32 version(folder_sd["version"].asInteger()); + S32 descendents(folder_sd["descendents"].asInteger()); + LLPointer tcategory = new LLViewerInventoryCategory(owner_id); + + if (parent_id.isNull()) + { + LLSD items(folder_sd["items"]); + LLPointer titem = new LLViewerInventoryItem; + + for (LLSD::array_const_iterator item_it = items.beginArray(); + item_it != items.endArray(); + ++item_it) + { + const LLUUID lost_uuid(gInventory.findCategoryUUIDForType(LLFolderType::FT_LOST_AND_FOUND)); + + if (lost_uuid.notNull()) + { + LLSD item(*item_it); + + titem->unpackMessage(item); + + LLInventoryModel::update_list_t update; + LLInventoryModel::LLCategoryUpdate new_folder(lost_uuid, 1); + update.push_back(new_folder); + gInventory.accountForUpdate(update); + + titem->setParent(lost_uuid); + titem->updateParentOnServer(FALSE); + gInventory.updateItem(titem); + gInventory.notifyObservers(); + } + } + } + + LLViewerInventoryCategory * pcat(gInventory.getCategory(parent_id)); + if (! pcat) + { + continue; + } + + LLSD categories(folder_sd["categories"]); + for (LLSD::array_const_iterator category_it = categories.beginArray(); + category_it != categories.endArray(); + ++category_it) + { + LLSD category(*category_it); + tcategory->fromLLSD(category); + + const bool recursive(getIsRecursive(tcategory->getUUID())); + if (recursive) + { + fetcher->addRequestAtBack(tcategory->getUUID(), recursive, true); + } + else if (! gInventory.isCategoryComplete(tcategory->getUUID())) + { + gInventory.updateCategory(tcategory); + } + } + + LLSD items(folder_sd["items"]); + LLPointer titem = new LLViewerInventoryItem; + for (LLSD::array_const_iterator item_it = items.beginArray(); + item_it != items.endArray(); + ++item_it) + { + LLSD item(*item_it); + titem->unpackMessage(item); + + gInventory.updateItem(titem); + } + + // Set version and descendentcount according to message. + LLViewerInventoryCategory * cat(gInventory.getCategory(parent_id)); + if (cat) + { + cat->setVersion(version); + cat->setDescendentCount(descendents); + cat->determineFolderType(); + } + } + } + + if (content.has("bad_folders")) + { + LLSD bad_folders(content["bad_folders"]); + for (LLSD::array_const_iterator folder_it = bad_folders.beginArray(); + folder_it != bad_folders.endArray(); + ++folder_it) + { + // *TODO: Stop copying data + LLSD folder_sd(*folder_it); + + // These folders failed on the dataserver. We probably don't want to retry them. + LL_WARNS(LOG_INV) << "Folder " << folder_sd["folder_id"].asString() + << "Error: " << folder_sd["error"].asString() << LL_ENDL; + } + } + + if (fetcher->isBulkFetchProcessingComplete()) + { + LL_INFOS(LOG_INV) << "Inventory fetch completed" << LL_ENDL; + fetcher->setAllFoldersFetched(); + } + + gInventory.notifyObservers(); +} + + +void BGFolderHttpHandler::processFailure(LLCore::HttpStatus status, LLCore::HttpResponse * response) +{ + LL_WARNS(LOG_INV) << dumpResponse() << LL_ENDL; + LLInventoryModelBackgroundFetch *fetcher = LLInventoryModelBackgroundFetch::getInstance(); + + LL_INFOS(LOG_INV) << dumpResponse() << LL_ENDL; + + // *FIX: Not the correct test here... + if (status == LLCore::HttpStatus(408)) // timed out or curl failure + { + for (LLSD::array_const_iterator folder_it = mRequestSD["folders"].beginArray(); + folder_it != mRequestSD["folders"].endArray(); + ++folder_it) + { + LLSD folder_sd(*folder_it); + LLUUID folder_id(folder_sd["folder_id"]); + const BOOL recursive = getIsRecursive(folder_id); + fetcher->addRequestAtFront(folder_id, recursive, true); + } + } + else + { + if (fetcher->isBulkFetchProcessingComplete()) + { + fetcher->setAllFoldersFetched(); + } + } + gInventory.notifyObservers(); +} + + +void BGFolderHttpHandler::processFailure(const char * const reason, LLCore::HttpResponse * response) +{ + LL_WARNS(LOG_INV) << dumpResponse() << LL_ENDL; + LLInventoryModelBackgroundFetch *fetcher = LLInventoryModelBackgroundFetch::getInstance(); + + LL_INFOS(LOG_INV) << dumpResponse() << LL_ENDL; + + if (true /* status == LLCore::HttpStatus(408)*/) // timed out or curl failure + { + for (LLSD::array_const_iterator folder_it = mRequestSD["folders"].beginArray(); + folder_it != mRequestSD["folders"].endArray(); + ++folder_it) + { + LLSD folder_sd(*folder_it); + LLUUID folder_id(folder_sd["folder_id"]); + const BOOL recursive = getIsRecursive(folder_id); + fetcher->addRequestAtFront(folder_id, recursive, true); + } + } + else + { + if (fetcher->isBulkFetchProcessingComplete()) + { + fetcher->setAllFoldersFetched(); + } + } + gInventory.notifyObservers(); +} + + +bool BGFolderHttpHandler::getIsRecursive(const LLUUID & cat_id) const +{ + return std::find(mRecursiveCatUUIDs.begin(), mRecursiveCatUUIDs.end(), cat_id) != mRecursiveCatUUIDs.end(); +} + + +// ==== BGItemHttpHandler ==== + +// Nothing to implement here. All ctor/dtor changes. + +} // end namespace anonymous diff --git a/indra/newview/llinventorymodelbackgroundfetch.h b/indra/newview/llinventorymodelbackgroundfetch.h index 9dfedddd6d..2139f85519 100755 --- a/indra/newview/llinventorymodelbackgroundfetch.h +++ b/indra/newview/llinventorymodelbackgroundfetch.h @@ -29,6 +29,11 @@ #include "llsingleton.h" #include "lluuid.h" +#include "httpcommon.h" +#include "httprequest.h" +#include "httpoptions.h" +#include "httpheaders.h" +#include "httphandler.h" //~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ // Class LLInventoryModelBackgroundFetch @@ -38,8 +43,6 @@ //~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ class LLInventoryModelBackgroundFetch : public LLSingleton { - friend class LLInventoryModelFetchDescendentsResponder; - public: LLInventoryModelBackgroundFetch(); ~LLInventoryModelBackgroundFetch(); @@ -60,16 +63,22 @@ public: bool inventoryFetchInProgress() const; void findLostItems(); - void incrFetchCount(S16 fetching); -protected: + void incrFetchCount(S32 fetching); + bool isBulkFetchProcessingComplete() const; + void setAllFoldersFetched(); + + void addRequestAtFront(const LLUUID & id, BOOL recursive, bool is_category); + void addRequestAtBack(const LLUUID & id, BOOL recursive, bool is_category); + +protected: void bulkFetch(); void backgroundFetch(); static void backgroundFetchCB(void*); // background fetch idle function - void setAllFoldersFetched(); bool fetchQueueContainsNoDescendentsOf(const LLUUID& cat_id) const; + private: BOOL mRecursiveInventoryFetchStarted; BOOL mRecursiveLibraryFetchStarted; @@ -77,7 +86,7 @@ private: BOOL mBackgroundFetchActive; bool mFolderFetchActive; - S16 mFetchCount; + S32 mFetchCount; BOOL mTimelyFetchPending; S32 mNumFetchRetries; @@ -87,9 +96,12 @@ private: struct FetchQueueInfo { - FetchQueueInfo(const LLUUID& id, BOOL recursive, bool is_category = true) : - mUUID(id), mRecursive(recursive), mIsCategory(is_category) + FetchQueueInfo(const LLUUID& id, BOOL recursive, bool is_category = true) + : mUUID(id), + mIsCategory(is_category), + mRecursive(recursive) {} + LLUUID mUUID; bool mIsCategory; BOOL mRecursive; diff --git a/indra/newview/llinventoryobserver.cpp b/indra/newview/llinventoryobserver.cpp index 2dd8dce42f..d81401b59b 100755 --- a/indra/newview/llinventoryobserver.cpp +++ b/indra/newview/llinventoryobserver.cpp @@ -237,7 +237,8 @@ void fetch_items_from_llsd(const LLSD& items_llsd) if (!url.empty()) { body[i]["agent_id"] = gAgent.getID(); - LLHTTPClient::post(url, body[i], new LLInventoryModel::fetchInventoryResponder(body[i])); + LLInventoryModel::FetchItemHttpHandler * handler(new LLInventoryModel::FetchItemHttpHandler(body[i])); + gInventory.requestPost(true, url, body[i], handler, (i ? "Library Item" : "Inventory Item")); continue; } diff --git a/indra/newview/llsnapshotlivepreview.cpp b/indra/newview/llsnapshotlivepreview.cpp index f61db77169..942360b650 100644 --- a/indra/newview/llsnapshotlivepreview.cpp +++ b/indra/newview/llsnapshotlivepreview.cpp @@ -5,7 +5,7 @@ * * $LicenseInfo:firstyear=2013&license=viewerlgpl$ * Second Life Viewer Source Code -* Copyright (C) 2013, Linden Research, Inc. +* Copyright (C) 2014, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -151,7 +151,7 @@ F32 LLSnapshotLivePreview::getImageAspect() void LLSnapshotLivePreview::updateSnapshot(BOOL new_snapshot, BOOL new_thumbnail, F32 delay) { - lldebugs << "updateSnapshot: mSnapshotUpToDate = " << getSnapshotUpToDate() << llendl; + LL_DEBUGS() << "updateSnapshot: mSnapshotUpToDate = " << getSnapshotUpToDate() << LL_ENDL; // Update snapshot if requested. if (new_snapshot) @@ -594,7 +594,7 @@ void LLSnapshotLivePreview::generateThumbnailImage(BOOL force_update) } else { - llwarns << "Couldn't find a path to the following filter : " << getFilter() << llendl; + LL_WARNS() << "Couldn't find a path to the following filter : " << getFilter() << LL_ENDL; } } // Scale to a power of 2 so it can be mapped to a texture @@ -642,7 +642,7 @@ LLViewerTexture* LLSnapshotLivePreview::getBigThumbnailImage() } else { - llwarns << "Couldn't find a path to the following filter : " << getFilter() << llendl; + LL_WARNS() << "Couldn't find a path to the following filter : " << getFilter() << LL_ENDL; } } // Scale to a power of 2 so it can be mapped to a texture @@ -695,7 +695,7 @@ BOOL LLSnapshotLivePreview::onIdle( void* snapshot_preview ) // time to produce a snapshot if(!previewp->getSnapshotUpToDate()) { - lldebugs << "producing snapshot" << llendl; + LL_DEBUGS() << "producing snapshot" << LL_ENDL; if (!previewp->mPreviewImage) { previewp->mPreviewImage = new LLImageRaw; @@ -775,7 +775,7 @@ BOOL LLSnapshotLivePreview::onIdle( void* snapshot_preview ) previewp->setVisible(gSavedSettings.getBOOL("UseFreezeFrame") && previewp->mAllowFullScreenPreview); // only show fullscreen preview when in freeze frame mode previewp->mSnapshotDelayTimer.stop(); previewp->mSnapshotActive = FALSE; - lldebugs << "done creating snapshot" << llendl; + LL_DEBUGS() << "done creating snapshot" << LL_ENDL; } if (!previewp->getThumbnailUpToDate()) @@ -910,13 +910,13 @@ LLPointer LLSnapshotLivePreview::getFormattedImage() } else { - llwarns << "Couldn't find a path to the following filter : " << getFilter() << llendl; + LL_WARNS() << "Couldn't find a path to the following filter : " << getFilter() << LL_ENDL; } } // Create the new formatted image of the appropriate format. LLFloaterSnapshot::ESnapshotFormat format = getSnapshotFormat(); - lldebugs << "Encoding new image of format " << format << llendl; + LL_DEBUGS() << "Encoding new image of format " << format << LL_ENDL; switch (format) { diff --git a/indra/newview/llviewerinventory.cpp b/indra/newview/llviewerinventory.cpp index 4e4c3471be..d364fce45a 100755 --- a/indra/newview/llviewerinventory.cpp +++ b/indra/newview/llviewerinventory.cpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2002&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2010, Linden Research, Inc. + * Copyright (C) 2014, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -75,6 +75,10 @@ void no_op_inventory_func(const LLUUID&) {} void no_op_llsd_func(const LLSD&) {} void no_op() {} +static const char * const LOG_INV("Inventory"); +static const char * const LOG_LOCAL("InventoryLocalize"); +static const char * const LOG_NOTECARD("copy_inventory_from_notecard"); + ///---------------------------------------------------------------------------- /// Helper class to store special inventory item names and their localized values. ///---------------------------------------------------------------------------- @@ -189,7 +193,7 @@ public: */ bool localizeInventoryObjectName(std::string& object_name) { - LL_DEBUGS("InventoryLocalize") << "Searching for localization: " << object_name << LL_ENDL; + LL_DEBUGS(LOG_LOCAL) << "Searching for localization: " << object_name << LL_ENDL; std::map::const_iterator dictionary_iter = mInventoryItemsDict.find(object_name); @@ -197,7 +201,7 @@ public: if(found) { object_name = dictionary_iter->second; - LL_DEBUGS("InventoryLocalize") << "Found, new name is: " << object_name << LL_ENDL; + LL_DEBUGS(LOG_LOCAL) << "Found, new name is: " << object_name << LL_ENDL; } return found; } @@ -307,8 +311,8 @@ LLViewerInventoryItem::LLViewerInventoryItem(const LLViewerInventoryItem* other) copyViewerItem(other); if (!mIsComplete) { - LL_WARNS() << "LLViewerInventoryItem copy constructor for incomplete item" - << mUUID << LL_ENDL; + LL_WARNS(LOG_INV) << "LLViewerInventoryItem copy constructor for incomplete item" + << mUUID << LL_ENDL; } } @@ -355,16 +359,16 @@ void LLViewerInventoryItem::updateServer(BOOL is_new) const { // *FIX: deal with this better. // If we're crashing here then the UI is incorrectly enabled. - LL_ERRS() << "LLViewerInventoryItem::updateServer() - for incomplete item" - << LL_ENDL; + LL_ERRS(LOG_INV) << "LLViewerInventoryItem::updateServer() - for incomplete item" + << LL_ENDL; return; } if(gAgent.getID() != mPermissions.getOwner()) { // *FIX: deal with this better. - LL_WARNS() << "LLViewerInventoryItem::updateServer() - for unowned item " - << ll_pretty_print_sd(this->asLLSD()) - << LL_ENDL; + LL_WARNS(LOG_INV) << "LLViewerInventoryItem::updateServer() - for unowned item " + << ll_pretty_print_sd(this->asLLSD()) + << LL_ENDL; return; } LLInventoryModel::LLCategoryUpdate up(mParentUUID, is_new ? 1 : 0); @@ -392,18 +396,18 @@ void LLViewerInventoryItem::fetchFromServer(void) const // we have to check region. It can be null after region was destroyed. See EXT-245 if (region) { - if(gAgent.getID() != mPermissions.getOwner()) - { + if (gAgent.getID() != mPermissions.getOwner()) + { url = region->getCapability("FetchLib2"); - } + } else - { + { url = region->getCapability("FetchInventory2"); - } + } } else { - LL_WARNS() << "Agent Region is absent" << LL_ENDL; + LL_WARNS(LOG_INV) << "Agent Region is absent" << LL_ENDL; } if (!url.empty()) @@ -413,7 +417,8 @@ void LLViewerInventoryItem::fetchFromServer(void) const body["items"][0]["owner_id"] = mPermissions.getOwner(); body["items"][0]["item_id"] = mUUID; - LLHTTPClient::post(url, body, new LLInventoryModel::fetchInventoryResponder(body)); + LLInventoryModel::FetchItemHttpHandler * handler(new LLInventoryModel::FetchItemHttpHandler(body)); + gInventory.requestPost(true, url, body, handler, "Inventory Item"); } else { @@ -649,7 +654,7 @@ bool LLViewerInventoryCategory::fetch() if((VERSION_UNKNOWN == getVersion()) && mDescendentsRequested.hasExpired()) //Expired check prevents multiple downloads. { - LL_DEBUGS("InventoryFetch") << "Fetching category children: " << mName << ", UUID: " << mUUID << LL_ENDL; + LL_DEBUGS(LOG_INV) << "Fetching category children: " << mName << ", UUID: " << mUUID << LL_ENDL; const F32 FETCH_TIMER_EXPIRY = 10.0f; mDescendentsRequested.reset(); mDescendentsRequested.setTimerExpirySec(FETCH_TIMER_EXPIRY); @@ -674,7 +679,7 @@ bool LLViewerInventoryCategory::fetch() } else { - LL_WARNS() << "agent region is null" << LL_ENDL; + LL_WARNS(LOG_INV) << "agent region is null" << LL_ENDL; } if (!url.empty()) //Capability found. Build up LLSD and use it. { @@ -682,7 +687,8 @@ bool LLViewerInventoryCategory::fetch() } else { //Deprecated, but if we don't have a capability, use the old system. - LL_INFOS() << "FetchInventoryDescendents2 capability not found. Using deprecated UDP message." << LL_ENDL; + LL_INFOS(LOG_INV) << "FetchInventoryDescendents2 capability not found. Using deprecated UDP message." << LL_ENDL; + LLMessageSystem* msg = gMessageSystem; msg->newMessage("FetchInventoryDescendents"); msg->nextBlock("AgentData"); @@ -777,8 +783,8 @@ bool LLViewerInventoryCategory::importFileLocal(LLFILE* fp) } else { - LL_WARNS() << "unknown keyword '" << keyword - << "' in inventory import category " << mUUID << LL_ENDL; + LL_WARNS(LOG_INV) << "unknown keyword '" << keyword + << "' in inventory import category " << mUUID << LL_ENDL; } } return true; @@ -906,7 +912,7 @@ LLInventoryCallbackManager::LLInventoryCallbackManager() : { if( sInstance != NULL ) { - LL_WARNS() << "LLInventoryCallbackManager::LLInventoryCallbackManager: unexpected multiple instances" << LL_ENDL; + LL_WARNS(LOG_INV) << "LLInventoryCallbackManager::LLInventoryCallbackManager: unexpected multiple instances" << LL_ENDL; return; } sInstance = this; @@ -916,7 +922,7 @@ LLInventoryCallbackManager::~LLInventoryCallbackManager() { if( sInstance != this ) { - LL_WARNS() << "LLInventoryCallbackManager::~LLInventoryCallbackManager: unexpected multiple instances" << LL_ENDL; + LL_WARNS(LOG_INV) << "LLInventoryCallbackManager::~LLInventoryCallbackManager: unexpected multiple instances" << LL_ENDL; return; } sInstance = NULL; @@ -1144,7 +1150,7 @@ void link_inventory_object(const LLUUID& category, { if (!baseobj) { - LL_WARNS() << "Attempt to link to non-existent object" << LL_ENDL; + LL_WARNS(LOG_INV) << "Attempt to link to non-existent object" << LL_ENDL; return; } @@ -1178,7 +1184,7 @@ void link_inventory_array(const LLUUID& category, const LLInventoryObject* baseobj = *it; if (!baseobj) { - LL_WARNS() << "attempt to link to unknown object" << LL_ENDL; + LL_WARNS(LOG_INV) << "attempt to link to unknown object" << LL_ENDL; continue; } @@ -1187,7 +1193,7 @@ void link_inventory_array(const LLUUID& category, // Fail if item can be found but is of a type that can't be linked. // Arguably should fail if the item can't be found too, but that could // be a larger behavioral change. - LL_WARNS() << "attempt to link an unlinkable object, type = " << baseobj->getActualType() << LL_ENDL; + LL_WARNS(LOG_INV) << "attempt to link an unlinkable object, type = " << baseobj->getActualType() << LL_ENDL; continue; } @@ -1223,7 +1229,7 @@ void link_inventory_array(const LLUUID& category, } else { - LL_WARNS() << "could not convert object into an item or category: " << baseobj->getUUID() << LL_ENDL; + LL_WARNS(LOG_INV) << "could not convert object into an item or category: " << baseobj->getUUID() << LL_ENDL; continue; } } @@ -1237,10 +1243,10 @@ void link_inventory_array(const LLUUID& category, links.append(link); #ifndef LL_RELEASE_FOR_DOWNLOAD - LL_DEBUGS("Inventory") << "Linking Object [ name:" << baseobj->getName() - << " UUID:" << baseobj->getUUID() - << " ] into Category [ name:" << cat_name - << " UUID:" << category << " ] " << LL_ENDL; + LL_DEBUGS(LOG_INV) << "Linking Object [ name:" << baseobj->getName() + << " UUID:" << baseobj->getUUID() + << " ] into Category [ name:" << cat_name + << " UUID:" << category << " ] " << LL_ENDL; #endif } @@ -1331,7 +1337,7 @@ void update_inventory_item( if (!ais_ran) { LLPointer obj = gInventory.getItem(item_id); - LL_DEBUGS("Inventory") << "item_id: [" << item_id << "] name " << (update_item ? update_item->getName() : "(NOT FOUND)") << LL_ENDL; + LL_DEBUGS(LOG_INV) << "item_id: [" << item_id << "] name " << (update_item ? update_item->getName() : "(NOT FOUND)") << LL_ENDL; if(obj) { LLMessageSystem* msg = gMessageSystem; @@ -1373,7 +1379,7 @@ void update_inventory_item( if (!ais_ran) { LLPointer obj = gInventory.getItem(item_id); - LL_DEBUGS("Inventory") << "item_id: [" << item_id << "] name " << (obj ? obj->getName() : "(NOT FOUND)") << LL_ENDL; + LL_DEBUGS(LOG_INV) << "item_id: [" << item_id << "] name " << (obj ? obj->getName() : "(NOT FOUND)") << LL_ENDL; if(obj) { LLPointer new_item(new LLViewerInventoryItem); @@ -1408,7 +1414,7 @@ void update_inventory_category( LLPointer cb) { LLPointer obj = gInventory.getCategory(cat_id); - LL_DEBUGS("Inventory") << "cat_id: [" << cat_id << "] name " << (obj ? obj->getName() : "(NOT FOUND)") << LL_ENDL; + LL_DEBUGS(LOG_INV) << "cat_id: [" << cat_id << "] name " << (obj ? obj->getName() : "(NOT FOUND)") << LL_ENDL; if(obj) { if (LLFolderType::lookupIsProtectedType(obj->getPreferredType())) @@ -1471,7 +1477,7 @@ void remove_inventory_item( } else { - LL_DEBUGS("Inventory") << "item_id: [" << item_id << "] name " << "(NOT FOUND)" << LL_ENDL; + LL_DEBUGS(LOG_INV) << "item_id: [" << item_id << "] name " << "(NOT FOUND)" << LL_ENDL; } } @@ -1482,7 +1488,7 @@ void remove_inventory_item( if(obj) { const LLUUID item_id(obj->getUUID()); - LL_DEBUGS("Inventory") << "item_id: [" << item_id << "] name " << obj->getName() << LL_ENDL; + LL_DEBUGS(LOG_INV) << "item_id: [" << item_id << "] name " << obj->getName() << LL_ENDL; if (AISCommand::isAPIAvailable()) { LLPointer cmd_ptr = new RemoveItemCommand(item_id, cb); @@ -1511,7 +1517,7 @@ void remove_inventory_item( else { // *TODO: Clean up callback? - LL_WARNS() << "remove_inventory_item called for invalid or nonexistent item." << LL_ENDL; + LL_WARNS(LOG_INV) << "remove_inventory_item called for invalid or nonexistent item." << LL_ENDL; } } @@ -1529,7 +1535,7 @@ public: LLInventoryModel::EHasChildren children = gInventory.categoryHasChildren(mID); if(children != LLInventoryModel::CHILDREN_NO) { - LL_WARNS() << "remove descendents failed, cannot remove category " << LL_ENDL; + LL_WARNS(LOG_INV) << "remove descendents failed, cannot remove category " << LL_ENDL; } else { @@ -1545,7 +1551,7 @@ void remove_inventory_category( const LLUUID& cat_id, LLPointer cb) { - LL_DEBUGS("Inventory") << "cat_id: [" << cat_id << "] " << LL_ENDL; + LL_DEBUGS(LOG_INV) << "cat_id: [" << cat_id << "] " << LL_ENDL; LLPointer obj = gInventory.getCategory(cat_id); if(obj) { @@ -1566,7 +1572,7 @@ void remove_inventory_category( LLInventoryModel::EHasChildren children = gInventory.categoryHasChildren(cat_id); if(children != LLInventoryModel::CHILDREN_NO) { - LL_DEBUGS("Inventory") << "Will purge descendents first before deleting category " << cat_id << LL_ENDL; + LL_DEBUGS(LOG_INV) << "Will purge descendents first before deleting category " << cat_id << LL_ENDL; LLPointer wrap_cb = new LLRemoveCategoryOnDestroy(cat_id, cb); purge_descendents_of(cat_id, wrap_cb); return; @@ -1592,7 +1598,7 @@ void remove_inventory_category( } else { - LL_WARNS() << "remove_inventory_category called for invalid or nonexistent item " << cat_id << LL_ENDL; + LL_WARNS(LOG_INV) << "remove_inventory_category called for invalid or nonexistent item " << cat_id << LL_ENDL; } } @@ -1620,7 +1626,7 @@ void purge_descendents_of(const LLUUID& id, LLPointer cb) LLInventoryModel::EHasChildren children = gInventory.categoryHasChildren(id); if(children == LLInventoryModel::CHILDREN_NO) { - LL_DEBUGS("Inventory") << "No descendents to purge for " << id << LL_ENDL; + LL_DEBUGS(LOG_INV) << "No descendents to purge for " << id << LL_ENDL; return; } LLPointer cat = gInventory.getCategory(id); @@ -1629,8 +1635,8 @@ void purge_descendents_of(const LLUUID& id, LLPointer cb) if (LLClipboard::instance().hasContents() && LLClipboard::instance().isCutMode()) { // Something on the clipboard is in "cut mode" and needs to be preserved - LL_DEBUGS("Inventory") << "purge_descendents_of clipboard case " << cat->getName() - << " iterate and purge non hidden items" << LL_ENDL; + LL_DEBUGS(LOG_INV) << "purge_descendents_of clipboard case " << cat->getName() + << " iterate and purge non hidden items" << LL_ENDL; LLInventoryModel::cat_array_t* categories; LLInventoryModel::item_array_t* items; // Get the list of direct descendants in tha categoy passed as argument @@ -1665,7 +1671,7 @@ void purge_descendents_of(const LLUUID& id, LLPointer cb) else // no cap { // Fast purge - LL_DEBUGS("Inventory") << "purge_descendents_of fast case " << cat->getName() << LL_ENDL; + LL_DEBUGS(LOG_INV) << "purge_descendents_of fast case " << cat->getName() << LL_ENDL; // send it upstream LLMessageSystem* msg = gMessageSystem; @@ -1708,9 +1714,9 @@ void copy_inventory_from_notecard(const LLUUID& destination_id, { if (NULL == src) { - LL_WARNS("copy_inventory_from_notecard") << "Null pointer to item was passed for object_id " - << object_id << " and notecard_inv_id " - << notecard_inv_id << LL_ENDL; + LL_WARNS(LOG_NOTECARD) << "Null pointer to item was passed for object_id " + << object_id << " and notecard_inv_id " + << notecard_inv_id << LL_ENDL; return; } @@ -1730,9 +1736,9 @@ void copy_inventory_from_notecard(const LLUUID& destination_id, if (! viewer_region) { - LL_WARNS("copy_inventory_from_notecard") << "Can't find region from object_id " - << object_id << " or gAgent" - << LL_ENDL; + LL_WARNS(LOG_NOTECARD) << "Can't find region from object_id " + << object_id << " or gAgent" + << LL_ENDL; return; } @@ -1740,9 +1746,9 @@ void copy_inventory_from_notecard(const LLUUID& destination_id, std::string url = viewer_region->getCapability("CopyInventoryFromNotecard"); if (url.empty()) { - LL_WARNS("copy_inventory_from_notecard") << "There is no 'CopyInventoryFromNotecard' capability" - << " for region: " << viewer_region->getName() - << LL_ENDL; + LL_WARNS(LOG_NOTECARD) << "There is no 'CopyInventoryFromNotecard' capability" + << " for region: " << viewer_region->getName() + << LL_ENDL; return; } @@ -1816,15 +1822,15 @@ void slam_inventory_folder(const LLUUID& folder_id, { if (AISCommand::isAPIAvailable()) { - LL_DEBUGS("Avatar") << "using AISv3 to slam folder, id " << folder_id - << " new contents: " << ll_pretty_print_sd(contents) << LL_ENDL; + LL_DEBUGS(LOG_INV) << "using AISv3 to slam folder, id " << folder_id + << " new contents: " << ll_pretty_print_sd(contents) << LL_ENDL; LLPointer cmd_ptr = new SlamFolderCommand(folder_id, contents, cb); cmd_ptr->run_command(); } else // no cap { - LL_DEBUGS("Avatar") << "using item-by-item calls to slam folder, id " << folder_id - << " new contents: " << ll_pretty_print_sd(contents) << LL_ENDL; + LL_DEBUGS(LOG_INV) << "using item-by-item calls to slam folder, id " << folder_id + << " new contents: " << ll_pretty_print_sd(contents) << LL_ENDL; for (LLSD::array_const_iterator it = contents.beginArray(); it != contents.endArray(); ++it) @@ -1926,7 +1932,7 @@ void menu_create_inventory_item(LLInventoryPanel* panel, LLFolderBridge *bridge, } else { - LL_WARNS() << "Can't create unrecognized type " << type_name << LL_ENDL; + LL_WARNS(LOG_INV) << "Can't create unrecognized type " << type_name << LL_ENDL; } } panel->getRootFolder()->setNeedsAutoRename(TRUE); @@ -2161,7 +2167,7 @@ LLViewerInventoryItem *LLViewerInventoryItem::getLinkedItem() const LLViewerInventoryItem *linked_item = gInventory.getItem(mAssetUUID); if (linked_item && linked_item->getIsLinkType()) { - LL_WARNS() << "Warning: Accessing link to link" << LL_ENDL; + LL_WARNS(LOG_INV) << "Warning: Accessing link to link" << LL_ENDL; return NULL; } return linked_item; diff --git a/indra/newview/llviewermenu.cpp b/indra/newview/llviewermenu.cpp index 8c9429c05d..10c77da5f3 100755 --- a/indra/newview/llviewermenu.cpp +++ b/indra/newview/llviewermenu.cpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2002&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2010, Linden Research, Inc. + * Copyright (C) 2014, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -8258,7 +8258,7 @@ class LLWorldEnableEnvSettings : public view_listener_t } else { - llwarns << "Unknown item" << llendl; + LL_WARNS() << "Unknown time-of-day item: " << tod << LL_ENDL; } } return result; -- cgit v1.2.3 From b64ef2ecd4af5265483527f2ef030554133ab137 Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Fri, 22 Aug 2014 19:00:11 -0400 Subject: Fix ambiguous constructor due to LLSD access which broke *ix builds. --- indra/newview/llinventorymodelbackgroundfetch.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'indra') diff --git a/indra/newview/llinventorymodelbackgroundfetch.cpp b/indra/newview/llinventorymodelbackgroundfetch.cpp index 443c54df42..7b944edf45 100755 --- a/indra/newview/llinventorymodelbackgroundfetch.cpp +++ b/indra/newview/llinventorymodelbackgroundfetch.cpp @@ -870,7 +870,7 @@ void BGFolderHttpHandler::processFailure(LLCore::HttpStatus status, LLCore::Http ++folder_it) { LLSD folder_sd(*folder_it); - LLUUID folder_id(folder_sd["folder_id"]); + LLUUID folder_id(folder_sd["folder_id"].asUUID()); const BOOL recursive = getIsRecursive(folder_id); fetcher->addRequestAtFront(folder_id, recursive, true); } @@ -900,7 +900,7 @@ void BGFolderHttpHandler::processFailure(const char * const reason, LLCore::Http ++folder_it) { LLSD folder_sd(*folder_it); - LLUUID folder_id(folder_sd["folder_id"]); + LLUUID folder_id(folder_sd["folder_id"].asUUID()); const BOOL recursive = getIsRecursive(folder_id); fetcher->addRequestAtFront(folder_id, recursive, true); } -- cgit v1.2.3 From bbf9de9c6717f38a77a39d42d8493d275d558db9 Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Tue, 26 Aug 2014 18:33:14 -0400 Subject: Bring better error handling to inventory item and folder fetching. First, introduced some LLSD-based interfaces to the llcorehttp code using utils classes (in llcorehttputil). I've kept LLSD out of the llcorehttp library up to now and will continue to do that. Functions provide a requestPost based on LLSD body and conversion utils for HttpResponse-to-LLSD and HttpResponse-to-string conversions. Inventory fetch operations now do more thorough error checking including 200-with-error status checking. Still do retry forever on folders though I don't like that. --- indra/llmessage/CMakeLists.txt | 4 + indra/llmessage/llcorehttputil.cpp | 137 ++++++++++++++++++++++ indra/llmessage/llcorehttputil.h | 115 ++++++++++++++++++ indra/newview/llinventorymodel.cpp | 76 +++++++----- indra/newview/llinventorymodelbackgroundfetch.cpp | 115 +++++++++++++----- 5 files changed, 386 insertions(+), 61 deletions(-) create mode 100644 indra/llmessage/llcorehttputil.cpp create mode 100644 indra/llmessage/llcorehttputil.h (limited to 'indra') diff --git a/indra/llmessage/CMakeLists.txt b/indra/llmessage/CMakeLists.txt index 8bd134dc84..40eddcb0ab 100755 --- a/indra/llmessage/CMakeLists.txt +++ b/indra/llmessage/CMakeLists.txt @@ -6,6 +6,7 @@ include(00-Common) include(GoogleMock) include(LLAddBuildTest) include(LLCommon) +include(LLCoreHttp) include(LLMath) include(LLMessage) include(LLVFS) @@ -18,6 +19,7 @@ include_directories (${CMAKE_CURRENT_SOURCE_DIR}) include_directories( ${LLCOMMON_INCLUDE_DIRS} + ${LLCOREHTTP_INCLUDE_DIRS} ${LLMATH_INCLUDE_DIRS} ${LLMESSAGE_INCLUDE_DIRS} ${LLVFS_INCLUDE_DIRS} @@ -36,6 +38,7 @@ set(llmessage_SOURCE_FILES llchainio.cpp llcircuit.cpp llclassifiedflags.cpp + llcorehttputil.cpp llcurl.cpp lldatapacker.cpp lldispatcher.cpp @@ -124,6 +127,7 @@ set(llmessage_HEADER_FILES llcipher.h llcircuit.h llclassifiedflags.h + llcorehttputil.h llcurl.h lldatapacker.h lldbstrings.h diff --git a/indra/llmessage/llcorehttputil.cpp b/indra/llmessage/llcorehttputil.cpp new file mode 100644 index 0000000000..8d09aac971 --- /dev/null +++ b/indra/llmessage/llcorehttputil.cpp @@ -0,0 +1,137 @@ +/** + * @file llcorehttputil.cpp + * @date 2014-08-25 + * @brief Implementation of adapter and utility classes expanding the llcorehttp interfaces. + * + * $LicenseInfo:firstyear=2014&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2014, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include "linden_common.h" + +#include + +#include "llcorehttputil.h" +#include "llsdserialize.h" + + +using namespace LLCore; + + +namespace LLCoreHttpUtil +{ + + +bool responseToLLSD(HttpResponse * response, bool log, LLSD & out_llsd) +{ + // Convert response to LLSD + BufferArray * body(response->getBody()); + if (! body || ! body->size()) + { + return false; + } + + LLCore::BufferArrayStream bas(body); + LLSD body_llsd; + S32 parse_status(LLSDSerialize::fromXML(body_llsd, bas, log)); + if (LLSDParser::PARSE_FAILURE == parse_status){ + return false; + } + out_llsd = body_llsd; + return true; +} + + +HttpHandle requestPostWithLLSD(HttpRequest * request, + HttpRequest::policy_t policy_id, + HttpRequest::priority_t priority, + const std::string & url, + const LLSD & body, + HttpOptions * options, + HttpHeaders * headers, + HttpHandler * handler) +{ + HttpHandle handle(LLCORE_HTTP_HANDLE_INVALID); + + BufferArray * ba = new BufferArray(); + BufferArrayStream bas(ba); + LLSDSerialize::toXML(body, bas); + + handle = request->requestPost(policy_id, + priority, + url, + ba, + options, + headers, + handler); + ba->release(); + return handle; +} + + +std::string responseToString(LLCore::HttpResponse * response) +{ + static const std::string empty("[Empty]"); + + if (! response) + { + return empty; + } + + BufferArray * body(response->getBody()); + if (! body || ! body->size()) + { + return empty; + } + + // Attempt to parse as LLSD regardless of content-type + LLSD body_llsd; + if (responseToLLSD(response, false, body_llsd)) + { + std::ostringstream tmp; + + LLSDSerialize::toPrettyNotation(body_llsd, tmp); + std::size_t temp_len(tmp.tellp()); + + if (temp_len) + { + return tmp.str().substr(0, std::min(temp_len, std::size_t(1024))); + } + } + else + { + // *TODO: More elaborate forms based on Content-Type as needed. + char content[1024]; + + size_t len(body->read(0, content, sizeof(content))); + if (len) + { + return std::string(content, 0, len); + } + } + + // Default + return empty; +} + + +} // end namespace LLCoreHttpUtil + diff --git a/indra/llmessage/llcorehttputil.h b/indra/llmessage/llcorehttputil.h new file mode 100644 index 0000000000..d40172bc7a --- /dev/null +++ b/indra/llmessage/llcorehttputil.h @@ -0,0 +1,115 @@ +/** + * @file llcorehttputil.h + * @date 2014-08-25 + * @brief Adapter and utility classes expanding the llcorehttp interfaces. + * + * $LicenseInfo:firstyear=2014&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2014, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef LL_LLCOREHTTPUTIL_H +#define LL_LLCOREHTTPUTIL_H + +#include + +#include "httpcommon.h" +#include "httprequest.h" +#include "httpresponse.h" +#include "httpheaders.h" +#include "httpoptions.h" +#include "httphandler.h" +#include "bufferarray.h" +#include "bufferstream.h" +#include "llsd.h" + +/// +/// The base llcorehttp library implements many HTTP idioms +/// used in the viewer but not all. That library intentionally +/// avoids the use of LLSD and its conventions which aren't +/// universally applicable. This module, using namespace +/// LLCoreHttpUtil, provides the additional helper functions +/// that support idiomatic LLSD transport via the newer +/// llcorehttp library. +/// +namespace LLCoreHttpUtil +{ + +/// Attempt to convert a response object's contents to LLSD. +/// It is expected that the response body will be of non-zero +/// length on input but basic checks will be performed and +/// and error (false status) returned if there is no data. +/// If there is data but it cannot be successfully parsed, +/// an error is also returned. If successfully parsed, +/// the output LLSD object, out_llsd, is written with the +/// result and true is returned. +/// +/// @arg response Response object as returned in +/// in an HttpHandler onCompleted() callback. +/// @arg log If true, LLSD parser will emit errors +/// as LL_INFOS-level messages as it parses. +/// Otherwise, it *should* be a quiet parse. +/// @arg out_llsd Output LLSD object written only upon +/// successful parse of the response object. +/// +/// @return Returns true (and writes to out_llsd) if +/// parse was successful. False otherwise. +/// +bool responseToLLSD(LLCore::HttpResponse * response, + bool log, + LLSD & out_llsd); + +/// Create a std::string representation of a response object +/// suitable for logging. Mainly intended for logging of +/// failures and debug information. This won't be fast, +/// just adequate. +std::string responseToString(LLCore::HttpResponse * response); + + +/// Issue a standard HttpRequest::requestPost() call but using +/// and LLSD object as the request body. Conventions are the +/// same as with that method. Caller is expected to provide +/// an HttpHeaders object with a correct 'Content-Type:' header. +/// One will not be provided by this call. You might look after +/// the 'Accept:' header as well. +/// +/// @return If request is successfully issued, the +/// HttpHandle representing the request. +/// On error, LLCORE_HTTP_HANDLE_INVALID +/// is returned and caller can fetch detailed +/// status with the getStatus() method on the +/// request object. In case of error, no +/// request is queued and caller may need to +/// perform additional cleanup such as freeing +/// a now-useless HttpHandler object. +/// +LLCore::HttpHandle requestPostWithLLSD(LLCore::HttpRequest * request, + LLCore::HttpRequest::policy_t policy_id, + LLCore::HttpRequest::priority_t priority, + const std::string & url, + const LLSD & body, + LLCore::HttpOptions * options, + LLCore::HttpHeaders * headers, + LLCore::HttpHandler * handler); + +} // end namespace LLCoreHttpUtil + + +#endif // LL_LLCOREHTTPUTIL_H diff --git a/indra/newview/llinventorymodel.cpp b/indra/newview/llinventorymodel.cpp index 2dd31f047f..5273fb6d96 100755 --- a/indra/newview/llinventorymodel.cpp +++ b/indra/newview/llinventorymodel.cpp @@ -55,7 +55,7 @@ #include "llsdutil.h" #include "bufferarray.h" #include "bufferstream.h" -#include "llsdserialize.h" +#include "llcorehttputil.h" //#define DIFF_INVENTORY_FILES #ifdef DIFF_INVENTORY_FILES @@ -75,12 +75,6 @@ BOOL LLInventoryModel::sFirstTimeInViewer2 = TRUE; static const char CACHE_FORMAT_STRING[] = "%s.inv"; static const char * const LOG_INV("Inventory"); -static std::string dumpResponse() -{ - return std::string("ADD SOMETHING MEANINGFUL HERE"); -} - - struct InventoryIDPtrLess { bool operator()(const LLViewerInventoryCategory* i1, const LLViewerInventoryCategory* i2) const @@ -2468,18 +2462,15 @@ LLCore::HttpHandle LLInventoryModel::requestPost(bool foreground, LLCore::HttpRequest * request(foreground ? mHttpRequestFG : mHttpRequestBG); LLCore::HttpHandle handle(LLCORE_HTTP_HANDLE_INVALID); - LLCore::BufferArray * ba = new LLCore::BufferArray; - LLCore::BufferArrayStream bas(ba); - LLSDSerialize::toXML(body, bas); - handle = request->requestPost(mHttpPolicyClass, - (foreground ? mHttpPriorityFG : mHttpPriorityBG), - url, - ba, - mHttpOptions, - mHttpHeaders, - handler); - ba->release(); + handle = LLCoreHttpUtil::requestPostWithLLSD(request, + mHttpPolicyClass, + (foreground ? mHttpPriorityFG : mHttpPriorityBG), + url, + body, + mHttpOptions, + mHttpHeaders, + handler); if (LLCORE_HTTP_HANDLE_INVALID == handle) { LLCore::HttpStatus status(request->getStatus()); @@ -3981,6 +3972,7 @@ void LLInventoryModel::FetchItemHttpHandler::onCompleted(LLCore::HttpHandle hand LLCore::HttpResponse * response) { LLCore::HttpStatus status(response->getStatus()); + // status = LLCore::HttpStatus(404); // Dev tool to force error handling if (! status) { processFailure(status, response); @@ -3988,23 +3980,41 @@ void LLInventoryModel::FetchItemHttpHandler::onCompleted(LLCore::HttpHandle hand else { LLCore::BufferArray * body(response->getBody()); + // body = NULL; // Dev tool to force error handling if (! body || ! body->size()) { LL_WARNS(LOG_INV) << "Missing data in inventory item query." << LL_ENDL; processFailure("HTTP response for inventory item query missing body", response); goto only_exit; } - - LLCore::BufferArrayStream bas(body); + + // body->write(0, "Garbage Response", 16); // Dev tool to force error handling LLSD body_llsd; - S32 parse_status(LLSDSerialize::fromXML(body_llsd, bas)); - if (LLSDParser::PARSE_FAILURE == parse_status) + if (! LLCoreHttpUtil::responseToLLSD(response, true, body_llsd)) { // INFOS-level logging will occur on the parsed failure processFailure("HTTP response for inventory item query has malformed LLSD", response); goto only_exit; } + // Expect top-level structure to be a map + // body_llsd = LLSD::emptyArray(); // Dev tool to force error handling + if (! body_llsd.isMap()) + { + processFailure("LLSD response for inventory item not a map", response); + goto only_exit; + } + + // Check for 200-with-error failures + // body_llsd["error"] = LLSD::emptyMap(); // Dev tool to force error handling + // body_llsd["error"]["identifier"] = "Development"; + // body_llsd["error"]["message"] = "You left development code in the viewer"; + if (body_llsd.has("error")) + { + processFailure("Inventory application error (200-with-error)", response); + goto only_exit; + } + // Okay, process data if possible processData(body_llsd, response); } @@ -4016,12 +4026,6 @@ only_exit: void LLInventoryModel::FetchItemHttpHandler::processData(LLSD & content, LLCore::HttpResponse * response) { - if (! content.isMap()) - { - processFailure("LLSD response for inventory item not a map", response); - return; - } - start_new_inventory_observer(); #if 0 @@ -4083,7 +4087,7 @@ void LLInventoryModel::FetchItemHttpHandler::processData(LLSD & content, LLCore: { changes |= gInventory.updateItem(*it); } - // *HUH: Have computed changes, nothing uses it. + // *HUH: Have computed 'changes', nothing uses it. gInventory.notifyObservers(); gViewerWindow->getWindow()->decBusyCount(); @@ -4092,13 +4096,23 @@ void LLInventoryModel::FetchItemHttpHandler::processData(LLSD & content, LLCore: void LLInventoryModel::FetchItemHttpHandler::processFailure(LLCore::HttpStatus status, LLCore::HttpResponse * response) { - LL_WARNS(LOG_INV) << dumpResponse() << LL_ENDL; + const std::string & ct(response->getContentType()); + LL_WARNS(LOG_INV) << "Inventory item fetch failure\n" + << "[Status: " << status.toTerseString() << "]\n" + << "[Reason: " << status.toString() << "]\n" + << "[Content-type: " << ct << "]\n" + << "[Content (abridged): " + << LLCoreHttpUtil::responseToString(response) << "]" << LL_ENDL; gInventory.notifyObservers(); } void LLInventoryModel::FetchItemHttpHandler::processFailure(const char * const reason, LLCore::HttpResponse * response) { - LL_WARNS(LOG_INV) << dumpResponse() << LL_ENDL; + LL_WARNS(LOG_INV) << "Inventory item fetch failure\n" + << "[Status: internal error]\n" + << "[Reason: " << reason << "]\n" + << "[Content (abridged): " + << LLCoreHttpUtil::responseToString(response) << "]" << LL_ENDL; gInventory.notifyObservers(); } diff --git a/indra/newview/llinventorymodelbackgroundfetch.cpp b/indra/newview/llinventorymodelbackgroundfetch.cpp index 7b944edf45..0c04a9c039 100755 --- a/indra/newview/llinventorymodelbackgroundfetch.cpp +++ b/indra/newview/llinventorymodelbackgroundfetch.cpp @@ -40,17 +40,24 @@ #include "llhttpconstants.h" #include "bufferarray.h" #include "bufferstream.h" -#include "llsdserialize.h" +#include "llcorehttputil.h" namespace { +// // Http request handler class for single inventory item requests. // // We'll use a handler-per-request pattern here rather than // a shared handler. Mainly convenient as this was converted // from a Responder class model. // +// Derives from and is identical to the normal FetchItemHttpHandler +// except that: 1) it uses the background request object which is +// updated more slowly than the foreground and 2) keeps a count of +// active requests on the LLInventoryModelBackgroundFetch object +// to indicate outstanding operations are in-flight. +// class BGItemHttpHandler : public LLInventoryModel::FetchItemHttpHandler { LOG_CLASS(BGItemHttpHandler); @@ -74,6 +81,10 @@ protected: // Http request handler class for folders. +// +// Handler for FetchInventoryDescendents2 and FetchLibDescendents2 +// caps requests for folders. +// class BGFolderHttpHandler : public LLCore::HttpHandler { LOG_CLASS(BGFolderHttpHandler); @@ -116,12 +127,6 @@ const S32 MAX_FETCH_RETRIES = 10; const char * const LOG_INV("Inventory"); -std::string dumpResponse() -{ - return std::string("ADD SOMETHING MEANINGFUL HERE"); -} - - } // end of namespace anonymous @@ -471,6 +476,10 @@ void LLInventoryModelBackgroundFetch::bulkFetch() return; } + // *TODO: These values could be tweaked at runtime to effect + // a fast/slow fetch throttle. Once login is complete and the scene + // is mostly loaded, we could turn up the throttle and fill missing + // inventory more quickly. static const S32 max_concurrent_fetches(12); // Outstanding requests, not connections static const F32 new_min_time(0.5f); // *HACK: Clean this up when old code goes away entirely. static const U32 max_batch_size(10); @@ -482,7 +491,7 @@ void LLInventoryModelBackgroundFetch::bulkFetch() if (mFetchCount) { - // Process completed HTTP requests + // Process completed background HTTP requests gInventory.handleResponses(false); } @@ -497,6 +506,8 @@ void LLInventoryModelBackgroundFetch::bulkFetch() const U32 sort_order(gSavedSettings.getU32(LLInventoryPanel::DEFAULT_SORT_ORDER) & 0x1); + // *TODO: Think I'd like to get a shared pointer to this and share it + // among all the folder requests. uuid_vec_t recursive_cats; LLSD folder_request_body; @@ -673,7 +684,9 @@ bool LLInventoryModelBackgroundFetch::fetchQueueContainsNoDescendentsOf(const LL } +// =============================== // Anonymous Namespace Definitions +// =============================== namespace { @@ -683,30 +696,53 @@ namespace void BGFolderHttpHandler::onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * response) { LLCore::HttpStatus status(response->getStatus()); + // status = LLCore::HttpStatus(404); // Dev tool to force error handling if (! status) { processFailure(status, response); } else { + // Response body should be present. LLCore::BufferArray * body(response->getBody()); + // body = NULL; // Dev tool to force error handling if (! body || ! body->size()) { LL_WARNS(LOG_INV) << "Missing data in inventory folder query." << LL_ENDL; processFailure("HTTP response missing expected body", response); goto only_exit; } - - LLCore::BufferArrayStream bas(body); + + // Could test 'Content-Type' header but probably unreliable. + + // Convert response to LLSD + // body->write(0, "Garbage Response", 16); // Dev tool to force error handling LLSD body_llsd; - S32 parse_status(LLSDSerialize::fromXML(body_llsd, bas)); - if (LLSDParser::PARSE_FAILURE == parse_status) + if (! LLCoreHttpUtil::responseToLLSD(response, true, body_llsd)) { // INFOS-level logging will occur on the parsed failure processFailure("HTTP response contained malformed LLSD", response); goto only_exit; } + // Expect top-level structure to be a map + // body_llsd = LLSD::emptyArray(); // Dev tool to force error handling + if (! body_llsd.isMap()) + { + processFailure("LLSD response not a map", response); + goto only_exit; + } + + // Check for 200-with-error failures + // body_llsd["error"] = LLSD::emptyMap(); // Dev tool to force error handling + // body_llsd["error"]["identifier"] = "Development"; + // body_llsd["error"]["message"] = "You left development code in the viewer"; + if (body_llsd.has("error")) + { + processFailure("Inventory application error (200-with-error)", response); + goto only_exit; + } + // Okay, process data if possible processData(body_llsd, response); } @@ -719,13 +755,12 @@ only_exit: void BGFolderHttpHandler::processData(LLSD & content, LLCore::HttpResponse * response) { - if (! content.isMap()) - { - processFailure("LLSD response not a map", response); - return; - } - LLInventoryModelBackgroundFetch * fetcher(LLInventoryModelBackgroundFetch::getInstance()); + + // API V2 and earlier should probably be testing for "error" map + // in response as an application-level error. + + // Instead, we assume success and attempt to extract information. if (content.has("folders")) { LLSD folders(content["folders"]); @@ -836,7 +871,7 @@ void BGFolderHttpHandler::processData(LLSD & content, LLCore::HttpResponse * res folder_it != bad_folders.endArray(); ++folder_it) { - // *TODO: Stop copying data + // *TODO: Stop copying data [ed: this isn't copying data] LLSD folder_sd(*folder_it); // These folders failed on the dataserver. We probably don't want to retry them. @@ -857,14 +892,26 @@ void BGFolderHttpHandler::processData(LLSD & content, LLCore::HttpResponse * res void BGFolderHttpHandler::processFailure(LLCore::HttpStatus status, LLCore::HttpResponse * response) { - LL_WARNS(LOG_INV) << dumpResponse() << LL_ENDL; + const std::string & ct(response->getContentType()); + LL_WARNS(LOG_INV) << "Inventory folder fetch failure\n" + << "[Status: " << status.toTerseString() << "]\n" + << "[Reason: " << status.toString() << "]\n" + << "[Content-type: " << ct << "]\n" + << "[Content (abridged): " + << LLCoreHttpUtil::responseToString(response) << "]" << LL_ENDL; + + // Could use a 404 test here to try to detect revoked caps... + + // This was originally the request retry logic for the inventory + // request which tested on HTTP_INTERNAL_ERROR status. This + // retry logic was unbounded and lacked discrimination as to the + // cause of the retry. The new http library should be doing + // adquately on retries but I want to keep the structure of a + // retry for reference. LLInventoryModelBackgroundFetch *fetcher = LLInventoryModelBackgroundFetch::getInstance(); - - LL_INFOS(LOG_INV) << dumpResponse() << LL_ENDL; - - // *FIX: Not the correct test here... - if (status == LLCore::HttpStatus(408)) // timed out or curl failure + if (false) { + // timed out or curl failure for (LLSD::array_const_iterator folder_it = mRequestSD["folders"].beginArray(); folder_it != mRequestSD["folders"].endArray(); ++folder_it) @@ -888,12 +935,20 @@ void BGFolderHttpHandler::processFailure(LLCore::HttpStatus status, LLCore::Http void BGFolderHttpHandler::processFailure(const char * const reason, LLCore::HttpResponse * response) { - LL_WARNS(LOG_INV) << dumpResponse() << LL_ENDL; + LL_WARNS(LOG_INV) << "Inventory folder fetch failure\n" + << "[Status: internal error]\n" + << "[Reason: " << reason << "]\n" + << "[Content (abridged): " + << LLCoreHttpUtil::responseToString(response) << "]" << LL_ENDL; + + // Reverse of previous processFailure() method, this is invoked + // when response structure is found to be invalid. Original + // always re-issued the request (without limit). This does + // the same but be aware that this may be a source of problems. + // Philosophy is that inventory folders are so essential to + // operation that this is a reasonable action. LLInventoryModelBackgroundFetch *fetcher = LLInventoryModelBackgroundFetch::getInstance(); - - LL_INFOS(LOG_INV) << dumpResponse() << LL_ENDL; - - if (true /* status == LLCore::HttpStatus(408)*/) // timed out or curl failure + if (true) { for (LLSD::array_const_iterator folder_it = mRequestSD["folders"].beginArray(); folder_it != mRequestSD["folders"].endArray(); -- cgit v1.2.3 From 0c20beda6800149ee71a307ca4e943b5bba56908 Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Thu, 4 Sep 2014 16:57:44 -0400 Subject: Pipelining work. Extend transfer timeout by the pipeline depth as transfers can appear delayed with deep pipelining and more requests in the pool. Added bad HTTP status error (typically getting a 0 back as HTTP status from libcurl) to the list of retryable errors. There's a response stream problem with libcurl and pipelining that induces this problem. Retrying helps but may not be entirely safe. Watch bug 1420 on the libcurl sourceforge bug tracker. Extend options of test/example program to include un-ranged requests. Document the excessive data transfer induced when ranged requests are disabled. This is an abnormal mode for very rare users so we'll just eat that for now. --- indra/llcorehttp/_httplibcurl.cpp | 15 +++++++---- indra/llcorehttp/_httpoprequest.cpp | 34 ++++++++++++++++++------- indra/llcorehttp/examples/http_texture_load.cpp | 21 ++++++++++++--- indra/llcorehttp/httpcommon.cpp | 13 ++++++++-- indra/newview/app_settings/settings.xml | 4 +-- indra/newview/llmeshrepository.cpp | 13 +++++++++- 6 files changed, 78 insertions(+), 22 deletions(-) (limited to 'indra') diff --git a/indra/llcorehttp/_httplibcurl.cpp b/indra/llcorehttp/_httplibcurl.cpp index b46833a1f3..cfbe0fd2bb 100755 --- a/indra/llcorehttp/_httplibcurl.cpp +++ b/indra/llcorehttp/_httplibcurl.cpp @@ -217,8 +217,17 @@ void HttpLibcurl::addOp(HttpOpRequest * op) } // Make the request live - curl_multi_add_handle(mMultiHandles[op->mReqPolicy], op->mCurlHandle); + CURLMcode code; + code = curl_multi_add_handle(mMultiHandles[op->mReqPolicy], op->mCurlHandle); + if (CURLM_OK != code) + { + // *TODO: Better cleanup and recovery but not much we can do here. + check_curl_multi_code(code); + return; + } op->mCurlActive = true; + mActiveOps.insert(op); + ++mActiveHandles[op->mReqPolicy]; if (op->mTracing > HTTP_TRACE_OFF) { @@ -230,10 +239,6 @@ void HttpLibcurl::addOp(HttpOpRequest * op) << ", Readies: " << policy.getReadyCount(op->mReqPolicy) << LL_ENDL; } - - // On success, make operation active - mActiveOps.insert(op); - ++mActiveHandles[op->mReqPolicy]; } diff --git a/indra/llcorehttp/_httpoprequest.cpp b/indra/llcorehttp/_httpoprequest.cpp index eb664fdced..38c1f1e78a 100755 --- a/indra/llcorehttp/_httpoprequest.cpp +++ b/indra/llcorehttp/_httpoprequest.cpp @@ -378,6 +378,7 @@ void HttpOpRequest::setupCommon(HttpRequest::policy_t policy_id, // Junk may be left around from a failed request and that // needs to be cleaned out. // +// *TODO: Move this to _httplibcurl where it belongs. HttpStatus HttpOpRequest::prepareRequest(HttpService * service) { CURLcode code; @@ -411,8 +412,9 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service) // *FIXME: better error handling later HttpStatus status; - // Get global policy options - HttpPolicyGlobal & policy(service->getPolicy().getGlobalOptions()); + // Get global and class policy options + HttpPolicyGlobal & gpolicy(service->getPolicy().getGlobalOptions()); + HttpPolicyClass & cpolicy(service->getPolicy().getClassOptions(mReqPolicy)); mCurlHandle = LLCurl::createStandardCurlHandle(); if (! mCurlHandle) @@ -462,30 +464,30 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service) code = curl_easy_setopt(mCurlHandle, CURLOPT_SSL_VERIFYHOST, 0); check_curl_easy_code(code, CURLOPT_SSL_VERIFYHOST); - if (policy.mUseLLProxy) + if (gpolicy.mUseLLProxy) { // Use the viewer-based thread-safe API which has a // fast/safe check for proxy enable. Would like to // encapsulate this someway... LLProxy::getInstance()->applyProxySettings(mCurlHandle); } - else if (policy.mHttpProxy.size()) + else if (gpolicy.mHttpProxy.size()) { // *TODO: This is fine for now but get fuller socks5/ // authentication thing going later.... - code = curl_easy_setopt(mCurlHandle, CURLOPT_PROXY, policy.mHttpProxy.c_str()); + code = curl_easy_setopt(mCurlHandle, CURLOPT_PROXY, gpolicy.mHttpProxy.c_str()); check_curl_easy_code(code, CURLOPT_PROXY); code = curl_easy_setopt(mCurlHandle, CURLOPT_PROXYTYPE, CURLPROXY_HTTP); check_curl_easy_code(code, CURLOPT_PROXYTYPE); } - if (policy.mCAPath.size()) + if (gpolicy.mCAPath.size()) { - code = curl_easy_setopt(mCurlHandle, CURLOPT_CAPATH, policy.mCAPath.c_str()); + code = curl_easy_setopt(mCurlHandle, CURLOPT_CAPATH, gpolicy.mCAPath.c_str()); check_curl_easy_code(code, CURLOPT_CAPATH); } - if (policy.mCAFile.size()) + if (gpolicy.mCAFile.size()) { - code = curl_easy_setopt(mCurlHandle, CURLOPT_CAINFO, policy.mCAFile.c_str()); + code = curl_easy_setopt(mCurlHandle, CURLOPT_CAINFO, gpolicy.mCAFile.c_str()); check_curl_easy_code(code, CURLOPT_CAINFO); } @@ -594,6 +596,20 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service) { xfer_timeout = timeout; } + if (cpolicy.mPipelining > 1L) + { + // Pipelining affects both connection and transfer timeout values. + // Requests that are added to a pipeling immediately have completed + // their connection so the connection delay tends to be less than + // the non-pipelined value. Transfers are the opposite. Transfer + // timeout starts once the connection is established and completion + // can be delayed due to the pipelined requests ahead. So, it's + // a handwave but bump the transfer timeout up by the pipelining + // depth to give some room. + // + // *TODO: Find a better scheme than timeouts to guarantee liveness. + xfer_timeout *= cpolicy.mPipelining; + } code = curl_easy_setopt(mCurlHandle, CURLOPT_TIMEOUT, xfer_timeout); check_curl_easy_code(code, CURLOPT_TIMEOUT); code = curl_easy_setopt(mCurlHandle, CURLOPT_CONNECTTIMEOUT, timeout); diff --git a/indra/llcorehttp/examples/http_texture_load.cpp b/indra/llcorehttp/examples/http_texture_load.cpp index 88692c3f69..b76c874557 100755 --- a/indra/llcorehttp/examples/http_texture_load.cpp +++ b/indra/llcorehttp/examples/http_texture_load.cpp @@ -102,6 +102,7 @@ public: public: bool mVerbose; bool mRandomRange; + bool mNoRange; int mRequestLowWater; int mRequestHighWater; handle_set_t mHandles; @@ -162,10 +163,11 @@ int main(int argc, char** argv) { LLCore::HttpStatus status; bool do_random(false); + bool do_whole(false); bool do_verbose(false); int option(-1); - while (-1 != (option = getopt(argc, argv, "u:c:h?RvH:p:t:"))) + while (-1 != (option = getopt(argc, argv, "u:c:h?RwvH:p:t:"))) { switch (option) { @@ -236,6 +238,12 @@ int main(int argc, char** argv) case 'R': do_random = true; + do_whole = false; + break; + + case 'w': + do_whole = true; + do_random = false; break; case 'v': @@ -307,6 +315,7 @@ int main(int argc, char** argv) ws.mUrl = url_format; ws.loadAssetUuids(uuids); ws.mRandomRange = do_random; + ws.mNoRange = do_whole; ws.mVerbose = do_verbose; ws.mRequestHighWater = highwater; ws.mRequestLowWater = ws.mRequestHighWater / 2; @@ -381,6 +390,7 @@ void usage(std::ostream & out) " -u printf-style format string for URL generation\n" " Default: " << url_format << "\n" " -R Issue GETs with random Range: headers\n" + " -w Issue GETs without Range: headers to get whole object\n" " -c Maximum connection concurrency. Range: [1..100]\n" " Default: " << concurrency_limit << "\n" " -H HTTP request highwater (requests fed to llcorehttp).\n" @@ -400,6 +410,7 @@ WorkingSet::WorkingSet() : LLCore::HttpHandler(), mVerbose(false), mRandomRange(false), + mNoRange(false), mRemaining(200), mLimit(200), mAt(0), @@ -449,8 +460,12 @@ bool WorkingSet::reload(LLCore::HttpRequest * hr, LLCore::HttpOptions * opt) #else snprintf(buffer, sizeof(buffer), mUrl.c_str(), mAssets[mAt].mUuid.c_str()); #endif - int offset(mRandomRange ? ((unsigned long) rand()) % 1000000UL : mAssets[mAt].mOffset); - int length(mRandomRange ? ((unsigned long) rand()) % 1000000UL : mAssets[mAt].mLength); + int offset(mNoRange + ? 0 + : (mRandomRange ? ((unsigned long) rand()) % 1000000UL : mAssets[mAt].mOffset)); + int length(mNoRange + ? 0 + : (mRandomRange ? ((unsigned long) rand()) % 1000000UL : mAssets[mAt].mLength)); LLCore::HttpHandle handle; if (offset || length) diff --git a/indra/llcorehttp/httpcommon.cpp b/indra/llcorehttp/httpcommon.cpp index c2f15155ac..9bcf7ac5e3 100755 --- a/indra/llcorehttp/httpcommon.cpp +++ b/indra/llcorehttp/httpcommon.cpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012-2013, Linden Research, Inc. + * Copyright (C) 2012-2014, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -219,6 +219,13 @@ std::string HttpStatus::toTerseString() const // Pass true on statuses that might actually be cleared by a // retry. Library failures, calling problems, etc. aren't // going to be fixed by squirting bits all over the Net. +// +// HE_INVALID_HTTP_STATUS is special. As of 7.37.0, there are +// some scenarios where response processing in libcurl appear +// to go wrong and response data is corrupted. A side-effect +// of this is that the HTTP status is read as 0 from the library. +// See libcurl bug report 1420 (https://sourceforge.net/p/curl/bugs/1420/) +// for details. bool HttpStatus::isRetryable() const { static const HttpStatus cant_connect(HttpStatus::EXT_CURL_EASY, CURLE_COULDNT_CONNECT); @@ -231,6 +238,7 @@ bool HttpStatus::isRetryable() const static const HttpStatus post_error(HttpStatus::EXT_CURL_EASY, CURLE_HTTP_POST_ERROR); static const HttpStatus partial_file(HttpStatus::EXT_CURL_EASY, CURLE_PARTIAL_FILE); static const HttpStatus inv_cont_range(HttpStatus::LLCORE, HE_INV_CONTENT_RANGE_HDR); + static const HttpStatus inv_status(HttpStatus::LLCORE, HE_INVALID_HTTP_STATUS); return ((isHttpStatus() && mType >= 499 && mType <= 599) || // Include special 499 in retryables *this == cant_connect || // Connection reset/endpoint problems @@ -242,7 +250,8 @@ bool HttpStatus::isRetryable() const *this == op_timedout || // Timer expired *this == post_error || // Transport problem *this == partial_file || // Data inconsistency in response - *this == inv_cont_range); // Short data read disagrees with content-range + *this == inv_cont_range || // Short data read disagrees with content-range + *this == inv_status); // Inv status can reflect internal state problem in libcurl } } // end namespace LLCore diff --git a/indra/newview/app_settings/settings.xml b/indra/newview/app_settings/settings.xml index 0607579a08..ab15a03229 100755 --- a/indra/newview/app_settings/settings.xml +++ b/indra/newview/app_settings/settings.xml @@ -4459,7 +4459,7 @@ HttpPipelining Comment - If true, viewer will pipeline HTTP requests to servers. + If true, viewer will attempt to pipeline HTTP requests. Persist 1 Type @@ -4470,7 +4470,7 @@ HttpRangeRequestsDisable Comment - If true, viewer will not issued range GET requests for meshes and textures. May resolve problems with certain ISPs and networking gear. + If true, viewer will not issue GET requests with 'Range:' headers for meshes and textures. May resolve problems with certain ISPs and networking gear. Persist 1 Type diff --git a/indra/newview/llmeshrepository.cpp b/indra/newview/llmeshrepository.cpp index 6477389d4c..a6707392fe 100755 --- a/indra/newview/llmeshrepository.cpp +++ b/indra/newview/llmeshrepository.cpp @@ -2644,6 +2644,17 @@ void LLMeshRepository::cacheOutgoingMesh(LLMeshUploadData& data, LLSD& header) } +// Handle failed or successful requests for mesh assets. +// +// Support for 200 responses was added for several reasons. One, +// a service or cache can ignore range headers and give us a +// 200 with full asset should it elect to. We also support +// a debug flag which disables range requests for those very +// few users that have some sort of problem with their networking +// services. But the 200 response handling is suboptimal: rather +// than cache the whole asset, we just extract the part that would +// have been sent in a 206 and process that. Inefficient but these +// are cases far off the norm. void LLMeshHandlerBase::onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * response) { mProcessed = true; @@ -2685,7 +2696,7 @@ void LLMeshHandlerBase::onCompleted(LLCore::HttpHandle handle, LLCore::HttpRespo if (par_status == status) { - // 216 case + // 206 case response->getRange(&offset, &length, &full_length); if (! offset && ! length) { -- cgit v1.2.3 From bd0a9b6b4c929f640164b623f5183d371404bb96 Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Tue, 9 Sep 2014 15:30:54 -0400 Subject: LLSlurl and LLViewerNetwork unit test fixes. Got tired of the TC builds failing on these tests. Appears to be a conflict/race condition on a temporary file. Changed the name of the file. Should use a tempname()-like API but this will do for a unit test. --- indra/newview/tests/llslurl_test.cpp | 23 ++++++++++++++++------- indra/newview/tests/llviewernetwork_test.cpp | 23 +++++++++++++++-------- 2 files changed, 31 insertions(+), 15 deletions(-) (limited to 'indra') diff --git a/indra/newview/tests/llslurl_test.cpp b/indra/newview/tests/llslurl_test.cpp index 86229ad636..2bc0d5a086 100755 --- a/indra/newview/tests/llslurl_test.cpp +++ b/indra/newview/tests/llslurl_test.cpp @@ -6,7 +6,7 @@ * * $LicenseInfo:firstyear=2009&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2010, Linden Research, Inc. + * Copyright (C) 2014, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -31,6 +31,15 @@ #include "../llslurl.h" #include "../../llxml/llcontrol.h" #include "llsdserialize.h" + +namespace +{ + +// Should not collide with other test programs creating temp files. +static const char * const TEST_FILENAME("llslurl_test.xml"); + +} + //---------------------------------------------------------------------------- // Mock objects for the dependencies of the code we're testing @@ -143,11 +152,11 @@ namespace tut template<> template<> void slurlTestObject::test<1>() { - llofstream gridfile("grid_test.xml"); + llofstream gridfile(TEST_FILENAME); gridfile << gSampleGridFile; gridfile.close(); - LLGridManager::getInstance()->initialize("grid_test.xml"); + LLGridManager::getInstance()->initialize(TEST_FILENAME); LLGridManager::getInstance()->setGridChoice("util.agni.lindenlab.com"); @@ -260,11 +269,11 @@ namespace tut template<> template<> void slurlTestObject::test<2>() { - llofstream gridfile("grid_test.xml"); + llofstream gridfile(TEST_FILENAME); gridfile << gSampleGridFile; gridfile.close(); - LLGridManager::getInstance()->initialize("grid_test.xml"); + LLGridManager::getInstance()->initialize(TEST_FILENAME); LLSLURL slurl = LLSLURL("my.grid.com", "my region"); ensure_equals("grid/region - type", slurl.getType(), LLSLURL::LOCATION); @@ -293,11 +302,11 @@ namespace tut template<> template<> void slurlTestObject::test<3>() { - llofstream gridfile("grid_test.xml"); + llofstream gridfile(TEST_FILENAME); gridfile << gSampleGridFile; gridfile.close(); - LLGridManager::getInstance()->initialize("grid_test.xml"); + LLGridManager::getInstance()->initialize(TEST_FILENAME); LLGridManager::getInstance()->setGridChoice("my.grid.com"); LLSLURL slurl = LLSLURL("https://my.grid.com/region/my%20region/1/2/3"); diff --git a/indra/newview/tests/llviewernetwork_test.cpp b/indra/newview/tests/llviewernetwork_test.cpp index 7ad7947ca4..0eb0ab6500 100755 --- a/indra/newview/tests/llviewernetwork_test.cpp +++ b/indra/newview/tests/llviewernetwork_test.cpp @@ -6,7 +6,7 @@ * * $LicenseInfo:firstyear=2009&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2010, Linden Research, Inc. + * Copyright (C) 2014, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -31,6 +31,13 @@ #include "../../llxml/llcontrol.h" #include "llfile.h" +namespace +{ + +// Should not collide with other test programs creating temp files. +static const char * const TEST_FILENAME("llviewernetwork_test.xml"); + +} //---------------------------------------------------------------------------- // Mock objects for the dependencies of the code we're testing @@ -143,7 +150,7 @@ namespace tut { viewerNetworkTest() { - LLFile::remove("grid_test.xml"); + LLFile::remove(TEST_FILENAME); gCmdLineLoginURI.clear(); gCmdLineGridChoice.clear(); gCmdLineHelperURI.clear(); @@ -152,7 +159,7 @@ namespace tut } ~viewerNetworkTest() { - LLFile::remove("grid_test.xml"); + LLFile::remove(TEST_FILENAME); } }; @@ -170,7 +177,7 @@ namespace tut { LLGridManager *manager = LLGridManager::getInstance(); // grid file doesn't exist - manager->initialize("grid_test.xml"); + manager->initialize(TEST_FILENAME); // validate that some of the defaults are available. std::map known_grids = manager->getKnownGrids(); ensure_equals("Known grids is a string-string map of size 2", known_grids.size(), 2); @@ -238,11 +245,11 @@ namespace tut template<> template<> void viewerNetworkTestObject::test<2>() { - llofstream gridfile("grid_test.xml"); + llofstream gridfile(TEST_FILENAME); gridfile << gSampleGridFile; gridfile.close(); - LLGridManager::getInstance()->initialize("grid_test.xml"); + LLGridManager::getInstance()->initialize(TEST_FILENAME); std::map known_grids = LLGridManager::getInstance()->getKnownGrids(); ensure_equals("adding a grid via a grid file increases known grid size",4, known_grids.size()); @@ -369,11 +376,11 @@ namespace tut void viewerNetworkTestObject::test<7>() { // adding a grid with simply a name will populate the values. - llofstream gridfile("grid_test.xml"); + llofstream gridfile(TEST_FILENAME); gridfile << gSampleGridFile; gridfile.close(); - LLGridManager::getInstance()->initialize("grid_test.xml"); + LLGridManager::getInstance()->initialize(TEST_FILENAME); LLGridManager::getInstance()->setGridChoice("util.agni.lindenlab.com"); ensure_equals("getGridLabel", -- cgit v1.2.3 From f71c6c745bc390fadc571801a0d7c043249ade24 Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Tue, 9 Sep 2014 15:36:35 -0400 Subject: Cleanup pass. Documentation. Get older llcorehttp-using code to use utils for any LLSD interfaces. --- indra/llmessage/llcorehttputil.cpp | 4 +- indra/newview/llinventorymodel.cpp | 14 ++++- indra/newview/llinventorymodelbackgroundfetch.cpp | 28 +++++++--- indra/newview/llmeshrepository.cpp | 64 +++++++++-------------- indra/newview/lltexturefetch.cpp | 34 ++++++------ 5 files changed, 78 insertions(+), 66 deletions(-) (limited to 'indra') diff --git a/indra/llmessage/llcorehttputil.cpp b/indra/llmessage/llcorehttputil.cpp index 8d09aac971..ee80b0fd94 100644 --- a/indra/llmessage/llcorehttputil.cpp +++ b/indra/llmessage/llcorehttputil.cpp @@ -39,7 +39,9 @@ using namespace LLCore; namespace LLCoreHttpUtil { - +// *TODO: Currently converts only from XML content. A mode +// to convert using fromBinary() might be useful as well. Mesh +// headers could use it. bool responseToLLSD(HttpResponse * response, bool log, LLSD & out_llsd) { // Convert response to LLSD diff --git a/indra/newview/llinventorymodel.cpp b/indra/newview/llinventorymodel.cpp index 5273fb6d96..1625ae0d1b 100755 --- a/indra/newview/llinventorymodel.cpp +++ b/indra/newview/llinventorymodel.cpp @@ -3958,7 +3958,10 @@ BOOL decompress_file(const char* src_filename, const char* dst_filename) } #endif -// ==== FetchItemHttpHandler ==== + +///---------------------------------------------------------------------------- +/// Class LLInventoryModel::FetchItemHttpHandler +///---------------------------------------------------------------------------- LLInventoryModel::FetchItemHttpHandler::FetchItemHttpHandler(const LLSD & request_sd) : LLCore::HttpHandler(), @@ -4006,6 +4009,15 @@ void LLInventoryModel::FetchItemHttpHandler::onCompleted(LLCore::HttpHandle hand } // Check for 200-with-error failures + // + // Original Responder-based serivce model didn't check for these errors. + // It may be more robust to ignore this condition. With aggregated requests, + // an error in one inventory item might take down the entire request. + // So if this instead broke up the aggregated items into single requests, + // maybe that would make progress. Or perhaps there's structured information + // that can tell us what went wrong. Need to dig into this and firm up + // the API. + // // body_llsd["error"] = LLSD::emptyMap(); // Dev tool to force error handling // body_llsd["error"]["identifier"] = "Development"; // body_llsd["error"]["message"] = "You left development code in the viewer"; diff --git a/indra/newview/llinventorymodelbackgroundfetch.cpp b/indra/newview/llinventorymodelbackgroundfetch.cpp index 0c04a9c039..013134843a 100755 --- a/indra/newview/llinventorymodelbackgroundfetch.cpp +++ b/indra/newview/llinventorymodelbackgroundfetch.cpp @@ -45,6 +45,10 @@ namespace { +///---------------------------------------------------------------------------- +/// Class ::BGItemHttpHandler +///---------------------------------------------------------------------------- + // // Http request handler class for single inventory item requests. // @@ -80,6 +84,10 @@ protected: }; +///---------------------------------------------------------------------------- +/// Class ::BGFolderHttpHandler +///---------------------------------------------------------------------------- + // Http request handler class for folders. // // Handler for FetchInventoryDescendents2 and FetchLibDescendents2 @@ -130,6 +138,10 @@ const char * const LOG_INV("Inventory"); } // end of namespace anonymous +///---------------------------------------------------------------------------- +/// Class LLInventoryModelBackgroundFetch +///---------------------------------------------------------------------------- + LLInventoryModelBackgroundFetch::LLInventoryModelBackgroundFetch(): mBackgroundFetchActive(FALSE), mFolderFetchActive(false), @@ -684,14 +696,12 @@ bool LLInventoryModelBackgroundFetch::fetchQueueContainsNoDescendentsOf(const LL } -// =============================== -// Anonymous Namespace Definitions -// =============================== - namespace { -// ==== BGFolderHttpHandler ==== +///---------------------------------------------------------------------------- +/// Class ::BGFolderHttpHandler +///---------------------------------------------------------------------------- void BGFolderHttpHandler::onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * response) { @@ -734,6 +744,9 @@ void BGFolderHttpHandler::onCompleted(LLCore::HttpHandle handle, LLCore::HttpRes } // Check for 200-with-error failures + // + // See comments in llinventorymodel.cpp about this mode of error. + // // body_llsd["error"] = LLSD::emptyMap(); // Dev tool to force error handling // body_llsd["error"]["identifier"] = "Development"; // body_llsd["error"]["message"] = "You left development code in the viewer"; @@ -976,8 +989,9 @@ bool BGFolderHttpHandler::getIsRecursive(const LLUUID & cat_id) const return std::find(mRecursiveCatUUIDs.begin(), mRecursiveCatUUIDs.end(), cat_id) != mRecursiveCatUUIDs.end(); } - -// ==== BGItemHttpHandler ==== +///---------------------------------------------------------------------------- +/// Class ::BGItemHttpHandler +///---------------------------------------------------------------------------- // Nothing to implement here. All ctor/dtor changes. diff --git a/indra/newview/llmeshrepository.cpp b/indra/newview/llmeshrepository.cpp index a6707392fe..2b044c6916 100755 --- a/indra/newview/llmeshrepository.cpp +++ b/indra/newview/llmeshrepository.cpp @@ -71,6 +71,7 @@ #include "bufferarray.h" #include "bufferstream.h" #include "llfasttimer.h" +#include "llcorehttputil.h" #include "boost/lexical_cast.hpp" @@ -2236,21 +2237,17 @@ void LLMeshUploadThread::doWholeModelUpload() mModelData = LLSD::emptyMap(); wholeModelToLLSD(mModelData, true); LLSD body = mModelData["asset_resources"]; - dump_llsd_to_file(body,make_dump_name("whole_model_body_",dump_num)); - - LLCore::BufferArray * ba = new LLCore::BufferArray; - LLCore::BufferArrayStream bas(ba); - LLSDSerialize::toXML(body, bas); - // LLSDSerialize::toXML(mModelData, bas); // <- Enabling this will generate a convenient upload error - LLCore::HttpHandle handle = mHttpRequest->requestPost(mHttpPolicyClass, - mHttpPriority, - mWholeModelUploadURL, - ba, - mHttpOptions, - mHttpHeaders, - this); - ba->release(); - + + dump_llsd_to_file(body, make_dump_name("whole_model_body_", dump_num)); + + LLCore::HttpHandle handle = LLCoreHttpUtil::requestPostWithLLSD(mHttpRequest, + mHttpPolicyClass, + mHttpPriority, + mWholeModelUploadURL, + body, + mHttpOptions, + mHttpHeaders, + this); if (LLCORE_HTTP_HANDLE_INVALID == handle) { mHttpStatus = mHttpRequest->getStatus(); @@ -2294,19 +2291,14 @@ void LLMeshUploadThread::requestWholeModelFee() mModelData = LLSD::emptyMap(); wholeModelToLLSD(mModelData, false); dump_llsd_to_file(mModelData, make_dump_name("whole_model_fee_request_", dump_num)); - - LLCore::BufferArray * ba = new LLCore::BufferArray; - LLCore::BufferArrayStream bas(ba); - LLSDSerialize::toXML(mModelData, bas); - - LLCore::HttpHandle handle = mHttpRequest->requestPost(mHttpPolicyClass, - mHttpPriority, - mWholeModelFeeCapability, - ba, - mHttpOptions, - mHttpHeaders, - this); - ba->release(); + LLCore::HttpHandle handle = LLCoreHttpUtil::requestPostWithLLSD(mHttpRequest, + mHttpPolicyClass, + mHttpPriority, + mWholeModelFeeCapability, + mModelData, + mHttpOptions, + mHttpHeaders, + this); if (LLCORE_HTTP_HANDLE_INVALID == handle) { mHttpStatus = mHttpRequest->getStatus(); @@ -2379,12 +2371,8 @@ void LLMeshUploadThread::onCompleted(LLCore::HttpHandle handle, LLCore::HttpResp } else { - LLCore::BufferArray * ba(response->getBody()); - if (ba && ba->size()) - { - LLCore::BufferArrayStream bas(ba); - LLSDSerialize::fromXML(body, bas); - } + // *TODO: handle error in conversion process + LLCoreHttpUtil::responseToLLSD(response, true, body); } dump_llsd_to_file(body, make_dump_name("whole_model_upload_response_", dump_num)); @@ -2443,12 +2431,8 @@ void LLMeshUploadThread::onCompleted(LLCore::HttpHandle handle, LLCore::HttpResp } else { - LLCore::BufferArray * ba(response->getBody()); - if (ba && ba->size()) - { - LLCore::BufferArrayStream bas(ba); - LLSDSerialize::fromXML(body, bas); - } + // *TODO: handle error in conversion process + LLCoreHttpUtil::responseToLLSD(response, true, body); } dump_llsd_to_file(body, make_dump_name("whole_model_fee_response_", dump_num)); diff --git a/indra/newview/lltexturefetch.cpp b/indra/newview/lltexturefetch.cpp index a64a6ee091..a955b907d4 100755 --- a/indra/newview/lltexturefetch.cpp +++ b/indra/newview/lltexturefetch.cpp @@ -56,13 +56,13 @@ #include "llsdparam.h" #include "llsdutil.h" #include "llstartup.h" -#include "llsdserialize.h" #include "httprequest.h" #include "httphandler.h" #include "httpresponse.h" #include "bufferarray.h" #include "bufferstream.h" +#include "llcorehttputil.h" #include "llhttpretrypolicy.h" @@ -1349,20 +1349,20 @@ bool LLTextureFetchWorker::doWork(S32 param) LL_WARNS(LOG_TXT) << "trying to seek a non-default texture on the sim. Bad!" << LL_ENDL; } setUrl(http_url + "/?texture_id=" + mID.asString().c_str()); - LL_DEBUGS("Texture") << "Texture URL " << mUrl << LL_ENDL; + LL_DEBUGS(LOG_TXT) << "Texture URL: " << mUrl << LL_ENDL; mWriteToCacheState = CAN_WRITE ; //because this texture has a fixed texture id. } else { mCanUseHTTP = false ; - LL_DEBUGS("Texture") << "Texture not available via HTTP: no URL " << mUrl << LL_ENDL; + LL_DEBUGS(LOG_TXT) << "Texture not available via HTTP: empty URL." << LL_ENDL; } } else { // This will happen if not logged in or if a region deoes not have HTTP Texture enabled //LL_WARNS(LOG_TXT) << "Region not found for host: " << mHost << LL_ENDL; - LL_DEBUGS("Texture") << "Texture not available via HTTP: no region " << mUrl << LL_ENDL; + LL_DEBUGS(LOG_TXT) << "Texture not available via HTTP: no region " << mUrl << LL_ENDL; mCanUseHTTP = false; } } @@ -4026,7 +4026,9 @@ TFReqSendMetrics::doWork(LLTextureFetch * fetcher) // Update sequence number if (S32_MAX == ++report_sequence) + { report_sequence = 0; + } reporting_started = true; // Limit the size of the stats report if necessary. @@ -4035,18 +4037,16 @@ TFReqSendMetrics::doWork(LLTextureFetch * fetcher) if (! mCapsURL.empty()) { - LLCore::BufferArray * ba = new LLCore::BufferArray; - LLCore::BufferArrayStream bas(ba); - LLSDSerialize::toXML(sd, bas); - - fetcher->getHttpRequest().requestPost(fetcher->getPolicyClass(), - report_priority, - mCapsURL, - ba, - NULL, - fetcher->getMetricsHeaders(), - handler); - ba->release(); + // *TODO: Move this to a different class that expects POSTs sometime. + // Don't care about handle, this is a fire-and-forget operation. + LLCoreHttpUtil::requestPostWithLLSD(&fetcher->getHttpRequest(), + fetcher->getPolicyClass(), + report_priority, + mCapsURL, + sd, + NULL, + fetcher->getMetricsHeaders(), + handler); LLTextureFetch::svMetricsDataBreak = false; } else @@ -4057,7 +4057,7 @@ TFReqSendMetrics::doWork(LLTextureFetch * fetcher) // In QA mode, Metrics submode, log the result for ease of testing if (fetcher->isQAMode()) { - LL_INFOS("Textures") << ll_pretty_print_sd(sd) << LL_ENDL; + LL_INFOS(LOG_TXT) << ll_pretty_print_sd(sd) << LL_ENDL; } return true; -- cgit v1.2.3 From 3057d246f0c63be1b3015313b9ad9824a66f4e0f Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Thu, 18 Sep 2014 18:42:30 -0400 Subject: Documentation. Describe curl bug 1420 testing and how to reproduce data corruption via timeouts. --- indra/llcorehttp/_httpoprequest.cpp | 15 +++++++++++++++ indra/llcorehttp/httpcommon.cpp | 4 ++++ 2 files changed, 19 insertions(+) (limited to 'indra') diff --git a/indra/llcorehttp/_httpoprequest.cpp b/indra/llcorehttp/_httpoprequest.cpp index 38c1f1e78a..4453bf2922 100755 --- a/indra/llcorehttp/_httpoprequest.cpp +++ b/indra/llcorehttp/_httpoprequest.cpp @@ -47,6 +47,19 @@ #include "llhttpconstants.h" #include "llproxy.h" +// *DEBUG: "[curl:bugs] #1420" problem and testing. +// +// A pipelining problem, https://sourceforge.net/p/curl/bugs/1420/, +// was a source of Core_9 failures. Code related to this can be +// identified and tested by: +// * Looking for '[curl:bugs]' strings in source and following +// instructions there. +// * Set 'QAModeHttpTrace' to 2 or 3 in settings.xml and look for +// 'timed out' events in the log. +// * Enable the HttpRangeRequestsDisable debug setting which causes +// full asset fetches. These slow the pipelines down a bit. +// + namespace { @@ -610,6 +623,8 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service) // *TODO: Find a better scheme than timeouts to guarantee liveness. xfer_timeout *= cpolicy.mPipelining; } + // *DEBUG: Useful for timeout handling and "[curl:bugs] #1420" tests + // xfer_timeout = 3L; code = curl_easy_setopt(mCurlHandle, CURLOPT_TIMEOUT, xfer_timeout); check_curl_easy_code(code, CURLOPT_TIMEOUT); code = curl_easy_setopt(mCurlHandle, CURLOPT_CONNECTTIMEOUT, timeout); diff --git a/indra/llcorehttp/httpcommon.cpp b/indra/llcorehttp/httpcommon.cpp index 9bcf7ac5e3..8714915fa2 100755 --- a/indra/llcorehttp/httpcommon.cpp +++ b/indra/llcorehttp/httpcommon.cpp @@ -240,6 +240,10 @@ bool HttpStatus::isRetryable() const static const HttpStatus inv_cont_range(HttpStatus::LLCORE, HE_INV_CONTENT_RANGE_HDR); static const HttpStatus inv_status(HttpStatus::LLCORE, HE_INVALID_HTTP_STATUS); + // *DEBUG: For "[curl:bugs] #1420" tests. + // Disable the '*this == inv_status' test and look for 'Core_9' + // failures in log files. + return ((isHttpStatus() && mType >= 499 && mType <= 599) || // Include special 499 in retryables *this == cant_connect || // Connection reset/endpoint problems *this == cant_res_proxy || // DNS problems -- cgit v1.2.3 From 4caa2746f3b1d336947dcc9e277af8b8c08b44e0 Mon Sep 17 00:00:00 2001 From: davep Date: Fri, 19 Sep 2014 13:01:41 -0500 Subject: MAINT-3131 Discard improbably high memory bandwidth measurements on OSX and default to Class 3 --- indra/newview/llfeaturemanager.cpp | 6 +++++- indra/newview/llglsandbox.cpp | 15 +++++++++++++-- 2 files changed, 18 insertions(+), 3 deletions(-) (limited to 'indra') diff --git a/indra/newview/llfeaturemanager.cpp b/indra/newview/llfeaturemanager.cpp index 92e6f70566..4db0422634 100755 --- a/indra/newview/llfeaturemanager.cpp +++ b/indra/newview/llfeaturemanager.cpp @@ -426,7 +426,10 @@ bool LLFeatureManager::loadGPUClass() if (gbps < 0.f) { //couldn't bench, use GLVersion - +#if LL_DARWIN + //GLVersion is misleading on OSX, just default to class 3 if we can't bench + mGPUClass = GPU_CLASS_3; +#else if (gGLManager.mGLVersion < 2.f) { mGPUClass = GPU_CLASS_0; @@ -447,6 +450,7 @@ bool LLFeatureManager::loadGPUClass() { mGPUClass = GPU_CLASS_4; } +#endif } else if (gbps < 5.f) { diff --git a/indra/newview/llglsandbox.cpp b/indra/newview/llglsandbox.cpp index 0548de3895..4b8ac2b3cf 100755 --- a/indra/newview/llglsandbox.cpp +++ b/indra/newview/llglsandbox.cpp @@ -928,7 +928,7 @@ F32 gpu_benchmark() std::vector results; //build a random texture - U8 pixels[res*res*4]; + U8* pixels = new U8[res*res*4]; for (U32 i = 0; i < res*res*4; ++i) { @@ -950,6 +950,8 @@ F32 gpu_benchmark() LLImageGL::setManualImage(GL_TEXTURE_2D, 0, GL_RGBA, res,res,GL_RGBA, GL_UNSIGNED_BYTE, pixels); } + delete [] pixels; + //make a dummy triangle to draw with LLPointer buff = new LLVertexBuffer(LLVertexBuffer::MAP_VERTEX | LLVertexBuffer::MAP_TEXCOORD0, GL_STATIC_DRAW_ARB); buff->allocateBuffer(3, 0, true); @@ -1031,7 +1033,16 @@ F32 gpu_benchmark() F32 gbps = results[results.size()/2]; LL_INFOS() << "Memory bandwidth is " << llformat("%.3f", gbps) << "GB/sec according to CPU timers" << LL_ENDL; - + +#if LL_DARWIN + if (gbps > 512.f) + { + LL_INFOS() << "Memory bandwidth is improbably high and likely incorrect." << LL_ENDL; + //OSX is probably lying, discard result + gbps = -1.f; + } +#endif + if (gGLManager.mHasTimerQuery) { F32 ms = gBenchmarkProgram.mTimeElapsed/1000000.f; -- cgit v1.2.3 From 79ab7c20703c092a4416a4f9a885e0246fc17ee0 Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Fri, 19 Sep 2014 15:34:09 -0400 Subject: Introduce libcurl handle cache. Create a private cache of used handles and a fast handle factory that's thread- correct. --- indra/llcorehttp/_httplibcurl.cpp | 83 +++++++++++++++++++++++++++++++++++-- indra/llcorehttp/_httplibcurl.h | 78 ++++++++++++++++++++++++++++++++-- indra/llcorehttp/_httpoprequest.cpp | 5 ++- 3 files changed, 158 insertions(+), 8 deletions(-) (limited to 'indra') diff --git a/indra/llcorehttp/_httplibcurl.cpp b/indra/llcorehttp/_httplibcurl.cpp index cfbe0fd2bb..81b44ab90b 100755 --- a/indra/llcorehttp/_httplibcurl.cpp +++ b/indra/llcorehttp/_httplibcurl.cpp @@ -51,6 +51,7 @@ namespace LLCore HttpLibcurl::HttpLibcurl(HttpService * service) : mService(service), + mHandleCache(), mPolicyCount(0), mMultiHandles(NULL), mActiveHandles(NULL), @@ -61,7 +62,7 @@ HttpLibcurl::HttpLibcurl(HttpService * service) HttpLibcurl::~HttpLibcurl() { shutdown(); - + mService = NULL; } @@ -279,7 +280,7 @@ void HttpLibcurl::cancelRequest(HttpOpRequest * op) // Detach from multi and recycle handle curl_multi_remove_handle(mMultiHandles[op->mReqPolicy], op->mCurlHandle); - curl_easy_cleanup(op->mCurlHandle); + mHandleCache.freeHandle(op->mCurlHandle); op->mCurlHandle = NULL; // Tracing @@ -356,7 +357,7 @@ bool HttpLibcurl::completeRequest(CURLM * multi_handle, CURL * handle, CURLcode // Detach from multi and recycle handle curl_multi_remove_handle(multi_handle, handle); - curl_easy_cleanup(handle); + mHandleCache.freeHandle(op->mCurlHandle); op->mCurlHandle = NULL; // Tracing @@ -471,6 +472,82 @@ void HttpLibcurl::policyUpdated(int policy_class) } } +// --------------------------------------- +// HttpLibcurl::HandleCache +// --------------------------------------- + +HttpLibcurl::HandleCache::HandleCache() + : mHandleTemplate(NULL) +{ + mCache.reserve(50); +} + + +HttpLibcurl::HandleCache::~HandleCache() +{ + if (mHandleTemplate) + { + curl_easy_cleanup(mHandleTemplate); + mHandleTemplate = NULL; + } + + for (handle_cache_t::iterator it(mCache.begin()); mCache.end() != it; ++it) + { + curl_easy_cleanup(*it); + } + mCache.clear(); +} + + +CURL * HttpLibcurl::HandleCache::getHandle() +{ + CURL * ret(NULL); + + if (! mCache.empty()) + { + // Fastest path to handle + ret = mCache.back(); + mCache.pop_back(); + } + else if (mHandleTemplate) + { + // Still fast path + ret = curl_easy_duphandle(mHandleTemplate); + } + else + { + // When all else fails + ret = curl_easy_init(); + } + + return ret; +} + + +void HttpLibcurl::HandleCache::freeHandle(CURL * handle) +{ + if (! handle) + { + return; + } + + curl_easy_reset(handle); + if (! mHandleTemplate) + { + // Save the first freed handle as a template. + mHandleTemplate = handle; + } + else + { + // Otherwise add it to the cache + if (mCache.size() >= mCache.capacity()) + { + mCache.reserve(mCache.capacity() + 50); + } + mCache.push_back(handle); + } +} + // --------------------------------------- // Free functions diff --git a/indra/llcorehttp/_httplibcurl.h b/indra/llcorehttp/_httplibcurl.h index 2c7ad1fa8e..ffc24c63a8 100755 --- a/indra/llcorehttp/_httplibcurl.h +++ b/indra/llcorehttp/_httplibcurl.h @@ -124,6 +124,23 @@ public: /// Threading: called by worker thread. void policyUpdated(int policy_class); + /// Allocate a curl handle for caller. May be freed using + /// either the freeHandle() method or calling curl_easy_cleanup() + /// directly. + /// + /// @return Libcurl handle (CURL *) or NULL on allocation + /// problem. Handle will be in curl_easy_reset() + /// condition. + /// + /// Threading: callable by worker thread. + /// + /// Deprecation: Expect this to go away after _httpoprequest is + /// refactored bringing code into this class. + CURL * getHandle() + { + return mHandleCache.getHandle(); + } + protected: /// Invoked when libcurl has indicated a request has been processed /// to completion and we need to move the request to a new state. @@ -135,14 +152,67 @@ protected: protected: typedef std::set active_set_t; + + /// Simple request handle cache for libcurl. + /// + /// Handle creation is somewhat slow and chunky in libcurl and there's + /// a pretty good speedup to be had from handle re-use. So, a simple + /// vector is kept of 'freed' handles to be reused as needed. When + /// that is empty, the first freed handle is kept as a template for + /// handle duplication. This is still faster than creation from nothing. + /// And when that fails, we init fresh from curl_easy_init(). + /// + /// Handles allocated with getHandle() may be freed with either + /// freeHandle() or curl_easy_cleanup(). Choice may be dictated + /// by thread constraints. + /// + /// Threading: Single-threaded. May only be used by a single thread, + /// typically the worker thread. If freeing requests' handles in an + /// unknown threading context, use curl_easy_cleanup() for safety. + + class HandleCache + { + public: + HandleCache(); + ~HandleCache(); + + private: + HandleCache(const HandleCache &); // Not defined + void operator=(const HandleCache &); // Not defined + + public: + /// Allocate a curl handle for caller. May be freed using + /// either the freeHandle() method or calling curl_easy_cleanup() + /// directly. + /// + /// @return Libcurl handle (CURL *) or NULL on allocation + /// problem. + /// + /// Threading: Single-thread (worker) only. + CURL * getHandle(); + + /// Free a libcurl handle acquired by whatever means. Thread + /// safety is left to the caller. + /// + /// Threading: Single-thread (worker) only. + void freeHandle(CURL * handle); + + protected: + typedef std::vector handle_cache_t; + + protected: + CURL * mHandleTemplate; // Template for duplicating new handles + handle_cache_t mCache; // Cache of old handles + }; // end class HandleCache protected: - HttpService * mService; // Simple reference, not owner + HttpService * mService; // Simple reference, not owner + HandleCache mHandleCache; // Handle allocator, owner active_set_t mActiveOps; int mPolicyCount; - CURLM ** mMultiHandles; // One handle per policy class - int * mActiveHandles; // Active count per policy class - bool * mDirtyPolicy; // Dirty policy update waiting for stall (per pc) + CURLM ** mMultiHandles; // One handle per policy class + int * mActiveHandles; // Active count per policy class + bool * mDirtyPolicy; // Dirty policy update waiting for stall (per pc) }; // end class HttpLibcurl diff --git a/indra/llcorehttp/_httpoprequest.cpp b/indra/llcorehttp/_httpoprequest.cpp index 4453bf2922..bbda0b82fd 100755 --- a/indra/llcorehttp/_httpoprequest.cpp +++ b/indra/llcorehttp/_httpoprequest.cpp @@ -170,6 +170,8 @@ HttpOpRequest::~HttpOpRequest() if (mCurlHandle) { + // Uncertain of thread context so free using + // safest method. curl_easy_cleanup(mCurlHandle); mCurlHandle = NULL; } @@ -429,7 +431,7 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service) HttpPolicyGlobal & gpolicy(service->getPolicy().getGlobalOptions()); HttpPolicyClass & cpolicy(service->getPolicy().getClassOptions(mReqPolicy)); - mCurlHandle = LLCurl::createStandardCurlHandle(); + mCurlHandle = service->getTransport().getHandle(); if (! mCurlHandle) { // We're in trouble. We'll continue but it won't go well. @@ -437,6 +439,7 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service) << LL_ENDL; return HttpStatus(HttpStatus::LLCORE, HE_BAD_ALLOC); } + code = curl_easy_setopt(mCurlHandle, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4); check_curl_easy_code(code, CURLOPT_IPRESOLVE); code = curl_easy_setopt(mCurlHandle, CURLOPT_NOSIGNAL, 1); -- cgit v1.2.3 From 11036d7bf471953ada9b877b8d9ce9de4b94dc5b Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Fri, 19 Sep 2014 19:43:25 -0400 Subject: Cleanup work. Use http constants for content-type and accept headers in mesh and textures. For texture metrics reporting, use the AP_INVENTORY policy class which is non-pipelined and pointing (usually) in the right direction. Use a do-while(false) structure to manage common exit path code in onCompleted() methods. Identical to a 'goto' but might amuse the pedantic. Tuning on background fetch to have it cycle faster. This is experimental. I suspect with HTTP balancing in llcorehttp, we can do away with the timers here. --- indra/llmessage/llhttpconstants.cpp | 6 +- indra/llmessage/llhttpconstants.h | 3 +- indra/newview/llappcorehttp.h | 9 +- indra/newview/llinventorymodel.cpp | 102 +++++++++++---------- indra/newview/llinventorymodelbackgroundfetch.cpp | 106 +++++++++++----------- indra/newview/llmeshrepository.cpp | 4 +- indra/newview/lltexturefetch.cpp | 15 +-- indra/newview/lltexturefetch.h | 6 +- 8 files changed, 130 insertions(+), 121 deletions(-) (limited to 'indra') diff --git a/indra/llmessage/llhttpconstants.cpp b/indra/llmessage/llhttpconstants.cpp index 01f4a080b0..32f76f0d70 100755 --- a/indra/llmessage/llhttpconstants.cpp +++ b/indra/llmessage/llhttpconstants.cpp @@ -3,11 +3,8 @@ * @brief Implementation of the HTTP request / response constant lookups * * $LicenseInfo:firstyear=2013&license=viewerlgpl$ - * - * Copyright (c) 2013, Linden Research, Inc. - * * Second Life Viewer Source Code - * Copyright (C) 2013, Linden Research, Inc. + * Copyright (C) 2013-2014, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -107,6 +104,7 @@ const std::string HTTP_IN_HEADER_X_FORWARDED_FOR("x-forwarded-for"); const std::string HTTP_CONTENT_LLSD_XML("application/llsd+xml"); const std::string HTTP_CONTENT_OCTET_STREAM("application/octet-stream"); +const std::string HTTP_CONTENT_VND_LL_MESH("application/vnd.ll.mesh"); const std::string HTTP_CONTENT_XML("application/xml"); const std::string HTTP_CONTENT_JSON("application/json"); const std::string HTTP_CONTENT_TEXT_HTML("text/html"); diff --git a/indra/llmessage/llhttpconstants.h b/indra/llmessage/llhttpconstants.h index 4aa3cc6394..d6bcbd3c19 100755 --- a/indra/llmessage/llhttpconstants.h +++ b/indra/llmessage/llhttpconstants.h @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2001&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2001-2013, Linden Research, Inc. + * Copyright (C) 2001-2014, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -203,6 +203,7 @@ extern const std::string HTTP_IN_HEADER_X_FORWARDED_FOR; extern const std::string HTTP_CONTENT_LLSD_XML; extern const std::string HTTP_CONTENT_OCTET_STREAM; +extern const std::string HTTP_CONTENT_VND_LL_MESH; extern const std::string HTTP_CONTENT_XML; extern const std::string HTTP_CONTENT_JSON; extern const std::string HTTP_CONTENT_TEXT_HTML; diff --git a/indra/newview/llappcorehttp.h b/indra/newview/llappcorehttp.h index e822a40300..37d7a737e7 100755 --- a/indra/newview/llappcorehttp.h +++ b/indra/newview/llappcorehttp.h @@ -66,7 +66,7 @@ public: /// baking service. Deep queueing of requests. /// Do not share. GET requests only. /// - /// Destination: simhost:12046 & bake-texture:80 + /// Destination: simhost:12046 & {bake-texture,cdn}:80 /// Protocol: http: /// Transfer size: KB-MB /// Long poll: no @@ -95,7 +95,7 @@ public: /// Very deeply queued. Do not share. GET /// requests only. /// - /// Destination: simhost:12046 + /// Destination: simhost:12046 & cdn:80 /// Protocol: http: /// Transfer size: KB-MB /// Long poll: no @@ -113,7 +113,7 @@ public: /// traffic that can wait for longish stalls /// (default timeout 600S). /// - /// Destination: simhost:12046 + /// Destination: simhost:12046 & cdn:80 /// Protocol: http: /// Transfer size: MB /// Long poll: no @@ -163,7 +163,8 @@ public: /// Request rate: high /// Pipelined: no AP_INVENTORY, - + AP_REPORTING = AP_INVENTORY, // Piggy-back on inventory + AP_COUNT // Must be last }; diff --git a/indra/newview/llinventorymodel.cpp b/indra/newview/llinventorymodel.cpp index 1625ae0d1b..ee28cef640 100755 --- a/indra/newview/llinventorymodel.cpp +++ b/indra/newview/llinventorymodel.cpp @@ -3974,64 +3974,68 @@ LLInventoryModel::FetchItemHttpHandler::~FetchItemHttpHandler() void LLInventoryModel::FetchItemHttpHandler::onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * response) { - LLCore::HttpStatus status(response->getStatus()); - // status = LLCore::HttpStatus(404); // Dev tool to force error handling - if (! status) - { - processFailure(status, response); - } - else + // Single-pass do-while used for common exit handling + do { - LLCore::BufferArray * body(response->getBody()); - // body = NULL; // Dev tool to force error handling - if (! body || ! body->size()) + LLCore::HttpStatus status(response->getStatus()); + // status = LLCore::HttpStatus(404); // Dev tool to force error handling + if (! status) { - LL_WARNS(LOG_INV) << "Missing data in inventory item query." << LL_ENDL; - processFailure("HTTP response for inventory item query missing body", response); - goto only_exit; + processFailure(status, response); } - - // body->write(0, "Garbage Response", 16); // Dev tool to force error handling - LLSD body_llsd; - if (! LLCoreHttpUtil::responseToLLSD(response, true, body_llsd)) + else { - // INFOS-level logging will occur on the parsed failure - processFailure("HTTP response for inventory item query has malformed LLSD", response); - goto only_exit; - } + LLCore::BufferArray * body(response->getBody()); + // body = NULL; // Dev tool to force error handling + if (! body || ! body->size()) + { + LL_WARNS(LOG_INV) << "Missing data in inventory item query." << LL_ENDL; + processFailure("HTTP response for inventory item query missing body", response); + break; // Goto common exit + } - // Expect top-level structure to be a map - // body_llsd = LLSD::emptyArray(); // Dev tool to force error handling - if (! body_llsd.isMap()) - { - processFailure("LLSD response for inventory item not a map", response); - goto only_exit; - } + // body->write(0, "Garbage Response", 16); // Dev tool to force error handling + LLSD body_llsd; + if (! LLCoreHttpUtil::responseToLLSD(response, true, body_llsd)) + { + // INFOS-level logging will occur on the parsed failure + processFailure("HTTP response for inventory item query has malformed LLSD", response); + break; // Goto common exit + } - // Check for 200-with-error failures - // - // Original Responder-based serivce model didn't check for these errors. - // It may be more robust to ignore this condition. With aggregated requests, - // an error in one inventory item might take down the entire request. - // So if this instead broke up the aggregated items into single requests, - // maybe that would make progress. Or perhaps there's structured information - // that can tell us what went wrong. Need to dig into this and firm up - // the API. - // - // body_llsd["error"] = LLSD::emptyMap(); // Dev tool to force error handling - // body_llsd["error"]["identifier"] = "Development"; - // body_llsd["error"]["message"] = "You left development code in the viewer"; - if (body_llsd.has("error")) - { - processFailure("Inventory application error (200-with-error)", response); - goto only_exit; - } + // Expect top-level structure to be a map + // body_llsd = LLSD::emptyArray(); // Dev tool to force error handling + if (! body_llsd.isMap()) + { + processFailure("LLSD response for inventory item not a map", response); + break; // Goto common exit + } - // Okay, process data if possible - processData(body_llsd, response); + // Check for 200-with-error failures + // + // Original Responder-based serivce model didn't check for these errors. + // It may be more robust to ignore this condition. With aggregated requests, + // an error in one inventory item might take down the entire request. + // So if this instead broke up the aggregated items into single requests, + // maybe that would make progress. Or perhaps there's structured information + // that can tell us what went wrong. Need to dig into this and firm up + // the API. + // + // body_llsd["error"] = LLSD::emptyMap(); // Dev tool to force error handling + // body_llsd["error"]["identifier"] = "Development"; + // body_llsd["error"]["message"] = "You left development code in the viewer"; + if (body_llsd.has("error")) + { + processFailure("Inventory application error (200-with-error)", response); + break; // Goto common exit + } + + // Okay, process data if possible + processData(body_llsd, response); + } } + while (false); -only_exit: // Must delete on completion. delete this; } diff --git a/indra/newview/llinventorymodelbackgroundfetch.cpp b/indra/newview/llinventorymodelbackgroundfetch.cpp index 013134843a..de1d123fe5 100755 --- a/indra/newview/llinventorymodelbackgroundfetch.cpp +++ b/indra/newview/llinventorymodelbackgroundfetch.cpp @@ -288,6 +288,7 @@ void LLInventoryModelBackgroundFetch::setAllFoldersFetched() } mFolderFetchActive = false; mBackgroundFetchActive = false; + LL_INFOS(LOG_INV) << "Inventory background fetch completed" << LL_ENDL; } void LLInventoryModelBackgroundFetch::backgroundFetchCB(void *) @@ -314,12 +315,7 @@ void LLInventoryModelBackgroundFetch::backgroundFetch() // No more categories to fetch, stop fetch process. if (mFetchQueue.empty()) { - LL_INFOS(LOG_INV) << "Inventory fetch completed" << LL_ENDL; - setAllFoldersFetched(); - mBackgroundFetchActive = false; - mFolderFetchActive = false; - return; } @@ -493,9 +489,10 @@ void LLInventoryModelBackgroundFetch::bulkFetch() // is mostly loaded, we could turn up the throttle and fill missing // inventory more quickly. static const S32 max_concurrent_fetches(12); // Outstanding requests, not connections - static const F32 new_min_time(0.5f); // *HACK: Clean this up when old code goes away entirely. + static const F32 new_min_time(0.05f); // *HACK: Clean this up when old code goes away entirely. static const U32 max_batch_size(10); + mMinTimeBetweenFetches = 0.01f; if (mMinTimeBetweenFetches < new_min_time) { mMinTimeBetweenFetches = new_min_time; // *HACK: See above. @@ -705,62 +702,66 @@ namespace void BGFolderHttpHandler::onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * response) { - LLCore::HttpStatus status(response->getStatus()); - // status = LLCore::HttpStatus(404); // Dev tool to force error handling - if (! status) - { - processFailure(status, response); - } - else + // Single-pass do-while used for common exit handling + do { - // Response body should be present. - LLCore::BufferArray * body(response->getBody()); - // body = NULL; // Dev tool to force error handling - if (! body || ! body->size()) + LLCore::HttpStatus status(response->getStatus()); + // status = LLCore::HttpStatus(404); // Dev tool to force error handling + if (! status) { - LL_WARNS(LOG_INV) << "Missing data in inventory folder query." << LL_ENDL; - processFailure("HTTP response missing expected body", response); - goto only_exit; + processFailure(status, response); } + else + { + // Response body should be present. + LLCore::BufferArray * body(response->getBody()); + // body = NULL; // Dev tool to force error handling + if (! body || ! body->size()) + { + LL_WARNS(LOG_INV) << "Missing data in inventory folder query." << LL_ENDL; + processFailure("HTTP response missing expected body", response); + break; // Goto common exit + } - // Could test 'Content-Type' header but probably unreliable. + // Could test 'Content-Type' header but probably unreliable. - // Convert response to LLSD - // body->write(0, "Garbage Response", 16); // Dev tool to force error handling - LLSD body_llsd; - if (! LLCoreHttpUtil::responseToLLSD(response, true, body_llsd)) - { - // INFOS-level logging will occur on the parsed failure - processFailure("HTTP response contained malformed LLSD", response); - goto only_exit; - } + // Convert response to LLSD + // body->write(0, "Garbage Response", 16); // Dev tool to force error handling + LLSD body_llsd; + if (! LLCoreHttpUtil::responseToLLSD(response, true, body_llsd)) + { + // INFOS-level logging will occur on the parsed failure + processFailure("HTTP response contained malformed LLSD", response); + break; // goto common exit + } - // Expect top-level structure to be a map - // body_llsd = LLSD::emptyArray(); // Dev tool to force error handling - if (! body_llsd.isMap()) - { - processFailure("LLSD response not a map", response); - goto only_exit; - } + // Expect top-level structure to be a map + // body_llsd = LLSD::emptyArray(); // Dev tool to force error handling + if (! body_llsd.isMap()) + { + processFailure("LLSD response not a map", response); + break; // goto common exit + } - // Check for 200-with-error failures - // - // See comments in llinventorymodel.cpp about this mode of error. - // - // body_llsd["error"] = LLSD::emptyMap(); // Dev tool to force error handling - // body_llsd["error"]["identifier"] = "Development"; - // body_llsd["error"]["message"] = "You left development code in the viewer"; - if (body_llsd.has("error")) - { - processFailure("Inventory application error (200-with-error)", response); - goto only_exit; - } + // Check for 200-with-error failures + // + // See comments in llinventorymodel.cpp about this mode of error. + // + // body_llsd["error"] = LLSD::emptyMap(); // Dev tool to force error handling + // body_llsd["error"]["identifier"] = "Development"; + // body_llsd["error"]["message"] = "You left development code in the viewer"; + if (body_llsd.has("error")) + { + processFailure("Inventory application error (200-with-error)", response); + break; // goto common exit + } - // Okay, process data if possible - processData(body_llsd, response); + // Okay, process data if possible + processData(body_llsd, response); + } } + while (false); -only_exit: // Must delete on completion. delete this; } @@ -895,7 +896,6 @@ void BGFolderHttpHandler::processData(LLSD & content, LLCore::HttpResponse * res if (fetcher->isBulkFetchProcessingComplete()) { - LL_INFOS(LOG_INV) << "Inventory fetch completed" << LL_ENDL; fetcher->setAllFoldersFetched(); } diff --git a/indra/newview/llmeshrepository.cpp b/indra/newview/llmeshrepository.cpp index 2b044c6916..648056484e 100755 --- a/indra/newview/llmeshrepository.cpp +++ b/indra/newview/llmeshrepository.cpp @@ -760,7 +760,7 @@ LLMeshRepoThread::LLMeshRepoThread() mHttpLargeOptions->setTransferTimeout(LARGE_MESH_XFER_TIMEOUT); mHttpLargeOptions->setUseRetryAfter(gSavedSettings.getBOOL("MeshUseHttpRetryAfter")); mHttpHeaders = new LLCore::HttpHeaders; - mHttpHeaders->append("Accept", "application/vnd.ll.mesh"); + mHttpHeaders->append(HTTP_OUT_HEADER_ACCEPT, HTTP_CONTENT_VND_LL_MESH); mHttpPolicyClass = app_core_http.getPolicy(LLAppCoreHttp::AP_MESH2); mHttpLegacyPolicyClass = app_core_http.getPolicy(LLAppCoreHttp::AP_MESH1); mHttpLargePolicyClass = app_core_http.getPolicy(LLAppCoreHttp::AP_LARGE_MESH); @@ -1887,7 +1887,7 @@ LLMeshUploadThread::LLMeshUploadThread(LLMeshUploadThread::instance_list& data, mHttpOptions->setUseRetryAfter(gSavedSettings.getBOOL("MeshUseHttpRetryAfter")); mHttpOptions->setRetries(UPLOAD_RETRY_LIMIT); mHttpHeaders = new LLCore::HttpHeaders; - mHttpHeaders->append("Content-Type", "application/llsd+xml"); + mHttpHeaders->append(HTTP_OUT_HEADER_CONTENT_TYPE, HTTP_CONTENT_LLSD_XML); mHttpPolicyClass = LLAppViewer::instance()->getAppCoreHttp().getPolicy(LLAppCoreHttp::AP_UPLOADS); mHttpPriority = 0; } diff --git a/indra/newview/lltexturefetch.cpp b/indra/newview/lltexturefetch.cpp index a955b907d4..acd4cf2d8d 100755 --- a/indra/newview/lltexturefetch.cpp +++ b/indra/newview/lltexturefetch.cpp @@ -2514,8 +2514,9 @@ LLTextureFetch::LLTextureFetch(LLTextureCache* cache, LLImageDecodeThread* image mHttpOptions(NULL), mHttpOptionsWithHeaders(NULL), mHttpHeaders(NULL), - mHttpMetricsHeaders(NULL), mHttpPolicyClass(LLCore::HttpRequest::DEFAULT_POLICY_ID), + mHttpMetricsHeaders(NULL), + mHttpMetricsPolicyClass(LLCore::HttpRequest::DEFAULT_POLICY_ID), mTotalCacheReadCount(0U), mTotalCacheWriteCount(0U), mTotalResourceWaitCount(0U), @@ -2528,15 +2529,16 @@ LLTextureFetch::LLTextureFetch(LLTextureCache* cache, LLImageDecodeThread* image mTextureInfo.setUpLogging(gSavedSettings.getBOOL("LogTextureDownloadsToViewerLog"), gSavedSettings.getBOOL("LogTextureDownloadsToSimulator"), U32Bytes(gSavedSettings.getU32("TextureLoggingThreshold"))); LLAppCoreHttp & app_core_http(LLAppViewer::instance()->getAppCoreHttp()); - mHttpPolicyClass = app_core_http.getPolicy(LLAppCoreHttp::AP_TEXTURE); mHttpRequest = new LLCore::HttpRequest; mHttpOptions = new LLCore::HttpOptions; mHttpOptionsWithHeaders = new LLCore::HttpOptions; mHttpOptionsWithHeaders->setWantHeaders(true); mHttpHeaders = new LLCore::HttpHeaders; - mHttpHeaders->append("Accept", "image/x-j2c"); + mHttpHeaders->append(HTTP_OUT_HEADER_ACCEPT, HTTP_CONTENT_IMAGE_X_J2C); + mHttpPolicyClass = app_core_http.getPolicy(LLAppCoreHttp::AP_TEXTURE); mHttpMetricsHeaders = new LLCore::HttpHeaders; - mHttpMetricsHeaders->append("Content-Type", "application/llsd+xml"); + mHttpMetricsHeaders->append(HTTP_OUT_HEADER_CONTENT_TYPE, HTTP_CONTENT_LLSD_XML); + mHttpMetricsPolicyClass = app_core_http.getPolicy(LLAppCoreHttp::AP_REPORTING); mHttpHighWater = HTTP_NONPIPE_REQUESTS_HIGH_WATER; mHttpLowWater = HTTP_NONPIPE_REQUESTS_LOW_WATER; mHttpSemaphore = 0; @@ -4037,10 +4039,9 @@ TFReqSendMetrics::doWork(LLTextureFetch * fetcher) if (! mCapsURL.empty()) { - // *TODO: Move this to a different class that expects POSTs sometime. // Don't care about handle, this is a fire-and-forget operation. LLCoreHttpUtil::requestPostWithLLSD(&fetcher->getHttpRequest(), - fetcher->getPolicyClass(), + fetcher->getMetricsPolicyClass(), report_priority, mCapsURL, sd, @@ -4227,7 +4228,7 @@ void LLTextureFetchDebugger::init() if (! mHttpHeaders) { mHttpHeaders = new LLCore::HttpHeaders; - mHttpHeaders->append("Accept", "image/x-j2c"); + mHttpHeaders->append(HTTP_OUT_HEADER_ACCEPT, HTTP_CONTENT_IMAGE_X_J2C); } } diff --git a/indra/newview/lltexturefetch.h b/indra/newview/lltexturefetch.h index 89d18e2c67..27779a31e0 100755 --- a/indra/newview/lltexturefetch.h +++ b/indra/newview/lltexturefetch.h @@ -179,6 +179,9 @@ public: // Threads: T* LLCore::HttpHeaders * getMetricsHeaders() const { return mHttpMetricsHeaders; } + // Threads: T* + LLCore::HttpRequest::policy_t getMetricsPolicyClass() const { return mHttpMetricsPolicyClass; } + bool isQAMode() const { return mQAMode; } // ---------------------------------- @@ -354,8 +357,9 @@ private: LLCore::HttpOptions * mHttpOptions; // Ttf LLCore::HttpOptions * mHttpOptionsWithHeaders; // Ttf LLCore::HttpHeaders * mHttpHeaders; // Ttf - LLCore::HttpHeaders * mHttpMetricsHeaders; // Ttf LLCore::HttpRequest::policy_t mHttpPolicyClass; // T* + LLCore::HttpHeaders * mHttpMetricsHeaders; // Ttf + LLCore::HttpRequest::policy_t mHttpMetricsPolicyClass; // T* S32 mHttpHighWater; // Ttf S32 mHttpLowWater; // Ttf -- cgit v1.2.3 From 329608d24668b044e16b54ff7a7d0ac592b2b88d Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Mon, 22 Sep 2014 18:49:45 -0400 Subject: Tuning and documentation. Use a fast poll frequency (0.05S) on the HTTP requests for inventory. We'll benchmark with that and see how it goes. Document some of the history of the background fetcher for future devs. Suggest some future projects to make things faster. Pointers on using LLSD with the llcorehttp library in the readme. And restructured the LLSD onCompleted() processing phases using do{}while(false) which produced a code flow that is fairly attractive. --- indra/llcorehttp/README.Linden | 8 ++ indra/newview/llinventorymodel.cpp | 92 +++++++------- indra/newview/llinventorymodelbackgroundfetch.cpp | 140 +++++++++++++++------- 3 files changed, 147 insertions(+), 93 deletions(-) (limited to 'indra') diff --git a/indra/llcorehttp/README.Linden b/indra/llcorehttp/README.Linden index eb6ccab3bc..c3aaa9158d 100644 --- a/indra/llcorehttp/README.Linden +++ b/indra/llcorehttp/README.Linden @@ -529,6 +529,14 @@ HttpOperation::addAsReply: TRACE, ToReplyQueue, Handle: 086D3148 data = NULL; + There are now helper functions in llmessage/llcorehttputil.h to + assist with LLSD usage. requestPostWithLLSD(...) provides a + requestPost()-like interface that takes an LLSD object rather than + a BufferArray. And responseToLLSD(...) attempts to convert a + BufferArray received from a server into an LLSD object. You can + find examples in llmeshrepository.cpp, llinventorymodel.cpp, + llinventorymodelbackgroundfetch.cpp and lltexturefetch.cpp. + LLSD will often go hand-in-hand with BufferArray and data transport. But you can also do all the streaming I/O you'd expect of a std::iostream object: diff --git a/indra/newview/llinventorymodel.cpp b/indra/newview/llinventorymodel.cpp index ee28cef640..dab3a4c06d 100755 --- a/indra/newview/llinventorymodel.cpp +++ b/indra/newview/llinventorymodel.cpp @@ -3974,65 +3974,63 @@ LLInventoryModel::FetchItemHttpHandler::~FetchItemHttpHandler() void LLInventoryModel::FetchItemHttpHandler::onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * response) { - // Single-pass do-while used for common exit handling - do + do // Single-pass do-while used for common exit handling { LLCore::HttpStatus status(response->getStatus()); // status = LLCore::HttpStatus(404); // Dev tool to force error handling if (! status) { processFailure(status, response); + break; // Goto common exit } - else - { - LLCore::BufferArray * body(response->getBody()); - // body = NULL; // Dev tool to force error handling - if (! body || ! body->size()) - { - LL_WARNS(LOG_INV) << "Missing data in inventory item query." << LL_ENDL; - processFailure("HTTP response for inventory item query missing body", response); - break; // Goto common exit - } - // body->write(0, "Garbage Response", 16); // Dev tool to force error handling - LLSD body_llsd; - if (! LLCoreHttpUtil::responseToLLSD(response, true, body_llsd)) - { - // INFOS-level logging will occur on the parsed failure - processFailure("HTTP response for inventory item query has malformed LLSD", response); - break; // Goto common exit - } + LLCore::BufferArray * body(response->getBody()); + // body = NULL; // Dev tool to force error handling + if (! body || ! body->size()) + { + LL_WARNS(LOG_INV) << "Missing data in inventory item query." << LL_ENDL; + processFailure("HTTP response for inventory item query missing body", response); + break; // Goto common exit + } - // Expect top-level structure to be a map - // body_llsd = LLSD::emptyArray(); // Dev tool to force error handling - if (! body_llsd.isMap()) - { - processFailure("LLSD response for inventory item not a map", response); - break; // Goto common exit - } + // body->write(0, "Garbage Response", 16); // Dev tool to force error handling + LLSD body_llsd; + if (! LLCoreHttpUtil::responseToLLSD(response, true, body_llsd)) + { + // INFOS-level logging will occur on the parsed failure + processFailure("HTTP response for inventory item query has malformed LLSD", response); + break; // Goto common exit + } - // Check for 200-with-error failures - // - // Original Responder-based serivce model didn't check for these errors. - // It may be more robust to ignore this condition. With aggregated requests, - // an error in one inventory item might take down the entire request. - // So if this instead broke up the aggregated items into single requests, - // maybe that would make progress. Or perhaps there's structured information - // that can tell us what went wrong. Need to dig into this and firm up - // the API. - // - // body_llsd["error"] = LLSD::emptyMap(); // Dev tool to force error handling - // body_llsd["error"]["identifier"] = "Development"; - // body_llsd["error"]["message"] = "You left development code in the viewer"; - if (body_llsd.has("error")) - { - processFailure("Inventory application error (200-with-error)", response); - break; // Goto common exit - } + // Expect top-level structure to be a map + // body_llsd = LLSD::emptyArray(); // Dev tool to force error handling + if (! body_llsd.isMap()) + { + processFailure("LLSD response for inventory item not a map", response); + break; // Goto common exit + } - // Okay, process data if possible - processData(body_llsd, response); + // Check for 200-with-error failures + // + // Original Responder-based serivce model didn't check for these errors. + // It may be more robust to ignore this condition. With aggregated requests, + // an error in one inventory item might take down the entire request. + // So if this instead broke up the aggregated items into single requests, + // maybe that would make progress. Or perhaps there's structured information + // that can tell us what went wrong. Need to dig into this and firm up + // the API. + // + // body_llsd["error"] = LLSD::emptyMap(); // Dev tool to force error handling + // body_llsd["error"]["identifier"] = "Development"; + // body_llsd["error"]["message"] = "You left development code in the viewer"; + if (body_llsd.has("error")) + { + processFailure("Inventory application error (200-with-error)", response); + break; // Goto common exit } + + // Okay, process data if possible + processData(body_llsd, response); } while (false); diff --git a/indra/newview/llinventorymodelbackgroundfetch.cpp b/indra/newview/llinventorymodelbackgroundfetch.cpp index de1d123fe5..f18832fe95 100755 --- a/indra/newview/llinventorymodelbackgroundfetch.cpp +++ b/indra/newview/llinventorymodelbackgroundfetch.cpp @@ -42,6 +42,56 @@ #include "bufferstream.h" #include "llcorehttputil.h" +// History (may be apocryphal) +// +// Around V2, an HTTP inventory download mechanism was added +// along with inventory LINK items referencing other inventory +// items. As part of this, at login, the entire inventory +// structure is downloaded 'in the background' using the +// backgroundFetch()/bulkFetch() methods. The UDP path can +// still be used and is found in the 'DEPRECATED OLD CODE' +// section. +// +// The old UDP path implemented a throttle that adapted +// itself during running. The mechanism survived info HTTP +// somewhat but was pinned to poll the HTTP plumbing at +// 0.5S intervals. The reasons for this particular value +// have been lost. It's possible to switch between UDP +// and HTTP while this is happening but there may be +// surprises in what happens in that case. +// +// Conversion to llcorehttp reduced the number of connections +// used but batches more data and queues more requests (but +// doesn't due pipelining due to libcurl restrictions). The +// poll interval above was re-examined and reduced to get +// inventory into the viewer more quickly. +// +// Possible future work: +// +// * Don't download the entire heirarchy in one go (which +// might have been how V1 worked). Implications for +// links (which may not have a valid target) and search +// which would then be missing data. +// +// * Review the download rate throttling. Slow then fast? +// Detect bandwidth usage and speed up when it drops? +// +// * A lot of calls to notifyObservers(). It looks like +// these could be collapsed by maintaining a 'dirty' +// bit and there appears to be an attempt to do this. +// But it isn't used or is used in a limited fashion. +// Are there semanic issues requiring a call after certain +// updateItem() calls? +// +// * An error on a fetch could be due to one item in the batch. +// If the batch were broken up, perhaps more of the inventory +// would download. (Handwave here, not certain this is an +// issue in practice.) +// +// * Conversion to AISv3. +// + + namespace { @@ -488,11 +538,11 @@ void LLInventoryModelBackgroundFetch::bulkFetch() // a fast/slow fetch throttle. Once login is complete and the scene // is mostly loaded, we could turn up the throttle and fill missing // inventory more quickly. + static const U32 max_batch_size(10); static const S32 max_concurrent_fetches(12); // Outstanding requests, not connections static const F32 new_min_time(0.05f); // *HACK: Clean this up when old code goes away entirely. - static const U32 max_batch_size(10); - mMinTimeBetweenFetches = 0.01f; + mMinTimeBetweenFetches = new_min_time; if (mMinTimeBetweenFetches < new_min_time) { mMinTimeBetweenFetches = new_min_time; // *HACK: See above. @@ -702,63 +752,61 @@ namespace void BGFolderHttpHandler::onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * response) { - // Single-pass do-while used for common exit handling - do + do // Single-pass do-while used for common exit handling { LLCore::HttpStatus status(response->getStatus()); // status = LLCore::HttpStatus(404); // Dev tool to force error handling if (! status) { processFailure(status, response); + break; // Goto common exit } - else - { - // Response body should be present. - LLCore::BufferArray * body(response->getBody()); - // body = NULL; // Dev tool to force error handling - if (! body || ! body->size()) - { - LL_WARNS(LOG_INV) << "Missing data in inventory folder query." << LL_ENDL; - processFailure("HTTP response missing expected body", response); - break; // Goto common exit - } - // Could test 'Content-Type' header but probably unreliable. + // Response body should be present. + LLCore::BufferArray * body(response->getBody()); + // body = NULL; // Dev tool to force error handling + if (! body || ! body->size()) + { + LL_WARNS(LOG_INV) << "Missing data in inventory folder query." << LL_ENDL; + processFailure("HTTP response missing expected body", response); + break; // Goto common exit + } - // Convert response to LLSD - // body->write(0, "Garbage Response", 16); // Dev tool to force error handling - LLSD body_llsd; - if (! LLCoreHttpUtil::responseToLLSD(response, true, body_llsd)) - { - // INFOS-level logging will occur on the parsed failure - processFailure("HTTP response contained malformed LLSD", response); - break; // goto common exit - } + // Could test 'Content-Type' header but probably unreliable. - // Expect top-level structure to be a map - // body_llsd = LLSD::emptyArray(); // Dev tool to force error handling - if (! body_llsd.isMap()) - { - processFailure("LLSD response not a map", response); - break; // goto common exit - } + // Convert response to LLSD + // body->write(0, "Garbage Response", 16); // Dev tool to force error handling + LLSD body_llsd; + if (! LLCoreHttpUtil::responseToLLSD(response, true, body_llsd)) + { + // INFOS-level logging will occur on the parsed failure + processFailure("HTTP response contained malformed LLSD", response); + break; // goto common exit + } - // Check for 200-with-error failures - // - // See comments in llinventorymodel.cpp about this mode of error. - // - // body_llsd["error"] = LLSD::emptyMap(); // Dev tool to force error handling - // body_llsd["error"]["identifier"] = "Development"; - // body_llsd["error"]["message"] = "You left development code in the viewer"; - if (body_llsd.has("error")) - { - processFailure("Inventory application error (200-with-error)", response); - break; // goto common exit - } + // Expect top-level structure to be a map + // body_llsd = LLSD::emptyArray(); // Dev tool to force error handling + if (! body_llsd.isMap()) + { + processFailure("LLSD response not a map", response); + break; // goto common exit + } - // Okay, process data if possible - processData(body_llsd, response); + // Check for 200-with-error failures + // + // See comments in llinventorymodel.cpp about this mode of error. + // + // body_llsd["error"] = LLSD::emptyMap(); // Dev tool to force error handling + // body_llsd["error"]["identifier"] = "Development"; + // body_llsd["error"]["message"] = "You left development code in the viewer"; + if (body_llsd.has("error")) + { + processFailure("Inventory application error (200-with-error)", response); + break; // goto common exit } + + // Okay, process data if possible + processData(body_llsd, response); } while (false); -- cgit v1.2.3 From f9c73f089a5a213e965aa59d6128ba5dd381186e Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Mon, 6 Oct 2014 05:44:54 +0000 Subject: BUG-7454 Viewer crashes while logging with a turned off "HTTP Inventory" The HttpRequest objects used in inventory ops are created lazily when needed. The update() operation didn't protect against their not being created. Added some guard tests around the update() operations and we're good. --- indra/newview/llinventorymodel.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'indra') diff --git a/indra/newview/llinventorymodel.cpp b/indra/newview/llinventorymodel.cpp index dab3a4c06d..f92332dea5 100755 --- a/indra/newview/llinventorymodel.cpp +++ b/indra/newview/llinventorymodel.cpp @@ -2436,11 +2436,11 @@ void LLInventoryModel::initHttpRequest() void LLInventoryModel::handleResponses(bool foreground) { - if (foreground) + if (foreground && mHttpRequestFG) { mHttpRequestFG->update(0); } - else + else if (! foreground && mHttpRequestBG) { mHttpRequestBG->update(50000L); } -- cgit v1.2.3 From ec4fd2f0e226bb2cae5982760317e1d6ea2d2d69 Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Fri, 10 Oct 2014 16:43:04 -0400 Subject: MAINT-4564 HTTP Pipelining is not happening in Drano HTTP Phase 4 Incorporate the new libcurl 7.38.0 build with curl bug 1420 workaround. Add developer-centric testing code to evaluate the workaround or a future fix for 1420. --- indra/llcorehttp/_httpoprequest.cpp | 4 ++-- indra/llcorehttp/_httppolicy.cpp | 12 ++++++++++++ indra/llcorehttp/httpcommon.cpp | 5 +++-- 3 files changed, 17 insertions(+), 4 deletions(-) (limited to 'indra') diff --git a/indra/llcorehttp/_httpoprequest.cpp b/indra/llcorehttp/_httpoprequest.cpp index bbda0b82fd..fbbb1614fb 100755 --- a/indra/llcorehttp/_httpoprequest.cpp +++ b/indra/llcorehttp/_httpoprequest.cpp @@ -626,8 +626,8 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service) // *TODO: Find a better scheme than timeouts to guarantee liveness. xfer_timeout *= cpolicy.mPipelining; } - // *DEBUG: Useful for timeout handling and "[curl:bugs] #1420" tests - // xfer_timeout = 3L; + // *DEBUG: Enable following override for timeout handling and "[curl:bugs] #1420" tests + // xfer_timeout = 1L; code = curl_easy_setopt(mCurlHandle, CURLOPT_TIMEOUT, xfer_timeout); check_curl_easy_code(code, CURLOPT_TIMEOUT); code = curl_easy_setopt(mCurlHandle, CURLOPT_CONNECTTIMEOUT, timeout); diff --git a/indra/llcorehttp/_httppolicy.cpp b/indra/llcorehttp/_httppolicy.cpp index 09b9206f63..e5d6321401 100755 --- a/indra/llcorehttp/_httppolicy.cpp +++ b/indra/llcorehttp/_httppolicy.cpp @@ -414,6 +414,18 @@ bool HttpPolicy::stageAfterCompletion(HttpOpRequest * op) // Retry or finalize if (! op->mStatus) { + // *DEBUG: For "[curl:bugs] #1420" tests. This will interfere + // with unit tests due to allocation retention by logging code. + // But you won't be checking this in enabled. +#if 0 + if (op->mStatus == HttpStatus(HttpStatus::EXT_CURL_EASY, CURLE_OPERATION_TIMEDOUT)) + { + LL_WARNS(LOG_CORE) << "HTTP request " << static_cast(op) + << " timed out." + << LL_ENDL; + } +#endif + // If this failed, we might want to retry. if (op->mPolicyRetries < op->mPolicyRetryLimit && op->mStatus.isRetryable()) { diff --git a/indra/llcorehttp/httpcommon.cpp b/indra/llcorehttp/httpcommon.cpp index 8714915fa2..7907e958a4 100755 --- a/indra/llcorehttp/httpcommon.cpp +++ b/indra/llcorehttp/httpcommon.cpp @@ -254,8 +254,9 @@ bool HttpStatus::isRetryable() const *this == op_timedout || // Timer expired *this == post_error || // Transport problem *this == partial_file || // Data inconsistency in response - *this == inv_cont_range || // Short data read disagrees with content-range - *this == inv_status); // Inv status can reflect internal state problem in libcurl + // *DEBUG: Comment out 'inv_status' test for [curl:bugs] #1420 testing. + *this == inv_status || // Inv status can reflect internal state problem in libcurl + *this == inv_cont_range); // Short data read disagrees with content-range } } // end namespace LLCore -- cgit v1.2.3 From ecba0d552471d005fde077203fd9a8da468faabb Mon Sep 17 00:00:00 2001 From: Oz Linden Date: Wed, 15 Oct 2014 15:12:37 -0400 Subject: MAINT-4560: turn on display of group title by default --- indra/newview/app_settings/settings.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'indra') diff --git a/indra/newview/app_settings/settings.xml b/indra/newview/app_settings/settings.xml index 22d64d4d14..49077cf416 100755 --- a/indra/newview/app_settings/settings.xml +++ b/indra/newview/app_settings/settings.xml @@ -9478,7 +9478,7 @@ Type Boolean Value - 0 + 1 NameTagShowDisplayNames -- cgit v1.2.3 From 02e2235277a90f2e291557a429ae4e5de3e0d3b6 Mon Sep 17 00:00:00 2001 From: Oz Linden Date: Tue, 28 Oct 2014 09:55:55 -0400 Subject: increment viewer version to 3.7.20 --- indra/newview/VIEWER_VERSION.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'indra') diff --git a/indra/newview/VIEWER_VERSION.txt b/indra/newview/VIEWER_VERSION.txt index 7707ee0e29..82a60c0bb1 100644 --- a/indra/newview/VIEWER_VERSION.txt +++ b/indra/newview/VIEWER_VERSION.txt @@ -1 +1 @@ -3.7.19 +3.7.20 -- cgit v1.2.3 From 2301cf800f40101baed7a5936683d0b1e4968be1 Mon Sep 17 00:00:00 2001 From: Oz Linden Date: Mon, 10 Nov 2014 14:41:41 -0500 Subject: increment viewer version to 3.7.21 --- indra/newview/VIEWER_VERSION.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'indra') diff --git a/indra/newview/VIEWER_VERSION.txt b/indra/newview/VIEWER_VERSION.txt index 82a60c0bb1..c6cff55cf7 100644 --- a/indra/newview/VIEWER_VERSION.txt +++ b/indra/newview/VIEWER_VERSION.txt @@ -1 +1 @@ -3.7.20 +3.7.21 -- cgit v1.2.3