diff options
38 files changed, 2811 insertions, 1157 deletions
@@ -491,3 +491,4 @@ a7872554f3665588f1e8347d472cec3a299254b3 3.7.14-release 562e7dace7465060ac9adb2e8eca800b699ff024 3.7.16-release bcc2770e21c125e0bab59141c51db9145aec068d 3.7.17-release 2729c1daf0257d68a40bdbc4acf1a16184974bbd 3.7.18-release +82973b38a6c9a457333e3519e4f2b16bb5eedf47 3.7.19-release diff --git a/autobuild.xml b/autobuild.xml index 571921032e..ccf08f8c94 100755 --- a/autobuild.xml +++ b/autobuild.xml @@ -282,9 +282,9 @@ <key>archive</key> <map> <key>hash</key> - <string>f5a699c93beb1a854d0b51382b5cecc8</string> + <string>40b1c6b3727ebedafc2f1a172797ccd1</string> <key>url</key> - <string>http://automated-builds-secondlife-com.s3.amazonaws.com/hg/repo/3pl_3p-curl-update/rev/290664/arch/Darwin/installer/curl-7.37.0-darwin-20140605.tar.bz2</string> + <string>http://automated-builds-secondlife-com.s3.amazonaws.com/hg/repo/3pl_3p-curl-update/rev/295367/arch/Darwin/installer/curl-7.38.0-darwin-20141010.tar.bz2</string> </map> <key>name</key> <string>darwin</string> @@ -294,9 +294,9 @@ <key>archive</key> <map> <key>hash</key> - <string>2bc285edffd0e55e0cd6290f39854a89</string> + <string>06149da3d7a34adf40853f813ae55328</string> <key>url</key> - <string>http://automated-builds-secondlife-com.s3.amazonaws.com/hg/repo/3pl_3p-curl-update/rev/290664/arch/Linux/installer/curl-7.37.0-linux-20140605.tar.bz2</string> + <string>http://automated-builds-secondlife-com.s3.amazonaws.com/hg/repo/3pl_3p-curl-update/rev/295367/arch/Linux/installer/curl-7.38.0-linux-20141010.tar.bz2</string> </map> <key>name</key> <string>linux</string> @@ -306,9 +306,9 @@ <key>archive</key> <map> <key>hash</key> - <string>8d3b197d7a114d2b688d2831a0a59757</string> + <string>e4280eae792a5f13bc9d01d8cfb7c557</string> <key>url</key> - <string>http://automated-builds-secondlife-com.s3.amazonaws.com/hg/repo/3pl_3p-curl-update/rev/290664/arch/CYGWIN/installer/curl-7.37.0-windows-20140605.tar.bz2</string> + <string>http://automated-builds-secondlife-com.s3.amazonaws.com/hg/repo/3pl_3p-curl-update/rev/295367/arch/CYGWIN/installer/curl-7.38.0-windows-20141010.tar.bz2</string> </map> <key>name</key> <string>windows</string> diff --git a/indra/cmake/00-COMPILE-LINK-RUN.txt b/indra/cmake/00-COMPILE-LINK-RUN.txt index d08cc2dc0c..49b899c50d 100644 --- a/indra/cmake/00-COMPILE-LINK-RUN.txt +++ b/indra/cmake/00-COMPILE-LINK-RUN.txt @@ -115,12 +115,12 @@ Compilation ---------------------------------------------------------------------------- Notes: - 1. We’re also building dylibs in a somewhat unusual way. They’re + 1. We're also building dylibs in a somewhat unusual way. They're currently being generated with a link path of - ‘@executable_path/../Resources/<library>’. If we were to follow - the recommendations in dyld’s man page, we’d instead reference - ‘@loader_path/<library>’, use -rpath on the executable link - (pointing to the ‘Resources’ subdir of the main executable), and + '@executable_path/../Resources/<library>'. If we were to follow + the recommendations in dyld's man page, we’d instead reference + '@loader_path/<library>', use -rpath on the executable link + (pointing to the 'Resources' subdir of the main executable), and be able to avoid some symlinking in the .app tree. 2. Use the -headerpad_max_install_names link option on all .dylibs. @@ -184,7 +184,7 @@ Linking second, incompatible version of the library. Switching colladadom to a static library ended the re-export problem. - * Preventing re-export is not sufficient. other libraries will + * Preventing re-export is not sufficient. Other libraries will still be shipped as shared and they can still have Singleton and Fragile Base Class issues. A DLL may be built with a static archive of a library that has global data. That same static diff --git a/indra/llcorehttp/README.Linden b/indra/llcorehttp/README.Linden index eb6ccab3bc..c3aaa9158d 100644 --- a/indra/llcorehttp/README.Linden +++ b/indra/llcorehttp/README.Linden @@ -529,6 +529,14 @@ HttpOperation::addAsReply: TRACE, ToReplyQueue, Handle: 086D3148 data = NULL; + There are now helper functions in llmessage/llcorehttputil.h to + assist with LLSD usage. requestPostWithLLSD(...) provides a + requestPost()-like interface that takes an LLSD object rather than + a BufferArray. And responseToLLSD(...) attempts to convert a + BufferArray received from a server into an LLSD object. You can + find examples in llmeshrepository.cpp, llinventorymodel.cpp, + llinventorymodelbackgroundfetch.cpp and lltexturefetch.cpp. + LLSD will often go hand-in-hand with BufferArray and data transport. But you can also do all the streaming I/O you'd expect of a std::iostream object: diff --git a/indra/llcorehttp/_httpinternal.h b/indra/llcorehttp/_httpinternal.h index f80d7f60f5..a2a60ca056 100755 --- a/indra/llcorehttp/_httpinternal.h +++ b/indra/llcorehttp/_httpinternal.h @@ -145,8 +145,11 @@ const int HTTP_CONNECTION_LIMIT_DEFAULT = 8; const int HTTP_CONNECTION_LIMIT_MIN = 1; const int HTTP_CONNECTION_LIMIT_MAX = 256; -// Miscellaneous defaults +// Pipelining limits const long HTTP_PIPELINING_DEFAULT = 0L; +const long HTTP_PIPELINING_MAX = 20L; + +// Miscellaneous defaults const bool HTTP_USE_RETRY_AFTER_DEFAULT = true; const long HTTP_THROTTLE_RATE_DEFAULT = 0L; diff --git a/indra/llcorehttp/_httplibcurl.cpp b/indra/llcorehttp/_httplibcurl.cpp index e56bc84174..81b44ab90b 100755 --- a/indra/llcorehttp/_httplibcurl.cpp +++ b/indra/llcorehttp/_httplibcurl.cpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012-2013, Linden Research, Inc. + * Copyright (C) 2012-2014, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -33,6 +33,17 @@ #include "llhttpconstants.h" +namespace +{ + +// Error testing and reporting for libcurl status codes +void check_curl_multi_code(CURLMcode code); +void check_curl_multi_code(CURLMcode code, int curl_setopt_option); + +static const char * const LOG_CORE("CoreHttp"); + +} // end anonymous namespace + namespace LLCore { @@ -40,16 +51,18 @@ namespace LLCore HttpLibcurl::HttpLibcurl(HttpService * service) : mService(service), + mHandleCache(), mPolicyCount(0), mMultiHandles(NULL), - mActiveHandles(NULL) + mActiveHandles(NULL), + mDirtyPolicy(NULL) {} HttpLibcurl::~HttpLibcurl() { shutdown(); - + mService = NULL; } @@ -81,6 +94,9 @@ void HttpLibcurl::shutdown() delete [] mActiveHandles; mActiveHandles = NULL; + + delete [] mDirtyPolicy; + mDirtyPolicy = NULL; } mPolicyCount = 0; @@ -95,11 +111,18 @@ void HttpLibcurl::start(int policy_count) mPolicyCount = policy_count; mMultiHandles = new CURLM * [mPolicyCount]; mActiveHandles = new int [mPolicyCount]; + mDirtyPolicy = new bool [mPolicyCount]; for (int policy_class(0); policy_class < mPolicyCount; ++policy_class) { - mMultiHandles[policy_class] = curl_multi_init(); + if (NULL == (mMultiHandles[policy_class] = curl_multi_init())) + { + LL_ERRS(LOG_CORE) << "Failed to allocate multi handle in libcurl." + << LL_ENDL; + } mActiveHandles[policy_class] = 0; + mDirtyPolicy[policy_class] = false; + policyUpdated(policy_class); } } @@ -117,8 +140,19 @@ HttpService::ELoopSpeed HttpLibcurl::processTransport() // Give libcurl some cycles to do I/O & callbacks for (int policy_class(0); policy_class < mPolicyCount; ++policy_class) { - if (! mActiveHandles[policy_class] || ! mMultiHandles[policy_class]) + if (! mMultiHandles[policy_class]) + { + // No handle, nothing to do. + continue; + } + if (! mActiveHandles[policy_class]) { + // If we've gone quiet and there's a dirty update, apply it, + // otherwise we're done. + if (mDirtyPolicy[policy_class]) + { + policyUpdated(policy_class); + } continue; } @@ -153,9 +187,9 @@ HttpService::ELoopSpeed HttpLibcurl::processTransport() } else { - LL_WARNS_ONCE("CoreHttp") << "Unexpected message from libcurl. Msg code: " - << msg->msg - << LL_ENDL; + LL_WARNS_ONCE(LOG_CORE) << "Unexpected message from libcurl. Msg code: " + << msg->msg + << LL_ENDL; } msgs_in_queue = 0; } @@ -184,23 +218,28 @@ void HttpLibcurl::addOp(HttpOpRequest * op) } // Make the request live - curl_multi_add_handle(mMultiHandles[op->mReqPolicy], op->mCurlHandle); + CURLMcode code; + code = curl_multi_add_handle(mMultiHandles[op->mReqPolicy], op->mCurlHandle); + if (CURLM_OK != code) + { + // *TODO: Better cleanup and recovery but not much we can do here. + check_curl_multi_code(code); + return; + } op->mCurlActive = true; + mActiveOps.insert(op); + ++mActiveHandles[op->mReqPolicy]; if (op->mTracing > HTTP_TRACE_OFF) { HttpPolicy & policy(mService->getPolicy()); - LL_INFOS("CoreHttp") << "TRACE, ToActiveQueue, Handle: " - << static_cast<HttpHandle>(op) - << ", Actives: " << mActiveOps.size() - << ", Readies: " << policy.getReadyCount(op->mReqPolicy) - << LL_ENDL; + LL_INFOS(LOG_CORE) << "TRACE, ToActiveQueue, Handle: " + << static_cast<HttpHandle>(op) + << ", Actives: " << mActiveOps.size() + << ", Readies: " << policy.getReadyCount(op->mReqPolicy) + << LL_ENDL; } - - // On success, make operation active - mActiveOps.insert(op); - ++mActiveHandles[op->mReqPolicy]; } @@ -241,16 +280,16 @@ void HttpLibcurl::cancelRequest(HttpOpRequest * op) // Detach from multi and recycle handle curl_multi_remove_handle(mMultiHandles[op->mReqPolicy], op->mCurlHandle); - curl_easy_cleanup(op->mCurlHandle); + mHandleCache.freeHandle(op->mCurlHandle); op->mCurlHandle = NULL; // Tracing if (op->mTracing > HTTP_TRACE_OFF) { - LL_INFOS("CoreHttp") << "TRACE, RequestCanceled, Handle: " - << static_cast<HttpHandle>(op) - << ", Status: " << op->mStatus.toTerseString() - << LL_ENDL; + LL_INFOS(LOG_CORE) << "TRACE, RequestCanceled, Handle: " + << static_cast<HttpHandle>(op) + << ", Status: " << op->mStatus.toTerseString() + << LL_ENDL; } // Cancel op and deliver for notification @@ -267,18 +306,18 @@ bool HttpLibcurl::completeRequest(CURLM * multi_handle, CURL * handle, CURLcode if (handle != op->mCurlHandle || ! op->mCurlActive) { - LL_WARNS("CoreHttp") << "libcurl handle and HttpOpRequest handle in disagreement or inactive request." - << " Handle: " << static_cast<HttpHandle>(handle) - << LL_ENDL; + LL_WARNS(LOG_CORE) << "libcurl handle and HttpOpRequest handle in disagreement or inactive request." + << " Handle: " << static_cast<HttpHandle>(handle) + << LL_ENDL; return false; } active_set_t::iterator it(mActiveOps.find(op)); if (mActiveOps.end() == it) { - LL_WARNS("CoreHttp") << "libcurl completion for request not on active list. Continuing." - << " Handle: " << static_cast<HttpHandle>(handle) - << LL_ENDL; + LL_WARNS(LOG_CORE) << "libcurl completion for request not on active list. Continuing." + << " Handle: " << static_cast<HttpHandle>(handle) + << LL_ENDL; return false; } @@ -309,25 +348,25 @@ bool HttpLibcurl::completeRequest(CURLM * multi_handle, CURL * handle, CURLcode } else { - LL_WARNS("CoreHttp") << "Invalid HTTP response code (" - << http_status << ") received from server." - << LL_ENDL; + LL_WARNS(LOG_CORE) << "Invalid HTTP response code (" + << http_status << ") received from server." + << LL_ENDL; op->mStatus = HttpStatus(HttpStatus::LLCORE, HE_INVALID_HTTP_STATUS); } } // Detach from multi and recycle handle curl_multi_remove_handle(multi_handle, handle); - curl_easy_cleanup(handle); + mHandleCache.freeHandle(op->mCurlHandle); op->mCurlHandle = NULL; // Tracing if (op->mTracing > HTTP_TRACE_OFF) { - LL_INFOS("CoreHttp") << "TRACE, RequestComplete, Handle: " - << static_cast<HttpHandle>(op) - << ", Status: " << op->mStatus.toTerseString() - << LL_ENDL; + LL_INFOS(LOG_CORE) << "TRACE, RequestComplete, Handle: " + << static_cast<HttpHandle>(op) + << ", Status: " << op->mStatus.toTerseString() + << LL_ENDL; } // Dispatch to next stage @@ -351,6 +390,164 @@ int HttpLibcurl::getActiveCountInClass(int policy_class) const return mActiveHandles ? mActiveHandles[policy_class] : 0; } +void HttpLibcurl::policyUpdated(int policy_class) +{ + if (policy_class < 0 || policy_class >= mPolicyCount || ! mMultiHandles) + { + return; + } + + HttpPolicy & policy(mService->getPolicy()); + + if (! mActiveHandles[policy_class]) + { + // Clear to set options. As of libcurl 7.37.0, if a pipelining + // multi handle has active requests and you try to set the + // multi handle to non-pipelining, the library gets very angry + // and goes off the rails corrupting memory. A clue that you're + // about to crash is that you'll get a missing server response + // error (curl code 9). So, if options are to be set, we let + // the multi handle run out of requests, then set options, and + // re-enable request processing. + // + // All of this stall mechanism exists for this reason. If + // libcurl becomes more resilient later, it should be possible + // to remove all of this. The connection limit settings are fine, + // it's just that pipelined-to-non-pipelined transition that + // is fatal at the moment. + + HttpPolicyClass & options(policy.getClassOptions(policy_class)); + CURLM * multi_handle(mMultiHandles[policy_class]); + CURLMcode code; + + // Enable policy if stalled + policy.stallPolicy(policy_class, false); + mDirtyPolicy[policy_class] = false; + + if (options.mPipelining > 1) + { + // We'll try to do pipelining on this multihandle + code = curl_multi_setopt(multi_handle, + CURLMOPT_PIPELINING, + 1L); + check_curl_multi_code(code, CURLMOPT_PIPELINING); + code = curl_multi_setopt(multi_handle, + CURLMOPT_MAX_PIPELINE_LENGTH, + long(options.mPipelining)); + check_curl_multi_code(code, CURLMOPT_MAX_PIPELINE_LENGTH); + code = curl_multi_setopt(multi_handle, + CURLMOPT_MAX_HOST_CONNECTIONS, + long(options.mPerHostConnectionLimit)); + check_curl_multi_code(code, CURLMOPT_MAX_HOST_CONNECTIONS); + code = curl_multi_setopt(multi_handle, + CURLMOPT_MAX_TOTAL_CONNECTIONS, + long(options.mConnectionLimit)); + check_curl_multi_code(code, CURLMOPT_MAX_TOTAL_CONNECTIONS); + } + else + { + code = curl_multi_setopt(multi_handle, + CURLMOPT_PIPELINING, + 0L); + check_curl_multi_code(code, CURLMOPT_PIPELINING); + code = curl_multi_setopt(multi_handle, + CURLMOPT_MAX_HOST_CONNECTIONS, + 0L); + check_curl_multi_code(code, CURLMOPT_MAX_HOST_CONNECTIONS); + code = curl_multi_setopt(multi_handle, + CURLMOPT_MAX_TOTAL_CONNECTIONS, + long(options.mConnectionLimit)); + check_curl_multi_code(code, CURLMOPT_MAX_TOTAL_CONNECTIONS); + } + } + else if (! mDirtyPolicy[policy_class]) + { + // Mark policy dirty and request a stall in the policy. + // When policy goes idle, we'll re-invoke this method + // and perform the change. Don't allow this thread to + // sleep while we're waiting for quiescence, we'll just + // stop processing. + mDirtyPolicy[policy_class] = true; + policy.stallPolicy(policy_class, true); + } +} + +// --------------------------------------- +// HttpLibcurl::HandleCache +// --------------------------------------- + +HttpLibcurl::HandleCache::HandleCache() + : mHandleTemplate(NULL) +{ + mCache.reserve(50); +} + + +HttpLibcurl::HandleCache::~HandleCache() +{ + if (mHandleTemplate) + { + curl_easy_cleanup(mHandleTemplate); + mHandleTemplate = NULL; + } + + for (handle_cache_t::iterator it(mCache.begin()); mCache.end() != it; ++it) + { + curl_easy_cleanup(*it); + } + mCache.clear(); +} + + +CURL * HttpLibcurl::HandleCache::getHandle() +{ + CURL * ret(NULL); + + if (! mCache.empty()) + { + // Fastest path to handle + ret = mCache.back(); + mCache.pop_back(); + } + else if (mHandleTemplate) + { + // Still fast path + ret = curl_easy_duphandle(mHandleTemplate); + } + else + { + // When all else fails + ret = curl_easy_init(); + } + + return ret; +} + + +void HttpLibcurl::HandleCache::freeHandle(CURL * handle) +{ + if (! handle) + { + return; + } + + curl_easy_reset(handle); + if (! mHandleTemplate) + { + // Save the first freed handle as a template. + mHandleTemplate = handle; + } + else + { + // Otherwise add it to the cache + if (mCache.size() >= mCache.capacity()) + { + mCache.reserve(mCache.capacity() + 50); + } + mCache.push_back(handle); + } +} + // --------------------------------------- // Free functions @@ -376,3 +573,29 @@ struct curl_slist * append_headers_to_slist(const HttpHeaders * headers, struct } // end namespace LLCore + + +namespace +{ + +void check_curl_multi_code(CURLMcode code, int curl_setopt_option) +{ + if (CURLM_OK != code) + { + LL_WARNS(LOG_CORE) << "libcurl multi error detected: " << curl_multi_strerror(code) + << ", curl_multi_setopt option: " << curl_setopt_option + << LL_ENDL; + } +} + + +void check_curl_multi_code(CURLMcode code) +{ + if (CURLM_OK != code) + { + LL_WARNS(LOG_CORE) << "libcurl multi error detected: " << curl_multi_strerror(code) + << LL_ENDL; + } +} + +} // end anonymous namespace diff --git a/indra/llcorehttp/_httplibcurl.h b/indra/llcorehttp/_httplibcurl.h index 67f98dd4f0..ffc24c63a8 100755 --- a/indra/llcorehttp/_httplibcurl.h +++ b/indra/llcorehttp/_httplibcurl.h @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012-2013, Linden Research, Inc. + * Copyright (C) 2012-2014, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -116,6 +116,31 @@ public: /// Threading: called by worker thread. bool cancel(HttpHandle handle); + /// Informs transport that a particular policy class has had + /// options changed and so should effect any transport state + /// change necessary to effect those changes. Used mainly for + /// initialization and dynamic option setting. + /// + /// Threading: called by worker thread. + void policyUpdated(int policy_class); + + /// Allocate a curl handle for caller. May be freed using + /// either the freeHandle() method or calling curl_easy_cleanup() + /// directly. + /// + /// @return Libcurl handle (CURL *) or NULL on allocation + /// problem. Handle will be in curl_easy_reset() + /// condition. + /// + /// Threading: callable by worker thread. + /// + /// Deprecation: Expect this to go away after _httpoprequest is + /// refactored bringing code into this class. + CURL * getHandle() + { + return mHandleCache.getHandle(); + } + protected: /// Invoked when libcurl has indicated a request has been processed /// to completion and we need to move the request to a new state. @@ -127,13 +152,68 @@ protected: protected: typedef std::set<HttpOpRequest *> active_set_t; + + /// Simple request handle cache for libcurl. + /// + /// Handle creation is somewhat slow and chunky in libcurl and there's + /// a pretty good speedup to be had from handle re-use. So, a simple + /// vector is kept of 'freed' handles to be reused as needed. When + /// that is empty, the first freed handle is kept as a template for + /// handle duplication. This is still faster than creation from nothing. + /// And when that fails, we init fresh from curl_easy_init(). + /// + /// Handles allocated with getHandle() may be freed with either + /// freeHandle() or curl_easy_cleanup(). Choice may be dictated + /// by thread constraints. + /// + /// Threading: Single-threaded. May only be used by a single thread, + /// typically the worker thread. If freeing requests' handles in an + /// unknown threading context, use curl_easy_cleanup() for safety. + + class HandleCache + { + public: + HandleCache(); + ~HandleCache(); + + private: + HandleCache(const HandleCache &); // Not defined + void operator=(const HandleCache &); // Not defined + + public: + /// Allocate a curl handle for caller. May be freed using + /// either the freeHandle() method or calling curl_easy_cleanup() + /// directly. + /// + /// @return Libcurl handle (CURL *) or NULL on allocation + /// problem. + /// + /// Threading: Single-thread (worker) only. + CURL * getHandle(); + + /// Free a libcurl handle acquired by whatever means. Thread + /// safety is left to the caller. + /// + /// Threading: Single-thread (worker) only. + void freeHandle(CURL * handle); + + protected: + typedef std::vector<CURL *> handle_cache_t; + + protected: + CURL * mHandleTemplate; // Template for duplicating new handles + handle_cache_t mCache; // Cache of old handles + }; // end class HandleCache protected: - HttpService * mService; // Simple reference, not owner + HttpService * mService; // Simple reference, not owner + HandleCache mHandleCache; // Handle allocator, owner active_set_t mActiveOps; int mPolicyCount; - CURLM ** mMultiHandles; // One handle per policy class - int * mActiveHandles; // Active count per policy class + CURLM ** mMultiHandles; // One handle per policy class + int * mActiveHandles; // Active count per policy class + bool * mDirtyPolicy; // Dirty policy update waiting for stall (per pc) + }; // end class HttpLibcurl } // end namespace LLCore diff --git a/indra/llcorehttp/_httpoperation.cpp b/indra/llcorehttp/_httpoperation.cpp index 5bb0654652..fefe561f80 100755 --- a/indra/llcorehttp/_httpoperation.cpp +++ b/indra/llcorehttp/_httpoperation.cpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012-2013, Linden Research, Inc. + * Copyright (C) 2012-2014, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -38,6 +38,14 @@ #include "lltimer.h" +namespace +{ + +static const char * const LOG_CORE("CoreHttp"); + +} // end anonymous namespace + + namespace LLCore { @@ -94,8 +102,8 @@ void HttpOperation::stageFromRequest(HttpService *) // Default implementation should never be called. This // indicates an operation making a transition that isn't // defined. - LL_ERRS("CoreHttp") << "Default stageFromRequest method may not be called." - << LL_ENDL; + LL_ERRS(LOG_CORE) << "Default stageFromRequest method may not be called." + << LL_ENDL; } @@ -104,8 +112,8 @@ void HttpOperation::stageFromReady(HttpService *) // Default implementation should never be called. This // indicates an operation making a transition that isn't // defined. - LL_ERRS("CoreHttp") << "Default stageFromReady method may not be called." - << LL_ENDL; + LL_ERRS(LOG_CORE) << "Default stageFromReady method may not be called." + << LL_ENDL; } @@ -114,8 +122,8 @@ void HttpOperation::stageFromActive(HttpService *) // Default implementation should never be called. This // indicates an operation making a transition that isn't // defined. - LL_ERRS("CoreHttp") << "Default stageFromActive method may not be called." - << LL_ENDL; + LL_ERRS(LOG_CORE) << "Default stageFromActive method may not be called." + << LL_ENDL; } @@ -145,9 +153,9 @@ void HttpOperation::addAsReply() { if (mTracing > HTTP_TRACE_OFF) { - LL_INFOS("CoreHttp") << "TRACE, ToReplyQueue, Handle: " - << static_cast<HttpHandle>(this) - << LL_ENDL; + LL_INFOS(LOG_CORE) << "TRACE, ToReplyQueue, Handle: " + << static_cast<HttpHandle>(this) + << LL_ENDL; } if (mReplyQueue) diff --git a/indra/llcorehttp/_httpoprequest.cpp b/indra/llcorehttp/_httpoprequest.cpp index 43dd069bc6..fbbb1614fb 100755 --- a/indra/llcorehttp/_httpoprequest.cpp +++ b/indra/llcorehttp/_httpoprequest.cpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012-2013, Linden Research, Inc. + * Copyright (C) 2012-2014, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -47,6 +47,19 @@ #include "llhttpconstants.h" #include "llproxy.h" +// *DEBUG: "[curl:bugs] #1420" problem and testing. +// +// A pipelining problem, https://sourceforge.net/p/curl/bugs/1420/, +// was a source of Core_9 failures. Code related to this can be +// identified and tested by: +// * Looking for '[curl:bugs]' strings in source and following +// instructions there. +// * Set 'QAModeHttpTrace' to 2 or 3 in settings.xml and look for +// 'timed out' events in the log. +// * Enable the HttpRangeRequestsDisable debug setting which causes +// full asset fetches. These slow the pipelines down a bit. +// + namespace { @@ -94,6 +107,8 @@ void os_strlower(char * str); void check_curl_easy_code(CURLcode code); void check_curl_easy_code(CURLcode code, int curl_setopt_option); +static const char * const LOG_CORE("CoreHttp"); + } // end anonymous namespace @@ -155,6 +170,8 @@ HttpOpRequest::~HttpOpRequest() if (mCurlHandle) { + // Uncertain of thread context so free using + // safest method. curl_easy_cleanup(mCurlHandle); mCurlHandle = NULL; } @@ -376,6 +393,7 @@ void HttpOpRequest::setupCommon(HttpRequest::policy_t policy_id, // Junk may be left around from a failed request and that // needs to be cleaned out. // +// *TODO: Move this to _httplibcurl where it belongs. HttpStatus HttpOpRequest::prepareRequest(HttpService * service) { CURLcode code; @@ -409,17 +427,19 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service) // *FIXME: better error handling later HttpStatus status; - // Get global policy options - HttpPolicyGlobal & policy(service->getPolicy().getGlobalOptions()); + // Get global and class policy options + HttpPolicyGlobal & gpolicy(service->getPolicy().getGlobalOptions()); + HttpPolicyClass & cpolicy(service->getPolicy().getClassOptions(mReqPolicy)); - mCurlHandle = LLCurl::createStandardCurlHandle(); + mCurlHandle = service->getTransport().getHandle(); if (! mCurlHandle) { // We're in trouble. We'll continue but it won't go well. - LL_WARNS("CoreHttp") << "Failed to allocate libcurl easy handle. Continuing." - << LL_ENDL; + LL_WARNS(LOG_CORE) << "Failed to allocate libcurl easy handle. Continuing." + << LL_ENDL; return HttpStatus(HttpStatus::LLCORE, HE_BAD_ALLOC); } + code = curl_easy_setopt(mCurlHandle, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4); check_curl_easy_code(code, CURLOPT_IPRESOLVE); code = curl_easy_setopt(mCurlHandle, CURLOPT_NOSIGNAL, 1); @@ -460,30 +480,30 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service) code = curl_easy_setopt(mCurlHandle, CURLOPT_SSL_VERIFYHOST, 0); check_curl_easy_code(code, CURLOPT_SSL_VERIFYHOST); - if (policy.mUseLLProxy) + if (gpolicy.mUseLLProxy) { // Use the viewer-based thread-safe API which has a // fast/safe check for proxy enable. Would like to // encapsulate this someway... LLProxy::getInstance()->applyProxySettings(mCurlHandle); } - else if (policy.mHttpProxy.size()) + else if (gpolicy.mHttpProxy.size()) { // *TODO: This is fine for now but get fuller socks5/ // authentication thing going later.... - code = curl_easy_setopt(mCurlHandle, CURLOPT_PROXY, policy.mHttpProxy.c_str()); + code = curl_easy_setopt(mCurlHandle, CURLOPT_PROXY, gpolicy.mHttpProxy.c_str()); check_curl_easy_code(code, CURLOPT_PROXY); code = curl_easy_setopt(mCurlHandle, CURLOPT_PROXYTYPE, CURLPROXY_HTTP); check_curl_easy_code(code, CURLOPT_PROXYTYPE); } - if (policy.mCAPath.size()) + if (gpolicy.mCAPath.size()) { - code = curl_easy_setopt(mCurlHandle, CURLOPT_CAPATH, policy.mCAPath.c_str()); + code = curl_easy_setopt(mCurlHandle, CURLOPT_CAPATH, gpolicy.mCAPath.c_str()); check_curl_easy_code(code, CURLOPT_CAPATH); } - if (policy.mCAFile.size()) + if (gpolicy.mCAFile.size()) { - code = curl_easy_setopt(mCurlHandle, CURLOPT_CAINFO, policy.mCAFile.c_str()); + code = curl_easy_setopt(mCurlHandle, CURLOPT_CAINFO, gpolicy.mCAFile.c_str()); check_curl_easy_code(code, CURLOPT_CAINFO); } @@ -538,9 +558,9 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service) break; default: - LL_ERRS("CoreHttp") << "Invalid HTTP method in request: " - << int(mReqMethod) << ". Can't recover." - << LL_ENDL; + LL_ERRS(LOG_CORE) << "Invalid HTTP method in request: " + << int(mReqMethod) << ". Can't recover." + << LL_ENDL; break; } @@ -592,6 +612,22 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service) { xfer_timeout = timeout; } + if (cpolicy.mPipelining > 1L) + { + // Pipelining affects both connection and transfer timeout values. + // Requests that are added to a pipeling immediately have completed + // their connection so the connection delay tends to be less than + // the non-pipelined value. Transfers are the opposite. Transfer + // timeout starts once the connection is established and completion + // can be delayed due to the pipelined requests ahead. So, it's + // a handwave but bump the transfer timeout up by the pipelining + // depth to give some room. + // + // *TODO: Find a better scheme than timeouts to guarantee liveness. + xfer_timeout *= cpolicy.mPipelining; + } + // *DEBUG: Enable following override for timeout handling and "[curl:bugs] #1420" tests + // xfer_timeout = 1L; code = curl_easy_setopt(mCurlHandle, CURLOPT_TIMEOUT, xfer_timeout); check_curl_easy_code(code, CURLOPT_TIMEOUT); code = curl_easy_setopt(mCurlHandle, CURLOPT_CONNECTTIMEOUT, timeout); @@ -652,8 +688,8 @@ size_t HttpOpRequest::readCallback(void * data, size_t size, size_t nmemb, void { // Warn but continue if the read position moves beyond end-of-body // for some reason. - LL_WARNS("CoreHttp") << "Request body position beyond body size. Truncating request body." - << LL_ENDL; + LL_WARNS(LOG_CORE) << "Request body position beyond body size. Truncating request body." + << LL_ENDL; } return 0; } @@ -790,10 +826,10 @@ size_t HttpOpRequest::headerCallback(void * data, size_t size, size_t nmemb, voi else { // Ignore the unparsable. - LL_INFOS_ONCE("CoreHttp") << "Problem parsing odd Content-Range header: '" - << std::string(hdr_data, wanted_hdr_size) - << "'. Ignoring." - << LL_ENDL; + LL_INFOS_ONCE(LOG_CORE) << "Problem parsing odd Content-Range header: '" + << std::string(hdr_data, wanted_hdr_size) + << "'. Ignoring." + << LL_ENDL; } } @@ -895,11 +931,11 @@ int HttpOpRequest::debugCallback(CURL * handle, curl_infotype info, char * buffe if (logit) { - LL_INFOS("CoreHttp") << "TRACE, LibcurlDebug, Handle: " - << static_cast<HttpHandle>(op) - << ", Type: " << tag - << ", Data: " << safe_line - << LL_ENDL; + LL_INFOS(LOG_CORE) << "TRACE, LibcurlDebug, Handle: " + << static_cast<HttpHandle>(op) + << ", Type: " << tag + << ", Data: " << safe_line + << LL_ENDL; } return 0; @@ -1094,9 +1130,9 @@ void check_curl_easy_code(CURLcode code, int curl_setopt_option) // // linux appears to throw a curl error once per session for a bad initialization // at a pretty random time (when enabling cookies). - LL_WARNS("CoreHttp") << "libcurl error detected: " << curl_easy_strerror(code) - << ", curl_easy_setopt option: " << curl_setopt_option - << LL_ENDL; + LL_WARNS(LOG_CORE) << "libcurl error detected: " << curl_easy_strerror(code) + << ", curl_easy_setopt option: " << curl_setopt_option + << LL_ENDL; } } @@ -1109,8 +1145,8 @@ void check_curl_easy_code(CURLcode code) // // linux appears to throw a curl error once per session for a bad initialization // at a pretty random time (when enabling cookies). - LL_WARNS("CoreHttp") << "libcurl error detected: " << curl_easy_strerror(code) - << LL_ENDL; + LL_WARNS(LOG_CORE) << "libcurl error detected: " << curl_easy_strerror(code) + << LL_ENDL; } } diff --git a/indra/llcorehttp/_httppolicy.cpp b/indra/llcorehttp/_httppolicy.cpp index fd5a93e192..e5d6321401 100755 --- a/indra/llcorehttp/_httppolicy.cpp +++ b/indra/llcorehttp/_httppolicy.cpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012-2013, Linden Research, Inc. + * Copyright (C) 2012-2014, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -35,6 +35,13 @@ #include "lltimer.h" +namespace +{ + +static const char * const LOG_CORE("CoreHttp"); + +} // end anonymous namespace + namespace LLCore { @@ -51,7 +58,8 @@ public: ClassState() : mThrottleEnd(0), mThrottleLeft(0L), - mRequestCount(0L) + mRequestCount(0L), + mStallStaging(false) {} HttpReadyQueue mReadyQueue; @@ -61,6 +69,7 @@ public: HttpTime mThrottleEnd; long mThrottleLeft; long mRequestCount; + bool mStallStaging; }; @@ -128,7 +137,8 @@ void HttpPolicy::shutdown() void HttpPolicy::start() -{} +{ +} void HttpPolicy::addOp(HttpOpRequest * op) @@ -170,19 +180,19 @@ void HttpPolicy::retryOp(HttpOpRequest * op) { ++op->mPolicy503Retries; } - LL_DEBUGS("CoreHttp") << "HTTP request " << static_cast<HttpHandle>(op) - << " retry " << op->mPolicyRetries - << " scheduled in " << (delta / HttpTime(1000)) - << " mS (" << (external_delta ? "external" : "internal") - << "). Status: " << op->mStatus.toTerseString() - << LL_ENDL; + LL_DEBUGS(LOG_CORE) << "HTTP request " << static_cast<HttpHandle>(op) + << " retry " << op->mPolicyRetries + << " scheduled in " << (delta / HttpTime(1000)) + << " mS (" << (external_delta ? "external" : "internal") + << "). Status: " << op->mStatus.toTerseString() + << LL_ENDL; if (op->mTracing > HTTP_TRACE_OFF) { - LL_INFOS("CoreHttp") << "TRACE, ToRetryQueue, Handle: " - << static_cast<HttpHandle>(op) - << ", Delta: " << (delta / HttpTime(1000)) - << ", Retries: " << op->mPolicyRetries - << LL_ENDL; + LL_INFOS(LOG_CORE) << "TRACE, ToRetryQueue, Handle: " + << static_cast<HttpHandle>(op) + << ", Delta: " << (delta / HttpTime(1000)) + << ", Retries: " << op->mPolicyRetries + << LL_ENDL; } mClasses[policy_class]->mRetryQueue.push(op); } @@ -218,6 +228,15 @@ HttpService::ELoopSpeed HttpPolicy::processReadyQueue() HttpRetryQueue & retryq(state.mRetryQueue); HttpReadyQueue & readyq(state.mReadyQueue); + if (state.mStallStaging) + { + // Stalling but don't sleep. Need to complete operations + // and get back to servicing queues. Do this test before + // the retryq/readyq test or you'll get stalls until you + // click a setting or an asset request comes in. + result = HttpService::NORMAL; + continue; + } if (retryq.empty() && readyq.empty()) { continue; @@ -234,7 +253,11 @@ HttpService::ELoopSpeed HttpPolicy::processReadyQueue() } int active(transport.getActiveCountInClass(policy_class)); - int needed(state.mOptions.mConnectionLimit - active); // Expect negatives here + int active_limit(state.mOptions.mPipelining > 1L + ? (state.mOptions.mPerHostConnectionLimit + * state.mOptions.mPipelining) + : state.mOptions.mConnectionLimit); + int needed(active_limit - active); // Expect negatives here if (needed > 0) { @@ -257,9 +280,9 @@ HttpService::ELoopSpeed HttpPolicy::processReadyQueue() if (now >= state.mThrottleEnd) { // Throttle expired, move to next window - LL_DEBUGS("CoreHttp") << "Throttle expired with " << state.mThrottleLeft - << " requests to go and " << state.mRequestCount - << " requests issued." << LL_ENDL; + LL_DEBUGS(LOG_CORE) << "Throttle expired with " << state.mThrottleLeft + << " requests to go and " << state.mRequestCount + << " requests issued." << LL_ENDL; state.mThrottleLeft = state.mOptions.mThrottleRate; state.mThrottleEnd = now + HttpTime(1000000); } @@ -286,9 +309,9 @@ HttpService::ELoopSpeed HttpPolicy::processReadyQueue() if (now >= state.mThrottleEnd) { // Throttle expired, move to next window - LL_DEBUGS("CoreHttp") << "Throttle expired with " << state.mThrottleLeft - << " requests to go and " << state.mRequestCount - << " requests issued." << LL_ENDL; + LL_DEBUGS(LOG_CORE) << "Throttle expired with " << state.mThrottleLeft + << " requests to go and " << state.mRequestCount + << " requests issued." << LL_ENDL; state.mThrottleLeft = state.mOptions.mThrottleRate; state.mThrottleEnd = now + HttpTime(1000000); } @@ -391,6 +414,18 @@ bool HttpPolicy::stageAfterCompletion(HttpOpRequest * op) // Retry or finalize if (! op->mStatus) { + // *DEBUG: For "[curl:bugs] #1420" tests. This will interfere + // with unit tests due to allocation retention by logging code. + // But you won't be checking this in enabled. +#if 0 + if (op->mStatus == HttpStatus(HttpStatus::EXT_CURL_EASY, CURLE_OPERATION_TIMEDOUT)) + { + LL_WARNS(LOG_CORE) << "HTTP request " << static_cast<HttpHandle>(op) + << " timed out." + << LL_ENDL; + } +#endif + // If this failed, we might want to retry. if (op->mPolicyRetries < op->mPolicyRetryLimit && op->mStatus.isRetryable()) { @@ -403,17 +438,17 @@ bool HttpPolicy::stageAfterCompletion(HttpOpRequest * op) // This op is done, finalize it delivering it to the reply queue... if (! op->mStatus) { - LL_WARNS("CoreHttp") << "HTTP request " << static_cast<HttpHandle>(op) - << " failed after " << op->mPolicyRetries - << " retries. Reason: " << op->mStatus.toString() - << " (" << op->mStatus.toTerseString() << ")" - << LL_ENDL; + LL_WARNS(LOG_CORE) << "HTTP request " << static_cast<HttpHandle>(op) + << " failed after " << op->mPolicyRetries + << " retries. Reason: " << op->mStatus.toString() + << " (" << op->mStatus.toTerseString() << ")" + << LL_ENDL; } else if (op->mPolicyRetries) { - LL_DEBUGS("CoreHttp") << "HTTP request " << static_cast<HttpHandle>(op) - << " succeeded on retry " << op->mPolicyRetries << "." - << LL_ENDL; + LL_DEBUGS(LOG_CORE) << "HTTP request " << static_cast<HttpHandle>(op) + << " succeeded on retry " << op->mPolicyRetries << "." + << LL_ENDL; } op->stageFromActive(mService); @@ -441,4 +476,17 @@ int HttpPolicy::getReadyCount(HttpRequest::policy_t policy_class) const } +bool HttpPolicy::stallPolicy(HttpRequest::policy_t policy_class, bool stall) +{ + bool ret(false); + + if (policy_class < mClasses.size()) + { + ret = mClasses[policy_class]->mStallStaging; + mClasses[policy_class]->mStallStaging = stall; + } + return ret; +} + + } // end namespace LLCore diff --git a/indra/llcorehttp/_httppolicy.h b/indra/llcorehttp/_httppolicy.h index bf1aa74267..11cd89bbd1 100755 --- a/indra/llcorehttp/_httppolicy.h +++ b/indra/llcorehttp/_httppolicy.h @@ -158,6 +158,14 @@ public: /// Threading: called by worker thread int getReadyCount(HttpRequest::policy_t policy_class) const; + /// Stall (or unstall) a policy class preventing requests from + /// transitioning to an active state. Used to allow an HTTP + /// request policy to empty prior to changing settings or state + /// that isn't tolerant of changes when work is outstanding. + /// + /// Threading: called by worker thread + bool stallPolicy(HttpRequest::policy_t policy_class, bool stall); + protected: struct ClassState; typedef std::vector<ClassState *> class_list_t; diff --git a/indra/llcorehttp/_httppolicyclass.cpp b/indra/llcorehttp/_httppolicyclass.cpp index f34a8e9f1e..2c0f650155 100755 --- a/indra/llcorehttp/_httppolicyclass.cpp +++ b/indra/llcorehttp/_httppolicyclass.cpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012-2013, Linden Research, Inc. + * Copyright (C) 2012-2014, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -78,8 +78,8 @@ HttpStatus HttpPolicyClass::set(HttpRequest::EPolicyOption opt, long value) mPerHostConnectionLimit = llclamp(value, long(HTTP_CONNECTION_LIMIT_MIN), mConnectionLimit); break; - case HttpRequest::PO_ENABLE_PIPELINING: - mPipelining = llclamp(value, 0L, 1L); + case HttpRequest::PO_PIPELINING_DEPTH: + mPipelining = llclamp(value, 0L, HTTP_PIPELINING_MAX); break; case HttpRequest::PO_THROTTLE_RATE: @@ -106,7 +106,7 @@ HttpStatus HttpPolicyClass::get(HttpRequest::EPolicyOption opt, long * value) co *value = mPerHostConnectionLimit; break; - case HttpRequest::PO_ENABLE_PIPELINING: + case HttpRequest::PO_PIPELINING_DEPTH: *value = mPipelining; break; diff --git a/indra/llcorehttp/_httpservice.cpp b/indra/llcorehttp/_httpservice.cpp index c94249dc2d..c673e1be1d 100755 --- a/indra/llcorehttp/_httpservice.cpp +++ b/indra/llcorehttp/_httpservice.cpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012-2013, Linden Research, Inc. + * Copyright (C) 2012-2014, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -40,6 +40,14 @@ #include "llthread.h" +namespace +{ + +static const char * const LOG_CORE("CoreHttp"); + +} // end anonymous namespace + + namespace LLCore { @@ -87,8 +95,8 @@ HttpService::~HttpService() // Failed to join, expect problems ahead so do a hard termination. mThread->cancel(); - LL_WARNS("CoreHttp") << "Destroying HttpService with running thread. Expect problems." - << LL_ENDL; + LL_WARNS(LOG_CORE) << "Destroying HttpService with running thread. Expect problems." + << LL_ENDL; } } } @@ -328,9 +336,9 @@ HttpService::ELoopSpeed HttpService::processRequestQueue(ELoopSpeed loop) if (op->mTracing > HTTP_TRACE_OFF) { - LL_INFOS("CoreHttp") << "TRACE, FromRequestQueue, Handle: " - << static_cast<HttpHandle>(op) - << LL_ENDL; + LL_INFOS(LOG_CORE) << "TRACE, FromRequestQueue, Handle: " + << static_cast<HttpHandle>(op) + << LL_ENDL; } // Stage @@ -437,9 +445,13 @@ HttpStatus HttpService::setPolicyOption(HttpRequest::EPolicyOption opt, HttpRequ HttpPolicyClass & opts(mPolicy->getClassOptions(pclass)); status = opts.set(opt, value); - if (status && ret_value) + if (status) { - status = opts.get(opt, ret_value); + mTransport->policyUpdated(pclass); + if (ret_value) + { + status = opts.get(opt, ret_value); + } } } @@ -463,7 +475,7 @@ HttpStatus HttpService::setPolicyOption(HttpRequest::EPolicyOption opt, HttpRequ return status; } - // Only string values are global at this time + // String values are always global (at this time). if (pclass == HttpRequest::GLOBAL_POLICY_ID) { HttpPolicyGlobal & opts(mPolicy->getGlobalOptions()); diff --git a/indra/llcorehttp/examples/http_texture_load.cpp b/indra/llcorehttp/examples/http_texture_load.cpp index 73c49687d7..b76c874557 100755 --- a/indra/llcorehttp/examples/http_texture_load.cpp +++ b/indra/llcorehttp/examples/http_texture_load.cpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012-2013, Linden Research, Inc. + * Copyright (C) 2012-2014, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -59,11 +59,13 @@ void usage(std::ostream & out); // Default command line settings static int concurrency_limit(40); static int highwater(100); +static int pipeline_depth(0); +static int tracing(0); static char url_format[1024] = "http://example.com/some/path?texture_id=%s.texture"; #if defined(WIN32) -#define strncpy(_a, _b, _c) strncpy_s(_a, _b, _c) +#define strncpy(_a, _b, _c) strncpy_s(_a, _b, _c) #define strtok_r(_a, _b, _c) strtok_s(_a, _b, _c) int getopt(int argc, char * const argv[], const char *optstring); @@ -100,6 +102,7 @@ public: public: bool mVerbose; bool mRandomRange; + bool mNoRange; int mRequestLowWater; int mRequestHighWater; handle_set_t mHandles; @@ -160,10 +163,11 @@ int main(int argc, char** argv) { LLCore::HttpStatus status; bool do_random(false); + bool do_whole(false); bool do_verbose(false); int option(-1); - while (-1 != (option = getopt(argc, argv, "u:c:h?RvH:"))) + while (-1 != (option = getopt(argc, argv, "u:c:h?RwvH:p:t:"))) { switch (option) { @@ -193,7 +197,7 @@ int main(int argc, char** argv) char * end; value = strtoul(optarg, &end, 10); - if (value < 1 || value > 100 || *end != '\0') + if (value < 1 || value > 200 || *end != '\0') { usage(std::cerr); return 1; @@ -202,8 +206,44 @@ int main(int argc, char** argv) } break; + case 'p': + { + unsigned long value; + char * end; + + value = strtoul(optarg, &end, 10); + if (value < 0 || value > 100 || *end != '\0') + { + usage(std::cerr); + return 1; + } + pipeline_depth = value; + } + break; + + case '5': + { + unsigned long value; + char * end; + + value = strtoul(optarg, &end, 10); + if (value < 0 || value > 3 || *end != '\0') + { + usage(std::cerr); + return 1; + } + tracing = value; + } + break; + case 'R': do_random = true; + do_whole = false; + break; + + case 'w': + do_whole = true; + do_random = false; break; case 'v': @@ -240,6 +280,24 @@ int main(int argc, char** argv) LLCore::HttpRequest::DEFAULT_POLICY_ID, concurrency_limit, NULL); + LLCore::HttpRequest::setStaticPolicyOption(LLCore::HttpRequest::PO_PER_HOST_CONNECTION_LIMIT, + LLCore::HttpRequest::DEFAULT_POLICY_ID, + concurrency_limit, + NULL); + if (pipeline_depth) + { + LLCore::HttpRequest::setStaticPolicyOption(LLCore::HttpRequest::PO_PIPELINING_DEPTH, + LLCore::HttpRequest::DEFAULT_POLICY_ID, + pipeline_depth, + NULL); + } + if (tracing) + { + LLCore::HttpRequest::setStaticPolicyOption(LLCore::HttpRequest::PO_TRACE, + LLCore::HttpRequest::DEFAULT_POLICY_ID, + tracing, + NULL); + } LLCore::HttpRequest::startThread(); // Get service point @@ -257,6 +315,7 @@ int main(int argc, char** argv) ws.mUrl = url_format; ws.loadAssetUuids(uuids); ws.mRandomRange = do_random; + ws.mNoRange = do_whole; ws.mVerbose = do_verbose; ws.mRequestHighWater = highwater; ws.mRequestLowWater = ws.mRequestHighWater / 2; @@ -331,10 +390,15 @@ void usage(std::ostream & out) " -u <url_format> printf-style format string for URL generation\n" " Default: " << url_format << "\n" " -R Issue GETs with random Range: headers\n" + " -w Issue GETs without Range: headers to get whole object\n" " -c <limit> Maximum connection concurrency. Range: [1..100]\n" " Default: " << concurrency_limit << "\n" " -H <limit> HTTP request highwater (requests fed to llcorehttp).\n" - " Range: [1..100] Default: " << highwater << "\n" + " Range: [1..200] Default: " << highwater << "\n" + " -p <depth> If <depth> is positive, enables and sets pipelineing\n" + " depth on HTTP requests. Default: " << pipeline_depth << "\n" + " -t <level> If <level> is positive ([1..3]), enables and sets HTTP\n" + " tracing on HTTP requests. Default: " << tracing << "\n" " -v Verbose mode. Issue some chatter while running\n" " -h print this help\n" "\n" @@ -346,6 +410,7 @@ WorkingSet::WorkingSet() : LLCore::HttpHandler(), mVerbose(false), mRandomRange(false), + mNoRange(false), mRemaining(200), mLimit(200), mAt(0), @@ -395,8 +460,12 @@ bool WorkingSet::reload(LLCore::HttpRequest * hr, LLCore::HttpOptions * opt) #else snprintf(buffer, sizeof(buffer), mUrl.c_str(), mAssets[mAt].mUuid.c_str()); #endif - int offset(mRandomRange ? ((unsigned long) rand()) % 1000000UL : mAssets[mAt].mOffset); - int length(mRandomRange ? ((unsigned long) rand()) % 1000000UL : mAssets[mAt].mLength); + int offset(mNoRange + ? 0 + : (mRandomRange ? ((unsigned long) rand()) % 1000000UL : mAssets[mAt].mOffset)); + int length(mNoRange + ? 0 + : (mRandomRange ? ((unsigned long) rand()) % 1000000UL : mAssets[mAt].mLength)); LLCore::HttpHandle handle; if (offset || length) diff --git a/indra/llcorehttp/httpcommon.cpp b/indra/llcorehttp/httpcommon.cpp index c2f15155ac..7907e958a4 100755 --- a/indra/llcorehttp/httpcommon.cpp +++ b/indra/llcorehttp/httpcommon.cpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012-2013, Linden Research, Inc. + * Copyright (C) 2012-2014, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -219,6 +219,13 @@ std::string HttpStatus::toTerseString() const // Pass true on statuses that might actually be cleared by a // retry. Library failures, calling problems, etc. aren't // going to be fixed by squirting bits all over the Net. +// +// HE_INVALID_HTTP_STATUS is special. As of 7.37.0, there are +// some scenarios where response processing in libcurl appear +// to go wrong and response data is corrupted. A side-effect +// of this is that the HTTP status is read as 0 from the library. +// See libcurl bug report 1420 (https://sourceforge.net/p/curl/bugs/1420/) +// for details. bool HttpStatus::isRetryable() const { static const HttpStatus cant_connect(HttpStatus::EXT_CURL_EASY, CURLE_COULDNT_CONNECT); @@ -231,6 +238,11 @@ bool HttpStatus::isRetryable() const static const HttpStatus post_error(HttpStatus::EXT_CURL_EASY, CURLE_HTTP_POST_ERROR); static const HttpStatus partial_file(HttpStatus::EXT_CURL_EASY, CURLE_PARTIAL_FILE); static const HttpStatus inv_cont_range(HttpStatus::LLCORE, HE_INV_CONTENT_RANGE_HDR); + static const HttpStatus inv_status(HttpStatus::LLCORE, HE_INVALID_HTTP_STATUS); + + // *DEBUG: For "[curl:bugs] #1420" tests. + // Disable the '*this == inv_status' test and look for 'Core_9' + // failures in log files. return ((isHttpStatus() && mType >= 499 && mType <= 599) || // Include special 499 in retryables *this == cant_connect || // Connection reset/endpoint problems @@ -242,6 +254,8 @@ bool HttpStatus::isRetryable() const *this == op_timedout || // Timer expired *this == post_error || // Transport problem *this == partial_file || // Data inconsistency in response + // *DEBUG: Comment out 'inv_status' test for [curl:bugs] #1420 testing. + *this == inv_status || // Inv status can reflect internal state problem in libcurl *this == inv_cont_range); // Short data read disagrees with content-range } diff --git a/indra/llcorehttp/httprequest.h b/indra/llcorehttp/httprequest.h index 651654844a..7f23723b0b 100755 --- a/indra/llcorehttp/httprequest.h +++ b/indra/llcorehttp/httprequest.h @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012-2013, Linden Research, Inc. + * Copyright (C) 2012-2014, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -183,11 +183,38 @@ public: /// Global only PO_TRACE, - /// Suitable requests are allowed to pipeline on their - /// connections when they ask for it. + /// If greater than 1, suitable requests are allowed to + /// pipeline on their connections when they ask for it. + /// Value gives the maximum number of outstanding requests + /// on a connection. + /// + /// There is some interaction between PO_CONNECTION_LIMIT, + /// PO_PER_HOST_CONNECTION_LIMIT, and PO_PIPELINING_DEPTH. + /// When PIPELINING_DEPTH is 0 or 1 (no pipelining), this + /// library manages connection lifecycle and honors the + /// PO_CONNECTION_LIMIT setting as the maximum in-flight + /// request limit. Libcurl itself may be caching additional + /// connections under its connection cache policy. + /// + /// When PIPELINING_DEPTH is 2 or more, libcurl performs + /// connection management and both PO_CONNECTION_LIMIT and + /// PO_PER_HOST_CONNECTION_LIMIT should be set and non-zero. + /// In this case (as of libcurl 7.37.0), libcurl will + /// open new connections in preference to pipelining, up + /// to the above limits at which time pipelining begins. + /// And as usual, an additional cache of open but inactive + /// connections may still be maintained within libcurl. + /// For SL, a good rule-of-thumb is to set + /// PO_PER_HOST_CONNECTION_LIMIT to the user-visible + /// concurrency value and PO_CONNECTION_LIMIT to twice + /// that for baked texture loads and region crossings where + /// additional connection load will be tolerated. If + /// either limit is 0, libcurl will prefer pipelining + /// over connection creation, which is still interesting, + /// but won't be pursued at this time. /// /// Per-class only - PO_ENABLE_PIPELINING, + PO_PIPELINING_DEPTH, /// Controls whether client-side throttling should be /// performed on this policy class. Positive values diff --git a/indra/llmessage/CMakeLists.txt b/indra/llmessage/CMakeLists.txt index 8bd134dc84..40eddcb0ab 100755 --- a/indra/llmessage/CMakeLists.txt +++ b/indra/llmessage/CMakeLists.txt @@ -6,6 +6,7 @@ include(00-Common) include(GoogleMock) include(LLAddBuildTest) include(LLCommon) +include(LLCoreHttp) include(LLMath) include(LLMessage) include(LLVFS) @@ -18,6 +19,7 @@ include_directories (${CMAKE_CURRENT_SOURCE_DIR}) include_directories( ${LLCOMMON_INCLUDE_DIRS} + ${LLCOREHTTP_INCLUDE_DIRS} ${LLMATH_INCLUDE_DIRS} ${LLMESSAGE_INCLUDE_DIRS} ${LLVFS_INCLUDE_DIRS} @@ -36,6 +38,7 @@ set(llmessage_SOURCE_FILES llchainio.cpp llcircuit.cpp llclassifiedflags.cpp + llcorehttputil.cpp llcurl.cpp lldatapacker.cpp lldispatcher.cpp @@ -124,6 +127,7 @@ set(llmessage_HEADER_FILES llcipher.h llcircuit.h llclassifiedflags.h + llcorehttputil.h llcurl.h lldatapacker.h lldbstrings.h diff --git a/indra/llmessage/llcorehttputil.cpp b/indra/llmessage/llcorehttputil.cpp new file mode 100644 index 0000000000..ee80b0fd94 --- /dev/null +++ b/indra/llmessage/llcorehttputil.cpp @@ -0,0 +1,139 @@ +/** + * @file llcorehttputil.cpp + * @date 2014-08-25 + * @brief Implementation of adapter and utility classes expanding the llcorehttp interfaces. + * + * $LicenseInfo:firstyear=2014&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2014, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include "linden_common.h" + +#include <sstream> + +#include "llcorehttputil.h" +#include "llsdserialize.h" + + +using namespace LLCore; + + +namespace LLCoreHttpUtil +{ + +// *TODO: Currently converts only from XML content. A mode +// to convert using fromBinary() might be useful as well. Mesh +// headers could use it. +bool responseToLLSD(HttpResponse * response, bool log, LLSD & out_llsd) +{ + // Convert response to LLSD + BufferArray * body(response->getBody()); + if (! body || ! body->size()) + { + return false; + } + + LLCore::BufferArrayStream bas(body); + LLSD body_llsd; + S32 parse_status(LLSDSerialize::fromXML(body_llsd, bas, log)); + if (LLSDParser::PARSE_FAILURE == parse_status){ + return false; + } + out_llsd = body_llsd; + return true; +} + + +HttpHandle requestPostWithLLSD(HttpRequest * request, + HttpRequest::policy_t policy_id, + HttpRequest::priority_t priority, + const std::string & url, + const LLSD & body, + HttpOptions * options, + HttpHeaders * headers, + HttpHandler * handler) +{ + HttpHandle handle(LLCORE_HTTP_HANDLE_INVALID); + + BufferArray * ba = new BufferArray(); + BufferArrayStream bas(ba); + LLSDSerialize::toXML(body, bas); + + handle = request->requestPost(policy_id, + priority, + url, + ba, + options, + headers, + handler); + ba->release(); + return handle; +} + + +std::string responseToString(LLCore::HttpResponse * response) +{ + static const std::string empty("[Empty]"); + + if (! response) + { + return empty; + } + + BufferArray * body(response->getBody()); + if (! body || ! body->size()) + { + return empty; + } + + // Attempt to parse as LLSD regardless of content-type + LLSD body_llsd; + if (responseToLLSD(response, false, body_llsd)) + { + std::ostringstream tmp; + + LLSDSerialize::toPrettyNotation(body_llsd, tmp); + std::size_t temp_len(tmp.tellp()); + + if (temp_len) + { + return tmp.str().substr(0, std::min(temp_len, std::size_t(1024))); + } + } + else + { + // *TODO: More elaborate forms based on Content-Type as needed. + char content[1024]; + + size_t len(body->read(0, content, sizeof(content))); + if (len) + { + return std::string(content, 0, len); + } + } + + // Default + return empty; +} + + +} // end namespace LLCoreHttpUtil + diff --git a/indra/llmessage/llcorehttputil.h b/indra/llmessage/llcorehttputil.h new file mode 100644 index 0000000000..d40172bc7a --- /dev/null +++ b/indra/llmessage/llcorehttputil.h @@ -0,0 +1,115 @@ +/** + * @file llcorehttputil.h + * @date 2014-08-25 + * @brief Adapter and utility classes expanding the llcorehttp interfaces. + * + * $LicenseInfo:firstyear=2014&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2014, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef LL_LLCOREHTTPUTIL_H +#define LL_LLCOREHTTPUTIL_H + +#include <string> + +#include "httpcommon.h" +#include "httprequest.h" +#include "httpresponse.h" +#include "httpheaders.h" +#include "httpoptions.h" +#include "httphandler.h" +#include "bufferarray.h" +#include "bufferstream.h" +#include "llsd.h" + +/// +/// The base llcorehttp library implements many HTTP idioms +/// used in the viewer but not all. That library intentionally +/// avoids the use of LLSD and its conventions which aren't +/// universally applicable. This module, using namespace +/// LLCoreHttpUtil, provides the additional helper functions +/// that support idiomatic LLSD transport via the newer +/// llcorehttp library. +/// +namespace LLCoreHttpUtil +{ + +/// Attempt to convert a response object's contents to LLSD. +/// It is expected that the response body will be of non-zero +/// length on input but basic checks will be performed and +/// and error (false status) returned if there is no data. +/// If there is data but it cannot be successfully parsed, +/// an error is also returned. If successfully parsed, +/// the output LLSD object, out_llsd, is written with the +/// result and true is returned. +/// +/// @arg response Response object as returned in +/// in an HttpHandler onCompleted() callback. +/// @arg log If true, LLSD parser will emit errors +/// as LL_INFOS-level messages as it parses. +/// Otherwise, it *should* be a quiet parse. +/// @arg out_llsd Output LLSD object written only upon +/// successful parse of the response object. +/// +/// @return Returns true (and writes to out_llsd) if +/// parse was successful. False otherwise. +/// +bool responseToLLSD(LLCore::HttpResponse * response, + bool log, + LLSD & out_llsd); + +/// Create a std::string representation of a response object +/// suitable for logging. Mainly intended for logging of +/// failures and debug information. This won't be fast, +/// just adequate. +std::string responseToString(LLCore::HttpResponse * response); + + +/// Issue a standard HttpRequest::requestPost() call but using +/// and LLSD object as the request body. Conventions are the +/// same as with that method. Caller is expected to provide +/// an HttpHeaders object with a correct 'Content-Type:' header. +/// One will not be provided by this call. You might look after +/// the 'Accept:' header as well. +/// +/// @return If request is successfully issued, the +/// HttpHandle representing the request. +/// On error, LLCORE_HTTP_HANDLE_INVALID +/// is returned and caller can fetch detailed +/// status with the getStatus() method on the +/// request object. In case of error, no +/// request is queued and caller may need to +/// perform additional cleanup such as freeing +/// a now-useless HttpHandler object. +/// +LLCore::HttpHandle requestPostWithLLSD(LLCore::HttpRequest * request, + LLCore::HttpRequest::policy_t policy_id, + LLCore::HttpRequest::priority_t priority, + const std::string & url, + const LLSD & body, + LLCore::HttpOptions * options, + LLCore::HttpHeaders * headers, + LLCore::HttpHandler * handler); + +} // end namespace LLCoreHttpUtil + + +#endif // LL_LLCOREHTTPUTIL_H diff --git a/indra/llmessage/llhttpconstants.cpp b/indra/llmessage/llhttpconstants.cpp index 01f4a080b0..32f76f0d70 100755 --- a/indra/llmessage/llhttpconstants.cpp +++ b/indra/llmessage/llhttpconstants.cpp @@ -3,11 +3,8 @@ * @brief Implementation of the HTTP request / response constant lookups * * $LicenseInfo:firstyear=2013&license=viewerlgpl$ - * - * Copyright (c) 2013, Linden Research, Inc. - * * Second Life Viewer Source Code - * Copyright (C) 2013, Linden Research, Inc. + * Copyright (C) 2013-2014, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -107,6 +104,7 @@ const std::string HTTP_IN_HEADER_X_FORWARDED_FOR("x-forwarded-for"); const std::string HTTP_CONTENT_LLSD_XML("application/llsd+xml"); const std::string HTTP_CONTENT_OCTET_STREAM("application/octet-stream"); +const std::string HTTP_CONTENT_VND_LL_MESH("application/vnd.ll.mesh"); const std::string HTTP_CONTENT_XML("application/xml"); const std::string HTTP_CONTENT_JSON("application/json"); const std::string HTTP_CONTENT_TEXT_HTML("text/html"); diff --git a/indra/llmessage/llhttpconstants.h b/indra/llmessage/llhttpconstants.h index 4aa3cc6394..d6bcbd3c19 100755 --- a/indra/llmessage/llhttpconstants.h +++ b/indra/llmessage/llhttpconstants.h @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2001&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2001-2013, Linden Research, Inc. + * Copyright (C) 2001-2014, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -203,6 +203,7 @@ extern const std::string HTTP_IN_HEADER_X_FORWARDED_FOR; extern const std::string HTTP_CONTENT_LLSD_XML; extern const std::string HTTP_CONTENT_OCTET_STREAM; +extern const std::string HTTP_CONTENT_VND_LL_MESH; extern const std::string HTTP_CONTENT_XML; extern const std::string HTTP_CONTENT_JSON; extern const std::string HTTP_CONTENT_TEXT_HTML; diff --git a/indra/newview/VIEWER_VERSION.txt b/indra/newview/VIEWER_VERSION.txt index 7707ee0e29..82a60c0bb1 100644 --- a/indra/newview/VIEWER_VERSION.txt +++ b/indra/newview/VIEWER_VERSION.txt @@ -1 +1 @@ -3.7.19 +3.7.20 diff --git a/indra/newview/app_settings/settings.xml b/indra/newview/app_settings/settings.xml index 22d64d4d14..b55e72f82a 100755 --- a/indra/newview/app_settings/settings.xml +++ b/indra/newview/app_settings/settings.xml @@ -4467,6 +4467,28 @@ <key>Value</key> <string /> </map> + <key>HttpPipelining</key> + <map> + <key>Comment</key> + <string>If true, viewer will attempt to pipeline HTTP requests.</string> + <key>Persist</key> + <integer>1</integer> + <key>Type</key> + <string>Boolean</string> + <key>Value</key> + <integer>1</integer> + </map> + <key>HttpRangeRequestsDisable</key> + <map> + <key>Comment</key> + <string>If true, viewer will not issue GET requests with 'Range:' headers for meshes and textures. May resolve problems with certain ISPs and networking gear.</string> + <key>Persist</key> + <integer>1</integer> + <key>Type</key> + <string>Boolean</string> + <key>Value</key> + <integer>0</integer> + </map> <key>IMShowTimestamps</key> <map> <key>Comment</key> diff --git a/indra/newview/llappcorehttp.cpp b/indra/newview/llappcorehttp.cpp index 70dcffefb2..f5f224b83e 100755 --- a/indra/newview/llappcorehttp.cpp +++ b/indra/newview/llappcorehttp.cpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012-2013, Linden Research, Inc. + * Copyright (C) 2012-2014, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -40,69 +40,79 @@ // be open at a time. const F64 LLAppCoreHttp::MAX_THREAD_WAIT_TIME(10.0); +const long LLAppCoreHttp::PIPELINING_DEPTH(5L); + +// Default and dynamic values for classes static const struct { - LLAppCoreHttp::EAppPolicy mPolicy; U32 mDefault; U32 mMin; U32 mMax; U32 mRate; + bool mPipelined; std::string mKey; const char * mUsage; -} init_data[] = // Default and dynamic values for classes +} init_data[LLAppCoreHttp::AP_COUNT] = { - { - LLAppCoreHttp::AP_DEFAULT, 8, 8, 8, 0, + { // AP_DEFAULT + 8, 8, 8, 0, false, "", "other" }, - { - LLAppCoreHttp::AP_TEXTURE, 8, 1, 12, 0, + { // AP_TEXTURE + 8, 1, 12, 0, true, "TextureFetchConcurrency", "texture fetch" }, - { - LLAppCoreHttp::AP_MESH1, 32, 1, 128, 100, + { // AP_MESH1 + 32, 1, 128, 0, false, "MeshMaxConcurrentRequests", "mesh fetch" }, - { - LLAppCoreHttp::AP_MESH2, 8, 1, 32, 100, + { // AP_MESH2 + 8, 1, 32, 0, true, "Mesh2MaxConcurrentRequests", "mesh2 fetch" }, - { - LLAppCoreHttp::AP_LARGE_MESH, 2, 1, 8, 0, + { // AP_LARGE_MESH + 2, 1, 8, 0, false, "", "large mesh fetch" }, - { - LLAppCoreHttp::AP_UPLOADS, 2, 1, 8, 0, + { // AP_UPLOADS + 2, 1, 8, 0, false, "", "asset upload" }, - { - LLAppCoreHttp::AP_LONG_POLL, 32, 32, 32, 0, + { // AP_LONG_POLL + 32, 32, 32, 0, false, "", "long poll" + }, + { // AP_INVENTORY + 4, 1, 4, 0, false, + "", + "inventory" } }; static void setting_changed(); +LLAppCoreHttp::HttpClass::HttpClass() + : mPolicy(LLCore::HttpRequest::DEFAULT_POLICY_ID), + mConnLimit(0U), + mPipelined(false) +{} + + LLAppCoreHttp::LLAppCoreHttp() : mRequest(NULL), mStopHandle(LLCORE_HTTP_HANDLE_INVALID), mStopRequested(0.0), - mStopped(false) -{ - for (int i(0); i < LL_ARRAY_SIZE(mPolicies); ++i) - { - mPolicies[i] = LLCore::HttpRequest::DEFAULT_POLICY_ID; - mSettings[i] = 0U; - } -} + mStopped(false), + mPipelined(true) +{} LLAppCoreHttp::~LLAppCoreHttp() @@ -157,27 +167,28 @@ void LLAppCoreHttp::init() } // Setup default policy and constrain if directed to - mPolicies[AP_DEFAULT] = LLCore::HttpRequest::DEFAULT_POLICY_ID; + mHttpClasses[AP_DEFAULT].mPolicy = LLCore::HttpRequest::DEFAULT_POLICY_ID; // Setup additional policies based on table and some special rules + llassert(LL_ARRAY_SIZE(init_data) == AP_COUNT); for (int i(0); i < LL_ARRAY_SIZE(init_data); ++i) { - const EAppPolicy policy(init_data[i].mPolicy); + const EAppPolicy app_policy(static_cast<EAppPolicy>(i)); - if (AP_DEFAULT == policy) + if (AP_DEFAULT == app_policy) { // Pre-created continue; } - mPolicies[policy] = LLCore::HttpRequest::createPolicyClass(); - if (! mPolicies[policy]) + mHttpClasses[app_policy].mPolicy = LLCore::HttpRequest::createPolicyClass(); + if (! mHttpClasses[app_policy].mPolicy) { // Use default policy (but don't accidentally modify default) LL_WARNS("Init") << "Failed to create HTTP policy class for " << init_data[i].mUsage << ". Using default policy." << LL_ENDL; - mPolicies[policy] = mPolicies[AP_DEFAULT]; + mHttpClasses[app_policy].mPolicy = mHttpClasses[AP_DEFAULT].mPolicy; continue; } } @@ -196,9 +207,27 @@ void LLAppCoreHttp::init() << LL_ENDL; } + // Signal for global pipelining preference from settings + static const std::string http_pipelining("HttpPipelining"); + if (gSavedSettings.controlExists(http_pipelining)) + { + LLPointer<LLControlVariable> cntrl_ptr = gSavedSettings.getControl(http_pipelining); + if (cntrl_ptr.isNull()) + { + LL_WARNS("Init") << "Unable to set signal on global setting '" << http_pipelining + << "'" << LL_ENDL; + } + else + { + mPipelinedSignal = cntrl_ptr->getCommitSignal()->connect(boost::bind(&setting_changed)); + } + } + // Register signals for settings and state changes for (int i(0); i < LL_ARRAY_SIZE(init_data); ++i) { + const EAppPolicy app_policy(static_cast<EAppPolicy>(i)); + if (! init_data[i].mKey.empty() && gSavedSettings.controlExists(init_data[i].mKey)) { LLPointer<LLControlVariable> cntrl_ptr = gSavedSettings.getControl(init_data[i].mKey); @@ -209,7 +238,7 @@ void LLAppCoreHttp::init() } else { - mSettingsSignal[i] = cntrl_ptr->getCommitSignal()->connect(boost::bind(&setting_changed)); + mHttpClasses[app_policy].mSettingsSignal = cntrl_ptr->getCommitSignal()->connect(boost::bind(&setting_changed)); } } } @@ -261,10 +290,11 @@ void LLAppCoreHttp::cleanup() } } - for (int i(0); i < LL_ARRAY_SIZE(init_data); ++i) + for (int i(0); i < LL_ARRAY_SIZE(mHttpClasses); ++i) { - mSettingsSignal[i].disconnect(); + mHttpClasses[i].mSettingsSignal.disconnect(); } + mPipelinedSignal.disconnect(); delete mRequest; mRequest = NULL; @@ -278,30 +308,84 @@ void LLAppCoreHttp::cleanup() } } + void LLAppCoreHttp::refreshSettings(bool initial) { LLCore::HttpStatus status; + + // Global pipelining setting + bool pipeline_changed(false); + static const std::string http_pipelining("HttpPipelining"); + if (gSavedSettings.controlExists(http_pipelining)) + { + // Default to true (in ctor) if absent. + bool pipelined(gSavedSettings.getBOOL(http_pipelining)); + if (pipelined != mPipelined) + { + mPipelined = pipelined; + pipeline_changed = true; + } + } for (int i(0); i < LL_ARRAY_SIZE(init_data); ++i) { - const EAppPolicy policy(init_data[i].mPolicy); + const EAppPolicy app_policy(static_cast<EAppPolicy>(i)); - // Set any desired throttle - if (initial && init_data[i].mRate) + if (initial) { - // Init-time only, can use the static setters here - status = LLCore::HttpRequest::setStaticPolicyOption(LLCore::HttpRequest::PO_THROTTLE_RATE, - mPolicies[policy], - init_data[i].mRate, - NULL); - if (! status) + // Init-time only settings, can use the static setters here + + if (init_data[i].mRate) { - LL_WARNS("Init") << "Unable to set " << init_data[i].mUsage - << " throttle rate. Reason: " << status.toString() - << LL_ENDL; + // Set any desired throttle + status = LLCore::HttpRequest::setStaticPolicyOption(LLCore::HttpRequest::PO_THROTTLE_RATE, + mHttpClasses[app_policy].mPolicy, + init_data[i].mRate, + NULL); + if (! status) + { + LL_WARNS("Init") << "Unable to set " << init_data[i].mUsage + << " throttle rate. Reason: " << status.toString() + << LL_ENDL; + } } + } + // Init- or run-time settings. Must use the queued request API. + + // Pipelining changes + if (initial || pipeline_changed) + { + const bool to_pipeline(mPipelined && init_data[i].mPipelined); + if (to_pipeline != mHttpClasses[app_policy].mPipelined) + { + // Pipeline election changing, set dynamic option via request + + LLCore::HttpHandle handle; + const long new_depth(to_pipeline ? PIPELINING_DEPTH : 0); + + handle = mRequest->setPolicyOption(LLCore::HttpRequest::PO_PIPELINING_DEPTH, + mHttpClasses[app_policy].mPolicy, + new_depth, + NULL); + if (LLCORE_HTTP_HANDLE_INVALID == handle) + { + status = mRequest->getStatus(); + LL_WARNS("Init") << "Unable to set " << init_data[i].mUsage + << " pipelining. Reason: " << status.toString() + << LL_ENDL; + } + else + { + LL_DEBUGS("Init") << "Changed " << init_data[i].mUsage + << " pipelining. New value: " << new_depth + << LL_ENDL; + mHttpClasses[app_policy].mPipelined = to_pipeline; + } + } + } + // Get target connection concurrency value U32 setting(init_data[i].mDefault); if (! init_data[i].mKey.empty() && gSavedSettings.controlExists(init_data[i].mKey)) @@ -314,38 +398,61 @@ void LLAppCoreHttp::refreshSettings(bool initial) } } - if (! initial && setting == mSettings[policy]) + if (initial || setting != mHttpClasses[app_policy].mConnLimit || pipeline_changed) { - // Unchanged, try next setting - continue; - } - - // Set it and report - // *TODO: These are intended to be per-host limits when we can - // support that in llcorehttp/libcurl. - LLCore::HttpHandle handle; - handle = mRequest->setPolicyOption(LLCore::HttpRequest::PO_CONNECTION_LIMIT, - mPolicies[policy], - setting, NULL); - if (LLCORE_HTTP_HANDLE_INVALID == handle) - { - status = mRequest->getStatus(); - LL_WARNS("Init") << "Unable to set " << init_data[i].mUsage - << " concurrency. Reason: " << status.toString() - << LL_ENDL; - } - else - { - LL_DEBUGS("Init") << "Changed " << init_data[i].mUsage - << " concurrency. New value: " << setting - << LL_ENDL; - mSettings[policy] = setting; - if (initial && setting != init_data[i].mDefault) + // Set it and report. Strategies depend on pipelining: + // + // No Pipelining. Llcorehttp manages connections itself based + // on the PO_CONNECTION_LIMIT setting. Set both limits to the + // same value for logical consistency. In the future, may + // hand over connection management to libcurl after the + // connection cache has been better vetted. + // + // Pipelining. Libcurl is allowed to manage connections to a + // great degree. Steady state will connection limit based on + // the per-host setting. Transitions (region crossings, new + // avatars, etc.) can request additional outbound connections + // to other servers via 2X total connection limit. + // + LLCore::HttpHandle handle; + handle = mRequest->setPolicyOption(LLCore::HttpRequest::PO_CONNECTION_LIMIT, + mHttpClasses[app_policy].mPolicy, + (mHttpClasses[app_policy].mPipelined ? 2 * setting : setting), + NULL); + if (LLCORE_HTTP_HANDLE_INVALID == handle) { - LL_INFOS("Init") << "Application settings overriding default " << init_data[i].mUsage - << " concurrency. New value: " << setting + status = mRequest->getStatus(); + LL_WARNS("Init") << "Unable to set " << init_data[i].mUsage + << " concurrency. Reason: " << status.toString() << LL_ENDL; } + else + { + handle = mRequest->setPolicyOption(LLCore::HttpRequest::PO_PER_HOST_CONNECTION_LIMIT, + mHttpClasses[app_policy].mPolicy, + setting, + NULL); + if (LLCORE_HTTP_HANDLE_INVALID == handle) + { + status = mRequest->getStatus(); + LL_WARNS("Init") << "Unable to set " << init_data[i].mUsage + << " per-host concurrency. Reason: " << status.toString() + << LL_ENDL; + } + else + { + LL_DEBUGS("Init") << "Changed " << init_data[i].mUsage + << " concurrency. New value: " << setting + << LL_ENDL; + mHttpClasses[app_policy].mConnLimit = setting; + if (initial && setting != init_data[i].mDefault) + { + LL_INFOS("Init") << "Application settings overriding default " << init_data[i].mUsage + << " concurrency. New value: " << setting + << LL_ENDL; + } + } + } } } } diff --git a/indra/newview/llappcorehttp.h b/indra/newview/llappcorehttp.h index 40e3042b84..37d7a737e7 100755 --- a/indra/newview/llappcorehttp.h +++ b/indra/newview/llappcorehttp.h @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012-2013, Linden Research, Inc. + * Copyright (C) 2012-2014, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -41,6 +41,8 @@ class LLAppCoreHttp : public LLCore::HttpHandler { public: + static const long PIPELINING_DEPTH; + typedef LLCore::HttpRequest::policy_t policy_t; enum EAppPolicy @@ -62,15 +64,15 @@ public: /// Texture fetching policy class. Used to /// download textures via capability or SSA /// baking service. Deep queueing of requests. - /// Do not share. + /// Do not share. GET requests only. /// - /// Destination: simhost:12046 & bake-texture:80 + /// Destination: simhost:12046 & {bake-texture,cdn}:80 /// Protocol: http: /// Transfer size: KB-MB /// Long poll: no /// Concurrency: high /// Request rate: high - /// Pipelined: soon + /// Pipelined: yes AP_TEXTURE, /// Legacy mesh fetching policy class. Used to @@ -90,15 +92,16 @@ public: /// download textures via 'GetMesh2' capability. /// Used when fetch request (typically one LOD) /// is 'small', currently defined as 2MB. - /// Very deeply queued. Do not share. + /// Very deeply queued. Do not share. GET + /// requests only. /// - /// Destination: simhost:12046 + /// Destination: simhost:12046 & cdn:80 /// Protocol: http: /// Transfer size: KB-MB /// Long poll: no /// Concurrency: high /// Request rate: high - /// Pipelined: soon + /// Pipelined: yes AP_MESH2, /// Large mesh fetching policy class. Used to @@ -110,13 +113,13 @@ public: /// traffic that can wait for longish stalls /// (default timeout 600S). /// - /// Destination: simhost:12046 + /// Destination: simhost:12046 & cdn:80 /// Protocol: http: /// Transfer size: MB /// Long poll: no /// Concurrency: low /// Request rate: low - /// Pipelined: soon + /// Pipelined: no AP_LARGE_MESH, /// Asset upload policy class. Used to store @@ -148,6 +151,20 @@ public: /// Pipelined: no AP_LONG_POLL, + /// Inventory operations (really Capabilities- + /// related operations). Mix of high-priority + /// and low-priority operations. + /// + /// Destination: simhost:12043 + /// Protocol: https: + /// Transfer size: KB-MB + /// Long poll: no + /// Concurrency: high + /// Request rate: high + /// Pipelined: no + AP_INVENTORY, + AP_REPORTING = AP_INVENTORY, // Piggy-back on inventory + AP_COUNT // Must be last }; @@ -180,7 +197,13 @@ public: // application function. policy_t getPolicy(EAppPolicy policy) const { - return mPolicies[policy]; + return mHttpClasses[policy].mPolicy; + } + + // Return whether a policy is using pipelined operations. + bool isPipelined(EAppPolicy policy) const + { + return mHttpClasses[policy].mPipelined; } // Apply initial or new settings from the environment. @@ -190,13 +213,27 @@ private: static const F64 MAX_THREAD_WAIT_TIME; private: - LLCore::HttpRequest * mRequest; // Request queue to issue shutdowns + + // PODish container for per-class settings and state. + struct HttpClass + { + public: + HttpClass(); + + public: + policy_t mPolicy; // Policy class id for the class + U32 mConnLimit; + bool mPipelined; + boost::signals2::connection mSettingsSignal; // Signal to global setting that affect this class (if any) + }; + + LLCore::HttpRequest * mRequest; // Request queue to issue shutdowns LLCore::HttpHandle mStopHandle; F64 mStopRequested; bool mStopped; - policy_t mPolicies[AP_COUNT]; // Policy class id for each connection set - U32 mSettings[AP_COUNT]; - boost::signals2::connection mSettingsSignal[AP_COUNT]; // Signals to global settings that affect us + HttpClass mHttpClasses[AP_COUNT]; + bool mPipelined; // Global setting + boost::signals2::connection mPipelinedSignal; // Signal for 'HttpPipelining' setting }; diff --git a/indra/newview/llinventorymodel.cpp b/indra/newview/llinventorymodel.cpp index 14ca0095ae..f92332dea5 100755 --- a/indra/newview/llinventorymodel.cpp +++ b/indra/newview/llinventorymodel.cpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2002&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2010, Linden Research, Inc. + * Copyright (C) 2014, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -25,6 +25,9 @@ */ #include "llviewerprecompiledheaders.h" + +#include <typeinfo> + #include "llinventorymodel.h" #include "llaisapi.h" @@ -50,7 +53,9 @@ #include "llvoavatarself.h" #include "llgesturemgr.h" #include "llsdutil.h" -#include <typeinfo> +#include "bufferarray.h" +#include "bufferstream.h" +#include "llcorehttputil.h" //#define DIFF_INVENTORY_FILES #ifdef DIFF_INVENTORY_FILES @@ -67,7 +72,8 @@ BOOL LLInventoryModel::sFirstTimeInViewer2 = TRUE; ///---------------------------------------------------------------------------- //BOOL decompress_file(const char* src_filename, const char* dst_filename); -const char CACHE_FORMAT_STRING[] = "%s.inv"; +static const char CACHE_FORMAT_STRING[] = "%s.inv"; +static const char * const LOG_INV("Inventory"); struct InventoryIDPtrLess { @@ -125,24 +131,32 @@ LLInventoryModel gInventory; // Default constructor LLInventoryModel::LLInventoryModel() -: mModifyMask(LLInventoryObserver::ALL), - mChangedItemIDs(), +: // These are now ordered, keep them that way. mBacklinkMMap(), + mIsAgentInvUsable(false), + mRootFolderID(), + mLibraryRootFolderID(), + mLibraryOwnerID(), mCategoryMap(), mItemMap(), - mCategoryLock(), - mItemLock(), - mLastItem(NULL), mParentChildCategoryTree(), mParentChildItemTree(), - mObservers(), - mRootFolderID(), - mLibraryRootFolderID(), - mLibraryOwnerID(), + mLastItem(NULL), mIsNotifyObservers(FALSE), - mIsAgentInvUsable(false) -{ -} + mModifyMask(LLInventoryObserver::ALL), + mChangedItemIDs(), + mObservers(), + mHttpRequestFG(NULL), + mHttpRequestBG(NULL), + mHttpOptions(NULL), + mHttpHeaders(NULL), + mHttpPolicyClass(LLCore::HttpRequest::DEFAULT_POLICY_ID), + mHttpPriorityFG(0), + mHttpPriorityBG(0), + mCategoryLock(), + mItemLock() +{} + // Destroys the object LLInventoryModel::~LLInventoryModel() @@ -162,6 +176,22 @@ void LLInventoryModel::cleanupInventory() delete observer; } mObservers.clear(); + + // Run down HTTP transport + if (mHttpHeaders) + { + mHttpHeaders->release(); + mHttpHeaders = NULL; + } + if (mHttpOptions) + { + mHttpOptions->release(); + mHttpOptions = NULL; + } + delete mHttpRequestFG; + mHttpRequestFG = NULL; + delete mHttpRequestBG; + mHttpRequestBG = NULL; } // This is a convenience function to check if one object has a parent @@ -253,7 +283,7 @@ bool LLInventoryModel::getObjectTopmostAncestor(const LLUUID& object_id, LLUUID& LLInventoryObject *parent_object = getObject(object->getParentUUID()); if (!parent_object) { - LL_WARNS() << "unable to trace topmost ancestor, missing item for uuid " << object->getParentUUID() << LL_ENDL; + LL_WARNS(LOG_INV) << "unable to trace topmost ancestor, missing item for uuid " << object->getParentUUID() << LL_ENDL; return false; } object = parent_object; @@ -508,7 +538,7 @@ public: protected: virtual void httpFailure() { - LL_WARNS("InvAPI") << dumpResponse() << LL_ENDL; + LL_WARNS(LOG_INV) << dumpResponse() << LL_ENDL; } virtual void httpSuccess() @@ -522,7 +552,7 @@ protected: } LLUUID category_id = content["folder_id"].asUUID(); - LL_DEBUGS("Avatar") << ll_pretty_print_sd(content) << LL_ENDL; + LL_DEBUGS(LOG_INV) << ll_pretty_print_sd(content) << LL_ENDL; // Add the category to the internal representation LLPointer<LLViewerInventoryCategory> cat = new LLViewerInventoryCategory( category_id, @@ -560,13 +590,13 @@ LLUUID LLInventoryModel::createNewCategory(const LLUUID& parent_id, LLUUID id; if(!isInventoryUsable()) { - LL_WARNS() << "Inventory is broken." << LL_ENDL; + LL_WARNS(LOG_INV) << "Inventory is broken." << LL_ENDL; return id; } if(LLFolderType::lookup(preferred_type) == LLFolderType::badLookup()) { - LL_DEBUGS() << "Attempt to create undefined category." << LL_ENDL; + LL_DEBUGS(LOG_INV) << "Attempt to create undefined category." << LL_ENDL; return id; } @@ -599,7 +629,7 @@ LLUUID LLInventoryModel::createNewCategory(const LLUUID& parent_id, request["message"] = "CreateInventoryCategory"; request["payload"] = body; - LL_DEBUGS("Avatar") << "create category request: " << ll_pretty_print_sd(request) << LL_ENDL; + LL_DEBUGS(LOG_INV) << "create category request: " << ll_pretty_print_sd(request) << LL_ENDL; // viewer_region->getCapAPI().post(request); LLHTTPClient::post( url, @@ -820,7 +850,7 @@ U32 LLInventoryModel::updateItem(const LLViewerInventoryItem* item, U32 mask) if(!isInventoryUsable()) { - LL_WARNS() << "Inventory is broken." << LL_ENDL; + LL_WARNS(LOG_INV) << "Inventory is broken." << LL_ENDL; return mask; } @@ -883,7 +913,7 @@ U32 LLInventoryModel::updateItem(const LLViewerInventoryItem* item, U32 mask) } else { - LL_WARNS() << "Couldn't find parent-child item tree for " << new_item->getName() << LL_ENDL; + LL_WARNS(LOG_INV) << "Couldn't find parent-child item tree for " << new_item->getName() << LL_ENDL; } } else @@ -912,8 +942,8 @@ U32 LLInventoryModel::updateItem(const LLViewerInventoryItem* item, U32 mask) else { // Whoops! No such parent, make one. - LL_INFOS() << "Lost item: " << new_item->getUUID() << " - " - << new_item->getName() << LL_ENDL; + LL_INFOS(LOG_INV) << "Lost item: " << new_item->getUUID() << " - " + << new_item->getName() << LL_ENDL; parent_id = findCategoryUUIDForType(LLFolderType::FT_LOST_AND_FOUND); new_item->setParent(parent_id); item_array = get_ptr_in_map(mParentChildItemTree, parent_id); @@ -926,7 +956,7 @@ U32 LLInventoryModel::updateItem(const LLViewerInventoryItem* item, U32 mask) } else { - LL_WARNS() << "Lost and found Not there!!" << LL_ENDL; + LL_WARNS(LOG_INV) << "Lost and found Not there!!" << LL_ENDL; } } } @@ -1000,7 +1030,7 @@ void LLInventoryModel::updateCategory(const LLViewerInventoryCategory* cat, U32 if(!isInventoryUsable()) { - LL_WARNS() << "Inventory is broken." << LL_ENDL; + LL_WARNS(LOG_INV) << "Inventory is broken." << LL_ENDL; return; } @@ -1062,17 +1092,17 @@ void LLInventoryModel::updateCategory(const LLViewerInventoryCategory* cat, U32 void LLInventoryModel::moveObject(const LLUUID& object_id, const LLUUID& cat_id) { - LL_DEBUGS() << "LLInventoryModel::moveObject()" << LL_ENDL; + LL_DEBUGS(LOG_INV) << "LLInventoryModel::moveObject()" << LL_ENDL; if(!isInventoryUsable()) { - LL_WARNS() << "Inventory is broken." << LL_ENDL; + LL_WARNS(LOG_INV) << "Inventory is broken." << LL_ENDL; return; } if((object_id == cat_id) || !is_in_map(mCategoryMap, cat_id)) { - LL_WARNS() << "Could not move inventory object " << object_id << " to " - << cat_id << LL_ENDL; + LL_WARNS(LOG_INV) << "Could not move inventory object " << object_id << " to " + << cat_id << LL_ENDL; return; } LLPointer<LLViewerInventoryCategory> cat = getCategory(object_id); @@ -1108,14 +1138,14 @@ void LLInventoryModel::changeItemParent(LLViewerInventoryItem* item, { if (item->getParentUUID() == new_parent_id) { - LL_DEBUGS("Inventory") << "'" << item->getName() << "' (" << item->getUUID() - << ") is already in folder " << new_parent_id << LL_ENDL; + LL_DEBUGS(LOG_INV) << "'" << item->getName() << "' (" << item->getUUID() + << ") is already in folder " << new_parent_id << LL_ENDL; } else { - LL_INFOS("Inventory") << "Moving '" << item->getName() << "' (" << item->getUUID() - << ") from " << item->getParentUUID() << " to folder " - << new_parent_id << LL_ENDL; + LL_INFOS(LOG_INV) << "Moving '" << item->getName() << "' (" << item->getUUID() + << ") from " << item->getParentUUID() << " to folder " + << new_parent_id << LL_ENDL; LLInventoryModel::update_list_t update; LLInventoryModel::LLCategoryUpdate old_folder(item->getParentUUID(),-1); update.push_back(old_folder); @@ -1171,7 +1201,7 @@ void LLInventoryModel::onAISUpdateReceived(const std::string& context, const LLS AISUpdate ais_update(update); // parse update llsd into stuff to do. ais_update.doUpdate(); // execute the updates in the appropriate order. - LL_INFOS() << "elapsed: " << timer.getElapsedTimeF32() << LL_ENDL; + LL_INFOS(LOG_INV) << "elapsed: " << timer.getElapsedTimeF32() << LL_ENDL; } // Does not appear to be used currently. @@ -1180,7 +1210,7 @@ void LLInventoryModel::onItemUpdated(const LLUUID& item_id, const LLSD& updates, U32 mask = LLInventoryObserver::NONE; LLPointer<LLViewerInventoryItem> item = gInventory.getItem(item_id); - LL_DEBUGS("Inventory") << "item_id: [" << item_id << "] name " << (item ? item->getName() : "(NOT FOUND)") << LL_ENDL; + LL_DEBUGS(LOG_INV) << "item_id: [" << item_id << "] name " << (item ? item->getName() : "(NOT FOUND)") << LL_ENDL; if(item) { for (LLSD::map_const_iterator it = updates.beginMap(); @@ -1188,19 +1218,19 @@ void LLInventoryModel::onItemUpdated(const LLUUID& item_id, const LLSD& updates, { if (it->first == "name") { - LL_INFOS() << "Updating name from " << item->getName() << " to " << it->second.asString() << LL_ENDL; + LL_INFOS(LOG_INV) << "Updating name from " << item->getName() << " to " << it->second.asString() << LL_ENDL; item->rename(it->second.asString()); mask |= LLInventoryObserver::LABEL; } else if (it->first == "desc") { - LL_INFOS() << "Updating description from " << item->getActualDescription() - << " to " << it->second.asString() << LL_ENDL; + LL_INFOS(LOG_INV) << "Updating description from " << item->getActualDescription() + << " to " << it->second.asString() << LL_ENDL; item->setDescription(it->second.asString()); } else { - LL_ERRS() << "unhandled updates for field: " << it->first << LL_ENDL; + LL_ERRS(LOG_INV) << "unhandled updates for field: " << it->first << LL_ENDL; } } mask |= LLInventoryObserver::INTERNAL; @@ -1211,7 +1241,7 @@ void LLInventoryModel::onItemUpdated(const LLUUID& item_id, const LLSD& updates, LLInventoryModel::LLCategoryUpdate up(item->getParentUUID(), 0); accountForUpdate(up); } - gInventory.notifyObservers(); // do we want to be able to make this optional? + notifyObservers(); // do we want to be able to make this optional? } } @@ -1221,7 +1251,7 @@ void LLInventoryModel::onCategoryUpdated(const LLUUID& cat_id, const LLSD& updat U32 mask = LLInventoryObserver::NONE; LLPointer<LLViewerInventoryCategory> cat = gInventory.getCategory(cat_id); - LL_DEBUGS("Inventory") << "cat_id: [" << cat_id << "] name " << (cat ? cat->getName() : "(NOT FOUND)") << LL_ENDL; + LL_DEBUGS(LOG_INV) << "cat_id: [" << cat_id << "] name " << (cat ? cat->getName() : "(NOT FOUND)") << LL_ENDL; if(cat) { for (LLSD::map_const_iterator it = updates.beginMap(); @@ -1229,18 +1259,18 @@ void LLInventoryModel::onCategoryUpdated(const LLUUID& cat_id, const LLSD& updat { if (it->first == "name") { - LL_INFOS() << "Updating name from " << cat->getName() << " to " << it->second.asString() << LL_ENDL; + LL_INFOS(LOG_INV) << "Updating name from " << cat->getName() << " to " << it->second.asString() << LL_ENDL; cat->rename(it->second.asString()); mask |= LLInventoryObserver::LABEL; } else { - LL_ERRS() << "unhandled updates for field: " << it->first << LL_ENDL; + LL_ERRS(LOG_INV) << "unhandled updates for field: " << it->first << LL_ENDL; } } mask |= LLInventoryObserver::INTERNAL; addChangedMask(mask, cat->getUUID()); - gInventory.notifyObservers(); // do we want to be able to make this optional? + notifyObservers(); // do we want to be able to make this optional? } } @@ -1317,8 +1347,8 @@ void LLInventoryModel::onDescendentsPurgedFromServer(const LLUUID& object_id, bo while (deleted_count > 0); if (total_deleted_count != count) { - LL_WARNS() << "Unexpected count of categories deleted, got " - << total_deleted_count << " expected " << count << LL_ENDL; + LL_WARNS(LOG_INV) << "Unexpected count of categories deleted, got " + << total_deleted_count << " expected " << count << LL_ENDL; } //gInventory.validate(); } @@ -1355,15 +1385,15 @@ void LLInventoryModel::onObjectDeletedFromServer(const LLUUID& object_id, bool f // Delete a particular inventory object by ID. void LLInventoryModel::deleteObject(const LLUUID& id, bool fix_broken_links, bool do_notify_observers) { - LL_DEBUGS() << "LLInventoryModel::deleteObject()" << LL_ENDL; + LL_DEBUGS(LOG_INV) << "LLInventoryModel::deleteObject()" << LL_ENDL; LLPointer<LLInventoryObject> obj = getObject(id); if (!obj) { - LL_WARNS() << "Deleting non-existent object [ id: " << id << " ] " << LL_ENDL; + LL_WARNS(LOG_INV) << "Deleting non-existent object [ id: " << id << " ] " << LL_ENDL; return; } - LL_DEBUGS() << "Deleting inventory object " << id << LL_ENDL; + LL_DEBUGS(LOG_INV) << "Deleting inventory object " << id << LL_ENDL; mLastItem = NULL; LLUUID parent_id = obj->getParentUUID(); mCategoryMap.erase(id); @@ -1386,7 +1416,7 @@ void LLInventoryModel::deleteObject(const LLUUID& id, bool fix_broken_links, boo { if (item_list->size()) { - LL_WARNS() << "Deleting cat " << id << " while it still has child items" << LL_ENDL; + LL_WARNS(LOG_INV) << "Deleting cat " << id << " while it still has child items" << LL_ENDL; } delete item_list; mParentChildItemTree.erase(id); @@ -1396,7 +1426,7 @@ void LLInventoryModel::deleteObject(const LLUUID& id, bool fix_broken_links, boo { if (cat_list->size()) { - LL_WARNS() << "Deleting cat " << id << " while it still has child cats" << LL_ENDL; + LL_WARNS(LOG_INV) << "Deleting cat " << id << " while it still has child cats" << LL_ENDL; } delete cat_list; mParentChildCategoryTree.erase(id); @@ -1431,7 +1461,7 @@ void LLInventoryModel::updateLinkedObjectsFromPurge(const LLUUID &baseobj_id) // everything else on the changelist will also get rebuilt. if (item_array.size() > 0) { - gInventory.notifyObservers(); + notifyObservers(); for (LLInventoryModel::item_array_t::const_iterator iter = item_array.begin(); iter != item_array.end(); iter++) @@ -1441,7 +1471,7 @@ void LLInventoryModel::updateLinkedObjectsFromPurge(const LLUUID &baseobj_id) if (item_id == baseobj_id) continue; addChangedMask(LLInventoryObserver::REBUILD, item_id); } - gInventory.notifyObservers(); + notifyObservers(); } } @@ -1464,6 +1494,9 @@ BOOL LLInventoryModel::containsObserver(LLInventoryObserver* observer) const void LLInventoryModel::idleNotifyObservers() { + // *FIX: Think I want this conditional or moved elsewhere... + handleResponses(true); + if (mModifyMask == LLInventoryObserver::NONE && (mChangedItemIDs.size() == 0)) { return; @@ -1479,7 +1512,7 @@ void LLInventoryModel::notifyObservers() // Within notifyObservers, something called notifyObservers // again. This type of recursion is unsafe because it causes items to be // processed twice, and this can easily lead to infinite loops. - LL_WARNS() << "Call was made to notifyObservers within notifyObservers!" << LL_ENDL; + LL_WARNS(LOG_INV) << "Call was made to notifyObservers within notifyObservers!" << LL_ENDL; return; } @@ -1509,18 +1542,18 @@ void LLInventoryModel::addChangedMask(U32 mask, const LLUUID& referent) // Something marked an item for change within a call to notifyObservers // (which is in the process of processing the list of items marked for change). // This means the change may fail to be processed. - LL_WARNS() << "Adding changed mask within notify observers! Change will likely be lost." << LL_ENDL; + LL_WARNS(LOG_INV) << "Adding changed mask within notify observers! Change will likely be lost." << LL_ENDL; LLViewerInventoryItem *item = getItem(referent); if (item) { - LL_WARNS() << "Item " << item->getName() << LL_ENDL; + LL_WARNS(LOG_INV) << "Item " << item->getName() << LL_ENDL; } else { LLViewerInventoryCategory *cat = getCategory(referent); if (cat) { - LL_WARNS() << "Category " << cat->getName() << LL_ENDL; + LL_WARNS(LOG_INV) << "Category " << cat->getName() << LL_ENDL; } } } @@ -1544,91 +1577,18 @@ void LLInventoryModel::addChangedMask(U32 mask, const LLUUID& referent) } } -// If we get back a normal response, handle it here -void LLInventoryModel::fetchInventoryResponder::httpSuccess() -{ - const LLSD& content = getContent(); - if (!content.isMap()) - { - failureResult(HTTP_INTERNAL_ERROR, "Malformed response contents", content); - return; - } - start_new_inventory_observer(); - - /*LLUUID agent_id; - agent_id = content["agent_id"].asUUID(); - if(agent_id != gAgent.getID()) - { - LL_WARNS() << "Got a inventory update for the wrong agent: " << agent_id - << LL_ENDL; - return; - }*/ - item_array_t items; - update_map_t update; - S32 count = content["items"].size(); - LLUUID folder_id; - // Does this loop ever execute more than once? - for(S32 i = 0; i < count; ++i) - { - LLPointer<LLViewerInventoryItem> titem = new LLViewerInventoryItem; - titem->unpackMessage(content["items"][i]); - - LL_DEBUGS() << "LLInventoryModel::fetchInventoryResponder item id: " - << titem->getUUID() << LL_ENDL; - items.push_back(titem); - // examine update for changes. - LLViewerInventoryItem* itemp = gInventory.getItem(titem->getUUID()); - if(itemp) - { - if(titem->getParentUUID() == itemp->getParentUUID()) - { - update[titem->getParentUUID()]; - } - else - { - ++update[titem->getParentUUID()]; - --update[itemp->getParentUUID()]; - } - } - else - { - ++update[titem->getParentUUID()]; - } - if (folder_id.isNull()) - { - folder_id = titem->getParentUUID(); - } - } - - U32 changes = 0x0; - //as above, this loop never seems to loop more than once per call - for (item_array_t::iterator it = items.begin(); it != items.end(); ++it) - { - changes |= gInventory.updateItem(*it); - } - gInventory.notifyObservers(); - gViewerWindow->getWindow()->decBusyCount(); -} - -//If we get back an error (not found, etc...), handle it here -void LLInventoryModel::fetchInventoryResponder::httpFailure() -{ - LL_WARNS() << dumpResponse() << LL_ENDL; - gInventory.notifyObservers(); -} - bool LLInventoryModel::fetchDescendentsOf(const LLUUID& folder_id) const { if(folder_id.isNull()) { - LL_WARNS() << "Calling fetch descendents on NULL folder id!" << LL_ENDL; + LL_WARNS(LOG_INV) << "Calling fetch descendents on NULL folder id!" << LL_ENDL; return false; } LLViewerInventoryCategory* cat = getCategory(folder_id); if(!cat) { - LL_WARNS() << "Asked to fetch descendents of non-existent folder: " - << folder_id << LL_ENDL; + LL_WARNS(LOG_INV) << "Asked to fetch descendents of non-existent folder: " + << folder_id << LL_ENDL; return false; } //S32 known_descendents = 0; @@ -1649,8 +1609,8 @@ void LLInventoryModel::cache( const LLUUID& parent_folder_id, const LLUUID& agent_id) { - LL_DEBUGS() << "Caching " << parent_folder_id << " for " << agent_id - << LL_ENDL; + LL_DEBUGS(LOG_INV) << "Caching " << parent_folder_id << " for " << agent_id + << LL_ENDL; LLViewerInventoryCategory* root_cat = getCategory(parent_folder_id); if(!root_cat) return; cat_array_t categories; @@ -1675,19 +1635,19 @@ void LLInventoryModel::cache( gzip_filename.append(".gz"); if(gzip_file(inventory_filename, gzip_filename)) { - LL_DEBUGS() << "Successfully compressed " << inventory_filename << LL_ENDL; + LL_DEBUGS(LOG_INV) << "Successfully compressed " << inventory_filename << LL_ENDL; LLFile::remove(inventory_filename); } else { - LL_WARNS() << "Unable to compress " << inventory_filename << LL_ENDL; + LL_WARNS(LOG_INV) << "Unable to compress " << inventory_filename << LL_ENDL; } } void LLInventoryModel::addCategory(LLViewerInventoryCategory* category) { - //LL_INFOS() << "LLInventoryModel::addCategory()" << LL_ENDL; + //LL_INFOS(LOG_INV) << "LLInventoryModel::addCategory()" << LL_ENDL; if(category) { // We aren't displaying the Meshes folder @@ -1757,7 +1717,9 @@ void LLInventoryModel::addItem(LLViewerInventoryItem* item) if ((item->getType() == LLAssetType::AT_NONE) || LLAssetType::lookup(item->getType()) == LLAssetType::badLookup()) { - LL_WARNS() << "Got bad asset type for item [ name: " << item->getName() << " type: " << item->getType() << " inv-type: " << item->getInventoryType() << " ], ignoring." << LL_ENDL; + LL_WARNS(LOG_INV) << "Got bad asset type for item [ name: " << item->getName() + << " type: " << item->getType() + << " inv-type: " << item->getInventoryType() << " ], ignoring." << LL_ENDL; return; } @@ -1765,7 +1727,9 @@ void LLInventoryModel::addItem(LLViewerInventoryItem* item) // The item will show up as a broken link. if (item->getIsBrokenLink()) { - LL_INFOS() << "Adding broken link [ name: " << item->getName() << " itemID: " << item->getUUID() << " assetID: " << item->getAssetUUID() << " ) parent: " << item->getParentUUID() << LL_ENDL; + LL_INFOS(LOG_INV) << "Adding broken link [ name: " << item->getName() + << " itemID: " << item->getUUID() + << " assetID: " << item->getAssetUUID() << " ) parent: " << item->getParentUUID() << LL_ENDL; } if (item->getIsLinkType()) { @@ -1781,7 +1745,7 @@ void LLInventoryModel::addItem(LLViewerInventoryItem* item) // Empty the entire contents void LLInventoryModel::empty() { -// LL_INFOS() << "LLInventoryModel::empty()" << LL_ENDL; +// LL_INFOS(LOG_INV) << "LLInventoryModel::empty()" << LL_ENDL; std::for_each( mParentChildCategoryTree.begin(), mParentChildCategoryTree.end(), @@ -1814,29 +1778,29 @@ void LLInventoryModel::accountForUpdate(const LLCategoryUpdate& update) const descendents_actual += update.mDescendentDelta; cat->setDescendentCount(descendents_actual); cat->setVersion(++version); - LL_DEBUGS("Inventory") << "accounted: '" << cat->getName() << "' " - << version << " with " << descendents_actual - << " descendents." << LL_ENDL; + LL_DEBUGS(LOG_INV) << "accounted: '" << cat->getName() << "' " + << version << " with " << descendents_actual + << " descendents." << LL_ENDL; } else { // Error condition, this means that the category did not register that // it got new descendents (perhaps because it is still being loaded) // which means its descendent count will be wrong. - LL_WARNS() << "Accounting failed for '" << cat->getName() << "' version:" - << version << " due to mismatched descendent count: server == " - << descendents_server << ", viewer == " << descendents_actual << LL_ENDL; + LL_WARNS(LOG_INV) << "Accounting failed for '" << cat->getName() << "' version:" + << version << " due to mismatched descendent count: server == " + << descendents_server << ", viewer == " << descendents_actual << LL_ENDL; } } else { - LL_WARNS() << "Accounting failed for '" << cat->getName() << "' version: unknown (" - << version << ")" << LL_ENDL; + LL_WARNS(LOG_INV) << "Accounting failed for '" << cat->getName() << "' version: unknown (" + << version << ")" << LL_ENDL; } } else { - LL_WARNS() << "No category found for update " << update.mCategoryID << LL_ENDL; + LL_WARNS(LOG_INV) << "No category found for update " << update.mCategoryID << LL_ENDL; } } @@ -1918,7 +1882,7 @@ bool LLInventoryModel::loadSkeleton( const LLSD& options, const LLUUID& owner_id) { - LL_DEBUGS() << "importing inventory skeleton for " << owner_id << LL_ENDL; + LL_DEBUGS(LOG_INV) << "importing inventory skeleton for " << owner_id << LL_ENDL; typedef std::set<LLPointer<LLViewerInventoryCategory>, InventoryIDPtrLess> cat_set_t; cat_set_t temp_cats; @@ -1955,7 +1919,7 @@ bool LLInventoryModel::loadSkeleton( } else { - LL_WARNS() << "Unable to import near " << name.asString() << LL_ENDL; + LL_WARNS(LOG_INV) << "Unable to import near " << name.asString() << LL_ENDL; rv = false; } } @@ -1992,7 +1956,7 @@ bool LLInventoryModel::loadSkeleton( } else { - LL_INFOS() << "Unable to gunzip " << gzip_filename << LL_ENDL; + LL_INFOS(LOG_INV) << "Unable to gunzip " << gzip_filename << LL_ENDL; } } bool is_cache_obsolete = false; @@ -2073,10 +2037,10 @@ bool LLInventoryModel::loadSkeleton( if (item->getIsBrokenLink()) { //bad_link_count++; - LL_DEBUGS() << "Attempted to add cached link item without baseobj present ( name: " - << item->getName() << " itemID: " << item->getUUID() - << " assetID: " << item->getAssetUUID() - << " ). Ignoring and invalidating " << cat->getName() << " . " << LL_ENDL; + LL_DEBUGS(LOG_INV) << "Attempted to add cached link item without baseobj present ( name: " + << item->getName() << " itemID: " << item->getUUID() + << " assetID: " << item->getAssetUUID() + << " ). Ignoring and invalidating " << cat->getName() << " . " << LL_ENDL; possible_broken_links.push_back(item); continue; } @@ -2103,7 +2067,7 @@ bool LLInventoryModel::loadSkeleton( { bad_link_count++; invalid_categories.insert(cit->second); - //LL_INFOS() << "link still broken: " << item->getName() << " in folder " << cat->getName() << LL_ENDL; + //LL_INFOS(LOG_INV) << "link still broken: " << item->getName() << " in folder " << cat->getName() << LL_ENDL; } else { @@ -2115,11 +2079,11 @@ bool LLInventoryModel::loadSkeleton( } } - LL_INFOS() << "Attempted to add " << bad_link_count - << " cached link items without baseobj present. " - << good_link_count << " link items were successfully added. " - << recovered_link_count << " links added in recovery. " - << "The corresponding categories were invalidated." << LL_ENDL; + LL_INFOS(LOG_INV) << "Attempted to add " << bad_link_count + << " cached link items without baseobj present. " + << good_link_count << " link items were successfully added. " + << recovered_link_count << " links added in recovery. " + << "The corresponding categories were invalidated." << LL_ENDL; } } @@ -2143,9 +2107,9 @@ bool LLInventoryModel::loadSkeleton( { LLViewerInventoryCategory* cat = (*invalid_cat_it).get(); cat->setVersion(NO_VERSION); - LL_DEBUGS("Inventory") << "Invalidating category name: " << cat->getName() << " UUID: " << cat->getUUID() << " due to invalid descendents cache" << LL_ENDL; + LL_DEBUGS(LOG_INV) << "Invalidating category name: " << cat->getName() << " UUID: " << cat->getUUID() << " due to invalid descendents cache" << LL_ENDL; } - LL_INFOS("Inventory") << "Invalidated " << invalid_categories.size() << " categories due to invalid descendents cache" << LL_ENDL; + LL_INFOS(LOG_INV) << "Invalidated " << invalid_categories.size() << " categories due to invalid descendents cache" << LL_ENDL; // At this point, we need to set the known descendents for each // category which successfully cached so that we do not @@ -2177,15 +2141,15 @@ bool LLInventoryModel::loadSkeleton( if(is_cache_obsolete) { // If out of date, remove the gzipped file too. - LL_WARNS() << "Inv cache out of date, removing" << LL_ENDL; + LL_WARNS(LOG_INV) << "Inv cache out of date, removing" << LL_ENDL; LLFile::remove(gzip_filename); } categories.clear(); // will unref and delete entries } - LL_INFOS() << "Successfully loaded " << cached_category_count - << " categories and " << cached_item_count << " items from cache." - << LL_ENDL; + LL_INFOS(LOG_INV) << "Successfully loaded " << cached_category_count + << " categories and " << cached_item_count << " items from cache." + << LL_ENDL; return rv; } @@ -2195,7 +2159,7 @@ bool LLInventoryModel::loadSkeleton( // should be sufficient for our needs. void LLInventoryModel::buildParentChildMap() { - LL_INFOS() << "LLInventoryModel::buildParentChildMap()" << LL_ENDL; + LL_INFOS(LOG_INV) << "LLInventoryModel::buildParentChildMap()" << LL_ENDL; // *NOTE: I am skipping the logic around folder version // synchronization here because it seems if a folder is lost, we @@ -2264,15 +2228,15 @@ void LLInventoryModel::buildParentChildMap() // implement it, we would need a set or map of uuid pairs // which would be (folder_id, new_parent_id) to be sent up // to the server. - LL_INFOS() << "Lost category: " << cat->getUUID() << " - " - << cat->getName() << LL_ENDL; + LL_INFOS(LOG_INV) << "Lost category: " << cat->getUUID() << " - " + << cat->getName() << LL_ENDL; ++lost; lost_cats.push_back(cat); } } if(lost) { - LL_WARNS() << "Found " << lost << " lost categories." << LL_ENDL; + LL_WARNS(LOG_INV) << "Found " << lost << " lost categories." << LL_ENDL; } // Do moves in a separate pass to make sure we've properly filed @@ -2307,7 +2271,7 @@ void LLInventoryModel::buildParentChildMap() } else { - LL_WARNS() << "Lost and found Not there!!" << LL_ENDL; + LL_WARNS(LOG_INV) << "Lost and found Not there!!" << LL_ENDL; } } @@ -2342,8 +2306,8 @@ void LLInventoryModel::buildParentChildMap() } else { - LL_INFOS() << "Lost item: " << item->getUUID() << " - " - << item->getName() << LL_ENDL; + LL_INFOS(LOG_INV) << "Lost item: " << item->getUUID() << " - " + << item->getName() << LL_ENDL; ++lost; // plop it into the lost & found. // @@ -2359,13 +2323,13 @@ void LLInventoryModel::buildParentChildMap() } else { - LL_WARNS() << "Lost and found Not there!!" << LL_ENDL; + LL_WARNS(LOG_INV) << "Lost and found Not there!!" << LL_ENDL; } } } if(lost) { - LL_WARNS() << "Found " << lost << " lost items." << LL_ENDL; + LL_WARNS(LOG_INV) << "Found " << lost << " lost items." << LL_ENDL; LLMessageSystem* msg = gMessageSystem; BOOL start_new_message = TRUE; const LLUUID lnf = findCategoryUUIDForType(LLFolderType::FT_LOST_AND_FOUND); @@ -2443,10 +2407,82 @@ void LLInventoryModel::buildParentChildMap() if (!gInventory.validate()) { - LL_WARNS() << "model failed validity check!" << LL_ENDL; + LL_WARNS(LOG_INV) << "model failed validity check!" << LL_ENDL; } } +// Would normally do this at construction but that's too early +// in the process for gInventory. Have the first requestPost() +// call set things up. +void LLInventoryModel::initHttpRequest() +{ + if (! mHttpRequestFG) + { + // Haven't initialized, get to it + LLAppCoreHttp & app_core_http(LLAppViewer::instance()->getAppCoreHttp()); + + mHttpRequestFG = new LLCore::HttpRequest; + mHttpRequestBG = new LLCore::HttpRequest; + mHttpOptions = new LLCore::HttpOptions; + mHttpOptions->setTransferTimeout(300); + mHttpOptions->setUseRetryAfter(true); + // mHttpOptions->setTrace(2); // Do tracing of requests + mHttpHeaders = new LLCore::HttpHeaders; + mHttpHeaders->append(HTTP_OUT_HEADER_CONTENT_TYPE, HTTP_CONTENT_LLSD_XML); + mHttpHeaders->append(HTTP_OUT_HEADER_ACCEPT, HTTP_CONTENT_LLSD_XML); + mHttpPolicyClass = app_core_http.getPolicy(LLAppCoreHttp::AP_INVENTORY); + } +} + +void LLInventoryModel::handleResponses(bool foreground) +{ + if (foreground && mHttpRequestFG) + { + mHttpRequestFG->update(0); + } + else if (! foreground && mHttpRequestBG) + { + mHttpRequestBG->update(50000L); + } +} + +LLCore::HttpHandle LLInventoryModel::requestPost(bool foreground, + const std::string & url, + const LLSD & body, + LLCore::HttpHandler * handler, + const char * const message) +{ + if (! mHttpRequestFG) + { + // We do the initialization late and lazily as this class is + // statically-constructed and not all the bits are ready at + // that time. + initHttpRequest(); + } + + LLCore::HttpRequest * request(foreground ? mHttpRequestFG : mHttpRequestBG); + LLCore::HttpHandle handle(LLCORE_HTTP_HANDLE_INVALID); + + handle = LLCoreHttpUtil::requestPostWithLLSD(request, + mHttpPolicyClass, + (foreground ? mHttpPriorityFG : mHttpPriorityBG), + url, + body, + mHttpOptions, + mHttpHeaders, + handler); + if (LLCORE_HTTP_HANDLE_INVALID == handle) + { + LLCore::HttpStatus status(request->getStatus()); + LL_WARNS(LOG_INV) << "HTTP POST request failed for " << message + << ", Status: " << status.toTerseString() + << " Reason: '" << status.toString() << "'" + << LL_ENDL; + delete handler; + } + return handle; +} + void LLInventoryModel::createCommonSystemCategories() { gInventory.findCategoryUUIDForType(LLFolderType::FT_TRASH,true); @@ -2495,14 +2531,14 @@ bool LLInventoryModel::loadFromFile(const std::string& filename, { if(filename.empty()) { - LL_ERRS() << "Filename is Null!" << LL_ENDL; + LL_ERRS(LOG_INV) << "Filename is Null!" << LL_ENDL; return false; } - LL_INFOS() << "LLInventoryModel::loadFromFile(" << filename << ")" << LL_ENDL; + LL_INFOS(LOG_INV) << "LLInventoryModel::loadFromFile(" << filename << ")" << LL_ENDL; LLFILE* file = LLFile::fopen(filename, "rb"); /*Flawfinder: ignore*/ if(!file) { - LL_INFOS() << "unable to load inventory from: " << filename << LL_ENDL; + LL_INFOS(LOG_INV) << "unable to load inventory from: " << filename << LL_ENDL; return false; } // *NOTE: This buffer size is hard coded into scanf() below. @@ -2541,7 +2577,7 @@ bool LLInventoryModel::loadFromFile(const std::string& filename, } else { - LL_WARNS() << "loadInventoryFromFile(). Ignoring invalid inventory category: " << inv_cat->getName() << LL_ENDL; + LL_WARNS(LOG_INV) << "loadInventoryFromFile(). Ignoring invalid inventory category: " << inv_cat->getName() << LL_ENDL; //delete inv_cat; // automatic when inv_cat is reassigned or destroyed } } @@ -2559,8 +2595,8 @@ bool LLInventoryModel::loadFromFile(const std::string& filename, if(inv_item->getUUID().isNull()) { //delete inv_item; // automatic when inv_cat is reassigned or destroyed - LL_WARNS() << "Ignoring inventory with null item id: " - << inv_item->getName() << LL_ENDL; + LL_WARNS(LOG_INV) << "Ignoring inventory with null item id: " + << inv_item->getName() << LL_ENDL; } else @@ -2570,14 +2606,14 @@ bool LLInventoryModel::loadFromFile(const std::string& filename, } else { - LL_WARNS() << "loadInventoryFromFile(). Ignoring invalid inventory item: " << inv_item->getName() << LL_ENDL; + LL_WARNS(LOG_INV) << "loadInventoryFromFile(). Ignoring invalid inventory item: " << inv_item->getName() << LL_ENDL; //delete inv_item; // automatic when inv_cat is reassigned or destroyed } } else { - LL_WARNS() << "Unknown token in inventory file '" << keyword << "'" - << LL_ENDL; + LL_WARNS(LOG_INV) << "Unknown token in inventory file '" << keyword << "'" + << LL_ENDL; } } fclose(file); @@ -2593,14 +2629,14 @@ bool LLInventoryModel::saveToFile(const std::string& filename, { if(filename.empty()) { - LL_ERRS() << "Filename is Null!" << LL_ENDL; + LL_ERRS(LOG_INV) << "Filename is Null!" << LL_ENDL; return false; } - LL_INFOS() << "LLInventoryModel::saveToFile(" << filename << ")" << LL_ENDL; + LL_INFOS(LOG_INV) << "LLInventoryModel::saveToFile(" << filename << ")" << LL_ENDL; LLFILE* file = LLFile::fopen(filename, "wb"); /*Flawfinder: ignore*/ if(!file) { - LL_WARNS() << "unable to save inventory to: " << filename << LL_ENDL; + LL_WARNS(LOG_INV) << "unable to save inventory to: " << filename << LL_ENDL; return false; } @@ -2705,8 +2741,8 @@ bool LLInventoryModel::messageUpdateCore(LLMessageSystem* msg, bool account, U32 msg->getUUIDFast(_PREHASH_AgentData, _PREHASH_AgentID, agent_id); if(agent_id != gAgent.getID()) { - LL_WARNS() << "Got a inventory update for the wrong agent: " << agent_id - << LL_ENDL; + LL_WARNS(LOG_INV) << "Got a inventory update for the wrong agent: " << agent_id + << LL_ENDL; return false; } item_array_t items; @@ -2718,8 +2754,8 @@ bool LLInventoryModel::messageUpdateCore(LLMessageSystem* msg, bool account, U32 { LLPointer<LLViewerInventoryItem> titem = new LLViewerInventoryItem; titem->unpackMessage(msg, _PREHASH_InventoryData, i); - LL_DEBUGS() << "LLInventoryModel::messageUpdateCore() item id:" - << titem->getUUID() << LL_ENDL; + LL_DEBUGS(LOG_INV) << "LLInventoryModel::messageUpdateCore() item id: " + << titem->getUUID() << LL_ENDL; items.push_back(titem); // examine update for changes. LLViewerInventoryItem* itemp = gInventory.getItem(titem->getUUID()); @@ -2770,17 +2806,17 @@ void LLInventoryModel::removeInventoryItem(LLUUID agent_id, LLMessageSystem* msg { LLUUID item_id; S32 count = msg->getNumberOfBlocksFast(msg_label); - LL_DEBUGS() << "Message has " << count << " item blocks" << LL_ENDL; + LL_DEBUGS(LOG_INV) << "Message has " << count << " item blocks" << LL_ENDL; uuid_vec_t item_ids; update_map_t update; for(S32 i = 0; i < count; ++i) { msg->getUUIDFast(msg_label, _PREHASH_ItemID, item_id, i); - LL_DEBUGS() << "Checking for item-to-be-removed " << item_id << LL_ENDL; + LL_DEBUGS(LOG_INV) << "Checking for item-to-be-removed " << item_id << LL_ENDL; LLViewerInventoryItem* itemp = gInventory.getItem(item_id); if(itemp) { - LL_DEBUGS() << "Item will be removed " << item_id << LL_ENDL; + LL_DEBUGS(LOG_INV) << "Item will be removed " << item_id << LL_ENDL; // we only bother with the delete and account if we found // the item - this is usually a back-up for permissions, // so frequently the item will already be gone. @@ -2791,7 +2827,7 @@ void LLInventoryModel::removeInventoryItem(LLUUID agent_id, LLMessageSystem* msg gInventory.accountForUpdate(update); for(uuid_vec_t::iterator it = item_ids.begin(); it != item_ids.end(); ++it) { - LL_DEBUGS() << "Calling deleteObject " << *it << LL_ENDL; + LL_DEBUGS(LOG_INV) << "Calling deleteObject " << *it << LL_ENDL; gInventory.deleteObject(*it); } } @@ -2799,13 +2835,13 @@ void LLInventoryModel::removeInventoryItem(LLUUID agent_id, LLMessageSystem* msg // static void LLInventoryModel::processRemoveInventoryItem(LLMessageSystem* msg, void**) { - LL_DEBUGS() << "LLInventoryModel::processRemoveInventoryItem()" << LL_ENDL; + LL_DEBUGS(LOG_INV) << "LLInventoryModel::processRemoveInventoryItem()" << LL_ENDL; LLUUID agent_id, item_id; msg->getUUIDFast(_PREHASH_AgentData, _PREHASH_AgentID, agent_id); if(agent_id != gAgent.getID()) { - LL_WARNS() << "Got a RemoveInventoryItem for the wrong agent." - << LL_ENDL; + LL_WARNS(LOG_INV) << "Got a RemoveInventoryItem for the wrong agent." + << LL_ENDL; return; } LLInventoryModel::removeInventoryItem(agent_id, msg, _PREHASH_InventoryData); @@ -2816,14 +2852,14 @@ void LLInventoryModel::processRemoveInventoryItem(LLMessageSystem* msg, void**) void LLInventoryModel::processUpdateInventoryFolder(LLMessageSystem* msg, void**) { - LL_DEBUGS() << "LLInventoryModel::processUpdateInventoryFolder()" << LL_ENDL; + LL_DEBUGS(LOG_INV) << "LLInventoryModel::processUpdateInventoryFolder()" << LL_ENDL; LLUUID agent_id, folder_id, parent_id; //char name[DB_INV_ITEM_NAME_BUF_SIZE]; msg->getUUIDFast(_PREHASH_FolderData, _PREHASH_AgentID, agent_id); if(agent_id != gAgent.getID()) { - LL_WARNS() << "Got an UpdateInventoryFolder for the wrong agent." - << LL_ENDL; + LL_WARNS(LOG_INV) << "Got an UpdateInventoryFolder for the wrong agent." + << LL_ENDL; return; } LLPointer<LLViewerInventoryCategory> lastfolder; // hack @@ -3881,7 +3917,7 @@ bool LLInventoryModel::validate() const ///---------------------------------------------------------------------------- -/* +#if 0 BOOL decompress_file(const char* src_filename, const char* dst_filename) { BOOL rv = FALSE; @@ -3920,4 +3956,177 @@ BOOL decompress_file(const char* src_filename, const char* dst_filename) if(dst != NULL) fclose(dst); return rv; } -*/ +#endif + + +///---------------------------------------------------------------------------- +/// Class LLInventoryModel::FetchItemHttpHandler +///---------------------------------------------------------------------------- + +LLInventoryModel::FetchItemHttpHandler::FetchItemHttpHandler(const LLSD & request_sd) + : LLCore::HttpHandler(), + mRequestSD(request_sd) +{} + +LLInventoryModel::FetchItemHttpHandler::~FetchItemHttpHandler() +{} + +void LLInventoryModel::FetchItemHttpHandler::onCompleted(LLCore::HttpHandle handle, + LLCore::HttpResponse * response) +{ + do // Single-pass do-while used for common exit handling + { + LLCore::HttpStatus status(response->getStatus()); + // status = LLCore::HttpStatus(404); // Dev tool to force error handling + if (! status) + { + processFailure(status, response); + break; // Goto common exit + } + + LLCore::BufferArray * body(response->getBody()); + // body = NULL; // Dev tool to force error handling + if (! body || ! body->size()) + { + LL_WARNS(LOG_INV) << "Missing data in inventory item query." << LL_ENDL; + processFailure("HTTP response for inventory item query missing body", response); + break; // Goto common exit + } + + // body->write(0, "Garbage Response", 16); // Dev tool to force error handling + LLSD body_llsd; + if (! LLCoreHttpUtil::responseToLLSD(response, true, body_llsd)) + { + // INFOS-level logging will occur on the parsed failure + processFailure("HTTP response for inventory item query has malformed LLSD", response); + break; // Goto common exit + } + + // Expect top-level structure to be a map + // body_llsd = LLSD::emptyArray(); // Dev tool to force error handling + if (! body_llsd.isMap()) + { + processFailure("LLSD response for inventory item not a map", response); + break; // Goto common exit + } + + // Check for 200-with-error failures + // + // Original Responder-based serivce model didn't check for these errors. + // It may be more robust to ignore this condition. With aggregated requests, + // an error in one inventory item might take down the entire request. + // So if this instead broke up the aggregated items into single requests, + // maybe that would make progress. Or perhaps there's structured information + // that can tell us what went wrong. Need to dig into this and firm up + // the API. + // + // body_llsd["error"] = LLSD::emptyMap(); // Dev tool to force error handling + // body_llsd["error"]["identifier"] = "Development"; + // body_llsd["error"]["message"] = "You left development code in the viewer"; + if (body_llsd.has("error")) + { + processFailure("Inventory application error (200-with-error)", response); + break; // Goto common exit + } + + // Okay, process data if possible + processData(body_llsd, response); + } + while (false); + + // Must delete on completion. + delete this; +} + +void LLInventoryModel::FetchItemHttpHandler::processData(LLSD & content, LLCore::HttpResponse * response) +{ + start_new_inventory_observer(); + +#if 0 + LLUUID agent_id; + agent_id = content["agent_id"].asUUID(); + if (agent_id != gAgent.getID()) + { + LL_WARNS(LOG_INV) << "Got a inventory update for the wrong agent: " << agent_id + << LL_ENDL; + return; + } +#endif + + LLInventoryModel::item_array_t items; + LLInventoryModel::update_map_t update; + LLUUID folder_id; + LLSD content_items(content["items"]); + const S32 count(content_items.size()); + + // Does this loop ever execute more than once? + for (S32 i(0); i < count; ++i) + { + LLPointer<LLViewerInventoryItem> titem = new LLViewerInventoryItem; + titem->unpackMessage(content_items[i]); + + LL_DEBUGS(LOG_INV) << "ItemHttpHandler::httpSuccess item id: " + << titem->getUUID() << LL_ENDL; + items.push_back(titem); + + // examine update for changes. + LLViewerInventoryItem * itemp(gInventory.getItem(titem->getUUID())); + + if (itemp) + { + if (titem->getParentUUID() == itemp->getParentUUID()) + { + update[titem->getParentUUID()]; + } + else + { + ++update[titem->getParentUUID()]; + --update[itemp->getParentUUID()]; + } + } + else + { + ++update[titem->getParentUUID()]; + } + + if (folder_id.isNull()) + { + folder_id = titem->getParentUUID(); + } + } + + // as above, this loop never seems to loop more than once per call + U32 changes(0U); + for (LLInventoryModel::item_array_t::iterator it = items.begin(); it != items.end(); ++it) + { + changes |= gInventory.updateItem(*it); + } + // *HUH: Have computed 'changes', nothing uses it. + + gInventory.notifyObservers(); + gViewerWindow->getWindow()->decBusyCount(); +} + + +void LLInventoryModel::FetchItemHttpHandler::processFailure(LLCore::HttpStatus status, LLCore::HttpResponse * response) +{ + const std::string & ct(response->getContentType()); + LL_WARNS(LOG_INV) << "Inventory item fetch failure\n" + << "[Status: " << status.toTerseString() << "]\n" + << "[Reason: " << status.toString() << "]\n" + << "[Content-type: " << ct << "]\n" + << "[Content (abridged): " + << LLCoreHttpUtil::responseToString(response) << "]" << LL_ENDL; + gInventory.notifyObservers(); +} + +void LLInventoryModel::FetchItemHttpHandler::processFailure(const char * const reason, LLCore::HttpResponse * response) +{ + LL_WARNS(LOG_INV) << "Inventory item fetch failure\n" + << "[Status: internal error]\n" + << "[Reason: " << reason << "]\n" + << "[Content (abridged): " + << LLCoreHttpUtil::responseToString(response) << "]" << LL_ENDL; + gInventory.notifyObservers(); +} + diff --git a/indra/newview/llinventorymodel.h b/indra/newview/llinventorymodel.h index 2e957809be..ac336e347c 100755 --- a/indra/newview/llinventorymodel.h +++ b/indra/newview/llinventorymodel.h @@ -27,6 +27,11 @@ #ifndef LL_LLINVENTORYMODEL_H #define LL_LLINVENTORYMODEL_H +#include <map> +#include <set> +#include <string> +#include <vector> + #include "llassettype.h" #include "llfoldertype.h" #include "llframetimer.h" @@ -36,10 +41,11 @@ #include "llviewerinventory.h" #include "llstring.h" #include "llmd5.h" -#include <map> -#include <set> -#include <string> -#include <vector> +#include "httpcommon.h" +#include "httprequest.h" +#include "httpoptions.h" +#include "httpheaders.h" +#include "httphandler.h" class LLInventoryObserver; class LLInventoryObject; @@ -60,9 +66,8 @@ class LLInventoryCollectFunctor; class LLInventoryModel { LOG_CLASS(LLInventoryModel); -public: - friend class LLInventoryModelFetchDescendentsResponder; +public: enum EHasChildren { CHILDREN_NO, @@ -74,14 +79,31 @@ public: typedef std::vector<LLPointer<LLViewerInventoryItem> > item_array_t; typedef std::set<LLUUID> changed_items_t; - class fetchInventoryResponder : public LLCurl::Responder + // HTTP handler for individual item requests (inventory or library). + // Background item requests are derived from this in the background + // inventory system. All folder requests are also located there + // but have their own handler derived from HttpHandler. + class FetchItemHttpHandler : public LLCore::HttpHandler { - LOG_CLASS(fetchInventoryResponder); public: - fetchInventoryResponder(const LLSD& request_sd) : mRequestSD(request_sd) {}; + LOG_CLASS(FetchItemHttpHandler); + + FetchItemHttpHandler(const LLSD & request_sd); + virtual ~FetchItemHttpHandler(); + protected: - virtual void httpSuccess(); - virtual void httpFailure(); + FetchItemHttpHandler(const FetchItemHttpHandler &); // Not defined + void operator=(const FetchItemHttpHandler &); // Not defined + + public: + virtual void onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * response); + + private: + void processData(LLSD & body, LLCore::HttpResponse * response); + void processFailure(LLCore::HttpStatus status, LLCore::HttpResponse * response); + void processFailure(const char * const reason, LLCore::HttpResponse * response); + + private: LLSD mRequestSD; }; @@ -109,6 +131,9 @@ public: private: bool mIsAgentInvUsable; // used to handle an invalid inventory state + // One-time initialization of HTTP system. + void initHttpRequest(); + //-------------------------------------------------------------------- // Root Folders //-------------------------------------------------------------------- @@ -520,6 +545,41 @@ private: /******************************************************************************** ** ** + ** HTTP Transport + **/ +public: + // Invoke handler completion method (onCompleted) for all + // requests that are ready. + void handleResponses(bool foreground); + + // Request an inventory HTTP operation to either the + // foreground or background processor. These are actually + // the same service queue but the background requests are + // seviced more slowly effectively de-prioritizing new + // requests. + LLCore::HttpHandle requestPost(bool foreground, + const std::string & url, + const LLSD & body, + LLCore::HttpHandler * handler, + const char * const message); + +private: + // Usual plumbing for LLCore:: HTTP operations. + LLCore::HttpRequest * mHttpRequestFG; + LLCore::HttpRequest * mHttpRequestBG; + LLCore::HttpOptions * mHttpOptions; + LLCore::HttpHeaders * mHttpHeaders; + LLCore::HttpRequest::policy_t mHttpPolicyClass; + LLCore::HttpRequest::priority_t mHttpPriorityFG; + LLCore::HttpRequest::priority_t mHttpPriorityBG; + +/** HTTP Transport + ** ** + *******************************************************************************/ + + +/******************************************************************************** + ** ** ** MISCELLANEOUS **/ diff --git a/indra/newview/llinventorymodelbackgroundfetch.cpp b/indra/newview/llinventorymodelbackgroundfetch.cpp index 2de37b0790..f18832fe95 100755 --- a/indra/newview/llinventorymodelbackgroundfetch.cpp +++ b/indra/newview/llinventorymodelbackgroundfetch.cpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2002&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2010, Linden Research, Inc. + * Copyright (C) 2014, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -37,31 +37,180 @@ #include "llviewermessage.h" #include "llviewerregion.h" #include "llviewerwindow.h" +#include "llhttpconstants.h" +#include "bufferarray.h" +#include "bufferstream.h" +#include "llcorehttputil.h" + +// History (may be apocryphal) +// +// Around V2, an HTTP inventory download mechanism was added +// along with inventory LINK items referencing other inventory +// items. As part of this, at login, the entire inventory +// structure is downloaded 'in the background' using the +// backgroundFetch()/bulkFetch() methods. The UDP path can +// still be used and is found in the 'DEPRECATED OLD CODE' +// section. +// +// The old UDP path implemented a throttle that adapted +// itself during running. The mechanism survived info HTTP +// somewhat but was pinned to poll the HTTP plumbing at +// 0.5S intervals. The reasons for this particular value +// have been lost. It's possible to switch between UDP +// and HTTP while this is happening but there may be +// surprises in what happens in that case. +// +// Conversion to llcorehttp reduced the number of connections +// used but batches more data and queues more requests (but +// doesn't due pipelining due to libcurl restrictions). The +// poll interval above was re-examined and reduced to get +// inventory into the viewer more quickly. +// +// Possible future work: +// +// * Don't download the entire heirarchy in one go (which +// might have been how V1 worked). Implications for +// links (which may not have a valid target) and search +// which would then be missing data. +// +// * Review the download rate throttling. Slow then fast? +// Detect bandwidth usage and speed up when it drops? +// +// * A lot of calls to notifyObservers(). It looks like +// these could be collapsed by maintaining a 'dirty' +// bit and there appears to be an attempt to do this. +// But it isn't used or is used in a limited fashion. +// Are there semanic issues requiring a call after certain +// updateItem() calls? +// +// * An error on a fetch could be due to one item in the batch. +// If the batch were broken up, perhaps more of the inventory +// would download. (Handwave here, not certain this is an +// issue in practice.) +// +// * Conversion to AISv3. +// + + +namespace +{ + +///---------------------------------------------------------------------------- +/// Class <anonymous>::BGItemHttpHandler +///---------------------------------------------------------------------------- + +// +// Http request handler class for single inventory item requests. +// +// We'll use a handler-per-request pattern here rather than +// a shared handler. Mainly convenient as this was converted +// from a Responder class model. +// +// Derives from and is identical to the normal FetchItemHttpHandler +// except that: 1) it uses the background request object which is +// updated more slowly than the foreground and 2) keeps a count of +// active requests on the LLInventoryModelBackgroundFetch object +// to indicate outstanding operations are in-flight. +// +class BGItemHttpHandler : public LLInventoryModel::FetchItemHttpHandler +{ + LOG_CLASS(BGItemHttpHandler); + +public: + BGItemHttpHandler(const LLSD & request_sd) + : LLInventoryModel::FetchItemHttpHandler(request_sd) + { + LLInventoryModelBackgroundFetch::instance().incrFetchCount(1); + } + + virtual ~BGItemHttpHandler() + { + LLInventoryModelBackgroundFetch::instance().incrFetchCount(-1); + } + +protected: + BGItemHttpHandler(const BGItemHttpHandler &); // Not defined + void operator=(const BGItemHttpHandler &); // Not defined +}; + + +///---------------------------------------------------------------------------- +/// Class <anonymous>::BGFolderHttpHandler +///---------------------------------------------------------------------------- + +// Http request handler class for folders. +// +// Handler for FetchInventoryDescendents2 and FetchLibDescendents2 +// caps requests for folders. +// +class BGFolderHttpHandler : public LLCore::HttpHandler +{ + LOG_CLASS(BGFolderHttpHandler); + +public: + BGFolderHttpHandler(const LLSD & request_sd, const uuid_vec_t & recursive_cats) + : LLCore::HttpHandler(), + mRequestSD(request_sd), + mRecursiveCatUUIDs(recursive_cats) + { + LLInventoryModelBackgroundFetch::instance().incrFetchCount(1); + } + + virtual ~BGFolderHttpHandler() + { + LLInventoryModelBackgroundFetch::instance().incrFetchCount(-1); + } + +protected: + BGFolderHttpHandler(const BGFolderHttpHandler &); // Not defined + void operator=(const BGFolderHttpHandler &); // Not defined + +public: + virtual void onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * response); + + bool getIsRecursive(const LLUUID & cat_id) const; + +private: + void processData(LLSD & body, LLCore::HttpResponse * response); + void processFailure(LLCore::HttpStatus status, LLCore::HttpResponse * response); + void processFailure(const char * const reason, LLCore::HttpResponse * response); + +private: + LLSD mRequestSD; + const uuid_vec_t mRecursiveCatUUIDs; // hack for storing away which cat fetches are recursive +}; + -const F32 MAX_TIME_FOR_SINGLE_FETCH = 10.f; const S32 MAX_FETCH_RETRIES = 10; -LLInventoryModelBackgroundFetch::LLInventoryModelBackgroundFetch() : +const char * const LOG_INV("Inventory"); + +} // end of namespace anonymous + + +///---------------------------------------------------------------------------- +/// Class LLInventoryModelBackgroundFetch +///---------------------------------------------------------------------------- + +LLInventoryModelBackgroundFetch::LLInventoryModelBackgroundFetch(): mBackgroundFetchActive(FALSE), mFolderFetchActive(false), + mFetchCount(0), mAllFoldersFetched(FALSE), mRecursiveInventoryFetchStarted(FALSE), mRecursiveLibraryFetchStarted(FALSE), mNumFetchRetries(0), mMinTimeBetweenFetches(0.3f), mMaxTimeBetweenFetches(10.f), - mTimelyFetchPending(FALSE), - mFetchCount(0) -{ -} + mTimelyFetchPending(FALSE) +{} LLInventoryModelBackgroundFetch::~LLInventoryModelBackgroundFetch() -{ -} +{} bool LLInventoryModelBackgroundFetch::isBulkFetchProcessingComplete() const { - return mFetchQueue.empty() && mFetchCount<=0; + return mFetchQueue.empty() && mFetchCount <= 0; } bool LLInventoryModelBackgroundFetch::libraryFetchStarted() const @@ -91,7 +240,7 @@ bool LLInventoryModelBackgroundFetch::inventoryFetchCompleted() const bool LLInventoryModelBackgroundFetch::inventoryFetchInProgress() const { - return inventoryFetchStarted() && !inventoryFetchCompleted(); + return inventoryFetchStarted() && ! inventoryFetchCompleted(); } bool LLInventoryModelBackgroundFetch::isEverythingFetched() const @@ -104,24 +253,36 @@ BOOL LLInventoryModelBackgroundFetch::folderFetchActive() const return mFolderFetchActive; } +void LLInventoryModelBackgroundFetch::addRequestAtFront(const LLUUID & id, BOOL recursive, bool is_category) +{ + mFetchQueue.push_front(FetchQueueInfo(id, recursive, is_category)); +} + +void LLInventoryModelBackgroundFetch::addRequestAtBack(const LLUUID & id, BOOL recursive, bool is_category) +{ + mFetchQueue.push_back(FetchQueueInfo(id, recursive, is_category)); +} + void LLInventoryModelBackgroundFetch::start(const LLUUID& id, BOOL recursive) { - LLViewerInventoryCategory* cat = gInventory.getCategory(id); - if (cat || (id.isNull() && !isEverythingFetched())) - { // it's a folder, do a bulk fetch - LL_DEBUGS("InventoryFetch") << "Start fetching category: " << id << ", recursive: " << recursive << LL_ENDL; + LLViewerInventoryCategory * cat(gInventory.getCategory(id)); + + if (cat || (id.isNull() && ! isEverythingFetched())) + { + // it's a folder, do a bulk fetch + LL_DEBUGS(LOG_INV) << "Start fetching category: " << id << ", recursive: " << recursive << LL_ENDL; mBackgroundFetchActive = TRUE; mFolderFetchActive = true; if (id.isNull()) { - if (!mRecursiveInventoryFetchStarted) + if (! mRecursiveInventoryFetchStarted) { mRecursiveInventoryFetchStarted |= recursive; mFetchQueue.push_back(FetchQueueInfo(gInventory.getRootFolderID(), recursive)); gIdleCallbacks.addFunction(&LLInventoryModelBackgroundFetch::backgroundFetchCB, NULL); } - if (!mRecursiveLibraryFetchStarted) + if (! mRecursiveLibraryFetchStarted) { mRecursiveLibraryFetchStarted |= recursive; mFetchQueue.push_back(FetchQueueInfo(gInventory.getLibraryRootFolderID(), recursive)); @@ -146,9 +307,9 @@ void LLInventoryModelBackgroundFetch::start(const LLUUID& id, BOOL recursive) } } } - else if (LLViewerInventoryItem* itemp = gInventory.getItem(id)) + else if (LLViewerInventoryItem * itemp = gInventory.getItem(id)) { - if (!itemp->mIsComplete && (mFetchQueue.empty() || mFetchQueue.front().mUUID != id)) + if (! itemp->mIsComplete && (mFetchQueue.empty() || mFetchQueue.front().mUUID != id)) { mBackgroundFetchActive = TRUE; @@ -172,11 +333,12 @@ void LLInventoryModelBackgroundFetch::setAllFoldersFetched() mRecursiveLibraryFetchStarted) { mAllFoldersFetched = TRUE; - //LL_INFOS() << "All folders fetched, validating" << LL_ENDL; + //LL_INFOS(LOG_INV) << "All folders fetched, validating" << LL_ENDL; //gInventory.validate(); } mFolderFetchActive = false; mBackgroundFetchActive = false; + LL_INFOS(LOG_INV) << "Inventory background fetch completed" << LL_ENDL; } void LLInventoryModelBackgroundFetch::backgroundFetchCB(void *) @@ -203,12 +365,7 @@ void LLInventoryModelBackgroundFetch::backgroundFetch() // No more categories to fetch, stop fetch process. if (mFetchQueue.empty()) { - LL_INFOS() << "Inventory fetch completed" << LL_ENDL; - setAllFoldersFetched(); - mBackgroundFetchActive = false; - mFolderFetchActive = false; - return; } @@ -219,7 +376,7 @@ void LLInventoryModelBackgroundFetch::backgroundFetch() // Double timeouts on failure. mMinTimeBetweenFetches = llmin(mMinTimeBetweenFetches * 2.f, 10.f); mMaxTimeBetweenFetches = llmin(mMaxTimeBetweenFetches * 2.f, 120.f); - LL_DEBUGS() << "Inventory fetch times grown to (" << mMinTimeBetweenFetches << ", " << mMaxTimeBetweenFetches << ")" << LL_ENDL; + LL_DEBUGS(LOG_INV) << "Inventory fetch times grown to (" << mMinTimeBetweenFetches << ", " << mMaxTimeBetweenFetches << ")" << LL_ENDL; // fetch is no longer considered "timely" although we will wait for full time-out. mTimelyFetchPending = FALSE; } @@ -231,7 +388,7 @@ void LLInventoryModelBackgroundFetch::backgroundFetch() break; } - if(gDisconnected) + if (gDisconnected) { // Just bail if we are disconnected. break; @@ -292,7 +449,7 @@ void LLInventoryModelBackgroundFetch::backgroundFetch() // Shrink timeouts based on success. mMinTimeBetweenFetches = llmax(mMinTimeBetweenFetches * 0.8f, 0.3f); mMaxTimeBetweenFetches = llmax(mMaxTimeBetweenFetches * 0.8f, 10.f); - LL_DEBUGS() << "Inventory fetch times shrunk to (" << mMinTimeBetweenFetches << ", " << mMaxTimeBetweenFetches << ")" << LL_ENDL; + LL_DEBUGS(LOG_INV) << "Inventory fetch times shrunk to (" << mMinTimeBetweenFetches << ", " << mMaxTimeBetweenFetches << ")" << LL_ENDL; } mTimelyFetchPending = FALSE; @@ -355,258 +512,61 @@ void LLInventoryModelBackgroundFetch::backgroundFetch() } } -void LLInventoryModelBackgroundFetch::incrFetchCount(S16 fetching) +void LLInventoryModelBackgroundFetch::incrFetchCount(S32 fetching) { mFetchCount += fetching; if (mFetchCount < 0) { + LL_WARNS_ONCE(LOG_INV) << "Inventory fetch count fell below zero (0)." << LL_ENDL; mFetchCount = 0; } } -class LLInventoryModelFetchItemResponder : public LLInventoryModel::fetchInventoryResponder -{ - LOG_CLASS(LLInventoryModelFetchItemResponder); -public: - LLInventoryModelFetchItemResponder(const LLSD& request_sd) : - LLInventoryModel::fetchInventoryResponder(request_sd) - { - LLInventoryModelBackgroundFetch::instance().incrFetchCount(1); - } -private: - /* virtual */ void httpCompleted() - { - LLInventoryModelBackgroundFetch::instance().incrFetchCount(-1); - LLInventoryModel::fetchInventoryResponder::httpCompleted(); - } -}; - -class LLInventoryModelFetchDescendentsResponder: public LLHTTPClient::Responder -{ - LOG_CLASS(LLInventoryModelFetchDescendentsResponder); -public: - LLInventoryModelFetchDescendentsResponder(const LLSD& request_sd, uuid_vec_t recursive_cats) : - mRequestSD(request_sd), - mRecursiveCatUUIDs(recursive_cats) - { - LLInventoryModelBackgroundFetch::instance().incrFetchCount(1); - } - //LLInventoryModelFetchDescendentsResponder() {}; -private: - /* virtual */ void httpCompleted() - { - LLInventoryModelBackgroundFetch::instance().incrFetchCount(-1); - LLHTTPClient::Responder::httpCompleted(); - } - /* virtual */ void httpSuccess(); - /* virtual */ void httpFailure(); -protected: - BOOL getIsRecursive(const LLUUID& cat_id) const; -private: - LLSD mRequestSD; - uuid_vec_t mRecursiveCatUUIDs; // hack for storing away which cat fetches are recursive -}; - -// If we get back a normal response, handle it here. -void LLInventoryModelFetchDescendentsResponder::httpSuccess() +// Bundle up a bunch of requests to send all at once. +void LLInventoryModelBackgroundFetch::bulkFetch() { - const LLSD& content = getContent(); - if (!content.isMap()) + //Background fetch is called from gIdleCallbacks in a loop until background fetch is stopped. + //If there are items in mFetchQueue, we want to check the time since the last bulkFetch was + //sent. If it exceeds our retry time, go ahead and fire off another batch. + LLViewerRegion * region(gAgent.getRegion()); + if (! region || gDisconnected) { - failureResult(HTTP_INTERNAL_ERROR, "Malformed response contents", content); return; } - LLInventoryModelBackgroundFetch *fetcher = LLInventoryModelBackgroundFetch::getInstance(); - if (content.has("folders")) - { - - for(LLSD::array_const_iterator folder_it = content["folders"].beginArray(); - folder_it != content["folders"].endArray(); - ++folder_it) - { - LLSD folder_sd = *folder_it; - - - //LLUUID agent_id = folder_sd["agent_id"]; - - //if(agent_id != gAgent.getID()) //This should never happen. - //{ - // LL_WARNS() << "Got a UpdateInventoryItem for the wrong agent." - // << LL_ENDL; - // break; - //} - - LLUUID parent_id = folder_sd["folder_id"]; - LLUUID owner_id = folder_sd["owner_id"]; - S32 version = (S32)folder_sd["version"].asInteger(); - S32 descendents = (S32)folder_sd["descendents"].asInteger(); - LLPointer<LLViewerInventoryCategory> tcategory = new LLViewerInventoryCategory(owner_id); - if (parent_id.isNull()) - { - LLPointer<LLViewerInventoryItem> titem = new LLViewerInventoryItem; - for(LLSD::array_const_iterator item_it = folder_sd["items"].beginArray(); - item_it != folder_sd["items"].endArray(); - ++item_it) - { - const LLUUID lost_uuid = gInventory.findCategoryUUIDForType(LLFolderType::FT_LOST_AND_FOUND); - if (lost_uuid.notNull()) - { - LLSD item = *item_it; - titem->unpackMessage(item); - - LLInventoryModel::update_list_t update; - LLInventoryModel::LLCategoryUpdate new_folder(lost_uuid, 1); - update.push_back(new_folder); - gInventory.accountForUpdate(update); - - titem->setParent(lost_uuid); - titem->updateParentOnServer(FALSE); - gInventory.updateItem(titem); - gInventory.notifyObservers(); - - } - } - } - - LLViewerInventoryCategory* pcat = gInventory.getCategory(parent_id); - if (!pcat) - { - continue; - } - - for(LLSD::array_const_iterator category_it = folder_sd["categories"].beginArray(); - category_it != folder_sd["categories"].endArray(); - ++category_it) - { - LLSD category = *category_it; - tcategory->fromLLSD(category); - - const BOOL recursive = getIsRecursive(tcategory->getUUID()); - - if (recursive) - { - fetcher->mFetchQueue.push_back(LLInventoryModelBackgroundFetch::FetchQueueInfo(tcategory->getUUID(), recursive)); - } - else if ( !gInventory.isCategoryComplete(tcategory->getUUID()) ) - { - gInventory.updateCategory(tcategory); - } - - } - LLPointer<LLViewerInventoryItem> titem = new LLViewerInventoryItem; - for(LLSD::array_const_iterator item_it = folder_sd["items"].beginArray(); - item_it != folder_sd["items"].endArray(); - ++item_it) - { - LLSD item = *item_it; - titem->unpackMessage(item); - - gInventory.updateItem(titem); - } - - // Set version and descendentcount according to message. - LLViewerInventoryCategory* cat = gInventory.getCategory(parent_id); - if(cat) - { - cat->setVersion(version); - cat->setDescendentCount(descendents); - cat->determineFolderType(); - } - - } - } - - if (content.has("bad_folders")) - { - for(LLSD::array_const_iterator folder_it = content["bad_folders"].beginArray(); - folder_it != content["bad_folders"].endArray(); - ++folder_it) - { - // *TODO: Stop copying data - LLSD folder_sd = *folder_it; - - // These folders failed on the dataserver. We probably don't want to retry them. - LL_WARNS() << "Folder " << folder_sd["folder_id"].asString() - << "Error: " << folder_sd["error"].asString() << LL_ENDL; - } - } + // *TODO: These values could be tweaked at runtime to effect + // a fast/slow fetch throttle. Once login is complete and the scene + // is mostly loaded, we could turn up the throttle and fill missing + // inventory more quickly. + static const U32 max_batch_size(10); + static const S32 max_concurrent_fetches(12); // Outstanding requests, not connections + static const F32 new_min_time(0.05f); // *HACK: Clean this up when old code goes away entirely. - if (fetcher->isBulkFetchProcessingComplete()) + mMinTimeBetweenFetches = new_min_time; + if (mMinTimeBetweenFetches < new_min_time) { - LL_INFOS() << "Inventory fetch completed" << LL_ENDL; - fetcher->setAllFoldersFetched(); + mMinTimeBetweenFetches = new_min_time; // *HACK: See above. } - - gInventory.notifyObservers(); -} - -// If we get back an error (not found, etc...), handle it here. -void LLInventoryModelFetchDescendentsResponder::httpFailure() -{ - LL_WARNS() << dumpResponse() << LL_ENDL; - LLInventoryModelBackgroundFetch *fetcher = LLInventoryModelBackgroundFetch::getInstance(); - LL_INFOS() << dumpResponse() << LL_ENDL; - - fetcher->incrFetchCount(-1); - - if (getStatus()==HTTP_INTERNAL_ERROR) // timed out or curl failure - { - for(LLSD::array_const_iterator folder_it = mRequestSD["folders"].beginArray(); - folder_it != mRequestSD["folders"].endArray(); - ++folder_it) - { - LLSD folder_sd = *folder_it; - LLUUID folder_id = folder_sd["folder_id"]; - const BOOL recursive = getIsRecursive(folder_id); - fetcher->mFetchQueue.push_front(LLInventoryModelBackgroundFetch::FetchQueueInfo(folder_id, recursive)); - } - } - else + if (mFetchCount) { - if (fetcher->isBulkFetchProcessingComplete()) - { - fetcher->setAllFoldersFetched(); - } - } - gInventory.notifyObservers(); -} - -BOOL LLInventoryModelFetchDescendentsResponder::getIsRecursive(const LLUUID& cat_id) const -{ - return (std::find(mRecursiveCatUUIDs.begin(),mRecursiveCatUUIDs.end(), cat_id) != mRecursiveCatUUIDs.end()); -} -// Bundle up a bunch of requests to send all at once. -// static -void LLInventoryModelBackgroundFetch::bulkFetch() -{ - //Background fetch is called from gIdleCallbacks in a loop until background fetch is stopped. - //If there are items in mFetchQueue, we want to check the time since the last bulkFetch was - //sent. If it exceeds our retry time, go ahead and fire off another batch. - LLViewerRegion* region = gAgent.getRegion(); - if (!region) return; - - S16 max_concurrent_fetches=8; - F32 new_min_time = 0.5f; //HACK! Clean this up when old code goes away entirely. - if (mMinTimeBetweenFetches < new_min_time) - { - mMinTimeBetweenFetches=new_min_time; //HACK! See above. + // Process completed background HTTP requests + gInventory.handleResponses(false); } - if (gDisconnected || - (mFetchCount > max_concurrent_fetches) || + if ((mFetchCount > max_concurrent_fetches) || (mFetchTimer.getElapsedTimeF32() < mMinTimeBetweenFetches)) { - return; // just bail if we are disconnected + return; } - U32 item_count=0; - U32 folder_count=0; - U32 max_batch_size=5; + U32 item_count(0); + U32 folder_count(0); - U32 sort_order = gSavedSettings.getU32(LLInventoryPanel::DEFAULT_SORT_ORDER) & 0x1; + const U32 sort_order(gSavedSettings.getU32(LLInventoryPanel::DEFAULT_SORT_ORDER) & 0x1); + // *TODO: Think I'd like to get a shared pointer to this and share it + // among all the folder requests. uuid_vec_t recursive_cats; LLSD folder_request_body; @@ -614,27 +574,27 @@ void LLInventoryModelBackgroundFetch::bulkFetch() LLSD item_request_body; LLSD item_request_body_lib; - while (!mFetchQueue.empty() + while (! mFetchQueue.empty() && (item_count + folder_count) < max_batch_size) { - const FetchQueueInfo& fetch_info = mFetchQueue.front(); + const FetchQueueInfo & fetch_info(mFetchQueue.front()); if (fetch_info.mIsCategory) { - const LLUUID &cat_id = fetch_info.mUUID; + const LLUUID & cat_id(fetch_info.mUUID); if (cat_id.isNull()) //DEV-17797 { LLSD folder_sd; folder_sd["folder_id"] = LLUUID::null.asString(); folder_sd["owner_id"] = gAgent.getID(); - folder_sd["sort_order"] = (LLSD::Integer)sort_order; - folder_sd["fetch_folders"] = (LLSD::Boolean)FALSE; - folder_sd["fetch_items"] = (LLSD::Boolean)TRUE; + folder_sd["sort_order"] = LLSD::Integer(sort_order); + folder_sd["fetch_folders"] = LLSD::Boolean(FALSE); + folder_sd["fetch_items"] = LLSD::Boolean(TRUE); folder_request_body["folders"].append(folder_sd); folder_count++; } else { - const LLViewerInventoryCategory* cat = gInventory.getCategory(cat_id); + const LLViewerInventoryCategory * cat(gInventory.getCategory(cat_id)); if (cat) { @@ -643,21 +603,26 @@ void LLInventoryModelBackgroundFetch::bulkFetch() LLSD folder_sd; folder_sd["folder_id"] = cat->getUUID(); folder_sd["owner_id"] = cat->getOwnerID(); - folder_sd["sort_order"] = (LLSD::Integer)sort_order; - folder_sd["fetch_folders"] = TRUE; //(LLSD::Boolean)sFullFetchStarted; - folder_sd["fetch_items"] = (LLSD::Boolean)TRUE; + folder_sd["sort_order"] = LLSD::Integer(sort_order); + folder_sd["fetch_folders"] = LLSD::Boolean(TRUE); //(LLSD::Boolean)sFullFetchStarted; + folder_sd["fetch_items"] = LLSD::Boolean(TRUE); if (ALEXANDRIA_LINDEN_ID == cat->getOwnerID()) + { folder_request_body_lib["folders"].append(folder_sd); + } else + { folder_request_body["folders"].append(folder_sd); + } folder_count++; } + // May already have this folder, but append child folders to list. if (fetch_info.mRecursive) { - LLInventoryModel::cat_array_t* categories; - LLInventoryModel::item_array_t* items; + LLInventoryModel::cat_array_t * categories(NULL); + LLInventoryModel::item_array_t * items(NULL); gInventory.getDirectDescendentsOf(cat->getUUID(), categories, items); for (LLInventoryModel::cat_array_t::const_iterator it = categories->begin(); it != categories->end(); @@ -669,11 +634,14 @@ void LLInventoryModelBackgroundFetch::bulkFetch() } } if (fetch_info.mRecursive) + { recursive_cats.push_back(cat_id); + } } else { - LLViewerInventoryItem* itemp = gInventory.getItem(fetch_info.mUUID); + LLViewerInventoryItem * itemp(gInventory.getItem(fetch_info.mUUID)); + if (itemp) { LLSD item_sd; @@ -694,72 +662,80 @@ void LLInventoryModelBackgroundFetch::bulkFetch() mFetchQueue.pop_front(); } - + + // Issue HTTP POST requests to fetch folders and items + if (item_count + folder_count > 0) { if (folder_count) { - std::string url = region->getCapability("FetchInventoryDescendents2"); - if ( !url.empty() ) + if (folder_request_body["folders"].size()) { - if (folder_request_body["folders"].size()) + const std::string url(region->getCapability("FetchInventoryDescendents2")); + + if (! url.empty()) { - LLInventoryModelFetchDescendentsResponder *fetcher = new LLInventoryModelFetchDescendentsResponder(folder_request_body, recursive_cats); - LLHTTPClient::post(url, folder_request_body, fetcher, 300.0); + BGFolderHttpHandler * handler(new BGFolderHttpHandler(folder_request_body, recursive_cats)); + gInventory.requestPost(false, url, folder_request_body, handler, "Inventory Folder"); } - if (folder_request_body_lib["folders"].size()) - { - std::string url_lib = gAgent.getRegion()->getCapability("FetchLibDescendents2"); + } + + if (folder_request_body_lib["folders"].size()) + { + const std::string url(region->getCapability("FetchLibDescendents2")); - LLInventoryModelFetchDescendentsResponder *fetcher = new LLInventoryModelFetchDescendentsResponder(folder_request_body_lib, recursive_cats); - LLHTTPClient::post(url_lib, folder_request_body_lib, fetcher, 300.0); + if (! url.empty()) + { + BGFolderHttpHandler * handler(new BGFolderHttpHandler(folder_request_body_lib, recursive_cats)); + gInventory.requestPost(false, url, folder_request_body_lib, handler, "Library Folder"); } } - } + } // if (folder_count) + if (item_count) { - std::string url; - if (item_request_body.size()) { - url = region->getCapability("FetchInventory2"); - if (!url.empty()) + const std::string url(region->getCapability("FetchInventory2")); + + if (! url.empty()) { LLSD body; body["items"] = item_request_body; - - LLHTTPClient::post(url, body, new LLInventoryModelFetchItemResponder(body)); + BGItemHttpHandler * handler(new BGItemHttpHandler(body)); + gInventory.requestPost(false, url, body, handler, "Inventory Item"); } } if (item_request_body_lib.size()) { + const std::string url(region->getCapability("FetchLib2")); - url = region->getCapability("FetchLib2"); - if (!url.empty()) + if (! url.empty()) { LLSD body; body["items"] = item_request_body_lib; - - LLHTTPClient::post(url, body, new LLInventoryModelFetchItemResponder(body)); + BGItemHttpHandler * handler(new BGItemHttpHandler(body)); + gInventory.requestPost(false, url, body, handler, "Library Item"); } } - } + } // if (item_count) + mFetchTimer.reset(); } - else if (isBulkFetchProcessingComplete()) { setAllFoldersFetched(); } } -bool LLInventoryModelBackgroundFetch::fetchQueueContainsNoDescendentsOf(const LLUUID& cat_id) const +bool LLInventoryModelBackgroundFetch::fetchQueueContainsNoDescendentsOf(const LLUUID & cat_id) const { for (fetch_queue_t::const_iterator it = mFetchQueue.begin(); - it != mFetchQueue.end(); ++it) + it != mFetchQueue.end(); + ++it) { - const LLUUID& fetch_id = (*it).mUUID; + const LLUUID & fetch_id = (*it).mUUID; if (gInventory.isObjectDescendentOf(fetch_id, cat_id)) return false; } @@ -767,3 +743,304 @@ bool LLInventoryModelBackgroundFetch::fetchQueueContainsNoDescendentsOf(const LL } +namespace +{ + +///---------------------------------------------------------------------------- +/// Class <anonymous>::BGFolderHttpHandler +///---------------------------------------------------------------------------- + +void BGFolderHttpHandler::onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * response) +{ + do // Single-pass do-while used for common exit handling + { + LLCore::HttpStatus status(response->getStatus()); + // status = LLCore::HttpStatus(404); // Dev tool to force error handling + if (! status) + { + processFailure(status, response); + break; // Goto common exit + } + + // Response body should be present. + LLCore::BufferArray * body(response->getBody()); + // body = NULL; // Dev tool to force error handling + if (! body || ! body->size()) + { + LL_WARNS(LOG_INV) << "Missing data in inventory folder query." << LL_ENDL; + processFailure("HTTP response missing expected body", response); + break; // Goto common exit + } + + // Could test 'Content-Type' header but probably unreliable. + + // Convert response to LLSD + // body->write(0, "Garbage Response", 16); // Dev tool to force error handling + LLSD body_llsd; + if (! LLCoreHttpUtil::responseToLLSD(response, true, body_llsd)) + { + // INFOS-level logging will occur on the parsed failure + processFailure("HTTP response contained malformed LLSD", response); + break; // goto common exit + } + + // Expect top-level structure to be a map + // body_llsd = LLSD::emptyArray(); // Dev tool to force error handling + if (! body_llsd.isMap()) + { + processFailure("LLSD response not a map", response); + break; // goto common exit + } + + // Check for 200-with-error failures + // + // See comments in llinventorymodel.cpp about this mode of error. + // + // body_llsd["error"] = LLSD::emptyMap(); // Dev tool to force error handling + // body_llsd["error"]["identifier"] = "Development"; + // body_llsd["error"]["message"] = "You left development code in the viewer"; + if (body_llsd.has("error")) + { + processFailure("Inventory application error (200-with-error)", response); + break; // goto common exit + } + + // Okay, process data if possible + processData(body_llsd, response); + } + while (false); + + // Must delete on completion. + delete this; +} + + +void BGFolderHttpHandler::processData(LLSD & content, LLCore::HttpResponse * response) +{ + LLInventoryModelBackgroundFetch * fetcher(LLInventoryModelBackgroundFetch::getInstance()); + + // API V2 and earlier should probably be testing for "error" map + // in response as an application-level error. + + // Instead, we assume success and attempt to extract information. + if (content.has("folders")) + { + LLSD folders(content["folders"]); + + for (LLSD::array_const_iterator folder_it = folders.beginArray(); + folder_it != folders.endArray(); + ++folder_it) + { + LLSD folder_sd(*folder_it); + + //LLUUID agent_id = folder_sd["agent_id"]; + + //if(agent_id != gAgent.getID()) //This should never happen. + //{ + // LL_WARNS(LOG_INV) << "Got a UpdateInventoryItem for the wrong agent." + // << LL_ENDL; + // break; + //} + + LLUUID parent_id(folder_sd["folder_id"].asUUID()); + LLUUID owner_id(folder_sd["owner_id"].asUUID()); + S32 version(folder_sd["version"].asInteger()); + S32 descendents(folder_sd["descendents"].asInteger()); + LLPointer<LLViewerInventoryCategory> tcategory = new LLViewerInventoryCategory(owner_id); + + if (parent_id.isNull()) + { + LLSD items(folder_sd["items"]); + LLPointer<LLViewerInventoryItem> titem = new LLViewerInventoryItem; + + for (LLSD::array_const_iterator item_it = items.beginArray(); + item_it != items.endArray(); + ++item_it) + { + const LLUUID lost_uuid(gInventory.findCategoryUUIDForType(LLFolderType::FT_LOST_AND_FOUND)); + + if (lost_uuid.notNull()) + { + LLSD item(*item_it); + + titem->unpackMessage(item); + + LLInventoryModel::update_list_t update; + LLInventoryModel::LLCategoryUpdate new_folder(lost_uuid, 1); + update.push_back(new_folder); + gInventory.accountForUpdate(update); + + titem->setParent(lost_uuid); + titem->updateParentOnServer(FALSE); + gInventory.updateItem(titem); + gInventory.notifyObservers(); + } + } + } + + LLViewerInventoryCategory * pcat(gInventory.getCategory(parent_id)); + if (! pcat) + { + continue; + } + + LLSD categories(folder_sd["categories"]); + for (LLSD::array_const_iterator category_it = categories.beginArray(); + category_it != categories.endArray(); + ++category_it) + { + LLSD category(*category_it); + tcategory->fromLLSD(category); + + const bool recursive(getIsRecursive(tcategory->getUUID())); + if (recursive) + { + fetcher->addRequestAtBack(tcategory->getUUID(), recursive, true); + } + else if (! gInventory.isCategoryComplete(tcategory->getUUID())) + { + gInventory.updateCategory(tcategory); + } + } + + LLSD items(folder_sd["items"]); + LLPointer<LLViewerInventoryItem> titem = new LLViewerInventoryItem; + for (LLSD::array_const_iterator item_it = items.beginArray(); + item_it != items.endArray(); + ++item_it) + { + LLSD item(*item_it); + titem->unpackMessage(item); + + gInventory.updateItem(titem); + } + + // Set version and descendentcount according to message. + LLViewerInventoryCategory * cat(gInventory.getCategory(parent_id)); + if (cat) + { + cat->setVersion(version); + cat->setDescendentCount(descendents); + cat->determineFolderType(); + } + } + } + + if (content.has("bad_folders")) + { + LLSD bad_folders(content["bad_folders"]); + for (LLSD::array_const_iterator folder_it = bad_folders.beginArray(); + folder_it != bad_folders.endArray(); + ++folder_it) + { + // *TODO: Stop copying data [ed: this isn't copying data] + LLSD folder_sd(*folder_it); + + // These folders failed on the dataserver. We probably don't want to retry them. + LL_WARNS(LOG_INV) << "Folder " << folder_sd["folder_id"].asString() + << "Error: " << folder_sd["error"].asString() << LL_ENDL; + } + } + + if (fetcher->isBulkFetchProcessingComplete()) + { + fetcher->setAllFoldersFetched(); + } + + gInventory.notifyObservers(); +} + + +void BGFolderHttpHandler::processFailure(LLCore::HttpStatus status, LLCore::HttpResponse * response) +{ + const std::string & ct(response->getContentType()); + LL_WARNS(LOG_INV) << "Inventory folder fetch failure\n" + << "[Status: " << status.toTerseString() << "]\n" + << "[Reason: " << status.toString() << "]\n" + << "[Content-type: " << ct << "]\n" + << "[Content (abridged): " + << LLCoreHttpUtil::responseToString(response) << "]" << LL_ENDL; + + // Could use a 404 test here to try to detect revoked caps... + + // This was originally the request retry logic for the inventory + // request which tested on HTTP_INTERNAL_ERROR status. This + // retry logic was unbounded and lacked discrimination as to the + // cause of the retry. The new http library should be doing + // adquately on retries but I want to keep the structure of a + // retry for reference. + LLInventoryModelBackgroundFetch *fetcher = LLInventoryModelBackgroundFetch::getInstance(); + if (false) + { + // timed out or curl failure + for (LLSD::array_const_iterator folder_it = mRequestSD["folders"].beginArray(); + folder_it != mRequestSD["folders"].endArray(); + ++folder_it) + { + LLSD folder_sd(*folder_it); + LLUUID folder_id(folder_sd["folder_id"].asUUID()); + const BOOL recursive = getIsRecursive(folder_id); + fetcher->addRequestAtFront(folder_id, recursive, true); + } + } + else + { + if (fetcher->isBulkFetchProcessingComplete()) + { + fetcher->setAllFoldersFetched(); + } + } + gInventory.notifyObservers(); +} + + +void BGFolderHttpHandler::processFailure(const char * const reason, LLCore::HttpResponse * response) +{ + LL_WARNS(LOG_INV) << "Inventory folder fetch failure\n" + << "[Status: internal error]\n" + << "[Reason: " << reason << "]\n" + << "[Content (abridged): " + << LLCoreHttpUtil::responseToString(response) << "]" << LL_ENDL; + + // Reverse of previous processFailure() method, this is invoked + // when response structure is found to be invalid. Original + // always re-issued the request (without limit). This does + // the same but be aware that this may be a source of problems. + // Philosophy is that inventory folders are so essential to + // operation that this is a reasonable action. + LLInventoryModelBackgroundFetch *fetcher = LLInventoryModelBackgroundFetch::getInstance(); + if (true) + { + for (LLSD::array_const_iterator folder_it = mRequestSD["folders"].beginArray(); + folder_it != mRequestSD["folders"].endArray(); + ++folder_it) + { + LLSD folder_sd(*folder_it); + LLUUID folder_id(folder_sd["folder_id"].asUUID()); + const BOOL recursive = getIsRecursive(folder_id); + fetcher->addRequestAtFront(folder_id, recursive, true); + } + } + else + { + if (fetcher->isBulkFetchProcessingComplete()) + { + fetcher->setAllFoldersFetched(); + } + } + gInventory.notifyObservers(); +} + + +bool BGFolderHttpHandler::getIsRecursive(const LLUUID & cat_id) const +{ + return std::find(mRecursiveCatUUIDs.begin(), mRecursiveCatUUIDs.end(), cat_id) != mRecursiveCatUUIDs.end(); +} + +///---------------------------------------------------------------------------- +/// Class <anonymous>::BGItemHttpHandler +///---------------------------------------------------------------------------- + +// Nothing to implement here. All ctor/dtor changes. + +} // end namespace anonymous diff --git a/indra/newview/llinventorymodelbackgroundfetch.h b/indra/newview/llinventorymodelbackgroundfetch.h index 9dfedddd6d..2139f85519 100755 --- a/indra/newview/llinventorymodelbackgroundfetch.h +++ b/indra/newview/llinventorymodelbackgroundfetch.h @@ -29,6 +29,11 @@ #include "llsingleton.h" #include "lluuid.h" +#include "httpcommon.h" +#include "httprequest.h" +#include "httpoptions.h" +#include "httpheaders.h" +#include "httphandler.h" //~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ // Class LLInventoryModelBackgroundFetch @@ -38,8 +43,6 @@ //~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ class LLInventoryModelBackgroundFetch : public LLSingleton<LLInventoryModelBackgroundFetch> { - friend class LLInventoryModelFetchDescendentsResponder; - public: LLInventoryModelBackgroundFetch(); ~LLInventoryModelBackgroundFetch(); @@ -60,16 +63,22 @@ public: bool inventoryFetchInProgress() const; void findLostItems(); - void incrFetchCount(S16 fetching); -protected: + void incrFetchCount(S32 fetching); + bool isBulkFetchProcessingComplete() const; + void setAllFoldersFetched(); + + void addRequestAtFront(const LLUUID & id, BOOL recursive, bool is_category); + void addRequestAtBack(const LLUUID & id, BOOL recursive, bool is_category); + +protected: void bulkFetch(); void backgroundFetch(); static void backgroundFetchCB(void*); // background fetch idle function - void setAllFoldersFetched(); bool fetchQueueContainsNoDescendentsOf(const LLUUID& cat_id) const; + private: BOOL mRecursiveInventoryFetchStarted; BOOL mRecursiveLibraryFetchStarted; @@ -77,7 +86,7 @@ private: BOOL mBackgroundFetchActive; bool mFolderFetchActive; - S16 mFetchCount; + S32 mFetchCount; BOOL mTimelyFetchPending; S32 mNumFetchRetries; @@ -87,9 +96,12 @@ private: struct FetchQueueInfo { - FetchQueueInfo(const LLUUID& id, BOOL recursive, bool is_category = true) : - mUUID(id), mRecursive(recursive), mIsCategory(is_category) + FetchQueueInfo(const LLUUID& id, BOOL recursive, bool is_category = true) + : mUUID(id), + mIsCategory(is_category), + mRecursive(recursive) {} + LLUUID mUUID; bool mIsCategory; BOOL mRecursive; diff --git a/indra/newview/llinventoryobserver.cpp b/indra/newview/llinventoryobserver.cpp index 2dd8dce42f..d81401b59b 100755 --- a/indra/newview/llinventoryobserver.cpp +++ b/indra/newview/llinventoryobserver.cpp @@ -237,7 +237,8 @@ void fetch_items_from_llsd(const LLSD& items_llsd) if (!url.empty()) { body[i]["agent_id"] = gAgent.getID(); - LLHTTPClient::post(url, body[i], new LLInventoryModel::fetchInventoryResponder(body[i])); + LLInventoryModel::FetchItemHttpHandler * handler(new LLInventoryModel::FetchItemHttpHandler(body[i])); + gInventory.requestPost(true, url, body[i], handler, (i ? "Library Item" : "Inventory Item")); continue; } diff --git a/indra/newview/llmeshrepository.cpp b/indra/newview/llmeshrepository.cpp index 8f50555a73..648056484e 100755 --- a/indra/newview/llmeshrepository.cpp +++ b/indra/newview/llmeshrepository.cpp @@ -1,4 +1,3 @@ - /** * @file llmeshrepository.cpp * @brief Mesh repository implementation. @@ -72,6 +71,7 @@ #include "bufferarray.h" #include "bufferstream.h" #include "llfasttimer.h" +#include "llcorehttputil.h" #include "boost/lexical_cast.hpp" @@ -338,14 +338,17 @@ static LLFastTimer::DeclareTimer FTM_MESH_FETCH("Mesh Fetch"); LLMeshRepository gMeshRepo; const S32 MESH_HEADER_SIZE = 4096; // Important: assumption is that headers fit in this space + const S32 REQUEST_HIGH_WATER_MIN = 32; // Limits for GetMesh regions const S32 REQUEST_HIGH_WATER_MAX = 150; // Should remain under 2X throttle const S32 REQUEST_LOW_WATER_MIN = 16; const S32 REQUEST_LOW_WATER_MAX = 75; + const S32 REQUEST2_HIGH_WATER_MIN = 32; // Limits for GetMesh2 regions -const S32 REQUEST2_HIGH_WATER_MAX = 80; +const S32 REQUEST2_HIGH_WATER_MAX = 100; const S32 REQUEST2_LOW_WATER_MIN = 16; -const S32 REQUEST2_LOW_WATER_MAX = 40; +const S32 REQUEST2_LOW_WATER_MAX = 50; + const U32 LARGE_MESH_FETCH_THRESHOLD = 1U << 21; // Size at which requests goes to narrow/slow queue const long SMALL_MESH_XFER_TIMEOUT = 120L; // Seconds to complete xfer, small mesh downloads const long LARGE_MESH_XFER_TIMEOUT = 600L; // Seconds to complete xfer, large downloads @@ -518,11 +521,13 @@ class LLMeshHandlerBase : public LLCore::HttpHandler { public: LOG_CLASS(LLMeshHandlerBase); - LLMeshHandlerBase() + LLMeshHandlerBase(U32 offset, U32 requested_bytes) : LLCore::HttpHandler(), mMeshParams(), mProcessed(false), - mHttpHandle(LLCORE_HTTP_HANDLE_INVALID) + mHttpHandle(LLCORE_HTTP_HANDLE_INVALID), + mOffset(offset), + mRequestedBytes(requested_bytes) {} virtual ~LLMeshHandlerBase() @@ -534,13 +539,15 @@ protected: public: virtual void onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * response); - virtual void processData(LLCore::BufferArray * body, U8 * data, S32 data_size) = 0; + virtual void processData(LLCore::BufferArray * body, S32 body_offset, U8 * data, S32 data_size) = 0; virtual void processFailure(LLCore::HttpStatus status) = 0; public: LLVolumeParams mMeshParams; bool mProcessed; - LLCore::HttpHandle mHttpHandle; + LLCore::HttpHandle mHttpHandle; + U32 mOffset; + U32 mRequestedBytes; }; @@ -551,8 +558,8 @@ class LLMeshHeaderHandler : public LLMeshHandlerBase { public: LOG_CLASS(LLMeshHeaderHandler); - LLMeshHeaderHandler(const LLVolumeParams & mesh_params) - : LLMeshHandlerBase() + LLMeshHeaderHandler(const LLVolumeParams & mesh_params, U32 offset, U32 requested_bytes) + : LLMeshHandlerBase(offset, requested_bytes) { mMeshParams = mesh_params; LLMeshRepoThread::incActiveHeaderRequests(); @@ -564,7 +571,7 @@ protected: void operator=(const LLMeshHeaderHandler &); // Not defined public: - virtual void processData(LLCore::BufferArray * body, U8 * data, S32 data_size); + virtual void processData(LLCore::BufferArray * body, S32 body_offset, U8 * data, S32 data_size); virtual void processFailure(LLCore::HttpStatus status); }; @@ -573,17 +580,16 @@ public: // // Thread: repo class LLMeshLODHandler : public LLMeshHandlerBase - { +{ public: + LOG_CLASS(LLMeshLODHandler); LLMeshLODHandler(const LLVolumeParams & mesh_params, S32 lod, U32 offset, U32 requested_bytes) - : LLMeshHandlerBase(), - mLOD(lod), - mRequestedBytes(requested_bytes), - mOffset(offset) + : LLMeshHandlerBase(offset, requested_bytes), + mLOD(lod) { - mMeshParams = mesh_params; - LLMeshRepoThread::incActiveLODRequests(); - } + mMeshParams = mesh_params; + LLMeshRepoThread::incActiveLODRequests(); + } virtual ~LLMeshLODHandler(); protected: @@ -591,13 +597,11 @@ protected: void operator=(const LLMeshLODHandler &); // Not defined public: - virtual void processData(LLCore::BufferArray * body, U8 * data, S32 data_size); + virtual void processData(LLCore::BufferArray * body, S32 body_offset, U8 * data, S32 data_size); virtual void processFailure(LLCore::HttpStatus status); public: S32 mLOD; - U32 mRequestedBytes; - U32 mOffset; }; @@ -605,14 +609,12 @@ public: // // Thread: repo class LLMeshSkinInfoHandler : public LLMeshHandlerBase - { +{ public: LOG_CLASS(LLMeshSkinInfoHandler); - LLMeshSkinInfoHandler(const LLUUID& id, U32 offset, U32 size) - : LLMeshHandlerBase(), - mMeshID(id), - mRequestedBytes(size), - mOffset(offset) + LLMeshSkinInfoHandler(const LLUUID& id, U32 offset, U32 requested_bytes) + : LLMeshHandlerBase(offset, requested_bytes), + mMeshID(id) {} virtual ~LLMeshSkinInfoHandler(); @@ -621,13 +623,11 @@ protected: void operator=(const LLMeshSkinInfoHandler &); // Not defined public: - virtual void processData(LLCore::BufferArray * body, U8 * data, S32 data_size); + virtual void processData(LLCore::BufferArray * body, S32 body_offset, U8 * data, S32 data_size); virtual void processFailure(LLCore::HttpStatus status); public: LLUUID mMeshID; - U32 mRequestedBytes; - U32 mOffset; }; @@ -635,14 +635,12 @@ public: // // Thread: repo class LLMeshDecompositionHandler : public LLMeshHandlerBase - { +{ public: LOG_CLASS(LLMeshDecompositionHandler); - LLMeshDecompositionHandler(const LLUUID& id, U32 offset, U32 size) - : LLMeshHandlerBase(), - mMeshID(id), - mRequestedBytes(size), - mOffset(offset) + LLMeshDecompositionHandler(const LLUUID& id, U32 offset, U32 requested_bytes) + : LLMeshHandlerBase(offset, requested_bytes), + mMeshID(id) {} virtual ~LLMeshDecompositionHandler(); @@ -651,13 +649,11 @@ protected: void operator=(const LLMeshDecompositionHandler &); // Not defined public: - virtual void processData(LLCore::BufferArray * body, U8 * data, S32 data_size); + virtual void processData(LLCore::BufferArray * body, S32 body_offset, U8 * data, S32 data_size); virtual void processFailure(LLCore::HttpStatus status); public: LLUUID mMeshID; - U32 mRequestedBytes; - U32 mOffset; }; @@ -665,14 +661,12 @@ public: // // Thread: repo class LLMeshPhysicsShapeHandler : public LLMeshHandlerBase - { +{ public: LOG_CLASS(LLMeshPhysicsShapeHandler); - LLMeshPhysicsShapeHandler(const LLUUID& id, U32 offset, U32 size) - : LLMeshHandlerBase(), - mMeshID(id), - mRequestedBytes(size), - mOffset(offset) + LLMeshPhysicsShapeHandler(const LLUUID& id, U32 offset, U32 requested_bytes) + : LLMeshHandlerBase(offset, requested_bytes), + mMeshID(id) {} virtual ~LLMeshPhysicsShapeHandler(); @@ -681,13 +675,11 @@ protected: void operator=(const LLMeshPhysicsShapeHandler &); // Not defined public: - virtual void processData(LLCore::BufferArray * body, U8 * data, S32 data_size); + virtual void processData(LLCore::BufferArray * body, S32 body_offset, U8 * data, S32 data_size); virtual void processFailure(LLCore::HttpStatus status); public: LLUUID mMeshID; - U32 mRequestedBytes; - U32 mOffset; }; @@ -713,8 +705,8 @@ void log_upload_error(LLCore::HttpStatus status, const LLSD& content, LL_WARNS(LOG_MESH) << "error: " << err << LL_ENDL; LL_WARNS(LOG_MESH) << " mesh upload failed, stage '" << stage << "', error '" << err["error"].asString() - << "', message '" << err["message"].asString() - << "', id '" << err["identifier"].asString() + << "', message '" << err["message"].asString() + << "', id '" << err["identifier"].asString() << "'" << LL_ENDL; if (err.has("errors")) { @@ -754,7 +746,9 @@ LLMeshRepoThread::LLMeshRepoThread() mHttpLargePolicyClass(LLCore::HttpRequest::DEFAULT_POLICY_ID), mHttpPriority(0), mGetMeshVersion(2) - { +{ + LLAppCoreHttp & app_core_http(LLAppViewer::instance()->getAppCoreHttp()); + mMutex = new LLMutex(NULL); mHeaderMutex = new LLMutex(NULL); mSignal = new LLCondition(NULL); @@ -766,15 +760,15 @@ LLMeshRepoThread::LLMeshRepoThread() mHttpLargeOptions->setTransferTimeout(LARGE_MESH_XFER_TIMEOUT); mHttpLargeOptions->setUseRetryAfter(gSavedSettings.getBOOL("MeshUseHttpRetryAfter")); mHttpHeaders = new LLCore::HttpHeaders; - mHttpHeaders->append("Accept", "application/vnd.ll.mesh"); - mHttpPolicyClass = LLAppViewer::instance()->getAppCoreHttp().getPolicy(LLAppCoreHttp::AP_MESH2); - mHttpLegacyPolicyClass = LLAppViewer::instance()->getAppCoreHttp().getPolicy(LLAppCoreHttp::AP_MESH1); - mHttpLargePolicyClass = LLAppViewer::instance()->getAppCoreHttp().getPolicy(LLAppCoreHttp::AP_LARGE_MESH); - } + mHttpHeaders->append(HTTP_OUT_HEADER_ACCEPT, HTTP_CONTENT_VND_LL_MESH); + mHttpPolicyClass = app_core_http.getPolicy(LLAppCoreHttp::AP_MESH2); + mHttpLegacyPolicyClass = app_core_http.getPolicy(LLAppCoreHttp::AP_MESH1); + mHttpLargePolicyClass = app_core_http.getPolicy(LLAppCoreHttp::AP_LARGE_MESH); +} LLMeshRepoThread::~LLMeshRepoThread() - { +{ LL_INFOS(LOG_MESH) << "Small GETs issued: " << LLMeshRepository::sHTTPRequestCount << ", Large GETs issued: " << LLMeshRepository::sHTTPLargeRequestCount << ", Max Lock Holdoffs: " << LLMeshRepository::sMaxLockHoldoffs @@ -785,23 +779,23 @@ LLMeshRepoThread::~LLMeshRepoThread() ++iter) { delete *iter; - } + } mHttpRequestSet.clear(); if (mHttpHeaders) - { + { mHttpHeaders->release(); mHttpHeaders = NULL; - } + } if (mHttpOptions) - { + { mHttpOptions->release(); mHttpOptions = NULL; - } + } if (mHttpLargeOptions) -{ + { mHttpLargeOptions->release(); mHttpLargeOptions = NULL; -} + } delete mHttpRequest; mHttpRequest = NULL; delete mMutex; @@ -846,48 +840,49 @@ void LLMeshRepoThread::run() { // Dispatch all HttpHandler notifications mHttpRequest->update(0L); - } + } sRequestWaterLevel = mHttpRequestSet.size(); // Stats data update // NOTE: order of queue processing intentionally favors LOD requests over header requests while (!mLODReqQ.empty() && mHttpRequestSet.size() < sRequestHighWater) - { + { if (! mMutex) - { + { break; } - mMutex->lock(); - LODRequest req = mLODReqQ.front(); - mLODReqQ.pop(); - LLMeshRepository::sLODProcessing--; - mMutex->unlock(); + mMutex->lock(); + LODRequest req = mLODReqQ.front(); + mLODReqQ.pop(); + LLMeshRepository::sLODProcessing--; + mMutex->unlock(); + if (!fetchMeshLOD(req.mMeshParams, req.mLOD)) // failed, resubmit - { - mMutex->lock(); - mLODReqQ.push(req); + { + mMutex->lock(); + mLODReqQ.push(req); ++LLMeshRepository::sLODProcessing; - mMutex->unlock(); - } - } + mMutex->unlock(); + } + } while (!mHeaderReqQ.empty() && mHttpRequestSet.size() < sRequestHighWater) - { + { if (! mMutex) - { + { break; } - mMutex->lock(); - HeaderRequest req = mHeaderReqQ.front(); - mHeaderReqQ.pop(); - mMutex->unlock(); + mMutex->lock(); + HeaderRequest req = mHeaderReqQ.front(); + mHeaderReqQ.pop(); + mMutex->unlock(); if (!fetchMeshHeader(req.mMeshParams))//failed, resubmit - { - mMutex->lock(); - mHeaderReqQ.push(req) ; - mMutex->unlock(); - } - } + { + mMutex->lock(); + mHeaderReqQ.push(req) ; + mMutex->unlock(); + } + } // For the final three request lists, similar goal to above but // slightly different queue structures. Stay off the mutex when @@ -983,7 +978,7 @@ void LLMeshRepoThread::run() } } mMutex->unlock(); - } + } // For dev purposes only. A dynamic change could make this false // and that shouldn't assert. @@ -1131,6 +1126,9 @@ LLCore::HttpHandle LLMeshRepoThread::getByteRange(const std::string & url, int c size_t offset, size_t len, LLCore::HttpHandler * handler) { + // Also used in lltexturefetch.cpp + static LLCachedControl<bool> disable_range_req(gSavedSettings, "HttpRangeRequestsDisable", false); + LLCore::HttpHandle handle(LLCORE_HTTP_HANDLE_INVALID); if (len < LARGE_MESH_FETCH_THRESHOLD) @@ -1140,8 +1138,8 @@ LLCore::HttpHandle LLMeshRepoThread::getByteRange(const std::string & url, int c : mHttpLegacyPolicyClass), mHttpPriority, url, - offset, - len, + (disable_range_req ? size_t(0) : offset), + (disable_range_req ? size_t(0) : len), mHttpOptions, mHttpHeaders, handler); @@ -1155,8 +1153,8 @@ LLCore::HttpHandle LLMeshRepoThread::getByteRange(const std::string & url, int c handle = mHttpRequest->requestGetByteRange(mHttpLargePolicyClass, mHttpPriority, url, - offset, - len, + (disable_range_req ? size_t(0) : offset), + (disable_range_req ? size_t(0) : len), mHttpLargeOptions, mHttpHeaders, handler); @@ -1250,7 +1248,6 @@ bool LLMeshRepoThread::fetchMeshSkinInfo(const LLUUID& mesh_id) << LL_ENDL; delete handler; ret = false; - } else { @@ -1527,7 +1524,7 @@ bool LLMeshRepoThread::fetchMeshHeader(const LLVolumeParams& mesh_params) //within the first 4KB //NOTE -- this will break of headers ever exceed 4KB - LLMeshHeaderHandler * handler = new LLMeshHeaderHandler(mesh_params); + LLMeshHeaderHandler * handler = new LLMeshHeaderHandler(mesh_params, 0, MESH_HEADER_SIZE); LLCore::HttpHandle handle = getByteRange(http_url, cap_version, 0, MESH_HEADER_SIZE, handler); if (LLCORE_HTTP_HANDLE_INVALID == handle) { @@ -1860,7 +1857,7 @@ LLMeshUploadThread::LLMeshUploadThread(LLMeshUploadThread::instance_list& data, bool upload_skin, bool upload_joints, const std::string & upload_url, bool do_upload, LLHandle<LLWholeModelFeeObserver> fee_observer, LLHandle<LLWholeModelUploadObserver> upload_observer) -: LLThread("mesh upload"), + : LLThread("mesh upload"), LLCore::HttpHandler(), mDiscarded(false), mDoUpload(do_upload), @@ -1890,7 +1887,7 @@ LLMeshUploadThread::LLMeshUploadThread(LLMeshUploadThread::instance_list& data, mHttpOptions->setUseRetryAfter(gSavedSettings.getBOOL("MeshUseHttpRetryAfter")); mHttpOptions->setRetries(UPLOAD_RETRY_LIMIT); mHttpHeaders = new LLCore::HttpHeaders; - mHttpHeaders->append("Content-Type", "application/llsd+xml"); + mHttpHeaders->append(HTTP_OUT_HEADER_CONTENT_TYPE, HTTP_CONTENT_LLSD_XML); mHttpPolicyClass = LLAppViewer::instance()->getAppCoreHttp().getPolicy(LLAppCoreHttp::AP_UPLOADS); mHttpPriority = 0; } @@ -2240,21 +2237,17 @@ void LLMeshUploadThread::doWholeModelUpload() mModelData = LLSD::emptyMap(); wholeModelToLLSD(mModelData, true); LLSD body = mModelData["asset_resources"]; - dump_llsd_to_file(body,make_dump_name("whole_model_body_",dump_num)); - - LLCore::BufferArray * ba = new LLCore::BufferArray; - LLCore::BufferArrayStream bas(ba); - LLSDSerialize::toXML(body, bas); - // LLSDSerialize::toXML(mModelData, bas); // <- Enabling this will generate a convenient upload error - LLCore::HttpHandle handle = mHttpRequest->requestPost(mHttpPolicyClass, - mHttpPriority, - mWholeModelUploadURL, - ba, - mHttpOptions, - mHttpHeaders, - this); - ba->release(); - + + dump_llsd_to_file(body, make_dump_name("whole_model_body_", dump_num)); + + LLCore::HttpHandle handle = LLCoreHttpUtil::requestPostWithLLSD(mHttpRequest, + mHttpPolicyClass, + mHttpPriority, + mWholeModelUploadURL, + body, + mHttpOptions, + mHttpHeaders, + this); if (LLCORE_HTTP_HANDLE_INVALID == handle) { mHttpStatus = mHttpRequest->getStatus(); @@ -2271,7 +2264,7 @@ void LLMeshUploadThread::doWholeModelUpload() mHttpRequest->update(0); while (! LLApp::isQuitting() && ! finished() && ! isDiscarded()) - { + { ms_sleep(sleep_time); sleep_time = llmin(250U, sleep_time + sleep_time); mHttpRequest->update(0); @@ -2287,7 +2280,7 @@ void LLMeshUploadThread::doWholeModelUpload() } } } - } +} void LLMeshUploadThread::requestWholeModelFee() { @@ -2298,19 +2291,14 @@ void LLMeshUploadThread::requestWholeModelFee() mModelData = LLSD::emptyMap(); wholeModelToLLSD(mModelData, false); dump_llsd_to_file(mModelData, make_dump_name("whole_model_fee_request_", dump_num)); - - LLCore::BufferArray * ba = new LLCore::BufferArray; - LLCore::BufferArrayStream bas(ba); - LLSDSerialize::toXML(mModelData, bas); - - LLCore::HttpHandle handle = mHttpRequest->requestPost(mHttpPolicyClass, - mHttpPriority, - mWholeModelFeeCapability, - ba, - mHttpOptions, - mHttpHeaders, - this); - ba->release(); + LLCore::HttpHandle handle = LLCoreHttpUtil::requestPostWithLLSD(mHttpRequest, + mHttpPolicyClass, + mHttpPriority, + mWholeModelFeeCapability, + mModelData, + mHttpOptions, + mHttpHeaders, + this); if (LLCORE_HTTP_HANDLE_INVALID == handle) { mHttpStatus = mHttpRequest->getStatus(); @@ -2318,7 +2306,7 @@ void LLMeshUploadThread::requestWholeModelFee() LL_WARNS(LOG_MESH) << "Couldn't issue request for model fee. Reason: " << mHttpStatus.toString() << " (" << mHttpStatus.toTerseString() << ")" << LL_ENDL; - } + } else { U32 sleep_time(10); @@ -2335,7 +2323,7 @@ void LLMeshUploadThread::requestWholeModelFee() LL_DEBUGS(LOG_MESH) << "Mesh fee query operation discarded." << LL_ENDL; } } - } +} // Does completion duty for both fee queries and actual uploads. @@ -2383,17 +2371,13 @@ void LLMeshUploadThread::onCompleted(LLCore::HttpHandle handle, LLCore::HttpResp } else { - LLCore::BufferArray * ba(response->getBody()); - if (ba && ba->size()) - { - LLCore::BufferArrayStream bas(ba); - LLSDSerialize::fromXML(body, bas); -} + // *TODO: handle error in conversion process + LLCoreHttpUtil::responseToLLSD(response, true, body); } dump_llsd_to_file(body, make_dump_name("whole_model_upload_response_", dump_num)); if (body["state"].asString() == "complete") -{ + { // requested "mesh" asset type isn't actually the type // of the resultant object, fix it up here. mModelData["asset_type"] = "object"; @@ -2446,18 +2430,14 @@ void LLMeshUploadThread::onCompleted(LLCore::HttpHandle handle, LLCore::HttpResp body = llsd_from_file("fake_upload_error.xml"); } else - { - LLCore::BufferArray * ba(response->getBody()); - if (ba && ba->size()) - { - LLCore::BufferArrayStream bas(ba); - LLSDSerialize::fromXML(body, bas); - } - } + { + // *TODO: handle error in conversion process + LLCoreHttpUtil::responseToLLSD(response, true, body); + } dump_llsd_to_file(body, make_dump_name("whole_model_fee_response_", dump_num)); if (body["state"].asString() == "upload") - { + { mWholeModelUploadURL = body["uploader"].asString(); if (observer) @@ -2543,18 +2523,18 @@ void LLMeshRepoThread::notifyLoadedMeshes() skin_info_q.swap(mSkinInfoQ); } if (! mDecompositionQ.empty()) - { + { decomp_q.swap(mDecompositionQ); - } + } mMutex->unlock(); // Process the elements free of the lock while (! skin_info_q.empty()) - { + { gMeshRepo.notifySkinInfoReceived(skin_info_q.front()); skin_info_q.pop_front(); - } + } while (! decomp_q.empty()) { @@ -2648,6 +2628,17 @@ void LLMeshRepository::cacheOutgoingMesh(LLMeshUploadData& data, LLSD& header) } +// Handle failed or successful requests for mesh assets. +// +// Support for 200 responses was added for several reasons. One, +// a service or cache can ignore range headers and give us a +// 200 with full asset should it elect to. We also support +// a debug flag which disables range requests for those very +// few users that have some sort of problem with their networking +// services. But the 200 response handling is suboptimal: rather +// than cache the whole asset, we just extract the part that would +// have been sent in a 206 and process that. Inefficient but these +// are cases far off the norm. void LLMeshHandlerBase::onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * response) { mProcessed = true; @@ -2676,35 +2667,78 @@ void LLMeshHandlerBase::onCompleted(LLCore::HttpHandle handle, LLCore::HttpRespo // rather than partial) and 416 (request completely unsatisfyable). // Always been exposed to these but are less likely here where // speculative loads aren't done. - static const LLCore::HttpStatus par_status(HTTP_PARTIAL_CONTENT); + LLCore::BufferArray * body(response->getBody()); + S32 body_offset(0); + U8 * data(NULL); + S32 data_size(body ? body->size() : 0); - if (par_status != status) + if (data_size > 0) { - LL_WARNS_ONCE(LOG_MESH) << "Non-206 successful status received for fetch: " - << status.toTerseString() << LL_ENDL; - } + static const LLCore::HttpStatus par_status(HTTP_PARTIAL_CONTENT); + + unsigned int offset(0), length(0), full_length(0); + + if (par_status == status) + { + // 206 case + response->getRange(&offset, &length, &full_length); + if (! offset && ! length) + { + // This is the case where we receive a 206 status but + // there wasn't a useful Content-Range header in the response. + // This could be because it was badly formatted but is more + // likely due to capabilities services which scrub headers + // from responses. Assume we got what we asked for...` + // length = data_size; + offset = mOffset; + } + } + else + { + // 200 case, typically + offset = 0; + } - LLCore::BufferArray * body(response->getBody()); - S32 data_size(body ? body->size() : 0); - U8 * data(NULL); + // *DEBUG: To test validation below + // offset += 1; - if (data_size > 0) - { + // Validate that what we think we received is consistent with + // what we've asked for. I.e. first byte we wanted lies somewhere + // in the response. + if (offset > mOffset + || (offset + data_size) <= mOffset + || (mOffset - offset) >= data_size) + { + // No overlap with requested range. Fail request with + // suitable error. Shouldn't happen unless server/cache/ISP + // is doing something awful. + LL_WARNS(LOG_MESH) << "Mesh response (bytes [" + << offset << ".." << (offset + length - 1) + << "]) didn't overlap with request's origin (bytes [" + << mOffset << ".." << (mOffset + mRequestedBytes - 1) + << "])." << LL_ENDL; + processFailure(LLCore::HttpStatus(LLCore::HttpStatus::LLCORE, LLCore::HE_INV_CONTENT_RANGE_HDR)); + ++LLMeshRepository::sHTTPErrorCount; + goto common_exit; + } + // *TODO: Try to get rid of data copying and add interfaces // that support BufferArray directly. Introduce a two-phase // handler, optional first that takes a body, fallback second // that requires a temporary allocation and data copy. - data = new U8[data_size]; - body->read(0, (char *) data, data_size); + body_offset = mOffset - offset; + data = new U8[data_size - body_offset]; + body->read(body_offset, (char *) data, data_size - body_offset); LLMeshRepository::sBytesReceived += data_size; } - processData(body, data, data_size); + processData(body, body_offset, data, data_size - body_offset); delete [] data; } // Release handler +common_exit: gMeshRepo.mThread->mHttpRequestSet.erase(this); delete this; // Must be last statement } @@ -2739,9 +2773,10 @@ void LLMeshHeaderHandler::processFailure(LLCore::HttpStatus status) { gMeshRepo.mThread->mUnavailableQ.push(LLMeshRepoThread::LODRequest(mMeshParams, i)); } - } +} -void LLMeshHeaderHandler::processData(LLCore::BufferArray * body, U8 * data, S32 data_size) +void LLMeshHeaderHandler::processData(LLCore::BufferArray * /* body */, S32 /* body_offset */, + U8 * data, S32 data_size) { LLUUID mesh_id = mMeshParams.getSculptID(); bool success = (! MESH_HEADER_PROCESS_FAILED) && gMeshRepo.mThread->headerReceived(mMeshParams, data, data_size); @@ -2756,12 +2791,12 @@ void LLMeshHeaderHandler::processData(LLCore::BufferArray * body, U8 * data, S32 // Can't get the header so none of the LODs will be available LLMutexLock lock(gMeshRepo.mThread->mMutex); for (int i(0); i < 4; ++i) - { + { gMeshRepo.mThread->mUnavailableQ.push(LLMeshRepoThread::LODRequest(mMeshParams, i)); - } } + } else if (data && data_size > 0) - { + { // header was successfully retrieved from sim, cache in vfs LLSD header = gMeshRepo.mThread->mMeshHeader[mesh_id]; @@ -2774,11 +2809,11 @@ void LLMeshHeaderHandler::processData(LLCore::BufferArray * body, U8 * data, S32 S32 lod_bytes = 0; for (U32 i = 0; i < LLModel::LOD_PHYSICS; ++i) - { + { // figure out how many bytes we'll need to reserve in the file const std::string & lod_name = header_lod[i]; lod_bytes = llmax(lod_bytes, header[lod_name]["offset"].asInteger()+header[lod_name]["size"].asInteger()); - } + } // just in case skin info or decomposition is at the end of the file (which it shouldn't be) lod_bytes = llmax(lod_bytes, header["skin"]["offset"].asInteger() + header["skin"]["size"].asInteger()); @@ -2794,7 +2829,7 @@ void LLMeshHeaderHandler::processData(LLCore::BufferArray * body, U8 * data, S32 LLVFile file(gVFS, mesh_id, LLAssetType::AT_MESH, LLVFile::WRITE); if (file.getMaxSize() >= bytes || file.setMaxSize(bytes)) - { + { LLMeshRepository::sCacheBytesWritten += data_size; ++LLMeshRepository::sCacheWrites; @@ -2805,19 +2840,19 @@ void LLMeshHeaderHandler::processData(LLCore::BufferArray * body, U8 * data, S32 memset(block, 0, sizeof(block)); while (bytes-file.tell() > sizeof(block)) - { + { file.write(block, sizeof(block)); - } + } S32 remaining = bytes-file.tell(); if (remaining > 0) - { + { file.write(block, remaining); } } } } - } +} LLMeshLODHandler::~LLMeshLODHandler() { @@ -2843,8 +2878,9 @@ void LLMeshLODHandler::processFailure(LLCore::HttpStatus status) gMeshRepo.mThread->mUnavailableQ.push(LLMeshRepoThread::LODRequest(mMeshParams, mLOD)); } -void LLMeshLODHandler::processData(LLCore::BufferArray * body, U8 * data, S32 data_size) - { +void LLMeshLODHandler::processData(LLCore::BufferArray * /* body */, S32 /* body_offset */, + U8 * data, S32 data_size) +{ if ((! MESH_LOD_PROCESS_FAILED) && gMeshRepo.mThread->lodReceived(mMeshParams, mLOD, data, data_size)) { //good fetch from sim, write to VFS for caching @@ -2860,7 +2896,7 @@ void LLMeshLODHandler::processData(LLCore::BufferArray * body, U8 * data, S32 da LLMeshRepository::sCacheBytesWritten += size; ++LLMeshRepository::sCacheWrites; } - } + } else { LL_WARNS(LOG_MESH) << "Error during mesh LOD processing. ID: " << mMeshParams.getSculptID() @@ -2872,12 +2908,12 @@ void LLMeshLODHandler::processData(LLCore::BufferArray * body, U8 * data, S32 da } LLMeshSkinInfoHandler::~LLMeshSkinInfoHandler() - { - llassert(mProcessed); - } +{ + llassert(mProcessed); +} void LLMeshSkinInfoHandler::processFailure(LLCore::HttpStatus status) - { +{ LL_WARNS(LOG_MESH) << "Error during mesh skin info handling. ID: " << mMeshID << ", Reason: " << status.toString() << " (" << status.toTerseString() << "). Not retrying." @@ -2885,10 +2921,11 @@ void LLMeshSkinInfoHandler::processFailure(LLCore::HttpStatus status) // *TODO: Mark mesh unavailable on error. For now, simply leave // request unfulfilled rather than retry forever. - } +} -void LLMeshSkinInfoHandler::processData(LLCore::BufferArray * body, U8 * data, S32 data_size) - { +void LLMeshSkinInfoHandler::processData(LLCore::BufferArray * /* body */, S32 /* body_offset */, + U8 * data, S32 data_size) +{ if ((! MESH_SKIN_INFO_PROCESS_FAILED) && gMeshRepo.mThread->skinInfoReceived(mMeshID, data, data_size)) { //good fetch from sim, write to VFS for caching @@ -2916,20 +2953,21 @@ void LLMeshSkinInfoHandler::processData(LLCore::BufferArray * body, U8 * data, S LLMeshDecompositionHandler::~LLMeshDecompositionHandler() { - llassert(mProcessed); + llassert(mProcessed); } void LLMeshDecompositionHandler::processFailure(LLCore::HttpStatus status) - { +{ LL_WARNS(LOG_MESH) << "Error during mesh decomposition handling. ID: " << mMeshID << ", Reason: " << status.toString() << " (" << status.toTerseString() << "). Not retrying." << LL_ENDL; // *TODO: Mark mesh unavailable on error. For now, simply leave // request unfulfilled rather than retry forever. - } +} -void LLMeshDecompositionHandler::processData(LLCore::BufferArray * body, U8 * data, S32 data_size) +void LLMeshDecompositionHandler::processData(LLCore::BufferArray * /* body */, S32 /* body_offset */, + U8 * data, S32 data_size) { if ((! MESH_DECOMP_PROCESS_FAILED) && gMeshRepo.mThread->decompositionReceived(mMeshID, data, data_size)) { @@ -2946,34 +2984,35 @@ void LLMeshDecompositionHandler::processData(LLCore::BufferArray * body, U8 * da file.seek(offset); file.write(data, size); } - } - else - { + } + else + { LL_WARNS(LOG_MESH) << "Error during mesh decomposition processing. ID: " << mMeshID << ", Unknown reason. Not retrying." << LL_ENDL; // *TODO: Mark mesh unavailable on error - } } +} LLMeshPhysicsShapeHandler::~LLMeshPhysicsShapeHandler() - { - llassert(mProcessed); - } +{ + llassert(mProcessed); +} void LLMeshPhysicsShapeHandler::processFailure(LLCore::HttpStatus status) - { +{ LL_WARNS(LOG_MESH) << "Error during mesh physics shape handling. ID: " << mMeshID << ", Reason: " << status.toString() << " (" << status.toTerseString() << "). Not retrying." << LL_ENDL; // *TODO: Mark mesh unavailable on error - } +} -void LLMeshPhysicsShapeHandler::processData(LLCore::BufferArray * body, U8 * data, S32 data_size) - { +void LLMeshPhysicsShapeHandler::processData(LLCore::BufferArray * /* body */, S32 /* body_offset */, + U8 * data, S32 data_size) +{ if ((! MESH_PHYS_SHAPE_PROCESS_FAILED) && gMeshRepo.mThread->physicsShapeReceived(mMeshID, data, data_size)) - { + { // good fetch from sim, write to VFS for caching LLVFile file(gVFS, mMeshID, LLAssetType::AT_MESH, LLVFile::WRITE); @@ -2981,13 +3020,13 @@ void LLMeshPhysicsShapeHandler::processData(LLCore::BufferArray * body, U8 * dat S32 size = mRequestedBytes; if (file.getSize() >= offset+size) - { + { LLMeshRepository::sCacheBytesWritten += size; ++LLMeshRepository::sCacheWrites; file.seek(offset); file.write(data, size); - } } + } else { LL_WARNS(LOG_MESH) << "Error during mesh physics shape processing. ID: " << mMeshID @@ -3187,7 +3226,7 @@ void LLMeshRepository::notifyLoadedMeshes() if (1 == mGetMeshVersion) { // Legacy GetMesh operation with high connection concurrency - LLMeshRepoThread::sMaxConcurrentRequests = gSavedSettings.getU32("MeshMaxConcurrentRequests"); + LLMeshRepoThread::sMaxConcurrentRequests = gSavedSettings.getU32("MeshMaxConcurrentRequests"); LLMeshRepoThread::sRequestHighWater = llclamp(2 * S32(LLMeshRepoThread::sMaxConcurrentRequests), REQUEST_HIGH_WATER_MIN, REQUEST_HIGH_WATER_MAX); @@ -3198,9 +3237,15 @@ void LLMeshRepository::notifyLoadedMeshes() else { // GetMesh2 operation with keepalives, etc. With pipelining, - // we'll increase this. + // we'll increase this. See llappcorehttp and llcorehttp for + // discussion on connection strategies. + LLAppCoreHttp & app_core_http(LLAppViewer::instance()->getAppCoreHttp()); + S32 scale(app_core_http.isPipelined(LLAppCoreHttp::AP_MESH2) + ? (2 * LLAppCoreHttp::PIPELINING_DEPTH) + : 5); + LLMeshRepoThread::sMaxConcurrentRequests = gSavedSettings.getU32("Mesh2MaxConcurrentRequests"); - LLMeshRepoThread::sRequestHighWater = llclamp(5 * S32(LLMeshRepoThread::sMaxConcurrentRequests), + LLMeshRepoThread::sRequestHighWater = llclamp(scale * S32(LLMeshRepoThread::sMaxConcurrentRequests), REQUEST2_HIGH_WATER_MIN, REQUEST2_HIGH_WATER_MAX); LLMeshRepoThread::sRequestLowWater = llclamp(LLMeshRepoThread::sRequestHighWater / 2, @@ -3300,18 +3345,18 @@ void LLMeshRepository::notifyLoadedMeshes() // If we can't get the locks, skip and pick this up later. ++hold_offs; sMaxLockHoldoffs = llmax(sMaxLockHoldoffs, hold_offs); - return; - } + return; + } hold_offs = 0; if (gAgent.getRegion()) { // Update capability urls - static std::string region_name("never name a region this"); + static std::string region_name("never name a region this"); - if (gAgent.getRegion()->getName() != region_name && gAgent.getRegion()->capabilitiesReceived()) - { - region_name = gAgent.getRegion()->getName(); + if (gAgent.getRegion()->getName() != region_name && gAgent.getRegion()->capabilitiesReceived()) + { + region_name = gAgent.getRegion()->getName(); const bool use_v1(gSavedSettings.getBOOL("MeshUseGetMesh1")); const std::string mesh1(gAgent.getRegion()->getCapability("GetMesh")); const std::string mesh2(gAgent.getRegion()->getCapability("GetMesh2")); @@ -3322,8 +3367,8 @@ void LLMeshRepository::notifyLoadedMeshes() << ", GetMesh: " << mesh1 << ", using version: " << mGetMeshVersion << LL_ENDL; + } } - } //popup queued error messages from background threads while (!mUploadErrorQ.empty()) @@ -3338,46 +3383,46 @@ void LLMeshRepository::notifyLoadedMeshes() S32 push_count = LLMeshRepoThread::sRequestHighWater - active_count; if (mPendingRequests.size() > push_count) - { + { // More requests than the high-water limit allows so // sort and forward the most important. - //calculate "score" for pending requests + //calculate "score" for pending requests - //create score map - std::map<LLUUID, F32> score_map; + //create score map + std::map<LLUUID, F32> score_map; - for (U32 i = 0; i < 4; ++i) - { - for (mesh_load_map::iterator iter = mLoadingMeshes[i].begin(); iter != mLoadingMeshes[i].end(); ++iter) + for (U32 i = 0; i < 4; ++i) { - F32 max_score = 0.f; - for (std::set<LLUUID>::iterator obj_iter = iter->second.begin(); obj_iter != iter->second.end(); ++obj_iter) + for (mesh_load_map::iterator iter = mLoadingMeshes[i].begin(); iter != mLoadingMeshes[i].end(); ++iter) { - LLViewerObject* object = gObjectList.findObject(*obj_iter); - - if (object) + F32 max_score = 0.f; + for (std::set<LLUUID>::iterator obj_iter = iter->second.begin(); obj_iter != iter->second.end(); ++obj_iter) { - LLDrawable* drawable = object->mDrawable; - if (drawable) + LLViewerObject* object = gObjectList.findObject(*obj_iter); + + if (object) { - F32 cur_score = drawable->getRadius()/llmax(drawable->mDistanceWRTCamera, 1.f); - max_score = llmax(max_score, cur_score); + LLDrawable* drawable = object->mDrawable; + if (drawable) + { + F32 cur_score = drawable->getRadius()/llmax(drawable->mDistanceWRTCamera, 1.f); + max_score = llmax(max_score, cur_score); + } } } - } - score_map[iter->first.getSculptID()] = max_score; + score_map[iter->first.getSculptID()] = max_score; + } } - } - //set "score" for pending requests - for (std::vector<LLMeshRepoThread::LODRequest>::iterator iter = mPendingRequests.begin(); iter != mPendingRequests.end(); ++iter) - { - iter->mScore = score_map[iter->mMeshParams.getSculptID()]; - } + //set "score" for pending requests + for (std::vector<LLMeshRepoThread::LODRequest>::iterator iter = mPendingRequests.begin(); iter != mPendingRequests.end(); ++iter) + { + iter->mScore = score_map[iter->mMeshParams.getSculptID()]; + } - //sort by "score" + //sort by "score" std::partial_sort(mPendingRequests.begin(), mPendingRequests.begin() + push_count, mPendingRequests.end(), LLMeshRepoThread::CompareScoreGreater()); } @@ -3588,7 +3633,6 @@ void LLMeshRepository::fetchPhysicsShape(const LLUUID& mesh_id) } } } - } LLModel::Decomposition* LLMeshRepository::getDecomposition(const LLUUID& mesh_id) diff --git a/indra/newview/llsnapshotlivepreview.cpp b/indra/newview/llsnapshotlivepreview.cpp index 1aa7041175..600ebf5914 100644 --- a/indra/newview/llsnapshotlivepreview.cpp +++ b/indra/newview/llsnapshotlivepreview.cpp @@ -5,7 +5,7 @@ * * $LicenseInfo:firstyear=2013&license=viewerlgpl$ * Second Life Viewer Source Code -* Copyright (C) 2013, Linden Research, Inc. +* Copyright (C) 2014, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/indra/newview/lltexturefetch.cpp b/indra/newview/lltexturefetch.cpp index d9a874be49..acd4cf2d8d 100755 --- a/indra/newview/lltexturefetch.cpp +++ b/indra/newview/lltexturefetch.cpp @@ -56,13 +56,13 @@ #include "llsdparam.h" #include "llsdutil.h" #include "llstartup.h" -#include "llsdserialize.h" #include "httprequest.h" #include "httphandler.h" #include "httpresponse.h" #include "bufferarray.h" #include "bufferstream.h" +#include "llcorehttputil.h" #include "llhttpretrypolicy.h" @@ -241,8 +241,10 @@ LLTrace::EventStatHandle<F64Milliseconds > LLTextureFetch::sCacheReadLatency("te // Tuning/Parameterization Constants -static const S32 HTTP_REQUESTS_IN_QUEUE_HIGH_WATER = 40; // Maximum requests to have active in HTTP -static const S32 HTTP_REQUESTS_IN_QUEUE_LOW_WATER = 20; // Active level at which to refill +static const S32 HTTP_PIPE_REQUESTS_HIGH_WATER = 100; // Maximum requests to have active in HTTP (pipelined) +static const S32 HTTP_PIPE_REQUESTS_LOW_WATER = 50; // Active level at which to refill +static const S32 HTTP_NONPIPE_REQUESTS_HIGH_WATER = 40; +static const S32 HTTP_NONPIPE_REQUESTS_LOW_WATER = 20; // BUG-3323/SH-4375 // *NOTE: This is a heuristic value. Texture fetches have a habit of using a @@ -481,12 +483,12 @@ private: bool acquireHttpSemaphore() { llassert(! mHttpHasResource); - if (mFetcher->mHttpSemaphore <= 0) + if (mFetcher->mHttpSemaphore >= mFetcher->mHttpHighWater) { return false; } mHttpHasResource = true; - mFetcher->mHttpSemaphore--; + mFetcher->mHttpSemaphore++; return true; } @@ -496,7 +498,8 @@ private: { llassert(mHttpHasResource); mHttpHasResource = false; - mFetcher->mHttpSemaphore++; + mFetcher->mHttpSemaphore--; + llassert_always(mFetcher->mHttpSemaphore >= 0); } private: @@ -608,16 +611,16 @@ private: LLCore::HttpHandle mHttpHandle; // Handle of any active request LLCore::BufferArray * mHttpBufferArray; // Refcounted pointer to response data - S32 mHttpPolicyClass; + S32 mHttpPolicyClass; bool mHttpActive; // Active request to http library - U32 mHttpReplySize, // Actual received data size - mHttpReplyOffset; // Actual received data offset + U32 mHttpReplySize, // Actual received data size + mHttpReplyOffset; // Actual received data offset bool mHttpHasResource; // Counts against Fetcher's mHttpSemaphore // State history - U32 mCacheReadCount, - mCacheWriteCount, - mResourceWaitCount; // Requests entering WAIT_HTTP_RESOURCE2 + U32 mCacheReadCount, + mCacheWriteCount, + mResourceWaitCount; // Requests entering WAIT_HTTP_RESOURCE2 }; ////////////////////////////////////////////////////////////////////////////// @@ -1325,7 +1328,7 @@ bool LLTextureFetchWorker::doWork(S32 param) } } - static LLCachedControl<bool> use_http(gSavedSettings,"ImagePipelineUseHTTP", true); + static LLCachedControl<bool> use_http(gSavedSettings, "ImagePipelineUseHTTP", true); // if (mHost != LLHost::invalid) get_url = false; if ( use_http && mCanUseHTTP && mUrl.empty())//get http url. @@ -1346,20 +1349,20 @@ bool LLTextureFetchWorker::doWork(S32 param) LL_WARNS(LOG_TXT) << "trying to seek a non-default texture on the sim. Bad!" << LL_ENDL; } setUrl(http_url + "/?texture_id=" + mID.asString().c_str()); - LL_DEBUGS("Texture") << "Texture URL " << mUrl << LL_ENDL; + LL_DEBUGS(LOG_TXT) << "Texture URL: " << mUrl << LL_ENDL; mWriteToCacheState = CAN_WRITE ; //because this texture has a fixed texture id. } else { mCanUseHTTP = false ; - LL_DEBUGS("Texture") << "Texture not available via HTTP: no URL " << mUrl << LL_ENDL; + LL_DEBUGS(LOG_TXT) << "Texture not available via HTTP: empty URL." << LL_ENDL; } } else { // This will happen if not logged in or if a region deoes not have HTTP Texture enabled //LL_WARNS(LOG_TXT) << "Region not found for host: " << mHost << LL_ENDL; - LL_DEBUGS("Texture") << "Texture not available via HTTP: no region " << mUrl << LL_ENDL; + LL_DEBUGS(LOG_TXT) << "Texture not available via HTTP: no region " << mUrl << LL_ENDL; mCanUseHTTP = false; } } @@ -1473,6 +1476,9 @@ bool LLTextureFetchWorker::doWork(S32 param) if (mState == SEND_HTTP_REQ) { + // Also used in llmeshrepository + static LLCachedControl<bool> disable_range_req(gSavedSettings, "HttpRangeRequestsDisable", false); + if (! mCanUseHTTP) { releaseHttpSemaphore(); @@ -1528,22 +1534,47 @@ bool LLTextureFetchWorker::doWork(S32 param) mRequestedOffset -= 1; mRequestedSize += 1; } - mHttpHandle = LLCORE_HTTP_HANDLE_INVALID; - if (!mUrl.empty()) - { - mRequestedTimer.reset(); - mLoaded = FALSE; - mGetStatus = LLCore::HttpStatus(); - mGetReason.clear(); - LL_DEBUGS(LOG_TXT) << "HTTP GET: " << mID << " Offset: " << mRequestedOffset - << " Bytes: " << mRequestedSize - << " Bandwidth(kbps): " << mFetcher->getTextureBandwidth() << "/" << mFetcher->mMaxBandwidth - << LL_ENDL; - // Will call callbackHttpGet when curl request completes - // Only server bake images use the returned headers currently, for getting retry-after field. - LLCore::HttpOptions *options = (mFTType == FTT_SERVER_BAKE) ? mFetcher->mHttpOptionsWithHeaders: mFetcher->mHttpOptions; + if (mUrl.empty()) + { + // *FIXME: This should not be reachable except it has become + // so after some recent 'work'. Need to track this down + // and illuminate the unenlightened. + LL_WARNS(LOG_TXT) << "HTTP GET request failed for " << mID + << " on empty URL." << LL_ENDL; + resetFormattedData(); + releaseHttpSemaphore(); + return true; // failed + } + + mRequestedTimer.reset(); + mLoaded = FALSE; + mGetStatus = LLCore::HttpStatus(); + mGetReason.clear(); + LL_DEBUGS(LOG_TXT) << "HTTP GET: " << mID << " Offset: " << mRequestedOffset + << " Bytes: " << mRequestedSize + << " Bandwidth(kbps): " << mFetcher->getTextureBandwidth() << "/" << mFetcher->mMaxBandwidth + << LL_ENDL; + + // Will call callbackHttpGet when curl request completes + // Only server bake images use the returned headers currently, for getting retry-after field. + LLCore::HttpOptions *options = (mFTType == FTT_SERVER_BAKE) ? mFetcher->mHttpOptionsWithHeaders: mFetcher->mHttpOptions; + if (disable_range_req) + { + // 'Range:' requests may be disabled in which case all HTTP + // texture fetches result in full fetches. This can be used + // by people with questionable ISPs or networking gear that + // doesn't handle these well. + mHttpHandle = mFetcher->mHttpRequest->requestGet(mHttpPolicyClass, + mWorkPriority, + mUrl, + options, + mFetcher->mHttpHeaders, + this); + } + else + { mHttpHandle = mFetcher->mHttpRequest->requestGetByteRange(mHttpPolicyClass, mWorkPriority, mUrl, @@ -1557,7 +1588,11 @@ bool LLTextureFetchWorker::doWork(S32 param) } if (LLCORE_HTTP_HANDLE_INVALID == mHttpHandle) { - LL_WARNS(LOG_TXT) << "HTTP GET request failed for " << mID << LL_ENDL; + LLCore::HttpStatus status(mFetcher->mHttpRequest->getStatus()); + LL_WARNS(LOG_TXT) << "HTTP GET request failed for " << mID + << ", Status: " << status.toTerseString() + << " Reason: '" << status.toString() << "'" + << LL_ENDL; resetFormattedData(); releaseHttpSemaphore(); return true; // failed @@ -1613,10 +1648,6 @@ bool LLTextureFetchWorker::doWork(S32 param) else if (http_service_unavail == mGetStatus) { LL_INFOS_ONCE(LOG_TXT) << "Texture server busy (503): " << mUrl << LL_ENDL; - LL_INFOS(LOG_TXT) << "503: HTTP GET failed for: " << mUrl - << " Status: " << mGetStatus.toHex() - << " Reason: '" << mGetReason << "'" - << LL_ENDL; } else if (http_not_sat == mGetStatus) { @@ -1774,7 +1805,7 @@ bool LLTextureFetchWorker::doWork(S32 param) if (mState == DECODE_IMAGE) { - static LLCachedControl<bool> textures_decode_disabled(gSavedSettings,"TextureDecodeDisabled", false); + static LLCachedControl<bool> textures_decode_disabled(gSavedSettings, "TextureDecodeDisabled", false); setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); // Set priority first since Responder may change it if (textures_decode_disabled) @@ -2483,9 +2514,9 @@ LLTextureFetch::LLTextureFetch(LLTextureCache* cache, LLImageDecodeThread* image mHttpOptions(NULL), mHttpOptionsWithHeaders(NULL), mHttpHeaders(NULL), - mHttpMetricsHeaders(NULL), mHttpPolicyClass(LLCore::HttpRequest::DEFAULT_POLICY_ID), - mHttpSemaphore(HTTP_REQUESTS_IN_QUEUE_HIGH_WATER), + mHttpMetricsHeaders(NULL), + mHttpMetricsPolicyClass(LLCore::HttpRequest::DEFAULT_POLICY_ID), mTotalCacheReadCount(0U), mTotalCacheWriteCount(0U), mTotalResourceWaitCount(0U), @@ -2497,6 +2528,23 @@ LLTextureFetch::LLTextureFetch(LLTextureCache* cache, LLImageDecodeThread* image mMaxBandwidth = gSavedSettings.getF32("ThrottleBandwidthKBPS"); mTextureInfo.setUpLogging(gSavedSettings.getBOOL("LogTextureDownloadsToViewerLog"), gSavedSettings.getBOOL("LogTextureDownloadsToSimulator"), U32Bytes(gSavedSettings.getU32("TextureLoggingThreshold"))); + LLAppCoreHttp & app_core_http(LLAppViewer::instance()->getAppCoreHttp()); + mHttpRequest = new LLCore::HttpRequest; + mHttpOptions = new LLCore::HttpOptions; + mHttpOptionsWithHeaders = new LLCore::HttpOptions; + mHttpOptionsWithHeaders->setWantHeaders(true); + mHttpHeaders = new LLCore::HttpHeaders; + mHttpHeaders->append(HTTP_OUT_HEADER_ACCEPT, HTTP_CONTENT_IMAGE_X_J2C); + mHttpPolicyClass = app_core_http.getPolicy(LLAppCoreHttp::AP_TEXTURE); + mHttpMetricsHeaders = new LLCore::HttpHeaders; + mHttpMetricsHeaders->append(HTTP_OUT_HEADER_CONTENT_TYPE, HTTP_CONTENT_LLSD_XML); + mHttpMetricsPolicyClass = app_core_http.getPolicy(LLAppCoreHttp::AP_REPORTING); + mHttpHighWater = HTTP_NONPIPE_REQUESTS_HIGH_WATER; + mHttpLowWater = HTTP_NONPIPE_REQUESTS_LOW_WATER; + mHttpSemaphore = 0; + + // Conditionally construct debugger object after 'this' is + // fully initialized. LLTextureFetchDebugger::sDebuggerEnabled = gSavedSettings.getBOOL("TextureFetchDebuggerEnabled"); if(LLTextureFetchDebugger::isEnabled()) { @@ -2509,16 +2557,6 @@ LLTextureFetch::LLTextureFetch(LLTextureCache* cache, LLImageDecodeThread* image } mOriginFetchSource = mFetchSource; } - - mHttpRequest = new LLCore::HttpRequest; - mHttpOptions = new LLCore::HttpOptions; - mHttpOptionsWithHeaders = new LLCore::HttpOptions; - mHttpOptionsWithHeaders->setWantHeaders(true); - mHttpHeaders = new LLCore::HttpHeaders; - mHttpHeaders->append("Accept", "image/x-j2c"); - mHttpMetricsHeaders = new LLCore::HttpHeaders; - mHttpMetricsHeaders->append("Content-Type", "application/llsd+xml"); - mHttpPolicyClass = LLAppViewer::instance()->getAppCoreHttp().getPolicy(LLAppCoreHttp::AP_TEXTURE); } LLTextureFetch::~LLTextureFetch() @@ -2990,6 +3028,20 @@ bool LLTextureFetch::runCondition() // Threads: Ttf void LLTextureFetch::commonUpdate() { + // Update low/high water levels based on pipelining. We pick + // up setting eventually, so the semaphore/request level can + // fall outside the [0..HIGH_WATER] range. Expect that. + if (LLAppViewer::instance()->getAppCoreHttp().isPipelined(LLAppCoreHttp::AP_TEXTURE)) + { + mHttpHighWater = HTTP_PIPE_REQUESTS_HIGH_WATER; + mHttpLowWater = HTTP_PIPE_REQUESTS_LOW_WATER; + } + else + { + mHttpHighWater = HTTP_NONPIPE_REQUESTS_HIGH_WATER; + mHttpLowWater = HTTP_NONPIPE_REQUESTS_LOW_WATER; + } + // Release waiters releaseHttpWaiters(); @@ -3651,8 +3703,16 @@ void LLTextureFetch::releaseHttpWaiters() { // Use mHttpSemaphore rather than mHTTPTextureQueue.size() // to avoid a lock. - if (mHttpSemaphore < (HTTP_REQUESTS_IN_QUEUE_HIGH_WATER - HTTP_REQUESTS_IN_QUEUE_LOW_WATER)) + if (mHttpSemaphore >= mHttpLowWater) return; + S32 needed(mHttpHighWater - mHttpSemaphore); + if (needed <= 0) + { + // Would only happen if High/LowWater were changed behind + // our back. In that case, defer fill until usage falls within + // limits. + return; + } // Quickly make a copy of all the LLUIDs. Get off the // mutex as early as possible. @@ -3701,10 +3761,10 @@ void LLTextureFetch::releaseHttpWaiters() tids.clear(); // Sort into priority order, if necessary and only as much as needed - if (tids2.size() > mHttpSemaphore) + if (tids2.size() > needed) { LLTextureFetchWorker::Compare compare; - std::partial_sort(tids2.begin(), tids2.begin() + mHttpSemaphore, tids2.end(), compare); + std::partial_sort(tids2.begin(), tids2.begin() + needed, tids2.end(), compare); } // Release workers up to the high water mark. Since we aren't @@ -3968,7 +4028,9 @@ TFReqSendMetrics::doWork(LLTextureFetch * fetcher) // Update sequence number if (S32_MAX == ++report_sequence) + { report_sequence = 0; + } reporting_started = true; // Limit the size of the stats report if necessary. @@ -3977,18 +4039,15 @@ TFReqSendMetrics::doWork(LLTextureFetch * fetcher) if (! mCapsURL.empty()) { - LLCore::BufferArray * ba = new LLCore::BufferArray; - LLCore::BufferArrayStream bas(ba); - LLSDSerialize::toXML(sd, bas); - - fetcher->getHttpRequest().requestPost(fetcher->getPolicyClass(), - report_priority, - mCapsURL, - ba, - NULL, - fetcher->getMetricsHeaders(), - handler); - ba->release(); + // Don't care about handle, this is a fire-and-forget operation. + LLCoreHttpUtil::requestPostWithLLSD(&fetcher->getHttpRequest(), + fetcher->getMetricsPolicyClass(), + report_priority, + mCapsURL, + sd, + NULL, + fetcher->getMetricsHeaders(), + handler); LLTextureFetch::svMetricsDataBreak = false; } else @@ -3999,7 +4058,7 @@ TFReqSendMetrics::doWork(LLTextureFetch * fetcher) // In QA mode, Metrics submode, log the result for ease of testing if (fetcher->isQAMode()) { - LL_INFOS("Textures") << ll_pretty_print_sd(sd) << LL_ENDL; + LL_INFOS(LOG_TXT) << ll_pretty_print_sd(sd) << LL_ENDL; } return true; @@ -4169,7 +4228,7 @@ void LLTextureFetchDebugger::init() if (! mHttpHeaders) { mHttpHeaders = new LLCore::HttpHeaders; - mHttpHeaders->append("Accept", "image/x-j2c"); + mHttpHeaders->append(HTTP_OUT_HEADER_ACCEPT, HTTP_CONTENT_IMAGE_X_J2C); } } @@ -4544,7 +4603,7 @@ S32 LLTextureFetchDebugger::fillCurlQueue() mNbCurlCompleted = mFetchingHistory.size(); return 0; } - if (mNbCurlRequests > HTTP_REQUESTS_IN_QUEUE_LOW_WATER) + if (mNbCurlRequests > HTTP_NONPIPE_REQUESTS_LOW_WATER) { return mNbCurlRequests; } @@ -4577,7 +4636,7 @@ S32 LLTextureFetchDebugger::fillCurlQueue() mFetchingHistory[i].mHttpHandle = handle; mFetchingHistory[i].mCurlState = FetchEntry::CURL_IN_PROGRESS; mNbCurlRequests++; - if (mNbCurlRequests >= HTTP_REQUESTS_IN_QUEUE_HIGH_WATER) // emulate normal pipeline + if (mNbCurlRequests >= HTTP_NONPIPE_REQUESTS_HIGH_WATER) // emulate normal pipeline { break; } diff --git a/indra/newview/lltexturefetch.h b/indra/newview/lltexturefetch.h index c4da2e8685..27779a31e0 100755 --- a/indra/newview/lltexturefetch.h +++ b/indra/newview/lltexturefetch.h @@ -179,6 +179,9 @@ public: // Threads: T* LLCore::HttpHeaders * getMetricsHeaders() const { return mHttpMetricsHeaders; } + // Threads: T* + LLCore::HttpRequest::policy_t getMetricsPolicyClass() const { return mHttpMetricsPolicyClass; } + bool isQAMode() const { return mQAMode; } // ---------------------------------- @@ -354,9 +357,12 @@ private: LLCore::HttpOptions * mHttpOptions; // Ttf LLCore::HttpOptions * mHttpOptionsWithHeaders; // Ttf LLCore::HttpHeaders * mHttpHeaders; // Ttf - LLCore::HttpHeaders * mHttpMetricsHeaders; // Ttf LLCore::HttpRequest::policy_t mHttpPolicyClass; // T* - + LLCore::HttpHeaders * mHttpMetricsHeaders; // Ttf + LLCore::HttpRequest::policy_t mHttpMetricsPolicyClass; // T* + S32 mHttpHighWater; // Ttf + S32 mHttpLowWater; // Ttf + // We use a resource semaphore to keep HTTP requests in // WAIT_HTTP_RESOURCE2 if there aren't sufficient slots in the // transport. This keeps them near where they can be cheaply @@ -364,7 +370,11 @@ private: // where it's more expensive to get at them. Requests in either // SEND_HTTP_REQ or WAIT_HTTP_REQ charge against the semaphore // and tracking state transitions is critical to liveness. - LLAtomicS32 mHttpSemaphore; // Ttf + Tmain + // + // Originally implemented as a traditional semaphore (heading towards + // zero), it now is an outstanding request count that is allowed to + // exceed the high water level (but not go below zero). + LLAtomicS32 mHttpSemaphore; // Ttf typedef std::set<LLUUID> wait_http_res_queue_t; wait_http_res_queue_t mHttpWaitResource; // Mfnq diff --git a/indra/newview/llviewerinventory.cpp b/indra/newview/llviewerinventory.cpp index 39267e8834..d6c8ba10f6 100755 --- a/indra/newview/llviewerinventory.cpp +++ b/indra/newview/llviewerinventory.cpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2002&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2010, Linden Research, Inc. + * Copyright (C) 2014, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -75,6 +75,10 @@ void no_op_inventory_func(const LLUUID&) {} void no_op_llsd_func(const LLSD&) {} void no_op() {} +static const char * const LOG_INV("Inventory"); +static const char * const LOG_LOCAL("InventoryLocalize"); +static const char * const LOG_NOTECARD("copy_inventory_from_notecard"); + ///---------------------------------------------------------------------------- /// Helper class to store special inventory item names and their localized values. ///---------------------------------------------------------------------------- @@ -189,7 +193,7 @@ public: */ bool localizeInventoryObjectName(std::string& object_name) { - LL_DEBUGS("InventoryLocalize") << "Searching for localization: " << object_name << LL_ENDL; + LL_DEBUGS(LOG_LOCAL) << "Searching for localization: " << object_name << LL_ENDL; std::map<std::string, std::string>::const_iterator dictionary_iter = mInventoryItemsDict.find(object_name); @@ -197,7 +201,7 @@ public: if(found) { object_name = dictionary_iter->second; - LL_DEBUGS("InventoryLocalize") << "Found, new name is: " << object_name << LL_ENDL; + LL_DEBUGS(LOG_LOCAL) << "Found, new name is: " << object_name << LL_ENDL; } return found; } @@ -307,8 +311,8 @@ LLViewerInventoryItem::LLViewerInventoryItem(const LLViewerInventoryItem* other) copyViewerItem(other); if (!mIsComplete) { - LL_WARNS() << "LLViewerInventoryItem copy constructor for incomplete item" - << mUUID << LL_ENDL; + LL_WARNS(LOG_INV) << "LLViewerInventoryItem copy constructor for incomplete item" + << mUUID << LL_ENDL; } } @@ -355,16 +359,16 @@ void LLViewerInventoryItem::updateServer(BOOL is_new) const { // *FIX: deal with this better. // If we're crashing here then the UI is incorrectly enabled. - LL_ERRS() << "LLViewerInventoryItem::updateServer() - for incomplete item" - << LL_ENDL; + LL_ERRS(LOG_INV) << "LLViewerInventoryItem::updateServer() - for incomplete item" + << LL_ENDL; return; } if(gAgent.getID() != mPermissions.getOwner()) { // *FIX: deal with this better. - LL_WARNS() << "LLViewerInventoryItem::updateServer() - for unowned item " - << ll_pretty_print_sd(this->asLLSD()) - << LL_ENDL; + LL_WARNS(LOG_INV) << "LLViewerInventoryItem::updateServer() - for unowned item " + << ll_pretty_print_sd(this->asLLSD()) + << LL_ENDL; return; } LLInventoryModel::LLCategoryUpdate up(mParentUUID, is_new ? 1 : 0); @@ -392,18 +396,18 @@ void LLViewerInventoryItem::fetchFromServer(void) const // we have to check region. It can be null after region was destroyed. See EXT-245 if (region) { - if(gAgent.getID() != mPermissions.getOwner()) - { + if (gAgent.getID() != mPermissions.getOwner()) + { url = region->getCapability("FetchLib2"); - } + } else - { + { url = region->getCapability("FetchInventory2"); - } + } } else { - LL_WARNS() << "Agent Region is absent" << LL_ENDL; + LL_WARNS(LOG_INV) << "Agent Region is absent" << LL_ENDL; } if (!url.empty()) @@ -413,7 +417,8 @@ void LLViewerInventoryItem::fetchFromServer(void) const body["items"][0]["owner_id"] = mPermissions.getOwner(); body["items"][0]["item_id"] = mUUID; - LLHTTPClient::post(url, body, new LLInventoryModel::fetchInventoryResponder(body)); + LLInventoryModel::FetchItemHttpHandler * handler(new LLInventoryModel::FetchItemHttpHandler(body)); + gInventory.requestPost(true, url, body, handler, "Inventory Item"); } else { @@ -649,7 +654,7 @@ bool LLViewerInventoryCategory::fetch() if((VERSION_UNKNOWN == getVersion()) && mDescendentsRequested.hasExpired()) //Expired check prevents multiple downloads. { - LL_DEBUGS("InventoryFetch") << "Fetching category children: " << mName << ", UUID: " << mUUID << LL_ENDL; + LL_DEBUGS(LOG_INV) << "Fetching category children: " << mName << ", UUID: " << mUUID << LL_ENDL; const F32 FETCH_TIMER_EXPIRY = 10.0f; mDescendentsRequested.reset(); mDescendentsRequested.setTimerExpirySec(FETCH_TIMER_EXPIRY); @@ -674,7 +679,7 @@ bool LLViewerInventoryCategory::fetch() } else { - LL_WARNS() << "agent region is null" << LL_ENDL; + LL_WARNS(LOG_INV) << "agent region is null" << LL_ENDL; } if (!url.empty()) //Capability found. Build up LLSD and use it. { @@ -682,7 +687,8 @@ bool LLViewerInventoryCategory::fetch() } else { //Deprecated, but if we don't have a capability, use the old system. - LL_INFOS() << "FetchInventoryDescendents2 capability not found. Using deprecated UDP message." << LL_ENDL; + LL_INFOS(LOG_INV) << "FetchInventoryDescendents2 capability not found. Using deprecated UDP message." << LL_ENDL; + LLMessageSystem* msg = gMessageSystem; msg->newMessage("FetchInventoryDescendents"); msg->nextBlock("AgentData"); @@ -777,8 +783,8 @@ bool LLViewerInventoryCategory::importFileLocal(LLFILE* fp) } else { - LL_WARNS() << "unknown keyword '" << keyword - << "' in inventory import category " << mUUID << LL_ENDL; + LL_WARNS(LOG_INV) << "unknown keyword '" << keyword + << "' in inventory import category " << mUUID << LL_ENDL; } } return true; @@ -906,7 +912,7 @@ LLInventoryCallbackManager::LLInventoryCallbackManager() : { if( sInstance != NULL ) { - LL_WARNS() << "LLInventoryCallbackManager::LLInventoryCallbackManager: unexpected multiple instances" << LL_ENDL; + LL_WARNS(LOG_INV) << "LLInventoryCallbackManager::LLInventoryCallbackManager: unexpected multiple instances" << LL_ENDL; return; } sInstance = this; @@ -916,7 +922,7 @@ LLInventoryCallbackManager::~LLInventoryCallbackManager() { if( sInstance != this ) { - LL_WARNS() << "LLInventoryCallbackManager::~LLInventoryCallbackManager: unexpected multiple instances" << LL_ENDL; + LL_WARNS(LOG_INV) << "LLInventoryCallbackManager::~LLInventoryCallbackManager: unexpected multiple instances" << LL_ENDL; return; } sInstance = NULL; @@ -1144,7 +1150,7 @@ void link_inventory_object(const LLUUID& category, { if (!baseobj) { - LL_WARNS() << "Attempt to link to non-existent object" << LL_ENDL; + LL_WARNS(LOG_INV) << "Attempt to link to non-existent object" << LL_ENDL; return; } @@ -1178,7 +1184,7 @@ void link_inventory_array(const LLUUID& category, const LLInventoryObject* baseobj = *it; if (!baseobj) { - LL_WARNS() << "attempt to link to unknown object" << LL_ENDL; + LL_WARNS(LOG_INV) << "attempt to link to unknown object" << LL_ENDL; continue; } @@ -1187,7 +1193,7 @@ void link_inventory_array(const LLUUID& category, // Fail if item can be found but is of a type that can't be linked. // Arguably should fail if the item can't be found too, but that could // be a larger behavioral change. - LL_WARNS() << "attempt to link an unlinkable object, type = " << baseobj->getActualType() << LL_ENDL; + LL_WARNS(LOG_INV) << "attempt to link an unlinkable object, type = " << baseobj->getActualType() << LL_ENDL; continue; } @@ -1223,7 +1229,7 @@ void link_inventory_array(const LLUUID& category, } else { - LL_WARNS() << "could not convert object into an item or category: " << baseobj->getUUID() << LL_ENDL; + LL_WARNS(LOG_INV) << "could not convert object into an item or category: " << baseobj->getUUID() << LL_ENDL; continue; } } @@ -1237,10 +1243,10 @@ void link_inventory_array(const LLUUID& category, links.append(link); #ifndef LL_RELEASE_FOR_DOWNLOAD - LL_DEBUGS("Inventory") << "Linking Object [ name:" << baseobj->getName() - << " UUID:" << baseobj->getUUID() - << " ] into Category [ name:" << cat_name - << " UUID:" << category << " ] " << LL_ENDL; + LL_DEBUGS(LOG_INV) << "Linking Object [ name:" << baseobj->getName() + << " UUID:" << baseobj->getUUID() + << " ] into Category [ name:" << cat_name + << " UUID:" << category << " ] " << LL_ENDL; #endif } @@ -1331,7 +1337,7 @@ void update_inventory_item( if (!ais_ran) { LLPointer<LLViewerInventoryItem> obj = gInventory.getItem(item_id); - LL_DEBUGS("Inventory") << "item_id: [" << item_id << "] name " << (update_item ? update_item->getName() : "(NOT FOUND)") << LL_ENDL; + LL_DEBUGS(LOG_INV) << "item_id: [" << item_id << "] name " << (update_item ? update_item->getName() : "(NOT FOUND)") << LL_ENDL; if(obj) { LLMessageSystem* msg = gMessageSystem; @@ -1373,7 +1379,7 @@ void update_inventory_item( if (!ais_ran) { LLPointer<LLViewerInventoryItem> obj = gInventory.getItem(item_id); - LL_DEBUGS("Inventory") << "item_id: [" << item_id << "] name " << (obj ? obj->getName() : "(NOT FOUND)") << LL_ENDL; + LL_DEBUGS(LOG_INV) << "item_id: [" << item_id << "] name " << (obj ? obj->getName() : "(NOT FOUND)") << LL_ENDL; if(obj) { LLPointer<LLViewerInventoryItem> new_item(new LLViewerInventoryItem); @@ -1408,7 +1414,7 @@ void update_inventory_category( LLPointer<LLInventoryCallback> cb) { LLPointer<LLViewerInventoryCategory> obj = gInventory.getCategory(cat_id); - LL_DEBUGS("Inventory") << "cat_id: [" << cat_id << "] name " << (obj ? obj->getName() : "(NOT FOUND)") << LL_ENDL; + LL_DEBUGS(LOG_INV) << "cat_id: [" << cat_id << "] name " << (obj ? obj->getName() : "(NOT FOUND)") << LL_ENDL; if(obj) { if (LLFolderType::lookupIsProtectedType(obj->getPreferredType())) @@ -1473,7 +1479,7 @@ void remove_inventory_item( } else { - LL_DEBUGS("Inventory") << "item_id: [" << item_id << "] name " << "(NOT FOUND)" << LL_ENDL; + LL_DEBUGS(LOG_INV) << "item_id: [" << item_id << "] name " << "(NOT FOUND)" << LL_ENDL; } } @@ -1485,7 +1491,7 @@ void remove_inventory_item( if(obj) { const LLUUID item_id(obj->getUUID()); - LL_DEBUGS("Inventory") << "item_id: [" << item_id << "] name " << obj->getName() << LL_ENDL; + LL_DEBUGS(LOG_INV) << "item_id: [" << item_id << "] name " << obj->getName() << LL_ENDL; if (AISCommand::isAPIAvailable()) { LLPointer<AISCommand> cmd_ptr = new RemoveItemCommand(item_id, cb); @@ -1519,7 +1525,7 @@ void remove_inventory_item( else { // *TODO: Clean up callback? - LL_WARNS() << "remove_inventory_item called for invalid or nonexistent item." << LL_ENDL; + LL_WARNS(LOG_INV) << "remove_inventory_item called for invalid or nonexistent item." << LL_ENDL; } } @@ -1537,7 +1543,7 @@ public: LLInventoryModel::EHasChildren children = gInventory.categoryHasChildren(mID); if(children != LLInventoryModel::CHILDREN_NO) { - LL_WARNS() << "remove descendents failed, cannot remove category " << LL_ENDL; + LL_WARNS(LOG_INV) << "remove descendents failed, cannot remove category " << LL_ENDL; } else { @@ -1553,7 +1559,7 @@ void remove_inventory_category( const LLUUID& cat_id, LLPointer<LLInventoryCallback> cb) { - LL_DEBUGS("Inventory") << "cat_id: [" << cat_id << "] " << LL_ENDL; + LL_DEBUGS(LOG_INV) << "cat_id: [" << cat_id << "] " << LL_ENDL; LLPointer<LLViewerInventoryCategory> obj = gInventory.getCategory(cat_id); if(obj) { @@ -1574,7 +1580,7 @@ void remove_inventory_category( LLInventoryModel::EHasChildren children = gInventory.categoryHasChildren(cat_id); if(children != LLInventoryModel::CHILDREN_NO) { - LL_DEBUGS("Inventory") << "Will purge descendents first before deleting category " << cat_id << LL_ENDL; + LL_DEBUGS(LOG_INV) << "Will purge descendents first before deleting category " << cat_id << LL_ENDL; LLPointer<LLInventoryCallback> wrap_cb = new LLRemoveCategoryOnDestroy(cat_id, cb); purge_descendents_of(cat_id, wrap_cb); return; @@ -1600,7 +1606,7 @@ void remove_inventory_category( } else { - LL_WARNS() << "remove_inventory_category called for invalid or nonexistent item " << cat_id << LL_ENDL; + LL_WARNS(LOG_INV) << "remove_inventory_category called for invalid or nonexistent item " << cat_id << LL_ENDL; } } @@ -1628,7 +1634,7 @@ void purge_descendents_of(const LLUUID& id, LLPointer<LLInventoryCallback> cb) LLInventoryModel::EHasChildren children = gInventory.categoryHasChildren(id); if(children == LLInventoryModel::CHILDREN_NO) { - LL_DEBUGS("Inventory") << "No descendents to purge for " << id << LL_ENDL; + LL_DEBUGS(LOG_INV) << "No descendents to purge for " << id << LL_ENDL; return; } LLPointer<LLViewerInventoryCategory> cat = gInventory.getCategory(id); @@ -1637,8 +1643,8 @@ void purge_descendents_of(const LLUUID& id, LLPointer<LLInventoryCallback> cb) if (LLClipboard::instance().hasContents() && LLClipboard::instance().isCutMode()) { // Something on the clipboard is in "cut mode" and needs to be preserved - LL_DEBUGS("Inventory") << "purge_descendents_of clipboard case " << cat->getName() - << " iterate and purge non hidden items" << LL_ENDL; + LL_DEBUGS(LOG_INV) << "purge_descendents_of clipboard case " << cat->getName() + << " iterate and purge non hidden items" << LL_ENDL; LLInventoryModel::cat_array_t* categories; LLInventoryModel::item_array_t* items; // Get the list of direct descendants in tha categoy passed as argument @@ -1673,7 +1679,7 @@ void purge_descendents_of(const LLUUID& id, LLPointer<LLInventoryCallback> cb) else // no cap { // Fast purge - LL_DEBUGS("Inventory") << "purge_descendents_of fast case " << cat->getName() << LL_ENDL; + LL_DEBUGS(LOG_INV) << "purge_descendents_of fast case " << cat->getName() << LL_ENDL; // send it upstream LLMessageSystem* msg = gMessageSystem; @@ -1716,9 +1722,9 @@ void copy_inventory_from_notecard(const LLUUID& destination_id, { if (NULL == src) { - LL_WARNS("copy_inventory_from_notecard") << "Null pointer to item was passed for object_id " - << object_id << " and notecard_inv_id " - << notecard_inv_id << LL_ENDL; + LL_WARNS(LOG_NOTECARD) << "Null pointer to item was passed for object_id " + << object_id << " and notecard_inv_id " + << notecard_inv_id << LL_ENDL; return; } @@ -1738,9 +1744,9 @@ void copy_inventory_from_notecard(const LLUUID& destination_id, if (! viewer_region) { - LL_WARNS("copy_inventory_from_notecard") << "Can't find region from object_id " - << object_id << " or gAgent" - << LL_ENDL; + LL_WARNS(LOG_NOTECARD) << "Can't find region from object_id " + << object_id << " or gAgent" + << LL_ENDL; return; } @@ -1748,9 +1754,9 @@ void copy_inventory_from_notecard(const LLUUID& destination_id, std::string url = viewer_region->getCapability("CopyInventoryFromNotecard"); if (url.empty()) { - LL_WARNS("copy_inventory_from_notecard") << "There is no 'CopyInventoryFromNotecard' capability" - << " for region: " << viewer_region->getName() - << LL_ENDL; + LL_WARNS(LOG_NOTECARD) << "There is no 'CopyInventoryFromNotecard' capability" + << " for region: " << viewer_region->getName() + << LL_ENDL; return; } @@ -1824,15 +1830,15 @@ void slam_inventory_folder(const LLUUID& folder_id, { if (AISCommand::isAPIAvailable()) { - LL_DEBUGS("Avatar") << "using AISv3 to slam folder, id " << folder_id - << " new contents: " << ll_pretty_print_sd(contents) << LL_ENDL; + LL_DEBUGS(LOG_INV) << "using AISv3 to slam folder, id " << folder_id + << " new contents: " << ll_pretty_print_sd(contents) << LL_ENDL; LLPointer<AISCommand> cmd_ptr = new SlamFolderCommand(folder_id, contents, cb); cmd_ptr->run_command(); } else // no cap { - LL_DEBUGS("Avatar") << "using item-by-item calls to slam folder, id " << folder_id - << " new contents: " << ll_pretty_print_sd(contents) << LL_ENDL; + LL_DEBUGS(LOG_INV) << "using item-by-item calls to slam folder, id " << folder_id + << " new contents: " << ll_pretty_print_sd(contents) << LL_ENDL; for (LLSD::array_const_iterator it = contents.beginArray(); it != contents.endArray(); ++it) @@ -1934,7 +1940,7 @@ void menu_create_inventory_item(LLInventoryPanel* panel, LLFolderBridge *bridge, } else { - LL_WARNS() << "Can't create unrecognized type " << type_name << LL_ENDL; + LL_WARNS(LOG_INV) << "Can't create unrecognized type " << type_name << LL_ENDL; } } panel->getRootFolder()->setNeedsAutoRename(TRUE); @@ -2169,7 +2175,7 @@ LLViewerInventoryItem *LLViewerInventoryItem::getLinkedItem() const LLViewerInventoryItem *linked_item = gInventory.getItem(mAssetUUID); if (linked_item && linked_item->getIsLinkType()) { - LL_WARNS() << "Warning: Accessing link to link" << LL_ENDL; + LL_WARNS(LOG_INV) << "Warning: Accessing link to link" << LL_ENDL; return NULL; } return linked_item; diff --git a/indra/newview/llviewermenu.cpp b/indra/newview/llviewermenu.cpp index 38aaff9279..faf929d8f9 100755 --- a/indra/newview/llviewermenu.cpp +++ b/indra/newview/llviewermenu.cpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2002&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2010, Linden Research, Inc. + * Copyright (C) 2014, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -8287,7 +8287,7 @@ class LLWorldEnableEnvSettings : public view_listener_t } else { - llwarns << "Unknown item" << llendl; + LL_WARNS() << "Unknown time-of-day item: " << tod << LL_ENDL; } } return result; diff --git a/indra/newview/tests/llslurl_test.cpp b/indra/newview/tests/llslurl_test.cpp index 86229ad636..2bc0d5a086 100755 --- a/indra/newview/tests/llslurl_test.cpp +++ b/indra/newview/tests/llslurl_test.cpp @@ -6,7 +6,7 @@ * * $LicenseInfo:firstyear=2009&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2010, Linden Research, Inc. + * Copyright (C) 2014, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -31,6 +31,15 @@ #include "../llslurl.h" #include "../../llxml/llcontrol.h" #include "llsdserialize.h" + +namespace +{ + +// Should not collide with other test programs creating temp files. +static const char * const TEST_FILENAME("llslurl_test.xml"); + +} + //---------------------------------------------------------------------------- // Mock objects for the dependencies of the code we're testing @@ -143,11 +152,11 @@ namespace tut template<> template<> void slurlTestObject::test<1>() { - llofstream gridfile("grid_test.xml"); + llofstream gridfile(TEST_FILENAME); gridfile << gSampleGridFile; gridfile.close(); - LLGridManager::getInstance()->initialize("grid_test.xml"); + LLGridManager::getInstance()->initialize(TEST_FILENAME); LLGridManager::getInstance()->setGridChoice("util.agni.lindenlab.com"); @@ -260,11 +269,11 @@ namespace tut template<> template<> void slurlTestObject::test<2>() { - llofstream gridfile("grid_test.xml"); + llofstream gridfile(TEST_FILENAME); gridfile << gSampleGridFile; gridfile.close(); - LLGridManager::getInstance()->initialize("grid_test.xml"); + LLGridManager::getInstance()->initialize(TEST_FILENAME); LLSLURL slurl = LLSLURL("my.grid.com", "my region"); ensure_equals("grid/region - type", slurl.getType(), LLSLURL::LOCATION); @@ -293,11 +302,11 @@ namespace tut template<> template<> void slurlTestObject::test<3>() { - llofstream gridfile("grid_test.xml"); + llofstream gridfile(TEST_FILENAME); gridfile << gSampleGridFile; gridfile.close(); - LLGridManager::getInstance()->initialize("grid_test.xml"); + LLGridManager::getInstance()->initialize(TEST_FILENAME); LLGridManager::getInstance()->setGridChoice("my.grid.com"); LLSLURL slurl = LLSLURL("https://my.grid.com/region/my%20region/1/2/3"); diff --git a/indra/newview/tests/llviewernetwork_test.cpp b/indra/newview/tests/llviewernetwork_test.cpp index 7ad7947ca4..0eb0ab6500 100755 --- a/indra/newview/tests/llviewernetwork_test.cpp +++ b/indra/newview/tests/llviewernetwork_test.cpp @@ -6,7 +6,7 @@ * * $LicenseInfo:firstyear=2009&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2010, Linden Research, Inc. + * Copyright (C) 2014, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -31,6 +31,13 @@ #include "../../llxml/llcontrol.h" #include "llfile.h" +namespace +{ + +// Should not collide with other test programs creating temp files. +static const char * const TEST_FILENAME("llviewernetwork_test.xml"); + +} //---------------------------------------------------------------------------- // Mock objects for the dependencies of the code we're testing @@ -143,7 +150,7 @@ namespace tut { viewerNetworkTest() { - LLFile::remove("grid_test.xml"); + LLFile::remove(TEST_FILENAME); gCmdLineLoginURI.clear(); gCmdLineGridChoice.clear(); gCmdLineHelperURI.clear(); @@ -152,7 +159,7 @@ namespace tut } ~viewerNetworkTest() { - LLFile::remove("grid_test.xml"); + LLFile::remove(TEST_FILENAME); } }; @@ -170,7 +177,7 @@ namespace tut { LLGridManager *manager = LLGridManager::getInstance(); // grid file doesn't exist - manager->initialize("grid_test.xml"); + manager->initialize(TEST_FILENAME); // validate that some of the defaults are available. std::map<std::string, std::string> known_grids = manager->getKnownGrids(); ensure_equals("Known grids is a string-string map of size 2", known_grids.size(), 2); @@ -238,11 +245,11 @@ namespace tut template<> template<> void viewerNetworkTestObject::test<2>() { - llofstream gridfile("grid_test.xml"); + llofstream gridfile(TEST_FILENAME); gridfile << gSampleGridFile; gridfile.close(); - LLGridManager::getInstance()->initialize("grid_test.xml"); + LLGridManager::getInstance()->initialize(TEST_FILENAME); std::map<std::string, std::string> known_grids = LLGridManager::getInstance()->getKnownGrids(); ensure_equals("adding a grid via a grid file increases known grid size",4, known_grids.size()); @@ -369,11 +376,11 @@ namespace tut void viewerNetworkTestObject::test<7>() { // adding a grid with simply a name will populate the values. - llofstream gridfile("grid_test.xml"); + llofstream gridfile(TEST_FILENAME); gridfile << gSampleGridFile; gridfile.close(); - LLGridManager::getInstance()->initialize("grid_test.xml"); + LLGridManager::getInstance()->initialize(TEST_FILENAME); LLGridManager::getInstance()->setGridChoice("util.agni.lindenlab.com"); ensure_equals("getGridLabel", |