From 438a6431e418eac5a3a4e00f7adfe379994869d7 Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Thu, 26 Apr 2012 11:35:07 -0400 Subject: Bring llcorehttp into the compile and link phases. Windows looks okay though it's a dead library so far. --- indra/newview/CMakeLists.txt | 3 +++ 1 file changed, 3 insertions(+) (limited to 'indra/newview') diff --git a/indra/newview/CMakeLists.txt b/indra/newview/CMakeLists.txt index f85b943c70..fc9d89aea0 100644 --- a/indra/newview/CMakeLists.txt +++ b/indra/newview/CMakeLists.txt @@ -18,6 +18,7 @@ include(LLAudio) include(LLCharacter) include(LLCommon) include(LLConvexDecomposition) +include(LLCoreHttp) include(LLImage) include(LLImageJ2COJ) include(LLInventory) @@ -52,6 +53,7 @@ include_directories( ${LLCHARACTER_INCLUDE_DIRS} ${LLCOMMON_INCLUDE_DIRS} ${LLCONVEXDECOMP_INCLUDE_DIRS} + ${LLCOREHTTP_INCLUDE_DIRS} ${FMOD_INCLUDE_DIR} ${LLIMAGE_INCLUDE_DIRS} ${LLKDU_INCLUDE_DIRS} @@ -1747,6 +1749,7 @@ target_link_libraries(${VIEWER_BINARY_NAME} ${LLXUIXML_LIBRARIES} ${LSCRIPT_LIBRARIES} ${LLMATH_LIBRARIES} + ${LLCOREHTTP_LIBRARIES} ${LLCOMMON_LIBRARIES} ${NDOF_LIBRARY} ${viewer_LIBRARIES} -- cgit v1.3 From cc878fed1b65779cd4535678117e99a68a69c62c Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Tue, 8 May 2012 11:32:46 -0700 Subject: Now trying to get linux working. libllcommon is picking up .so boost libraries when we tend to want statics. Try to work around that for the moment. --- indra/newview/viewer_manifest.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'indra/newview') diff --git a/indra/newview/viewer_manifest.py b/indra/newview/viewer_manifest.py index 0931c4ec9b..b3ca24a783 100644 --- a/indra/newview/viewer_manifest.py +++ b/indra/newview/viewer_manifest.py @@ -1031,6 +1031,8 @@ class Linux_i686Manifest(LinuxManifest): self.path("libaprutil-1.so") self.path("libaprutil-1.so.0") self.path("libaprutil-1.so.0.3.10") + self.path("libboost_program_options-mt.so.1.48.0") + self.path("libboost_regex.so.1.48.0") self.path("libbreakpad_client.so.0.0.0") self.path("libbreakpad_client.so.0") self.path("libbreakpad_client.so") -- cgit v1.3 From e1a978bbcfe4640e3163fd0a3ed41460f1d6942a Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Tue, 8 May 2012 13:39:06 -0700 Subject: And more boost library work. --- indra/cmake/Copy3rdPartyLibs.cmake | 1 + indra/newview/viewer_manifest.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) (limited to 'indra/newview') diff --git a/indra/cmake/Copy3rdPartyLibs.cmake b/indra/cmake/Copy3rdPartyLibs.cmake index 0866b35e9d..fe42cc5251 100644 --- a/indra/cmake/Copy3rdPartyLibs.cmake +++ b/indra/cmake/Copy3rdPartyLibs.cmake @@ -253,6 +253,7 @@ elseif(LINUX) libatk-1.0.so libboost_program_options-mt.so.1.48.0 libboost_regex-mt.so.1.48.0 + libboost_thread-mt.so.1.48.0 libbreakpad_client.so.0 libcollada14dom.so libcrypto.so.1.0.0 diff --git a/indra/newview/viewer_manifest.py b/indra/newview/viewer_manifest.py index b3ca24a783..2be89b77de 100644 --- a/indra/newview/viewer_manifest.py +++ b/indra/newview/viewer_manifest.py @@ -1032,7 +1032,8 @@ class Linux_i686Manifest(LinuxManifest): self.path("libaprutil-1.so.0") self.path("libaprutil-1.so.0.3.10") self.path("libboost_program_options-mt.so.1.48.0") - self.path("libboost_regex.so.1.48.0") + self.path("libboost_regex-mt.so.1.48.0") + self.path("libboost_thread-mt.so.1.48.0") self.path("libbreakpad_client.so.0.0.0") self.path("libbreakpad_client.so.0") self.path("libbreakpad_client.so") -- cgit v1.3 From 2f496ecaeeb0ab90c29ca6f0414cad1fe16cd4b0 Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Wed, 9 May 2012 06:49:18 -0700 Subject: Add more boost libs to the lib copy and manifest. --- indra/cmake/Copy3rdPartyLibs.cmake | 3 +++ indra/newview/viewer_manifest.py | 3 +++ 2 files changed, 6 insertions(+) (limited to 'indra/newview') diff --git a/indra/cmake/Copy3rdPartyLibs.cmake b/indra/cmake/Copy3rdPartyLibs.cmake index fe42cc5251..17ea48f31f 100644 --- a/indra/cmake/Copy3rdPartyLibs.cmake +++ b/indra/cmake/Copy3rdPartyLibs.cmake @@ -254,6 +254,9 @@ elseif(LINUX) libboost_program_options-mt.so.1.48.0 libboost_regex-mt.so.1.48.0 libboost_thread-mt.so.1.48.0 + libboost_filesystem-mt.so.1.48.0 + libboost_signals-mt.so.1.48.0 + libboost_system-mt.so.1.48.0 libbreakpad_client.so.0 libcollada14dom.so libcrypto.so.1.0.0 diff --git a/indra/newview/viewer_manifest.py b/indra/newview/viewer_manifest.py index 2be89b77de..fffb61e89c 100644 --- a/indra/newview/viewer_manifest.py +++ b/indra/newview/viewer_manifest.py @@ -1034,6 +1034,9 @@ class Linux_i686Manifest(LinuxManifest): self.path("libboost_program_options-mt.so.1.48.0") self.path("libboost_regex-mt.so.1.48.0") self.path("libboost_thread-mt.so.1.48.0") + self.path("libboost_filesystem-mt.so.1.48.0") + self.path("libboost_signals-mt.so.1.48.0") + self.path("libboost_system-mt.so.1.48.0") self.path("libbreakpad_client.so.0.0.0") self.path("libbreakpad_client.so.0") self.path("libbreakpad_client.so") -- cgit v1.3 From 8fc350125c671baeae6b7f8b1814251009f4f50a Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Wed, 23 May 2012 19:12:09 -0400 Subject: Integrate llcorehttp library into lltexturefetch design. This is the first functional viewer pass with the HTTP work of the texture fetch code performed by the llcorehttp library. Not exactly a 'drop-in' replacement but a work-alike with some changes (e.g. handler notification in consumer thread versus responder notification in worker thread). This also includes some temporary changes in the priority scheme to prevent the kind of priority inversion found in VWR-28996. Scheme used here does provide liveness if not optimal responsiveness or order-of-operation. The llcorehttp library at this point is far from optimally performing. Its worker thread is making relatively poor use of cycles it gets and it doesn't idle or sleep intelligently yet. This early integration step helps shake out the interfaces, implementation niceties will be covered soon. --- indra/llcorehttp/CMakeLists.txt | 2 + indra/llcorehttp/_httplibcurl.cpp | 11 +- indra/llcorehttp/_httplibcurl.h | 3 +- indra/llcorehttp/_httpopcancel.cpp | 82 ++++ indra/llcorehttp/_httpopcancel.h | 75 ++++ indra/llcorehttp/_httpoperation.h | 25 -- indra/llcorehttp/_httpoprequest.cpp | 105 +++-- indra/llcorehttp/_httpoprequest.h | 9 +- indra/llcorehttp/_httppolicy.h | 3 +- indra/llcorehttp/_httpreplyqueue.h | 2 +- indra/llcorehttp/_httprequestqueue.h | 2 +- indra/llcorehttp/_httpservice.h | 4 +- indra/llcorehttp/httpcommon.cpp | 90 ++++- indra/llcorehttp/httpcommon.h | 59 ++- indra/llcorehttp/httprequest.cpp | 17 + indra/llcorehttp/httpresponse.cpp | 3 +- indra/llcorehttp/httpresponse.h | 32 +- indra/llcorehttp/tests/test_httpstatus.hpp | 102 +++++ indra/newview/llappviewer.cpp | 159 ++++++++ indra/newview/lltexturefetch.cpp | 610 +++++++++++++++++------------ indra/newview/lltexturefetch.h | 19 +- 21 files changed, 1057 insertions(+), 357 deletions(-) create mode 100644 indra/llcorehttp/_httpopcancel.cpp create mode 100644 indra/llcorehttp/_httpopcancel.h (limited to 'indra/newview') diff --git a/indra/llcorehttp/CMakeLists.txt b/indra/llcorehttp/CMakeLists.txt index 81c502b642..ae92fb96fd 100644 --- a/indra/llcorehttp/CMakeLists.txt +++ b/indra/llcorehttp/CMakeLists.txt @@ -30,6 +30,7 @@ set(llcorehttp_SOURCE_FILES _httprequestqueue.cpp _httpoperation.cpp _httpoprequest.cpp + _httpopcancel.cpp _httpreplyqueue.cpp _httppolicy.cpp _httplibcurl.cpp @@ -49,6 +50,7 @@ set(llcorehttp_HEADER_FILES httpresponse.h _httpoperation.h _httpoprequest.h + _httpopcancel.h _httprequestqueue.h _httpreplyqueue.h _httpservice.h diff --git a/indra/llcorehttp/_httplibcurl.cpp b/indra/llcorehttp/_httplibcurl.cpp index 15be977adf..1b951818e4 100644 --- a/indra/llcorehttp/_httplibcurl.cpp +++ b/indra/llcorehttp/_httplibcurl.cpp @@ -27,7 +27,6 @@ #include "_httplibcurl.h" #include "httpheaders.h" - #include "_httpoprequest.h" #include "_httpservice.h" @@ -173,8 +172,11 @@ void HttpLibcurl::completeRequest(CURLM * multi_handle, CURL * handle, CURLcode { int http_status(200); - curl_easy_getinfo(handle, CURLINFO_RESPONSE_CODE, &status); - op->mReplyStatus = http_status; + curl_easy_getinfo(handle, CURLINFO_RESPONSE_CODE, &http_status); + op->mStatus = LLCore::HttpStatus(http_status, + (http_status >= 200 && http_status <= 299 + ? HE_SUCCESS + : HE_REPLY_ERROR)); } // Detach from multi and recycle handle @@ -202,7 +204,8 @@ int HttpLibcurl::activeCount() const struct curl_slist * append_headers_to_slist(const HttpHeaders * headers, struct curl_slist * slist) { for (HttpHeaders::container_t::const_iterator it(headers->mHeaders.begin()); - headers->mHeaders.end() != it; + + headers->mHeaders.end() != it; ++it) { slist = curl_slist_append(slist, (*it).c_str()); diff --git a/indra/llcorehttp/_httplibcurl.h b/indra/llcorehttp/_httplibcurl.h index 01c68320af..807196628d 100644 --- a/indra/llcorehttp/_httplibcurl.h +++ b/indra/llcorehttp/_httplibcurl.h @@ -45,6 +45,7 @@ class HttpOpRequest; class HttpHeaders; +/// Implements libcurl-based transport for an HttpService instance. class HttpLibcurl { public: @@ -71,7 +72,7 @@ protected: typedef std::set active_set_t; protected: - HttpService * mService; + HttpService * mService; // Simple reference, not owner active_set_t mActiveOps; CURLM * mMultiHandles[1]; }; // end class HttpLibcurl diff --git a/indra/llcorehttp/_httpopcancel.cpp b/indra/llcorehttp/_httpopcancel.cpp new file mode 100644 index 0000000000..69dbff4bb4 --- /dev/null +++ b/indra/llcorehttp/_httpopcancel.cpp @@ -0,0 +1,82 @@ +/** + * @file _httpopcancel.cpp + * @brief Definitions for internal class HttpOpCancel + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include "_httpopcancel.h" + +#include +#include + +#include "httpcommon.h" +#include "httphandler.h" +#include "httpresponse.h" + +#include "_httprequestqueue.h" +#include "_httpreplyqueue.h" +#include "_httpservice.h" +#include "_httppolicy.h" +#include "_httplibcurl.h" + + +namespace LLCore +{ + + +// ================================== +// HttpOpCancel +// ================================== + + +HttpOpCancel::HttpOpCancel(HttpHandle handle) + : HttpOperation(), + mHandle(handle) +{} + + +HttpOpCancel::~HttpOpCancel() +{} + + +void HttpOpCancel::stageFromRequest(HttpService * service) +{ + // *FIXME: Need cancel functionality into services + addAsReply(); +} + + +void HttpOpCancel::visitNotifier(HttpRequest * request) +{ + if (mLibraryHandler) + { + HttpResponse * response = new HttpResponse(); + mLibraryHandler->onCompleted(static_cast(this), response); + response->release(); + } +} + + +} // end namespace LLCore + + diff --git a/indra/llcorehttp/_httpopcancel.h b/indra/llcorehttp/_httpopcancel.h new file mode 100644 index 0000000000..38ccc585ed --- /dev/null +++ b/indra/llcorehttp/_httpopcancel.h @@ -0,0 +1,75 @@ +/** + * @file _httpopcancel.h + * @brief Internal declarations for the HttpOpCancel subclass + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef _LLCORE_HTTP_OPCANCEL_H_ +#define _LLCORE_HTTP_OPCANCEL_H_ + + +#include "linden_common.h" // Modifies curl/curl.h interfaces + +#include "httpcommon.h" + +#include + +#include "_httpoperation.h" +#include "_refcounted.h" + + +namespace LLCore +{ + + +/// HttpOpCancel requests that a previously issued request +/// be canceled, if possible. Requests that have been made +/// active and are available for sending on the wire cannot +/// be canceled. + +class HttpOpCancel : public HttpOperation +{ +public: + HttpOpCancel(HttpHandle handle); + virtual ~HttpOpCancel(); + +private: + HttpOpCancel(const HttpOpCancel &); // Not defined + void operator=(const HttpOpCancel &); // Not defined + +public: + virtual void stageFromRequest(HttpService *); + + virtual void visitNotifier(HttpRequest * request); + +public: + // Request data + HttpHandle mHandle; + +}; // end class HttpOpCancel + + +} // end namespace LLCore + +#endif // _LLCORE_HTTP_OPCANCEL_H_ + diff --git a/indra/llcorehttp/_httpoperation.h b/indra/llcorehttp/_httpoperation.h index d04961c47b..5d06a28586 100644 --- a/indra/llcorehttp/_httpoperation.h +++ b/indra/llcorehttp/_httpoperation.h @@ -87,31 +87,6 @@ public: }; // end class HttpOperation -/// HttpOpCancel requests that a previously issued request -/// be canceled, if possible. Requests that have been made -/// active and are available for sending on the wire cannot -/// be canceled. - -class HttpOpCancel : public HttpOperation -{ -public: - HttpOpCancel(); - virtual ~HttpOpCancel(); - -private: - HttpOpCancel(const HttpOpCancel &); // Not defined - void operator=(const HttpOpCancel &); // Not defined - -public: - virtual void stageFromRequest(HttpService *); - virtual void stageFromReady(HttpService *); - virtual void stageFromActive(HttpService *); - -public: - HttpHandle mHandle; -}; // end class HttpOpCancel - - /// HttpOpStop requests the servicing thread to shutdown /// operations, cease pulling requests from the request /// queue and release shared resources (particularly diff --git a/indra/llcorehttp/_httpoprequest.cpp b/indra/llcorehttp/_httpoprequest.cpp index 3c9eb71b9a..521bd5b879 100644 --- a/indra/llcorehttp/_httpoprequest.cpp +++ b/indra/llcorehttp/_httpoprequest.cpp @@ -86,53 +86,53 @@ HttpOpRequest::HttpOpRequest() mReqMethod(HOR_GET), mReqBody(NULL), mReqOffset(0), - mReqLen(0), + mReqLength(0), mReqHeaders(NULL), mReqOptions(NULL), mCurlActive(false), mCurlHandle(NULL), - mCurlHeaders(NULL), mCurlService(NULL), - mReplyStatus(200), + mCurlHeaders(NULL), mReplyBody(NULL), mReplyOffset(0), - mReplyLen(0), + mReplyLength(0), mReplyHeaders(NULL) {} HttpOpRequest::~HttpOpRequest() { - if (mCurlHandle) - { - curl_easy_cleanup(mCurlHandle); - mCurlHandle = NULL; - } - - if (mCurlHeaders) - { - curl_slist_free_all(mCurlHeaders); - mCurlHeaders = NULL; - } - - mCurlService = NULL; - if (mReqBody) { mReqBody->release(); mReqBody = NULL; } + if (mReqOptions) + { + mReqOptions->release(); + mReqOptions = NULL; + } + if (mReqHeaders) { - curl_slist_free_all(mReqHeaders); + mReqHeaders->release(); mReqHeaders = NULL; } - if (mReqOptions) + if (mCurlHandle) { - mReqOptions->release(); - mReqOptions = NULL; + curl_easy_cleanup(mCurlHandle); + mCurlHandle = NULL; + } + + mCurlService = NULL; + + + if (mCurlHeaders) + { + curl_slist_free_all(mCurlHeaders); + mCurlHeaders = NULL; } if (mReplyBody) @@ -165,28 +165,29 @@ void HttpOpRequest::stageFromReady(HttpService * service) void HttpOpRequest::stageFromActive(HttpService * service) { - if (mReplyLen) + if (mReplyLength) { // If non-zero, we received and processed a Content-Range // header with the response. Verify that what it says // is consistent with the received data. - if (mReplyLen != mReplyBody->size()) + if (mReplyLength != mReplyBody->size()) { // Not as expected, fail the request mStatus = HttpStatus(HttpStatus::LLCORE, HE_INV_CONTENT_RANGE_HDR); } } - if (mReqHeaders) + if (mCurlHeaders) { // We take these headers out of the request now as they were // allocated originally in this thread and the notifier doesn't // need them. This eliminates one source of heap moving across // threads. - curl_slist_free_all(mReqHeaders); - mReqHeaders = NULL; + curl_slist_free_all(mCurlHeaders); + mCurlHeaders = NULL; } + addAsReply(); } @@ -196,12 +197,24 @@ void HttpOpRequest::visitNotifier(HttpRequest * request) if (mLibraryHandler) { HttpResponse * response = new HttpResponse(); - - // *FIXME: add http status, offset, length response->setStatus(mStatus); - response->setReplyStatus(mReplyStatus); response->setBody(mReplyBody); response->setHeaders(mReplyHeaders); + unsigned int offset(0), length(0); + if (mReplyOffset || mReplyLength) + { + // Got an explicit offset/length in response + offset = mReplyOffset; + length = mReplyLength; + } + else if (mReplyBody) + { + // Provide implicit offset/length from request/response + offset = mReqOffset; + length = mReplyBody->size(); + } + response->setRange(offset, length); + mLibraryHandler->onCompleted(static_cast(this), response); response->release(); @@ -235,14 +248,15 @@ HttpStatus HttpOpRequest::setupGetByteRange(unsigned int policy_id, mReqMethod = HOR_GET; mReqURL = url; mReqOffset = offset; - mReqLen = len; + mReqLength = len; if (offset || len) { mProcFlags |= PF_SCAN_RANGE_HEADER; } if (headers && ! mReqHeaders) { - mReqHeaders = append_headers_to_slist(headers, mReqHeaders); + headers->addRef(); + mReqHeaders = headers; } if (options && ! mReqOptions) { @@ -257,7 +271,7 @@ HttpStatus HttpOpRequest::prepareForGet(HttpService * service) { // *FIXME: better error handling later HttpStatus status; - + mCurlHandle = curl_easy_init(); curl_easy_setopt(mCurlHandle, CURLOPT_TIMEOUT, 30); curl_easy_setopt(mCurlHandle, CURLOPT_CONNECTTIMEOUT, 30); @@ -268,14 +282,18 @@ HttpStatus HttpOpRequest::prepareForGet(HttpService * service) curl_easy_setopt(mCurlHandle, CURLOPT_ENCODING, ""); // curl_easy_setopt(handle, CURLOPT_PROXY, ""); - mCurlHeaders = curl_slist_append(mCurlHeaders, "Pragma:"); curl_easy_setopt(mCurlHandle, CURLOPT_DNS_CACHE_TIMEOUT, 0); - curl_easy_setopt(mCurlHandle, CURLOPT_HTTPHEADER, mCurlHeaders); curl_easy_setopt(mCurlHandle, CURLOPT_FOLLOWLOCATION, 1); curl_easy_setopt(mCurlHandle, CURLOPT_WRITEFUNCTION, writeCallback); curl_easy_setopt(mCurlHandle, CURLOPT_WRITEDATA, mCurlHandle); - if (mReqOffset || mReqLen) + if (mReqHeaders) + { + mCurlHeaders = append_headers_to_slist(mReqHeaders, mCurlHeaders); + } + mCurlHeaders = curl_slist_append(mCurlHeaders, "Pragma:"); + + if (mReqOffset || mReqLength) { static const char * fmt1("Range: bytes=%d-%d"); static const char * fmt2("Range: bytes=%d-"); @@ -284,16 +302,17 @@ HttpStatus HttpOpRequest::prepareForGet(HttpService * service) #if defined(WIN32) _snprintf_s(range_line, sizeof(range_line), sizeof(range_line) - 1, - (mReqLen ? fmt1 : fmt2), - mReqOffset, mReqOffset + mReqLen - 1); + (mReqLength ? fmt1 : fmt2), + mReqOffset, mReqOffset + mReqLength - 1); #else snprintf(range_line, sizeof(range_line), - (mReqLen ? fmt1 : fmt2), - mReqOffset, mReqOffset + mReqLen - 1); + (mReqLength ? fmt1 : fmt2), + mReqOffset, mReqOffset + mReqLength - 1); #endif // defined(WIN32) range_line[sizeof(range_line) - 1] = '\0'; mCurlHeaders = curl_slist_append(mCurlHeaders, range_line); } + curl_easy_setopt(mCurlHandle, CURLOPT_HTTPHEADER, mCurlHeaders); if (mProcFlags & (PF_SCAN_RANGE_HEADER | PF_SAVE_HEADERS)) { @@ -347,8 +366,9 @@ size_t HttpOpRequest::headerCallback(void * data, size_t size, size_t nmemb, voi if (hdr_size >= status_line_len && ! strncmp(status_line, hdr_data, status_line_len)) { // One of possibly several status lines. Reset what we know and start over + // taking results from the last header stanza we receive. op->mReplyOffset = 0; - op->mReplyLen = 0; + op->mReplyLength = 0; op->mStatus = HttpStatus(); } else if (op->mProcFlags & PF_SCAN_RANGE_HEADER) @@ -371,7 +391,7 @@ size_t HttpOpRequest::headerCallback(void * data, size_t size, size_t nmemb, voi { // Success, record the fragment position op->mReplyOffset = first; - op->mReplyLen = last - first + 1; + op->mReplyLength = last - first + 1; } else if (-1 == status) { @@ -390,6 +410,7 @@ size_t HttpOpRequest::headerCallback(void * data, size_t size, size_t nmemb, voi if (op->mProcFlags & PF_SAVE_HEADERS) { // Save headers in response + // *FIXME: Implement this... ; } diff --git a/indra/llcorehttp/_httpoprequest.h b/indra/llcorehttp/_httpoprequest.h index 232ee841d6..601937a943 100644 --- a/indra/llcorehttp/_httpoprequest.h +++ b/indra/llcorehttp/_httpoprequest.h @@ -103,22 +103,21 @@ public: std::string mReqURL; BufferArray * mReqBody; off_t mReqOffset; - size_t mReqLen; - curl_slist * mReqHeaders; + size_t mReqLength; + HttpHeaders * mReqHeaders; HttpOptions * mReqOptions; // Transport data bool mCurlActive; CURL * mCurlHandle; - curl_slist * mCurlHeaders; HttpService * mCurlService; + curl_slist * mCurlHeaders; // Result data HttpStatus mStatus; - int mReplyStatus; BufferArray * mReplyBody; off_t mReplyOffset; - size_t mReplyLen; + size_t mReplyLength; HttpHeaders * mReplyHeaders; }; // end class HttpOpRequest diff --git a/indra/llcorehttp/_httppolicy.h b/indra/llcorehttp/_httppolicy.h index 28aea27f38..192bc73b31 100644 --- a/indra/llcorehttp/_httppolicy.h +++ b/indra/llcorehttp/_httppolicy.h @@ -39,6 +39,7 @@ class HttpService; class HttpOpRequest; +/// Implements class-based queuing policies for an HttpService instance. class HttpPolicy { public: @@ -58,7 +59,7 @@ protected: typedef std::vector ready_queue_t; protected: - HttpService * mService; // Naked pointer, not refcounted + HttpService * mService; // Naked pointer, not refcounted, not owner ready_queue_t mReadyQueue; }; // end class HttpPolicy diff --git a/indra/llcorehttp/_httpreplyqueue.h b/indra/llcorehttp/_httpreplyqueue.h index 56dadec87c..28cb1d68b7 100644 --- a/indra/llcorehttp/_httpreplyqueue.h +++ b/indra/llcorehttp/_httpreplyqueue.h @@ -58,7 +58,7 @@ class HttpOperation; /// will be coded anyway so it shouldn't be too much of a /// burden. -class HttpReplyQueue: public LLCoreInt::RefCounted +class HttpReplyQueue : public LLCoreInt::RefCounted { public: /// Caller acquires a Refcount on construction diff --git a/indra/llcorehttp/_httprequestqueue.h b/indra/llcorehttp/_httprequestqueue.h index 3a9ce0c3c6..f96bd7520c 100644 --- a/indra/llcorehttp/_httprequestqueue.h +++ b/indra/llcorehttp/_httprequestqueue.h @@ -46,7 +46,7 @@ class HttpOperation; /// requests from all HttpRequest instances into the /// singleton HttpService instance. -class HttpRequestQueue: public LLCoreInt::RefCounted +class HttpRequestQueue : public LLCoreInt::RefCounted { protected: /// Caller acquires a Refcount on construction diff --git a/indra/llcorehttp/_httpservice.h b/indra/llcorehttp/_httpservice.h index c052e35452..ba76e1eeca 100644 --- a/indra/llcorehttp/_httpservice.h +++ b/indra/llcorehttp/_httpservice.h @@ -152,8 +152,8 @@ protected: LLCoreInt::HttpThread * mThread; // === working-thread-only data === - HttpPolicy * mPolicy; - HttpLibcurl * mTransport; + HttpPolicy * mPolicy; // Simple pointer, has ownership + HttpLibcurl * mTransport; // Simple pointer, has ownership }; // end class HttpService diff --git a/indra/llcorehttp/httpcommon.cpp b/indra/llcorehttp/httpcommon.cpp index c37d081150..b5872606b8 100644 --- a/indra/llcorehttp/httpcommon.cpp +++ b/indra/llcorehttp/httpcommon.cpp @@ -37,22 +37,82 @@ HttpStatus::type_enum_t EXT_CURL_EASY; HttpStatus::type_enum_t EXT_CURL_MULTI; HttpStatus::type_enum_t LLCORE; +HttpStatus::operator unsigned long() const +{ + static const int shift(sizeof(unsigned long) * 4); + + unsigned long result(((unsigned long) mType) << shift | (unsigned long) (int) mStatus); + return result; +} + + std::string HttpStatus::toString() const { static const char * llcore_errors[] = { "", + "HTTP error reply status", "Services shutting down", "Operation canceled", "Invalid Content-Range header encountered" }; static const int llcore_errors_count(sizeof(llcore_errors) / sizeof(llcore_errors[0])); + + static const struct + { + type_enum_t mCode; + char * mText; + } + http_errors[] = + { + // Keep sorted by mCode, we binary search this list. + { 100, "Continue" }, + { 101, "Switching Protocols" }, + { 200, "OK" }, + { 201, "Created" }, + { 202, "Accepted" }, + { 203, "Non-Authoritative Information" }, + { 204, "No Content" }, + { 205, "Reset Content" }, + { 206, "Partial Content" }, + { 300, "Multiple Choices" }, + { 301, "Moved Permanently" }, + { 302, "Found" }, + { 303, "See Other" }, + { 304, "Not Modified" }, + { 305, "Use Proxy" }, + { 307, "Temporary Redirect" }, + { 400, "Bad Request" }, + { 401, "Unauthorized" }, + { 402, "Payment Required" }, + { 403, "Forbidden" }, + { 404, "Not Found" }, + { 405, "Method Not Allowed" }, + { 406, "Not Acceptable" }, + { 407, "Proxy Authentication Required" }, + { 408, "Request Time-out" }, + { 409, "Conflict" }, + { 410, "Gone" }, + { 411, "Length Required" }, + { 412, "Precondition Failed" }, + { 413, "Request Entity Too Large" }, + { 414, "Request-URI Too Large" }, + { 415, "Unsupported Media Type" }, + { 416, "Requested range not satisfiable" }, + { 417, "Expectation Failed" }, + { 500, "Internal Server Error" }, + { 501, "Not Implemented" }, + { 502, "Bad Gateway" }, + { 503, "Service Unavailable" }, + { 504, "Gateway Time-out" }, + { 505, "HTTP Version not supported" } + }; + static const int http_errors_count(sizeof(http_errors) / sizeof(http_errors[0])); if (*this) { return std::string(""); } - switch (mType) { case EXT_CURL_EASY: @@ -67,7 +127,33 @@ std::string HttpStatus::toString() const return std::string(llcore_errors[mStatus]); } break; - + + default: + if (isHttpStatus()) + { + int bottom(0), top(http_errors_count); + while (true) + { + int at((bottom + top) / 2); + if (mType == http_errors[at].mCode) + { + return std::string(http_errors[at].mText); + } + if (at == bottom) + { + break; + } + else if (mType < http_errors[at].mCode) + { + top = at; + } + else + { + bottom = at; + } + } + } + break; } return std::string("Unknown error"); } diff --git a/indra/llcorehttp/httpcommon.h b/indra/llcorehttp/httpcommon.h index 9de5769d57..f81be7103e 100644 --- a/indra/llcorehttp/httpcommon.h +++ b/indra/llcorehttp/httpcommon.h @@ -122,23 +122,38 @@ enum HttpError // Successful value compatible with the libcurl codes. HE_SUCCESS = 0, + // Intended for HTTP reply codes 100-999, indicates that + // the reply should be considered an error by the application. + HE_REPLY_ERROR = 1, + // Service is shutting down and requested operation will // not be queued or performed. - HE_SHUTTING_DOWN = 1, + HE_SHUTTING_DOWN = 2, // Operation was canceled by request. - HE_OP_CANCELED = 2, + HE_OP_CANCELED = 3, // Invalid content range header received. - HE_INV_CONTENT_RANGE_HDR = 3 + HE_INV_CONTENT_RANGE_HDR = 4 }; // end enum HttpError -/// HttpStatus encapsulates errors from libcurl (easy, multi) as well as -/// internal errors. The encapsulation isn't expected to completely -/// isolate the caller from libcurl but basic operational tests (success -/// or failure) are provided. +/// HttpStatus encapsulates errors from libcurl (easy, multi), HTTP +/// reply status codes and internal errors as well. The encapsulation +/// isn't expected to completely isolate the caller from libcurl but +/// basic operational tests (success or failure) are provided. +/// +/// Non-HTTP status are encoded as (type, status) with type being +/// one of: EXT_CURL_EASY, EXT_CURL_MULTI or LLCORE and status +/// being the success/error code from that domain. HTTP status +/// is encoded as (status, error_flag). Status should be in the +/// range [100, 999] and error_flag is either HE_SUCCESS or +/// HE_REPLY_ERROR to indicate whether this should be treated as +/// a successful status or an error. The application is responsible +/// for making that determination and a range like [200, 299] isn't +/// automatically assumed to be definitive. + struct HttpStatus { typedef unsigned short type_enum_t; @@ -192,12 +207,42 @@ struct HttpStatus return 0 != mStatus; } + /// Equality and inequality tests to bypass bool conversion + /// which will do the wrong thing in conditional expressions. + bool operator==(const HttpStatus & rhs) const + { + return mType == rhs.mType && mStatus == rhs.mStatus; + } + + bool operator!=(const HttpStatus & rhs) const + { + return ! operator==(rhs); + } + + /// Convert to single numeric representation. Mainly + /// for logging or other informal purposes. Also + /// creates an ambiguous second path to integer conversion + /// which tends to find programming errors such as formatting + /// the status to a stream (operator<<). + operator unsigned long() const; + unsigned long toULong() const + { + return operator unsigned long(); + } + /// Convert status to a string representation. For /// success, returns an empty string. For failure /// statuses, a string as appropriate for the source of /// the error code (libcurl easy, libcurl multi, or /// LLCore itself). std::string toString() const; + + /// Returns true if the status value represents an + /// HTTP response status (100 - 999). + bool isHttpStatus() const + { + return mType >= type_enum_t(100) && mType <= type_enum_t(999); + } }; // end struct HttpStatus diff --git a/indra/llcorehttp/httprequest.cpp b/indra/llcorehttp/httprequest.cpp index 2a87f5231a..6c62f931ff 100644 --- a/indra/llcorehttp/httprequest.cpp +++ b/indra/llcorehttp/httprequest.cpp @@ -31,6 +31,7 @@ #include "_httpservice.h" #include "_httpoperation.h" #include "_httpoprequest.h" +#include "_httpopcancel.h" namespace @@ -189,6 +190,22 @@ HttpHandle HttpRequest::requestGetByteRange(unsigned int policy_id, } +HttpHandle HttpRequest::requestCancel(HttpHandle handle, HttpHandler * user_handler) +{ + HttpStatus status; + HttpHandle ret_handle(LLCORE_HTTP_HANDLE_INVALID); + + HttpOpCancel * op = new HttpOpCancel(handle); + op->setHandlers(mReplyQueue, mSelfHandler, user_handler); + mRequestQueue->addOp(op); // transfer refcount as well + + mLastReqStatus = status; + ret_handle = static_cast(op); + + return ret_handle; +} + + HttpHandle HttpRequest::requestNoOp(HttpHandler * user_handler) { HttpStatus status; diff --git a/indra/llcorehttp/httpresponse.cpp b/indra/llcorehttp/httpresponse.cpp index 9ac8276f05..3dcdadb337 100644 --- a/indra/llcorehttp/httpresponse.cpp +++ b/indra/llcorehttp/httpresponse.cpp @@ -35,7 +35,8 @@ namespace LLCore HttpResponse::HttpResponse() : LLCoreInt::RefCounted(true), - mReplyStatus(0U), + mReplyOffset(0U), + mReplyLength(0U), mBufferArray(NULL), mHeaders(NULL) {} diff --git a/indra/llcorehttp/httpresponse.h b/indra/llcorehttp/httpresponse.h index a25e22aef6..5cf3a919f4 100644 --- a/indra/llcorehttp/httpresponse.h +++ b/indra/llcorehttp/httpresponse.h @@ -67,8 +67,6 @@ protected: public: /// Returns the final status of the requested operation. /// - // *FIXME: Haven't incorporated HTTP status into this yet. - // Will do soon. HttpStatus getStatus() const { return mStatus; @@ -79,19 +77,6 @@ public: mStatus = status; } - /// Fetch the HTTP reply status. This is only guaranteed to be - /// valid if the HttpStatus tests successful and was the result - /// of a completed HTTP request. - unsigned int getReplyStatus() const - { - return mReplyStatus; - } - - void setReplyStatus(unsigned int status) - { - mReplyStatus = status; - } - /// Simple getter for the response body returned as a scatter/gather /// buffer. If the operation doesn't produce data (such as the Null /// or StopThread operations), this may be NULL. @@ -122,11 +107,26 @@ public: /// Behaves like @see setResponse() but for header data. void setHeaders(HttpHeaders * headers); + + /// If a 'Range:' header was used, these methods are involved + /// in setting and returning data about the actual response. + void getRange(unsigned int * offset, unsigned int * length) const + { + *offset = mReplyOffset; + *length = mReplyLength; + } + + void setRange(unsigned int offset, unsigned int length) + { + mReplyOffset = offset; + mReplyLength = length; + } protected: // Response data here HttpStatus mStatus; - unsigned int mReplyStatus; + unsigned int mReplyOffset; + unsigned int mReplyLength; BufferArray * mBufferArray; HttpHeaders * mHeaders; }; diff --git a/indra/llcorehttp/tests/test_httpstatus.hpp b/indra/llcorehttp/tests/test_httpstatus.hpp index 38bf494dec..f7b542d3b5 100644 --- a/indra/llcorehttp/tests/test_httpstatus.hpp +++ b/indra/llcorehttp/tests/test_httpstatus.hpp @@ -157,6 +157,108 @@ void HttpStatusTestObjectType::test<4>() ensure(! msg.empty()); } +template <> template <> +void HttpStatusTestObjectType::test<5>() +{ + set_test_name("HttpStatus equality/inequality testing"); + + // Make certain equality/inequality tests do not pass + // through the bool conversion. Distinct successful + // and error statuses should compare unequal. + + HttpStatus status1(HttpStatus::LLCORE, HE_SUCCESS); + HttpStatus status2(HttpStatus::EXT_CURL_EASY, HE_SUCCESS); + ensure(status1 != status2); + + status1.mType = HttpStatus::LLCORE; + status1.mStatus = HE_REPLY_ERROR; + status2.mType = HttpStatus::LLCORE; + status2.mStatus= HE_SHUTTING_DOWN; + ensure(status1 != status2); +} + +template <> template <> +void HttpStatusTestObjectType::test<6>() +{ + set_test_name("HttpStatus basic HTTP status encoding"); + + HttpStatus status; + status.mType = 200; + status.mStatus = HE_SUCCESS; + std::string msg = status.toString(); + ensure(msg.empty()); + ensure(bool(status)); + + // Normally a success but application says error + status.mStatus = HE_REPLY_ERROR; + msg = status.toString(); + ensure(! msg.empty()); + ensure(! bool(status)); + ensure(status.toULong() > 1UL); // Biggish number, not a bool-to-ulong + + // Same statuses with distinct success/fail are distinct + status.mType = 200; + status.mStatus = HE_SUCCESS; + HttpStatus status2(200, HE_REPLY_ERROR); + ensure(status != status2); + + // Normally an error but application says okay + status.mType = 406; + status.mStatus = HE_SUCCESS; + msg = status.toString(); + ensure(msg.empty()); + ensure(bool(status)); + + // Different statuses but both successful are distinct + status.mType = 200; + status.mStatus = HE_SUCCESS; + status2.mType = 201; + status2.mStatus = HE_SUCCESS; + ensure(status != status2); + + // Different statuses but both failed are distinct + status.mType = 200; + status.mStatus = HE_REPLY_ERROR; + status2.mType = 201; + status2.mStatus = HE_REPLY_ERROR; + ensure(status != status2); +} + +template <> template <> +void HttpStatusTestObjectType::test<7>() +{ + set_test_name("HttpStatus HTTP error text strings"); + + HttpStatus status(100, HE_REPLY_ERROR); + std::string msg(status.toString()); + ensure(! msg.empty()); // Should be something + ensure(msg == "Continue"); + + status.mStatus = HE_SUCCESS; + msg = status.toString(); + ensure(msg.empty()); // Success is empty + + status.mType = 199; + status.mStatus = HE_REPLY_ERROR; + msg = status.toString(); + ensure(msg == "Unknown error"); + + status.mType = 505; // Last defined string + status.mStatus = HE_REPLY_ERROR; + msg = status.toString(); + ensure(msg == "HTTP Version not supported"); + + status.mType = 506; // One beyond + status.mStatus = HE_REPLY_ERROR; + msg = status.toString(); + ensure(msg == "Unknown error"); + + status.mType = 999; // Last HTTP status + status.mStatus = HE_REPLY_ERROR; + msg = status.toString(); + ensure(msg == "Unknown error"); +} + } // end namespace tut #endif // TEST_HTTP_STATUS_H diff --git a/indra/newview/llappviewer.cpp b/indra/newview/llappviewer.cpp index 1174d108d2..8e6deb9cce 100644 --- a/indra/newview/llappviewer.cpp +++ b/indra/newview/llappviewer.cpp @@ -212,6 +212,11 @@ #include "llmachineid.h" #include "llmainlooprepeater.h" +// LLCore::HTTP +#include "httpcommon.h" +#include "httprequest.h" +#include "httphandler.h" + // *FIX: These extern globals should be cleaned up. // The globals either represent state/config/resource-storage of either // this app, or another 'component' of the viewer. App globals should be @@ -326,6 +331,53 @@ static std::string gLaunchFileOnQuit; // Used on Win32 for other apps to identify our window (eg, win_setup) const char* const VIEWER_WINDOW_CLASSNAME = "Second Life"; +namespace +{ + +// This class manages the lifecyle of the core http library. +// Slightly different style than traditional code but reflects +// the use of handler classes and light-weight interface +// object instances of the new libraries. To be used +// as a singleton and static construction is fine. +class CoreHttp : public LLCore::HttpHandler +{ +public: + CoreHttp(); + ~CoreHttp(); + + // Initialize the LLCore::HTTP library creating service classes + // and starting the servicing thread. Caller is expected to do + // other initializations (SSL mutex, thread hash function) appropriate + // for the application. + void init(); + + // Request that the servicing thread stop servicing requests, + // release resource references and stop. + void requestStop(); + + // Terminate LLCore::HTTP library services. Caller is expected + // to have made a best-effort to shutdown the servicing thread + // by issuing a requestThreadStop() and waiting for completion + // notification that the stop has completed. + void cleanup(); + + // Notification when the stop request is complete. + virtual void onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * response); + +private: + static const F64 MAX_THREAD_WAIT_TIME; + +private: + LLCore::HttpRequest * mRequest; + LLCore::HttpHandle mStopHandle; + F64 mStopRequested; + bool mStopped; +}; + +CoreHttp coreHttpLib; + +} // end anonymous namespace + //-- LLDeferredTaskList ------------------------------------------------------ /** @@ -720,6 +772,9 @@ bool LLAppViewer::init() LLViewerStatsRecorder::initClass(); #endif + // Initialize the non-LLCurl libcurl library + coreHttpLib.init(); + // *NOTE:Mani - LLCurl::initClass is not thread safe. // Called before threads are created. LLCurl::initClass(gSavedSettings.getF32("CurlRequestTimeOut"), @@ -1807,6 +1862,7 @@ bool LLAppViewer::cleanup() // Delete workers first // shotdown all worker threads before deleting them in case of co-dependencies + coreHttpLib.requestStop(); sTextureFetch->shutdown(); sTextureCache->shutdown(); sImageDecodeThread->shutdown(); @@ -1890,6 +1946,9 @@ bool LLAppViewer::cleanup() // *NOTE:Mani - The following call is not thread safe. LLCurl::cleanupClass(); + // Non-LLCurl libcurl library + coreHttpLib.cleanup(); + // If we're exiting to launch an URL, do that here so the screen // is at the right resolution before we launch IE. if (!gLaunchFileOnQuit.empty()) @@ -5267,3 +5326,103 @@ void LLAppViewer::metricsSend(bool enable_reporting) gViewerAssetStatsMain->reset(); } +namespace +{ + +const F64 CoreHttp::MAX_THREAD_WAIT_TIME(10.0); + +CoreHttp::CoreHttp() + : mRequest(NULL), + mStopHandle(LLCORE_HTTP_HANDLE_INVALID), + mStopRequested(0.0), + mStopped(false) +{} + + +CoreHttp::~CoreHttp() +{ + delete mRequest; + mRequest = NULL; +} + + +void CoreHttp::init() +{ + LLCore::HttpStatus status = LLCore::HttpRequest::createService(); + if (! status) + { + LL_ERRS("Init") << "Failed to initialize HTTP services. Reason: " + << status.toString() + << LL_ENDL; + } + + status = LLCore::HttpRequest::startThread(); + if (! status) + { + LL_ERRS("Init") << "Failed to start HTTP servicing thread. Reason: " + << status.toString() + << LL_ENDL; + } + + mRequest = new LLCore::HttpRequest; +} + + +void CoreHttp::requestStop() +{ + llassert_always(mRequest); + + mStopHandle = mRequest->requestStopThread(this); + if (LLCORE_HTTP_HANDLE_INVALID != mStopHandle) + { + mStopRequested = LLTimer::getTotalSeconds(); + } +} + + +void CoreHttp::cleanup() +{ + if (LLCORE_HTTP_HANDLE_INVALID == mStopHandle) + { + // Should have been started already... + requestStop(); + } + + if (LLCORE_HTTP_HANDLE_INVALID == mStopHandle) + { + LL_WARNS("Cleanup") << "Attempting to cleanup HTTP services without thread shutdown" + << LL_ENDL; + } + else + { + while (! mStopped && LLTimer::getTotalSeconds() < (mStopRequested + MAX_THREAD_WAIT_TIME)) + { + mRequest->update(200); + ms_sleep(50); + } + if (! mStopped) + { + LL_WARNS("Cleanup") << "Attempting to cleanup HTTP services with thread shutdown incomplete" + << LL_ENDL; + } + } + + delete mRequest; + mRequest = NULL; + + LLCore::HttpStatus status = LLCore::HttpRequest::destroyService(); + if (! status) + { + LL_WARNS("Cleanup") << "Failed to shutdown HTTP services, continuing. Reason: " + << status.toString() + << LL_ENDL; + } +} + + +void CoreHttp::onCompleted(LLCore::HttpHandle, LLCore::HttpResponse *) +{ + mStopped = true; +} + +} // end anonymous namespace diff --git a/indra/newview/lltexturefetch.cpp b/indra/newview/lltexturefetch.cpp index f18aa8b4e6..17c68f7c22 100644 --- a/indra/newview/lltexturefetch.cpp +++ b/indra/newview/lltexturefetch.cpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2000&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2010, Linden Research, Inc. + * Copyright (C) 2012, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -33,7 +33,6 @@ #include "lltexturefetch.h" -#include "llcurl.h" #include "lldir.h" #include "llhttpclient.h" #include "llhttpstatuscodes.h" @@ -53,11 +52,17 @@ #include "llviewerassetstats.h" #include "llworld.h" +#include "httprequest.h" +#include "httphandler.h" +#include "httpresponse.h" +#include "bufferarray.h" + + ////////////////////////////////////////////////////////////////////////////// -class LLTextureFetchWorker : public LLWorkerClass +class LLTextureFetchWorker : public LLWorkerClass, public LLCore::HttpHandler + { friend class LLTextureFetch; - friend class HTTPGetResponder; private: class CacheReadResponder : public LLTextureCache::ReadResponder @@ -147,15 +152,14 @@ public: ~LLTextureFetchWorker(); // void relese() { --mActiveCount; } - S32 callbackHttpGet(const LLChannelDescriptors& channels, - const LLIOPipe::buffer_ptr_t& buffer, - bool partial, bool success); + S32 callbackHttpGet(LLCore::HttpResponse * response, + bool partial, bool success); void callbackCacheRead(bool success, LLImageFormatted* image, S32 imagesize, BOOL islocal); void callbackCacheWrite(bool success); void callbackDecoded(bool success, LLImageRaw* raw, LLImageRaw* aux); - void setGetStatus(U32 status, const std::string& reason) + void setGetStatus(LLCore::HttpStatus status, const std::string& reason) { LLMutexLock lock(&mWorkMutex); @@ -167,6 +171,9 @@ public: bool getCanUseHTTP() const { return mCanUseHTTP; } LLTextureFetch & getFetcher() { return *mFetcher; } + + // Inherited from LLCore::HttpHandler + virtual void onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * response); protected: LLTextureFetchWorker(LLTextureFetch* fetcher, const std::string& url, const LLUUID& id, const LLHost& host, @@ -191,9 +198,15 @@ private: void lockWorkMutex() { mWorkMutex.lock(); } void unlockWorkMutex() { mWorkMutex.unlock(); } + void recordTextureStart(bool is_http); + void recordTextureDone(bool is_http); + private: enum e_state // mState { + // *NOTE: Do not change the order/value of state variables, some code + // depends upon specific ordering/adjacency. + // NOTE: Affects LLTextureBar::draw in lltextureview.cpp (debug hack) INVALID = 0, INIT, @@ -244,9 +257,8 @@ private: LLFrameTimer mFetchTimer; LLTextureCache::handle_t mCacheReadHandle; LLTextureCache::handle_t mCacheWriteHandle; - U8* mBuffer; - S32 mBufferSize; S32 mRequestedSize; + S32 mRequestedOffset; S32 mDesiredSize; S32 mFileSize; S32 mCachedSize; @@ -263,7 +275,7 @@ private: S32 mHTTPFailCount; S32 mRetryAttempt; S32 mActiveCount; - U32 mGetStatus; + LLCore::HttpStatus mGetStatus; std::string mGetReason; // Work Data @@ -283,106 +295,17 @@ private: U8 mImageCodec; LLViewerAssetStats::duration_t mMetricsStartTime; + + LLCore::HttpHandle mHttpHandle; + LLCore::BufferArray * mHttpBufferArray; + int mHttpPolicyClass; + bool mHttpActive; + unsigned int mHttpReplySize; + unsigned int mHttpReplyOffset; }; ////////////////////////////////////////////////////////////////////////////// -class HTTPGetResponder : public LLCurl::Responder -{ - LOG_CLASS(HTTPGetResponder); -public: - HTTPGetResponder(LLTextureFetch* fetcher, const LLUUID& id, U64 startTime, S32 requestedSize, U32 offset, bool redir) - : mFetcher(fetcher), mID(id), mStartTime(startTime), mRequestedSize(requestedSize), mOffset(offset), mFollowRedir(redir) - { - } - ~HTTPGetResponder() - { - } - - virtual void completedRaw(U32 status, const std::string& reason, - const LLChannelDescriptors& channels, - const LLIOPipe::buffer_ptr_t& buffer) - { - static LLCachedControl log_to_viewer_log(gSavedSettings,"LogTextureDownloadsToViewerLog"); - static LLCachedControl log_to_sim(gSavedSettings,"LogTextureDownloadsToSimulator"); - static LLCachedControl log_texture_traffic(gSavedSettings,"LogTextureNetworkTraffic") ; - - if (log_to_viewer_log || log_to_sim) - { - mFetcher->mTextureInfo.setRequestStartTime(mID, mStartTime); - U64 timeNow = LLTimer::getTotalTime(); - mFetcher->mTextureInfo.setRequestType(mID, LLTextureInfoDetails::REQUEST_TYPE_HTTP); - mFetcher->mTextureInfo.setRequestSize(mID, mRequestedSize); - mFetcher->mTextureInfo.setRequestOffset(mID, mOffset); - mFetcher->mTextureInfo.setRequestCompleteTimeAndLog(mID, timeNow); - } - - lldebugs << "HTTP COMPLETE: " << mID << llendl; - LLTextureFetchWorker* worker = mFetcher->getWorker(mID); - if (worker) - { - bool success = false; - bool partial = false; - if (HTTP_OK <= status && status < HTTP_MULTIPLE_CHOICES) - { - success = true; - if (HTTP_PARTIAL_CONTENT == status) // partial information - { - partial = true; - } - } - - if (!success) - { - worker->setGetStatus(status, reason); -// llwarns << "CURL GET FAILED, status:" << status << " reason:" << reason << llendl; - } - - S32 data_size = worker->callbackHttpGet(channels, buffer, partial, success); - - if(log_texture_traffic && data_size > 0) - { - LLViewerTexture* tex = LLViewerTextureManager::findTexture(mID) ; - if(tex) - { - gTotalTextureBytesPerBoostLevel[tex->getBoostLevel()] += data_size ; - } - } - - mFetcher->removeFromHTTPQueue(mID, data_size); - - if (worker->mMetricsStartTime) - { - LLViewerAssetStatsFF::record_response_thread1(LLViewerAssetType::AT_TEXTURE, - true, - LLImageBase::TYPE_AVATAR_BAKE == worker->mType, - LLViewerAssetStatsFF::get_timestamp() - worker->mMetricsStartTime); - worker->mMetricsStartTime = 0; - } - LLViewerAssetStatsFF::record_dequeue_thread1(LLViewerAssetType::AT_TEXTURE, - true, - LLImageBase::TYPE_AVATAR_BAKE == worker->mType); - } - else - { - mFetcher->removeFromHTTPQueue(mID); - llwarns << "Worker not found: " << mID << llendl; - } - } - - virtual bool followRedir() - { - return mFollowRedir; - } - -private: - LLTextureFetch* mFetcher; - LLUUID mID; - U64 mStartTime; - S32 mRequestedSize; - U32 mOffset; - bool mFollowRedir; -}; ////////////////////////////////////////////////////////////////////////////// @@ -639,6 +562,7 @@ LLTextureFetchWorker::LLTextureFetchWorker(LLTextureFetch* fetcher, S32 discard, // Desired discard S32 size) // Desired size : LLWorkerClass(fetcher, "TextureFetch"), + LLCore::HttpHandler(), mState(INIT), mWriteToCacheState(NOT_WRITE), mFetcher(fetcher), @@ -655,9 +579,8 @@ LLTextureFetchWorker::LLTextureFetchWorker(LLTextureFetch* fetcher, mDecodedDiscard(-1), mCacheReadHandle(LLTextureCache::nullHandle()), mCacheWriteHandle(LLTextureCache::nullHandle()), - mBuffer(NULL), - mBufferSize(0), mRequestedSize(0), + mRequestedOffset(0), mDesiredSize(TEXTURE_CACHE_ENTRY_SIZE), mFileSize(0), mCachedSize(0), @@ -673,13 +596,18 @@ LLTextureFetchWorker::LLTextureFetchWorker(LLTextureFetch* fetcher, mHTTPFailCount(0), mRetryAttempt(0), mActiveCount(0), - mGetStatus(0), mWorkMutex(NULL), mFirstPacket(0), mLastPacket(-1), mTotalPackets(0), mImageCodec(IMG_CODEC_INVALID), - mMetricsStartTime(0) + mMetricsStartTime(0), + mHttpHandle(LLCORE_HTTP_HANDLE_INVALID), + mHttpBufferArray(NULL), + mHttpPolicyClass(LLCore::HttpRequest::DEFAULT_POLICY_ID), + mHttpActive(false), + mHttpReplySize(0U), + mHttpReplyOffset(0U) { mCanUseNET = mUrl.empty() ; @@ -701,6 +629,11 @@ LLTextureFetchWorker::~LLTextureFetchWorker() // << " Requested=" << mRequestedDiscard // << " Desired=" << mDesiredDiscard << llendl; llassert_always(!haveWork()); + if (mHttpActive) + { + LL_WARNS("Texture") << "Deleting worker object while HTTP request is active." + << LL_ENDL; + } lockWorkMutex(); if (mCacheReadHandle != LLTextureCache::nullHandle() && mFetcher->mTextureCache) { @@ -714,6 +647,11 @@ LLTextureFetchWorker::~LLTextureFetchWorker() clearPackets(); unlockWorkMutex(); mFetcher->removeFromHTTPQueue(mID); + if (mHttpBufferArray) + { + mHttpBufferArray->release(); + mHttpBufferArray = NULL; + } } void LLTextureFetchWorker::clearPackets() @@ -797,7 +735,7 @@ void LLTextureFetchWorker::setDesiredDiscard(S32 discard, S32 size) if ((prioritize && mState == INIT) || mState == DONE) { mState = INIT; - U32 work_priority = mWorkPriority | LLWorkerThread::PRIORITY_HIGH; + U32 work_priority = mWorkPriority | LLWorkerThread::PRIORITY_LOW; setPriority(work_priority); } } @@ -810,16 +748,18 @@ void LLTextureFetchWorker::setImagePriority(F32 priority) { mImagePriority = priority; calcWorkPriority(); - U32 work_priority = mWorkPriority | (getPriority() & LLWorkerThread::PRIORITY_HIGHBITS); + U32 work_priority = mWorkPriority | LLWorkerThread::PRIORITY_LOW; setPriority(work_priority); } } void LLTextureFetchWorker::resetFormattedData() { - FREE_MEM(LLImageBase::getPrivatePool(), mBuffer); - mBuffer = NULL; - mBufferSize = 0; + if (mHttpBufferArray) + { + mHttpBufferArray->release(); + mHttpBufferArray = NULL; + } if (mFormattedImage.notNull()) { mFormattedImage->deleteData(); @@ -875,6 +815,14 @@ bool LLTextureFetchWorker::doWork(S32 param) mFetchTimer.reset(); } + static LLUUID last_id; + if (mID != last_id) + { + // LL_WARNS("Texture") << "DOWORK SWITCH: " << last_id << " to: " << mID + // << LL_ENDL; + last_id = mID; + } + if (mState == INIT) { mRawImage = NULL ; @@ -882,15 +830,18 @@ bool LLTextureFetchWorker::doWork(S32 param) mLoadedDiscard = -1; mDecodedDiscard = -1; mRequestedSize = 0; + mRequestedOffset = 0; mFileSize = 0; mCachedSize = 0; mLoaded = FALSE; mSentRequest = UNSENT; mDecoded = FALSE; mWritten = FALSE; - FREE_MEM(LLImageBase::getPrivatePool(), mBuffer); - mBuffer = NULL; - mBufferSize = 0; + if (mHttpBufferArray) + { + mHttpBufferArray->release(); + mHttpBufferArray = NULL; + } mHaveAllData = FALSE; clearPackets(); // TODO: Shouldn't be necessary mCacheReadHandle = LLTextureCache::nullHandle(); @@ -904,6 +855,7 @@ bool LLTextureFetchWorker::doWork(S32 param) if (mState == LOAD_FROM_TEXTURE_CACHE) { + setPriority(0); // Set priority first since Responder may change it if (mCacheReadHandle == LLTextureCache::nullHandle()) { U32 cache_priority = mWorkPriority; @@ -919,8 +871,6 @@ bool LLTextureFetchWorker::doWork(S32 param) if (mUrl.compare(0, 7, "file://") == 0) { - setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); // Set priority first since Responder may change it - // read file from local disk std::string filename = mUrl.substr(7, std::string::npos); CacheReadResponder* responder = new CacheReadResponder(mFetcher, mID, mFormattedImage); @@ -929,8 +879,6 @@ bool LLTextureFetchWorker::doWork(S32 param) } else if (mUrl.empty()) { - setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); // Set priority first since Responder may change it - CacheReadResponder* responder = new CacheReadResponder(mFetcher, mID, mFormattedImage); mCacheReadHandle = mFetcher->mTextureCache->readFromCache(mID, cache_priority, offset, size, responder); @@ -942,12 +890,12 @@ bool LLTextureFetchWorker::doWork(S32 param) // *TODO:?remove this warning llwarns << "Unknown URL Type: " << mUrl << llendl; } - setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); + setPriority(LLWorkerThread::PRIORITY_HIGH); mState = SEND_HTTP_REQ; } else { - setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); + setPriority(LLWorkerThread::PRIORITY_HIGH); mState = LOAD_FROM_NETWORK; } } @@ -959,6 +907,7 @@ bool LLTextureFetchWorker::doWork(S32 param) { mCacheReadHandle = LLTextureCache::nullHandle(); mState = CACHE_POST; + setPriority(LLWorkerThread::PRIORITY_HIGH); // fall through } else @@ -982,6 +931,7 @@ bool LLTextureFetchWorker::doWork(S32 param) llassert_always(mFormattedImage->getDataSize() > 0); mLoadedDiscard = mDesiredDiscard; mState = DECODE_IMAGE; + setPriority(LLWorkerThread::PRIORITY_HIGH); mWriteToCacheState = NOT_WRITE ; LL_DEBUGS("Texture") << mID << ": Cached. Bytes: " << mFormattedImage->getDataSize() << " Size: " << llformat("%dx%d",mFormattedImage->getWidth(),mFormattedImage->getHeight()) @@ -999,6 +949,7 @@ bool LLTextureFetchWorker::doWork(S32 param) else { LL_DEBUGS("Texture") << mID << ": Not in Cache" << LL_ENDL; + setPriority(LLWorkerThread::PRIORITY_HIGH); mState = LOAD_FROM_NETWORK; } // fall through @@ -1009,6 +960,7 @@ bool LLTextureFetchWorker::doWork(S32 param) { static LLCachedControl use_http(gSavedSettings,"ImagePipelineUseHTTP"); + setPriority(0); // if (mHost != LLHost::invalid) get_url = false; if ( use_http && mCanUseHTTP && mUrl.empty())//get http url. { @@ -1040,8 +992,8 @@ bool LLTextureFetchWorker::doWork(S32 param) } if (mCanUseHTTP && !mUrl.empty()) { - mState = LLTextureFetchWorker::SEND_HTTP_REQ; - setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); + mState = SEND_HTTP_REQ; + setPriority(LLWorkerThread::PRIORITY_HIGH); if(mWriteToCacheState != NOT_WRITE) { mWriteToCacheState = CAN_WRITE ; @@ -1057,14 +1009,7 @@ bool LLTextureFetchWorker::doWork(S32 param) mRequestedDiscard = mDesiredDiscard; mSentRequest = QUEUED; mFetcher->addToNetworkQueue(this); - if (! mMetricsStartTime) - { - mMetricsStartTime = LLViewerAssetStatsFF::get_timestamp(); - } - LLViewerAssetStatsFF::record_enqueue_thread1(LLViewerAssetType::AT_TEXTURE, - false, - LLImageBase::TYPE_AVATAR_BAKE == mType); - setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); + recordTextureStart(false); return false; } @@ -1074,12 +1019,7 @@ bool LLTextureFetchWorker::doWork(S32 param) //llassert_always(mFetcher->mNetworkQueue.find(mID) != mFetcher->mNetworkQueue.end()); // Make certain this is in the network queue //mFetcher->addToNetworkQueue(this); - //if (! mMetricsStartTime) - //{ - // mMetricsStartTime = LLViewerAssetStatsFF::get_timestamp(); - //} - //LLViewerAssetStatsFF::record_enqueue_thread1(LLViewerAssetType::AT_TEXTURE, false, - // LLImageBase::TYPE_AVATAR_BAKE == mType); + //recordTextureStart(false); //setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); return false; } @@ -1087,6 +1027,7 @@ bool LLTextureFetchWorker::doWork(S32 param) if (mState == LOAD_FROM_SIMULATOR) { + setPriority(0); if (mFormattedImage.isNull()) { mFormattedImage = new LLImageJ2C; @@ -1101,39 +1042,22 @@ bool LLTextureFetchWorker::doWork(S32 param) // llwarns << "processSimulatorPackets() failed to load buffer" << llendl; return true; // failed } - setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); + setPriority(LLWorkerThread::PRIORITY_HIGH); mState = DECODE_IMAGE; mWriteToCacheState = SHOULD_WRITE; - - if (mMetricsStartTime) - { - LLViewerAssetStatsFF::record_response_thread1(LLViewerAssetType::AT_TEXTURE, - false, - LLImageBase::TYPE_AVATAR_BAKE == mType, - LLViewerAssetStatsFF::get_timestamp() - mMetricsStartTime); - mMetricsStartTime = 0; - } - LLViewerAssetStatsFF::record_dequeue_thread1(LLViewerAssetType::AT_TEXTURE, - false, - LLImageBase::TYPE_AVATAR_BAKE == mType); + recordTextureDone(false); } else { mFetcher->addToNetworkQueue(this); // failsafe - if (! mMetricsStartTime) - { - mMetricsStartTime = LLViewerAssetStatsFF::get_timestamp(); - } - LLViewerAssetStatsFF::record_enqueue_thread1(LLViewerAssetType::AT_TEXTURE, - false, - LLImageBase::TYPE_AVATAR_BAKE == mType); - setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); + recordTextureStart(false); } return false; } if (mState == SEND_HTTP_REQ) { + setPriority(0); if(mCanUseHTTP) { //NOTE: @@ -1159,6 +1083,7 @@ bool LLTextureFetchWorker::doWork(S32 param) { // We already have all the data, just decode it mLoadedDiscard = mFormattedImage->getDiscardLevel(); + setPriority(LLWorkerThread::PRIORITY_HIGH); mState = DECODE_IMAGE; return false; } @@ -1171,44 +1096,48 @@ bool LLTextureFetchWorker::doWork(S32 param) mRequestedSize = mDesiredSize; mRequestedDiscard = mDesiredDiscard; mRequestedSize -= cur_size; - S32 offset = cur_size; - mBufferSize = cur_size; // This will get modified by callbackHttpGet() + mRequestedOffset = cur_size; - bool res = false; + mHttpHandle = LLCORE_HTTP_HANDLE_INVALID; if (!mUrl.empty()) { mLoaded = FALSE; - mGetStatus = 0; + mGetStatus = LLCore::HttpStatus(); mGetReason.clear(); - LL_DEBUGS("Texture") << "HTTP GET: " << mID << " Offset: " << offset + LL_DEBUGS("Texture") << "HTTP GET: " << mID << " Offset: " << mRequestedOffset << " Bytes: " << mRequestedSize << " Bandwidth(kbps): " << mFetcher->getTextureBandwidth() << "/" << mFetcher->mMaxBandwidth << LL_ENDL; - setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); - mState = WAIT_HTTP_REQ; - - mFetcher->addToHTTPQueue(mID); - if (! mMetricsStartTime) - { - mMetricsStartTime = LLViewerAssetStatsFF::get_timestamp(); - } - LLViewerAssetStatsFF::record_enqueue_thread1(LLViewerAssetType::AT_TEXTURE, - true, - LLImageBase::TYPE_AVATAR_BAKE == mType); +// LL_WARNS("Texture") << "HTTP GET: " << mID << " Offset: " << mRequestedOffset +// << " Bytes: " << mRequestedSize +// << " Bandwidth(kbps): " << mFetcher->getTextureBandwidth() << "/" << mFetcher->mMaxBandwidth +// << LL_ENDL; // Will call callbackHttpGet when curl request completes - std::vector headers; - headers.push_back("Accept: image/x-j2c"); - res = mFetcher->mCurlGetRequest->getByteRange(mUrl, headers, offset, mRequestedSize, - new HTTPGetResponder(mFetcher, mID, LLTimer::getTotalTime(), mRequestedSize, offset, true)); + // *FIXME: enable redirection follow + mHttpHandle = mFetcher->mHttpRequest->requestGetByteRange(mHttpPolicyClass, + mRequestedPriority, + mUrl, + mRequestedOffset, + mRequestedSize, + mFetcher->mHttpOptions, + mFetcher->mHttpHeaders, + this); } - if (!res) + if (LLCORE_HTTP_HANDLE_INVALID == mHttpHandle) { llwarns << "HTTP GET request failed for " << mID << llendl; resetFormattedData(); ++mHTTPFailCount; return true; // failed } + + mHttpActive = true; + mFetcher->addToHTTPQueue(mID); + recordTextureStart(true); + setPriority(LLWorkerThread::PRIORITY_HIGH); + mState = WAIT_HTTP_REQ; + // fall through } else //can not use http fetch. @@ -1219,27 +1148,28 @@ bool LLTextureFetchWorker::doWork(S32 param) if (mState == WAIT_HTTP_REQ) { + setPriority(0); if (mLoaded) { S32 cur_size = mFormattedImage.notNull() ? mFormattedImage->getDataSize() : 0; if (mRequestedSize < 0) { S32 max_attempts; - if (mGetStatus == HTTP_NOT_FOUND) + if (mGetStatus == LLCore::HttpStatus(HTTP_NOT_FOUND, LLCore::HE_REPLY_ERROR)) { mHTTPFailCount = max_attempts = 1; // Don't retry llwarns << "Texture missing from server (404): " << mUrl << llendl; //roll back to try UDP - if(mCanUseNET) + if (mCanUseNET) { - mState = INIT ; - mCanUseHTTP = false ; - setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); - return false ; + mState = INIT; + mCanUseHTTP = false; + setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); + return false; } } - else if (mGetStatus == HTTP_SERVICE_UNAVAILABLE) + else if (mGetStatus == LLCore::HttpStatus(HTTP_SERVICE_UNAVAILABLE, LLCore::HE_REPLY_ERROR)) { // *TODO: Should probably introduce a timer here to delay future HTTP requsts // for a short time (~1s) to ease server load? Ideally the server would queue @@ -1254,7 +1184,7 @@ bool LLTextureFetchWorker::doWork(S32 param) max_attempts = HTTP_MAX_RETRY_COUNT + 1; ++mHTTPFailCount; llinfos << "HTTP GET failed for: " << mUrl - << " Status: " << mGetStatus << " Reason: '" << mGetReason << "'" + << " Status: " << mGetStatus.toULong() << " Reason: '" << mGetReason << "'" << " Attempt:" << mHTTPFailCount+1 << "/" << max_attempts << llendl; } @@ -1264,12 +1194,14 @@ bool LLTextureFetchWorker::doWork(S32 param) { // Use available data mLoadedDiscard = mFormattedImage->getDiscardLevel(); + setPriority(LLWorkerThread::PRIORITY_HIGH); mState = DECODE_IMAGE; return false; } else { resetFormattedData(); + setPriority(LLWorkerThread::PRIORITY_HIGH); mState = DONE; return true; // failed } @@ -1281,17 +1213,25 @@ bool LLTextureFetchWorker::doWork(S32 param) } } - llassert_always(mBufferSize == cur_size + mRequestedSize); - if(!mBufferSize)//no data received. + if (! mHttpBufferArray || ! mHttpBufferArray->size()) { - FREE_MEM(LLImageBase::getPrivatePool(), mBuffer); - mBuffer = NULL; + // no data received. + if (mHttpBufferArray) + { + mHttpBufferArray->release(); + mHttpBufferArray = NULL; + } - //abort. + // abort. + setPriority(LLWorkerThread::PRIORITY_HIGH); mState = DONE; return true; } + const S32 append_size(mHttpBufferArray->size()); + const S32 total_size(cur_size + append_size); + llassert_always(append_size == mRequestedSize); + if (mFormattedImage.isNull()) { // For now, create formatted image based on extension @@ -1305,49 +1245,52 @@ bool LLTextureFetchWorker::doWork(S32 param) if (mHaveAllData && mRequestedDiscard == 0) //the image file is fully loaded. { - mFileSize = mBufferSize; + mFileSize = total_size; } else //the file size is unknown. { - mFileSize = mBufferSize + 1 ; //flag the file is not fully loaded. + mFileSize = total_size + 1 ; //flag the file is not fully loaded. } - U8* buffer = (U8*)ALLOCATE_MEM(LLImageBase::getPrivatePool(), mBufferSize); + U8 * buffer = (U8 *) ALLOCATE_MEM(LLImageBase::getPrivatePool(), total_size); if (cur_size > 0) { memcpy(buffer, mFormattedImage->getData(), cur_size); } - memcpy(buffer + cur_size, mBuffer, mRequestedSize); // append + mHttpBufferArray->seek(0); + mHttpBufferArray->read((char *) buffer + cur_size, append_size); + // NOTE: setData releases current data and owns new data (buffer) - mFormattedImage->setData(buffer, mBufferSize); - // delete temp data - FREE_MEM(LLImageBase::getPrivatePool(), mBuffer); // Note: not 'buffer' (assigned in setData()) - mBuffer = NULL; - mBufferSize = 0; + mFormattedImage->setData(buffer, total_size); + + // Done with buffer array + mHttpBufferArray->release(); + mHttpBufferArray = NULL; + mLoadedDiscard = mRequestedDiscard; + setPriority(LLWorkerThread::PRIORITY_HIGH); mState = DECODE_IMAGE; - if(mWriteToCacheState != NOT_WRITE) + if (mWriteToCacheState != NOT_WRITE) { mWriteToCacheState = SHOULD_WRITE ; } - setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); return false; } else { - setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); return false; } } if (mState == DECODE_IMAGE) { + setPriority(0); static LLCachedControl textures_decode_disabled(gSavedSettings,"TextureDecodeDisabled"); - if(textures_decode_disabled) + if (textures_decode_disabled) { // for debug use, don't decode mState = DONE; - setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); + setPriority(LLWorkerThread::PRIORITY_HIGH); return true; } @@ -1355,7 +1298,7 @@ bool LLTextureFetchWorker::doWork(S32 param) { // We aborted, don't decode mState = DONE; - setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); + setPriority(LLWorkerThread::PRIORITY_HIGH); return true; } @@ -1365,7 +1308,7 @@ bool LLTextureFetchWorker::doWork(S32 param) //abort, don't decode mState = DONE; - setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); + setPriority(LLWorkerThread::PRIORITY_HIGH); return true; } if (mLoadedDiscard < 0) @@ -1374,10 +1317,10 @@ bool LLTextureFetchWorker::doWork(S32 param) //abort, don't decode mState = DONE; - setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); + setPriority(LLWorkerThread::PRIORITY_HIGH); return true; } - setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); // Set priority first since Responder may change it + mRawImage = NULL; mAuxImage = NULL; llassert_always(mFormattedImage.notNull()); @@ -1394,6 +1337,7 @@ bool LLTextureFetchWorker::doWork(S32 param) if (mState == DECODE_IMAGE_UPDATE) { + setPriority(0); if (mDecoded) { if (mDecodedDiscard < 0) @@ -1406,13 +1350,14 @@ bool LLTextureFetchWorker::doWork(S32 param) llassert_always(mDecodeHandle == 0); mFormattedImage = NULL; ++mRetryAttempt; - setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); + setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); mState = INIT; return false; } else { // llwarns << "UNABLE TO LOAD TEXTURE: " << mID << " RETRIES: " << mRetryAttempt << llendl; + setPriority(LLWorkerThread::PRIORITY_HIGH); mState = DONE; // failed } } @@ -1421,7 +1366,7 @@ bool LLTextureFetchWorker::doWork(S32 param) llassert_always(mRawImage.notNull()); LL_DEBUGS("Texture") << mID << ": Decoded. Discard: " << mDecodedDiscard << " Raw Image: " << llformat("%dx%d",mRawImage->getWidth(),mRawImage->getHeight()) << LL_ENDL; - setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); + setPriority(LLWorkerThread::PRIORITY_HIGH); mState = WRITE_TO_CACHE; } // fall through @@ -1434,10 +1379,12 @@ bool LLTextureFetchWorker::doWork(S32 param) if (mState == WRITE_TO_CACHE) { + setPriority(0); if (mWriteToCacheState != SHOULD_WRITE || mFormattedImage.isNull()) { // If we're in a local cache or we didn't actually receive any new data, // or we failed to load anything, skip + setPriority(LLWorkerThread::PRIORITY_HIGH); mState = DONE; return false; } @@ -1457,6 +1404,7 @@ bool LLTextureFetchWorker::doWork(S32 param) setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); // Set priority first since Responder may change it U32 cache_priority = mWorkPriority; mWritten = FALSE; + setPriority(LLWorkerThread::PRIORITY_HIGH); mState = WAIT_ON_WRITE; CacheWriteResponder* responder = new CacheWriteResponder(mFetcher, mID); mCacheWriteHandle = mFetcher->mTextureCache->writeToCache(mID, cache_priority, @@ -1467,8 +1415,10 @@ bool LLTextureFetchWorker::doWork(S32 param) if (mState == WAIT_ON_WRITE) { + setPriority(0); if (writeToCacheComplete()) { + setPriority(LLWorkerThread::PRIORITY_HIGH); mState = DONE; // fall through } @@ -1487,16 +1437,15 @@ bool LLTextureFetchWorker::doWork(S32 param) if (mState == DONE) { + setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); if (mDecodedDiscard >= 0 && mDesiredDiscard < mDecodedDiscard) { // More data was requested, return to INIT mState = INIT; - setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); return false; } else { - setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); return true; } } @@ -1504,6 +1453,71 @@ bool LLTextureFetchWorker::doWork(S32 param) return false; } +// virtual +void LLTextureFetchWorker::onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * response) +{ + static LLCachedControl log_to_viewer_log(gSavedSettings, "LogTextureDownloadsToViewerLog"); + static LLCachedControl log_to_sim(gSavedSettings, "LogTextureDownloadsToSimulator"); + static LLCachedControl log_texture_traffic(gSavedSettings, "LogTextureNetworkTraffic") ; + + mHttpActive = false; + + if (log_to_viewer_log || log_to_sim) + { + U64 timeNow = LLTimer::getTotalTime(); + mFetcher->mTextureInfo.setRequestStartTime(mID, mMetricsStartTime); + mFetcher->mTextureInfo.setRequestType(mID, LLTextureInfoDetails::REQUEST_TYPE_HTTP); + mFetcher->mTextureInfo.setRequestSize(mID, mRequestedSize); + mFetcher->mTextureInfo.setRequestOffset(mID, mRequestedOffset); + mFetcher->mTextureInfo.setRequestCompleteTimeAndLog(mID, timeNow); + } + + bool success = true; + bool partial = false; + LLCore::HttpStatus status(response->getStatus()); + + lldebugs << "HTTP COMPLETE: " << mID + << " status: " << status.toULong() << " '" << status.toString() << "'" + << llendl; + unsigned int offset(0), length(0); + response->getRange(&offset, &length); +// llwarns << "HTTP COMPLETE: " << mID << " handle: " << handle +// << " status: " << status.toULong() << " '" << status.toString() << "'" +// << " req offset: " << mRequestedOffset << " req length: " << mRequestedSize +// << " offset: " << offset << " length: " << length +// << llendl; + + if (! status) + { + success = false; + std::string reason(status.toString()); + setGetStatus(status, reason); + llwarns << "CURL GET FAILED, status:" << status.toULong() << " reason:" << reason << llendl; + } + else + { + static const LLCore::HttpStatus par_status(LLCore::HttpStatus(HTTP_PARTIAL_CONTENT, LLCore::HE_SUCCESS)); + + partial = (par_status == status); + } + + S32 data_size = callbackHttpGet(response, partial, success); + + if (log_texture_traffic && data_size > 0) + { + LLViewerTexture* tex = LLViewerTextureManager::findTexture(mID); + if (tex) + { + gTotalTextureBytesPerBoostLevel[tex->getBoostLevel()] += data_size ; + } + } + + mFetcher->removeFromHTTPQueue(mID); + + recordTextureDone(true); +} + + // Called from MAIN thread void LLTextureFetchWorker::endWork(S32 param, bool aborted) { @@ -1537,6 +1551,14 @@ void LLTextureFetchWorker::finishWork(S32 param, bool completed) bool LLTextureFetchWorker::deleteOK() { bool delete_ok = true; + + if (mHttpActive) + { + // HTTP library has a pointer to this worker + // and will dereference it to do notification. + delete_ok = false; + } + // Allow any pending reads or writes to complete if (mCacheReadHandle != LLTextureCache::nullHandle()) { @@ -1642,9 +1664,8 @@ bool LLTextureFetchWorker::processSimulatorPackets() ////////////////////////////////////////////////////////////////////////////// -S32 LLTextureFetchWorker::callbackHttpGet(const LLChannelDescriptors& channels, - const LLIOPipe::buffer_ptr_t& buffer, - bool partial, bool success) +S32 LLTextureFetchWorker::callbackHttpGet(LLCore::HttpResponse * response, + bool partial, bool success) { S32 data_size = 0 ; @@ -1664,15 +1685,22 @@ S32 LLTextureFetchWorker::callbackHttpGet(const LLChannelDescriptors& channels, if (success) { // get length of stream: - data_size = buffer->countAfter(channels.in(), NULL); - + LLCore::BufferArray * body(response->getBody()); + data_size = body ? body->size() : 0; + LL_DEBUGS("Texture") << "HTTP RECEIVED: " << mID.asString() << " Bytes: " << data_size << LL_ENDL; if (data_size > 0) { // *TODO: set the formatted image data here directly to avoid the copy - mBuffer = (U8*)ALLOCATE_MEM(LLImageBase::getPrivatePool(), data_size); - buffer->readAfter(channels.in(), NULL, mBuffer, data_size); - mBufferSize += data_size; + // *FIXME: deal with actual offset and actual datasize, don't assume + // server gave exactly what was asked for. + + llassert_always(NULL == mHttpBufferArray); + + // Hold on to body for later copy + body->addRef(); + mHttpBufferArray = body; + if (data_size < mRequestedSize && mRequestedDiscard == 0) { mHaveAllData = TRUE; @@ -1684,7 +1712,6 @@ S32 LLTextureFetchWorker::callbackHttpGet(const LLChannelDescriptors& channels, mHaveAllData = TRUE; llassert_always(mDecodeHandle == 0); mFormattedImage = NULL; // discard any previous data we had - mBufferSize = data_size; } } else @@ -1700,7 +1727,7 @@ S32 LLTextureFetchWorker::callbackHttpGet(const LLChannelDescriptors& channels, mRequestedSize = -1; // error } mLoaded = TRUE; - setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); + setPriority(LLWorkerThread::PRIORITY_HIGH); return data_size ; } @@ -1729,7 +1756,7 @@ void LLTextureFetchWorker::callbackCacheRead(bool success, LLImageFormatted* ima } } mLoaded = TRUE; - setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); + setPriority(LLWorkerThread::PRIORITY_HIGH); } void LLTextureFetchWorker::callbackCacheWrite(bool success) @@ -1741,7 +1768,7 @@ void LLTextureFetchWorker::callbackCacheWrite(bool success) return; } mWritten = TRUE; - setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); + setPriority(LLWorkerThread::PRIORITY_HIGH); } ////////////////////////////////////////////////////////////////////////////// @@ -1779,7 +1806,7 @@ void LLTextureFetchWorker::callbackDecoded(bool success, LLImageRaw* raw, LLImag } mDecoded = TRUE; // llinfos << mID << " : DECODE COMPLETE " << llendl; - setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); + setPriority(LLWorkerThread::PRIORITY_HIGH); } ////////////////////////////////////////////////////////////////////////////// @@ -1806,6 +1833,34 @@ bool LLTextureFetchWorker::writeToCacheComplete() } +void LLTextureFetchWorker::recordTextureStart(bool is_http) +{ + if (! mMetricsStartTime) + { + mMetricsStartTime = LLViewerAssetStatsFF::get_timestamp(); + } + LLViewerAssetStatsFF::record_enqueue_thread1(LLViewerAssetType::AT_TEXTURE, + is_http, + LLImageBase::TYPE_AVATAR_BAKE == mType); +} + + +void LLTextureFetchWorker::recordTextureDone(bool is_http) +{ + if (mMetricsStartTime) + { + LLViewerAssetStatsFF::record_response_thread1(LLViewerAssetType::AT_TEXTURE, + is_http, + LLImageBase::TYPE_AVATAR_BAKE == mType, + LLViewerAssetStatsFF::get_timestamp() - mMetricsStartTime); + mMetricsStartTime = 0; + } + LLViewerAssetStatsFF::record_dequeue_thread1(LLViewerAssetType::AT_TEXTURE, + is_http, + LLImageBase::TYPE_AVATAR_BAKE == mType); +} + + ////////////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////////// // public @@ -1823,17 +1878,26 @@ LLTextureFetch::LLTextureFetch(LLTextureCache* cache, LLImageDecodeThread* image mTextureBandwidth(0), mHTTPTextureBits(0), mTotalHTTPRequests(0), - mCurlGetRequest(NULL), - mQAMode(qa_mode) + mQAMode(qa_mode), + mHttpRequest(NULL), + mHttpOptions(NULL), + mHttpHeaders(NULL) { mCurlPOSTRequestCount = 0; mMaxBandwidth = gSavedSettings.getF32("ThrottleBandwidthKBPS"); mTextureInfo.setUpLogging(gSavedSettings.getBOOL("LogTextureDownloadsToViewerLog"), gSavedSettings.getBOOL("LogTextureDownloadsToSimulator"), gSavedSettings.getU32("TextureLoggingThreshold")); + + mHttpRequest = new LLCore::HttpRequest; + mHttpOptions = new LLCore::HttpOptions; + mHttpHeaders = new LLCore::HttpHeaders; + mHttpHeaders->mHeaders.push_back("Accept: image/x-j2c"); } LLTextureFetch::~LLTextureFetch() { - clearDeleteList() ; + cancelHttpRequests(); + + clearDeleteList(); while (! mCommands.empty()) { @@ -1841,7 +1905,22 @@ LLTextureFetch::~LLTextureFetch() mCommands.erase(mCommands.begin()); delete req; } - + + if (mHttpOptions) + { + mHttpOptions->release(); + mHttpOptions = NULL; + } + + if (mHttpHeaders) + { + mHttpHeaders->release(); + mHttpHeaders = NULL; + } + + delete mHttpRequest; + mHttpRequest = NULL; + // ~LLQueuedThread() called here } @@ -1984,6 +2063,28 @@ void LLTextureFetch::removeFromHTTPQueue(const LLUUID& id, S32 received_size) mHTTPTextureBits += received_size * 8; // Approximate - does not include header bits } +void LLTextureFetch::cancelHttpRequests() +{ + for (queue_t::iterator iter(mHTTPTextureQueue.begin()); + mHTTPTextureQueue.end() != iter; + ++iter) + { + LLTextureFetchWorker* worker = getWorker(*iter); + if (worker && worker->mHttpActive) + { + mHttpRequest->requestCancel(worker->mHttpHandle, NULL); + } + } + + // *FIXME: Do this better with less time wasting. + int tries(10); + while (! mHTTPTextureQueue.empty() && --tries) + { + mHttpRequest->update(100); + ms_sleep(100); + } +} + void LLTextureFetch::deleteRequest(const LLUUID& id, bool cancel) { lockQueue() ; @@ -2194,11 +2295,21 @@ void LLTextureFetch::commonUpdate() cmdDoWork(); // Update Curl on same thread as mCurlGetRequest was constructed - S32 processed = mCurlGetRequest->process(); + LLCore::HttpStatus status = mHttpRequest->update(200); + if (! status) + { + LL_INFOS_ONCE("Texture") << "Problem during HTTP servicing. Reason: " + << status.toString() + << LL_ENDL; + } + +#if 0 + // *FIXME: maybe implement this another way... if (processed > 0) { lldebugs << "processed: " << processed << " messages." << llendl; } +#endif } @@ -2256,22 +2367,22 @@ void LLTextureFetch::shutDownImageDecodeThread() // WORKER THREAD void LLTextureFetch::startThread() { - // Construct mCurlGetRequest from Worker Thread - mCurlGetRequest = new LLCurlRequest(); } // WORKER THREAD +// +// This detaches the texture fetch thread from the LLCore +// HTTP library but doesn't stop the thread running in that +// library... void LLTextureFetch::endThread() { - // Destroy mCurlGetRequest from Worker Thread - delete mCurlGetRequest; - mCurlGetRequest = NULL; + cancelHttpRequests(); } // WORKER THREAD void LLTextureFetch::threadedUpdate() { - llassert_always(mCurlGetRequest); + llassert_always(mHttpRequest); // Limit update frequency const F32 PROCESS_TIME = 0.05f; @@ -2579,7 +2690,7 @@ bool LLTextureFetch::receiveImageHeader(const LLHost& host, const LLUUID& id, U8 llassert_always(totalbytes > 0); llassert_always(data_size == FIRST_PACKET_SIZE || data_size == worker->mFileSize); res = worker->insertPacket(0, data, data_size); - worker->setPriority(LLWorkerThread::PRIORITY_HIGH | worker->mWorkPriority); + worker->setPriority(LLWorkerThread::PRIORITY_HIGH); worker->mState = LLTextureFetchWorker::LOAD_FROM_SIMULATOR; worker->unlockWorkMutex(); return res; @@ -2623,7 +2734,7 @@ bool LLTextureFetch::receiveImagePacket(const LLHost& host, const LLUUID& id, U1 if ((worker->mState == LLTextureFetchWorker::LOAD_FROM_SIMULATOR) || (worker->mState == LLTextureFetchWorker::LOAD_FROM_NETWORK)) { - worker->setPriority(LLWorkerThread::PRIORITY_HIGH | worker->mWorkPriority); + worker->setPriority(LLWorkerThread::PRIORITY_HIGH); worker->mState = LLTextureFetchWorker::LOAD_FROM_SIMULATOR; } else @@ -2730,6 +2841,14 @@ void LLTextureFetch::dump() << " STATE: " << worker->sStateDescs[worker->mState] << llendl; } + + llinfos << "LLTextureFetch ACTIVE_HTTP:" << llendl; + for (queue_t::const_iterator iter(mHTTPTextureQueue.begin()); + mHTTPTextureQueue.end() != iter; + ++iter) + { + llinfos << " ID: " << (*iter) << llendl; + } } ////////////////////////////////////////////////////////////////////////////// @@ -2942,6 +3061,8 @@ TFReqSendMetrics::doWork(LLTextureFetch * fetcher) if (! mCapsURL.empty()) { LLCurlRequest::headers_t headers; +#if 0 + // *FIXME: Going to need a post op after all... fetcher->getCurlRequest().post(mCapsURL, headers, merged_llsd, @@ -2950,6 +3071,7 @@ TFReqSendMetrics::doWork(LLTextureFetch * fetcher) report_sequence, LLTextureFetch::svMetricsDataBreak, reporting_started)); +#endif } else { diff --git a/indra/newview/lltexturefetch.h b/indra/newview/lltexturefetch.h index 35df7d816f..402b198246 100644 --- a/indra/newview/lltexturefetch.h +++ b/indra/newview/lltexturefetch.h @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2000&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2010, Linden Research, Inc. + * Copyright (C) 2012, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -31,9 +31,11 @@ #include "llimage.h" #include "lluuid.h" #include "llworkerthread.h" -#include "llcurl.h" #include "lltextureinfo.h" #include "llapr.h" +#include "httprequest.h" +#include "httpoptions.h" +#include "httpheaders.h" class LLViewerTexture; class LLTextureFetchWorker; @@ -98,7 +100,7 @@ public: LLViewerAssetStats * main_stats); void commandDataBreak(); - LLCurlRequest & getCurlRequest() { return *mCurlGetRequest; } + LLCore::HttpRequest & getHttpRequest() { return *mHttpRequest; } bool isQAMode() const { return mQAMode; } @@ -112,7 +114,8 @@ protected: void addToHTTPQueue(const LLUUID& id); void removeFromHTTPQueue(const LLUUID& id, S32 received_size = 0); void removeRequest(LLTextureFetchWorker* worker, bool cancel); - + void cancelHttpRequests(); + // Overrides from the LLThread tree bool runCondition(); @@ -166,7 +169,6 @@ private: LLTextureCache* mTextureCache; LLImageDecodeThread* mImageDecodeThread; - LLCurlRequest* mCurlGetRequest; // Map of all requests by UUID typedef std::map map_t; @@ -203,6 +205,13 @@ private: // use the LLCurl module's request counter as it isn't thread compatible. // *NOTE: Don't mix Atomic and static, apr_initialize must be called first. LLAtomic32 mCurlPOSTRequestCount; + + // Interfaces and objects into the core http library used + // to make our HTTP requests. These replace the various + // LLCurl interfaces used in the past. + LLCore::HttpRequest * mHttpRequest; + LLCore::HttpOptions * mHttpOptions; + LLCore::HttpHeaders * mHttpHeaders; public: // A probabilistically-correct indicator that the current -- cgit v1.3 From 510e1b9e0ec14e98793041b9560694454a9aec2b Mon Sep 17 00:00:00 2001 From: Ansariel Date: Wed, 12 Sep 2012 19:44:53 +0200 Subject: Possible crash fix in LLAgent::doTeleportViaLocation --- indra/newview/llagent.cpp | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'indra/newview') diff --git a/indra/newview/llagent.cpp b/indra/newview/llagent.cpp index 447836910d..11fa50b51a 100755 --- a/indra/newview/llagent.cpp +++ b/indra/newview/llagent.cpp @@ -4045,6 +4045,12 @@ void LLAgent::teleportViaLocation(const LLVector3d& pos_global) void LLAgent::doTeleportViaLocation(const LLVector3d& pos_global) { LLViewerRegion* regionp = getRegion(); + + if (!regionp) + { + return; + } + U64 handle = to_region_handle(pos_global); LLSimInfo* info = LLWorldMap::getInstance()->simInfoFromHandle(handle); if(regionp && info) -- cgit v1.3 From 48631db986f4f16ba6810c4f91844066407d430d Mon Sep 17 00:00:00 2001 From: Nicky Date: Tue, 28 Aug 2012 23:53:16 +0200 Subject: Crashfix; During TP, or shortly after, gAgent.getRegion can be invalid. Handle that instead of crashing. --- indra/newview/llwlhandlers.cpp | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) (limited to 'indra/newview') diff --git a/indra/newview/llwlhandlers.cpp b/indra/newview/llwlhandlers.cpp index 2425b96678..be3e3ff30e 100644 --- a/indra/newview/llwlhandlers.cpp +++ b/indra/newview/llwlhandlers.cpp @@ -105,10 +105,16 @@ LLEnvironmentRequestResponder::LLEnvironmentRequestResponder() return; } - if (unvalidated_content[0]["regionID"].asUUID() != gAgent.getRegion()->getRegionID()) + LLUUID regionId; + if( gAgent.getRegion() ) + { + regionId = gAgent.getRegion()->getRegionID(); + } + + if (unvalidated_content[0]["regionID"].asUUID() != regionId ) { LL_WARNS("WindlightCaps") << "Not in the region from where this data was received (wanting " - << gAgent.getRegion()->getRegionID() << " but got " << unvalidated_content[0]["regionID"].asUUID() + << regionId << " but got " << unvalidated_content[0]["regionID"].asUUID() << ") - ignoring..." << LL_ENDL; return; } -- cgit v1.3 From 33263bb5e7dad106ef82c62f789ae430bad63d60 Mon Sep 17 00:00:00 2001 From: Nicky Date: Tue, 28 Aug 2012 23:52:30 +0200 Subject: Don't crash if star_brightness misses in a WL-Param set. Instead warn to log and exit the fuction gracefully. --- indra/newview/lldrawpoolwlsky.cpp | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) (limited to 'indra/newview') diff --git a/indra/newview/lldrawpoolwlsky.cpp b/indra/newview/lldrawpoolwlsky.cpp index caf15fe1cb..b5faff7968 100644 --- a/indra/newview/lldrawpoolwlsky.cpp +++ b/indra/newview/lldrawpoolwlsky.cpp @@ -192,14 +192,18 @@ void LLDrawPoolWLSky::renderStars(void) const bool error; LLColor4 star_alpha(LLColor4::black); star_alpha.mV[3] = LLWLParamManager::getInstance()->mCurParams.getFloat("star_brightness", error) / 2.f; - llassert_always(!error); + + // If start_brightness is not set, exit + if( error ) + { + llwarns << "star_brightness missing in mCurParams" << llendl; + return; + } gGL.getTexUnit(0)->bind(gSky.mVOSkyp->getBloomTex()); gGL.pushMatrix(); gGL.rotatef(gFrameTimeSeconds*0.01f, 0.f, 0.f, 1.f); - // gl_FragColor.rgb = gl_Color.rgb; - // gl_FragColor.a = gl_Color.a * star_alpha.a; if (LLGLSLShader::sNoFixedFunction) { gCustomAlphaProgram.bind(); -- cgit v1.3 From 57d36df3bc3c601d87dcf35a4e6692e1ae4d65ff Mon Sep 17 00:00:00 2001 From: Nicky Date: Tue, 21 Aug 2012 19:47:16 +0200 Subject: Make sure mWearableItem/mNameEditor are valid before dereferencing them. --- indra/newview/llpaneleditwearable.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'indra/newview') diff --git a/indra/newview/llpaneleditwearable.cpp b/indra/newview/llpaneleditwearable.cpp index d58d6d536c..d42056ef9d 100644 --- a/indra/newview/llpaneleditwearable.cpp +++ b/indra/newview/llpaneleditwearable.cpp @@ -835,11 +835,11 @@ BOOL LLPanelEditWearable::isDirty() const BOOL isDirty = FALSE; if (mWearablePtr) { - if (mWearablePtr->isDirty() || - mWearableItem->getName().compare(mNameEditor->getText()) != 0) - { - isDirty = TRUE; - } + if (mWearablePtr->isDirty() || + ( mWearableItem && mNameEditor && mWearableItem->getName().compare(mNameEditor->getText()) != 0 )) + { + isDirty = TRUE; + } } return isDirty; } -- cgit v1.3 From f9c9a894442c7ff51255451663110a553c4c35c9 Mon Sep 17 00:00:00 2001 From: Nicky Date: Tue, 21 Aug 2012 19:45:45 +0200 Subject: Crashfix in animation preview floater. --- indra/newview/llfloaterbvhpreview.cpp | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) (limited to 'indra/newview') diff --git a/indra/newview/llfloaterbvhpreview.cpp b/indra/newview/llfloaterbvhpreview.cpp index fa0ad20fdb..62848586cd 100644 --- a/indra/newview/llfloaterbvhpreview.cpp +++ b/indra/newview/llfloaterbvhpreview.cpp @@ -422,13 +422,14 @@ void LLFloaterBvhPreview::resetMotion() LLVOAvatar* avatarp = mAnimPreview->getDummyAvatar(); BOOL paused = avatarp->areAnimationsPaused(); - // *TODO: Fix awful casting hack - LLKeyframeMotion* motionp = (LLKeyframeMotion*)avatarp->findMotion(mMotionID); - - // Set emotion - std::string emote = getChild("emote_combo")->getValue().asString(); - motionp->setEmote(mIDList[emote]); - + LLKeyframeMotion* motionp = dynamic_cast(avatarp->findMotion(mMotionID)); + if( motionp ) + { + // Set emotion + std::string emote = getChild("emote_combo")->getValue().asString(); + motionp->setEmote(mIDList[emote]); + } + LLUUID base_id = mIDList[getChild("preview_base_anim")->getValue().asString()]; avatarp->deactivateAllMotions(); avatarp->startMotion(mMotionID, 0.0f); @@ -438,8 +439,12 @@ void LLFloaterBvhPreview::resetMotion() // Set pose std::string handpose = getChild("hand_pose_combo")->getValue().asString(); avatarp->startMotion( ANIM_AGENT_HAND_MOTION, 0.0f ); - motionp->setHandPose(LLHandMotion::getHandPose(handpose)); + if( motionp ) + { + motionp->setHandPose(LLHandMotion::getHandPose(handpose)); + } + if (paused) { mPauseRequest = avatarp->requestPause(); -- cgit v1.3 From 944469b84f226cd96a62a9a1bb6302781000c497 Mon Sep 17 00:00:00 2001 From: Nicky Date: Tue, 14 Aug 2012 19:57:43 +0200 Subject: Make sure only one thread access mPendingLOD at a time. --- indra/newview/llmeshrepository.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'indra/newview') diff --git a/indra/newview/llmeshrepository.cpp b/indra/newview/llmeshrepository.cpp index bc7f522848..52077c620f 100755 --- a/indra/newview/llmeshrepository.cpp +++ b/indra/newview/llmeshrepository.cpp @@ -1097,11 +1097,13 @@ bool LLMeshRepoThread::headerReceived(const LLVolumeParams& mesh_params, U8* dat mMeshHeader[mesh_id] = header; } + + LLMutexLock lock(mMutex); // make sure only one thread access mPendingLOD at the same time. + //check for pending requests pending_lod_map::iterator iter = mPendingLOD.find(mesh_params); if (iter != mPendingLOD.end()) { - LLMutexLock lock(mMutex); for (U32 i = 0; i < iter->second.size(); ++i) { LODRequest req(mesh_params, iter->second[i]); -- cgit v1.3 From c8ba971e84b6dba31636dbfba20b2d9ca219f67f Mon Sep 17 00:00:00 2001 From: Nicky Date: Tue, 24 Jul 2012 17:22:18 +0200 Subject: Crashfix; don't crash in LLRightClickInventoryFetchDescendentsObserver::execute when there's a 0 pointer for item_array or cat_array. --- indra/newview/llinventorybridge.cpp | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) (limited to 'indra/newview') diff --git a/indra/newview/llinventorybridge.cpp b/indra/newview/llinventorybridge.cpp index fce0b7c9c9..14a228df1c 100644 --- a/indra/newview/llinventorybridge.cpp +++ b/indra/newview/llinventorybridge.cpp @@ -2577,14 +2577,23 @@ void LLRightClickInventoryFetchDescendentsObserver::execute(bool clear_observer) LLInventoryModel::item_array_t* item_array; gInventory.getDirectDescendentsOf(*current_folder, cat_array, item_array); - S32 item_count = item_array->count(); - S32 cat_count = cat_array->count(); - + S32 item_count(0); + if( item_array ) + { + item_count = item_array->count(); + } + + S32 cat_count(0); + if( cat_array ) + { + cat_count = cat_array->count(); + } + // Move to next if current folder empty if ((item_count == 0) && (cat_count == 0)) - { + { continue; - } + } uuid_vec_t ids; LLRightClickInventoryFetchObserver* outfit = NULL; -- cgit v1.3 From fdb0218bfc156842a70661e339ba4592e02dc9c4 Mon Sep 17 00:00:00 2001 From: Nicky Date: Mon, 22 Oct 2012 15:40:18 -0400 Subject: Crashfix; Guard against 0 pointer textures. --- indra/newview/llface.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'indra/newview') diff --git a/indra/newview/llface.cpp b/indra/newview/llface.cpp index 188f943f13..605cb81c10 100755 --- a/indra/newview/llface.cpp +++ b/indra/newview/llface.cpp @@ -1062,7 +1062,11 @@ bool LLFace::canRenderAsMask() } const LLTextureEntry* te = getTextureEntry(); - + if( !te || !getViewerObject() || !getTexture() ) + { + return false; + } + if ((te->getColor().mV[3] == 1.0f) && // can't treat as mask if we have face alpha (te->getGlow() == 0.f) && // glowing masks are hard to implement - don't mask getTexture()->getIsAlphaMask()) // texture actually qualifies for masking (lazily recalculated but expensive) -- cgit v1.3 From bef60207e309db96d1aace39c24906903ed3bdc1 Mon Sep 17 00:00:00 2001 From: Nicky Date: Sat, 21 Jul 2012 13:35:14 +0200 Subject: Crashfix; Guard against 0 pointer in LLRiggedVolume::update. --- indra/newview/llvovolume.cpp | 2 ++ 1 file changed, 2 insertions(+) (limited to 'indra/newview') diff --git a/indra/newview/llvovolume.cpp b/indra/newview/llvovolume.cpp index e99898a83c..afe62bbb2e 100644 --- a/indra/newview/llvovolume.cpp +++ b/indra/newview/llvovolume.cpp @@ -4880,6 +4880,7 @@ void LLVolumeGeometryManager::genDrawInfo(LLSpatialGroup* group, U32 mask, std:: std::vector texture_list; + if( pos && weight && dst_face.mExtents ) { LLFastTimer t(FTM_GEN_DRAW_INFO_FACE_SIZE); if (batch_textures) @@ -4976,6 +4977,7 @@ void LLVolumeGeometryManager::genDrawInfo(LLSpatialGroup* group, U32 mask, std:: //create vertex buffer LLVertexBuffer* buffer = NULL; + if( dst_face.mExtents ) { LLFastTimer t(FTM_GEN_DRAW_INFO_ALLOCATE); buffer = createVertexBuffer(mask, buffer_usage); -- cgit v1.3 From 6fab95060fe2793e1206a5329f81877e3980f0cd Mon Sep 17 00:00:00 2001 From: Nicky Date: Thu, 19 Jul 2012 22:54:17 +0200 Subject: Crashfix; Make sure getImage returns a valid result. --- indra/newview/llvoavatar.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'indra/newview') diff --git a/indra/newview/llvoavatar.cpp b/indra/newview/llvoavatar.cpp index 366b6004be..2c1291c954 100755 --- a/indra/newview/llvoavatar.cpp +++ b/indra/newview/llvoavatar.cpp @@ -4421,7 +4421,9 @@ U32 LLVOAvatar::renderTransparent(BOOL first_pass) } // Can't test for baked hair being defined, since that won't always be the case (not all viewers send baked hair) // TODO: 1.25 will be able to switch this logic back to calling isTextureVisible(); - if (getImage(TEX_HAIR_BAKED, 0)->getID() != IMG_INVISIBLE || LLDrawPoolAlpha::sShowDebugAlpha) + + if ( getImage(TEX_HAIR_BAKED, 0) + && getImage(TEX_HAIR_BAKED, 0)->getID() != IMG_INVISIBLE || LLDrawPoolAlpha::sShowDebugAlpha) { num_indices += mMeshLOD[MESH_ID_HAIR]->render(mAdjustedPixelArea, first_pass, mIsDummy); first_pass = FALSE; -- cgit v1.3 From 9b856a124ac11f976c7913d1ce2af8d57ab0464c Mon Sep 17 00:00:00 2001 From: Nicky Date: Sat, 14 Jul 2012 03:10:24 +0200 Subject: Crashfix: showToastsTop/Bottom, don't iterate over mToastList. The member can change during recursive calls, invalidating iterators. --- indra/newview/llscreenchannel.cpp | 40 +++++++++++++++++++++++++-------------- 1 file changed, 26 insertions(+), 14 deletions(-) (limited to 'indra/newview') diff --git a/indra/newview/llscreenchannel.cpp b/indra/newview/llscreenchannel.cpp index d340b304ca..d2280ea089 100644 --- a/indra/newview/llscreenchannel.cpp +++ b/indra/newview/llscreenchannel.cpp @@ -571,9 +571,13 @@ void LLScreenChannel::showToastsBottom() LLDockableFloater* floater = dynamic_cast(LLDockableFloater::getInstanceHandle().get()); - for(it = mToastList.rbegin(); it != mToastList.rend(); ++it) + // Use a local variable instead of mToastList. + // mToastList can be modified during recursive calls and then all iteratos will be invalidated. + std::vector vToastList( mToastList ); + + for(it = vToastList.rbegin(); it != vToastList.rend(); ++it) { - if(it != mToastList.rbegin()) + if(it != vToastList.rbegin()) { LLToast* toast = (it-1)->getToast(); if (!toast) @@ -601,7 +605,7 @@ void LLScreenChannel::showToastsBottom() if(floater && floater->overlapsScreenChannel()) { - if(it == mToastList.rbegin()) + if(it == vToastList.rbegin()) { // move first toast above docked floater S32 shift = floater->getRect().getHeight(); @@ -624,7 +628,7 @@ void LLScreenChannel::showToastsBottom() if(!stop_showing_toasts) { - if( it != mToastList.rend()-1) + if( it != vToastList.rend()-1) { S32 toast_top = toast->getRect().mTop + gSavedSettings.getS32("ToastGap"); stop_showing_toasts = toast_top > getRect().mTop; @@ -632,7 +636,8 @@ void LLScreenChannel::showToastsBottom() } // at least one toast should be visible - if(it == mToastList.rbegin()) + + if(it == vToastList.rbegin()) { stop_showing_toasts = false; } @@ -655,10 +660,11 @@ void LLScreenChannel::showToastsBottom() } // Dismiss toasts we don't have space for (STORM-391). - if(it != mToastList.rend()) + if(it != vToastList.rend()) { mHiddenToastsNum = 0; - for(; it != mToastList.rend(); it++) + + for(; it != vToastList.rend(); it++) { LLToast* toast = it->getToast(); if (toast) @@ -714,9 +720,13 @@ void LLScreenChannel::showToastsTop() LLDockableFloater* floater = dynamic_cast(LLDockableFloater::getInstanceHandle().get()); - for(it = mToastList.rbegin(); it != mToastList.rend(); ++it) + // Use a local variable instead of mToastList. + // mToastList can be modified during recursive calls and then all iteratos will be invalidated. + std::vector vToastList( mToastList ); + + for(it = vToastList.rbegin(); it != vToastList.rend(); ++it) { - if(it != mToastList.rbegin()) + if(it != vToastList.rbegin()) { LLToast* toast = (it-1)->getToast(); if (!toast) @@ -744,7 +754,7 @@ void LLScreenChannel::showToastsTop() if(floater && floater->overlapsScreenChannel()) { - if(it == mToastList.rbegin()) + if(it == vToastList.rbegin()) { // move first toast above docked floater S32 shift = -floater->getRect().getHeight(); @@ -767,7 +777,7 @@ void LLScreenChannel::showToastsTop() if(!stop_showing_toasts) { - if( it != mToastList.rend()-1) + if( it != vToastList.rend()-1) { S32 toast_bottom = toast->getRect().mBottom - gSavedSettings.getS32("ToastGap"); stop_showing_toasts = toast_bottom < channel_rect.mBottom; @@ -775,7 +785,7 @@ void LLScreenChannel::showToastsTop() } // at least one toast should be visible - if(it == mToastList.rbegin()) + if(it == vToastList.rbegin()) { stop_showing_toasts = false; } @@ -799,10 +809,12 @@ void LLScreenChannel::showToastsTop() // Dismiss toasts we don't have space for (STORM-391). std::vector toasts_to_hide; - if(it != mToastList.rend()) + + if(it != vToastList.rend()) { mHiddenToastsNum = 0; - for(; it != mToastList.rend(); it++) + + for(; it != vToastList.rend(); it++) { LLToast* toast = it->getToast(); if (toast) -- cgit v1.3 From c0e8656477e41fffa26be9a23bef1c5bb1357330 Mon Sep 17 00:00:00 2001 From: Nicky Date: Fri, 13 Jul 2012 14:04:45 +0200 Subject: Crashfix: Stop notifications when we're about to exit. --- indra/newview/llnotificationmanager.cpp | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'indra/newview') diff --git a/indra/newview/llnotificationmanager.cpp b/indra/newview/llnotificationmanager.cpp index f792f53ac5..3d8150eed3 100644 --- a/indra/newview/llnotificationmanager.cpp +++ b/indra/newview/llnotificationmanager.cpp @@ -97,6 +97,13 @@ bool LLNotificationManager::onNotification(const LLSD& notify) { LLSysHandler* handle = NULL; + // Don't bother if we're going down. + // Otherwise we might crash when trying to use handlers that are already dead. + if( LLApp::isExiting() ) + { + return false; + } + if (LLNotifications::destroyed()) return false; -- cgit v1.3 From 85025ffc7be40c0953ca8905f149701b4b8f568f Mon Sep 17 00:00:00 2001 From: Nicky Date: Fri, 13 Jul 2012 13:46:38 +0200 Subject: Crashfix: LLVOVolume::calcLOD make sure avatar and avatar->mDrawable are valid. --- indra/newview/llvovolume.cpp | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'indra/newview') diff --git a/indra/newview/llvovolume.cpp b/indra/newview/llvovolume.cpp index afe62bbb2e..49513eb206 100644 --- a/indra/newview/llvovolume.cpp +++ b/indra/newview/llvovolume.cpp @@ -1227,6 +1227,13 @@ BOOL LLVOVolume::calcLOD() if (mDrawable->isState(LLDrawable::RIGGED)) { LLVOAvatar* avatar = getAvatar(); + + // Not sure how this can really happen, but alas it does. Better exit here than crashing. + if( !avatar || !avatar->mDrawable ) + { + return FALSE; + } + distance = avatar->mDrawable->mDistanceWRTCamera; radius = avatar->getBinRadius(); } -- cgit v1.3 From 60393abdad98c61d9cb01d004e3d69bd8d34bfda Mon Sep 17 00:00:00 2001 From: Nicky Date: Fri, 13 Jul 2012 13:37:18 +0200 Subject: Crashfix: In LLAttachmentsMgr::onIdle make sure we've a valid region before dereferencing it. --- indra/newview/llattachmentsmgr.cpp | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'indra/newview') diff --git a/indra/newview/llattachmentsmgr.cpp b/indra/newview/llattachmentsmgr.cpp index c9543988a6..ea0b8f00a4 100644 --- a/indra/newview/llattachmentsmgr.cpp +++ b/indra/newview/llattachmentsmgr.cpp @@ -62,6 +62,12 @@ void LLAttachmentsMgr::onIdle(void *) void LLAttachmentsMgr::onIdle() { + // Make sure we got a region before trying anything else + if( !gAgent.getRegion() ) + { + return; + } + S32 obj_count = mPendingAttachments.size(); if (obj_count == 0) { -- cgit v1.3 From 8902f4c6c0fed60ae6d8b7596e61c7b8387079c1 Mon Sep 17 00:00:00 2001 From: Nicky Date: Tue, 10 Jul 2012 20:12:31 +0200 Subject: Crashfix: Don't blindly trust getImage() returns a valid pointer. --- indra/newview/llvoavatar.cpp | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'indra/newview') diff --git a/indra/newview/llvoavatar.cpp b/indra/newview/llvoavatar.cpp index 2c1291c954..39222c25fd 100755 --- a/indra/newview/llvoavatar.cpp +++ b/indra/newview/llvoavatar.cpp @@ -8707,6 +8707,12 @@ BOOL LLVOAvatar::isTextureDefined(LLVOAvatarDefines::ETextureIndex te, U32 index return FALSE; } + if( !getImage( te, index ) ) + { + llwarns << "getImage( " << te << ", " << index << " ) returned 0" << llendl; + return FALSE; + } + return (getImage(te, index)->getID() != IMG_DEFAULT_AVATAR && getImage(te, index)->getID() != IMG_DEFAULT); } -- cgit v1.3 From 82c51c8d29d7c9a59373f45fa794bbc0729c97d5 Mon Sep 17 00:00:00 2001 From: Nicky Date: Mon, 9 Jul 2012 13:37:21 +0200 Subject: Crashfix: LLVOAvatar::updateTextures in some odd cases getTE can return 0. Safeguard against that. --- indra/newview/llvoavatar.cpp | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) (limited to 'indra/newview') diff --git a/indra/newview/llvoavatar.cpp b/indra/newview/llvoavatar.cpp index 39222c25fd..627238b0f5 100755 --- a/indra/newview/llvoavatar.cpp +++ b/indra/newview/llvoavatar.cpp @@ -4567,7 +4567,20 @@ void LLVOAvatar::updateTextures() LLWearableType::EType wearable_type = LLVOAvatarDictionary::getTEWearableType((ETextureIndex)texture_index); U32 num_wearables = gAgentWearables.getWearableCount(wearable_type); const LLTextureEntry *te = getTE(texture_index); - const F32 texel_area_ratio = fabs(te->mScaleS * te->mScaleT); + + // getTE can return 0. + // Not sure yet why it does, but of course it crashes when te->mScale? gets used. + // Put safeguard in place so this corner case get better handling and does not result in a crash. + F32 texel_area_ratio = 1.0f; + if( te ) + { + texel_area_ratio = fabs(te->mScaleS * te->mScaleT); + } + else + { + llwarns << "getTE( " << texture_index << " ) returned 0" < Date: Sun, 8 Jul 2012 12:48:43 +0200 Subject: Crashfix: Make sure drawable exists before calling any method on it. --- indra/newview/llviewertexture.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'indra/newview') diff --git a/indra/newview/llviewertexture.cpp b/indra/newview/llviewertexture.cpp index 8eb8717de2..32cf8cc1b3 100644 --- a/indra/newview/llviewertexture.cpp +++ b/indra/newview/llviewertexture.cpp @@ -1904,7 +1904,7 @@ void LLViewerFetchedTexture::updateVirtualSize() for(U32 i = 0 ; i < mNumFaces ; i++) { LLFace* facep = mFaceList[i] ; - if(facep->getDrawable()->isRecentlyVisible()) + if( facep && facep->getDrawable() && facep->getDrawable()->isRecentlyVisible()) { addTextureStats(facep->getVirtualSize()) ; setAdditionalDecodePriority(facep->getImportanceToCamera()) ; -- cgit v1.3 From d1e924a71ef8ce41ba43bba1fbc1e1825401625c Mon Sep 17 00:00:00 2001 From: Nicky Date: Sat, 7 Jul 2012 22:10:43 +0200 Subject: Crashfix: During cleanup gAgentAvatarp can already be 0. --- indra/newview/llappearancemgr.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'indra/newview') diff --git a/indra/newview/llappearancemgr.cpp b/indra/newview/llappearancemgr.cpp index 6d67e098a6..06a9892c7e 100755 --- a/indra/newview/llappearancemgr.cpp +++ b/indra/newview/llappearancemgr.cpp @@ -52,7 +52,8 @@ std::string self_av_string() { - return gAgentAvatarp->avString(); + // On logout gAgentAvatarp can already be invalid + return isAgentAvatarValid() ? gAgentAvatarp->avString() : ""; } // RAII thingy to guarantee that a variable gets reset when the Setter -- cgit v1.3 From d40a620483729d87bb7bca5de42d338f32ab09dc Mon Sep 17 00:00:00 2001 From: Nicky Date: Sun, 17 Jun 2012 18:20:30 +0200 Subject: Don't reference any LLFace via LLDrawable::getFace whose index is out of bounds. --- indra/newview/llvovolume.cpp | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) (limited to 'indra/newview') diff --git a/indra/newview/llvovolume.cpp b/indra/newview/llvovolume.cpp index 49513eb206..8465a9dadd 100644 --- a/indra/newview/llvovolume.cpp +++ b/indra/newview/llvovolume.cpp @@ -1342,7 +1342,8 @@ BOOL LLVOVolume::setDrawableParent(LLDrawable* parentp) void LLVOVolume::updateFaceFlags() { - for (S32 i = 0; i < getVolume()->getNumFaces(); i++) + // There's no guarantee that getVolume()->getNumFaces() == mDrawable->getNumFaces() + for (S32 i = 0; i < getVolume()->getNumFaces() && i < mDrawable->getNumFaces(); i++) { LLFace *face = mDrawable->getFace(i); if (face) @@ -1443,7 +1444,10 @@ BOOL LLVOVolume::genBBoxes(BOOL force_global) volume = getVolume(); } - for (S32 i = 0; i < getVolume()->getNumVolumeFaces(); i++) + // There's no guarantee that getVolume()->getNumFaces() == mDrawable->getNumFaces() + for (S32 i = 0; + i < getVolume()->getNumVolumeFaces() && i < mDrawable->getNumFaces() && i < getNumTEs(); + i++) { LLFace *face = mDrawable->getFace(i); if (!face) @@ -1744,6 +1748,11 @@ BOOL LLVOVolume::updateGeometry(LLDrawable *drawable) void LLVOVolume::updateFaceSize(S32 idx) { + if( mDrawable->getNumFaces() <= idx ) + { + return; + } + LLFace* facep = mDrawable->getFace(idx); if (facep) { @@ -2434,7 +2443,12 @@ void LLVOVolume::addMediaImpl(LLViewerMediaImpl* media_impl, S32 texture_index) //add the face to show the media if it is in playing if(mDrawable) { - LLFace* facep = mDrawable->getFace(texture_index) ; + LLFace* facep(NULL); + if( texture_index < mDrawable->getNumFaces() ) + { + facep = mDrawable->getFace(texture_index) ; + } + if(facep) { LLViewerMediaTexture* media_tex = LLViewerTextureManager::findMediaTexture(mMediaImplList[texture_index]->getMediaTextureID()) ; -- cgit v1.3 From e963cefea596e41922e91f794356ff2533594587 Mon Sep 17 00:00:00 2001 From: Nicky Date: Fri, 1 Jun 2012 19:37:10 +0200 Subject: check mesh repo thread before actively using it. --- indra/newview/llmeshrepository.cpp | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) (limited to 'indra/newview') diff --git a/indra/newview/llmeshrepository.cpp b/indra/newview/llmeshrepository.cpp index 52077c620f..ba0a590910 100755 --- a/indra/newview/llmeshrepository.cpp +++ b/indra/newview/llmeshrepository.cpp @@ -1798,7 +1798,12 @@ void LLMeshLODResponder::completedRaw(U32 status, const std::string& reason, const LLChannelDescriptors& channels, const LLIOPipe::buffer_ptr_t& buffer) { - + // thread could have already be destroyed during logout + if( !gMeshRepo.mThread ) + { + return; + } + S32 data_size = buffer->countAfter(channels.in(), NULL); if (status < 200 || status > 400) @@ -1853,6 +1858,12 @@ void LLMeshSkinInfoResponder::completedRaw(U32 status, const std::string& reason const LLChannelDescriptors& channels, const LLIOPipe::buffer_ptr_t& buffer) { + // thread could have already be destroyed during logout + if( !gMeshRepo.mThread ) + { + return; + } + S32 data_size = buffer->countAfter(channels.in(), NULL); if (status < 200 || status > 400) @@ -1907,6 +1918,11 @@ void LLMeshDecompositionResponder::completedRaw(U32 status, const std::string& r const LLChannelDescriptors& channels, const LLIOPipe::buffer_ptr_t& buffer) { + if( !gMeshRepo.mThread ) + { + return; + } + S32 data_size = buffer->countAfter(channels.in(), NULL); if (status < 200 || status > 400) @@ -1961,6 +1977,12 @@ void LLMeshPhysicsShapeResponder::completedRaw(U32 status, const std::string& re const LLChannelDescriptors& channels, const LLIOPipe::buffer_ptr_t& buffer) { + // thread could have already be destroyed during logout + if( !gMeshRepo.mThread ) + { + return; + } + S32 data_size = buffer->countAfter(channels.in(), NULL); if (status < 200 || status > 400) @@ -2015,6 +2037,12 @@ void LLMeshHeaderResponder::completedRaw(U32 status, const std::string& reason, const LLChannelDescriptors& channels, const LLIOPipe::buffer_ptr_t& buffer) { + // thread could have already be destroyed during logout + if( !gMeshRepo.mThread ) + { + return; + } + if (status < 200 || status > 400) { //llwarns -- cgit v1.3 From e8bebc5b3c17cbb3c3d1021104be2de99dcc3efe Mon Sep 17 00:00:00 2001 From: Nicky Date: Wed, 30 May 2012 23:46:41 +0200 Subject: Make sure to not use floaters of different type than LLPreview as LLPreview instance. --- indra/newview/llpreview.cpp | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) (limited to 'indra/newview') diff --git a/indra/newview/llpreview.cpp b/indra/newview/llpreview.cpp index 18626e3273..04934b13f1 100644 --- a/indra/newview/llpreview.cpp +++ b/indra/newview/llpreview.cpp @@ -464,7 +464,9 @@ LLMultiPreview::LLMultiPreview() void LLMultiPreview::onOpen(const LLSD& key) { - LLPreview* frontmost_preview = (LLPreview*)mTabContainer->getCurrentPanel(); + // Floater could be something else than LLPreview, eg LLFloaterProfile. + LLPreview* frontmost_preview = dynamic_cast(mTabContainer->getCurrentPanel()); + if (frontmost_preview && frontmost_preview->getAssetStatus() == LLPreview::PREVIEW_ASSET_UNLOADED) { frontmost_preview->loadAsset(); @@ -477,8 +479,13 @@ void LLMultiPreview::handleReshape(const LLRect& new_rect, bool by_user) { if(new_rect.getWidth() != getRect().getWidth() || new_rect.getHeight() != getRect().getHeight()) { - LLPreview* frontmost_preview = (LLPreview*)mTabContainer->getCurrentPanel(); - if (frontmost_preview) frontmost_preview->userResized(); + // Floater could be something else than LLPreview, eg LLFloaterProfile. + LLPreview* frontmost_preview = dynamic_cast(mTabContainer->getCurrentPanel()); + + if (frontmost_preview) + { + frontmost_preview->userResized(); + } } LLFloater::handleReshape(new_rect, by_user); } @@ -486,7 +493,9 @@ void LLMultiPreview::handleReshape(const LLRect& new_rect, bool by_user) void LLMultiPreview::tabOpen(LLFloater* opened_floater, bool from_click) { - LLPreview* opened_preview = (LLPreview*)opened_floater; + // Floater could be something else than LLPreview, eg LLFloaterProfile. + LLPreview* opened_preview = dynamic_cast(opened_floater); + if (opened_preview && opened_preview->getAssetStatus() == LLPreview::PREVIEW_ASSET_UNLOADED) { opened_preview->loadAsset(); -- cgit v1.3 From b8edacd0bb4feacc3ac1d61421e600c75ab87f7c Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Fri, 1 Jun 2012 14:07:34 -0400 Subject: Major steps towards implementing the policy component. Identified and reacted to the priority inversion problem we have in texturefetch. Includes the introduction of a priority_queue for the requests that are ready. Start some parameterization in anticipation of having policy_class everywhere. Removed _assert.h which isn't really needed in indra codebase. Implemented async setPriority request (which I hope I can get rid of eventually along with all priorities in this library). Converted to using unsigned int for priority rather than float. Implemented POST and did groundwork for PUT. --- indra/llcorehttp/CMakeLists.txt | 27 ++-- indra/llcorehttp/_assert.h | 39 ----- indra/llcorehttp/_httplibcurl.cpp | 52 ++++-- indra/llcorehttp/_httplibcurl.h | 27 +++- indra/llcorehttp/_httpopcancel.h | 2 +- indra/llcorehttp/_httpoperation.cpp | 8 +- indra/llcorehttp/_httpoperation.h | 42 ++++- indra/llcorehttp/_httpoprequest.cpp | 113 ++++++++++++- indra/llcorehttp/_httpoprequest.h | 11 +- indra/llcorehttp/_httpopsetpriority.cpp | 77 +++++++++ indra/llcorehttp/_httpopsetpriority.h | 70 +++++++++ indra/llcorehttp/_httppolicy.cpp | 97 ++++++++++-- indra/llcorehttp/_httppolicy.h | 28 +++- indra/llcorehttp/_httpreadyqueue.h | 85 ++++++++++ indra/llcorehttp/_httprequestqueue.cpp | 2 +- indra/llcorehttp/_httpservice.cpp | 63 +++++--- indra/llcorehttp/_httpservice.h | 26 ++- indra/llcorehttp/_refcounted.h | 14 +- indra/llcorehttp/httpcommon.cpp | 15 +- indra/llcorehttp/httpcommon.h | 8 +- indra/llcorehttp/httprequest.cpp | 52 +++++- indra/llcorehttp/httprequest.h | 49 +++++- indra/llcorehttp/tests/test_httprequest.hpp | 2 +- indra/newview/lltexturefetch.cpp | 235 ++++++++++++---------------- indra/newview/lltexturefetch.h | 15 -- 25 files changed, 869 insertions(+), 290 deletions(-) delete mode 100644 indra/llcorehttp/_assert.h create mode 100644 indra/llcorehttp/_httpopsetpriority.cpp create mode 100644 indra/llcorehttp/_httpopsetpriority.h create mode 100644 indra/llcorehttp/_httpreadyqueue.h (limited to 'indra/newview') diff --git a/indra/llcorehttp/CMakeLists.txt b/indra/llcorehttp/CMakeLists.txt index ae92fb96fd..85df5364db 100644 --- a/indra/llcorehttp/CMakeLists.txt +++ b/indra/llcorehttp/CMakeLists.txt @@ -23,17 +23,18 @@ include_directories( set(llcorehttp_SOURCE_FILES bufferarray.cpp httpcommon.cpp + httpheaders.cpp + httpoptions.cpp httprequest.cpp httpresponse.cpp - httpoptions.cpp - httpheaders.cpp - _httprequestqueue.cpp + _httplibcurl.cpp + _httpopcancel.cpp _httpoperation.cpp _httpoprequest.cpp - _httpopcancel.cpp - _httpreplyqueue.cpp + _httpopsetpriority.cpp _httppolicy.cpp - _httplibcurl.cpp + _httpreplyqueue.cpp + _httprequestqueue.cpp _httpservice.cpp _refcounted.cpp ) @@ -48,17 +49,19 @@ set(llcorehttp_HEADER_FILES httpoptions.h httprequest.h httpresponse.h + _assert.h + _httplibcurl.h + _httpopcancel.h _httpoperation.h _httpoprequest.h - _httpopcancel.h - _httprequestqueue.h + _httpopsetpriority.h + _httppolicy.h + _httpreadyqueue.h _httpreplyqueue.h + _httprequestqueue.h _httpservice.h - _httppolicy.h - _httplibcurl.h - _assert.h - _refcounted.h _mutex.h + _refcounted.h _thread.h ) diff --git a/indra/llcorehttp/_assert.h b/indra/llcorehttp/_assert.h deleted file mode 100644 index 054f23ef32..0000000000 --- a/indra/llcorehttp/_assert.h +++ /dev/null @@ -1,39 +0,0 @@ -/** - * @file _assert - * @brief assert abstraction - * - * $LicenseInfo:firstyear=2012&license=viewerlgpl$ - * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; - * version 2.1 of the License only. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - * - * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA - * $/LicenseInfo$ - */ - -#ifndef LLCOREINT__ASSERT_H_ -#define LLCOREINT__ASSERT_H_ - -#ifdef DEBUG_ASSERT -#include -#define LLINT_ASSERT(x) assert(x) -#else -#define LLINT_ASSERT(x) -#endif - -#endif // LLCOREINT__ASSERT_H_ - - diff --git a/indra/llcorehttp/_httplibcurl.cpp b/indra/llcorehttp/_httplibcurl.cpp index 1b951818e4..704f9baac9 100644 --- a/indra/llcorehttp/_httplibcurl.cpp +++ b/indra/llcorehttp/_httplibcurl.cpp @@ -28,7 +28,6 @@ #include "httpheaders.h" #include "_httpoprequest.h" -#include "_httpservice.h" namespace LLCore @@ -38,6 +37,12 @@ namespace LLCore HttpLibcurl::HttpLibcurl(HttpService * service) : mService(service) { + for (int policy_class(0); policy_class < HttpRequest::POLICY_CLASS_LIMIT; ++policy_class) + { + mMultiHandles[policy_class] = 0; + } + + // Create multi handle for default class mMultiHandles[0] = curl_multi_init(); } @@ -51,15 +56,18 @@ HttpLibcurl::~HttpLibcurl() (*item)->cancel(); (*item)->release(); + mActiveOps.erase(item); } - if (mMultiHandles[0]) + for (int policy_class(0); policy_class < HttpRequest::POLICY_CLASS_LIMIT; ++policy_class) { - // *FIXME: Do some multi cleanup here first - + if (mMultiHandles[policy_class]) + { + // *FIXME: Do some multi cleanup here first - curl_multi_cleanup(mMultiHandles[0]); - mMultiHandles[0] = NULL; + curl_multi_cleanup(mMultiHandles[policy_class]); + mMultiHandles[policy_class] = 0; + } } mService = NULL; @@ -74,31 +82,34 @@ void HttpLibcurl::term() {} -void HttpLibcurl::processTransport() +HttpService::ELoopSpeed HttpLibcurl::processTransport() { - if (mMultiHandles[0]) + // Give libcurl some cycles to do I/O & callbacks + for (int policy_class(0); policy_class < HttpRequest::POLICY_CLASS_LIMIT; ++policy_class) { - // Give libcurl some cycles to do I/O & callbacks + if (! mMultiHandles[policy_class]) + continue; + int running(0); CURLMcode status(CURLM_CALL_MULTI_PERFORM); do { running = 0; - status = curl_multi_perform(mMultiHandles[0], &running); + status = curl_multi_perform(mMultiHandles[policy_class], &running); } while (0 != running && CURLM_CALL_MULTI_PERFORM == status); // Run completion on anything done CURLMsg * msg(NULL); int msgs_in_queue(0); - while ((msg = curl_multi_info_read(mMultiHandles[0], &msgs_in_queue))) + while ((msg = curl_multi_info_read(mMultiHandles[policy_class], &msgs_in_queue))) { if (CURLMSG_DONE == msg->msg) { CURL * handle(msg->easy_handle); CURLcode result(msg->data.result); - completeRequest(mMultiHandles[0], handle, result); + completeRequest(mMultiHandles[policy_class], handle, result); handle = NULL; // No longer valid on return } else if (CURLMSG_NONE == msg->msg) @@ -114,13 +125,18 @@ void HttpLibcurl::processTransport() msgs_in_queue = 0; } } + + return mActiveOps.empty() ? HttpService::REQUEST_SLEEP : HttpService::NORMAL; } void HttpLibcurl::addOp(HttpOpRequest * op) { + llassert_always(op->mReqPolicy < HttpRequest::POLICY_CLASS_LIMIT); + llassert_always(mMultiHandles[op->mReqPolicy] != NULL); + // Create standard handle - if (! op->prepareForGet(mService)) + if (! op->prepareRequest(mService)) { // Couldn't issue request, fail with notification // *FIXME: Need failure path @@ -128,7 +144,7 @@ void HttpLibcurl::addOp(HttpOpRequest * op) } // Make the request live - curl_multi_add_handle(mMultiHandles[0], op->mCurlHandle); + curl_multi_add_handle(mMultiHandles[op->mReqPolicy], op->mCurlHandle); op->mCurlActive = true; // On success, make operation active @@ -190,12 +206,18 @@ void HttpLibcurl::completeRequest(CURLM * multi_handle, CURL * handle, CURLcode } -int HttpLibcurl::activeCount() const +int HttpLibcurl::getActiveCount() const { return mActiveOps.size(); } +int HttpLibcurl::getActiveCountInClass(int /* policy_class */) const +{ + return getActiveCount(); +} + + // --------------------------------------- // Free functions // --------------------------------------- diff --git a/indra/llcorehttp/_httplibcurl.h b/indra/llcorehttp/_httplibcurl.h index 807196628d..ec325c1946 100644 --- a/indra/llcorehttp/_httplibcurl.h +++ b/indra/llcorehttp/_httplibcurl.h @@ -34,18 +34,25 @@ #include +#include "httprequest.h" +#include "_httpservice.h" + namespace LLCore { -class HttpService; class HttpPolicy; class HttpOpRequest; class HttpHeaders; /// Implements libcurl-based transport for an HttpService instance. +/// +/// Threading: Single-threaded. Other than for construction/destruction, +/// all methods are expected to be invoked in a single thread, typically +/// a worker thread of some sort. + class HttpLibcurl { public: @@ -60,12 +67,22 @@ public: static void init(); static void term(); - void processTransport(); - void addOp(HttpOpRequest * op); + /// Give cycles to libcurl to run active requests. Completed + /// operations (successful or failed) will be retried or handed + /// over to the reply queue as final responses. + HttpService::ELoopSpeed processTransport(); - int activeCount() const; + /// Add request to the active list. Caller is expected to have + /// provided us with a reference count to hold the request. (No + /// additional references will be added.) + void addOp(HttpOpRequest * op); + int getActiveCount() const; + int getActiveCountInClass(int policy_class) const; + protected: + /// Invoked when libcurl has indicated a request has been processed + /// to completion and we need to move the request to a new state. void completeRequest(CURLM * multi_handle, CURL * handle, CURLcode status); protected: @@ -74,7 +91,7 @@ protected: protected: HttpService * mService; // Simple reference, not owner active_set_t mActiveOps; - CURLM * mMultiHandles[1]; + CURLM * mMultiHandles[HttpRequest::POLICY_CLASS_LIMIT]; }; // end class HttpLibcurl diff --git a/indra/llcorehttp/_httpopcancel.h b/indra/llcorehttp/_httpopcancel.h index 38ccc585ed..fab6f1f362 100644 --- a/indra/llcorehttp/_httpopcancel.h +++ b/indra/llcorehttp/_httpopcancel.h @@ -45,7 +45,7 @@ namespace LLCore /// HttpOpCancel requests that a previously issued request /// be canceled, if possible. Requests that have been made /// active and are available for sending on the wire cannot -/// be canceled. +/// be canceled. class HttpOpCancel : public HttpOperation { diff --git a/indra/llcorehttp/_httpoperation.cpp b/indra/llcorehttp/_httpoperation.cpp index 17c65b0379..d966efd12b 100644 --- a/indra/llcorehttp/_httpoperation.cpp +++ b/indra/llcorehttp/_httpoperation.cpp @@ -50,7 +50,7 @@ HttpOperation::HttpOperation() mLibraryHandler(NULL), mUserHandler(NULL), mReqPolicy(HttpRequest::DEFAULT_POLICY_ID), - mReqPriority(0.0f) + mReqPriority(0U) { } @@ -95,7 +95,7 @@ void HttpOperation::stageFromRequest(HttpService *) // Default implementation should never be called. This // indicates an operation making a transition that isn't // defined. - LLINT_ASSERT(false); + llassert_always(false); } @@ -105,7 +105,7 @@ void HttpOperation::stageFromReady(HttpService *) // Default implementation should never be called. This // indicates an operation making a transition that isn't // defined. - LLINT_ASSERT(false); + llassert_always(false); } @@ -115,7 +115,7 @@ void HttpOperation::stageFromActive(HttpService *) // Default implementation should never be called. This // indicates an operation making a transition that isn't // defined. - LLINT_ASSERT(false); + llassert_always(false); } diff --git a/indra/llcorehttp/_httpoperation.h b/indra/llcorehttp/_httpoperation.h index 5d06a28586..6c0c3183b7 100644 --- a/indra/llcorehttp/_httpoperation.h +++ b/indra/llcorehttp/_httpoperation.h @@ -44,6 +44,32 @@ class HttpRequest; /// HttpOperation is the base class for all request/reply /// pairs. /// +/// Operations are expected to be of two types: immediate +/// and queued. Immediate requests go to the singleton +/// request queue and when picked up by the worker thread +/// are executed immediately and there results placed on +/// the supplied reply queue. Queued requests (namely for +/// HTTP operations), go to the request queue, are picked +/// up and moved to a ready queue where they're ordered by +/// priority and managed by the policy component, are +/// then activated issuing HTTP requests and moved to an +/// active list managed by the transport (libcurl) component +/// and eventually finalized when a response is available +/// and status and data return via reply queue. +/// +/// To manage these transitions, derived classes implement +/// three methods: stageFromRequest, stageFromReady and +/// stageFromActive. Immediate requests will only override +/// stageFromRequest which will perform the operation and +/// return the result by invoking addAsReply() to put the +/// request on a reply queue. Queued requests will involve +/// all three stage methods. +/// +/// Threading: not thread-safe. Base and derived classes +/// provide no locking. Instances move across threads +/// via queue-like interfaces that are thread compatible +/// and those interfaces establish the access rules. + class HttpOperation : public LLCoreInt::RefCounted { public: @@ -82,7 +108,7 @@ protected: public: unsigned int mReqPolicy; - float mReqPriority; + unsigned int mReqPriority; }; // end class HttpOperation @@ -133,6 +159,20 @@ public: }; // end class HttpOpNull + +/// HttpOpCompare isn't an operation but a uniform comparison +/// functor for STL containers that order by priority. Mainly +/// used for the ready queue container but defined here. +class HttpOpCompare +{ +public: + bool operator()(const HttpOperation * lhs, const HttpOperation * rhs) + { + return lhs->mReqPriority > rhs->mReqPriority; + } +}; // end class HttpOpCompare + + } // end namespace LLCore #endif // _LLCORE_HTTP_OPERATION_H_ diff --git a/indra/llcorehttp/_httpoprequest.cpp b/indra/llcorehttp/_httpoprequest.cpp index 521bd5b879..54b9990057 100644 --- a/indra/llcorehttp/_httpoprequest.cpp +++ b/indra/llcorehttp/_httpoprequest.cpp @@ -93,6 +93,7 @@ HttpOpRequest::HttpOpRequest() mCurlHandle(NULL), mCurlService(NULL), mCurlHeaders(NULL), + mCurlBodyPos(0), mReplyBody(NULL), mReplyOffset(0), mReplyLength(0), @@ -267,12 +268,46 @@ HttpStatus HttpOpRequest::setupGetByteRange(unsigned int policy_id, } -HttpStatus HttpOpRequest::prepareForGet(HttpService * service) +HttpStatus HttpOpRequest::setupPost(unsigned int policy_id, + float priority, + const std::string & url, + BufferArray * body, + HttpOptions * options, + HttpHeaders * headers) +{ + HttpStatus status; + + mProcFlags = 0; + mReqPolicy = policy_id; + mReqPriority = priority; + mReqMethod = HOR_POST; + mReqURL = url; + if (body) + { + body->addRef(); + mReqBody = body; + } + if (headers && ! mReqHeaders) + { + headers->addRef(); + mReqHeaders = headers; + } + if (options && ! mReqOptions) + { + mReqOptions = new HttpOptions(*options); + } + + return status; +} + + +HttpStatus HttpOpRequest::prepareRequest(HttpService * service) { // *FIXME: better error handling later HttpStatus status; mCurlHandle = curl_easy_init(); + // curl_easy_setopt(mCurlHandle, CURLOPT_VERBOSE, 1); curl_easy_setopt(mCurlHandle, CURLOPT_TIMEOUT, 30); curl_easy_setopt(mCurlHandle, CURLOPT_CONNECTTIMEOUT, 30); curl_easy_setopt(mCurlHandle, CURLOPT_NOSIGNAL, 1); @@ -280,20 +315,68 @@ HttpStatus HttpOpRequest::prepareForGet(HttpService * service) curl_easy_setopt(mCurlHandle, CURLOPT_URL, mReqURL.c_str()); curl_easy_setopt(mCurlHandle, CURLOPT_PRIVATE, this); curl_easy_setopt(mCurlHandle, CURLOPT_ENCODING, ""); + // *FIXME: Need to deal with proxy setup... // curl_easy_setopt(handle, CURLOPT_PROXY, ""); + // *FIXME: Revisit this old DNS timeout setting - may no longer be valid curl_easy_setopt(mCurlHandle, CURLOPT_DNS_CACHE_TIMEOUT, 0); + curl_easy_setopt(mCurlHandle, CURLOPT_AUTOREFERER, 1); curl_easy_setopt(mCurlHandle, CURLOPT_FOLLOWLOCATION, 1); + curl_easy_setopt(mCurlHandle, CURLOPT_MAXREDIRS, 10); curl_easy_setopt(mCurlHandle, CURLOPT_WRITEFUNCTION, writeCallback); curl_easy_setopt(mCurlHandle, CURLOPT_WRITEDATA, mCurlHandle); + curl_easy_setopt(mCurlHandle, CURLOPT_READFUNCTION, readCallback); + curl_easy_setopt(mCurlHandle, CURLOPT_READDATA, mCurlHandle); + switch (mReqMethod) + { + case HOR_GET: + curl_easy_setopt(mCurlHandle, CURLOPT_HTTPGET, 1); + break; + + case HOR_POST: + { + curl_easy_setopt(mCurlHandle, CURLOPT_POST, 1); + long data_size(0); + if (mReqBody) + { + mReqBody->seek(0); + data_size = mReqBody->size(); + } + curl_easy_setopt(mCurlHandle, CURLOPT_POSTFIELDS, static_cast(NULL)); + curl_easy_setopt(mCurlHandle, CURLOPT_POSTFIELDSIZE, data_size); + mCurlHeaders = curl_slist_append(mCurlHeaders, "Transfer-Encoding: chunked"); + mCurlHeaders = curl_slist_append(mCurlHeaders, "Expect:"); + } + break; + + case HOR_PUT: + { + curl_easy_setopt(mCurlHandle, CURLOPT_UPLOAD, 1); + long data_size(0); + if (mReqBody) + { + mReqBody->seek(0); + data_size = mReqBody->size(); + } + curl_easy_setopt(mCurlHandle, CURLOPT_INFILESIZE, data_size); + mCurlHeaders = curl_slist_append(mCurlHeaders, "Transfer-Encoding: chunked"); + mCurlHeaders = curl_slist_append(mCurlHeaders, "Expect:"); + } + break; + + default: + // *FIXME: fail out here + break; + } + if (mReqHeaders) { mCurlHeaders = append_headers_to_slist(mReqHeaders, mCurlHeaders); } mCurlHeaders = curl_slist_append(mCurlHeaders, "Pragma:"); - if (mReqOffset || mReqLength) + if ((mReqOffset || mReqLength) && HOR_GET == mReqMethod) { static const char * fmt1("Range: bytes=%d-%d"); static const char * fmt2("Range: bytes=%d-"); @@ -347,6 +430,32 @@ size_t HttpOpRequest::writeCallback(void * data, size_t size, size_t nmemb, void } +size_t HttpOpRequest::readCallback(void * data, size_t size, size_t nmemb, void * userdata) +{ + CURL * handle(static_cast(userdata)); + HttpOpRequest * op(NULL); + curl_easy_getinfo(handle, CURLINFO_PRIVATE, &op); + // *FIXME: check the pointer + + if (! op->mReqBody) + { + return 0; + } + const size_t req_size(size * nmemb); + const size_t body_size(op->mReqBody->size()); + if (body_size <= op->mCurlBodyPos) + { + // *FIXME: should probably log this event - unexplained + return 0; + } + + const size_t do_size((std::min)(req_size, body_size - op->mCurlBodyPos)); + op->mReqBody->read(static_cast(data), do_size); + op->mCurlBodyPos += do_size; + return do_size; +} + + size_t HttpOpRequest::headerCallback(void * data, size_t size, size_t nmemb, void * userdata) { static const char status_line[] = "HTTP/"; diff --git a/indra/llcorehttp/_httpoprequest.h b/indra/llcorehttp/_httpoprequest.h index 601937a943..7efed0b1d9 100644 --- a/indra/llcorehttp/_httpoprequest.h +++ b/indra/llcorehttp/_httpoprequest.h @@ -84,12 +84,20 @@ public: HttpOptions * options, HttpHeaders * headers); - HttpStatus prepareForGet(HttpService * service); + HttpStatus setupPost(unsigned int policy_id, + float priority, + const std::string & url, + BufferArray * body, + HttpOptions * options, + HttpHeaders * headers); + + HttpStatus prepareRequest(HttpService * service); virtual HttpStatus cancel(); protected: static size_t writeCallback(void * data, size_t size, size_t nmemb, void * userdata); + static size_t readCallback(void * data, size_t size, size_t nmemb, void * userdata); static size_t headerCallback(void * data, size_t size, size_t nmemb, void * userdata); protected: @@ -112,6 +120,7 @@ public: CURL * mCurlHandle; HttpService * mCurlService; curl_slist * mCurlHeaders; + size_t mCurlBodyPos; // Result data HttpStatus mStatus; diff --git a/indra/llcorehttp/_httpopsetpriority.cpp b/indra/llcorehttp/_httpopsetpriority.cpp new file mode 100644 index 0000000000..dc609421ed --- /dev/null +++ b/indra/llcorehttp/_httpopsetpriority.cpp @@ -0,0 +1,77 @@ +/** + * @file _httpopsetpriority.cpp + * @brief Definitions for internal classes based on HttpOpSetPriority + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include "_httpopsetpriority.h" + +#include "httpresponse.h" +#include "httphandler.h" +#include "_httpservice.h" + + +namespace LLCore +{ + + +HttpOpSetPriority::HttpOpSetPriority(HttpHandle handle, unsigned int priority) + : HttpOperation(), + mHandle(handle), + mPriority(priority) +{} + + +HttpOpSetPriority::~HttpOpSetPriority() +{} + + +void HttpOpSetPriority::stageFromRequest(HttpService * service) +{ + // Do operations + if (! service->changePriority(mHandle, mPriority)) + { + // Request not found, fail the final status + mStatus = HttpStatus(HttpStatus::LLCORE, HE_HANDLE_NOT_FOUND); + } + + // Move directly to response queue + addAsReply(); +} + + +void HttpOpSetPriority::visitNotifier(HttpRequest * request) +{ + if (mLibraryHandler) + { + HttpResponse * response = new HttpResponse(); + + response->setStatus(mStatus); + mLibraryHandler->onCompleted(static_cast(this), response); + + response->release(); + } +} + + +} // end namespace LLCore diff --git a/indra/llcorehttp/_httpopsetpriority.h b/indra/llcorehttp/_httpopsetpriority.h new file mode 100644 index 0000000000..e5d8e5fc1f --- /dev/null +++ b/indra/llcorehttp/_httpopsetpriority.h @@ -0,0 +1,70 @@ +/** + * @file _httpsetpriority.h + * @brief Internal declarations for HttpSetPriority + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef _LLCORE_HTTP_SETPRIORITY_H_ +#define _LLCORE_HTTP_SETPRIORITY_H_ + + +#include "httpcommon.h" + +#include "_httpoperation.h" +#include "_refcounted.h" + + +namespace LLCore +{ + + +/// HttpOpSetPriority is an immediate request that +/// searches the various queues looking for a given +/// request handle and changing it's priority if +/// found. + +class HttpOpSetPriority : public HttpOperation +{ +public: + HttpOpSetPriority(HttpHandle handle, unsigned int priority); + virtual ~HttpOpSetPriority(); + +private: + HttpOpSetPriority(const HttpOpSetPriority &); // Not defined + void operator=(const HttpOpSetPriority &); // Not defined + +public: + virtual void stageFromRequest(HttpService *); + + virtual void visitNotifier(HttpRequest * request); + +protected: + HttpStatus mStatus; + HttpHandle mHandle; + unsigned int mPriority; +}; // end class HttpOpSetPriority + +} // end namespace LLCore + +#endif // _LLCORE_HTTP_SETPRIORITY_H_ + diff --git a/indra/llcorehttp/_httppolicy.cpp b/indra/llcorehttp/_httppolicy.cpp index d965a6cf3a..873b519c51 100644 --- a/indra/llcorehttp/_httppolicy.cpp +++ b/indra/llcorehttp/_httppolicy.cpp @@ -28,48 +28,113 @@ #include "_httpoprequest.h" #include "_httpservice.h" +#include "_httplibcurl.h" namespace LLCore { - HttpPolicy::HttpPolicy(HttpService * service) : mService(service) -{} +{ + for (int policy_class(0); policy_class < HttpRequest::POLICY_CLASS_LIMIT; ++policy_class) + { + mReadyInClass[policy_class] = 0; + } +} HttpPolicy::~HttpPolicy() { - for (ready_queue_t::reverse_iterator i(mReadyQueue.rbegin()); - mReadyQueue.rend() != i;) + for (int policy_class(0); policy_class < HttpRequest::POLICY_CLASS_LIMIT; ++policy_class) { - ready_queue_t::reverse_iterator cur(i++); - - (*cur)->cancel(); - (*cur)->release(); + HttpReadyQueue & readyq(mReadyQueue[policy_class]); + + while (! readyq.empty()) + { + HttpOpRequest * op(readyq.top()); + + op->cancel(); + op->release(); + mReadyInClass[policy_class]--; + readyq.pop(); + } } - mService = NULL; } void HttpPolicy::addOp(HttpOpRequest * op) { - mReadyQueue.push_back(op); + const int policy_class(op->mReqPolicy); + + mReadyQueue[policy_class].push(op); + ++mReadyInClass[policy_class]; } -void HttpPolicy::processReadyQueue() +HttpService::ELoopSpeed HttpPolicy::processReadyQueue() { - while (! mReadyQueue.empty()) + HttpService::ELoopSpeed result(HttpService::REQUEST_SLEEP); + HttpLibcurl * pTransport(mService->getTransport()); + + for (int policy_class(0); policy_class < HttpRequest::POLICY_CLASS_LIMIT; ++policy_class) { - HttpOpRequest * op(mReadyQueue.front()); - mReadyQueue.erase(mReadyQueue.begin()); + HttpReadyQueue & readyq(mReadyQueue[policy_class]); + int active(pTransport->getActiveCountInClass(policy_class)); + int needed(8 - active); + + if (needed > 0 && mReadyInClass[policy_class] > 0) + { + // Scan ready queue for requests that match policy + + while (! readyq.empty() && needed > 0 && mReadyInClass[policy_class] > 0) + { + HttpOpRequest * op(readyq.top()); + readyq.pop(); + + op->stageFromReady(mService); + op->release(); + + --mReadyInClass[policy_class]; + --needed; + } + } + + if (! readyq.empty()) + { + // If anything is ready, continue looping... + result = (std::min)(result, HttpService::NORMAL); + } + } + + return result; +} - op->stageFromReady(mService); - op->release(); + +bool HttpPolicy::changePriority(HttpHandle handle, unsigned int priority) +{ + for (int policy_class(0); policy_class < HttpRequest::POLICY_CLASS_LIMIT; ++policy_class) + { + HttpReadyQueue::container_type & c(mReadyQueue[policy_class].get_container()); + + // Scan ready queue for requests that match policy + for (HttpReadyQueue::container_type::iterator iter(c.begin()); c.end() != iter;) + { + HttpReadyQueue::container_type::iterator cur(iter++); + + if (static_cast(*cur) == handle) + { + HttpOpRequest * op(*cur); + c.erase(cur); // All iterators are now invalidated + op->mReqPriority = priority; + mReadyQueue[policy_class].push(op); // Re-insert using adapter class + return true; + } + } } + + return false; } diff --git a/indra/llcorehttp/_httppolicy.h b/indra/llcorehttp/_httppolicy.h index 192bc73b31..c5e82d0a65 100644 --- a/indra/llcorehttp/_httppolicy.h +++ b/indra/llcorehttp/_httppolicy.h @@ -28,18 +28,23 @@ #define _LLCORE_HTTP_POLICY_H_ -#include +#include "httprequest.h" +#include "_httpservice.h" +#include "_httpreadyqueue.h" namespace LLCore { - -class HttpService; +class HttpReadyQueue; class HttpOpRequest; /// Implements class-based queuing policies for an HttpService instance. +/// +/// Threading: Single-threaded. Other than for construction/destruction, +/// all methods are expected to be invoked in a single thread, typically +/// a worker thread of some sort. class HttpPolicy { public: @@ -51,16 +56,23 @@ private: void operator=(const HttpPolicy &); // Not defined public: - void processReadyQueue(); + /// Give the policy layer some cycles to scan the ready + /// queue promoting higher-priority requests to active + /// as permited. + HttpService::ELoopSpeed processReadyQueue(); + /// Add request to a ready queue. Caller is expected to have + /// provided us with a reference count to hold the request. (No + /// additional references will be added.) void addOp(HttpOpRequest *); + + // Shadows HttpService's method + bool changePriority(HttpHandle handle, unsigned int priority); protected: - typedef std::vector ready_queue_t; - -protected: + int mReadyInClass[HttpRequest::POLICY_CLASS_LIMIT]; + HttpReadyQueue mReadyQueue[HttpRequest::POLICY_CLASS_LIMIT]; HttpService * mService; // Naked pointer, not refcounted, not owner - ready_queue_t mReadyQueue; }; // end class HttpPolicy diff --git a/indra/llcorehttp/_httpreadyqueue.h b/indra/llcorehttp/_httpreadyqueue.h new file mode 100644 index 0000000000..283e868b4c --- /dev/null +++ b/indra/llcorehttp/_httpreadyqueue.h @@ -0,0 +1,85 @@ +/** + * @file _httpreadyqueue.h + * @brief Internal declaration for the operation ready queue + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef _LLCORE_HTTP_READY_QUEUE_H_ +#define _LLCORE_HTTP_READY_QUEUE_H_ + + +#include + +#include "_httpoperation.h" + + +namespace LLCore +{ + +class HttpOpRequest; + +/// HttpReadyQueue provides a simple priority queue for HttpOpRequest objects. +/// +/// This uses the priority_queue adaptor class to provide the queue +/// as well as the ordering scheme while allowing us access to the +/// raw container if we follow a few simple rules. One of the more +/// important of those rules is that any iterator becomes invalid +/// on element erasure. So pay attention. +/// +/// Threading: not thread-safe. Expected to be used entirely by +/// a single thread, typically a worker thread of some sort. + +class HttpReadyQueue : public std::priority_queue, + LLCore::HttpOpCompare> +{ +public: + HttpReadyQueue() + : priority_queue() + {} + + ~HttpReadyQueue() + {} + +protected: + HttpReadyQueue(const HttpReadyQueue &); // Not defined + void operator=(const HttpReadyQueue &); // Not defined + +public: + const container_type & get_container() const + { + return c; + } + + container_type & get_container() + { + return c; + } + +}; // end class HttpReadyQueue + + +} // end namespace LLCore + + +#endif // _LLCORE_HTTP_READY_QUEUE_H_ diff --git a/indra/llcorehttp/_httprequestqueue.cpp b/indra/llcorehttp/_httprequestqueue.cpp index c36814aee3..92bb5ec5c1 100644 --- a/indra/llcorehttp/_httprequestqueue.cpp +++ b/indra/llcorehttp/_httprequestqueue.cpp @@ -57,7 +57,7 @@ HttpRequestQueue::~HttpRequestQueue() void HttpRequestQueue::init() { - LLINT_ASSERT(! sInstance); + llassert_always(! sInstance); sInstance = new HttpRequestQueue(); } diff --git a/indra/llcorehttp/_httpservice.cpp b/indra/llcorehttp/_httpservice.cpp index 6ebc0ec6cb..48884ca060 100644 --- a/indra/llcorehttp/_httpservice.cpp +++ b/indra/llcorehttp/_httpservice.cpp @@ -35,6 +35,8 @@ #include "_httplibcurl.h" #include "_thread.h" +#include "lltimer.h" + namespace LLCore { @@ -89,8 +91,8 @@ HttpService::~HttpService() void HttpService::init(HttpRequestQueue * queue) { - LLINT_ASSERT(! sInstance); - LLINT_ASSERT(NOT_INITIALIZED == sState); + llassert_always(! sInstance); + llassert_always(NOT_INITIALIZED == sState); sInstance = new HttpService(); queue->addRef(); @@ -103,7 +105,7 @@ void HttpService::init(HttpRequestQueue * queue) void HttpService::term() { - LLINT_ASSERT(RUNNING != sState); + llassert_always(RUNNING != sState); if (sInstance) { delete sInstance; @@ -132,8 +134,8 @@ bool HttpService::isStopped() void HttpService::startThread() { - LLINT_ASSERT(! mThread || STOPPED == sState); - LLINT_ASSERT(INITIALIZED == sState || STOPPED == sState); + llassert_always(! mThread || STOPPED == sState); + llassert_always(INITIALIZED == sState || STOPPED == sState); if (mThread) { @@ -150,6 +152,20 @@ void HttpService::stopRequested() mExitRequested = true; } +bool HttpService::changePriority(HttpHandle handle, unsigned int priority) +{ + bool found(false); + + // Skip the request queue as we currently don't leave earlier + // requests sitting there. Start with the ready queue... + found = mPolicy->changePriority(handle, priority); + + // If not there, we could try the transport/active queue but priority + // doesn't really have much effect there so we don't waste cycles. + + return found; +} + void HttpService::shutdown() { @@ -157,38 +173,46 @@ void HttpService::shutdown() } +// Working thread loop-forever method. Gives time to +// each of the request queue, policy layer and transport +// layer pieces and then either sleeps for a small time +// or waits for a request to come in. Repeats until +// requested to stop. void HttpService::threadRun(LLCoreInt::HttpThread * thread) { boost::this_thread::disable_interruption di; - + ELoopSpeed loop(REQUEST_SLEEP); + while (! mExitRequested) { - processRequestQueue(); + loop = processRequestQueue(loop); // Process ready queue issuing new requests as needed - mPolicy->processReadyQueue(); + ELoopSpeed new_loop = mPolicy->processReadyQueue(); + loop = (std::min)(loop, new_loop); // Give libcurl some cycles - mTransport->processTransport(); + new_loop = mTransport->processTransport(); + loop = (std::min)(loop, new_loop); // Determine whether to spin, sleep briefly or sleep for next request - // *FIXME: For now, do this -#if defined(WIN32) - Sleep(50); -#else - usleep(5000); -#endif + if (REQUEST_SLEEP != loop) + { + ms_sleep(50); + } } + shutdown(); sState = STOPPED; } -void HttpService::processRequestQueue() +HttpService::ELoopSpeed HttpService::processRequestQueue(ELoopSpeed loop) { HttpRequestQueue::OpContainer ops; - - mRequestQueue->fetchAll(false, ops); + const bool wait_for_req(REQUEST_SLEEP == loop); + + mRequestQueue->fetchAll(wait_for_req, ops); while (! ops.empty()) { HttpOperation * op(ops.front()); @@ -203,6 +227,9 @@ void HttpService::processRequestQueue() // Done with operation op->release(); } + + // Queue emptied, allow polling loop to sleep + return REQUEST_SLEEP; } diff --git a/indra/llcorehttp/_httpservice.h b/indra/llcorehttp/_httpservice.h index ba76e1eeca..3e5a5457d7 100644 --- a/indra/llcorehttp/_httpservice.h +++ b/indra/llcorehttp/_httpservice.h @@ -28,6 +28,9 @@ #define _LLCORE_HTTP_SERVICE_H_ +#include "httpcommon.h" + + namespace LLCoreInt { @@ -86,7 +89,17 @@ public: RUNNING, ///< thread created and running STOPPED ///< thread has committed to exiting }; - + + // Ordered enumeration of idling strategies available to + // threadRun's loop. Ordered so that std::min on values + // produces the most conservative result of multiple + // requests. + enum ELoopSpeed + { + NORMAL, ///< continuous polling of request, ready, active queues + REQUEST_SLEEP ///< can sleep indefinitely waiting for request queue write + }; + static void init(HttpRequestQueue *); static void term(); @@ -124,6 +137,15 @@ public: /// Threading: callable by worker thread. void shutdown(); + + /// Try to find the given request handle on any of the request + /// queues and reset the priority (and queue position) of the + /// request if found. + /// + /// @return True if the request was found somewhere. + /// + /// Threading: callable by worker thread. + bool changePriority(HttpHandle handle, unsigned int priority); HttpPolicy * getPolicy() { @@ -138,7 +160,7 @@ public: protected: void threadRun(LLCoreInt::HttpThread * thread); - void processRequestQueue(); + ELoopSpeed processRequestQueue(ELoopSpeed loop); protected: static HttpService * sInstance; diff --git a/indra/llcorehttp/_refcounted.h b/indra/llcorehttp/_refcounted.h index 4a6ce8420a..72cef6b342 100644 --- a/indra/llcorehttp/_refcounted.h +++ b/indra/llcorehttp/_refcounted.h @@ -30,7 +30,7 @@ #include -#include "_assert.h" +#include "linden_common.h" namespace LLCoreInt @@ -71,7 +71,7 @@ private: inline void RefCounted::addRef() const { boost::mutex::scoped_lock lock(mRefLock); - LLINT_ASSERT(mRefCount >= 0); + llassert_always(mRefCount >= 0); ++mRefCount; } @@ -82,8 +82,8 @@ inline void RefCounted::release() const { // CRITICAL SECTION boost::mutex::scoped_lock lock(mRefLock); - LLINT_ASSERT(mRefCount != NOT_REF_COUNTED); - LLINT_ASSERT(mRefCount > 0); + llassert_always(mRefCount != NOT_REF_COUNTED); + llassert_always(mRefCount > 0); count = --mRefCount; // CRITICAL SECTION } @@ -104,8 +104,8 @@ inline bool RefCounted::isLastRef() const // CRITICAL SECTION boost::mutex::scoped_lock lock(mRefLock); - LLINT_ASSERT(mRefCount != NOT_REF_COUNTED); - LLINT_ASSERT(mRefCount >= 1); + llassert_always(mRefCount != NOT_REF_COUNTED); + llassert_always(mRefCount >= 1); count = mRefCount; // CRITICAL SECTION } @@ -125,7 +125,7 @@ inline int RefCounted::getRefCount() const inline void RefCounted::noRef() const { boost::mutex::scoped_lock lock(mRefLock); - LLINT_ASSERT(mRefCount <= 1); + llassert_always(mRefCount <= 1); mRefCount = NOT_REF_COUNTED; } diff --git a/indra/llcorehttp/httpcommon.cpp b/indra/llcorehttp/httpcommon.cpp index b5872606b8..273acae132 100644 --- a/indra/llcorehttp/httpcommon.cpp +++ b/indra/llcorehttp/httpcommon.cpp @@ -28,6 +28,7 @@ #include #include +#include namespace LLCore @@ -46,6 +47,16 @@ HttpStatus::operator unsigned long() const } +std::string HttpStatus::toHex() const +{ + std::ostringstream result; + result.width(8); + result.fill('0'); + result << std::hex << operator unsigned long(); + return result.str(); + +} + std::string HttpStatus::toString() const { static const char * llcore_errors[] = @@ -54,7 +65,8 @@ std::string HttpStatus::toString() const "HTTP error reply status", "Services shutting down", "Operation canceled", - "Invalid Content-Range header encountered" + "Invalid Content-Range header encountered", + "Request handle not found" }; static const int llcore_errors_count(sizeof(llcore_errors) / sizeof(llcore_errors[0])); @@ -131,6 +143,7 @@ std::string HttpStatus::toString() const default: if (isHttpStatus()) { + // Binary search for the error code and string int bottom(0), top(http_errors_count); while (true) { diff --git a/indra/llcorehttp/httpcommon.h b/indra/llcorehttp/httpcommon.h index f81be7103e..c01a5f85d3 100644 --- a/indra/llcorehttp/httpcommon.h +++ b/indra/llcorehttp/httpcommon.h @@ -134,7 +134,10 @@ enum HttpError HE_OP_CANCELED = 3, // Invalid content range header received. - HE_INV_CONTENT_RANGE_HDR = 4 + HE_INV_CONTENT_RANGE_HDR = 4, + + // Request handle not found + HE_HANDLE_NOT_FOUND = 5 }; // end enum HttpError @@ -229,6 +232,9 @@ struct HttpStatus { return operator unsigned long(); } + + /// And to convert to a hex string. + std::string toHex() const; /// Convert status to a string representation. For /// success, returns an empty string. For failure diff --git a/indra/llcorehttp/httprequest.cpp b/indra/llcorehttp/httprequest.cpp index 6c62f931ff..a06b859a91 100644 --- a/indra/llcorehttp/httprequest.cpp +++ b/indra/llcorehttp/httprequest.cpp @@ -31,6 +31,7 @@ #include "_httpservice.h" #include "_httpoperation.h" #include "_httpoprequest.h" +#include "_httpopsetpriority.h" #include "_httpopcancel.h" @@ -162,7 +163,7 @@ HttpStatus HttpRequest::getStatus() const HttpHandle HttpRequest::requestGetByteRange(unsigned int policy_id, - float priority, + unsigned int priority, const std::string & url, size_t offset, size_t len, @@ -190,6 +191,34 @@ HttpHandle HttpRequest::requestGetByteRange(unsigned int policy_id, } +HttpHandle HttpRequest::requestPost(unsigned int policy_id, + unsigned int priority, + const std::string & url, + BufferArray * body, + HttpOptions * options, + HttpHeaders * headers, + HttpHandler * user_handler) +{ + HttpStatus status; + HttpHandle handle(LLCORE_HTTP_HANDLE_INVALID); + + HttpOpRequest * op = new HttpOpRequest(); + if (! (status = op->setupPost(policy_id, priority, url, body, options, headers))) + { + op->release(); + mLastReqStatus = status; + return handle; + } + op->setHandlers(mReplyQueue, mSelfHandler, user_handler); + mRequestQueue->addOp(op); // transfers refcount + + mLastReqStatus = status; + handle = static_cast(op); + + return handle; +} + + HttpHandle HttpRequest::requestCancel(HttpHandle handle, HttpHandler * user_handler) { HttpStatus status; @@ -222,6 +251,23 @@ HttpHandle HttpRequest::requestNoOp(HttpHandler * user_handler) } +HttpHandle HttpRequest::requestSetPriority(HttpHandle request, unsigned int priority, + HttpHandler * handler) +{ + HttpStatus status; + HttpHandle ret_handle(LLCORE_HTTP_HANDLE_INVALID); + + HttpOpSetPriority * op = new HttpOpSetPriority(request, priority); + op->setHandlers(mReplyQueue, mSelfHandler, handler); + mRequestQueue->addOp(op); // transfer refcount as well + + mLastReqStatus = status; + ret_handle = static_cast(op); + + return ret_handle; +} + + HttpStatus HttpRequest::update(long millis) { HttpStatus status; @@ -259,7 +305,7 @@ HttpStatus HttpRequest::createService() { HttpStatus status; - LLINT_ASSERT(! has_inited); + llassert_always(! has_inited); HttpRequestQueue::init(); HttpRequestQueue * rq = HttpRequestQueue::instanceOf(); HttpService::init(rq); @@ -273,7 +319,7 @@ HttpStatus HttpRequest::destroyService() { HttpStatus status; - LLINT_ASSERT(has_inited); + llassert_always(has_inited); HttpService::term(); HttpRequestQueue::term(); has_inited = false; diff --git a/indra/llcorehttp/httprequest.h b/indra/llcorehttp/httprequest.h index 4bbd13a13a..e2ab9be533 100644 --- a/indra/llcorehttp/httprequest.h +++ b/indra/llcorehttp/httprequest.h @@ -41,6 +41,7 @@ class HttpService; class HttpOptions; class HttpHeaders; class HttpOperation; +class BufferArray; /// HttpRequest supplies the entry into the HTTP transport /// services in the LLCore libraries. Services provided include: @@ -96,6 +97,10 @@ public: /// Represents a default, catch-all policy class that guarantees /// eventual service for any HTTP request. static const int DEFAULT_POLICY_ID = 0; + + /// Maximum number of policies that may be defined. No policy + /// ID will equal or exceed this value. + static const int POLICY_CLASS_LIMIT = 1; enum EGlobalPolicy { @@ -177,7 +182,7 @@ public: /// @param policy_id Default or user-defined policy class under /// which this request is to be serviced. /// @param priority Standard priority scheme inherited from - /// Indra code base. + /// Indra code base (U32-type scheme). /// @param url /// @param offset /// @param len @@ -190,7 +195,7 @@ public: /// case, @see getStatus() will return more info. /// HttpHandle requestGetByteRange(unsigned int policy_id, - float priority, + unsigned int priority, const std::string & url, size_t offset, size_t len, @@ -199,6 +204,32 @@ public: HttpHandler * handler); + /// + /// @param policy_id Default or user-defined policy class under + /// which this request is to be serviced. + /// @param priority Standard priority scheme inherited from + /// Indra code base. + /// @param url + /// @param body Byte stream to be sent as the body. No + /// further encoding or escaping will be done + /// to the content. + /// @param options (optional) + /// @param headers (optional) + /// @param handler (optional) + /// @return The handle of the request if successfully + /// queued or LLCORE_HTTP_HANDLE_INVALID if the + /// request could not be queued. In the latter + /// case, @see getStatus() will return more info. + /// + HttpHandle requestPost(unsigned int policy_id, + unsigned int priority, + const std::string & url, + BufferArray * body, + HttpOptions * options, + HttpHeaders * headers, + HttpHandler * handler); + + /// Queue a NoOp request. /// The request is queued and serviced by the working thread which /// immediately processes it and returns the request to the reply @@ -235,6 +266,20 @@ public: HttpHandle requestCancel(HttpHandle request, HttpHandler *); + /// Request that a previously-issued request be reprioritized. + /// The status of whether the change itself succeeded arrives + /// via notification. + /// + /// @param request Handle of previously-issued request to + /// be changed. + /// @param priority New priority value. + /// @param handler (optional) + /// @return The handle of the request if successfully + /// queued or LLCORE_HTTP_HANDLE_INVALID if the + /// request could not be queued. + /// + HttpHandle requestSetPriority(HttpHandle request, unsigned int priority, HttpHandler * handler); + /// @} /// @name UtilityMethods diff --git a/indra/llcorehttp/tests/test_httprequest.hpp b/indra/llcorehttp/tests/test_httprequest.hpp index a73d90957e..0e9d7d8979 100644 --- a/indra/llcorehttp/tests/test_httprequest.hpp +++ b/indra/llcorehttp/tests/test_httprequest.hpp @@ -370,7 +370,7 @@ void HttpRequestTestObjectType::test<5>() // Issue a GET that can't connect mStatus = HttpStatus(HttpStatus::EXT_CURL_EASY, CURLE_COULDNT_CONNECT); HttpHandle handle = req->requestGetByteRange(HttpRequest::DEFAULT_POLICY_ID, - 0.0f, + 0U, "http://127.0.0.1:2/nothing/here", 0, 0, diff --git a/indra/newview/lltexturefetch.cpp b/indra/newview/lltexturefetch.cpp index 17c68f7c22..381364b5c3 100644 --- a/indra/newview/lltexturefetch.cpp +++ b/indra/newview/lltexturefetch.cpp @@ -51,6 +51,7 @@ #include "llviewerstats.h" #include "llviewerassetstats.h" #include "llworld.h" +#include "llsdserialize.h" #include "httprequest.h" #include "httphandler.h" @@ -748,7 +749,7 @@ void LLTextureFetchWorker::setImagePriority(F32 priority) { mImagePriority = priority; calcWorkPriority(); - U32 work_priority = mWorkPriority | LLWorkerThread::PRIORITY_LOW; + U32 work_priority = mWorkPriority | (getPriority() & LLWorkerThread::PRIORITY_HIGHBITS); setPriority(work_priority); } } @@ -855,7 +856,6 @@ bool LLTextureFetchWorker::doWork(S32 param) if (mState == LOAD_FROM_TEXTURE_CACHE) { - setPriority(0); // Set priority first since Responder may change it if (mCacheReadHandle == LLTextureCache::nullHandle()) { U32 cache_priority = mWorkPriority; @@ -871,6 +871,8 @@ bool LLTextureFetchWorker::doWork(S32 param) if (mUrl.compare(0, 7, "file://") == 0) { + setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); // Set priority first since Responder may change it + // read file from local disk std::string filename = mUrl.substr(7, std::string::npos); CacheReadResponder* responder = new CacheReadResponder(mFetcher, mID, mFormattedImage); @@ -879,6 +881,8 @@ bool LLTextureFetchWorker::doWork(S32 param) } else if (mUrl.empty()) { + setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); // Set priority first since Responder may change it + CacheReadResponder* responder = new CacheReadResponder(mFetcher, mID, mFormattedImage); mCacheReadHandle = mFetcher->mTextureCache->readFromCache(mID, cache_priority, offset, size, responder); @@ -890,12 +894,12 @@ bool LLTextureFetchWorker::doWork(S32 param) // *TODO:?remove this warning llwarns << "Unknown URL Type: " << mUrl << llendl; } - setPriority(LLWorkerThread::PRIORITY_HIGH); + setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); mState = SEND_HTTP_REQ; } else { - setPriority(LLWorkerThread::PRIORITY_HIGH); + setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); mState = LOAD_FROM_NETWORK; } } @@ -907,7 +911,6 @@ bool LLTextureFetchWorker::doWork(S32 param) { mCacheReadHandle = LLTextureCache::nullHandle(); mState = CACHE_POST; - setPriority(LLWorkerThread::PRIORITY_HIGH); // fall through } else @@ -931,7 +934,6 @@ bool LLTextureFetchWorker::doWork(S32 param) llassert_always(mFormattedImage->getDataSize() > 0); mLoadedDiscard = mDesiredDiscard; mState = DECODE_IMAGE; - setPriority(LLWorkerThread::PRIORITY_HIGH); mWriteToCacheState = NOT_WRITE ; LL_DEBUGS("Texture") << mID << ": Cached. Bytes: " << mFormattedImage->getDataSize() << " Size: " << llformat("%dx%d",mFormattedImage->getWidth(),mFormattedImage->getHeight()) @@ -949,7 +951,6 @@ bool LLTextureFetchWorker::doWork(S32 param) else { LL_DEBUGS("Texture") << mID << ": Not in Cache" << LL_ENDL; - setPriority(LLWorkerThread::PRIORITY_HIGH); mState = LOAD_FROM_NETWORK; } // fall through @@ -960,7 +961,6 @@ bool LLTextureFetchWorker::doWork(S32 param) { static LLCachedControl use_http(gSavedSettings,"ImagePipelineUseHTTP"); - setPriority(0); // if (mHost != LLHost::invalid) get_url = false; if ( use_http && mCanUseHTTP && mUrl.empty())//get http url. { @@ -993,7 +993,7 @@ bool LLTextureFetchWorker::doWork(S32 param) if (mCanUseHTTP && !mUrl.empty()) { mState = SEND_HTTP_REQ; - setPriority(LLWorkerThread::PRIORITY_HIGH); + setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); if(mWriteToCacheState != NOT_WRITE) { mWriteToCacheState = CAN_WRITE ; @@ -1010,6 +1010,7 @@ bool LLTextureFetchWorker::doWork(S32 param) mSentRequest = QUEUED; mFetcher->addToNetworkQueue(this); recordTextureStart(false); + setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); return false; } @@ -1027,7 +1028,6 @@ bool LLTextureFetchWorker::doWork(S32 param) if (mState == LOAD_FROM_SIMULATOR) { - setPriority(0); if (mFormattedImage.isNull()) { mFormattedImage = new LLImageJ2C; @@ -1042,7 +1042,7 @@ bool LLTextureFetchWorker::doWork(S32 param) // llwarns << "processSimulatorPackets() failed to load buffer" << llendl; return true; // failed } - setPriority(LLWorkerThread::PRIORITY_HIGH); + setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); mState = DECODE_IMAGE; mWriteToCacheState = SHOULD_WRITE; recordTextureDone(false); @@ -1050,6 +1050,7 @@ bool LLTextureFetchWorker::doWork(S32 param) else { mFetcher->addToNetworkQueue(this); // failsafe + setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); recordTextureStart(false); } return false; @@ -1057,7 +1058,6 @@ bool LLTextureFetchWorker::doWork(S32 param) if (mState == SEND_HTTP_REQ) { - setPriority(0); if(mCanUseHTTP) { //NOTE: @@ -1065,11 +1065,13 @@ bool LLTextureFetchWorker::doWork(S32 param) //1, not openning too many file descriptors at the same time; //2, control the traffic of http so udp gets bandwidth. // - static const S32 MAX_NUM_OF_HTTP_REQUESTS_IN_QUEUE = 8 ; - if(mFetcher->getNumHTTPRequests() > MAX_NUM_OF_HTTP_REQUESTS_IN_QUEUE) - { - return false ; //wait. - } + static const S32 MAX_NUM_OF_HTTP_REQUESTS_IN_QUEUE = 8; + // *FIXME: For the moment, allow everything to transition into HTTP + // and have the new library order and throttle. + //if(mFetcher->getNumHTTPRequests() > MAX_NUM_OF_HTTP_REQUESTS_IN_QUEUE) + //{ + //return false ; //wait. + //} mFetcher->removeFromNetworkQueue(this, false); @@ -1083,7 +1085,6 @@ bool LLTextureFetchWorker::doWork(S32 param) { // We already have all the data, just decode it mLoadedDiscard = mFormattedImage->getDiscardLevel(); - setPriority(LLWorkerThread::PRIORITY_HIGH); mState = DECODE_IMAGE; return false; } @@ -1135,7 +1136,7 @@ bool LLTextureFetchWorker::doWork(S32 param) mHttpActive = true; mFetcher->addToHTTPQueue(mID); recordTextureStart(true); - setPriority(LLWorkerThread::PRIORITY_HIGH); + setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); mState = WAIT_HTTP_REQ; // fall through @@ -1148,7 +1149,6 @@ bool LLTextureFetchWorker::doWork(S32 param) if (mState == WAIT_HTTP_REQ) { - setPriority(0); if (mLoaded) { S32 cur_size = mFormattedImage.notNull() ? mFormattedImage->getDataSize() : 0; @@ -1165,7 +1165,7 @@ bool LLTextureFetchWorker::doWork(S32 param) { mState = INIT; mCanUseHTTP = false; - setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); + setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); return false; } } @@ -1184,7 +1184,8 @@ bool LLTextureFetchWorker::doWork(S32 param) max_attempts = HTTP_MAX_RETRY_COUNT + 1; ++mHTTPFailCount; llinfos << "HTTP GET failed for: " << mUrl - << " Status: " << mGetStatus.toULong() << " Reason: '" << mGetReason << "'" + << " Status: " << mGetStatus.toHex() + << " Reason: '" << mGetReason << "'" << " Attempt:" << mHTTPFailCount+1 << "/" << max_attempts << llendl; } @@ -1194,14 +1195,12 @@ bool LLTextureFetchWorker::doWork(S32 param) { // Use available data mLoadedDiscard = mFormattedImage->getDiscardLevel(); - setPriority(LLWorkerThread::PRIORITY_HIGH); mState = DECODE_IMAGE; return false; } else { resetFormattedData(); - setPriority(LLWorkerThread::PRIORITY_HIGH); mState = DONE; return true; // failed } @@ -1223,7 +1222,6 @@ bool LLTextureFetchWorker::doWork(S32 param) } // abort. - setPriority(LLWorkerThread::PRIORITY_HIGH); mState = DONE; return true; } @@ -1268,29 +1266,30 @@ bool LLTextureFetchWorker::doWork(S32 param) mHttpBufferArray = NULL; mLoadedDiscard = mRequestedDiscard; - setPriority(LLWorkerThread::PRIORITY_HIGH); mState = DECODE_IMAGE; if (mWriteToCacheState != NOT_WRITE) { mWriteToCacheState = SHOULD_WRITE ; } + setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); return false; } else { + setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); return false; } } if (mState == DECODE_IMAGE) { - setPriority(0); static LLCachedControl textures_decode_disabled(gSavedSettings,"TextureDecodeDisabled"); + + setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); // Set priority first since Responder may change it if (textures_decode_disabled) { // for debug use, don't decode mState = DONE; - setPriority(LLWorkerThread::PRIORITY_HIGH); return true; } @@ -1298,7 +1297,6 @@ bool LLTextureFetchWorker::doWork(S32 param) { // We aborted, don't decode mState = DONE; - setPriority(LLWorkerThread::PRIORITY_HIGH); return true; } @@ -1308,7 +1306,6 @@ bool LLTextureFetchWorker::doWork(S32 param) //abort, don't decode mState = DONE; - setPriority(LLWorkerThread::PRIORITY_HIGH); return true; } if (mLoadedDiscard < 0) @@ -1317,7 +1314,6 @@ bool LLTextureFetchWorker::doWork(S32 param) //abort, don't decode mState = DONE; - setPriority(LLWorkerThread::PRIORITY_HIGH); return true; } @@ -1337,7 +1333,6 @@ bool LLTextureFetchWorker::doWork(S32 param) if (mState == DECODE_IMAGE_UPDATE) { - setPriority(0); if (mDecoded) { if (mDecodedDiscard < 0) @@ -1350,14 +1345,13 @@ bool LLTextureFetchWorker::doWork(S32 param) llassert_always(mDecodeHandle == 0); mFormattedImage = NULL; ++mRetryAttempt; - setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); + setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); mState = INIT; return false; } else { // llwarns << "UNABLE TO LOAD TEXTURE: " << mID << " RETRIES: " << mRetryAttempt << llendl; - setPriority(LLWorkerThread::PRIORITY_HIGH); mState = DONE; // failed } } @@ -1366,7 +1360,7 @@ bool LLTextureFetchWorker::doWork(S32 param) llassert_always(mRawImage.notNull()); LL_DEBUGS("Texture") << mID << ": Decoded. Discard: " << mDecodedDiscard << " Raw Image: " << llformat("%dx%d",mRawImage->getWidth(),mRawImage->getHeight()) << LL_ENDL; - setPriority(LLWorkerThread::PRIORITY_HIGH); + setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); mState = WRITE_TO_CACHE; } // fall through @@ -1379,12 +1373,10 @@ bool LLTextureFetchWorker::doWork(S32 param) if (mState == WRITE_TO_CACHE) { - setPriority(0); if (mWriteToCacheState != SHOULD_WRITE || mFormattedImage.isNull()) { // If we're in a local cache or we didn't actually receive any new data, // or we failed to load anything, skip - setPriority(LLWorkerThread::PRIORITY_HIGH); mState = DONE; return false; } @@ -1404,7 +1396,6 @@ bool LLTextureFetchWorker::doWork(S32 param) setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); // Set priority first since Responder may change it U32 cache_priority = mWorkPriority; mWritten = FALSE; - setPriority(LLWorkerThread::PRIORITY_HIGH); mState = WAIT_ON_WRITE; CacheWriteResponder* responder = new CacheWriteResponder(mFetcher, mID); mCacheWriteHandle = mFetcher->mTextureCache->writeToCache(mID, cache_priority, @@ -1415,10 +1406,8 @@ bool LLTextureFetchWorker::doWork(S32 param) if (mState == WAIT_ON_WRITE) { - setPriority(0); if (writeToCacheComplete()) { - setPriority(LLWorkerThread::PRIORITY_HIGH); mState = DONE; // fall through } @@ -1437,15 +1426,16 @@ bool LLTextureFetchWorker::doWork(S32 param) if (mState == DONE) { - setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); if (mDecodedDiscard >= 0 && mDesiredDiscard < mDecodedDiscard) { // More data was requested, return to INIT mState = INIT; + setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); return false; } else { + setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); return true; } } @@ -1477,7 +1467,8 @@ void LLTextureFetchWorker::onCompleted(LLCore::HttpHandle handle, LLCore::HttpRe LLCore::HttpStatus status(response->getStatus()); lldebugs << "HTTP COMPLETE: " << mID - << " status: " << status.toULong() << " '" << status.toString() << "'" + << " status: " << status.toHex() + << " '" << status.toString() << "'" << llendl; unsigned int offset(0), length(0); response->getRange(&offset, &length); @@ -1492,7 +1483,8 @@ void LLTextureFetchWorker::onCompleted(LLCore::HttpHandle handle, LLCore::HttpRe success = false; std::string reason(status.toString()); setGetStatus(status, reason); - llwarns << "CURL GET FAILED, status:" << status.toULong() << " reason:" << reason << llendl; + llwarns << "CURL GET FAILED, status: " << status.toHex() + << " reason: " << reason << llendl; } else { @@ -1727,7 +1719,7 @@ S32 LLTextureFetchWorker::callbackHttpGet(LLCore::HttpResponse * response, mRequestedSize = -1; // error } mLoaded = TRUE; - setPriority(LLWorkerThread::PRIORITY_HIGH); + setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); return data_size ; } @@ -1756,7 +1748,7 @@ void LLTextureFetchWorker::callbackCacheRead(bool success, LLImageFormatted* ima } } mLoaded = TRUE; - setPriority(LLWorkerThread::PRIORITY_HIGH); + setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); } void LLTextureFetchWorker::callbackCacheWrite(bool success) @@ -1768,7 +1760,7 @@ void LLTextureFetchWorker::callbackCacheWrite(bool success) return; } mWritten = TRUE; - setPriority(LLWorkerThread::PRIORITY_HIGH); + setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); } ////////////////////////////////////////////////////////////////////////////// @@ -1806,7 +1798,7 @@ void LLTextureFetchWorker::callbackDecoded(bool success, LLImageRaw* raw, LLImag } mDecoded = TRUE; // llinfos << mID << " : DECODE COMPLETE " << llendl; - setPriority(LLWorkerThread::PRIORITY_HIGH); + setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); } ////////////////////////////////////////////////////////////////////////////// @@ -1883,7 +1875,6 @@ LLTextureFetch::LLTextureFetch(LLTextureCache* cache, LLImageDecodeThread* image mHttpOptions(NULL), mHttpHeaders(NULL) { - mCurlPOSTRequestCount = 0; mMaxBandwidth = gSavedSettings.getF32("ThrottleBandwidthKBPS"); mTextureInfo.setUpLogging(gSavedSettings.getBOOL("LogTextureDownloadsToViewerLog"), gSavedSettings.getBOOL("LogTextureDownloadsToSimulator"), gSavedSettings.getU32("TextureLoggingThreshold")); @@ -2253,7 +2244,6 @@ S32 LLTextureFetch::getPending() LLMutexLock lock(&mQueueMutex); res = mRequestQueue.size(); - res += mCurlPOSTRequestCount; res += mCommands.size(); } unlockData(); @@ -2279,10 +2269,7 @@ bool LLTextureFetch::runCondition() have_no_commands = mCommands.empty(); } - bool have_no_curl_requests(0 == mCurlPOSTRequestCount); - return ! (have_no_commands - && have_no_curl_requests && (mRequestQueue.empty() && mIdleThread)); // From base class } @@ -2690,7 +2677,7 @@ bool LLTextureFetch::receiveImageHeader(const LLHost& host, const LLUUID& id, U8 llassert_always(totalbytes > 0); llassert_always(data_size == FIRST_PACKET_SIZE || data_size == worker->mFileSize); res = worker->insertPacket(0, data, data_size); - worker->setPriority(LLWorkerThread::PRIORITY_HIGH); + worker->setPriority(LLWorkerThread::PRIORITY_HIGH | worker->mWorkPriority); worker->mState = LLTextureFetchWorker::LOAD_FROM_SIMULATOR; worker->unlockWorkMutex(); return res; @@ -2734,7 +2721,7 @@ bool LLTextureFetch::receiveImagePacket(const LLHost& host, const LLUUID& id, U1 if ((worker->mState == LLTextureFetchWorker::LOAD_FROM_SIMULATOR) || (worker->mState == LLTextureFetchWorker::LOAD_FROM_NETWORK)) { - worker->setPriority(LLWorkerThread::PRIORITY_HIGH); + worker->setPriority(LLWorkerThread::PRIORITY_HIGH | worker->mWorkPriority); worker->mState = LLTextureFetchWorker::LOAD_FROM_SIMULATOR; } else @@ -2930,6 +2917,35 @@ void LLTextureFetch::cmdDoWork() namespace { + +// Example of a simple notification handler for metrics +// delivery notification. Earlier versions of the code used +// a Responder that tried harder to detect delivery breaks +// but it really isn't that important. If someone wants to +// revisit that effort, here is a place to start. +class AssetReportHandler : public LLCore::HttpHandler +{ +public: + virtual void onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * response) + { + LLCore::HttpStatus status(response->getStatus()); + + if (status) + { + LL_WARNS("Texture") << "Successfully delivered asset metrics to grid." + << LL_ENDL; + } + else + { + LL_WARNS("Texture") << "Error delivering asset metrics to grid. Reason: " + << status.toString() << LL_ENDL; + } + } +}; // end class AssetReportHandler + +AssetReportHandler stats_handler; + + /** * Implements the 'Set Region' command. * @@ -2960,73 +2976,9 @@ TFReqSendMetrics::~TFReqSendMetrics() bool TFReqSendMetrics::doWork(LLTextureFetch * fetcher) { - /* - * HTTP POST responder. Doesn't do much but tries to - * detect simple breaks in recording the metrics stream. - * - * The 'volatile' modifiers don't indicate signals, - * mmap'd memory or threads, really. They indicate that - * the referenced data is part of a pseudo-closure for - * this responder rather than being required for correct - * operation. - * - * We don't try very hard with the POST request. We give - * it one shot and that's more-or-less it. With a proper - * refactoring of the LLQueuedThread usage, these POSTs - * could be put in a request object and made more reliable. - */ - class lcl_responder : public LLCurl::Responder - { - public: - lcl_responder(LLTextureFetch * fetcher, - S32 expected_sequence, - volatile const S32 & live_sequence, - volatile bool & reporting_break, - volatile bool & reporting_started) - : LLCurl::Responder(), - mFetcher(fetcher), - mExpectedSequence(expected_sequence), - mLiveSequence(live_sequence), - mReportingBreak(reporting_break), - mReportingStarted(reporting_started) - { - mFetcher->incrCurlPOSTCount(); - } - - ~lcl_responder() - { - mFetcher->decrCurlPOSTCount(); - } - - // virtual - void error(U32 status_num, const std::string & reason) - { - if (mLiveSequence == mExpectedSequence) - { - mReportingBreak = true; - } - LL_WARNS("Texture") << "Break in metrics stream due to POST failure to metrics collection service. Reason: " - << reason << LL_ENDL; - } - - // virtual - void result(const LLSD & content) - { - if (mLiveSequence == mExpectedSequence) - { - mReportingBreak = false; - mReportingStarted = true; - } - } - - private: - LLTextureFetch * mFetcher; - S32 mExpectedSequence; - volatile const S32 & mLiveSequence; - volatile bool & mReportingBreak; - volatile bool & mReportingStarted; - - }; // class lcl_responder + static const U32 report_priority(LLWorkerThread::PRIORITY_LOW); + static const int report_policy_class(LLCore::HttpRequest::DEFAULT_POLICY_ID); + static LLCore::HttpHandler * const handler(fetcher->isQAMode() || true ? &stats_handler : NULL); if (! gViewerAssetStatsThread1) return true; @@ -3054,24 +3006,37 @@ TFReqSendMetrics::doWork(LLTextureFetch * fetcher) // Update sequence number if (S32_MAX == ++report_sequence) report_sequence = 0; - + reporting_started = true; + // Limit the size of the stats report if necessary. merged_llsd["truncated"] = truncate_viewer_metrics(10, merged_llsd); if (! mCapsURL.empty()) { - LLCurlRequest::headers_t headers; -#if 0 - // *FIXME: Going to need a post op after all... - fetcher->getCurlRequest().post(mCapsURL, - headers, - merged_llsd, - new lcl_responder(fetcher, - report_sequence, - report_sequence, - LLTextureFetch::svMetricsDataBreak, - reporting_started)); -#endif + // *FIXME: This mess to get an llsd into a string though + // it's actually no worse than what we currently do... + std::stringstream body; + LLSDSerialize::toXML(merged_llsd, body); + std::string body_str(body.str()); + body.clear(); + + LLCore::HttpHeaders * headers = new LLCore::HttpHeaders; + headers->mHeaders.push_back("Content-Type: application/llsd+xml"); + + LLCore::BufferArray * ba = new LLCore::BufferArray; + ba->append(body_str.c_str(), body_str.length()); + body_str.clear(); + + fetcher->getHttpRequest().requestPost(report_policy_class, + report_priority, + mCapsURL, + ba, + NULL, + headers, + handler); + ba->release(); + headers->release(); + LLTextureFetch::svMetricsDataBreak = false; } else { @@ -3079,7 +3044,7 @@ TFReqSendMetrics::doWork(LLTextureFetch * fetcher) } // In QA mode, Metrics submode, log the result for ease of testing - if (fetcher->isQAMode()) + if (fetcher->isQAMode() || true) { LL_INFOS("Textures") << merged_llsd << LL_ENDL; } diff --git a/indra/newview/lltexturefetch.h b/indra/newview/lltexturefetch.h index 402b198246..cfea3aad9d 100644 --- a/indra/newview/lltexturefetch.h +++ b/indra/newview/lltexturefetch.h @@ -39,7 +39,6 @@ class LLViewerTexture; class LLTextureFetchWorker; -class HTTPGetResponder; class LLTextureCache; class LLImageDecodeThread; class LLHost; @@ -49,7 +48,6 @@ class LLViewerAssetStats; class LLTextureFetch : public LLWorkerThread { friend class LLTextureFetchWorker; - friend class HTTPGetResponder; public: LLTextureFetch(LLTextureCache* cache, LLImageDecodeThread* imagedecodethread, bool threaded, bool qa_mode); @@ -90,8 +88,6 @@ public: LLTextureFetchWorker* getWorker(const LLUUID& id); LLTextureFetchWorker* getWorkerAfterLock(const LLUUID& id); - LLTextureInfo* getTextureInfo() { return &mTextureInfo; } - // Commands available to other threads to control metrics gathering operations. void commandSetRegion(U64 region_handle); void commandSendMetrics(const std::string & caps_url, @@ -104,10 +100,6 @@ public: bool isQAMode() const { return mQAMode; } - // Curl POST counter maintenance - inline void incrCurlPOSTCount() { mCurlPOSTRequestCount++; } - inline void decrCurlPOSTCount() { mCurlPOSTRequestCount--; } - protected: void addToNetworkQueue(LLTextureFetchWorker* worker); void removeFromNetworkQueue(LLTextureFetchWorker* worker, bool cancel); @@ -199,13 +191,6 @@ private: // If true, modifies some behaviors that help with QA tasks. const bool mQAMode; - // Count of POST requests outstanding. We maintain the count - // indirectly in the CURL request responder's ctor and dtor and - // use it when determining whether or not to sleep the thread. Can't - // use the LLCurl module's request counter as it isn't thread compatible. - // *NOTE: Don't mix Atomic and static, apr_initialize must be called first. - LLAtomic32 mCurlPOSTRequestCount; - // Interfaces and objects into the core http library used // to make our HTTP requests. These replace the various // LLCurl interfaces used in the past. -- cgit v1.3 From fb5a29c069d27611b6328fbc313382ef0914ffe9 Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Fri, 1 Jun 2012 23:06:24 +0000 Subject: Platform fixups Linux: unused variables, make error strings constant. --- indra/llcommon/llsdserialize.cpp | 5 ++++- indra/llcommon/tests/bitpack_test.cpp | 1 + indra/llcommon/tests/reflection_test.cpp | 2 +- indra/llcorehttp/httpcommon.cpp | 2 +- indra/llmessage/llhttpassetstorage.cpp | 2 +- indra/newview/lltexturefetch.cpp | 10 +++++----- 6 files changed, 13 insertions(+), 9 deletions(-) (limited to 'indra/newview') diff --git a/indra/llcommon/llsdserialize.cpp b/indra/llcommon/llsdserialize.cpp index bf62600514..f29c96f17c 100644 --- a/indra/llcommon/llsdserialize.cpp +++ b/indra/llcommon/llsdserialize.cpp @@ -1447,9 +1447,12 @@ S32 LLSDBinaryFormatter::format(const LLSD& data, std::ostream& ostr, U32 option } case LLSD::TypeUUID: + { ostr.put('u'); - ostr.write((const char*)(&(data.asUUID().mData)), UUID_BYTES); + LLSD::UUID value = data.asUUID(); + ostr.write((const char*)(&value.mData), UUID_BYTES); break; + } case LLSD::TypeString: ostr.put('s'); diff --git a/indra/llcommon/tests/bitpack_test.cpp b/indra/llcommon/tests/bitpack_test.cpp index 05289881d0..4c3bc674af 100644 --- a/indra/llcommon/tests/bitpack_test.cpp +++ b/indra/llcommon/tests/bitpack_test.cpp @@ -95,6 +95,7 @@ namespace tut ensure("bitPack: individual unpack: 5", unpackbuffer[0] == (U8) str[5]); unpack_bufsize = bitunpack.bitUnpack(unpackbuffer, 8*4); // Life ensure_memory_matches("bitPack: 4 bytes unpack:", unpackbuffer, 4, str+6, 4); + ensure("keep compiler quiet", unpack_bufsize == unpack_bufsize); } // U32 packing diff --git a/indra/llcommon/tests/reflection_test.cpp b/indra/llcommon/tests/reflection_test.cpp index 59491cd1fe..8980ebb1f1 100644 --- a/indra/llcommon/tests/reflection_test.cpp +++ b/indra/llcommon/tests/reflection_test.cpp @@ -207,7 +207,7 @@ namespace tut const LLReflective* reflective = property->get(aggregated_data); // Wrong reflective type, should throw exception. // useless op to get rid of compiler warning. - reflective = NULL; + reflective = reflective; } catch(...) { diff --git a/indra/llcorehttp/httpcommon.cpp b/indra/llcorehttp/httpcommon.cpp index 273acae132..a01182cf23 100644 --- a/indra/llcorehttp/httpcommon.cpp +++ b/indra/llcorehttp/httpcommon.cpp @@ -73,7 +73,7 @@ std::string HttpStatus::toString() const static const struct { type_enum_t mCode; - char * mText; + const char * mText; } http_errors[] = { diff --git a/indra/llmessage/llhttpassetstorage.cpp b/indra/llmessage/llhttpassetstorage.cpp index 612d765969..d6ed08055e 100644 --- a/indra/llmessage/llhttpassetstorage.cpp +++ b/indra/llmessage/llhttpassetstorage.cpp @@ -749,7 +749,7 @@ LLAssetRequest* LLHTTPAssetStorage::findNextRequest(LLAssetStorage::request_list request_list_t::iterator pending_iter = pending.begin(); request_list_t::iterator pending_end = pending.end(); // Loop over all pending requests until we miss finding it in the running list. - for (; pending_iter != pending.end(); ++pending_iter) + for (; pending_iter != pending_end; ++pending_iter) { LLAssetRequest* req = *pending_iter; // Look for this pending request in the running list. diff --git a/indra/newview/lltexturefetch.cpp b/indra/newview/lltexturefetch.cpp index 381364b5c3..daad463e0d 100644 --- a/indra/newview/lltexturefetch.cpp +++ b/indra/newview/lltexturefetch.cpp @@ -1065,13 +1065,13 @@ bool LLTextureFetchWorker::doWork(S32 param) //1, not openning too many file descriptors at the same time; //2, control the traffic of http so udp gets bandwidth. // - static const S32 MAX_NUM_OF_HTTP_REQUESTS_IN_QUEUE = 8; + // static const S32 MAX_NUM_OF_HTTP_REQUESTS_IN_QUEUE = 8; // *FIXME: For the moment, allow everything to transition into HTTP // and have the new library order and throttle. - //if(mFetcher->getNumHTTPRequests() > MAX_NUM_OF_HTTP_REQUESTS_IN_QUEUE) - //{ - //return false ; //wait. - //} + // if(mFetcher->getNumHTTPRequests() > MAX_NUM_OF_HTTP_REQUESTS_IN_QUEUE) + // { + // return false ; //wait. + // } mFetcher->removeFromNetworkQueue(this, false); -- cgit v1.3 From fe5c1683f0e13e8a3f0523095c1c7e3a3fd17cf3 Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Fri, 1 Jun 2012 20:49:00 -0400 Subject: Another float/int issue and move the POST priority in line with what normal requests do... --- indra/newview/lltexturefetch.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'indra/newview') diff --git a/indra/newview/lltexturefetch.cpp b/indra/newview/lltexturefetch.cpp index daad463e0d..97d7ec5531 100644 --- a/indra/newview/lltexturefetch.cpp +++ b/indra/newview/lltexturefetch.cpp @@ -1117,7 +1117,7 @@ bool LLTextureFetchWorker::doWork(S32 param) // Will call callbackHttpGet when curl request completes // *FIXME: enable redirection follow mHttpHandle = mFetcher->mHttpRequest->requestGetByteRange(mHttpPolicyClass, - mRequestedPriority, + mWorkPriority, mUrl, mRequestedOffset, mRequestedSize, @@ -2976,7 +2976,7 @@ TFReqSendMetrics::~TFReqSendMetrics() bool TFReqSendMetrics::doWork(LLTextureFetch * fetcher) { - static const U32 report_priority(LLWorkerThread::PRIORITY_LOW); + static const U32 report_priority(1); static const int report_policy_class(LLCore::HttpRequest::DEFAULT_POLICY_ID); static LLCore::HttpHandler * const handler(fetcher->isQAMode() || true ? &stats_handler : NULL); -- cgit v1.3 From 4155301015525a242a79b9b3134e66d366bc0ebd Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Fri, 1 Jun 2012 21:30:45 -0400 Subject: Do some work on BufferArray to make it a bit less naive about chunking data. Remove the stateful use of a seek pointer so that shared read is possible (though maybe not interesting). --- indra/llcorehttp/_httpoprequest.cpp | 14 ++-- indra/llcorehttp/bufferarray.cpp | 126 +++++++++++++++++++--------- indra/llcorehttp/bufferarray.h | 53 ++---------- indra/llcorehttp/tests/test_bufferarray.hpp | 89 ++++++++------------ indra/newview/lltexturefetch.cpp | 3 +- 5 files changed, 133 insertions(+), 152 deletions(-) (limited to 'indra/newview') diff --git a/indra/llcorehttp/_httpoprequest.cpp b/indra/llcorehttp/_httpoprequest.cpp index 791ee5f860..196011f953 100644 --- a/indra/llcorehttp/_httpoprequest.cpp +++ b/indra/llcorehttp/_httpoprequest.cpp @@ -340,7 +340,6 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service) long data_size(0); if (mReqBody) { - mReqBody->seek(0); data_size = mReqBody->size(); } curl_easy_setopt(mCurlHandle, CURLOPT_POSTFIELDS, static_cast(NULL)); @@ -356,7 +355,6 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service) long data_size(0); if (mReqBody) { - mReqBody->seek(0); data_size = mReqBody->size(); } curl_easy_setopt(mCurlHandle, CURLOPT_INFILESIZE, data_size); @@ -423,10 +421,8 @@ size_t HttpOpRequest::writeCallback(void * data, size_t size, size_t nmemb, void op->mReplyBody = new BufferArray(); } const size_t req_size(size * nmemb); - char * lump(op->mReplyBody->appendBufferAlloc(req_size)); - memcpy(lump, data, req_size); - - return req_size; + const size_t write_size(op->mReplyBody->append(static_cast(data), req_size)); + return write_size; } @@ -450,9 +446,9 @@ size_t HttpOpRequest::readCallback(void * data, size_t size, size_t nmemb, void } const size_t do_size((std::min)(req_size, body_size - op->mCurlBodyPos)); - op->mReqBody->read(static_cast(data), do_size); - op->mCurlBodyPos += do_size; - return do_size; + const size_t read_size(op->mReqBody->read(op->mCurlBodyPos, static_cast(data), do_size)); + op->mCurlBodyPos += read_size; + return read_size; } diff --git a/indra/llcorehttp/bufferarray.cpp b/indra/llcorehttp/bufferarray.cpp index 4c20350b13..c13b1a3540 100644 --- a/indra/llcorehttp/bufferarray.cpp +++ b/indra/llcorehttp/bufferarray.cpp @@ -27,6 +27,18 @@ #include "bufferarray.h" +// BufferArray is a list of chunks, each a BufferArray::Block, of contiguous +// data presented as a single array. Chunks are at least BufferArray::BLOCK_ALLOC_SIZE +// in length and can be larger. Any chunk may be partially filled or even +// empty. +// +// The BufferArray itself is sharable as a RefCounted entity. As shared +// reads don't work with the concept of a current position/seek value, +// none is kept with the object. Instead, the read and write operations +// all take position arguments. Single write/shared read isn't supported +// directly and any such attempts have to be serialized outside of this +// implementation. + namespace LLCore { @@ -58,7 +70,8 @@ public: static Block * alloc(size_t len); public: - size_t mLen; + size_t mUsed; + size_t mAlloced; // *NOTE: Must be last member of the object. We'll // overallocate as requested via operator new and index @@ -74,7 +87,6 @@ public: BufferArray::BufferArray() : LLCoreInt::RefCounted(true), - mPos(0), mLen(0) {} @@ -94,19 +106,45 @@ BufferArray::~BufferArray() size_t BufferArray::append(const char * src, size_t len) { - if (len) + const size_t ret(len); + + // First, try to copy into the last block + if (len && ! mBlocks.empty()) + { + Block & last(*mBlocks.back()); + if (last.mUsed < last.mAlloced) + { + // Some will fit... + const size_t copy_len((std::min)(len, (last.mAlloced - last.mUsed))); + + memcpy(&last.mData[last.mUsed], src, copy_len); + last.mUsed += copy_len; + llassert_always(last.mUsed <= last.mAlloced); + mLen += copy_len; + src += copy_len; + len -= copy_len; + } + } + + // Then get new blocks as needed + while (len) { + const size_t copy_len((std::min)(len, BLOCK_ALLOC_SIZE)); + if (mBlocks.size() >= mBlocks.capacity()) { mBlocks.reserve(mBlocks.size() + 5); } - Block * block = Block::alloc(len); - memcpy(block->mData, src, len); + Block * block = Block::alloc(BLOCK_ALLOC_SIZE); + memcpy(block->mData, src, copy_len); + block->mUsed = copy_len; + llassert_always(block->mUsed <= block->mAlloced); mBlocks.push_back(block); - mLen += len; - mPos = mLen; + mLen += copy_len; + src += copy_len; + len -= copy_len; } - return len; + return ret; } @@ -117,41 +155,33 @@ char * BufferArray::appendBufferAlloc(size_t len) { mBlocks.reserve(mBlocks.size() + 5); } - Block * block = Block::alloc(len); + Block * block = Block::alloc((std::max)(BLOCK_ALLOC_SIZE, len)); + block->mUsed = len; mBlocks.push_back(block); mLen += len; - mPos = mLen; return block->mData; } -size_t BufferArray::seek(size_t pos) +size_t BufferArray::read(size_t pos, char * dst, size_t len) { - if (pos > mLen) - pos = mLen; - mPos = pos; - return mPos; -} - - -size_t BufferArray::read(char * dst, size_t len) -{ - size_t result(0), offset(0); - size_t len_limit(mLen - mPos); + if (pos >= mLen) + return 0; + size_t len_limit(mLen - pos); len = std::min(len, len_limit); - - if (mPos >= mLen || 0 == len) + if (0 == len) return 0; + size_t result(0), offset(0); const int block_limit(mBlocks.size()); - int block_start(findBlock(mPos, &offset)); + int block_start(findBlock(pos, &offset)); if (block_start < 0) return 0; do { Block & block(*mBlocks[block_start]); - size_t block_limit(block.mLen - offset); + size_t block_limit(block.mUsed - offset); size_t block_len(std::min(block_limit, len)); memcpy(dst, &block.mData[offset], block_len); @@ -163,19 +193,18 @@ size_t BufferArray::read(char * dst, size_t len) } while (len && block_start < block_limit); - mPos += result; return result; } -size_t BufferArray::write(const char * src, size_t len) +size_t BufferArray::write(size_t pos, const char * src, size_t len) { - size_t result(0), offset(0); - if (mPos > mLen || 0 == len) + if (pos > mLen || 0 == len) return 0; + size_t result(0), offset(0); const int block_limit(mBlocks.size()); - int block_start(findBlock(mPos, &offset)); + int block_start(findBlock(pos, &offset)); if (block_start >= 0) { @@ -184,20 +213,39 @@ size_t BufferArray::write(const char * src, size_t len) do { Block & block(*mBlocks[block_start]); - size_t block_limit(block.mLen - offset); + size_t block_limit(block.mUsed - offset); size_t block_len(std::min(block_limit, len)); memcpy(&block.mData[offset], src, block_len); result += block_len; - len -= block_len; src += block_len; + len -= block_len; offset = 0; ++block_start; } while (len && block_start < block_limit); } - mPos += result; + // Something left, see if it will fit in the free + // space of the last block. + if (len && ! mBlocks.empty()) + { + Block & last(*mBlocks.back()); + if (last.mUsed < last.mAlloced) + { + // Some will fit... + const size_t copy_len((std::min)(len, (last.mAlloced - last.mUsed))); + + memcpy(&last.mData[last.mUsed], src, copy_len); + last.mUsed += copy_len; + result += copy_len; + llassert_always(last.mUsed <= last.mAlloced); + mLen += copy_len; + src += copy_len; + len -= copy_len; + } + } + if (len) { // Some or all of the remaining write data will @@ -218,12 +266,12 @@ int BufferArray::findBlock(size_t pos, size_t * ret_offset) const int block_limit(mBlocks.size()); for (int i(0); i < block_limit; ++i) { - if (pos < mBlocks[i]->mLen) + if (pos < mBlocks[i]->mUsed) { *ret_offset = pos; return i; } - pos -= mBlocks[i]->mLen; + pos -= mBlocks[i]->mUsed; } // Shouldn't get here but... @@ -237,7 +285,8 @@ int BufferArray::findBlock(size_t pos, size_t * ret_offset) BufferArray::Block::Block(size_t len) - : mLen(len) + : mUsed(0), + mAlloced(len) { memset(mData, 0, len); } @@ -245,7 +294,8 @@ BufferArray::Block::Block(size_t len) BufferArray::Block::~Block() { - mLen = 0; + mUsed = 0; + mAlloced = 0; } diff --git a/indra/llcorehttp/bufferarray.h b/indra/llcorehttp/bufferarray.h index b26ad1b297..9ccd85d4f8 100644 --- a/indra/llcorehttp/bufferarray.h +++ b/indra/llcorehttp/bufferarray.h @@ -73,6 +73,9 @@ private: void operator=(const BufferArray &); // Not defined public: + // Internal magic number, may be used by unit tests. + static const size_t BLOCK_ALLOC_SIZE = 1504; + /// Appends the indicated data to the BufferArray /// modifying current position and total size. New /// position is one beyond the final byte of the buffer. @@ -96,24 +99,16 @@ public: return mLen; } - /// Set the current position for subsequent read and - /// write operations. 'pos' values before the beginning - /// or greater than the size of the buffer are coerced - /// to a value within the buffer. - /// - /// @return Actual current position after seek. - size_t seek(size_t pos); - - /// Copies data from the current position in the instance + /// Copies data from the given position in the instance /// to the caller's buffer. Will return a short count of /// bytes copied if the 'len' extends beyond the data. - size_t read(char * dst, size_t len); + size_t read(size_t pos, char * dst, size_t len); /// Copies data from the caller's buffer to the instance /// at the current position. May overwrite existing data, /// append data when current position is equal to the /// size of the instance or do a mix of both. - size_t write(const char * src, size_t len); + size_t write(size_t pos, const char * src, size_t len); protected: int findBlock(size_t pos, size_t * ret_offset); @@ -123,46 +118,10 @@ protected: typedef std::vector container_t; container_t mBlocks; - size_t mPos; size_t mLen; }; // end class BufferArray -#if 0 - -// Conceptual for now. Another possibility is going with -// something like Boost::asio's buffers interface. They're -// trying to achieve the same thing above and below.... - -class BufferStream : public std::streambuf -{ -public: - BufferStream(BufferArray * buffer); - virtual ~BufferStream(); - -private: - BufferStream(const BufferStream &); // Not defined - void operator=(const BufferStream &); // Not defined - -public: - // Types - typedef std::streambuf::pos_type pos_type; - typedef std::streambuf::off_type off_type; - - virtual int underflow(); - - virtual int overflow(int c); - - virtual int sync(); - - virtual pos_type seekoff(off_type off, std::ios::seekdir way, std::ios::openmode which); - -protected: - BufferArray * mBufferArray; -}; // end class BufferStream - -#endif // 0 - } // end namespace LLCore #endif // _LLCORE_BUFFER_ARRAY_H_ diff --git a/indra/llcorehttp/tests/test_bufferarray.hpp b/indra/llcorehttp/tests/test_bufferarray.hpp index ecbb5ef250..2ad9391d1c 100644 --- a/indra/llcorehttp/tests/test_bufferarray.hpp +++ b/indra/llcorehttp/tests/test_bufferarray.hpp @@ -67,7 +67,7 @@ void BufferArrayTestObjectType::test<1>() // Try to read char buffer[20]; - size_t read_len(ba->read(buffer, sizeof(buffer))); + size_t read_len(ba->read(0, buffer, sizeof(buffer))); ensure("Read returns empty", 0 == read_len); // release the implicit reference, causing the object to be released @@ -92,16 +92,13 @@ void BufferArrayTestObjectType::test<2>() char str1[] = "abcdefghij"; char buffer[256]; - size_t len = ba->write(str1, strlen(str1)); + size_t len = ba->write(0, str1, strlen(str1)); ensure("Wrote length correct", strlen(str1) == len); ensure("Recorded size correct", strlen(str1) == ba->size()); // read some data back - len = ba->seek(2); - ensure("Seek worked", 2 == len); - memset(buffer, 'X', sizeof(buffer)); - len = ba->read(buffer, 2); + len = ba->read(2, buffer, 2); ensure("Read length correct", 2 == len); ensure("Read content correct", 'c' == buffer[0] && 'd' == buffer[1]); ensure("Read didn't overwrite", 'X' == buffer[2]); @@ -130,32 +127,26 @@ void BufferArrayTestObjectType::test<3>() size_t str1_len(strlen(str1)); char buffer[256]; - size_t len = ba->write(str1, str1_len); + size_t len = ba->write(0, str1, str1_len); ensure("Wrote length correct", str1_len == len); ensure("Recorded size correct", str1_len == ba->size()); // again... - len = ba->write(str1, strlen(str1)); + len = ba->write(str1_len, str1, strlen(str1)); ensure("Wrote length correct", str1_len == len); ensure("Recorded size correct", (2 * str1_len) == ba->size()); // read some data back - len = ba->seek(8); - ensure("Seek worked", 8 == len); - memset(buffer, 'X', sizeof(buffer)); - len = ba->read(buffer, 4); + len = ba->read(8, buffer, 4); ensure("Read length correct", 4 == len); ensure("Read content correct", 'i' == buffer[0] && 'j' == buffer[1]); ensure("Read content correct", 'a' == buffer[2] && 'b' == buffer[3]); ensure("Read didn't overwrite", 'X' == buffer[4]); // Read whole thing - len = ba->seek(0); - ensure("Seek worked (2)", 0 == len); - memset(buffer, 'X', sizeof(buffer)); - len = ba->read(buffer, sizeof(buffer)); + len = ba->read(0, buffer, sizeof(buffer)); ensure("Read length correct", (2 * str1_len) == len); ensure("Read content correct (3)", 0 == strncmp(buffer, str1, str1_len)); ensure("Read content correct (4)", 0 == strncmp(&buffer[str1_len], str1, str1_len)); @@ -185,33 +176,29 @@ void BufferArrayTestObjectType::test<4>() char str2[] = "ABCDEFGHIJ"; char buffer[256]; - size_t len = ba->write(str1, str1_len); + size_t len = ba->write(0, str1, str1_len); ensure("Wrote length correct", str1_len == len); ensure("Recorded size correct", str1_len == ba->size()); // again... - len = ba->write(str1, strlen(str1)); + len = ba->write(str1_len, str1, strlen(str1)); ensure("Wrote length correct", str1_len == len); ensure("Recorded size correct", (2 * str1_len) == ba->size()); // reposition and overwrite - len = ba->seek(8); - ensure("Seek worked", 8 == len); - len = ba->write(str2, 4); + len = ba->write(8, str2, 4); ensure("Overwrite length correct", 4 == len); - // Leave position and read verifying content + // Leave position and read verifying content (stale really from seek() days) memset(buffer, 'X', sizeof(buffer)); - len = ba->read(buffer, 4); + len = ba->read(12, buffer, 4); ensure("Read length correct", 4 == len); ensure("Read content correct", 'c' == buffer[0] && 'd' == buffer[1]); ensure("Read content correct.2", 'e' == buffer[2] && 'f' == buffer[3]); ensure("Read didn't overwrite", 'X' == buffer[4]); // reposition and check - len = ba->seek(6); - memset(buffer, 'X', sizeof(buffer)); - len = ba->read(buffer, 8); + len = ba->read(6, buffer, 8); ensure("Read length correct.2", 8 == len); ensure("Read content correct.3", 'g' == buffer[0] && 'h' == buffer[1]); ensure("Read content correct.4", 'A' == buffer[2] && 'B' == buffer[3]); @@ -242,21 +229,18 @@ void BufferArrayTestObjectType::test<5>() size_t str1_len(strlen(str1)); char buffer[256]; - size_t len = ba->write(str1, str1_len); + size_t len = ba->write(0, str1, str1_len); ensure("Wrote length correct", str1_len == len); ensure("Recorded size correct", str1_len == ba->size()); // again... - len = ba->write(str1, strlen(str1)); + len = ba->write(str1_len, str1, str1_len); ensure("Wrote length correct", str1_len == len); ensure("Recorded size correct", (2 * str1_len) == ba->size()); // read some data back - len = ba->seek(8); - ensure("Seek worked", 8 == len); - memset(buffer, 'X', sizeof(buffer)); - len = ba->read(buffer, 4); + len = ba->read(8, buffer, 4); ensure("Read length correct", 4 == len); ensure("Read content correct", 'i' == buffer[0] && 'j' == buffer[1]); ensure("Read content correct.2", 'a' == buffer[2] && 'b' == buffer[3]); @@ -264,7 +248,7 @@ void BufferArrayTestObjectType::test<5>() // Read some more without repositioning memset(buffer, 'X', sizeof(buffer)); - len = ba->read(buffer, sizeof(buffer)); + len = ba->read(12, buffer, sizeof(buffer)); ensure("Read length correct", (str1_len - 2) == len); ensure("Read content correct.3", 0 == strncmp(buffer, str1+2, str1_len-2)); ensure("Read didn't overwrite.2", 'X' == buffer[str1_len-1]); @@ -294,31 +278,27 @@ void BufferArrayTestObjectType::test<6>() size_t str2_len(strlen(str2)); char buffer[256]; - size_t len = ba->write(str1, str1_len); + size_t len = ba->write(0, str1, str1_len); ensure("Wrote length correct", str1_len == len); ensure("Recorded size correct", str1_len == ba->size()); // again... - len = ba->write(str1, strlen(str1)); + len = ba->write(str1_len, str1, strlen(str1)); ensure("Wrote length correct", str1_len == len); ensure("Recorded size correct", (2 * str1_len) == ba->size()); // reposition and overwrite - len = ba->seek(8); - ensure("Seek worked", 8 == len); - len = ba->write(str2, str2_len); + len = ba->write(8, str2, str2_len); ensure("Overwrite length correct", str2_len == len); // Leave position and read verifying content memset(buffer, 'X', sizeof(buffer)); - len = ba->read(buffer, 0); + len = ba->read(8 + str2_len, buffer, 0); ensure("Read length correct", 0 == len); ensure("Read didn't overwrite", 'X' == buffer[0]); // reposition and check - len = ba->seek(0); - memset(buffer, 'X', sizeof(buffer)); - len = ba->read(buffer, sizeof(buffer)); + len = ba->read(0, buffer, sizeof(buffer)); ensure("Read length correct.2", (str1_len + str2_len - 2) == len); ensure("Read content correct", 0 == strncmp(buffer, str1, str1_len-2)); ensure("Read content correct.2", 0 == strncmp(buffer+str1_len-2, str2, str2_len)); @@ -350,18 +330,17 @@ void BufferArrayTestObjectType::test<7>() char buffer[256]; // 2x str1 - size_t len = ba->write(str1, str1_len); - len = ba->write(str1, strlen(str1)); + size_t len = ba->write(0, str1, str1_len); + len = ba->write(str1_len, str1, str1_len); // reposition and overwrite - len = ba->seek(6); - len = ba->write(str2, 2); + len = ba->write(6, str2, 2); ensure("Overwrite length correct", 2 == len); - len = ba->write(str2, 2); + len = ba->write(8, str2, 2); ensure("Overwrite length correct.2", 2 == len); - len = ba->write(str2, 2); + len = ba->write(10, str2, 2); ensure("Overwrite length correct.3", 2 == len); // append some data @@ -373,13 +352,12 @@ void BufferArrayTestObjectType::test<7>() memcpy(out_buf, str1, str1_len); // And some final writes - len = ba->write(str2, 2); + len = ba->write(3 * str1_len + str2_len, str2, 2); ensure("Write length correct.2", 2 == len); // Check contents memset(buffer, 'X', sizeof(buffer)); - ba->seek(0); - len = ba->read(buffer, sizeof(buffer)); + len = ba->read(0, buffer, sizeof(buffer)); ensure("Final buffer length correct", (3 * str1_len + str2_len + 2) == len); ensure("Read content correct", 0 == strncmp(buffer, str1, 6)); ensure("Read content correct.2", 0 == strncmp(buffer + 6, str2, 2)); @@ -417,8 +395,8 @@ void BufferArrayTestObjectType::test<8>() char buffer[256]; // 2x str1 - size_t len = ba->write(str1, str1_len); - len = ba->write(str1, strlen(str1)); + size_t len = ba->write(0, str1, str1_len); + len = ba->write(str1_len, str1, str1_len); // zero-length allocate (we allow this with a valid pointer returned) char * out_buf(ba->appendBufferAlloc(0)); @@ -430,12 +408,11 @@ void BufferArrayTestObjectType::test<8>() ensure("Two zero-length appendBufferAlloc buffers distinct", out_buf != out_buf2); // And some final writes - len = ba->write(str2, str2_len); + len = ba->write(2 * str1_len, str2, str2_len); // Check contents memset(buffer, 'X', sizeof(buffer)); - ba->seek(0); - len = ba->read(buffer, sizeof(buffer)); + len = ba->read(0, buffer, sizeof(buffer)); ensure("Final buffer length correct", (2 * str1_len + str2_len) == len); ensure("Read content correct.1", 0 == strncmp(buffer, str1, str1_len)); ensure("Read content correct.2", 0 == strncmp(buffer + str1_len, str1, str1_len)); diff --git a/indra/newview/lltexturefetch.cpp b/indra/newview/lltexturefetch.cpp index daad463e0d..e27f0aec1e 100644 --- a/indra/newview/lltexturefetch.cpp +++ b/indra/newview/lltexturefetch.cpp @@ -1255,8 +1255,7 @@ bool LLTextureFetchWorker::doWork(S32 param) { memcpy(buffer, mFormattedImage->getData(), cur_size); } - mHttpBufferArray->seek(0); - mHttpBufferArray->read((char *) buffer + cur_size, append_size); + mHttpBufferArray->read(0, (char *) buffer + cur_size, append_size); // NOTE: setData releases current data and owns new data (buffer) mFormattedImage->setData(buffer, total_size); -- cgit v1.3 From 9a11a2946f4dec334ce1ac449b355ba16eaae23a Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Tue, 5 Jun 2012 12:06:42 -0400 Subject: Faster spin in worker thread when doing I/O and a priority bump needed when fixing priorities. --- indra/llcorehttp/_httpservice.cpp | 6 +++++- indra/newview/lltexturefetch.cpp | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) (limited to 'indra/newview') diff --git a/indra/llcorehttp/_httpservice.cpp b/indra/llcorehttp/_httpservice.cpp index 337493ca12..b038bdb720 100644 --- a/indra/llcorehttp/_httpservice.cpp +++ b/indra/llcorehttp/_httpservice.cpp @@ -38,6 +38,10 @@ #include "lltimer.h" +// Tuning parameters +static const int LOOP_SLEEP_NORMAL_MS = 10; // Normal per-loop sleep in milliseconds + + namespace LLCore { @@ -198,7 +202,7 @@ void HttpService::threadRun(LLCoreInt::HttpThread * thread) // Determine whether to spin, sleep briefly or sleep for next request if (REQUEST_SLEEP != loop) { - ms_sleep(50); + ms_sleep(LOOP_SLEEP_NORMAL_MS); } } diff --git a/indra/newview/lltexturefetch.cpp b/indra/newview/lltexturefetch.cpp index 08f3ca1583..34fb21798f 100644 --- a/indra/newview/lltexturefetch.cpp +++ b/indra/newview/lltexturefetch.cpp @@ -736,7 +736,7 @@ void LLTextureFetchWorker::setDesiredDiscard(S32 discard, S32 size) if ((prioritize && mState == INIT) || mState == DONE) { mState = INIT; - U32 work_priority = mWorkPriority | LLWorkerThread::PRIORITY_LOW; + U32 work_priority = mWorkPriority | LLWorkerThread::PRIORITY_HIGH; setPriority(work_priority); } } -- cgit v1.3 From 05af16a23abe37210e0b880aa27387d8994419dd Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Wed, 6 Jun 2012 13:52:38 -0400 Subject: Policy + caching fixes + https support + POST working Implemented first global policy definitions to support SSL CA certificate configuration to support https: operations. Fixed HTTP 206 status handling to match what is currently being done by grid services and to lay a foundation for fixes that will be a response to ER-1824. More libcurl CURLOPT options set on easy handles to do peer verification in the traditional way. HTTP POST working and now reporting asset metrics back to grid for the viewer's asset system. This uses LLSD so that is also showing as compatible with the new library. --- indra/llcorehttp/CMakeLists.txt | 2 ++ indra/llcorehttp/_httplibcurl.cpp | 1 + indra/llcorehttp/_httpoprequest.cpp | 45 ++++++++++++++++++++++++++++++++----- indra/llcorehttp/_httppolicy.cpp | 4 ++-- indra/llcorehttp/_httppolicy.h | 10 +++++++++ indra/llcorehttp/_httpservice.h | 8 +++---- indra/llcorehttp/httpcommon.cpp | 5 ++++- indra/llcorehttp/httpcommon.h | 11 ++++++++- indra/llcorehttp/httprequest.cpp | 13 +++++++++-- indra/llcorehttp/httprequest.h | 16 +++++++------ indra/newview/llappviewer.cpp | 12 +++++++++- indra/newview/lltexturefetch.cpp | 37 +++++++++++++++++------------- 12 files changed, 126 insertions(+), 38 deletions(-) (limited to 'indra/newview') diff --git a/indra/llcorehttp/CMakeLists.txt b/indra/llcorehttp/CMakeLists.txt index 9a073eb850..3fda524ddf 100644 --- a/indra/llcorehttp/CMakeLists.txt +++ b/indra/llcorehttp/CMakeLists.txt @@ -33,6 +33,7 @@ set(llcorehttp_SOURCE_FILES _httpoprequest.cpp _httpopsetpriority.cpp _httppolicy.cpp + _httppolicyglobal.cpp _httpreplyqueue.cpp _httprequestqueue.cpp _httpservice.cpp @@ -55,6 +56,7 @@ set(llcorehttp_HEADER_FILES _httpoprequest.h _httpopsetpriority.h _httppolicy.h + _httppolicyglobal.h _httpreadyqueue.h _httpreplyqueue.h _httprequestqueue.h diff --git a/indra/llcorehttp/_httplibcurl.cpp b/indra/llcorehttp/_httplibcurl.cpp index 704f9baac9..5272c391e8 100644 --- a/indra/llcorehttp/_httplibcurl.cpp +++ b/indra/llcorehttp/_httplibcurl.cpp @@ -27,6 +27,7 @@ #include "_httplibcurl.h" #include "httpheaders.h" +#include "bufferarray.h" #include "_httpoprequest.h" diff --git a/indra/llcorehttp/_httpoprequest.cpp b/indra/llcorehttp/_httpoprequest.cpp index f52ff5a44c..4bdc4a5257 100644 --- a/indra/llcorehttp/_httpoprequest.cpp +++ b/indra/llcorehttp/_httpoprequest.cpp @@ -40,8 +40,10 @@ #include "_httpreplyqueue.h" #include "_httpservice.h" #include "_httppolicy.h" +#include "_httppolicyglobal.h" #include "_httplibcurl.h" +#include "llhttpstatuscodes.h" namespace { @@ -153,14 +155,14 @@ HttpOpRequest::~HttpOpRequest() void HttpOpRequest::stageFromRequest(HttpService * service) { addRef(); - service->getPolicy()->addOp(this); // transfers refcount + service->getPolicy().addOp(this); // transfers refcount } void HttpOpRequest::stageFromReady(HttpService * service) { addRef(); - service->getTransport()->addOp(this); // transfers refcount + service->getTransport().addOp(this); // transfers refcount } @@ -195,6 +197,8 @@ void HttpOpRequest::stageFromActive(HttpService * service) void HttpOpRequest::visitNotifier(HttpRequest * request) { + static const HttpStatus partial_content(HTTP_PARTIAL_CONTENT, HE_SUCCESS); + if (mLibraryHandler) { HttpResponse * response = new HttpResponse(); @@ -208,9 +212,15 @@ void HttpOpRequest::visitNotifier(HttpRequest * request) offset = mReplyOffset; length = mReplyLength; } - else if (mReplyBody) + else if (mReplyBody && partial_content == mStatus) { - // Provide implicit offset/length from request/response + // Legacy grid services did not provide a 'Content-Range' + // header in responses to full- or partly-satisfyiable + // 'Range' requests. For these, we have to hope that + // the data starts where requested and the length is simply + // whatever we received. A bit of sanity could be provided + // by overlapping ranged requests and verifying that the + // overlap matches. offset = mReqOffset; length = mReplyBody->size(); } @@ -306,6 +316,9 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service) // *FIXME: better error handling later HttpStatus status; + // Get policy options + HttpPolicyGlobal & policy(service->getPolicy().getGlobalOptions()); + mCurlHandle = curl_easy_init(); // curl_easy_setopt(mCurlHandle, CURLOPT_VERBOSE, 1); curl_easy_setopt(mCurlHandle, CURLOPT_TIMEOUT, 30); @@ -322,21 +335,40 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service) curl_easy_setopt(mCurlHandle, CURLOPT_DNS_CACHE_TIMEOUT, 0); curl_easy_setopt(mCurlHandle, CURLOPT_AUTOREFERER, 1); curl_easy_setopt(mCurlHandle, CURLOPT_FOLLOWLOCATION, 1); - curl_easy_setopt(mCurlHandle, CURLOPT_MAXREDIRS, 10); + curl_easy_setopt(mCurlHandle, CURLOPT_MAXREDIRS, 10); // *FIXME: parameterize this later curl_easy_setopt(mCurlHandle, CURLOPT_WRITEFUNCTION, writeCallback); curl_easy_setopt(mCurlHandle, CURLOPT_WRITEDATA, mCurlHandle); curl_easy_setopt(mCurlHandle, CURLOPT_READFUNCTION, readCallback); curl_easy_setopt(mCurlHandle, CURLOPT_READDATA, mCurlHandle); + curl_easy_setopt(mCurlHandle, CURLOPT_SSL_VERIFYPEER, 1); + curl_easy_setopt(mCurlHandle, CURLOPT_SSL_VERIFYHOST, 0); + std::string opt_value; + if (policy.get(HttpRequest::GP_CA_PATH, opt_value)) + { + curl_easy_setopt(mCurlHandle, CURLOPT_CAPATH, opt_value.c_str()); + } + if (policy.get(HttpRequest::GP_CA_FILE, opt_value)) + { + curl_easy_setopt(mCurlHandle, CURLOPT_CAINFO, opt_value.c_str()); + } + if (policy.get(HttpRequest::GP_HTTP_PROXY, opt_value)) + { + curl_easy_setopt(mCurlHandle, CURLOPT_PROXY, opt_value.c_str()); + } + switch (mReqMethod) { case HOR_GET: curl_easy_setopt(mCurlHandle, CURLOPT_HTTPGET, 1); + mCurlHeaders = curl_slist_append(mCurlHeaders, "Connection: keep-alive"); + mCurlHeaders = curl_slist_append(mCurlHeaders, "Keep-alive: 300"); break; case HOR_POST: { curl_easy_setopt(mCurlHandle, CURLOPT_POST, 1); + curl_easy_setopt(mCurlHandle, CURLOPT_ENCODING, ""); long data_size(0); if (mReqBody) { @@ -358,8 +390,11 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service) data_size = mReqBody->size(); } curl_easy_setopt(mCurlHandle, CURLOPT_INFILESIZE, data_size); + curl_easy_setopt(mCurlHandle, CURLOPT_POSTFIELDS, (void *) NULL); mCurlHeaders = curl_slist_append(mCurlHeaders, "Transfer-Encoding: chunked"); mCurlHeaders = curl_slist_append(mCurlHeaders, "Expect:"); + mCurlHeaders = curl_slist_append(mCurlHeaders, "Connection: keep-alive"); + mCurlHeaders = curl_slist_append(mCurlHeaders, "Keep-alive: 300"); } break; diff --git a/indra/llcorehttp/_httppolicy.cpp b/indra/llcorehttp/_httppolicy.cpp index 1d28f23d56..51f5e487dc 100644 --- a/indra/llcorehttp/_httppolicy.cpp +++ b/indra/llcorehttp/_httppolicy.cpp @@ -76,12 +76,12 @@ void HttpPolicy::addOp(HttpOpRequest * op) HttpService::ELoopSpeed HttpPolicy::processReadyQueue() { HttpService::ELoopSpeed result(HttpService::REQUEST_SLEEP); - HttpLibcurl * pTransport(mService->getTransport()); + HttpLibcurl & transport(mService->getTransport()); for (int policy_class(0); policy_class < HttpRequest::POLICY_CLASS_LIMIT; ++policy_class) { HttpReadyQueue & readyq(mReadyQueue[policy_class]); - int active(pTransport->getActiveCountInClass(policy_class)); + int active(transport.getActiveCountInClass(policy_class)); int needed(8 - active); if (needed > 0 && mReadyInClass[policy_class] > 0) diff --git a/indra/llcorehttp/_httppolicy.h b/indra/llcorehttp/_httppolicy.h index 2bc03c531f..425079ec63 100644 --- a/indra/llcorehttp/_httppolicy.h +++ b/indra/llcorehttp/_httppolicy.h @@ -31,6 +31,7 @@ #include "httprequest.h" #include "_httpservice.h" #include "_httpreadyqueue.h" +#include "_httppolicyglobal.h" namespace LLCore @@ -68,11 +69,20 @@ public: // Shadows HttpService's method bool changePriority(HttpHandle handle, HttpRequest::priority_t priority); + + // Get pointer to global policy options. Caller is expected + // to do context checks like no setting once running. + HttpPolicyGlobal & getGlobalOptions() + { + return mGlobalOptions; + } + protected: int mReadyInClass[HttpRequest::POLICY_CLASS_LIMIT]; HttpReadyQueue mReadyQueue[HttpRequest::POLICY_CLASS_LIMIT]; HttpService * mService; // Naked pointer, not refcounted, not owner + HttpPolicyGlobal mGlobalOptions; }; // end class HttpPolicy diff --git a/indra/llcorehttp/_httpservice.h b/indra/llcorehttp/_httpservice.h index 095316c8a7..748354a8e4 100644 --- a/indra/llcorehttp/_httpservice.h +++ b/indra/llcorehttp/_httpservice.h @@ -148,14 +148,14 @@ public: /// Threading: callable by worker thread. bool changePriority(HttpHandle handle, HttpRequest::priority_t priority); - HttpPolicy * getPolicy() + HttpPolicy & getPolicy() { - return mPolicy; + return *mPolicy; } - HttpLibcurl * getTransport() + HttpLibcurl & getTransport() { - return mTransport; + return *mTransport; } protected: diff --git a/indra/llcorehttp/httpcommon.cpp b/indra/llcorehttp/httpcommon.cpp index a01182cf23..9f17b5c842 100644 --- a/indra/llcorehttp/httpcommon.cpp +++ b/indra/llcorehttp/httpcommon.cpp @@ -66,7 +66,10 @@ std::string HttpStatus::toString() const "Services shutting down", "Operation canceled", "Invalid Content-Range header encountered", - "Request handle not found" + "Request handle not found", + "Invalid datatype for argument or option", + "Option has not been explicitly set", + "Option is not dynamic and must be set early" }; static const int llcore_errors_count(sizeof(llcore_errors) / sizeof(llcore_errors[0])); diff --git a/indra/llcorehttp/httpcommon.h b/indra/llcorehttp/httpcommon.h index c01a5f85d3..fd2661b700 100644 --- a/indra/llcorehttp/httpcommon.h +++ b/indra/llcorehttp/httpcommon.h @@ -137,7 +137,16 @@ enum HttpError HE_INV_CONTENT_RANGE_HDR = 4, // Request handle not found - HE_HANDLE_NOT_FOUND = 5 + HE_HANDLE_NOT_FOUND = 5, + + // Invalid datatype for option/setting + HE_INVALID_ARG = 6, + + // Option hasn't been explicitly set + HE_OPT_NOT_SET = 7, + + // Option not dynamic, must be set during init phase + HE_OPT_NOT_DYNAMIC = 8 }; // end enum HttpError diff --git a/indra/llcorehttp/httprequest.cpp b/indra/llcorehttp/httprequest.cpp index 0e512d97ed..baa0fe1a84 100644 --- a/indra/llcorehttp/httprequest.cpp +++ b/indra/llcorehttp/httprequest.cpp @@ -29,6 +29,7 @@ #include "_httprequestqueue.h" #include "_httpreplyqueue.h" #include "_httpservice.h" +#include "_httppolicy.h" #include "_httpoperation.h" #include "_httpoprequest.h" #include "_httpopsetpriority.h" @@ -127,9 +128,17 @@ HttpRequest::~HttpRequest() HttpStatus HttpRequest::setPolicyGlobalOption(EGlobalPolicy opt, long value) { - HttpStatus status; + // *FIXME: Fail if thread is running. - return status; + return HttpService::instanceOf()->getPolicy().getGlobalOptions().set(opt, value); +} + + +HttpStatus HttpRequest::setPolicyGlobalOption(EGlobalPolicy opt, const std::string & value) +{ + // *FIXME: Fail if thread is running. + + return HttpService::instanceOf()->getPolicy().getGlobalOptions().set(opt, value); } diff --git a/indra/llcorehttp/httprequest.h b/indra/llcorehttp/httprequest.h index 57d2da245b..3592d5c6a3 100644 --- a/indra/llcorehttp/httprequest.h +++ b/indra/llcorehttp/httprequest.h @@ -111,7 +111,10 @@ public: /// Maximum number of connections the library will use to /// perform operations. This is somewhat soft as the underlying /// transport will cache some connections (up to 5). - GLOBAL_CONNECTION_LIMIT + GP_CONNECTION_LIMIT, ///< Takes long giving number of connections + GP_CA_PATH, ///< System path/directory where SSL certs are stored. + GP_CA_FILE, ///< System path/file containing certs. + GP_HTTP_PROXY ///< String giving host/port to use for HTTP proxy }; /// Set a parameter on a global policy option. Calls @@ -122,6 +125,7 @@ public: /// @param value Desired value of option. /// @return Standard status code. HttpStatus setPolicyGlobalOption(EGlobalPolicy opt, long value); + HttpStatus setPolicyGlobalOption(EGlobalPolicy opt, const std::string & value); /// Create a new policy class into which requests can be made. /// @@ -134,15 +138,15 @@ public: enum EClassPolicy { /// Limits the number of connections used for the class. - CLASS_CONNECTION_LIMIT, + CP_CONNECTION_LIMIT, /// Limits the number of connections used for a single /// literal address/port pair within the class. - PER_HOST_CONNECTION_LIMIT, + CP_PER_HOST_CONNECTION_LIMIT, /// Suitable requests are allowed to pipeline on their /// connections when they ask for it. - ENABLE_PIPELINING + CP_ENABLE_PIPELINING }; /// Set a parameter on a class-based policy option. Calls @@ -153,9 +157,7 @@ public: /// @param opt Enum of option to be set. /// @param value Desired value of option. /// @return Standard status code. - HttpStatus setPolicyClassOption(policy_t policy_id, - EClassPolicy opt, - long value); + HttpStatus setPolicyClassOption(policy_t policy_id, EClassPolicy opt, long value); /// @} diff --git a/indra/newview/llappviewer.cpp b/indra/newview/llappviewer.cpp index 8e6deb9cce..7a44415fba 100644 --- a/indra/newview/llappviewer.cpp +++ b/indra/newview/llappviewer.cpp @@ -5356,6 +5356,17 @@ void CoreHttp::init() << LL_ENDL; } + mRequest = new LLCore::HttpRequest; + + status = mRequest->setPolicyGlobalOption(LLCore::HttpRequest::GP_CA_FILE, + gDirUtilp->getCAFile()); + if (! status) + { + LL_ERRS("Init") << "Failed to set CA File for HTTP services. Reason: " + << status.toString() + << LL_ENDL; + } + status = LLCore::HttpRequest::startThread(); if (! status) { @@ -5364,7 +5375,6 @@ void CoreHttp::init() << LL_ENDL; } - mRequest = new LLCore::HttpRequest; } diff --git a/indra/newview/lltexturefetch.cpp b/indra/newview/lltexturefetch.cpp index 34fb21798f..f9294b4cd1 100644 --- a/indra/newview/lltexturefetch.cpp +++ b/indra/newview/lltexturefetch.cpp @@ -816,14 +816,6 @@ bool LLTextureFetchWorker::doWork(S32 param) mFetchTimer.reset(); } - static LLUUID last_id; - if (mID != last_id) - { - // LL_WARNS("Texture") << "DOWORK SWITCH: " << last_id << " to: " << mID - // << LL_ENDL; - last_id = mID; - } - if (mState == INIT) { mRawImage = NULL ; @@ -1109,10 +1101,6 @@ bool LLTextureFetchWorker::doWork(S32 param) << " Bytes: " << mRequestedSize << " Bandwidth(kbps): " << mFetcher->getTextureBandwidth() << "/" << mFetcher->mMaxBandwidth << LL_ENDL; -// LL_WARNS("Texture") << "HTTP GET: " << mID << " Offset: " << mRequestedOffset -// << " Bytes: " << mRequestedSize -// << " Bandwidth(kbps): " << mFetcher->getTextureBandwidth() << "/" << mFetcher->mMaxBandwidth -// << LL_ENDL; // Will call callbackHttpGet when curl request completes // *FIXME: enable redirection follow @@ -1241,7 +1229,7 @@ bool LLTextureFetchWorker::doWork(S32 param) } } - if (mHaveAllData && mRequestedDiscard == 0) //the image file is fully loaded. + if (mHaveAllData /* && mRequestedDiscard == 0*/) //the image file is fully loaded. { mFileSize = total_size; } @@ -1692,13 +1680,32 @@ S32 LLTextureFetchWorker::callbackHttpGet(LLCore::HttpResponse * response, body->addRef(); mHttpBufferArray = body; - if (data_size < mRequestedSize && mRequestedDiscard == 0) + if (! partial) + { + // Response indicates this is the entire asset regardless + // of our asking for a byte range. Mark it so and drop + // any partial data we might have so that the current + // response body becomes the entire dataset. + if (data_size <= mRequestedOffset) + { + LL_WARNS("Texture") << "Fetched entire texture " << mID + << " when it was expected to be marked complete. mImageSize: " + << mFileSize << " datasize: " << mFormattedImage->getDataSize() + << LL_ENDL; + } + mHaveAllData = TRUE; + llassert_always(mDecodeHandle == 0); + mFormattedImage = NULL; // discard any previous data we had + } + else if (data_size < mRequestedSize && mRequestedDiscard == 0) { + // *FIXME: I think we can treat this as complete regardless + // of requested discard level. Revisit this... mHaveAllData = TRUE; } else if (data_size > mRequestedSize) { - // *TODO: This shouldn't be happening any more + // *TODO: This shouldn't be happening any more (REALLY don't expect this anymore) llwarns << "data_size = " << data_size << " > requested: " << mRequestedSize << llendl; mHaveAllData = TRUE; llassert_always(mDecodeHandle == 0); -- cgit v1.3 From 196e1d46bdc8b35b2e79d8a9d6a693ebeeeb19d3 Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Thu, 7 Jun 2012 17:46:22 -0400 Subject: Lock/Mutex fixes, documentation, new resource wait state, dtor cleanups Went through all the code and tried to document lock and thread usage in the module. There's a huge comment block introducing all of this at the beginning and I believe it's correct (though not quite complete). Keep it updated, people. Added a new state, WAIT_HTTP_RESOURCE, that's sort of a side-state of SEND_HTTP_REQ. If we hit a high-water mark for HTTP requests, the extra are shunted to the new state once. Once levels fall to a low-water mark, we run through a wait list of UUIDs, sort the valid ones by priority and release them for service. This keeps the HTTP layer busy while leaving the active queue shallow enough that requests can still be re-prioritzed cheaply. Priority model changed. The new state uses the PRIORITY_LOW mask, the old users of _LOW are now at PRIORITY_NORMAL and sleepers woken up after an external event are kicked off at PRIORITY_HIGH. This combination along with the new state should avoid priority inversion and keep things running without resorting to an infinite pipeline. New state displays as "HTW" with green text in the texture console. Request cancelation and worker run-down should now be more correct but this edge case may need more attention. --- indra/newview/lltexturefetch.cpp | 942 ++++++++++++++++++++++++++------------- indra/newview/lltexturefetch.h | 153 ++++++- indra/newview/lltextureview.cpp | 3 +- 3 files changed, 777 insertions(+), 321 deletions(-) (limited to 'indra/newview') diff --git a/indra/newview/lltexturefetch.cpp b/indra/newview/lltexturefetch.cpp index f9294b4cd1..23232cb590 100644 --- a/indra/newview/lltexturefetch.cpp +++ b/indra/newview/lltexturefetch.cpp @@ -52,6 +52,7 @@ #include "llviewerassetstats.h" #include "llworld.h" #include "llsdserialize.h" +#include "llviewertexturelist.h" // debug #include "httprequest.h" #include "httphandler.h" @@ -60,6 +61,102 @@ ////////////////////////////////////////////////////////////////////////////// +// +// Introduction +// +// This is an attempt to document what's going on in here after-the-fact. +// It's a sincere attempt to be accurate but there will be mistakes. +// +// Purpose +// +// (What is this solving?) +// +// Threads +// +// Several threads are actively invoking code in this module. They +// include: +// +// 1. Tmain Main thread of execution +// 2. Ttf LLTextureFetch's worker thread provided by LLQueuedThread +// 3. Tcurl LLCurl's worker thread (should disappear over time) +// 4. Ttc LLTextureCache's worker thread +// 5. Tid Image decoder's worker thread +// 6. Thl HTTP library's worker thread +// +// Mutexes/Condition Variables +// +// 1. Mt Mutex defined for LLThread's condition variable (base class of +// LLTextureFetch) +// 2. Ct Condition variable for LLThread and used by lock/unlockData(). +// 3. Mwtd Special LLWorkerThread mutex used for request deletion +// operations (base class of LLTextureFetch) +// 4. Mfq LLTextureFetch's mutex covering request and command queue +// data. +// 5. Mfnq LLTextureFetch's mutex covering udp and http request +// queue data. +// 6. Mwc Mutex covering LLWorkerClass's members (base class of +// LLTextureFetchWorker). One per request. +// 7. Mw LLTextureFetchWorker's mutex. One per request. +// +// Lock Ordering Rules +// +// Not an exhaustive list but shows the order of lock acquisition +// needed to prevent deadlocks. 'A < B' means acquire 'A' before +// acquiring 'B'. +// +// 1. Mw < Mfnq +// +// Method and Member Definitions +// +// With the above, we'll try to document what threads can call what +// methods (using T* for any), what locks must be held on entry and +// are taken out during execution and what data is covered by which +// lock (if any). This latter category will be especially prone to +// error so be skeptical. +// +// A line like: "// Locks: M" indicates a method that must +// be invoked by a caller holding the 'M' lock. Similarly, +// "// Threads: T" means that a caller should be running in +// the indicated thread. +// +// For data members, a trailing comment like "// M" means that +// the data member is covered by the specified lock. Absence of a +// comment can mean the member is unlocked or that I didn't bother +// to do the archaeology. In the case of LLTextureFetchWorker, +// most data members added by the leaf class are actually covered +// by the Mw lock. +// +// In code, a trailing comment like "// [-+]M" indicates a +// lock acquision or release point. +// +// +// Worker Lifecycle +// +// (Can't unilaterally delete, cleanup is two-phase, etc.) +// +// Worker State Machine +// +// (ASCII art needed) +// +// Priority Scheme +// +// [PRIORITY_LOW, PRIORITY_NORMAL) - for WAIT_HTTP_RESOURCE state +// [PRIORITY_NORMAL, PRIORITY_HIGH) - waiting for external event +// [PRIORITY_HIGH, PRIORITY_URGENT) - rapidly transitioning through states, +// no waiting allowed +// +// + +////////////////////////////////////////////////////////////////////////////// + +// Tuning/Parameterization Constants + +static const S32 HTTP_REQUESTS_IN_QUEUE_HIGH_WATER = 40; +static const S32 HTTP_REQUESTS_IN_QUEUE_LOW_WATER = 20; + + +////////////////////////////////////////////////////////////////////////////// + class LLTextureFetchWorker : public LLWorkerClass, public LLCore::HttpHandler { @@ -69,11 +166,15 @@ private: class CacheReadResponder : public LLTextureCache::ReadResponder { public: + + // Threads: Ttf CacheReadResponder(LLTextureFetch* fetcher, const LLUUID& id, LLImageFormatted* image) : mFetcher(fetcher), mID(id) { setImage(image); } + + // Threads: Ttc virtual void completed(bool success) { LLTextureFetchWorker* worker = mFetcher->getWorker(mID); @@ -90,10 +191,14 @@ private: class CacheWriteResponder : public LLTextureCache::WriteResponder { public: + + // Threads: Ttf CacheWriteResponder(LLTextureFetch* fetcher, const LLUUID& id) : mFetcher(fetcher), mID(id) { } + + // Threads: Ttc virtual void completed(bool success) { LLTextureFetchWorker* worker = mFetcher->getWorker(mID); @@ -110,10 +215,14 @@ private: class DecodeResponder : public LLImageDecodeThread::Responder { public: + + // Threads: Ttf DecodeResponder(LLTextureFetch* fetcher, const LLUUID& id, LLTextureFetchWorker* worker) : mFetcher(fetcher), mID(id), mWorker(worker) { } + + // Threads: Tid virtual void completed(bool success, LLImageRaw* raw, LLImageRaw* aux) { LLTextureFetchWorker* worker = mFetcher->getWorker(mID); @@ -146,20 +255,34 @@ private: }; public: + + // Threads: Ttf /*virtual*/ bool doWork(S32 param); // Called from LLWorkerThread::processRequest() + + // Threads: Ttf /*virtual*/ void finishWork(S32 param, bool completed); // called from finishRequest() (WORK THREAD) - /*virtual*/ bool deleteOK(); // called from update() (WORK THREAD) + + // Threads: Tmain + /*virtual*/ bool deleteOK(); // called from update() ~LLTextureFetchWorker(); - // void relese() { --mActiveCount; } + // Threads: Ttf + // Locks: Mw S32 callbackHttpGet(LLCore::HttpResponse * response, bool partial, bool success); + + // Threads: Ttc void callbackCacheRead(bool success, LLImageFormatted* image, S32 imagesize, BOOL islocal); + + // Threads: Ttc void callbackCacheWrite(bool success); + + // Threads: Tid void callbackDecoded(bool success, LLImageRaw* raw, LLImageRaw* aux); + // Threads: T* void setGetStatus(LLCore::HttpStatus status, const std::string& reason) { LLMutexLock lock(&mWorkMutex); @@ -174,6 +297,7 @@ public: LLTextureFetch & getFetcher() { return *mFetcher; } // Inherited from LLCore::HttpHandler + // Threads: Ttf virtual void onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * response); protected: @@ -181,27 +305,54 @@ protected: F32 priority, S32 discard, S32 size); private: + + // Threads: Tmain /*virtual*/ void startWork(S32 param); // called from addWork() (MAIN THREAD) + + // Threads: Tmain /*virtual*/ void endWork(S32 param, bool aborted); // called from doWork() (MAIN THREAD) + // Locks: Mw void resetFormattedData(); + // Locks: Mw void setImagePriority(F32 priority); + + // Locks: Mw (ctor invokes without lock) void setDesiredDiscard(S32 discard, S32 size); + + // Threads: T* + // Locks: Mw bool insertPacket(S32 index, U8* data, S32 size); + + // Locks: Mw void clearPackets(); + + // Locks: Mw void setupPacketData(); + + // Locks: Mw (ctor invokes without lock) U32 calcWorkPriority(); + + // Locks: Mw void removeFromCache(); + + // Threads: Ttf + // Locks: Mw bool processSimulatorPackets(); + + // Threads: Ttf bool writeToCacheComplete(); - void lockWorkMutex() { mWorkMutex.lock(); } - void unlockWorkMutex() { mWorkMutex.unlock(); } - + // Threads: Ttf void recordTextureStart(bool is_http); + + // Threads: Ttf void recordTextureDone(bool is_http); + void lockWorkMutex() { mWorkMutex.lock(); } + void unlockWorkMutex() { mWorkMutex.unlock(); } + private: enum e_state // mState { @@ -215,8 +366,9 @@ private: CACHE_POST, LOAD_FROM_NETWORK, LOAD_FROM_SIMULATOR, - SEND_HTTP_REQ, - WAIT_HTTP_REQ, + SEND_HTTP_REQ, // Commit to sending as HTTP + WAIT_HTTP_RESOURCE, // Waiting for HTTP resources + WAIT_HTTP_REQ, // Request sent, wait for completion DECODE_IMAGE, DECODE_IMAGE_UPDATE, WRITE_TO_CACHE, @@ -297,17 +449,15 @@ private: LLViewerAssetStats::duration_t mMetricsStartTime; - LLCore::HttpHandle mHttpHandle; - LLCore::BufferArray * mHttpBufferArray; + LLCore::HttpHandle mHttpHandle; // Handle of any active request + LLCore::BufferArray * mHttpBufferArray; // Refcounted pointer to response data int mHttpPolicyClass; - bool mHttpActive; + bool mHttpActive; // Active request to http library unsigned int mHttpReplySize; unsigned int mHttpReplyOffset; + bool mHttpReleased; // Has been released from resource wait once }; -////////////////////////////////////////////////////////////////////////////// - - ////////////////////////////////////////////////////////////////////////////// // Cross-thread messaging for asset metrics. @@ -542,12 +692,13 @@ const char* LLTextureFetchWorker::sStateDescs[] = { "LOAD_FROM_NETWORK", "LOAD_FROM_SIMULATOR", "SEND_HTTP_REQ", + "WAIT_HTTP_RESOURCE", "WAIT_HTTP_REQ", "DECODE_IMAGE", "DECODE_IMAGE_UPDATE", "WRITE_TO_CACHE", "WAIT_ON_WRITE", - "DONE", + "DONE" }; // static @@ -608,7 +759,8 @@ LLTextureFetchWorker::LLTextureFetchWorker(LLTextureFetch* fetcher, mHttpPolicyClass(LLCore::HttpRequest::DEFAULT_POLICY_ID), mHttpActive(false), mHttpReplySize(0U), - mHttpReplyOffset(0U) + mHttpReplyOffset(0U), + mHttpReleased(true) { mCanUseNET = mUrl.empty() ; @@ -630,12 +782,13 @@ LLTextureFetchWorker::~LLTextureFetchWorker() // << " Requested=" << mRequestedDiscard // << " Desired=" << mDesiredDiscard << llendl; llassert_always(!haveWork()); + + lockWorkMutex(); // +Mw (should be useless) if (mHttpActive) { - LL_WARNS("Texture") << "Deleting worker object while HTTP request is active." - << LL_ENDL; + // Issue a cancel on a live request... + mFetcher->getHttpRequest().requestCancel(mHttpHandle, NULL); } - lockWorkMutex(); if (mCacheReadHandle != LLTextureCache::nullHandle() && mFetcher->mTextureCache) { mFetcher->mTextureCache->readComplete(mCacheReadHandle, true); @@ -646,15 +799,17 @@ LLTextureFetchWorker::~LLTextureFetchWorker() } mFormattedImage = NULL; clearPackets(); - unlockWorkMutex(); - mFetcher->removeFromHTTPQueue(mID); if (mHttpBufferArray) { mHttpBufferArray->release(); mHttpBufferArray = NULL; } + unlockWorkMutex(); // -Mw + mFetcher->removeFromHTTPQueue(mID); + mFetcher->removeHttpWaiter(mID); } +// Locks: Mw void LLTextureFetchWorker::clearPackets() { for_each(mPackets.begin(), mPackets.end(), DeletePointer()); @@ -664,6 +819,7 @@ void LLTextureFetchWorker::clearPackets() mFirstPacket = 0; } +// Locks: Mw void LLTextureFetchWorker::setupPacketData() { S32 data_size = 0; @@ -696,6 +852,7 @@ void LLTextureFetchWorker::setupPacketData() } } +// Locks: Mw (ctor invokes without lock) U32 LLTextureFetchWorker::calcWorkPriority() { //llassert_always(mImagePriority >= 0 && mImagePriority <= LLViewerFetchedTexture::maxDecodePriority()); @@ -705,7 +862,7 @@ U32 LLTextureFetchWorker::calcWorkPriority() return mWorkPriority; } -// mWorkMutex is locked +// Locks: Mw (ctor invokes without lock) void LLTextureFetchWorker::setDesiredDiscard(S32 discard, S32 size) { bool prioritize = false; @@ -741,6 +898,7 @@ void LLTextureFetchWorker::setDesiredDiscard(S32 discard, S32 size) } } +// Locks: Mw void LLTextureFetchWorker::setImagePriority(F32 priority) { // llassert_always(priority >= 0 && priority <= LLViewerTexture::maxDecodePriority()); @@ -754,6 +912,7 @@ void LLTextureFetchWorker::setImagePriority(F32 priority) } } +// Locks: Mw void LLTextureFetchWorker::resetFormattedData() { if (mHttpBufferArray) @@ -768,18 +927,19 @@ void LLTextureFetchWorker::resetFormattedData() mHaveAllData = FALSE; } -// Called from MAIN thread +// Threads: Tmain void LLTextureFetchWorker::startWork(S32 param) { llassert(mFormattedImage.isNull()); } -#include "llviewertexturelist.h" // debug - -// Called from LLWorkerThread::processRequest() +// Threads: Ttf bool LLTextureFetchWorker::doWork(S32 param) { - LLMutexLock lock(&mWorkMutex); + // Release waiters while we aren't holding the Mw lock. + mFetcher->releaseHttpWaiters(); + + LLMutexLock lock(&mWorkMutex); // +Mw if ((mFetcher->isQuitting() || getFlags(LLWorkerClass::WCF_DELETE_REQUESTED))) { @@ -863,7 +1023,7 @@ bool LLTextureFetchWorker::doWork(S32 param) if (mUrl.compare(0, 7, "file://") == 0) { - setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); // Set priority first since Responder may change it + setPriority(LLWorkerThread::PRIORITY_NORMAL | mWorkPriority); // Set priority first since Responder may change it // read file from local disk std::string filename = mUrl.substr(7, std::string::npos); @@ -873,7 +1033,7 @@ bool LLTextureFetchWorker::doWork(S32 param) } else if (mUrl.empty()) { - setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); // Set priority first since Responder may change it + setPriority(LLWorkerThread::PRIORITY_NORMAL | mWorkPriority); // Set priority first since Responder may change it CacheReadResponder* responder = new CacheReadResponder(mFetcher, mID, mFormattedImage); mCacheReadHandle = mFetcher->mTextureCache->readFromCache(mID, cache_priority, @@ -1002,7 +1162,7 @@ bool LLTextureFetchWorker::doWork(S32 param) mSentRequest = QUEUED; mFetcher->addToNetworkQueue(this); recordTextureStart(false); - setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); + setPriority(LLWorkerThread::PRIORITY_NORMAL | mWorkPriority); return false; } @@ -1013,7 +1173,7 @@ bool LLTextureFetchWorker::doWork(S32 param) // Make certain this is in the network queue //mFetcher->addToNetworkQueue(this); //recordTextureStart(false); - //setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); + //setPriority(LLWorkerThread::PRIORITY_NORMAL | mWorkPriority); return false; } } @@ -1042,7 +1202,7 @@ bool LLTextureFetchWorker::doWork(S32 param) else { mFetcher->addToNetworkQueue(this); // failsafe - setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); + setPriority(LLWorkerThread::PRIORITY_NORMAL | mWorkPriority); recordTextureStart(false); } return false; @@ -1050,89 +1210,100 @@ bool LLTextureFetchWorker::doWork(S32 param) if (mState == SEND_HTTP_REQ) { - if(mCanUseHTTP) - { - //NOTE: - //control the number of the http requests issued for: - //1, not openning too many file descriptors at the same time; - //2, control the traffic of http so udp gets bandwidth. - // - // static const S32 MAX_NUM_OF_HTTP_REQUESTS_IN_QUEUE = 8; - // *FIXME: For the moment, allow everything to transition into HTTP - // and have the new library order and throttle. - // if(mFetcher->getNumHTTPRequests() > MAX_NUM_OF_HTTP_REQUESTS_IN_QUEUE) - // { - // return false ; //wait. - // } + if (! mCanUseHTTP) + { + return true; // abort + } - mFetcher->removeFromNetworkQueue(this, false); - - S32 cur_size = 0; - if (mFormattedImage.notNull()) + // NOTE: + // control the number of the http requests issued for: + // 1, not openning too many file descriptors at the same time; + // 2, control the traffic of http so udp gets bandwidth. + // + if (! mHttpReleased) + { + // If this request hasn't been released before and it looks like + // we're busy, put this request into resource wait and allow something + // else to come to the front. + if (mFetcher->getNumHTTPRequests() >= HTTP_REQUESTS_IN_QUEUE_HIGH_WATER || + mFetcher->getHttpWaitersCount()) { - cur_size = mFormattedImage->getDataSize(); // amount of data we already have - if (mFormattedImage->getDiscardLevel() == 0) - { - if(cur_size > 0) - { - // We already have all the data, just decode it - mLoadedDiscard = mFormattedImage->getDiscardLevel(); - mState = DECODE_IMAGE; - return false; - } - else - { - return true ; //abort. - } - } + mState = WAIT_HTTP_RESOURCE; + setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); + mFetcher->addHttpWaiter(this->mID); + return false; } - mRequestedSize = mDesiredSize; - mRequestedDiscard = mDesiredDiscard; - mRequestedSize -= cur_size; - mRequestedOffset = cur_size; + } + + mFetcher->removeFromNetworkQueue(this, false); - mHttpHandle = LLCORE_HTTP_HANDLE_INVALID; - if (!mUrl.empty()) - { - mLoaded = FALSE; - mGetStatus = LLCore::HttpStatus(); - mGetReason.clear(); - LL_DEBUGS("Texture") << "HTTP GET: " << mID << " Offset: " << mRequestedOffset - << " Bytes: " << mRequestedSize - << " Bandwidth(kbps): " << mFetcher->getTextureBandwidth() << "/" << mFetcher->mMaxBandwidth - << LL_ENDL; - - // Will call callbackHttpGet when curl request completes - // *FIXME: enable redirection follow - mHttpHandle = mFetcher->mHttpRequest->requestGetByteRange(mHttpPolicyClass, - mWorkPriority, - mUrl, - mRequestedOffset, - mRequestedSize, - mFetcher->mHttpOptions, - mFetcher->mHttpHeaders, - this); - } - if (LLCORE_HTTP_HANDLE_INVALID == mHttpHandle) + S32 cur_size = 0; + if (mFormattedImage.notNull()) + { + cur_size = mFormattedImage->getDataSize(); // amount of data we already have + if (mFormattedImage->getDiscardLevel() == 0) { - llwarns << "HTTP GET request failed for " << mID << llendl; - resetFormattedData(); - ++mHTTPFailCount; - return true; // failed + if (cur_size > 0) + { + // We already have all the data, just decode it + mLoadedDiscard = mFormattedImage->getDiscardLevel(); + mState = DECODE_IMAGE; + return false; + } + else + { + return true; // abort. + } } - - mHttpActive = true; - mFetcher->addToHTTPQueue(mID); - recordTextureStart(true); - setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); - mState = WAIT_HTTP_REQ; - - // fall through } - else //can not use http fetch. - { - return true ; //abort + mRequestedSize = mDesiredSize; + mRequestedDiscard = mDesiredDiscard; + mRequestedSize -= cur_size; + mRequestedOffset = cur_size; + + mHttpHandle = LLCORE_HTTP_HANDLE_INVALID; + if (!mUrl.empty()) + { + mLoaded = FALSE; + mGetStatus = LLCore::HttpStatus(); + mGetReason.clear(); + LL_DEBUGS("Texture") << "HTTP GET: " << mID << " Offset: " << mRequestedOffset + << " Bytes: " << mRequestedSize + << " Bandwidth(kbps): " << mFetcher->getTextureBandwidth() << "/" << mFetcher->mMaxBandwidth + << LL_ENDL; + + // Will call callbackHttpGet when curl request completes + mHttpHandle = mFetcher->mHttpRequest->requestGetByteRange(mHttpPolicyClass, + mWorkPriority, + mUrl, + mRequestedOffset, + mRequestedSize, + mFetcher->mHttpOptions, + mFetcher->mHttpHeaders, + this); + } + if (LLCORE_HTTP_HANDLE_INVALID == mHttpHandle) + { + llwarns << "HTTP GET request failed for " << mID << llendl; + resetFormattedData(); + ++mHTTPFailCount; + return true; // failed } + + mHttpActive = true; + mFetcher->addToHTTPQueue(mID); + recordTextureStart(true); + setPriority(LLWorkerThread::PRIORITY_NORMAL | mWorkPriority); + mState = WAIT_HTTP_REQ; + + // fall through + } + + if (mState == WAIT_HTTP_RESOURCE) + { + // Nothing to do until releaseHttpWaiters() puts us back + // into the flow... + return false; } if (mState == WAIT_HTTP_REQ) @@ -1148,7 +1319,7 @@ bool LLTextureFetchWorker::doWork(S32 param) mHTTPFailCount = max_attempts = 1; // Don't retry llwarns << "Texture missing from server (404): " << mUrl << llendl; - //roll back to try UDP + // roll back to try UDP if (mCanUseNET) { mState = INIT; @@ -1263,7 +1434,7 @@ bool LLTextureFetchWorker::doWork(S32 param) } else { - setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); + setPriority(LLWorkerThread::PRIORITY_NORMAL | mWorkPriority); return false; } } @@ -1272,7 +1443,7 @@ bool LLTextureFetchWorker::doWork(S32 param) { static LLCachedControl textures_decode_disabled(gSavedSettings,"TextureDecodeDisabled"); - setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); // Set priority first since Responder may change it + setPriority(LLWorkerThread::PRIORITY_NORMAL | mWorkPriority); // Set priority first since Responder may change it if (textures_decode_disabled) { // for debug use, don't decode @@ -1380,7 +1551,7 @@ bool LLTextureFetchWorker::doWork(S32 param) } } llassert_always(datasize); - setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); // Set priority first since Responder may change it + setPriority(LLWorkerThread::PRIORITY_NORMAL | mWorkPriority); // Set priority first since Responder may change it U32 cache_priority = mWorkPriority; mWritten = FALSE; mState = WAIT_ON_WRITE; @@ -1422,14 +1593,15 @@ bool LLTextureFetchWorker::doWork(S32 param) } else { - setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); + setPriority(LLWorkerThread::PRIORITY_NORMAL | mWorkPriority); return true; } } return false; -} +} // -Mw +// Threads: Ttf // virtual void LLTextureFetchWorker::onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * response) { @@ -1437,6 +1609,8 @@ void LLTextureFetchWorker::onCompleted(LLCore::HttpHandle handle, LLCore::HttpRe static LLCachedControl log_to_sim(gSavedSettings, "LogTextureDownloadsToSimulator"); static LLCachedControl log_texture_traffic(gSavedSettings, "LogTextureNetworkTraffic") ; + LLMutexLock lock(&mWorkMutex); // +Mw + mHttpActive = false; if (log_to_viewer_log || log_to_sim) @@ -1494,10 +1668,10 @@ void LLTextureFetchWorker::onCompleted(LLCore::HttpHandle handle, LLCore::HttpRe mFetcher->removeFromHTTPQueue(mID); recordTextureDone(true); -} +} // -Mw -// Called from MAIN thread +// Threads: Tmain void LLTextureFetchWorker::endWork(S32 param, bool aborted) { if (mDecodeHandle != 0) @@ -1510,6 +1684,8 @@ void LLTextureFetchWorker::endWork(S32 param, bool aborted) ////////////////////////////////////////////////////////////////////////////// +// Threads: Ttf + // virtual void LLTextureFetchWorker::finishWork(S32 param, bool completed) { @@ -1526,6 +1702,13 @@ void LLTextureFetchWorker::finishWork(S32 param, bool completed) } } +// LLQueuedThread's update() method is asking if it's okay to +// delete this worker. You'll notice we're not locking in here +// which is a slight concern. Caller is expected to have made +// this request 'quiet' by whatever means... +// +// Threads: Tmain + // virtual bool LLTextureFetchWorker::deleteOK() { @@ -1572,6 +1755,7 @@ bool LLTextureFetchWorker::deleteOK() return delete_ok; } +// Threads: Ttf void LLTextureFetchWorker::removeFromCache() { if (!mInLocalCache) @@ -1583,6 +1767,8 @@ void LLTextureFetchWorker::removeFromCache() ////////////////////////////////////////////////////////////////////////////// +// Threads: Ttf +// Locks: Mw bool LLTextureFetchWorker::processSimulatorPackets() { if (mFormattedImage.isNull() || mRequestedSize < 0) @@ -1643,13 +1829,13 @@ bool LLTextureFetchWorker::processSimulatorPackets() ////////////////////////////////////////////////////////////////////////////// +// Threads: Ttf +// Locks: Mw S32 LLTextureFetchWorker::callbackHttpGet(LLCore::HttpResponse * response, bool partial, bool success) { S32 data_size = 0 ; - LLMutexLock lock(&mWorkMutex); - if (mState != WAIT_HTTP_REQ) { llwarns << "callbackHttpGet for unrequested fetch worker: " << mID @@ -1732,10 +1918,11 @@ S32 LLTextureFetchWorker::callbackHttpGet(LLCore::HttpResponse * response, ////////////////////////////////////////////////////////////////////////////// +// Threads: Ttc void LLTextureFetchWorker::callbackCacheRead(bool success, LLImageFormatted* image, S32 imagesize, BOOL islocal) { - LLMutexLock lock(&mWorkMutex); + LLMutexLock lock(&mWorkMutex); // +Mw if (mState != LOAD_FROM_TEXTURE_CACHE) { // llwarns << "Read callback for " << mID << " with state = " << mState << llendl; @@ -1755,11 +1942,12 @@ void LLTextureFetchWorker::callbackCacheRead(bool success, LLImageFormatted* ima } mLoaded = TRUE; setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); -} +} // -Mw +// Threads: Ttc void LLTextureFetchWorker::callbackCacheWrite(bool success) { - LLMutexLock lock(&mWorkMutex); + LLMutexLock lock(&mWorkMutex); // +Mw if (mState != WAIT_ON_WRITE) { // llwarns << "Write callback for " << mID << " with state = " << mState << llendl; @@ -1767,13 +1955,14 @@ void LLTextureFetchWorker::callbackCacheWrite(bool success) } mWritten = TRUE; setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); -} +} // -Mw ////////////////////////////////////////////////////////////////////////////// +// Threads: Tid void LLTextureFetchWorker::callbackDecoded(bool success, LLImageRaw* raw, LLImageRaw* aux) { - LLMutexLock lock(&mWorkMutex); + LLMutexLock lock(&mWorkMutex); // +Mw if (mDecodeHandle == 0) { return; // aborted, ignore @@ -1805,10 +1994,11 @@ void LLTextureFetchWorker::callbackDecoded(bool success, LLImageRaw* raw, LLImag mDecoded = TRUE; // llinfos << mID << " : DECODE COMPLETE " << llendl; setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); -} +} // -Mw ////////////////////////////////////////////////////////////////////////////// +// Threads: Ttf bool LLTextureFetchWorker::writeToCacheComplete() { // Complete write to cache @@ -1831,6 +2021,7 @@ bool LLTextureFetchWorker::writeToCacheComplete() } +// Threads: Ttf void LLTextureFetchWorker::recordTextureStart(bool is_http) { if (! mMetricsStartTime) @@ -1843,6 +2034,7 @@ void LLTextureFetchWorker::recordTextureStart(bool is_http) } +// Threads: Ttf void LLTextureFetchWorker::recordTextureDone(bool is_http) { if (mMetricsStartTime) @@ -1892,8 +2084,6 @@ LLTextureFetch::LLTextureFetch(LLTextureCache* cache, LLImageDecodeThread* image LLTextureFetch::~LLTextureFetch() { - cancelHttpRequests(); - clearDeleteList(); while (! mCommands.empty()) @@ -1915,6 +2105,8 @@ LLTextureFetch::~LLTextureFetch() mHttpHeaders = NULL; } + mHttpWaitResource.clear(); + delete mHttpRequest; mHttpRequest = NULL; @@ -1978,7 +2170,7 @@ bool LLTextureFetch::createRequest(const std::string& url, const LLUUID& id, con { return false; // need to wait for previous aborted request to complete } - worker->lockWorkMutex(); + worker->lockWorkMutex(); // +Mw worker->mActiveCount++; worker->mNeedsAux = needs_aux; worker->setImagePriority(priority); @@ -1987,41 +2179,44 @@ bool LLTextureFetch::createRequest(const std::string& url, const LLUUID& id, con if (!worker->haveWork()) { worker->mState = LLTextureFetchWorker::INIT; - worker->unlockWorkMutex(); + worker->unlockWorkMutex(); // -Mw worker->addWork(0, LLWorkerThread::PRIORITY_HIGH | worker->mWorkPriority); } else { - worker->unlockWorkMutex(); + worker->unlockWorkMutex(); // -Mw } } else { worker = new LLTextureFetchWorker(this, url, id, host, priority, desired_discard, desired_size); - lockQueue() ; + lockQueue(); // +Mfq mRequestMap[id] = worker; - unlockQueue() ; + unlockQueue(); // -Mfq - worker->lockWorkMutex(); + worker->lockWorkMutex(); // +Mw worker->mActiveCount++; worker->mNeedsAux = needs_aux; worker->setCanUseHTTP(can_use_http) ; - worker->unlockWorkMutex(); + worker->unlockWorkMutex(); // -Mw } // llinfos << "REQUESTED: " << id << " Discard: " << desired_discard << llendl; return true; } + +// Threads: T* (but Ttf in practice) + // protected void LLTextureFetch::addToNetworkQueue(LLTextureFetchWorker* worker) { - lockQueue() ; + lockQueue(); // +Mfq bool in_request_map = (mRequestMap.find(worker->mID) != mRequestMap.end()) ; - unlockQueue() ; + unlockQueue(); // -Mfq - LLMutexLock lock(&mNetworkQueueMutex); + LLMutexLock lock(&mNetworkQueueMutex); // +Mfnq if (in_request_map) { // only add to the queue if in the request map @@ -2033,63 +2228,50 @@ void LLTextureFetch::addToNetworkQueue(LLTextureFetchWorker* worker) { iter1->second.erase(worker->mID); } -} +} // -Mfnq +// Threads: T* void LLTextureFetch::removeFromNetworkQueue(LLTextureFetchWorker* worker, bool cancel) { - LLMutexLock lock(&mNetworkQueueMutex); + LLMutexLock lock(&mNetworkQueueMutex); // +Mfnq size_t erased = mNetworkQueue.erase(worker->mID); if (cancel && erased > 0) { mCancelQueue[worker->mHost].insert(worker->mID); } -} +} // -Mfnq +// Threads: T* +// // protected void LLTextureFetch::addToHTTPQueue(const LLUUID& id) { - LLMutexLock lock(&mNetworkQueueMutex); + LLMutexLock lock(&mNetworkQueueMutex); // +Mfnq mHTTPTextureQueue.insert(id); mTotalHTTPRequests++; -} +} // -Mfnq +// Threads: T* void LLTextureFetch::removeFromHTTPQueue(const LLUUID& id, S32 received_size) { - LLMutexLock lock(&mNetworkQueueMutex); + LLMutexLock lock(&mNetworkQueueMutex); // +Mfnq mHTTPTextureQueue.erase(id); mHTTPTextureBits += received_size * 8; // Approximate - does not include header bits -} - -void LLTextureFetch::cancelHttpRequests() -{ - for (queue_t::iterator iter(mHTTPTextureQueue.begin()); - mHTTPTextureQueue.end() != iter; - ++iter) - { - LLTextureFetchWorker* worker = getWorker(*iter); - if (worker && worker->mHttpActive) - { - mHttpRequest->requestCancel(worker->mHttpHandle, NULL); - } - } - - // *FIXME: Do this better with less time wasting. - int tries(10); - while (! mHTTPTextureQueue.empty() && --tries) - { - mHttpRequest->update(100); - ms_sleep(100); - } -} +} // -Mfnq +// NB: If you change deleteRequest() you should probably make +// parallel changes in removeRequest(). They're functionally +// identical with only argument variations. +// +// Threads: T* void LLTextureFetch::deleteRequest(const LLUUID& id, bool cancel) { - lockQueue() ; + lockQueue(); // +Mfq LLTextureFetchWorker* worker = getWorkerAfterLock(id); if (worker) { size_t erased_1 = mRequestMap.erase(worker->mID); - unlockQueue() ; + unlockQueue(); // -Mfq llassert_always(erased_1 > 0) ; @@ -2100,15 +2282,20 @@ void LLTextureFetch::deleteRequest(const LLUUID& id, bool cancel) } else { - unlockQueue() ; + unlockQueue(); // -Mfq } } +// NB: If you change removeRequest() you should probably make +// parallel changes in deleteRequest(). They're functionally +// identical with only argument variations. +// +// Threads: T* void LLTextureFetch::removeRequest(LLTextureFetchWorker* worker, bool cancel) { - lockQueue() ; + lockQueue(); // +Mfq size_t erased_1 = mRequestMap.erase(worker->mID); - unlockQueue() ; + unlockQueue(); // -Mfq llassert_always(erased_1 > 0) ; removeFromNetworkQueue(worker, cancel); @@ -2117,34 +2304,39 @@ void LLTextureFetch::removeRequest(LLTextureFetchWorker* worker, bool cancel) worker->scheduleDelete(); } +// Threads: T* S32 LLTextureFetch::getNumRequests() { - lockQueue() ; + lockQueue(); // +Mfq S32 size = (S32)mRequestMap.size(); - unlockQueue() ; + unlockQueue(); // -Mfq - return size ; + return size; } +// Threads: T* S32 LLTextureFetch::getNumHTTPRequests() { - mNetworkQueueMutex.lock() ; + mNetworkQueueMutex.lock(); // +Mfq S32 size = (S32)mHTTPTextureQueue.size(); - mNetworkQueueMutex.unlock() ; + mNetworkQueueMutex.unlock(); // -Mfq - return size ; + return size; } +// Threads: T* U32 LLTextureFetch::getTotalNumHTTPRequests() { - mNetworkQueueMutex.lock() ; - U32 size = mTotalHTTPRequests ; - mNetworkQueueMutex.unlock() ; + mNetworkQueueMutex.lock(); // +Mfq + U32 size = mTotalHTTPRequests; + mNetworkQueueMutex.unlock(); // -Mfq - return size ; + return size; } // call lockQueue() first! +// Threads: T* +// Locks: Mfq LLTextureFetchWorker* LLTextureFetch::getWorkerAfterLock(const LLUUID& id) { LLTextureFetchWorker* res = NULL; @@ -2156,14 +2348,16 @@ LLTextureFetchWorker* LLTextureFetch::getWorkerAfterLock(const LLUUID& id) return res; } +// Threads: T* LLTextureFetchWorker* LLTextureFetch::getWorker(const LLUUID& id) { - LLMutexLock lock(&mQueueMutex) ; + LLMutexLock lock(&mQueueMutex); // +Mfq - return getWorkerAfterLock(id) ; -} + return getWorkerAfterLock(id); +} // -Mfq +// Threads: T* bool LLTextureFetch::getRequestFinished(const LLUUID& id, S32& discard_level, LLPointer& raw, LLPointer& aux) { @@ -2186,17 +2380,17 @@ bool LLTextureFetch::getRequestFinished(const LLUUID& id, S32& discard_level, } else if (worker->checkWork()) { - worker->lockWorkMutex(); + worker->lockWorkMutex(); // +Mw discard_level = worker->mDecodedDiscard; raw = worker->mRawImage; aux = worker->mAuxImage; res = true; LL_DEBUGS("Texture") << id << ": Request Finished. State: " << worker->mState << " Discard: " << discard_level << LL_ENDL; - worker->unlockWorkMutex(); + worker->unlockWorkMutex(); // -Mw } else { - worker->lockWorkMutex(); + worker->lockWorkMutex(); // +Mw if ((worker->mDecodedDiscard >= 0) && (worker->mDecodedDiscard < discard_level || discard_level < 0) && (worker->mState >= LLTextureFetchWorker::WAIT_ON_WRITE)) @@ -2206,7 +2400,7 @@ bool LLTextureFetch::getRequestFinished(const LLUUID& id, S32& discard_level, raw = worker->mRawImage; aux = worker->mAuxImage; } - worker->unlockWorkMutex(); + worker->unlockWorkMutex(); // -Mw } } else @@ -2216,15 +2410,16 @@ bool LLTextureFetch::getRequestFinished(const LLUUID& id, S32& discard_level, return res; } +// Threads: T* bool LLTextureFetch::updateRequestPriority(const LLUUID& id, F32 priority) { bool res = false; LLTextureFetchWorker* worker = getWorker(id); if (worker) { - worker->lockWorkMutex(); + worker->lockWorkMutex(); // +Mw worker->setImagePriority(priority); - worker->unlockWorkMutex(); + worker->unlockWorkMutex(); // -Mw res = true; } return res; @@ -2239,23 +2434,24 @@ bool LLTextureFetch::updateRequestPriority(const LLUUID& id, F32 priority) // in step, at least until this can be refactored and // the redundancy eliminated. // -// May be called from any thread +// Threads: T* //virtual S32 LLTextureFetch::getPending() { S32 res; - lockData(); + lockData(); // +Ct { - LLMutexLock lock(&mQueueMutex); + LLMutexLock lock(&mQueueMutex); // +Mfq res = mRequestQueue.size(); res += mCommands.size(); - } - unlockData(); + } // -Mfq + unlockData(); // -Ct return res; } +// Locks: Ct // virtual bool LLTextureFetch::runCondition() { @@ -2270,10 +2466,10 @@ bool LLTextureFetch::runCondition() bool have_no_commands(false); { - LLMutexLock lock(&mQueueMutex); + LLMutexLock lock(&mQueueMutex); // +Mfq have_no_commands = mCommands.empty(); - } + } // -Mfq return ! (have_no_commands && (mRequestQueue.empty() && mIdleThread)); // From base class @@ -2281,7 +2477,7 @@ bool LLTextureFetch::runCondition() ////////////////////////////////////////////////////////////////////////////// -// MAIN THREAD (unthreaded envs), WORKER THREAD (threaded envs) +// Threads: Ttf void LLTextureFetch::commonUpdate() { // Run a cross-thread command, if any. @@ -2306,20 +2502,21 @@ void LLTextureFetch::commonUpdate() } -// MAIN THREAD +// Threads: Tmain + //virtual S32 LLTextureFetch::update(F32 max_time_ms) { static LLCachedControl band_width(gSavedSettings,"ThrottleBandwidthKBPS"); { - mNetworkQueueMutex.lock() ; - mMaxBandwidth = band_width ; + mNetworkQueueMutex.lock(); // +Mfnq + mMaxBandwidth = band_width; - gTextureList.sTextureBits += mHTTPTextureBits ; - mHTTPTextureBits = 0 ; + gTextureList.sTextureBits += mHTTPTextureBits; + mHTTPTextureBits = 0; - mNetworkQueueMutex.unlock() ; + mNetworkQueueMutex.unlock(); // -Mfnq } S32 res = LLWorkerThread::update(max_time_ms); @@ -2337,7 +2534,9 @@ S32 LLTextureFetch::update(F32 max_time_ms) return res; } -//called in the MAIN thread after the TextureCacheThread shuts down. +// called in the MAIN thread after the TextureCacheThread shuts down. +// +// Threads: Tmain void LLTextureFetch::shutDownTextureCacheThread() { if(mTextureCache) @@ -2347,7 +2546,9 @@ void LLTextureFetch::shutDownTextureCacheThread() } } -//called in the MAIN thread after the ImageDecodeThread shuts down. +// called in the MAIN thread after the ImageDecodeThread shuts down. +// +// Threads: Tmain void LLTextureFetch::shutDownImageDecodeThread() { if(mImageDecodeThread) @@ -2357,22 +2558,21 @@ void LLTextureFetch::shutDownImageDecodeThread() } } -// WORKER THREAD +// Threads: Ttf void LLTextureFetch::startThread() { } -// WORKER THREAD +// Threads: Ttf // // This detaches the texture fetch thread from the LLCore // HTTP library but doesn't stop the thread running in that // library... void LLTextureFetch::endThread() { - cancelHttpRequests(); } -// WORKER THREAD +// Threads: Ttf void LLTextureFetch::threadedUpdate() { llassert_always(mHttpRequest); @@ -2406,6 +2606,7 @@ void LLTextureFetch::threadedUpdate() ////////////////////////////////////////////////////////////////////////////// +// Threads: Tmain void LLTextureFetch::sendRequestListToSimulators() { // All requests @@ -2431,48 +2632,48 @@ void LLTextureFetch::sendRequestListToSimulators() typedef std::map< LLHost, request_list_t > work_request_map_t; work_request_map_t requests; { - LLMutexLock lock2(&mNetworkQueueMutex); - for (queue_t::iterator iter = mNetworkQueue.begin(); iter != mNetworkQueue.end(); ) - { - queue_t::iterator curiter = iter++; - LLTextureFetchWorker* req = getWorker(*curiter); - if (!req) - { - mNetworkQueue.erase(curiter); - continue; // paranoia - } - if ((req->mState != LLTextureFetchWorker::LOAD_FROM_NETWORK) && - (req->mState != LLTextureFetchWorker::LOAD_FROM_SIMULATOR)) + LLMutexLock lock2(&mNetworkQueueMutex); // +Mfnq + for (queue_t::iterator iter = mNetworkQueue.begin(); iter != mNetworkQueue.end(); ) { - // We already received our URL, remove from the queue - llwarns << "Worker: " << req->mID << " in mNetworkQueue but in wrong state: " << req->mState << llendl; - mNetworkQueue.erase(curiter); - continue; - } - if (req->mID == mDebugID) - { - mDebugCount++; // for setting breakpoints - } - if (req->mSentRequest == LLTextureFetchWorker::SENT_SIM && - req->mTotalPackets > 0 && - req->mLastPacket >= req->mTotalPackets-1) - { - // We have all the packets... make sure this is high priority + queue_t::iterator curiter = iter++; + LLTextureFetchWorker* req = getWorker(*curiter); + if (!req) + { + mNetworkQueue.erase(curiter); + continue; // paranoia + } + if ((req->mState != LLTextureFetchWorker::LOAD_FROM_NETWORK) && + (req->mState != LLTextureFetchWorker::LOAD_FROM_SIMULATOR)) + { + // We already received our URL, remove from the queue + llwarns << "Worker: " << req->mID << " in mNetworkQueue but in wrong state: " << req->mState << llendl; + mNetworkQueue.erase(curiter); + continue; + } + if (req->mID == mDebugID) + { + mDebugCount++; // for setting breakpoints + } + if (req->mSentRequest == LLTextureFetchWorker::SENT_SIM && + req->mTotalPackets > 0 && + req->mLastPacket >= req->mTotalPackets-1) + { + // We have all the packets... make sure this is high priority // req->setPriority(LLWorkerThread::PRIORITY_HIGH | req->mWorkPriority); - continue; - } - F32 elapsed = req->mRequestedTimer.getElapsedTimeF32(); - { - F32 delta_priority = llabs(req->mRequestedPriority - req->mImagePriority); - if ((req->mSimRequestedDiscard != req->mDesiredDiscard) || - (delta_priority > MIN_DELTA_PRIORITY && elapsed >= MIN_REQUEST_TIME) || - (elapsed >= SIM_LAZY_FLUSH_TIMEOUT)) + continue; + } + F32 elapsed = req->mRequestedTimer.getElapsedTimeF32(); { - requests[req->mHost].insert(req); + F32 delta_priority = llabs(req->mRequestedPriority - req->mImagePriority); + if ((req->mSimRequestedDiscard != req->mDesiredDiscard) || + (delta_priority > MIN_DELTA_PRIORITY && elapsed >= MIN_REQUEST_TIME) || + (elapsed >= SIM_LAZY_FLUSH_TIMEOUT)) + { + requests[req->mHost].insert(req); + } } } - } - } + } // -Mfnq for (work_request_map_t::iterator iter1 = requests.begin(); iter1 != requests.end(); ++iter1) @@ -2495,9 +2696,9 @@ void LLTextureFetch::sendRequestListToSimulators() if (req->mSentRequest != LLTextureFetchWorker::SENT_SIM) { // Initialize packet data based on data read from cache - req->lockWorkMutex(); + req->lockWorkMutex(); // +Mw req->setupPacketData(); - req->unlockWorkMutex(); + req->unlockWorkMutex(); // -Mw } if (0 == sim_request_count) { @@ -2526,12 +2727,12 @@ void LLTextureFetch::sendRequestListToSimulators() mTextureInfo.setRequestType(req->mID, LLTextureInfoDetails::REQUEST_TYPE_UDP); } - req->lockWorkMutex(); + req->lockWorkMutex(); // +Mw req->mSentRequest = LLTextureFetchWorker::SENT_SIM; req->mSimRequestedDiscard = req->mDesiredDiscard; req->mRequestedPriority = req->mImagePriority; req->mRequestedTimer.reset(); - req->unlockWorkMutex(); + req->unlockWorkMutex(); // -Mw sim_request_count++; if (sim_request_count >= IMAGES_PER_REQUEST) { @@ -2552,55 +2753,57 @@ void LLTextureFetch::sendRequestListToSimulators() // Send cancelations { - LLMutexLock lock2(&mNetworkQueueMutex); - if (gMessageSystem && !mCancelQueue.empty()) - { - for (cancel_queue_t::iterator iter1 = mCancelQueue.begin(); - iter1 != mCancelQueue.end(); ++iter1) + LLMutexLock lock2(&mNetworkQueueMutex); // +Mfnq + if (gMessageSystem && !mCancelQueue.empty()) { - LLHost host = iter1->first; - if (host == LLHost::invalid) - { - host = gAgent.getRegionHost(); - } - S32 request_count = 0; - for (queue_t::iterator iter2 = iter1->second.begin(); - iter2 != iter1->second.end(); ++iter2) + for (cancel_queue_t::iterator iter1 = mCancelQueue.begin(); + iter1 != mCancelQueue.end(); ++iter1) { - if (0 == request_count) + LLHost host = iter1->first; + if (host == LLHost::invalid) { - gMessageSystem->newMessageFast(_PREHASH_RequestImage); - gMessageSystem->nextBlockFast(_PREHASH_AgentData); - gMessageSystem->addUUIDFast(_PREHASH_AgentID, gAgent.getID()); - gMessageSystem->addUUIDFast(_PREHASH_SessionID, gAgent.getSessionID()); + host = gAgent.getRegionHost(); } - gMessageSystem->nextBlockFast(_PREHASH_RequestImage); - gMessageSystem->addUUIDFast(_PREHASH_Image, *iter2); - gMessageSystem->addS8Fast(_PREHASH_DiscardLevel, -1); - gMessageSystem->addF32Fast(_PREHASH_DownloadPriority, 0); - gMessageSystem->addU32Fast(_PREHASH_Packet, 0); - gMessageSystem->addU8Fast(_PREHASH_Type, 0); + S32 request_count = 0; + for (queue_t::iterator iter2 = iter1->second.begin(); + iter2 != iter1->second.end(); ++iter2) + { + if (0 == request_count) + { + gMessageSystem->newMessageFast(_PREHASH_RequestImage); + gMessageSystem->nextBlockFast(_PREHASH_AgentData); + gMessageSystem->addUUIDFast(_PREHASH_AgentID, gAgent.getID()); + gMessageSystem->addUUIDFast(_PREHASH_SessionID, gAgent.getSessionID()); + } + gMessageSystem->nextBlockFast(_PREHASH_RequestImage); + gMessageSystem->addUUIDFast(_PREHASH_Image, *iter2); + gMessageSystem->addS8Fast(_PREHASH_DiscardLevel, -1); + gMessageSystem->addF32Fast(_PREHASH_DownloadPriority, 0); + gMessageSystem->addU32Fast(_PREHASH_Packet, 0); + gMessageSystem->addU8Fast(_PREHASH_Type, 0); // llinfos << "CANCELING IMAGE REQUEST: " << (*iter2) << llendl; - request_count++; - if (request_count >= IMAGES_PER_REQUEST) + request_count++; + if (request_count >= IMAGES_PER_REQUEST) + { + gMessageSystem->sendSemiReliable(host, NULL, NULL); + request_count = 0; + } + } + if (request_count > 0 && request_count < IMAGES_PER_REQUEST) { gMessageSystem->sendSemiReliable(host, NULL, NULL); - request_count = 0; } } - if (request_count > 0 && request_count < IMAGES_PER_REQUEST) - { - gMessageSystem->sendSemiReliable(host, NULL, NULL); - } + mCancelQueue.clear(); } - mCancelQueue.clear(); - } - } + } // -Mfnq } ////////////////////////////////////////////////////////////////////////////// +// Threads: T* +// Locks: Mw bool LLTextureFetchWorker::insertPacket(S32 index, U8* data, S32 size) { mRequestedTimer.reset(); @@ -2633,6 +2836,7 @@ bool LLTextureFetchWorker::insertPacket(S32 index, U8* data, S32 size) return true; } +// Threads: T* bool LLTextureFetch::receiveImageHeader(const LLHost& host, const LLUUID& id, U8 codec, U16 packets, U32 totalbytes, U16 data_size, U8* data) { @@ -2667,14 +2871,14 @@ bool LLTextureFetch::receiveImageHeader(const LLHost& host, const LLUUID& id, U8 } if (!res) { + mNetworkQueueMutex.lock(); // +Mfnq ++mBadPacketCount; - mNetworkQueueMutex.lock() ; mCancelQueue[host].insert(id); - mNetworkQueueMutex.unlock() ; + mNetworkQueueMutex.unlock(); // -Mfnq return false; } - worker->lockWorkMutex(); + worker->lockWorkMutex(); // +Mw // Copy header data into image object worker->mImageCodec = codec; @@ -2685,10 +2889,12 @@ bool LLTextureFetch::receiveImageHeader(const LLHost& host, const LLUUID& id, U8 res = worker->insertPacket(0, data, data_size); worker->setPriority(LLWorkerThread::PRIORITY_HIGH | worker->mWorkPriority); worker->mState = LLTextureFetchWorker::LOAD_FROM_SIMULATOR; - worker->unlockWorkMutex(); + worker->unlockWorkMutex(); // -Mw return res; } + +// Threads: T* bool LLTextureFetch::receiveImagePacket(const LLHost& host, const LLUUID& id, U16 packet_num, U16 data_size, U8* data) { LLTextureFetchWorker* worker = getWorker(id); @@ -2713,14 +2919,14 @@ bool LLTextureFetch::receiveImagePacket(const LLHost& host, const LLUUID& id, U1 } if (!res) { + mNetworkQueueMutex.lock(); // +Mfnq ++mBadPacketCount; - mNetworkQueueMutex.lock() ; mCancelQueue[host].insert(id); - mNetworkQueueMutex.unlock() ; + mNetworkQueueMutex.unlock(); // -Mfnq return false; } - worker->lockWorkMutex(); + worker->lockWorkMutex(); // +Mw res = worker->insertPacket(packet_num, data, data_size); @@ -2737,7 +2943,7 @@ bool LLTextureFetch::receiveImagePacket(const LLHost& host, const LLUUID& id, U1 removeFromNetworkQueue(worker, true); // failsafe } - if(packet_num >= (worker->mTotalPackets - 1)) + if (packet_num >= (worker->mTotalPackets - 1)) { static LLCachedControl log_to_viewer_log(gSavedSettings,"LogTextureDownloadsToViewerLog"); static LLCachedControl log_to_sim(gSavedSettings,"LogTextureDownloadsToSimulator"); @@ -2749,12 +2955,14 @@ bool LLTextureFetch::receiveImagePacket(const LLHost& host, const LLUUID& id, U1 mTextureInfo.setRequestCompleteTimeAndLog(id, timeNow); } } - worker->unlockWorkMutex(); + worker->unlockWorkMutex(); // -Mw return res; } ////////////////////////////////////////////////////////////////////////////// + +// Threads: T* BOOL LLTextureFetch::isFromLocalCache(const LLUUID& id) { BOOL from_cache = FALSE ; @@ -2762,14 +2970,15 @@ BOOL LLTextureFetch::isFromLocalCache(const LLUUID& id) LLTextureFetchWorker* worker = getWorker(id); if (worker) { - worker->lockWorkMutex() ; - from_cache = worker->mInLocalCache ; - worker->unlockWorkMutex() ; + worker->lockWorkMutex(); // +Mw + from_cache = worker->mInLocalCache; + worker->unlockWorkMutex(); // -Mw } return from_cache ; } +// Threads: T* S32 LLTextureFetch::getFetchState(const LLUUID& id, F32& data_progress_p, F32& requested_priority_p, U32& fetch_priority_p, F32& fetch_dtime_p, F32& request_dtime_p, bool& can_use_http) { @@ -2783,7 +2992,7 @@ S32 LLTextureFetch::getFetchState(const LLUUID& id, F32& data_progress_p, F32& r LLTextureFetchWorker* worker = getWorker(id); if (worker && worker->haveWork()) { - worker->lockWorkMutex(); + worker->lockWorkMutex(); // +Mw state = worker->mState; fetch_dtime = worker->mFetchTimer.getElapsedTimeF32(); request_dtime = worker->mRequestedTimer.getElapsedTimeF32(); @@ -2810,7 +3019,7 @@ S32 LLTextureFetch::getFetchState(const LLUUID& id, F32& data_progress_p, F32& r } fetch_priority = worker->getPriority(); can_use_http = worker->getCanUseHTTP() ; - worker->unlockWorkMutex(); + worker->unlockWorkMutex(); // -Mw } data_progress_p = data_progress; requested_priority_p = requested_priority; @@ -2842,12 +3051,136 @@ void LLTextureFetch::dump() { llinfos << " ID: " << (*iter) << llendl; } + + llinfos << "LLTextureFetch WAIT_HTTP_RESOURCE:" << llendl; + for (wait_http_res_queue_t::const_iterator iter(mHttpWaitResource.begin()); + mHttpWaitResource.end() != iter; + ++iter) + { + llinfos << " ID: " << (*iter) << llendl; + } } +////////////////////////////////////////////////////////////////////////////// + +// HTTP Resource Waiting Methods + +// Threads: Ttf +void LLTextureFetch::addHttpWaiter(const LLUUID & tid) +{ + mNetworkQueueMutex.lock(); // +Mfnq + mHttpWaitResource.insert(tid); + mNetworkQueueMutex.unlock(); // -Mfnq +} + +// Threads: Ttf +void LLTextureFetch::removeHttpWaiter(const LLUUID & tid) +{ + mNetworkQueueMutex.lock(); // +Mfnq + wait_http_res_queue_t::iterator iter(mHttpWaitResource.find(tid)); + if (mHttpWaitResource.end() != iter) + { + mHttpWaitResource.erase(iter); + } + mNetworkQueueMutex.unlock(); // -Mfnq +} + +// Threads: Ttf +// Locks: -Mw (must not hold any worker when called) +void LLTextureFetch::releaseHttpWaiters() +{ + if (HTTP_REQUESTS_IN_QUEUE_LOW_WATER < getNumHTTPRequests()) + return; + + // Quickly make a copy of all the LLUIDs. Get off the + // mutex as early as possible. + + typedef std::vector uuid_vec_t; + uuid_vec_t tids; + + { + LLMutexLock lock(&mNetworkQueueMutex); // +Mfnq + + if (mHttpWaitResource.empty()) + return; + + const size_t limit(mHttpWaitResource.size()); + tids.resize(limit); + wait_http_res_queue_t::iterator iter(mHttpWaitResource.begin()); + for (int i(0); + i < limit && mHttpWaitResource.end() != iter; + ++i, ++iter) + { + tids[i] = *iter; + } + } // -Mfnq + + // Now lookup the UUUIDs to find valid requests and sort + // them in priority order, highest to lowest. + typedef std::set worker_set_t; + worker_set_t tids2; + + for (uuid_vec_t::const_iterator iter(tids.begin()); + tids.end() != iter; + ++iter) + { + LLTextureFetchWorker * worker(getWorker(* iter)); + if (worker) + { + tids2.insert(worker); + } + } + + // Release workers up to the high water mark. Since we aren't + // holding any locks at this point, we can be in competition + // with other callers. Do defensive things like getting + // refreshed counts of requests and checking if someone else + // has moved any worker state around.... + tids.clear(); + for (worker_set_t::iterator iter2(tids2.begin()); + tids2.end() != iter2 && 0 < (HTTP_REQUESTS_IN_QUEUE_HIGH_WATER - getNumHTTPRequests()); + ++iter2) + { + LLTextureFetchWorker * worker(* iter2); + + worker->lockWorkMutex(); // +Mw + if (LLTextureFetchWorker::WAIT_HTTP_RESOURCE != worker->mState) + { + worker->unlockWorkMutex(); // -Mw + continue; + } + worker->mHttpReleased = true; + worker->mState = LLTextureFetchWorker::SEND_HTTP_REQ; + worker->setPriority(LLWorkerThread::PRIORITY_HIGH | worker->mWorkPriority); + worker->unlockWorkMutex(); // -Mw + + removeHttpWaiter(worker->mID); + } +} + +// Threads: T* +void LLTextureFetch::cancelHttpWaiters() +{ + mNetworkQueueMutex.lock(); // +Mfnq + mHttpWaitResource.clear(); + mNetworkQueueMutex.unlock(); // -Mfnq +} + +// Threads: T* +int LLTextureFetch::getHttpWaitersCount() +{ + mNetworkQueueMutex.lock(); // +Mfnq + int ret(mHttpWaitResource.size()); + mNetworkQueueMutex.unlock(); // -Mfnq + return ret; +} + + ////////////////////////////////////////////////////////////////////////////// // cross-thread command methods +// Threads: T* void LLTextureFetch::commandSetRegion(U64 region_handle) { TFReqSetRegion * req = new TFReqSetRegion(region_handle); @@ -2855,6 +3188,7 @@ void LLTextureFetch::commandSetRegion(U64 region_handle) cmdEnqueue(req); } +// Threads: T* void LLTextureFetch::commandSendMetrics(const std::string & caps_url, const LLUUID & session_id, const LLUUID & agent_id, @@ -2865,6 +3199,7 @@ void LLTextureFetch::commandSendMetrics(const std::string & caps_url, cmdEnqueue(req); } +// Threads: T* void LLTextureFetch::commandDataBreak() { // The pedantically correct way to implement this is to create a command @@ -2875,30 +3210,33 @@ void LLTextureFetch::commandDataBreak() LLTextureFetch::svMetricsDataBreak = true; } +// Threads: T* void LLTextureFetch::cmdEnqueue(TFRequest * req) { - lockQueue(); + lockQueue(); // +Mfq mCommands.push_back(req); - unlockQueue(); + unlockQueue(); // -Mfq unpause(); } +// Threads: T* LLTextureFetch::TFRequest * LLTextureFetch::cmdDequeue() { TFRequest * ret = 0; - lockQueue(); + lockQueue(); // +Mfq if (! mCommands.empty()) { ret = mCommands.front(); mCommands.erase(mCommands.begin()); } - unlockQueue(); + unlockQueue(); // -Mfq return ret; } +// Threads: Ttf void LLTextureFetch::cmdDoWork() { if (mDebugPause) @@ -2932,6 +3270,8 @@ namespace class AssetReportHandler : public LLCore::HttpHandler { public: + + // Threads: Ttf virtual void onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * response) { LLCore::HttpStatus status(response->getStatus()); @@ -3100,5 +3440,3 @@ truncate_viewer_metrics(int max_regions, LLSD & metrics) } // end of anonymous namespace - - diff --git a/indra/newview/lltexturefetch.h b/indra/newview/lltexturefetch.h index cfea3aad9d..53b0f7885f 100644 --- a/indra/newview/lltexturefetch.h +++ b/indra/newview/lltexturefetch.h @@ -45,6 +45,7 @@ class LLHost; class LLViewerAssetStats; // Interface class + class LLTextureFetch : public LLWorkerThread { friend class LLTextureFetchWorker; @@ -55,67 +56,174 @@ public: class TFRequest; - /*virtual*/ S32 update(F32 max_time_ms); - void shutDownTextureCacheThread() ; //called in the main thread after the TextureCacheThread shuts down. - void shutDownImageDecodeThread() ; //called in the main thread after the ImageDecodeThread shuts down. + // Threads: Tmain + /*virtual*/ S32 update(F32 max_time_ms); + + // called in the main thread after the TextureCacheThread shuts down. + // Threads: Tmain + void shutDownTextureCacheThread(); + + //called in the main thread after the ImageDecodeThread shuts down. + // Threads: Tmain + void shutDownImageDecodeThread(); + // Threads: T* (but Tmain mostly) bool createRequest(const std::string& url, const LLUUID& id, const LLHost& host, F32 priority, S32 w, S32 h, S32 c, S32 discard, bool needs_aux, bool can_use_http); + + // Requests that a fetch operation be deleted from the queue. + // If @cancel is true, also stops any I/O operations pending. + // Actual delete will be scheduled and performed later. + // + // Note: This *looks* like an override/variant of the + // base class's deleteRequest() but is functionally quite + // different. + // + // Threads: T* void deleteRequest(const LLUUID& id, bool cancel); + + // Threads: T* bool getRequestFinished(const LLUUID& id, S32& discard_level, LLPointer& raw, LLPointer& aux); + + // Threads: T* bool updateRequestPriority(const LLUUID& id, F32 priority); + // Threads: T* bool receiveImageHeader(const LLHost& host, const LLUUID& id, U8 codec, U16 packets, U32 totalbytes, U16 data_size, U8* data); + + // Threads: T* bool receiveImagePacket(const LLHost& host, const LLUUID& id, U16 packet_num, U16 data_size, U8* data); + // Threads: T* (but not safe) void setTextureBandwidth(F32 bandwidth) { mTextureBandwidth = bandwidth; } + + // Threads: T* (but not safe) F32 getTextureBandwidth() { return mTextureBandwidth; } - // Debug + // Threads: T* BOOL isFromLocalCache(const LLUUID& id); + + // @return Magic number giving the internal state of the + // request. We should make these codes public if we're + // going to return them as a status value. + // + // Threads: T* S32 getFetchState(const LLUUID& id, F32& decode_progress_p, F32& requested_priority_p, U32& fetch_priority_p, F32& fetch_dtime_p, F32& request_dtime_p, bool& can_use_http); + + // Debug utility - generally not safe void dump(); - S32 getNumRequests() ; - S32 getNumHTTPRequests() ; - U32 getTotalNumHTTPRequests() ; + + // Threads: T* + S32 getNumRequests(); + + // Threads: T* + S32 getNumHTTPRequests(); + + // Threads: T* + U32 getTotalNumHTTPRequests(); - // Public for access by callbacks + // Threads: T* S32 getPending(); + + // Threads: T* void lockQueue() { mQueueMutex.lock(); } + + // Threads: T* void unlockQueue() { mQueueMutex.unlock(); } + + // Threads: T* LLTextureFetchWorker* getWorker(const LLUUID& id); + + // Threads: T* + // Locks: Mfq LLTextureFetchWorker* getWorkerAfterLock(const LLUUID& id); // Commands available to other threads to control metrics gathering operations. + + // Threads: T* void commandSetRegion(U64 region_handle); + + // Threads: T* void commandSendMetrics(const std::string & caps_url, const LLUUID & session_id, const LLUUID & agent_id, LLViewerAssetStats * main_stats); + + // Threads: T* void commandDataBreak(); + // Threads: T* LLCore::HttpRequest & getHttpRequest() { return *mHttpRequest; } bool isQAMode() const { return mQAMode; } + // ---------------------------------- + // HTTP resource waiting methods + + // Threads: T* + void addHttpWaiter(const LLUUID & tid); + + // Threads: T* + void removeHttpWaiter(const LLUUID & tid); + + // If there are slots, release one or more LLTextureFetchWorker + // requests from resource wait state (WAIT_HTTP_RESOURCE) to + // active (SEND_HTTP_REQ). + // + // Because this will modify state of many workers, you may not + // hold any Mw lock while calling. This makes it a little + // inconvenient to use but that's the rule. + // + // Threads: T* + // Locks: -Mw (must not hold any worker when called) + void releaseHttpWaiters(); + + // Threads: T* + void cancelHttpWaiters(); + + // Threads: T* + int getHttpWaitersCount(); + // ---------------------------------- + protected: + // Threads: T* (but Ttf in practice) void addToNetworkQueue(LLTextureFetchWorker* worker); + + // Threads: T* void removeFromNetworkQueue(LLTextureFetchWorker* worker, bool cancel); + + // Threads: T* void addToHTTPQueue(const LLUUID& id); + + // Threads: T* void removeFromHTTPQueue(const LLUUID& id, S32 received_size = 0); + + // Identical to @deleteRequest but with different arguments + // (caller already has the worker pointer). + // + // Threads: T* void removeRequest(LLTextureFetchWorker* worker, bool cancel); - void cancelHttpRequests(); // Overrides from the LLThread tree + // Locks: Ct bool runCondition(); private: + // Threads: Tmain void sendRequestListToSimulators(); + + // Threads: Ttf /*virtual*/ void startThread(void); + + // Threads: Ttf /*virtual*/ void endThread(void); + + // Threads: Ttf /*virtual*/ void threadedUpdate(void); + + // Threads: Ttf void commonUpdate(); // Metrics command helpers @@ -126,6 +234,8 @@ private: * Takes ownership of the TFRequest object. * * Method locks the command queue. + * + * Threads: T* */ void cmdEnqueue(TFRequest *); @@ -136,6 +246,8 @@ private: * Caller acquires ownership of the object and must dispose of it. * * Method locks the command queue. + * + * Threads: T* */ TFRequest * cmdDequeue(); @@ -145,6 +257,8 @@ private: * additional commands. * * Method locks the command queue. + * + * Threads: Ttf */ void cmdDoWork(); @@ -164,29 +278,29 @@ private: // Map of all requests by UUID typedef std::map map_t; - map_t mRequestMap; + map_t mRequestMap; // Mfq // Set of requests that require network data typedef std::set queue_t; - queue_t mNetworkQueue; - queue_t mHTTPTextureQueue; + queue_t mNetworkQueue; // Mfnq + queue_t mHTTPTextureQueue; // Mfnq typedef std::map > cancel_queue_t; - cancel_queue_t mCancelQueue; - F32 mTextureBandwidth; - F32 mMaxBandwidth; + cancel_queue_t mCancelQueue; // Mfnq + F32 mTextureBandwidth; // + F32 mMaxBandwidth; // Mfnq LLTextureInfo mTextureInfo; - U32 mHTTPTextureBits; + U32 mHTTPTextureBits; // Mfnq //debug use - U32 mTotalHTTPRequests ; + U32 mTotalHTTPRequests; // Out-of-band cross-thread command queue. This command queue // is logically tied to LLQueuedThread's list of // QueuedRequest instances and so must be covered by the // same locks. typedef std::vector command_queue_t; - command_queue_t mCommands; + command_queue_t mCommands; // Mfq // If true, modifies some behaviors that help with QA tasks. const bool mQAMode; @@ -197,6 +311,9 @@ private: LLCore::HttpRequest * mHttpRequest; LLCore::HttpOptions * mHttpOptions; LLCore::HttpHeaders * mHttpHeaders; + + typedef std::set wait_http_res_queue_t; + wait_http_res_queue_t mHttpWaitResource; // Mfnq public: // A probabilistically-correct indicator that the current diff --git a/indra/newview/lltextureview.cpp b/indra/newview/lltextureview.cpp index 1c89766b26..5f1d7829ed 100644 --- a/indra/newview/lltextureview.cpp +++ b/indra/newview/lltextureview.cpp @@ -235,13 +235,14 @@ void LLTextureBar::draw() { "NET", LLColor4::green }, // LOAD_FROM_NETWORK { "SIM", LLColor4::green }, // LOAD_FROM_SIMULATOR { "REQ", LLColor4::yellow },// SEND_HTTP_REQ + { "HTW", LLColor4::green }, // WAIT_HTTP_RES { "HTP", LLColor4::green }, // WAIT_HTTP_REQ { "DEC", LLColor4::yellow },// DECODE_IMAGE { "DEC", LLColor4::green }, // DECODE_IMAGE_UPDATE { "WRT", LLColor4::purple },// WRITE_TO_CACHE { "WRT", LLColor4::orange },// WAIT_ON_WRITE { "END", LLColor4::red }, // DONE -#define LAST_STATE 12 +#define LAST_STATE 13 { "CRE", LLColor4::magenta }, // LAST_STATE+1 { "FUL", LLColor4::green }, // LAST_STATE+2 { "BAD", LLColor4::red }, // LAST_STATE+3 -- cgit v1.3 From cab68bb04ed3c3734b48c703a43d41f231911647 Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Thu, 7 Jun 2012 18:10:32 -0400 Subject: First efforts at getting the texture cache working. This is now avoiding doing HTTP fetches for read data. Not certain it's completely correct but the difference is already significant. --- indra/newview/lltexturefetch.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'indra/newview') diff --git a/indra/newview/lltexturefetch.cpp b/indra/newview/lltexturefetch.cpp index 23232cb590..92b847345d 100644 --- a/indra/newview/lltexturefetch.cpp +++ b/indra/newview/lltexturefetch.cpp @@ -1584,7 +1584,7 @@ bool LLTextureFetchWorker::doWork(S32 param) if (mState == DONE) { - if (mDecodedDiscard >= 0 && mDesiredDiscard < mDecodedDiscard) + if (mDecodedDiscard > 0 && mDesiredDiscard < mDecodedDiscard) { // More data was requested, return to INIT mState = INIT; @@ -1883,7 +1883,7 @@ S32 LLTextureFetchWorker::callbackHttpGet(LLCore::HttpResponse * response, llassert_always(mDecodeHandle == 0); mFormattedImage = NULL; // discard any previous data we had } - else if (data_size < mRequestedSize && mRequestedDiscard == 0) + else if (data_size < mRequestedSize /*&& mRequestedDiscard == 0*/) { // *FIXME: I think we can treat this as complete regardless // of requested discard level. Revisit this... -- cgit v1.3 From 28a04400b4160dd34166483ddcf0c12637bcc363 Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Fri, 8 Jun 2012 20:21:54 -0400 Subject: Implemented HTTP retry for requests. Went in rather easily which surprised me. Added a retry queue similar to ready queue to the policy object which is sorted by retry time. Currently do five retries (after the initial try) delayed by .25, .5, 1, 2 and 5 seconds. Removed the retry logic from the lltexturefetch module. Upped the waiting time in the unit test for the retries. People won't like this but tough, need tests. --- indra/llcorehttp/_httplibcurl.cpp | 58 +++++++++-- indra/llcorehttp/_httplibcurl.h | 4 +- indra/llcorehttp/_httpoprequest.cpp | 51 ++++++++-- indra/llcorehttp/_httpoprequest.h | 5 + indra/llcorehttp/_httppolicy.cpp | 98 ++++++++++++++----- indra/llcorehttp/_httppolicy.h | 19 +++- indra/llcorehttp/_httpreadyqueue.h | 2 - indra/llcorehttp/_httpretryqueue.h | 94 ++++++++++++++++++ indra/llcorehttp/httpcommon.h | 12 +++ indra/llcorehttp/tests/test_httprequest.hpp | 2 +- indra/newview/lltexturefetch.cpp | 145 ++++++++++++++++++++-------- indra/newview/lltexturefetch.h | 6 +- 12 files changed, 402 insertions(+), 94 deletions(-) create mode 100644 indra/llcorehttp/_httpretryqueue.h (limited to 'indra/newview') diff --git a/indra/llcorehttp/_httplibcurl.cpp b/indra/llcorehttp/_httplibcurl.cpp index 5272c391e8..05b2c2be69 100644 --- a/indra/llcorehttp/_httplibcurl.cpp +++ b/indra/llcorehttp/_httplibcurl.cpp @@ -29,6 +29,7 @@ #include "httpheaders.h" #include "bufferarray.h" #include "_httpoprequest.h" +#include "_httppolicy.h" namespace LLCore @@ -85,6 +86,8 @@ void HttpLibcurl::term() HttpService::ELoopSpeed HttpLibcurl::processTransport() { + HttpService::ELoopSpeed ret(HttpService::REQUEST_SLEEP); + // Give libcurl some cycles to do I/O & callbacks for (int policy_class(0); policy_class < HttpRequest::POLICY_CLASS_LIMIT; ++policy_class) { @@ -110,7 +113,8 @@ HttpService::ELoopSpeed HttpLibcurl::processTransport() CURL * handle(msg->easy_handle); CURLcode result(msg->data.result); - completeRequest(mMultiHandles[policy_class], handle, result); + HttpService::ELoopSpeed speed(completeRequest(mMultiHandles[policy_class], handle, result)); + ret = (std::min)(ret, speed); handle = NULL; // No longer valid on return } else if (CURLMSG_NONE == msg->msg) @@ -127,7 +131,11 @@ HttpService::ELoopSpeed HttpLibcurl::processTransport() } } - return mActiveOps.empty() ? HttpService::REQUEST_SLEEP : HttpService::NORMAL; + if (! mActiveOps.empty()) + { + ret = (std::min)(ret, HttpService::NORMAL); + } + return ret; } @@ -153,8 +161,12 @@ void HttpLibcurl::addOp(HttpOpRequest * op) } -void HttpLibcurl::completeRequest(CURLM * multi_handle, CURL * handle, CURLcode status) +HttpService::ELoopSpeed HttpLibcurl::completeRequest(CURLM * multi_handle, CURL * handle, CURLcode status) { + static const HttpStatus cant_connect(HttpStatus::EXT_CURL_EASY, CURLE_COULDNT_CONNECT); + static const HttpStatus cant_res_proxy(HttpStatus::EXT_CURL_EASY, CURLE_COULDNT_RESOLVE_PROXY); + static const HttpStatus cant_res_host(HttpStatus::EXT_CURL_EASY, CURLE_COULDNT_RESOLVE_HOST); + HttpOpRequest * op(NULL); curl_easy_getinfo(handle, CURLINFO_PRIVATE, &op); // *FIXME: check the pointer @@ -190,10 +202,7 @@ void HttpLibcurl::completeRequest(CURLM * multi_handle, CURL * handle, CURLcode int http_status(200); curl_easy_getinfo(handle, CURLINFO_RESPONSE_CODE, &http_status); - op->mStatus = LLCore::HttpStatus(http_status, - (http_status >= 200 && http_status <= 299 - ? HE_SUCCESS - : HE_REPLY_ERROR)); + op->mStatus = LLCore::HttpStatus(http_status); } // Detach from multi and recycle handle @@ -201,9 +210,42 @@ void HttpLibcurl::completeRequest(CURLM * multi_handle, CURL * handle, CURLcode curl_easy_cleanup(handle); op->mCurlHandle = NULL; - // Deliver to reply queue and release + // Retry or finalize + if (! op->mStatus) + { + // If this failed, we might want to retry. Have to inspect + // the status a little more deeply for those reasons worth retrying... + if (op->mPolicyRetries < op->mPolicyRetryLimit && + ((op->mStatus.isHttpStatus() && op->mStatus.mType >= 499 && op->mStatus.mType <= 599) || + cant_connect == op->mStatus || + cant_res_proxy == op->mStatus || + cant_res_host == op->mStatus)) + { + // Okay, worth a retry. We include 499 in this test as + // it's the old 'who knows?' error from many grid services... + HttpPolicy & policy(mService->getPolicy()); + + policy.retryOp(op); + return HttpService::NORMAL; // Having pushed to retry, keep things running + } + } + + // This op is done, finalize it delivering it to the reply queue... + if (! op->mStatus) + { + LL_WARNS("CoreHttp") << "URL op failed after " << op->mPolicyRetries + << " retries. Reason: " << op->mStatus.toString() + << LL_ENDL; + } + else if (op->mPolicyRetries) + { + LL_WARNS("CoreHttp") << "URL op succeeded after " << op->mPolicyRetries << " retries." + << LL_ENDL; + } + op->stageFromActive(mService); op->release(); + return HttpService::REQUEST_SLEEP; } diff --git a/indra/llcorehttp/_httplibcurl.h b/indra/llcorehttp/_httplibcurl.h index ec325c1946..fe628b9ab0 100644 --- a/indra/llcorehttp/_httplibcurl.h +++ b/indra/llcorehttp/_httplibcurl.h @@ -83,7 +83,9 @@ public: protected: /// Invoked when libcurl has indicated a request has been processed /// to completion and we need to move the request to a new state. - void completeRequest(CURLM * multi_handle, CURL * handle, CURLcode status); + HttpService::ELoopSpeed completeRequest(CURLM * multi_handle, + CURL * handle, + CURLcode status); protected: typedef std::set active_set_t; diff --git a/indra/llcorehttp/_httpoprequest.cpp b/indra/llcorehttp/_httpoprequest.cpp index 4bdc4a5257..895629c514 100644 --- a/indra/llcorehttp/_httpoprequest.cpp +++ b/indra/llcorehttp/_httpoprequest.cpp @@ -99,8 +99,15 @@ HttpOpRequest::HttpOpRequest() mReplyBody(NULL), mReplyOffset(0), mReplyLength(0), - mReplyHeaders(NULL) -{} + mReplyHeaders(NULL), + mPolicyRetries(0), + mPolicyRetryAt(HttpTime(0)), + mPolicyRetryLimit(5) // *FIXME: Get from policy definitions +{ + // *NOTE: As members are added, retry initialization/cleanup + // may need to be extended in @prepareRequest(). +} + HttpOpRequest::~HttpOpRequest() @@ -130,7 +137,6 @@ HttpOpRequest::~HttpOpRequest() } mCurlService = NULL; - if (mCurlHeaders) { @@ -313,6 +319,30 @@ HttpStatus HttpOpRequest::setupPost(HttpRequest::policy_t policy_id, HttpStatus HttpOpRequest::prepareRequest(HttpService * service) { + // Scrub transport and result data for retried op case + mCurlActive = false; + mCurlHandle = NULL; + mCurlService = NULL; + if (mCurlHeaders) + { + curl_slist_free_all(mCurlHeaders); + mCurlHeaders = NULL; + } + mCurlBodyPos = 0; + + if (mReplyBody) + { + mReplyBody->release(); + mReplyBody = NULL; + } + mReplyOffset = 0; + mReplyLength = 0; + if (mReplyHeaders) + { + mReplyHeaders->release(); + mReplyHeaders = NULL; + } + // *FIXME: better error handling later HttpStatus status; @@ -321,6 +351,7 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service) mCurlHandle = curl_easy_init(); // curl_easy_setopt(mCurlHandle, CURLOPT_VERBOSE, 1); + curl_easy_setopt(mCurlHandle, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4); curl_easy_setopt(mCurlHandle, CURLOPT_TIMEOUT, 30); curl_easy_setopt(mCurlHandle, CURLOPT_CONNECTTIMEOUT, 30); curl_easy_setopt(mCurlHandle, CURLOPT_NOSIGNAL, 1); @@ -403,12 +434,7 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service) break; } - if (mReqHeaders) - { - mCurlHeaders = append_headers_to_slist(mReqHeaders, mCurlHeaders); - } - mCurlHeaders = curl_slist_append(mCurlHeaders, "Pragma:"); - + // There's a CURLOPT for this now... if ((mReqOffset || mReqLength) && HOR_GET == mReqMethod) { static const char * const fmt1("Range: bytes=%lu-%lu"); @@ -428,6 +454,13 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service) range_line[sizeof(range_line) - 1] = '\0'; mCurlHeaders = curl_slist_append(mCurlHeaders, range_line); } + + mCurlHeaders = curl_slist_append(mCurlHeaders, "Pragma:"); + if (mReqHeaders) + { + // Caller's headers last to override + mCurlHeaders = append_headers_to_slist(mReqHeaders, mCurlHeaders); + } curl_easy_setopt(mCurlHandle, CURLOPT_HTTPHEADER, mCurlHeaders); if (mProcFlags & (PF_SCAN_RANGE_HEADER | PF_SAVE_HEADERS)) diff --git a/indra/llcorehttp/_httpoprequest.h b/indra/llcorehttp/_httpoprequest.h index 0cad4e8459..6dcf30ca0c 100644 --- a/indra/llcorehttp/_httpoprequest.h +++ b/indra/llcorehttp/_httpoprequest.h @@ -128,6 +128,11 @@ public: off_t mReplyOffset; size_t mReplyLength; HttpHeaders * mReplyHeaders; + + // Policy data + int mPolicyRetries; + HttpTime mPolicyRetryAt; + const int mPolicyRetryLimit; }; // end class HttpOpRequest diff --git a/indra/llcorehttp/_httppolicy.cpp b/indra/llcorehttp/_httppolicy.cpp index 51f5e487dc..1f4cd34a4b 100644 --- a/indra/llcorehttp/_httppolicy.cpp +++ b/indra/llcorehttp/_httppolicy.cpp @@ -24,39 +24,46 @@ * $/LicenseInfo$ */ +#include "linden_common.h" + #include "_httppolicy.h" #include "_httpoprequest.h" #include "_httpservice.h" #include "_httplibcurl.h" +#include "lltimer.h" + namespace LLCore { HttpPolicy::HttpPolicy(HttpService * service) : mService(service) -{ - for (int policy_class(0); policy_class < HttpRequest::POLICY_CLASS_LIMIT; ++policy_class) - { - mReadyInClass[policy_class] = 0; - } -} +{} HttpPolicy::~HttpPolicy() { - for (int policy_class(0); policy_class < HttpRequest::POLICY_CLASS_LIMIT; ++policy_class) + for (int policy_class(0); policy_class < LL_ARRAY_SIZE(mState); ++policy_class) { - HttpReadyQueue & readyq(mReadyQueue[policy_class]); + HttpRetryQueue & retryq(mState[policy_class].mRetryQueue); + while (! retryq.empty()) + { + HttpOpRequest * op(retryq.top()); + op->cancel(); + op->release(); + retryq.pop(); + } + + HttpReadyQueue & readyq(mState[policy_class].mReadyQueue); while (! readyq.empty()) { HttpOpRequest * op(readyq.top()); op->cancel(); op->release(); - mReadyInClass[policy_class]--; readyq.pop(); } } @@ -68,27 +75,69 @@ void HttpPolicy::addOp(HttpOpRequest * op) { const int policy_class(op->mReqPolicy); - mReadyQueue[policy_class].push(op); - ++mReadyInClass[policy_class]; + op->mPolicyRetries = 0; + mState[policy_class].mReadyQueue.push(op); +} + + +void HttpPolicy::retryOp(HttpOpRequest * op) +{ + static const HttpTime retry_deltas[] = + { + 250000, // 1st retry in 0.25 S, etc... + 500000, + 1000000, + 2000000, + 5000000 // ... to every 5.0 S. + }; + static const int delta_max(int(LL_ARRAY_SIZE(retry_deltas)) - 1); + + const HttpTime now(totalTime()); + const int policy_class(op->mReqPolicy); + + const HttpTime delta(retry_deltas[llclamp(op->mPolicyRetries, 0, delta_max)]); + op->mPolicyRetryAt = now + delta; + ++op->mPolicyRetries; + LL_WARNS("CoreHttp") << "URL op retry #" << op->mPolicyRetries + << " being scheduled for " << delta << " uSecs from now." + << LL_ENDL; + mState[policy_class].mRetryQueue.push(op); } HttpService::ELoopSpeed HttpPolicy::processReadyQueue() { + const HttpTime now(totalTime()); HttpService::ELoopSpeed result(HttpService::REQUEST_SLEEP); HttpLibcurl & transport(mService->getTransport()); - for (int policy_class(0); policy_class < HttpRequest::POLICY_CLASS_LIMIT; ++policy_class) + for (int policy_class(0); policy_class < LL_ARRAY_SIZE(mState); ++policy_class) { - HttpReadyQueue & readyq(mReadyQueue[policy_class]); int active(transport.getActiveCountInClass(policy_class)); int needed(8 - active); - if (needed > 0 && mReadyInClass[policy_class] > 0) + HttpRetryQueue & retryq(mState[policy_class].mRetryQueue); + HttpReadyQueue & readyq(mState[policy_class].mReadyQueue); + + if (needed > 0) { - // Scan ready queue for requests that match policy - - while (! readyq.empty() && needed > 0 && mReadyInClass[policy_class] > 0) + // First see if we have any retries... + while (needed > 0 && ! retryq.empty()) + { + HttpOpRequest * op(retryq.top()); + if (op->mPolicyRetryAt > now) + break; + + retryq.pop(); + + op->stageFromReady(mService); + op->release(); + + --needed; + } + + // Now go on to the new requests... + while (needed > 0 && ! readyq.empty()) { HttpOpRequest * op(readyq.top()); readyq.pop(); @@ -96,17 +145,16 @@ HttpService::ELoopSpeed HttpPolicy::processReadyQueue() op->stageFromReady(mService); op->release(); - --mReadyInClass[policy_class]; --needed; } } - - if (! readyq.empty()) + + if (! readyq.empty() || ! retryq.empty()) { // If anything is ready, continue looping... result = (std::min)(result, HttpService::NORMAL); } - } + } // end foreach policy_class return result; } @@ -114,9 +162,9 @@ HttpService::ELoopSpeed HttpPolicy::processReadyQueue() bool HttpPolicy::changePriority(HttpHandle handle, HttpRequest::priority_t priority) { - for (int policy_class(0); policy_class < HttpRequest::POLICY_CLASS_LIMIT; ++policy_class) + for (int policy_class(0); policy_class < LL_ARRAY_SIZE(mState); ++policy_class) { - HttpReadyQueue::container_type & c(mReadyQueue[policy_class].get_container()); + HttpReadyQueue::container_type & c(mState[policy_class].mReadyQueue.get_container()); // Scan ready queue for requests that match policy for (HttpReadyQueue::container_type::iterator iter(c.begin()); c.end() != iter;) @@ -126,9 +174,9 @@ bool HttpPolicy::changePriority(HttpHandle handle, HttpRequest::priority_t prior if (static_cast(*cur) == handle) { HttpOpRequest * op(*cur); - c.erase(cur); // All iterators are now invalidated + c.erase(cur); // All iterators are now invalidated op->mReqPriority = priority; - mReadyQueue[policy_class].push(op); // Re-insert using adapter class + mState[policy_class].mReadyQueue.push(op); // Re-insert using adapter class return true; } } diff --git a/indra/llcorehttp/_httppolicy.h b/indra/llcorehttp/_httppolicy.h index 425079ec63..6f18264f3d 100644 --- a/indra/llcorehttp/_httppolicy.h +++ b/indra/llcorehttp/_httppolicy.h @@ -31,6 +31,7 @@ #include "httprequest.h" #include "_httpservice.h" #include "_httpreadyqueue.h" +#include "_httpretryqueue.h" #include "_httppolicyglobal.h" @@ -67,6 +68,14 @@ public: /// additional references will be added.) void addOp(HttpOpRequest *); + /// Similar to addOp, used when a caller wants to retry a + /// request that has failed. It's placed on a special retry + /// queue but ordered by retry time not priority. Otherwise, + /// handling is the same and retried operations are considered + /// before new ones but that doesn't guarantee completion + /// order. + void retryOp(HttpOpRequest *); + // Shadows HttpService's method bool changePriority(HttpHandle handle, HttpRequest::priority_t priority); @@ -77,10 +86,14 @@ public: return mGlobalOptions; } - protected: - int mReadyInClass[HttpRequest::POLICY_CLASS_LIMIT]; - HttpReadyQueue mReadyQueue[HttpRequest::POLICY_CLASS_LIMIT]; + struct State + { + HttpReadyQueue mReadyQueue; + HttpRetryQueue mRetryQueue; + }; + + State mState[HttpRequest::POLICY_CLASS_LIMIT]; HttpService * mService; // Naked pointer, not refcounted, not owner HttpPolicyGlobal mGlobalOptions; diff --git a/indra/llcorehttp/_httpreadyqueue.h b/indra/llcorehttp/_httpreadyqueue.h index 2cd96aefe3..87828834dc 100644 --- a/indra/llcorehttp/_httpreadyqueue.h +++ b/indra/llcorehttp/_httpreadyqueue.h @@ -36,8 +36,6 @@ namespace LLCore { -class HttpOpRequest; - /// HttpReadyQueue provides a simple priority queue for HttpOpRequest objects. /// /// This uses the priority_queue adaptor class to provide the queue diff --git a/indra/llcorehttp/_httpretryqueue.h b/indra/llcorehttp/_httpretryqueue.h new file mode 100644 index 0000000000..745adec09d --- /dev/null +++ b/indra/llcorehttp/_httpretryqueue.h @@ -0,0 +1,94 @@ +/** + * @file _httpretryqueue.h + * @brief Internal declaration for the operation retry queue + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef _LLCORE_HTTP_RETRY_QUEUE_H_ +#define _LLCORE_HTTP_RETRY_QUEUE_H_ + + +#include + +#include "_httpoprequest.h" + + +namespace LLCore +{ + +/// HttpRetryQueue provides a simple priority queue for HttpOpRequest objects. +/// +/// This uses the priority_queue adaptor class to provide the queue +/// as well as the ordering scheme while allowing us access to the +/// raw container if we follow a few simple rules. One of the more +/// important of those rules is that any iterator becomes invalid +/// on element erasure. So pay attention. +/// +/// Threading: not thread-safe. Expected to be used entirely by +/// a single thread, typically a worker thread of some sort. + +struct HttpOpRetryCompare +{ + bool operator()(const HttpOpRequest * lhs, const HttpOpRequest * rhs) + { + return lhs->mPolicyRetryAt < rhs->mPolicyRetryAt; + } +}; + + +typedef std::priority_queue, + LLCore::HttpOpRetryCompare> HttpRetryQueueBase; + +class HttpRetryQueue : public HttpRetryQueueBase +{ +public: + HttpRetryQueue() + : HttpRetryQueueBase() + {} + + ~HttpRetryQueue() + {} + +protected: + HttpRetryQueue(const HttpRetryQueue &); // Not defined + void operator=(const HttpRetryQueue &); // Not defined + +public: + const container_type & get_container() const + { + return c; + } + + container_type & get_container() + { + return c; + } + +}; // end class HttpRetryQueue + + +} // end namespace LLCore + + +#endif // _LLCORE_HTTP_RETRY_QUEUE_H_ diff --git a/indra/llcorehttp/httpcommon.h b/indra/llcorehttp/httpcommon.h index fd2661b700..42b75edb41 100644 --- a/indra/llcorehttp/httpcommon.h +++ b/indra/llcorehttp/httpcommon.h @@ -114,6 +114,9 @@ namespace LLCore typedef void * HttpHandle; #define LLCORE_HTTP_HANDLE_INVALID (NULL) +/// For internal scheduling and metrics, we use a microsecond +/// timebase compatible with the environment. +typedef U64 HttpTime; /// Error codes defined by the library itself as distinct from /// libcurl (or any other transport provider). @@ -180,6 +183,15 @@ struct HttpStatus mStatus(status) {} + HttpStatus(int http_status) + : mType(http_status), + mStatus(http_status >= 200 && http_status <= 299 + ? HE_SUCCESS + : HE_REPLY_ERROR) + { + llassert(http_status >= 100 && http_status <= 999); + } + HttpStatus(const HttpStatus & rhs) : mType(rhs.mType), mStatus(rhs.mStatus) diff --git a/indra/llcorehttp/tests/test_httprequest.hpp b/indra/llcorehttp/tests/test_httprequest.hpp index 0e9d7d8979..2d91b95347 100644 --- a/indra/llcorehttp/tests/test_httprequest.hpp +++ b/indra/llcorehttp/tests/test_httprequest.hpp @@ -381,7 +381,7 @@ void HttpRequestTestObjectType::test<5>() // Run the notification pump. int count(0); - int limit(20); + int limit(180); // With retries, can take more than 10 seconds to give up while (count++ < limit && mHandlerCalls < 1) { req->update(1000); diff --git a/indra/newview/lltexturefetch.cpp b/indra/newview/lltexturefetch.cpp index 92b847345d..4a46ea0e97 100644 --- a/indra/newview/lltexturefetch.cpp +++ b/indra/newview/lltexturefetch.cpp @@ -67,9 +67,53 @@ // This is an attempt to document what's going on in here after-the-fact. // It's a sincere attempt to be accurate but there will be mistakes. // +// // Purpose // -// (What is this solving?) +// What is this module trying to do? It accepts requests to load textures +// at a given priority and discard level and notifies the caller when done +// (successfully or not). Additional constraints are: +// +// * Support a local texture cache. Don't hit network when possible +// to avoid it. +// * Use UDP or HTTP as directed or as fallback. HTTP is tried when +// not disabled and a URL is available. UDP when a URL isn't +// available or HTTP attempts fail. +// * Asynchronous (using threads). Main thread is not to be blocked or +// burdened. +// * High concurrency. Many requests need to be in-flight and at various +// stages of completion. +// * Tolerate frequent re-prioritizations of requests. Priority is +// a reflection of a camera's viewpoint and as that viewpoint changes, +// objects and textures become more and less relevant and that is +// expressed at this level by priority changes and request cancelations. +// +// The caller interfaces that fall out of the above and shape the +// implementation are: +// * createRequest - Load j2c image via UDP or HTTP at given discard level and priority +// * deleteRequest - Request removal of prior request +// * getRequestFinished - Test if request is finished returning data to caller +// * updateRequestPriority - Change priority of existing request +// * getFetchState - Retrieve progress on existing request +// +// Everything else in here is mostly plumbing, metrics and debug. +// +// +// The Work Queue +// +// The two central classes are LLTextureFetch and LLTextureFetchWorker. +// LLTextureFetch combines threading with a priority queue of work +// requests. The priority queue is sorted by a U32 priority derived +// from the F32 priority in the APIs. The *only* work request that +// receives service time by this thread is the highest priority +// request. All others wait until it is complete or a dynamic priority +// change has re-ordered work. +// +// LLTextureFetchWorker implements the work request and is 1:1 with +// texture fetch requests. Embedded in each is a state machine that +// walks it through the cache, HTTP, UDP, image decode and retry +// steps of texture acquisition. +// // // Threads // @@ -83,6 +127,7 @@ // 5. Tid Image decoder's worker thread // 6. Thl HTTP library's worker thread // +// // Mutexes/Condition Variables // // 1. Mt Mutex defined for LLThread's condition variable (base class of @@ -98,6 +143,7 @@ // LLTextureFetchWorker). One per request. // 7. Mw LLTextureFetchWorker's mutex. One per request. // +// // Lock Ordering Rules // // Not an exhaustive list but shows the order of lock acquisition @@ -105,6 +151,8 @@ // acquiring 'B'. // // 1. Mw < Mfnq +// (there are many more...) +// // // Method and Member Definitions // @@ -124,7 +172,10 @@ // comment can mean the member is unlocked or that I didn't bother // to do the archaeology. In the case of LLTextureFetchWorker, // most data members added by the leaf class are actually covered -// by the Mw lock. +// by the Mw lock. You may also see "// T" which means that +// the member's usage is restricted to one thread (except for +// perhaps construction and destruction) and so explicit locking +// isn't used. // // In code, a trailing comment like "// [-+]M" indicates a // lock acquision or release point. @@ -132,27 +183,54 @@ // // Worker Lifecycle // -// (Can't unilaterally delete, cleanup is two-phase, etc.) +// The threading and responder model makes it very likely that +// other components are holding on to a pointer to a worker request. +// So, uncoordinated deletions of requests is a guarantee of memory +// corruption in a short time. So destroying a request involves +// invocations's of LLQueuedThread/LLWorkerThread's abort/stop +// logic that removes workers and puts them ona delete queue for +// 2-phase destruction. That second phase is deferrable by calls +// to deleteOK() which only allow final destruction (via dtor) +// once deleteOK has determined that the request is in a safe +// state. +// // // Worker State Machine // // (ASCII art needed) // +// // Priority Scheme // // [PRIORITY_LOW, PRIORITY_NORMAL) - for WAIT_HTTP_RESOURCE state // [PRIORITY_NORMAL, PRIORITY_HIGH) - waiting for external event -// [PRIORITY_HIGH, PRIORITY_URGENT) - rapidly transitioning through states, +// [PRIORITY_HIGH, PRIORITY_URGENT) - External event delivered, +// rapidly transitioning through states, // no waiting allowed // +// By itself, the above work queue model would fail the concurrency +// and liveness requirements of the interface. A high priority +// request could find itself on the head and stalled for external +// reasons (see VWR-28996). So a few additional constraints are +// required to keep things running: +// * Anything that can make forward progress must be kept at a +// higher priority than anything that can't. +// * On completion of external events, the associated request +// needs to be elevated beyond the normal range to handle +// any data delivery and release any external resource. +// +// This effort is made to keep higher-priority entities moving +// forward in their state machines at every possible step of +// processing. It's not entirely proven that this produces the +// experiencial benefits promised. // ////////////////////////////////////////////////////////////////////////////// // Tuning/Parameterization Constants -static const S32 HTTP_REQUESTS_IN_QUEUE_HIGH_WATER = 40; -static const S32 HTTP_REQUESTS_IN_QUEUE_LOW_WATER = 20; +static const S32 HTTP_REQUESTS_IN_QUEUE_HIGH_WATER = 40; // Maximum requests to have active in HTTP +static const S32 HTTP_REQUESTS_IN_QUEUE_LOW_WATER = 20; // Active level at which to refill ////////////////////////////////////////////////////////////////////////////// @@ -425,7 +503,6 @@ private: BOOL mInLocalCache; bool mCanUseHTTP ; bool mCanUseNET ; //can get from asset server. - S32 mHTTPFailCount; S32 mRetryAttempt; S32 mActiveCount; LLCore::HttpStatus mGetStatus; @@ -745,7 +822,6 @@ LLTextureFetchWorker::LLTextureFetchWorker(LLTextureFetch* fetcher, mHaveAllData(FALSE), mInLocalCache(FALSE), mCanUseHTTP(true), - mHTTPFailCount(0), mRetryAttempt(0), mActiveCount(0), mWorkMutex(NULL), @@ -936,6 +1012,9 @@ void LLTextureFetchWorker::startWork(S32 param) // Threads: Ttf bool LLTextureFetchWorker::doWork(S32 param) { + static const LLCore::HttpStatus http_not_found(HTTP_NOT_FOUND); + static const LLCore::HttpStatus http_service_unavail(HTTP_SERVICE_UNAVAILABLE); + // Release waiters while we aren't holding the Mw lock. mFetcher->releaseHttpWaiters(); @@ -1286,7 +1365,6 @@ bool LLTextureFetchWorker::doWork(S32 param) { llwarns << "HTTP GET request failed for " << mID << llendl; resetFormattedData(); - ++mHTTPFailCount; return true; // failed } @@ -1313,10 +1391,8 @@ bool LLTextureFetchWorker::doWork(S32 param) S32 cur_size = mFormattedImage.notNull() ? mFormattedImage->getDataSize() : 0; if (mRequestedSize < 0) { - S32 max_attempts; - if (mGetStatus == LLCore::HttpStatus(HTTP_NOT_FOUND, LLCore::HE_REPLY_ERROR)) + if (http_not_found == mGetStatus) { - mHTTPFailCount = max_attempts = 1; // Don't retry llwarns << "Texture missing from server (404): " << mUrl << llendl; // roll back to try UDP @@ -1328,47 +1404,32 @@ bool LLTextureFetchWorker::doWork(S32 param) return false; } } - else if (mGetStatus == LLCore::HttpStatus(HTTP_SERVICE_UNAVAILABLE, LLCore::HE_REPLY_ERROR)) + else if (http_service_unavail == mGetStatus) { - // *TODO: Should probably introduce a timer here to delay future HTTP requsts - // for a short time (~1s) to ease server load? Ideally the server would queue - // requests instead of returning 503... we already limit the number pending. - ++mHTTPFailCount; - max_attempts = mHTTPFailCount+1; // Keep retrying LL_INFOS_ONCE("Texture") << "Texture server busy (503): " << mUrl << LL_ENDL; } else { - const S32 HTTP_MAX_RETRY_COUNT = 3; - max_attempts = HTTP_MAX_RETRY_COUNT + 1; - ++mHTTPFailCount; llinfos << "HTTP GET failed for: " << mUrl << " Status: " << mGetStatus.toHex() << " Reason: '" << mGetReason << "'" - << " Attempt:" << mHTTPFailCount+1 << "/" << max_attempts << llendl; + // *FIXME: Add retry info for reporting purposes... + // << " Attempt:" << mHTTPFailCount+1 << "/" << max_attempts + << llendl; } - if (mHTTPFailCount >= max_attempts) - { - if (cur_size > 0) - { - // Use available data - mLoadedDiscard = mFormattedImage->getDiscardLevel(); - mState = DECODE_IMAGE; - return false; - } - else - { - resetFormattedData(); - mState = DONE; - return true; // failed - } - } - else + if (cur_size > 0) { - mState = SEND_HTTP_REQ; - return false; // retry + // Use available data + mLoadedDiscard = mFormattedImage->getDiscardLevel(); + mState = DECODE_IMAGE; + return false; } + + // Fail harder + resetFormattedData(); + mState = DONE; + return true; // failed } if (! mHttpBufferArray || ! mHttpBufferArray->size()) @@ -1649,7 +1710,7 @@ void LLTextureFetchWorker::onCompleted(LLCore::HttpHandle handle, LLCore::HttpRe } else { - static const LLCore::HttpStatus par_status(LLCore::HttpStatus(HTTP_PARTIAL_CONTENT, LLCore::HE_SUCCESS)); + static const LLCore::HttpStatus par_status(HTTP_PARTIAL_CONTENT); partial = (par_status == status); } diff --git a/indra/newview/lltexturefetch.h b/indra/newview/lltexturefetch.h index 53b0f7885f..4ee13d171e 100644 --- a/indra/newview/lltexturefetch.h +++ b/indra/newview/lltexturefetch.h @@ -308,9 +308,9 @@ private: // Interfaces and objects into the core http library used // to make our HTTP requests. These replace the various // LLCurl interfaces used in the past. - LLCore::HttpRequest * mHttpRequest; - LLCore::HttpOptions * mHttpOptions; - LLCore::HttpHeaders * mHttpHeaders; + LLCore::HttpRequest * mHttpRequest; // Ttf + LLCore::HttpOptions * mHttpOptions; // Ttf + LLCore::HttpHeaders * mHttpHeaders; // Ttf typedef std::set wait_http_res_queue_t; wait_http_res_queue_t mHttpWaitResource; // Mfnq -- cgit v1.3 From 89187229dd630845177ecd7a16e2b9cb01dc47ce Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Mon, 11 Jun 2012 15:28:06 -0400 Subject: Refactoring of the request completion thread and removal of 206/content-range hack in xport. Retry/response handling is decided in policy so moved that there. Removed special case 206-without-content-range response in transport. Have this sitation recognizable in the API and let callers deal with it as needed. --- indra/llcorehttp/_httplibcurl.cpp | 51 +++++++------------------------------ indra/llcorehttp/_httplibcurl.h | 4 +-- indra/llcorehttp/_httpoprequest.cpp | 19 +++----------- indra/llcorehttp/_httppolicy.cpp | 42 ++++++++++++++++++++++++++++++ indra/llcorehttp/_httppolicy.h | 10 ++++++++ indra/newview/lltexturefetch.cpp | 9 +++++-- 6 files changed, 72 insertions(+), 63 deletions(-) (limited to 'indra/newview') diff --git a/indra/llcorehttp/_httplibcurl.cpp b/indra/llcorehttp/_httplibcurl.cpp index 332b6f3856..e134a28401 100644 --- a/indra/llcorehttp/_httplibcurl.cpp +++ b/indra/llcorehttp/_httplibcurl.cpp @@ -113,8 +113,11 @@ HttpService::ELoopSpeed HttpLibcurl::processTransport() CURL * handle(msg->easy_handle); CURLcode result(msg->data.result); - HttpService::ELoopSpeed speed(completeRequest(mMultiHandles[policy_class], handle, result)); - ret = (std::min)(ret, speed); + if (completeRequest(mMultiHandles[policy_class], handle, result)) + { + // Request is still active, don't get too sleepy + ret = (std::min)(ret, HttpService::NORMAL); + } handle = NULL; // No longer valid on return } else if (CURLMSG_NONE == msg->msg) @@ -161,12 +164,8 @@ void HttpLibcurl::addOp(HttpOpRequest * op) } -HttpService::ELoopSpeed HttpLibcurl::completeRequest(CURLM * multi_handle, CURL * handle, CURLcode status) +bool HttpLibcurl::completeRequest(CURLM * multi_handle, CURL * handle, CURLcode status) { - static const HttpStatus cant_connect(HttpStatus::EXT_CURL_EASY, CURLE_COULDNT_CONNECT); - static const HttpStatus cant_res_proxy(HttpStatus::EXT_CURL_EASY, CURLE_COULDNT_RESOLVE_PROXY); - static const HttpStatus cant_res_host(HttpStatus::EXT_CURL_EASY, CURLE_COULDNT_RESOLVE_HOST); - HttpOpRequest * op(NULL); curl_easy_getinfo(handle, CURLINFO_PRIVATE, &op); // *FIXME: check the pointer @@ -209,43 +208,11 @@ HttpService::ELoopSpeed HttpLibcurl::completeRequest(CURLM * multi_handle, CURL curl_multi_remove_handle(multi_handle, handle); curl_easy_cleanup(handle); op->mCurlHandle = NULL; - - // Retry or finalize - if (! op->mStatus) - { - // If this failed, we might want to retry. Have to inspect - // the status a little more deeply for those reasons worth retrying... - if (op->mPolicyRetries < op->mPolicyRetryLimit && - ((op->mStatus.isHttpStatus() && op->mStatus.mType >= 499 && op->mStatus.mType <= 599) || - cant_connect == op->mStatus || - cant_res_proxy == op->mStatus || - cant_res_host == op->mStatus)) - { - // Okay, worth a retry. We include 499 in this test as - // it's the old 'who knows?' error from many grid services... - HttpPolicy & policy(mService->getPolicy()); - - policy.retryOp(op); - return HttpService::NORMAL; // Having pushed to retry, keep things running - } - } - // This op is done, finalize it delivering it to the reply queue... - if (! op->mStatus) - { - LL_WARNS("CoreHttp") << "URL op failed after " << op->mPolicyRetries - << " retries. Reason: " << op->mStatus.toString() - << LL_ENDL; - } - else if (op->mPolicyRetries) - { - LL_WARNS("CoreHttp") << "URL op succeeded after " << op->mPolicyRetries << " retries." - << LL_ENDL; - } + HttpPolicy & policy(mService->getPolicy()); + bool still_active(policy.stageAfterCompletion(op)); - op->stageFromActive(mService); - op->release(); - return HttpService::REQUEST_SLEEP; + return still_active; } diff --git a/indra/llcorehttp/_httplibcurl.h b/indra/llcorehttp/_httplibcurl.h index fe628b9ab0..16b68bde43 100644 --- a/indra/llcorehttp/_httplibcurl.h +++ b/indra/llcorehttp/_httplibcurl.h @@ -83,9 +83,7 @@ public: protected: /// Invoked when libcurl has indicated a request has been processed /// to completion and we need to move the request to a new state. - HttpService::ELoopSpeed completeRequest(CURLM * multi_handle, - CURL * handle, - CURLcode status); + bool completeRequest(CURLM * multi_handle, CURL * handle, CURLcode status); protected: typedef std::set active_set_t; diff --git a/indra/llcorehttp/_httpoprequest.cpp b/indra/llcorehttp/_httpoprequest.cpp index 895629c514..ea0b99303e 100644 --- a/indra/llcorehttp/_httpoprequest.cpp +++ b/indra/llcorehttp/_httpoprequest.cpp @@ -144,6 +144,8 @@ HttpOpRequest::~HttpOpRequest() mCurlHeaders = NULL; } + mReplyOffset = 0; + mReplyLength = 0; if (mReplyBody) { mReplyBody->release(); @@ -211,26 +213,11 @@ void HttpOpRequest::visitNotifier(HttpRequest * request) response->setStatus(mStatus); response->setBody(mReplyBody); response->setHeaders(mReplyHeaders); - unsigned int offset(0), length(0); if (mReplyOffset || mReplyLength) { // Got an explicit offset/length in response - offset = mReplyOffset; - length = mReplyLength; - } - else if (mReplyBody && partial_content == mStatus) - { - // Legacy grid services did not provide a 'Content-Range' - // header in responses to full- or partly-satisfyiable - // 'Range' requests. For these, we have to hope that - // the data starts where requested and the length is simply - // whatever we received. A bit of sanity could be provided - // by overlapping ranged requests and verifying that the - // overlap matches. - offset = mReqOffset; - length = mReplyBody->size(); + response->setRange(mReplyOffset, mReplyLength); } - response->setRange(offset, length); mLibraryHandler->onCompleted(static_cast(this), response); diff --git a/indra/llcorehttp/_httppolicy.cpp b/indra/llcorehttp/_httppolicy.cpp index 1f4cd34a4b..72bb6f14e4 100644 --- a/indra/llcorehttp/_httppolicy.cpp +++ b/indra/llcorehttp/_httppolicy.cpp @@ -185,5 +185,47 @@ bool HttpPolicy::changePriority(HttpHandle handle, HttpRequest::priority_t prior return false; } +bool HttpPolicy::stageAfterCompletion(HttpOpRequest * op) +{ + static const HttpStatus cant_connect(HttpStatus::EXT_CURL_EASY, CURLE_COULDNT_CONNECT); + static const HttpStatus cant_res_proxy(HttpStatus::EXT_CURL_EASY, CURLE_COULDNT_RESOLVE_PROXY); + static const HttpStatus cant_res_host(HttpStatus::EXT_CURL_EASY, CURLE_COULDNT_RESOLVE_HOST); + + // Retry or finalize + if (! op->mStatus) + { + // If this failed, we might want to retry. Have to inspect + // the status a little more deeply for those reasons worth retrying... + if (op->mPolicyRetries < op->mPolicyRetryLimit && + ((op->mStatus.isHttpStatus() && op->mStatus.mType >= 499 && op->mStatus.mType <= 599) || + cant_connect == op->mStatus || + cant_res_proxy == op->mStatus || + cant_res_host == op->mStatus)) + { + // Okay, worth a retry. We include 499 in this test as + // it's the old 'who knows?' error from many grid services... + retryOp(op); + return true; // still active/ready + } + } + + // This op is done, finalize it delivering it to the reply queue... + if (! op->mStatus) + { + LL_WARNS("CoreHttp") << "URL op failed after " << op->mPolicyRetries + << " retries. Reason: " << op->mStatus.toString() + << LL_ENDL; + } + else if (op->mPolicyRetries) + { + LL_WARNS("CoreHttp") << "URL op succeeded after " << op->mPolicyRetries << " retries." + << LL_ENDL; + } + + op->stageFromActive(mService); + op->release(); + return false; // not active +} + } // end namespace LLCore diff --git a/indra/llcorehttp/_httppolicy.h b/indra/llcorehttp/_httppolicy.h index 6f18264f3d..14f6a9a676 100644 --- a/indra/llcorehttp/_httppolicy.h +++ b/indra/llcorehttp/_httppolicy.h @@ -79,6 +79,16 @@ public: // Shadows HttpService's method bool changePriority(HttpHandle handle, HttpRequest::priority_t priority); + /// When transport is finished with an op and takes it off the + /// active queue, it is delivered here for dispatch. Policy + /// may send it back to the ready/retry queues if it needs another + /// go or we may finalize it and send it on to the reply queue. + /// + /// @return Returns true of the request is still active + /// or ready after staging, false if has been + /// sent on to the reply queue. + bool stageAfterCompletion(HttpOpRequest * op); + // Get pointer to global policy options. Caller is expected // to do context checks like no setting once running. HttpPolicyGlobal & getGlobalOptions() diff --git a/indra/newview/lltexturefetch.cpp b/indra/newview/lltexturefetch.cpp index 4a46ea0e97..571f9ab3b5 100644 --- a/indra/newview/lltexturefetch.cpp +++ b/indra/newview/lltexturefetch.cpp @@ -1692,8 +1692,8 @@ void LLTextureFetchWorker::onCompleted(LLCore::HttpHandle handle, LLCore::HttpRe << " status: " << status.toHex() << " '" << status.toString() << "'" << llendl; - unsigned int offset(0), length(0); - response->getRange(&offset, &length); +// unsigned int offset(0), length(0); +// response->getRange(&offset, &length); // llwarns << "HTTP COMPLETE: " << mID << " handle: " << handle // << " status: " << status.toULong() << " '" << status.toString() << "'" // << " req offset: " << mRequestedOffset << " req length: " << mRequestedSize @@ -1710,6 +1710,11 @@ void LLTextureFetchWorker::onCompleted(LLCore::HttpHandle handle, LLCore::HttpRe } else { + // A warning about partial (HTTP 206) data. Some grid services + // do *not* return a 'Content-Range' header in the response to + // Range requests with a 206 status. We're forced to assume + // we get what we asked for in these cases until we can fix + // the services. static const LLCore::HttpStatus par_status(HTTP_PARTIAL_CONTENT); partial = (par_status == status); -- cgit v1.3 From 7adeb3923728ca84a309a6af141c148ce38066fc Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Tue, 12 Jun 2012 17:42:33 -0400 Subject: HTTP Proxy, PUT & POST, unit tests and refactoring. Implemented/modified PUT & POST to not used chunked encoding for the request. Made the unit test much happier and probably a better thing for the pipeline. Have a cheesy static & dynamic proxy capability using both local options and a way to wire into LLProxy in llmessages. Not a clean thing but it will get the proxy path working with both socks5 & http proxies. Refactoring to get rid of unneeded library handler and unified an HttpStatus return for all requests. Big batch of code removed as a result of that and more is possible as well as some syscall avoidance with a bit more work. Boosted the unit tests for simple PUT & POST test which revealed the test harness does *not* like chunked encoding so we'll avoid it for now (and don't really need it in any of our schemes). --- indra/llcorehttp/CMakeLists.txt | 5 + indra/llcorehttp/_httpopcancel.cpp | 11 -- indra/llcorehttp/_httpopcancel.h | 3 - indra/llcorehttp/_httpoperation.cpp | 18 +- indra/llcorehttp/_httpoperation.h | 13 +- indra/llcorehttp/_httpoprequest.cpp | 63 +++++-- indra/llcorehttp/_httpoprequest.h | 7 + indra/llcorehttp/_httpopsetget.cpp | 105 +++++++++++ indra/llcorehttp/_httpopsetget.h | 78 ++++++++ indra/llcorehttp/_httpopsetpriority.cpp | 14 -- indra/llcorehttp/_httpopsetpriority.h | 4 +- indra/llcorehttp/_httppolicy.cpp | 6 + indra/llcorehttp/_httppolicy.h | 4 +- indra/llcorehttp/_httppolicyglobal.cpp | 38 ++-- indra/llcorehttp/_httppolicyglobal.h | 7 +- indra/llcorehttp/_httpservice.cpp | 11 +- indra/llcorehttp/_httpservice.h | 10 +- indra/llcorehttp/httprequest.cpp | 151 ++++++++-------- indra/llcorehttp/httprequest.h | 44 ++++- indra/llcorehttp/tests/llcorehttp_test.cpp | 6 + indra/llcorehttp/tests/test_httpoperation.hpp | 5 +- indra/llcorehttp/tests/test_httprequest.hpp | 239 +++++++++++++++++++++++++ indra/llcorehttp/tests/test_llcorehttp_peer.py | 10 +- indra/newview/llappviewer.cpp | 13 ++ 24 files changed, 702 insertions(+), 163 deletions(-) create mode 100644 indra/llcorehttp/_httpopsetget.cpp create mode 100644 indra/llcorehttp/_httpopsetget.h (limited to 'indra/newview') diff --git a/indra/llcorehttp/CMakeLists.txt b/indra/llcorehttp/CMakeLists.txt index a0827286e3..4273b32fe3 100644 --- a/indra/llcorehttp/CMakeLists.txt +++ b/indra/llcorehttp/CMakeLists.txt @@ -10,12 +10,14 @@ include(OpenSSL) include(ZLIB) include(LLCoreHttp) include(LLAddBuildTest) +include(LLMessage) include(LLCommon) include(Tut) include_directories (${CMAKE_CURRENT_SOURCE_DIR}) include_directories( + ${LLMESSAGE_INCLUDE_DIRS} ${LLCOMMON_INCLUDE_DIRS} ${LLCOREHTTP_INCLUDE_DIRS} ) @@ -31,6 +33,7 @@ set(llcorehttp_SOURCE_FILES _httpopcancel.cpp _httpoperation.cpp _httpoprequest.cpp + _httpopsetget.cpp _httpopsetpriority.cpp _httppolicy.cpp _httppolicyglobal.cpp @@ -54,6 +57,7 @@ set(llcorehttp_HEADER_FILES _httpopcancel.h _httpoperation.h _httpoprequest.h + _httpopsetget.h _httpopsetpriority.h _httppolicy.h _httppolicyglobal.h @@ -113,6 +117,7 @@ if (LL_TESTS) set(test_libs ${LLCOREHTTP_LIBRARIES} ${WINDOWS_LIBRARIES} + ${LLMESSAGE_LIBRARIES} ${LLCOMMON_LIBRARIES} ${GOOGLEMOCK_LIBRARIES} ${CURL_LIBRARIES} diff --git a/indra/llcorehttp/_httpopcancel.cpp b/indra/llcorehttp/_httpopcancel.cpp index 69dbff4bb4..ad624d2e57 100644 --- a/indra/llcorehttp/_httpopcancel.cpp +++ b/indra/llcorehttp/_httpopcancel.cpp @@ -66,17 +66,6 @@ void HttpOpCancel::stageFromRequest(HttpService * service) } -void HttpOpCancel::visitNotifier(HttpRequest * request) -{ - if (mLibraryHandler) - { - HttpResponse * response = new HttpResponse(); - mLibraryHandler->onCompleted(static_cast(this), response); - response->release(); - } -} - - } // end namespace LLCore diff --git a/indra/llcorehttp/_httpopcancel.h b/indra/llcorehttp/_httpopcancel.h index fab6f1f362..6d1e0f8774 100644 --- a/indra/llcorehttp/_httpopcancel.h +++ b/indra/llcorehttp/_httpopcancel.h @@ -59,13 +59,10 @@ private: public: virtual void stageFromRequest(HttpService *); - - virtual void visitNotifier(HttpRequest * request); public: // Request data HttpHandle mHandle; - }; // end class HttpOpCancel diff --git a/indra/llcorehttp/_httpoperation.cpp b/indra/llcorehttp/_httpoperation.cpp index d966efd12b..b5c58013d4 100644 --- a/indra/llcorehttp/_httpoperation.cpp +++ b/indra/llcorehttp/_httpoperation.cpp @@ -47,7 +47,6 @@ namespace LLCore HttpOperation::HttpOperation() : LLCoreInt::RefCounted(true), mReplyQueue(NULL), - mLibraryHandler(NULL), mUserHandler(NULL), mReqPolicy(HttpRequest::DEFAULT_POLICY_ID), mReqPriority(0U) @@ -57,13 +56,12 @@ HttpOperation::HttpOperation() HttpOperation::~HttpOperation() { - setHandlers(NULL, NULL, NULL); + setReplyPath(NULL, NULL); } -void HttpOperation::setHandlers(HttpReplyQueue * reply_queue, - HttpHandler * lib_handler, - HttpHandler * user_handler) +void HttpOperation::setReplyPath(HttpReplyQueue * reply_queue, + HttpHandler * user_handler) { if (reply_queue != mReplyQueue) { @@ -80,9 +78,6 @@ void HttpOperation::setHandlers(HttpReplyQueue * reply_queue, mReplyQueue = reply_queue; } - // Not refcounted - mLibraryHandler = lib_handler; - // Not refcounted mUserHandler = user_handler; } @@ -121,11 +116,12 @@ void HttpOperation::stageFromActive(HttpService *) void HttpOperation::visitNotifier(HttpRequest *) { - if (mLibraryHandler) + if (mUserHandler) { HttpResponse * response = new HttpResponse(); - mLibraryHandler->onCompleted(static_cast(this), response); + response->setStatus(mStatus); + mUserHandler->onCompleted(static_cast(this), response); response->release(); } @@ -142,7 +138,7 @@ HttpStatus HttpOperation::cancel() void HttpOperation::addAsReply() { - if (mReplyQueue && mLibraryHandler) + if (mReplyQueue) { addRef(); mReplyQueue->addOp(this); diff --git a/indra/llcorehttp/_httpoperation.h b/indra/llcorehttp/_httpoperation.h index 01e26029d2..c93aa2def9 100644 --- a/indra/llcorehttp/_httpoperation.h +++ b/indra/llcorehttp/_httpoperation.h @@ -80,9 +80,8 @@ private: void operator=(const HttpOperation &); // Not defined public: - void setHandlers(HttpReplyQueue * reply_queue, - HttpHandler * lib_handler, - HttpHandler * user_handler); + void setReplyPath(HttpReplyQueue * reply_queue, + HttpHandler * handler); HttpHandler * getUserHandler() const { @@ -102,13 +101,15 @@ protected: protected: HttpReplyQueue * mReplyQueue; // Have refcount - HttpHandler * mLibraryHandler; // Have refcount - HttpHandler * mUserHandler; // Have refcount + HttpHandler * mUserHandler; public: + // Request Data HttpRequest::policy_t mReqPolicy; HttpRequest::priority_t mReqPriority; - + + // Reply Data + HttpStatus mStatus; }; // end class HttpOperation diff --git a/indra/llcorehttp/_httpoprequest.cpp b/indra/llcorehttp/_httpoprequest.cpp index ea0b99303e..e2550d057e 100644 --- a/indra/llcorehttp/_httpoprequest.cpp +++ b/indra/llcorehttp/_httpoprequest.cpp @@ -44,6 +44,7 @@ #include "_httplibcurl.h" #include "llhttpstatuscodes.h" +#include "llproxy.h" namespace { @@ -207,7 +208,7 @@ void HttpOpRequest::visitNotifier(HttpRequest * request) { static const HttpStatus partial_content(HTTP_PARTIAL_CONTENT, HE_SUCCESS); - if (mLibraryHandler) + if (mUserHandler) { HttpResponse * response = new HttpResponse(); response->setStatus(mStatus); @@ -219,7 +220,7 @@ void HttpOpRequest::visitNotifier(HttpRequest * request) response->setRange(mReplyOffset, mReplyLength); } - mLibraryHandler->onCompleted(static_cast(this), response); + mUserHandler->onCompleted(static_cast(this), response); response->release(); } @@ -304,6 +305,39 @@ HttpStatus HttpOpRequest::setupPost(HttpRequest::policy_t policy_id, } +HttpStatus HttpOpRequest::setupPut(HttpRequest::policy_t policy_id, + HttpRequest::priority_t priority, + const std::string & url, + BufferArray * body, + HttpOptions * options, + HttpHeaders * headers) +{ + HttpStatus status; + + mProcFlags = 0; + mReqPolicy = policy_id; + mReqPriority = priority; + mReqMethod = HOR_PUT; + mReqURL = url; + if (body) + { + body->addRef(); + mReqBody = body; + } + if (headers && ! mReqHeaders) + { + headers->addRef(); + mReqHeaders = headers; + } + if (options && ! mReqOptions) + { + mReqOptions = new HttpOptions(*options); + } + + return status; +} + + HttpStatus HttpOpRequest::prepareRequest(HttpService * service) { // Scrub transport and result data for retried op case @@ -346,8 +380,6 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service) curl_easy_setopt(mCurlHandle, CURLOPT_URL, mReqURL.c_str()); curl_easy_setopt(mCurlHandle, CURLOPT_PRIVATE, this); curl_easy_setopt(mCurlHandle, CURLOPT_ENCODING, ""); - // *FIXME: Need to deal with proxy setup... - // curl_easy_setopt(handle, CURLOPT_PROXY, ""); // *FIXME: Revisit this old DNS timeout setting - may no longer be valid curl_easy_setopt(mCurlHandle, CURLOPT_DNS_CACHE_TIMEOUT, 0); @@ -361,18 +393,31 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service) curl_easy_setopt(mCurlHandle, CURLOPT_SSL_VERIFYPEER, 1); curl_easy_setopt(mCurlHandle, CURLOPT_SSL_VERIFYHOST, 0); - std::string opt_value; + const std::string * opt_value(NULL); if (policy.get(HttpRequest::GP_CA_PATH, opt_value)) { - curl_easy_setopt(mCurlHandle, CURLOPT_CAPATH, opt_value.c_str()); + curl_easy_setopt(mCurlHandle, CURLOPT_CAPATH, opt_value->c_str()); } if (policy.get(HttpRequest::GP_CA_FILE, opt_value)) { - curl_easy_setopt(mCurlHandle, CURLOPT_CAINFO, opt_value.c_str()); + curl_easy_setopt(mCurlHandle, CURLOPT_CAINFO, opt_value->c_str()); } if (policy.get(HttpRequest::GP_HTTP_PROXY, opt_value)) { - curl_easy_setopt(mCurlHandle, CURLOPT_PROXY, opt_value.c_str()); + if (*opt_value == "LLProxy") + { + // Use the viewer-based thread-safe API which has a + // fast/safe check for proxy enable. Would like to + // encapsulate this someway... + LLProxy::getInstance()->applyProxySettings(mCurlHandle); + } + else + { + // *TODO: This is fine for now but get fuller socks/ + // authentication thing going later.... + curl_easy_setopt(mCurlHandle, CURLOPT_PROXY, opt_value->c_str()); + curl_easy_setopt(mCurlHandle, CURLOPT_PROXYTYPE, CURLPROXY_HTTP); + } } switch (mReqMethod) @@ -394,7 +439,6 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service) } curl_easy_setopt(mCurlHandle, CURLOPT_POSTFIELDS, static_cast(NULL)); curl_easy_setopt(mCurlHandle, CURLOPT_POSTFIELDSIZE, data_size); - mCurlHeaders = curl_slist_append(mCurlHeaders, "Transfer-Encoding: chunked"); mCurlHeaders = curl_slist_append(mCurlHeaders, "Expect:"); } break; @@ -409,7 +453,6 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service) } curl_easy_setopt(mCurlHandle, CURLOPT_INFILESIZE, data_size); curl_easy_setopt(mCurlHandle, CURLOPT_POSTFIELDS, (void *) NULL); - mCurlHeaders = curl_slist_append(mCurlHeaders, "Transfer-Encoding: chunked"); mCurlHeaders = curl_slist_append(mCurlHeaders, "Expect:"); mCurlHeaders = curl_slist_append(mCurlHeaders, "Connection: keep-alive"); mCurlHeaders = curl_slist_append(mCurlHeaders, "Keep-alive: 300"); diff --git a/indra/llcorehttp/_httpoprequest.h b/indra/llcorehttp/_httpoprequest.h index 6dcf30ca0c..80893beb40 100644 --- a/indra/llcorehttp/_httpoprequest.h +++ b/indra/llcorehttp/_httpoprequest.h @@ -91,6 +91,13 @@ public: HttpOptions * options, HttpHeaders * headers); + HttpStatus setupPut(HttpRequest::policy_t policy_id, + HttpRequest::priority_t priority, + const std::string & url, + BufferArray * body, + HttpOptions * options, + HttpHeaders * headers); + HttpStatus prepareRequest(HttpService * service); virtual HttpStatus cancel(); diff --git a/indra/llcorehttp/_httpopsetget.cpp b/indra/llcorehttp/_httpopsetget.cpp new file mode 100644 index 0000000000..21e058b2be --- /dev/null +++ b/indra/llcorehttp/_httpopsetget.cpp @@ -0,0 +1,105 @@ +/** + * @file _httpopsetget.cpp + * @brief Definitions for internal class HttpOpSetGet + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include "_httpopsetget.h" + +#include +#include + +#include "httpcommon.h" +#include "httphandler.h" +#include "httpresponse.h" + +#include "_httprequestqueue.h" +#include "_httpreplyqueue.h" +#include "_httpservice.h" +#include "_httppolicy.h" +#include "_httplibcurl.h" + + +namespace LLCore +{ + + +// ================================== +// HttpOpSetget +// ================================== + + +HttpOpSetGet::HttpOpSetGet() + : HttpOperation(), + mIsGlobal(false), + mDoSet(false), + mSetting(-1), // Nothing requested + mLongValue(0L) +{} + + +HttpOpSetGet::~HttpOpSetGet() +{} + + +void HttpOpSetGet::setupGet(HttpRequest::EGlobalPolicy setting) +{ + mIsGlobal = true; + mSetting = setting; +} + + +void HttpOpSetGet::setupSet(HttpRequest::EGlobalPolicy setting, const std::string & value) +{ + mIsGlobal = true; + mDoSet = true; + mSetting = setting; + mStrValue = value; +} + + +void HttpOpSetGet::stageFromRequest(HttpService * service) +{ + HttpPolicyGlobal & pol_opt(service->getPolicy().getGlobalOptions()); + HttpRequest::EGlobalPolicy setting(static_cast(mSetting)); + + if (mDoSet) + { + mStatus = pol_opt.set(setting, mStrValue); + } + if (mStatus) + { + const std::string * value; + if ((mStatus = pol_opt.get(setting, value))) + { + mStrValue = *value; + } + } + + addAsReply(); +} + + +} // end namespace LLCore + + diff --git a/indra/llcorehttp/_httpopsetget.h b/indra/llcorehttp/_httpopsetget.h new file mode 100644 index 0000000000..e065eb4c30 --- /dev/null +++ b/indra/llcorehttp/_httpopsetget.h @@ -0,0 +1,78 @@ +/** + * @file _httpopsetget.h + * @brief Internal declarations for the HttpOpSetGet subclass + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef _LLCORE_HTTP_OPSETGET_H_ +#define _LLCORE_HTTP_OPSETGET_H_ + + +#include "linden_common.h" // Modifies curl/curl.h interfaces + +#include "httpcommon.h" + +#include + +#include "_httpoperation.h" +#include "_refcounted.h" + + +namespace LLCore +{ + + +/// HttpOpSetGet requests dynamic changes to policy and +/// configuration settings. + +class HttpOpSetGet : public HttpOperation +{ +public: + HttpOpSetGet(); + virtual ~HttpOpSetGet(); + +private: + HttpOpSetGet(const HttpOpSetGet &); // Not defined + void operator=(const HttpOpSetGet &); // Not defined + +public: + void setupGet(HttpRequest::EGlobalPolicy setting); + void setupSet(HttpRequest::EGlobalPolicy setting, const std::string & value); + + virtual void stageFromRequest(HttpService *); + +public: + // Request data + bool mIsGlobal; + bool mDoSet; + int mSetting; + long mLongValue; + std::string mStrValue; + +}; // end class HttpOpSetGet + + +} // end namespace LLCore + +#endif // _LLCORE_HTTP_OPSETGET_H_ + diff --git a/indra/llcorehttp/_httpopsetpriority.cpp b/indra/llcorehttp/_httpopsetpriority.cpp index b0ee577087..d48c7a0b7d 100644 --- a/indra/llcorehttp/_httpopsetpriority.cpp +++ b/indra/llcorehttp/_httpopsetpriority.cpp @@ -60,18 +60,4 @@ void HttpOpSetPriority::stageFromRequest(HttpService * service) } -void HttpOpSetPriority::visitNotifier(HttpRequest * request) -{ - if (mLibraryHandler) - { - HttpResponse * response = new HttpResponse(); - - response->setStatus(mStatus); - mLibraryHandler->onCompleted(static_cast(this), response); - - response->release(); - } -} - - } // end namespace LLCore diff --git a/indra/llcorehttp/_httpopsetpriority.h b/indra/llcorehttp/_httpopsetpriority.h index b972f50fff..f1e94b6e43 100644 --- a/indra/llcorehttp/_httpopsetpriority.h +++ b/indra/llcorehttp/_httpopsetpriority.h @@ -56,10 +56,8 @@ private: public: virtual void stageFromRequest(HttpService *); - virtual void visitNotifier(HttpRequest * request); - protected: - HttpStatus mStatus; + // Request Data HttpHandle mHandle; HttpRequest::priority_t mPriority; }; // end class HttpOpSetPriority diff --git a/indra/llcorehttp/_httppolicy.cpp b/indra/llcorehttp/_httppolicy.cpp index 72bb6f14e4..8ee3f88658 100644 --- a/indra/llcorehttp/_httppolicy.cpp +++ b/indra/llcorehttp/_httppolicy.cpp @@ -71,6 +71,12 @@ HttpPolicy::~HttpPolicy() } +void HttpPolicy::setPolicies(const HttpPolicyGlobal & global) +{ + mGlobalOptions = global; +} + + void HttpPolicy::addOp(HttpOpRequest * op) { const int policy_class(op->mReqPolicy); diff --git a/indra/llcorehttp/_httppolicy.h b/indra/llcorehttp/_httppolicy.h index 14f6a9a676..73c22bab78 100644 --- a/indra/llcorehttp/_httppolicy.h +++ b/indra/llcorehttp/_httppolicy.h @@ -95,7 +95,9 @@ public: { return mGlobalOptions; } - + + void setPolicies(const HttpPolicyGlobal & global); + protected: struct State { diff --git a/indra/llcorehttp/_httppolicyglobal.cpp b/indra/llcorehttp/_httppolicyglobal.cpp index 877b85896f..d95d73cfba 100644 --- a/indra/llcorehttp/_httppolicyglobal.cpp +++ b/indra/llcorehttp/_httppolicyglobal.cpp @@ -32,7 +32,7 @@ namespace LLCore HttpPolicyGlobal::HttpPolicyGlobal() - : mValidMask(0UL), + : mSetMask(0UL), mConnectionLimit(32L) {} @@ -41,6 +41,20 @@ HttpPolicyGlobal::~HttpPolicyGlobal() {} +HttpPolicyGlobal & HttpPolicyGlobal::operator=(const HttpPolicyGlobal & other) +{ + if (this != &other) + { + mSetMask = other.mSetMask; + mConnectionLimit = other.mConnectionLimit; + mCAPath = other.mCAPath; + mCAFile = other.mCAFile; + mHttpProxy = other.mHttpProxy; + } + return *this; +} + + HttpStatus HttpPolicyGlobal::set(HttpRequest::EGlobalPolicy opt, long value) { switch (opt) @@ -53,7 +67,7 @@ HttpStatus HttpPolicyGlobal::set(HttpRequest::EGlobalPolicy opt, long value) return HttpStatus(HttpStatus::LLCORE, HE_INVALID_ARG); } - mValidMask |= 1UL << int(opt); + mSetMask |= 1UL << int(opt); return HttpStatus(); } @@ -78,7 +92,7 @@ HttpStatus HttpPolicyGlobal::set(HttpRequest::EGlobalPolicy opt, const std::stri return HttpStatus(HttpStatus::LLCORE, HE_INVALID_ARG); } - mValidMask |= 1UL << int(opt); + mSetMask |= 1UL << int(opt); return HttpStatus(); } @@ -90,7 +104,7 @@ HttpStatus HttpPolicyGlobal::get(HttpRequest::EGlobalPolicy opt, long & value) switch (opt) { case HttpRequest::GP_CONNECTION_LIMIT: - if (! (mValidMask & (1UL << int(opt)))) + if (! (mSetMask & (1UL << int(opt)))) return not_set; value = mConnectionLimit; break; @@ -103,28 +117,28 @@ HttpStatus HttpPolicyGlobal::get(HttpRequest::EGlobalPolicy opt, long & value) } -HttpStatus HttpPolicyGlobal::get(HttpRequest::EGlobalPolicy opt, std::string & value) +HttpStatus HttpPolicyGlobal::get(HttpRequest::EGlobalPolicy opt, const std::string *& value) { static const HttpStatus not_set(HttpStatus::LLCORE, HE_OPT_NOT_SET); - + switch (opt) { case HttpRequest::GP_CA_PATH: - if (! (mValidMask & (1UL << int(opt)))) + if (! (mSetMask & (1UL << int(opt)))) return not_set; - value = mCAPath; + value = &mCAPath; break; case HttpRequest::GP_CA_FILE: - if (! (mValidMask & (1UL << int(opt)))) + if (! (mSetMask & (1UL << int(opt)))) return not_set; - value = mCAFile; + value = &mCAFile; break; case HttpRequest::GP_HTTP_PROXY: - if (! (mValidMask & (1UL << int(opt)))) + if (! (mSetMask & (1UL << int(opt)))) return not_set; - value = mHttpProxy; + value = &mHttpProxy; break; default: diff --git a/indra/llcorehttp/_httppolicyglobal.h b/indra/llcorehttp/_httppolicyglobal.h index 39ffbcb9bb..f4bb4d4b25 100644 --- a/indra/llcorehttp/_httppolicyglobal.h +++ b/indra/llcorehttp/_httppolicyglobal.h @@ -40,18 +40,19 @@ public: HttpPolicyGlobal(); ~HttpPolicyGlobal(); + HttpPolicyGlobal & operator=(const HttpPolicyGlobal &); + private: HttpPolicyGlobal(const HttpPolicyGlobal &); // Not defined - void operator=(const HttpPolicyGlobal &); // Not defined public: HttpStatus set(HttpRequest::EGlobalPolicy opt, long value); HttpStatus set(HttpRequest::EGlobalPolicy opt, const std::string & value); HttpStatus get(HttpRequest::EGlobalPolicy opt, long & value); - HttpStatus get(HttpRequest::EGlobalPolicy opt, std::string & value); + HttpStatus get(HttpRequest::EGlobalPolicy opt, const std::string *& value); public: - unsigned long mValidMask; + unsigned long mSetMask; long mConnectionLimit; std::string mCAPath; std::string mCAFile; diff --git a/indra/llcorehttp/_httpservice.cpp b/indra/llcorehttp/_httpservice.cpp index b038bdb720..920a3f3b6d 100644 --- a/indra/llcorehttp/_httpservice.cpp +++ b/indra/llcorehttp/_httpservice.cpp @@ -79,11 +79,8 @@ HttpService::~HttpService() mTransport = NULL; } - if (mPolicy) - { - delete mPolicy; - mPolicy = NULL; - } + delete mPolicy; + mPolicy = NULL; if (mThread) { @@ -145,6 +142,10 @@ void HttpService::startThread() { mThread->release(); } + + // Push current policy definitions + mPolicy->setPolicies(mPolicyGlobal); + mThread = new LLCoreInt::HttpThread(boost::bind(&HttpService::threadRun, this, _1)); mThread->addRef(); // Need an explicit reference, implicit one is used internally sState = RUNNING; diff --git a/indra/llcorehttp/_httpservice.h b/indra/llcorehttp/_httpservice.h index 748354a8e4..3f953ec1a7 100644 --- a/indra/llcorehttp/_httpservice.h +++ b/indra/llcorehttp/_httpservice.h @@ -30,6 +30,7 @@ #include "httpcommon.h" #include "httprequest.h" +#include "_httppolicyglobal.h" namespace LLCoreInt @@ -157,6 +158,11 @@ public: { return *mTransport; } + + HttpPolicyGlobal & getGlobalOptions() + { + return mPolicyGlobal; + } protected: void threadRun(LLCoreInt::HttpThread * thread); @@ -173,11 +179,11 @@ protected: // === calling-thread-only data === LLCoreInt::HttpThread * mThread; - + HttpPolicyGlobal mPolicyGlobal; + // === working-thread-only data === HttpPolicy * mPolicy; // Simple pointer, has ownership HttpLibcurl * mTransport; // Simple pointer, has ownership - }; // end class HttpService } // end namespace LLCore diff --git a/indra/llcorehttp/httprequest.cpp b/indra/llcorehttp/httprequest.cpp index 2f36168f8b..089eee76f3 100644 --- a/indra/llcorehttp/httprequest.cpp +++ b/indra/llcorehttp/httprequest.cpp @@ -34,6 +34,7 @@ #include "_httpoprequest.h" #include "_httpopsetpriority.h" #include "_httpopcancel.h" +#include "_httpopsetget.h" #include "lltimer.h" @@ -48,39 +49,6 @@ bool has_inited(false); namespace LLCore { -// ==================================== -// InternalHandler Implementation -// ==================================== - - -class HttpRequest::InternalHandler : public HttpHandler -{ -public: - InternalHandler(HttpRequest & request) - : mRequest(request) - {} - -protected: - InternalHandler(const InternalHandler &); // Not defined - void operator=(const InternalHandler &); // Not defined - -public: - void onCompleted(HttpHandle handle, HttpResponse * response) - { - HttpOperation * op(static_cast(handle)); - HttpHandler * user_handler(op->getUserHandler()); - if (user_handler) - { - user_handler->onCompleted(handle, response); - } - } - -protected: - HttpRequest & mRequest; - -}; // end class HttpRequest::InternalHandler - - // ==================================== // HttpRequest Implementation // ==================================== @@ -92,15 +60,12 @@ HttpRequest::policy_t HttpRequest::sNextPolicyID(1); HttpRequest::HttpRequest() : //HttpHandler(), mReplyQueue(NULL), - mRequestQueue(NULL), - mSelfHandler(NULL) + mRequestQueue(NULL) { mRequestQueue = HttpRequestQueue::instanceOf(); mRequestQueue->addRef(); mReplyQueue = new HttpReplyQueue(); - - mSelfHandler = new InternalHandler(*this); } @@ -117,9 +82,6 @@ HttpRequest::~HttpRequest() mReplyQueue->release(); mReplyQueue = NULL; } - - delete mSelfHandler; - mSelfHandler = NULL; } @@ -132,7 +94,7 @@ HttpStatus HttpRequest::setPolicyGlobalOption(EGlobalPolicy opt, long value) { // *FIXME: Fail if thread is running. - return HttpService::instanceOf()->getPolicy().getGlobalOptions().set(opt, value); + return HttpService::instanceOf()->getGlobalOptions().set(opt, value); } @@ -140,7 +102,7 @@ HttpStatus HttpRequest::setPolicyGlobalOption(EGlobalPolicy opt, const std::stri { // *FIXME: Fail if thread is running. - return HttpService::instanceOf()->getPolicy().getGlobalOptions().set(opt, value); + return HttpService::instanceOf()->getGlobalOptions().set(opt, value); } @@ -192,7 +154,7 @@ HttpHandle HttpRequest::requestGetByteRange(policy_t policy_id, mLastReqStatus = status; return handle; } - op->setHandlers(mReplyQueue, mSelfHandler, user_handler); + op->setReplyPath(mReplyQueue, user_handler); mRequestQueue->addOp(op); // transfers refcount mLastReqStatus = status; @@ -220,7 +182,7 @@ HttpHandle HttpRequest::requestPost(policy_t policy_id, mLastReqStatus = status; return handle; } - op->setHandlers(mReplyQueue, mSelfHandler, user_handler); + op->setReplyPath(mReplyQueue, user_handler); mRequestQueue->addOp(op); // transfers refcount mLastReqStatus = status; @@ -230,19 +192,31 @@ HttpHandle HttpRequest::requestPost(policy_t policy_id, } -HttpHandle HttpRequest::requestCancel(HttpHandle handle, HttpHandler * user_handler) +HttpHandle HttpRequest::requestPut(policy_t policy_id, + priority_t priority, + const std::string & url, + BufferArray * body, + HttpOptions * options, + HttpHeaders * headers, + HttpHandler * user_handler) { HttpStatus status; - HttpHandle ret_handle(LLCORE_HTTP_HANDLE_INVALID); - - HttpOpCancel * op = new HttpOpCancel(handle); - op->setHandlers(mReplyQueue, mSelfHandler, user_handler); - mRequestQueue->addOp(op); // transfer refcount as well + HttpHandle handle(LLCORE_HTTP_HANDLE_INVALID); + HttpOpRequest * op = new HttpOpRequest(); + if (! (status = op->setupPut(policy_id, priority, url, body, options, headers))) + { + op->release(); + mLastReqStatus = status; + return handle; + } + op->setReplyPath(mReplyQueue, user_handler); + mRequestQueue->addOp(op); // transfers refcount + mLastReqStatus = status; - ret_handle = static_cast(op); + handle = static_cast(op); - return ret_handle; + return handle; } @@ -252,7 +226,7 @@ HttpHandle HttpRequest::requestNoOp(HttpHandler * user_handler) HttpHandle handle(LLCORE_HTTP_HANDLE_INVALID); HttpOpNull * op = new HttpOpNull(); - op->setHandlers(mReplyQueue, mSelfHandler, user_handler); + op->setReplyPath(mReplyQueue, user_handler); mRequestQueue->addOp(op); // transfer refcount as well mLastReqStatus = status; @@ -262,23 +236,6 @@ HttpHandle HttpRequest::requestNoOp(HttpHandler * user_handler) } -HttpHandle HttpRequest::requestSetPriority(HttpHandle request, priority_t priority, - HttpHandler * handler) -{ - HttpStatus status; - HttpHandle ret_handle(LLCORE_HTTP_HANDLE_INVALID); - - HttpOpSetPriority * op = new HttpOpSetPriority(request, priority); - op->setHandlers(mReplyQueue, mSelfHandler, handler); - mRequestQueue->addOp(op); // transfer refcount as well - - mLastReqStatus = status; - ret_handle = static_cast(op); - - return ret_handle; -} - - HttpStatus HttpRequest::update(long millis) { const HttpTime limit(totalTime() + (1000 * HttpTime(millis))); @@ -302,6 +259,38 @@ HttpStatus HttpRequest::update(long millis) // Request Management Methods // ==================================== +HttpHandle HttpRequest::requestCancel(HttpHandle handle, HttpHandler * user_handler) +{ + HttpStatus status; + HttpHandle ret_handle(LLCORE_HTTP_HANDLE_INVALID); + + HttpOpCancel * op = new HttpOpCancel(handle); + op->setReplyPath(mReplyQueue, user_handler); + mRequestQueue->addOp(op); // transfer refcount as well + + mLastReqStatus = status; + ret_handle = static_cast(op); + + return ret_handle; +} + + +HttpHandle HttpRequest::requestSetPriority(HttpHandle request, priority_t priority, + HttpHandler * handler) +{ + HttpStatus status; + HttpHandle ret_handle(LLCORE_HTTP_HANDLE_INVALID); + + HttpOpSetPriority * op = new HttpOpSetPriority(request, priority); + op->setReplyPath(mReplyQueue, handler); + mRequestQueue->addOp(op); // transfer refcount as well + + mLastReqStatus = status; + ret_handle = static_cast(op); + + return ret_handle; +} + // ==================================== // Utility Methods @@ -350,7 +339,27 @@ HttpHandle HttpRequest::requestStopThread(HttpHandler * user_handler) HttpHandle handle(LLCORE_HTTP_HANDLE_INVALID); HttpOpStop * op = new HttpOpStop(); - op->setHandlers(mReplyQueue, mSelfHandler, user_handler); + op->setReplyPath(mReplyQueue, user_handler); + mRequestQueue->addOp(op); // transfer refcount as well + + mLastReqStatus = status; + handle = static_cast(op); + + return handle; +} + +// ==================================== +// Dynamic Policy Methods +// ==================================== + +HttpHandle HttpRequest::requestSetHttpProxy(const std::string & proxy, HttpHandler * handler) +{ + HttpStatus status; + HttpHandle handle(LLCORE_HTTP_HANDLE_INVALID); + + HttpOpSetGet * op = new HttpOpSetGet(); + op->setupSet(GP_HTTP_PROXY, proxy); + op->setReplyPath(mReplyQueue, handler); mRequestQueue->addOp(op); // transfer refcount as well mLastReqStatus = status; diff --git a/indra/llcorehttp/httprequest.h b/indra/llcorehttp/httprequest.h index 01dbfba6dd..a953aa28d0 100644 --- a/indra/llcorehttp/httprequest.h +++ b/indra/llcorehttp/httprequest.h @@ -124,8 +124,8 @@ public: /// @param opt Enum of option to be set. /// @param value Desired value of option. /// @return Standard status code. - HttpStatus setPolicyGlobalOption(EGlobalPolicy opt, long value); - HttpStatus setPolicyGlobalOption(EGlobalPolicy opt, const std::string & value); + static HttpStatus setPolicyGlobalOption(EGlobalPolicy opt, long value); + static HttpStatus setPolicyGlobalOption(EGlobalPolicy opt, const std::string & value); /// Create a new policy class into which requests can be made. /// @@ -236,6 +236,32 @@ public: HttpHandler * handler); + /// + /// @param policy_id Default or user-defined policy class under + /// which this request is to be serviced. + /// @param priority Standard priority scheme inherited from + /// Indra code base. + /// @param url + /// @param body Byte stream to be sent as the body. No + /// further encoding or escaping will be done + /// to the content. + /// @param options (optional) + /// @param headers (optional) + /// @param handler (optional) + /// @return The handle of the request if successfully + /// queued or LLCORE_HTTP_HANDLE_INVALID if the + /// request could not be queued. In the latter + /// case, @see getStatus() will return more info. + /// + HttpHandle requestPut(policy_t policy_id, + priority_t priority, + const std::string & url, + BufferArray * body, + HttpOptions * options, + HttpHeaders * headers, + HttpHandler * handler); + + /// Queue a NoOp request. /// The request is queued and serviced by the working thread which /// immediately processes it and returns the request to the reply @@ -325,13 +351,20 @@ public: HttpHandle requestStopThread(HttpHandler * handler); /// @} + + /// @name DynamicPolicyMethods + /// + /// @{ + + /// Request that a running transport pick up a new proxy setting. + /// An empty string will indicate no proxy is to be used. + HttpHandle requestSetHttpProxy(const std::string & proxy, HttpHandler * handler); + + /// @} protected: void generateNotification(HttpOperation * op); - class InternalHandler; - friend class InternalHandler; - private: /// @name InstanceData /// @@ -339,7 +372,6 @@ private: HttpStatus mLastReqStatus; HttpReplyQueue * mReplyQueue; HttpRequestQueue * mRequestQueue; - InternalHandler * mSelfHandler; /// @} diff --git a/indra/llcorehttp/tests/llcorehttp_test.cpp b/indra/llcorehttp/tests/llcorehttp_test.cpp index f59361ab53..2b36d3a982 100644 --- a/indra/llcorehttp/tests/llcorehttp_test.cpp +++ b/indra/llcorehttp/tests/llcorehttp_test.cpp @@ -44,6 +44,8 @@ #include "test_bufferarray.hpp" #include "test_httprequestqueue.hpp" +#include "llproxy.h" + unsigned long ssl_thread_id_callback(void); void ssl_locking_callback(int mode, int type, const char * file, int line); @@ -91,11 +93,15 @@ void init_curl() CRYPTO_set_locking_callback(ssl_locking_callback); CRYPTO_set_id_callback(ssl_thread_id_callback); } + + LLProxy::getInstance(); } void term_curl() { + LLProxy::cleanupClass(); + CRYPTO_set_locking_callback(NULL); for (int i(0); i < ssl_mutex_count; ++i) { diff --git a/indra/llcorehttp/tests/test_httpoperation.hpp b/indra/llcorehttp/tests/test_httpoperation.hpp index 6c3df1e9e3..17b1a96878 100644 --- a/indra/llcorehttp/tests/test_httpoperation.hpp +++ b/indra/llcorehttp/tests/test_httpoperation.hpp @@ -97,13 +97,12 @@ namespace tut // Get some handlers TestHandler * h1 = new TestHandler(); - TestHandler * h2 = new TestHandler(); // create a new ref counted object with an implicit reference HttpOpNull * op = new HttpOpNull(); // Add the handlers - op->setHandlers(NULL, h1, h2); + op->setReplyPath(NULL, h1); // Check ref count ensure(op->getRefCount() == 1); @@ -117,8 +116,6 @@ namespace tut // release the handlers delete h1; h1 = NULL; - delete h2; - h2 = NULL; ensure(mMemTotal == GetMemTotal()); } diff --git a/indra/llcorehttp/tests/test_httprequest.hpp b/indra/llcorehttp/tests/test_httprequest.hpp index 81f8fe4a85..61698f34d8 100644 --- a/indra/llcorehttp/tests/test_httprequest.hpp +++ b/indra/llcorehttp/tests/test_httprequest.hpp @@ -27,6 +27,7 @@ #define TEST_LLCORE_HTTP_REQUEST_H_ #include "httprequest.h" +#include "bufferarray.h" #include "httphandler.h" #include "httpresponse.h" #include "_httpservice.h" @@ -604,6 +605,244 @@ void HttpRequestTestObjectType::test<6>() } } +template <> template <> +void HttpRequestTestObjectType::test<7>() +{ + ScopedCurlInit ready; + + std::string url_base(get_base_url()); + std::cerr << "Base: " << url_base << std::endl; + + set_test_name("HttpRequest PUT to real service"); + + // Handler can be stack-allocated *if* there are no dangling + // references to it after completion of this method. + // Create before memory record as the string copy will bump numbers. + TestHandler2 handler(this, "handler"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + mHandlerCalls = 0; + + HttpRequest * req = NULL; + BufferArray * body = new BufferArray; + + try + { + // Get singletons created + HttpRequest::createService(); + + // Start threading early so that thread memory is invariant + // over the test. + HttpRequest::startThread(); + + // create a new ref counted object with an implicit reference + req = new HttpRequest(); + ensure("Memory allocated on construction", mMemTotal < GetMemTotal()); + + // Issue a GET that *can* connect + static const char * body_text("Now is the time for all good men..."); + body->append(body_text, strlen(body_text)); + mStatus = HttpStatus(200); + HttpHandle handle = req->requestPut(HttpRequest::DEFAULT_POLICY_ID, + 0U, + url_base, + body, + NULL, + NULL, + &handler); + ensure("Valid handle returned for ranged request", handle != LLCORE_HTTP_HANDLE_INVALID); + + // Run the notification pump. + int count(0); + int limit(10); + while (count++ < limit && mHandlerCalls < 1) + { + req->update(1000); + usleep(100000); + } + ensure("Request executed in reasonable time", count < limit); + ensure("One handler invocation for request", mHandlerCalls == 1); + + // Okay, request a shutdown of the servicing thread + mStatus = HttpStatus(); + handle = req->requestStopThread(&handler); + ensure("Valid handle returned for second request", handle != LLCORE_HTTP_HANDLE_INVALID); + + // Run the notification pump again + count = 0; + limit = 10; + while (count++ < limit && mHandlerCalls < 2) + { + req->update(1000); + usleep(100000); + } + ensure("Second request executed in reasonable time", count < limit); + ensure("Second handler invocation", mHandlerCalls == 2); + + // See that we actually shutdown the thread + count = 0; + limit = 10; + while (count++ < limit && ! HttpService::isStopped()) + { + usleep(100000); + } + ensure("Thread actually stopped running", HttpService::isStopped()); + + // Lose the request body + body->release(); + body = NULL; + + // release the request object + delete req; + req = NULL; + + // Shut down service + HttpRequest::destroyService(); + + ensure("Two handler calls on the way out", 2 == mHandlerCalls); + +#if defined(WIN32) + // Can only do this memory test on Windows. On other platforms, + // the LL logging system holds on to memory and produces what looks + // like memory leaks... + + // printf("Old mem: %d, New mem: %d\n", mMemTotal, GetMemTotal()); + ensure("Memory usage back to that at entry", mMemTotal == GetMemTotal()); +#endif + } + catch (...) + { + if (body) + { + body->release(); + } + stop_thread(req); + delete req; + HttpRequest::destroyService(); + throw; + } +} + +template <> template <> +void HttpRequestTestObjectType::test<8>() +{ + ScopedCurlInit ready; + + std::string url_base(get_base_url()); + std::cerr << "Base: " << url_base << std::endl; + + set_test_name("HttpRequest POST to real service"); + + // Handler can be stack-allocated *if* there are no dangling + // references to it after completion of this method. + // Create before memory record as the string copy will bump numbers. + TestHandler2 handler(this, "handler"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + mHandlerCalls = 0; + + HttpRequest * req = NULL; + BufferArray * body = new BufferArray; + + try + { + // Get singletons created + HttpRequest::createService(); + + // Start threading early so that thread memory is invariant + // over the test. + HttpRequest::startThread(); + + // create a new ref counted object with an implicit reference + req = new HttpRequest(); + ensure("Memory allocated on construction", mMemTotal < GetMemTotal()); + + // Issue a GET that *can* connect + static const char * body_text("Now is the time for all good men..."); + body->append(body_text, strlen(body_text)); + mStatus = HttpStatus(200); + HttpHandle handle = req->requestPost(HttpRequest::DEFAULT_POLICY_ID, + 0U, + url_base, + body, + NULL, + NULL, + &handler); + ensure("Valid handle returned for ranged request", handle != LLCORE_HTTP_HANDLE_INVALID); + + // Run the notification pump. + int count(0); + int limit(10); + while (count++ < limit && mHandlerCalls < 1) + { + req->update(1000); + usleep(100000); + } + ensure("Request executed in reasonable time", count < limit); + ensure("One handler invocation for request", mHandlerCalls == 1); + + // Okay, request a shutdown of the servicing thread + mStatus = HttpStatus(); + handle = req->requestStopThread(&handler); + ensure("Valid handle returned for second request", handle != LLCORE_HTTP_HANDLE_INVALID); + + // Run the notification pump again + count = 0; + limit = 10; + while (count++ < limit && mHandlerCalls < 2) + { + req->update(1000); + usleep(100000); + } + ensure("Second request executed in reasonable time", count < limit); + ensure("Second handler invocation", mHandlerCalls == 2); + + // See that we actually shutdown the thread + count = 0; + limit = 10; + while (count++ < limit && ! HttpService::isStopped()) + { + usleep(100000); + } + ensure("Thread actually stopped running", HttpService::isStopped()); + + // Lose the request body + body->release(); + body = NULL; + + // release the request object + delete req; + req = NULL; + + // Shut down service + HttpRequest::destroyService(); + + ensure("Two handler calls on the way out", 2 == mHandlerCalls); + +#if defined(WIN32) + // Can only do this memory test on Windows. On other platforms, + // the LL logging system holds on to memory and produces what looks + // like memory leaks... + + // printf("Old mem: %d, New mem: %d\n", mMemTotal, GetMemTotal()); + ensure("Memory usage back to that at entry", mMemTotal == GetMemTotal()); +#endif + } + catch (...) + { + if (body) + { + body->release(); + } + stop_thread(req); + delete req; + HttpRequest::destroyService(); + throw; + } +} + } // end namespace tut namespace diff --git a/indra/llcorehttp/tests/test_llcorehttp_peer.py b/indra/llcorehttp/tests/test_llcorehttp_peer.py index 3e200a5c19..8c3ad805b3 100644 --- a/indra/llcorehttp/tests/test_llcorehttp_peer.py +++ b/indra/llcorehttp/tests/test_llcorehttp_peer.py @@ -85,7 +85,15 @@ class TestHTTPRequestHandler(BaseHTTPRequestHandler): def do_POST(self): # Read the provided POST data. - self.answer(self.read()) + # self.answer(self.read()) + self.answer(dict(reply="success", status=200, + reason=self.read())) + + def do_PUT(self): + # Read the provided PUT data. + # self.answer(self.read()) + self.answer(dict(reply="success", status=200, + reason=self.read())) def answer(self, data): debug("%s.answer(%s): self.path = %r", self.__class__.__name__, data, self.path) diff --git a/indra/newview/llappviewer.cpp b/indra/newview/llappviewer.cpp index 7a44415fba..e2c13e77e3 100644 --- a/indra/newview/llappviewer.cpp +++ b/indra/newview/llappviewer.cpp @@ -5358,6 +5358,7 @@ void CoreHttp::init() mRequest = new LLCore::HttpRequest; + // Point to our certs or SSH/https: will fail on connect status = mRequest->setPolicyGlobalOption(LLCore::HttpRequest::GP_CA_FILE, gDirUtilp->getCAFile()); if (! status) @@ -5366,6 +5367,18 @@ void CoreHttp::init() << status.toString() << LL_ENDL; } + + // Establish HTTP Proxy. "LLProxy" is a special string which directs + // the code to use LLProxy::applyProxySettings() to establish any + // HTTP or SOCKS proxy for http operations. + status = mRequest->setPolicyGlobalOption(LLCore::HttpRequest::GP_HTTP_PROXY, + std::string("LLProxy")); + if (! status) + { + LL_ERRS("Init") << "Failed to set HTTP proxy for HTTP services. Reason: " + << status.toString() + << LL_ENDL; + } status = LLCore::HttpRequest::startThread(); if (! status) -- cgit v1.3 From 682ae001cf3a618ff6f96469e3c1f074fbb76a4e Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Wed, 13 Jun 2012 11:46:38 -0400 Subject: Restore original priority scheme of LOW/HIGH. The NORMAL range doesn't do any sleeping at all and so we'll spin the core harder than we already are. Bring all idlers into the same range. --- indra/newview/lltexturefetch.cpp | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) (limited to 'indra/newview') diff --git a/indra/newview/lltexturefetch.cpp b/indra/newview/lltexturefetch.cpp index 571f9ab3b5..f5e7540e85 100644 --- a/indra/newview/lltexturefetch.cpp +++ b/indra/newview/lltexturefetch.cpp @@ -203,7 +203,7 @@ // Priority Scheme // // [PRIORITY_LOW, PRIORITY_NORMAL) - for WAIT_HTTP_RESOURCE state -// [PRIORITY_NORMAL, PRIORITY_HIGH) - waiting for external event +// and other wait states // [PRIORITY_HIGH, PRIORITY_URGENT) - External event delivered, // rapidly transitioning through states, // no waiting allowed @@ -1102,7 +1102,7 @@ bool LLTextureFetchWorker::doWork(S32 param) if (mUrl.compare(0, 7, "file://") == 0) { - setPriority(LLWorkerThread::PRIORITY_NORMAL | mWorkPriority); // Set priority first since Responder may change it + setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); // Set priority first since Responder may change it // read file from local disk std::string filename = mUrl.substr(7, std::string::npos); @@ -1112,7 +1112,7 @@ bool LLTextureFetchWorker::doWork(S32 param) } else if (mUrl.empty()) { - setPriority(LLWorkerThread::PRIORITY_NORMAL | mWorkPriority); // Set priority first since Responder may change it + setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); // Set priority first since Responder may change it CacheReadResponder* responder = new CacheReadResponder(mFetcher, mID, mFormattedImage); mCacheReadHandle = mFetcher->mTextureCache->readFromCache(mID, cache_priority, @@ -1241,7 +1241,7 @@ bool LLTextureFetchWorker::doWork(S32 param) mSentRequest = QUEUED; mFetcher->addToNetworkQueue(this); recordTextureStart(false); - setPriority(LLWorkerThread::PRIORITY_NORMAL | mWorkPriority); + setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); return false; } @@ -1252,7 +1252,7 @@ bool LLTextureFetchWorker::doWork(S32 param) // Make certain this is in the network queue //mFetcher->addToNetworkQueue(this); //recordTextureStart(false); - //setPriority(LLWorkerThread::PRIORITY_NORMAL | mWorkPriority); + //setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); return false; } } @@ -1281,7 +1281,7 @@ bool LLTextureFetchWorker::doWork(S32 param) else { mFetcher->addToNetworkQueue(this); // failsafe - setPriority(LLWorkerThread::PRIORITY_NORMAL | mWorkPriority); + setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); recordTextureStart(false); } return false; @@ -1371,7 +1371,7 @@ bool LLTextureFetchWorker::doWork(S32 param) mHttpActive = true; mFetcher->addToHTTPQueue(mID); recordTextureStart(true); - setPriority(LLWorkerThread::PRIORITY_NORMAL | mWorkPriority); + setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); mState = WAIT_HTTP_REQ; // fall through @@ -1495,7 +1495,7 @@ bool LLTextureFetchWorker::doWork(S32 param) } else { - setPriority(LLWorkerThread::PRIORITY_NORMAL | mWorkPriority); + setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); return false; } } @@ -1504,7 +1504,7 @@ bool LLTextureFetchWorker::doWork(S32 param) { static LLCachedControl textures_decode_disabled(gSavedSettings,"TextureDecodeDisabled"); - setPriority(LLWorkerThread::PRIORITY_NORMAL | mWorkPriority); // Set priority first since Responder may change it + setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); // Set priority first since Responder may change it if (textures_decode_disabled) { // for debug use, don't decode @@ -1540,7 +1540,7 @@ bool LLTextureFetchWorker::doWork(S32 param) mAuxImage = NULL; llassert_always(mFormattedImage.notNull()); S32 discard = mHaveAllData ? 0 : mLoadedDiscard; - U32 image_priority = LLWorkerThread::PRIORITY_NORMAL | mWorkPriority; + U32 image_priority = LLWorkerThread::PRIORITY_LOW | mWorkPriority; mDecoded = FALSE; mState = DECODE_IMAGE_UPDATE; LL_DEBUGS("Texture") << mID << ": Decoding. Bytes: " << mFormattedImage->getDataSize() << " Discard: " << discard @@ -1612,7 +1612,7 @@ bool LLTextureFetchWorker::doWork(S32 param) } } llassert_always(datasize); - setPriority(LLWorkerThread::PRIORITY_NORMAL | mWorkPriority); // Set priority first since Responder may change it + setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); // Set priority first since Responder may change it U32 cache_priority = mWorkPriority; mWritten = FALSE; mState = WAIT_ON_WRITE; @@ -1654,7 +1654,7 @@ bool LLTextureFetchWorker::doWork(S32 param) } else { - setPriority(LLWorkerThread::PRIORITY_NORMAL | mWorkPriority); + setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); return true; } } -- cgit v1.3 From b08125a5874a89ce5210f8fb2c961ae17fb80fde Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Thu, 14 Jun 2012 16:31:48 -0400 Subject: LLMutex recursive lock, global & per-request tracing, simple GET request, LLProxy support, HttpOptions starting to work, HTTP resource waiting fixed. Non-LLThread-based threads need to do some registration or LLMutex locks taken out in these threads will not work as expected (SH-3154). We'll get a better solution later, this fixes some things for now. Tracing of operations now supported. Global and per-request (via HttpOptions) tracing levels of [0..3]. The 2 and 3 levels use libcurl's VERBOSE mode combined with CURLOPT_DEBUGFUNCTION to stream high levels of detail into the log. *Very* laggy but useful. Simple GET request supported (no Range: header). Really just a degenrate case of a ranged get but supplied an API anyway. Global option to use the LLProxy interface to setup CURL handles for either socks5 or http proxy usage. This isn't really the most encapsulated way to do this but a better solution will have to come later. The wantHeaders and tracing options are now supported in HttpOptions giving per-request controls. Big refactoring of the HTTP resource waiter in lltexturefetch. What I was doing before wasn't correct. Instead, I'm implementing the resource wait after the Semaphore model (though not using system semaphores). So instead of having a sequence like: SEND_HTTP_REQ -> WAIT_HTTP_RESOURCE -> SEND_HTTP_REQ, we now do WAIT_HTTP_RESOURCE -> WAIT_HTTP_RESOURCE2 (actual wait) -> SEND_HTTP_REQ. Works well but the prioritized filling of the corehttp library needs some performance work later. --- indra/llcommon/llthread.cpp | 7 + indra/llcommon/llthread.h | 5 + indra/llcorehttp/_httplibcurl.cpp | 24 ++- indra/llcorehttp/_httpoperation.cpp | 15 +- indra/llcorehttp/_httpoperation.h | 4 + indra/llcorehttp/_httpoprequest.cpp | 271 +++++++++++++++++++++------- indra/llcorehttp/_httpoprequest.h | 16 +- indra/llcorehttp/_httpopsetget.cpp | 4 +- indra/llcorehttp/_httppolicy.cpp | 21 ++- indra/llcorehttp/_httppolicy.h | 5 +- indra/llcorehttp/_httppolicyglobal.cpp | 52 ++++-- indra/llcorehttp/_httppolicyglobal.h | 6 +- indra/llcorehttp/_httpservice.cpp | 18 +- indra/llcorehttp/httpoptions.cpp | 18 +- indra/llcorehttp/httpoptions.h | 14 +- indra/llcorehttp/httprequest.cpp | 27 +++ indra/llcorehttp/httprequest.h | 75 +++++++- indra/llcorehttp/tests/test_httprequest.hpp | 228 ++++++++++++++++++++++- indra/newview/llappviewer.cpp | 20 +- indra/newview/lltexturefetch.cpp | 120 +++++++----- indra/newview/lltexturefetch.h | 9 + indra/newview/lltextureview.cpp | 5 +- 22 files changed, 803 insertions(+), 161 deletions(-) (limited to 'indra/newview') diff --git a/indra/llcommon/llthread.cpp b/indra/llcommon/llthread.cpp index a6ad6b125c..b27b64b26f 100644 --- a/indra/llcommon/llthread.cpp +++ b/indra/llcommon/llthread.cpp @@ -71,6 +71,13 @@ LL_COMMON_API void assert_main_thread() } } +void LLThread::registerThreadID() +{ +#if !LL_DARWIN + sThreadID = ++sIDIter; +#endif +} + // // Handed to the APR thread creation function // diff --git a/indra/llcommon/llthread.h b/indra/llcommon/llthread.h index b52e70ab2e..54af41ec59 100644 --- a/indra/llcommon/llthread.h +++ b/indra/llcommon/llthread.h @@ -88,6 +88,11 @@ public: U32 getID() const { return mID; } + // Called by threads *not* created via LLThread to register some + // internal state used by LLMutex. You must call this once early + // in the running thread to prevent collisions with the main thread. + static void registerThreadID(); + private: BOOL mPaused; diff --git a/indra/llcorehttp/_httplibcurl.cpp b/indra/llcorehttp/_httplibcurl.cpp index e134a28401..a176dd5b2a 100644 --- a/indra/llcorehttp/_httplibcurl.cpp +++ b/indra/llcorehttp/_httplibcurl.cpp @@ -159,6 +159,17 @@ void HttpLibcurl::addOp(HttpOpRequest * op) curl_multi_add_handle(mMultiHandles[op->mReqPolicy], op->mCurlHandle); op->mCurlActive = true; + if (op->mTracing > 0) + { + HttpPolicy & policy(mService->getPolicy()); + + LL_INFOS("CoreHttp") << "TRACE, ToActiveQueue, Handle: " + << static_cast(op) + << ", Actives: " << mActiveOps.size() + << ", Readies: " << policy.getReadyCount(op->mReqPolicy) + << LL_ENDL; + } + // On success, make operation active mActiveOps.insert(op); } @@ -190,10 +201,9 @@ bool HttpLibcurl::completeRequest(CURLM * multi_handle, CURL * handle, CURLcode // Deactivate request op->mCurlActive = false; - // Set final status of request + // Set final status of request if it hasn't failed by other mechanisms yet if (op->mStatus) { - // Only set if it hasn't failed by other mechanisms yet op->mStatus = HttpStatus(HttpStatus::EXT_CURL_EASY, status); } if (op->mStatus) @@ -209,6 +219,16 @@ bool HttpLibcurl::completeRequest(CURLM * multi_handle, CURL * handle, CURLcode curl_easy_cleanup(handle); op->mCurlHandle = NULL; + // Tracing + if (op->mTracing > 0) + { + LL_INFOS("CoreHttp") << "TRACE, RequestComplete, Handle: " + << static_cast(op) + << ", Status: " << op->mStatus.toHex() + << LL_ENDL; + } + + // Dispatch to next stage HttpPolicy & policy(mService->getPolicy()); bool still_active(policy.stageAfterCompletion(op)); diff --git a/indra/llcorehttp/_httpoperation.cpp b/indra/llcorehttp/_httpoperation.cpp index b5c58013d4..5a31bf90e7 100644 --- a/indra/llcorehttp/_httpoperation.cpp +++ b/indra/llcorehttp/_httpoperation.cpp @@ -34,6 +34,8 @@ #include "_httpreplyqueue.h" #include "_httpservice.h" +#include "lltimer.h" + namespace LLCore { @@ -49,8 +51,10 @@ HttpOperation::HttpOperation() mReplyQueue(NULL), mUserHandler(NULL), mReqPolicy(HttpRequest::DEFAULT_POLICY_ID), - mReqPriority(0U) + mReqPriority(0U), + mTracing(0) { + mMetricCreated = totalTime(); } @@ -113,7 +117,7 @@ void HttpOperation::stageFromActive(HttpService *) llassert_always(false); } - + void HttpOperation::visitNotifier(HttpRequest *) { if (mUserHandler) @@ -138,6 +142,13 @@ HttpStatus HttpOperation::cancel() void HttpOperation::addAsReply() { + if (mTracing > 0) + { + LL_INFOS("CoreHttp") << "TRACE, ToReplyQueue, Handle: " + << static_cast(this) + << LL_ENDL; + } + if (mReplyQueue) { addRef(); diff --git a/indra/llcorehttp/_httpoperation.h b/indra/llcorehttp/_httpoperation.h index c93aa2def9..de4939a0ac 100644 --- a/indra/llcorehttp/_httpoperation.h +++ b/indra/llcorehttp/_httpoperation.h @@ -110,6 +110,10 @@ public: // Reply Data HttpStatus mStatus; + + // Tracing, debug and metrics + HttpTime mMetricCreated; + int mTracing; }; // end class HttpOperation diff --git a/indra/llcorehttp/_httpoprequest.cpp b/indra/llcorehttp/_httpoprequest.cpp index e2550d057e..f78971d8f2 100644 --- a/indra/llcorehttp/_httpoprequest.cpp +++ b/indra/llcorehttp/_httpoprequest.cpp @@ -63,6 +63,16 @@ int parse_content_range_header(char * buffer, unsigned int * last, unsigned int * length); + +// Take data from libcurl's CURLOPT_DEBUGFUNCTION callback and +// escape and format it for a tracing line in logging. Absolutely +// anything including NULs can be in the data. If @scrub is true, +// non-printing or non-ascii characters are replaced with spaces +// otherwise a %XX form of escaping is used. +void escape_libcurl_debug_data(char * buffer, size_t len, bool scrub, + std::string & safe_line); + + #if defined(WIN32) // Not available on windows where the legacy strtok interface @@ -78,11 +88,6 @@ namespace LLCore { -// ================================== -// HttpOpRequest -// ================================== - - HttpOpRequest::HttpOpRequest() : HttpOperation(), mProcFlags(0U), @@ -237,6 +242,19 @@ HttpStatus HttpOpRequest::cancel() } +HttpStatus HttpOpRequest::setupGet(HttpRequest::policy_t policy_id, + HttpRequest::priority_t priority, + const std::string & url, + HttpOptions * options, + HttpHeaders * headers) +{ + setupCommon(policy_id, priority, url, NULL, options, headers); + mReqMethod = HOR_GET; + + return HttpStatus(); +} + + HttpStatus HttpOpRequest::setupGetByteRange(HttpRequest::policy_t policy_id, HttpRequest::priority_t priority, const std::string & url, @@ -245,30 +263,16 @@ HttpStatus HttpOpRequest::setupGetByteRange(HttpRequest::policy_t policy_id, HttpOptions * options, HttpHeaders * headers) { - HttpStatus status; - - mProcFlags = 0; - mReqPolicy = policy_id; - mReqPriority = priority; + setupCommon(policy_id, priority, url, NULL, options, headers); mReqMethod = HOR_GET; - mReqURL = url; mReqOffset = offset; mReqLength = len; if (offset || len) { mProcFlags |= PF_SCAN_RANGE_HEADER; } - if (headers && ! mReqHeaders) - { - headers->addRef(); - mReqHeaders = headers; - } - if (options && ! mReqOptions) - { - mReqOptions = new HttpOptions(*options); - } - return status; + return HttpStatus(); } @@ -279,29 +283,10 @@ HttpStatus HttpOpRequest::setupPost(HttpRequest::policy_t policy_id, HttpOptions * options, HttpHeaders * headers) { - HttpStatus status; - - mProcFlags = 0; - mReqPolicy = policy_id; - mReqPriority = priority; + setupCommon(policy_id, priority, url, body, options, headers); mReqMethod = HOR_POST; - mReqURL = url; - if (body) - { - body->addRef(); - mReqBody = body; - } - if (headers && ! mReqHeaders) - { - headers->addRef(); - mReqHeaders = headers; - } - if (options && ! mReqOptions) - { - mReqOptions = new HttpOptions(*options); - } - return status; + return HttpStatus(); } @@ -312,12 +297,23 @@ HttpStatus HttpOpRequest::setupPut(HttpRequest::policy_t policy_id, HttpOptions * options, HttpHeaders * headers) { - HttpStatus status; + setupCommon(policy_id, priority, url, body, options, headers); + mReqMethod = HOR_PUT; + + return HttpStatus(); +} + +void HttpOpRequest::setupCommon(HttpRequest::policy_t policy_id, + HttpRequest::priority_t priority, + const std::string & url, + BufferArray * body, + HttpOptions * options, + HttpHeaders * headers) +{ mProcFlags = 0; mReqPolicy = policy_id; mReqPriority = priority; - mReqMethod = HOR_PUT; mReqURL = url; if (body) { @@ -331,10 +327,14 @@ HttpStatus HttpOpRequest::setupPut(HttpRequest::policy_t policy_id, } if (options && ! mReqOptions) { - mReqOptions = new HttpOptions(*options); + options->addRef(); + mReqOptions = options; + if (options->getWantHeaders()) + { + mProcFlags |= PF_SAVE_HEADERS; + } + mTracing = (std::max)(mTracing, llclamp(options->getTrace(), 0, 3)); } - - return status; } @@ -394,30 +394,29 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service) curl_easy_setopt(mCurlHandle, CURLOPT_SSL_VERIFYHOST, 0); const std::string * opt_value(NULL); - if (policy.get(HttpRequest::GP_CA_PATH, opt_value)) + long opt_long(0L); + policy.get(HttpRequest::GP_LLPROXY, &opt_long); + if (opt_long) { - curl_easy_setopt(mCurlHandle, CURLOPT_CAPATH, opt_value->c_str()); + // Use the viewer-based thread-safe API which has a + // fast/safe check for proxy enable. Would like to + // encapsulate this someway... + LLProxy::getInstance()->applyProxySettings(mCurlHandle); } - if (policy.get(HttpRequest::GP_CA_FILE, opt_value)) + else if (policy.get(HttpRequest::GP_HTTP_PROXY, &opt_value)) { - curl_easy_setopt(mCurlHandle, CURLOPT_CAINFO, opt_value->c_str()); + // *TODO: This is fine for now but get fuller socks/ + // authentication thing going later.... + curl_easy_setopt(mCurlHandle, CURLOPT_PROXY, opt_value->c_str()); + curl_easy_setopt(mCurlHandle, CURLOPT_PROXYTYPE, CURLPROXY_HTTP); } - if (policy.get(HttpRequest::GP_HTTP_PROXY, opt_value)) + if (policy.get(HttpRequest::GP_CA_PATH, &opt_value)) { - if (*opt_value == "LLProxy") - { - // Use the viewer-based thread-safe API which has a - // fast/safe check for proxy enable. Would like to - // encapsulate this someway... - LLProxy::getInstance()->applyProxySettings(mCurlHandle); - } - else - { - // *TODO: This is fine for now but get fuller socks/ - // authentication thing going later.... - curl_easy_setopt(mCurlHandle, CURLOPT_PROXY, opt_value->c_str()); - curl_easy_setopt(mCurlHandle, CURLOPT_PROXYTYPE, CURLPROXY_HTTP); - } + curl_easy_setopt(mCurlHandle, CURLOPT_CAPATH, opt_value->c_str()); + } + if (policy.get(HttpRequest::GP_CA_FILE, &opt_value)) + { + curl_easy_setopt(mCurlHandle, CURLOPT_CAINFO, opt_value->c_str()); } switch (mReqMethod) @@ -463,6 +462,14 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service) // *FIXME: fail out here break; } + + // Tracing + if (mTracing > 1) + { + curl_easy_setopt(mCurlHandle, CURLOPT_VERBOSE, 1); + curl_easy_setopt(mCurlHandle, CURLOPT_DEBUGDATA, mCurlHandle); + curl_easy_setopt(mCurlHandle, CURLOPT_DEBUGFUNCTION, debugCallback); + } // There's a CURLOPT for this now... if ((mReqOffset || mReqLength) && HOR_GET == mReqMethod) @@ -621,6 +628,101 @@ size_t HttpOpRequest::headerCallback(void * data, size_t size, size_t nmemb, voi return hdr_size; } + +int HttpOpRequest::debugCallback(CURL * handle, curl_infotype info, char * buffer, size_t len, void * userdata) +{ + HttpOpRequest * op(NULL); + curl_easy_getinfo(handle, CURLINFO_PRIVATE, &op); + // *FIXME: check the pointer + + std::string safe_line; + std::string tag; + bool logit(false); + len = (std::min)(len, size_t(256)); // Keep things reasonable in all cases + + switch (info) + { + case CURLINFO_TEXT: + if (op->mTracing > 1) + { + tag = "TEXT"; + escape_libcurl_debug_data(buffer, len, true, safe_line); + logit = true; + } + break; + + case CURLINFO_HEADER_IN: + if (op->mTracing > 1) + { + tag = "HEADERIN"; + escape_libcurl_debug_data(buffer, len, true, safe_line); + logit = true; + } + break; + + case CURLINFO_HEADER_OUT: + if (op->mTracing > 1) + { + tag = "HEADEROUT"; + escape_libcurl_debug_data(buffer, len, true, safe_line); + logit = true; + } + break; + + case CURLINFO_DATA_IN: + if (op->mTracing > 1) + { + tag = "DATAIN"; + logit = true; + if (op->mTracing > 2) + { + escape_libcurl_debug_data(buffer, len, false, safe_line); + } + else + { + std::ostringstream out; + out << len << " Bytes"; + safe_line = out.str(); + } + } + break; + + case CURLINFO_DATA_OUT: + if (op->mTracing > 1) + { + tag = "DATAOUT"; + logit = true; + if (op->mTracing > 2) + { + escape_libcurl_debug_data(buffer, len, false, safe_line); + } + else + { + std::ostringstream out; + out << len << " Bytes"; + safe_line = out.str(); + } + } + break; + + default: + logit = false; + break; + } + + if (logit) + { + LL_INFOS("CoreHttp") << "TRACE, LibcurlDebug, Handle: " + << static_cast(op) + << ", Type: " << tag + << ", Data: " << safe_line + << LL_ENDL; + } + + return 0; +} + + } // end namespace LLCore @@ -694,6 +796,43 @@ char *strtok_r(char *str, const char *delim, char ** savestate) #endif + +void escape_libcurl_debug_data(char * buffer, size_t len, bool scrub, std::string & safe_line) +{ + std::string out; + len = (std::min)(len, size_t(200)); + out.reserve(3 * len); + for (int i(0); i < len; ++i) + { + unsigned char uc(static_cast(buffer[i])); + + if (uc < 32 || uc > 126) + { + if (scrub) + { + out.append(1, ' '); + } + else + { + static const char hex[] = "0123456789ABCDEF"; + char convert[4]; + + convert[0] = '%'; + convert[1] = hex[(uc >> 4) % 16]; + convert[2] = hex[uc % 16]; + convert[3] = '\0'; + out.append(convert); + } + } + else + { + out.append(1, buffer[i]); + } + } + safe_line.swap(out); +} + + } // end anonymous namespace diff --git a/indra/llcorehttp/_httpoprequest.h b/indra/llcorehttp/_httpoprequest.h index 80893beb40..fc2301057c 100644 --- a/indra/llcorehttp/_httpoprequest.h +++ b/indra/llcorehttp/_httpoprequest.h @@ -76,6 +76,12 @@ public: public: // Setup Methods + HttpStatus setupGet(HttpRequest::policy_t policy_id, + HttpRequest::priority_t priority, + const std::string & url, + HttpOptions * options, + HttpHeaders * headers); + HttpStatus setupGetByteRange(HttpRequest::policy_t policy_id, HttpRequest::priority_t priority, const std::string & url, @@ -103,15 +109,23 @@ public: virtual HttpStatus cancel(); protected: + void setupCommon(HttpRequest::policy_t policy_id, + HttpRequest::priority_t priority, + const std::string & url, + BufferArray * body, + HttpOptions * options, + HttpHeaders * headers); + static size_t writeCallback(void * data, size_t size, size_t nmemb, void * userdata); static size_t readCallback(void * data, size_t size, size_t nmemb, void * userdata); static size_t headerCallback(void * data, size_t size, size_t nmemb, void * userdata); + static int debugCallback(CURL *, curl_infotype info, char * buffer, size_t len, void * userdata); protected: unsigned int mProcFlags; static const unsigned int PF_SCAN_RANGE_HEADER = 0x00000001U; static const unsigned int PF_SAVE_HEADERS = 0x00000002U; - + public: // Request data EMethod mReqMethod; diff --git a/indra/llcorehttp/_httpopsetget.cpp b/indra/llcorehttp/_httpopsetget.cpp index 21e058b2be..c1357f9ae5 100644 --- a/indra/llcorehttp/_httpopsetget.cpp +++ b/indra/llcorehttp/_httpopsetget.cpp @@ -89,8 +89,8 @@ void HttpOpSetGet::stageFromRequest(HttpService * service) } if (mStatus) { - const std::string * value; - if ((mStatus = pol_opt.get(setting, value))) + const std::string * value(NULL); + if ((mStatus = pol_opt.get(setting, &value))) { mStrValue = *value; } diff --git a/indra/llcorehttp/_httppolicy.cpp b/indra/llcorehttp/_httppolicy.cpp index 8ee3f88658..0e08d88276 100644 --- a/indra/llcorehttp/_httppolicy.cpp +++ b/indra/llcorehttp/_httppolicy.cpp @@ -107,6 +107,12 @@ void HttpPolicy::retryOp(HttpOpRequest * op) LL_WARNS("CoreHttp") << "URL op retry #" << op->mPolicyRetries << " being scheduled for " << delta << " uSecs from now." << LL_ENDL; + if (op->mTracing > 0) + { + LL_INFOS("CoreHttp") << "TRACE, ToRetryQueue, Handle: " + << static_cast(op) + << LL_ENDL; + } mState[policy_class].mRetryQueue.push(op); } @@ -224,8 +230,8 @@ bool HttpPolicy::stageAfterCompletion(HttpOpRequest * op) } else if (op->mPolicyRetries) { - LL_WARNS("CoreHttp") << "URL op succeeded after " << op->mPolicyRetries << " retries." - << LL_ENDL; + LL_DEBUGS("CoreHttp") << "URL op succeeded after " << op->mPolicyRetries << " retries." + << LL_ENDL; } op->stageFromActive(mService); @@ -234,4 +240,15 @@ bool HttpPolicy::stageAfterCompletion(HttpOpRequest * op) } +int HttpPolicy::getReadyCount(HttpRequest::policy_t policy_class) +{ + if (policy_class < HttpRequest::POLICY_CLASS_LIMIT) + { + return (mState[policy_class].mReadyQueue.size() + + mState[policy_class].mRetryQueue.size()); + } + return 0; +} + + } // end namespace LLCore diff --git a/indra/llcorehttp/_httppolicy.h b/indra/llcorehttp/_httppolicy.h index 73c22bab78..4114f64848 100644 --- a/indra/llcorehttp/_httppolicy.h +++ b/indra/llcorehttp/_httppolicy.h @@ -97,7 +97,10 @@ public: } void setPolicies(const HttpPolicyGlobal & global); - + + // Get ready counts for a particular class + int getReadyCount(HttpRequest::policy_t policy_class); + protected: struct State { diff --git a/indra/llcorehttp/_httppolicyglobal.cpp b/indra/llcorehttp/_httppolicyglobal.cpp index d95d73cfba..6b1de38fd6 100644 --- a/indra/llcorehttp/_httppolicyglobal.cpp +++ b/indra/llcorehttp/_httppolicyglobal.cpp @@ -33,7 +33,9 @@ namespace LLCore HttpPolicyGlobal::HttpPolicyGlobal() : mSetMask(0UL), - mConnectionLimit(32L) + mConnectionLimit(32L), + mTrace(0), + mUseLLProxy(0) {} @@ -50,6 +52,8 @@ HttpPolicyGlobal & HttpPolicyGlobal::operator=(const HttpPolicyGlobal & other) mCAPath = other.mCAPath; mCAFile = other.mCAFile; mHttpProxy = other.mHttpProxy; + mTrace = other.mTrace; + mUseLLProxy = other.mUseLLProxy; } return *this; } @@ -63,6 +67,14 @@ HttpStatus HttpPolicyGlobal::set(HttpRequest::EGlobalPolicy opt, long value) mConnectionLimit = value; break; + case HttpRequest::GP_TRACE: + mTrace = llclamp(value, 0L, 3L); + break; + + case HttpRequest::GP_LLPROXY: + mUseLLProxy = llclamp(value, 0L, 1L); + break; + default: return HttpStatus(HttpStatus::LLCORE, HE_INVALID_ARG); } @@ -97,54 +109,64 @@ HttpStatus HttpPolicyGlobal::set(HttpRequest::EGlobalPolicy opt, const std::stri } -HttpStatus HttpPolicyGlobal::get(HttpRequest::EGlobalPolicy opt, long & value) +HttpStatus HttpPolicyGlobal::get(HttpRequest::EGlobalPolicy opt, long * value) { static const HttpStatus not_set(HttpStatus::LLCORE, HE_OPT_NOT_SET); + long * src(NULL); switch (opt) { case HttpRequest::GP_CONNECTION_LIMIT: - if (! (mSetMask & (1UL << int(opt)))) - return not_set; - value = mConnectionLimit; + src = &mConnectionLimit; + break; + + case HttpRequest::GP_TRACE: + src = &mTrace; + break; + + case HttpRequest::GP_LLPROXY: + src = &mUseLLProxy; break; default: return HttpStatus(HttpStatus::LLCORE, HE_INVALID_ARG); } + if (! (mSetMask & (1UL << int(opt)))) + return not_set; + + *value = *src; return HttpStatus(); } -HttpStatus HttpPolicyGlobal::get(HttpRequest::EGlobalPolicy opt, const std::string *& value) +HttpStatus HttpPolicyGlobal::get(HttpRequest::EGlobalPolicy opt, const std::string ** value) { static const HttpStatus not_set(HttpStatus::LLCORE, HE_OPT_NOT_SET); + const std::string * src(NULL); switch (opt) { case HttpRequest::GP_CA_PATH: - if (! (mSetMask & (1UL << int(opt)))) - return not_set; - value = &mCAPath; + src = &mCAPath; break; case HttpRequest::GP_CA_FILE: - if (! (mSetMask & (1UL << int(opt)))) - return not_set; - value = &mCAFile; + src = &mCAFile; break; case HttpRequest::GP_HTTP_PROXY: - if (! (mSetMask & (1UL << int(opt)))) - return not_set; - value = &mHttpProxy; + src = &mHttpProxy; break; default: return HttpStatus(HttpStatus::LLCORE, HE_INVALID_ARG); } + if (! (mSetMask & (1UL << int(opt)))) + return not_set; + + *value = src; return HttpStatus(); } diff --git a/indra/llcorehttp/_httppolicyglobal.h b/indra/llcorehttp/_httppolicyglobal.h index f4bb4d4b25..a50d0e4188 100644 --- a/indra/llcorehttp/_httppolicyglobal.h +++ b/indra/llcorehttp/_httppolicyglobal.h @@ -48,8 +48,8 @@ private: public: HttpStatus set(HttpRequest::EGlobalPolicy opt, long value); HttpStatus set(HttpRequest::EGlobalPolicy opt, const std::string & value); - HttpStatus get(HttpRequest::EGlobalPolicy opt, long & value); - HttpStatus get(HttpRequest::EGlobalPolicy opt, const std::string *& value); + HttpStatus get(HttpRequest::EGlobalPolicy opt, long * value); + HttpStatus get(HttpRequest::EGlobalPolicy opt, const std::string ** value); public: unsigned long mSetMask; @@ -57,6 +57,8 @@ public: std::string mCAPath; std::string mCAFile; std::string mHttpProxy; + long mTrace; + long mUseLLProxy; }; // end class HttpPolicyGlobal } // end namespace LLCore diff --git a/indra/llcorehttp/_httpservice.cpp b/indra/llcorehttp/_httpservice.cpp index 920a3f3b6d..beba8f08f4 100644 --- a/indra/llcorehttp/_httpservice.cpp +++ b/indra/llcorehttp/_httpservice.cpp @@ -36,6 +36,7 @@ #include "_thread.h" #include "lltimer.h" +#include "llthread.h" // Tuning parameters @@ -186,8 +187,10 @@ void HttpService::shutdown() void HttpService::threadRun(LLCoreInt::HttpThread * thread) { boost::this_thread::disable_interruption di; - ELoopSpeed loop(REQUEST_SLEEP); + + LLThread::registerThreadID(); + ELoopSpeed loop(REQUEST_SLEEP); while (! mExitRequested) { loop = processRequestQueue(loop); @@ -226,6 +229,19 @@ HttpService::ELoopSpeed HttpService::processRequestQueue(ELoopSpeed loop) // Process operation if (! mExitRequested) { + // Setup for subsequent tracing + long tracing(0); + mPolicy->getGlobalOptions().get(HttpRequest::GP_TRACE, &tracing); + op->mTracing = (std::max)(op->mTracing, int(tracing)); + + if (op->mTracing > 0) + { + LL_INFOS("CoreHttp") << "TRACE, FromRequestQueue, Handle: " + << static_cast(op) + << LL_ENDL; + } + + // Stage op->stageFromRequest(this); } diff --git a/indra/llcorehttp/httpoptions.cpp b/indra/llcorehttp/httpoptions.cpp index 15b505f5bf..155fbda7f1 100644 --- a/indra/llcorehttp/httpoptions.cpp +++ b/indra/llcorehttp/httpoptions.cpp @@ -33,13 +33,15 @@ namespace LLCore HttpOptions::HttpOptions() : RefCounted(true), - mWantHeaders(false) + mWantHeaders(false), + mTracing(0) {} HttpOptions::HttpOptions(const HttpOptions & rhs) : RefCounted(true), - mWantHeaders(rhs.mWantHeaders) + mWantHeaders(rhs.mWantHeaders), + mTracing(rhs.mTracing) {} @@ -47,4 +49,16 @@ HttpOptions::~HttpOptions() {} +void HttpOptions::setWantHeaders() +{ + mWantHeaders = true; +} + + +void HttpOptions::setTrace(long level) +{ + mTracing = int(level); +} + + } // end namespace LLCore diff --git a/indra/llcorehttp/httpoptions.h b/indra/llcorehttp/httpoptions.h index 267a982dd5..0b9dfdc1de 100644 --- a/indra/llcorehttp/httpoptions.h +++ b/indra/llcorehttp/httpoptions.h @@ -67,11 +67,21 @@ protected: void operator=(const HttpOptions &); // Not defined public: + void setWantHeaders(); + bool getWantHeaders() const + { + return mWantHeaders; + } + + void setTrace(long level); + int getTrace() const + { + return mTracing; + } protected: - // *TODO: add some options bool mWantHeaders; - + long int mTracing; }; // end class HttpOptions diff --git a/indra/llcorehttp/httprequest.cpp b/indra/llcorehttp/httprequest.cpp index 089eee76f3..2036ecfd1c 100644 --- a/indra/llcorehttp/httprequest.cpp +++ b/indra/llcorehttp/httprequest.cpp @@ -135,6 +135,33 @@ HttpStatus HttpRequest::getStatus() const } +HttpHandle HttpRequest::requestGet(policy_t policy_id, + priority_t priority, + const std::string & url, + HttpOptions * options, + HttpHeaders * headers, + HttpHandler * user_handler) +{ + HttpStatus status; + HttpHandle handle(LLCORE_HTTP_HANDLE_INVALID); + + HttpOpRequest * op = new HttpOpRequest(); + if (! (status = op->setupGet(policy_id, priority, url, options, headers))) + { + op->release(); + mLastReqStatus = status; + return handle; + } + op->setReplyPath(mReplyQueue, user_handler); + mRequestQueue->addOp(op); // transfers refcount + + mLastReqStatus = status; + handle = static_cast(op); + + return handle; +} + + HttpHandle HttpRequest::requestGetByteRange(policy_t policy_id, priority_t priority, const std::string & url, diff --git a/indra/llcorehttp/httprequest.h b/indra/llcorehttp/httprequest.h index a953aa28d0..4e78ed3719 100644 --- a/indra/llcorehttp/httprequest.h +++ b/indra/llcorehttp/httprequest.h @@ -111,10 +111,43 @@ public: /// Maximum number of connections the library will use to /// perform operations. This is somewhat soft as the underlying /// transport will cache some connections (up to 5). - GP_CONNECTION_LIMIT, ///< Takes long giving number of connections - GP_CA_PATH, ///< System path/directory where SSL certs are stored. - GP_CA_FILE, ///< System path/file containing certs. - GP_HTTP_PROXY ///< String giving host/port to use for HTTP proxy + + /// A long value setting the maximum number of connections + /// allowed over all policy classes. Note that this will be + /// a somewhat soft value. There may be an additional five + /// connections per policy class depending upon runtime + /// behavior. + GP_CONNECTION_LIMIT, + + /// String containing a system-appropriate directory name + /// where SSL certs are stored. + GP_CA_PATH, + + /// String giving a full path to a file containing SSL certs. + GP_CA_FILE, + + /// String of host/port to use as simple HTTP proxy. This is + /// going to change in the future into something more elaborate + /// that may support richer schemes. + GP_HTTP_PROXY, + + /// Long value that if non-zero enables the use of the + /// traditional LLProxy code for http/socks5 support. If + /// enabled, has priority over GP_HTTP_PROXY. + GP_LLPROXY, + + /// Long value setting the logging trace level for the + /// library. Possible values are: + /// 0 - No tracing (default) + /// 1 - Basic tracing of request start, stop and major events. + /// 2 - Connection, header and payload size information from + /// HTTP transactions. + /// 3 - Partial logging of payload itself. + /// + /// These values are also used in the trace modes for + /// individual requests in HttpOptions. Also be aware that + /// tracing tends to impact performance of the viewer. + GP_TRACE }; /// Set a parameter on a global policy option. Calls @@ -133,7 +166,7 @@ public: /// the class in other methods. If -1, an error /// occurred and @see getStatus() may provide more /// detail on the reason. - policy_t createPolicyClass(); + static policy_t createPolicyClass(); enum EClassPolicy { @@ -157,7 +190,7 @@ public: /// @param opt Enum of option to be set. /// @param value Desired value of option. /// @return Standard status code. - HttpStatus setPolicyClassOption(policy_t policy_id, EClassPolicy opt, long value); + static HttpStatus setPolicyClassOption(policy_t policy_id, EClassPolicy opt, long value); /// @} @@ -176,6 +209,36 @@ public: /// HttpStatus getStatus() const; + /// Queue a full HTTP GET request to be issued for entire entity. + /// The request is queued and serviced by the working thread and + /// notification of completion delivered to the optional HttpHandler + /// argument during @see update() calls. + /// + /// With a valid handle returned, it can be used to reference the + /// request in other requests (like cancellation) and will be an + /// argument when any HttpHandler object is invoked. + /// + /// @param policy_id Default or user-defined policy class under + /// which this request is to be serviced. + /// @param priority Standard priority scheme inherited from + /// Indra code base (U32-type scheme). + /// @param url + /// @param options (optional) + /// @param headers (optional) + /// @param handler (optional) + /// @return The handle of the request if successfully + /// queued or LLCORE_HTTP_HANDLE_INVALID if the + /// request could not be queued. In the latter + /// case, @see getStatus() will return more info. + /// + HttpHandle requestGet(policy_t policy_id, + priority_t priority, + const std::string & url, + HttpOptions * options, + HttpHeaders * headers, + HttpHandler * handler); + + /// Queue a full HTTP GET request to be issued with a 'Range' header. /// The request is queued and serviced by the working thread and /// notification of completion delivered to the optional HttpHandler diff --git a/indra/llcorehttp/tests/test_httprequest.hpp b/indra/llcorehttp/tests/test_httprequest.hpp index 61698f34d8..5b04796c8a 100644 --- a/indra/llcorehttp/tests/test_httprequest.hpp +++ b/indra/llcorehttp/tests/test_httprequest.hpp @@ -395,7 +395,7 @@ void HttpRequestTestObjectType::test<5>() { ScopedCurlInit ready; - set_test_name("HttpRequest GET + Stop execution"); + set_test_name("HttpRequest GET to dead port + Stop execution"); // Handler can be stack-allocated *if* there are no dangling // references to it after completion of this method. @@ -496,6 +496,7 @@ void HttpRequestTestObjectType::test<5>() } } + template <> template <> void HttpRequestTestObjectType::test<6>() { @@ -517,6 +518,114 @@ void HttpRequestTestObjectType::test<6>() HttpRequest * req = NULL; + try + { + // Get singletons created + HttpRequest::createService(); + + // Start threading early so that thread memory is invariant + // over the test. + HttpRequest::startThread(); + + // create a new ref counted object with an implicit reference + req = new HttpRequest(); + ensure("Memory allocated on construction", mMemTotal < GetMemTotal()); + + // Issue a GET that *can* connect + mStatus = HttpStatus(200); + HttpHandle handle = req->requestGet(HttpRequest::DEFAULT_POLICY_ID, + 0U, + url_base, + NULL, + NULL, + &handler); + ensure("Valid handle returned for ranged request", handle != LLCORE_HTTP_HANDLE_INVALID); + + // Run the notification pump. + int count(0); + int limit(10); + while (count++ < limit && mHandlerCalls < 1) + { + req->update(1000); + usleep(100000); + } + ensure("Request executed in reasonable time", count < limit); + ensure("One handler invocation for request", mHandlerCalls == 1); + + // Okay, request a shutdown of the servicing thread + mStatus = HttpStatus(); + handle = req->requestStopThread(&handler); + ensure("Valid handle returned for second request", handle != LLCORE_HTTP_HANDLE_INVALID); + + // Run the notification pump again + count = 0; + limit = 10; + while (count++ < limit && mHandlerCalls < 2) + { + req->update(1000); + usleep(100000); + } + ensure("Second request executed in reasonable time", count < limit); + ensure("Second handler invocation", mHandlerCalls == 2); + + // See that we actually shutdown the thread + count = 0; + limit = 10; + while (count++ < limit && ! HttpService::isStopped()) + { + usleep(100000); + } + ensure("Thread actually stopped running", HttpService::isStopped()); + + // release the request object + delete req; + req = NULL; + + // Shut down service + HttpRequest::destroyService(); + + ensure("Two handler calls on the way out", 2 == mHandlerCalls); + +#if defined(WIN32) + // Can only do this memory test on Windows. On other platforms, + // the LL logging system holds on to memory and produces what looks + // like memory leaks... + + // printf("Old mem: %d, New mem: %d\n", mMemTotal, GetMemTotal()); + ensure("Memory usage back to that at entry", mMemTotal == GetMemTotal()); +#endif + } + catch (...) + { + stop_thread(req); + delete req; + HttpRequest::destroyService(); + throw; + } +} + + +template <> template <> +void HttpRequestTestObjectType::test<7>() +{ + ScopedCurlInit ready; + + std::string url_base(get_base_url()); + std::cerr << "Base: " << url_base << std::endl; + + set_test_name("HttpRequest GET with Range: header to real service"); + + // Handler can be stack-allocated *if* there are no dangling + // references to it after completion of this method. + // Create before memory record as the string copy will bump numbers. + TestHandler2 handler(this, "handler"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + mHandlerCalls = 0; + + HttpRequest * req = NULL; + try { // Get singletons created @@ -605,8 +714,9 @@ void HttpRequestTestObjectType::test<6>() } } + template <> template <> -void HttpRequestTestObjectType::test<7>() +void HttpRequestTestObjectType::test<8>() { ScopedCurlInit ready; @@ -725,7 +835,7 @@ void HttpRequestTestObjectType::test<7>() } template <> template <> -void HttpRequestTestObjectType::test<8>() +void HttpRequestTestObjectType::test<9>() { ScopedCurlInit ready; @@ -843,6 +953,118 @@ void HttpRequestTestObjectType::test<8>() } } +template <> template <> +void HttpRequestTestObjectType::test<10>() +{ + ScopedCurlInit ready; + + std::string url_base(get_base_url()); + std::cerr << "Base: " << url_base << std::endl; + + set_test_name("HttpRequest GET with some tracing"); + + // Handler can be stack-allocated *if* there are no dangling + // references to it after completion of this method. + // Create before memory record as the string copy will bump numbers. + TestHandler2 handler(this, "handler"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + mHandlerCalls = 0; + + HttpRequest * req = NULL; + + try + { + // Get singletons created + HttpRequest::createService(); + + // Enable tracing + HttpRequest::setPolicyGlobalOption(LLCore::HttpRequest::GP_TRACE, 2); + + // Start threading early so that thread memory is invariant + // over the test. + HttpRequest::startThread(); + + // create a new ref counted object with an implicit reference + req = new HttpRequest(); + ensure("Memory allocated on construction", mMemTotal < GetMemTotal()); + + // Issue a GET that *can* connect + mStatus = HttpStatus(200); + HttpHandle handle = req->requestGetByteRange(HttpRequest::DEFAULT_POLICY_ID, + 0U, + url_base, + 0, + 0, + NULL, + NULL, + &handler); + ensure("Valid handle returned for ranged request", handle != LLCORE_HTTP_HANDLE_INVALID); + + // Run the notification pump. + int count(0); + int limit(10); + while (count++ < limit && mHandlerCalls < 1) + { + req->update(1000); + usleep(100000); + } + ensure("Request executed in reasonable time", count < limit); + ensure("One handler invocation for request", mHandlerCalls == 1); + + // Okay, request a shutdown of the servicing thread + mStatus = HttpStatus(); + handle = req->requestStopThread(&handler); + ensure("Valid handle returned for second request", handle != LLCORE_HTTP_HANDLE_INVALID); + + // Run the notification pump again + count = 0; + limit = 10; + while (count++ < limit && mHandlerCalls < 2) + { + req->update(1000); + usleep(100000); + } + ensure("Second request executed in reasonable time", count < limit); + ensure("Second handler invocation", mHandlerCalls == 2); + + // See that we actually shutdown the thread + count = 0; + limit = 10; + while (count++ < limit && ! HttpService::isStopped()) + { + usleep(100000); + } + ensure("Thread actually stopped running", HttpService::isStopped()); + + // release the request object + delete req; + req = NULL; + + // Shut down service + HttpRequest::destroyService(); + + ensure("Two handler calls on the way out", 2 == mHandlerCalls); + +#if defined(WIN32) + // Can only do this memory test on Windows. On other platforms, + // the LL logging system holds on to memory and produces what looks + // like memory leaks... + + // printf("Old mem: %d, New mem: %d\n", mMemTotal, GetMemTotal()); + ensure("Memory usage back to that at entry", mMemTotal == GetMemTotal()); +#endif + } + catch (...) + { + stop_thread(req); + delete req; + HttpRequest::destroyService(); + throw; + } +} + } // end namespace tut namespace diff --git a/indra/newview/llappviewer.cpp b/indra/newview/llappviewer.cpp index e2c13e77e3..430dd89c3e 100644 --- a/indra/newview/llappviewer.cpp +++ b/indra/newview/llappviewer.cpp @@ -5356,11 +5356,9 @@ void CoreHttp::init() << LL_ENDL; } - mRequest = new LLCore::HttpRequest; - // Point to our certs or SSH/https: will fail on connect - status = mRequest->setPolicyGlobalOption(LLCore::HttpRequest::GP_CA_FILE, - gDirUtilp->getCAFile()); + status = LLCore::HttpRequest::setPolicyGlobalOption(LLCore::HttpRequest::GP_CA_FILE, + gDirUtilp->getCAFile()); if (! status) { LL_ERRS("Init") << "Failed to set CA File for HTTP services. Reason: " @@ -5371,15 +5369,22 @@ void CoreHttp::init() // Establish HTTP Proxy. "LLProxy" is a special string which directs // the code to use LLProxy::applyProxySettings() to establish any // HTTP or SOCKS proxy for http operations. - status = mRequest->setPolicyGlobalOption(LLCore::HttpRequest::GP_HTTP_PROXY, - std::string("LLProxy")); + status = LLCore::HttpRequest::setPolicyGlobalOption(LLCore::HttpRequest::GP_LLPROXY, 1); if (! status) { LL_ERRS("Init") << "Failed to set HTTP proxy for HTTP services. Reason: " << status.toString() << LL_ENDL; } - + + // Tracing levels for library & libcurl (note that 2 & 3 are beyond spammy): + // 0 - None + // 1 - Basic start, stop simple transitions + // 2 - libcurl CURLOPT_VERBOSE mode with brief lines + // 3 - with partial data content + status = LLCore::HttpRequest::setPolicyGlobalOption(LLCore::HttpRequest::GP_TRACE, 0); + + // Kick the thread status = LLCore::HttpRequest::startThread(); if (! status) { @@ -5388,6 +5393,7 @@ void CoreHttp::init() << LL_ENDL; } + mRequest = new LLCore::HttpRequest; } diff --git a/indra/newview/lltexturefetch.cpp b/indra/newview/lltexturefetch.cpp index f5e7540e85..664af02f78 100644 --- a/indra/newview/lltexturefetch.cpp +++ b/indra/newview/lltexturefetch.cpp @@ -431,6 +431,22 @@ private: void lockWorkMutex() { mWorkMutex.lock(); } void unlockWorkMutex() { mWorkMutex.unlock(); } + // Locks: Mw + void acquireHttpSemaphore() + { + llassert(! mHttpHasResource); + mHttpHasResource = true; + --mFetcher->mHttpSemaphore; + } + + // Locks: Mw + void releaseHttpSemaphore() + { + llassert(mHttpHasResource); + mHttpHasResource = false; + ++mFetcher->mHttpSemaphore; + } + private: enum e_state // mState { @@ -444,8 +460,9 @@ private: CACHE_POST, LOAD_FROM_NETWORK, LOAD_FROM_SIMULATOR, - SEND_HTTP_REQ, // Commit to sending as HTTP WAIT_HTTP_RESOURCE, // Waiting for HTTP resources + WAIT_HTTP_RESOURCE2, // Waiting for HTTP resources + SEND_HTTP_REQ, // Commit to sending as HTTP WAIT_HTTP_REQ, // Request sent, wait for completion DECODE_IMAGE, DECODE_IMAGE_UPDATE, @@ -532,7 +549,7 @@ private: bool mHttpActive; // Active request to http library unsigned int mHttpReplySize; unsigned int mHttpReplyOffset; - bool mHttpReleased; // Has been released from resource wait once + bool mHttpHasResource; // Counts against Fetcher's mHttpSemaphore }; ////////////////////////////////////////////////////////////////////////////// @@ -768,8 +785,9 @@ const char* LLTextureFetchWorker::sStateDescs[] = { "CACHE_POST", "LOAD_FROM_NETWORK", "LOAD_FROM_SIMULATOR", - "SEND_HTTP_REQ", "WAIT_HTTP_RESOURCE", + "WAIT_HTTP_RESOURCE2", + "SEND_HTTP_REQ", "WAIT_HTTP_REQ", "DECODE_IMAGE", "DECODE_IMAGE_UPDATE", @@ -836,7 +854,7 @@ LLTextureFetchWorker::LLTextureFetchWorker(LLTextureFetch* fetcher, mHttpActive(false), mHttpReplySize(0U), mHttpReplyOffset(0U), - mHttpReleased(true) + mHttpHasResource(false) { mCanUseNET = mUrl.empty() ; @@ -860,6 +878,10 @@ LLTextureFetchWorker::~LLTextureFetchWorker() llassert_always(!haveWork()); lockWorkMutex(); // +Mw (should be useless) + if (mHttpHasResource) + { + releaseHttpSemaphore(); + } if (mHttpActive) { // Issue a cancel on a live request... @@ -1126,7 +1148,7 @@ bool LLTextureFetchWorker::doWork(S32 param) llwarns << "Unknown URL Type: " << mUrl << llendl; } setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); - mState = SEND_HTTP_REQ; + mState = WAIT_HTTP_RESOURCE; } else { @@ -1223,7 +1245,7 @@ bool LLTextureFetchWorker::doWork(S32 param) } if (mCanUseHTTP && !mUrl.empty()) { - mState = SEND_HTTP_REQ; + mState = WAIT_HTTP_RESOURCE; setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); if(mWriteToCacheState != NOT_WRITE) { @@ -1287,31 +1309,38 @@ bool LLTextureFetchWorker::doWork(S32 param) return false; } - if (mState == SEND_HTTP_REQ) + if (mState == WAIT_HTTP_RESOURCE) { - if (! mCanUseHTTP) - { - return true; // abort - } - // NOTE: // control the number of the http requests issued for: // 1, not openning too many file descriptors at the same time; // 2, control the traffic of http so udp gets bandwidth. // - if (! mHttpReleased) + // If it looks like we're busy, keep this request here. + // Otherwise, advance into the HTTP states. + if (mFetcher->mHttpSemaphore <= 0 || mFetcher->getHttpWaitersCount()) { - // If this request hasn't been released before and it looks like - // we're busy, put this request into resource wait and allow something - // else to come to the front. - if (mFetcher->getNumHTTPRequests() >= HTTP_REQUESTS_IN_QUEUE_HIGH_WATER || - mFetcher->getHttpWaitersCount()) - { - mState = WAIT_HTTP_RESOURCE; - setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); - mFetcher->addHttpWaiter(this->mID); - return false; - } + mState = WAIT_HTTP_RESOURCE2; + setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); + mFetcher->addHttpWaiter(this->mID); + return false; + } + mState = SEND_HTTP_REQ; + acquireHttpSemaphore(); + } + + if (mState == WAIT_HTTP_RESOURCE2) + { + // Just idle it if we make it to the head... + return false; + } + + if (mState == SEND_HTTP_REQ) + { + if (! mCanUseHTTP) + { + releaseHttpSemaphore(); + return true; // abort } mFetcher->removeFromNetworkQueue(this, false); @@ -1327,10 +1356,12 @@ bool LLTextureFetchWorker::doWork(S32 param) // We already have all the data, just decode it mLoadedDiscard = mFormattedImage->getDiscardLevel(); mState = DECODE_IMAGE; + releaseHttpSemaphore(); return false; } else { + releaseHttpSemaphore(); return true; // abort. } } @@ -1365,6 +1396,7 @@ bool LLTextureFetchWorker::doWork(S32 param) { llwarns << "HTTP GET request failed for " << mID << llendl; resetFormattedData(); + releaseHttpSemaphore(); return true; // failed } @@ -1377,13 +1409,6 @@ bool LLTextureFetchWorker::doWork(S32 param) // fall through } - if (mState == WAIT_HTTP_RESOURCE) - { - // Nothing to do until releaseHttpWaiters() puts us back - // into the flow... - return false; - } - if (mState == WAIT_HTTP_REQ) { if (mLoaded) @@ -1401,6 +1426,7 @@ bool LLTextureFetchWorker::doWork(S32 param) mState = INIT; mCanUseHTTP = false; setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); + releaseHttpSemaphore(); return false; } } @@ -1423,12 +1449,14 @@ bool LLTextureFetchWorker::doWork(S32 param) // Use available data mLoadedDiscard = mFormattedImage->getDiscardLevel(); mState = DECODE_IMAGE; + releaseHttpSemaphore(); return false; } // Fail harder resetFormattedData(); mState = DONE; + releaseHttpSemaphore(); return true; // failed } @@ -1443,6 +1471,7 @@ bool LLTextureFetchWorker::doWork(S32 param) // abort. mState = DONE; + releaseHttpSemaphore(); return true; } @@ -1491,6 +1520,7 @@ bool LLTextureFetchWorker::doWork(S32 param) mWriteToCacheState = SHOULD_WRITE ; } setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); + releaseHttpSemaphore(); return false; } else @@ -2137,7 +2167,8 @@ LLTextureFetch::LLTextureFetch(LLTextureCache* cache, LLImageDecodeThread* image mQAMode(qa_mode), mHttpRequest(NULL), mHttpOptions(NULL), - mHttpHeaders(NULL) + mHttpHeaders(NULL), + mHttpSemaphore(HTTP_REQUESTS_IN_QUEUE_HIGH_WATER) { mMaxBandwidth = gSavedSettings.getF32("ThrottleBandwidthKBPS"); mTextureInfo.setUpLogging(gSavedSettings.getBOOL("LogTextureDownloadsToViewerLog"), gSavedSettings.getBOOL("LogTextureDownloadsToSimulator"), gSavedSettings.getU32("TextureLoggingThreshold")); @@ -3155,12 +3186,11 @@ void LLTextureFetch::removeHttpWaiter(const LLUUID & tid) // Locks: -Mw (must not hold any worker when called) void LLTextureFetch::releaseHttpWaiters() { - if (HTTP_REQUESTS_IN_QUEUE_LOW_WATER < getNumHTTPRequests()) + if (mHttpSemaphore < HTTP_REQUESTS_IN_QUEUE_LOW_WATER) return; // Quickly make a copy of all the LLUIDs. Get off the // mutex as early as possible. - typedef std::vector uuid_vec_t; uuid_vec_t tids; @@ -3171,13 +3201,12 @@ void LLTextureFetch::releaseHttpWaiters() return; const size_t limit(mHttpWaitResource.size()); - tids.resize(limit); - wait_http_res_queue_t::iterator iter(mHttpWaitResource.begin()); - for (int i(0); - i < limit && mHttpWaitResource.end() != iter; - ++i, ++iter) + tids.reserve(limit); + for (wait_http_res_queue_t::iterator iter(mHttpWaitResource.begin()); + mHttpWaitResource.end() != iter; + ++iter) { - tids[i] = *iter; + tids.push_back(*iter); } } // -Mfnq @@ -3196,28 +3225,29 @@ void LLTextureFetch::releaseHttpWaiters() tids2.insert(worker); } } + tids.clear(); // Release workers up to the high water mark. Since we aren't // holding any locks at this point, we can be in competition // with other callers. Do defensive things like getting // refreshed counts of requests and checking if someone else // has moved any worker state around.... - tids.clear(); for (worker_set_t::iterator iter2(tids2.begin()); - tids2.end() != iter2 && 0 < (HTTP_REQUESTS_IN_QUEUE_HIGH_WATER - getNumHTTPRequests()); + tids2.end() != iter2 && mHttpSemaphore > 0; ++iter2) { LLTextureFetchWorker * worker(* iter2); worker->lockWorkMutex(); // +Mw - if (LLTextureFetchWorker::WAIT_HTTP_RESOURCE != worker->mState) + if (LLTextureFetchWorker::WAIT_HTTP_RESOURCE2 != worker->mState) { worker->unlockWorkMutex(); // -Mw continue; } - worker->mHttpReleased = true; + worker->mState = LLTextureFetchWorker::SEND_HTTP_REQ; worker->setPriority(LLWorkerThread::PRIORITY_HIGH | worker->mWorkPriority); + worker->acquireHttpSemaphore(); worker->unlockWorkMutex(); // -Mw removeHttpWaiter(worker->mID); @@ -3456,7 +3486,7 @@ TFReqSendMetrics::doWork(LLTextureFetch * fetcher) } // In QA mode, Metrics submode, log the result for ease of testing - if (fetcher->isQAMode() || true) + if (fetcher->isQAMode()) { LL_INFOS("Textures") << merged_llsd << LL_ENDL; } diff --git a/indra/newview/lltexturefetch.h b/indra/newview/lltexturefetch.h index 4ee13d171e..50e3181623 100644 --- a/indra/newview/lltexturefetch.h +++ b/indra/newview/lltexturefetch.h @@ -312,6 +312,15 @@ private: LLCore::HttpOptions * mHttpOptions; // Ttf LLCore::HttpHeaders * mHttpHeaders; // Ttf + // We use a resource semaphore to keep HTTP requests in + // WAIT_HTTP_RESOURCE2 if there aren't sufficient slots in the + // transport. This keeps them near where they can be cheaply + // reprioritized rather than dumping them all across a thread + // where it's more expensive to get at them. Requests in either + // SEND_HTTP_REQ or WAIT_HTTP_REQ charge against the semaphore + // and tracking state transitions is critical to liveness. + int mHttpSemaphore; // Ttf + typedef std::set wait_http_res_queue_t; wait_http_res_queue_t mHttpWaitResource; // Mfnq diff --git a/indra/newview/lltextureview.cpp b/indra/newview/lltextureview.cpp index 5f1d7829ed..bb1535d23d 100644 --- a/indra/newview/lltextureview.cpp +++ b/indra/newview/lltextureview.cpp @@ -234,15 +234,16 @@ void LLTextureBar::draw() { "DSK", LLColor4::blue }, // CACHE_POST { "NET", LLColor4::green }, // LOAD_FROM_NETWORK { "SIM", LLColor4::green }, // LOAD_FROM_SIMULATOR + { "HTW", LLColor4::green }, // WAIT_HTTP_RESOURCE + { "HTW", LLColor4::green }, // WAIT_HTTP_RESOURCE2 { "REQ", LLColor4::yellow },// SEND_HTTP_REQ - { "HTW", LLColor4::green }, // WAIT_HTTP_RES { "HTP", LLColor4::green }, // WAIT_HTTP_REQ { "DEC", LLColor4::yellow },// DECODE_IMAGE { "DEC", LLColor4::green }, // DECODE_IMAGE_UPDATE { "WRT", LLColor4::purple },// WRITE_TO_CACHE { "WRT", LLColor4::orange },// WAIT_ON_WRITE { "END", LLColor4::red }, // DONE -#define LAST_STATE 13 +#define LAST_STATE 14 { "CRE", LLColor4::magenta }, // LAST_STATE+1 { "FUL", LLColor4::green }, // LAST_STATE+2 { "BAD", LLColor4::red }, // LAST_STATE+3 -- cgit v1.3 From 1cf8e785bad3562fac23feeb2343cfaec1b971bc Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Mon, 18 Jun 2012 18:38:24 -0400 Subject: Tidy Texture Console, add cache & resource wait stats, issue stats line to log on exit. With much trial-and-error, cleaned up the banner on the texture console and made everything mostly fit. Added global cache read, cache write and resource wait count events to the console display to show if cache is working. On clean exit, emit a log line to report stats to log file (intended for automated tests, maybe): LLTextureFetch::endThread: CacheReads: 2618, CacheWrites: 117, ResWaits: 0, TotalHTTPReq: 117 --- indra/newview/lltexturefetch.cpp | 57 +++++++++++++++++++++++++++++++++++----- indra/newview/lltexturefetch.h | 17 ++++++++++++ indra/newview/lltextureview.cpp | 34 +++++++++++++++++++----- 3 files changed, 95 insertions(+), 13 deletions(-) (limited to 'indra/newview') diff --git a/indra/newview/lltexturefetch.cpp b/indra/newview/lltexturefetch.cpp index 664af02f78..6e14c5fc63 100644 --- a/indra/newview/lltexturefetch.cpp +++ b/indra/newview/lltexturefetch.cpp @@ -550,6 +550,11 @@ private: unsigned int mHttpReplySize; unsigned int mHttpReplyOffset; bool mHttpHasResource; // Counts against Fetcher's mHttpSemaphore + + // State history + U32 mCacheReadCount; + U32 mCacheWriteCount; + U32 mResourceWaitCount; }; ////////////////////////////////////////////////////////////////////////////// @@ -854,7 +859,10 @@ LLTextureFetchWorker::LLTextureFetchWorker(LLTextureFetch* fetcher, mHttpActive(false), mHttpReplySize(0U), mHttpReplyOffset(0U), - mHttpHasResource(false) + mHttpHasResource(false), + mCacheReadCount(0U), + mCacheWriteCount(0U), + mResourceWaitCount(0U) { mCanUseNET = mUrl.empty() ; @@ -905,6 +913,7 @@ LLTextureFetchWorker::~LLTextureFetchWorker() unlockWorkMutex(); // -Mw mFetcher->removeFromHTTPQueue(mID); mFetcher->removeHttpWaiter(mID); + mFetcher->updateStateStats(mCacheReadCount, mCacheWriteCount, mResourceWaitCount); } // Locks: Mw @@ -1127,6 +1136,7 @@ bool LLTextureFetchWorker::doWork(S32 param) setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); // Set priority first since Responder may change it // read file from local disk + ++mCacheReadCount; std::string filename = mUrl.substr(7, std::string::npos); CacheReadResponder* responder = new CacheReadResponder(mFetcher, mID, mFormattedImage); mCacheReadHandle = mFetcher->mTextureCache->readFromCache(filename, mID, cache_priority, @@ -1136,6 +1146,7 @@ bool LLTextureFetchWorker::doWork(S32 param) { setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); // Set priority first since Responder may change it + ++mCacheReadCount; CacheReadResponder* responder = new CacheReadResponder(mFetcher, mID, mFormattedImage); mCacheReadHandle = mFetcher->mTextureCache->readFromCache(mID, cache_priority, offset, size, responder); @@ -1323,6 +1334,7 @@ bool LLTextureFetchWorker::doWork(S32 param) mState = WAIT_HTTP_RESOURCE2; setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); mFetcher->addHttpWaiter(this->mID); + ++mResourceWaitCount; return false; } mState = SEND_HTTP_REQ; @@ -1646,6 +1658,7 @@ bool LLTextureFetchWorker::doWork(S32 param) U32 cache_priority = mWorkPriority; mWritten = FALSE; mState = WAIT_ON_WRITE; + ++mCacheWriteCount; CacheWriteResponder* responder = new CacheWriteResponder(mFetcher, mID); mCacheWriteHandle = mFetcher->mTextureCache->writeToCache(mID, cache_priority, mFormattedImage->getData(), datasize, @@ -2168,7 +2181,10 @@ LLTextureFetch::LLTextureFetch(LLTextureCache* cache, LLImageDecodeThread* image mHttpRequest(NULL), mHttpOptions(NULL), mHttpHeaders(NULL), - mHttpSemaphore(HTTP_REQUESTS_IN_QUEUE_HIGH_WATER) + mHttpSemaphore(HTTP_REQUESTS_IN_QUEUE_HIGH_WATER), + mTotalCacheReadCount(0U), + mTotalCacheWriteCount(0U), + mTotalResourceWaitCount(0U) { mMaxBandwidth = gSavedSettings.getF32("ThrottleBandwidthKBPS"); mTextureInfo.setUpLogging(gSavedSettings.getBOOL("LogTextureDownloadsToViewerLog"), gSavedSettings.getBOOL("LogTextureDownloadsToSimulator"), gSavedSettings.getU32("TextureLoggingThreshold")); @@ -2661,12 +2677,13 @@ void LLTextureFetch::startThread() } // Threads: Ttf -// -// This detaches the texture fetch thread from the LLCore -// HTTP library but doesn't stop the thread running in that -// library... void LLTextureFetch::endThread() { + LL_INFOS("Texture") << "CacheReads: " << mTotalCacheReadCount + << ", CacheWrites: " << mTotalCacheWriteCount + << ", ResWaits: " << mTotalResourceWaitCount + << ", TotalHTTPReq: " << getTotalNumHTTPRequests() + << LL_ENDL; } // Threads: Ttf @@ -3272,6 +3289,34 @@ int LLTextureFetch::getHttpWaitersCount() } +// Threads: T* +void LLTextureFetch::updateStateStats(U32 cache_read, U32 cache_write, U32 res_wait) +{ + LLMutexLock lock(&mQueueMutex); // +Mfq + + mTotalCacheReadCount += cache_read; + mTotalCacheWriteCount += cache_write; + mTotalResourceWaitCount += res_wait; +} // -Mfq + + +// Threads: T* +void LLTextureFetch::getStateStats(U32 * cache_read, U32 * cache_write, U32 * res_wait) +{ + U32 ret1(0U), ret2(0U), ret3(0U); + + { + LLMutexLock lock(&mQueueMutex); // +Mfq + ret1 = mTotalCacheReadCount; + ret2 = mTotalCacheWriteCount; + ret3 = mTotalResourceWaitCount; + } // -Mfq + + *cache_read = ret1; + *cache_write = ret2; + *res_wait = ret3; +} + ////////////////////////////////////////////////////////////////////////////// // cross-thread command methods diff --git a/indra/newview/lltexturefetch.h b/indra/newview/lltexturefetch.h index 50e3181623..e17c71113a 100644 --- a/indra/newview/lltexturefetch.h +++ b/indra/newview/lltexturefetch.h @@ -186,6 +186,17 @@ public: // Threads: T* int getHttpWaitersCount(); // ---------------------------------- + // Stats management + + // Add given counts to the global totals for the states/requests + // Threads: T* + void updateStateStats(U32 cache_read, U32 cache_write, U32 res_wait); + + // Return the global counts + // Threads: T* + void getStateStats(U32 * cache_read, U32 * cache_write, U32 * res_wait); + + // ---------------------------------- protected: // Threads: T* (but Ttf in practice) @@ -323,6 +334,12 @@ private: typedef std::set wait_http_res_queue_t; wait_http_res_queue_t mHttpWaitResource; // Mfnq + + // Cumulative stats on the states/requests issued by + // textures running through here. + U32 mTotalCacheReadCount; // Mfq + U32 mTotalCacheWriteCount; // Mfq + U32 mTotalResourceWaitCount; // Mfq public: // A probabilistically-correct indicator that the current diff --git a/indra/newview/lltextureview.cpp b/indra/newview/lltextureview.cpp index bb1535d23d..a4227ec2ff 100644 --- a/indra/newview/lltextureview.cpp +++ b/indra/newview/lltextureview.cpp @@ -76,7 +76,7 @@ static std::string title_string4(" W x H (Dis) Mem"); static S32 title_x1 = 0; static S32 title_x2 = 460; static S32 title_x3 = title_x2 + 40; -static S32 title_x4 = title_x3 + 50; +static S32 title_x4 = title_x3 + 46; static S32 texture_bar_height = 8; //////////////////////////////////////////////////////////////////////////// @@ -349,7 +349,7 @@ void LLTextureBar::draw() // draw the image size at the end { - std::string num_str = llformat("%3dx%3d (%d) %7d", mImagep->getWidth(), mImagep->getHeight(), + std::string num_str = llformat("%3dx%3d (%2d) %7d", mImagep->getWidth(), mImagep->getHeight(), mImagep->getDiscardLevel(), mImagep->hasGLTexture() ? mImagep->getTextureMemory() : 0); LLFontGL::getFontMonospace()->renderUTF8(num_str, 0, title_x4, getRect().getHeight(), color, LLFontGL::LEFT, LLFontGL::TOP); @@ -523,22 +523,42 @@ void LLGLTexMemBar::draw() LLGLSUIDefault gls_ui; LLColor4 text_color(1.f, 1.f, 1.f, 0.75f); LLColor4 color; - - std::string text = ""; + // Gray background using completely magic numbers + gGL.color4f(0.f, 0.f, 0.f, 0.25f); + const LLRect & rect(getRect()); + gl_rect_2d(-4, v_offset, rect.mRight - rect.mLeft + 2, v_offset + line_height*4); + + std::string text = ""; LLFontGL::getFontMonospace()->renderUTF8(text, 0, 0, v_offset + line_height*6, text_color, LLFontGL::LEFT, LLFontGL::TOP); - text = llformat("GL Tot: %d/%d MB Bound: %d/%d MB FBO: %d MB Raw Tot: %d MB Bias: %.2f Cache: %.1f/%.1f MB Net Tot Tex: %.1f MB Tot Obj: %.1f MB Tot Htp: %d", + text = llformat("GL Tot: %d/%d MB Bound: %d/%d MB FBO: %d MB Raw Tot: %d MB Bias: %.2f Cache: %.1f/%.1f MB", total_mem, max_total_mem, bound_mem, max_bound_mem, LLRenderTarget::sBytesAllocated/(1024*1024), - LLImageRaw::sGlobalRawMemory >> 20, discard_bias, - cache_usage, cache_max_usage, total_texture_downloaded, total_object_downloaded, total_http_requests); + LLImageRaw::sGlobalRawMemory >> 20, + discard_bias, + cache_usage, + cache_max_usage); //, cache_entries, cache_max_entries + LLFontGL::getFontMonospace()->renderUTF8(text, 0, 0, v_offset + line_height*4, + text_color, LLFontGL::LEFT, LLFontGL::TOP); + + U32 cache_read(0U), cache_write(0U), res_wait(0U); + LLAppViewer::getTextureFetch()->getStateStats(&cache_read, &cache_write, &res_wait); + + text = llformat("Net Tot Tex: %.1f MB Tot Obj: %.1f MB Tot Htp: %d Cread: %u Cwrite: %u Rwait: %u", + total_texture_downloaded, + total_object_downloaded, + total_http_requests, + cache_read, + cache_write, + res_wait); + LLFontGL::getFontMonospace()->renderUTF8(text, 0, 0, v_offset + line_height*3, text_color, LLFontGL::LEFT, LLFontGL::TOP); -- cgit v1.3 From a50944e078b98435685f04eda0ba93e95d4c61f2 Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Tue, 19 Jun 2012 17:01:02 -0400 Subject: Cleanup: move magic nubmers to new _httpinternal.h header file. --- indra/llcorehttp/CMakeLists.txt | 1 + indra/llcorehttp/_httpinternal.h | 82 ++++++++++++++++++++++++++++++++++ indra/llcorehttp/_httplibcurl.cpp | 17 ++++--- indra/llcorehttp/_httplibcurl.h | 14 ++---- indra/llcorehttp/_httpoperation.cpp | 3 +- indra/llcorehttp/_httpoprequest.cpp | 62 ++++++++++++------------- indra/llcorehttp/_httpoprequest.h | 6 +++ indra/llcorehttp/_httppolicy.cpp | 4 +- indra/llcorehttp/_httppolicy.h | 3 +- indra/llcorehttp/_httppolicyglobal.cpp | 10 +++-- indra/llcorehttp/_httpservice.cpp | 9 ++-- indra/llcorehttp/_refcounted.cpp | 4 +- indra/llcorehttp/bufferarray.cpp | 4 +- indra/llcorehttp/httpcommon.cpp | 2 +- indra/llcorehttp/httpoptions.cpp | 8 ++-- indra/llcorehttp/httprequest.cpp | 1 + indra/llcorehttp/httprequest.h | 6 +-- indra/newview/lltextureview.cpp | 2 +- 18 files changed, 161 insertions(+), 77 deletions(-) create mode 100644 indra/llcorehttp/_httpinternal.h (limited to 'indra/newview') diff --git a/indra/llcorehttp/CMakeLists.txt b/indra/llcorehttp/CMakeLists.txt index 9d8bae973e..acf4c0d6a3 100644 --- a/indra/llcorehttp/CMakeLists.txt +++ b/indra/llcorehttp/CMakeLists.txt @@ -53,6 +53,7 @@ set(llcorehttp_HEADER_FILES httpoptions.h httprequest.h httpresponse.h + _httpinternal.h _httplibcurl.h _httpopcancel.h _httpoperation.h diff --git a/indra/llcorehttp/_httpinternal.h b/indra/llcorehttp/_httpinternal.h new file mode 100644 index 0000000000..bc0bd6a2ab --- /dev/null +++ b/indra/llcorehttp/_httpinternal.h @@ -0,0 +1,82 @@ +/** + * @file httpinternal.h + * @brief Implementation constants and magic numbers + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef _LLCORE_HTTP_INTERNAL_H_ +#define _LLCORE_HTTP_INTERNAL_H_ + + +// If you find this included in a public interface header, +// something wrong is probably happening. + + +namespace LLCore +{ + +// Maxium number of policy classes that can be defined. +// *FIXME: Currently limited to the default class, extend. +const int POLICY_CLASS_LIMIT = 1; + +// Debug/informational tracing. Used both +// as a global option and in per-request traces. +const int TRACE_OFF = 0; +const int TRACE_LOW = 1; +const int TRACE_CURL_HEADERS = 2; +const int TRACE_CURL_BODIES = 3; + +const int TRACE_MIN = TRACE_OFF; +const int TRACE_MAX = TRACE_CURL_BODIES; + +// Request retry limits +const int DEFAULT_RETRY_COUNT = 5; +const int LIMIT_RETRY_MIN = 0; +const int LIMIT_RETRY_MAX = 100; + +const int DEFAULT_HTTP_REDIRECTS = 10; + +// Timeout value used for both connect and protocol exchange. +// Retries and time-on-queue are not included and aren't +// accounted for. +const long DEFAULT_TIMEOUT = 30L; +const long LIMIT_TIMEOUT_MIN = 0L; +const long LIMIT_TIMEOUT_MAX = 3600L; + +// Limits on connection counts +const int DEFAULT_CONNECTIONS = 8; +const int LIMIT_CONNECTIONS_MIN = 1; +const int LIMIT_CONNECTIONS_MAX = 256; + +// Tuning parameters + +// Time worker thread sleeps after a pass through the +// request, ready and active queues. +const int LOOP_SLEEP_NORMAL_MS = 2; + +// Block allocation size (a tuning parameter) is found +// in bufferarray.h. + +} // end namespace LLCore + +#endif // _LLCORE_HTTP_INTERNAL_H_ diff --git a/indra/llcorehttp/_httplibcurl.cpp b/indra/llcorehttp/_httplibcurl.cpp index a176dd5b2a..65eb642056 100644 --- a/indra/llcorehttp/_httplibcurl.cpp +++ b/indra/llcorehttp/_httplibcurl.cpp @@ -31,6 +31,8 @@ #include "_httpoprequest.h" #include "_httppolicy.h" +#include "llhttpstatuscodes.h" + namespace LLCore { @@ -39,7 +41,8 @@ namespace LLCore HttpLibcurl::HttpLibcurl(HttpService * service) : mService(service) { - for (int policy_class(0); policy_class < HttpRequest::POLICY_CLASS_LIMIT; ++policy_class) + // *FIXME: Use active policy class count later + for (int policy_class(0); policy_class < LL_ARRAY_SIZE(mMultiHandles); ++policy_class) { mMultiHandles[policy_class] = 0; } @@ -61,7 +64,7 @@ HttpLibcurl::~HttpLibcurl() mActiveOps.erase(item); } - for (int policy_class(0); policy_class < HttpRequest::POLICY_CLASS_LIMIT; ++policy_class) + for (int policy_class(0); policy_class < LL_ARRAY_SIZE(mMultiHandles); ++policy_class) { if (mMultiHandles[policy_class]) { @@ -89,7 +92,7 @@ HttpService::ELoopSpeed HttpLibcurl::processTransport() HttpService::ELoopSpeed ret(HttpService::REQUEST_SLEEP); // Give libcurl some cycles to do I/O & callbacks - for (int policy_class(0); policy_class < HttpRequest::POLICY_CLASS_LIMIT; ++policy_class) + for (int policy_class(0); policy_class < LL_ARRAY_SIZE(mMultiHandles); ++policy_class) { if (! mMultiHandles[policy_class]) continue; @@ -144,7 +147,7 @@ HttpService::ELoopSpeed HttpLibcurl::processTransport() void HttpLibcurl::addOp(HttpOpRequest * op) { - llassert_always(op->mReqPolicy < HttpRequest::POLICY_CLASS_LIMIT); + llassert_always(op->mReqPolicy < POLICY_CLASS_LIMIT); llassert_always(mMultiHandles[op->mReqPolicy] != NULL); // Create standard handle @@ -159,7 +162,7 @@ void HttpLibcurl::addOp(HttpOpRequest * op) curl_multi_add_handle(mMultiHandles[op->mReqPolicy], op->mCurlHandle); op->mCurlActive = true; - if (op->mTracing > 0) + if (op->mTracing > TRACE_OFF) { HttpPolicy & policy(mService->getPolicy()); @@ -208,7 +211,7 @@ bool HttpLibcurl::completeRequest(CURLM * multi_handle, CURL * handle, CURLcode } if (op->mStatus) { - int http_status(200); + int http_status(HTTP_OK); curl_easy_getinfo(handle, CURLINFO_RESPONSE_CODE, &http_status); op->mStatus = LLCore::HttpStatus(http_status); @@ -220,7 +223,7 @@ bool HttpLibcurl::completeRequest(CURLM * multi_handle, CURL * handle, CURLcode op->mCurlHandle = NULL; // Tracing - if (op->mTracing > 0) + if (op->mTracing > TRACE_OFF) { LL_INFOS("CoreHttp") << "TRACE, RequestComplete, Handle: " << static_cast(op) diff --git a/indra/llcorehttp/_httplibcurl.h b/indra/llcorehttp/_httplibcurl.h index 16b68bde43..0d0c4cad6d 100644 --- a/indra/llcorehttp/_httplibcurl.h +++ b/indra/llcorehttp/_httplibcurl.h @@ -36,6 +36,7 @@ #include "httprequest.h" #include "_httpservice.h" +#include "_httpinternal.h" namespace LLCore @@ -91,17 +92,8 @@ protected: protected: HttpService * mService; // Simple reference, not owner active_set_t mActiveOps; - CURLM * mMultiHandles[HttpRequest::POLICY_CLASS_LIMIT]; -}; // end class HttpLibcurl - - -// --------------------------------------- -// Free functions -// --------------------------------------- - - -curl_slist * append_headers_to_slist(const HttpHeaders *, curl_slist * slist); - + CURLM * mMultiHandles[POLICY_CLASS_LIMIT]; +}; // end class HttpLibcurl } // end namespace LLCore diff --git a/indra/llcorehttp/_httpoperation.cpp b/indra/llcorehttp/_httpoperation.cpp index 5a31bf90e7..0d9553434e 100644 --- a/indra/llcorehttp/_httpoperation.cpp +++ b/indra/llcorehttp/_httpoperation.cpp @@ -33,6 +33,7 @@ #include "_httprequestqueue.h" #include "_httpreplyqueue.h" #include "_httpservice.h" +#include "_httpinternal.h" #include "lltimer.h" @@ -142,7 +143,7 @@ HttpStatus HttpOperation::cancel() void HttpOperation::addAsReply() { - if (mTracing > 0) + if (mTracing > TRACE_OFF) { LL_INFOS("CoreHttp") << "TRACE, ToReplyQueue, Handle: " << static_cast(this) diff --git a/indra/llcorehttp/_httpoprequest.cpp b/indra/llcorehttp/_httpoprequest.cpp index ce41ebcce0..04a8e0baff 100644 --- a/indra/llcorehttp/_httpoprequest.cpp +++ b/indra/llcorehttp/_httpoprequest.cpp @@ -42,6 +42,7 @@ #include "_httppolicy.h" #include "_httppolicyglobal.h" #include "_httplibcurl.h" +#include "_httpinternal.h" #include "llhttpstatuscodes.h" #include "llproxy.h" @@ -73,13 +74,14 @@ void escape_libcurl_debug_data(char * buffer, size_t len, bool scrub, std::string & safe_line); -#if defined(WIN32) +#if LL_WINDOWS // Not available on windows where the legacy strtok interface // is thread-safe. char *strtok_r(char *str, const char *delim, char **saveptr); -#endif +#endif // LL_WINDOWS + } @@ -108,7 +110,7 @@ HttpOpRequest::HttpOpRequest() mReplyHeaders(NULL), mPolicyRetries(0), mPolicyRetryAt(HttpTime(0)), - mPolicyRetryLimit(5) + mPolicyRetryLimit(DEFAULT_RETRY_COUNT) { // *NOTE: As members are added, retry initialization/cleanup // may need to be extended in @prepareRequest(). @@ -334,8 +336,8 @@ void HttpOpRequest::setupCommon(HttpRequest::policy_t policy_id, mProcFlags |= PF_SAVE_HEADERS; } mPolicyRetryLimit = options->getRetries(); - mPolicyRetryLimit = llclamp(mPolicyRetryLimit, 0, 100); - mTracing = (std::max)(mTracing, llclamp(options->getTrace(), 0, 3)); + mPolicyRetryLimit = llclamp(mPolicyRetryLimit, LIMIT_RETRY_MIN, LIMIT_RETRY_MAX); + mTracing = (std::max)(mTracing, llclamp(options->getTrace(), TRACE_MIN, TRACE_MAX)); } } @@ -384,7 +386,7 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service) curl_easy_setopt(mCurlHandle, CURLOPT_DNS_CACHE_TIMEOUT, 0); curl_easy_setopt(mCurlHandle, CURLOPT_AUTOREFERER, 1); curl_easy_setopt(mCurlHandle, CURLOPT_FOLLOWLOCATION, 1); - curl_easy_setopt(mCurlHandle, CURLOPT_MAXREDIRS, 10); // *FIXME: parameterize this later + curl_easy_setopt(mCurlHandle, CURLOPT_MAXREDIRS, DEFAULT_HTTP_REDIRECTS); // *FIXME: parameterize this later curl_easy_setopt(mCurlHandle, CURLOPT_WRITEFUNCTION, writeCallback); curl_easy_setopt(mCurlHandle, CURLOPT_WRITEDATA, mCurlHandle); curl_easy_setopt(mCurlHandle, CURLOPT_READFUNCTION, readCallback); @@ -452,8 +454,6 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service) curl_easy_setopt(mCurlHandle, CURLOPT_INFILESIZE, data_size); curl_easy_setopt(mCurlHandle, CURLOPT_POSTFIELDS, (void *) NULL); mCurlHeaders = curl_slist_append(mCurlHeaders, "Expect:"); - mCurlHeaders = curl_slist_append(mCurlHeaders, "Connection: keep-alive"); - mCurlHeaders = curl_slist_append(mCurlHeaders, "Keep-alive: 300"); } break; @@ -463,7 +463,7 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service) } // Tracing - if (mTracing > 1) + if (mTracing >= TRACE_CURL_HEADERS) { curl_easy_setopt(mCurlHandle, CURLOPT_VERBOSE, 1); curl_easy_setopt(mCurlHandle, CURLOPT_DEBUGDATA, mCurlHandle); @@ -478,7 +478,7 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service) char range_line[64]; -#if defined(WIN32) +#if LL_WINDOWS _snprintf_s(range_line, sizeof(range_line), sizeof(range_line) - 1, (mReqLength ? fmt1 : fmt2), (unsigned long) mReqOffset, (unsigned long) (mReqOffset + mReqLength - 1)); @@ -486,7 +486,7 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service) snprintf(range_line, sizeof(range_line), (mReqLength ? fmt1 : fmt2), (unsigned long) mReqOffset, (unsigned long) (mReqOffset + mReqLength - 1)); -#endif // defined(WIN32) +#endif // LL_WINDOWS range_line[sizeof(range_line) - 1] = '\0'; mCurlHeaders = curl_slist_append(mCurlHeaders, range_line); } @@ -494,11 +494,11 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service) mCurlHeaders = curl_slist_append(mCurlHeaders, "Pragma:"); // Request options - long timeout(30); + long timeout(DEFAULT_TIMEOUT); if (mReqOptions) { timeout = mReqOptions->getTimeout(); - timeout = llclamp(timeout, 0L, 3600L); + timeout = llclamp(timeout, LIMIT_TIMEOUT_MIN, LIMIT_TIMEOUT_MAX); } curl_easy_setopt(mCurlHandle, CURLOPT_TIMEOUT, timeout); curl_easy_setopt(mCurlHandle, CURLOPT_CONNECTTIMEOUT, timeout); @@ -599,11 +599,11 @@ size_t HttpOpRequest::headerCallback(void * data, size_t size, size_t nmemb, voi memcpy(hdr_buffer, hdr_data, frag_size); hdr_buffer[frag_size] = '\0'; -#if defined(WIN32) +#if LL_WINDOWS if (! _strnicmp(hdr_buffer, con_ran_line, (std::min)(frag_size, con_ran_line_len))) #else if (! strncasecmp(hdr_buffer, con_ran_line, (std::min)(frag_size, con_ran_line_len))) -#endif +#endif // LL_WINDOWS { unsigned int first(0), last(0), length(0); int status; @@ -654,7 +654,7 @@ int HttpOpRequest::debugCallback(CURL * handle, curl_infotype info, char * buffe switch (info) { case CURLINFO_TEXT: - if (op->mTracing > 1) + if (op->mTracing >= TRACE_CURL_HEADERS) { tag = "TEXT"; escape_libcurl_debug_data(buffer, len, true, safe_line); @@ -663,7 +663,7 @@ int HttpOpRequest::debugCallback(CURL * handle, curl_infotype info, char * buffe break; case CURLINFO_HEADER_IN: - if (op->mTracing > 1) + if (op->mTracing >= TRACE_CURL_HEADERS) { tag = "HEADERIN"; escape_libcurl_debug_data(buffer, len, true, safe_line); @@ -672,20 +672,20 @@ int HttpOpRequest::debugCallback(CURL * handle, curl_infotype info, char * buffe break; case CURLINFO_HEADER_OUT: - if (op->mTracing > 1) + if (op->mTracing >= TRACE_CURL_HEADERS) { tag = "HEADEROUT"; - escape_libcurl_debug_data(buffer, len, true, safe_line); + escape_libcurl_debug_data(buffer, 2 * len, true, safe_line); // Goes out as one line logit = true; } break; case CURLINFO_DATA_IN: - if (op->mTracing > 1) + if (op->mTracing >= TRACE_CURL_HEADERS) { tag = "DATAIN"; logit = true; - if (op->mTracing > 2) + if (op->mTracing >= TRACE_CURL_BODIES) { escape_libcurl_debug_data(buffer, len, false, safe_line); } @@ -699,11 +699,11 @@ int HttpOpRequest::debugCallback(CURL * handle, curl_infotype info, char * buffe break; case CURLINFO_DATA_OUT: - if (op->mTracing > 1) + if (op->mTracing >= TRACE_CURL_HEADERS) { tag = "DATAOUT"; logit = true; - if (op->mTracing > 2) + if (op->mTracing >= TRACE_CURL_BODIES) { escape_libcurl_debug_data(buffer, len, false, safe_line); } @@ -755,22 +755,22 @@ int parse_content_range_header(char * buffer, if (! strtok_r(buffer, ": \t", &tok_state)) match = false; if (match && (tok = strtok_r(NULL, " \t", &tok_state))) -#if defined(WIN32) +#if LL_WINDOWS match = 0 == _stricmp("bytes", tok); #else match = 0 == strcasecmp("bytes", tok); -#endif +#endif // LL_WINDOWS if (match && ! (tok = strtok_r(NULL, " \t", &tok_state))) match = false; if (match) { unsigned int lcl_first(0), lcl_last(0), lcl_len(0); -#if defined(WIN32) +#if LL_WINDOWS if (3 == sscanf_s(tok, "%u-%u/%u", &lcl_first, &lcl_last, &lcl_len)) #else if (3 == sscanf(tok, "%u-%u/%u", &lcl_first, &lcl_last, &lcl_len)) -#endif +#endif // LL_WINDOWS { if (lcl_first > lcl_last || lcl_last >= lcl_len) return -1; @@ -779,11 +779,11 @@ int parse_content_range_header(char * buffer, *length = lcl_len; return 0; } -#if defined(WIN32) +#if LL_WINDOWS if (2 == sscanf_s(tok, "%u-%u/*", &lcl_first, &lcl_last)) #else if (2 == sscanf(tok, "%u-%u/*", &lcl_first, &lcl_last)) -#endif +#endif // LL_WINDOWS { if (lcl_first > lcl_last) return -1; @@ -798,14 +798,14 @@ int parse_content_range_header(char * buffer, return 1; } -#if defined(WIN32) +#if LL_WINDOWS char *strtok_r(char *str, const char *delim, char ** savestate) { return strtok_s(str, delim, savestate); } -#endif +#endif // LL_WINDOWS void escape_libcurl_debug_data(char * buffer, size_t len, bool scrub, std::string & safe_line) diff --git a/indra/llcorehttp/_httpoprequest.h b/indra/llcorehttp/_httpoprequest.h index 9278445763..f2b709a3a2 100644 --- a/indra/llcorehttp/_httpoprequest.h +++ b/indra/llcorehttp/_httpoprequest.h @@ -172,6 +172,12 @@ public: }; // end class HttpOpRequestCompare +// --------------------------------------- +// Free functions +// --------------------------------------- + +curl_slist * append_headers_to_slist(const HttpHeaders *, curl_slist * slist); + } // end namespace LLCore #endif // _LLCORE_HTTP_OPREQUEST_H_ diff --git a/indra/llcorehttp/_httppolicy.cpp b/indra/llcorehttp/_httppolicy.cpp index 0e08d88276..4be9f1d45f 100644 --- a/indra/llcorehttp/_httppolicy.cpp +++ b/indra/llcorehttp/_httppolicy.cpp @@ -126,7 +126,7 @@ HttpService::ELoopSpeed HttpPolicy::processReadyQueue() for (int policy_class(0); policy_class < LL_ARRAY_SIZE(mState); ++policy_class) { int active(transport.getActiveCountInClass(policy_class)); - int needed(8 - active); + int needed(DEFAULT_CONNECTIONS - active); // *FIXME: move to policy class HttpRetryQueue & retryq(mState[policy_class].mRetryQueue); HttpReadyQueue & readyq(mState[policy_class].mReadyQueue); @@ -242,7 +242,7 @@ bool HttpPolicy::stageAfterCompletion(HttpOpRequest * op) int HttpPolicy::getReadyCount(HttpRequest::policy_t policy_class) { - if (policy_class < HttpRequest::POLICY_CLASS_LIMIT) + if (policy_class < POLICY_CLASS_LIMIT) // *FIXME: use actual active class count { return (mState[policy_class].mReadyQueue.size() + mState[policy_class].mRetryQueue.size()); diff --git a/indra/llcorehttp/_httppolicy.h b/indra/llcorehttp/_httppolicy.h index 4114f64848..05de9303b5 100644 --- a/indra/llcorehttp/_httppolicy.h +++ b/indra/llcorehttp/_httppolicy.h @@ -33,6 +33,7 @@ #include "_httpreadyqueue.h" #include "_httpretryqueue.h" #include "_httppolicyglobal.h" +#include "_httpinternal.h" namespace LLCore @@ -108,7 +109,7 @@ protected: HttpRetryQueue mRetryQueue; }; - State mState[HttpRequest::POLICY_CLASS_LIMIT]; + State mState[POLICY_CLASS_LIMIT]; HttpService * mService; // Naked pointer, not refcounted, not owner HttpPolicyGlobal mGlobalOptions; diff --git a/indra/llcorehttp/_httppolicyglobal.cpp b/indra/llcorehttp/_httppolicyglobal.cpp index 6b1de38fd6..ca04839eaf 100644 --- a/indra/llcorehttp/_httppolicyglobal.cpp +++ b/indra/llcorehttp/_httppolicyglobal.cpp @@ -26,6 +26,8 @@ #include "_httppolicyglobal.h" +#include "_httpinternal.h" + namespace LLCore { @@ -33,8 +35,8 @@ namespace LLCore HttpPolicyGlobal::HttpPolicyGlobal() : mSetMask(0UL), - mConnectionLimit(32L), - mTrace(0), + mConnectionLimit(DEFAULT_CONNECTIONS), + mTrace(TRACE_OFF), mUseLLProxy(0) {} @@ -64,11 +66,11 @@ HttpStatus HttpPolicyGlobal::set(HttpRequest::EGlobalPolicy opt, long value) switch (opt) { case HttpRequest::GP_CONNECTION_LIMIT: - mConnectionLimit = value; + mConnectionLimit = llclamp(value, long(LIMIT_CONNECTIONS_MIN), long(LIMIT_CONNECTIONS_MAX)); break; case HttpRequest::GP_TRACE: - mTrace = llclamp(value, 0L, 3L); + mTrace = llclamp(value, long(TRACE_MIN), long(TRACE_MAX)); break; case HttpRequest::GP_LLPROXY: diff --git a/indra/llcorehttp/_httpservice.cpp b/indra/llcorehttp/_httpservice.cpp index 87a78820f5..25f64acc42 100644 --- a/indra/llcorehttp/_httpservice.cpp +++ b/indra/llcorehttp/_httpservice.cpp @@ -34,15 +34,12 @@ #include "_httppolicy.h" #include "_httplibcurl.h" #include "_thread.h" +#include "_httpinternal.h" #include "lltimer.h" #include "llthread.h" -// Tuning parameters -static const int LOOP_SLEEP_NORMAL_MS = 2; // Normal per-loop sleep in milliseconds - - namespace LLCore { @@ -230,11 +227,11 @@ HttpService::ELoopSpeed HttpService::processRequestQueue(ELoopSpeed loop) if (! mExitRequested) { // Setup for subsequent tracing - long tracing(0); + long tracing(TRACE_OFF); mPolicy->getGlobalOptions().get(HttpRequest::GP_TRACE, &tracing); op->mTracing = (std::max)(op->mTracing, int(tracing)); - if (op->mTracing > 0) + if (op->mTracing > TRACE_OFF) { LL_INFOS("CoreHttp") << "TRACE, FromRequestQueue, Handle: " << static_cast(op) diff --git a/indra/llcorehttp/_refcounted.cpp b/indra/llcorehttp/_refcounted.cpp index 11d75fdf97..e7d0b72741 100644 --- a/indra/llcorehttp/_refcounted.cpp +++ b/indra/llcorehttp/_refcounted.cpp @@ -30,11 +30,11 @@ namespace LLCoreInt { -#if ! defined(WIN32) +#if ! LL_WINDOWS const S32 RefCounted::NOT_REF_COUNTED; -#endif // ! defined(WIN32) +#endif // ! LL_WINDOWS RefCounted::~RefCounted() {} diff --git a/indra/llcorehttp/bufferarray.cpp b/indra/llcorehttp/bufferarray.cpp index ae92057df0..5eaa60c9ba 100644 --- a/indra/llcorehttp/bufferarray.cpp +++ b/indra/llcorehttp/bufferarray.cpp @@ -85,9 +85,9 @@ public: // ================================== -#if ! defined(WIN32) +#if ! LL_WINDOWS const size_t BufferArray::BLOCK_ALLOC_SIZE; -#endif // ! defined(WIN32) +#endif // ! LL_WINDOWS BufferArray::BufferArray() : LLCoreInt::RefCounted(true), diff --git a/indra/llcorehttp/httpcommon.cpp b/indra/llcorehttp/httpcommon.cpp index 9f17b5c842..1b18976359 100644 --- a/indra/llcorehttp/httpcommon.cpp +++ b/indra/llcorehttp/httpcommon.cpp @@ -54,9 +54,9 @@ std::string HttpStatus::toHex() const result.fill('0'); result << std::hex << operator unsigned long(); return result.str(); - } + std::string HttpStatus::toString() const { static const char * llcore_errors[] = diff --git a/indra/llcorehttp/httpoptions.cpp b/indra/llcorehttp/httpoptions.cpp index c11d89e619..f2771c1f29 100644 --- a/indra/llcorehttp/httpoptions.cpp +++ b/indra/llcorehttp/httpoptions.cpp @@ -26,6 +26,8 @@ #include "httpoptions.h" +#include "_httpinternal.h" + namespace LLCore { @@ -34,9 +36,9 @@ namespace LLCore HttpOptions::HttpOptions() : RefCounted(true), mWantHeaders(false), - mTracing(0), - mTimeout(30), - mRetries(5) + mTracing(TRACE_OFF), + mTimeout(DEFAULT_TIMEOUT), + mRetries(DEFAULT_RETRY_COUNT) {} diff --git a/indra/llcorehttp/httprequest.cpp b/indra/llcorehttp/httprequest.cpp index 2036ecfd1c..e906ff8a1e 100644 --- a/indra/llcorehttp/httprequest.cpp +++ b/indra/llcorehttp/httprequest.cpp @@ -108,6 +108,7 @@ HttpStatus HttpRequest::setPolicyGlobalOption(EGlobalPolicy opt, const std::stri HttpRequest::policy_t HttpRequest::createPolicyClass() { + // *FIXME: Implement classes policy_t policy_id = 1; return policy_id; diff --git a/indra/llcorehttp/httprequest.h b/indra/llcorehttp/httprequest.h index 4e78ed3719..24fff24b83 100644 --- a/indra/llcorehttp/httprequest.h +++ b/indra/llcorehttp/httprequest.h @@ -102,10 +102,6 @@ public: /// eventual service for any HTTP request. static const int DEFAULT_POLICY_ID = 0; - /// Maximum number of policies that may be defined. No policy - /// ID will equal or exceed this value. - static const int POLICY_CLASS_LIMIT = 1; - enum EGlobalPolicy { /// Maximum number of connections the library will use to @@ -163,7 +159,7 @@ public: /// Create a new policy class into which requests can be made. /// /// @return If positive, the policy_id used to reference - /// the class in other methods. If -1, an error + /// the class in other methods. If 0, an error /// occurred and @see getStatus() may provide more /// detail on the reason. static policy_t createPolicyClass(); diff --git a/indra/newview/lltextureview.cpp b/indra/newview/lltextureview.cpp index a4227ec2ff..ae2da84edd 100644 --- a/indra/newview/lltextureview.cpp +++ b/indra/newview/lltextureview.cpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2001&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2010, Linden Research, Inc. + * Copyright (C) 2012, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public -- cgit v1.3 From 6b4fe9fadc2301eb13a2854457b67fbebe045b08 Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Tue, 19 Jun 2012 17:12:20 -0400 Subject: When a Content-Range header is received, make available the full triplet of . --- indra/llcorehttp/_httpoprequest.cpp | 7 ++++++- indra/llcorehttp/_httpoprequest.h | 1 + indra/llcorehttp/httpresponse.cpp | 1 + indra/llcorehttp/httpresponse.h | 7 +++++-- indra/newview/lltexturefetch.cpp | 4 ++-- 5 files changed, 15 insertions(+), 5 deletions(-) (limited to 'indra/newview') diff --git a/indra/llcorehttp/_httpoprequest.cpp b/indra/llcorehttp/_httpoprequest.cpp index 04a8e0baff..516daadf9b 100644 --- a/indra/llcorehttp/_httpoprequest.cpp +++ b/indra/llcorehttp/_httpoprequest.cpp @@ -107,6 +107,7 @@ HttpOpRequest::HttpOpRequest() mReplyBody(NULL), mReplyOffset(0), mReplyLength(0), + mReplyFullLength(0), mReplyHeaders(NULL), mPolicyRetries(0), mPolicyRetryAt(HttpTime(0)), @@ -154,6 +155,7 @@ HttpOpRequest::~HttpOpRequest() mReplyOffset = 0; mReplyLength = 0; + mReplyFullLength = 0; if (mReplyBody) { mReplyBody->release(); @@ -224,7 +226,7 @@ void HttpOpRequest::visitNotifier(HttpRequest * request) if (mReplyOffset || mReplyLength) { // Got an explicit offset/length in response - response->setRange(mReplyOffset, mReplyLength); + response->setRange(mReplyOffset, mReplyLength, mReplyFullLength); } mUserHandler->onCompleted(static_cast(this), response); @@ -362,6 +364,7 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service) } mReplyOffset = 0; mReplyLength = 0; + mReplyFullLength = 0; if (mReplyHeaders) { mReplyHeaders->release(); @@ -590,6 +593,7 @@ size_t HttpOpRequest::headerCallback(void * data, size_t size, size_t nmemb, voi // taking results from the last header stanza we receive. op->mReplyOffset = 0; op->mReplyLength = 0; + op->mReplyFullLength = 0; op->mStatus = HttpStatus(); } else if (op->mProcFlags & PF_SCAN_RANGE_HEADER) @@ -613,6 +617,7 @@ size_t HttpOpRequest::headerCallback(void * data, size_t size, size_t nmemb, voi // Success, record the fragment position op->mReplyOffset = first; op->mReplyLength = last - first + 1; + op->mReplyFullLength = length; } else if (-1 == status) { diff --git a/indra/llcorehttp/_httpoprequest.h b/indra/llcorehttp/_httpoprequest.h index f2b709a3a2..5d2417466c 100644 --- a/indra/llcorehttp/_httpoprequest.h +++ b/indra/llcorehttp/_httpoprequest.h @@ -150,6 +150,7 @@ public: BufferArray * mReplyBody; off_t mReplyOffset; size_t mReplyLength; + size_t mReplyFullLength; HttpHeaders * mReplyHeaders; // Policy data diff --git a/indra/llcorehttp/httpresponse.cpp b/indra/llcorehttp/httpresponse.cpp index 3dcdadb337..a552e48a1b 100644 --- a/indra/llcorehttp/httpresponse.cpp +++ b/indra/llcorehttp/httpresponse.cpp @@ -37,6 +37,7 @@ HttpResponse::HttpResponse() : LLCoreInt::RefCounted(true), mReplyOffset(0U), mReplyLength(0U), + mReplyFullLength(0U), mBufferArray(NULL), mHeaders(NULL) {} diff --git a/indra/llcorehttp/httpresponse.h b/indra/llcorehttp/httpresponse.h index 5bcd7c4eb8..925cf81586 100644 --- a/indra/llcorehttp/httpresponse.h +++ b/indra/llcorehttp/httpresponse.h @@ -111,16 +111,18 @@ public: /// If a 'Range:' header was used, these methods are involved /// in setting and returning data about the actual response. - void getRange(unsigned int * offset, unsigned int * length) const + void getRange(unsigned int * offset, unsigned int * length, unsigned int * full) const { *offset = mReplyOffset; *length = mReplyLength; + *full = mReplyFullLength; } - void setRange(unsigned int offset, unsigned int length) + void setRange(unsigned int offset, unsigned int length, unsigned int full_length) { mReplyOffset = offset; mReplyLength = length; + mReplyFullLength = full_length; } protected: @@ -128,6 +130,7 @@ protected: HttpStatus mStatus; unsigned int mReplyOffset; unsigned int mReplyLength; + unsigned int mReplyFullLength; BufferArray * mBufferArray; HttpHeaders * mHeaders; }; diff --git a/indra/newview/lltexturefetch.cpp b/indra/newview/lltexturefetch.cpp index 6e14c5fc63..2eda6a9b80 100644 --- a/indra/newview/lltexturefetch.cpp +++ b/indra/newview/lltexturefetch.cpp @@ -1735,8 +1735,8 @@ void LLTextureFetchWorker::onCompleted(LLCore::HttpHandle handle, LLCore::HttpRe << " status: " << status.toHex() << " '" << status.toString() << "'" << llendl; -// unsigned int offset(0), length(0); -// response->getRange(&offset, &length); +// unsigned int offset(0), length(0), full_length(0); +// response->getRange(&offset, &length, &full_length); // llwarns << "HTTP COMPLETE: " << mID << " handle: " << handle // << " status: " << status.toULong() << " '" << status.toString() << "'" // << " req offset: " << mRequestedOffset << " req length: " << mRequestedSize -- cgit v1.3 From a066bc1994fccae7967921980332505aac97953f Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Wed, 20 Jun 2012 18:43:28 -0400 Subject: SH-3181 More interface options for API. Also includes returned headers. Only thing interesting in this changeset is the discovery that a sleep in the fake HTTP server ties up tests. Need to thread that or fail on client disconnect or something to speed that up and make it usable for bigger test scenarios. But good enough for now... --- indra/llcorehttp/_httpoprequest.cpp | 20 +++- indra/llcorehttp/tests/test_httprequest.hpp | 156 ++++++++++++++++++++++++- indra/llcorehttp/tests/test_llcorehttp_peer.py | 1 + indra/newview/lltexturefetch.cpp | 5 +- 4 files changed, 175 insertions(+), 7 deletions(-) (limited to 'indra/newview') diff --git a/indra/llcorehttp/_httpoprequest.cpp b/indra/llcorehttp/_httpoprequest.cpp index 516daadf9b..86ecee5b26 100644 --- a/indra/llcorehttp/_httpoprequest.cpp +++ b/indra/llcorehttp/_httpoprequest.cpp @@ -595,6 +595,10 @@ size_t HttpOpRequest::headerCallback(void * data, size_t size, size_t nmemb, voi op->mReplyLength = 0; op->mReplyFullLength = 0; op->mStatus = HttpStatus(); + if (op->mReplyHeaders) + { + op->mReplyHeaders->mHeaders.clear(); + } } else if (op->mProcFlags & PF_SCAN_RANGE_HEADER) { @@ -636,9 +640,19 @@ size_t HttpOpRequest::headerCallback(void * data, size_t size, size_t nmemb, voi if (op->mProcFlags & PF_SAVE_HEADERS) { // Save headers in response - // *FIXME: Implement this... - ; - + if (! op->mReplyHeaders) + { + op->mReplyHeaders = new HttpHeaders; + } + size_t wanted_size(hdr_size); + if (wanted_size && '\n' == hdr_data[wanted_size - 1]) + { + if (--wanted_size && '\r' == hdr_data[wanted_size - 1]) + { + --wanted_size; + } + } + op->mReplyHeaders->mHeaders.push_back(std::string(hdr_data, wanted_size)); } return hdr_size; diff --git a/indra/llcorehttp/tests/test_httprequest.hpp b/indra/llcorehttp/tests/test_httprequest.hpp index 42e4857037..cac927cfca 100644 --- a/indra/llcorehttp/tests/test_httprequest.hpp +++ b/indra/llcorehttp/tests/test_httprequest.hpp @@ -29,6 +29,7 @@ #include "httprequest.h" #include "bufferarray.h" #include "httphandler.h" +#include "httpheaders.h" #include "httpresponse.h" #include "httpoptions.h" #include "_httpservice.h" @@ -73,7 +74,8 @@ public: const std::string & name) : mState(state), mName(name), - mExpectHandle(LLCORE_HTTP_HANDLE_INVALID) + mExpectHandle(LLCORE_HTTP_HANDLE_INVALID), + mCheckHeader(false) {} virtual void onCompleted(HttpHandle handle, HttpResponse * response) @@ -95,12 +97,33 @@ public: { mState->mHandlerCalls++; } + if (mCheckHeader) + { + ensure("Response required with header check", response != NULL); + HttpHeaders * header(response->getHeaders()); // Will not hold onto this + ensure("Some quantity of headers returned", header != NULL); + bool found_special(false); + + for (HttpHeaders::container_t::const_iterator iter(header->mHeaders.begin()); + header->mHeaders.end() != iter; + ++iter) + { + if (std::string::npos != (*iter).find("X-LL-Special")) + { + found_special = true; + break; + } + } + ensure("Special header X-LL-Special in response", found_special); + } + // std::cout << "TestHandler2::onCompleted() invoked" << std::endl; } HttpRequestTestData * mState; std::string mName; HttpHandle mExpectHandle; + bool mCheckHeader; }; typedef test_group HttpRequestTestGroupType; @@ -1085,6 +1108,135 @@ void HttpRequestTestObjectType::test<11>() { ScopedCurlInit ready; + std::string url_base(get_base_url()); + // std::cerr << "Base: " << url_base << std::endl; + + set_test_name("HttpRequest GET with returned headers"); + + // Handler can be stack-allocated *if* there are no dangling + // references to it after completion of this method. + // Create before memory record as the string copy will bump numbers. + TestHandler2 handler(this, "handler"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + mHandlerCalls = 0; + + HttpRequest * req = NULL; + HttpOptions * opts = NULL; + + try + { + // Get singletons created + HttpRequest::createService(); + + // Enable tracing + HttpRequest::setPolicyGlobalOption(LLCore::HttpRequest::GP_TRACE, 2); + + // Start threading early so that thread memory is invariant + // over the test. + HttpRequest::startThread(); + + // create a new ref counted object with an implicit reference + req = new HttpRequest(); + ensure("Memory allocated on construction", mMemTotal < GetMemTotal()); + + opts = new HttpOptions(); + opts->setWantHeaders(); + + // Issue a GET that succeeds + mStatus = HttpStatus(200); + handler.mCheckHeader = true; + HttpHandle handle = req->requestGetByteRange(HttpRequest::DEFAULT_POLICY_ID, + 0U, + url_base, + 0, + 0, + opts, + NULL, + &handler); + ensure("Valid handle returned for ranged request", handle != LLCORE_HTTP_HANDLE_INVALID); + + // release options + opts->release(); + opts = NULL; + + // Run the notification pump. + int count(0); + int limit(10); + while (count++ < limit && mHandlerCalls < 1) + { + req->update(1000); + usleep(100000); + } + ensure("Request executed in reasonable time", count < limit); + ensure("One handler invocation for request", mHandlerCalls == 1); + + // Okay, request a shutdown of the servicing thread + mStatus = HttpStatus(); + handler.mCheckHeader = false; + handle = req->requestStopThread(&handler); + ensure("Valid handle returned for second request", handle != LLCORE_HTTP_HANDLE_INVALID); + + // Run the notification pump again + count = 0; + limit = 10; + while (count++ < limit && mHandlerCalls < 2) + { + req->update(1000); + usleep(100000); + } + ensure("Second request executed in reasonable time", count < limit); + ensure("Second handler invocation", mHandlerCalls == 2); + + // See that we actually shutdown the thread + count = 0; + limit = 10; + while (count++ < limit && ! HttpService::isStopped()) + { + usleep(100000); + } + ensure("Thread actually stopped running", HttpService::isStopped()); + + // release the request object + delete req; + req = NULL; + + // Shut down service + HttpRequest::destroyService(); + + ensure("Two handler calls on the way out", 2 == mHandlerCalls); + +#if defined(WIN32) + // Can only do this memory test on Windows. On other platforms, + // the LL logging system holds on to memory and produces what looks + // like memory leaks... + + // printf("Old mem: %d, New mem: %d\n", mMemTotal, GetMemTotal()); + ensure("Memory usage back to that at entry", mMemTotal == GetMemTotal()); +#endif + } + catch (...) + { + stop_thread(req); + if (opts) + { + opts->release(); + opts = NULL; + } + delete req; + HttpRequest::destroyService(); + throw; + } +} + +// *NB: This test must be last. The sleeping webserver +// won't respond for a long time. +template <> template <> +void HttpRequestTestObjectType::test<12>() +{ + ScopedCurlInit ready; + set_test_name("HttpRequest GET timeout"); // Handler can be stack-allocated *if* there are no dangling @@ -1117,7 +1269,7 @@ void HttpRequestTestObjectType::test<11>() opts->setRetries(0); // Don't retry opts->setTimeout(2); - // Issue a GET that can't connect + // Issue a GET that sleeps mStatus = HttpStatus(HttpStatus::EXT_CURL_EASY, CURLE_OPERATION_TIMEDOUT); HttpHandle handle = req->requestGetByteRange(HttpRequest::DEFAULT_POLICY_ID, 0U, diff --git a/indra/llcorehttp/tests/test_llcorehttp_peer.py b/indra/llcorehttp/tests/test_llcorehttp_peer.py index 5f0116d384..aedd8acd83 100644 --- a/indra/llcorehttp/tests/test_llcorehttp_peer.py +++ b/indra/llcorehttp/tests/test_llcorehttp_peer.py @@ -107,6 +107,7 @@ class TestHTTPRequestHandler(BaseHTTPRequestHandler): self.send_response(200) self.send_header("Content-type", "application/llsd+xml") self.send_header("Content-Length", str(len(response))) + self.send_header("X-LL-Special", "Mememememe"); self.end_headers() self.wfile.write(response) else: # fail requested diff --git a/indra/newview/lltexturefetch.cpp b/indra/newview/lltexturefetch.cpp index 2eda6a9b80..4995d9f5ea 100644 --- a/indra/newview/lltexturefetch.cpp +++ b/indra/newview/lltexturefetch.cpp @@ -1043,8 +1043,9 @@ void LLTextureFetchWorker::startWork(S32 param) // Threads: Ttf bool LLTextureFetchWorker::doWork(S32 param) { - static const LLCore::HttpStatus http_not_found(HTTP_NOT_FOUND); - static const LLCore::HttpStatus http_service_unavail(HTTP_SERVICE_UNAVAILABLE); + static const LLCore::HttpStatus http_not_found(HTTP_NOT_FOUND); // 404 + static const LLCore::HttpStatus http_service_unavail(HTTP_SERVICE_UNAVAILABLE); // 503 + static const LLCore::HttpStatus http_not_sat(HTTP_REQUESTED_RANGE_NOT_SATISFIABLE); // 416; // Release waiters while we aren't holding the Mw lock. mFetcher->releaseHttpWaiters(); -- cgit v1.3 From 5ff1758b633f1984f601aacbb7920c3c744b87f7 Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Fri, 22 Jun 2012 14:41:08 -0400 Subject: SH-3177, SH-3180 std::iostream and LLSD serialization for BufferArray objects. Seems to be working correctly. Not certain this is the fastest possible way to provide a std::streambuf interface but it's visually acceptable. --- indra/llcorehttp/bufferstream.h | 59 ++++++++++++++++++++++++++++ indra/llcorehttp/tests/test_bufferstream.hpp | 54 ++++++++++++++++++++++++- indra/newview/lltexturefetch.cpp | 27 ++++++------- indra/newview/lltexturefetch.h | 8 ++++ 4 files changed, 133 insertions(+), 15 deletions(-) (limited to 'indra/newview') diff --git a/indra/llcorehttp/bufferstream.h b/indra/llcorehttp/bufferstream.h index 60bda9ff9a..9327a798aa 100644 --- a/indra/llcorehttp/bufferstream.h +++ b/indra/llcorehttp/bufferstream.h @@ -34,13 +34,59 @@ #include "bufferarray.h" +/// @file bufferstream.h +/// +/// std::streambuf and std::iostream adapters for BufferArray +/// objects. +/// +/// BufferArrayStreamBuf inherits std::streambuf and implements +/// an unbuffered interface for streambuf. This may or may not +/// be the most time efficient implementation and it is a little +/// challenging. +/// +/// BufferArrayStream inherits std::iostream and will be the +/// adapter object most callers will be interested in (though +/// it uses BufferArrayStreamBuf internally). Instances allow +/// for the usual streaming operators ('<<', '>>') and serialization +/// methods. +/// +/// Example of LLSD serialization to a BufferArray: +/// +/// BufferArray * ba = new BufferArray; +/// BufferArrayStream bas(ba); +/// LLSDSerialize::toXML(llsd, bas); +/// operationOnBufferArray(ba); +/// ba->release(); +/// ba = NULL; +/// // operationOnBufferArray and bas are each holding +/// // references to the ba instance at this point. +/// + namespace LLCore { +// ===================================================== +// BufferArrayStreamBuf +// ===================================================== + +/// Adapter class to put a std::streambuf interface on a BufferArray +/// +/// Application developers will rarely be interested in anything +/// other than the constructor and even that will rarely be used +/// except indirectly via the @BufferArrayStream class. The +/// choice of interfaces implemented yields a bufferless adapter +/// that doesn't used either the input or output pointer triplets +/// of the more common buffered implementations. This may or may +/// not be faster and that question could stand to be looked at +/// sometime. +/// + class BufferArrayStreamBuf : public std::streambuf { public: + /// Constructor increments the reference count on the + /// BufferArray argument and calls release() on destruction. BufferArrayStreamBuf(BufferArray * array); virtual ~BufferArrayStreamBuf(); @@ -74,9 +120,22 @@ protected: }; // end class BufferArrayStreamBuf +// ===================================================== +// BufferArrayStream +// ===================================================== + +/// Adapter class that supplies streaming operators to BufferArray +/// +/// Provides a streaming adapter to an existing BufferArray +/// instance so that the convenient '<<' and '>>' conversions +/// can be applied to a BufferArray. Very convenient for LLSD +/// serialization and parsing as well. + class BufferArrayStream : public std::iostream { public: + /// Constructor increments the reference count on the + /// BufferArray argument and calls release() on destruction. BufferArrayStream(BufferArray * ba); ~BufferArrayStream(); diff --git a/indra/llcorehttp/tests/test_bufferstream.hpp b/indra/llcorehttp/tests/test_bufferstream.hpp index 45ddb7fd80..831c901b9d 100644 --- a/indra/llcorehttp/tests/test_bufferstream.hpp +++ b/indra/llcorehttp/tests/test_bufferstream.hpp @@ -31,6 +31,8 @@ #include #include "test_allocator.h" +#include "llsd.h" +#include "llsdserialize.h" using namespace LLCore; @@ -173,6 +175,8 @@ void BufferStreamTestObjectType::test<5>() const char * content("This is a string. A fragment."); const size_t c_len(strlen(content)); ba->append(content, c_len); + + // Creat an adapter for the BufferArray BufferArrayStreamBuf * bsb = new BufferArrayStreamBuf(ba); ensure("Memory being used", mMemTotal < GetMemTotal()); @@ -223,7 +227,7 @@ void BufferStreamTestObjectType::test<6>() //ba->append(content, strlen(content)); { - // create a new ref counted object with an implicit reference + // Creat an adapter for the BufferArray BufferArrayStream bas(ba); ensure("Memory being used", mMemTotal < GetMemTotal()); @@ -246,6 +250,54 @@ void BufferStreamTestObjectType::test<6>() } +template <> template <> +void BufferStreamTestObjectType::test<7>() +{ + set_test_name("BufferArrayStream with LLSD serialization"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + + // create a new ref counted BufferArray with implicit reference + BufferArray * ba = new BufferArray; + + { + // Creat an adapter for the BufferArray + BufferArrayStream bas(ba); + ensure("Memory being used", mMemTotal < GetMemTotal()); + + // LLSD + LLSD llsd = LLSD::emptyMap(); + + llsd["int"] = LLSD::Integer(3); + llsd["float"] = LLSD::Real(923289.28992); + llsd["string"] = LLSD::String("aksjdl;ajsdgfjgfal;sdgjakl;sdfjkl;ajsdfkl;ajsdfkl;jaskl;dfj"); + + LLSD llsd_map = LLSD::emptyMap(); + llsd_map["int"] = LLSD::Integer(-2889); + llsd_map["float"] = LLSD::Real(2.37829e32); + llsd_map["string"] = LLSD::String("OHIGODHSPDGHOSDHGOPSHDGP"); + + llsd["map"] = llsd_map; + + // Serialize it + LLSDSerialize::toXML(llsd, bas); + + std::string str; + bas >> str; + // std::cout << "SERIALIZED LLSD: " << str << std::endl; + ensure("Extracted string has reasonable length", str.size() > 60); + } + + // release the implicit reference, causing the object to be released + ba->release(); + ba = NULL; + + // make sure we didn't leak any memory + // ensure("Allocated memory returned", mMemTotal == GetMemTotal()); +} + + } // end namespace tut diff --git a/indra/newview/lltexturefetch.cpp b/indra/newview/lltexturefetch.cpp index 4995d9f5ea..6b186811f1 100644 --- a/indra/newview/lltexturefetch.cpp +++ b/indra/newview/lltexturefetch.cpp @@ -58,6 +58,7 @@ #include "httphandler.h" #include "httpresponse.h" #include "bufferarray.h" +#include "bufferstream.h" ////////////////////////////////////////////////////////////////////////////// @@ -2182,6 +2183,7 @@ LLTextureFetch::LLTextureFetch(LLTextureCache* cache, LLImageDecodeThread* image mHttpRequest(NULL), mHttpOptions(NULL), mHttpHeaders(NULL), + mHttpMetricsHeaders(NULL), mHttpSemaphore(HTTP_REQUESTS_IN_QUEUE_HIGH_WATER), mTotalCacheReadCount(0U), mTotalCacheWriteCount(0U), @@ -2194,6 +2196,8 @@ LLTextureFetch::LLTextureFetch(LLTextureCache* cache, LLImageDecodeThread* image mHttpOptions = new LLCore::HttpOptions; mHttpHeaders = new LLCore::HttpHeaders; mHttpHeaders->mHeaders.push_back("Accept: image/x-j2c"); + mHttpMetricsHeaders = new LLCore::HttpHeaders; + mHttpMetricsHeaders->mHeaders.push_back("Content-Type: application/llsd+xml"); } LLTextureFetch::~LLTextureFetch() @@ -2219,6 +2223,12 @@ LLTextureFetch::~LLTextureFetch() mHttpHeaders = NULL; } + if (mHttpMetricsHeaders) + { + mHttpMetricsHeaders->release(); + mHttpMetricsHeaders = NULL; + } + mHttpWaitResource.clear(); delete mHttpRequest; @@ -3501,29 +3511,18 @@ TFReqSendMetrics::doWork(LLTextureFetch * fetcher) if (! mCapsURL.empty()) { - // *FIXME: This mess to get an llsd into a string though - // it's actually no worse than what we currently do... - std::stringstream body; - LLSDSerialize::toXML(merged_llsd, body); - std::string body_str(body.str()); - body.clear(); - - LLCore::HttpHeaders * headers = new LLCore::HttpHeaders; - headers->mHeaders.push_back("Content-Type: application/llsd+xml"); - LLCore::BufferArray * ba = new LLCore::BufferArray; - ba->append(body_str.c_str(), body_str.length()); - body_str.clear(); + LLCore::BufferArrayStream bas(ba); + LLSDSerialize::toXML(merged_llsd, bas); fetcher->getHttpRequest().requestPost(report_policy_class, report_priority, mCapsURL, ba, NULL, - headers, + fetcher->getMetricsHeaders(), handler); ba->release(); - headers->release(); LLTextureFetch::svMetricsDataBreak = false; } else diff --git a/indra/newview/lltexturefetch.h b/indra/newview/lltexturefetch.h index e17c71113a..4d762a0e05 100644 --- a/indra/newview/lltexturefetch.h +++ b/indra/newview/lltexturefetch.h @@ -157,6 +157,13 @@ public: // Threads: T* LLCore::HttpRequest & getHttpRequest() { return *mHttpRequest; } + // Return a pointer to the shared metrics headers definition. + // Does not increment the reference count, caller is required + // to do that to hold a reference for any length of time. + // + // Threads: T* + LLCore::HttpHeaders * getMetricsHeaders() const { return mHttpMetricsHeaders; } + bool isQAMode() const { return mQAMode; } // ---------------------------------- @@ -322,6 +329,7 @@ private: LLCore::HttpRequest * mHttpRequest; // Ttf LLCore::HttpOptions * mHttpOptions; // Ttf LLCore::HttpHeaders * mHttpHeaders; // Ttf + LLCore::HttpHeaders * mHttpMetricsHeaders; // Ttf // We use a resource semaphore to keep HTTP requests in // WAIT_HTTP_RESOURCE2 if there aren't sufficient slots in the -- cgit v1.3 From 7997a9c4e58f67a8cf4d13cdc3e2a1b536bc1e4d Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Mon, 2 Jul 2012 18:08:36 -0400 Subject: Dos-to-unix line ending conversion. --- .../xui/en/floater_texture_fetch_debugger.xml | 682 ++++++++++----------- 1 file changed, 341 insertions(+), 341 deletions(-) (limited to 'indra/newview') diff --git a/indra/newview/skins/default/xui/en/floater_texture_fetch_debugger.xml b/indra/newview/skins/default/xui/en/floater_texture_fetch_debugger.xml index 44b6a63bca..f3f8d4ddca 100644 --- a/indra/newview/skins/default/xui/en/floater_texture_fetch_debugger.xml +++ b/indra/newview/skins/default/xui/en/floater_texture_fetch_debugger.xml @@ -1,341 +1,341 @@ - - - - 1, Total number of fetched textures: [NUM] - - - 2, Total number of fetching requests: [NUM] - - - 3, Total number of cache hits: [NUM] - - - 4, Total number of visible textures: [NUM] - - - 5, Total number of visible texture fetching requests: [NUM] - - - 6, Total number of fetched data: [SIZE1]KB, Decoded Data: [SIZE2]KB, [PIXEL]MPixels - - - 7, Total number of visible data: [SIZE1]KB, Decoded Data: [SIZE2]KB - - - 8, Total number of rendered data: [SIZE1]KB, Decoded Data: [SIZE2]KB, [PIXEL]MPixels - - - 9, Total time on cache readings: [TIME] seconds - - - 10, Total time on cache writings: [TIME] seconds - - - 11, Total time on decodings: [TIME] seconds - - - 12, Total time on gl texture creation: [TIME] seconds - - - 13, Total time on HTTP fetching: [TIME] seconds - - - 14, Total time on entire fetching: [TIME] seconds - - - 15, Refetching visibles from cache, Time: [TIME] seconds, Fetched: [SIZE]KB, [PIXEL]MPixels - - - 16, Refetching visibles from HTTP, Time: [TIME] seconds, Fetched: [SIZE]KB, [PIXEL]MPixels - - - - - - - - - - - - - - - + + + + 1, Total number of fetched textures: [NUM] + + + 2, Total number of fetching requests: [NUM] + + + 3, Total number of cache hits: [NUM] + + + 4, Total number of visible textures: [NUM] + + + 5, Total number of visible texture fetching requests: [NUM] + + + 6, Total number of fetched data: [SIZE1]KB, Decoded Data: [SIZE2]KB, [PIXEL]MPixels + + + 7, Total number of visible data: [SIZE1]KB, Decoded Data: [SIZE2]KB + + + 8, Total number of rendered data: [SIZE1]KB, Decoded Data: [SIZE2]KB, [PIXEL]MPixels + + + 9, Total time on cache readings: [TIME] seconds + + + 10, Total time on cache writings: [TIME] seconds + + + 11, Total time on decodings: [TIME] seconds + + + 12, Total time on gl texture creation: [TIME] seconds + + + 13, Total time on HTTP fetching: [TIME] seconds + + + 14, Total time on entire fetching: [TIME] seconds + + + 15, Refetching visibles from cache, Time: [TIME] seconds, Fetched: [SIZE]KB, [PIXEL]MPixels + + + 16, Refetching visibles from HTTP, Time: [TIME] seconds, Fetched: [SIZE]KB, [PIXEL]MPixels + + + + + + + + + + + + + + + -- cgit v1.3 From 2d7b7de20327a40be12a620debaae9917af16cd6 Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Tue, 3 Jul 2012 13:06:46 -0400 Subject: More integration work for texture fetch timeouts. The fetch state machine received a new timeout during the WAIT_HTTP_REQ state. For the integration, rather than jump the state to done, we issue a request cancel and let the notification plumbing do the rest without any race conditions or special-case logic. --- indra/llcorehttp/_httplibcurl.cpp | 24 +++++++++++++++++++ indra/llcorehttp/_httplibcurl.h | 3 +++ indra/llcorehttp/_httpopcancel.cpp | 6 ++++- indra/llcorehttp/_httppolicy.cpp | 49 ++++++++++++++++++++++++++++++++++++-- indra/llcorehttp/_httppolicy.h | 3 +++ indra/llcorehttp/_httpservice.cpp | 25 +++++++++++++++++++ indra/llcorehttp/_httpservice.h | 8 +++++++ indra/newview/lltexturefetch.cpp | 20 ++++++++++++++-- 8 files changed, 133 insertions(+), 5 deletions(-) (limited to 'indra/newview') diff --git a/indra/llcorehttp/_httplibcurl.cpp b/indra/llcorehttp/_httplibcurl.cpp index 39abca12c5..3c69ae1c96 100644 --- a/indra/llcorehttp/_httplibcurl.cpp +++ b/indra/llcorehttp/_httplibcurl.cpp @@ -189,6 +189,30 @@ void HttpLibcurl::addOp(HttpOpRequest * op) } +// Implements the transport part of any cancel operation. +// See if the handle is an active operation and if so, +// use the more complicated transport-based cancelation +// method to kill the request. +bool HttpLibcurl::cancel(HttpHandle handle) +{ + HttpOpRequest * op(static_cast(handle)); + active_set_t::iterator it(mActiveOps.find(op)); + if (mActiveOps.end() == it) + { + return false; + } + + // Cancel request + cancelRequest(op); + + // Drop references + mActiveOps.erase(it); + op->release(); + + return true; +} + + // *NOTE: cancelRequest logic parallels completeRequest logic. // Keep them synchronized as necessary. Caller is expected to // remove to op from the active list and release the op *after* diff --git a/indra/llcorehttp/_httplibcurl.h b/indra/llcorehttp/_httplibcurl.h index 69f7bb2b6d..53972b1ffa 100644 --- a/indra/llcorehttp/_httplibcurl.h +++ b/indra/llcorehttp/_httplibcurl.h @@ -85,6 +85,9 @@ public: int getActiveCount() const; int getActiveCountInClass(int policy_class) const; + // Shadows HttpService's method + bool cancel(HttpHandle handle); + protected: /// Invoked when libcurl has indicated a request has been processed /// to completion and we need to move the request to a new state. diff --git a/indra/llcorehttp/_httpopcancel.cpp b/indra/llcorehttp/_httpopcancel.cpp index ad624d2e57..5c1f484109 100644 --- a/indra/llcorehttp/_httpopcancel.cpp +++ b/indra/llcorehttp/_httpopcancel.cpp @@ -61,7 +61,11 @@ HttpOpCancel::~HttpOpCancel() void HttpOpCancel::stageFromRequest(HttpService * service) { - // *FIXME: Need cancel functionality into services + if (! service->cancel(mHandle)) + { + mStatus = HttpStatus(HttpStatus::LLCORE, HE_HANDLE_NOT_FOUND); + } + addAsReply(); } diff --git a/indra/llcorehttp/_httppolicy.cpp b/indra/llcorehttp/_httppolicy.cpp index 4350ff617b..1b10805b72 100644 --- a/indra/llcorehttp/_httppolicy.cpp +++ b/indra/llcorehttp/_httppolicy.cpp @@ -231,9 +231,12 @@ bool HttpPolicy::changePriority(HttpHandle handle, HttpRequest::priority_t prior for (int policy_class(0); policy_class < mActiveClasses; ++policy_class) { State & state(mState[policy_class]); - HttpReadyQueue::container_type & c(state.mReadyQueue.get_container()); - + // We don't scan retry queue because a priority change there + // is meaningless. The request will be issued based on retry + // intervals not priority value, which is now moot. + // Scan ready queue for requests that match policy + HttpReadyQueue::container_type & c(state.mReadyQueue.get_container()); for (HttpReadyQueue::container_type::iterator iter(c.begin()); c.end() != iter;) { HttpReadyQueue::container_type::iterator cur(iter++); @@ -253,6 +256,48 @@ bool HttpPolicy::changePriority(HttpHandle handle, HttpRequest::priority_t prior } +bool HttpPolicy::cancel(HttpHandle handle) +{ + for (int policy_class(0); policy_class < mActiveClasses; ++policy_class) + { + State & state(mState[policy_class]); + + // Scan retry queue + HttpRetryQueue::container_type & c1(state.mRetryQueue.get_container()); + for (HttpRetryQueue::container_type::iterator iter(c1.begin()); c1.end() != iter;) + { + HttpRetryQueue::container_type::iterator cur(iter++); + + if (static_cast(*cur) == handle) + { + HttpOpRequest * op(*cur); + c1.erase(cur); // All iterators are now invalidated + op->cancel(); + op->release(); + return true; + } + } + + // Scan ready queue + HttpReadyQueue::container_type & c2(state.mReadyQueue.get_container()); + for (HttpReadyQueue::container_type::iterator iter(c2.begin()); c2.end() != iter;) + { + HttpReadyQueue::container_type::iterator cur(iter++); + + if (static_cast(*cur) == handle) + { + HttpOpRequest * op(*cur); + c2.erase(cur); // All iterators are now invalidated + op->cancel(); + op->release(); + return true; + } + } + } + + return false; +} + bool HttpPolicy::stageAfterCompletion(HttpOpRequest * op) { static const HttpStatus cant_connect(HttpStatus::EXT_CURL_EASY, CURLE_COULDNT_CONNECT); diff --git a/indra/llcorehttp/_httppolicy.h b/indra/llcorehttp/_httppolicy.h index 90bb3b571d..a02bf084c1 100644 --- a/indra/llcorehttp/_httppolicy.h +++ b/indra/llcorehttp/_httppolicy.h @@ -92,6 +92,9 @@ public: // Shadows HttpService's method bool changePriority(HttpHandle handle, HttpRequest::priority_t priority); + // Shadows HttpService's method as well + bool cancel(HttpHandle handle); + /// When transport is finished with an op and takes it off the /// active queue, it is delivered here for dispatch. Policy /// may send it back to the ready/retry queues if it needs another diff --git a/indra/llcorehttp/_httpservice.cpp b/indra/llcorehttp/_httpservice.cpp index 92c15b5b8f..f7d9813db0 100644 --- a/indra/llcorehttp/_httpservice.cpp +++ b/indra/llcorehttp/_httpservice.cpp @@ -219,6 +219,31 @@ bool HttpService::changePriority(HttpHandle handle, HttpRequest::priority_t prio } + /// Try to find the given request handle on any of the request + /// queues and cancel the operation. + /// + /// @return True if the request was canceled. + /// + /// Threading: callable by worker thread. +bool HttpService::cancel(HttpHandle handle) +{ + bool canceled(false); + + // Request can't be on request queue so skip that. + + // Check the policy component's queues first + canceled = mPolicy->cancel(handle); + + if (! canceled) + { + // If that didn't work, check transport's. + canceled = mTransport->cancel(handle); + } + + return canceled; +} + + /// Threading: callable by worker thread. void HttpService::shutdown() { diff --git a/indra/llcorehttp/_httpservice.h b/indra/llcorehttp/_httpservice.h index d67e6e95a5..d24c497ca9 100644 --- a/indra/llcorehttp/_httpservice.h +++ b/indra/llcorehttp/_httpservice.h @@ -154,6 +154,14 @@ public: /// Threading: callable by worker thread. bool changePriority(HttpHandle handle, HttpRequest::priority_t priority); + /// Try to find the given request handle on any of the request + /// queues and cancel the operation. + /// + /// @return True if the request was found and canceled. + /// + /// Threading: callable by worker thread. + bool cancel(HttpHandle handle); + /// Threading: callable by worker thread. HttpPolicy & getPolicy() { diff --git a/indra/newview/lltexturefetch.cpp b/indra/newview/lltexturefetch.cpp index 36b878d6f2..b30b25e543 100755 --- a/indra/newview/lltexturefetch.cpp +++ b/indra/newview/lltexturefetch.cpp @@ -1560,8 +1560,24 @@ bool LLTextureFetchWorker::doWork(S32 param) if(FETCHING_TIMEOUT < mRequestedTimer.getElapsedTimeF32()) { //timeout, abort. - mState = DONE; - return true; + LL_WARNS("Texture") << "Fetch of texture " << mID << " timed out after " + << mRequestedTimer.getElapsedTimeF32() + << " seconds. Canceling request." << LL_ENDL; + + if (LLCORE_HTTP_HANDLE_INVALID != mHttpHandle) + { + // Issue cancel on any outstanding request. Asynchronous + // so cancel may not actually take effect if operation is + // complete & queued. Either way, notification will + // complete and the request can be transitioned. + mFetcher->mHttpRequest->requestCancel(mHttpHandle, NULL); + } + else + { + // Shouldn't happen but if it does, cancel quickly. + mState = DONE; + return true; + } } setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); -- cgit v1.3 From 348db20b92f1f1f85712c5a9a862ef079102bbee Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Mon, 9 Jul 2012 11:47:47 -0400 Subject: SH-3187 Issue smarter 'Range' requests for textures. First, try to issue ranged GETs that are always at least partially satisfiable. This will keep Varnish-type caches from simply sending back 200/full asset responses to unsatisfiable requests. Implement awareness of Content-Range headers as well. Currently they're not coming back but they will be someday. --- indra/llcorehttp/httpresponse.h | 6 +++ indra/newview/lltexturefetch.cpp | 90 +++++++++++++++++++++++++++++++++------- 2 files changed, 80 insertions(+), 16 deletions(-) (limited to 'indra/newview') diff --git a/indra/llcorehttp/httpresponse.h b/indra/llcorehttp/httpresponse.h index 925cf81586..65e403cec8 100644 --- a/indra/llcorehttp/httpresponse.h +++ b/indra/llcorehttp/httpresponse.h @@ -111,6 +111,12 @@ public: /// If a 'Range:' header was used, these methods are involved /// in setting and returning data about the actual response. + /// If both @offset and @length are returned as 0, we probably + /// didn't get a Content-Range header in the response. This + /// occurs with various Capabilities-based services and the + /// caller is going to have to make assumptions on receipt of + /// a 206 status. The @full value may also be zero in cases of + /// parsing problems or a wild-carded length response. void getRange(unsigned int * offset, unsigned int * length, unsigned int * full) const { *offset = mReplyOffset; diff --git a/indra/newview/lltexturefetch.cpp b/indra/newview/lltexturefetch.cpp index b30b25e543..214a6099d0 100755 --- a/indra/newview/lltexturefetch.cpp +++ b/indra/newview/lltexturefetch.cpp @@ -557,14 +557,14 @@ private: LLCore::BufferArray * mHttpBufferArray; // Refcounted pointer to response data int mHttpPolicyClass; bool mHttpActive; // Active request to http library - unsigned int mHttpReplySize; - unsigned int mHttpReplyOffset; + unsigned int mHttpReplySize; // Actual received data size + unsigned int mHttpReplyOffset; // Actual received data offset bool mHttpHasResource; // Counts against Fetcher's mHttpSemaphore // State history - U32 mCacheReadCount; - U32 mCacheWriteCount; - U32 mResourceWaitCount; + U32 mCacheReadCount; + U32 mCacheWriteCount; + U32 mResourceWaitCount; // Requests entering WAIT_HTTP_RESOURCE2 }; ////////////////////////////////////////////////////////////////////////////// @@ -1043,6 +1043,8 @@ void LLTextureFetchWorker::resetFormattedData() { mFormattedImage->deleteData(); } + mHttpReplySize = 0; + mHttpReplyOffset = 0; mHaveAllData = FALSE; } @@ -1120,6 +1122,8 @@ bool LLTextureFetchWorker::doWork(S32 param) mHttpBufferArray->release(); mHttpBufferArray = NULL; } + mHttpReplySize = 0; + mHttpReplyOffset = 0; mHaveAllData = FALSE; clearPackets(); // TODO: Shouldn't be necessary mCacheReadHandle = LLTextureCache::nullHandle(); @@ -1402,6 +1406,21 @@ bool LLTextureFetchWorker::doWork(S32 param) mRequestedDiscard = mDesiredDiscard; mRequestedSize -= cur_size; mRequestedOffset = cur_size; + if (mRequestedOffset) + { + // Texture fetching often issues 'speculative' loads that + // start beyond the end of the actual asset. Some cache/web + // systems, e.g. Varnish, will respond to this not with a + // 416 but with a 200 and the entire asset in the response + // body. By ensuring that we always have a partially + // satisfiable Range request, we avoid that hit to the network. + // We just have to deal with the overlapping data which is made + // somewhat harder by the fact that grid services don't necessarily + // return the Content-Range header on 206 responses. *Sigh* + mRequestedOffset -= 1; + mRequestedSize += 1; + } + mHttpHandle = LLCORE_HTTP_HANDLE_INVALID; if (!mUrl.empty()) { @@ -1507,9 +1526,29 @@ bool LLTextureFetchWorker::doWork(S32 param) return true; } - const S32 append_size(mHttpBufferArray->size()); - const S32 total_size(cur_size + append_size); + S32 append_size(mHttpBufferArray->size()); + S32 total_size(cur_size + append_size); + S32 src_offset(0); llassert_always(append_size == mRequestedSize); + if (mHttpReplyOffset && mHttpReplyOffset != cur_size) + { + // In case of a partial response, our offset may + // not be trivially contiguous with the data we have. + // Get back into alignment. + if (mHttpReplyOffset > cur_size) + { + LL_WARNS("Texture") << "Partial HTTP response produces break in image data for texture " + << mID << ". Aborting load." << LL_ENDL; + mState = DONE; + releaseHttpSemaphore(); + return true; + } + src_offset = cur_size - mHttpReplyOffset; + append_size -= src_offset; + total_size -= src_offset; + mRequestedSize -= src_offset; // Make requested values reflect useful part + mRequestedOffset += src_offset; + } if (mFormattedImage.isNull()) { @@ -1522,7 +1561,7 @@ bool LLTextureFetchWorker::doWork(S32 param) } } - if (mHaveAllData /* && mRequestedDiscard == 0*/) //the image file is fully loaded. + if (mHaveAllData) //the image file is fully loaded. { mFileSize = total_size; } @@ -1536,7 +1575,7 @@ bool LLTextureFetchWorker::doWork(S32 param) { memcpy(buffer, mFormattedImage->getData(), cur_size); } - mHttpBufferArray->read(0, (char *) buffer + cur_size, append_size); + mHttpBufferArray->read(src_offset, (char *) buffer + cur_size, append_size); // NOTE: setData releases current data and owns new data (buffer) mFormattedImage->setData(buffer, total_size); @@ -1544,6 +1583,8 @@ bool LLTextureFetchWorker::doWork(S32 param) // Done with buffer array mHttpBufferArray->release(); mHttpBufferArray = NULL; + mHttpReplySize = 0; + mHttpReplyOffset = 0; mLoadedDiscard = mRequestedDiscard; mState = DECODE_IMAGE; @@ -1576,6 +1617,7 @@ bool LLTextureFetchWorker::doWork(S32 param) { // Shouldn't happen but if it does, cancel quickly. mState = DONE; + releaseHttpSemaphore(); return true; } } @@ -2014,15 +2056,33 @@ S32 LLTextureFetchWorker::callbackHttpGet(LLCore::HttpResponse * response, if (data_size > 0) { // *TODO: set the formatted image data here directly to avoid the copy - // *FIXME: deal with actual offset and actual datasize, don't assume - // server gave exactly what was asked for. - - llassert_always(NULL == mHttpBufferArray); // Hold on to body for later copy + llassert_always(NULL == mHttpBufferArray); body->addRef(); mHttpBufferArray = body; + if (partial) + { + unsigned int offset(0), length(0), full_length(0); + response->getRange(&offset, &length, &full_length); + if (! offset && ! length) + { + // This is the case where we receive a 206 status but + // there wasn't a useful Content-Range header in the response. + // This could be because it was badly formatted but is more + // likely due to capabilities services which scrub headers + // from responses. Assume we got what we asked for... + mHttpReplySize = mRequestedSize; + mHttpReplyOffset = mRequestedOffset; + } + else + { + mHttpReplySize = length; + mHttpReplyOffset = offset; + } + } + if (! partial) { // Response indicates this is the entire asset regardless @@ -2040,10 +2100,8 @@ S32 LLTextureFetchWorker::callbackHttpGet(LLCore::HttpResponse * response, llassert_always(mDecodeHandle == 0); mFormattedImage = NULL; // discard any previous data we had } - else if (data_size < mRequestedSize /*&& mRequestedDiscard == 0*/) + else if (data_size < mRequestedSize) { - // *FIXME: I think we can treat this as complete regardless - // of requested discard level. Revisit this... mHaveAllData = TRUE; } else if (data_size > mRequestedSize) -- cgit v1.3 From b3659f2eba2a0a43f48e2c395f1a327dc1114098 Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Mon, 9 Jul 2012 17:04:07 -0400 Subject: Safe implementation of the HTTP resource waiter release method. Doesn't use sets or maps and so there's no ordering assumption to be violated when priorities are changed. Should also be faster. Still want to get rid of the ancillary list, however... --- indra/newview/lltexturefetch.cpp | 36 ++++++++++++++++++++---------------- 1 file changed, 20 insertions(+), 16 deletions(-) (limited to 'indra/newview') diff --git a/indra/newview/lltexturefetch.cpp b/indra/newview/lltexturefetch.cpp index 214a6099d0..9d3e7eb2b6 100755 --- a/indra/newview/lltexturefetch.cpp +++ b/indra/newview/lltexturefetch.cpp @@ -28,6 +28,7 @@ #include #include +#include #include "llstl.h" @@ -3358,32 +3359,35 @@ void LLTextureFetch::releaseHttpWaiters() if (mHttpWaitResource.empty()) return; - - const size_t limit(mHttpWaitResource.size()); - tids.reserve(limit); - for (wait_http_res_queue_t::iterator iter(mHttpWaitResource.begin()); - mHttpWaitResource.end() != iter; - ++iter) - { - tids.push_back(*iter); - } + tids.reserve(mHttpWaitResource.size()); + tids.assign(mHttpWaitResource.begin(), mHttpWaitResource.end()); } // -Mfnq // Now lookup the UUUIDs to find valid requests and sort - // them in priority order, highest to lowest. - typedef std::set worker_set_t; - worker_set_t tids2; - - for (uuid_vec_t::const_iterator iter(tids.begin()); + // them in priority order, highest to lowest. We're going + // to modify priority later as a side-effect of releasing + // these objects. That, in turn, would violate the partial + // ordering assumption of std::set, std::map, etc. so we + // don't use those containers. We use a vector and an explicit + // sort to keep the containers valid later. + typedef std::vector worker_list_t; + worker_list_t tids2; + + tids2.reserve(tids.size()); + for (uuid_vec_t::iterator iter(tids.begin()); tids.end() != iter; ++iter) { LLTextureFetchWorker * worker(getWorker(* iter)); if (worker) { - tids2.insert(worker); + tids2.push_back(worker); } } + + // Sort into priority order + LLTextureFetchWorker::Compare compare; + std::sort(tids2.begin(), tids2.end(), compare); tids.clear(); // Release workers up to the high water mark. Since we aren't @@ -3391,7 +3395,7 @@ void LLTextureFetch::releaseHttpWaiters() // with other callers. Do defensive things like getting // refreshed counts of requests and checking if someone else // has moved any worker state around.... - for (worker_set_t::iterator iter2(tids2.begin()); + for (worker_list_t::iterator iter2(tids2.begin()); tids2.end() != iter2 && mHttpSemaphore > 0; ++iter2) { -- cgit v1.3 From d6cbe006d3a9448b8897b42c3a817f914a279f02 Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Mon, 9 Jul 2012 17:29:21 -0400 Subject: Take body size as the reply size when Content-Range header isn't available. --- indra/newview/lltexturefetch.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'indra/newview') diff --git a/indra/newview/lltexturefetch.cpp b/indra/newview/lltexturefetch.cpp index 9d3e7eb2b6..a1f99eeb25 100755 --- a/indra/newview/lltexturefetch.cpp +++ b/indra/newview/lltexturefetch.cpp @@ -2074,7 +2074,7 @@ S32 LLTextureFetchWorker::callbackHttpGet(LLCore::HttpResponse * response, // This could be because it was badly formatted but is more // likely due to capabilities services which scrub headers // from responses. Assume we got what we asked for... - mHttpReplySize = mRequestedSize; + mHttpReplySize = data_size; mHttpReplyOffset = mRequestedOffset; } else -- cgit v1.3 From a5ba9c0eb327d2fb38a39560a34712e844a71a79 Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Tue, 10 Jul 2012 16:56:38 -0400 Subject: SH-3276 Handle 416 status back from texture fetches as okay. A 416 will just mean there's no more data and whatever we have is complete. --- indra/newview/lltexturefetch.cpp | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) (limited to 'indra/newview') diff --git a/indra/newview/lltexturefetch.cpp b/indra/newview/lltexturefetch.cpp index a1f99eeb25..5c39504243 100755 --- a/indra/newview/lltexturefetch.cpp +++ b/indra/newview/lltexturefetch.cpp @@ -1364,6 +1364,10 @@ bool LLTextureFetchWorker::doWork(S32 param) } mState = SEND_HTTP_REQ; acquireHttpSemaphore(); + + // *NOTE: You must invoke releaseHttpSemaphore() if you transition + // to a state other than SEND_HTTP_REQ or WAIT_HTTP_REQ or abort + // the request. } if (mState == WAIT_HTTP_RESOURCE2) @@ -1486,13 +1490,16 @@ bool LLTextureFetchWorker::doWork(S32 param) { LL_INFOS_ONCE("Texture") << "Texture server busy (503): " << mUrl << LL_ENDL; } + else if (http_not_sat == mGetStatus) + { + // Allowed, we'll accept whatever data we have as complete. + mHaveAllData = TRUE; + } else { llinfos << "HTTP GET failed for: " << mUrl << " Status: " << mGetStatus.toHex() << " Reason: '" << mGetReason << "'" - // *FIXME: Add retry info for reporting purposes... - // << " Attempt:" << mHTTPFailCount+1 << "/" << max_attempts << llendl; } @@ -1839,7 +1846,7 @@ void LLTextureFetchWorker::onCompleted(LLCore::HttpHandle handle, LLCore::HttpRe success = false; std::string reason(status.toString()); setGetStatus(status, reason); - llwarns << "CURL GET FAILED, status: " << status.toHex() + llwarns << "CURL GET FAILED, status: " << status.toHex() << " reason: " << reason << llendl; } else -- cgit v1.3 From bc72acbfd2410e01946375bcfa29cf37a7c01c17 Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Tue, 10 Jul 2012 18:50:21 -0400 Subject: SH-3244 Syscall avoidance in HttpRequest::update() method Well, achieved that by doing work in bulk when needed. But turned into some additional things. Change timebase from mS to uS as, well, things are headed that way. Implement an HttpReplyQueue::fetchAll method (advertised one, hadn't implemented it). --- indra/llcorehttp/_httpreplyqueue.cpp | 20 +++++++++++ indra/llcorehttp/_httprequestqueue.cpp | 7 ++-- indra/llcorehttp/examples/http_texture_load.cpp | 2 +- indra/llcorehttp/httprequest.cpp | 40 +++++++++++++++++---- indra/llcorehttp/httprequest.h | 7 ++-- indra/llcorehttp/tests/test_httprequest.hpp | 48 ++++++++++++------------- indra/newview/llappviewer.cpp | 2 +- indra/newview/lltexturefetch.cpp | 19 ++++------ 8 files changed, 92 insertions(+), 53 deletions(-) (limited to 'indra/newview') diff --git a/indra/llcorehttp/_httpreplyqueue.cpp b/indra/llcorehttp/_httpreplyqueue.cpp index a354ed7e10..558b7bdee9 100644 --- a/indra/llcorehttp/_httpreplyqueue.cpp +++ b/indra/llcorehttp/_httpreplyqueue.cpp @@ -84,4 +84,24 @@ HttpOperation * HttpReplyQueue::fetchOp() return result; } + +void HttpReplyQueue::fetchAll(OpContainer & ops) +{ + // Not valid putting something back on the queue... + llassert_always(ops.empty()); + + { + HttpScopedLock lock(mQueueMutex); + + if (! mQueue.empty()) + { + mQueue.swap(ops); + } + } + + // Caller also acquires the reference counts on each op. + return; +} + + } // end namespace LLCore diff --git a/indra/llcorehttp/_httprequestqueue.cpp b/indra/llcorehttp/_httprequestqueue.cpp index 9acac665a9..c16966d078 100644 --- a/indra/llcorehttp/_httprequestqueue.cpp +++ b/indra/llcorehttp/_httprequestqueue.cpp @@ -120,10 +120,9 @@ HttpOperation * HttpRequestQueue::fetchOp(bool wait) void HttpRequestQueue::fetchAll(bool wait, OpContainer & ops) { - // Note: Should probably test whether we're empty or not here. - // A target passed in with entries is likely also carrying - // reference counts and we're going to leak something. - ops.clear(); + // Not valid putting something back on the queue... + llassert_always(ops.empty()); + { HttpScopedLock lock(mQueueMutex); diff --git a/indra/llcorehttp/examples/http_texture_load.cpp b/indra/llcorehttp/examples/http_texture_load.cpp index e5951e8415..bcb322bd5c 100644 --- a/indra/llcorehttp/examples/http_texture_load.cpp +++ b/indra/llcorehttp/examples/http_texture_load.cpp @@ -244,7 +244,7 @@ int main(int argc, char** argv) int passes(0); while (! ws.reload(hr)) { - hr->update(5000); + hr->update(5000000); ms_sleep(2); if (0 == (++passes % 200)) { diff --git a/indra/llcorehttp/httprequest.cpp b/indra/llcorehttp/httprequest.cpp index 3a55a849b9..9b739a8825 100644 --- a/indra/llcorehttp/httprequest.cpp +++ b/indra/llcorehttp/httprequest.cpp @@ -296,17 +296,43 @@ HttpHandle HttpRequest::requestNoOp(HttpHandler * user_handler) } -HttpStatus HttpRequest::update(long millis) +HttpStatus HttpRequest::update(long usecs) { - const HttpTime limit(totalTime() + (1000 * HttpTime(millis))); HttpOperation * op(NULL); - while (limit >= totalTime() && (op = mReplyQueue->fetchOp())) + + if (usecs) { - // Process operation - op->visitNotifier(this); + const HttpTime limit(totalTime() + HttpTime(usecs)); + while (limit >= totalTime() && (op = mReplyQueue->fetchOp())) + { + // Process operation + op->visitNotifier(this); - // We're done with the operation - op->release(); + // We're done with the operation + op->release(); + } + } + else + { + // Same as above, just no time limit + HttpReplyQueue::OpContainer replies; + mReplyQueue->fetchAll(replies); + if (! replies.empty()) + { + for (HttpReplyQueue::OpContainer::iterator iter(replies.begin()); + replies.end() != iter; + ++iter) + { + // Swap op pointer for NULL; + op = *iter; *iter = NULL; + + // Process operation + op->visitNotifier(this); + + // We're done with the operation + op->release(); + } + } } return HttpStatus(); diff --git a/indra/llcorehttp/httprequest.h b/indra/llcorehttp/httprequest.h index 134a61b618..9dd53f4483 100644 --- a/indra/llcorehttp/httprequest.h +++ b/indra/llcorehttp/httprequest.h @@ -341,13 +341,14 @@ public: /// are expected to return 'quickly' and do any significant processing /// outside of the notification callback to onCompleted(). /// - /// @param millis Maximum number of wallclock milliseconds to + /// @param usecs Maximum number of wallclock microseconds to /// spend in the call. As hinted at above, this /// is partly a function of application code so it's - /// a soft limit. + /// a soft limit. A '0' value will run without + /// time limit. /// /// @return Standard status code. - HttpStatus update(long millis); + HttpStatus update(long usecs); /// @} diff --git a/indra/llcorehttp/tests/test_httprequest.hpp b/indra/llcorehttp/tests/test_httprequest.hpp index 914f35ec3d..ba7c757af4 100644 --- a/indra/llcorehttp/tests/test_httprequest.hpp +++ b/indra/llcorehttp/tests/test_httprequest.hpp @@ -260,7 +260,7 @@ void HttpRequestTestObjectType::test<3>() int limit(20); while (count++ < limit && mHandlerCalls < 1) { - req->update(1000); + req->update(1000000); usleep(100000); } ensure("Request executed in reasonable time", count < limit); @@ -275,7 +275,7 @@ void HttpRequestTestObjectType::test<3>() limit = 100; while (count++ < limit && mHandlerCalls < 2) { - req->update(1000); + req->update(1000000); usleep(100000); } ensure("Second request executed in reasonable time", count < limit); @@ -358,8 +358,8 @@ void HttpRequestTestObjectType::test<4>() int limit(20); while (count++ < limit && mHandlerCalls < 2) { - req1->update(1000); - req2->update(1000); + req1->update(1000000); + req2->update(1000000); usleep(100000); } ensure("Request executed in reasonable time", count < limit); @@ -375,8 +375,8 @@ void HttpRequestTestObjectType::test<4>() limit = 100; while (count++ < limit && mHandlerCalls < 3) { - req1->update(1000); - req2->update(1000); + req1->update(1000000); + req2->update(1000000); usleep(100000); } ensure("Second request executed in reasonable time", count < limit); @@ -459,7 +459,7 @@ void HttpRequestTestObjectType::test<5>() int limit(10); while (count++ < limit && mHandlerCalls < 1) { - req->update(1000); + req->update(1000000); usleep(100000); } ensure("NoOp notification received", mHandlerCalls == 1); @@ -535,7 +535,7 @@ void HttpRequestTestObjectType::test<6>() int limit(10); while (count++ < limit && mHandlerCalls < 1) { - req->update(1000); + req->update(1000000); usleep(100000); } ensure("No notifications received", mHandlerCalls == 0); @@ -616,7 +616,7 @@ void HttpRequestTestObjectType::test<7>() int limit(50); // With one retry, should fail quickish while (count++ < limit && mHandlerCalls < 1) { - req->update(1000); + req->update(1000000); usleep(100000); } ensure("Request executed in reasonable time", count < limit); @@ -632,7 +632,7 @@ void HttpRequestTestObjectType::test<7>() limit = 100; while (count++ < limit && mHandlerCalls < 2) { - req->update(1000); + req->update(1000000); usleep(100000); } ensure("Second request executed in reasonable time", count < limit); @@ -733,7 +733,7 @@ void HttpRequestTestObjectType::test<8>() int limit(10); while (count++ < limit && mHandlerCalls < 1) { - req->update(1000); + req->update(1000000); usleep(100000); } ensure("Request executed in reasonable time", count < limit); @@ -749,7 +749,7 @@ void HttpRequestTestObjectType::test<8>() limit = 10; while (count++ < limit && mHandlerCalls < 2) { - req->update(1000); + req->update(1000000); usleep(100000); } ensure("Second request executed in reasonable time", count < limit); @@ -843,7 +843,7 @@ void HttpRequestTestObjectType::test<9>() int limit(10); while (count++ < limit && mHandlerCalls < 1) { - req->update(1000); + req->update(1000000); usleep(100000); } ensure("Request executed in reasonable time", count < limit); @@ -859,7 +859,7 @@ void HttpRequestTestObjectType::test<9>() limit = 10; while (count++ < limit && mHandlerCalls < 2) { - req->update(1000); + req->update(1000000); usleep(100000); } ensure("Second request executed in reasonable time", count < limit); @@ -955,7 +955,7 @@ void HttpRequestTestObjectType::test<10>() int limit(10); while (count++ < limit && mHandlerCalls < 1) { - req->update(1000); + req->update(1000000); usleep(100000); } ensure("Request executed in reasonable time", count < limit); @@ -971,7 +971,7 @@ void HttpRequestTestObjectType::test<10>() limit = 10; while (count++ < limit && mHandlerCalls < 2) { - req->update(1000); + req->update(1000000); usleep(100000); } ensure("Second request executed in reasonable time", count < limit); @@ -1074,7 +1074,7 @@ void HttpRequestTestObjectType::test<11>() int limit(10); while (count++ < limit && mHandlerCalls < 1) { - req->update(1000); + req->update(1000000); usleep(100000); } ensure("Request executed in reasonable time", count < limit); @@ -1090,7 +1090,7 @@ void HttpRequestTestObjectType::test<11>() limit = 10; while (count++ < limit && mHandlerCalls < 2) { - req->update(1000); + req->update(1000000); usleep(100000); } ensure("Second request executed in reasonable time", count < limit); @@ -1194,7 +1194,7 @@ void HttpRequestTestObjectType::test<12>() int limit(10); while (count++ < limit && mHandlerCalls < 1) { - req->update(1000); + req->update(1000000); usleep(100000); } ensure("Request executed in reasonable time", count < limit); @@ -1210,7 +1210,7 @@ void HttpRequestTestObjectType::test<12>() limit = 10; while (count++ < limit && mHandlerCalls < 2) { - req->update(1000); + req->update(1000000); usleep(100000); } ensure("Second request executed in reasonable time", count < limit); @@ -1316,7 +1316,7 @@ void HttpRequestTestObjectType::test<13>() int limit(10); while (count++ < limit && mHandlerCalls < 1) { - req->update(1000); + req->update(1000000); usleep(100000); } ensure("Request executed in reasonable time", count < limit); @@ -1333,7 +1333,7 @@ void HttpRequestTestObjectType::test<13>() limit = 10; while (count++ < limit && mHandlerCalls < 2) { - req->update(1000); + req->update(1000000); usleep(100000); } ensure("Second request executed in reasonable time", count < limit); @@ -1435,7 +1435,7 @@ void HttpRequestTestObjectType::test<14>() int limit(50); // With one retry, should fail quickish while (count++ < limit && mHandlerCalls < 1) { - req->update(1000); + req->update(1000000); usleep(100000); } ensure("Request executed in reasonable time", count < limit); @@ -1451,7 +1451,7 @@ void HttpRequestTestObjectType::test<14>() limit = 100; while (count++ < limit && mHandlerCalls < 2) { - req->update(1000); + req->update(1000000); usleep(100000); } ensure("Second request executed in reasonable time", count < limit); diff --git a/indra/newview/llappviewer.cpp b/indra/newview/llappviewer.cpp index 8243d4f2f3..0549a972e1 100644 --- a/indra/newview/llappviewer.cpp +++ b/indra/newview/llappviewer.cpp @@ -5398,7 +5398,7 @@ void CoreHttp::cleanup() { while (! mStopped && LLTimer::getTotalSeconds() < (mStopRequested + MAX_THREAD_WAIT_TIME)) { - mRequest->update(200); + mRequest->update(200000); ms_sleep(50); } if (! mStopped) diff --git a/indra/newview/lltexturefetch.cpp b/indra/newview/lltexturefetch.cpp index 5c39504243..8314031f14 100755 --- a/indra/newview/lltexturefetch.cpp +++ b/indra/newview/lltexturefetch.cpp @@ -2740,22 +2740,14 @@ void LLTextureFetch::commonUpdate() // Run a cross-thread command, if any. cmdDoWork(); - // Update Curl on same thread as mCurlGetRequest was constructed - LLCore::HttpStatus status = mHttpRequest->update(200); + // Deliver all completion notifications + LLCore::HttpStatus status = mHttpRequest->update(0); if (! status) { LL_INFOS_ONCE("Texture") << "Problem during HTTP servicing. Reason: " << status.toString() << LL_ENDL; } - -#if 0 - // *FIXME: maybe implement this another way... - if (processed > 0) - { - lldebugs << "processed: " << processed << " messages." << llendl; - } -#endif } @@ -2840,7 +2832,8 @@ void LLTextureFetch::endThread() void LLTextureFetch::threadedUpdate() { llassert_always(mHttpRequest); - + +#if 0 // Limit update frequency const F32 PROCESS_TIME = 0.05f; static LLFrameTimer process_timer; @@ -2849,9 +2842,10 @@ void LLTextureFetch::threadedUpdate() return; } process_timer.reset(); +#endif commonUpdate(); - + #if 0 const F32 INFO_TIME = 1.0f; static LLFrameTimer info_timer; @@ -2865,7 +2859,6 @@ void LLTextureFetch::threadedUpdate() } } #endif - } ////////////////////////////////////////////////////////////////////////////// -- cgit v1.3 From d238341afaecedfe227141126c4c35dcde4a0671 Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Mon, 16 Jul 2012 11:53:04 -0400 Subject: SH-3189 Remove/improve naive data structures When releasing HTTP waiters, avoid unnecessary sort activity. For Content-Type in responses, let libcurl do the work and removed my parsing of headers. Drop Content-Encoding as libcurl will deal with that. If anyone is interested, they can parse. --- indra/llcorehttp/_httplibcurl.cpp | 18 ++++++++++- indra/llcorehttp/_httpoprequest.cpp | 48 ++--------------------------- indra/llcorehttp/_httpoprequest.h | 2 -- indra/llcorehttp/httpcommon.cpp | 3 +- indra/llcorehttp/httpcommon.h | 5 ++- indra/llcorehttp/httpresponse.h | 9 ++---- indra/llcorehttp/tests/test_httprequest.hpp | 3 +- indra/newview/lltexturefetch.cpp | 22 ++++++++++--- 8 files changed, 48 insertions(+), 62 deletions(-) (limited to 'indra/newview') diff --git a/indra/llcorehttp/_httplibcurl.cpp b/indra/llcorehttp/_httplibcurl.cpp index 3c69ae1c96..e031efbc91 100644 --- a/indra/llcorehttp/_httplibcurl.cpp +++ b/indra/llcorehttp/_httplibcurl.cpp @@ -280,7 +280,23 @@ bool HttpLibcurl::completeRequest(CURLM * multi_handle, CURL * handle, CURLcode int http_status(HTTP_OK); curl_easy_getinfo(handle, CURLINFO_RESPONSE_CODE, &http_status); - op->mStatus = LLCore::HttpStatus(http_status); + if (http_status >= 100 && http_status <= 999) + { + char * cont_type(NULL); + curl_easy_getinfo(handle, CURLINFO_CONTENT_TYPE, &cont_type); + if (cont_type) + { + op->mReplyConType = cont_type; + } + op->mStatus = HttpStatus(http_status); + } + else + { + LL_WARNS("CoreHttp") << "Invalid HTTP response code (" + << http_status << ") received from server." + << LL_ENDL; + op->mStatus = HttpStatus(HttpStatus::LLCORE, HE_INVALID_HTTP_STATUS); + } } // Detach from multi and recycle handle diff --git a/indra/llcorehttp/_httpoprequest.cpp b/indra/llcorehttp/_httpoprequest.cpp index 1854d7ada4..1a770f67be 100644 --- a/indra/llcorehttp/_httpoprequest.cpp +++ b/indra/llcorehttp/_httpoprequest.cpp @@ -228,7 +228,7 @@ void HttpOpRequest::visitNotifier(HttpRequest * request) // Got an explicit offset/length in response response->setRange(mReplyOffset, mReplyLength, mReplyFullLength); } - response->setContent(mReplyConType, mReplyConEncode); + response->setContentType(mReplyConType); mUserHandler->onCompleted(static_cast(this), response); @@ -316,7 +316,7 @@ void HttpOpRequest::setupCommon(HttpRequest::policy_t policy_id, HttpOptions * options, HttpHeaders * headers) { - mProcFlags = PF_SCAN_CONTENT_HEADERS; // Always scan for content headers + mProcFlags = 0U; mReqPolicy = policy_id; mReqPriority = priority; mReqURL = url; @@ -379,7 +379,6 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service) mReplyHeaders = NULL; } mReplyConType.clear(); - mReplyConEncode.clear(); // *FIXME: better error handling later HttpStatus status; @@ -542,7 +541,7 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service) } curl_easy_setopt(mCurlHandle, CURLOPT_HTTPHEADER, mCurlHeaders); - if (mProcFlags & (PF_SCAN_RANGE_HEADER | PF_SAVE_HEADERS | PF_SCAN_CONTENT_HEADERS)) + if (mProcFlags & (PF_SCAN_RANGE_HEADER | PF_SAVE_HEADERS)) { curl_easy_setopt(mCurlHandle, CURLOPT_HEADERFUNCTION, headerCallback); curl_easy_setopt(mCurlHandle, CURLOPT_HEADERDATA, this); @@ -620,8 +619,6 @@ size_t HttpOpRequest::headerCallback(void * data, size_t size, size_t nmemb, voi op->mReplyOffset = 0; op->mReplyLength = 0; op->mReplyFullLength = 0; - op->mReplyConType.clear(); - op->mReplyConEncode.clear(); op->mStatus = HttpStatus(); if (op->mReplyHeaders) { @@ -688,45 +685,6 @@ size_t HttpOpRequest::headerCallback(void * data, size_t size, size_t nmemb, voi } } - // Detect and parse 'Content-Type' and 'Content-Encoding' headers - if (op->mProcFlags & PF_SCAN_CONTENT_HEADERS) - { - if (wanted_hdr_size > con_type_line_len && - ! os_strncasecmp(hdr_data, con_type_line, con_type_line_len)) - { - // Found 'Content-Type:', extract single-token value - std::string rhs(hdr_data + con_type_line_len, wanted_hdr_size - con_type_line_len); - std::string::size_type begin(0), end(rhs.size()), pos; - - if ((pos = rhs.find_first_not_of(hdr_whitespace)) != std::string::npos) - { - begin = pos; - } - if ((pos = rhs.find_first_of(hdr_whitespace, begin)) != std::string::npos) - { - end = pos; - } - op->mReplyConType.assign(rhs, begin, end - begin); - } - else if (wanted_hdr_size > con_enc_line_len && - ! os_strncasecmp(hdr_data, con_enc_line, con_enc_line_len)) - { - // Found 'Content-Encoding:', extract single-token value - std::string rhs(hdr_data + con_enc_line_len, wanted_hdr_size - con_enc_line_len); - std::string::size_type begin(0), end(rhs.size()), pos; - - if ((pos = rhs.find_first_not_of(hdr_whitespace)) != std::string::npos) - { - begin = pos; - } - if ((pos = rhs.find_first_of(hdr_whitespace, begin)) != std::string::npos) - { - end = pos; - } - op->mReplyConEncode.assign(rhs, begin, end - begin); - } - } - return hdr_size; } diff --git a/indra/llcorehttp/_httpoprequest.h b/indra/llcorehttp/_httpoprequest.h index 200b925c4e..36dc5dc876 100644 --- a/indra/llcorehttp/_httpoprequest.h +++ b/indra/llcorehttp/_httpoprequest.h @@ -138,7 +138,6 @@ protected: unsigned int mProcFlags; static const unsigned int PF_SCAN_RANGE_HEADER = 0x00000001U; static const unsigned int PF_SAVE_HEADERS = 0x00000002U; - static const unsigned int PF_SCAN_CONTENT_HEADERS = 0x00000004U; public: // Request data @@ -165,7 +164,6 @@ public: size_t mReplyFullLength; HttpHeaders * mReplyHeaders; std::string mReplyConType; - std::string mReplyConEncode; // Policy data int mPolicyRetries; diff --git a/indra/llcorehttp/httpcommon.cpp b/indra/llcorehttp/httpcommon.cpp index 1b18976359..f2fcbf77a3 100644 --- a/indra/llcorehttp/httpcommon.cpp +++ b/indra/llcorehttp/httpcommon.cpp @@ -69,7 +69,8 @@ std::string HttpStatus::toString() const "Request handle not found", "Invalid datatype for argument or option", "Option has not been explicitly set", - "Option is not dynamic and must be set early" + "Option is not dynamic and must be set early", + "Invalid HTTP status code received from server" }; static const int llcore_errors_count(sizeof(llcore_errors) / sizeof(llcore_errors[0])); diff --git a/indra/llcorehttp/httpcommon.h b/indra/llcorehttp/httpcommon.h index 576a113e54..dd5798edf9 100644 --- a/indra/llcorehttp/httpcommon.h +++ b/indra/llcorehttp/httpcommon.h @@ -149,7 +149,10 @@ enum HttpError HE_OPT_NOT_SET = 7, // Option not dynamic, must be set during init phase - HE_OPT_NOT_DYNAMIC = 8 + HE_OPT_NOT_DYNAMIC = 8, + + // Invalid HTTP status code returned by server + HE_INVALID_HTTP_STATUS = 9 }; // end enum HttpError diff --git a/indra/llcorehttp/httpresponse.h b/indra/llcorehttp/httpresponse.h index 3d35050c17..4a481db6ac 100644 --- a/indra/llcorehttp/httpresponse.h +++ b/indra/llcorehttp/httpresponse.h @@ -134,16 +134,14 @@ public: } /// - void getContent(std::string & con_type, std::string & con_encode) const + const std::string & getContentType() const { - con_type = mContentType; - con_encode = mContentEncoding; + return mContentType; } - void setContent(const std::string & con_type, const std::string & con_encode) + void setContentType(const std::string & con_type) { mContentType = con_type; - mContentEncoding = con_encode; } protected: @@ -155,7 +153,6 @@ protected: BufferArray * mBufferArray; HttpHeaders * mHeaders; std::string mContentType; - std::string mContentEncoding; }; diff --git a/indra/llcorehttp/tests/test_httprequest.hpp b/indra/llcorehttp/tests/test_httprequest.hpp index 0f9eb93996..1acf4f9d4b 100644 --- a/indra/llcorehttp/tests/test_httprequest.hpp +++ b/indra/llcorehttp/tests/test_httprequest.hpp @@ -147,8 +147,7 @@ public: if (! mCheckContentType.empty()) { ensure("Response required with content type check", response != NULL); - std::string con_type, con_enc; - response->getContent(con_type, con_enc); + std::string con_type(response->getContentType()); ensure("Content-Type as expected (" + mCheckContentType + ")", mCheckContentType == con_type); } diff --git a/indra/newview/lltexturefetch.cpp b/indra/newview/lltexturefetch.cpp index 8314031f14..225ea46558 100755 --- a/indra/newview/lltexturefetch.cpp +++ b/indra/newview/lltexturefetch.cpp @@ -3342,6 +3342,17 @@ void LLTextureFetch::removeHttpWaiter(const LLUUID & tid) mNetworkQueueMutex.unlock(); // -Mfnq } +// Release as many requests as permitted from the WAIT_HTTP_RESOURCE2 +// state to the SEND_HTTP_REQ state based on their current priority. +// +// This data structures and code associated with this looks a bit +// indirect and naive but it's done in the name of safety. An +// ordered container may become invalid from time to time due to +// priority changes caused by actions in other threads. State itself +// could also suffer the same fate with canceled operations. Even +// done this way, I'm not fully trusting we're truly safe. This +// module is due for a major refactoring and we'll deal with it then. +// // Threads: Ttf // Locks: -Mw (must not hold any worker when called) void LLTextureFetch::releaseHttpWaiters() @@ -3384,12 +3395,15 @@ void LLTextureFetch::releaseHttpWaiters() tids2.push_back(worker); } } - - // Sort into priority order - LLTextureFetchWorker::Compare compare; - std::sort(tids2.begin(), tids2.end(), compare); tids.clear(); + // Sort into priority order, if necessary and only as much as needed + if (tids2.size() > mHttpSemaphore) + { + LLTextureFetchWorker::Compare compare; + std::partial_sort(tids2.begin(), tids2.begin() + mHttpSemaphore, tids2.end(), compare); + } + // Release workers up to the high water mark. Since we aren't // holding any locks at this point, we can be in competition // with other callers. Do defensive things like getting -- cgit v1.3 From c9f5bc7cae793b6965ceb9490243b4c52017c254 Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Tue, 17 Jul 2012 11:24:52 -0400 Subject: SH-3189 Improve naive data structures Move releaseHttpWaiters() to commonUpdate from doWork. More appropriate home for it. Have deleteOK() defer deletion of anything in WAIT_HTTP_RESOURCE2 state to keep pointers valid for the releaseHttpWaiters() method. It will then transition canceled operations to SEND_HTTP_REQ where they can be deleted. --- indra/newview/lltexturefetch.cpp | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) (limited to 'indra/newview') diff --git a/indra/newview/lltexturefetch.cpp b/indra/newview/lltexturefetch.cpp index 225ea46558..51d57ccf68 100755 --- a/indra/newview/lltexturefetch.cpp +++ b/indra/newview/lltexturefetch.cpp @@ -1064,9 +1064,6 @@ bool LLTextureFetchWorker::doWork(S32 param) static const LLCore::HttpStatus http_service_unavail(HTTP_SERVICE_UNAVAILABLE); // 503 static const LLCore::HttpStatus http_not_sat(HTTP_REQUESTED_RANGE_NOT_SATISFIABLE); // 416; - // Release waiters while we aren't holding the Mw lock. - mFetcher->releaseHttpWaiters(); - LLMutexLock lock(&mWorkMutex); // +Mw if ((mFetcher->isQuitting() || getFlags(LLWorkerClass::WCF_DELETE_REQUESTED))) @@ -1927,6 +1924,14 @@ bool LLTextureFetchWorker::deleteOK() // and will dereference it to do notification. delete_ok = false; } + + if (WAIT_HTTP_RESOURCE2 == mState) + { + // Don't delete the worker out from under the + // releaseHttpWaiters() method. Keep the pointers + // valid, clean up after transition. + delete_ok = false; + } // Allow any pending reads or writes to complete if (mCacheReadHandle != LLTextureCache::nullHandle()) @@ -2737,6 +2742,9 @@ bool LLTextureFetch::runCondition() // Threads: Ttf void LLTextureFetch::commonUpdate() { + // Release waiters + releaseHttpWaiters(); + // Run a cross-thread command, if any. cmdDoWork(); -- cgit v1.3 From 3f032e33f2b2f929b229cf4d358b9c9d297856b8 Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Thu, 19 Jul 2012 13:41:18 -0400 Subject: SH-3280 Better init/shutdown functionality for llcorehttp by llappviewer Isolate llcorehttp initialization into a utility class (LLAppCoreHttp) that provides glue between app and library (sets up policies, handles notifications). Introduce 'TextureFetchConcurrency' debug setting to provide some field control when absolutely necessary. --- indra/newview/CMakeLists.txt | 2 + indra/newview/app_settings/settings.xml | 11 ++ indra/newview/llappcorehttp.cpp | 186 +++++++++++++++++++++++++++++++ indra/newview/llappcorehttp.h | 86 +++++++++++++++ indra/newview/llappviewer.cpp | 189 +------------------------------- indra/newview/llappviewer.h | 7 ++ indra/newview/lltexturefetch.cpp | 9 +- indra/newview/lltexturefetch.h | 12 +- 8 files changed, 310 insertions(+), 192 deletions(-) create mode 100644 indra/newview/llappcorehttp.cpp create mode 100644 indra/newview/llappcorehttp.h (limited to 'indra/newview') diff --git a/indra/newview/CMakeLists.txt b/indra/newview/CMakeLists.txt index ab81cadc96..94286f0ff4 100644 --- a/indra/newview/CMakeLists.txt +++ b/indra/newview/CMakeLists.txt @@ -91,6 +91,7 @@ set(viewer_SOURCE_FILES llagentwearables.cpp llagentwearablesfetch.cpp llanimstatelabels.cpp + llappcorehttp.cpp llappearancemgr.cpp llappviewer.cpp llappviewerlistener.cpp @@ -648,6 +649,7 @@ set(viewer_HEADER_FILES llagentwearables.h llagentwearablesfetch.h llanimstatelabels.h + llappcorehttp.h llappearance.h llappearancemgr.h llappviewer.h diff --git a/indra/newview/app_settings/settings.xml b/indra/newview/app_settings/settings.xml index c9b4de0140..fc32e65410 100644 --- a/indra/newview/app_settings/settings.xml +++ b/indra/newview/app_settings/settings.xml @@ -10743,6 +10743,17 @@ Value 0 + TextureFetchConcurrency + + Comment + Maximum number of HTTP connections used for texture fetches + Persist + 1 + Type + U32 + Value + 0 + TextureFetchDebuggerEnabled Comment diff --git a/indra/newview/llappcorehttp.cpp b/indra/newview/llappcorehttp.cpp new file mode 100644 index 0000000000..e89c8d5ac2 --- /dev/null +++ b/indra/newview/llappcorehttp.cpp @@ -0,0 +1,186 @@ +/** + * @file llappcorehttp.cpp + * @brief + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include "llviewerprecompiledheaders.h" + +#include "llappcorehttp.h" + +#include "llviewercontrol.h" + + +const F64 LLAppCoreHttp::MAX_THREAD_WAIT_TIME(10.0); + +LLAppCoreHttp::LLAppCoreHttp() + : mRequest(NULL), + mStopHandle(LLCORE_HTTP_HANDLE_INVALID), + mStopRequested(0.0), + mStopped(false), + mPolicyDefault(-1) +{} + + +LLAppCoreHttp::~LLAppCoreHttp() +{ + delete mRequest; + mRequest = NULL; +} + + +void LLAppCoreHttp::init() +{ + LLCore::HttpStatus status = LLCore::HttpRequest::createService(); + if (! status) + { + LL_ERRS("Init") << "Failed to initialize HTTP services. Reason: " + << status.toString() + << LL_ENDL; + } + + // Point to our certs or SSH/https: will fail on connect + status = LLCore::HttpRequest::setPolicyGlobalOption(LLCore::HttpRequest::GP_CA_FILE, + gDirUtilp->getCAFile()); + if (! status) + { + LL_ERRS("Init") << "Failed to set CA File for HTTP services. Reason: " + << status.toString() + << LL_ENDL; + } + + // Establish HTTP Proxy. "LLProxy" is a special string which directs + // the code to use LLProxy::applyProxySettings() to establish any + // HTTP or SOCKS proxy for http operations. + status = LLCore::HttpRequest::setPolicyGlobalOption(LLCore::HttpRequest::GP_LLPROXY, 1); + if (! status) + { + LL_ERRS("Init") << "Failed to set HTTP proxy for HTTP services. Reason: " + << status.toString() + << LL_ENDL; + } + + // Tracing levels for library & libcurl (note that 2 & 3 are beyond spammy): + // 0 - None + // 1 - Basic start, stop simple transitions + // 2 - libcurl CURLOPT_VERBOSE mode with brief lines + // 3 - with partial data content + status = LLCore::HttpRequest::setPolicyGlobalOption(LLCore::HttpRequest::GP_TRACE, 0); + + // Setup default policy and constrain if directed to + mPolicyDefault = LLCore::HttpRequest::DEFAULT_POLICY_ID; + static const std::string texture_concur("TextureFetchConcurrency"); + if (gSavedSettings.controlExists(texture_concur)) + { + U32 concur(llmin(gSavedSettings.getU32(texture_concur), U32(12))); + + if (concur > 0) + { + LLCore::HttpStatus status; + status = LLCore::HttpRequest::setPolicyClassOption(mPolicyDefault, + LLCore::HttpRequest::CP_CONNECTION_LIMIT, + concur); + if (! status) + { + LL_WARNS("Init") << "Unable to set texture fetch concurrency. Reason: " + << status.toString() + << LL_ENDL; + } + else + { + LL_INFOS("Init") << "Application settings overriding default texture fetch concurrency. New value: " + << concur + << LL_ENDL; + } + } + } + + // Kick the thread + status = LLCore::HttpRequest::startThread(); + if (! status) + { + LL_ERRS("Init") << "Failed to start HTTP servicing thread. Reason: " + << status.toString() + << LL_ENDL; + } + + mRequest = new LLCore::HttpRequest; +} + + +void LLAppCoreHttp::requestStop() +{ + llassert_always(mRequest); + + mStopHandle = mRequest->requestStopThread(this); + if (LLCORE_HTTP_HANDLE_INVALID != mStopHandle) + { + mStopRequested = LLTimer::getTotalSeconds(); + } +} + + +void LLAppCoreHttp::cleanup() +{ + if (LLCORE_HTTP_HANDLE_INVALID == mStopHandle) + { + // Should have been started already... + requestStop(); + } + + if (LLCORE_HTTP_HANDLE_INVALID == mStopHandle) + { + LL_WARNS("Cleanup") << "Attempting to cleanup HTTP services without thread shutdown" + << LL_ENDL; + } + else + { + while (! mStopped && LLTimer::getTotalSeconds() < (mStopRequested + MAX_THREAD_WAIT_TIME)) + { + mRequest->update(200000); + ms_sleep(50); + } + if (! mStopped) + { + LL_WARNS("Cleanup") << "Attempting to cleanup HTTP services with thread shutdown incomplete" + << LL_ENDL; + } + } + + delete mRequest; + mRequest = NULL; + + LLCore::HttpStatus status = LLCore::HttpRequest::destroyService(); + if (! status) + { + LL_WARNS("Cleanup") << "Failed to shutdown HTTP services, continuing. Reason: " + << status.toString() + << LL_ENDL; + } +} + + +void LLAppCoreHttp::onCompleted(LLCore::HttpHandle, LLCore::HttpResponse *) +{ + mStopped = true; +} diff --git a/indra/newview/llappcorehttp.h b/indra/newview/llappcorehttp.h new file mode 100644 index 0000000000..241d73ad52 --- /dev/null +++ b/indra/newview/llappcorehttp.h @@ -0,0 +1,86 @@ +/** + * @file llappcorehttp.h + * @brief Singleton initialization/shutdown class for llcorehttp library + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef _LL_APP_COREHTTP_H_ +#define _LL_APP_COREHTTP_H_ + + +#include "httprequest.h" +#include "httphandler.h" +#include "httpresponse.h" + + +// This class manages the lifecyle of the core http library. +// Slightly different style than traditional code but reflects +// the use of handler classes and light-weight interface +// object instances of the new libraries. To be used +// as a singleton and static construction is fine. +class LLAppCoreHttp : public LLCore::HttpHandler +{ +public: + LLAppCoreHttp(); + ~LLAppCoreHttp(); + + // Initialize the LLCore::HTTP library creating service classes + // and starting the servicing thread. Caller is expected to do + // other initializations (SSL mutex, thread hash function) appropriate + // for the application. + void init(); + + // Request that the servicing thread stop servicing requests, + // release resource references and stop. Request is asynchronous + // and @see cleanup() will perform a limited wait loop for this + // request to stop the thread. + void requestStop(); + + // Terminate LLCore::HTTP library services. Caller is expected + // to have made a best-effort to shutdown the servicing thread + // by issuing a requestThreadStop() and waiting for completion + // notification that the stop has completed. + void cleanup(); + + // Notification when the stop request is complete. + virtual void onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * response); + + // Retrieve the policy class for default operations. + int getPolicyDefault() const + { + return mPolicyDefault; + } + +private: + static const F64 MAX_THREAD_WAIT_TIME; + +private: + LLCore::HttpRequest * mRequest; + LLCore::HttpHandle mStopHandle; + F64 mStopRequested; + bool mStopped; + int mPolicyDefault; +}; + + +#endif // _LL_APP_COREHTTP_H_ diff --git a/indra/newview/llappviewer.cpp b/indra/newview/llappviewer.cpp index 0549a972e1..f8ee1a477f 100644 --- a/indra/newview/llappviewer.cpp +++ b/indra/newview/llappviewer.cpp @@ -215,10 +215,6 @@ #include "llmachineid.h" #include "llmainlooprepeater.h" -// LLCore::HTTP -#include "httpcommon.h" -#include "httprequest.h" -#include "httphandler.h" // *FIX: These extern globals should be cleaned up. // The globals either represent state/config/resource-storage of either @@ -334,53 +330,6 @@ static std::string gLaunchFileOnQuit; // Used on Win32 for other apps to identify our window (eg, win_setup) const char* const VIEWER_WINDOW_CLASSNAME = "Second Life"; -namespace -{ - -// This class manages the lifecyle of the core http library. -// Slightly different style than traditional code but reflects -// the use of handler classes and light-weight interface -// object instances of the new libraries. To be used -// as a singleton and static construction is fine. -class CoreHttp : public LLCore::HttpHandler -{ -public: - CoreHttp(); - ~CoreHttp(); - - // Initialize the LLCore::HTTP library creating service classes - // and starting the servicing thread. Caller is expected to do - // other initializations (SSL mutex, thread hash function) appropriate - // for the application. - void init(); - - // Request that the servicing thread stop servicing requests, - // release resource references and stop. - void requestStop(); - - // Terminate LLCore::HTTP library services. Caller is expected - // to have made a best-effort to shutdown the servicing thread - // by issuing a requestThreadStop() and waiting for completion - // notification that the stop has completed. - void cleanup(); - - // Notification when the stop request is complete. - virtual void onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * response); - -private: - static const F64 MAX_THREAD_WAIT_TIME; - -private: - LLCore::HttpRequest * mRequest; - LLCore::HttpHandle mStopHandle; - F64 mStopRequested; - bool mStopped; -}; - -CoreHttp coreHttpLib; - -} // end anonymous namespace - //-- LLDeferredTaskList ------------------------------------------------------ /** @@ -775,8 +724,9 @@ bool LLAppViewer::init() LLViewerStatsRecorder::initClass(); #endif - // Initialize the non-LLCurl libcurl library - coreHttpLib.init(); + // Initialize the non-LLCurl libcurl library. Should be called + // before consumers (LLTextureFetch). + mAppCoreHttp.init(); // *NOTE:Mani - LLCurl::initClass is not thread safe. // Called before threads are created. @@ -1915,7 +1865,7 @@ bool LLAppViewer::cleanup() // Delete workers first // shotdown all worker threads before deleting them in case of co-dependencies - coreHttpLib.requestStop(); + mAppCoreHttp.requestStop(); sTextureFetch->shutdown(); sTextureCache->shutdown(); sImageDecodeThread->shutdown(); @@ -2000,7 +1950,7 @@ bool LLAppViewer::cleanup() LLCurl::cleanupClass(); // Non-LLCurl libcurl library - coreHttpLib.cleanup(); + mAppCoreHttp.cleanup(); // If we're exiting to launch an URL, do that here so the screen // is at the right resolution before we launch IE. @@ -5298,132 +5248,3 @@ void LLAppViewer::metricsSend(bool enable_reporting) gViewerAssetStatsMain->reset(); } -namespace -{ - -const F64 CoreHttp::MAX_THREAD_WAIT_TIME(10.0); - -CoreHttp::CoreHttp() - : mRequest(NULL), - mStopHandle(LLCORE_HTTP_HANDLE_INVALID), - mStopRequested(0.0), - mStopped(false) -{} - - -CoreHttp::~CoreHttp() -{ - delete mRequest; - mRequest = NULL; -} - - -void CoreHttp::init() -{ - LLCore::HttpStatus status = LLCore::HttpRequest::createService(); - if (! status) - { - LL_ERRS("Init") << "Failed to initialize HTTP services. Reason: " - << status.toString() - << LL_ENDL; - } - - // Point to our certs or SSH/https: will fail on connect - status = LLCore::HttpRequest::setPolicyGlobalOption(LLCore::HttpRequest::GP_CA_FILE, - gDirUtilp->getCAFile()); - if (! status) - { - LL_ERRS("Init") << "Failed to set CA File for HTTP services. Reason: " - << status.toString() - << LL_ENDL; - } - - // Establish HTTP Proxy. "LLProxy" is a special string which directs - // the code to use LLProxy::applyProxySettings() to establish any - // HTTP or SOCKS proxy for http operations. - status = LLCore::HttpRequest::setPolicyGlobalOption(LLCore::HttpRequest::GP_LLPROXY, 1); - if (! status) - { - LL_ERRS("Init") << "Failed to set HTTP proxy for HTTP services. Reason: " - << status.toString() - << LL_ENDL; - } - - // Tracing levels for library & libcurl (note that 2 & 3 are beyond spammy): - // 0 - None - // 1 - Basic start, stop simple transitions - // 2 - libcurl CURLOPT_VERBOSE mode with brief lines - // 3 - with partial data content - status = LLCore::HttpRequest::setPolicyGlobalOption(LLCore::HttpRequest::GP_TRACE, 0); - - // Kick the thread - status = LLCore::HttpRequest::startThread(); - if (! status) - { - LL_ERRS("Init") << "Failed to start HTTP servicing thread. Reason: " - << status.toString() - << LL_ENDL; - } - - mRequest = new LLCore::HttpRequest; -} - - -void CoreHttp::requestStop() -{ - llassert_always(mRequest); - - mStopHandle = mRequest->requestStopThread(this); - if (LLCORE_HTTP_HANDLE_INVALID != mStopHandle) - { - mStopRequested = LLTimer::getTotalSeconds(); - } -} - - -void CoreHttp::cleanup() -{ - if (LLCORE_HTTP_HANDLE_INVALID == mStopHandle) - { - // Should have been started already... - requestStop(); - } - - if (LLCORE_HTTP_HANDLE_INVALID == mStopHandle) - { - LL_WARNS("Cleanup") << "Attempting to cleanup HTTP services without thread shutdown" - << LL_ENDL; - } - else - { - while (! mStopped && LLTimer::getTotalSeconds() < (mStopRequested + MAX_THREAD_WAIT_TIME)) - { - mRequest->update(200000); - ms_sleep(50); - } - if (! mStopped) - { - LL_WARNS("Cleanup") << "Attempting to cleanup HTTP services with thread shutdown incomplete" - << LL_ENDL; - } - } - - delete mRequest; - mRequest = NULL; - - LLCore::HttpStatus status = LLCore::HttpRequest::destroyService(); - if (! status) - { - LL_WARNS("Cleanup") << "Failed to shutdown HTTP services, continuing. Reason: " - << status.toString() - << LL_ENDL; - } -} - - -void CoreHttp::onCompleted(LLCore::HttpHandle, LLCore::HttpResponse *) -{ - mStopped = true; -} - -} // end anonymous namespace diff --git a/indra/newview/llappviewer.h b/indra/newview/llappviewer.h index f7d019ccba..a694ea9fe6 100644 --- a/indra/newview/llappviewer.h +++ b/indra/newview/llappviewer.h @@ -31,6 +31,7 @@ #include "llcontrol.h" #include "llsys.h" // for LLOSInfo #include "lltimer.h" +#include "llappcorehttp.h" class LLCommandLineParser; class LLFrameTimer; @@ -173,6 +174,9 @@ public: // Metrics policy helper statics. static void metricsUpdateRegion(U64 region_handle); static void metricsSend(bool enable_reporting); + + // llcorehttp init/shutdown/config information. + LLAppCoreHttp & getAppCoreHttp() { return mAppCoreHttp; } protected: virtual bool initWindow(); // Initialize the viewer's window. @@ -270,6 +274,9 @@ private: boost::scoped_ptr mUpdater; + // llcorehttp library init/shutdown helper + LLAppCoreHttp mAppCoreHttp; + //--------------------------------------------- //*NOTE: Mani - legacy updater stuff // Still useable? diff --git a/indra/newview/lltexturefetch.cpp b/indra/newview/lltexturefetch.cpp index 51d57ccf68..b5586d1250 100755 --- a/indra/newview/lltexturefetch.cpp +++ b/indra/newview/lltexturefetch.cpp @@ -868,7 +868,7 @@ LLTextureFetchWorker::LLTextureFetchWorker(LLTextureFetch* fetcher, mMetricsStartTime(0), mHttpHandle(LLCORE_HTTP_HANDLE_INVALID), mHttpBufferArray(NULL), - mHttpPolicyClass(LLCore::HttpRequest::DEFAULT_POLICY_ID), + mHttpPolicyClass(mFetcher->mHttpPolicyClass), mHttpActive(false), mHttpReplySize(0U), mHttpReplyOffset(0U), @@ -2302,6 +2302,7 @@ LLTextureFetch::LLTextureFetch(LLTextureCache* cache, LLImageDecodeThread* image mHttpOptions(NULL), mHttpHeaders(NULL), mHttpMetricsHeaders(NULL), + mHttpPolicyClass(LLCore::HttpRequest::DEFAULT_POLICY_ID), mHttpSemaphore(HTTP_REQUESTS_IN_QUEUE_HIGH_WATER), mTotalCacheReadCount(0U), mTotalCacheWriteCount(0U), @@ -2324,6 +2325,7 @@ LLTextureFetch::LLTextureFetch(LLTextureCache* cache, LLImageDecodeThread* image mHttpHeaders->mHeaders.push_back("Accept: image/x-j2c"); mHttpMetricsHeaders = new LLCore::HttpHeaders; mHttpMetricsHeaders->mHeaders.push_back("Content-Type: application/llsd+xml"); + mHttpPolicyClass = LLAppViewer::instance()->getAppCoreHttp().getPolicyDefault(); } LLTextureFetch::~LLTextureFetch() @@ -3631,7 +3633,6 @@ bool TFReqSendMetrics::doWork(LLTextureFetch * fetcher) { static const U32 report_priority(1); - static const int report_policy_class(LLCore::HttpRequest::DEFAULT_POLICY_ID); static LLCore::HttpHandler * const handler(fetcher->isQAMode() || true ? &stats_handler : NULL); if (! gViewerAssetStatsThread1) @@ -3671,7 +3672,7 @@ TFReqSendMetrics::doWork(LLTextureFetch * fetcher) LLCore::BufferArrayStream bas(ba); LLSDSerialize::toXML(merged_llsd, bas); - fetcher->getHttpRequest().requestPost(report_policy_class, + fetcher->getHttpRequest().requestPost(fetcher->getPolicyClass(), report_priority, mCapsURL, ba, @@ -3797,7 +3798,7 @@ LLTextureFetchDebugger::LLTextureFetchDebugger(LLTextureFetch* fetcher, LLTextur mTextureCache(cache), mImageDecodeThread(imagedecodethread), mHttpHeaders(NULL), - mHttpPolicyClass(LLCore::HttpRequest::DEFAULT_POLICY_ID) + mHttpPolicyClass(fetcher->getPolicyClass()) { init(); } diff --git a/indra/newview/lltexturefetch.h b/indra/newview/lltexturefetch.h index 54ffeda8df..115e471bc9 100644 --- a/indra/newview/lltexturefetch.h +++ b/indra/newview/lltexturefetch.h @@ -164,6 +164,9 @@ public: // Threads: T* LLCore::HttpRequest & getHttpRequest() { return *mHttpRequest; } + // Threads: T* + LLCore::HttpRequest::policy_t getPolicyClass() const { return mHttpPolicyClass; } + // Return a pointer to the shared metrics headers definition. // Does not increment the reference count, caller is required // to do that to hold a reference for any length of time. @@ -339,10 +342,11 @@ private: // Interfaces and objects into the core http library used // to make our HTTP requests. These replace the various // LLCurl interfaces used in the past. - LLCore::HttpRequest * mHttpRequest; // Ttf - LLCore::HttpOptions * mHttpOptions; // Ttf - LLCore::HttpHeaders * mHttpHeaders; // Ttf - LLCore::HttpHeaders * mHttpMetricsHeaders; // Ttf + LLCore::HttpRequest * mHttpRequest; // Ttf + LLCore::HttpOptions * mHttpOptions; // Ttf + LLCore::HttpHeaders * mHttpHeaders; // Ttf + LLCore::HttpHeaders * mHttpMetricsHeaders; // Ttf + LLCore::HttpRequest::policy_t mHttpPolicyClass; // T* // We use a resource semaphore to keep HTTP requests in // WAIT_HTTP_RESOURCE2 if there aren't sufficient slots in the -- cgit v1.3 From 1d5490e752deeff316658f4850aac5fc96a91866 Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Wed, 25 Jul 2012 22:29:53 +0000 Subject: SH-3304 drano-http viewer fails to launch on linux libapr-1.so.0.4.5 and libaprutil-1.so.0.4.1 missing from manifest. --- indra/newview/viewer_manifest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'indra/newview') diff --git a/indra/newview/viewer_manifest.py b/indra/newview/viewer_manifest.py index a0dfc19e9c..42691fb51b 100644 --- a/indra/newview/viewer_manifest.py +++ b/indra/newview/viewer_manifest.py @@ -1027,10 +1027,10 @@ class Linux_i686Manifest(LinuxManifest): if self.prefix("../packages/lib/release", dst="lib"): self.path("libapr-1.so") self.path("libapr-1.so.0") - self.path("libapr-1.so.0.4.2") + self.path("libapr-1.so.0.4.5") self.path("libaprutil-1.so") self.path("libaprutil-1.so.0") - self.path("libaprutil-1.so.0.3.10") + self.path("libaprutil-1.so.0.4.1") self.path("libboost_program_options-mt.so.1.48.0") self.path("libboost_regex-mt.so.1.48.0") self.path("libboost_thread-mt.so.1.48.0") -- cgit v1.3 From 7d98d1afb036ce4b221e101e6de153a3d978ba9f Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Fri, 3 Aug 2012 14:02:36 -0400 Subject: Uniform gray background for texture console window. --- indra/newview/lltextureview.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'indra/newview') diff --git a/indra/newview/lltextureview.cpp b/indra/newview/lltextureview.cpp index f7636b2473..29925657b5 100644 --- a/indra/newview/lltextureview.cpp +++ b/indra/newview/lltextureview.cpp @@ -524,8 +524,8 @@ void LLGLTexMemBar::draw() // Gray background using completely magic numbers gGL.color4f(0.f, 0.f, 0.f, 0.25f); - const LLRect & rect(getRect()); - gl_rect_2d(-4, v_offset, rect.mRight - rect.mLeft + 2, v_offset + line_height*4); + // const LLRect & rect(getRect()); + // gl_rect_2d(-4, v_offset, rect.mRight - rect.mLeft + 2, v_offset + line_height*4); std::string text = ""; LLFontGL::getFontMonospace()->renderUTF8(text, 0, 0, v_offset + line_height*6, -- cgit v1.3 From 9a6881bdedf530872c442bddbe0335ca475c736e Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Fri, 17 Aug 2012 12:01:22 -0400 Subject: SH-3328 Permanently add 'QAModeHttpTrace' setting to control llcorehttp trace functionality llcorehttp implements a nice trace facility but it hasn't been attached to an external control to date. This hands over control to a *non-persistent* setting that can be used for QA or field diagnostics. --- indra/newview/llappcorehttp.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'indra/newview') diff --git a/indra/newview/llappcorehttp.cpp b/indra/newview/llappcorehttp.cpp index e89c8d5ac2..a3cd0638d5 100644 --- a/indra/newview/llappcorehttp.cpp +++ b/indra/newview/llappcorehttp.cpp @@ -85,7 +85,9 @@ void LLAppCoreHttp::init() // 1 - Basic start, stop simple transitions // 2 - libcurl CURLOPT_VERBOSE mode with brief lines // 3 - with partial data content - status = LLCore::HttpRequest::setPolicyGlobalOption(LLCore::HttpRequest::GP_TRACE, 0); + long trace_level(0L); + trace_level = long(gSavedSettings.getU32("QAModeHttpTrace")); + status = LLCore::HttpRequest::setPolicyGlobalOption(LLCore::HttpRequest::GP_TRACE, trace_level); // Setup default policy and constrain if directed to mPolicyDefault = LLCore::HttpRequest::DEFAULT_POLICY_ID; -- cgit v1.3 From 9de14fe45b6fc3b97e8c1ca91161e1fd5a9aab65 Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Fri, 17 Aug 2012 12:33:12 -0400 Subject: SH-3327 HTTP transfers not showing up in texture console 'net tot tex' value. Dropped an argument during integration which made the total byte count read lower than expected. Everything else is fine, however. --- indra/newview/lltexturefetch.cpp | 4 ++-- indra/newview/lltexturefetch.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'indra/newview') diff --git a/indra/newview/lltexturefetch.cpp b/indra/newview/lltexturefetch.cpp index efaeafbe78..718b15825d 100755 --- a/indra/newview/lltexturefetch.cpp +++ b/indra/newview/lltexturefetch.cpp @@ -924,7 +924,7 @@ LLTextureFetchWorker::~LLTextureFetchWorker() mHttpBufferArray = NULL; } unlockWorkMutex(); // -Mw - mFetcher->removeFromHTTPQueue(mID); + mFetcher->removeFromHTTPQueue(mID, 0); mFetcher->removeHttpWaiter(mID); mFetcher->updateStateStats(mCacheReadCount, mCacheWriteCount, mResourceWaitCount); } @@ -1875,7 +1875,7 @@ void LLTextureFetchWorker::onCompleted(LLCore::HttpHandle handle, LLCore::HttpRe } } - mFetcher->removeFromHTTPQueue(mID); + mFetcher->removeFromHTTPQueue(mID, data_size); recordTextureDone(true); } // -Mw diff --git a/indra/newview/lltexturefetch.h b/indra/newview/lltexturefetch.h index 115e471bc9..95ec8c65c0 100644 --- a/indra/newview/lltexturefetch.h +++ b/indra/newview/lltexturefetch.h @@ -226,7 +226,7 @@ protected: void addToHTTPQueue(const LLUUID& id); // Threads: T* - void removeFromHTTPQueue(const LLUUID& id, S32 received_size = 0); + void removeFromHTTPQueue(const LLUUID& id, S32 received_size); // Identical to @deleteRequest but with different arguments // (caller already has the worker pointer). -- cgit v1.3 From d06739facfc13f5e7081783f85c1dd3576c3c155 Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Fri, 17 Aug 2012 13:16:22 -0400 Subject: SH-3328 llcorehttp tracing control needed defensive logic for missing setting Guard for the typical case of a missing setting with a bracketing gSavedSettings.controlExists() call. --- indra/newview/llappcorehttp.cpp | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) (limited to 'indra/newview') diff --git a/indra/newview/llappcorehttp.cpp b/indra/newview/llappcorehttp.cpp index a3cd0638d5..0d7d41304d 100644 --- a/indra/newview/llappcorehttp.cpp +++ b/indra/newview/llappcorehttp.cpp @@ -85,10 +85,14 @@ void LLAppCoreHttp::init() // 1 - Basic start, stop simple transitions // 2 - libcurl CURLOPT_VERBOSE mode with brief lines // 3 - with partial data content - long trace_level(0L); - trace_level = long(gSavedSettings.getU32("QAModeHttpTrace")); - status = LLCore::HttpRequest::setPolicyGlobalOption(LLCore::HttpRequest::GP_TRACE, trace_level); - + static const std::string http_trace("QAModeHttpTrace"); + if (gSavedSettings.controlExists(http_trace)) + { + long trace_level(0L); + trace_level = long(gSavedSettings.getU32(http_trace)); + status = LLCore::HttpRequest::setPolicyGlobalOption(LLCore::HttpRequest::GP_TRACE, trace_level); + } + // Setup default policy and constrain if directed to mPolicyDefault = LLCore::HttpRequest::DEFAULT_POLICY_ID; static const std::string texture_concur("TextureFetchConcurrency"); -- cgit v1.3 From 4b86f8983ad343b675e3f4960e91f0d4cb876dea Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Sun, 19 Aug 2012 18:51:21 -0400 Subject: SH-3329 Cached scene loads slower than same scene with cleared cache. A/B comparison with original code showed the newer issuing lower-priority requests of the cache reader and some other minor changes. Brought them into agreement (this is cargo-cult programming). Made the HTTP resource semaphore an atomic int for rigorous correctness across threads. I swear I'm going to tear down this code someday. --- indra/newview/lltexturefetch.cpp | 47 ++++++++++++++++++++++++++++------------ indra/newview/lltexturefetch.h | 4 ++-- 2 files changed, 35 insertions(+), 16 deletions(-) (limited to 'indra/newview') diff --git a/indra/newview/lltexturefetch.cpp b/indra/newview/lltexturefetch.cpp index 718b15825d..6ae20edb9f 100755 --- a/indra/newview/lltexturefetch.cpp +++ b/indra/newview/lltexturefetch.cpp @@ -439,20 +439,27 @@ private: void lockWorkMutex() { mWorkMutex.lock(); } void unlockWorkMutex() { mWorkMutex.unlock(); } + // Threads: Ttf // Locks: Mw - void acquireHttpSemaphore() + bool acquireHttpSemaphore() { llassert(! mHttpHasResource); + if (mFetcher->mHttpSemaphore <= 0) + { + return false; + } mHttpHasResource = true; - --mFetcher->mHttpSemaphore; + mFetcher->mHttpSemaphore--; + return true; } + // Threads: Ttf // Locks: Mw void releaseHttpSemaphore() { llassert(mHttpHasResource); mHttpHasResource = false; - ++mFetcher->mHttpSemaphore; + mFetcher->mHttpSemaphore++; } private: @@ -901,6 +908,9 @@ LLTextureFetchWorker::~LLTextureFetchWorker() lockWorkMutex(); // +Mw (should be useless) if (mHttpHasResource) { + // Last-chance catchall to recover the resource. Using an + // atomic datatype solely because this can be running in + // another thread. releaseHttpSemaphore(); } if (mHttpActive) @@ -1351,7 +1361,7 @@ bool LLTextureFetchWorker::doWork(S32 param) // // If it looks like we're busy, keep this request here. // Otherwise, advance into the HTTP states. - if (mFetcher->mHttpSemaphore <= 0 || mFetcher->getHttpWaitersCount()) + if (mFetcher->getHttpWaitersCount() || ! acquireHttpSemaphore()) { mState = WAIT_HTTP_RESOURCE2; setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); @@ -1359,9 +1369,8 @@ bool LLTextureFetchWorker::doWork(S32 param) ++mResourceWaitCount; return false; } + mState = SEND_HTTP_REQ; - acquireHttpSemaphore(); - // *NOTE: You must invoke releaseHttpSemaphore() if you transition // to a state other than SEND_HTTP_REQ or WAIT_HTTP_REQ or abort // the request. @@ -1464,6 +1473,8 @@ bool LLTextureFetchWorker::doWork(S32 param) if (mState == WAIT_HTTP_REQ) { + // *NOTE: As stated above, all transitions out of this state should + // call releaseHttpSemaphore(). if (mLoaded) { S32 cur_size = mFormattedImage.notNull() ? mFormattedImage->getDataSize() : 0; @@ -1474,6 +1485,7 @@ bool LLTextureFetchWorker::doWork(S32 param) if(mWriteToCacheState == NOT_WRITE) //map tiles { mState = DONE; + releaseHttpSemaphore(); return true; // failed, means no map tile on the empty region. } @@ -1678,7 +1690,7 @@ bool LLTextureFetchWorker::doWork(S32 param) mAuxImage = NULL; llassert_always(mFormattedImage.notNull()); S32 discard = mHaveAllData ? 0 : mLoadedDiscard; - U32 image_priority = LLWorkerThread::PRIORITY_LOW | mWorkPriority; + U32 image_priority = LLWorkerThread::PRIORITY_NORMAL | mWorkPriority; mDecoded = FALSE; mState = DECODE_IMAGE_UPDATE; LL_DEBUGS("Texture") << mID << ": Decoding. Bytes: " << mFormattedImage->getDataSize() << " Discard: " << discard @@ -1789,7 +1801,7 @@ bool LLTextureFetchWorker::doWork(S32 param) if (mState == DONE) { - if (mDecodedDiscard > 0 && mDesiredDiscard < mDecodedDiscard) + if (mDecodedDiscard >= 0 && mDesiredDiscard < mDecodedDiscard) { // More data was requested, return to INIT mState = INIT; @@ -3373,7 +3385,9 @@ void LLTextureFetch::removeHttpWaiter(const LLUUID & tid) // Locks: -Mw (must not hold any worker when called) void LLTextureFetch::releaseHttpWaiters() { - if (mHttpSemaphore < HTTP_REQUESTS_IN_QUEUE_LOW_WATER) + // Use mHttpSemaphore rather than mHTTPTextureQueue.size() + // to avoid a lock. + if (mHttpSemaphore < (HTTP_REQUESTS_IN_QUEUE_HIGH_WATER - HTTP_REQUESTS_IN_QUEUE_LOW_WATER)) return; // Quickly make a copy of all the LLUIDs. Get off the @@ -3425,22 +3439,27 @@ void LLTextureFetch::releaseHttpWaiters() // with other callers. Do defensive things like getting // refreshed counts of requests and checking if someone else // has moved any worker state around.... - for (worker_list_t::iterator iter2(tids2.begin()); - tids2.end() != iter2 && mHttpSemaphore > 0; - ++iter2) + for (worker_list_t::iterator iter2(tids2.begin()); tids2.end() != iter2; ++iter2) { LLTextureFetchWorker * worker(* iter2); worker->lockWorkMutex(); // +Mw if (LLTextureFetchWorker::WAIT_HTTP_RESOURCE2 != worker->mState) { + // Not in expected state, try the next one worker->unlockWorkMutex(); // -Mw continue; } + if (! worker->acquireHttpSemaphore()) + { + // Out of active slots, quit + worker->unlockWorkMutex(); // -Mw + break; + } + worker->mState = LLTextureFetchWorker::SEND_HTTP_REQ; worker->setPriority(LLWorkerThread::PRIORITY_HIGH | worker->mWorkPriority); - worker->acquireHttpSemaphore(); worker->unlockWorkMutex(); // -Mw removeHttpWaiter(worker->mID); @@ -4154,7 +4173,7 @@ S32 LLTextureFetchDebugger::fillCurlQueue() mFetchingHistory[i].mCurlState = FetchEntry::CURL_IN_PROGRESS; mNbCurlRequests++; // Hack - if (mNbCurlRequests == 40) + if (mNbCurlRequests == HTTP_REQUESTS_IN_QUEUE_HIGH_WATER) // emulate normal pipeline break; } else diff --git a/indra/newview/lltexturefetch.h b/indra/newview/lltexturefetch.h index 95ec8c65c0..4294209f04 100644 --- a/indra/newview/lltexturefetch.h +++ b/indra/newview/lltexturefetch.h @@ -355,10 +355,10 @@ private: // where it's more expensive to get at them. Requests in either // SEND_HTTP_REQ or WAIT_HTTP_REQ charge against the semaphore // and tracking state transitions is critical to liveness. - int mHttpSemaphore; // Ttf + LLAtomicS32 mHttpSemaphore; // Ttf + Tmain typedef std::set wait_http_res_queue_t; - wait_http_res_queue_t mHttpWaitResource; // Mfnq + wait_http_res_queue_t mHttpWaitResource; // Mfnq // Cumulative stats on the states/requests issued by // textures running through here. -- cgit v1.3 From 7bee4b58ff1e36ca39abc090991833c43c8903cc Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Tue, 21 Aug 2012 12:28:51 -0400 Subject: SH-3325 texture load slow on some machines This doesn't really address 3325 directly but it is the result of research done while hunting it down. First, this is a thread safety improvement for canceled requests that have gone into resource wait state. I don't think we've seen a failure there but there was a window. It also cleans the resource wait queue earlier which lets us do less work and get requests more quickly into llcorehttp by bypassing the resource wait state. With this, I finally feel comfortable about rundown of requests. --- indra/newview/lltexturefetch.cpp | 39 +++++++++++++++++++++++++++++++++------ indra/newview/lltexturefetch.h | 3 +++ 2 files changed, 36 insertions(+), 6 deletions(-) (limited to 'indra/newview') diff --git a/indra/newview/lltexturefetch.cpp b/indra/newview/lltexturefetch.cpp index 6ae20edb9f..faaa9ed86b 100755 --- a/indra/newview/lltexturefetch.cpp +++ b/indra/newview/lltexturefetch.cpp @@ -1945,10 +1945,14 @@ bool LLTextureFetchWorker::deleteOK() if (WAIT_HTTP_RESOURCE2 == mState) { - // Don't delete the worker out from under the - // releaseHttpWaiters() method. Keep the pointers - // valid, clean up after transition. - delete_ok = false; + if (mFetcher->isHttpWaiter(mID)) + { + // Don't delete the worker out from under the releaseHttpWaiters() + // method. Keep the pointers valid, clean up after that method + // has recognized the cancelation and removed the UUID from the + // waiter list. + delete_ok = false; + } } // Allow any pending reads or writes to complete @@ -2551,7 +2555,6 @@ void LLTextureFetch::deleteRequest(const LLUUID& id, bool cancel) unlockQueue(); // -Mfq llassert_always(erased_1 > 0) ; - removeFromNetworkQueue(worker, cancel); llassert_always(!(worker->getFlags(LLWorkerClass::WCF_DELETE_REQUESTED))) ; @@ -3370,6 +3373,16 @@ void LLTextureFetch::removeHttpWaiter(const LLUUID & tid) mNetworkQueueMutex.unlock(); // -Mfnq } +// Threads: T* +bool LLTextureFetch::isHttpWaiter(const LLUUID & tid) +{ + mNetworkQueueMutex.lock(); // +Mfnq + wait_http_res_queue_t::iterator iter(mHttpWaitResource.find(tid)); + const bool ret(mHttpWaitResource.end() != iter); + mNetworkQueueMutex.unlock(); // -Mfnq + return ret; +} + // Release as many requests as permitted from the WAIT_HTTP_RESOURCE2 // state to the SEND_HTTP_REQ state based on their current priority. // @@ -3424,6 +3437,15 @@ void LLTextureFetch::releaseHttpWaiters() { tids2.push_back(worker); } + else + { + // If worker isn't found, this should be due to a request + // for deletion. We signal our recognition that this + // uuid shouldn't be used for resource waiting anymore by + // erasing it from the resource waiter list. That allows + // deleteOK to do final deletion on the worker. + removeHttpWaiter(* iter); + } } tids.clear(); @@ -3446,8 +3468,13 @@ void LLTextureFetch::releaseHttpWaiters() worker->lockWorkMutex(); // +Mw if (LLTextureFetchWorker::WAIT_HTTP_RESOURCE2 != worker->mState) { - // Not in expected state, try the next one + // Not in expected state, remove it, try the next one worker->unlockWorkMutex(); // -Mw + LL_WARNS("Texture") << "Resource-waited texture " << worker->mID + << " in unexpected state: " << worker->mState + << ". Removing from wait list." + << LL_ENDL; + removeHttpWaiter(worker->mID); continue; } diff --git a/indra/newview/lltexturefetch.h b/indra/newview/lltexturefetch.h index 4294209f04..fd2223ecb0 100644 --- a/indra/newview/lltexturefetch.h +++ b/indra/newview/lltexturefetch.h @@ -185,6 +185,9 @@ public: // Threads: T* void removeHttpWaiter(const LLUUID & tid); + // Threads: T* + bool isHttpWaiter(const LLUUID & tid); + // If there are slots, release one or more LLTextureFetchWorker // requests from resource wait state (WAIT_HTTP_RESOURCE) to // active (SEND_HTTP_REQ). -- cgit v1.3 From 67553c99f6e0a3146741c448f6d20c90488b8d5b Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Tue, 21 Aug 2012 14:02:53 -0400 Subject: Pointless edit to tickle TC/codeticket. --- indra/newview/lltexturefetch.cpp | 1 + 1 file changed, 1 insertion(+) (limited to 'indra/newview') diff --git a/indra/newview/lltexturefetch.cpp b/indra/newview/lltexturefetch.cpp index faaa9ed86b..422655879c 100755 --- a/indra/newview/lltexturefetch.cpp +++ b/indra/newview/lltexturefetch.cpp @@ -232,6 +232,7 @@ LLStat LLTextureFetch::sCacheReadLatency("texture_cache_read_latency", 128); // experiencial benefits promised. // + ////////////////////////////////////////////////////////////////////////////// // Tuning/Parameterization Constants -- cgit v1.3 From 09cd2a4b1a9f1ddf046fb0ce5d12988b968269a3 Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Fri, 7 Sep 2012 15:08:12 -0400 Subject: DRTVWR-209 Additional merge of viewer-development with SH-3316 drano-http code. Restore original deleteRequest/removeRequest implementation removing a small race. Remove a short-lived additional timeout scheme on requests which really isn't appropriate as originally implemented as we can have very long-lived requests on big regions. --- indra/newview/lltexturefetch.cpp | 61 ++++++++++++++++++---------------------- 1 file changed, 28 insertions(+), 33 deletions(-) (limited to 'indra/newview') diff --git a/indra/newview/lltexturefetch.cpp b/indra/newview/lltexturefetch.cpp index 43198d3725..8884c978b2 100644 --- a/indra/newview/lltexturefetch.cpp +++ b/indra/newview/lltexturefetch.cpp @@ -1068,8 +1068,6 @@ void LLTextureFetchWorker::startWork(S32 param) // Threads: Ttf bool LLTextureFetchWorker::doWork(S32 param) { - static const F32 FETCHING_TIMEOUT = 120.f;//seconds - static const LLCore::HttpStatus http_not_found(HTTP_NOT_FOUND); // 404 static const LLCore::HttpStatus http_service_unavail(HTTP_SERVICE_UNAVAILABLE); // 503 static const LLCore::HttpStatus http_not_sat(HTTP_REQUESTED_RANGE_NOT_SATISFIABLE); // 416; @@ -1637,32 +1635,12 @@ bool LLTextureFetchWorker::doWork(S32 param) } else { - // *FIXME: This auxiliary timeout logic appeared recently and then - // quickly disappeared. While I haven't seen it invoked, I'm leaving - // it active for now. - if(FETCHING_TIMEOUT < mRequestedTimer.getElapsedTimeF32()) - { - //timeout, abort. - LL_WARNS("Texture") << "Fetch of texture " << mID << " timed out after " - << mRequestedTimer.getElapsedTimeF32() - << " seconds. Canceling request." << LL_ENDL; - - if (LLCORE_HTTP_HANDLE_INVALID != mHttpHandle) - { - // Issue cancel on any outstanding request. Asynchronous - // so cancel may not actually take effect if operation is - // complete & queued. Either way, notification will - // complete and the request can be transitioned. - mFetcher->mHttpRequest->requestCancel(mHttpHandle, NULL); - } - else - { - // Shouldn't happen but if it does, cancel quickly. - mState = DONE; - releaseHttpSemaphore(); - return true; - } - } + // *HISTORY: There was a texture timeout test here originally that + // would cancel a request that was over 120 seconds old. That's + // probably not a good idea. Particularly rich regions can take + // an enormous amount of time to load textures. We'll revisit the + // various possible timeout components (total request time, connection + // time, I/O time, with and without retries, etc.) in the future. setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); return false; @@ -2571,19 +2549,36 @@ void LLTextureFetch::removeFromHTTPQueue(const LLUUID& id, S32 received_size) mHTTPTextureBits += received_size * 8; // Approximate - does not include header bits } // -Mfnq +// NB: If you change deleteRequest() you should probably make +// parallel changes in removeRequest(). They're functionally +// identical with only argument variations. +// // Threads: T* void LLTextureFetch::deleteRequest(const LLUUID& id, bool cancel) { lockQueue(); // +Mfq LLTextureFetchWorker* worker = getWorkerAfterLock(id); - unlockQueue(); // -Mfq + if (worker) + { + size_t erased_1 = mRequestMap.erase(worker->mID); + unlockQueue(); // -Mfq + + llassert_always(erased_1 > 0) ; + removeFromNetworkQueue(worker, cancel); + llassert_always(!(worker->getFlags(LLWorkerClass::WCF_DELETE_REQUESTED))) ; - // *TODO: Refactoring this code may have introduced a thread race - // here where other code can run between the lookup above and the - // removeRequest() below. - removeRequest(worker, cancel); + worker->scheduleDelete(); + } + else + { + unlockQueue(); // -Mfq + } } +// NB: If you change removeRequest() you should probably make +// parallel changes in deleteRequest(). They're functionally +// identical with only argument variations. +// // Threads: T* void LLTextureFetch::removeRequest(LLTextureFetchWorker* worker, bool cancel) { -- cgit v1.3 From a5e2ef6042c9c3fe69acd9daafebc7694151a21f Mon Sep 17 00:00:00 2001 From: Oz Linden Date: Tue, 23 Oct 2012 16:34:04 -0400 Subject: correct faulty merge --- indra/newview/llvovolume.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'indra/newview') diff --git a/indra/newview/llvovolume.cpp b/indra/newview/llvovolume.cpp index 8465a9dadd..5d1c335078 100644 --- a/indra/newview/llvovolume.cpp +++ b/indra/newview/llvovolume.cpp @@ -3847,6 +3847,7 @@ void LLRiggedVolume::update(const LLMeshSkinInfo* skin, LLVOAvatar* avatar, cons LLVector4a* pos = dst_face.mPositions; + if( pos && weight && dst_face.mExtents ) { LLFastTimer t(FTM_SKIN_RIGGED); @@ -4901,7 +4902,6 @@ void LLVolumeGeometryManager::genDrawInfo(LLSpatialGroup* group, U32 mask, std:: std::vector texture_list; - if( pos && weight && dst_face.mExtents ) { LLFastTimer t(FTM_GEN_DRAW_INFO_FACE_SIZE); if (batch_textures) @@ -4998,7 +4998,6 @@ void LLVolumeGeometryManager::genDrawInfo(LLSpatialGroup* group, U32 mask, std:: //create vertex buffer LLVertexBuffer* buffer = NULL; - if( dst_face.mExtents ) { LLFastTimer t(FTM_GEN_DRAW_INFO_ALLOCATE); buffer = createVertexBuffer(mask, buffer_usage); -- cgit v1.3 From 07410567c087a877ac8af25cd85c0100123d4a63 Mon Sep 17 00:00:00 2001 From: Nicky Date: Mon, 22 Oct 2012 21:39:48 +0200 Subject: LLDrawable::destroy: Illegal deletion of LLDrawable! --- indra/newview/pipeline.cpp | 2 ++ indra/newview/pipeline.h | 2 ++ 2 files changed, 4 insertions(+) (limited to 'indra/newview') diff --git a/indra/newview/pipeline.cpp b/indra/newview/pipeline.cpp index 678898797f..91dd441319 100644 --- a/indra/newview/pipeline.cpp +++ b/indra/newview/pipeline.cpp @@ -2011,6 +2011,7 @@ void LLPipeline::grabReferences(LLCullResult& result) void LLPipeline::clearReferences() { sCull = NULL; + mGroupSaveQ1.clear(); } void check_references(LLSpatialGroup* group, LLDrawable* drawable) @@ -2619,6 +2620,7 @@ void LLPipeline::rebuildPriorityGroups() group->clearState(LLSpatialGroup::IN_BUILD_Q1); } + mGroupSaveQ1 = mGroupQ1; mGroupQ1.clear(); mGroupQ1Locked = false; diff --git a/indra/newview/pipeline.h b/indra/newview/pipeline.h index fd2a1e06cd..7a0ca86231 100644 --- a/indra/newview/pipeline.h +++ b/indra/newview/pipeline.h @@ -668,6 +668,8 @@ protected: LLSpatialGroup::sg_vector_t mGroupQ1; //priority LLSpatialGroup::sg_vector_t mGroupQ2; // non-priority + LLSpatialGroup::sg_vector_t mGroupSaveQ1; // a place to save mGroupQ1 until it is safe to unref + LLSpatialGroup::sg_vector_t mMeshDirtyGroup; //groups that need rebuildMesh called U32 mMeshDirtyQueryObject; -- cgit v1.3