diff options
Diffstat (limited to 'indra/newview/llmeshrepository.cpp')
| -rwxr-xr-x | indra/newview/llmeshrepository.cpp | 2521 |
1 files changed, 1677 insertions, 844 deletions
diff --git a/indra/newview/llmeshrepository.cpp b/indra/newview/llmeshrepository.cpp index 2e02805c02..5afd2cb329 100755 --- a/indra/newview/llmeshrepository.cpp +++ b/indra/newview/llmeshrepository.cpp @@ -5,7 +5,7 @@ * * $LicenseInfo:firstyear=2005&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2010, Linden Research, Inc. + * Copyright (C) 2010-2014, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -38,11 +38,13 @@ #include "llcallbacklist.h" #include "llcurl.h" #include "lldatapacker.h" +#include "lldeadmantimer.h" #include "llfloatermodelpreview.h" #include "llfloaterperms.h" #include "lleconomy.h" #include "llimagej2c.h" #include "llhost.h" +#include "llmath.h" #include "llnotificationsutil.h" #include "llsd.h" #include "llsdutil_math.h" @@ -52,6 +54,7 @@ #include "llviewercontrol.h" #include "llviewerinventory.h" #include "llviewermenufile.h" +#include "llviewermessage.h" #include "llviewerobjectlist.h" #include "llviewerregion.h" #include "llviewertexturelist.h" @@ -65,6 +68,9 @@ #include "llfoldertype.h" #include "llviewerparcelmgr.h" #include "lluploadfloaterobservers.h" +#include "bufferarray.h" +#include "bufferstream.h" +#include "llfasttimer.h" #include "boost/lexical_cast.hpp" @@ -72,11 +78,296 @@ #include "netdb.h" #endif -#include <queue> + +// Purpose +// +// The purpose of this module is to provide access between the viewer +// and the asset system as regards to mesh objects. +// +// * High-throughput download of mesh assets from servers while +// following best industry practices for network profile. +// * Reliable expensing and upload of new mesh assets. +// * Recovery and retry from errors when appropriate. +// * Decomposition of mesh assets for preview and uploads. +// * And most important: all of the above without exposing the +// main thread to stalls due to deep processing or thread +// locking actions. In particular, the following operations +// on LLMeshRepository are very averse to any stalls: +// * loadMesh +// * getMeshHeader (For structural details, see: +// http://wiki.secondlife.com/wiki/Mesh/Mesh_Asset_Format) +// * notifyLoadedMeshes +// * getSkinInfo +// +// Threads +// +// main Main rendering thread, very sensitive to locking and other stalls +// repo Overseeing worker thread associated with the LLMeshRepoThread class +// decom Worker thread for mesh decomposition requests +// core HTTP worker thread: does the work but doesn't intrude here +// uploadN 0-N temporary mesh upload threads (0-1 in practice) +// +// Sequence of Operations +// +// What follows is a description of the retrieval of one LOD for +// a new mesh object. Work is performed by a series of short, quick +// actions distributed over a number of threads. Each is meant +// to proceed without stalling and the whole forms a deep request +// pipeline to achieve throughput. Ellipsis indicates a return +// or break in processing which is resumed elsewhere. +// +// main thread repo thread (run() method) +// +// loadMesh() invoked to request LOD +// append LODRequest to mPendingRequests +// ... +// other mesh requests may be made +// ... +// notifyLoadedMeshes() invoked to stage work +// append HeaderRequest to mHeaderReqQ +// ... +// scan mHeaderReqQ +// issue 4096-byte GET for header +// ... +// onCompleted() invoked for GET +// data copied +// headerReceived() invoked +// LLSD parsed +// mMeshHeader, mMeshHeaderSize updated +// scan mPendingLOD for LOD request +// push LODRequest to mLODReqQ +// ... +// scan mLODReqQ +// fetchMeshLOD() invoked +// issue Byte-Range GET for LOD +// ... +// onCompleted() invoked for GET +// data copied +// lodReceived() invoked +// unpack data into LLVolume +// append LoadedMesh to mLoadedQ +// ... +// notifyLoadedMeshes() invoked again +// scan mLoadedQ +// notifyMeshLoaded() for LOD +// setMeshAssetLoaded() invoked for system volume +// notifyMeshLoaded() invoked for each interested object +// ... +// +// Mutexes +// +// LLMeshRepository::mMeshMutex +// LLMeshRepoThread::mMutex +// LLMeshRepoThread::mHeaderMutex +// LLMeshRepoThread::mSignal (LLCondition) +// LLPhysicsDecomp::mSignal (LLCondition) +// LLPhysicsDecomp::mMutex +// LLMeshUploadThread::mMutex +// +// Mutex Order Rules +// +// 1. LLMeshRepoThread::mMutex before LLMeshRepoThread::mHeaderMutex +// 2. LLMeshRepository::mMeshMutex before LLMeshRepoThread::mMutex +// (There are more rules, haven't been extracted.) +// +// Data Member Access/Locking +// +// Description of how shared access to static and instance data +// members is performed. Each member is followed by the name of +// the mutex, if any, covering the data and then a list of data +// access models each of which is a triplet of the following form: +// +// {ro, wo, rw}.{main, repo, any}.{mutex, none} +// Type of access: read-only, write-only, read-write. +// Accessing thread or 'any' +// Relevant mutex held during access (several may be held) or 'none' +// +// A careful eye will notice some unsafe operations. Many of these +// have an alibi of some form. Several types of alibi are identified +// and listed here: +// +// [0] No alibi. Probably unsafe. +// [1] Single-writer, self-consistent readers. Old data must +// be tolerated by any reader but data will come true eventually. +// [2] Like [1] but provides a hint about thread state. These +// may be unsafe. +// [3] empty() check outside of lock. Can me made safish when +// done in double-check lock style. But this depends on +// std:: implementation and memory model. +// [4] Appears to be covered by a mutex but doesn't need one. +// [5] Read of a double-checked lock. +// +// So, in addition to documentation, take this as a to-do/review +// list and see if you can improve things. For porters to non-x86 +// architectures, the weaker memory models will make these platforms +// probabilistically more susceptible to hitting race conditions. +// True here and in other multi-thread code such as texture fetching. +// (Strong memory models make weak programmers. Weak memory models +// make strong programmers. Ref: arm, ppc, mips, alpha) +// +// LLMeshRepository: +// +// sBytesReceived none rw.repo.none, ro.main.none [1] +// sMeshRequestCount " +// sHTTPRequestCount " +// sHTTPLargeRequestCount " +// sHTTPRetryCount " +// sHTTPErrorCount " +// sLODPending mMeshMutex [4] rw.main.mMeshMutex +// sLODProcessing Repo::mMutex rw.any.Repo::mMutex +// sCacheBytesRead none rw.repo.none, ro.main.none [1] +// sCacheBytesWritten " +// sCacheReads " +// sCacheWrites " +// mLoadingMeshes mMeshMutex [4] rw.main.none, rw.any.mMeshMutex +// mSkinMap none rw.main.none +// mDecompositionMap none rw.main.none +// mPendingRequests mMeshMutex [4] rw.main.mMeshMutex +// mLoadingSkins mMeshMutex [4] rw.main.mMeshMutex +// mPendingSkinRequests mMeshMutex [4] rw.main.mMeshMutex +// mLoadingDecompositions mMeshMutex [4] rw.main.mMeshMutex +// mPendingDecompositionRequests mMeshMutex [4] rw.main.mMeshMutex +// mLoadingPhysicsShapes mMeshMutex [4] rw.main.mMeshMutex +// mPendingPhysicsShapeRequests mMeshMutex [4] rw.main.mMeshMutex +// mUploads none rw.main.none (upload thread accessing objects) +// mUploadWaitList none rw.main.none (upload thread accessing objects) +// mInventoryQ mMeshMutex [4] rw.main.mMeshMutex, ro.main.none [5] +// mUploadErrorQ mMeshMutex rw.main.mMeshMutex, rw.any.mMeshMutex +// mGetMeshVersion none rw.main.none +// +// LLMeshRepoThread: +// +// sActiveHeaderRequests mMutex rw.any.mMutex, ro.repo.none [1] +// sActiveLODRequests mMutex rw.any.mMutex, ro.repo.none [1] +// sMaxConcurrentRequests mMutex wo.main.none, ro.repo.none, ro.main.mMutex +// mMeshHeader mHeaderMutex rw.repo.mHeaderMutex, ro.main.mHeaderMutex, ro.main.none [0] +// mMeshHeaderSize mHeaderMutex rw.repo.mHeaderMutex +// mSkinRequests mMutex rw.repo.mMutex, ro.repo.none [5] +// mSkinInfoQ mMutex rw.repo.mMutex, rw.main.mMutex [5] (was: [0]) +// mDecompositionRequests mMutex rw.repo.mMutex, ro.repo.none [5] +// mPhysicsShapeRequests mMutex rw.repo.mMutex, ro.repo.none [5] +// mDecompositionQ mMutex rw.repo.mMutex, rw.main.mMutex [5] (was: [0]) +// mHeaderReqQ mMutex ro.repo.none [5], rw.repo.mMutex, rw.any.mMutex +// mLODReqQ mMutex ro.repo.none [5], rw.repo.mMutex, rw.any.mMutex +// mUnavailableQ mMutex rw.repo.none [0], ro.main.none [5], rw.main.mMutex +// mLoadedQ mMutex rw.repo.mMutex, ro.main.none [5], rw.main.mMutex +// mPendingLOD mMutex rw.repo.mMutex, rw.any.mMutex +// mGetMeshCapability mMutex rw.main.mMutex, ro.repo.mMutex (was: [0]) +// mGetMesh2Capability mMutex rw.main.mMutex, ro.repo.mMutex (was: [0]) +// mGetMeshVersion mMutex rw.main.mMutex, ro.repo.mMutex +// mHttp* none rw.repo.none +// +// LLMeshUploadThread: +// +// mDiscarded mMutex rw.main.mMutex, ro.uploadN.none [1] +// ... more ... +// +// QA/Development Testing +// +// Debug variable 'MeshUploadFakeErrors' takes a mask of bits that will +// simulate an error on fee query or upload. Defined bits are: +// +// 0x01 Simulate application error on fee check reading +// response body from file "fake_upload_error.xml" +// 0x02 Same as 0x01 but for actual upload attempt. +// 0x04 Simulate a transport problem on fee check with a +// locally-generated 500 status. +// 0x08 As with 0x04 but for the upload operation. +// +// For major changes, see the LL_MESH_FASTTIMER_ENABLE below and +// instructions for looking for frame stalls using fast timers. +// +// *TODO: Work list for followup actions: +// * Review anything marked as unsafe above, verify if there are real issues. +// * See if we can put ::run() into a hard sleep. May not actually perform better +// than the current scheme so be prepared for disappointment. You'll likely +// need to introduce a condition variable class that references a mutex in +// methods rather than derives from mutex which isn't correct. +// * On upload failures, make more information available to the alerting +// dialog. Get the structured information going into the log into a +// tree there. +// * Header parse failures come without much explanation. Elaborate. +// * Work queue for uploads? Any need for this or is the current scheme good +// enough? +// * Various temp buffers used in VFS I/O might be allocated once or even +// statically. Look for some wins here. +// * Move data structures holding mesh data used by main thread into main- +// thread-only access so that no locking is needed. May require duplication +// of some data so that worker thread has a minimal data set to guide +// operations. +// +// -------------------------------------------------------------------------- +// Development/Debug/QA Tools +// +// Enable here or in build environment to get fasttimer data on mesh fetches. +// +// Typically, this is used to perform A/B testing using the +// fasttimer console (shift-ctrl-9). This is done by looking +// for stalls due to lock contention between the main thread +// and the repository and HTTP code. In a release viewer, +// these appear as ping-time or worse spikes in frame time. +// With this instrumentation enabled, a stall will appear +// under the 'Mesh Fetch' timer which will be either top-level +// or under 'Render' time. + +#ifndef LL_MESH_FASTTIMER_ENABLE +#define LL_MESH_FASTTIMER_ENABLE 1 +#endif +#if LL_MESH_FASTTIMER_ENABLE +static LLFastTimer::DeclareTimer FTM_MESH_FETCH("Mesh Fetch"); + +#define MESH_FASTTIMER_DEFBLOCK LLFastTimer meshtimer(FTM_MESH_FETCH) +#else +#define MESH_FASTTIMER_DEFBLOCK +#endif // LL_MESH_FASTTIMER_ENABLE + + +// Random failure testing for development/QA. +// +// Set the MESH_*_FAILED macros to either 'false' or to +// an invocation of MESH_RANDOM_NTH_TRUE() with some +// suitable number. In production, all must be false. +// +// Example: +// #define MESH_HTTP_RESPONSE_FAILED MESH_RANDOM_NTH_TRUE(9) + +// 1-in-N calls will test true +#define MESH_RANDOM_NTH_TRUE(_N) ( ll_rand(S32(_N)) == 0 ) + +#define MESH_HTTP_RESPONSE_FAILED false +#define MESH_HEADER_PROCESS_FAILED false +#define MESH_LOD_PROCESS_FAILED false +#define MESH_SKIN_INFO_PROCESS_FAILED false +#define MESH_DECOMP_PROCESS_FAILED false +#define MESH_PHYS_SHAPE_PROCESS_FAILED false + +// -------------------------------------------------------------------------- + LLMeshRepository gMeshRepo; -const U32 MAX_MESH_REQUESTS_PER_SECOND = 100; +const S32 MESH_HEADER_SIZE = 4096; // Important: assumption is that headers fit in this space +const S32 REQUEST_HIGH_WATER_MIN = 32; // Limits for GetMesh regions +const S32 REQUEST_HIGH_WATER_MAX = 150; // Should remain under 2X throttle +const S32 REQUEST_LOW_WATER_MIN = 16; +const S32 REQUEST_LOW_WATER_MAX = 75; +const S32 REQUEST2_HIGH_WATER_MIN = 32; // Limits for GetMesh2 regions +const S32 REQUEST2_HIGH_WATER_MAX = 80; +const S32 REQUEST2_LOW_WATER_MIN = 16; +const S32 REQUEST2_LOW_WATER_MAX = 40; +const U32 LARGE_MESH_FETCH_THRESHOLD = 1U << 21; // Size at which requests goes to narrow/slow queue +const long SMALL_MESH_XFER_TIMEOUT = 120L; // Seconds to complete xfer, small mesh downloads +const long LARGE_MESH_XFER_TIMEOUT = 600L; // Seconds to complete xfer, large downloads + +// Would normally like to retry on uploads as some +// retryable failures would be recoverable. Unfortunately, +// the mesh service is using 500 (retryable) rather than +// 400/bad request (permanent) for a bad payload and +// retrying that just leads to revocation of the one-shot +// cap which then produces a 404 on retry destroying some +// (occasionally) useful error information. We'll leave +// upload retries to the user as in the past. SH-4667. +const long UPLOAD_RETRY_LIMIT = 0L; // Maximum mesh version to support. Three least significant digits are reserved for the minor version, // with major version changes indicating a format change that is not backwards compatible and should not @@ -87,35 +378,45 @@ const U32 MAX_MESH_REQUESTS_PER_SECOND = 100; const S32 MAX_MESH_VERSION = 999; U32 LLMeshRepository::sBytesReceived = 0; +U32 LLMeshRepository::sMeshRequestCount = 0; U32 LLMeshRepository::sHTTPRequestCount = 0; +U32 LLMeshRepository::sHTTPLargeRequestCount = 0; U32 LLMeshRepository::sHTTPRetryCount = 0; +U32 LLMeshRepository::sHTTPErrorCount = 0; U32 LLMeshRepository::sLODProcessing = 0; U32 LLMeshRepository::sLODPending = 0; U32 LLMeshRepository::sCacheBytesRead = 0; U32 LLMeshRepository::sCacheBytesWritten = 0; -U32 LLMeshRepository::sPeakKbps = 0; - +U32 LLMeshRepository::sCacheReads = 0; +U32 LLMeshRepository::sCacheWrites = 0; +U32 LLMeshRepository::sMaxLockHoldoffs = 0; -const U32 MAX_TEXTURE_UPLOAD_RETRIES = 5; +LLDeadmanTimer LLMeshRepository::sQuiescentTimer(15.0, false); // true -> gather cpu metrics + static S32 dump_num = 0; std::string make_dump_name(std::string prefix, S32 num) { return prefix + boost::lexical_cast<std::string>(num) + std::string(".xml"); - } void dump_llsd_to_file(const LLSD& content, std::string filename); LLSD llsd_from_file(std::string filename); -std::string header_lod[] = +const std::string header_lod[] = { "lowest_lod", "low_lod", "medium_lod", "high_lod" }; +const char * const LOG_MESH = "Mesh"; +// Static data and functions to measure mesh load +// time metrics for a new region scene. +static unsigned int metrics_teleport_start_count = 0; +boost::signals2::connection metrics_teleport_started_signal; +static void teleport_started(); //get the number of bytes resident in memory for given volume U32 get_volume_memory_size(const LLVolume* volume) @@ -197,200 +498,228 @@ void get_vertex_buffer_from_mesh(LLCDMeshData& mesh, LLModel::PhysicsMesh& res, } } -S32 LLMeshRepoThread::sActiveHeaderRequests = 0; -S32 LLMeshRepoThread::sActiveLODRequests = 0; +volatile S32 LLMeshRepoThread::sActiveHeaderRequests = 0; +volatile S32 LLMeshRepoThread::sActiveLODRequests = 0; U32 LLMeshRepoThread::sMaxConcurrentRequests = 1; - -class LLMeshHeaderResponder : public LLCurl::Responder +S32 LLMeshRepoThread::sRequestLowWater = REQUEST2_LOW_WATER_MIN; +S32 LLMeshRepoThread::sRequestHighWater = REQUEST2_HIGH_WATER_MIN; +S32 LLMeshRepoThread::sRequestWaterLevel = 0; + +// Base handler class for all mesh users of llcorehttp. +// This is roughly equivalent to a Responder class in +// traditional LL code. The base is going to perform +// common response/data handling in the inherited +// onCompleted() method. Derived classes, one for each +// type of HTTP action, define processData() and +// processFailure() methods to customize handling and +// error messages. +// +// LLCore::HttpHandler +// LLMeshHandlerBase +// LLMeshHeaderHandler +// LLMeshLODHandler +// LLMeshSkinInfoHandler +// LLMeshDecompositionHandler +// LLMeshPhysicsShapeHandler +// LLMeshUploadThread + +class LLMeshHandlerBase : public LLCore::HttpHandler { public: - LLVolumeParams mMeshParams; - bool mProcessed; + LLMeshHandlerBase() + : LLCore::HttpHandler(), + mMeshParams(), + mProcessed(false), + mHttpHandle(LLCORE_HTTP_HANDLE_INVALID) + {} + + virtual ~LLMeshHandlerBase() + {} + +protected: + LLMeshHandlerBase(const LLMeshHandlerBase &); // Not defined + void operator=(const LLMeshHandlerBase &); // Not defined + +public: + virtual void onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * response); + virtual void processData(LLCore::BufferArray * body, U8 * data, S32 data_size) = 0; + virtual void processFailure(LLCore::HttpStatus status) = 0; + +public: + LLVolumeParams mMeshParams; + bool mProcessed; + LLCore::HttpHandle mHttpHandle; +}; - LLMeshHeaderResponder(const LLVolumeParams& mesh_params) - : mMeshParams(mesh_params) - { - LLMeshRepoThread::incActiveHeaderRequests(); - mProcessed = false; - } - ~LLMeshHeaderResponder() +// Subclass for header fetches. +// +// Thread: repo +class LLMeshHeaderHandler : public LLMeshHandlerBase +{ +public: + LLMeshHeaderHandler(const LLVolumeParams & mesh_params) + : LLMeshHandlerBase() { - if (!LLApp::isQuitting()) - { - if (!mProcessed) - { //something went wrong, retry - llwarns << "Timeout or service unavailable, retrying." << llendl; - LLMeshRepository::sHTTPRetryCount++; - LLMeshRepoThread::HeaderRequest req(mMeshParams); - LLMutexLock lock(gMeshRepo.mThread->mMutex); - gMeshRepo.mThread->mHeaderReqQ.push(req); - } - - LLMeshRepoThread::decActiveHeaderRequests(); - } + mMeshParams = mesh_params; + LLMeshRepoThread::incActiveHeaderRequests(); } + virtual ~LLMeshHeaderHandler(); - virtual void completedRaw(U32 status, const std::string& reason, - const LLChannelDescriptors& channels, - const LLIOPipe::buffer_ptr_t& buffer); - +protected: + LLMeshHeaderHandler(const LLMeshHeaderHandler &); // Not defined + void operator=(const LLMeshHeaderHandler &); // Not defined + +public: + virtual void processData(LLCore::BufferArray * body, U8 * data, S32 data_size); + virtual void processFailure(LLCore::HttpStatus status); }; -class LLMeshLODResponder : public LLCurl::Responder + +// Subclass for LOD fetches. +// +// Thread: repo +class LLMeshLODHandler : public LLMeshHandlerBase { public: - LLVolumeParams mMeshParams; - S32 mLOD; - U32 mRequestedBytes; - U32 mOffset; - bool mProcessed; - - LLMeshLODResponder(const LLVolumeParams& mesh_params, S32 lod, U32 offset, U32 requested_bytes) - : mMeshParams(mesh_params), mLOD(lod), mOffset(offset), mRequestedBytes(requested_bytes) + LLMeshLODHandler(const LLVolumeParams & mesh_params, S32 lod, U32 offset, U32 requested_bytes) + : LLMeshHandlerBase(), + mLOD(lod), + mRequestedBytes(requested_bytes), + mOffset(offset) { + mMeshParams = mesh_params; LLMeshRepoThread::incActiveLODRequests(); - mProcessed = false; } + virtual ~LLMeshLODHandler(); + +protected: + LLMeshLODHandler(const LLMeshLODHandler &); // Not defined + void operator=(const LLMeshLODHandler &); // Not defined + +public: + virtual void processData(LLCore::BufferArray * body, U8 * data, S32 data_size); + virtual void processFailure(LLCore::HttpStatus status); - ~LLMeshLODResponder() - { - if (!LLApp::isQuitting()) - { - if (!mProcessed) - { - llwarns << "Killed without being processed, retrying." << llendl; - LLMeshRepository::sHTTPRetryCount++; - gMeshRepo.mThread->lockAndLoadMeshLOD(mMeshParams, mLOD); - } - LLMeshRepoThread::decActiveLODRequests(); - } - } - - virtual void completedRaw(U32 status, const std::string& reason, - const LLChannelDescriptors& channels, - const LLIOPipe::buffer_ptr_t& buffer); - -}; - -class LLMeshSkinInfoResponder : public LLCurl::Responder -{ public: - LLUUID mMeshID; + S32 mLOD; U32 mRequestedBytes; U32 mOffset; - bool mProcessed; - - LLMeshSkinInfoResponder(const LLUUID& id, U32 offset, U32 size) - : mMeshID(id), mRequestedBytes(size), mOffset(offset) - { - mProcessed = false; - } - - ~LLMeshSkinInfoResponder() - { - if (!LLApp::isQuitting() && - !mProcessed && - mMeshID.notNull()) - { // Something went wrong, retry - llwarns << "Timeout or service unavailable, retrying loadMeshSkinInfo() for " << mMeshID << llendl; - LLMeshRepository::sHTTPRetryCount++; - gMeshRepo.mThread->loadMeshSkinInfo(mMeshID); - } - } - - virtual void completedRaw(U32 status, const std::string& reason, - const LLChannelDescriptors& channels, - const LLIOPipe::buffer_ptr_t& buffer); - }; -class LLMeshDecompositionResponder : public LLCurl::Responder + +// Subclass for skin info fetches. +// +// Thread: repo +class LLMeshSkinInfoHandler : public LLMeshHandlerBase { public: + LLMeshSkinInfoHandler(const LLUUID& id, U32 offset, U32 size) + : LLMeshHandlerBase(), + mMeshID(id), + mRequestedBytes(size), + mOffset(offset) + {} + virtual ~LLMeshSkinInfoHandler(); + +protected: + LLMeshSkinInfoHandler(const LLMeshSkinInfoHandler &); // Not defined + void operator=(const LLMeshSkinInfoHandler &); // Not defined + +public: + virtual void processData(LLCore::BufferArray * body, U8 * data, S32 data_size); + virtual void processFailure(LLCore::HttpStatus status); + +public: LLUUID mMeshID; U32 mRequestedBytes; U32 mOffset; - bool mProcessed; - - LLMeshDecompositionResponder(const LLUUID& id, U32 offset, U32 size) - : mMeshID(id), mRequestedBytes(size), mOffset(offset) - { - mProcessed = false; - } - - ~LLMeshDecompositionResponder() - { - if (!LLApp::isQuitting() && - !mProcessed && - mMeshID.notNull()) - { // Something went wrong, retry - llwarns << "Timeout or service unavailable, retrying loadMeshDecomposition() for " << mMeshID << llendl; - LLMeshRepository::sHTTPRetryCount++; - gMeshRepo.mThread->loadMeshDecomposition(mMeshID); - } - } - - virtual void completedRaw(U32 status, const std::string& reason, - const LLChannelDescriptors& channels, - const LLIOPipe::buffer_ptr_t& buffer); - }; -class LLMeshPhysicsShapeResponder : public LLCurl::Responder + +// Subclass for decomposition fetches. +// +// Thread: repo +class LLMeshDecompositionHandler : public LLMeshHandlerBase { public: + LLMeshDecompositionHandler(const LLUUID& id, U32 offset, U32 size) + : LLMeshHandlerBase(), + mMeshID(id), + mRequestedBytes(size), + mOffset(offset) + {} + virtual ~LLMeshDecompositionHandler(); + +protected: + LLMeshDecompositionHandler(const LLMeshDecompositionHandler &); // Not defined + void operator=(const LLMeshDecompositionHandler &); // Not defined + +public: + virtual void processData(LLCore::BufferArray * body, U8 * data, S32 data_size); + virtual void processFailure(LLCore::HttpStatus status); + +public: LLUUID mMeshID; U32 mRequestedBytes; U32 mOffset; - bool mProcessed; +}; - LLMeshPhysicsShapeResponder(const LLUUID& id, U32 offset, U32 size) - : mMeshID(id), mRequestedBytes(size), mOffset(offset) - { - mProcessed = false; - } - ~LLMeshPhysicsShapeResponder() - { - if (!LLApp::isQuitting() && - !mProcessed && - mMeshID.notNull()) - { // Something went wrong, retry - llwarns << "Timeout or service unavailable, retrying loadMeshPhysicsShape() for " << mMeshID << llendl; - LLMeshRepository::sHTTPRetryCount++; - gMeshRepo.mThread->loadMeshPhysicsShape(mMeshID); - } - } - - virtual void completedRaw(U32 status, const std::string& reason, - const LLChannelDescriptors& channels, - const LLIOPipe::buffer_ptr_t& buffer); +// Subclass for physics shape fetches. +// +// Thread: repo +class LLMeshPhysicsShapeHandler : public LLMeshHandlerBase +{ +public: + LLMeshPhysicsShapeHandler(const LLUUID& id, U32 offset, U32 size) + : LLMeshHandlerBase(), + mMeshID(id), + mRequestedBytes(size), + mOffset(offset) + {} + virtual ~LLMeshPhysicsShapeHandler(); + +protected: + LLMeshPhysicsShapeHandler(const LLMeshPhysicsShapeHandler &); // Not defined + void operator=(const LLMeshPhysicsShapeHandler &); // Not defined + +public: + virtual void processData(LLCore::BufferArray * body, U8 * data, S32 data_size); + virtual void processFailure(LLCore::HttpStatus status); +public: + LLUUID mMeshID; + U32 mRequestedBytes; + U32 mOffset; }; -void log_upload_error(S32 status, const LLSD& content, std::string stage, std::string model_name) + +void log_upload_error(LLCore::HttpStatus status, const LLSD& content, + const char * const stage, const std::string & model_name) { // Add notification popup. LLSD args; - std::string message = content["error"]["message"]; - std::string identifier = content["error"]["identifier"]; + std::string message = content["error"]["message"].asString(); + std::string identifier = content["error"]["identifier"].asString(); args["MESSAGE"] = message; args["IDENTIFIER"] = identifier; args["LABEL"] = model_name; gMeshRepo.uploadError(args); // Log details. - llwarns << "stage: " << stage << " http status: " << status << llendl; + LL_WARNS(LOG_MESH) << "Error in stage: " << stage + << ", Reason: " << status.toString() + << " (" << status.toTerseString() << ")" << LL_ENDL; if (content.has("error")) { const LLSD& err = content["error"]; - llwarns << "err: " << err << llendl; - llwarns << "mesh upload failed, stage '" << stage - << "' error '" << err["error"].asString() - << "', message '" << err["message"].asString() - << "', id '" << err["identifier"].asString() - << "'" << llendl; + LL_WARNS(LOG_MESH) << "error: " << err << LL_ENDL; + LL_WARNS(LOG_MESH) << " mesh upload failed, stage '" << stage + << "', error '" << err["error"].asString() + << "', message '" << err["message"].asString() + << "', id '" << err["identifier"].asString() + << "'" << LL_ENDL; if (err.has("errors")) { S32 error_num = 0; @@ -400,13 +729,13 @@ void log_upload_error(S32 status, const LLSD& content, std::string stage, std::s ++it) { const LLSD& err_entry = *it; - llwarns << "error[" << error_num << "]:" << llendl; + LL_WARNS(LOG_MESH) << " error[" << error_num << "]:" << LL_ENDL; for (LLSD::map_const_iterator map_it = err_entry.beginMap(); map_it != err_entry.endMap(); ++map_it) { - llwarns << "\t" << map_it->first << ": " - << map_it->second << llendl; + LL_WARNS(LOG_MESH) << " " << map_it->first << ": " + << map_it->second << LL_ENDL; } error_num++; } @@ -414,153 +743,72 @@ void log_upload_error(S32 status, const LLSD& content, std::string stage, std::s } else { - llwarns << "bad mesh, no error information available" << llendl; + LL_WARNS(LOG_MESH) << "Bad response to mesh request, no additional error information available." << LL_ENDL; } } -class LLWholeModelFeeResponder: public LLCurl::Responder -{ - LLMeshUploadThread* mThread; - LLSD mModelData; - LLHandle<LLWholeModelFeeObserver> mObserverHandle; -public: - LLWholeModelFeeResponder(LLMeshUploadThread* thread, LLSD& model_data, LLHandle<LLWholeModelFeeObserver> observer_handle): - mThread(thread), - mModelData(model_data), - mObserverHandle(observer_handle) - { - if (mThread) - { - mThread->startRequest(); - } - } - - ~LLWholeModelFeeResponder() - { - if (mThread) - { - mThread->stopRequest(); - } - } - - virtual void completed(U32 status, - const std::string& reason, - const LLSD& content) - { - LLSD cc = content; - if (gSavedSettings.getS32("MeshUploadFakeErrors")&1) - { - cc = llsd_from_file("fake_upload_error.xml"); - } - - dump_llsd_to_file(cc,make_dump_name("whole_model_fee_response_",dump_num)); - LLWholeModelFeeObserver* observer = mObserverHandle.get(); +LLMeshRepoThread::LLMeshRepoThread() +: LLThread("mesh repo"), + mHttpRequest(NULL), + mHttpOptions(NULL), + mHttpLargeOptions(NULL), + mHttpHeaders(NULL), + mHttpPolicyClass(LLCore::HttpRequest::DEFAULT_POLICY_ID), + mHttpLegacyPolicyClass(LLCore::HttpRequest::DEFAULT_POLICY_ID), + mHttpLargePolicyClass(LLCore::HttpRequest::DEFAULT_POLICY_ID), + mHttpPriority(0), + mGetMeshVersion(2) +{ + mMutex = new LLMutex(NULL); + mHeaderMutex = new LLMutex(NULL); + mSignal = new LLCondition(NULL); + mHttpRequest = new LLCore::HttpRequest; + mHttpOptions = new LLCore::HttpOptions; + mHttpOptions->setTransferTimeout(SMALL_MESH_XFER_TIMEOUT); + mHttpOptions->setUseRetryAfter(gSavedSettings.getBOOL("MeshUseHttpRetryAfter")); + mHttpLargeOptions = new LLCore::HttpOptions; + mHttpLargeOptions->setTransferTimeout(LARGE_MESH_XFER_TIMEOUT); + mHttpLargeOptions->setUseRetryAfter(gSavedSettings.getBOOL("MeshUseHttpRetryAfter")); + mHttpHeaders = new LLCore::HttpHeaders; + mHttpHeaders->append("Accept", "application/vnd.ll.mesh"); + mHttpPolicyClass = LLAppViewer::instance()->getAppCoreHttp().getPolicy(LLAppCoreHttp::AP_MESH2); + mHttpLegacyPolicyClass = LLAppViewer::instance()->getAppCoreHttp().getPolicy(LLAppCoreHttp::AP_MESH1); + mHttpLargePolicyClass = LLAppViewer::instance()->getAppCoreHttp().getPolicy(LLAppCoreHttp::AP_LARGE_MESH); +} - if (isGoodStatus(status) && - cc["state"].asString() == "upload") - { - mThread->mWholeModelUploadURL = cc["uploader"].asString(); - if (observer) - { - cc["data"]["upload_price"] = cc["upload_price"]; - observer->onModelPhysicsFeeReceived(cc["data"], mThread->mWholeModelUploadURL); - } - } - else - { - llwarns << "fee request failed" << llendl; - log_upload_error(status,cc,"fee",mModelData["name"]); - mThread->mWholeModelUploadURL = ""; +LLMeshRepoThread::~LLMeshRepoThread() +{ + LL_INFOS(LOG_MESH) << "Small GETs issued: " << LLMeshRepository::sHTTPRequestCount + << ", Large GETs issued: " << LLMeshRepository::sHTTPLargeRequestCount + << ", Max Lock Holdoffs: " << LLMeshRepository::sMaxLockHoldoffs + << LL_ENDL; - if (observer) - { - observer->setModelPhysicsFeeErrorStatus(status, reason); - } - } + for (http_request_set::iterator iter(mHttpRequestSet.begin()); + iter != mHttpRequestSet.end(); + ++iter) + { + delete *iter; } - -}; - -class LLWholeModelUploadResponder: public LLCurl::Responder -{ - LLMeshUploadThread* mThread; - LLSD mModelData; - LLHandle<LLWholeModelUploadObserver> mObserverHandle; - -public: - LLWholeModelUploadResponder(LLMeshUploadThread* thread, LLSD& model_data, LLHandle<LLWholeModelUploadObserver> observer_handle): - mThread(thread), - mModelData(model_data), - mObserverHandle(observer_handle) + mHttpRequestSet.clear(); + if (mHttpHeaders) { - if (mThread) - { - mThread->startRequest(); - } + mHttpHeaders->release(); + mHttpHeaders = NULL; } - - ~LLWholeModelUploadResponder() + if (mHttpOptions) { - if (mThread) - { - mThread->stopRequest(); - } + mHttpOptions->release(); + mHttpOptions = NULL; } - - virtual void completed(U32 status, - const std::string& reason, - const LLSD& content) + if (mHttpLargeOptions) { - LLSD cc = content; - if (gSavedSettings.getS32("MeshUploadFakeErrors")&2) - { - cc = llsd_from_file("fake_upload_error.xml"); - } - - dump_llsd_to_file(cc,make_dump_name("whole_model_upload_response_",dump_num)); - - LLWholeModelUploadObserver* observer = mObserverHandle.get(); - - // requested "mesh" asset type isn't actually the type - // of the resultant object, fix it up here. - if (isGoodStatus(status) && - cc["state"].asString() == "complete") - { - mModelData["asset_type"] = "object"; - gMeshRepo.updateInventory(LLMeshRepository::inventory_data(mModelData,cc)); - - if (observer) - { - doOnIdleOneTime(boost::bind(&LLWholeModelUploadObserver::onModelUploadSuccess, observer)); - } - } - else - { - llwarns << "upload failed" << llendl; - std::string model_name = mModelData["name"].asString(); - log_upload_error(status,cc,"upload",model_name); - - if (observer) - { - doOnIdleOneTime(boost::bind(&LLWholeModelUploadObserver::onModelUploadFailure, observer)); - } - } + mHttpLargeOptions->release(); + mHttpLargeOptions = NULL; } -}; - -LLMeshRepoThread::LLMeshRepoThread() -: LLThread("mesh repo") -{ - mWaiting = false; - mMutex = new LLMutex(NULL); - mHeaderMutex = new LLMutex(NULL); - mSignal = new LLCondition(NULL); -} - -LLMeshRepoThread::~LLMeshRepoThread() -{ + delete mHttpRequest; + mHttpRequest = NULL; delete mMutex; mMutex = NULL; delete mHeaderMutex; @@ -571,109 +819,180 @@ LLMeshRepoThread::~LLMeshRepoThread() void LLMeshRepoThread::run() { - mCurlRequest = new LLCurlRequest(); LLCDResult res = LLConvexDecomposition::initThread(); if (res != LLCD_OK) { - llwarns << "convex decomposition unable to be loaded" << llendl; + LL_WARNS(LOG_MESH) << "Convex decomposition unable to be loaded. Expect severe problems." << LL_ENDL; } while (!LLApp::isQuitting()) { - mWaiting = true; + // *TODO: Revise sleep/wake strategy and try to move away + // from polling operations in this thread. We can sleep + // this thread hard when: + // * All Http requests are serviced + // * LOD request queue empty + // * Header request queue empty + // * Skin info request queue empty + // * Decomposition request queue empty + // * Physics shape request queue empty + // We wake the thread when any of the above become untrue. + // Will likely need a correctly-implemented condition variable to do this. + // On the other hand, this may actually be an effective and efficient scheme... + mSignal->wait(); - mWaiting = false; - if (!LLApp::isQuitting()) + if (LLApp::isQuitting()) { - static U32 count = 0; + break; + } + + if (! mHttpRequestSet.empty()) + { + // Dispatch all HttpHandler notifications + mHttpRequest->update(0L); + } + sRequestWaterLevel = mHttpRequestSet.size(); // Stats data update + + // NOTE: order of queue processing intentionally favors LOD requests over header requests - static F32 last_hundred = gFrameTimeSeconds; + while (!mLODReqQ.empty() && mHttpRequestSet.size() < sRequestHighWater) + { + if (! mMutex) + { + break; + } + mMutex->lock(); + LODRequest req = mLODReqQ.front(); + mLODReqQ.pop(); + LLMeshRepository::sLODProcessing--; + mMutex->unlock(); + if (!fetchMeshLOD(req.mMeshParams, req.mLOD)) // failed, resubmit + { + mMutex->lock(); + mLODReqQ.push(req) ; + ++LLMeshRepository::sLODProcessing; + mMutex->unlock(); + } + } - if (gFrameTimeSeconds - last_hundred > 1.f) - { //a second has gone by, clear count - last_hundred = gFrameTimeSeconds; - count = 0; + while (!mHeaderReqQ.empty() && mHttpRequestSet.size() < sRequestHighWater) + { + if (! mMutex) + { + break; + } + mMutex->lock(); + HeaderRequest req = mHeaderReqQ.front(); + mHeaderReqQ.pop(); + mMutex->unlock(); + if (!fetchMeshHeader(req.mMeshParams))//failed, resubmit + { + mMutex->lock(); + mHeaderReqQ.push(req) ; + mMutex->unlock(); } + } - // NOTE: throttling intentionally favors LOD requests over header requests - - while (!mLODReqQ.empty() && count < MAX_MESH_REQUESTS_PER_SECOND && sActiveLODRequests < sMaxConcurrentRequests) + // For the final three request lists, similar goal to above but + // slightly different queue structures. Stay off the mutex when + // performing long-duration actions. + + if (mHttpRequestSet.size() < sRequestHighWater + && (! mSkinRequests.empty() + || ! mDecompositionRequests.empty() + || ! mPhysicsShapeRequests.empty())) + { + // Something to do probably, lock and double-check. We don't want + // to hold the lock long here. That will stall main thread activities + // so we bounce it. + + mMutex->lock(); + if (! mSkinRequests.empty() && mHttpRequestSet.size() < sRequestHighWater) { - if (mMutex) + std::set<LLUUID> incomplete; + std::set<LLUUID>::iterator iter(mSkinRequests.begin()); + while (iter != mSkinRequests.end() && mHttpRequestSet.size() < sRequestHighWater) { - mMutex->lock(); - LODRequest req = mLODReqQ.front(); - mLODReqQ.pop(); - LLMeshRepository::sLODProcessing--; + LLUUID mesh_id = *iter; + mSkinRequests.erase(iter); mMutex->unlock(); - if (!fetchMeshLOD(req.mMeshParams, req.mLOD, count))//failed, resubmit + + if (! fetchMeshSkinInfo(mesh_id)) { - mMutex->lock(); - mLODReqQ.push(req); - mMutex->unlock(); + incomplete.insert(mesh_id); } + + mMutex->lock(); + iter = mSkinRequests.begin(); } - } - while (!mHeaderReqQ.empty() && count < MAX_MESH_REQUESTS_PER_SECOND && sActiveHeaderRequests < sMaxConcurrentRequests) - { - if (mMutex) + if (! incomplete.empty()) { - mMutex->lock(); - HeaderRequest req = mHeaderReqQ.front(); - mHeaderReqQ.pop(); - mMutex->unlock(); - if (!fetchMeshHeader(req.mMeshParams, count))//failed, resubmit - { - mMutex->lock(); - mHeaderReqQ.push(req) ; - mMutex->unlock(); - } + mSkinRequests.insert(incomplete.begin(), incomplete.end()); } } - { //mSkinRequests is protected by mSignal + // holding lock, try next list + // *TODO: For UI/debug-oriented lists, we might drop the fine- + // grained locking as there's a lowered expectation of smoothness + // in these cases. + if (! mDecompositionRequests.empty() && mHttpRequestSet.size() < sRequestHighWater) + { std::set<LLUUID> incomplete; - for (std::set<LLUUID>::iterator iter = mSkinRequests.begin(); iter != mSkinRequests.end(); ++iter) + std::set<LLUUID>::iterator iter(mDecompositionRequests.begin()); + while (iter != mDecompositionRequests.end() && mHttpRequestSet.size() < sRequestHighWater) { LLUUID mesh_id = *iter; - if (!fetchMeshSkinInfo(mesh_id)) + mDecompositionRequests.erase(iter); + mMutex->unlock(); + + if (! fetchMeshDecomposition(mesh_id)) { incomplete.insert(mesh_id); } + + mMutex->lock(); + iter = mDecompositionRequests.begin(); } - mSkinRequests = incomplete; - } - { //mDecompositionRequests is protected by mSignal - std::set<LLUUID> incomplete; - for (std::set<LLUUID>::iterator iter = mDecompositionRequests.begin(); iter != mDecompositionRequests.end(); ++iter) + if (! incomplete.empty()) { - LLUUID mesh_id = *iter; - if (!fetchMeshDecomposition(mesh_id)) - { - incomplete.insert(mesh_id); - } + mDecompositionRequests.insert(incomplete.begin(), incomplete.end()); } - mDecompositionRequests = incomplete; } - { //mPhysicsShapeRequests is protected by mSignal + // holding lock, final list + if (! mPhysicsShapeRequests.empty() && mHttpRequestSet.size() < sRequestHighWater) + { std::set<LLUUID> incomplete; - for (std::set<LLUUID>::iterator iter = mPhysicsShapeRequests.begin(); iter != mPhysicsShapeRequests.end(); ++iter) + std::set<LLUUID>::iterator iter(mPhysicsShapeRequests.begin()); + while (iter != mPhysicsShapeRequests.end() && mHttpRequestSet.size() < sRequestHighWater) { LLUUID mesh_id = *iter; - if (!fetchMeshPhysicsShape(mesh_id)) + mPhysicsShapeRequests.erase(iter); + mMutex->unlock(); + + if (! fetchMeshPhysicsShape(mesh_id)) { incomplete.insert(mesh_id); } + + mMutex->lock(); + iter = mPhysicsShapeRequests.begin(); } - mPhysicsShapeRequests = incomplete; - } - mCurlRequest->process(); + if (! incomplete.empty()) + { + mPhysicsShapeRequests.insert(incomplete.begin(), incomplete.end()); + } + } + mMutex->unlock(); } + + // For dev purposes only. A dynamic change could make this false + // and that shouldn't assert. + // llassert_always(mHttpRequestSet.size() <= sRequestHighWater); } if (mSignal->isLocked()) @@ -684,25 +1003,25 @@ void LLMeshRepoThread::run() res = LLConvexDecomposition::quitThread(); if (res != LLCD_OK) { - llwarns << "convex decomposition unable to be quit" << llendl; + LL_WARNS(LOG_MESH) << "Convex decomposition unable to be quit." << LL_ENDL; } - - delete mCurlRequest; - mCurlRequest = NULL; } +// Mutex: LLMeshRepoThread::mMutex must be held on entry void LLMeshRepoThread::loadMeshSkinInfo(const LLUUID& mesh_id) -{ //protected by mSignal, no locking needed here +{ mSkinRequests.insert(mesh_id); } +// Mutex: LLMeshRepoThread::mMutex must be held on entry void LLMeshRepoThread::loadMeshDecomposition(const LLUUID& mesh_id) -{ //protected by mSignal, no locking needed here +{ mDecompositionRequests.insert(mesh_id); } +// Mutex: LLMeshRepoThread::mMutex must be held on entry void LLMeshRepoThread::loadMeshPhysicsShape(const LLUUID& mesh_id) -{ //protected by mSignal, no locking needed here +{ mPhysicsShapeRequests.insert(mesh_id); } @@ -715,7 +1034,6 @@ void LLMeshRepoThread::lockAndLoadMeshLOD(const LLVolumeParams& mesh_params, S32 } - void LLMeshRepoThread::loadMeshLOD(const LLVolumeParams& mesh_params, S32 lod) { //could be called from any thread LLMutexLock lock(mMutex); @@ -747,31 +1065,122 @@ void LLMeshRepoThread::loadMeshLOD(const LLVolumeParams& mesh_params, S32 lod) } } -//static -std::string LLMeshRepoThread::constructUrl(LLUUID mesh_id) +// Mutex: must be holding mMutex when called +void LLMeshRepoThread::setGetMeshCaps(const std::string & get_mesh1, + const std::string & get_mesh2, + int pref_version) { - std::string http_url; + mGetMeshCapability = get_mesh1; + mGetMesh2Capability = get_mesh2; + mGetMeshVersion = pref_version; +} + + +// Constructs a Cap URL for the mesh. Prefers a GetMesh2 cap +// over a GetMesh cap. +// +// Mutex: acquires mMutex +void LLMeshRepoThread::constructUrl(LLUUID mesh_id, std::string * url, int * version) +{ + std::string res_url; + int res_version(2); if (gAgent.getRegion()) { - http_url = gMeshRepo.mGetMeshCapability; + LLMutexLock lock(mMutex); + + // Get a consistent pair of (cap string, version). The + // locking could be eliminated here without loss of safety + // by using a set of staging values in setGetMeshCaps(). + + if (! mGetMesh2Capability.empty() && mGetMeshVersion > 1) + { + res_url = mGetMesh2Capability; + res_version = 2; + } + else + { + res_url = mGetMeshCapability; + res_version = 1; + } } - if (!http_url.empty()) + if (! res_url.empty()) { - http_url += "/?mesh_id="; - http_url += mesh_id.asString().c_str(); + res_url += "/?mesh_id="; + res_url += mesh_id.asString().c_str(); } else { - llwarns << "Current region does not have GetMesh capability! Cannot load " << mesh_id << ".mesh" << llendl; + LL_WARNS_ONCE(LOG_MESH) << "Current region does not have GetMesh capability! Cannot load " + << mesh_id << ".mesh" << LL_ENDL; } - return http_url; + *url = res_url; + *version = res_version; +} + +// Issue an HTTP GET request with byte range using the right +// policy class. Large requests go to the large request class. +// If the current region supports GetMesh2, we prefer that for +// smaller requests otherwise we try to use the traditional +// GetMesh capability and connection concurrency. +// +// @return Valid handle or LLCORE_HTTP_HANDLE_INVALID. +// If the latter, actual status is found in +// mHttpStatus member which is valid until the +// next call to this method. +// +// Thread: repo +LLCore::HttpHandle LLMeshRepoThread::getByteRange(const std::string & url, int cap_version, + size_t offset, size_t len, + LLCore::HttpHandler * handler) +{ + LLCore::HttpHandle handle(LLCORE_HTTP_HANDLE_INVALID); + + if (len < LARGE_MESH_FETCH_THRESHOLD) + { + handle = mHttpRequest->requestGetByteRange((2 == cap_version + ? mHttpPolicyClass + : mHttpLegacyPolicyClass), + mHttpPriority, + url, + offset, + len, + mHttpOptions, + mHttpHeaders, + handler); + if (LLCORE_HTTP_HANDLE_INVALID != handle) + { + ++LLMeshRepository::sHTTPRequestCount; + } + } + else + { + handle = mHttpRequest->requestGetByteRange(mHttpLargePolicyClass, + mHttpPriority, + url, + offset, + len, + mHttpLargeOptions, + mHttpHeaders, + handler); + if (LLCORE_HTTP_HANDLE_INVALID != handle) + { + ++LLMeshRepository::sHTTPLargeRequestCount; + } + } + if (LLCORE_HTTP_HANDLE_INVALID == handle) + { + // Something went wrong, capture the error code for caller. + mHttpStatus = mHttpRequest->getStatus(); + } + return handle; } + bool LLMeshRepoThread::fetchMeshSkinInfo(const LLUUID& mesh_id) -{ //protected by mMutex +{ if (!mHeaderMutex) { @@ -786,7 +1195,8 @@ bool LLMeshRepoThread::fetchMeshSkinInfo(const LLUUID& mesh_id) return false; } - bool ret = true ; + ++LLMeshRepository::sMeshRequestCount; + bool ret = true; U32 header_size = mMeshHeaderSize[mesh_id]; if (header_size > 0) @@ -804,6 +1214,7 @@ bool LLMeshRepoThread::fetchMeshSkinInfo(const LLUUID& mesh_id) if (file.getSize() >= offset+size) { LLMeshRepository::sCacheBytesRead += size; + ++LLMeshRepository::sCacheReads; file.seek(offset); U8* buffer = new U8[size]; file.read(buffer, size); @@ -828,17 +1239,28 @@ bool LLMeshRepoThread::fetchMeshSkinInfo(const LLUUID& mesh_id) } //reading from VFS failed for whatever reason, fetch from sim - std::vector<std::string> headers; - headers.push_back("Accept: application/octet-stream"); + int cap_version(2); + std::string http_url; + constructUrl(mesh_id, &http_url, &cap_version); - std::string http_url = constructUrl(mesh_id); if (!http_url.empty()) - { - ret = mCurlRequest->getByteRange(http_url, headers, offset, size, - new LLMeshSkinInfoResponder(mesh_id, offset, size)); - if(ret) + { + LLMeshSkinInfoHandler * handler = new LLMeshSkinInfoHandler(mesh_id, offset, size); + LLCore::HttpHandle handle = getByteRange(http_url, cap_version, offset, size, handler); + if (LLCORE_HTTP_HANDLE_INVALID == handle) + { + LL_WARNS(LOG_MESH) << "HTTP GET request failed for skin info on mesh " << mID + << ". Reason: " << mHttpStatus.toString() + << " (" << mHttpStatus.toTerseString() << ")" + << LL_ENDL; + delete handler; + ret = false; + + } + else { - LLMeshRepository::sHTTPRequestCount++; + handler->mHttpHandle = handle; + mHttpRequestSet.insert(handler); } } } @@ -853,7 +1275,7 @@ bool LLMeshRepoThread::fetchMeshSkinInfo(const LLUUID& mesh_id) } bool LLMeshRepoThread::fetchMeshDecomposition(const LLUUID& mesh_id) -{ //protected by mMutex +{ if (!mHeaderMutex) { return false; @@ -867,8 +1289,9 @@ bool LLMeshRepoThread::fetchMeshDecomposition(const LLUUID& mesh_id) return false; } + ++LLMeshRepository::sMeshRequestCount; U32 header_size = mMeshHeaderSize[mesh_id]; - bool ret = true ; + bool ret = true; if (header_size > 0) { @@ -885,6 +1308,7 @@ bool LLMeshRepoThread::fetchMeshDecomposition(const LLUUID& mesh_id) if (file.getSize() >= offset+size) { LLMeshRepository::sCacheBytesRead += size; + ++LLMeshRepository::sCacheReads; file.seek(offset); U8* buffer = new U8[size]; @@ -910,17 +1334,27 @@ bool LLMeshRepoThread::fetchMeshDecomposition(const LLUUID& mesh_id) } //reading from VFS failed for whatever reason, fetch from sim - std::vector<std::string> headers; - headers.push_back("Accept: application/octet-stream"); - - std::string http_url = constructUrl(mesh_id); + int cap_version(2); + std::string http_url; + constructUrl(mesh_id, &http_url, &cap_version); + if (!http_url.empty()) - { - ret = mCurlRequest->getByteRange(http_url, headers, offset, size, - new LLMeshDecompositionResponder(mesh_id, offset, size)); - if(ret) + { + LLMeshDecompositionHandler * handler = new LLMeshDecompositionHandler(mesh_id, offset, size); + LLCore::HttpHandle handle = getByteRange(http_url, cap_version, offset, size, handler); + if (LLCORE_HTTP_HANDLE_INVALID == handle) + { + LL_WARNS(LOG_MESH) << "HTTP GET request failed for decomposition mesh " << mID + << ". Reason: " << mHttpStatus.toString() + << " (" << mHttpStatus.toTerseString() << ")" + << LL_ENDL; + delete handler; + ret = false; + } + else { - LLMeshRepository::sHTTPRequestCount++; + handler->mHttpHandle = handle; + mHttpRequestSet.insert(handler); } } } @@ -935,7 +1369,7 @@ bool LLMeshRepoThread::fetchMeshDecomposition(const LLUUID& mesh_id) } bool LLMeshRepoThread::fetchMeshPhysicsShape(const LLUUID& mesh_id) -{ //protected by mMutex +{ if (!mHeaderMutex) { return false; @@ -949,8 +1383,9 @@ bool LLMeshRepoThread::fetchMeshPhysicsShape(const LLUUID& mesh_id) return false; } + ++LLMeshRepository::sMeshRequestCount; U32 header_size = mMeshHeaderSize[mesh_id]; - bool ret = true ; + bool ret = true; if (header_size > 0) { @@ -967,6 +1402,7 @@ bool LLMeshRepoThread::fetchMeshPhysicsShape(const LLUUID& mesh_id) if (file.getSize() >= offset+size) { LLMeshRepository::sCacheBytesRead += size; + ++LLMeshRepository::sCacheReads; file.seek(offset); U8* buffer = new U8[size]; file.read(buffer, size); @@ -991,18 +1427,27 @@ bool LLMeshRepoThread::fetchMeshPhysicsShape(const LLUUID& mesh_id) } //reading from VFS failed for whatever reason, fetch from sim - std::vector<std::string> headers; - headers.push_back("Accept: application/octet-stream"); - - std::string http_url = constructUrl(mesh_id); + int cap_version(2); + std::string http_url; + constructUrl(mesh_id, &http_url, &cap_version); + if (!http_url.empty()) - { - ret = mCurlRequest->getByteRange(http_url, headers, offset, size, - new LLMeshPhysicsShapeResponder(mesh_id, offset, size)); - - if(ret) + { + LLMeshPhysicsShapeHandler * handler = new LLMeshPhysicsShapeHandler(mesh_id, offset, size); + LLCore::HttpHandle handle = getByteRange(http_url, cap_version, offset, size, handler); + if (LLCORE_HTTP_HANDLE_INVALID == handle) { - LLMeshRepository::sHTTPRequestCount++; + LL_WARNS(LOG_MESH) << "HTTP GET request failed for physics shape on mesh " << mID + << ". Reason: " << mHttpStatus.toString() + << " (" << mHttpStatus.toTerseString() << ")" + << LL_ENDL; + delete handler; + ret = false; + } + else + { + handler->mHttpHandle = handle; + mHttpRequestSet.insert(handler); } } } @@ -1049,8 +1494,10 @@ void LLMeshRepoThread::decActiveHeaderRequests() } //return false if failed to get header -bool LLMeshRepoThread::fetchMeshHeader(const LLVolumeParams& mesh_params, U32& count) +bool LLMeshRepoThread::fetchMeshHeader(const LLVolumeParams& mesh_params) { + ++LLMeshRepository::sMeshRequestCount; + { //look for mesh in asset in vfs LLVFile file(gVFS, mesh_params.getSculptID(), LLAssetType::AT_MESH); @@ -1058,43 +1505,57 @@ bool LLMeshRepoThread::fetchMeshHeader(const LLVolumeParams& mesh_params, U32& c S32 size = file.getSize(); if (size > 0) - { //NOTE -- if the header size is ever more than 4KB, this will break - U8 buffer[4096]; - S32 bytes = llmin(size, 4096); + { + // *NOTE: if the header size is ever more than 4KB, this will break + U8 buffer[MESH_HEADER_SIZE]; + S32 bytes = llmin(size, MESH_HEADER_SIZE); LLMeshRepository::sCacheBytesRead += bytes; + ++LLMeshRepository::sCacheReads; file.read(buffer, bytes); if (headerReceived(mesh_params, buffer, bytes)) - { //did not do an HTTP request, return false + { + // Found mesh in VFS cache return true; } } } //either cache entry doesn't exist or is corrupt, request header from simulator - bool retval = true ; - std::vector<std::string> headers; - headers.push_back("Accept: application/octet-stream"); - - std::string http_url = constructUrl(mesh_params.getSculptID()); + bool retval = true; + int cap_version(2); + std::string http_url; + constructUrl(mesh_params.getSculptID(), &http_url, &cap_version); + if (!http_url.empty()) { //grab first 4KB if we're going to bother with a fetch. Cache will prevent future fetches if a full mesh fits //within the first 4KB //NOTE -- this will break of headers ever exceed 4KB - retval = mCurlRequest->getByteRange(http_url, headers, 0, 4096, new LLMeshHeaderResponder(mesh_params)); - if(retval) + + LLMeshHeaderHandler * handler = new LLMeshHeaderHandler(mesh_params); + LLCore::HttpHandle handle = getByteRange(http_url, cap_version, 0, MESH_HEADER_SIZE, handler); + if (LLCORE_HTTP_HANDLE_INVALID == handle) + { + LL_WARNS(LOG_MESH) << "HTTP GET request failed for mesh header " << mID + << ". Reason: " << mHttpStatus.toString() + << " (" << mHttpStatus.toTerseString() << ")" + << LL_ENDL; + delete handler; + retval = false; + } + else { - LLMeshRepository::sHTTPRequestCount++; + handler->mHttpHandle = handle; + mHttpRequestSet.insert(handler); } - count++; } return retval; } //return false if failed to get mesh lod. -bool LLMeshRepoThread::fetchMeshLOD(const LLVolumeParams& mesh_params, S32 lod, U32& count) -{ //protected by mMutex +bool LLMeshRepoThread::fetchMeshLOD(const LLVolumeParams& mesh_params, S32 lod) +{ if (!mHeaderMutex) { return false; @@ -1102,6 +1563,7 @@ bool LLMeshRepoThread::fetchMeshLOD(const LLVolumeParams& mesh_params, S32 lod, mHeaderMutex->lock(); + ++LLMeshRepository::sMeshRequestCount; bool retval = true; LLUUID mesh_id = mesh_params.getSculptID(); @@ -1123,6 +1585,7 @@ bool LLMeshRepoThread::fetchMeshLOD(const LLVolumeParams& mesh_params, S32 lod, if (file.getSize() >= offset+size) { LLMeshRepository::sCacheBytesRead += size; + ++LLMeshRepository::sCacheReads; file.seek(offset); U8* buffer = new U8[size]; file.read(buffer, size); @@ -1147,20 +1610,29 @@ bool LLMeshRepoThread::fetchMeshLOD(const LLVolumeParams& mesh_params, S32 lod, } //reading from VFS failed for whatever reason, fetch from sim - std::vector<std::string> headers; - headers.push_back("Accept: application/octet-stream"); - - std::string http_url = constructUrl(mesh_id); + int cap_version(2); + std::string http_url; + constructUrl(mesh_id, &http_url, &cap_version); + if (!http_url.empty()) - { - retval = mCurlRequest->getByteRange(constructUrl(mesh_id), headers, offset, size, - new LLMeshLODResponder(mesh_params, lod, offset, size)); - - if(retval) + { + LLMeshLODHandler * handler = new LLMeshLODHandler(mesh_params, lod, offset, size); + LLCore::HttpHandle handle = getByteRange(http_url, cap_version, offset, size, handler); + if (LLCORE_HTTP_HANDLE_INVALID == handle) + { + LL_WARNS(LOG_MESH) << "HTTP GET request failed for LOD on mesh " << mID + << ". Reason: " << mHttpStatus.toString() + << " (" << mHttpStatus.toTerseString() << ")" + << LL_ENDL; + delete handler; + retval = false; + } + else { - LLMeshRepository::sHTTPRequestCount++; + handler->mHttpHandle = handle; + mHttpRequestSet.insert(handler); + // *NOTE: Allowing a re-request, not marking as unavailable. Is that correct? } - count++; } else { @@ -1182,6 +1654,7 @@ bool LLMeshRepoThread::fetchMeshLOD(const LLVolumeParams& mesh_params, S32 lod, bool LLMeshRepoThread::headerReceived(const LLVolumeParams& mesh_params, U8* data, S32 data_size) { + const LLUUID mesh_id = mesh_params.getSculptID(); LLSD header; U32 header_size = 0; @@ -1202,7 +1675,8 @@ bool LLMeshRepoThread::headerReceived(const LLVolumeParams& mesh_params, U8* dat if (!LLSDSerialize::fromBinary(header, stream, data_size)) { - llwarns << "Mesh header parse error. Not a valid mesh asset!" << llendl; + LL_WARNS(LOG_MESH) << "Mesh header parse error. Not a valid mesh asset! ID: " << mesh_id + << LL_ENDL; return false; } @@ -1210,13 +1684,12 @@ bool LLMeshRepoThread::headerReceived(const LLVolumeParams& mesh_params, U8* dat } else { - llinfos - << "Marking header as non-existent, will not retry." << llendl; + LL_INFOS(LOG_MESH) << "Non-positive data size. Marking header as non-existent, will not retry. ID: " << mesh_id + << LL_ENDL; header["404"] = 1; } { - LLUUID mesh_id = mesh_params.getSculptID(); { LLMutexLock lock(mHeaderMutex); @@ -1224,6 +1697,7 @@ bool LLMeshRepoThread::headerReceived(const LLVolumeParams& mesh_params, U8* dat mMeshHeader[mesh_id] = header; } + LLMutexLock lock(mMutex); // make sure only one thread access mPendingLOD at the same time. //check for pending requests @@ -1277,7 +1751,8 @@ bool LLMeshRepoThread::skinInfoReceived(const LLUUID& mesh_id, U8* data, S32 dat if (!unzip_llsd(skin, stream, data_size)) { - llwarns << "Mesh skin info parse error. Not a valid mesh asset!" << llendl; + LL_WARNS(LOG_MESH) << "Mesh skin info parse error. Not a valid mesh asset! ID: " << mesh_id + << LL_ENDL; return false; } } @@ -1286,8 +1761,11 @@ bool LLMeshRepoThread::skinInfoReceived(const LLUUID& mesh_id, U8* data, S32 dat LLMeshSkinInfo info(skin); info.mMeshID = mesh_id; - //llinfos<<"info pelvis offset"<<info.mPelvisOffset<<llendl; - mSkinInfoQ.push(info); + // LL_DEBUGS(LOG_MESH) << "info pelvis offset" << info.mPelvisOffset << LL_ENDL; + { + LLMutexLock lock(mMutex); + mSkinInfoQ.push_back(info); + } } return true; @@ -1305,7 +1783,8 @@ bool LLMeshRepoThread::decompositionReceived(const LLUUID& mesh_id, U8* data, S3 if (!unzip_llsd(decomp, stream, data_size)) { - llwarns << "Mesh decomposition parse error. Not a valid mesh asset!" << llendl; + LL_WARNS(LOG_MESH) << "Mesh decomposition parse error. Not a valid mesh asset! ID: " << mesh_id + << LL_ENDL; return false; } } @@ -1313,7 +1792,10 @@ bool LLMeshRepoThread::decompositionReceived(const LLUUID& mesh_id, U8* data, S3 { LLModel::Decomposition* d = new LLModel::Decomposition(decomp); d->mMeshID = mesh_id; - mDecompositionQ.push(d); + { + LLMutexLock lock(mMutex); + mDecompositionQ.push_back(d); + } } return true; @@ -1372,15 +1854,20 @@ bool LLMeshRepoThread::physicsShapeReceived(const LLUUID& mesh_id, U8* data, S32 } } - mDecompositionQ.push(d); + { + LLMutexLock lock(mMutex); + mDecompositionQ.push_back(d); + } return true; } LLMeshUploadThread::LLMeshUploadThread(LLMeshUploadThread::instance_list& data, LLVector3& scale, bool upload_textures, - bool upload_skin, bool upload_joints, std::string upload_url, bool do_upload, - LLHandle<LLWholeModelFeeObserver> fee_observer, LLHandle<LLWholeModelUploadObserver> upload_observer) -: LLThread("mesh upload"), - mDiscarded(FALSE), + bool upload_skin, bool upload_joints, const std::string & upload_url, bool do_upload, + LLHandle<LLWholeModelFeeObserver> fee_observer, + LLHandle<LLWholeModelUploadObserver> upload_observer) + : LLThread("mesh upload"), + LLCore::HttpHandler(), + mDiscarded(false), mDoUpload(do_upload), mWholeModelUploadURL(upload_url), mFeeObserverHandle(fee_observer), @@ -1391,7 +1878,6 @@ LLMeshUploadThread::LLMeshUploadThread(LLMeshUploadThread::instance_list& data, mUploadSkin = upload_skin; mUploadJoints = upload_joints; mMutex = new LLMutex(NULL); - mCurlRequest = NULL; mPendingUploads = 0; mFinished = false; mOrigin = gAgent.getPositionAgent(); @@ -1401,12 +1887,33 @@ LLMeshUploadThread::LLMeshUploadThread(LLMeshUploadThread::instance_list& data, mOrigin += gAgent.getAtAxis() * scale.magVec(); - mMeshUploadTimeOut = gSavedSettings.getS32("MeshUploadTimeOut") ; + mMeshUploadTimeOut = gSavedSettings.getS32("MeshUploadTimeOut"); + + mHttpRequest = new LLCore::HttpRequest; + mHttpOptions = new LLCore::HttpOptions; + mHttpOptions->setTransferTimeout(mMeshUploadTimeOut); + mHttpOptions->setUseRetryAfter(gSavedSettings.getBOOL("MeshUseHttpRetryAfter")); + mHttpOptions->setRetries(UPLOAD_RETRY_LIMIT); + mHttpHeaders = new LLCore::HttpHeaders; + mHttpHeaders->append("Content-Type", "application/llsd+xml"); + mHttpPolicyClass = LLAppViewer::instance()->getAppCoreHttp().getPolicy(LLAppCoreHttp::AP_UPLOADS); + mHttpPriority = 0; } LLMeshUploadThread::~LLMeshUploadThread() { - + if (mHttpHeaders) + { + mHttpHeaders->release(); + mHttpHeaders = NULL; + } + if (mHttpOptions) + { + mHttpOptions->release(); + mHttpOptions = NULL; + } + delete mHttpRequest; + mHttpRequest = NULL; } LLMeshUploadThread::DecompRequest::DecompRequest(LLModel* mdl, LLModel* base_model, LLMeshUploadThread* thread) @@ -1448,14 +1955,14 @@ void LLMeshUploadThread::preStart() void LLMeshUploadThread::discard() { - LLMutexLock lock(mMutex) ; - mDiscarded = TRUE ; + LLMutexLock lock(mMutex); + mDiscarded = true; } -BOOL LLMeshUploadThread::isDiscarded() +bool LLMeshUploadThread::isDiscarded() const { - LLMutexLock lock(mMutex) ; - return mDiscarded ; + LLMutexLock lock(mMutex); + return mDiscarded; } void LLMeshUploadThread::run() @@ -1706,9 +2213,15 @@ void LLMeshUploadThread::generateHulls() } } - if(has_valid_requests) - { - while (!mPhysicsComplete) + if (has_valid_requests) + { + // *NOTE: Interesting livelock condition on shutdown. If there + // is an upload request in generateHulls() when shutdown starts, + // the main thread isn't available to manage communication between + // the decomposition thread and the upload thread and this loop + // wouldn't complete in turn stalling the main thread. The check + // on isDiscarded() prevents that. + while (! mPhysicsComplete && ! isDiscarded()) { apr_sleep(100); } @@ -1717,86 +2230,266 @@ void LLMeshUploadThread::generateHulls() void LLMeshUploadThread::doWholeModelUpload() { - mCurlRequest = new LLCurlRequest(); + LL_DEBUGS(LOG_MESH) << "Starting model upload. Instances: " << mInstance.size() << LL_ENDL; if (mWholeModelUploadURL.empty()) { - llinfos << "unable to upload, fee request failed" << llendl; + LL_WARNS(LOG_MESH) << "Missing mesh upload capability, unable to upload, fee request failed." + << LL_ENDL; } else { generateHulls(); - - LLSD full_model_data; - wholeModelToLLSD(full_model_data, true); - LLSD body = full_model_data["asset_resources"]; - dump_llsd_to_file(body,make_dump_name("whole_model_body_",dump_num)); - LLCurlRequest::headers_t headers; - + LL_DEBUGS(LOG_MESH) << "Hull generation completed." << LL_ENDL; + + mModelData = LLSD::emptyMap(); + wholeModelToLLSD(mModelData, true); + LLSD body = mModelData["asset_resources"]; + dump_llsd_to_file(body, make_dump_name("whole_model_body_", dump_num)); + + LLCore::BufferArray * ba = new LLCore::BufferArray; + LLCore::BufferArrayStream bas(ba); + LLSDSerialize::toXML(body, bas); + // LLSDSerialize::toXML(mModelData, bas); // <- Enabling this will generate a convenient upload error + LLCore::HttpHandle handle = mHttpRequest->requestPost(mHttpPolicyClass, + mHttpPriority, + mWholeModelUploadURL, + ba, + mHttpOptions, + mHttpHeaders, + this); + ba->release(); + + if (LLCORE_HTTP_HANDLE_INVALID == handle) + { + mHttpStatus = mHttpRequest->getStatus(); + + LL_WARNS(LOG_MESH) << "Couldn't issue request for full model upload. Reason: " << mHttpStatus.toString() + << " (" << mHttpStatus.toTerseString() << ")" + << LL_ENDL; + } + else { - LLCurl::ResponderPtr responder = new LLWholeModelUploadResponder(this, full_model_data, mUploadObserverHandle) ; + U32 sleep_time(10); + + LL_DEBUGS(LOG_MESH) << "POST request issued." << LL_ENDL; + + mHttpRequest->update(0); + while (! LLApp::isQuitting() && ! finished() && ! isDiscarded()) + { + ms_sleep(sleep_time); + sleep_time = llmin(250U, sleep_time + sleep_time); + mHttpRequest->update(0); + } - while(!mCurlRequest->post(mWholeModelUploadURL, headers, body, responder, mMeshUploadTimeOut)) + if (isDiscarded()) + { + LL_DEBUGS(LOG_MESH) << "Mesh upload operation discarded." << LL_ENDL; + } + else { - //sleep for 10ms to prevent eating a whole core - apr_sleep(10000); + LL_DEBUGS(LOG_MESH) << "Mesh upload operation completed." << LL_ENDL; } } - - do - { - mCurlRequest->process(); - //sleep for 10ms to prevent eating a whole core - apr_sleep(10000); - } while (!LLAppViewer::isQuitting() && mPendingUploads > 0); } - - delete mCurlRequest; - mCurlRequest = NULL; - - // Currently a no-op. - mFinished = true; } void LLMeshUploadThread::requestWholeModelFee() { dump_num++; - mCurlRequest = new LLCurlRequest(); - generateHulls(); - LLSD model_data; - wholeModelToLLSD(model_data,false); - dump_llsd_to_file(model_data,make_dump_name("whole_model_fee_request_",dump_num)); - - LLCurlRequest::headers_t headers; + mModelData = LLSD::emptyMap(); + wholeModelToLLSD(mModelData, false); + dump_llsd_to_file(mModelData, make_dump_name("whole_model_fee_request_", dump_num)); + LLCore::BufferArray * ba = new LLCore::BufferArray; + LLCore::BufferArrayStream bas(ba); + LLSDSerialize::toXML(mModelData, bas); + + LLCore::HttpHandle handle = mHttpRequest->requestPost(mHttpPolicyClass, + mHttpPriority, + mWholeModelFeeCapability, + ba, + mHttpOptions, + mHttpHeaders, + this); + ba->release(); + if (LLCORE_HTTP_HANDLE_INVALID == handle) + { + mHttpStatus = mHttpRequest->getStatus(); + + LL_WARNS(LOG_MESH) << "Couldn't issue request for model fee. Reason: " << mHttpStatus.toString() + << " (" << mHttpStatus.toTerseString() << ")" + << LL_ENDL; + } + else { - LLCurl::ResponderPtr responder = new LLWholeModelFeeResponder(this,model_data, mFeeObserverHandle) ; - while(!mCurlRequest->post(mWholeModelFeeCapability, headers, model_data, responder, mMeshUploadTimeOut)) + U32 sleep_time(10); + + mHttpRequest->update(0); + while (! LLApp::isQuitting() && ! finished() && ! isDiscarded()) + { + ms_sleep(sleep_time); + sleep_time = llmin(250U, sleep_time + sleep_time); + mHttpRequest->update(0); + } + if (isDiscarded()) { - //sleep for 10ms to prevent eating a whole core - apr_sleep(10000); + LL_DEBUGS(LOG_MESH) << "Mesh fee query operation discarded." << LL_ENDL; } } +} - do - { - mCurlRequest->process(); - //sleep for 10ms to prevent eating a whole core - apr_sleep(10000); - } while (!LLApp::isQuitting() && mPendingUploads > 0); - delete mCurlRequest; - mCurlRequest = NULL; +// Does completion duty for both fee queries and actual uploads. +void LLMeshUploadThread::onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * response) +{ + // QA/Devel: 0x2 to enable fake error import on upload, 0x1 on fee check + const S32 fake_error(gSavedSettings.getS32("MeshUploadFakeErrors") & (mDoUpload ? 0xa : 0x5)); + LLCore::HttpStatus status(response->getStatus()); + if (fake_error) + { + status = (fake_error & 0x0c) ? LLCore::HttpStatus(500) : LLCore::HttpStatus(200); + } + std::string reason(status.toString()); + LLSD body; - // Currently a no-op. mFinished = true; + + if (mDoUpload) + { + // model upload case + LLWholeModelUploadObserver * observer(mUploadObserverHandle.get()); + + if (! status) + { + LL_WARNS(LOG_MESH) << "Upload failed. Reason: " << reason + << " (" << status.toTerseString() << ")" + << LL_ENDL; + + // Build a fake body for the alert generator + body["error"] = LLSD::emptyMap(); + body["error"]["message"] = reason; + body["error"]["identifier"] = "NetworkError"; // from asset-upload/upload_util.py + log_upload_error(status, body, "upload", mModelData["name"].asString()); + + if (observer) + { + doOnIdleOneTime(boost::bind(&LLWholeModelUploadObserver::onModelUploadFailure, observer)); + } + } + else + { + if (fake_error & 0x2) + { + body = llsd_from_file("fake_upload_error.xml"); + } + else + { + LLCore::BufferArray * ba(response->getBody()); + if (ba && ba->size()) + { + LLCore::BufferArrayStream bas(ba); + LLSDSerialize::fromXML(body, bas); + } + } + dump_llsd_to_file(body, make_dump_name("whole_model_upload_response_", dump_num)); + + if (body["state"].asString() == "complete") + { + // requested "mesh" asset type isn't actually the type + // of the resultant object, fix it up here. + mModelData["asset_type"] = "object"; + gMeshRepo.updateInventory(LLMeshRepository::inventory_data(mModelData, body)); + + if (observer) + { + doOnIdleOneTime(boost::bind(&LLWholeModelUploadObserver::onModelUploadSuccess, observer)); + } + } + else + { + LL_WARNS(LOG_MESH) << "Upload failed. Not in expected 'complete' state." << LL_ENDL; + log_upload_error(status, body, "upload", mModelData["name"].asString()); + + if (observer) + { + doOnIdleOneTime(boost::bind(&LLWholeModelUploadObserver::onModelUploadFailure, observer)); + } + } + } + } + else + { + // model fee case + LLWholeModelFeeObserver* observer(mFeeObserverHandle.get()); + mWholeModelUploadURL.clear(); + + if (! status) + { + LL_WARNS(LOG_MESH) << "Fee request failed. Reason: " << reason + << " (" << status.toTerseString() << ")" + << LL_ENDL; + + // Build a fake body for the alert generator + body["error"] = LLSD::emptyMap(); + body["error"]["message"] = reason; + body["error"]["identifier"] = "NetworkError"; // from asset-upload/upload_util.py + log_upload_error(status, body, "fee", mModelData["name"].asString()); + + if (observer) + { + observer->setModelPhysicsFeeErrorStatus(status.toULong(), reason); + } + } + else + { + if (fake_error & 0x1) + { + body = llsd_from_file("fake_upload_error.xml"); + } + else + { + LLCore::BufferArray * ba(response->getBody()); + if (ba && ba->size()) + { + LLCore::BufferArrayStream bas(ba); + LLSDSerialize::fromXML(body, bas); + } + } + dump_llsd_to_file(body, make_dump_name("whole_model_fee_response_", dump_num)); + + if (body["state"].asString() == "upload") + { + mWholeModelUploadURL = body["uploader"].asString(); + + if (observer) + { + body["data"]["upload_price"] = body["upload_price"]; + observer->onModelPhysicsFeeReceived(body["data"], mWholeModelUploadURL); + } + } + else + { + LL_WARNS(LOG_MESH) << "Fee request failed. Not in expected 'upload' state." << LL_ENDL; + log_upload_error(status, body, "fee", mModelData["name"].asString()); + + if (observer) + { + observer->setModelPhysicsFeeErrorStatus(status.toULong(), reason); + } + } + } + } } + void LLMeshRepoThread::notifyLoadedMeshes() { + bool update_metrics(false); + if (!mMutex) { return; @@ -1805,10 +2498,16 @@ void LLMeshRepoThread::notifyLoadedMeshes() while (!mLoadedQ.empty()) { mMutex->lock(); + if (mLoadedQ.empty()) + { + mMutex->unlock(); + break; + } LoadedMesh mesh = mLoadedQ.front(); mLoadedQ.pop(); mMutex->unlock(); + update_metrics = true; if (mesh.mVolume && mesh.mVolume->getNumVolumeFaces() > 0) { gMeshRepo.notifyMeshLoaded(mesh.mMeshParams, mesh.mVolume); @@ -1823,24 +2522,59 @@ void LLMeshRepoThread::notifyLoadedMeshes() while (!mUnavailableQ.empty()) { mMutex->lock(); + if (mUnavailableQ.empty()) + { + mMutex->unlock(); + break; + } + LODRequest req = mUnavailableQ.front(); mUnavailableQ.pop(); mMutex->unlock(); - + + update_metrics = true; gMeshRepo.notifyMeshUnavailable(req.mMeshParams, req.mLOD); } - while (!mSkinInfoQ.empty()) + if (! mSkinInfoQ.empty() || ! mDecompositionQ.empty()) { - gMeshRepo.notifySkinInfoReceived(mSkinInfoQ.front()); - mSkinInfoQ.pop(); + if (mMutex->trylock()) + { + std::list<LLMeshSkinInfo> skin_info_q; + std::list<LLModel::Decomposition*> decomp_q; + + if (! mSkinInfoQ.empty()) + { + skin_info_q.swap(mSkinInfoQ); + } + if (! mDecompositionQ.empty()) + { + decomp_q.swap(mDecompositionQ); + } + + mMutex->unlock(); + + // Process the elements free of the lock + while (! skin_info_q.empty()) + { + gMeshRepo.notifySkinInfoReceived(skin_info_q.front()); + skin_info_q.pop_front(); + } + + while (! decomp_q.empty()) + { + gMeshRepo.notifyDecompositionReceived(decomp_q.front()); + decomp_q.pop_front(); + } + } } - while (!mDecompositionQ.empty()) + if (update_metrics) { - gMeshRepo.notifyDecompositionReceived(mDecompositionQ.front()); - mDecompositionQ.pop(); + // Ping time-to-load metrics for mesh download operations. + LLMeshRepository::metricsProgress(0); } + } S32 LLMeshRepoThread::getActualMeshLOD(const LLVolumeParams& mesh_params, S32 lod) @@ -1919,245 +2653,250 @@ void LLMeshRepository::cacheOutgoingMesh(LLMeshUploadData& data, LLSD& header) } -void LLMeshLODResponder::completedRaw(U32 status, const std::string& reason, - const LLChannelDescriptors& channels, - const LLIOPipe::buffer_ptr_t& buffer) +void LLMeshHandlerBase::onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * response) { mProcessed = true; - - // thread could have already be destroyed during logout - if( !gMeshRepo.mThread ) - { - return; - } - - S32 data_size = buffer->countAfter(channels.in(), NULL); - if (status < 200 || status > 400) + unsigned int retries(0U); + response->getRetries(NULL, &retries); + LLMeshRepository::sHTTPRetryCount += retries; + + LLCore::HttpStatus status(response->getStatus()); + if (! status || MESH_HTTP_RESPONSE_FAILED) { - llwarns << status << ": " << reason << llendl; + processFailure(status); + ++LLMeshRepository::sHTTPErrorCount; } - - if (data_size < mRequestedBytes) + else { - if (status == HTTP_INTERNAL_ERROR || status == HTTP_SERVICE_UNAVAILABLE) - { //timeout or service unavailable, try again - llwarns << "Timeout or service unavailable, retrying." << llendl; - LLMeshRepository::sHTTPRetryCount++; - gMeshRepo.mThread->loadMeshLOD(mMeshParams, mLOD); - } - else + // From texture fetch code and may apply here: + // + // A warning about partial (HTTP 206) data. Some grid services + // do *not* return a 'Content-Range' header in the response to + // Range requests with a 206 status. We're forced to assume + // we get what we asked for in these cases until we can fix + // the services. + // + // May also need to deal with 200 status (full asset returned + // rather than partial) and 416 (request completely unsatisfyable). + // Always been exposed to these but are less likely here where + // speculative loads aren't done. + static const LLCore::HttpStatus par_status(HTTP_PARTIAL_CONTENT); + + if (par_status != status) { - llassert(status == HTTP_INTERNAL_ERROR || status == HTTP_SERVICE_UNAVAILABLE); //intentionally trigger a breakpoint - llwarns << "Unhandled status " << status << llendl; + LL_WARNS_ONCE(LOG_MESH) << "Non-206 successful status received for fetch: " + << status.toTerseString() << LL_ENDL; } - return; - } + + LLCore::BufferArray * body(response->getBody()); + S32 data_size(body ? body->size() : 0); + U8 * data(NULL); - LLMeshRepository::sBytesReceived += mRequestedBytes; + if (data_size > 0) + { + // *TODO: Try to get rid of data copying and add interfaces + // that support BufferArray directly. Introduce a two-phase + // handler, optional first that takes a body, fallback second + // that requires a temporary allocation and data copy. + data = new U8[data_size]; + body->read(0, (char *) data, data_size); + LLMeshRepository::sBytesReceived += data_size; + } - U8* data = NULL; + processData(body, data, data_size); - if (data_size > 0) - { - data = new U8[data_size]; - buffer->readAfter(channels.in(), NULL, data, data_size); + delete [] data; } - if (gMeshRepo.mThread->lodReceived(mMeshParams, mLOD, data, data_size)) - { - //good fetch from sim, write to VFS for caching - LLVFile file(gVFS, mMeshParams.getSculptID(), LLAssetType::AT_MESH, LLVFile::WRITE); + // Release handler + gMeshRepo.mThread->mHttpRequestSet.erase(this); + delete this; // Must be last statement +} - S32 offset = mOffset; - S32 size = mRequestedBytes; - if (file.getSize() >= offset+size) +LLMeshHeaderHandler::~LLMeshHeaderHandler() +{ + if (!LLApp::isQuitting()) + { + if (! mProcessed) { - file.seek(offset); - file.write(data, size); - LLMeshRepository::sCacheBytesWritten += size; + // something went wrong, retry + LL_WARNS(LOG_MESH) << "Mesh header fetch canceled unexpectedly, retrying." << LL_ENDL; + LLMeshRepoThread::HeaderRequest req(mMeshParams); + LLMutexLock lock(gMeshRepo.mThread->mMutex); + gMeshRepo.mThread->mHeaderReqQ.push(req); } + LLMeshRepoThread::decActiveHeaderRequests(); } - - delete [] data; } -void LLMeshSkinInfoResponder::completedRaw(U32 status, const std::string& reason, - const LLChannelDescriptors& channels, - const LLIOPipe::buffer_ptr_t& buffer) +void LLMeshHeaderHandler::processFailure(LLCore::HttpStatus status) { - mProcessed = true; + LL_WARNS(LOG_MESH) << "Error during mesh header handling. ID: " << mMeshParams.getSculptID() + << ", Reason: " << status.toString() + << " (" << status.toTerseString() << "). Not retrying." + << LL_ENDL; - // thread could have already be destroyed during logout - if( !gMeshRepo.mThread ) + // Can't get the header so none of the LODs will be available + LLMutexLock lock(gMeshRepo.mThread->mMutex); + for (int i(0); i < 4; ++i) { - return; + gMeshRepo.mThread->mUnavailableQ.push(LLMeshRepoThread::LODRequest(mMeshParams, i)); } +} - S32 data_size = buffer->countAfter(channels.in(), NULL); - - if (status < 200 || status > 400) +void LLMeshHeaderHandler::processData(LLCore::BufferArray * body, U8 * data, S32 data_size) +{ + LLUUID mesh_id = mMeshParams.getSculptID(); + bool success = (! MESH_HEADER_PROCESS_FAILED) && gMeshRepo.mThread->headerReceived(mMeshParams, data, data_size); + llassert(success); + if (! success) { - llwarns << status << ": " << reason << llendl; - } + // *TODO: Get real reason for parse failure here. Might we want to retry? + LL_WARNS(LOG_MESH) << "Unable to parse mesh header. ID: " << mesh_id + << ", Unknown reason. Not retrying." + << LL_ENDL; - if (data_size < mRequestedBytes) - { - if (status == HTTP_INTERNAL_ERROR || status == HTTP_SERVICE_UNAVAILABLE) - { //timeout or service unavailable, try again - llwarns << "Timeout or service unavailable, retrying loadMeshSkinInfo() for " << mMeshID << llendl; - LLMeshRepository::sHTTPRetryCount++; - gMeshRepo.mThread->loadMeshSkinInfo(mMeshID); - } - else + // Can't get the header so none of the LODs will be available + LLMutexLock lock(gMeshRepo.mThread->mMutex); + for (int i(0); i < 4; ++i) { - llassert(status == HTTP_INTERNAL_ERROR || status == HTTP_SERVICE_UNAVAILABLE); //intentionally trigger a breakpoint - llwarns << "Unhandled status " << status << llendl; + gMeshRepo.mThread->mUnavailableQ.push(LLMeshRepoThread::LODRequest(mMeshParams, i)); } - return; } + else if (data && data_size > 0) + { + // header was successfully retrieved from sim, cache in vfs + LLSD header = gMeshRepo.mThread->mMeshHeader[mesh_id]; - LLMeshRepository::sBytesReceived += mRequestedBytes; + S32 version = header["version"].asInteger(); - U8* data = NULL; + if (version <= MAX_MESH_VERSION) + { + std::stringstream str; - if (data_size > 0) - { - data = new U8[data_size]; - buffer->readAfter(channels.in(), NULL, data, data_size); - } + S32 lod_bytes = 0; - if (gMeshRepo.mThread->skinInfoReceived(mMeshID, data, data_size)) - { - //good fetch from sim, write to VFS for caching - LLVFile file(gVFS, mMeshID, LLAssetType::AT_MESH, LLVFile::WRITE); + for (U32 i = 0; i < LLModel::LOD_PHYSICS; ++i) + { + // figure out how many bytes we'll need to reserve in the file + const std::string & lod_name = header_lod[i]; + lod_bytes = llmax(lod_bytes, header[lod_name]["offset"].asInteger()+header[lod_name]["size"].asInteger()); + } + + // just in case skin info or decomposition is at the end of the file (which it shouldn't be) + lod_bytes = llmax(lod_bytes, header["skin"]["offset"].asInteger() + header["skin"]["size"].asInteger()); + lod_bytes = llmax(lod_bytes, header["physics_convex"]["offset"].asInteger() + header["physics_convex"]["size"].asInteger()); - S32 offset = mOffset; - S32 size = mRequestedBytes; + S32 header_bytes = (S32) gMeshRepo.mThread->mMeshHeaderSize[mesh_id]; + S32 bytes = lod_bytes + header_bytes; - if (file.getSize() >= offset+size) - { - LLMeshRepository::sCacheBytesWritten += size; - file.seek(offset); - file.write(data, size); - } - } + + // It's possible for the remote asset to have more data than is needed for the local cache + // only allocate as much space in the VFS as is needed for the local cache + data_size = llmin(data_size, bytes); - delete [] data; -} + LLVFile file(gVFS, mesh_id, LLAssetType::AT_MESH, LLVFile::WRITE); + if (file.getMaxSize() >= bytes || file.setMaxSize(bytes)) + { + LLMeshRepository::sCacheBytesWritten += data_size; + ++LLMeshRepository::sCacheWrites; -void LLMeshDecompositionResponder::completedRaw(U32 status, const std::string& reason, - const LLChannelDescriptors& channels, - const LLIOPipe::buffer_ptr_t& buffer) -{ - mProcessed = true; - - if( !gMeshRepo.mThread ) - { - return; - } + file.write(data, data_size); + + // zero out the rest of the file + U8 block[MESH_HEADER_SIZE]; + memset(block, 0, sizeof(block)); - S32 data_size = buffer->countAfter(channels.in(), NULL); + while (bytes-file.tell() > sizeof(block)) + { + file.write(block, sizeof(block)); + } - if (status < 200 || status > 400) - { - llwarns << status << ": " << reason << llendl; + S32 remaining = bytes-file.tell(); + if (remaining > 0) + { + file.write(block, remaining); + } + } + } } +} - if (data_size < mRequestedBytes) +LLMeshLODHandler::~LLMeshLODHandler() +{ + if (! LLApp::isQuitting()) { - if (status == HTTP_INTERNAL_ERROR || status == HTTP_SERVICE_UNAVAILABLE) - { //timeout or service unavailable, try again - llwarns << "Timeout or service unavailable, retrying loadMeshDecomposition() for " << mMeshID << llendl; - LLMeshRepository::sHTTPRetryCount++; - gMeshRepo.mThread->loadMeshDecomposition(mMeshID); - } - else + if (! mProcessed) { - llassert(status == HTTP_INTERNAL_ERROR || status == HTTP_SERVICE_UNAVAILABLE); //intentionally trigger a breakpoint - llwarns << "Unhandled status " << status << llendl; + LL_WARNS(LOG_MESH) << "Mesh LOD fetch canceled unexpectedly, retrying." << LL_ENDL; + gMeshRepo.mThread->lockAndLoadMeshLOD(mMeshParams, mLOD); } - return; + LLMeshRepoThread::decActiveLODRequests(); } +} - LLMeshRepository::sBytesReceived += mRequestedBytes; - - U8* data = NULL; +void LLMeshLODHandler::processFailure(LLCore::HttpStatus status) +{ + LL_WARNS(LOG_MESH) << "Error during mesh LOD handling. ID: " << mMeshParams.getSculptID() + << ", Reason: " << status.toString() + << " (" << status.toTerseString() << "). Not retrying." + << LL_ENDL; - if (data_size > 0) - { - data = new U8[data_size]; - buffer->readAfter(channels.in(), NULL, data, data_size); - } + LLMutexLock lock(gMeshRepo.mThread->mMutex); + gMeshRepo.mThread->mUnavailableQ.push(LLMeshRepoThread::LODRequest(mMeshParams, mLOD)); +} - if (gMeshRepo.mThread->decompositionReceived(mMeshID, data, data_size)) +void LLMeshLODHandler::processData(LLCore::BufferArray * body, U8 * data, S32 data_size) +{ + if ((! MESH_LOD_PROCESS_FAILED) && gMeshRepo.mThread->lodReceived(mMeshParams, mLOD, data, data_size)) { - //good fetch from sim, write to VFS for caching - LLVFile file(gVFS, mMeshID, LLAssetType::AT_MESH, LLVFile::WRITE); + // good fetch from sim, write to VFS for caching + LLVFile file(gVFS, mMeshParams.getSculptID(), LLAssetType::AT_MESH, LLVFile::WRITE); S32 offset = mOffset; S32 size = mRequestedBytes; if (file.getSize() >= offset+size) { - LLMeshRepository::sCacheBytesWritten += size; file.seek(offset); file.write(data, size); + LLMeshRepository::sCacheBytesWritten += size; + ++LLMeshRepository::sCacheWrites; } } - - delete [] data; -} - -void LLMeshPhysicsShapeResponder::completedRaw(U32 status, const std::string& reason, - const LLChannelDescriptors& channels, - const LLIOPipe::buffer_ptr_t& buffer) -{ - mProcessed = true; - - // thread could have already be destroyed during logout - if( !gMeshRepo.mThread ) - { - return; - } - - S32 data_size = buffer->countAfter(channels.in(), NULL); - - if (status < 200 || status > 400) - { - llwarns << status << ": " << reason << llendl; - } - - if (data_size < mRequestedBytes) + else { - if (status == HTTP_INTERNAL_ERROR || status == HTTP_SERVICE_UNAVAILABLE) - { //timeout or service unavailable, try again - llwarns << "Timeout or service unavailable, retrying loadMeshPhysicsShape() for " << mMeshID << llendl; - LLMeshRepository::sHTTPRetryCount++; - gMeshRepo.mThread->loadMeshPhysicsShape(mMeshID); - } - else - { - llassert(status == HTTP_INTERNAL_ERROR || status == HTTP_SERVICE_UNAVAILABLE); //intentionally trigger a breakpoint - llwarns << "Unhandled status " << status << llendl; - } - return; + LL_WARNS(LOG_MESH) << "Error during mesh LOD processing. ID: " << mMeshParams.getSculptID() + << ", Unknown reason. Not retrying." + << LL_ENDL; + LLMutexLock lock(gMeshRepo.mThread->mMutex); + gMeshRepo.mThread->mUnavailableQ.push(LLMeshRepoThread::LODRequest(mMeshParams, mLOD)); } +} - LLMeshRepository::sBytesReceived += mRequestedBytes; +LLMeshSkinInfoHandler::~LLMeshSkinInfoHandler() +{ + llassert(mProcessed); +} - U8* data = NULL; +void LLMeshSkinInfoHandler::processFailure(LLCore::HttpStatus status) +{ + LL_WARNS(LOG_MESH) << "Error during mesh skin info handling. ID: " << mMeshID + << ", Reason: " << status.toString() + << " (" << status.toTerseString() << "). Not retrying." + << LL_ENDL; - if (data_size > 0) - { - data = new U8[data_size]; - buffer->readAfter(channels.in(), NULL, data, data_size); - } + // *TODO: Mark mesh unavailable on error. For now, simply leave + // request unfulfilled rather than retry forever. +} - if (gMeshRepo.mThread->physicsShapeReceived(mMeshID, data, data_size)) +void LLMeshSkinInfoHandler::processData(LLCore::BufferArray * body, U8 * data, S32 data_size) +{ + if ((! MESH_SKIN_INFO_PROCESS_FAILED) && gMeshRepo.mThread->skinInfoReceived(mMeshID, data, data_size)) { - //good fetch from sim, write to VFS for caching + // good fetch from sim, write to VFS for caching LLVFile file(gVFS, mMeshID, LLAssetType::AT_MESH, LLVFile::WRITE); S32 offset = mOffset; @@ -2166,145 +2905,108 @@ void LLMeshPhysicsShapeResponder::completedRaw(U32 status, const std::string& re if (file.getSize() >= offset+size) { LLMeshRepository::sCacheBytesWritten += size; + ++LLMeshRepository::sCacheWrites; file.seek(offset); file.write(data, size); } } - - delete [] data; + else + { + LL_WARNS(LOG_MESH) << "Error during mesh skin info processing. ID: " << mMeshID + << ", Unknown reason. Not retrying." + << LL_ENDL; + // *TODO: Mark mesh unavailable on error + } } -void LLMeshHeaderResponder::completedRaw(U32 status, const std::string& reason, - const LLChannelDescriptors& channels, - const LLIOPipe::buffer_ptr_t& buffer) +LLMeshDecompositionHandler::~LLMeshDecompositionHandler() { - mProcessed = true; + llassert(mProcessed); +} - // thread could have already be destroyed during logout - if( !gMeshRepo.mThread ) - { - return; - } +void LLMeshDecompositionHandler::processFailure(LLCore::HttpStatus status) +{ + LL_WARNS(LOG_MESH) << "Error during mesh decomposition handling. ID: " << mMeshID + << ", Reason: " << status.toString() + << " (" << status.toTerseString() << "). Not retrying." + << LL_ENDL; + // *TODO: Mark mesh unavailable on error. For now, simply leave + // request unfulfilled rather than retry forever. +} - if (status < 200 || status > 400) +void LLMeshDecompositionHandler::processData(LLCore::BufferArray * body, U8 * data, S32 data_size) +{ + if ((! MESH_DECOMP_PROCESS_FAILED) && gMeshRepo.mThread->decompositionReceived(mMeshID, data, data_size)) { - //llwarns - // << "Header responder failed with status: " - // << status << ": " << reason << llendl; - - // 503 (service unavailable) or 499 (internal Linden-generated error) - // can be due to server load and can be retried - - // TODO*: Add maximum retry logic, exponential backoff - // and (somewhat more optional than the others) retries - // again after some set period of time - - llassert(status == HTTP_NOT_FOUND || status == HTTP_SERVICE_UNAVAILABLE || status == HTTP_REQUEST_TIME_OUT || status == HTTP_INTERNAL_ERROR); + // good fetch from sim, write to VFS for caching + LLVFile file(gVFS, mMeshID, LLAssetType::AT_MESH, LLVFile::WRITE); - if (status == HTTP_SERVICE_UNAVAILABLE || status == HTTP_REQUEST_TIME_OUT || status == HTTP_INTERNAL_ERROR) - { //retry - llwarns << "Timeout or service unavailable, retrying." << llendl; - LLMeshRepository::sHTTPRetryCount++; - LLMeshRepoThread::HeaderRequest req(mMeshParams); - LLMutexLock lock(gMeshRepo.mThread->mMutex); - gMeshRepo.mThread->mHeaderReqQ.push(req); + S32 offset = mOffset; + S32 size = mRequestedBytes; - return; - } - else + if (file.getSize() >= offset+size) { - llwarns << "Unhandled status: " << status << llendl; + LLMeshRepository::sCacheBytesWritten += size; + ++LLMeshRepository::sCacheWrites; + file.seek(offset); + file.write(data, size); } } - - S32 data_size = buffer->countAfter(channels.in(), NULL); - - U8* data = NULL; - - if (data_size > 0) + else { - data = new U8[data_size]; - buffer->readAfter(channels.in(), NULL, data, data_size); + LL_WARNS(LOG_MESH) << "Error during mesh decomposition processing. ID: " << mMeshID + << ", Unknown reason. Not retrying." + << LL_ENDL; + // *TODO: Mark mesh unavailable on error } +} - LLMeshRepository::sBytesReceived += llmin(data_size, 4096); +LLMeshPhysicsShapeHandler::~LLMeshPhysicsShapeHandler() +{ + llassert(mProcessed); +} - bool success = gMeshRepo.mThread->headerReceived(mMeshParams, data, data_size); - - llassert(success); +void LLMeshPhysicsShapeHandler::processFailure(LLCore::HttpStatus status) +{ + LL_WARNS(LOG_MESH) << "Error during mesh physics shape handling. ID: " << mMeshID + << ", Reason: " << status.toString() + << " (" << status.toTerseString() << "). Not retrying." + << LL_ENDL; + // *TODO: Mark mesh unavailable on error +} - if (!success) - { - llwarns - << "Unable to parse mesh header: " - << status << ": " << reason << llendl; - } - else if (data && data_size > 0) +void LLMeshPhysicsShapeHandler::processData(LLCore::BufferArray * body, U8 * data, S32 data_size) +{ + if ((! MESH_PHYS_SHAPE_PROCESS_FAILED) && gMeshRepo.mThread->physicsShapeReceived(mMeshID, data, data_size)) { - //header was successfully retrieved from sim, cache in vfs - LLUUID mesh_id = mMeshParams.getSculptID(); - LLSD header = gMeshRepo.mThread->mMeshHeader[mesh_id]; + // good fetch from sim, write to VFS for caching + LLVFile file(gVFS, mMeshID, LLAssetType::AT_MESH, LLVFile::WRITE); - S32 version = header["version"].asInteger(); + S32 offset = mOffset; + S32 size = mRequestedBytes; - if (version <= MAX_MESH_VERSION) + if (file.getSize() >= offset+size) { - std::stringstream str; - - S32 lod_bytes = 0; - - for (U32 i = 0; i < LLModel::LOD_PHYSICS; ++i) - { //figure out how many bytes we'll need to reserve in the file - std::string lod_name = header_lod[i]; - lod_bytes = llmax(lod_bytes, header[lod_name]["offset"].asInteger()+header[lod_name]["size"].asInteger()); - } - - //just in case skin info or decomposition is at the end of the file (which it shouldn't be) - lod_bytes = llmax(lod_bytes, header["skin"]["offset"].asInteger() + header["skin"]["size"].asInteger()); - lod_bytes = llmax(lod_bytes, header["physics_convex"]["offset"].asInteger() + header["physics_convex"]["size"].asInteger()); - - S32 header_bytes = (S32) gMeshRepo.mThread->mMeshHeaderSize[mesh_id]; - S32 bytes = lod_bytes + header_bytes; - - - //it's possible for the remote asset to have more data than is needed for the local cache - //only allocate as much space in the VFS as is needed for the local cache - data_size = llmin(data_size, bytes); - - LLVFile file(gVFS, mesh_id, LLAssetType::AT_MESH, LLVFile::WRITE); - if (file.getMaxSize() >= bytes || file.setMaxSize(bytes)) - { - LLMeshRepository::sCacheBytesWritten += data_size; - - file.write((const U8*) data, data_size); - - //zero out the rest of the file - U8 block[4096]; - memset(block, 0, 4096); - - while (bytes-file.tell() > 4096) - { - file.write(block, 4096); - } - - S32 remaining = bytes-file.tell(); - - if (remaining > 0) - { - file.write(block, remaining); - } - } + LLMeshRepository::sCacheBytesWritten += size; + ++LLMeshRepository::sCacheWrites; + file.seek(offset); + file.write(data, size); } } - - delete [] data; + else + { + LL_WARNS(LOG_MESH) << "Error during mesh physics shape processing. ID: " << mMeshID + << ", Unknown reason. Not retrying." + << LL_ENDL; + // *TODO: Mark mesh unavailable on error + } } - LLMeshRepository::LLMeshRepository() : mMeshMutex(NULL), mMeshThreadCount(0), - mThread(NULL) + mThread(NULL), + mGetMeshVersion(2) { } @@ -2323,7 +3025,7 @@ void LLMeshRepository::init() apr_sleep(100); } - + metrics_teleport_started_signal = LLViewerMessage::getInstance()->setTeleportStartedCallback(teleport_started); mThread = new LLMeshRepoThread(); mThread->start(); @@ -2331,11 +3033,13 @@ void LLMeshRepository::init() void LLMeshRepository::shutdown() { - llinfos << "Shutting down mesh repository." << llendl; + LL_INFOS(LOG_MESH) << "Shutting down mesh repository." << LL_ENDL; + + metrics_teleport_started_signal.disconnect(); for (U32 i = 0; i < mUploads.size(); ++i) { - llinfos << "Discard the pending mesh uploads " << llendl; + LL_INFOS(LOG_MESH) << "Discard the pending mesh uploads." << LL_ENDL; mUploads[i]->discard() ; //discard the uploading requests. } @@ -2350,7 +3054,7 @@ void LLMeshRepository::shutdown() for (U32 i = 0; i < mUploads.size(); ++i) { - llinfos << "Waiting for pending mesh upload " << i << "/" << mUploads.size() << llendl; + LL_INFOS(LOG_MESH) << "Waiting for pending mesh upload " << (i + 1) << "/" << mUploads.size() << LL_ENDL; while (!mUploads[i]->isStopped()) { apr_sleep(10); @@ -2363,7 +3067,7 @@ void LLMeshRepository::shutdown() delete mMeshMutex; mMeshMutex = NULL; - llinfos << "Shutting down decomposition system." << llendl; + LL_INFOS(LOG_MESH) << "Shutting down decomposition system." << LL_ENDL; if (mDecompThread) { @@ -2378,6 +3082,9 @@ void LLMeshRepository::shutdown() //called in the main thread. S32 LLMeshRepository::update() { + // Conditionally log a mesh metrics event + metricsUpdate(); + if(mUploadWaitList.empty()) { return 0 ; @@ -2397,7 +3104,12 @@ S32 LLMeshRepository::update() S32 LLMeshRepository::loadMesh(LLVOVolume* vobj, const LLVolumeParams& mesh_params, S32 detail, S32 last_lod) { - if (detail < 0 || detail > 4) + MESH_FASTTIMER_DEFBLOCK; + + // Manage time-to-load metrics for mesh download operations. + metricsProgress(1); + + if (detail < 0 || detail >= 4) { return detail; } @@ -2475,9 +3187,32 @@ S32 LLMeshRepository::loadMesh(LLVOVolume* vobj, const LLVolumeParams& mesh_para void LLMeshRepository::notifyLoadedMeshes() { //called from main thread + MESH_FASTTIMER_DEFBLOCK; - LLMeshRepoThread::sMaxConcurrentRequests = gSavedSettings.getU32("MeshMaxConcurrentRequests"); - + if (1 == mGetMeshVersion) + { + // Legacy GetMesh operation with high connection concurrency + LLMeshRepoThread::sMaxConcurrentRequests = gSavedSettings.getU32("MeshMaxConcurrentRequests"); + LLMeshRepoThread::sRequestHighWater = llclamp(2 * S32(LLMeshRepoThread::sMaxConcurrentRequests), + REQUEST_HIGH_WATER_MIN, + REQUEST_HIGH_WATER_MAX); + LLMeshRepoThread::sRequestLowWater = llclamp(LLMeshRepoThread::sRequestHighWater / 2, + REQUEST_LOW_WATER_MIN, + REQUEST_LOW_WATER_MAX); + } + else + { + // GetMesh2 operation with keepalives, etc. With pipelining, + // we'll increase this. + LLMeshRepoThread::sMaxConcurrentRequests = gSavedSettings.getU32("Mesh2MaxConcurrentRequests"); + LLMeshRepoThread::sRequestHighWater = llclamp(5 * S32(LLMeshRepoThread::sMaxConcurrentRequests), + REQUEST2_HIGH_WATER_MIN, + REQUEST2_HIGH_WATER_MAX); + LLMeshRepoThread::sRequestLowWater = llclamp(LLMeshRepoThread::sRequestHighWater / 2, + REQUEST2_LOW_WATER_MIN, + REQUEST2_LOW_WATER_MAX); + } + //clean up completed upload threads for (std::vector<LLMeshUploadThread*>::iterator iter = mUploads.begin(); iter != mUploads.end(); ) { @@ -2554,26 +3289,46 @@ void LLMeshRepository::notifyLoadedMeshes() //call completed callbacks on finished decompositions mDecompThread->notifyCompleted(); - - if (!mThread->mWaiting) - { //curl thread is churning, wait for it to go idle - return; - } - static std::string region_name("never name a region this"); + // For major operations, attempt to get the required locks + // without blocking and punt if they're not available. The + // longest run of holdoffs is kept in sMaxLockHoldoffs just + // to collect the data. In testing, I've never seen a value + // greater than 2 (written to log on exit). + { + LLMutexTrylock lock1(mMeshMutex); + LLMutexTrylock lock2(mThread->mMutex); - if (gAgent.getRegion()) - { //update capability url - if (gAgent.getRegion()->getName() != region_name && gAgent.getRegion()->capabilitiesReceived()) + static U32 hold_offs(0); + if (! lock1.isLocked() || ! lock2.isLocked()) { - region_name = gAgent.getRegion()->getName(); - mGetMeshCapability = gAgent.getRegion()->getCapability("GetMesh"); + // If we can't get the locks, skip and pick this up later. + ++hold_offs; + sMaxLockHoldoffs = llmax(sMaxLockHoldoffs, hold_offs); + return; + } + hold_offs = 0; + + if (gAgent.getRegion()) + { + // Update capability urls + static std::string region_name("never name a region this"); + + if (gAgent.getRegion()->getName() != region_name && gAgent.getRegion()->capabilitiesReceived()) + { + region_name = gAgent.getRegion()->getName(); + const bool use_v1(gSavedSettings.getBOOL("MeshUseGetMesh1")); + const std::string mesh1(gAgent.getRegion()->getCapability("GetMesh")); + const std::string mesh2(gAgent.getRegion()->getCapability("GetMesh2")); + mGetMeshVersion = (mesh2.empty() || use_v1) ? 1 : 2; + mThread->setGetMeshCaps(mesh1, mesh2, mGetMeshVersion); + LL_DEBUGS(LOG_MESH) << "Retrieving caps for region '" << region_name + << "', GetMesh2: " << mesh2 + << ", GetMesh: " << mesh1 + << ", using version: " << mGetMeshVersion + << LL_ENDL; + } } - } - - { - LLMutexLock lock1(mMeshMutex); - LLMutexLock lock2(mThread->mMutex); //popup queued error messages from background threads while (!mUploadErrorQ.empty()) @@ -2582,47 +3337,55 @@ void LLMeshRepository::notifyLoadedMeshes() mUploadErrorQ.pop(); } - S32 push_count = LLMeshRepoThread::sMaxConcurrentRequests-(LLMeshRepoThread::sActiveHeaderRequests+LLMeshRepoThread::sActiveLODRequests); - - if (push_count > 0) + S32 active_count = LLMeshRepoThread::sActiveHeaderRequests + LLMeshRepoThread::sActiveLODRequests; + if (active_count < LLMeshRepoThread::sRequestLowWater) { - //calculate "score" for pending requests - - //create score map - std::map<LLUUID, F32> score_map; + S32 push_count = LLMeshRepoThread::sRequestHighWater - active_count; - for (U32 i = 0; i < 4; ++i) + if (mPendingRequests.size() > push_count) { - for (mesh_load_map::iterator iter = mLoadingMeshes[i].begin(); iter != mLoadingMeshes[i].end(); ++iter) + // More requests than the high-water limit allows so + // sort and forward the most important. + + //calculate "score" for pending requests + + //create score map + std::map<LLUUID, F32> score_map; + + for (U32 i = 0; i < 4; ++i) { - F32 max_score = 0.f; - for (std::set<LLUUID>::iterator obj_iter = iter->second.begin(); obj_iter != iter->second.end(); ++obj_iter) + for (mesh_load_map::iterator iter = mLoadingMeshes[i].begin(); iter != mLoadingMeshes[i].end(); ++iter) { - LLViewerObject* object = gObjectList.findObject(*obj_iter); - - if (object) + F32 max_score = 0.f; + for (std::set<LLUUID>::iterator obj_iter = iter->second.begin(); obj_iter != iter->second.end(); ++obj_iter) { - LLDrawable* drawable = object->mDrawable; - if (drawable) + LLViewerObject* object = gObjectList.findObject(*obj_iter); + + if (object) { - F32 cur_score = drawable->getRadius()/llmax(drawable->mDistanceWRTCamera, 1.f); - max_score = llmax(max_score, cur_score); + LLDrawable* drawable = object->mDrawable; + if (drawable) + { + F32 cur_score = drawable->getRadius()/llmax(drawable->mDistanceWRTCamera, 1.f); + max_score = llmax(max_score, cur_score); + } } } - } - score_map[iter->first.getSculptID()] = max_score; + score_map[iter->first.getSculptID()] = max_score; + } } - } - //set "score" for pending requests - for (std::vector<LLMeshRepoThread::LODRequest>::iterator iter = mPendingRequests.begin(); iter != mPendingRequests.end(); ++iter) - { - iter->mScore = score_map[iter->mMeshParams.getSculptID()]; - } + //set "score" for pending requests + for (std::vector<LLMeshRepoThread::LODRequest>::iterator iter = mPendingRequests.begin(); iter != mPendingRequests.end(); ++iter) + { + iter->mScore = score_map[iter->mMeshParams.getSculptID()]; + } - //sort by "score" - std::sort(mPendingRequests.begin(), mPendingRequests.end(), LLMeshRepoThread::CompareScoreGreater()); + //sort by "score" + std::partial_sort(mPendingRequests.begin(), mPendingRequests.begin() + push_count, + mPendingRequests.end(), LLMeshRepoThread::CompareScoreGreater()); + } while (!mPendingRequests.empty() && push_count > 0) { @@ -2676,9 +3439,8 @@ void LLMeshRepository::notifySkinInfoReceived(LLMeshSkinInfo& info) vobj->notifyMeshLoaded(); } } + mLoadingSkins.erase(info.mMeshID); } - - mLoadingSkins.erase(info.mMeshID); } void LLMeshRepository::notifyDecompositionReceived(LLModel::Decomposition* decomp) @@ -2687,14 +3449,14 @@ void LLMeshRepository::notifyDecompositionReceived(LLModel::Decomposition* decom if (iter == mDecompositionMap.end()) { //just insert decomp into map mDecompositionMap[decomp->mMeshID] = decomp; + mLoadingDecompositions.erase(decomp->mMeshID); } else { //merge decomp with existing entry iter->second->merge(decomp); + mLoadingDecompositions.erase(decomp->mMeshID); delete decomp; } - - mLoadingDecompositions.erase(decomp->mMeshID); } void LLMeshRepository::notifyMeshLoaded(const LLVolumeParams& mesh_params, LLVolume* volume) @@ -2709,7 +3471,8 @@ void LLMeshRepository::notifyMeshLoaded(const LLVolumeParams& mesh_params, LLVol //make sure target volume is still valid if (volume->getNumVolumeFaces() <= 0) { - llwarns << "Mesh loading returned empty volume." << llendl; + LL_WARNS(LOG_MESH) << "Mesh loading returned empty volume. ID: " << mesh_params.getSculptID() + << LL_ENDL; } { //update system volume @@ -2722,7 +3485,8 @@ void LLMeshRepository::notifyMeshLoaded(const LLVolumeParams& mesh_params, LLVol } else { - llwarns << "Couldn't find system volume for given mesh." << llendl; + LL_WARNS(LOG_MESH) << "Couldn't find system volume for mesh " << mesh_params.getSculptID() + << LL_ENDL; } } @@ -2776,6 +3540,8 @@ S32 LLMeshRepository::getActualMeshLOD(const LLVolumeParams& mesh_params, S32 lo const LLMeshSkinInfo* LLMeshRepository::getSkinInfo(const LLUUID& mesh_id, const LLVOVolume* requesting_obj) { + MESH_FASTTIMER_DEFBLOCK; + if (mesh_id.notNull()) { skin_map::iterator iter = mSkinMap.find(mesh_id); @@ -2802,6 +3568,8 @@ const LLMeshSkinInfo* LLMeshRepository::getSkinInfo(const LLUUID& mesh_id, const void LLMeshRepository::fetchPhysicsShape(const LLUUID& mesh_id) { + MESH_FASTTIMER_DEFBLOCK; + if (mesh_id.notNull()) { LLModel::Decomposition* decomp = NULL; @@ -2819,6 +3587,7 @@ void LLMeshRepository::fetchPhysicsShape(const LLUUID& mesh_id) std::set<LLUUID>::iterator iter = mLoadingPhysicsShapes.find(mesh_id); if (iter == mLoadingPhysicsShapes.end()) { //no request pending for this skin info + // *FIXME: Nothing ever deletes entries, can't be right mLoadingPhysicsShapes.insert(mesh_id); mPendingPhysicsShapeRequests.push(mesh_id); } @@ -2829,6 +3598,8 @@ void LLMeshRepository::fetchPhysicsShape(const LLUUID& mesh_id) LLModel::Decomposition* LLMeshRepository::getDecomposition(const LLUUID& mesh_id) { + MESH_FASTTIMER_DEFBLOCK; + LLModel::Decomposition* ret = NULL; if (mesh_id.notNull()) @@ -2891,6 +3662,8 @@ bool LLMeshRepository::hasPhysicsShape(const LLUUID& mesh_id) LLSD& LLMeshRepository::getMeshHeader(const LLUUID& mesh_id) { + MESH_FASTTIMER_DEFBLOCK; + return mThread->getMeshHeader(mesh_id); } @@ -2912,7 +3685,7 @@ LLSD& LLMeshRepoThread::getMeshHeader(const LLUUID& mesh_id) void LLMeshRepository::uploadModel(std::vector<LLModelInstance>& data, LLVector3& scale, bool upload_textures, - bool upload_skin, bool upload_joints, std::string upload_url, bool do_upload, + bool upload_skin, bool upload_joints, std::string upload_url, bool do_upload, LLHandle<LLWholeModelFeeObserver> fee_observer, LLHandle<LLWholeModelUploadObserver> upload_observer) { LLMeshUploadThread* thread = new LLMeshUploadThread(data, scale, upload_textures, upload_skin, upload_joints, upload_url, @@ -2941,7 +3714,6 @@ S32 LLMeshRepository::getMeshSize(const LLUUID& mesh_id, S32 lod) } return -1; - } void LLMeshUploadThread::decomposeMeshMatrix(LLMatrix4& transformation, @@ -3206,7 +3978,7 @@ void LLPhysicsDecomp::setMeshData(LLCDMeshData& mesh, bool vertex_based) if (ret) { - llerrs << "Convex Decomposition thread valid but could not set mesh data" << llendl; + LL_ERRS(LOG_MESH) << "Convex Decomposition thread valid but could not set mesh data." << LL_ENDL; } } } @@ -3282,7 +4054,8 @@ void LLPhysicsDecomp::doDecomposition() if (ret) { - llwarns << "Convex Decomposition thread valid but could not execute stage " << stage << llendl; + LL_WARNS(LOG_MESH) << "Convex Decomposition thread valid but could not execute stage " << stage << "." + << LL_ENDL; LLMutexLock lock(mMutex); mCurRequest->mHull.clear(); @@ -3411,9 +4184,9 @@ void LLPhysicsDecomp::doDecompositionSingleHull() setMeshData(mesh, true); LLCDResult ret = decomp->buildSingleHull() ; - if(ret) + if (ret) { - llwarns << "Could not execute decomposition stage when attempting to create single hull." << llendl; + LL_WARNS(LOG_MESH) << "Could not execute decomposition stage when attempting to create single hull." << LL_ENDL; make_box(mCurRequest); } else @@ -3718,3 +4491,63 @@ bool LLMeshRepository::meshRezEnabled() } return false; } + +// Threading: main thread only +// static +void LLMeshRepository::metricsStart() +{ + ++metrics_teleport_start_count; + sQuiescentTimer.start(0); +} + +// Threading: main thread only +// static +void LLMeshRepository::metricsStop() +{ + sQuiescentTimer.stop(0); +} + +// Threading: main thread only +// static +void LLMeshRepository::metricsProgress(unsigned int this_count) +{ + static bool first_start(true); + + if (first_start) + { + metricsStart(); + first_start = false; + } + sQuiescentTimer.ringBell(0, this_count); +} + +// Threading: main thread only +// static +void LLMeshRepository::metricsUpdate() +{ + F64 started, stopped; + U64 total_count(U64L(0)), user_cpu(U64L(0)), sys_cpu(U64L(0)); + + if (sQuiescentTimer.isExpired(0, started, stopped, total_count, user_cpu, sys_cpu)) + { + LLSD metrics; + + metrics["reason"] = "Mesh Download Quiescent"; + metrics["scope"] = metrics_teleport_start_count > 1 ? "Teleport" : "Login"; + metrics["start"] = started; + metrics["stop"] = stopped; + metrics["fetches"] = LLSD::Integer(total_count); + metrics["teleports"] = LLSD::Integer(metrics_teleport_start_count); + metrics["user_cpu"] = double(user_cpu) / 1.0e6; + metrics["sys_cpu"] = double(sys_cpu) / 1.0e6; + LL_INFOS(LOG_MESH) << "EventMarker " << metrics << LL_ENDL; + } +} + +// Threading: main thread only +// static +void teleport_started() +{ + LLMeshRepository::metricsStart(); +} + |