diff options
Diffstat (limited to 'indra/newview/llmeshrepository.cpp')
| -rwxr-xr-x | indra/newview/llmeshrepository.cpp | 2493 |
1 files changed, 1692 insertions, 801 deletions
diff --git a/indra/newview/llmeshrepository.cpp b/indra/newview/llmeshrepository.cpp index 1223615079..648056484e 100755 --- a/indra/newview/llmeshrepository.cpp +++ b/indra/newview/llmeshrepository.cpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2005&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2010, Linden Research, Inc. + * Copyright (C) 2010-2014, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -28,7 +28,8 @@ #include "apr_pools.h" #include "apr_dso.h" -#include "llhttpstatuscodes.h" +#include "llhttpconstants.h" +#include "llapr.h" #include "llmeshrepository.h" #include "llagent.h" @@ -37,11 +38,13 @@ #include "llcallbacklist.h" #include "llcurl.h" #include "lldatapacker.h" +#include "lldeadmantimer.h" #include "llfloatermodelpreview.h" #include "llfloaterperms.h" #include "lleconomy.h" #include "llimagej2c.h" #include "llhost.h" +#include "llmath.h" #include "llnotificationsutil.h" #include "llsd.h" #include "llsdutil_math.h" @@ -51,6 +54,7 @@ #include "llviewercontrol.h" #include "llviewerinventory.h" #include "llviewermenufile.h" +#include "llviewermessage.h" #include "llviewerobjectlist.h" #include "llviewerregion.h" #include "llviewertexturelist.h" @@ -64,6 +68,10 @@ #include "llfoldertype.h" #include "llviewerparcelmgr.h" #include "lluploadfloaterobservers.h" +#include "bufferarray.h" +#include "bufferstream.h" +#include "llfasttimer.h" +#include "llcorehttputil.h" #include "boost/lexical_cast.hpp" @@ -71,11 +79,289 @@ #include "netdb.h" #endif -#include <queue> + +// Purpose +// +// The purpose of this module is to provide access between the viewer +// and the asset system as regards to mesh objects. +// +// * High-throughput download of mesh assets from servers while +// following best industry practices for network profile. +// * Reliable expensing and upload of new mesh assets. +// * Recovery and retry from errors when appropriate. +// * Decomposition of mesh assets for preview and uploads. +// * And most important: all of the above without exposing the +// main thread to stalls due to deep processing or thread +// locking actions. In particular, the following operations +// on LLMeshRepository are very averse to any stalls: +// * loadMesh +// * getMeshHeader (For structural details, see: +// http://wiki.secondlife.com/wiki/Mesh/Mesh_Asset_Format) +// * notifyLoadedMeshes +// * getSkinInfo +// +// Threads +// +// main Main rendering thread, very sensitive to locking and other stalls +// repo Overseeing worker thread associated with the LLMeshRepoThread class +// decom Worker thread for mesh decomposition requests +// core HTTP worker thread: does the work but doesn't intrude here +// uploadN 0-N temporary mesh upload threads (0-1 in practice) +// +// Sequence of Operations +// +// What follows is a description of the retrieval of one LOD for +// a new mesh object. Work is performed by a series of short, quick +// actions distributed over a number of threads. Each is meant +// to proceed without stalling and the whole forms a deep request +// pipeline to achieve throughput. Ellipsis indicates a return +// or break in processing which is resumed elsewhere. +// +// main thread repo thread (run() method) +// +// loadMesh() invoked to request LOD +// append LODRequest to mPendingRequests +// ... +// other mesh requests may be made +// ... +// notifyLoadedMeshes() invoked to stage work +// append HeaderRequest to mHeaderReqQ +// ... +// scan mHeaderReqQ +// issue 4096-byte GET for header +// ... +// onCompleted() invoked for GET +// data copied +// headerReceived() invoked +// LLSD parsed +// mMeshHeader, mMeshHeaderSize updated +// scan mPendingLOD for LOD request +// push LODRequest to mLODReqQ +// ... +// scan mLODReqQ +// fetchMeshLOD() invoked +// issue Byte-Range GET for LOD +// ... +// onCompleted() invoked for GET +// data copied +// lodReceived() invoked +// unpack data into LLVolume +// append LoadedMesh to mLoadedQ +// ... +// notifyLoadedMeshes() invoked again +// scan mLoadedQ +// notifyMeshLoaded() for LOD +// setMeshAssetLoaded() invoked for system volume +// notifyMeshLoaded() invoked for each interested object +// ... +// +// Mutexes +// +// LLMeshRepository::mMeshMutex +// LLMeshRepoThread::mMutex +// LLMeshRepoThread::mHeaderMutex +// LLMeshRepoThread::mSignal (LLCondition) +// LLPhysicsDecomp::mSignal (LLCondition) +// LLPhysicsDecomp::mMutex +// LLMeshUploadThread::mMutex +// +// Mutex Order Rules +// +// 1. LLMeshRepoThread::mMutex before LLMeshRepoThread::mHeaderMutex +// 2. LLMeshRepository::mMeshMutex before LLMeshRepoThread::mMutex +// (There are more rules, haven't been extracted.) +// +// Data Member Access/Locking +// +// Description of how shared access to static and instance data +// members is performed. Each member is followed by the name of +// the mutex, if any, covering the data and then a list of data +// access models each of which is a triplet of the following form: +// +// {ro, wo, rw}.{main, repo, any}.{mutex, none} +// Type of access: read-only, write-only, read-write. +// Accessing thread or 'any' +// Relevant mutex held during access (several may be held) or 'none' +// +// A careful eye will notice some unsafe operations. Many of these +// have an alibi of some form. Several types of alibi are identified +// and listed here: +// +// [0] No alibi. Probably unsafe. +// [1] Single-writer, self-consistent readers. Old data must +// be tolerated by any reader but data will come true eventually. +// [2] Like [1] but provides a hint about thread state. These +// may be unsafe. +// [3] empty() check outside of lock. Can me made safish when +// done in double-check lock style. But this depends on +// std:: implementation and memory model. +// [4] Appears to be covered by a mutex but doesn't need one. +// [5] Read of a double-checked lock. +// +// So, in addition to documentation, take this as a to-do/review +// list and see if you can improve things. For porters to non-x86 +// architectures, the weaker memory models will make these platforms +// probabilistically more susceptible to hitting race conditions. +// True here and in other multi-thread code such as texture fetching. +// (Strong memory models make weak programmers. Weak memory models +// make strong programmers. Ref: arm, ppc, mips, alpha) +// +// LLMeshRepository: +// +// sBytesReceived none rw.repo.none, ro.main.none [1] +// sMeshRequestCount " +// sHTTPRequestCount " +// sHTTPLargeRequestCount " +// sHTTPRetryCount " +// sHTTPErrorCount " +// sLODPending mMeshMutex [4] rw.main.mMeshMutex +// sLODProcessing Repo::mMutex rw.any.Repo::mMutex +// sCacheBytesRead none rw.repo.none, ro.main.none [1] +// sCacheBytesWritten " +// sCacheReads " +// sCacheWrites " +// mLoadingMeshes mMeshMutex [4] rw.main.none, rw.any.mMeshMutex +// mSkinMap none rw.main.none +// mDecompositionMap none rw.main.none +// mPendingRequests mMeshMutex [4] rw.main.mMeshMutex +// mLoadingSkins mMeshMutex [4] rw.main.mMeshMutex +// mPendingSkinRequests mMeshMutex [4] rw.main.mMeshMutex +// mLoadingDecompositions mMeshMutex [4] rw.main.mMeshMutex +// mPendingDecompositionRequests mMeshMutex [4] rw.main.mMeshMutex +// mLoadingPhysicsShapes mMeshMutex [4] rw.main.mMeshMutex +// mPendingPhysicsShapeRequests mMeshMutex [4] rw.main.mMeshMutex +// mUploads none rw.main.none (upload thread accessing objects) +// mUploadWaitList none rw.main.none (upload thread accessing objects) +// mInventoryQ mMeshMutex [4] rw.main.mMeshMutex, ro.main.none [5] +// mUploadErrorQ mMeshMutex rw.main.mMeshMutex, rw.any.mMeshMutex +// mGetMeshVersion none rw.main.none +// +// LLMeshRepoThread: +// +// sActiveHeaderRequests mMutex rw.any.mMutex, ro.repo.none [1] +// sActiveLODRequests mMutex rw.any.mMutex, ro.repo.none [1] +// sMaxConcurrentRequests mMutex wo.main.none, ro.repo.none, ro.main.mMutex +// mMeshHeader mHeaderMutex rw.repo.mHeaderMutex, ro.main.mHeaderMutex, ro.main.none [0] +// mMeshHeaderSize mHeaderMutex rw.repo.mHeaderMutex +// mSkinRequests mMutex rw.repo.mMutex, ro.repo.none [5] +// mSkinInfoQ mMutex rw.repo.mMutex, rw.main.mMutex [5] (was: [0]) +// mDecompositionRequests mMutex rw.repo.mMutex, ro.repo.none [5] +// mPhysicsShapeRequests mMutex rw.repo.mMutex, ro.repo.none [5] +// mDecompositionQ mMutex rw.repo.mMutex, rw.main.mMutex [5] (was: [0]) +// mHeaderReqQ mMutex ro.repo.none [5], rw.repo.mMutex, rw.any.mMutex +// mLODReqQ mMutex ro.repo.none [5], rw.repo.mMutex, rw.any.mMutex +// mUnavailableQ mMutex rw.repo.none [0], ro.main.none [5], rw.main.mMutex +// mLoadedQ mMutex rw.repo.mMutex, ro.main.none [5], rw.main.mMutex +// mPendingLOD mMutex rw.repo.mMutex, rw.any.mMutex +// mGetMeshCapability mMutex rw.main.mMutex, ro.repo.mMutex (was: [0]) +// mGetMesh2Capability mMutex rw.main.mMutex, ro.repo.mMutex (was: [0]) +// mGetMeshVersion mMutex rw.main.mMutex, ro.repo.mMutex +// mHttp* none rw.repo.none +// +// LLMeshUploadThread: +// +// mDiscarded mMutex rw.main.mMutex, ro.uploadN.none [1] +// ... more ... +// +// QA/Development Testing +// +// Debug variable 'MeshUploadFakeErrors' takes a mask of bits that will +// simulate an error on fee query or upload. Defined bits are: +// +// 0x01 Simulate application error on fee check reading +// response body from file "fake_upload_error.xml" +// 0x02 Same as 0x01 but for actual upload attempt. +// 0x04 Simulate a transport problem on fee check with a +// locally-generated 500 status. +// 0x08 As with 0x04 but for the upload operation. +// +// For major changes, see the LL_MESH_FASTTIMER_ENABLE below and +// instructions for looking for frame stalls using fast timers. +// +// *TODO: Work list for followup actions: +// * Review anything marked as unsafe above, verify if there are real issues. +// * See if we can put ::run() into a hard sleep. May not actually perform better +// than the current scheme so be prepared for disappointment. You'll likely +// need to introduce a condition variable class that references a mutex in +// methods rather than derives from mutex which isn't correct. +// * On upload failures, make more information available to the alerting +// dialog. Get the structured information going into the log into a +// tree there. +// * Header parse failures come without much explanation. Elaborate. +// * Work queue for uploads? Any need for this or is the current scheme good +// enough? +// * Various temp buffers used in VFS I/O might be allocated once or even +// statically. Look for some wins here. +// * Move data structures holding mesh data used by main thread into main- +// thread-only access so that no locking is needed. May require duplication +// of some data so that worker thread has a minimal data set to guide +// operations. +// +// -------------------------------------------------------------------------- +// Development/Debug/QA Tools +// +// Enable here or in build environment to get fasttimer data on mesh fetches. +// +// Typically, this is used to perform A/B testing using the +// fasttimer console (shift-ctrl-9). This is done by looking +// for stalls due to lock contention between the main thread +// and the repository and HTTP code. In a release viewer, +// these appear as ping-time or worse spikes in frame time. +// With this instrumentation enabled, a stall will appear +// under the 'Mesh Fetch' timer which will be either top-level +// or under 'Render' time. + +static LLFastTimer::DeclareTimer FTM_MESH_FETCH("Mesh Fetch"); + +// Random failure testing for development/QA. +// +// Set the MESH_*_FAILED macros to either 'false' or to +// an invocation of MESH_RANDOM_NTH_TRUE() with some +// suitable number. In production, all must be false. +// +// Example: +// #define MESH_HTTP_RESPONSE_FAILED MESH_RANDOM_NTH_TRUE(9) + +// 1-in-N calls will test true +#define MESH_RANDOM_NTH_TRUE(_N) ( ll_rand(S32(_N)) == 0 ) + +#define MESH_HTTP_RESPONSE_FAILED false +#define MESH_HEADER_PROCESS_FAILED false +#define MESH_LOD_PROCESS_FAILED false +#define MESH_SKIN_INFO_PROCESS_FAILED false +#define MESH_DECOMP_PROCESS_FAILED false +#define MESH_PHYS_SHAPE_PROCESS_FAILED false + +// -------------------------------------------------------------------------- + LLMeshRepository gMeshRepo; -const U32 MAX_MESH_REQUESTS_PER_SECOND = 100; +const S32 MESH_HEADER_SIZE = 4096; // Important: assumption is that headers fit in this space + +const S32 REQUEST_HIGH_WATER_MIN = 32; // Limits for GetMesh regions +const S32 REQUEST_HIGH_WATER_MAX = 150; // Should remain under 2X throttle +const S32 REQUEST_LOW_WATER_MIN = 16; +const S32 REQUEST_LOW_WATER_MAX = 75; + +const S32 REQUEST2_HIGH_WATER_MIN = 32; // Limits for GetMesh2 regions +const S32 REQUEST2_HIGH_WATER_MAX = 100; +const S32 REQUEST2_LOW_WATER_MIN = 16; +const S32 REQUEST2_LOW_WATER_MAX = 50; + +const U32 LARGE_MESH_FETCH_THRESHOLD = 1U << 21; // Size at which requests goes to narrow/slow queue +const long SMALL_MESH_XFER_TIMEOUT = 120L; // Seconds to complete xfer, small mesh downloads +const long LARGE_MESH_XFER_TIMEOUT = 600L; // Seconds to complete xfer, large downloads + +// Would normally like to retry on uploads as some +// retryable failures would be recoverable. Unfortunately, +// the mesh service is using 500 (retryable) rather than +// 400/bad request (permanent) for a bad payload and +// retrying that just leads to revocation of the one-shot +// cap which then produces a 404 on retry destroying some +// (occasionally) useful error information. We'll leave +// upload retries to the user as in the past. SH-4667. +const long UPLOAD_RETRY_LIMIT = 0L; // Maximum mesh version to support. Three least significant digits are reserved for the minor version, // with major version changes indicating a format change that is not backwards compatible and should not @@ -86,35 +372,45 @@ const U32 MAX_MESH_REQUESTS_PER_SECOND = 100; const S32 MAX_MESH_VERSION = 999; U32 LLMeshRepository::sBytesReceived = 0; +U32 LLMeshRepository::sMeshRequestCount = 0; U32 LLMeshRepository::sHTTPRequestCount = 0; +U32 LLMeshRepository::sHTTPLargeRequestCount = 0; U32 LLMeshRepository::sHTTPRetryCount = 0; +U32 LLMeshRepository::sHTTPErrorCount = 0; U32 LLMeshRepository::sLODProcessing = 0; U32 LLMeshRepository::sLODPending = 0; U32 LLMeshRepository::sCacheBytesRead = 0; U32 LLMeshRepository::sCacheBytesWritten = 0; -U32 LLMeshRepository::sPeakKbps = 0; +U32 LLMeshRepository::sCacheReads = 0; +U32 LLMeshRepository::sCacheWrites = 0; +U32 LLMeshRepository::sMaxLockHoldoffs = 0; +LLDeadmanTimer LLMeshRepository::sQuiescentTimer(15.0, false); // true -> gather cpu metrics -const U32 MAX_TEXTURE_UPLOAD_RETRIES = 5; static S32 dump_num = 0; std::string make_dump_name(std::string prefix, S32 num) { return prefix + boost::lexical_cast<std::string>(num) + std::string(".xml"); - } void dump_llsd_to_file(const LLSD& content, std::string filename); LLSD llsd_from_file(std::string filename); -std::string header_lod[] = +const std::string header_lod[] = { "lowest_lod", "low_lod", "medium_lod", "high_lod" }; +const char * const LOG_MESH = "Mesh"; +// Static data and functions to measure mesh load +// time metrics for a new region scene. +static unsigned int metrics_teleport_start_count = 0; +boost::signals2::connection metrics_teleport_started_signal; +static void teleport_started(); //get the number of bytes resident in memory for given volume U32 get_volume_memory_size(const LLVolume* volume) @@ -196,179 +492,222 @@ void get_vertex_buffer_from_mesh(LLCDMeshData& mesh, LLModel::PhysicsMesh& res, } } -S32 LLMeshRepoThread::sActiveHeaderRequests = 0; -S32 LLMeshRepoThread::sActiveLODRequests = 0; +volatile S32 LLMeshRepoThread::sActiveHeaderRequests = 0; +volatile S32 LLMeshRepoThread::sActiveLODRequests = 0; U32 LLMeshRepoThread::sMaxConcurrentRequests = 1; - -class LLMeshHeaderResponder : public LLCurl::Responder +S32 LLMeshRepoThread::sRequestLowWater = REQUEST2_LOW_WATER_MIN; +S32 LLMeshRepoThread::sRequestHighWater = REQUEST2_HIGH_WATER_MIN; +S32 LLMeshRepoThread::sRequestWaterLevel = 0; + +// Base handler class for all mesh users of llcorehttp. +// This is roughly equivalent to a Responder class in +// traditional LL code. The base is going to perform +// common response/data handling in the inherited +// onCompleted() method. Derived classes, one for each +// type of HTTP action, define processData() and +// processFailure() methods to customize handling and +// error messages. +// +// LLCore::HttpHandler +// LLMeshHandlerBase +// LLMeshHeaderHandler +// LLMeshLODHandler +// LLMeshSkinInfoHandler +// LLMeshDecompositionHandler +// LLMeshPhysicsShapeHandler +// LLMeshUploadThread + +class LLMeshHandlerBase : public LLCore::HttpHandler { public: + LOG_CLASS(LLMeshHandlerBase); + LLMeshHandlerBase(U32 offset, U32 requested_bytes) + : LLCore::HttpHandler(), + mMeshParams(), + mProcessed(false), + mHttpHandle(LLCORE_HTTP_HANDLE_INVALID), + mOffset(offset), + mRequestedBytes(requested_bytes) + {} + + virtual ~LLMeshHandlerBase() + {} + +protected: + LLMeshHandlerBase(const LLMeshHandlerBase &); // Not defined + void operator=(const LLMeshHandlerBase &); // Not defined + +public: + virtual void onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * response); + virtual void processData(LLCore::BufferArray * body, S32 body_offset, U8 * data, S32 data_size) = 0; + virtual void processFailure(LLCore::HttpStatus status) = 0; + +public: LLVolumeParams mMeshParams; bool mProcessed; + LLCore::HttpHandle mHttpHandle; + U32 mOffset; + U32 mRequestedBytes; +}; - LLMeshHeaderResponder(const LLVolumeParams& mesh_params) - : mMeshParams(mesh_params) - { - LLMeshRepoThread::incActiveHeaderRequests(); - mProcessed = false; - } - ~LLMeshHeaderResponder() +// Subclass for header fetches. +// +// Thread: repo +class LLMeshHeaderHandler : public LLMeshHandlerBase +{ +public: + LOG_CLASS(LLMeshHeaderHandler); + LLMeshHeaderHandler(const LLVolumeParams & mesh_params, U32 offset, U32 requested_bytes) + : LLMeshHandlerBase(offset, requested_bytes) { - if (!LLApp::isQuitting()) - { - if (!mProcessed) - { //something went wrong, retry - llwarns << "Timeout or service unavailable, retrying." << llendl; - LLMeshRepository::sHTTPRetryCount++; - LLMeshRepoThread::HeaderRequest req(mMeshParams); - LLMutexLock lock(gMeshRepo.mThread->mMutex); - gMeshRepo.mThread->mHeaderReqQ.push(req); - } - - LLMeshRepoThread::decActiveHeaderRequests(); - } + mMeshParams = mesh_params; + LLMeshRepoThread::incActiveHeaderRequests(); } + virtual ~LLMeshHeaderHandler(); - virtual void completedRaw(U32 status, const std::string& reason, - const LLChannelDescriptors& channels, - const LLIOPipe::buffer_ptr_t& buffer); - +protected: + LLMeshHeaderHandler(const LLMeshHeaderHandler &); // Not defined + void operator=(const LLMeshHeaderHandler &); // Not defined + +public: + virtual void processData(LLCore::BufferArray * body, S32 body_offset, U8 * data, S32 data_size); + virtual void processFailure(LLCore::HttpStatus status); }; -class LLMeshLODResponder : public LLCurl::Responder + +// Subclass for LOD fetches. +// +// Thread: repo +class LLMeshLODHandler : public LLMeshHandlerBase { public: - LLVolumeParams mMeshParams; - S32 mLOD; - U32 mRequestedBytes; - U32 mOffset; - bool mProcessed; - - LLMeshLODResponder(const LLVolumeParams& mesh_params, S32 lod, U32 offset, U32 requested_bytes) - : mMeshParams(mesh_params), mLOD(lod), mOffset(offset), mRequestedBytes(requested_bytes) - { - LLMeshRepoThread::incActiveLODRequests(); - mProcessed = false; - } - - ~LLMeshLODResponder() - { - if (!LLApp::isQuitting()) + LOG_CLASS(LLMeshLODHandler); + LLMeshLODHandler(const LLVolumeParams & mesh_params, S32 lod, U32 offset, U32 requested_bytes) + : LLMeshHandlerBase(offset, requested_bytes), + mLOD(lod) { - if (!mProcessed) - { - llwarns << "Killed without being processed, retrying." << llendl; - LLMeshRepository::sHTTPRetryCount++; - gMeshRepo.mThread->lockAndLoadMeshLOD(mMeshParams, mLOD); - } - LLMeshRepoThread::decActiveLODRequests(); + mMeshParams = mesh_params; + LLMeshRepoThread::incActiveLODRequests(); } - } + virtual ~LLMeshLODHandler(); + +protected: + LLMeshLODHandler(const LLMeshLODHandler &); // Not defined + void operator=(const LLMeshLODHandler &); // Not defined - virtual void completedRaw(U32 status, const std::string& reason, - const LLChannelDescriptors& channels, - const LLIOPipe::buffer_ptr_t& buffer); +public: + virtual void processData(LLCore::BufferArray * body, S32 body_offset, U8 * data, S32 data_size); + virtual void processFailure(LLCore::HttpStatus status); +public: + S32 mLOD; }; -class LLMeshSkinInfoResponder : public LLCurl::Responder + +// Subclass for skin info fetches. +// +// Thread: repo +class LLMeshSkinInfoHandler : public LLMeshHandlerBase { public: - LLUUID mMeshID; - U32 mRequestedBytes; - U32 mOffset; - bool mProcessed; - - LLMeshSkinInfoResponder(const LLUUID& id, U32 offset, U32 size) - : mMeshID(id), mRequestedBytes(size), mOffset(offset) - { - mProcessed = false; - } + LOG_CLASS(LLMeshSkinInfoHandler); + LLMeshSkinInfoHandler(const LLUUID& id, U32 offset, U32 requested_bytes) + : LLMeshHandlerBase(offset, requested_bytes), + mMeshID(id) + {} + virtual ~LLMeshSkinInfoHandler(); - ~LLMeshSkinInfoResponder() - { - llassert(mProcessed); - } +protected: + LLMeshSkinInfoHandler(const LLMeshSkinInfoHandler &); // Not defined + void operator=(const LLMeshSkinInfoHandler &); // Not defined - virtual void completedRaw(U32 status, const std::string& reason, - const LLChannelDescriptors& channels, - const LLIOPipe::buffer_ptr_t& buffer); +public: + virtual void processData(LLCore::BufferArray * body, S32 body_offset, U8 * data, S32 data_size); + virtual void processFailure(LLCore::HttpStatus status); +public: + LLUUID mMeshID; }; -class LLMeshDecompositionResponder : public LLCurl::Responder + +// Subclass for decomposition fetches. +// +// Thread: repo +class LLMeshDecompositionHandler : public LLMeshHandlerBase { public: - LLUUID mMeshID; - U32 mRequestedBytes; - U32 mOffset; - bool mProcessed; + LOG_CLASS(LLMeshDecompositionHandler); + LLMeshDecompositionHandler(const LLUUID& id, U32 offset, U32 requested_bytes) + : LLMeshHandlerBase(offset, requested_bytes), + mMeshID(id) + {} + virtual ~LLMeshDecompositionHandler(); - LLMeshDecompositionResponder(const LLUUID& id, U32 offset, U32 size) - : mMeshID(id), mRequestedBytes(size), mOffset(offset) - { - mProcessed = false; - } - - ~LLMeshDecompositionResponder() - { - llassert(mProcessed); - } +protected: + LLMeshDecompositionHandler(const LLMeshDecompositionHandler &); // Not defined + void operator=(const LLMeshDecompositionHandler &); // Not defined - virtual void completedRaw(U32 status, const std::string& reason, - const LLChannelDescriptors& channels, - const LLIOPipe::buffer_ptr_t& buffer); +public: + virtual void processData(LLCore::BufferArray * body, S32 body_offset, U8 * data, S32 data_size); + virtual void processFailure(LLCore::HttpStatus status); +public: + LLUUID mMeshID; }; -class LLMeshPhysicsShapeResponder : public LLCurl::Responder + +// Subclass for physics shape fetches. +// +// Thread: repo +class LLMeshPhysicsShapeHandler : public LLMeshHandlerBase { public: - LLUUID mMeshID; - U32 mRequestedBytes; - U32 mOffset; - bool mProcessed; + LOG_CLASS(LLMeshPhysicsShapeHandler); + LLMeshPhysicsShapeHandler(const LLUUID& id, U32 offset, U32 requested_bytes) + : LLMeshHandlerBase(offset, requested_bytes), + mMeshID(id) + {} + virtual ~LLMeshPhysicsShapeHandler(); - LLMeshPhysicsShapeResponder(const LLUUID& id, U32 offset, U32 size) - : mMeshID(id), mRequestedBytes(size), mOffset(offset) - { - mProcessed = false; - } +protected: + LLMeshPhysicsShapeHandler(const LLMeshPhysicsShapeHandler &); // Not defined + void operator=(const LLMeshPhysicsShapeHandler &); // Not defined - ~LLMeshPhysicsShapeResponder() - { - llassert(mProcessed); - } - - virtual void completedRaw(U32 status, const std::string& reason, - const LLChannelDescriptors& channels, - const LLIOPipe::buffer_ptr_t& buffer); +public: + virtual void processData(LLCore::BufferArray * body, S32 body_offset, U8 * data, S32 data_size); + virtual void processFailure(LLCore::HttpStatus status); +public: + LLUUID mMeshID; }; -void log_upload_error(S32 status, const LLSD& content, std::string stage, std::string model_name) + +void log_upload_error(LLCore::HttpStatus status, const LLSD& content, + const char * const stage, const std::string & model_name) { // Add notification popup. LLSD args; - std::string message = content["error"]["message"]; - std::string identifier = content["error"]["identifier"]; + std::string message = content["error"]["message"].asString(); + std::string identifier = content["error"]["identifier"].asString(); args["MESSAGE"] = message; args["IDENTIFIER"] = identifier; args["LABEL"] = model_name; gMeshRepo.uploadError(args); // Log details. - llwarns << "stage: " << stage << " http status: " << status << llendl; + LL_WARNS(LOG_MESH) << "Error in stage: " << stage + << ", Reason: " << status.toString() + << " (" << status.toTerseString() << ")" << LL_ENDL; if (content.has("error")) { const LLSD& err = content["error"]; - llwarns << "err: " << err << llendl; - llwarns << "mesh upload failed, stage '" << stage - << "' error '" << err["error"].asString() - << "', message '" << err["message"].asString() - << "', id '" << err["identifier"].asString() - << "'" << llendl; + LL_WARNS(LOG_MESH) << "error: " << err << LL_ENDL; + LL_WARNS(LOG_MESH) << " mesh upload failed, stage '" << stage + << "', error '" << err["error"].asString() + << "', message '" << err["message"].asString() + << "', id '" << err["identifier"].asString() + << "'" << LL_ENDL; if (err.has("errors")) { S32 error_num = 0; @@ -378,13 +717,13 @@ void log_upload_error(S32 status, const LLSD& content, std::string stage, std::s ++it) { const LLSD& err_entry = *it; - llwarns << "error[" << error_num << "]:" << llendl; + LL_WARNS(LOG_MESH) << " error[" << error_num << "]:" << LL_ENDL; for (LLSD::map_const_iterator map_it = err_entry.beginMap(); map_it != err_entry.endMap(); ++map_it) { - llwarns << "\t" << map_it->first << ": " - << map_it->second << llendl; + LL_WARNS(LOG_MESH) << " " << map_it->first << ": " + << map_it->second << LL_ENDL; } error_num++; } @@ -392,153 +731,73 @@ void log_upload_error(S32 status, const LLSD& content, std::string stage, std::s } else { - llwarns << "bad mesh, no error information available" << llendl; + LL_WARNS(LOG_MESH) << "Bad response to mesh request, no additional error information available." << LL_ENDL; } } -class LLWholeModelFeeResponder: public LLCurl::Responder +LLMeshRepoThread::LLMeshRepoThread() +: LLThread("mesh repo"), + mHttpRequest(NULL), + mHttpOptions(NULL), + mHttpLargeOptions(NULL), + mHttpHeaders(NULL), + mHttpPolicyClass(LLCore::HttpRequest::DEFAULT_POLICY_ID), + mHttpLegacyPolicyClass(LLCore::HttpRequest::DEFAULT_POLICY_ID), + mHttpLargePolicyClass(LLCore::HttpRequest::DEFAULT_POLICY_ID), + mHttpPriority(0), + mGetMeshVersion(2) { - LLMeshUploadThread* mThread; - LLSD mModelData; - LLHandle<LLWholeModelFeeObserver> mObserverHandle; -public: - LLWholeModelFeeResponder(LLMeshUploadThread* thread, LLSD& model_data, LLHandle<LLWholeModelFeeObserver> observer_handle): - mThread(thread), - mModelData(model_data), - mObserverHandle(observer_handle) - { - if (mThread) - { - mThread->startRequest(); - } - } + LLAppCoreHttp & app_core_http(LLAppViewer::instance()->getAppCoreHttp()); - ~LLWholeModelFeeResponder() - { - if (mThread) - { - mThread->stopRequest(); - } - } + mMutex = new LLMutex(NULL); + mHeaderMutex = new LLMutex(NULL); + mSignal = new LLCondition(NULL); + mHttpRequest = new LLCore::HttpRequest; + mHttpOptions = new LLCore::HttpOptions; + mHttpOptions->setTransferTimeout(SMALL_MESH_XFER_TIMEOUT); + mHttpOptions->setUseRetryAfter(gSavedSettings.getBOOL("MeshUseHttpRetryAfter")); + mHttpLargeOptions = new LLCore::HttpOptions; + mHttpLargeOptions->setTransferTimeout(LARGE_MESH_XFER_TIMEOUT); + mHttpLargeOptions->setUseRetryAfter(gSavedSettings.getBOOL("MeshUseHttpRetryAfter")); + mHttpHeaders = new LLCore::HttpHeaders; + mHttpHeaders->append(HTTP_OUT_HEADER_ACCEPT, HTTP_CONTENT_VND_LL_MESH); + mHttpPolicyClass = app_core_http.getPolicy(LLAppCoreHttp::AP_MESH2); + mHttpLegacyPolicyClass = app_core_http.getPolicy(LLAppCoreHttp::AP_MESH1); + mHttpLargePolicyClass = app_core_http.getPolicy(LLAppCoreHttp::AP_LARGE_MESH); +} - virtual void completed(U32 status, - const std::string& reason, - const LLSD& content) - { - LLSD cc = content; - if (gSavedSettings.getS32("MeshUploadFakeErrors")&1) - { - cc = llsd_from_file("fake_upload_error.xml"); - } - dump_llsd_to_file(cc,make_dump_name("whole_model_fee_response_",dump_num)); - - LLWholeModelFeeObserver* observer = mObserverHandle.get(); - - if (isGoodStatus(status) && - cc["state"].asString() == "upload") - { - mThread->mWholeModelUploadURL = cc["uploader"].asString(); - - if (observer) - { - cc["data"]["upload_price"] = cc["upload_price"]; - observer->onModelPhysicsFeeReceived(cc["data"], mThread->mWholeModelUploadURL); - } - } - else - { - llwarns << "fee request failed" << llendl; - log_upload_error(status,cc,"fee",mModelData["name"]); - mThread->mWholeModelUploadURL = ""; +LLMeshRepoThread::~LLMeshRepoThread() +{ + LL_INFOS(LOG_MESH) << "Small GETs issued: " << LLMeshRepository::sHTTPRequestCount + << ", Large GETs issued: " << LLMeshRepository::sHTTPLargeRequestCount + << ", Max Lock Holdoffs: " << LLMeshRepository::sMaxLockHoldoffs + << LL_ENDL; - if (observer) - { - observer->setModelPhysicsFeeErrorStatus(status, reason); - } - } + for (http_request_set::iterator iter(mHttpRequestSet.begin()); + iter != mHttpRequestSet.end(); + ++iter) + { + delete *iter; } - -}; - -class LLWholeModelUploadResponder: public LLCurl::Responder -{ - LLMeshUploadThread* mThread; - LLSD mModelData; - LLHandle<LLWholeModelUploadObserver> mObserverHandle; - -public: - LLWholeModelUploadResponder(LLMeshUploadThread* thread, LLSD& model_data, LLHandle<LLWholeModelUploadObserver> observer_handle): - mThread(thread), - mModelData(model_data), - mObserverHandle(observer_handle) + mHttpRequestSet.clear(); + if (mHttpHeaders) { - if (mThread) - { - mThread->startRequest(); - } + mHttpHeaders->release(); + mHttpHeaders = NULL; } - - ~LLWholeModelUploadResponder() + if (mHttpOptions) { - if (mThread) - { - mThread->stopRequest(); - } + mHttpOptions->release(); + mHttpOptions = NULL; } - - virtual void completed(U32 status, - const std::string& reason, - const LLSD& content) + if (mHttpLargeOptions) { - LLSD cc = content; - if (gSavedSettings.getS32("MeshUploadFakeErrors")&2) - { - cc = llsd_from_file("fake_upload_error.xml"); - } - - dump_llsd_to_file(cc,make_dump_name("whole_model_upload_response_",dump_num)); - - LLWholeModelUploadObserver* observer = mObserverHandle.get(); - - // requested "mesh" asset type isn't actually the type - // of the resultant object, fix it up here. - if (isGoodStatus(status) && - cc["state"].asString() == "complete") - { - mModelData["asset_type"] = "object"; - gMeshRepo.updateInventory(LLMeshRepository::inventory_data(mModelData,cc)); - - if (observer) - { - doOnIdleOneTime(boost::bind(&LLWholeModelUploadObserver::onModelUploadSuccess, observer)); - } - } - else - { - llwarns << "upload failed" << llendl; - std::string model_name = mModelData["name"].asString(); - log_upload_error(status,cc,"upload",model_name); - - if (observer) - { - doOnIdleOneTime(boost::bind(&LLWholeModelUploadObserver::onModelUploadFailure, observer)); - } - } + mHttpLargeOptions->release(); + mHttpLargeOptions = NULL; } -}; - -LLMeshRepoThread::LLMeshRepoThread() -: LLThread("mesh repo") -{ - mWaiting = false; - mMutex = new LLMutex(NULL); - mHeaderMutex = new LLMutex(NULL); - mSignal = new LLCondition(NULL); -} - -LLMeshRepoThread::~LLMeshRepoThread() -{ + delete mHttpRequest; + mHttpRequest = NULL; delete mMutex; mMutex = NULL; delete mHeaderMutex; @@ -549,109 +808,181 @@ LLMeshRepoThread::~LLMeshRepoThread() void LLMeshRepoThread::run() { - mCurlRequest = new LLCurlRequest(); LLCDResult res = LLConvexDecomposition::initThread(); if (res != LLCD_OK) { - llwarns << "convex decomposition unable to be loaded" << llendl; + LL_WARNS(LOG_MESH) << "Convex decomposition unable to be loaded. Expect severe problems." << LL_ENDL; } while (!LLApp::isQuitting()) { - mWaiting = true; + // *TODO: Revise sleep/wake strategy and try to move away + // from polling operations in this thread. We can sleep + // this thread hard when: + // * All Http requests are serviced + // * LOD request queue empty + // * Header request queue empty + // * Skin info request queue empty + // * Decomposition request queue empty + // * Physics shape request queue empty + // We wake the thread when any of the above become untrue. + // Will likely need a correctly-implemented condition variable to do this. + // On the other hand, this may actually be an effective and efficient scheme... + mSignal->wait(); - mWaiting = false; - if (!LLApp::isQuitting()) + if (LLApp::isQuitting()) { - static U32 count = 0; + break; + } - static F32 last_hundred = gFrameTimeSeconds; + if (! mHttpRequestSet.empty()) + { + // Dispatch all HttpHandler notifications + mHttpRequest->update(0L); + } + sRequestWaterLevel = mHttpRequestSet.size(); // Stats data update - if (gFrameTimeSeconds - last_hundred > 1.f) - { //a second has gone by, clear count - last_hundred = gFrameTimeSeconds; - count = 0; + // NOTE: order of queue processing intentionally favors LOD requests over header requests + + while (!mLODReqQ.empty() && mHttpRequestSet.size() < sRequestHighWater) + { + if (! mMutex) + { + break; } + mMutex->lock(); + LODRequest req = mLODReqQ.front(); + mLODReqQ.pop(); + LLMeshRepository::sLODProcessing--; + mMutex->unlock(); - // NOTE: throttling intentionally favors LOD requests over header requests - - while (!mLODReqQ.empty() && count < MAX_MESH_REQUESTS_PER_SECOND && sActiveLODRequests < sMaxConcurrentRequests) + if (!fetchMeshLOD(req.mMeshParams, req.mLOD)) // failed, resubmit { - if (mMutex) - { - mMutex->lock(); - LODRequest req = mLODReqQ.front(); - mLODReqQ.pop(); - LLMeshRepository::sLODProcessing--; - mMutex->unlock(); - if (!fetchMeshLOD(req.mMeshParams, req.mLOD, count))//failed, resubmit - { - mMutex->lock(); - mLODReqQ.push(req) ; - mMutex->unlock(); - } - } + mMutex->lock(); + mLODReqQ.push(req); + ++LLMeshRepository::sLODProcessing; + mMutex->unlock(); } + } - while (!mHeaderReqQ.empty() && count < MAX_MESH_REQUESTS_PER_SECOND && sActiveHeaderRequests < sMaxConcurrentRequests) + while (!mHeaderReqQ.empty() && mHttpRequestSet.size() < sRequestHighWater) + { + if (! mMutex) { - if (mMutex) - { - mMutex->lock(); - HeaderRequest req = mHeaderReqQ.front(); - mHeaderReqQ.pop(); - mMutex->unlock(); - if (!fetchMeshHeader(req.mMeshParams, count))//failed, resubmit - { - mMutex->lock(); - mHeaderReqQ.push(req) ; - mMutex->unlock(); - } - } + break; + } + mMutex->lock(); + HeaderRequest req = mHeaderReqQ.front(); + mHeaderReqQ.pop(); + mMutex->unlock(); + if (!fetchMeshHeader(req.mMeshParams))//failed, resubmit + { + mMutex->lock(); + mHeaderReqQ.push(req) ; + mMutex->unlock(); } + } + + // For the final three request lists, similar goal to above but + // slightly different queue structures. Stay off the mutex when + // performing long-duration actions. + + if (mHttpRequestSet.size() < sRequestHighWater + && (! mSkinRequests.empty() + || ! mDecompositionRequests.empty() + || ! mPhysicsShapeRequests.empty())) + { + // Something to do probably, lock and double-check. We don't want + // to hold the lock long here. That will stall main thread activities + // so we bounce it. - { //mSkinRequests is protected by mSignal + mMutex->lock(); + if (! mSkinRequests.empty() && mHttpRequestSet.size() < sRequestHighWater) + { std::set<LLUUID> incomplete; - for (std::set<LLUUID>::iterator iter = mSkinRequests.begin(); iter != mSkinRequests.end(); ++iter) + std::set<LLUUID>::iterator iter(mSkinRequests.begin()); + while (iter != mSkinRequests.end() && mHttpRequestSet.size() < sRequestHighWater) { LLUUID mesh_id = *iter; + mSkinRequests.erase(iter); + mMutex->unlock(); + if (!fetchMeshSkinInfo(mesh_id)) { incomplete.insert(mesh_id); } + + mMutex->lock(); + iter = mSkinRequests.begin(); + } + + if (! incomplete.empty()) + { + mSkinRequests.insert(incomplete.begin(), incomplete.end()); } - mSkinRequests = incomplete; } - { //mDecompositionRequests is protected by mSignal + // holding lock, try next list + // *TODO: For UI/debug-oriented lists, we might drop the fine- + // grained locking as there's a lowered expectation of smoothness + // in these cases. + if (! mDecompositionRequests.empty() && mHttpRequestSet.size() < sRequestHighWater) + { std::set<LLUUID> incomplete; - for (std::set<LLUUID>::iterator iter = mDecompositionRequests.begin(); iter != mDecompositionRequests.end(); ++iter) + std::set<LLUUID>::iterator iter(mDecompositionRequests.begin()); + while (iter != mDecompositionRequests.end() && mHttpRequestSet.size() < sRequestHighWater) { LLUUID mesh_id = *iter; + mDecompositionRequests.erase(iter); + mMutex->unlock(); + if (!fetchMeshDecomposition(mesh_id)) { incomplete.insert(mesh_id); } + + mMutex->lock(); + iter = mDecompositionRequests.begin(); + } + + if (! incomplete.empty()) + { + mDecompositionRequests.insert(incomplete.begin(), incomplete.end()); } - mDecompositionRequests = incomplete; } - { //mPhysicsShapeRequests is protected by mSignal + // holding lock, final list + if (! mPhysicsShapeRequests.empty() && mHttpRequestSet.size() < sRequestHighWater) + { std::set<LLUUID> incomplete; - for (std::set<LLUUID>::iterator iter = mPhysicsShapeRequests.begin(); iter != mPhysicsShapeRequests.end(); ++iter) + std::set<LLUUID>::iterator iter(mPhysicsShapeRequests.begin()); + while (iter != mPhysicsShapeRequests.end() && mHttpRequestSet.size() < sRequestHighWater) { LLUUID mesh_id = *iter; + mPhysicsShapeRequests.erase(iter); + mMutex->unlock(); + if (!fetchMeshPhysicsShape(mesh_id)) { incomplete.insert(mesh_id); } + + mMutex->lock(); + iter = mPhysicsShapeRequests.begin(); } - mPhysicsShapeRequests = incomplete; - } - mCurlRequest->process(); + if (! incomplete.empty()) + { + mPhysicsShapeRequests.insert(incomplete.begin(), incomplete.end()); + } + } + mMutex->unlock(); } + + // For dev purposes only. A dynamic change could make this false + // and that shouldn't assert. + // llassert_always(mHttpRequestSet.size() <= sRequestHighWater); } if (mSignal->isLocked()) @@ -662,25 +993,25 @@ void LLMeshRepoThread::run() res = LLConvexDecomposition::quitThread(); if (res != LLCD_OK) { - llwarns << "convex decomposition unable to be quit" << llendl; + LL_WARNS(LOG_MESH) << "Convex decomposition unable to be quit." << LL_ENDL; } - - delete mCurlRequest; - mCurlRequest = NULL; } +// Mutex: LLMeshRepoThread::mMutex must be held on entry void LLMeshRepoThread::loadMeshSkinInfo(const LLUUID& mesh_id) -{ //protected by mSignal, no locking needed here +{ mSkinRequests.insert(mesh_id); } +// Mutex: LLMeshRepoThread::mMutex must be held on entry void LLMeshRepoThread::loadMeshDecomposition(const LLUUID& mesh_id) -{ //protected by mSignal, no locking needed here +{ mDecompositionRequests.insert(mesh_id); } +// Mutex: LLMeshRepoThread::mMutex must be held on entry void LLMeshRepoThread::loadMeshPhysicsShape(const LLUUID& mesh_id) -{ //protected by mSignal, no locking needed here +{ mPhysicsShapeRequests.insert(mesh_id); } @@ -693,7 +1024,6 @@ void LLMeshRepoThread::lockAndLoadMeshLOD(const LLVolumeParams& mesh_params, S32 } - void LLMeshRepoThread::loadMeshLOD(const LLVolumeParams& mesh_params, S32 lod) { //could be called from any thread LLMutexLock lock(mMutex); @@ -715,7 +1045,7 @@ void LLMeshRepoThread::loadMeshLOD(const LLVolumeParams& mesh_params, S32 lod) if (pending != mPendingLOD.end()) { //append this lod request to existing header request pending->second.push_back(lod); - llassert(pending->second.size() <= LLModel::NUM_LODS) + llassert(pending->second.size() <= LLModel::NUM_LODS); } else { //if no header request is pending, fetch header @@ -725,31 +1055,125 @@ void LLMeshRepoThread::loadMeshLOD(const LLVolumeParams& mesh_params, S32 lod) } } -//static -std::string LLMeshRepoThread::constructUrl(LLUUID mesh_id) +// Mutex: must be holding mMutex when called +void LLMeshRepoThread::setGetMeshCaps(const std::string & get_mesh1, + const std::string & get_mesh2, + int pref_version) { - std::string http_url; + mGetMeshCapability = get_mesh1; + mGetMesh2Capability = get_mesh2; + mGetMeshVersion = pref_version; +} + + +// Constructs a Cap URL for the mesh. Prefers a GetMesh2 cap +// over a GetMesh cap. +// +// Mutex: acquires mMutex +void LLMeshRepoThread::constructUrl(LLUUID mesh_id, std::string * url, int * version) +{ + std::string res_url; + int res_version(2); if (gAgent.getRegion()) { - http_url = gMeshRepo.mGetMeshCapability; + LLMutexLock lock(mMutex); + + // Get a consistent pair of (cap string, version). The + // locking could be eliminated here without loss of safety + // by using a set of staging values in setGetMeshCaps(). + + if (! mGetMesh2Capability.empty() && mGetMeshVersion > 1) + { + res_url = mGetMesh2Capability; + res_version = 2; + } + else + { + res_url = mGetMeshCapability; + res_version = 1; + } } - if (!http_url.empty()) + if (! res_url.empty()) { - http_url += "/?mesh_id="; - http_url += mesh_id.asString().c_str(); + res_url += "/?mesh_id="; + res_url += mesh_id.asString().c_str(); } else { - llwarns << "Current region does not have GetMesh capability! Cannot load " << mesh_id << ".mesh" << llendl; + LL_WARNS_ONCE(LOG_MESH) << "Current region does not have GetMesh capability! Cannot load " + << mesh_id << ".mesh" << LL_ENDL; } - return http_url; + *url = res_url; + *version = res_version; } +// Issue an HTTP GET request with byte range using the right +// policy class. Large requests go to the large request class. +// If the current region supports GetMesh2, we prefer that for +// smaller requests otherwise we try to use the traditional +// GetMesh capability and connection concurrency. +// +// @return Valid handle or LLCORE_HTTP_HANDLE_INVALID. +// If the latter, actual status is found in +// mHttpStatus member which is valid until the +// next call to this method. +// +// Thread: repo +LLCore::HttpHandle LLMeshRepoThread::getByteRange(const std::string & url, int cap_version, + size_t offset, size_t len, + LLCore::HttpHandler * handler) +{ + // Also used in lltexturefetch.cpp + static LLCachedControl<bool> disable_range_req(gSavedSettings, "HttpRangeRequestsDisable", false); + + LLCore::HttpHandle handle(LLCORE_HTTP_HANDLE_INVALID); + + if (len < LARGE_MESH_FETCH_THRESHOLD) + { + handle = mHttpRequest->requestGetByteRange((2 == cap_version + ? mHttpPolicyClass + : mHttpLegacyPolicyClass), + mHttpPriority, + url, + (disable_range_req ? size_t(0) : offset), + (disable_range_req ? size_t(0) : len), + mHttpOptions, + mHttpHeaders, + handler); + if (LLCORE_HTTP_HANDLE_INVALID != handle) + { + ++LLMeshRepository::sHTTPRequestCount; + } + } + else + { + handle = mHttpRequest->requestGetByteRange(mHttpLargePolicyClass, + mHttpPriority, + url, + (disable_range_req ? size_t(0) : offset), + (disable_range_req ? size_t(0) : len), + mHttpLargeOptions, + mHttpHeaders, + handler); + if (LLCORE_HTTP_HANDLE_INVALID != handle) + { + ++LLMeshRepository::sHTTPLargeRequestCount; + } + } + if (LLCORE_HTTP_HANDLE_INVALID == handle) + { + // Something went wrong, capture the error code for caller. + mHttpStatus = mHttpRequest->getStatus(); + } + return handle; +} + + bool LLMeshRepoThread::fetchMeshSkinInfo(const LLUUID& mesh_id) -{ //protected by mMutex +{ if (!mHeaderMutex) { @@ -764,6 +1188,7 @@ bool LLMeshRepoThread::fetchMeshSkinInfo(const LLUUID& mesh_id) return false; } + ++LLMeshRepository::sMeshRequestCount; bool ret = true ; U32 header_size = mMeshHeaderSize[mesh_id]; @@ -782,6 +1207,7 @@ bool LLMeshRepoThread::fetchMeshSkinInfo(const LLUUID& mesh_id) if (file.getSize() >= offset+size) { LLMeshRepository::sCacheBytesRead += size; + ++LLMeshRepository::sCacheReads; file.seek(offset); U8* buffer = new U8[size]; file.read(buffer, size); @@ -806,17 +1232,27 @@ bool LLMeshRepoThread::fetchMeshSkinInfo(const LLUUID& mesh_id) } //reading from VFS failed for whatever reason, fetch from sim - std::vector<std::string> headers; - headers.push_back("Accept: application/octet-stream"); + int cap_version(2); + std::string http_url; + constructUrl(mesh_id, &http_url, &cap_version); - std::string http_url = constructUrl(mesh_id); if (!http_url.empty()) { - ret = mCurlRequest->getByteRange(http_url, headers, offset, size, - new LLMeshSkinInfoResponder(mesh_id, offset, size)); - if(ret) + LLMeshSkinInfoHandler * handler = new LLMeshSkinInfoHandler(mesh_id, offset, size); + LLCore::HttpHandle handle = getByteRange(http_url, cap_version, offset, size, handler); + if (LLCORE_HTTP_HANDLE_INVALID == handle) { - LLMeshRepository::sHTTPRequestCount++; + LL_WARNS(LOG_MESH) << "HTTP GET request failed for skin info on mesh " << mID + << ". Reason: " << mHttpStatus.toString() + << " (" << mHttpStatus.toTerseString() << ")" + << LL_ENDL; + delete handler; + ret = false; + } + else + { + handler->mHttpHandle = handle; + mHttpRequestSet.insert(handler); } } } @@ -831,7 +1267,7 @@ bool LLMeshRepoThread::fetchMeshSkinInfo(const LLUUID& mesh_id) } bool LLMeshRepoThread::fetchMeshDecomposition(const LLUUID& mesh_id) -{ //protected by mMutex +{ if (!mHeaderMutex) { return false; @@ -845,6 +1281,7 @@ bool LLMeshRepoThread::fetchMeshDecomposition(const LLUUID& mesh_id) return false; } + ++LLMeshRepository::sMeshRequestCount; U32 header_size = mMeshHeaderSize[mesh_id]; bool ret = true ; @@ -863,6 +1300,7 @@ bool LLMeshRepoThread::fetchMeshDecomposition(const LLUUID& mesh_id) if (file.getSize() >= offset+size) { LLMeshRepository::sCacheBytesRead += size; + ++LLMeshRepository::sCacheReads; file.seek(offset); U8* buffer = new U8[size]; @@ -888,17 +1326,27 @@ bool LLMeshRepoThread::fetchMeshDecomposition(const LLUUID& mesh_id) } //reading from VFS failed for whatever reason, fetch from sim - std::vector<std::string> headers; - headers.push_back("Accept: application/octet-stream"); + int cap_version(2); + std::string http_url; + constructUrl(mesh_id, &http_url, &cap_version); - std::string http_url = constructUrl(mesh_id); if (!http_url.empty()) { - ret = mCurlRequest->getByteRange(http_url, headers, offset, size, - new LLMeshDecompositionResponder(mesh_id, offset, size)); - if(ret) + LLMeshDecompositionHandler * handler = new LLMeshDecompositionHandler(mesh_id, offset, size); + LLCore::HttpHandle handle = getByteRange(http_url, cap_version, offset, size, handler); + if (LLCORE_HTTP_HANDLE_INVALID == handle) { - LLMeshRepository::sHTTPRequestCount++; + LL_WARNS(LOG_MESH) << "HTTP GET request failed for decomposition mesh " << mID + << ". Reason: " << mHttpStatus.toString() + << " (" << mHttpStatus.toTerseString() << ")" + << LL_ENDL; + delete handler; + ret = false; + } + else + { + handler->mHttpHandle = handle; + mHttpRequestSet.insert(handler); } } } @@ -913,7 +1361,7 @@ bool LLMeshRepoThread::fetchMeshDecomposition(const LLUUID& mesh_id) } bool LLMeshRepoThread::fetchMeshPhysicsShape(const LLUUID& mesh_id) -{ //protected by mMutex +{ if (!mHeaderMutex) { return false; @@ -927,6 +1375,7 @@ bool LLMeshRepoThread::fetchMeshPhysicsShape(const LLUUID& mesh_id) return false; } + ++LLMeshRepository::sMeshRequestCount; U32 header_size = mMeshHeaderSize[mesh_id]; bool ret = true ; @@ -945,6 +1394,7 @@ bool LLMeshRepoThread::fetchMeshPhysicsShape(const LLUUID& mesh_id) if (file.getSize() >= offset+size) { LLMeshRepository::sCacheBytesRead += size; + ++LLMeshRepository::sCacheReads; file.seek(offset); U8* buffer = new U8[size]; file.read(buffer, size); @@ -969,18 +1419,27 @@ bool LLMeshRepoThread::fetchMeshPhysicsShape(const LLUUID& mesh_id) } //reading from VFS failed for whatever reason, fetch from sim - std::vector<std::string> headers; - headers.push_back("Accept: application/octet-stream"); + int cap_version(2); + std::string http_url; + constructUrl(mesh_id, &http_url, &cap_version); - std::string http_url = constructUrl(mesh_id); if (!http_url.empty()) { - ret = mCurlRequest->getByteRange(http_url, headers, offset, size, - new LLMeshPhysicsShapeResponder(mesh_id, offset, size)); - - if(ret) + LLMeshPhysicsShapeHandler * handler = new LLMeshPhysicsShapeHandler(mesh_id, offset, size); + LLCore::HttpHandle handle = getByteRange(http_url, cap_version, offset, size, handler); + if (LLCORE_HTTP_HANDLE_INVALID == handle) + { + LL_WARNS(LOG_MESH) << "HTTP GET request failed for physics shape on mesh " << mID + << ". Reason: " << mHttpStatus.toString() + << " (" << mHttpStatus.toTerseString() << ")" + << LL_ENDL; + delete handler; + ret = false; + } + else { - LLMeshRepository::sHTTPRequestCount++; + handler->mHttpHandle = handle; + mHttpRequestSet.insert(handler); } } } @@ -1027,8 +1486,10 @@ void LLMeshRepoThread::decActiveHeaderRequests() } //return false if failed to get header -bool LLMeshRepoThread::fetchMeshHeader(const LLVolumeParams& mesh_params, U32& count) +bool LLMeshRepoThread::fetchMeshHeader(const LLVolumeParams& mesh_params) { + ++LLMeshRepository::sMeshRequestCount; + { //look for mesh in asset in vfs LLVFile file(gVFS, mesh_params.getSculptID(), LLAssetType::AT_MESH); @@ -1036,13 +1497,16 @@ bool LLMeshRepoThread::fetchMeshHeader(const LLVolumeParams& mesh_params, U32& c S32 size = file.getSize(); if (size > 0) - { //NOTE -- if the header size is ever more than 4KB, this will break - U8 buffer[4096]; - S32 bytes = llmin(size, 4096); + { + // *NOTE: if the header size is ever more than 4KB, this will break + U8 buffer[MESH_HEADER_SIZE]; + S32 bytes = llmin(size, MESH_HEADER_SIZE); LLMeshRepository::sCacheBytesRead += bytes; + ++LLMeshRepository::sCacheReads; file.read(buffer, bytes); if (headerReceived(mesh_params, buffer, bytes)) - { //did not do an HTTP request, return false + { + // Found mesh in VFS cache return true; } } @@ -1050,29 +1514,40 @@ bool LLMeshRepoThread::fetchMeshHeader(const LLVolumeParams& mesh_params, U32& c //either cache entry doesn't exist or is corrupt, request header from simulator bool retval = true ; - std::vector<std::string> headers; - headers.push_back("Accept: application/octet-stream"); + int cap_version(2); + std::string http_url; + constructUrl(mesh_params.getSculptID(), &http_url, &cap_version); - std::string http_url = constructUrl(mesh_params.getSculptID()); if (!http_url.empty()) { //grab first 4KB if we're going to bother with a fetch. Cache will prevent future fetches if a full mesh fits //within the first 4KB //NOTE -- this will break of headers ever exceed 4KB - retval = mCurlRequest->getByteRange(http_url, headers, 0, 4096, new LLMeshHeaderResponder(mesh_params)); - if(retval) + + LLMeshHeaderHandler * handler = new LLMeshHeaderHandler(mesh_params, 0, MESH_HEADER_SIZE); + LLCore::HttpHandle handle = getByteRange(http_url, cap_version, 0, MESH_HEADER_SIZE, handler); + if (LLCORE_HTTP_HANDLE_INVALID == handle) + { + LL_WARNS(LOG_MESH) << "HTTP GET request failed for mesh header " << mID + << ". Reason: " << mHttpStatus.toString() + << " (" << mHttpStatus.toTerseString() << ")" + << LL_ENDL; + delete handler; + retval = false; + } + else { - LLMeshRepository::sHTTPRequestCount++; + handler->mHttpHandle = handle; + mHttpRequestSet.insert(handler); } - count++; } return retval; } //return false if failed to get mesh lod. -bool LLMeshRepoThread::fetchMeshLOD(const LLVolumeParams& mesh_params, S32 lod, U32& count) -{ //protected by mMutex +bool LLMeshRepoThread::fetchMeshLOD(const LLVolumeParams& mesh_params, S32 lod) +{ if (!mHeaderMutex) { return false; @@ -1080,6 +1555,7 @@ bool LLMeshRepoThread::fetchMeshLOD(const LLVolumeParams& mesh_params, S32 lod, mHeaderMutex->lock(); + ++LLMeshRepository::sMeshRequestCount; bool retval = true; LLUUID mesh_id = mesh_params.getSculptID(); @@ -1101,6 +1577,7 @@ bool LLMeshRepoThread::fetchMeshLOD(const LLVolumeParams& mesh_params, S32 lod, if (file.getSize() >= offset+size) { LLMeshRepository::sCacheBytesRead += size; + ++LLMeshRepository::sCacheReads; file.seek(offset); U8* buffer = new U8[size]; file.read(buffer, size); @@ -1125,20 +1602,29 @@ bool LLMeshRepoThread::fetchMeshLOD(const LLVolumeParams& mesh_params, S32 lod, } //reading from VFS failed for whatever reason, fetch from sim - std::vector<std::string> headers; - headers.push_back("Accept: application/octet-stream"); + int cap_version(2); + std::string http_url; + constructUrl(mesh_id, &http_url, &cap_version); - std::string http_url = constructUrl(mesh_id); if (!http_url.empty()) { - retval = mCurlRequest->getByteRange(constructUrl(mesh_id), headers, offset, size, - new LLMeshLODResponder(mesh_params, lod, offset, size)); - - if(retval) + LLMeshLODHandler * handler = new LLMeshLODHandler(mesh_params, lod, offset, size); + LLCore::HttpHandle handle = getByteRange(http_url, cap_version, offset, size, handler); + if (LLCORE_HTTP_HANDLE_INVALID == handle) { - LLMeshRepository::sHTTPRequestCount++; + LL_WARNS(LOG_MESH) << "HTTP GET request failed for LOD on mesh " << mID + << ". Reason: " << mHttpStatus.toString() + << " (" << mHttpStatus.toTerseString() << ")" + << LL_ENDL; + delete handler; + retval = false; + } + else + { + handler->mHttpHandle = handle; + mHttpRequestSet.insert(handler); + // *NOTE: Allowing a re-request, not marking as unavailable. Is that correct? } - count++; } else { @@ -1160,6 +1646,7 @@ bool LLMeshRepoThread::fetchMeshLOD(const LLVolumeParams& mesh_params, S32 lod, bool LLMeshRepoThread::headerReceived(const LLVolumeParams& mesh_params, U8* data, S32 data_size) { + const LLUUID mesh_id = mesh_params.getSculptID(); LLSD header; U32 header_size = 0; @@ -1180,7 +1667,8 @@ bool LLMeshRepoThread::headerReceived(const LLVolumeParams& mesh_params, U8* dat if (!LLSDSerialize::fromBinary(header, stream, data_size)) { - llwarns << "Mesh header parse error. Not a valid mesh asset!" << llendl; + LL_WARNS(LOG_MESH) << "Mesh header parse error. Not a valid mesh asset! ID: " << mesh_id + << LL_ENDL; return false; } @@ -1188,21 +1676,20 @@ bool LLMeshRepoThread::headerReceived(const LLVolumeParams& mesh_params, U8* dat } else { - llinfos - << "Marking header as non-existent, will not retry." << llendl; + LL_INFOS(LOG_MESH) << "Non-positive data size. Marking header as non-existent, will not retry. ID: " << mesh_id + << LL_ENDL; header["404"] = 1; } { - LLUUID mesh_id = mesh_params.getSculptID(); { LLMutexLock lock(mHeaderMutex); mMeshHeaderSize[mesh_id] = header_size; mMeshHeader[mesh_id] = header; - } - + } + LLMutexLock lock(mMutex); // make sure only one thread access mPendingLOD at the same time. //check for pending requests @@ -1215,8 +1702,8 @@ bool LLMeshRepoThread::headerReceived(const LLVolumeParams& mesh_params, U8* dat mLODReqQ.push(req); LLMeshRepository::sLODProcessing++; } + mPendingLOD.erase(iter); } - mPendingLOD.erase(iter); } return true; @@ -1256,7 +1743,8 @@ bool LLMeshRepoThread::skinInfoReceived(const LLUUID& mesh_id, U8* data, S32 dat if (!unzip_llsd(skin, stream, data_size)) { - llwarns << "Mesh skin info parse error. Not a valid mesh asset!" << llendl; + LL_WARNS(LOG_MESH) << "Mesh skin info parse error. Not a valid mesh asset! ID: " << mesh_id + << LL_ENDL; return false; } } @@ -1265,8 +1753,11 @@ bool LLMeshRepoThread::skinInfoReceived(const LLUUID& mesh_id, U8* data, S32 dat LLMeshSkinInfo info(skin); info.mMeshID = mesh_id; - //llinfos<<"info pelvis offset"<<info.mPelvisOffset<<llendl; - mSkinInfoQ.push(info); + // LL_DEBUGS(LOG_MESH) << "info pelvis offset" << info.mPelvisOffset << LL_ENDL; + { + LLMutexLock lock(mMutex); + mSkinInfoQ.push_back(info); + } } return true; @@ -1284,7 +1775,8 @@ bool LLMeshRepoThread::decompositionReceived(const LLUUID& mesh_id, U8* data, S3 if (!unzip_llsd(decomp, stream, data_size)) { - llwarns << "Mesh decomposition parse error. Not a valid mesh asset!" << llendl; + LL_WARNS(LOG_MESH) << "Mesh decomposition parse error. Not a valid mesh asset! ID: " << mesh_id + << LL_ENDL; return false; } } @@ -1292,7 +1784,10 @@ bool LLMeshRepoThread::decompositionReceived(const LLUUID& mesh_id, U8* data, S3 { LLModel::Decomposition* d = new LLModel::Decomposition(decomp); d->mMeshID = mesh_id; - mDecompositionQ.push(d); + { + LLMutexLock lock(mMutex); + mDecompositionQ.push_back(d); + } } return true; @@ -1351,15 +1846,20 @@ bool LLMeshRepoThread::physicsShapeReceived(const LLUUID& mesh_id, U8* data, S32 } } - mDecompositionQ.push(d); + { + LLMutexLock lock(mMutex); + mDecompositionQ.push_back(d); + } return true; } LLMeshUploadThread::LLMeshUploadThread(LLMeshUploadThread::instance_list& data, LLVector3& scale, bool upload_textures, - bool upload_skin, bool upload_joints, std::string upload_url, bool do_upload, - LLHandle<LLWholeModelFeeObserver> fee_observer, LLHandle<LLWholeModelUploadObserver> upload_observer) -: LLThread("mesh upload"), - mDiscarded(FALSE), + bool upload_skin, bool upload_joints, const std::string & upload_url, bool do_upload, + LLHandle<LLWholeModelFeeObserver> fee_observer, + LLHandle<LLWholeModelUploadObserver> upload_observer) + : LLThread("mesh upload"), + LLCore::HttpHandler(), + mDiscarded(false), mDoUpload(do_upload), mWholeModelUploadURL(upload_url), mFeeObserverHandle(fee_observer), @@ -1370,7 +1870,6 @@ LLMeshUploadThread::LLMeshUploadThread(LLMeshUploadThread::instance_list& data, mUploadSkin = upload_skin; mUploadJoints = upload_joints; mMutex = new LLMutex(NULL); - mCurlRequest = NULL; mPendingUploads = 0; mFinished = false; mOrigin = gAgent.getPositionAgent(); @@ -1381,11 +1880,32 @@ LLMeshUploadThread::LLMeshUploadThread(LLMeshUploadThread::instance_list& data, mOrigin += gAgent.getAtAxis() * scale.magVec(); mMeshUploadTimeOut = gSavedSettings.getS32("MeshUploadTimeOut") ; + + mHttpRequest = new LLCore::HttpRequest; + mHttpOptions = new LLCore::HttpOptions; + mHttpOptions->setTransferTimeout(mMeshUploadTimeOut); + mHttpOptions->setUseRetryAfter(gSavedSettings.getBOOL("MeshUseHttpRetryAfter")); + mHttpOptions->setRetries(UPLOAD_RETRY_LIMIT); + mHttpHeaders = new LLCore::HttpHeaders; + mHttpHeaders->append(HTTP_OUT_HEADER_CONTENT_TYPE, HTTP_CONTENT_LLSD_XML); + mHttpPolicyClass = LLAppViewer::instance()->getAppCoreHttp().getPolicy(LLAppCoreHttp::AP_UPLOADS); + mHttpPriority = 0; } LLMeshUploadThread::~LLMeshUploadThread() { - + if (mHttpHeaders) + { + mHttpHeaders->release(); + mHttpHeaders = NULL; + } + if (mHttpOptions) + { + mHttpOptions->release(); + mHttpOptions = NULL; + } + delete mHttpRequest; + mHttpRequest = NULL; } LLMeshUploadThread::DecompRequest::DecompRequest(LLModel* mdl, LLModel* base_model, LLMeshUploadThread* thread) @@ -1428,10 +1948,10 @@ void LLMeshUploadThread::preStart() void LLMeshUploadThread::discard() { LLMutexLock lock(mMutex) ; - mDiscarded = TRUE ; + mDiscarded = true; } -BOOL LLMeshUploadThread::isDiscarded() +bool LLMeshUploadThread::isDiscarded() const { LLMutexLock lock(mMutex) ; return mDiscarded ; @@ -1476,9 +1996,9 @@ void LLMeshUploadThread::wholeModelToLLSD(LLSD& dest, bool include_textures) result["asset_type"] = "mesh"; result["inventory_type"] = "object"; result["description"] = "(No Description)"; - result["next_owner_mask"] = LLSD::Integer(LLFloaterPerms::getNextOwnerPerms()); - result["group_mask"] = LLSD::Integer(LLFloaterPerms::getGroupPerms()); - result["everyone_mask"] = LLSD::Integer(LLFloaterPerms::getEveryonePerms()); + result["next_owner_mask"] = LLSD::Integer(LLFloaterPerms::getNextOwnerPerms("Uploads")); + result["group_mask"] = LLSD::Integer(LLFloaterPerms::getGroupPerms("Uploads")); + result["everyone_mask"] = LLSD::Integer(LLFloaterPerms::getEveryonePerms("Uploads")); res["mesh_list"] = LLSD::emptyArray(); res["texture_list"] = LLSD::emptyArray(); @@ -1687,7 +2207,13 @@ void LLMeshUploadThread::generateHulls() if(has_valid_requests) { - while (!mPhysicsComplete) + // *NOTE: Interesting livelock condition on shutdown. If there + // is an upload request in generateHulls() when shutdown starts, + // the main thread isn't available to manage communication between + // the decomposition thread and the upload thread and this loop + // wouldn't complete in turn stalling the main thread. The check + // on isDiscarded() prevents that. + while (! mPhysicsComplete && ! isDiscarded()) { apr_sleep(100); } @@ -1696,86 +2222,249 @@ void LLMeshUploadThread::generateHulls() void LLMeshUploadThread::doWholeModelUpload() { - mCurlRequest = new LLCurlRequest(); + LL_DEBUGS(LOG_MESH) << "Starting model upload. Instances: " << mInstance.size() << LL_ENDL; if (mWholeModelUploadURL.empty()) { - llinfos << "unable to upload, fee request failed" << llendl; + LL_WARNS(LOG_MESH) << "Missing mesh upload capability, unable to upload, fee request failed." + << LL_ENDL; } else { generateHulls(); - - LLSD full_model_data; - wholeModelToLLSD(full_model_data, true); - LLSD body = full_model_data["asset_resources"]; - dump_llsd_to_file(body,make_dump_name("whole_model_body_",dump_num)); - LLCurlRequest::headers_t headers; - + LL_DEBUGS(LOG_MESH) << "Hull generation completed." << LL_ENDL; + + mModelData = LLSD::emptyMap(); + wholeModelToLLSD(mModelData, true); + LLSD body = mModelData["asset_resources"]; + + dump_llsd_to_file(body, make_dump_name("whole_model_body_", dump_num)); + + LLCore::HttpHandle handle = LLCoreHttpUtil::requestPostWithLLSD(mHttpRequest, + mHttpPolicyClass, + mHttpPriority, + mWholeModelUploadURL, + body, + mHttpOptions, + mHttpHeaders, + this); + if (LLCORE_HTTP_HANDLE_INVALID == handle) + { + mHttpStatus = mHttpRequest->getStatus(); + + LL_WARNS(LOG_MESH) << "Couldn't issue request for full model upload. Reason: " << mHttpStatus.toString() + << " (" << mHttpStatus.toTerseString() << ")" + << LL_ENDL; + } + else { - LLCurl::ResponderPtr responder = new LLWholeModelUploadResponder(this, full_model_data, mUploadObserverHandle) ; + U32 sleep_time(10); + + LL_DEBUGS(LOG_MESH) << "POST request issued." << LL_ENDL; + + mHttpRequest->update(0); + while (! LLApp::isQuitting() && ! finished() && ! isDiscarded()) + { + ms_sleep(sleep_time); + sleep_time = llmin(250U, sleep_time + sleep_time); + mHttpRequest->update(0); + } - while(!mCurlRequest->post(mWholeModelUploadURL, headers, body, responder, mMeshUploadTimeOut)) + if (isDiscarded()) + { + LL_DEBUGS(LOG_MESH) << "Mesh upload operation discarded." << LL_ENDL; + } + else { - //sleep for 10ms to prevent eating a whole core - apr_sleep(10000); + LL_DEBUGS(LOG_MESH) << "Mesh upload operation completed." << LL_ENDL; } } - - do - { - mCurlRequest->process(); - //sleep for 10ms to prevent eating a whole core - apr_sleep(10000); - } while (!LLAppViewer::isQuitting() && mPendingUploads > 0); } - - delete mCurlRequest; - mCurlRequest = NULL; - - // Currently a no-op. - mFinished = true; } void LLMeshUploadThread::requestWholeModelFee() { dump_num++; - mCurlRequest = new LLCurlRequest(); - generateHulls(); - LLSD model_data; - wholeModelToLLSD(model_data,false); - dump_llsd_to_file(model_data,make_dump_name("whole_model_fee_request_",dump_num)); + mModelData = LLSD::emptyMap(); + wholeModelToLLSD(mModelData, false); + dump_llsd_to_file(mModelData, make_dump_name("whole_model_fee_request_", dump_num)); + LLCore::HttpHandle handle = LLCoreHttpUtil::requestPostWithLLSD(mHttpRequest, + mHttpPolicyClass, + mHttpPriority, + mWholeModelFeeCapability, + mModelData, + mHttpOptions, + mHttpHeaders, + this); + if (LLCORE_HTTP_HANDLE_INVALID == handle) + { + mHttpStatus = mHttpRequest->getStatus(); + + LL_WARNS(LOG_MESH) << "Couldn't issue request for model fee. Reason: " << mHttpStatus.toString() + << " (" << mHttpStatus.toTerseString() << ")" + << LL_ENDL; + } + else + { + U32 sleep_time(10); - LLCurlRequest::headers_t headers; + mHttpRequest->update(0); + while (! LLApp::isQuitting() && ! finished() && ! isDiscarded()) + { + ms_sleep(sleep_time); + sleep_time = llmin(250U, sleep_time + sleep_time); + mHttpRequest->update(0); + } + if (isDiscarded()) + { + LL_DEBUGS(LOG_MESH) << "Mesh fee query operation discarded." << LL_ENDL; + } + } +} + + +// Does completion duty for both fee queries and actual uploads. +void LLMeshUploadThread::onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * response) +{ + // QA/Devel: 0x2 to enable fake error import on upload, 0x1 on fee check + const S32 fake_error(gSavedSettings.getS32("MeshUploadFakeErrors") & (mDoUpload ? 0xa : 0x5)); + LLCore::HttpStatus status(response->getStatus()); + if (fake_error) + { + status = (fake_error & 0x0c) ? LLCore::HttpStatus(500) : LLCore::HttpStatus(200); + } + std::string reason(status.toString()); + LLSD body; + mFinished = true; + + if (mDoUpload) { - LLCurl::ResponderPtr responder = new LLWholeModelFeeResponder(this,model_data, mFeeObserverHandle) ; - while(!mCurlRequest->post(mWholeModelFeeCapability, headers, model_data, responder, mMeshUploadTimeOut)) + // model upload case + LLWholeModelUploadObserver * observer(mUploadObserverHandle.get()); + + if (! status) { - //sleep for 10ms to prevent eating a whole core - apr_sleep(10000); + LL_WARNS(LOG_MESH) << "Upload failed. Reason: " << reason + << " (" << status.toTerseString() << ")" + << LL_ENDL; + + // Build a fake body for the alert generator + body["error"] = LLSD::emptyMap(); + body["error"]["message"] = reason; + body["error"]["identifier"] = "NetworkError"; // from asset-upload/upload_util.py + log_upload_error(status, body, "upload", mModelData["name"].asString()); + + if (observer) + { + doOnIdleOneTime(boost::bind(&LLWholeModelUploadObserver::onModelUploadFailure, observer)); + } } - } + else + { + if (fake_error & 0x2) + { + body = llsd_from_file("fake_upload_error.xml"); + } + else + { + // *TODO: handle error in conversion process + LLCoreHttpUtil::responseToLLSD(response, true, body); + } + dump_llsd_to_file(body, make_dump_name("whole_model_upload_response_", dump_num)); + + if (body["state"].asString() == "complete") + { + // requested "mesh" asset type isn't actually the type + // of the resultant object, fix it up here. + mModelData["asset_type"] = "object"; + gMeshRepo.updateInventory(LLMeshRepository::inventory_data(mModelData, body)); - do + if (observer) + { + doOnIdleOneTime(boost::bind(&LLWholeModelUploadObserver::onModelUploadSuccess, observer)); + } + } + else + { + LL_WARNS(LOG_MESH) << "Upload failed. Not in expected 'complete' state." << LL_ENDL; + log_upload_error(status, body, "upload", mModelData["name"].asString()); + + if (observer) + { + doOnIdleOneTime(boost::bind(&LLWholeModelUploadObserver::onModelUploadFailure, observer)); + } + } + } + } + else { - mCurlRequest->process(); - //sleep for 10ms to prevent eating a whole core - apr_sleep(10000); - } while (!LLApp::isQuitting() && mPendingUploads > 0); + // model fee case + LLWholeModelFeeObserver* observer(mFeeObserverHandle.get()); + mWholeModelUploadURL.clear(); - delete mCurlRequest; - mCurlRequest = NULL; + if (! status) + { + LL_WARNS(LOG_MESH) << "Fee request failed. Reason: " << reason + << " (" << status.toTerseString() << ")" + << LL_ENDL; - // Currently a no-op. - mFinished = true; + // Build a fake body for the alert generator + body["error"] = LLSD::emptyMap(); + body["error"]["message"] = reason; + body["error"]["identifier"] = "NetworkError"; // from asset-upload/upload_util.py + log_upload_error(status, body, "fee", mModelData["name"].asString()); + + if (observer) + { + observer->setModelPhysicsFeeErrorStatus(status.toULong(), reason); + } + } + else + { + if (fake_error & 0x1) + { + body = llsd_from_file("fake_upload_error.xml"); + } + else + { + // *TODO: handle error in conversion process + LLCoreHttpUtil::responseToLLSD(response, true, body); + } + dump_llsd_to_file(body, make_dump_name("whole_model_fee_response_", dump_num)); + + if (body["state"].asString() == "upload") + { + mWholeModelUploadURL = body["uploader"].asString(); + + if (observer) + { + body["data"]["upload_price"] = body["upload_price"]; + observer->onModelPhysicsFeeReceived(body["data"], mWholeModelUploadURL); + } + } + else + { + LL_WARNS(LOG_MESH) << "Fee request failed. Not in expected 'upload' state." << LL_ENDL; + log_upload_error(status, body, "fee", mModelData["name"].asString()); + + if (observer) + { + observer->setModelPhysicsFeeErrorStatus(status.toULong(), reason); + } + } + } + } } + void LLMeshRepoThread::notifyLoadedMeshes() { + bool update_metrics(false); + if (!mMutex) { return; @@ -1784,10 +2473,16 @@ void LLMeshRepoThread::notifyLoadedMeshes() while (!mLoadedQ.empty()) { mMutex->lock(); + if (mLoadedQ.empty()) + { + mMutex->unlock(); + break; + } LoadedMesh mesh = mLoadedQ.front(); mLoadedQ.pop(); mMutex->unlock(); + update_metrics = true; if (mesh.mVolume && mesh.mVolume->getNumVolumeFaces() > 0) { gMeshRepo.notifyMeshLoaded(mesh.mMeshParams, mesh.mVolume); @@ -1802,24 +2497,59 @@ void LLMeshRepoThread::notifyLoadedMeshes() while (!mUnavailableQ.empty()) { mMutex->lock(); + if (mUnavailableQ.empty()) + { + mMutex->unlock(); + break; + } + LODRequest req = mUnavailableQ.front(); mUnavailableQ.pop(); mMutex->unlock(); + update_metrics = true; gMeshRepo.notifyMeshUnavailable(req.mMeshParams, req.mLOD); } - while (!mSkinInfoQ.empty()) + if (! mSkinInfoQ.empty() || ! mDecompositionQ.empty()) { - gMeshRepo.notifySkinInfoReceived(mSkinInfoQ.front()); - mSkinInfoQ.pop(); + if (mMutex->trylock()) + { + std::list<LLMeshSkinInfo> skin_info_q; + std::list<LLModel::Decomposition*> decomp_q; + + if (! mSkinInfoQ.empty()) + { + skin_info_q.swap(mSkinInfoQ); + } + if (! mDecompositionQ.empty()) + { + decomp_q.swap(mDecompositionQ); + } + + mMutex->unlock(); + + // Process the elements free of the lock + while (! skin_info_q.empty()) + { + gMeshRepo.notifySkinInfoReceived(skin_info_q.front()); + skin_info_q.pop_front(); + } + + while (! decomp_q.empty()) + { + gMeshRepo.notifyDecompositionReceived(decomp_q.front()); + decomp_q.pop_front(); + } + } } - while (!mDecompositionQ.empty()) + if (update_metrics) { - gMeshRepo.notifyDecompositionReceived(mDecompositionQ.front()); - mDecompositionQ.pop(); + // Ping time-to-load metrics for mesh download operations. + LLMeshRepository::metricsProgress(0); } + } S32 LLMeshRepoThread::getActualMeshLOD(const LLVolumeParams& mesh_params, S32 lod) @@ -1898,243 +2628,305 @@ void LLMeshRepository::cacheOutgoingMesh(LLMeshUploadData& data, LLSD& header) } -void LLMeshLODResponder::completedRaw(U32 status, const std::string& reason, - const LLChannelDescriptors& channels, - const LLIOPipe::buffer_ptr_t& buffer) +// Handle failed or successful requests for mesh assets. +// +// Support for 200 responses was added for several reasons. One, +// a service or cache can ignore range headers and give us a +// 200 with full asset should it elect to. We also support +// a debug flag which disables range requests for those very +// few users that have some sort of problem with their networking +// services. But the 200 response handling is suboptimal: rather +// than cache the whole asset, we just extract the part that would +// have been sent in a 206 and process that. Inefficient but these +// are cases far off the norm. +void LLMeshHandlerBase::onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * response) { mProcessed = true; - // thread could have already be destroyed during logout - if( !gMeshRepo.mThread ) - { - return; - } - - S32 data_size = buffer->countAfter(channels.in(), NULL); + unsigned int retries(0U); + response->getRetries(NULL, &retries); + LLMeshRepository::sHTTPRetryCount += retries; - if (status < 200 || status > 400) + LLCore::HttpStatus status(response->getStatus()); + if (! status || MESH_HTTP_RESPONSE_FAILED) { - llwarns << status << ": " << reason << llendl; + processFailure(status); + ++LLMeshRepository::sHTTPErrorCount; } - - if (data_size < mRequestedBytes) + else { - if (status == 499 || status == 503) - { //timeout or service unavailable, try again - llwarns << "Timeout or service unavailable, retrying." << llendl; - LLMeshRepository::sHTTPRetryCount++; - gMeshRepo.mThread->loadMeshLOD(mMeshParams, mLOD); - } - else + // From texture fetch code and may apply here: + // + // A warning about partial (HTTP 206) data. Some grid services + // do *not* return a 'Content-Range' header in the response to + // Range requests with a 206 status. We're forced to assume + // we get what we asked for in these cases until we can fix + // the services. + // + // May also need to deal with 200 status (full asset returned + // rather than partial) and 416 (request completely unsatisfyable). + // Always been exposed to these but are less likely here where + // speculative loads aren't done. + LLCore::BufferArray * body(response->getBody()); + S32 body_offset(0); + U8 * data(NULL); + S32 data_size(body ? body->size() : 0); + + if (data_size > 0) { - llassert(status == 499 || status == 503); //intentionally trigger a breakpoint - llwarns << "Unhandled status " << status << llendl; - } - return; - } + static const LLCore::HttpStatus par_status(HTTP_PARTIAL_CONTENT); + + unsigned int offset(0), length(0), full_length(0); + + if (par_status == status) + { + // 206 case + response->getRange(&offset, &length, &full_length); + if (! offset && ! length) + { + // This is the case where we receive a 206 status but + // there wasn't a useful Content-Range header in the response. + // This could be because it was badly formatted but is more + // likely due to capabilities services which scrub headers + // from responses. Assume we got what we asked for...` + // length = data_size; + offset = mOffset; + } + } + else + { + // 200 case, typically + offset = 0; + } - LLMeshRepository::sBytesReceived += mRequestedBytes; + // *DEBUG: To test validation below + // offset += 1; - U8* data = NULL; + // Validate that what we think we received is consistent with + // what we've asked for. I.e. first byte we wanted lies somewhere + // in the response. + if (offset > mOffset + || (offset + data_size) <= mOffset + || (mOffset - offset) >= data_size) + { + // No overlap with requested range. Fail request with + // suitable error. Shouldn't happen unless server/cache/ISP + // is doing something awful. + LL_WARNS(LOG_MESH) << "Mesh response (bytes [" + << offset << ".." << (offset + length - 1) + << "]) didn't overlap with request's origin (bytes [" + << mOffset << ".." << (mOffset + mRequestedBytes - 1) + << "])." << LL_ENDL; + processFailure(LLCore::HttpStatus(LLCore::HttpStatus::LLCORE, LLCore::HE_INV_CONTENT_RANGE_HDR)); + ++LLMeshRepository::sHTTPErrorCount; + goto common_exit; + } + + // *TODO: Try to get rid of data copying and add interfaces + // that support BufferArray directly. Introduce a two-phase + // handler, optional first that takes a body, fallback second + // that requires a temporary allocation and data copy. + body_offset = mOffset - offset; + data = new U8[data_size - body_offset]; + body->read(body_offset, (char *) data, data_size - body_offset); + LLMeshRepository::sBytesReceived += data_size; + } - if (data_size > 0) - { - data = new U8[data_size]; - buffer->readAfter(channels.in(), NULL, data, data_size); + processData(body, body_offset, data, data_size - body_offset); + + delete [] data; } - if (gMeshRepo.mThread->lodReceived(mMeshParams, mLOD, data, data_size)) - { - //good fetch from sim, write to VFS for caching - LLVFile file(gVFS, mMeshParams.getSculptID(), LLAssetType::AT_MESH, LLVFile::WRITE); + // Release handler +common_exit: + gMeshRepo.mThread->mHttpRequestSet.erase(this); + delete this; // Must be last statement +} - S32 offset = mOffset; - S32 size = mRequestedBytes; - if (file.getSize() >= offset+size) +LLMeshHeaderHandler::~LLMeshHeaderHandler() +{ + if (!LLApp::isQuitting()) + { + if (! mProcessed) { - file.seek(offset); - file.write(data, size); - LLMeshRepository::sCacheBytesWritten += size; + // something went wrong, retry + LL_WARNS(LOG_MESH) << "Mesh header fetch canceled unexpectedly, retrying." << LL_ENDL; + LLMeshRepoThread::HeaderRequest req(mMeshParams); + LLMutexLock lock(gMeshRepo.mThread->mMutex); + gMeshRepo.mThread->mHeaderReqQ.push(req); } + LLMeshRepoThread::decActiveHeaderRequests(); } - - delete [] data; } -void LLMeshSkinInfoResponder::completedRaw(U32 status, const std::string& reason, - const LLChannelDescriptors& channels, - const LLIOPipe::buffer_ptr_t& buffer) +void LLMeshHeaderHandler::processFailure(LLCore::HttpStatus status) { - mProcessed = true; + LL_WARNS(LOG_MESH) << "Error during mesh header handling. ID: " << mMeshParams.getSculptID() + << ", Reason: " << status.toString() + << " (" << status.toTerseString() << "). Not retrying." + << LL_ENDL; - // thread could have already be destroyed during logout - if( !gMeshRepo.mThread ) + // Can't get the header so none of the LODs will be available + LLMutexLock lock(gMeshRepo.mThread->mMutex); + for (int i(0); i < 4; ++i) { - return; + gMeshRepo.mThread->mUnavailableQ.push(LLMeshRepoThread::LODRequest(mMeshParams, i)); } +} - S32 data_size = buffer->countAfter(channels.in(), NULL); - - if (status < 200 || status > 400) +void LLMeshHeaderHandler::processData(LLCore::BufferArray * /* body */, S32 /* body_offset */, + U8 * data, S32 data_size) +{ + LLUUID mesh_id = mMeshParams.getSculptID(); + bool success = (! MESH_HEADER_PROCESS_FAILED) && gMeshRepo.mThread->headerReceived(mMeshParams, data, data_size); + llassert(success); + if (! success) { - llwarns << status << ": " << reason << llendl; - } + // *TODO: Get real reason for parse failure here. Might we want to retry? + LL_WARNS(LOG_MESH) << "Unable to parse mesh header. ID: " << mesh_id + << ", Unknown reason. Not retrying." + << LL_ENDL; - if (data_size < mRequestedBytes) - { - if (status == 499 || status == 503) - { //timeout or service unavailable, try again - llwarns << "Timeout or service unavailable, retrying." << llendl; - LLMeshRepository::sHTTPRetryCount++; - gMeshRepo.mThread->loadMeshSkinInfo(mMeshID); - } - else + // Can't get the header so none of the LODs will be available + LLMutexLock lock(gMeshRepo.mThread->mMutex); + for (int i(0); i < 4; ++i) { - llassert(status == 499 || status == 503); //intentionally trigger a breakpoint - llwarns << "Unhandled status " << status << llendl; + gMeshRepo.mThread->mUnavailableQ.push(LLMeshRepoThread::LODRequest(mMeshParams, i)); } - return; } + else if (data && data_size > 0) + { + // header was successfully retrieved from sim, cache in vfs + LLSD header = gMeshRepo.mThread->mMeshHeader[mesh_id]; - LLMeshRepository::sBytesReceived += mRequestedBytes; + S32 version = header["version"].asInteger(); - U8* data = NULL; + if (version <= MAX_MESH_VERSION) + { + std::stringstream str; - if (data_size > 0) - { - data = new U8[data_size]; - buffer->readAfter(channels.in(), NULL, data, data_size); - } + S32 lod_bytes = 0; - if (gMeshRepo.mThread->skinInfoReceived(mMeshID, data, data_size)) - { - //good fetch from sim, write to VFS for caching - LLVFile file(gVFS, mMeshID, LLAssetType::AT_MESH, LLVFile::WRITE); + for (U32 i = 0; i < LLModel::LOD_PHYSICS; ++i) + { + // figure out how many bytes we'll need to reserve in the file + const std::string & lod_name = header_lod[i]; + lod_bytes = llmax(lod_bytes, header[lod_name]["offset"].asInteger()+header[lod_name]["size"].asInteger()); + } - S32 offset = mOffset; - S32 size = mRequestedBytes; + // just in case skin info or decomposition is at the end of the file (which it shouldn't be) + lod_bytes = llmax(lod_bytes, header["skin"]["offset"].asInteger() + header["skin"]["size"].asInteger()); + lod_bytes = llmax(lod_bytes, header["physics_convex"]["offset"].asInteger() + header["physics_convex"]["size"].asInteger()); - if (file.getSize() >= offset+size) - { - LLMeshRepository::sCacheBytesWritten += size; - file.seek(offset); - file.write(data, size); - } - } + S32 header_bytes = (S32) gMeshRepo.mThread->mMeshHeaderSize[mesh_id]; + S32 bytes = lod_bytes + header_bytes; - delete [] data; -} -void LLMeshDecompositionResponder::completedRaw(U32 status, const std::string& reason, - const LLChannelDescriptors& channels, - const LLIOPipe::buffer_ptr_t& buffer) -{ - mProcessed = true; - - if( !gMeshRepo.mThread ) - { - return; - } + // It's possible for the remote asset to have more data than is needed for the local cache + // only allocate as much space in the VFS as is needed for the local cache + data_size = llmin(data_size, bytes); - S32 data_size = buffer->countAfter(channels.in(), NULL); + LLVFile file(gVFS, mesh_id, LLAssetType::AT_MESH, LLVFile::WRITE); + if (file.getMaxSize() >= bytes || file.setMaxSize(bytes)) + { + LLMeshRepository::sCacheBytesWritten += data_size; + ++LLMeshRepository::sCacheWrites; - if (status < 200 || status > 400) - { - llwarns << status << ": " << reason << llendl; + file.write(data, data_size); + + // zero out the rest of the file + U8 block[MESH_HEADER_SIZE]; + memset(block, 0, sizeof(block)); + + while (bytes-file.tell() > sizeof(block)) + { + file.write(block, sizeof(block)); + } + + S32 remaining = bytes-file.tell(); + if (remaining > 0) + { + file.write(block, remaining); + } + } + } } +} - if (data_size < mRequestedBytes) +LLMeshLODHandler::~LLMeshLODHandler() +{ + if (! LLApp::isQuitting()) { - if (status == 499 || status == 503) - { //timeout or service unavailable, try again - llwarns << "Timeout or service unavailable, retrying." << llendl; - LLMeshRepository::sHTTPRetryCount++; - gMeshRepo.mThread->loadMeshDecomposition(mMeshID); - } - else + if (! mProcessed) { - llassert(status == 499 || status == 503); //intentionally trigger a breakpoint - llwarns << "Unhandled status " << status << llendl; + LL_WARNS(LOG_MESH) << "Mesh LOD fetch canceled unexpectedly, retrying." << LL_ENDL; + gMeshRepo.mThread->lockAndLoadMeshLOD(mMeshParams, mLOD); + } + LLMeshRepoThread::decActiveLODRequests(); } - return; } - LLMeshRepository::sBytesReceived += mRequestedBytes; - - U8* data = NULL; +void LLMeshLODHandler::processFailure(LLCore::HttpStatus status) +{ + LL_WARNS(LOG_MESH) << "Error during mesh LOD handling. ID: " << mMeshParams.getSculptID() + << ", Reason: " << status.toString() + << " (" << status.toTerseString() << "). Not retrying." + << LL_ENDL; - if (data_size > 0) - { - data = new U8[data_size]; - buffer->readAfter(channels.in(), NULL, data, data_size); - } + LLMutexLock lock(gMeshRepo.mThread->mMutex); + gMeshRepo.mThread->mUnavailableQ.push(LLMeshRepoThread::LODRequest(mMeshParams, mLOD)); +} - if (gMeshRepo.mThread->decompositionReceived(mMeshID, data, data_size)) +void LLMeshLODHandler::processData(LLCore::BufferArray * /* body */, S32 /* body_offset */, + U8 * data, S32 data_size) +{ + if ((! MESH_LOD_PROCESS_FAILED) && gMeshRepo.mThread->lodReceived(mMeshParams, mLOD, data, data_size)) { //good fetch from sim, write to VFS for caching - LLVFile file(gVFS, mMeshID, LLAssetType::AT_MESH, LLVFile::WRITE); + LLVFile file(gVFS, mMeshParams.getSculptID(), LLAssetType::AT_MESH, LLVFile::WRITE); S32 offset = mOffset; S32 size = mRequestedBytes; if (file.getSize() >= offset+size) { - LLMeshRepository::sCacheBytesWritten += size; file.seek(offset); file.write(data, size); + LLMeshRepository::sCacheBytesWritten += size; + ++LLMeshRepository::sCacheWrites; } } - - delete [] data; -} - -void LLMeshPhysicsShapeResponder::completedRaw(U32 status, const std::string& reason, - const LLChannelDescriptors& channels, - const LLIOPipe::buffer_ptr_t& buffer) -{ - mProcessed = true; - - // thread could have already be destroyed during logout - if( !gMeshRepo.mThread ) - { - return; - } - - S32 data_size = buffer->countAfter(channels.in(), NULL); - - if (status < 200 || status > 400) - { - llwarns << status << ": " << reason << llendl; - } - - if (data_size < mRequestedBytes) + else { - if (status == 499 || status == 503) - { //timeout or service unavailable, try again - llwarns << "Timeout or service unavailable, retrying." << llendl; - LLMeshRepository::sHTTPRetryCount++; - gMeshRepo.mThread->loadMeshPhysicsShape(mMeshID); - } - else - { - llassert(status == 499 || status == 503); //intentionally trigger a breakpoint - llwarns << "Unhandled status " << status << llendl; - } - return; + LL_WARNS(LOG_MESH) << "Error during mesh LOD processing. ID: " << mMeshParams.getSculptID() + << ", Unknown reason. Not retrying." + << LL_ENDL; + LLMutexLock lock(gMeshRepo.mThread->mMutex); + gMeshRepo.mThread->mUnavailableQ.push(LLMeshRepoThread::LODRequest(mMeshParams, mLOD)); } +} - LLMeshRepository::sBytesReceived += mRequestedBytes; +LLMeshSkinInfoHandler::~LLMeshSkinInfoHandler() +{ + llassert(mProcessed); +} - U8* data = NULL; +void LLMeshSkinInfoHandler::processFailure(LLCore::HttpStatus status) +{ + LL_WARNS(LOG_MESH) << "Error during mesh skin info handling. ID: " << mMeshID + << ", Reason: " << status.toString() + << " (" << status.toTerseString() << "). Not retrying." + << LL_ENDL; - if (data_size > 0) - { - data = new U8[data_size]; - buffer->readAfter(channels.in(), NULL, data, data_size); - } + // *TODO: Mark mesh unavailable on error. For now, simply leave + // request unfulfilled rather than retry forever. +} - if (gMeshRepo.mThread->physicsShapeReceived(mMeshID, data, data_size)) +void LLMeshSkinInfoHandler::processData(LLCore::BufferArray * /* body */, S32 /* body_offset */, + U8 * data, S32 data_size) +{ + if ((! MESH_SKIN_INFO_PROCESS_FAILED) && gMeshRepo.mThread->skinInfoReceived(mMeshID, data, data_size)) { //good fetch from sim, write to VFS for caching LLVFile file(gVFS, mMeshID, LLAssetType::AT_MESH, LLVFile::WRITE); @@ -2145,145 +2937,110 @@ void LLMeshPhysicsShapeResponder::completedRaw(U32 status, const std::string& re if (file.getSize() >= offset+size) { LLMeshRepository::sCacheBytesWritten += size; + ++LLMeshRepository::sCacheWrites; file.seek(offset); file.write(data, size); } } - - delete [] data; + else + { + LL_WARNS(LOG_MESH) << "Error during mesh skin info processing. ID: " << mMeshID + << ", Unknown reason. Not retrying." + << LL_ENDL; + // *TODO: Mark mesh unavailable on error + } } -void LLMeshHeaderResponder::completedRaw(U32 status, const std::string& reason, - const LLChannelDescriptors& channels, - const LLIOPipe::buffer_ptr_t& buffer) +LLMeshDecompositionHandler::~LLMeshDecompositionHandler() { - mProcessed = true; + llassert(mProcessed); +} - // thread could have already be destroyed during logout - if( !gMeshRepo.mThread ) - { - return; - } +void LLMeshDecompositionHandler::processFailure(LLCore::HttpStatus status) +{ + LL_WARNS(LOG_MESH) << "Error during mesh decomposition handling. ID: " << mMeshID + << ", Reason: " << status.toString() + << " (" << status.toTerseString() << "). Not retrying." + << LL_ENDL; + // *TODO: Mark mesh unavailable on error. For now, simply leave + // request unfulfilled rather than retry forever. +} - if (status < 200 || status > 400) +void LLMeshDecompositionHandler::processData(LLCore::BufferArray * /* body */, S32 /* body_offset */, + U8 * data, S32 data_size) +{ + if ((! MESH_DECOMP_PROCESS_FAILED) && gMeshRepo.mThread->decompositionReceived(mMeshID, data, data_size)) { - //llwarns - // << "Header responder failed with status: " - // << status << ": " << reason << llendl; - - // 503 (service unavailable) or 499 (timeout) - // can be due to server load and can be retried - - // TODO*: Add maximum retry logic, exponential backoff - // and (somewhat more optional than the others) retries - // again after some set period of time - - llassert(status == 503 || status == 499); + // good fetch from sim, write to VFS for caching + LLVFile file(gVFS, mMeshID, LLAssetType::AT_MESH, LLVFile::WRITE); - if (status == 503 || status == 499) - { //retry - llwarns << "Timeout or service unavailable, retrying." << llendl; - LLMeshRepository::sHTTPRetryCount++; - LLMeshRepoThread::HeaderRequest req(mMeshParams); - LLMutexLock lock(gMeshRepo.mThread->mMutex); - gMeshRepo.mThread->mHeaderReqQ.push(req); + S32 offset = mOffset; + S32 size = mRequestedBytes; - return; - } - else + if (file.getSize() >= offset+size) { - llwarns << "Unhandled status." << llendl; + LLMeshRepository::sCacheBytesWritten += size; + ++LLMeshRepository::sCacheWrites; + file.seek(offset); + file.write(data, size); } } - - S32 data_size = buffer->countAfter(channels.in(), NULL); - - U8* data = NULL; - - if (data_size > 0) + else { - data = new U8[data_size]; - buffer->readAfter(channels.in(), NULL, data, data_size); + LL_WARNS(LOG_MESH) << "Error during mesh decomposition processing. ID: " << mMeshID + << ", Unknown reason. Not retrying." + << LL_ENDL; + // *TODO: Mark mesh unavailable on error } +} - LLMeshRepository::sBytesReceived += llmin(data_size, 4096); +LLMeshPhysicsShapeHandler::~LLMeshPhysicsShapeHandler() +{ + llassert(mProcessed); +} - bool success = gMeshRepo.mThread->headerReceived(mMeshParams, data, data_size); - - llassert(success); +void LLMeshPhysicsShapeHandler::processFailure(LLCore::HttpStatus status) +{ + LL_WARNS(LOG_MESH) << "Error during mesh physics shape handling. ID: " << mMeshID + << ", Reason: " << status.toString() + << " (" << status.toTerseString() << "). Not retrying." + << LL_ENDL; + // *TODO: Mark mesh unavailable on error +} - if (!success) - { - llwarns - << "Unable to parse mesh header: " - << status << ": " << reason << llendl; - } - else if (data && data_size > 0) +void LLMeshPhysicsShapeHandler::processData(LLCore::BufferArray * /* body */, S32 /* body_offset */, + U8 * data, S32 data_size) +{ + if ((! MESH_PHYS_SHAPE_PROCESS_FAILED) && gMeshRepo.mThread->physicsShapeReceived(mMeshID, data, data_size)) { - //header was successfully retrieved from sim, cache in vfs - LLUUID mesh_id = mMeshParams.getSculptID(); - LLSD header = gMeshRepo.mThread->mMeshHeader[mesh_id]; + // good fetch from sim, write to VFS for caching + LLVFile file(gVFS, mMeshID, LLAssetType::AT_MESH, LLVFile::WRITE); - S32 version = header["version"].asInteger(); + S32 offset = mOffset; + S32 size = mRequestedBytes; - if (version <= MAX_MESH_VERSION) + if (file.getSize() >= offset+size) { - std::stringstream str; - - S32 lod_bytes = 0; - - for (U32 i = 0; i < LLModel::LOD_PHYSICS; ++i) - { //figure out how many bytes we'll need to reserve in the file - std::string lod_name = header_lod[i]; - lod_bytes = llmax(lod_bytes, header[lod_name]["offset"].asInteger()+header[lod_name]["size"].asInteger()); - } - - //just in case skin info or decomposition is at the end of the file (which it shouldn't be) - lod_bytes = llmax(lod_bytes, header["skin"]["offset"].asInteger() + header["skin"]["size"].asInteger()); - lod_bytes = llmax(lod_bytes, header["physics_convex"]["offset"].asInteger() + header["physics_convex"]["size"].asInteger()); - - S32 header_bytes = (S32) gMeshRepo.mThread->mMeshHeaderSize[mesh_id]; - S32 bytes = lod_bytes + header_bytes; - - - //it's possible for the remote asset to have more data than is needed for the local cache - //only allocate as much space in the VFS as is needed for the local cache - data_size = llmin(data_size, bytes); - - LLVFile file(gVFS, mesh_id, LLAssetType::AT_MESH, LLVFile::WRITE); - if (file.getMaxSize() >= bytes || file.setMaxSize(bytes)) - { - LLMeshRepository::sCacheBytesWritten += data_size; - - file.write((const U8*) data, data_size); - - //zero out the rest of the file - U8 block[4096]; - memset(block, 0, 4096); - - while (bytes-file.tell() > 4096) - { - file.write(block, 4096); - } - - S32 remaining = bytes-file.tell(); - - if (remaining > 0) - { - file.write(block, remaining); - } - } + LLMeshRepository::sCacheBytesWritten += size; + ++LLMeshRepository::sCacheWrites; + file.seek(offset); + file.write(data, size); } } - - delete [] data; + else + { + LL_WARNS(LOG_MESH) << "Error during mesh physics shape processing. ID: " << mMeshID + << ", Unknown reason. Not retrying." + << LL_ENDL; + // *TODO: Mark mesh unavailable on error + } } - LLMeshRepository::LLMeshRepository() : mMeshMutex(NULL), mMeshThreadCount(0), - mThread(NULL) + mThread(NULL), + mGetMeshVersion(2) { } @@ -2302,7 +3059,7 @@ void LLMeshRepository::init() apr_sleep(100); } - + metrics_teleport_started_signal = LLViewerMessage::getInstance()->setTeleportStartedCallback(teleport_started); mThread = new LLMeshRepoThread(); mThread->start(); @@ -2310,11 +3067,13 @@ void LLMeshRepository::init() void LLMeshRepository::shutdown() { - llinfos << "Shutting down mesh repository." << llendl; + LL_INFOS(LOG_MESH) << "Shutting down mesh repository." << LL_ENDL; + + metrics_teleport_started_signal.disconnect(); for (U32 i = 0; i < mUploads.size(); ++i) { - llinfos << "Discard the pending mesh uploads " << llendl; + LL_INFOS(LOG_MESH) << "Discard the pending mesh uploads." << LL_ENDL; mUploads[i]->discard() ; //discard the uploading requests. } @@ -2329,7 +3088,7 @@ void LLMeshRepository::shutdown() for (U32 i = 0; i < mUploads.size(); ++i) { - llinfos << "Waiting for pending mesh upload " << i << "/" << mUploads.size() << llendl; + LL_INFOS(LOG_MESH) << "Waiting for pending mesh upload " << (i + 1) << "/" << mUploads.size() << LL_ENDL; while (!mUploads[i]->isStopped()) { apr_sleep(10); @@ -2342,7 +3101,7 @@ void LLMeshRepository::shutdown() delete mMeshMutex; mMeshMutex = NULL; - llinfos << "Shutting down decomposition system." << llendl; + LL_INFOS(LOG_MESH) << "Shutting down decomposition system." << LL_ENDL; if (mDecompThread) { @@ -2357,6 +3116,9 @@ void LLMeshRepository::shutdown() //called in the main thread. S32 LLMeshRepository::update() { + // Conditionally log a mesh metrics event + metricsUpdate(); + if(mUploadWaitList.empty()) { return 0 ; @@ -2376,7 +3138,12 @@ S32 LLMeshRepository::update() S32 LLMeshRepository::loadMesh(LLVOVolume* vobj, const LLVolumeParams& mesh_params, S32 detail, S32 last_lod) { - if (detail < 0 || detail > 4) + LL_RECORD_BLOCK_TIME(FTM_MESH_FETCH); + + // Manage time-to-load metrics for mesh download operations. + metricsProgress(1); + + if (detail < 0 || detail >= 4) { return detail; } @@ -2454,8 +3221,37 @@ S32 LLMeshRepository::loadMesh(LLVOVolume* vobj, const LLVolumeParams& mesh_para void LLMeshRepository::notifyLoadedMeshes() { //called from main thread + LL_RECORD_BLOCK_TIME(FTM_MESH_FETCH); + + if (1 == mGetMeshVersion) + { + // Legacy GetMesh operation with high connection concurrency + LLMeshRepoThread::sMaxConcurrentRequests = gSavedSettings.getU32("MeshMaxConcurrentRequests"); + LLMeshRepoThread::sRequestHighWater = llclamp(2 * S32(LLMeshRepoThread::sMaxConcurrentRequests), + REQUEST_HIGH_WATER_MIN, + REQUEST_HIGH_WATER_MAX); + LLMeshRepoThread::sRequestLowWater = llclamp(LLMeshRepoThread::sRequestHighWater / 2, + REQUEST_LOW_WATER_MIN, + REQUEST_LOW_WATER_MAX); + } + else + { + // GetMesh2 operation with keepalives, etc. With pipelining, + // we'll increase this. See llappcorehttp and llcorehttp for + // discussion on connection strategies. + LLAppCoreHttp & app_core_http(LLAppViewer::instance()->getAppCoreHttp()); + S32 scale(app_core_http.isPipelined(LLAppCoreHttp::AP_MESH2) + ? (2 * LLAppCoreHttp::PIPELINING_DEPTH) + : 5); - LLMeshRepoThread::sMaxConcurrentRequests = gSavedSettings.getU32("MeshMaxConcurrentRequests"); + LLMeshRepoThread::sMaxConcurrentRequests = gSavedSettings.getU32("Mesh2MaxConcurrentRequests"); + LLMeshRepoThread::sRequestHighWater = llclamp(scale * S32(LLMeshRepoThread::sMaxConcurrentRequests), + REQUEST2_HIGH_WATER_MIN, + REQUEST2_HIGH_WATER_MAX); + LLMeshRepoThread::sRequestLowWater = llclamp(LLMeshRepoThread::sRequestHighWater / 2, + REQUEST2_LOW_WATER_MIN, + REQUEST2_LOW_WATER_MAX); + } //clean up completed upload threads for (std::vector<LLMeshUploadThread*>::iterator iter = mUploads.begin(); iter != mUploads.end(); ) @@ -2534,26 +3330,46 @@ void LLMeshRepository::notifyLoadedMeshes() //call completed callbacks on finished decompositions mDecompThread->notifyCompleted(); - if (!mThread->mWaiting) - { //curl thread is churning, wait for it to go idle - return; - } + // For major operations, attempt to get the required locks + // without blocking and punt if they're not available. The + // longest run of holdoffs is kept in sMaxLockHoldoffs just + // to collect the data. In testing, I've never seen a value + // greater than 2 (written to log on exit). + { + LLMutexTrylock lock1(mMeshMutex); + LLMutexTrylock lock2(mThread->mMutex); - static std::string region_name("never name a region this"); + static U32 hold_offs(0); + if (! lock1.isLocked() || ! lock2.isLocked()) + { + // If we can't get the locks, skip and pick this up later. + ++hold_offs; + sMaxLockHoldoffs = llmax(sMaxLockHoldoffs, hold_offs); + return; + } + hold_offs = 0; - if (gAgent.getRegion()) - { //update capability url - if (gAgent.getRegion()->getName() != region_name && gAgent.getRegion()->capabilitiesReceived()) + if (gAgent.getRegion()) { - region_name = gAgent.getRegion()->getName(); - mGetMeshCapability = gAgent.getRegion()->getCapability("GetMesh"); + // Update capability urls + static std::string region_name("never name a region this"); + + if (gAgent.getRegion()->getName() != region_name && gAgent.getRegion()->capabilitiesReceived()) + { + region_name = gAgent.getRegion()->getName(); + const bool use_v1(gSavedSettings.getBOOL("MeshUseGetMesh1")); + const std::string mesh1(gAgent.getRegion()->getCapability("GetMesh")); + const std::string mesh2(gAgent.getRegion()->getCapability("GetMesh2")); + mGetMeshVersion = (mesh2.empty() || use_v1) ? 1 : 2; + mThread->setGetMeshCaps(mesh1, mesh2, mGetMeshVersion); + LL_DEBUGS(LOG_MESH) << "Retrieving caps for region '" << region_name + << "', GetMesh2: " << mesh2 + << ", GetMesh: " << mesh1 + << ", using version: " << mGetMeshVersion + << LL_ENDL; + } } - } - { - LLMutexLock lock1(mMeshMutex); - LLMutexLock lock2(mThread->mMutex); - //popup queued error messages from background threads while (!mUploadErrorQ.empty()) { @@ -2561,47 +3377,55 @@ void LLMeshRepository::notifyLoadedMeshes() mUploadErrorQ.pop(); } - S32 push_count = LLMeshRepoThread::sMaxConcurrentRequests-(LLMeshRepoThread::sActiveHeaderRequests+LLMeshRepoThread::sActiveLODRequests); - - if (push_count > 0) + S32 active_count = LLMeshRepoThread::sActiveHeaderRequests + LLMeshRepoThread::sActiveLODRequests; + if (active_count < LLMeshRepoThread::sRequestLowWater) { - //calculate "score" for pending requests + S32 push_count = LLMeshRepoThread::sRequestHighWater - active_count; - //create score map - std::map<LLUUID, F32> score_map; - - for (U32 i = 0; i < 4; ++i) + if (mPendingRequests.size() > push_count) { - for (mesh_load_map::iterator iter = mLoadingMeshes[i].begin(); iter != mLoadingMeshes[i].end(); ++iter) + // More requests than the high-water limit allows so + // sort and forward the most important. + + //calculate "score" for pending requests + + //create score map + std::map<LLUUID, F32> score_map; + + for (U32 i = 0; i < 4; ++i) { - F32 max_score = 0.f; - for (std::set<LLUUID>::iterator obj_iter = iter->second.begin(); obj_iter != iter->second.end(); ++obj_iter) + for (mesh_load_map::iterator iter = mLoadingMeshes[i].begin(); iter != mLoadingMeshes[i].end(); ++iter) { - LLViewerObject* object = gObjectList.findObject(*obj_iter); - - if (object) + F32 max_score = 0.f; + for (std::set<LLUUID>::iterator obj_iter = iter->second.begin(); obj_iter != iter->second.end(); ++obj_iter) { - LLDrawable* drawable = object->mDrawable; - if (drawable) + LLViewerObject* object = gObjectList.findObject(*obj_iter); + + if (object) { - F32 cur_score = drawable->getRadius()/llmax(drawable->mDistanceWRTCamera, 1.f); - max_score = llmax(max_score, cur_score); + LLDrawable* drawable = object->mDrawable; + if (drawable) + { + F32 cur_score = drawable->getRadius()/llmax(drawable->mDistanceWRTCamera, 1.f); + max_score = llmax(max_score, cur_score); + } } } - } - score_map[iter->first.getSculptID()] = max_score; + score_map[iter->first.getSculptID()] = max_score; + } } - } - //set "score" for pending requests - for (std::vector<LLMeshRepoThread::LODRequest>::iterator iter = mPendingRequests.begin(); iter != mPendingRequests.end(); ++iter) - { - iter->mScore = score_map[iter->mMeshParams.getSculptID()]; - } + //set "score" for pending requests + for (std::vector<LLMeshRepoThread::LODRequest>::iterator iter = mPendingRequests.begin(); iter != mPendingRequests.end(); ++iter) + { + iter->mScore = score_map[iter->mMeshParams.getSculptID()]; + } - //sort by "score" - std::sort(mPendingRequests.begin(), mPendingRequests.end(), LLMeshRepoThread::CompareScoreGreater()); + //sort by "score" + std::partial_sort(mPendingRequests.begin(), mPendingRequests.begin() + push_count, + mPendingRequests.end(), LLMeshRepoThread::CompareScoreGreater()); + } while (!mPendingRequests.empty() && push_count > 0) { @@ -2655,9 +3479,8 @@ void LLMeshRepository::notifySkinInfoReceived(LLMeshSkinInfo& info) vobj->notifyMeshLoaded(); } } + mLoadingSkins.erase(info.mMeshID); } - - mLoadingSkins.erase(info.mMeshID); } void LLMeshRepository::notifyDecompositionReceived(LLModel::Decomposition* decomp) @@ -2666,14 +3489,14 @@ void LLMeshRepository::notifyDecompositionReceived(LLModel::Decomposition* decom if (iter == mDecompositionMap.end()) { //just insert decomp into map mDecompositionMap[decomp->mMeshID] = decomp; + mLoadingDecompositions.erase(decomp->mMeshID); } else { //merge decomp with existing entry iter->second->merge(decomp); + mLoadingDecompositions.erase(decomp->mMeshID); delete decomp; } - - mLoadingDecompositions.erase(decomp->mMeshID); } void LLMeshRepository::notifyMeshLoaded(const LLVolumeParams& mesh_params, LLVolume* volume) @@ -2688,7 +3511,8 @@ void LLMeshRepository::notifyMeshLoaded(const LLVolumeParams& mesh_params, LLVol //make sure target volume is still valid if (volume->getNumVolumeFaces() <= 0) { - llwarns << "Mesh loading returned empty volume." << llendl; + LL_WARNS(LOG_MESH) << "Mesh loading returned empty volume. ID: " << mesh_params.getSculptID() + << LL_ENDL; } { //update system volume @@ -2701,7 +3525,8 @@ void LLMeshRepository::notifyMeshLoaded(const LLVolumeParams& mesh_params, LLVol } else { - llwarns << "Couldn't find system volume for given mesh." << llendl; + LL_WARNS(LOG_MESH) << "Couldn't find system volume for mesh " << mesh_params.getSculptID() + << LL_ENDL; } } @@ -2755,6 +3580,8 @@ S32 LLMeshRepository::getActualMeshLOD(const LLVolumeParams& mesh_params, S32 lo const LLMeshSkinInfo* LLMeshRepository::getSkinInfo(const LLUUID& mesh_id, const LLVOVolume* requesting_obj) { + LL_RECORD_BLOCK_TIME(FTM_MESH_FETCH); + if (mesh_id.notNull()) { skin_map::iterator iter = mSkinMap.find(mesh_id); @@ -2781,6 +3608,8 @@ const LLMeshSkinInfo* LLMeshRepository::getSkinInfo(const LLUUID& mesh_id, const void LLMeshRepository::fetchPhysicsShape(const LLUUID& mesh_id) { + LL_RECORD_BLOCK_TIME(FTM_MESH_FETCH); + if (mesh_id.notNull()) { LLModel::Decomposition* decomp = NULL; @@ -2798,16 +3627,18 @@ void LLMeshRepository::fetchPhysicsShape(const LLUUID& mesh_id) std::set<LLUUID>::iterator iter = mLoadingPhysicsShapes.find(mesh_id); if (iter == mLoadingPhysicsShapes.end()) { //no request pending for this skin info + // *FIXME: Nothing ever deletes entries, can't be right mLoadingPhysicsShapes.insert(mesh_id); mPendingPhysicsShapeRequests.push(mesh_id); } } } - } LLModel::Decomposition* LLMeshRepository::getDecomposition(const LLUUID& mesh_id) { + LL_RECORD_BLOCK_TIME(FTM_MESH_FETCH); + LLModel::Decomposition* ret = NULL; if (mesh_id.notNull()) @@ -2870,6 +3701,8 @@ bool LLMeshRepository::hasPhysicsShape(const LLUUID& mesh_id) LLSD& LLMeshRepository::getMeshHeader(const LLUUID& mesh_id) { + LL_RECORD_BLOCK_TIME(FTM_MESH_FETCH); + return mThread->getMeshHeader(mesh_id); } @@ -2920,7 +3753,6 @@ S32 LLMeshRepository::getMeshSize(const LLUUID& mesh_id, S32 lod) } return -1; - } void LLMeshUploadThread::decomposeMeshMatrix(LLMatrix4& transformation, @@ -3185,7 +4017,7 @@ void LLPhysicsDecomp::setMeshData(LLCDMeshData& mesh, bool vertex_based) if (ret) { - llerrs << "Convex Decomposition thread valid but could not set mesh data" << llendl; + LL_ERRS(LOG_MESH) << "Convex Decomposition thread valid but could not set mesh data." << LL_ENDL; } } } @@ -3222,6 +4054,7 @@ void LLPhysicsDecomp::doDecomposition() param_map[params[i].mName] = params+i; } + U32 ret = LLCD_OK; //set parameter values for (decomp_params::iterator iter = mCurRequest->mParams.begin(); iter != mCurRequest->mParams.end(); ++iter) { @@ -3235,7 +4068,6 @@ void LLPhysicsDecomp::doDecomposition() continue; } - U32 ret = LLCD_OK; if (param->mType == LLCDParam::LLCD_FLOAT) { @@ -3254,8 +4086,6 @@ void LLPhysicsDecomp::doDecomposition() mCurRequest->setStatusMessage("Executing."); - LLCDResult ret = LLCD_OK; - if (LLConvexDecomposition::getInstance() != NULL) { ret = LLConvexDecomposition::getInstance()->executeStage(stage); @@ -3263,7 +4093,8 @@ void LLPhysicsDecomp::doDecomposition() if (ret) { - llwarns << "Convex Decomposition thread valid but could not execute stage " << stage << llendl; + LL_WARNS(LOG_MESH) << "Convex Decomposition thread valid but could not execute stage " << stage << "." + << LL_ENDL; LLMutexLock lock(mMutex); mCurRequest->mHull.clear(); @@ -3394,7 +4225,7 @@ void LLPhysicsDecomp::doDecompositionSingleHull() LLCDResult ret = decomp->buildSingleHull() ; if(ret) { - llwarns << "Could not execute decomposition stage when attempting to create single hull." << llendl; + LL_WARNS(LOG_MESH) << "Could not execute decomposition stage when attempting to create single hull." << LL_ENDL; make_box(mCurRequest); } else @@ -3699,3 +4530,63 @@ bool LLMeshRepository::meshRezEnabled() } return false; } + +// Threading: main thread only +// static +void LLMeshRepository::metricsStart() +{ + ++metrics_teleport_start_count; + sQuiescentTimer.start(0); +} + +// Threading: main thread only +// static +void LLMeshRepository::metricsStop() +{ + sQuiescentTimer.stop(0); +} + +// Threading: main thread only +// static +void LLMeshRepository::metricsProgress(unsigned int this_count) +{ + static bool first_start(true); + + if (first_start) + { + metricsStart(); + first_start = false; + } + sQuiescentTimer.ringBell(0, this_count); +} + +// Threading: main thread only +// static +void LLMeshRepository::metricsUpdate() +{ + F64 started, stopped; + U64 total_count(U64L(0)), user_cpu(U64L(0)), sys_cpu(U64L(0)); + + if (sQuiescentTimer.isExpired(0, started, stopped, total_count, user_cpu, sys_cpu)) + { + LLSD metrics; + + metrics["reason"] = "Mesh Download Quiescent"; + metrics["scope"] = metrics_teleport_start_count > 1 ? "Teleport" : "Login"; + metrics["start"] = started; + metrics["stop"] = stopped; + metrics["fetches"] = LLSD::Integer(total_count); + metrics["teleports"] = LLSD::Integer(metrics_teleport_start_count); + metrics["user_cpu"] = double(user_cpu) / 1.0e6; + metrics["sys_cpu"] = double(sys_cpu) / 1.0e6; + LL_INFOS(LOG_MESH) << "EventMarker " << metrics << LL_ENDL; + } +} + +// Threading: main thread only +// static +void teleport_started() +{ + LLMeshRepository::metricsStart(); +} + |