diff options
Diffstat (limited to 'indra')
| -rw-r--r-- | indra/llcorehttp/_httpinternal.h | 6 | ||||
| -rw-r--r-- | indra/llcorehttp/httprequest.h | 15 | ||||
| -rw-r--r-- | indra/newview/llappcorehttp.cpp | 48 | ||||
| -rw-r--r-- | indra/newview/llappcorehttp.h | 16 | ||||
| -rwxr-xr-x | indra/newview/llmeshrepository.cpp | 854 | ||||
| -rw-r--r-- | indra/newview/llmeshrepository.h | 16 |
6 files changed, 929 insertions, 26 deletions
diff --git a/indra/llcorehttp/_httpinternal.h b/indra/llcorehttp/_httpinternal.h index 14f744a9f1..30b0905c12 100644 --- a/indra/llcorehttp/_httpinternal.h +++ b/indra/llcorehttp/_httpinternal.h @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -97,8 +97,8 @@ namespace LLCore { // Maxium number of policy classes that can be defined. -// *TODO: Currently limited to the default class, extend. -const int HTTP_POLICY_CLASS_LIMIT = 1; +// *TODO: Currently limited to the default class + 1, extend. +const int HTTP_POLICY_CLASS_LIMIT = 2; // Debug/informational tracing. Used both // as a global option and in per-request traces. diff --git a/indra/llcorehttp/httprequest.h b/indra/llcorehttp/httprequest.h index ab2f302d34..5000f47d0d 100644 --- a/indra/llcorehttp/httprequest.h +++ b/indra/llcorehttp/httprequest.h @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -158,10 +158,17 @@ public: /// Create a new policy class into which requests can be made. /// + /// All class creation must occur before threads are started and + /// transport begins. Policy classes are limited to a small value. + /// Currently that limit is the default class + 1. + /// /// @return If positive, the policy_id used to reference - /// the class in other methods. If 0, an error - /// occurred and @see getStatus() may provide more - /// detail on the reason. + /// the class in other methods. If 0, requests + /// for classes have exceeded internal limits + /// or caller has tried to create a class after + /// threads have been started. Caller must fallback + /// and recover. + /// static policy_t createPolicyClass(); enum EClassPolicy diff --git a/indra/newview/llappcorehttp.cpp b/indra/newview/llappcorehttp.cpp index 0d7d41304d..4386e3283b 100644 --- a/indra/newview/llappcorehttp.cpp +++ b/indra/newview/llappcorehttp.cpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -38,7 +38,9 @@ LLAppCoreHttp::LLAppCoreHttp() mStopHandle(LLCORE_HTTP_HANDLE_INVALID), mStopRequested(0.0), mStopped(false), - mPolicyDefault(-1) + mPolicyDefault(-1), + mPolicyTexture(-1), + mPolicyMesh(-1) {} @@ -95,6 +97,9 @@ void LLAppCoreHttp::init() // Setup default policy and constrain if directed to mPolicyDefault = LLCore::HttpRequest::DEFAULT_POLICY_ID; + + // Texture policy will use default for now. + mPolicyTexture = mPolicyDefault; static const std::string texture_concur("TextureFetchConcurrency"); if (gSavedSettings.controlExists(texture_concur)) { @@ -103,7 +108,7 @@ void LLAppCoreHttp::init() if (concur > 0) { LLCore::HttpStatus status; - status = LLCore::HttpRequest::setPolicyClassOption(mPolicyDefault, + status = LLCore::HttpRequest::setPolicyClassOption(mPolicyTexture, LLCore::HttpRequest::CP_CONNECTION_LIMIT, concur); if (! status) @@ -120,6 +125,43 @@ void LLAppCoreHttp::init() } } } + + // Create the mesh class + mPolicyMesh = LLCore::HttpRequest::createPolicyClass(); + if (! mPolicyMesh) + { + LL_WARNS("Init") << "Failed to create HTTP policy class for Mesh. Using default policy." + << LL_ENDL; + mPolicyMesh = mPolicyDefault; + } + else + { + static const std::string mesh_concur("MeshMaxConcurrentRequests"); + if (gSavedSettings.controlExists(mesh_concur)) + { + U32 setting(llmin(gSavedSettings.getU32(mesh_concur), U32(32))); + + if (setting > 0) + { + LLCore::HttpStatus status; + status = LLCore::HttpRequest::setPolicyClassOption(mPolicyMesh, + LLCore::HttpRequest::CP_CONNECTION_LIMIT, + setting); + if (! status) + { + LL_WARNS("Init") << "Unable to set mesh fetch concurrency. Reason: " + << status.toString() + << LL_ENDL; + } + else + { + LL_INFOS("Init") << "Application settings overriding default mesh fetch concurrency. New value: " + << setting + << LL_ENDL; + } + } + } + } // Kick the thread status = LLCore::HttpRequest::startThread(); diff --git a/indra/newview/llappcorehttp.h b/indra/newview/llappcorehttp.h index 241d73ad52..d90af9e5ca 100644 --- a/indra/newview/llappcorehttp.h +++ b/indra/newview/llappcorehttp.h @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -70,6 +70,18 @@ public: { return mPolicyDefault; } + + // Get the texture fetch policy class. + int getPolicyTexture() const + { + return mPolicyTexture; + } + + // Get the mesh fetch policy class. + int getPolicyMesh() const + { + return mPolicyMesh; + } private: static const F64 MAX_THREAD_WAIT_TIME; @@ -80,6 +92,8 @@ private: F64 mStopRequested; bool mStopped; int mPolicyDefault; + int mPolicyTexture; + int mPolicyMesh; }; diff --git a/indra/newview/llmeshrepository.cpp b/indra/newview/llmeshrepository.cpp index 0f33128057..21b7b120f6 100755 --- a/indra/newview/llmeshrepository.cpp +++ b/indra/newview/llmeshrepository.cpp @@ -66,6 +66,7 @@ #include "llfoldertype.h" #include "llviewerparcelmgr.h" #include "lluploadfloaterobservers.h" +#include "bufferarray.h" #include "boost/lexical_cast.hpp" @@ -77,6 +78,7 @@ LLMeshRepository gMeshRepo; +const S32 MESH_HEADER_SIZE = 4096; const U32 MAX_MESH_REQUESTS_PER_SECOND = 100; // Maximum mesh version to support. Three least significant digits are reserved for the minor version, @@ -124,6 +126,7 @@ static bool metrics_inited(false); static boost::signals2::connection metrics_teleport_connection; static unsigned int metrics_teleport_start_count(0); static void metrics_teleport_started(); +static bool is_retryable(LLCore::HttpStatus status); //get the number of bytes resident in memory for given volume U32 get_volume_memory_size(const LLVolume* volume) @@ -209,6 +212,160 @@ S32 LLMeshRepoThread::sActiveHeaderRequests = 0; S32 LLMeshRepoThread::sActiveLODRequests = 0; U32 LLMeshRepoThread::sMaxConcurrentRequests = 1; +class LLMeshHandlerBase : public LLCore::HttpHandler +{ +public: + LLMeshHandlerBase() + : LLCore::HttpHandler(), + mMeshParams(), + mProcessed(false) + {} + virtual ~LLMeshHandlerBase(); + +protected: + LLMeshHandlerBase(const LLMeshHandlerBase &); // Not defined + void operator=(const LLMeshHandlerBase &); // Not defined + +public: + virtual void onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * response); + virtual void processData(LLCore::BufferArray * body, U8 * data, S32 data_size) = 0; + virtual void processFailure(LLCore::HttpStatus status) = 0; + +public: + LLVolumeParams mMeshParams; + bool mProcessed; + LLCore::HttpHandle mHttpHandle; +}; + + +class LLMeshHeaderHandler : public LLMeshHandlerBase +{ +public: + LLMeshHeaderHandler(const LLVolumeParams & mesh_params) + : LLMeshHandlerBase() + { + mMeshParams = mesh_params; + LLMeshRepoThread::incActiveHeaderRequests(); + } + virtual ~LLMeshHeaderHandler(); + +protected: + LLMeshHeaderHandler(const LLMeshHeaderHandler &); // Not defined + void operator=(const LLMeshHeaderHandler &); // Not defined + +public: + virtual void processData(LLCore::BufferArray * body, U8 * data, S32 data_size); + virtual void processFailure(LLCore::HttpStatus status); +}; + + +class LLMeshLODHandler : public LLMeshHandlerBase +{ +public: + LLMeshLODHandler(const LLVolumeParams & mesh_params, S32 lod, U32 offset, U32 requested_bytes) + : LLMeshHandlerBase(), + mLOD(lod), + mRequestedBytes(requested_bytes), + mOffset(offset) + { + mMeshParams = mesh_params; + LLMeshRepoThread::incActiveLODRequests(); + } + virtual ~LLMeshLODHandler(); + +protected: + LLMeshLODHandler(const LLMeshLODHandler &); // Not defined + void operator=(const LLMeshLODHandler &); // Not defined + +public: + virtual void processData(LLCore::BufferArray * body, U8 * data, S32 data_size); + virtual void processFailure(LLCore::HttpStatus status); + +public: + S32 mLOD; + U32 mRequestedBytes; + U32 mOffset; +}; + + +class LLMeshSkinInfoHandler : public LLMeshHandlerBase +{ +public: + LLMeshSkinInfoHandler(const LLUUID& id, U32 offset, U32 size) + : LLMeshHandlerBase(), + mMeshID(id), + mRequestedBytes(size), + mOffset(offset) + {} + virtual ~LLMeshSkinInfoHandler(); + +protected: + LLMeshSkinInfoHandler(const LLMeshSkinInfoHandler &); // Not defined + void operator=(const LLMeshSkinInfoHandler &); // Not defined + +public: + virtual void processData(LLCore::BufferArray * body, U8 * data, S32 data_size); + virtual void processFailure(LLCore::HttpStatus status); + +public: + LLUUID mMeshID; + U32 mRequestedBytes; + U32 mOffset; +}; + + +class LLMeshDecompositionHandler : public LLMeshHandlerBase +{ +public: + LLMeshDecompositionHandler(const LLUUID& id, U32 offset, U32 size) + : LLMeshHandlerBase(), + mMeshID(id), + mRequestedBytes(size), + mOffset(offset) + {} + virtual ~LLMeshDecompositionHandler(); + +protected: + LLMeshDecompositionHandler(const LLMeshDecompositionHandler &); // Not defined + void operator=(const LLMeshDecompositionHandler &); // Not defined + +public: + virtual void processData(LLCore::BufferArray * body, U8 * data, S32 data_size); + virtual void processFailure(LLCore::HttpStatus status); + +public: + LLUUID mMeshID; + U32 mRequestedBytes; + U32 mOffset; +}; + + +class LLMeshPhysicsShapeHandler : public LLMeshHandlerBase +{ +public: + LLMeshPhysicsShapeHandler(const LLUUID& id, U32 offset, U32 size) + : LLMeshHandlerBase(), + mMeshID(id), + mRequestedBytes(size), + mOffset(offset) + {} + virtual ~LLMeshPhysicsShapeHandler(); + +protected: + LLMeshPhysicsShapeHandler(const LLMeshPhysicsShapeHandler &); // Not defined + void operator=(const LLMeshPhysicsShapeHandler &); // Not defined + +public: + virtual void processData(LLCore::BufferArray * body, U8 * data, S32 data_size); + virtual void processFailure(LLCore::HttpStatus status); + +public: + LLUUID mMeshID; + U32 mRequestedBytes; + U32 mOffset; +}; + + class LLMeshHeaderResponder : public LLCurl::Responder { public: @@ -538,16 +695,45 @@ public: }; LLMeshRepoThread::LLMeshRepoThread() -: LLThread("mesh repo") +: LLThread("mesh repo"), + mCurlRequest(NULL), + mWaiting(false), + mHttpRequest(NULL), + mHttpOptions(NULL), + mHttpHeaders(NULL), + mHttpPolicyClass(LLCore::HttpRequest::DEFAULT_POLICY_ID) { - mWaiting = false; mMutex = new LLMutex(NULL); mHeaderMutex = new LLMutex(NULL); mSignal = new LLCondition(NULL); + mHttpRequest = new LLCore::HttpRequest; + mHttpOptions = new LLCore::HttpOptions; + mHttpHeaders = new LLCore::HttpHeaders; + mHttpHeaders->mHeaders.push_back("Accept: application/vnd.ll.mesh"); + mHttpPolicyClass = LLAppViewer::instance()->getAppCoreHttp().getPolicyMesh(); } LLMeshRepoThread::~LLMeshRepoThread() { + for (http_request_set::iterator iter(mHttpRequestSet.begin()); + iter != mHttpRequestSet.end(); + ++iter) + { + delete *iter; + } + mHttpRequestSet.clear(); + if (mHttpHeaders) + { + mHttpHeaders->release(); + mHttpHeaders = NULL; + } + if (mHttpOptions) + { + mHttpOptions->release(); + mHttpOptions = NULL; + } + delete mHttpRequest; + mHttpRequest = NULL; delete mMutex; mMutex = NULL; delete mHeaderMutex; @@ -571,7 +757,7 @@ void LLMeshRepoThread::run() mSignal->wait(); mWaiting = false; - if (!LLApp::isQuitting()) + if (! LLApp::isQuitting() && ! mHttpRequestSet.empty()) { static U32 count = 0; @@ -660,6 +846,7 @@ void LLMeshRepoThread::run() } mCurlRequest->process(); + mHttpRequest->update(0L); } } @@ -1045,13 +1232,15 @@ bool LLMeshRepoThread::fetchMeshHeader(const LLVolumeParams& mesh_params, U32& c S32 size = file.getSize(); if (size > 0) - { //NOTE -- if the header size is ever more than 4KB, this will break - U8 buffer[4096]; - S32 bytes = llmin(size, 4096); + { + // *NOTE: if the header size is ever more than 4KB, this will break + U8 buffer[MESH_HEADER_SIZE]; + S32 bytes = llmin(size, MESH_HEADER_SIZE); LLMeshRepository::sCacheBytesRead += bytes; file.read(buffer, bytes); if (headerReceived(mesh_params, buffer, bytes)) - { //did not do an HTTP request, return false + { + // Found mesh in VFS cache return true; } } @@ -1068,7 +1257,31 @@ bool LLMeshRepoThread::fetchMeshHeader(const LLVolumeParams& mesh_params, U32& c //grab first 4KB if we're going to bother with a fetch. Cache will prevent future fetches if a full mesh fits //within the first 4KB //NOTE -- this will break of headers ever exceed 4KB - retval = mCurlRequest->getByteRange(http_url, headers, 0, 4096, new LLMeshHeaderResponder(mesh_params)); +#if 0 + retval = mCurlRequest->getByteRange(http_url, headers, 0, MESH_HEADER_SIZE, new LLMeshHeaderResponder(mesh_params)); +#else + LLMeshHeaderHandler * handler = new LLMeshHeaderHandler(mesh_params); + LLCore::HttpHandle handle = mHttpRequest->requestGetByteRange(mHttpPolicyClass, + 0, // *TODO: Get better priority value + http_url, + 0, + MESH_HEADER_SIZE, + mHttpOptions, + mHttpHeaders, + handler); + if (LLCORE_HTTP_HANDLE_INVALID == handle) + { + // *TODO: Better error message + llwarns << "HTTP GET request failed for mesh " << mID << llendl; + delete handler; + retval = false; + } + else + { + handler->mHttpHandle = handle; + mHttpRequestSet.insert(handler); + } +#endif if(retval) { LLMeshRepository::sHTTPRequestCount++; @@ -1209,7 +1422,7 @@ bool LLMeshRepoThread::headerReceived(const LLVolumeParams& mesh_params, U8* dat LLMutexLock lock(mHeaderMutex); mMeshHeaderSize[mesh_id] = header_size; mMeshHeader[mesh_id] = header; - } + } LLMutexLock lock(mMutex); // make sure only one thread access mPendingLOD at the same time. @@ -2162,6 +2375,336 @@ void LLMeshPhysicsShapeResponder::completedRaw(U32 status, const std::string& re delete [] data; } + +LLMeshHandlerBase::~LLMeshHandlerBase() +{} + + +void LLMeshHandlerBase::onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * response) +{ + mProcessed = true; + + LLCore::HttpStatus status(response->getStatus()); + if (! status) + { + processFailure(status); + } + else + { + // From texture fetch code and applies here: + // + // A warning about partial (HTTP 206) data. Some grid services + // do *not* return a 'Content-Range' header in the response to + // Range requests with a 206 status. We're forced to assume + // we get what we asked for in these cases until we can fix + // the services. + static const LLCore::HttpStatus par_status(HTTP_PARTIAL_CONTENT); + + LLCore::BufferArray * body(response->getBody()); + S32 data_size(body ? body->size() : 0); + U8 * data(NULL); + + if (data_size > 0) + { + // *TODO: Try to get rid of data copying and add interfaces + // that support BufferArray directly. + data = new U8[data_size]; + body->read(0, (char *) data, data_size); + LLMeshRepository::sBytesReceived += llmin(data_size, MESH_HEADER_SIZE); + } + + processData(body, data, data_size); + + delete [] data; + } + + // Release handler + gMeshRepo.mThread->mHttpRequestSet.erase(this); + + delete this; // Must be last statement +} + + +LLMeshHeaderHandler::~LLMeshHeaderHandler() +{ + if (!LLApp::isQuitting()) + { + if (! mProcessed) + { + // something went wrong, retry + llwarns << "Timeout or service unavailable, retrying." << llendl; + LLMeshRepository::sHTTPRetryCount++; + LLMeshRepoThread::HeaderRequest req(mMeshParams); + LLMutexLock lock(gMeshRepo.mThread->mMutex); + gMeshRepo.mThread->mHeaderReqQ.push(req); + } + LLMeshRepoThread::decActiveHeaderRequests(); + } +} + + +void LLMeshHeaderHandler::processFailure(LLCore::HttpStatus status) +{ + if (is_retryable(status)) + { + llwarns << "Timeout or service unavailable, retrying." << llendl; + LLMeshRepository::sHTTPRetryCount++; + LLMeshRepoThread::HeaderRequest req(mMeshParams); + LLMutexLock lock(gMeshRepo.mThread->mMutex); + gMeshRepo.mThread->mHeaderReqQ.push(req); + } + else + { + // *TODO: better error message + llwarns << "Unhandled status." << llendl; + } +} + + +void LLMeshHeaderHandler::processData(LLCore::BufferArray * body, U8 * data, S32 data_size) +{ + bool success = gMeshRepo.mThread->headerReceived(mMeshParams, data, data_size); + llassert(success); + if (! success) + { + // *TODO: Get real reason for parse failure here + llwarns << "Unable to parse mesh header: " << llendl; + } + else if (data && data_size > 0) + { + // header was successfully retrieved from sim, cache in vfs + LLUUID mesh_id = mMeshParams.getSculptID(); + LLSD header = gMeshRepo.mThread->mMeshHeader[mesh_id]; + + S32 version = header["version"].asInteger(); + + if (version <= MAX_MESH_VERSION) + { + std::stringstream str; + + S32 lod_bytes = 0; + + for (U32 i = 0; i < LLModel::LOD_PHYSICS; ++i) + { + // figure out how many bytes we'll need to reserve in the file + std::string lod_name = header_lod[i]; + lod_bytes = llmax(lod_bytes, header[lod_name]["offset"].asInteger()+header[lod_name]["size"].asInteger()); + } + + // just in case skin info or decomposition is at the end of the file (which it shouldn't be) + lod_bytes = llmax(lod_bytes, header["skin"]["offset"].asInteger() + header["skin"]["size"].asInteger()); + lod_bytes = llmax(lod_bytes, header["physics_convex"]["offset"].asInteger() + header["physics_convex"]["size"].asInteger()); + + S32 header_bytes = (S32) gMeshRepo.mThread->mMeshHeaderSize[mesh_id]; + S32 bytes = lod_bytes + header_bytes; + + + // It's possible for the remote asset to have more data than is needed for the local cache + // only allocate as much space in the VFS as is needed for the local cache + data_size = llmin(data_size, bytes); + + LLVFile file(gVFS, mesh_id, LLAssetType::AT_MESH, LLVFile::WRITE); + if (file.getMaxSize() >= bytes || file.setMaxSize(bytes)) + { + LLMeshRepository::sCacheBytesWritten += data_size; + + file.write(data, data_size); + + // zero out the rest of the file + U8 block[MESH_HEADER_SIZE]; + memset(block, 0, MESH_HEADER_SIZE); + + while (bytes-file.tell() > MESH_HEADER_SIZE) + { + file.write(block, MESH_HEADER_SIZE); + } + + S32 remaining = bytes-file.tell(); + + if (remaining > 0) + { + file.write(block, remaining); + } + } + } + } +} + + +LLMeshLODHandler::~LLMeshLODHandler() +{ + if (! LLApp::isQuitting()) + { + if (! mProcessed) + { + llwarns << "Killed without being processed, retrying." << llendl; + LLMeshRepository::sHTTPRetryCount++; + gMeshRepo.mThread->lockAndLoadMeshLOD(mMeshParams, mLOD); + } + LLMeshRepoThread::decActiveLODRequests(); + } +} + +void LLMeshLODHandler::processFailure(LLCore::HttpStatus status) +{ + if (is_retryable(status)) + { + llwarns << "Timeout or service unavailable, retrying." << llendl; + LLMeshRepository::sHTTPRetryCount++; + // *FIXME: Is this safe? Does this need locking? + gMeshRepo.mThread->loadMeshLOD(mMeshParams, mLOD); + } + else + { + // *TODO: better error message + llwarns << "Unhandled status." << llendl; + } +} + +void LLMeshLODHandler::processData(LLCore::BufferArray * body, U8 * data, S32 data_size) +{ + if (gMeshRepo.mThread->lodReceived(mMeshParams, mLOD, data, data_size)) + { + // good fetch from sim, write to VFS for caching + LLVFile file(gVFS, mMeshParams.getSculptID(), LLAssetType::AT_MESH, LLVFile::WRITE); + + S32 offset = mOffset; + S32 size = mRequestedBytes; + + if (file.getSize() >= offset+size) + { + file.seek(offset); + file.write(data, size); + LLMeshRepository::sCacheBytesWritten += size; + } + } +} + + +LLMeshSkinInfoHandler::~LLMeshSkinInfoHandler() +{ + llassert(mProcessed); +} + +void LLMeshSkinInfoHandler::processFailure(LLCore::HttpStatus status) +{ + if (is_retryable(status)) + { + llwarns << "Timeout or service unavailable, retrying." << llendl; + LLMeshRepository::sHTTPRetryCount++; + // *FIXME: Is this safe? Does this need locking? + gMeshRepo.mThread->loadMeshSkinInfo(mMeshID); + } + else + { + // *TODO: better error message + llwarns << "Unhandled status." << llendl; + } +} + +void LLMeshSkinInfoHandler::processData(LLCore::BufferArray * body, U8 * data, S32 data_size) +{ + if (gMeshRepo.mThread->skinInfoReceived(mMeshID, data, data_size)) + { + // good fetch from sim, write to VFS for caching + LLVFile file(gVFS, mMeshID, LLAssetType::AT_MESH, LLVFile::WRITE); + + S32 offset = mOffset; + S32 size = mRequestedBytes; + + if (file.getSize() >= offset+size) + { + LLMeshRepository::sCacheBytesWritten += size; + file.seek(offset); + file.write(data, size); + } + } +} + + +LLMeshDecompositionHandler::~LLMeshDecompositionHandler() +{ + llassert(mProcessed); +} + +void LLMeshDecompositionHandler::processFailure(LLCore::HttpStatus status) +{ + if (is_retryable(status)) + { + llwarns << "Timeout or service unavailable, retrying." << llendl; + LLMeshRepository::sHTTPRetryCount++; + // *FIXME: Is this safe? Does this need locking? + gMeshRepo.mThread->loadMeshDecomposition(mMeshID); + } + else + { + // *TODO: better error message + llwarns << "Unhandled status." << llendl; + } +} + +void LLMeshDecompositionHandler::processData(LLCore::BufferArray * body, U8 * data, S32 data_size) +{ + if (gMeshRepo.mThread->decompositionReceived(mMeshID, data, data_size)) + { + // good fetch from sim, write to VFS for caching + LLVFile file(gVFS, mMeshID, LLAssetType::AT_MESH, LLVFile::WRITE); + + S32 offset = mOffset; + S32 size = mRequestedBytes; + + if (file.getSize() >= offset+size) + { + LLMeshRepository::sCacheBytesWritten += size; + file.seek(offset); + file.write(data, size); + } + } +} + + +LLMeshPhysicsShapeHandler::~LLMeshPhysicsShapeHandler() +{ + llassert(mProcessed); +} + +void LLMeshPhysicsShapeHandler::processFailure(LLCore::HttpStatus status) +{ + if (is_retryable(status)) + { + llwarns << "Timeout or service unavailable, retrying." << llendl; + LLMeshRepository::sHTTPRetryCount++; + // *FIXME: Is this safe? Does this need locking? + gMeshRepo.mThread->loadMeshPhysicsShape(mMeshID); + } + else + { + // *TODO: better error message + llwarns << "Unhandled status." << llendl; + } +} + + +void LLMeshPhysicsShapeHandler::processData(LLCore::BufferArray * body, U8 * data, S32 data_size) +{ + if (gMeshRepo.mThread->physicsShapeReceived(mMeshID, data, data_size)) + { + // good fetch from sim, write to VFS for caching + LLVFile file(gVFS, mMeshID, LLAssetType::AT_MESH, LLVFile::WRITE); + + S32 offset = mOffset; + S32 size = mRequestedBytes; + + if (file.getSize() >= offset+size) + { + LLMeshRepository::sCacheBytesWritten += size; + file.seek(offset); + file.write(data, size); + } + } +} + + void LLMeshHeaderResponder::completedRaw(U32 status, const std::string& reason, const LLChannelDescriptors& channels, const LLIOPipe::buffer_ptr_t& buffer) @@ -2215,7 +2758,7 @@ void LLMeshHeaderResponder::completedRaw(U32 status, const std::string& reason, buffer->readAfter(channels.in(), NULL, data, data_size); } - LLMeshRepository::sBytesReceived += llmin(data_size, 4096); + LLMeshRepository::sBytesReceived += llmin(data_size, MESH_HEADER_SIZE); bool success = gMeshRepo.mThread->headerReceived(mMeshParams, data, data_size); @@ -2267,12 +2810,12 @@ void LLMeshHeaderResponder::completedRaw(U32 status, const std::string& reason, file.write((const U8*) data, data_size); //zero out the rest of the file - U8 block[4096]; - memset(block, 0, 4096); + U8 block[MESH_HEADER_SIZE]; + memset(block, 0, MESH_HEADER_SIZE); - while (bytes-file.tell() > 4096) + while (bytes-file.tell() > MESH_HEADER_SIZE) { - file.write(block, 4096); + file.write(block, MESH_HEADER_SIZE); } S32 remaining = bytes-file.tell(); @@ -3797,3 +4340,286 @@ void metrics_teleport_started() ++metrics_teleport_start_count; } + +// This comes from an edit in viewer-cat. Unify this once that's +// available everywhere. +bool is_retryable(LLCore::HttpStatus status) +{ + static const LLCore::HttpStatus cant_connect(LLCore::HttpStatus::EXT_CURL_EASY, CURLE_COULDNT_CONNECT); + static const LLCore::HttpStatus cant_res_proxy(LLCore::HttpStatus::EXT_CURL_EASY, CURLE_COULDNT_RESOLVE_PROXY); + static const LLCore::HttpStatus cant_res_host(LLCore::HttpStatus::EXT_CURL_EASY, CURLE_COULDNT_RESOLVE_HOST); + static const LLCore::HttpStatus send_error(LLCore::HttpStatus::EXT_CURL_EASY, CURLE_SEND_ERROR); + static const LLCore::HttpStatus recv_error(LLCore::HttpStatus::EXT_CURL_EASY, CURLE_RECV_ERROR); + static const LLCore::HttpStatus upload_failed(LLCore::HttpStatus::EXT_CURL_EASY, CURLE_UPLOAD_FAILED); + static const LLCore::HttpStatus op_timedout(LLCore::HttpStatus::EXT_CURL_EASY, CURLE_OPERATION_TIMEDOUT); + static const LLCore::HttpStatus post_error(LLCore::HttpStatus::EXT_CURL_EASY, CURLE_HTTP_POST_ERROR); + static const LLCore::HttpStatus partial_file(LLCore::HttpStatus::EXT_CURL_EASY, CURLE_PARTIAL_FILE); + static const LLCore::HttpStatus inv_cont_range(LLCore::HttpStatus::LLCORE, LLCore::HE_INV_CONTENT_RANGE_HDR); + + return ((! status) && + ((status.isHttpStatus() && status.mType >= 499 && status.mType <= 599) || // Include special 499 in retryables + status == cant_connect || // Connection reset/endpoint problems + status == cant_res_proxy || // DNS problems + status == cant_res_host || // DNS problems + status == send_error || // General socket problems + status == recv_error || // General socket problems + status == upload_failed || // Transport problem + status == op_timedout || // Timer expired + status == post_error || // Transport problem + status == partial_file || // Data inconsistency in response + status == inv_cont_range)); // Short data read disagrees with content-range +} + + + + +// =========== +// +// HTTP fragments I'll be needing +// +// +#if 0 + mHttpPolicyClass(LLCore::HttpRequest::DEFAULT_POLICY_ID), + mHttpRequest = new LLCore::HttpRequest; + mHttpOptions = new LLCore::HttpOptions; + mHttpHeaders = new LLCore::HttpHeaders; + mHttpHeaders->mHeaders.push_back("Accept: image/x-j2c"); + mHttpMetricsHeaders = new LLCore::HttpHeaders; + mHttpMetricsHeaders->mHeaders.push_back("Content-Type: application/llsd+xml"); + mHttpPolicyClass = LLAppViewer::instance()->getAppCoreHttp().getPolicyDefault(); + + + LLCore::HttpHandle mHttpHandle; // Handle of any active request + LLCore::BufferArray * mHttpBufferArray; // Refcounted pointer to response data + int mHttpPolicyClass; + bool mHttpActive; // Active request to http library + unsigned int mHttpReplySize; // Actual received data size + unsigned int mHttpReplyOffset; // Actual received data offset + bool mHttpHasResource; // Counts against Fetcher's mHttpSemaphore + + + mHttpHandle = LLCORE_HTTP_HANDLE_INVALID; + if (!mUrl.empty()) + { + mRequestedTimer.reset(); + mLoaded = FALSE; + mGetStatus = LLCore::HttpStatus(); + mGetReason.clear(); + LL_DEBUGS("Texture") << "HTTP GET: " << mID << " Offset: " << mRequestedOffset + << " Bytes: " << mRequestedSize + << " Bandwidth(kbps): " << mFetcher->getTextureBandwidth() << "/" << mFetcher->mMaxBandwidth + << LL_ENDL; + + // Will call callbackHttpGet when curl request completes + mHttpHandle = mFetcher->mHttpRequest->requestGetByteRange(mHttpPolicyClass, + mWorkPriority, + mUrl, + mRequestedOffset, + mRequestedSize, + mFetcher->mHttpOptions, + mFetcher->mHttpHeaders, + this); + } + if (LLCORE_HTTP_HANDLE_INVALID == mHttpHandle) + { + llwarns << "HTTP GET request failed for " << mID << llendl; + resetFormattedData(); + releaseHttpSemaphore(); + return true; // failed + } + + mHttpActive = true; + mFetcher->addToHTTPQueue(mID); + recordTextureStart(true); + setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); + mState = WAIT_HTTP_REQ; + + // fall through + } + + + +// Threads: Ttf +// virtual +void LLTextureFetchWorker::onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * response) +{ + static LLCachedControl<bool> log_to_viewer_log(gSavedSettings, "LogTextureDownloadsToViewerLog"); + static LLCachedControl<bool> log_to_sim(gSavedSettings, "LogTextureDownloadsToSimulator"); + static LLCachedControl<bool> log_texture_traffic(gSavedSettings, "LogTextureNetworkTraffic") ; + + LLMutexLock lock(&mWorkMutex); // +Mw + + mHttpActive = false; + + if (log_to_viewer_log || log_to_sim) + { + U64 timeNow = LLTimer::getTotalTime(); + mFetcher->mTextureInfo.setRequestStartTime(mID, mMetricsStartTime); + mFetcher->mTextureInfo.setRequestType(mID, LLTextureInfoDetails::REQUEST_TYPE_HTTP); + mFetcher->mTextureInfo.setRequestSize(mID, mRequestedSize); + mFetcher->mTextureInfo.setRequestOffset(mID, mRequestedOffset); + mFetcher->mTextureInfo.setRequestCompleteTimeAndLog(mID, timeNow); + } + + bool success = true; + bool partial = false; + LLCore::HttpStatus status(response->getStatus()); + + lldebugs << "HTTP COMPLETE: " << mID + << " status: " << status.toHex() + << " '" << status.toString() << "'" + << llendl; +// unsigned int offset(0), length(0), full_length(0); +// response->getRange(&offset, &length, &full_length); +// llwarns << "HTTP COMPLETE: " << mID << " handle: " << handle +// << " status: " << status.toULong() << " '" << status.toString() << "'" +// << " req offset: " << mRequestedOffset << " req length: " << mRequestedSize +// << " offset: " << offset << " length: " << length +// << llendl; + + if (! status) + { + success = false; + std::string reason(status.toString()); + setGetStatus(status, reason); + llwarns << "CURL GET FAILED, status: " << status.toHex() + << " reason: " << reason << llendl; + } + else + { + // A warning about partial (HTTP 206) data. Some grid services + // do *not* return a 'Content-Range' header in the response to + // Range requests with a 206 status. We're forced to assume + // we get what we asked for in these cases until we can fix + // the services. + static const LLCore::HttpStatus par_status(HTTP_PARTIAL_CONTENT); + + partial = (par_status == status); + } + + S32 data_size = callbackHttpGet(response, partial, success); + + if (log_texture_traffic && data_size > 0) + { + LLViewerTexture* tex = LLViewerTextureManager::findTexture(mID); + if (tex) + { + gTotalTextureBytesPerBoostLevel[tex->getBoostLevel()] += data_size ; + } + } + + mFetcher->removeFromHTTPQueue(mID, data_size); + + recordTextureDone(true); +} // -Mw + + +// Threads: Ttf +// Locks: Mw +S32 LLTextureFetchWorker::callbackHttpGet(LLCore::HttpResponse * response, + bool partial, bool success) +{ + S32 data_size = 0 ; + + if (mState != WAIT_HTTP_REQ) + { + llwarns << "callbackHttpGet for unrequested fetch worker: " << mID + << " req=" << mSentRequest << " state= " << mState << llendl; + return data_size; + } + if (mLoaded) + { + llwarns << "Duplicate callback for " << mID.asString() << llendl; + return data_size ; // ignore duplicate callback + } + if (success) + { + // get length of stream: + LLCore::BufferArray * body(response->getBody()); + data_size = body ? body->size() : 0; + + LL_DEBUGS("Texture") << "HTTP RECEIVED: " << mID.asString() << " Bytes: " << data_size << LL_ENDL; + if (data_size > 0) + { + LLViewerStatsRecorder::instance().textureFetch(data_size); + // *TODO: set the formatted image data here directly to avoid the copy + + // Hold on to body for later copy + llassert_always(NULL == mHttpBufferArray); + body->addRef(); + mHttpBufferArray = body; + + if (partial) + { + unsigned int offset(0), length(0), full_length(0); + response->getRange(&offset, &length, &full_length); + if (! offset && ! length) + { + // This is the case where we receive a 206 status but + // there wasn't a useful Content-Range header in the response. + // This could be because it was badly formatted but is more + // likely due to capabilities services which scrub headers + // from responses. Assume we got what we asked for... + mHttpReplySize = data_size; + mHttpReplyOffset = mRequestedOffset; + } + else + { + mHttpReplySize = length; + mHttpReplyOffset = offset; + } + } + + if (! partial) + { + // Response indicates this is the entire asset regardless + // of our asking for a byte range. Mark it so and drop + // any partial data we might have so that the current + // response body becomes the entire dataset. + if (data_size <= mRequestedOffset) + { + LL_WARNS("Texture") << "Fetched entire texture " << mID + << " when it was expected to be marked complete. mImageSize: " + << mFileSize << " datasize: " << mFormattedImage->getDataSize() + << LL_ENDL; + } + mHaveAllData = TRUE; + llassert_always(mDecodeHandle == 0); + mFormattedImage = NULL; // discard any previous data we had + } + else if (data_size < mRequestedSize) + { + mHaveAllData = TRUE; + } + else if (data_size > mRequestedSize) + { + // *TODO: This shouldn't be happening any more (REALLY don't expect this anymore) + llwarns << "data_size = " << data_size << " > requested: " << mRequestedSize << llendl; + mHaveAllData = TRUE; + llassert_always(mDecodeHandle == 0); + mFormattedImage = NULL; // discard any previous data we had + } + } + else + { + // We requested data but received none (and no error), + // so presumably we have all of it + mHaveAllData = TRUE; + } + mRequestedSize = data_size; + } + else + { + mRequestedSize = -1; // error + } + + mLoaded = TRUE; + setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); + + LLViewerStatsRecorder::instance().log(0.2f); + return data_size ; +} + +#endif +// ============ + diff --git a/indra/newview/llmeshrepository.h b/indra/newview/llmeshrepository.h index 3cdc66e1f0..8ffe2d68cb 100644 --- a/indra/newview/llmeshrepository.h +++ b/indra/newview/llmeshrepository.h @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2001&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2010, Linden Research, Inc. + * Copyright (C) 2010-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -33,6 +33,11 @@ #include "llviewertexture.h" #include "llvolume.h" #include "lldeadmantimer.h" +#include "httpcommon.h" +#include "httprequest.h" +#include "httpoptions.h" +#include "httpheaders.h" +#include "httphandler.h" #define LLCONVEXDECOMPINTER_STATIC 1 @@ -316,6 +321,15 @@ public: typedef std::map<LLVolumeParams, std::vector<S32> > pending_lod_map; pending_lod_map mPendingLOD; + // llcorehttp library interface objects. + LLCore::HttpRequest * mHttpRequest; + LLCore::HttpOptions * mHttpOptions; + LLCore::HttpHeaders * mHttpHeaders; + LLCore::HttpRequest::policy_t mHttpPolicyClass; + + typedef std::set<LLCore::HttpHandler *> http_request_set; + http_request_set mHttpRequestSet; // Outstanding HTTP requests + static std::string constructUrl(LLUUID mesh_id); LLMeshRepoThread(); |