/** * @file llviewerassetstorage.cpp * @brief Subclass capable of loading asset data to/from an external source. * * $LicenseInfo:firstyear=2003&license=viewerlgpl$ * Second Life Viewer Source Code * Copyright (C) 2010, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; * version 2.1 of the License only. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA * * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA * $/LicenseInfo$ */ #include "llviewerprecompiledheaders.h" #include "llviewerassetstorage.h" #include "llvfile.h" #include "llvfs.h" #include "message.h" #include "llagent.h" #include "llappcorehttp.h" #include "llviewerregion.h" #include "lltransfersourceasset.h" #include "lltransfertargetvfile.h" #include "llviewerassetstats.h" #include "llcoros.h" #include "llcoproceduremanager.h" #include "lleventcoro.h" #include "llsdutil.h" #include "llworld.h" ///---------------------------------------------------------------------------- /// LLViewerAssetRequest ///---------------------------------------------------------------------------- // There is also PoolSizeAssetStorage value in setting that should mirror this name static const std::string VIEWER_ASSET_STORAGE_CORO_POOL = "AssetStorage"; /** * @brief Local class to encapsulate asset fetch requests with a timestamp. * * Derived from the common LLAssetRequest class, this is currently used * only for fetch/get operations and its only function is to wrap remote * asset fetch requests so that they can be timed. */ class LLViewerAssetRequest : public LLAssetRequest { public: LLViewerAssetRequest(const LLUUID &uuid, const LLAssetType::EType type, bool with_http) : LLAssetRequest(uuid, type), mMetricsStartTime(0), mWithHTTP(with_http) { } LLViewerAssetRequest & operator=(const LLViewerAssetRequest &); // Not defined // Default assignment operator valid // virtual ~LLViewerAssetRequest() { recordMetrics(); } protected: void recordMetrics() { if (mMetricsStartTime.value()) { // Okay, it appears this request was used for useful things. Record // the expected dequeue and duration of request processing. LLViewerAssetStatsFF::record_dequeue(mType, mWithHTTP, false); LLViewerAssetStatsFF::record_response(mType, mWithHTTP, false, (LLViewerAssetStatsFF::get_timestamp() - mMetricsStartTime), mBytesFetched); mMetricsStartTime = (U32Seconds)0; } } public: LLViewerAssetStats::duration_t mMetricsStartTime; bool mWithHTTP; }; ///---------------------------------------------------------------------------- /// LLViewerAssetStorage ///---------------------------------------------------------------------------- S32 LLViewerAssetStorage::sAssetCoroCount = 0; // Unused? LLViewerAssetStorage::LLViewerAssetStorage(LLMessageSystem *msg, LLXferManager *xfer, LLVFS *vfs, LLVFS *static_vfs, const LLHost &upstream_host) : LLAssetStorage(msg, xfer, vfs, static_vfs, upstream_host), mCountRequests(0), mCountStarted(0), mCountCompleted(0), mCountSucceeded(0), mTotalBytesFetched(0) { LLCoprocedureManager::instance().initializePool(VIEWER_ASSET_STORAGE_CORO_POOL); } LLViewerAssetStorage::LLViewerAssetStorage(LLMessageSystem *msg, LLXferManager *xfer, LLVFS *vfs, LLVFS *static_vfs) : LLAssetStorage(msg, xfer, vfs, static_vfs), mCountRequests(0), mCountStarted(0), mCountCompleted(0), mCountSucceeded(0), mTotalBytesFetched(0) { LLCoprocedureManager::instance().initializePool(VIEWER_ASSET_STORAGE_CORO_POOL); } LLViewerAssetStorage::~LLViewerAssetStorage() { if (!LLCoprocedureManager::wasDeleted()) { // This class has dedicated coroutine pool, clean it up, otherwise coroutines will crash later. LLCoprocedureManager::instance().close(VIEWER_ASSET_STORAGE_CORO_POOL); } while (mCoroWaitList.size() > 0) { CoroWaitList &request = mCoroWaitList.front(); // Clean up pending downloads, delete request and trigger callbacks removeAndCallbackPendingDownloads(request.mId, request.mType, request.mId, request.mType, LL_ERR_NOERR, LLExtStat::NONE); mCoroWaitList.pop_front(); } } // virtual void LLViewerAssetStorage::storeAssetData( const LLTransactionID& tid, LLAssetType::EType asset_type, LLStoreAssetCallback callback, void* user_data, bool temp_file, bool is_priority, bool store_local, bool user_waiting, F64Seconds timeout) { LLAssetID asset_id = tid.makeAssetID(gAgent.getSecureSessionID()); LL_DEBUGS("AssetStorage") << "LLViewerAssetStorage::storeAssetData (legacy) " << tid << ":" << LLAssetType::lookup(asset_type) << " ASSET_ID: " << asset_id << LL_ENDL; if (mUpstreamHost.isOk()) { if (mVFS->getExists(asset_id, asset_type)) { // Pack data into this packet if we can fit it. U8 buffer[MTUBYTES]; buffer[0] = 0; LLVFile vfile(mVFS, asset_id, asset_type, LLVFile::READ); S32 asset_size = vfile.getSize(); LLAssetRequest *req = new LLAssetRequest(asset_id, asset_type); req->mUpCallback = callback; req->mUserData = user_data; if (asset_size < 1) { // This can happen if there's a bug in our code or if the VFS has been corrupted. LL_WARNS("AssetStorage") << "LLViewerAssetStorage::storeAssetData() Data _should_ already be in the VFS, but it's not! " << asset_id << LL_ENDL; // LLAssetStorage metric: Zero size VFS reportMetric( asset_id, asset_type, LLStringUtil::null, LLUUID::null, 0, MR_ZERO_SIZE, __FILE__, __LINE__, "The file didn't exist or was zero length (VFS - can't tell which)" ); delete req; if (callback) { callback(asset_id, user_data, LL_ERR_ASSET_REQUEST_FAILED, LLExtStat::VFS_CORRUPT); } return; } else { // LLAssetStorage metric: Successful Request S32 size = mVFS->getSize(asset_id, asset_type); const char *message = "Added to upload queue"; reportMetric( asset_id, asset_type, LLStringUtil::null, LLUUID::null, size, MR_OKAY, __FILE__, __LINE__, message ); if(is_priority) { mPendingUploads.push_front(req); } else { mPendingUploads.push_back(req); } } // Read the data from the VFS if it'll fit in this packet. if (asset_size + 100 < MTUBYTES) { BOOL res = vfile.read(buffer, asset_size); /* Flawfinder: ignore */ S32 bytes_read = res ? vfile.getLastBytesRead() : 0; if( bytes_read == asset_size ) { req->mDataSentInFirstPacket = TRUE; //LL_INFOS() << "LLViewerAssetStorage::createAsset sending data in first packet" << LL_ENDL; } else { LL_WARNS("AssetStorage") << "Probable corruption in VFS file, aborting store asset data" << LL_ENDL; // LLAssetStorage metric: VFS corrupt - bogus size reportMetric( asset_id, asset_type, LLStringUtil::null, LLUUID::null, asset_size, MR_VFS_CORRUPTION, __FILE__, __LINE__, "VFS corruption" ); if (callback) { callback(asset_id, user_data, LL_ERR_ASSET_REQUEST_NONEXISTENT_FILE, LLExtStat::VFS_CORRUPT); } return; } } else { // Too big, do an xfer buffer[0] = 0; asset_size = 0; } mMessageSys->newMessageFast(_PREHASH_AssetUploadRequest); mMessageSys->nextBlockFast(_PREHASH_AssetBlock); mMessageSys->addUUIDFast(_PREHASH_TransactionID, tid); mMessageSys->addS8Fast(_PREHASH_Type, (S8)asset_type); mMessageSys->addBOOLFast(_PREHASH_Tempfile, temp_file); mMessageSys->addBOOLFast(_PREHASH_StoreLocal, store_local); mMessageSys->addBinaryDataFast( _PREHASH_AssetData, buffer, asset_size ); mMessageSys->sendReliable(mUpstreamHost); } else { LL_WARNS("AssetStorage") << "AssetStorage: attempt to upload non-existent vfile " << asset_id << ":" << LLAssetType::lookup(asset_type) << LL_ENDL; // LLAssetStorage metric: Zero size VFS reportMetric( asset_id, asset_type, LLStringUtil::null, LLUUID::null, 0, MR_ZERO_SIZE, __FILE__, __LINE__, "The file didn't exist or was zero length (VFS - can't tell which)" ); if (callback) { callback(asset_id, user_data, LL_ERR_ASSET_REQUEST_NONEXISTENT_FILE, LLExtStat::NONEXISTENT_FILE); } } } else { LL_WARNS("AssetStorage") << "Attempt to move asset store request upstream w/o valid upstream provider" << LL_ENDL; // LLAssetStorage metric: Upstream provider dead reportMetric( asset_id, asset_type, LLStringUtil::null, LLUUID::null, 0, MR_NO_UPSTREAM, __FILE__, __LINE__, "No upstream provider" ); if (callback) { callback(asset_id, user_data, LL_ERR_CIRCUIT_GONE, LLExtStat::NO_UPSTREAM); } } } void LLViewerAssetStorage::storeAssetData( const std::string& filename, const LLTransactionID& tid, LLAssetType::EType asset_type, LLStoreAssetCallback callback, void* user_data, bool temp_file, bool is_priority, bool user_waiting, F64Seconds timeout) { if(filename.empty()) { // LLAssetStorage metric: no filename reportMetric( LLUUID::null, asset_type, LLStringUtil::null, LLUUID::null, 0, MR_VFS_CORRUPTION, __FILE__, __LINE__, "Filename missing" ); LL_ERRS() << "No filename specified" << LL_ENDL; return; } LLAssetID asset_id = tid.makeAssetID(gAgent.getSecureSessionID()); LL_DEBUGS("AssetStorage") << "LLViewerAssetStorage::storeAssetData (legacy)" << asset_id << ":" << LLAssetType::lookup(asset_type) << LL_ENDL; LL_DEBUGS("AssetStorage") << "ASSET_ID: " << asset_id << LL_ENDL; S32 size = 0; LLFILE* fp = LLFile::fopen(filename, "rb"); if (fp) { fseek(fp, 0, SEEK_END); size = ftell(fp); fseek(fp, 0, SEEK_SET); } if( size ) { LLLegacyAssetRequest *legacy = new LLLegacyAssetRequest; legacy->mUpCallback = callback; legacy->mUserData = user_data; LLVFile file(mVFS, asset_id, asset_type, LLVFile::WRITE); file.setMaxSize(size); const S32 buf_size = 65536; U8 copy_buf[buf_size]; while ((size = (S32)fread(copy_buf, 1, buf_size, fp))) { file.write(copy_buf, size); } fclose(fp); // if this upload fails, the caller needs to setup a new tempfile for us if (temp_file) { LLFile::remove(filename); } // LLAssetStorage metric: Success not needed; handled in the overloaded method here: LLViewerAssetStorage::storeAssetData( tid, asset_type, legacyStoreDataCallback, (void**)legacy, temp_file, is_priority); } else // size == 0 (but previous block changes size) { if( fp ) { // LLAssetStorage metric: Zero size reportMetric( asset_id, asset_type, filename, LLUUID::null, 0, MR_ZERO_SIZE, __FILE__, __LINE__, "The file was zero length" ); } else { // LLAssetStorage metric: Missing File reportMetric( asset_id, asset_type, filename, LLUUID::null, 0, MR_FILE_NONEXIST, __FILE__, __LINE__, "The file didn't exist" ); } if (callback) { callback(asset_id, user_data, LL_ERR_CANNOT_OPEN_FILE, LLExtStat::BLOCKED_FILE); } } } void LLViewerAssetStorage::checkForTimeouts() { LLAssetStorage::checkForTimeouts(); // Restore requests LLCoprocedureManager* manager = LLCoprocedureManager::getInstance(); while (mCoroWaitList.size() > 0 && manager->count(VIEWER_ASSET_STORAGE_CORO_POOL) < LLCoprocedureManager::DEFAULT_QUEUE_SIZE) { CoroWaitList &request = mCoroWaitList.front(); bool with_http = true; bool is_temp = false; LLViewerAssetStatsFF::record_enqueue(request.mType, with_http, is_temp); manager->enqueueCoprocedure(VIEWER_ASSET_STORAGE_CORO_POOL, "LLViewerAssetStorage::assetRequestCoro", boost::bind(&LLViewerAssetStorage::assetRequestCoro, this, request.mRequest, request.mId, request.mType, request.mCallback, request.mUserData)); mCoroWaitList.pop_front(); } } /** * @brief Allocate and queue an asset fetch request for the viewer * * This is a nearly-verbatim copy of the base class's implementation * with the following changes: * - Use a locally-derived request class * - Start timing for metrics when request is queued * * This is an unfortunate implementation choice but it's forced by * current conditions. A refactoring that might clean up the layers * of responsibility or introduce factories or more virtualization * of methods would enable a more attractive solution. * * If LLAssetStorage::_queueDataRequest changes, this must change * as well. */ // virtual void LLViewerAssetStorage::_queueDataRequest( const LLUUID& uuid, LLAssetType::EType atype, LLGetAssetCallback callback, void *user_data, BOOL duplicate, BOOL is_priority) { mCountRequests++; queueRequestHttp(uuid, atype, callback, user_data, duplicate, is_priority); } void LLViewerAssetStorage::queueRequestHttp( const LLUUID& uuid, LLAssetType::EType atype, LLGetAssetCallback callback, void *user_data, BOOL duplicate, BOOL is_priority) { LL_DEBUGS("ViewerAsset") << "Request asset via HTTP " << uuid << " type " << LLAssetType::lookup(atype) << LL_ENDL; bool with_http = true; LLViewerAssetRequest *req = new LLViewerAssetRequest(uuid, atype, with_http); req->mDownCallback = callback; req->mUserData = user_data; req->mIsPriority = is_priority; if (!duplicate) { // Only collect metrics for non-duplicate requests. Others // are piggy-backing and will artificially lower averages. req->mMetricsStartTime = LLViewerAssetStatsFF::get_timestamp(); } mPendingDownloads.push_back(req); // This is the same as the current UDP logic - don't re-request a duplicate. if (!duplicate) { // Coroutine buffer has fixed size (synchronization buffer, so we have no alternatives), so buffer any request above limit if (LLCoprocedureManager::instance().count(VIEWER_ASSET_STORAGE_CORO_POOL) < LLCoprocedureManager::DEFAULT_QUEUE_SIZE) { bool with_http = true; bool is_temp = false; LLViewerAssetStatsFF::record_enqueue(atype, with_http, is_temp); LLCoprocedureManager::instance().enqueueCoprocedure(VIEWER_ASSET_STORAGE_CORO_POOL, "LLViewerAssetStorage::assetRequestCoro", boost::bind(&LLViewerAssetStorage::assetRequestCoro, this, req, uuid, atype, callback, user_data)); } else { mCoroWaitList.emplace_back(req, uuid, atype, callback, user_data); } } } void LLViewerAssetStorage::capsRecvForRegion(const LLUUID& region_id, std::string pumpname) { LLViewerRegion *regionp = LLWorld::instance().getRegionFromID(region_id); if (!regionp) { LL_WARNS("ViewerAsset") << "region not found for region_id " << region_id << LL_ENDL; } else { mViewerAssetUrl = regionp->getViewerAssetUrl(); } LLEventPumps::instance().obtain(pumpname).post(LLSD()); } struct LLScopedIncrement { LLScopedIncrement(S32& counter): mCounter(counter) { ++mCounter; } ~LLScopedIncrement() { --mCounter; } S32& mCounter; }; void LLViewerAssetStorage::assetRequestCoro( LLViewerAssetRequest *req, const LLUUID uuid, LLAssetType::EType atype, LLGetAssetCallback callback, void *user_data) { LLScopedIncrement coro_count_boost(sAssetCoroCount); // static counter since corotine can outlive LLViewerAssetStorage S32 result_code = LL_ERR_NOERR; LLExtStat ext_status = LLExtStat::NONE; if (!gAssetStorage) { LL_WARNS_ONCE("ViewerAsset") << "Asset request fails: asset storage no longer exists" << LL_ENDL; return; } mCountStarted++; if (!gAgent.getRegion()) { LL_WARNS_ONCE("ViewerAsset") << "Asset request fails: no region set" << LL_ENDL; result_code = LL_ERR_ASSET_REQUEST_FAILED; ext_status = LLExtStat::NONE; removeAndCallbackPendingDownloads(uuid, atype, uuid, atype, result_code, ext_status); return; } else if (!gAgent.getRegion()->capabilitiesReceived()) { LL_WARNS_ONCE("ViewerAsset") << "Waiting for capabilities" << LL_ENDL; LLEventStream capsRecv("waitForCaps", true); gAgent.getRegion()->setCapabilitiesReceivedCallback( boost::bind(&LLViewerAssetStorage::capsRecvForRegion, this, _1, capsRecv.getName())); llcoro::suspendUntilEventOn(capsRecv); if (LLApp::isExiting() || !gAssetStorage) { return; } LL_WARNS_ONCE("ViewerAsset") << "capsRecv got event" << LL_ENDL; LL_WARNS_ONCE("ViewerAsset") << "region " << gAgent.getRegion() << " mViewerAssetUrl " << mViewerAssetUrl << LL_ENDL; } if (mViewerAssetUrl.empty() && gAgent.getRegion()) { mViewerAssetUrl = gAgent.getRegion()->getViewerAssetUrl(); } if (mViewerAssetUrl.empty()) { LL_WARNS_ONCE("ViewerAsset") << "asset request fails: caps received but no viewer asset cap found" << LL_ENDL; result_code = LL_ERR_ASSET_REQUEST_FAILED; ext_status = LLExtStat::NONE; removeAndCallbackPendingDownloads(uuid, atype, uuid, atype, result_code, ext_status); return; } std::string url = getAssetURL(mViewerAssetUrl, uuid,atype); LL_DEBUGS("ViewerAsset") << "request url: " << url << LL_ENDL; LLCore::HttpRequest::policy_t httpPolicy(LLAppCoreHttp::AP_TEXTURE); LLCoreHttpUtil::HttpCoroutineAdapter::ptr_t httpAdapter(new LLCoreHttpUtil::HttpCoroutineAdapter("assetRequestCoro", httpPolicy)); LLCore::HttpRequest::ptr_t httpRequest(new LLCore::HttpRequest); LLCore::HttpOptions::ptr_t httpOpts = LLCore::HttpOptions::ptr_t(new LLCore::HttpOptions); LLSD result = httpAdapter->getRawAndSuspend(httpRequest, url, httpOpts); if (LLApp::isExiting() || !gAssetStorage) { // Bail out if result arrives after shutdown has been started. return; } mCountCompleted++; LLSD httpResults = result[LLCoreHttpUtil::HttpCoroutineAdapter::HTTP_RESULTS]; LLCore::HttpStatus status = LLCoreHttpUtil::HttpCoroutineAdapter::getStatusFromLLSD(httpResults); if (!status) { LL_DEBUGS("ViewerAsset") << "request failed, status " << status.toTerseString() << LL_ENDL; result_code = LL_ERR_ASSET_REQUEST_FAILED; ext_status = LLExtStat::NONE; } else if (!result.has(LLCoreHttpUtil::HttpCoroutineAdapter::HTTP_RESULTS_RAW)) { LL_DEBUGS("ViewerAsset") << "request failed, no data returned!" << LL_ENDL; result_code = LL_ERR_ASSET_REQUEST_FAILED; ext_status = LLExtStat::NONE; } else if (!result[LLCoreHttpUtil::HttpCoroutineAdapter::HTTP_RESULTS_RAW].isBinary()) { LL_DEBUGS("ViewerAsset") << "request failed, invalid data format!" << LL_ENDL; result_code = LL_ERR_ASSET_REQUEST_FAILED; ext_status = LLExtStat::NONE; } else { LL_DEBUGS("ViewerAsset") << "request succeeded, url " << url << LL_ENDL; const LLSD::Binary &raw = result[LLCoreHttpUtil::HttpCoroutineAdapter::HTTP_RESULTS_RAW].asBinary(); S32 size = raw.size(); if (size > 0) { mTotalBytesFetched += size; // This create-then-rename flow is modeled on // LLTransferTargetVFile, which is what was used in the UDP // case. LLUUID temp_id; temp_id.generate(); LLVFile vf(gAssetStorage->mVFS, temp_id, atype, LLVFile::WRITE); vf.setMaxSize(size); req->mBytesFetched = size; if (!vf.write(raw.data(),size)) { // TODO asset-http: handle error LL_WARNS("ViewerAsset") << "Failure in vf.write()" << LL_ENDL; result_code = LL_ERR_ASSET_REQUEST_FAILED; ext_status = LLExtStat::VFS_CORRUPT; } else if (!vf.rename(uuid, atype)) { LL_WARNS("ViewerAsset") << "rename failed" << LL_ENDL; result_code = LL_ERR_ASSET_REQUEST_FAILED; ext_status = LLExtStat::VFS_CORRUPT; } else { mCountSucceeded++; } } else { // TODO asset-http: handle invalid size case LL_WARNS("ViewerAsset") << "bad size" << LL_ENDL; result_code = LL_ERR_ASSET_REQUEST_FAILED; ext_status = LLExtStat::NONE; } } // Clean up pending downloads and trigger callbacks removeAndCallbackPendingDownloads(uuid, atype, uuid, atype, result_code, ext_status); } std::string LLViewerAssetStorage::getAssetURL(const std::string& cap_url, const LLUUID& uuid, LLAssetType::EType atype) { std::string type_name = LLAssetType::lookup(atype); std::string url = cap_url + "/?" + type_name + "_id=" + uuid.asString(); return url; } void LLViewerAssetStorage::logAssetStorageInfo() { LLMemory::logMemoryInfo(true); LL_INFOS("AssetStorage") << "Active coros " << sAssetCoroCount << LL_ENDL; LL_INFOS("AssetStorage") << "mPendingDownloads size " << mPendingDownloads.size() << LL_ENDL; LL_INFOS("AssetStorage") << "mCountStarted " << mCountStarted << LL_ENDL; LL_INFOS("AssetStorage") << "mCountCompleted " << mCountCompleted << LL_ENDL; LL_INFOS("AssetStorage") << "mCountSucceeded " << mCountSucceeded << LL_ENDL; LL_INFOS("AssetStorage") << "mTotalBytesFetched " << mTotalBytesFetched << LL_ENDL; }