diff options
Diffstat (limited to 'indra/newview/lltexturefetch.cpp')
| -rw-r--r-- | indra/newview/lltexturefetch.cpp | 902 |
1 files changed, 805 insertions, 97 deletions
diff --git a/indra/newview/lltexturefetch.cpp b/indra/newview/lltexturefetch.cpp index 7fa04ce574..18c3a3b87d 100644 --- a/indra/newview/lltexturefetch.cpp +++ b/indra/newview/lltexturefetch.cpp @@ -2,37 +2,32 @@ * @file lltexturefetch.cpp * @brief Object which fetches textures from the cache and/or network * - * $LicenseInfo:firstyear=2000&license=viewergpl$ - * - * Copyright (c) 2000-2009, Linden Research, Inc. - * + * $LicenseInfo:firstyear=2000&license=viewerlgpl$ * Second Life Viewer Source Code - * The source code in this file ("Source Code") is provided by Linden Lab - * to you under the terms of the GNU General Public License, version 2.0 - * ("GPL"), unless you have obtained a separate licensing agreement - * ("Other License"), formally executed by you and Linden Lab. Terms of - * the GPL can be found in doc/GPL-license.txt in this distribution, or - * online at http://secondlifegrid.net/programs/open_source/licensing/gplv2 + * Copyright (C) 2010, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. * - * There are special exceptions to the terms and conditions of the GPL as - * it is applied to this Source Code. View the full text of the exception - * in the file doc/FLOSS-exception.txt in this software distribution, or - * online at - * http://secondlifegrid.net/programs/open_source/licensing/flossexception + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. * - * By copying, modifying or distributing this software, you acknowledge - * that you have read and understood your obligations described above, - * and agree to abide by those obligations. + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA * - * ALL LINDEN LAB SOURCE CODE IS PROVIDED "AS IS." LINDEN LAB MAKES NO - * WARRANTIES, EXPRESS, IMPLIED OR OTHERWISE, REGARDING ITS ACCURACY, - * COMPLETENESS OR PERFORMANCE. + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA * $/LicenseInfo$ */ #include "llviewerprecompiledheaders.h" #include <iostream> +#include <map> #include "llstl.h" @@ -54,6 +49,8 @@ #include "llviewertexturelist.h" #include "llviewertexture.h" #include "llviewerregion.h" +#include "llviewerstats.h" +#include "llviewerassetstats.h" #include "llworld.h" ////////////////////////////////////////////////////////////////////////////// @@ -148,9 +145,9 @@ public: /*virtual*/ bool deleteOK(); // called from update() (WORK THREAD) ~LLTextureFetchWorker(); - void relese() { --mActiveCount; } + // void relese() { --mActiveCount; } - void callbackHttpGet(const LLChannelDescriptors& channels, + S32 callbackHttpGet(const LLChannelDescriptors& channels, const LLIOPipe::buffer_ptr_t& buffer, bool partial, bool success); void callbackCacheRead(bool success, LLImageFormatted* image, @@ -166,9 +163,11 @@ public: mGetReason = reason; } - void setCanUseHTTP(bool can_use_http) {mCanUseHTTP = can_use_http;} - bool getCanUseHTTP()const {return mCanUseHTTP ;} + void setCanUseHTTP(bool can_use_http) { mCanUseHTTP = can_use_http; } + bool getCanUseHTTP() const { return mCanUseHTTP; } + LLTextureFetch & getFetcher() { return *mFetcher; } + protected: LLTextureFetchWorker(LLTextureFetch* fetcher, const std::string& url, const LLUUID& id, const LLHost& host, F32 priority, S32 discard, S32 size); @@ -260,6 +259,7 @@ private: BOOL mHaveAllData; BOOL mInLocalCache; bool mCanUseHTTP ; + bool mCanUseNET ; //can get from asset server. S32 mHTTPFailCount; S32 mRetryAttempt; S32 mActiveCount; @@ -281,6 +281,8 @@ private: S32 mLastPacket; U16 mTotalPackets; U8 mImageCodec; + + LLViewerAssetStats::duration_t mMetricsStartTime; }; ////////////////////////////////////////////////////////////////////////////// @@ -289,8 +291,8 @@ class HTTPGetResponder : public LLCurl::Responder { LOG_CLASS(HTTPGetResponder); public: - HTTPGetResponder(LLTextureFetch* fetcher, const LLUUID& id, U64 startTime, S32 requestedSize, U32 offset) - : mFetcher(fetcher), mID(id), mStartTime(startTime), mRequestedSize(requestedSize), mOffset(offset) + HTTPGetResponder(LLTextureFetch* fetcher, const LLUUID& id, U64 startTime, S32 requestedSize, U32 offset, bool redir) + : mFetcher(fetcher), mID(id), mStartTime(startTime), mRequestedSize(requestedSize), mOffset(offset), mFollowRedir(redir) { } ~HTTPGetResponder() @@ -303,6 +305,7 @@ public: { static LLCachedControl<bool> log_to_viewer_log(gSavedSettings,"LogTextureDownloadsToViewerLog"); static LLCachedControl<bool> log_to_sim(gSavedSettings,"LogTextureDownloadsToSimulator"); + static LLCachedControl<bool> log_texture_traffic(gSavedSettings,"LogTextureNetworkTraffic") ; if (log_to_viewer_log || log_to_sim) { @@ -328,18 +331,37 @@ public: partial = true; } } - else - { - worker->setGetStatus(status, reason); -// llwarns << status << ": " << reason << llendl; - } + if (!success) { worker->setGetStatus(status, reason); // llwarns << "CURL GET FAILED, status:" << status << " reason:" << reason << llendl; } - mFetcher->removeFromHTTPQueue(mID); - worker->callbackHttpGet(channels, buffer, partial, success); + + S32 data_size = worker->callbackHttpGet(channels, buffer, partial, success); + + if(log_texture_traffic && data_size > 0) + { + LLViewerTexture* tex = LLViewerTextureManager::findTexture(mID) ; + if(tex) + { + gTotalTextureBytesPerBoostLevel[tex->getBoostLevel()] += data_size ; + } + } + + mFetcher->removeFromHTTPQueue(mID, data_size); + + if (worker->mMetricsStartTime) + { + LLViewerAssetStatsFF::record_response_thread1(LLViewerAssetType::AT_TEXTURE, + true, + LLImageBase::TYPE_AVATAR_BAKE == worker->mType, + LLViewerAssetStatsFF::get_timestamp() - worker->mMetricsStartTime); + worker->mMetricsStartTime = 0; + } + LLViewerAssetStatsFF::record_dequeue_thread1(LLViewerAssetType::AT_TEXTURE, + true, + LLImageBase::TYPE_AVATAR_BAKE == worker->mType); } else { @@ -347,6 +369,11 @@ public: llwarns << "Worker not found: " << mID << llendl; } } + + virtual bool followRedir() + { + return mFollowRedir; + } private: LLTextureFetch* mFetcher; @@ -354,10 +381,234 @@ private: U64 mStartTime; S32 mRequestedSize; U32 mOffset; + bool mFollowRedir; }; ////////////////////////////////////////////////////////////////////////////// +// Cross-thread messaging for asset metrics. + +/** + * @brief Base class for cross-thread requests made of the fetcher + * + * I believe the intent of the LLQueuedThread class was to + * have these operations derived from LLQueuedThread::QueuedRequest + * but the texture fetcher has elected to manage the queue + * in its own manner. So these are free-standing objects which are + * managed in simple FIFO order on the mCommands queue of the + * LLTextureFetch object. + * + * What each represents is a simple command sent from an + * outside thread into the TextureFetch thread to be processed + * in order and in a timely fashion (though not an absolute + * higher priority than other operations of the thread). + * Each operation derives a new class from the base customizing + * members, constructors and the doWork() method to effect + * the command. + * + * The flow is one-directional. There are two global instances + * of the LLViewerAssetStats collector, one for the main program's + * thread pointed to by gViewerAssetStatsMain and one for the + * TextureFetch thread pointed to by gViewerAssetStatsThread1. + * Common operations has each thread recording metrics events + * into the respective collector unconcerned with locking and + * the state of any other thread. But when the agent moves into + * a different region or the metrics timer expires and a report + * needs to be sent back to the grid, messaging across threads + * is required to distribute data and perform global actions. + * In pseudo-UML, it looks like: + * + * Main Thread1 + * . . + * . . + * +-----+ . + * | AM | . + * +--+--+ . + * +-------+ | . + * | Main | +--+--+ . + * | | | SRE |---. . + * | Stats | +-----+ \ . + * | | | \ (uuid) +-----+ + * | Coll. | +--+--+ `-------->| SR | + * +-------+ | MSC | +--+--+ + * | ^ +-----+ | + * | | (uuid) / . +-----+ (uuid) + * | `--------' . | MSC |---------. + * | . +-----+ | + * | +-----+ . v + * | | TE | . +-------+ + * | +--+--+ . | Thd1 | + * | | . | | + * | +-----+ . | Stats | + * `--------->| RSC | . | | + * +--+--+ . | Coll. | + * | . +-------+ + * +--+--+ . | + * | SME |---. . | + * +-----+ \ . | + * . \ (clone) +-----+ | + * . `-------->| SM | | + * . +--+--+ | + * . | | + * . +-----+ | + * . | RSC |<--------' + * . +-----+ + * . | + * . +-----+ + * . | CP |--> HTTP POST + * . +-----+ + * . . + * . . + * + * + * Key: + * + * SRE - Set Region Enqueued. Enqueue a 'Set Region' command in + * the other thread providing the new UUID of the region. + * TFReqSetRegion carries the data. + * SR - Set Region. New region UUID is sent to the thread-local + * collector. + * SME - Send Metrics Enqueued. Enqueue a 'Send Metrics' command + * including an ownership transfer of a cloned LLViewerAssetStats. + * TFReqSendMetrics carries the data. + * SM - Send Metrics. Global metrics reporting operation. Takes + * the cloned stats from the command, merges it with the + * thread's local stats, converts to LLSD and sends it on + * to the grid. + * AM - Agent Moved. Agent has completed some sort of move to a + * new region. + * TE - Timer Expired. Metrics timer has expired (on the order + * of 10 minutes). + * CP - CURL Post + * MSC - Modify Stats Collector. State change in the thread-local + * collector. Typically a region change which affects the + * global pointers used to find the 'current stats'. + * RSC - Read Stats Collector. Extract collector data cloning it + * (i.e. deep copy) when necessary. + * + */ +class LLTextureFetch::TFRequest // : public LLQueuedThread::QueuedRequest +{ +public: + // Default ctors and assignment operator are correct. + + virtual ~TFRequest() + {} + + // Patterned after QueuedRequest's method but expected behavior + // is different. Always expected to complete on the first call + // and work dispatcher will assume the same and delete the + // request after invocation. + virtual bool doWork(LLTextureFetch * fetcher) = 0; +}; + +namespace +{ + +/** + * @brief Implements a 'Set Region' cross-thread command. + * + * When an agent moves to a new region, subsequent metrics need + * to be binned into a new or existing stats collection in 1:1 + * relationship with the region. We communicate this region + * change across the threads involved in the communication with + * this message. + * + * Corresponds to LLTextureFetch::commandSetRegion() + */ +class TFReqSetRegion : public LLTextureFetch::TFRequest +{ +public: + TFReqSetRegion(U64 region_handle) + : LLTextureFetch::TFRequest(), + mRegionHandle(region_handle) + {} + TFReqSetRegion & operator=(const TFReqSetRegion &); // Not defined + + virtual ~TFReqSetRegion() + {} + + virtual bool doWork(LLTextureFetch * fetcher); + +public: + const U64 mRegionHandle; +}; + + +/** + * @brief Implements a 'Send Metrics' cross-thread command. + * + * This is the big operation. The main thread gathers metrics + * for a period of minutes into LLViewerAssetStats and other + * objects then makes a snapshot of the data by cloning the + * collector. This command transfers the clone, along with a few + * additional arguments (UUIDs), handing ownership to the + * TextureFetch thread. It then merges its own data into the + * cloned copy, converts to LLSD and kicks off an HTTP POST of + * the resulting data to the currently active metrics collector. + * + * Corresponds to LLTextureFetch::commandSendMetrics() + */ +class TFReqSendMetrics : public LLTextureFetch::TFRequest +{ +public: + /** + * Construct the 'Send Metrics' command to have the TextureFetch + * thread add and log metrics data. + * + * @param caps_url URL of a "ViewerMetrics" Caps target + * to receive the data. Does not have to + * be associated with a particular region. + * + * @param session_id UUID of the agent's session. + * + * @param agent_id UUID of the agent. (Being pure here...) + * + * @param main_stats Pointer to a clone of the main thread's + * LLViewerAssetStats data. Thread1 takes + * ownership of the copy and disposes of it + * when done. + */ + TFReqSendMetrics(const std::string & caps_url, + const LLUUID & session_id, + const LLUUID & agent_id, + LLViewerAssetStats * main_stats) + : LLTextureFetch::TFRequest(), + mCapsURL(caps_url), + mSessionID(session_id), + mAgentID(agent_id), + mMainStats(main_stats) + {} + TFReqSendMetrics & operator=(const TFReqSendMetrics &); // Not defined + + virtual ~TFReqSendMetrics(); + + virtual bool doWork(LLTextureFetch * fetcher); + +public: + const std::string mCapsURL; + const LLUUID mSessionID; + const LLUUID mAgentID; + LLViewerAssetStats * mMainStats; +}; + +/* + * Examines the merged viewer metrics report and if found to be too long, + * will attempt to truncate it in some reasonable fashion. + * + * @param max_regions Limit of regions allowed in report. + * + * @param metrics Full, merged viewer metrics report. + * + * @returns If data was truncated, returns true. + */ +bool truncate_viewer_metrics(int max_regions, LLSD & metrics); + +} // end of anonymous namespace + + +////////////////////////////////////////////////////////////////////////////// + //static const char* LLTextureFetchWorker::sStateDescs[] = { "INVALID", @@ -375,6 +626,9 @@ const char* LLTextureFetchWorker::sStateDescs[] = { "DONE", }; +// static +volatile bool LLTextureFetch::svMetricsDataBreak(true); // Start with a data break + // called from MAIN THREAD LLTextureFetchWorker::LLTextureFetchWorker(LLTextureFetch* fetcher, @@ -424,8 +678,11 @@ LLTextureFetchWorker::LLTextureFetchWorker(LLTextureFetch* fetcher, mFirstPacket(0), mLastPacket(-1), mTotalPackets(0), - mImageCodec(IMG_CODEC_INVALID) + mImageCodec(IMG_CODEC_INVALID), + mMetricsStartTime(0) { + mCanUseNET = mUrl.empty() ; + calcWorkPriority(); mType = host.isOk() ? LLImageBase::TYPE_AVATAR_BAKE : LLImageBase::TYPE_NORMAL; // llinfos << "Create: " << mID << " mHost:" << host << " Discard=" << discard << llendl; @@ -583,14 +840,27 @@ bool LLTextureFetchWorker::doWork(S32 param) { LLMutexLock lock(&mWorkMutex); - if ((mFetcher->isQuitting() || mImagePriority < 1.0f || getFlags(LLWorkerClass::WCF_DELETE_REQUESTED))) + if ((mFetcher->isQuitting() || getFlags(LLWorkerClass::WCF_DELETE_REQUESTED))) { if (mState < DECODE_IMAGE) { return true; // abort } } - + + if(mImagePriority < F_ALMOST_ZERO) + { + if (mState == INIT || mState == LOAD_FROM_NETWORK || mState == LOAD_FROM_SIMULATOR) + { + return true; // abort + } + } + if(mState > CACHE_POST && !mCanUseNET && !mCanUseHTTP) + { + //nowhere to get data, abort. + return true ; + } + if (mFetcher->mDebugPause) { return false; // debug: don't do any work @@ -750,17 +1020,22 @@ bool LLTextureFetchWorker::doWork(S32 param) if (region) { - std::string http_url = region->getCapability("GetTexture"); + std::string http_url = region->getHttpUrl() ; if (!http_url.empty()) { mUrl = http_url + "/?texture_id=" + mID.asString().c_str(); mWriteToCacheState = CAN_WRITE ; //because this texture has a fixed texture id. } + else + { + mCanUseHTTP = false ; + } } else { // This will happen if not logged in or if a region deoes not have HTTP Texture enabled //llwarns << "Region not found for host: " << mHost << llendl; + mCanUseHTTP = false; } } if (mCanUseHTTP && !mUrl.empty()) @@ -773,7 +1048,7 @@ bool LLTextureFetchWorker::doWork(S32 param) } // don't return, fall through to next state } - else if (mSentRequest == UNSENT) + else if (mSentRequest == UNSENT && mCanUseNET) { // Add this to the network queue and sit here. // LLTextureFetch::update() will send off a request which will change our state @@ -782,7 +1057,15 @@ bool LLTextureFetchWorker::doWork(S32 param) mRequestedDiscard = mDesiredDiscard; mSentRequest = QUEUED; mFetcher->addToNetworkQueue(this); + if (! mMetricsStartTime) + { + mMetricsStartTime = LLViewerAssetStatsFF::get_timestamp(); + } + LLViewerAssetStatsFF::record_enqueue_thread1(LLViewerAssetType::AT_TEXTURE, + false, + LLImageBase::TYPE_AVATAR_BAKE == mType); setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); + return false; } else @@ -791,6 +1074,12 @@ bool LLTextureFetchWorker::doWork(S32 param) //llassert_always(mFetcher->mNetworkQueue.find(mID) != mFetcher->mNetworkQueue.end()); // Make certain this is in the network queue //mFetcher->addToNetworkQueue(this); + //if (! mMetricsStartTime) + //{ + // mMetricsStartTime = LLViewerAssetStatsFF::get_timestamp(); + //} + //LLViewerAssetStatsFF::record_enqueue_thread1(LLViewerAssetType::AT_TEXTURE, false, + // LLImageBase::TYPE_AVATAR_BAKE == mType); //setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); return false; } @@ -814,11 +1103,30 @@ bool LLTextureFetchWorker::doWork(S32 param) } setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); mState = DECODE_IMAGE; - mWriteToCacheState = SHOULD_WRITE ; + mWriteToCacheState = SHOULD_WRITE; + + if (mMetricsStartTime) + { + LLViewerAssetStatsFF::record_response_thread1(LLViewerAssetType::AT_TEXTURE, + false, + LLImageBase::TYPE_AVATAR_BAKE == mType, + LLViewerAssetStatsFF::get_timestamp() - mMetricsStartTime); + mMetricsStartTime = 0; + } + LLViewerAssetStatsFF::record_dequeue_thread1(LLViewerAssetType::AT_TEXTURE, + false, + LLImageBase::TYPE_AVATAR_BAKE == mType); } else { mFetcher->addToNetworkQueue(this); // failsafe + if (! mMetricsStartTime) + { + mMetricsStartTime = LLViewerAssetStatsFF::get_timestamp(); + } + LLViewerAssetStatsFF::record_enqueue_thread1(LLViewerAssetType::AT_TEXTURE, + false, + LLImageBase::TYPE_AVATAR_BAKE == mType); setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); } return false; @@ -826,22 +1134,19 @@ bool LLTextureFetchWorker::doWork(S32 param) if (mState == SEND_HTTP_REQ) { + if(mCanUseHTTP) { - const S32 HTTP_QUEUE_MAX_SIZE = 8; - // *TODO: Integrate this with llviewerthrottle - // Note: LLViewerThrottle uses dynamic throttling which makes sense for UDP, - // but probably not for Textures. - // Set the throttle to the entire bandwidth, assuming UDP packets will get priority - // when they are needed - F32 max_bandwidth = mFetcher->mMaxBandwidth; - if ((mFetcher->getHTTPQueueSize() >= HTTP_QUEUE_MAX_SIZE) || - (mFetcher->getTextureBandwidth() > max_bandwidth)) + //NOTE: + //control the number of the http requests issued for: + //1, not openning too many file descriptors at the same time; + //2, control the traffic of http so udp gets bandwidth. + // + static const S32 MAX_NUM_OF_HTTP_REQUESTS_IN_QUEUE = 8 ; + if(mFetcher->getNumHTTPRequests() > MAX_NUM_OF_HTTP_REQUESTS_IN_QUEUE) { - // Make normal priority and return (i.e. wait until there is room in the queue) - setPriority(LLWorkerThread::PRIORITY_NORMAL | mWorkPriority); - return false; + return false ; //wait. } - + mFetcher->removeFromNetworkQueue(this, false); S32 cur_size = 0; @@ -850,10 +1155,17 @@ bool LLTextureFetchWorker::doWork(S32 param) cur_size = mFormattedImage->getDataSize(); // amount of data we already have if (mFormattedImage->getDiscardLevel() == 0) { - // We already have all the data, just decode it - mLoadedDiscard = mFormattedImage->getDiscardLevel(); - mState = DECODE_IMAGE; - return false; + if(cur_size > 0) + { + // We already have all the data, just decode it + mLoadedDiscard = mFormattedImage->getDiscardLevel(); + mState = DECODE_IMAGE; + return false; + } + else + { + return true ; //abort. + } } } mRequestedSize = mDesiredSize; @@ -870,17 +1182,25 @@ bool LLTextureFetchWorker::doWork(S32 param) mGetReason.clear(); LL_DEBUGS("Texture") << "HTTP GET: " << mID << " Offset: " << offset << " Bytes: " << mRequestedSize - << " Bandwidth(kbps): " << mFetcher->getTextureBandwidth() << "/" << max_bandwidth + << " Bandwidth(kbps): " << mFetcher->getTextureBandwidth() << "/" << mFetcher->mMaxBandwidth << LL_ENDL; setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); mState = WAIT_HTTP_REQ; mFetcher->addToHTTPQueue(mID); + if (! mMetricsStartTime) + { + mMetricsStartTime = LLViewerAssetStatsFF::get_timestamp(); + } + LLViewerAssetStatsFF::record_enqueue_thread1(LLViewerAssetType::AT_TEXTURE, + true, + LLImageBase::TYPE_AVATAR_BAKE == mType); + // Will call callbackHttpGet when curl request completes std::vector<std::string> headers; headers.push_back("Accept: image/x-j2c"); res = mFetcher->mCurlGetRequest->getByteRange(mUrl, headers, offset, mRequestedSize, - new HTTPGetResponder(mFetcher, mID, LLTimer::getTotalTime(), mRequestedSize, offset)); + new HTTPGetResponder(mFetcher, mID, LLTimer::getTotalTime(), mRequestedSize, offset, true)); } if (!res) { @@ -891,6 +1211,10 @@ bool LLTextureFetchWorker::doWork(S32 param) } // fall through } + else //can not use http fetch. + { + return true ; //abort + } } if (mState == WAIT_HTTP_REQ) @@ -904,13 +1228,16 @@ bool LLTextureFetchWorker::doWork(S32 param) if (mGetStatus == HTTP_NOT_FOUND) { mHTTPFailCount = max_attempts = 1; // Don't retry - //llinfos << "Texture missing from server (404): " << mUrl << llendl; + llwarns << "Texture missing from server (404): " << mUrl << llendl; //roll back to try UDP - mState = INIT ; - mCanUseHTTP = false ; - setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); - return false ; + if(mCanUseNET) + { + mState = INIT ; + mCanUseHTTP = false ; + setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); + return false ; + } } else if (mGetStatus == HTTP_SERVICE_UNAVAILABLE) { @@ -930,6 +1257,7 @@ bool LLTextureFetchWorker::doWork(S32 param) << " Status: " << mGetStatus << " Reason: '" << mGetReason << "'" << " Attempt:" << mHTTPFailCount+1 << "/" << max_attempts << llendl; } + if (mHTTPFailCount >= max_attempts) { if (cur_size > 0) @@ -942,6 +1270,7 @@ bool LLTextureFetchWorker::doWork(S32 param) else { resetFormattedData(); + mState = DONE; return true; // failed } } @@ -952,6 +1281,17 @@ bool LLTextureFetchWorker::doWork(S32 param) } } + llassert_always(mBufferSize == cur_size + mRequestedSize); + if(!mBufferSize)//no data received. + { + delete[] mBuffer; + mBuffer = NULL; + + //abort. + mState = DONE; + return true; + } + if (mFormattedImage.isNull()) { // For now, create formatted image based on extension @@ -962,8 +1302,7 @@ bool LLTextureFetchWorker::doWork(S32 param) mFormattedImage = new LLImageJ2C; // default } } - - llassert_always(mBufferSize == cur_size + mRequestedSize); + if (mHaveAllData && mRequestedDiscard == 0) //the image file is fully loaded. { mFileSize = mBufferSize; @@ -1022,7 +1361,12 @@ bool LLTextureFetchWorker::doWork(S32 param) if (mFormattedImage->getDataSize() <= 0) { - llerrs << "Decode entered with invalid mFormattedImage. ID = " << mID << llendl; + //llerrs << "Decode entered with invalid mFormattedImage. ID = " << mID << llendl; + + //abort, don't decode + mState = DONE; + setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); + return true; } if (mLoadedDiscard < 0) { @@ -1219,8 +1563,7 @@ bool LLTextureFetchWorker::deleteOK() if ((haveWork() && // not ok to delete from these states - ((mState >= SEND_HTTP_REQ && mState <= WAIT_HTTP_REQ) || - (mState >= WRITE_TO_CACHE && mState <= WAIT_ON_WRITE)))) + ((mState >= WRITE_TO_CACHE && mState <= WAIT_ON_WRITE)))) { delete_ok = false; } @@ -1299,29 +1642,29 @@ bool LLTextureFetchWorker::processSimulatorPackets() ////////////////////////////////////////////////////////////////////////////// -void LLTextureFetchWorker::callbackHttpGet(const LLChannelDescriptors& channels, +S32 LLTextureFetchWorker::callbackHttpGet(const LLChannelDescriptors& channels, const LLIOPipe::buffer_ptr_t& buffer, bool partial, bool success) { + S32 data_size = 0 ; + LLMutexLock lock(&mWorkMutex); if (mState != WAIT_HTTP_REQ) { llwarns << "callbackHttpGet for unrequested fetch worker: " << mID << " req=" << mSentRequest << " state= " << mState << llendl; - return; + return data_size; } if (mLoaded) { llwarns << "Duplicate callback for " << mID.asString() << llendl; - return; // ignore duplicate callback + return data_size ; // ignore duplicate callback } if (success) { // get length of stream: - S32 data_size = buffer->countAfter(channels.in(), NULL); - - gTextureList.sTextureBits += data_size * 8; // Approximate - does not include header bits + data_size = buffer->countAfter(channels.in(), NULL); LL_DEBUGS("Texture") << "HTTP RECEIVED: " << mID.asString() << " Bytes: " << data_size << LL_ENDL; if (data_size > 0) @@ -1358,6 +1701,8 @@ void LLTextureFetchWorker::callbackHttpGet(const LLChannelDescriptors& channels, } mLoaded = TRUE; setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); + + return data_size ; } ////////////////////////////////////////////////////////////////////////////// @@ -1465,7 +1810,7 @@ bool LLTextureFetchWorker::writeToCacheComplete() ////////////////////////////////////////////////////////////////////////////// // public -LLTextureFetch::LLTextureFetch(LLTextureCache* cache, LLImageDecodeThread* imagedecodethread, bool threaded) +LLTextureFetch::LLTextureFetch(LLTextureCache* cache, LLImageDecodeThread* imagedecodethread, bool threaded, bool qa_mode) : LLWorkerThread("TextureFetch", threaded), mDebugCount(0), mDebugPause(FALSE), @@ -1476,8 +1821,12 @@ LLTextureFetch::LLTextureFetch(LLTextureCache* cache, LLImageDecodeThread* image mTextureCache(cache), mImageDecodeThread(imagedecodethread), mTextureBandwidth(0), - mCurlGetRequest(NULL) + mHTTPTextureBits(0), + mTotalHTTPRequests(0), + mCurlGetRequest(NULL), + mQAMode(qa_mode) { + mCurlPOSTRequestCount = 0; mMaxBandwidth = gSavedSettings.getF32("ThrottleBandwidthKBPS"); mTextureInfo.setUpLogging(gSavedSettings.getBOOL("LogTextureDownloadsToViewerLog"), gSavedSettings.getBOOL("LogTextureDownloadsToSimulator"), gSavedSettings.getU32("TextureLoggingThreshold")); } @@ -1486,6 +1835,13 @@ LLTextureFetch::~LLTextureFetch() { clearDeleteList() ; + while (! mCommands.empty()) + { + TFRequest * req(mCommands.front()); + mCommands.erase(mCommands.begin()); + delete req; + } + // ~LLQueuedThread() called here } @@ -1515,7 +1871,6 @@ bool LLTextureFetch::createRequest(const std::string& url, const LLUUID& id, con if (!url.empty() && (!exten.empty() && LLImageBase::getCodecFromExtension(exten) != IMG_CODEC_J2C)) { // Only do partial requests for J2C at the moment - //llinfos << "Merov : LLTextureFetch::createRequest(), blocking fetch on " << url << llendl; desired_size = MAX_IMAGE_DATA_SIZE; desired_discard = 0; } @@ -1619,12 +1974,14 @@ void LLTextureFetch::addToHTTPQueue(const LLUUID& id) { LLMutexLock lock(&mNetworkQueueMutex); mHTTPTextureQueue.insert(id); + mTotalHTTPRequests++; } -void LLTextureFetch::removeFromHTTPQueue(const LLUUID& id) +void LLTextureFetch::removeFromHTTPQueue(const LLUUID& id, S32 received_size) { LLMutexLock lock(&mNetworkQueueMutex); mHTTPTextureQueue.erase(id); + mHTTPTextureBits += received_size * 8; // Approximate - does not include header bits } void LLTextureFetch::deleteRequest(const LLUUID& id, bool cancel) @@ -1680,6 +2037,15 @@ S32 LLTextureFetch::getNumHTTPRequests() return size ; } +U32 LLTextureFetch::getTotalNumHTTPRequests() +{ + mNetworkQueueMutex.lock() ; + U32 size = mTotalHTTPRequests ; + mNetworkQueueMutex.unlock() ; + + return size ; +} + // call lockQueue() first! LLTextureFetchWorker* LLTextureFetch::getWorkerAfterLock(const LLUUID& id) { @@ -1766,18 +2132,93 @@ bool LLTextureFetch::updateRequestPriority(const LLUUID& id, F32 priority) return res; } +// Replicates and expands upon the base class's +// getPending() implementation. getPending() and +// runCondition() replicate one another's logic to +// an extent and are sometimes used for the same +// function (deciding whether or not to sleep/pause +// a thread). So the implementations need to stay +// in step, at least until this can be refactored and +// the redundancy eliminated. +// +// May be called from any thread + +//virtual +S32 LLTextureFetch::getPending() +{ + S32 res; + lockData(); + { + LLMutexLock lock(&mQueueMutex); + + res = mRequestQueue.size(); + res += mCurlPOSTRequestCount; + res += mCommands.size(); + } + unlockData(); + return res; +} + +// virtual +bool LLTextureFetch::runCondition() +{ + // Caller is holding the lock on LLThread's condition variable. + + // LLQueuedThread, unlike its base class LLThread, makes this a + // private method which is unfortunate. I want to use it directly + // but I'm going to have to re-implement the logic here (or change + // declarations, which I don't want to do right now). + // + // Changes here may need to be reflected in getPending(). + + bool have_no_commands(false); + { + LLMutexLock lock(&mQueueMutex); + + have_no_commands = mCommands.empty(); + } + + bool have_no_curl_requests(0 == mCurlPOSTRequestCount); + + return ! (have_no_commands + && have_no_curl_requests + && (mRequestQueue.empty() && mIdleThread)); // From base class +} + ////////////////////////////////////////////////////////////////////////////// +// MAIN THREAD (unthreaded envs), WORKER THREAD (threaded envs) +void LLTextureFetch::commonUpdate() +{ + // Run a cross-thread command, if any. + cmdDoWork(); + + // Update Curl on same thread as mCurlGetRequest was constructed + S32 processed = mCurlGetRequest->process(); + if (processed > 0) + { + lldebugs << "processed: " << processed << " messages." << llendl; + } +} + + // MAIN THREAD //virtual S32 LLTextureFetch::update(U32 max_time_ms) { - S32 res; - static LLCachedControl<F32> band_width(gSavedSettings,"ThrottleBandwidthKBPS"); - mMaxBandwidth = band_width ; - - res = LLWorkerThread::update(max_time_ms); + + { + mNetworkQueueMutex.lock() ; + mMaxBandwidth = band_width ; + + gTextureList.sTextureBits += mHTTPTextureBits ; + mHTTPTextureBits = 0 ; + + mNetworkQueueMutex.unlock() ; + } + + S32 res = LLWorkerThread::update(max_time_ms); if (!mDebugPause) { @@ -1786,14 +2227,9 @@ S32 LLTextureFetch::update(U32 max_time_ms) if (!mThreaded) { - // Update Curl on same thread as mCurlGetRequest was constructed - S32 processed = mCurlGetRequest->process(); - if (processed > 0) - { - lldebugs << "processed: " << processed << " messages." << llendl; - } + commonUpdate(); } - + return res; } @@ -1846,12 +2282,7 @@ void LLTextureFetch::threadedUpdate() } process_timer.reset(); - // Update Curl on same thread as mCurlGetRequest was constructed - S32 processed = mCurlGetRequest->process(); - if (processed > 0) - { - lldebugs << "processed: " << processed << " messages." << llendl; - } + commonUpdate(); #if 0 const F32 INFO_TIME = 1.0f; @@ -2301,3 +2732,280 @@ void LLTextureFetch::dump() } } +////////////////////////////////////////////////////////////////////////////// + +// cross-thread command methods + +void LLTextureFetch::commandSetRegion(U64 region_handle) +{ + TFReqSetRegion * req = new TFReqSetRegion(region_handle); + + cmdEnqueue(req); +} + +void LLTextureFetch::commandSendMetrics(const std::string & caps_url, + const LLUUID & session_id, + const LLUUID & agent_id, + LLViewerAssetStats * main_stats) +{ + TFReqSendMetrics * req = new TFReqSendMetrics(caps_url, session_id, agent_id, main_stats); + + cmdEnqueue(req); +} + +void LLTextureFetch::commandDataBreak() +{ + // The pedantically correct way to implement this is to create a command + // request object in the above fashion and enqueue it. However, this is + // simple data of an advisorial not operational nature and this case + // of shared-write access is tolerable. + + LLTextureFetch::svMetricsDataBreak = true; +} + +void LLTextureFetch::cmdEnqueue(TFRequest * req) +{ + lockQueue(); + mCommands.push_back(req); + unlockQueue(); + + unpause(); +} + +LLTextureFetch::TFRequest * LLTextureFetch::cmdDequeue() +{ + TFRequest * ret = 0; + + lockQueue(); + if (! mCommands.empty()) + { + ret = mCommands.front(); + mCommands.erase(mCommands.begin()); + } + unlockQueue(); + + return ret; +} + +void LLTextureFetch::cmdDoWork() +{ + if (mDebugPause) + { + return; // debug: don't do any work + } + + TFRequest * req = cmdDequeue(); + if (req) + { + // One request per pass should really be enough for this. + req->doWork(this); + delete req; + } +} + + +////////////////////////////////////////////////////////////////////////////// + +// Private (anonymous) class methods implementing the command scheme. + +namespace +{ + +/** + * Implements the 'Set Region' command. + * + * Thread: Thread1 (TextureFetch) + */ +bool +TFReqSetRegion::doWork(LLTextureFetch *) +{ + LLViewerAssetStatsFF::set_region_thread1(mRegionHandle); + + return true; +} + + +TFReqSendMetrics::~TFReqSendMetrics() +{ + delete mMainStats; + mMainStats = 0; +} + + +/** + * Implements the 'Send Metrics' command. Takes over + * ownership of the passed LLViewerAssetStats pointer. + * + * Thread: Thread1 (TextureFetch) + */ +bool +TFReqSendMetrics::doWork(LLTextureFetch * fetcher) +{ + /* + * HTTP POST responder. Doesn't do much but tries to + * detect simple breaks in recording the metrics stream. + * + * The 'volatile' modifiers don't indicate signals, + * mmap'd memory or threads, really. They indicate that + * the referenced data is part of a pseudo-closure for + * this responder rather than being required for correct + * operation. + * + * We don't try very hard with the POST request. We give + * it one shot and that's more-or-less it. With a proper + * refactoring of the LLQueuedThread usage, these POSTs + * could be put in a request object and made more reliable. + */ + class lcl_responder : public LLCurl::Responder + { + public: + lcl_responder(LLTextureFetch * fetcher, + S32 expected_sequence, + volatile const S32 & live_sequence, + volatile bool & reporting_break, + volatile bool & reporting_started) + : LLCurl::Responder(), + mFetcher(fetcher), + mExpectedSequence(expected_sequence), + mLiveSequence(live_sequence), + mReportingBreak(reporting_break), + mReportingStarted(reporting_started) + { + mFetcher->incrCurlPOSTCount(); + } + + ~lcl_responder() + { + mFetcher->decrCurlPOSTCount(); + } + + // virtual + void error(U32 status_num, const std::string & reason) + { + if (mLiveSequence == mExpectedSequence) + { + mReportingBreak = true; + } + LL_WARNS("Texture") << "Break in metrics stream due to POST failure to metrics collection service. Reason: " + << reason << LL_ENDL; + } + + // virtual + void result(const LLSD & content) + { + if (mLiveSequence == mExpectedSequence) + { + mReportingBreak = false; + mReportingStarted = true; + } + } + + private: + LLTextureFetch * mFetcher; + S32 mExpectedSequence; + volatile const S32 & mLiveSequence; + volatile bool & mReportingBreak; + volatile bool & mReportingStarted; + + }; // class lcl_responder + + if (! gViewerAssetStatsThread1) + return true; + + static volatile bool reporting_started(false); + static volatile S32 report_sequence(0); + + // We've taken over ownership of the stats copy at this + // point. Get a working reference to it for merging here + // but leave it in 'this'. Destructor will rid us of it. + LLViewerAssetStats & main_stats = *mMainStats; + + // Merge existing stats into those from main, convert to LLSD + main_stats.merge(*gViewerAssetStatsThread1); + LLSD merged_llsd = main_stats.asLLSD(true); + + // Add some additional meta fields to the content + merged_llsd["session_id"] = mSessionID; + merged_llsd["agent_id"] = mAgentID; + merged_llsd["message"] = "ViewerAssetMetrics"; // Identifies the type of metrics + merged_llsd["sequence"] = report_sequence; // Sequence number + merged_llsd["initial"] = ! reporting_started; // Initial data from viewer + merged_llsd["break"] = LLTextureFetch::svMetricsDataBreak; // Break in data prior to this report + + // Update sequence number + if (S32_MAX == ++report_sequence) + report_sequence = 0; + + // Limit the size of the stats report if necessary. + merged_llsd["truncated"] = truncate_viewer_metrics(10, merged_llsd); + + if (! mCapsURL.empty()) + { + LLCurlRequest::headers_t headers; + fetcher->getCurlRequest().post(mCapsURL, + headers, + merged_llsd, + new lcl_responder(fetcher, + report_sequence, + report_sequence, + LLTextureFetch::svMetricsDataBreak, + reporting_started)); + } + else + { + LLTextureFetch::svMetricsDataBreak = true; + } + + // In QA mode, Metrics submode, log the result for ease of testing + if (fetcher->isQAMode()) + { + LL_INFOS("Textures") << merged_llsd << LL_ENDL; + } + + gViewerAssetStatsThread1->reset(); + + return true; +} + + +bool +truncate_viewer_metrics(int max_regions, LLSD & metrics) +{ + static const LLSD::String reg_tag("regions"); + static const LLSD::String duration_tag("duration"); + + LLSD & reg_map(metrics[reg_tag]); + if (reg_map.size() <= max_regions) + { + return false; + } + + // Build map of region hashes ordered by duration + typedef std::multimap<LLSD::Real, int> reg_ordered_list_t; + reg_ordered_list_t regions_by_duration; + + int ind(0); + LLSD::array_const_iterator it_end(reg_map.endArray()); + for (LLSD::array_const_iterator it(reg_map.beginArray()); it_end != it; ++it, ++ind) + { + LLSD::Real duration = (*it)[duration_tag].asReal(); + regions_by_duration.insert(reg_ordered_list_t::value_type(duration, ind)); + } + + // Build a replacement regions array with the longest-persistence regions + LLSD new_region(LLSD::emptyArray()); + reg_ordered_list_t::const_reverse_iterator it2_end(regions_by_duration.rend()); + reg_ordered_list_t::const_reverse_iterator it2(regions_by_duration.rbegin()); + for (int i(0); i < max_regions && it2_end != it2; ++i, ++it2) + { + new_region.append(reg_map[it2->second]); + } + reg_map = new_region; + + return true; +} + +} // end of anonymous namespace + + + |