From dc75235e38e3f2d74476607e6196b26d36955619 Mon Sep 17 00:00:00 2001 From: Xiaohong Bao Date: Mon, 13 Dec 2010 21:09:52 -0700 Subject: debug code for SH-639: http requests 3X more in viewer 2.4 --- indra/newview/lltexturefetch.cpp | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) (limited to 'indra/newview/lltexturefetch.cpp') diff --git a/indra/newview/lltexturefetch.cpp b/indra/newview/lltexturefetch.cpp index d6d38de225..c1346b6768 100644 --- a/indra/newview/lltexturefetch.cpp +++ b/indra/newview/lltexturefetch.cpp @@ -862,7 +862,7 @@ bool LLTextureFetchWorker::doWork(S32 param) //1, not openning too many file descriptors at the same time; //2, control the traffic of http so udp gets bandwidth. // - static const S32 MAX_NUM_OF_HTTP_REQUESTS_IN_QUEUE = 32 ; + static const S32 MAX_NUM_OF_HTTP_REQUESTS_IN_QUEUE = 8 ; if(mFetcher->getNumHTTPRequests() > MAX_NUM_OF_HTTP_REQUESTS_IN_QUEUE) { return false ; //wait. @@ -1535,6 +1535,7 @@ LLTextureFetch::LLTextureFetch(LLTextureCache* cache, LLImageDecodeThread* image mImageDecodeThread(imagedecodethread), mTextureBandwidth(0), mHTTPTextureBits(0), + mTotalHTTPRequests(0), mCurlGetRequest(NULL) { mMaxBandwidth = gSavedSettings.getF32("ThrottleBandwidthKBPS"); @@ -1678,6 +1679,7 @@ void LLTextureFetch::addToHTTPQueue(const LLUUID& id) { LLMutexLock lock(&mNetworkQueueMutex); mHTTPTextureQueue.insert(id); + mTotalHTTPRequests++; } void LLTextureFetch::removeFromHTTPQueue(const LLUUID& id, S32 received_size) @@ -1740,6 +1742,15 @@ S32 LLTextureFetch::getNumHTTPRequests() return size ; } +U32 LLTextureFetch::getTotalNumHTTPRequests() +{ + mNetworkQueueMutex.lock() ; + U32 size = mTotalHTTPRequests ; + mNetworkQueueMutex.unlock() ; + + return size ; +} + // call lockQueue() first! LLTextureFetchWorker* LLTextureFetch::getWorkerAfterLock(const LLUUID& id) { -- cgit v1.2.3 From 2e2d990eb10d101982b4f1915780a49615b55d6e Mon Sep 17 00:00:00 2001 From: Oz Linden Date: Sat, 29 Jan 2011 13:09:55 -0500 Subject: correct DOS line endings --- indra/newview/lltexturefetch.cpp | 6022 +++++++++++++++++++------------------- 1 file changed, 3011 insertions(+), 3011 deletions(-) (limited to 'indra/newview/lltexturefetch.cpp') diff --git a/indra/newview/lltexturefetch.cpp b/indra/newview/lltexturefetch.cpp index 5cdf1706e6..18c3a3b87d 100644 --- a/indra/newview/lltexturefetch.cpp +++ b/indra/newview/lltexturefetch.cpp @@ -1,3011 +1,3011 @@ -/** - * @file lltexturefetch.cpp - * @brief Object which fetches textures from the cache and/or network - * - * $LicenseInfo:firstyear=2000&license=viewerlgpl$ - * Second Life Viewer Source Code - * Copyright (C) 2010, Linden Research, Inc. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; - * version 2.1 of the License only. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - * - * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA - * $/LicenseInfo$ - */ - -#include "llviewerprecompiledheaders.h" - -#include -#include - -#include "llstl.h" - -#include "lltexturefetch.h" - -#include "llcurl.h" -#include "lldir.h" -#include "llhttpclient.h" -#include "llhttpstatuscodes.h" -#include "llimage.h" -#include "llimagej2c.h" -#include "llimageworker.h" -#include "llworkerthread.h" -#include "message.h" - -#include "llagent.h" -#include "lltexturecache.h" -#include "llviewercontrol.h" -#include "llviewertexturelist.h" -#include "llviewertexture.h" -#include "llviewerregion.h" -#include "llviewerstats.h" -#include "llviewerassetstats.h" -#include "llworld.h" - -////////////////////////////////////////////////////////////////////////////// -class LLTextureFetchWorker : public LLWorkerClass -{ - friend class LLTextureFetch; - friend class HTTPGetResponder; - -private: - class CacheReadResponder : public LLTextureCache::ReadResponder - { - public: - CacheReadResponder(LLTextureFetch* fetcher, const LLUUID& id, LLImageFormatted* image) - : mFetcher(fetcher), mID(id) - { - setImage(image); - } - virtual void completed(bool success) - { - LLTextureFetchWorker* worker = mFetcher->getWorker(mID); - if (worker) - { - worker->callbackCacheRead(success, mFormattedImage, mImageSize, mImageLocal); - } - } - private: - LLTextureFetch* mFetcher; - LLUUID mID; - }; - - class CacheWriteResponder : public LLTextureCache::WriteResponder - { - public: - CacheWriteResponder(LLTextureFetch* fetcher, const LLUUID& id) - : mFetcher(fetcher), mID(id) - { - } - virtual void completed(bool success) - { - LLTextureFetchWorker* worker = mFetcher->getWorker(mID); - if (worker) - { - worker->callbackCacheWrite(success); - } - } - private: - LLTextureFetch* mFetcher; - LLUUID mID; - }; - - class DecodeResponder : public LLImageDecodeThread::Responder - { - public: - DecodeResponder(LLTextureFetch* fetcher, const LLUUID& id, LLTextureFetchWorker* worker) - : mFetcher(fetcher), mID(id), mWorker(worker) - { - } - virtual void completed(bool success, LLImageRaw* raw, LLImageRaw* aux) - { - LLTextureFetchWorker* worker = mFetcher->getWorker(mID); - if (worker) - { - worker->callbackDecoded(success, raw, aux); - } - } - private: - LLTextureFetch* mFetcher; - LLUUID mID; - LLTextureFetchWorker* mWorker; // debug only (may get deleted from under us, use mFetcher/mID) - }; - - struct Compare - { - // lhs < rhs - bool operator()(const LLTextureFetchWorker* lhs, const LLTextureFetchWorker* rhs) const - { - // greater priority is "less" - const F32 lpriority = lhs->mImagePriority; - const F32 rpriority = rhs->mImagePriority; - if (lpriority > rpriority) // higher priority - return true; - else if (lpriority < rpriority) - return false; - else - return lhs < rhs; - } - }; - -public: - /*virtual*/ bool doWork(S32 param); // Called from LLWorkerThread::processRequest() - /*virtual*/ void finishWork(S32 param, bool completed); // called from finishRequest() (WORK THREAD) - /*virtual*/ bool deleteOK(); // called from update() (WORK THREAD) - - ~LLTextureFetchWorker(); - // void relese() { --mActiveCount; } - - S32 callbackHttpGet(const LLChannelDescriptors& channels, - const LLIOPipe::buffer_ptr_t& buffer, - bool partial, bool success); - void callbackCacheRead(bool success, LLImageFormatted* image, - S32 imagesize, BOOL islocal); - void callbackCacheWrite(bool success); - void callbackDecoded(bool success, LLImageRaw* raw, LLImageRaw* aux); - - void setGetStatus(U32 status, const std::string& reason) - { - LLMutexLock lock(&mWorkMutex); - - mGetStatus = status; - mGetReason = reason; - } - - void setCanUseHTTP(bool can_use_http) { mCanUseHTTP = can_use_http; } - bool getCanUseHTTP() const { return mCanUseHTTP; } - - LLTextureFetch & getFetcher() { return *mFetcher; } - -protected: - LLTextureFetchWorker(LLTextureFetch* fetcher, const std::string& url, const LLUUID& id, const LLHost& host, - F32 priority, S32 discard, S32 size); - -private: - /*virtual*/ void startWork(S32 param); // called from addWork() (MAIN THREAD) - /*virtual*/ void endWork(S32 param, bool aborted); // called from doWork() (MAIN THREAD) - - void resetFormattedData(); - - void setImagePriority(F32 priority); - void setDesiredDiscard(S32 discard, S32 size); - bool insertPacket(S32 index, U8* data, S32 size); - void clearPackets(); - void setupPacketData(); - U32 calcWorkPriority(); - void removeFromCache(); - bool processSimulatorPackets(); - bool writeToCacheComplete(); - - void lockWorkMutex() { mWorkMutex.lock(); } - void unlockWorkMutex() { mWorkMutex.unlock(); } - -private: - enum e_state // mState - { - // NOTE: Affects LLTextureBar::draw in lltextureview.cpp (debug hack) - INVALID = 0, - INIT, - LOAD_FROM_TEXTURE_CACHE, - CACHE_POST, - LOAD_FROM_NETWORK, - LOAD_FROM_SIMULATOR, - SEND_HTTP_REQ, - WAIT_HTTP_REQ, - DECODE_IMAGE, - DECODE_IMAGE_UPDATE, - WRITE_TO_CACHE, - WAIT_ON_WRITE, - DONE - }; - enum e_request_state // mSentRequest - { - UNSENT = 0, - QUEUED = 1, - SENT_SIM = 2 - }; - enum e_write_to_cache_state //mWriteToCacheState - { - NOT_WRITE = 0, - CAN_WRITE = 1, - SHOULD_WRITE = 2 - }; - static const char* sStateDescs[]; - e_state mState; - e_write_to_cache_state mWriteToCacheState; - LLTextureFetch* mFetcher; - LLPointer mFormattedImage; - LLPointer mRawImage; - LLPointer mAuxImage; - LLUUID mID; - LLHost mHost; - std::string mUrl; - U8 mType; - F32 mImagePriority; - U32 mWorkPriority; - F32 mRequestedPriority; - S32 mDesiredDiscard; - S32 mSimRequestedDiscard; - S32 mRequestedDiscard; - S32 mLoadedDiscard; - S32 mDecodedDiscard; - LLFrameTimer mRequestedTimer; - LLFrameTimer mFetchTimer; - LLTextureCache::handle_t mCacheReadHandle; - LLTextureCache::handle_t mCacheWriteHandle; - U8* mBuffer; - S32 mBufferSize; - S32 mRequestedSize; - S32 mDesiredSize; - S32 mFileSize; - S32 mCachedSize; - e_request_state mSentRequest; - handle_t mDecodeHandle; - BOOL mLoaded; - BOOL mDecoded; - BOOL mWritten; - BOOL mNeedsAux; - BOOL mHaveAllData; - BOOL mInLocalCache; - bool mCanUseHTTP ; - bool mCanUseNET ; //can get from asset server. - S32 mHTTPFailCount; - S32 mRetryAttempt; - S32 mActiveCount; - U32 mGetStatus; - std::string mGetReason; - - // Work Data - LLMutex mWorkMutex; - struct PacketData - { - PacketData(U8* data, S32 size) { mData = data; mSize = size; } - ~PacketData() { clearData(); } - void clearData() { delete[] mData; mData = NULL; } - U8* mData; - U32 mSize; - }; - std::vector mPackets; - S32 mFirstPacket; - S32 mLastPacket; - U16 mTotalPackets; - U8 mImageCodec; - - LLViewerAssetStats::duration_t mMetricsStartTime; -}; - -////////////////////////////////////////////////////////////////////////////// - -class HTTPGetResponder : public LLCurl::Responder -{ - LOG_CLASS(HTTPGetResponder); -public: - HTTPGetResponder(LLTextureFetch* fetcher, const LLUUID& id, U64 startTime, S32 requestedSize, U32 offset, bool redir) - : mFetcher(fetcher), mID(id), mStartTime(startTime), mRequestedSize(requestedSize), mOffset(offset), mFollowRedir(redir) - { - } - ~HTTPGetResponder() - { - } - - virtual void completedRaw(U32 status, const std::string& reason, - const LLChannelDescriptors& channels, - const LLIOPipe::buffer_ptr_t& buffer) - { - static LLCachedControl log_to_viewer_log(gSavedSettings,"LogTextureDownloadsToViewerLog"); - static LLCachedControl log_to_sim(gSavedSettings,"LogTextureDownloadsToSimulator"); - static LLCachedControl log_texture_traffic(gSavedSettings,"LogTextureNetworkTraffic") ; - - if (log_to_viewer_log || log_to_sim) - { - mFetcher->mTextureInfo.setRequestStartTime(mID, mStartTime); - U64 timeNow = LLTimer::getTotalTime(); - mFetcher->mTextureInfo.setRequestType(mID, LLTextureInfoDetails::REQUEST_TYPE_HTTP); - mFetcher->mTextureInfo.setRequestSize(mID, mRequestedSize); - mFetcher->mTextureInfo.setRequestOffset(mID, mOffset); - mFetcher->mTextureInfo.setRequestCompleteTimeAndLog(mID, timeNow); - } - - lldebugs << "HTTP COMPLETE: " << mID << llendl; - LLTextureFetchWorker* worker = mFetcher->getWorker(mID); - if (worker) - { - bool success = false; - bool partial = false; - if (HTTP_OK <= status && status < HTTP_MULTIPLE_CHOICES) - { - success = true; - if (HTTP_PARTIAL_CONTENT == status) // partial information - { - partial = true; - } - } - - if (!success) - { - worker->setGetStatus(status, reason); -// llwarns << "CURL GET FAILED, status:" << status << " reason:" << reason << llendl; - } - - S32 data_size = worker->callbackHttpGet(channels, buffer, partial, success); - - if(log_texture_traffic && data_size > 0) - { - LLViewerTexture* tex = LLViewerTextureManager::findTexture(mID) ; - if(tex) - { - gTotalTextureBytesPerBoostLevel[tex->getBoostLevel()] += data_size ; - } - } - - mFetcher->removeFromHTTPQueue(mID, data_size); - - if (worker->mMetricsStartTime) - { - LLViewerAssetStatsFF::record_response_thread1(LLViewerAssetType::AT_TEXTURE, - true, - LLImageBase::TYPE_AVATAR_BAKE == worker->mType, - LLViewerAssetStatsFF::get_timestamp() - worker->mMetricsStartTime); - worker->mMetricsStartTime = 0; - } - LLViewerAssetStatsFF::record_dequeue_thread1(LLViewerAssetType::AT_TEXTURE, - true, - LLImageBase::TYPE_AVATAR_BAKE == worker->mType); - } - else - { - mFetcher->removeFromHTTPQueue(mID); - llwarns << "Worker not found: " << mID << llendl; - } - } - - virtual bool followRedir() - { - return mFollowRedir; - } - -private: - LLTextureFetch* mFetcher; - LLUUID mID; - U64 mStartTime; - S32 mRequestedSize; - U32 mOffset; - bool mFollowRedir; -}; - -////////////////////////////////////////////////////////////////////////////// - -// Cross-thread messaging for asset metrics. - -/** - * @brief Base class for cross-thread requests made of the fetcher - * - * I believe the intent of the LLQueuedThread class was to - * have these operations derived from LLQueuedThread::QueuedRequest - * but the texture fetcher has elected to manage the queue - * in its own manner. So these are free-standing objects which are - * managed in simple FIFO order on the mCommands queue of the - * LLTextureFetch object. - * - * What each represents is a simple command sent from an - * outside thread into the TextureFetch thread to be processed - * in order and in a timely fashion (though not an absolute - * higher priority than other operations of the thread). - * Each operation derives a new class from the base customizing - * members, constructors and the doWork() method to effect - * the command. - * - * The flow is one-directional. There are two global instances - * of the LLViewerAssetStats collector, one for the main program's - * thread pointed to by gViewerAssetStatsMain and one for the - * TextureFetch thread pointed to by gViewerAssetStatsThread1. - * Common operations has each thread recording metrics events - * into the respective collector unconcerned with locking and - * the state of any other thread. But when the agent moves into - * a different region or the metrics timer expires and a report - * needs to be sent back to the grid, messaging across threads - * is required to distribute data and perform global actions. - * In pseudo-UML, it looks like: - * - * Main Thread1 - * . . - * . . - * +-----+ . - * | AM | . - * +--+--+ . - * +-------+ | . - * | Main | +--+--+ . - * | | | SRE |---. . - * | Stats | +-----+ \ . - * | | | \ (uuid) +-----+ - * | Coll. | +--+--+ `-------->| SR | - * +-------+ | MSC | +--+--+ - * | ^ +-----+ | - * | | (uuid) / . +-----+ (uuid) - * | `--------' . | MSC |---------. - * | . +-----+ | - * | +-----+ . v - * | | TE | . +-------+ - * | +--+--+ . | Thd1 | - * | | . | | - * | +-----+ . | Stats | - * `--------->| RSC | . | | - * +--+--+ . | Coll. | - * | . +-------+ - * +--+--+ . | - * | SME |---. . | - * +-----+ \ . | - * . \ (clone) +-----+ | - * . `-------->| SM | | - * . +--+--+ | - * . | | - * . +-----+ | - * . | RSC |<--------' - * . +-----+ - * . | - * . +-----+ - * . | CP |--> HTTP POST - * . +-----+ - * . . - * . . - * - * - * Key: - * - * SRE - Set Region Enqueued. Enqueue a 'Set Region' command in - * the other thread providing the new UUID of the region. - * TFReqSetRegion carries the data. - * SR - Set Region. New region UUID is sent to the thread-local - * collector. - * SME - Send Metrics Enqueued. Enqueue a 'Send Metrics' command - * including an ownership transfer of a cloned LLViewerAssetStats. - * TFReqSendMetrics carries the data. - * SM - Send Metrics. Global metrics reporting operation. Takes - * the cloned stats from the command, merges it with the - * thread's local stats, converts to LLSD and sends it on - * to the grid. - * AM - Agent Moved. Agent has completed some sort of move to a - * new region. - * TE - Timer Expired. Metrics timer has expired (on the order - * of 10 minutes). - * CP - CURL Post - * MSC - Modify Stats Collector. State change in the thread-local - * collector. Typically a region change which affects the - * global pointers used to find the 'current stats'. - * RSC - Read Stats Collector. Extract collector data cloning it - * (i.e. deep copy) when necessary. - * - */ -class LLTextureFetch::TFRequest // : public LLQueuedThread::QueuedRequest -{ -public: - // Default ctors and assignment operator are correct. - - virtual ~TFRequest() - {} - - // Patterned after QueuedRequest's method but expected behavior - // is different. Always expected to complete on the first call - // and work dispatcher will assume the same and delete the - // request after invocation. - virtual bool doWork(LLTextureFetch * fetcher) = 0; -}; - -namespace -{ - -/** - * @brief Implements a 'Set Region' cross-thread command. - * - * When an agent moves to a new region, subsequent metrics need - * to be binned into a new or existing stats collection in 1:1 - * relationship with the region. We communicate this region - * change across the threads involved in the communication with - * this message. - * - * Corresponds to LLTextureFetch::commandSetRegion() - */ -class TFReqSetRegion : public LLTextureFetch::TFRequest -{ -public: - TFReqSetRegion(U64 region_handle) - : LLTextureFetch::TFRequest(), - mRegionHandle(region_handle) - {} - TFReqSetRegion & operator=(const TFReqSetRegion &); // Not defined - - virtual ~TFReqSetRegion() - {} - - virtual bool doWork(LLTextureFetch * fetcher); - -public: - const U64 mRegionHandle; -}; - - -/** - * @brief Implements a 'Send Metrics' cross-thread command. - * - * This is the big operation. The main thread gathers metrics - * for a period of minutes into LLViewerAssetStats and other - * objects then makes a snapshot of the data by cloning the - * collector. This command transfers the clone, along with a few - * additional arguments (UUIDs), handing ownership to the - * TextureFetch thread. It then merges its own data into the - * cloned copy, converts to LLSD and kicks off an HTTP POST of - * the resulting data to the currently active metrics collector. - * - * Corresponds to LLTextureFetch::commandSendMetrics() - */ -class TFReqSendMetrics : public LLTextureFetch::TFRequest -{ -public: - /** - * Construct the 'Send Metrics' command to have the TextureFetch - * thread add and log metrics data. - * - * @param caps_url URL of a "ViewerMetrics" Caps target - * to receive the data. Does not have to - * be associated with a particular region. - * - * @param session_id UUID of the agent's session. - * - * @param agent_id UUID of the agent. (Being pure here...) - * - * @param main_stats Pointer to a clone of the main thread's - * LLViewerAssetStats data. Thread1 takes - * ownership of the copy and disposes of it - * when done. - */ - TFReqSendMetrics(const std::string & caps_url, - const LLUUID & session_id, - const LLUUID & agent_id, - LLViewerAssetStats * main_stats) - : LLTextureFetch::TFRequest(), - mCapsURL(caps_url), - mSessionID(session_id), - mAgentID(agent_id), - mMainStats(main_stats) - {} - TFReqSendMetrics & operator=(const TFReqSendMetrics &); // Not defined - - virtual ~TFReqSendMetrics(); - - virtual bool doWork(LLTextureFetch * fetcher); - -public: - const std::string mCapsURL; - const LLUUID mSessionID; - const LLUUID mAgentID; - LLViewerAssetStats * mMainStats; -}; - -/* - * Examines the merged viewer metrics report and if found to be too long, - * will attempt to truncate it in some reasonable fashion. - * - * @param max_regions Limit of regions allowed in report. - * - * @param metrics Full, merged viewer metrics report. - * - * @returns If data was truncated, returns true. - */ -bool truncate_viewer_metrics(int max_regions, LLSD & metrics); - -} // end of anonymous namespace - - -////////////////////////////////////////////////////////////////////////////// - -//static -const char* LLTextureFetchWorker::sStateDescs[] = { - "INVALID", - "INIT", - "LOAD_FROM_TEXTURE_CACHE", - "CACHE_POST", - "LOAD_FROM_NETWORK", - "LOAD_FROM_SIMULATOR", - "SEND_HTTP_REQ", - "WAIT_HTTP_REQ", - "DECODE_IMAGE", - "DECODE_IMAGE_UPDATE", - "WRITE_TO_CACHE", - "WAIT_ON_WRITE", - "DONE", -}; - -// static -volatile bool LLTextureFetch::svMetricsDataBreak(true); // Start with a data break - -// called from MAIN THREAD - -LLTextureFetchWorker::LLTextureFetchWorker(LLTextureFetch* fetcher, - const std::string& url, // Optional URL - const LLUUID& id, // Image UUID - const LLHost& host, // Simulator host - F32 priority, // Priority - S32 discard, // Desired discard - S32 size) // Desired size - : LLWorkerClass(fetcher, "TextureFetch"), - mState(INIT), - mWriteToCacheState(NOT_WRITE), - mFetcher(fetcher), - mID(id), - mHost(host), - mUrl(url), - mImagePriority(priority), - mWorkPriority(0), - mRequestedPriority(0.f), - mDesiredDiscard(-1), - mSimRequestedDiscard(-1), - mRequestedDiscard(-1), - mLoadedDiscard(-1), - mDecodedDiscard(-1), - mCacheReadHandle(LLTextureCache::nullHandle()), - mCacheWriteHandle(LLTextureCache::nullHandle()), - mBuffer(NULL), - mBufferSize(0), - mRequestedSize(0), - mDesiredSize(TEXTURE_CACHE_ENTRY_SIZE), - mFileSize(0), - mCachedSize(0), - mLoaded(FALSE), - mSentRequest(UNSENT), - mDecodeHandle(0), - mDecoded(FALSE), - mWritten(FALSE), - mNeedsAux(FALSE), - mHaveAllData(FALSE), - mInLocalCache(FALSE), - mCanUseHTTP(true), - mHTTPFailCount(0), - mRetryAttempt(0), - mActiveCount(0), - mGetStatus(0), - mWorkMutex(NULL), - mFirstPacket(0), - mLastPacket(-1), - mTotalPackets(0), - mImageCodec(IMG_CODEC_INVALID), - mMetricsStartTime(0) -{ - mCanUseNET = mUrl.empty() ; - - calcWorkPriority(); - mType = host.isOk() ? LLImageBase::TYPE_AVATAR_BAKE : LLImageBase::TYPE_NORMAL; -// llinfos << "Create: " << mID << " mHost:" << host << " Discard=" << discard << llendl; - if (!mFetcher->mDebugPause) - { - U32 work_priority = mWorkPriority | LLWorkerThread::PRIORITY_HIGH; - addWork(0, work_priority ); - } - setDesiredDiscard(discard, size); -} - -LLTextureFetchWorker::~LLTextureFetchWorker() -{ -// llinfos << "Destroy: " << mID -// << " Decoded=" << mDecodedDiscard -// << " Requested=" << mRequestedDiscard -// << " Desired=" << mDesiredDiscard << llendl; - llassert_always(!haveWork()); - lockWorkMutex(); - if (mCacheReadHandle != LLTextureCache::nullHandle() && mFetcher->mTextureCache) - { - mFetcher->mTextureCache->readComplete(mCacheReadHandle, true); - } - if (mCacheWriteHandle != LLTextureCache::nullHandle() && mFetcher->mTextureCache) - { - mFetcher->mTextureCache->writeComplete(mCacheWriteHandle, true); - } - mFormattedImage = NULL; - clearPackets(); - unlockWorkMutex(); - mFetcher->removeFromHTTPQueue(mID); -} - -void LLTextureFetchWorker::clearPackets() -{ - for_each(mPackets.begin(), mPackets.end(), DeletePointer()); - mPackets.clear(); - mTotalPackets = 0; - mLastPacket = -1; - mFirstPacket = 0; -} - -void LLTextureFetchWorker::setupPacketData() -{ - S32 data_size = 0; - if (mFormattedImage.notNull()) - { - data_size = mFormattedImage->getDataSize(); - } - if (data_size > 0) - { - // Only used for simulator requests - mFirstPacket = (data_size - FIRST_PACKET_SIZE) / MAX_IMG_PACKET_SIZE + 1; - if (FIRST_PACKET_SIZE + (mFirstPacket-1) * MAX_IMG_PACKET_SIZE != data_size) - { - llwarns << "Bad CACHED TEXTURE size: " << data_size << " removing." << llendl; - removeFromCache(); - resetFormattedData(); - clearPackets(); - } - else if (mFileSize > 0) - { - mLastPacket = mFirstPacket-1; - mTotalPackets = (mFileSize - FIRST_PACKET_SIZE + MAX_IMG_PACKET_SIZE-1) / MAX_IMG_PACKET_SIZE + 1; - } - else - { - // This file was cached using HTTP so we have to refetch the first packet - resetFormattedData(); - clearPackets(); - } - } -} - -U32 LLTextureFetchWorker::calcWorkPriority() -{ - //llassert_always(mImagePriority >= 0 && mImagePriority <= LLViewerFetchedTexture::maxDecodePriority()); - static const F32 PRIORITY_SCALE = (F32)LLWorkerThread::PRIORITY_LOWBITS / LLViewerFetchedTexture::maxDecodePriority(); - - mWorkPriority = llmin((U32)LLWorkerThread::PRIORITY_LOWBITS, (U32)(mImagePriority * PRIORITY_SCALE)); - return mWorkPriority; -} - -// mWorkMutex is locked -void LLTextureFetchWorker::setDesiredDiscard(S32 discard, S32 size) -{ - bool prioritize = false; - if (mDesiredDiscard != discard) - { - if (!haveWork()) - { - calcWorkPriority(); - if (!mFetcher->mDebugPause) - { - U32 work_priority = mWorkPriority | LLWorkerThread::PRIORITY_HIGH; - addWork(0, work_priority); - } - } - else if (mDesiredDiscard < discard) - { - prioritize = true; - } - mDesiredDiscard = discard; - mDesiredSize = size; - } - else if (size > mDesiredSize) - { - mDesiredSize = size; - prioritize = true; - } - mDesiredSize = llmax(mDesiredSize, TEXTURE_CACHE_ENTRY_SIZE); - if ((prioritize && mState == INIT) || mState == DONE) - { - mState = INIT; - U32 work_priority = mWorkPriority | LLWorkerThread::PRIORITY_HIGH; - setPriority(work_priority); - } -} - -void LLTextureFetchWorker::setImagePriority(F32 priority) -{ -// llassert_always(priority >= 0 && priority <= LLViewerTexture::maxDecodePriority()); - F32 delta = fabs(priority - mImagePriority); - if (delta > (mImagePriority * .05f) || mState == DONE) - { - mImagePriority = priority; - calcWorkPriority(); - U32 work_priority = mWorkPriority | (getPriority() & LLWorkerThread::PRIORITY_HIGHBITS); - setPriority(work_priority); - } -} - -void LLTextureFetchWorker::resetFormattedData() -{ - delete[] mBuffer; - mBuffer = NULL; - mBufferSize = 0; - if (mFormattedImage.notNull()) - { - mFormattedImage->deleteData(); - } - mHaveAllData = FALSE; -} - -// Called from MAIN thread -void LLTextureFetchWorker::startWork(S32 param) -{ - llassert(mFormattedImage.isNull()); -} - -#include "llviewertexturelist.h" // debug - -// Called from LLWorkerThread::processRequest() -bool LLTextureFetchWorker::doWork(S32 param) -{ - LLMutexLock lock(&mWorkMutex); - - if ((mFetcher->isQuitting() || getFlags(LLWorkerClass::WCF_DELETE_REQUESTED))) - { - if (mState < DECODE_IMAGE) - { - return true; // abort - } - } - - if(mImagePriority < F_ALMOST_ZERO) - { - if (mState == INIT || mState == LOAD_FROM_NETWORK || mState == LOAD_FROM_SIMULATOR) - { - return true; // abort - } - } - if(mState > CACHE_POST && !mCanUseNET && !mCanUseHTTP) - { - //nowhere to get data, abort. - return true ; - } - - if (mFetcher->mDebugPause) - { - return false; // debug: don't do any work - } - if (mID == mFetcher->mDebugID) - { - mFetcher->mDebugCount++; // for setting breakpoints - } - - if (mState != DONE) - { - mFetchTimer.reset(); - } - - if (mState == INIT) - { - mRawImage = NULL ; - mRequestedDiscard = -1; - mLoadedDiscard = -1; - mDecodedDiscard = -1; - mRequestedSize = 0; - mFileSize = 0; - mCachedSize = 0; - mLoaded = FALSE; - mSentRequest = UNSENT; - mDecoded = FALSE; - mWritten = FALSE; - delete[] mBuffer; - mBuffer = NULL; - mBufferSize = 0; - mHaveAllData = FALSE; - clearPackets(); // TODO: Shouldn't be necessary - mCacheReadHandle = LLTextureCache::nullHandle(); - mCacheWriteHandle = LLTextureCache::nullHandle(); - mState = LOAD_FROM_TEXTURE_CACHE; - mDesiredSize = llmax(mDesiredSize, TEXTURE_CACHE_ENTRY_SIZE); // min desired size is TEXTURE_CACHE_ENTRY_SIZE - LL_DEBUGS("Texture") << mID << ": Priority: " << llformat("%8.0f",mImagePriority) - << " Desired Discard: " << mDesiredDiscard << " Desired Size: " << mDesiredSize << LL_ENDL; - // fall through - } - - if (mState == LOAD_FROM_TEXTURE_CACHE) - { - if (mCacheReadHandle == LLTextureCache::nullHandle()) - { - U32 cache_priority = mWorkPriority; - S32 offset = mFormattedImage.notNull() ? mFormattedImage->getDataSize() : 0; - S32 size = mDesiredSize - offset; - if (size <= 0) - { - mState = CACHE_POST; - return false; - } - mFileSize = 0; - mLoaded = FALSE; - - if (mUrl.compare(0, 7, "file://") == 0) - { - setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); // Set priority first since Responder may change it - - // read file from local disk - std::string filename = mUrl.substr(7, std::string::npos); - CacheReadResponder* responder = new CacheReadResponder(mFetcher, mID, mFormattedImage); - mCacheReadHandle = mFetcher->mTextureCache->readFromCache(filename, mID, cache_priority, - offset, size, responder); - } - else if (mUrl.empty()) - { - setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); // Set priority first since Responder may change it - - CacheReadResponder* responder = new CacheReadResponder(mFetcher, mID, mFormattedImage); - mCacheReadHandle = mFetcher->mTextureCache->readFromCache(mID, cache_priority, - offset, size, responder); - } - else if(mCanUseHTTP) - { - if (!(mUrl.compare(0, 7, "http://") == 0)) - { - // *TODO:?remove this warning - llwarns << "Unknown URL Type: " << mUrl << llendl; - } - setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); - mState = SEND_HTTP_REQ; - } - else - { - setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); - mState = LOAD_FROM_NETWORK; - } - } - - if (mLoaded) - { - // Make sure request is complete. *TODO: make this auto-complete - if (mFetcher->mTextureCache->readComplete(mCacheReadHandle, false)) - { - mCacheReadHandle = LLTextureCache::nullHandle(); - mState = CACHE_POST; - // fall through - } - else - { - return false; - } - } - else - { - return false; - } - } - - if (mState == CACHE_POST) - { - mCachedSize = mFormattedImage.notNull() ? mFormattedImage->getDataSize() : 0; - // Successfully loaded - if ((mCachedSize >= mDesiredSize) || mHaveAllData) - { - // we have enough data, decode it - llassert_always(mFormattedImage->getDataSize() > 0); - mLoadedDiscard = mDesiredDiscard; - mState = DECODE_IMAGE; - mWriteToCacheState = NOT_WRITE ; - LL_DEBUGS("Texture") << mID << ": Cached. Bytes: " << mFormattedImage->getDataSize() - << " Size: " << llformat("%dx%d",mFormattedImage->getWidth(),mFormattedImage->getHeight()) - << " Desired Discard: " << mDesiredDiscard << " Desired Size: " << mDesiredSize << LL_ENDL; - // fall through - } - else - { - if (mUrl.compare(0, 7, "file://") == 0) - { - // failed to load local file, we're done. - return true; - } - // need more data - else - { - LL_DEBUGS("Texture") << mID << ": Not in Cache" << LL_ENDL; - mState = LOAD_FROM_NETWORK; - } - // fall through - } - } - - if (mState == LOAD_FROM_NETWORK) - { - static LLCachedControl use_http(gSavedSettings,"ImagePipelineUseHTTP"); - -// if (mHost != LLHost::invalid) get_url = false; - if ( use_http && mCanUseHTTP && mUrl.empty())//get http url. - { - LLViewerRegion* region = NULL; - if (mHost == LLHost::invalid) - region = gAgent.getRegion(); - else - region = LLWorld::getInstance()->getRegion(mHost); - - if (region) - { - std::string http_url = region->getHttpUrl() ; - if (!http_url.empty()) - { - mUrl = http_url + "/?texture_id=" + mID.asString().c_str(); - mWriteToCacheState = CAN_WRITE ; //because this texture has a fixed texture id. - } - else - { - mCanUseHTTP = false ; - } - } - else - { - // This will happen if not logged in or if a region deoes not have HTTP Texture enabled - //llwarns << "Region not found for host: " << mHost << llendl; - mCanUseHTTP = false; - } - } - if (mCanUseHTTP && !mUrl.empty()) - { - mState = LLTextureFetchWorker::SEND_HTTP_REQ; - setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); - if(mWriteToCacheState != NOT_WRITE) - { - mWriteToCacheState = CAN_WRITE ; - } - // don't return, fall through to next state - } - else if (mSentRequest == UNSENT && mCanUseNET) - { - // Add this to the network queue and sit here. - // LLTextureFetch::update() will send off a request which will change our state - mWriteToCacheState = CAN_WRITE ; - mRequestedSize = mDesiredSize; - mRequestedDiscard = mDesiredDiscard; - mSentRequest = QUEUED; - mFetcher->addToNetworkQueue(this); - if (! mMetricsStartTime) - { - mMetricsStartTime = LLViewerAssetStatsFF::get_timestamp(); - } - LLViewerAssetStatsFF::record_enqueue_thread1(LLViewerAssetType::AT_TEXTURE, - false, - LLImageBase::TYPE_AVATAR_BAKE == mType); - setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); - - return false; - } - else - { - // Shouldn't need to do anything here - //llassert_always(mFetcher->mNetworkQueue.find(mID) != mFetcher->mNetworkQueue.end()); - // Make certain this is in the network queue - //mFetcher->addToNetworkQueue(this); - //if (! mMetricsStartTime) - //{ - // mMetricsStartTime = LLViewerAssetStatsFF::get_timestamp(); - //} - //LLViewerAssetStatsFF::record_enqueue_thread1(LLViewerAssetType::AT_TEXTURE, false, - // LLImageBase::TYPE_AVATAR_BAKE == mType); - //setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); - return false; - } - } - - if (mState == LOAD_FROM_SIMULATOR) - { - if (mFormattedImage.isNull()) - { - mFormattedImage = new LLImageJ2C; - } - if (processSimulatorPackets()) - { - LL_DEBUGS("Texture") << mID << ": Loaded from Sim. Bytes: " << mFormattedImage->getDataSize() << LL_ENDL; - mFetcher->removeFromNetworkQueue(this, false); - if (mFormattedImage.isNull() || !mFormattedImage->getDataSize()) - { - // processSimulatorPackets() failed -// llwarns << "processSimulatorPackets() failed to load buffer" << llendl; - return true; // failed - } - setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); - mState = DECODE_IMAGE; - mWriteToCacheState = SHOULD_WRITE; - - if (mMetricsStartTime) - { - LLViewerAssetStatsFF::record_response_thread1(LLViewerAssetType::AT_TEXTURE, - false, - LLImageBase::TYPE_AVATAR_BAKE == mType, - LLViewerAssetStatsFF::get_timestamp() - mMetricsStartTime); - mMetricsStartTime = 0; - } - LLViewerAssetStatsFF::record_dequeue_thread1(LLViewerAssetType::AT_TEXTURE, - false, - LLImageBase::TYPE_AVATAR_BAKE == mType); - } - else - { - mFetcher->addToNetworkQueue(this); // failsafe - if (! mMetricsStartTime) - { - mMetricsStartTime = LLViewerAssetStatsFF::get_timestamp(); - } - LLViewerAssetStatsFF::record_enqueue_thread1(LLViewerAssetType::AT_TEXTURE, - false, - LLImageBase::TYPE_AVATAR_BAKE == mType); - setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); - } - return false; - } - - if (mState == SEND_HTTP_REQ) - { - if(mCanUseHTTP) - { - //NOTE: - //control the number of the http requests issued for: - //1, not openning too many file descriptors at the same time; - //2, control the traffic of http so udp gets bandwidth. - // - static const S32 MAX_NUM_OF_HTTP_REQUESTS_IN_QUEUE = 8 ; - if(mFetcher->getNumHTTPRequests() > MAX_NUM_OF_HTTP_REQUESTS_IN_QUEUE) - { - return false ; //wait. - } - - mFetcher->removeFromNetworkQueue(this, false); - - S32 cur_size = 0; - if (mFormattedImage.notNull()) - { - cur_size = mFormattedImage->getDataSize(); // amount of data we already have - if (mFormattedImage->getDiscardLevel() == 0) - { - if(cur_size > 0) - { - // We already have all the data, just decode it - mLoadedDiscard = mFormattedImage->getDiscardLevel(); - mState = DECODE_IMAGE; - return false; - } - else - { - return true ; //abort. - } - } - } - mRequestedSize = mDesiredSize; - mRequestedDiscard = mDesiredDiscard; - mRequestedSize -= cur_size; - S32 offset = cur_size; - mBufferSize = cur_size; // This will get modified by callbackHttpGet() - - bool res = false; - if (!mUrl.empty()) - { - mLoaded = FALSE; - mGetStatus = 0; - mGetReason.clear(); - LL_DEBUGS("Texture") << "HTTP GET: " << mID << " Offset: " << offset - << " Bytes: " << mRequestedSize - << " Bandwidth(kbps): " << mFetcher->getTextureBandwidth() << "/" << mFetcher->mMaxBandwidth - << LL_ENDL; - setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); - mState = WAIT_HTTP_REQ; - - mFetcher->addToHTTPQueue(mID); - if (! mMetricsStartTime) - { - mMetricsStartTime = LLViewerAssetStatsFF::get_timestamp(); - } - LLViewerAssetStatsFF::record_enqueue_thread1(LLViewerAssetType::AT_TEXTURE, - true, - LLImageBase::TYPE_AVATAR_BAKE == mType); - - // Will call callbackHttpGet when curl request completes - std::vector headers; - headers.push_back("Accept: image/x-j2c"); - res = mFetcher->mCurlGetRequest->getByteRange(mUrl, headers, offset, mRequestedSize, - new HTTPGetResponder(mFetcher, mID, LLTimer::getTotalTime(), mRequestedSize, offset, true)); - } - if (!res) - { - llwarns << "HTTP GET request failed for " << mID << llendl; - resetFormattedData(); - ++mHTTPFailCount; - return true; // failed - } - // fall through - } - else //can not use http fetch. - { - return true ; //abort - } - } - - if (mState == WAIT_HTTP_REQ) - { - if (mLoaded) - { - S32 cur_size = mFormattedImage.notNull() ? mFormattedImage->getDataSize() : 0; - if (mRequestedSize < 0) - { - S32 max_attempts; - if (mGetStatus == HTTP_NOT_FOUND) - { - mHTTPFailCount = max_attempts = 1; // Don't retry - llwarns << "Texture missing from server (404): " << mUrl << llendl; - - //roll back to try UDP - if(mCanUseNET) - { - mState = INIT ; - mCanUseHTTP = false ; - setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); - return false ; - } - } - else if (mGetStatus == HTTP_SERVICE_UNAVAILABLE) - { - // *TODO: Should probably introduce a timer here to delay future HTTP requsts - // for a short time (~1s) to ease server load? Ideally the server would queue - // requests instead of returning 503... we already limit the number pending. - ++mHTTPFailCount; - max_attempts = mHTTPFailCount+1; // Keep retrying - LL_INFOS_ONCE("Texture") << "Texture server busy (503): " << mUrl << LL_ENDL; - } - else - { - const S32 HTTP_MAX_RETRY_COUNT = 3; - max_attempts = HTTP_MAX_RETRY_COUNT + 1; - ++mHTTPFailCount; - llinfos << "HTTP GET failed for: " << mUrl - << " Status: " << mGetStatus << " Reason: '" << mGetReason << "'" - << " Attempt:" << mHTTPFailCount+1 << "/" << max_attempts << llendl; - } - - if (mHTTPFailCount >= max_attempts) - { - if (cur_size > 0) - { - // Use available data - mLoadedDiscard = mFormattedImage->getDiscardLevel(); - mState = DECODE_IMAGE; - return false; - } - else - { - resetFormattedData(); - mState = DONE; - return true; // failed - } - } - else - { - mState = SEND_HTTP_REQ; - return false; // retry - } - } - - llassert_always(mBufferSize == cur_size + mRequestedSize); - if(!mBufferSize)//no data received. - { - delete[] mBuffer; - mBuffer = NULL; - - //abort. - mState = DONE; - return true; - } - - if (mFormattedImage.isNull()) - { - // For now, create formatted image based on extension - std::string extension = gDirUtilp->getExtension(mUrl); - mFormattedImage = LLImageFormatted::createFromType(LLImageBase::getCodecFromExtension(extension)); - if (mFormattedImage.isNull()) - { - mFormattedImage = new LLImageJ2C; // default - } - } - - if (mHaveAllData && mRequestedDiscard == 0) //the image file is fully loaded. - { - mFileSize = mBufferSize; - } - else //the file size is unknown. - { - mFileSize = mBufferSize + 1 ; //flag the file is not fully loaded. - } - - U8* buffer = new U8[mBufferSize]; - if (cur_size > 0) - { - memcpy(buffer, mFormattedImage->getData(), cur_size); - } - memcpy(buffer + cur_size, mBuffer, mRequestedSize); // append - // NOTE: setData releases current data and owns new data (buffer) - mFormattedImage->setData(buffer, mBufferSize); - // delete temp data - delete[] mBuffer; // Note: not 'buffer' (assigned in setData()) - mBuffer = NULL; - mBufferSize = 0; - mLoadedDiscard = mRequestedDiscard; - mState = DECODE_IMAGE; - if(mWriteToCacheState != NOT_WRITE) - { - mWriteToCacheState = SHOULD_WRITE ; - } - setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); - return false; - } - else - { - setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); - return false; - } - } - - if (mState == DECODE_IMAGE) - { - static LLCachedControl textures_decode_disabled(gSavedSettings,"TextureDecodeDisabled"); - if(textures_decode_disabled) - { - // for debug use, don't decode - mState = DONE; - setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); - return true; - } - - if (mDesiredDiscard < 0) - { - // We aborted, don't decode - mState = DONE; - setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); - return true; - } - - if (mFormattedImage->getDataSize() <= 0) - { - //llerrs << "Decode entered with invalid mFormattedImage. ID = " << mID << llendl; - - //abort, don't decode - mState = DONE; - setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); - return true; - } - if (mLoadedDiscard < 0) - { - //llerrs << "Decode entered with invalid mLoadedDiscard. ID = " << mID << llendl; - - //abort, don't decode - mState = DONE; - setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); - return true; - } - setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); // Set priority first since Responder may change it - mRawImage = NULL; - mAuxImage = NULL; - llassert_always(mFormattedImage.notNull()); - S32 discard = mHaveAllData ? 0 : mLoadedDiscard; - U32 image_priority = LLWorkerThread::PRIORITY_NORMAL | mWorkPriority; - mDecoded = FALSE; - mState = DECODE_IMAGE_UPDATE; - LL_DEBUGS("Texture") << mID << ": Decoding. Bytes: " << mFormattedImage->getDataSize() << " Discard: " << discard - << " All Data: " << mHaveAllData << LL_ENDL; - mDecodeHandle = mFetcher->mImageDecodeThread->decodeImage(mFormattedImage, image_priority, discard, mNeedsAux, - new DecodeResponder(mFetcher, mID, this)); - // fall though - } - - if (mState == DECODE_IMAGE_UPDATE) - { - if (mDecoded) - { - if (mDecodedDiscard < 0) - { - LL_DEBUGS("Texture") << mID << ": Failed to Decode." << LL_ENDL; - if (mCachedSize > 0 && !mInLocalCache && mRetryAttempt == 0) - { - // Cache file should be deleted, try again -// llwarns << mID << ": Decode of cached file failed (removed), retrying" << llendl; - llassert_always(mDecodeHandle == 0); - mFormattedImage = NULL; - ++mRetryAttempt; - setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); - mState = INIT; - return false; - } - else - { -// llwarns << "UNABLE TO LOAD TEXTURE: " << mID << " RETRIES: " << mRetryAttempt << llendl; - mState = DONE; // failed - } - } - else - { - llassert_always(mRawImage.notNull()); - LL_DEBUGS("Texture") << mID << ": Decoded. Discard: " << mDecodedDiscard - << " Raw Image: " << llformat("%dx%d",mRawImage->getWidth(),mRawImage->getHeight()) << LL_ENDL; - setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); - mState = WRITE_TO_CACHE; - } - // fall through - } - else - { - return false; - } - } - - if (mState == WRITE_TO_CACHE) - { - if (mWriteToCacheState != SHOULD_WRITE || mFormattedImage.isNull()) - { - // If we're in a local cache or we didn't actually receive any new data, - // or we failed to load anything, skip - mState = DONE; - return false; - } - S32 datasize = mFormattedImage->getDataSize(); - if(mFileSize < datasize)//This could happen when http fetching and sim fetching mixed. - { - if(mHaveAllData) - { - mFileSize = datasize ; - } - else - { - mFileSize = datasize + 1 ; //flag not fully loaded. - } - } - llassert_always(datasize); - setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); // Set priority first since Responder may change it - U32 cache_priority = mWorkPriority; - mWritten = FALSE; - mState = WAIT_ON_WRITE; - CacheWriteResponder* responder = new CacheWriteResponder(mFetcher, mID); - mCacheWriteHandle = mFetcher->mTextureCache->writeToCache(mID, cache_priority, - mFormattedImage->getData(), datasize, - mFileSize, responder); - // fall through - } - - if (mState == WAIT_ON_WRITE) - { - if (writeToCacheComplete()) - { - mState = DONE; - // fall through - } - else - { - if (mDesiredDiscard < mDecodedDiscard) - { - // We're waiting for this write to complete before we can receive more data - // (we can't touch mFormattedImage until the write completes) - // Prioritize the write - mFetcher->mTextureCache->prioritizeWrite(mCacheWriteHandle); - } - return false; - } - } - - if (mState == DONE) - { - if (mDecodedDiscard >= 0 && mDesiredDiscard < mDecodedDiscard) - { - // More data was requested, return to INIT - mState = INIT; - setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); - return false; - } - else - { - setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); - return true; - } - } - - return false; -} - -// Called from MAIN thread -void LLTextureFetchWorker::endWork(S32 param, bool aborted) -{ - if (mDecodeHandle != 0) - { - mFetcher->mImageDecodeThread->abortRequest(mDecodeHandle, false); - mDecodeHandle = 0; - } - mFormattedImage = NULL; -} - -////////////////////////////////////////////////////////////////////////////// - -// virtual -void LLTextureFetchWorker::finishWork(S32 param, bool completed) -{ - // The following are required in case the work was aborted - if (mCacheReadHandle != LLTextureCache::nullHandle()) - { - mFetcher->mTextureCache->readComplete(mCacheReadHandle, true); - mCacheReadHandle = LLTextureCache::nullHandle(); - } - if (mCacheWriteHandle != LLTextureCache::nullHandle()) - { - mFetcher->mTextureCache->writeComplete(mCacheWriteHandle, true); - mCacheWriteHandle = LLTextureCache::nullHandle(); - } -} - -// virtual -bool LLTextureFetchWorker::deleteOK() -{ - bool delete_ok = true; - // Allow any pending reads or writes to complete - if (mCacheReadHandle != LLTextureCache::nullHandle()) - { - if (mFetcher->mTextureCache->readComplete(mCacheReadHandle, true)) - { - mCacheReadHandle = LLTextureCache::nullHandle(); - } - else - { - delete_ok = false; - } - } - if (mCacheWriteHandle != LLTextureCache::nullHandle()) - { - if (mFetcher->mTextureCache->writeComplete(mCacheWriteHandle)) - { - mCacheWriteHandle = LLTextureCache::nullHandle(); - } - else - { - delete_ok = false; - } - } - - if ((haveWork() && - // not ok to delete from these states - ((mState >= WRITE_TO_CACHE && mState <= WAIT_ON_WRITE)))) - { - delete_ok = false; - } - - return delete_ok; -} - -void LLTextureFetchWorker::removeFromCache() -{ - if (!mInLocalCache) - { - mFetcher->mTextureCache->removeFromCache(mID); - } -} - - -////////////////////////////////////////////////////////////////////////////// - -bool LLTextureFetchWorker::processSimulatorPackets() -{ - if (mFormattedImage.isNull() || mRequestedSize < 0) - { - // not sure how we got here, but not a valid state, abort! - llassert_always(mDecodeHandle == 0); - mFormattedImage = NULL; - return true; - } - - if (mLastPacket >= mFirstPacket) - { - S32 buffer_size = mFormattedImage->getDataSize(); - for (S32 i = mFirstPacket; i<=mLastPacket; i++) - { - llassert_always(mPackets[i]); - buffer_size += mPackets[i]->mSize; - } - bool have_all_data = mLastPacket >= mTotalPackets-1; - if (mRequestedSize <= 0) - { - // We received a packed but haven't requested anything yet (edge case) - // Return true (we're "done") since we didn't request anything - return true; - } - if (buffer_size >= mRequestedSize || have_all_data) - { - /// We have enough (or all) data - if (have_all_data) - { - mHaveAllData = TRUE; - } - S32 cur_size = mFormattedImage->getDataSize(); - if (buffer_size > cur_size) - { - /// We have new data - U8* buffer = new U8[buffer_size]; - S32 offset = 0; - if (cur_size > 0 && mFirstPacket > 0) - { - memcpy(buffer, mFormattedImage->getData(), cur_size); - offset = cur_size; - } - for (S32 i=mFirstPacket; i<=mLastPacket; i++) - { - memcpy(buffer + offset, mPackets[i]->mData, mPackets[i]->mSize); - offset += mPackets[i]->mSize; - } - // NOTE: setData releases current data - mFormattedImage->setData(buffer, buffer_size); - } - mLoadedDiscard = mRequestedDiscard; - return true; - } - } - return false; -} - -////////////////////////////////////////////////////////////////////////////// - -S32 LLTextureFetchWorker::callbackHttpGet(const LLChannelDescriptors& channels, - const LLIOPipe::buffer_ptr_t& buffer, - bool partial, bool success) -{ - S32 data_size = 0 ; - - LLMutexLock lock(&mWorkMutex); - - if (mState != WAIT_HTTP_REQ) - { - llwarns << "callbackHttpGet for unrequested fetch worker: " << mID - << " req=" << mSentRequest << " state= " << mState << llendl; - return data_size; - } - if (mLoaded) - { - llwarns << "Duplicate callback for " << mID.asString() << llendl; - return data_size ; // ignore duplicate callback - } - if (success) - { - // get length of stream: - data_size = buffer->countAfter(channels.in(), NULL); - - LL_DEBUGS("Texture") << "HTTP RECEIVED: " << mID.asString() << " Bytes: " << data_size << LL_ENDL; - if (data_size > 0) - { - // *TODO: set the formatted image data here directly to avoid the copy - mBuffer = new U8[data_size]; - buffer->readAfter(channels.in(), NULL, mBuffer, data_size); - mBufferSize += data_size; - if (data_size < mRequestedSize && mRequestedDiscard == 0) - { - mHaveAllData = TRUE; - } - else if (data_size > mRequestedSize) - { - // *TODO: This shouldn't be happening any more - llwarns << "data_size = " << data_size << " > requested: " << mRequestedSize << llendl; - mHaveAllData = TRUE; - llassert_always(mDecodeHandle == 0); - mFormattedImage = NULL; // discard any previous data we had - mBufferSize = data_size; - } - } - else - { - // We requested data but received none (and no error), - // so presumably we have all of it - mHaveAllData = TRUE; - } - mRequestedSize = data_size; - } - else - { - mRequestedSize = -1; // error - } - mLoaded = TRUE; - setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); - - return data_size ; -} - -////////////////////////////////////////////////////////////////////////////// - -void LLTextureFetchWorker::callbackCacheRead(bool success, LLImageFormatted* image, - S32 imagesize, BOOL islocal) -{ - LLMutexLock lock(&mWorkMutex); - if (mState != LOAD_FROM_TEXTURE_CACHE) - { -// llwarns << "Read callback for " << mID << " with state = " << mState << llendl; - return; - } - if (success) - { - llassert_always(imagesize >= 0); - mFileSize = imagesize; - mFormattedImage = image; - mImageCodec = image->getCodec(); - mInLocalCache = islocal; - if (mFileSize != 0 && mFormattedImage->getDataSize() >= mFileSize) - { - mHaveAllData = TRUE; - } - } - mLoaded = TRUE; - setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); -} - -void LLTextureFetchWorker::callbackCacheWrite(bool success) -{ - LLMutexLock lock(&mWorkMutex); - if (mState != WAIT_ON_WRITE) - { -// llwarns << "Write callback for " << mID << " with state = " << mState << llendl; - return; - } - mWritten = TRUE; - setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); -} - -////////////////////////////////////////////////////////////////////////////// - -void LLTextureFetchWorker::callbackDecoded(bool success, LLImageRaw* raw, LLImageRaw* aux) -{ - LLMutexLock lock(&mWorkMutex); - if (mDecodeHandle == 0) - { - return; // aborted, ignore - } - if (mState != DECODE_IMAGE_UPDATE) - { -// llwarns << "Decode callback for " << mID << " with state = " << mState << llendl; - mDecodeHandle = 0; - return; - } - llassert_always(mFormattedImage.notNull()); - - mDecodeHandle = 0; - if (success) - { - llassert_always(raw); - mRawImage = raw; - mAuxImage = aux; - mDecodedDiscard = mFormattedImage->getDiscardLevel(); - LL_DEBUGS("Texture") << mID << ": Decode Finished. Discard: " << mDecodedDiscard - << " Raw Image: " << llformat("%dx%d",mRawImage->getWidth(),mRawImage->getHeight()) << LL_ENDL; - } - else - { - llwarns << "DECODE FAILED: " << mID << " Discard: " << (S32)mFormattedImage->getDiscardLevel() << llendl; - removeFromCache(); - mDecodedDiscard = -1; // Redundant, here for clarity and paranoia - } - mDecoded = TRUE; -// llinfos << mID << " : DECODE COMPLETE " << llendl; - setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); -} - -////////////////////////////////////////////////////////////////////////////// - -bool LLTextureFetchWorker::writeToCacheComplete() -{ - // Complete write to cache - if (mCacheWriteHandle != LLTextureCache::nullHandle()) - { - if (!mWritten) - { - return false; - } - if (mFetcher->mTextureCache->writeComplete(mCacheWriteHandle)) - { - mCacheWriteHandle = LLTextureCache::nullHandle(); - } - else - { - return false; - } - } - return true; -} - - -////////////////////////////////////////////////////////////////////////////// -////////////////////////////////////////////////////////////////////////////// -// public - -LLTextureFetch::LLTextureFetch(LLTextureCache* cache, LLImageDecodeThread* imagedecodethread, bool threaded, bool qa_mode) - : LLWorkerThread("TextureFetch", threaded), - mDebugCount(0), - mDebugPause(FALSE), - mPacketCount(0), - mBadPacketCount(0), - mQueueMutex(getAPRPool()), - mNetworkQueueMutex(getAPRPool()), - mTextureCache(cache), - mImageDecodeThread(imagedecodethread), - mTextureBandwidth(0), - mHTTPTextureBits(0), - mTotalHTTPRequests(0), - mCurlGetRequest(NULL), - mQAMode(qa_mode) -{ - mCurlPOSTRequestCount = 0; - mMaxBandwidth = gSavedSettings.getF32("ThrottleBandwidthKBPS"); - mTextureInfo.setUpLogging(gSavedSettings.getBOOL("LogTextureDownloadsToViewerLog"), gSavedSettings.getBOOL("LogTextureDownloadsToSimulator"), gSavedSettings.getU32("TextureLoggingThreshold")); -} - -LLTextureFetch::~LLTextureFetch() -{ - clearDeleteList() ; - - while (! mCommands.empty()) - { - TFRequest * req(mCommands.front()); - mCommands.erase(mCommands.begin()); - delete req; - } - - // ~LLQueuedThread() called here -} - -bool LLTextureFetch::createRequest(const std::string& url, const LLUUID& id, const LLHost& host, F32 priority, - S32 w, S32 h, S32 c, S32 desired_discard, bool needs_aux, bool can_use_http) -{ - if (mDebugPause) - { - return false; - } - - LLTextureFetchWorker* worker = getWorker(id) ; - if (worker) - { - if (worker->mHost != host) - { - llwarns << "LLTextureFetch::createRequest " << id << " called with multiple hosts: " - << host << " != " << worker->mHost << llendl; - removeRequest(worker, true); - worker = NULL; - return false; - } - } - - S32 desired_size; - std::string exten = gDirUtilp->getExtension(url); - if (!url.empty() && (!exten.empty() && LLImageBase::getCodecFromExtension(exten) != IMG_CODEC_J2C)) - { - // Only do partial requests for J2C at the moment - desired_size = MAX_IMAGE_DATA_SIZE; - desired_discard = 0; - } - else if (desired_discard == 0) - { - // if we want the entire image, and we know its size, then get it all - // (calcDataSizeJ2C() below makes assumptions about how the image - // was compressed - this code ensures that when we request the entire image, - // we really do get it.) - desired_size = MAX_IMAGE_DATA_SIZE; - } - else if (w*h*c > 0) - { - // If the requester knows the dimensions of the image, - // this will calculate how much data we need without having to parse the header - - desired_size = LLImageJ2C::calcDataSizeJ2C(w, h, c, desired_discard); - } - else - { - desired_size = TEXTURE_CACHE_ENTRY_SIZE; - desired_discard = MAX_DISCARD_LEVEL; - } - - - if (worker) - { - if (worker->wasAborted()) - { - return false; // need to wait for previous aborted request to complete - } - worker->lockWorkMutex(); - worker->mActiveCount++; - worker->mNeedsAux = needs_aux; - worker->setImagePriority(priority); - worker->setDesiredDiscard(desired_discard, desired_size); - worker->setCanUseHTTP(can_use_http) ; - if (!worker->haveWork()) - { - worker->mState = LLTextureFetchWorker::INIT; - worker->unlockWorkMutex(); - - worker->addWork(0, LLWorkerThread::PRIORITY_HIGH | worker->mWorkPriority); - } - else - { - worker->unlockWorkMutex(); - } - } - else - { - worker = new LLTextureFetchWorker(this, url, id, host, priority, desired_discard, desired_size); - lockQueue() ; - mRequestMap[id] = worker; - unlockQueue() ; - - worker->lockWorkMutex(); - worker->mActiveCount++; - worker->mNeedsAux = needs_aux; - worker->setCanUseHTTP(can_use_http) ; - worker->unlockWorkMutex(); - } - -// llinfos << "REQUESTED: " << id << " Discard: " << desired_discard << llendl; - return true; -} - -// protected -void LLTextureFetch::addToNetworkQueue(LLTextureFetchWorker* worker) -{ - lockQueue() ; - bool in_request_map = (mRequestMap.find(worker->mID) != mRequestMap.end()) ; - unlockQueue() ; - - LLMutexLock lock(&mNetworkQueueMutex); - if (in_request_map) - { - // only add to the queue if in the request map - // i.e. a delete has not been requested - mNetworkQueue.insert(worker->mID); - } - for (cancel_queue_t::iterator iter1 = mCancelQueue.begin(); - iter1 != mCancelQueue.end(); ++iter1) - { - iter1->second.erase(worker->mID); - } -} - -void LLTextureFetch::removeFromNetworkQueue(LLTextureFetchWorker* worker, bool cancel) -{ - LLMutexLock lock(&mNetworkQueueMutex); - size_t erased = mNetworkQueue.erase(worker->mID); - if (cancel && erased > 0) - { - mCancelQueue[worker->mHost].insert(worker->mID); - } -} - -// protected -void LLTextureFetch::addToHTTPQueue(const LLUUID& id) -{ - LLMutexLock lock(&mNetworkQueueMutex); - mHTTPTextureQueue.insert(id); - mTotalHTTPRequests++; -} - -void LLTextureFetch::removeFromHTTPQueue(const LLUUID& id, S32 received_size) -{ - LLMutexLock lock(&mNetworkQueueMutex); - mHTTPTextureQueue.erase(id); - mHTTPTextureBits += received_size * 8; // Approximate - does not include header bits -} - -void LLTextureFetch::deleteRequest(const LLUUID& id, bool cancel) -{ - lockQueue() ; - LLTextureFetchWorker* worker = getWorkerAfterLock(id); - if (worker) - { - size_t erased_1 = mRequestMap.erase(worker->mID); - unlockQueue() ; - - llassert_always(erased_1 > 0) ; - - removeFromNetworkQueue(worker, cancel); - llassert_always(!(worker->getFlags(LLWorkerClass::WCF_DELETE_REQUESTED))) ; - - worker->scheduleDelete(); - } - else - { - unlockQueue() ; - } -} - -void LLTextureFetch::removeRequest(LLTextureFetchWorker* worker, bool cancel) -{ - lockQueue() ; - size_t erased_1 = mRequestMap.erase(worker->mID); - unlockQueue() ; - - llassert_always(erased_1 > 0) ; - removeFromNetworkQueue(worker, cancel); - llassert_always(!(worker->getFlags(LLWorkerClass::WCF_DELETE_REQUESTED))) ; - - worker->scheduleDelete(); -} - -S32 LLTextureFetch::getNumRequests() -{ - lockQueue() ; - S32 size = (S32)mRequestMap.size(); - unlockQueue() ; - - return size ; -} - -S32 LLTextureFetch::getNumHTTPRequests() -{ - mNetworkQueueMutex.lock() ; - S32 size = (S32)mHTTPTextureQueue.size(); - mNetworkQueueMutex.unlock() ; - - return size ; -} - -U32 LLTextureFetch::getTotalNumHTTPRequests() -{ - mNetworkQueueMutex.lock() ; - U32 size = mTotalHTTPRequests ; - mNetworkQueueMutex.unlock() ; - - return size ; -} - -// call lockQueue() first! -LLTextureFetchWorker* LLTextureFetch::getWorkerAfterLock(const LLUUID& id) -{ - LLTextureFetchWorker* res = NULL; - map_t::iterator iter = mRequestMap.find(id); - if (iter != mRequestMap.end()) - { - res = iter->second; - } - return res; -} - -LLTextureFetchWorker* LLTextureFetch::getWorker(const LLUUID& id) -{ - LLMutexLock lock(&mQueueMutex) ; - - return getWorkerAfterLock(id) ; -} - - -bool LLTextureFetch::getRequestFinished(const LLUUID& id, S32& discard_level, - LLPointer& raw, LLPointer& aux) -{ - bool res = false; - LLTextureFetchWorker* worker = getWorker(id); - if (worker) - { - if (worker->wasAborted()) - { - res = true; - } - else if (!worker->haveWork()) - { - // Should only happen if we set mDebugPause... - if (!mDebugPause) - { -// llwarns << "Adding work for inactive worker: " << id << llendl; - worker->addWork(0, LLWorkerThread::PRIORITY_HIGH | worker->mWorkPriority); - } - } - else if (worker->checkWork()) - { - worker->lockWorkMutex(); - discard_level = worker->mDecodedDiscard; - raw = worker->mRawImage; - aux = worker->mAuxImage; - res = true; - LL_DEBUGS("Texture") << id << ": Request Finished. State: " << worker->mState << " Discard: " << discard_level << LL_ENDL; - worker->unlockWorkMutex(); - } - else - { - worker->lockWorkMutex(); - if ((worker->mDecodedDiscard >= 0) && - (worker->mDecodedDiscard < discard_level || discard_level < 0) && - (worker->mState >= LLTextureFetchWorker::WAIT_ON_WRITE)) - { - // Not finished, but data is ready - discard_level = worker->mDecodedDiscard; - raw = worker->mRawImage; - aux = worker->mAuxImage; - } - worker->unlockWorkMutex(); - } - } - else - { - res = true; - } - return res; -} - -bool LLTextureFetch::updateRequestPriority(const LLUUID& id, F32 priority) -{ - bool res = false; - LLTextureFetchWorker* worker = getWorker(id); - if (worker) - { - worker->lockWorkMutex(); - worker->setImagePriority(priority); - worker->unlockWorkMutex(); - res = true; - } - return res; -} - -// Replicates and expands upon the base class's -// getPending() implementation. getPending() and -// runCondition() replicate one another's logic to -// an extent and are sometimes used for the same -// function (deciding whether or not to sleep/pause -// a thread). So the implementations need to stay -// in step, at least until this can be refactored and -// the redundancy eliminated. -// -// May be called from any thread - -//virtual -S32 LLTextureFetch::getPending() -{ - S32 res; - lockData(); - { - LLMutexLock lock(&mQueueMutex); - - res = mRequestQueue.size(); - res += mCurlPOSTRequestCount; - res += mCommands.size(); - } - unlockData(); - return res; -} - -// virtual -bool LLTextureFetch::runCondition() -{ - // Caller is holding the lock on LLThread's condition variable. - - // LLQueuedThread, unlike its base class LLThread, makes this a - // private method which is unfortunate. I want to use it directly - // but I'm going to have to re-implement the logic here (or change - // declarations, which I don't want to do right now). - // - // Changes here may need to be reflected in getPending(). - - bool have_no_commands(false); - { - LLMutexLock lock(&mQueueMutex); - - have_no_commands = mCommands.empty(); - } - - bool have_no_curl_requests(0 == mCurlPOSTRequestCount); - - return ! (have_no_commands - && have_no_curl_requests - && (mRequestQueue.empty() && mIdleThread)); // From base class -} - -////////////////////////////////////////////////////////////////////////////// - -// MAIN THREAD (unthreaded envs), WORKER THREAD (threaded envs) -void LLTextureFetch::commonUpdate() -{ - // Run a cross-thread command, if any. - cmdDoWork(); - - // Update Curl on same thread as mCurlGetRequest was constructed - S32 processed = mCurlGetRequest->process(); - if (processed > 0) - { - lldebugs << "processed: " << processed << " messages." << llendl; - } -} - - -// MAIN THREAD -//virtual -S32 LLTextureFetch::update(U32 max_time_ms) -{ - static LLCachedControl band_width(gSavedSettings,"ThrottleBandwidthKBPS"); - - { - mNetworkQueueMutex.lock() ; - mMaxBandwidth = band_width ; - - gTextureList.sTextureBits += mHTTPTextureBits ; - mHTTPTextureBits = 0 ; - - mNetworkQueueMutex.unlock() ; - } - - S32 res = LLWorkerThread::update(max_time_ms); - - if (!mDebugPause) - { - sendRequestListToSimulators(); - } - - if (!mThreaded) - { - commonUpdate(); - } - - return res; -} - -//called in the MAIN thread after the TextureCacheThread shuts down. -void LLTextureFetch::shutDownTextureCacheThread() -{ - if(mTextureCache) - { - llassert_always(mTextureCache->isQuitting() || mTextureCache->isStopped()) ; - mTextureCache = NULL ; - } -} - -//called in the MAIN thread after the ImageDecodeThread shuts down. -void LLTextureFetch::shutDownImageDecodeThread() -{ - if(mImageDecodeThread) - { - llassert_always(mImageDecodeThread->isQuitting() || mImageDecodeThread->isStopped()) ; - mImageDecodeThread = NULL ; - } -} - -// WORKER THREAD -void LLTextureFetch::startThread() -{ - // Construct mCurlGetRequest from Worker Thread - mCurlGetRequest = new LLCurlRequest(); -} - -// WORKER THREAD -void LLTextureFetch::endThread() -{ - // Destroy mCurlGetRequest from Worker Thread - delete mCurlGetRequest; - mCurlGetRequest = NULL; -} - -// WORKER THREAD -void LLTextureFetch::threadedUpdate() -{ - llassert_always(mCurlGetRequest); - - // Limit update frequency - const F32 PROCESS_TIME = 0.05f; - static LLFrameTimer process_timer; - if (process_timer.getElapsedTimeF32() < PROCESS_TIME) - { - return; - } - process_timer.reset(); - - commonUpdate(); - -#if 0 - const F32 INFO_TIME = 1.0f; - static LLFrameTimer info_timer; - if (info_timer.getElapsedTimeF32() >= INFO_TIME) - { - S32 q = mCurlGetRequest->getQueued(); - if (q > 0) - { - llinfos << "Queued gets: " << q << llendl; - info_timer.reset(); - } - } -#endif - -} - -////////////////////////////////////////////////////////////////////////////// - -void LLTextureFetch::sendRequestListToSimulators() -{ - // All requests - const F32 REQUEST_DELTA_TIME = 0.10f; // 10 fps - - // Sim requests - const S32 IMAGES_PER_REQUEST = 50; - const F32 SIM_LAZY_FLUSH_TIMEOUT = 10.0f; // temp - const F32 MIN_REQUEST_TIME = 1.0f; - const F32 MIN_DELTA_PRIORITY = 1000.f; - - // Periodically, gather the list of textures that need data from the network - // And send the requests out to the simulators - static LLFrameTimer timer; - if (timer.getElapsedTimeF32() < REQUEST_DELTA_TIME) - { - return; - } - timer.reset(); - - // Send requests - typedef std::set request_list_t; - typedef std::map< LLHost, request_list_t > work_request_map_t; - work_request_map_t requests; - { - LLMutexLock lock2(&mNetworkQueueMutex); - for (queue_t::iterator iter = mNetworkQueue.begin(); iter != mNetworkQueue.end(); ) - { - queue_t::iterator curiter = iter++; - LLTextureFetchWorker* req = getWorker(*curiter); - if (!req) - { - mNetworkQueue.erase(curiter); - continue; // paranoia - } - if ((req->mState != LLTextureFetchWorker::LOAD_FROM_NETWORK) && - (req->mState != LLTextureFetchWorker::LOAD_FROM_SIMULATOR)) - { - // We already received our URL, remove from the queue - llwarns << "Worker: " << req->mID << " in mNetworkQueue but in wrong state: " << req->mState << llendl; - mNetworkQueue.erase(curiter); - continue; - } - if (req->mID == mDebugID) - { - mDebugCount++; // for setting breakpoints - } - if (req->mSentRequest == LLTextureFetchWorker::SENT_SIM && - req->mTotalPackets > 0 && - req->mLastPacket >= req->mTotalPackets-1) - { - // We have all the packets... make sure this is high priority -// req->setPriority(LLWorkerThread::PRIORITY_HIGH | req->mWorkPriority); - continue; - } - F32 elapsed = req->mRequestedTimer.getElapsedTimeF32(); - { - F32 delta_priority = llabs(req->mRequestedPriority - req->mImagePriority); - if ((req->mSimRequestedDiscard != req->mDesiredDiscard) || - (delta_priority > MIN_DELTA_PRIORITY && elapsed >= MIN_REQUEST_TIME) || - (elapsed >= SIM_LAZY_FLUSH_TIMEOUT)) - { - requests[req->mHost].insert(req); - } - } - } - } - - for (work_request_map_t::iterator iter1 = requests.begin(); - iter1 != requests.end(); ++iter1) - { - LLHost host = iter1->first; - // invalid host = use agent host - if (host == LLHost::invalid) - { - host = gAgent.getRegionHost(); - } - - S32 sim_request_count = 0; - - for (request_list_t::iterator iter2 = iter1->second.begin(); - iter2 != iter1->second.end(); ++iter2) - { - LLTextureFetchWorker* req = *iter2; - if (gMessageSystem) - { - if (req->mSentRequest != LLTextureFetchWorker::SENT_SIM) - { - // Initialize packet data based on data read from cache - req->lockWorkMutex(); - req->setupPacketData(); - req->unlockWorkMutex(); - } - if (0 == sim_request_count) - { - gMessageSystem->newMessageFast(_PREHASH_RequestImage); - gMessageSystem->nextBlockFast(_PREHASH_AgentData); - gMessageSystem->addUUIDFast(_PREHASH_AgentID, gAgent.getID()); - gMessageSystem->addUUIDFast(_PREHASH_SessionID, gAgent.getSessionID()); - } - S32 packet = req->mLastPacket + 1; - gMessageSystem->nextBlockFast(_PREHASH_RequestImage); - gMessageSystem->addUUIDFast(_PREHASH_Image, req->mID); - gMessageSystem->addS8Fast(_PREHASH_DiscardLevel, (S8)req->mDesiredDiscard); - gMessageSystem->addF32Fast(_PREHASH_DownloadPriority, req->mImagePriority); - gMessageSystem->addU32Fast(_PREHASH_Packet, packet); - gMessageSystem->addU8Fast(_PREHASH_Type, req->mType); -// llinfos << "IMAGE REQUEST: " << req->mID << " Discard: " << req->mDesiredDiscard -// << " Packet: " << packet << " Priority: " << req->mImagePriority << llendl; - - static LLCachedControl log_to_viewer_log(gSavedSettings,"LogTextureDownloadsToViewerLog"); - static LLCachedControl log_to_sim(gSavedSettings,"LogTextureDownloadsToSimulator"); - if (log_to_viewer_log || log_to_sim) - { - mTextureInfo.setRequestStartTime(req->mID, LLTimer::getTotalTime()); - mTextureInfo.setRequestOffset(req->mID, 0); - mTextureInfo.setRequestSize(req->mID, 0); - mTextureInfo.setRequestType(req->mID, LLTextureInfoDetails::REQUEST_TYPE_UDP); - } - - req->lockWorkMutex(); - req->mSentRequest = LLTextureFetchWorker::SENT_SIM; - req->mSimRequestedDiscard = req->mDesiredDiscard; - req->mRequestedPriority = req->mImagePriority; - req->mRequestedTimer.reset(); - req->unlockWorkMutex(); - sim_request_count++; - if (sim_request_count >= IMAGES_PER_REQUEST) - { -// llinfos << "REQUESTING " << sim_request_count << " IMAGES FROM HOST: " << host.getIPString() << llendl; - - gMessageSystem->sendSemiReliable(host, NULL, NULL); - sim_request_count = 0; - } - } - } - if (gMessageSystem && sim_request_count > 0 && sim_request_count < IMAGES_PER_REQUEST) - { -// llinfos << "REQUESTING " << sim_request_count << " IMAGES FROM HOST: " << host.getIPString() << llendl; - gMessageSystem->sendSemiReliable(host, NULL, NULL); - sim_request_count = 0; - } - } - - // Send cancelations - { - LLMutexLock lock2(&mNetworkQueueMutex); - if (gMessageSystem && !mCancelQueue.empty()) - { - for (cancel_queue_t::iterator iter1 = mCancelQueue.begin(); - iter1 != mCancelQueue.end(); ++iter1) - { - LLHost host = iter1->first; - if (host == LLHost::invalid) - { - host = gAgent.getRegionHost(); - } - S32 request_count = 0; - for (queue_t::iterator iter2 = iter1->second.begin(); - iter2 != iter1->second.end(); ++iter2) - { - if (0 == request_count) - { - gMessageSystem->newMessageFast(_PREHASH_RequestImage); - gMessageSystem->nextBlockFast(_PREHASH_AgentData); - gMessageSystem->addUUIDFast(_PREHASH_AgentID, gAgent.getID()); - gMessageSystem->addUUIDFast(_PREHASH_SessionID, gAgent.getSessionID()); - } - gMessageSystem->nextBlockFast(_PREHASH_RequestImage); - gMessageSystem->addUUIDFast(_PREHASH_Image, *iter2); - gMessageSystem->addS8Fast(_PREHASH_DiscardLevel, -1); - gMessageSystem->addF32Fast(_PREHASH_DownloadPriority, 0); - gMessageSystem->addU32Fast(_PREHASH_Packet, 0); - gMessageSystem->addU8Fast(_PREHASH_Type, 0); -// llinfos << "CANCELING IMAGE REQUEST: " << (*iter2) << llendl; - - request_count++; - if (request_count >= IMAGES_PER_REQUEST) - { - gMessageSystem->sendSemiReliable(host, NULL, NULL); - request_count = 0; - } - } - if (request_count > 0 && request_count < IMAGES_PER_REQUEST) - { - gMessageSystem->sendSemiReliable(host, NULL, NULL); - } - } - mCancelQueue.clear(); - } - } -} - -////////////////////////////////////////////////////////////////////////////// - -bool LLTextureFetchWorker::insertPacket(S32 index, U8* data, S32 size) -{ - mRequestedTimer.reset(); - if (index >= mTotalPackets) - { -// llwarns << "Received Image Packet " << index << " > max: " << mTotalPackets << " for image: " << mID << llendl; - return false; - } - if (index > 0 && index < mTotalPackets-1 && size != MAX_IMG_PACKET_SIZE) - { -// llwarns << "Received bad sized packet: " << index << ", " << size << " != " << MAX_IMG_PACKET_SIZE << " for image: " << mID << llendl; - return false; - } - - if (index >= (S32)mPackets.size()) - { - mPackets.resize(index+1, (PacketData*)NULL); // initializes v to NULL pointers - } - else if (mPackets[index] != NULL) - { -// llwarns << "Received duplicate packet: " << index << " for image: " << mID << llendl; - return false; - } - - mPackets[index] = new PacketData(data, size); - while (mLastPacket+1 < (S32)mPackets.size() && mPackets[mLastPacket+1] != NULL) - { - ++mLastPacket; - } - return true; -} - -bool LLTextureFetch::receiveImageHeader(const LLHost& host, const LLUUID& id, U8 codec, U16 packets, U32 totalbytes, - U16 data_size, U8* data) -{ - LLTextureFetchWorker* worker = getWorker(id); - bool res = true; - - ++mPacketCount; - - if (!worker) - { -// llwarns << "Received header for non active worker: " << id << llendl; - res = false; - } - else if (worker->mState != LLTextureFetchWorker::LOAD_FROM_NETWORK || - worker->mSentRequest != LLTextureFetchWorker::SENT_SIM) - { -// llwarns << "receiveImageHeader for worker: " << id -// << " in state: " << LLTextureFetchWorker::sStateDescs[worker->mState] -// << " sent: " << worker->mSentRequest << llendl; - res = false; - } - else if (worker->mLastPacket != -1) - { - // check to see if we've gotten this packet before -// llwarns << "Received duplicate header for: " << id << llendl; - res = false; - } - else if (!data_size) - { -// llwarns << "Img: " << id << ":" << " Empty Image Header" << llendl; - res = false; - } - if (!res) - { - ++mBadPacketCount; - mNetworkQueueMutex.lock() ; - mCancelQueue[host].insert(id); - mNetworkQueueMutex.unlock() ; - return false; - } - - worker->lockWorkMutex(); - - // Copy header data into image object - worker->mImageCodec = codec; - worker->mTotalPackets = packets; - worker->mFileSize = (S32)totalbytes; - llassert_always(totalbytes > 0); - llassert_always(data_size == FIRST_PACKET_SIZE || data_size == worker->mFileSize); - res = worker->insertPacket(0, data, data_size); - worker->setPriority(LLWorkerThread::PRIORITY_HIGH | worker->mWorkPriority); - worker->mState = LLTextureFetchWorker::LOAD_FROM_SIMULATOR; - worker->unlockWorkMutex(); - return res; -} - -bool LLTextureFetch::receiveImagePacket(const LLHost& host, const LLUUID& id, U16 packet_num, U16 data_size, U8* data) -{ - LLTextureFetchWorker* worker = getWorker(id); - bool res = true; - - ++mPacketCount; - - if (!worker) - { -// llwarns << "Received packet " << packet_num << " for non active worker: " << id << llendl; - res = false; - } - else if (worker->mLastPacket == -1) - { -// llwarns << "Received packet " << packet_num << " before header for: " << id << llendl; - res = false; - } - else if (!data_size) - { -// llwarns << "Img: " << id << ":" << " Empty Image Header" << llendl; - res = false; - } - if (!res) - { - ++mBadPacketCount; - mNetworkQueueMutex.lock() ; - mCancelQueue[host].insert(id); - mNetworkQueueMutex.unlock() ; - return false; - } - - worker->lockWorkMutex(); - - res = worker->insertPacket(packet_num, data, data_size); - - if ((worker->mState == LLTextureFetchWorker::LOAD_FROM_SIMULATOR) || - (worker->mState == LLTextureFetchWorker::LOAD_FROM_NETWORK)) - { - worker->setPriority(LLWorkerThread::PRIORITY_HIGH | worker->mWorkPriority); - worker->mState = LLTextureFetchWorker::LOAD_FROM_SIMULATOR; - } - else - { -// llwarns << "receiveImagePacket " << packet_num << "/" << worker->mLastPacket << " for worker: " << id -// << " in state: " << LLTextureFetchWorker::sStateDescs[worker->mState] << llendl; - removeFromNetworkQueue(worker, true); // failsafe - } - - if(packet_num >= (worker->mTotalPackets - 1)) - { - static LLCachedControl log_to_viewer_log(gSavedSettings,"LogTextureDownloadsToViewerLog"); - static LLCachedControl log_to_sim(gSavedSettings,"LogTextureDownloadsToSimulator"); - - if (log_to_viewer_log || log_to_sim) - { - U64 timeNow = LLTimer::getTotalTime(); - mTextureInfo.setRequestSize(id, worker->mFileSize); - mTextureInfo.setRequestCompleteTimeAndLog(id, timeNow); - } - } - worker->unlockWorkMutex(); - - return res; -} - -////////////////////////////////////////////////////////////////////////////// -BOOL LLTextureFetch::isFromLocalCache(const LLUUID& id) -{ - BOOL from_cache = FALSE ; - - LLTextureFetchWorker* worker = getWorker(id); - if (worker) - { - worker->lockWorkMutex() ; - from_cache = worker->mInLocalCache ; - worker->unlockWorkMutex() ; - } - - return from_cache ; -} - -S32 LLTextureFetch::getFetchState(const LLUUID& id, F32& data_progress_p, F32& requested_priority_p, - U32& fetch_priority_p, F32& fetch_dtime_p, F32& request_dtime_p, bool& can_use_http) -{ - S32 state = LLTextureFetchWorker::INVALID; - F32 data_progress = 0.0f; - F32 requested_priority = 0.0f; - F32 fetch_dtime = 999999.f; - F32 request_dtime = 999999.f; - U32 fetch_priority = 0; - - LLTextureFetchWorker* worker = getWorker(id); - if (worker && worker->haveWork()) - { - worker->lockWorkMutex(); - state = worker->mState; - fetch_dtime = worker->mFetchTimer.getElapsedTimeF32(); - request_dtime = worker->mRequestedTimer.getElapsedTimeF32(); - if (worker->mFileSize > 0) - { - if (state == LLTextureFetchWorker::LOAD_FROM_SIMULATOR) - { - S32 data_size = FIRST_PACKET_SIZE + (worker->mLastPacket-1) * MAX_IMG_PACKET_SIZE; - data_size = llmax(data_size, 0); - data_progress = (F32)data_size / (F32)worker->mFileSize; - } - else if (worker->mFormattedImage.notNull()) - { - data_progress = (F32)worker->mFormattedImage->getDataSize() / (F32)worker->mFileSize; - } - } - if (state >= LLTextureFetchWorker::LOAD_FROM_NETWORK && state <= LLTextureFetchWorker::WAIT_HTTP_REQ) - { - requested_priority = worker->mRequestedPriority; - } - else - { - requested_priority = worker->mImagePriority; - } - fetch_priority = worker->getPriority(); - can_use_http = worker->getCanUseHTTP() ; - worker->unlockWorkMutex(); - } - data_progress_p = data_progress; - requested_priority_p = requested_priority; - fetch_priority_p = fetch_priority; - fetch_dtime_p = fetch_dtime; - request_dtime_p = request_dtime; - return state; -} - -void LLTextureFetch::dump() -{ - llinfos << "LLTextureFetch REQUESTS:" << llendl; - for (request_queue_t::iterator iter = mRequestQueue.begin(); - iter != mRequestQueue.end(); ++iter) - { - LLQueuedThread::QueuedRequest* qreq = *iter; - LLWorkerThread::WorkRequest* wreq = (LLWorkerThread::WorkRequest*)qreq; - LLTextureFetchWorker* worker = (LLTextureFetchWorker*)wreq->getWorkerClass(); - llinfos << " ID: " << worker->mID - << " PRI: " << llformat("0x%08x",wreq->getPriority()) - << " STATE: " << worker->sStateDescs[worker->mState] - << llendl; - } -} - -////////////////////////////////////////////////////////////////////////////// - -// cross-thread command methods - -void LLTextureFetch::commandSetRegion(U64 region_handle) -{ - TFReqSetRegion * req = new TFReqSetRegion(region_handle); - - cmdEnqueue(req); -} - -void LLTextureFetch::commandSendMetrics(const std::string & caps_url, - const LLUUID & session_id, - const LLUUID & agent_id, - LLViewerAssetStats * main_stats) -{ - TFReqSendMetrics * req = new TFReqSendMetrics(caps_url, session_id, agent_id, main_stats); - - cmdEnqueue(req); -} - -void LLTextureFetch::commandDataBreak() -{ - // The pedantically correct way to implement this is to create a command - // request object in the above fashion and enqueue it. However, this is - // simple data of an advisorial not operational nature and this case - // of shared-write access is tolerable. - - LLTextureFetch::svMetricsDataBreak = true; -} - -void LLTextureFetch::cmdEnqueue(TFRequest * req) -{ - lockQueue(); - mCommands.push_back(req); - unlockQueue(); - - unpause(); -} - -LLTextureFetch::TFRequest * LLTextureFetch::cmdDequeue() -{ - TFRequest * ret = 0; - - lockQueue(); - if (! mCommands.empty()) - { - ret = mCommands.front(); - mCommands.erase(mCommands.begin()); - } - unlockQueue(); - - return ret; -} - -void LLTextureFetch::cmdDoWork() -{ - if (mDebugPause) - { - return; // debug: don't do any work - } - - TFRequest * req = cmdDequeue(); - if (req) - { - // One request per pass should really be enough for this. - req->doWork(this); - delete req; - } -} - - -////////////////////////////////////////////////////////////////////////////// - -// Private (anonymous) class methods implementing the command scheme. - -namespace -{ - -/** - * Implements the 'Set Region' command. - * - * Thread: Thread1 (TextureFetch) - */ -bool -TFReqSetRegion::doWork(LLTextureFetch *) -{ - LLViewerAssetStatsFF::set_region_thread1(mRegionHandle); - - return true; -} - - -TFReqSendMetrics::~TFReqSendMetrics() -{ - delete mMainStats; - mMainStats = 0; -} - - -/** - * Implements the 'Send Metrics' command. Takes over - * ownership of the passed LLViewerAssetStats pointer. - * - * Thread: Thread1 (TextureFetch) - */ -bool -TFReqSendMetrics::doWork(LLTextureFetch * fetcher) -{ - /* - * HTTP POST responder. Doesn't do much but tries to - * detect simple breaks in recording the metrics stream. - * - * The 'volatile' modifiers don't indicate signals, - * mmap'd memory or threads, really. They indicate that - * the referenced data is part of a pseudo-closure for - * this responder rather than being required for correct - * operation. - * - * We don't try very hard with the POST request. We give - * it one shot and that's more-or-less it. With a proper - * refactoring of the LLQueuedThread usage, these POSTs - * could be put in a request object and made more reliable. - */ - class lcl_responder : public LLCurl::Responder - { - public: - lcl_responder(LLTextureFetch * fetcher, - S32 expected_sequence, - volatile const S32 & live_sequence, - volatile bool & reporting_break, - volatile bool & reporting_started) - : LLCurl::Responder(), - mFetcher(fetcher), - mExpectedSequence(expected_sequence), - mLiveSequence(live_sequence), - mReportingBreak(reporting_break), - mReportingStarted(reporting_started) - { - mFetcher->incrCurlPOSTCount(); - } - - ~lcl_responder() - { - mFetcher->decrCurlPOSTCount(); - } - - // virtual - void error(U32 status_num, const std::string & reason) - { - if (mLiveSequence == mExpectedSequence) - { - mReportingBreak = true; - } - LL_WARNS("Texture") << "Break in metrics stream due to POST failure to metrics collection service. Reason: " - << reason << LL_ENDL; - } - - // virtual - void result(const LLSD & content) - { - if (mLiveSequence == mExpectedSequence) - { - mReportingBreak = false; - mReportingStarted = true; - } - } - - private: - LLTextureFetch * mFetcher; - S32 mExpectedSequence; - volatile const S32 & mLiveSequence; - volatile bool & mReportingBreak; - volatile bool & mReportingStarted; - - }; // class lcl_responder - - if (! gViewerAssetStatsThread1) - return true; - - static volatile bool reporting_started(false); - static volatile S32 report_sequence(0); - - // We've taken over ownership of the stats copy at this - // point. Get a working reference to it for merging here - // but leave it in 'this'. Destructor will rid us of it. - LLViewerAssetStats & main_stats = *mMainStats; - - // Merge existing stats into those from main, convert to LLSD - main_stats.merge(*gViewerAssetStatsThread1); - LLSD merged_llsd = main_stats.asLLSD(true); - - // Add some additional meta fields to the content - merged_llsd["session_id"] = mSessionID; - merged_llsd["agent_id"] = mAgentID; - merged_llsd["message"] = "ViewerAssetMetrics"; // Identifies the type of metrics - merged_llsd["sequence"] = report_sequence; // Sequence number - merged_llsd["initial"] = ! reporting_started; // Initial data from viewer - merged_llsd["break"] = LLTextureFetch::svMetricsDataBreak; // Break in data prior to this report - - // Update sequence number - if (S32_MAX == ++report_sequence) - report_sequence = 0; - - // Limit the size of the stats report if necessary. - merged_llsd["truncated"] = truncate_viewer_metrics(10, merged_llsd); - - if (! mCapsURL.empty()) - { - LLCurlRequest::headers_t headers; - fetcher->getCurlRequest().post(mCapsURL, - headers, - merged_llsd, - new lcl_responder(fetcher, - report_sequence, - report_sequence, - LLTextureFetch::svMetricsDataBreak, - reporting_started)); - } - else - { - LLTextureFetch::svMetricsDataBreak = true; - } - - // In QA mode, Metrics submode, log the result for ease of testing - if (fetcher->isQAMode()) - { - LL_INFOS("Textures") << merged_llsd << LL_ENDL; - } - - gViewerAssetStatsThread1->reset(); - - return true; -} - - -bool -truncate_viewer_metrics(int max_regions, LLSD & metrics) -{ - static const LLSD::String reg_tag("regions"); - static const LLSD::String duration_tag("duration"); - - LLSD & reg_map(metrics[reg_tag]); - if (reg_map.size() <= max_regions) - { - return false; - } - - // Build map of region hashes ordered by duration - typedef std::multimap reg_ordered_list_t; - reg_ordered_list_t regions_by_duration; - - int ind(0); - LLSD::array_const_iterator it_end(reg_map.endArray()); - for (LLSD::array_const_iterator it(reg_map.beginArray()); it_end != it; ++it, ++ind) - { - LLSD::Real duration = (*it)[duration_tag].asReal(); - regions_by_duration.insert(reg_ordered_list_t::value_type(duration, ind)); - } - - // Build a replacement regions array with the longest-persistence regions - LLSD new_region(LLSD::emptyArray()); - reg_ordered_list_t::const_reverse_iterator it2_end(regions_by_duration.rend()); - reg_ordered_list_t::const_reverse_iterator it2(regions_by_duration.rbegin()); - for (int i(0); i < max_regions && it2_end != it2; ++i, ++it2) - { - new_region.append(reg_map[it2->second]); - } - reg_map = new_region; - - return true; -} - -} // end of anonymous namespace - - - +/** + * @file lltexturefetch.cpp + * @brief Object which fetches textures from the cache and/or network + * + * $LicenseInfo:firstyear=2000&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2010, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include "llviewerprecompiledheaders.h" + +#include +#include + +#include "llstl.h" + +#include "lltexturefetch.h" + +#include "llcurl.h" +#include "lldir.h" +#include "llhttpclient.h" +#include "llhttpstatuscodes.h" +#include "llimage.h" +#include "llimagej2c.h" +#include "llimageworker.h" +#include "llworkerthread.h" +#include "message.h" + +#include "llagent.h" +#include "lltexturecache.h" +#include "llviewercontrol.h" +#include "llviewertexturelist.h" +#include "llviewertexture.h" +#include "llviewerregion.h" +#include "llviewerstats.h" +#include "llviewerassetstats.h" +#include "llworld.h" + +////////////////////////////////////////////////////////////////////////////// +class LLTextureFetchWorker : public LLWorkerClass +{ + friend class LLTextureFetch; + friend class HTTPGetResponder; + +private: + class CacheReadResponder : public LLTextureCache::ReadResponder + { + public: + CacheReadResponder(LLTextureFetch* fetcher, const LLUUID& id, LLImageFormatted* image) + : mFetcher(fetcher), mID(id) + { + setImage(image); + } + virtual void completed(bool success) + { + LLTextureFetchWorker* worker = mFetcher->getWorker(mID); + if (worker) + { + worker->callbackCacheRead(success, mFormattedImage, mImageSize, mImageLocal); + } + } + private: + LLTextureFetch* mFetcher; + LLUUID mID; + }; + + class CacheWriteResponder : public LLTextureCache::WriteResponder + { + public: + CacheWriteResponder(LLTextureFetch* fetcher, const LLUUID& id) + : mFetcher(fetcher), mID(id) + { + } + virtual void completed(bool success) + { + LLTextureFetchWorker* worker = mFetcher->getWorker(mID); + if (worker) + { + worker->callbackCacheWrite(success); + } + } + private: + LLTextureFetch* mFetcher; + LLUUID mID; + }; + + class DecodeResponder : public LLImageDecodeThread::Responder + { + public: + DecodeResponder(LLTextureFetch* fetcher, const LLUUID& id, LLTextureFetchWorker* worker) + : mFetcher(fetcher), mID(id), mWorker(worker) + { + } + virtual void completed(bool success, LLImageRaw* raw, LLImageRaw* aux) + { + LLTextureFetchWorker* worker = mFetcher->getWorker(mID); + if (worker) + { + worker->callbackDecoded(success, raw, aux); + } + } + private: + LLTextureFetch* mFetcher; + LLUUID mID; + LLTextureFetchWorker* mWorker; // debug only (may get deleted from under us, use mFetcher/mID) + }; + + struct Compare + { + // lhs < rhs + bool operator()(const LLTextureFetchWorker* lhs, const LLTextureFetchWorker* rhs) const + { + // greater priority is "less" + const F32 lpriority = lhs->mImagePriority; + const F32 rpriority = rhs->mImagePriority; + if (lpriority > rpriority) // higher priority + return true; + else if (lpriority < rpriority) + return false; + else + return lhs < rhs; + } + }; + +public: + /*virtual*/ bool doWork(S32 param); // Called from LLWorkerThread::processRequest() + /*virtual*/ void finishWork(S32 param, bool completed); // called from finishRequest() (WORK THREAD) + /*virtual*/ bool deleteOK(); // called from update() (WORK THREAD) + + ~LLTextureFetchWorker(); + // void relese() { --mActiveCount; } + + S32 callbackHttpGet(const LLChannelDescriptors& channels, + const LLIOPipe::buffer_ptr_t& buffer, + bool partial, bool success); + void callbackCacheRead(bool success, LLImageFormatted* image, + S32 imagesize, BOOL islocal); + void callbackCacheWrite(bool success); + void callbackDecoded(bool success, LLImageRaw* raw, LLImageRaw* aux); + + void setGetStatus(U32 status, const std::string& reason) + { + LLMutexLock lock(&mWorkMutex); + + mGetStatus = status; + mGetReason = reason; + } + + void setCanUseHTTP(bool can_use_http) { mCanUseHTTP = can_use_http; } + bool getCanUseHTTP() const { return mCanUseHTTP; } + + LLTextureFetch & getFetcher() { return *mFetcher; } + +protected: + LLTextureFetchWorker(LLTextureFetch* fetcher, const std::string& url, const LLUUID& id, const LLHost& host, + F32 priority, S32 discard, S32 size); + +private: + /*virtual*/ void startWork(S32 param); // called from addWork() (MAIN THREAD) + /*virtual*/ void endWork(S32 param, bool aborted); // called from doWork() (MAIN THREAD) + + void resetFormattedData(); + + void setImagePriority(F32 priority); + void setDesiredDiscard(S32 discard, S32 size); + bool insertPacket(S32 index, U8* data, S32 size); + void clearPackets(); + void setupPacketData(); + U32 calcWorkPriority(); + void removeFromCache(); + bool processSimulatorPackets(); + bool writeToCacheComplete(); + + void lockWorkMutex() { mWorkMutex.lock(); } + void unlockWorkMutex() { mWorkMutex.unlock(); } + +private: + enum e_state // mState + { + // NOTE: Affects LLTextureBar::draw in lltextureview.cpp (debug hack) + INVALID = 0, + INIT, + LOAD_FROM_TEXTURE_CACHE, + CACHE_POST, + LOAD_FROM_NETWORK, + LOAD_FROM_SIMULATOR, + SEND_HTTP_REQ, + WAIT_HTTP_REQ, + DECODE_IMAGE, + DECODE_IMAGE_UPDATE, + WRITE_TO_CACHE, + WAIT_ON_WRITE, + DONE + }; + enum e_request_state // mSentRequest + { + UNSENT = 0, + QUEUED = 1, + SENT_SIM = 2 + }; + enum e_write_to_cache_state //mWriteToCacheState + { + NOT_WRITE = 0, + CAN_WRITE = 1, + SHOULD_WRITE = 2 + }; + static const char* sStateDescs[]; + e_state mState; + e_write_to_cache_state mWriteToCacheState; + LLTextureFetch* mFetcher; + LLPointer mFormattedImage; + LLPointer mRawImage; + LLPointer mAuxImage; + LLUUID mID; + LLHost mHost; + std::string mUrl; + U8 mType; + F32 mImagePriority; + U32 mWorkPriority; + F32 mRequestedPriority; + S32 mDesiredDiscard; + S32 mSimRequestedDiscard; + S32 mRequestedDiscard; + S32 mLoadedDiscard; + S32 mDecodedDiscard; + LLFrameTimer mRequestedTimer; + LLFrameTimer mFetchTimer; + LLTextureCache::handle_t mCacheReadHandle; + LLTextureCache::handle_t mCacheWriteHandle; + U8* mBuffer; + S32 mBufferSize; + S32 mRequestedSize; + S32 mDesiredSize; + S32 mFileSize; + S32 mCachedSize; + e_request_state mSentRequest; + handle_t mDecodeHandle; + BOOL mLoaded; + BOOL mDecoded; + BOOL mWritten; + BOOL mNeedsAux; + BOOL mHaveAllData; + BOOL mInLocalCache; + bool mCanUseHTTP ; + bool mCanUseNET ; //can get from asset server. + S32 mHTTPFailCount; + S32 mRetryAttempt; + S32 mActiveCount; + U32 mGetStatus; + std::string mGetReason; + + // Work Data + LLMutex mWorkMutex; + struct PacketData + { + PacketData(U8* data, S32 size) { mData = data; mSize = size; } + ~PacketData() { clearData(); } + void clearData() { delete[] mData; mData = NULL; } + U8* mData; + U32 mSize; + }; + std::vector mPackets; + S32 mFirstPacket; + S32 mLastPacket; + U16 mTotalPackets; + U8 mImageCodec; + + LLViewerAssetStats::duration_t mMetricsStartTime; +}; + +////////////////////////////////////////////////////////////////////////////// + +class HTTPGetResponder : public LLCurl::Responder +{ + LOG_CLASS(HTTPGetResponder); +public: + HTTPGetResponder(LLTextureFetch* fetcher, const LLUUID& id, U64 startTime, S32 requestedSize, U32 offset, bool redir) + : mFetcher(fetcher), mID(id), mStartTime(startTime), mRequestedSize(requestedSize), mOffset(offset), mFollowRedir(redir) + { + } + ~HTTPGetResponder() + { + } + + virtual void completedRaw(U32 status, const std::string& reason, + const LLChannelDescriptors& channels, + const LLIOPipe::buffer_ptr_t& buffer) + { + static LLCachedControl log_to_viewer_log(gSavedSettings,"LogTextureDownloadsToViewerLog"); + static LLCachedControl log_to_sim(gSavedSettings,"LogTextureDownloadsToSimulator"); + static LLCachedControl log_texture_traffic(gSavedSettings,"LogTextureNetworkTraffic") ; + + if (log_to_viewer_log || log_to_sim) + { + mFetcher->mTextureInfo.setRequestStartTime(mID, mStartTime); + U64 timeNow = LLTimer::getTotalTime(); + mFetcher->mTextureInfo.setRequestType(mID, LLTextureInfoDetails::REQUEST_TYPE_HTTP); + mFetcher->mTextureInfo.setRequestSize(mID, mRequestedSize); + mFetcher->mTextureInfo.setRequestOffset(mID, mOffset); + mFetcher->mTextureInfo.setRequestCompleteTimeAndLog(mID, timeNow); + } + + lldebugs << "HTTP COMPLETE: " << mID << llendl; + LLTextureFetchWorker* worker = mFetcher->getWorker(mID); + if (worker) + { + bool success = false; + bool partial = false; + if (HTTP_OK <= status && status < HTTP_MULTIPLE_CHOICES) + { + success = true; + if (HTTP_PARTIAL_CONTENT == status) // partial information + { + partial = true; + } + } + + if (!success) + { + worker->setGetStatus(status, reason); +// llwarns << "CURL GET FAILED, status:" << status << " reason:" << reason << llendl; + } + + S32 data_size = worker->callbackHttpGet(channels, buffer, partial, success); + + if(log_texture_traffic && data_size > 0) + { + LLViewerTexture* tex = LLViewerTextureManager::findTexture(mID) ; + if(tex) + { + gTotalTextureBytesPerBoostLevel[tex->getBoostLevel()] += data_size ; + } + } + + mFetcher->removeFromHTTPQueue(mID, data_size); + + if (worker->mMetricsStartTime) + { + LLViewerAssetStatsFF::record_response_thread1(LLViewerAssetType::AT_TEXTURE, + true, + LLImageBase::TYPE_AVATAR_BAKE == worker->mType, + LLViewerAssetStatsFF::get_timestamp() - worker->mMetricsStartTime); + worker->mMetricsStartTime = 0; + } + LLViewerAssetStatsFF::record_dequeue_thread1(LLViewerAssetType::AT_TEXTURE, + true, + LLImageBase::TYPE_AVATAR_BAKE == worker->mType); + } + else + { + mFetcher->removeFromHTTPQueue(mID); + llwarns << "Worker not found: " << mID << llendl; + } + } + + virtual bool followRedir() + { + return mFollowRedir; + } + +private: + LLTextureFetch* mFetcher; + LLUUID mID; + U64 mStartTime; + S32 mRequestedSize; + U32 mOffset; + bool mFollowRedir; +}; + +////////////////////////////////////////////////////////////////////////////// + +// Cross-thread messaging for asset metrics. + +/** + * @brief Base class for cross-thread requests made of the fetcher + * + * I believe the intent of the LLQueuedThread class was to + * have these operations derived from LLQueuedThread::QueuedRequest + * but the texture fetcher has elected to manage the queue + * in its own manner. So these are free-standing objects which are + * managed in simple FIFO order on the mCommands queue of the + * LLTextureFetch object. + * + * What each represents is a simple command sent from an + * outside thread into the TextureFetch thread to be processed + * in order and in a timely fashion (though not an absolute + * higher priority than other operations of the thread). + * Each operation derives a new class from the base customizing + * members, constructors and the doWork() method to effect + * the command. + * + * The flow is one-directional. There are two global instances + * of the LLViewerAssetStats collector, one for the main program's + * thread pointed to by gViewerAssetStatsMain and one for the + * TextureFetch thread pointed to by gViewerAssetStatsThread1. + * Common operations has each thread recording metrics events + * into the respective collector unconcerned with locking and + * the state of any other thread. But when the agent moves into + * a different region or the metrics timer expires and a report + * needs to be sent back to the grid, messaging across threads + * is required to distribute data and perform global actions. + * In pseudo-UML, it looks like: + * + * Main Thread1 + * . . + * . . + * +-----+ . + * | AM | . + * +--+--+ . + * +-------+ | . + * | Main | +--+--+ . + * | | | SRE |---. . + * | Stats | +-----+ \ . + * | | | \ (uuid) +-----+ + * | Coll. | +--+--+ `-------->| SR | + * +-------+ | MSC | +--+--+ + * | ^ +-----+ | + * | | (uuid) / . +-----+ (uuid) + * | `--------' . | MSC |---------. + * | . +-----+ | + * | +-----+ . v + * | | TE | . +-------+ + * | +--+--+ . | Thd1 | + * | | . | | + * | +-----+ . | Stats | + * `--------->| RSC | . | | + * +--+--+ . | Coll. | + * | . +-------+ + * +--+--+ . | + * | SME |---. . | + * +-----+ \ . | + * . \ (clone) +-----+ | + * . `-------->| SM | | + * . +--+--+ | + * . | | + * . +-----+ | + * . | RSC |<--------' + * . +-----+ + * . | + * . +-----+ + * . | CP |--> HTTP POST + * . +-----+ + * . . + * . . + * + * + * Key: + * + * SRE - Set Region Enqueued. Enqueue a 'Set Region' command in + * the other thread providing the new UUID of the region. + * TFReqSetRegion carries the data. + * SR - Set Region. New region UUID is sent to the thread-local + * collector. + * SME - Send Metrics Enqueued. Enqueue a 'Send Metrics' command + * including an ownership transfer of a cloned LLViewerAssetStats. + * TFReqSendMetrics carries the data. + * SM - Send Metrics. Global metrics reporting operation. Takes + * the cloned stats from the command, merges it with the + * thread's local stats, converts to LLSD and sends it on + * to the grid. + * AM - Agent Moved. Agent has completed some sort of move to a + * new region. + * TE - Timer Expired. Metrics timer has expired (on the order + * of 10 minutes). + * CP - CURL Post + * MSC - Modify Stats Collector. State change in the thread-local + * collector. Typically a region change which affects the + * global pointers used to find the 'current stats'. + * RSC - Read Stats Collector. Extract collector data cloning it + * (i.e. deep copy) when necessary. + * + */ +class LLTextureFetch::TFRequest // : public LLQueuedThread::QueuedRequest +{ +public: + // Default ctors and assignment operator are correct. + + virtual ~TFRequest() + {} + + // Patterned after QueuedRequest's method but expected behavior + // is different. Always expected to complete on the first call + // and work dispatcher will assume the same and delete the + // request after invocation. + virtual bool doWork(LLTextureFetch * fetcher) = 0; +}; + +namespace +{ + +/** + * @brief Implements a 'Set Region' cross-thread command. + * + * When an agent moves to a new region, subsequent metrics need + * to be binned into a new or existing stats collection in 1:1 + * relationship with the region. We communicate this region + * change across the threads involved in the communication with + * this message. + * + * Corresponds to LLTextureFetch::commandSetRegion() + */ +class TFReqSetRegion : public LLTextureFetch::TFRequest +{ +public: + TFReqSetRegion(U64 region_handle) + : LLTextureFetch::TFRequest(), + mRegionHandle(region_handle) + {} + TFReqSetRegion & operator=(const TFReqSetRegion &); // Not defined + + virtual ~TFReqSetRegion() + {} + + virtual bool doWork(LLTextureFetch * fetcher); + +public: + const U64 mRegionHandle; +}; + + +/** + * @brief Implements a 'Send Metrics' cross-thread command. + * + * This is the big operation. The main thread gathers metrics + * for a period of minutes into LLViewerAssetStats and other + * objects then makes a snapshot of the data by cloning the + * collector. This command transfers the clone, along with a few + * additional arguments (UUIDs), handing ownership to the + * TextureFetch thread. It then merges its own data into the + * cloned copy, converts to LLSD and kicks off an HTTP POST of + * the resulting data to the currently active metrics collector. + * + * Corresponds to LLTextureFetch::commandSendMetrics() + */ +class TFReqSendMetrics : public LLTextureFetch::TFRequest +{ +public: + /** + * Construct the 'Send Metrics' command to have the TextureFetch + * thread add and log metrics data. + * + * @param caps_url URL of a "ViewerMetrics" Caps target + * to receive the data. Does not have to + * be associated with a particular region. + * + * @param session_id UUID of the agent's session. + * + * @param agent_id UUID of the agent. (Being pure here...) + * + * @param main_stats Pointer to a clone of the main thread's + * LLViewerAssetStats data. Thread1 takes + * ownership of the copy and disposes of it + * when done. + */ + TFReqSendMetrics(const std::string & caps_url, + const LLUUID & session_id, + const LLUUID & agent_id, + LLViewerAssetStats * main_stats) + : LLTextureFetch::TFRequest(), + mCapsURL(caps_url), + mSessionID(session_id), + mAgentID(agent_id), + mMainStats(main_stats) + {} + TFReqSendMetrics & operator=(const TFReqSendMetrics &); // Not defined + + virtual ~TFReqSendMetrics(); + + virtual bool doWork(LLTextureFetch * fetcher); + +public: + const std::string mCapsURL; + const LLUUID mSessionID; + const LLUUID mAgentID; + LLViewerAssetStats * mMainStats; +}; + +/* + * Examines the merged viewer metrics report and if found to be too long, + * will attempt to truncate it in some reasonable fashion. + * + * @param max_regions Limit of regions allowed in report. + * + * @param metrics Full, merged viewer metrics report. + * + * @returns If data was truncated, returns true. + */ +bool truncate_viewer_metrics(int max_regions, LLSD & metrics); + +} // end of anonymous namespace + + +////////////////////////////////////////////////////////////////////////////// + +//static +const char* LLTextureFetchWorker::sStateDescs[] = { + "INVALID", + "INIT", + "LOAD_FROM_TEXTURE_CACHE", + "CACHE_POST", + "LOAD_FROM_NETWORK", + "LOAD_FROM_SIMULATOR", + "SEND_HTTP_REQ", + "WAIT_HTTP_REQ", + "DECODE_IMAGE", + "DECODE_IMAGE_UPDATE", + "WRITE_TO_CACHE", + "WAIT_ON_WRITE", + "DONE", +}; + +// static +volatile bool LLTextureFetch::svMetricsDataBreak(true); // Start with a data break + +// called from MAIN THREAD + +LLTextureFetchWorker::LLTextureFetchWorker(LLTextureFetch* fetcher, + const std::string& url, // Optional URL + const LLUUID& id, // Image UUID + const LLHost& host, // Simulator host + F32 priority, // Priority + S32 discard, // Desired discard + S32 size) // Desired size + : LLWorkerClass(fetcher, "TextureFetch"), + mState(INIT), + mWriteToCacheState(NOT_WRITE), + mFetcher(fetcher), + mID(id), + mHost(host), + mUrl(url), + mImagePriority(priority), + mWorkPriority(0), + mRequestedPriority(0.f), + mDesiredDiscard(-1), + mSimRequestedDiscard(-1), + mRequestedDiscard(-1), + mLoadedDiscard(-1), + mDecodedDiscard(-1), + mCacheReadHandle(LLTextureCache::nullHandle()), + mCacheWriteHandle(LLTextureCache::nullHandle()), + mBuffer(NULL), + mBufferSize(0), + mRequestedSize(0), + mDesiredSize(TEXTURE_CACHE_ENTRY_SIZE), + mFileSize(0), + mCachedSize(0), + mLoaded(FALSE), + mSentRequest(UNSENT), + mDecodeHandle(0), + mDecoded(FALSE), + mWritten(FALSE), + mNeedsAux(FALSE), + mHaveAllData(FALSE), + mInLocalCache(FALSE), + mCanUseHTTP(true), + mHTTPFailCount(0), + mRetryAttempt(0), + mActiveCount(0), + mGetStatus(0), + mWorkMutex(NULL), + mFirstPacket(0), + mLastPacket(-1), + mTotalPackets(0), + mImageCodec(IMG_CODEC_INVALID), + mMetricsStartTime(0) +{ + mCanUseNET = mUrl.empty() ; + + calcWorkPriority(); + mType = host.isOk() ? LLImageBase::TYPE_AVATAR_BAKE : LLImageBase::TYPE_NORMAL; +// llinfos << "Create: " << mID << " mHost:" << host << " Discard=" << discard << llendl; + if (!mFetcher->mDebugPause) + { + U32 work_priority = mWorkPriority | LLWorkerThread::PRIORITY_HIGH; + addWork(0, work_priority ); + } + setDesiredDiscard(discard, size); +} + +LLTextureFetchWorker::~LLTextureFetchWorker() +{ +// llinfos << "Destroy: " << mID +// << " Decoded=" << mDecodedDiscard +// << " Requested=" << mRequestedDiscard +// << " Desired=" << mDesiredDiscard << llendl; + llassert_always(!haveWork()); + lockWorkMutex(); + if (mCacheReadHandle != LLTextureCache::nullHandle() && mFetcher->mTextureCache) + { + mFetcher->mTextureCache->readComplete(mCacheReadHandle, true); + } + if (mCacheWriteHandle != LLTextureCache::nullHandle() && mFetcher->mTextureCache) + { + mFetcher->mTextureCache->writeComplete(mCacheWriteHandle, true); + } + mFormattedImage = NULL; + clearPackets(); + unlockWorkMutex(); + mFetcher->removeFromHTTPQueue(mID); +} + +void LLTextureFetchWorker::clearPackets() +{ + for_each(mPackets.begin(), mPackets.end(), DeletePointer()); + mPackets.clear(); + mTotalPackets = 0; + mLastPacket = -1; + mFirstPacket = 0; +} + +void LLTextureFetchWorker::setupPacketData() +{ + S32 data_size = 0; + if (mFormattedImage.notNull()) + { + data_size = mFormattedImage->getDataSize(); + } + if (data_size > 0) + { + // Only used for simulator requests + mFirstPacket = (data_size - FIRST_PACKET_SIZE) / MAX_IMG_PACKET_SIZE + 1; + if (FIRST_PACKET_SIZE + (mFirstPacket-1) * MAX_IMG_PACKET_SIZE != data_size) + { + llwarns << "Bad CACHED TEXTURE size: " << data_size << " removing." << llendl; + removeFromCache(); + resetFormattedData(); + clearPackets(); + } + else if (mFileSize > 0) + { + mLastPacket = mFirstPacket-1; + mTotalPackets = (mFileSize - FIRST_PACKET_SIZE + MAX_IMG_PACKET_SIZE-1) / MAX_IMG_PACKET_SIZE + 1; + } + else + { + // This file was cached using HTTP so we have to refetch the first packet + resetFormattedData(); + clearPackets(); + } + } +} + +U32 LLTextureFetchWorker::calcWorkPriority() +{ + //llassert_always(mImagePriority >= 0 && mImagePriority <= LLViewerFetchedTexture::maxDecodePriority()); + static const F32 PRIORITY_SCALE = (F32)LLWorkerThread::PRIORITY_LOWBITS / LLViewerFetchedTexture::maxDecodePriority(); + + mWorkPriority = llmin((U32)LLWorkerThread::PRIORITY_LOWBITS, (U32)(mImagePriority * PRIORITY_SCALE)); + return mWorkPriority; +} + +// mWorkMutex is locked +void LLTextureFetchWorker::setDesiredDiscard(S32 discard, S32 size) +{ + bool prioritize = false; + if (mDesiredDiscard != discard) + { + if (!haveWork()) + { + calcWorkPriority(); + if (!mFetcher->mDebugPause) + { + U32 work_priority = mWorkPriority | LLWorkerThread::PRIORITY_HIGH; + addWork(0, work_priority); + } + } + else if (mDesiredDiscard < discard) + { + prioritize = true; + } + mDesiredDiscard = discard; + mDesiredSize = size; + } + else if (size > mDesiredSize) + { + mDesiredSize = size; + prioritize = true; + } + mDesiredSize = llmax(mDesiredSize, TEXTURE_CACHE_ENTRY_SIZE); + if ((prioritize && mState == INIT) || mState == DONE) + { + mState = INIT; + U32 work_priority = mWorkPriority | LLWorkerThread::PRIORITY_HIGH; + setPriority(work_priority); + } +} + +void LLTextureFetchWorker::setImagePriority(F32 priority) +{ +// llassert_always(priority >= 0 && priority <= LLViewerTexture::maxDecodePriority()); + F32 delta = fabs(priority - mImagePriority); + if (delta > (mImagePriority * .05f) || mState == DONE) + { + mImagePriority = priority; + calcWorkPriority(); + U32 work_priority = mWorkPriority | (getPriority() & LLWorkerThread::PRIORITY_HIGHBITS); + setPriority(work_priority); + } +} + +void LLTextureFetchWorker::resetFormattedData() +{ + delete[] mBuffer; + mBuffer = NULL; + mBufferSize = 0; + if (mFormattedImage.notNull()) + { + mFormattedImage->deleteData(); + } + mHaveAllData = FALSE; +} + +// Called from MAIN thread +void LLTextureFetchWorker::startWork(S32 param) +{ + llassert(mFormattedImage.isNull()); +} + +#include "llviewertexturelist.h" // debug + +// Called from LLWorkerThread::processRequest() +bool LLTextureFetchWorker::doWork(S32 param) +{ + LLMutexLock lock(&mWorkMutex); + + if ((mFetcher->isQuitting() || getFlags(LLWorkerClass::WCF_DELETE_REQUESTED))) + { + if (mState < DECODE_IMAGE) + { + return true; // abort + } + } + + if(mImagePriority < F_ALMOST_ZERO) + { + if (mState == INIT || mState == LOAD_FROM_NETWORK || mState == LOAD_FROM_SIMULATOR) + { + return true; // abort + } + } + if(mState > CACHE_POST && !mCanUseNET && !mCanUseHTTP) + { + //nowhere to get data, abort. + return true ; + } + + if (mFetcher->mDebugPause) + { + return false; // debug: don't do any work + } + if (mID == mFetcher->mDebugID) + { + mFetcher->mDebugCount++; // for setting breakpoints + } + + if (mState != DONE) + { + mFetchTimer.reset(); + } + + if (mState == INIT) + { + mRawImage = NULL ; + mRequestedDiscard = -1; + mLoadedDiscard = -1; + mDecodedDiscard = -1; + mRequestedSize = 0; + mFileSize = 0; + mCachedSize = 0; + mLoaded = FALSE; + mSentRequest = UNSENT; + mDecoded = FALSE; + mWritten = FALSE; + delete[] mBuffer; + mBuffer = NULL; + mBufferSize = 0; + mHaveAllData = FALSE; + clearPackets(); // TODO: Shouldn't be necessary + mCacheReadHandle = LLTextureCache::nullHandle(); + mCacheWriteHandle = LLTextureCache::nullHandle(); + mState = LOAD_FROM_TEXTURE_CACHE; + mDesiredSize = llmax(mDesiredSize, TEXTURE_CACHE_ENTRY_SIZE); // min desired size is TEXTURE_CACHE_ENTRY_SIZE + LL_DEBUGS("Texture") << mID << ": Priority: " << llformat("%8.0f",mImagePriority) + << " Desired Discard: " << mDesiredDiscard << " Desired Size: " << mDesiredSize << LL_ENDL; + // fall through + } + + if (mState == LOAD_FROM_TEXTURE_CACHE) + { + if (mCacheReadHandle == LLTextureCache::nullHandle()) + { + U32 cache_priority = mWorkPriority; + S32 offset = mFormattedImage.notNull() ? mFormattedImage->getDataSize() : 0; + S32 size = mDesiredSize - offset; + if (size <= 0) + { + mState = CACHE_POST; + return false; + } + mFileSize = 0; + mLoaded = FALSE; + + if (mUrl.compare(0, 7, "file://") == 0) + { + setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); // Set priority first since Responder may change it + + // read file from local disk + std::string filename = mUrl.substr(7, std::string::npos); + CacheReadResponder* responder = new CacheReadResponder(mFetcher, mID, mFormattedImage); + mCacheReadHandle = mFetcher->mTextureCache->readFromCache(filename, mID, cache_priority, + offset, size, responder); + } + else if (mUrl.empty()) + { + setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); // Set priority first since Responder may change it + + CacheReadResponder* responder = new CacheReadResponder(mFetcher, mID, mFormattedImage); + mCacheReadHandle = mFetcher->mTextureCache->readFromCache(mID, cache_priority, + offset, size, responder); + } + else if(mCanUseHTTP) + { + if (!(mUrl.compare(0, 7, "http://") == 0)) + { + // *TODO:?remove this warning + llwarns << "Unknown URL Type: " << mUrl << llendl; + } + setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); + mState = SEND_HTTP_REQ; + } + else + { + setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); + mState = LOAD_FROM_NETWORK; + } + } + + if (mLoaded) + { + // Make sure request is complete. *TODO: make this auto-complete + if (mFetcher->mTextureCache->readComplete(mCacheReadHandle, false)) + { + mCacheReadHandle = LLTextureCache::nullHandle(); + mState = CACHE_POST; + // fall through + } + else + { + return false; + } + } + else + { + return false; + } + } + + if (mState == CACHE_POST) + { + mCachedSize = mFormattedImage.notNull() ? mFormattedImage->getDataSize() : 0; + // Successfully loaded + if ((mCachedSize >= mDesiredSize) || mHaveAllData) + { + // we have enough data, decode it + llassert_always(mFormattedImage->getDataSize() > 0); + mLoadedDiscard = mDesiredDiscard; + mState = DECODE_IMAGE; + mWriteToCacheState = NOT_WRITE ; + LL_DEBUGS("Texture") << mID << ": Cached. Bytes: " << mFormattedImage->getDataSize() + << " Size: " << llformat("%dx%d",mFormattedImage->getWidth(),mFormattedImage->getHeight()) + << " Desired Discard: " << mDesiredDiscard << " Desired Size: " << mDesiredSize << LL_ENDL; + // fall through + } + else + { + if (mUrl.compare(0, 7, "file://") == 0) + { + // failed to load local file, we're done. + return true; + } + // need more data + else + { + LL_DEBUGS("Texture") << mID << ": Not in Cache" << LL_ENDL; + mState = LOAD_FROM_NETWORK; + } + // fall through + } + } + + if (mState == LOAD_FROM_NETWORK) + { + static LLCachedControl use_http(gSavedSettings,"ImagePipelineUseHTTP"); + +// if (mHost != LLHost::invalid) get_url = false; + if ( use_http && mCanUseHTTP && mUrl.empty())//get http url. + { + LLViewerRegion* region = NULL; + if (mHost == LLHost::invalid) + region = gAgent.getRegion(); + else + region = LLWorld::getInstance()->getRegion(mHost); + + if (region) + { + std::string http_url = region->getHttpUrl() ; + if (!http_url.empty()) + { + mUrl = http_url + "/?texture_id=" + mID.asString().c_str(); + mWriteToCacheState = CAN_WRITE ; //because this texture has a fixed texture id. + } + else + { + mCanUseHTTP = false ; + } + } + else + { + // This will happen if not logged in or if a region deoes not have HTTP Texture enabled + //llwarns << "Region not found for host: " << mHost << llendl; + mCanUseHTTP = false; + } + } + if (mCanUseHTTP && !mUrl.empty()) + { + mState = LLTextureFetchWorker::SEND_HTTP_REQ; + setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); + if(mWriteToCacheState != NOT_WRITE) + { + mWriteToCacheState = CAN_WRITE ; + } + // don't return, fall through to next state + } + else if (mSentRequest == UNSENT && mCanUseNET) + { + // Add this to the network queue and sit here. + // LLTextureFetch::update() will send off a request which will change our state + mWriteToCacheState = CAN_WRITE ; + mRequestedSize = mDesiredSize; + mRequestedDiscard = mDesiredDiscard; + mSentRequest = QUEUED; + mFetcher->addToNetworkQueue(this); + if (! mMetricsStartTime) + { + mMetricsStartTime = LLViewerAssetStatsFF::get_timestamp(); + } + LLViewerAssetStatsFF::record_enqueue_thread1(LLViewerAssetType::AT_TEXTURE, + false, + LLImageBase::TYPE_AVATAR_BAKE == mType); + setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); + + return false; + } + else + { + // Shouldn't need to do anything here + //llassert_always(mFetcher->mNetworkQueue.find(mID) != mFetcher->mNetworkQueue.end()); + // Make certain this is in the network queue + //mFetcher->addToNetworkQueue(this); + //if (! mMetricsStartTime) + //{ + // mMetricsStartTime = LLViewerAssetStatsFF::get_timestamp(); + //} + //LLViewerAssetStatsFF::record_enqueue_thread1(LLViewerAssetType::AT_TEXTURE, false, + // LLImageBase::TYPE_AVATAR_BAKE == mType); + //setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); + return false; + } + } + + if (mState == LOAD_FROM_SIMULATOR) + { + if (mFormattedImage.isNull()) + { + mFormattedImage = new LLImageJ2C; + } + if (processSimulatorPackets()) + { + LL_DEBUGS("Texture") << mID << ": Loaded from Sim. Bytes: " << mFormattedImage->getDataSize() << LL_ENDL; + mFetcher->removeFromNetworkQueue(this, false); + if (mFormattedImage.isNull() || !mFormattedImage->getDataSize()) + { + // processSimulatorPackets() failed +// llwarns << "processSimulatorPackets() failed to load buffer" << llendl; + return true; // failed + } + setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); + mState = DECODE_IMAGE; + mWriteToCacheState = SHOULD_WRITE; + + if (mMetricsStartTime) + { + LLViewerAssetStatsFF::record_response_thread1(LLViewerAssetType::AT_TEXTURE, + false, + LLImageBase::TYPE_AVATAR_BAKE == mType, + LLViewerAssetStatsFF::get_timestamp() - mMetricsStartTime); + mMetricsStartTime = 0; + } + LLViewerAssetStatsFF::record_dequeue_thread1(LLViewerAssetType::AT_TEXTURE, + false, + LLImageBase::TYPE_AVATAR_BAKE == mType); + } + else + { + mFetcher->addToNetworkQueue(this); // failsafe + if (! mMetricsStartTime) + { + mMetricsStartTime = LLViewerAssetStatsFF::get_timestamp(); + } + LLViewerAssetStatsFF::record_enqueue_thread1(LLViewerAssetType::AT_TEXTURE, + false, + LLImageBase::TYPE_AVATAR_BAKE == mType); + setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); + } + return false; + } + + if (mState == SEND_HTTP_REQ) + { + if(mCanUseHTTP) + { + //NOTE: + //control the number of the http requests issued for: + //1, not openning too many file descriptors at the same time; + //2, control the traffic of http so udp gets bandwidth. + // + static const S32 MAX_NUM_OF_HTTP_REQUESTS_IN_QUEUE = 8 ; + if(mFetcher->getNumHTTPRequests() > MAX_NUM_OF_HTTP_REQUESTS_IN_QUEUE) + { + return false ; //wait. + } + + mFetcher->removeFromNetworkQueue(this, false); + + S32 cur_size = 0; + if (mFormattedImage.notNull()) + { + cur_size = mFormattedImage->getDataSize(); // amount of data we already have + if (mFormattedImage->getDiscardLevel() == 0) + { + if(cur_size > 0) + { + // We already have all the data, just decode it + mLoadedDiscard = mFormattedImage->getDiscardLevel(); + mState = DECODE_IMAGE; + return false; + } + else + { + return true ; //abort. + } + } + } + mRequestedSize = mDesiredSize; + mRequestedDiscard = mDesiredDiscard; + mRequestedSize -= cur_size; + S32 offset = cur_size; + mBufferSize = cur_size; // This will get modified by callbackHttpGet() + + bool res = false; + if (!mUrl.empty()) + { + mLoaded = FALSE; + mGetStatus = 0; + mGetReason.clear(); + LL_DEBUGS("Texture") << "HTTP GET: " << mID << " Offset: " << offset + << " Bytes: " << mRequestedSize + << " Bandwidth(kbps): " << mFetcher->getTextureBandwidth() << "/" << mFetcher->mMaxBandwidth + << LL_ENDL; + setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); + mState = WAIT_HTTP_REQ; + + mFetcher->addToHTTPQueue(mID); + if (! mMetricsStartTime) + { + mMetricsStartTime = LLViewerAssetStatsFF::get_timestamp(); + } + LLViewerAssetStatsFF::record_enqueue_thread1(LLViewerAssetType::AT_TEXTURE, + true, + LLImageBase::TYPE_AVATAR_BAKE == mType); + + // Will call callbackHttpGet when curl request completes + std::vector headers; + headers.push_back("Accept: image/x-j2c"); + res = mFetcher->mCurlGetRequest->getByteRange(mUrl, headers, offset, mRequestedSize, + new HTTPGetResponder(mFetcher, mID, LLTimer::getTotalTime(), mRequestedSize, offset, true)); + } + if (!res) + { + llwarns << "HTTP GET request failed for " << mID << llendl; + resetFormattedData(); + ++mHTTPFailCount; + return true; // failed + } + // fall through + } + else //can not use http fetch. + { + return true ; //abort + } + } + + if (mState == WAIT_HTTP_REQ) + { + if (mLoaded) + { + S32 cur_size = mFormattedImage.notNull() ? mFormattedImage->getDataSize() : 0; + if (mRequestedSize < 0) + { + S32 max_attempts; + if (mGetStatus == HTTP_NOT_FOUND) + { + mHTTPFailCount = max_attempts = 1; // Don't retry + llwarns << "Texture missing from server (404): " << mUrl << llendl; + + //roll back to try UDP + if(mCanUseNET) + { + mState = INIT ; + mCanUseHTTP = false ; + setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); + return false ; + } + } + else if (mGetStatus == HTTP_SERVICE_UNAVAILABLE) + { + // *TODO: Should probably introduce a timer here to delay future HTTP requsts + // for a short time (~1s) to ease server load? Ideally the server would queue + // requests instead of returning 503... we already limit the number pending. + ++mHTTPFailCount; + max_attempts = mHTTPFailCount+1; // Keep retrying + LL_INFOS_ONCE("Texture") << "Texture server busy (503): " << mUrl << LL_ENDL; + } + else + { + const S32 HTTP_MAX_RETRY_COUNT = 3; + max_attempts = HTTP_MAX_RETRY_COUNT + 1; + ++mHTTPFailCount; + llinfos << "HTTP GET failed for: " << mUrl + << " Status: " << mGetStatus << " Reason: '" << mGetReason << "'" + << " Attempt:" << mHTTPFailCount+1 << "/" << max_attempts << llendl; + } + + if (mHTTPFailCount >= max_attempts) + { + if (cur_size > 0) + { + // Use available data + mLoadedDiscard = mFormattedImage->getDiscardLevel(); + mState = DECODE_IMAGE; + return false; + } + else + { + resetFormattedData(); + mState = DONE; + return true; // failed + } + } + else + { + mState = SEND_HTTP_REQ; + return false; // retry + } + } + + llassert_always(mBufferSize == cur_size + mRequestedSize); + if(!mBufferSize)//no data received. + { + delete[] mBuffer; + mBuffer = NULL; + + //abort. + mState = DONE; + return true; + } + + if (mFormattedImage.isNull()) + { + // For now, create formatted image based on extension + std::string extension = gDirUtilp->getExtension(mUrl); + mFormattedImage = LLImageFormatted::createFromType(LLImageBase::getCodecFromExtension(extension)); + if (mFormattedImage.isNull()) + { + mFormattedImage = new LLImageJ2C; // default + } + } + + if (mHaveAllData && mRequestedDiscard == 0) //the image file is fully loaded. + { + mFileSize = mBufferSize; + } + else //the file size is unknown. + { + mFileSize = mBufferSize + 1 ; //flag the file is not fully loaded. + } + + U8* buffer = new U8[mBufferSize]; + if (cur_size > 0) + { + memcpy(buffer, mFormattedImage->getData(), cur_size); + } + memcpy(buffer + cur_size, mBuffer, mRequestedSize); // append + // NOTE: setData releases current data and owns new data (buffer) + mFormattedImage->setData(buffer, mBufferSize); + // delete temp data + delete[] mBuffer; // Note: not 'buffer' (assigned in setData()) + mBuffer = NULL; + mBufferSize = 0; + mLoadedDiscard = mRequestedDiscard; + mState = DECODE_IMAGE; + if(mWriteToCacheState != NOT_WRITE) + { + mWriteToCacheState = SHOULD_WRITE ; + } + setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); + return false; + } + else + { + setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); + return false; + } + } + + if (mState == DECODE_IMAGE) + { + static LLCachedControl textures_decode_disabled(gSavedSettings,"TextureDecodeDisabled"); + if(textures_decode_disabled) + { + // for debug use, don't decode + mState = DONE; + setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); + return true; + } + + if (mDesiredDiscard < 0) + { + // We aborted, don't decode + mState = DONE; + setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); + return true; + } + + if (mFormattedImage->getDataSize() <= 0) + { + //llerrs << "Decode entered with invalid mFormattedImage. ID = " << mID << llendl; + + //abort, don't decode + mState = DONE; + setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); + return true; + } + if (mLoadedDiscard < 0) + { + //llerrs << "Decode entered with invalid mLoadedDiscard. ID = " << mID << llendl; + + //abort, don't decode + mState = DONE; + setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); + return true; + } + setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); // Set priority first since Responder may change it + mRawImage = NULL; + mAuxImage = NULL; + llassert_always(mFormattedImage.notNull()); + S32 discard = mHaveAllData ? 0 : mLoadedDiscard; + U32 image_priority = LLWorkerThread::PRIORITY_NORMAL | mWorkPriority; + mDecoded = FALSE; + mState = DECODE_IMAGE_UPDATE; + LL_DEBUGS("Texture") << mID << ": Decoding. Bytes: " << mFormattedImage->getDataSize() << " Discard: " << discard + << " All Data: " << mHaveAllData << LL_ENDL; + mDecodeHandle = mFetcher->mImageDecodeThread->decodeImage(mFormattedImage, image_priority, discard, mNeedsAux, + new DecodeResponder(mFetcher, mID, this)); + // fall though + } + + if (mState == DECODE_IMAGE_UPDATE) + { + if (mDecoded) + { + if (mDecodedDiscard < 0) + { + LL_DEBUGS("Texture") << mID << ": Failed to Decode." << LL_ENDL; + if (mCachedSize > 0 && !mInLocalCache && mRetryAttempt == 0) + { + // Cache file should be deleted, try again +// llwarns << mID << ": Decode of cached file failed (removed), retrying" << llendl; + llassert_always(mDecodeHandle == 0); + mFormattedImage = NULL; + ++mRetryAttempt; + setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); + mState = INIT; + return false; + } + else + { +// llwarns << "UNABLE TO LOAD TEXTURE: " << mID << " RETRIES: " << mRetryAttempt << llendl; + mState = DONE; // failed + } + } + else + { + llassert_always(mRawImage.notNull()); + LL_DEBUGS("Texture") << mID << ": Decoded. Discard: " << mDecodedDiscard + << " Raw Image: " << llformat("%dx%d",mRawImage->getWidth(),mRawImage->getHeight()) << LL_ENDL; + setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); + mState = WRITE_TO_CACHE; + } + // fall through + } + else + { + return false; + } + } + + if (mState == WRITE_TO_CACHE) + { + if (mWriteToCacheState != SHOULD_WRITE || mFormattedImage.isNull()) + { + // If we're in a local cache or we didn't actually receive any new data, + // or we failed to load anything, skip + mState = DONE; + return false; + } + S32 datasize = mFormattedImage->getDataSize(); + if(mFileSize < datasize)//This could happen when http fetching and sim fetching mixed. + { + if(mHaveAllData) + { + mFileSize = datasize ; + } + else + { + mFileSize = datasize + 1 ; //flag not fully loaded. + } + } + llassert_always(datasize); + setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); // Set priority first since Responder may change it + U32 cache_priority = mWorkPriority; + mWritten = FALSE; + mState = WAIT_ON_WRITE; + CacheWriteResponder* responder = new CacheWriteResponder(mFetcher, mID); + mCacheWriteHandle = mFetcher->mTextureCache->writeToCache(mID, cache_priority, + mFormattedImage->getData(), datasize, + mFileSize, responder); + // fall through + } + + if (mState == WAIT_ON_WRITE) + { + if (writeToCacheComplete()) + { + mState = DONE; + // fall through + } + else + { + if (mDesiredDiscard < mDecodedDiscard) + { + // We're waiting for this write to complete before we can receive more data + // (we can't touch mFormattedImage until the write completes) + // Prioritize the write + mFetcher->mTextureCache->prioritizeWrite(mCacheWriteHandle); + } + return false; + } + } + + if (mState == DONE) + { + if (mDecodedDiscard >= 0 && mDesiredDiscard < mDecodedDiscard) + { + // More data was requested, return to INIT + mState = INIT; + setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); + return false; + } + else + { + setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); + return true; + } + } + + return false; +} + +// Called from MAIN thread +void LLTextureFetchWorker::endWork(S32 param, bool aborted) +{ + if (mDecodeHandle != 0) + { + mFetcher->mImageDecodeThread->abortRequest(mDecodeHandle, false); + mDecodeHandle = 0; + } + mFormattedImage = NULL; +} + +////////////////////////////////////////////////////////////////////////////// + +// virtual +void LLTextureFetchWorker::finishWork(S32 param, bool completed) +{ + // The following are required in case the work was aborted + if (mCacheReadHandle != LLTextureCache::nullHandle()) + { + mFetcher->mTextureCache->readComplete(mCacheReadHandle, true); + mCacheReadHandle = LLTextureCache::nullHandle(); + } + if (mCacheWriteHandle != LLTextureCache::nullHandle()) + { + mFetcher->mTextureCache->writeComplete(mCacheWriteHandle, true); + mCacheWriteHandle = LLTextureCache::nullHandle(); + } +} + +// virtual +bool LLTextureFetchWorker::deleteOK() +{ + bool delete_ok = true; + // Allow any pending reads or writes to complete + if (mCacheReadHandle != LLTextureCache::nullHandle()) + { + if (mFetcher->mTextureCache->readComplete(mCacheReadHandle, true)) + { + mCacheReadHandle = LLTextureCache::nullHandle(); + } + else + { + delete_ok = false; + } + } + if (mCacheWriteHandle != LLTextureCache::nullHandle()) + { + if (mFetcher->mTextureCache->writeComplete(mCacheWriteHandle)) + { + mCacheWriteHandle = LLTextureCache::nullHandle(); + } + else + { + delete_ok = false; + } + } + + if ((haveWork() && + // not ok to delete from these states + ((mState >= WRITE_TO_CACHE && mState <= WAIT_ON_WRITE)))) + { + delete_ok = false; + } + + return delete_ok; +} + +void LLTextureFetchWorker::removeFromCache() +{ + if (!mInLocalCache) + { + mFetcher->mTextureCache->removeFromCache(mID); + } +} + + +////////////////////////////////////////////////////////////////////////////// + +bool LLTextureFetchWorker::processSimulatorPackets() +{ + if (mFormattedImage.isNull() || mRequestedSize < 0) + { + // not sure how we got here, but not a valid state, abort! + llassert_always(mDecodeHandle == 0); + mFormattedImage = NULL; + return true; + } + + if (mLastPacket >= mFirstPacket) + { + S32 buffer_size = mFormattedImage->getDataSize(); + for (S32 i = mFirstPacket; i<=mLastPacket; i++) + { + llassert_always(mPackets[i]); + buffer_size += mPackets[i]->mSize; + } + bool have_all_data = mLastPacket >= mTotalPackets-1; + if (mRequestedSize <= 0) + { + // We received a packed but haven't requested anything yet (edge case) + // Return true (we're "done") since we didn't request anything + return true; + } + if (buffer_size >= mRequestedSize || have_all_data) + { + /// We have enough (or all) data + if (have_all_data) + { + mHaveAllData = TRUE; + } + S32 cur_size = mFormattedImage->getDataSize(); + if (buffer_size > cur_size) + { + /// We have new data + U8* buffer = new U8[buffer_size]; + S32 offset = 0; + if (cur_size > 0 && mFirstPacket > 0) + { + memcpy(buffer, mFormattedImage->getData(), cur_size); + offset = cur_size; + } + for (S32 i=mFirstPacket; i<=mLastPacket; i++) + { + memcpy(buffer + offset, mPackets[i]->mData, mPackets[i]->mSize); + offset += mPackets[i]->mSize; + } + // NOTE: setData releases current data + mFormattedImage->setData(buffer, buffer_size); + } + mLoadedDiscard = mRequestedDiscard; + return true; + } + } + return false; +} + +////////////////////////////////////////////////////////////////////////////// + +S32 LLTextureFetchWorker::callbackHttpGet(const LLChannelDescriptors& channels, + const LLIOPipe::buffer_ptr_t& buffer, + bool partial, bool success) +{ + S32 data_size = 0 ; + + LLMutexLock lock(&mWorkMutex); + + if (mState != WAIT_HTTP_REQ) + { + llwarns << "callbackHttpGet for unrequested fetch worker: " << mID + << " req=" << mSentRequest << " state= " << mState << llendl; + return data_size; + } + if (mLoaded) + { + llwarns << "Duplicate callback for " << mID.asString() << llendl; + return data_size ; // ignore duplicate callback + } + if (success) + { + // get length of stream: + data_size = buffer->countAfter(channels.in(), NULL); + + LL_DEBUGS("Texture") << "HTTP RECEIVED: " << mID.asString() << " Bytes: " << data_size << LL_ENDL; + if (data_size > 0) + { + // *TODO: set the formatted image data here directly to avoid the copy + mBuffer = new U8[data_size]; + buffer->readAfter(channels.in(), NULL, mBuffer, data_size); + mBufferSize += data_size; + if (data_size < mRequestedSize && mRequestedDiscard == 0) + { + mHaveAllData = TRUE; + } + else if (data_size > mRequestedSize) + { + // *TODO: This shouldn't be happening any more + llwarns << "data_size = " << data_size << " > requested: " << mRequestedSize << llendl; + mHaveAllData = TRUE; + llassert_always(mDecodeHandle == 0); + mFormattedImage = NULL; // discard any previous data we had + mBufferSize = data_size; + } + } + else + { + // We requested data but received none (and no error), + // so presumably we have all of it + mHaveAllData = TRUE; + } + mRequestedSize = data_size; + } + else + { + mRequestedSize = -1; // error + } + mLoaded = TRUE; + setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); + + return data_size ; +} + +////////////////////////////////////////////////////////////////////////////// + +void LLTextureFetchWorker::callbackCacheRead(bool success, LLImageFormatted* image, + S32 imagesize, BOOL islocal) +{ + LLMutexLock lock(&mWorkMutex); + if (mState != LOAD_FROM_TEXTURE_CACHE) + { +// llwarns << "Read callback for " << mID << " with state = " << mState << llendl; + return; + } + if (success) + { + llassert_always(imagesize >= 0); + mFileSize = imagesize; + mFormattedImage = image; + mImageCodec = image->getCodec(); + mInLocalCache = islocal; + if (mFileSize != 0 && mFormattedImage->getDataSize() >= mFileSize) + { + mHaveAllData = TRUE; + } + } + mLoaded = TRUE; + setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); +} + +void LLTextureFetchWorker::callbackCacheWrite(bool success) +{ + LLMutexLock lock(&mWorkMutex); + if (mState != WAIT_ON_WRITE) + { +// llwarns << "Write callback for " << mID << " with state = " << mState << llendl; + return; + } + mWritten = TRUE; + setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); +} + +////////////////////////////////////////////////////////////////////////////// + +void LLTextureFetchWorker::callbackDecoded(bool success, LLImageRaw* raw, LLImageRaw* aux) +{ + LLMutexLock lock(&mWorkMutex); + if (mDecodeHandle == 0) + { + return; // aborted, ignore + } + if (mState != DECODE_IMAGE_UPDATE) + { +// llwarns << "Decode callback for " << mID << " with state = " << mState << llendl; + mDecodeHandle = 0; + return; + } + llassert_always(mFormattedImage.notNull()); + + mDecodeHandle = 0; + if (success) + { + llassert_always(raw); + mRawImage = raw; + mAuxImage = aux; + mDecodedDiscard = mFormattedImage->getDiscardLevel(); + LL_DEBUGS("Texture") << mID << ": Decode Finished. Discard: " << mDecodedDiscard + << " Raw Image: " << llformat("%dx%d",mRawImage->getWidth(),mRawImage->getHeight()) << LL_ENDL; + } + else + { + llwarns << "DECODE FAILED: " << mID << " Discard: " << (S32)mFormattedImage->getDiscardLevel() << llendl; + removeFromCache(); + mDecodedDiscard = -1; // Redundant, here for clarity and paranoia + } + mDecoded = TRUE; +// llinfos << mID << " : DECODE COMPLETE " << llendl; + setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); +} + +////////////////////////////////////////////////////////////////////////////// + +bool LLTextureFetchWorker::writeToCacheComplete() +{ + // Complete write to cache + if (mCacheWriteHandle != LLTextureCache::nullHandle()) + { + if (!mWritten) + { + return false; + } + if (mFetcher->mTextureCache->writeComplete(mCacheWriteHandle)) + { + mCacheWriteHandle = LLTextureCache::nullHandle(); + } + else + { + return false; + } + } + return true; +} + + +////////////////////////////////////////////////////////////////////////////// +////////////////////////////////////////////////////////////////////////////// +// public + +LLTextureFetch::LLTextureFetch(LLTextureCache* cache, LLImageDecodeThread* imagedecodethread, bool threaded, bool qa_mode) + : LLWorkerThread("TextureFetch", threaded), + mDebugCount(0), + mDebugPause(FALSE), + mPacketCount(0), + mBadPacketCount(0), + mQueueMutex(getAPRPool()), + mNetworkQueueMutex(getAPRPool()), + mTextureCache(cache), + mImageDecodeThread(imagedecodethread), + mTextureBandwidth(0), + mHTTPTextureBits(0), + mTotalHTTPRequests(0), + mCurlGetRequest(NULL), + mQAMode(qa_mode) +{ + mCurlPOSTRequestCount = 0; + mMaxBandwidth = gSavedSettings.getF32("ThrottleBandwidthKBPS"); + mTextureInfo.setUpLogging(gSavedSettings.getBOOL("LogTextureDownloadsToViewerLog"), gSavedSettings.getBOOL("LogTextureDownloadsToSimulator"), gSavedSettings.getU32("TextureLoggingThreshold")); +} + +LLTextureFetch::~LLTextureFetch() +{ + clearDeleteList() ; + + while (! mCommands.empty()) + { + TFRequest * req(mCommands.front()); + mCommands.erase(mCommands.begin()); + delete req; + } + + // ~LLQueuedThread() called here +} + +bool LLTextureFetch::createRequest(const std::string& url, const LLUUID& id, const LLHost& host, F32 priority, + S32 w, S32 h, S32 c, S32 desired_discard, bool needs_aux, bool can_use_http) +{ + if (mDebugPause) + { + return false; + } + + LLTextureFetchWorker* worker = getWorker(id) ; + if (worker) + { + if (worker->mHost != host) + { + llwarns << "LLTextureFetch::createRequest " << id << " called with multiple hosts: " + << host << " != " << worker->mHost << llendl; + removeRequest(worker, true); + worker = NULL; + return false; + } + } + + S32 desired_size; + std::string exten = gDirUtilp->getExtension(url); + if (!url.empty() && (!exten.empty() && LLImageBase::getCodecFromExtension(exten) != IMG_CODEC_J2C)) + { + // Only do partial requests for J2C at the moment + desired_size = MAX_IMAGE_DATA_SIZE; + desired_discard = 0; + } + else if (desired_discard == 0) + { + // if we want the entire image, and we know its size, then get it all + // (calcDataSizeJ2C() below makes assumptions about how the image + // was compressed - this code ensures that when we request the entire image, + // we really do get it.) + desired_size = MAX_IMAGE_DATA_SIZE; + } + else if (w*h*c > 0) + { + // If the requester knows the dimensions of the image, + // this will calculate how much data we need without having to parse the header + + desired_size = LLImageJ2C::calcDataSizeJ2C(w, h, c, desired_discard); + } + else + { + desired_size = TEXTURE_CACHE_ENTRY_SIZE; + desired_discard = MAX_DISCARD_LEVEL; + } + + + if (worker) + { + if (worker->wasAborted()) + { + return false; // need to wait for previous aborted request to complete + } + worker->lockWorkMutex(); + worker->mActiveCount++; + worker->mNeedsAux = needs_aux; + worker->setImagePriority(priority); + worker->setDesiredDiscard(desired_discard, desired_size); + worker->setCanUseHTTP(can_use_http) ; + if (!worker->haveWork()) + { + worker->mState = LLTextureFetchWorker::INIT; + worker->unlockWorkMutex(); + + worker->addWork(0, LLWorkerThread::PRIORITY_HIGH | worker->mWorkPriority); + } + else + { + worker->unlockWorkMutex(); + } + } + else + { + worker = new LLTextureFetchWorker(this, url, id, host, priority, desired_discard, desired_size); + lockQueue() ; + mRequestMap[id] = worker; + unlockQueue() ; + + worker->lockWorkMutex(); + worker->mActiveCount++; + worker->mNeedsAux = needs_aux; + worker->setCanUseHTTP(can_use_http) ; + worker->unlockWorkMutex(); + } + +// llinfos << "REQUESTED: " << id << " Discard: " << desired_discard << llendl; + return true; +} + +// protected +void LLTextureFetch::addToNetworkQueue(LLTextureFetchWorker* worker) +{ + lockQueue() ; + bool in_request_map = (mRequestMap.find(worker->mID) != mRequestMap.end()) ; + unlockQueue() ; + + LLMutexLock lock(&mNetworkQueueMutex); + if (in_request_map) + { + // only add to the queue if in the request map + // i.e. a delete has not been requested + mNetworkQueue.insert(worker->mID); + } + for (cancel_queue_t::iterator iter1 = mCancelQueue.begin(); + iter1 != mCancelQueue.end(); ++iter1) + { + iter1->second.erase(worker->mID); + } +} + +void LLTextureFetch::removeFromNetworkQueue(LLTextureFetchWorker* worker, bool cancel) +{ + LLMutexLock lock(&mNetworkQueueMutex); + size_t erased = mNetworkQueue.erase(worker->mID); + if (cancel && erased > 0) + { + mCancelQueue[worker->mHost].insert(worker->mID); + } +} + +// protected +void LLTextureFetch::addToHTTPQueue(const LLUUID& id) +{ + LLMutexLock lock(&mNetworkQueueMutex); + mHTTPTextureQueue.insert(id); + mTotalHTTPRequests++; +} + +void LLTextureFetch::removeFromHTTPQueue(const LLUUID& id, S32 received_size) +{ + LLMutexLock lock(&mNetworkQueueMutex); + mHTTPTextureQueue.erase(id); + mHTTPTextureBits += received_size * 8; // Approximate - does not include header bits +} + +void LLTextureFetch::deleteRequest(const LLUUID& id, bool cancel) +{ + lockQueue() ; + LLTextureFetchWorker* worker = getWorkerAfterLock(id); + if (worker) + { + size_t erased_1 = mRequestMap.erase(worker->mID); + unlockQueue() ; + + llassert_always(erased_1 > 0) ; + + removeFromNetworkQueue(worker, cancel); + llassert_always(!(worker->getFlags(LLWorkerClass::WCF_DELETE_REQUESTED))) ; + + worker->scheduleDelete(); + } + else + { + unlockQueue() ; + } +} + +void LLTextureFetch::removeRequest(LLTextureFetchWorker* worker, bool cancel) +{ + lockQueue() ; + size_t erased_1 = mRequestMap.erase(worker->mID); + unlockQueue() ; + + llassert_always(erased_1 > 0) ; + removeFromNetworkQueue(worker, cancel); + llassert_always(!(worker->getFlags(LLWorkerClass::WCF_DELETE_REQUESTED))) ; + + worker->scheduleDelete(); +} + +S32 LLTextureFetch::getNumRequests() +{ + lockQueue() ; + S32 size = (S32)mRequestMap.size(); + unlockQueue() ; + + return size ; +} + +S32 LLTextureFetch::getNumHTTPRequests() +{ + mNetworkQueueMutex.lock() ; + S32 size = (S32)mHTTPTextureQueue.size(); + mNetworkQueueMutex.unlock() ; + + return size ; +} + +U32 LLTextureFetch::getTotalNumHTTPRequests() +{ + mNetworkQueueMutex.lock() ; + U32 size = mTotalHTTPRequests ; + mNetworkQueueMutex.unlock() ; + + return size ; +} + +// call lockQueue() first! +LLTextureFetchWorker* LLTextureFetch::getWorkerAfterLock(const LLUUID& id) +{ + LLTextureFetchWorker* res = NULL; + map_t::iterator iter = mRequestMap.find(id); + if (iter != mRequestMap.end()) + { + res = iter->second; + } + return res; +} + +LLTextureFetchWorker* LLTextureFetch::getWorker(const LLUUID& id) +{ + LLMutexLock lock(&mQueueMutex) ; + + return getWorkerAfterLock(id) ; +} + + +bool LLTextureFetch::getRequestFinished(const LLUUID& id, S32& discard_level, + LLPointer& raw, LLPointer& aux) +{ + bool res = false; + LLTextureFetchWorker* worker = getWorker(id); + if (worker) + { + if (worker->wasAborted()) + { + res = true; + } + else if (!worker->haveWork()) + { + // Should only happen if we set mDebugPause... + if (!mDebugPause) + { +// llwarns << "Adding work for inactive worker: " << id << llendl; + worker->addWork(0, LLWorkerThread::PRIORITY_HIGH | worker->mWorkPriority); + } + } + else if (worker->checkWork()) + { + worker->lockWorkMutex(); + discard_level = worker->mDecodedDiscard; + raw = worker->mRawImage; + aux = worker->mAuxImage; + res = true; + LL_DEBUGS("Texture") << id << ": Request Finished. State: " << worker->mState << " Discard: " << discard_level << LL_ENDL; + worker->unlockWorkMutex(); + } + else + { + worker->lockWorkMutex(); + if ((worker->mDecodedDiscard >= 0) && + (worker->mDecodedDiscard < discard_level || discard_level < 0) && + (worker->mState >= LLTextureFetchWorker::WAIT_ON_WRITE)) + { + // Not finished, but data is ready + discard_level = worker->mDecodedDiscard; + raw = worker->mRawImage; + aux = worker->mAuxImage; + } + worker->unlockWorkMutex(); + } + } + else + { + res = true; + } + return res; +} + +bool LLTextureFetch::updateRequestPriority(const LLUUID& id, F32 priority) +{ + bool res = false; + LLTextureFetchWorker* worker = getWorker(id); + if (worker) + { + worker->lockWorkMutex(); + worker->setImagePriority(priority); + worker->unlockWorkMutex(); + res = true; + } + return res; +} + +// Replicates and expands upon the base class's +// getPending() implementation. getPending() and +// runCondition() replicate one another's logic to +// an extent and are sometimes used for the same +// function (deciding whether or not to sleep/pause +// a thread). So the implementations need to stay +// in step, at least until this can be refactored and +// the redundancy eliminated. +// +// May be called from any thread + +//virtual +S32 LLTextureFetch::getPending() +{ + S32 res; + lockData(); + { + LLMutexLock lock(&mQueueMutex); + + res = mRequestQueue.size(); + res += mCurlPOSTRequestCount; + res += mCommands.size(); + } + unlockData(); + return res; +} + +// virtual +bool LLTextureFetch::runCondition() +{ + // Caller is holding the lock on LLThread's condition variable. + + // LLQueuedThread, unlike its base class LLThread, makes this a + // private method which is unfortunate. I want to use it directly + // but I'm going to have to re-implement the logic here (or change + // declarations, which I don't want to do right now). + // + // Changes here may need to be reflected in getPending(). + + bool have_no_commands(false); + { + LLMutexLock lock(&mQueueMutex); + + have_no_commands = mCommands.empty(); + } + + bool have_no_curl_requests(0 == mCurlPOSTRequestCount); + + return ! (have_no_commands + && have_no_curl_requests + && (mRequestQueue.empty() && mIdleThread)); // From base class +} + +////////////////////////////////////////////////////////////////////////////// + +// MAIN THREAD (unthreaded envs), WORKER THREAD (threaded envs) +void LLTextureFetch::commonUpdate() +{ + // Run a cross-thread command, if any. + cmdDoWork(); + + // Update Curl on same thread as mCurlGetRequest was constructed + S32 processed = mCurlGetRequest->process(); + if (processed > 0) + { + lldebugs << "processed: " << processed << " messages." << llendl; + } +} + + +// MAIN THREAD +//virtual +S32 LLTextureFetch::update(U32 max_time_ms) +{ + static LLCachedControl band_width(gSavedSettings,"ThrottleBandwidthKBPS"); + + { + mNetworkQueueMutex.lock() ; + mMaxBandwidth = band_width ; + + gTextureList.sTextureBits += mHTTPTextureBits ; + mHTTPTextureBits = 0 ; + + mNetworkQueueMutex.unlock() ; + } + + S32 res = LLWorkerThread::update(max_time_ms); + + if (!mDebugPause) + { + sendRequestListToSimulators(); + } + + if (!mThreaded) + { + commonUpdate(); + } + + return res; +} + +//called in the MAIN thread after the TextureCacheThread shuts down. +void LLTextureFetch::shutDownTextureCacheThread() +{ + if(mTextureCache) + { + llassert_always(mTextureCache->isQuitting() || mTextureCache->isStopped()) ; + mTextureCache = NULL ; + } +} + +//called in the MAIN thread after the ImageDecodeThread shuts down. +void LLTextureFetch::shutDownImageDecodeThread() +{ + if(mImageDecodeThread) + { + llassert_always(mImageDecodeThread->isQuitting() || mImageDecodeThread->isStopped()) ; + mImageDecodeThread = NULL ; + } +} + +// WORKER THREAD +void LLTextureFetch::startThread() +{ + // Construct mCurlGetRequest from Worker Thread + mCurlGetRequest = new LLCurlRequest(); +} + +// WORKER THREAD +void LLTextureFetch::endThread() +{ + // Destroy mCurlGetRequest from Worker Thread + delete mCurlGetRequest; + mCurlGetRequest = NULL; +} + +// WORKER THREAD +void LLTextureFetch::threadedUpdate() +{ + llassert_always(mCurlGetRequest); + + // Limit update frequency + const F32 PROCESS_TIME = 0.05f; + static LLFrameTimer process_timer; + if (process_timer.getElapsedTimeF32() < PROCESS_TIME) + { + return; + } + process_timer.reset(); + + commonUpdate(); + +#if 0 + const F32 INFO_TIME = 1.0f; + static LLFrameTimer info_timer; + if (info_timer.getElapsedTimeF32() >= INFO_TIME) + { + S32 q = mCurlGetRequest->getQueued(); + if (q > 0) + { + llinfos << "Queued gets: " << q << llendl; + info_timer.reset(); + } + } +#endif + +} + +////////////////////////////////////////////////////////////////////////////// + +void LLTextureFetch::sendRequestListToSimulators() +{ + // All requests + const F32 REQUEST_DELTA_TIME = 0.10f; // 10 fps + + // Sim requests + const S32 IMAGES_PER_REQUEST = 50; + const F32 SIM_LAZY_FLUSH_TIMEOUT = 10.0f; // temp + const F32 MIN_REQUEST_TIME = 1.0f; + const F32 MIN_DELTA_PRIORITY = 1000.f; + + // Periodically, gather the list of textures that need data from the network + // And send the requests out to the simulators + static LLFrameTimer timer; + if (timer.getElapsedTimeF32() < REQUEST_DELTA_TIME) + { + return; + } + timer.reset(); + + // Send requests + typedef std::set request_list_t; + typedef std::map< LLHost, request_list_t > work_request_map_t; + work_request_map_t requests; + { + LLMutexLock lock2(&mNetworkQueueMutex); + for (queue_t::iterator iter = mNetworkQueue.begin(); iter != mNetworkQueue.end(); ) + { + queue_t::iterator curiter = iter++; + LLTextureFetchWorker* req = getWorker(*curiter); + if (!req) + { + mNetworkQueue.erase(curiter); + continue; // paranoia + } + if ((req->mState != LLTextureFetchWorker::LOAD_FROM_NETWORK) && + (req->mState != LLTextureFetchWorker::LOAD_FROM_SIMULATOR)) + { + // We already received our URL, remove from the queue + llwarns << "Worker: " << req->mID << " in mNetworkQueue but in wrong state: " << req->mState << llendl; + mNetworkQueue.erase(curiter); + continue; + } + if (req->mID == mDebugID) + { + mDebugCount++; // for setting breakpoints + } + if (req->mSentRequest == LLTextureFetchWorker::SENT_SIM && + req->mTotalPackets > 0 && + req->mLastPacket >= req->mTotalPackets-1) + { + // We have all the packets... make sure this is high priority +// req->setPriority(LLWorkerThread::PRIORITY_HIGH | req->mWorkPriority); + continue; + } + F32 elapsed = req->mRequestedTimer.getElapsedTimeF32(); + { + F32 delta_priority = llabs(req->mRequestedPriority - req->mImagePriority); + if ((req->mSimRequestedDiscard != req->mDesiredDiscard) || + (delta_priority > MIN_DELTA_PRIORITY && elapsed >= MIN_REQUEST_TIME) || + (elapsed >= SIM_LAZY_FLUSH_TIMEOUT)) + { + requests[req->mHost].insert(req); + } + } + } + } + + for (work_request_map_t::iterator iter1 = requests.begin(); + iter1 != requests.end(); ++iter1) + { + LLHost host = iter1->first; + // invalid host = use agent host + if (host == LLHost::invalid) + { + host = gAgent.getRegionHost(); + } + + S32 sim_request_count = 0; + + for (request_list_t::iterator iter2 = iter1->second.begin(); + iter2 != iter1->second.end(); ++iter2) + { + LLTextureFetchWorker* req = *iter2; + if (gMessageSystem) + { + if (req->mSentRequest != LLTextureFetchWorker::SENT_SIM) + { + // Initialize packet data based on data read from cache + req->lockWorkMutex(); + req->setupPacketData(); + req->unlockWorkMutex(); + } + if (0 == sim_request_count) + { + gMessageSystem->newMessageFast(_PREHASH_RequestImage); + gMessageSystem->nextBlockFast(_PREHASH_AgentData); + gMessageSystem->addUUIDFast(_PREHASH_AgentID, gAgent.getID()); + gMessageSystem->addUUIDFast(_PREHASH_SessionID, gAgent.getSessionID()); + } + S32 packet = req->mLastPacket + 1; + gMessageSystem->nextBlockFast(_PREHASH_RequestImage); + gMessageSystem->addUUIDFast(_PREHASH_Image, req->mID); + gMessageSystem->addS8Fast(_PREHASH_DiscardLevel, (S8)req->mDesiredDiscard); + gMessageSystem->addF32Fast(_PREHASH_DownloadPriority, req->mImagePriority); + gMessageSystem->addU32Fast(_PREHASH_Packet, packet); + gMessageSystem->addU8Fast(_PREHASH_Type, req->mType); +// llinfos << "IMAGE REQUEST: " << req->mID << " Discard: " << req->mDesiredDiscard +// << " Packet: " << packet << " Priority: " << req->mImagePriority << llendl; + + static LLCachedControl log_to_viewer_log(gSavedSettings,"LogTextureDownloadsToViewerLog"); + static LLCachedControl log_to_sim(gSavedSettings,"LogTextureDownloadsToSimulator"); + if (log_to_viewer_log || log_to_sim) + { + mTextureInfo.setRequestStartTime(req->mID, LLTimer::getTotalTime()); + mTextureInfo.setRequestOffset(req->mID, 0); + mTextureInfo.setRequestSize(req->mID, 0); + mTextureInfo.setRequestType(req->mID, LLTextureInfoDetails::REQUEST_TYPE_UDP); + } + + req->lockWorkMutex(); + req->mSentRequest = LLTextureFetchWorker::SENT_SIM; + req->mSimRequestedDiscard = req->mDesiredDiscard; + req->mRequestedPriority = req->mImagePriority; + req->mRequestedTimer.reset(); + req->unlockWorkMutex(); + sim_request_count++; + if (sim_request_count >= IMAGES_PER_REQUEST) + { +// llinfos << "REQUESTING " << sim_request_count << " IMAGES FROM HOST: " << host.getIPString() << llendl; + + gMessageSystem->sendSemiReliable(host, NULL, NULL); + sim_request_count = 0; + } + } + } + if (gMessageSystem && sim_request_count > 0 && sim_request_count < IMAGES_PER_REQUEST) + { +// llinfos << "REQUESTING " << sim_request_count << " IMAGES FROM HOST: " << host.getIPString() << llendl; + gMessageSystem->sendSemiReliable(host, NULL, NULL); + sim_request_count = 0; + } + } + + // Send cancelations + { + LLMutexLock lock2(&mNetworkQueueMutex); + if (gMessageSystem && !mCancelQueue.empty()) + { + for (cancel_queue_t::iterator iter1 = mCancelQueue.begin(); + iter1 != mCancelQueue.end(); ++iter1) + { + LLHost host = iter1->first; + if (host == LLHost::invalid) + { + host = gAgent.getRegionHost(); + } + S32 request_count = 0; + for (queue_t::iterator iter2 = iter1->second.begin(); + iter2 != iter1->second.end(); ++iter2) + { + if (0 == request_count) + { + gMessageSystem->newMessageFast(_PREHASH_RequestImage); + gMessageSystem->nextBlockFast(_PREHASH_AgentData); + gMessageSystem->addUUIDFast(_PREHASH_AgentID, gAgent.getID()); + gMessageSystem->addUUIDFast(_PREHASH_SessionID, gAgent.getSessionID()); + } + gMessageSystem->nextBlockFast(_PREHASH_RequestImage); + gMessageSystem->addUUIDFast(_PREHASH_Image, *iter2); + gMessageSystem->addS8Fast(_PREHASH_DiscardLevel, -1); + gMessageSystem->addF32Fast(_PREHASH_DownloadPriority, 0); + gMessageSystem->addU32Fast(_PREHASH_Packet, 0); + gMessageSystem->addU8Fast(_PREHASH_Type, 0); +// llinfos << "CANCELING IMAGE REQUEST: " << (*iter2) << llendl; + + request_count++; + if (request_count >= IMAGES_PER_REQUEST) + { + gMessageSystem->sendSemiReliable(host, NULL, NULL); + request_count = 0; + } + } + if (request_count > 0 && request_count < IMAGES_PER_REQUEST) + { + gMessageSystem->sendSemiReliable(host, NULL, NULL); + } + } + mCancelQueue.clear(); + } + } +} + +////////////////////////////////////////////////////////////////////////////// + +bool LLTextureFetchWorker::insertPacket(S32 index, U8* data, S32 size) +{ + mRequestedTimer.reset(); + if (index >= mTotalPackets) + { +// llwarns << "Received Image Packet " << index << " > max: " << mTotalPackets << " for image: " << mID << llendl; + return false; + } + if (index > 0 && index < mTotalPackets-1 && size != MAX_IMG_PACKET_SIZE) + { +// llwarns << "Received bad sized packet: " << index << ", " << size << " != " << MAX_IMG_PACKET_SIZE << " for image: " << mID << llendl; + return false; + } + + if (index >= (S32)mPackets.size()) + { + mPackets.resize(index+1, (PacketData*)NULL); // initializes v to NULL pointers + } + else if (mPackets[index] != NULL) + { +// llwarns << "Received duplicate packet: " << index << " for image: " << mID << llendl; + return false; + } + + mPackets[index] = new PacketData(data, size); + while (mLastPacket+1 < (S32)mPackets.size() && mPackets[mLastPacket+1] != NULL) + { + ++mLastPacket; + } + return true; +} + +bool LLTextureFetch::receiveImageHeader(const LLHost& host, const LLUUID& id, U8 codec, U16 packets, U32 totalbytes, + U16 data_size, U8* data) +{ + LLTextureFetchWorker* worker = getWorker(id); + bool res = true; + + ++mPacketCount; + + if (!worker) + { +// llwarns << "Received header for non active worker: " << id << llendl; + res = false; + } + else if (worker->mState != LLTextureFetchWorker::LOAD_FROM_NETWORK || + worker->mSentRequest != LLTextureFetchWorker::SENT_SIM) + { +// llwarns << "receiveImageHeader for worker: " << id +// << " in state: " << LLTextureFetchWorker::sStateDescs[worker->mState] +// << " sent: " << worker->mSentRequest << llendl; + res = false; + } + else if (worker->mLastPacket != -1) + { + // check to see if we've gotten this packet before +// llwarns << "Received duplicate header for: " << id << llendl; + res = false; + } + else if (!data_size) + { +// llwarns << "Img: " << id << ":" << " Empty Image Header" << llendl; + res = false; + } + if (!res) + { + ++mBadPacketCount; + mNetworkQueueMutex.lock() ; + mCancelQueue[host].insert(id); + mNetworkQueueMutex.unlock() ; + return false; + } + + worker->lockWorkMutex(); + + // Copy header data into image object + worker->mImageCodec = codec; + worker->mTotalPackets = packets; + worker->mFileSize = (S32)totalbytes; + llassert_always(totalbytes > 0); + llassert_always(data_size == FIRST_PACKET_SIZE || data_size == worker->mFileSize); + res = worker->insertPacket(0, data, data_size); + worker->setPriority(LLWorkerThread::PRIORITY_HIGH | worker->mWorkPriority); + worker->mState = LLTextureFetchWorker::LOAD_FROM_SIMULATOR; + worker->unlockWorkMutex(); + return res; +} + +bool LLTextureFetch::receiveImagePacket(const LLHost& host, const LLUUID& id, U16 packet_num, U16 data_size, U8* data) +{ + LLTextureFetchWorker* worker = getWorker(id); + bool res = true; + + ++mPacketCount; + + if (!worker) + { +// llwarns << "Received packet " << packet_num << " for non active worker: " << id << llendl; + res = false; + } + else if (worker->mLastPacket == -1) + { +// llwarns << "Received packet " << packet_num << " before header for: " << id << llendl; + res = false; + } + else if (!data_size) + { +// llwarns << "Img: " << id << ":" << " Empty Image Header" << llendl; + res = false; + } + if (!res) + { + ++mBadPacketCount; + mNetworkQueueMutex.lock() ; + mCancelQueue[host].insert(id); + mNetworkQueueMutex.unlock() ; + return false; + } + + worker->lockWorkMutex(); + + res = worker->insertPacket(packet_num, data, data_size); + + if ((worker->mState == LLTextureFetchWorker::LOAD_FROM_SIMULATOR) || + (worker->mState == LLTextureFetchWorker::LOAD_FROM_NETWORK)) + { + worker->setPriority(LLWorkerThread::PRIORITY_HIGH | worker->mWorkPriority); + worker->mState = LLTextureFetchWorker::LOAD_FROM_SIMULATOR; + } + else + { +// llwarns << "receiveImagePacket " << packet_num << "/" << worker->mLastPacket << " for worker: " << id +// << " in state: " << LLTextureFetchWorker::sStateDescs[worker->mState] << llendl; + removeFromNetworkQueue(worker, true); // failsafe + } + + if(packet_num >= (worker->mTotalPackets - 1)) + { + static LLCachedControl log_to_viewer_log(gSavedSettings,"LogTextureDownloadsToViewerLog"); + static LLCachedControl log_to_sim(gSavedSettings,"LogTextureDownloadsToSimulator"); + + if (log_to_viewer_log || log_to_sim) + { + U64 timeNow = LLTimer::getTotalTime(); + mTextureInfo.setRequestSize(id, worker->mFileSize); + mTextureInfo.setRequestCompleteTimeAndLog(id, timeNow); + } + } + worker->unlockWorkMutex(); + + return res; +} + +////////////////////////////////////////////////////////////////////////////// +BOOL LLTextureFetch::isFromLocalCache(const LLUUID& id) +{ + BOOL from_cache = FALSE ; + + LLTextureFetchWorker* worker = getWorker(id); + if (worker) + { + worker->lockWorkMutex() ; + from_cache = worker->mInLocalCache ; + worker->unlockWorkMutex() ; + } + + return from_cache ; +} + +S32 LLTextureFetch::getFetchState(const LLUUID& id, F32& data_progress_p, F32& requested_priority_p, + U32& fetch_priority_p, F32& fetch_dtime_p, F32& request_dtime_p, bool& can_use_http) +{ + S32 state = LLTextureFetchWorker::INVALID; + F32 data_progress = 0.0f; + F32 requested_priority = 0.0f; + F32 fetch_dtime = 999999.f; + F32 request_dtime = 999999.f; + U32 fetch_priority = 0; + + LLTextureFetchWorker* worker = getWorker(id); + if (worker && worker->haveWork()) + { + worker->lockWorkMutex(); + state = worker->mState; + fetch_dtime = worker->mFetchTimer.getElapsedTimeF32(); + request_dtime = worker->mRequestedTimer.getElapsedTimeF32(); + if (worker->mFileSize > 0) + { + if (state == LLTextureFetchWorker::LOAD_FROM_SIMULATOR) + { + S32 data_size = FIRST_PACKET_SIZE + (worker->mLastPacket-1) * MAX_IMG_PACKET_SIZE; + data_size = llmax(data_size, 0); + data_progress = (F32)data_size / (F32)worker->mFileSize; + } + else if (worker->mFormattedImage.notNull()) + { + data_progress = (F32)worker->mFormattedImage->getDataSize() / (F32)worker->mFileSize; + } + } + if (state >= LLTextureFetchWorker::LOAD_FROM_NETWORK && state <= LLTextureFetchWorker::WAIT_HTTP_REQ) + { + requested_priority = worker->mRequestedPriority; + } + else + { + requested_priority = worker->mImagePriority; + } + fetch_priority = worker->getPriority(); + can_use_http = worker->getCanUseHTTP() ; + worker->unlockWorkMutex(); + } + data_progress_p = data_progress; + requested_priority_p = requested_priority; + fetch_priority_p = fetch_priority; + fetch_dtime_p = fetch_dtime; + request_dtime_p = request_dtime; + return state; +} + +void LLTextureFetch::dump() +{ + llinfos << "LLTextureFetch REQUESTS:" << llendl; + for (request_queue_t::iterator iter = mRequestQueue.begin(); + iter != mRequestQueue.end(); ++iter) + { + LLQueuedThread::QueuedRequest* qreq = *iter; + LLWorkerThread::WorkRequest* wreq = (LLWorkerThread::WorkRequest*)qreq; + LLTextureFetchWorker* worker = (LLTextureFetchWorker*)wreq->getWorkerClass(); + llinfos << " ID: " << worker->mID + << " PRI: " << llformat("0x%08x",wreq->getPriority()) + << " STATE: " << worker->sStateDescs[worker->mState] + << llendl; + } +} + +////////////////////////////////////////////////////////////////////////////// + +// cross-thread command methods + +void LLTextureFetch::commandSetRegion(U64 region_handle) +{ + TFReqSetRegion * req = new TFReqSetRegion(region_handle); + + cmdEnqueue(req); +} + +void LLTextureFetch::commandSendMetrics(const std::string & caps_url, + const LLUUID & session_id, + const LLUUID & agent_id, + LLViewerAssetStats * main_stats) +{ + TFReqSendMetrics * req = new TFReqSendMetrics(caps_url, session_id, agent_id, main_stats); + + cmdEnqueue(req); +} + +void LLTextureFetch::commandDataBreak() +{ + // The pedantically correct way to implement this is to create a command + // request object in the above fashion and enqueue it. However, this is + // simple data of an advisorial not operational nature and this case + // of shared-write access is tolerable. + + LLTextureFetch::svMetricsDataBreak = true; +} + +void LLTextureFetch::cmdEnqueue(TFRequest * req) +{ + lockQueue(); + mCommands.push_back(req); + unlockQueue(); + + unpause(); +} + +LLTextureFetch::TFRequest * LLTextureFetch::cmdDequeue() +{ + TFRequest * ret = 0; + + lockQueue(); + if (! mCommands.empty()) + { + ret = mCommands.front(); + mCommands.erase(mCommands.begin()); + } + unlockQueue(); + + return ret; +} + +void LLTextureFetch::cmdDoWork() +{ + if (mDebugPause) + { + return; // debug: don't do any work + } + + TFRequest * req = cmdDequeue(); + if (req) + { + // One request per pass should really be enough for this. + req->doWork(this); + delete req; + } +} + + +////////////////////////////////////////////////////////////////////////////// + +// Private (anonymous) class methods implementing the command scheme. + +namespace +{ + +/** + * Implements the 'Set Region' command. + * + * Thread: Thread1 (TextureFetch) + */ +bool +TFReqSetRegion::doWork(LLTextureFetch *) +{ + LLViewerAssetStatsFF::set_region_thread1(mRegionHandle); + + return true; +} + + +TFReqSendMetrics::~TFReqSendMetrics() +{ + delete mMainStats; + mMainStats = 0; +} + + +/** + * Implements the 'Send Metrics' command. Takes over + * ownership of the passed LLViewerAssetStats pointer. + * + * Thread: Thread1 (TextureFetch) + */ +bool +TFReqSendMetrics::doWork(LLTextureFetch * fetcher) +{ + /* + * HTTP POST responder. Doesn't do much but tries to + * detect simple breaks in recording the metrics stream. + * + * The 'volatile' modifiers don't indicate signals, + * mmap'd memory or threads, really. They indicate that + * the referenced data is part of a pseudo-closure for + * this responder rather than being required for correct + * operation. + * + * We don't try very hard with the POST request. We give + * it one shot and that's more-or-less it. With a proper + * refactoring of the LLQueuedThread usage, these POSTs + * could be put in a request object and made more reliable. + */ + class lcl_responder : public LLCurl::Responder + { + public: + lcl_responder(LLTextureFetch * fetcher, + S32 expected_sequence, + volatile const S32 & live_sequence, + volatile bool & reporting_break, + volatile bool & reporting_started) + : LLCurl::Responder(), + mFetcher(fetcher), + mExpectedSequence(expected_sequence), + mLiveSequence(live_sequence), + mReportingBreak(reporting_break), + mReportingStarted(reporting_started) + { + mFetcher->incrCurlPOSTCount(); + } + + ~lcl_responder() + { + mFetcher->decrCurlPOSTCount(); + } + + // virtual + void error(U32 status_num, const std::string & reason) + { + if (mLiveSequence == mExpectedSequence) + { + mReportingBreak = true; + } + LL_WARNS("Texture") << "Break in metrics stream due to POST failure to metrics collection service. Reason: " + << reason << LL_ENDL; + } + + // virtual + void result(const LLSD & content) + { + if (mLiveSequence == mExpectedSequence) + { + mReportingBreak = false; + mReportingStarted = true; + } + } + + private: + LLTextureFetch * mFetcher; + S32 mExpectedSequence; + volatile const S32 & mLiveSequence; + volatile bool & mReportingBreak; + volatile bool & mReportingStarted; + + }; // class lcl_responder + + if (! gViewerAssetStatsThread1) + return true; + + static volatile bool reporting_started(false); + static volatile S32 report_sequence(0); + + // We've taken over ownership of the stats copy at this + // point. Get a working reference to it for merging here + // but leave it in 'this'. Destructor will rid us of it. + LLViewerAssetStats & main_stats = *mMainStats; + + // Merge existing stats into those from main, convert to LLSD + main_stats.merge(*gViewerAssetStatsThread1); + LLSD merged_llsd = main_stats.asLLSD(true); + + // Add some additional meta fields to the content + merged_llsd["session_id"] = mSessionID; + merged_llsd["agent_id"] = mAgentID; + merged_llsd["message"] = "ViewerAssetMetrics"; // Identifies the type of metrics + merged_llsd["sequence"] = report_sequence; // Sequence number + merged_llsd["initial"] = ! reporting_started; // Initial data from viewer + merged_llsd["break"] = LLTextureFetch::svMetricsDataBreak; // Break in data prior to this report + + // Update sequence number + if (S32_MAX == ++report_sequence) + report_sequence = 0; + + // Limit the size of the stats report if necessary. + merged_llsd["truncated"] = truncate_viewer_metrics(10, merged_llsd); + + if (! mCapsURL.empty()) + { + LLCurlRequest::headers_t headers; + fetcher->getCurlRequest().post(mCapsURL, + headers, + merged_llsd, + new lcl_responder(fetcher, + report_sequence, + report_sequence, + LLTextureFetch::svMetricsDataBreak, + reporting_started)); + } + else + { + LLTextureFetch::svMetricsDataBreak = true; + } + + // In QA mode, Metrics submode, log the result for ease of testing + if (fetcher->isQAMode()) + { + LL_INFOS("Textures") << merged_llsd << LL_ENDL; + } + + gViewerAssetStatsThread1->reset(); + + return true; +} + + +bool +truncate_viewer_metrics(int max_regions, LLSD & metrics) +{ + static const LLSD::String reg_tag("regions"); + static const LLSD::String duration_tag("duration"); + + LLSD & reg_map(metrics[reg_tag]); + if (reg_map.size() <= max_regions) + { + return false; + } + + // Build map of region hashes ordered by duration + typedef std::multimap reg_ordered_list_t; + reg_ordered_list_t regions_by_duration; + + int ind(0); + LLSD::array_const_iterator it_end(reg_map.endArray()); + for (LLSD::array_const_iterator it(reg_map.beginArray()); it_end != it; ++it, ++ind) + { + LLSD::Real duration = (*it)[duration_tag].asReal(); + regions_by_duration.insert(reg_ordered_list_t::value_type(duration, ind)); + } + + // Build a replacement regions array with the longest-persistence regions + LLSD new_region(LLSD::emptyArray()); + reg_ordered_list_t::const_reverse_iterator it2_end(regions_by_duration.rend()); + reg_ordered_list_t::const_reverse_iterator it2(regions_by_duration.rbegin()); + for (int i(0); i < max_regions && it2_end != it2; ++i, ++it2) + { + new_region.append(reg_map[it2->second]); + } + reg_map = new_region; + + return true; +} + +} // end of anonymous namespace + + + -- cgit v1.2.3