diff options
| -rw-r--r-- | indra/llcommon/llassettype.cpp | 14 | ||||
| -rw-r--r-- | indra/llcommon/llassettype.h | 1 | ||||
| -rw-r--r-- | indra/llmessage/llassetstorage.cpp | 314 | ||||
| -rw-r--r-- | indra/llmessage/llassetstorage.h | 72 | ||||
| -rw-r--r-- | indra/llmessage/llhttpassetstorage.cpp | 426 | ||||
| -rw-r--r-- | indra/llmessage/llhttpassetstorage.h | 32 |
6 files changed, 717 insertions, 142 deletions
diff --git a/indra/llcommon/llassettype.cpp b/indra/llcommon/llassettype.cpp index 628ebb7aec..b5378943ca 100644 --- a/indra/llcommon/llassettype.cpp +++ b/indra/llcommon/llassettype.cpp @@ -184,6 +184,20 @@ const char* LLAssetType::lookupHumanReadable(LLAssetType::EType type) } } +// static +LLAssetType::EType LLAssetType::lookupHumanReadable( const char* name ) +{ + for( S32 i = 0; i < AT_COUNT; i++ ) + { + if( 0 == strcmp(name, mAssetTypeHumanNames[i]) ) + { + // match + return (EType)i; + } + } + return AT_NONE; +} + EDragAndDropType LLAssetType::lookupDragAndDropType( EType asset ) { switch( asset ) diff --git a/indra/llcommon/llassettype.h b/indra/llcommon/llassettype.h index abc118fe9c..aedee06483 100644 --- a/indra/llcommon/llassettype.h +++ b/indra/llcommon/llassettype.h @@ -126,6 +126,7 @@ public: static const char* lookup(EType type); // translation from a type to a human readable form. + static EType lookupHumanReadable( const char* name ); static const char* lookupHumanReadable(EType type); static EDragAndDropType lookupDragAndDropType( EType ); diff --git a/indra/llmessage/llassetstorage.cpp b/indra/llmessage/llassetstorage.cpp index 4ec5ec34ec..d1bb575c36 100644 --- a/indra/llmessage/llassetstorage.cpp +++ b/indra/llmessage/llassetstorage.cpp @@ -155,10 +155,40 @@ LLAssetRequest::LLAssetRequest(const LLUUID &uuid, const LLAssetType::EType type mTime = LLMessageSystem::getMessageTimeSeconds(TRUE); } +// virtual LLAssetRequest::~LLAssetRequest() { } +// virtual +LLSD LLAssetRequest::getTerseDetails() const +{ + LLSD sd; + sd["asset_id"] = getUUID(); + sd["type_long"] = LLAssetType::lookupHumanReadable(getType()); + sd["type"] = LLAssetType::lookup(getType()); + sd["time"] = mTime; + time_t timestamp = (time_t) mTime; + std::ostringstream time_string; + time_string << ctime(×tamp); + sd["time_string"] = time_string.str(); + return sd; +} + +// virtual +LLSD LLAssetRequest::getFullDetails() const +{ + LLSD sd = getTerseDetails(); + sd["host"] = mHost.getIPandPort(); + sd["requesting_agent"] = mRequestingAgentID; + sd["is_temp"] = mIsTemp; + sd["is_local"] = mIsLocal; + sd["is_priority"] = mIsPriority; + sd["data_send_in_first_packet"] = mDataSentInFirstPacket; + sd["data_is_in_vfs"] = mDataIsInVFS; + + return sd; +} ///---------------------------------------------------------------------------- /// LLInvItemRequest @@ -279,47 +309,41 @@ void LLAssetStorage::checkForTimeouts() void LLAssetStorage::_cleanupRequests(BOOL all, S32 error) { - const S32 NUM_QUEUES = 3; F64 mt_secs = LLMessageSystem::getMessageTimeSeconds(); - std::list<LLAssetRequest*>* requests[NUM_QUEUES]; - requests[0] = &mPendingDownloads; - requests[1] = &mPendingUploads; - requests[2] = &mPendingLocalUploads; - static const char* REQUEST_TYPE[NUM_QUEUES] = { "download", "upload", "localuploads"}; - - std::list<LLAssetRequest*> timed_out; - - for (S32 ii = 0; ii < NUM_QUEUES; ++ii) + request_list_t timed_out; + S32 rt; + for (rt = 0; rt < RT_COUNT; rt++) { - for (std::list<LLAssetRequest*>::iterator iter = requests[ii]->begin(); - iter != requests[ii]->end(); ) + request_list_t* requests = getRequestList((ERequestType)rt); + for (request_list_t::iterator iter = requests->begin(); + iter != requests->end(); ) { - std::list<LLAssetRequest*>::iterator curiter = iter++; + request_list_t::iterator curiter = iter++; LLAssetRequest* tmp = *curiter; // if all is true, we want to clean up everything // otherwise just check for timed out requests // EXCEPT for upload timeouts if (all - || ((0 == ii) + || ((RT_DOWNLOAD == rt) && LL_ASSET_STORAGE_TIMEOUT < (mt_secs - tmp->mTime))) { - llwarns << "Asset " << REQUEST_TYPE[ii] << " request " + llwarns << "Asset " << getRequestName((ERequestType)rt) << " request " << (all ? "aborted" : "timed out") << " for " << tmp->getUUID() << "." << LLAssetType::lookup(tmp->getType()) << llendl; timed_out.push_front(tmp); - iter = requests[ii]->erase(curiter); + iter = requests->erase(curiter); } } } LLAssetInfo info; - for (std::list<LLAssetRequest*>::iterator iter = timed_out.begin(); + for (request_list_t::iterator iter = timed_out.begin(); iter != timed_out.end(); ) { - std::list<LLAssetRequest*>::iterator curiter = iter++; + request_list_t::iterator curiter = iter++; LLAssetRequest* tmp = *curiter; if (tmp->mUpCallback) { @@ -382,7 +406,7 @@ void LLAssetStorage::getAssetData(const LLUUID uuid, LLAssetType::EType type, vo BOOL duplicate = FALSE; // check to see if there's a pending download of this uuid already - for (std::list<LLAssetRequest*>::iterator iter = mPendingDownloads.begin(); + for (request_list_t::iterator iter = mPendingDownloads.begin(); iter != mPendingDownloads.end(); ++iter ) { LLAssetRequest *tmp = *iter; @@ -504,11 +528,11 @@ void LLAssetStorage::downloadCompleteCallback( // find and callback ALL pending requests for this UUID // SJB: We process the callbacks in reverse order, I do not know if this is important, // but I didn't want to mess with it. - std::list<LLAssetRequest*> requests; - for (std::list<LLAssetRequest*>::iterator iter = gAssetStorage->mPendingDownloads.begin(); + request_list_t requests; + for (request_list_t::iterator iter = gAssetStorage->mPendingDownloads.begin(); iter != gAssetStorage->mPendingDownloads.end(); ) { - std::list<LLAssetRequest*>::iterator curiter = iter++; + request_list_t::iterator curiter = iter++; LLAssetRequest* tmp = *curiter; if ((tmp->getUUID() == req->getUUID()) && (tmp->getType()== req->getType())) { @@ -516,10 +540,10 @@ void LLAssetStorage::downloadCompleteCallback( iter = gAssetStorage->mPendingDownloads.erase(curiter); } } - for (std::list<LLAssetRequest*>::iterator iter = requests.begin(); + for (request_list_t::iterator iter = requests.begin(); iter != requests.end(); ) { - std::list<LLAssetRequest*>::iterator curiter = iter++; + request_list_t::iterator curiter = iter++; LLAssetRequest* tmp = *curiter; if (tmp->mDownCallback) { @@ -877,11 +901,11 @@ void LLAssetStorage::_callUploadCallbacks(const LLUUID &uuid, LLAssetType::EType { // SJB: We process the callbacks in reverse order, I do not know if this is important, // but I didn't want to mess with it. - std::list<LLAssetRequest*> requests; - for (std::list<LLAssetRequest*>::iterator iter = mPendingUploads.begin(); + request_list_t requests; + for (request_list_t::iterator iter = mPendingUploads.begin(); iter != mPendingUploads.end(); ) { - std::list<LLAssetRequest*>::iterator curiter = iter++; + request_list_t::iterator curiter = iter++; LLAssetRequest* req = *curiter; if ((req->getUUID() == uuid) && (req->getType() == asset_type)) { @@ -889,10 +913,10 @@ void LLAssetStorage::_callUploadCallbacks(const LLUUID &uuid, LLAssetType::EType iter = mPendingUploads.erase(curiter); } } - for (std::list<LLAssetRequest*>::iterator iter = mPendingLocalUploads.begin(); + for (request_list_t::iterator iter = mPendingLocalUploads.begin(); iter != mPendingLocalUploads.end(); ) { - std::list<LLAssetRequest*>::iterator curiter = iter++; + request_list_t::iterator curiter = iter++; LLAssetRequest* req = *curiter; if ((req->getUUID() == uuid) && (req->getType() == asset_type)) { @@ -900,10 +924,10 @@ void LLAssetStorage::_callUploadCallbacks(const LLUUID &uuid, LLAssetType::EType iter = mPendingLocalUploads.erase(curiter); } } - for (std::list<LLAssetRequest*>::iterator iter = requests.begin(); + for (request_list_t::iterator iter = requests.begin(); iter != requests.end(); ) { - std::list<LLAssetRequest*>::iterator curiter = iter++; + request_list_t::iterator curiter = iter++; LLAssetRequest* req = *curiter; if (req->mUpCallback) { @@ -913,45 +937,239 @@ void LLAssetStorage::_callUploadCallbacks(const LLUUID &uuid, LLAssetType::EType } } +LLAssetStorage::request_list_t* LLAssetStorage::getRequestList(LLAssetStorage::ERequestType rt) +{ + switch (rt) + { + case RT_DOWNLOAD: + return &mPendingDownloads; + case RT_UPLOAD: + return &mPendingUploads; + case RT_LOCALUPLOAD: + return &mPendingLocalUploads; + default: + llwarns << "Unable to find request list for request type '" << rt << "'" << llendl; + return NULL; + } +} + +const LLAssetStorage::request_list_t* LLAssetStorage::getRequestList(LLAssetStorage::ERequestType rt) const +{ + switch (rt) + { + case RT_DOWNLOAD: + return &mPendingDownloads; + case RT_UPLOAD: + return &mPendingUploads; + case RT_LOCALUPLOAD: + return &mPendingLocalUploads; + default: + llwarns << "Unable to find request list for request type '" << rt << "'" << llendl; + return NULL; + } +} + +// static +std::string LLAssetStorage::getRequestName(LLAssetStorage::ERequestType rt) +{ + switch (rt) + { + case RT_DOWNLOAD: + return "download"; + case RT_UPLOAD: + return "upload"; + case RT_LOCALUPLOAD: + return "localupload"; + default: + llwarns << "Unable to find request name for request type '" << rt << "'" << llendl; + return ""; + } +} + +S32 LLAssetStorage::getNumPending(LLAssetStorage::ERequestType rt) const +{ + const request_list_t* requests = getRequestList(rt); + S32 num_pending = -1; + if (requests) + { + num_pending = requests->size(); + } + return num_pending; +} S32 LLAssetStorage::getNumPendingDownloads() const { - return mPendingDownloads.size(); + return getNumPending(RT_DOWNLOAD); } S32 LLAssetStorage::getNumPendingUploads() const { - return mPendingUploads.size(); + return getNumPending(RT_UPLOAD); } S32 LLAssetStorage::getNumPendingLocalUploads() { - return mPendingLocalUploads.size(); + return getNumPending(RT_LOCALUPLOAD); +} + +// virtual +LLSD LLAssetStorage::getPendingDetails(LLAssetStorage::ERequestType rt, + LLAssetType::EType asset_type, + const std::string& detail_prefix) const +{ + const request_list_t* requests = getRequestList(rt); + LLSD sd; + sd["requests"] = getPendingDetails(requests, asset_type, detail_prefix); + return sd; +} + +// virtual +LLSD LLAssetStorage::getPendingDetails(const LLAssetStorage::request_list_t* requests, + LLAssetType::EType asset_type, + const std::string& detail_prefix) const +{ + LLSD details; + if (requests) + { + request_list_t::const_iterator it = requests->begin(); + request_list_t::const_iterator end = requests->end(); + for ( ; it != end; ++it) + { + LLAssetRequest* req = *it; + if ( (LLAssetType::AT_NONE == asset_type) + || (req->getType() == asset_type) ) + { + LLSD row = req->getTerseDetails(); + + std::ostringstream detail; + detail << detail_prefix << "/" << LLAssetType::lookup(req->getType()) + << "/" << req->getUUID(); + row["detail"] = LLURI(detail.str()); + + details.append(row); + } + } + } + return details; +} + + +// static +const LLAssetRequest* LLAssetStorage::findRequest(const LLAssetStorage::request_list_t* requests, + LLAssetType::EType asset_type, + const LLUUID& asset_id) +{ + if (requests) + { + // Search the requests list for the asset. + request_list_t::const_iterator iter = requests->begin(); + request_list_t::const_iterator end = requests->end(); + for (; iter != end; ++iter) + { + const LLAssetRequest* req = *iter; + if (asset_type == req->getType() && + asset_id == req->getUUID() ) + { + return req; + } + } + } + return NULL; } -LLSD LLAssetStorage::getPendingTypes(const std::list<LLAssetRequest*>& requests) const +// static +LLAssetRequest* LLAssetStorage::findRequest(LLAssetStorage::request_list_t* requests, + LLAssetType::EType asset_type, + const LLUUID& asset_id) { - LLSD type_counts; - std::list<LLAssetRequest*>::const_iterator it = requests.begin(); - std::list<LLAssetRequest*>::const_iterator end = requests.end(); - for ( ; it != end; ++it) + if (requests) { - LLAssetRequest* req = *it; + // Search the requests list for the asset. + request_list_t::iterator iter = requests->begin(); + request_list_t::iterator end = requests->end(); + for (; iter != end; ++iter) + { + LLAssetRequest* req = *iter; + if (asset_type == req->getType() && + asset_id == req->getUUID() ) + { + return req; + } + } + } + return NULL; +} + - const char* type_name = LLAssetType::lookupHumanReadable(req->getType()); - type_counts[type_name] = type_counts[type_name].asInteger() + 1; +// virtual +LLSD LLAssetStorage::getPendingRequest(LLAssetStorage::ERequestType rt, + LLAssetType::EType asset_type, + const LLUUID& asset_id) const +{ + const request_list_t* requests = getRequestList(rt); + return getPendingRequest(requests, asset_type, asset_id); +} + +// virtual +LLSD LLAssetStorage::getPendingRequest(const LLAssetStorage::request_list_t* requests, + LLAssetType::EType asset_type, + const LLUUID& asset_id) const +{ + LLSD sd; + const LLAssetRequest* req = findRequest(requests, asset_type, asset_id); + if (req) + { + sd = req->getFullDetails(); } - return type_counts; + return sd; } -LLSD LLAssetStorage::getPendingDownloadTypes() const +// virtual +bool LLAssetStorage::deletePendingRequest(LLAssetStorage::ERequestType rt, + LLAssetType::EType asset_type, + const LLUUID& asset_id) { - return getPendingTypes(mPendingDownloads); + request_list_t* requests = getRequestList(rt); + if (deletePendingRequest(requests, asset_type, asset_id)) + { + llinfos << "Asset " << getRequestName(rt) << " request for " + << asset_id << "." << LLAssetType::lookup(asset_type) + << " removed from pending queue." << llendl; + return true; + } + return false; } -LLSD LLAssetStorage::getPendingUploadTypes() const +// virtual +bool LLAssetStorage::deletePendingRequest(LLAssetStorage::request_list_t* requests, + LLAssetType::EType asset_type, + const LLUUID& asset_id) { - return getPendingTypes(mPendingUploads); + LLAssetRequest* req = findRequest(requests, asset_type, asset_id); + if (req) + { + // Remove the request from this list. + requests->remove(req); + S32 error = LL_ERR_TCP_TIMEOUT; + // Run callbacks. + if (req->mUpCallback) + { + req->mUpCallback(req->getUUID(), req->mUserData, error); + } + if (req->mDownCallback) + { + req->mDownCallback(mVFS, req->getUUID(), req->getType(), req->mUserData, error); + } + if (req->mInfoCallback) + { + LLAssetInfo info; + req->mInfoCallback(&info, req->mUserData, error); + } + delete req; + return true; + } + + return false; } // static @@ -996,7 +1214,7 @@ const char* LLAssetStorage::getErrorString(S32 status) void LLAssetStorage::getAssetData(const LLUUID uuid, LLAssetType::EType type, void (*callback)(const char*, const LLUUID&, void *, S32), void *user_data, BOOL is_priority) { // check for duplicates here, since we're about to fool the normal duplicate checker - for (std::list<LLAssetRequest*>::iterator iter = mPendingDownloads.begin(); + for (request_list_t::iterator iter = mPendingDownloads.begin(); iter != mPendingDownloads.end(); ) { LLAssetRequest* tmp = *iter++; diff --git a/indra/llmessage/llassetstorage.h b/indra/llmessage/llassetstorage.h index 59baddd69c..9d73bf71eb 100644 --- a/indra/llmessage/llassetstorage.h +++ b/indra/llmessage/llassetstorage.h @@ -90,6 +90,19 @@ public: BOOL mDataSentInFirstPacket; BOOL mDataIsInVFS; LLUUID mRequestingAgentID; // Only valid for uploads from an agent + + virtual LLSD getTerseDetails() const; + virtual LLSD getFullDetails() const; +}; + +template <class T> +struct ll_asset_request_equal : public std::equal_to<T> +{ + bool operator()(const T& x, const T& y) const + { + return ( x->getType() == y->getType() + && x->getUUID() == y->getUUID() ); + } }; @@ -165,6 +178,15 @@ public: LLVFS *mVFS; typedef void (*LLStoreAssetCallback)(const LLUUID &asset_id, void *user_data, S32 status); + enum ERequestType + { + RT_INVALID = -1, + RT_DOWNLOAD = 0, + RT_UPLOAD = 1, + RT_LOCALUPLOAD = 2, + RT_COUNT = 3 + }; + protected: BOOL mShutDown; LLHost mUpstreamHost; @@ -172,9 +194,11 @@ protected: LLMessageSystem *mMessageSys; LLXferManager *mXferManager; - std::list<LLAssetRequest*> mPendingDownloads; - std::list<LLAssetRequest*> mPendingUploads; - std::list<LLAssetRequest*> mPendingLocalUploads; + + typedef std::list<LLAssetRequest*> request_list_t; + request_list_t mPendingDownloads; + request_list_t mPendingUploads; + request_list_t mPendingLocalUploads; public: LLAssetStorage(LLMessageSystem *msg, LLXferManager *xfer, @@ -239,14 +263,48 @@ public: const LLUUID &asset_id, LLAssetType::EType atype, LLGetAssetCallback cb, void *user_data, BOOL is_priority = FALSE); // Get a particular inventory item. +protected: + virtual LLSD getPendingDetails(const request_list_t* requests, + LLAssetType::EType asset_type, + const std::string& detail_prefix) const; + + virtual LLSD getPendingRequest(const request_list_t* requests, + LLAssetType::EType asset_type, + const LLUUID& asset_id) const; + + virtual bool deletePendingRequest(request_list_t* requests, + LLAssetType::EType asset_type, + const LLUUID& asset_id); + +public: + static const LLAssetRequest* findRequest(const request_list_t* requests, + LLAssetType::EType asset_type, + const LLUUID& asset_id); + static LLAssetRequest* findRequest(request_list_t* requests, + LLAssetType::EType asset_type, + const LLUUID& asset_id); + + request_list_t* getRequestList(ERequestType rt); + const request_list_t* getRequestList(ERequestType rt) const; + static std::string getRequestName(ERequestType rt); S32 getNumPendingDownloads() const; S32 getNumPendingUploads() const; S32 getNumPendingLocalUploads(); + S32 getNumPending(ERequestType rt) const; + + virtual LLSD getPendingDetails(ERequestType rt, + LLAssetType::EType asset_type, + const std::string& detail_prefix) const; + + virtual LLSD getPendingRequest(ERequestType rt, + LLAssetType::EType asset_type, + const LLUUID& asset_id) const; + + virtual bool deletePendingRequest(ERequestType rt, + LLAssetType::EType asset_type, + const LLUUID& asset_id); - // Returns a map from type to num pending, eg 'texture' => 5, 'object' => 10 - LLSD getPendingDownloadTypes() const; - LLSD getPendingUploadTypes() const; // download process callbacks static void downloadCompleteCallback( @@ -330,8 +388,6 @@ private: LLXferManager *xfer, LLVFS *vfs, const LLHost &upstream_host); - LLSD getPendingTypes(const std::list<LLAssetRequest*>& requests) const; - }; //////////////////////////////////////////////////////////////////////// diff --git a/indra/llmessage/llhttpassetstorage.cpp b/indra/llmessage/llhttpassetstorage.cpp index d429dd5b63..e786cf64b3 100644 --- a/indra/llmessage/llhttpassetstorage.cpp +++ b/indra/llmessage/llhttpassetstorage.cpp @@ -11,12 +11,15 @@ #include "llhttpassetstorage.h" +#include <sys/stat.h> + #include "indra_constants.h" #include "llvfile.h" #include "llvfs.h" #include "zlib/zlib.h" +const U32 MAX_RUNNING_REQUESTS = 4; const F32 MAX_PROCESSING_TIME = 0.005f; const S32 CURL_XFER_BUFFER_SIZE = 65536; // Try for 30 minutes for now. @@ -49,7 +52,9 @@ struct LLTempAssetData class LLHTTPAssetRequest : public LLAssetRequest { public: - LLHTTPAssetRequest(LLHTTPAssetStorage *asp, const LLUUID &uuid, LLAssetType::EType type, const char *url, CURLM *curl_multi); + LLHTTPAssetRequest(LLHTTPAssetStorage *asp, const LLUUID &uuid, + LLAssetType::EType type, LLAssetStorage::ERequestType rt, + const char *url, CURLM *curl_multi); virtual ~LLHTTPAssetRequest(); void setupCurlHandle(); @@ -61,6 +66,9 @@ public: static size_t curlCompressedUploadCallback( void *data, size_t size, size_t nmemb, void *user_data); + virtual LLSD getTerseDetails() const; + virtual LLSD getFullDetails() const; + public: LLHTTPAssetStorage *mAssetStoragep; @@ -70,9 +78,7 @@ public: struct curl_slist *mHTTPHeaders; LLVFile *mVFile; LLUUID mTmpUUID; - BOOL mIsUpload; - BOOL mIsLocalUpload; - BOOL mIsDownload; + LLAssetStorage::ERequestType mRequestType; bool mZInitialized; z_stream mZStream; @@ -83,7 +89,12 @@ public: }; -LLHTTPAssetRequest::LLHTTPAssetRequest(LLHTTPAssetStorage *asp, const LLUUID &uuid, LLAssetType::EType type, const char *url, CURLM *curl_multi) +LLHTTPAssetRequest::LLHTTPAssetRequest(LLHTTPAssetStorage *asp, + const LLUUID &uuid, + LLAssetType::EType type, + LLAssetStorage::ERequestType rt, + const char *url, + CURLM *curl_multi) : LLAssetRequest(uuid, type), mZInitialized(false) { @@ -91,10 +102,11 @@ LLHTTPAssetRequest::LLHTTPAssetRequest(LLHTTPAssetStorage *asp, const LLUUID &uu mCurlHandle = NULL; mCurlMultiHandle = curl_multi; mVFile = NULL; - mIsUpload = FALSE; - mIsLocalUpload = FALSE; - mIsDownload = FALSE; + mRequestType = rt; mHTTPHeaders = NULL; + mFP = NULL; + mZInputBuffer = NULL; + mZInputExhausted = false; mURLBuffer = new char[strlen(url) + 1]; /*Flawfinder: ignore*/ if (mURLBuffer) @@ -113,22 +125,7 @@ LLHTTPAssetRequest::~LLHTTPAssetRequest() if (mAssetStoragep) { // Terminating a request. Thus upload or download is no longer pending. - if (mIsUpload) - { - mAssetStoragep->clearPendingUpload(); - } - else if (mIsLocalUpload) - { - mAssetStoragep->clearPendingLocalUpload(); - } - else if (mIsDownload) - { - mAssetStoragep->clearPendingDownload(); - } - else - { - llerrs << "LLHTTPAssetRequest::~LLHTTPAssetRequest - Destroyed request is not upload OR download, this is bad!" << llendl; - } + mAssetStoragep->removeRunningRequest(mRequestType, this); } else { @@ -144,6 +141,82 @@ LLHTTPAssetRequest::~LLHTTPAssetRequest() finishCompressedUpload(); } +// virtual +LLSD LLHTTPAssetRequest::getTerseDetails() const +{ + LLSD sd = LLAssetRequest::getTerseDetails(); + + sd["url"] = mURLBuffer; + + return sd; +} + +// virtual +LLSD LLHTTPAssetRequest::getFullDetails() const +{ + LLSD sd = LLAssetRequest::getFullDetails(); + + if (mCurlHandle) + { + long curl_response = -1; + long curl_connect = -1; + double curl_total_time = -1.0f; + double curl_size_upload = -1.0f; + double curl_size_download = -1.0f; + long curl_content_length_upload = -1; + long curl_content_length_download = -1; + long curl_request_size = -1; + const char* curl_content_type = NULL; + + curl_easy_getinfo(mCurlHandle, CURLINFO_HTTP_CODE, &curl_response); + curl_easy_getinfo(mCurlHandle, CURLINFO_HTTP_CONNECTCODE, &curl_connect); + curl_easy_getinfo(mCurlHandle, CURLINFO_TOTAL_TIME, &curl_total_time); + curl_easy_getinfo(mCurlHandle, CURLINFO_SIZE_UPLOAD, &curl_size_upload); + curl_easy_getinfo(mCurlHandle, CURLINFO_SIZE_DOWNLOAD, &curl_size_download); + curl_easy_getinfo(mCurlHandle, CURLINFO_CONTENT_LENGTH_UPLOAD, &curl_content_length_upload); + curl_easy_getinfo(mCurlHandle, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &curl_content_length_download); + curl_easy_getinfo(mCurlHandle, CURLINFO_REQUEST_SIZE, &curl_request_size); + curl_easy_getinfo(mCurlHandle, CURLINFO_CONTENT_TYPE, &curl_content_type); + + sd["curl_response_code"] = (int) curl_response; + sd["curl_http_connect_code"] = (int) curl_connect; + sd["curl_total_time"] = curl_total_time; + sd["curl_size_upload"] = curl_size_upload; + sd["curl_size_download"] = curl_size_download; + sd["curl_content_length_upload"] = (int) curl_content_length_upload; + sd["curl_content_length_download"] = (int) curl_content_length_download; + sd["curl_request_size"] = (int) curl_request_size; + if (curl_content_type) + { + sd["curl_content_type"] = curl_content_type; + } + else + { + sd["curl_content_type"] = ""; + } + } + + sd["temp_id"] = mTmpUUID; + sd["request_type"] = LLAssetStorage::getRequestName(mRequestType); + sd["z_initialized"] = mZInitialized; + sd["z_input_exhausted"] = mZInputExhausted; + + S32 file_size = -1; + if (mFP) + { + struct stat file_stat; + int file_desc = fileno(mFP); + if ( fstat(file_desc, &file_stat) == 0) + { + file_size = file_stat.st_size; + } + } + sd["file_size"] = file_size; + + return sd; +} + + void LLHTTPAssetRequest::setupCurlHandle() { mCurlHandle = curl_easy_init(); @@ -151,7 +224,7 @@ void LLHTTPAssetRequest::setupCurlHandle() curl_easy_setopt(mCurlHandle, CURLOPT_NOPROGRESS, 1); curl_easy_setopt(mCurlHandle, CURLOPT_URL, mURLBuffer); curl_easy_setopt(mCurlHandle, CURLOPT_PRIVATE, this); - if (mIsDownload) + if (LLAssetStorage::RT_DOWNLOAD == mRequestType) { curl_easy_setopt(mCurlHandle, CURLOPT_ENCODING, ""); // only do this on downloads, as uploads @@ -174,22 +247,7 @@ void LLHTTPAssetRequest::setupCurlHandle() if (mAssetStoragep) { // Set the appropriate pending upload or download flag - if (mIsUpload) - { - mAssetStoragep->setPendingUpload(); - } - else if (mIsLocalUpload) - { - mAssetStoragep->setPendingLocalUpload(); - } - else if (mIsDownload) - { - mAssetStoragep->setPendingDownload(); - } - else - { - llerrs << "LLHTTPAssetRequest::setupCurlHandle - Request is not upload OR download, this is bad!" << llendl; - } + mAssetStoragep->addRunningRequest(mRequestType, this); } else { @@ -323,10 +381,6 @@ void LLHTTPAssetStorage::_init(const char *web_host, const char *local_web_host, curl_global_init(CURL_GLOBAL_ALL); mCurlMultiHandle = curl_multi_init(); - - mPendingDownload = FALSE; - mPendingUpload = FALSE; - mPendingLocalUpload = FALSE; } LLHTTPAssetStorage::~LLHTTPAssetStorage() @@ -438,6 +492,113 @@ void LLHTTPAssetStorage::storeAssetData( } } +// virtual +LLSD LLHTTPAssetStorage::getPendingDetails(LLAssetStorage::ERequestType rt, + LLAssetType::EType asset_type, + const std::string& detail_prefix) const +{ + LLSD sd = LLAssetStorage::getPendingDetails(rt, asset_type, detail_prefix); + const request_list_t* running = getRunningList(rt); + if (running) + { + // Loop through the pending requests sd, and add extra info about its running status. + S32 num_pending = sd["requests"].size(); + S32 i; + for (i = 0; i < num_pending; ++i) + { + LLSD& pending = sd["requests"][i]; + // See if this pending request is running. + const LLAssetRequest* req = findRequest(running, + LLAssetType::lookup(pending["type"].asString().c_str()), + pending["asset_id"]); + if (req) + { + // Keep the detail_url so we don't have to rebuild it. + LLURI detail_url = pending["detail"]; + pending = req->getTerseDetails(); + pending["detail"] = detail_url; + pending["is_running"] = true; + } + else + { + pending["is_running"] = false; + } + } + } + return sd; +} + +// virtual +LLSD LLHTTPAssetStorage::getPendingRequest(LLAssetStorage::ERequestType rt, + LLAssetType::EType asset_type, + const LLUUID& asset_id) const +{ + // Look for this asset in the running list first. + const request_list_t* running = getRunningList(rt); + if (running) + { + LLSD sd = LLAssetStorage::getPendingRequest(running, asset_type, asset_id); + if (sd) + { + sd["is_running"] = true; + return sd; + } + } + LLSD sd = LLAssetStorage::getPendingRequest(rt, asset_type, asset_id); + if (sd) + { + sd["is_running"] = false; + } + return sd; +} + +// virtual +bool LLHTTPAssetStorage::deletePendingRequest(LLAssetStorage::ERequestType rt, + LLAssetType::EType asset_type, + const LLUUID& asset_id) +{ + // Try removing this from the running list first. + request_list_t* running = getRunningList(rt); + if (running) + { + LLAssetRequest* req = findRequest(running, asset_type, asset_id); + if (req) + { + // Remove this request from the running list to get it out of curl. + running->remove(req); + + // Find this request in the pending list, so we can move it to the end of the line. + request_list_t* pending = getRequestList(rt); + if (pending) + { + request_list_t::iterator result = std::find_if(pending->begin(), pending->end(), + std::bind2nd(ll_asset_request_equal<LLAssetRequest*>(), req)); + if (pending->end() != result) + { + // This request was found in the pending list. Move it to the end! + LLAssetRequest* pending_req = *result; + pending->remove(pending_req); + pending->push_back(pending_req); + + llinfos << "Asset " << getRequestName(rt) << " request for " + << asset_id << "." << LLAssetType::lookup(asset_type) + << " removed from curl and placed at the end of the pending queue." + << llendl; + } + else + { + llwarns << "Unable to find pending " << getRequestName(rt) << " request for " + << asset_id << "." << LLAssetType::lookup(asset_type) << llendl; + } + } + delete req; + + return true; + } + } + return LLAssetStorage::deletePendingRequest(rt, asset_type, asset_id); +} + // internal requester, used by getAssetData in superclass void LLHTTPAssetStorage::_queueDataRequest(const LLUUID& uuid, LLAssetType::EType type, void (*callback)(LLVFS *vfs, const LLUUID&, LLAssetType::EType, void *, S32), @@ -469,13 +630,41 @@ void LLHTTPAssetStorage::_queueDataRequest(const LLUUID& uuid, LLAssetType::ETyp } } +LLAssetRequest* LLHTTPAssetStorage::findNextRequest(LLAssetStorage::request_list_t& pending, + LLAssetStorage::request_list_t& running) +{ + // Early exit if the running list is full, or we don't have more pending than running. + if (running.size() >= MAX_RUNNING_REQUESTS + || pending.size() <= running.size()) return NULL; + + // Look for the first pending request that is not already running. + request_list_t::iterator running_begin = running.begin(); + request_list_t::iterator running_end = running.end(); + + request_list_t::iterator pending_iter = pending.begin(); + request_list_t::iterator pending_end = pending.end(); + // Loop over all pending requests until we miss finding it in the running list. + for (; pending_iter != pending.end(); ++pending_iter) + { + LLAssetRequest* req = *pending_iter; + // Look for this pending request in the running list. + if (running_end == std::find_if(running_begin, running_end, + std::bind2nd(ll_asset_request_equal<LLAssetRequest*>(), req))) + { + // It isn't running! Return it. + return req; + } + } + return NULL; +} + // overloaded to additionally move data to/from the webserver void LLHTTPAssetStorage::checkForTimeouts() { - LLAssetRequest *req = NULL; - if (mPendingDownloads.size() > 0 && !mPendingDownload) - { - req = mPendingDownloads.front(); + CURLMcode mcode; + LLAssetRequest *req; + while (req = findNextRequest(mPendingDownloads, mRunningDownloads)) + { // Setup this curl download request // We need to generate a new request here // since the one in the list could go away @@ -485,9 +674,9 @@ void LLHTTPAssetStorage::checkForTimeouts() std::string base_url = getBaseURL(req->getUUID(), req->getType()); snprintf(tmp_url, sizeof(tmp_url), "%s/%36s.%s", base_url.c_str() , uuid_str, LLAssetType::lookup(req->getType())); /*Flawfinder: ignore*/ - LLHTTPAssetRequest *new_req = new LLHTTPAssetRequest(this, req->getUUID(), req->getType(), tmp_url, mCurlMultiHandle); + LLHTTPAssetRequest *new_req = new LLHTTPAssetRequest(this, req->getUUID(), + req->getType(), RT_DOWNLOAD, tmp_url, mCurlMultiHandle); new_req->mTmpUUID.generate(); - new_req->mIsDownload = TRUE; // Sets pending download flag internally new_req->setupCurlHandle(); @@ -495,15 +684,22 @@ void LLHTTPAssetStorage::checkForTimeouts() curl_easy_setopt(new_req->mCurlHandle, CURLOPT_WRITEFUNCTION, &curlDownCallback); curl_easy_setopt(new_req->mCurlHandle, CURLOPT_WRITEDATA, new_req->mCurlHandle); - curl_multi_add_handle(mCurlMultiHandle, new_req->mCurlHandle); - llinfos << "Requesting " << new_req->mURLBuffer << llendl; - + mcode = curl_multi_add_handle(mCurlMultiHandle, new_req->mCurlHandle); + if (mcode > CURLM_OK) + { + // Failure. Deleting the pending request will remove it from the running + // queue, and push it to the end of the pending queue. + deletePendingRequest(RT_DOWNLOAD, req->getType(), req->getUUID()); + break; + } + else + { + llinfos << "Requesting " << new_req->mURLBuffer << llendl; + } } - - if (mPendingUploads.size() > 0 && !mPendingUpload) + while (req = findNextRequest(mPendingUploads, mRunningUploads)) { - req = mPendingUploads.front(); // setup this curl upload request bool do_compress = req->getType() == LLAssetType::AT_OBJECT; @@ -515,8 +711,8 @@ void LLHTTPAssetStorage::checkForTimeouts() do_compress ? "%s/%s.%s.gz" : "%s/%s.%s", mBaseURL.c_str(), uuid_str, LLAssetType::lookup(req->getType())); - LLHTTPAssetRequest *new_req = new LLHTTPAssetRequest(this, req->getUUID(), req->getType(), tmp_url, mCurlMultiHandle); - new_req->mIsUpload = TRUE; + LLHTTPAssetRequest *new_req = new LLHTTPAssetRequest(this, req->getUUID(), + req->getType(), RT_UPLOAD, tmp_url, mCurlMultiHandle); if (do_compress) { new_req->prepareCompressedUpload(); @@ -541,15 +737,23 @@ void LLHTTPAssetStorage::checkForTimeouts() } curl_easy_setopt(new_req->mCurlHandle, CURLOPT_READDATA, new_req->mCurlHandle); - curl_multi_add_handle(mCurlMultiHandle, new_req->mCurlHandle); - llinfos << "Requesting PUT " << new_req->mURLBuffer << llendl; + mcode = curl_multi_add_handle(mCurlMultiHandle, new_req->mCurlHandle); + if (mcode > CURLM_OK) + { + // Failure. Deleting the pending request will remove it from the running + // queue, and push it to the end of the pending queue. + deletePendingRequest(RT_UPLOAD, req->getType(), req->getUUID()); + break; + } + else + { + llinfos << "Requesting PUT " << new_req->mURLBuffer << llendl; + } // Pending upload will have been flagged by the request } - - if (mPendingLocalUploads.size() > 0 && !mPendingLocalUpload) + while (req = findNextRequest(mPendingLocalUploads, mRunningLocalUploads)) { - req = mPendingLocalUploads.front(); // setup this curl upload request LLVFile file(mVFS, req->getUUID(), req->getType()); @@ -560,8 +764,8 @@ void LLHTTPAssetStorage::checkForTimeouts() // KLW - All temporary uploads are saved locally "http://localhost:12041/asset" snprintf(tmp_url, sizeof(tmp_url), "%s/%36s.%s", mLocalBaseURL.c_str(), uuid_str, LLAssetType::lookup(req->getType())); /*Flawfinder: ignore*/ - LLHTTPAssetRequest *new_req = new LLHTTPAssetRequest(this, req->getUUID(), req->getType(), tmp_url, mCurlMultiHandle); - new_req->mIsLocalUpload = TRUE; + LLHTTPAssetRequest *new_req = new LLHTTPAssetRequest(this, req->getUUID(), + req->getType(), RT_LOCALUPLOAD, tmp_url, mCurlMultiHandle); new_req->mRequestingAgentID = req->mRequestingAgentID; // Sets pending upload flag internally @@ -572,13 +776,22 @@ void LLHTTPAssetStorage::checkForTimeouts() curl_easy_setopt(new_req->mCurlHandle, CURLOPT_READFUNCTION, &curlUpCallback); curl_easy_setopt(new_req->mCurlHandle, CURLOPT_READDATA, new_req->mCurlHandle); - curl_multi_add_handle(mCurlMultiHandle, new_req->mCurlHandle); - llinfos << "TAT: LLHTTPAssetStorage::checkForTimeouts() : pending local!" - << " Requesting PUT " << new_req->mURLBuffer << llendl; + mcode = curl_multi_add_handle(mCurlMultiHandle, new_req->mCurlHandle); + if (mcode > CURLM_OK) + { + // Failure. Deleting the pending request will remove it from the running + // queue, and push it to the end of the pending queue. + deletePendingRequest(RT_LOCALUPLOAD, req->getType(), req->getUUID()); + break; + } + else + { + llinfos << "TAT: LLHTTPAssetStorage::checkForTimeouts() : pending local!" + << " Requesting PUT " << new_req->mURLBuffer << llendl; + } // Pending upload will have been flagged by the request } S32 count = 0; - CURLMcode mcode; int queue_length; do { @@ -599,12 +812,15 @@ void LLHTTPAssetStorage::checkForTimeouts() curl_easy_getinfo(curl_msg->easy_handle, CURLINFO_PRIVATE, &req); curl_easy_getinfo(curl_msg->easy_handle, CURLINFO_HTTP_CODE, &curl_result); - if (req->mIsUpload || req->mIsLocalUpload) + if (RT_UPLOAD == req->mRequestType || RT_LOCALUPLOAD == req->mRequestType) { - if (curl_msg->data.result == CURLE_OK && (curl_result == HTTP_OK || curl_result == HTTP_PUT_OK || curl_result == HTTP_NO_CONTENT)) + if (curl_msg->data.result == CURLE_OK && + ( curl_result == HTTP_OK + || curl_result == HTTP_PUT_OK + || curl_result == HTTP_NO_CONTENT)) { llinfos << "Success uploading " << req->getUUID() << " to " << req->mURLBuffer << llendl; - if (req->mIsLocalUpload) + if (RT_LOCALUPLOAD == req->mRequestType) { addTempAssetData(req->getUUID(), req->mRequestingAgentID, mHostName); } @@ -636,7 +852,7 @@ void LLHTTPAssetStorage::checkForTimeouts() // Pending upload flag will get cleared when the request is deleted } } - else if (req->mIsDownload) + else if (RT_DOWNLOAD == req->mRequestType) { if (curl_result == HTTP_OK && curl_msg->data.result == CURLE_OK) { @@ -648,7 +864,7 @@ void LLHTTPAssetStorage::checkForTimeouts() } else { - // TODO: if this actually indicates a bad asset on the server + // *TODO: if this actually indicates a bad asset on the server // (not certain at this point), then delete it llwarns << "Found " << req->mURLBuffer << " to be zero size" << llendl; xfer_result = LL_ERR_ASSET_REQUEST_NOT_IN_DATABASE; @@ -774,9 +990,8 @@ S32 LLHTTPAssetStorage::getURLToFile(const LLUUID& uuid, LLAssetType::EType asse } // make sure we use the normal curl setup, even though we don't really need a request object - LLHTTPAssetRequest req(this, uuid, asset_type, url.c_str(), mCurlMultiHandle); + LLHTTPAssetRequest req(this, uuid, asset_type, RT_DOWNLOAD, url.c_str(), mCurlMultiHandle); req.mFP = fp; - req.mIsDownload = TRUE; req.setupCurlHandle(); curl_easy_setopt(req.mCurlHandle, CURLOPT_FOLLOWLOCATION, TRUE); @@ -868,6 +1083,63 @@ size_t LLHTTPAssetStorage::curlFileDownCallback(void *data, size_t size, size_t return fwrite(data, size, nmemb, req->mFP); } +LLAssetStorage::request_list_t* LLHTTPAssetStorage::getRunningList(LLAssetStorage::ERequestType rt) +{ + switch (rt) + { + case RT_DOWNLOAD: + return &mRunningDownloads; + case RT_UPLOAD: + return &mRunningUploads; + case RT_LOCALUPLOAD: + return &mRunningLocalUploads; + default: + return NULL; + } +} + +const LLAssetStorage::request_list_t* LLHTTPAssetStorage::getRunningList(LLAssetStorage::ERequestType rt) const +{ + switch (rt) + { + case RT_DOWNLOAD: + return &mRunningDownloads; + case RT_UPLOAD: + return &mRunningUploads; + case RT_LOCALUPLOAD: + return &mRunningLocalUploads; + default: + return NULL; + } +} + + +void LLHTTPAssetStorage::addRunningRequest(ERequestType rt, LLHTTPAssetRequest* request) +{ + request_list_t* requests = getRunningList(rt); + if (requests) + { + requests->push_back(request); + } + else + { + llerrs << "LLHTTPAssetStorage::addRunningRequest - Request is not an upload OR download, this is bad!" << llendl; + } +} + +void LLHTTPAssetStorage::removeRunningRequest(ERequestType rt, LLHTTPAssetRequest* request) +{ + request_list_t* requests = getRunningList(rt); + if (requests) + { + requests->remove(request); + } + else + { + llerrs << "LLHTTPAssetStorage::removeRunningRequest - Destroyed request is not an upload OR download, this is bad!" << llendl; + } +} + // virtual void LLHTTPAssetStorage::addTempAssetData(const LLUUID& asset_id, const LLUUID& agent_id, const std::string& host_name) { diff --git a/indra/llmessage/llhttpassetstorage.h b/indra/llmessage/llhttpassetstorage.h index a6ba5c795c..b1be7431b9 100644 --- a/indra/llmessage/llhttpassetstorage.h +++ b/indra/llmessage/llhttpassetstorage.h @@ -13,6 +13,7 @@ #include "curl/curl.h" class LLVFile; +class LLHTTPAssetRequest; typedef void (*progress_callback)(void* userdata); struct LLTempAssetData; @@ -56,11 +57,25 @@ public: bool temp_file, bool is_priority); + virtual LLSD getPendingDetails(ERequestType rt, + LLAssetType::EType asset_type, + const std::string& detail_prefix) const; + + virtual LLSD getPendingRequest(ERequestType rt, + LLAssetType::EType asset_type, + const LLUUID& asset_id) const; + + virtual bool deletePendingRequest(ERequestType rt, + LLAssetType::EType asset_type, + const LLUUID& asset_id); + // Hack. One off curl download an URL to a file. Probably should be elsewhere. // Only used by lldynamicstate. The API is broken, and should be replaced with // a generic HTTP file fetch - Doug 9/25/06 S32 getURLToFile(const LLUUID& uuid, LLAssetType::EType asset_type, const LLString &url, const char *filename, progress_callback callback, void *userdata); + LLAssetRequest* findNextRequest(request_list_t& pending, request_list_t& running); + void checkForTimeouts(); static size_t curlDownCallback(void *data, size_t size, size_t nmemb, void *user_data); @@ -69,12 +84,11 @@ public: static size_t nullOutputCallback(void *data, size_t size, size_t nmemb, void *user_data); // Should only be used by the LLHTTPAssetRequest - void setPendingUpload() { mPendingUpload = TRUE; } - void setPendingLocalUpload() { mPendingLocalUpload = TRUE; } - void setPendingDownload() { mPendingDownload = TRUE; } - void clearPendingUpload() { mPendingUpload = FALSE; } - void clearPendingLocalUpload() { mPendingLocalUpload = FALSE; } - void clearPendingDownload() { mPendingDownload = FALSE; } + void addRunningRequest(ERequestType rt, LLHTTPAssetRequest* request); + void removeRunningRequest(ERequestType rt, LLHTTPAssetRequest* request); + + request_list_t* getRunningList(ERequestType rt); + const request_list_t* getRunningList(ERequestType rt) const; // Temp assets are stored on sim nodes, they have agent ID and location data associated with them. virtual void addTempAssetData(const LLUUID& asset_id, const LLUUID& agent_id, const std::string& host_name); @@ -106,9 +120,9 @@ protected: CURLM *mCurlMultiHandle; - BOOL mPendingDownload; - BOOL mPendingUpload; - BOOL mPendingLocalUpload; + request_list_t mRunningDownloads; + request_list_t mRunningUploads; + request_list_t mRunningLocalUploads; uuid_tempdata_map mTempAssets; }; |