diff options
Diffstat (limited to 'indra/newview/llinventorymodelbackgroundfetch.cpp')
| -rwxr-xr-x[-rw-r--r--] | indra/newview/llinventorymodelbackgroundfetch.cpp | 1007 |
1 files changed, 560 insertions, 447 deletions
diff --git a/indra/newview/llinventorymodelbackgroundfetch.cpp b/indra/newview/llinventorymodelbackgroundfetch.cpp index f4d0110b0f..40edb13a80 100644..100755 --- a/indra/newview/llinventorymodelbackgroundfetch.cpp +++ b/indra/newview/llinventorymodelbackgroundfetch.cpp @@ -1,10 +1,10 @@ /** - * @file llinventorymodel.cpp - * @brief Implementation of the inventory model used to track agent inventory. + * @file llinventorymodelbackgroundfetch.cpp + * @brief Implementation of background fetching of inventory. * * $LicenseInfo:firstyear=2002&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2010, Linden Research, Inc. + * Copyright (C) 2014, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -37,31 +37,175 @@ #include "llviewermessage.h" #include "llviewerregion.h" #include "llviewerwindow.h" +#include "llhttpconstants.h" +#include "bufferarray.h" +#include "bufferstream.h" +#include "llcorehttputil.h" + +// History (may be apocryphal) +// +// Around V2, an HTTP inventory download mechanism was added +// along with inventory LINK items referencing other inventory +// items. As part of this, at login, the entire inventory +// structure is downloaded 'in the background' using the +// backgroundFetch()/bulkFetch() methods. The UDP path can +// still be used and is found in the 'DEPRECATED OLD CODE' +// section. +// +// The old UDP path implemented a throttle that adapted +// itself during running. The mechanism survived info HTTP +// somewhat but was pinned to poll the HTTP plumbing at +// 0.5S intervals. The reasons for this particular value +// have been lost. It's possible to switch between UDP +// and HTTP while this is happening but there may be +// surprises in what happens in that case. +// +// Conversion to llcorehttp reduced the number of connections +// used but batches more data and queues more requests (but +// doesn't due pipelining due to libcurl restrictions). The +// poll interval above was re-examined and reduced to get +// inventory into the viewer more quickly. +// +// Possible future work: +// +// * Don't download the entire heirarchy in one go (which +// might have been how V1 worked). Implications for +// links (which may not have a valid target) and search +// which would then be missing data. +// +// * Review the download rate throttling. Slow then fast? +// Detect bandwidth usage and speed up when it drops? +// +// * A lot of calls to notifyObservers(). It looks like +// these could be collapsed by maintaining a 'dirty' +// bit and there appears to be an attempt to do this. +// But it isn't used or is used in a limited fashion. +// Are there semanic issues requiring a call after certain +// updateItem() calls? +// +// * An error on a fetch could be due to one item in the batch. +// If the batch were broken up, perhaps more of the inventory +// would download. (Handwave here, not certain this is an +// issue in practice.) +// +// * Conversion to AISv3. +// + + +namespace +{ + +///---------------------------------------------------------------------------- +/// Class <anonymous>::BGItemHttpHandler +///---------------------------------------------------------------------------- + +// +// Http request handler class for single inventory item requests. +// +// We'll use a handler-per-request pattern here rather than +// a shared handler. Mainly convenient as this was converted +// from a Responder class model. +// +// Derives from and is identical to the normal FetchItemHttpHandler +// except that: 1) it uses the background request object which is +// updated more slowly than the foreground and 2) keeps a count of +// active requests on the LLInventoryModelBackgroundFetch object +// to indicate outstanding operations are in-flight. +// +class BGItemHttpHandler : public LLInventoryModel::FetchItemHttpHandler +{ + LOG_CLASS(BGItemHttpHandler); + +public: + BGItemHttpHandler(const LLSD & request_sd) + : LLInventoryModel::FetchItemHttpHandler(request_sd) + { + LLInventoryModelBackgroundFetch::instance().incrFetchCount(1); + } + + virtual ~BGItemHttpHandler() + { + LLInventoryModelBackgroundFetch::instance().incrFetchCount(-1); + } + +protected: + BGItemHttpHandler(const BGItemHttpHandler &); // Not defined + void operator=(const BGItemHttpHandler &); // Not defined +}; + + +///---------------------------------------------------------------------------- +/// Class <anonymous>::BGFolderHttpHandler +///---------------------------------------------------------------------------- + +// Http request handler class for folders. +// +// Handler for FetchInventoryDescendents2 and FetchLibDescendents2 +// caps requests for folders. +// +class BGFolderHttpHandler : public LLCore::HttpHandler +{ + LOG_CLASS(BGFolderHttpHandler); + +public: + BGFolderHttpHandler(const LLSD & request_sd, const uuid_vec_t & recursive_cats) + : LLCore::HttpHandler(), + mRequestSD(request_sd), + mRecursiveCatUUIDs(recursive_cats) + { + LLInventoryModelBackgroundFetch::instance().incrFetchCount(1); + } + + virtual ~BGFolderHttpHandler() + { + LLInventoryModelBackgroundFetch::instance().incrFetchCount(-1); + } + +protected: + BGFolderHttpHandler(const BGFolderHttpHandler &); // Not defined + void operator=(const BGFolderHttpHandler &); // Not defined -const F32 MAX_TIME_FOR_SINGLE_FETCH = 10.f; -const S32 MAX_FETCH_RETRIES = 10; +public: + virtual void onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * response); + + bool getIsRecursive(const LLUUID & cat_id) const; + +private: + void processData(LLSD & body, LLCore::HttpResponse * response); + void processFailure(LLCore::HttpStatus status, LLCore::HttpResponse * response); + void processFailure(const char * const reason, LLCore::HttpResponse * response); + +private: + LLSD mRequestSD; + const uuid_vec_t mRecursiveCatUUIDs; // hack for storing away which cat fetches are recursive +}; + + +const char * const LOG_INV("Inventory"); + +} // end of namespace anonymous + + +///---------------------------------------------------------------------------- +/// Class LLInventoryModelBackgroundFetch +///---------------------------------------------------------------------------- -LLInventoryModelBackgroundFetch::LLInventoryModelBackgroundFetch() : +LLInventoryModelBackgroundFetch::LLInventoryModelBackgroundFetch(): mBackgroundFetchActive(FALSE), mFolderFetchActive(false), + mFetchCount(0), mAllFoldersFetched(FALSE), mRecursiveInventoryFetchStarted(FALSE), mRecursiveLibraryFetchStarted(FALSE), - mNumFetchRetries(0), - mMinTimeBetweenFetches(0.3f), - mMaxTimeBetweenFetches(10.f), - mTimelyFetchPending(FALSE), - mFetchCount(0) -{ -} + mMinTimeBetweenFetches(0.3f) +{} LLInventoryModelBackgroundFetch::~LLInventoryModelBackgroundFetch() -{ -} +{} bool LLInventoryModelBackgroundFetch::isBulkFetchProcessingComplete() const { - return mFetchQueue.empty() && mFetchCount<=0; + return mFetchQueue.empty() && mFetchCount <= 0; } bool LLInventoryModelBackgroundFetch::libraryFetchStarted() const @@ -91,7 +235,7 @@ bool LLInventoryModelBackgroundFetch::inventoryFetchCompleted() const bool LLInventoryModelBackgroundFetch::inventoryFetchInProgress() const { - return inventoryFetchStarted() && !inventoryFetchCompleted(); + return inventoryFetchStarted() && ! inventoryFetchCompleted(); } bool LLInventoryModelBackgroundFetch::isEverythingFetched() const @@ -104,24 +248,36 @@ BOOL LLInventoryModelBackgroundFetch::folderFetchActive() const return mFolderFetchActive; } +void LLInventoryModelBackgroundFetch::addRequestAtFront(const LLUUID & id, BOOL recursive, bool is_category) +{ + mFetchQueue.push_front(FetchQueueInfo(id, recursive, is_category)); +} + +void LLInventoryModelBackgroundFetch::addRequestAtBack(const LLUUID & id, BOOL recursive, bool is_category) +{ + mFetchQueue.push_back(FetchQueueInfo(id, recursive, is_category)); +} + void LLInventoryModelBackgroundFetch::start(const LLUUID& id, BOOL recursive) { - LLViewerInventoryCategory* cat = gInventory.getCategory(id); - if (cat || (id.isNull() && !isEverythingFetched())) - { // it's a folder, do a bulk fetch - LL_DEBUGS("InventoryFetch") << "Start fetching category: " << id << ", recursive: " << recursive << LL_ENDL; + LLViewerInventoryCategory * cat(gInventory.getCategory(id)); + + if (cat || (id.isNull() && ! isEverythingFetched())) + { + // it's a folder, do a bulk fetch + LL_DEBUGS(LOG_INV) << "Start fetching category: " << id << ", recursive: " << recursive << LL_ENDL; mBackgroundFetchActive = TRUE; mFolderFetchActive = true; if (id.isNull()) { - if (!mRecursiveInventoryFetchStarted) + if (! mRecursiveInventoryFetchStarted) { mRecursiveInventoryFetchStarted |= recursive; mFetchQueue.push_back(FetchQueueInfo(gInventory.getRootFolderID(), recursive)); gIdleCallbacks.addFunction(&LLInventoryModelBackgroundFetch::backgroundFetchCB, NULL); } - if (!mRecursiveLibraryFetchStarted) + if (! mRecursiveLibraryFetchStarted) { mRecursiveLibraryFetchStarted |= recursive; mFetchQueue.push_back(FetchQueueInfo(gInventory.getLibraryRootFolderID(), recursive)); @@ -146,9 +302,9 @@ void LLInventoryModelBackgroundFetch::start(const LLUUID& id, BOOL recursive) } } } - else if (LLViewerInventoryItem* itemp = gInventory.getItem(id)) + else if (LLViewerInventoryItem * itemp = gInventory.getItem(id)) { - if (!itemp->mIsComplete && (mFetchQueue.empty() || mFetchQueue.front().mUUID != id)) + if (! itemp->mIsComplete && (mFetchQueue.empty() || mFetchQueue.front().mUUID != id)) { mBackgroundFetchActive = TRUE; @@ -172,8 +328,12 @@ void LLInventoryModelBackgroundFetch::setAllFoldersFetched() mRecursiveLibraryFetchStarted) { mAllFoldersFetched = TRUE; + //LL_INFOS(LOG_INV) << "All folders fetched, validating" << LL_ENDL; + //gInventory.validate(); } mFolderFetchActive = false; + mBackgroundFetchActive = false; + LL_INFOS(LOG_INV) << "Inventory background fetch completed" << LL_ENDL; } void LLInventoryModelBackgroundFetch::backgroundFetchCB(void *) @@ -183,262 +343,364 @@ void LLInventoryModelBackgroundFetch::backgroundFetchCB(void *) void LLInventoryModelBackgroundFetch::backgroundFetch() { - if (mBackgroundFetchActive && gAgent.getRegion()) + if (mBackgroundFetchActive && gAgent.getRegion() && gAgent.getRegion()->capabilitiesReceived()) { // If we'll be using the capability, we'll be sending batches and the background thing isn't as important. - if (gSavedSettings.getBOOL("UseHTTPInventory")) - { - bulkFetch(); - return; - } - -#if 1 - //-------------------------------------------------------------------------------- - // DEPRECATED OLD CODE - // + bulkFetch(); + } +} - // No more categories to fetch, stop fetch process. - if (mFetchQueue.empty()) - { - llinfos << "Inventory fetch completed" << llendl; +void LLInventoryModelBackgroundFetch::incrFetchCount(S32 fetching) +{ + mFetchCount += fetching; + if (mFetchCount < 0) + { + LL_WARNS_ONCE(LOG_INV) << "Inventory fetch count fell below zero (0)." << LL_ENDL; + mFetchCount = 0; + } +} - setAllFoldersFetched(); - mBackgroundFetchActive = false; - mFolderFetchActive = false; +// Bundle up a bunch of requests to send all at once. +void LLInventoryModelBackgroundFetch::bulkFetch() +{ + //Background fetch is called from gIdleCallbacks in a loop until background fetch is stopped. + //If there are items in mFetchQueue, we want to check the time since the last bulkFetch was + //sent. If it exceeds our retry time, go ahead and fire off another batch. + LLViewerRegion * region(gAgent.getRegion()); + if (! region || gDisconnected) + { + return; + } - return; - } + // *TODO: These values could be tweaked at runtime to effect + // a fast/slow fetch throttle. Once login is complete and the scene + // is mostly loaded, we could turn up the throttle and fill missing + // inventory more quickly. + static const U32 max_batch_size(10); + static const S32 max_concurrent_fetches(12); // Outstanding requests, not connections + static const F32 new_min_time(0.05f); // *HACK: Clean this up when old code goes away entirely. + + mMinTimeBetweenFetches = new_min_time; + if (mMinTimeBetweenFetches < new_min_time) + { + mMinTimeBetweenFetches = new_min_time; // *HACK: See above. + } - F32 fast_fetch_time = lerp(mMinTimeBetweenFetches, mMaxTimeBetweenFetches, 0.1f); - F32 slow_fetch_time = lerp(mMinTimeBetweenFetches, mMaxTimeBetweenFetches, 0.5f); - if (mTimelyFetchPending && mFetchTimer.getElapsedTimeF32() > slow_fetch_time) - { - // Double timeouts on failure. - mMinTimeBetweenFetches = llmin(mMinTimeBetweenFetches * 2.f, 10.f); - mMaxTimeBetweenFetches = llmin(mMaxTimeBetweenFetches * 2.f, 120.f); - lldebugs << "Inventory fetch times grown to (" << mMinTimeBetweenFetches << ", " << mMaxTimeBetweenFetches << ")" << llendl; - // fetch is no longer considered "timely" although we will wait for full time-out. - mTimelyFetchPending = FALSE; - } + if (mFetchCount) + { + // Process completed background HTTP requests + gInventory.handleResponses(false); + } + + if ((mFetchCount > max_concurrent_fetches) || + (mFetchTimer.getElapsedTimeF32() < mMinTimeBetweenFetches)) + { + return; + } + + U32 item_count(0); + U32 folder_count(0); + + const U32 sort_order(gSavedSettings.getU32(LLInventoryPanel::DEFAULT_SORT_ORDER) & 0x1); + + // *TODO: Think I'd like to get a shared pointer to this and share it + // among all the folder requests. + uuid_vec_t recursive_cats; + + LLSD folder_request_body; + LLSD folder_request_body_lib; + LLSD item_request_body; + LLSD item_request_body_lib; - while(1) + while (! mFetchQueue.empty() + && (item_count + folder_count) < max_batch_size) + { + const FetchQueueInfo & fetch_info(mFetchQueue.front()); + if (fetch_info.mIsCategory) { - if (mFetchQueue.empty()) + const LLUUID & cat_id(fetch_info.mUUID); + if (cat_id.isNull()) //DEV-17797 { - break; + LLSD folder_sd; + folder_sd["folder_id"] = LLUUID::null.asString(); + folder_sd["owner_id"] = gAgent.getID(); + folder_sd["sort_order"] = LLSD::Integer(sort_order); + folder_sd["fetch_folders"] = LLSD::Boolean(FALSE); + folder_sd["fetch_items"] = LLSD::Boolean(TRUE); + folder_request_body["folders"].append(folder_sd); + folder_count++; } + else + { + const LLViewerInventoryCategory * cat(gInventory.getCategory(cat_id)); + + if (cat) + { + if (LLViewerInventoryCategory::VERSION_UNKNOWN == cat->getVersion()) + { + LLSD folder_sd; + folder_sd["folder_id"] = cat->getUUID(); + folder_sd["owner_id"] = cat->getOwnerID(); + folder_sd["sort_order"] = LLSD::Integer(sort_order); + folder_sd["fetch_folders"] = LLSD::Boolean(TRUE); //(LLSD::Boolean)sFullFetchStarted; + folder_sd["fetch_items"] = LLSD::Boolean(TRUE); + + if (ALEXANDRIA_LINDEN_ID == cat->getOwnerID()) + { + folder_request_body_lib["folders"].append(folder_sd); + } + else + { + folder_request_body["folders"].append(folder_sd); + } + folder_count++; + } - if(gDisconnected) + // May already have this folder, but append child folders to list. + if (fetch_info.mRecursive) + { + LLInventoryModel::cat_array_t * categories(NULL); + LLInventoryModel::item_array_t * items(NULL); + gInventory.getDirectDescendentsOf(cat->getUUID(), categories, items); + for (LLInventoryModel::cat_array_t::const_iterator it = categories->begin(); + it != categories->end(); + ++it) + { + mFetchQueue.push_back(FetchQueueInfo((*it)->getUUID(), fetch_info.mRecursive)); + } + } + } + } + if (fetch_info.mRecursive) { - // Just bail if we are disconnected. - break; + recursive_cats.push_back(cat_id); } + } + else + { + LLViewerInventoryItem * itemp(gInventory.getItem(fetch_info.mUUID)); - const FetchQueueInfo info = mFetchQueue.front(); - - if (info.mIsCategory) + if (itemp) { - - LLViewerInventoryCategory* cat = gInventory.getCategory(info.mUUID); - - // Category has been deleted, remove from queue. - if (!cat) + LLSD item_sd; + item_sd["owner_id"] = itemp->getPermissions().getOwner(); + item_sd["item_id"] = itemp->getUUID(); + if (itemp->getPermissions().getOwner() == gAgent.getID()) { - mFetchQueue.pop_front(); - continue; + item_request_body.append(item_sd); } - - if (mFetchTimer.getElapsedTimeF32() > mMinTimeBetweenFetches && - LLViewerInventoryCategory::VERSION_UNKNOWN == cat->getVersion()) + else { - // Category exists but has no children yet, fetch the descendants - // for now, just request every time and rely on retry timer to throttle. - if (cat->fetch()) - { - mFetchTimer.reset(); - mTimelyFetchPending = TRUE; - } - else - { - // The catagory also tracks if it has expired and here it says it hasn't - // yet. Get out of here because nothing is going to happen until we - // update the timers. - break; - } + item_request_body_lib.append(item_sd); } - // Do I have all my children? - else if (gInventory.isCategoryComplete(info.mUUID)) - { - // Finished with this category, remove from queue. - mFetchQueue.pop_front(); - - // Add all children to queue. - LLInventoryModel::cat_array_t* categories; - LLInventoryModel::item_array_t* items; - gInventory.getDirectDescendentsOf(cat->getUUID(), categories, items); - for (LLInventoryModel::cat_array_t::const_iterator it = categories->begin(); - it != categories->end(); - ++it) - { - mFetchQueue.push_back(FetchQueueInfo((*it)->getUUID(),info.mRecursive)); - } + //itemp->fetchFromServer(); + item_count++; + } + } - // We received a response in less than the fast time. - if (mTimelyFetchPending && mFetchTimer.getElapsedTimeF32() < fast_fetch_time) - { - // Shrink timeouts based on success. - mMinTimeBetweenFetches = llmax(mMinTimeBetweenFetches * 0.8f, 0.3f); - mMaxTimeBetweenFetches = llmax(mMaxTimeBetweenFetches * 0.8f, 10.f); - lldebugs << "Inventory fetch times shrunk to (" << mMinTimeBetweenFetches << ", " << mMaxTimeBetweenFetches << ")" << llendl; - } + mFetchQueue.pop_front(); + } - mTimelyFetchPending = FALSE; - continue; - } - else if (mFetchTimer.getElapsedTimeF32() > mMaxTimeBetweenFetches) - { - // Received first packet, but our num descendants does not match db's num descendants - // so try again later. - mFetchQueue.pop_front(); + // Issue HTTP POST requests to fetch folders and items + + if (item_count + folder_count > 0) + { + if (folder_count) + { + if (folder_request_body["folders"].size()) + { + const std::string url(region->getCapability("FetchInventoryDescendents2")); - if (mNumFetchRetries++ < MAX_FETCH_RETRIES) - { - // push on back of queue - mFetchQueue.push_back(info); - } - mTimelyFetchPending = FALSE; - mFetchTimer.reset(); - break; + if (! url.empty()) + { + BGFolderHttpHandler * handler(new BGFolderHttpHandler(folder_request_body, recursive_cats)); + gInventory.requestPost(false, url, folder_request_body, handler, "Inventory Folder"); } - - // Not enough time has elapsed to do a new fetch - break; } - else + + if (folder_request_body_lib["folders"].size()) { - LLViewerInventoryItem* itemp = gInventory.getItem(info.mUUID); + const std::string url(region->getCapability("FetchLibDescendents2")); - mFetchQueue.pop_front(); - if (!itemp) + if (! url.empty()) { - continue; + BGFolderHttpHandler * handler(new BGFolderHttpHandler(folder_request_body_lib, recursive_cats)); + gInventory.requestPost(false, url, folder_request_body_lib, handler, "Library Folder"); } + } + } // if (folder_count) - if (mFetchTimer.getElapsedTimeF32() > mMinTimeBetweenFetches) - { - itemp->fetchFromServer(); - mFetchTimer.reset(); - mTimelyFetchPending = TRUE; - } - else if (itemp->mIsComplete) + if (item_count) + { + if (item_request_body.size()) + { + const std::string url(region->getCapability("FetchInventory2")); + + if (! url.empty()) { - mTimelyFetchPending = FALSE; + LLSD body; + body["items"] = item_request_body; + BGItemHttpHandler * handler(new BGItemHttpHandler(body)); + gInventory.requestPost(false, url, body, handler, "Inventory Item"); } - else if (mFetchTimer.getElapsedTimeF32() > mMaxTimeBetweenFetches) + } + + if (item_request_body_lib.size()) + { + const std::string url(region->getCapability("FetchLib2")); + + if (! url.empty()) { - mFetchQueue.push_back(info); - mFetchTimer.reset(); - mTimelyFetchPending = FALSE; + LLSD body; + body["items"] = item_request_body_lib; + BGItemHttpHandler * handler(new BGItemHttpHandler(body)); + gInventory.requestPost(false, url, body, handler, "Library Item"); } - // Not enough time has elapsed to do a new fetch - break; } - } - - // - // DEPRECATED OLD CODE - //-------------------------------------------------------------------------------- -#endif + } // if (item_count) + + mFetchTimer.reset(); + } + else if (isBulkFetchProcessingComplete()) + { + setAllFoldersFetched(); } } -void LLInventoryModelBackgroundFetch::incrFetchCount(S16 fetching) -{ - mFetchCount += fetching; - if (mFetchCount < 0) +bool LLInventoryModelBackgroundFetch::fetchQueueContainsNoDescendentsOf(const LLUUID & cat_id) const +{ + for (fetch_queue_t::const_iterator it = mFetchQueue.begin(); + it != mFetchQueue.end(); + ++it) { - mFetchCount = 0; + const LLUUID & fetch_id = (*it).mUUID; + if (gInventory.isObjectDescendentOf(fetch_id, cat_id)) + return false; } + return true; } -class LLInventoryModelFetchItemResponder : public LLInventoryModel::fetchInventoryResponder -{ -public: - LLInventoryModelFetchItemResponder(const LLSD& request_sd) : LLInventoryModel::fetchInventoryResponder(request_sd) {}; - void result(const LLSD& content); - void error(U32 status, const std::string& reason); -}; -void LLInventoryModelFetchItemResponder::result( const LLSD& content ) +namespace { - LLInventoryModel::fetchInventoryResponder::result(content); - LLInventoryModelBackgroundFetch::instance().incrFetchCount(-1); -} -void LLInventoryModelFetchItemResponder::error( U32 status, const std::string& reason ) +///---------------------------------------------------------------------------- +/// Class <anonymous>::BGFolderHttpHandler +///---------------------------------------------------------------------------- + +void BGFolderHttpHandler::onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * response) { - LLInventoryModel::fetchInventoryResponder::error(status, reason); - LLInventoryModelBackgroundFetch::instance().incrFetchCount(-1); + do // Single-pass do-while used for common exit handling + { + LLCore::HttpStatus status(response->getStatus()); + // status = LLCore::HttpStatus(404); // Dev tool to force error handling + if (! status) + { + processFailure(status, response); + break; // Goto common exit + } + + // Response body should be present. + LLCore::BufferArray * body(response->getBody()); + // body = NULL; // Dev tool to force error handling + if (! body || ! body->size()) + { + LL_WARNS(LOG_INV) << "Missing data in inventory folder query." << LL_ENDL; + processFailure("HTTP response missing expected body", response); + break; // Goto common exit + } + + // Could test 'Content-Type' header but probably unreliable. + + // Convert response to LLSD + // body->write(0, "Garbage Response", 16); // Dev tool to force error handling + LLSD body_llsd; + if (! LLCoreHttpUtil::responseToLLSD(response, true, body_llsd)) + { + // INFOS-level logging will occur on the parsed failure + processFailure("HTTP response contained malformed LLSD", response); + break; // goto common exit + } + + // Expect top-level structure to be a map + // body_llsd = LLSD::emptyArray(); // Dev tool to force error handling + if (! body_llsd.isMap()) + { + processFailure("LLSD response not a map", response); + break; // goto common exit + } + + // Check for 200-with-error failures + // + // See comments in llinventorymodel.cpp about this mode of error. + // + // body_llsd["error"] = LLSD::emptyMap(); // Dev tool to force error handling + // body_llsd["error"]["identifier"] = "Development"; + // body_llsd["error"]["message"] = "You left development code in the viewer"; + if (body_llsd.has("error")) + { + processFailure("Inventory application error (200-with-error)", response); + break; // goto common exit + } + + // Okay, process data if possible + processData(body_llsd, response); + } + while (false); + + // Must delete on completion. + delete this; } -class LLInventoryModelFetchDescendentsResponder: public LLHTTPClient::Responder +void BGFolderHttpHandler::processData(LLSD & content, LLCore::HttpResponse * response) { -public: - LLInventoryModelFetchDescendentsResponder(const LLSD& request_sd, uuid_vec_t recursive_cats) : - mRequestSD(request_sd), - mRecursiveCatUUIDs(recursive_cats) - {}; - //LLInventoryModelFetchDescendentsResponder() {}; - void result(const LLSD& content); - void error(U32 status, const std::string& reason); -protected: - BOOL getIsRecursive(const LLUUID& cat_id) const; -private: - LLSD mRequestSD; - uuid_vec_t mRecursiveCatUUIDs; // hack for storing away which cat fetches are recursive -}; + LLInventoryModelBackgroundFetch * fetcher(LLInventoryModelBackgroundFetch::getInstance()); -// If we get back a normal response, handle it here. -void LLInventoryModelFetchDescendentsResponder::result(const LLSD& content) -{ - LLInventoryModelBackgroundFetch *fetcher = LLInventoryModelBackgroundFetch::getInstance(); + // API V2 and earlier should probably be testing for "error" map + // in response as an application-level error. + + // Instead, we assume success and attempt to extract information. if (content.has("folders")) { - - for(LLSD::array_const_iterator folder_it = content["folders"].beginArray(); - folder_it != content["folders"].endArray(); + LLSD folders(content["folders"]); + + for (LLSD::array_const_iterator folder_it = folders.beginArray(); + folder_it != folders.endArray(); ++folder_it) { - LLSD folder_sd = *folder_it; - + LLSD folder_sd(*folder_it); //LLUUID agent_id = folder_sd["agent_id"]; //if(agent_id != gAgent.getID()) //This should never happen. //{ - // llwarns << "Got a UpdateInventoryItem for the wrong agent." - // << llendl; + // LL_WARNS(LOG_INV) << "Got a UpdateInventoryItem for the wrong agent." + // << LL_ENDL; // break; //} - LLUUID parent_id = folder_sd["folder_id"]; - LLUUID owner_id = folder_sd["owner_id"]; - S32 version = (S32)folder_sd["version"].asInteger(); - S32 descendents = (S32)folder_sd["descendents"].asInteger(); + LLUUID parent_id(folder_sd["folder_id"].asUUID()); + LLUUID owner_id(folder_sd["owner_id"].asUUID()); + S32 version(folder_sd["version"].asInteger()); + S32 descendents(folder_sd["descendents"].asInteger()); LLPointer<LLViewerInventoryCategory> tcategory = new LLViewerInventoryCategory(owner_id); if (parent_id.isNull()) { + LLSD items(folder_sd["items"]); LLPointer<LLViewerInventoryItem> titem = new LLViewerInventoryItem; - for(LLSD::array_const_iterator item_it = folder_sd["items"].beginArray(); - item_it != folder_sd["items"].endArray(); + + for (LLSD::array_const_iterator item_it = items.beginArray(); + item_it != items.endArray(); ++item_it) { - const LLUUID lost_uuid = gInventory.findCategoryUUIDForType(LLFolderType::FT_LOST_AND_FOUND); + const LLUUID lost_uuid(gInventory.findCategoryUUIDForType(LLFolderType::FT_LOST_AND_FOUND)); + if (lost_uuid.notNull()) { - LLSD item = *item_it; + LLSD item(*item_it); + titem->unpackMessage(item); LLInventoryModel::update_list_t update; @@ -450,104 +712,113 @@ void LLInventoryModelFetchDescendentsResponder::result(const LLSD& content) titem->updateParentOnServer(FALSE); gInventory.updateItem(titem); gInventory.notifyObservers(); - } } } - LLViewerInventoryCategory* pcat = gInventory.getCategory(parent_id); - if (!pcat) + LLViewerInventoryCategory * pcat(gInventory.getCategory(parent_id)); + if (! pcat) { continue; } - for(LLSD::array_const_iterator category_it = folder_sd["categories"].beginArray(); - category_it != folder_sd["categories"].endArray(); + LLSD categories(folder_sd["categories"]); + for (LLSD::array_const_iterator category_it = categories.beginArray(); + category_it != categories.endArray(); ++category_it) { - LLSD category = *category_it; + LLSD category(*category_it); tcategory->fromLLSD(category); - const BOOL recursive = getIsRecursive(tcategory->getUUID()); - + const bool recursive(getIsRecursive(tcategory->getUUID())); if (recursive) { - fetcher->mFetchQueue.push_back(LLInventoryModelBackgroundFetch::FetchQueueInfo(tcategory->getUUID(), recursive)); + fetcher->addRequestAtBack(tcategory->getUUID(), recursive, true); } - else if ( !gInventory.isCategoryComplete(tcategory->getUUID()) ) + else if (! gInventory.isCategoryComplete(tcategory->getUUID())) { gInventory.updateCategory(tcategory); } - } + + LLSD items(folder_sd["items"]); LLPointer<LLViewerInventoryItem> titem = new LLViewerInventoryItem; - for(LLSD::array_const_iterator item_it = folder_sd["items"].beginArray(); - item_it != folder_sd["items"].endArray(); - ++item_it) + for (LLSD::array_const_iterator item_it = items.beginArray(); + item_it != items.endArray(); + ++item_it) { - LLSD item = *item_it; + LLSD item(*item_it); titem->unpackMessage(item); gInventory.updateItem(titem); } // Set version and descendentcount according to message. - LLViewerInventoryCategory* cat = gInventory.getCategory(parent_id); - if(cat) + LLViewerInventoryCategory * cat(gInventory.getCategory(parent_id)); + if (cat) { cat->setVersion(version); cat->setDescendentCount(descendents); cat->determineFolderType(); } - } } if (content.has("bad_folders")) { - for(LLSD::array_const_iterator folder_it = content["bad_folders"].beginArray(); - folder_it != content["bad_folders"].endArray(); - ++folder_it) - { - LLSD folder_sd = *folder_it; + LLSD bad_folders(content["bad_folders"]); + for (LLSD::array_const_iterator folder_it = bad_folders.beginArray(); + folder_it != bad_folders.endArray(); + ++folder_it) + { + // *TODO: Stop copying data [ed: this isn't copying data] + LLSD folder_sd(*folder_it); // These folders failed on the dataserver. We probably don't want to retry them. - llinfos << "Folder " << folder_sd["folder_id"].asString() - << "Error: " << folder_sd["error"].asString() << llendl; + LL_WARNS(LOG_INV) << "Folder " << folder_sd["folder_id"].asString() + << "Error: " << folder_sd["error"].asString() << LL_ENDL; } } - - fetcher->incrFetchCount(-1); if (fetcher->isBulkFetchProcessingComplete()) { - llinfos << "Inventory fetch completed" << llendl; fetcher->setAllFoldersFetched(); } gInventory.notifyObservers(); } -// If we get back an error (not found, etc...), handle it here. -void LLInventoryModelFetchDescendentsResponder::error(U32 status, const std::string& reason) + +void BGFolderHttpHandler::processFailure(LLCore::HttpStatus status, LLCore::HttpResponse * response) { + const std::string & ct(response->getContentType()); + LL_WARNS(LOG_INV) << "Inventory folder fetch failure\n" + << "[Status: " << status.toTerseString() << "]\n" + << "[Reason: " << status.toString() << "]\n" + << "[Content-type: " << ct << "]\n" + << "[Content (abridged): " + << LLCoreHttpUtil::responseToString(response) << "]" << LL_ENDL; + + // Could use a 404 test here to try to detect revoked caps... + + // This was originally the request retry logic for the inventory + // request which tested on HTTP_INTERNAL_ERROR status. This + // retry logic was unbounded and lacked discrimination as to the + // cause of the retry. The new http library should be doing + // adquately on retries but I want to keep the structure of a + // retry for reference. LLInventoryModelBackgroundFetch *fetcher = LLInventoryModelBackgroundFetch::getInstance(); - - llinfos << "LLInventoryModelFetchDescendentsResponder::error " - << status << ": " << reason << llendl; - - fetcher->incrFetchCount(-1); - - if (status==499) // timed out + if (false) { - for(LLSD::array_const_iterator folder_it = mRequestSD["folders"].beginArray(); - folder_it != mRequestSD["folders"].endArray(); - ++folder_it) - { - LLSD folder_sd = *folder_it; - LLUUID folder_id = folder_sd["folder_id"]; + // timed out or curl failure + for (LLSD::array_const_iterator folder_it = mRequestSD["folders"].beginArray(); + folder_it != mRequestSD["folders"].endArray(); + ++folder_it) + { + LLSD folder_sd(*folder_it); + LLUUID folder_id(folder_sd["folder_id"].asUUID()); const BOOL recursive = getIsRecursive(folder_id); - fetcher->mFetchQueue.push_front(LLInventoryModelBackgroundFetch::FetchQueueInfo(folder_id, recursive)); + fetcher->addRequestAtFront(folder_id, recursive, true); } } else @@ -560,212 +831,54 @@ void LLInventoryModelFetchDescendentsResponder::error(U32 status, const std::str gInventory.notifyObservers(); } -BOOL LLInventoryModelFetchDescendentsResponder::getIsRecursive(const LLUUID& cat_id) const -{ - return (std::find(mRecursiveCatUUIDs.begin(),mRecursiveCatUUIDs.end(), cat_id) != mRecursiveCatUUIDs.end()); -} -// Bundle up a bunch of requests to send all at once. -// static -void LLInventoryModelBackgroundFetch::bulkFetch() +void BGFolderHttpHandler::processFailure(const char * const reason, LLCore::HttpResponse * response) { - //Background fetch is called from gIdleCallbacks in a loop until background fetch is stopped. - //If there are items in mFetchQueue, we want to check the time since the last bulkFetch was - //sent. If it exceeds our retry time, go ahead and fire off another batch. - LLViewerRegion* region = gAgent.getRegion(); - if (!region) return; - - S16 max_concurrent_fetches=8; - F32 new_min_time = 0.5f; //HACK! Clean this up when old code goes away entirely. - if (mMinTimeBetweenFetches < new_min_time) - { - mMinTimeBetweenFetches=new_min_time; //HACK! See above. - } - - if (gDisconnected || - (mFetchCount > max_concurrent_fetches) || - (mFetchTimer.getElapsedTimeF32() < mMinTimeBetweenFetches)) - { - return; // just bail if we are disconnected - } - - U32 item_count=0; - U32 folder_count=0; - U32 max_batch_size=5; - - U32 sort_order = gSavedSettings.getU32(LLInventoryPanel::DEFAULT_SORT_ORDER) & 0x1; - - uuid_vec_t recursive_cats; - - LLSD folder_request_body; - LLSD folder_request_body_lib; - LLSD item_request_body; - LLSD item_request_body_lib; - - while (!mFetchQueue.empty() - && (item_count + folder_count) < max_batch_size) + LL_WARNS(LOG_INV) << "Inventory folder fetch failure\n" + << "[Status: internal error]\n" + << "[Reason: " << reason << "]\n" + << "[Content (abridged): " + << LLCoreHttpUtil::responseToString(response) << "]" << LL_ENDL; + + // Reverse of previous processFailure() method, this is invoked + // when response structure is found to be invalid. Original + // always re-issued the request (without limit). This does + // the same but be aware that this may be a source of problems. + // Philosophy is that inventory folders are so essential to + // operation that this is a reasonable action. + LLInventoryModelBackgroundFetch *fetcher = LLInventoryModelBackgroundFetch::getInstance(); + if (true) { - const FetchQueueInfo& fetch_info = mFetchQueue.front(); - if (fetch_info.mIsCategory) + for (LLSD::array_const_iterator folder_it = mRequestSD["folders"].beginArray(); + folder_it != mRequestSD["folders"].endArray(); + ++folder_it) { - const LLUUID &cat_id = fetch_info.mUUID; - if (cat_id.isNull()) //DEV-17797 - { - LLSD folder_sd; - folder_sd["folder_id"] = LLUUID::null.asString(); - folder_sd["owner_id"] = gAgent.getID(); - folder_sd["sort_order"] = (LLSD::Integer)sort_order; - folder_sd["fetch_folders"] = (LLSD::Boolean)FALSE; - folder_sd["fetch_items"] = (LLSD::Boolean)TRUE; - folder_request_body["folders"].append(folder_sd); - folder_count++; - } - else - { - const LLViewerInventoryCategory* cat = gInventory.getCategory(cat_id); - - if (cat) - { - if (LLViewerInventoryCategory::VERSION_UNKNOWN == cat->getVersion()) - { - LLSD folder_sd; - folder_sd["folder_id"] = cat->getUUID(); - folder_sd["owner_id"] = cat->getOwnerID(); - folder_sd["sort_order"] = (LLSD::Integer)sort_order; - folder_sd["fetch_folders"] = TRUE; //(LLSD::Boolean)sFullFetchStarted; - folder_sd["fetch_items"] = (LLSD::Boolean)TRUE; - - if (ALEXANDRIA_LINDEN_ID == cat->getOwnerID()) - folder_request_body_lib["folders"].append(folder_sd); - else - folder_request_body["folders"].append(folder_sd); - folder_count++; - } - // May already have this folder, but append child folders to list. - if (fetch_info.mRecursive) - { - LLInventoryModel::cat_array_t* categories; - LLInventoryModel::item_array_t* items; - gInventory.getDirectDescendentsOf(cat->getUUID(), categories, items); - for (LLInventoryModel::cat_array_t::const_iterator it = categories->begin(); - it != categories->end(); - ++it) - { - mFetchQueue.push_back(FetchQueueInfo((*it)->getUUID(), fetch_info.mRecursive)); - } - } - } - } - if (fetch_info.mRecursive) - recursive_cats.push_back(cat_id); - } - else - { - LLViewerInventoryItem* itemp = gInventory.getItem(fetch_info.mUUID); - if (itemp) - { - LLSD item_sd; - item_sd["owner_id"] = itemp->getPermissions().getOwner(); - item_sd["item_id"] = itemp->getUUID(); - if (itemp->getPermissions().getOwner() == gAgent.getID()) - { - item_request_body.append(item_sd); - } - else - { - item_request_body_lib.append(item_sd); - } - //itemp->fetchFromServer(); - item_count++; - } + LLSD folder_sd(*folder_it); + LLUUID folder_id(folder_sd["folder_id"].asUUID()); + const BOOL recursive = getIsRecursive(folder_id); + fetcher->addRequestAtFront(folder_id, recursive, true); } - - mFetchQueue.pop_front(); } - - if (item_count + folder_count > 0) + else { - if (folder_count) - { - std::string url = region->getCapability("FetchInventoryDescendents2"); - mFetchCount++; - if (folder_request_body["folders"].size()) - { - LLInventoryModelFetchDescendentsResponder *fetcher = new LLInventoryModelFetchDescendentsResponder(folder_request_body, recursive_cats); - LLHTTPClient::post(url, folder_request_body, fetcher, 300.0); - } - if (folder_request_body_lib["folders"].size()) - { - std::string url_lib = gAgent.getRegion()->getCapability("FetchLibDescendents2"); - - LLInventoryModelFetchDescendentsResponder *fetcher = new LLInventoryModelFetchDescendentsResponder(folder_request_body_lib, recursive_cats); - LLHTTPClient::post(url_lib, folder_request_body_lib, fetcher, 300.0); - } - } - if (item_count) + if (fetcher->isBulkFetchProcessingComplete()) { - std::string url; - - if (item_request_body.size()) - { - mFetchCount++; - url = region->getCapability("FetchInventory2"); - if (!url.empty()) - { - LLSD body; - body["agent_id"] = gAgent.getID(); - body["items"] = item_request_body; - - LLHTTPClient::post(url, body, new LLInventoryModelFetchItemResponder(body)); - } - //else - //{ - // LLMessageSystem* msg = gMessageSystem; - // msg->newMessage("FetchInventory"); - // msg->nextBlock("AgentData"); - // msg->addUUID("AgentID", gAgent.getID()); - // msg->addUUID("SessionID", gAgent.getSessionID()); - // msg->nextBlock("InventoryData"); - // msg->addUUID("OwnerID", mPermissions.getOwner()); - // msg->addUUID("ItemID", mUUID); - // gAgent.sendReliableMessage(); - //} - } - - if (item_request_body_lib.size()) - { - mFetchCount++; - - url = region->getCapability("FetchLib2"); - if (!url.empty()) - { - LLSD body; - body["agent_id"] = gAgent.getID(); - body["items"] = item_request_body_lib; - - LLHTTPClient::post(url, body, new LLInventoryModelFetchItemResponder(body)); - } - } + fetcher->setAllFoldersFetched(); } - mFetchTimer.reset(); - } - - else if (isBulkFetchProcessingComplete()) - { - setAllFoldersFetched(); } + gInventory.notifyObservers(); } -bool LLInventoryModelBackgroundFetch::fetchQueueContainsNoDescendentsOf(const LLUUID& cat_id) const + +bool BGFolderHttpHandler::getIsRecursive(const LLUUID & cat_id) const { - for (fetch_queue_t::const_iterator it = mFetchQueue.begin(); - it != mFetchQueue.end(); ++it) - { - const LLUUID& fetch_id = (*it).mUUID; - if (gInventory.isObjectDescendentOf(fetch_id, cat_id)) - return false; - } - return true; + return std::find(mRecursiveCatUUIDs.begin(), mRecursiveCatUUIDs.end(), cat_id) != mRecursiveCatUUIDs.end(); } +///---------------------------------------------------------------------------- +/// Class <anonymous>::BGItemHttpHandler +///---------------------------------------------------------------------------- + +// Nothing to implement here. All ctor/dtor changes. +} // end namespace anonymous |