summaryrefslogtreecommitdiff
path: root/indra/newview/llinventorymodelbackgroundfetch.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'indra/newview/llinventorymodelbackgroundfetch.cpp')
-rw-r--r--indra/newview/llinventorymodelbackgroundfetch.cpp3238
1 files changed, 1619 insertions, 1619 deletions
diff --git a/indra/newview/llinventorymodelbackgroundfetch.cpp b/indra/newview/llinventorymodelbackgroundfetch.cpp
index 5975f685e3..9b0dc2fd50 100644
--- a/indra/newview/llinventorymodelbackgroundfetch.cpp
+++ b/indra/newview/llinventorymodelbackgroundfetch.cpp
@@ -1,1619 +1,1619 @@
-/**
- * @file llinventorymodelbackgroundfetch.cpp
- * @brief Implementation of background fetching of inventory.
- *
- * $LicenseInfo:firstyear=2002&license=viewerlgpl$
- * Second Life Viewer Source Code
- * Copyright (C) 2014, Linden Research, Inc.
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation;
- * version 2.1 of the License only.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- *
- * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA
- * $/LicenseInfo$
- */
-
-#include "llviewerprecompiledheaders.h"
-#include "llinventorymodelbackgroundfetch.h"
-
-#include "llaisapi.h"
-#include "llagent.h"
-#include "llappviewer.h"
-#include "llcallbacklist.h"
-#include "llinventorymodel.h"
-#include "llinventorypanel.h"
-#include "llnotificationsutil.h"
-#include "llstartup.h"
-#include "llviewercontrol.h"
-#include "llviewerinventory.h"
-#include "llviewermessage.h"
-#include "llviewerregion.h"
-#include "llviewerwindow.h"
-#include "llhttpconstants.h"
-#include "bufferarray.h"
-#include "bufferstream.h"
-#include "llcorehttputil.h"
-
-// History (may be apocryphal)
-//
-// Around V2, an HTTP inventory download mechanism was added
-// along with inventory LINK items referencing other inventory
-// items. As part of this, at login, the entire inventory
-// structure is downloaded 'in the background' using the
-// backgroundFetch()/bulkFetch() methods. The UDP path can
-// still be used and is found in the 'DEPRECATED OLD CODE'
-// section.
-//
-// The old UDP path implemented a throttle that adapted
-// itself during running. The mechanism survived info HTTP
-// somewhat but was pinned to poll the HTTP plumbing at
-// 0.5S intervals. The reasons for this particular value
-// have been lost. It's possible to switch between UDP
-// and HTTP while this is happening but there may be
-// surprises in what happens in that case.
-//
-// Conversion to llcorehttp reduced the number of connections
-// used but batches more data and queues more requests (but
-// doesn't due pipelining due to libcurl restrictions). The
-// poll interval above was re-examined and reduced to get
-// inventory into the viewer more quickly.
-//
-// Possible future work:
-//
-// * Don't download the entire heirarchy in one go (which
-// might have been how V1 worked). Implications for
-// links (which may not have a valid target) and search
-// which would then be missing data.
-//
-// * Review the download rate throttling. Slow then fast?
-// Detect bandwidth usage and speed up when it drops?
-//
-// * An error on a fetch could be due to one item in the batch.
-// If the batch were broken up, perhaps more of the inventory
-// would download. (Handwave here, not certain this is an
-// issue in practice.)
-//
-// * Conversion to AISv3.
-//
-
-
-namespace
-{
-
-///----------------------------------------------------------------------------
-/// Class <anonymous>::BGItemHttpHandler
-///----------------------------------------------------------------------------
-
-//
-// Http request handler class for single inventory item requests.
-//
-// We'll use a handler-per-request pattern here rather than
-// a shared handler. Mainly convenient as this was converted
-// from a Responder class model.
-//
-// Derives from and is identical to the normal FetchItemHttpHandler
-// except that: 1) it uses the background request object which is
-// updated more slowly than the foreground and 2) keeps a count of
-// active requests on the LLInventoryModelBackgroundFetch object
-// to indicate outstanding operations are in-flight.
-//
-class BGItemHttpHandler : public LLInventoryModel::FetchItemHttpHandler
-{
- LOG_CLASS(BGItemHttpHandler);
-
-public:
- BGItemHttpHandler(const LLSD & request_sd)
- : LLInventoryModel::FetchItemHttpHandler(request_sd)
- {
- LLInventoryModelBackgroundFetch::instance().incrFetchCount(1);
- }
-
- virtual ~BGItemHttpHandler()
- {
- LLInventoryModelBackgroundFetch::instance().incrFetchCount(-1);
- }
-
-protected:
- BGItemHttpHandler(const BGItemHttpHandler &); // Not defined
- void operator=(const BGItemHttpHandler &); // Not defined
-};
-
-
-///----------------------------------------------------------------------------
-/// Class <anonymous>::BGFolderHttpHandler
-///----------------------------------------------------------------------------
-
-// Http request handler class for folders.
-//
-// Handler for FetchInventoryDescendents2 and FetchLibDescendents2
-// caps requests for folders.
-//
-class BGFolderHttpHandler : public LLCore::HttpHandler
-{
- LOG_CLASS(BGFolderHttpHandler);
-
-public:
- BGFolderHttpHandler(const LLSD & request_sd, const uuid_vec_t & recursive_cats)
- : LLCore::HttpHandler(),
- mRequestSD(request_sd),
- mRecursiveCatUUIDs(recursive_cats)
- {
- LLInventoryModelBackgroundFetch::instance().incrFetchCount(1);
- }
-
- virtual ~BGFolderHttpHandler()
- {
- LLInventoryModelBackgroundFetch::instance().incrFetchCount(-1);
- }
-
-protected:
- BGFolderHttpHandler(const BGFolderHttpHandler &); // Not defined
- void operator=(const BGFolderHttpHandler &); // Not defined
-
-public:
- virtual void onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * response);
-
- bool getIsRecursive(const LLUUID & cat_id) const;
-
-private:
- void processData(LLSD & body, LLCore::HttpResponse * response);
- void processFailure(LLCore::HttpStatus status, LLCore::HttpResponse * response);
- void processFailure(const char * const reason, LLCore::HttpResponse * response);
-
-private:
- LLSD mRequestSD;
- const uuid_vec_t mRecursiveCatUUIDs; // hack for storing away which cat fetches are recursive
-};
-
-
-const char * const LOG_INV("Inventory");
-
-} // end of namespace anonymous
-
-
-///----------------------------------------------------------------------------
-/// Class LLInventoryModelBackgroundFetch
-///----------------------------------------------------------------------------
-
-LLInventoryModelBackgroundFetch::LLInventoryModelBackgroundFetch():
- mBackgroundFetchActive(false),
- mFolderFetchActive(false),
- mFetchCount(0),
- mLastFetchCount(0),
- mFetchFolderCount(0),
- mAllRecursiveFoldersFetched(false),
- mRecursiveInventoryFetchStarted(false),
- mRecursiveLibraryFetchStarted(false),
- mRecursiveMarketplaceFetchStarted(false),
- mMinTimeBetweenFetches(0.3f)
-{}
-
-LLInventoryModelBackgroundFetch::~LLInventoryModelBackgroundFetch()
-{
- gIdleCallbacks.deleteFunction(&LLInventoryModelBackgroundFetch::backgroundFetchCB, NULL);
-}
-
-bool LLInventoryModelBackgroundFetch::isBulkFetchProcessingComplete() const
-{
- return mFetchFolderQueue.empty() && mFetchItemQueue.empty() && mFetchCount <= 0;
-}
-
-bool LLInventoryModelBackgroundFetch::isFolderFetchProcessingComplete() const
-{
- return mFetchFolderQueue.empty() && mFetchFolderCount <= 0;
-}
-
-bool LLInventoryModelBackgroundFetch::libraryFetchStarted() const
-{
- return mRecursiveLibraryFetchStarted;
-}
-
-bool LLInventoryModelBackgroundFetch::libraryFetchCompleted() const
-{
- return libraryFetchStarted() && fetchQueueContainsNoDescendentsOf(gInventory.getLibraryRootFolderID());
-}
-
-bool LLInventoryModelBackgroundFetch::libraryFetchInProgress() const
-{
- return libraryFetchStarted() && !libraryFetchCompleted();
-}
-
-bool LLInventoryModelBackgroundFetch::inventoryFetchStarted() const
-{
- return mRecursiveInventoryFetchStarted;
-}
-
-bool LLInventoryModelBackgroundFetch::inventoryFetchCompleted() const
-{
- return inventoryFetchStarted() && fetchQueueContainsNoDescendentsOf(gInventory.getRootFolderID());
-}
-
-bool LLInventoryModelBackgroundFetch::inventoryFetchInProgress() const
-{
- return inventoryFetchStarted() && ! inventoryFetchCompleted();
-}
-
-bool LLInventoryModelBackgroundFetch::isEverythingFetched() const
-{
- return mAllRecursiveFoldersFetched;
-}
-
-bool LLInventoryModelBackgroundFetch::folderFetchActive() const
-{
- return mFolderFetchActive;
-}
-
-void LLInventoryModelBackgroundFetch::addRequestAtFront(const LLUUID & id, bool recursive, bool is_category)
-{
- EFetchType recursion_type = recursive ? FT_RECURSIVE : FT_DEFAULT;
- if (is_category)
- {
- mFetchFolderQueue.push_front(FetchQueueInfo(id, recursion_type, is_category));
- }
- else
- {
- mFetchItemQueue.push_front(FetchQueueInfo(id, recursion_type, is_category));
- }
-}
-
-void LLInventoryModelBackgroundFetch::addRequestAtBack(const LLUUID & id, bool recursive, bool is_category)
-{
- EFetchType recursion_type = recursive ? FT_RECURSIVE : FT_DEFAULT;
- if (is_category)
- {
- mFetchFolderQueue.push_back(FetchQueueInfo(id, recursion_type, is_category));
- }
- else
- {
- mFetchItemQueue.push_back(FetchQueueInfo(id, recursion_type, is_category));
- }
-}
-
-void LLInventoryModelBackgroundFetch::start(const LLUUID& id, bool recursive)
-{
- LLViewerInventoryCategory * cat(gInventory.getCategory(id));
-
- if (cat || (id.isNull() && ! isEverythingFetched()))
- {
- // it's a folder, do a bulk fetch
- LL_DEBUGS(LOG_INV) << "Start fetching category: " << id << ", recursive: " << recursive << LL_ENDL;
-
- mBackgroundFetchActive = true;
- mFolderFetchActive = true;
- EFetchType recursion_type = recursive ? FT_RECURSIVE : FT_DEFAULT;
- if (id.isNull())
- {
- if (! mRecursiveInventoryFetchStarted)
- {
- mRecursiveInventoryFetchStarted |= recursive;
- if (recursive && AISAPI::isAvailable())
- {
- // Not only root folder can be massive, but
- // most system folders will be requested independently
- // so request root folder and content separately
- mFetchFolderQueue.push_front(FetchQueueInfo(gInventory.getRootFolderID(), FT_FOLDER_AND_CONTENT));
- }
- else
- {
- mFetchFolderQueue.push_back(FetchQueueInfo(gInventory.getRootFolderID(), recursion_type));
- }
- gIdleCallbacks.addFunction(&LLInventoryModelBackgroundFetch::backgroundFetchCB, NULL);
- }
- if (! mRecursiveLibraryFetchStarted)
- {
- mRecursiveLibraryFetchStarted |= recursive;
- mFetchFolderQueue.push_back(FetchQueueInfo(gInventory.getLibraryRootFolderID(), recursion_type));
- gIdleCallbacks.addFunction(&LLInventoryModelBackgroundFetch::backgroundFetchCB, NULL);
- }
- }
- else if (recursive && cat && cat->getPreferredType() == LLFolderType::FT_MARKETPLACE_LISTINGS)
- {
- if (mFetchFolderQueue.empty() || mFetchFolderQueue.back().mUUID != id)
- {
- if (recursive && AISAPI::isAvailable())
- {
- // Request marketplace folder and content separately
- mFetchFolderQueue.push_front(FetchQueueInfo(id, FT_FOLDER_AND_CONTENT));
- }
- else
- {
- mFetchFolderQueue.push_front(FetchQueueInfo(id, recursion_type));
- }
- gIdleCallbacks.addFunction(&LLInventoryModelBackgroundFetch::backgroundFetchCB, NULL);
- mRecursiveMarketplaceFetchStarted = true;
- }
- }
- else
- {
- if (AISAPI::isAvailable())
- {
- if (mFetchFolderQueue.empty() || mFetchFolderQueue.back().mUUID != id)
- {
- // On AIS make sure root goes to the top and follow up recursive
- // fetches, not individual requests
- mFetchFolderQueue.push_back(FetchQueueInfo(id, recursion_type));
- gIdleCallbacks.addFunction(&LLInventoryModelBackgroundFetch::backgroundFetchCB, NULL);
- }
- }
- else if (mFetchFolderQueue.empty() || mFetchFolderQueue.front().mUUID != id)
- {
- // Specific folder requests go to front of queue.
- mFetchFolderQueue.push_front(FetchQueueInfo(id, recursion_type));
- gIdleCallbacks.addFunction(&LLInventoryModelBackgroundFetch::backgroundFetchCB, NULL);
- }
-
- if (id == gInventory.getLibraryRootFolderID())
- {
- mRecursiveLibraryFetchStarted |= recursive;
- }
- if (id == gInventory.getRootFolderID())
- {
- mRecursiveInventoryFetchStarted |= recursive;
- }
- }
- }
- else if (LLViewerInventoryItem * itemp = gInventory.getItem(id))
- {
- if (! itemp->mIsComplete)
- {
- scheduleItemFetch(id);
- }
- }
-}
-
-void LLInventoryModelBackgroundFetch::scheduleFolderFetch(const LLUUID& cat_id, bool forced)
-{
- if (mFetchFolderQueue.empty() || mFetchFolderQueue.front().mUUID != cat_id)
- {
- mBackgroundFetchActive = true;
- mFolderFetchActive = true;
-
- if (forced)
- {
- // check if already requested
- if (mForceFetchSet.find(cat_id) == mForceFetchSet.end())
- {
- mForceFetchSet.insert(cat_id);
- mFetchItemQueue.push_front(FetchQueueInfo(cat_id, FT_FORCED));
- }
- }
- else
- {
- // Specific folder requests go to front of queue.
- // version presence acts as dupplicate prevention for normal fetches
- mFetchItemQueue.push_front(FetchQueueInfo(cat_id, FT_DEFAULT));
- }
-
- gIdleCallbacks.addFunction(&LLInventoryModelBackgroundFetch::backgroundFetchCB, NULL);
- }
-}
-
-void LLInventoryModelBackgroundFetch::scheduleItemFetch(const LLUUID& item_id, bool forced)
-{
- if (mFetchItemQueue.empty() || mFetchItemQueue.front().mUUID != item_id)
- {
- mBackgroundFetchActive = true;
- if (forced)
- {
- // check if already requested
- if (mForceFetchSet.find(item_id) == mForceFetchSet.end())
- {
- mForceFetchSet.insert(item_id);
- mFetchItemQueue.push_front(FetchQueueInfo(item_id, FT_FORCED, false));
- }
- }
- else
- {
- // 'isFinished' being set acts as dupplicate prevention for normal fetches
- mFetchItemQueue.push_front(FetchQueueInfo(item_id, FT_DEFAULT, false));
- }
-
- gIdleCallbacks.addFunction(&LLInventoryModelBackgroundFetch::backgroundFetchCB, NULL);
- }
-}
-
-void LLInventoryModelBackgroundFetch::fetchFolderAndLinks(const LLUUID& cat_id, nullary_func_t callback)
-{
- LLViewerInventoryCategory* cat = gInventory.getCategory(cat_id);
- if (cat)
- {
- // Mark folder (update timer) so that background fetch won't request it
- cat->setFetching(LLViewerInventoryCategory::FETCH_RECURSIVE);
- }
- incrFetchFolderCount(1);
- mExpectedFolderIds.push_back(cat_id);
-
- // Assume that we have no relevant cache. Fetch folder, and items folder's links point to.
- AISAPI::FetchCategoryLinks(cat_id,
- [callback, cat_id](const LLUUID& id)
- {
- callback();
- if (id.isNull())
- {
- LL_WARNS() << "Failed to fetch category links " << cat_id << LL_ENDL;
- }
- LLInventoryModelBackgroundFetch::getInstance()->onAISFolderCalback(cat_id, id, FT_DEFAULT);
- });
-
- // start idle loop to track completion
- mBackgroundFetchActive = true;
- mFolderFetchActive = true;
- gIdleCallbacks.addFunction(&LLInventoryModelBackgroundFetch::backgroundFetchCB, NULL);
-}
-
-void LLInventoryModelBackgroundFetch::fetchCOF(nullary_func_t callback)
-{
- LLUUID cat_id = gInventory.findCategoryUUIDForType(LLFolderType::FT_CURRENT_OUTFIT);
- LLViewerInventoryCategory* cat = gInventory.getCategory(cat_id);
- if (cat)
- {
- // Mark cof (update timer) so that background fetch won't request it
- cat->setFetching(LLViewerInventoryCategory::FETCH_RECURSIVE);
- }
- incrFetchFolderCount(1);
- mExpectedFolderIds.push_back(cat_id);
- // For reliability assume that we have no relevant cache, so
- // fetch cof along with items cof's links point to.
- AISAPI::FetchCOF([callback](const LLUUID& id)
- {
- callback();
- LLUUID cat_id = gInventory.findCategoryUUIDForType(LLFolderType::FT_CURRENT_OUTFIT);
- LLInventoryModelBackgroundFetch::getInstance()->onAISFolderCalback(cat_id, id, FT_DEFAULT);
- });
-
- // start idle loop to track completion
- mBackgroundFetchActive = true;
- mFolderFetchActive = true;
- gIdleCallbacks.addFunction(&LLInventoryModelBackgroundFetch::backgroundFetchCB, NULL);
-}
-
-void LLInventoryModelBackgroundFetch::findLostItems()
-{
- mBackgroundFetchActive = true;
- mFolderFetchActive = true;
- mFetchFolderQueue.push_back(FetchQueueInfo(LLUUID::null, FT_RECURSIVE));
- gIdleCallbacks.addFunction(&LLInventoryModelBackgroundFetch::backgroundFetchCB, NULL);
-}
-
-void LLInventoryModelBackgroundFetch::setAllFoldersFetched()
-{
- if (mRecursiveInventoryFetchStarted &&
- mRecursiveLibraryFetchStarted)
- {
- mAllRecursiveFoldersFetched = true;
- //LL_INFOS(LOG_INV) << "All folders fetched, validating" << LL_ENDL;
- //gInventory.validate();
- }
-
- mFolderFetchActive = false;
- if (isBulkFetchProcessingComplete())
- {
- mBackgroundFetchActive = false;
- }
-
- // For now only informs about initial fetch being done
- mFoldersFetchedSignal();
-
- LL_INFOS(LOG_INV) << "Inventory background fetch completed" << LL_ENDL;
-}
-
-boost::signals2::connection LLInventoryModelBackgroundFetch::setFetchCompletionCallback(folders_fetched_callback_t cb)
-{
- return mFoldersFetchedSignal.connect(cb);
-}
-
-void LLInventoryModelBackgroundFetch::backgroundFetchCB(void *)
-{
- LLInventoryModelBackgroundFetch::instance().backgroundFetch();
-}
-
-void LLInventoryModelBackgroundFetch::backgroundFetch()
-{
- if (mBackgroundFetchActive)
- {
- if (AISAPI::isAvailable())
- {
- bulkFetchViaAis();
- }
- else if (gAgent.getRegion() && gAgent.getRegion()->capabilitiesReceived())
- {
- // If we'll be using the capability, we'll be sending batches and the background thing isn't as important.
- bulkFetch();
- }
- }
-}
-
-void LLInventoryModelBackgroundFetch::incrFetchCount(S32 fetching)
-{
- mFetchCount += fetching;
- if (mFetchCount < 0)
- {
- LL_WARNS_ONCE(LOG_INV) << "Inventory fetch count fell below zero (0)." << LL_ENDL;
- mFetchCount = 0;
- }
-}
-void LLInventoryModelBackgroundFetch::incrFetchFolderCount(S32 fetching)
-{
- incrFetchCount(fetching);
- mFetchFolderCount += fetching;
- if (mFetchCount < 0)
- {
- LL_WARNS_ONCE(LOG_INV) << "Inventory fetch count fell below zero (0)." << LL_ENDL;
- mFetchFolderCount = 0;
- }
-}
-
-void ais_simple_item_callback(const LLUUID& inv_id)
-{
- LL_DEBUGS(LOG_INV , "AIS3") << "Response for " << inv_id << LL_ENDL;
- LLInventoryModelBackgroundFetch::instance().incrFetchCount(-1);
-}
-
-void LLInventoryModelBackgroundFetch::onAISContentCalback(
- const LLUUID& request_id,
- const uuid_vec_t& content_ids,
- const LLUUID& response_id,
- EFetchType fetch_type)
-{
- // Don't push_front on failure - there is a chance it was fired from inside bulkFetchViaAis
- incrFetchFolderCount(-1);
-
- uuid_vec_t::const_iterator folder_iter = content_ids.begin();
- uuid_vec_t::const_iterator folder_end = content_ids.end();
- while (folder_iter != folder_end)
- {
- std::list<LLUUID>::const_iterator found = std::find(mExpectedFolderIds.begin(), mExpectedFolderIds.end(), *folder_iter);
- if (found != mExpectedFolderIds.end())
- {
- mExpectedFolderIds.erase(found);
- }
-
- LLViewerInventoryCategory* cat(gInventory.getCategory(*folder_iter));
- if (cat)
- {
- cat->setFetching(LLViewerInventoryCategory::FETCH_NONE);
- }
- if (response_id.isNull())
- {
- // Failed to fetch, get it individually
- mFetchFolderQueue.push_back(FetchQueueInfo(*folder_iter, FT_RECURSIVE));
- }
- else
- {
- // push descendant back to verify they are fetched fully (ex: didn't encounter depth limit)
- LLInventoryModel::cat_array_t* categories(NULL);
- LLInventoryModel::item_array_t* items(NULL);
- gInventory.getDirectDescendentsOf(*folder_iter, categories, items);
- if (categories)
- {
- for (LLInventoryModel::cat_array_t::const_iterator it = categories->begin();
- it != categories->end();
- ++it)
- {
- mFetchFolderQueue.push_back(FetchQueueInfo((*it)->getUUID(), FT_RECURSIVE));
- }
- }
- }
-
- folder_iter++;
- }
-
- if (!mFetchFolderQueue.empty())
- {
- mBackgroundFetchActive = true;
- mFolderFetchActive = true;
- gIdleCallbacks.addFunction(&LLInventoryModelBackgroundFetch::backgroundFetchCB, NULL);
- }
-}
-void LLInventoryModelBackgroundFetch::onAISFolderCalback(const LLUUID &request_id, const LLUUID &response_id, EFetchType fetch_type)
-{
- // Don't push_front on failure - there is a chance it was fired from inside bulkFetchViaAis
- incrFetchFolderCount(-1);
- std::list<LLUUID>::const_iterator found = std::find(mExpectedFolderIds.begin(), mExpectedFolderIds.end(), request_id);
- if (found != mExpectedFolderIds.end())
- {
- mExpectedFolderIds.erase(found);
- }
- else
- {
- // ais shouldn't respond twice
- llassert(false);
- LL_WARNS() << "Unexpected folder response for " << request_id << LL_ENDL;
- }
-
- if (request_id.isNull())
- {
- // orhans, no other actions needed
- return;
- }
-
- LLViewerInventoryCategory::EFetchType new_state = LLViewerInventoryCategory::FETCH_NONE;
- bool request_descendants = false;
- if (response_id.isNull()) // Failure
- {
- LL_DEBUGS(LOG_INV , "AIS3") << "Failure response for folder " << request_id << LL_ENDL;
- if (fetch_type == FT_RECURSIVE)
- {
- // A full recursive request failed.
- // Try requesting folder and nested content separately
- mFetchFolderQueue.push_back(FetchQueueInfo(request_id, FT_FOLDER_AND_CONTENT));
- }
- else if (fetch_type == FT_FOLDER_AND_CONTENT)
- {
- LL_WARNS() << "Failed to download folder: " << request_id << " Requesting known content separately" << LL_ENDL;
- mFetchFolderQueue.push_back(FetchQueueInfo(request_id, FT_CONTENT_RECURSIVE));
-
- // set folder's version to prevent viewer from trying to request folder indefinetely
- LLViewerInventoryCategory* cat(gInventory.getCategory(request_id));
- if (cat && cat->getVersion() == LLViewerInventoryCategory::VERSION_UNKNOWN)
- {
- cat->setVersion(0);
- }
- // back off for a bit in case something tries to force-request immediately
- new_state = LLViewerInventoryCategory::FETCH_FAILED;
- }
- }
- else
- {
- if (fetch_type == FT_RECURSIVE)
- {
- // Got the folder and content, now verify content
- // Request content even for FT_RECURSIVE in case of changes, failures
- // or if depth limit gets imlemented.
- // This shouldn't redownload folders if they already have version
- request_descendants = true;
- LL_DEBUGS(LOG_INV, "AIS3") << "Got folder " << request_id << ". Requesting content" << LL_ENDL;
- }
- else if (fetch_type == FT_FOLDER_AND_CONTENT)
- {
- // readd folder for content request
- mFetchFolderQueue.push_front(FetchQueueInfo(request_id, FT_CONTENT_RECURSIVE));
- }
- else
- {
- LL_DEBUGS(LOG_INV, "AIS3") << "Got folder " << request_id << "." << LL_ENDL;
- }
-
- }
-
- if (request_descendants)
- {
- LLInventoryModel::cat_array_t* categories(NULL);
- LLInventoryModel::item_array_t* items(NULL);
- gInventory.getDirectDescendentsOf(request_id, categories, items);
- if (categories)
- {
- for (LLInventoryModel::cat_array_t::const_iterator it = categories->begin();
- it != categories->end();
- ++it)
- {
- mFetchFolderQueue.push_back(FetchQueueInfo((*it)->getUUID(), FT_RECURSIVE));
- }
- }
- }
-
- if (!mFetchFolderQueue.empty())
- {
- mBackgroundFetchActive = true;
- mFolderFetchActive = true;
- gIdleCallbacks.addFunction(&LLInventoryModelBackgroundFetch::backgroundFetchCB, NULL);
- }
-
- // done
- LLViewerInventoryCategory * cat(gInventory.getCategory(request_id));
- if (cat)
- {
- cat->setFetching(new_state);
- }
-}
-
-static LLTrace::BlockTimerStatHandle FTM_BULK_FETCH("Bulk Fetch");
-
-void LLInventoryModelBackgroundFetch::bulkFetchViaAis()
-{
- LL_RECORD_BLOCK_TIME(FTM_BULK_FETCH);
- //Background fetch is called from gIdleCallbacks in a loop until background fetch is stopped.
- if (gDisconnected)
- {
- return;
- }
-
- static LLCachedControl<U32> ais_pool(gSavedSettings, "PoolSizeAIS", 20);
- // Don't have too many requests at once, AIS throttles
- // Reserve one request for actions outside of fetch (like renames)
- const U32 max_concurrent_fetches = llclamp(ais_pool - 1, 1, 50);
-
- if (mFetchCount >= max_concurrent_fetches)
- {
- return;
- }
-
- // Don't loop for too long (in case of large, fully loaded inventory)
- F64 curent_time = LLTimer::getTotalSeconds();
- const F64 max_time = LLStartUp::getStartupState() > STATE_WEARABLES_WAIT
- ? 0.006f // 6 ms
- : 1.f;
- const F64 end_time = curent_time + max_time;
- S32 last_fetch_count = mFetchCount;
-
- while (!mFetchFolderQueue.empty() && mFetchCount < max_concurrent_fetches && curent_time < end_time)
- {
- const FetchQueueInfo & fetch_info(mFetchFolderQueue.front());
- bulkFetchViaAis(fetch_info);
- mFetchFolderQueue.pop_front();
- curent_time = LLTimer::getTotalSeconds();
- }
-
- // Ideally we shouldn't fetch items if recursive fetch isn't done,
- // but there is a chance some request will start timeouting and recursive
- // fetch will get stuck on a signle folder, don't block item fetch in such case
- while (!mFetchItemQueue.empty() && mFetchCount < max_concurrent_fetches && curent_time < end_time)
- {
- const FetchQueueInfo& fetch_info(mFetchItemQueue.front());
- bulkFetchViaAis(fetch_info);
- mFetchItemQueue.pop_front();
- curent_time = LLTimer::getTotalSeconds();
- }
-
- if (last_fetch_count != mFetchCount // if anything was added
- || mLastFetchCount != mFetchCount) // if anything was substracted
- {
- LL_DEBUGS(LOG_INV , "AIS3") << "Total active fetches: " << mLastFetchCount << "->" << last_fetch_count << "->" << mFetchCount
- << ", scheduled folder fetches: " << (S32)mFetchFolderQueue.size()
- << ", scheduled item fetches: " << (S32)mFetchItemQueue.size()
- << LL_ENDL;
- mLastFetchCount = mFetchCount;
-
- if (!mExpectedFolderIds.empty())
- {
- // A folder seem to be stack fetching on QA account, print oldest folder out
- LL_DEBUGS(LOG_INV , "AIS3") << "Oldest expected folder: ";
- std::list<LLUUID>::const_iterator iter = mExpectedFolderIds.begin();
- LL_CONT << *iter;
- if ((*iter).notNull())
- {
- LLViewerInventoryCategory* cat(gInventory.getCategory(*iter));
- if (cat)
- {
- LL_CONT << " Folder name: " << cat->getName() << " Parent: " << cat->getParentUUID();
- }
- else
- {
- LL_CONT << " This folder doesn't exist";
- }
- }
- else
- {
- LL_CONT << " Orphans request";
- }
- LL_CONT << LL_ENDL;
- }
- }
-
- if (isFolderFetchProcessingComplete() && mFolderFetchActive)
- {
- if (!mRecursiveInventoryFetchStarted || mRecursiveMarketplaceFetchStarted)
- {
- setAllFoldersFetched();
- }
- else
- {
- // Intent is for marketplace request to happen after
- // main inventory is done, unless requested by floater
- mRecursiveMarketplaceFetchStarted = true;
- const LLUUID& marketplacelistings_id = gInventory.findCategoryUUIDForType(LLFolderType::FT_MARKETPLACE_LISTINGS);
- if (marketplacelistings_id.notNull())
- {
- mFetchFolderQueue.push_front(FetchQueueInfo(marketplacelistings_id, FT_FOLDER_AND_CONTENT));
- }
- else
- {
- setAllFoldersFetched();
- }
- }
-
- }
-
- if (isBulkFetchProcessingComplete())
- {
- mBackgroundFetchActive = false;
- }
-}
-
-void LLInventoryModelBackgroundFetch::bulkFetchViaAis(const FetchQueueInfo& fetch_info)
-{
- if (fetch_info.mIsCategory)
- {
- const LLUUID & cat_id(fetch_info.mUUID);
- if (cat_id.isNull())
- {
- incrFetchFolderCount(1);
- mExpectedFolderIds.push_back(cat_id);
- // Lost and found
- // Should it actually be recursive?
- AISAPI::FetchOrphans([](const LLUUID& response_id)
- {
- LLInventoryModelBackgroundFetch::instance().onAISFolderCalback(LLUUID::null,
- response_id,
- FT_DEFAULT);
- });
- }
- else
- {
- LLViewerInventoryCategory * cat(gInventory.getCategory(cat_id));
- if (cat)
- {
- if (fetch_info.mFetchType == FT_CONTENT_RECURSIVE)
- {
- // fetch content only, ignore cat itself
- uuid_vec_t children;
- LLInventoryModel::cat_array_t* categories(NULL);
- LLInventoryModel::item_array_t* items(NULL);
- gInventory.getDirectDescendentsOf(cat_id, categories, items);
-
- LLViewerInventoryCategory::EFetchType target_state = LLViewerInventoryCategory::FETCH_RECURSIVE;
- bool content_done = true;
-
- // Top limit is 'as many as you can put into url'
- static LLCachedControl<S32> ais_batch(gSavedSettings, "BatchSizeAIS3", 20);
- S32 batch_limit = llclamp(ais_batch(), 1, 40);
-
- for (LLInventoryModel::cat_array_t::iterator it = categories->begin();
- it != categories->end();
- ++it)
- {
- LLViewerInventoryCategory* child_cat = (*it);
- if (LLViewerInventoryCategory::VERSION_UNKNOWN != child_cat->getVersion()
- || child_cat->getFetching() >= target_state)
- {
- continue;
- }
-
- if (child_cat->getPreferredType() == LLFolderType::FT_MARKETPLACE_LISTINGS)
- {
- // special case, marketplace will fetch that as needed
- continue;
- }
-
- children.push_back(child_cat->getUUID());
- mExpectedFolderIds.push_back(child_cat->getUUID());
- child_cat->setFetching(target_state);
-
- if (children.size() >= batch_limit)
- {
- content_done = false;
- break;
- }
- }
-
- if (!children.empty())
- {
- // increment before call in case of immediate callback
- incrFetchFolderCount(1);
-
- EFetchType type = fetch_info.mFetchType;
- LLUUID cat_id = cat->getUUID(); // need a copy for lambda
- AISAPI::completion_t cb = [cat_id, children, type](const LLUUID& response_id)
- {
- LLInventoryModelBackgroundFetch::instance().onAISContentCalback(cat_id, children, response_id, type);
- };
-
- AISAPI::ITEM_TYPE item_type = AISAPI::INVENTORY;
- if (ALEXANDRIA_LINDEN_ID == cat->getOwnerID())
- {
- item_type = AISAPI::LIBRARY;
- }
-
- AISAPI::FetchCategorySubset(cat_id, children, item_type, true, cb, 0);
- }
-
- if (content_done)
- {
- // This will have a bit of overlap with onAISContentCalback,
- // but something else might have dowloaded folders, so verify
- // every child that is complete has it's children done as well
- for (LLInventoryModel::cat_array_t::iterator it = categories->begin();
- it != categories->end();
- ++it)
- {
- LLViewerInventoryCategory* child_cat = (*it);
- if (LLViewerInventoryCategory::VERSION_UNKNOWN != child_cat->getVersion())
- {
- mFetchFolderQueue.push_back(FetchQueueInfo(child_cat->getUUID(), FT_RECURSIVE));
- }
- }
- }
- else
- {
- // send it back to get the rest
- mFetchFolderQueue.push_back(FetchQueueInfo(cat_id, FT_CONTENT_RECURSIVE));
- }
- }
- else if (LLViewerInventoryCategory::VERSION_UNKNOWN == cat->getVersion()
- || fetch_info.mFetchType == FT_FORCED)
- {
- LLViewerInventoryCategory::EFetchType target_state =
- fetch_info.mFetchType > FT_CONTENT_RECURSIVE
- ? LLViewerInventoryCategory::FETCH_RECURSIVE
- : LLViewerInventoryCategory::FETCH_NORMAL;
- // start again if we did a non-recursive fetch before
- // to get all children in a single request
- if (cat->getFetching() < target_state)
- {
- // increment before call in case of immediate callback
- incrFetchFolderCount(1);
- cat->setFetching(target_state);
- mExpectedFolderIds.push_back(cat_id);
-
- EFetchType type = fetch_info.mFetchType;
- LLUUID cat_cb_id = cat_id;
- AISAPI::completion_t cb = [cat_cb_id, type](const LLUUID& response_id)
- {
- LLInventoryModelBackgroundFetch::instance().onAISFolderCalback(cat_cb_id, response_id , type);
- };
-
- AISAPI::ITEM_TYPE item_type = AISAPI::INVENTORY;
- if (ALEXANDRIA_LINDEN_ID == cat->getOwnerID())
- {
- item_type = AISAPI::LIBRARY;
- }
-
- AISAPI::FetchCategoryChildren(cat_id , item_type , type == FT_RECURSIVE , cb, 0);
- }
- }
- else
- {
- // Already fetched, check if anything inside needs fetching
- if (fetch_info.mFetchType == FT_RECURSIVE
- || fetch_info.mFetchType == FT_FOLDER_AND_CONTENT)
- {
- LLInventoryModel::cat_array_t * categories(NULL);
- LLInventoryModel::item_array_t * items(NULL);
- gInventory.getDirectDescendentsOf(cat_id, categories, items);
- for (LLInventoryModel::cat_array_t::const_iterator it = categories->begin();
- it != categories->end();
- ++it)
- {
- // not push_front to not cause an infinite loop
- mFetchFolderQueue.push_back(FetchQueueInfo((*it)->getUUID(), FT_RECURSIVE));
- }
- }
- }
- } // else try to fetch folder either way?
- }
- }
- else
- {
- LLViewerInventoryItem * itemp(gInventory.getItem(fetch_info.mUUID));
-
- if (itemp)
- {
- if (!itemp->isFinished() || fetch_info.mFetchType == FT_FORCED)
- {
- mFetchCount++;
- if (itemp->getPermissions().getOwner() == gAgent.getID())
- {
- AISAPI::FetchItem(fetch_info.mUUID, AISAPI::INVENTORY, ais_simple_item_callback);
- }
- else
- {
- AISAPI::FetchItem(fetch_info.mUUID, AISAPI::LIBRARY, ais_simple_item_callback);
- }
- }
- }
- else // We don't know it, assume incomplete
- {
- // Assume agent's inventory, library wouldn't have gotten here
- mFetchCount++;
- AISAPI::FetchItem(fetch_info.mUUID, AISAPI::INVENTORY, ais_simple_item_callback);
- }
- }
-
- if (fetch_info.mFetchType == FT_FORCED)
- {
- mForceFetchSet.erase(fetch_info.mUUID);
- }
-}
-
-// Bundle up a bunch of requests to send all at once.
-void LLInventoryModelBackgroundFetch::bulkFetch()
-{
- LL_RECORD_BLOCK_TIME(FTM_BULK_FETCH);
- //Background fetch is called from gIdleCallbacks in a loop until background fetch is stopped.
- //If there are items in mFetchQueue, we want to check the time since the last bulkFetch was
- //sent. If it exceeds our retry time, go ahead and fire off another batch.
- LLViewerRegion * region(gAgent.getRegion());
- if (! region || gDisconnected || LLApp::isExiting())
- {
- return;
- }
-
- // *TODO: These values could be tweaked at runtime to effect
- // a fast/slow fetch throttle. Once login is complete and the scene
- // is mostly loaded, we could turn up the throttle and fill missing
- // inventory more quickly.
- static const U32 max_batch_size(10);
- static const S32 max_concurrent_fetches(12); // Outstanding requests, not connections
-
- if (mFetchCount)
- {
- // Process completed background HTTP requests
- gInventory.handleResponses(false);
- // Just processed a bunch of items.
- // Note: do we really need notifyObservers() here?
- // OnIdle it will be called anyway due to Add flag for processed item.
- // It seems like in some cases we are updaiting on fail (no flag),
- // but is there anything to update?
- gInventory.notifyObservers();
- }
-
- if (mFetchCount > max_concurrent_fetches)
- {
- return;
- }
-
- U32 item_count(0);
- U32 folder_count(0);
-
- const U32 sort_order(gSavedSettings.getU32(LLInventoryPanel::DEFAULT_SORT_ORDER) & 0x1);
-
- // *TODO: Think I'd like to get a shared pointer to this and share it
- // among all the folder requests.
- uuid_vec_t recursive_cats;
- uuid_vec_t all_cats; // dupplicate avoidance
-
- LLSD folder_request_body;
- LLSD folder_request_body_lib;
- LLSD item_request_body;
- LLSD item_request_body_lib;
-
- while (! mFetchFolderQueue.empty()
- && (item_count + folder_count) < max_batch_size)
- {
- const FetchQueueInfo & fetch_info(mFetchFolderQueue.front());
- if (fetch_info.mIsCategory)
- {
- const LLUUID & cat_id(fetch_info.mUUID);
- if (cat_id.isNull()) //DEV-17797 Lost and found
- {
- LLSD folder_sd;
- folder_sd["folder_id"] = LLUUID::null.asString();
- folder_sd["owner_id"] = gAgent.getID();
- folder_sd["sort_order"] = LLSD::Integer(sort_order);
- folder_sd["fetch_folders"] = LLSD::Boolean(false);
- folder_sd["fetch_items"] = LLSD::Boolean(true);
- folder_request_body["folders"].append(folder_sd);
- folder_count++;
- }
- else
- {
- const LLViewerInventoryCategory * cat(gInventory.getCategory(cat_id));
- if (cat)
- {
- if (LLViewerInventoryCategory::VERSION_UNKNOWN == cat->getVersion())
- {
- if (std::find(all_cats.begin(), all_cats.end(), cat_id) == all_cats.end())
- {
- LLSD folder_sd;
- folder_sd["folder_id"] = cat->getUUID();
- folder_sd["owner_id"] = cat->getOwnerID();
- folder_sd["sort_order"] = LLSD::Integer(sort_order);
- folder_sd["fetch_folders"] = LLSD::Boolean(true); //(LLSD::Boolean)sFullFetchStarted;
- folder_sd["fetch_items"] = LLSD::Boolean(true);
-
- if (ALEXANDRIA_LINDEN_ID == cat->getOwnerID())
- {
- folder_request_body_lib["folders"].append(folder_sd);
- }
- else
- {
- folder_request_body["folders"].append(folder_sd);
- }
- folder_count++;
- }
- }
- else
- {
- // May already have this folder, but append child folders to list.
- if (fetch_info.mFetchType >= FT_CONTENT_RECURSIVE)
- {
- LLInventoryModel::cat_array_t * categories(NULL);
- LLInventoryModel::item_array_t * items(NULL);
- gInventory.getDirectDescendentsOf(cat_id, categories, items);
- for (LLInventoryModel::cat_array_t::const_iterator it = categories->begin();
- it != categories->end();
- ++it)
- {
- mFetchFolderQueue.push_back(FetchQueueInfo((*it)->getUUID(), fetch_info.mFetchType));
- }
- }
- }
- }
- }
- if (fetch_info.mFetchType >= FT_CONTENT_RECURSIVE)
- {
- recursive_cats.push_back(cat_id);
- }
- all_cats.push_back(cat_id);
- }
-
- mFetchFolderQueue.pop_front();
- }
-
-
- while (!mFetchItemQueue.empty()
- && (item_count + folder_count) < max_batch_size)
- {
- const FetchQueueInfo & fetch_info(mFetchItemQueue.front());
-
- LLViewerInventoryItem * itemp(gInventory.getItem(fetch_info.mUUID));
-
- if (itemp)
- {
- LLSD item_sd;
- item_sd["owner_id"] = itemp->getPermissions().getOwner();
- item_sd["item_id"] = itemp->getUUID();
- if (itemp->getPermissions().getOwner() == gAgent.getID())
- {
- item_request_body.append(item_sd);
- }
- else
- {
- item_request_body_lib.append(item_sd);
- }
- //itemp->fetchFromServer();
- item_count++;
- }
-
- mFetchItemQueue.pop_front();
- }
-
- // Issue HTTP POST requests to fetch folders and items
-
- if (item_count + folder_count > 0)
- {
- if (folder_count)
- {
- if (folder_request_body["folders"].size())
- {
- const std::string url(region->getCapability("FetchInventoryDescendents2"));
-
- if (! url.empty())
- {
- LLCore::HttpHandler::ptr_t handler(new BGFolderHttpHandler(folder_request_body, recursive_cats));
- gInventory.requestPost(false, url, folder_request_body, handler, "Inventory Folder");
- }
- }
-
- if (folder_request_body_lib["folders"].size())
- {
- const std::string url(region->getCapability("FetchLibDescendents2"));
-
- if (! url.empty())
- {
- LLCore::HttpHandler::ptr_t handler(new BGFolderHttpHandler(folder_request_body_lib, recursive_cats));
- gInventory.requestPost(false, url, folder_request_body_lib, handler, "Library Folder");
- }
- }
- } // if (folder_count)
-
- if (item_count)
- {
- if (item_request_body.size())
- {
- const std::string url(region->getCapability("FetchInventory2"));
-
- if (! url.empty())
- {
- LLSD body;
- body["items"] = item_request_body;
- LLCore::HttpHandler::ptr_t handler(new BGItemHttpHandler(body));
- gInventory.requestPost(false, url, body, handler, "Inventory Item");
- }
- }
-
- if (item_request_body_lib.size())
- {
- const std::string url(region->getCapability("FetchLib2"));
-
- if (! url.empty())
- {
- LLSD body;
- body["items"] = item_request_body_lib;
- LLCore::HttpHandler::ptr_t handler(new BGItemHttpHandler(body));
- gInventory.requestPost(false, url, body, handler, "Library Item");
- }
- }
- } // if (item_count)
-
- mFetchTimer.reset();
- }
- else if (isBulkFetchProcessingComplete())
- {
- setAllFoldersFetched();
- }
-}
-
-bool LLInventoryModelBackgroundFetch::fetchQueueContainsNoDescendentsOf(const LLUUID & cat_id) const
-{
- for (fetch_queue_t::const_iterator it = mFetchFolderQueue.begin();
- it != mFetchFolderQueue.end();
- ++it)
- {
- const LLUUID & fetch_id = (*it).mUUID;
- if (gInventory.isObjectDescendentOf(fetch_id, cat_id))
- return false;
- }
- for (fetch_queue_t::const_iterator it = mFetchItemQueue.begin();
- it != mFetchItemQueue.end();
- ++it)
- {
- const LLUUID & fetch_id = (*it).mUUID;
- if (gInventory.isObjectDescendentOf(fetch_id, cat_id))
- return false;
- }
- return true;
-}
-
-
-namespace
-{
-
-///----------------------------------------------------------------------------
-/// Class <anonymous>::BGFolderHttpHandler
-///----------------------------------------------------------------------------
-
-void BGFolderHttpHandler::onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * response)
-{
- do // Single-pass do-while used for common exit handling
- {
- LLCore::HttpStatus status(response->getStatus());
- // status = LLCore::HttpStatus(404); // Dev tool to force error handling
- if (! status)
- {
- processFailure(status, response);
- break; // Goto common exit
- }
-
- // Response body should be present.
- LLCore::BufferArray * body(response->getBody());
- // body = NULL; // Dev tool to force error handling
- if (! body || ! body->size())
- {
- LL_WARNS(LOG_INV) << "Missing data in inventory folder query." << LL_ENDL;
- processFailure("HTTP response missing expected body", response);
- break; // Goto common exit
- }
-
- // Could test 'Content-Type' header but probably unreliable.
-
- // Convert response to LLSD
- // body->write(0, "Garbage Response", 16); // Dev tool to force error handling
- LLSD body_llsd;
- if (! LLCoreHttpUtil::responseToLLSD(response, true, body_llsd))
- {
- // INFOS-level logging will occur on the parsed failure
- processFailure("HTTP response contained malformed LLSD", response);
- break; // goto common exit
- }
-
- // Expect top-level structure to be a map
- // body_llsd = LLSD::emptyArray(); // Dev tool to force error handling
- if (! body_llsd.isMap())
- {
- processFailure("LLSD response not a map", response);
- break; // goto common exit
- }
-
- // Check for 200-with-error failures
- //
- // See comments in llinventorymodel.cpp about this mode of error.
- //
- // body_llsd["error"] = LLSD::emptyMap(); // Dev tool to force error handling
- // body_llsd["error"]["identifier"] = "Development";
- // body_llsd["error"]["message"] = "You left development code in the viewer";
- if (body_llsd.has("error"))
- {
- processFailure("Inventory application error (200-with-error)", response);
- break; // goto common exit
- }
-
- // Okay, process data if possible
- processData(body_llsd, response);
- }
- while (false);
-}
-
-
-void BGFolderHttpHandler::processData(LLSD & content, LLCore::HttpResponse * response)
-{
- LLInventoryModelBackgroundFetch * fetcher(LLInventoryModelBackgroundFetch::getInstance());
-
- // API V2 and earlier should probably be testing for "error" map
- // in response as an application-level error.
-
- // Instead, we assume success and attempt to extract information.
- if (content.has("folders"))
- {
- LLSD folders(content["folders"]);
-
- for (LLSD::array_const_iterator folder_it = folders.beginArray();
- folder_it != folders.endArray();
- ++folder_it)
- {
- LLSD folder_sd(*folder_it);
-
- //LLUUID agent_id = folder_sd["agent_id"];
-
- //if(agent_id != gAgent.getID()) //This should never happen.
- //{
- // LL_WARNS(LOG_INV) << "Got a UpdateInventoryItem for the wrong agent."
- // << LL_ENDL;
- // break;
- //}
-
- LLUUID parent_id(folder_sd["folder_id"].asUUID());
- LLUUID owner_id(folder_sd["owner_id"].asUUID());
- S32 version(folder_sd["version"].asInteger());
- S32 descendents(folder_sd["descendents"].asInteger());
- LLPointer<LLViewerInventoryCategory> tcategory = new LLViewerInventoryCategory(owner_id);
-
- if (parent_id.isNull())
- {
- LLSD items(folder_sd["items"]);
- LLPointer<LLViewerInventoryItem> titem = new LLViewerInventoryItem;
-
- for (LLSD::array_const_iterator item_it = items.beginArray();
- item_it != items.endArray();
- ++item_it)
- {
- const LLUUID lost_uuid(gInventory.findCategoryUUIDForType(LLFolderType::FT_LOST_AND_FOUND));
-
- if (lost_uuid.notNull())
- {
- LLSD item(*item_it);
-
- titem->unpackMessage(item);
-
- LLInventoryModel::update_list_t update;
- LLInventoryModel::LLCategoryUpdate new_folder(lost_uuid, 1);
- update.push_back(new_folder);
- gInventory.accountForUpdate(update);
-
- titem->setParent(lost_uuid);
- titem->updateParentOnServer(false);
- gInventory.updateItem(titem);
- }
- }
- }
-
- LLViewerInventoryCategory * pcat(gInventory.getCategory(parent_id));
- if (! pcat)
- {
- continue;
- }
-
- LLSD categories(folder_sd["categories"]);
- for (LLSD::array_const_iterator category_it = categories.beginArray();
- category_it != categories.endArray();
- ++category_it)
- {
- LLSD category(*category_it);
- tcategory->fromLLSD(category);
-
- const bool recursive(getIsRecursive(tcategory->getUUID()));
- if (recursive)
- {
- fetcher->addRequestAtBack(tcategory->getUUID(), recursive, true);
- }
- else if (! gInventory.isCategoryComplete(tcategory->getUUID()))
- {
- gInventory.updateCategory(tcategory);
- }
- }
-
- LLSD items(folder_sd["items"]);
- LLPointer<LLViewerInventoryItem> titem = new LLViewerInventoryItem;
- for (LLSD::array_const_iterator item_it = items.beginArray();
- item_it != items.endArray();
- ++item_it)
- {
- LLSD item(*item_it);
- titem->unpackMessage(item);
-
- gInventory.updateItem(titem);
- }
-
- // Set version and descendentcount according to message.
- LLViewerInventoryCategory * cat(gInventory.getCategory(parent_id));
- if (cat)
- {
- cat->setVersion(version);
- cat->setDescendentCount(descendents);
- cat->determineFolderType();
- }
- }
- }
-
- if (content.has("bad_folders"))
- {
- LLSD bad_folders(content["bad_folders"]);
- for (LLSD::array_const_iterator folder_it = bad_folders.beginArray();
- folder_it != bad_folders.endArray();
- ++folder_it)
- {
- // *TODO: Stop copying data [ed: this isn't copying data]
- LLSD folder_sd(*folder_it);
-
- // These folders failed on the dataserver. We probably don't want to retry them.
- LL_WARNS(LOG_INV) << "Folder " << folder_sd["folder_id"].asString()
- << "Error: " << folder_sd["error"].asString() << LL_ENDL;
- }
- }
-
- if (fetcher->isBulkFetchProcessingComplete())
- {
- fetcher->setAllFoldersFetched();
- }
-}
-
-
-void BGFolderHttpHandler::processFailure(LLCore::HttpStatus status, LLCore::HttpResponse * response)
-{
- const std::string & ct(response->getContentType());
- LL_WARNS(LOG_INV) << "Inventory folder fetch failure\n"
- << "[Status: " << status.toTerseString() << "]\n"
- << "[Reason: " << status.toString() << "]\n"
- << "[Content-type: " << ct << "]\n"
- << "[Content (abridged): "
- << LLCoreHttpUtil::responseToString(response) << "]" << LL_ENDL;
-
- // Could use a 404 test here to try to detect revoked caps...
-
- if(status == LLCore::HttpStatus(HTTP_FORBIDDEN))
- {
- // Too large, split into two if possible
- if (gDisconnected || LLApp::isExiting())
- {
- return;
- }
-
- const std::string url(gAgent.getRegionCapability("FetchInventoryDescendents2"));
- if (url.empty())
- {
- LL_WARNS(LOG_INV) << "Failed to get AIS2 cap" << LL_ENDL;
- return;
- }
-
- S32 size = mRequestSD["folders"].size();
-
- if (size > 1)
- {
- // Can split, assume that this isn't the library
- LLSD folders;
- uuid_vec_t recursive_cats;
- LLSD::array_iterator iter = mRequestSD["folders"].beginArray();
- LLSD::array_iterator end = mRequestSD["folders"].endArray();
- while (iter != end)
- {
- folders.append(*iter);
- LLUUID folder_id = iter->get("folder_id").asUUID();
- if (std::find(mRecursiveCatUUIDs.begin(), mRecursiveCatUUIDs.end(), folder_id) != mRecursiveCatUUIDs.end())
- {
- recursive_cats.push_back(folder_id);
- }
- if (folders.size() == (S32)(size / 2))
- {
- LLSD request_body;
- request_body["folders"] = folders;
- LLCore::HttpHandler::ptr_t handler(new BGFolderHttpHandler(request_body, recursive_cats));
- gInventory.requestPost(false, url, request_body, handler, "Inventory Folder");
- recursive_cats.clear();
- folders.clear();
- }
- iter++;
- }
-
- LLSD request_body;
- request_body["folders"] = folders;
- LLCore::HttpHandler::ptr_t handler(new BGFolderHttpHandler(request_body, recursive_cats));
- gInventory.requestPost(false, url, request_body, handler, "Inventory Folder");
- return;
- }
- else
- {
- // Can't split
- LLNotificationsUtil::add("InventoryLimitReachedAIS");
- }
- }
-
- // This was originally the request retry logic for the inventory
- // request which tested on HTTP_INTERNAL_ERROR status. This
- // retry logic was unbounded and lacked discrimination as to the
- // cause of the retry. The new http library should be doing
- // adquately on retries but I want to keep the structure of a
- // retry for reference.
- LLInventoryModelBackgroundFetch *fetcher = LLInventoryModelBackgroundFetch::getInstance();
- if (false)
- {
- // timed out or curl failure
- for (LLSD::array_const_iterator folder_it = mRequestSD["folders"].beginArray();
- folder_it != mRequestSD["folders"].endArray();
- ++folder_it)
- {
- LLSD folder_sd(*folder_it);
- LLUUID folder_id(folder_sd["folder_id"].asUUID());
- const bool recursive = getIsRecursive(folder_id);
- fetcher->addRequestAtFront(folder_id, recursive, true);
- }
- }
- else
- {
- if (fetcher->isBulkFetchProcessingComplete())
- {
- fetcher->setAllFoldersFetched();
- }
- }
-}
-
-
-void BGFolderHttpHandler::processFailure(const char * const reason, LLCore::HttpResponse * response)
-{
- LL_WARNS(LOG_INV) << "Inventory folder fetch failure\n"
- << "[Status: internal error]\n"
- << "[Reason: " << reason << "]\n"
- << "[Content (abridged): "
- << LLCoreHttpUtil::responseToString(response) << "]" << LL_ENDL;
-
- // Reverse of previous processFailure() method, this is invoked
- // when response structure is found to be invalid. Original
- // always re-issued the request (without limit). This does
- // the same but be aware that this may be a source of problems.
- // Philosophy is that inventory folders are so essential to
- // operation that this is a reasonable action.
- LLInventoryModelBackgroundFetch *fetcher = LLInventoryModelBackgroundFetch::getInstance();
- if (true)
- {
- for (LLSD::array_const_iterator folder_it = mRequestSD["folders"].beginArray();
- folder_it != mRequestSD["folders"].endArray();
- ++folder_it)
- {
- LLSD folder_sd(*folder_it);
- LLUUID folder_id(folder_sd["folder_id"].asUUID());
- const bool recursive = getIsRecursive(folder_id);
- fetcher->addRequestAtFront(folder_id, recursive, true);
- }
- }
- else
- {
- if (fetcher->isBulkFetchProcessingComplete())
- {
- fetcher->setAllFoldersFetched();
- }
- }
-}
-
-
-bool BGFolderHttpHandler::getIsRecursive(const LLUUID & cat_id) const
-{
- return std::find(mRecursiveCatUUIDs.begin(), mRecursiveCatUUIDs.end(), cat_id) != mRecursiveCatUUIDs.end();
-}
-
-///----------------------------------------------------------------------------
-/// Class <anonymous>::BGItemHttpHandler
-///----------------------------------------------------------------------------
-
-// Nothing to implement here. All ctor/dtor changes.
-
-} // end namespace anonymous
+/**
+ * @file llinventorymodelbackgroundfetch.cpp
+ * @brief Implementation of background fetching of inventory.
+ *
+ * $LicenseInfo:firstyear=2002&license=viewerlgpl$
+ * Second Life Viewer Source Code
+ * Copyright (C) 2014, Linden Research, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation;
+ * version 2.1 of the License only.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ *
+ * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA
+ * $/LicenseInfo$
+ */
+
+#include "llviewerprecompiledheaders.h"
+#include "llinventorymodelbackgroundfetch.h"
+
+#include "llaisapi.h"
+#include "llagent.h"
+#include "llappviewer.h"
+#include "llcallbacklist.h"
+#include "llinventorymodel.h"
+#include "llinventorypanel.h"
+#include "llnotificationsutil.h"
+#include "llstartup.h"
+#include "llviewercontrol.h"
+#include "llviewerinventory.h"
+#include "llviewermessage.h"
+#include "llviewerregion.h"
+#include "llviewerwindow.h"
+#include "llhttpconstants.h"
+#include "bufferarray.h"
+#include "bufferstream.h"
+#include "llcorehttputil.h"
+
+// History (may be apocryphal)
+//
+// Around V2, an HTTP inventory download mechanism was added
+// along with inventory LINK items referencing other inventory
+// items. As part of this, at login, the entire inventory
+// structure is downloaded 'in the background' using the
+// backgroundFetch()/bulkFetch() methods. The UDP path can
+// still be used and is found in the 'DEPRECATED OLD CODE'
+// section.
+//
+// The old UDP path implemented a throttle that adapted
+// itself during running. The mechanism survived info HTTP
+// somewhat but was pinned to poll the HTTP plumbing at
+// 0.5S intervals. The reasons for this particular value
+// have been lost. It's possible to switch between UDP
+// and HTTP while this is happening but there may be
+// surprises in what happens in that case.
+//
+// Conversion to llcorehttp reduced the number of connections
+// used but batches more data and queues more requests (but
+// doesn't due pipelining due to libcurl restrictions). The
+// poll interval above was re-examined and reduced to get
+// inventory into the viewer more quickly.
+//
+// Possible future work:
+//
+// * Don't download the entire heirarchy in one go (which
+// might have been how V1 worked). Implications for
+// links (which may not have a valid target) and search
+// which would then be missing data.
+//
+// * Review the download rate throttling. Slow then fast?
+// Detect bandwidth usage and speed up when it drops?
+//
+// * An error on a fetch could be due to one item in the batch.
+// If the batch were broken up, perhaps more of the inventory
+// would download. (Handwave here, not certain this is an
+// issue in practice.)
+//
+// * Conversion to AISv3.
+//
+
+
+namespace
+{
+
+///----------------------------------------------------------------------------
+/// Class <anonymous>::BGItemHttpHandler
+///----------------------------------------------------------------------------
+
+//
+// Http request handler class for single inventory item requests.
+//
+// We'll use a handler-per-request pattern here rather than
+// a shared handler. Mainly convenient as this was converted
+// from a Responder class model.
+//
+// Derives from and is identical to the normal FetchItemHttpHandler
+// except that: 1) it uses the background request object which is
+// updated more slowly than the foreground and 2) keeps a count of
+// active requests on the LLInventoryModelBackgroundFetch object
+// to indicate outstanding operations are in-flight.
+//
+class BGItemHttpHandler : public LLInventoryModel::FetchItemHttpHandler
+{
+ LOG_CLASS(BGItemHttpHandler);
+
+public:
+ BGItemHttpHandler(const LLSD & request_sd)
+ : LLInventoryModel::FetchItemHttpHandler(request_sd)
+ {
+ LLInventoryModelBackgroundFetch::instance().incrFetchCount(1);
+ }
+
+ virtual ~BGItemHttpHandler()
+ {
+ LLInventoryModelBackgroundFetch::instance().incrFetchCount(-1);
+ }
+
+protected:
+ BGItemHttpHandler(const BGItemHttpHandler &); // Not defined
+ void operator=(const BGItemHttpHandler &); // Not defined
+};
+
+
+///----------------------------------------------------------------------------
+/// Class <anonymous>::BGFolderHttpHandler
+///----------------------------------------------------------------------------
+
+// Http request handler class for folders.
+//
+// Handler for FetchInventoryDescendents2 and FetchLibDescendents2
+// caps requests for folders.
+//
+class BGFolderHttpHandler : public LLCore::HttpHandler
+{
+ LOG_CLASS(BGFolderHttpHandler);
+
+public:
+ BGFolderHttpHandler(const LLSD & request_sd, const uuid_vec_t & recursive_cats)
+ : LLCore::HttpHandler(),
+ mRequestSD(request_sd),
+ mRecursiveCatUUIDs(recursive_cats)
+ {
+ LLInventoryModelBackgroundFetch::instance().incrFetchCount(1);
+ }
+
+ virtual ~BGFolderHttpHandler()
+ {
+ LLInventoryModelBackgroundFetch::instance().incrFetchCount(-1);
+ }
+
+protected:
+ BGFolderHttpHandler(const BGFolderHttpHandler &); // Not defined
+ void operator=(const BGFolderHttpHandler &); // Not defined
+
+public:
+ virtual void onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * response);
+
+ bool getIsRecursive(const LLUUID & cat_id) const;
+
+private:
+ void processData(LLSD & body, LLCore::HttpResponse * response);
+ void processFailure(LLCore::HttpStatus status, LLCore::HttpResponse * response);
+ void processFailure(const char * const reason, LLCore::HttpResponse * response);
+
+private:
+ LLSD mRequestSD;
+ const uuid_vec_t mRecursiveCatUUIDs; // hack for storing away which cat fetches are recursive
+};
+
+
+const char * const LOG_INV("Inventory");
+
+} // end of namespace anonymous
+
+
+///----------------------------------------------------------------------------
+/// Class LLInventoryModelBackgroundFetch
+///----------------------------------------------------------------------------
+
+LLInventoryModelBackgroundFetch::LLInventoryModelBackgroundFetch():
+ mBackgroundFetchActive(false),
+ mFolderFetchActive(false),
+ mFetchCount(0),
+ mLastFetchCount(0),
+ mFetchFolderCount(0),
+ mAllRecursiveFoldersFetched(false),
+ mRecursiveInventoryFetchStarted(false),
+ mRecursiveLibraryFetchStarted(false),
+ mRecursiveMarketplaceFetchStarted(false),
+ mMinTimeBetweenFetches(0.3f)
+{}
+
+LLInventoryModelBackgroundFetch::~LLInventoryModelBackgroundFetch()
+{
+ gIdleCallbacks.deleteFunction(&LLInventoryModelBackgroundFetch::backgroundFetchCB, NULL);
+}
+
+bool LLInventoryModelBackgroundFetch::isBulkFetchProcessingComplete() const
+{
+ return mFetchFolderQueue.empty() && mFetchItemQueue.empty() && mFetchCount <= 0;
+}
+
+bool LLInventoryModelBackgroundFetch::isFolderFetchProcessingComplete() const
+{
+ return mFetchFolderQueue.empty() && mFetchFolderCount <= 0;
+}
+
+bool LLInventoryModelBackgroundFetch::libraryFetchStarted() const
+{
+ return mRecursiveLibraryFetchStarted;
+}
+
+bool LLInventoryModelBackgroundFetch::libraryFetchCompleted() const
+{
+ return libraryFetchStarted() && fetchQueueContainsNoDescendentsOf(gInventory.getLibraryRootFolderID());
+}
+
+bool LLInventoryModelBackgroundFetch::libraryFetchInProgress() const
+{
+ return libraryFetchStarted() && !libraryFetchCompleted();
+}
+
+bool LLInventoryModelBackgroundFetch::inventoryFetchStarted() const
+{
+ return mRecursiveInventoryFetchStarted;
+}
+
+bool LLInventoryModelBackgroundFetch::inventoryFetchCompleted() const
+{
+ return inventoryFetchStarted() && fetchQueueContainsNoDescendentsOf(gInventory.getRootFolderID());
+}
+
+bool LLInventoryModelBackgroundFetch::inventoryFetchInProgress() const
+{
+ return inventoryFetchStarted() && ! inventoryFetchCompleted();
+}
+
+bool LLInventoryModelBackgroundFetch::isEverythingFetched() const
+{
+ return mAllRecursiveFoldersFetched;
+}
+
+bool LLInventoryModelBackgroundFetch::folderFetchActive() const
+{
+ return mFolderFetchActive;
+}
+
+void LLInventoryModelBackgroundFetch::addRequestAtFront(const LLUUID & id, bool recursive, bool is_category)
+{
+ EFetchType recursion_type = recursive ? FT_RECURSIVE : FT_DEFAULT;
+ if (is_category)
+ {
+ mFetchFolderQueue.push_front(FetchQueueInfo(id, recursion_type, is_category));
+ }
+ else
+ {
+ mFetchItemQueue.push_front(FetchQueueInfo(id, recursion_type, is_category));
+ }
+}
+
+void LLInventoryModelBackgroundFetch::addRequestAtBack(const LLUUID & id, bool recursive, bool is_category)
+{
+ EFetchType recursion_type = recursive ? FT_RECURSIVE : FT_DEFAULT;
+ if (is_category)
+ {
+ mFetchFolderQueue.push_back(FetchQueueInfo(id, recursion_type, is_category));
+ }
+ else
+ {
+ mFetchItemQueue.push_back(FetchQueueInfo(id, recursion_type, is_category));
+ }
+}
+
+void LLInventoryModelBackgroundFetch::start(const LLUUID& id, bool recursive)
+{
+ LLViewerInventoryCategory * cat(gInventory.getCategory(id));
+
+ if (cat || (id.isNull() && ! isEverythingFetched()))
+ {
+ // it's a folder, do a bulk fetch
+ LL_DEBUGS(LOG_INV) << "Start fetching category: " << id << ", recursive: " << recursive << LL_ENDL;
+
+ mBackgroundFetchActive = true;
+ mFolderFetchActive = true;
+ EFetchType recursion_type = recursive ? FT_RECURSIVE : FT_DEFAULT;
+ if (id.isNull())
+ {
+ if (! mRecursiveInventoryFetchStarted)
+ {
+ mRecursiveInventoryFetchStarted |= recursive;
+ if (recursive && AISAPI::isAvailable())
+ {
+ // Not only root folder can be massive, but
+ // most system folders will be requested independently
+ // so request root folder and content separately
+ mFetchFolderQueue.push_front(FetchQueueInfo(gInventory.getRootFolderID(), FT_FOLDER_AND_CONTENT));
+ }
+ else
+ {
+ mFetchFolderQueue.push_back(FetchQueueInfo(gInventory.getRootFolderID(), recursion_type));
+ }
+ gIdleCallbacks.addFunction(&LLInventoryModelBackgroundFetch::backgroundFetchCB, NULL);
+ }
+ if (! mRecursiveLibraryFetchStarted)
+ {
+ mRecursiveLibraryFetchStarted |= recursive;
+ mFetchFolderQueue.push_back(FetchQueueInfo(gInventory.getLibraryRootFolderID(), recursion_type));
+ gIdleCallbacks.addFunction(&LLInventoryModelBackgroundFetch::backgroundFetchCB, NULL);
+ }
+ }
+ else if (recursive && cat && cat->getPreferredType() == LLFolderType::FT_MARKETPLACE_LISTINGS)
+ {
+ if (mFetchFolderQueue.empty() || mFetchFolderQueue.back().mUUID != id)
+ {
+ if (recursive && AISAPI::isAvailable())
+ {
+ // Request marketplace folder and content separately
+ mFetchFolderQueue.push_front(FetchQueueInfo(id, FT_FOLDER_AND_CONTENT));
+ }
+ else
+ {
+ mFetchFolderQueue.push_front(FetchQueueInfo(id, recursion_type));
+ }
+ gIdleCallbacks.addFunction(&LLInventoryModelBackgroundFetch::backgroundFetchCB, NULL);
+ mRecursiveMarketplaceFetchStarted = true;
+ }
+ }
+ else
+ {
+ if (AISAPI::isAvailable())
+ {
+ if (mFetchFolderQueue.empty() || mFetchFolderQueue.back().mUUID != id)
+ {
+ // On AIS make sure root goes to the top and follow up recursive
+ // fetches, not individual requests
+ mFetchFolderQueue.push_back(FetchQueueInfo(id, recursion_type));
+ gIdleCallbacks.addFunction(&LLInventoryModelBackgroundFetch::backgroundFetchCB, NULL);
+ }
+ }
+ else if (mFetchFolderQueue.empty() || mFetchFolderQueue.front().mUUID != id)
+ {
+ // Specific folder requests go to front of queue.
+ mFetchFolderQueue.push_front(FetchQueueInfo(id, recursion_type));
+ gIdleCallbacks.addFunction(&LLInventoryModelBackgroundFetch::backgroundFetchCB, NULL);
+ }
+
+ if (id == gInventory.getLibraryRootFolderID())
+ {
+ mRecursiveLibraryFetchStarted |= recursive;
+ }
+ if (id == gInventory.getRootFolderID())
+ {
+ mRecursiveInventoryFetchStarted |= recursive;
+ }
+ }
+ }
+ else if (LLViewerInventoryItem * itemp = gInventory.getItem(id))
+ {
+ if (! itemp->mIsComplete)
+ {
+ scheduleItemFetch(id);
+ }
+ }
+}
+
+void LLInventoryModelBackgroundFetch::scheduleFolderFetch(const LLUUID& cat_id, bool forced)
+{
+ if (mFetchFolderQueue.empty() || mFetchFolderQueue.front().mUUID != cat_id)
+ {
+ mBackgroundFetchActive = true;
+ mFolderFetchActive = true;
+
+ if (forced)
+ {
+ // check if already requested
+ if (mForceFetchSet.find(cat_id) == mForceFetchSet.end())
+ {
+ mForceFetchSet.insert(cat_id);
+ mFetchItemQueue.push_front(FetchQueueInfo(cat_id, FT_FORCED));
+ }
+ }
+ else
+ {
+ // Specific folder requests go to front of queue.
+ // version presence acts as dupplicate prevention for normal fetches
+ mFetchItemQueue.push_front(FetchQueueInfo(cat_id, FT_DEFAULT));
+ }
+
+ gIdleCallbacks.addFunction(&LLInventoryModelBackgroundFetch::backgroundFetchCB, NULL);
+ }
+}
+
+void LLInventoryModelBackgroundFetch::scheduleItemFetch(const LLUUID& item_id, bool forced)
+{
+ if (mFetchItemQueue.empty() || mFetchItemQueue.front().mUUID != item_id)
+ {
+ mBackgroundFetchActive = true;
+ if (forced)
+ {
+ // check if already requested
+ if (mForceFetchSet.find(item_id) == mForceFetchSet.end())
+ {
+ mForceFetchSet.insert(item_id);
+ mFetchItemQueue.push_front(FetchQueueInfo(item_id, FT_FORCED, false));
+ }
+ }
+ else
+ {
+ // 'isFinished' being set acts as dupplicate prevention for normal fetches
+ mFetchItemQueue.push_front(FetchQueueInfo(item_id, FT_DEFAULT, false));
+ }
+
+ gIdleCallbacks.addFunction(&LLInventoryModelBackgroundFetch::backgroundFetchCB, NULL);
+ }
+}
+
+void LLInventoryModelBackgroundFetch::fetchFolderAndLinks(const LLUUID& cat_id, nullary_func_t callback)
+{
+ LLViewerInventoryCategory* cat = gInventory.getCategory(cat_id);
+ if (cat)
+ {
+ // Mark folder (update timer) so that background fetch won't request it
+ cat->setFetching(LLViewerInventoryCategory::FETCH_RECURSIVE);
+ }
+ incrFetchFolderCount(1);
+ mExpectedFolderIds.push_back(cat_id);
+
+ // Assume that we have no relevant cache. Fetch folder, and items folder's links point to.
+ AISAPI::FetchCategoryLinks(cat_id,
+ [callback, cat_id](const LLUUID& id)
+ {
+ callback();
+ if (id.isNull())
+ {
+ LL_WARNS() << "Failed to fetch category links " << cat_id << LL_ENDL;
+ }
+ LLInventoryModelBackgroundFetch::getInstance()->onAISFolderCalback(cat_id, id, FT_DEFAULT);
+ });
+
+ // start idle loop to track completion
+ mBackgroundFetchActive = true;
+ mFolderFetchActive = true;
+ gIdleCallbacks.addFunction(&LLInventoryModelBackgroundFetch::backgroundFetchCB, NULL);
+}
+
+void LLInventoryModelBackgroundFetch::fetchCOF(nullary_func_t callback)
+{
+ LLUUID cat_id = gInventory.findCategoryUUIDForType(LLFolderType::FT_CURRENT_OUTFIT);
+ LLViewerInventoryCategory* cat = gInventory.getCategory(cat_id);
+ if (cat)
+ {
+ // Mark cof (update timer) so that background fetch won't request it
+ cat->setFetching(LLViewerInventoryCategory::FETCH_RECURSIVE);
+ }
+ incrFetchFolderCount(1);
+ mExpectedFolderIds.push_back(cat_id);
+ // For reliability assume that we have no relevant cache, so
+ // fetch cof along with items cof's links point to.
+ AISAPI::FetchCOF([callback](const LLUUID& id)
+ {
+ callback();
+ LLUUID cat_id = gInventory.findCategoryUUIDForType(LLFolderType::FT_CURRENT_OUTFIT);
+ LLInventoryModelBackgroundFetch::getInstance()->onAISFolderCalback(cat_id, id, FT_DEFAULT);
+ });
+
+ // start idle loop to track completion
+ mBackgroundFetchActive = true;
+ mFolderFetchActive = true;
+ gIdleCallbacks.addFunction(&LLInventoryModelBackgroundFetch::backgroundFetchCB, NULL);
+}
+
+void LLInventoryModelBackgroundFetch::findLostItems()
+{
+ mBackgroundFetchActive = true;
+ mFolderFetchActive = true;
+ mFetchFolderQueue.push_back(FetchQueueInfo(LLUUID::null, FT_RECURSIVE));
+ gIdleCallbacks.addFunction(&LLInventoryModelBackgroundFetch::backgroundFetchCB, NULL);
+}
+
+void LLInventoryModelBackgroundFetch::setAllFoldersFetched()
+{
+ if (mRecursiveInventoryFetchStarted &&
+ mRecursiveLibraryFetchStarted)
+ {
+ mAllRecursiveFoldersFetched = true;
+ //LL_INFOS(LOG_INV) << "All folders fetched, validating" << LL_ENDL;
+ //gInventory.validate();
+ }
+
+ mFolderFetchActive = false;
+ if (isBulkFetchProcessingComplete())
+ {
+ mBackgroundFetchActive = false;
+ }
+
+ // For now only informs about initial fetch being done
+ mFoldersFetchedSignal();
+
+ LL_INFOS(LOG_INV) << "Inventory background fetch completed" << LL_ENDL;
+}
+
+boost::signals2::connection LLInventoryModelBackgroundFetch::setFetchCompletionCallback(folders_fetched_callback_t cb)
+{
+ return mFoldersFetchedSignal.connect(cb);
+}
+
+void LLInventoryModelBackgroundFetch::backgroundFetchCB(void *)
+{
+ LLInventoryModelBackgroundFetch::instance().backgroundFetch();
+}
+
+void LLInventoryModelBackgroundFetch::backgroundFetch()
+{
+ if (mBackgroundFetchActive)
+ {
+ if (AISAPI::isAvailable())
+ {
+ bulkFetchViaAis();
+ }
+ else if (gAgent.getRegion() && gAgent.getRegion()->capabilitiesReceived())
+ {
+ // If we'll be using the capability, we'll be sending batches and the background thing isn't as important.
+ bulkFetch();
+ }
+ }
+}
+
+void LLInventoryModelBackgroundFetch::incrFetchCount(S32 fetching)
+{
+ mFetchCount += fetching;
+ if (mFetchCount < 0)
+ {
+ LL_WARNS_ONCE(LOG_INV) << "Inventory fetch count fell below zero (0)." << LL_ENDL;
+ mFetchCount = 0;
+ }
+}
+void LLInventoryModelBackgroundFetch::incrFetchFolderCount(S32 fetching)
+{
+ incrFetchCount(fetching);
+ mFetchFolderCount += fetching;
+ if (mFetchCount < 0)
+ {
+ LL_WARNS_ONCE(LOG_INV) << "Inventory fetch count fell below zero (0)." << LL_ENDL;
+ mFetchFolderCount = 0;
+ }
+}
+
+void ais_simple_item_callback(const LLUUID& inv_id)
+{
+ LL_DEBUGS(LOG_INV , "AIS3") << "Response for " << inv_id << LL_ENDL;
+ LLInventoryModelBackgroundFetch::instance().incrFetchCount(-1);
+}
+
+void LLInventoryModelBackgroundFetch::onAISContentCalback(
+ const LLUUID& request_id,
+ const uuid_vec_t& content_ids,
+ const LLUUID& response_id,
+ EFetchType fetch_type)
+{
+ // Don't push_front on failure - there is a chance it was fired from inside bulkFetchViaAis
+ incrFetchFolderCount(-1);
+
+ uuid_vec_t::const_iterator folder_iter = content_ids.begin();
+ uuid_vec_t::const_iterator folder_end = content_ids.end();
+ while (folder_iter != folder_end)
+ {
+ std::list<LLUUID>::const_iterator found = std::find(mExpectedFolderIds.begin(), mExpectedFolderIds.end(), *folder_iter);
+ if (found != mExpectedFolderIds.end())
+ {
+ mExpectedFolderIds.erase(found);
+ }
+
+ LLViewerInventoryCategory* cat(gInventory.getCategory(*folder_iter));
+ if (cat)
+ {
+ cat->setFetching(LLViewerInventoryCategory::FETCH_NONE);
+ }
+ if (response_id.isNull())
+ {
+ // Failed to fetch, get it individually
+ mFetchFolderQueue.push_back(FetchQueueInfo(*folder_iter, FT_RECURSIVE));
+ }
+ else
+ {
+ // push descendant back to verify they are fetched fully (ex: didn't encounter depth limit)
+ LLInventoryModel::cat_array_t* categories(NULL);
+ LLInventoryModel::item_array_t* items(NULL);
+ gInventory.getDirectDescendentsOf(*folder_iter, categories, items);
+ if (categories)
+ {
+ for (LLInventoryModel::cat_array_t::const_iterator it = categories->begin();
+ it != categories->end();
+ ++it)
+ {
+ mFetchFolderQueue.push_back(FetchQueueInfo((*it)->getUUID(), FT_RECURSIVE));
+ }
+ }
+ }
+
+ folder_iter++;
+ }
+
+ if (!mFetchFolderQueue.empty())
+ {
+ mBackgroundFetchActive = true;
+ mFolderFetchActive = true;
+ gIdleCallbacks.addFunction(&LLInventoryModelBackgroundFetch::backgroundFetchCB, NULL);
+ }
+}
+void LLInventoryModelBackgroundFetch::onAISFolderCalback(const LLUUID &request_id, const LLUUID &response_id, EFetchType fetch_type)
+{
+ // Don't push_front on failure - there is a chance it was fired from inside bulkFetchViaAis
+ incrFetchFolderCount(-1);
+ std::list<LLUUID>::const_iterator found = std::find(mExpectedFolderIds.begin(), mExpectedFolderIds.end(), request_id);
+ if (found != mExpectedFolderIds.end())
+ {
+ mExpectedFolderIds.erase(found);
+ }
+ else
+ {
+ // ais shouldn't respond twice
+ llassert(false);
+ LL_WARNS() << "Unexpected folder response for " << request_id << LL_ENDL;
+ }
+
+ if (request_id.isNull())
+ {
+ // orhans, no other actions needed
+ return;
+ }
+
+ LLViewerInventoryCategory::EFetchType new_state = LLViewerInventoryCategory::FETCH_NONE;
+ bool request_descendants = false;
+ if (response_id.isNull()) // Failure
+ {
+ LL_DEBUGS(LOG_INV , "AIS3") << "Failure response for folder " << request_id << LL_ENDL;
+ if (fetch_type == FT_RECURSIVE)
+ {
+ // A full recursive request failed.
+ // Try requesting folder and nested content separately
+ mFetchFolderQueue.push_back(FetchQueueInfo(request_id, FT_FOLDER_AND_CONTENT));
+ }
+ else if (fetch_type == FT_FOLDER_AND_CONTENT)
+ {
+ LL_WARNS() << "Failed to download folder: " << request_id << " Requesting known content separately" << LL_ENDL;
+ mFetchFolderQueue.push_back(FetchQueueInfo(request_id, FT_CONTENT_RECURSIVE));
+
+ // set folder's version to prevent viewer from trying to request folder indefinetely
+ LLViewerInventoryCategory* cat(gInventory.getCategory(request_id));
+ if (cat && cat->getVersion() == LLViewerInventoryCategory::VERSION_UNKNOWN)
+ {
+ cat->setVersion(0);
+ }
+ // back off for a bit in case something tries to force-request immediately
+ new_state = LLViewerInventoryCategory::FETCH_FAILED;
+ }
+ }
+ else
+ {
+ if (fetch_type == FT_RECURSIVE)
+ {
+ // Got the folder and content, now verify content
+ // Request content even for FT_RECURSIVE in case of changes, failures
+ // or if depth limit gets imlemented.
+ // This shouldn't redownload folders if they already have version
+ request_descendants = true;
+ LL_DEBUGS(LOG_INV, "AIS3") << "Got folder " << request_id << ". Requesting content" << LL_ENDL;
+ }
+ else if (fetch_type == FT_FOLDER_AND_CONTENT)
+ {
+ // readd folder for content request
+ mFetchFolderQueue.push_front(FetchQueueInfo(request_id, FT_CONTENT_RECURSIVE));
+ }
+ else
+ {
+ LL_DEBUGS(LOG_INV, "AIS3") << "Got folder " << request_id << "." << LL_ENDL;
+ }
+
+ }
+
+ if (request_descendants)
+ {
+ LLInventoryModel::cat_array_t* categories(NULL);
+ LLInventoryModel::item_array_t* items(NULL);
+ gInventory.getDirectDescendentsOf(request_id, categories, items);
+ if (categories)
+ {
+ for (LLInventoryModel::cat_array_t::const_iterator it = categories->begin();
+ it != categories->end();
+ ++it)
+ {
+ mFetchFolderQueue.push_back(FetchQueueInfo((*it)->getUUID(), FT_RECURSIVE));
+ }
+ }
+ }
+
+ if (!mFetchFolderQueue.empty())
+ {
+ mBackgroundFetchActive = true;
+ mFolderFetchActive = true;
+ gIdleCallbacks.addFunction(&LLInventoryModelBackgroundFetch::backgroundFetchCB, NULL);
+ }
+
+ // done
+ LLViewerInventoryCategory * cat(gInventory.getCategory(request_id));
+ if (cat)
+ {
+ cat->setFetching(new_state);
+ }
+}
+
+static LLTrace::BlockTimerStatHandle FTM_BULK_FETCH("Bulk Fetch");
+
+void LLInventoryModelBackgroundFetch::bulkFetchViaAis()
+{
+ LL_RECORD_BLOCK_TIME(FTM_BULK_FETCH);
+ //Background fetch is called from gIdleCallbacks in a loop until background fetch is stopped.
+ if (gDisconnected)
+ {
+ return;
+ }
+
+ static LLCachedControl<U32> ais_pool(gSavedSettings, "PoolSizeAIS", 20);
+ // Don't have too many requests at once, AIS throttles
+ // Reserve one request for actions outside of fetch (like renames)
+ const U32 max_concurrent_fetches = llclamp(ais_pool - 1, 1, 50);
+
+ if (mFetchCount >= max_concurrent_fetches)
+ {
+ return;
+ }
+
+ // Don't loop for too long (in case of large, fully loaded inventory)
+ F64 curent_time = LLTimer::getTotalSeconds();
+ const F64 max_time = LLStartUp::getStartupState() > STATE_WEARABLES_WAIT
+ ? 0.006f // 6 ms
+ : 1.f;
+ const F64 end_time = curent_time + max_time;
+ S32 last_fetch_count = mFetchCount;
+
+ while (!mFetchFolderQueue.empty() && mFetchCount < max_concurrent_fetches && curent_time < end_time)
+ {
+ const FetchQueueInfo & fetch_info(mFetchFolderQueue.front());
+ bulkFetchViaAis(fetch_info);
+ mFetchFolderQueue.pop_front();
+ curent_time = LLTimer::getTotalSeconds();
+ }
+
+ // Ideally we shouldn't fetch items if recursive fetch isn't done,
+ // but there is a chance some request will start timeouting and recursive
+ // fetch will get stuck on a signle folder, don't block item fetch in such case
+ while (!mFetchItemQueue.empty() && mFetchCount < max_concurrent_fetches && curent_time < end_time)
+ {
+ const FetchQueueInfo& fetch_info(mFetchItemQueue.front());
+ bulkFetchViaAis(fetch_info);
+ mFetchItemQueue.pop_front();
+ curent_time = LLTimer::getTotalSeconds();
+ }
+
+ if (last_fetch_count != mFetchCount // if anything was added
+ || mLastFetchCount != mFetchCount) // if anything was substracted
+ {
+ LL_DEBUGS(LOG_INV , "AIS3") << "Total active fetches: " << mLastFetchCount << "->" << last_fetch_count << "->" << mFetchCount
+ << ", scheduled folder fetches: " << (S32)mFetchFolderQueue.size()
+ << ", scheduled item fetches: " << (S32)mFetchItemQueue.size()
+ << LL_ENDL;
+ mLastFetchCount = mFetchCount;
+
+ if (!mExpectedFolderIds.empty())
+ {
+ // A folder seem to be stack fetching on QA account, print oldest folder out
+ LL_DEBUGS(LOG_INV , "AIS3") << "Oldest expected folder: ";
+ std::list<LLUUID>::const_iterator iter = mExpectedFolderIds.begin();
+ LL_CONT << *iter;
+ if ((*iter).notNull())
+ {
+ LLViewerInventoryCategory* cat(gInventory.getCategory(*iter));
+ if (cat)
+ {
+ LL_CONT << " Folder name: " << cat->getName() << " Parent: " << cat->getParentUUID();
+ }
+ else
+ {
+ LL_CONT << " This folder doesn't exist";
+ }
+ }
+ else
+ {
+ LL_CONT << " Orphans request";
+ }
+ LL_CONT << LL_ENDL;
+ }
+ }
+
+ if (isFolderFetchProcessingComplete() && mFolderFetchActive)
+ {
+ if (!mRecursiveInventoryFetchStarted || mRecursiveMarketplaceFetchStarted)
+ {
+ setAllFoldersFetched();
+ }
+ else
+ {
+ // Intent is for marketplace request to happen after
+ // main inventory is done, unless requested by floater
+ mRecursiveMarketplaceFetchStarted = true;
+ const LLUUID& marketplacelistings_id = gInventory.findCategoryUUIDForType(LLFolderType::FT_MARKETPLACE_LISTINGS);
+ if (marketplacelistings_id.notNull())
+ {
+ mFetchFolderQueue.push_front(FetchQueueInfo(marketplacelistings_id, FT_FOLDER_AND_CONTENT));
+ }
+ else
+ {
+ setAllFoldersFetched();
+ }
+ }
+
+ }
+
+ if (isBulkFetchProcessingComplete())
+ {
+ mBackgroundFetchActive = false;
+ }
+}
+
+void LLInventoryModelBackgroundFetch::bulkFetchViaAis(const FetchQueueInfo& fetch_info)
+{
+ if (fetch_info.mIsCategory)
+ {
+ const LLUUID & cat_id(fetch_info.mUUID);
+ if (cat_id.isNull())
+ {
+ incrFetchFolderCount(1);
+ mExpectedFolderIds.push_back(cat_id);
+ // Lost and found
+ // Should it actually be recursive?
+ AISAPI::FetchOrphans([](const LLUUID& response_id)
+ {
+ LLInventoryModelBackgroundFetch::instance().onAISFolderCalback(LLUUID::null,
+ response_id,
+ FT_DEFAULT);
+ });
+ }
+ else
+ {
+ LLViewerInventoryCategory * cat(gInventory.getCategory(cat_id));
+ if (cat)
+ {
+ if (fetch_info.mFetchType == FT_CONTENT_RECURSIVE)
+ {
+ // fetch content only, ignore cat itself
+ uuid_vec_t children;
+ LLInventoryModel::cat_array_t* categories(NULL);
+ LLInventoryModel::item_array_t* items(NULL);
+ gInventory.getDirectDescendentsOf(cat_id, categories, items);
+
+ LLViewerInventoryCategory::EFetchType target_state = LLViewerInventoryCategory::FETCH_RECURSIVE;
+ bool content_done = true;
+
+ // Top limit is 'as many as you can put into url'
+ static LLCachedControl<S32> ais_batch(gSavedSettings, "BatchSizeAIS3", 20);
+ S32 batch_limit = llclamp(ais_batch(), 1, 40);
+
+ for (LLInventoryModel::cat_array_t::iterator it = categories->begin();
+ it != categories->end();
+ ++it)
+ {
+ LLViewerInventoryCategory* child_cat = (*it);
+ if (LLViewerInventoryCategory::VERSION_UNKNOWN != child_cat->getVersion()
+ || child_cat->getFetching() >= target_state)
+ {
+ continue;
+ }
+
+ if (child_cat->getPreferredType() == LLFolderType::FT_MARKETPLACE_LISTINGS)
+ {
+ // special case, marketplace will fetch that as needed
+ continue;
+ }
+
+ children.push_back(child_cat->getUUID());
+ mExpectedFolderIds.push_back(child_cat->getUUID());
+ child_cat->setFetching(target_state);
+
+ if (children.size() >= batch_limit)
+ {
+ content_done = false;
+ break;
+ }
+ }
+
+ if (!children.empty())
+ {
+ // increment before call in case of immediate callback
+ incrFetchFolderCount(1);
+
+ EFetchType type = fetch_info.mFetchType;
+ LLUUID cat_id = cat->getUUID(); // need a copy for lambda
+ AISAPI::completion_t cb = [cat_id, children, type](const LLUUID& response_id)
+ {
+ LLInventoryModelBackgroundFetch::instance().onAISContentCalback(cat_id, children, response_id, type);
+ };
+
+ AISAPI::ITEM_TYPE item_type = AISAPI::INVENTORY;
+ if (ALEXANDRIA_LINDEN_ID == cat->getOwnerID())
+ {
+ item_type = AISAPI::LIBRARY;
+ }
+
+ AISAPI::FetchCategorySubset(cat_id, children, item_type, true, cb, 0);
+ }
+
+ if (content_done)
+ {
+ // This will have a bit of overlap with onAISContentCalback,
+ // but something else might have dowloaded folders, so verify
+ // every child that is complete has it's children done as well
+ for (LLInventoryModel::cat_array_t::iterator it = categories->begin();
+ it != categories->end();
+ ++it)
+ {
+ LLViewerInventoryCategory* child_cat = (*it);
+ if (LLViewerInventoryCategory::VERSION_UNKNOWN != child_cat->getVersion())
+ {
+ mFetchFolderQueue.push_back(FetchQueueInfo(child_cat->getUUID(), FT_RECURSIVE));
+ }
+ }
+ }
+ else
+ {
+ // send it back to get the rest
+ mFetchFolderQueue.push_back(FetchQueueInfo(cat_id, FT_CONTENT_RECURSIVE));
+ }
+ }
+ else if (LLViewerInventoryCategory::VERSION_UNKNOWN == cat->getVersion()
+ || fetch_info.mFetchType == FT_FORCED)
+ {
+ LLViewerInventoryCategory::EFetchType target_state =
+ fetch_info.mFetchType > FT_CONTENT_RECURSIVE
+ ? LLViewerInventoryCategory::FETCH_RECURSIVE
+ : LLViewerInventoryCategory::FETCH_NORMAL;
+ // start again if we did a non-recursive fetch before
+ // to get all children in a single request
+ if (cat->getFetching() < target_state)
+ {
+ // increment before call in case of immediate callback
+ incrFetchFolderCount(1);
+ cat->setFetching(target_state);
+ mExpectedFolderIds.push_back(cat_id);
+
+ EFetchType type = fetch_info.mFetchType;
+ LLUUID cat_cb_id = cat_id;
+ AISAPI::completion_t cb = [cat_cb_id, type](const LLUUID& response_id)
+ {
+ LLInventoryModelBackgroundFetch::instance().onAISFolderCalback(cat_cb_id, response_id , type);
+ };
+
+ AISAPI::ITEM_TYPE item_type = AISAPI::INVENTORY;
+ if (ALEXANDRIA_LINDEN_ID == cat->getOwnerID())
+ {
+ item_type = AISAPI::LIBRARY;
+ }
+
+ AISAPI::FetchCategoryChildren(cat_id , item_type , type == FT_RECURSIVE , cb, 0);
+ }
+ }
+ else
+ {
+ // Already fetched, check if anything inside needs fetching
+ if (fetch_info.mFetchType == FT_RECURSIVE
+ || fetch_info.mFetchType == FT_FOLDER_AND_CONTENT)
+ {
+ LLInventoryModel::cat_array_t * categories(NULL);
+ LLInventoryModel::item_array_t * items(NULL);
+ gInventory.getDirectDescendentsOf(cat_id, categories, items);
+ for (LLInventoryModel::cat_array_t::const_iterator it = categories->begin();
+ it != categories->end();
+ ++it)
+ {
+ // not push_front to not cause an infinite loop
+ mFetchFolderQueue.push_back(FetchQueueInfo((*it)->getUUID(), FT_RECURSIVE));
+ }
+ }
+ }
+ } // else try to fetch folder either way?
+ }
+ }
+ else
+ {
+ LLViewerInventoryItem * itemp(gInventory.getItem(fetch_info.mUUID));
+
+ if (itemp)
+ {
+ if (!itemp->isFinished() || fetch_info.mFetchType == FT_FORCED)
+ {
+ mFetchCount++;
+ if (itemp->getPermissions().getOwner() == gAgent.getID())
+ {
+ AISAPI::FetchItem(fetch_info.mUUID, AISAPI::INVENTORY, ais_simple_item_callback);
+ }
+ else
+ {
+ AISAPI::FetchItem(fetch_info.mUUID, AISAPI::LIBRARY, ais_simple_item_callback);
+ }
+ }
+ }
+ else // We don't know it, assume incomplete
+ {
+ // Assume agent's inventory, library wouldn't have gotten here
+ mFetchCount++;
+ AISAPI::FetchItem(fetch_info.mUUID, AISAPI::INVENTORY, ais_simple_item_callback);
+ }
+ }
+
+ if (fetch_info.mFetchType == FT_FORCED)
+ {
+ mForceFetchSet.erase(fetch_info.mUUID);
+ }
+}
+
+// Bundle up a bunch of requests to send all at once.
+void LLInventoryModelBackgroundFetch::bulkFetch()
+{
+ LL_RECORD_BLOCK_TIME(FTM_BULK_FETCH);
+ //Background fetch is called from gIdleCallbacks in a loop until background fetch is stopped.
+ //If there are items in mFetchQueue, we want to check the time since the last bulkFetch was
+ //sent. If it exceeds our retry time, go ahead and fire off another batch.
+ LLViewerRegion * region(gAgent.getRegion());
+ if (! region || gDisconnected || LLApp::isExiting())
+ {
+ return;
+ }
+
+ // *TODO: These values could be tweaked at runtime to effect
+ // a fast/slow fetch throttle. Once login is complete and the scene
+ // is mostly loaded, we could turn up the throttle and fill missing
+ // inventory more quickly.
+ static const U32 max_batch_size(10);
+ static const S32 max_concurrent_fetches(12); // Outstanding requests, not connections
+
+ if (mFetchCount)
+ {
+ // Process completed background HTTP requests
+ gInventory.handleResponses(false);
+ // Just processed a bunch of items.
+ // Note: do we really need notifyObservers() here?
+ // OnIdle it will be called anyway due to Add flag for processed item.
+ // It seems like in some cases we are updaiting on fail (no flag),
+ // but is there anything to update?
+ gInventory.notifyObservers();
+ }
+
+ if (mFetchCount > max_concurrent_fetches)
+ {
+ return;
+ }
+
+ U32 item_count(0);
+ U32 folder_count(0);
+
+ const U32 sort_order(gSavedSettings.getU32(LLInventoryPanel::DEFAULT_SORT_ORDER) & 0x1);
+
+ // *TODO: Think I'd like to get a shared pointer to this and share it
+ // among all the folder requests.
+ uuid_vec_t recursive_cats;
+ uuid_vec_t all_cats; // dupplicate avoidance
+
+ LLSD folder_request_body;
+ LLSD folder_request_body_lib;
+ LLSD item_request_body;
+ LLSD item_request_body_lib;
+
+ while (! mFetchFolderQueue.empty()
+ && (item_count + folder_count) < max_batch_size)
+ {
+ const FetchQueueInfo & fetch_info(mFetchFolderQueue.front());
+ if (fetch_info.mIsCategory)
+ {
+ const LLUUID & cat_id(fetch_info.mUUID);
+ if (cat_id.isNull()) //DEV-17797 Lost and found
+ {
+ LLSD folder_sd;
+ folder_sd["folder_id"] = LLUUID::null.asString();
+ folder_sd["owner_id"] = gAgent.getID();
+ folder_sd["sort_order"] = LLSD::Integer(sort_order);
+ folder_sd["fetch_folders"] = LLSD::Boolean(false);
+ folder_sd["fetch_items"] = LLSD::Boolean(true);
+ folder_request_body["folders"].append(folder_sd);
+ folder_count++;
+ }
+ else
+ {
+ const LLViewerInventoryCategory * cat(gInventory.getCategory(cat_id));
+ if (cat)
+ {
+ if (LLViewerInventoryCategory::VERSION_UNKNOWN == cat->getVersion())
+ {
+ if (std::find(all_cats.begin(), all_cats.end(), cat_id) == all_cats.end())
+ {
+ LLSD folder_sd;
+ folder_sd["folder_id"] = cat->getUUID();
+ folder_sd["owner_id"] = cat->getOwnerID();
+ folder_sd["sort_order"] = LLSD::Integer(sort_order);
+ folder_sd["fetch_folders"] = LLSD::Boolean(true); //(LLSD::Boolean)sFullFetchStarted;
+ folder_sd["fetch_items"] = LLSD::Boolean(true);
+
+ if (ALEXANDRIA_LINDEN_ID == cat->getOwnerID())
+ {
+ folder_request_body_lib["folders"].append(folder_sd);
+ }
+ else
+ {
+ folder_request_body["folders"].append(folder_sd);
+ }
+ folder_count++;
+ }
+ }
+ else
+ {
+ // May already have this folder, but append child folders to list.
+ if (fetch_info.mFetchType >= FT_CONTENT_RECURSIVE)
+ {
+ LLInventoryModel::cat_array_t * categories(NULL);
+ LLInventoryModel::item_array_t * items(NULL);
+ gInventory.getDirectDescendentsOf(cat_id, categories, items);
+ for (LLInventoryModel::cat_array_t::const_iterator it = categories->begin();
+ it != categories->end();
+ ++it)
+ {
+ mFetchFolderQueue.push_back(FetchQueueInfo((*it)->getUUID(), fetch_info.mFetchType));
+ }
+ }
+ }
+ }
+ }
+ if (fetch_info.mFetchType >= FT_CONTENT_RECURSIVE)
+ {
+ recursive_cats.push_back(cat_id);
+ }
+ all_cats.push_back(cat_id);
+ }
+
+ mFetchFolderQueue.pop_front();
+ }
+
+
+ while (!mFetchItemQueue.empty()
+ && (item_count + folder_count) < max_batch_size)
+ {
+ const FetchQueueInfo & fetch_info(mFetchItemQueue.front());
+
+ LLViewerInventoryItem * itemp(gInventory.getItem(fetch_info.mUUID));
+
+ if (itemp)
+ {
+ LLSD item_sd;
+ item_sd["owner_id"] = itemp->getPermissions().getOwner();
+ item_sd["item_id"] = itemp->getUUID();
+ if (itemp->getPermissions().getOwner() == gAgent.getID())
+ {
+ item_request_body.append(item_sd);
+ }
+ else
+ {
+ item_request_body_lib.append(item_sd);
+ }
+ //itemp->fetchFromServer();
+ item_count++;
+ }
+
+ mFetchItemQueue.pop_front();
+ }
+
+ // Issue HTTP POST requests to fetch folders and items
+
+ if (item_count + folder_count > 0)
+ {
+ if (folder_count)
+ {
+ if (folder_request_body["folders"].size())
+ {
+ const std::string url(region->getCapability("FetchInventoryDescendents2"));
+
+ if (! url.empty())
+ {
+ LLCore::HttpHandler::ptr_t handler(new BGFolderHttpHandler(folder_request_body, recursive_cats));
+ gInventory.requestPost(false, url, folder_request_body, handler, "Inventory Folder");
+ }
+ }
+
+ if (folder_request_body_lib["folders"].size())
+ {
+ const std::string url(region->getCapability("FetchLibDescendents2"));
+
+ if (! url.empty())
+ {
+ LLCore::HttpHandler::ptr_t handler(new BGFolderHttpHandler(folder_request_body_lib, recursive_cats));
+ gInventory.requestPost(false, url, folder_request_body_lib, handler, "Library Folder");
+ }
+ }
+ } // if (folder_count)
+
+ if (item_count)
+ {
+ if (item_request_body.size())
+ {
+ const std::string url(region->getCapability("FetchInventory2"));
+
+ if (! url.empty())
+ {
+ LLSD body;
+ body["items"] = item_request_body;
+ LLCore::HttpHandler::ptr_t handler(new BGItemHttpHandler(body));
+ gInventory.requestPost(false, url, body, handler, "Inventory Item");
+ }
+ }
+
+ if (item_request_body_lib.size())
+ {
+ const std::string url(region->getCapability("FetchLib2"));
+
+ if (! url.empty())
+ {
+ LLSD body;
+ body["items"] = item_request_body_lib;
+ LLCore::HttpHandler::ptr_t handler(new BGItemHttpHandler(body));
+ gInventory.requestPost(false, url, body, handler, "Library Item");
+ }
+ }
+ } // if (item_count)
+
+ mFetchTimer.reset();
+ }
+ else if (isBulkFetchProcessingComplete())
+ {
+ setAllFoldersFetched();
+ }
+}
+
+bool LLInventoryModelBackgroundFetch::fetchQueueContainsNoDescendentsOf(const LLUUID & cat_id) const
+{
+ for (fetch_queue_t::const_iterator it = mFetchFolderQueue.begin();
+ it != mFetchFolderQueue.end();
+ ++it)
+ {
+ const LLUUID & fetch_id = (*it).mUUID;
+ if (gInventory.isObjectDescendentOf(fetch_id, cat_id))
+ return false;
+ }
+ for (fetch_queue_t::const_iterator it = mFetchItemQueue.begin();
+ it != mFetchItemQueue.end();
+ ++it)
+ {
+ const LLUUID & fetch_id = (*it).mUUID;
+ if (gInventory.isObjectDescendentOf(fetch_id, cat_id))
+ return false;
+ }
+ return true;
+}
+
+
+namespace
+{
+
+///----------------------------------------------------------------------------
+/// Class <anonymous>::BGFolderHttpHandler
+///----------------------------------------------------------------------------
+
+void BGFolderHttpHandler::onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * response)
+{
+ do // Single-pass do-while used for common exit handling
+ {
+ LLCore::HttpStatus status(response->getStatus());
+ // status = LLCore::HttpStatus(404); // Dev tool to force error handling
+ if (! status)
+ {
+ processFailure(status, response);
+ break; // Goto common exit
+ }
+
+ // Response body should be present.
+ LLCore::BufferArray * body(response->getBody());
+ // body = NULL; // Dev tool to force error handling
+ if (! body || ! body->size())
+ {
+ LL_WARNS(LOG_INV) << "Missing data in inventory folder query." << LL_ENDL;
+ processFailure("HTTP response missing expected body", response);
+ break; // Goto common exit
+ }
+
+ // Could test 'Content-Type' header but probably unreliable.
+
+ // Convert response to LLSD
+ // body->write(0, "Garbage Response", 16); // Dev tool to force error handling
+ LLSD body_llsd;
+ if (! LLCoreHttpUtil::responseToLLSD(response, true, body_llsd))
+ {
+ // INFOS-level logging will occur on the parsed failure
+ processFailure("HTTP response contained malformed LLSD", response);
+ break; // goto common exit
+ }
+
+ // Expect top-level structure to be a map
+ // body_llsd = LLSD::emptyArray(); // Dev tool to force error handling
+ if (! body_llsd.isMap())
+ {
+ processFailure("LLSD response not a map", response);
+ break; // goto common exit
+ }
+
+ // Check for 200-with-error failures
+ //
+ // See comments in llinventorymodel.cpp about this mode of error.
+ //
+ // body_llsd["error"] = LLSD::emptyMap(); // Dev tool to force error handling
+ // body_llsd["error"]["identifier"] = "Development";
+ // body_llsd["error"]["message"] = "You left development code in the viewer";
+ if (body_llsd.has("error"))
+ {
+ processFailure("Inventory application error (200-with-error)", response);
+ break; // goto common exit
+ }
+
+ // Okay, process data if possible
+ processData(body_llsd, response);
+ }
+ while (false);
+}
+
+
+void BGFolderHttpHandler::processData(LLSD & content, LLCore::HttpResponse * response)
+{
+ LLInventoryModelBackgroundFetch * fetcher(LLInventoryModelBackgroundFetch::getInstance());
+
+ // API V2 and earlier should probably be testing for "error" map
+ // in response as an application-level error.
+
+ // Instead, we assume success and attempt to extract information.
+ if (content.has("folders"))
+ {
+ LLSD folders(content["folders"]);
+
+ for (LLSD::array_const_iterator folder_it = folders.beginArray();
+ folder_it != folders.endArray();
+ ++folder_it)
+ {
+ LLSD folder_sd(*folder_it);
+
+ //LLUUID agent_id = folder_sd["agent_id"];
+
+ //if(agent_id != gAgent.getID()) //This should never happen.
+ //{
+ // LL_WARNS(LOG_INV) << "Got a UpdateInventoryItem for the wrong agent."
+ // << LL_ENDL;
+ // break;
+ //}
+
+ LLUUID parent_id(folder_sd["folder_id"].asUUID());
+ LLUUID owner_id(folder_sd["owner_id"].asUUID());
+ S32 version(folder_sd["version"].asInteger());
+ S32 descendents(folder_sd["descendents"].asInteger());
+ LLPointer<LLViewerInventoryCategory> tcategory = new LLViewerInventoryCategory(owner_id);
+
+ if (parent_id.isNull())
+ {
+ LLSD items(folder_sd["items"]);
+ LLPointer<LLViewerInventoryItem> titem = new LLViewerInventoryItem;
+
+ for (LLSD::array_const_iterator item_it = items.beginArray();
+ item_it != items.endArray();
+ ++item_it)
+ {
+ const LLUUID lost_uuid(gInventory.findCategoryUUIDForType(LLFolderType::FT_LOST_AND_FOUND));
+
+ if (lost_uuid.notNull())
+ {
+ LLSD item(*item_it);
+
+ titem->unpackMessage(item);
+
+ LLInventoryModel::update_list_t update;
+ LLInventoryModel::LLCategoryUpdate new_folder(lost_uuid, 1);
+ update.push_back(new_folder);
+ gInventory.accountForUpdate(update);
+
+ titem->setParent(lost_uuid);
+ titem->updateParentOnServer(false);
+ gInventory.updateItem(titem);
+ }
+ }
+ }
+
+ LLViewerInventoryCategory * pcat(gInventory.getCategory(parent_id));
+ if (! pcat)
+ {
+ continue;
+ }
+
+ LLSD categories(folder_sd["categories"]);
+ for (LLSD::array_const_iterator category_it = categories.beginArray();
+ category_it != categories.endArray();
+ ++category_it)
+ {
+ LLSD category(*category_it);
+ tcategory->fromLLSD(category);
+
+ const bool recursive(getIsRecursive(tcategory->getUUID()));
+ if (recursive)
+ {
+ fetcher->addRequestAtBack(tcategory->getUUID(), recursive, true);
+ }
+ else if (! gInventory.isCategoryComplete(tcategory->getUUID()))
+ {
+ gInventory.updateCategory(tcategory);
+ }
+ }
+
+ LLSD items(folder_sd["items"]);
+ LLPointer<LLViewerInventoryItem> titem = new LLViewerInventoryItem;
+ for (LLSD::array_const_iterator item_it = items.beginArray();
+ item_it != items.endArray();
+ ++item_it)
+ {
+ LLSD item(*item_it);
+ titem->unpackMessage(item);
+
+ gInventory.updateItem(titem);
+ }
+
+ // Set version and descendentcount according to message.
+ LLViewerInventoryCategory * cat(gInventory.getCategory(parent_id));
+ if (cat)
+ {
+ cat->setVersion(version);
+ cat->setDescendentCount(descendents);
+ cat->determineFolderType();
+ }
+ }
+ }
+
+ if (content.has("bad_folders"))
+ {
+ LLSD bad_folders(content["bad_folders"]);
+ for (LLSD::array_const_iterator folder_it = bad_folders.beginArray();
+ folder_it != bad_folders.endArray();
+ ++folder_it)
+ {
+ // *TODO: Stop copying data [ed: this isn't copying data]
+ LLSD folder_sd(*folder_it);
+
+ // These folders failed on the dataserver. We probably don't want to retry them.
+ LL_WARNS(LOG_INV) << "Folder " << folder_sd["folder_id"].asString()
+ << "Error: " << folder_sd["error"].asString() << LL_ENDL;
+ }
+ }
+
+ if (fetcher->isBulkFetchProcessingComplete())
+ {
+ fetcher->setAllFoldersFetched();
+ }
+}
+
+
+void BGFolderHttpHandler::processFailure(LLCore::HttpStatus status, LLCore::HttpResponse * response)
+{
+ const std::string & ct(response->getContentType());
+ LL_WARNS(LOG_INV) << "Inventory folder fetch failure\n"
+ << "[Status: " << status.toTerseString() << "]\n"
+ << "[Reason: " << status.toString() << "]\n"
+ << "[Content-type: " << ct << "]\n"
+ << "[Content (abridged): "
+ << LLCoreHttpUtil::responseToString(response) << "]" << LL_ENDL;
+
+ // Could use a 404 test here to try to detect revoked caps...
+
+ if(status == LLCore::HttpStatus(HTTP_FORBIDDEN))
+ {
+ // Too large, split into two if possible
+ if (gDisconnected || LLApp::isExiting())
+ {
+ return;
+ }
+
+ const std::string url(gAgent.getRegionCapability("FetchInventoryDescendents2"));
+ if (url.empty())
+ {
+ LL_WARNS(LOG_INV) << "Failed to get AIS2 cap" << LL_ENDL;
+ return;
+ }
+
+ S32 size = mRequestSD["folders"].size();
+
+ if (size > 1)
+ {
+ // Can split, assume that this isn't the library
+ LLSD folders;
+ uuid_vec_t recursive_cats;
+ LLSD::array_iterator iter = mRequestSD["folders"].beginArray();
+ LLSD::array_iterator end = mRequestSD["folders"].endArray();
+ while (iter != end)
+ {
+ folders.append(*iter);
+ LLUUID folder_id = iter->get("folder_id").asUUID();
+ if (std::find(mRecursiveCatUUIDs.begin(), mRecursiveCatUUIDs.end(), folder_id) != mRecursiveCatUUIDs.end())
+ {
+ recursive_cats.push_back(folder_id);
+ }
+ if (folders.size() == (S32)(size / 2))
+ {
+ LLSD request_body;
+ request_body["folders"] = folders;
+ LLCore::HttpHandler::ptr_t handler(new BGFolderHttpHandler(request_body, recursive_cats));
+ gInventory.requestPost(false, url, request_body, handler, "Inventory Folder");
+ recursive_cats.clear();
+ folders.clear();
+ }
+ iter++;
+ }
+
+ LLSD request_body;
+ request_body["folders"] = folders;
+ LLCore::HttpHandler::ptr_t handler(new BGFolderHttpHandler(request_body, recursive_cats));
+ gInventory.requestPost(false, url, request_body, handler, "Inventory Folder");
+ return;
+ }
+ else
+ {
+ // Can't split
+ LLNotificationsUtil::add("InventoryLimitReachedAIS");
+ }
+ }
+
+ // This was originally the request retry logic for the inventory
+ // request which tested on HTTP_INTERNAL_ERROR status. This
+ // retry logic was unbounded and lacked discrimination as to the
+ // cause of the retry. The new http library should be doing
+ // adquately on retries but I want to keep the structure of a
+ // retry for reference.
+ LLInventoryModelBackgroundFetch *fetcher = LLInventoryModelBackgroundFetch::getInstance();
+ if (false)
+ {
+ // timed out or curl failure
+ for (LLSD::array_const_iterator folder_it = mRequestSD["folders"].beginArray();
+ folder_it != mRequestSD["folders"].endArray();
+ ++folder_it)
+ {
+ LLSD folder_sd(*folder_it);
+ LLUUID folder_id(folder_sd["folder_id"].asUUID());
+ const bool recursive = getIsRecursive(folder_id);
+ fetcher->addRequestAtFront(folder_id, recursive, true);
+ }
+ }
+ else
+ {
+ if (fetcher->isBulkFetchProcessingComplete())
+ {
+ fetcher->setAllFoldersFetched();
+ }
+ }
+}
+
+
+void BGFolderHttpHandler::processFailure(const char * const reason, LLCore::HttpResponse * response)
+{
+ LL_WARNS(LOG_INV) << "Inventory folder fetch failure\n"
+ << "[Status: internal error]\n"
+ << "[Reason: " << reason << "]\n"
+ << "[Content (abridged): "
+ << LLCoreHttpUtil::responseToString(response) << "]" << LL_ENDL;
+
+ // Reverse of previous processFailure() method, this is invoked
+ // when response structure is found to be invalid. Original
+ // always re-issued the request (without limit). This does
+ // the same but be aware that this may be a source of problems.
+ // Philosophy is that inventory folders are so essential to
+ // operation that this is a reasonable action.
+ LLInventoryModelBackgroundFetch *fetcher = LLInventoryModelBackgroundFetch::getInstance();
+ if (true)
+ {
+ for (LLSD::array_const_iterator folder_it = mRequestSD["folders"].beginArray();
+ folder_it != mRequestSD["folders"].endArray();
+ ++folder_it)
+ {
+ LLSD folder_sd(*folder_it);
+ LLUUID folder_id(folder_sd["folder_id"].asUUID());
+ const bool recursive = getIsRecursive(folder_id);
+ fetcher->addRequestAtFront(folder_id, recursive, true);
+ }
+ }
+ else
+ {
+ if (fetcher->isBulkFetchProcessingComplete())
+ {
+ fetcher->setAllFoldersFetched();
+ }
+ }
+}
+
+
+bool BGFolderHttpHandler::getIsRecursive(const LLUUID & cat_id) const
+{
+ return std::find(mRecursiveCatUUIDs.begin(), mRecursiveCatUUIDs.end(), cat_id) != mRecursiveCatUUIDs.end();
+}
+
+///----------------------------------------------------------------------------
+/// Class <anonymous>::BGItemHttpHandler
+///----------------------------------------------------------------------------
+
+// Nothing to implement here. All ctor/dtor changes.
+
+} // end namespace anonymous
generated by cgit v1.2.3 (git 2.25.1) at 2025-02-02 03:55:00 +0000