SL-18003 Bulk download items when possible

And signal fodler fetch completion when folder of recursive fetch is done, do not hold it for individual items

7 files changed, 191 insertions, 80 deletions

diff --git a/indra/newview/llappearancemgr.cpp b/indra/newview/llappearancemgr.cpp
index b36ea1c457..8e44411d52 100644
--- a/indra/newview/llappearancemgr.cpp
+++ b/indra/newview/llappearancemgr.cpp
@@ -2420,7 +2420,7 @@ void LLAppearanceMgr::updateAppearanceFromCOF(bool enforce_item_restrictions,
     if (gInventory.hasPosiblyBrockenLinks())
     {
         // Inventory has either broken links or links that
-        // haven't loaded yet and fetch is still in progress.
+        // haven't loaded yet.
         // Check if LLAppearanceMgr needs to wait.
         LLUUID current_outfit_id = getCOF();
         LLInventoryModel::item_array_t cof_items;
@@ -2438,22 +2438,34 @@ void LLAppearanceMgr::updateAppearanceFromCOF(bool enforce_item_restrictions,
             // links are likely fine and we will have to wait for them to
             // load (if inventory takes too long to load, might be a good
             // idea to make this check periodical)
-            if (!mBulkFecthCallbackSlot.connected())
+            if (LLInventoryModelBackgroundFetch::getInstance()->folderFetchActive())
             {
-                nullary_func_t cb = post_update_func;
-                mBulkFecthCallbackSlot =
-                    LLInventoryModelBackgroundFetch::getInstance()->setAllFoldersFetchedCallback(
-                        [this, enforce_ordering, post_update_func, cb]()
+                if (!mBulkFecthCallbackSlot.connected())
                 {
-                    // inventory model should be already tracking this
-                    // callback, but make sure rebuildBrockenLinks gets
-                    // called before a cof update
-                    gInventory.rebuildBrockenLinks();
-                    updateAppearanceFromCOF(enforce_ordering, post_update_func, post_update_func);
-                    mBulkFecthCallbackSlot.disconnect();
-                });
+                    nullary_func_t cb = post_update_func;
+                    mBulkFecthCallbackSlot =
+                        LLInventoryModelBackgroundFetch::getInstance()->setFetchCompletionCallback(
+                            [this, enforce_ordering, post_update_func, cb]()
+                    {
+                        // inventory model should be already tracking this
+                        // callback, but make sure rebuildBrockenLinks gets
+                        // called before a cof update
+                        gInventory.rebuildBrockenLinks();
+                        updateAppearanceFromCOF(enforce_ordering, post_update_func, post_update_func);
+                        mBulkFecthCallbackSlot.disconnect();
+                    });
+                }
+                return;
+            }
+            else
+            {
+                // this should have happened on completion callback,
+                // check why it didn't then fix it
+                llassert(false); 
+
+                // try to recover now
+                gInventory.rebuildBrockenLinks();
             }
-            return;
         }
     }
 
@@ -4433,7 +4445,7 @@ public:
                         // To prevent premature removal from mIncomplete and
                         // since we are doing a full refetch anyway, mark unknown
                         cat->setVersion(LLViewerInventoryCategory::VERSION_UNKNOWN);
-                        LLInventoryModelBackgroundFetch::instance().start(*it, false);
+                        cat->fetch();
                         mIncomplete.push_back(*it);
                     }
                     else
diff --git a/indra/newview/llinventorymodel.cpp b/indra/newview/llinventorymodel.cpp
index 47fd17ef86..bfc7840708 100644
--- a/indra/newview/llinventorymodel.cpp
+++ b/indra/newview/llinventorymodel.cpp
@@ -1762,7 +1762,7 @@ void LLInventoryModel::rebuildBrockenLinks()
     // make sure we aren't adding expensive Rebuild to anything else.
     notifyObservers();
 
-    for (LLUUID link_id : mPossiblyBrockenLinks)
+    for (const LLUUID &link_id : mPossiblyBrockenLinks)
     {
         addChangedMask(LLInventoryObserver::REBUILD, link_id);
     }
@@ -2388,7 +2388,7 @@ void LLInventoryModel::addItem(LLViewerInventoryItem* item)
 		// The item will show up as a broken link.
 		if (item->getIsBrokenLink())
 		{
-            if (!LLInventoryModelBackgroundFetch::getInstance()->isEverythingFetched())
+            if (LLInventoryModelBackgroundFetch::getInstance()->folderFetchActive())
             {
                 // isEverythingFetched is actually 'initial' fetch only.
                 // Schedule this link for a recheck once inventory gets loaded
@@ -2399,7 +2399,7 @@ void LLInventoryModel::addItem(LLViewerInventoryItem* item)
                     // might be a lot of them. A better option might be to check
                     // links periodically with final check on fetch completion.
                     mBulkFecthCallbackSlot =
-                        LLInventoryModelBackgroundFetch::getInstance()->setAllFoldersFetchedCallback(
+                        LLInventoryModelBackgroundFetch::getInstance()->setFetchCompletionCallback(
                             [this]()
                     {
                         rebuildBrockenLinks();
@@ -3798,7 +3798,10 @@ void LLInventoryModel::processBulkUpdateInventory(LLMessageSystem* msg, void**)
         // Temporary workaround: just fetch the item using AIS to get missing fields.
         // If this works fine we might want to extract ids only from the message
         // then use AIS as a primary fetcher
-        LLInventoryModelBackgroundFetch::instance().start((*cit)->getUUID(), false);
+
+        // Use AIS derectly to not reset folder's version
+        // Todo: May be LLInventoryModelBackgroundFetch needs a 'forced' option
+        AISAPI::FetchCategoryChildren((*cit)->getUUID(), AISAPI::INVENTORY);
 	}
 	for (item_array_t::iterator iit = items.begin(); iit != items.end(); ++iit)
 	{
@@ -3807,7 +3810,7 @@ void LLInventoryModel::processBulkUpdateInventory(LLMessageSystem* msg, void**)
         // Temporary workaround: just fetch the item using AIS to get missing fields.
         // If this works fine we might want to extract ids only from the message
         // then use AIS as a primary fetcher
-        LLInventoryModelBackgroundFetch::instance().start((*iit)->getUUID(), false);
+        LLInventoryModelBackgroundFetch::instance().scheduleItemFetch((*iit)->getUUID());
 	}
 	gInventory.notifyObservers();
 
diff --git a/indra/newview/llinventorymodelbackgroundfetch.cpp b/indra/newview/llinventorymodelbackgroundfetch.cpp
index 25e35915c3..d58dd22ca1 100644
--- a/indra/newview/llinventorymodelbackgroundfetch.cpp
+++ b/indra/newview/llinventorymodelbackgroundfetch.cpp
@@ -190,6 +190,7 @@ LLInventoryModelBackgroundFetch::LLInventoryModelBackgroundFetch():
 	mBackgroundFetchActive(false),
 	mFolderFetchActive(false),
 	mFetchCount(0),
+    mFetchFolderCount(0),
 	mAllFoldersFetched(false),
 	mRecursiveInventoryFetchStarted(false),
 	mRecursiveLibraryFetchStarted(false),
@@ -204,6 +205,11 @@ bool LLInventoryModelBackgroundFetch::isBulkFetchProcessingComplete() const
 	return mFetchFolderQueue.empty() && mFetchItemQueue.empty() && mFetchCount <= 0;
 }
 
+bool LLInventoryModelBackgroundFetch::isFolderFetchProcessingComplete() const
+{
+    return mFetchFolderQueue.empty() && mFetchFolderCount <= 0;
+}
+
 bool LLInventoryModelBackgroundFetch::libraryFetchStarted() const
 {
 	return mRecursiveLibraryFetchStarted;
@@ -338,16 +344,24 @@ void LLInventoryModelBackgroundFetch::start(const LLUUID& id, bool recursive)
 	}
 	else if (LLViewerInventoryItem * itemp = gInventory.getItem(id))
 	{
-		if (! itemp->mIsComplete && (mFetchItemQueue.empty() || mFetchItemQueue.front().mUUID != id))
+		if (! itemp->mIsComplete)
 		{
-			mBackgroundFetchActive = true;
-
-			mFetchItemQueue.push_front(FetchQueueInfo(id, RT_NONE, false));
-			gIdleCallbacks.addFunction(&LLInventoryModelBackgroundFetch::backgroundFetchCB, NULL);
+            scheduleItemFetch(id);
 		}
 	}
 }
 
+void LLInventoryModelBackgroundFetch::scheduleItemFetch(const LLUUID& item_id)
+{
+    if (mFetchItemQueue.empty() || mFetchItemQueue.front().mUUID != item_id)
+    {
+        mBackgroundFetchActive = true;
+
+        mFetchItemQueue.push_front(FetchQueueInfo(item_id, RT_NONE, false));
+        gIdleCallbacks.addFunction(&LLInventoryModelBackgroundFetch::backgroundFetchCB, NULL);
+    }
+}
+
 void LLInventoryModelBackgroundFetch::findLostItems()
 {
     if (AISAPI::isAvailable())
@@ -371,18 +385,23 @@ void LLInventoryModelBackgroundFetch::setAllFoldersFetched()
 		mAllFoldersFetched = true;
 		//LL_INFOS(LOG_INV) << "All folders fetched, validating" << LL_ENDL;
 		//gInventory.validate();
-
-        // For now only informs about initial fetch being done
-        mAllFoldersFetchedSignal();
 	}
+
 	mFolderFetchActive = false;
-	mBackgroundFetchActive = false;
+    if (isBulkFetchProcessingComplete())
+    {
+        mBackgroundFetchActive = false;
+    }
+
+    // For now only informs about initial fetch being done
+    mFoldersFetchedSignal();
+
 	LL_INFOS(LOG_INV) << "Inventory background fetch completed" << LL_ENDL;
 }
 
-boost::signals2::connection LLInventoryModelBackgroundFetch::setAllFoldersFetchedCallback(folders_fetched_callback_t cb)
+boost::signals2::connection LLInventoryModelBackgroundFetch::setFetchCompletionCallback(folders_fetched_callback_t cb)
 {
-    return mAllFoldersFetchedSignal.connect(cb);
+    return mFoldersFetchedSignal.connect(cb);
 }
 
 void LLInventoryModelBackgroundFetch::backgroundFetchCB(void *)
@@ -415,10 +434,20 @@ void LLInventoryModelBackgroundFetch::incrFetchCount(S32 fetching)
 		mFetchCount = 0; 
 	}
 }
+void LLInventoryModelBackgroundFetch::incrFetchFolderCount(S32 fetching)
+{
+    incrFetchCount(fetching);
+    mFetchFolderCount += fetching;
+    if (mFetchCount < 0)
+    {
+        LL_WARNS_ONCE(LOG_INV) << "Inventory fetch count fell below zero (0)." << LL_ENDL;
+        mFetchFolderCount = 0;
+    }
+}
 
 void ais_simple_folder_callback(const LLUUID& inv_id)
 {
-    LLInventoryModelBackgroundFetch::instance().incrFetchCount(-1);
+    LLInventoryModelBackgroundFetch::instance().incrFetchFolderCount(-1);
     LLViewerInventoryCategory * cat(gInventory.getCategory(inv_id));
     if (cat) cat->setFetching(false);
 }
@@ -430,7 +459,7 @@ void ais_simple_item_callback(const LLUUID& inv_id)
 
 void LLInventoryModelBackgroundFetch::onAISFodlerCalback(const LLUUID &request_id, const LLUUID &response_id, ERecursionType recursion)
 {
-    incrFetchCount(-1);
+    incrFetchFolderCount(-1);
     if (response_id.isNull()) // Failure
     {
         if (recursion == RT_RECURSIVE)
@@ -464,13 +493,13 @@ void LLInventoryModelBackgroundFetch::onAISFodlerCalback(const LLUUID &request_i
                 gIdleCallbacks.addFunction(&LLInventoryModelBackgroundFetch::backgroundFetchCB, NULL);
             }
         }
+    }
 
-        // done
-        LLViewerInventoryCategory * cat(gInventory.getCategory(request_id));
-        if (cat)
-        {
-            cat->setFetching(false);
-        }
+    // done
+    LLViewerInventoryCategory * cat(gInventory.getCategory(request_id));
+    if (cat)
+    {
+        cat->setFetching(false);
     }
 }
 
@@ -516,10 +545,15 @@ void LLInventoryModelBackgroundFetch::bulkFetchViaAis()
         curent_time = LLTimer::getTotalSeconds();
     }
     
-    if (isBulkFetchProcessingComplete())
+    if (isFolderFetchProcessingComplete() && mFolderFetchActive)
     {
         setAllFoldersFetched();
     }
+
+    if (isBulkFetchProcessingComplete())
+    {
+        mBackgroundFetchActive = false;
+    }
 }
 
 void LLInventoryModelBackgroundFetch::bulkFetchViaAis(const FetchQueueInfo& fetch_info)
@@ -532,7 +566,7 @@ void LLInventoryModelBackgroundFetch::bulkFetchViaAis(const FetchQueueInfo& fetc
             LL_WARNS() << "Lost and found not implemented yet" << LL_ENDL;
             // todo: needs to be requested from ais in special manner?
             /*AISAPI::FetchCategoryChildren(LLUUID::null, AISAPI::INVENTORY, false, ais_simple_callback);
-            mFetchCount++;*/
+            incrFetchFolderCount(1);*/
         }
         else
         {
@@ -559,7 +593,7 @@ void LLInventoryModelBackgroundFetch::bulkFetchViaAis(const FetchQueueInfo& fetc
                             LLInventoryModelBackgroundFetch::instance().onAISFodlerCalback(cat_id, response_id, type);
                         });
                     }
-                    mFetchCount++;
+                    incrFetchFolderCount(1);
                     cat->setFetching(true);
                 }
                 else
@@ -590,14 +624,19 @@ void LLInventoryModelBackgroundFetch::bulkFetchViaAis(const FetchQueueInfo& fetc
         {
             if (itemp->getPermissions().getOwner() == gAgent.getID())
             {
-                AISAPI::FetchItem(itemp->getUUID(), AISAPI::INVENTORY, ais_simple_item_callback);
+                AISAPI::FetchItem(fetch_info.mUUID, AISAPI::INVENTORY, ais_simple_item_callback);
             }
             else
             {
-                AISAPI::FetchItem(itemp->getUUID(), AISAPI::LIBRARY, ais_simple_item_callback);
+                AISAPI::FetchItem(fetch_info.mUUID, AISAPI::LIBRARY, ais_simple_item_callback);
             }
-            mFetchCount++;
         }
+        else
+        {
+            // Assume agent's inventory, library wouldn't have gotten here
+            AISAPI::FetchItem(fetch_info.mUUID, AISAPI::INVENTORY, ais_simple_item_callback);
+        }
+        mFetchCount++;
     }
 }
 
diff --git a/indra/newview/llinventorymodelbackgroundfetch.h b/indra/newview/llinventorymodelbackgroundfetch.h
index 0a5608c0c6..6a8b616a82 100644
--- a/indra/newview/llinventorymodelbackgroundfetch.h
+++ b/indra/newview/llinventorymodelbackgroundfetch.h
@@ -47,9 +47,10 @@ class LLInventoryModelBackgroundFetch : public LLSingleton<LLInventoryModelBackg
 	~LLInventoryModelBackgroundFetch();
 public:
 
-	// Start and stop background breadth-first fetching of inventory contents.
+    // Start  background breadth-first fetching of inventory contents.
 	// This gets triggered when performing a filter-search.
 	void start(const LLUUID& cat_id = LLUUID::null, bool recursive = true);
+    void scheduleItemFetch(const LLUUID& item_id);
 
 	BOOL folderFetchActive() const;
 	bool isEverythingFetched() const; // completing the fetch once per session should be sufficient
@@ -64,12 +65,14 @@ public:
 
     void findLostItems();	
 	void incrFetchCount(S32 fetching);
+    void incrFetchFolderCount(S32 fetching);
 
 	bool isBulkFetchProcessingComplete() const;
+    bool isFolderFetchProcessingComplete() const;
 	void setAllFoldersFetched();
 
     typedef boost::function<void()> folders_fetched_callback_t;
-    boost::signals2::connection setAllFoldersFetchedCallback(folders_fetched_callback_t cb);
+    boost::signals2::connection setFetchCompletionCallback(folders_fetched_callback_t cb);
 
 	void addRequestAtFront(const LLUUID & id, bool recursive, bool is_category);
 	void addRequestAtBack(const LLUUID & id, bool recursive, bool is_category);
@@ -110,11 +113,12 @@ private:
 	bool mRecursiveLibraryFetchStarted;
 	bool mAllFoldersFetched;
     typedef boost::signals2::signal<void()> folders_fetched_signal_t;
-    folders_fetched_signal_t mAllFoldersFetchedSignal;
+    folders_fetched_signal_t mFoldersFetchedSignal;
 
     bool mBackgroundFetchActive;
 	bool mFolderFetchActive;
 	S32 mFetchCount;
+    S32 mFetchFolderCount;
 
 	LLFrameTimer mFetchTimer;
 	F32 mMinTimeBetweenFetches;
diff --git a/indra/newview/llinventoryobserver.cpp b/indra/newview/llinventoryobserver.cpp
index fe33be4abe..4db913ed0d 100644
--- a/indra/newview/llinventoryobserver.cpp
+++ b/indra/newview/llinventoryobserver.cpp
@@ -255,29 +255,19 @@ void LLInventoryFetchItemsObserver::startFetch()
 {
     bool aisv3 = AISAPI::isAvailable();
 
-	LLUUID owner_id;
 	LLSD items_llsd;
+
+    typedef std::map<LLUUID, uuid_vec_t> requests_by_fodlers_t;
+    requests_by_fodlers_t requests;
 	for (uuid_vec_t::const_iterator it = mIDs.begin(); it < mIDs.end(); ++it)
 	{
-		LLViewerInventoryItem* item = gInventory.getItem(*it);
-		if (item)
-		{
-			if (item->isFinished())
-			{
-				// It's complete, so put it on the complete container.
-				mComplete.push_back(*it);
-				continue;
-			}
-			else
-			{
-				owner_id = item->getPermissions().getOwner();
-			}
-		}
-		else
-		{
-			// assume it's agent inventory.
-			owner_id = gAgent.getID();
-		}
+        LLViewerInventoryItem* item = gInventory.getItem(*it);
+        if (item && item->isFinished())
+        {
+            // It's complete, so put it on the complete container.
+            mComplete.push_back(*it);
+            continue;
+        }
 
 		// Ignore categories since they're not items.  We
 		// could also just add this to mComplete but not sure what the
@@ -300,13 +290,31 @@ void LLInventoryFetchItemsObserver::startFetch()
 
         if (aisv3)
         {
-            LLInventoryModelBackgroundFetch::getInstance()->start(*it);
+            if (item)
+            {
+                LLUUID parent_id = item->getParentUUID();
+                requests[parent_id].push_back(*it);
+            }
+            else
+            {
+                // Can happen for gestures and calling cards if server notified us before they fetched
+                // Request by id without checking for an item.
+                LLInventoryModelBackgroundFetch::getInstance()->scheduleItemFetch(*it);
+            }
         }
         else
         {
             // Prepare the data to fetch
             LLSD item_entry;
-            item_entry["owner_id"] = owner_id;
+            if (item)
+            {
+                item_entry["owner_id"] = item->getPermissions().getOwner();
+            }
+            else
+            {
+                // assume it's agent inventory.
+                item_entry["owner_id"] = gAgent.getID();
+            }
             item_entry["item_id"] = (*it);
             items_llsd.append(item_entry);
         }
@@ -315,7 +323,59 @@ void LLInventoryFetchItemsObserver::startFetch()
 	mFetchingPeriod.reset();
 	mFetchingPeriod.setTimerExpirySec(FETCH_TIMER_EXPIRY);
 
-    if (!aisv3)
+    if (aisv3)
+    {
+        const S32 MAX_INDIVIDUAL_REQUESTS = 10;
+        for (requests_by_fodlers_t::value_type &folder : requests)
+        {
+            if (folder.second.size() > MAX_INDIVIDUAL_REQUESTS)
+            {
+                // requesting one by one will take a while
+                // do whole folder
+                LLInventoryModelBackgroundFetch::getInstance()->start(folder.first);
+            }
+            else
+            {
+                LLViewerInventoryCategory* cat = gInventory.getCategory(folder.first);
+                if (cat)
+                {
+                    if (cat->getVersion() == LLViewerInventoryCategory::VERSION_UNKNOWN)
+                    {
+                        // start fetching whole folder since it's not ready either way
+                        cat->fetch();
+                    }
+                    else if (cat->getViewerDescendentCount() <= folder.second.size())
+                    {
+                        // start fetching whole folder since we need all items
+                        cat->setVersion(LLViewerInventoryCategory::VERSION_UNKNOWN);
+                        cat->fetch();
+
+                    }
+                    else
+                    {
+                        // get items one by one
+                        for (LLUUID &item_id : folder.second)
+                        {
+                            LLInventoryModelBackgroundFetch::getInstance()->scheduleItemFetch(item_id);
+                        }
+                    }
+                }
+                else
+                {
+                    // Isn't supposed to happen? We should have all folders
+                    // and if item exists, folder is supposed to exist as well.
+                    llassert(false);
+
+                    // get items one by one
+                    for (LLUUID &item_id : folder.second)
+                    {
+                        LLInventoryModelBackgroundFetch::getInstance()->scheduleItemFetch(item_id);
+                    }
+                }
+            }
+        }
+    }
+    else
     {
         fetch_items_from_llsd(items_llsd);
     }
diff --git a/indra/newview/llsidepanelinventory.cpp b/indra/newview/llsidepanelinventory.cpp
index de0330588b..bd6f846268 100644
--- a/indra/newview/llsidepanelinventory.cpp
+++ b/indra/newview/llsidepanelinventory.cpp
@@ -545,7 +545,7 @@ void LLSidepanelInventory::clearSelections(bool clearMain, bool clearInbox)
 		}
 	}
 	
-	if (clearInbox && mInboxEnabled && mInventoryPanelInbox.get())
+	if (clearInbox && mInboxEnabled && !mInventoryPanelInbox.isDead())
 	{
 		mInventoryPanelInbox.get()->getRootFolder()->clearSelection();
 	}
diff --git a/indra/newview/llviewerinventory.cpp b/indra/newview/llviewerinventory.cpp
index 6bd86f7902..6f62ba5409 100644
--- a/indra/newview/llviewerinventory.cpp
+++ b/indra/newview/llviewerinventory.cpp
@@ -432,14 +432,7 @@ void LLViewerInventoryItem::fetchFromServer(void) const
 	{
         if (AISAPI::isAvailable()) // AIS v 3
         {
-            if (gAgent.getID() != mPermissions.getOwner())
-            {
-                AISAPI::FetchItem(mUUID, AISAPI::LIBRARY);
-            }
-            else
-            {
-                AISAPI::FetchItem(mUUID, AISAPI::INVENTORY);
-            }
+            LLInventoryModelBackgroundFetch::getInstance()->scheduleItemFetch(mUUID);
         }
         else
         {