SL-18629 WIP Fetch Inventory using AIS caps #2

4 files changed, 164 insertions, 124 deletions

diff --git a/indra/newview/llaisapi.cpp b/indra/newview/llaisapi.cpp
index 87811e9c0b..23bb123dee 100644
--- a/indra/newview/llaisapi.cpp
+++ b/indra/newview/llaisapi.cpp
@@ -484,41 +484,6 @@ void AISAPI::FetchCategoryCategories(const LLUUID &catId, ITEM_TYPE type, bool r
 }
 
 /*static*/
-void AISAPI::FetchCOF(completion_t callback)
-{
-    std::string cap;
-    cap = getInvCap();
-    if (cap.empty())
-    {
-        LL_WARNS("Inventory") << "Inventory cap not found!" << LL_ENDL;
-        return;
-    }
-    LLUUID cof_id = gInventory.findCategoryUUIDForType(LLFolderType::FT_CURRENT_OUTFIT);
-
-    std::string url = cap + std::string("/category/") + cof_id.asString() +  "/links";
-    
-    //LLUUID root_id = gInventory.findCategoryUUIDForType(LLFolderType::FT_ROOT_INVENTORY);
-    //std::string url = cap + std::string("/category/") + root_id.asString() +  "/children?depth=*";
-
-    invokationFn_t patchFn = boost::bind(
-        // Humans ignore next line.  It is just a cast to specify which LLCoreHttpUtil::HttpCoroutineAdapter routine overload.
-        static_cast<LLSD(LLCoreHttpUtil::HttpCoroutineAdapter::*)(LLCore::HttpRequest::ptr_t, const std::string &, LLCore::HttpOptions::ptr_t, LLCore::HttpHeaders::ptr_t)>
-        //----
-        // _1 -> httpAdapter
-        // _2 -> httpRequest
-        // _3 -> url
-        // _4 -> body 
-        // _5 -> httpOptions
-        // _6 -> httpHeaders
-        (&LLCoreHttpUtil::HttpCoroutineAdapter::getAndSuspend), _1, _2, _3, _5, _6);
-
-    LLCoprocedureManager::CoProcedure_t proc(boost::bind(&AISAPI::InvokeAISCommandCoro,
-        _1, patchFn, url, cof_id, LLSD(), callback, FETCHCOF));
-
-    EnqueueAISCommand("FetchCOF", proc);
-}
-
-/*static*/
 void AISAPI::EnqueueAISCommand(const std::string &procName, LLCoprocedureManager::CoProcedure_t proc)
 {
     LLCoprocedureManager &inst = LLCoprocedureManager::instance();
@@ -578,8 +543,7 @@ void AISAPI::onUpdateReceived(const std::string& context, const LLSD& update, CO
     }
     bool is_fetch = (type == FETCHITEM)
         || (type == FETCHCATEGORYCHILDREN)
-        || (type == FETCHCATEGORYCATEGORIES)
-        || (type == FETCHCOF);
+        || (type == FETCHCATEGORYCATEGORIES);
     // parse update llsd into stuff to do or parse received items.
     AISUpdate ais_update(update, is_fetch);
     ais_update.doUpdate(); // execute the updates in the appropriate order.
@@ -648,36 +612,16 @@ void AISAPI::InvokeAISCommandCoro(LLCoreHttpUtil::HttpCoroutineAdapter::ptr_t ht
                 }
             }
         }
-        /*else if (status.getType() == 403)
+        else if (status.getType() == 403)
         {
             if (type == FETCHCATEGORYCHILDREN)
             {
-                if (url.find("?depth=*") != std::string::npos)
-                {
-                    LL_WARNS() << "too much" << LL_ENDL;
-                    AISAPI::FetchCategoryChildren(gInventory.getRootFolderID(), AISAPI::ITEM_TYPE::INVENTORY, false, boost::bind(&LLInventoryModel::fetchDescendentsDepthOne, &gInventory, targetId));
-
-                    if (result["oversize_inventory"].asBoolean() == true)
-                    {
-                        LL_WARNS("Inventory") << "Can't fetch the oversized inventory folder" << LL_ENDL;
-                    }
-                }
-                else
-                {
-                    if (result["oversize_inventory"].asBoolean() == true)
-                    {
-                        LL_WARNS("Inventory") << "Can't fetch the oversized inventory folder" << LL_ENDL;
-                    }
-                }
+                LL_DEBUGS("Inventory") << "Fetch failed, content is over imit" << LL_ENDL;
             }
-        }*/
+        }
         LL_WARNS("Inventory") << "Inventory error: " << status.toString() << LL_ENDL;
         LL_WARNS("Inventory") << ll_pretty_print_sd(result) << LL_ENDL;
     }
-    else if (type == FETCHCOF)
-    {
-        //LL_WARNS("Inventory") << ll_pretty_print_sd(result) << LL_ENDL;      
-    }
 
 	LL_DEBUGS("Inventory") << result << LL_ENDL;
     onUpdateReceived("AISCommand", result, type);
@@ -687,7 +631,10 @@ void AISAPI::InvokeAISCommandCoro(LLCoreHttpUtil::HttpCoroutineAdapter::ptr_t ht
         bool needs_callback = true;
         LLUUID id(LLUUID::null);
 
-        if (type == COPYLIBRARYCATEGORY && result.has("category_id"))
+        if ( ( (type == COPYLIBRARYCATEGORY)
+               || (type == FETCHCATEGORYCATEGORIES)
+               || (type == FETCHCATEGORYCHILDREN))
+             && result.has("category_id"))
         {
             id = result["category_id"];
 	    }
@@ -871,7 +818,12 @@ void AISUpdate::parseItem(const LLSD& item_map)
 	BOOL rv = new_item->unpackMessage(item_map);
 	if (rv)
 	{
-		if (!mFetch && curr_item)
+        if (mFetch)
+        {
+            mItemsCreated[item_id] = new_item;
+            mCatDescendentDeltas[new_item->getParentUUID()];
+        }
+        else if (curr_item)
 		{
 			mItemsUpdated[item_id] = new_item;
 			// This statement is here to cause a new entry with 0
@@ -906,7 +858,19 @@ void AISUpdate::parseLink(const LLSD& link_map)
 	if (rv)
 	{
 		const LLUUID& parent_id = new_link->getParentUUID();
-		if (!mFetch && curr_link)
+        if (mFetch)
+        {
+            LLPermissions default_perms;
+            default_perms.init(gAgent.getID(), gAgent.getID(), LLUUID::null, LLUUID::null);
+            default_perms.initMasks(PERM_NONE, PERM_NONE, PERM_NONE, PERM_NONE, PERM_NONE);
+            new_link->setPermissions(default_perms);
+            LLSaleInfo default_sale_info;
+            new_link->setSaleInfo(default_sale_info);
+            //LL_DEBUGS("Inventory") << "creating link from llsd: " << ll_pretty_print_sd(link_map) << LL_ENDL;
+            mItemsCreated[item_id] = new_link;
+            mCatDescendentDeltas[parent_id];
+        }
+		else if (curr_link)
 		{
 			mItemsUpdated[item_id] = new_link;
 			// This statement is here to cause a new entry with 0
@@ -973,7 +937,28 @@ void AISUpdate::parseCategory(const LLSD& category_map)
 	//}
 	if (rv)
 	{
-		if (curr_cat && !mFetch)
+        if (mFetch)
+        {
+            // Set version/descendents for newly created categories.
+            if (category_map.has("version"))
+            {
+                S32 version = category_map["version"].asInteger();
+                LL_DEBUGS("Inventory") << "Setting version to " << version
+                    << " for new category " << category_id << LL_ENDL;
+                new_cat->setVersion(version);
+            }
+            uuid_int_map_t::const_iterator lookup_it = mCatDescendentsKnown.find(category_id);
+            if (mCatDescendentsKnown.end() != lookup_it)
+            {
+                S32 descendent_count = lookup_it->second;
+                LL_DEBUGS("Inventory") << "Setting descendents count to " << descendent_count
+                    << " for new category " << category_id << LL_ENDL;
+                new_cat->setDescendentCount(descendent_count);
+            }
+            mCategoriesCreated[category_id] = new_cat;
+            mCatDescendentDeltas[new_cat->getParentUUID()];
+        }
+		else if (curr_cat)
 		{
 			mCategoriesUpdated[category_id] = new_cat;
 			// This statement is here to cause a new entry with 0
diff --git a/indra/newview/llaisapi.h b/indra/newview/llaisapi.h
index 3de3366a0e..58f6a17650 100644
--- a/indra/newview/llaisapi.h
+++ b/indra/newview/llaisapi.h
@@ -75,7 +75,6 @@ private:
         FETCHITEM,
         FETCHCATEGORYCHILDREN,
         FETCHCATEGORYCATEGORIES,
-        FETCHCOF,
     } COMMAND_TYPE;
 
     static const std::string INVENTORY_CAP_NAME;
diff --git a/indra/newview/llinventorymodelbackgroundfetch.cpp b/indra/newview/llinventorymodelbackgroundfetch.cpp
index 1905a9c9a4..5ea8fe4cba 100644
--- a/indra/newview/llinventorymodelbackgroundfetch.cpp
+++ b/indra/newview/llinventorymodelbackgroundfetch.cpp
@@ -244,12 +244,14 @@ BOOL LLInventoryModelBackgroundFetch::folderFetchActive() const
 
 void LLInventoryModelBackgroundFetch::addRequestAtFront(const LLUUID & id, bool recursive, bool is_category)
 {
-	mFetchQueue.push_front(FetchQueueInfo(id, recursive, is_category));
+    ERecursionType recursion_type = recursive ? RT_RECURSIVE : RT_NONE;
+	mFetchQueue.push_front(FetchQueueInfo(id, recursion_type, is_category));
 }
 
 void LLInventoryModelBackgroundFetch::addRequestAtBack(const LLUUID & id, bool recursive, bool is_category)
 {
-	mFetchQueue.push_back(FetchQueueInfo(id, recursive, is_category));
+    ERecursionType recursion_type = recursive ? RT_RECURSIVE : RT_NONE;
+	mFetchQueue.push_back(FetchQueueInfo(id, recursion_type, is_category));
 }
 
 void LLInventoryModelBackgroundFetch::start(const LLUUID& id, bool recursive)
@@ -263,6 +265,7 @@ void LLInventoryModelBackgroundFetch::start(const LLUUID& id, bool recursive)
 
 		mBackgroundFetchActive = true;
 		mFolderFetchActive = true;
+        ERecursionType recursion_type = recursive ? RT_RECURSIVE : RT_NONE;
 		if (id.isNull())
 		{
 			if (! mRecursiveInventoryFetchStarted)
@@ -270,11 +273,14 @@ void LLInventoryModelBackgroundFetch::start(const LLUUID& id, bool recursive)
 				mRecursiveInventoryFetchStarted |= recursive;
                 if (recursive && AISAPI::isAvailable())
                 {
-                    mRecursiveFetchQueue.push_back(FetchQueueInfo(gInventory.getRootFolderID(), recursive));
+                    // Not only root folder can be massive, but
+                    // most system folders will be requested independently
+                    // so request root folder and content separately
+                    mRecursiveFetchQueue.push_back(FetchQueueInfo(gInventory.getRootFolderID(), RT_CONTENT));
                 }
                 else
                 {
-                    mFetchQueue.push_back(FetchQueueInfo(gInventory.getRootFolderID(), recursive));
+                    mFetchQueue.push_back(FetchQueueInfo(gInventory.getRootFolderID(), recursion_type));
                 }
 				gIdleCallbacks.addFunction(&LLInventoryModelBackgroundFetch::backgroundFetchCB, NULL);
 			}
@@ -283,11 +289,11 @@ void LLInventoryModelBackgroundFetch::start(const LLUUID& id, bool recursive)
 				mRecursiveLibraryFetchStarted |= recursive;
                 if (recursive && AISAPI::isAvailable())
                 {
-                    mRecursiveFetchQueue.push_back(FetchQueueInfo(gInventory.getLibraryRootFolderID(), recursive));
+                    mRecursiveFetchQueue.push_back(FetchQueueInfo(gInventory.getLibraryRootFolderID(), recursion_type));
                 }
                 else
                 {
-                    mFetchQueue.push_back(FetchQueueInfo(gInventory.getLibraryRootFolderID(), recursive));
+                    mFetchQueue.push_back(FetchQueueInfo(gInventory.getLibraryRootFolderID(), recursion_type));
                 }
 				gIdleCallbacks.addFunction(&LLInventoryModelBackgroundFetch::backgroundFetchCB, NULL);
 			}
@@ -299,7 +305,7 @@ void LLInventoryModelBackgroundFetch::start(const LLUUID& id, bool recursive)
                 // AIS does depth requests, recursive requests will need to be prioritizes
                 if (mRecursiveFetchQueue.empty() || mRecursiveFetchQueue.back().mUUID != id)
                 {
-                    mRecursiveFetchQueue.push_back(FetchQueueInfo(id, recursive));
+                    mRecursiveFetchQueue.push_back(FetchQueueInfo(id, recursion_type));
                     gIdleCallbacks.addFunction(&LLInventoryModelBackgroundFetch::backgroundFetchCB, NULL);
                 }
             }
@@ -308,7 +314,7 @@ void LLInventoryModelBackgroundFetch::start(const LLUUID& id, bool recursive)
                 // Specific folder requests go to front of queue.
                 if (mFetchQueue.empty() || mFetchQueue.front().mUUID != id)
                 {
-                    mFetchQueue.push_front(FetchQueueInfo(id, recursive));
+                    mFetchQueue.push_front(FetchQueueInfo(id, recursion_type));
                     gIdleCallbacks.addFunction(&LLInventoryModelBackgroundFetch::backgroundFetchCB, NULL);
                 }
             }
@@ -328,7 +334,7 @@ void LLInventoryModelBackgroundFetch::start(const LLUUID& id, bool recursive)
 		{
 			mBackgroundFetchActive = true;
 
-			mFetchQueue.push_front(FetchQueueInfo(id, false, false));
+			mFetchQueue.push_front(FetchQueueInfo(id, RT_NONE, false));
 			gIdleCallbacks.addFunction(&LLInventoryModelBackgroundFetch::backgroundFetchCB, NULL);
 		}
 	}
@@ -338,7 +344,7 @@ void LLInventoryModelBackgroundFetch::findLostItems()
 {
 	mBackgroundFetchActive = true;
 	mFolderFetchActive = true;
-    mFetchQueue.push_back(FetchQueueInfo(LLUUID::null, true));
+    mFetchQueue.push_back(FetchQueueInfo(LLUUID::null, RT_RECURSIVE));
     gIdleCallbacks.addFunction(&LLInventoryModelBackgroundFetch::backgroundFetchCB, NULL);
 }
 
@@ -386,11 +392,45 @@ void LLInventoryModelBackgroundFetch::incrFetchCount(S32 fetching)
 		mFetchCount = 0; 
 	}
 }
-void ais_callback(const LLUUID& inv_id)
+
+void ais_simple_callback(const LLUUID& inv_id)
 {
     LLInventoryModelBackgroundFetch::instance().incrFetchCount(-1);
 }
 
+void LLInventoryModelBackgroundFetch::onAISCalback(const LLUUID &request_id, const LLUUID &response_id, ERecursionType recursion)
+{
+    incrFetchCount(-1);
+    if (response_id.isNull()) // Failure
+    {
+        if (recursion == RT_RECURSIVE)
+        {
+            // A full recursive request failed.
+            // Try requesting folder and nested content separately
+            mBackgroundFetchActive = true;
+            mFolderFetchActive = true;
+            mRecursiveFetchQueue.push_back(FetchQueueInfo(request_id, RT_CONTENT));
+            gIdleCallbacks.addFunction(&LLInventoryModelBackgroundFetch::backgroundFetchCB, NULL);
+        }
+    }
+    else
+    {
+        if (recursion == RT_CONTENT)
+        {
+            // Got the folder, now recursively request content
+            LLInventoryModel::cat_array_t * categories(NULL);
+            LLInventoryModel::item_array_t * items(NULL);
+            gInventory.getDirectDescendentsOf(request_id, categories, items);
+            for (LLInventoryModel::cat_array_t::const_iterator it = categories->begin();
+                it != categories->end();
+                ++it)
+            {
+                mRecursiveFetchQueue.push_back(FetchQueueInfo((*it)->getUUID(), RT_RECURSIVE));
+            }
+        }
+    }
+}
+
 static LLTrace::BlockTimerStatHandle FTM_BULK_FETCH("Bulk Fetch");
 
 void LLInventoryModelBackgroundFetch::bulkFetchViaAis()
@@ -438,45 +478,54 @@ void LLInventoryModelBackgroundFetch::bulkFetchViaAis(const FetchQueueInfo& fetc
         const LLUUID & cat_id(fetch_info.mUUID);
         if (cat_id.isNull()) // Lost and found
         {
-            AISAPI::FetchCategoryChildren(LLUUID::null, AISAPI::INVENTORY, false, ais_callback);
+            AISAPI::FetchCategoryChildren(LLUUID::null, AISAPI::INVENTORY, false, ais_simple_callback);
             mFetchCount++;
         }
         else
         {
-            const LLViewerInventoryCategory * cat(gInventory.getCategory(cat_id));
 
-            if (cat)
+            if (!gInventory.isCategoryComplete(cat_id))
             {
-                if (LLViewerInventoryCategory::VERSION_UNKNOWN == cat->getVersion())
+                const LLViewerInventoryCategory * cat(gInventory.getCategory(cat_id));
+                if (cat)
                 {
                     if (ALEXANDRIA_LINDEN_ID == cat->getOwnerID())
                     {
-                        AISAPI::FetchCategoryChildren(cat->getUUID(), AISAPI::LIBRARY, fetch_info.mRecursive, ais_callback);
+                        AISAPI::FetchCategoryChildren(cat->getUUID(), AISAPI::LIBRARY, fetch_info.mRecursive == RT_RECURSIVE, ais_simple_callback);
                     }
                     else
                     {
-                        AISAPI::FetchCategoryChildren(cat->getUUID(), AISAPI::INVENTORY, fetch_info.mRecursive, ais_callback);
+                        LLUUID cat_id = cat->getUUID();
+                        ERecursionType type = fetch_info.mRecursive;
+                        AISAPI::FetchCategoryChildren(
+                            cat_id,
+                            AISAPI::INVENTORY,
+                            type == RT_RECURSIVE,
+                            [cat_id, type](const LLUUID &response_id)
+                        {
+                            LLInventoryModelBackgroundFetch::instance().onAISCalback(cat_id, response_id, type);
+                        });
                     }
                     mFetchCount++;
                 }
-                else
+                // else?
+            }
+            else
+            {
+                // Already fetched, check if anything inside needs fetching
+                if (fetch_info.mRecursive)
                 {
-                    // Already fetched, check if anything inside needs fetching
-                    if (fetch_info.mRecursive)
+                    LLInventoryModel::cat_array_t * categories(NULL);
+                    LLInventoryModel::item_array_t * items(NULL);
+                    gInventory.getDirectDescendentsOf(cat_id, categories, items);
+                    for (LLInventoryModel::cat_array_t::const_iterator it = categories->begin();
+                        it != categories->end();
+                        ++it)
                     {
-                        LLInventoryModel::cat_array_t * categories(NULL);
-                        LLInventoryModel::item_array_t * items(NULL);
-                        gInventory.getDirectDescendentsOf(cat->getUUID(), categories, items);
-                        for (LLInventoryModel::cat_array_t::const_iterator it = categories->begin();
-                            it != categories->end();
-                            ++it)
-                        {
-                            mFetchQueue.push_back(FetchQueueInfo((*it)->getUUID(), fetch_info.mRecursive));
-                        }
+                        mRecursiveFetchQueue.push_back(FetchQueueInfo((*it)->getUUID(), fetch_info.mRecursive));
                     }
                 }
             }
-            // else???
         }
     }
     else
@@ -487,11 +536,11 @@ void LLInventoryModelBackgroundFetch::bulkFetchViaAis(const FetchQueueInfo& fetc
         {
             if (itemp->getPermissions().getOwner() == gAgent.getID())
             {
-                AISAPI::FetchItem(itemp->getUUID(), AISAPI::INVENTORY, ais_callback);
+                AISAPI::FetchItem(itemp->getUUID(), AISAPI::INVENTORY, ais_simple_callback);
             }
             else
             {
-                AISAPI::FetchItem(itemp->getUUID(), AISAPI::LIBRARY, ais_callback);
+                AISAPI::FetchItem(itemp->getUUID(), AISAPI::LIBRARY, ais_simple_callback);
             }
             mFetchCount++;
         }
@@ -576,36 +625,37 @@ void LLInventoryModelBackgroundFetch::bulkFetch()
 			}
 			else
 			{
-				const LLViewerInventoryCategory * cat(gInventory.getCategory(cat_id));
-		
-				if (cat)
-				{
-					if (LLViewerInventoryCategory::VERSION_UNKNOWN == cat->getVersion())
-					{
-						LLSD folder_sd;
-						folder_sd["folder_id"]		= cat->getUUID();
-						folder_sd["owner_id"]		= cat->getOwnerID();
-						folder_sd["sort_order"]		= LLSD::Integer(sort_order);
-						folder_sd["fetch_folders"]	= LLSD::Boolean(true); //(LLSD::Boolean)sFullFetchStarted;
-						folder_sd["fetch_items"]	= LLSD::Boolean(true);
-				    
-						if (ALEXANDRIA_LINDEN_ID == cat->getOwnerID())
-						{
-							folder_request_body_lib["folders"].append(folder_sd);
-						}
-						else
-						{
-							folder_request_body["folders"].append(folder_sd);
-						}
-						folder_count++;
-					}
-
+                if (!gInventory.isCategoryComplete(cat_id))
+                {
+                    const LLViewerInventoryCategory * cat(gInventory.getCategory(cat_id));
+                    if (cat)
+                    {
+                        LLSD folder_sd;
+                        folder_sd["folder_id"] = cat->getUUID();
+                        folder_sd["owner_id"] = cat->getOwnerID();
+                        folder_sd["sort_order"] = LLSD::Integer(sort_order);
+                        folder_sd["fetch_folders"] = LLSD::Boolean(true); //(LLSD::Boolean)sFullFetchStarted;
+                        folder_sd["fetch_items"] = LLSD::Boolean(true);
+
+                        if (ALEXANDRIA_LINDEN_ID == cat->getOwnerID())
+                        {
+                            folder_request_body_lib["folders"].append(folder_sd);
+                        }
+                        else
+                        {
+                            folder_request_body["folders"].append(folder_sd);
+                        }
+                        folder_count++;
+                    }
+                }
+                else
+                {
 					// May already have this folder, but append child folders to list.
 					if (fetch_info.mRecursive)
 					{	
 						LLInventoryModel::cat_array_t * categories(NULL);
 						LLInventoryModel::item_array_t * items(NULL);
-						gInventory.getDirectDescendentsOf(cat->getUUID(), categories, items);
+						gInventory.getDirectDescendentsOf(cat_id, categories, items);
 						for (LLInventoryModel::cat_array_t::const_iterator it = categories->begin();
 							 it != categories->end();
 							 ++it)
diff --git a/indra/newview/llinventorymodelbackgroundfetch.h b/indra/newview/llinventorymodelbackgroundfetch.h
index ac1c42e0d7..4c4de6ac7c 100644
--- a/indra/newview/llinventorymodelbackgroundfetch.h
+++ b/indra/newview/llinventorymodelbackgroundfetch.h
@@ -73,9 +73,14 @@ public:
 
 protected:
 
+    typedef enum {
+        RT_NONE = 0,
+        RT_CONTENT, // request content recursively
+        RT_RECURSIVE, // request everything recursively
+    } ERecursionType;
     struct FetchQueueInfo
     {
-        FetchQueueInfo(const LLUUID& id, bool recursive, bool is_category = true)
+        FetchQueueInfo(const LLUUID& id, ERecursionType recursive, bool is_category = true)
             : mUUID(id),
             mIsCategory(is_category),
             mRecursive(recursive)
@@ -83,10 +88,11 @@ protected:
 
         LLUUID mUUID;
         bool mIsCategory;
-        bool mRecursive;
+        ERecursionType mRecursive;
     };
     typedef std::deque<FetchQueueInfo> fetch_queue_t;
 
+    void onAISCalback(const LLUUID &request_id, const LLUUID &response_id, ERecursionType recursion);
     void bulkFetchViaAis();
     void bulkFetchViaAis(const FetchQueueInfo& fetch_info);
 	void bulkFetch();