Give certain LLInventory queries an API based on result sets.

Introduce abstract base class InvResultSet, derived from LLIntTracker so each
instance has a unique int key. InvResultSet supports virtual getLength() and
getSlice() operations. getSlice() returns an LLSD array limited to
MAX_ITEM_LIMIT result set entries. It permits retrieving a "slice" of the
contained result set starting at an arbitrary index. A sequence of getSlice()
calls can eventually retrieve a whole result set.

InvResultSet has subclasses CatResultSet containing cat_array_t, and
ItemResultSet containing item_array_t. Each implements a virtual method that
produces an LLSD map from a single array item.

Make LLInventoryListener::getItemsInfo(), getDirectDescendants() and
collectDescendantsIf() instantiate heap CatResultSet and ItemResultSet objects
containing the resultant LLPointer arrays, and return their int keys for
categories and items.

Add LLInventoryListener::getSlice() and closeResult() methods that accept the
int keys of result sets. getSlice() returns the requested LLSD array to its
caller, while closeResult() is fire-and-forget.

Because bulk data transfer is now performed by getSlice() rather than by
collectDescendantsIf(), change the latter's "limit" default to unlimited.
Allow the C++ code to collect an arbitrary number of LLPointer array entries,
as long as getSlice() limits retrieval overhead.

Spell "descendants" correctly, unlike the "descendents" spelling embedded in
the rest of the viewer... sigh. Make the Lua module provide both spellings.

Make MAX_ITEM_LIMIT a U32 instead of F32.

In LLInventory.lua, store int result set keys from 'getItemsInfo',
'getDirectDescendants' and 'collectDescendantsIf' in a table with a close()
function. The close() function invokes 'closeResult' with the bound int keys.

Give that table an __index() metamethod that recognizes only 'categories' and
'items' keys: anything else returns nil. For either of the recognized keys,
call 'getSlice' with the corresponding result set key to retrieve (the initial
slice of) the actual result set. Cache that result. Lazy retrieval means that
if the caller only cares about categories, or only about items, the other
result set need never be retrieved at all.

This is a first step: like the previous code, it still retrieves only up to
the first 100 result set entries. But the C++ code now supports retrieval of
additional slices, so extending result set retrieval is mostly Lua work.

Finally, wrap the table-with-metamethod in an LL.setdtor() proxy whose
destructor calls its close() method to tell LLInventoryListener to destroy the
CatResultSet and ItemResultSet with the bound keys.

1 files changed, 214 insertions, 47 deletions

diff --git a/indra/newview/llinventorylistener.cpp b/indra/newview/llinventorylistener.cpp
index 157e04dce3..c330ef42a0 100644
--- a/indra/newview/llinventorylistener.cpp
+++ b/indra/newview/llinventorylistener.cpp
@@ -28,12 +28,13 @@
 #include "llinventorylistener.h"
 
 #include "llappearancemgr.h"
+#include "llinttracker.h"
 #include "llinventoryfunctions.h"
 #include "lltransutil.h"
 #include "llwearableitemslist.h"
 #include "stringize.h"
 
-static const F32 MAX_ITEM_LIMIT = 100;
+constexpr U32 MAX_ITEM_LIMIT = 100;
 
 LLInventoryListener::LLInventoryListener()
   : LLEventAPI("LLInventory",
@@ -41,7 +42,7 @@ LLInventoryListener::LLInventoryListener()
 {
     add("getItemsInfo",
         "Return information about items or folders defined in [\"item_ids\"]:\n"
-        "reply will contain [\"items\"] and [\"categories\"] tables accordingly",
+        "reply will contain [\"items\"] and [\"categories\"] result set keys",
         &LLInventoryListener::getItemsInfo,
         llsd::map("item_ids", LLSD(), "reply", LLSD()));
 
@@ -61,78 +62,190 @@ LLInventoryListener::LLInventoryListener()
         &LLInventoryListener::getBasicFolderID,
         llsd::map("ft_name", LLSD(), "reply", LLSD()));
 
-    add("getDirectDescendents",
-        "Return the direct descendents(both items and folders) of the [\"folder_id\"]",
-        &LLInventoryListener::getDirectDescendents,
+    add("getDirectDescendants",
+        "Return result set keys [\"categories\"] and [\"items\"] for the direct\n"
+        "descendants of the [\"folder_id\"]",
+        &LLInventoryListener::getDirectDescendants,
         llsd::map("folder_id", LLSD(), "reply", LLSD()));
 
-    add("collectDescendentsIf",
-        "Return the descendents(both items and folders) of the [\"folder_id\"], if it passes specified filters:\n"
+    add("collectDescendantsIf",
+        "Return result set keys [\"categories\"] and [\"items\"] for the descendants\n"
+        "of the [\"folder_id\"], if it passes specified filters:\n"
         "[\"name\"] is a substring of object's name,\n"
         "[\"desc\"] is a substring of object's description,\n"
         "asset [\"type\"] corresponds to the string name of the object's asset type\n"
-        "[\"limit\"] sets item count limit in reply, maximum and default is 100\n"
+        "[\"limit\"] sets item count limit in result set (default unlimited)\n"
         "[\"filter_links\"]: EXCLUDE_LINKS - don't show links, ONLY_LINKS - only show links, INCLUDE_LINKS - show links too (default)",
-        &LLInventoryListener::collectDescendentsIf,
+        &LLInventoryListener::collectDescendantsIf,
         llsd::map("folder_id", LLSD(), "reply", LLSD()));
- }
 
-
-void add_item_info(LLEventAPI::Response& response, LLViewerInventoryItem* item)
-{
-    response["items"].insert(item->getUUID().asString(),
-                             llsd::map("name", item->getName(),
-                                       "parent_id", item->getParentUUID(),
-                                       "desc", item->getDescription(),
-                                       "inv_type", LLInventoryType::lookup(item->getInventoryType()),
-                                       "asset_type", LLAssetType::lookup(item->getType()),
-                                       "creation_date", (S32) item->getCreationDate(),
-                                       "asset_id", item->getAssetUUID(),
-                                       "is_link", item->getIsLinkType(),
-                                       "linked_id", item->getLinkedUUID()));
+/*==========================================================================*|
+    add("getSingle",
+        "Return LLSD [\"single\"] for a single folder or item from the specified\n"
+        "[\"result\"] key at the specified 0-relative [\"index\"].",
+        &LLInventoryListener::getSingle,
+        llsd::map("result", LLSD::Integer(), "index", LLSD::Integer(),
+                  "reply", LLSD::String()));
+|*==========================================================================*/
+
+    add("getSlice",
+        stringize(
+        "Return an LLSD array [\"slice\"] from the specified [\"result\"] key\n"
+        "starting at 0-relative [\"index\"] with (up to) [\"count\"] entries.\n"
+        "count is limited to ", MAX_ITEM_LIMIT, " (default and max)."),
+        &LLInventoryListener::getSlice,
+        llsd::map("result", LLSD::Integer(), "index", LLSD::Integer(),
+                  "reply", LLSD::String()));
+
+    add("closeResult",
+        "Release resources associated with specified [\"result\"] key,\n"
+        "or keys if [\"result\"] is an array.",
+        &LLInventoryListener::closeResult,
+        llsd::map("result", LLSD()));
 }
 
-void add_cat_info(LLEventAPI::Response &response, LLViewerInventoryCategory *cat)
+// This abstract base class defines the interface for CatResultSet and
+// ItemResultSet. It isa LLIntTracker so we can pass its unique int key to a
+// consuming script via LLSD.
+struct InvResultSet: public LLIntTracker<InvResultSet>
 {
-    response["categories"].insert(cat->getUUID().asString(),
-                                  llsd::map("name", cat->getName(),
-                                            "parent_id", cat->getParentUUID(),
-                                            "type", LLFolderType::lookup(cat->getPreferredType())));
-}
+    // Get the length of the result set. Indexes are 0-relative.
+    virtual int getLength() const = 0;
+/*==========================================================================*|
+    // Retrieve LLSD corresponding to a single entry from the result set,
+    // with index validation.
+    LLSD getSingle(int index) const
+    {
+        if (0 <= index && index < getLength())
+        {
+            return getSingle_(index);
+        }
+        else
+        {
+            return {};
+        }
+    }
+|*==========================================================================*/
+    // Retrieve LLSD corresponding to a single entry from the result set,
+    // once we're sure the index is valid.
+    virtual LLSD getSingle(int index) const = 0;
+    // Retrieve LLSD corresponding to a "slice" of the result set: a
+    // contiguous sub-array starting at index. The returned LLSD array might
+    // be shorter than count entries if count > MAX_ITEM_LIMIT, or if the
+    // specified slice contains the end of the result set.
+    LLSD getSlice(int index, int count) const
+    {
+        // only call getLength() once
+        auto length = getLength();
+        // Adjust bounds [start, end) to overlap the actual result set from
+        // [0, getLength()). Permit negative index; e.g. with a result set
+        // containing 5 entries, getSlice(-2, 5) will adjust start to 0 and
+        // end to 3.
+        int start = llclamp(index, 0, length);
+        // Constrain count to MAX_ITEM_LIMIT even before clamping end.
+        int end = llclamp(index + llclamp(count, 0, MAX_ITEM_LIMIT), 0, length);
+        LLSD result{ LLSD::emptyArray() };
+        // beware of count == 0, or an [index, count) range that doesn't even
+        // overlap [0, length) at all
+        if (end > start)
+        {
+            // right away expand the result array to the size we'll need
+            result[end - 1] = LLSD();
+            for (int i = start; i < end; ++i)
+            {
+                result[i] = getSingle(i);
+            }
+        }
+        return result;
+    }
+
+    /*---------------- the rest is solely for debug logging ----------------*/
+    std::string mName;
+
+    friend std::ostream& operator<<(std::ostream& out, const InvResultSet& self)
+    {
+        return out << "InvResultSet(" << self.mName << ", " << self.getKey() << ")";
+    }
+
+    InvResultSet(const std::string& name):
+        mName(name)
+    {
+        LL_DEBUGS("Lua") << *this << LL_ENDL;
+    }
+    virtual ~InvResultSet()
+    {
+        // We want to be able to observe that the consuming script uses
+        // LL.setdtor() to eventually destroy each of these InvResultSets.
+        LL_DEBUGS("Lua") << "~" << *this << LL_ENDL;
+    }
+};
 
-void add_objects_info(LLEventAPI::Response& response, LLInventoryModel::cat_array_t cat_array, LLInventoryModel::item_array_t item_array)
+// This struct captures (possibly large) category results from
+// getDirectDescendants() and collectDescendantsIf().
+struct CatResultSet: public InvResultSet
 {
-    for (auto &p : item_array)
+    CatResultSet(): InvResultSet("categories") {}
+    LLInventoryModel::cat_array_t mCategories;
+
+    int getLength() const override { return narrow(mCategories.size()); }
+    LLSD getSingle(int index) const override
     {
-        add_item_info(response, p);
+        auto cat = mCategories[index];
+        return llsd::map("name", cat->getName(),
+                         "parent_id", cat->getParentUUID(),
+                         "type", LLFolderType::lookup(cat->getPreferredType()));
     }
-    for (auto &p : cat_array)
+};
+
+// This struct captures (possibly large) item results from
+// getDirectDescendants() and collectDescendantsIf().
+struct ItemResultSet: public InvResultSet
+{
+    ItemResultSet(): InvResultSet("items") {}
+    LLInventoryModel::item_array_t mItems;
+
+    int getLength() const override { return narrow(mItems.size()); }
+    LLSD getSingle(int index) const override
     {
-        add_cat_info(response, p);
+        auto item = mItems[index];
+        return llsd::map("name", item->getName(),
+                         "parent_id", item->getParentUUID(),
+                         "desc", item->getDescription(),
+                         "inv_type", LLInventoryType::lookup(item->getInventoryType()),
+                         "asset_type", LLAssetType::lookup(item->getType()),
+                         "creation_date", LLSD::Integer(item->getCreationDate()),
+                         "asset_id", item->getAssetUUID(),
+                         "is_link", item->getIsLinkType(),
+                         "linked_id", item->getLinkedUUID());
     }
-}
+};
 
 void LLInventoryListener::getItemsInfo(LLSD const &data)
 {
     Response response(LLSD(), data);
 
+    auto catresult = new CatResultSet;
+    auto itemresult = new ItemResultSet;
+
     uuid_vec_t ids = LLSDParam<uuid_vec_t>(data["item_ids"]);
     for (auto &it : ids)
     {
         LLViewerInventoryItem* item = gInventory.getItem(it);
         if (item)
         {
-            add_item_info(response, item);
+            itemresult->mItems.push_back(item);
         }
         else
         {
             LLViewerInventoryCategory *cat = gInventory.getCategory(it);
             if (cat)
             {
-                add_cat_info(response, cat);
+                catresult->mCategories.push_back(cat);
             }
         }
     }
+    response["categories"] = catresult->getKey();
+    response["items"] = itemresult->getKey();
 }
 
 void LLInventoryListener::getFolderTypeNames(LLSD const &data)
@@ -151,14 +264,21 @@ void LLInventoryListener::getBasicFolderID(LLSD const &data)
 }
 
 
-void LLInventoryListener::getDirectDescendents(LLSD const &data)
+void LLInventoryListener::getDirectDescendants(LLSD const &data)
 {
     Response response(LLSD(), data);
     LLInventoryModel::cat_array_t* cats; 
     LLInventoryModel::item_array_t* items;
     gInventory.getDirectDescendentsOf(data["folder_id"], cats, items);
 
-    add_objects_info(response, *cats, *items);
+    auto catresult = new CatResultSet;
+    auto itemresult = new ItemResultSet;
+
+    catresult->mCategories = *cats;
+    itemresult->mItems = *items;
+
+    response["categories"] = catresult->getKey();
+    response["items"] = itemresult->getKey();
 }
 
 struct LLFilteredCollector : public LLInventoryCollectFunctor
@@ -173,7 +293,11 @@ struct LLFilteredCollector : public LLInventoryCollectFunctor
     LLFilteredCollector(LLSD const &data);
     virtual ~LLFilteredCollector() {}
     virtual bool operator()(LLInventoryCategory *cat, LLInventoryItem *item) override;
-    virtual bool exceedsLimit() override { return (mItemLimit <= mItemCount); };
+    virtual bool exceedsLimit() override
+    {
+        // mItemLimit == 0 means unlimited
+        return (mItemLimit && mItemLimit <= mItemCount);
+    }
 
   protected:
     bool checkagainstType(LLInventoryCategory *cat, LLInventoryItem *item);
@@ -189,7 +313,7 @@ struct LLFilteredCollector : public LLInventoryCollectFunctor
     S32 mItemCount;
 };
 
-void LLInventoryListener::collectDescendentsIf(LLSD const &data)
+void LLInventoryListener::collectDescendantsIf(LLSD const &data)
 {
     Response response(LLSD(), data);
     LLUUID folder_id(data["folder_id"].asUUID());
@@ -198,20 +322,63 @@ void LLInventoryListener::collectDescendentsIf(LLSD const &data)
     {
         return response.error(stringize("Folder ", std::quoted(data["folder_id"].asString()), " was not found"));
     }
-    LLInventoryModel::cat_array_t  cat_array;
-    LLInventoryModel::item_array_t item_array;
+    auto catresult = new CatResultSet;
+    auto itemresult = new ItemResultSet;
 
     LLFilteredCollector collector = LLFilteredCollector(data);
 
-    gInventory.collectDescendentsIf(folder_id, cat_array, item_array, LLInventoryModel::EXCLUDE_TRASH, collector);
+    // Populate results directly into the catresult and itemresult arrays.
+    // TODO: sprinkle count-based coroutine yields into the real
+    // collectDescendentsIf() method so it doesn't steal too many cycles.
+    gInventory.collectDescendentsIf(
+        folder_id,
+        catresult->mCategories,
+        itemresult->mItems,
+        LLInventoryModel::EXCLUDE_TRASH,
+        collector);
+
+    response["categories"] = catresult->getKey();
+    response["items"] = itemresult->getKey();
+}
+
+/*==========================================================================*|
+void LLInventoryListener::getSingle(LLSD const& data)
+{
+    auto result = InvResultSet::getInstance(data["result"]);
+    sendReply(llsd::map("single", result->getSingle(data["index"])), data);
+}
+|*==========================================================================*/
 
-    add_objects_info(response, cat_array, item_array);
+void LLInventoryListener::getSlice(LLSD const& data)
+{
+    auto result = InvResultSet::getInstance(data["result"]);
+    int count = data.has("count")? data["count"].asInteger() : MAX_ITEM_LIMIT;
+    LL_DEBUGS("Lua") << *result << ".getSlice(" << data["index"].asInteger()
+                     << ", " << count << ')' << LL_ENDL;
+    sendReply(llsd::map("slice", result->getSlice(data["index"], count)), data);
+}
+
+void LLInventoryListener::closeResult(LLSD const& data)
+{
+    LLSD results = data["result"];
+    if (results.isInteger())
+    {
+        results = llsd::array(results);
+    }
+    for (const auto& result : llsd::inArray(results))
+    {
+        auto ptr = InvResultSet::getInstance(result);
+        if (ptr)
+        {
+            delete ptr.get();
+        }
+    }
 }
 
 LLFilteredCollector::LLFilteredCollector(LLSD const &data) :
     mType(LLAssetType::EType::AT_UNKNOWN),
     mLinkFilter(INCLUDE_LINKS),
-    mItemLimit(MAX_ITEM_LIMIT),
+    mItemLimit(0),
     mItemCount(0)
 {
 
@@ -235,7 +402,7 @@ LLFilteredCollector::LLFilteredCollector(LLSD const &data) :
     }
     if (data["limit"].isInteger())
     {
-        mItemLimit = llclamp(data["limit"].asInteger(), 1, MAX_ITEM_LIMIT);
+        mItemLimit = std::max(data["limit"].asInteger(), 1);
     }
 }