#3969 Make inventory creation from cache faster

Still has a lot of space for improvements, but should be 2.5 times faster

8 files changed, 220 insertions, 144 deletions

diff --git a/indra/llinventory/llinventory.cpp b/indra/llinventory/llinventory.cpp
index fe60800700..3defad8f3b 100644
--- a/indra/llinventory/llinventory.cpp
+++ b/indra/llinventory/llinventory.cpp
@@ -995,6 +995,7 @@ bool LLInventoryItem::fromLLSD(const LLSD& sd, bool is_new)
     // TODO - figure out if this should be moved into the noclobber fields above
     mThumbnailUUID.setNull();
     mFavorite = false;
+    mPermissions.init(LLUUID::null, LLUUID::null, LLUUID::null, LLUUID::null);
 
     // iterate as map to avoid making unnecessary temp copies of everything
     LLSD::map_const_iterator i, end;
@@ -1053,7 +1054,7 @@ bool LLInventoryItem::fromLLSD(const LLSD& sd, bool is_new)
 
         if (i->first == INV_PERMISSIONS_LABEL)
         {
-            mPermissions = ll_permissions_from_sd(i->second);
+            mPermissions.importLLSD(i->second);
             continue;
         }
 
@@ -1522,53 +1523,68 @@ void LLInventoryCategory::exportLLSD(LLSD& cat_data) const
     }
 }
 
-bool LLInventoryCategory::importLLSD(const LLSD& cat_data)
+bool LLInventoryCategory::importLLSDMap(const LLSD& cat_data)
 {
-    if (cat_data.has(INV_FOLDER_ID_LABEL))
+    LLSD::map_const_iterator i, end;
+    end = cat_data.endMap();
+    for ( i = cat_data.beginMap(); i != end; ++i)
+    {
+        importLLSD(i->first, i->second);
+    }
+    return true;
+}
+
+bool LLInventoryCategory::importLLSD(const std::string& label, const LLSD& value)
+{
+    if (label == INV_FOLDER_ID_LABEL)
     {
-        setUUID(cat_data[INV_FOLDER_ID_LABEL].asUUID());
+        setUUID(value.asUUID());
+        return true;
     }
-    if (cat_data.has(INV_PARENT_ID_LABEL))
+    else if (label == INV_PARENT_ID_LABEL)
     {
-        setParent(cat_data[INV_PARENT_ID_LABEL].asUUID());
+        setParent(value.asUUID());
+        return true;
     }
-    if (cat_data.has(INV_ASSET_TYPE_LABEL))
+    else if (label == INV_ASSET_TYPE_LABEL)
     {
-        setType(LLAssetType::lookup(cat_data[INV_ASSET_TYPE_LABEL].asString()));
+        setType(LLAssetType::lookup(value.asString()));
+        return true;
     }
-    if (cat_data.has(INV_PREFERRED_TYPE_LABEL))
+    else if (label == INV_PREFERRED_TYPE_LABEL)
     {
-        setPreferredType(LLFolderType::lookup(cat_data[INV_PREFERRED_TYPE_LABEL].asString()));
+        setPreferredType(LLFolderType::lookup(value.asString()));
+        return true;
     }
-    if (cat_data.has(INV_THUMBNAIL_LABEL))
+    else if (label == INV_THUMBNAIL_LABEL)
     {
         LLUUID thumbnail_uuid;
-        const LLSD &thumbnail_data = cat_data[INV_THUMBNAIL_LABEL];
-        if (thumbnail_data.has(INV_ASSET_ID_LABEL))
+        if (value.has(INV_ASSET_ID_LABEL))
         {
-            thumbnail_uuid = thumbnail_data[INV_ASSET_ID_LABEL].asUUID();
+            thumbnail_uuid = value[INV_ASSET_ID_LABEL].asUUID();
         }
         setThumbnailUUID(thumbnail_uuid);
+        return true;
     }
-    if (cat_data.has(INV_FAVORITE_LABEL))
+    if (label == INV_FAVORITE_LABEL)
     {
         bool favorite = false;
-        const LLSD& favorite_data = cat_data[INV_FAVORITE_LABEL];
-        if (favorite_data.has(INV_TOGGLED_LABEL))
+        if (value.has(INV_TOGGLED_LABEL))
         {
-            favorite = favorite_data[INV_TOGGLED_LABEL].asBoolean();
+            favorite = value[INV_TOGGLED_LABEL].asBoolean();
         }
         setFavorite(favorite);
     }
-    if (cat_data.has(INV_NAME_LABEL))
+    else if (label == INV_NAME_LABEL)
     {
-        mName = cat_data[INV_NAME_LABEL].asString();
+        mName = value.asString();
         LLStringUtil::replaceNonstandardASCII(mName, ' ');
         LLStringUtil::replaceChar(mName, '|', ' ');
+        return true;
     }
-
-    return true;
+    return false;
 }
+
 ///----------------------------------------------------------------------------
 /// Local function definitions for testing purposes
 ///----------------------------------------------------------------------------
diff --git a/indra/llinventory/llinventory.h b/indra/llinventory/llinventory.h
index 17670d2ea1..181c46226c 100644
--- a/indra/llinventory/llinventory.h
+++ b/indra/llinventory/llinventory.h
@@ -274,7 +274,8 @@ public:
     virtual bool exportLegacyStream(std::ostream& output_stream, bool include_asset_key = true) const;
 
     virtual void exportLLSD(LLSD& sd) const;
-    bool importLLSD(const LLSD& cat_data);
+    bool importLLSDMap(const LLSD& cat_data);
+    virtual bool importLLSD(const std::string& label, const LLSD& value);
     //--------------------------------------------------------------------
     // Member Variables
     //--------------------------------------------------------------------
diff --git a/indra/llinventory/llpermissions.cpp b/indra/llinventory/llpermissions.cpp
index d800ca02c9..ebf7445c65 100644
--- a/indra/llinventory/llpermissions.cpp
+++ b/indra/llinventory/llpermissions.cpp
@@ -704,6 +704,79 @@ bool LLPermissions::exportLegacyStream(std::ostream& output_stream) const
     return true;
 }
 
+static const std::string PERM_CREATOR_ID_LABEL("creator_id");
+static const std::string PERM_OWNER_ID_LABEL("owner_id");
+static const std::string PERM_LAST_OWNER_ID_LABEL("last_owner_id");
+static const std::string PERM_GROUP_ID_LABEL("group_id");
+static const std::string PERM_IS_OWNER_GROUP_LABEL("is_owner_group");
+static const std::string PERM_BASE_MASK_LABEL("base_mask");
+static const std::string PERM_OWNER_MASK_LABEL("owner_mask");
+static const std::string PERM_GROUP_MASK_LABEL("group_mask");
+static const std::string PERM_EVERYONE_MASK_LABEL("everyone_mask");
+static const std::string PERM_NEXT_OWNER_MASK_LABEL("next_owner_mask");
+
+void LLPermissions::importLLSD(const LLSD& sd_perm)
+{
+    LLSD::map_const_iterator i, end;
+    end = sd_perm.endMap();
+    for (i = sd_perm.beginMap(); i != end; ++i)
+    {
+        const std::string& label = i->first;
+        if (label == PERM_CREATOR_ID_LABEL)
+        {
+            mCreator = i->second.asUUID();
+            continue;
+        }
+        if (label == PERM_OWNER_ID_LABEL)
+        {
+            mOwner = i->second.asUUID();
+            continue;
+        }
+        if (label == PERM_LAST_OWNER_ID_LABEL)
+        {
+            mLastOwner = i->second.asUUID();
+            continue;
+        }
+        if (label == PERM_GROUP_ID_LABEL)
+        {
+            mGroup = i->second.asUUID();
+            continue;
+        }
+        if (label == PERM_BASE_MASK_LABEL)
+        {
+            PermissionMask mask = i->second.asInteger();
+            mMaskBase = mask;
+            continue;
+        }
+        if (label == PERM_OWNER_MASK_LABEL)
+        {
+            PermissionMask mask = i->second.asInteger();
+            mMaskOwner = mask;
+            continue;
+        }
+        if (label == PERM_EVERYONE_MASK_LABEL)
+        {
+            PermissionMask mask = i->second.asInteger();
+            mMaskEveryone = mask;
+            continue;
+        }
+        if (label == PERM_GROUP_MASK_LABEL)
+        {
+            PermissionMask mask = i->second.asInteger();
+            mMaskGroup = mask;
+            continue;
+        }
+        if (label == PERM_NEXT_OWNER_MASK_LABEL)
+        {
+            PermissionMask mask = i->second.asInteger();
+            mMaskNextOwner = mask;
+            continue;
+        }
+    }
+
+    fix();
+}
+
 bool LLPermissions::operator==(const LLPermissions &rhs) const
 {
     return
@@ -998,16 +1071,6 @@ std::string mask_to_string(U32 mask)
 ///----------------------------------------------------------------------------
 /// exported functions
 ///----------------------------------------------------------------------------
-static const std::string PERM_CREATOR_ID_LABEL("creator_id");
-static const std::string PERM_OWNER_ID_LABEL("owner_id");
-static const std::string PERM_LAST_OWNER_ID_LABEL("last_owner_id");
-static const std::string PERM_GROUP_ID_LABEL("group_id");
-static const std::string PERM_IS_OWNER_GROUP_LABEL("is_owner_group");
-static const std::string PERM_BASE_MASK_LABEL("base_mask");
-static const std::string PERM_OWNER_MASK_LABEL("owner_mask");
-static const std::string PERM_GROUP_MASK_LABEL("group_mask");
-static const std::string PERM_EVERYONE_MASK_LABEL("everyone_mask");
-static const std::string PERM_NEXT_OWNER_MASK_LABEL("next_owner_mask");
 
 LLSD ll_create_sd_from_permissions(const LLPermissions& perm)
 {
@@ -1032,25 +1095,6 @@ void ll_fill_sd_from_permissions(LLSD& rv, const LLPermissions& perm)
 LLPermissions ll_permissions_from_sd(const LLSD& sd_perm)
 {
     LLPermissions rv;
-    rv.init(
-        sd_perm[PERM_CREATOR_ID_LABEL].asUUID(),
-        sd_perm[PERM_OWNER_ID_LABEL].asUUID(),
-        sd_perm[PERM_LAST_OWNER_ID_LABEL].asUUID(),
-        sd_perm[PERM_GROUP_ID_LABEL].asUUID());
-
-    // We do a cast to U32 here since LLSD does not attempt to
-    // represent unsigned ints.
-    PermissionMask mask;
-    mask = (U32)(sd_perm[PERM_BASE_MASK_LABEL].asInteger());
-    rv.setMaskBase(mask);
-    mask = (U32)(sd_perm[PERM_OWNER_MASK_LABEL].asInteger());
-    rv.setMaskOwner(mask);
-    mask = (U32)(sd_perm[PERM_EVERYONE_MASK_LABEL].asInteger());
-    rv.setMaskEveryone(mask);
-    mask = (U32)(sd_perm[PERM_GROUP_MASK_LABEL].asInteger());
-    rv.setMaskGroup(mask);
-    mask = (U32)(sd_perm[PERM_NEXT_OWNER_MASK_LABEL].asInteger());
-    rv.setMaskNext(mask);
-    rv.fix();
+    rv.importLLSD(sd_perm);
     return rv;
 }
diff --git a/indra/llinventory/llpermissions.h b/indra/llinventory/llpermissions.h
index f3e10af25c..82cdc03727 100644
--- a/indra/llinventory/llpermissions.h
+++ b/indra/llinventory/llpermissions.h
@@ -299,6 +299,8 @@ public:
     bool    importLegacyStream(std::istream& input_stream);
     bool    exportLegacyStream(std::ostream& output_stream) const;
 
+    void importLLSD(const LLSD& sd_perm);
+
     bool operator==(const LLPermissions &rhs) const;
     bool operator!=(const LLPermissions &rhs) const;
 
diff --git a/indra/llinventory/tests/inventorymisc_test.cpp b/indra/llinventory/tests/inventorymisc_test.cpp
index e41500b4c5..f11a4c3bf7 100644
--- a/indra/llinventory/tests/inventorymisc_test.cpp
+++ b/indra/llinventory/tests/inventorymisc_test.cpp
@@ -518,7 +518,7 @@ namespace tut
         file.close();
 
         LLPointer<LLInventoryCategory> src2 = new LLInventoryCategory();
-        src2->importLLSD(s_item);
+        src2->importLLSDMap(s_item);
 
         ensure_equals("1.item id::getUUID() failed", src1->getUUID(), src2->getUUID());
         ensure_equals("2.parent::getParentUUID() failed", src1->getParentUUID(), src2->getParentUUID());
diff --git a/indra/newview/llinventorymodel.cpp b/indra/newview/llinventorymodel.cpp
index 117f2d1adb..2dfc3b014f 100644
--- a/indra/newview/llinventorymodel.cpp
+++ b/indra/newview/llinventorymodel.cpp
@@ -78,7 +78,7 @@
 
 // Increment this if the inventory contents change in a non-backwards-compatible way.
 // For viewer 2, the addition of link items makes a pre-viewer-2 cache incorrect.
-const S32 LLInventoryModel::sCurrentInvCacheVersion = 4;
+const S32 LLInventoryModel::sCurrentInvCacheVersion = 5;
 bool LLInventoryModel::sFirstTimeInViewer2 = true;
 
 S32 LLInventoryModel::sPendingSystemFolders = 0;
@@ -3384,7 +3384,7 @@ bool LLInventoryModel::loadFromFile(const std::string& filename,
     }
     LL_INFOS(LOG_INV) << "loading inventory from: (" << filename << ")" << LL_ENDL;
 
-    llifstream file(filename.c_str());
+    llifstream file(filename.c_str(), std::ifstream::in | std::ifstream::binary);
 
     if (!file.is_open())
     {
@@ -3393,80 +3393,92 @@ bool LLInventoryModel::loadFromFile(const std::string& filename,
     }
 
     is_cache_obsolete = true; // Obsolete until proven current
-
-    //U64 lines_count = 0U;
-    std::string line;
-    LLPointer<LLSDParser> parser = new LLSDNotationParser();
-    while (std::getline(file, line))
+    U32 value_nbo = 0;
+    file.read((char*)&value_nbo, sizeof(U32));
+    if (file.fail())
+    {
+        LL_WARNS(LOG_INV) << "Failed to read cache version. Unable to load inventory from: " << filename << LL_ENDL;
+    }
+    else
     {
-        LLSD s_item;
-        std::istringstream iss(line);
-        if (parser->parse(iss, s_item, line.length()) == LLSDParser::PARSE_FAILURE)
+        S32 version = (S32)ntohl(value_nbo);
+        if (version == sCurrentInvCacheVersion)
         {
-            LL_WARNS(LOG_INV)<< "Parsing inventory cache failed" << LL_ENDL;
-            break;
+            // Cache is up to date
+            is_cache_obsolete = false;
         }
-
-        if (s_item.has("inv_cache_version"))
+        else
         {
-            S32 version = s_item["inv_cache_version"].asInteger();
-            if (version == sCurrentInvCacheVersion)
-            {
-                // Cache is up to date
-                is_cache_obsolete = false;
-                continue;
-            }
-            else
-            {
-                LL_WARNS(LOG_INV)<< "Inventory cache is out of date" << LL_ENDL;
-                break;
-            }
+            LL_WARNS(LOG_INV) << "Inventory cache is out of date" << LL_ENDL;
         }
-        else if (s_item.has("cat_id"))
+    }
+
+    LLSD inventory;
+    if (!is_cache_obsolete)
+    {
+        LLPointer<LLSDParser> parser = new LLSDBinaryParser();
+
+        if (parser->parse(file, inventory, LLSDSerialize::SIZE_UNLIMITED) == LLSDParser::PARSE_FAILURE)
         {
-            if (is_cache_obsolete)
-                break;
+            is_cache_obsolete = true;
+            LL_WARNS(LOG_INV) << "Parsing inventory cache failed" << LL_ENDL;
+        }
+    }
 
-            LLPointer<LLViewerInventoryCategory> inv_cat = new LLViewerInventoryCategory(LLUUID::null);
-            if(inv_cat->importLLSD(s_item))
+    if (!is_cache_obsolete)
+    {
+        const LLSD& llsd_cats = inventory["categories"];
+        if (llsd_cats.isArray())
+        {
+            LLSD::array_const_iterator iter = llsd_cats.beginArray();
+            LLSD::array_const_iterator end = llsd_cats.endArray();
+            for (; iter != end; ++iter)
             {
-                categories.push_back(inv_cat);
+                LLPointer<LLViewerInventoryCategory> inv_cat = new LLViewerInventoryCategory(LLUUID::null);
+                if (inv_cat->importLLSDMap(*iter))
+                {
+                    categories.push_back(inv_cat);
+                }
             }
         }
-        else if (s_item.has("item_id"))
-        {
-            if (is_cache_obsolete)
-                break;
 
-            LLPointer<LLViewerInventoryItem> inv_item = new LLViewerInventoryItem;
-            if( inv_item->fromLLSD(s_item) )
+        const LLSD& llsd_items = inventory["items"];
+        if (llsd_items.isArray())
+        {
+            LLSD::array_const_iterator iter = llsd_items.beginArray();
+            LLSD::array_const_iterator end = llsd_items.endArray();
+            for (; iter != end; ++iter)
             {
-                if(inv_item->getUUID().isNull())
-                {
-                    LL_DEBUGS(LOG_INV) << "Ignoring inventory with null item id: "
-                        << inv_item->getName() << LL_ENDL;
-                }
-                else
+                LLPointer<LLViewerInventoryItem> inv_item = new LLViewerInventoryItem;
+                if (inv_item->fromLLSD(*iter))
                 {
-                    if (inv_item->getType() == LLAssetType::AT_UNKNOWN)
+                    if (inv_item->getUUID().isNull())
                     {
-                        cats_to_update.insert(inv_item->getParentUUID());
+                        LL_DEBUGS(LOG_INV) << "Ignoring inventory with null item id: "
+                            << inv_item->getName() << LL_ENDL;
                     }
                     else
                     {
-                        items.push_back(inv_item);
+                        if (inv_item->getType() == LLAssetType::AT_UNKNOWN)
+                        {
+                            cats_to_update.insert(inv_item->getParentUUID());
+                        }
+                        else
+                        {
+                            items.push_back(inv_item);
+                        }
                     }
                 }
+
+                //      TODO(brad) - figure out how to reenable this without breaking everything else
+                //      static constexpr U64 BATCH_SIZE = 512U;
+                //      if ((++lines_count % BATCH_SIZE) == 0)
+                //      {
+                //          // SL-19968 - make sure message system code gets a chance to run every so often
+                //          pump_idle_startup_network();
+                //      }
             }
         }
-
-//      TODO(brad) - figure out how to reenable this without breaking everything else
-//      static constexpr U64 BATCH_SIZE = 512U;
-//      if ((++lines_count % BATCH_SIZE) == 0)
-//      {
-//          // SL-19968 - make sure message system code gets a chance to run every so often
-//          pump_idle_startup_network();
-//      }
     }
 
     file.close();
@@ -3489,58 +3501,54 @@ bool LLInventoryModel::saveToFile(const std::string& filename,
 
     try
     {
-        llofstream fileXML(filename.c_str());
-        if (!fileXML.is_open())
+        llofstream fileSD(filename.c_str(), std::ios_base::out | std::ios_base::binary);
+        if (!fileSD.is_open())
         {
             LL_WARNS(LOG_INV) << "Failed to open file. Unable to save inventory to: " << filename << LL_ENDL;
             return false;
         }
-
-        LLSD cache_ver;
-        cache_ver["inv_cache_version"] = sCurrentInvCacheVersion;
-
-        if (fileXML.fail())
+        U32 value_nbo = htonl(sCurrentInvCacheVersion);
+        fileSD.write((const char*)(&value_nbo), sizeof(U32));
+        if (fileSD.fail())
         {
-            LL_WARNS(LOG_INV) << "Failed to write cache version to file. Unable to save inventory to: " << filename << LL_ENDL;
+            LL_WARNS(LOG_INV) << "Failed to write cache. Unable to save inventory to: " << filename << LL_ENDL;
             return false;
         }
 
-        fileXML << LLSDOStreamer<LLSDNotationFormatter>(cache_ver) << std::endl;
+        LLSD inventory;
+        inventory["categories"] = LLSD::emptyArray();
+        LLSD& cat_array = inventory["categories"];
 
         S32 cat_count = 0;
         for (auto& cat : categories)
         {
             if (cat->getVersion() != LLViewerInventoryCategory::VERSION_UNKNOWN)
             {
-                LLSD sd = LLSD::emptyMap();
+                LLSD sd;
                 cat->exportLLSD(sd);
-                fileXML << LLSDOStreamer<LLSDNotationFormatter>(sd) << std::endl;
+                cat_array.append(sd);
                 cat_count++;
             }
-
-            if (fileXML.fail())
-            {
-                LL_WARNS(LOG_INV) << "Failed to write a folder to file. Unable to save inventory to: " << filename << LL_ENDL;
-                return false;
-            }
         }
 
+        inventory["items"] = LLSD::emptyArray();
+        LLSD& item_array = inventory["items"];
         auto it_count = items.size();
         for (auto& item : items)
         {
-            LLSD sd = LLSD::emptyMap();
+            LLSD sd;
             item->asLLSD(sd);
-            fileXML << LLSDOStreamer<LLSDNotationFormatter>(sd) << std::endl;
-
-            if (fileXML.fail())
-            {
-                LL_WARNS(LOG_INV) << "Failed to write an item to file. Unable to save inventory to: " << filename << LL_ENDL;
-                return false;
-            }
+            item_array.append(sd);
+        }
+        fileSD << LLSDOStreamer<LLSDBinaryFormatter>(inventory) << std::endl;
+        if (fileSD.fail())
+        {
+            LL_WARNS(LOG_INV) << "Failed to write cache. Unable to save inventory to: " << filename << LL_ENDL;
+            return false;
         }
-        fileXML.flush();
+        fileSD.flush();
 
-        fileXML.close();
+        fileSD.close();
 
         LL_INFOS(LOG_INV) << "Inventory saved: " << (S32)cat_count << " categories, " << (S32)it_count << " items." << LL_ENDL;
     }
diff --git a/indra/newview/llviewerinventory.cpp b/indra/newview/llviewerinventory.cpp
index 5f61aeaf13..6155058f14 100644
--- a/indra/newview/llviewerinventory.cpp
+++ b/indra/newview/llviewerinventory.cpp
@@ -761,18 +761,23 @@ void LLViewerInventoryCategory::exportLLSD(LLSD & cat_data) const
     cat_data[INV_VERSION] = mVersion;
 }
 
-bool LLViewerInventoryCategory::importLLSD(const LLSD& cat_data)
+bool LLViewerInventoryCategory::importLLSD(const std::string& label, const LLSD& value)
 {
-    LLInventoryCategory::importLLSD(cat_data);
-    if (cat_data.has(INV_OWNER_ID))
+    if (LLInventoryCategory::importLLSD(label, value))
     {
-        mOwnerID = cat_data[INV_OWNER_ID].asUUID();
+        return true;
     }
-    if (cat_data.has(INV_VERSION))
+    else if (label == INV_OWNER_ID)
     {
-        setVersion(cat_data[INV_VERSION].asInteger());
+        mOwnerID = value.asUUID();
+        return true;
     }
-    return true;
+    else if (label == INV_VERSION)
+    {
+        setVersion(value.asInteger());
+        return true;
+    }
+    return false;
 }
 
 bool LLViewerInventoryCategory::acceptItem(LLInventoryItem* inv_item)
diff --git a/indra/newview/llviewerinventory.h b/indra/newview/llviewerinventory.h
index 5cd31353f8..0dfbf0cced 100644
--- a/indra/newview/llviewerinventory.h
+++ b/indra/newview/llviewerinventory.h
@@ -233,7 +233,7 @@ public:
     S32 getViewerDescendentCount() const;
 
     virtual void exportLLSD(LLSD &sd) const;
-    virtual bool importLLSD(const LLSD& cat_data);
+    virtual bool importLLSD(const std::string& label, const LLSD& value);
 
     void determineFolderType();
     void changeType(LLFolderType::EType new_folder_type);