Merge remote-tracking branch 'origin/master' into brad/DRTVWR-550-mfa

1 files changed, 410 insertions, 0 deletions

diff --git a/indra/llfilesystem/lldiskcache.cpp b/indra/llfilesystem/lldiskcache.cpp
new file mode 100644
index 0000000000..ee43a599f7
--- /dev/null
+++ b/indra/llfilesystem/lldiskcache.cpp
@@ -0,0 +1,410 @@
+/**
+ * @file lldiskcache.cpp
+ * @brief The disk cache implementation.
+ *
+ * Note: Rather than keep the top level function comments up
+ * to date in both the source and header files, I elected to
+ * only have explicit comments about each function and variable
+ * in the header - look there for details. The same is true for
+ * description of how this code is supposed to work.
+ *
+ * $LicenseInfo:firstyear=2009&license=viewerlgpl$
+ * Second Life Viewer Source Code
+ * Copyright (C) 2020, Linden Research, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation;
+ * version 2.1 of the License only.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ * Linden Research, Inc., 945 Battery Street, San Francisco, CA  94111  USA
+ * $/LicenseInfo$
+ */
+
+#include "linden_common.h"
+#include "llapp.h"
+#include "llassettype.h"
+#include "lldir.h"
+#include <boost/filesystem.hpp>
+#include <boost/range/iterator_range.hpp>
+#include <chrono>
+
+#include "lldiskcache.h"
+
+LLDiskCache::LLDiskCache(const std::string cache_dir,
+                         const uintmax_t max_size_bytes,
+                         const bool enable_cache_debug_info) :
+    mCacheDir(cache_dir),
+    mMaxSizeBytes(max_size_bytes),
+    mEnableCacheDebugInfo(enable_cache_debug_info)
+{
+    mCacheFilenamePrefix = "sl_cache";
+
+    LLFile::mkdir(cache_dir);
+}
+
+// WARNING: purge() is called by LLPurgeDiskCacheThread. As such it must
+// NOT touch any LLDiskCache data without introducing and locking a mutex!
+
+// Interaction through the filesystem itself should be safe. Let’s say thread
+// A is accessing the cache file for reading/writing and thread B is trimming
+// the cache. Let’s also assume using llifstream to open a file and
+// boost::filesystem::remove are not atomic (which will be pretty much the
+// case).
+
+// Now, A is trying to open the file using llifstream ctor. It does some
+// checks if the file exists and whatever else it might be doing, but has not
+// issued the call to the OS to actually open the file yet. Now B tries to
+// delete the file: If the file has been already marked as in use by the OS,
+// deleting the file will fail and B will continue with the next file. A can
+// safely continue opening the file. If the file has not yet been marked as in
+// use, B will delete the file. Now A actually wants to open it, operation
+// will fail, subsequent check via llifstream.is_open will fail, asset will
+// have to be re-requested. (Assuming here the viewer will actually handle
+// this situation properly, that can also happen if there is a file containing
+// garbage.)
+
+// Other situation: B is trimming the cache and A wants to read a file that is
+// about to get deleted. boost::filesystem::remove does whatever it is doing
+// before actually deleting the file. If A opens the file before the file is
+// actually gone, the OS call from B to delete the file will fail since the OS
+// will prevent this. B continues with the next file. If the file is already
+// gone before A finally gets to open it, this operation will fail and the
+// asset will have to be re-requested.
+void LLDiskCache::purge()
+{
+    if (mEnableCacheDebugInfo)
+    {
+        LL_INFOS() << "Total dir size before purge is " << dirFileSize(mCacheDir) << LL_ENDL;
+    }
+
+    boost::system::error_code ec;
+    auto start_time = std::chrono::high_resolution_clock::now();
+
+    typedef std::pair<std::time_t, std::pair<uintmax_t, std::string>> file_info_t;
+    std::vector<file_info_t> file_info;
+
+#if LL_WINDOWS
+    std::wstring cache_path(utf8str_to_utf16str(mCacheDir));
+#else
+    std::string cache_path(mCacheDir);
+#endif
+    if (boost::filesystem::is_directory(cache_path, ec) && !ec.failed())
+    {
+        for (auto& entry : boost::make_iterator_range(boost::filesystem::directory_iterator(cache_path, ec), {}))
+        {
+            if (boost::filesystem::is_regular_file(entry, ec) && !ec.failed())
+            {
+                if (entry.path().string().find(mCacheFilenamePrefix) != std::string::npos)
+                {
+                    uintmax_t file_size = boost::filesystem::file_size(entry, ec);
+                    if (ec.failed())
+                    {
+                        continue;
+                    }
+                    const std::string file_path = entry.path().string();
+                    const std::time_t file_time = boost::filesystem::last_write_time(entry, ec);
+                    if (ec.failed())
+                    {
+                        continue;
+                    }
+
+                    file_info.push_back(file_info_t(file_time, { file_size, file_path }));
+                }
+            }
+        }
+    }
+
+    std::sort(file_info.begin(), file_info.end(), [](file_info_t& x, file_info_t& y)
+    {
+        return x.first > y.first;
+    });
+
+    LL_INFOS() << "Purging cache to a maximum of " << mMaxSizeBytes << " bytes" << LL_ENDL;
+
+    uintmax_t file_size_total = 0;
+    for (file_info_t& entry : file_info)
+    {
+        file_size_total += entry.second.first;
+
+        std::string action = "";
+        if (file_size_total > mMaxSizeBytes)
+        {
+            action = "DELETE:";
+            boost::filesystem::remove(entry.second.second, ec);
+            if (ec.failed())
+            {
+                LL_WARNS() << "Failed to delete cache file " << entry.second.second << ": " << ec.message() << LL_ENDL;
+            }
+        }
+        else
+        {
+            action = "  KEEP:";
+        }
+
+        if (mEnableCacheDebugInfo)
+        {
+            // have to do this because of LL_INFO/LL_END weirdness
+            std::ostringstream line;
+
+            line << action << "  ";
+            line << entry.first << "  ";
+            line << entry.second.first << "  ";
+            line << entry.second.second;
+            line << " (" << file_size_total << "/" << mMaxSizeBytes << ")";
+            LL_INFOS() << line.str() << LL_ENDL;
+        }
+    }
+
+    if (mEnableCacheDebugInfo)
+    {
+        auto end_time = std::chrono::high_resolution_clock::now();
+        auto execute_time = std::chrono::duration_cast<std::chrono::milliseconds>(end_time - start_time).count();
+        LL_INFOS() << "Total dir size after purge is " << dirFileSize(mCacheDir) << LL_ENDL;
+        LL_INFOS() << "Cache purge took " << execute_time << " ms to execute for " << file_info.size() << " files" << LL_ENDL;
+    }
+}
+
+const std::string LLDiskCache::assetTypeToString(LLAssetType::EType at)
+{
+    /**
+     * Make use of the handy C++17  feature that allows
+     * for inline initialization of an std::map<>
+     */
+    typedef std::map<LLAssetType::EType, std::string> asset_type_to_name_t;
+    asset_type_to_name_t asset_type_to_name =
+    {
+        { LLAssetType::AT_TEXTURE, "TEXTURE" },
+        { LLAssetType::AT_SOUND, "SOUND" },
+        { LLAssetType::AT_CALLINGCARD, "CALLINGCARD" },
+        { LLAssetType::AT_LANDMARK, "LANDMARK" },
+        { LLAssetType::AT_SCRIPT, "SCRIPT" },
+        { LLAssetType::AT_CLOTHING, "CLOTHING" },
+        { LLAssetType::AT_OBJECT, "OBJECT" },
+        { LLAssetType::AT_NOTECARD, "NOTECARD" },
+        { LLAssetType::AT_CATEGORY, "CATEGORY" },
+        { LLAssetType::AT_LSL_TEXT, "LSL_TEXT" },
+        { LLAssetType::AT_LSL_BYTECODE, "LSL_BYTECODE" },
+        { LLAssetType::AT_TEXTURE_TGA, "TEXTURE_TGA" },
+        { LLAssetType::AT_BODYPART, "BODYPART" },
+        { LLAssetType::AT_SOUND_WAV, "SOUND_WAV" },
+        { LLAssetType::AT_IMAGE_TGA, "IMAGE_TGA" },
+        { LLAssetType::AT_IMAGE_JPEG, "IMAGE_JPEG" },
+        { LLAssetType::AT_ANIMATION, "ANIMATION" },
+        { LLAssetType::AT_GESTURE, "GESTURE" },
+        { LLAssetType::AT_SIMSTATE, "SIMSTATE" },
+        { LLAssetType::AT_LINK, "LINK" },
+        { LLAssetType::AT_LINK_FOLDER, "LINK_FOLDER" },
+        { LLAssetType::AT_MARKETPLACE_FOLDER, "MARKETPLACE_FOLDER" },
+        { LLAssetType::AT_WIDGET, "WIDGET" },
+        { LLAssetType::AT_PERSON, "PERSON" },
+        { LLAssetType::AT_MESH, "MESH" },
+        { LLAssetType::AT_SETTINGS, "SETTINGS" },
+        { LLAssetType::AT_UNKNOWN, "UNKNOWN" }
+    };
+
+    asset_type_to_name_t::iterator iter = asset_type_to_name.find(at);
+    if (iter != asset_type_to_name.end())
+    {
+        return iter->second;
+    }
+
+    return std::string("UNKNOWN");
+}
+
+const std::string LLDiskCache::metaDataToFilepath(const std::string id,
+        LLAssetType::EType at,
+        const std::string extra_info)
+{
+    std::ostringstream file_path;
+
+    file_path << mCacheDir;
+    file_path << gDirUtilp->getDirDelimiter();
+    file_path << mCacheFilenamePrefix;
+    file_path << "_";
+    file_path << id;
+    file_path << "_";
+    file_path << (extra_info.empty() ? "0" : extra_info);
+    //file_path << "_";
+    //file_path << assetTypeToString(at); // see  SL-14210 Prune descriptive tag from new cache filenames
+                                          // for details of why it was removed. Note that if you put it
+                                          // back or change the format of the filename, the cache files
+                                          // files will be invalidated (and perhaps, more importantly,
+                                          // never deleted unless you delete them manually).
+    file_path << ".asset";
+
+    return file_path.str();
+}
+
+void LLDiskCache::updateFileAccessTime(const std::string file_path)
+{
+    /**
+     * Threshold in time_t units that is used to decide if the last access time
+     * time of the file is updated or not. Added as a precaution for the concern
+     * outlined in SL-14582  about frequent writes on older SSDs reducing their
+     * lifespan. I think this is the right place for the threshold value - rather
+     * than it being a pref - do comment on that Jira if you disagree...
+     *
+     * Let's start with 1 hour in time_t units and see how that unfolds
+     */
+    const std::time_t time_threshold = 1 * 60 * 60;
+
+    // current time
+    const std::time_t cur_time = std::time(nullptr);
+
+    boost::system::error_code ec;
+#if LL_WINDOWS
+    // file last write time
+    const std::time_t last_write_time = boost::filesystem::last_write_time(utf8str_to_utf16str(file_path), ec);
+    if (ec.failed())
+    {
+        LL_WARNS() << "Failed to read last write time for cache file " << file_path << ": " << ec.message() << LL_ENDL;
+        return;
+    }
+
+    // delta between cur time and last time the file was written
+    const std::time_t delta_time = cur_time - last_write_time;
+
+    // we only write the new value if the time in time_threshold has elapsed
+    // before the last one
+    if (delta_time > time_threshold)
+    {
+        boost::filesystem::last_write_time(utf8str_to_utf16str(file_path), cur_time, ec);
+    }
+#else
+    // file last write time
+    const std::time_t last_write_time = boost::filesystem::last_write_time(file_path, ec);
+    if (ec.failed())
+    {
+        LL_WARNS() << "Failed to read last write time for cache file " << file_path << ": " << ec.message() << LL_ENDL;
+        return;
+    }
+
+    // delta between cur time and last time the file was written
+    const std::time_t delta_time = cur_time - last_write_time;
+
+    // we only write the new value if the time in time_threshold has elapsed
+    // before the last one
+    if (delta_time > time_threshold)
+    {
+        boost::filesystem::last_write_time(file_path, cur_time, ec);
+    }
+#endif
+
+    if (ec.failed())
+    {
+        LL_WARNS() << "Failed to update last write time for cache file " << file_path << ": " << ec.message() << LL_ENDL;
+    }
+}
+
+const std::string LLDiskCache::getCacheInfo()
+{
+    std::ostringstream cache_info;
+
+    F32 max_in_mb = (F32)mMaxSizeBytes / (1024.0 * 1024.0);
+    F32 percent_used = ((F32)dirFileSize(mCacheDir) / (F32)mMaxSizeBytes) * 100.0;
+
+    cache_info << std::fixed;
+    cache_info << std::setprecision(1);
+    cache_info << "Max size " << max_in_mb << " MB ";
+    cache_info << "(" << percent_used << "% used)";
+
+    return cache_info.str();
+}
+
+void LLDiskCache::clearCache()
+{
+    /**
+     * See notes on performance in dirFileSize(..) - there may be
+     * a quicker way to do this by operating on the parent dir vs
+     * the component files but it's called infrequently so it's
+     * likely just fine
+     */
+    boost::system::error_code ec;
+#if LL_WINDOWS
+    std::wstring cache_path(utf8str_to_utf16str(mCacheDir));
+#else
+    std::string cache_path(mCacheDir);
+#endif
+    if (boost::filesystem::is_directory(cache_path, ec) && !ec.failed())
+    {
+        for (auto& entry : boost::make_iterator_range(boost::filesystem::directory_iterator(cache_path, ec), {}))
+        {
+            if (boost::filesystem::is_regular_file(entry, ec) && !ec.failed())
+            {
+                if (entry.path().string().find(mCacheFilenamePrefix) != std::string::npos)
+                {
+                    boost::filesystem::remove(entry, ec);
+                    if (ec.failed())
+                    {
+                        LL_WARNS() << "Failed to delete cache file " << entry << ": " << ec.message() << LL_ENDL;
+                    }
+                }
+            }
+        }
+    }
+}
+
+uintmax_t LLDiskCache::dirFileSize(const std::string dir)
+{
+    uintmax_t total_file_size = 0;
+
+    /**
+     * There may be a better way that works directly on the folder (similar to
+     * right clicking on a folder in the OS and asking for size vs right clicking
+     * on all files and adding up manually) but this is very fast - less than 100ms
+     * for 10,000 files in my testing so, so long as it's not called frequently,
+     * it should be okay. Note that's it's only currently used for logging/debugging
+     * so if performance is ever an issue, optimizing this or removing it altogether,
+     * is an easy win.
+     */
+    boost::system::error_code ec;
+#if LL_WINDOWS
+    std::wstring dir_path(utf8str_to_utf16str(dir));
+#else
+    std::string dir_path(dir);
+#endif
+    if (boost::filesystem::is_directory(dir_path, ec) && !ec.failed())
+    {
+        for (auto& entry : boost::make_iterator_range(boost::filesystem::directory_iterator(dir_path, ec), {}))
+        {
+            if (boost::filesystem::is_regular_file(entry, ec) && !ec.failed())
+            {
+                if (entry.path().string().find(mCacheFilenamePrefix) != std::string::npos)
+                {
+                    uintmax_t file_size = boost::filesystem::file_size(entry, ec);
+                    if (!ec.failed())
+                    {
+                        total_file_size += file_size;
+                    }
+                }
+            }
+        }
+    }
+
+    return total_file_size;
+}
+
+LLPurgeDiskCacheThread::LLPurgeDiskCacheThread() :
+    LLThread("PurgeDiskCacheThread", nullptr)
+{
+}
+
+void LLPurgeDiskCacheThread::run()
+{
+    constexpr std::chrono::seconds CHECK_INTERVAL{60};
+
+    while (LLApp::instance()->sleep(CHECK_INTERVAL))
+    {
+        LLDiskCache::instance().purge();
+    }
+}