/** * @file lldiskcache.cpp * @brief The disk cache implementation. * * Note: Rather than keep the top level function comments up * to date in both the source and header files, I elected to * only have explicit comments about each function and variable * in the header - look there for details. The same is true for * description of how this code is supposed to work. * * $LicenseInfo:firstyear=2009&license=viewerlgpl$ * Second Life Viewer Source Code * Copyright (C) 2020, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; * version 2.1 of the License only. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA * * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA * $/LicenseInfo$ */ #include "linden_common.h" #include "llapp.h" #include "llassettype.h" #include "lldir.h" #include <boost/filesystem.hpp> #include <chrono> #include "lldiskcache.h" LLDiskCache::LLDiskCache(const std::string cache_dir, const uintmax_t max_size_bytes, const bool enable_cache_debug_info) : mCacheDir(cache_dir), mMaxSizeBytes(max_size_bytes), mEnableCacheDebugInfo(enable_cache_debug_info) { mCacheFilenamePrefix = "sl_cache"; LLFile::mkdir(cache_dir); } // WARNING: purge() is called by LLPurgeDiskCacheThread. As such it must // NOT touch any LLDiskCache data without introducing and locking a mutex! // Interaction through the filesystem itself should be safe. Let’s say thread // A is accessing the cache file for reading/writing and thread B is trimming // the cache. Let’s also assume using llifstream to open a file and // boost::filesystem::remove are not atomic (which will be pretty much the // case). // Now, A is trying to open the file using llifstream ctor. It does some // checks if the file exists and whatever else it might be doing, but has not // issued the call to the OS to actually open the file yet. Now B tries to // delete the file: If the file has been already marked as in use by the OS, // deleting the file will fail and B will continue with the next file. A can // safely continue opening the file. If the file has not yet been marked as in // use, B will delete the file. Now A actually wants to open it, operation // will fail, subsequent check via llifstream.is_open will fail, asset will // have to be re-requested. (Assuming here the viewer will actually handle // this situation properly, that can also happen if there is a file containing // garbage.) // Other situation: B is trimming the cache and A wants to read a file that is // about to get deleted. boost::filesystem::remove does whatever it is doing // before actually deleting the file. If A opens the file before the file is // actually gone, the OS call from B to delete the file will fail since the OS // will prevent this. B continues with the next file. If the file is already // gone before A finally gets to open it, this operation will fail and the // asset will have to be re-requested. void LLDiskCache::purge() { if (mEnableCacheDebugInfo) { LL_INFOS() << "Total dir size before purge is " << dirFileSize(mCacheDir) << LL_ENDL; } boost::system::error_code ec; auto start_time = std::chrono::high_resolution_clock::now(); typedef std::pair<std::time_t, std::pair<uintmax_t, std::string>> file_info_t; std::vector<file_info_t> file_info; #if LL_WINDOWS std::wstring cache_path(utf8str_to_utf16str(mCacheDir)); #else std::string cache_path(mCacheDir); #endif if (boost::filesystem::is_directory(cache_path, ec) && !ec.failed()) { boost::filesystem::directory_iterator iter(cache_path, ec); while (iter != boost::filesystem::directory_iterator() && !ec.failed()) { if (boost::filesystem::is_regular_file(*iter, ec) && !ec.failed()) { if ((*iter).path().string().find(mCacheFilenamePrefix) != std::string::npos) { uintmax_t file_size = boost::filesystem::file_size(*iter, ec); if (ec.failed()) { continue; } const std::string file_path = (*iter).path().string(); const std::time_t file_time = boost::filesystem::last_write_time(*iter, ec); if (ec.failed()) { continue; } file_info.push_back(file_info_t(file_time, { file_size, file_path })); } } iter.increment(ec); } } std::sort(file_info.begin(), file_info.end(), [](file_info_t& x, file_info_t& y) { return x.first > y.first; }); LL_INFOS() << "Purging cache to a maximum of " << mMaxSizeBytes << " bytes" << LL_ENDL; std::vector<bool> file_removed; if (mEnableCacheDebugInfo) { file_removed.reserve(file_info.size()); } uintmax_t file_size_total = 0; for (file_info_t& entry : file_info) { file_size_total += entry.second.first; bool should_remove = file_size_total > mMaxSizeBytes; if (mEnableCacheDebugInfo) { file_removed.push_back(should_remove); } if (should_remove) { boost::filesystem::remove(entry.second.second, ec); if (ec.failed()) { LL_WARNS() << "Failed to delete cache file " << entry.second.second << ": " << ec.message() << LL_ENDL; } } } if (mEnableCacheDebugInfo) { auto end_time = std::chrono::high_resolution_clock::now(); auto execute_time = std::chrono::duration_cast<std::chrono::milliseconds>(end_time - start_time).count(); // Log afterward so it doesn't affect the time measurement // Logging thousands of file results can take hundreds of milliseconds for (size_t i = 0; i < file_info.size(); ++i) { const file_info_t& entry = file_info[i]; const bool removed = file_removed[i]; const std::string action = removed ? "DELETE:" : "KEEP:"; // have to do this because of LL_INFO/LL_END weirdness std::ostringstream line; line << action << " "; line << entry.first << " "; line << entry.second.first << " "; line << entry.second.second; line << " (" << file_size_total << "/" << mMaxSizeBytes << ")"; LL_INFOS() << line.str() << LL_ENDL; } LL_INFOS() << "Total dir size after purge is " << dirFileSize(mCacheDir) << LL_ENDL; LL_INFOS() << "Cache purge took " << execute_time << " ms to execute for " << file_info.size() << " files" << LL_ENDL; } } const std::string LLDiskCache::assetTypeToString(LLAssetType::EType at) { /** * Make use of the handy C++17 feature that allows * for inline initialization of an std::map<> */ typedef std::map<LLAssetType::EType, std::string> asset_type_to_name_t; asset_type_to_name_t asset_type_to_name = { { LLAssetType::AT_TEXTURE, "TEXTURE" }, { LLAssetType::AT_SOUND, "SOUND" }, { LLAssetType::AT_CALLINGCARD, "CALLINGCARD" }, { LLAssetType::AT_LANDMARK, "LANDMARK" }, { LLAssetType::AT_SCRIPT, "SCRIPT" }, { LLAssetType::AT_CLOTHING, "CLOTHING" }, { LLAssetType::AT_OBJECT, "OBJECT" }, { LLAssetType::AT_NOTECARD, "NOTECARD" }, { LLAssetType::AT_CATEGORY, "CATEGORY" }, { LLAssetType::AT_LSL_TEXT, "LSL_TEXT" }, { LLAssetType::AT_LSL_BYTECODE, "LSL_BYTECODE" }, { LLAssetType::AT_TEXTURE_TGA, "TEXTURE_TGA" }, { LLAssetType::AT_BODYPART, "BODYPART" }, { LLAssetType::AT_SOUND_WAV, "SOUND_WAV" }, { LLAssetType::AT_IMAGE_TGA, "IMAGE_TGA" }, { LLAssetType::AT_IMAGE_JPEG, "IMAGE_JPEG" }, { LLAssetType::AT_ANIMATION, "ANIMATION" }, { LLAssetType::AT_GESTURE, "GESTURE" }, { LLAssetType::AT_SIMSTATE, "SIMSTATE" }, { LLAssetType::AT_LINK, "LINK" }, { LLAssetType::AT_LINK_FOLDER, "LINK_FOLDER" }, { LLAssetType::AT_MARKETPLACE_FOLDER, "MARKETPLACE_FOLDER" }, { LLAssetType::AT_WIDGET, "WIDGET" }, { LLAssetType::AT_PERSON, "PERSON" }, { LLAssetType::AT_MESH, "MESH" }, { LLAssetType::AT_SETTINGS, "SETTINGS" }, { LLAssetType::AT_MATERIAL, "MATERIAL" }, { LLAssetType::AT_GLTF, "GLTF" }, { LLAssetType::AT_GLTF_BIN, "GLTF_BIN" }, { LLAssetType::AT_UNKNOWN, "UNKNOWN" } }; asset_type_to_name_t::iterator iter = asset_type_to_name.find(at); if (iter != asset_type_to_name.end()) { return iter->second; } return std::string("UNKNOWN"); } const std::string LLDiskCache::metaDataToFilepath(const std::string id, LLAssetType::EType at, const std::string extra_info) { std::ostringstream file_path; file_path << mCacheDir; file_path << gDirUtilp->getDirDelimiter(); file_path << mCacheFilenamePrefix; file_path << "_"; file_path << id; file_path << "_"; file_path << (extra_info.empty() ? "0" : extra_info); //file_path << "_"; //file_path << assetTypeToString(at); // see SL-14210 Prune descriptive tag from new cache filenames // for details of why it was removed. Note that if you put it // back or change the format of the filename, the cache files // files will be invalidated (and perhaps, more importantly, // never deleted unless you delete them manually). file_path << ".asset"; return file_path.str(); } void LLDiskCache::updateFileAccessTime(const std::string file_path) { /** * Threshold in time_t units that is used to decide if the last access time * time of the file is updated or not. Added as a precaution for the concern * outlined in SL-14582 about frequent writes on older SSDs reducing their * lifespan. I think this is the right place for the threshold value - rather * than it being a pref - do comment on that Jira if you disagree... * * Let's start with 1 hour in time_t units and see how that unfolds */ const std::time_t time_threshold = 1 * 60 * 60; // current time const std::time_t cur_time = std::time(nullptr); boost::system::error_code ec; #if LL_WINDOWS // file last write time const std::time_t last_write_time = boost::filesystem::last_write_time(utf8str_to_utf16str(file_path), ec); if (ec.failed()) { LL_WARNS() << "Failed to read last write time for cache file " << file_path << ": " << ec.message() << LL_ENDL; return; } // delta between cur time and last time the file was written const std::time_t delta_time = cur_time - last_write_time; // we only write the new value if the time in time_threshold has elapsed // before the last one if (delta_time > time_threshold) { boost::filesystem::last_write_time(utf8str_to_utf16str(file_path), cur_time, ec); } #else // file last write time const std::time_t last_write_time = boost::filesystem::last_write_time(file_path, ec); if (ec.failed()) { LL_WARNS() << "Failed to read last write time for cache file " << file_path << ": " << ec.message() << LL_ENDL; return; } // delta between cur time and last time the file was written const std::time_t delta_time = cur_time - last_write_time; // we only write the new value if the time in time_threshold has elapsed // before the last one if (delta_time > time_threshold) { boost::filesystem::last_write_time(file_path, cur_time, ec); } #endif if (ec.failed()) { LL_WARNS() << "Failed to update last write time for cache file " << file_path << ": " << ec.message() << LL_ENDL; } } const std::string LLDiskCache::getCacheInfo() { std::ostringstream cache_info; F32 max_in_mb = (F32)mMaxSizeBytes / (1024.0f * 1024.0f); F32 percent_used = ((F32)dirFileSize(mCacheDir) / (F32)mMaxSizeBytes) * 100.0f; cache_info << std::fixed; cache_info << std::setprecision(1); cache_info << "Max size " << max_in_mb << " MB "; cache_info << "(" << percent_used << "% used)"; return cache_info.str(); } void LLDiskCache::clearCache() { /** * See notes on performance in dirFileSize(..) - there may be * a quicker way to do this by operating on the parent dir vs * the component files but it's called infrequently so it's * likely just fine */ boost::system::error_code ec; #if LL_WINDOWS std::wstring cache_path(utf8str_to_utf16str(mCacheDir)); #else std::string cache_path(mCacheDir); #endif if (boost::filesystem::is_directory(cache_path, ec) && !ec.failed()) { boost::filesystem::directory_iterator iter(cache_path, ec); while (iter != boost::filesystem::directory_iterator() && !ec.failed()) { if (boost::filesystem::is_regular_file(*iter, ec) && !ec.failed()) { if ((*iter).path().string().find(mCacheFilenamePrefix) != std::string::npos) { boost::filesystem::remove(*iter, ec); if (ec.failed()) { LL_WARNS() << "Failed to delete cache file " << *iter << ": " << ec.message() << LL_ENDL; } } } iter.increment(ec); } } } void LLDiskCache::removeOldVFSFiles() { //VFS files won't be created, so consider removing this code later static const char CACHE_FORMAT[] = "inv.llsd"; static const char DB_FORMAT[] = "db2.x"; boost::system::error_code ec; #if LL_WINDOWS std::wstring cache_path(utf8str_to_utf16str(gDirUtilp->getExpandedFilename(LL_PATH_CACHE, ""))); #else std::string cache_path(gDirUtilp->getExpandedFilename(LL_PATH_CACHE, "")); #endif if (boost::filesystem::is_directory(cache_path, ec) && !ec.failed()) { boost::filesystem::directory_iterator iter(cache_path, ec); while (iter != boost::filesystem::directory_iterator() && !ec.failed()) { if (boost::filesystem::is_regular_file(*iter, ec) && !ec.failed()) { if (((*iter).path().string().find(CACHE_FORMAT) != std::string::npos) || ((*iter).path().string().find(DB_FORMAT) != std::string::npos)) { boost::filesystem::remove(*iter, ec); if (ec.failed()) { LL_WARNS() << "Failed to delete cache file " << *iter << ": " << ec.message() << LL_ENDL; } } } iter.increment(ec); } } } uintmax_t LLDiskCache::dirFileSize(const std::string dir) { uintmax_t total_file_size = 0; /** * There may be a better way that works directly on the folder (similar to * right clicking on a folder in the OS and asking for size vs right clicking * on all files and adding up manually) but this is very fast - less than 100ms * for 10,000 files in my testing so, so long as it's not called frequently, * it should be okay. Note that's it's only currently used for logging/debugging * so if performance is ever an issue, optimizing this or removing it altogether, * is an easy win. */ boost::system::error_code ec; #if LL_WINDOWS std::wstring dir_path(utf8str_to_utf16str(dir)); #else std::string dir_path(dir); #endif if (boost::filesystem::is_directory(dir_path, ec) && !ec.failed()) { boost::filesystem::directory_iterator iter(dir_path, ec); while (iter != boost::filesystem::directory_iterator() && !ec.failed()) { if (boost::filesystem::is_regular_file(*iter, ec) && !ec.failed()) { if ((*iter).path().string().find(mCacheFilenamePrefix) != std::string::npos) { uintmax_t file_size = boost::filesystem::file_size(*iter, ec); if (!ec.failed()) { total_file_size += file_size; } } } iter.increment(ec); } } return total_file_size; } LLPurgeDiskCacheThread::LLPurgeDiskCacheThread() : LLThread("PurgeDiskCacheThread", nullptr) { } void LLPurgeDiskCacheThread::run() { constexpr std::chrono::seconds CHECK_INTERVAL{60}; while (LLApp::instance()->sleep(CHECK_INTERVAL)) { LLDiskCache::instance().purge(); } }