From 3092aa8aae496803707980eb456cddbb9960ef1c Mon Sep 17 00:00:00 2001 From: Callum Prentice Date: Tue, 6 Oct 2020 17:16:53 -0700 Subject: Add in the C++ filesystem based cache and clean up some indempotent functions in llfilesystem --- indra/llfilesystem/lldiskcache.cpp | 809 +++++------------------------- indra/llfilesystem/lldiskcache.h | 146 ++---- indra/llfilesystem/llfilesystem.cpp | 32 +- indra/llfilesystem/llfilesystem.h | 10 +- indra/llmessage/lltransfertargetvfile.cpp | 1 - indra/llmessage/llxfer_vfile.cpp | 1 - indra/newview/app_settings/settings.xml | 11 + indra/newview/llfloaterbvhpreview.cpp | 1 - indra/newview/llmeshrepository.cpp | 2 +- indra/newview/llpreviewgesture.cpp | 1 - indra/newview/llpreviewnotecard.cpp | 1 - indra/newview/llviewerassetstorage.cpp | 3 - indra/newview/llviewerassetupload.cpp | 3 - 13 files changed, 196 insertions(+), 825 deletions(-) (limited to 'indra') diff --git a/indra/llfilesystem/lldiskcache.cpp b/indra/llfilesystem/lldiskcache.cpp index 72982db069..4b2ba0dd79 100644 --- a/indra/llfilesystem/lldiskcache.cpp +++ b/indra/llfilesystem/lldiskcache.cpp @@ -1,9 +1,6 @@ /** * @file lldiskcache.cpp - * @brief The SQLite based disk cache implementation. - * @Note Eventually, this component might split into an interface - * file and multiple implemtations but for now, this is the - * only one so I think it's okay to combine everything. + * @brief The disk cache implementation. * * $LicenseInfo:firstyear=2009&license=viewerlgpl$ * Second Life Viewer Source Code @@ -27,747 +24,167 @@ * $/LicenseInfo$ */ -#if (defined(LL_WINDOWS) || defined(LL_LINUX) || defined(LL_DARWIN)) #include "linden_common.h" -#endif +#include "lluuid.h" +#include "lldir.h" #include "lldiskcache.h" -#include -#include -#include -#include -#include - -/////////////////////////////////////////////////////////////////////////////// -// -llDiskCache::llDiskCache() : - mDataStorePath("") -{ -} +#include +#include -llDiskCache::~llDiskCache() -{ -} +#include +#include +#include -/////////////////////////////////////////////////////////////////////////////// -// Opens the database - typically done when the application starts -// and is complementary to close() which is called when the application -// is finisahed and exits. -// Pass in the folder and filename of the SQLite database you want to -// use or create (file doesn't have to exist but the folder must) -// Returns true or false and writes a message to console on error -bool llDiskCache::open(const std::string db_folder, const std::string db_filename) +LLDiskCache::LLDiskCache(const std::string cache_dir) : + mCacheDir(cache_dir), + mMaxSizeBytes(mDefaultSizeBytes) { - mDataStorePath = db_folder; - std::string db_pathname = makeFullPath(db_filename); - - // simple flags for the moment - these will likely be extended - // later on to support the SQLite mutex model for reading/writing - // simultaneously - perhaps when we look at supporting textures too - int flags = SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE; - - if (sqlite3_open_v2( - db_pathname.c_str(), - &mDb, - flags, - nullptr // Name of VFS module to use - ) != SQLITE_OK) - { - printError(__FUNCDNAME__ , "open_v2", true); - close(); - return false; - } - - // I elected to store the code that generates the statement - // in sepsrate functions throughout - this seemed like a cleaner - // approach than having hundreds of stmt << "REPLACE AS" lines - // interspersed in the logic code. They are all prefixed with - // 'sqlCompose' and followed by a short description. - const std::string stmt = sqlComposeCreateTable(); - if (! sqliteExec(stmt, __FUNCDNAME__ )) + // no access to LLControlGroup / gSavedSettings so use the system environment + // to trigger additional debugging on top of the default "we purged the cache" + mCacheDebugInfo = true; + if (getenv("LL_CACHE_DEBUGGING") != nullptr) { - // Not sure if we need close here - if open fails, then we - // perhaps don't need to close it - TODO: look in SQLite docs - close(); - return false; + mCacheDebugInfo = true; } - return true; + // create cache dir if it does not exist + boost::filesystem::create_directory(cache_dir); } -/////////////////////////////////////////////////////////////////////////////// -// Determines if an entry exists. -// Pass in the id and a variable that will indicate existance -// Returns true/false if function succeeded and the boolean -// you pass in will be set to true/false if entry exists or not -bool llDiskCache::exists(const std::string id, bool& exists) +LLDiskCache::~LLDiskCache() { - if (!mDb) - { - printError(__FUNCDNAME__ , "mDb is invalid", false); - return false; - } - - if (id.empty()) - { - printError(__FUNCDNAME__ , "id is empty", false); - return false; - } - - // As well as the separate function to compose the SQL statement, - // worth pointing out that the code to 'prepare' and 'step' the - // SQLite "prepared statement" has been factored out into its own - // function and used in several other functions. - const std::string stmt = sqlComposeExists(id); - sqlite3_stmt* prepared_statement = sqlitePrepareStep(__FUNCDNAME__ , stmt, SQLITE_ROW); - if (! prepared_statement) - { - return false; - } - - int result_column_index = 0; - int result_count = sqlite3_column_int(prepared_statement, result_column_index); - if (sqlite3_finalize(prepared_statement) != SQLITE_OK) - { - printError(__FUNCDNAME__ , "sqlite3_finalize()", true); - return false; - } - - // given the uniqueness of the ID, this can only ever be - // either 1 or 0 so this might be a bit confusing - exists = result_count > 0 ? true : false; - - return true; -} - -/////////////////////////////////////////////////////////////////////////////// -// Given an id (likely a UUID + decoration but can be any string), a -// pointer to a blob of binary data along with its length, write the -// entry to the cache -// Returns true/false for success/failure -bool llDiskCache::put(const std::string id, - char* binary_data, - int binary_data_size) -{ - if (!mDb) - { - printError(__FUNCDNAME__ , "mDb is invalid", false); - return false; - } - - if (id.empty()) - { - printError(__FUNCDNAME__ , "id is empty", false); - return false; - } - - // < 0 is obvious but we assert the data must be at least 1 byte long - if (binary_data_size <= 0) - { - printError(__FUNCDNAME__ , "size of binary file to write is invalid", false); - return false; - } - - // we generate a unique filename for the actual data itself - // which is stored on disk directly and not in the database. - // TODO: consider making the filename more like the ID passed in - // although the problem with that is we would have to parse the id - // to remove invalid filename chars, consider length etc. As it - // stands, we can write a simple SQL statement to return the filename - // given the ID. - const std::string filename = makeUniqueFilename(); - const std::string filepath = makeFullPath(filename); - std::ofstream file(filepath, std::ios::out | std::ios::binary); - if (! file.is_open()) - { - std::ostringstream error; - error << "Unable to open " << filepath << " for writing"; - printError(__FUNCDNAME__ , error.str(), false); - return false; - } - - file.write((char*)binary_data, binary_data_size); - file.close(); - - // I think is a catchall "wasn't able to write the file to disk" - // conditional but should revisit when we hook this up to viewer - // code to make sure - we never want to write bad/no data to the - // disk and not indicate something has gone wrong - if (file.bad()) - { - std::ostringstream error; - error << "Unable to write " << binary_data_size; - error << " bytes to " << filepath; - printError(__FUNCDNAME__ , error.str(), false); - - return false; - } - - // this is where the filename/size is written to the database along - // with the current date/time for the created/last access times - const std::string stmt = sqlComposePut(id, filename, binary_data_size); - if (! sqlitePrepareStep(__FUNCDNAME__ , stmt, SQLITE_DONE)) - { - return false; - } - - return true; } -/////////////////////////////////////////////////////////////////////////////// -// Given an id (likely a UUID + decoration but can be any string), the -// address of a pointer that will be used to allocate memory and a -// varialble that will contain the length of the data, get an item from -// the cache. Note that the memory buffer returned belongs to the calling -// function and it is its responsiblity to clean it up when it's no -// longer needed. -// Returns true/false for success/failure -const bool llDiskCache::get(const std::string id, - char** mem_buffer, - int& mem_buffer_size) +void LLDiskCache::purge() { - // Check if the entry exists first to avoid dealing with a bunch - // of conditions that look like failure but aren't in the main code. - // Note exists() is a public method and also tests for mDB and id - // being valid so we can safely put this about the same tests - // in this function - bool get_exists = false; - if (! exists(id, get_exists)) + if (mCacheDebugInfo) { - return false; - } - if (! get_exists) - { - return false; + LL_INFOS() << "Total dir size before purge is " << dirFileSize(mCacheDir) << LL_ENDL; } - if (!mDb) - { - printError(__FUNCDNAME__ , "mDb is invalid", false); - return false; - } - - if (id.empty()) - { - printError(__FUNCDNAME__ , "id is empty", false); - return false; - } + auto start_time = std::chrono::high_resolution_clock::now(); - const std::string stmt_select = sqlComposeGetSelect(id); - sqlite3_stmt* prepared_statement = sqlitePrepareStep(__FUNCDNAME__ , stmt_select, SQLITE_ROW); - if (! prepared_statement) - { - return false; - } + typedef std::pair> file_info_t; + std::vector file_info; - int result_column_index = 0; - const unsigned char* text = sqlite3_column_text(prepared_statement, result_column_index); - if (text == nullptr) + if (boost::filesystem::is_directory(mCacheDir)) { - printError(__FUNCDNAME__ , "filename is nullptr", true); - return false; - } - const std::string filename = std::string(reinterpret_cast(text)); - const std::string filepath = makeFullPath(filename); + for (auto& entry : boost::make_iterator_range(boost::filesystem::directory_iterator(mCacheDir), {})) + { + if (boost::filesystem::is_regular_file(entry)) + { + uintmax_t file_size = boost::filesystem::file_size(entry); + const std::string file_path = entry.path().string(); + const std::time_t file_time = boost::filesystem::last_write_time(entry); - result_column_index = 1; - int filesize_db = sqlite3_column_int(prepared_statement, result_column_index); - if (filesize_db <= 0) - { - printError(__FUNCDNAME__ , "filesize is invalid", true); - return false; + file_info.push_back(file_info_t(file_time, { file_size, file_path })); + } + } } - if (sqlite3_finalize(prepared_statement) != SQLITE_OK) + std::sort(file_info.begin(), file_info.end(), [](file_info_t& x, file_info_t& y) { - printError(__FUNCDNAME__ , "sqlite3_finalize()", true); - return false; - } + return x.first > y.first; + }); - // Now we have the fiename, we can read the file from disk - std::ifstream file(filepath, std::ios::in | std::ios::binary | std::ios::ate); - if (! file.is_open()) - { - std::ostringstream error; - error << "Unable to open " << filepath << " for reading"; - printError(__FUNCDNAME__ , error.str(), false); - return false; - } + LL_INFOS() << "Purging cache to a maximum of " << mMaxSizeBytes << " bytes" << LL_ENDL; - // we get the expected filesize from the database but we can also - // get it (easily) when we read the file from the disk. We compare - // the two and return false if they don't match - std::streampos filesize_file = file.tellg(); - if (filesize_db != filesize_file) + uintmax_t file_size_total = 0; + for (file_info_t& entry : file_info) { - std::ostringstream error; - error << "File size from DB (" << filesize_db << ")"; - error << " and "; - error << "file size from file (" << filesize_file << ")"; - error << " in file " << filepath << " are different"; - printError(__FUNCDNAME__ , error.str(), false); - - return false; - } - - // doest matter if we choose DB or file version - they must be - // identical if we get this far - just used for clarity - int filesize = filesize_db; + file_size_total += entry.second.first; - // allocate a block of memory that we pass back for the calling - // function to use then delete when it's no longer needed - *mem_buffer = new char[filesize]; - mem_buffer_size = filesize; + std::string action = ""; + if (file_size_total > mMaxSizeBytes) + { + action = "DELETE:"; + boost::filesystem::remove(entry.second.second); + } + else + { + action = " KEEP:"; + } - file.seekg(0, std::ios::beg); - file.read(*mem_buffer, filesize); - file.close(); + if (mCacheDebugInfo) + { + // have to do this because of LL_INFO/LL_END weirdness + std::ostringstream line; - if (file.bad()) - { - std::ostringstream error; - error << "Unable to read " << filesize; - error << " bytes from " << filepath; - printError(__FUNCDNAME__ , error.str(), false); - - return false; + line << action << " "; + line << entry.first << " "; + line << entry.second.first << " "; + line << entry.second.second; + line << " (" << file_size_total << "/" << mMaxSizeBytes << ")"; + LL_INFOS() << line.str() << LL_ENDL; + } } - // here we update the count of times the file is accessed so - // we can keep track of how many times it's been requested. - // This will be useful for metrics and perhaps determining - // if a file should not be purged even though its age - // might suggest that it should. - // In addition, this is where the time of last access is updated - // in the database and that us used to determine what is purged - // in an LRU fashion when the purge function is called. - const std::string stmt_update = sqlComposeGetUpdate(id); - if (! sqliteExec(stmt_update, __FUNCDNAME__ )) + if (mCacheDebugInfo) { - return false; + auto end_time = std::chrono::high_resolution_clock::now(); + auto execute_time = std::chrono::duration_cast(end_time - start_time).count(); + LL_INFOS() << "Total dir size after purge is " << dirFileSize(mCacheDir) << LL_ENDL; + LL_INFOS() << "Cache purge took " << execute_time << " ms to execute for " << file_info.size() << " files" << LL_ENDL; } - - return true; -} - -/////////////////////////////////////////////////////////////////////////////// -// Purges the database of older entries using an LRU approach. -// Pass in the number of entries to retain. -// This is called now after open to "clean up" the cache when the -// application starts. -// TODO: IMPORTANT: compose a list of files that will be deleted -// and delete them from disk too - not just from the DB -bool llDiskCache::purge(int num_entries) -{ - if (num_entries < 0) - { - printError(__FUNCDNAME__ , "number of entries to purge is invalid", false); - return false; - } - - // find the rows affected and get the filenames for them -//swww - - // delete oldest entries leaving the correct number in place - const std::string stmt = sqlComposePurge(num_entries); - if (! sqliteExec(stmt, __FUNCDNAME__ )) - { - return false; - } - - return true; -} - -/////////////////////////////////////////////////////////////////////////////// -// Call at application shutdown -void llDiskCache::close() -{ - sqlite3_close(mDb); -} - -/////////////////////////////////////////////////////////////////////////////// -// Determine the version of SQLite in use -// TODO: make this a static so we can get to it from the Viewer About -// box without instantiating the whole thing. -const std::string llDiskCache::dbVersion() -{ - std::ostringstream version; - - version << sqlite3_libversion(); - - return version.str(); } -/////////////////////////////////////////////////////////////////////////////// -// Given an id, return the matching filename -const std::string llDiskCache::getFilenameById(const std::string id) -{ - // TODO: - return std::string(); -} - -/////////////////////////////////////////////////////////////////////////////// -// Given an id, return the number of times that entry has been -// accessed from the cache -const int llDiskCache::getAccessCountById(const std::string id) -{ - // TODO: - return -1; -} - -/////////////////////////////////////////////////////////////////////////////// -// Return the number of entries currently in the cache as well as -// the maximum possible entries. -void llDiskCache::getNumEntries(int& num_entries, int& max_entries) -{ - num_entries = -1; - max_entries = -1; -} - -/////////////////////////////////////////////////////////////////////////////// -// Wraps the sqlite3_exec(..) used in many places -bool llDiskCache::sqliteExec(const std::string stmt, - const std::string funcname) -{ - if (sqlite3_exec( - mDb, - stmt.c_str(), - nullptr, // Callback function (unused) - nullptr, // 1st argument to callback (unused) - nullptr // Error msg written here (unused) - ) != SQLITE_OK) - { - printError(funcname, "sqlite3_exec", true); - return false; - } - - return true; -} - -/////////////////////////////////////////////////////////////////////////////// -// Wraps the sqlite3_prepare_v2 and sqlite3_step calls used in many places -sqlite3_stmt* llDiskCache::sqlitePrepareStep(const std::string funcname, - const std::string stmt, - int sqlite_success_condition) -{ - sqlite3_stmt* prepared_statement; - - if (sqlite3_prepare_v2( - mDb, - stmt.c_str(), - -1, // Maximum length of zSql in bytes. - &prepared_statement, - 0 // OUT: Pointer to unused portion of zSql - ) != SQLITE_OK) - { - printError(funcname, "sqlite3_prepare_v2", true); - return nullptr; - } - - if (sqlite3_step(prepared_statement) != sqlite_success_condition) - { - printError(funcname, "sqlite3_step", true); - sqlite3_finalize(prepared_statement); - return nullptr; - } - return prepared_statement; -} - -/////////////////////////////////////////////////////////////////////////////// -// When an "error" occurss - e.g. database cannot be found, file cannot be -// written, invalid argument passed into a function etc. a message is -// written to stderr that should end up in the viewer log -// TODO: Set the verbosity using the usual Viewer mechanism -void llDiskCache::printError(const std::string funcname, - const std::string desc, - bool is_sqlite_err) -{ - std::ostringstream err_msg; - - err_msg << "llDiskCache error in "; - err_msg << __FUNCDNAME__ << "(...) "; - err_msg << desc; - - if (is_sqlite_err) - { - err_msg << " - "; - err_msg << std::string(sqlite3_errmsg(mDb)); - } - - // TODO: set via viewer verbosity level - const int verbosity = 1; - if (verbosity > 0) - { - std::cerr << err_msg.str() << std::endl; - } -} - -/////////////////////////////////////////////////////////////////////////////// -// wrapper for SQLite code to begin an SQL transaction - not used yet but -// it will be eventually -bool llDiskCache::beginTransaction() +/** + * Update the "last write time" of a file to "now". This must be called whenever a + * file in the cache is read (not written) so that the last time the file was + * accessed which is used in the mechanism for purging the cache, is up to date. + */ +void LLDiskCache::updateFileAccessTime(const std::string file_path) { - const std::string stmt("BEGIN TRANSACTION"); - if (! sqliteExec(stmt, __FUNCDNAME__ )) - { - return false; - } - - return true; + const std::time_t file_time = std::time(nullptr); + boost::filesystem::last_write_time(file_path, file_time); } -/////////////////////////////////////////////////////////////////////////////// -// wrapper for SQLite code to end an SQL transaction - not used yet but -// it will be eventually -bool llDiskCache::endTransaction() +/** + * Clear the cache by removing all the files in the cache directory + * individually. It's important to maintain control of which directory + * if passed in and not let the user inadvertently (or maliciously) set + * it to an random location like your project source or OS system directory + */ +void LLDiskCache::clearCache(const std::string cache_dir) { - const std::string stmt("COMMIT"); - if (! sqliteExec(stmt, __FUNCDNAME__ )) + if (boost::filesystem::is_directory(cache_dir)) { - return false; + for (auto& entry : boost::make_iterator_range(boost::filesystem::directory_iterator(cache_dir), {})) + { + if (boost::filesystem::is_regular_file(entry)) + { + boost::filesystem::remove(entry); + } + } } - - return true; } -/////////////////////////////////////////////////////////////////////////////// -// Build a unique filename that will be used to store the actual file -// on disk (as opposed to the meta data in the database) -// TODO: I think this needs more work once we move it to the viewer -// and espcially to make it cross platform - see 'std::tmpnam' -// depreciation comments in compiler output for example -std::string llDiskCache::makeUniqueFilename() +/** + * Utility function to get the total filesize of all files in a directory. It + * used to test file extensions to only check cache files but that was removed. + * There may be a better way that works directly on the folder (similar to + * right clicking on a folder in the OS and asking for size vs right clicking + * on all files and adding up manually) but this is very fast - less than 100ms + * in my testing so, so long as it's not called frequently, it should be okay. + * Note that's it's only currently used for logging/debugging so if performance + * is ever an issue, optimizing this or removing it altogether, is an easy win. + */ +uintmax_t LLDiskCache::dirFileSize(const std::string dir) { - // TODO: replace with boost - this is marked as deprecated? - std::string base = std::tmpnam(nullptr); - - // C++11 random number generation!!! - static std::random_device dev; - static std::mt19937 rng(dev()); - std::uniform_int_distribution dist(100000, 999999); + uintmax_t total_file_size = 0; - // currently the tmp filename from std::tmpnam() on macOS - // is of the form `/tmp/foo/bar.12345 and the following code - // strips all the preceding dirs - we likely want a more - // robust (and cross platform solution) when we move to the - // viewer code - std::size_t found = base.rfind(systemSeparator()); - if (found != std::string::npos && found < base.size() - 2) - { - base = base.substr(found + 1); - } - else + if (boost::filesystem::is_directory(dir)) { - base = ""; + for (auto& entry : boost::make_iterator_range(boost::filesystem::directory_iterator(dir), {})) + { + if (boost::filesystem::is_regular_file(entry)) + { + total_file_size += boost::filesystem::file_size(entry); + } + } } - // we mix in a random number for some more entropy.. - // (i know, i know...) - std::ostringstream unique_filename; - unique_filename << base; - unique_filename << "."; - unique_filename << dist(rng); - - return unique_filename.str(); -} - -/////////////////////////////////////////////////////////////////////////////// -// Return system file/path separator - likely replaced by the version -// in the viewer -const std::string llDiskCache::systemSeparator() -{ -// TODO: replace in viewer with relevant call -#ifdef _WIN32 - return "\\"; -#else - return "/"; -#endif -} - -/////////////////////////////////////////////////////////////////////////////// -// Given a filename, compose a full path based on the path name passed -// in when the database was opened and the separator in play. -const std::string llDiskCache::makeFullPath(const std::string filename) -{ - std::string pathname = mDataStorePath + systemSeparator() + filename; - - return pathname; -} - -/////////////////////////////////////////////////////////////////////////////// -// -const std::string llDiskCache::sqlComposeCreateTable() -{ - std::ostringstream stmt; - stmt << "CREATE TABLE IF NOT EXISTS "; - stmt << mTableName; - stmt << "("; - stmt << mIdFieldName; - stmt << " TEXT PRIMARY KEY, "; - stmt << mFilenameFieldName; - stmt << " TEXT, "; - stmt << mFilesizeFieldName; - stmt << " INTEGER DEFAULT 0, "; - stmt << mInsertionDateTimeFieldName; - stmt << " TEXT, "; - stmt << mLastAccessDateTimeFieldName; - stmt << " TEXT, "; - stmt << mAccessCountFieldName; - stmt << " INTEGER DEFAULT 0"; - stmt << ")"; - -#ifdef SHOW_STATEMENTS - std::cout << stmt.str() << std::endl; -#endif - - return stmt.str(); -} - -/////////////////////////////////////////////////////////////////////////////// -// -const std::string llDiskCache::sqlComposeExists(const std::string id) -{ - std::ostringstream stmt; - stmt << "SELECT COUNT(*) FROM "; - stmt << mTableName; - stmt << " WHERE "; - stmt << mIdFieldName; - stmt << "='"; - stmt << id; - stmt << "'"; - -#ifdef SHOW_STATEMENTS - std::cout << stmt.str() << std::endl; -#endif - - return stmt.str(); -} - -/////////////////////////////////////////////////////////////////////////////// -// SQL statement to write an entry to the DB -// Saves id, filename (generated), file size and create/last access date -const std::string llDiskCache::sqlComposePut(const std::string id, - const std::string filename, - int binary_data_size) -{ - std::ostringstream stmt; - stmt << "REPLACE INTO "; - stmt << mTableName; - stmt << "("; - stmt << mIdFieldName; - stmt << ","; - stmt << mFilenameFieldName; - stmt << ","; - stmt << mFilesizeFieldName; - stmt << ","; - stmt << mInsertionDateTimeFieldName; - stmt << ","; - stmt << mLastAccessDateTimeFieldName; - stmt << ") "; - stmt << "VALUES("; - stmt << "'"; - stmt << id; - stmt << "', "; - stmt << "'"; - stmt << filename; - stmt << "', "; - stmt << binary_data_size; - stmt << ", "; - stmt << "strftime('%Y-%m-%d %H:%M:%f', 'now')"; - stmt << ", "; - stmt << "strftime('%Y-%m-%d %H:%M:%f', 'now')"; - stmt << ")"; - -#ifdef SHOW_STATEMENTS - std::cout << stmt.str() << std::endl; -#endif - - return stmt.str(); -} - -/////////////////////////////////////////////////////////////////////////////// -// SQL statement used in get() to look up the filename and file size -const std::string llDiskCache::sqlComposeGetSelect(const std::string id) -{ - std::ostringstream stmt; - stmt << "SELECT "; - stmt << mFilenameFieldName; - stmt << ", "; - stmt << mFilesizeFieldName; - stmt << " FROM "; - stmt << mTableName; - stmt << " WHERE "; - stmt << mIdFieldName; - stmt << "='"; - stmt << id; - stmt << "'"; - -#ifdef SHOW_STATEMENTS - std::cout << stmt.str() << std::endl; -#endif - - return stmt.str(); -} - -/////////////////////////////////////////////////////////////////////////////// -// SQL statement to update the date/time of last access as well as the -// count of number of times the file has been accessed. -// Note: the more accurate representation of date/time is used to -// ensure ms accuracy vs the standard INTEGER days since epoch approach -const std::string llDiskCache::sqlComposeGetUpdate(const std::string id) -{ - std::ostringstream stmt; - stmt << "UPDATE "; - stmt << mTableName; - stmt << " SET "; - stmt << mAccessCountFieldName; - stmt << "="; - stmt << mAccessCountFieldName; - stmt << "+1"; - stmt << ", "; - stmt << mLastAccessDateTimeFieldName; - stmt << "="; - stmt << "strftime('%Y-%m-%d %H:%M:%f', 'now')"; - stmt << " WHERE "; - stmt << mIdFieldName; - stmt << "='"; - stmt << id; - stmt << "'"; - -#ifdef SHOW_STATEMENTS - std::cout << stmt.str() << std::endl; -#endif - - return stmt.str(); -} - -/////////////////////////////////////////////////////////////////////////////// -// SQL statement to remove items from the database that are older -// than the newest num_elements entries -const std::string llDiskCache::sqlComposePurge(int num_entries) -{ - std::ostringstream stmt; - stmt << "DELETE FROM "; - stmt << mTableName; - stmt << " WHERE "; - stmt << mLastAccessDateTimeFieldName; - stmt << " NOT IN "; - stmt << "("; - stmt << "SELECT "; - stmt << mLastAccessDateTimeFieldName; - stmt << " FROM "; - stmt << mTableName; - stmt << " ORDER BY "; - stmt << mLastAccessDateTimeFieldName; - stmt << " DESC"; - stmt << " LIMIT "; - stmt << num_entries; - stmt << ")"; - -//#ifdef SHOW_STATEMENTS - std::cout << stmt.str() << std::endl; -//#endif - - return stmt.str(); + return total_file_size; } diff --git a/indra/llfilesystem/lldiskcache.h b/indra/llfilesystem/lldiskcache.h index 39b8f7ef48..1618cec6a6 100644 --- a/indra/llfilesystem/lldiskcache.h +++ b/indra/llfilesystem/lldiskcache.h @@ -1,11 +1,6 @@ /** * @file lldiskcache.h - * @brief Declaration SQLite meta data / file storage API - * @brief Declaration of the generic disk cache interface - as well the SQLite header/implementation. - * @Note Eventually, this component might split into an interface - * file and multiple implemtations but for now, this is the - * only one so I think it's okay to combine everything. + * @brief The disk cache implementation. * * $LicenseInfo:firstyear=2009&license=viewerlgpl$ * Second Life Viewer Source Code @@ -32,99 +27,68 @@ #ifndef _LLDISKCACHE #define _LLDISKCACHE -#include -#include +class LLDiskCache +{ + public: + LLDiskCache(const std::string cache_dir); + ~LLDiskCache(); -#include "sqlite3.h" + /** + * Update the "last write time" of a file to "now". This must be called whenever a + * file in the cache is read (not written) so that the last time the file was + * accessed which is used in the mechanism for purging the cache, is up to date. + */ + void updateFileAccessTime(const std::string file_path); -// Enable this line to display each SQL statement when it is executed -// It can lead to a lot of spam but useful for debugging -//#define SHOW_STATEMENTS + /** + * Purge the oldest items in the cache so that the combined size of all files + * is no bigger than mMaxSizeBytes. + */ + void purge(); -// I toyed with the idea of a variety of approaches and thought have -// an abstract base class with which to hang different implementations -// off was a good idea but I think we settled on a single approach -// so I will likely remove this level of indirection if not other -// interesting implementation ideas present themselves. -class llDiskCacheBase -{ - public: - llDiskCacheBase() {}; - virtual ~llDiskCacheBase() {}; - virtual bool open(const std::string db_folder, - const std::string db_filename) = 0; - virtual bool exists(const std::string id, bool& exists) = 0; - virtual bool put(const std::string id, - char* binary_data, - int binary_data_size) = 0; - virtual const bool get(const std::string id, - char** mem_buffer, - int& mem_buffer_size) = 0; - virtual bool purge(int num_entries) = 0; - virtual void close() = 0; - virtual const std::string dbVersion() = 0; - virtual const std::string getFilenameById(const std::string id) = 0; - virtual const int getAccessCountById(const std::string id) = 0; - virtual void getNumEntries(int& num_entries, int& max_entries) = 0; -}; + /** + * Clear the cache by removing the files in the cache directory individually + */ + void clearCache(const std::string cache_dir); -// implementation for the SQLite/disk blended case -// see source file for detailed comments -class llDiskCache : - public llDiskCacheBase -{ - public: - llDiskCache(); - virtual ~llDiskCache(); - virtual bool open(const std::string db_folder, - const std::string db_filename) override; - virtual bool exists(const std::string id, bool& exists) override; - virtual bool put(const std::string id, - char* binary_data, - int binary_data_size) override; - virtual const bool get(const std::string id, - char** mem_buffer, - int& mem_buffer_size) override; - virtual bool purge(int num_entries) override; - virtual void close() override; - virtual const std::string dbVersion() override; - virtual const std::string getFilenameById(const std::string id) override; - virtual const int getAccessCountById(const std::string id) override; - virtual void getNumEntries(int& num_entries, int& max_entries) override; + /** + * Manage max size in bytes of cache - use a discrete setter/getter so the value can + * be changed in the preferences and cache cleared/purged without restarting viewer + * to instantiate this class again. + */ + void setMaxSizeBytes(const uintmax_t size_bytes) { mMaxSizeBytes = size_bytes; } + uintmax_t getMaxSizeBytes() const { return mMaxSizeBytes; } private: - sqlite3* mDb; - std::string mDataStorePath; - const std::string mTableName = "lldiskcache"; - const std::string mIdFieldName = "id"; - const std::string mFilenameFieldName = "filename"; - const std::string mFilesizeFieldName = "filesize"; - const std::string mInsertionDateTimeFieldName = "insert_datetime"; - const std::string mLastAccessDateTimeFieldName = "last_access_datetime"; - const std::string mAccessCountFieldName = "access_count"; + /** + * Utility function to gather the total size the files in a given + * directory. Primarily used here to determine the directory size + * before and after the cache purge + */ + uintmax_t dirFileSize(const std::string dir); private: - void printError(const std::string funcname, - const std::string desc, - bool is_sqlite_err); - bool beginTransaction(); - bool endTransaction(); - std::string makeUniqueFilename(); - const std::string systemSeparator(); - const std::string makeFullPath(const std::string filename); - bool sqliteExec(const std::string stmt, - const std::string funcname); - sqlite3_stmt* sqlitePrepareStep(const std::string funcname, - const std::string stmt, - int sqlite_success_condition); - const std::string sqlComposeCreateTable(); - const std::string sqlComposeExists(const std::string id); - const std::string sqlComposePut(const std::string id, - const std::string filename, - int binary_data_size); - const std::string sqlComposeGetSelect(const std::string id); - const std::string sqlComposeGetUpdate(const std::string id); - const std::string sqlComposePurge(int num_entries); + /** + * Default of 20MB seems reasonable - it will likely be reset + * immediately anyway using a value from the Viewer settings + */ + const uintmax_t mDefaultSizeBytes = 20 * 1024 * 1024; + uintmax_t mMaxSizeBytes; + + /** + * The folder that holds the cached files. The consumer of this + * class must avoid letting the user set this location as a malicious + * setting could potentially point it at a non-cache directory (for example, + * the Windows System dir) with disastrous results. + */ + std::string mCacheDir; + + /** + * This is set from the CacheDebugInfo global setting and + * when enabled, displays additional debugging information in + * various parts of the code + */ + bool mCacheDebugInfo; }; #endif // _LLDISKCACHE diff --git a/indra/llfilesystem/llfilesystem.cpp b/indra/llfilesystem/llfilesystem.cpp index f0037c9a22..ffc3dee12b 100644 --- a/indra/llfilesystem/llfilesystem.cpp +++ b/indra/llfilesystem/llfilesystem.cpp @@ -42,6 +42,8 @@ const S32 LLFileSystem::READ_WRITE = 0x00000003; // LLFileSystem::READ & LLFile const S32 LLFileSystem::APPEND = 0x00000006; // 0x00000004 & LLFileSystem::WRITE static LLTrace::BlockTimerStatHandle FTM_VFILE_WAIT("VFile Wait"); +LLDiskCache* LLFileSystem::mDiskCache = 0; +std::string LLFileSystem::mCacheDirName = "cache"; LLFileSystem::LLFileSystem(const LLUUID &file_id, const LLAssetType::EType file_type, S32 mode) { @@ -104,7 +106,7 @@ const std::string assetTypeToString(LLAssetType::EType at) return std::string("UNKNOWN"); } -const std::string idToFilepath(const std::string id, LLAssetType::EType at) +const std::string LLFileSystem::idToFilepath(const std::string id, LLAssetType::EType at) { /** * For the moment this is just {UUID}_{ASSET_TYPE}.txt but of @@ -117,7 +119,7 @@ const std::string idToFilepath(const std::string id, LLAssetType::EType at) ss << assetTypeToString(at); ss << ".txt"; - const std::string filepath = gDirUtilp->getExpandedFilename(LL_PATH_CACHE, ss.str()); + const std::string filepath = gDirUtilp->getExpandedFilename(LL_PATH_CACHE, mCacheDirName, ss.str()); return filepath; } @@ -289,7 +291,6 @@ BOOL LLFileSystem::write(const U8 *buffer, S32 bytes) BOOL LLFileSystem::writeFile(const U8 *buffer, S32 bytes, const LLUUID &uuid, LLAssetType::EType type) { LLFileSystem file(uuid, type, LLFileSystem::WRITE); - file.setMaxSize(bytes); return file.write(buffer, bytes); } @@ -339,13 +340,6 @@ S32 LLFileSystem::getMaxSize() return INT_MAX; } -BOOL LLFileSystem::setMaxSize(S32 size) -{ - // we don't care what the max size is so we do nothing - // and return true to indicate all was okay - return TRUE; -} - BOOL LLFileSystem::rename(const LLUUID &new_id, const LLAssetType::EType new_type) { LLFileSystem::renameFile(mFileID, mFileType, new_id, new_type); @@ -366,21 +360,15 @@ BOOL LLFileSystem::remove() // static void LLFileSystem::initClass() { -} + const std::string cache_dir = gDirUtilp->getExpandedFilename(LL_PATH_CACHE, mCacheDirName); -// static -void LLFileSystem::cleanupClass() -{ -} + LLFileSystem::mDiskCache = new LLDiskCache(cache_dir); -bool LLFileSystem::isLocked() -{ - // I don't think we care about this test since there is no locking - // TODO: remove this function and calling sites? - return FALSE; + mDiskCache->purge(); } -void LLFileSystem::waitForLock() +// static +void LLFileSystem::cleanupClass() { - // TODO: remove this function and calling sites? + delete LLFileSystem::mDiskCache; } diff --git a/indra/llfilesystem/llfilesystem.h b/indra/llfilesystem/llfilesystem.h index 9cddef74a7..789f9456c7 100644 --- a/indra/llfilesystem/llfilesystem.h +++ b/indra/llfilesystem/llfilesystem.h @@ -32,6 +32,7 @@ #include "lluuid.h" #include "llassettype.h" +#include "lldiskcache.h" class LLFileSystem { @@ -51,13 +52,9 @@ public: S32 getSize(); S32 getMaxSize(); - BOOL setMaxSize(S32 size); BOOL rename(const LLUUID &new_id, const LLAssetType::EType new_type); BOOL remove(); - bool isLocked(); - void waitForLock(); - static bool getExists(const LLUUID &file_id, const LLAssetType::EType file_type); static bool removeFile(const LLUUID &file_id, const LLAssetType::EType file_type); static bool renameFile(const LLUUID &old_file_id, const LLAssetType::EType old_file_type, @@ -80,6 +77,11 @@ protected: S32 mPosition; S32 mMode; S32 mBytesRead; + +private: + static const std::string idToFilepath(const std::string id, LLAssetType::EType at); + static std::string mCacheDirName; + static LLDiskCache* mDiskCache; }; #endif // LL_FILESYSTEM_H diff --git a/indra/llmessage/lltransfertargetvfile.cpp b/indra/llmessage/lltransfertargetvfile.cpp index 471d687d67..f6faadf87f 100644 --- a/indra/llmessage/lltransfertargetvfile.cpp +++ b/indra/llmessage/lltransfertargetvfile.cpp @@ -141,7 +141,6 @@ LLTSCode LLTransferTargetVFile::dataCallback(const S32 packet_id, U8 *in_datap, LLFileSystem vf(mTempID, mParams.getAssetType(), LLFileSystem::APPEND); if (mNeedsCreate) { - vf.setMaxSize(mSize); mNeedsCreate = FALSE; } diff --git a/indra/llmessage/llxfer_vfile.cpp b/indra/llmessage/llxfer_vfile.cpp index 9de9ed379b..12419b342d 100644 --- a/indra/llmessage/llxfer_vfile.cpp +++ b/indra/llmessage/llxfer_vfile.cpp @@ -261,7 +261,6 @@ void LLXfer_VFile::setXferSize (S32 xfer_size) if (! mVFile) { LLFileSystem file(mTempID, mType, LLFileSystem::APPEND); - file.setMaxSize(xfer_size); } } diff --git a/indra/newview/app_settings/settings.xml b/indra/newview/app_settings/settings.xml index c45d6dd7af..0123bc32af 100644 --- a/indra/newview/app_settings/settings.xml +++ b/indra/newview/app_settings/settings.xml @@ -1351,6 +1351,17 @@ Value 23 + CacheDebugInfo + + Comment + When enabled, display additional cache debugging information + Persist + 1 + Type + Boolean + Value + 0 + CacheLocation Comment diff --git a/indra/newview/llfloaterbvhpreview.cpp b/indra/newview/llfloaterbvhpreview.cpp index 08f3b577b4..687d820a18 100644 --- a/indra/newview/llfloaterbvhpreview.cpp +++ b/indra/newview/llfloaterbvhpreview.cpp @@ -1000,7 +1000,6 @@ void LLFloaterBvhPreview::onBtnOK(void* userdata) LLFileSystem file(motionp->getID(), LLAssetType::AT_ANIMATION, LLFileSystem::APPEND); S32 size = dp.getCurrentSize(); - file.setMaxSize(size); if (file.write((U8*)buffer, size)) { std::string name = floaterp->getChild("name_form")->getValue().asString(); diff --git a/indra/newview/llmeshrepository.cpp b/indra/newview/llmeshrepository.cpp index a0d591dc47..3183e6d8fd 100644 --- a/indra/newview/llmeshrepository.cpp +++ b/indra/newview/llmeshrepository.cpp @@ -3241,7 +3241,7 @@ void LLMeshHeaderHandler::processData(LLCore::BufferArray * /* body */, S32 /* b data_size = llmin(data_size, bytes); LLFileSystem file(mesh_id, LLAssetType::AT_MESH, LLFileSystem::WRITE); - if (file.getMaxSize() >= bytes || file.setMaxSize(bytes)) + if (file.getMaxSize() >= bytes) { LLMeshRepository::sCacheBytesWritten += data_size; ++LLMeshRepository::sCacheWrites; diff --git a/indra/newview/llpreviewgesture.cpp b/indra/newview/llpreviewgesture.cpp index 371153aac3..4318a55704 100644 --- a/indra/newview/llpreviewgesture.cpp +++ b/indra/newview/llpreviewgesture.cpp @@ -1140,7 +1140,6 @@ void LLPreviewGesture::saveIfNeeded() LLFileSystem file(assetId, LLAssetType::AT_GESTURE, LLFileSystem::APPEND); S32 size = dp.getCurrentSize(); - file.setMaxSize(size); file.write((U8*)buffer, size); LLLineEditor* descEditor = getChild("desc"); diff --git a/indra/newview/llpreviewnotecard.cpp b/indra/newview/llpreviewnotecard.cpp index 0bccf1d06f..32e1a4a186 100644 --- a/indra/newview/llpreviewnotecard.cpp +++ b/indra/newview/llpreviewnotecard.cpp @@ -563,7 +563,6 @@ bool LLPreviewNotecard::saveIfNeeded(LLInventoryItem* copyitem, bool sync) tid, copyitem); S32 size = buffer.length() + 1; - file.setMaxSize(size); file.write((U8*)buffer.c_str(), size); gAssetStorage->storeAssetData(tid, LLAssetType::AT_NOTECARD, diff --git a/indra/newview/llviewerassetstorage.cpp b/indra/newview/llviewerassetstorage.cpp index df3ff1a3c7..5b76d57196 100644 --- a/indra/newview/llviewerassetstorage.cpp +++ b/indra/newview/llviewerassetstorage.cpp @@ -293,8 +293,6 @@ void LLViewerAssetStorage::storeAssetData( LLFileSystem file(asset_id, asset_type, LLFileSystem::WRITE); - file.setMaxSize(size); - const S32 buf_size = 65536; U8 copy_buf[buf_size]; while ((size = (S32)fread(copy_buf, 1, buf_size, fp))) @@ -528,7 +526,6 @@ void LLViewerAssetStorage::assetRequestCoro( LLUUID temp_id; temp_id.generate(); LLFileSystem vf(temp_id, atype, LLFileSystem::WRITE); - vf.setMaxSize(size); req->mBytesFetched = size; if (!vf.write(raw.data(),size)) { diff --git a/indra/newview/llviewerassetupload.cpp b/indra/newview/llviewerassetupload.cpp index fb3ca69d5d..67ee06e255 100644 --- a/indra/newview/llviewerassetupload.cpp +++ b/indra/newview/llviewerassetupload.cpp @@ -474,8 +474,6 @@ LLSD LLNewFileResourceUploadInfo::exportTempFile() { LLFileSystem file(getAssetId(), assetType, LLFileSystem::WRITE); - file.setMaxSize(file_size); - const S32 buf_size = 65536; U8 copy_buf[buf_size]; while ((file_size = infile.read(copy_buf, buf_size))) @@ -568,7 +566,6 @@ LLSD LLBufferedAssetUploadInfo::prepareUpload() LLFileSystem file(getAssetId(), getAssetType(), LLFileSystem::APPEND); S32 size = mContents.length() + 1; - file.setMaxSize(size); file.write((U8*)mContents.c_str(), size); mStoredToCache = true; -- cgit v1.2.3