From 3092aa8aae496803707980eb456cddbb9960ef1c Mon Sep 17 00:00:00 2001
From: Callum Prentice <callum@gmail.com>
Date: Tue, 6 Oct 2020 17:16:53 -0700
Subject: Add in the C++ filesystem based cache and clean up some indempotent
functions in llfilesystem
---
indra/llfilesystem/lldiskcache.cpp | 809 +++++-------------------------
indra/llfilesystem/lldiskcache.h | 146 ++----
indra/llfilesystem/llfilesystem.cpp | 32 +-
indra/llfilesystem/llfilesystem.h | 10 +-
indra/llmessage/lltransfertargetvfile.cpp | 1 -
indra/llmessage/llxfer_vfile.cpp | 1 -
indra/newview/app_settings/settings.xml | 11 +
indra/newview/llfloaterbvhpreview.cpp | 1 -
indra/newview/llmeshrepository.cpp | 2 +-
indra/newview/llpreviewgesture.cpp | 1 -
indra/newview/llpreviewnotecard.cpp | 1 -
indra/newview/llviewerassetstorage.cpp | 3 -
indra/newview/llviewerassetupload.cpp | 3 -
13 files changed, 196 insertions(+), 825 deletions(-)
(limited to 'indra')
diff --git a/indra/llfilesystem/lldiskcache.cpp b/indra/llfilesystem/lldiskcache.cpp
index 72982db069..4b2ba0dd79 100644
--- a/indra/llfilesystem/lldiskcache.cpp
+++ b/indra/llfilesystem/lldiskcache.cpp
@@ -1,9 +1,6 @@
/**
* @file lldiskcache.cpp
- * @brief The SQLite based disk cache implementation.
- * @Note Eventually, this component might split into an interface
- * file and multiple implemtations but for now, this is the
- * only one so I think it's okay to combine everything.
+ * @brief The disk cache implementation.
*
* $LicenseInfo:firstyear=2009&license=viewerlgpl$
* Second Life Viewer Source Code
@@ -27,747 +24,167 @@
* $/LicenseInfo$
*/
-#if (defined(LL_WINDOWS) || defined(LL_LINUX) || defined(LL_DARWIN))
#include "linden_common.h"
-#endif
+#include "lluuid.h"
+#include "lldir.h"
#include "lldiskcache.h"
-#include <string>
-#include <sstream>
-#include <random>
-#include <algorithm>
-#include <fstream>
-
-///////////////////////////////////////////////////////////////////////////////
-//
-llDiskCache::llDiskCache() :
- mDataStorePath("")
-{
-}
+#include <boost/filesystem.hpp>
+#include <boost/range/iterator_range.hpp>
-llDiskCache::~llDiskCache()
-{
-}
+#include <chrono>
+#include <thread>
+#include <algorithm>
-///////////////////////////////////////////////////////////////////////////////
-// Opens the database - typically done when the application starts
-// and is complementary to close() which is called when the application
-// is finisahed and exits.
-// Pass in the folder and filename of the SQLite database you want to
-// use or create (file doesn't have to exist but the folder must)
-// Returns true or false and writes a message to console on error
-bool llDiskCache::open(const std::string db_folder, const std::string db_filename)
+LLDiskCache::LLDiskCache(const std::string cache_dir) :
+ mCacheDir(cache_dir),
+ mMaxSizeBytes(mDefaultSizeBytes)
{
- mDataStorePath = db_folder;
- std::string db_pathname = makeFullPath(db_filename);
-
- // simple flags for the moment - these will likely be extended
- // later on to support the SQLite mutex model for reading/writing
- // simultaneously - perhaps when we look at supporting textures too
- int flags = SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE;
-
- if (sqlite3_open_v2(
- db_pathname.c_str(),
- &mDb,
- flags,
- nullptr // Name of VFS module to use
- ) != SQLITE_OK)
- {
- printError(__FUNCDNAME__ , "open_v2", true);
- close();
- return false;
- }
-
- // I elected to store the code that generates the statement
- // in sepsrate functions throughout - this seemed like a cleaner
- // approach than having hundreds of stmt << "REPLACE AS" lines
- // interspersed in the logic code. They are all prefixed with
- // 'sqlCompose' and followed by a short description.
- const std::string stmt = sqlComposeCreateTable();
- if (! sqliteExec(stmt, __FUNCDNAME__ ))
+ // no access to LLControlGroup / gSavedSettings so use the system environment
+ // to trigger additional debugging on top of the default "we purged the cache"
+ mCacheDebugInfo = true;
+ if (getenv("LL_CACHE_DEBUGGING") != nullptr)
{
- // Not sure if we need close here - if open fails, then we
- // perhaps don't need to close it - TODO: look in SQLite docs
- close();
- return false;
+ mCacheDebugInfo = true;
}
- return true;
+ // create cache dir if it does not exist
+ boost::filesystem::create_directory(cache_dir);
}
-///////////////////////////////////////////////////////////////////////////////
-// Determines if an entry exists.
-// Pass in the id and a variable that will indicate existance
-// Returns true/false if function succeeded and the boolean
-// you pass in will be set to true/false if entry exists or not
-bool llDiskCache::exists(const std::string id, bool& exists)
+LLDiskCache::~LLDiskCache()
{
- if (!mDb)
- {
- printError(__FUNCDNAME__ , "mDb is invalid", false);
- return false;
- }
-
- if (id.empty())
- {
- printError(__FUNCDNAME__ , "id is empty", false);
- return false;
- }
-
- // As well as the separate function to compose the SQL statement,
- // worth pointing out that the code to 'prepare' and 'step' the
- // SQLite "prepared statement" has been factored out into its own
- // function and used in several other functions.
- const std::string stmt = sqlComposeExists(id);
- sqlite3_stmt* prepared_statement = sqlitePrepareStep(__FUNCDNAME__ , stmt, SQLITE_ROW);
- if (! prepared_statement)
- {
- return false;
- }
-
- int result_column_index = 0;
- int result_count = sqlite3_column_int(prepared_statement, result_column_index);
- if (sqlite3_finalize(prepared_statement) != SQLITE_OK)
- {
- printError(__FUNCDNAME__ , "sqlite3_finalize()", true);
- return false;
- }
-
- // given the uniqueness of the ID, this can only ever be
- // either 1 or 0 so this might be a bit confusing
- exists = result_count > 0 ? true : false;
-
- return true;
-}
-
-///////////////////////////////////////////////////////////////////////////////
-// Given an id (likely a UUID + decoration but can be any string), a
-// pointer to a blob of binary data along with its length, write the
-// entry to the cache
-// Returns true/false for success/failure
-bool llDiskCache::put(const std::string id,
- char* binary_data,
- int binary_data_size)
-{
- if (!mDb)
- {
- printError(__FUNCDNAME__ , "mDb is invalid", false);
- return false;
- }
-
- if (id.empty())
- {
- printError(__FUNCDNAME__ , "id is empty", false);
- return false;
- }
-
- // < 0 is obvious but we assert the data must be at least 1 byte long
- if (binary_data_size <= 0)
- {
- printError(__FUNCDNAME__ , "size of binary file to write is invalid", false);
- return false;
- }
-
- // we generate a unique filename for the actual data itself
- // which is stored on disk directly and not in the database.
- // TODO: consider making the filename more like the ID passed in
- // although the problem with that is we would have to parse the id
- // to remove invalid filename chars, consider length etc. As it
- // stands, we can write a simple SQL statement to return the filename
- // given the ID.
- const std::string filename = makeUniqueFilename();
- const std::string filepath = makeFullPath(filename);
- std::ofstream file(filepath, std::ios::out | std::ios::binary);
- if (! file.is_open())
- {
- std::ostringstream error;
- error << "Unable to open " << filepath << " for writing";
- printError(__FUNCDNAME__ , error.str(), false);
- return false;
- }
-
- file.write((char*)binary_data, binary_data_size);
- file.close();
-
- // I think is a catchall "wasn't able to write the file to disk"
- // conditional but should revisit when we hook this up to viewer
- // code to make sure - we never want to write bad/no data to the
- // disk and not indicate something has gone wrong
- if (file.bad())
- {
- std::ostringstream error;
- error << "Unable to write " << binary_data_size;
- error << " bytes to " << filepath;
- printError(__FUNCDNAME__ , error.str(), false);
-
- return false;
- }
-
- // this is where the filename/size is written to the database along
- // with the current date/time for the created/last access times
- const std::string stmt = sqlComposePut(id, filename, binary_data_size);
- if (! sqlitePrepareStep(__FUNCDNAME__ , stmt, SQLITE_DONE))
- {
- return false;
- }
-
- return true;
}
-///////////////////////////////////////////////////////////////////////////////
-// Given an id (likely a UUID + decoration but can be any string), the
-// address of a pointer that will be used to allocate memory and a
-// varialble that will contain the length of the data, get an item from
-// the cache. Note that the memory buffer returned belongs to the calling
-// function and it is its responsiblity to clean it up when it's no
-// longer needed.
-// Returns true/false for success/failure
-const bool llDiskCache::get(const std::string id,
- char** mem_buffer,
- int& mem_buffer_size)
+void LLDiskCache::purge()
{
- // Check if the entry exists first to avoid dealing with a bunch
- // of conditions that look like failure but aren't in the main code.
- // Note exists() is a public method and also tests for mDB and id
- // being valid so we can safely put this about the same tests
- // in this function
- bool get_exists = false;
- if (! exists(id, get_exists))
+ if (mCacheDebugInfo)
{
- return false;
- }
- if (! get_exists)
- {
- return false;
+ LL_INFOS() << "Total dir size before purge is " << dirFileSize(mCacheDir) << LL_ENDL;
}
- if (!mDb)
- {
- printError(__FUNCDNAME__ , "mDb is invalid", false);
- return false;
- }
-
- if (id.empty())
- {
- printError(__FUNCDNAME__ , "id is empty", false);
- return false;
- }
+ auto start_time = std::chrono::high_resolution_clock::now();
- const std::string stmt_select = sqlComposeGetSelect(id);
- sqlite3_stmt* prepared_statement = sqlitePrepareStep(__FUNCDNAME__ , stmt_select, SQLITE_ROW);
- if (! prepared_statement)
- {
- return false;
- }
+ typedef std::pair<std::time_t, std::pair<uintmax_t, std::string>> file_info_t;
+ std::vector<file_info_t> file_info;
- int result_column_index = 0;
- const unsigned char* text = sqlite3_column_text(prepared_statement, result_column_index);
- if (text == nullptr)
+ if (boost::filesystem::is_directory(mCacheDir))
{
- printError(__FUNCDNAME__ , "filename is nullptr", true);
- return false;
- }
- const std::string filename = std::string(reinterpret_cast<const char*>(text));
- const std::string filepath = makeFullPath(filename);
+ for (auto& entry : boost::make_iterator_range(boost::filesystem::directory_iterator(mCacheDir), {}))
+ {
+ if (boost::filesystem::is_regular_file(entry))
+ {
+ uintmax_t file_size = boost::filesystem::file_size(entry);
+ const std::string file_path = entry.path().string();
+ const std::time_t file_time = boost::filesystem::last_write_time(entry);
- result_column_index = 1;
- int filesize_db = sqlite3_column_int(prepared_statement, result_column_index);
- if (filesize_db <= 0)
- {
- printError(__FUNCDNAME__ , "filesize is invalid", true);
- return false;
+ file_info.push_back(file_info_t(file_time, { file_size, file_path }));
+ }
+ }
}
- if (sqlite3_finalize(prepared_statement) != SQLITE_OK)
+ std::sort(file_info.begin(), file_info.end(), [](file_info_t& x, file_info_t& y)
{
- printError(__FUNCDNAME__ , "sqlite3_finalize()", true);
- return false;
- }
+ return x.first > y.first;
+ });
- // Now we have the fiename, we can read the file from disk
- std::ifstream file(filepath, std::ios::in | std::ios::binary | std::ios::ate);
- if (! file.is_open())
- {
- std::ostringstream error;
- error << "Unable to open " << filepath << " for reading";
- printError(__FUNCDNAME__ , error.str(), false);
- return false;
- }
+ LL_INFOS() << "Purging cache to a maximum of " << mMaxSizeBytes << " bytes" << LL_ENDL;
- // we get the expected filesize from the database but we can also
- // get it (easily) when we read the file from the disk. We compare
- // the two and return false if they don't match
- std::streampos filesize_file = file.tellg();
- if (filesize_db != filesize_file)
+ uintmax_t file_size_total = 0;
+ for (file_info_t& entry : file_info)
{
- std::ostringstream error;
- error << "File size from DB (" << filesize_db << ")";
- error << " and ";
- error << "file size from file (" << filesize_file << ")";
- error << " in file " << filepath << " are different";
- printError(__FUNCDNAME__ , error.str(), false);
-
- return false;
- }
-
- // doest matter if we choose DB or file version - they must be
- // identical if we get this far - just used for clarity
- int filesize = filesize_db;
+ file_size_total += entry.second.first;
- // allocate a block of memory that we pass back for the calling
- // function to use then delete when it's no longer needed
- *mem_buffer = new char[filesize];
- mem_buffer_size = filesize;
+ std::string action = "";
+ if (file_size_total > mMaxSizeBytes)
+ {
+ action = "DELETE:";
+ boost::filesystem::remove(entry.second.second);
+ }
+ else
+ {
+ action = " KEEP:";
+ }
- file.seekg(0, std::ios::beg);
- file.read(*mem_buffer, filesize);
- file.close();
+ if (mCacheDebugInfo)
+ {
+ // have to do this because of LL_INFO/LL_END weirdness
+ std::ostringstream line;
- if (file.bad())
- {
- std::ostringstream error;
- error << "Unable to read " << filesize;
- error << " bytes from " << filepath;
- printError(__FUNCDNAME__ , error.str(), false);
-
- return false;
+ line << action << " ";
+ line << entry.first << " ";
+ line << entry.second.first << " ";
+ line << entry.second.second;
+ line << " (" << file_size_total << "/" << mMaxSizeBytes << ")";
+ LL_INFOS() << line.str() << LL_ENDL;
+ }
}
- // here we update the count of times the file is accessed so
- // we can keep track of how many times it's been requested.
- // This will be useful for metrics and perhaps determining
- // if a file should not be purged even though its age
- // might suggest that it should.
- // In addition, this is where the time of last access is updated
- // in the database and that us used to determine what is purged
- // in an LRU fashion when the purge function is called.
- const std::string stmt_update = sqlComposeGetUpdate(id);
- if (! sqliteExec(stmt_update, __FUNCDNAME__ ))
+ if (mCacheDebugInfo)
{
- return false;
+ auto end_time = std::chrono::high_resolution_clock::now();
+ auto execute_time = std::chrono::duration_cast<std::chrono::milliseconds>(end_time - start_time).count();
+ LL_INFOS() << "Total dir size after purge is " << dirFileSize(mCacheDir) << LL_ENDL;
+ LL_INFOS() << "Cache purge took " << execute_time << " ms to execute for " << file_info.size() << " files" << LL_ENDL;
}
-
- return true;
-}
-
-///////////////////////////////////////////////////////////////////////////////
-// Purges the database of older entries using an LRU approach.
-// Pass in the number of entries to retain.
-// This is called now after open to "clean up" the cache when the
-// application starts.
-// TODO: IMPORTANT: compose a list of files that will be deleted
-// and delete them from disk too - not just from the DB
-bool llDiskCache::purge(int num_entries)
-{
- if (num_entries < 0)
- {
- printError(__FUNCDNAME__ , "number of entries to purge is invalid", false);
- return false;
- }
-
- // find the rows affected and get the filenames for them
-//swww
-
- // delete oldest entries leaving the correct number in place
- const std::string stmt = sqlComposePurge(num_entries);
- if (! sqliteExec(stmt, __FUNCDNAME__ ))
- {
- return false;
- }
-
- return true;
-}
-
-///////////////////////////////////////////////////////////////////////////////
-// Call at application shutdown
-void llDiskCache::close()
-{
- sqlite3_close(mDb);
-}
-
-///////////////////////////////////////////////////////////////////////////////
-// Determine the version of SQLite in use
-// TODO: make this a static so we can get to it from the Viewer About
-// box without instantiating the whole thing.
-const std::string llDiskCache::dbVersion()
-{
- std::ostringstream version;
-
- version << sqlite3_libversion();
-
- return version.str();
}
-///////////////////////////////////////////////////////////////////////////////
-// Given an id, return the matching filename
-const std::string llDiskCache::getFilenameById(const std::string id)
-{
- // TODO:
- return std::string();
-}
-
-///////////////////////////////////////////////////////////////////////////////
-// Given an id, return the number of times that entry has been
-// accessed from the cache
-const int llDiskCache::getAccessCountById(const std::string id)
-{
- // TODO:
- return -1;
-}
-
-///////////////////////////////////////////////////////////////////////////////
-// Return the number of entries currently in the cache as well as
-// the maximum possible entries.
-void llDiskCache::getNumEntries(int& num_entries, int& max_entries)
-{
- num_entries = -1;
- max_entries = -1;
-}
-
-///////////////////////////////////////////////////////////////////////////////
-// Wraps the sqlite3_exec(..) used in many places
-bool llDiskCache::sqliteExec(const std::string stmt,
- const std::string funcname)
-{
- if (sqlite3_exec(
- mDb,
- stmt.c_str(),
- nullptr, // Callback function (unused)
- nullptr, // 1st argument to callback (unused)
- nullptr // Error msg written here (unused)
- ) != SQLITE_OK)
- {
- printError(funcname, "sqlite3_exec", true);
- return false;
- }
-
- return true;
-}
-
-///////////////////////////////////////////////////////////////////////////////
-// Wraps the sqlite3_prepare_v2 and sqlite3_step calls used in many places
-sqlite3_stmt* llDiskCache::sqlitePrepareStep(const std::string funcname,
- const std::string stmt,
- int sqlite_success_condition)
-{
- sqlite3_stmt* prepared_statement;
-
- if (sqlite3_prepare_v2(
- mDb,
- stmt.c_str(),
- -1, // Maximum length of zSql in bytes.
- &prepared_statement,
- 0 // OUT: Pointer to unused portion of zSql
- ) != SQLITE_OK)
- {
- printError(funcname, "sqlite3_prepare_v2", true);
- return nullptr;
- }
-
- if (sqlite3_step(prepared_statement) != sqlite_success_condition)
- {
- printError(funcname, "sqlite3_step", true);
- sqlite3_finalize(prepared_statement);
- return nullptr;
- }
- return prepared_statement;
-}
-
-///////////////////////////////////////////////////////////////////////////////
-// When an "error" occurss - e.g. database cannot be found, file cannot be
-// written, invalid argument passed into a function etc. a message is
-// written to stderr that should end up in the viewer log
-// TODO: Set the verbosity using the usual Viewer mechanism
-void llDiskCache::printError(const std::string funcname,
- const std::string desc,
- bool is_sqlite_err)
-{
- std::ostringstream err_msg;
-
- err_msg << "llDiskCache error in ";
- err_msg << __FUNCDNAME__ << "(...) ";
- err_msg << desc;
-
- if (is_sqlite_err)
- {
- err_msg << " - ";
- err_msg << std::string(sqlite3_errmsg(mDb));
- }
-
- // TODO: set via viewer verbosity level
- const int verbosity = 1;
- if (verbosity > 0)
- {
- std::cerr << err_msg.str() << std::endl;
- }
-}
-
-///////////////////////////////////////////////////////////////////////////////
-// wrapper for SQLite code to begin an SQL transaction - not used yet but
-// it will be eventually
-bool llDiskCache::beginTransaction()
+/**
+ * Update the "last write time" of a file to "now". This must be called whenever a
+ * file in the cache is read (not written) so that the last time the file was
+ * accessed which is used in the mechanism for purging the cache, is up to date.
+ */
+void LLDiskCache::updateFileAccessTime(const std::string file_path)
{
- const std::string stmt("BEGIN TRANSACTION");
- if (! sqliteExec(stmt, __FUNCDNAME__ ))
- {
- return false;
- }
-
- return true;
+ const std::time_t file_time = std::time(nullptr);
+ boost::filesystem::last_write_time(file_path, file_time);
}
-///////////////////////////////////////////////////////////////////////////////
-// wrapper for SQLite code to end an SQL transaction - not used yet but
-// it will be eventually
-bool llDiskCache::endTransaction()
+/**
+ * Clear the cache by removing all the files in the cache directory
+ * individually. It's important to maintain control of which directory
+ * if passed in and not let the user inadvertently (or maliciously) set
+ * it to an random location like your project source or OS system directory
+ */
+void LLDiskCache::clearCache(const std::string cache_dir)
{
- const std::string stmt("COMMIT");
- if (! sqliteExec(stmt, __FUNCDNAME__ ))
+ if (boost::filesystem::is_directory(cache_dir))
{
- return false;
+ for (auto& entry : boost::make_iterator_range(boost::filesystem::directory_iterator(cache_dir), {}))
+ {
+ if (boost::filesystem::is_regular_file(entry))
+ {
+ boost::filesystem::remove(entry);
+ }
+ }
}
-
- return true;
}
-///////////////////////////////////////////////////////////////////////////////
-// Build a unique filename that will be used to store the actual file
-// on disk (as opposed to the meta data in the database)
-// TODO: I think this needs more work once we move it to the viewer
-// and espcially to make it cross platform - see 'std::tmpnam'
-// depreciation comments in compiler output for example
-std::string llDiskCache::makeUniqueFilename()
+/**
+ * Utility function to get the total filesize of all files in a directory. It
+ * used to test file extensions to only check cache files but that was removed.
+ * There may be a better way that works directly on the folder (similar to
+ * right clicking on a folder in the OS and asking for size vs right clicking
+ * on all files and adding up manually) but this is very fast - less than 100ms
+ * in my testing so, so long as it's not called frequently, it should be okay.
+ * Note that's it's only currently used for logging/debugging so if performance
+ * is ever an issue, optimizing this or removing it altogether, is an easy win.
+ */
+uintmax_t LLDiskCache::dirFileSize(const std::string dir)
{
- // TODO: replace with boost - this is marked as deprecated?
- std::string base = std::tmpnam(nullptr);
-
- // C++11 random number generation!!!
- static std::random_device dev;
- static std::mt19937 rng(dev());
- std::uniform_int_distribution<int> dist(100000, 999999);
+ uintmax_t total_file_size = 0;
- // currently the tmp filename from std::tmpnam() on macOS
- // is of the form `/tmp/foo/bar.12345 and the following code
- // strips all the preceding dirs - we likely want a more
- // robust (and cross platform solution) when we move to the
- // viewer code
- std::size_t found = base.rfind(systemSeparator());
- if (found != std::string::npos && found < base.size() - 2)
- {
- base = base.substr(found + 1);
- }
- else
+ if (boost::filesystem::is_directory(dir))
{
- base = "";
+ for (auto& entry : boost::make_iterator_range(boost::filesystem::directory_iterator(dir), {}))
+ {
+ if (boost::filesystem::is_regular_file(entry))
+ {
+ total_file_size += boost::filesystem::file_size(entry);
+ }
+ }
}
- // we mix in a random number for some more entropy..
- // (i know, i know...)
- std::ostringstream unique_filename;
- unique_filename << base;
- unique_filename << ".";
- unique_filename << dist(rng);
-
- return unique_filename.str();
-}
-
-///////////////////////////////////////////////////////////////////////////////
-// Return system file/path separator - likely replaced by the version
-// in the viewer
-const std::string llDiskCache::systemSeparator()
-{
-// TODO: replace in viewer with relevant call
-#ifdef _WIN32
- return "\\";
-#else
- return "/";
-#endif
-}
-
-///////////////////////////////////////////////////////////////////////////////
-// Given a filename, compose a full path based on the path name passed
-// in when the database was opened and the separator in play.
-const std::string llDiskCache::makeFullPath(const std::string filename)
-{
- std::string pathname = mDataStorePath + systemSeparator() + filename;
-
- return pathname;
-}
-
-///////////////////////////////////////////////////////////////////////////////
-//
-const std::string llDiskCache::sqlComposeCreateTable()
-{
- std::ostringstream stmt;
- stmt << "CREATE TABLE IF NOT EXISTS ";
- stmt << mTableName;
- stmt << "(";
- stmt << mIdFieldName;
- stmt << " TEXT PRIMARY KEY, ";
- stmt << mFilenameFieldName;
- stmt << " TEXT, ";
- stmt << mFilesizeFieldName;
- stmt << " INTEGER DEFAULT 0, ";
- stmt << mInsertionDateTimeFieldName;
- stmt << " TEXT, ";
- stmt << mLastAccessDateTimeFieldName;
- stmt << " TEXT, ";
- stmt << mAccessCountFieldName;
- stmt << " INTEGER DEFAULT 0";
- stmt << ")";
-
-#ifdef SHOW_STATEMENTS
- std::cout << stmt.str() << std::endl;
-#endif
-
- return stmt.str();
-}
-
-///////////////////////////////////////////////////////////////////////////////
-//
-const std::string llDiskCache::sqlComposeExists(const std::string id)
-{
- std::ostringstream stmt;
- stmt << "SELECT COUNT(*) FROM ";
- stmt << mTableName;
- stmt << " WHERE ";
- stmt << mIdFieldName;
- stmt << "='";
- stmt << id;
- stmt << "'";
-
-#ifdef SHOW_STATEMENTS
- std::cout << stmt.str() << std::endl;
-#endif
-
- return stmt.str();
-}
-
-///////////////////////////////////////////////////////////////////////////////
-// SQL statement to write an entry to the DB
-// Saves id, filename (generated), file size and create/last access date
-const std::string llDiskCache::sqlComposePut(const std::string id,
- const std::string filename,
- int binary_data_size)
-{
- std::ostringstream stmt;
- stmt << "REPLACE INTO ";
- stmt << mTableName;
- stmt << "(";
- stmt << mIdFieldName;
- stmt << ",";
- stmt << mFilenameFieldName;
- stmt << ",";
- stmt << mFilesizeFieldName;
- stmt << ",";
- stmt << mInsertionDateTimeFieldName;
- stmt << ",";
- stmt << mLastAccessDateTimeFieldName;
- stmt << ") ";
- stmt << "VALUES(";
- stmt << "'";
- stmt << id;
- stmt << "', ";
- stmt << "'";
- stmt << filename;
- stmt << "', ";
- stmt << binary_data_size;
- stmt << ", ";
- stmt << "strftime('%Y-%m-%d %H:%M:%f', 'now')";
- stmt << ", ";
- stmt << "strftime('%Y-%m-%d %H:%M:%f', 'now')";
- stmt << ")";
-
-#ifdef SHOW_STATEMENTS
- std::cout << stmt.str() << std::endl;
-#endif
-
- return stmt.str();
-}
-
-///////////////////////////////////////////////////////////////////////////////
-// SQL statement used in get() to look up the filename and file size
-const std::string llDiskCache::sqlComposeGetSelect(const std::string id)
-{
- std::ostringstream stmt;
- stmt << "SELECT ";
- stmt << mFilenameFieldName;
- stmt << ", ";
- stmt << mFilesizeFieldName;
- stmt << " FROM ";
- stmt << mTableName;
- stmt << " WHERE ";
- stmt << mIdFieldName;
- stmt << "='";
- stmt << id;
- stmt << "'";
-
-#ifdef SHOW_STATEMENTS
- std::cout << stmt.str() << std::endl;
-#endif
-
- return stmt.str();
-}
-
-///////////////////////////////////////////////////////////////////////////////
-// SQL statement to update the date/time of last access as well as the
-// count of number of times the file has been accessed.
-// Note: the more accurate representation of date/time is used to
-// ensure ms accuracy vs the standard INTEGER days since epoch approach
-const std::string llDiskCache::sqlComposeGetUpdate(const std::string id)
-{
- std::ostringstream stmt;
- stmt << "UPDATE ";
- stmt << mTableName;
- stmt << " SET ";
- stmt << mAccessCountFieldName;
- stmt << "=";
- stmt << mAccessCountFieldName;
- stmt << "+1";
- stmt << ", ";
- stmt << mLastAccessDateTimeFieldName;
- stmt << "=";
- stmt << "strftime('%Y-%m-%d %H:%M:%f', 'now')";
- stmt << " WHERE ";
- stmt << mIdFieldName;
- stmt << "='";
- stmt << id;
- stmt << "'";
-
-#ifdef SHOW_STATEMENTS
- std::cout << stmt.str() << std::endl;
-#endif
-
- return stmt.str();
-}
-
-///////////////////////////////////////////////////////////////////////////////
-// SQL statement to remove items from the database that are older
-// than the newest num_elements entries
-const std::string llDiskCache::sqlComposePurge(int num_entries)
-{
- std::ostringstream stmt;
- stmt << "DELETE FROM ";
- stmt << mTableName;
- stmt << " WHERE ";
- stmt << mLastAccessDateTimeFieldName;
- stmt << " NOT IN ";
- stmt << "(";
- stmt << "SELECT ";
- stmt << mLastAccessDateTimeFieldName;
- stmt << " FROM ";
- stmt << mTableName;
- stmt << " ORDER BY ";
- stmt << mLastAccessDateTimeFieldName;
- stmt << " DESC";
- stmt << " LIMIT ";
- stmt << num_entries;
- stmt << ")";
-
-//#ifdef SHOW_STATEMENTS
- std::cout << stmt.str() << std::endl;
-//#endif
-
- return stmt.str();
+ return total_file_size;
}
diff --git a/indra/llfilesystem/lldiskcache.h b/indra/llfilesystem/lldiskcache.h
index 39b8f7ef48..1618cec6a6 100644
--- a/indra/llfilesystem/lldiskcache.h
+++ b/indra/llfilesystem/lldiskcache.h
@@ -1,11 +1,6 @@
/**
* @file lldiskcache.h
- * @brief Declaration SQLite meta data / file storage API
- * @brief Declaration of the generic disk cache interface
- as well the SQLite header/implementation.
- * @Note Eventually, this component might split into an interface
- * file and multiple implemtations but for now, this is the
- * only one so I think it's okay to combine everything.
+ * @brief The disk cache implementation.
*
* $LicenseInfo:firstyear=2009&license=viewerlgpl$
* Second Life Viewer Source Code
@@ -32,99 +27,68 @@
#ifndef _LLDISKCACHE
#define _LLDISKCACHE
-#include <iostream>
-#include <string>
+class LLDiskCache
+{
+ public:
+ LLDiskCache(const std::string cache_dir);
+ ~LLDiskCache();
-#include "sqlite3.h"
+ /**
+ * Update the "last write time" of a file to "now". This must be called whenever a
+ * file in the cache is read (not written) so that the last time the file was
+ * accessed which is used in the mechanism for purging the cache, is up to date.
+ */
+ void updateFileAccessTime(const std::string file_path);
-// Enable this line to display each SQL statement when it is executed
-// It can lead to a lot of spam but useful for debugging
-//#define SHOW_STATEMENTS
+ /**
+ * Purge the oldest items in the cache so that the combined size of all files
+ * is no bigger than mMaxSizeBytes.
+ */
+ void purge();
-// I toyed with the idea of a variety of approaches and thought have
-// an abstract base class with which to hang different implementations
-// off was a good idea but I think we settled on a single approach
-// so I will likely remove this level of indirection if not other
-// interesting implementation ideas present themselves.
-class llDiskCacheBase
-{
- public:
- llDiskCacheBase() {};
- virtual ~llDiskCacheBase() {};
- virtual bool open(const std::string db_folder,
- const std::string db_filename) = 0;
- virtual bool exists(const std::string id, bool& exists) = 0;
- virtual bool put(const std::string id,
- char* binary_data,
- int binary_data_size) = 0;
- virtual const bool get(const std::string id,
- char** mem_buffer,
- int& mem_buffer_size) = 0;
- virtual bool purge(int num_entries) = 0;
- virtual void close() = 0;
- virtual const std::string dbVersion() = 0;
- virtual const std::string getFilenameById(const std::string id) = 0;
- virtual const int getAccessCountById(const std::string id) = 0;
- virtual void getNumEntries(int& num_entries, int& max_entries) = 0;
-};
+ /**
+ * Clear the cache by removing the files in the cache directory individually
+ */
+ void clearCache(const std::string cache_dir);
-// implementation for the SQLite/disk blended case
-// see source file for detailed comments
-class llDiskCache :
- public llDiskCacheBase
-{
- public:
- llDiskCache();
- virtual ~llDiskCache();
- virtual bool open(const std::string db_folder,
- const std::string db_filename) override;
- virtual bool exists(const std::string id, bool& exists) override;
- virtual bool put(const std::string id,
- char* binary_data,
- int binary_data_size) override;
- virtual const bool get(const std::string id,
- char** mem_buffer,
- int& mem_buffer_size) override;
- virtual bool purge(int num_entries) override;
- virtual void close() override;
- virtual const std::string dbVersion() override;
- virtual const std::string getFilenameById(const std::string id) override;
- virtual const int getAccessCountById(const std::string id) override;
- virtual void getNumEntries(int& num_entries, int& max_entries) override;
+ /**
+ * Manage max size in bytes of cache - use a discrete setter/getter so the value can
+ * be changed in the preferences and cache cleared/purged without restarting viewer
+ * to instantiate this class again.
+ */
+ void setMaxSizeBytes(const uintmax_t size_bytes) { mMaxSizeBytes = size_bytes; }
+ uintmax_t getMaxSizeBytes() const { return mMaxSizeBytes; }
private:
- sqlite3* mDb;
- std::string mDataStorePath;
- const std::string mTableName = "lldiskcache";
- const std::string mIdFieldName = "id";
- const std::string mFilenameFieldName = "filename";
- const std::string mFilesizeFieldName = "filesize";
- const std::string mInsertionDateTimeFieldName = "insert_datetime";
- const std::string mLastAccessDateTimeFieldName = "last_access_datetime";
- const std::string mAccessCountFieldName = "access_count";
+ /**
+ * Utility function to gather the total size the files in a given
+ * directory. Primarily used here to determine the directory size
+ * before and after the cache purge
+ */
+ uintmax_t dirFileSize(const std::string dir);
private:
- void printError(const std::string funcname,
- const std::string desc,
- bool is_sqlite_err);
- bool beginTransaction();
- bool endTransaction();
- std::string makeUniqueFilename();
- const std::string systemSeparator();
- const std::string makeFullPath(const std::string filename);
- bool sqliteExec(const std::string stmt,
- const std::string funcname);
- sqlite3_stmt* sqlitePrepareStep(const std::string funcname,
- const std::string stmt,
- int sqlite_success_condition);
- const std::string sqlComposeCreateTable();
- const std::string sqlComposeExists(const std::string id);
- const std::string sqlComposePut(const std::string id,
- const std::string filename,
- int binary_data_size);
- const std::string sqlComposeGetSelect(const std::string id);
- const std::string sqlComposeGetUpdate(const std::string id);
- const std::string sqlComposePurge(int num_entries);
+ /**
+ * Default of 20MB seems reasonable - it will likely be reset
+ * immediately anyway using a value from the Viewer settings
+ */
+ const uintmax_t mDefaultSizeBytes = 20 * 1024 * 1024;
+ uintmax_t mMaxSizeBytes;
+
+ /**
+ * The folder that holds the cached files. The consumer of this
+ * class must avoid letting the user set this location as a malicious
+ * setting could potentially point it at a non-cache directory (for example,
+ * the Windows System dir) with disastrous results.
+ */
+ std::string mCacheDir;
+
+ /**
+ * This is set from the CacheDebugInfo global setting and
+ * when enabled, displays additional debugging information in
+ * various parts of the code
+ */
+ bool mCacheDebugInfo;
};
#endif // _LLDISKCACHE
diff --git a/indra/llfilesystem/llfilesystem.cpp b/indra/llfilesystem/llfilesystem.cpp
index f0037c9a22..ffc3dee12b 100644
--- a/indra/llfilesystem/llfilesystem.cpp
+++ b/indra/llfilesystem/llfilesystem.cpp
@@ -42,6 +42,8 @@ const S32 LLFileSystem::READ_WRITE = 0x00000003; // LLFileSystem::READ & LLFile
const S32 LLFileSystem::APPEND = 0x00000006; // 0x00000004 & LLFileSystem::WRITE
static LLTrace::BlockTimerStatHandle FTM_VFILE_WAIT("VFile Wait");
+LLDiskCache* LLFileSystem::mDiskCache = 0;
+std::string LLFileSystem::mCacheDirName = "cache";
LLFileSystem::LLFileSystem(const LLUUID &file_id, const LLAssetType::EType file_type, S32 mode)
{
@@ -104,7 +106,7 @@ const std::string assetTypeToString(LLAssetType::EType at)
return std::string("UNKNOWN");
}
-const std::string idToFilepath(const std::string id, LLAssetType::EType at)
+const std::string LLFileSystem::idToFilepath(const std::string id, LLAssetType::EType at)
{
/**
* For the moment this is just {UUID}_{ASSET_TYPE}.txt but of
@@ -117,7 +119,7 @@ const std::string idToFilepath(const std::string id, LLAssetType::EType at)
ss << assetTypeToString(at);
ss << ".txt";
- const std::string filepath = gDirUtilp->getExpandedFilename(LL_PATH_CACHE, ss.str());
+ const std::string filepath = gDirUtilp->getExpandedFilename(LL_PATH_CACHE, mCacheDirName, ss.str());
return filepath;
}
@@ -289,7 +291,6 @@ BOOL LLFileSystem::write(const U8 *buffer, S32 bytes)
BOOL LLFileSystem::writeFile(const U8 *buffer, S32 bytes, const LLUUID &uuid, LLAssetType::EType type)
{
LLFileSystem file(uuid, type, LLFileSystem::WRITE);
- file.setMaxSize(bytes);
return file.write(buffer, bytes);
}
@@ -339,13 +340,6 @@ S32 LLFileSystem::getMaxSize()
return INT_MAX;
}
-BOOL LLFileSystem::setMaxSize(S32 size)
-{
- // we don't care what the max size is so we do nothing
- // and return true to indicate all was okay
- return TRUE;
-}
-
BOOL LLFileSystem::rename(const LLUUID &new_id, const LLAssetType::EType new_type)
{
LLFileSystem::renameFile(mFileID, mFileType, new_id, new_type);
@@ -366,21 +360,15 @@ BOOL LLFileSystem::remove()
// static
void LLFileSystem::initClass()
{
-}
+ const std::string cache_dir = gDirUtilp->getExpandedFilename(LL_PATH_CACHE, mCacheDirName);
-// static
-void LLFileSystem::cleanupClass()
-{
-}
+ LLFileSystem::mDiskCache = new LLDiskCache(cache_dir);
-bool LLFileSystem::isLocked()
-{
- // I don't think we care about this test since there is no locking
- // TODO: remove this function and calling sites?
- return FALSE;
+ mDiskCache->purge();
}
-void LLFileSystem::waitForLock()
+// static
+void LLFileSystem::cleanupClass()
{
- // TODO: remove this function and calling sites?
+ delete LLFileSystem::mDiskCache;
}
diff --git a/indra/llfilesystem/llfilesystem.h b/indra/llfilesystem/llfilesystem.h
index 9cddef74a7..789f9456c7 100644
--- a/indra/llfilesystem/llfilesystem.h
+++ b/indra/llfilesystem/llfilesystem.h
@@ -32,6 +32,7 @@
#include "lluuid.h"
#include "llassettype.h"
+#include "lldiskcache.h"
class LLFileSystem
{
@@ -51,13 +52,9 @@ public:
S32 getSize();
S32 getMaxSize();
- BOOL setMaxSize(S32 size);
BOOL rename(const LLUUID &new_id, const LLAssetType::EType new_type);
BOOL remove();
- bool isLocked();
- void waitForLock();
-
static bool getExists(const LLUUID &file_id, const LLAssetType::EType file_type);
static bool removeFile(const LLUUID &file_id, const LLAssetType::EType file_type);
static bool renameFile(const LLUUID &old_file_id, const LLAssetType::EType old_file_type,
@@ -80,6 +77,11 @@ protected:
S32 mPosition;
S32 mMode;
S32 mBytesRead;
+
+private:
+ static const std::string idToFilepath(const std::string id, LLAssetType::EType at);
+ static std::string mCacheDirName;
+ static LLDiskCache* mDiskCache;
};
#endif // LL_FILESYSTEM_H
diff --git a/indra/llmessage/lltransfertargetvfile.cpp b/indra/llmessage/lltransfertargetvfile.cpp
index 471d687d67..f6faadf87f 100644
--- a/indra/llmessage/lltransfertargetvfile.cpp
+++ b/indra/llmessage/lltransfertargetvfile.cpp
@@ -141,7 +141,6 @@ LLTSCode LLTransferTargetVFile::dataCallback(const S32 packet_id, U8 *in_datap,
LLFileSystem vf(mTempID, mParams.getAssetType(), LLFileSystem::APPEND);
if (mNeedsCreate)
{
- vf.setMaxSize(mSize);
mNeedsCreate = FALSE;
}
diff --git a/indra/llmessage/llxfer_vfile.cpp b/indra/llmessage/llxfer_vfile.cpp
index 9de9ed379b..12419b342d 100644
--- a/indra/llmessage/llxfer_vfile.cpp
+++ b/indra/llmessage/llxfer_vfile.cpp
@@ -261,7 +261,6 @@ void LLXfer_VFile::setXferSize (S32 xfer_size)
if (! mVFile)
{
LLFileSystem file(mTempID, mType, LLFileSystem::APPEND);
- file.setMaxSize(xfer_size);
}
}
diff --git a/indra/newview/app_settings/settings.xml b/indra/newview/app_settings/settings.xml
index c45d6dd7af..0123bc32af 100644
--- a/indra/newview/app_settings/settings.xml
+++ b/indra/newview/app_settings/settings.xml
@@ -1351,6 +1351,17 @@
<key>Value</key>
<integer>23</integer>
</map>
+ <key>CacheDebugInfo</key>
+ <map>
+ <key>Comment</key>
+ <string>When enabled, display additional cache debugging information</string>
+ <key>Persist</key>
+ <integer>1</integer>
+ <key>Type</key>
+ <string>Boolean</string>
+ <key>Value</key>
+ <integer>0</integer>
+ </map>
<key>CacheLocation</key>
<map>
<key>Comment</key>
diff --git a/indra/newview/llfloaterbvhpreview.cpp b/indra/newview/llfloaterbvhpreview.cpp
index 08f3b577b4..687d820a18 100644
--- a/indra/newview/llfloaterbvhpreview.cpp
+++ b/indra/newview/llfloaterbvhpreview.cpp
@@ -1000,7 +1000,6 @@ void LLFloaterBvhPreview::onBtnOK(void* userdata)
LLFileSystem file(motionp->getID(), LLAssetType::AT_ANIMATION, LLFileSystem::APPEND);
S32 size = dp.getCurrentSize();
- file.setMaxSize(size);
if (file.write((U8*)buffer, size))
{
std::string name = floaterp->getChild<LLUICtrl>("name_form")->getValue().asString();
diff --git a/indra/newview/llmeshrepository.cpp b/indra/newview/llmeshrepository.cpp
index a0d591dc47..3183e6d8fd 100644
--- a/indra/newview/llmeshrepository.cpp
+++ b/indra/newview/llmeshrepository.cpp
@@ -3241,7 +3241,7 @@ void LLMeshHeaderHandler::processData(LLCore::BufferArray * /* body */, S32 /* b
data_size = llmin(data_size, bytes);
LLFileSystem file(mesh_id, LLAssetType::AT_MESH, LLFileSystem::WRITE);
- if (file.getMaxSize() >= bytes || file.setMaxSize(bytes))
+ if (file.getMaxSize() >= bytes)
{
LLMeshRepository::sCacheBytesWritten += data_size;
++LLMeshRepository::sCacheWrites;
diff --git a/indra/newview/llpreviewgesture.cpp b/indra/newview/llpreviewgesture.cpp
index 371153aac3..4318a55704 100644
--- a/indra/newview/llpreviewgesture.cpp
+++ b/indra/newview/llpreviewgesture.cpp
@@ -1140,7 +1140,6 @@ void LLPreviewGesture::saveIfNeeded()
LLFileSystem file(assetId, LLAssetType::AT_GESTURE, LLFileSystem::APPEND);
S32 size = dp.getCurrentSize();
- file.setMaxSize(size);
file.write((U8*)buffer, size);
LLLineEditor* descEditor = getChild<LLLineEditor>("desc");
diff --git a/indra/newview/llpreviewnotecard.cpp b/indra/newview/llpreviewnotecard.cpp
index 0bccf1d06f..32e1a4a186 100644
--- a/indra/newview/llpreviewnotecard.cpp
+++ b/indra/newview/llpreviewnotecard.cpp
@@ -563,7 +563,6 @@ bool LLPreviewNotecard::saveIfNeeded(LLInventoryItem* copyitem, bool sync)
tid, copyitem);
S32 size = buffer.length() + 1;
- file.setMaxSize(size);
file.write((U8*)buffer.c_str(), size);
gAssetStorage->storeAssetData(tid, LLAssetType::AT_NOTECARD,
diff --git a/indra/newview/llviewerassetstorage.cpp b/indra/newview/llviewerassetstorage.cpp
index df3ff1a3c7..5b76d57196 100644
--- a/indra/newview/llviewerassetstorage.cpp
+++ b/indra/newview/llviewerassetstorage.cpp
@@ -293,8 +293,6 @@ void LLViewerAssetStorage::storeAssetData(
LLFileSystem file(asset_id, asset_type, LLFileSystem::WRITE);
- file.setMaxSize(size);
-
const S32 buf_size = 65536;
U8 copy_buf[buf_size];
while ((size = (S32)fread(copy_buf, 1, buf_size, fp)))
@@ -528,7 +526,6 @@ void LLViewerAssetStorage::assetRequestCoro(
LLUUID temp_id;
temp_id.generate();
LLFileSystem vf(temp_id, atype, LLFileSystem::WRITE);
- vf.setMaxSize(size);
req->mBytesFetched = size;
if (!vf.write(raw.data(),size))
{
diff --git a/indra/newview/llviewerassetupload.cpp b/indra/newview/llviewerassetupload.cpp
index fb3ca69d5d..67ee06e255 100644
--- a/indra/newview/llviewerassetupload.cpp
+++ b/indra/newview/llviewerassetupload.cpp
@@ -474,8 +474,6 @@ LLSD LLNewFileResourceUploadInfo::exportTempFile()
{
LLFileSystem file(getAssetId(), assetType, LLFileSystem::WRITE);
- file.setMaxSize(file_size);
-
const S32 buf_size = 65536;
U8 copy_buf[buf_size];
while ((file_size = infile.read(copy_buf, buf_size)))
@@ -568,7 +566,6 @@ LLSD LLBufferedAssetUploadInfo::prepareUpload()
LLFileSystem file(getAssetId(), getAssetType(), LLFileSystem::APPEND);
S32 size = mContents.length() + 1;
- file.setMaxSize(size);
file.write((U8*)mContents.c_str(), size);
mStoredToCache = true;
--
cgit v1.2.3