diff options
Diffstat (limited to 'indra/llfilesystem')
| -rw-r--r-- | indra/llfilesystem/CMakeLists.txt | 3 | ||||
| -rw-r--r-- | indra/llfilesystem/lldiskcache.cpp | 773 | ||||
| -rw-r--r-- | indra/llfilesystem/lldiskcache.h | 130 | ||||
| -rw-r--r-- | indra/llfilesystem/llfilesystem.cpp | 79 | ||||
| -rw-r--r-- | indra/llfilesystem/llfilesystem.h | 19 |
5 files changed, 955 insertions, 49 deletions
diff --git a/indra/llfilesystem/CMakeLists.txt b/indra/llfilesystem/CMakeLists.txt index 306b483097..d1dece5bba 100644 --- a/indra/llfilesystem/CMakeLists.txt +++ b/indra/llfilesystem/CMakeLists.txt @@ -18,16 +18,17 @@ set(llfilesystem_SOURCE_FILES lldiriterator.cpp lllfsthread.cpp lldiskcache.cpp + llfilesystem.cpp ) set(llfilesystem_HEADER_FILES CMakeLists.txt - lldir.h lldirguard.h lldiriterator.h lllfsthread.h lldiskcache.h + llfilesystem.h ) if (DARWIN) diff --git a/indra/llfilesystem/lldiskcache.cpp b/indra/llfilesystem/lldiskcache.cpp new file mode 100644 index 0000000000..72982db069 --- /dev/null +++ b/indra/llfilesystem/lldiskcache.cpp @@ -0,0 +1,773 @@ +/** + * @file lldiskcache.cpp + * @brief The SQLite based disk cache implementation. + * @Note Eventually, this component might split into an interface + * file and multiple implemtations but for now, this is the + * only one so I think it's okay to combine everything. + * + * $LicenseInfo:firstyear=2009&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2020, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#if (defined(LL_WINDOWS) || defined(LL_LINUX) || defined(LL_DARWIN)) +#include "linden_common.h" +#endif + +#include "lldiskcache.h" + +#include <string> +#include <sstream> +#include <random> +#include <algorithm> +#include <fstream> + +/////////////////////////////////////////////////////////////////////////////// +// +llDiskCache::llDiskCache() : + mDataStorePath("") +{ +} + +llDiskCache::~llDiskCache() +{ +} + +/////////////////////////////////////////////////////////////////////////////// +// Opens the database - typically done when the application starts +// and is complementary to close() which is called when the application +// is finisahed and exits. +// Pass in the folder and filename of the SQLite database you want to +// use or create (file doesn't have to exist but the folder must) +// Returns true or false and writes a message to console on error +bool llDiskCache::open(const std::string db_folder, const std::string db_filename) +{ + mDataStorePath = db_folder; + std::string db_pathname = makeFullPath(db_filename); + + // simple flags for the moment - these will likely be extended + // later on to support the SQLite mutex model for reading/writing + // simultaneously - perhaps when we look at supporting textures too + int flags = SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE; + + if (sqlite3_open_v2( + db_pathname.c_str(), + &mDb, + flags, + nullptr // Name of VFS module to use + ) != SQLITE_OK) + { + printError(__FUNCDNAME__ , "open_v2", true); + close(); + return false; + } + + // I elected to store the code that generates the statement + // in sepsrate functions throughout - this seemed like a cleaner + // approach than having hundreds of stmt << "REPLACE AS" lines + // interspersed in the logic code. They are all prefixed with + // 'sqlCompose' and followed by a short description. + const std::string stmt = sqlComposeCreateTable(); + if (! sqliteExec(stmt, __FUNCDNAME__ )) + { + // Not sure if we need close here - if open fails, then we + // perhaps don't need to close it - TODO: look in SQLite docs + close(); + return false; + } + + return true; +} + +/////////////////////////////////////////////////////////////////////////////// +// Determines if an entry exists. +// Pass in the id and a variable that will indicate existance +// Returns true/false if function succeeded and the boolean +// you pass in will be set to true/false if entry exists or not +bool llDiskCache::exists(const std::string id, bool& exists) +{ + if (!mDb) + { + printError(__FUNCDNAME__ , "mDb is invalid", false); + return false; + } + + if (id.empty()) + { + printError(__FUNCDNAME__ , "id is empty", false); + return false; + } + + // As well as the separate function to compose the SQL statement, + // worth pointing out that the code to 'prepare' and 'step' the + // SQLite "prepared statement" has been factored out into its own + // function and used in several other functions. + const std::string stmt = sqlComposeExists(id); + sqlite3_stmt* prepared_statement = sqlitePrepareStep(__FUNCDNAME__ , stmt, SQLITE_ROW); + if (! prepared_statement) + { + return false; + } + + int result_column_index = 0; + int result_count = sqlite3_column_int(prepared_statement, result_column_index); + if (sqlite3_finalize(prepared_statement) != SQLITE_OK) + { + printError(__FUNCDNAME__ , "sqlite3_finalize()", true); + return false; + } + + // given the uniqueness of the ID, this can only ever be + // either 1 or 0 so this might be a bit confusing + exists = result_count > 0 ? true : false; + + return true; +} + +/////////////////////////////////////////////////////////////////////////////// +// Given an id (likely a UUID + decoration but can be any string), a +// pointer to a blob of binary data along with its length, write the +// entry to the cache +// Returns true/false for success/failure +bool llDiskCache::put(const std::string id, + char* binary_data, + int binary_data_size) +{ + if (!mDb) + { + printError(__FUNCDNAME__ , "mDb is invalid", false); + return false; + } + + if (id.empty()) + { + printError(__FUNCDNAME__ , "id is empty", false); + return false; + } + + // < 0 is obvious but we assert the data must be at least 1 byte long + if (binary_data_size <= 0) + { + printError(__FUNCDNAME__ , "size of binary file to write is invalid", false); + return false; + } + + // we generate a unique filename for the actual data itself + // which is stored on disk directly and not in the database. + // TODO: consider making the filename more like the ID passed in + // although the problem with that is we would have to parse the id + // to remove invalid filename chars, consider length etc. As it + // stands, we can write a simple SQL statement to return the filename + // given the ID. + const std::string filename = makeUniqueFilename(); + const std::string filepath = makeFullPath(filename); + std::ofstream file(filepath, std::ios::out | std::ios::binary); + if (! file.is_open()) + { + std::ostringstream error; + error << "Unable to open " << filepath << " for writing"; + printError(__FUNCDNAME__ , error.str(), false); + return false; + } + + file.write((char*)binary_data, binary_data_size); + file.close(); + + // I think is a catchall "wasn't able to write the file to disk" + // conditional but should revisit when we hook this up to viewer + // code to make sure - we never want to write bad/no data to the + // disk and not indicate something has gone wrong + if (file.bad()) + { + std::ostringstream error; + error << "Unable to write " << binary_data_size; + error << " bytes to " << filepath; + printError(__FUNCDNAME__ , error.str(), false); + + return false; + } + + // this is where the filename/size is written to the database along + // with the current date/time for the created/last access times + const std::string stmt = sqlComposePut(id, filename, binary_data_size); + if (! sqlitePrepareStep(__FUNCDNAME__ , stmt, SQLITE_DONE)) + { + return false; + } + + return true; +} + +/////////////////////////////////////////////////////////////////////////////// +// Given an id (likely a UUID + decoration but can be any string), the +// address of a pointer that will be used to allocate memory and a +// varialble that will contain the length of the data, get an item from +// the cache. Note that the memory buffer returned belongs to the calling +// function and it is its responsiblity to clean it up when it's no +// longer needed. +// Returns true/false for success/failure +const bool llDiskCache::get(const std::string id, + char** mem_buffer, + int& mem_buffer_size) +{ + // Check if the entry exists first to avoid dealing with a bunch + // of conditions that look like failure but aren't in the main code. + // Note exists() is a public method and also tests for mDB and id + // being valid so we can safely put this about the same tests + // in this function + bool get_exists = false; + if (! exists(id, get_exists)) + { + return false; + } + if (! get_exists) + { + return false; + } + + if (!mDb) + { + printError(__FUNCDNAME__ , "mDb is invalid", false); + return false; + } + + if (id.empty()) + { + printError(__FUNCDNAME__ , "id is empty", false); + return false; + } + + const std::string stmt_select = sqlComposeGetSelect(id); + sqlite3_stmt* prepared_statement = sqlitePrepareStep(__FUNCDNAME__ , stmt_select, SQLITE_ROW); + if (! prepared_statement) + { + return false; + } + + int result_column_index = 0; + const unsigned char* text = sqlite3_column_text(prepared_statement, result_column_index); + if (text == nullptr) + { + printError(__FUNCDNAME__ , "filename is nullptr", true); + return false; + } + const std::string filename = std::string(reinterpret_cast<const char*>(text)); + const std::string filepath = makeFullPath(filename); + + result_column_index = 1; + int filesize_db = sqlite3_column_int(prepared_statement, result_column_index); + if (filesize_db <= 0) + { + printError(__FUNCDNAME__ , "filesize is invalid", true); + return false; + } + + if (sqlite3_finalize(prepared_statement) != SQLITE_OK) + { + printError(__FUNCDNAME__ , "sqlite3_finalize()", true); + return false; + } + + // Now we have the fiename, we can read the file from disk + std::ifstream file(filepath, std::ios::in | std::ios::binary | std::ios::ate); + if (! file.is_open()) + { + std::ostringstream error; + error << "Unable to open " << filepath << " for reading"; + printError(__FUNCDNAME__ , error.str(), false); + return false; + } + + // we get the expected filesize from the database but we can also + // get it (easily) when we read the file from the disk. We compare + // the two and return false if they don't match + std::streampos filesize_file = file.tellg(); + if (filesize_db != filesize_file) + { + std::ostringstream error; + error << "File size from DB (" << filesize_db << ")"; + error << " and "; + error << "file size from file (" << filesize_file << ")"; + error << " in file " << filepath << " are different"; + printError(__FUNCDNAME__ , error.str(), false); + + return false; + } + + // doest matter if we choose DB or file version - they must be + // identical if we get this far - just used for clarity + int filesize = filesize_db; + + // allocate a block of memory that we pass back for the calling + // function to use then delete when it's no longer needed + *mem_buffer = new char[filesize]; + mem_buffer_size = filesize; + + file.seekg(0, std::ios::beg); + file.read(*mem_buffer, filesize); + file.close(); + + if (file.bad()) + { + std::ostringstream error; + error << "Unable to read " << filesize; + error << " bytes from " << filepath; + printError(__FUNCDNAME__ , error.str(), false); + + return false; + } + + // here we update the count of times the file is accessed so + // we can keep track of how many times it's been requested. + // This will be useful for metrics and perhaps determining + // if a file should not be purged even though its age + // might suggest that it should. + // In addition, this is where the time of last access is updated + // in the database and that us used to determine what is purged + // in an LRU fashion when the purge function is called. + const std::string stmt_update = sqlComposeGetUpdate(id); + if (! sqliteExec(stmt_update, __FUNCDNAME__ )) + { + return false; + } + + return true; +} + +/////////////////////////////////////////////////////////////////////////////// +// Purges the database of older entries using an LRU approach. +// Pass in the number of entries to retain. +// This is called now after open to "clean up" the cache when the +// application starts. +// TODO: IMPORTANT: compose a list of files that will be deleted +// and delete them from disk too - not just from the DB +bool llDiskCache::purge(int num_entries) +{ + if (num_entries < 0) + { + printError(__FUNCDNAME__ , "number of entries to purge is invalid", false); + return false; + } + + // find the rows affected and get the filenames for them +//swww + + // delete oldest entries leaving the correct number in place + const std::string stmt = sqlComposePurge(num_entries); + if (! sqliteExec(stmt, __FUNCDNAME__ )) + { + return false; + } + + return true; +} + +/////////////////////////////////////////////////////////////////////////////// +// Call at application shutdown +void llDiskCache::close() +{ + sqlite3_close(mDb); +} + +/////////////////////////////////////////////////////////////////////////////// +// Determine the version of SQLite in use +// TODO: make this a static so we can get to it from the Viewer About +// box without instantiating the whole thing. +const std::string llDiskCache::dbVersion() +{ + std::ostringstream version; + + version << sqlite3_libversion(); + + return version.str(); +} + +/////////////////////////////////////////////////////////////////////////////// +// Given an id, return the matching filename +const std::string llDiskCache::getFilenameById(const std::string id) +{ + // TODO: + return std::string(); +} + +/////////////////////////////////////////////////////////////////////////////// +// Given an id, return the number of times that entry has been +// accessed from the cache +const int llDiskCache::getAccessCountById(const std::string id) +{ + // TODO: + return -1; +} + +/////////////////////////////////////////////////////////////////////////////// +// Return the number of entries currently in the cache as well as +// the maximum possible entries. +void llDiskCache::getNumEntries(int& num_entries, int& max_entries) +{ + num_entries = -1; + max_entries = -1; +} + +/////////////////////////////////////////////////////////////////////////////// +// Wraps the sqlite3_exec(..) used in many places +bool llDiskCache::sqliteExec(const std::string stmt, + const std::string funcname) +{ + if (sqlite3_exec( + mDb, + stmt.c_str(), + nullptr, // Callback function (unused) + nullptr, // 1st argument to callback (unused) + nullptr // Error msg written here (unused) + ) != SQLITE_OK) + { + printError(funcname, "sqlite3_exec", true); + return false; + } + + return true; +} + +/////////////////////////////////////////////////////////////////////////////// +// Wraps the sqlite3_prepare_v2 and sqlite3_step calls used in many places +sqlite3_stmt* llDiskCache::sqlitePrepareStep(const std::string funcname, + const std::string stmt, + int sqlite_success_condition) +{ + sqlite3_stmt* prepared_statement; + + if (sqlite3_prepare_v2( + mDb, + stmt.c_str(), + -1, // Maximum length of zSql in bytes. + &prepared_statement, + 0 // OUT: Pointer to unused portion of zSql + ) != SQLITE_OK) + { + printError(funcname, "sqlite3_prepare_v2", true); + return nullptr; + } + + if (sqlite3_step(prepared_statement) != sqlite_success_condition) + { + printError(funcname, "sqlite3_step", true); + sqlite3_finalize(prepared_statement); + return nullptr; + } + return prepared_statement; +} + +/////////////////////////////////////////////////////////////////////////////// +// When an "error" occurss - e.g. database cannot be found, file cannot be +// written, invalid argument passed into a function etc. a message is +// written to stderr that should end up in the viewer log +// TODO: Set the verbosity using the usual Viewer mechanism +void llDiskCache::printError(const std::string funcname, + const std::string desc, + bool is_sqlite_err) +{ + std::ostringstream err_msg; + + err_msg << "llDiskCache error in "; + err_msg << __FUNCDNAME__ << "(...) "; + err_msg << desc; + + if (is_sqlite_err) + { + err_msg << " - "; + err_msg << std::string(sqlite3_errmsg(mDb)); + } + + // TODO: set via viewer verbosity level + const int verbosity = 1; + if (verbosity > 0) + { + std::cerr << err_msg.str() << std::endl; + } +} + +/////////////////////////////////////////////////////////////////////////////// +// wrapper for SQLite code to begin an SQL transaction - not used yet but +// it will be eventually +bool llDiskCache::beginTransaction() +{ + const std::string stmt("BEGIN TRANSACTION"); + if (! sqliteExec(stmt, __FUNCDNAME__ )) + { + return false; + } + + return true; +} + +/////////////////////////////////////////////////////////////////////////////// +// wrapper for SQLite code to end an SQL transaction - not used yet but +// it will be eventually +bool llDiskCache::endTransaction() +{ + const std::string stmt("COMMIT"); + if (! sqliteExec(stmt, __FUNCDNAME__ )) + { + return false; + } + + return true; +} + +/////////////////////////////////////////////////////////////////////////////// +// Build a unique filename that will be used to store the actual file +// on disk (as opposed to the meta data in the database) +// TODO: I think this needs more work once we move it to the viewer +// and espcially to make it cross platform - see 'std::tmpnam' +// depreciation comments in compiler output for example +std::string llDiskCache::makeUniqueFilename() +{ + // TODO: replace with boost - this is marked as deprecated? + std::string base = std::tmpnam(nullptr); + + // C++11 random number generation!!! + static std::random_device dev; + static std::mt19937 rng(dev()); + std::uniform_int_distribution<int> dist(100000, 999999); + + // currently the tmp filename from std::tmpnam() on macOS + // is of the form `/tmp/foo/bar.12345 and the following code + // strips all the preceding dirs - we likely want a more + // robust (and cross platform solution) when we move to the + // viewer code + std::size_t found = base.rfind(systemSeparator()); + if (found != std::string::npos && found < base.size() - 2) + { + base = base.substr(found + 1); + } + else + { + base = ""; + } + + // we mix in a random number for some more entropy.. + // (i know, i know...) + std::ostringstream unique_filename; + unique_filename << base; + unique_filename << "."; + unique_filename << dist(rng); + + return unique_filename.str(); +} + +/////////////////////////////////////////////////////////////////////////////// +// Return system file/path separator - likely replaced by the version +// in the viewer +const std::string llDiskCache::systemSeparator() +{ +// TODO: replace in viewer with relevant call +#ifdef _WIN32 + return "\\"; +#else + return "/"; +#endif +} + +/////////////////////////////////////////////////////////////////////////////// +// Given a filename, compose a full path based on the path name passed +// in when the database was opened and the separator in play. +const std::string llDiskCache::makeFullPath(const std::string filename) +{ + std::string pathname = mDataStorePath + systemSeparator() + filename; + + return pathname; +} + +/////////////////////////////////////////////////////////////////////////////// +// +const std::string llDiskCache::sqlComposeCreateTable() +{ + std::ostringstream stmt; + stmt << "CREATE TABLE IF NOT EXISTS "; + stmt << mTableName; + stmt << "("; + stmt << mIdFieldName; + stmt << " TEXT PRIMARY KEY, "; + stmt << mFilenameFieldName; + stmt << " TEXT, "; + stmt << mFilesizeFieldName; + stmt << " INTEGER DEFAULT 0, "; + stmt << mInsertionDateTimeFieldName; + stmt << " TEXT, "; + stmt << mLastAccessDateTimeFieldName; + stmt << " TEXT, "; + stmt << mAccessCountFieldName; + stmt << " INTEGER DEFAULT 0"; + stmt << ")"; + +#ifdef SHOW_STATEMENTS + std::cout << stmt.str() << std::endl; +#endif + + return stmt.str(); +} + +/////////////////////////////////////////////////////////////////////////////// +// +const std::string llDiskCache::sqlComposeExists(const std::string id) +{ + std::ostringstream stmt; + stmt << "SELECT COUNT(*) FROM "; + stmt << mTableName; + stmt << " WHERE "; + stmt << mIdFieldName; + stmt << "='"; + stmt << id; + stmt << "'"; + +#ifdef SHOW_STATEMENTS + std::cout << stmt.str() << std::endl; +#endif + + return stmt.str(); +} + +/////////////////////////////////////////////////////////////////////////////// +// SQL statement to write an entry to the DB +// Saves id, filename (generated), file size and create/last access date +const std::string llDiskCache::sqlComposePut(const std::string id, + const std::string filename, + int binary_data_size) +{ + std::ostringstream stmt; + stmt << "REPLACE INTO "; + stmt << mTableName; + stmt << "("; + stmt << mIdFieldName; + stmt << ","; + stmt << mFilenameFieldName; + stmt << ","; + stmt << mFilesizeFieldName; + stmt << ","; + stmt << mInsertionDateTimeFieldName; + stmt << ","; + stmt << mLastAccessDateTimeFieldName; + stmt << ") "; + stmt << "VALUES("; + stmt << "'"; + stmt << id; + stmt << "', "; + stmt << "'"; + stmt << filename; + stmt << "', "; + stmt << binary_data_size; + stmt << ", "; + stmt << "strftime('%Y-%m-%d %H:%M:%f', 'now')"; + stmt << ", "; + stmt << "strftime('%Y-%m-%d %H:%M:%f', 'now')"; + stmt << ")"; + +#ifdef SHOW_STATEMENTS + std::cout << stmt.str() << std::endl; +#endif + + return stmt.str(); +} + +/////////////////////////////////////////////////////////////////////////////// +// SQL statement used in get() to look up the filename and file size +const std::string llDiskCache::sqlComposeGetSelect(const std::string id) +{ + std::ostringstream stmt; + stmt << "SELECT "; + stmt << mFilenameFieldName; + stmt << ", "; + stmt << mFilesizeFieldName; + stmt << " FROM "; + stmt << mTableName; + stmt << " WHERE "; + stmt << mIdFieldName; + stmt << "='"; + stmt << id; + stmt << "'"; + +#ifdef SHOW_STATEMENTS + std::cout << stmt.str() << std::endl; +#endif + + return stmt.str(); +} + +/////////////////////////////////////////////////////////////////////////////// +// SQL statement to update the date/time of last access as well as the +// count of number of times the file has been accessed. +// Note: the more accurate representation of date/time is used to +// ensure ms accuracy vs the standard INTEGER days since epoch approach +const std::string llDiskCache::sqlComposeGetUpdate(const std::string id) +{ + std::ostringstream stmt; + stmt << "UPDATE "; + stmt << mTableName; + stmt << " SET "; + stmt << mAccessCountFieldName; + stmt << "="; + stmt << mAccessCountFieldName; + stmt << "+1"; + stmt << ", "; + stmt << mLastAccessDateTimeFieldName; + stmt << "="; + stmt << "strftime('%Y-%m-%d %H:%M:%f', 'now')"; + stmt << " WHERE "; + stmt << mIdFieldName; + stmt << "='"; + stmt << id; + stmt << "'"; + +#ifdef SHOW_STATEMENTS + std::cout << stmt.str() << std::endl; +#endif + + return stmt.str(); +} + +/////////////////////////////////////////////////////////////////////////////// +// SQL statement to remove items from the database that are older +// than the newest num_elements entries +const std::string llDiskCache::sqlComposePurge(int num_entries) +{ + std::ostringstream stmt; + stmt << "DELETE FROM "; + stmt << mTableName; + stmt << " WHERE "; + stmt << mLastAccessDateTimeFieldName; + stmt << " NOT IN "; + stmt << "("; + stmt << "SELECT "; + stmt << mLastAccessDateTimeFieldName; + stmt << " FROM "; + stmt << mTableName; + stmt << " ORDER BY "; + stmt << mLastAccessDateTimeFieldName; + stmt << " DESC"; + stmt << " LIMIT "; + stmt << num_entries; + stmt << ")"; + +//#ifdef SHOW_STATEMENTS + std::cout << stmt.str() << std::endl; +//#endif + + return stmt.str(); +} diff --git a/indra/llfilesystem/lldiskcache.h b/indra/llfilesystem/lldiskcache.h new file mode 100644 index 0000000000..39b8f7ef48 --- /dev/null +++ b/indra/llfilesystem/lldiskcache.h @@ -0,0 +1,130 @@ +/** + * @file lldiskcache.h + * @brief Declaration SQLite meta data / file storage API + * @brief Declaration of the generic disk cache interface + as well the SQLite header/implementation. + * @Note Eventually, this component might split into an interface + * file and multiple implemtations but for now, this is the + * only one so I think it's okay to combine everything. + * + * $LicenseInfo:firstyear=2009&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2020, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef _LLDISKCACHE +#define _LLDISKCACHE + +#include <iostream> +#include <string> + +#include "sqlite3.h" + +// Enable this line to display each SQL statement when it is executed +// It can lead to a lot of spam but useful for debugging +//#define SHOW_STATEMENTS + +// I toyed with the idea of a variety of approaches and thought have +// an abstract base class with which to hang different implementations +// off was a good idea but I think we settled on a single approach +// so I will likely remove this level of indirection if not other +// interesting implementation ideas present themselves. +class llDiskCacheBase +{ + public: + llDiskCacheBase() {}; + virtual ~llDiskCacheBase() {}; + virtual bool open(const std::string db_folder, + const std::string db_filename) = 0; + virtual bool exists(const std::string id, bool& exists) = 0; + virtual bool put(const std::string id, + char* binary_data, + int binary_data_size) = 0; + virtual const bool get(const std::string id, + char** mem_buffer, + int& mem_buffer_size) = 0; + virtual bool purge(int num_entries) = 0; + virtual void close() = 0; + virtual const std::string dbVersion() = 0; + virtual const std::string getFilenameById(const std::string id) = 0; + virtual const int getAccessCountById(const std::string id) = 0; + virtual void getNumEntries(int& num_entries, int& max_entries) = 0; +}; + +// implementation for the SQLite/disk blended case +// see source file for detailed comments +class llDiskCache : + public llDiskCacheBase +{ + public: + llDiskCache(); + virtual ~llDiskCache(); + virtual bool open(const std::string db_folder, + const std::string db_filename) override; + virtual bool exists(const std::string id, bool& exists) override; + virtual bool put(const std::string id, + char* binary_data, + int binary_data_size) override; + virtual const bool get(const std::string id, + char** mem_buffer, + int& mem_buffer_size) override; + virtual bool purge(int num_entries) override; + virtual void close() override; + virtual const std::string dbVersion() override; + virtual const std::string getFilenameById(const std::string id) override; + virtual const int getAccessCountById(const std::string id) override; + virtual void getNumEntries(int& num_entries, int& max_entries) override; + + private: + sqlite3* mDb; + std::string mDataStorePath; + const std::string mTableName = "lldiskcache"; + const std::string mIdFieldName = "id"; + const std::string mFilenameFieldName = "filename"; + const std::string mFilesizeFieldName = "filesize"; + const std::string mInsertionDateTimeFieldName = "insert_datetime"; + const std::string mLastAccessDateTimeFieldName = "last_access_datetime"; + const std::string mAccessCountFieldName = "access_count"; + + private: + void printError(const std::string funcname, + const std::string desc, + bool is_sqlite_err); + bool beginTransaction(); + bool endTransaction(); + std::string makeUniqueFilename(); + const std::string systemSeparator(); + const std::string makeFullPath(const std::string filename); + bool sqliteExec(const std::string stmt, + const std::string funcname); + sqlite3_stmt* sqlitePrepareStep(const std::string funcname, + const std::string stmt, + int sqlite_success_condition); + const std::string sqlComposeCreateTable(); + const std::string sqlComposeExists(const std::string id); + const std::string sqlComposePut(const std::string id, + const std::string filename, + int binary_data_size); + const std::string sqlComposeGetSelect(const std::string id); + const std::string sqlComposeGetUpdate(const std::string id); + const std::string sqlComposePurge(int num_entries); +}; + +#endif // _LLDISKCACHE diff --git a/indra/llfilesystem/llfilesystem.cpp b/indra/llfilesystem/llfilesystem.cpp index af93049e07..f0037c9a22 100644 --- a/indra/llfilesystem/llfilesystem.cpp +++ b/indra/llfilesystem/llfilesystem.cpp @@ -1,6 +1,9 @@ /** - * @file lldiskcache.cpp - * @brief Implementation of virtual file + * @file filesystem.h + * @brief Simulate local file system operations. + * @Note The initial implementation does actually use standard C++ + * file operations but eventually, there will be another + * layer that caches and manages file meta data too. * * $LicenseInfo:firstyear=2002&license=viewerlgpl$ * Second Life Viewer Source Code @@ -26,25 +29,21 @@ #include "linden_common.h" -#include "lldiskcache.h" - -#include "llerror.h" -#include "llthread.h" -#include "lltimer.h" +#include "lldir.h" +#include "llfilesystem.h" #include "llfasttimer.h" -#include "llmemory.h" +#include "lldiskcache.h" #include <fstream> -#include "lldir.h" -const S32 LLDiskCache::READ = 0x00000001; -const S32 LLDiskCache::WRITE = 0x00000002; -const S32 LLDiskCache::READ_WRITE = 0x00000003; // LLDiskCache::READ & LLDiskCache::WRITE -const S32 LLDiskCache::APPEND = 0x00000006; // 0x00000004 & LLDiskCache::WRITE +const S32 LLFileSystem::READ = 0x00000001; +const S32 LLFileSystem::WRITE = 0x00000002; +const S32 LLFileSystem::READ_WRITE = 0x00000003; // LLFileSystem::READ & LLFileSystem::WRITE +const S32 LLFileSystem::APPEND = 0x00000006; // 0x00000004 & LLFileSystem::WRITE static LLTrace::BlockTimerStatHandle FTM_VFILE_WAIT("VFile Wait"); -LLDiskCache::LLDiskCache(const LLUUID &file_id, const LLAssetType::EType file_type, S32 mode) +LLFileSystem::LLFileSystem(const LLUUID &file_id, const LLAssetType::EType file_type, S32 mode) { mFileType = file_type; mFileID = file_id; @@ -54,7 +53,7 @@ LLDiskCache::LLDiskCache(const LLUUID &file_id, const LLAssetType::EType file_ty mMode = mode; } -LLDiskCache::~LLDiskCache() +LLFileSystem::~LLFileSystem() { } @@ -124,7 +123,7 @@ const std::string idToFilepath(const std::string id, LLAssetType::EType at) } // static -bool LLDiskCache::getExists(const LLUUID &file_id, const LLAssetType::EType file_type) +bool LLFileSystem::getExists(const LLUUID &file_id, const LLAssetType::EType file_type) { std::string id_str; file_id.toString(id_str); @@ -140,7 +139,7 @@ bool LLDiskCache::getExists(const LLUUID &file_id, const LLAssetType::EType file } // static -bool LLDiskCache::removeFile(const LLUUID &file_id, const LLAssetType::EType file_type) +bool LLFileSystem::removeFile(const LLUUID &file_id, const LLAssetType::EType file_type) { std::string id_str; file_id.toString(id_str); @@ -152,7 +151,7 @@ bool LLDiskCache::removeFile(const LLUUID &file_id, const LLAssetType::EType fil } // static -bool LLDiskCache::renameFile(const LLUUID &old_file_id, const LLAssetType::EType old_file_type, +bool LLFileSystem::renameFile(const LLUUID &old_file_id, const LLAssetType::EType old_file_type, const LLUUID &new_file_id, const LLAssetType::EType new_file_type) { std::string old_id_str; @@ -175,7 +174,7 @@ bool LLDiskCache::renameFile(const LLUUID &old_file_id, const LLAssetType::EType } // static -S32 LLDiskCache::getFileSize(const LLUUID &file_id, const LLAssetType::EType file_type) +S32 LLFileSystem::getFileSize(const LLUUID &file_id, const LLAssetType::EType file_type) { std::string id_str; file_id.toString(id_str); @@ -192,7 +191,7 @@ S32 LLDiskCache::getFileSize(const LLUUID &file_id, const LLAssetType::EType fil return file_size; } -BOOL LLDiskCache::read(U8 *buffer, S32 bytes, BOOL async, F32 priority) +BOOL LLFileSystem::read(U8 *buffer, S32 bytes, BOOL async, F32 priority) { BOOL success = TRUE; @@ -232,7 +231,7 @@ BOOL LLDiskCache::read(U8 *buffer, S32 bytes, BOOL async, F32 priority) return success; } -BOOL LLDiskCache::isReadComplete() +BOOL LLFileSystem::isReadComplete() { if (mReadComplete) { @@ -242,17 +241,17 @@ BOOL LLDiskCache::isReadComplete() return FALSE; } -S32 LLDiskCache::getLastBytesRead() +S32 LLFileSystem::getLastBytesRead() { return mBytesRead; } -BOOL LLDiskCache::eof() +BOOL LLFileSystem::eof() { return mPosition >= getSize(); } -BOOL LLDiskCache::write(const U8 *buffer, S32 bytes) +BOOL LLFileSystem::write(const U8 *buffer, S32 bytes) { std::string id_str; mFileID.toString(id_str); @@ -287,14 +286,14 @@ BOOL LLDiskCache::write(const U8 *buffer, S32 bytes) } //static -BOOL LLDiskCache::writeFile(const U8 *buffer, S32 bytes, const LLUUID &uuid, LLAssetType::EType type) +BOOL LLFileSystem::writeFile(const U8 *buffer, S32 bytes, const LLUUID &uuid, LLAssetType::EType type) { - LLDiskCache file(uuid, type, LLDiskCache::WRITE); + LLFileSystem file(uuid, type, LLFileSystem::WRITE); file.setMaxSize(bytes); return file.write(buffer, bytes); } -BOOL LLDiskCache::seek(S32 offset, S32 origin) +BOOL LLFileSystem::seek(S32 offset, S32 origin) { if (-1 == origin) { @@ -324,32 +323,32 @@ BOOL LLDiskCache::seek(S32 offset, S32 origin) return TRUE; } -S32 LLDiskCache::tell() const +S32 LLFileSystem::tell() const { return mPosition; } -S32 LLDiskCache::getSize() +S32 LLFileSystem::getSize() { - return LLDiskCache::getFileSize(mFileID, mFileType); + return LLFileSystem::getFileSize(mFileID, mFileType); } -S32 LLDiskCache::getMaxSize() +S32 LLFileSystem::getMaxSize() { // offer up a huge size since we don't care what the max is return INT_MAX; } -BOOL LLDiskCache::setMaxSize(S32 size) +BOOL LLFileSystem::setMaxSize(S32 size) { // we don't care what the max size is so we do nothing // and return true to indicate all was okay return TRUE; } -BOOL LLDiskCache::rename(const LLUUID &new_id, const LLAssetType::EType new_type) +BOOL LLFileSystem::rename(const LLUUID &new_id, const LLAssetType::EType new_type) { - LLDiskCache::renameFile(mFileID, mFileType, new_id, new_type); + LLFileSystem::renameFile(mFileID, mFileType, new_id, new_type); mFileID = new_id; mFileType = new_type; @@ -357,31 +356,31 @@ BOOL LLDiskCache::rename(const LLUUID &new_id, const LLAssetType::EType new_type return TRUE; } -BOOL LLDiskCache::remove() +BOOL LLFileSystem::remove() { - LLDiskCache::removeFile(mFileID, mFileType); + LLFileSystem::removeFile(mFileID, mFileType); return TRUE; } // static -void LLDiskCache::initClass() +void LLFileSystem::initClass() { } // static -void LLDiskCache::cleanupClass() +void LLFileSystem::cleanupClass() { } -bool LLDiskCache::isLocked() +bool LLFileSystem::isLocked() { // I don't think we care about this test since there is no locking // TODO: remove this function and calling sites? return FALSE; } -void LLDiskCache::waitForLock() +void LLFileSystem::waitForLock() { // TODO: remove this function and calling sites? } diff --git a/indra/llfilesystem/llfilesystem.h b/indra/llfilesystem/llfilesystem.h index 7ad06a8689..9cddef74a7 100644 --- a/indra/llfilesystem/llfilesystem.h +++ b/indra/llfilesystem/llfilesystem.h @@ -1,6 +1,9 @@ /** - * @file lldiskcacke.h - * @brief Definition of virtual file + * @file filesystem.h + * @brief Simulate local file system operations. + * @Note The initial implementation does actually use standard C++ + * file operations but eventually, there will be another + * layer that caches and manages file meta data too. * * $LicenseInfo:firstyear=2002&license=viewerlgpl$ * Second Life Viewer Source Code @@ -24,17 +27,17 @@ * $/LicenseInfo$ */ -#ifndef LL_LLDISKCACHE_H -#define LL_LLDISKCACHE_H +#ifndef LL_FILESYSTEM_H +#define LL_FILESYSTEM_H #include "lluuid.h" #include "llassettype.h" -class LLDiskCache +class LLFileSystem { public: - LLDiskCache(const LLUUID &file_id, const LLAssetType::EType file_type, S32 mode = LLDiskCache::READ); - ~LLDiskCache(); + LLFileSystem(const LLUUID &file_id, const LLAssetType::EType file_type, S32 mode = LLFileSystem::READ); + ~LLFileSystem(); BOOL read(U8 *buffer, S32 bytes, BOOL async = FALSE, F32 priority = 128.f); /* Flawfinder: ignore */ BOOL isReadComplete(); @@ -79,4 +82,4 @@ protected: S32 mBytesRead; }; -#endif // LL_LLDISKCACHE_H +#endif // LL_FILESYSTEM_H |