Complete the change from lldiskcache -> llfilesystem and then addition of new lldiskcache implementation

5 files changed, 955 insertions, 49 deletions

diff --git a/indra/llfilesystem/CMakeLists.txt b/indra/llfilesystem/CMakeLists.txt
index 306b483097..d1dece5bba 100644
--- a/indra/llfilesystem/CMakeLists.txt
+++ b/indra/llfilesystem/CMakeLists.txt
@@ -18,16 +18,17 @@ set(llfilesystem_SOURCE_FILES
     lldiriterator.cpp
     lllfsthread.cpp
     lldiskcache.cpp
+    llfilesystem.cpp
     )
 
 set(llfilesystem_HEADER_FILES
     CMakeLists.txt
-
     lldir.h
     lldirguard.h
     lldiriterator.h
     lllfsthread.h
     lldiskcache.h
+    llfilesystem.h
     )
 
 if (DARWIN)
diff --git a/indra/llfilesystem/lldiskcache.cpp b/indra/llfilesystem/lldiskcache.cpp
new file mode 100644
index 0000000000..72982db069
--- /dev/null
+++ b/indra/llfilesystem/lldiskcache.cpp
@@ -0,0 +1,773 @@
+/**
+ * @file lldiskcache.cpp
+ * @brief The SQLite based disk cache implementation.
+ * @Note  Eventually, this component might split into an interface
+ *        file and multiple implemtations but for now, this is the
+ *        only one so I think it's okay to combine everything. 
+ *
+ * $LicenseInfo:firstyear=2009&license=viewerlgpl$
+ * Second Life Viewer Source Code
+ * Copyright (C) 2020, Linden Research, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation;
+ * version 2.1 of the License only.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ * Linden Research, Inc., 945 Battery Street, San Francisco, CA  94111  USA
+ * $/LicenseInfo$
+ */
+
+#if (defined(LL_WINDOWS) || defined(LL_LINUX)  || defined(LL_DARWIN))
+#include "linden_common.h"
+#endif
+
+#include "lldiskcache.h"
+
+#include <string>
+#include <sstream>
+#include <random>
+#include <algorithm>
+#include <fstream>
+
+///////////////////////////////////////////////////////////////////////////////
+//
+llDiskCache::llDiskCache() :
+    mDataStorePath("")
+{
+}
+
+llDiskCache::~llDiskCache()
+{
+}
+
+///////////////////////////////////////////////////////////////////////////////
+// Opens the database - typically done when the application starts
+// and is complementary to close() which is called when the application
+// is finisahed and exits.
+// Pass in the folder and filename of the SQLite database you want to
+// use or create (file doesn't have to exist but the folder must)
+// Returns true or false and writes a message to console on error
+bool llDiskCache::open(const std::string db_folder, const std::string db_filename)
+{
+    mDataStorePath = db_folder;
+    std::string db_pathname = makeFullPath(db_filename);
+
+    // simple flags for the moment - these will likely be extended
+    // later on to support the SQLite mutex model for reading/writing
+    // simultaneously - perhaps when we look at supporting textures too
+    int flags = SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE;
+
+    if (sqlite3_open_v2(
+                db_pathname.c_str(),
+                &mDb,
+                flags,
+                nullptr // Name of VFS module to use
+            ) != SQLITE_OK)
+    {
+        printError(__FUNCDNAME__ , "open_v2", true);
+        close();
+        return false;
+    }
+
+    // I elected to store the code that generates the statement
+    // in sepsrate functions throughout - this seemed like a cleaner
+    // approach than having hundreds of stmt << "REPLACE AS" lines
+    // interspersed in the logic code. They are all prefixed with
+    // 'sqlCompose' and followed by a short description.
+    const std::string stmt = sqlComposeCreateTable();
+    if (! sqliteExec(stmt, __FUNCDNAME__ ))
+    {
+        // Not sure if we need close here - if open fails, then we
+        // perhaps don't need to close it - TODO: look in SQLite docs
+        close();
+        return false;
+    }
+
+    return true;
+}
+
+///////////////////////////////////////////////////////////////////////////////
+// Determines if an entry exists.
+// Pass in the id and a variable that will indicate existance
+// Returns true/false if function succeeded and the boolean
+// you pass in will be set to true/false if entry exists or not
+bool llDiskCache::exists(const std::string id, bool& exists)
+{
+    if (!mDb)
+    {
+        printError(__FUNCDNAME__ , "mDb is invalid", false);
+        return false;
+    }
+
+    if (id.empty())
+    {
+        printError(__FUNCDNAME__ , "id is empty", false);
+        return false;
+    }
+
+    // As well as the separate function to compose the SQL statement,
+    // worth pointing out that the code to 'prepare' and 'step' the
+    // SQLite "prepared statement" has been factored out into its own
+    // function and used in several other functions.
+    const std::string stmt = sqlComposeExists(id);
+    sqlite3_stmt* prepared_statement = sqlitePrepareStep(__FUNCDNAME__ , stmt, SQLITE_ROW);
+    if (! prepared_statement)
+    {
+        return false;
+    }
+
+    int result_column_index = 0;
+    int result_count = sqlite3_column_int(prepared_statement, result_column_index);
+    if (sqlite3_finalize(prepared_statement) != SQLITE_OK)
+    {
+        printError(__FUNCDNAME__ , "sqlite3_finalize()", true);
+        return false;
+    }
+
+    // given the uniqueness of the ID, this can only ever be
+    // either 1 or 0 so this might be a bit confusing
+    exists = result_count > 0 ? true : false;
+
+    return true;
+}
+
+///////////////////////////////////////////////////////////////////////////////
+// Given an id (likely a UUID + decoration but can be any string), a
+// pointer to a blob of binary data along with its length, write the
+// entry to the cache
+// Returns true/false for success/failure
+bool llDiskCache::put(const std::string id,
+                      char* binary_data,
+                      int binary_data_size)
+{
+    if (!mDb)
+    {
+        printError(__FUNCDNAME__ , "mDb is invalid", false);
+        return false;
+    }
+
+    if (id.empty())
+    {
+        printError(__FUNCDNAME__ , "id is empty", false);
+        return false;
+    }
+
+    // < 0 is obvious but we assert the data must be at least 1 byte long
+    if (binary_data_size <= 0)
+    {
+        printError(__FUNCDNAME__ , "size of binary file to write is invalid", false);
+        return false;
+    }
+
+    // we generate a unique filename for the actual data itself
+    // which is stored on disk directly and not in the database.
+    // TODO: consider making the filename more like the ID passed in
+    // although the problem with that is we would have to parse the id
+    // to remove invalid filename chars, consider length etc.  As it
+    // stands, we can write a simple SQL statement to return the filename
+    // given the ID.
+    const std::string filename = makeUniqueFilename();
+    const std::string filepath = makeFullPath(filename);
+    std::ofstream file(filepath, std::ios::out | std::ios::binary);
+    if (! file.is_open())
+    {
+        std::ostringstream error;
+        error << "Unable to open " << filepath << " for writing";
+        printError(__FUNCDNAME__ , error.str(), false);
+        return false;
+    }
+
+    file.write((char*)binary_data, binary_data_size);
+    file.close();
+
+    // I think is a catchall "wasn't able to write the file to disk"
+    // conditional but should revisit when we hook this up to viewer
+    // code to make sure - we never want to write bad/no data to the
+    // disk and not indicate something has gone wrong
+    if (file.bad())
+    {
+        std::ostringstream error;
+        error << "Unable to write " << binary_data_size;
+        error << " bytes to " << filepath;
+        printError(__FUNCDNAME__ , error.str(), false);
+
+        return false;
+    }
+
+    // this is where the filename/size is written to the database along
+    // with the current date/time for the created/last access times
+    const std::string stmt = sqlComposePut(id, filename, binary_data_size);
+    if (! sqlitePrepareStep(__FUNCDNAME__ , stmt, SQLITE_DONE))
+    {
+        return false;
+    }
+
+    return true;
+}
+
+///////////////////////////////////////////////////////////////////////////////
+// Given an id (likely a UUID + decoration but can be any string), the
+// address of a pointer that will be used to allocate memory and a
+// varialble that will contain the length of the data, get an item from
+// the cache. Note that the memory buffer returned belongs to the calling
+// function and it is its responsiblity to clean it up when it's no
+// longer needed.
+// Returns true/false for success/failure
+const bool llDiskCache::get(const std::string id,
+                            char** mem_buffer,
+                            int& mem_buffer_size)
+{
+    // Check if the entry exists first to avoid dealing with a bunch
+    // of conditions that look like failure but aren't in the main code.
+    // Note exists() is a public method and also tests for mDB and id
+    // being valid so we can safely put this about the same tests
+    // in this function
+    bool get_exists = false;
+    if (! exists(id, get_exists))
+    {
+        return false;
+    }
+    if (! get_exists)
+    {
+        return false;
+    }
+
+    if (!mDb)
+    {
+        printError(__FUNCDNAME__ , "mDb is invalid", false);
+        return false;
+    }
+
+    if (id.empty())
+    {
+        printError(__FUNCDNAME__ , "id is empty", false);
+        return false;
+    }
+
+    const std::string stmt_select = sqlComposeGetSelect(id);
+    sqlite3_stmt* prepared_statement = sqlitePrepareStep(__FUNCDNAME__ , stmt_select, SQLITE_ROW);
+    if (! prepared_statement)
+    {
+        return false;
+    }
+
+    int result_column_index = 0;
+    const unsigned char* text = sqlite3_column_text(prepared_statement, result_column_index);
+    if (text == nullptr)
+    {
+        printError(__FUNCDNAME__ , "filename is nullptr", true);
+        return false;
+    }
+    const std::string filename = std::string(reinterpret_cast<const char*>(text));
+    const std::string filepath = makeFullPath(filename);
+
+    result_column_index = 1;
+    int filesize_db = sqlite3_column_int(prepared_statement, result_column_index);
+    if (filesize_db <= 0)
+    {
+        printError(__FUNCDNAME__ , "filesize is invalid", true);
+        return false;
+    }
+
+    if (sqlite3_finalize(prepared_statement) != SQLITE_OK)
+    {
+        printError(__FUNCDNAME__ , "sqlite3_finalize()", true);
+        return false;
+    }
+
+    // Now we have the fiename, we can read the file from disk
+    std::ifstream file(filepath, std::ios::in | std::ios::binary | std::ios::ate);
+    if (! file.is_open())
+    {
+        std::ostringstream error;
+        error << "Unable to open " << filepath << " for reading";
+        printError(__FUNCDNAME__ , error.str(), false);
+        return false;
+    }
+
+    // we get the expected filesize from the database but we can also
+    // get it (easily) when we read the file from the disk. We compare
+    // the two and return false if they don't match
+    std::streampos filesize_file = file.tellg();
+    if (filesize_db != filesize_file)
+    {
+        std::ostringstream error;
+        error << "File size from DB (" << filesize_db << ")";
+        error << " and ";
+        error << "file size from file (" << filesize_file << ")";
+        error << " in file " << filepath << " are different";
+        printError(__FUNCDNAME__ , error.str(), false);
+
+        return false;
+    }
+
+    // doest matter if we choose DB or file version - they must be
+    // identical if we get this far - just used for clarity
+    int filesize = filesize_db;
+
+    // allocate a block of memory that we pass back for the calling
+    // function to use then delete when it's no longer needed
+    *mem_buffer = new char[filesize];
+    mem_buffer_size = filesize;
+
+    file.seekg(0, std::ios::beg);
+    file.read(*mem_buffer, filesize);
+    file.close();
+
+    if (file.bad())
+    {
+        std::ostringstream error;
+        error << "Unable to read " << filesize;
+        error << " bytes from " << filepath;
+        printError(__FUNCDNAME__ , error.str(), false);
+
+        return false;
+    }
+
+    // here we update the count of times the file is accessed so
+    // we can keep track of how many times it's been requested.
+    // This will be useful for metrics and perhaps determining
+    // if a file should not be purged even though its age
+    // might suggest that it should.
+    // In addition, this is where the time of last access is updated
+    // in the database and that us used to determine what is purged
+    // in an LRU fashion when the purge function is called.
+    const std::string stmt_update = sqlComposeGetUpdate(id);
+    if (! sqliteExec(stmt_update, __FUNCDNAME__ ))
+    {
+        return false;
+    }
+
+    return true;
+}
+
+///////////////////////////////////////////////////////////////////////////////
+// Purges the database of older entries using an LRU approach.
+// Pass in the number of entries to retain.
+// This is called now after open to "clean up" the cache when the
+// application starts.
+// TODO: IMPORTANT: compose a list of files that will be deleted
+// and delete them from disk too - not just from the DB
+bool llDiskCache::purge(int num_entries)
+{
+    if (num_entries < 0)
+    {
+        printError(__FUNCDNAME__ , "number of entries to purge is invalid", false);
+        return false;
+    }
+
+    // find the rows affected and get the filenames for them
+//swww
+
+    // delete oldest entries leaving the correct number in place
+    const std::string stmt = sqlComposePurge(num_entries);
+    if (! sqliteExec(stmt, __FUNCDNAME__ ))
+    {
+        return false;
+    }
+
+    return true;
+}
+
+///////////////////////////////////////////////////////////////////////////////
+// Call at application shutdown
+void llDiskCache::close()
+{
+    sqlite3_close(mDb);
+}
+
+///////////////////////////////////////////////////////////////////////////////
+// Determine the version of SQLite in use
+// TODO: make this a static so we can get to it from the Viewer About
+// box without instantiating the whole thing.
+const std::string llDiskCache::dbVersion()
+{
+    std::ostringstream version;
+
+    version << sqlite3_libversion();
+
+    return version.str();
+}
+
+///////////////////////////////////////////////////////////////////////////////
+// Given an id, return the matching filename
+const std::string llDiskCache::getFilenameById(const std::string id)
+{
+    // TODO:
+    return std::string();
+}
+
+///////////////////////////////////////////////////////////////////////////////
+// Given an id, return the number of times that entry has been
+// accessed from the cache
+const int llDiskCache::getAccessCountById(const std::string id)
+{
+    // TODO:
+    return -1;
+}
+
+///////////////////////////////////////////////////////////////////////////////
+// Return the number of entries currently in the cache as well as
+// the maximum possible entries.
+void llDiskCache::getNumEntries(int& num_entries, int& max_entries)
+{
+    num_entries = -1;
+    max_entries = -1;
+}
+
+///////////////////////////////////////////////////////////////////////////////
+// Wraps the sqlite3_exec(..) used in many places
+bool llDiskCache::sqliteExec(const std::string stmt,
+                             const std::string funcname)
+{
+    if (sqlite3_exec(
+                mDb,
+                stmt.c_str(),
+                nullptr,    // Callback function (unused)
+                nullptr,    // 1st argument to callback (unused)
+                nullptr     // Error msg written here (unused)
+            ) != SQLITE_OK)
+    {
+        printError(funcname, "sqlite3_exec", true);
+        return false;
+    }
+
+    return true;
+}
+
+///////////////////////////////////////////////////////////////////////////////
+// Wraps the sqlite3_prepare_v2 and sqlite3_step calls used in many places
+sqlite3_stmt* llDiskCache::sqlitePrepareStep(const std::string funcname,
+        const std::string stmt,
+        int sqlite_success_condition)
+{
+    sqlite3_stmt* prepared_statement;
+
+    if (sqlite3_prepare_v2(
+                mDb,
+                stmt.c_str(),
+                -1,                     // Maximum length of zSql in bytes.
+                &prepared_statement,
+                0                       // OUT: Pointer to unused portion of zSql
+            ) != SQLITE_OK)
+    {
+        printError(funcname, "sqlite3_prepare_v2", true);
+        return nullptr;
+    }
+
+    if (sqlite3_step(prepared_statement) != sqlite_success_condition)
+    {
+        printError(funcname, "sqlite3_step", true);
+        sqlite3_finalize(prepared_statement);
+        return nullptr;
+    }
+    return prepared_statement;
+}
+
+///////////////////////////////////////////////////////////////////////////////
+// When an "error" occurss - e.g. database cannot be found, file cannot be
+// written, invalid argument passed into a function etc. a message is
+// written to stderr that should end up in the viewer log
+// TODO: Set the verbosity using the usual Viewer mechanism
+void llDiskCache::printError(const std::string funcname,
+                             const std::string desc,
+                             bool is_sqlite_err)
+{
+    std::ostringstream err_msg;
+
+    err_msg << "llDiskCache error in ";
+    err_msg << __FUNCDNAME__  << "(...) ";
+    err_msg << desc;
+
+    if (is_sqlite_err)
+    {
+        err_msg << " - ";
+        err_msg << std::string(sqlite3_errmsg(mDb));
+    }
+
+    // TODO: set via viewer verbosity level
+    const int verbosity = 1;
+    if (verbosity > 0)
+    {
+        std::cerr << err_msg.str() << std::endl;
+    }
+}
+
+///////////////////////////////////////////////////////////////////////////////
+// wrapper for SQLite code to begin an SQL transaction - not used yet but
+// it will be eventually
+bool llDiskCache::beginTransaction()
+{
+    const std::string stmt("BEGIN TRANSACTION");
+    if (! sqliteExec(stmt, __FUNCDNAME__ ))
+    {
+        return false;
+    }
+
+    return true;
+}
+
+///////////////////////////////////////////////////////////////////////////////
+// wrapper for SQLite code to end an SQL transaction - not used yet but
+// it will be eventually
+bool llDiskCache::endTransaction()
+{
+    const std::string stmt("COMMIT");
+    if (! sqliteExec(stmt, __FUNCDNAME__ ))
+    {
+        return false;
+    }
+
+    return true;
+}
+
+///////////////////////////////////////////////////////////////////////////////
+// Build a unique filename that will be used to store the actual file
+// on disk (as opposed to the meta data in the database)
+// TODO: I think this needs more work once we move it to the viewer
+// and espcially to make it cross platform - see 'std::tmpnam'
+// depreciation comments in compiler output for example
+std::string llDiskCache::makeUniqueFilename()
+{
+    // TODO: replace with boost - this is marked as deprecated?
+    std::string base = std::tmpnam(nullptr);
+
+    // C++11 random number generation!!!
+    static std::random_device dev;
+    static std::mt19937 rng(dev());
+    std::uniform_int_distribution<int> dist(100000, 999999);
+
+    // currently the tmp filename from std::tmpnam() on macOS
+    // is of the form `/tmp/foo/bar.12345 and the following code
+    // strips all the preceding dirs - we likely want a more
+    // robust (and cross platform solution) when we move to the
+    // viewer code
+    std::size_t found = base.rfind(systemSeparator());
+    if (found != std::string::npos && found < base.size() - 2)
+    {
+        base = base.substr(found + 1);
+    }
+    else
+    {
+        base = "";
+    }
+
+    // we mix in a random number for some more entropy..
+    // (i know, i know...)
+    std::ostringstream unique_filename;
+    unique_filename << base;
+    unique_filename << ".";
+    unique_filename <<      dist(rng);
+
+    return unique_filename.str();
+}
+
+///////////////////////////////////////////////////////////////////////////////
+// Return system file/path separator - likely replaced by the version
+// in the viewer
+const std::string llDiskCache::systemSeparator()
+{
+// TODO: replace in viewer with relevant call
+#ifdef _WIN32
+    return "\\";
+#else
+    return "/";
+#endif
+}
+
+///////////////////////////////////////////////////////////////////////////////
+// Given a filename, compose a full path based on the path name passed
+// in when the database was opened and the separator in play.
+const std::string llDiskCache::makeFullPath(const std::string filename)
+{
+    std::string pathname = mDataStorePath + systemSeparator() + filename;
+
+    return pathname;
+}
+
+///////////////////////////////////////////////////////////////////////////////
+//
+const std::string llDiskCache::sqlComposeCreateTable()
+{
+    std::ostringstream stmt;
+    stmt << "CREATE TABLE IF NOT EXISTS ";
+    stmt << mTableName;
+    stmt << "(";
+    stmt << mIdFieldName;
+    stmt << " TEXT PRIMARY KEY, ";
+    stmt << mFilenameFieldName;
+    stmt << " TEXT, ";
+    stmt << mFilesizeFieldName;
+    stmt << " INTEGER DEFAULT 0, ";
+    stmt << mInsertionDateTimeFieldName;
+    stmt << " TEXT, ";
+    stmt << mLastAccessDateTimeFieldName;
+    stmt << " TEXT, ";
+    stmt << mAccessCountFieldName;
+    stmt << " INTEGER DEFAULT 0";
+    stmt << ")";
+
+#ifdef SHOW_STATEMENTS
+    std::cout << stmt.str() << std::endl;
+#endif
+
+    return stmt.str();
+}
+
+///////////////////////////////////////////////////////////////////////////////
+//
+const std::string llDiskCache::sqlComposeExists(const std::string id)
+{
+    std::ostringstream stmt;
+    stmt << "SELECT COUNT(*) FROM ";
+    stmt << mTableName;
+    stmt << " WHERE ";
+    stmt << mIdFieldName;
+    stmt << "='";
+    stmt << id;
+    stmt << "'";
+
+#ifdef SHOW_STATEMENTS
+    std::cout << stmt.str() << std::endl;
+#endif
+
+    return stmt.str();
+}
+
+///////////////////////////////////////////////////////////////////////////////
+// SQL statement to write an entry to the DB
+// Saves id, filename (generated), file size and create/last access date
+const std::string llDiskCache::sqlComposePut(const std::string id,
+        const std::string filename,
+        int binary_data_size)
+{
+    std::ostringstream stmt;
+    stmt << "REPLACE INTO ";
+    stmt << mTableName;
+    stmt << "(";
+    stmt << mIdFieldName;
+    stmt << ",";
+    stmt << mFilenameFieldName;
+    stmt << ",";
+    stmt << mFilesizeFieldName;
+    stmt << ",";
+    stmt << mInsertionDateTimeFieldName;
+    stmt << ",";
+    stmt << mLastAccessDateTimeFieldName;
+    stmt << ") ";
+    stmt << "VALUES(";
+    stmt << "'";
+    stmt << id;
+    stmt << "', ";
+    stmt << "'";
+    stmt << filename;
+    stmt << "', ";
+    stmt << binary_data_size;
+    stmt << ", ";
+    stmt << "strftime('%Y-%m-%d %H:%M:%f', 'now')";
+    stmt << ", ";
+    stmt << "strftime('%Y-%m-%d %H:%M:%f', 'now')";
+    stmt << ")";
+
+#ifdef SHOW_STATEMENTS
+    std::cout << stmt.str() << std::endl;
+#endif
+
+    return stmt.str();
+}
+
+///////////////////////////////////////////////////////////////////////////////
+// SQL statement used in get() to look up the filename and file size
+const std::string llDiskCache::sqlComposeGetSelect(const std::string id)
+{
+    std::ostringstream stmt;
+    stmt << "SELECT ";
+    stmt << mFilenameFieldName;
+    stmt << ", ";
+    stmt << mFilesizeFieldName;
+    stmt << " FROM ";
+    stmt << mTableName;
+    stmt << " WHERE ";
+    stmt << mIdFieldName;
+    stmt << "='";
+    stmt << id;
+    stmt << "'";
+
+#ifdef SHOW_STATEMENTS
+    std::cout << stmt.str() << std::endl;
+#endif
+
+    return stmt.str();
+}
+
+///////////////////////////////////////////////////////////////////////////////
+// SQL statement to update the date/time of last access as well as the
+// count of number of times the file has been accessed.
+// Note: the more accurate representation of date/time is used to
+// ensure ms accuracy vs the standard INTEGER days since epoch approach
+const std::string llDiskCache::sqlComposeGetUpdate(const std::string id)
+{
+    std::ostringstream stmt;
+    stmt << "UPDATE ";
+    stmt << mTableName;
+    stmt << " SET ";
+    stmt << mAccessCountFieldName;
+    stmt << "=";
+    stmt << mAccessCountFieldName;
+    stmt << "+1";
+    stmt << ", ";
+    stmt << mLastAccessDateTimeFieldName;
+    stmt << "=";
+    stmt << "strftime('%Y-%m-%d %H:%M:%f', 'now')";
+    stmt << " WHERE ";
+    stmt << mIdFieldName;
+    stmt << "='";
+    stmt << id;
+    stmt << "'";
+
+#ifdef SHOW_STATEMENTS
+    std::cout << stmt.str() << std::endl;
+#endif
+
+    return stmt.str();
+}
+
+///////////////////////////////////////////////////////////////////////////////
+// SQL statement to remove items from the database that are older
+// than the newest num_elements entries
+const std::string llDiskCache::sqlComposePurge(int num_entries)
+{
+    std::ostringstream stmt;
+    stmt << "DELETE FROM ";
+    stmt << mTableName;
+    stmt << " WHERE ";
+    stmt << mLastAccessDateTimeFieldName;
+    stmt << " NOT IN ";
+    stmt << "(";
+    stmt << "SELECT ";
+    stmt << mLastAccessDateTimeFieldName;
+    stmt << " FROM ";
+    stmt << mTableName;
+    stmt << " ORDER BY ";
+    stmt << mLastAccessDateTimeFieldName;
+    stmt << " DESC";
+    stmt << " LIMIT ";
+    stmt << num_entries;
+    stmt << ")";
+
+//#ifdef SHOW_STATEMENTS
+    std::cout << stmt.str() << std::endl;
+//#endif
+
+    return stmt.str();
+}
diff --git a/indra/llfilesystem/lldiskcache.h b/indra/llfilesystem/lldiskcache.h
new file mode 100644
index 0000000000..39b8f7ef48
--- /dev/null
+++ b/indra/llfilesystem/lldiskcache.h
@@ -0,0 +1,130 @@
+/**
+ * @file lldiskcache.h
+ * @brief Declaration SQLite meta data / file storage API
+ * @brief Declaration of the generic disk cache interface
+          as well the SQLite header/implementation.
+ * @Note  Eventually, this component might split into an interface
+ *        file and multiple implemtations but for now, this is the
+ *        only one so I think it's okay to combine everything. 
+ *
+ * $LicenseInfo:firstyear=2009&license=viewerlgpl$
+ * Second Life Viewer Source Code
+ * Copyright (C) 2020, Linden Research, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation;
+ * version 2.1 of the License only.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ * Linden Research, Inc., 945 Battery Street, San Francisco, CA  94111  USA
+ * $/LicenseInfo$
+ */
+
+#ifndef _LLDISKCACHE
+#define _LLDISKCACHE
+
+#include <iostream>
+#include <string>
+
+#include "sqlite3.h"
+
+// Enable this line to display each SQL statement when it is executed
+// It can lead to a lot of spam but useful for debugging
+//#define SHOW_STATEMENTS
+
+// I toyed with the idea of a variety of approaches and thought have
+// an abstract base class with which to hang different implementations
+// off was a good idea but I think we settled on a single approach
+// so I will likely remove this level of indirection if not other 
+// interesting implementation ideas present themselves.
+class llDiskCacheBase
+{
+    public:
+        llDiskCacheBase() {};
+        virtual ~llDiskCacheBase() {};
+        virtual bool open(const std::string db_folder,
+                          const std::string db_filename) = 0;
+        virtual bool exists(const std::string id, bool& exists) = 0;
+        virtual bool put(const std::string id,
+                         char* binary_data,
+                         int binary_data_size) = 0;
+        virtual const bool get(const std::string id,
+                               char** mem_buffer,
+                               int& mem_buffer_size) = 0;
+        virtual bool purge(int num_entries) = 0;
+        virtual void close() = 0;
+        virtual const std::string dbVersion() = 0;
+        virtual const std::string getFilenameById(const std::string id) = 0;
+        virtual const int getAccessCountById(const std::string id) = 0;
+        virtual void getNumEntries(int& num_entries, int& max_entries) = 0;
+};
+
+// implementation for the SQLite/disk blended case
+// see source file for detailed comments
+class llDiskCache :
+    public llDiskCacheBase
+{
+    public:
+        llDiskCache();
+        virtual ~llDiskCache();
+        virtual bool open(const std::string db_folder,
+                          const std::string db_filename) override;
+        virtual bool exists(const std::string id, bool& exists) override;
+        virtual bool put(const std::string id,
+                         char* binary_data,
+                         int binary_data_size) override;
+        virtual const bool get(const std::string id,
+                               char** mem_buffer,
+                               int& mem_buffer_size) override;
+        virtual bool purge(int num_entries) override;
+        virtual void close() override;
+        virtual const std::string dbVersion() override;
+        virtual const std::string getFilenameById(const std::string id) override;
+        virtual const int getAccessCountById(const std::string id) override;
+        virtual void getNumEntries(int& num_entries, int& max_entries) override;
+
+    private:
+        sqlite3* mDb;
+        std::string mDataStorePath;
+        const std::string mTableName = "lldiskcache";
+        const std::string mIdFieldName = "id";
+        const std::string mFilenameFieldName = "filename";
+        const std::string mFilesizeFieldName = "filesize";
+        const std::string mInsertionDateTimeFieldName = "insert_datetime";
+        const std::string mLastAccessDateTimeFieldName = "last_access_datetime";
+        const std::string mAccessCountFieldName = "access_count";
+
+    private:
+        void printError(const std::string funcname,
+                        const std::string desc,
+                        bool is_sqlite_err);
+        bool beginTransaction();
+        bool endTransaction();
+        std::string makeUniqueFilename();
+        const std::string systemSeparator();
+        const std::string makeFullPath(const std::string filename);
+        bool sqliteExec(const std::string stmt,
+                        const std::string funcname);
+        sqlite3_stmt* sqlitePrepareStep(const std::string funcname,
+                                        const std::string stmt,
+                                        int sqlite_success_condition);
+        const std::string sqlComposeCreateTable();
+        const std::string sqlComposeExists(const std::string id);
+        const std::string sqlComposePut(const std::string id,
+                                        const std::string filename,
+                                        int binary_data_size);
+        const std::string sqlComposeGetSelect(const std::string id);
+        const std::string sqlComposeGetUpdate(const std::string id);
+        const std::string sqlComposePurge(int num_entries);
+};
+
+#endif // _LLDISKCACHE
diff --git a/indra/llfilesystem/llfilesystem.cpp b/indra/llfilesystem/llfilesystem.cpp
index af93049e07..f0037c9a22 100644
--- a/indra/llfilesystem/llfilesystem.cpp
+++ b/indra/llfilesystem/llfilesystem.cpp
@@ -1,6 +1,9 @@
 /** 
- * @file lldiskcache.cpp
- * @brief Implementation of virtual file
+ * @file filesystem.h
+ * @brief Simulate local file system operations.
+ * @Note The initial implementation does actually use standard C++
+ *       file operations but eventually, there will be another
+ *       layer that caches and manages file meta data too.
  *
  * $LicenseInfo:firstyear=2002&license=viewerlgpl$
  * Second Life Viewer Source Code
@@ -26,25 +29,21 @@
 
 #include "linden_common.h"
 
-#include "lldiskcache.h"
-
-#include "llerror.h"
-#include "llthread.h"
-#include "lltimer.h"
+#include "lldir.h"
+#include "llfilesystem.h"
 #include "llfasttimer.h"
-#include "llmemory.h"
+#include "lldiskcache.h"
 
 #include <fstream>
-#include "lldir.h"
 
-const S32 LLDiskCache::READ			= 0x00000001;
-const S32 LLDiskCache::WRITE		= 0x00000002;
-const S32 LLDiskCache::READ_WRITE	= 0x00000003;  // LLDiskCache::READ & LLDiskCache::WRITE
-const S32 LLDiskCache::APPEND		= 0x00000006;  // 0x00000004 & LLDiskCache::WRITE
+const S32 LLFileSystem::READ		= 0x00000001;
+const S32 LLFileSystem::WRITE		= 0x00000002;
+const S32 LLFileSystem::READ_WRITE	= 0x00000003;  // LLFileSystem::READ & LLFileSystem::WRITE
+const S32 LLFileSystem::APPEND		= 0x00000006;  // 0x00000004 & LLFileSystem::WRITE
 
 static LLTrace::BlockTimerStatHandle FTM_VFILE_WAIT("VFile Wait");
 
-LLDiskCache::LLDiskCache(const LLUUID &file_id, const LLAssetType::EType file_type, S32 mode)
+LLFileSystem::LLFileSystem(const LLUUID &file_id, const LLAssetType::EType file_type, S32 mode)
 {
 	mFileType =	file_type;
 	mFileID = file_id;
@@ -54,7 +53,7 @@ LLDiskCache::LLDiskCache(const LLUUID &file_id, const LLAssetType::EType file_ty
 	mMode = mode;
 }
 
-LLDiskCache::~LLDiskCache()
+LLFileSystem::~LLFileSystem()
 {
 }
 
@@ -124,7 +123,7 @@ const std::string idToFilepath(const std::string id, LLAssetType::EType at)
 }
 
 // static
-bool LLDiskCache::getExists(const LLUUID &file_id, const LLAssetType::EType file_type)
+bool LLFileSystem::getExists(const LLUUID &file_id, const LLAssetType::EType file_type)
 {
 	std::string id_str;
 	file_id.toString(id_str);
@@ -140,7 +139,7 @@ bool LLDiskCache::getExists(const LLUUID &file_id, const LLAssetType::EType file
 }
 
 // static
-bool LLDiskCache::removeFile(const LLUUID &file_id, const LLAssetType::EType file_type)
+bool LLFileSystem::removeFile(const LLUUID &file_id, const LLAssetType::EType file_type)
 {
     std::string id_str;
     file_id.toString(id_str);
@@ -152,7 +151,7 @@ bool LLDiskCache::removeFile(const LLUUID &file_id, const LLAssetType::EType fil
 }
 
 // static
-bool LLDiskCache::renameFile(const LLUUID &old_file_id, const LLAssetType::EType old_file_type,
+bool LLFileSystem::renameFile(const LLUUID &old_file_id, const LLAssetType::EType old_file_type,
                          const LLUUID &new_file_id, const LLAssetType::EType new_file_type)
 {
     std::string old_id_str;
@@ -175,7 +174,7 @@ bool LLDiskCache::renameFile(const LLUUID &old_file_id, const LLAssetType::EType
 }
 
 // static
-S32 LLDiskCache::getFileSize(const LLUUID &file_id, const LLAssetType::EType file_type)
+S32 LLFileSystem::getFileSize(const LLUUID &file_id, const LLAssetType::EType file_type)
 {
     std::string id_str;
     file_id.toString(id_str);
@@ -192,7 +191,7 @@ S32 LLDiskCache::getFileSize(const LLUUID &file_id, const LLAssetType::EType fil
     return file_size;
 }
 
-BOOL LLDiskCache::read(U8 *buffer, S32 bytes, BOOL async, F32 priority)
+BOOL LLFileSystem::read(U8 *buffer, S32 bytes, BOOL async, F32 priority)
 {
 	BOOL success = TRUE;
 
@@ -232,7 +231,7 @@ BOOL LLDiskCache::read(U8 *buffer, S32 bytes, BOOL async, F32 priority)
     return success;
 }
 
-BOOL LLDiskCache::isReadComplete()
+BOOL LLFileSystem::isReadComplete()
 {
     if (mReadComplete)
     {
@@ -242,17 +241,17 @@ BOOL LLDiskCache::isReadComplete()
     return FALSE;
 }
 
-S32 LLDiskCache::getLastBytesRead()
+S32 LLFileSystem::getLastBytesRead()
 {
 	return mBytesRead;
 }
 
-BOOL LLDiskCache::eof()
+BOOL LLFileSystem::eof()
 {
 	return mPosition >= getSize();
 }
 
-BOOL LLDiskCache::write(const U8 *buffer, S32 bytes)
+BOOL LLFileSystem::write(const U8 *buffer, S32 bytes)
 {
     std::string id_str;
     mFileID.toString(id_str);
@@ -287,14 +286,14 @@ BOOL LLDiskCache::write(const U8 *buffer, S32 bytes)
 }
 
 //static
-BOOL LLDiskCache::writeFile(const U8 *buffer, S32 bytes, const LLUUID &uuid, LLAssetType::EType type)
+BOOL LLFileSystem::writeFile(const U8 *buffer, S32 bytes, const LLUUID &uuid, LLAssetType::EType type)
 {
-	LLDiskCache file(uuid, type, LLDiskCache::WRITE);
+	LLFileSystem file(uuid, type, LLFileSystem::WRITE);
 	file.setMaxSize(bytes);
 	return file.write(buffer, bytes);
 }
 
-BOOL LLDiskCache::seek(S32 offset, S32 origin)
+BOOL LLFileSystem::seek(S32 offset, S32 origin)
 {
 	if (-1 == origin)
 	{
@@ -324,32 +323,32 @@ BOOL LLDiskCache::seek(S32 offset, S32 origin)
 	return TRUE;
 }
 
-S32 LLDiskCache::tell() const
+S32 LLFileSystem::tell() const
 {
 	return mPosition;
 }
 
-S32 LLDiskCache::getSize()
+S32 LLFileSystem::getSize()
 {
-    return LLDiskCache::getFileSize(mFileID, mFileType);
+    return LLFileSystem::getFileSize(mFileID, mFileType);
 }
 
-S32 LLDiskCache::getMaxSize()
+S32 LLFileSystem::getMaxSize()
 {
     // offer up a huge size since we don't care what the max is 
     return INT_MAX;
 }
 
-BOOL LLDiskCache::setMaxSize(S32 size)
+BOOL LLFileSystem::setMaxSize(S32 size)
 {
     // we don't care what the max size is so we do nothing
     // and return true to indicate all was okay
     return TRUE;
 }
 
-BOOL LLDiskCache::rename(const LLUUID &new_id, const LLAssetType::EType new_type)
+BOOL LLFileSystem::rename(const LLUUID &new_id, const LLAssetType::EType new_type)
 {
-    LLDiskCache::renameFile(mFileID, mFileType, new_id, new_type);
+    LLFileSystem::renameFile(mFileID, mFileType, new_id, new_type);
 
     mFileID = new_id;
     mFileType = new_type;
@@ -357,31 +356,31 @@ BOOL LLDiskCache::rename(const LLUUID &new_id, const LLAssetType::EType new_type
     return TRUE;
 }
 
-BOOL LLDiskCache::remove()
+BOOL LLFileSystem::remove()
 {
-    LLDiskCache::removeFile(mFileID, mFileType);
+    LLFileSystem::removeFile(mFileID, mFileType);
 
     return TRUE;
 }
 
 // static
-void LLDiskCache::initClass()
+void LLFileSystem::initClass()
 {
 }
 
 // static
-void LLDiskCache::cleanupClass()
+void LLFileSystem::cleanupClass()
 {
 }
 
-bool LLDiskCache::isLocked()
+bool LLFileSystem::isLocked()
 {
     // I don't think we care about this test since there is no locking
     // TODO: remove this function and calling sites?
     return FALSE;
 }
 
-void LLDiskCache::waitForLock()
+void LLFileSystem::waitForLock()
 {
     // TODO: remove this function and calling sites?
 }
diff --git a/indra/llfilesystem/llfilesystem.h b/indra/llfilesystem/llfilesystem.h
index 7ad06a8689..9cddef74a7 100644
--- a/indra/llfilesystem/llfilesystem.h
+++ b/indra/llfilesystem/llfilesystem.h
@@ -1,6 +1,9 @@
 /** 
- * @file lldiskcacke.h
- * @brief Definition of virtual file
+ * @file filesystem.h
+ * @brief Simulate local file system operations.
+ * @Note The initial implementation does actually use standard C++
+ *       file operations but eventually, there will be another
+ *       layer that caches and manages file meta data too.
  *
  * $LicenseInfo:firstyear=2002&license=viewerlgpl$
  * Second Life Viewer Source Code
@@ -24,17 +27,17 @@
  * $/LicenseInfo$
  */
 
-#ifndef LL_LLDISKCACHE_H
-#define LL_LLDISKCACHE_H
+#ifndef LL_FILESYSTEM_H
+#define LL_FILESYSTEM_H
 
 #include "lluuid.h"
 #include "llassettype.h"
 
-class LLDiskCache
+class LLFileSystem
 {
 public:
-	LLDiskCache(const LLUUID &file_id, const LLAssetType::EType file_type, S32 mode = LLDiskCache::READ);
-	~LLDiskCache();
+	LLFileSystem(const LLUUID &file_id, const LLAssetType::EType file_type, S32 mode = LLFileSystem::READ);
+	~LLFileSystem();
 
 	BOOL read(U8 *buffer, S32 bytes, BOOL async = FALSE, F32 priority = 128.f);	/* Flawfinder: ignore */ 
 	BOOL isReadComplete();
@@ -79,4 +82,4 @@ protected:
 	S32		mBytesRead;
 };
 
-#endif  // LL_LLDISKCACHE_H
+#endif  // LL_FILESYSTEM_H