Add in the C++ filesystem based cache and clean up some indempotent functions in llfilesystem

13 files changed, 196 insertions, 825 deletions

diff --git a/indra/llfilesystem/lldiskcache.cpp b/indra/llfilesystem/lldiskcache.cpp
index 72982db069..4b2ba0dd79 100644
--- a/indra/llfilesystem/lldiskcache.cpp
+++ b/indra/llfilesystem/lldiskcache.cpp
@@ -1,9 +1,6 @@
 /**
  * @file lldiskcache.cpp
- * @brief The SQLite based disk cache implementation.
- * @Note  Eventually, this component might split into an interface
- *        file and multiple implemtations but for now, this is the
- *        only one so I think it's okay to combine everything. 
+ * @brief The disk cache implementation.
  *
  * $LicenseInfo:firstyear=2009&license=viewerlgpl$
  * Second Life Viewer Source Code
@@ -27,747 +24,167 @@
  * $/LicenseInfo$
  */
 
-#if (defined(LL_WINDOWS) || defined(LL_LINUX)  || defined(LL_DARWIN))
 #include "linden_common.h"
-#endif
+#include "lluuid.h"
+#include "lldir.h"
 
 #include "lldiskcache.h"
 
-#include <string>
-#include <sstream>
-#include <random>
-#include <algorithm>
-#include <fstream>
-
-///////////////////////////////////////////////////////////////////////////////
-//
-llDiskCache::llDiskCache() :
-    mDataStorePath("")
-{
-}
+#include <boost/filesystem.hpp>
+#include <boost/range/iterator_range.hpp>
 
-llDiskCache::~llDiskCache()
-{
-}
+#include <chrono>
+#include <thread>
+#include <algorithm>
 
-///////////////////////////////////////////////////////////////////////////////
-// Opens the database - typically done when the application starts
-// and is complementary to close() which is called when the application
-// is finisahed and exits.
-// Pass in the folder and filename of the SQLite database you want to
-// use or create (file doesn't have to exist but the folder must)
-// Returns true or false and writes a message to console on error
-bool llDiskCache::open(const std::string db_folder, const std::string db_filename)
+LLDiskCache::LLDiskCache(const std::string cache_dir) :
+    mCacheDir(cache_dir),
+    mMaxSizeBytes(mDefaultSizeBytes)
 {
-    mDataStorePath = db_folder;
-    std::string db_pathname = makeFullPath(db_filename);
-
-    // simple flags for the moment - these will likely be extended
-    // later on to support the SQLite mutex model for reading/writing
-    // simultaneously - perhaps when we look at supporting textures too
-    int flags = SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE;
-
-    if (sqlite3_open_v2(
-                db_pathname.c_str(),
-                &mDb,
-                flags,
-                nullptr // Name of VFS module to use
-            ) != SQLITE_OK)
-    {
-        printError(__FUNCDNAME__ , "open_v2", true);
-        close();
-        return false;
-    }
-
-    // I elected to store the code that generates the statement
-    // in sepsrate functions throughout - this seemed like a cleaner
-    // approach than having hundreds of stmt << "REPLACE AS" lines
-    // interspersed in the logic code. They are all prefixed with
-    // 'sqlCompose' and followed by a short description.
-    const std::string stmt = sqlComposeCreateTable();
-    if (! sqliteExec(stmt, __FUNCDNAME__ ))
+    // no access to LLControlGroup / gSavedSettings so use the system environment
+    // to trigger additional debugging on top of the default "we purged the cache"
+    mCacheDebugInfo = true;
+    if (getenv("LL_CACHE_DEBUGGING") != nullptr)
     {
-        // Not sure if we need close here - if open fails, then we
-        // perhaps don't need to close it - TODO: look in SQLite docs
-        close();
-        return false;
+        mCacheDebugInfo = true;
     }
 
-    return true;
+    // create cache dir if it does not exist
+    boost::filesystem::create_directory(cache_dir);
 }
 
-///////////////////////////////////////////////////////////////////////////////
-// Determines if an entry exists.
-// Pass in the id and a variable that will indicate existance
-// Returns true/false if function succeeded and the boolean
-// you pass in will be set to true/false if entry exists or not
-bool llDiskCache::exists(const std::string id, bool& exists)
+LLDiskCache::~LLDiskCache()
 {
-    if (!mDb)
-    {
-        printError(__FUNCDNAME__ , "mDb is invalid", false);
-        return false;
-    }
-
-    if (id.empty())
-    {
-        printError(__FUNCDNAME__ , "id is empty", false);
-        return false;
-    }
-
-    // As well as the separate function to compose the SQL statement,
-    // worth pointing out that the code to 'prepare' and 'step' the
-    // SQLite "prepared statement" has been factored out into its own
-    // function and used in several other functions.
-    const std::string stmt = sqlComposeExists(id);
-    sqlite3_stmt* prepared_statement = sqlitePrepareStep(__FUNCDNAME__ , stmt, SQLITE_ROW);
-    if (! prepared_statement)
-    {
-        return false;
-    }
-
-    int result_column_index = 0;
-    int result_count = sqlite3_column_int(prepared_statement, result_column_index);
-    if (sqlite3_finalize(prepared_statement) != SQLITE_OK)
-    {
-        printError(__FUNCDNAME__ , "sqlite3_finalize()", true);
-        return false;
-    }
-
-    // given the uniqueness of the ID, this can only ever be
-    // either 1 or 0 so this might be a bit confusing
-    exists = result_count > 0 ? true : false;
-
-    return true;
-}
-
-///////////////////////////////////////////////////////////////////////////////
-// Given an id (likely a UUID + decoration but can be any string), a
-// pointer to a blob of binary data along with its length, write the
-// entry to the cache
-// Returns true/false for success/failure
-bool llDiskCache::put(const std::string id,
-                      char* binary_data,
-                      int binary_data_size)
-{
-    if (!mDb)
-    {
-        printError(__FUNCDNAME__ , "mDb is invalid", false);
-        return false;
-    }
-
-    if (id.empty())
-    {
-        printError(__FUNCDNAME__ , "id is empty", false);
-        return false;
-    }
-
-    // < 0 is obvious but we assert the data must be at least 1 byte long
-    if (binary_data_size <= 0)
-    {
-        printError(__FUNCDNAME__ , "size of binary file to write is invalid", false);
-        return false;
-    }
-
-    // we generate a unique filename for the actual data itself
-    // which is stored on disk directly and not in the database.
-    // TODO: consider making the filename more like the ID passed in
-    // although the problem with that is we would have to parse the id
-    // to remove invalid filename chars, consider length etc.  As it
-    // stands, we can write a simple SQL statement to return the filename
-    // given the ID.
-    const std::string filename = makeUniqueFilename();
-    const std::string filepath = makeFullPath(filename);
-    std::ofstream file(filepath, std::ios::out | std::ios::binary);
-    if (! file.is_open())
-    {
-        std::ostringstream error;
-        error << "Unable to open " << filepath << " for writing";
-        printError(__FUNCDNAME__ , error.str(), false);
-        return false;
-    }
-
-    file.write((char*)binary_data, binary_data_size);
-    file.close();
-
-    // I think is a catchall "wasn't able to write the file to disk"
-    // conditional but should revisit when we hook this up to viewer
-    // code to make sure - we never want to write bad/no data to the
-    // disk and not indicate something has gone wrong
-    if (file.bad())
-    {
-        std::ostringstream error;
-        error << "Unable to write " << binary_data_size;
-        error << " bytes to " << filepath;
-        printError(__FUNCDNAME__ , error.str(), false);
-
-        return false;
-    }
-
-    // this is where the filename/size is written to the database along
-    // with the current date/time for the created/last access times
-    const std::string stmt = sqlComposePut(id, filename, binary_data_size);
-    if (! sqlitePrepareStep(__FUNCDNAME__ , stmt, SQLITE_DONE))
-    {
-        return false;
-    }
-
-    return true;
 }
 
-///////////////////////////////////////////////////////////////////////////////
-// Given an id (likely a UUID + decoration but can be any string), the
-// address of a pointer that will be used to allocate memory and a
-// varialble that will contain the length of the data, get an item from
-// the cache. Note that the memory buffer returned belongs to the calling
-// function and it is its responsiblity to clean it up when it's no
-// longer needed.
-// Returns true/false for success/failure
-const bool llDiskCache::get(const std::string id,
-                            char** mem_buffer,
-                            int& mem_buffer_size)
+void LLDiskCache::purge()
 {
-    // Check if the entry exists first to avoid dealing with a bunch
-    // of conditions that look like failure but aren't in the main code.
-    // Note exists() is a public method and also tests for mDB and id
-    // being valid so we can safely put this about the same tests
-    // in this function
-    bool get_exists = false;
-    if (! exists(id, get_exists))
+    if (mCacheDebugInfo)
     {
-        return false;
-    }
-    if (! get_exists)
-    {
-        return false;
+        LL_INFOS() << "Total dir size before purge is " << dirFileSize(mCacheDir) << LL_ENDL;
     }
 
-    if (!mDb)
-    {
-        printError(__FUNCDNAME__ , "mDb is invalid", false);
-        return false;
-    }
-
-    if (id.empty())
-    {
-        printError(__FUNCDNAME__ , "id is empty", false);
-        return false;
-    }
+    auto start_time = std::chrono::high_resolution_clock::now();
 
-    const std::string stmt_select = sqlComposeGetSelect(id);
-    sqlite3_stmt* prepared_statement = sqlitePrepareStep(__FUNCDNAME__ , stmt_select, SQLITE_ROW);
-    if (! prepared_statement)
-    {
-        return false;
-    }
+    typedef std::pair<std::time_t, std::pair<uintmax_t, std::string>> file_info_t;
+    std::vector<file_info_t> file_info;
 
-    int result_column_index = 0;
-    const unsigned char* text = sqlite3_column_text(prepared_statement, result_column_index);
-    if (text == nullptr)
+    if (boost::filesystem::is_directory(mCacheDir))
     {
-        printError(__FUNCDNAME__ , "filename is nullptr", true);
-        return false;
-    }
-    const std::string filename = std::string(reinterpret_cast<const char*>(text));
-    const std::string filepath = makeFullPath(filename);
+        for (auto& entry : boost::make_iterator_range(boost::filesystem::directory_iterator(mCacheDir), {}))
+        {
+            if (boost::filesystem::is_regular_file(entry))
+            {
+                uintmax_t file_size = boost::filesystem::file_size(entry);
+                const std::string file_path = entry.path().string();
+                const std::time_t file_time = boost::filesystem::last_write_time(entry);
 
-    result_column_index = 1;
-    int filesize_db = sqlite3_column_int(prepared_statement, result_column_index);
-    if (filesize_db <= 0)
-    {
-        printError(__FUNCDNAME__ , "filesize is invalid", true);
-        return false;
+                file_info.push_back(file_info_t(file_time, { file_size, file_path }));
+            }
+        }
     }
 
-    if (sqlite3_finalize(prepared_statement) != SQLITE_OK)
+    std::sort(file_info.begin(), file_info.end(), [](file_info_t& x, file_info_t& y)
     {
-        printError(__FUNCDNAME__ , "sqlite3_finalize()", true);
-        return false;
-    }
+        return x.first > y.first;
+    });
 
-    // Now we have the fiename, we can read the file from disk
-    std::ifstream file(filepath, std::ios::in | std::ios::binary | std::ios::ate);
-    if (! file.is_open())
-    {
-        std::ostringstream error;
-        error << "Unable to open " << filepath << " for reading";
-        printError(__FUNCDNAME__ , error.str(), false);
-        return false;
-    }
+    LL_INFOS() << "Purging cache to a maximum of " << mMaxSizeBytes << " bytes" << LL_ENDL;
 
-    // we get the expected filesize from the database but we can also
-    // get it (easily) when we read the file from the disk. We compare
-    // the two and return false if they don't match
-    std::streampos filesize_file = file.tellg();
-    if (filesize_db != filesize_file)
+    uintmax_t file_size_total = 0;
+    for (file_info_t& entry : file_info)
     {
-        std::ostringstream error;
-        error << "File size from DB (" << filesize_db << ")";
-        error << " and ";
-        error << "file size from file (" << filesize_file << ")";
-        error << " in file " << filepath << " are different";
-        printError(__FUNCDNAME__ , error.str(), false);
-
-        return false;
-    }
-
-    // doest matter if we choose DB or file version - they must be
-    // identical if we get this far - just used for clarity
-    int filesize = filesize_db;
+        file_size_total += entry.second.first;
 
-    // allocate a block of memory that we pass back for the calling
-    // function to use then delete when it's no longer needed
-    *mem_buffer = new char[filesize];
-    mem_buffer_size = filesize;
+        std::string action = "";
+        if (file_size_total > mMaxSizeBytes)
+        {
+            action = "DELETE:";
+            boost::filesystem::remove(entry.second.second);
+        }
+        else
+        {
+            action = "  KEEP:";
+        }
 
-    file.seekg(0, std::ios::beg);
-    file.read(*mem_buffer, filesize);
-    file.close();
+        if (mCacheDebugInfo)
+        {
+            // have to do this because of LL_INFO/LL_END weirdness
+            std::ostringstream line;
 
-    if (file.bad())
-    {
-        std::ostringstream error;
-        error << "Unable to read " << filesize;
-        error << " bytes from " << filepath;
-        printError(__FUNCDNAME__ , error.str(), false);
-
-        return false;
+            line << action << "  ";
+            line << entry.first << "  ";
+            line << entry.second.first << "  ";
+            line << entry.second.second;
+            line << " (" << file_size_total << "/" << mMaxSizeBytes << ")";
+            LL_INFOS() << line.str() << LL_ENDL;
+        }
     }
 
-    // here we update the count of times the file is accessed so
-    // we can keep track of how many times it's been requested.
-    // This will be useful for metrics and perhaps determining
-    // if a file should not be purged even though its age
-    // might suggest that it should.
-    // In addition, this is where the time of last access is updated
-    // in the database and that us used to determine what is purged
-    // in an LRU fashion when the purge function is called.
-    const std::string stmt_update = sqlComposeGetUpdate(id);
-    if (! sqliteExec(stmt_update, __FUNCDNAME__ ))
+    if (mCacheDebugInfo)
     {
-        return false;
+        auto end_time = std::chrono::high_resolution_clock::now();
+        auto execute_time = std::chrono::duration_cast<std::chrono::milliseconds>(end_time - start_time).count();
+        LL_INFOS() << "Total dir size after purge is " << dirFileSize(mCacheDir) << LL_ENDL;
+        LL_INFOS() << "Cache purge took " << execute_time << " ms to execute for " << file_info.size() << " files" << LL_ENDL;
     }
-
-    return true;
-}
-
-///////////////////////////////////////////////////////////////////////////////
-// Purges the database of older entries using an LRU approach.
-// Pass in the number of entries to retain.
-// This is called now after open to "clean up" the cache when the
-// application starts.
-// TODO: IMPORTANT: compose a list of files that will be deleted
-// and delete them from disk too - not just from the DB
-bool llDiskCache::purge(int num_entries)
-{
-    if (num_entries < 0)
-    {
-        printError(__FUNCDNAME__ , "number of entries to purge is invalid", false);
-        return false;
-    }
-
-    // find the rows affected and get the filenames for them
-//swww
-
-    // delete oldest entries leaving the correct number in place
-    const std::string stmt = sqlComposePurge(num_entries);
-    if (! sqliteExec(stmt, __FUNCDNAME__ ))
-    {
-        return false;
-    }
-
-    return true;
-}
-
-///////////////////////////////////////////////////////////////////////////////
-// Call at application shutdown
-void llDiskCache::close()
-{
-    sqlite3_close(mDb);
-}
-
-///////////////////////////////////////////////////////////////////////////////
-// Determine the version of SQLite in use
-// TODO: make this a static so we can get to it from the Viewer About
-// box without instantiating the whole thing.
-const std::string llDiskCache::dbVersion()
-{
-    std::ostringstream version;
-
-    version << sqlite3_libversion();
-
-    return version.str();
 }
 
-///////////////////////////////////////////////////////////////////////////////
-// Given an id, return the matching filename
-const std::string llDiskCache::getFilenameById(const std::string id)
-{
-    // TODO:
-    return std::string();
-}
-
-///////////////////////////////////////////////////////////////////////////////
-// Given an id, return the number of times that entry has been
-// accessed from the cache
-const int llDiskCache::getAccessCountById(const std::string id)
-{
-    // TODO:
-    return -1;
-}
-
-///////////////////////////////////////////////////////////////////////////////
-// Return the number of entries currently in the cache as well as
-// the maximum possible entries.
-void llDiskCache::getNumEntries(int& num_entries, int& max_entries)
-{
-    num_entries = -1;
-    max_entries = -1;
-}
-
-///////////////////////////////////////////////////////////////////////////////
-// Wraps the sqlite3_exec(..) used in many places
-bool llDiskCache::sqliteExec(const std::string stmt,
-                             const std::string funcname)
-{
-    if (sqlite3_exec(
-                mDb,
-                stmt.c_str(),
-                nullptr,    // Callback function (unused)
-                nullptr,    // 1st argument to callback (unused)
-                nullptr     // Error msg written here (unused)
-            ) != SQLITE_OK)
-    {
-        printError(funcname, "sqlite3_exec", true);
-        return false;
-    }
-
-    return true;
-}
-
-///////////////////////////////////////////////////////////////////////////////
-// Wraps the sqlite3_prepare_v2 and sqlite3_step calls used in many places
-sqlite3_stmt* llDiskCache::sqlitePrepareStep(const std::string funcname,
-        const std::string stmt,
-        int sqlite_success_condition)
-{
-    sqlite3_stmt* prepared_statement;
-
-    if (sqlite3_prepare_v2(
-                mDb,
-                stmt.c_str(),
-                -1,                     // Maximum length of zSql in bytes.
-                &prepared_statement,
-                0                       // OUT: Pointer to unused portion of zSql
-            ) != SQLITE_OK)
-    {
-        printError(funcname, "sqlite3_prepare_v2", true);
-        return nullptr;
-    }
-
-    if (sqlite3_step(prepared_statement) != sqlite_success_condition)
-    {
-        printError(funcname, "sqlite3_step", true);
-        sqlite3_finalize(prepared_statement);
-        return nullptr;
-    }
-    return prepared_statement;
-}
-
-///////////////////////////////////////////////////////////////////////////////
-// When an "error" occurss - e.g. database cannot be found, file cannot be
-// written, invalid argument passed into a function etc. a message is
-// written to stderr that should end up in the viewer log
-// TODO: Set the verbosity using the usual Viewer mechanism
-void llDiskCache::printError(const std::string funcname,
-                             const std::string desc,
-                             bool is_sqlite_err)
-{
-    std::ostringstream err_msg;
-
-    err_msg << "llDiskCache error in ";
-    err_msg << __FUNCDNAME__  << "(...) ";
-    err_msg << desc;
-
-    if (is_sqlite_err)
-    {
-        err_msg << " - ";
-        err_msg << std::string(sqlite3_errmsg(mDb));
-    }
-
-    // TODO: set via viewer verbosity level
-    const int verbosity = 1;
-    if (verbosity > 0)
-    {
-        std::cerr << err_msg.str() << std::endl;
-    }
-}
-
-///////////////////////////////////////////////////////////////////////////////
-// wrapper for SQLite code to begin an SQL transaction - not used yet but
-// it will be eventually
-bool llDiskCache::beginTransaction()
+/**
+ * Update the "last write time" of a file to "now". This must be called whenever a 
+ * file in the cache is read (not written) so that the last time the file was 
+ * accessed which is used in the mechanism for purging the cache, is up to date.
+ */
+void LLDiskCache::updateFileAccessTime(const std::string file_path)
 {
-    const std::string stmt("BEGIN TRANSACTION");
-    if (! sqliteExec(stmt, __FUNCDNAME__ ))
-    {
-        return false;
-    }
-
-    return true;
+    const std::time_t file_time = std::time(nullptr);
+    boost::filesystem::last_write_time(file_path, file_time);
 }
 
-///////////////////////////////////////////////////////////////////////////////
-// wrapper for SQLite code to end an SQL transaction - not used yet but
-// it will be eventually
-bool llDiskCache::endTransaction()
+/**
+ * Clear the cache by removing all the files in the cache directory
+ * individually. It's important to maintain control of which directory
+ * if passed in and not let the user inadvertently (or maliciously) set 
+ * it to an random location like your project source or OS system directory 
+ */
+void LLDiskCache::clearCache(const std::string cache_dir)
 {
-    const std::string stmt("COMMIT");
-    if (! sqliteExec(stmt, __FUNCDNAME__ ))
+    if (boost::filesystem::is_directory(cache_dir))
     {
-        return false;
+        for (auto& entry : boost::make_iterator_range(boost::filesystem::directory_iterator(cache_dir), {}))
+        {
+            if (boost::filesystem::is_regular_file(entry))
+            {
+                boost::filesystem::remove(entry);
+            }
+        }
     }
-
-    return true;
 }
 
-///////////////////////////////////////////////////////////////////////////////
-// Build a unique filename that will be used to store the actual file
-// on disk (as opposed to the meta data in the database)
-// TODO: I think this needs more work once we move it to the viewer
-// and espcially to make it cross platform - see 'std::tmpnam'
-// depreciation comments in compiler output for example
-std::string llDiskCache::makeUniqueFilename()
+/**
+ * Utility function to get the total filesize of all files in a directory. It
+ * used to test file extensions to only check cache files but that was removed.
+ * There may be a better way that works directly on the folder (similar to
+ * right clicking on a folder in the OS and asking for size vs right clicking
+ * on all files and adding up manually) but this is very fast - less than 100ms
+ * in my testing so, so long as it's not called frequently, it should be okay.
+ * Note that's it's only currently used for logging/debugging so if performance
+ * is ever an issue, optimizing this or removing it altogether, is an easy win.
+ */
+uintmax_t LLDiskCache::dirFileSize(const std::string dir)
 {
-    // TODO: replace with boost - this is marked as deprecated?
-    std::string base = std::tmpnam(nullptr);
-
-    // C++11 random number generation!!!
-    static std::random_device dev;
-    static std::mt19937 rng(dev());
-    std::uniform_int_distribution<int> dist(100000, 999999);
+    uintmax_t total_file_size = 0;
 
-    // currently the tmp filename from std::tmpnam() on macOS
-    // is of the form `/tmp/foo/bar.12345 and the following code
-    // strips all the preceding dirs - we likely want a more
-    // robust (and cross platform solution) when we move to the
-    // viewer code
-    std::size_t found = base.rfind(systemSeparator());
-    if (found != std::string::npos && found < base.size() - 2)
-    {
-        base = base.substr(found + 1);
-    }
-    else
+    if (boost::filesystem::is_directory(dir))
     {
-        base = "";
+        for (auto& entry : boost::make_iterator_range(boost::filesystem::directory_iterator(dir), {}))
+        {
+            if (boost::filesystem::is_regular_file(entry))
+            {
+                total_file_size += boost::filesystem::file_size(entry);
+            }
+        }
     }
 
-    // we mix in a random number for some more entropy..
-    // (i know, i know...)
-    std::ostringstream unique_filename;
-    unique_filename << base;
-    unique_filename << ".";
-    unique_filename <<      dist(rng);
-
-    return unique_filename.str();
-}
-
-///////////////////////////////////////////////////////////////////////////////
-// Return system file/path separator - likely replaced by the version
-// in the viewer
-const std::string llDiskCache::systemSeparator()
-{
-// TODO: replace in viewer with relevant call
-#ifdef _WIN32
-    return "\\";
-#else
-    return "/";
-#endif
-}
-
-///////////////////////////////////////////////////////////////////////////////
-// Given a filename, compose a full path based on the path name passed
-// in when the database was opened and the separator in play.
-const std::string llDiskCache::makeFullPath(const std::string filename)
-{
-    std::string pathname = mDataStorePath + systemSeparator() + filename;
-
-    return pathname;
-}
-
-///////////////////////////////////////////////////////////////////////////////
-//
-const std::string llDiskCache::sqlComposeCreateTable()
-{
-    std::ostringstream stmt;
-    stmt << "CREATE TABLE IF NOT EXISTS ";
-    stmt << mTableName;
-    stmt << "(";
-    stmt << mIdFieldName;
-    stmt << " TEXT PRIMARY KEY, ";
-    stmt << mFilenameFieldName;
-    stmt << " TEXT, ";
-    stmt << mFilesizeFieldName;
-    stmt << " INTEGER DEFAULT 0, ";
-    stmt << mInsertionDateTimeFieldName;
-    stmt << " TEXT, ";
-    stmt << mLastAccessDateTimeFieldName;
-    stmt << " TEXT, ";
-    stmt << mAccessCountFieldName;
-    stmt << " INTEGER DEFAULT 0";
-    stmt << ")";
-
-#ifdef SHOW_STATEMENTS
-    std::cout << stmt.str() << std::endl;
-#endif
-
-    return stmt.str();
-}
-
-///////////////////////////////////////////////////////////////////////////////
-//
-const std::string llDiskCache::sqlComposeExists(const std::string id)
-{
-    std::ostringstream stmt;
-    stmt << "SELECT COUNT(*) FROM ";
-    stmt << mTableName;
-    stmt << " WHERE ";
-    stmt << mIdFieldName;
-    stmt << "='";
-    stmt << id;
-    stmt << "'";
-
-#ifdef SHOW_STATEMENTS
-    std::cout << stmt.str() << std::endl;
-#endif
-
-    return stmt.str();
-}
-
-///////////////////////////////////////////////////////////////////////////////
-// SQL statement to write an entry to the DB
-// Saves id, filename (generated), file size and create/last access date
-const std::string llDiskCache::sqlComposePut(const std::string id,
-        const std::string filename,
-        int binary_data_size)
-{
-    std::ostringstream stmt;
-    stmt << "REPLACE INTO ";
-    stmt << mTableName;
-    stmt << "(";
-    stmt << mIdFieldName;
-    stmt << ",";
-    stmt << mFilenameFieldName;
-    stmt << ",";
-    stmt << mFilesizeFieldName;
-    stmt << ",";
-    stmt << mInsertionDateTimeFieldName;
-    stmt << ",";
-    stmt << mLastAccessDateTimeFieldName;
-    stmt << ") ";
-    stmt << "VALUES(";
-    stmt << "'";
-    stmt << id;
-    stmt << "', ";
-    stmt << "'";
-    stmt << filename;
-    stmt << "', ";
-    stmt << binary_data_size;
-    stmt << ", ";
-    stmt << "strftime('%Y-%m-%d %H:%M:%f', 'now')";
-    stmt << ", ";
-    stmt << "strftime('%Y-%m-%d %H:%M:%f', 'now')";
-    stmt << ")";
-
-#ifdef SHOW_STATEMENTS
-    std::cout << stmt.str() << std::endl;
-#endif
-
-    return stmt.str();
-}
-
-///////////////////////////////////////////////////////////////////////////////
-// SQL statement used in get() to look up the filename and file size
-const std::string llDiskCache::sqlComposeGetSelect(const std::string id)
-{
-    std::ostringstream stmt;
-    stmt << "SELECT ";
-    stmt << mFilenameFieldName;
-    stmt << ", ";
-    stmt << mFilesizeFieldName;
-    stmt << " FROM ";
-    stmt << mTableName;
-    stmt << " WHERE ";
-    stmt << mIdFieldName;
-    stmt << "='";
-    stmt << id;
-    stmt << "'";
-
-#ifdef SHOW_STATEMENTS
-    std::cout << stmt.str() << std::endl;
-#endif
-
-    return stmt.str();
-}
-
-///////////////////////////////////////////////////////////////////////////////
-// SQL statement to update the date/time of last access as well as the
-// count of number of times the file has been accessed.
-// Note: the more accurate representation of date/time is used to
-// ensure ms accuracy vs the standard INTEGER days since epoch approach
-const std::string llDiskCache::sqlComposeGetUpdate(const std::string id)
-{
-    std::ostringstream stmt;
-    stmt << "UPDATE ";
-    stmt << mTableName;
-    stmt << " SET ";
-    stmt << mAccessCountFieldName;
-    stmt << "=";
-    stmt << mAccessCountFieldName;
-    stmt << "+1";
-    stmt << ", ";
-    stmt << mLastAccessDateTimeFieldName;
-    stmt << "=";
-    stmt << "strftime('%Y-%m-%d %H:%M:%f', 'now')";
-    stmt << " WHERE ";
-    stmt << mIdFieldName;
-    stmt << "='";
-    stmt << id;
-    stmt << "'";
-
-#ifdef SHOW_STATEMENTS
-    std::cout << stmt.str() << std::endl;
-#endif
-
-    return stmt.str();
-}
-
-///////////////////////////////////////////////////////////////////////////////
-// SQL statement to remove items from the database that are older
-// than the newest num_elements entries
-const std::string llDiskCache::sqlComposePurge(int num_entries)
-{
-    std::ostringstream stmt;
-    stmt << "DELETE FROM ";
-    stmt << mTableName;
-    stmt << " WHERE ";
-    stmt << mLastAccessDateTimeFieldName;
-    stmt << " NOT IN ";
-    stmt << "(";
-    stmt << "SELECT ";
-    stmt << mLastAccessDateTimeFieldName;
-    stmt << " FROM ";
-    stmt << mTableName;
-    stmt << " ORDER BY ";
-    stmt << mLastAccessDateTimeFieldName;
-    stmt << " DESC";
-    stmt << " LIMIT ";
-    stmt << num_entries;
-    stmt << ")";
-
-//#ifdef SHOW_STATEMENTS
-    std::cout << stmt.str() << std::endl;
-//#endif
-
-    return stmt.str();
+    return total_file_size;
 }
diff --git a/indra/llfilesystem/lldiskcache.h b/indra/llfilesystem/lldiskcache.h
index 39b8f7ef48..1618cec6a6 100644
--- a/indra/llfilesystem/lldiskcache.h
+++ b/indra/llfilesystem/lldiskcache.h
@@ -1,11 +1,6 @@
 /**
  * @file lldiskcache.h
- * @brief Declaration SQLite meta data / file storage API
- * @brief Declaration of the generic disk cache interface
-          as well the SQLite header/implementation.
- * @Note  Eventually, this component might split into an interface
- *        file and multiple implemtations but for now, this is the
- *        only one so I think it's okay to combine everything. 
+ * @brief The disk cache implementation.
  *
  * $LicenseInfo:firstyear=2009&license=viewerlgpl$
  * Second Life Viewer Source Code
@@ -32,99 +27,68 @@
 #ifndef _LLDISKCACHE
 #define _LLDISKCACHE
 
-#include <iostream>
-#include <string>
+class LLDiskCache
+{
+    public:
+        LLDiskCache(const std::string cache_dir);
+        ~LLDiskCache();
 
-#include "sqlite3.h"
+        /**
+         * Update the "last write time" of a file to "now". This must be called whenever a 
+         * file in the cache is read (not written) so that the last time the file was 
+         * accessed which is used in the mechanism for purging the cache, is up to date.
+         */
+        void updateFileAccessTime(const std::string file_path);
 
-// Enable this line to display each SQL statement when it is executed
-// It can lead to a lot of spam but useful for debugging
-//#define SHOW_STATEMENTS
+        /**
+         * Purge the oldest items in the cache so that the combined size of all files
+         * is no bigger than mMaxSizeBytes.
+         */
+        void purge();
 
-// I toyed with the idea of a variety of approaches and thought have
-// an abstract base class with which to hang different implementations
-// off was a good idea but I think we settled on a single approach
-// so I will likely remove this level of indirection if not other 
-// interesting implementation ideas present themselves.
-class llDiskCacheBase
-{
-    public:
-        llDiskCacheBase() {};
-        virtual ~llDiskCacheBase() {};
-        virtual bool open(const std::string db_folder,
-                          const std::string db_filename) = 0;
-        virtual bool exists(const std::string id, bool& exists) = 0;
-        virtual bool put(const std::string id,
-                         char* binary_data,
-                         int binary_data_size) = 0;
-        virtual const bool get(const std::string id,
-                               char** mem_buffer,
-                               int& mem_buffer_size) = 0;
-        virtual bool purge(int num_entries) = 0;
-        virtual void close() = 0;
-        virtual const std::string dbVersion() = 0;
-        virtual const std::string getFilenameById(const std::string id) = 0;
-        virtual const int getAccessCountById(const std::string id) = 0;
-        virtual void getNumEntries(int& num_entries, int& max_entries) = 0;
-};
+        /**
+         * Clear the cache by removing the files in the cache directory individually
+         */
+        void clearCache(const std::string cache_dir);
 
-// implementation for the SQLite/disk blended case
-// see source file for detailed comments
-class llDiskCache :
-    public llDiskCacheBase
-{
-    public:
-        llDiskCache();
-        virtual ~llDiskCache();
-        virtual bool open(const std::string db_folder,
-                          const std::string db_filename) override;
-        virtual bool exists(const std::string id, bool& exists) override;
-        virtual bool put(const std::string id,
-                         char* binary_data,
-                         int binary_data_size) override;
-        virtual const bool get(const std::string id,
-                               char** mem_buffer,
-                               int& mem_buffer_size) override;
-        virtual bool purge(int num_entries) override;
-        virtual void close() override;
-        virtual const std::string dbVersion() override;
-        virtual const std::string getFilenameById(const std::string id) override;
-        virtual const int getAccessCountById(const std::string id) override;
-        virtual void getNumEntries(int& num_entries, int& max_entries) override;
+        /**
+         * Manage max size in bytes of cache - use a discrete setter/getter so the value can
+         * be changed in the preferences and cache cleared/purged without restarting viewer
+         * to instantiate this class again.
+         */
+        void setMaxSizeBytes(const uintmax_t size_bytes) { mMaxSizeBytes = size_bytes; }
+        uintmax_t getMaxSizeBytes() const { return mMaxSizeBytes; }
 
     private:
-        sqlite3* mDb;
-        std::string mDataStorePath;
-        const std::string mTableName = "lldiskcache";
-        const std::string mIdFieldName = "id";
-        const std::string mFilenameFieldName = "filename";
-        const std::string mFilesizeFieldName = "filesize";
-        const std::string mInsertionDateTimeFieldName = "insert_datetime";
-        const std::string mLastAccessDateTimeFieldName = "last_access_datetime";
-        const std::string mAccessCountFieldName = "access_count";
+        /**
+         * Utility function to gather the total size the files in a given
+         * directory. Primarily used here to determine the directory size 
+         * before and after the cache purge
+         */
+        uintmax_t dirFileSize(const std::string dir);
 
     private:
-        void printError(const std::string funcname,
-                        const std::string desc,
-                        bool is_sqlite_err);
-        bool beginTransaction();
-        bool endTransaction();
-        std::string makeUniqueFilename();
-        const std::string systemSeparator();
-        const std::string makeFullPath(const std::string filename);
-        bool sqliteExec(const std::string stmt,
-                        const std::string funcname);
-        sqlite3_stmt* sqlitePrepareStep(const std::string funcname,
-                                        const std::string stmt,
-                                        int sqlite_success_condition);
-        const std::string sqlComposeCreateTable();
-        const std::string sqlComposeExists(const std::string id);
-        const std::string sqlComposePut(const std::string id,
-                                        const std::string filename,
-                                        int binary_data_size);
-        const std::string sqlComposeGetSelect(const std::string id);
-        const std::string sqlComposeGetUpdate(const std::string id);
-        const std::string sqlComposePurge(int num_entries);
+        /**
+         * Default of 20MB seems reasonable - it will likely be reset
+         * immediately anyway using a value from the Viewer settings
+         */
+        const uintmax_t mDefaultSizeBytes = 20 * 1024 * 1024;
+        uintmax_t mMaxSizeBytes;
+
+        /**
+         * The folder that holds the cached files. The consumer of this
+         * class must avoid letting the user set this location as a malicious
+         * setting could potentially point it at a non-cache directory (for example,
+         * the Windows System dir) with disastrous results.
+         */
+        std::string mCacheDir;
+
+        /**
+         * This is set from the CacheDebugInfo global setting and
+         * when enabled, displays additional debugging information in
+         * various parts of the code
+         */
+        bool mCacheDebugInfo;
 };
 
 #endif // _LLDISKCACHE
diff --git a/indra/llfilesystem/llfilesystem.cpp b/indra/llfilesystem/llfilesystem.cpp
index f0037c9a22..ffc3dee12b 100644
--- a/indra/llfilesystem/llfilesystem.cpp
+++ b/indra/llfilesystem/llfilesystem.cpp
@@ -42,6 +42,8 @@ const S32 LLFileSystem::READ_WRITE	= 0x00000003;  // LLFileSystem::READ & LLFile
 const S32 LLFileSystem::APPEND		= 0x00000006;  // 0x00000004 & LLFileSystem::WRITE
 
 static LLTrace::BlockTimerStatHandle FTM_VFILE_WAIT("VFile Wait");
+LLDiskCache* LLFileSystem::mDiskCache = 0;
+std::string LLFileSystem::mCacheDirName = "cache";
 
 LLFileSystem::LLFileSystem(const LLUUID &file_id, const LLAssetType::EType file_type, S32 mode)
 {
@@ -104,7 +106,7 @@ const std::string assetTypeToString(LLAssetType::EType at)
     return std::string("UNKNOWN");
 }
 
-const std::string idToFilepath(const std::string id, LLAssetType::EType at)
+const std::string LLFileSystem::idToFilepath(const std::string id, LLAssetType::EType at)
 {
     /**
      * For the moment this is just {UUID}_{ASSET_TYPE}.txt but of
@@ -117,7 +119,7 @@ const std::string idToFilepath(const std::string id, LLAssetType::EType at)
     ss << assetTypeToString(at);
     ss << ".txt";
 
-    const std::string filepath = gDirUtilp->getExpandedFilename(LL_PATH_CACHE, ss.str());
+    const std::string filepath = gDirUtilp->getExpandedFilename(LL_PATH_CACHE, mCacheDirName, ss.str());
 
     return filepath;
 }
@@ -289,7 +291,6 @@ BOOL LLFileSystem::write(const U8 *buffer, S32 bytes)
 BOOL LLFileSystem::writeFile(const U8 *buffer, S32 bytes, const LLUUID &uuid, LLAssetType::EType type)
 {
 	LLFileSystem file(uuid, type, LLFileSystem::WRITE);
-	file.setMaxSize(bytes);
 	return file.write(buffer, bytes);
 }
 
@@ -339,13 +340,6 @@ S32 LLFileSystem::getMaxSize()
     return INT_MAX;
 }
 
-BOOL LLFileSystem::setMaxSize(S32 size)
-{
-    // we don't care what the max size is so we do nothing
-    // and return true to indicate all was okay
-    return TRUE;
-}
-
 BOOL LLFileSystem::rename(const LLUUID &new_id, const LLAssetType::EType new_type)
 {
     LLFileSystem::renameFile(mFileID, mFileType, new_id, new_type);
@@ -366,21 +360,15 @@ BOOL LLFileSystem::remove()
 // static
 void LLFileSystem::initClass()
 {
-}
+    const std::string cache_dir = gDirUtilp->getExpandedFilename(LL_PATH_CACHE, mCacheDirName);
 
-// static
-void LLFileSystem::cleanupClass()
-{
-}
+    LLFileSystem::mDiskCache = new LLDiskCache(cache_dir);
 
-bool LLFileSystem::isLocked()
-{
-    // I don't think we care about this test since there is no locking
-    // TODO: remove this function and calling sites?
-    return FALSE;
+    mDiskCache->purge();
 }
 
-void LLFileSystem::waitForLock()
+// static
+void LLFileSystem::cleanupClass()
 {
-    // TODO: remove this function and calling sites?
+    delete LLFileSystem::mDiskCache;
 }
diff --git a/indra/llfilesystem/llfilesystem.h b/indra/llfilesystem/llfilesystem.h
index 9cddef74a7..789f9456c7 100644
--- a/indra/llfilesystem/llfilesystem.h
+++ b/indra/llfilesystem/llfilesystem.h
@@ -32,6 +32,7 @@
 
 #include "lluuid.h"
 #include "llassettype.h"
+#include "lldiskcache.h"
 
 class LLFileSystem
 {
@@ -51,13 +52,9 @@ public:
 
 	S32 getSize();
 	S32 getMaxSize();
-	BOOL setMaxSize(S32 size);
 	BOOL rename(const LLUUID &new_id, const LLAssetType::EType new_type);
 	BOOL remove();
 
-	bool isLocked();
-	void waitForLock();
-
     static bool getExists(const LLUUID &file_id, const LLAssetType::EType file_type);
     static bool removeFile(const LLUUID &file_id, const LLAssetType::EType file_type);
     static bool renameFile(const LLUUID &old_file_id, const LLAssetType::EType old_file_type,
@@ -80,6 +77,11 @@ protected:
 	S32		mPosition;
 	S32		mMode;
 	S32		mBytesRead;
+
+private:
+    static const std::string idToFilepath(const std::string id, LLAssetType::EType at);
+    static std::string mCacheDirName;
+    static LLDiskCache* mDiskCache;
 };
 
 #endif  // LL_FILESYSTEM_H
diff --git a/indra/llmessage/lltransfertargetvfile.cpp b/indra/llmessage/lltransfertargetvfile.cpp
index 471d687d67..f6faadf87f 100644
--- a/indra/llmessage/lltransfertargetvfile.cpp
+++ b/indra/llmessage/lltransfertargetvfile.cpp
@@ -141,7 +141,6 @@ LLTSCode LLTransferTargetVFile::dataCallback(const S32 packet_id, U8 *in_datap,
 	LLFileSystem vf(mTempID, mParams.getAssetType(), LLFileSystem::APPEND);
 	if (mNeedsCreate)
 	{
-		vf.setMaxSize(mSize);
 		mNeedsCreate = FALSE;
 	}
 
diff --git a/indra/llmessage/llxfer_vfile.cpp b/indra/llmessage/llxfer_vfile.cpp
index 9de9ed379b..12419b342d 100644
--- a/indra/llmessage/llxfer_vfile.cpp
+++ b/indra/llmessage/llxfer_vfile.cpp
@@ -261,7 +261,6 @@ void LLXfer_VFile::setXferSize (S32 xfer_size)
 	if (! mVFile)
 	{
 		LLFileSystem file(mTempID, mType, LLFileSystem::APPEND);
-		file.setMaxSize(xfer_size);
 	}
 }
 
diff --git a/indra/newview/app_settings/settings.xml b/indra/newview/app_settings/settings.xml
index c45d6dd7af..0123bc32af 100644
--- a/indra/newview/app_settings/settings.xml
+++ b/indra/newview/app_settings/settings.xml
@@ -1351,6 +1351,17 @@
       <key>Value</key>
       <integer>23</integer>
     </map>
+    <key>CacheDebugInfo</key>
+    <map>
+      <key>Comment</key>
+      <string>When enabled, display additional cache debugging information</string>
+      <key>Persist</key>
+      <integer>1</integer>
+      <key>Type</key>
+      <string>Boolean</string>
+      <key>Value</key>
+      <integer>0</integer>
+    </map>
     <key>CacheLocation</key>
     <map>
       <key>Comment</key>
diff --git a/indra/newview/llfloaterbvhpreview.cpp b/indra/newview/llfloaterbvhpreview.cpp
index 08f3b577b4..687d820a18 100644
--- a/indra/newview/llfloaterbvhpreview.cpp
+++ b/indra/newview/llfloaterbvhpreview.cpp
@@ -1000,7 +1000,6 @@ void LLFloaterBvhPreview::onBtnOK(void* userdata)
 			LLFileSystem file(motionp->getID(), LLAssetType::AT_ANIMATION, LLFileSystem::APPEND);
 
 			S32 size = dp.getCurrentSize();
-			file.setMaxSize(size);
 			if (file.write((U8*)buffer, size))
 			{
 				std::string name = floaterp->getChild<LLUICtrl>("name_form")->getValue().asString();
diff --git a/indra/newview/llmeshrepository.cpp b/indra/newview/llmeshrepository.cpp
index a0d591dc47..3183e6d8fd 100644
--- a/indra/newview/llmeshrepository.cpp
+++ b/indra/newview/llmeshrepository.cpp
@@ -3241,7 +3241,7 @@ void LLMeshHeaderHandler::processData(LLCore::BufferArray * /* body */, S32 /* b
 			data_size = llmin(data_size, bytes);
 
 			LLFileSystem file(mesh_id, LLAssetType::AT_MESH, LLFileSystem::WRITE);
-			if (file.getMaxSize() >= bytes || file.setMaxSize(bytes))
+			if (file.getMaxSize() >= bytes)
 			{
 				LLMeshRepository::sCacheBytesWritten += data_size;
 				++LLMeshRepository::sCacheWrites;
diff --git a/indra/newview/llpreviewgesture.cpp b/indra/newview/llpreviewgesture.cpp
index 371153aac3..4318a55704 100644
--- a/indra/newview/llpreviewgesture.cpp
+++ b/indra/newview/llpreviewgesture.cpp
@@ -1140,7 +1140,6 @@ void LLPreviewGesture::saveIfNeeded()
             LLFileSystem file(assetId, LLAssetType::AT_GESTURE, LLFileSystem::APPEND);
 
             S32 size = dp.getCurrentSize();
-            file.setMaxSize(size);
             file.write((U8*)buffer, size);
 
             LLLineEditor* descEditor = getChild<LLLineEditor>("desc");
diff --git a/indra/newview/llpreviewnotecard.cpp b/indra/newview/llpreviewnotecard.cpp
index 0bccf1d06f..32e1a4a186 100644
--- a/indra/newview/llpreviewnotecard.cpp
+++ b/indra/newview/llpreviewnotecard.cpp
@@ -563,7 +563,6 @@ bool LLPreviewNotecard::saveIfNeeded(LLInventoryItem* copyitem, bool sync)
 																tid, copyitem);
 
                 S32 size = buffer.length() + 1;
-                file.setMaxSize(size);
                 file.write((U8*)buffer.c_str(), size);
 
 				gAssetStorage->storeAssetData(tid, LLAssetType::AT_NOTECARD,
diff --git a/indra/newview/llviewerassetstorage.cpp b/indra/newview/llviewerassetstorage.cpp
index df3ff1a3c7..5b76d57196 100644
--- a/indra/newview/llviewerassetstorage.cpp
+++ b/indra/newview/llviewerassetstorage.cpp
@@ -293,8 +293,6 @@ void LLViewerAssetStorage::storeAssetData(
 
         LLFileSystem file(asset_id, asset_type, LLFileSystem::WRITE);
 
-        file.setMaxSize(size);
-
         const S32 buf_size = 65536;
         U8 copy_buf[buf_size];
         while ((size = (S32)fread(copy_buf, 1, buf_size, fp)))
@@ -528,7 +526,6 @@ void LLViewerAssetStorage::assetRequestCoro(
             LLUUID temp_id;
             temp_id.generate();
             LLFileSystem vf(temp_id, atype, LLFileSystem::WRITE);
-            vf.setMaxSize(size);
             req->mBytesFetched = size;
             if (!vf.write(raw.data(),size))
             {
diff --git a/indra/newview/llviewerassetupload.cpp b/indra/newview/llviewerassetupload.cpp
index fb3ca69d5d..67ee06e255 100644
--- a/indra/newview/llviewerassetupload.cpp
+++ b/indra/newview/llviewerassetupload.cpp
@@ -474,8 +474,6 @@ LLSD LLNewFileResourceUploadInfo::exportTempFile()
     {
         LLFileSystem file(getAssetId(), assetType, LLFileSystem::WRITE);
 
-        file.setMaxSize(file_size);
-
         const S32 buf_size = 65536;
         U8 copy_buf[buf_size];
         while ((file_size = infile.read(copy_buf, buf_size)))
@@ -568,7 +566,6 @@ LLSD LLBufferedAssetUploadInfo::prepareUpload()
     LLFileSystem file(getAssetId(), getAssetType(), LLFileSystem::APPEND);
 
     S32 size = mContents.length() + 1;
-    file.setMaxSize(size);
     file.write((U8*)mContents.c_str(), size);
 
     mStoredToCache = true;