From d9448c6f52218146113d1d5c5ca4c4d5f01dc5cf Mon Sep 17 00:00:00 2001 From: Callum Prentice Date: Thu, 17 Sep 2020 09:45:06 -0700 Subject: The folder where the disk cache lives was originally renamed from llvfs to llcache but @henri's suggestion that that doesn't reflect the other files in the same place and it should be llfilesystem is a good one so I changed it over --- indra/llfilesystem/lldiskcache.cpp | 387 +++++++++++++++++++++++++++++++++++++ 1 file changed, 387 insertions(+) create mode 100644 indra/llfilesystem/lldiskcache.cpp (limited to 'indra/llfilesystem/lldiskcache.cpp') diff --git a/indra/llfilesystem/lldiskcache.cpp b/indra/llfilesystem/lldiskcache.cpp new file mode 100644 index 0000000000..af93049e07 --- /dev/null +++ b/indra/llfilesystem/lldiskcache.cpp @@ -0,0 +1,387 @@ +/** + * @file lldiskcache.cpp + * @brief Implementation of virtual file + * + * $LicenseInfo:firstyear=2002&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2010, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include "linden_common.h" + +#include "lldiskcache.h" + +#include "llerror.h" +#include "llthread.h" +#include "lltimer.h" +#include "llfasttimer.h" +#include "llmemory.h" + +#include +#include "lldir.h" + +const S32 LLDiskCache::READ = 0x00000001; +const S32 LLDiskCache::WRITE = 0x00000002; +const S32 LLDiskCache::READ_WRITE = 0x00000003; // LLDiskCache::READ & LLDiskCache::WRITE +const S32 LLDiskCache::APPEND = 0x00000006; // 0x00000004 & LLDiskCache::WRITE + +static LLTrace::BlockTimerStatHandle FTM_VFILE_WAIT("VFile Wait"); + +LLDiskCache::LLDiskCache(const LLUUID &file_id, const LLAssetType::EType file_type, S32 mode) +{ + mFileType = file_type; + mFileID = file_id; + mPosition = 0; + mBytesRead = 0; + mReadComplete = FALSE; + mMode = mode; +} + +LLDiskCache::~LLDiskCache() +{ +} + +const std::string assetTypeToString(LLAssetType::EType at) +{ + /** + * Make use of the C++17 (or is it 14) feature that allows + * for inline initialization of an std::map<> + */ + typedef std::map asset_type_to_name_t; + asset_type_to_name_t asset_type_to_name = + { + { LLAssetType::AT_TEXTURE, "TEXTURE" }, + { LLAssetType::AT_SOUND, "SOUND" }, + { LLAssetType::AT_CALLINGCARD, "CALLINGCARD" }, + { LLAssetType::AT_LANDMARK, "LANDMARK" }, + { LLAssetType::AT_SCRIPT, "SCRIPT" }, + { LLAssetType::AT_CLOTHING, "CLOTHING" }, + { LLAssetType::AT_OBJECT, "OBJECT" }, + { LLAssetType::AT_NOTECARD, "NOTECARD" }, + { LLAssetType::AT_CATEGORY, "CATEGORY" }, + { LLAssetType::AT_LSL_TEXT, "LSL_TEXT" }, + { LLAssetType::AT_LSL_BYTECODE, "LSL_BYTECODE" }, + { LLAssetType::AT_TEXTURE_TGA, "TEXTURE_TGA" }, + { LLAssetType::AT_BODYPART, "BODYPART" }, + { LLAssetType::AT_SOUND_WAV, "SOUND_WAV" }, + { LLAssetType::AT_IMAGE_TGA, "IMAGE_TGA" }, + { LLAssetType::AT_IMAGE_JPEG, "IMAGE_JPEG" }, + { LLAssetType::AT_ANIMATION, "ANIMATION" }, + { LLAssetType::AT_GESTURE, "GESTURE" }, + { LLAssetType::AT_SIMSTATE, "SIMSTATE" }, + { LLAssetType::AT_LINK, "LINK" }, + { LLAssetType::AT_LINK_FOLDER, "LINK_FOLDER" }, + { LLAssetType::AT_MARKETPLACE_FOLDER, "MARKETPLACE_FOLDER" }, + { LLAssetType::AT_WIDGET, "WIDGET" }, + { LLAssetType::AT_PERSON, "PERSON" }, + { LLAssetType::AT_MESH, "MESH" }, + { LLAssetType::AT_SETTINGS, "SETTINGS" }, + { LLAssetType::AT_UNKNOWN, "UNKNOWN" } + }; + + asset_type_to_name_t::iterator iter = asset_type_to_name.find(at); + if (iter != asset_type_to_name.end()) + { + return iter->second; + } + + return std::string("UNKNOWN"); +} + +const std::string idToFilepath(const std::string id, LLAssetType::EType at) +{ + /** + * For the moment this is just {UUID}_{ASSET_TYPE}.txt but of + * course, will be greatly expanded upon + */ + std::ostringstream ss; + ss << "00cache_"; + ss << id; + ss << "_"; + ss << assetTypeToString(at); + ss << ".txt"; + + const std::string filepath = gDirUtilp->getExpandedFilename(LL_PATH_CACHE, ss.str()); + + return filepath; +} + +// static +bool LLDiskCache::getExists(const LLUUID &file_id, const LLAssetType::EType file_type) +{ + std::string id_str; + file_id.toString(id_str); + const std::string filename = idToFilepath(id_str, file_type); + + std::ifstream file(filename, std::ios::binary); + if (file.is_open()) + { + file.seekg(0, std::ios::end); + return file.tellg() > 0; + } + return false; +} + +// static +bool LLDiskCache::removeFile(const LLUUID &file_id, const LLAssetType::EType file_type) +{ + std::string id_str; + file_id.toString(id_str); + const std::string filename = idToFilepath(id_str, file_type); + + std::remove(filename.c_str()); + + return true; +} + +// static +bool LLDiskCache::renameFile(const LLUUID &old_file_id, const LLAssetType::EType old_file_type, + const LLUUID &new_file_id, const LLAssetType::EType new_file_type) +{ + std::string old_id_str; + old_file_id.toString(old_id_str); + const std::string old_filename = idToFilepath(old_id_str, old_file_type); + + std::string new_id_str; + new_file_id.toString(new_id_str); + const std::string new_filename = idToFilepath(new_id_str, new_file_type); + + if (std::rename(old_filename.c_str(), new_filename.c_str())) + { + // We would like to return FALSE here indicating the operation + // failed but the original code does not and doing so seems to + // break a lot of things so we go with the flow... + //return FALSE; + } + + return TRUE; +} + +// static +S32 LLDiskCache::getFileSize(const LLUUID &file_id, const LLAssetType::EType file_type) +{ + std::string id_str; + file_id.toString(id_str); + const std::string filename = idToFilepath(id_str, file_type); + + S32 file_size = 0; + std::ifstream file(filename, std::ios::binary); + if (file.is_open()) + { + file.seekg(0, std::ios::end); + file_size = file.tellg(); + } + + return file_size; +} + +BOOL LLDiskCache::read(U8 *buffer, S32 bytes, BOOL async, F32 priority) +{ + BOOL success = TRUE; + + mReadComplete = FALSE; + + std::string id; + mFileID.toString(id); + const std::string filename = idToFilepath(id, mFileType); + + std::ifstream file(filename, std::ios::binary); + if (file.is_open()) + { + file.seekg(mPosition, std::ios::beg); + + file.read((char*)buffer, bytes); + + if (file) + { + mBytesRead = bytes; + } + else + { + mBytesRead = file.gcount(); + } + + file.close(); + + mPosition += mBytesRead; + if (!mBytesRead) + { + success = FALSE; + } + + mReadComplete = TRUE; + } + + return success; +} + +BOOL LLDiskCache::isReadComplete() +{ + if (mReadComplete) + { + return TRUE; + } + + return FALSE; +} + +S32 LLDiskCache::getLastBytesRead() +{ + return mBytesRead; +} + +BOOL LLDiskCache::eof() +{ + return mPosition >= getSize(); +} + +BOOL LLDiskCache::write(const U8 *buffer, S32 bytes) +{ + std::string id_str; + mFileID.toString(id_str); + const std::string filename = idToFilepath(id_str, mFileType); + + BOOL success = FALSE; + + if (mMode == APPEND) + { + std::ofstream ofs(filename, std::ios::app | std::ios::binary); + if (ofs) + { + ofs.write((const char*)buffer, bytes); + + success = TRUE; + } + } + else + { + std::ofstream ofs(filename, std::ios::binary); + if (ofs) + { + ofs.write((const char*)buffer, bytes); + + mPosition += bytes; + + success = TRUE; + } + } + + return success; +} + +//static +BOOL LLDiskCache::writeFile(const U8 *buffer, S32 bytes, const LLUUID &uuid, LLAssetType::EType type) +{ + LLDiskCache file(uuid, type, LLDiskCache::WRITE); + file.setMaxSize(bytes); + return file.write(buffer, bytes); +} + +BOOL LLDiskCache::seek(S32 offset, S32 origin) +{ + if (-1 == origin) + { + origin = mPosition; + } + + S32 new_pos = origin + offset; + + S32 size = getSize(); + + if (new_pos > size) + { + LL_WARNS() << "Attempt to seek past end of file" << LL_ENDL; + + mPosition = size; + return FALSE; + } + else if (new_pos < 0) + { + LL_WARNS() << "Attempt to seek past beginning of file" << LL_ENDL; + + mPosition = 0; + return FALSE; + } + + mPosition = new_pos; + return TRUE; +} + +S32 LLDiskCache::tell() const +{ + return mPosition; +} + +S32 LLDiskCache::getSize() +{ + return LLDiskCache::getFileSize(mFileID, mFileType); +} + +S32 LLDiskCache::getMaxSize() +{ + // offer up a huge size since we don't care what the max is + return INT_MAX; +} + +BOOL LLDiskCache::setMaxSize(S32 size) +{ + // we don't care what the max size is so we do nothing + // and return true to indicate all was okay + return TRUE; +} + +BOOL LLDiskCache::rename(const LLUUID &new_id, const LLAssetType::EType new_type) +{ + LLDiskCache::renameFile(mFileID, mFileType, new_id, new_type); + + mFileID = new_id; + mFileType = new_type; + + return TRUE; +} + +BOOL LLDiskCache::remove() +{ + LLDiskCache::removeFile(mFileID, mFileType); + + return TRUE; +} + +// static +void LLDiskCache::initClass() +{ +} + +// static +void LLDiskCache::cleanupClass() +{ +} + +bool LLDiskCache::isLocked() +{ + // I don't think we care about this test since there is no locking + // TODO: remove this function and calling sites? + return FALSE; +} + +void LLDiskCache::waitForLock() +{ + // TODO: remove this function and calling sites? +} -- cgit v1.2.3 From 96e2873bfa2c6b8823aed3b4190c43cd5dab54e6 Mon Sep 17 00:00:00 2001 From: Callum Prentice Date: Thu, 24 Sep 2020 10:23:39 -0700 Subject: Rename lldiskcache.* to llfilesystem.* - i think this is the right name since it's responsible for performing file system operations and (will eventually) delegrate to a separate disk cache component to save/load data and keep track of metadata etc. --- indra/llfilesystem/lldiskcache.cpp | 387 ------------------------------------- 1 file changed, 387 deletions(-) delete mode 100644 indra/llfilesystem/lldiskcache.cpp (limited to 'indra/llfilesystem/lldiskcache.cpp') diff --git a/indra/llfilesystem/lldiskcache.cpp b/indra/llfilesystem/lldiskcache.cpp deleted file mode 100644 index af93049e07..0000000000 --- a/indra/llfilesystem/lldiskcache.cpp +++ /dev/null @@ -1,387 +0,0 @@ -/** - * @file lldiskcache.cpp - * @brief Implementation of virtual file - * - * $LicenseInfo:firstyear=2002&license=viewerlgpl$ - * Second Life Viewer Source Code - * Copyright (C) 2010, Linden Research, Inc. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; - * version 2.1 of the License only. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - * - * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA - * $/LicenseInfo$ - */ - -#include "linden_common.h" - -#include "lldiskcache.h" - -#include "llerror.h" -#include "llthread.h" -#include "lltimer.h" -#include "llfasttimer.h" -#include "llmemory.h" - -#include -#include "lldir.h" - -const S32 LLDiskCache::READ = 0x00000001; -const S32 LLDiskCache::WRITE = 0x00000002; -const S32 LLDiskCache::READ_WRITE = 0x00000003; // LLDiskCache::READ & LLDiskCache::WRITE -const S32 LLDiskCache::APPEND = 0x00000006; // 0x00000004 & LLDiskCache::WRITE - -static LLTrace::BlockTimerStatHandle FTM_VFILE_WAIT("VFile Wait"); - -LLDiskCache::LLDiskCache(const LLUUID &file_id, const LLAssetType::EType file_type, S32 mode) -{ - mFileType = file_type; - mFileID = file_id; - mPosition = 0; - mBytesRead = 0; - mReadComplete = FALSE; - mMode = mode; -} - -LLDiskCache::~LLDiskCache() -{ -} - -const std::string assetTypeToString(LLAssetType::EType at) -{ - /** - * Make use of the C++17 (or is it 14) feature that allows - * for inline initialization of an std::map<> - */ - typedef std::map asset_type_to_name_t; - asset_type_to_name_t asset_type_to_name = - { - { LLAssetType::AT_TEXTURE, "TEXTURE" }, - { LLAssetType::AT_SOUND, "SOUND" }, - { LLAssetType::AT_CALLINGCARD, "CALLINGCARD" }, - { LLAssetType::AT_LANDMARK, "LANDMARK" }, - { LLAssetType::AT_SCRIPT, "SCRIPT" }, - { LLAssetType::AT_CLOTHING, "CLOTHING" }, - { LLAssetType::AT_OBJECT, "OBJECT" }, - { LLAssetType::AT_NOTECARD, "NOTECARD" }, - { LLAssetType::AT_CATEGORY, "CATEGORY" }, - { LLAssetType::AT_LSL_TEXT, "LSL_TEXT" }, - { LLAssetType::AT_LSL_BYTECODE, "LSL_BYTECODE" }, - { LLAssetType::AT_TEXTURE_TGA, "TEXTURE_TGA" }, - { LLAssetType::AT_BODYPART, "BODYPART" }, - { LLAssetType::AT_SOUND_WAV, "SOUND_WAV" }, - { LLAssetType::AT_IMAGE_TGA, "IMAGE_TGA" }, - { LLAssetType::AT_IMAGE_JPEG, "IMAGE_JPEG" }, - { LLAssetType::AT_ANIMATION, "ANIMATION" }, - { LLAssetType::AT_GESTURE, "GESTURE" }, - { LLAssetType::AT_SIMSTATE, "SIMSTATE" }, - { LLAssetType::AT_LINK, "LINK" }, - { LLAssetType::AT_LINK_FOLDER, "LINK_FOLDER" }, - { LLAssetType::AT_MARKETPLACE_FOLDER, "MARKETPLACE_FOLDER" }, - { LLAssetType::AT_WIDGET, "WIDGET" }, - { LLAssetType::AT_PERSON, "PERSON" }, - { LLAssetType::AT_MESH, "MESH" }, - { LLAssetType::AT_SETTINGS, "SETTINGS" }, - { LLAssetType::AT_UNKNOWN, "UNKNOWN" } - }; - - asset_type_to_name_t::iterator iter = asset_type_to_name.find(at); - if (iter != asset_type_to_name.end()) - { - return iter->second; - } - - return std::string("UNKNOWN"); -} - -const std::string idToFilepath(const std::string id, LLAssetType::EType at) -{ - /** - * For the moment this is just {UUID}_{ASSET_TYPE}.txt but of - * course, will be greatly expanded upon - */ - std::ostringstream ss; - ss << "00cache_"; - ss << id; - ss << "_"; - ss << assetTypeToString(at); - ss << ".txt"; - - const std::string filepath = gDirUtilp->getExpandedFilename(LL_PATH_CACHE, ss.str()); - - return filepath; -} - -// static -bool LLDiskCache::getExists(const LLUUID &file_id, const LLAssetType::EType file_type) -{ - std::string id_str; - file_id.toString(id_str); - const std::string filename = idToFilepath(id_str, file_type); - - std::ifstream file(filename, std::ios::binary); - if (file.is_open()) - { - file.seekg(0, std::ios::end); - return file.tellg() > 0; - } - return false; -} - -// static -bool LLDiskCache::removeFile(const LLUUID &file_id, const LLAssetType::EType file_type) -{ - std::string id_str; - file_id.toString(id_str); - const std::string filename = idToFilepath(id_str, file_type); - - std::remove(filename.c_str()); - - return true; -} - -// static -bool LLDiskCache::renameFile(const LLUUID &old_file_id, const LLAssetType::EType old_file_type, - const LLUUID &new_file_id, const LLAssetType::EType new_file_type) -{ - std::string old_id_str; - old_file_id.toString(old_id_str); - const std::string old_filename = idToFilepath(old_id_str, old_file_type); - - std::string new_id_str; - new_file_id.toString(new_id_str); - const std::string new_filename = idToFilepath(new_id_str, new_file_type); - - if (std::rename(old_filename.c_str(), new_filename.c_str())) - { - // We would like to return FALSE here indicating the operation - // failed but the original code does not and doing so seems to - // break a lot of things so we go with the flow... - //return FALSE; - } - - return TRUE; -} - -// static -S32 LLDiskCache::getFileSize(const LLUUID &file_id, const LLAssetType::EType file_type) -{ - std::string id_str; - file_id.toString(id_str); - const std::string filename = idToFilepath(id_str, file_type); - - S32 file_size = 0; - std::ifstream file(filename, std::ios::binary); - if (file.is_open()) - { - file.seekg(0, std::ios::end); - file_size = file.tellg(); - } - - return file_size; -} - -BOOL LLDiskCache::read(U8 *buffer, S32 bytes, BOOL async, F32 priority) -{ - BOOL success = TRUE; - - mReadComplete = FALSE; - - std::string id; - mFileID.toString(id); - const std::string filename = idToFilepath(id, mFileType); - - std::ifstream file(filename, std::ios::binary); - if (file.is_open()) - { - file.seekg(mPosition, std::ios::beg); - - file.read((char*)buffer, bytes); - - if (file) - { - mBytesRead = bytes; - } - else - { - mBytesRead = file.gcount(); - } - - file.close(); - - mPosition += mBytesRead; - if (!mBytesRead) - { - success = FALSE; - } - - mReadComplete = TRUE; - } - - return success; -} - -BOOL LLDiskCache::isReadComplete() -{ - if (mReadComplete) - { - return TRUE; - } - - return FALSE; -} - -S32 LLDiskCache::getLastBytesRead() -{ - return mBytesRead; -} - -BOOL LLDiskCache::eof() -{ - return mPosition >= getSize(); -} - -BOOL LLDiskCache::write(const U8 *buffer, S32 bytes) -{ - std::string id_str; - mFileID.toString(id_str); - const std::string filename = idToFilepath(id_str, mFileType); - - BOOL success = FALSE; - - if (mMode == APPEND) - { - std::ofstream ofs(filename, std::ios::app | std::ios::binary); - if (ofs) - { - ofs.write((const char*)buffer, bytes); - - success = TRUE; - } - } - else - { - std::ofstream ofs(filename, std::ios::binary); - if (ofs) - { - ofs.write((const char*)buffer, bytes); - - mPosition += bytes; - - success = TRUE; - } - } - - return success; -} - -//static -BOOL LLDiskCache::writeFile(const U8 *buffer, S32 bytes, const LLUUID &uuid, LLAssetType::EType type) -{ - LLDiskCache file(uuid, type, LLDiskCache::WRITE); - file.setMaxSize(bytes); - return file.write(buffer, bytes); -} - -BOOL LLDiskCache::seek(S32 offset, S32 origin) -{ - if (-1 == origin) - { - origin = mPosition; - } - - S32 new_pos = origin + offset; - - S32 size = getSize(); - - if (new_pos > size) - { - LL_WARNS() << "Attempt to seek past end of file" << LL_ENDL; - - mPosition = size; - return FALSE; - } - else if (new_pos < 0) - { - LL_WARNS() << "Attempt to seek past beginning of file" << LL_ENDL; - - mPosition = 0; - return FALSE; - } - - mPosition = new_pos; - return TRUE; -} - -S32 LLDiskCache::tell() const -{ - return mPosition; -} - -S32 LLDiskCache::getSize() -{ - return LLDiskCache::getFileSize(mFileID, mFileType); -} - -S32 LLDiskCache::getMaxSize() -{ - // offer up a huge size since we don't care what the max is - return INT_MAX; -} - -BOOL LLDiskCache::setMaxSize(S32 size) -{ - // we don't care what the max size is so we do nothing - // and return true to indicate all was okay - return TRUE; -} - -BOOL LLDiskCache::rename(const LLUUID &new_id, const LLAssetType::EType new_type) -{ - LLDiskCache::renameFile(mFileID, mFileType, new_id, new_type); - - mFileID = new_id; - mFileType = new_type; - - return TRUE; -} - -BOOL LLDiskCache::remove() -{ - LLDiskCache::removeFile(mFileID, mFileType); - - return TRUE; -} - -// static -void LLDiskCache::initClass() -{ -} - -// static -void LLDiskCache::cleanupClass() -{ -} - -bool LLDiskCache::isLocked() -{ - // I don't think we care about this test since there is no locking - // TODO: remove this function and calling sites? - return FALSE; -} - -void LLDiskCache::waitForLock() -{ - // TODO: remove this function and calling sites? -} -- cgit v1.2.3 From 6be1f88a5ef99e1e40bb5701a250ba0728f56005 Mon Sep 17 00:00:00 2001 From: Callum Prentice Date: Thu, 24 Sep 2020 14:45:39 -0700 Subject: Complete the change from lldiskcache -> llfilesystem and then addition of new lldiskcache implementation --- indra/llfilesystem/lldiskcache.cpp | 773 +++++++++++++++++++++++++++++++++++++ 1 file changed, 773 insertions(+) create mode 100644 indra/llfilesystem/lldiskcache.cpp (limited to 'indra/llfilesystem/lldiskcache.cpp') diff --git a/indra/llfilesystem/lldiskcache.cpp b/indra/llfilesystem/lldiskcache.cpp new file mode 100644 index 0000000000..72982db069 --- /dev/null +++ b/indra/llfilesystem/lldiskcache.cpp @@ -0,0 +1,773 @@ +/** + * @file lldiskcache.cpp + * @brief The SQLite based disk cache implementation. + * @Note Eventually, this component might split into an interface + * file and multiple implemtations but for now, this is the + * only one so I think it's okay to combine everything. + * + * $LicenseInfo:firstyear=2009&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2020, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#if (defined(LL_WINDOWS) || defined(LL_LINUX) || defined(LL_DARWIN)) +#include "linden_common.h" +#endif + +#include "lldiskcache.h" + +#include +#include +#include +#include +#include + +/////////////////////////////////////////////////////////////////////////////// +// +llDiskCache::llDiskCache() : + mDataStorePath("") +{ +} + +llDiskCache::~llDiskCache() +{ +} + +/////////////////////////////////////////////////////////////////////////////// +// Opens the database - typically done when the application starts +// and is complementary to close() which is called when the application +// is finisahed and exits. +// Pass in the folder and filename of the SQLite database you want to +// use or create (file doesn't have to exist but the folder must) +// Returns true or false and writes a message to console on error +bool llDiskCache::open(const std::string db_folder, const std::string db_filename) +{ + mDataStorePath = db_folder; + std::string db_pathname = makeFullPath(db_filename); + + // simple flags for the moment - these will likely be extended + // later on to support the SQLite mutex model for reading/writing + // simultaneously - perhaps when we look at supporting textures too + int flags = SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE; + + if (sqlite3_open_v2( + db_pathname.c_str(), + &mDb, + flags, + nullptr // Name of VFS module to use + ) != SQLITE_OK) + { + printError(__FUNCDNAME__ , "open_v2", true); + close(); + return false; + } + + // I elected to store the code that generates the statement + // in sepsrate functions throughout - this seemed like a cleaner + // approach than having hundreds of stmt << "REPLACE AS" lines + // interspersed in the logic code. They are all prefixed with + // 'sqlCompose' and followed by a short description. + const std::string stmt = sqlComposeCreateTable(); + if (! sqliteExec(stmt, __FUNCDNAME__ )) + { + // Not sure if we need close here - if open fails, then we + // perhaps don't need to close it - TODO: look in SQLite docs + close(); + return false; + } + + return true; +} + +/////////////////////////////////////////////////////////////////////////////// +// Determines if an entry exists. +// Pass in the id and a variable that will indicate existance +// Returns true/false if function succeeded and the boolean +// you pass in will be set to true/false if entry exists or not +bool llDiskCache::exists(const std::string id, bool& exists) +{ + if (!mDb) + { + printError(__FUNCDNAME__ , "mDb is invalid", false); + return false; + } + + if (id.empty()) + { + printError(__FUNCDNAME__ , "id is empty", false); + return false; + } + + // As well as the separate function to compose the SQL statement, + // worth pointing out that the code to 'prepare' and 'step' the + // SQLite "prepared statement" has been factored out into its own + // function and used in several other functions. + const std::string stmt = sqlComposeExists(id); + sqlite3_stmt* prepared_statement = sqlitePrepareStep(__FUNCDNAME__ , stmt, SQLITE_ROW); + if (! prepared_statement) + { + return false; + } + + int result_column_index = 0; + int result_count = sqlite3_column_int(prepared_statement, result_column_index); + if (sqlite3_finalize(prepared_statement) != SQLITE_OK) + { + printError(__FUNCDNAME__ , "sqlite3_finalize()", true); + return false; + } + + // given the uniqueness of the ID, this can only ever be + // either 1 or 0 so this might be a bit confusing + exists = result_count > 0 ? true : false; + + return true; +} + +/////////////////////////////////////////////////////////////////////////////// +// Given an id (likely a UUID + decoration but can be any string), a +// pointer to a blob of binary data along with its length, write the +// entry to the cache +// Returns true/false for success/failure +bool llDiskCache::put(const std::string id, + char* binary_data, + int binary_data_size) +{ + if (!mDb) + { + printError(__FUNCDNAME__ , "mDb is invalid", false); + return false; + } + + if (id.empty()) + { + printError(__FUNCDNAME__ , "id is empty", false); + return false; + } + + // < 0 is obvious but we assert the data must be at least 1 byte long + if (binary_data_size <= 0) + { + printError(__FUNCDNAME__ , "size of binary file to write is invalid", false); + return false; + } + + // we generate a unique filename for the actual data itself + // which is stored on disk directly and not in the database. + // TODO: consider making the filename more like the ID passed in + // although the problem with that is we would have to parse the id + // to remove invalid filename chars, consider length etc. As it + // stands, we can write a simple SQL statement to return the filename + // given the ID. + const std::string filename = makeUniqueFilename(); + const std::string filepath = makeFullPath(filename); + std::ofstream file(filepath, std::ios::out | std::ios::binary); + if (! file.is_open()) + { + std::ostringstream error; + error << "Unable to open " << filepath << " for writing"; + printError(__FUNCDNAME__ , error.str(), false); + return false; + } + + file.write((char*)binary_data, binary_data_size); + file.close(); + + // I think is a catchall "wasn't able to write the file to disk" + // conditional but should revisit when we hook this up to viewer + // code to make sure - we never want to write bad/no data to the + // disk and not indicate something has gone wrong + if (file.bad()) + { + std::ostringstream error; + error << "Unable to write " << binary_data_size; + error << " bytes to " << filepath; + printError(__FUNCDNAME__ , error.str(), false); + + return false; + } + + // this is where the filename/size is written to the database along + // with the current date/time for the created/last access times + const std::string stmt = sqlComposePut(id, filename, binary_data_size); + if (! sqlitePrepareStep(__FUNCDNAME__ , stmt, SQLITE_DONE)) + { + return false; + } + + return true; +} + +/////////////////////////////////////////////////////////////////////////////// +// Given an id (likely a UUID + decoration but can be any string), the +// address of a pointer that will be used to allocate memory and a +// varialble that will contain the length of the data, get an item from +// the cache. Note that the memory buffer returned belongs to the calling +// function and it is its responsiblity to clean it up when it's no +// longer needed. +// Returns true/false for success/failure +const bool llDiskCache::get(const std::string id, + char** mem_buffer, + int& mem_buffer_size) +{ + // Check if the entry exists first to avoid dealing with a bunch + // of conditions that look like failure but aren't in the main code. + // Note exists() is a public method and also tests for mDB and id + // being valid so we can safely put this about the same tests + // in this function + bool get_exists = false; + if (! exists(id, get_exists)) + { + return false; + } + if (! get_exists) + { + return false; + } + + if (!mDb) + { + printError(__FUNCDNAME__ , "mDb is invalid", false); + return false; + } + + if (id.empty()) + { + printError(__FUNCDNAME__ , "id is empty", false); + return false; + } + + const std::string stmt_select = sqlComposeGetSelect(id); + sqlite3_stmt* prepared_statement = sqlitePrepareStep(__FUNCDNAME__ , stmt_select, SQLITE_ROW); + if (! prepared_statement) + { + return false; + } + + int result_column_index = 0; + const unsigned char* text = sqlite3_column_text(prepared_statement, result_column_index); + if (text == nullptr) + { + printError(__FUNCDNAME__ , "filename is nullptr", true); + return false; + } + const std::string filename = std::string(reinterpret_cast(text)); + const std::string filepath = makeFullPath(filename); + + result_column_index = 1; + int filesize_db = sqlite3_column_int(prepared_statement, result_column_index); + if (filesize_db <= 0) + { + printError(__FUNCDNAME__ , "filesize is invalid", true); + return false; + } + + if (sqlite3_finalize(prepared_statement) != SQLITE_OK) + { + printError(__FUNCDNAME__ , "sqlite3_finalize()", true); + return false; + } + + // Now we have the fiename, we can read the file from disk + std::ifstream file(filepath, std::ios::in | std::ios::binary | std::ios::ate); + if (! file.is_open()) + { + std::ostringstream error; + error << "Unable to open " << filepath << " for reading"; + printError(__FUNCDNAME__ , error.str(), false); + return false; + } + + // we get the expected filesize from the database but we can also + // get it (easily) when we read the file from the disk. We compare + // the two and return false if they don't match + std::streampos filesize_file = file.tellg(); + if (filesize_db != filesize_file) + { + std::ostringstream error; + error << "File size from DB (" << filesize_db << ")"; + error << " and "; + error << "file size from file (" << filesize_file << ")"; + error << " in file " << filepath << " are different"; + printError(__FUNCDNAME__ , error.str(), false); + + return false; + } + + // doest matter if we choose DB or file version - they must be + // identical if we get this far - just used for clarity + int filesize = filesize_db; + + // allocate a block of memory that we pass back for the calling + // function to use then delete when it's no longer needed + *mem_buffer = new char[filesize]; + mem_buffer_size = filesize; + + file.seekg(0, std::ios::beg); + file.read(*mem_buffer, filesize); + file.close(); + + if (file.bad()) + { + std::ostringstream error; + error << "Unable to read " << filesize; + error << " bytes from " << filepath; + printError(__FUNCDNAME__ , error.str(), false); + + return false; + } + + // here we update the count of times the file is accessed so + // we can keep track of how many times it's been requested. + // This will be useful for metrics and perhaps determining + // if a file should not be purged even though its age + // might suggest that it should. + // In addition, this is where the time of last access is updated + // in the database and that us used to determine what is purged + // in an LRU fashion when the purge function is called. + const std::string stmt_update = sqlComposeGetUpdate(id); + if (! sqliteExec(stmt_update, __FUNCDNAME__ )) + { + return false; + } + + return true; +} + +/////////////////////////////////////////////////////////////////////////////// +// Purges the database of older entries using an LRU approach. +// Pass in the number of entries to retain. +// This is called now after open to "clean up" the cache when the +// application starts. +// TODO: IMPORTANT: compose a list of files that will be deleted +// and delete them from disk too - not just from the DB +bool llDiskCache::purge(int num_entries) +{ + if (num_entries < 0) + { + printError(__FUNCDNAME__ , "number of entries to purge is invalid", false); + return false; + } + + // find the rows affected and get the filenames for them +//swww + + // delete oldest entries leaving the correct number in place + const std::string stmt = sqlComposePurge(num_entries); + if (! sqliteExec(stmt, __FUNCDNAME__ )) + { + return false; + } + + return true; +} + +/////////////////////////////////////////////////////////////////////////////// +// Call at application shutdown +void llDiskCache::close() +{ + sqlite3_close(mDb); +} + +/////////////////////////////////////////////////////////////////////////////// +// Determine the version of SQLite in use +// TODO: make this a static so we can get to it from the Viewer About +// box without instantiating the whole thing. +const std::string llDiskCache::dbVersion() +{ + std::ostringstream version; + + version << sqlite3_libversion(); + + return version.str(); +} + +/////////////////////////////////////////////////////////////////////////////// +// Given an id, return the matching filename +const std::string llDiskCache::getFilenameById(const std::string id) +{ + // TODO: + return std::string(); +} + +/////////////////////////////////////////////////////////////////////////////// +// Given an id, return the number of times that entry has been +// accessed from the cache +const int llDiskCache::getAccessCountById(const std::string id) +{ + // TODO: + return -1; +} + +/////////////////////////////////////////////////////////////////////////////// +// Return the number of entries currently in the cache as well as +// the maximum possible entries. +void llDiskCache::getNumEntries(int& num_entries, int& max_entries) +{ + num_entries = -1; + max_entries = -1; +} + +/////////////////////////////////////////////////////////////////////////////// +// Wraps the sqlite3_exec(..) used in many places +bool llDiskCache::sqliteExec(const std::string stmt, + const std::string funcname) +{ + if (sqlite3_exec( + mDb, + stmt.c_str(), + nullptr, // Callback function (unused) + nullptr, // 1st argument to callback (unused) + nullptr // Error msg written here (unused) + ) != SQLITE_OK) + { + printError(funcname, "sqlite3_exec", true); + return false; + } + + return true; +} + +/////////////////////////////////////////////////////////////////////////////// +// Wraps the sqlite3_prepare_v2 and sqlite3_step calls used in many places +sqlite3_stmt* llDiskCache::sqlitePrepareStep(const std::string funcname, + const std::string stmt, + int sqlite_success_condition) +{ + sqlite3_stmt* prepared_statement; + + if (sqlite3_prepare_v2( + mDb, + stmt.c_str(), + -1, // Maximum length of zSql in bytes. + &prepared_statement, + 0 // OUT: Pointer to unused portion of zSql + ) != SQLITE_OK) + { + printError(funcname, "sqlite3_prepare_v2", true); + return nullptr; + } + + if (sqlite3_step(prepared_statement) != sqlite_success_condition) + { + printError(funcname, "sqlite3_step", true); + sqlite3_finalize(prepared_statement); + return nullptr; + } + return prepared_statement; +} + +/////////////////////////////////////////////////////////////////////////////// +// When an "error" occurss - e.g. database cannot be found, file cannot be +// written, invalid argument passed into a function etc. a message is +// written to stderr that should end up in the viewer log +// TODO: Set the verbosity using the usual Viewer mechanism +void llDiskCache::printError(const std::string funcname, + const std::string desc, + bool is_sqlite_err) +{ + std::ostringstream err_msg; + + err_msg << "llDiskCache error in "; + err_msg << __FUNCDNAME__ << "(...) "; + err_msg << desc; + + if (is_sqlite_err) + { + err_msg << " - "; + err_msg << std::string(sqlite3_errmsg(mDb)); + } + + // TODO: set via viewer verbosity level + const int verbosity = 1; + if (verbosity > 0) + { + std::cerr << err_msg.str() << std::endl; + } +} + +/////////////////////////////////////////////////////////////////////////////// +// wrapper for SQLite code to begin an SQL transaction - not used yet but +// it will be eventually +bool llDiskCache::beginTransaction() +{ + const std::string stmt("BEGIN TRANSACTION"); + if (! sqliteExec(stmt, __FUNCDNAME__ )) + { + return false; + } + + return true; +} + +/////////////////////////////////////////////////////////////////////////////// +// wrapper for SQLite code to end an SQL transaction - not used yet but +// it will be eventually +bool llDiskCache::endTransaction() +{ + const std::string stmt("COMMIT"); + if (! sqliteExec(stmt, __FUNCDNAME__ )) + { + return false; + } + + return true; +} + +/////////////////////////////////////////////////////////////////////////////// +// Build a unique filename that will be used to store the actual file +// on disk (as opposed to the meta data in the database) +// TODO: I think this needs more work once we move it to the viewer +// and espcially to make it cross platform - see 'std::tmpnam' +// depreciation comments in compiler output for example +std::string llDiskCache::makeUniqueFilename() +{ + // TODO: replace with boost - this is marked as deprecated? + std::string base = std::tmpnam(nullptr); + + // C++11 random number generation!!! + static std::random_device dev; + static std::mt19937 rng(dev()); + std::uniform_int_distribution dist(100000, 999999); + + // currently the tmp filename from std::tmpnam() on macOS + // is of the form `/tmp/foo/bar.12345 and the following code + // strips all the preceding dirs - we likely want a more + // robust (and cross platform solution) when we move to the + // viewer code + std::size_t found = base.rfind(systemSeparator()); + if (found != std::string::npos && found < base.size() - 2) + { + base = base.substr(found + 1); + } + else + { + base = ""; + } + + // we mix in a random number for some more entropy.. + // (i know, i know...) + std::ostringstream unique_filename; + unique_filename << base; + unique_filename << "."; + unique_filename << dist(rng); + + return unique_filename.str(); +} + +/////////////////////////////////////////////////////////////////////////////// +// Return system file/path separator - likely replaced by the version +// in the viewer +const std::string llDiskCache::systemSeparator() +{ +// TODO: replace in viewer with relevant call +#ifdef _WIN32 + return "\\"; +#else + return "/"; +#endif +} + +/////////////////////////////////////////////////////////////////////////////// +// Given a filename, compose a full path based on the path name passed +// in when the database was opened and the separator in play. +const std::string llDiskCache::makeFullPath(const std::string filename) +{ + std::string pathname = mDataStorePath + systemSeparator() + filename; + + return pathname; +} + +/////////////////////////////////////////////////////////////////////////////// +// +const std::string llDiskCache::sqlComposeCreateTable() +{ + std::ostringstream stmt; + stmt << "CREATE TABLE IF NOT EXISTS "; + stmt << mTableName; + stmt << "("; + stmt << mIdFieldName; + stmt << " TEXT PRIMARY KEY, "; + stmt << mFilenameFieldName; + stmt << " TEXT, "; + stmt << mFilesizeFieldName; + stmt << " INTEGER DEFAULT 0, "; + stmt << mInsertionDateTimeFieldName; + stmt << " TEXT, "; + stmt << mLastAccessDateTimeFieldName; + stmt << " TEXT, "; + stmt << mAccessCountFieldName; + stmt << " INTEGER DEFAULT 0"; + stmt << ")"; + +#ifdef SHOW_STATEMENTS + std::cout << stmt.str() << std::endl; +#endif + + return stmt.str(); +} + +/////////////////////////////////////////////////////////////////////////////// +// +const std::string llDiskCache::sqlComposeExists(const std::string id) +{ + std::ostringstream stmt; + stmt << "SELECT COUNT(*) FROM "; + stmt << mTableName; + stmt << " WHERE "; + stmt << mIdFieldName; + stmt << "='"; + stmt << id; + stmt << "'"; + +#ifdef SHOW_STATEMENTS + std::cout << stmt.str() << std::endl; +#endif + + return stmt.str(); +} + +/////////////////////////////////////////////////////////////////////////////// +// SQL statement to write an entry to the DB +// Saves id, filename (generated), file size and create/last access date +const std::string llDiskCache::sqlComposePut(const std::string id, + const std::string filename, + int binary_data_size) +{ + std::ostringstream stmt; + stmt << "REPLACE INTO "; + stmt << mTableName; + stmt << "("; + stmt << mIdFieldName; + stmt << ","; + stmt << mFilenameFieldName; + stmt << ","; + stmt << mFilesizeFieldName; + stmt << ","; + stmt << mInsertionDateTimeFieldName; + stmt << ","; + stmt << mLastAccessDateTimeFieldName; + stmt << ") "; + stmt << "VALUES("; + stmt << "'"; + stmt << id; + stmt << "', "; + stmt << "'"; + stmt << filename; + stmt << "', "; + stmt << binary_data_size; + stmt << ", "; + stmt << "strftime('%Y-%m-%d %H:%M:%f', 'now')"; + stmt << ", "; + stmt << "strftime('%Y-%m-%d %H:%M:%f', 'now')"; + stmt << ")"; + +#ifdef SHOW_STATEMENTS + std::cout << stmt.str() << std::endl; +#endif + + return stmt.str(); +} + +/////////////////////////////////////////////////////////////////////////////// +// SQL statement used in get() to look up the filename and file size +const std::string llDiskCache::sqlComposeGetSelect(const std::string id) +{ + std::ostringstream stmt; + stmt << "SELECT "; + stmt << mFilenameFieldName; + stmt << ", "; + stmt << mFilesizeFieldName; + stmt << " FROM "; + stmt << mTableName; + stmt << " WHERE "; + stmt << mIdFieldName; + stmt << "='"; + stmt << id; + stmt << "'"; + +#ifdef SHOW_STATEMENTS + std::cout << stmt.str() << std::endl; +#endif + + return stmt.str(); +} + +/////////////////////////////////////////////////////////////////////////////// +// SQL statement to update the date/time of last access as well as the +// count of number of times the file has been accessed. +// Note: the more accurate representation of date/time is used to +// ensure ms accuracy vs the standard INTEGER days since epoch approach +const std::string llDiskCache::sqlComposeGetUpdate(const std::string id) +{ + std::ostringstream stmt; + stmt << "UPDATE "; + stmt << mTableName; + stmt << " SET "; + stmt << mAccessCountFieldName; + stmt << "="; + stmt << mAccessCountFieldName; + stmt << "+1"; + stmt << ", "; + stmt << mLastAccessDateTimeFieldName; + stmt << "="; + stmt << "strftime('%Y-%m-%d %H:%M:%f', 'now')"; + stmt << " WHERE "; + stmt << mIdFieldName; + stmt << "='"; + stmt << id; + stmt << "'"; + +#ifdef SHOW_STATEMENTS + std::cout << stmt.str() << std::endl; +#endif + + return stmt.str(); +} + +/////////////////////////////////////////////////////////////////////////////// +// SQL statement to remove items from the database that are older +// than the newest num_elements entries +const std::string llDiskCache::sqlComposePurge(int num_entries) +{ + std::ostringstream stmt; + stmt << "DELETE FROM "; + stmt << mTableName; + stmt << " WHERE "; + stmt << mLastAccessDateTimeFieldName; + stmt << " NOT IN "; + stmt << "("; + stmt << "SELECT "; + stmt << mLastAccessDateTimeFieldName; + stmt << " FROM "; + stmt << mTableName; + stmt << " ORDER BY "; + stmt << mLastAccessDateTimeFieldName; + stmt << " DESC"; + stmt << " LIMIT "; + stmt << num_entries; + stmt << ")"; + +//#ifdef SHOW_STATEMENTS + std::cout << stmt.str() << std::endl; +//#endif + + return stmt.str(); +} -- cgit v1.2.3 From 3092aa8aae496803707980eb456cddbb9960ef1c Mon Sep 17 00:00:00 2001 From: Callum Prentice Date: Tue, 6 Oct 2020 17:16:53 -0700 Subject: Add in the C++ filesystem based cache and clean up some indempotent functions in llfilesystem --- indra/llfilesystem/lldiskcache.cpp | 809 ++++++------------------------------- 1 file changed, 113 insertions(+), 696 deletions(-) (limited to 'indra/llfilesystem/lldiskcache.cpp') diff --git a/indra/llfilesystem/lldiskcache.cpp b/indra/llfilesystem/lldiskcache.cpp index 72982db069..4b2ba0dd79 100644 --- a/indra/llfilesystem/lldiskcache.cpp +++ b/indra/llfilesystem/lldiskcache.cpp @@ -1,9 +1,6 @@ /** * @file lldiskcache.cpp - * @brief The SQLite based disk cache implementation. - * @Note Eventually, this component might split into an interface - * file and multiple implemtations but for now, this is the - * only one so I think it's okay to combine everything. + * @brief The disk cache implementation. * * $LicenseInfo:firstyear=2009&license=viewerlgpl$ * Second Life Viewer Source Code @@ -27,747 +24,167 @@ * $/LicenseInfo$ */ -#if (defined(LL_WINDOWS) || defined(LL_LINUX) || defined(LL_DARWIN)) #include "linden_common.h" -#endif +#include "lluuid.h" +#include "lldir.h" #include "lldiskcache.h" -#include -#include -#include -#include -#include - -/////////////////////////////////////////////////////////////////////////////// -// -llDiskCache::llDiskCache() : - mDataStorePath("") -{ -} +#include +#include -llDiskCache::~llDiskCache() -{ -} +#include +#include +#include -/////////////////////////////////////////////////////////////////////////////// -// Opens the database - typically done when the application starts -// and is complementary to close() which is called when the application -// is finisahed and exits. -// Pass in the folder and filename of the SQLite database you want to -// use or create (file doesn't have to exist but the folder must) -// Returns true or false and writes a message to console on error -bool llDiskCache::open(const std::string db_folder, const std::string db_filename) +LLDiskCache::LLDiskCache(const std::string cache_dir) : + mCacheDir(cache_dir), + mMaxSizeBytes(mDefaultSizeBytes) { - mDataStorePath = db_folder; - std::string db_pathname = makeFullPath(db_filename); - - // simple flags for the moment - these will likely be extended - // later on to support the SQLite mutex model for reading/writing - // simultaneously - perhaps when we look at supporting textures too - int flags = SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE; - - if (sqlite3_open_v2( - db_pathname.c_str(), - &mDb, - flags, - nullptr // Name of VFS module to use - ) != SQLITE_OK) - { - printError(__FUNCDNAME__ , "open_v2", true); - close(); - return false; - } - - // I elected to store the code that generates the statement - // in sepsrate functions throughout - this seemed like a cleaner - // approach than having hundreds of stmt << "REPLACE AS" lines - // interspersed in the logic code. They are all prefixed with - // 'sqlCompose' and followed by a short description. - const std::string stmt = sqlComposeCreateTable(); - if (! sqliteExec(stmt, __FUNCDNAME__ )) + // no access to LLControlGroup / gSavedSettings so use the system environment + // to trigger additional debugging on top of the default "we purged the cache" + mCacheDebugInfo = true; + if (getenv("LL_CACHE_DEBUGGING") != nullptr) { - // Not sure if we need close here - if open fails, then we - // perhaps don't need to close it - TODO: look in SQLite docs - close(); - return false; + mCacheDebugInfo = true; } - return true; + // create cache dir if it does not exist + boost::filesystem::create_directory(cache_dir); } -/////////////////////////////////////////////////////////////////////////////// -// Determines if an entry exists. -// Pass in the id and a variable that will indicate existance -// Returns true/false if function succeeded and the boolean -// you pass in will be set to true/false if entry exists or not -bool llDiskCache::exists(const std::string id, bool& exists) +LLDiskCache::~LLDiskCache() { - if (!mDb) - { - printError(__FUNCDNAME__ , "mDb is invalid", false); - return false; - } - - if (id.empty()) - { - printError(__FUNCDNAME__ , "id is empty", false); - return false; - } - - // As well as the separate function to compose the SQL statement, - // worth pointing out that the code to 'prepare' and 'step' the - // SQLite "prepared statement" has been factored out into its own - // function and used in several other functions. - const std::string stmt = sqlComposeExists(id); - sqlite3_stmt* prepared_statement = sqlitePrepareStep(__FUNCDNAME__ , stmt, SQLITE_ROW); - if (! prepared_statement) - { - return false; - } - - int result_column_index = 0; - int result_count = sqlite3_column_int(prepared_statement, result_column_index); - if (sqlite3_finalize(prepared_statement) != SQLITE_OK) - { - printError(__FUNCDNAME__ , "sqlite3_finalize()", true); - return false; - } - - // given the uniqueness of the ID, this can only ever be - // either 1 or 0 so this might be a bit confusing - exists = result_count > 0 ? true : false; - - return true; -} - -/////////////////////////////////////////////////////////////////////////////// -// Given an id (likely a UUID + decoration but can be any string), a -// pointer to a blob of binary data along with its length, write the -// entry to the cache -// Returns true/false for success/failure -bool llDiskCache::put(const std::string id, - char* binary_data, - int binary_data_size) -{ - if (!mDb) - { - printError(__FUNCDNAME__ , "mDb is invalid", false); - return false; - } - - if (id.empty()) - { - printError(__FUNCDNAME__ , "id is empty", false); - return false; - } - - // < 0 is obvious but we assert the data must be at least 1 byte long - if (binary_data_size <= 0) - { - printError(__FUNCDNAME__ , "size of binary file to write is invalid", false); - return false; - } - - // we generate a unique filename for the actual data itself - // which is stored on disk directly and not in the database. - // TODO: consider making the filename more like the ID passed in - // although the problem with that is we would have to parse the id - // to remove invalid filename chars, consider length etc. As it - // stands, we can write a simple SQL statement to return the filename - // given the ID. - const std::string filename = makeUniqueFilename(); - const std::string filepath = makeFullPath(filename); - std::ofstream file(filepath, std::ios::out | std::ios::binary); - if (! file.is_open()) - { - std::ostringstream error; - error << "Unable to open " << filepath << " for writing"; - printError(__FUNCDNAME__ , error.str(), false); - return false; - } - - file.write((char*)binary_data, binary_data_size); - file.close(); - - // I think is a catchall "wasn't able to write the file to disk" - // conditional but should revisit when we hook this up to viewer - // code to make sure - we never want to write bad/no data to the - // disk and not indicate something has gone wrong - if (file.bad()) - { - std::ostringstream error; - error << "Unable to write " << binary_data_size; - error << " bytes to " << filepath; - printError(__FUNCDNAME__ , error.str(), false); - - return false; - } - - // this is where the filename/size is written to the database along - // with the current date/time for the created/last access times - const std::string stmt = sqlComposePut(id, filename, binary_data_size); - if (! sqlitePrepareStep(__FUNCDNAME__ , stmt, SQLITE_DONE)) - { - return false; - } - - return true; } -/////////////////////////////////////////////////////////////////////////////// -// Given an id (likely a UUID + decoration but can be any string), the -// address of a pointer that will be used to allocate memory and a -// varialble that will contain the length of the data, get an item from -// the cache. Note that the memory buffer returned belongs to the calling -// function and it is its responsiblity to clean it up when it's no -// longer needed. -// Returns true/false for success/failure -const bool llDiskCache::get(const std::string id, - char** mem_buffer, - int& mem_buffer_size) +void LLDiskCache::purge() { - // Check if the entry exists first to avoid dealing with a bunch - // of conditions that look like failure but aren't in the main code. - // Note exists() is a public method and also tests for mDB and id - // being valid so we can safely put this about the same tests - // in this function - bool get_exists = false; - if (! exists(id, get_exists)) + if (mCacheDebugInfo) { - return false; - } - if (! get_exists) - { - return false; + LL_INFOS() << "Total dir size before purge is " << dirFileSize(mCacheDir) << LL_ENDL; } - if (!mDb) - { - printError(__FUNCDNAME__ , "mDb is invalid", false); - return false; - } - - if (id.empty()) - { - printError(__FUNCDNAME__ , "id is empty", false); - return false; - } + auto start_time = std::chrono::high_resolution_clock::now(); - const std::string stmt_select = sqlComposeGetSelect(id); - sqlite3_stmt* prepared_statement = sqlitePrepareStep(__FUNCDNAME__ , stmt_select, SQLITE_ROW); - if (! prepared_statement) - { - return false; - } + typedef std::pair> file_info_t; + std::vector file_info; - int result_column_index = 0; - const unsigned char* text = sqlite3_column_text(prepared_statement, result_column_index); - if (text == nullptr) + if (boost::filesystem::is_directory(mCacheDir)) { - printError(__FUNCDNAME__ , "filename is nullptr", true); - return false; - } - const std::string filename = std::string(reinterpret_cast(text)); - const std::string filepath = makeFullPath(filename); + for (auto& entry : boost::make_iterator_range(boost::filesystem::directory_iterator(mCacheDir), {})) + { + if (boost::filesystem::is_regular_file(entry)) + { + uintmax_t file_size = boost::filesystem::file_size(entry); + const std::string file_path = entry.path().string(); + const std::time_t file_time = boost::filesystem::last_write_time(entry); - result_column_index = 1; - int filesize_db = sqlite3_column_int(prepared_statement, result_column_index); - if (filesize_db <= 0) - { - printError(__FUNCDNAME__ , "filesize is invalid", true); - return false; + file_info.push_back(file_info_t(file_time, { file_size, file_path })); + } + } } - if (sqlite3_finalize(prepared_statement) != SQLITE_OK) + std::sort(file_info.begin(), file_info.end(), [](file_info_t& x, file_info_t& y) { - printError(__FUNCDNAME__ , "sqlite3_finalize()", true); - return false; - } + return x.first > y.first; + }); - // Now we have the fiename, we can read the file from disk - std::ifstream file(filepath, std::ios::in | std::ios::binary | std::ios::ate); - if (! file.is_open()) - { - std::ostringstream error; - error << "Unable to open " << filepath << " for reading"; - printError(__FUNCDNAME__ , error.str(), false); - return false; - } + LL_INFOS() << "Purging cache to a maximum of " << mMaxSizeBytes << " bytes" << LL_ENDL; - // we get the expected filesize from the database but we can also - // get it (easily) when we read the file from the disk. We compare - // the two and return false if they don't match - std::streampos filesize_file = file.tellg(); - if (filesize_db != filesize_file) + uintmax_t file_size_total = 0; + for (file_info_t& entry : file_info) { - std::ostringstream error; - error << "File size from DB (" << filesize_db << ")"; - error << " and "; - error << "file size from file (" << filesize_file << ")"; - error << " in file " << filepath << " are different"; - printError(__FUNCDNAME__ , error.str(), false); - - return false; - } - - // doest matter if we choose DB or file version - they must be - // identical if we get this far - just used for clarity - int filesize = filesize_db; + file_size_total += entry.second.first; - // allocate a block of memory that we pass back for the calling - // function to use then delete when it's no longer needed - *mem_buffer = new char[filesize]; - mem_buffer_size = filesize; + std::string action = ""; + if (file_size_total > mMaxSizeBytes) + { + action = "DELETE:"; + boost::filesystem::remove(entry.second.second); + } + else + { + action = " KEEP:"; + } - file.seekg(0, std::ios::beg); - file.read(*mem_buffer, filesize); - file.close(); + if (mCacheDebugInfo) + { + // have to do this because of LL_INFO/LL_END weirdness + std::ostringstream line; - if (file.bad()) - { - std::ostringstream error; - error << "Unable to read " << filesize; - error << " bytes from " << filepath; - printError(__FUNCDNAME__ , error.str(), false); - - return false; + line << action << " "; + line << entry.first << " "; + line << entry.second.first << " "; + line << entry.second.second; + line << " (" << file_size_total << "/" << mMaxSizeBytes << ")"; + LL_INFOS() << line.str() << LL_ENDL; + } } - // here we update the count of times the file is accessed so - // we can keep track of how many times it's been requested. - // This will be useful for metrics and perhaps determining - // if a file should not be purged even though its age - // might suggest that it should. - // In addition, this is where the time of last access is updated - // in the database and that us used to determine what is purged - // in an LRU fashion when the purge function is called. - const std::string stmt_update = sqlComposeGetUpdate(id); - if (! sqliteExec(stmt_update, __FUNCDNAME__ )) + if (mCacheDebugInfo) { - return false; + auto end_time = std::chrono::high_resolution_clock::now(); + auto execute_time = std::chrono::duration_cast(end_time - start_time).count(); + LL_INFOS() << "Total dir size after purge is " << dirFileSize(mCacheDir) << LL_ENDL; + LL_INFOS() << "Cache purge took " << execute_time << " ms to execute for " << file_info.size() << " files" << LL_ENDL; } - - return true; -} - -/////////////////////////////////////////////////////////////////////////////// -// Purges the database of older entries using an LRU approach. -// Pass in the number of entries to retain. -// This is called now after open to "clean up" the cache when the -// application starts. -// TODO: IMPORTANT: compose a list of files that will be deleted -// and delete them from disk too - not just from the DB -bool llDiskCache::purge(int num_entries) -{ - if (num_entries < 0) - { - printError(__FUNCDNAME__ , "number of entries to purge is invalid", false); - return false; - } - - // find the rows affected and get the filenames for them -//swww - - // delete oldest entries leaving the correct number in place - const std::string stmt = sqlComposePurge(num_entries); - if (! sqliteExec(stmt, __FUNCDNAME__ )) - { - return false; - } - - return true; -} - -/////////////////////////////////////////////////////////////////////////////// -// Call at application shutdown -void llDiskCache::close() -{ - sqlite3_close(mDb); -} - -/////////////////////////////////////////////////////////////////////////////// -// Determine the version of SQLite in use -// TODO: make this a static so we can get to it from the Viewer About -// box without instantiating the whole thing. -const std::string llDiskCache::dbVersion() -{ - std::ostringstream version; - - version << sqlite3_libversion(); - - return version.str(); } -/////////////////////////////////////////////////////////////////////////////// -// Given an id, return the matching filename -const std::string llDiskCache::getFilenameById(const std::string id) -{ - // TODO: - return std::string(); -} - -/////////////////////////////////////////////////////////////////////////////// -// Given an id, return the number of times that entry has been -// accessed from the cache -const int llDiskCache::getAccessCountById(const std::string id) -{ - // TODO: - return -1; -} - -/////////////////////////////////////////////////////////////////////////////// -// Return the number of entries currently in the cache as well as -// the maximum possible entries. -void llDiskCache::getNumEntries(int& num_entries, int& max_entries) -{ - num_entries = -1; - max_entries = -1; -} - -/////////////////////////////////////////////////////////////////////////////// -// Wraps the sqlite3_exec(..) used in many places -bool llDiskCache::sqliteExec(const std::string stmt, - const std::string funcname) -{ - if (sqlite3_exec( - mDb, - stmt.c_str(), - nullptr, // Callback function (unused) - nullptr, // 1st argument to callback (unused) - nullptr // Error msg written here (unused) - ) != SQLITE_OK) - { - printError(funcname, "sqlite3_exec", true); - return false; - } - - return true; -} - -/////////////////////////////////////////////////////////////////////////////// -// Wraps the sqlite3_prepare_v2 and sqlite3_step calls used in many places -sqlite3_stmt* llDiskCache::sqlitePrepareStep(const std::string funcname, - const std::string stmt, - int sqlite_success_condition) -{ - sqlite3_stmt* prepared_statement; - - if (sqlite3_prepare_v2( - mDb, - stmt.c_str(), - -1, // Maximum length of zSql in bytes. - &prepared_statement, - 0 // OUT: Pointer to unused portion of zSql - ) != SQLITE_OK) - { - printError(funcname, "sqlite3_prepare_v2", true); - return nullptr; - } - - if (sqlite3_step(prepared_statement) != sqlite_success_condition) - { - printError(funcname, "sqlite3_step", true); - sqlite3_finalize(prepared_statement); - return nullptr; - } - return prepared_statement; -} - -/////////////////////////////////////////////////////////////////////////////// -// When an "error" occurss - e.g. database cannot be found, file cannot be -// written, invalid argument passed into a function etc. a message is -// written to stderr that should end up in the viewer log -// TODO: Set the verbosity using the usual Viewer mechanism -void llDiskCache::printError(const std::string funcname, - const std::string desc, - bool is_sqlite_err) -{ - std::ostringstream err_msg; - - err_msg << "llDiskCache error in "; - err_msg << __FUNCDNAME__ << "(...) "; - err_msg << desc; - - if (is_sqlite_err) - { - err_msg << " - "; - err_msg << std::string(sqlite3_errmsg(mDb)); - } - - // TODO: set via viewer verbosity level - const int verbosity = 1; - if (verbosity > 0) - { - std::cerr << err_msg.str() << std::endl; - } -} - -/////////////////////////////////////////////////////////////////////////////// -// wrapper for SQLite code to begin an SQL transaction - not used yet but -// it will be eventually -bool llDiskCache::beginTransaction() +/** + * Update the "last write time" of a file to "now". This must be called whenever a + * file in the cache is read (not written) so that the last time the file was + * accessed which is used in the mechanism for purging the cache, is up to date. + */ +void LLDiskCache::updateFileAccessTime(const std::string file_path) { - const std::string stmt("BEGIN TRANSACTION"); - if (! sqliteExec(stmt, __FUNCDNAME__ )) - { - return false; - } - - return true; + const std::time_t file_time = std::time(nullptr); + boost::filesystem::last_write_time(file_path, file_time); } -/////////////////////////////////////////////////////////////////////////////// -// wrapper for SQLite code to end an SQL transaction - not used yet but -// it will be eventually -bool llDiskCache::endTransaction() +/** + * Clear the cache by removing all the files in the cache directory + * individually. It's important to maintain control of which directory + * if passed in and not let the user inadvertently (or maliciously) set + * it to an random location like your project source or OS system directory + */ +void LLDiskCache::clearCache(const std::string cache_dir) { - const std::string stmt("COMMIT"); - if (! sqliteExec(stmt, __FUNCDNAME__ )) + if (boost::filesystem::is_directory(cache_dir)) { - return false; + for (auto& entry : boost::make_iterator_range(boost::filesystem::directory_iterator(cache_dir), {})) + { + if (boost::filesystem::is_regular_file(entry)) + { + boost::filesystem::remove(entry); + } + } } - - return true; } -/////////////////////////////////////////////////////////////////////////////// -// Build a unique filename that will be used to store the actual file -// on disk (as opposed to the meta data in the database) -// TODO: I think this needs more work once we move it to the viewer -// and espcially to make it cross platform - see 'std::tmpnam' -// depreciation comments in compiler output for example -std::string llDiskCache::makeUniqueFilename() +/** + * Utility function to get the total filesize of all files in a directory. It + * used to test file extensions to only check cache files but that was removed. + * There may be a better way that works directly on the folder (similar to + * right clicking on a folder in the OS and asking for size vs right clicking + * on all files and adding up manually) but this is very fast - less than 100ms + * in my testing so, so long as it's not called frequently, it should be okay. + * Note that's it's only currently used for logging/debugging so if performance + * is ever an issue, optimizing this or removing it altogether, is an easy win. + */ +uintmax_t LLDiskCache::dirFileSize(const std::string dir) { - // TODO: replace with boost - this is marked as deprecated? - std::string base = std::tmpnam(nullptr); - - // C++11 random number generation!!! - static std::random_device dev; - static std::mt19937 rng(dev()); - std::uniform_int_distribution dist(100000, 999999); + uintmax_t total_file_size = 0; - // currently the tmp filename from std::tmpnam() on macOS - // is of the form `/tmp/foo/bar.12345 and the following code - // strips all the preceding dirs - we likely want a more - // robust (and cross platform solution) when we move to the - // viewer code - std::size_t found = base.rfind(systemSeparator()); - if (found != std::string::npos && found < base.size() - 2) - { - base = base.substr(found + 1); - } - else + if (boost::filesystem::is_directory(dir)) { - base = ""; + for (auto& entry : boost::make_iterator_range(boost::filesystem::directory_iterator(dir), {})) + { + if (boost::filesystem::is_regular_file(entry)) + { + total_file_size += boost::filesystem::file_size(entry); + } + } } - // we mix in a random number for some more entropy.. - // (i know, i know...) - std::ostringstream unique_filename; - unique_filename << base; - unique_filename << "."; - unique_filename << dist(rng); - - return unique_filename.str(); -} - -/////////////////////////////////////////////////////////////////////////////// -// Return system file/path separator - likely replaced by the version -// in the viewer -const std::string llDiskCache::systemSeparator() -{ -// TODO: replace in viewer with relevant call -#ifdef _WIN32 - return "\\"; -#else - return "/"; -#endif -} - -/////////////////////////////////////////////////////////////////////////////// -// Given a filename, compose a full path based on the path name passed -// in when the database was opened and the separator in play. -const std::string llDiskCache::makeFullPath(const std::string filename) -{ - std::string pathname = mDataStorePath + systemSeparator() + filename; - - return pathname; -} - -/////////////////////////////////////////////////////////////////////////////// -// -const std::string llDiskCache::sqlComposeCreateTable() -{ - std::ostringstream stmt; - stmt << "CREATE TABLE IF NOT EXISTS "; - stmt << mTableName; - stmt << "("; - stmt << mIdFieldName; - stmt << " TEXT PRIMARY KEY, "; - stmt << mFilenameFieldName; - stmt << " TEXT, "; - stmt << mFilesizeFieldName; - stmt << " INTEGER DEFAULT 0, "; - stmt << mInsertionDateTimeFieldName; - stmt << " TEXT, "; - stmt << mLastAccessDateTimeFieldName; - stmt << " TEXT, "; - stmt << mAccessCountFieldName; - stmt << " INTEGER DEFAULT 0"; - stmt << ")"; - -#ifdef SHOW_STATEMENTS - std::cout << stmt.str() << std::endl; -#endif - - return stmt.str(); -} - -/////////////////////////////////////////////////////////////////////////////// -// -const std::string llDiskCache::sqlComposeExists(const std::string id) -{ - std::ostringstream stmt; - stmt << "SELECT COUNT(*) FROM "; - stmt << mTableName; - stmt << " WHERE "; - stmt << mIdFieldName; - stmt << "='"; - stmt << id; - stmt << "'"; - -#ifdef SHOW_STATEMENTS - std::cout << stmt.str() << std::endl; -#endif - - return stmt.str(); -} - -/////////////////////////////////////////////////////////////////////////////// -// SQL statement to write an entry to the DB -// Saves id, filename (generated), file size and create/last access date -const std::string llDiskCache::sqlComposePut(const std::string id, - const std::string filename, - int binary_data_size) -{ - std::ostringstream stmt; - stmt << "REPLACE INTO "; - stmt << mTableName; - stmt << "("; - stmt << mIdFieldName; - stmt << ","; - stmt << mFilenameFieldName; - stmt << ","; - stmt << mFilesizeFieldName; - stmt << ","; - stmt << mInsertionDateTimeFieldName; - stmt << ","; - stmt << mLastAccessDateTimeFieldName; - stmt << ") "; - stmt << "VALUES("; - stmt << "'"; - stmt << id; - stmt << "', "; - stmt << "'"; - stmt << filename; - stmt << "', "; - stmt << binary_data_size; - stmt << ", "; - stmt << "strftime('%Y-%m-%d %H:%M:%f', 'now')"; - stmt << ", "; - stmt << "strftime('%Y-%m-%d %H:%M:%f', 'now')"; - stmt << ")"; - -#ifdef SHOW_STATEMENTS - std::cout << stmt.str() << std::endl; -#endif - - return stmt.str(); -} - -/////////////////////////////////////////////////////////////////////////////// -// SQL statement used in get() to look up the filename and file size -const std::string llDiskCache::sqlComposeGetSelect(const std::string id) -{ - std::ostringstream stmt; - stmt << "SELECT "; - stmt << mFilenameFieldName; - stmt << ", "; - stmt << mFilesizeFieldName; - stmt << " FROM "; - stmt << mTableName; - stmt << " WHERE "; - stmt << mIdFieldName; - stmt << "='"; - stmt << id; - stmt << "'"; - -#ifdef SHOW_STATEMENTS - std::cout << stmt.str() << std::endl; -#endif - - return stmt.str(); -} - -/////////////////////////////////////////////////////////////////////////////// -// SQL statement to update the date/time of last access as well as the -// count of number of times the file has been accessed. -// Note: the more accurate representation of date/time is used to -// ensure ms accuracy vs the standard INTEGER days since epoch approach -const std::string llDiskCache::sqlComposeGetUpdate(const std::string id) -{ - std::ostringstream stmt; - stmt << "UPDATE "; - stmt << mTableName; - stmt << " SET "; - stmt << mAccessCountFieldName; - stmt << "="; - stmt << mAccessCountFieldName; - stmt << "+1"; - stmt << ", "; - stmt << mLastAccessDateTimeFieldName; - stmt << "="; - stmt << "strftime('%Y-%m-%d %H:%M:%f', 'now')"; - stmt << " WHERE "; - stmt << mIdFieldName; - stmt << "='"; - stmt << id; - stmt << "'"; - -#ifdef SHOW_STATEMENTS - std::cout << stmt.str() << std::endl; -#endif - - return stmt.str(); -} - -/////////////////////////////////////////////////////////////////////////////// -// SQL statement to remove items from the database that are older -// than the newest num_elements entries -const std::string llDiskCache::sqlComposePurge(int num_entries) -{ - std::ostringstream stmt; - stmt << "DELETE FROM "; - stmt << mTableName; - stmt << " WHERE "; - stmt << mLastAccessDateTimeFieldName; - stmt << " NOT IN "; - stmt << "("; - stmt << "SELECT "; - stmt << mLastAccessDateTimeFieldName; - stmt << " FROM "; - stmt << mTableName; - stmt << " ORDER BY "; - stmt << mLastAccessDateTimeFieldName; - stmt << " DESC"; - stmt << " LIMIT "; - stmt << num_entries; - stmt << ")"; - -//#ifdef SHOW_STATEMENTS - std::cout << stmt.str() << std::endl; -//#endif - - return stmt.str(); + return total_file_size; } -- cgit v1.2.3 From 08dfc0836fb12855d0c07d811e2909400d5b74f3 Mon Sep 17 00:00:00 2001 From: Callum Prentice Date: Wed, 7 Oct 2020 15:25:12 -0700 Subject: This changeset hooks up many things that have been in progress and moves things about between llfilesystem and lldiskcache - there is still some bookkeeping work left but this is the first version that appears to work and actively manage the cache --- indra/llfilesystem/lldiskcache.cpp | 134 ++++++++++++++++++++++++++++--------- 1 file changed, 101 insertions(+), 33 deletions(-) (limited to 'indra/llfilesystem/lldiskcache.cpp') diff --git a/indra/llfilesystem/lldiskcache.cpp b/indra/llfilesystem/lldiskcache.cpp index 4b2ba0dd79..4b82cf3cce 100644 --- a/indra/llfilesystem/lldiskcache.cpp +++ b/indra/llfilesystem/lldiskcache.cpp @@ -25,41 +25,31 @@ */ #include "linden_common.h" -#include "lluuid.h" +#include "llassettype.h" #include "lldir.h" - -#include "lldiskcache.h" - #include #include - #include -#include -#include -LLDiskCache::LLDiskCache(const std::string cache_dir) : +#include "lldiskcache.h" + +LLDiskCache::LLDiskCache(const std::string cache_dir, + const int max_size_bytes, + const bool enable_cache_debug_info) : mCacheDir(cache_dir), - mMaxSizeBytes(mDefaultSizeBytes) + mMaxSizeBytes(max_size_bytes), + mEnableCacheDebugInfo(enable_cache_debug_info) { - // no access to LLControlGroup / gSavedSettings so use the system environment - // to trigger additional debugging on top of the default "we purged the cache" - mCacheDebugInfo = true; - if (getenv("LL_CACHE_DEBUGGING") != nullptr) - { - mCacheDebugInfo = true; - } + // the prefix used for cache filenames to disambiguate them from other files + mCacheFilenamePrefix = "sl_cache"; // create cache dir if it does not exist boost::filesystem::create_directory(cache_dir); } -LLDiskCache::~LLDiskCache() -{ -} - void LLDiskCache::purge() { - if (mCacheDebugInfo) + if (mEnableCacheDebugInfo) { LL_INFOS() << "Total dir size before purge is " << dirFileSize(mCacheDir) << LL_ENDL; } @@ -75,11 +65,14 @@ void LLDiskCache::purge() { if (boost::filesystem::is_regular_file(entry)) { - uintmax_t file_size = boost::filesystem::file_size(entry); - const std::string file_path = entry.path().string(); - const std::time_t file_time = boost::filesystem::last_write_time(entry); + if (entry.path().string().find(mCacheFilenamePrefix) != std::string::npos) + { + uintmax_t file_size = boost::filesystem::file_size(entry); + const std::string file_path = entry.path().string(); + const std::time_t file_time = boost::filesystem::last_write_time(entry); - file_info.push_back(file_info_t(file_time, { file_size, file_path })); + file_info.push_back(file_info_t(file_time, { file_size, file_path })); + } } } } @@ -107,7 +100,7 @@ void LLDiskCache::purge() action = " KEEP:"; } - if (mCacheDebugInfo) + if (mEnableCacheDebugInfo) { // have to do this because of LL_INFO/LL_END weirdness std::ostringstream line; @@ -121,7 +114,7 @@ void LLDiskCache::purge() } } - if (mCacheDebugInfo) + if (mEnableCacheDebugInfo) { auto end_time = std::chrono::high_resolution_clock::now(); auto execute_time = std::chrono::duration_cast(end_time - start_time).count(); @@ -130,9 +123,78 @@ void LLDiskCache::purge() } } +const std::string LLDiskCache::assetTypeToString(LLAssetType::EType at) +{ + /** + * Make use of the C++17 (or is it 14) feature that allows + * for inline initialization of an std::map<> + */ + typedef std::map asset_type_to_name_t; + asset_type_to_name_t asset_type_to_name = + { + { LLAssetType::AT_TEXTURE, "TEXTURE" }, + { LLAssetType::AT_SOUND, "SOUND" }, + { LLAssetType::AT_CALLINGCARD, "CALLINGCARD" }, + { LLAssetType::AT_LANDMARK, "LANDMARK" }, + { LLAssetType::AT_SCRIPT, "SCRIPT" }, + { LLAssetType::AT_CLOTHING, "CLOTHING" }, + { LLAssetType::AT_OBJECT, "OBJECT" }, + { LLAssetType::AT_NOTECARD, "NOTECARD" }, + { LLAssetType::AT_CATEGORY, "CATEGORY" }, + { LLAssetType::AT_LSL_TEXT, "LSL_TEXT" }, + { LLAssetType::AT_LSL_BYTECODE, "LSL_BYTECODE" }, + { LLAssetType::AT_TEXTURE_TGA, "TEXTURE_TGA" }, + { LLAssetType::AT_BODYPART, "BODYPART" }, + { LLAssetType::AT_SOUND_WAV, "SOUND_WAV" }, + { LLAssetType::AT_IMAGE_TGA, "IMAGE_TGA" }, + { LLAssetType::AT_IMAGE_JPEG, "IMAGE_JPEG" }, + { LLAssetType::AT_ANIMATION, "ANIMATION" }, + { LLAssetType::AT_GESTURE, "GESTURE" }, + { LLAssetType::AT_SIMSTATE, "SIMSTATE" }, + { LLAssetType::AT_LINK, "LINK" }, + { LLAssetType::AT_LINK_FOLDER, "LINK_FOLDER" }, + { LLAssetType::AT_MARKETPLACE_FOLDER, "MARKETPLACE_FOLDER" }, + { LLAssetType::AT_WIDGET, "WIDGET" }, + { LLAssetType::AT_PERSON, "PERSON" }, + { LLAssetType::AT_MESH, "MESH" }, + { LLAssetType::AT_SETTINGS, "SETTINGS" }, + { LLAssetType::AT_UNKNOWN, "UNKNOWN" } + }; + + asset_type_to_name_t::iterator iter = asset_type_to_name.find(at); + if (iter != asset_type_to_name.end()) + { + return iter->second; + } + + return std::string("UNKNOWN"); +} + +const std::string LLDiskCache::metaDataToFilepath(const std::string id, + LLAssetType::EType at, + const std::string extra_info) +{ + std::ostringstream file_path; + + file_path << mCacheDir; + file_path << gDirUtilp->getDirDelimiter(); + file_path << mCacheFilenamePrefix; + file_path << "_"; + file_path << id; + file_path << "_"; + file_path << (extra_info.empty() ? "0" : extra_info); + file_path << "_"; + file_path << assetTypeToString(at); + file_path << ".asset"; + + LL_INFOS() << "filepath.str() = " << file_path.str() << LL_ENDL; + + return file_path.str(); +} + /** - * Update the "last write time" of a file to "now". This must be called whenever a - * file in the cache is read (not written) so that the last time the file was + * Update the "last write time" of a file to "now". This must be called whenever a + * file in the cache is read (not written) so that the last time the file was * accessed which is used in the mechanism for purging the cache, is up to date. */ void LLDiskCache::updateFileAccessTime(const std::string file_path) @@ -144,8 +206,8 @@ void LLDiskCache::updateFileAccessTime(const std::string file_path) /** * Clear the cache by removing all the files in the cache directory * individually. It's important to maintain control of which directory - * if passed in and not let the user inadvertently (or maliciously) set - * it to an random location like your project source or OS system directory + * if passed in and not let the user inadvertently (or maliciously) set + * it to an random location like your project source or OS system directory */ void LLDiskCache::clearCache(const std::string cache_dir) { @@ -155,7 +217,10 @@ void LLDiskCache::clearCache(const std::string cache_dir) { if (boost::filesystem::is_regular_file(entry)) { - boost::filesystem::remove(entry); + if (entry.path().string().find(mCacheFilenamePrefix) != std::string::npos) + { + boost::filesystem::remove(entry); + } } } } @@ -181,7 +246,10 @@ uintmax_t LLDiskCache::dirFileSize(const std::string dir) { if (boost::filesystem::is_regular_file(entry)) { - total_file_size += boost::filesystem::file_size(entry); + if (entry.path().string().find(mCacheFilenamePrefix) != std::string::npos) + { + total_file_size += boost::filesystem::file_size(entry); + } } } } -- cgit v1.2.3 From 5fdc4a1fb4a1788b20ca7a2e7764fd1391c35785 Mon Sep 17 00:00:00 2001 From: Callum Prentice Date: Wed, 7 Oct 2020 16:03:28 -0700 Subject: Add in some cache stats for the about box --- indra/llfilesystem/lldiskcache.cpp | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) (limited to 'indra/llfilesystem/lldiskcache.cpp') diff --git a/indra/llfilesystem/lldiskcache.cpp b/indra/llfilesystem/lldiskcache.cpp index 4b82cf3cce..455e27221e 100644 --- a/indra/llfilesystem/lldiskcache.cpp +++ b/indra/llfilesystem/lldiskcache.cpp @@ -187,8 +187,6 @@ const std::string LLDiskCache::metaDataToFilepath(const std::string id, file_path << assetTypeToString(at); file_path << ".asset"; - LL_INFOS() << "filepath.str() = " << file_path.str() << LL_ENDL; - return file_path.str(); } @@ -203,6 +201,24 @@ void LLDiskCache::updateFileAccessTime(const std::string file_path) boost::filesystem::last_write_time(file_path, file_time); } +/** + * + */ +const std::string LLDiskCache::getCacheInfo() +{ + std::ostringstream cache_info; + + F32 max_in_mb = (F32)mMaxSizeBytes / (1024.0 * 1024.0); + F32 percent_used = ((F32)dirFileSize(mCacheDir) / (F32)mMaxSizeBytes) * 100.0; + + cache_info << std::fixed; + cache_info << std::setprecision(1); + cache_info << "Max size " << max_in_mb << " MB "; + cache_info << "(" << percent_used << "% used)"; + + return cache_info.str(); +} + /** * Clear the cache by removing all the files in the cache directory * individually. It's important to maintain control of which directory -- cgit v1.2.3 From 1a9942a51c2b5db51adb75356f342665743d1f16 Mon Sep 17 00:00:00 2001 From: Callum Prentice Date: Wed, 7 Oct 2020 16:43:01 -0700 Subject: Improve, rationalize and expand comments --- indra/llfilesystem/lldiskcache.cpp | 49 +++++++++++++++++--------------------- 1 file changed, 22 insertions(+), 27 deletions(-) (limited to 'indra/llfilesystem/lldiskcache.cpp') diff --git a/indra/llfilesystem/lldiskcache.cpp b/indra/llfilesystem/lldiskcache.cpp index 455e27221e..e2e50c775d 100644 --- a/indra/llfilesystem/lldiskcache.cpp +++ b/indra/llfilesystem/lldiskcache.cpp @@ -2,6 +2,12 @@ * @file lldiskcache.cpp * @brief The disk cache implementation. * + * Note: Rather than keep the top level function comments up + * to date in both the source and header files, I elected to + * only have explicit comments about each function and variable + * in the header - look there for details. The same is true for + * description of how this code is supposed to work. + * * $LicenseInfo:firstyear=2009&license=viewerlgpl$ * Second Life Viewer Source Code * Copyright (C) 2020, Linden Research, Inc. @@ -40,10 +46,8 @@ LLDiskCache::LLDiskCache(const std::string cache_dir, mMaxSizeBytes(max_size_bytes), mEnableCacheDebugInfo(enable_cache_debug_info) { - // the prefix used for cache filenames to disambiguate them from other files mCacheFilenamePrefix = "sl_cache"; - // create cache dir if it does not exist boost::filesystem::create_directory(cache_dir); } @@ -126,7 +130,7 @@ void LLDiskCache::purge() const std::string LLDiskCache::assetTypeToString(LLAssetType::EType at) { /** - * Make use of the C++17 (or is it 14) feature that allows + * Make use of the handy C++17 feature that allows * for inline initialization of an std::map<> */ typedef std::map asset_type_to_name_t; @@ -190,20 +194,12 @@ const std::string LLDiskCache::metaDataToFilepath(const std::string id, return file_path.str(); } -/** - * Update the "last write time" of a file to "now". This must be called whenever a - * file in the cache is read (not written) so that the last time the file was - * accessed which is used in the mechanism for purging the cache, is up to date. - */ void LLDiskCache::updateFileAccessTime(const std::string file_path) { const std::time_t file_time = std::time(nullptr); boost::filesystem::last_write_time(file_path, file_time); } -/** - * - */ const std::string LLDiskCache::getCacheInfo() { std::ostringstream cache_info; @@ -219,14 +215,14 @@ const std::string LLDiskCache::getCacheInfo() return cache_info.str(); } -/** - * Clear the cache by removing all the files in the cache directory - * individually. It's important to maintain control of which directory - * if passed in and not let the user inadvertently (or maliciously) set - * it to an random location like your project source or OS system directory - */ void LLDiskCache::clearCache(const std::string cache_dir) { + /** + * See notes on performance in dirFileSize(..) - there may be + * a quicker way to do this by operating on the parent dir vs + * the component files but it's called infrequently so it's + * likely just fine + */ if (boost::filesystem::is_directory(cache_dir)) { for (auto& entry : boost::make_iterator_range(boost::filesystem::directory_iterator(cache_dir), {})) @@ -242,20 +238,19 @@ void LLDiskCache::clearCache(const std::string cache_dir) } } -/** - * Utility function to get the total filesize of all files in a directory. It - * used to test file extensions to only check cache files but that was removed. - * There may be a better way that works directly on the folder (similar to - * right clicking on a folder in the OS and asking for size vs right clicking - * on all files and adding up manually) but this is very fast - less than 100ms - * in my testing so, so long as it's not called frequently, it should be okay. - * Note that's it's only currently used for logging/debugging so if performance - * is ever an issue, optimizing this or removing it altogether, is an easy win. - */ uintmax_t LLDiskCache::dirFileSize(const std::string dir) { uintmax_t total_file_size = 0; + /** + * There may be a better way that works directly on the folder (similar to + * right clicking on a folder in the OS and asking for size vs right clicking + * on all files and adding up manually) but this is very fast - less than 100ms + * for 10,000 files in my testing so, so long as it's not called frequently, + * it should be okay. Note that's it's only currently used for logging/debugging + * so if performance is ever an issue, optimizing this or removing it altogether, + * is an easy win. + */ if (boost::filesystem::is_directory(dir)) { for (auto& entry : boost::make_iterator_range(boost::filesystem::directory_iterator(dir), {})) -- cgit v1.2.3 From 3051db7b61ee43fffd28f0a12c0714b11b6b7df7 Mon Sep 17 00:00:00 2001 From: Mnikolenko Productengine Date: Thu, 8 Oct 2020 15:26:54 +0300 Subject: Purge excessive files from disc cache each startup --- indra/llfilesystem/lldiskcache.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'indra/llfilesystem/lldiskcache.cpp') diff --git a/indra/llfilesystem/lldiskcache.cpp b/indra/llfilesystem/lldiskcache.cpp index e2e50c775d..efe5e7092c 100644 --- a/indra/llfilesystem/lldiskcache.cpp +++ b/indra/llfilesystem/lldiskcache.cpp @@ -215,7 +215,7 @@ const std::string LLDiskCache::getCacheInfo() return cache_info.str(); } -void LLDiskCache::clearCache(const std::string cache_dir) +void LLDiskCache::clearCache() { /** * See notes on performance in dirFileSize(..) - there may be @@ -223,9 +223,9 @@ void LLDiskCache::clearCache(const std::string cache_dir) * the component files but it's called infrequently so it's * likely just fine */ - if (boost::filesystem::is_directory(cache_dir)) + if (boost::filesystem::is_directory(mCacheDir)) { - for (auto& entry : boost::make_iterator_range(boost::filesystem::directory_iterator(cache_dir), {})) + for (auto& entry : boost::make_iterator_range(boost::filesystem::directory_iterator(mCacheDir), {})) { if (boost::filesystem::is_regular_file(entry)) { -- cgit v1.2.3 From aae7259a0adaddb4b2d9a2a62a0d4ff95fe5e2b3 Mon Sep 17 00:00:00 2001 From: Callum Prentice Date: Tue, 27 Oct 2020 14:02:24 -0700 Subject: Fix for meta issue: SL-14210 Prune descriptive tag from new cache filenames --- indra/llfilesystem/lldiskcache.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'indra/llfilesystem/lldiskcache.cpp') diff --git a/indra/llfilesystem/lldiskcache.cpp b/indra/llfilesystem/lldiskcache.cpp index efe5e7092c..91fc1b15d1 100644 --- a/indra/llfilesystem/lldiskcache.cpp +++ b/indra/llfilesystem/lldiskcache.cpp @@ -187,8 +187,12 @@ const std::string LLDiskCache::metaDataToFilepath(const std::string id, file_path << id; file_path << "_"; file_path << (extra_info.empty() ? "0" : extra_info); - file_path << "_"; - file_path << assetTypeToString(at); + //file_path << "_"; + //file_path << assetTypeToString(at); // see SL-14210 Prune descriptive tag from new cache filenames + // for details of why it was removed. Note that if you put it + // back or change the format of the filename, the cache files + // files will be invalidated (and perhaps, more importantly, + // never deleted unless you delete them manually). file_path << ".asset"; return file_path.str(); -- cgit v1.2.3 From 53cae8b21f0f77fbb1be22c64deee9b6a3f237f7 Mon Sep 17 00:00:00 2001 From: Mnikolenko Productengine Date: Fri, 11 Dec 2020 16:42:10 +0200 Subject: SL-14505 FIXED [Win10] The viewer isn't started on the non-English system locale --- indra/llfilesystem/lldiskcache.cpp | 34 +++++++++++++++++++++++++++------- 1 file changed, 27 insertions(+), 7 deletions(-) (limited to 'indra/llfilesystem/lldiskcache.cpp') diff --git a/indra/llfilesystem/lldiskcache.cpp b/indra/llfilesystem/lldiskcache.cpp index 91fc1b15d1..34ff80b250 100644 --- a/indra/llfilesystem/lldiskcache.cpp +++ b/indra/llfilesystem/lldiskcache.cpp @@ -48,7 +48,7 @@ LLDiskCache::LLDiskCache(const std::string cache_dir, { mCacheFilenamePrefix = "sl_cache"; - boost::filesystem::create_directory(cache_dir); + LLFile::mkdir(cache_dir); } void LLDiskCache::purge() @@ -63,9 +63,14 @@ void LLDiskCache::purge() typedef std::pair> file_info_t; std::vector file_info; - if (boost::filesystem::is_directory(mCacheDir)) +#if LL_WINDOWS + std::wstring cache_path(utf8str_to_utf16str(mCacheDir)); +#else + std::string cache_path(mCacheDir); +#endif + if (boost::filesystem::is_directory(cache_path)) { - for (auto& entry : boost::make_iterator_range(boost::filesystem::directory_iterator(mCacheDir), {})) + for (auto& entry : boost::make_iterator_range(boost::filesystem::directory_iterator(cache_path), {})) { if (boost::filesystem::is_regular_file(entry)) { @@ -201,7 +206,12 @@ const std::string LLDiskCache::metaDataToFilepath(const std::string id, void LLDiskCache::updateFileAccessTime(const std::string file_path) { const std::time_t file_time = std::time(nullptr); + +#if LL_WINDOWS + boost::filesystem::last_write_time(utf8str_to_utf16str(file_path), file_time); +#else boost::filesystem::last_write_time(file_path, file_time); +#endif } const std::string LLDiskCache::getCacheInfo() @@ -227,9 +237,14 @@ void LLDiskCache::clearCache() * the component files but it's called infrequently so it's * likely just fine */ - if (boost::filesystem::is_directory(mCacheDir)) +#if LL_WINDOWS + std::wstring cache_path(utf8str_to_utf16str(mCacheDir)); +#else + std::string cache_path(mCacheDir); +#endif + if (boost::filesystem::is_directory(cache_path)) { - for (auto& entry : boost::make_iterator_range(boost::filesystem::directory_iterator(mCacheDir), {})) + for (auto& entry : boost::make_iterator_range(boost::filesystem::directory_iterator(cache_path), {})) { if (boost::filesystem::is_regular_file(entry)) { @@ -255,9 +270,14 @@ uintmax_t LLDiskCache::dirFileSize(const std::string dir) * so if performance is ever an issue, optimizing this or removing it altogether, * is an easy win. */ - if (boost::filesystem::is_directory(dir)) +#if LL_WINDOWS + std::wstring dir_path(utf8str_to_utf16str(dir)); +#else + std::string dir_path(dir); +#endif + if (boost::filesystem::is_directory(dir_path)) { - for (auto& entry : boost::make_iterator_range(boost::filesystem::directory_iterator(dir), {})) + for (auto& entry : boost::make_iterator_range(boost::filesystem::directory_iterator(dir_path), {})) { if (boost::filesystem::is_regular_file(entry)) { -- cgit v1.2.3 From 60e8f990fdbc5f00b69c1a7355330ff421133cea Mon Sep 17 00:00:00 2001 From: Callum Prentice Date: Fri, 8 Jan 2021 14:21:58 -0800 Subject: Addresses SL-14582: Add code to only write the file last access time occasionally --- indra/llfilesystem/lldiskcache.cpp | 40 +++++++++++++++++++++++++++++++++++--- 1 file changed, 37 insertions(+), 3 deletions(-) (limited to 'indra/llfilesystem/lldiskcache.cpp') diff --git a/indra/llfilesystem/lldiskcache.cpp b/indra/llfilesystem/lldiskcache.cpp index 34ff80b250..c9f7684b5a 100644 --- a/indra/llfilesystem/lldiskcache.cpp +++ b/indra/llfilesystem/lldiskcache.cpp @@ -205,12 +205,46 @@ const std::string LLDiskCache::metaDataToFilepath(const std::string id, void LLDiskCache::updateFileAccessTime(const std::string file_path) { - const std::time_t file_time = std::time(nullptr); + /** + * Threshold in time_t units that is used to decide if the last access time + * time of the file is updated or not. Added as a precaution for the concern + * outlined in SL-14582 about frequent writes on older SSDs reducing their + * lifespan. I think this is the right place for the threshold value - rather + * than it being a pref - do comment on that Jira if you disagree... + * + * Let's start with 1 hour in time_t units and see how that unfolds + */ + const std::time_t time_threshold = 1 * 60 * 60; + + // current time + const std::time_t cur_time = std::time(nullptr); #if LL_WINDOWS - boost::filesystem::last_write_time(utf8str_to_utf16str(file_path), file_time); + // file last write time + const std::time_t last_write_time = boost::filesystem::last_write_time(utf8str_to_utf16str(file_path)); + + // delta between cur time and last time the file was written + const std::time_t delta_time = cur_time - last_write_time; + + // we only write the new value if the time in time_threshold has elapsed + // before the last one + if (delta_time > time_threshold) + { + boost::filesystem::last_write_time(utf8str_to_utf16str(file_path), cur_time); + } #else - boost::filesystem::last_write_time(file_path, file_time); + // file last write time + const std::time_t last_write_time = boost::filesystem::last_write_time(file_path); + + // delta between cur time and last time the file was written + const std::time_t delta_time = cur_time - last_write_time; + + // we only write the new value if the time in time_threshold has elapsed + // before the last one + if (delta_time > time_threshold) + { + boost::filesystem::last_write_time(file_path, cur_time); + } #endif } -- cgit v1.2.3