From d9448c6f52218146113d1d5c5ca4c4d5f01dc5cf Mon Sep 17 00:00:00 2001 From: Callum Prentice Date: Thu, 17 Sep 2020 09:45:06 -0700 Subject: The folder where the disk cache lives was originally renamed from llvfs to llcache but @henri's suggestion that that doesn't reflect the other files in the same place and it should be llfilesystem is a good one so I changed it over --- indra/llfilesystem/CMakeLists.txt | 98 ++ indra/llfilesystem/lldir.cpp | 1134 +++++++++++++++++++++++ indra/llfilesystem/lldir.h | 280 ++++++ indra/llfilesystem/lldir_linux.cpp | 269 ++++++ indra/llfilesystem/lldir_linux.h | 64 ++ indra/llfilesystem/lldir_mac.cpp | 205 ++++ indra/llfilesystem/lldir_mac.h | 56 ++ indra/llfilesystem/lldir_solaris.cpp | 266 ++++++ indra/llfilesystem/lldir_solaris.h | 61 ++ indra/llfilesystem/lldir_utils_objc.h | 43 + indra/llfilesystem/lldir_utils_objc.mm | 108 +++ indra/llfilesystem/lldir_win32.cpp | 452 +++++++++ indra/llfilesystem/lldir_win32.h | 59 ++ indra/llfilesystem/lldirguard.h | 72 ++ indra/llfilesystem/lldiriterator.cpp | 243 +++++ indra/llfilesystem/lldiriterator.h | 87 ++ indra/llfilesystem/lldiskcache.cpp | 387 ++++++++ indra/llfilesystem/lldiskcache.h | 82 ++ indra/llfilesystem/lllfsthread.cpp | 245 +++++ indra/llfilesystem/lllfsthread.h | 147 +++ indra/llfilesystem/tests/lldir_test.cpp | 767 +++++++++++++++ indra/llfilesystem/tests/lldiriterator_test.cpp | 65 ++ 22 files changed, 5190 insertions(+) create mode 100644 indra/llfilesystem/CMakeLists.txt create mode 100644 indra/llfilesystem/lldir.cpp create mode 100644 indra/llfilesystem/lldir.h create mode 100644 indra/llfilesystem/lldir_linux.cpp create mode 100644 indra/llfilesystem/lldir_linux.h create mode 100644 indra/llfilesystem/lldir_mac.cpp create mode 100644 indra/llfilesystem/lldir_mac.h create mode 100644 indra/llfilesystem/lldir_solaris.cpp create mode 100644 indra/llfilesystem/lldir_solaris.h create mode 100644 indra/llfilesystem/lldir_utils_objc.h create mode 100644 indra/llfilesystem/lldir_utils_objc.mm create mode 100644 indra/llfilesystem/lldir_win32.cpp create mode 100644 indra/llfilesystem/lldir_win32.h create mode 100644 indra/llfilesystem/lldirguard.h create mode 100644 indra/llfilesystem/lldiriterator.cpp create mode 100644 indra/llfilesystem/lldiriterator.h create mode 100644 indra/llfilesystem/lldiskcache.cpp create mode 100644 indra/llfilesystem/lldiskcache.h create mode 100644 indra/llfilesystem/lllfsthread.cpp create mode 100644 indra/llfilesystem/lllfsthread.h create mode 100644 indra/llfilesystem/tests/lldir_test.cpp create mode 100644 indra/llfilesystem/tests/lldiriterator_test.cpp (limited to 'indra/llfilesystem') diff --git a/indra/llfilesystem/CMakeLists.txt b/indra/llfilesystem/CMakeLists.txt new file mode 100644 index 0000000000..4af14d6d3a --- /dev/null +++ b/indra/llfilesystem/CMakeLists.txt @@ -0,0 +1,98 @@ +# -*- cmake -*- + +project(llfilesystem) + +include(00-Common) +include(LLCommon) +include(UnixInstall) + +include_directories( + ${LLCOMMON_INCLUDE_DIRS} + ${LLCOMMON_SYSTEM_INCLUDE_DIRS} + ) + +set(llfilesystem_SOURCE_FILES + lldir.cpp + lldiriterator.cpp + lllfsthread.cpp + lldiskcache.cpp + ) + +set(llfilesystem_HEADER_FILES + CMakeLists.txt + + lldir.h + lldirguard.h + lldiriterator.h + lllfsthread.h + lldiskcache.h + ) + +if (DARWIN) + LIST(APPEND llfilesystem_SOURCE_FILES lldir_utils_objc.mm) + LIST(APPEND llfilesystem_SOURCE_FILES lldir_utils_objc.h) + LIST(APPEND llfilesystem_SOURCE_FILES lldir_mac.cpp) + LIST(APPEND llfilesystem_HEADER_FILES lldir_mac.h) +endif (DARWIN) + +if (LINUX) + LIST(APPEND llfilesystem_SOURCE_FILES lldir_linux.cpp) + LIST(APPEND llfilesystem_HEADER_FILES lldir_linux.h) + + if (INSTALL) + set_source_files_properties(lldir_linux.cpp + PROPERTIES COMPILE_FLAGS + "-DAPP_RO_DATA_DIR=\\\"${APP_SHARE_DIR}\\\"" + ) + endif (INSTALL) +endif (LINUX) + +if (WINDOWS) + LIST(APPEND llfilesystem_SOURCE_FILES lldir_win32.cpp) + LIST(APPEND llfilesystem_HEADER_FILES lldir_win32.h) +endif (WINDOWS) + +set_source_files_properties(${llfilesystem_HEADER_FILES} + PROPERTIES HEADER_FILE_ONLY TRUE) + +list(APPEND llfilesystem_SOURCE_FILES ${llfilesystem_HEADER_FILES}) + +add_library (llfilesystem ${llfilesystem_SOURCE_FILES}) + +set(cache_BOOST_LIBRARIES + ${BOOST_FILESYSTEM_LIBRARY} + ${BOOST_SYSTEM_LIBRARY} + ) + +target_link_libraries(llfilesystem + ${LLCOMMON_LIBRARIES} + ${cache_BOOST_LIBRARIES} + ) + +if (DARWIN) + include(CMakeFindFrameworks) + find_library(COCOA_LIBRARY Cocoa) + target_link_libraries(llfilesystem ${COCOA_LIBRARY}) +endif (DARWIN) + + +# Add tests +if (LL_TESTS) + include(LLAddBuildTest) + # UNIT TESTS + SET(llfilesystem_TEST_SOURCE_FILES + lldiriterator.cpp + ) + + set_source_files_properties(lldiriterator.cpp + PROPERTIES + LL_TEST_ADDITIONAL_LIBRARIES "${cache_BOOST_LIBRARIES}" + ) + LL_ADD_PROJECT_UNIT_TESTS(llfilesystem "${llfilesystem_TEST_SOURCE_FILES}") + + # INTEGRATION TESTS + set(test_libs llmath llcommon llfilesystem ${LLCOMMON_LIBRARIES} ${WINDOWS_LIBRARIES}) + + # TODO: Some of these need refactoring to be proper Unit tests rather than Integration tests. + LL_ADD_INTEGRATION_TEST(lldir "" "${test_libs}") +endif (LL_TESTS) diff --git a/indra/llfilesystem/lldir.cpp b/indra/llfilesystem/lldir.cpp new file mode 100644 index 0000000000..10fbc06c61 --- /dev/null +++ b/indra/llfilesystem/lldir.cpp @@ -0,0 +1,1134 @@ +/** + * @file lldir.cpp + * @brief implementation of directory utilities base class + * + * $LicenseInfo:firstyear=2002&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2010, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include "linden_common.h" + +#if !LL_WINDOWS +#include +#include +#include +#else +#include +#endif + +#include "lldir.h" + +#include "llerror.h" +#include "lltimer.h" // ms_sleep() +#include "lluuid.h" + +#include "lldiriterator.h" +#include "stringize.h" +#include "llstring.h" +#include +#include +#include +#include +#include +#include +#include +#include + +using boost::assign::list_of; +using boost::assign::map_list_of; + +#if LL_WINDOWS +#include "lldir_win32.h" +LLDir_Win32 gDirUtil; +#elif LL_DARWIN +#include "lldir_mac.h" +LLDir_Mac gDirUtil; +#elif LL_SOLARIS +#include "lldir_solaris.h" +LLDir_Solaris gDirUtil; +#else +#include "lldir_linux.h" +LLDir_Linux gDirUtil; +#endif + +LLDir *gDirUtilp = (LLDir *)&gDirUtil; + +/// Values for findSkinnedFilenames(subdir) parameter +const char + *LLDir::XUI = "xui", + *LLDir::TEXTURES = "textures", + *LLDir::SKINBASE = ""; + +static const char* const empty = ""; +std::string LLDir::sDumpDir = ""; + +LLDir::LLDir() +: mAppName(""), + mExecutablePathAndName(""), + mExecutableFilename(""), + mExecutableDir(""), + mAppRODataDir(""), + mOSUserDir(""), + mOSUserAppDir(""), + mLindenUserDir(""), + mOSCacheDir(""), + mCAFile(""), + mTempDir(""), + mDirDelimiter("/"), // fallback to forward slash if not overridden + mLanguage("en"), + mUserName("undefined") +{ +} + +LLDir::~LLDir() +{ +} + +std::vector LLDir::getFilesInDir(const std::string &dirname) +{ + //Returns a vector of fullpath filenames. + +#ifdef LL_WINDOWS // or BOOST_WINDOWS_API + boost::filesystem::path p(utf8str_to_utf16str(dirname)); +#else + boost::filesystem::path p(dirname); +#endif + + std::vector v; + + if (exists(p)) + { + if (is_directory(p)) + { + boost::filesystem::directory_iterator end_iter; + for (boost::filesystem::directory_iterator dir_itr(p); + dir_itr != end_iter; + ++dir_itr) + { + if (boost::filesystem::is_regular_file(dir_itr->status())) + { + v.push_back(dir_itr->path().filename().string()); + } + } + } + } + return v; +} + +S32 LLDir::deleteFilesInDir(const std::string &dirname, const std::string &mask) +{ + S32 count = 0; + std::string filename; + std::string fullpath; + S32 result; + + // File masks starting with "/" will match nothing, so we consider them invalid. + if (LLStringUtil::startsWith(mask, getDirDelimiter())) + { + LL_WARNS() << "Invalid file mask: " << mask << LL_ENDL; + llassert(!"Invalid file mask"); + } + + LLDirIterator iter(dirname, mask); + while (iter.next(filename)) + { + fullpath = add(dirname, filename); + + if(LLFile::isdir(fullpath)) + { + // skipping directory traversal filenames + count++; + continue; + } + + S32 retry_count = 0; + while (retry_count < 5) + { + if (0 != LLFile::remove(fullpath)) + { + retry_count++; + result = errno; + LL_WARNS() << "Problem removing " << fullpath << " - errorcode: " + << result << " attempt " << retry_count << LL_ENDL; + + if(retry_count >= 5) + { + LL_WARNS() << "Failed to remove " << fullpath << LL_ENDL ; + return count ; + } + + ms_sleep(100); + } + else + { + if (retry_count) + { + LL_WARNS() << "Successfully removed " << fullpath << LL_ENDL; + } + break; + } + } + count++; + } + return count; +} + +U32 LLDir::deleteDirAndContents(const std::string& dir_name) +{ + //Removes the directory and its contents. Returns number of files deleted. + + U32 num_deleted = 0; + + try + { +#ifdef LL_WINDOWS // or BOOST_WINDOWS_API + boost::filesystem::path dir_path(utf8str_to_utf16str(dir_name)); +#else + boost::filesystem::path dir_path(dir_name); +#endif + + if (boost::filesystem::exists (dir_path)) + { + if (!boost::filesystem::is_empty (dir_path)) + { // Directory has content + num_deleted = boost::filesystem::remove_all (dir_path); + } + else + { // Directory is empty + boost::filesystem::remove (dir_path); + } + } + } + catch (boost::filesystem::filesystem_error &er) + { + LL_WARNS() << "Failed to delete " << dir_name << " with error " << er.code().message() << LL_ENDL; + } + return num_deleted; +} + +const std::string LLDir::findFile(const std::string &filename, + const std::string& searchPath1, + const std::string& searchPath2, + const std::string& searchPath3) const +{ + std::vector search_paths; + search_paths.push_back(searchPath1); + search_paths.push_back(searchPath2); + search_paths.push_back(searchPath3); + return findFile(filename, search_paths); +} + +const std::string LLDir::findFile(const std::string& filename, const std::vector search_paths) const +{ + std::vector::const_iterator search_path_iter; + for (search_path_iter = search_paths.begin(); + search_path_iter != search_paths.end(); + ++search_path_iter) + { + if (!search_path_iter->empty()) + { + std::string filename_and_path = (*search_path_iter); + if (!filename.empty()) + { + filename_and_path += getDirDelimiter() + filename; + } + if (fileExists(filename_and_path)) + { + return filename_and_path; + } + } + } + return ""; +} + + +const std::string &LLDir::getExecutablePathAndName() const +{ + return mExecutablePathAndName; +} + +const std::string &LLDir::getExecutableFilename() const +{ + return mExecutableFilename; +} + +const std::string &LLDir::getExecutableDir() const +{ + return mExecutableDir; +} + +const std::string &LLDir::getWorkingDir() const +{ + return mWorkingDir; +} + +const std::string &LLDir::getAppName() const +{ + return mAppName; +} + +const std::string &LLDir::getAppRODataDir() const +{ + return mAppRODataDir; +} + +const std::string &LLDir::getOSUserDir() const +{ + return mOSUserDir; +} + +const std::string &LLDir::getOSUserAppDir() const +{ + return mOSUserAppDir; +} + +const std::string &LLDir::getLindenUserDir() const +{ + if (mLindenUserDir.empty()) + { + LL_DEBUGS() << "getLindenUserDir() called early, we don't have the user name yet - returning empty string to caller" << LL_ENDL; + } + + return mLindenUserDir; +} + +const std::string& LLDir::getChatLogsDir() const +{ + return mChatLogsDir; +} + +void LLDir::setDumpDir( const std::string& path ) +{ + LLDir::sDumpDir = path; + if (LLStringUtil::endsWith(sDumpDir, mDirDelimiter)) + { + sDumpDir.erase(sDumpDir.size() - mDirDelimiter.size()); + } +} + +const std::string &LLDir::getDumpDir() const +{ + if (sDumpDir.empty() ) + { + LLUUID uid; + uid.generate(); + + sDumpDir = gDirUtilp->getExpandedFilename(LL_PATH_LOGS, "") + + "dump-" + uid.asString(); + + dir_exists_or_crash(sDumpDir); + } + + return LLDir::sDumpDir; +} + +const std::string &LLDir::getPerAccountChatLogsDir() const +{ + return mPerAccountChatLogsDir; +} + +const std::string &LLDir::getTempDir() const +{ + return mTempDir; +} + +const std::string LLDir::getCacheDir(bool get_default) const +{ + if (mCacheDir.empty() || get_default) + { + if (!mDefaultCacheDir.empty()) + { // Set at startup - can't set here due to const API + return mDefaultCacheDir; + } + + std::string res = buildSLOSCacheDir(); + return res; + } + else + { + return mCacheDir; + } +} + +// Return the default cache directory +std::string LLDir::buildSLOSCacheDir() const +{ + std::string res; + if (getOSCacheDir().empty()) + { + if (getOSUserAppDir().empty()) + { + res = "data"; + } + else + { + res = add(getOSUserAppDir(), "cache"); + } + } + else + { + res = add(getOSCacheDir(), "SecondLife"); + } + return res; +} + + + +const std::string &LLDir::getOSCacheDir() const +{ + return mOSCacheDir; +} + + +const std::string &LLDir::getCAFile() const +{ + return mCAFile; +} + +const std::string &LLDir::getDirDelimiter() const +{ + return mDirDelimiter; +} + +const std::string& LLDir::getDefaultSkinDir() const +{ + return mDefaultSkinDir; +} + +const std::string &LLDir::getSkinDir() const +{ + return mSkinDir; +} + +const std::string &LLDir::getUserDefaultSkinDir() const +{ + return mUserDefaultSkinDir; +} + +const std::string &LLDir::getUserSkinDir() const +{ + return mUserSkinDir; +} + +const std::string LLDir::getSkinBaseDir() const +{ + return mSkinBaseDir; +} + +const std::string &LLDir::getLLPluginDir() const +{ + return mLLPluginDir; +} + +const std::string &LLDir::getUserName() const +{ + return mUserName; +} + +static std::string ELLPathToString(ELLPath location) +{ + typedef std::map ELLPathMap; +#define ENT(symbol) (symbol, #symbol) + static const ELLPathMap sMap = map_list_of + ENT(LL_PATH_NONE) + ENT(LL_PATH_USER_SETTINGS) + ENT(LL_PATH_APP_SETTINGS) + ENT(LL_PATH_PER_SL_ACCOUNT) // returns/expands to blank string if we don't know the account name yet + ENT(LL_PATH_CACHE) + ENT(LL_PATH_CHARACTER) + ENT(LL_PATH_HELP) + ENT(LL_PATH_LOGS) + ENT(LL_PATH_TEMP) + ENT(LL_PATH_SKINS) + ENT(LL_PATH_TOP_SKIN) + ENT(LL_PATH_CHAT_LOGS) + ENT(LL_PATH_PER_ACCOUNT_CHAT_LOGS) + ENT(LL_PATH_USER_SKIN) + ENT(LL_PATH_LOCAL_ASSETS) + ENT(LL_PATH_EXECUTABLE) + ENT(LL_PATH_DEFAULT_SKIN) + ENT(LL_PATH_FONTS) + ENT(LL_PATH_LAST) + ; +#undef ENT + + ELLPathMap::const_iterator found = sMap.find(location); + if (found != sMap.end()) + return found->second; + return STRINGIZE("Invalid ELLPath value " << location); +} + +std::string LLDir::getExpandedFilename(ELLPath location, const std::string& filename) const +{ + return getExpandedFilename(location, "", filename); +} + +std::string LLDir::getExpandedFilename(ELLPath location, const std::string& subdir, const std::string& filename) const +{ + return getExpandedFilename(location, "", subdir, filename); +} + +std::string LLDir::getExpandedFilename(ELLPath location, const std::string& subdir1, const std::string& subdir2, const std::string& in_filename) const +{ + std::string prefix; + switch (location) + { + case LL_PATH_NONE: + // Do nothing + break; + + case LL_PATH_APP_SETTINGS: + prefix = add(getAppRODataDir(), "app_settings"); + break; + + case LL_PATH_CHARACTER: + prefix = add(getAppRODataDir(), "character"); + break; + + case LL_PATH_HELP: + prefix = "help"; + break; + + case LL_PATH_CACHE: + prefix = getCacheDir(); + break; + + case LL_PATH_DUMP: + prefix=getDumpDir(); + break; + + case LL_PATH_USER_SETTINGS: + prefix = add(getOSUserAppDir(), "user_settings"); + break; + + case LL_PATH_PER_SL_ACCOUNT: + prefix = getLindenUserDir(); + if (prefix.empty()) + { + // if we're asking for the per-SL-account directory but we haven't + // logged in yet (or otherwise don't know the account name from + // which to build this string), then intentionally return a blank + // string to the caller and skip the below warning about a blank + // prefix. + LL_DEBUGS("LLDir") << "getLindenUserDir() not yet set: " + << ELLPathToString(location) + << ", '" << subdir1 << "', '" << subdir2 << "', '" << in_filename + << "' => ''" << LL_ENDL; + return std::string(); + } + break; + + case LL_PATH_CHAT_LOGS: + prefix = getChatLogsDir(); + break; + + case LL_PATH_PER_ACCOUNT_CHAT_LOGS: + prefix = getPerAccountChatLogsDir(); + if (prefix.empty()) + { + // potentially directory was not set yet + // intentionally return a blank string to the caller + LL_DEBUGS("LLDir") << "Conversation log directory is not yet set" << LL_ENDL; + return std::string(); + } + break; + + case LL_PATH_LOGS: + prefix = add(getOSUserAppDir(), "logs"); + break; + + case LL_PATH_TEMP: + prefix = getTempDir(); + break; + + case LL_PATH_TOP_SKIN: + prefix = getSkinDir(); + break; + + case LL_PATH_DEFAULT_SKIN: + prefix = getDefaultSkinDir(); + break; + + case LL_PATH_USER_SKIN: + prefix = getUserSkinDir(); + break; + + case LL_PATH_SKINS: + prefix = getSkinBaseDir(); + break; + + case LL_PATH_LOCAL_ASSETS: + prefix = add(getAppRODataDir(), "local_assets"); + break; + + case LL_PATH_EXECUTABLE: + prefix = getExecutableDir(); + break; + + case LL_PATH_FONTS: + prefix = add(getAppRODataDir(), "fonts"); + break; + + default: + llassert(0); + } + + if (prefix.empty()) + { + LL_WARNS() << ELLPathToString(location) + << ", '" << subdir1 << "', '" << subdir2 << "', '" << in_filename + << "': prefix is empty, possible bad filename" << LL_ENDL; + } + + std::string expanded_filename = add(prefix, subdir1, subdir2); + if (expanded_filename.empty() && in_filename.empty()) + { + return ""; + } + // Use explicit concatenation here instead of another add() call. Callers + // passing in_filename as "" expect to obtain a pathname ending with + // mDirSeparator so they can later directly concatenate with a specific + // filename. A caller using add() doesn't care, but there's still code + // loose in the system that uses std::string::operator+(). + expanded_filename += mDirDelimiter; + expanded_filename += in_filename; + + LL_DEBUGS("LLDir") << ELLPathToString(location) + << ", '" << subdir1 << "', '" << subdir2 << "', '" << in_filename + << "' => '" << expanded_filename << "'" << LL_ENDL; + return expanded_filename; +} + +std::string LLDir::getBaseFileName(const std::string& filepath, bool strip_exten) const +{ + std::size_t offset = filepath.find_last_of(getDirDelimiter()); + offset = (offset == std::string::npos) ? 0 : offset+1; + std::string res = filepath.substr(offset, std::string::npos); + if (strip_exten) + { + offset = res.find_last_of('.'); + if (offset != std::string::npos && + offset != 0) // if basename STARTS with '.', don't strip + { + res = res.substr(0, offset); + } + } + return res; +} + +std::string LLDir::getDirName(const std::string& filepath) const +{ + std::size_t offset = filepath.find_last_of(getDirDelimiter()); + S32 len = (offset == std::string::npos) ? 0 : offset; + std::string dirname = filepath.substr(0, len); + return dirname; +} + +std::string LLDir::getExtension(const std::string& filepath) const +{ + if (filepath.empty()) + return std::string(); + std::string basename = getBaseFileName(filepath, false); + std::size_t offset = basename.find_last_of('.'); + std::string exten = (offset == std::string::npos || offset == 0) ? "" : basename.substr(offset+1); + LLStringUtil::toLower(exten); + return exten; +} + +std::string LLDir::findSkinnedFilenameBaseLang(const std::string &subdir, + const std::string &filename, + ESkinConstraint constraint) const +{ + // This implementation is basically just as described in the declaration comments. + std::vector found(findSkinnedFilenames(subdir, filename, constraint)); + if (found.empty()) + { + return ""; + } + return found.front(); +} + +std::string LLDir::findSkinnedFilename(const std::string &subdir, + const std::string &filename, + ESkinConstraint constraint) const +{ + // This implementation is basically just as described in the declaration comments. + std::vector found(findSkinnedFilenames(subdir, filename, constraint)); + if (found.empty()) + { + return ""; + } + return found.back(); +} + +// This method exists because the two code paths for +// findSkinnedFilenames(ALL_SKINS) and findSkinnedFilenames(CURRENT_SKIN) must +// generate the list of candidate pathnames in identical ways. The only +// difference is in the body of the inner loop. +template +void LLDir::walkSearchSkinDirs(const std::string& subdir, + const std::vector& subsubdirs, + const std::string& filename, + const FUNCTION& function) const +{ + BOOST_FOREACH(std::string skindir, mSearchSkinDirs) + { + std::string subdir_path(add(skindir, subdir)); + BOOST_FOREACH(std::string subsubdir, subsubdirs) + { + std::string full_path(add(subdir_path, subsubdir, filename)); + if (fileExists(full_path)) + { + function(subsubdir, full_path); + } + } + } +} + +// ridiculous little helper function that should go away when we can use lambda +inline void push_back(std::vector& vector, const std::string& value) +{ + vector.push_back(value); +} + +typedef std::map StringMap; +// ridiculous little helper function that should go away when we can use lambda +inline void store_in_map(StringMap& map, const std::string& key, const std::string& value) +{ + map[key] = value; +} + +std::vector LLDir::findSkinnedFilenames(const std::string& subdir, + const std::string& filename, + ESkinConstraint constraint) const +{ + // Recognize subdirs that have no localization. + static const std::set sUnlocalized = list_of + ("") // top-level directory not localized + ("textures") // textures not localized + ; + + LL_DEBUGS("LLDir") << "subdir '" << subdir << "', filename '" << filename + << "', constraint " + << ((constraint == CURRENT_SKIN)? "CURRENT_SKIN" : "ALL_SKINS") + << LL_ENDL; + + // Build results vector. + std::vector results; + // Disallow filenames that may escape subdir + if (filename.find("..") != std::string::npos) + { + LL_WARNS("LLDir") << "Ignoring potentially relative filename '" << filename << "'" << LL_ENDL; + return results; + } + + // Cache the default language directory for each subdir we've encountered. + // A cache entry whose value is the empty string means "not localized, + // don't bother checking again." + static StringMap sLocalized; + + // Check whether we've already discovered if this subdir is localized. + StringMap::const_iterator found = sLocalized.find(subdir); + if (found == sLocalized.end()) + { + // We have not yet determined that. Is it one of the subdirs "known" + // to be unlocalized? + if (sUnlocalized.find(subdir) != sUnlocalized.end()) + { + // This subdir is known to be unlocalized. Remember that. + found = sLocalized.insert(StringMap::value_type(subdir, "")).first; + } + else + { + // We do not recognize this subdir. Investigate. + std::string subdir_path(add(getDefaultSkinDir(), subdir)); + if (fileExists(add(subdir_path, "en"))) + { + // defaultSkinDir/subdir contains subdir "en". That's our + // default language; this subdir is localized. + found = sLocalized.insert(StringMap::value_type(subdir, "en")).first; + } + else if (fileExists(add(subdir_path, "en-us"))) + { + // defaultSkinDir/subdir contains subdir "en-us" but not "en". + // Set as default language; this subdir is localized. + found = sLocalized.insert(StringMap::value_type(subdir, "en-us")).first; + } + else + { + // defaultSkinDir/subdir contains neither "en" nor "en-us". + // Assume it's not localized. Remember that assumption. + found = sLocalized.insert(StringMap::value_type(subdir, "")).first; + } + } + } + // Every code path above should have resulted in 'found' becoming a valid + // iterator to an entry in sLocalized. + llassert(found != sLocalized.end()); + + // Now -- is this subdir localized, or not? The answer determines what + // subdirectories we check (under subdir) for the requested filename. + std::vector subsubdirs; + if (found->second.empty()) + { + // subdir is not localized. filename should be located directly within it. + subsubdirs.push_back(""); + } + else + { + // subdir is localized, and found->second is the default language + // directory within it. Check both the default language and the + // current language -- if it differs from the default, of course. + subsubdirs.push_back(found->second); + if (mLanguage != found->second) + { + subsubdirs.push_back(mLanguage); + } + } + + // The process we use depends on 'constraint'. + if (constraint != CURRENT_SKIN) // meaning ALL_SKINS + { + // ALL_SKINS is simpler: just return every pathname generated by + // walkSearchSkinDirs(). Tricky bit: walkSearchSkinDirs() passes its + // FUNCTION the subsubdir as well as the full pathname. We just want + // the full pathname. + walkSearchSkinDirs(subdir, subsubdirs, filename, + boost::bind(push_back, boost::ref(results), _2)); + } + else // CURRENT_SKIN + { + // CURRENT_SKIN turns out to be a bit of a misnomer because we might + // still return files from two different skins. In any case, this + // value of 'constraint' means we will return at most two paths: one + // for the default language, one for the current language (supposing + // those differ). + // It is important to allow a user to override only the localization + // for a particular file, for all viewer installs, without also + // overriding the default-language file. + // It is important to allow a user to override only the default- + // language file, for all viewer installs, without also overriding the + // applicable localization of that file. + // Therefore, treat the default language and the current language as + // two separate cases. For each, capture the most-specialized file + // that exists. + // Use a map keyed by subsubdir (i.e. language code). This allows us + // to handle the case of a single subsubdirs entry with the same logic + // that handles two. For every real file path generated by + // walkSearchSkinDirs(), update the map entry for its subsubdir. + StringMap path_for; + walkSearchSkinDirs(subdir, subsubdirs, filename, + boost::bind(store_in_map, boost::ref(path_for), _1, _2)); + // Now that we have a path for each of the default language and the + // current language, copy them -- in proper order -- into results. + // Don't drive this by walking the map itself: it matters that we + // generate results in the same order as subsubdirs. + BOOST_FOREACH(std::string subsubdir, subsubdirs) + { + StringMap::const_iterator found(path_for.find(subsubdir)); + if (found != path_for.end()) + { + results.push_back(found->second); + } + } + } + + LL_DEBUGS("LLDir") << empty; + const char* comma = ""; + BOOST_FOREACH(std::string path, results) + { + LL_CONT << comma << "'" << path << "'"; + comma = ", "; + } + LL_CONT << LL_ENDL; + + return results; +} + +std::string LLDir::getTempFilename() const +{ + LLUUID random_uuid; + std::string uuid_str; + + random_uuid.generate(); + random_uuid.toString(uuid_str); + + return add(getTempDir(), uuid_str + ".tmp"); +} + +// static +std::string LLDir::getScrubbedFileName(const std::string uncleanFileName) +{ + std::string name(uncleanFileName); + const std::string illegalChars(getForbiddenFileChars()); + // replace any illegal file chars with and underscore '_' + for( unsigned int i = 0; i < illegalChars.length(); i++ ) + { + int j = -1; + while((j = name.find(illegalChars[i])) > -1) + { + name[j] = '_'; + } + } + return name; +} + +// static +std::string LLDir::getForbiddenFileChars() +{ + return "\\/:*?\"<>|"; +} + +void LLDir::setLindenUserDir(const std::string &username) +{ + // if the username isn't set, that's bad + if (!username.empty()) + { + // some platforms have case-sensitive filesystems, so be + // utterly consistent with our firstname/lastname case. + std::string userlower(username); + LLStringUtil::toLower(userlower); + LLStringUtil::replaceChar(userlower, ' ', '_'); + mLindenUserDir = add(getOSUserAppDir(), userlower); + } + else + { + LL_ERRS() << "NULL name for LLDir::setLindenUserDir" << LL_ENDL; + } + + dumpCurrentDirectories(); +} + +void LLDir::setChatLogsDir(const std::string &path) +{ + if (!path.empty() ) + { + mChatLogsDir = path; + } + else + { + LL_WARNS() << "Invalid name for LLDir::setChatLogsDir" << LL_ENDL; + } +} + +void LLDir::updatePerAccountChatLogsDir() +{ + mPerAccountChatLogsDir = add(getChatLogsDir(), mUserName); +} + +void LLDir::setPerAccountChatLogsDir(const std::string &username) +{ + // if both first and last aren't set, assume we're grabbing the cached dir + if (!username.empty()) + { + // some platforms have case-sensitive filesystems, so be + // utterly consistent with our firstname/lastname case. + std::string userlower(username); + LLStringUtil::toLower(userlower); + LLStringUtil::replaceChar(userlower, ' ', '_'); + + mUserName = userlower; + updatePerAccountChatLogsDir(); + } + else + { + LL_ERRS() << "NULL name for LLDir::setPerAccountChatLogsDir" << LL_ENDL; + } +} + +void LLDir::setSkinFolder(const std::string &skin_folder, const std::string& language) +{ + LL_DEBUGS("LLDir") << "Setting skin '" << skin_folder << "', language '" << language << "'" + << LL_ENDL; + mSkinName = skin_folder; + mLanguage = language; + + // This method is called multiple times during viewer initialization. Each + // time it's called, reset mSearchSkinDirs. + mSearchSkinDirs.clear(); + + // base skin which is used as fallback for all skinned files + // e.g. c:\program files\secondlife\skins\default + mDefaultSkinDir = getSkinBaseDir(); + append(mDefaultSkinDir, "default"); + // This is always the most general of the search skin directories. + addSearchSkinDir(mDefaultSkinDir); + + mSkinDir = getSkinBaseDir(); + append(mSkinDir, skin_folder); + // Next level of generality is a skin installed with the viewer. + addSearchSkinDir(mSkinDir); + + // user modifications to skins, current and default + // e.g. c:\documents and settings\users\username\application data\second life\skins\dazzle + mUserSkinDir = getOSUserAppDir(); + append(mUserSkinDir, "skins"); + mUserDefaultSkinDir = mUserSkinDir; + append(mUserDefaultSkinDir, "default"); + append(mUserSkinDir, skin_folder); + // Next level of generality is user modifications to default skin... + addSearchSkinDir(mUserDefaultSkinDir); + // then user-defined skins. + addSearchSkinDir(mUserSkinDir); +} + +void LLDir::addSearchSkinDir(const std::string& skindir) +{ + if (std::find(mSearchSkinDirs.begin(), mSearchSkinDirs.end(), skindir) == mSearchSkinDirs.end()) + { + LL_DEBUGS("LLDir") << "search skin: '" << skindir << "'" << LL_ENDL; + mSearchSkinDirs.push_back(skindir); + } +} + +std::string LLDir::getSkinFolder() const +{ + return mSkinName; +} + +std::string LLDir::getLanguage() const +{ + return mLanguage; +} + +bool LLDir::setCacheDir(const std::string &path) +{ + if (path.empty() ) + { + // reset to default + mCacheDir = ""; + return true; + } + else + { + LLFile::mkdir(path); + std::string tempname = add(path, "temp"); + LLFILE* file = LLFile::fopen(tempname,"wt"); + if (file) + { + fclose(file); + LLFile::remove(tempname); + mCacheDir = path; + return true; + } + return false; + } +} + +void LLDir::dumpCurrentDirectories(LLError::ELevel level) +{ + LL_VLOGS(level, "AppInit","Directories") << "Current Directories:" << LL_ENDL; + + LL_VLOGS(level, "AppInit", "Directories") << " CurPath: " << getCurPath() << LL_ENDL; + LL_VLOGS(level, "AppInit", "Directories") << " AppName: " << getAppName() << LL_ENDL; + LL_VLOGS(level, "AppInit", "Directories") << " ExecutableFilename: " << getExecutableFilename() << LL_ENDL; + LL_VLOGS(level, "AppInit", "Directories") << " ExecutableDir: " << getExecutableDir() << LL_ENDL; + LL_VLOGS(level, "AppInit", "Directories") << " ExecutablePathAndName: " << getExecutablePathAndName() << LL_ENDL; + LL_VLOGS(level, "AppInit", "Directories") << " WorkingDir: " << getWorkingDir() << LL_ENDL; + LL_VLOGS(level, "AppInit", "Directories") << " AppRODataDir: " << getAppRODataDir() << LL_ENDL; + LL_VLOGS(level, "AppInit", "Directories") << " OSUserDir: " << getOSUserDir() << LL_ENDL; + LL_VLOGS(level, "AppInit", "Directories") << " OSUserAppDir: " << getOSUserAppDir() << LL_ENDL; + LL_VLOGS(level, "AppInit", "Directories") << " LindenUserDir: " << getLindenUserDir() << LL_ENDL; + LL_VLOGS(level, "AppInit", "Directories") << " TempDir: " << getTempDir() << LL_ENDL; + LL_VLOGS(level, "AppInit", "Directories") << " CAFile: " << getCAFile() << LL_ENDL; + LL_VLOGS(level, "AppInit", "Directories") << " SkinBaseDir: " << getSkinBaseDir() << LL_ENDL; + LL_VLOGS(level, "AppInit", "Directories") << " SkinDir: " << getSkinDir() << LL_ENDL; +} + +void LLDir::append(std::string& destpath, const std::string& name) const +{ + // Delegate question of whether we need a separator to helper method. + SepOff sepoff(needSep(destpath, name)); + if (sepoff.first) // do we need a separator? + { + destpath += mDirDelimiter; + } + // If destpath ends with a separator, AND name starts with one, skip + // name's leading separator. + destpath += name.substr(sepoff.second); +} + +LLDir::SepOff LLDir::needSep(const std::string& path, const std::string& name) const +{ + if (path.empty() || name.empty()) + { + // If either path or name are empty, we do not need a separator + // between them. + return SepOff(false, 0); + } + // Here we know path and name are both non-empty. But if path already ends + // with a separator, or if name already starts with a separator, we need + // not add one. + std::string::size_type seplen(mDirDelimiter.length()); + bool path_ends_sep(path.substr(path.length() - seplen) == mDirDelimiter); + bool name_starts_sep(name.substr(0, seplen) == mDirDelimiter); + if ((! path_ends_sep) && (! name_starts_sep)) + { + // If neither path nor name brings a separator to the junction, then + // we need one. + return SepOff(true, 0); + } + if (path_ends_sep && name_starts_sep) + { + // But if BOTH path and name bring a separator, we need not add one. + // Moreover, we should actually skip the leading separator of 'name'. + return SepOff(false, (unsigned short)seplen); + } + // Here we know that either path_ends_sep or name_starts_sep is true -- + // but not both. So don't add a separator, and don't skip any characters: + // simple concatenation will do the trick. + return SepOff(false, 0); +} + +void dir_exists_or_crash(const std::string &dir_name) +{ +#if LL_WINDOWS + // *FIX: lame - it doesn't do the same thing on windows. not so + // important since we don't deploy simulator to windows boxes. + LLFile::mkdir(dir_name, 0700); +#else + struct stat dir_stat; + if(0 != LLFile::stat(dir_name, &dir_stat)) + { + S32 stat_rv = errno; + if(ENOENT == stat_rv) + { + if(0 != LLFile::mkdir(dir_name, 0700)) // octal + { + LL_ERRS() << "Unable to create directory: " << dir_name << LL_ENDL; + } + } + else + { + LL_ERRS() << "Unable to stat: " << dir_name << " errno = " << stat_rv + << LL_ENDL; + } + } + else + { + // data_dir exists, make sure it's a directory. + if(!S_ISDIR(dir_stat.st_mode)) + { + LL_ERRS() << "Data directory collision: " << dir_name << LL_ENDL; + } + } +#endif +} diff --git a/indra/llfilesystem/lldir.h b/indra/llfilesystem/lldir.h new file mode 100644 index 0000000000..38e204ef04 --- /dev/null +++ b/indra/llfilesystem/lldir.h @@ -0,0 +1,280 @@ +/** + * @file lldir.h + * @brief Definition of directory utilities class + * + * $LicenseInfo:firstyear=2000&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2010, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef LL_LLDIR_H +#define LL_LLDIR_H + +#if LL_SOLARIS +#include +#define MAX_PATH MAXPATHLEN +#endif + +// these numbers are read from settings_files.xml, so we need to be explicit +typedef enum ELLPath +{ + LL_PATH_NONE = 0, + LL_PATH_USER_SETTINGS = 1, + LL_PATH_APP_SETTINGS = 2, + LL_PATH_PER_SL_ACCOUNT = 3, // returns/expands to blank string if we don't know the account name yet + LL_PATH_CACHE = 4, + LL_PATH_CHARACTER = 5, + LL_PATH_HELP = 6, + LL_PATH_LOGS = 7, + LL_PATH_TEMP = 8, + LL_PATH_SKINS = 9, + LL_PATH_TOP_SKIN = 10, + LL_PATH_CHAT_LOGS = 11, + LL_PATH_PER_ACCOUNT_CHAT_LOGS = 12, + LL_PATH_USER_SKIN = 14, + LL_PATH_LOCAL_ASSETS = 15, + LL_PATH_EXECUTABLE = 16, + LL_PATH_DEFAULT_SKIN = 17, + LL_PATH_FONTS = 18, + LL_PATH_DUMP = 19, + LL_PATH_LAST +} ELLPath; + +/// Directory operations +class LLDir +{ + public: + LLDir(); + virtual ~LLDir(); + + // app_name - Usually SecondLife, used for creating settings directories + // in OS-specific location, such as C:\Documents and Settings + // app_read_only_data_dir - Usually the source code directory, used + // for test applications to read newview data files. + virtual void initAppDirs(const std::string &app_name, + const std::string& app_read_only_data_dir = "") = 0; + + virtual S32 deleteFilesInDir(const std::string &dirname, const std::string &mask); + U32 deleteDirAndContents(const std::string& dir_name); + std::vector getFilesInDir(const std::string &dirname); +// pure virtual functions + virtual std::string getCurPath() = 0; + virtual bool fileExists(const std::string &filename) const = 0; + + const std::string findFile(const std::string& filename, const std::vector filenames) const; + const std::string findFile(const std::string& filename, const std::string& searchPath1 = "", const std::string& searchPath2 = "", const std::string& searchPath3 = "") const; + + virtual std::string getLLPluginLauncher() = 0; // full path and name for the plugin shell + virtual std::string getLLPluginFilename(std::string base_name) = 0; // full path and name to the plugin DSO for this base_name (i.e. 'FOO' -> '/bar/baz/libFOO.so') + + const std::string &getExecutablePathAndName() const; // Full pathname of the executable + const std::string &getAppName() const; // install directory under progams/ ie "SecondLife" + const std::string &getExecutableDir() const; // Directory where the executable is located + const std::string &getExecutableFilename() const;// Filename of .exe + const std::string &getWorkingDir() const; // Current working directory + const std::string &getAppRODataDir() const; // Location of read-only data files + const std::string &getOSUserDir() const; // Location of the os-specific user dir + const std::string &getOSUserAppDir() const; // Location of the os-specific user app dir + const std::string &getLindenUserDir() const; // Location of the Linden user dir. + const std::string &getChatLogsDir() const; // Location of the chat logs dir. + const std::string &getDumpDir() const; // Location of the per-run dump dir. + const std::string &getPerAccountChatLogsDir() const; // Location of the per account chat logs dir. + const std::string &getTempDir() const; // Common temporary directory + const std::string getCacheDir(bool get_default = false) const; // Location of the cache. + const std::string &getOSCacheDir() const; // location of OS-specific cache folder (may be empty string) + const std::string &getCAFile() const; // File containing TLS certificate authorities + const std::string &getDirDelimiter() const; // directory separator for platform (ie. '\' or '/' or ':') + const std::string &getDefaultSkinDir() const; // folder for default skin. e.g. c:\program files\second life\skins\default + const std::string &getSkinDir() const; // User-specified skin folder. + const std::string &getUserDefaultSkinDir() const; // dir with user modifications to default skin + const std::string &getUserSkinDir() const; // User-specified skin folder with user modifications. e.g. c:\documents and settings\username\application data\second life\skins\curskin + const std::string getSkinBaseDir() const; // folder that contains all installed skins (not user modifications). e.g. c:\program files\second life\skins + const std::string &getLLPluginDir() const; // Directory containing plugins and plugin shell + const std::string &getUserName() const; + + // Expanded filename + std::string getExpandedFilename(ELLPath location, const std::string &filename) const; + std::string getExpandedFilename(ELLPath location, const std::string &subdir, const std::string &filename) const; + std::string getExpandedFilename(ELLPath location, const std::string &subdir1, const std::string &subdir2, const std::string &filename) const; + + // Base and Directory name extraction + std::string getBaseFileName(const std::string& filepath, bool strip_exten = false) const; + std::string getDirName(const std::string& filepath) const; + std::string getExtension(const std::string& filepath) const; // Excludes '.', e.g getExtension("foo.wav") == "wav" + + // these methods search the various skin paths for the specified file in the following order: + // getUserSkinDir(), getUserDefaultSkinDir(), getSkinDir(), getDefaultSkinDir() + /// param value for findSkinnedFilenames(), explained below + enum ESkinConstraint { CURRENT_SKIN, ALL_SKINS }; + /** + * Given a filename within skin, return an ordered sequence of paths to + * search. Nonexistent files will be filtered out -- which means that the + * vector might be empty. + * + * @param subdir Identify top-level skin subdirectory by passing one of + * LLDir::XUI (file lives under "xui" subtree), LLDir::TEXTURES (file + * lives under "textures" subtree), LLDir::SKINBASE (file lives at top + * level of skin subdirectory). + * @param filename Desired filename within subdir within skin, e.g. + * "panel_login.xml". DO NOT prepend (e.g.) "xui" or the desired language. + * @param constraint Callers perform two different kinds of processing. + * When fetching a XUI file, for instance, the existence of @a filename in + * the specified skin completely supercedes any @a filename in the default + * skin. For that case, leave the default @a constraint=CURRENT_SKIN. The + * returned vector will contain only + * ".../current_skin/xui/en/filename", + * ".../current_skin/xui/current_language/filename". + * But for (e.g.) "strings.xml", we want a given skin to be able to + * override only specific entries from the default skin. Any string not + * defined in the specified skin will be sought in the default skin. For + * that case, pass @a constraint=ALL_SKINS. The returned vector will + * contain at least ".../default/xui/en/strings.xml", + * ".../default/xui/current_language/strings.xml", + * ".../current_skin/xui/en/strings.xml", + * ".../current_skin/xui/current_language/strings.xml". + */ + std::vector findSkinnedFilenames(const std::string& subdir, + const std::string& filename, + ESkinConstraint constraint=CURRENT_SKIN) const; + /// Values for findSkinnedFilenames(subdir) parameter + static const char *XUI, *TEXTURES, *SKINBASE; + /** + * Return the base-language pathname from findSkinnedFilenames(), or + * the empty string if no such file exists. Parameters are identical to + * findSkinnedFilenames(). This is shorthand for capturing the vector + * returned by findSkinnedFilenames(), checking for empty() and then + * returning front(). + */ + std::string findSkinnedFilenameBaseLang(const std::string &subdir, + const std::string &filename, + ESkinConstraint constraint=CURRENT_SKIN) const; + /** + * Return the "most localized" pathname from findSkinnedFilenames(), or + * the empty string if no such file exists. Parameters are identical to + * findSkinnedFilenames(). This is shorthand for capturing the vector + * returned by findSkinnedFilenames(), checking for empty() and then + * returning back(). + */ + std::string findSkinnedFilename(const std::string &subdir, + const std::string &filename, + ESkinConstraint constraint=CURRENT_SKIN) const; + + // random filename in common temporary directory + std::string getTempFilename() const; + + // For producing safe download file names from potentially unsafe ones + static std::string getScrubbedFileName(const std::string uncleanFileName); + static std::string getForbiddenFileChars(); + void setDumpDir( const std::string& path ); + + + virtual void setChatLogsDir(const std::string &path); // Set the chat logs dir to this user's dir + virtual void setPerAccountChatLogsDir(const std::string &username); // Set the per user chat log directory. + virtual void setLindenUserDir(const std::string &username); // Set the linden user dir to this user's dir + virtual void setSkinFolder(const std::string &skin_folder, const std::string& language); + virtual std::string getSkinFolder() const; + virtual std::string getLanguage() const; + virtual bool setCacheDir(const std::string &path); + virtual void updatePerAccountChatLogsDir(); + + virtual void dumpCurrentDirectories(LLError::ELevel level = LLError::LEVEL_DEBUG); + + // Utility routine + std::string buildSLOSCacheDir() const; + + /// Append specified @a name to @a destpath, separated by getDirDelimiter() + /// if both are non-empty. + void append(std::string& destpath, const std::string& name) const; + /// Variadic form: append @a name0 and @a name1 and arbitrary other @a + /// names to @a destpath, separated by getDirDelimiter() as needed. + template + void append(std::string& destpath, const std::string& name0, const std::string& name1, + const NAMES& ... names) const + { + // In a typical recursion case, we'd accept (destpath, name0, names). + // We accept (destpath, name0, name1, names) because it's important to + // delegate the two-argument case to the non-template implementation. + append(destpath, name0); + append(destpath, name1, names...); + } + + /// Append specified @a names to @a path, separated by getDirDelimiter() + /// as needed. Return result, leaving @a path unmodified. + template + std::string add(const std::string& path, const NAMES& ... names) const + { + std::string destpath(path); + append(destpath, names...); + return destpath; + } + +protected: + // Does an add() or append() call need a directory delimiter? + typedef std::pair SepOff; + SepOff needSep(const std::string& path, const std::string& name) const; + // build mSearchSkinDirs without adding duplicates + void addSearchSkinDir(const std::string& skindir); + + // Internal to findSkinnedFilenames() + template + void walkSearchSkinDirs(const std::string& subdir, + const std::vector& subsubdirs, + const std::string& filename, + const FUNCTION& function) const; + + std::string mAppName; // install directory under progams/ ie "SecondLife" + std::string mExecutablePathAndName; // full path + Filename of .exe + std::string mExecutableFilename; // Filename of .exe + std::string mExecutableDir; // Location of executable + std::string mWorkingDir; // Current working directory + std::string mAppRODataDir; // Location for static app data + std::string mOSUserDir; // OS Specific user directory + std::string mOSUserAppDir; // OS Specific user app directory + std::string mLindenUserDir; // Location for Linden user-specific data + std::string mPerAccountChatLogsDir; // Location for chat logs. + std::string mChatLogsDir; // Location for chat logs. + std::string mCAFile; // Location of the TLS certificate authority PEM file. + std::string mTempDir; + std::string mCacheDir; // cache directory as set by user preference + std::string mDefaultCacheDir; // default cache diretory + std::string mOSCacheDir; // operating system cache dir + std::string mDirDelimiter; + std::string mSkinName; // caller-specified skin name + std::string mSkinBaseDir; // Base for skins paths. + std::string mDefaultSkinDir; // Location for default skin info. + std::string mSkinDir; // Location for current skin info. + std::string mUserDefaultSkinDir; // Location for default skin info. + std::string mUserSkinDir; // Location for user-modified skin info. + // Skin directories to search, most general to most specific. This order + // works well for composing fine-grained files, in which an individual item + // in a specific file overrides the corresponding item in more general + // files. Of course, for a file-level search, iterate backwards. + std::vector mSearchSkinDirs; + std::string mLanguage; // Current viewer language + std::string mLLPluginDir; // Location for plugins and plugin shell + static std::string sDumpDir; // Per-run crash report subdir of log directory. + std::string mUserName; // Current user name +}; + +void dir_exists_or_crash(const std::string &dir_name); + +extern LLDir *gDirUtilp; + +#endif // LL_LLDIR_H diff --git a/indra/llfilesystem/lldir_linux.cpp b/indra/llfilesystem/lldir_linux.cpp new file mode 100644 index 0000000000..80ad05345a --- /dev/null +++ b/indra/llfilesystem/lldir_linux.cpp @@ -0,0 +1,269 @@ +/** + * @file lldir_linux.cpp + * @brief Implementation of directory utilities for linux + * + * $LicenseInfo:firstyear=2002&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2010, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include "linden_common.h" + +#include "lldir_linux.h" +#include "llerror.h" +#include "llrand.h" +#include "llstring.h" +#include +#include +#include +#include +#include + + +static std::string getCurrentUserHome(char* fallback) +{ + const uid_t uid = getuid(); + struct passwd *pw; + + pw = getpwuid(uid); + if ((pw != NULL) && (pw->pw_dir != NULL)) + { + return pw->pw_dir; + } + + LL_INFOS() << "Couldn't detect home directory from passwd - trying $HOME" << LL_ENDL; + auto home_env = LLStringUtil::getoptenv("HOME"); + if (home_env) + { + return *home_env; + } + else + { + LL_WARNS() << "Couldn't detect home directory! Falling back to " << fallback << LL_ENDL; + return fallback; + } +} + + +LLDir_Linux::LLDir_Linux() +{ + mDirDelimiter = "/"; + mCurrentDirIndex = -1; + mCurrentDirCount = -1; + mDirp = NULL; + + char tmp_str[LL_MAX_PATH]; /* Flawfinder: ignore */ + if (getcwd(tmp_str, LL_MAX_PATH) == NULL) + { + strcpy(tmp_str, "/tmp"); + LL_WARNS() << "Could not get current directory; changing to " + << tmp_str << LL_ENDL; + if (chdir(tmp_str) == -1) + { + LL_ERRS() << "Could not change directory to " << tmp_str << LL_ENDL; + } + } + + mExecutableFilename = ""; + mExecutablePathAndName = ""; + mExecutableDir = tmp_str; + mWorkingDir = tmp_str; +#ifdef APP_RO_DATA_DIR + mAppRODataDir = APP_RO_DATA_DIR; +#else + mAppRODataDir = tmp_str; +#endif + std::string::size_type build_dir_pos = mExecutableDir.rfind("/build-linux-"); + if (build_dir_pos != std::string::npos) + { + // ...we're in a dev checkout + mSkinBaseDir = mExecutableDir.substr(0, build_dir_pos) + "/indra/newview/skins"; + LL_INFOS() << "Running in dev checkout with mSkinBaseDir " + << mSkinBaseDir << LL_ENDL; + } + else + { + // ...normal installation running + mSkinBaseDir = mAppRODataDir + mDirDelimiter + "skins"; + } + + mOSUserDir = getCurrentUserHome(tmp_str); + mOSUserAppDir = ""; + mLindenUserDir = ""; + + char path [32]; /* Flawfinder: ignore */ + + // *NOTE: /proc/%d/exe doesn't work on FreeBSD. But that's ok, + // because this is the linux implementation. + + snprintf (path, sizeof(path), "/proc/%d/exe", (int) getpid ()); + int rc = readlink (path, tmp_str, sizeof (tmp_str)-1); /* Flawfinder: ignore */ + if ( (rc != -1) && (rc <= ((int) sizeof (tmp_str)-1)) ) + { + tmp_str[rc] = '\0'; //readlink() doesn't 0-terminate the buffer + mExecutablePathAndName = tmp_str; + char *path_end; + if ((path_end = strrchr(tmp_str,'/'))) + { + *path_end = '\0'; + mExecutableDir = tmp_str; + mWorkingDir = tmp_str; + mExecutableFilename = path_end+1; + } + else + { + mExecutableFilename = tmp_str; + } + } + + mLLPluginDir = mExecutableDir + mDirDelimiter + "llplugin"; + + // *TODO: don't use /tmp, use $HOME/.secondlife/tmp or something. + mTempDir = "/tmp"; +} + +LLDir_Linux::~LLDir_Linux() +{ +} + +// Implementation + + +void LLDir_Linux::initAppDirs(const std::string &app_name, + const std::string& app_read_only_data_dir) +{ + // Allow override so test apps can read newview directory + if (!app_read_only_data_dir.empty()) + { + mAppRODataDir = app_read_only_data_dir; + mSkinBaseDir = add(mAppRODataDir, "skins"); + } + mAppName = app_name; + + std::string upper_app_name(app_name); + LLStringUtil::toUpper(upper_app_name); + + auto app_home_env(LLStringUtil::getoptenv(upper_app_name + "_USER_DIR")); + if (app_home_env) + { + // user has specified own userappdir i.e. $SECONDLIFE_USER_DIR + mOSUserAppDir = *app_home_env; + } + else + { + // traditionally on unixoids, MyApp gets ~/.myapp dir for data + mOSUserAppDir = mOSUserDir; + mOSUserAppDir += "/"; + mOSUserAppDir += "."; + std::string lower_app_name(app_name); + LLStringUtil::toLower(lower_app_name); + mOSUserAppDir += lower_app_name; + } + + // create any directories we expect to write to. + + int res = LLFile::mkdir(mOSUserAppDir); + if (res == -1) + { + LL_WARNS() << "Couldn't create app user dir " << mOSUserAppDir << LL_ENDL; + LL_WARNS() << "Default to base dir" << mOSUserDir << LL_ENDL; + mOSUserAppDir = mOSUserDir; + } + + res = LLFile::mkdir(getExpandedFilename(LL_PATH_LOGS,"")); + if (res == -1) + { + LL_WARNS() << "Couldn't create LL_PATH_LOGS dir " << getExpandedFilename(LL_PATH_LOGS,"") << LL_ENDL; + } + + res = LLFile::mkdir(getExpandedFilename(LL_PATH_USER_SETTINGS,"")); + if (res == -1) + { + LL_WARNS() << "Couldn't create LL_PATH_USER_SETTINGS dir " << getExpandedFilename(LL_PATH_USER_SETTINGS,"") << LL_ENDL; + } + + res = LLFile::mkdir(getExpandedFilename(LL_PATH_CACHE,"")); + if (res == -1) + { + LL_WARNS() << "Couldn't create LL_PATH_CACHE dir " << getExpandedFilename(LL_PATH_CACHE,"") << LL_ENDL; + } + + mCAFile = getExpandedFilename(LL_PATH_EXECUTABLE, "ca-bundle.crt"); +} + +U32 LLDir_Linux::countFilesInDir(const std::string &dirname, const std::string &mask) +{ + U32 file_count = 0; + glob_t g; + + std::string tmp_str; + tmp_str = dirname; + tmp_str += mask; + + if(glob(tmp_str.c_str(), GLOB_NOSORT, NULL, &g) == 0) + { + file_count = g.gl_pathc; + + globfree(&g); + } + + return (file_count); +} + +std::string LLDir_Linux::getCurPath() +{ + char tmp_str[LL_MAX_PATH]; /* Flawfinder: ignore */ + if (getcwd(tmp_str, LL_MAX_PATH) == NULL) + { + LL_WARNS() << "Could not get current directory" << LL_ENDL; + tmp_str[0] = '\0'; + } + return tmp_str; +} + + +bool LLDir_Linux::fileExists(const std::string &filename) const +{ + struct stat stat_data; + // Check the age of the file + // Now, we see if the files we've gathered are recent... + int res = stat(filename.c_str(), &stat_data); + if (!res) + { + return TRUE; + } + else + { + return FALSE; + } +} + + +/*virtual*/ std::string LLDir_Linux::getLLPluginLauncher() +{ + return gDirUtilp->getExecutableDir() + gDirUtilp->getDirDelimiter() + + "SLPlugin"; +} + +/*virtual*/ std::string LLDir_Linux::getLLPluginFilename(std::string base_name) +{ + return gDirUtilp->getLLPluginDir() + gDirUtilp->getDirDelimiter() + + "lib" + base_name + ".so"; +} diff --git a/indra/llfilesystem/lldir_linux.h b/indra/llfilesystem/lldir_linux.h new file mode 100644 index 0000000000..e83a020ba4 --- /dev/null +++ b/indra/llfilesystem/lldir_linux.h @@ -0,0 +1,64 @@ +/** + * @file lldir_linux.h + * @brief Definition of directory utilities class for linux + * + * $LicenseInfo:firstyear=2000&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2010, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#if !LL_LINUX +#error This header must not be included when compiling for any target other than Linux. Consider including lldir.h instead. +#endif // !LL_LINUX + +#ifndef LL_LLDIR_LINUX_H +#define LL_LLDIR_LINUX_H + +#include "lldir.h" + +#include +#include + +class LLDir_Linux : public LLDir +{ +public: + LLDir_Linux(); + virtual ~LLDir_Linux(); + + /*virtual*/ void initAppDirs(const std::string &app_name, + const std::string& app_read_only_data_dir); + + virtual std::string getCurPath(); + virtual U32 countFilesInDir(const std::string &dirname, const std::string &mask); + /*virtual*/ bool fileExists(const std::string &filename) const; + + /*virtual*/ std::string getLLPluginLauncher(); + /*virtual*/ std::string getLLPluginFilename(std::string base_name); + +private: + DIR *mDirp; + int mCurrentDirIndex; + int mCurrentDirCount; + std::string mCurrentDir; +}; + +#endif // LL_LLDIR_LINUX_H + + diff --git a/indra/llfilesystem/lldir_mac.cpp b/indra/llfilesystem/lldir_mac.cpp new file mode 100644 index 0000000000..3bc4ee844e --- /dev/null +++ b/indra/llfilesystem/lldir_mac.cpp @@ -0,0 +1,205 @@ +/** + * @file lldir_mac.cpp + * @brief Implementation of directory utilities for Mac OS X + * + * $LicenseInfo:firstyear=2002&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2010, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#if LL_DARWIN + +#include "linden_common.h" + +#include "lldir_mac.h" +#include "llerror.h" +#include "llrand.h" +#include +#include +#include +#include +#include +#include "lldir_utils_objc.h" + +// -------------------------------------------------------------------------------- + +static bool CreateDirectory(const std::string &parent, + const std::string &child, + std::string *fullname) +{ + + boost::filesystem::path p(parent); + p /= child; + + if (fullname) + *fullname = std::string(p.string()); + + if (! boost::filesystem::create_directory(p)) + { + return (boost::filesystem::is_directory(p)); + } + return true; +} + +// -------------------------------------------------------------------------------- + +LLDir_Mac::LLDir_Mac() +{ + mDirDelimiter = "/"; + + const std::string secondLifeString = "SecondLife"; + + std::string *executablepathstr = getSystemExecutableFolder(); + + //NOTE: LLINFOS/LLERRS will not output to log here. The streams are not initialized. + + if (executablepathstr) + { + // mExecutablePathAndName + mExecutablePathAndName = *executablepathstr; + + boost::filesystem::path executablepath(*executablepathstr); + +# ifndef BOOST_SYSTEM_NO_DEPRECATED +#endif + mExecutableFilename = executablepath.filename().string(); + mExecutableDir = executablepath.parent_path().string(); + + // mAppRODataDir + std::string *resourcepath = getSystemResourceFolder(); + mAppRODataDir = *resourcepath; + + // *NOTE: When running in a dev tree, use the copy of + // skins in indra/newview/ rather than in the application bundle. This + // mirrors Windows dev environment behavior and allows direct checkin + // of edited skins/xui files. JC + + // MBW -- This keeps the mac application from finding other things. + // If this is really for skins, it should JUST apply to skins. + + std::string::size_type build_dir_pos = mExecutableDir.rfind("/build-darwin-"); + if (build_dir_pos != std::string::npos) + { + // ...we're in a dev checkout + mSkinBaseDir = mExecutableDir.substr(0, build_dir_pos) + + "/indra/newview/skins"; + LL_INFOS() << "Running in dev checkout with mSkinBaseDir " + << mSkinBaseDir << LL_ENDL; + } + else + { + // ...normal installation running + mSkinBaseDir = mAppRODataDir + mDirDelimiter + "skins"; + } + + // mOSUserDir + std::string *appdir = getSystemApplicationSupportFolder(); + std::string rootdir; + + //Create root directory + if (CreateDirectory(*appdir, secondLifeString, &rootdir)) + { + + // Save the full path to the folder + mOSUserDir = rootdir; + + // Create our sub-dirs + CreateDirectory(rootdir, std::string("data"), NULL); + CreateDirectory(rootdir, std::string("logs"), NULL); + CreateDirectory(rootdir, std::string("user_settings"), NULL); + CreateDirectory(rootdir, std::string("browser_profile"), NULL); + } + + //mOSCacheDir + std::string *cachedir = getSystemCacheFolder(); + + if (cachedir) + + { + mOSCacheDir = *cachedir; + //TODO: This changes from ~/Library/Cache/Secondlife to ~/Library/Cache/com.app.secondlife/Secondlife. Last dir level could go away. + CreateDirectory(mOSCacheDir, secondLifeString, NULL); + } + + // mOSUserAppDir + mOSUserAppDir = mOSUserDir; + + // mTempDir + //Aura 120920 boost::filesystem::temp_directory_path() not yet implemented on mac. :( + std::string *tmpdir = getSystemTempFolder(); + if (tmpdir) + { + + CreateDirectory(*tmpdir, secondLifeString, &mTempDir); + if (tmpdir) delete tmpdir; + } + + mWorkingDir = getCurPath(); + + mLLPluginDir = mAppRODataDir + mDirDelimiter + "llplugin"; + } +} + +LLDir_Mac::~LLDir_Mac() +{ +} + +// Implementation + + +void LLDir_Mac::initAppDirs(const std::string &app_name, + const std::string& app_read_only_data_dir) +{ + // Allow override so test apps can read newview directory + if (!app_read_only_data_dir.empty()) + { + mAppRODataDir = app_read_only_data_dir; + mSkinBaseDir = add(mAppRODataDir, "skins"); + } + mCAFile = add(mAppRODataDir, "ca-bundle.crt"); +} + +std::string LLDir_Mac::getCurPath() +{ + return boost::filesystem::path( boost::filesystem::current_path() ).string(); +} + + + +bool LLDir_Mac::fileExists(const std::string &filename) const +{ + return boost::filesystem::exists(filename); +} + + +/*virtual*/ std::string LLDir_Mac::getLLPluginLauncher() +{ + return gDirUtilp->getAppRODataDir() + gDirUtilp->getDirDelimiter() + + "SLPlugin.app/Contents/MacOS/SLPlugin"; +} + +/*virtual*/ std::string LLDir_Mac::getLLPluginFilename(std::string base_name) +{ + return gDirUtilp->getLLPluginDir() + gDirUtilp->getDirDelimiter() + + base_name + ".dylib"; +} + + +#endif // LL_DARWIN diff --git a/indra/llfilesystem/lldir_mac.h b/indra/llfilesystem/lldir_mac.h new file mode 100644 index 0000000000..558727ebbc --- /dev/null +++ b/indra/llfilesystem/lldir_mac.h @@ -0,0 +1,56 @@ +/** + * @file lldir_mac.h + * @brief Definition of directory utilities class for Mac OS X + * + * $LicenseInfo:firstyear=2000&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2010, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#if !LL_DARWIN +#error This header must not be included when compiling for any target other than Mac OS. Consider including lldir.h instead. +#endif // !LL_DARWIN + +#ifndef LL_LLDIR_MAC_H +#define LL_LLDIR_MAC_H + +#include "lldir.h" + +#include + +class LLDir_Mac : public LLDir +{ +public: + LLDir_Mac(); + virtual ~LLDir_Mac(); + + /*virtual*/ void initAppDirs(const std::string &app_name, + const std::string& app_read_only_data_dir); + + virtual std::string getCurPath(); + virtual bool fileExists(const std::string &filename) const; + + /*virtual*/ std::string getLLPluginLauncher(); + /*virtual*/ std::string getLLPluginFilename(std::string base_name); +}; + +#endif // LL_LLDIR_MAC_H + + diff --git a/indra/llfilesystem/lldir_solaris.cpp b/indra/llfilesystem/lldir_solaris.cpp new file mode 100644 index 0000000000..f18560ff20 --- /dev/null +++ b/indra/llfilesystem/lldir_solaris.cpp @@ -0,0 +1,266 @@ +/** + * @file fmodwrapper.cpp + * @brief dummy source file for building a shared library to wrap libfmod.a + * + * $LicenseInfo:firstyear=2005&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2010, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include "linden_common.h" + +#include "lldir_solaris.h" +#include "llerror.h" +#include "llrand.h" +#include "llstring.h" +#include +#include +#include +#include +#include +#include +#define _STRUCTURED_PROC 1 +#include +#include + +static std::string getCurrentUserHome(char* fallback) +{ + // fwiw this exactly duplicates getCurrentUserHome() in lldir_linux.cpp... + // we should either derive both from LLDir_Posix or just axe Solaris. + const uid_t uid = getuid(); + struct passwd *pw; + + pw = getpwuid(uid); + if ((pw != NULL) && (pw->pw_dir != NULL)) + { + return pw->pw_dir; + } + + LL_INFOS() << "Couldn't detect home directory from passwd - trying $HOME" << LL_ENDL; + auto home_env = LLStringUtil::getoptenv("HOME"); + if (home_env) + { + return *home_env; + } + else + { + LL_WARNS() << "Couldn't detect home directory! Falling back to " << fallback << LL_ENDL; + return fallback; + } +} + + +LLDir_Solaris::LLDir_Solaris() +{ + mDirDelimiter = "/"; + mCurrentDirIndex = -1; + mCurrentDirCount = -1; + mDirp = NULL; + + char tmp_str[LL_MAX_PATH]; /* Flawfinder: ignore */ + if (getcwd(tmp_str, LL_MAX_PATH) == NULL) + { + strcpy(tmp_str, "/tmp"); + LL_WARNS() << "Could not get current directory; changing to " + << tmp_str << LL_ENDL; + if (chdir(tmp_str) == -1) + { + LL_ERRS() << "Could not change directory to " << tmp_str << LL_ENDL; + } + } + + mExecutableFilename = ""; + mExecutablePathAndName = ""; + mExecutableDir = strdup(tmp_str); + mWorkingDir = strdup(tmp_str); + mAppRODataDir = strdup(tmp_str); + mOSUserDir = getCurrentUserHome(tmp_str); + mOSUserAppDir = ""; + mLindenUserDir = ""; + + char path [LL_MAX_PATH]; /* Flawfinder: ignore */ + + sprintf(path, "/proc/%d/psinfo", (int)getpid()); + int proc_fd = -1; + if((proc_fd = open(path, O_RDONLY)) == -1){ + LL_WARNS() << "unable to open " << path << LL_ENDL; + return; + } + psinfo_t proc_psinfo; + if(read(proc_fd, &proc_psinfo, sizeof(psinfo_t)) != sizeof(psinfo_t)){ + LL_WARNS() << "Unable to read " << path << LL_ENDL; + close(proc_fd); + return; + } + + close(proc_fd); + + mExecutableFilename = strdup(proc_psinfo.pr_fname); + LL_INFOS() << "mExecutableFilename = [" << mExecutableFilename << "]" << LL_ENDL; + + sprintf(path, "/proc/%d/path/a.out", (int)getpid()); + + char execpath[LL_MAX_PATH]; + if(readlink(path, execpath, LL_MAX_PATH) == -1){ + LL_WARNS() << "Unable to read link from " << path << LL_ENDL; + return; + } + + char *p = execpath; // nuke trash in link, if any exists + int i = 0; + while(*p != NULL && ++i < LL_MAX_PATH && isprint((int)(*p++))); + *p = NULL; + + mExecutablePathAndName = strdup(execpath); + LL_INFOS() << "mExecutablePathAndName = [" << mExecutablePathAndName << "]" << LL_ENDL; + + //NOTE: Why force people to cd into the package directory? + // Look for SECONDLIFE env variable and use it, if set. + + auto SECONDLIFE(LLDirUtil::getoptenv("SECONDLIFE")); + if(SECONDLIFE){ + mExecutableDir = add(*SECONDLIFE, "bin"); //NOTE: make sure we point at the bin + }else{ + mExecutableDir = getDirName(execpath); + LL_INFOS() << "mExecutableDir = [" << mExecutableDir << "]" << LL_ENDL; + } + + mLLPluginDir = add(mExecutableDir, "llplugin"); + + // *TODO: don't use /tmp, use $HOME/.secondlife/tmp or something. + mTempDir = "/tmp"; +} + +LLDir_Solaris::~LLDir_Solaris() +{ +} + +// Implementation + + +void LLDir_Solaris::initAppDirs(const std::string &app_name, + const std::string& app_read_only_data_dir) +{ + // Allow override so test apps can read newview directory + if (!app_read_only_data_dir.empty()) + { + mAppRODataDir = app_read_only_data_dir; + mSkinBaseDir = add(mAppRODataDir, "skins"); + } + mAppName = app_name; + + std::string upper_app_name(app_name); + LLStringUtil::toUpper(upper_app_name); + + auto app_home_env(LLStringUtil::getoptenv(upper_app_name + "_USER_DIR")); + if (app_home_env) + { + // user has specified own userappdir i.e. $SECONDLIFE_USER_DIR + mOSUserAppDir = *app_home_env; + } + else + { + // traditionally on unixoids, MyApp gets ~/.myapp dir for data + mOSUserAppDir = mOSUserDir; + mOSUserAppDir += "/"; + mOSUserAppDir += "."; + std::string lower_app_name(app_name); + LLStringUtil::toLower(lower_app_name); + mOSUserAppDir += lower_app_name; + } + + // create any directories we expect to write to. + + int res = LLFile::mkdir(mOSUserAppDir); + if (res == -1) + { + LL_WARNS() << "Couldn't create app user dir " << mOSUserAppDir << LL_ENDL; + LL_WARNS() << "Default to base dir" << mOSUserDir << LL_ENDL; + mOSUserAppDir = mOSUserDir; + } + + res = LLFile::mkdir(getExpandedFilename(LL_PATH_LOGS,"")); + if (res == -1) + { + LL_WARNS() << "Couldn't create LL_PATH_LOGS dir " << getExpandedFilename(LL_PATH_LOGS,"") << LL_ENDL; + } + + res = LLFile::mkdir(getExpandedFilename(LL_PATH_USER_SETTINGS,"")); + if (res == -1) + { + LL_WARNS() << "Couldn't create LL_PATH_USER_SETTINGS dir " << getExpandedFilename(LL_PATH_USER_SETTINGS,"") << LL_ENDL; + } + + res = LLFile::mkdir(getExpandedFilename(LL_PATH_CACHE,"")); + if (res == -1) + { + LL_WARNS() << "Couldn't create LL_PATH_CACHE dir " << getExpandedFilename(LL_PATH_CACHE,"") << LL_ENDL; + } + + mCAFile = getExpandedFilename(LL_PATH_EXECUTABLE, "ca-bundle.crt"); +} + +U32 LLDir_Solaris::countFilesInDir(const std::string &dirname, const std::string &mask) +{ + U32 file_count = 0; + glob_t g; + + std::string tmp_str; + tmp_str = dirname; + tmp_str += mask; + + if(glob(tmp_str.c_str(), GLOB_NOSORT, NULL, &g) == 0) + { + file_count = g.gl_pathc; + + globfree(&g); + } + + return (file_count); +} + +std::string LLDir_Solaris::getCurPath() +{ + char tmp_str[LL_MAX_PATH]; /* Flawfinder: ignore */ + if (getcwd(tmp_str, LL_MAX_PATH) == NULL) + { + LL_WARNS() << "Could not get current directory" << LL_ENDL; + tmp_str[0] = '\0'; + } + return tmp_str; +} + + +bool LLDir_Solaris::fileExists(const std::string &filename) const +{ + struct stat stat_data; + // Check the age of the file + // Now, we see if the files we've gathered are recent... + int res = stat(filename.c_str(), &stat_data); + if (!res) + { + return TRUE; + } + else + { + return FALSE; + } +} + diff --git a/indra/llfilesystem/lldir_solaris.h b/indra/llfilesystem/lldir_solaris.h new file mode 100644 index 0000000000..c6dac57e14 --- /dev/null +++ b/indra/llfilesystem/lldir_solaris.h @@ -0,0 +1,61 @@ +/** + * @file fmodwrapper.cpp + * @brief dummy source file for building a shared library to wrap libfmod.a + * + * $LicenseInfo:firstyear=2005&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2010, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#if !LL_SOLARIS +#error This header must not be included when compiling for any target other than Solaris. Consider including lldir.h instead. +#endif // !LL_SOLARIS + +#ifndef LL_LLDIR_SOLARIS_H +#define LL_LLDIR_SOLARIS_H + +#include "lldir.h" + +#include +#include + +class LLDir_Solaris : public LLDir +{ +public: + LLDir_Solaris(); + virtual ~LLDir_Solaris(); + + /*virtual*/ void initAppDirs(const std::string &app_name, + const std::string& app_read_only_data_dir); + + virtual std::string getCurPath(); + virtual U32 countFilesInDir(const std::string &dirname, const std::string &mask); + /*virtual*/ bool fileExists(const std::string &filename) const; + +private: + DIR *mDirp; + int mCurrentDirIndex; + int mCurrentDirCount; + std::string mCurrentDir; +}; + +#endif // LL_LLDIR_SOLARIS_H + + diff --git a/indra/llfilesystem/lldir_utils_objc.h b/indra/llfilesystem/lldir_utils_objc.h new file mode 100644 index 0000000000..12019c4284 --- /dev/null +++ b/indra/llfilesystem/lldir_utils_objc.h @@ -0,0 +1,43 @@ +/** + * @file lldir_utils_objc.h + * @brief Definition of directory utilities class for Mac OS X + * + * $LicenseInfo:firstyear=2020&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2020, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#if !LL_DARWIN +#error This header must not be included when compiling for any target other than Mac OS. Consider including lldir.h instead. +#endif // !LL_DARWIN + +#ifndef LL_LLDIR_UTILS_OBJC_H +#define LL_LLDIR_UTILS_OBJC_H + +#include + +std::string* getSystemTempFolder(); +std::string* getSystemCacheFolder(); +std::string* getSystemApplicationSupportFolder(); +std::string* getSystemResourceFolder(); +std::string* getSystemExecutableFolder(); + + +#endif // LL_LLDIR_UTILS_OBJC_H diff --git a/indra/llfilesystem/lldir_utils_objc.mm b/indra/llfilesystem/lldir_utils_objc.mm new file mode 100644 index 0000000000..da55a2f897 --- /dev/null +++ b/indra/llfilesystem/lldir_utils_objc.mm @@ -0,0 +1,108 @@ +/** + * @file lldir_utils_objc.mm + * @brief Cocoa implementation of directory utilities for Mac OS X + * + * $LicenseInfo:firstyear=2020&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2020, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ +#if LL_DARWIN + +//WARNING: This file CANNOT use standard linden includes due to conflicts between definitions of BOOL + +#include "lldir_utils_objc.h" +#import + +std::string* getSystemTempFolder() +{ + NSAutoreleasePool *pool = [[NSAutoreleasePool alloc] init]; + NSString * tempDir = NSTemporaryDirectory(); + if (tempDir == nil) + tempDir = @"/tmp"; + std::string *result = ( new std::string([tempDir UTF8String]) ); + [pool release]; + + return result; +} + +//findSystemDirectory scoped exclusively to this file. +std::string* findSystemDirectory(NSSearchPathDirectory searchPathDirectory, + NSSearchPathDomainMask domainMask) +{ + NSAutoreleasePool *pool = [[NSAutoreleasePool alloc] init]; + + std::string *result = nil; + NSString *path = nil; + + // Search for the path + NSArray* paths = NSSearchPathForDirectoriesInDomains(searchPathDirectory, + domainMask, + YES); + if ([paths count]) + { + path = [paths objectAtIndex:0]; + //HACK: Always attempt to create directory, ignore errors. + NSError *error = nil; + + [[NSFileManager defaultManager] createDirectoryAtPath:path withIntermediateDirectories:YES attributes:nil error:&error]; + + + result = new std::string([path UTF8String]); + } + [pool release]; + return result; +} + +std::string* getSystemExecutableFolder() +{ + NSAutoreleasePool *pool = [[NSAutoreleasePool alloc] init]; + + NSString *bundlePath = [[NSBundle mainBundle] executablePath]; + std::string *result = (new std::string([bundlePath UTF8String])); + [pool release]; + + return result; +} + +std::string* getSystemResourceFolder() +{ + NSAutoreleasePool *pool = [[NSAutoreleasePool alloc] init]; + + NSString *bundlePath = [[NSBundle mainBundle] resourcePath]; + std::string *result = (new std::string([bundlePath UTF8String])); + [pool release]; + + return result; +} + +std::string* getSystemCacheFolder() +{ + return findSystemDirectory (NSCachesDirectory, + NSUserDomainMask); +} + +std::string* getSystemApplicationSupportFolder() +{ + return findSystemDirectory (NSApplicationSupportDirectory, + NSUserDomainMask); + +} + +#endif // LL_DARWIN diff --git a/indra/llfilesystem/lldir_win32.cpp b/indra/llfilesystem/lldir_win32.cpp new file mode 100644 index 0000000000..b3b3afb37e --- /dev/null +++ b/indra/llfilesystem/lldir_win32.cpp @@ -0,0 +1,452 @@ +/** + * @file lldir_win32.cpp + * @brief Implementation of directory utilities for windows + * + * $LicenseInfo:firstyear=2002&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2010, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#if LL_WINDOWS + +#include "linden_common.h" + +#include "lldir_win32.h" +#include "llerror.h" +#include "llstring.h" +#include "stringize.h" +#include "llfile.h" +#include +#include + +#include +#include +#include +#include + +// Utility stuff to get versions of the sh +#define PACKVERSION(major,minor) MAKELONG(minor,major) +DWORD GetDllVersion(LPCTSTR lpszDllName); + +namespace +{ // anonymous + enum class prst { INIT, OPEN, SKIP } state = prst::INIT; + // This is called so early that we can't count on static objects being + // properly constructed yet, so declare a pointer instead of an instance. + std::ofstream* prelogf = nullptr; + + void prelog(const std::string& message) + { + boost::optional prelog_name; + + switch (state) + { + case prst::INIT: + // assume we failed, until we succeed + state = prst::SKIP; + + prelog_name = LLStringUtil::getoptenv("PRELOG"); + if (! prelog_name) + // no PRELOG variable set, carry on + return; + prelogf = new llofstream(*prelog_name, std::ios_base::app); + if (! (prelogf && prelogf->is_open())) + // can't complain to anybody; how? + return; + // got the log file open, cool! + state = prst::OPEN; + (*prelogf) << "========================================================================" + << std::endl; + // fall through, don't break + + case prst::OPEN: + (*prelogf) << message << std::endl; + break; + + case prst::SKIP: + // either PRELOG isn't set, or we failed to open that pathname + break; + } + } +} // anonymous namespace + +#define PRELOG(expression) prelog(STRINGIZE(expression)) + +LLDir_Win32::LLDir_Win32() +{ + // set this first: used by append() and add() methods + mDirDelimiter = "\\"; + + WCHAR w_str[MAX_PATH]; + // Application Data is where user settings go. We rely on $APPDATA being + // correct. + auto APPDATA = LLStringUtil::getoptenv("APPDATA"); + if (APPDATA) + { + mOSUserDir = *APPDATA; + } + PRELOG("APPDATA='" << mOSUserDir << "'"); + // On Windows, we could have received a plain-ASCII pathname in which + // non-ASCII characters have been munged to '?', or the pathname could + // have been badly encoded and decoded such that we now have garbage + // instead of a valid path. Check that mOSUserDir actually exists. + if (mOSUserDir.empty() || ! fileExists(mOSUserDir)) + { + PRELOG("APPDATA does not exist"); + //HRESULT okay = SHGetFolderPath(NULL, CSIDL_APPDATA, NULL, 0, w_str); + wchar_t *pwstr = NULL; + HRESULT okay = SHGetKnownFolderPath(FOLDERID_RoamingAppData, 0, NULL, &pwstr); + PRELOG("SHGetKnownFolderPath(FOLDERID_RoamingAppData) returned " << okay); + if (SUCCEEDED(okay) && pwstr) + { + // But of course, only update mOSUserDir if SHGetKnownFolderPath() works. + mOSUserDir = ll_convert_wide_to_string(pwstr); + // Not only that: update our environment so that child processes + // will see a reasonable value as well. + _wputenv_s(L"APPDATA", pwstr); + // SHGetKnownFolderPath() contract requires us to free pwstr + CoTaskMemFree(pwstr); + PRELOG("mOSUserDir='" << mOSUserDir << "'"); + } + } + + // We want cache files to go on the local disk, even if the + // user is on a network with a "roaming profile". + // + // On Vista this is: + // C:\Users\James\AppData\Local + // + // We used to store the cache in AppData\Roaming, and the installer + // cleans up that version on upgrade. JC + auto LOCALAPPDATA = LLStringUtil::getoptenv("LOCALAPPDATA"); + if (LOCALAPPDATA) + { + mOSCacheDir = *LOCALAPPDATA; + } + PRELOG("LOCALAPPDATA='" << mOSCacheDir << "'"); + // Windows really does not deal well with pathnames containing non-ASCII + // characters. See above remarks about APPDATA. + if (mOSCacheDir.empty() || ! fileExists(mOSCacheDir)) + { + PRELOG("LOCALAPPDATA does not exist"); + //HRESULT okay = SHGetFolderPath(NULL, CSIDL_LOCAL_APPDATA, NULL, 0, w_str); + wchar_t *pwstr = NULL; + HRESULT okay = SHGetKnownFolderPath(FOLDERID_LocalAppData, 0, NULL, &pwstr); + PRELOG("SHGetKnownFolderPath(FOLDERID_LocalAppData) returned " << okay); + if (SUCCEEDED(okay) && pwstr) + { + // But of course, only update mOSCacheDir if SHGetKnownFolderPath() works. + mOSCacheDir = ll_convert_wide_to_string(pwstr); + // Update our environment so that child processes will see a + // reasonable value as well. + _wputenv_s(L"LOCALAPPDATA", pwstr); + // SHGetKnownFolderPath() contract requires us to free pwstr + CoTaskMemFree(pwstr); + PRELOG("mOSCacheDir='" << mOSCacheDir << "'"); + } + } + + if (GetTempPath(MAX_PATH, w_str)) + { + if (wcslen(w_str)) /* Flawfinder: ignore */ + { + w_str[wcslen(w_str)-1] = '\0'; /* Flawfinder: ignore */ // remove trailing slash + } + mTempDir = utf16str_to_utf8str(llutf16string(w_str)); + + if (mOSUserDir.empty()) + { + mOSUserDir = mTempDir; + } + + if (mOSCacheDir.empty()) + { + mOSCacheDir = mTempDir; + } + } + else + { + mTempDir = mOSUserDir; + } + +/*==========================================================================*| + // Now that we've got mOSUserDir, one way or another, let's see how we did + // with our environment variables. + { + auto report = [this](std::ostream& out){ + out << "mOSUserDir = '" << mOSUserDir << "'\n" + << "mOSCacheDir = '" << mOSCacheDir << "'\n" + << "mTempDir = '" << mTempDir << "'" << std::endl; + }; + int res = LLFile::mkdir(mOSUserDir); + if (res == -1) + { + // If we couldn't even create the directory, just blurt to stderr + report(std::cerr); + } + else + { + // successfully created logdir, plunk a log file there + std::string logfilename(add(mOSUserDir, "lldir.log")); + std::ofstream logfile(logfilename.c_str()); + if (! logfile.is_open()) + { + report(std::cerr); + } + else + { + report(logfile); + } + } + } +|*==========================================================================*/ + +// fprintf(stderr, "mTempDir = <%s>",mTempDir); + + // Set working directory, for LLDir::getWorkingDir() + GetCurrentDirectory(MAX_PATH, w_str); + mWorkingDir = utf16str_to_utf8str(llutf16string(w_str)); + + // Set the executable directory + S32 size = GetModuleFileName(NULL, w_str, MAX_PATH); + if (size) + { + w_str[size] = '\0'; + mExecutablePathAndName = utf16str_to_utf8str(llutf16string(w_str)); + S32 path_end = mExecutablePathAndName.find_last_of('\\'); + if (path_end != std::string::npos) + { + mExecutableDir = mExecutablePathAndName.substr(0, path_end); + mExecutableFilename = mExecutablePathAndName.substr(path_end+1, std::string::npos); + } + else + { + mExecutableFilename = mExecutablePathAndName; + } + + } + else + { + fprintf(stderr, "Couldn't get APP path, assuming current directory!"); + mExecutableDir = mWorkingDir; + // Assume it's the current directory + } + + // mAppRODataDir = "."; + + // Determine the location of the App-Read-Only-Data + // Try the working directory then the exe's dir. + mAppRODataDir = mWorkingDir; + + +// if (mExecutableDir.find("indra") == std::string::npos) + + // *NOTE:Mani - It is a mistake to put viewer specific code in + // the LLDir implementation. The references to 'skins' and + // 'llplugin' need to go somewhere else. + // alas... this also gets called during static initialization + // time due to the construction of gDirUtil in lldir.cpp. + if(! LLFile::isdir(add(mAppRODataDir, "skins"))) + { + // What? No skins in the working dir? + // Try the executable's directory. + mAppRODataDir = mExecutableDir; + } + +// LL_INFOS() << "mAppRODataDir = " << mAppRODataDir << LL_ENDL; + + mSkinBaseDir = add(mAppRODataDir, "skins"); + + // Build the default cache directory + mDefaultCacheDir = buildSLOSCacheDir(); + + // Make sure it exists + int res = LLFile::mkdir(mDefaultCacheDir); + if (res == -1) + { + LL_WARNS() << "Couldn't create LL_PATH_CACHE dir " << mDefaultCacheDir << LL_ENDL; + } + + mLLPluginDir = add(mExecutableDir, "llplugin"); +} + +LLDir_Win32::~LLDir_Win32() +{ +} + +// Implementation + +void LLDir_Win32::initAppDirs(const std::string &app_name, + const std::string& app_read_only_data_dir) +{ + // Allow override so test apps can read newview directory + if (!app_read_only_data_dir.empty()) + { + mAppRODataDir = app_read_only_data_dir; + mSkinBaseDir = add(mAppRODataDir, "skins"); + } + mAppName = app_name; + mOSUserAppDir = add(mOSUserDir, app_name); + + int res = LLFile::mkdir(mOSUserAppDir); + if (res == -1) + { + LL_WARNS() << "Couldn't create app user dir " << mOSUserAppDir << LL_ENDL; + LL_WARNS() << "Default to base dir" << mOSUserDir << LL_ENDL; + mOSUserAppDir = mOSUserDir; + } + //dumpCurrentDirectories(); + + res = LLFile::mkdir(getExpandedFilename(LL_PATH_LOGS,"")); + if (res == -1) + { + LL_WARNS() << "Couldn't create LL_PATH_LOGS dir " << getExpandedFilename(LL_PATH_LOGS,"") << LL_ENDL; + } + + res = LLFile::mkdir(getExpandedFilename(LL_PATH_USER_SETTINGS,"")); + if (res == -1) + { + LL_WARNS() << "Couldn't create LL_PATH_USER_SETTINGS dir " << getExpandedFilename(LL_PATH_USER_SETTINGS,"") << LL_ENDL; + } + + res = LLFile::mkdir(getExpandedFilename(LL_PATH_CACHE,"")); + if (res == -1) + { + LL_WARNS() << "Couldn't create LL_PATH_CACHE dir " << getExpandedFilename(LL_PATH_CACHE,"") << LL_ENDL; + } + + mCAFile = getExpandedFilename( LL_PATH_EXECUTABLE, "ca-bundle.crt" ); +} + +U32 LLDir_Win32::countFilesInDir(const std::string &dirname, const std::string &mask) +{ + HANDLE count_search_h; + U32 file_count; + + file_count = 0; + + WIN32_FIND_DATA FileData; + + llutf16string pathname = utf8str_to_utf16str(dirname); + pathname += utf8str_to_utf16str(mask); + + if ((count_search_h = FindFirstFile(pathname.c_str(), &FileData)) != INVALID_HANDLE_VALUE) + { + file_count++; + + while (FindNextFile(count_search_h, &FileData)) + { + file_count++; + } + + FindClose(count_search_h); + } + + return (file_count); +} + +std::string LLDir_Win32::getCurPath() +{ + WCHAR w_str[MAX_PATH]; + GetCurrentDirectory(MAX_PATH, w_str); + + return utf16str_to_utf8str(llutf16string(w_str)); +} + + +bool LLDir_Win32::fileExists(const std::string &filename) const +{ + llstat stat_data; + // Check the age of the file + // Now, we see if the files we've gathered are recent... + int res = LLFile::stat(filename, &stat_data); + if (!res) + { + return TRUE; + } + else + { + return FALSE; + } +} + + +/*virtual*/ std::string LLDir_Win32::getLLPluginLauncher() +{ + return gDirUtilp->getExecutableDir() + gDirUtilp->getDirDelimiter() + + "SLPlugin.exe"; +} + +/*virtual*/ std::string LLDir_Win32::getLLPluginFilename(std::string base_name) +{ + return gDirUtilp->getLLPluginDir() + gDirUtilp->getDirDelimiter() + + base_name + ".dll"; +} + + +#if 0 +// Utility function to get version number of a DLL + +#define PACKVERSION(major,minor) MAKELONG(minor,major) + +DWORD GetDllVersion(LPCTSTR lpszDllName) +{ + + HINSTANCE hinstDll; + DWORD dwVersion = 0; + + hinstDll = LoadLibrary(lpszDllName); /* Flawfinder: ignore */ + + if(hinstDll) + { + DLLGETVERSIONPROC pDllGetVersion; + + pDllGetVersion = (DLLGETVERSIONPROC) GetProcAddress(hinstDll, "DllGetVersion"); + +/*Because some DLLs might not implement this function, you + must test for it explicitly. Depending on the particular + DLL, the lack of a DllGetVersion function can be a useful + indicator of the version. +*/ + if(pDllGetVersion) + { + DLLVERSIONINFO dvi; + HRESULT hr; + + ZeroMemory(&dvi, sizeof(dvi)); + dvi.cbSize = sizeof(dvi); + + hr = (*pDllGetVersion)(&dvi); + + if(SUCCEEDED(hr)) + { + dwVersion = PACKVERSION(dvi.dwMajorVersion, dvi.dwMinorVersion); + } + } + + FreeLibrary(hinstDll); + } + return dwVersion; +} +#endif + +#endif + + diff --git a/indra/llfilesystem/lldir_win32.h b/indra/llfilesystem/lldir_win32.h new file mode 100644 index 0000000000..450efaf9da --- /dev/null +++ b/indra/llfilesystem/lldir_win32.h @@ -0,0 +1,59 @@ +/** + * @file lldir_win32.h + * @brief Definition of directory utilities class for windows + * + * $LicenseInfo:firstyear=2000&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2010, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#if !LL_WINDOWS +#error This header must not be included when compiling for any target other than Windows. Consider including lldir.h instead. +#endif // !LL_WINDOWS + +#ifndef LL_LLDIR_WIN32_H +#define LL_LLDIR_WIN32_H + +#include "lldir.h" + +class LLDir_Win32 : public LLDir +{ +public: + LLDir_Win32(); + virtual ~LLDir_Win32(); + + /*virtual*/ void initAppDirs(const std::string &app_name, + const std::string& app_read_only_data_dir); + + /*virtual*/ std::string getCurPath(); + /*virtual*/ U32 countFilesInDir(const std::string &dirname, const std::string &mask); + /*virtual*/ bool fileExists(const std::string &filename) const; + + /*virtual*/ std::string getLLPluginLauncher(); + /*virtual*/ std::string getLLPluginFilename(std::string base_name); + +private: + void* mDirSearch_h; + llutf16string mCurrentDir; +}; + +#endif // LL_LLDIR_WIN32_H + + diff --git a/indra/llfilesystem/lldirguard.h b/indra/llfilesystem/lldirguard.h new file mode 100644 index 0000000000..37b9e9b83e --- /dev/null +++ b/indra/llfilesystem/lldirguard.h @@ -0,0 +1,72 @@ +/** + * @file lldirguard.h + * @brief Protect working directory from being changed in scope. + * + * $LicenseInfo:firstyear=2009&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2010, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef LL_DIRGUARD_H +#define LL_DIRGUARD_H + +#include "linden_common.h" +#include "llerror.h" + +#if LL_WINDOWS +class LLDirectoryGuard +{ +public: + LLDirectoryGuard() + { + mOrigDirLen = GetCurrentDirectory(MAX_PATH, mOrigDir); + } + + ~LLDirectoryGuard() + { + mFinalDirLen = GetCurrentDirectory(MAX_PATH, mFinalDir); + if ((mOrigDirLen!=mFinalDirLen) || + (wcsncmp(mOrigDir,mFinalDir,mOrigDirLen)!=0)) + { + // Dir has changed + std::string mOrigDirUtf8 = utf16str_to_utf8str(llutf16string(mOrigDir)); + std::string mFinalDirUtf8 = utf16str_to_utf8str(llutf16string(mFinalDir)); + LL_INFOS() << "Resetting working dir from " << mFinalDirUtf8 << " to " << mOrigDirUtf8 << LL_ENDL; + SetCurrentDirectory(mOrigDir); + } + } + +private: + TCHAR mOrigDir[MAX_PATH]; + DWORD mOrigDirLen; + TCHAR mFinalDir[MAX_PATH]; + DWORD mFinalDirLen; +}; +#else // No-op outside Windows. +class LLDirectoryGuard +{ +public: + LLDirectoryGuard() {} + ~LLDirectoryGuard() {} +}; +#endif + + +#endif diff --git a/indra/llfilesystem/lldiriterator.cpp b/indra/llfilesystem/lldiriterator.cpp new file mode 100644 index 0000000000..3eb64e69d9 --- /dev/null +++ b/indra/llfilesystem/lldiriterator.cpp @@ -0,0 +1,243 @@ +/** + * @file lldiriterator.cpp + * @brief Iterator through directory entries matching the search pattern. + * + * $LicenseInfo:firstyear=2010&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2010, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include "lldiriterator.h" + +#include "fix_macros.h" +#include +#include + +namespace fs = boost::filesystem; + +static std::string glob_to_regex(const std::string& glob); + +class LLDirIterator::Impl +{ +public: + Impl(const std::string &dirname, const std::string &mask); + ~Impl(); + + bool next(std::string &fname); + +private: + boost::regex mFilterExp; + fs::directory_iterator mIter; + bool mIsValid; +}; + +LLDirIterator::Impl::Impl(const std::string &dirname, const std::string &mask) + : mIsValid(false) +{ +#ifdef LL_WINDOWS // or BOOST_WINDOWS_API + fs::path dir_path(utf8str_to_utf16str(dirname)); +#else + fs::path dir_path(dirname); +#endif + + bool is_dir = false; + + // Check if path is a directory. + try + { + is_dir = fs::is_directory(dir_path); + } + catch (const fs::filesystem_error& e) + { + LL_WARNS() << e.what() << LL_ENDL; + return; + } + + if (!is_dir) + { + LL_WARNS() << "Invalid path: \"" << dir_path.string() << "\"" << LL_ENDL; + return; + } + + // Initialize the directory iterator for the given path. + try + { + mIter = fs::directory_iterator(dir_path); + } + catch (const fs::filesystem_error& e) + { + LL_WARNS() << e.what() << LL_ENDL; + return; + } + + // Convert the glob mask to a regular expression + std::string exp = glob_to_regex(mask); + + // Initialize boost::regex with the expression converted from + // the glob mask. + // An exception is thrown if the expression is not valid. + try + { + mFilterExp.assign(exp); + } + catch (boost::regex_error& e) + { + LL_WARNS() << "\"" << exp << "\" is not a valid regular expression: " + << e.what() << LL_ENDL; + return; + } + + mIsValid = true; +} + +LLDirIterator::Impl::~Impl() +{ +} + +bool LLDirIterator::Impl::next(std::string &fname) +{ + fname = ""; + + if (!mIsValid) + { + LL_WARNS() << "The iterator is not correctly initialized." << LL_ENDL; + return false; + } + + fs::directory_iterator end_itr; // default construction yields past-the-end + bool found = false; + + // Check if path is a directory. + try + { + while (mIter != end_itr && !found) + { + boost::smatch match; + std::string name = mIter->path().filename().string(); + found = boost::regex_match(name, match, mFilterExp); + if (found) + { + fname = name; + } + + ++mIter; + } + } + catch (const fs::filesystem_error& e) + { + LL_WARNS() << e.what() << LL_ENDL; + } + + return found; +} + +/** +Converts the incoming glob into a regex. This involves +converting incoming glob expressions to regex equivilents and +at the same time, escaping any regex meaningful characters which +do not have glob meaning, i.e. + .()+|^$ +in the input. +*/ +std::string glob_to_regex(const std::string& glob) +{ + std::string regex; + regex.reserve(glob.size()<<1); + S32 braces = 0; + bool escaped = false; + bool square_brace_open = false; + + for (std::string::const_iterator i = glob.begin(); i != glob.end(); ++i) + { + char c = *i; + + switch (c) + { + case '*': + if (glob.begin() == i) + { + regex+="[^.].*"; + } + else + { + regex+= escaped ? "*" : ".*"; + } + break; + case '?': + regex+= escaped ? '?' : '.'; + break; + case '{': + braces++; + regex+='('; + break; + case '}': + if (!braces) + { + LL_ERRS() << "glob_to_regex: Closing brace without an equivalent opening brace: " << glob << LL_ENDL; + } + + regex+=')'; + braces--; + break; + case ',': + regex+= braces ? '|' : c; + break; + case '!': + regex+= square_brace_open ? '^' : c; + break; + case '.': // This collection have different regex meaning + case '^': // and so need escaping. + case '(': + case ')': + case '+': + case '|': + case '$': + regex += '\\'; + default: + regex += c; + break; + } + + escaped = ('\\' == c); + square_brace_open = ('[' == c); + } + + if (braces) + { + LL_ERRS() << "glob_to_regex: Unterminated brace expression: " << glob << LL_ENDL; + } + + return regex; +} + +LLDirIterator::LLDirIterator(const std::string &dirname, const std::string &mask) +{ + mImpl = new Impl(dirname, mask); +} + +LLDirIterator::~LLDirIterator() +{ + delete mImpl; +} + +bool LLDirIterator::next(std::string &fname) +{ + return mImpl->next(fname); +} diff --git a/indra/llfilesystem/lldiriterator.h b/indra/llfilesystem/lldiriterator.h new file mode 100644 index 0000000000..0b48be41b3 --- /dev/null +++ b/indra/llfilesystem/lldiriterator.h @@ -0,0 +1,87 @@ +/** + * @file lldiriterator.h + * @brief Iterator through directory entries matching the search pattern. + * + * $LicenseInfo:firstyear=2010&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2010, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef LL_LLDIRITERATOR_H +#define LL_LLDIRITERATOR_H + +#include "linden_common.h" + +/** + * Class LLDirIterator + * + * Iterates through directory entries matching the search pattern. + */ +class LLDirIterator +{ +public: + /** + * Constructs LLDirIterator object to search for glob pattern + * matches in a directory. + * + * @param dirname - name of a directory to search in. + * @param mask - search pattern, a glob expression + * + * Wildcards supported in glob expressions: + * -------------------------------------------------------------- + * | Wildcard | Matches | + * -------------------------------------------------------------- + * | * |zero or more characters | + * | ? |exactly one character | + * | [abcde] |exactly one character listed | + * | [a-e] |exactly one character in the given range | + * | [!abcde] |any character that is not listed | + * | [!a-e] |any character that is not in the given range | + * | {abc,xyz} |exactly one entire word in the options given | + * -------------------------------------------------------------- + */ + LLDirIterator(const std::string &dirname, const std::string &mask); + + ~LLDirIterator(); + + /** + * Searches for the next directory entry matching the glob mask + * specified upon iterator construction. + * Returns true if a match is found, sets fname + * parameter to the name of the matched directory entry and + * increments the iterator position. + * + * Typical usage: + * + * LLDirIterator iter(directory, pattern); + * if ( iter.next(scanResult) ) + * + * + * @param fname - name of the matched directory entry. + * @return true if a match is found, false otherwise. + */ + bool next(std::string &fname); + +protected: + class Impl; + Impl* mImpl; +}; + +#endif //LL_LLDIRITERATOR_H diff --git a/indra/llfilesystem/lldiskcache.cpp b/indra/llfilesystem/lldiskcache.cpp new file mode 100644 index 0000000000..af93049e07 --- /dev/null +++ b/indra/llfilesystem/lldiskcache.cpp @@ -0,0 +1,387 @@ +/** + * @file lldiskcache.cpp + * @brief Implementation of virtual file + * + * $LicenseInfo:firstyear=2002&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2010, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include "linden_common.h" + +#include "lldiskcache.h" + +#include "llerror.h" +#include "llthread.h" +#include "lltimer.h" +#include "llfasttimer.h" +#include "llmemory.h" + +#include +#include "lldir.h" + +const S32 LLDiskCache::READ = 0x00000001; +const S32 LLDiskCache::WRITE = 0x00000002; +const S32 LLDiskCache::READ_WRITE = 0x00000003; // LLDiskCache::READ & LLDiskCache::WRITE +const S32 LLDiskCache::APPEND = 0x00000006; // 0x00000004 & LLDiskCache::WRITE + +static LLTrace::BlockTimerStatHandle FTM_VFILE_WAIT("VFile Wait"); + +LLDiskCache::LLDiskCache(const LLUUID &file_id, const LLAssetType::EType file_type, S32 mode) +{ + mFileType = file_type; + mFileID = file_id; + mPosition = 0; + mBytesRead = 0; + mReadComplete = FALSE; + mMode = mode; +} + +LLDiskCache::~LLDiskCache() +{ +} + +const std::string assetTypeToString(LLAssetType::EType at) +{ + /** + * Make use of the C++17 (or is it 14) feature that allows + * for inline initialization of an std::map<> + */ + typedef std::map asset_type_to_name_t; + asset_type_to_name_t asset_type_to_name = + { + { LLAssetType::AT_TEXTURE, "TEXTURE" }, + { LLAssetType::AT_SOUND, "SOUND" }, + { LLAssetType::AT_CALLINGCARD, "CALLINGCARD" }, + { LLAssetType::AT_LANDMARK, "LANDMARK" }, + { LLAssetType::AT_SCRIPT, "SCRIPT" }, + { LLAssetType::AT_CLOTHING, "CLOTHING" }, + { LLAssetType::AT_OBJECT, "OBJECT" }, + { LLAssetType::AT_NOTECARD, "NOTECARD" }, + { LLAssetType::AT_CATEGORY, "CATEGORY" }, + { LLAssetType::AT_LSL_TEXT, "LSL_TEXT" }, + { LLAssetType::AT_LSL_BYTECODE, "LSL_BYTECODE" }, + { LLAssetType::AT_TEXTURE_TGA, "TEXTURE_TGA" }, + { LLAssetType::AT_BODYPART, "BODYPART" }, + { LLAssetType::AT_SOUND_WAV, "SOUND_WAV" }, + { LLAssetType::AT_IMAGE_TGA, "IMAGE_TGA" }, + { LLAssetType::AT_IMAGE_JPEG, "IMAGE_JPEG" }, + { LLAssetType::AT_ANIMATION, "ANIMATION" }, + { LLAssetType::AT_GESTURE, "GESTURE" }, + { LLAssetType::AT_SIMSTATE, "SIMSTATE" }, + { LLAssetType::AT_LINK, "LINK" }, + { LLAssetType::AT_LINK_FOLDER, "LINK_FOLDER" }, + { LLAssetType::AT_MARKETPLACE_FOLDER, "MARKETPLACE_FOLDER" }, + { LLAssetType::AT_WIDGET, "WIDGET" }, + { LLAssetType::AT_PERSON, "PERSON" }, + { LLAssetType::AT_MESH, "MESH" }, + { LLAssetType::AT_SETTINGS, "SETTINGS" }, + { LLAssetType::AT_UNKNOWN, "UNKNOWN" } + }; + + asset_type_to_name_t::iterator iter = asset_type_to_name.find(at); + if (iter != asset_type_to_name.end()) + { + return iter->second; + } + + return std::string("UNKNOWN"); +} + +const std::string idToFilepath(const std::string id, LLAssetType::EType at) +{ + /** + * For the moment this is just {UUID}_{ASSET_TYPE}.txt but of + * course, will be greatly expanded upon + */ + std::ostringstream ss; + ss << "00cache_"; + ss << id; + ss << "_"; + ss << assetTypeToString(at); + ss << ".txt"; + + const std::string filepath = gDirUtilp->getExpandedFilename(LL_PATH_CACHE, ss.str()); + + return filepath; +} + +// static +bool LLDiskCache::getExists(const LLUUID &file_id, const LLAssetType::EType file_type) +{ + std::string id_str; + file_id.toString(id_str); + const std::string filename = idToFilepath(id_str, file_type); + + std::ifstream file(filename, std::ios::binary); + if (file.is_open()) + { + file.seekg(0, std::ios::end); + return file.tellg() > 0; + } + return false; +} + +// static +bool LLDiskCache::removeFile(const LLUUID &file_id, const LLAssetType::EType file_type) +{ + std::string id_str; + file_id.toString(id_str); + const std::string filename = idToFilepath(id_str, file_type); + + std::remove(filename.c_str()); + + return true; +} + +// static +bool LLDiskCache::renameFile(const LLUUID &old_file_id, const LLAssetType::EType old_file_type, + const LLUUID &new_file_id, const LLAssetType::EType new_file_type) +{ + std::string old_id_str; + old_file_id.toString(old_id_str); + const std::string old_filename = idToFilepath(old_id_str, old_file_type); + + std::string new_id_str; + new_file_id.toString(new_id_str); + const std::string new_filename = idToFilepath(new_id_str, new_file_type); + + if (std::rename(old_filename.c_str(), new_filename.c_str())) + { + // We would like to return FALSE here indicating the operation + // failed but the original code does not and doing so seems to + // break a lot of things so we go with the flow... + //return FALSE; + } + + return TRUE; +} + +// static +S32 LLDiskCache::getFileSize(const LLUUID &file_id, const LLAssetType::EType file_type) +{ + std::string id_str; + file_id.toString(id_str); + const std::string filename = idToFilepath(id_str, file_type); + + S32 file_size = 0; + std::ifstream file(filename, std::ios::binary); + if (file.is_open()) + { + file.seekg(0, std::ios::end); + file_size = file.tellg(); + } + + return file_size; +} + +BOOL LLDiskCache::read(U8 *buffer, S32 bytes, BOOL async, F32 priority) +{ + BOOL success = TRUE; + + mReadComplete = FALSE; + + std::string id; + mFileID.toString(id); + const std::string filename = idToFilepath(id, mFileType); + + std::ifstream file(filename, std::ios::binary); + if (file.is_open()) + { + file.seekg(mPosition, std::ios::beg); + + file.read((char*)buffer, bytes); + + if (file) + { + mBytesRead = bytes; + } + else + { + mBytesRead = file.gcount(); + } + + file.close(); + + mPosition += mBytesRead; + if (!mBytesRead) + { + success = FALSE; + } + + mReadComplete = TRUE; + } + + return success; +} + +BOOL LLDiskCache::isReadComplete() +{ + if (mReadComplete) + { + return TRUE; + } + + return FALSE; +} + +S32 LLDiskCache::getLastBytesRead() +{ + return mBytesRead; +} + +BOOL LLDiskCache::eof() +{ + return mPosition >= getSize(); +} + +BOOL LLDiskCache::write(const U8 *buffer, S32 bytes) +{ + std::string id_str; + mFileID.toString(id_str); + const std::string filename = idToFilepath(id_str, mFileType); + + BOOL success = FALSE; + + if (mMode == APPEND) + { + std::ofstream ofs(filename, std::ios::app | std::ios::binary); + if (ofs) + { + ofs.write((const char*)buffer, bytes); + + success = TRUE; + } + } + else + { + std::ofstream ofs(filename, std::ios::binary); + if (ofs) + { + ofs.write((const char*)buffer, bytes); + + mPosition += bytes; + + success = TRUE; + } + } + + return success; +} + +//static +BOOL LLDiskCache::writeFile(const U8 *buffer, S32 bytes, const LLUUID &uuid, LLAssetType::EType type) +{ + LLDiskCache file(uuid, type, LLDiskCache::WRITE); + file.setMaxSize(bytes); + return file.write(buffer, bytes); +} + +BOOL LLDiskCache::seek(S32 offset, S32 origin) +{ + if (-1 == origin) + { + origin = mPosition; + } + + S32 new_pos = origin + offset; + + S32 size = getSize(); + + if (new_pos > size) + { + LL_WARNS() << "Attempt to seek past end of file" << LL_ENDL; + + mPosition = size; + return FALSE; + } + else if (new_pos < 0) + { + LL_WARNS() << "Attempt to seek past beginning of file" << LL_ENDL; + + mPosition = 0; + return FALSE; + } + + mPosition = new_pos; + return TRUE; +} + +S32 LLDiskCache::tell() const +{ + return mPosition; +} + +S32 LLDiskCache::getSize() +{ + return LLDiskCache::getFileSize(mFileID, mFileType); +} + +S32 LLDiskCache::getMaxSize() +{ + // offer up a huge size since we don't care what the max is + return INT_MAX; +} + +BOOL LLDiskCache::setMaxSize(S32 size) +{ + // we don't care what the max size is so we do nothing + // and return true to indicate all was okay + return TRUE; +} + +BOOL LLDiskCache::rename(const LLUUID &new_id, const LLAssetType::EType new_type) +{ + LLDiskCache::renameFile(mFileID, mFileType, new_id, new_type); + + mFileID = new_id; + mFileType = new_type; + + return TRUE; +} + +BOOL LLDiskCache::remove() +{ + LLDiskCache::removeFile(mFileID, mFileType); + + return TRUE; +} + +// static +void LLDiskCache::initClass() +{ +} + +// static +void LLDiskCache::cleanupClass() +{ +} + +bool LLDiskCache::isLocked() +{ + // I don't think we care about this test since there is no locking + // TODO: remove this function and calling sites? + return FALSE; +} + +void LLDiskCache::waitForLock() +{ + // TODO: remove this function and calling sites? +} diff --git a/indra/llfilesystem/lldiskcache.h b/indra/llfilesystem/lldiskcache.h new file mode 100644 index 0000000000..7ad06a8689 --- /dev/null +++ b/indra/llfilesystem/lldiskcache.h @@ -0,0 +1,82 @@ +/** + * @file lldiskcacke.h + * @brief Definition of virtual file + * + * $LicenseInfo:firstyear=2002&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2010, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef LL_LLDISKCACHE_H +#define LL_LLDISKCACHE_H + +#include "lluuid.h" +#include "llassettype.h" + +class LLDiskCache +{ +public: + LLDiskCache(const LLUUID &file_id, const LLAssetType::EType file_type, S32 mode = LLDiskCache::READ); + ~LLDiskCache(); + + BOOL read(U8 *buffer, S32 bytes, BOOL async = FALSE, F32 priority = 128.f); /* Flawfinder: ignore */ + BOOL isReadComplete(); + S32 getLastBytesRead(); + BOOL eof(); + + BOOL write(const U8 *buffer, S32 bytes); + static BOOL writeFile(const U8 *buffer, S32 bytes, const LLUUID &uuid, LLAssetType::EType type); + BOOL seek(S32 offset, S32 origin = -1); + S32 tell() const; + + S32 getSize(); + S32 getMaxSize(); + BOOL setMaxSize(S32 size); + BOOL rename(const LLUUID &new_id, const LLAssetType::EType new_type); + BOOL remove(); + + bool isLocked(); + void waitForLock(); + + static bool getExists(const LLUUID &file_id, const LLAssetType::EType file_type); + static bool removeFile(const LLUUID &file_id, const LLAssetType::EType file_type); + static bool renameFile(const LLUUID &old_file_id, const LLAssetType::EType old_file_type, + const LLUUID &new_file_id, const LLAssetType::EType new_file_type); + static S32 getFileSize(const LLUUID &file_id, const LLAssetType::EType file_type); + + static void initClass(); + static void cleanupClass(); + +public: + static const S32 READ; + static const S32 WRITE; + static const S32 READ_WRITE; + static const S32 APPEND; + +protected: + LLAssetType::EType mFileType; + BOOL mReadComplete; + LLUUID mFileID; + S32 mPosition; + S32 mMode; + S32 mBytesRead; +}; + +#endif // LL_LLDISKCACHE_H diff --git a/indra/llfilesystem/lllfsthread.cpp b/indra/llfilesystem/lllfsthread.cpp new file mode 100644 index 0000000000..be8e83a56f --- /dev/null +++ b/indra/llfilesystem/lllfsthread.cpp @@ -0,0 +1,245 @@ +/** + * @file lllfsthread.cpp + * @brief LLLFSThread base class + * + * $LicenseInfo:firstyear=2001&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2010, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include "linden_common.h" +#include "lllfsthread.h" +#include "llstl.h" +#include "llapr.h" + +//============================================================================ + +/*static*/ LLLFSThread* LLLFSThread::sLocal = NULL; + +//============================================================================ +// Run on MAIN thread +//static +void LLLFSThread::initClass(bool local_is_threaded) +{ + llassert(sLocal == NULL); + sLocal = new LLLFSThread(local_is_threaded); +} + +//static +S32 LLLFSThread::updateClass(U32 ms_elapsed) +{ + sLocal->update((F32)ms_elapsed); + return sLocal->getPending(); +} + +//static +void LLLFSThread::cleanupClass() +{ + llassert(sLocal != NULL); + sLocal->setQuitting(); + while (sLocal->getPending()) + { + sLocal->update(0); + } + delete sLocal; + sLocal = NULL; +} + +//---------------------------------------------------------------------------- + +LLLFSThread::LLLFSThread(bool threaded) : + LLQueuedThread("LFS", threaded), + mPriorityCounter(PRIORITY_LOWBITS) +{ + if(!mLocalAPRFilePoolp) + { + mLocalAPRFilePoolp = new LLVolatileAPRPool() ; + } +} + +LLLFSThread::~LLLFSThread() +{ + // mLocalAPRFilePoolp cleanup in LLThread + // ~LLQueuedThread() will be called here +} + +//---------------------------------------------------------------------------- + +LLLFSThread::handle_t LLLFSThread::read(const std::string& filename, /* Flawfinder: ignore */ + U8* buffer, S32 offset, S32 numbytes, + Responder* responder, U32 priority) +{ + handle_t handle = generateHandle(); + + if (priority == 0) priority = PRIORITY_NORMAL | priorityCounter(); + else if (priority < PRIORITY_LOW) priority |= PRIORITY_LOW; // All reads are at least PRIORITY_LOW + + Request* req = new Request(this, handle, priority, + FILE_READ, filename, + buffer, offset, numbytes, + responder); + + bool res = addRequest(req); + if (!res) + { + LL_ERRS() << "LLLFSThread::read called after LLLFSThread::cleanupClass()" << LL_ENDL; + } + + return handle; +} + +LLLFSThread::handle_t LLLFSThread::write(const std::string& filename, + U8* buffer, S32 offset, S32 numbytes, + Responder* responder, U32 priority) +{ + handle_t handle = generateHandle(); + + if (priority == 0) priority = PRIORITY_LOW | priorityCounter(); + + Request* req = new Request(this, handle, priority, + FILE_WRITE, filename, + buffer, offset, numbytes, + responder); + + bool res = addRequest(req); + if (!res) + { + LL_ERRS() << "LLLFSThread::read called after LLLFSThread::cleanupClass()" << LL_ENDL; + } + + return handle; +} + +//============================================================================ + +LLLFSThread::Request::Request(LLLFSThread* thread, + handle_t handle, U32 priority, + operation_t op, const std::string& filename, + U8* buffer, S32 offset, S32 numbytes, + Responder* responder) : + QueuedRequest(handle, priority, FLAG_AUTO_COMPLETE), + mThread(thread), + mOperation(op), + mFileName(filename), + mBuffer(buffer), + mOffset(offset), + mBytes(numbytes), + mBytesRead(0), + mResponder(responder) +{ + if (numbytes <= 0) + { + LL_WARNS() << "LLLFSThread: Request with numbytes = " << numbytes << LL_ENDL; + } +} + +LLLFSThread::Request::~Request() +{ +} + +// virtual, called from own thread +void LLLFSThread::Request::finishRequest(bool completed) +{ + if (mResponder.notNull()) + { + mResponder->completed(completed ? mBytesRead : 0); + mResponder = NULL; + } +} + +void LLLFSThread::Request::deleteRequest() +{ + if (getStatus() == STATUS_QUEUED) + { + LL_ERRS() << "Attempt to delete a queued LLLFSThread::Request!" << LL_ENDL; + } + if (mResponder.notNull()) + { + mResponder->completed(0); + mResponder = NULL; + } + LLQueuedThread::QueuedRequest::deleteRequest(); +} + +bool LLLFSThread::Request::processRequest() +{ + bool complete = false; + if (mOperation == FILE_READ) + { + llassert(mOffset >= 0); + LLAPRFile infile ; // auto-closes + infile.open(mFileName, LL_APR_RB, mThread->getLocalAPRFilePool()); + if (!infile.getFileHandle()) + { + LL_WARNS() << "LLLFS: Unable to read file: " << mFileName << LL_ENDL; + mBytesRead = 0; // fail + return true; + } + S32 off; + if (mOffset < 0) + off = infile.seek(APR_END, 0); + else + off = infile.seek(APR_SET, mOffset); + llassert_always(off >= 0); + mBytesRead = infile.read(mBuffer, mBytes ); + complete = true; +// LL_INFOS() << "LLLFSThread::READ:" << mFileName << " Bytes: " << mBytesRead << LL_ENDL; + } + else if (mOperation == FILE_WRITE) + { + apr_int32_t flags = APR_CREATE|APR_WRITE|APR_BINARY; + if (mOffset < 0) + flags |= APR_APPEND; + LLAPRFile outfile ; // auto-closes + outfile.open(mFileName, flags, mThread->getLocalAPRFilePool()); + if (!outfile.getFileHandle()) + { + LL_WARNS() << "LLLFS: Unable to write file: " << mFileName << LL_ENDL; + mBytesRead = 0; // fail + return true; + } + if (mOffset >= 0) + { + S32 seek = outfile.seek(APR_SET, mOffset); + if (seek < 0) + { + LL_WARNS() << "LLLFS: Unable to write file (seek failed): " << mFileName << LL_ENDL; + mBytesRead = 0; // fail + return true; + } + } + mBytesRead = outfile.write(mBuffer, mBytes ); + complete = true; +// LL_INFOS() << "LLLFSThread::WRITE:" << mFileName << " Bytes: " << mBytesRead << "/" << mBytes << " Offset:" << mOffset << LL_ENDL; + } + else + { + LL_ERRS() << "LLLFSThread::unknown operation: " << (S32)mOperation << LL_ENDL; + } + return complete; +} + +//============================================================================ + +LLLFSThread::Responder::~Responder() +{ +} + +//============================================================================ diff --git a/indra/llfilesystem/lllfsthread.h b/indra/llfilesystem/lllfsthread.h new file mode 100644 index 0000000000..58f658f7ba --- /dev/null +++ b/indra/llfilesystem/lllfsthread.h @@ -0,0 +1,147 @@ +/** + * @file lllfsthread.h + * @brief LLLFSThread base class + * + * $LicenseInfo:firstyear=2000&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2010, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef LL_LLLFSTHREAD_H +#define LL_LLLFSTHREAD_H + +#include +#include +#include +#include + +#include "llpointer.h" +#include "llqueuedthread.h" + +//============================================================================ +// Threaded Local File System +//============================================================================ + +class LLLFSThread : public LLQueuedThread +{ + //------------------------------------------------------------------------ +public: + enum operation_t { + FILE_READ, + FILE_WRITE, + FILE_RENAME, + FILE_REMOVE + }; + + //------------------------------------------------------------------------ +public: + + class Responder : public LLThreadSafeRefCount + { + protected: + ~Responder(); + public: + virtual void completed(S32 bytes) = 0; + }; + + class Request : public QueuedRequest + { + protected: + virtual ~Request(); // use deleteRequest() + + public: + Request(LLLFSThread* thread, + handle_t handle, U32 priority, + operation_t op, const std::string& filename, + U8* buffer, S32 offset, S32 numbytes, + Responder* responder); + + S32 getBytes() + { + return mBytes; + } + S32 getBytesRead() + { + return mBytesRead; + } + S32 getOperation() + { + return mOperation; + } + U8* getBuffer() + { + return mBuffer; + } + const std::string& getFilename() + { + return mFileName; + } + + /*virtual*/ bool processRequest(); + /*virtual*/ void finishRequest(bool completed); + /*virtual*/ void deleteRequest(); + + private: + LLLFSThread* mThread; + operation_t mOperation; + + std::string mFileName; + + U8* mBuffer; // dest for reads, source for writes, new UUID for rename + S32 mOffset; // offset into file, -1 = append (WRITE only) + S32 mBytes; // bytes to read from file, -1 = all + S32 mBytesRead; // bytes read from file + + LLPointer mResponder; + }; + + //------------------------------------------------------------------------ +public: + LLLFSThread(bool threaded = TRUE); + ~LLLFSThread(); + + // Return a Request handle + handle_t read(const std::string& filename, /* Flawfinder: ignore */ + U8* buffer, S32 offset, S32 numbytes, + Responder* responder, U32 pri=0); + handle_t write(const std::string& filename, + U8* buffer, S32 offset, S32 numbytes, + Responder* responder, U32 pri=0); + + // Misc + U32 priorityCounter() { return mPriorityCounter-- & PRIORITY_LOWBITS; } // Use to order IO operations + + // static initializers + static void initClass(bool local_is_threaded = TRUE); // Setup sLocal + static S32 updateClass(U32 ms_elapsed); + static void cleanupClass(); // Delete sLocal + + +private: + U32 mPriorityCounter; + +public: + static LLLFSThread* sLocal; // Default local file thread +}; + +//============================================================================ + + +#endif // LL_LLLFSTHREAD_H diff --git a/indra/llfilesystem/tests/lldir_test.cpp b/indra/llfilesystem/tests/lldir_test.cpp new file mode 100644 index 0000000000..3cff622a4b --- /dev/null +++ b/indra/llfilesystem/tests/lldir_test.cpp @@ -0,0 +1,767 @@ +/** + * @file lldir_test.cpp + * @date 2008-05 + * @brief LLDir test cases. + * + * $LicenseInfo:firstyear=2008&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2010, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include "linden_common.h" + +#include "llstring.h" +#include "tests/StringVec.h" +#include "../lldir.h" +#include "../lldiriterator.h" + +#include "../test/lltut.h" +#include "stringize.h" +#include +#include + +using boost::assign::list_of; + +// We use ensure_equals(..., vec(list_of(...))) not because it's functionally +// required, but because ensure_equals() knows how to format a StringVec. +// Turns out that when ensure_equals() displays a test failure with just +// list_of("string")("another"), you see 'stringanother' vs. '("string", +// "another")'. +StringVec vec(const StringVec& v) +{ + return v; +} + +// For some tests, use a dummy LLDir that uses memory data instead of touching +// the filesystem +struct LLDir_Dummy: public LLDir +{ + /*----------------------------- LLDir API ------------------------------*/ + LLDir_Dummy() + { + // Initialize important LLDir data members based on the filesystem + // data below. + mDirDelimiter = "/"; + mExecutableDir = "install"; + mExecutableFilename = "test"; + mExecutablePathAndName = add(mExecutableDir, mExecutableFilename); + mWorkingDir = mExecutableDir; + mAppRODataDir = "install"; + mSkinBaseDir = add(mAppRODataDir, "skins"); + mOSUserDir = "user"; + mOSUserAppDir = mOSUserDir; + mLindenUserDir = ""; + + // Make the dummy filesystem look more or less like what we expect in + // the real one. + static const char* preload[] = + { + // We group these fixture-data pathnames by basename, rather than + // sorting by full path as you might expect, because the outcome + // of each test strongly depends on which skins/languages provide + // a given basename. + "install/skins/default/colors.xml", + "install/skins/steam/colors.xml", + "user/skins/default/colors.xml", + "user/skins/steam/colors.xml", + + "install/skins/default/xui/en/strings.xml", + "install/skins/default/xui/fr/strings.xml", + "install/skins/steam/xui/en/strings.xml", + "install/skins/steam/xui/fr/strings.xml", + "user/skins/default/xui/en/strings.xml", + "user/skins/default/xui/fr/strings.xml", + "user/skins/steam/xui/en/strings.xml", + "user/skins/steam/xui/fr/strings.xml", + + "install/skins/default/xui/en/floater.xml", + "install/skins/default/xui/fr/floater.xml", + "user/skins/default/xui/fr/floater.xml", + + "install/skins/default/xui/en/newfile.xml", + "install/skins/default/xui/fr/newfile.xml", + "user/skins/default/xui/en/newfile.xml", + + "install/skins/default/html/en-us/welcome.html", + "install/skins/default/html/fr/welcome.html", + + "install/skins/default/textures/only_default.jpeg", + "install/skins/steam/textures/only_steam.jpeg", + "user/skins/default/textures/only_user_default.jpeg", + "user/skins/steam/textures/only_user_steam.jpeg", + + "install/skins/default/future/somefile.txt" + }; + BOOST_FOREACH(const char* path, preload) + { + buildFilesystem(path); + } + } + + virtual ~LLDir_Dummy() {} + + virtual void initAppDirs(const std::string& app_name, const std::string& app_read_only_data_dir) + { + // Implement this when we write a test that needs it + } + + virtual std::string getCurPath() + { + // Implement this when we write a test that needs it + return ""; + } + + virtual U32 countFilesInDir(const std::string& dirname, const std::string& mask) + { + // Implement this when we write a test that needs it + return 0; + } + + virtual bool fileExists(const std::string& pathname) const + { + // Record fileExists() calls so we can check whether caching is + // working right. Certain LLDir calls should be able to make decisions + // without calling fileExists() again, having already checked existence. + mChecked.insert(pathname); + // For our simple flat set of strings, see whether the identical + // pathname exists in our set. + return (mFilesystem.find(pathname) != mFilesystem.end()); + } + + virtual std::string getLLPluginLauncher() + { + // Implement this when we write a test that needs it + return ""; + } + + virtual std::string getLLPluginFilename(std::string base_name) + { + // Implement this when we write a test that needs it + return ""; + } + + /*----------------------------- Dummy data -----------------------------*/ + void clearFilesystem() { mFilesystem.clear(); } + void buildFilesystem(const std::string& path) + { + // Split the pathname on slashes, ignoring leading, trailing, doubles + StringVec components; + LLStringUtil::getTokens(path, components, "/"); + // Ensure we have an entry representing every level of this path + std::string partial; + BOOST_FOREACH(std::string component, components) + { + append(partial, component); + mFilesystem.insert(partial); + } + } + + void clear_checked() { mChecked.clear(); } + void ensure_checked(const std::string& pathname) const + { + tut::ensure(STRINGIZE(pathname << " was not checked but should have been"), + mChecked.find(pathname) != mChecked.end()); + } + void ensure_not_checked(const std::string& pathname) const + { + tut::ensure(STRINGIZE(pathname << " was checked but should not have been"), + mChecked.find(pathname) == mChecked.end()); + } + + std::set mFilesystem; + mutable std::set mChecked; +}; + +namespace tut +{ + struct LLDirTest + { + }; + typedef test_group LLDirTest_t; + typedef LLDirTest_t::object LLDirTest_object_t; + tut::LLDirTest_t tut_LLDirTest("LLDir"); + + template<> template<> + void LLDirTest_object_t::test<1>() + // getDirDelimiter + { + ensure("getDirDelimiter", !gDirUtilp->getDirDelimiter().empty()); + } + + template<> template<> + void LLDirTest_object_t::test<2>() + // getBaseFileName + { + std::string delim = gDirUtilp->getDirDelimiter(); + std::string rawFile = "foo"; + std::string rawFileExt = "foo.bAr"; + std::string rawFileNullExt = "foo."; + std::string rawExt = ".bAr"; + std::string rawDot = "."; + std::string pathNoExt = "aa" + delim + "bb" + delim + "cc" + delim + "dd" + delim + "ee"; + std::string pathExt = pathNoExt + ".eXt"; + std::string dottedPathNoExt = "aa" + delim + "bb" + delim + "cc.dd" + delim + "ee"; + std::string dottedPathExt = dottedPathNoExt + ".eXt"; + + // foo[.bAr] + + ensure_equals("getBaseFileName/r-no-ext/no-strip-exten", + gDirUtilp->getBaseFileName(rawFile, false), + "foo"); + + ensure_equals("getBaseFileName/r-no-ext/strip-exten", + gDirUtilp->getBaseFileName(rawFile, true), + "foo"); + + ensure_equals("getBaseFileName/r-ext/no-strip-exten", + gDirUtilp->getBaseFileName(rawFileExt, false), + "foo.bAr"); + + ensure_equals("getBaseFileName/r-ext/strip-exten", + gDirUtilp->getBaseFileName(rawFileExt, true), + "foo"); + + // foo. + + ensure_equals("getBaseFileName/rn-no-ext/no-strip-exten", + gDirUtilp->getBaseFileName(rawFileNullExt, false), + "foo."); + + ensure_equals("getBaseFileName/rn-no-ext/strip-exten", + gDirUtilp->getBaseFileName(rawFileNullExt, true), + "foo"); + + // .bAr + // interesting case - with no basename, this IS the basename, not the extension. + + ensure_equals("getBaseFileName/e-ext/no-strip-exten", + gDirUtilp->getBaseFileName(rawExt, false), + ".bAr"); + + ensure_equals("getBaseFileName/e-ext/strip-exten", + gDirUtilp->getBaseFileName(rawExt, true), + ".bAr"); + + // . + + ensure_equals("getBaseFileName/d/no-strip-exten", + gDirUtilp->getBaseFileName(rawDot, false), + "."); + + ensure_equals("getBaseFileName/d/strip-exten", + gDirUtilp->getBaseFileName(rawDot, true), + "."); + + // aa/bb/cc/dd/ee[.eXt] + + ensure_equals("getBaseFileName/no-ext/no-strip-exten", + gDirUtilp->getBaseFileName(pathNoExt, false), + "ee"); + + ensure_equals("getBaseFileName/no-ext/strip-exten", + gDirUtilp->getBaseFileName(pathNoExt, true), + "ee"); + + ensure_equals("getBaseFileName/ext/no-strip-exten", + gDirUtilp->getBaseFileName(pathExt, false), + "ee.eXt"); + + ensure_equals("getBaseFileName/ext/strip-exten", + gDirUtilp->getBaseFileName(pathExt, true), + "ee"); + + // aa/bb/cc.dd/ee[.eXt] + + ensure_equals("getBaseFileName/d-no-ext/no-strip-exten", + gDirUtilp->getBaseFileName(dottedPathNoExt, false), + "ee"); + + ensure_equals("getBaseFileName/d-no-ext/strip-exten", + gDirUtilp->getBaseFileName(dottedPathNoExt, true), + "ee"); + + ensure_equals("getBaseFileName/d-ext/no-strip-exten", + gDirUtilp->getBaseFileName(dottedPathExt, false), + "ee.eXt"); + + ensure_equals("getBaseFileName/d-ext/strip-exten", + gDirUtilp->getBaseFileName(dottedPathExt, true), + "ee"); + } + + template<> template<> + void LLDirTest_object_t::test<3>() + // getDirName + { + std::string delim = gDirUtilp->getDirDelimiter(); + std::string rawFile = "foo"; + std::string rawFileExt = "foo.bAr"; + std::string pathNoExt = "aa" + delim + "bb" + delim + "cc" + delim + "dd" + delim + "ee"; + std::string pathExt = pathNoExt + ".eXt"; + std::string dottedPathNoExt = "aa" + delim + "bb" + delim + "cc.dd" + delim + "ee"; + std::string dottedPathExt = dottedPathNoExt + ".eXt"; + + // foo[.bAr] + + ensure_equals("getDirName/r-no-ext", + gDirUtilp->getDirName(rawFile), + ""); + + ensure_equals("getDirName/r-ext", + gDirUtilp->getDirName(rawFileExt), + ""); + + // aa/bb/cc/dd/ee[.eXt] + + ensure_equals("getDirName/no-ext", + gDirUtilp->getDirName(pathNoExt), + "aa" + delim + "bb" + delim + "cc" + delim + "dd"); + + ensure_equals("getDirName/ext", + gDirUtilp->getDirName(pathExt), + "aa" + delim + "bb" + delim + "cc" + delim + "dd"); + + // aa/bb/cc.dd/ee[.eXt] + + ensure_equals("getDirName/d-no-ext", + gDirUtilp->getDirName(dottedPathNoExt), + "aa" + delim + "bb" + delim + "cc.dd"); + + ensure_equals("getDirName/d-ext", + gDirUtilp->getDirName(dottedPathExt), + "aa" + delim + "bb" + delim + "cc.dd"); + } + + template<> template<> + void LLDirTest_object_t::test<4>() + // getExtension + { + std::string delim = gDirUtilp->getDirDelimiter(); + std::string rawFile = "foo"; + std::string rawFileExt = "foo.bAr"; + std::string rawFileNullExt = "foo."; + std::string rawExt = ".bAr"; + std::string rawDot = "."; + std::string pathNoExt = "aa" + delim + "bb" + delim + "cc" + delim + "dd" + delim + "ee"; + std::string pathExt = pathNoExt + ".eXt"; + std::string dottedPathNoExt = "aa" + delim + "bb" + delim + "cc.dd" + delim + "ee"; + std::string dottedPathExt = dottedPathNoExt + ".eXt"; + + // foo[.bAr] + + ensure_equals("getExtension/r-no-ext", + gDirUtilp->getExtension(rawFile), + ""); + + ensure_equals("getExtension/r-ext", + gDirUtilp->getExtension(rawFileExt), + "bar"); + + // foo. + + ensure_equals("getExtension/rn-no-ext", + gDirUtilp->getExtension(rawFileNullExt), + ""); + + // .bAr + // interesting case - with no basename, this IS the basename, not the extension. + + ensure_equals("getExtension/e-ext", + gDirUtilp->getExtension(rawExt), + ""); + + // . + + ensure_equals("getExtension/d", + gDirUtilp->getExtension(rawDot), + ""); + + // aa/bb/cc/dd/ee[.eXt] + + ensure_equals("getExtension/no-ext", + gDirUtilp->getExtension(pathNoExt), + ""); + + ensure_equals("getExtension/ext", + gDirUtilp->getExtension(pathExt), + "ext"); + + // aa/bb/cc.dd/ee[.eXt] + + ensure_equals("getExtension/d-no-ext", + gDirUtilp->getExtension(dottedPathNoExt), + ""); + + ensure_equals("getExtension/d-ext", + gDirUtilp->getExtension(dottedPathExt), + "ext"); + } + + std::string makeTestFile( const std::string& dir, const std::string& file ) + { + std::string path = dir + file; + LLFILE* handle = LLFile::fopen( path, "w" ); + ensure("failed to open test file '"+path+"'", handle != NULL ); + // Harbison & Steele, 4th ed., p. 366: "If an error occurs, fputs + // returns EOF; otherwise, it returns some other, nonnegative value." + ensure("failed to write to test file '"+path+"'", EOF != fputs("test file", handle) ); + fclose(handle); + return path; + } + + std::string makeTestDir( const std::string& dirbase ) + { + int counter; + std::string uniqueDir; + bool foundUnused; + std::string delim = gDirUtilp->getDirDelimiter(); + + for (counter=0, foundUnused=false; !foundUnused; counter++ ) + { + char counterStr[3]; + sprintf(counterStr, "%02d", counter); + uniqueDir = dirbase + counterStr; + foundUnused = ! ( LLFile::isdir(uniqueDir) || LLFile::isfile(uniqueDir) ); + } + ensure("test directory '" + uniqueDir + "' creation failed", !LLFile::mkdir(uniqueDir)); + + return uniqueDir + delim; // HACK - apparently, the trailing delimiter is needed... + } + + static const char* DirScanFilename[5] = { "file1.abc", "file2.abc", "file1.xyz", "file2.xyz", "file1.mno" }; + + void scanTest(const std::string& directory, const std::string& pattern, bool correctResult[5]) + { + + // Scan directory and see if any file1.* files are found + std::string scanResult; + int found = 0; + bool filesFound[5] = { false, false, false, false, false }; + //std::cerr << "searching '"+directory+"' for '"+pattern+"'\n"; + + LLDirIterator iter(directory, pattern); + while ( found <= 5 && iter.next(scanResult) ) + { + found++; + //std::cerr << " found '"+scanResult+"'\n"; + int check; + for (check=0; check < 5 && ! ( scanResult == DirScanFilename[check] ); check++) + { + } + // check is now either 5 (not found) or the index of the matching name + if (check < 5) + { + ensure( "found file '"+(std::string)DirScanFilename[check]+"' twice", ! filesFound[check] ); + filesFound[check] = true; + } + else // check is 5 - should not happen + { + fail( "found unknown file '"+scanResult+"'"); + } + } + for (int i=0; i<5; i++) + { + if (correctResult[i]) + { + ensure("scan of '"+directory+"' using '"+pattern+"' did not return '"+DirScanFilename[i]+"'", filesFound[i]); + } + else + { + ensure("scan of '"+directory+"' using '"+pattern+"' incorrectly returned '"+DirScanFilename[i]+"'", !filesFound[i]); + } + } + } + + template<> template<> + void LLDirTest_object_t::test<5>() + // LLDirIterator::next + { + std::string delim = gDirUtilp->getDirDelimiter(); + std::string dirTemp = LLFile::tmpdir(); + + // Create the same 5 file names of the two directories + + std::string dir1 = makeTestDir(dirTemp + "LLDirIterator"); + std::string dir2 = makeTestDir(dirTemp + "LLDirIterator"); + std::string dir1files[5]; + std::string dir2files[5]; + for (int i=0; i<5; i++) + { + dir1files[i] = makeTestFile(dir1, DirScanFilename[i]); + dir2files[i] = makeTestFile(dir2, DirScanFilename[i]); + } + + // Scan dir1 and see if each of the 5 files is found exactly once + bool expected1[5] = { true, true, true, true, true }; + scanTest(dir1, "*", expected1); + + // Scan dir2 and see if only the 2 *.xyz files are found + bool expected2[5] = { false, false, true, true, false }; + scanTest(dir1, "*.xyz", expected2); + + // Scan dir2 and see if only the 1 *.mno file is found + bool expected3[5] = { false, false, false, false, true }; + scanTest(dir2, "*.mno", expected3); + + // Scan dir1 and see if any *.foo files are found + bool expected4[5] = { false, false, false, false, false }; + scanTest(dir1, "*.foo", expected4); + + // Scan dir1 and see if any file1.* files are found + bool expected5[5] = { true, false, true, false, true }; + scanTest(dir1, "file1.*", expected5); + + // Scan dir1 and see if any file1.* files are found + bool expected6[5] = { true, true, false, false, false }; + scanTest(dir1, "file?.abc", expected6); + + // Scan dir2 and see if any file?.x?z files are found + bool expected7[5] = { false, false, true, true, false }; + scanTest(dir2, "file?.x?z", expected7); + + // Scan dir2 and see if any file?.??c files are found + bool expected8[5] = { true, true, false, false, false }; + scanTest(dir2, "file?.??c", expected8); + scanTest(dir2, "*.??c", expected8); + + // Scan dir1 and see if any *.?n? files are found + bool expected9[5] = { false, false, false, false, true }; + scanTest(dir1, "*.?n?", expected9); + + // Scan dir1 and see if any *.???? files are found + bool expected10[5] = { false, false, false, false, false }; + scanTest(dir1, "*.????", expected10); + + // Scan dir1 and see if any ?????.* files are found + bool expected11[5] = { true, true, true, true, true }; + scanTest(dir1, "?????.*", expected11); + + // Scan dir1 and see if any ??l??.xyz files are found + bool expected12[5] = { false, false, true, true, false }; + scanTest(dir1, "??l??.xyz", expected12); + + bool expected13[5] = { true, false, true, false, false }; + scanTest(dir1, "file1.{abc,xyz}", expected13); + + bool expected14[5] = { true, true, false, false, false }; + scanTest(dir1, "file[0-9].abc", expected14); + + bool expected15[5] = { true, true, false, false, false }; + scanTest(dir1, "file[!a-z].abc", expected15); + + // clean up all test files and directories + for (int i=0; i<5; i++) + { + LLFile::remove(dir1files[i]); + LLFile::remove(dir2files[i]); + } + LLFile::rmdir(dir1); + LLFile::rmdir(dir2); + } + + template<> template<> + void LLDirTest_object_t::test<6>() + { + set_test_name("findSkinnedFilenames()"); + LLDir_Dummy lldir; + /*------------------------ "default", "en" -------------------------*/ + // Setting "default" means we shouldn't consider any "*/skins/steam" + // directories; setting "en" means we shouldn't consider any "xui/fr" + // directories. + lldir.setSkinFolder("default", "en"); + ensure_equals(lldir.getSkinFolder(), "default"); + ensure_equals(lldir.getLanguage(), "en"); + + // top-level directory of a skin isn't localized + ensure_equals(lldir.findSkinnedFilenames(LLDir::SKINBASE, "colors.xml", LLDir::ALL_SKINS), + vec(list_of("install/skins/default/colors.xml") + ("user/skins/default/colors.xml"))); + // We should not have needed to check for skins/default/en. We should + // just "know" that SKINBASE is not localized. + lldir.ensure_not_checked("install/skins/default/en"); + + ensure_equals(lldir.findSkinnedFilenames(LLDir::TEXTURES, "only_default.jpeg"), + vec(list_of("install/skins/default/textures/only_default.jpeg"))); + // Nor should we have needed to check skins/default/textures/en + // because textures is known not to be localized. + lldir.ensure_not_checked("install/skins/default/textures/en"); + + StringVec expected(vec(list_of("install/skins/default/xui/en/strings.xml") + ("user/skins/default/xui/en/strings.xml"))); + ensure_equals(lldir.findSkinnedFilenames(LLDir::XUI, "strings.xml", LLDir::ALL_SKINS), + expected); + // The first time, we had to probe to find out whether xui was localized. + lldir.ensure_checked("install/skins/default/xui/en"); + lldir.clear_checked(); + // Now make the same call again -- should return same result -- + ensure_equals(lldir.findSkinnedFilenames(LLDir::XUI, "strings.xml", LLDir::ALL_SKINS), + expected); + // but this time it should remember that xui is localized. + lldir.ensure_not_checked("install/skins/default/xui/en"); + + // localized subdir with "en-us" instead of "en" + ensure_equals(lldir.findSkinnedFilenames("html", "welcome.html"), + vec(list_of("install/skins/default/html/en-us/welcome.html"))); + lldir.ensure_checked("install/skins/default/html/en"); + lldir.ensure_checked("install/skins/default/html/en-us"); + lldir.clear_checked(); + ensure_equals(lldir.findSkinnedFilenames("html", "welcome.html"), + vec(list_of("install/skins/default/html/en-us/welcome.html"))); + lldir.ensure_not_checked("install/skins/default/html/en"); + lldir.ensure_not_checked("install/skins/default/html/en-us"); + + ensure_equals(lldir.findSkinnedFilenames("future", "somefile.txt"), + vec(list_of("install/skins/default/future/somefile.txt"))); + // Test probing for an unrecognized unlocalized future subdir. + lldir.ensure_checked("install/skins/default/future/en"); + lldir.clear_checked(); + ensure_equals(lldir.findSkinnedFilenames("future", "somefile.txt"), + vec(list_of("install/skins/default/future/somefile.txt"))); + // Second time it should remember that future is unlocalized. + lldir.ensure_not_checked("install/skins/default/future/en"); + + // When language is set to "en", requesting an html file pulls up the + // "en-us" version -- not because it magically matches those strings, + // but because there's no "en" localization and it falls back on the + // default "en-us"! Note that it would probably still be better to + // make the default localization be "en" and allow "en-gb" (or + // whatever) localizations, which would work much more the way you'd + // expect. + ensure_equals(lldir.findSkinnedFilenames("html", "welcome.html"), + vec(list_of("install/skins/default/html/en-us/welcome.html"))); + + /*------------------------ "default", "fr" -------------------------*/ + // We start being able to distinguish localized subdirs from + // unlocalized when we ask for a non-English language. + lldir.setSkinFolder("default", "fr"); + ensure_equals(lldir.getLanguage(), "fr"); + + // pass merge=true to request this filename in all relevant skins + ensure_equals(lldir.findSkinnedFilenames(LLDir::XUI, "strings.xml", LLDir::ALL_SKINS), + vec(list_of + ("install/skins/default/xui/en/strings.xml") + ("install/skins/default/xui/fr/strings.xml") + ("user/skins/default/xui/en/strings.xml") + ("user/skins/default/xui/fr/strings.xml"))); + + // pass (or default) merge=false to request only most specific skin + ensure_equals(lldir.findSkinnedFilenames(LLDir::XUI, "strings.xml"), + vec(list_of + ("user/skins/default/xui/en/strings.xml") + ("user/skins/default/xui/fr/strings.xml"))); + + // Our dummy floater.xml has a user localization (for "fr") but no + // English override. This is a case in which CURRENT_SKIN nonetheless + // returns paths from two different skins. + ensure_equals(lldir.findSkinnedFilenames(LLDir::XUI, "floater.xml"), + vec(list_of + ("install/skins/default/xui/en/floater.xml") + ("user/skins/default/xui/fr/floater.xml"))); + + // Our dummy newfile.xml has an English override but no user + // localization. This is another case in which CURRENT_SKIN + // nonetheless returns paths from two different skins. + ensure_equals(lldir.findSkinnedFilenames(LLDir::XUI, "newfile.xml"), + vec(list_of + ("user/skins/default/xui/en/newfile.xml") + ("install/skins/default/xui/fr/newfile.xml"))); + + ensure_equals(lldir.findSkinnedFilenames("html", "welcome.html"), + vec(list_of + ("install/skins/default/html/en-us/welcome.html") + ("install/skins/default/html/fr/welcome.html"))); + + /*------------------------ "default", "zh" -------------------------*/ + lldir.setSkinFolder("default", "zh"); + // Because strings.xml has only a "fr" override but no "zh" override + // in any skin, the most localized version we can find is "en". + ensure_equals(lldir.findSkinnedFilenames(LLDir::XUI, "strings.xml"), + vec(list_of("user/skins/default/xui/en/strings.xml"))); + + /*------------------------- "steam", "en" --------------------------*/ + lldir.setSkinFolder("steam", "en"); + + ensure_equals(lldir.findSkinnedFilenames(LLDir::SKINBASE, "colors.xml", LLDir::ALL_SKINS), + vec(list_of + ("install/skins/default/colors.xml") + ("install/skins/steam/colors.xml") + ("user/skins/default/colors.xml") + ("user/skins/steam/colors.xml"))); + + ensure_equals(lldir.findSkinnedFilenames(LLDir::TEXTURES, "only_default.jpeg"), + vec(list_of("install/skins/default/textures/only_default.jpeg"))); + + ensure_equals(lldir.findSkinnedFilenames(LLDir::TEXTURES, "only_steam.jpeg"), + vec(list_of("install/skins/steam/textures/only_steam.jpeg"))); + + ensure_equals(lldir.findSkinnedFilenames(LLDir::TEXTURES, "only_user_default.jpeg"), + vec(list_of("user/skins/default/textures/only_user_default.jpeg"))); + + ensure_equals(lldir.findSkinnedFilenames(LLDir::TEXTURES, "only_user_steam.jpeg"), + vec(list_of("user/skins/steam/textures/only_user_steam.jpeg"))); + + // CURRENT_SKIN + ensure_equals(lldir.findSkinnedFilenames(LLDir::XUI, "strings.xml"), + vec(list_of("user/skins/steam/xui/en/strings.xml"))); + + // pass constraint=ALL_SKINS to request this filename in all relevant skins + ensure_equals(lldir.findSkinnedFilenames(LLDir::XUI, "strings.xml", LLDir::ALL_SKINS), + vec(list_of + ("install/skins/default/xui/en/strings.xml") + ("install/skins/steam/xui/en/strings.xml") + ("user/skins/default/xui/en/strings.xml") + ("user/skins/steam/xui/en/strings.xml"))); + + /*------------------------- "steam", "fr" --------------------------*/ + lldir.setSkinFolder("steam", "fr"); + + // pass CURRENT_SKIN to request only the most specialized files + ensure_equals(lldir.findSkinnedFilenames(LLDir::XUI, "strings.xml"), + vec(list_of + ("user/skins/steam/xui/en/strings.xml") + ("user/skins/steam/xui/fr/strings.xml"))); + + // pass ALL_SKINS to request this filename in all relevant skins + ensure_equals(lldir.findSkinnedFilenames(LLDir::XUI, "strings.xml", LLDir::ALL_SKINS), + vec(list_of + ("install/skins/default/xui/en/strings.xml") + ("install/skins/default/xui/fr/strings.xml") + ("install/skins/steam/xui/en/strings.xml") + ("install/skins/steam/xui/fr/strings.xml") + ("user/skins/default/xui/en/strings.xml") + ("user/skins/default/xui/fr/strings.xml") + ("user/skins/steam/xui/en/strings.xml") + ("user/skins/steam/xui/fr/strings.xml"))); + } + + template<> template<> + void LLDirTest_object_t::test<7>() + { + set_test_name("add()"); + LLDir_Dummy lldir; + ensure_equals("both empty", lldir.add("", ""), ""); + ensure_equals("path empty", lldir.add("", "b"), "b"); + ensure_equals("name empty", lldir.add("a", ""), "a"); + ensure_equals("both simple", lldir.add("a", "b"), "a/b"); + ensure_equals("name leading slash", lldir.add("a", "/b"), "a/b"); + ensure_equals("path trailing slash", lldir.add("a/", "b"), "a/b"); + ensure_equals("both bring slashes", lldir.add("a/", "/b"), "a/b"); + } +} diff --git a/indra/llfilesystem/tests/lldiriterator_test.cpp b/indra/llfilesystem/tests/lldiriterator_test.cpp new file mode 100644 index 0000000000..a65e3dada5 --- /dev/null +++ b/indra/llfilesystem/tests/lldiriterator_test.cpp @@ -0,0 +1,65 @@ +/** + * @file lldiriterator_test.cpp + * @date 2011-06 + * @brief LLDirIterator test cases. + * + * $LicenseInfo:firstyear=2011&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2011, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only., + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include "linden_common.h" +#include "lltut.h" +#include "../lldiriterator.h" + + +namespace tut +{ + + struct LLDirIteratorFixture + { + LLDirIteratorFixture() + { + } + }; + typedef test_group LLDirIteratorTest_factory; + typedef LLDirIteratorTest_factory::object LLDirIteratorTest_t; + LLDirIteratorTest_factory tf("LLDirIterator"); + + /* + CHOP-662 was originally introduced to deal with crashes deleting files from + a directory (VWR-25500). However, this introduced a crash looking for + old chat logs as the glob_to_regex function in lldiriterator wasn't escaping lots of regexp characters + */ + void test_chop_662(void) + { + // Check a selection of bad group names from the crash reports + LLDirIterator iter(".","+bad-group-name]+?\?-??.*"); + LLDirIterator iter1(".","))--@---bad-group-name2((?\?-??.*\\.txt"); + LLDirIterator iter2(".","__^v--x)Cuide d sua vida(x--v^__?\?-??.*"); + } + + template<> template<> + void LLDirIteratorTest_t::test<1>() + { + test_chop_662(); + } + +} -- cgit v1.2.3 From 5858bb87c7d9d222e61c7c4f7f0f1dc8ba02c77f Mon Sep 17 00:00:00 2001 From: Callum Prentice Date: Tue, 22 Sep 2020 16:52:10 -0700 Subject: Add SQLite third package to this viewer and pull in the most recent build (3.33) --- indra/llfilesystem/CMakeLists.txt | 3 +++ 1 file changed, 3 insertions(+) (limited to 'indra/llfilesystem') diff --git a/indra/llfilesystem/CMakeLists.txt b/indra/llfilesystem/CMakeLists.txt index 4af14d6d3a..306b483097 100644 --- a/indra/llfilesystem/CMakeLists.txt +++ b/indra/llfilesystem/CMakeLists.txt @@ -4,11 +4,13 @@ project(llfilesystem) include(00-Common) include(LLCommon) +include(SQLite) include(UnixInstall) include_directories( ${LLCOMMON_INCLUDE_DIRS} ${LLCOMMON_SYSTEM_INCLUDE_DIRS} + ${SQLITE_INCLUDE_DIR} ) set(llfilesystem_SOURCE_FILES @@ -67,6 +69,7 @@ set(cache_BOOST_LIBRARIES target_link_libraries(llfilesystem ${LLCOMMON_LIBRARIES} ${cache_BOOST_LIBRARIES} + ${SQLITE_LIBRARIES} ) if (DARWIN) -- cgit v1.2.3 From 96e2873bfa2c6b8823aed3b4190c43cd5dab54e6 Mon Sep 17 00:00:00 2001 From: Callum Prentice Date: Thu, 24 Sep 2020 10:23:39 -0700 Subject: Rename lldiskcache.* to llfilesystem.* - i think this is the right name since it's responsible for performing file system operations and (will eventually) delegrate to a separate disk cache component to save/load data and keep track of metadata etc. --- indra/llfilesystem/lldiskcache.cpp | 387 ------------------------------------ indra/llfilesystem/lldiskcache.h | 82 -------- indra/llfilesystem/llfilesystem.cpp | 387 ++++++++++++++++++++++++++++++++++++ indra/llfilesystem/llfilesystem.h | 82 ++++++++ 4 files changed, 469 insertions(+), 469 deletions(-) delete mode 100644 indra/llfilesystem/lldiskcache.cpp delete mode 100644 indra/llfilesystem/lldiskcache.h create mode 100644 indra/llfilesystem/llfilesystem.cpp create mode 100644 indra/llfilesystem/llfilesystem.h (limited to 'indra/llfilesystem') diff --git a/indra/llfilesystem/lldiskcache.cpp b/indra/llfilesystem/lldiskcache.cpp deleted file mode 100644 index af93049e07..0000000000 --- a/indra/llfilesystem/lldiskcache.cpp +++ /dev/null @@ -1,387 +0,0 @@ -/** - * @file lldiskcache.cpp - * @brief Implementation of virtual file - * - * $LicenseInfo:firstyear=2002&license=viewerlgpl$ - * Second Life Viewer Source Code - * Copyright (C) 2010, Linden Research, Inc. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; - * version 2.1 of the License only. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - * - * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA - * $/LicenseInfo$ - */ - -#include "linden_common.h" - -#include "lldiskcache.h" - -#include "llerror.h" -#include "llthread.h" -#include "lltimer.h" -#include "llfasttimer.h" -#include "llmemory.h" - -#include -#include "lldir.h" - -const S32 LLDiskCache::READ = 0x00000001; -const S32 LLDiskCache::WRITE = 0x00000002; -const S32 LLDiskCache::READ_WRITE = 0x00000003; // LLDiskCache::READ & LLDiskCache::WRITE -const S32 LLDiskCache::APPEND = 0x00000006; // 0x00000004 & LLDiskCache::WRITE - -static LLTrace::BlockTimerStatHandle FTM_VFILE_WAIT("VFile Wait"); - -LLDiskCache::LLDiskCache(const LLUUID &file_id, const LLAssetType::EType file_type, S32 mode) -{ - mFileType = file_type; - mFileID = file_id; - mPosition = 0; - mBytesRead = 0; - mReadComplete = FALSE; - mMode = mode; -} - -LLDiskCache::~LLDiskCache() -{ -} - -const std::string assetTypeToString(LLAssetType::EType at) -{ - /** - * Make use of the C++17 (or is it 14) feature that allows - * for inline initialization of an std::map<> - */ - typedef std::map asset_type_to_name_t; - asset_type_to_name_t asset_type_to_name = - { - { LLAssetType::AT_TEXTURE, "TEXTURE" }, - { LLAssetType::AT_SOUND, "SOUND" }, - { LLAssetType::AT_CALLINGCARD, "CALLINGCARD" }, - { LLAssetType::AT_LANDMARK, "LANDMARK" }, - { LLAssetType::AT_SCRIPT, "SCRIPT" }, - { LLAssetType::AT_CLOTHING, "CLOTHING" }, - { LLAssetType::AT_OBJECT, "OBJECT" }, - { LLAssetType::AT_NOTECARD, "NOTECARD" }, - { LLAssetType::AT_CATEGORY, "CATEGORY" }, - { LLAssetType::AT_LSL_TEXT, "LSL_TEXT" }, - { LLAssetType::AT_LSL_BYTECODE, "LSL_BYTECODE" }, - { LLAssetType::AT_TEXTURE_TGA, "TEXTURE_TGA" }, - { LLAssetType::AT_BODYPART, "BODYPART" }, - { LLAssetType::AT_SOUND_WAV, "SOUND_WAV" }, - { LLAssetType::AT_IMAGE_TGA, "IMAGE_TGA" }, - { LLAssetType::AT_IMAGE_JPEG, "IMAGE_JPEG" }, - { LLAssetType::AT_ANIMATION, "ANIMATION" }, - { LLAssetType::AT_GESTURE, "GESTURE" }, - { LLAssetType::AT_SIMSTATE, "SIMSTATE" }, - { LLAssetType::AT_LINK, "LINK" }, - { LLAssetType::AT_LINK_FOLDER, "LINK_FOLDER" }, - { LLAssetType::AT_MARKETPLACE_FOLDER, "MARKETPLACE_FOLDER" }, - { LLAssetType::AT_WIDGET, "WIDGET" }, - { LLAssetType::AT_PERSON, "PERSON" }, - { LLAssetType::AT_MESH, "MESH" }, - { LLAssetType::AT_SETTINGS, "SETTINGS" }, - { LLAssetType::AT_UNKNOWN, "UNKNOWN" } - }; - - asset_type_to_name_t::iterator iter = asset_type_to_name.find(at); - if (iter != asset_type_to_name.end()) - { - return iter->second; - } - - return std::string("UNKNOWN"); -} - -const std::string idToFilepath(const std::string id, LLAssetType::EType at) -{ - /** - * For the moment this is just {UUID}_{ASSET_TYPE}.txt but of - * course, will be greatly expanded upon - */ - std::ostringstream ss; - ss << "00cache_"; - ss << id; - ss << "_"; - ss << assetTypeToString(at); - ss << ".txt"; - - const std::string filepath = gDirUtilp->getExpandedFilename(LL_PATH_CACHE, ss.str()); - - return filepath; -} - -// static -bool LLDiskCache::getExists(const LLUUID &file_id, const LLAssetType::EType file_type) -{ - std::string id_str; - file_id.toString(id_str); - const std::string filename = idToFilepath(id_str, file_type); - - std::ifstream file(filename, std::ios::binary); - if (file.is_open()) - { - file.seekg(0, std::ios::end); - return file.tellg() > 0; - } - return false; -} - -// static -bool LLDiskCache::removeFile(const LLUUID &file_id, const LLAssetType::EType file_type) -{ - std::string id_str; - file_id.toString(id_str); - const std::string filename = idToFilepath(id_str, file_type); - - std::remove(filename.c_str()); - - return true; -} - -// static -bool LLDiskCache::renameFile(const LLUUID &old_file_id, const LLAssetType::EType old_file_type, - const LLUUID &new_file_id, const LLAssetType::EType new_file_type) -{ - std::string old_id_str; - old_file_id.toString(old_id_str); - const std::string old_filename = idToFilepath(old_id_str, old_file_type); - - std::string new_id_str; - new_file_id.toString(new_id_str); - const std::string new_filename = idToFilepath(new_id_str, new_file_type); - - if (std::rename(old_filename.c_str(), new_filename.c_str())) - { - // We would like to return FALSE here indicating the operation - // failed but the original code does not and doing so seems to - // break a lot of things so we go with the flow... - //return FALSE; - } - - return TRUE; -} - -// static -S32 LLDiskCache::getFileSize(const LLUUID &file_id, const LLAssetType::EType file_type) -{ - std::string id_str; - file_id.toString(id_str); - const std::string filename = idToFilepath(id_str, file_type); - - S32 file_size = 0; - std::ifstream file(filename, std::ios::binary); - if (file.is_open()) - { - file.seekg(0, std::ios::end); - file_size = file.tellg(); - } - - return file_size; -} - -BOOL LLDiskCache::read(U8 *buffer, S32 bytes, BOOL async, F32 priority) -{ - BOOL success = TRUE; - - mReadComplete = FALSE; - - std::string id; - mFileID.toString(id); - const std::string filename = idToFilepath(id, mFileType); - - std::ifstream file(filename, std::ios::binary); - if (file.is_open()) - { - file.seekg(mPosition, std::ios::beg); - - file.read((char*)buffer, bytes); - - if (file) - { - mBytesRead = bytes; - } - else - { - mBytesRead = file.gcount(); - } - - file.close(); - - mPosition += mBytesRead; - if (!mBytesRead) - { - success = FALSE; - } - - mReadComplete = TRUE; - } - - return success; -} - -BOOL LLDiskCache::isReadComplete() -{ - if (mReadComplete) - { - return TRUE; - } - - return FALSE; -} - -S32 LLDiskCache::getLastBytesRead() -{ - return mBytesRead; -} - -BOOL LLDiskCache::eof() -{ - return mPosition >= getSize(); -} - -BOOL LLDiskCache::write(const U8 *buffer, S32 bytes) -{ - std::string id_str; - mFileID.toString(id_str); - const std::string filename = idToFilepath(id_str, mFileType); - - BOOL success = FALSE; - - if (mMode == APPEND) - { - std::ofstream ofs(filename, std::ios::app | std::ios::binary); - if (ofs) - { - ofs.write((const char*)buffer, bytes); - - success = TRUE; - } - } - else - { - std::ofstream ofs(filename, std::ios::binary); - if (ofs) - { - ofs.write((const char*)buffer, bytes); - - mPosition += bytes; - - success = TRUE; - } - } - - return success; -} - -//static -BOOL LLDiskCache::writeFile(const U8 *buffer, S32 bytes, const LLUUID &uuid, LLAssetType::EType type) -{ - LLDiskCache file(uuid, type, LLDiskCache::WRITE); - file.setMaxSize(bytes); - return file.write(buffer, bytes); -} - -BOOL LLDiskCache::seek(S32 offset, S32 origin) -{ - if (-1 == origin) - { - origin = mPosition; - } - - S32 new_pos = origin + offset; - - S32 size = getSize(); - - if (new_pos > size) - { - LL_WARNS() << "Attempt to seek past end of file" << LL_ENDL; - - mPosition = size; - return FALSE; - } - else if (new_pos < 0) - { - LL_WARNS() << "Attempt to seek past beginning of file" << LL_ENDL; - - mPosition = 0; - return FALSE; - } - - mPosition = new_pos; - return TRUE; -} - -S32 LLDiskCache::tell() const -{ - return mPosition; -} - -S32 LLDiskCache::getSize() -{ - return LLDiskCache::getFileSize(mFileID, mFileType); -} - -S32 LLDiskCache::getMaxSize() -{ - // offer up a huge size since we don't care what the max is - return INT_MAX; -} - -BOOL LLDiskCache::setMaxSize(S32 size) -{ - // we don't care what the max size is so we do nothing - // and return true to indicate all was okay - return TRUE; -} - -BOOL LLDiskCache::rename(const LLUUID &new_id, const LLAssetType::EType new_type) -{ - LLDiskCache::renameFile(mFileID, mFileType, new_id, new_type); - - mFileID = new_id; - mFileType = new_type; - - return TRUE; -} - -BOOL LLDiskCache::remove() -{ - LLDiskCache::removeFile(mFileID, mFileType); - - return TRUE; -} - -// static -void LLDiskCache::initClass() -{ -} - -// static -void LLDiskCache::cleanupClass() -{ -} - -bool LLDiskCache::isLocked() -{ - // I don't think we care about this test since there is no locking - // TODO: remove this function and calling sites? - return FALSE; -} - -void LLDiskCache::waitForLock() -{ - // TODO: remove this function and calling sites? -} diff --git a/indra/llfilesystem/lldiskcache.h b/indra/llfilesystem/lldiskcache.h deleted file mode 100644 index 7ad06a8689..0000000000 --- a/indra/llfilesystem/lldiskcache.h +++ /dev/null @@ -1,82 +0,0 @@ -/** - * @file lldiskcacke.h - * @brief Definition of virtual file - * - * $LicenseInfo:firstyear=2002&license=viewerlgpl$ - * Second Life Viewer Source Code - * Copyright (C) 2010, Linden Research, Inc. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; - * version 2.1 of the License only. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - * - * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA - * $/LicenseInfo$ - */ - -#ifndef LL_LLDISKCACHE_H -#define LL_LLDISKCACHE_H - -#include "lluuid.h" -#include "llassettype.h" - -class LLDiskCache -{ -public: - LLDiskCache(const LLUUID &file_id, const LLAssetType::EType file_type, S32 mode = LLDiskCache::READ); - ~LLDiskCache(); - - BOOL read(U8 *buffer, S32 bytes, BOOL async = FALSE, F32 priority = 128.f); /* Flawfinder: ignore */ - BOOL isReadComplete(); - S32 getLastBytesRead(); - BOOL eof(); - - BOOL write(const U8 *buffer, S32 bytes); - static BOOL writeFile(const U8 *buffer, S32 bytes, const LLUUID &uuid, LLAssetType::EType type); - BOOL seek(S32 offset, S32 origin = -1); - S32 tell() const; - - S32 getSize(); - S32 getMaxSize(); - BOOL setMaxSize(S32 size); - BOOL rename(const LLUUID &new_id, const LLAssetType::EType new_type); - BOOL remove(); - - bool isLocked(); - void waitForLock(); - - static bool getExists(const LLUUID &file_id, const LLAssetType::EType file_type); - static bool removeFile(const LLUUID &file_id, const LLAssetType::EType file_type); - static bool renameFile(const LLUUID &old_file_id, const LLAssetType::EType old_file_type, - const LLUUID &new_file_id, const LLAssetType::EType new_file_type); - static S32 getFileSize(const LLUUID &file_id, const LLAssetType::EType file_type); - - static void initClass(); - static void cleanupClass(); - -public: - static const S32 READ; - static const S32 WRITE; - static const S32 READ_WRITE; - static const S32 APPEND; - -protected: - LLAssetType::EType mFileType; - BOOL mReadComplete; - LLUUID mFileID; - S32 mPosition; - S32 mMode; - S32 mBytesRead; -}; - -#endif // LL_LLDISKCACHE_H diff --git a/indra/llfilesystem/llfilesystem.cpp b/indra/llfilesystem/llfilesystem.cpp new file mode 100644 index 0000000000..af93049e07 --- /dev/null +++ b/indra/llfilesystem/llfilesystem.cpp @@ -0,0 +1,387 @@ +/** + * @file lldiskcache.cpp + * @brief Implementation of virtual file + * + * $LicenseInfo:firstyear=2002&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2010, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include "linden_common.h" + +#include "lldiskcache.h" + +#include "llerror.h" +#include "llthread.h" +#include "lltimer.h" +#include "llfasttimer.h" +#include "llmemory.h" + +#include +#include "lldir.h" + +const S32 LLDiskCache::READ = 0x00000001; +const S32 LLDiskCache::WRITE = 0x00000002; +const S32 LLDiskCache::READ_WRITE = 0x00000003; // LLDiskCache::READ & LLDiskCache::WRITE +const S32 LLDiskCache::APPEND = 0x00000006; // 0x00000004 & LLDiskCache::WRITE + +static LLTrace::BlockTimerStatHandle FTM_VFILE_WAIT("VFile Wait"); + +LLDiskCache::LLDiskCache(const LLUUID &file_id, const LLAssetType::EType file_type, S32 mode) +{ + mFileType = file_type; + mFileID = file_id; + mPosition = 0; + mBytesRead = 0; + mReadComplete = FALSE; + mMode = mode; +} + +LLDiskCache::~LLDiskCache() +{ +} + +const std::string assetTypeToString(LLAssetType::EType at) +{ + /** + * Make use of the C++17 (or is it 14) feature that allows + * for inline initialization of an std::map<> + */ + typedef std::map asset_type_to_name_t; + asset_type_to_name_t asset_type_to_name = + { + { LLAssetType::AT_TEXTURE, "TEXTURE" }, + { LLAssetType::AT_SOUND, "SOUND" }, + { LLAssetType::AT_CALLINGCARD, "CALLINGCARD" }, + { LLAssetType::AT_LANDMARK, "LANDMARK" }, + { LLAssetType::AT_SCRIPT, "SCRIPT" }, + { LLAssetType::AT_CLOTHING, "CLOTHING" }, + { LLAssetType::AT_OBJECT, "OBJECT" }, + { LLAssetType::AT_NOTECARD, "NOTECARD" }, + { LLAssetType::AT_CATEGORY, "CATEGORY" }, + { LLAssetType::AT_LSL_TEXT, "LSL_TEXT" }, + { LLAssetType::AT_LSL_BYTECODE, "LSL_BYTECODE" }, + { LLAssetType::AT_TEXTURE_TGA, "TEXTURE_TGA" }, + { LLAssetType::AT_BODYPART, "BODYPART" }, + { LLAssetType::AT_SOUND_WAV, "SOUND_WAV" }, + { LLAssetType::AT_IMAGE_TGA, "IMAGE_TGA" }, + { LLAssetType::AT_IMAGE_JPEG, "IMAGE_JPEG" }, + { LLAssetType::AT_ANIMATION, "ANIMATION" }, + { LLAssetType::AT_GESTURE, "GESTURE" }, + { LLAssetType::AT_SIMSTATE, "SIMSTATE" }, + { LLAssetType::AT_LINK, "LINK" }, + { LLAssetType::AT_LINK_FOLDER, "LINK_FOLDER" }, + { LLAssetType::AT_MARKETPLACE_FOLDER, "MARKETPLACE_FOLDER" }, + { LLAssetType::AT_WIDGET, "WIDGET" }, + { LLAssetType::AT_PERSON, "PERSON" }, + { LLAssetType::AT_MESH, "MESH" }, + { LLAssetType::AT_SETTINGS, "SETTINGS" }, + { LLAssetType::AT_UNKNOWN, "UNKNOWN" } + }; + + asset_type_to_name_t::iterator iter = asset_type_to_name.find(at); + if (iter != asset_type_to_name.end()) + { + return iter->second; + } + + return std::string("UNKNOWN"); +} + +const std::string idToFilepath(const std::string id, LLAssetType::EType at) +{ + /** + * For the moment this is just {UUID}_{ASSET_TYPE}.txt but of + * course, will be greatly expanded upon + */ + std::ostringstream ss; + ss << "00cache_"; + ss << id; + ss << "_"; + ss << assetTypeToString(at); + ss << ".txt"; + + const std::string filepath = gDirUtilp->getExpandedFilename(LL_PATH_CACHE, ss.str()); + + return filepath; +} + +// static +bool LLDiskCache::getExists(const LLUUID &file_id, const LLAssetType::EType file_type) +{ + std::string id_str; + file_id.toString(id_str); + const std::string filename = idToFilepath(id_str, file_type); + + std::ifstream file(filename, std::ios::binary); + if (file.is_open()) + { + file.seekg(0, std::ios::end); + return file.tellg() > 0; + } + return false; +} + +// static +bool LLDiskCache::removeFile(const LLUUID &file_id, const LLAssetType::EType file_type) +{ + std::string id_str; + file_id.toString(id_str); + const std::string filename = idToFilepath(id_str, file_type); + + std::remove(filename.c_str()); + + return true; +} + +// static +bool LLDiskCache::renameFile(const LLUUID &old_file_id, const LLAssetType::EType old_file_type, + const LLUUID &new_file_id, const LLAssetType::EType new_file_type) +{ + std::string old_id_str; + old_file_id.toString(old_id_str); + const std::string old_filename = idToFilepath(old_id_str, old_file_type); + + std::string new_id_str; + new_file_id.toString(new_id_str); + const std::string new_filename = idToFilepath(new_id_str, new_file_type); + + if (std::rename(old_filename.c_str(), new_filename.c_str())) + { + // We would like to return FALSE here indicating the operation + // failed but the original code does not and doing so seems to + // break a lot of things so we go with the flow... + //return FALSE; + } + + return TRUE; +} + +// static +S32 LLDiskCache::getFileSize(const LLUUID &file_id, const LLAssetType::EType file_type) +{ + std::string id_str; + file_id.toString(id_str); + const std::string filename = idToFilepath(id_str, file_type); + + S32 file_size = 0; + std::ifstream file(filename, std::ios::binary); + if (file.is_open()) + { + file.seekg(0, std::ios::end); + file_size = file.tellg(); + } + + return file_size; +} + +BOOL LLDiskCache::read(U8 *buffer, S32 bytes, BOOL async, F32 priority) +{ + BOOL success = TRUE; + + mReadComplete = FALSE; + + std::string id; + mFileID.toString(id); + const std::string filename = idToFilepath(id, mFileType); + + std::ifstream file(filename, std::ios::binary); + if (file.is_open()) + { + file.seekg(mPosition, std::ios::beg); + + file.read((char*)buffer, bytes); + + if (file) + { + mBytesRead = bytes; + } + else + { + mBytesRead = file.gcount(); + } + + file.close(); + + mPosition += mBytesRead; + if (!mBytesRead) + { + success = FALSE; + } + + mReadComplete = TRUE; + } + + return success; +} + +BOOL LLDiskCache::isReadComplete() +{ + if (mReadComplete) + { + return TRUE; + } + + return FALSE; +} + +S32 LLDiskCache::getLastBytesRead() +{ + return mBytesRead; +} + +BOOL LLDiskCache::eof() +{ + return mPosition >= getSize(); +} + +BOOL LLDiskCache::write(const U8 *buffer, S32 bytes) +{ + std::string id_str; + mFileID.toString(id_str); + const std::string filename = idToFilepath(id_str, mFileType); + + BOOL success = FALSE; + + if (mMode == APPEND) + { + std::ofstream ofs(filename, std::ios::app | std::ios::binary); + if (ofs) + { + ofs.write((const char*)buffer, bytes); + + success = TRUE; + } + } + else + { + std::ofstream ofs(filename, std::ios::binary); + if (ofs) + { + ofs.write((const char*)buffer, bytes); + + mPosition += bytes; + + success = TRUE; + } + } + + return success; +} + +//static +BOOL LLDiskCache::writeFile(const U8 *buffer, S32 bytes, const LLUUID &uuid, LLAssetType::EType type) +{ + LLDiskCache file(uuid, type, LLDiskCache::WRITE); + file.setMaxSize(bytes); + return file.write(buffer, bytes); +} + +BOOL LLDiskCache::seek(S32 offset, S32 origin) +{ + if (-1 == origin) + { + origin = mPosition; + } + + S32 new_pos = origin + offset; + + S32 size = getSize(); + + if (new_pos > size) + { + LL_WARNS() << "Attempt to seek past end of file" << LL_ENDL; + + mPosition = size; + return FALSE; + } + else if (new_pos < 0) + { + LL_WARNS() << "Attempt to seek past beginning of file" << LL_ENDL; + + mPosition = 0; + return FALSE; + } + + mPosition = new_pos; + return TRUE; +} + +S32 LLDiskCache::tell() const +{ + return mPosition; +} + +S32 LLDiskCache::getSize() +{ + return LLDiskCache::getFileSize(mFileID, mFileType); +} + +S32 LLDiskCache::getMaxSize() +{ + // offer up a huge size since we don't care what the max is + return INT_MAX; +} + +BOOL LLDiskCache::setMaxSize(S32 size) +{ + // we don't care what the max size is so we do nothing + // and return true to indicate all was okay + return TRUE; +} + +BOOL LLDiskCache::rename(const LLUUID &new_id, const LLAssetType::EType new_type) +{ + LLDiskCache::renameFile(mFileID, mFileType, new_id, new_type); + + mFileID = new_id; + mFileType = new_type; + + return TRUE; +} + +BOOL LLDiskCache::remove() +{ + LLDiskCache::removeFile(mFileID, mFileType); + + return TRUE; +} + +// static +void LLDiskCache::initClass() +{ +} + +// static +void LLDiskCache::cleanupClass() +{ +} + +bool LLDiskCache::isLocked() +{ + // I don't think we care about this test since there is no locking + // TODO: remove this function and calling sites? + return FALSE; +} + +void LLDiskCache::waitForLock() +{ + // TODO: remove this function and calling sites? +} diff --git a/indra/llfilesystem/llfilesystem.h b/indra/llfilesystem/llfilesystem.h new file mode 100644 index 0000000000..7ad06a8689 --- /dev/null +++ b/indra/llfilesystem/llfilesystem.h @@ -0,0 +1,82 @@ +/** + * @file lldiskcacke.h + * @brief Definition of virtual file + * + * $LicenseInfo:firstyear=2002&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2010, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef LL_LLDISKCACHE_H +#define LL_LLDISKCACHE_H + +#include "lluuid.h" +#include "llassettype.h" + +class LLDiskCache +{ +public: + LLDiskCache(const LLUUID &file_id, const LLAssetType::EType file_type, S32 mode = LLDiskCache::READ); + ~LLDiskCache(); + + BOOL read(U8 *buffer, S32 bytes, BOOL async = FALSE, F32 priority = 128.f); /* Flawfinder: ignore */ + BOOL isReadComplete(); + S32 getLastBytesRead(); + BOOL eof(); + + BOOL write(const U8 *buffer, S32 bytes); + static BOOL writeFile(const U8 *buffer, S32 bytes, const LLUUID &uuid, LLAssetType::EType type); + BOOL seek(S32 offset, S32 origin = -1); + S32 tell() const; + + S32 getSize(); + S32 getMaxSize(); + BOOL setMaxSize(S32 size); + BOOL rename(const LLUUID &new_id, const LLAssetType::EType new_type); + BOOL remove(); + + bool isLocked(); + void waitForLock(); + + static bool getExists(const LLUUID &file_id, const LLAssetType::EType file_type); + static bool removeFile(const LLUUID &file_id, const LLAssetType::EType file_type); + static bool renameFile(const LLUUID &old_file_id, const LLAssetType::EType old_file_type, + const LLUUID &new_file_id, const LLAssetType::EType new_file_type); + static S32 getFileSize(const LLUUID &file_id, const LLAssetType::EType file_type); + + static void initClass(); + static void cleanupClass(); + +public: + static const S32 READ; + static const S32 WRITE; + static const S32 READ_WRITE; + static const S32 APPEND; + +protected: + LLAssetType::EType mFileType; + BOOL mReadComplete; + LLUUID mFileID; + S32 mPosition; + S32 mMode; + S32 mBytesRead; +}; + +#endif // LL_LLDISKCACHE_H -- cgit v1.2.3 From 6be1f88a5ef99e1e40bb5701a250ba0728f56005 Mon Sep 17 00:00:00 2001 From: Callum Prentice Date: Thu, 24 Sep 2020 14:45:39 -0700 Subject: Complete the change from lldiskcache -> llfilesystem and then addition of new lldiskcache implementation --- indra/llfilesystem/CMakeLists.txt | 3 +- indra/llfilesystem/lldiskcache.cpp | 773 ++++++++++++++++++++++++++++++++++++ indra/llfilesystem/lldiskcache.h | 130 ++++++ indra/llfilesystem/llfilesystem.cpp | 79 ++-- indra/llfilesystem/llfilesystem.h | 19 +- 5 files changed, 955 insertions(+), 49 deletions(-) create mode 100644 indra/llfilesystem/lldiskcache.cpp create mode 100644 indra/llfilesystem/lldiskcache.h (limited to 'indra/llfilesystem') diff --git a/indra/llfilesystem/CMakeLists.txt b/indra/llfilesystem/CMakeLists.txt index 306b483097..d1dece5bba 100644 --- a/indra/llfilesystem/CMakeLists.txt +++ b/indra/llfilesystem/CMakeLists.txt @@ -18,16 +18,17 @@ set(llfilesystem_SOURCE_FILES lldiriterator.cpp lllfsthread.cpp lldiskcache.cpp + llfilesystem.cpp ) set(llfilesystem_HEADER_FILES CMakeLists.txt - lldir.h lldirguard.h lldiriterator.h lllfsthread.h lldiskcache.h + llfilesystem.h ) if (DARWIN) diff --git a/indra/llfilesystem/lldiskcache.cpp b/indra/llfilesystem/lldiskcache.cpp new file mode 100644 index 0000000000..72982db069 --- /dev/null +++ b/indra/llfilesystem/lldiskcache.cpp @@ -0,0 +1,773 @@ +/** + * @file lldiskcache.cpp + * @brief The SQLite based disk cache implementation. + * @Note Eventually, this component might split into an interface + * file and multiple implemtations but for now, this is the + * only one so I think it's okay to combine everything. + * + * $LicenseInfo:firstyear=2009&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2020, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#if (defined(LL_WINDOWS) || defined(LL_LINUX) || defined(LL_DARWIN)) +#include "linden_common.h" +#endif + +#include "lldiskcache.h" + +#include +#include +#include +#include +#include + +/////////////////////////////////////////////////////////////////////////////// +// +llDiskCache::llDiskCache() : + mDataStorePath("") +{ +} + +llDiskCache::~llDiskCache() +{ +} + +/////////////////////////////////////////////////////////////////////////////// +// Opens the database - typically done when the application starts +// and is complementary to close() which is called when the application +// is finisahed and exits. +// Pass in the folder and filename of the SQLite database you want to +// use or create (file doesn't have to exist but the folder must) +// Returns true or false and writes a message to console on error +bool llDiskCache::open(const std::string db_folder, const std::string db_filename) +{ + mDataStorePath = db_folder; + std::string db_pathname = makeFullPath(db_filename); + + // simple flags for the moment - these will likely be extended + // later on to support the SQLite mutex model for reading/writing + // simultaneously - perhaps when we look at supporting textures too + int flags = SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE; + + if (sqlite3_open_v2( + db_pathname.c_str(), + &mDb, + flags, + nullptr // Name of VFS module to use + ) != SQLITE_OK) + { + printError(__FUNCDNAME__ , "open_v2", true); + close(); + return false; + } + + // I elected to store the code that generates the statement + // in sepsrate functions throughout - this seemed like a cleaner + // approach than having hundreds of stmt << "REPLACE AS" lines + // interspersed in the logic code. They are all prefixed with + // 'sqlCompose' and followed by a short description. + const std::string stmt = sqlComposeCreateTable(); + if (! sqliteExec(stmt, __FUNCDNAME__ )) + { + // Not sure if we need close here - if open fails, then we + // perhaps don't need to close it - TODO: look in SQLite docs + close(); + return false; + } + + return true; +} + +/////////////////////////////////////////////////////////////////////////////// +// Determines if an entry exists. +// Pass in the id and a variable that will indicate existance +// Returns true/false if function succeeded and the boolean +// you pass in will be set to true/false if entry exists or not +bool llDiskCache::exists(const std::string id, bool& exists) +{ + if (!mDb) + { + printError(__FUNCDNAME__ , "mDb is invalid", false); + return false; + } + + if (id.empty()) + { + printError(__FUNCDNAME__ , "id is empty", false); + return false; + } + + // As well as the separate function to compose the SQL statement, + // worth pointing out that the code to 'prepare' and 'step' the + // SQLite "prepared statement" has been factored out into its own + // function and used in several other functions. + const std::string stmt = sqlComposeExists(id); + sqlite3_stmt* prepared_statement = sqlitePrepareStep(__FUNCDNAME__ , stmt, SQLITE_ROW); + if (! prepared_statement) + { + return false; + } + + int result_column_index = 0; + int result_count = sqlite3_column_int(prepared_statement, result_column_index); + if (sqlite3_finalize(prepared_statement) != SQLITE_OK) + { + printError(__FUNCDNAME__ , "sqlite3_finalize()", true); + return false; + } + + // given the uniqueness of the ID, this can only ever be + // either 1 or 0 so this might be a bit confusing + exists = result_count > 0 ? true : false; + + return true; +} + +/////////////////////////////////////////////////////////////////////////////// +// Given an id (likely a UUID + decoration but can be any string), a +// pointer to a blob of binary data along with its length, write the +// entry to the cache +// Returns true/false for success/failure +bool llDiskCache::put(const std::string id, + char* binary_data, + int binary_data_size) +{ + if (!mDb) + { + printError(__FUNCDNAME__ , "mDb is invalid", false); + return false; + } + + if (id.empty()) + { + printError(__FUNCDNAME__ , "id is empty", false); + return false; + } + + // < 0 is obvious but we assert the data must be at least 1 byte long + if (binary_data_size <= 0) + { + printError(__FUNCDNAME__ , "size of binary file to write is invalid", false); + return false; + } + + // we generate a unique filename for the actual data itself + // which is stored on disk directly and not in the database. + // TODO: consider making the filename more like the ID passed in + // although the problem with that is we would have to parse the id + // to remove invalid filename chars, consider length etc. As it + // stands, we can write a simple SQL statement to return the filename + // given the ID. + const std::string filename = makeUniqueFilename(); + const std::string filepath = makeFullPath(filename); + std::ofstream file(filepath, std::ios::out | std::ios::binary); + if (! file.is_open()) + { + std::ostringstream error; + error << "Unable to open " << filepath << " for writing"; + printError(__FUNCDNAME__ , error.str(), false); + return false; + } + + file.write((char*)binary_data, binary_data_size); + file.close(); + + // I think is a catchall "wasn't able to write the file to disk" + // conditional but should revisit when we hook this up to viewer + // code to make sure - we never want to write bad/no data to the + // disk and not indicate something has gone wrong + if (file.bad()) + { + std::ostringstream error; + error << "Unable to write " << binary_data_size; + error << " bytes to " << filepath; + printError(__FUNCDNAME__ , error.str(), false); + + return false; + } + + // this is where the filename/size is written to the database along + // with the current date/time for the created/last access times + const std::string stmt = sqlComposePut(id, filename, binary_data_size); + if (! sqlitePrepareStep(__FUNCDNAME__ , stmt, SQLITE_DONE)) + { + return false; + } + + return true; +} + +/////////////////////////////////////////////////////////////////////////////// +// Given an id (likely a UUID + decoration but can be any string), the +// address of a pointer that will be used to allocate memory and a +// varialble that will contain the length of the data, get an item from +// the cache. Note that the memory buffer returned belongs to the calling +// function and it is its responsiblity to clean it up when it's no +// longer needed. +// Returns true/false for success/failure +const bool llDiskCache::get(const std::string id, + char** mem_buffer, + int& mem_buffer_size) +{ + // Check if the entry exists first to avoid dealing with a bunch + // of conditions that look like failure but aren't in the main code. + // Note exists() is a public method and also tests for mDB and id + // being valid so we can safely put this about the same tests + // in this function + bool get_exists = false; + if (! exists(id, get_exists)) + { + return false; + } + if (! get_exists) + { + return false; + } + + if (!mDb) + { + printError(__FUNCDNAME__ , "mDb is invalid", false); + return false; + } + + if (id.empty()) + { + printError(__FUNCDNAME__ , "id is empty", false); + return false; + } + + const std::string stmt_select = sqlComposeGetSelect(id); + sqlite3_stmt* prepared_statement = sqlitePrepareStep(__FUNCDNAME__ , stmt_select, SQLITE_ROW); + if (! prepared_statement) + { + return false; + } + + int result_column_index = 0; + const unsigned char* text = sqlite3_column_text(prepared_statement, result_column_index); + if (text == nullptr) + { + printError(__FUNCDNAME__ , "filename is nullptr", true); + return false; + } + const std::string filename = std::string(reinterpret_cast(text)); + const std::string filepath = makeFullPath(filename); + + result_column_index = 1; + int filesize_db = sqlite3_column_int(prepared_statement, result_column_index); + if (filesize_db <= 0) + { + printError(__FUNCDNAME__ , "filesize is invalid", true); + return false; + } + + if (sqlite3_finalize(prepared_statement) != SQLITE_OK) + { + printError(__FUNCDNAME__ , "sqlite3_finalize()", true); + return false; + } + + // Now we have the fiename, we can read the file from disk + std::ifstream file(filepath, std::ios::in | std::ios::binary | std::ios::ate); + if (! file.is_open()) + { + std::ostringstream error; + error << "Unable to open " << filepath << " for reading"; + printError(__FUNCDNAME__ , error.str(), false); + return false; + } + + // we get the expected filesize from the database but we can also + // get it (easily) when we read the file from the disk. We compare + // the two and return false if they don't match + std::streampos filesize_file = file.tellg(); + if (filesize_db != filesize_file) + { + std::ostringstream error; + error << "File size from DB (" << filesize_db << ")"; + error << " and "; + error << "file size from file (" << filesize_file << ")"; + error << " in file " << filepath << " are different"; + printError(__FUNCDNAME__ , error.str(), false); + + return false; + } + + // doest matter if we choose DB or file version - they must be + // identical if we get this far - just used for clarity + int filesize = filesize_db; + + // allocate a block of memory that we pass back for the calling + // function to use then delete when it's no longer needed + *mem_buffer = new char[filesize]; + mem_buffer_size = filesize; + + file.seekg(0, std::ios::beg); + file.read(*mem_buffer, filesize); + file.close(); + + if (file.bad()) + { + std::ostringstream error; + error << "Unable to read " << filesize; + error << " bytes from " << filepath; + printError(__FUNCDNAME__ , error.str(), false); + + return false; + } + + // here we update the count of times the file is accessed so + // we can keep track of how many times it's been requested. + // This will be useful for metrics and perhaps determining + // if a file should not be purged even though its age + // might suggest that it should. + // In addition, this is where the time of last access is updated + // in the database and that us used to determine what is purged + // in an LRU fashion when the purge function is called. + const std::string stmt_update = sqlComposeGetUpdate(id); + if (! sqliteExec(stmt_update, __FUNCDNAME__ )) + { + return false; + } + + return true; +} + +/////////////////////////////////////////////////////////////////////////////// +// Purges the database of older entries using an LRU approach. +// Pass in the number of entries to retain. +// This is called now after open to "clean up" the cache when the +// application starts. +// TODO: IMPORTANT: compose a list of files that will be deleted +// and delete them from disk too - not just from the DB +bool llDiskCache::purge(int num_entries) +{ + if (num_entries < 0) + { + printError(__FUNCDNAME__ , "number of entries to purge is invalid", false); + return false; + } + + // find the rows affected and get the filenames for them +//swww + + // delete oldest entries leaving the correct number in place + const std::string stmt = sqlComposePurge(num_entries); + if (! sqliteExec(stmt, __FUNCDNAME__ )) + { + return false; + } + + return true; +} + +/////////////////////////////////////////////////////////////////////////////// +// Call at application shutdown +void llDiskCache::close() +{ + sqlite3_close(mDb); +} + +/////////////////////////////////////////////////////////////////////////////// +// Determine the version of SQLite in use +// TODO: make this a static so we can get to it from the Viewer About +// box without instantiating the whole thing. +const std::string llDiskCache::dbVersion() +{ + std::ostringstream version; + + version << sqlite3_libversion(); + + return version.str(); +} + +/////////////////////////////////////////////////////////////////////////////// +// Given an id, return the matching filename +const std::string llDiskCache::getFilenameById(const std::string id) +{ + // TODO: + return std::string(); +} + +/////////////////////////////////////////////////////////////////////////////// +// Given an id, return the number of times that entry has been +// accessed from the cache +const int llDiskCache::getAccessCountById(const std::string id) +{ + // TODO: + return -1; +} + +/////////////////////////////////////////////////////////////////////////////// +// Return the number of entries currently in the cache as well as +// the maximum possible entries. +void llDiskCache::getNumEntries(int& num_entries, int& max_entries) +{ + num_entries = -1; + max_entries = -1; +} + +/////////////////////////////////////////////////////////////////////////////// +// Wraps the sqlite3_exec(..) used in many places +bool llDiskCache::sqliteExec(const std::string stmt, + const std::string funcname) +{ + if (sqlite3_exec( + mDb, + stmt.c_str(), + nullptr, // Callback function (unused) + nullptr, // 1st argument to callback (unused) + nullptr // Error msg written here (unused) + ) != SQLITE_OK) + { + printError(funcname, "sqlite3_exec", true); + return false; + } + + return true; +} + +/////////////////////////////////////////////////////////////////////////////// +// Wraps the sqlite3_prepare_v2 and sqlite3_step calls used in many places +sqlite3_stmt* llDiskCache::sqlitePrepareStep(const std::string funcname, + const std::string stmt, + int sqlite_success_condition) +{ + sqlite3_stmt* prepared_statement; + + if (sqlite3_prepare_v2( + mDb, + stmt.c_str(), + -1, // Maximum length of zSql in bytes. + &prepared_statement, + 0 // OUT: Pointer to unused portion of zSql + ) != SQLITE_OK) + { + printError(funcname, "sqlite3_prepare_v2", true); + return nullptr; + } + + if (sqlite3_step(prepared_statement) != sqlite_success_condition) + { + printError(funcname, "sqlite3_step", true); + sqlite3_finalize(prepared_statement); + return nullptr; + } + return prepared_statement; +} + +/////////////////////////////////////////////////////////////////////////////// +// When an "error" occurss - e.g. database cannot be found, file cannot be +// written, invalid argument passed into a function etc. a message is +// written to stderr that should end up in the viewer log +// TODO: Set the verbosity using the usual Viewer mechanism +void llDiskCache::printError(const std::string funcname, + const std::string desc, + bool is_sqlite_err) +{ + std::ostringstream err_msg; + + err_msg << "llDiskCache error in "; + err_msg << __FUNCDNAME__ << "(...) "; + err_msg << desc; + + if (is_sqlite_err) + { + err_msg << " - "; + err_msg << std::string(sqlite3_errmsg(mDb)); + } + + // TODO: set via viewer verbosity level + const int verbosity = 1; + if (verbosity > 0) + { + std::cerr << err_msg.str() << std::endl; + } +} + +/////////////////////////////////////////////////////////////////////////////// +// wrapper for SQLite code to begin an SQL transaction - not used yet but +// it will be eventually +bool llDiskCache::beginTransaction() +{ + const std::string stmt("BEGIN TRANSACTION"); + if (! sqliteExec(stmt, __FUNCDNAME__ )) + { + return false; + } + + return true; +} + +/////////////////////////////////////////////////////////////////////////////// +// wrapper for SQLite code to end an SQL transaction - not used yet but +// it will be eventually +bool llDiskCache::endTransaction() +{ + const std::string stmt("COMMIT"); + if (! sqliteExec(stmt, __FUNCDNAME__ )) + { + return false; + } + + return true; +} + +/////////////////////////////////////////////////////////////////////////////// +// Build a unique filename that will be used to store the actual file +// on disk (as opposed to the meta data in the database) +// TODO: I think this needs more work once we move it to the viewer +// and espcially to make it cross platform - see 'std::tmpnam' +// depreciation comments in compiler output for example +std::string llDiskCache::makeUniqueFilename() +{ + // TODO: replace with boost - this is marked as deprecated? + std::string base = std::tmpnam(nullptr); + + // C++11 random number generation!!! + static std::random_device dev; + static std::mt19937 rng(dev()); + std::uniform_int_distribution dist(100000, 999999); + + // currently the tmp filename from std::tmpnam() on macOS + // is of the form `/tmp/foo/bar.12345 and the following code + // strips all the preceding dirs - we likely want a more + // robust (and cross platform solution) when we move to the + // viewer code + std::size_t found = base.rfind(systemSeparator()); + if (found != std::string::npos && found < base.size() - 2) + { + base = base.substr(found + 1); + } + else + { + base = ""; + } + + // we mix in a random number for some more entropy.. + // (i know, i know...) + std::ostringstream unique_filename; + unique_filename << base; + unique_filename << "."; + unique_filename << dist(rng); + + return unique_filename.str(); +} + +/////////////////////////////////////////////////////////////////////////////// +// Return system file/path separator - likely replaced by the version +// in the viewer +const std::string llDiskCache::systemSeparator() +{ +// TODO: replace in viewer with relevant call +#ifdef _WIN32 + return "\\"; +#else + return "/"; +#endif +} + +/////////////////////////////////////////////////////////////////////////////// +// Given a filename, compose a full path based on the path name passed +// in when the database was opened and the separator in play. +const std::string llDiskCache::makeFullPath(const std::string filename) +{ + std::string pathname = mDataStorePath + systemSeparator() + filename; + + return pathname; +} + +/////////////////////////////////////////////////////////////////////////////// +// +const std::string llDiskCache::sqlComposeCreateTable() +{ + std::ostringstream stmt; + stmt << "CREATE TABLE IF NOT EXISTS "; + stmt << mTableName; + stmt << "("; + stmt << mIdFieldName; + stmt << " TEXT PRIMARY KEY, "; + stmt << mFilenameFieldName; + stmt << " TEXT, "; + stmt << mFilesizeFieldName; + stmt << " INTEGER DEFAULT 0, "; + stmt << mInsertionDateTimeFieldName; + stmt << " TEXT, "; + stmt << mLastAccessDateTimeFieldName; + stmt << " TEXT, "; + stmt << mAccessCountFieldName; + stmt << " INTEGER DEFAULT 0"; + stmt << ")"; + +#ifdef SHOW_STATEMENTS + std::cout << stmt.str() << std::endl; +#endif + + return stmt.str(); +} + +/////////////////////////////////////////////////////////////////////////////// +// +const std::string llDiskCache::sqlComposeExists(const std::string id) +{ + std::ostringstream stmt; + stmt << "SELECT COUNT(*) FROM "; + stmt << mTableName; + stmt << " WHERE "; + stmt << mIdFieldName; + stmt << "='"; + stmt << id; + stmt << "'"; + +#ifdef SHOW_STATEMENTS + std::cout << stmt.str() << std::endl; +#endif + + return stmt.str(); +} + +/////////////////////////////////////////////////////////////////////////////// +// SQL statement to write an entry to the DB +// Saves id, filename (generated), file size and create/last access date +const std::string llDiskCache::sqlComposePut(const std::string id, + const std::string filename, + int binary_data_size) +{ + std::ostringstream stmt; + stmt << "REPLACE INTO "; + stmt << mTableName; + stmt << "("; + stmt << mIdFieldName; + stmt << ","; + stmt << mFilenameFieldName; + stmt << ","; + stmt << mFilesizeFieldName; + stmt << ","; + stmt << mInsertionDateTimeFieldName; + stmt << ","; + stmt << mLastAccessDateTimeFieldName; + stmt << ") "; + stmt << "VALUES("; + stmt << "'"; + stmt << id; + stmt << "', "; + stmt << "'"; + stmt << filename; + stmt << "', "; + stmt << binary_data_size; + stmt << ", "; + stmt << "strftime('%Y-%m-%d %H:%M:%f', 'now')"; + stmt << ", "; + stmt << "strftime('%Y-%m-%d %H:%M:%f', 'now')"; + stmt << ")"; + +#ifdef SHOW_STATEMENTS + std::cout << stmt.str() << std::endl; +#endif + + return stmt.str(); +} + +/////////////////////////////////////////////////////////////////////////////// +// SQL statement used in get() to look up the filename and file size +const std::string llDiskCache::sqlComposeGetSelect(const std::string id) +{ + std::ostringstream stmt; + stmt << "SELECT "; + stmt << mFilenameFieldName; + stmt << ", "; + stmt << mFilesizeFieldName; + stmt << " FROM "; + stmt << mTableName; + stmt << " WHERE "; + stmt << mIdFieldName; + stmt << "='"; + stmt << id; + stmt << "'"; + +#ifdef SHOW_STATEMENTS + std::cout << stmt.str() << std::endl; +#endif + + return stmt.str(); +} + +/////////////////////////////////////////////////////////////////////////////// +// SQL statement to update the date/time of last access as well as the +// count of number of times the file has been accessed. +// Note: the more accurate representation of date/time is used to +// ensure ms accuracy vs the standard INTEGER days since epoch approach +const std::string llDiskCache::sqlComposeGetUpdate(const std::string id) +{ + std::ostringstream stmt; + stmt << "UPDATE "; + stmt << mTableName; + stmt << " SET "; + stmt << mAccessCountFieldName; + stmt << "="; + stmt << mAccessCountFieldName; + stmt << "+1"; + stmt << ", "; + stmt << mLastAccessDateTimeFieldName; + stmt << "="; + stmt << "strftime('%Y-%m-%d %H:%M:%f', 'now')"; + stmt << " WHERE "; + stmt << mIdFieldName; + stmt << "='"; + stmt << id; + stmt << "'"; + +#ifdef SHOW_STATEMENTS + std::cout << stmt.str() << std::endl; +#endif + + return stmt.str(); +} + +/////////////////////////////////////////////////////////////////////////////// +// SQL statement to remove items from the database that are older +// than the newest num_elements entries +const std::string llDiskCache::sqlComposePurge(int num_entries) +{ + std::ostringstream stmt; + stmt << "DELETE FROM "; + stmt << mTableName; + stmt << " WHERE "; + stmt << mLastAccessDateTimeFieldName; + stmt << " NOT IN "; + stmt << "("; + stmt << "SELECT "; + stmt << mLastAccessDateTimeFieldName; + stmt << " FROM "; + stmt << mTableName; + stmt << " ORDER BY "; + stmt << mLastAccessDateTimeFieldName; + stmt << " DESC"; + stmt << " LIMIT "; + stmt << num_entries; + stmt << ")"; + +//#ifdef SHOW_STATEMENTS + std::cout << stmt.str() << std::endl; +//#endif + + return stmt.str(); +} diff --git a/indra/llfilesystem/lldiskcache.h b/indra/llfilesystem/lldiskcache.h new file mode 100644 index 0000000000..39b8f7ef48 --- /dev/null +++ b/indra/llfilesystem/lldiskcache.h @@ -0,0 +1,130 @@ +/** + * @file lldiskcache.h + * @brief Declaration SQLite meta data / file storage API + * @brief Declaration of the generic disk cache interface + as well the SQLite header/implementation. + * @Note Eventually, this component might split into an interface + * file and multiple implemtations but for now, this is the + * only one so I think it's okay to combine everything. + * + * $LicenseInfo:firstyear=2009&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2020, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef _LLDISKCACHE +#define _LLDISKCACHE + +#include +#include + +#include "sqlite3.h" + +// Enable this line to display each SQL statement when it is executed +// It can lead to a lot of spam but useful for debugging +//#define SHOW_STATEMENTS + +// I toyed with the idea of a variety of approaches and thought have +// an abstract base class with which to hang different implementations +// off was a good idea but I think we settled on a single approach +// so I will likely remove this level of indirection if not other +// interesting implementation ideas present themselves. +class llDiskCacheBase +{ + public: + llDiskCacheBase() {}; + virtual ~llDiskCacheBase() {}; + virtual bool open(const std::string db_folder, + const std::string db_filename) = 0; + virtual bool exists(const std::string id, bool& exists) = 0; + virtual bool put(const std::string id, + char* binary_data, + int binary_data_size) = 0; + virtual const bool get(const std::string id, + char** mem_buffer, + int& mem_buffer_size) = 0; + virtual bool purge(int num_entries) = 0; + virtual void close() = 0; + virtual const std::string dbVersion() = 0; + virtual const std::string getFilenameById(const std::string id) = 0; + virtual const int getAccessCountById(const std::string id) = 0; + virtual void getNumEntries(int& num_entries, int& max_entries) = 0; +}; + +// implementation for the SQLite/disk blended case +// see source file for detailed comments +class llDiskCache : + public llDiskCacheBase +{ + public: + llDiskCache(); + virtual ~llDiskCache(); + virtual bool open(const std::string db_folder, + const std::string db_filename) override; + virtual bool exists(const std::string id, bool& exists) override; + virtual bool put(const std::string id, + char* binary_data, + int binary_data_size) override; + virtual const bool get(const std::string id, + char** mem_buffer, + int& mem_buffer_size) override; + virtual bool purge(int num_entries) override; + virtual void close() override; + virtual const std::string dbVersion() override; + virtual const std::string getFilenameById(const std::string id) override; + virtual const int getAccessCountById(const std::string id) override; + virtual void getNumEntries(int& num_entries, int& max_entries) override; + + private: + sqlite3* mDb; + std::string mDataStorePath; + const std::string mTableName = "lldiskcache"; + const std::string mIdFieldName = "id"; + const std::string mFilenameFieldName = "filename"; + const std::string mFilesizeFieldName = "filesize"; + const std::string mInsertionDateTimeFieldName = "insert_datetime"; + const std::string mLastAccessDateTimeFieldName = "last_access_datetime"; + const std::string mAccessCountFieldName = "access_count"; + + private: + void printError(const std::string funcname, + const std::string desc, + bool is_sqlite_err); + bool beginTransaction(); + bool endTransaction(); + std::string makeUniqueFilename(); + const std::string systemSeparator(); + const std::string makeFullPath(const std::string filename); + bool sqliteExec(const std::string stmt, + const std::string funcname); + sqlite3_stmt* sqlitePrepareStep(const std::string funcname, + const std::string stmt, + int sqlite_success_condition); + const std::string sqlComposeCreateTable(); + const std::string sqlComposeExists(const std::string id); + const std::string sqlComposePut(const std::string id, + const std::string filename, + int binary_data_size); + const std::string sqlComposeGetSelect(const std::string id); + const std::string sqlComposeGetUpdate(const std::string id); + const std::string sqlComposePurge(int num_entries); +}; + +#endif // _LLDISKCACHE diff --git a/indra/llfilesystem/llfilesystem.cpp b/indra/llfilesystem/llfilesystem.cpp index af93049e07..f0037c9a22 100644 --- a/indra/llfilesystem/llfilesystem.cpp +++ b/indra/llfilesystem/llfilesystem.cpp @@ -1,6 +1,9 @@ /** - * @file lldiskcache.cpp - * @brief Implementation of virtual file + * @file filesystem.h + * @brief Simulate local file system operations. + * @Note The initial implementation does actually use standard C++ + * file operations but eventually, there will be another + * layer that caches and manages file meta data too. * * $LicenseInfo:firstyear=2002&license=viewerlgpl$ * Second Life Viewer Source Code @@ -26,25 +29,21 @@ #include "linden_common.h" -#include "lldiskcache.h" - -#include "llerror.h" -#include "llthread.h" -#include "lltimer.h" +#include "lldir.h" +#include "llfilesystem.h" #include "llfasttimer.h" -#include "llmemory.h" +#include "lldiskcache.h" #include -#include "lldir.h" -const S32 LLDiskCache::READ = 0x00000001; -const S32 LLDiskCache::WRITE = 0x00000002; -const S32 LLDiskCache::READ_WRITE = 0x00000003; // LLDiskCache::READ & LLDiskCache::WRITE -const S32 LLDiskCache::APPEND = 0x00000006; // 0x00000004 & LLDiskCache::WRITE +const S32 LLFileSystem::READ = 0x00000001; +const S32 LLFileSystem::WRITE = 0x00000002; +const S32 LLFileSystem::READ_WRITE = 0x00000003; // LLFileSystem::READ & LLFileSystem::WRITE +const S32 LLFileSystem::APPEND = 0x00000006; // 0x00000004 & LLFileSystem::WRITE static LLTrace::BlockTimerStatHandle FTM_VFILE_WAIT("VFile Wait"); -LLDiskCache::LLDiskCache(const LLUUID &file_id, const LLAssetType::EType file_type, S32 mode) +LLFileSystem::LLFileSystem(const LLUUID &file_id, const LLAssetType::EType file_type, S32 mode) { mFileType = file_type; mFileID = file_id; @@ -54,7 +53,7 @@ LLDiskCache::LLDiskCache(const LLUUID &file_id, const LLAssetType::EType file_ty mMode = mode; } -LLDiskCache::~LLDiskCache() +LLFileSystem::~LLFileSystem() { } @@ -124,7 +123,7 @@ const std::string idToFilepath(const std::string id, LLAssetType::EType at) } // static -bool LLDiskCache::getExists(const LLUUID &file_id, const LLAssetType::EType file_type) +bool LLFileSystem::getExists(const LLUUID &file_id, const LLAssetType::EType file_type) { std::string id_str; file_id.toString(id_str); @@ -140,7 +139,7 @@ bool LLDiskCache::getExists(const LLUUID &file_id, const LLAssetType::EType file } // static -bool LLDiskCache::removeFile(const LLUUID &file_id, const LLAssetType::EType file_type) +bool LLFileSystem::removeFile(const LLUUID &file_id, const LLAssetType::EType file_type) { std::string id_str; file_id.toString(id_str); @@ -152,7 +151,7 @@ bool LLDiskCache::removeFile(const LLUUID &file_id, const LLAssetType::EType fil } // static -bool LLDiskCache::renameFile(const LLUUID &old_file_id, const LLAssetType::EType old_file_type, +bool LLFileSystem::renameFile(const LLUUID &old_file_id, const LLAssetType::EType old_file_type, const LLUUID &new_file_id, const LLAssetType::EType new_file_type) { std::string old_id_str; @@ -175,7 +174,7 @@ bool LLDiskCache::renameFile(const LLUUID &old_file_id, const LLAssetType::EType } // static -S32 LLDiskCache::getFileSize(const LLUUID &file_id, const LLAssetType::EType file_type) +S32 LLFileSystem::getFileSize(const LLUUID &file_id, const LLAssetType::EType file_type) { std::string id_str; file_id.toString(id_str); @@ -192,7 +191,7 @@ S32 LLDiskCache::getFileSize(const LLUUID &file_id, const LLAssetType::EType fil return file_size; } -BOOL LLDiskCache::read(U8 *buffer, S32 bytes, BOOL async, F32 priority) +BOOL LLFileSystem::read(U8 *buffer, S32 bytes, BOOL async, F32 priority) { BOOL success = TRUE; @@ -232,7 +231,7 @@ BOOL LLDiskCache::read(U8 *buffer, S32 bytes, BOOL async, F32 priority) return success; } -BOOL LLDiskCache::isReadComplete() +BOOL LLFileSystem::isReadComplete() { if (mReadComplete) { @@ -242,17 +241,17 @@ BOOL LLDiskCache::isReadComplete() return FALSE; } -S32 LLDiskCache::getLastBytesRead() +S32 LLFileSystem::getLastBytesRead() { return mBytesRead; } -BOOL LLDiskCache::eof() +BOOL LLFileSystem::eof() { return mPosition >= getSize(); } -BOOL LLDiskCache::write(const U8 *buffer, S32 bytes) +BOOL LLFileSystem::write(const U8 *buffer, S32 bytes) { std::string id_str; mFileID.toString(id_str); @@ -287,14 +286,14 @@ BOOL LLDiskCache::write(const U8 *buffer, S32 bytes) } //static -BOOL LLDiskCache::writeFile(const U8 *buffer, S32 bytes, const LLUUID &uuid, LLAssetType::EType type) +BOOL LLFileSystem::writeFile(const U8 *buffer, S32 bytes, const LLUUID &uuid, LLAssetType::EType type) { - LLDiskCache file(uuid, type, LLDiskCache::WRITE); + LLFileSystem file(uuid, type, LLFileSystem::WRITE); file.setMaxSize(bytes); return file.write(buffer, bytes); } -BOOL LLDiskCache::seek(S32 offset, S32 origin) +BOOL LLFileSystem::seek(S32 offset, S32 origin) { if (-1 == origin) { @@ -324,32 +323,32 @@ BOOL LLDiskCache::seek(S32 offset, S32 origin) return TRUE; } -S32 LLDiskCache::tell() const +S32 LLFileSystem::tell() const { return mPosition; } -S32 LLDiskCache::getSize() +S32 LLFileSystem::getSize() { - return LLDiskCache::getFileSize(mFileID, mFileType); + return LLFileSystem::getFileSize(mFileID, mFileType); } -S32 LLDiskCache::getMaxSize() +S32 LLFileSystem::getMaxSize() { // offer up a huge size since we don't care what the max is return INT_MAX; } -BOOL LLDiskCache::setMaxSize(S32 size) +BOOL LLFileSystem::setMaxSize(S32 size) { // we don't care what the max size is so we do nothing // and return true to indicate all was okay return TRUE; } -BOOL LLDiskCache::rename(const LLUUID &new_id, const LLAssetType::EType new_type) +BOOL LLFileSystem::rename(const LLUUID &new_id, const LLAssetType::EType new_type) { - LLDiskCache::renameFile(mFileID, mFileType, new_id, new_type); + LLFileSystem::renameFile(mFileID, mFileType, new_id, new_type); mFileID = new_id; mFileType = new_type; @@ -357,31 +356,31 @@ BOOL LLDiskCache::rename(const LLUUID &new_id, const LLAssetType::EType new_type return TRUE; } -BOOL LLDiskCache::remove() +BOOL LLFileSystem::remove() { - LLDiskCache::removeFile(mFileID, mFileType); + LLFileSystem::removeFile(mFileID, mFileType); return TRUE; } // static -void LLDiskCache::initClass() +void LLFileSystem::initClass() { } // static -void LLDiskCache::cleanupClass() +void LLFileSystem::cleanupClass() { } -bool LLDiskCache::isLocked() +bool LLFileSystem::isLocked() { // I don't think we care about this test since there is no locking // TODO: remove this function and calling sites? return FALSE; } -void LLDiskCache::waitForLock() +void LLFileSystem::waitForLock() { // TODO: remove this function and calling sites? } diff --git a/indra/llfilesystem/llfilesystem.h b/indra/llfilesystem/llfilesystem.h index 7ad06a8689..9cddef74a7 100644 --- a/indra/llfilesystem/llfilesystem.h +++ b/indra/llfilesystem/llfilesystem.h @@ -1,6 +1,9 @@ /** - * @file lldiskcacke.h - * @brief Definition of virtual file + * @file filesystem.h + * @brief Simulate local file system operations. + * @Note The initial implementation does actually use standard C++ + * file operations but eventually, there will be another + * layer that caches and manages file meta data too. * * $LicenseInfo:firstyear=2002&license=viewerlgpl$ * Second Life Viewer Source Code @@ -24,17 +27,17 @@ * $/LicenseInfo$ */ -#ifndef LL_LLDISKCACHE_H -#define LL_LLDISKCACHE_H +#ifndef LL_FILESYSTEM_H +#define LL_FILESYSTEM_H #include "lluuid.h" #include "llassettype.h" -class LLDiskCache +class LLFileSystem { public: - LLDiskCache(const LLUUID &file_id, const LLAssetType::EType file_type, S32 mode = LLDiskCache::READ); - ~LLDiskCache(); + LLFileSystem(const LLUUID &file_id, const LLAssetType::EType file_type, S32 mode = LLFileSystem::READ); + ~LLFileSystem(); BOOL read(U8 *buffer, S32 bytes, BOOL async = FALSE, F32 priority = 128.f); /* Flawfinder: ignore */ BOOL isReadComplete(); @@ -79,4 +82,4 @@ protected: S32 mBytesRead; }; -#endif // LL_LLDISKCACHE_H +#endif // LL_FILESYSTEM_H -- cgit v1.2.3 From 3092aa8aae496803707980eb456cddbb9960ef1c Mon Sep 17 00:00:00 2001 From: Callum Prentice Date: Tue, 6 Oct 2020 17:16:53 -0700 Subject: Add in the C++ filesystem based cache and clean up some indempotent functions in llfilesystem --- indra/llfilesystem/lldiskcache.cpp | 809 +++++------------------------------- indra/llfilesystem/lldiskcache.h | 146 +++---- indra/llfilesystem/llfilesystem.cpp | 32 +- indra/llfilesystem/llfilesystem.h | 10 +- 4 files changed, 184 insertions(+), 813 deletions(-) (limited to 'indra/llfilesystem') diff --git a/indra/llfilesystem/lldiskcache.cpp b/indra/llfilesystem/lldiskcache.cpp index 72982db069..4b2ba0dd79 100644 --- a/indra/llfilesystem/lldiskcache.cpp +++ b/indra/llfilesystem/lldiskcache.cpp @@ -1,9 +1,6 @@ /** * @file lldiskcache.cpp - * @brief The SQLite based disk cache implementation. - * @Note Eventually, this component might split into an interface - * file and multiple implemtations but for now, this is the - * only one so I think it's okay to combine everything. + * @brief The disk cache implementation. * * $LicenseInfo:firstyear=2009&license=viewerlgpl$ * Second Life Viewer Source Code @@ -27,747 +24,167 @@ * $/LicenseInfo$ */ -#if (defined(LL_WINDOWS) || defined(LL_LINUX) || defined(LL_DARWIN)) #include "linden_common.h" -#endif +#include "lluuid.h" +#include "lldir.h" #include "lldiskcache.h" -#include -#include -#include -#include -#include - -/////////////////////////////////////////////////////////////////////////////// -// -llDiskCache::llDiskCache() : - mDataStorePath("") -{ -} +#include +#include -llDiskCache::~llDiskCache() -{ -} +#include +#include +#include -/////////////////////////////////////////////////////////////////////////////// -// Opens the database - typically done when the application starts -// and is complementary to close() which is called when the application -// is finisahed and exits. -// Pass in the folder and filename of the SQLite database you want to -// use or create (file doesn't have to exist but the folder must) -// Returns true or false and writes a message to console on error -bool llDiskCache::open(const std::string db_folder, const std::string db_filename) +LLDiskCache::LLDiskCache(const std::string cache_dir) : + mCacheDir(cache_dir), + mMaxSizeBytes(mDefaultSizeBytes) { - mDataStorePath = db_folder; - std::string db_pathname = makeFullPath(db_filename); - - // simple flags for the moment - these will likely be extended - // later on to support the SQLite mutex model for reading/writing - // simultaneously - perhaps when we look at supporting textures too - int flags = SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE; - - if (sqlite3_open_v2( - db_pathname.c_str(), - &mDb, - flags, - nullptr // Name of VFS module to use - ) != SQLITE_OK) - { - printError(__FUNCDNAME__ , "open_v2", true); - close(); - return false; - } - - // I elected to store the code that generates the statement - // in sepsrate functions throughout - this seemed like a cleaner - // approach than having hundreds of stmt << "REPLACE AS" lines - // interspersed in the logic code. They are all prefixed with - // 'sqlCompose' and followed by a short description. - const std::string stmt = sqlComposeCreateTable(); - if (! sqliteExec(stmt, __FUNCDNAME__ )) + // no access to LLControlGroup / gSavedSettings so use the system environment + // to trigger additional debugging on top of the default "we purged the cache" + mCacheDebugInfo = true; + if (getenv("LL_CACHE_DEBUGGING") != nullptr) { - // Not sure if we need close here - if open fails, then we - // perhaps don't need to close it - TODO: look in SQLite docs - close(); - return false; + mCacheDebugInfo = true; } - return true; + // create cache dir if it does not exist + boost::filesystem::create_directory(cache_dir); } -/////////////////////////////////////////////////////////////////////////////// -// Determines if an entry exists. -// Pass in the id and a variable that will indicate existance -// Returns true/false if function succeeded and the boolean -// you pass in will be set to true/false if entry exists or not -bool llDiskCache::exists(const std::string id, bool& exists) +LLDiskCache::~LLDiskCache() { - if (!mDb) - { - printError(__FUNCDNAME__ , "mDb is invalid", false); - return false; - } - - if (id.empty()) - { - printError(__FUNCDNAME__ , "id is empty", false); - return false; - } - - // As well as the separate function to compose the SQL statement, - // worth pointing out that the code to 'prepare' and 'step' the - // SQLite "prepared statement" has been factored out into its own - // function and used in several other functions. - const std::string stmt = sqlComposeExists(id); - sqlite3_stmt* prepared_statement = sqlitePrepareStep(__FUNCDNAME__ , stmt, SQLITE_ROW); - if (! prepared_statement) - { - return false; - } - - int result_column_index = 0; - int result_count = sqlite3_column_int(prepared_statement, result_column_index); - if (sqlite3_finalize(prepared_statement) != SQLITE_OK) - { - printError(__FUNCDNAME__ , "sqlite3_finalize()", true); - return false; - } - - // given the uniqueness of the ID, this can only ever be - // either 1 or 0 so this might be a bit confusing - exists = result_count > 0 ? true : false; - - return true; -} - -/////////////////////////////////////////////////////////////////////////////// -// Given an id (likely a UUID + decoration but can be any string), a -// pointer to a blob of binary data along with its length, write the -// entry to the cache -// Returns true/false for success/failure -bool llDiskCache::put(const std::string id, - char* binary_data, - int binary_data_size) -{ - if (!mDb) - { - printError(__FUNCDNAME__ , "mDb is invalid", false); - return false; - } - - if (id.empty()) - { - printError(__FUNCDNAME__ , "id is empty", false); - return false; - } - - // < 0 is obvious but we assert the data must be at least 1 byte long - if (binary_data_size <= 0) - { - printError(__FUNCDNAME__ , "size of binary file to write is invalid", false); - return false; - } - - // we generate a unique filename for the actual data itself - // which is stored on disk directly and not in the database. - // TODO: consider making the filename more like the ID passed in - // although the problem with that is we would have to parse the id - // to remove invalid filename chars, consider length etc. As it - // stands, we can write a simple SQL statement to return the filename - // given the ID. - const std::string filename = makeUniqueFilename(); - const std::string filepath = makeFullPath(filename); - std::ofstream file(filepath, std::ios::out | std::ios::binary); - if (! file.is_open()) - { - std::ostringstream error; - error << "Unable to open " << filepath << " for writing"; - printError(__FUNCDNAME__ , error.str(), false); - return false; - } - - file.write((char*)binary_data, binary_data_size); - file.close(); - - // I think is a catchall "wasn't able to write the file to disk" - // conditional but should revisit when we hook this up to viewer - // code to make sure - we never want to write bad/no data to the - // disk and not indicate something has gone wrong - if (file.bad()) - { - std::ostringstream error; - error << "Unable to write " << binary_data_size; - error << " bytes to " << filepath; - printError(__FUNCDNAME__ , error.str(), false); - - return false; - } - - // this is where the filename/size is written to the database along - // with the current date/time for the created/last access times - const std::string stmt = sqlComposePut(id, filename, binary_data_size); - if (! sqlitePrepareStep(__FUNCDNAME__ , stmt, SQLITE_DONE)) - { - return false; - } - - return true; } -/////////////////////////////////////////////////////////////////////////////// -// Given an id (likely a UUID + decoration but can be any string), the -// address of a pointer that will be used to allocate memory and a -// varialble that will contain the length of the data, get an item from -// the cache. Note that the memory buffer returned belongs to the calling -// function and it is its responsiblity to clean it up when it's no -// longer needed. -// Returns true/false for success/failure -const bool llDiskCache::get(const std::string id, - char** mem_buffer, - int& mem_buffer_size) +void LLDiskCache::purge() { - // Check if the entry exists first to avoid dealing with a bunch - // of conditions that look like failure but aren't in the main code. - // Note exists() is a public method and also tests for mDB and id - // being valid so we can safely put this about the same tests - // in this function - bool get_exists = false; - if (! exists(id, get_exists)) + if (mCacheDebugInfo) { - return false; - } - if (! get_exists) - { - return false; + LL_INFOS() << "Total dir size before purge is " << dirFileSize(mCacheDir) << LL_ENDL; } - if (!mDb) - { - printError(__FUNCDNAME__ , "mDb is invalid", false); - return false; - } - - if (id.empty()) - { - printError(__FUNCDNAME__ , "id is empty", false); - return false; - } + auto start_time = std::chrono::high_resolution_clock::now(); - const std::string stmt_select = sqlComposeGetSelect(id); - sqlite3_stmt* prepared_statement = sqlitePrepareStep(__FUNCDNAME__ , stmt_select, SQLITE_ROW); - if (! prepared_statement) - { - return false; - } + typedef std::pair> file_info_t; + std::vector file_info; - int result_column_index = 0; - const unsigned char* text = sqlite3_column_text(prepared_statement, result_column_index); - if (text == nullptr) + if (boost::filesystem::is_directory(mCacheDir)) { - printError(__FUNCDNAME__ , "filename is nullptr", true); - return false; - } - const std::string filename = std::string(reinterpret_cast(text)); - const std::string filepath = makeFullPath(filename); + for (auto& entry : boost::make_iterator_range(boost::filesystem::directory_iterator(mCacheDir), {})) + { + if (boost::filesystem::is_regular_file(entry)) + { + uintmax_t file_size = boost::filesystem::file_size(entry); + const std::string file_path = entry.path().string(); + const std::time_t file_time = boost::filesystem::last_write_time(entry); - result_column_index = 1; - int filesize_db = sqlite3_column_int(prepared_statement, result_column_index); - if (filesize_db <= 0) - { - printError(__FUNCDNAME__ , "filesize is invalid", true); - return false; + file_info.push_back(file_info_t(file_time, { file_size, file_path })); + } + } } - if (sqlite3_finalize(prepared_statement) != SQLITE_OK) + std::sort(file_info.begin(), file_info.end(), [](file_info_t& x, file_info_t& y) { - printError(__FUNCDNAME__ , "sqlite3_finalize()", true); - return false; - } + return x.first > y.first; + }); - // Now we have the fiename, we can read the file from disk - std::ifstream file(filepath, std::ios::in | std::ios::binary | std::ios::ate); - if (! file.is_open()) - { - std::ostringstream error; - error << "Unable to open " << filepath << " for reading"; - printError(__FUNCDNAME__ , error.str(), false); - return false; - } + LL_INFOS() << "Purging cache to a maximum of " << mMaxSizeBytes << " bytes" << LL_ENDL; - // we get the expected filesize from the database but we can also - // get it (easily) when we read the file from the disk. We compare - // the two and return false if they don't match - std::streampos filesize_file = file.tellg(); - if (filesize_db != filesize_file) + uintmax_t file_size_total = 0; + for (file_info_t& entry : file_info) { - std::ostringstream error; - error << "File size from DB (" << filesize_db << ")"; - error << " and "; - error << "file size from file (" << filesize_file << ")"; - error << " in file " << filepath << " are different"; - printError(__FUNCDNAME__ , error.str(), false); - - return false; - } - - // doest matter if we choose DB or file version - they must be - // identical if we get this far - just used for clarity - int filesize = filesize_db; + file_size_total += entry.second.first; - // allocate a block of memory that we pass back for the calling - // function to use then delete when it's no longer needed - *mem_buffer = new char[filesize]; - mem_buffer_size = filesize; + std::string action = ""; + if (file_size_total > mMaxSizeBytes) + { + action = "DELETE:"; + boost::filesystem::remove(entry.second.second); + } + else + { + action = " KEEP:"; + } - file.seekg(0, std::ios::beg); - file.read(*mem_buffer, filesize); - file.close(); + if (mCacheDebugInfo) + { + // have to do this because of LL_INFO/LL_END weirdness + std::ostringstream line; - if (file.bad()) - { - std::ostringstream error; - error << "Unable to read " << filesize; - error << " bytes from " << filepath; - printError(__FUNCDNAME__ , error.str(), false); - - return false; + line << action << " "; + line << entry.first << " "; + line << entry.second.first << " "; + line << entry.second.second; + line << " (" << file_size_total << "/" << mMaxSizeBytes << ")"; + LL_INFOS() << line.str() << LL_ENDL; + } } - // here we update the count of times the file is accessed so - // we can keep track of how many times it's been requested. - // This will be useful for metrics and perhaps determining - // if a file should not be purged even though its age - // might suggest that it should. - // In addition, this is where the time of last access is updated - // in the database and that us used to determine what is purged - // in an LRU fashion when the purge function is called. - const std::string stmt_update = sqlComposeGetUpdate(id); - if (! sqliteExec(stmt_update, __FUNCDNAME__ )) + if (mCacheDebugInfo) { - return false; + auto end_time = std::chrono::high_resolution_clock::now(); + auto execute_time = std::chrono::duration_cast(end_time - start_time).count(); + LL_INFOS() << "Total dir size after purge is " << dirFileSize(mCacheDir) << LL_ENDL; + LL_INFOS() << "Cache purge took " << execute_time << " ms to execute for " << file_info.size() << " files" << LL_ENDL; } - - return true; -} - -/////////////////////////////////////////////////////////////////////////////// -// Purges the database of older entries using an LRU approach. -// Pass in the number of entries to retain. -// This is called now after open to "clean up" the cache when the -// application starts. -// TODO: IMPORTANT: compose a list of files that will be deleted -// and delete them from disk too - not just from the DB -bool llDiskCache::purge(int num_entries) -{ - if (num_entries < 0) - { - printError(__FUNCDNAME__ , "number of entries to purge is invalid", false); - return false; - } - - // find the rows affected and get the filenames for them -//swww - - // delete oldest entries leaving the correct number in place - const std::string stmt = sqlComposePurge(num_entries); - if (! sqliteExec(stmt, __FUNCDNAME__ )) - { - return false; - } - - return true; -} - -/////////////////////////////////////////////////////////////////////////////// -// Call at application shutdown -void llDiskCache::close() -{ - sqlite3_close(mDb); -} - -/////////////////////////////////////////////////////////////////////////////// -// Determine the version of SQLite in use -// TODO: make this a static so we can get to it from the Viewer About -// box without instantiating the whole thing. -const std::string llDiskCache::dbVersion() -{ - std::ostringstream version; - - version << sqlite3_libversion(); - - return version.str(); } -/////////////////////////////////////////////////////////////////////////////// -// Given an id, return the matching filename -const std::string llDiskCache::getFilenameById(const std::string id) -{ - // TODO: - return std::string(); -} - -/////////////////////////////////////////////////////////////////////////////// -// Given an id, return the number of times that entry has been -// accessed from the cache -const int llDiskCache::getAccessCountById(const std::string id) -{ - // TODO: - return -1; -} - -/////////////////////////////////////////////////////////////////////////////// -// Return the number of entries currently in the cache as well as -// the maximum possible entries. -void llDiskCache::getNumEntries(int& num_entries, int& max_entries) -{ - num_entries = -1; - max_entries = -1; -} - -/////////////////////////////////////////////////////////////////////////////// -// Wraps the sqlite3_exec(..) used in many places -bool llDiskCache::sqliteExec(const std::string stmt, - const std::string funcname) -{ - if (sqlite3_exec( - mDb, - stmt.c_str(), - nullptr, // Callback function (unused) - nullptr, // 1st argument to callback (unused) - nullptr // Error msg written here (unused) - ) != SQLITE_OK) - { - printError(funcname, "sqlite3_exec", true); - return false; - } - - return true; -} - -/////////////////////////////////////////////////////////////////////////////// -// Wraps the sqlite3_prepare_v2 and sqlite3_step calls used in many places -sqlite3_stmt* llDiskCache::sqlitePrepareStep(const std::string funcname, - const std::string stmt, - int sqlite_success_condition) -{ - sqlite3_stmt* prepared_statement; - - if (sqlite3_prepare_v2( - mDb, - stmt.c_str(), - -1, // Maximum length of zSql in bytes. - &prepared_statement, - 0 // OUT: Pointer to unused portion of zSql - ) != SQLITE_OK) - { - printError(funcname, "sqlite3_prepare_v2", true); - return nullptr; - } - - if (sqlite3_step(prepared_statement) != sqlite_success_condition) - { - printError(funcname, "sqlite3_step", true); - sqlite3_finalize(prepared_statement); - return nullptr; - } - return prepared_statement; -} - -/////////////////////////////////////////////////////////////////////////////// -// When an "error" occurss - e.g. database cannot be found, file cannot be -// written, invalid argument passed into a function etc. a message is -// written to stderr that should end up in the viewer log -// TODO: Set the verbosity using the usual Viewer mechanism -void llDiskCache::printError(const std::string funcname, - const std::string desc, - bool is_sqlite_err) -{ - std::ostringstream err_msg; - - err_msg << "llDiskCache error in "; - err_msg << __FUNCDNAME__ << "(...) "; - err_msg << desc; - - if (is_sqlite_err) - { - err_msg << " - "; - err_msg << std::string(sqlite3_errmsg(mDb)); - } - - // TODO: set via viewer verbosity level - const int verbosity = 1; - if (verbosity > 0) - { - std::cerr << err_msg.str() << std::endl; - } -} - -/////////////////////////////////////////////////////////////////////////////// -// wrapper for SQLite code to begin an SQL transaction - not used yet but -// it will be eventually -bool llDiskCache::beginTransaction() +/** + * Update the "last write time" of a file to "now". This must be called whenever a + * file in the cache is read (not written) so that the last time the file was + * accessed which is used in the mechanism for purging the cache, is up to date. + */ +void LLDiskCache::updateFileAccessTime(const std::string file_path) { - const std::string stmt("BEGIN TRANSACTION"); - if (! sqliteExec(stmt, __FUNCDNAME__ )) - { - return false; - } - - return true; + const std::time_t file_time = std::time(nullptr); + boost::filesystem::last_write_time(file_path, file_time); } -/////////////////////////////////////////////////////////////////////////////// -// wrapper for SQLite code to end an SQL transaction - not used yet but -// it will be eventually -bool llDiskCache::endTransaction() +/** + * Clear the cache by removing all the files in the cache directory + * individually. It's important to maintain control of which directory + * if passed in and not let the user inadvertently (or maliciously) set + * it to an random location like your project source or OS system directory + */ +void LLDiskCache::clearCache(const std::string cache_dir) { - const std::string stmt("COMMIT"); - if (! sqliteExec(stmt, __FUNCDNAME__ )) + if (boost::filesystem::is_directory(cache_dir)) { - return false; + for (auto& entry : boost::make_iterator_range(boost::filesystem::directory_iterator(cache_dir), {})) + { + if (boost::filesystem::is_regular_file(entry)) + { + boost::filesystem::remove(entry); + } + } } - - return true; } -/////////////////////////////////////////////////////////////////////////////// -// Build a unique filename that will be used to store the actual file -// on disk (as opposed to the meta data in the database) -// TODO: I think this needs more work once we move it to the viewer -// and espcially to make it cross platform - see 'std::tmpnam' -// depreciation comments in compiler output for example -std::string llDiskCache::makeUniqueFilename() +/** + * Utility function to get the total filesize of all files in a directory. It + * used to test file extensions to only check cache files but that was removed. + * There may be a better way that works directly on the folder (similar to + * right clicking on a folder in the OS and asking for size vs right clicking + * on all files and adding up manually) but this is very fast - less than 100ms + * in my testing so, so long as it's not called frequently, it should be okay. + * Note that's it's only currently used for logging/debugging so if performance + * is ever an issue, optimizing this or removing it altogether, is an easy win. + */ +uintmax_t LLDiskCache::dirFileSize(const std::string dir) { - // TODO: replace with boost - this is marked as deprecated? - std::string base = std::tmpnam(nullptr); - - // C++11 random number generation!!! - static std::random_device dev; - static std::mt19937 rng(dev()); - std::uniform_int_distribution dist(100000, 999999); + uintmax_t total_file_size = 0; - // currently the tmp filename from std::tmpnam() on macOS - // is of the form `/tmp/foo/bar.12345 and the following code - // strips all the preceding dirs - we likely want a more - // robust (and cross platform solution) when we move to the - // viewer code - std::size_t found = base.rfind(systemSeparator()); - if (found != std::string::npos && found < base.size() - 2) - { - base = base.substr(found + 1); - } - else + if (boost::filesystem::is_directory(dir)) { - base = ""; + for (auto& entry : boost::make_iterator_range(boost::filesystem::directory_iterator(dir), {})) + { + if (boost::filesystem::is_regular_file(entry)) + { + total_file_size += boost::filesystem::file_size(entry); + } + } } - // we mix in a random number for some more entropy.. - // (i know, i know...) - std::ostringstream unique_filename; - unique_filename << base; - unique_filename << "."; - unique_filename << dist(rng); - - return unique_filename.str(); -} - -/////////////////////////////////////////////////////////////////////////////// -// Return system file/path separator - likely replaced by the version -// in the viewer -const std::string llDiskCache::systemSeparator() -{ -// TODO: replace in viewer with relevant call -#ifdef _WIN32 - return "\\"; -#else - return "/"; -#endif -} - -/////////////////////////////////////////////////////////////////////////////// -// Given a filename, compose a full path based on the path name passed -// in when the database was opened and the separator in play. -const std::string llDiskCache::makeFullPath(const std::string filename) -{ - std::string pathname = mDataStorePath + systemSeparator() + filename; - - return pathname; -} - -/////////////////////////////////////////////////////////////////////////////// -// -const std::string llDiskCache::sqlComposeCreateTable() -{ - std::ostringstream stmt; - stmt << "CREATE TABLE IF NOT EXISTS "; - stmt << mTableName; - stmt << "("; - stmt << mIdFieldName; - stmt << " TEXT PRIMARY KEY, "; - stmt << mFilenameFieldName; - stmt << " TEXT, "; - stmt << mFilesizeFieldName; - stmt << " INTEGER DEFAULT 0, "; - stmt << mInsertionDateTimeFieldName; - stmt << " TEXT, "; - stmt << mLastAccessDateTimeFieldName; - stmt << " TEXT, "; - stmt << mAccessCountFieldName; - stmt << " INTEGER DEFAULT 0"; - stmt << ")"; - -#ifdef SHOW_STATEMENTS - std::cout << stmt.str() << std::endl; -#endif - - return stmt.str(); -} - -/////////////////////////////////////////////////////////////////////////////// -// -const std::string llDiskCache::sqlComposeExists(const std::string id) -{ - std::ostringstream stmt; - stmt << "SELECT COUNT(*) FROM "; - stmt << mTableName; - stmt << " WHERE "; - stmt << mIdFieldName; - stmt << "='"; - stmt << id; - stmt << "'"; - -#ifdef SHOW_STATEMENTS - std::cout << stmt.str() << std::endl; -#endif - - return stmt.str(); -} - -/////////////////////////////////////////////////////////////////////////////// -// SQL statement to write an entry to the DB -// Saves id, filename (generated), file size and create/last access date -const std::string llDiskCache::sqlComposePut(const std::string id, - const std::string filename, - int binary_data_size) -{ - std::ostringstream stmt; - stmt << "REPLACE INTO "; - stmt << mTableName; - stmt << "("; - stmt << mIdFieldName; - stmt << ","; - stmt << mFilenameFieldName; - stmt << ","; - stmt << mFilesizeFieldName; - stmt << ","; - stmt << mInsertionDateTimeFieldName; - stmt << ","; - stmt << mLastAccessDateTimeFieldName; - stmt << ") "; - stmt << "VALUES("; - stmt << "'"; - stmt << id; - stmt << "', "; - stmt << "'"; - stmt << filename; - stmt << "', "; - stmt << binary_data_size; - stmt << ", "; - stmt << "strftime('%Y-%m-%d %H:%M:%f', 'now')"; - stmt << ", "; - stmt << "strftime('%Y-%m-%d %H:%M:%f', 'now')"; - stmt << ")"; - -#ifdef SHOW_STATEMENTS - std::cout << stmt.str() << std::endl; -#endif - - return stmt.str(); -} - -/////////////////////////////////////////////////////////////////////////////// -// SQL statement used in get() to look up the filename and file size -const std::string llDiskCache::sqlComposeGetSelect(const std::string id) -{ - std::ostringstream stmt; - stmt << "SELECT "; - stmt << mFilenameFieldName; - stmt << ", "; - stmt << mFilesizeFieldName; - stmt << " FROM "; - stmt << mTableName; - stmt << " WHERE "; - stmt << mIdFieldName; - stmt << "='"; - stmt << id; - stmt << "'"; - -#ifdef SHOW_STATEMENTS - std::cout << stmt.str() << std::endl; -#endif - - return stmt.str(); -} - -/////////////////////////////////////////////////////////////////////////////// -// SQL statement to update the date/time of last access as well as the -// count of number of times the file has been accessed. -// Note: the more accurate representation of date/time is used to -// ensure ms accuracy vs the standard INTEGER days since epoch approach -const std::string llDiskCache::sqlComposeGetUpdate(const std::string id) -{ - std::ostringstream stmt; - stmt << "UPDATE "; - stmt << mTableName; - stmt << " SET "; - stmt << mAccessCountFieldName; - stmt << "="; - stmt << mAccessCountFieldName; - stmt << "+1"; - stmt << ", "; - stmt << mLastAccessDateTimeFieldName; - stmt << "="; - stmt << "strftime('%Y-%m-%d %H:%M:%f', 'now')"; - stmt << " WHERE "; - stmt << mIdFieldName; - stmt << "='"; - stmt << id; - stmt << "'"; - -#ifdef SHOW_STATEMENTS - std::cout << stmt.str() << std::endl; -#endif - - return stmt.str(); -} - -/////////////////////////////////////////////////////////////////////////////// -// SQL statement to remove items from the database that are older -// than the newest num_elements entries -const std::string llDiskCache::sqlComposePurge(int num_entries) -{ - std::ostringstream stmt; - stmt << "DELETE FROM "; - stmt << mTableName; - stmt << " WHERE "; - stmt << mLastAccessDateTimeFieldName; - stmt << " NOT IN "; - stmt << "("; - stmt << "SELECT "; - stmt << mLastAccessDateTimeFieldName; - stmt << " FROM "; - stmt << mTableName; - stmt << " ORDER BY "; - stmt << mLastAccessDateTimeFieldName; - stmt << " DESC"; - stmt << " LIMIT "; - stmt << num_entries; - stmt << ")"; - -//#ifdef SHOW_STATEMENTS - std::cout << stmt.str() << std::endl; -//#endif - - return stmt.str(); + return total_file_size; } diff --git a/indra/llfilesystem/lldiskcache.h b/indra/llfilesystem/lldiskcache.h index 39b8f7ef48..1618cec6a6 100644 --- a/indra/llfilesystem/lldiskcache.h +++ b/indra/llfilesystem/lldiskcache.h @@ -1,11 +1,6 @@ /** * @file lldiskcache.h - * @brief Declaration SQLite meta data / file storage API - * @brief Declaration of the generic disk cache interface - as well the SQLite header/implementation. - * @Note Eventually, this component might split into an interface - * file and multiple implemtations but for now, this is the - * only one so I think it's okay to combine everything. + * @brief The disk cache implementation. * * $LicenseInfo:firstyear=2009&license=viewerlgpl$ * Second Life Viewer Source Code @@ -32,99 +27,68 @@ #ifndef _LLDISKCACHE #define _LLDISKCACHE -#include -#include +class LLDiskCache +{ + public: + LLDiskCache(const std::string cache_dir); + ~LLDiskCache(); -#include "sqlite3.h" + /** + * Update the "last write time" of a file to "now". This must be called whenever a + * file in the cache is read (not written) so that the last time the file was + * accessed which is used in the mechanism for purging the cache, is up to date. + */ + void updateFileAccessTime(const std::string file_path); -// Enable this line to display each SQL statement when it is executed -// It can lead to a lot of spam but useful for debugging -//#define SHOW_STATEMENTS + /** + * Purge the oldest items in the cache so that the combined size of all files + * is no bigger than mMaxSizeBytes. + */ + void purge(); -// I toyed with the idea of a variety of approaches and thought have -// an abstract base class with which to hang different implementations -// off was a good idea but I think we settled on a single approach -// so I will likely remove this level of indirection if not other -// interesting implementation ideas present themselves. -class llDiskCacheBase -{ - public: - llDiskCacheBase() {}; - virtual ~llDiskCacheBase() {}; - virtual bool open(const std::string db_folder, - const std::string db_filename) = 0; - virtual bool exists(const std::string id, bool& exists) = 0; - virtual bool put(const std::string id, - char* binary_data, - int binary_data_size) = 0; - virtual const bool get(const std::string id, - char** mem_buffer, - int& mem_buffer_size) = 0; - virtual bool purge(int num_entries) = 0; - virtual void close() = 0; - virtual const std::string dbVersion() = 0; - virtual const std::string getFilenameById(const std::string id) = 0; - virtual const int getAccessCountById(const std::string id) = 0; - virtual void getNumEntries(int& num_entries, int& max_entries) = 0; -}; + /** + * Clear the cache by removing the files in the cache directory individually + */ + void clearCache(const std::string cache_dir); -// implementation for the SQLite/disk blended case -// see source file for detailed comments -class llDiskCache : - public llDiskCacheBase -{ - public: - llDiskCache(); - virtual ~llDiskCache(); - virtual bool open(const std::string db_folder, - const std::string db_filename) override; - virtual bool exists(const std::string id, bool& exists) override; - virtual bool put(const std::string id, - char* binary_data, - int binary_data_size) override; - virtual const bool get(const std::string id, - char** mem_buffer, - int& mem_buffer_size) override; - virtual bool purge(int num_entries) override; - virtual void close() override; - virtual const std::string dbVersion() override; - virtual const std::string getFilenameById(const std::string id) override; - virtual const int getAccessCountById(const std::string id) override; - virtual void getNumEntries(int& num_entries, int& max_entries) override; + /** + * Manage max size in bytes of cache - use a discrete setter/getter so the value can + * be changed in the preferences and cache cleared/purged without restarting viewer + * to instantiate this class again. + */ + void setMaxSizeBytes(const uintmax_t size_bytes) { mMaxSizeBytes = size_bytes; } + uintmax_t getMaxSizeBytes() const { return mMaxSizeBytes; } private: - sqlite3* mDb; - std::string mDataStorePath; - const std::string mTableName = "lldiskcache"; - const std::string mIdFieldName = "id"; - const std::string mFilenameFieldName = "filename"; - const std::string mFilesizeFieldName = "filesize"; - const std::string mInsertionDateTimeFieldName = "insert_datetime"; - const std::string mLastAccessDateTimeFieldName = "last_access_datetime"; - const std::string mAccessCountFieldName = "access_count"; + /** + * Utility function to gather the total size the files in a given + * directory. Primarily used here to determine the directory size + * before and after the cache purge + */ + uintmax_t dirFileSize(const std::string dir); private: - void printError(const std::string funcname, - const std::string desc, - bool is_sqlite_err); - bool beginTransaction(); - bool endTransaction(); - std::string makeUniqueFilename(); - const std::string systemSeparator(); - const std::string makeFullPath(const std::string filename); - bool sqliteExec(const std::string stmt, - const std::string funcname); - sqlite3_stmt* sqlitePrepareStep(const std::string funcname, - const std::string stmt, - int sqlite_success_condition); - const std::string sqlComposeCreateTable(); - const std::string sqlComposeExists(const std::string id); - const std::string sqlComposePut(const std::string id, - const std::string filename, - int binary_data_size); - const std::string sqlComposeGetSelect(const std::string id); - const std::string sqlComposeGetUpdate(const std::string id); - const std::string sqlComposePurge(int num_entries); + /** + * Default of 20MB seems reasonable - it will likely be reset + * immediately anyway using a value from the Viewer settings + */ + const uintmax_t mDefaultSizeBytes = 20 * 1024 * 1024; + uintmax_t mMaxSizeBytes; + + /** + * The folder that holds the cached files. The consumer of this + * class must avoid letting the user set this location as a malicious + * setting could potentially point it at a non-cache directory (for example, + * the Windows System dir) with disastrous results. + */ + std::string mCacheDir; + + /** + * This is set from the CacheDebugInfo global setting and + * when enabled, displays additional debugging information in + * various parts of the code + */ + bool mCacheDebugInfo; }; #endif // _LLDISKCACHE diff --git a/indra/llfilesystem/llfilesystem.cpp b/indra/llfilesystem/llfilesystem.cpp index f0037c9a22..ffc3dee12b 100644 --- a/indra/llfilesystem/llfilesystem.cpp +++ b/indra/llfilesystem/llfilesystem.cpp @@ -42,6 +42,8 @@ const S32 LLFileSystem::READ_WRITE = 0x00000003; // LLFileSystem::READ & LLFile const S32 LLFileSystem::APPEND = 0x00000006; // 0x00000004 & LLFileSystem::WRITE static LLTrace::BlockTimerStatHandle FTM_VFILE_WAIT("VFile Wait"); +LLDiskCache* LLFileSystem::mDiskCache = 0; +std::string LLFileSystem::mCacheDirName = "cache"; LLFileSystem::LLFileSystem(const LLUUID &file_id, const LLAssetType::EType file_type, S32 mode) { @@ -104,7 +106,7 @@ const std::string assetTypeToString(LLAssetType::EType at) return std::string("UNKNOWN"); } -const std::string idToFilepath(const std::string id, LLAssetType::EType at) +const std::string LLFileSystem::idToFilepath(const std::string id, LLAssetType::EType at) { /** * For the moment this is just {UUID}_{ASSET_TYPE}.txt but of @@ -117,7 +119,7 @@ const std::string idToFilepath(const std::string id, LLAssetType::EType at) ss << assetTypeToString(at); ss << ".txt"; - const std::string filepath = gDirUtilp->getExpandedFilename(LL_PATH_CACHE, ss.str()); + const std::string filepath = gDirUtilp->getExpandedFilename(LL_PATH_CACHE, mCacheDirName, ss.str()); return filepath; } @@ -289,7 +291,6 @@ BOOL LLFileSystem::write(const U8 *buffer, S32 bytes) BOOL LLFileSystem::writeFile(const U8 *buffer, S32 bytes, const LLUUID &uuid, LLAssetType::EType type) { LLFileSystem file(uuid, type, LLFileSystem::WRITE); - file.setMaxSize(bytes); return file.write(buffer, bytes); } @@ -339,13 +340,6 @@ S32 LLFileSystem::getMaxSize() return INT_MAX; } -BOOL LLFileSystem::setMaxSize(S32 size) -{ - // we don't care what the max size is so we do nothing - // and return true to indicate all was okay - return TRUE; -} - BOOL LLFileSystem::rename(const LLUUID &new_id, const LLAssetType::EType new_type) { LLFileSystem::renameFile(mFileID, mFileType, new_id, new_type); @@ -366,21 +360,15 @@ BOOL LLFileSystem::remove() // static void LLFileSystem::initClass() { -} + const std::string cache_dir = gDirUtilp->getExpandedFilename(LL_PATH_CACHE, mCacheDirName); -// static -void LLFileSystem::cleanupClass() -{ -} + LLFileSystem::mDiskCache = new LLDiskCache(cache_dir); -bool LLFileSystem::isLocked() -{ - // I don't think we care about this test since there is no locking - // TODO: remove this function and calling sites? - return FALSE; + mDiskCache->purge(); } -void LLFileSystem::waitForLock() +// static +void LLFileSystem::cleanupClass() { - // TODO: remove this function and calling sites? + delete LLFileSystem::mDiskCache; } diff --git a/indra/llfilesystem/llfilesystem.h b/indra/llfilesystem/llfilesystem.h index 9cddef74a7..789f9456c7 100644 --- a/indra/llfilesystem/llfilesystem.h +++ b/indra/llfilesystem/llfilesystem.h @@ -32,6 +32,7 @@ #include "lluuid.h" #include "llassettype.h" +#include "lldiskcache.h" class LLFileSystem { @@ -51,13 +52,9 @@ public: S32 getSize(); S32 getMaxSize(); - BOOL setMaxSize(S32 size); BOOL rename(const LLUUID &new_id, const LLAssetType::EType new_type); BOOL remove(); - bool isLocked(); - void waitForLock(); - static bool getExists(const LLUUID &file_id, const LLAssetType::EType file_type); static bool removeFile(const LLUUID &file_id, const LLAssetType::EType file_type); static bool renameFile(const LLUUID &old_file_id, const LLAssetType::EType old_file_type, @@ -80,6 +77,11 @@ protected: S32 mPosition; S32 mMode; S32 mBytesRead; + +private: + static const std::string idToFilepath(const std::string id, LLAssetType::EType at); + static std::string mCacheDirName; + static LLDiskCache* mDiskCache; }; #endif // LL_FILESYSTEM_H -- cgit v1.2.3 From 56e30615530bf5d1c86fbafee89c9998a079e88f Mon Sep 17 00:00:00 2001 From: Callum Prentice Date: Tue, 6 Oct 2020 17:20:08 -0700 Subject: Remove SQLite from project and we are now not going to use it for the cache --- indra/llfilesystem/CMakeLists.txt | 3 --- 1 file changed, 3 deletions(-) (limited to 'indra/llfilesystem') diff --git a/indra/llfilesystem/CMakeLists.txt b/indra/llfilesystem/CMakeLists.txt index d1dece5bba..09c4c33ebf 100644 --- a/indra/llfilesystem/CMakeLists.txt +++ b/indra/llfilesystem/CMakeLists.txt @@ -4,13 +4,11 @@ project(llfilesystem) include(00-Common) include(LLCommon) -include(SQLite) include(UnixInstall) include_directories( ${LLCOMMON_INCLUDE_DIRS} ${LLCOMMON_SYSTEM_INCLUDE_DIRS} - ${SQLITE_INCLUDE_DIR} ) set(llfilesystem_SOURCE_FILES @@ -70,7 +68,6 @@ set(cache_BOOST_LIBRARIES target_link_libraries(llfilesystem ${LLCOMMON_LIBRARIES} ${cache_BOOST_LIBRARIES} - ${SQLITE_LIBRARIES} ) if (DARWIN) -- cgit v1.2.3 From a0ea119623b8bda445f86afdb0ea7b5833c8e171 Mon Sep 17 00:00:00 2001 From: Callum Prentice Date: Tue, 6 Oct 2020 18:18:18 -0700 Subject: Replace references to static writefile with write so we end up with only a single read and a single write function --- indra/llfilesystem/llfilesystem.cpp | 13 ++++++------- indra/llfilesystem/llfilesystem.h | 2 +- 2 files changed, 7 insertions(+), 8 deletions(-) (limited to 'indra/llfilesystem') diff --git a/indra/llfilesystem/llfilesystem.cpp b/indra/llfilesystem/llfilesystem.cpp index ffc3dee12b..6d6ff3d7e1 100644 --- a/indra/llfilesystem/llfilesystem.cpp +++ b/indra/llfilesystem/llfilesystem.cpp @@ -230,6 +230,12 @@ BOOL LLFileSystem::read(U8 *buffer, S32 bytes, BOOL async, F32 priority) mReadComplete = TRUE; } + // update the last access time for the file - this is required + // even though we are reading and not writing because this is the + // way the cache works - it relies on a valid "last accessed time" for + // each file so it knows how to remove the oldest, unused files + LLFileSystem::mDiskCache->updateFileAccessTime(filename); + return success; } @@ -287,13 +293,6 @@ BOOL LLFileSystem::write(const U8 *buffer, S32 bytes) return success; } -//static -BOOL LLFileSystem::writeFile(const U8 *buffer, S32 bytes, const LLUUID &uuid, LLAssetType::EType type) -{ - LLFileSystem file(uuid, type, LLFileSystem::WRITE); - return file.write(buffer, bytes); -} - BOOL LLFileSystem::seek(S32 offset, S32 origin) { if (-1 == origin) diff --git a/indra/llfilesystem/llfilesystem.h b/indra/llfilesystem/llfilesystem.h index 789f9456c7..5d87de9bf8 100644 --- a/indra/llfilesystem/llfilesystem.h +++ b/indra/llfilesystem/llfilesystem.h @@ -1,3 +1,4 @@ +/** /** * @file filesystem.h * @brief Simulate local file system operations. @@ -46,7 +47,6 @@ public: BOOL eof(); BOOL write(const U8 *buffer, S32 bytes); - static BOOL writeFile(const U8 *buffer, S32 bytes, const LLUUID &uuid, LLAssetType::EType type); BOOL seek(S32 offset, S32 origin = -1); S32 tell() const; -- cgit v1.2.3 From 08dfc0836fb12855d0c07d811e2909400d5b74f3 Mon Sep 17 00:00:00 2001 From: Callum Prentice Date: Wed, 7 Oct 2020 15:25:12 -0700 Subject: This changeset hooks up many things that have been in progress and moves things about between llfilesystem and lldiskcache - there is still some bookkeeping work left but this is the first version that appears to work and actively manage the cache --- indra/llfilesystem/lldiskcache.cpp | 134 +++++++++++++++----- indra/llfilesystem/lldiskcache.h | 82 ++++++++++--- indra/llfilesystem/llfilesystem.cpp | 238 +++++++++++------------------------- indra/llfilesystem/llfilesystem.h | 82 ++++++------- 4 files changed, 273 insertions(+), 263 deletions(-) (limited to 'indra/llfilesystem') diff --git a/indra/llfilesystem/lldiskcache.cpp b/indra/llfilesystem/lldiskcache.cpp index 4b2ba0dd79..4b82cf3cce 100644 --- a/indra/llfilesystem/lldiskcache.cpp +++ b/indra/llfilesystem/lldiskcache.cpp @@ -25,41 +25,31 @@ */ #include "linden_common.h" -#include "lluuid.h" +#include "llassettype.h" #include "lldir.h" - -#include "lldiskcache.h" - #include #include - #include -#include -#include -LLDiskCache::LLDiskCache(const std::string cache_dir) : +#include "lldiskcache.h" + +LLDiskCache::LLDiskCache(const std::string cache_dir, + const int max_size_bytes, + const bool enable_cache_debug_info) : mCacheDir(cache_dir), - mMaxSizeBytes(mDefaultSizeBytes) + mMaxSizeBytes(max_size_bytes), + mEnableCacheDebugInfo(enable_cache_debug_info) { - // no access to LLControlGroup / gSavedSettings so use the system environment - // to trigger additional debugging on top of the default "we purged the cache" - mCacheDebugInfo = true; - if (getenv("LL_CACHE_DEBUGGING") != nullptr) - { - mCacheDebugInfo = true; - } + // the prefix used for cache filenames to disambiguate them from other files + mCacheFilenamePrefix = "sl_cache"; // create cache dir if it does not exist boost::filesystem::create_directory(cache_dir); } -LLDiskCache::~LLDiskCache() -{ -} - void LLDiskCache::purge() { - if (mCacheDebugInfo) + if (mEnableCacheDebugInfo) { LL_INFOS() << "Total dir size before purge is " << dirFileSize(mCacheDir) << LL_ENDL; } @@ -75,11 +65,14 @@ void LLDiskCache::purge() { if (boost::filesystem::is_regular_file(entry)) { - uintmax_t file_size = boost::filesystem::file_size(entry); - const std::string file_path = entry.path().string(); - const std::time_t file_time = boost::filesystem::last_write_time(entry); + if (entry.path().string().find(mCacheFilenamePrefix) != std::string::npos) + { + uintmax_t file_size = boost::filesystem::file_size(entry); + const std::string file_path = entry.path().string(); + const std::time_t file_time = boost::filesystem::last_write_time(entry); - file_info.push_back(file_info_t(file_time, { file_size, file_path })); + file_info.push_back(file_info_t(file_time, { file_size, file_path })); + } } } } @@ -107,7 +100,7 @@ void LLDiskCache::purge() action = " KEEP:"; } - if (mCacheDebugInfo) + if (mEnableCacheDebugInfo) { // have to do this because of LL_INFO/LL_END weirdness std::ostringstream line; @@ -121,7 +114,7 @@ void LLDiskCache::purge() } } - if (mCacheDebugInfo) + if (mEnableCacheDebugInfo) { auto end_time = std::chrono::high_resolution_clock::now(); auto execute_time = std::chrono::duration_cast(end_time - start_time).count(); @@ -130,9 +123,78 @@ void LLDiskCache::purge() } } +const std::string LLDiskCache::assetTypeToString(LLAssetType::EType at) +{ + /** + * Make use of the C++17 (or is it 14) feature that allows + * for inline initialization of an std::map<> + */ + typedef std::map asset_type_to_name_t; + asset_type_to_name_t asset_type_to_name = + { + { LLAssetType::AT_TEXTURE, "TEXTURE" }, + { LLAssetType::AT_SOUND, "SOUND" }, + { LLAssetType::AT_CALLINGCARD, "CALLINGCARD" }, + { LLAssetType::AT_LANDMARK, "LANDMARK" }, + { LLAssetType::AT_SCRIPT, "SCRIPT" }, + { LLAssetType::AT_CLOTHING, "CLOTHING" }, + { LLAssetType::AT_OBJECT, "OBJECT" }, + { LLAssetType::AT_NOTECARD, "NOTECARD" }, + { LLAssetType::AT_CATEGORY, "CATEGORY" }, + { LLAssetType::AT_LSL_TEXT, "LSL_TEXT" }, + { LLAssetType::AT_LSL_BYTECODE, "LSL_BYTECODE" }, + { LLAssetType::AT_TEXTURE_TGA, "TEXTURE_TGA" }, + { LLAssetType::AT_BODYPART, "BODYPART" }, + { LLAssetType::AT_SOUND_WAV, "SOUND_WAV" }, + { LLAssetType::AT_IMAGE_TGA, "IMAGE_TGA" }, + { LLAssetType::AT_IMAGE_JPEG, "IMAGE_JPEG" }, + { LLAssetType::AT_ANIMATION, "ANIMATION" }, + { LLAssetType::AT_GESTURE, "GESTURE" }, + { LLAssetType::AT_SIMSTATE, "SIMSTATE" }, + { LLAssetType::AT_LINK, "LINK" }, + { LLAssetType::AT_LINK_FOLDER, "LINK_FOLDER" }, + { LLAssetType::AT_MARKETPLACE_FOLDER, "MARKETPLACE_FOLDER" }, + { LLAssetType::AT_WIDGET, "WIDGET" }, + { LLAssetType::AT_PERSON, "PERSON" }, + { LLAssetType::AT_MESH, "MESH" }, + { LLAssetType::AT_SETTINGS, "SETTINGS" }, + { LLAssetType::AT_UNKNOWN, "UNKNOWN" } + }; + + asset_type_to_name_t::iterator iter = asset_type_to_name.find(at); + if (iter != asset_type_to_name.end()) + { + return iter->second; + } + + return std::string("UNKNOWN"); +} + +const std::string LLDiskCache::metaDataToFilepath(const std::string id, + LLAssetType::EType at, + const std::string extra_info) +{ + std::ostringstream file_path; + + file_path << mCacheDir; + file_path << gDirUtilp->getDirDelimiter(); + file_path << mCacheFilenamePrefix; + file_path << "_"; + file_path << id; + file_path << "_"; + file_path << (extra_info.empty() ? "0" : extra_info); + file_path << "_"; + file_path << assetTypeToString(at); + file_path << ".asset"; + + LL_INFOS() << "filepath.str() = " << file_path.str() << LL_ENDL; + + return file_path.str(); +} + /** - * Update the "last write time" of a file to "now". This must be called whenever a - * file in the cache is read (not written) so that the last time the file was + * Update the "last write time" of a file to "now". This must be called whenever a + * file in the cache is read (not written) so that the last time the file was * accessed which is used in the mechanism for purging the cache, is up to date. */ void LLDiskCache::updateFileAccessTime(const std::string file_path) @@ -144,8 +206,8 @@ void LLDiskCache::updateFileAccessTime(const std::string file_path) /** * Clear the cache by removing all the files in the cache directory * individually. It's important to maintain control of which directory - * if passed in and not let the user inadvertently (or maliciously) set - * it to an random location like your project source or OS system directory + * if passed in and not let the user inadvertently (or maliciously) set + * it to an random location like your project source or OS system directory */ void LLDiskCache::clearCache(const std::string cache_dir) { @@ -155,7 +217,10 @@ void LLDiskCache::clearCache(const std::string cache_dir) { if (boost::filesystem::is_regular_file(entry)) { - boost::filesystem::remove(entry); + if (entry.path().string().find(mCacheFilenamePrefix) != std::string::npos) + { + boost::filesystem::remove(entry); + } } } } @@ -181,7 +246,10 @@ uintmax_t LLDiskCache::dirFileSize(const std::string dir) { if (boost::filesystem::is_regular_file(entry)) { - total_file_size += boost::filesystem::file_size(entry); + if (entry.path().string().find(mCacheFilenamePrefix) != std::string::npos) + { + total_file_size += boost::filesystem::file_size(entry); + } } } } diff --git a/indra/llfilesystem/lldiskcache.h b/indra/llfilesystem/lldiskcache.h index 1618cec6a6..12599a132e 100644 --- a/indra/llfilesystem/lldiskcache.h +++ b/indra/llfilesystem/lldiskcache.h @@ -27,15 +27,50 @@ #ifndef _LLDISKCACHE #define _LLDISKCACHE -class LLDiskCache +#include "llsingleton.h" + +class LLDiskCache : + public LLParamSingleton { public: - LLDiskCache(const std::string cache_dir); - ~LLDiskCache(); + LLSINGLETON(LLDiskCache, + const std::string cache_dir, + const int max_size_bytes, + const bool enable_cache_debug_info); + virtual ~LLDiskCache() = default; + + public: + ///** + // * The location of the cache dir is set in llappviewer at startup via the + // * saved settings parameters. We do not have access to those saved settings + // * here in LLCommon so we must provide an accessor for other classes to use + // * when they need it - e.g. LLFilesystem needs the path so it can write files + // * to it. + // */ + //const std::string getCacheDirName() { return mCacheDir; } + + ///** + // * Each cache filename has a prefix inserted (see definition of the + // * mCacheFilenamePrefix variable below for why) but the LLFileSystem + // * component needs access to it to in order to create the file so we + // * expose it by a getter here. + // */ + //const std::string getCacheFilenamePrefix() { return mCacheFilenamePrefix; } /** - * Update the "last write time" of a file to "now". This must be called whenever a - * file in the cache is read (not written) so that the last time the file was + * Construct a filename and path to it based on the file meta data + * (id, asset type, additional 'extra' info like discard level perhaps) + * Worth pointing out that this function used to be in LLFileSystem but + * so many things had to be pushed back there to accomodate it, that I + * decided to move it here. Still not sure that's completely right. + */ + const std::string metaDataToFilepath(const std::string id, + LLAssetType::EType at, + const std::string extra_info); + + /** + * Update the "last write time" of a file to "now". This must be called whenever a + * file in the cache is read (not written) so that the last time the file was * accessed which is used in the mechanism for purging the cache, is up to date. */ void updateFileAccessTime(const std::string file_path); @@ -51,28 +86,26 @@ class LLDiskCache */ void clearCache(const std::string cache_dir); - /** - * Manage max size in bytes of cache - use a discrete setter/getter so the value can - * be changed in the preferences and cache cleared/purged without restarting viewer - * to instantiate this class again. - */ - void setMaxSizeBytes(const uintmax_t size_bytes) { mMaxSizeBytes = size_bytes; } - uintmax_t getMaxSizeBytes() const { return mMaxSizeBytes; } - private: /** * Utility function to gather the total size the files in a given - * directory. Primarily used here to determine the directory size + * directory. Primarily used here to determine the directory size * before and after the cache purge */ uintmax_t dirFileSize(const std::string dir); + /** + * Utility function to convert an LLAssetType enum into a + * string that we use as part of the cache file filename + */ + const std::string assetTypeToString(LLAssetType::EType at); + private: /** - * Default of 20MB seems reasonable - it will likely be reset - * immediately anyway using a value from the Viewer settings + * The maximum size of the cache in bytes. After purge is called, the + * total size of the cache files in the cache directory will be + * less than this value */ - const uintmax_t mDefaultSizeBytes = 20 * 1024 * 1024; uintmax_t mMaxSizeBytes; /** @@ -84,11 +117,20 @@ class LLDiskCache std::string mCacheDir; /** - * This is set from the CacheDebugInfo global setting and - * when enabled, displays additional debugging information in + * The prefix inserted at the start of a cache file filename to + * help identify it as a cache file. It's probably not required + * (just the presence in the cache folder is enough) but I am + * paranoid about the cache folder being set to something bad + * like the users' OS system dir by mistake or maliciously and + * this will help to offset any damage if that happens. + */ + std::string mCacheFilenamePrefix; + + /** + * When enabled, displays additional debugging information in * various parts of the code */ - bool mCacheDebugInfo; + bool mEnableCacheDebugInfo; }; #endif // _LLDISKCACHE diff --git a/indra/llfilesystem/llfilesystem.cpp b/indra/llfilesystem/llfilesystem.cpp index 6d6ff3d7e1..c6b20caa69 100644 --- a/indra/llfilesystem/llfilesystem.cpp +++ b/indra/llfilesystem/llfilesystem.cpp @@ -1,4 +1,4 @@ -/** +/** * @file filesystem.h * @brief Simulate local file system operations. * @Note The initial implementation does actually use standard C++ @@ -8,21 +8,21 @@ * $LicenseInfo:firstyear=2002&license=viewerlgpl$ * Second Life Viewer Source Code * Copyright (C) 2010, Linden Research, Inc. - * + * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; * version 2.1 of the License only. - * + * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. - * + * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - * + * * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA * $/LicenseInfo$ */ @@ -36,139 +36,74 @@ #include -const S32 LLFileSystem::READ = 0x00000001; -const S32 LLFileSystem::WRITE = 0x00000002; -const S32 LLFileSystem::READ_WRITE = 0x00000003; // LLFileSystem::READ & LLFileSystem::WRITE -const S32 LLFileSystem::APPEND = 0x00000006; // 0x00000004 & LLFileSystem::WRITE +const S32 LLFileSystem::READ = 0x00000001; +const S32 LLFileSystem::WRITE = 0x00000002; +const S32 LLFileSystem::READ_WRITE = 0x00000003; // LLFileSystem::READ & LLFileSystem::WRITE +const S32 LLFileSystem::APPEND = 0x00000006; // 0x00000004 & LLFileSystem::WRITE static LLTrace::BlockTimerStatHandle FTM_VFILE_WAIT("VFile Wait"); -LLDiskCache* LLFileSystem::mDiskCache = 0; -std::string LLFileSystem::mCacheDirName = "cache"; -LLFileSystem::LLFileSystem(const LLUUID &file_id, const LLAssetType::EType file_type, S32 mode) +LLFileSystem::LLFileSystem(const LLUUID& file_id, const LLAssetType::EType file_type, S32 mode) { - mFileType = file_type; - mFileID = file_id; - mPosition = 0; + mFileType = file_type; + mFileID = file_id; + mPosition = 0; mBytesRead = 0; - mReadComplete = FALSE; - mMode = mode; + mMode = mode; } LLFileSystem::~LLFileSystem() { } -const std::string assetTypeToString(LLAssetType::EType at) +// static +bool LLFileSystem::getExists(const LLUUID& file_id, const LLAssetType::EType file_type) { - /** - * Make use of the C++17 (or is it 14) feature that allows - * for inline initialization of an std::map<> - */ - typedef std::map asset_type_to_name_t; - asset_type_to_name_t asset_type_to_name = - { - { LLAssetType::AT_TEXTURE, "TEXTURE" }, - { LLAssetType::AT_SOUND, "SOUND" }, - { LLAssetType::AT_CALLINGCARD, "CALLINGCARD" }, - { LLAssetType::AT_LANDMARK, "LANDMARK" }, - { LLAssetType::AT_SCRIPT, "SCRIPT" }, - { LLAssetType::AT_CLOTHING, "CLOTHING" }, - { LLAssetType::AT_OBJECT, "OBJECT" }, - { LLAssetType::AT_NOTECARD, "NOTECARD" }, - { LLAssetType::AT_CATEGORY, "CATEGORY" }, - { LLAssetType::AT_LSL_TEXT, "LSL_TEXT" }, - { LLAssetType::AT_LSL_BYTECODE, "LSL_BYTECODE" }, - { LLAssetType::AT_TEXTURE_TGA, "TEXTURE_TGA" }, - { LLAssetType::AT_BODYPART, "BODYPART" }, - { LLAssetType::AT_SOUND_WAV, "SOUND_WAV" }, - { LLAssetType::AT_IMAGE_TGA, "IMAGE_TGA" }, - { LLAssetType::AT_IMAGE_JPEG, "IMAGE_JPEG" }, - { LLAssetType::AT_ANIMATION, "ANIMATION" }, - { LLAssetType::AT_GESTURE, "GESTURE" }, - { LLAssetType::AT_SIMSTATE, "SIMSTATE" }, - { LLAssetType::AT_LINK, "LINK" }, - { LLAssetType::AT_LINK_FOLDER, "LINK_FOLDER" }, - { LLAssetType::AT_MARKETPLACE_FOLDER, "MARKETPLACE_FOLDER" }, - { LLAssetType::AT_WIDGET, "WIDGET" }, - { LLAssetType::AT_PERSON, "PERSON" }, - { LLAssetType::AT_MESH, "MESH" }, - { LLAssetType::AT_SETTINGS, "SETTINGS" }, - { LLAssetType::AT_UNKNOWN, "UNKNOWN" } - }; - - asset_type_to_name_t::iterator iter = asset_type_to_name.find(at); - if (iter != asset_type_to_name.end()) + std::string id_str; + file_id.toString(id_str); + const std::string extra_info = ""; + const std::string filename = LLDiskCache::getInstance()->metaDataToFilepath(id_str, file_type, extra_info); + + std::ifstream file(filename, std::ios::binary); + if (file.is_open()) { - return iter->second; + file.seekg(0, std::ios::end); + return file.tellg() > 0; } - - return std::string("UNKNOWN"); -} - -const std::string LLFileSystem::idToFilepath(const std::string id, LLAssetType::EType at) -{ - /** - * For the moment this is just {UUID}_{ASSET_TYPE}.txt but of - * course, will be greatly expanded upon - */ - std::ostringstream ss; - ss << "00cache_"; - ss << id; - ss << "_"; - ss << assetTypeToString(at); - ss << ".txt"; - - const std::string filepath = gDirUtilp->getExpandedFilename(LL_PATH_CACHE, mCacheDirName, ss.str()); - - return filepath; + return false; } // static -bool LLFileSystem::getExists(const LLUUID &file_id, const LLAssetType::EType file_type) -{ - std::string id_str; - file_id.toString(id_str); - const std::string filename = idToFilepath(id_str, file_type); - - std::ifstream file(filename, std::ios::binary); - if (file.is_open()) - { - file.seekg(0, std::ios::end); - return file.tellg() > 0; - } - return false; -} - -// static -bool LLFileSystem::removeFile(const LLUUID &file_id, const LLAssetType::EType file_type) +bool LLFileSystem::removeFile(const LLUUID& file_id, const LLAssetType::EType file_type) { std::string id_str; file_id.toString(id_str); - const std::string filename = idToFilepath(id_str, file_type); - + const std::string extra_info = ""; + const std::string filename = LLDiskCache::getInstance()->metaDataToFilepath(id_str, file_type, extra_info); + std::remove(filename.c_str()); return true; } // static -bool LLFileSystem::renameFile(const LLUUID &old_file_id, const LLAssetType::EType old_file_type, - const LLUUID &new_file_id, const LLAssetType::EType new_file_type) +bool LLFileSystem::renameFile(const LLUUID& old_file_id, const LLAssetType::EType old_file_type, + const LLUUID& new_file_id, const LLAssetType::EType new_file_type) { std::string old_id_str; old_file_id.toString(old_id_str); - const std::string old_filename = idToFilepath(old_id_str, old_file_type); + const std::string extra_info = ""; + const std::string old_filename = LLDiskCache::getInstance()->metaDataToFilepath(old_id_str, old_file_type, extra_info); std::string new_id_str; new_file_id.toString(new_id_str); - const std::string new_filename = idToFilepath(new_id_str, new_file_type); + const std::string new_filename = LLDiskCache::getInstance()->metaDataToFilepath(new_id_str, new_file_type, extra_info); if (std::rename(old_filename.c_str(), new_filename.c_str())) { // We would like to return FALSE here indicating the operation // failed but the original code does not and doing so seems to - // break a lot of things so we go with the flow... + // break a lot of things so we go with the flow... //return FALSE; } @@ -176,11 +111,12 @@ bool LLFileSystem::renameFile(const LLUUID &old_file_id, const LLAssetType::ETyp } // static -S32 LLFileSystem::getFileSize(const LLUUID &file_id, const LLAssetType::EType file_type) +S32 LLFileSystem::getFileSize(const LLUUID& file_id, const LLAssetType::EType file_type) { std::string id_str; file_id.toString(id_str); - const std::string filename = idToFilepath(id_str, file_type); + const std::string extra_info = ""; + const std::string filename = LLDiskCache::getInstance()->metaDataToFilepath(id_str, file_type, extra_info); S32 file_size = 0; std::ifstream file(filename, std::ios::binary); @@ -193,15 +129,14 @@ S32 LLFileSystem::getFileSize(const LLUUID &file_id, const LLAssetType::EType fi return file_size; } -BOOL LLFileSystem::read(U8 *buffer, S32 bytes, BOOL async, F32 priority) +BOOL LLFileSystem::read(U8* buffer, S32 bytes) { - BOOL success = TRUE; - - mReadComplete = FALSE; + BOOL success = TRUE; std::string id; mFileID.toString(id); - const std::string filename = idToFilepath(id, mFileType); + const std::string extra_info = ""; + const std::string filename = LLDiskCache::getInstance()->metaDataToFilepath(id, mFileType, extra_info); std::ifstream file(filename, std::ios::binary); if (file.is_open()) @@ -226,44 +161,33 @@ BOOL LLFileSystem::read(U8 *buffer, S32 bytes, BOOL async, F32 priority) { success = FALSE; } - - mReadComplete = TRUE; } - // update the last access time for the file - this is required + // update the last access time for the file - this is required // even though we are reading and not writing because this is the // way the cache works - it relies on a valid "last accessed time" for // each file so it knows how to remove the oldest, unused files - LLFileSystem::mDiskCache->updateFileAccessTime(filename); + LLDiskCache::getInstance()->updateFileAccessTime(filename); return success; } -BOOL LLFileSystem::isReadComplete() -{ - if (mReadComplete) - { - return TRUE; - } - - return FALSE; -} - S32 LLFileSystem::getLastBytesRead() { - return mBytesRead; + return mBytesRead; } BOOL LLFileSystem::eof() { - return mPosition >= getSize(); + return mPosition >= getSize(); } -BOOL LLFileSystem::write(const U8 *buffer, S32 bytes) +BOOL LLFileSystem::write(const U8* buffer, S32 bytes) { std::string id_str; mFileID.toString(id_str); - const std::string filename = idToFilepath(id_str, mFileType); + const std::string extra_info = ""; + const std::string filename = LLDiskCache::getInstance()->metaDataToFilepath(id_str, mFileType, extra_info); BOOL success = FALSE; @@ -295,37 +219,37 @@ BOOL LLFileSystem::write(const U8 *buffer, S32 bytes) BOOL LLFileSystem::seek(S32 offset, S32 origin) { - if (-1 == origin) - { - origin = mPosition; - } + if (-1 == origin) + { + origin = mPosition; + } - S32 new_pos = origin + offset; + S32 new_pos = origin + offset; - S32 size = getSize(); + S32 size = getSize(); - if (new_pos > size) - { - LL_WARNS() << "Attempt to seek past end of file" << LL_ENDL; + if (new_pos > size) + { + LL_WARNS() << "Attempt to seek past end of file" << LL_ENDL; - mPosition = size; - return FALSE; - } - else if (new_pos < 0) - { - LL_WARNS() << "Attempt to seek past beginning of file" << LL_ENDL; + mPosition = size; + return FALSE; + } + else if (new_pos < 0) + { + LL_WARNS() << "Attempt to seek past beginning of file" << LL_ENDL; - mPosition = 0; - return FALSE; - } + mPosition = 0; + return FALSE; + } - mPosition = new_pos; - return TRUE; + mPosition = new_pos; + return TRUE; } S32 LLFileSystem::tell() const { - return mPosition; + return mPosition; } S32 LLFileSystem::getSize() @@ -335,11 +259,11 @@ S32 LLFileSystem::getSize() S32 LLFileSystem::getMaxSize() { - // offer up a huge size since we don't care what the max is + // offer up a huge size since we don't care what the max is return INT_MAX; } -BOOL LLFileSystem::rename(const LLUUID &new_id, const LLAssetType::EType new_type) +BOOL LLFileSystem::rename(const LLUUID& new_id, const LLAssetType::EType new_type) { LLFileSystem::renameFile(mFileID, mFileType, new_id, new_type); @@ -355,19 +279,3 @@ BOOL LLFileSystem::remove() return TRUE; } - -// static -void LLFileSystem::initClass() -{ - const std::string cache_dir = gDirUtilp->getExpandedFilename(LL_PATH_CACHE, mCacheDirName); - - LLFileSystem::mDiskCache = new LLDiskCache(cache_dir); - - mDiskCache->purge(); -} - -// static -void LLFileSystem::cleanupClass() -{ - delete LLFileSystem::mDiskCache; -} diff --git a/indra/llfilesystem/llfilesystem.h b/indra/llfilesystem/llfilesystem.h index 5d87de9bf8..46bf1bd775 100644 --- a/indra/llfilesystem/llfilesystem.h +++ b/indra/llfilesystem/llfilesystem.h @@ -1,5 +1,5 @@ -/** -/** +/** +/** * @file filesystem.h * @brief Simulate local file system operations. * @Note The initial implementation does actually use standard C++ @@ -9,21 +9,21 @@ * $LicenseInfo:firstyear=2002&license=viewerlgpl$ * Second Life Viewer Source Code * Copyright (C) 2010, Linden Research, Inc. - * + * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; * version 2.1 of the License only. - * + * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. - * + * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - * + * * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA * $/LicenseInfo$ */ @@ -37,51 +37,43 @@ class LLFileSystem { -public: - LLFileSystem(const LLUUID &file_id, const LLAssetType::EType file_type, S32 mode = LLFileSystem::READ); - ~LLFileSystem(); + public: + LLFileSystem(const LLUUID& file_id, const LLAssetType::EType file_type, S32 mode = LLFileSystem::READ); + ~LLFileSystem(); - BOOL read(U8 *buffer, S32 bytes, BOOL async = FALSE, F32 priority = 128.f); /* Flawfinder: ignore */ - BOOL isReadComplete(); - S32 getLastBytesRead(); - BOOL eof(); + BOOL read(U8* buffer, S32 bytes); + S32 getLastBytesRead(); + BOOL eof(); - BOOL write(const U8 *buffer, S32 bytes); - BOOL seek(S32 offset, S32 origin = -1); - S32 tell() const; + BOOL write(const U8* buffer, S32 bytes); + BOOL seek(S32 offset, S32 origin = -1); + S32 tell() const; - S32 getSize(); - S32 getMaxSize(); - BOOL rename(const LLUUID &new_id, const LLAssetType::EType new_type); - BOOL remove(); + S32 getSize(); + S32 getMaxSize(); + BOOL rename(const LLUUID& new_id, const LLAssetType::EType new_type); + BOOL remove(); - static bool getExists(const LLUUID &file_id, const LLAssetType::EType file_type); - static bool removeFile(const LLUUID &file_id, const LLAssetType::EType file_type); - static bool renameFile(const LLUUID &old_file_id, const LLAssetType::EType old_file_type, - const LLUUID &new_file_id, const LLAssetType::EType new_file_type); - static S32 getFileSize(const LLUUID &file_id, const LLAssetType::EType file_type); - - static void initClass(); - static void cleanupClass(); + static bool getExists(const LLUUID& file_id, const LLAssetType::EType file_type); + static bool removeFile(const LLUUID& file_id, const LLAssetType::EType file_type); + static bool renameFile(const LLUUID& old_file_id, const LLAssetType::EType old_file_type, + const LLUUID& new_file_id, const LLAssetType::EType new_file_type); + static S32 getFileSize(const LLUUID& file_id, const LLAssetType::EType file_type); -public: - static const S32 READ; - static const S32 WRITE; - static const S32 READ_WRITE; - static const S32 APPEND; - -protected: - LLAssetType::EType mFileType; - BOOL mReadComplete; - LLUUID mFileID; - S32 mPosition; - S32 mMode; - S32 mBytesRead; + public: + static const S32 READ; + static const S32 WRITE; + static const S32 READ_WRITE; + static const S32 APPEND; -private: - static const std::string idToFilepath(const std::string id, LLAssetType::EType at); - static std::string mCacheDirName; - static LLDiskCache* mDiskCache; + protected: + LLAssetType::EType mFileType; + LLUUID mFileID; + S32 mPosition; + S32 mMode; + S32 mBytesRead; +//private: +// static const std::string idToFilepath(const std::string id, LLAssetType::EType at); }; #endif // LL_FILESYSTEM_H -- cgit v1.2.3 From 5fdc4a1fb4a1788b20ca7a2e7764fd1391c35785 Mon Sep 17 00:00:00 2001 From: Callum Prentice Date: Wed, 7 Oct 2020 16:03:28 -0700 Subject: Add in some cache stats for the about box --- indra/llfilesystem/lldiskcache.cpp | 20 ++++++++++++++++++-- indra/llfilesystem/lldiskcache.h | 22 +++++----------------- 2 files changed, 23 insertions(+), 19 deletions(-) (limited to 'indra/llfilesystem') diff --git a/indra/llfilesystem/lldiskcache.cpp b/indra/llfilesystem/lldiskcache.cpp index 4b82cf3cce..455e27221e 100644 --- a/indra/llfilesystem/lldiskcache.cpp +++ b/indra/llfilesystem/lldiskcache.cpp @@ -187,8 +187,6 @@ const std::string LLDiskCache::metaDataToFilepath(const std::string id, file_path << assetTypeToString(at); file_path << ".asset"; - LL_INFOS() << "filepath.str() = " << file_path.str() << LL_ENDL; - return file_path.str(); } @@ -203,6 +201,24 @@ void LLDiskCache::updateFileAccessTime(const std::string file_path) boost::filesystem::last_write_time(file_path, file_time); } +/** + * + */ +const std::string LLDiskCache::getCacheInfo() +{ + std::ostringstream cache_info; + + F32 max_in_mb = (F32)mMaxSizeBytes / (1024.0 * 1024.0); + F32 percent_used = ((F32)dirFileSize(mCacheDir) / (F32)mMaxSizeBytes) * 100.0; + + cache_info << std::fixed; + cache_info << std::setprecision(1); + cache_info << "Max size " << max_in_mb << " MB "; + cache_info << "(" << percent_used << "% used)"; + + return cache_info.str(); +} + /** * Clear the cache by removing all the files in the cache directory * individually. It's important to maintain control of which directory diff --git a/indra/llfilesystem/lldiskcache.h b/indra/llfilesystem/lldiskcache.h index 12599a132e..34a4fda756 100644 --- a/indra/llfilesystem/lldiskcache.h +++ b/indra/llfilesystem/lldiskcache.h @@ -40,23 +40,6 @@ class LLDiskCache : virtual ~LLDiskCache() = default; public: - ///** - // * The location of the cache dir is set in llappviewer at startup via the - // * saved settings parameters. We do not have access to those saved settings - // * here in LLCommon so we must provide an accessor for other classes to use - // * when they need it - e.g. LLFilesystem needs the path so it can write files - // * to it. - // */ - //const std::string getCacheDirName() { return mCacheDir; } - - ///** - // * Each cache filename has a prefix inserted (see definition of the - // * mCacheFilenamePrefix variable below for why) but the LLFileSystem - // * component needs access to it to in order to create the file so we - // * expose it by a getter here. - // */ - //const std::string getCacheFilenamePrefix() { return mCacheFilenamePrefix; } - /** * Construct a filename and path to it based on the file meta data * (id, asset type, additional 'extra' info like discard level perhaps) @@ -86,6 +69,11 @@ class LLDiskCache : */ void clearCache(const std::string cache_dir); + /** + * Return some information about the cache for use in About Box etc. + */ + const std::string getCacheInfo(); + private: /** * Utility function to gather the total size the files in a given -- cgit v1.2.3 From 1a9942a51c2b5db51adb75356f342665743d1f16 Mon Sep 17 00:00:00 2001 From: Callum Prentice Date: Wed, 7 Oct 2020 16:43:01 -0700 Subject: Improve, rationalize and expand comments --- indra/llfilesystem/lldiskcache.cpp | 49 +++++++++++++--------------- indra/llfilesystem/lldiskcache.h | 65 ++++++++++++++++++++++++++++++++++++-- 2 files changed, 84 insertions(+), 30 deletions(-) (limited to 'indra/llfilesystem') diff --git a/indra/llfilesystem/lldiskcache.cpp b/indra/llfilesystem/lldiskcache.cpp index 455e27221e..e2e50c775d 100644 --- a/indra/llfilesystem/lldiskcache.cpp +++ b/indra/llfilesystem/lldiskcache.cpp @@ -2,6 +2,12 @@ * @file lldiskcache.cpp * @brief The disk cache implementation. * + * Note: Rather than keep the top level function comments up + * to date in both the source and header files, I elected to + * only have explicit comments about each function and variable + * in the header - look there for details. The same is true for + * description of how this code is supposed to work. + * * $LicenseInfo:firstyear=2009&license=viewerlgpl$ * Second Life Viewer Source Code * Copyright (C) 2020, Linden Research, Inc. @@ -40,10 +46,8 @@ LLDiskCache::LLDiskCache(const std::string cache_dir, mMaxSizeBytes(max_size_bytes), mEnableCacheDebugInfo(enable_cache_debug_info) { - // the prefix used for cache filenames to disambiguate them from other files mCacheFilenamePrefix = "sl_cache"; - // create cache dir if it does not exist boost::filesystem::create_directory(cache_dir); } @@ -126,7 +130,7 @@ void LLDiskCache::purge() const std::string LLDiskCache::assetTypeToString(LLAssetType::EType at) { /** - * Make use of the C++17 (or is it 14) feature that allows + * Make use of the handy C++17 feature that allows * for inline initialization of an std::map<> */ typedef std::map asset_type_to_name_t; @@ -190,20 +194,12 @@ const std::string LLDiskCache::metaDataToFilepath(const std::string id, return file_path.str(); } -/** - * Update the "last write time" of a file to "now". This must be called whenever a - * file in the cache is read (not written) so that the last time the file was - * accessed which is used in the mechanism for purging the cache, is up to date. - */ void LLDiskCache::updateFileAccessTime(const std::string file_path) { const std::time_t file_time = std::time(nullptr); boost::filesystem::last_write_time(file_path, file_time); } -/** - * - */ const std::string LLDiskCache::getCacheInfo() { std::ostringstream cache_info; @@ -219,14 +215,14 @@ const std::string LLDiskCache::getCacheInfo() return cache_info.str(); } -/** - * Clear the cache by removing all the files in the cache directory - * individually. It's important to maintain control of which directory - * if passed in and not let the user inadvertently (or maliciously) set - * it to an random location like your project source or OS system directory - */ void LLDiskCache::clearCache(const std::string cache_dir) { + /** + * See notes on performance in dirFileSize(..) - there may be + * a quicker way to do this by operating on the parent dir vs + * the component files but it's called infrequently so it's + * likely just fine + */ if (boost::filesystem::is_directory(cache_dir)) { for (auto& entry : boost::make_iterator_range(boost::filesystem::directory_iterator(cache_dir), {})) @@ -242,20 +238,19 @@ void LLDiskCache::clearCache(const std::string cache_dir) } } -/** - * Utility function to get the total filesize of all files in a directory. It - * used to test file extensions to only check cache files but that was removed. - * There may be a better way that works directly on the folder (similar to - * right clicking on a folder in the OS and asking for size vs right clicking - * on all files and adding up manually) but this is very fast - less than 100ms - * in my testing so, so long as it's not called frequently, it should be okay. - * Note that's it's only currently used for logging/debugging so if performance - * is ever an issue, optimizing this or removing it altogether, is an easy win. - */ uintmax_t LLDiskCache::dirFileSize(const std::string dir) { uintmax_t total_file_size = 0; + /** + * There may be a better way that works directly on the folder (similar to + * right clicking on a folder in the OS and asking for size vs right clicking + * on all files and adding up manually) but this is very fast - less than 100ms + * for 10,000 files in my testing so, so long as it's not called frequently, + * it should be okay. Note that's it's only currently used for logging/debugging + * so if performance is ever an issue, optimizing this or removing it altogether, + * is an easy win. + */ if (boost::filesystem::is_directory(dir)) { for (auto& entry : boost::make_iterator_range(boost::filesystem::directory_iterator(dir), {})) diff --git a/indra/llfilesystem/lldiskcache.h b/indra/llfilesystem/lldiskcache.h index 34a4fda756..f718b7a328 100644 --- a/indra/llfilesystem/lldiskcache.h +++ b/indra/llfilesystem/lldiskcache.h @@ -1,6 +1,41 @@ /** * @file lldiskcache.h - * @brief The disk cache implementation. + * @brief The disk cache implementation declarations. + * + * @Description: + * This code implements a disk cache using the following ideas: + * 1/ The metadata for a file can be encapsulated in the filename. + The filenames will be composed of the following fields: + Prefix: Used to identify the file as a part of the cache. + An additional reason for using a prefix is that it + might be possible, either accidentally or maliciously + to end up with the cache dir set to a non-cache + location such as your OS system dir or a work folder. + Purging files from that would obviously be a disaster + so this is an extra step to help avoid that scenario. + ID: Typically the asset ID (UUID) of the asset being + saved but can be anything valid for a filename + Extra Info: A field for use in the future that can be used + to store extra identifiers - e.g. the discard + level of a JPEG2000 file + Asset Type: A text string created from the LLAssetType enum + that identifies the type of asset being stored. + .asset A file extension of .asset is used to help + identify this as a Viewer asset file + * 2/ The time of last access for a file can be updated instantly + * for file reads and automatically as part of the file writes. + * 3/ The purge algorithm collects a list of all files in the + * directory, sorts them by date of last access (write) and then + * deletes any files based on age until the total size of all + * the files is less than the maximum size specified. + * 4/ An LLSingleton idiom is used since there will only ever be + * a single cache and we want to access it from numerous places. + * 5/ Performance on my modest system seems very acceptable. For + * example, in testing, I was able to purge a directory of + * 10,000 files, deleting about half of them in ~ 1700ms. For + * the same sized directory of files, writing the last updated + * time to each took less than 600ms indicating that this + * important part of the mechanism has almost no overhead. * * $LicenseInfo:firstyear=2009&license=viewerlgpl$ * Second Life Viewer Source Code @@ -33,10 +68,32 @@ class LLDiskCache : public LLParamSingleton { public: + /** + * Since this is using the LLSingleton pattern but we + * want to allow the constructor to be called first + * with various parameters, we also invoke the + * LLParamSingleton idiom and use it to initialize + * the class via a call in LLAppViewer. + */ LLSINGLETON(LLDiskCache, + /** + * The full name of the cache folder - typically a + * a child of the main Viewer cache directory. Defined + * by the setting at 'DiskCacheDirName' + */ const std::string cache_dir, + /** + * The maximum size of the cache in bytes - Defined by + * the setting at 'DiskCacheMaxSizeMB' (* 1024 * 1024) + */ const int max_size_bytes, + /** + * A flag that enables extra cache debugging so that + * if there are bugs, we can ask uses to enable this + * setting and send us their logs + */ const bool enable_cache_debug_info); + virtual ~LLDiskCache() = default; public: @@ -54,7 +111,7 @@ class LLDiskCache : /** * Update the "last write time" of a file to "now". This must be called whenever a * file in the cache is read (not written) so that the last time the file was - * accessed which is used in the mechanism for purging the cache, is up to date. + * accessed is up to date (This is used in the mechanism for purging the cache) */ void updateFileAccessTime(const std::string file_path); @@ -65,7 +122,9 @@ class LLDiskCache : void purge(); /** - * Clear the cache by removing the files in the cache directory individually + * Clear the cache by removing all the files in the specified cache + * directory individually. Only the files that contain a prefix defined + * by mCacheFilenamePrefix will be removed. */ void clearCache(const std::string cache_dir); -- cgit v1.2.3 From 3051db7b61ee43fffd28f0a12c0714b11b6b7df7 Mon Sep 17 00:00:00 2001 From: Mnikolenko Productengine Date: Thu, 8 Oct 2020 15:26:54 +0300 Subject: Purge excessive files from disc cache each startup --- indra/llfilesystem/lldiskcache.cpp | 6 +++--- indra/llfilesystem/lldiskcache.h | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) (limited to 'indra/llfilesystem') diff --git a/indra/llfilesystem/lldiskcache.cpp b/indra/llfilesystem/lldiskcache.cpp index e2e50c775d..efe5e7092c 100644 --- a/indra/llfilesystem/lldiskcache.cpp +++ b/indra/llfilesystem/lldiskcache.cpp @@ -215,7 +215,7 @@ const std::string LLDiskCache::getCacheInfo() return cache_info.str(); } -void LLDiskCache::clearCache(const std::string cache_dir) +void LLDiskCache::clearCache() { /** * See notes on performance in dirFileSize(..) - there may be @@ -223,9 +223,9 @@ void LLDiskCache::clearCache(const std::string cache_dir) * the component files but it's called infrequently so it's * likely just fine */ - if (boost::filesystem::is_directory(cache_dir)) + if (boost::filesystem::is_directory(mCacheDir)) { - for (auto& entry : boost::make_iterator_range(boost::filesystem::directory_iterator(cache_dir), {})) + for (auto& entry : boost::make_iterator_range(boost::filesystem::directory_iterator(mCacheDir), {})) { if (boost::filesystem::is_regular_file(entry)) { diff --git a/indra/llfilesystem/lldiskcache.h b/indra/llfilesystem/lldiskcache.h index f718b7a328..b25eac8538 100644 --- a/indra/llfilesystem/lldiskcache.h +++ b/indra/llfilesystem/lldiskcache.h @@ -126,7 +126,7 @@ class LLDiskCache : * directory individually. Only the files that contain a prefix defined * by mCacheFilenamePrefix will be removed. */ - void clearCache(const std::string cache_dir); + void clearCache(); /** * Return some information about the cache for use in About Box etc. -- cgit v1.2.3 From 31d80c21b5e17e962c0fdb5a370d3ddea8694768 Mon Sep 17 00:00:00 2001 From: Mnikolenko ProductEngine Date: Thu, 8 Oct 2020 21:17:28 +0300 Subject: macos build fix --- indra/llfilesystem/llfilesystem.h | 1 - 1 file changed, 1 deletion(-) (limited to 'indra/llfilesystem') diff --git a/indra/llfilesystem/llfilesystem.h b/indra/llfilesystem/llfilesystem.h index 46bf1bd775..89bfff5798 100644 --- a/indra/llfilesystem/llfilesystem.h +++ b/indra/llfilesystem/llfilesystem.h @@ -1,4 +1,3 @@ -/** /** * @file filesystem.h * @brief Simulate local file system operations. -- cgit v1.2.3 From 3b4bd86a1de3fb1a9065024089fcfec2dae1da85 Mon Sep 17 00:00:00 2001 From: Mnikolenko Productengine Date: Tue, 27 Oct 2020 16:46:31 +0200 Subject: SL-14182 remove old script asset file after saving changes and allow renaming files if destination file exists --- indra/llfilesystem/llfilesystem.cpp | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'indra/llfilesystem') diff --git a/indra/llfilesystem/llfilesystem.cpp b/indra/llfilesystem/llfilesystem.cpp index c6b20caa69..932ef2a9c6 100644 --- a/indra/llfilesystem/llfilesystem.cpp +++ b/indra/llfilesystem/llfilesystem.cpp @@ -99,12 +99,16 @@ bool LLFileSystem::renameFile(const LLUUID& old_file_id, const LLAssetType::ETyp new_file_id.toString(new_id_str); const std::string new_filename = LLDiskCache::getInstance()->metaDataToFilepath(new_id_str, new_file_type, extra_info); + // Rename needs the new file to not exist. + LLFileSystem::removeFile(new_file_id, new_file_type); + if (std::rename(old_filename.c_str(), new_filename.c_str())) { // We would like to return FALSE here indicating the operation // failed but the original code does not and doing so seems to // break a lot of things so we go with the flow... //return FALSE; + LL_WARNS() << "Failed to rename " << old_file_id << " to " << new_id_str << " reason: " << strerror(errno) << LL_ENDL; } return TRUE; -- cgit v1.2.3 From 391ada1150a861e899dcde8558e9efd4c5efc981 Mon Sep 17 00:00:00 2001 From: Callum Prentice Date: Tue, 27 Oct 2020 13:53:17 -0700 Subject: Fix for meta issue: SL-14211 Determine optimum cache size for VFS replacement cache --- indra/llfilesystem/lldiskcache.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'indra/llfilesystem') diff --git a/indra/llfilesystem/lldiskcache.h b/indra/llfilesystem/lldiskcache.h index b25eac8538..997884da31 100644 --- a/indra/llfilesystem/lldiskcache.h +++ b/indra/llfilesystem/lldiskcache.h @@ -83,8 +83,8 @@ class LLDiskCache : */ const std::string cache_dir, /** - * The maximum size of the cache in bytes - Defined by - * the setting at 'DiskCacheMaxSizeMB' (* 1024 * 1024) + * The maximum size of the cache in bytes - Based on the + * setting at 'CacheSize' and 'DiskCachePercentOfTotal' */ const int max_size_bytes, /** -- cgit v1.2.3 From aae7259a0adaddb4b2d9a2a62a0d4ff95fe5e2b3 Mon Sep 17 00:00:00 2001 From: Callum Prentice Date: Tue, 27 Oct 2020 14:02:24 -0700 Subject: Fix for meta issue: SL-14210 Prune descriptive tag from new cache filenames --- indra/llfilesystem/lldiskcache.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'indra/llfilesystem') diff --git a/indra/llfilesystem/lldiskcache.cpp b/indra/llfilesystem/lldiskcache.cpp index efe5e7092c..91fc1b15d1 100644 --- a/indra/llfilesystem/lldiskcache.cpp +++ b/indra/llfilesystem/lldiskcache.cpp @@ -187,8 +187,12 @@ const std::string LLDiskCache::metaDataToFilepath(const std::string id, file_path << id; file_path << "_"; file_path << (extra_info.empty() ? "0" : extra_info); - file_path << "_"; - file_path << assetTypeToString(at); + //file_path << "_"; + //file_path << assetTypeToString(at); // see SL-14210 Prune descriptive tag from new cache filenames + // for details of why it was removed. Note that if you put it + // back or change the format of the filename, the cache files + // files will be invalidated (and perhaps, more importantly, + // never deleted unless you delete them manually). file_path << ".asset"; return file_path.str(); -- cgit v1.2.3 From 53cae8b21f0f77fbb1be22c64deee9b6a3f237f7 Mon Sep 17 00:00:00 2001 From: Mnikolenko Productengine Date: Fri, 11 Dec 2020 16:42:10 +0200 Subject: SL-14505 FIXED [Win10] The viewer isn't started on the non-English system locale --- indra/llfilesystem/lldiskcache.cpp | 34 +++++++++++++++++++++++++++------- indra/llfilesystem/llfilesystem.cpp | 16 +++++++--------- 2 files changed, 34 insertions(+), 16 deletions(-) (limited to 'indra/llfilesystem') diff --git a/indra/llfilesystem/lldiskcache.cpp b/indra/llfilesystem/lldiskcache.cpp index 91fc1b15d1..34ff80b250 100644 --- a/indra/llfilesystem/lldiskcache.cpp +++ b/indra/llfilesystem/lldiskcache.cpp @@ -48,7 +48,7 @@ LLDiskCache::LLDiskCache(const std::string cache_dir, { mCacheFilenamePrefix = "sl_cache"; - boost::filesystem::create_directory(cache_dir); + LLFile::mkdir(cache_dir); } void LLDiskCache::purge() @@ -63,9 +63,14 @@ void LLDiskCache::purge() typedef std::pair> file_info_t; std::vector file_info; - if (boost::filesystem::is_directory(mCacheDir)) +#if LL_WINDOWS + std::wstring cache_path(utf8str_to_utf16str(mCacheDir)); +#else + std::string cache_path(mCacheDir); +#endif + if (boost::filesystem::is_directory(cache_path)) { - for (auto& entry : boost::make_iterator_range(boost::filesystem::directory_iterator(mCacheDir), {})) + for (auto& entry : boost::make_iterator_range(boost::filesystem::directory_iterator(cache_path), {})) { if (boost::filesystem::is_regular_file(entry)) { @@ -201,7 +206,12 @@ const std::string LLDiskCache::metaDataToFilepath(const std::string id, void LLDiskCache::updateFileAccessTime(const std::string file_path) { const std::time_t file_time = std::time(nullptr); + +#if LL_WINDOWS + boost::filesystem::last_write_time(utf8str_to_utf16str(file_path), file_time); +#else boost::filesystem::last_write_time(file_path, file_time); +#endif } const std::string LLDiskCache::getCacheInfo() @@ -227,9 +237,14 @@ void LLDiskCache::clearCache() * the component files but it's called infrequently so it's * likely just fine */ - if (boost::filesystem::is_directory(mCacheDir)) +#if LL_WINDOWS + std::wstring cache_path(utf8str_to_utf16str(mCacheDir)); +#else + std::string cache_path(mCacheDir); +#endif + if (boost::filesystem::is_directory(cache_path)) { - for (auto& entry : boost::make_iterator_range(boost::filesystem::directory_iterator(mCacheDir), {})) + for (auto& entry : boost::make_iterator_range(boost::filesystem::directory_iterator(cache_path), {})) { if (boost::filesystem::is_regular_file(entry)) { @@ -255,9 +270,14 @@ uintmax_t LLDiskCache::dirFileSize(const std::string dir) * so if performance is ever an issue, optimizing this or removing it altogether, * is an easy win. */ - if (boost::filesystem::is_directory(dir)) +#if LL_WINDOWS + std::wstring dir_path(utf8str_to_utf16str(dir)); +#else + std::string dir_path(dir); +#endif + if (boost::filesystem::is_directory(dir_path)) { - for (auto& entry : boost::make_iterator_range(boost::filesystem::directory_iterator(dir), {})) + for (auto& entry : boost::make_iterator_range(boost::filesystem::directory_iterator(dir_path), {})) { if (boost::filesystem::is_regular_file(entry)) { diff --git a/indra/llfilesystem/llfilesystem.cpp b/indra/llfilesystem/llfilesystem.cpp index 932ef2a9c6..64e0b9f193 100644 --- a/indra/llfilesystem/llfilesystem.cpp +++ b/indra/llfilesystem/llfilesystem.cpp @@ -34,8 +34,6 @@ #include "llfasttimer.h" #include "lldiskcache.h" -#include - const S32 LLFileSystem::READ = 0x00000001; const S32 LLFileSystem::WRITE = 0x00000002; const S32 LLFileSystem::READ_WRITE = 0x00000003; // LLFileSystem::READ & LLFileSystem::WRITE @@ -64,7 +62,7 @@ bool LLFileSystem::getExists(const LLUUID& file_id, const LLAssetType::EType fil const std::string extra_info = ""; const std::string filename = LLDiskCache::getInstance()->metaDataToFilepath(id_str, file_type, extra_info); - std::ifstream file(filename, std::ios::binary); + llifstream file(filename, std::ios::binary); if (file.is_open()) { file.seekg(0, std::ios::end); @@ -81,7 +79,7 @@ bool LLFileSystem::removeFile(const LLUUID& file_id, const LLAssetType::EType fi const std::string extra_info = ""; const std::string filename = LLDiskCache::getInstance()->metaDataToFilepath(id_str, file_type, extra_info); - std::remove(filename.c_str()); + LLFile::remove(filename.c_str()); return true; } @@ -102,7 +100,7 @@ bool LLFileSystem::renameFile(const LLUUID& old_file_id, const LLAssetType::ETyp // Rename needs the new file to not exist. LLFileSystem::removeFile(new_file_id, new_file_type); - if (std::rename(old_filename.c_str(), new_filename.c_str())) + if (LLFile::rename(old_filename, new_filename) != 0) { // We would like to return FALSE here indicating the operation // failed but the original code does not and doing so seems to @@ -123,7 +121,7 @@ S32 LLFileSystem::getFileSize(const LLUUID& file_id, const LLAssetType::EType fi const std::string filename = LLDiskCache::getInstance()->metaDataToFilepath(id_str, file_type, extra_info); S32 file_size = 0; - std::ifstream file(filename, std::ios::binary); + llifstream file(filename, std::ios::binary); if (file.is_open()) { file.seekg(0, std::ios::end); @@ -142,7 +140,7 @@ BOOL LLFileSystem::read(U8* buffer, S32 bytes) const std::string extra_info = ""; const std::string filename = LLDiskCache::getInstance()->metaDataToFilepath(id, mFileType, extra_info); - std::ifstream file(filename, std::ios::binary); + llifstream file(filename, std::ios::binary); if (file.is_open()) { file.seekg(mPosition, std::ios::beg); @@ -197,7 +195,7 @@ BOOL LLFileSystem::write(const U8* buffer, S32 bytes) if (mMode == APPEND) { - std::ofstream ofs(filename, std::ios::app | std::ios::binary); + llofstream ofs(filename, std::ios::app | std::ios::binary); if (ofs) { ofs.write((const char*)buffer, bytes); @@ -207,7 +205,7 @@ BOOL LLFileSystem::write(const U8* buffer, S32 bytes) } else { - std::ofstream ofs(filename, std::ios::binary); + llofstream ofs(filename, std::ios::binary); if (ofs) { ofs.write((const char*)buffer, bytes); -- cgit v1.2.3 From 60e8f990fdbc5f00b69c1a7355330ff421133cea Mon Sep 17 00:00:00 2001 From: Callum Prentice Date: Fri, 8 Jan 2021 14:21:58 -0800 Subject: Addresses SL-14582: Add code to only write the file last access time occasionally --- indra/llfilesystem/lldiskcache.cpp | 40 +++++++++++++++++++++++++++++++++++--- 1 file changed, 37 insertions(+), 3 deletions(-) (limited to 'indra/llfilesystem') diff --git a/indra/llfilesystem/lldiskcache.cpp b/indra/llfilesystem/lldiskcache.cpp index 34ff80b250..c9f7684b5a 100644 --- a/indra/llfilesystem/lldiskcache.cpp +++ b/indra/llfilesystem/lldiskcache.cpp @@ -205,12 +205,46 @@ const std::string LLDiskCache::metaDataToFilepath(const std::string id, void LLDiskCache::updateFileAccessTime(const std::string file_path) { - const std::time_t file_time = std::time(nullptr); + /** + * Threshold in time_t units that is used to decide if the last access time + * time of the file is updated or not. Added as a precaution for the concern + * outlined in SL-14582 about frequent writes on older SSDs reducing their + * lifespan. I think this is the right place for the threshold value - rather + * than it being a pref - do comment on that Jira if you disagree... + * + * Let's start with 1 hour in time_t units and see how that unfolds + */ + const std::time_t time_threshold = 1 * 60 * 60; + + // current time + const std::time_t cur_time = std::time(nullptr); #if LL_WINDOWS - boost::filesystem::last_write_time(utf8str_to_utf16str(file_path), file_time); + // file last write time + const std::time_t last_write_time = boost::filesystem::last_write_time(utf8str_to_utf16str(file_path)); + + // delta between cur time and last time the file was written + const std::time_t delta_time = cur_time - last_write_time; + + // we only write the new value if the time in time_threshold has elapsed + // before the last one + if (delta_time > time_threshold) + { + boost::filesystem::last_write_time(utf8str_to_utf16str(file_path), cur_time); + } #else - boost::filesystem::last_write_time(file_path, file_time); + // file last write time + const std::time_t last_write_time = boost::filesystem::last_write_time(file_path); + + // delta between cur time and last time the file was written + const std::time_t delta_time = cur_time - last_write_time; + + // we only write the new value if the time in time_threshold has elapsed + // before the last one + if (delta_time > time_threshold) + { + boost::filesystem::last_write_time(file_path, cur_time); + } #endif } -- cgit v1.2.3