diff options
author | Andrey Lihatskiy <alihatskiy@productengine.com> | 2021-03-09 21:58:57 +0200 |
---|---|---|
committer | Andrey Lihatskiy <alihatskiy@productengine.com> | 2021-03-09 21:58:57 +0200 |
commit | 1fe0033a7e8dc2b68fb1e5a25144fa0bc719b1af (patch) | |
tree | 0a4c8381d5815663f6bc84488f5945c27f6f8080 /indra/llvfs | |
parent | 4f55a14f8281e7cccfa5c6980ad77921cb5ebdd8 (diff) | |
parent | 88d837c16e768c5262073a7df965066d4bd8842c (diff) |
Merge branch 'master' into DRTVWR-483
Diffstat (limited to 'indra/llvfs')
28 files changed, 8433 insertions, 0 deletions
diff --git a/indra/llvfs/CMakeLists.txt b/indra/llvfs/CMakeLists.txt new file mode 100644 index 0000000000..67dce8c073 --- /dev/null +++ b/indra/llvfs/CMakeLists.txt @@ -0,0 +1,104 @@ +# -*- cmake -*- + +project(llvfs) + +include(00-Common) +include(LLCommon) +include(UnixInstall) + +include_directories( + ${LLCOMMON_INCLUDE_DIRS} + ${LLCOMMON_SYSTEM_INCLUDE_DIRS} + ) + +set(llvfs_SOURCE_FILES + lldir.cpp + lldiriterator.cpp + lllfsthread.cpp + llpidlock.cpp + llvfile.cpp + llvfs.cpp + llvfsthread.cpp + ) + +set(llvfs_HEADER_FILES + CMakeLists.txt + + lldir.h + lldirguard.h + lldiriterator.h + lllfsthread.h + llpidlock.h + llvfile.h + llvfs.h + llvfsthread.h + ) + +if (DARWIN) + LIST(APPEND llvfs_SOURCE_FILES lldir_mac.cpp) + LIST(APPEND llvfs_HEADER_FILES lldir_mac.h) + LIST(APPEND llvfs_SOURCE_FILES llvfs_objc.mm) + LIST(APPEND llvfs_HEADER_FILES llvfs_objc.h) +endif (DARWIN) + +if (LINUX) + LIST(APPEND llvfs_SOURCE_FILES lldir_linux.cpp) + LIST(APPEND llvfs_HEADER_FILES lldir_linux.h) + + if (INSTALL) + set_source_files_properties(lldir_linux.cpp + PROPERTIES COMPILE_FLAGS + "-DAPP_RO_DATA_DIR=\\\"${APP_SHARE_DIR}\\\"" + ) + endif (INSTALL) +endif (LINUX) + +if (WINDOWS) + LIST(APPEND llvfs_SOURCE_FILES lldir_win32.cpp) + LIST(APPEND llvfs_HEADER_FILES lldir_win32.h) +endif (WINDOWS) + +set_source_files_properties(${llvfs_HEADER_FILES} + PROPERTIES HEADER_FILE_ONLY TRUE) + +list(APPEND llvfs_SOURCE_FILES ${llvfs_HEADER_FILES}) + +add_library (llvfs ${llvfs_SOURCE_FILES}) + +set(vfs_BOOST_LIBRARIES + ${BOOST_FILESYSTEM_LIBRARY} + ${BOOST_SYSTEM_LIBRARY} + ) + +target_link_libraries(llvfs + ${LLCOMMON_LIBRARIES} + ${vfs_BOOST_LIBRARIES} + ) + +if (DARWIN) + include(CMakeFindFrameworks) + find_library(COCOA_LIBRARY Cocoa) + target_link_libraries(llvfs ${COCOA_LIBRARY}) +endif (DARWIN) + + +# Add tests +if (LL_TESTS) + include(LLAddBuildTest) + # UNIT TESTS + SET(llvfs_TEST_SOURCE_FILES + lldiriterator.cpp + ) + + set_source_files_properties(lldiriterator.cpp + PROPERTIES + LL_TEST_ADDITIONAL_LIBRARIES "${vfs_BOOST_LIBRARIES}" + ) + LL_ADD_PROJECT_UNIT_TESTS(llvfs "${llvfs_TEST_SOURCE_FILES}") + + # INTEGRATION TESTS + set(test_libs llmath llcommon llvfs ${LLCOMMON_LIBRARIES} ${WINDOWS_LIBRARIES}) + + # TODO: Some of these need refactoring to be proper Unit tests rather than Integration tests. + LL_ADD_INTEGRATION_TEST(lldir "" "${test_libs}") +endif (LL_TESTS) diff --git a/indra/llvfs/lldir.cpp b/indra/llvfs/lldir.cpp new file mode 100644 index 0000000000..10fbc06c61 --- /dev/null +++ b/indra/llvfs/lldir.cpp @@ -0,0 +1,1134 @@ +/** + * @file lldir.cpp + * @brief implementation of directory utilities base class + * + * $LicenseInfo:firstyear=2002&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2010, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include "linden_common.h" + +#if !LL_WINDOWS +#include <sys/stat.h> +#include <sys/types.h> +#include <errno.h> +#else +#include <direct.h> +#endif + +#include "lldir.h" + +#include "llerror.h" +#include "lltimer.h" // ms_sleep() +#include "lluuid.h" + +#include "lldiriterator.h" +#include "stringize.h" +#include "llstring.h" +#include <boost/filesystem.hpp> +#include <boost/foreach.hpp> +#include <boost/range/begin.hpp> +#include <boost/range/end.hpp> +#include <boost/assign/list_of.hpp> +#include <boost/bind.hpp> +#include <boost/ref.hpp> +#include <algorithm> + +using boost::assign::list_of; +using boost::assign::map_list_of; + +#if LL_WINDOWS +#include "lldir_win32.h" +LLDir_Win32 gDirUtil; +#elif LL_DARWIN +#include "lldir_mac.h" +LLDir_Mac gDirUtil; +#elif LL_SOLARIS +#include "lldir_solaris.h" +LLDir_Solaris gDirUtil; +#else +#include "lldir_linux.h" +LLDir_Linux gDirUtil; +#endif + +LLDir *gDirUtilp = (LLDir *)&gDirUtil; + +/// Values for findSkinnedFilenames(subdir) parameter +const char + *LLDir::XUI = "xui", + *LLDir::TEXTURES = "textures", + *LLDir::SKINBASE = ""; + +static const char* const empty = ""; +std::string LLDir::sDumpDir = ""; + +LLDir::LLDir() +: mAppName(""), + mExecutablePathAndName(""), + mExecutableFilename(""), + mExecutableDir(""), + mAppRODataDir(""), + mOSUserDir(""), + mOSUserAppDir(""), + mLindenUserDir(""), + mOSCacheDir(""), + mCAFile(""), + mTempDir(""), + mDirDelimiter("/"), // fallback to forward slash if not overridden + mLanguage("en"), + mUserName("undefined") +{ +} + +LLDir::~LLDir() +{ +} + +std::vector<std::string> LLDir::getFilesInDir(const std::string &dirname) +{ + //Returns a vector of fullpath filenames. + +#ifdef LL_WINDOWS // or BOOST_WINDOWS_API + boost::filesystem::path p(utf8str_to_utf16str(dirname)); +#else + boost::filesystem::path p(dirname); +#endif + + std::vector<std::string> v; + + if (exists(p)) + { + if (is_directory(p)) + { + boost::filesystem::directory_iterator end_iter; + for (boost::filesystem::directory_iterator dir_itr(p); + dir_itr != end_iter; + ++dir_itr) + { + if (boost::filesystem::is_regular_file(dir_itr->status())) + { + v.push_back(dir_itr->path().filename().string()); + } + } + } + } + return v; +} + +S32 LLDir::deleteFilesInDir(const std::string &dirname, const std::string &mask) +{ + S32 count = 0; + std::string filename; + std::string fullpath; + S32 result; + + // File masks starting with "/" will match nothing, so we consider them invalid. + if (LLStringUtil::startsWith(mask, getDirDelimiter())) + { + LL_WARNS() << "Invalid file mask: " << mask << LL_ENDL; + llassert(!"Invalid file mask"); + } + + LLDirIterator iter(dirname, mask); + while (iter.next(filename)) + { + fullpath = add(dirname, filename); + + if(LLFile::isdir(fullpath)) + { + // skipping directory traversal filenames + count++; + continue; + } + + S32 retry_count = 0; + while (retry_count < 5) + { + if (0 != LLFile::remove(fullpath)) + { + retry_count++; + result = errno; + LL_WARNS() << "Problem removing " << fullpath << " - errorcode: " + << result << " attempt " << retry_count << LL_ENDL; + + if(retry_count >= 5) + { + LL_WARNS() << "Failed to remove " << fullpath << LL_ENDL ; + return count ; + } + + ms_sleep(100); + } + else + { + if (retry_count) + { + LL_WARNS() << "Successfully removed " << fullpath << LL_ENDL; + } + break; + } + } + count++; + } + return count; +} + +U32 LLDir::deleteDirAndContents(const std::string& dir_name) +{ + //Removes the directory and its contents. Returns number of files deleted. + + U32 num_deleted = 0; + + try + { +#ifdef LL_WINDOWS // or BOOST_WINDOWS_API + boost::filesystem::path dir_path(utf8str_to_utf16str(dir_name)); +#else + boost::filesystem::path dir_path(dir_name); +#endif + + if (boost::filesystem::exists (dir_path)) + { + if (!boost::filesystem::is_empty (dir_path)) + { // Directory has content + num_deleted = boost::filesystem::remove_all (dir_path); + } + else + { // Directory is empty + boost::filesystem::remove (dir_path); + } + } + } + catch (boost::filesystem::filesystem_error &er) + { + LL_WARNS() << "Failed to delete " << dir_name << " with error " << er.code().message() << LL_ENDL; + } + return num_deleted; +} + +const std::string LLDir::findFile(const std::string &filename, + const std::string& searchPath1, + const std::string& searchPath2, + const std::string& searchPath3) const +{ + std::vector<std::string> search_paths; + search_paths.push_back(searchPath1); + search_paths.push_back(searchPath2); + search_paths.push_back(searchPath3); + return findFile(filename, search_paths); +} + +const std::string LLDir::findFile(const std::string& filename, const std::vector<std::string> search_paths) const +{ + std::vector<std::string>::const_iterator search_path_iter; + for (search_path_iter = search_paths.begin(); + search_path_iter != search_paths.end(); + ++search_path_iter) + { + if (!search_path_iter->empty()) + { + std::string filename_and_path = (*search_path_iter); + if (!filename.empty()) + { + filename_and_path += getDirDelimiter() + filename; + } + if (fileExists(filename_and_path)) + { + return filename_and_path; + } + } + } + return ""; +} + + +const std::string &LLDir::getExecutablePathAndName() const +{ + return mExecutablePathAndName; +} + +const std::string &LLDir::getExecutableFilename() const +{ + return mExecutableFilename; +} + +const std::string &LLDir::getExecutableDir() const +{ + return mExecutableDir; +} + +const std::string &LLDir::getWorkingDir() const +{ + return mWorkingDir; +} + +const std::string &LLDir::getAppName() const +{ + return mAppName; +} + +const std::string &LLDir::getAppRODataDir() const +{ + return mAppRODataDir; +} + +const std::string &LLDir::getOSUserDir() const +{ + return mOSUserDir; +} + +const std::string &LLDir::getOSUserAppDir() const +{ + return mOSUserAppDir; +} + +const std::string &LLDir::getLindenUserDir() const +{ + if (mLindenUserDir.empty()) + { + LL_DEBUGS() << "getLindenUserDir() called early, we don't have the user name yet - returning empty string to caller" << LL_ENDL; + } + + return mLindenUserDir; +} + +const std::string& LLDir::getChatLogsDir() const +{ + return mChatLogsDir; +} + +void LLDir::setDumpDir( const std::string& path ) +{ + LLDir::sDumpDir = path; + if (LLStringUtil::endsWith(sDumpDir, mDirDelimiter)) + { + sDumpDir.erase(sDumpDir.size() - mDirDelimiter.size()); + } +} + +const std::string &LLDir::getDumpDir() const +{ + if (sDumpDir.empty() ) + { + LLUUID uid; + uid.generate(); + + sDumpDir = gDirUtilp->getExpandedFilename(LL_PATH_LOGS, "") + + "dump-" + uid.asString(); + + dir_exists_or_crash(sDumpDir); + } + + return LLDir::sDumpDir; +} + +const std::string &LLDir::getPerAccountChatLogsDir() const +{ + return mPerAccountChatLogsDir; +} + +const std::string &LLDir::getTempDir() const +{ + return mTempDir; +} + +const std::string LLDir::getCacheDir(bool get_default) const +{ + if (mCacheDir.empty() || get_default) + { + if (!mDefaultCacheDir.empty()) + { // Set at startup - can't set here due to const API + return mDefaultCacheDir; + } + + std::string res = buildSLOSCacheDir(); + return res; + } + else + { + return mCacheDir; + } +} + +// Return the default cache directory +std::string LLDir::buildSLOSCacheDir() const +{ + std::string res; + if (getOSCacheDir().empty()) + { + if (getOSUserAppDir().empty()) + { + res = "data"; + } + else + { + res = add(getOSUserAppDir(), "cache"); + } + } + else + { + res = add(getOSCacheDir(), "SecondLife"); + } + return res; +} + + + +const std::string &LLDir::getOSCacheDir() const +{ + return mOSCacheDir; +} + + +const std::string &LLDir::getCAFile() const +{ + return mCAFile; +} + +const std::string &LLDir::getDirDelimiter() const +{ + return mDirDelimiter; +} + +const std::string& LLDir::getDefaultSkinDir() const +{ + return mDefaultSkinDir; +} + +const std::string &LLDir::getSkinDir() const +{ + return mSkinDir; +} + +const std::string &LLDir::getUserDefaultSkinDir() const +{ + return mUserDefaultSkinDir; +} + +const std::string &LLDir::getUserSkinDir() const +{ + return mUserSkinDir; +} + +const std::string LLDir::getSkinBaseDir() const +{ + return mSkinBaseDir; +} + +const std::string &LLDir::getLLPluginDir() const +{ + return mLLPluginDir; +} + +const std::string &LLDir::getUserName() const +{ + return mUserName; +} + +static std::string ELLPathToString(ELLPath location) +{ + typedef std::map<ELLPath, const char*> ELLPathMap; +#define ENT(symbol) (symbol, #symbol) + static const ELLPathMap sMap = map_list_of + ENT(LL_PATH_NONE) + ENT(LL_PATH_USER_SETTINGS) + ENT(LL_PATH_APP_SETTINGS) + ENT(LL_PATH_PER_SL_ACCOUNT) // returns/expands to blank string if we don't know the account name yet + ENT(LL_PATH_CACHE) + ENT(LL_PATH_CHARACTER) + ENT(LL_PATH_HELP) + ENT(LL_PATH_LOGS) + ENT(LL_PATH_TEMP) + ENT(LL_PATH_SKINS) + ENT(LL_PATH_TOP_SKIN) + ENT(LL_PATH_CHAT_LOGS) + ENT(LL_PATH_PER_ACCOUNT_CHAT_LOGS) + ENT(LL_PATH_USER_SKIN) + ENT(LL_PATH_LOCAL_ASSETS) + ENT(LL_PATH_EXECUTABLE) + ENT(LL_PATH_DEFAULT_SKIN) + ENT(LL_PATH_FONTS) + ENT(LL_PATH_LAST) + ; +#undef ENT + + ELLPathMap::const_iterator found = sMap.find(location); + if (found != sMap.end()) + return found->second; + return STRINGIZE("Invalid ELLPath value " << location); +} + +std::string LLDir::getExpandedFilename(ELLPath location, const std::string& filename) const +{ + return getExpandedFilename(location, "", filename); +} + +std::string LLDir::getExpandedFilename(ELLPath location, const std::string& subdir, const std::string& filename) const +{ + return getExpandedFilename(location, "", subdir, filename); +} + +std::string LLDir::getExpandedFilename(ELLPath location, const std::string& subdir1, const std::string& subdir2, const std::string& in_filename) const +{ + std::string prefix; + switch (location) + { + case LL_PATH_NONE: + // Do nothing + break; + + case LL_PATH_APP_SETTINGS: + prefix = add(getAppRODataDir(), "app_settings"); + break; + + case LL_PATH_CHARACTER: + prefix = add(getAppRODataDir(), "character"); + break; + + case LL_PATH_HELP: + prefix = "help"; + break; + + case LL_PATH_CACHE: + prefix = getCacheDir(); + break; + + case LL_PATH_DUMP: + prefix=getDumpDir(); + break; + + case LL_PATH_USER_SETTINGS: + prefix = add(getOSUserAppDir(), "user_settings"); + break; + + case LL_PATH_PER_SL_ACCOUNT: + prefix = getLindenUserDir(); + if (prefix.empty()) + { + // if we're asking for the per-SL-account directory but we haven't + // logged in yet (or otherwise don't know the account name from + // which to build this string), then intentionally return a blank + // string to the caller and skip the below warning about a blank + // prefix. + LL_DEBUGS("LLDir") << "getLindenUserDir() not yet set: " + << ELLPathToString(location) + << ", '" << subdir1 << "', '" << subdir2 << "', '" << in_filename + << "' => ''" << LL_ENDL; + return std::string(); + } + break; + + case LL_PATH_CHAT_LOGS: + prefix = getChatLogsDir(); + break; + + case LL_PATH_PER_ACCOUNT_CHAT_LOGS: + prefix = getPerAccountChatLogsDir(); + if (prefix.empty()) + { + // potentially directory was not set yet + // intentionally return a blank string to the caller + LL_DEBUGS("LLDir") << "Conversation log directory is not yet set" << LL_ENDL; + return std::string(); + } + break; + + case LL_PATH_LOGS: + prefix = add(getOSUserAppDir(), "logs"); + break; + + case LL_PATH_TEMP: + prefix = getTempDir(); + break; + + case LL_PATH_TOP_SKIN: + prefix = getSkinDir(); + break; + + case LL_PATH_DEFAULT_SKIN: + prefix = getDefaultSkinDir(); + break; + + case LL_PATH_USER_SKIN: + prefix = getUserSkinDir(); + break; + + case LL_PATH_SKINS: + prefix = getSkinBaseDir(); + break; + + case LL_PATH_LOCAL_ASSETS: + prefix = add(getAppRODataDir(), "local_assets"); + break; + + case LL_PATH_EXECUTABLE: + prefix = getExecutableDir(); + break; + + case LL_PATH_FONTS: + prefix = add(getAppRODataDir(), "fonts"); + break; + + default: + llassert(0); + } + + if (prefix.empty()) + { + LL_WARNS() << ELLPathToString(location) + << ", '" << subdir1 << "', '" << subdir2 << "', '" << in_filename + << "': prefix is empty, possible bad filename" << LL_ENDL; + } + + std::string expanded_filename = add(prefix, subdir1, subdir2); + if (expanded_filename.empty() && in_filename.empty()) + { + return ""; + } + // Use explicit concatenation here instead of another add() call. Callers + // passing in_filename as "" expect to obtain a pathname ending with + // mDirSeparator so they can later directly concatenate with a specific + // filename. A caller using add() doesn't care, but there's still code + // loose in the system that uses std::string::operator+(). + expanded_filename += mDirDelimiter; + expanded_filename += in_filename; + + LL_DEBUGS("LLDir") << ELLPathToString(location) + << ", '" << subdir1 << "', '" << subdir2 << "', '" << in_filename + << "' => '" << expanded_filename << "'" << LL_ENDL; + return expanded_filename; +} + +std::string LLDir::getBaseFileName(const std::string& filepath, bool strip_exten) const +{ + std::size_t offset = filepath.find_last_of(getDirDelimiter()); + offset = (offset == std::string::npos) ? 0 : offset+1; + std::string res = filepath.substr(offset, std::string::npos); + if (strip_exten) + { + offset = res.find_last_of('.'); + if (offset != std::string::npos && + offset != 0) // if basename STARTS with '.', don't strip + { + res = res.substr(0, offset); + } + } + return res; +} + +std::string LLDir::getDirName(const std::string& filepath) const +{ + std::size_t offset = filepath.find_last_of(getDirDelimiter()); + S32 len = (offset == std::string::npos) ? 0 : offset; + std::string dirname = filepath.substr(0, len); + return dirname; +} + +std::string LLDir::getExtension(const std::string& filepath) const +{ + if (filepath.empty()) + return std::string(); + std::string basename = getBaseFileName(filepath, false); + std::size_t offset = basename.find_last_of('.'); + std::string exten = (offset == std::string::npos || offset == 0) ? "" : basename.substr(offset+1); + LLStringUtil::toLower(exten); + return exten; +} + +std::string LLDir::findSkinnedFilenameBaseLang(const std::string &subdir, + const std::string &filename, + ESkinConstraint constraint) const +{ + // This implementation is basically just as described in the declaration comments. + std::vector<std::string> found(findSkinnedFilenames(subdir, filename, constraint)); + if (found.empty()) + { + return ""; + } + return found.front(); +} + +std::string LLDir::findSkinnedFilename(const std::string &subdir, + const std::string &filename, + ESkinConstraint constraint) const +{ + // This implementation is basically just as described in the declaration comments. + std::vector<std::string> found(findSkinnedFilenames(subdir, filename, constraint)); + if (found.empty()) + { + return ""; + } + return found.back(); +} + +// This method exists because the two code paths for +// findSkinnedFilenames(ALL_SKINS) and findSkinnedFilenames(CURRENT_SKIN) must +// generate the list of candidate pathnames in identical ways. The only +// difference is in the body of the inner loop. +template <typename FUNCTION> +void LLDir::walkSearchSkinDirs(const std::string& subdir, + const std::vector<std::string>& subsubdirs, + const std::string& filename, + const FUNCTION& function) const +{ + BOOST_FOREACH(std::string skindir, mSearchSkinDirs) + { + std::string subdir_path(add(skindir, subdir)); + BOOST_FOREACH(std::string subsubdir, subsubdirs) + { + std::string full_path(add(subdir_path, subsubdir, filename)); + if (fileExists(full_path)) + { + function(subsubdir, full_path); + } + } + } +} + +// ridiculous little helper function that should go away when we can use lambda +inline void push_back(std::vector<std::string>& vector, const std::string& value) +{ + vector.push_back(value); +} + +typedef std::map<std::string, std::string> StringMap; +// ridiculous little helper function that should go away when we can use lambda +inline void store_in_map(StringMap& map, const std::string& key, const std::string& value) +{ + map[key] = value; +} + +std::vector<std::string> LLDir::findSkinnedFilenames(const std::string& subdir, + const std::string& filename, + ESkinConstraint constraint) const +{ + // Recognize subdirs that have no localization. + static const std::set<std::string> sUnlocalized = list_of + ("") // top-level directory not localized + ("textures") // textures not localized + ; + + LL_DEBUGS("LLDir") << "subdir '" << subdir << "', filename '" << filename + << "', constraint " + << ((constraint == CURRENT_SKIN)? "CURRENT_SKIN" : "ALL_SKINS") + << LL_ENDL; + + // Build results vector. + std::vector<std::string> results; + // Disallow filenames that may escape subdir + if (filename.find("..") != std::string::npos) + { + LL_WARNS("LLDir") << "Ignoring potentially relative filename '" << filename << "'" << LL_ENDL; + return results; + } + + // Cache the default language directory for each subdir we've encountered. + // A cache entry whose value is the empty string means "not localized, + // don't bother checking again." + static StringMap sLocalized; + + // Check whether we've already discovered if this subdir is localized. + StringMap::const_iterator found = sLocalized.find(subdir); + if (found == sLocalized.end()) + { + // We have not yet determined that. Is it one of the subdirs "known" + // to be unlocalized? + if (sUnlocalized.find(subdir) != sUnlocalized.end()) + { + // This subdir is known to be unlocalized. Remember that. + found = sLocalized.insert(StringMap::value_type(subdir, "")).first; + } + else + { + // We do not recognize this subdir. Investigate. + std::string subdir_path(add(getDefaultSkinDir(), subdir)); + if (fileExists(add(subdir_path, "en"))) + { + // defaultSkinDir/subdir contains subdir "en". That's our + // default language; this subdir is localized. + found = sLocalized.insert(StringMap::value_type(subdir, "en")).first; + } + else if (fileExists(add(subdir_path, "en-us"))) + { + // defaultSkinDir/subdir contains subdir "en-us" but not "en". + // Set as default language; this subdir is localized. + found = sLocalized.insert(StringMap::value_type(subdir, "en-us")).first; + } + else + { + // defaultSkinDir/subdir contains neither "en" nor "en-us". + // Assume it's not localized. Remember that assumption. + found = sLocalized.insert(StringMap::value_type(subdir, "")).first; + } + } + } + // Every code path above should have resulted in 'found' becoming a valid + // iterator to an entry in sLocalized. + llassert(found != sLocalized.end()); + + // Now -- is this subdir localized, or not? The answer determines what + // subdirectories we check (under subdir) for the requested filename. + std::vector<std::string> subsubdirs; + if (found->second.empty()) + { + // subdir is not localized. filename should be located directly within it. + subsubdirs.push_back(""); + } + else + { + // subdir is localized, and found->second is the default language + // directory within it. Check both the default language and the + // current language -- if it differs from the default, of course. + subsubdirs.push_back(found->second); + if (mLanguage != found->second) + { + subsubdirs.push_back(mLanguage); + } + } + + // The process we use depends on 'constraint'. + if (constraint != CURRENT_SKIN) // meaning ALL_SKINS + { + // ALL_SKINS is simpler: just return every pathname generated by + // walkSearchSkinDirs(). Tricky bit: walkSearchSkinDirs() passes its + // FUNCTION the subsubdir as well as the full pathname. We just want + // the full pathname. + walkSearchSkinDirs(subdir, subsubdirs, filename, + boost::bind(push_back, boost::ref(results), _2)); + } + else // CURRENT_SKIN + { + // CURRENT_SKIN turns out to be a bit of a misnomer because we might + // still return files from two different skins. In any case, this + // value of 'constraint' means we will return at most two paths: one + // for the default language, one for the current language (supposing + // those differ). + // It is important to allow a user to override only the localization + // for a particular file, for all viewer installs, without also + // overriding the default-language file. + // It is important to allow a user to override only the default- + // language file, for all viewer installs, without also overriding the + // applicable localization of that file. + // Therefore, treat the default language and the current language as + // two separate cases. For each, capture the most-specialized file + // that exists. + // Use a map keyed by subsubdir (i.e. language code). This allows us + // to handle the case of a single subsubdirs entry with the same logic + // that handles two. For every real file path generated by + // walkSearchSkinDirs(), update the map entry for its subsubdir. + StringMap path_for; + walkSearchSkinDirs(subdir, subsubdirs, filename, + boost::bind(store_in_map, boost::ref(path_for), _1, _2)); + // Now that we have a path for each of the default language and the + // current language, copy them -- in proper order -- into results. + // Don't drive this by walking the map itself: it matters that we + // generate results in the same order as subsubdirs. + BOOST_FOREACH(std::string subsubdir, subsubdirs) + { + StringMap::const_iterator found(path_for.find(subsubdir)); + if (found != path_for.end()) + { + results.push_back(found->second); + } + } + } + + LL_DEBUGS("LLDir") << empty; + const char* comma = ""; + BOOST_FOREACH(std::string path, results) + { + LL_CONT << comma << "'" << path << "'"; + comma = ", "; + } + LL_CONT << LL_ENDL; + + return results; +} + +std::string LLDir::getTempFilename() const +{ + LLUUID random_uuid; + std::string uuid_str; + + random_uuid.generate(); + random_uuid.toString(uuid_str); + + return add(getTempDir(), uuid_str + ".tmp"); +} + +// static +std::string LLDir::getScrubbedFileName(const std::string uncleanFileName) +{ + std::string name(uncleanFileName); + const std::string illegalChars(getForbiddenFileChars()); + // replace any illegal file chars with and underscore '_' + for( unsigned int i = 0; i < illegalChars.length(); i++ ) + { + int j = -1; + while((j = name.find(illegalChars[i])) > -1) + { + name[j] = '_'; + } + } + return name; +} + +// static +std::string LLDir::getForbiddenFileChars() +{ + return "\\/:*?\"<>|"; +} + +void LLDir::setLindenUserDir(const std::string &username) +{ + // if the username isn't set, that's bad + if (!username.empty()) + { + // some platforms have case-sensitive filesystems, so be + // utterly consistent with our firstname/lastname case. + std::string userlower(username); + LLStringUtil::toLower(userlower); + LLStringUtil::replaceChar(userlower, ' ', '_'); + mLindenUserDir = add(getOSUserAppDir(), userlower); + } + else + { + LL_ERRS() << "NULL name for LLDir::setLindenUserDir" << LL_ENDL; + } + + dumpCurrentDirectories(); +} + +void LLDir::setChatLogsDir(const std::string &path) +{ + if (!path.empty() ) + { + mChatLogsDir = path; + } + else + { + LL_WARNS() << "Invalid name for LLDir::setChatLogsDir" << LL_ENDL; + } +} + +void LLDir::updatePerAccountChatLogsDir() +{ + mPerAccountChatLogsDir = add(getChatLogsDir(), mUserName); +} + +void LLDir::setPerAccountChatLogsDir(const std::string &username) +{ + // if both first and last aren't set, assume we're grabbing the cached dir + if (!username.empty()) + { + // some platforms have case-sensitive filesystems, so be + // utterly consistent with our firstname/lastname case. + std::string userlower(username); + LLStringUtil::toLower(userlower); + LLStringUtil::replaceChar(userlower, ' ', '_'); + + mUserName = userlower; + updatePerAccountChatLogsDir(); + } + else + { + LL_ERRS() << "NULL name for LLDir::setPerAccountChatLogsDir" << LL_ENDL; + } +} + +void LLDir::setSkinFolder(const std::string &skin_folder, const std::string& language) +{ + LL_DEBUGS("LLDir") << "Setting skin '" << skin_folder << "', language '" << language << "'" + << LL_ENDL; + mSkinName = skin_folder; + mLanguage = language; + + // This method is called multiple times during viewer initialization. Each + // time it's called, reset mSearchSkinDirs. + mSearchSkinDirs.clear(); + + // base skin which is used as fallback for all skinned files + // e.g. c:\program files\secondlife\skins\default + mDefaultSkinDir = getSkinBaseDir(); + append(mDefaultSkinDir, "default"); + // This is always the most general of the search skin directories. + addSearchSkinDir(mDefaultSkinDir); + + mSkinDir = getSkinBaseDir(); + append(mSkinDir, skin_folder); + // Next level of generality is a skin installed with the viewer. + addSearchSkinDir(mSkinDir); + + // user modifications to skins, current and default + // e.g. c:\documents and settings\users\username\application data\second life\skins\dazzle + mUserSkinDir = getOSUserAppDir(); + append(mUserSkinDir, "skins"); + mUserDefaultSkinDir = mUserSkinDir; + append(mUserDefaultSkinDir, "default"); + append(mUserSkinDir, skin_folder); + // Next level of generality is user modifications to default skin... + addSearchSkinDir(mUserDefaultSkinDir); + // then user-defined skins. + addSearchSkinDir(mUserSkinDir); +} + +void LLDir::addSearchSkinDir(const std::string& skindir) +{ + if (std::find(mSearchSkinDirs.begin(), mSearchSkinDirs.end(), skindir) == mSearchSkinDirs.end()) + { + LL_DEBUGS("LLDir") << "search skin: '" << skindir << "'" << LL_ENDL; + mSearchSkinDirs.push_back(skindir); + } +} + +std::string LLDir::getSkinFolder() const +{ + return mSkinName; +} + +std::string LLDir::getLanguage() const +{ + return mLanguage; +} + +bool LLDir::setCacheDir(const std::string &path) +{ + if (path.empty() ) + { + // reset to default + mCacheDir = ""; + return true; + } + else + { + LLFile::mkdir(path); + std::string tempname = add(path, "temp"); + LLFILE* file = LLFile::fopen(tempname,"wt"); + if (file) + { + fclose(file); + LLFile::remove(tempname); + mCacheDir = path; + return true; + } + return false; + } +} + +void LLDir::dumpCurrentDirectories(LLError::ELevel level) +{ + LL_VLOGS(level, "AppInit","Directories") << "Current Directories:" << LL_ENDL; + + LL_VLOGS(level, "AppInit", "Directories") << " CurPath: " << getCurPath() << LL_ENDL; + LL_VLOGS(level, "AppInit", "Directories") << " AppName: " << getAppName() << LL_ENDL; + LL_VLOGS(level, "AppInit", "Directories") << " ExecutableFilename: " << getExecutableFilename() << LL_ENDL; + LL_VLOGS(level, "AppInit", "Directories") << " ExecutableDir: " << getExecutableDir() << LL_ENDL; + LL_VLOGS(level, "AppInit", "Directories") << " ExecutablePathAndName: " << getExecutablePathAndName() << LL_ENDL; + LL_VLOGS(level, "AppInit", "Directories") << " WorkingDir: " << getWorkingDir() << LL_ENDL; + LL_VLOGS(level, "AppInit", "Directories") << " AppRODataDir: " << getAppRODataDir() << LL_ENDL; + LL_VLOGS(level, "AppInit", "Directories") << " OSUserDir: " << getOSUserDir() << LL_ENDL; + LL_VLOGS(level, "AppInit", "Directories") << " OSUserAppDir: " << getOSUserAppDir() << LL_ENDL; + LL_VLOGS(level, "AppInit", "Directories") << " LindenUserDir: " << getLindenUserDir() << LL_ENDL; + LL_VLOGS(level, "AppInit", "Directories") << " TempDir: " << getTempDir() << LL_ENDL; + LL_VLOGS(level, "AppInit", "Directories") << " CAFile: " << getCAFile() << LL_ENDL; + LL_VLOGS(level, "AppInit", "Directories") << " SkinBaseDir: " << getSkinBaseDir() << LL_ENDL; + LL_VLOGS(level, "AppInit", "Directories") << " SkinDir: " << getSkinDir() << LL_ENDL; +} + +void LLDir::append(std::string& destpath, const std::string& name) const +{ + // Delegate question of whether we need a separator to helper method. + SepOff sepoff(needSep(destpath, name)); + if (sepoff.first) // do we need a separator? + { + destpath += mDirDelimiter; + } + // If destpath ends with a separator, AND name starts with one, skip + // name's leading separator. + destpath += name.substr(sepoff.second); +} + +LLDir::SepOff LLDir::needSep(const std::string& path, const std::string& name) const +{ + if (path.empty() || name.empty()) + { + // If either path or name are empty, we do not need a separator + // between them. + return SepOff(false, 0); + } + // Here we know path and name are both non-empty. But if path already ends + // with a separator, or if name already starts with a separator, we need + // not add one. + std::string::size_type seplen(mDirDelimiter.length()); + bool path_ends_sep(path.substr(path.length() - seplen) == mDirDelimiter); + bool name_starts_sep(name.substr(0, seplen) == mDirDelimiter); + if ((! path_ends_sep) && (! name_starts_sep)) + { + // If neither path nor name brings a separator to the junction, then + // we need one. + return SepOff(true, 0); + } + if (path_ends_sep && name_starts_sep) + { + // But if BOTH path and name bring a separator, we need not add one. + // Moreover, we should actually skip the leading separator of 'name'. + return SepOff(false, (unsigned short)seplen); + } + // Here we know that either path_ends_sep or name_starts_sep is true -- + // but not both. So don't add a separator, and don't skip any characters: + // simple concatenation will do the trick. + return SepOff(false, 0); +} + +void dir_exists_or_crash(const std::string &dir_name) +{ +#if LL_WINDOWS + // *FIX: lame - it doesn't do the same thing on windows. not so + // important since we don't deploy simulator to windows boxes. + LLFile::mkdir(dir_name, 0700); +#else + struct stat dir_stat; + if(0 != LLFile::stat(dir_name, &dir_stat)) + { + S32 stat_rv = errno; + if(ENOENT == stat_rv) + { + if(0 != LLFile::mkdir(dir_name, 0700)) // octal + { + LL_ERRS() << "Unable to create directory: " << dir_name << LL_ENDL; + } + } + else + { + LL_ERRS() << "Unable to stat: " << dir_name << " errno = " << stat_rv + << LL_ENDL; + } + } + else + { + // data_dir exists, make sure it's a directory. + if(!S_ISDIR(dir_stat.st_mode)) + { + LL_ERRS() << "Data directory collision: " << dir_name << LL_ENDL; + } + } +#endif +} diff --git a/indra/llvfs/lldir.h b/indra/llvfs/lldir.h new file mode 100644 index 0000000000..38e204ef04 --- /dev/null +++ b/indra/llvfs/lldir.h @@ -0,0 +1,280 @@ +/** + * @file lldir.h + * @brief Definition of directory utilities class + * + * $LicenseInfo:firstyear=2000&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2010, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef LL_LLDIR_H +#define LL_LLDIR_H + +#if LL_SOLARIS +#include <sys/param.h> +#define MAX_PATH MAXPATHLEN +#endif + +// these numbers are read from settings_files.xml, so we need to be explicit +typedef enum ELLPath +{ + LL_PATH_NONE = 0, + LL_PATH_USER_SETTINGS = 1, + LL_PATH_APP_SETTINGS = 2, + LL_PATH_PER_SL_ACCOUNT = 3, // returns/expands to blank string if we don't know the account name yet + LL_PATH_CACHE = 4, + LL_PATH_CHARACTER = 5, + LL_PATH_HELP = 6, + LL_PATH_LOGS = 7, + LL_PATH_TEMP = 8, + LL_PATH_SKINS = 9, + LL_PATH_TOP_SKIN = 10, + LL_PATH_CHAT_LOGS = 11, + LL_PATH_PER_ACCOUNT_CHAT_LOGS = 12, + LL_PATH_USER_SKIN = 14, + LL_PATH_LOCAL_ASSETS = 15, + LL_PATH_EXECUTABLE = 16, + LL_PATH_DEFAULT_SKIN = 17, + LL_PATH_FONTS = 18, + LL_PATH_DUMP = 19, + LL_PATH_LAST +} ELLPath; + +/// Directory operations +class LLDir +{ + public: + LLDir(); + virtual ~LLDir(); + + // app_name - Usually SecondLife, used for creating settings directories + // in OS-specific location, such as C:\Documents and Settings + // app_read_only_data_dir - Usually the source code directory, used + // for test applications to read newview data files. + virtual void initAppDirs(const std::string &app_name, + const std::string& app_read_only_data_dir = "") = 0; + + virtual S32 deleteFilesInDir(const std::string &dirname, const std::string &mask); + U32 deleteDirAndContents(const std::string& dir_name); + std::vector<std::string> getFilesInDir(const std::string &dirname); +// pure virtual functions + virtual std::string getCurPath() = 0; + virtual bool fileExists(const std::string &filename) const = 0; + + const std::string findFile(const std::string& filename, const std::vector<std::string> filenames) const; + const std::string findFile(const std::string& filename, const std::string& searchPath1 = "", const std::string& searchPath2 = "", const std::string& searchPath3 = "") const; + + virtual std::string getLLPluginLauncher() = 0; // full path and name for the plugin shell + virtual std::string getLLPluginFilename(std::string base_name) = 0; // full path and name to the plugin DSO for this base_name (i.e. 'FOO' -> '/bar/baz/libFOO.so') + + const std::string &getExecutablePathAndName() const; // Full pathname of the executable + const std::string &getAppName() const; // install directory under progams/ ie "SecondLife" + const std::string &getExecutableDir() const; // Directory where the executable is located + const std::string &getExecutableFilename() const;// Filename of .exe + const std::string &getWorkingDir() const; // Current working directory + const std::string &getAppRODataDir() const; // Location of read-only data files + const std::string &getOSUserDir() const; // Location of the os-specific user dir + const std::string &getOSUserAppDir() const; // Location of the os-specific user app dir + const std::string &getLindenUserDir() const; // Location of the Linden user dir. + const std::string &getChatLogsDir() const; // Location of the chat logs dir. + const std::string &getDumpDir() const; // Location of the per-run dump dir. + const std::string &getPerAccountChatLogsDir() const; // Location of the per account chat logs dir. + const std::string &getTempDir() const; // Common temporary directory + const std::string getCacheDir(bool get_default = false) const; // Location of the cache. + const std::string &getOSCacheDir() const; // location of OS-specific cache folder (may be empty string) + const std::string &getCAFile() const; // File containing TLS certificate authorities + const std::string &getDirDelimiter() const; // directory separator for platform (ie. '\' or '/' or ':') + const std::string &getDefaultSkinDir() const; // folder for default skin. e.g. c:\program files\second life\skins\default + const std::string &getSkinDir() const; // User-specified skin folder. + const std::string &getUserDefaultSkinDir() const; // dir with user modifications to default skin + const std::string &getUserSkinDir() const; // User-specified skin folder with user modifications. e.g. c:\documents and settings\username\application data\second life\skins\curskin + const std::string getSkinBaseDir() const; // folder that contains all installed skins (not user modifications). e.g. c:\program files\second life\skins + const std::string &getLLPluginDir() const; // Directory containing plugins and plugin shell + const std::string &getUserName() const; + + // Expanded filename + std::string getExpandedFilename(ELLPath location, const std::string &filename) const; + std::string getExpandedFilename(ELLPath location, const std::string &subdir, const std::string &filename) const; + std::string getExpandedFilename(ELLPath location, const std::string &subdir1, const std::string &subdir2, const std::string &filename) const; + + // Base and Directory name extraction + std::string getBaseFileName(const std::string& filepath, bool strip_exten = false) const; + std::string getDirName(const std::string& filepath) const; + std::string getExtension(const std::string& filepath) const; // Excludes '.', e.g getExtension("foo.wav") == "wav" + + // these methods search the various skin paths for the specified file in the following order: + // getUserSkinDir(), getUserDefaultSkinDir(), getSkinDir(), getDefaultSkinDir() + /// param value for findSkinnedFilenames(), explained below + enum ESkinConstraint { CURRENT_SKIN, ALL_SKINS }; + /** + * Given a filename within skin, return an ordered sequence of paths to + * search. Nonexistent files will be filtered out -- which means that the + * vector might be empty. + * + * @param subdir Identify top-level skin subdirectory by passing one of + * LLDir::XUI (file lives under "xui" subtree), LLDir::TEXTURES (file + * lives under "textures" subtree), LLDir::SKINBASE (file lives at top + * level of skin subdirectory). + * @param filename Desired filename within subdir within skin, e.g. + * "panel_login.xml". DO NOT prepend (e.g.) "xui" or the desired language. + * @param constraint Callers perform two different kinds of processing. + * When fetching a XUI file, for instance, the existence of @a filename in + * the specified skin completely supercedes any @a filename in the default + * skin. For that case, leave the default @a constraint=CURRENT_SKIN. The + * returned vector will contain only + * ".../<i>current_skin</i>/xui/en/<i>filename</i>", + * ".../<i>current_skin</i>/xui/<i>current_language</i>/<i>filename</i>". + * But for (e.g.) "strings.xml", we want a given skin to be able to + * override only specific entries from the default skin. Any string not + * defined in the specified skin will be sought in the default skin. For + * that case, pass @a constraint=ALL_SKINS. The returned vector will + * contain at least ".../default/xui/en/strings.xml", + * ".../default/xui/<i>current_language</i>/strings.xml", + * ".../<i>current_skin</i>/xui/en/strings.xml", + * ".../<i>current_skin</i>/xui/<i>current_language</i>/strings.xml". + */ + std::vector<std::string> findSkinnedFilenames(const std::string& subdir, + const std::string& filename, + ESkinConstraint constraint=CURRENT_SKIN) const; + /// Values for findSkinnedFilenames(subdir) parameter + static const char *XUI, *TEXTURES, *SKINBASE; + /** + * Return the base-language pathname from findSkinnedFilenames(), or + * the empty string if no such file exists. Parameters are identical to + * findSkinnedFilenames(). This is shorthand for capturing the vector + * returned by findSkinnedFilenames(), checking for empty() and then + * returning front(). + */ + std::string findSkinnedFilenameBaseLang(const std::string &subdir, + const std::string &filename, + ESkinConstraint constraint=CURRENT_SKIN) const; + /** + * Return the "most localized" pathname from findSkinnedFilenames(), or + * the empty string if no such file exists. Parameters are identical to + * findSkinnedFilenames(). This is shorthand for capturing the vector + * returned by findSkinnedFilenames(), checking for empty() and then + * returning back(). + */ + std::string findSkinnedFilename(const std::string &subdir, + const std::string &filename, + ESkinConstraint constraint=CURRENT_SKIN) const; + + // random filename in common temporary directory + std::string getTempFilename() const; + + // For producing safe download file names from potentially unsafe ones + static std::string getScrubbedFileName(const std::string uncleanFileName); + static std::string getForbiddenFileChars(); + void setDumpDir( const std::string& path ); + + + virtual void setChatLogsDir(const std::string &path); // Set the chat logs dir to this user's dir + virtual void setPerAccountChatLogsDir(const std::string &username); // Set the per user chat log directory. + virtual void setLindenUserDir(const std::string &username); // Set the linden user dir to this user's dir + virtual void setSkinFolder(const std::string &skin_folder, const std::string& language); + virtual std::string getSkinFolder() const; + virtual std::string getLanguage() const; + virtual bool setCacheDir(const std::string &path); + virtual void updatePerAccountChatLogsDir(); + + virtual void dumpCurrentDirectories(LLError::ELevel level = LLError::LEVEL_DEBUG); + + // Utility routine + std::string buildSLOSCacheDir() const; + + /// Append specified @a name to @a destpath, separated by getDirDelimiter() + /// if both are non-empty. + void append(std::string& destpath, const std::string& name) const; + /// Variadic form: append @a name0 and @a name1 and arbitrary other @a + /// names to @a destpath, separated by getDirDelimiter() as needed. + template <typename... NAMES> + void append(std::string& destpath, const std::string& name0, const std::string& name1, + const NAMES& ... names) const + { + // In a typical recursion case, we'd accept (destpath, name0, names). + // We accept (destpath, name0, name1, names) because it's important to + // delegate the two-argument case to the non-template implementation. + append(destpath, name0); + append(destpath, name1, names...); + } + + /// Append specified @a names to @a path, separated by getDirDelimiter() + /// as needed. Return result, leaving @a path unmodified. + template <typename... NAMES> + std::string add(const std::string& path, const NAMES& ... names) const + { + std::string destpath(path); + append(destpath, names...); + return destpath; + } + +protected: + // Does an add() or append() call need a directory delimiter? + typedef std::pair<bool, unsigned short> SepOff; + SepOff needSep(const std::string& path, const std::string& name) const; + // build mSearchSkinDirs without adding duplicates + void addSearchSkinDir(const std::string& skindir); + + // Internal to findSkinnedFilenames() + template <typename FUNCTION> + void walkSearchSkinDirs(const std::string& subdir, + const std::vector<std::string>& subsubdirs, + const std::string& filename, + const FUNCTION& function) const; + + std::string mAppName; // install directory under progams/ ie "SecondLife" + std::string mExecutablePathAndName; // full path + Filename of .exe + std::string mExecutableFilename; // Filename of .exe + std::string mExecutableDir; // Location of executable + std::string mWorkingDir; // Current working directory + std::string mAppRODataDir; // Location for static app data + std::string mOSUserDir; // OS Specific user directory + std::string mOSUserAppDir; // OS Specific user app directory + std::string mLindenUserDir; // Location for Linden user-specific data + std::string mPerAccountChatLogsDir; // Location for chat logs. + std::string mChatLogsDir; // Location for chat logs. + std::string mCAFile; // Location of the TLS certificate authority PEM file. + std::string mTempDir; + std::string mCacheDir; // cache directory as set by user preference + std::string mDefaultCacheDir; // default cache diretory + std::string mOSCacheDir; // operating system cache dir + std::string mDirDelimiter; + std::string mSkinName; // caller-specified skin name + std::string mSkinBaseDir; // Base for skins paths. + std::string mDefaultSkinDir; // Location for default skin info. + std::string mSkinDir; // Location for current skin info. + std::string mUserDefaultSkinDir; // Location for default skin info. + std::string mUserSkinDir; // Location for user-modified skin info. + // Skin directories to search, most general to most specific. This order + // works well for composing fine-grained files, in which an individual item + // in a specific file overrides the corresponding item in more general + // files. Of course, for a file-level search, iterate backwards. + std::vector<std::string> mSearchSkinDirs; + std::string mLanguage; // Current viewer language + std::string mLLPluginDir; // Location for plugins and plugin shell + static std::string sDumpDir; // Per-run crash report subdir of log directory. + std::string mUserName; // Current user name +}; + +void dir_exists_or_crash(const std::string &dir_name); + +extern LLDir *gDirUtilp; + +#endif // LL_LLDIR_H diff --git a/indra/llvfs/lldir_linux.cpp b/indra/llvfs/lldir_linux.cpp new file mode 100644 index 0000000000..80ad05345a --- /dev/null +++ b/indra/llvfs/lldir_linux.cpp @@ -0,0 +1,269 @@ +/** + * @file lldir_linux.cpp + * @brief Implementation of directory utilities for linux + * + * $LicenseInfo:firstyear=2002&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2010, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include "linden_common.h" + +#include "lldir_linux.h" +#include "llerror.h" +#include "llrand.h" +#include "llstring.h" +#include <sys/types.h> +#include <sys/stat.h> +#include <unistd.h> +#include <glob.h> +#include <pwd.h> + + +static std::string getCurrentUserHome(char* fallback) +{ + const uid_t uid = getuid(); + struct passwd *pw; + + pw = getpwuid(uid); + if ((pw != NULL) && (pw->pw_dir != NULL)) + { + return pw->pw_dir; + } + + LL_INFOS() << "Couldn't detect home directory from passwd - trying $HOME" << LL_ENDL; + auto home_env = LLStringUtil::getoptenv("HOME"); + if (home_env) + { + return *home_env; + } + else + { + LL_WARNS() << "Couldn't detect home directory! Falling back to " << fallback << LL_ENDL; + return fallback; + } +} + + +LLDir_Linux::LLDir_Linux() +{ + mDirDelimiter = "/"; + mCurrentDirIndex = -1; + mCurrentDirCount = -1; + mDirp = NULL; + + char tmp_str[LL_MAX_PATH]; /* Flawfinder: ignore */ + if (getcwd(tmp_str, LL_MAX_PATH) == NULL) + { + strcpy(tmp_str, "/tmp"); + LL_WARNS() << "Could not get current directory; changing to " + << tmp_str << LL_ENDL; + if (chdir(tmp_str) == -1) + { + LL_ERRS() << "Could not change directory to " << tmp_str << LL_ENDL; + } + } + + mExecutableFilename = ""; + mExecutablePathAndName = ""; + mExecutableDir = tmp_str; + mWorkingDir = tmp_str; +#ifdef APP_RO_DATA_DIR + mAppRODataDir = APP_RO_DATA_DIR; +#else + mAppRODataDir = tmp_str; +#endif + std::string::size_type build_dir_pos = mExecutableDir.rfind("/build-linux-"); + if (build_dir_pos != std::string::npos) + { + // ...we're in a dev checkout + mSkinBaseDir = mExecutableDir.substr(0, build_dir_pos) + "/indra/newview/skins"; + LL_INFOS() << "Running in dev checkout with mSkinBaseDir " + << mSkinBaseDir << LL_ENDL; + } + else + { + // ...normal installation running + mSkinBaseDir = mAppRODataDir + mDirDelimiter + "skins"; + } + + mOSUserDir = getCurrentUserHome(tmp_str); + mOSUserAppDir = ""; + mLindenUserDir = ""; + + char path [32]; /* Flawfinder: ignore */ + + // *NOTE: /proc/%d/exe doesn't work on FreeBSD. But that's ok, + // because this is the linux implementation. + + snprintf (path, sizeof(path), "/proc/%d/exe", (int) getpid ()); + int rc = readlink (path, tmp_str, sizeof (tmp_str)-1); /* Flawfinder: ignore */ + if ( (rc != -1) && (rc <= ((int) sizeof (tmp_str)-1)) ) + { + tmp_str[rc] = '\0'; //readlink() doesn't 0-terminate the buffer + mExecutablePathAndName = tmp_str; + char *path_end; + if ((path_end = strrchr(tmp_str,'/'))) + { + *path_end = '\0'; + mExecutableDir = tmp_str; + mWorkingDir = tmp_str; + mExecutableFilename = path_end+1; + } + else + { + mExecutableFilename = tmp_str; + } + } + + mLLPluginDir = mExecutableDir + mDirDelimiter + "llplugin"; + + // *TODO: don't use /tmp, use $HOME/.secondlife/tmp or something. + mTempDir = "/tmp"; +} + +LLDir_Linux::~LLDir_Linux() +{ +} + +// Implementation + + +void LLDir_Linux::initAppDirs(const std::string &app_name, + const std::string& app_read_only_data_dir) +{ + // Allow override so test apps can read newview directory + if (!app_read_only_data_dir.empty()) + { + mAppRODataDir = app_read_only_data_dir; + mSkinBaseDir = add(mAppRODataDir, "skins"); + } + mAppName = app_name; + + std::string upper_app_name(app_name); + LLStringUtil::toUpper(upper_app_name); + + auto app_home_env(LLStringUtil::getoptenv(upper_app_name + "_USER_DIR")); + if (app_home_env) + { + // user has specified own userappdir i.e. $SECONDLIFE_USER_DIR + mOSUserAppDir = *app_home_env; + } + else + { + // traditionally on unixoids, MyApp gets ~/.myapp dir for data + mOSUserAppDir = mOSUserDir; + mOSUserAppDir += "/"; + mOSUserAppDir += "."; + std::string lower_app_name(app_name); + LLStringUtil::toLower(lower_app_name); + mOSUserAppDir += lower_app_name; + } + + // create any directories we expect to write to. + + int res = LLFile::mkdir(mOSUserAppDir); + if (res == -1) + { + LL_WARNS() << "Couldn't create app user dir " << mOSUserAppDir << LL_ENDL; + LL_WARNS() << "Default to base dir" << mOSUserDir << LL_ENDL; + mOSUserAppDir = mOSUserDir; + } + + res = LLFile::mkdir(getExpandedFilename(LL_PATH_LOGS,"")); + if (res == -1) + { + LL_WARNS() << "Couldn't create LL_PATH_LOGS dir " << getExpandedFilename(LL_PATH_LOGS,"") << LL_ENDL; + } + + res = LLFile::mkdir(getExpandedFilename(LL_PATH_USER_SETTINGS,"")); + if (res == -1) + { + LL_WARNS() << "Couldn't create LL_PATH_USER_SETTINGS dir " << getExpandedFilename(LL_PATH_USER_SETTINGS,"") << LL_ENDL; + } + + res = LLFile::mkdir(getExpandedFilename(LL_PATH_CACHE,"")); + if (res == -1) + { + LL_WARNS() << "Couldn't create LL_PATH_CACHE dir " << getExpandedFilename(LL_PATH_CACHE,"") << LL_ENDL; + } + + mCAFile = getExpandedFilename(LL_PATH_EXECUTABLE, "ca-bundle.crt"); +} + +U32 LLDir_Linux::countFilesInDir(const std::string &dirname, const std::string &mask) +{ + U32 file_count = 0; + glob_t g; + + std::string tmp_str; + tmp_str = dirname; + tmp_str += mask; + + if(glob(tmp_str.c_str(), GLOB_NOSORT, NULL, &g) == 0) + { + file_count = g.gl_pathc; + + globfree(&g); + } + + return (file_count); +} + +std::string LLDir_Linux::getCurPath() +{ + char tmp_str[LL_MAX_PATH]; /* Flawfinder: ignore */ + if (getcwd(tmp_str, LL_MAX_PATH) == NULL) + { + LL_WARNS() << "Could not get current directory" << LL_ENDL; + tmp_str[0] = '\0'; + } + return tmp_str; +} + + +bool LLDir_Linux::fileExists(const std::string &filename) const +{ + struct stat stat_data; + // Check the age of the file + // Now, we see if the files we've gathered are recent... + int res = stat(filename.c_str(), &stat_data); + if (!res) + { + return TRUE; + } + else + { + return FALSE; + } +} + + +/*virtual*/ std::string LLDir_Linux::getLLPluginLauncher() +{ + return gDirUtilp->getExecutableDir() + gDirUtilp->getDirDelimiter() + + "SLPlugin"; +} + +/*virtual*/ std::string LLDir_Linux::getLLPluginFilename(std::string base_name) +{ + return gDirUtilp->getLLPluginDir() + gDirUtilp->getDirDelimiter() + + "lib" + base_name + ".so"; +} diff --git a/indra/llvfs/lldir_linux.h b/indra/llvfs/lldir_linux.h new file mode 100644 index 0000000000..e83a020ba4 --- /dev/null +++ b/indra/llvfs/lldir_linux.h @@ -0,0 +1,64 @@ +/** + * @file lldir_linux.h + * @brief Definition of directory utilities class for linux + * + * $LicenseInfo:firstyear=2000&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2010, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#if !LL_LINUX +#error This header must not be included when compiling for any target other than Linux. Consider including lldir.h instead. +#endif // !LL_LINUX + +#ifndef LL_LLDIR_LINUX_H +#define LL_LLDIR_LINUX_H + +#include "lldir.h" + +#include <dirent.h> +#include <errno.h> + +class LLDir_Linux : public LLDir +{ +public: + LLDir_Linux(); + virtual ~LLDir_Linux(); + + /*virtual*/ void initAppDirs(const std::string &app_name, + const std::string& app_read_only_data_dir); + + virtual std::string getCurPath(); + virtual U32 countFilesInDir(const std::string &dirname, const std::string &mask); + /*virtual*/ bool fileExists(const std::string &filename) const; + + /*virtual*/ std::string getLLPluginLauncher(); + /*virtual*/ std::string getLLPluginFilename(std::string base_name); + +private: + DIR *mDirp; + int mCurrentDirIndex; + int mCurrentDirCount; + std::string mCurrentDir; +}; + +#endif // LL_LLDIR_LINUX_H + + diff --git a/indra/llvfs/lldir_mac.cpp b/indra/llvfs/lldir_mac.cpp new file mode 100644 index 0000000000..87dc1b9795 --- /dev/null +++ b/indra/llvfs/lldir_mac.cpp @@ -0,0 +1,205 @@ +/** + * @file lldir_mac.cpp + * @brief Implementation of directory utilities for Mac OS X + * + * $LicenseInfo:firstyear=2002&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2010, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#if LL_DARWIN + +#include "linden_common.h" + +#include "lldir_mac.h" +#include "llerror.h" +#include "llrand.h" +#include <sys/types.h> +#include <sys/stat.h> +#include <unistd.h> +#include <glob.h> +#include <boost/filesystem.hpp> +#include "llvfs_objc.h" + +// -------------------------------------------------------------------------------- + +static bool CreateDirectory(const std::string &parent, + const std::string &child, + std::string *fullname) +{ + + boost::filesystem::path p(parent); + p /= child; + + if (fullname) + *fullname = std::string(p.string()); + + if (! boost::filesystem::create_directory(p)) + { + return (boost::filesystem::is_directory(p)); + } + return true; +} + +// -------------------------------------------------------------------------------- + +LLDir_Mac::LLDir_Mac() +{ + mDirDelimiter = "/"; + + const std::string secondLifeString = "SecondLife"; + + std::string *executablepathstr = getSystemExecutableFolder(); + + //NOTE: LLINFOS/LLERRS will not output to log here. The streams are not initialized. + + if (executablepathstr) + { + // mExecutablePathAndName + mExecutablePathAndName = *executablepathstr; + + boost::filesystem::path executablepath(*executablepathstr); + +# ifndef BOOST_SYSTEM_NO_DEPRECATED +#endif + mExecutableFilename = executablepath.filename().string(); + mExecutableDir = executablepath.parent_path().string(); + + // mAppRODataDir + std::string *resourcepath = getSystemResourceFolder(); + mAppRODataDir = *resourcepath; + + // *NOTE: When running in a dev tree, use the copy of + // skins in indra/newview/ rather than in the application bundle. This + // mirrors Windows dev environment behavior and allows direct checkin + // of edited skins/xui files. JC + + // MBW -- This keeps the mac application from finding other things. + // If this is really for skins, it should JUST apply to skins. + + std::string::size_type build_dir_pos = mExecutableDir.rfind("/build-darwin-"); + if (build_dir_pos != std::string::npos) + { + // ...we're in a dev checkout + mSkinBaseDir = mExecutableDir.substr(0, build_dir_pos) + + "/indra/newview/skins"; + LL_INFOS() << "Running in dev checkout with mSkinBaseDir " + << mSkinBaseDir << LL_ENDL; + } + else + { + // ...normal installation running + mSkinBaseDir = mAppRODataDir + mDirDelimiter + "skins"; + } + + // mOSUserDir + std::string *appdir = getSystemApplicationSupportFolder(); + std::string rootdir; + + //Create root directory + if (CreateDirectory(*appdir, secondLifeString, &rootdir)) + { + + // Save the full path to the folder + mOSUserDir = rootdir; + + // Create our sub-dirs + CreateDirectory(rootdir, std::string("data"), NULL); + CreateDirectory(rootdir, std::string("logs"), NULL); + CreateDirectory(rootdir, std::string("user_settings"), NULL); + CreateDirectory(rootdir, std::string("browser_profile"), NULL); + } + + //mOSCacheDir + std::string *cachedir = getSystemCacheFolder(); + + if (cachedir) + + { + mOSCacheDir = *cachedir; + //TODO: This changes from ~/Library/Cache/Secondlife to ~/Library/Cache/com.app.secondlife/Secondlife. Last dir level could go away. + CreateDirectory(mOSCacheDir, secondLifeString, NULL); + } + + // mOSUserAppDir + mOSUserAppDir = mOSUserDir; + + // mTempDir + //Aura 120920 boost::filesystem::temp_directory_path() not yet implemented on mac. :( + std::string *tmpdir = getSystemTempFolder(); + if (tmpdir) + { + + CreateDirectory(*tmpdir, secondLifeString, &mTempDir); + if (tmpdir) delete tmpdir; + } + + mWorkingDir = getCurPath(); + + mLLPluginDir = mAppRODataDir + mDirDelimiter + "llplugin"; + } +} + +LLDir_Mac::~LLDir_Mac() +{ +} + +// Implementation + + +void LLDir_Mac::initAppDirs(const std::string &app_name, + const std::string& app_read_only_data_dir) +{ + // Allow override so test apps can read newview directory + if (!app_read_only_data_dir.empty()) + { + mAppRODataDir = app_read_only_data_dir; + mSkinBaseDir = add(mAppRODataDir, "skins"); + } + mCAFile = add(mAppRODataDir, "ca-bundle.crt"); +} + +std::string LLDir_Mac::getCurPath() +{ + return boost::filesystem::path( boost::filesystem::current_path() ).string(); +} + + + +bool LLDir_Mac::fileExists(const std::string &filename) const +{ + return boost::filesystem::exists(filename); +} + + +/*virtual*/ std::string LLDir_Mac::getLLPluginLauncher() +{ + return gDirUtilp->getAppRODataDir() + gDirUtilp->getDirDelimiter() + + "SLPlugin.app/Contents/MacOS/SLPlugin"; +} + +/*virtual*/ std::string LLDir_Mac::getLLPluginFilename(std::string base_name) +{ + return gDirUtilp->getLLPluginDir() + gDirUtilp->getDirDelimiter() + + base_name + ".dylib"; +} + + +#endif // LL_DARWIN diff --git a/indra/llvfs/lldir_mac.h b/indra/llvfs/lldir_mac.h new file mode 100644 index 0000000000..558727ebbc --- /dev/null +++ b/indra/llvfs/lldir_mac.h @@ -0,0 +1,56 @@ +/** + * @file lldir_mac.h + * @brief Definition of directory utilities class for Mac OS X + * + * $LicenseInfo:firstyear=2000&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2010, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#if !LL_DARWIN +#error This header must not be included when compiling for any target other than Mac OS. Consider including lldir.h instead. +#endif // !LL_DARWIN + +#ifndef LL_LLDIR_MAC_H +#define LL_LLDIR_MAC_H + +#include "lldir.h" + +#include <dirent.h> + +class LLDir_Mac : public LLDir +{ +public: + LLDir_Mac(); + virtual ~LLDir_Mac(); + + /*virtual*/ void initAppDirs(const std::string &app_name, + const std::string& app_read_only_data_dir); + + virtual std::string getCurPath(); + virtual bool fileExists(const std::string &filename) const; + + /*virtual*/ std::string getLLPluginLauncher(); + /*virtual*/ std::string getLLPluginFilename(std::string base_name); +}; + +#endif // LL_LLDIR_MAC_H + + diff --git a/indra/llvfs/lldir_solaris.cpp b/indra/llvfs/lldir_solaris.cpp new file mode 100644 index 0000000000..f18560ff20 --- /dev/null +++ b/indra/llvfs/lldir_solaris.cpp @@ -0,0 +1,266 @@ +/** + * @file fmodwrapper.cpp + * @brief dummy source file for building a shared library to wrap libfmod.a + * + * $LicenseInfo:firstyear=2005&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2010, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include "linden_common.h" + +#include "lldir_solaris.h" +#include "llerror.h" +#include "llrand.h" +#include "llstring.h" +#include <sys/types.h> +#include <sys/stat.h> +#include <unistd.h> +#include <glob.h> +#include <pwd.h> +#include <sys/utsname.h> +#define _STRUCTURED_PROC 1 +#include <sys/procfs.h> +#include <fcntl.h> + +static std::string getCurrentUserHome(char* fallback) +{ + // fwiw this exactly duplicates getCurrentUserHome() in lldir_linux.cpp... + // we should either derive both from LLDir_Posix or just axe Solaris. + const uid_t uid = getuid(); + struct passwd *pw; + + pw = getpwuid(uid); + if ((pw != NULL) && (pw->pw_dir != NULL)) + { + return pw->pw_dir; + } + + LL_INFOS() << "Couldn't detect home directory from passwd - trying $HOME" << LL_ENDL; + auto home_env = LLStringUtil::getoptenv("HOME"); + if (home_env) + { + return *home_env; + } + else + { + LL_WARNS() << "Couldn't detect home directory! Falling back to " << fallback << LL_ENDL; + return fallback; + } +} + + +LLDir_Solaris::LLDir_Solaris() +{ + mDirDelimiter = "/"; + mCurrentDirIndex = -1; + mCurrentDirCount = -1; + mDirp = NULL; + + char tmp_str[LL_MAX_PATH]; /* Flawfinder: ignore */ + if (getcwd(tmp_str, LL_MAX_PATH) == NULL) + { + strcpy(tmp_str, "/tmp"); + LL_WARNS() << "Could not get current directory; changing to " + << tmp_str << LL_ENDL; + if (chdir(tmp_str) == -1) + { + LL_ERRS() << "Could not change directory to " << tmp_str << LL_ENDL; + } + } + + mExecutableFilename = ""; + mExecutablePathAndName = ""; + mExecutableDir = strdup(tmp_str); + mWorkingDir = strdup(tmp_str); + mAppRODataDir = strdup(tmp_str); + mOSUserDir = getCurrentUserHome(tmp_str); + mOSUserAppDir = ""; + mLindenUserDir = ""; + + char path [LL_MAX_PATH]; /* Flawfinder: ignore */ + + sprintf(path, "/proc/%d/psinfo", (int)getpid()); + int proc_fd = -1; + if((proc_fd = open(path, O_RDONLY)) == -1){ + LL_WARNS() << "unable to open " << path << LL_ENDL; + return; + } + psinfo_t proc_psinfo; + if(read(proc_fd, &proc_psinfo, sizeof(psinfo_t)) != sizeof(psinfo_t)){ + LL_WARNS() << "Unable to read " << path << LL_ENDL; + close(proc_fd); + return; + } + + close(proc_fd); + + mExecutableFilename = strdup(proc_psinfo.pr_fname); + LL_INFOS() << "mExecutableFilename = [" << mExecutableFilename << "]" << LL_ENDL; + + sprintf(path, "/proc/%d/path/a.out", (int)getpid()); + + char execpath[LL_MAX_PATH]; + if(readlink(path, execpath, LL_MAX_PATH) == -1){ + LL_WARNS() << "Unable to read link from " << path << LL_ENDL; + return; + } + + char *p = execpath; // nuke trash in link, if any exists + int i = 0; + while(*p != NULL && ++i < LL_MAX_PATH && isprint((int)(*p++))); + *p = NULL; + + mExecutablePathAndName = strdup(execpath); + LL_INFOS() << "mExecutablePathAndName = [" << mExecutablePathAndName << "]" << LL_ENDL; + + //NOTE: Why force people to cd into the package directory? + // Look for SECONDLIFE env variable and use it, if set. + + auto SECONDLIFE(LLDirUtil::getoptenv("SECONDLIFE")); + if(SECONDLIFE){ + mExecutableDir = add(*SECONDLIFE, "bin"); //NOTE: make sure we point at the bin + }else{ + mExecutableDir = getDirName(execpath); + LL_INFOS() << "mExecutableDir = [" << mExecutableDir << "]" << LL_ENDL; + } + + mLLPluginDir = add(mExecutableDir, "llplugin"); + + // *TODO: don't use /tmp, use $HOME/.secondlife/tmp or something. + mTempDir = "/tmp"; +} + +LLDir_Solaris::~LLDir_Solaris() +{ +} + +// Implementation + + +void LLDir_Solaris::initAppDirs(const std::string &app_name, + const std::string& app_read_only_data_dir) +{ + // Allow override so test apps can read newview directory + if (!app_read_only_data_dir.empty()) + { + mAppRODataDir = app_read_only_data_dir; + mSkinBaseDir = add(mAppRODataDir, "skins"); + } + mAppName = app_name; + + std::string upper_app_name(app_name); + LLStringUtil::toUpper(upper_app_name); + + auto app_home_env(LLStringUtil::getoptenv(upper_app_name + "_USER_DIR")); + if (app_home_env) + { + // user has specified own userappdir i.e. $SECONDLIFE_USER_DIR + mOSUserAppDir = *app_home_env; + } + else + { + // traditionally on unixoids, MyApp gets ~/.myapp dir for data + mOSUserAppDir = mOSUserDir; + mOSUserAppDir += "/"; + mOSUserAppDir += "."; + std::string lower_app_name(app_name); + LLStringUtil::toLower(lower_app_name); + mOSUserAppDir += lower_app_name; + } + + // create any directories we expect to write to. + + int res = LLFile::mkdir(mOSUserAppDir); + if (res == -1) + { + LL_WARNS() << "Couldn't create app user dir " << mOSUserAppDir << LL_ENDL; + LL_WARNS() << "Default to base dir" << mOSUserDir << LL_ENDL; + mOSUserAppDir = mOSUserDir; + } + + res = LLFile::mkdir(getExpandedFilename(LL_PATH_LOGS,"")); + if (res == -1) + { + LL_WARNS() << "Couldn't create LL_PATH_LOGS dir " << getExpandedFilename(LL_PATH_LOGS,"") << LL_ENDL; + } + + res = LLFile::mkdir(getExpandedFilename(LL_PATH_USER_SETTINGS,"")); + if (res == -1) + { + LL_WARNS() << "Couldn't create LL_PATH_USER_SETTINGS dir " << getExpandedFilename(LL_PATH_USER_SETTINGS,"") << LL_ENDL; + } + + res = LLFile::mkdir(getExpandedFilename(LL_PATH_CACHE,"")); + if (res == -1) + { + LL_WARNS() << "Couldn't create LL_PATH_CACHE dir " << getExpandedFilename(LL_PATH_CACHE,"") << LL_ENDL; + } + + mCAFile = getExpandedFilename(LL_PATH_EXECUTABLE, "ca-bundle.crt"); +} + +U32 LLDir_Solaris::countFilesInDir(const std::string &dirname, const std::string &mask) +{ + U32 file_count = 0; + glob_t g; + + std::string tmp_str; + tmp_str = dirname; + tmp_str += mask; + + if(glob(tmp_str.c_str(), GLOB_NOSORT, NULL, &g) == 0) + { + file_count = g.gl_pathc; + + globfree(&g); + } + + return (file_count); +} + +std::string LLDir_Solaris::getCurPath() +{ + char tmp_str[LL_MAX_PATH]; /* Flawfinder: ignore */ + if (getcwd(tmp_str, LL_MAX_PATH) == NULL) + { + LL_WARNS() << "Could not get current directory" << LL_ENDL; + tmp_str[0] = '\0'; + } + return tmp_str; +} + + +bool LLDir_Solaris::fileExists(const std::string &filename) const +{ + struct stat stat_data; + // Check the age of the file + // Now, we see if the files we've gathered are recent... + int res = stat(filename.c_str(), &stat_data); + if (!res) + { + return TRUE; + } + else + { + return FALSE; + } +} + diff --git a/indra/llvfs/lldir_solaris.h b/indra/llvfs/lldir_solaris.h new file mode 100644 index 0000000000..c6dac57e14 --- /dev/null +++ b/indra/llvfs/lldir_solaris.h @@ -0,0 +1,61 @@ +/** + * @file fmodwrapper.cpp + * @brief dummy source file for building a shared library to wrap libfmod.a + * + * $LicenseInfo:firstyear=2005&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2010, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#if !LL_SOLARIS +#error This header must not be included when compiling for any target other than Solaris. Consider including lldir.h instead. +#endif // !LL_SOLARIS + +#ifndef LL_LLDIR_SOLARIS_H +#define LL_LLDIR_SOLARIS_H + +#include "lldir.h" + +#include <dirent.h> +#include <errno.h> + +class LLDir_Solaris : public LLDir +{ +public: + LLDir_Solaris(); + virtual ~LLDir_Solaris(); + + /*virtual*/ void initAppDirs(const std::string &app_name, + const std::string& app_read_only_data_dir); + + virtual std::string getCurPath(); + virtual U32 countFilesInDir(const std::string &dirname, const std::string &mask); + /*virtual*/ bool fileExists(const std::string &filename) const; + +private: + DIR *mDirp; + int mCurrentDirIndex; + int mCurrentDirCount; + std::string mCurrentDir; +}; + +#endif // LL_LLDIR_SOLARIS_H + + diff --git a/indra/llvfs/lldir_win32.cpp b/indra/llvfs/lldir_win32.cpp new file mode 100644 index 0000000000..b3b3afb37e --- /dev/null +++ b/indra/llvfs/lldir_win32.cpp @@ -0,0 +1,452 @@ +/** + * @file lldir_win32.cpp + * @brief Implementation of directory utilities for windows + * + * $LicenseInfo:firstyear=2002&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2010, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#if LL_WINDOWS + +#include "linden_common.h" + +#include "lldir_win32.h" +#include "llerror.h" +#include "llstring.h" +#include "stringize.h" +#include "llfile.h" +#include <shlobj.h> +#include <fstream> + +#include <direct.h> +#include <errno.h> +#include <sys/types.h> +#include <sys/stat.h> + +// Utility stuff to get versions of the sh +#define PACKVERSION(major,minor) MAKELONG(minor,major) +DWORD GetDllVersion(LPCTSTR lpszDllName); + +namespace +{ // anonymous + enum class prst { INIT, OPEN, SKIP } state = prst::INIT; + // This is called so early that we can't count on static objects being + // properly constructed yet, so declare a pointer instead of an instance. + std::ofstream* prelogf = nullptr; + + void prelog(const std::string& message) + { + boost::optional<std::string> prelog_name; + + switch (state) + { + case prst::INIT: + // assume we failed, until we succeed + state = prst::SKIP; + + prelog_name = LLStringUtil::getoptenv("PRELOG"); + if (! prelog_name) + // no PRELOG variable set, carry on + return; + prelogf = new llofstream(*prelog_name, std::ios_base::app); + if (! (prelogf && prelogf->is_open())) + // can't complain to anybody; how? + return; + // got the log file open, cool! + state = prst::OPEN; + (*prelogf) << "========================================================================" + << std::endl; + // fall through, don't break + + case prst::OPEN: + (*prelogf) << message << std::endl; + break; + + case prst::SKIP: + // either PRELOG isn't set, or we failed to open that pathname + break; + } + } +} // anonymous namespace + +#define PRELOG(expression) prelog(STRINGIZE(expression)) + +LLDir_Win32::LLDir_Win32() +{ + // set this first: used by append() and add() methods + mDirDelimiter = "\\"; + + WCHAR w_str[MAX_PATH]; + // Application Data is where user settings go. We rely on $APPDATA being + // correct. + auto APPDATA = LLStringUtil::getoptenv("APPDATA"); + if (APPDATA) + { + mOSUserDir = *APPDATA; + } + PRELOG("APPDATA='" << mOSUserDir << "'"); + // On Windows, we could have received a plain-ASCII pathname in which + // non-ASCII characters have been munged to '?', or the pathname could + // have been badly encoded and decoded such that we now have garbage + // instead of a valid path. Check that mOSUserDir actually exists. + if (mOSUserDir.empty() || ! fileExists(mOSUserDir)) + { + PRELOG("APPDATA does not exist"); + //HRESULT okay = SHGetFolderPath(NULL, CSIDL_APPDATA, NULL, 0, w_str); + wchar_t *pwstr = NULL; + HRESULT okay = SHGetKnownFolderPath(FOLDERID_RoamingAppData, 0, NULL, &pwstr); + PRELOG("SHGetKnownFolderPath(FOLDERID_RoamingAppData) returned " << okay); + if (SUCCEEDED(okay) && pwstr) + { + // But of course, only update mOSUserDir if SHGetKnownFolderPath() works. + mOSUserDir = ll_convert_wide_to_string(pwstr); + // Not only that: update our environment so that child processes + // will see a reasonable value as well. + _wputenv_s(L"APPDATA", pwstr); + // SHGetKnownFolderPath() contract requires us to free pwstr + CoTaskMemFree(pwstr); + PRELOG("mOSUserDir='" << mOSUserDir << "'"); + } + } + + // We want cache files to go on the local disk, even if the + // user is on a network with a "roaming profile". + // + // On Vista this is: + // C:\Users\James\AppData\Local + // + // We used to store the cache in AppData\Roaming, and the installer + // cleans up that version on upgrade. JC + auto LOCALAPPDATA = LLStringUtil::getoptenv("LOCALAPPDATA"); + if (LOCALAPPDATA) + { + mOSCacheDir = *LOCALAPPDATA; + } + PRELOG("LOCALAPPDATA='" << mOSCacheDir << "'"); + // Windows really does not deal well with pathnames containing non-ASCII + // characters. See above remarks about APPDATA. + if (mOSCacheDir.empty() || ! fileExists(mOSCacheDir)) + { + PRELOG("LOCALAPPDATA does not exist"); + //HRESULT okay = SHGetFolderPath(NULL, CSIDL_LOCAL_APPDATA, NULL, 0, w_str); + wchar_t *pwstr = NULL; + HRESULT okay = SHGetKnownFolderPath(FOLDERID_LocalAppData, 0, NULL, &pwstr); + PRELOG("SHGetKnownFolderPath(FOLDERID_LocalAppData) returned " << okay); + if (SUCCEEDED(okay) && pwstr) + { + // But of course, only update mOSCacheDir if SHGetKnownFolderPath() works. + mOSCacheDir = ll_convert_wide_to_string(pwstr); + // Update our environment so that child processes will see a + // reasonable value as well. + _wputenv_s(L"LOCALAPPDATA", pwstr); + // SHGetKnownFolderPath() contract requires us to free pwstr + CoTaskMemFree(pwstr); + PRELOG("mOSCacheDir='" << mOSCacheDir << "'"); + } + } + + if (GetTempPath(MAX_PATH, w_str)) + { + if (wcslen(w_str)) /* Flawfinder: ignore */ + { + w_str[wcslen(w_str)-1] = '\0'; /* Flawfinder: ignore */ // remove trailing slash + } + mTempDir = utf16str_to_utf8str(llutf16string(w_str)); + + if (mOSUserDir.empty()) + { + mOSUserDir = mTempDir; + } + + if (mOSCacheDir.empty()) + { + mOSCacheDir = mTempDir; + } + } + else + { + mTempDir = mOSUserDir; + } + +/*==========================================================================*| + // Now that we've got mOSUserDir, one way or another, let's see how we did + // with our environment variables. + { + auto report = [this](std::ostream& out){ + out << "mOSUserDir = '" << mOSUserDir << "'\n" + << "mOSCacheDir = '" << mOSCacheDir << "'\n" + << "mTempDir = '" << mTempDir << "'" << std::endl; + }; + int res = LLFile::mkdir(mOSUserDir); + if (res == -1) + { + // If we couldn't even create the directory, just blurt to stderr + report(std::cerr); + } + else + { + // successfully created logdir, plunk a log file there + std::string logfilename(add(mOSUserDir, "lldir.log")); + std::ofstream logfile(logfilename.c_str()); + if (! logfile.is_open()) + { + report(std::cerr); + } + else + { + report(logfile); + } + } + } +|*==========================================================================*/ + +// fprintf(stderr, "mTempDir = <%s>",mTempDir); + + // Set working directory, for LLDir::getWorkingDir() + GetCurrentDirectory(MAX_PATH, w_str); + mWorkingDir = utf16str_to_utf8str(llutf16string(w_str)); + + // Set the executable directory + S32 size = GetModuleFileName(NULL, w_str, MAX_PATH); + if (size) + { + w_str[size] = '\0'; + mExecutablePathAndName = utf16str_to_utf8str(llutf16string(w_str)); + S32 path_end = mExecutablePathAndName.find_last_of('\\'); + if (path_end != std::string::npos) + { + mExecutableDir = mExecutablePathAndName.substr(0, path_end); + mExecutableFilename = mExecutablePathAndName.substr(path_end+1, std::string::npos); + } + else + { + mExecutableFilename = mExecutablePathAndName; + } + + } + else + { + fprintf(stderr, "Couldn't get APP path, assuming current directory!"); + mExecutableDir = mWorkingDir; + // Assume it's the current directory + } + + // mAppRODataDir = "."; + + // Determine the location of the App-Read-Only-Data + // Try the working directory then the exe's dir. + mAppRODataDir = mWorkingDir; + + +// if (mExecutableDir.find("indra") == std::string::npos) + + // *NOTE:Mani - It is a mistake to put viewer specific code in + // the LLDir implementation. The references to 'skins' and + // 'llplugin' need to go somewhere else. + // alas... this also gets called during static initialization + // time due to the construction of gDirUtil in lldir.cpp. + if(! LLFile::isdir(add(mAppRODataDir, "skins"))) + { + // What? No skins in the working dir? + // Try the executable's directory. + mAppRODataDir = mExecutableDir; + } + +// LL_INFOS() << "mAppRODataDir = " << mAppRODataDir << LL_ENDL; + + mSkinBaseDir = add(mAppRODataDir, "skins"); + + // Build the default cache directory + mDefaultCacheDir = buildSLOSCacheDir(); + + // Make sure it exists + int res = LLFile::mkdir(mDefaultCacheDir); + if (res == -1) + { + LL_WARNS() << "Couldn't create LL_PATH_CACHE dir " << mDefaultCacheDir << LL_ENDL; + } + + mLLPluginDir = add(mExecutableDir, "llplugin"); +} + +LLDir_Win32::~LLDir_Win32() +{ +} + +// Implementation + +void LLDir_Win32::initAppDirs(const std::string &app_name, + const std::string& app_read_only_data_dir) +{ + // Allow override so test apps can read newview directory + if (!app_read_only_data_dir.empty()) + { + mAppRODataDir = app_read_only_data_dir; + mSkinBaseDir = add(mAppRODataDir, "skins"); + } + mAppName = app_name; + mOSUserAppDir = add(mOSUserDir, app_name); + + int res = LLFile::mkdir(mOSUserAppDir); + if (res == -1) + { + LL_WARNS() << "Couldn't create app user dir " << mOSUserAppDir << LL_ENDL; + LL_WARNS() << "Default to base dir" << mOSUserDir << LL_ENDL; + mOSUserAppDir = mOSUserDir; + } + //dumpCurrentDirectories(); + + res = LLFile::mkdir(getExpandedFilename(LL_PATH_LOGS,"")); + if (res == -1) + { + LL_WARNS() << "Couldn't create LL_PATH_LOGS dir " << getExpandedFilename(LL_PATH_LOGS,"") << LL_ENDL; + } + + res = LLFile::mkdir(getExpandedFilename(LL_PATH_USER_SETTINGS,"")); + if (res == -1) + { + LL_WARNS() << "Couldn't create LL_PATH_USER_SETTINGS dir " << getExpandedFilename(LL_PATH_USER_SETTINGS,"") << LL_ENDL; + } + + res = LLFile::mkdir(getExpandedFilename(LL_PATH_CACHE,"")); + if (res == -1) + { + LL_WARNS() << "Couldn't create LL_PATH_CACHE dir " << getExpandedFilename(LL_PATH_CACHE,"") << LL_ENDL; + } + + mCAFile = getExpandedFilename( LL_PATH_EXECUTABLE, "ca-bundle.crt" ); +} + +U32 LLDir_Win32::countFilesInDir(const std::string &dirname, const std::string &mask) +{ + HANDLE count_search_h; + U32 file_count; + + file_count = 0; + + WIN32_FIND_DATA FileData; + + llutf16string pathname = utf8str_to_utf16str(dirname); + pathname += utf8str_to_utf16str(mask); + + if ((count_search_h = FindFirstFile(pathname.c_str(), &FileData)) != INVALID_HANDLE_VALUE) + { + file_count++; + + while (FindNextFile(count_search_h, &FileData)) + { + file_count++; + } + + FindClose(count_search_h); + } + + return (file_count); +} + +std::string LLDir_Win32::getCurPath() +{ + WCHAR w_str[MAX_PATH]; + GetCurrentDirectory(MAX_PATH, w_str); + + return utf16str_to_utf8str(llutf16string(w_str)); +} + + +bool LLDir_Win32::fileExists(const std::string &filename) const +{ + llstat stat_data; + // Check the age of the file + // Now, we see if the files we've gathered are recent... + int res = LLFile::stat(filename, &stat_data); + if (!res) + { + return TRUE; + } + else + { + return FALSE; + } +} + + +/*virtual*/ std::string LLDir_Win32::getLLPluginLauncher() +{ + return gDirUtilp->getExecutableDir() + gDirUtilp->getDirDelimiter() + + "SLPlugin.exe"; +} + +/*virtual*/ std::string LLDir_Win32::getLLPluginFilename(std::string base_name) +{ + return gDirUtilp->getLLPluginDir() + gDirUtilp->getDirDelimiter() + + base_name + ".dll"; +} + + +#if 0 +// Utility function to get version number of a DLL + +#define PACKVERSION(major,minor) MAKELONG(minor,major) + +DWORD GetDllVersion(LPCTSTR lpszDllName) +{ + + HINSTANCE hinstDll; + DWORD dwVersion = 0; + + hinstDll = LoadLibrary(lpszDllName); /* Flawfinder: ignore */ + + if(hinstDll) + { + DLLGETVERSIONPROC pDllGetVersion; + + pDllGetVersion = (DLLGETVERSIONPROC) GetProcAddress(hinstDll, "DllGetVersion"); + +/*Because some DLLs might not implement this function, you + must test for it explicitly. Depending on the particular + DLL, the lack of a DllGetVersion function can be a useful + indicator of the version. +*/ + if(pDllGetVersion) + { + DLLVERSIONINFO dvi; + HRESULT hr; + + ZeroMemory(&dvi, sizeof(dvi)); + dvi.cbSize = sizeof(dvi); + + hr = (*pDllGetVersion)(&dvi); + + if(SUCCEEDED(hr)) + { + dwVersion = PACKVERSION(dvi.dwMajorVersion, dvi.dwMinorVersion); + } + } + + FreeLibrary(hinstDll); + } + return dwVersion; +} +#endif + +#endif + + diff --git a/indra/llvfs/lldir_win32.h b/indra/llvfs/lldir_win32.h new file mode 100644 index 0000000000..450efaf9da --- /dev/null +++ b/indra/llvfs/lldir_win32.h @@ -0,0 +1,59 @@ +/** + * @file lldir_win32.h + * @brief Definition of directory utilities class for windows + * + * $LicenseInfo:firstyear=2000&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2010, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#if !LL_WINDOWS +#error This header must not be included when compiling for any target other than Windows. Consider including lldir.h instead. +#endif // !LL_WINDOWS + +#ifndef LL_LLDIR_WIN32_H +#define LL_LLDIR_WIN32_H + +#include "lldir.h" + +class LLDir_Win32 : public LLDir +{ +public: + LLDir_Win32(); + virtual ~LLDir_Win32(); + + /*virtual*/ void initAppDirs(const std::string &app_name, + const std::string& app_read_only_data_dir); + + /*virtual*/ std::string getCurPath(); + /*virtual*/ U32 countFilesInDir(const std::string &dirname, const std::string &mask); + /*virtual*/ bool fileExists(const std::string &filename) const; + + /*virtual*/ std::string getLLPluginLauncher(); + /*virtual*/ std::string getLLPluginFilename(std::string base_name); + +private: + void* mDirSearch_h; + llutf16string mCurrentDir; +}; + +#endif // LL_LLDIR_WIN32_H + + diff --git a/indra/llvfs/lldirguard.h b/indra/llvfs/lldirguard.h new file mode 100644 index 0000000000..37b9e9b83e --- /dev/null +++ b/indra/llvfs/lldirguard.h @@ -0,0 +1,72 @@ +/** + * @file lldirguard.h + * @brief Protect working directory from being changed in scope. + * + * $LicenseInfo:firstyear=2009&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2010, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef LL_DIRGUARD_H +#define LL_DIRGUARD_H + +#include "linden_common.h" +#include "llerror.h" + +#if LL_WINDOWS +class LLDirectoryGuard +{ +public: + LLDirectoryGuard() + { + mOrigDirLen = GetCurrentDirectory(MAX_PATH, mOrigDir); + } + + ~LLDirectoryGuard() + { + mFinalDirLen = GetCurrentDirectory(MAX_PATH, mFinalDir); + if ((mOrigDirLen!=mFinalDirLen) || + (wcsncmp(mOrigDir,mFinalDir,mOrigDirLen)!=0)) + { + // Dir has changed + std::string mOrigDirUtf8 = utf16str_to_utf8str(llutf16string(mOrigDir)); + std::string mFinalDirUtf8 = utf16str_to_utf8str(llutf16string(mFinalDir)); + LL_INFOS() << "Resetting working dir from " << mFinalDirUtf8 << " to " << mOrigDirUtf8 << LL_ENDL; + SetCurrentDirectory(mOrigDir); + } + } + +private: + TCHAR mOrigDir[MAX_PATH]; + DWORD mOrigDirLen; + TCHAR mFinalDir[MAX_PATH]; + DWORD mFinalDirLen; +}; +#else // No-op outside Windows. +class LLDirectoryGuard +{ +public: + LLDirectoryGuard() {} + ~LLDirectoryGuard() {} +}; +#endif + + +#endif diff --git a/indra/llvfs/lldiriterator.cpp b/indra/llvfs/lldiriterator.cpp new file mode 100644 index 0000000000..3eb64e69d9 --- /dev/null +++ b/indra/llvfs/lldiriterator.cpp @@ -0,0 +1,243 @@ +/** + * @file lldiriterator.cpp + * @brief Iterator through directory entries matching the search pattern. + * + * $LicenseInfo:firstyear=2010&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2010, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include "lldiriterator.h" + +#include "fix_macros.h" +#include <boost/filesystem.hpp> +#include <boost/regex.hpp> + +namespace fs = boost::filesystem; + +static std::string glob_to_regex(const std::string& glob); + +class LLDirIterator::Impl +{ +public: + Impl(const std::string &dirname, const std::string &mask); + ~Impl(); + + bool next(std::string &fname); + +private: + boost::regex mFilterExp; + fs::directory_iterator mIter; + bool mIsValid; +}; + +LLDirIterator::Impl::Impl(const std::string &dirname, const std::string &mask) + : mIsValid(false) +{ +#ifdef LL_WINDOWS // or BOOST_WINDOWS_API + fs::path dir_path(utf8str_to_utf16str(dirname)); +#else + fs::path dir_path(dirname); +#endif + + bool is_dir = false; + + // Check if path is a directory. + try + { + is_dir = fs::is_directory(dir_path); + } + catch (const fs::filesystem_error& e) + { + LL_WARNS() << e.what() << LL_ENDL; + return; + } + + if (!is_dir) + { + LL_WARNS() << "Invalid path: \"" << dir_path.string() << "\"" << LL_ENDL; + return; + } + + // Initialize the directory iterator for the given path. + try + { + mIter = fs::directory_iterator(dir_path); + } + catch (const fs::filesystem_error& e) + { + LL_WARNS() << e.what() << LL_ENDL; + return; + } + + // Convert the glob mask to a regular expression + std::string exp = glob_to_regex(mask); + + // Initialize boost::regex with the expression converted from + // the glob mask. + // An exception is thrown if the expression is not valid. + try + { + mFilterExp.assign(exp); + } + catch (boost::regex_error& e) + { + LL_WARNS() << "\"" << exp << "\" is not a valid regular expression: " + << e.what() << LL_ENDL; + return; + } + + mIsValid = true; +} + +LLDirIterator::Impl::~Impl() +{ +} + +bool LLDirIterator::Impl::next(std::string &fname) +{ + fname = ""; + + if (!mIsValid) + { + LL_WARNS() << "The iterator is not correctly initialized." << LL_ENDL; + return false; + } + + fs::directory_iterator end_itr; // default construction yields past-the-end + bool found = false; + + // Check if path is a directory. + try + { + while (mIter != end_itr && !found) + { + boost::smatch match; + std::string name = mIter->path().filename().string(); + found = boost::regex_match(name, match, mFilterExp); + if (found) + { + fname = name; + } + + ++mIter; + } + } + catch (const fs::filesystem_error& e) + { + LL_WARNS() << e.what() << LL_ENDL; + } + + return found; +} + +/** +Converts the incoming glob into a regex. This involves +converting incoming glob expressions to regex equivilents and +at the same time, escaping any regex meaningful characters which +do not have glob meaning, i.e. + .()+|^$ +in the input. +*/ +std::string glob_to_regex(const std::string& glob) +{ + std::string regex; + regex.reserve(glob.size()<<1); + S32 braces = 0; + bool escaped = false; + bool square_brace_open = false; + + for (std::string::const_iterator i = glob.begin(); i != glob.end(); ++i) + { + char c = *i; + + switch (c) + { + case '*': + if (glob.begin() == i) + { + regex+="[^.].*"; + } + else + { + regex+= escaped ? "*" : ".*"; + } + break; + case '?': + regex+= escaped ? '?' : '.'; + break; + case '{': + braces++; + regex+='('; + break; + case '}': + if (!braces) + { + LL_ERRS() << "glob_to_regex: Closing brace without an equivalent opening brace: " << glob << LL_ENDL; + } + + regex+=')'; + braces--; + break; + case ',': + regex+= braces ? '|' : c; + break; + case '!': + regex+= square_brace_open ? '^' : c; + break; + case '.': // This collection have different regex meaning + case '^': // and so need escaping. + case '(': + case ')': + case '+': + case '|': + case '$': + regex += '\\'; + default: + regex += c; + break; + } + + escaped = ('\\' == c); + square_brace_open = ('[' == c); + } + + if (braces) + { + LL_ERRS() << "glob_to_regex: Unterminated brace expression: " << glob << LL_ENDL; + } + + return regex; +} + +LLDirIterator::LLDirIterator(const std::string &dirname, const std::string &mask) +{ + mImpl = new Impl(dirname, mask); +} + +LLDirIterator::~LLDirIterator() +{ + delete mImpl; +} + +bool LLDirIterator::next(std::string &fname) +{ + return mImpl->next(fname); +} diff --git a/indra/llvfs/lldiriterator.h b/indra/llvfs/lldiriterator.h new file mode 100644 index 0000000000..0b48be41b3 --- /dev/null +++ b/indra/llvfs/lldiriterator.h @@ -0,0 +1,87 @@ +/** + * @file lldiriterator.h + * @brief Iterator through directory entries matching the search pattern. + * + * $LicenseInfo:firstyear=2010&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2010, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef LL_LLDIRITERATOR_H +#define LL_LLDIRITERATOR_H + +#include "linden_common.h" + +/** + * Class LLDirIterator + * + * Iterates through directory entries matching the search pattern. + */ +class LLDirIterator +{ +public: + /** + * Constructs LLDirIterator object to search for glob pattern + * matches in a directory. + * + * @param dirname - name of a directory to search in. + * @param mask - search pattern, a glob expression + * + * Wildcards supported in glob expressions: + * -------------------------------------------------------------- + * | Wildcard | Matches | + * -------------------------------------------------------------- + * | * |zero or more characters | + * | ? |exactly one character | + * | [abcde] |exactly one character listed | + * | [a-e] |exactly one character in the given range | + * | [!abcde] |any character that is not listed | + * | [!a-e] |any character that is not in the given range | + * | {abc,xyz} |exactly one entire word in the options given | + * -------------------------------------------------------------- + */ + LLDirIterator(const std::string &dirname, const std::string &mask); + + ~LLDirIterator(); + + /** + * Searches for the next directory entry matching the glob mask + * specified upon iterator construction. + * Returns true if a match is found, sets fname + * parameter to the name of the matched directory entry and + * increments the iterator position. + * + * Typical usage: + * <code> + * LLDirIterator iter(directory, pattern); + * if ( iter.next(scanResult) ) + * </code> + * + * @param fname - name of the matched directory entry. + * @return true if a match is found, false otherwise. + */ + bool next(std::string &fname); + +protected: + class Impl; + Impl* mImpl; +}; + +#endif //LL_LLDIRITERATOR_H diff --git a/indra/llvfs/lllfsthread.cpp b/indra/llvfs/lllfsthread.cpp new file mode 100644 index 0000000000..be8e83a56f --- /dev/null +++ b/indra/llvfs/lllfsthread.cpp @@ -0,0 +1,245 @@ +/** + * @file lllfsthread.cpp + * @brief LLLFSThread base class + * + * $LicenseInfo:firstyear=2001&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2010, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include "linden_common.h" +#include "lllfsthread.h" +#include "llstl.h" +#include "llapr.h" + +//============================================================================ + +/*static*/ LLLFSThread* LLLFSThread::sLocal = NULL; + +//============================================================================ +// Run on MAIN thread +//static +void LLLFSThread::initClass(bool local_is_threaded) +{ + llassert(sLocal == NULL); + sLocal = new LLLFSThread(local_is_threaded); +} + +//static +S32 LLLFSThread::updateClass(U32 ms_elapsed) +{ + sLocal->update((F32)ms_elapsed); + return sLocal->getPending(); +} + +//static +void LLLFSThread::cleanupClass() +{ + llassert(sLocal != NULL); + sLocal->setQuitting(); + while (sLocal->getPending()) + { + sLocal->update(0); + } + delete sLocal; + sLocal = NULL; +} + +//---------------------------------------------------------------------------- + +LLLFSThread::LLLFSThread(bool threaded) : + LLQueuedThread("LFS", threaded), + mPriorityCounter(PRIORITY_LOWBITS) +{ + if(!mLocalAPRFilePoolp) + { + mLocalAPRFilePoolp = new LLVolatileAPRPool() ; + } +} + +LLLFSThread::~LLLFSThread() +{ + // mLocalAPRFilePoolp cleanup in LLThread + // ~LLQueuedThread() will be called here +} + +//---------------------------------------------------------------------------- + +LLLFSThread::handle_t LLLFSThread::read(const std::string& filename, /* Flawfinder: ignore */ + U8* buffer, S32 offset, S32 numbytes, + Responder* responder, U32 priority) +{ + handle_t handle = generateHandle(); + + if (priority == 0) priority = PRIORITY_NORMAL | priorityCounter(); + else if (priority < PRIORITY_LOW) priority |= PRIORITY_LOW; // All reads are at least PRIORITY_LOW + + Request* req = new Request(this, handle, priority, + FILE_READ, filename, + buffer, offset, numbytes, + responder); + + bool res = addRequest(req); + if (!res) + { + LL_ERRS() << "LLLFSThread::read called after LLLFSThread::cleanupClass()" << LL_ENDL; + } + + return handle; +} + +LLLFSThread::handle_t LLLFSThread::write(const std::string& filename, + U8* buffer, S32 offset, S32 numbytes, + Responder* responder, U32 priority) +{ + handle_t handle = generateHandle(); + + if (priority == 0) priority = PRIORITY_LOW | priorityCounter(); + + Request* req = new Request(this, handle, priority, + FILE_WRITE, filename, + buffer, offset, numbytes, + responder); + + bool res = addRequest(req); + if (!res) + { + LL_ERRS() << "LLLFSThread::read called after LLLFSThread::cleanupClass()" << LL_ENDL; + } + + return handle; +} + +//============================================================================ + +LLLFSThread::Request::Request(LLLFSThread* thread, + handle_t handle, U32 priority, + operation_t op, const std::string& filename, + U8* buffer, S32 offset, S32 numbytes, + Responder* responder) : + QueuedRequest(handle, priority, FLAG_AUTO_COMPLETE), + mThread(thread), + mOperation(op), + mFileName(filename), + mBuffer(buffer), + mOffset(offset), + mBytes(numbytes), + mBytesRead(0), + mResponder(responder) +{ + if (numbytes <= 0) + { + LL_WARNS() << "LLLFSThread: Request with numbytes = " << numbytes << LL_ENDL; + } +} + +LLLFSThread::Request::~Request() +{ +} + +// virtual, called from own thread +void LLLFSThread::Request::finishRequest(bool completed) +{ + if (mResponder.notNull()) + { + mResponder->completed(completed ? mBytesRead : 0); + mResponder = NULL; + } +} + +void LLLFSThread::Request::deleteRequest() +{ + if (getStatus() == STATUS_QUEUED) + { + LL_ERRS() << "Attempt to delete a queued LLLFSThread::Request!" << LL_ENDL; + } + if (mResponder.notNull()) + { + mResponder->completed(0); + mResponder = NULL; + } + LLQueuedThread::QueuedRequest::deleteRequest(); +} + +bool LLLFSThread::Request::processRequest() +{ + bool complete = false; + if (mOperation == FILE_READ) + { + llassert(mOffset >= 0); + LLAPRFile infile ; // auto-closes + infile.open(mFileName, LL_APR_RB, mThread->getLocalAPRFilePool()); + if (!infile.getFileHandle()) + { + LL_WARNS() << "LLLFS: Unable to read file: " << mFileName << LL_ENDL; + mBytesRead = 0; // fail + return true; + } + S32 off; + if (mOffset < 0) + off = infile.seek(APR_END, 0); + else + off = infile.seek(APR_SET, mOffset); + llassert_always(off >= 0); + mBytesRead = infile.read(mBuffer, mBytes ); + complete = true; +// LL_INFOS() << "LLLFSThread::READ:" << mFileName << " Bytes: " << mBytesRead << LL_ENDL; + } + else if (mOperation == FILE_WRITE) + { + apr_int32_t flags = APR_CREATE|APR_WRITE|APR_BINARY; + if (mOffset < 0) + flags |= APR_APPEND; + LLAPRFile outfile ; // auto-closes + outfile.open(mFileName, flags, mThread->getLocalAPRFilePool()); + if (!outfile.getFileHandle()) + { + LL_WARNS() << "LLLFS: Unable to write file: " << mFileName << LL_ENDL; + mBytesRead = 0; // fail + return true; + } + if (mOffset >= 0) + { + S32 seek = outfile.seek(APR_SET, mOffset); + if (seek < 0) + { + LL_WARNS() << "LLLFS: Unable to write file (seek failed): " << mFileName << LL_ENDL; + mBytesRead = 0; // fail + return true; + } + } + mBytesRead = outfile.write(mBuffer, mBytes ); + complete = true; +// LL_INFOS() << "LLLFSThread::WRITE:" << mFileName << " Bytes: " << mBytesRead << "/" << mBytes << " Offset:" << mOffset << LL_ENDL; + } + else + { + LL_ERRS() << "LLLFSThread::unknown operation: " << (S32)mOperation << LL_ENDL; + } + return complete; +} + +//============================================================================ + +LLLFSThread::Responder::~Responder() +{ +} + +//============================================================================ diff --git a/indra/llvfs/lllfsthread.h b/indra/llvfs/lllfsthread.h new file mode 100644 index 0000000000..58f658f7ba --- /dev/null +++ b/indra/llvfs/lllfsthread.h @@ -0,0 +1,147 @@ +/** + * @file lllfsthread.h + * @brief LLLFSThread base class + * + * $LicenseInfo:firstyear=2000&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2010, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef LL_LLLFSTHREAD_H +#define LL_LLLFSTHREAD_H + +#include <queue> +#include <string> +#include <map> +#include <set> + +#include "llpointer.h" +#include "llqueuedthread.h" + +//============================================================================ +// Threaded Local File System +//============================================================================ + +class LLLFSThread : public LLQueuedThread +{ + //------------------------------------------------------------------------ +public: + enum operation_t { + FILE_READ, + FILE_WRITE, + FILE_RENAME, + FILE_REMOVE + }; + + //------------------------------------------------------------------------ +public: + + class Responder : public LLThreadSafeRefCount + { + protected: + ~Responder(); + public: + virtual void completed(S32 bytes) = 0; + }; + + class Request : public QueuedRequest + { + protected: + virtual ~Request(); // use deleteRequest() + + public: + Request(LLLFSThread* thread, + handle_t handle, U32 priority, + operation_t op, const std::string& filename, + U8* buffer, S32 offset, S32 numbytes, + Responder* responder); + + S32 getBytes() + { + return mBytes; + } + S32 getBytesRead() + { + return mBytesRead; + } + S32 getOperation() + { + return mOperation; + } + U8* getBuffer() + { + return mBuffer; + } + const std::string& getFilename() + { + return mFileName; + } + + /*virtual*/ bool processRequest(); + /*virtual*/ void finishRequest(bool completed); + /*virtual*/ void deleteRequest(); + + private: + LLLFSThread* mThread; + operation_t mOperation; + + std::string mFileName; + + U8* mBuffer; // dest for reads, source for writes, new UUID for rename + S32 mOffset; // offset into file, -1 = append (WRITE only) + S32 mBytes; // bytes to read from file, -1 = all + S32 mBytesRead; // bytes read from file + + LLPointer<Responder> mResponder; + }; + + //------------------------------------------------------------------------ +public: + LLLFSThread(bool threaded = TRUE); + ~LLLFSThread(); + + // Return a Request handle + handle_t read(const std::string& filename, /* Flawfinder: ignore */ + U8* buffer, S32 offset, S32 numbytes, + Responder* responder, U32 pri=0); + handle_t write(const std::string& filename, + U8* buffer, S32 offset, S32 numbytes, + Responder* responder, U32 pri=0); + + // Misc + U32 priorityCounter() { return mPriorityCounter-- & PRIORITY_LOWBITS; } // Use to order IO operations + + // static initializers + static void initClass(bool local_is_threaded = TRUE); // Setup sLocal + static S32 updateClass(U32 ms_elapsed); + static void cleanupClass(); // Delete sLocal + + +private: + U32 mPriorityCounter; + +public: + static LLLFSThread* sLocal; // Default local file thread +}; + +//============================================================================ + + +#endif // LL_LLLFSTHREAD_H diff --git a/indra/llvfs/llpidlock.cpp b/indra/llvfs/llpidlock.cpp new file mode 100644 index 0000000000..f770e93d45 --- /dev/null +++ b/indra/llvfs/llpidlock.cpp @@ -0,0 +1,276 @@ +/** + * @file llformat.cpp + * @date January 2007 + * @brief string formatting utility + * + * $LicenseInfo:firstyear=2007&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2010, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include "linden_common.h" + +#include "llapr.h" // thread-related functions +#include "llpidlock.h" +#include "lldir.h" +#include "llsd.h" +#include "llsdserialize.h" +#include "llnametable.h" +#include "llframetimer.h" + +#if LL_WINDOWS //For windows platform. + +#include <windows.h> + +bool isProcessAlive(U32 pid) +{ + return (bool) GetProcessVersion((DWORD)pid); +} + +#else //Everyone Else +bool isProcessAlive(U32 pid) +{ + return (bool) kill( (pid_t)pid, 0); +} +#endif //Everyone else. + + + +class LLPidLockFile +{ + public: + LLPidLockFile( ) : + mAutosave(false), + mSaving(false), + mWaiting(false), + mPID(getpid()), + mNameTable(NULL), + mClean(true) + { + mLockName = gDirUtilp->getTempDir() + gDirUtilp->getDirDelimiter() + "savelock"; + } + bool requestLock(LLNameTable<void *> *name_table, bool autosave, + bool force_immediate=FALSE, F32 timeout=300.0); + bool checkLock(); + void releaseLock(); + + private: + void writeLockFile(LLSD pids); + public: + static LLPidLockFile& instance(); // return the singleton black list file + + bool mAutosave; + bool mSaving; + bool mWaiting; + LLFrameTimer mTimer; + U32 mPID; + std::string mLockName; + std::string mSaveName; + LLSD mPIDS_sd; + LLNameTable<void*> *mNameTable; + bool mClean; +}; + +LLPidLockFile& LLPidLockFile::instance() +{ + static LLPidLockFile the_file; + return the_file; +} + +void LLPidLockFile::writeLockFile(LLSD pids) +{ + llofstream ofile(mLockName.c_str()); + + if (!LLSDSerialize::toXML(pids,ofile)) + { + LL_WARNS() << "Unable to write concurrent save lock file." << LL_ENDL; + } + ofile.close(); +} + +bool LLPidLockFile::requestLock(LLNameTable<void *> *name_table, bool autosave, + bool force_immediate, F32 timeout) +{ + bool readyToSave = FALSE; + + if (mSaving) return FALSE; //Bail out if we're currently saving. Will not queue another save. + + if (!mWaiting){ + mNameTable=name_table; + mAutosave = autosave; + } + + LLSD out_pids; + out_pids.append( (LLSD::Integer)mPID ); + + llifstream ifile(mLockName.c_str()); + + if (ifile.is_open()) + { //If file exists, we need to decide whether or not to continue. + if ( force_immediate + || mTimer.hasExpired() ) //Only deserialize if we REALLY need to. + { + + LLSD in_pids; + + LLSDSerialize::fromXML(in_pids, ifile); + + //Clean up any dead PIDS that might be in there. + for (LLSD::array_iterator i=in_pids.beginArray(); + i !=in_pids.endArray(); + ++i) + { + U32 stored_pid=(*i).asInteger(); + + if (isProcessAlive(stored_pid)) + { + out_pids.append( (*i) ); + } + } + + readyToSave=TRUE; + } + ifile.close(); + } + else + { + readyToSave=TRUE; + } + + if (!mWaiting) //Not presently waiting to save. Queue up. + { + mTimer.resetWithExpiry(timeout); + mWaiting=TRUE; + } + + if (readyToSave) + { //Potential race condition won't kill us. Ignore it. + writeLockFile(out_pids); + mSaving=TRUE; + } + + return readyToSave; +} + +bool LLPidLockFile::checkLock() +{ + return mWaiting; +} + +void LLPidLockFile::releaseLock() +{ + llifstream ifile(mLockName.c_str()); + LLSD in_pids; + LLSD out_pids; + bool write_file=FALSE; + + LLSDSerialize::fromXML(in_pids, ifile); + + //Clean up this PID and any dead ones. + for (LLSD::array_iterator i=in_pids.beginArray(); + i !=in_pids.endArray(); + ++i) + { + U32 stored_pid=(*i).asInteger(); + + if (stored_pid != mPID && isProcessAlive(stored_pid)) + { + out_pids.append( (*i) ); + write_file=TRUE; + } + } + ifile.close(); + + if (write_file) + { + writeLockFile(out_pids); + } + else + { + unlink(mLockName.c_str()); + } + + mSaving=FALSE; + mWaiting=FALSE; +} + +//LLPidLock + +void LLPidLock::initClass() { + (void) LLPidLockFile::instance(); +} + +bool LLPidLock::checkLock() +{ + return LLPidLockFile::instance().checkLock(); +} + +bool LLPidLock::requestLock(LLNameTable<void *> *name_table, bool autosave, + bool force_immediate, F32 timeout) +{ + return LLPidLockFile::instance().requestLock(name_table,autosave,force_immediate,timeout); +} + +void LLPidLock::releaseLock() +{ + return LLPidLockFile::instance().releaseLock(); +} + +bool LLPidLock::isClean() +{ + return LLPidLockFile::instance().mClean; +} + +//getters +LLNameTable<void *> * LLPidLock::getNameTable() +{ + return LLPidLockFile::instance().mNameTable; +} + +bool LLPidLock::getAutosave() +{ + return LLPidLockFile::instance().mAutosave; +} + +bool LLPidLock::getClean() +{ + return LLPidLockFile::instance().mClean; +} + +std::string LLPidLock::getSaveName() +{ + return LLPidLockFile::instance().mSaveName; +} + +//setters +void LLPidLock::setClean(bool clean) +{ + LLPidLockFile::instance().mClean=clean; +} + +void LLPidLock::setSaveName(std::string savename) +{ + LLPidLockFile::instance().mSaveName=savename; +} + +S32 LLPidLock::getPID() +{ + return (S32)getpid(); +} diff --git a/indra/llvfs/llpidlock.h b/indra/llvfs/llpidlock.h new file mode 100644 index 0000000000..334f26bb29 --- /dev/null +++ b/indra/llvfs/llpidlock.h @@ -0,0 +1,60 @@ +/** + * @file llpidlock.h + * @brief System information debugging classes. + * + * $LicenseInfo:firstyear=2001&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2010, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef LL_PIDLOCK_H +#define LL_PIDLOCK_H +#include "llnametable.h" + +class LLSD; +class LLFrameTimer; + +#if !LL_WINDOWS //For non-windows platforms. +#include <signal.h> +#endif + +namespace LLPidLock +{ + void initClass(); // { (void) LLPidLockFile::instance(); } + + bool requestLock( LLNameTable<void *> *name_table=NULL, bool autosave=TRUE, + bool force_immediate=FALSE, F32 timeout=300.0); + bool checkLock(); + void releaseLock(); + bool isClean(); + + //getters + LLNameTable<void *> * getNameTable(); + bool getAutosave(); + bool getClean(); + std::string getSaveName(); + S32 getPID(); + + //setters + void setClean(bool clean); + void setSaveName(std::string savename); +}; + +#endif // LL_PIDLOCK_H diff --git a/indra/llvfs/llvfile.cpp b/indra/llvfs/llvfile.cpp new file mode 100644 index 0000000000..b8588e99f4 --- /dev/null +++ b/indra/llvfs/llvfile.cpp @@ -0,0 +1,437 @@ +/** + * @file llvfile.cpp + * @brief Implementation of virtual file + * + * $LicenseInfo:firstyear=2002&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2010, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include "linden_common.h" + +#include "llvfile.h" + +#include "llerror.h" +#include "llthread.h" +#include "lltimer.h" +#include "llfasttimer.h" +#include "llmemory.h" +#include "llvfs.h" + +const S32 LLVFile::READ = 0x00000001; +const S32 LLVFile::WRITE = 0x00000002; +const S32 LLVFile::READ_WRITE = 0x00000003; // LLVFile::READ & LLVFile::WRITE +const S32 LLVFile::APPEND = 0x00000006; // 0x00000004 & LLVFile::WRITE + +static LLTrace::BlockTimerStatHandle FTM_VFILE_WAIT("VFile Wait"); + +//---------------------------------------------------------------------------- +LLVFSThread* LLVFile::sVFSThread = NULL; +BOOL LLVFile::sAllocdVFSThread = FALSE; +//---------------------------------------------------------------------------- + +//============================================================================ + +LLVFile::LLVFile(LLVFS *vfs, const LLUUID &file_id, const LLAssetType::EType file_type, S32 mode) +{ + mFileType = file_type; + + mFileID = file_id; + mPosition = 0; + mMode = mode; + mVFS = vfs; + + mBytesRead = 0; + mHandle = LLVFSThread::nullHandle(); + mPriority = 128.f; + + mVFS->incLock(mFileID, mFileType, VFSLOCK_OPEN); +} + +LLVFile::~LLVFile() +{ + if (!isReadComplete()) + { + if (mHandle != LLVFSThread::nullHandle()) + { + if (!(mMode & LLVFile::WRITE)) + { + //LL_WARNS() << "Destroying LLVFile with pending async read/write, aborting..." << LL_ENDL; + sVFSThread->setFlags(mHandle, LLVFSThread::FLAG_AUTO_COMPLETE | LLVFSThread::FLAG_ABORT); + } + else // WRITE + { + sVFSThread->setFlags(mHandle, LLVFSThread::FLAG_AUTO_COMPLETE); + } + } + } + mVFS->decLock(mFileID, mFileType, VFSLOCK_OPEN); +} + +BOOL LLVFile::read(U8 *buffer, S32 bytes, BOOL async, F32 priority) +{ + if (! (mMode & READ)) + { + LL_WARNS() << "Attempt to read from file " << mFileID << " opened with mode " << std::hex << mMode << std::dec << LL_ENDL; + return FALSE; + } + + if (mHandle != LLVFSThread::nullHandle()) + { + LL_WARNS() << "Attempt to read from vfile object " << mFileID << " with pending async operation" << LL_ENDL; + return FALSE; + } + mPriority = priority; + + BOOL success = TRUE; + + // We can't do a read while there are pending async writes + waitForLock(VFSLOCK_APPEND); + + // *FIX: (?) + if (async) + { + mHandle = sVFSThread->read(mVFS, mFileID, mFileType, buffer, mPosition, bytes, threadPri()); + } + else + { + // We can't do a read while there are pending async writes on this file + mBytesRead = sVFSThread->readImmediate(mVFS, mFileID, mFileType, buffer, mPosition, bytes); + mPosition += mBytesRead; + if (! mBytesRead) + { + success = FALSE; + } + } + + return success; +} + +//static +U8* LLVFile::readFile(LLVFS *vfs, const LLUUID &uuid, LLAssetType::EType type, S32* bytes_read) +{ + U8 *data; + LLVFile file(vfs, uuid, type, LLVFile::READ); + S32 file_size = file.getSize(); + if (file_size == 0) + { + // File is empty. + data = NULL; + } + else + { + data = (U8*) ll_aligned_malloc<16>(file_size); + file.read(data, file_size); /* Flawfinder: ignore */ + + if (file.getLastBytesRead() != (S32)file_size) + { + ll_aligned_free<16>(data); + data = NULL; + file_size = 0; + } + } + if (bytes_read) + { + *bytes_read = file_size; + } + return data; +} + +void LLVFile::setReadPriority(const F32 priority) +{ + mPriority = priority; + if (mHandle != LLVFSThread::nullHandle()) + { + sVFSThread->setPriority(mHandle, threadPri()); + } +} + +BOOL LLVFile::isReadComplete() +{ + BOOL res = TRUE; + if (mHandle != LLVFSThread::nullHandle()) + { + LLVFSThread::Request* req = (LLVFSThread::Request*)sVFSThread->getRequest(mHandle); + LLVFSThread::status_t status = req->getStatus(); + if (status == LLVFSThread::STATUS_COMPLETE) + { + mBytesRead = req->getBytesRead(); + mPosition += mBytesRead; + sVFSThread->completeRequest(mHandle); + mHandle = LLVFSThread::nullHandle(); + } + else + { + res = FALSE; + } + } + return res; +} + +S32 LLVFile::getLastBytesRead() +{ + return mBytesRead; +} + +BOOL LLVFile::eof() +{ + return mPosition >= getSize(); +} + +BOOL LLVFile::write(const U8 *buffer, S32 bytes) +{ + if (! (mMode & WRITE)) + { + LL_WARNS() << "Attempt to write to file " << mFileID << " opened with mode " << std::hex << mMode << std::dec << LL_ENDL; + } + if (mHandle != LLVFSThread::nullHandle()) + { + LL_ERRS() << "Attempt to write to vfile object " << mFileID << " with pending async operation" << LL_ENDL; + return FALSE; + } + BOOL success = TRUE; + + // *FIX: allow async writes? potential problem wit mPosition... + if (mMode == APPEND) // all appends are async (but WRITEs are not) + { + U8* writebuf = new U8[bytes]; + memcpy(writebuf, buffer, bytes); + S32 offset = -1; + mHandle = sVFSThread->write(mVFS, mFileID, mFileType, + writebuf, offset, bytes, + LLVFSThread::FLAG_AUTO_COMPLETE | LLVFSThread::FLAG_AUTO_DELETE); + mHandle = LLVFSThread::nullHandle(); // FLAG_AUTO_COMPLETE means we don't track this + } + else + { + // We can't do a write while there are pending reads or writes on this file + waitForLock(VFSLOCK_READ); + waitForLock(VFSLOCK_APPEND); + + S32 pos = (mMode & APPEND) == APPEND ? -1 : mPosition; + + S32 wrote = sVFSThread->writeImmediate(mVFS, mFileID, mFileType, (U8*)buffer, pos, bytes); + + mPosition += wrote; + + if (wrote < bytes) + { + LL_WARNS() << "Tried to write " << bytes << " bytes, actually wrote " << wrote << LL_ENDL; + + success = FALSE; + } + } + return success; +} + +//static +BOOL LLVFile::writeFile(const U8 *buffer, S32 bytes, LLVFS *vfs, const LLUUID &uuid, LLAssetType::EType type) +{ + LLVFile file(vfs, uuid, type, LLVFile::WRITE); + file.setMaxSize(bytes); + return file.write(buffer, bytes); +} + +BOOL LLVFile::seek(S32 offset, S32 origin) +{ + if (mMode == APPEND) + { + LL_WARNS() << "Attempt to seek on append-only file" << LL_ENDL; + return FALSE; + } + + if (-1 == origin) + { + origin = mPosition; + } + + S32 new_pos = origin + offset; + + S32 size = getSize(); // Calls waitForLock(VFSLOCK_APPEND) + + if (new_pos > size) + { + LL_WARNS() << "Attempt to seek past end of file" << LL_ENDL; + + mPosition = size; + return FALSE; + } + else if (new_pos < 0) + { + LL_WARNS() << "Attempt to seek past beginning of file" << LL_ENDL; + + mPosition = 0; + return FALSE; + } + + mPosition = new_pos; + return TRUE; +} + +S32 LLVFile::tell() const +{ + return mPosition; +} + +S32 LLVFile::getSize() +{ + waitForLock(VFSLOCK_APPEND); + S32 size = mVFS->getSize(mFileID, mFileType); + + return size; +} + +S32 LLVFile::getMaxSize() +{ + S32 size = mVFS->getMaxSize(mFileID, mFileType); + + return size; +} + +BOOL LLVFile::setMaxSize(S32 size) +{ + if (! (mMode & WRITE)) + { + LL_WARNS() << "Attempt to change size of file " << mFileID << " opened with mode " << std::hex << mMode << std::dec << LL_ENDL; + + return FALSE; + } + + if (!mVFS->checkAvailable(size)) + { + //LL_RECORD_BLOCK_TIME(FTM_VFILE_WAIT); + S32 count = 0; + while (sVFSThread->getPending() > 1000) + { + if (count % 100 == 0) + { + LL_INFOS() << "VFS catching up... Pending: " << sVFSThread->getPending() << LL_ENDL; + } + if (sVFSThread->isPaused()) + { + sVFSThread->update(0); + } + ms_sleep(10); + } + } + return mVFS->setMaxSize(mFileID, mFileType, size); +} + +BOOL LLVFile::rename(const LLUUID &new_id, const LLAssetType::EType new_type) +{ + if (! (mMode & WRITE)) + { + LL_WARNS() << "Attempt to rename file " << mFileID << " opened with mode " << std::hex << mMode << std::dec << LL_ENDL; + + return FALSE; + } + + if (mHandle != LLVFSThread::nullHandle()) + { + LL_WARNS() << "Renaming file with pending async read" << LL_ENDL; + } + + waitForLock(VFSLOCK_READ); + waitForLock(VFSLOCK_APPEND); + + // we need to release / replace our own lock + // since the renamed file will inherit locks from the new name + mVFS->decLock(mFileID, mFileType, VFSLOCK_OPEN); + mVFS->renameFile(mFileID, mFileType, new_id, new_type); + mVFS->incLock(new_id, new_type, VFSLOCK_OPEN); + + mFileID = new_id; + mFileType = new_type; + + return TRUE; +} + +BOOL LLVFile::remove() +{ +// LL_INFOS() << "Removing file " << mFileID << LL_ENDL; + + if (! (mMode & WRITE)) + { + // Leaving paranoia warning just because this should be a very infrequent + // operation. + LL_WARNS() << "Remove file " << mFileID << " opened with mode " << std::hex << mMode << std::dec << LL_ENDL; + } + + if (mHandle != LLVFSThread::nullHandle()) + { + LL_WARNS() << "Removing file with pending async read" << LL_ENDL; + } + + // why not seek back to the beginning of the file too? + mPosition = 0; + + waitForLock(VFSLOCK_READ); + waitForLock(VFSLOCK_APPEND); + mVFS->removeFile(mFileID, mFileType); + + return TRUE; +} + +// static +void LLVFile::initClass(LLVFSThread* vfsthread) +{ + if (!vfsthread) + { + if (LLVFSThread::sLocal != NULL) + { + vfsthread = LLVFSThread::sLocal; + } + else + { + vfsthread = new LLVFSThread(); + sAllocdVFSThread = TRUE; + } + } + sVFSThread = vfsthread; +} + +// static +void LLVFile::cleanupClass() +{ + if (sAllocdVFSThread) + { + delete sVFSThread; + } + sVFSThread = NULL; +} + +bool LLVFile::isLocked(EVFSLock lock) +{ + return mVFS->isLocked(mFileID, mFileType, lock) ? true : false; +} + +void LLVFile::waitForLock(EVFSLock lock) +{ + //LL_RECORD_BLOCK_TIME(FTM_VFILE_WAIT); + // spin until the lock clears + while (isLocked(lock)) + { + if (sVFSThread->isPaused()) + { + sVFSThread->update(0); + } + ms_sleep(1); + } +} diff --git a/indra/llvfs/llvfile.h b/indra/llvfs/llvfile.h new file mode 100644 index 0000000000..7e9d9f73e5 --- /dev/null +++ b/indra/llvfs/llvfile.h @@ -0,0 +1,90 @@ +/** + * @file llvfile.h + * @brief Definition of virtual file + * + * $LicenseInfo:firstyear=2002&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2010, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef LL_LLVFILE_H +#define LL_LLVFILE_H + +#include "lluuid.h" +#include "llassettype.h" +#include "llvfs.h" +#include "llvfsthread.h" + +class LLVFile +{ +public: + LLVFile(LLVFS *vfs, const LLUUID &file_id, const LLAssetType::EType file_type, S32 mode = LLVFile::READ); + ~LLVFile(); + + BOOL read(U8 *buffer, S32 bytes, BOOL async = FALSE, F32 priority = 128.f); /* Flawfinder: ignore */ + static U8* readFile(LLVFS *vfs, const LLUUID &uuid, LLAssetType::EType type, S32* bytes_read = 0); + void setReadPriority(const F32 priority); + BOOL isReadComplete(); + S32 getLastBytesRead(); + BOOL eof(); + + BOOL write(const U8 *buffer, S32 bytes); + static BOOL writeFile(const U8 *buffer, S32 bytes, LLVFS *vfs, const LLUUID &uuid, LLAssetType::EType type); + BOOL seek(S32 offset, S32 origin = -1); + S32 tell() const; + + S32 getSize(); + S32 getMaxSize(); + BOOL setMaxSize(S32 size); + BOOL rename(const LLUUID &new_id, const LLAssetType::EType new_type); + BOOL remove(); + + bool isLocked(EVFSLock lock); + void waitForLock(EVFSLock lock); + + static void initClass(LLVFSThread* vfsthread = NULL); + static void cleanupClass(); + static LLVFSThread* getVFSThread() { return sVFSThread; } + +protected: + static LLVFSThread* sVFSThread; + static BOOL sAllocdVFSThread; + U32 threadPri() { return LLVFSThread::PRIORITY_NORMAL + llmin((U32)mPriority,(U32)0xfff); } + +public: + static const S32 READ; + static const S32 WRITE; + static const S32 READ_WRITE; + static const S32 APPEND; + +protected: + LLAssetType::EType mFileType; + + LLUUID mFileID; + S32 mPosition; + S32 mMode; + LLVFS *mVFS; + F32 mPriority; + + S32 mBytesRead; + LLVFSThread::handle_t mHandle; +}; + +#endif diff --git a/indra/llvfs/llvfs.cpp b/indra/llvfs/llvfs.cpp new file mode 100644 index 0000000000..617056d94d --- /dev/null +++ b/indra/llvfs/llvfs.cpp @@ -0,0 +1,2220 @@ +/** + * @file llvfs.cpp + * @brief Implementation of virtual file system + * + * $LicenseInfo:firstyear=2002&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2010, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include "linden_common.h" + +#include "llvfs.h" + +#include <sys/stat.h> +#include <set> +#include <map> +#if LL_WINDOWS +#include <share.h> +#elif LL_SOLARIS +#include <sys/types.h> +#include <unistd.h> +#include <fcntl.h> +#else +#include <sys/file.h> +#endif + +#include "llstl.h" +#include "lltimer.h" + +const S32 FILE_BLOCK_MASK = 0x000003FF; // 1024-byte blocks +const S32 VFS_CLEANUP_SIZE = 5242880; // how much space we free up in a single stroke +const S32 BLOCK_LENGTH_INVALID = -1; // mLength for invalid LLVFSFileBlocks + +LLVFS *gVFS = NULL; + +// internal class definitions +class LLVFSBlock +{ +public: + LLVFSBlock() + { + mLocation = 0; + mLength = 0; + } + + LLVFSBlock(U32 loc, S32 size) + { + mLocation = loc; + mLength = size; + } + + static bool locationSortPredicate( + const LLVFSBlock* lhs, + const LLVFSBlock* rhs) + { + return lhs->mLocation < rhs->mLocation; + } + +public: + U32 mLocation; + S32 mLength; // allocated block size +}; + +LLVFSFileSpecifier::LLVFSFileSpecifier() +: mFileID(), + mFileType( LLAssetType::AT_NONE ) +{ +} + +LLVFSFileSpecifier::LLVFSFileSpecifier(const LLUUID &file_id, const LLAssetType::EType file_type) +{ + mFileID = file_id; + mFileType = file_type; +} + +bool LLVFSFileSpecifier::operator<(const LLVFSFileSpecifier &rhs) const +{ + return (mFileID == rhs.mFileID) + ? mFileType < rhs.mFileType + : mFileID < rhs.mFileID; +} + +bool LLVFSFileSpecifier::operator==(const LLVFSFileSpecifier &rhs) const +{ + return (mFileID == rhs.mFileID && + mFileType == rhs.mFileType); +} + + +class LLVFSFileBlock : public LLVFSBlock, public LLVFSFileSpecifier +{ +public: + LLVFSFileBlock() : LLVFSBlock(), LLVFSFileSpecifier() + { + init(); + } + + LLVFSFileBlock(const LLUUID &file_id, LLAssetType::EType file_type, U32 loc = 0, S32 size = 0) + : LLVFSBlock(loc, size), LLVFSFileSpecifier( file_id, file_type ) + { + init(); + } + + void init() + { + mSize = 0; + mIndexLocation = -1; + mAccessTime = (U32)time(NULL); + + for (S32 i = 0; i < (S32)VFSLOCK_COUNT; i++) + { + mLocks[(EVFSLock)i] = 0; + } + } + + #ifdef LL_LITTLE_ENDIAN + inline void swizzleCopy(void *dst, void *src, int size) { memcpy(dst, src, size); /* Flawfinder: ignore */} + + #else + + inline U32 swizzle32(U32 x) + { + return(((x >> 24) & 0x000000FF) | ((x >> 8) & 0x0000FF00) | ((x << 8) & 0x00FF0000) |((x << 24) & 0xFF000000)); + } + + inline U16 swizzle16(U16 x) + { + return( ((x >> 8) & 0x000000FF) | ((x << 8) & 0x0000FF00) ); + } + + inline void swizzleCopy(void *dst, void *src, int size) + { + if(size == 4) + { + ((U32*)dst)[0] = swizzle32(((U32*)src)[0]); + } + else if(size == 2) + { + ((U16*)dst)[0] = swizzle16(((U16*)src)[0]); + } + else + { + // Perhaps this should assert... + memcpy(dst, src, size); /* Flawfinder: ignore */ + } + } + + #endif + + void serialize(U8 *buffer) + { + swizzleCopy(buffer, &mLocation, 4); + buffer += 4; + swizzleCopy(buffer, &mLength, 4); + buffer +=4; + swizzleCopy(buffer, &mAccessTime, 4); + buffer +=4; + memcpy(buffer, &mFileID.mData, 16); /* Flawfinder: ignore */ + buffer += 16; + S16 temp_type = mFileType; + swizzleCopy(buffer, &temp_type, 2); + buffer += 2; + swizzleCopy(buffer, &mSize, 4); + } + + void deserialize(U8 *buffer, const S32 index_loc) + { + mIndexLocation = index_loc; + + swizzleCopy(&mLocation, buffer, 4); + buffer += 4; + swizzleCopy(&mLength, buffer, 4); + buffer += 4; + swizzleCopy(&mAccessTime, buffer, 4); + buffer += 4; + memcpy(&mFileID.mData, buffer, 16); + buffer += 16; + S16 temp_type; + swizzleCopy(&temp_type, buffer, 2); + mFileType = (LLAssetType::EType)temp_type; + buffer += 2; + swizzleCopy(&mSize, buffer, 4); + } + + static BOOL insertLRU(LLVFSFileBlock* const& first, + LLVFSFileBlock* const& second) + { + return (first->mAccessTime == second->mAccessTime) + ? *first < *second + : first->mAccessTime < second->mAccessTime; + } + +public: + S32 mSize; + S32 mIndexLocation; // location of index entry + U32 mAccessTime; + BOOL mLocks[VFSLOCK_COUNT]; // number of outstanding locks of each type + + static const S32 SERIAL_SIZE; +}; + +// Helper structure for doing lru w/ stl... is there a simpler way? +struct LLVFSFileBlock_less +{ + bool operator()(LLVFSFileBlock* const& lhs, LLVFSFileBlock* const& rhs) const + { + return (LLVFSFileBlock::insertLRU(lhs, rhs)) ? true : false; + } +}; + + +const S32 LLVFSFileBlock::SERIAL_SIZE = 34; + + +LLVFS::LLVFS(const std::string& index_filename, const std::string& data_filename, const BOOL read_only, const U32 presize, const BOOL remove_after_crash) +: mRemoveAfterCrash(remove_after_crash), + mDataFP(NULL), + mIndexFP(NULL) +{ + mDataMutex = new LLMutex(); + + S32 i; + for (i = 0; i < VFSLOCK_COUNT; i++) + { + mLockCounts[i] = 0; + } + mValid = VFSVALID_OK; + mReadOnly = read_only; + mIndexFilename = index_filename; + mDataFilename = data_filename; + + const char *file_mode = mReadOnly ? "rb" : "r+b"; + + LL_INFOS("VFS") << "Attempting to open VFS index file " << mIndexFilename << LL_ENDL; + LL_INFOS("VFS") << "Attempting to open VFS data file " << mDataFilename << LL_ENDL; + + mDataFP = openAndLock(mDataFilename, file_mode, mReadOnly); + if (!mDataFP) + { + if (mReadOnly) + { + LL_WARNS("VFS") << "Can't find " << mDataFilename << " to open read-only VFS" << LL_ENDL; + mValid = VFSVALID_BAD_CANNOT_OPEN_READONLY; + return; + } + + mDataFP = openAndLock(mDataFilename, "w+b", FALSE); + if (mDataFP) + { + // Since we're creating this data file, assume any index file is bogus + // remove the index, since this vfs is now blank + LLFile::remove(mIndexFilename); + } + else + { + LL_WARNS("VFS") << "Couldn't open vfs data file " + << mDataFilename << LL_ENDL; + mValid = VFSVALID_BAD_CANNOT_CREATE; + return; + } + + if (presize) + { + presizeDataFile(presize); + } + } + + // Did we leave this file open for writing last time? + // If so, close it and start over. + if (!mReadOnly && mRemoveAfterCrash) + { + llstat marker_info; + std::string marker = mDataFilename + ".open"; + if (!LLFile::stat(marker, &marker_info)) + { + // marker exists, kill the lock and the VFS files + unlockAndClose(mDataFP); + mDataFP = NULL; + + LL_WARNS("VFS") << "VFS: File left open on last run, removing old VFS file " << mDataFilename << LL_ENDL; + LLFile::remove(mIndexFilename); + LLFile::remove(mDataFilename); + LLFile::remove(marker); + + mDataFP = openAndLock(mDataFilename, "w+b", FALSE); + if (!mDataFP) + { + LL_WARNS("VFS") << "Can't open VFS data file in crash recovery" << LL_ENDL; + mValid = VFSVALID_BAD_CANNOT_CREATE; + return; + } + + if (presize) + { + presizeDataFile(presize); + } + } + } + + // determine the real file size + fseek(mDataFP, 0, SEEK_END); + U32 data_size = ftell(mDataFP); + + // read the index file + // make sure there's at least one file in it too + // if not, we'll treat this as a new vfs + llstat fbuf; + if (! LLFile::stat(mIndexFilename, &fbuf) && + fbuf.st_size >= LLVFSFileBlock::SERIAL_SIZE && + (mIndexFP = openAndLock(mIndexFilename, file_mode, mReadOnly)) // Yes, this is an assignment and not '==' + ) + { + std::vector<U8> buffer(fbuf.st_size); + size_t buf_offset = 0; + size_t nread = fread(&buffer[0], 1, fbuf.st_size, mIndexFP); + + std::vector<LLVFSFileBlock*> files_by_loc; + + while (buf_offset < nread) + { + LLVFSFileBlock *block = new LLVFSFileBlock(); + + block->deserialize(&buffer[buf_offset], (S32)buf_offset); + + // Do sanity check on this block. + // Note that this skips zero size blocks, which helps VFS + // to heal after some errors. JC + if (block->mLength > 0 && + (U32)block->mLength <= data_size && + block->mLocation < data_size && + block->mSize > 0 && + block->mSize <= block->mLength && + block->mFileType >= LLAssetType::AT_NONE && + block->mFileType < LLAssetType::AT_COUNT) + { + mFileBlocks.insert(fileblock_map::value_type(*block, block)); + files_by_loc.push_back(block); + } + else + if (block->mLength && block->mSize > 0) + { + // this is corrupt, not empty + LL_WARNS("VFS") << "VFS corruption: " << block->mFileID << " (" << block->mFileType << ") at index " << block->mIndexLocation << " DS: " << data_size << LL_ENDL; + LL_WARNS("VFS") << "Length: " << block->mLength << "\tLocation: " << block->mLocation << "\tSize: " << block->mSize << LL_ENDL; + LL_WARNS("VFS") << "File has bad data - VFS removed" << LL_ENDL; + + delete block; + + unlockAndClose( mIndexFP ); + mIndexFP = NULL; + LLFile::remove( mIndexFilename ); + + unlockAndClose( mDataFP ); + mDataFP = NULL; + LLFile::remove( mDataFilename ); + + LL_WARNS("VFS") << "Deleted corrupt VFS files " + << mDataFilename + << " and " + << mIndexFilename + << LL_ENDL; + + mValid = VFSVALID_BAD_CORRUPT; + return; + } + else + { + // this is a null or bad entry, skip it + mIndexHoles.push_back(buf_offset); + + delete block; + } + + buf_offset += LLVFSFileBlock::SERIAL_SIZE; + } + + std::sort( + files_by_loc.begin(), + files_by_loc.end(), + LLVFSFileBlock::locationSortPredicate); + + // There are 3 cases that have to be considered. + // 1. No blocks + // 2. One block. + // 3. Two or more blocks. + if (!files_by_loc.empty()) + { + // cur walks through the list. + std::vector<LLVFSFileBlock*>::iterator cur = files_by_loc.begin(); + std::vector<LLVFSFileBlock*>::iterator end = files_by_loc.end(); + LLVFSFileBlock* last_file_block = *cur; + + // Check to see if there is an empty space before the first file. + if (last_file_block->mLocation > 0) + { + // If so, create a free block. + addFreeBlock(new LLVFSBlock(0, last_file_block->mLocation)); + } + + // Walk through the 2nd+ block. If there is a free space + // between cur_file_block and last_file_block, add it to + // the free space collection. This block will not need to + // run in the case there is only one entry in the VFS. + ++cur; + while( cur != end ) + { + LLVFSFileBlock* cur_file_block = *cur; + + // Dupe check on the block + if (cur_file_block->mLocation == last_file_block->mLocation + && cur_file_block->mLength == last_file_block->mLength) + { + LL_WARNS("VFS") << "VFS: removing duplicate entry" + << " at " << cur_file_block->mLocation + << " length " << cur_file_block->mLength + << " size " << cur_file_block->mSize + << " ID " << cur_file_block->mFileID + << " type " << cur_file_block->mFileType + << LL_ENDL; + + // Duplicate entries. Nuke them both for safety. + mFileBlocks.erase(*cur_file_block); // remove ID/type entry + if (cur_file_block->mLength > 0) + { + // convert to hole + addFreeBlock( + new LLVFSBlock( + cur_file_block->mLocation, + cur_file_block->mLength)); + } + lockData(); // needed for sync() + sync(cur_file_block, TRUE); // remove first on disk + sync(last_file_block, TRUE); // remove last on disk + unlockData(); // needed for sync() + last_file_block = cur_file_block; + ++cur; + continue; + } + + // Figure out where the last block ended. + S32 loc = last_file_block->mLocation+last_file_block->mLength; + + // Figure out how much space there is between where + // the last block ended and this block begins. + S32 length = cur_file_block->mLocation - loc; + + // Check for more errors... Seeing if the current + // entry and the last entry make sense together. + if (length < 0 || loc < 0 || (U32)loc > data_size) + { + // Invalid VFS + unlockAndClose( mIndexFP ); + mIndexFP = NULL; + LLFile::remove( mIndexFilename ); + + unlockAndClose( mDataFP ); + mDataFP = NULL; + LLFile::remove( mDataFilename ); + + LL_WARNS("VFS") << "VFS: overlapping entries" + << " at " << cur_file_block->mLocation + << " length " << cur_file_block->mLength + << " ID " << cur_file_block->mFileID + << " type " << cur_file_block->mFileType + << LL_ENDL; + + LL_WARNS("VFS") << "Deleted corrupt VFS files " + << mDataFilename + << " and " + << mIndexFilename + << LL_ENDL; + + mValid = VFSVALID_BAD_CORRUPT; + return; + } + + // we don't want to add empty blocks to the list... + if (length > 0) + { + addFreeBlock(new LLVFSBlock(loc, length)); + } + last_file_block = cur_file_block; + ++cur; + } + + // also note any empty space at the end + U32 loc = last_file_block->mLocation + last_file_block->mLength; + if (loc < data_size) + { + addFreeBlock(new LLVFSBlock(loc, data_size - loc)); + } + } + else // There where no blocks in the file. + { + addFreeBlock(new LLVFSBlock(0, data_size)); + } + } + else // Pre-existing index file wasn't opened + { + if (mReadOnly) + { + LL_WARNS("VFS") << "Can't find " << mIndexFilename << " to open read-only VFS" << LL_ENDL; + mValid = VFSVALID_BAD_CANNOT_OPEN_READONLY; + return; + } + + + mIndexFP = openAndLock(mIndexFilename, "w+b", FALSE); + if (!mIndexFP) + { + LL_WARNS("VFS") << "Couldn't open an index file for the VFS, probably a sharing violation!" << LL_ENDL; + + unlockAndClose( mDataFP ); + mDataFP = NULL; + LLFile::remove( mDataFilename ); + + mValid = VFSVALID_BAD_CANNOT_CREATE; + return; + } + + // no index file, start from scratch w/ 1GB allocation + LLVFSBlock *first_block = new LLVFSBlock(0, data_size ? data_size : 0x40000000); + addFreeBlock(first_block); + } + + // Open marker file to look for bad shutdowns + if (!mReadOnly && mRemoveAfterCrash) + { + std::string marker = mDataFilename + ".open"; + LLFILE* marker_fp = LLFile::fopen(marker, "w"); /* Flawfinder: ignore */ + if (marker_fp) + { + fclose(marker_fp); + marker_fp = NULL; + } + } + + LL_INFOS("VFS") << "Using VFS index file " << mIndexFilename << LL_ENDL; + LL_INFOS("VFS") << "Using VFS data file " << mDataFilename << LL_ENDL; + + mValid = VFSVALID_OK; +} + +LLVFS::~LLVFS() +{ + if (mDataMutex->isLocked()) + { + LL_ERRS("VFS") << "LLVFS destroyed with mutex locked" << LL_ENDL; + } + + unlockAndClose(mIndexFP); + mIndexFP = NULL; + + fileblock_map::const_iterator it; + for (it = mFileBlocks.begin(); it != mFileBlocks.end(); ++it) + { + delete (*it).second; + } + mFileBlocks.clear(); + + mFreeBlocksByLength.clear(); + + for_each(mFreeBlocksByLocation.begin(), mFreeBlocksByLocation.end(), DeletePairedPointer()); + mFreeBlocksByLocation.clear(); + + unlockAndClose(mDataFP); + mDataFP = NULL; + + // Remove marker file + if (!mReadOnly && mRemoveAfterCrash) + { + std::string marker = mDataFilename + ".open"; + LLFile::remove(marker); + } + + delete mDataMutex; +} + + +// Use this function normally to create LLVFS files. +// Will append digits to the end of the filename with multiple re-trys +// static +LLVFS * LLVFS::createLLVFS(const std::string& index_filename, + const std::string& data_filename, + const BOOL read_only, + const U32 presize, + const BOOL remove_after_crash) +{ + LLVFS * new_vfs = new LLVFS(index_filename, data_filename, read_only, presize, remove_after_crash); + + if( !new_vfs->isValid() ) + { // First name failed, retry with new names + std::string retry_vfs_index_name; + std::string retry_vfs_data_name; + S32 count = 0; + while (!new_vfs->isValid() && + count < 256) + { // Append '.<number>' to end of filenames + retry_vfs_index_name = index_filename + llformat(".%u",count); + retry_vfs_data_name = data_filename + llformat(".%u", count); + + delete new_vfs; // Delete bad VFS and try again + new_vfs = new LLVFS(retry_vfs_index_name, retry_vfs_data_name, read_only, presize, remove_after_crash); + + count++; + } + } + + if( !new_vfs->isValid() ) + { + delete new_vfs; // Delete bad VFS + new_vfs = NULL; // Total failure + } + + return new_vfs; +} + + + +void LLVFS::presizeDataFile(const U32 size) +{ + if (!mDataFP) + { + LL_ERRS() << "LLVFS::presizeDataFile() with no data file open" << LL_ENDL; + return; + } + + // we're creating this file for the first time, size it + fseek(mDataFP, size-1, SEEK_SET); + S32 tmp = 0; + tmp = (S32)fwrite(&tmp, 1, 1, mDataFP); + // fflush(mDataFP); + + // also remove any index, since this vfs is now blank + LLFile::remove(mIndexFilename); + + if (tmp) + { + LL_INFOS() << "Pre-sized VFS data file to " << ftell(mDataFP) << " bytes" << LL_ENDL; + } + else + { + LL_WARNS() << "Failed to pre-size VFS data file" << LL_ENDL; + } +} + +BOOL LLVFS::getExists(const LLUUID &file_id, const LLAssetType::EType file_type) +{ + LLVFSFileBlock *block = NULL; + + if (!isValid()) + { + LL_ERRS() << "Attempting to use invalid VFS!" << LL_ENDL; + } + + lockData(); + + LLVFSFileSpecifier spec(file_id, file_type); + fileblock_map::iterator it = mFileBlocks.find(spec); + if (it != mFileBlocks.end()) + { + block = (*it).second; + block->mAccessTime = (U32)time(NULL); + } + + BOOL res = (block && block->mLength > 0) ? TRUE : FALSE; + + unlockData(); + + return res; +} + +S32 LLVFS::getSize(const LLUUID &file_id, const LLAssetType::EType file_type) +{ + S32 size = 0; + + if (!isValid()) + { + LL_ERRS() << "Attempting to use invalid VFS!" << LL_ENDL; + + } + + lockData(); + + LLVFSFileSpecifier spec(file_id, file_type); + fileblock_map::iterator it = mFileBlocks.find(spec); + if (it != mFileBlocks.end()) + { + LLVFSFileBlock *block = (*it).second; + + block->mAccessTime = (U32)time(NULL); + size = block->mSize; + } + + unlockData(); + + return size; +} + +S32 LLVFS::getMaxSize(const LLUUID &file_id, const LLAssetType::EType file_type) +{ + S32 size = 0; + + if (!isValid()) + { + LL_ERRS() << "Attempting to use invalid VFS!" << LL_ENDL; + } + + lockData(); + + LLVFSFileSpecifier spec(file_id, file_type); + fileblock_map::iterator it = mFileBlocks.find(spec); + if (it != mFileBlocks.end()) + { + LLVFSFileBlock *block = (*it).second; + + block->mAccessTime = (U32)time(NULL); + size = block->mLength; + } + + unlockData(); + + return size; +} + +BOOL LLVFS::checkAvailable(S32 max_size) +{ + lockData(); + + blocks_length_map_t::iterator iter = mFreeBlocksByLength.lower_bound(max_size); // first entry >= size + const BOOL res(iter == mFreeBlocksByLength.end() ? FALSE : TRUE); + + unlockData(); + + return res; +} + +BOOL LLVFS::setMaxSize(const LLUUID &file_id, const LLAssetType::EType file_type, S32 max_size) +{ + if (!isValid()) + { + LL_ERRS() << "Attempting to use invalid VFS!" << LL_ENDL; + } + if (mReadOnly) + { + LL_ERRS() << "Attempt to write to read-only VFS" << LL_ENDL; + } + if (max_size <= 0) + { + LL_WARNS() << "VFS: Attempt to assign size " << max_size << " to vfile " << file_id << LL_ENDL; + return FALSE; + } + + lockData(); + + LLVFSFileSpecifier spec(file_id, file_type); + LLVFSFileBlock *block = NULL; + fileblock_map::iterator it = mFileBlocks.find(spec); + if (it != mFileBlocks.end()) + { + block = (*it).second; + } + + // round all sizes upward to KB increments + // SJB: Need to not round for the new texture-pipeline code so we know the correct + // max file size. Need to investigate the potential problems with this... + if (file_type != LLAssetType::AT_TEXTURE) + { + if (max_size & FILE_BLOCK_MASK) + { + max_size += FILE_BLOCK_MASK; + max_size &= ~FILE_BLOCK_MASK; + } + } + + if (block && block->mLength > 0) + { + block->mAccessTime = (U32)time(NULL); + + if (max_size == block->mLength) + { + unlockData(); + return TRUE; + } + else if (max_size < block->mLength) + { + // this file is shrinking + LLVFSBlock *free_block = new LLVFSBlock(block->mLocation + max_size, block->mLength - max_size); + + addFreeBlock(free_block); + + block->mLength = max_size; + + if (block->mLength < block->mSize) + { + // JC: Was a warning, but Ian says it's bad. + LL_ERRS() << "Truncating virtual file " << file_id << " to " << block->mLength << " bytes" << LL_ENDL; + block->mSize = block->mLength; + } + + sync(block); + //mergeFreeBlocks(); + + unlockData(); + return TRUE; + } + else if (max_size > block->mLength) + { + // this file is growing + // first check for an adjacent free block to grow into + S32 size_increase = max_size - block->mLength; + + // Find the first free block with and addres > block->mLocation + LLVFSBlock *free_block; + blocks_location_map_t::iterator iter = mFreeBlocksByLocation.upper_bound(block->mLocation); + if (iter != mFreeBlocksByLocation.end()) + { + free_block = iter->second; + + if (free_block->mLocation == block->mLocation + block->mLength && + free_block->mLength >= size_increase) + { + // this free block is at the end of the file and is large enough + + // Must call useFreeSpace before sync(), as sync() + // unlocks data structures. + useFreeSpace(free_block, size_increase); + block->mLength += size_increase; + sync(block); + + unlockData(); + return TRUE; + } + } + + // no adjecent free block, find one in the list + free_block = findFreeBlock(max_size, block); + + if (free_block) + { + // Save location where data is going, useFreeSpace will move free_block->mLocation; + U32 new_data_location = free_block->mLocation; + + //mark the free block as used so it does not + //interfere with other operations such as addFreeBlock + useFreeSpace(free_block, max_size); // useFreeSpace takes ownership (and may delete) free_block + + if (block->mLength > 0) + { + // create a new free block where this file used to be + LLVFSBlock *new_free_block = new LLVFSBlock(block->mLocation, block->mLength); + + addFreeBlock(new_free_block); + + if (block->mSize > 0) + { + // move the file into the new block + std::vector<U8> buffer(block->mSize); + fseek(mDataFP, block->mLocation, SEEK_SET); + if (fread(&buffer[0], block->mSize, 1, mDataFP) == 1) + { + fseek(mDataFP, new_data_location, SEEK_SET); + if (fwrite(&buffer[0], block->mSize, 1, mDataFP) != 1) + { + LL_WARNS() << "Short write" << LL_ENDL; + } + } else { + LL_WARNS() << "Short read" << LL_ENDL; + } + } + } + + block->mLocation = new_data_location; + + block->mLength = max_size; + + + sync(block); + + unlockData(); + return TRUE; + } + else + { + LL_WARNS() << "VFS: No space (" << max_size << ") to resize existing vfile " << file_id << LL_ENDL; + //dumpMap(); + unlockData(); + dumpStatistics(); + return FALSE; + } + } + } + else + { + // find a free block in the list + LLVFSBlock *free_block = findFreeBlock(max_size); + + if (free_block) + { + if (block) + { + block->mLocation = free_block->mLocation; + block->mLength = max_size; + } + else + { + // this file doesn't exist, create it + block = new LLVFSFileBlock(file_id, file_type, free_block->mLocation, max_size); + mFileBlocks.insert(fileblock_map::value_type(spec, block)); + } + + // Must call useFreeSpace before sync(), as sync() + // unlocks data structures. + useFreeSpace(free_block, max_size); + block->mAccessTime = (U32)time(NULL); + + sync(block); + } + else + { + LL_WARNS() << "VFS: No space (" << max_size << ") for new virtual file " << file_id << LL_ENDL; + //dumpMap(); + unlockData(); + dumpStatistics(); + return FALSE; + } + } + unlockData(); + return TRUE; +} + + +// WARNING: HERE BE DRAGONS! +// rename is the weirdest VFS op, because the file moves but the locks don't! +void LLVFS::renameFile(const LLUUID &file_id, const LLAssetType::EType file_type, + const LLUUID &new_id, const LLAssetType::EType &new_type) +{ + if (!isValid()) + { + LL_ERRS() << "Attempting to use invalid VFS!" << LL_ENDL; + } + if (mReadOnly) + { + LL_ERRS() << "Attempt to write to read-only VFS" << LL_ENDL; + } + + lockData(); + + LLVFSFileSpecifier new_spec(new_id, new_type); + LLVFSFileSpecifier old_spec(file_id, file_type); + + fileblock_map::iterator it = mFileBlocks.find(old_spec); + if (it != mFileBlocks.end()) + { + LLVFSFileBlock *src_block = (*it).second; + + // this will purge the data but leave the file block in place, w/ locks, if any + // WAS: removeFile(new_id, new_type); NOW uses removeFileBlock() to avoid mutex lock recursion + fileblock_map::iterator new_it = mFileBlocks.find(new_spec); + if (new_it != mFileBlocks.end()) + { + LLVFSFileBlock *new_block = (*new_it).second; + removeFileBlock(new_block); + } + + // if there's something in the target location, remove it but inherit its locks + it = mFileBlocks.find(new_spec); + if (it != mFileBlocks.end()) + { + LLVFSFileBlock *dest_block = (*it).second; + + for (S32 i = 0; i < (S32)VFSLOCK_COUNT; i++) + { + if(dest_block->mLocks[i]) + { + LL_ERRS() << "Renaming VFS block to a locked file." << LL_ENDL; + } + dest_block->mLocks[i] = src_block->mLocks[i]; + } + + mFileBlocks.erase(new_spec); + delete dest_block; + } + + src_block->mFileID = new_id; + src_block->mFileType = new_type; + src_block->mAccessTime = (U32)time(NULL); + + mFileBlocks.erase(old_spec); + mFileBlocks.insert(fileblock_map::value_type(new_spec, src_block)); + + sync(src_block); + } + else + { + LL_WARNS() << "VFS: Attempt to rename nonexistent vfile " << file_id << ":" << file_type << LL_ENDL; + } + unlockData(); +} + +// mDataMutex must be LOCKED before calling this +void LLVFS::removeFileBlock(LLVFSFileBlock *fileblock) +{ + // convert this into an unsaved, dummy fileblock to preserve locks + // a more rubust solution would store the locks in a seperate data structure + sync(fileblock, TRUE); + + if (fileblock->mLength > 0) + { + // turn this file into an empty block + LLVFSBlock *free_block = new LLVFSBlock(fileblock->mLocation, fileblock->mLength); + + addFreeBlock(free_block); + } + + fileblock->mLocation = 0; + fileblock->mSize = 0; + fileblock->mLength = BLOCK_LENGTH_INVALID; + fileblock->mIndexLocation = -1; + + //mergeFreeBlocks(); +} + +void LLVFS::removeFile(const LLUUID &file_id, const LLAssetType::EType file_type) +{ + if (!isValid()) + { + LL_ERRS() << "Attempting to use invalid VFS!" << LL_ENDL; + } + if (mReadOnly) + { + LL_ERRS() << "Attempt to write to read-only VFS" << LL_ENDL; + } + + lockData(); + + LLVFSFileSpecifier spec(file_id, file_type); + fileblock_map::iterator it = mFileBlocks.find(spec); + if (it != mFileBlocks.end()) + { + LLVFSFileBlock *block = (*it).second; + removeFileBlock(block); + } + else + { + LL_WARNS() << "VFS: attempting to remove nonexistent file " << file_id << " type " << file_type << LL_ENDL; + } + + unlockData(); +} + + +S32 LLVFS::getData(const LLUUID &file_id, const LLAssetType::EType file_type, U8 *buffer, S32 location, S32 length) +{ + S32 bytesread = 0; + + if (!isValid()) + { + LL_ERRS() << "Attempting to use invalid VFS!" << LL_ENDL; + } + llassert(location >= 0); + llassert(length >= 0); + + BOOL do_read = FALSE; + + lockData(); + + LLVFSFileSpecifier spec(file_id, file_type); + fileblock_map::iterator it = mFileBlocks.find(spec); + if (it != mFileBlocks.end()) + { + LLVFSFileBlock *block = (*it).second; + + block->mAccessTime = (U32)time(NULL); + + if (location > block->mSize) + { + LL_WARNS() << "VFS: Attempt to read location " << location << " in file " << file_id << " of length " << block->mSize << LL_ENDL; + } + else + { + if (length > block->mSize - location) + { + length = block->mSize - location; + } + location += block->mLocation; + do_read = TRUE; + } + } + + if (do_read) + { + fseek(mDataFP, location, SEEK_SET); + bytesread = (S32)fread(buffer, 1, length, mDataFP); + } + + unlockData(); + + return bytesread; +} + +S32 LLVFS::storeData(const LLUUID &file_id, const LLAssetType::EType file_type, const U8 *buffer, S32 location, S32 length) +{ + if (!isValid()) + { + LL_ERRS() << "Attempting to use invalid VFS!" << LL_ENDL; + } + if (mReadOnly) + { + LL_ERRS() << "Attempt to write to read-only VFS" << LL_ENDL; + } + + llassert(length > 0); + + lockData(); + + LLVFSFileSpecifier spec(file_id, file_type); + fileblock_map::iterator it = mFileBlocks.find(spec); + if (it != mFileBlocks.end()) + { + LLVFSFileBlock *block = (*it).second; + + S32 in_loc = location; + if (location == -1) + { + location = block->mSize; + } + llassert(location >= 0); + + block->mAccessTime = (U32)time(NULL); + + if (block->mLength == BLOCK_LENGTH_INVALID) + { + // Block was removed, ignore write + LL_WARNS() << "VFS: Attempt to write to invalid block" + << " in file " << file_id + << " location: " << in_loc + << " bytes: " << length + << LL_ENDL; + unlockData(); + return length; + } + else if (location > block->mLength) + { + LL_WARNS() << "VFS: Attempt to write to location " << location + << " in file " << file_id + << " type " << S32(file_type) + << " of size " << block->mSize + << " block length " << block->mLength + << LL_ENDL; + unlockData(); + return length; + } + else + { + if (length > block->mLength - location ) + { + LL_WARNS() << "VFS: Truncating write to virtual file " << file_id << " type " << S32(file_type) << LL_ENDL; + length = block->mLength - location; + } + U32 file_location = location + block->mLocation; + + fseek(mDataFP, file_location, SEEK_SET); + S32 write_len = (S32)fwrite(buffer, 1, length, mDataFP); + if (write_len != length) + { + LL_WARNS() << llformat("VFS Write Error: %d != %d",write_len,length) << LL_ENDL; + } + // fflush(mDataFP); + + if (location + length > block->mSize) + { + block->mSize = location + write_len; + sync(block); + } + unlockData(); + + return write_len; + } + } + else + { + unlockData(); + return 0; + } +} + +void LLVFS::incLock(const LLUUID &file_id, const LLAssetType::EType file_type, EVFSLock lock) +{ + lockData(); + + LLVFSFileSpecifier spec(file_id, file_type); + LLVFSFileBlock *block; + + fileblock_map::iterator it = mFileBlocks.find(spec); + if (it != mFileBlocks.end()) + { + block = (*it).second; + } + else + { + // Create a dummy block which isn't saved + block = new LLVFSFileBlock(file_id, file_type, 0, BLOCK_LENGTH_INVALID); + block->mAccessTime = (U32)time(NULL); + mFileBlocks.insert(fileblock_map::value_type(spec, block)); + } + + block->mLocks[lock]++; + mLockCounts[lock]++; + + unlockData(); +} + +void LLVFS::decLock(const LLUUID &file_id, const LLAssetType::EType file_type, EVFSLock lock) +{ + lockData(); + + LLVFSFileSpecifier spec(file_id, file_type); + fileblock_map::iterator it = mFileBlocks.find(spec); + if (it != mFileBlocks.end()) + { + LLVFSFileBlock *block = (*it).second; + + if (block->mLocks[lock] > 0) + { + block->mLocks[lock]--; + } + else + { + LL_WARNS() << "VFS: Decrementing zero-value lock " << lock << LL_ENDL; + } + mLockCounts[lock]--; + } + + unlockData(); +} + +BOOL LLVFS::isLocked(const LLUUID &file_id, const LLAssetType::EType file_type, EVFSLock lock) +{ + lockData(); + + BOOL res = FALSE; + + LLVFSFileSpecifier spec(file_id, file_type); + fileblock_map::iterator it = mFileBlocks.find(spec); + if (it != mFileBlocks.end()) + { + LLVFSFileBlock *block = (*it).second; + res = (block->mLocks[lock] > 0); + } + + unlockData(); + + return res; +} + +//============================================================================ +// protected +//============================================================================ + +void LLVFS::eraseBlockLength(LLVFSBlock *block) +{ + // find the corresponding map entry in the length map and erase it + S32 length = block->mLength; + blocks_length_map_t::iterator iter = mFreeBlocksByLength.lower_bound(length); + blocks_length_map_t::iterator end = mFreeBlocksByLength.end(); + bool found_block = false; + while(iter != end) + { + LLVFSBlock *tblock = iter->second; + llassert(tblock->mLength == length); // there had -better- be an entry with our length! + if (tblock == block) + { + mFreeBlocksByLength.erase(iter); + found_block = true; + break; + } + ++iter; + } + if(!found_block) + { + LL_ERRS() << "eraseBlock could not find block" << LL_ENDL; + } +} + + +// Remove block from both free lists (by location and by length). +void LLVFS::eraseBlock(LLVFSBlock *block) +{ + eraseBlockLength(block); + // find the corresponding map entry in the location map and erase it + U32 location = block->mLocation; + llverify(mFreeBlocksByLocation.erase(location) == 1); // we should only have one entry per location. +} + + +// Add the region specified by block location and length to the free lists. +// Also incrementally defragment by merging with previous and next free blocks. +void LLVFS::addFreeBlock(LLVFSBlock *block) +{ +#if LL_DEBUG + size_t dbgcount = mFreeBlocksByLocation.count(block->mLocation); + if(dbgcount > 0) + { + LL_ERRS() << "addFreeBlock called with block already in list" << LL_ENDL; + } +#endif + + // Get a pointer to the next free block (by location). + blocks_location_map_t::iterator next_free_it = mFreeBlocksByLocation.lower_bound(block->mLocation); + + // We can merge with previous if it ends at our requested location. + LLVFSBlock* prev_block = NULL; + bool merge_prev = false; + if (next_free_it != mFreeBlocksByLocation.begin()) + { + blocks_location_map_t::iterator prev_free_it = next_free_it; + --prev_free_it; + prev_block = prev_free_it->second; + merge_prev = (prev_block->mLocation + prev_block->mLength == block->mLocation); + } + + // We can merge with next if our block ends at the next block's location. + LLVFSBlock* next_block = NULL; + bool merge_next = false; + if (next_free_it != mFreeBlocksByLocation.end()) + { + next_block = next_free_it->second; + merge_next = (block->mLocation + block->mLength == next_block->mLocation); + } + + if (merge_prev && merge_next) + { + // LL_INFOS() << "VFS merge BOTH" << LL_ENDL; + // Previous block is changing length (a lot), so only need to update length map. + // Next block is going away completely. JC + eraseBlockLength(prev_block); + eraseBlock(next_block); + prev_block->mLength += block->mLength + next_block->mLength; + mFreeBlocksByLength.insert(blocks_length_map_t::value_type(prev_block->mLength, prev_block)); + delete block; + block = NULL; + delete next_block; + next_block = NULL; + } + else if (merge_prev) + { + // LL_INFOS() << "VFS merge previous" << LL_ENDL; + // Previous block is maintaining location, only changing length, + // therefore only need to update the length map. JC + eraseBlockLength(prev_block); + prev_block->mLength += block->mLength; + mFreeBlocksByLength.insert(blocks_length_map_t::value_type(prev_block->mLength, prev_block)); // multimap insert + delete block; + block = NULL; + } + else if (merge_next) + { + // LL_INFOS() << "VFS merge next" << LL_ENDL; + // Next block is changing both location and length, + // so both free lists must update. JC + eraseBlock(next_block); + next_block->mLocation = block->mLocation; + next_block->mLength += block->mLength; + // Don't hint here, next_free_it iterator may be invalid. + mFreeBlocksByLocation.insert(blocks_location_map_t::value_type(next_block->mLocation, next_block)); // multimap insert + mFreeBlocksByLength.insert(blocks_length_map_t::value_type(next_block->mLength, next_block)); // multimap insert + delete block; + block = NULL; + } + else + { + // Can't merge with other free blocks. + // Hint that insert should go near next_free_it. + mFreeBlocksByLocation.insert(next_free_it, blocks_location_map_t::value_type(block->mLocation, block)); // multimap insert + mFreeBlocksByLength.insert(blocks_length_map_t::value_type(block->mLength, block)); // multimap insert + } +} + +// Superceeded by new addFreeBlock which does incremental free space merging. +// Incremental is faster overall. +//void LLVFS::mergeFreeBlocks() +//{ +// if (!isValid()) +// { +// LL_ERRS() << "Attempting to use invalid VFS!" << LL_ENDL; +// } +// // TODO: could we optimize this with hints from the calling code? +// blocks_location_map_t::iterator iter = mFreeBlocksByLocation.begin(); +// blocks_location_map_t::iterator end = mFreeBlocksByLocation.end(); +// LLVFSBlock *first_block = iter->second; +// while(iter != end) +// { +// blocks_location_map_t::iterator first_iter = iter; // save for if we do a merge +// if (++iter == end) +// break; +// LLVFSBlock *second_block = iter->second; +// if (first_block->mLocation + first_block->mLength == second_block->mLocation) +// { +// // remove the first block from the length map +// eraseBlockLength(first_block); +// // merge first_block with second_block, since they're adjacent +// first_block->mLength += second_block->mLength; +// // add the first block to the length map (with the new size) +// mFreeBlocksByLength.insert(blocks_length_map_t::value_type(first_block->mLength, first_block)); // multimap insert +// +// // erase and delete the second block +// eraseBlock(second_block); +// delete second_block; +// +// // reset iterator +// iter = first_iter; // haven't changed first_block, so corresponding iterator is still valid +// end = mFreeBlocksByLocation.end(); +// } +// first_block = second_block; +// } +//} + +// length bytes from free_block are going to be used (so they are no longer free) +void LLVFS::useFreeSpace(LLVFSBlock *free_block, S32 length) +{ + if (free_block->mLength == length) + { + eraseBlock(free_block); + delete free_block; + } + else + { + eraseBlock(free_block); + + free_block->mLocation += length; + free_block->mLength -= length; + + addFreeBlock(free_block); + } +} + +// NOTE! mDataMutex must be LOCKED before calling this +// sync this index entry out to the index file +// we need to do this constantly to avoid corruption on viewer crash +void LLVFS::sync(LLVFSFileBlock *block, BOOL remove) +{ + if (!isValid()) + { + LL_ERRS() << "Attempting to use invalid VFS!" << LL_ENDL; + } + if (mReadOnly) + { + LL_WARNS() << "Attempt to sync read-only VFS" << LL_ENDL; + return; + } + if (block->mLength == BLOCK_LENGTH_INVALID) + { + // This is a dummy file, don't save + return; + } + if (block->mLength == 0) + { + LL_ERRS() << "VFS syncing zero-length block" << LL_ENDL; + } + + BOOL set_index_to_end = FALSE; + long seek_pos = block->mIndexLocation; + + if (-1 == seek_pos) + { + if (!mIndexHoles.empty()) + { + seek_pos = mIndexHoles.front(); + mIndexHoles.pop_front(); + } + else + { + set_index_to_end = TRUE; + } + } + + if (set_index_to_end) + { + // Need fseek/ftell to update the seek_pos and hence data + // structures, so can't unlockData() before this. + fseek(mIndexFP, 0, SEEK_END); + seek_pos = ftell(mIndexFP); + } + + block->mIndexLocation = seek_pos; + if (remove) + { + mIndexHoles.push_back(seek_pos); + } + + U8 buffer[LLVFSFileBlock::SERIAL_SIZE]; + if (remove) + { + memset(buffer, 0, LLVFSFileBlock::SERIAL_SIZE); + } + else + { + block->serialize(buffer); + } + + // If set_index_to_end, file pointer is already at seek_pos + // and we don't need to do anything. Only seek if not at end. + if (!set_index_to_end) + { + fseek(mIndexFP, seek_pos, SEEK_SET); + } + + if (fwrite(buffer, LLVFSFileBlock::SERIAL_SIZE, 1, mIndexFP) != 1) + { + LL_WARNS() << "Short write" << LL_ENDL; + } + + // *NOTE: Why was this commented out? + // fflush(mIndexFP); + + return; +} + +// mDataMutex must be LOCKED before calling this +// Can initiate LRU-based file removal to make space. +// The immune file block will not be removed. +LLVFSBlock *LLVFS::findFreeBlock(S32 size, LLVFSFileBlock *immune) +{ + if (!isValid()) + { + LL_ERRS() << "Attempting to use invalid VFS!" << LL_ENDL; + } + + LLVFSBlock *block = NULL; + BOOL have_lru_list = FALSE; + + typedef std::set<LLVFSFileBlock*, LLVFSFileBlock_less> lru_set; + lru_set lru_list; + + LLTimer timer; + + while (! block) + { + // look for a suitable free block + blocks_length_map_t::iterator iter = mFreeBlocksByLength.lower_bound(size); // first entry >= size + if (iter != mFreeBlocksByLength.end()) + block = iter->second; + + // no large enough free blocks, time to clean out some junk + if (! block) + { + // create a list of files sorted by usage time + // this is far faster than sorting a linked list + if (! have_lru_list) + { + for (fileblock_map::iterator it = mFileBlocks.begin(); it != mFileBlocks.end(); ++it) + { + LLVFSFileBlock *tmp = (*it).second; + + if (tmp != immune && + tmp->mLength > 0 && + ! tmp->mLocks[VFSLOCK_READ] && + ! tmp->mLocks[VFSLOCK_APPEND] && + ! tmp->mLocks[VFSLOCK_OPEN]) + { + lru_list.insert(tmp); + } + } + + have_lru_list = TRUE; + } + + if (lru_list.size() == 0) + { + // No more files to delete, and still not enough room! + LL_WARNS() << "VFS: Can't make " << size << " bytes of free space in VFS, giving up" << LL_ENDL; + break; + } + + // is the oldest file big enough? (Should be about half the time) + lru_set::iterator it = lru_list.begin(); + LLVFSFileBlock *file_block = *it; + if (file_block->mLength >= size && file_block != immune) + { + // ditch this file and look again for a free block - should find it + // TODO: it'll be faster just to assign the free block and break + LL_INFOS() << "LRU: Removing " << file_block->mFileID << ":" << file_block->mFileType << LL_ENDL; + lru_list.erase(it); + removeFileBlock(file_block); + file_block = NULL; + continue; + } + + + LL_INFOS() << "VFS: LRU: Aggressive: " << (S32)lru_list.size() << " files remain" << LL_ENDL; + dumpLockCounts(); + + // Now it's time to aggressively make more space + // Delete the oldest 5MB of the vfs or enough to hold the file, which ever is larger + // This may yield too much free space, but we'll use it up soon enough + U32 cleanup_target = (size > VFS_CLEANUP_SIZE) ? size : VFS_CLEANUP_SIZE; + U32 cleaned_up = 0; + for (it = lru_list.begin(); + it != lru_list.end() && cleaned_up < cleanup_target; + ) + { + file_block = *it; + + // TODO: it would be great to be able to batch all these sync() calls + // LL_INFOS() << "LRU2: Removing " << file_block->mFileID << ":" << file_block->mFileType << " last accessed" << file_block->mAccessTime << LL_ENDL; + + cleaned_up += file_block->mLength; + lru_list.erase(it++); + removeFileBlock(file_block); + file_block = NULL; + } + //mergeFreeBlocks(); + } + } + + F32 time = timer.getElapsedTimeF32(); + if (time > 0.5f) + { + LL_WARNS() << "VFS: Spent " << time << " seconds in findFreeBlock!" << LL_ENDL; + } + + return block; +} + +//============================================================================ +// public +//============================================================================ + +void LLVFS::pokeFiles() +{ + if (!isValid()) + { + LL_ERRS() << "Attempting to use invalid VFS!" << LL_ENDL; + } + U32 word; + + // only write data if we actually read 4 bytes + // otherwise we're writing garbage and screwing up the file + fseek(mDataFP, 0, SEEK_SET); + if (fread(&word, sizeof(word), 1, mDataFP) == 1) + { + fseek(mDataFP, 0, SEEK_SET); + if (fwrite(&word, sizeof(word), 1, mDataFP) != 1) + { + LL_WARNS() << "Could not write to data file" << LL_ENDL; + } + fflush(mDataFP); + } + + fseek(mIndexFP, 0, SEEK_SET); + if (fread(&word, sizeof(word), 1, mIndexFP) == 1) + { + fseek(mIndexFP, 0, SEEK_SET); + if (fwrite(&word, sizeof(word), 1, mIndexFP) != 1) + { + LL_WARNS() << "Could not write to index file" << LL_ENDL; + } + fflush(mIndexFP); + } +} + + +void LLVFS::dumpMap() +{ + LL_INFOS() << "Files:" << LL_ENDL; + for (fileblock_map::iterator it = mFileBlocks.begin(); it != mFileBlocks.end(); ++it) + { + LLVFSFileBlock *file_block = (*it).second; + LL_INFOS() << "Location: " << file_block->mLocation << "\tLength: " << file_block->mLength << "\t" << file_block->mFileID << "\t" << file_block->mFileType << LL_ENDL; + } + + LL_INFOS() << "Free Blocks:" << LL_ENDL; + for (blocks_location_map_t::iterator iter = mFreeBlocksByLocation.begin(), + end = mFreeBlocksByLocation.end(); + iter != end; iter++) + { + LLVFSBlock *free_block = iter->second; + LL_INFOS() << "Location: " << free_block->mLocation << "\tLength: " << free_block->mLength << LL_ENDL; + } +} + +// verify that the index file contents match the in-memory file structure +// Very slow, do not call routinely. JC +void LLVFS::audit() +{ + // Lock the mutex through this whole function. + LLMutexLock lock_data(mDataMutex); + + fflush(mIndexFP); + + fseek(mIndexFP, 0, SEEK_END); + size_t index_size = ftell(mIndexFP); + fseek(mIndexFP, 0, SEEK_SET); + + BOOL vfs_corrupt = FALSE; + + // since we take the address of element 0, we need to have at least one element. + std::vector<U8> buffer(llmax<size_t>(index_size,1U)); + + if (fread(&buffer[0], 1, index_size, mIndexFP) != index_size) + { + LL_WARNS() << "Index truncated" << LL_ENDL; + vfs_corrupt = TRUE; + } + + size_t buf_offset = 0; + + std::map<LLVFSFileSpecifier, LLVFSFileBlock*> found_files; + U32 cur_time = (U32)time(NULL); + + std::vector<LLVFSFileBlock*> audit_blocks; + while (!vfs_corrupt && buf_offset < index_size) + { + LLVFSFileBlock *block = new LLVFSFileBlock(); + audit_blocks.push_back(block); + + block->deserialize(&buffer[buf_offset], (S32)buf_offset); + buf_offset += block->SERIAL_SIZE; + + // do sanity check on this block + if (block->mLength >= 0 && + block->mSize >= 0 && + block->mSize <= block->mLength && + block->mFileType >= LLAssetType::AT_NONE && + block->mFileType < LLAssetType::AT_COUNT && + block->mAccessTime <= cur_time && + block->mFileID != LLUUID::null) + { + if (mFileBlocks.find(*block) == mFileBlocks.end()) + { + LL_WARNS() << "VFile " << block->mFileID << ":" << block->mFileType << " on disk, not in memory, loc " << block->mIndexLocation << LL_ENDL; + } + else if (found_files.find(*block) != found_files.end()) + { + std::map<LLVFSFileSpecifier, LLVFSFileBlock*>::iterator it; + it = found_files.find(*block); + LLVFSFileBlock* dupe = it->second; + // try to keep data from being lost + unlockAndClose(mIndexFP); + mIndexFP = NULL; + unlockAndClose(mDataFP); + mDataFP = NULL; + LL_WARNS() << "VFS: Original block index " << block->mIndexLocation + << " location " << block->mLocation + << " length " << block->mLength + << " size " << block->mSize + << " id " << block->mFileID + << " type " << block->mFileType + << LL_ENDL; + LL_WARNS() << "VFS: Duplicate block index " << dupe->mIndexLocation + << " location " << dupe->mLocation + << " length " << dupe->mLength + << " size " << dupe->mSize + << " id " << dupe->mFileID + << " type " << dupe->mFileType + << LL_ENDL; + LL_WARNS() << "VFS: Index size " << index_size << LL_ENDL; + LL_WARNS() << "VFS: INDEX CORRUPT" << LL_ENDL; + vfs_corrupt = TRUE; + break; + } + else + { + found_files[*block] = block; + } + } + else + { + if (block->mLength) + { + LL_WARNS() << "VFile " << block->mFileID << ":" << block->mFileType << " corrupt on disk" << LL_ENDL; + } + // else this is just a hole + } + } + + if (!vfs_corrupt) + { + for (fileblock_map::iterator it = mFileBlocks.begin(); it != mFileBlocks.end(); ++it) + { + LLVFSFileBlock* block = (*it).second; + + if (block->mSize > 0) + { + if (! found_files.count(*block)) + { + LL_WARNS() << "VFile " << block->mFileID << ":" << block->mFileType << " in memory, not on disk, loc " << block->mIndexLocation<< LL_ENDL; + fseek(mIndexFP, block->mIndexLocation, SEEK_SET); + U8 buf[LLVFSFileBlock::SERIAL_SIZE]; + if (fread(buf, LLVFSFileBlock::SERIAL_SIZE, 1, mIndexFP) != 1) + { + LL_WARNS() << "VFile " << block->mFileID + << " gave short read" << LL_ENDL; + } + + LLVFSFileBlock disk_block; + disk_block.deserialize(buf, block->mIndexLocation); + + LL_WARNS() << "Instead found " << disk_block.mFileID << ":" << block->mFileType << LL_ENDL; + } + else + { + block = found_files.find(*block)->second; + found_files.erase(*block); + } + } + } + + for (std::map<LLVFSFileSpecifier, LLVFSFileBlock*>::iterator iter = found_files.begin(); + iter != found_files.end(); iter++) + { + LLVFSFileBlock* block = iter->second; + LL_WARNS() << "VFile " << block->mFileID << ":" << block->mFileType << " szie:" << block->mSize << " leftover" << LL_ENDL; + } + + LL_INFOS() << "VFS: audit OK" << LL_ENDL; + // mutex released by LLMutexLock() destructor. + } + + for_each(audit_blocks.begin(), audit_blocks.end(), DeletePointer()); + audit_blocks.clear(); +} + + +// quick check for uninitialized blocks +// Slow, do not call in release. +void LLVFS::checkMem() +{ + lockData(); + + for (fileblock_map::iterator it = mFileBlocks.begin(); it != mFileBlocks.end(); ++it) + { + LLVFSFileBlock *block = (*it).second; + llassert(block->mFileType >= LLAssetType::AT_NONE && + block->mFileType < LLAssetType::AT_COUNT && + block->mFileID != LLUUID::null); + + for (std::deque<S32>::iterator iter = mIndexHoles.begin(); + iter != mIndexHoles.end(); ++iter) + { + S32 index_loc = *iter; + if (index_loc == block->mIndexLocation) + { + LL_WARNS() << "VFile block " << block->mFileID << ":" << block->mFileType << " is marked as a hole" << LL_ENDL; + } + } + } + + LL_INFOS() << "VFS: mem check OK" << LL_ENDL; + + unlockData(); +} + +void LLVFS::dumpLockCounts() +{ + S32 i; + for (i = 0; i < VFSLOCK_COUNT; i++) + { + LL_INFOS() << "LockType: " << i << ": " << mLockCounts[i] << LL_ENDL; + } +} + +void LLVFS::dumpStatistics() +{ + lockData(); + + // Investigate file blocks. + std::map<S32, S32> size_counts; + std::map<U32, S32> location_counts; + std::map<LLAssetType::EType, std::pair<S32,S32> > filetype_counts; + + S32 max_file_size = 0; + S32 total_file_size = 0; + S32 invalid_file_count = 0; + for (fileblock_map::iterator it = mFileBlocks.begin(); it != mFileBlocks.end(); ++it) + { + LLVFSFileBlock *file_block = (*it).second; + if (file_block->mLength == BLOCK_LENGTH_INVALID) + { + invalid_file_count++; + } + else if (file_block->mLength <= 0) + { + LL_INFOS() << "Bad file block at: " << file_block->mLocation << "\tLength: " << file_block->mLength << "\t" << file_block->mFileID << "\t" << file_block->mFileType << LL_ENDL; + size_counts[file_block->mLength]++; + location_counts[file_block->mLocation]++; + } + else + { + total_file_size += file_block->mLength; + } + + if (file_block->mLength > max_file_size) + { + max_file_size = file_block->mLength; + } + + filetype_counts[file_block->mFileType].first++; + filetype_counts[file_block->mFileType].second += file_block->mLength; + } + + for (std::map<S32,S32>::iterator it = size_counts.begin(); it != size_counts.end(); ++it) + { + S32 size = it->first; + S32 size_count = it->second; + LL_INFOS() << "Bad files size " << size << " count " << size_count << LL_ENDL; + } + for (std::map<U32,S32>::iterator it = location_counts.begin(); it != location_counts.end(); ++it) + { + U32 location = it->first; + S32 location_count = it->second; + LL_INFOS() << "Bad files location " << location << " count " << location_count << LL_ENDL; + } + + // Investigate free list. + S32 max_free_size = 0; + S32 total_free_size = 0; + std::map<S32, S32> free_length_counts; + for (blocks_location_map_t::iterator iter = mFreeBlocksByLocation.begin(), + end = mFreeBlocksByLocation.end(); + iter != end; iter++) + { + LLVFSBlock *free_block = iter->second; + if (free_block->mLength <= 0) + { + LL_INFOS() << "Bad free block at: " << free_block->mLocation << "\tLength: " << free_block->mLength << LL_ENDL; + } + else + { + LL_INFOS() << "Block: " << free_block->mLocation + << "\tLength: " << free_block->mLength + << "\tEnd: " << free_block->mLocation + free_block->mLength + << LL_ENDL; + total_free_size += free_block->mLength; + } + + if (free_block->mLength > max_free_size) + { + max_free_size = free_block->mLength; + } + + free_length_counts[free_block->mLength]++; + } + + // Dump histogram of free block sizes + for (std::map<S32,S32>::iterator it = free_length_counts.begin(); it != free_length_counts.end(); ++it) + { + LL_INFOS() << "Free length " << it->first << " count " << it->second << LL_ENDL; + } + + LL_INFOS() << "Invalid blocks: " << invalid_file_count << LL_ENDL; + LL_INFOS() << "File blocks: " << mFileBlocks.size() << LL_ENDL; + + S32 length_list_count = (S32)mFreeBlocksByLength.size(); + S32 location_list_count = (S32)mFreeBlocksByLocation.size(); + if (length_list_count == location_list_count) + { + LL_INFOS() << "Free list lengths match, free blocks: " << location_list_count << LL_ENDL; + } + else + { + LL_WARNS() << "Free list lengths do not match!" << LL_ENDL; + LL_WARNS() << "By length: " << length_list_count << LL_ENDL; + LL_WARNS() << "By location: " << location_list_count << LL_ENDL; + } + LL_INFOS() << "Max file: " << max_file_size/1024 << "K" << LL_ENDL; + LL_INFOS() << "Max free: " << max_free_size/1024 << "K" << LL_ENDL; + LL_INFOS() << "Total file size: " << total_file_size/1024 << "K" << LL_ENDL; + LL_INFOS() << "Total free size: " << total_free_size/1024 << "K" << LL_ENDL; + LL_INFOS() << "Sum: " << (total_file_size + total_free_size) << " bytes" << LL_ENDL; + LL_INFOS() << llformat("%.0f%% full",((F32)(total_file_size)/(F32)(total_file_size+total_free_size))*100.f) << LL_ENDL; + + LL_INFOS() << " " << LL_ENDL; + for (std::map<LLAssetType::EType, std::pair<S32,S32> >::iterator iter = filetype_counts.begin(); + iter != filetype_counts.end(); ++iter) + { + LL_INFOS() << "Type: " << LLAssetType::getDesc(iter->first) + << " Count: " << iter->second.first + << " Bytes: " << (iter->second.second>>20) << " MB" << LL_ENDL; + } + + // Look for potential merges + { + blocks_location_map_t::iterator iter = mFreeBlocksByLocation.begin(); + blocks_location_map_t::iterator end = mFreeBlocksByLocation.end(); + LLVFSBlock *first_block = iter->second; + while(iter != end) + { + if (++iter == end) + break; + LLVFSBlock *second_block = iter->second; + if (first_block->mLocation + first_block->mLength == second_block->mLocation) + { + LL_INFOS() << "Potential merge at " << first_block->mLocation << LL_ENDL; + } + first_block = second_block; + } + } + unlockData(); +} + +// Debug Only! +std::string get_extension(LLAssetType::EType type) +{ + std::string extension; + switch(type) + { + case LLAssetType::AT_TEXTURE: + extension = ".jp2"; // formerly ".j2c" + break; + case LLAssetType::AT_SOUND: + extension = ".ogg"; + break; + case LLAssetType::AT_SOUND_WAV: + extension = ".wav"; + break; + case LLAssetType::AT_TEXTURE_TGA: + extension = ".tga"; + break; + case LLAssetType::AT_ANIMATION: + extension = ".lla"; + break; + case LLAssetType::AT_MESH: + extension = ".slm"; + break; + default: + // Just use the asset server filename extension in most cases + extension += "."; + extension += LLAssetType::lookup(type); + break; + } + return extension; +} + +void LLVFS::listFiles() +{ + lockData(); + + for (fileblock_map::iterator it = mFileBlocks.begin(); it != mFileBlocks.end(); ++it) + { + LLVFSFileSpecifier file_spec = it->first; + LLVFSFileBlock *file_block = it->second; + S32 length = file_block->mLength; + S32 size = file_block->mSize; + if (length != BLOCK_LENGTH_INVALID && size > 0) + { + LLUUID id = file_spec.mFileID; + std::string extension = get_extension(file_spec.mFileType); + LL_INFOS() << " File: " << id + << " Type: " << LLAssetType::getDesc(file_spec.mFileType) + << " Size: " << size + << LL_ENDL; + } + } + + unlockData(); +} + +#include "llapr.h" +void LLVFS::dumpFiles() +{ + lockData(); + + S32 files_extracted = 0; + for (fileblock_map::iterator it = mFileBlocks.begin(); it != mFileBlocks.end(); ++it) + { + LLVFSFileSpecifier file_spec = it->first; + LLVFSFileBlock *file_block = it->second; + S32 length = file_block->mLength; + S32 size = file_block->mSize; + if (length != BLOCK_LENGTH_INVALID && size > 0) + { + LLUUID id = file_spec.mFileID; + LLAssetType::EType type = file_spec.mFileType; + std::vector<U8> buffer(size); + + unlockData(); + getData(id, type, &buffer[0], 0, size); + lockData(); + + std::string extension = get_extension(type); + std::string filename = id.asString() + extension; + LL_INFOS() << " Writing " << filename << LL_ENDL; + + LLAPRFile outfile; + outfile.open(filename, LL_APR_WB); + outfile.write(&buffer[0], size); + outfile.close(); + + files_extracted++; + } + } + + unlockData(); + + LL_INFOS() << "Extracted " << files_extracted << " files out of " << mFileBlocks.size() << LL_ENDL; +} + +time_t LLVFS::creationTime() +{ + llstat data_file_stat; + int errors = LLFile::stat(mDataFilename, &data_file_stat); + if (0 == errors) + { + time_t creation_time = data_file_stat.st_ctime; +#if LL_DARWIN + creation_time = data_file_stat.st_birthtime; +#endif + return creation_time; + } + return 0; +} + +//============================================================================ +// protected +//============================================================================ + +// static +LLFILE *LLVFS::openAndLock(const std::string& filename, const char* mode, BOOL read_lock) +{ +#if LL_WINDOWS + + return LLFile::_fsopen(filename, mode, (read_lock ? _SH_DENYWR : _SH_DENYRW)); + +#else + + LLFILE *fp; + int fd; + + // first test the lock in a non-destructive way +#if LL_SOLARIS + struct flock fl; + fl.l_whence = SEEK_SET; + fl.l_start = 0; + fl.l_len = 1; +#else // !LL_SOLARIS + if (strchr(mode, 'w') != NULL) + { + fp = LLFile::fopen(filename, "rb"); /* Flawfinder: ignore */ + if (fp) + { + fd = fileno(fp); + if (flock(fd, (read_lock ? LOCK_SH : LOCK_EX) | LOCK_NB) == -1) + { + fclose(fp); + return NULL; + } + + fclose(fp); + } + } +#endif // !LL_SOLARIS + + // now actually open the file for use + fp = LLFile::fopen(filename, mode); /* Flawfinder: ignore */ + if (fp) + { + fd = fileno(fp); +#if LL_SOLARIS + fl.l_type = read_lock ? F_RDLCK : F_WRLCK; + if (fcntl(fd, F_SETLK, &fl) == -1) +#else + if (flock(fd, (read_lock ? LOCK_SH : LOCK_EX) | LOCK_NB) == -1) +#endif + { + fclose(fp); + fp = NULL; + } + } + + return fp; + +#endif +} + +// static +void LLVFS::unlockAndClose(LLFILE *fp) +{ + if (fp) + { + // IW: we don't actually want to unlock on linux + // this is because a forked process can kill the parent's lock + // with an explicit unlock + // however, fclose() will implicitly remove the lock + // but only once both parent and child have closed the file + /* + #if !LL_WINDOWS + int fd = fileno(fp); + flock(fd, LOCK_UN); + #endif + */ +#if LL_SOLARIS + struct flock fl; + fl.l_whence = SEEK_SET; + fl.l_start = 0; + fl.l_len = 1; + fl.l_type = F_UNLCK; + fcntl(fileno(fp), F_SETLK, &fl); +#endif + fclose(fp); + } +} diff --git a/indra/llvfs/llvfs.h b/indra/llvfs/llvfs.h new file mode 100644 index 0000000000..42feafe20b --- /dev/null +++ b/indra/llvfs/llvfs.h @@ -0,0 +1,183 @@ +/** + * @file llvfs.h + * @brief Definition of virtual file system + * + * $LicenseInfo:firstyear=2002&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2010, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef LL_LLVFS_H +#define LL_LLVFS_H + +#include <deque> +#include "lluuid.h" +#include "llassettype.h" +#include "llthread.h" +#include "llmutex.h" + +enum EVFSValid +{ + VFSVALID_UNKNOWN = 0, + VFSVALID_OK = 1, + VFSVALID_BAD_CORRUPT = 2, + VFSVALID_BAD_CANNOT_OPEN_READONLY = 3, + VFSVALID_BAD_CANNOT_CREATE = 4 +}; + +// Lock types for open vfiles, pending async reads, and pending async appends +// (There are no async normal writes, currently) +enum EVFSLock +{ + VFSLOCK_OPEN = 0, + VFSLOCK_READ = 1, + VFSLOCK_APPEND = 2, + + VFSLOCK_COUNT = 3 +}; + +// internal classes +class LLVFSBlock; +class LLVFSFileBlock; +class LLVFSFileSpecifier +{ +public: + LLVFSFileSpecifier(); + LLVFSFileSpecifier(const LLUUID &file_id, const LLAssetType::EType file_type); + bool operator<(const LLVFSFileSpecifier &rhs) const; + bool operator==(const LLVFSFileSpecifier &rhs) const; + +public: + LLUUID mFileID; + LLAssetType::EType mFileType; +}; + +class LLVFS +{ +private: + // Use createLLVFS() to open a VFS file + // Pass 0 to not presize + LLVFS(const std::string& index_filename, + const std::string& data_filename, + const BOOL read_only, + const U32 presize, + const BOOL remove_after_crash); +public: + ~LLVFS(); + + // Use this function normally to create LLVFS files + // Pass 0 to not presize + static LLVFS * createLLVFS(const std::string& index_filename, + const std::string& data_filename, + const BOOL read_only, + const U32 presize, + const BOOL remove_after_crash); + + BOOL isValid() const { return (VFSVALID_OK == mValid); } + EVFSValid getValidState() const { return mValid; } + + // ---------- The following fucntions lock/unlock mDataMutex ---------- + BOOL getExists(const LLUUID &file_id, const LLAssetType::EType file_type); + S32 getSize(const LLUUID &file_id, const LLAssetType::EType file_type); + + BOOL checkAvailable(S32 max_size); + + S32 getMaxSize(const LLUUID &file_id, const LLAssetType::EType file_type); + BOOL setMaxSize(const LLUUID &file_id, const LLAssetType::EType file_type, S32 max_size); + + void renameFile(const LLUUID &file_id, const LLAssetType::EType file_type, + const LLUUID &new_id, const LLAssetType::EType &new_type); + void removeFile(const LLUUID &file_id, const LLAssetType::EType file_type); + + S32 getData(const LLUUID &file_id, const LLAssetType::EType file_type, U8 *buffer, S32 location, S32 length); + S32 storeData(const LLUUID &file_id, const LLAssetType::EType file_type, const U8 *buffer, S32 location, S32 length); + + void incLock(const LLUUID &file_id, const LLAssetType::EType file_type, EVFSLock lock); + void decLock(const LLUUID &file_id, const LLAssetType::EType file_type, EVFSLock lock); + BOOL isLocked(const LLUUID &file_id, const LLAssetType::EType file_type, EVFSLock lock); + // ---------------------------------------------------------------- + + // Used to trigger evil WinXP behavior of "preloading" entire file into memory. + void pokeFiles(); + + // Verify that the index file contents match the in-memory file structure + // Very slow, do not call routinely. JC + void audit(); + // Check for uninitialized blocks. Slow, do not call in release. JC + void checkMem(); + // for debugging, prints a map of the vfs + void dumpMap(); + void dumpLockCounts(); + void dumpStatistics(); + void listFiles(); + void dumpFiles(); + time_t creationTime(); + +protected: + void removeFileBlock(LLVFSFileBlock *fileblock); + + void eraseBlockLength(LLVFSBlock *block); + void eraseBlock(LLVFSBlock *block); + void addFreeBlock(LLVFSBlock *block); + //void mergeFreeBlocks(); + void useFreeSpace(LLVFSBlock *free_block, S32 length); + void sync(LLVFSFileBlock *block, BOOL remove = FALSE); + void presizeDataFile(const U32 size); + + static LLFILE *openAndLock(const std::string& filename, const char* mode, BOOL read_lock); + static void unlockAndClose(FILE *fp); + + // Can initiate LRU-based file removal to make space. + // The immune file block will not be removed. + LLVFSBlock *findFreeBlock(S32 size, LLVFSFileBlock *immune = NULL); + + // lock/unlock data mutex (mDataMutex) + void lockData() { mDataMutex->lock(); } + void unlockData() { mDataMutex->unlock(); } + +protected: + LLMutex* mDataMutex; + + typedef std::map<LLVFSFileSpecifier, LLVFSFileBlock*> fileblock_map; + fileblock_map mFileBlocks; + + typedef std::multimap<S32, LLVFSBlock*> blocks_length_map_t; + blocks_length_map_t mFreeBlocksByLength; + typedef std::multimap<U32, LLVFSBlock*> blocks_location_map_t; + blocks_location_map_t mFreeBlocksByLocation; + + LLFILE *mDataFP; + LLFILE *mIndexFP; + + std::deque<S32> mIndexHoles; + + std::string mIndexFilename; + std::string mDataFilename; + BOOL mReadOnly; + + EVFSValid mValid; + + S32 mLockCounts[VFSLOCK_COUNT]; + BOOL mRemoveAfterCrash; +}; + +extern LLVFS *gVFS; + +#endif diff --git a/indra/llvfs/llvfs_objc.h b/indra/llvfs/llvfs_objc.h new file mode 100644 index 0000000000..56cdbebfc5 --- /dev/null +++ b/indra/llvfs/llvfs_objc.h @@ -0,0 +1,43 @@ +/** + * @file llvfs_objc.h + * @brief Definition of directory utilities class for Mac OS X + * + * $LicenseInfo:firstyear=2000&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2010, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#if !LL_DARWIN +#error This header must not be included when compiling for any target other than Mac OS. Consider including lldir.h instead. +#endif // !LL_DARWIN + +#ifndef LL_LLVFS_OBJC_H +#define LL_LLVFS_OBJC_H + +#include <iostream> + +std::string* getSystemTempFolder(); +std::string* getSystemCacheFolder(); +std::string* getSystemApplicationSupportFolder(); +std::string* getSystemResourceFolder(); +std::string* getSystemExecutableFolder(); + + +#endif // LL_LLVFS_OBJC_H diff --git a/indra/llvfs/llvfs_objc.mm b/indra/llvfs/llvfs_objc.mm new file mode 100644 index 0000000000..282ea41339 --- /dev/null +++ b/indra/llvfs/llvfs_objc.mm @@ -0,0 +1,108 @@ +/** + * @file llvfs_objc.cpp + * @brief Cocoa implementation of directory utilities for Mac OS X + * + * $LicenseInfo:firstyear=2002&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2010, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ +#if LL_DARWIN + +//WARNING: This file CANNOT use standard linden includes due to conflicts between definitions of BOOL + +#include "llvfs_objc.h" +#import <Cocoa/Cocoa.h> + +std::string* getSystemTempFolder() +{ + NSAutoreleasePool *pool = [[NSAutoreleasePool alloc] init]; + NSString * tempDir = NSTemporaryDirectory(); + if (tempDir == nil) + tempDir = @"/tmp"; + std::string *result = ( new std::string([tempDir UTF8String]) ); + [pool release]; + + return result; +} + +//findSystemDirectory scoped exclusively to this file. +std::string* findSystemDirectory(NSSearchPathDirectory searchPathDirectory, + NSSearchPathDomainMask domainMask) +{ + NSAutoreleasePool *pool = [[NSAutoreleasePool alloc] init]; + + std::string *result = nil; + NSString *path = nil; + + // Search for the path + NSArray* paths = NSSearchPathForDirectoriesInDomains(searchPathDirectory, + domainMask, + YES); + if ([paths count]) + { + path = [paths objectAtIndex:0]; + //HACK: Always attempt to create directory, ignore errors. + NSError *error = nil; + + [[NSFileManager defaultManager] createDirectoryAtPath:path withIntermediateDirectories:YES attributes:nil error:&error]; + + + result = new std::string([path UTF8String]); + } + [pool release]; + return result; +} + +std::string* getSystemExecutableFolder() +{ + NSAutoreleasePool *pool = [[NSAutoreleasePool alloc] init]; + + NSString *bundlePath = [[NSBundle mainBundle] executablePath]; + std::string *result = (new std::string([bundlePath UTF8String])); + [pool release]; + + return result; +} + +std::string* getSystemResourceFolder() +{ + NSAutoreleasePool *pool = [[NSAutoreleasePool alloc] init]; + + NSString *bundlePath = [[NSBundle mainBundle] resourcePath]; + std::string *result = (new std::string([bundlePath UTF8String])); + [pool release]; + + return result; +} + +std::string* getSystemCacheFolder() +{ + return findSystemDirectory (NSCachesDirectory, + NSUserDomainMask); +} + +std::string* getSystemApplicationSupportFolder() +{ + return findSystemDirectory (NSApplicationSupportDirectory, + NSUserDomainMask); + +} + +#endif // LL_DARWIN diff --git a/indra/llvfs/llvfsthread.cpp b/indra/llvfs/llvfsthread.cpp new file mode 100644 index 0000000000..8cd85929e2 --- /dev/null +++ b/indra/llvfs/llvfsthread.cpp @@ -0,0 +1,300 @@ +/** + * @file llvfsthread.cpp + * @brief LLVFSThread implementation + * + * $LicenseInfo:firstyear=2001&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2010, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include "linden_common.h" +#include "llvfsthread.h" +#include "llstl.h" + +//============================================================================ + +/*static*/ std::string LLVFSThread::sDataPath = ""; + +/*static*/ LLVFSThread* LLVFSThread::sLocal = NULL; + +//============================================================================ +// Run on MAIN thread +//static +void LLVFSThread::initClass(bool local_is_threaded) +{ + llassert(sLocal == NULL); + sLocal = new LLVFSThread(local_is_threaded); +} + +//static +S32 LLVFSThread::updateClass(U32 ms_elapsed) +{ + sLocal->update((F32)ms_elapsed); + return sLocal->getPending(); +} + +//static +void LLVFSThread::cleanupClass() +{ + sLocal->setQuitting(); + while (sLocal->getPending()) + { + sLocal->update(0); + } + delete sLocal; + sLocal = 0; +} + +//---------------------------------------------------------------------------- + +LLVFSThread::LLVFSThread(bool threaded) : + LLQueuedThread("VFS", threaded) +{ +} + +LLVFSThread::~LLVFSThread() +{ + // ~LLQueuedThread() will be called here +} + +//---------------------------------------------------------------------------- + +LLVFSThread::handle_t LLVFSThread::read(LLVFS* vfs, const LLUUID &file_id, const LLAssetType::EType file_type, + U8* buffer, S32 offset, S32 numbytes, U32 priority, U32 flags) +{ + handle_t handle = generateHandle(); + + priority = llmax(priority, (U32)PRIORITY_LOW); // All reads are at least PRIORITY_LOW + Request* req = new Request(handle, priority, flags, FILE_READ, vfs, file_id, file_type, + buffer, offset, numbytes); + + bool res = addRequest(req); + if (!res) + { + LL_ERRS() << "LLVFSThread::read called after LLVFSThread::cleanupClass()" << LL_ENDL; + req->deleteRequest(); + handle = nullHandle(); + } + + return handle; +} + +S32 LLVFSThread::readImmediate(LLVFS* vfs, const LLUUID &file_id, const LLAssetType::EType file_type, + U8* buffer, S32 offset, S32 numbytes) +{ + handle_t handle = generateHandle(); + + Request* req = new Request(handle, PRIORITY_IMMEDIATE, 0, FILE_READ, vfs, file_id, file_type, + buffer, offset, numbytes); + + S32 res = addRequest(req) ? 1 : 0; + if (res == 0) + { + LL_ERRS() << "LLVFSThread::read called after LLVFSThread::cleanupClass()" << LL_ENDL; + req->deleteRequest(); + } + else + { + llverify(waitForResult(handle, false) == true); + res = req->getBytesRead(); + completeRequest(handle); + } + return res; +} + +LLVFSThread::handle_t LLVFSThread::write(LLVFS* vfs, const LLUUID &file_id, const LLAssetType::EType file_type, + U8* buffer, S32 offset, S32 numbytes, U32 flags) +{ + handle_t handle = generateHandle(); + + Request* req = new Request(handle, 0, flags, FILE_WRITE, vfs, file_id, file_type, + buffer, offset, numbytes); + + bool res = addRequest(req); + if (!res) + { + LL_ERRS() << "LLVFSThread::read called after LLVFSThread::cleanupClass()" << LL_ENDL; + req->deleteRequest(); + handle = nullHandle(); + } + + return handle; +} + +S32 LLVFSThread::writeImmediate(LLVFS* vfs, const LLUUID &file_id, const LLAssetType::EType file_type, + U8* buffer, S32 offset, S32 numbytes) +{ + handle_t handle = generateHandle(); + + Request* req = new Request(handle, PRIORITY_IMMEDIATE, 0, FILE_WRITE, vfs, file_id, file_type, + buffer, offset, numbytes); + + S32 res = addRequest(req) ? 1 : 0; + if (res == 0) + { + LL_ERRS() << "LLVFSThread::read called after LLVFSThread::cleanupClass()" << LL_ENDL; + req->deleteRequest(); + } + else + { + llverify(waitForResult(handle, false) == true); + res = req->getBytesRead(); + completeRequest(handle); + } + return res; +} + + +// LLVFSThread::handle_t LLVFSThread::rename(LLVFS* vfs, const LLUUID &file_id, const LLAssetType::EType file_type, +// const LLUUID &new_id, const LLAssetType::EType new_type, U32 flags) +// { +// handle_t handle = generateHandle(); + +// LLUUID* new_idp = new LLUUID(new_id); // deleted with Request +// // new_type is passed as "numbytes" +// Request* req = new Request(handle, 0, flags, FILE_RENAME, vfs, file_id, file_type, +// (U8*)new_idp, 0, (S32)new_type); + +// bool res = addRequest(req); +// if (!res) +// { +// LL_ERRS() << "LLVFSThread::read called after LLVFSThread::cleanupClass()" << LL_ENDL; +// req->deleteRequest(); +// handle = nullHandle(); +// } + +// return handle; +// } + +//============================================================================ + +LLVFSThread::Request::Request(handle_t handle, U32 priority, U32 flags, + operation_t op, LLVFS* vfs, + const LLUUID &file_id, const LLAssetType::EType file_type, + U8* buffer, S32 offset, S32 numbytes) : + QueuedRequest(handle, priority, flags), + mOperation(op), + mVFS(vfs), + mFileID(file_id), + mFileType(file_type), + mBuffer(buffer), + mOffset(offset), + mBytes(numbytes), + mBytesRead(0) +{ + llassert(mBuffer); + + if (numbytes <= 0 && mOperation != FILE_RENAME) + { + LL_WARNS() << "LLVFSThread: Request with numbytes = " << numbytes + << " operation = " << op + << " offset " << offset + << " file_type " << file_type << LL_ENDL; + } + if (mOperation == FILE_WRITE) + { + S32 blocksize = mVFS->getMaxSize(mFileID, mFileType); + if (blocksize < 0) + { + LL_WARNS() << "VFS write to temporary block (shouldn't happen)" << LL_ENDL; + } + mVFS->incLock(mFileID, mFileType, VFSLOCK_APPEND); + } + else if (mOperation == FILE_RENAME) + { + mVFS->incLock(mFileID, mFileType, VFSLOCK_APPEND); + } + else // if (mOperation == FILE_READ) + { + mVFS->incLock(mFileID, mFileType, VFSLOCK_READ); + } +} + +// dec locks as soon as a request finishes +void LLVFSThread::Request::finishRequest(bool completed) +{ + if (mOperation == FILE_WRITE) + { + mVFS->decLock(mFileID, mFileType, VFSLOCK_APPEND); + } + else if (mOperation == FILE_RENAME) + { + mVFS->decLock(mFileID, mFileType, VFSLOCK_APPEND); + } + else // if (mOperation == FILE_READ) + { + mVFS->decLock(mFileID, mFileType, VFSLOCK_READ); + } +} + +void LLVFSThread::Request::deleteRequest() +{ + if (getStatus() == STATUS_QUEUED) + { + LL_ERRS() << "Attempt to delete a queued LLVFSThread::Request!" << LL_ENDL; + } + if (mOperation == FILE_WRITE) + { + if (mFlags & FLAG_AUTO_DELETE) + { + delete [] mBuffer; + } + } + else if (mOperation == FILE_RENAME) + { + LLUUID* new_idp = (LLUUID*)mBuffer; + delete new_idp; + } + LLQueuedThread::QueuedRequest::deleteRequest(); +} + +bool LLVFSThread::Request::processRequest() +{ + bool complete = false; + if (mOperation == FILE_READ) + { + llassert(mOffset >= 0); + mBytesRead = mVFS->getData(mFileID, mFileType, mBuffer, mOffset, mBytes); + complete = true; + //LL_INFOS() << llformat("LLVFSThread::READ '%s': %d bytes arg:%d",getFilename(),mBytesRead) << LL_ENDL; + } + else if (mOperation == FILE_WRITE) + { + mBytesRead = mVFS->storeData(mFileID, mFileType, mBuffer, mOffset, mBytes); + complete = true; + //LL_INFOS() << llformat("LLVFSThread::WRITE '%s': %d bytes arg:%d",getFilename(),mBytesRead) << LL_ENDL; + } + else if (mOperation == FILE_RENAME) + { + LLUUID* new_idp = (LLUUID*)mBuffer; + LLAssetType::EType new_type = (LLAssetType::EType)mBytes; + mVFS->renameFile(mFileID, mFileType, *new_idp, new_type); + mFileID = *new_idp; + complete = true; + //LL_INFOS() << llformat("LLVFSThread::RENAME '%s': %d bytes arg:%d",getFilename(),mBytesRead) << LL_ENDL; + } + else + { + LL_ERRS() << llformat("LLVFSThread::unknown operation: %d", mOperation) << LL_ENDL; + } + return complete; +} + +//============================================================================ diff --git a/indra/llvfs/llvfsthread.h b/indra/llvfs/llvfsthread.h new file mode 100644 index 0000000000..7814de4a2d --- /dev/null +++ b/indra/llvfs/llvfsthread.h @@ -0,0 +1,140 @@ +/** + * @file llvfsthread.h + * @brief LLVFSThread definition + * + * $LicenseInfo:firstyear=2001&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2010, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef LL_LLVFSTHREAD_H +#define LL_LLVFSTHREAD_H + +#include <queue> +#include <string> +#include <map> +#include <set> + +#include "llqueuedthread.h" + +#include "llvfs.h" + +//============================================================================ + +class LLVFSThread : public LLQueuedThread +{ + //------------------------------------------------------------------------ +public: + enum operation_t { + FILE_READ, + FILE_WRITE, + FILE_RENAME + }; + + //------------------------------------------------------------------------ +public: + + class Request : public QueuedRequest + { + protected: + ~Request() {}; // use deleteRequest() + + public: + Request(handle_t handle, U32 priority, U32 flags, + operation_t op, LLVFS* vfs, + const LLUUID &file_id, const LLAssetType::EType file_type, + U8* buffer, S32 offset, S32 numbytes); + + S32 getBytesRead() + { + return mBytesRead; + } + S32 getOperation() + { + return mOperation; + } + U8* getBuffer() + { + return mBuffer; + } + LLVFS* getVFS() + { + return mVFS; + } + std::string getFilename() + { + std::string tstring; + mFileID.toString(tstring); + return tstring; + } + + /*virtual*/ bool processRequest(); + /*virtual*/ void finishRequest(bool completed); + /*virtual*/ void deleteRequest(); + + private: + operation_t mOperation; + + LLVFS* mVFS; + LLUUID mFileID; + LLAssetType::EType mFileType; + + U8* mBuffer; // dest for reads, source for writes, new UUID for rename + S32 mOffset; // offset into file, -1 = append (WRITE only) + S32 mBytes; // bytes to read from file, -1 = all (new mFileType for rename) + S32 mBytesRead; // bytes read from file + }; + + //------------------------------------------------------------------------ +public: + static std::string sDataPath; + static LLVFSThread* sLocal; // Default worker thread + +public: + LLVFSThread(bool threaded = TRUE); + ~LLVFSThread(); + + // Return a Request handle + handle_t read(LLVFS* vfs, const LLUUID &file_id, const LLAssetType::EType file_type, /* Flawfinder: ignore */ + U8* buffer, S32 offset, S32 numbytes, U32 pri=PRIORITY_NORMAL, U32 flags = 0); + handle_t write(LLVFS* vfs, const LLUUID &file_id, const LLAssetType::EType file_type, + U8* buffer, S32 offset, S32 numbytes, U32 flags); + // SJB: rename seems to have issues, especially when threaded +// handle_t rename(LLVFS* vfs, const LLUUID &file_id, const LLAssetType::EType file_type, +// const LLUUID &new_id, const LLAssetType::EType new_type, U32 flags); + // Return number of bytes read + S32 readImmediate(LLVFS* vfs, const LLUUID &file_id, const LLAssetType::EType file_type, + U8* buffer, S32 offset, S32 numbytes); + S32 writeImmediate(LLVFS* vfs, const LLUUID &file_id, const LLAssetType::EType file_type, + U8* buffer, S32 offset, S32 numbytes); + + /*virtual*/ bool processRequest(QueuedRequest* req); + +public: + static void initClass(bool local_is_threaded = TRUE); // Setup sLocal + static S32 updateClass(U32 ms_elapsed); + static void cleanupClass(); // Delete sLocal + static void setDataPath(const std::string& path) { sDataPath = path; } +}; + +//============================================================================ + + +#endif // LL_LLVFSTHREAD_H diff --git a/indra/llvfs/tests/lldir_test.cpp b/indra/llvfs/tests/lldir_test.cpp new file mode 100644 index 0000000000..3cff622a4b --- /dev/null +++ b/indra/llvfs/tests/lldir_test.cpp @@ -0,0 +1,767 @@ +/** + * @file lldir_test.cpp + * @date 2008-05 + * @brief LLDir test cases. + * + * $LicenseInfo:firstyear=2008&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2010, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include "linden_common.h" + +#include "llstring.h" +#include "tests/StringVec.h" +#include "../lldir.h" +#include "../lldiriterator.h" + +#include "../test/lltut.h" +#include "stringize.h" +#include <boost/foreach.hpp> +#include <boost/assign/list_of.hpp> + +using boost::assign::list_of; + +// We use ensure_equals(..., vec(list_of(...))) not because it's functionally +// required, but because ensure_equals() knows how to format a StringVec. +// Turns out that when ensure_equals() displays a test failure with just +// list_of("string")("another"), you see 'stringanother' vs. '("string", +// "another")'. +StringVec vec(const StringVec& v) +{ + return v; +} + +// For some tests, use a dummy LLDir that uses memory data instead of touching +// the filesystem +struct LLDir_Dummy: public LLDir +{ + /*----------------------------- LLDir API ------------------------------*/ + LLDir_Dummy() + { + // Initialize important LLDir data members based on the filesystem + // data below. + mDirDelimiter = "/"; + mExecutableDir = "install"; + mExecutableFilename = "test"; + mExecutablePathAndName = add(mExecutableDir, mExecutableFilename); + mWorkingDir = mExecutableDir; + mAppRODataDir = "install"; + mSkinBaseDir = add(mAppRODataDir, "skins"); + mOSUserDir = "user"; + mOSUserAppDir = mOSUserDir; + mLindenUserDir = ""; + + // Make the dummy filesystem look more or less like what we expect in + // the real one. + static const char* preload[] = + { + // We group these fixture-data pathnames by basename, rather than + // sorting by full path as you might expect, because the outcome + // of each test strongly depends on which skins/languages provide + // a given basename. + "install/skins/default/colors.xml", + "install/skins/steam/colors.xml", + "user/skins/default/colors.xml", + "user/skins/steam/colors.xml", + + "install/skins/default/xui/en/strings.xml", + "install/skins/default/xui/fr/strings.xml", + "install/skins/steam/xui/en/strings.xml", + "install/skins/steam/xui/fr/strings.xml", + "user/skins/default/xui/en/strings.xml", + "user/skins/default/xui/fr/strings.xml", + "user/skins/steam/xui/en/strings.xml", + "user/skins/steam/xui/fr/strings.xml", + + "install/skins/default/xui/en/floater.xml", + "install/skins/default/xui/fr/floater.xml", + "user/skins/default/xui/fr/floater.xml", + + "install/skins/default/xui/en/newfile.xml", + "install/skins/default/xui/fr/newfile.xml", + "user/skins/default/xui/en/newfile.xml", + + "install/skins/default/html/en-us/welcome.html", + "install/skins/default/html/fr/welcome.html", + + "install/skins/default/textures/only_default.jpeg", + "install/skins/steam/textures/only_steam.jpeg", + "user/skins/default/textures/only_user_default.jpeg", + "user/skins/steam/textures/only_user_steam.jpeg", + + "install/skins/default/future/somefile.txt" + }; + BOOST_FOREACH(const char* path, preload) + { + buildFilesystem(path); + } + } + + virtual ~LLDir_Dummy() {} + + virtual void initAppDirs(const std::string& app_name, const std::string& app_read_only_data_dir) + { + // Implement this when we write a test that needs it + } + + virtual std::string getCurPath() + { + // Implement this when we write a test that needs it + return ""; + } + + virtual U32 countFilesInDir(const std::string& dirname, const std::string& mask) + { + // Implement this when we write a test that needs it + return 0; + } + + virtual bool fileExists(const std::string& pathname) const + { + // Record fileExists() calls so we can check whether caching is + // working right. Certain LLDir calls should be able to make decisions + // without calling fileExists() again, having already checked existence. + mChecked.insert(pathname); + // For our simple flat set of strings, see whether the identical + // pathname exists in our set. + return (mFilesystem.find(pathname) != mFilesystem.end()); + } + + virtual std::string getLLPluginLauncher() + { + // Implement this when we write a test that needs it + return ""; + } + + virtual std::string getLLPluginFilename(std::string base_name) + { + // Implement this when we write a test that needs it + return ""; + } + + /*----------------------------- Dummy data -----------------------------*/ + void clearFilesystem() { mFilesystem.clear(); } + void buildFilesystem(const std::string& path) + { + // Split the pathname on slashes, ignoring leading, trailing, doubles + StringVec components; + LLStringUtil::getTokens(path, components, "/"); + // Ensure we have an entry representing every level of this path + std::string partial; + BOOST_FOREACH(std::string component, components) + { + append(partial, component); + mFilesystem.insert(partial); + } + } + + void clear_checked() { mChecked.clear(); } + void ensure_checked(const std::string& pathname) const + { + tut::ensure(STRINGIZE(pathname << " was not checked but should have been"), + mChecked.find(pathname) != mChecked.end()); + } + void ensure_not_checked(const std::string& pathname) const + { + tut::ensure(STRINGIZE(pathname << " was checked but should not have been"), + mChecked.find(pathname) == mChecked.end()); + } + + std::set<std::string> mFilesystem; + mutable std::set<std::string> mChecked; +}; + +namespace tut +{ + struct LLDirTest + { + }; + typedef test_group<LLDirTest> LLDirTest_t; + typedef LLDirTest_t::object LLDirTest_object_t; + tut::LLDirTest_t tut_LLDirTest("LLDir"); + + template<> template<> + void LLDirTest_object_t::test<1>() + // getDirDelimiter + { + ensure("getDirDelimiter", !gDirUtilp->getDirDelimiter().empty()); + } + + template<> template<> + void LLDirTest_object_t::test<2>() + // getBaseFileName + { + std::string delim = gDirUtilp->getDirDelimiter(); + std::string rawFile = "foo"; + std::string rawFileExt = "foo.bAr"; + std::string rawFileNullExt = "foo."; + std::string rawExt = ".bAr"; + std::string rawDot = "."; + std::string pathNoExt = "aa" + delim + "bb" + delim + "cc" + delim + "dd" + delim + "ee"; + std::string pathExt = pathNoExt + ".eXt"; + std::string dottedPathNoExt = "aa" + delim + "bb" + delim + "cc.dd" + delim + "ee"; + std::string dottedPathExt = dottedPathNoExt + ".eXt"; + + // foo[.bAr] + + ensure_equals("getBaseFileName/r-no-ext/no-strip-exten", + gDirUtilp->getBaseFileName(rawFile, false), + "foo"); + + ensure_equals("getBaseFileName/r-no-ext/strip-exten", + gDirUtilp->getBaseFileName(rawFile, true), + "foo"); + + ensure_equals("getBaseFileName/r-ext/no-strip-exten", + gDirUtilp->getBaseFileName(rawFileExt, false), + "foo.bAr"); + + ensure_equals("getBaseFileName/r-ext/strip-exten", + gDirUtilp->getBaseFileName(rawFileExt, true), + "foo"); + + // foo. + + ensure_equals("getBaseFileName/rn-no-ext/no-strip-exten", + gDirUtilp->getBaseFileName(rawFileNullExt, false), + "foo."); + + ensure_equals("getBaseFileName/rn-no-ext/strip-exten", + gDirUtilp->getBaseFileName(rawFileNullExt, true), + "foo"); + + // .bAr + // interesting case - with no basename, this IS the basename, not the extension. + + ensure_equals("getBaseFileName/e-ext/no-strip-exten", + gDirUtilp->getBaseFileName(rawExt, false), + ".bAr"); + + ensure_equals("getBaseFileName/e-ext/strip-exten", + gDirUtilp->getBaseFileName(rawExt, true), + ".bAr"); + + // . + + ensure_equals("getBaseFileName/d/no-strip-exten", + gDirUtilp->getBaseFileName(rawDot, false), + "."); + + ensure_equals("getBaseFileName/d/strip-exten", + gDirUtilp->getBaseFileName(rawDot, true), + "."); + + // aa/bb/cc/dd/ee[.eXt] + + ensure_equals("getBaseFileName/no-ext/no-strip-exten", + gDirUtilp->getBaseFileName(pathNoExt, false), + "ee"); + + ensure_equals("getBaseFileName/no-ext/strip-exten", + gDirUtilp->getBaseFileName(pathNoExt, true), + "ee"); + + ensure_equals("getBaseFileName/ext/no-strip-exten", + gDirUtilp->getBaseFileName(pathExt, false), + "ee.eXt"); + + ensure_equals("getBaseFileName/ext/strip-exten", + gDirUtilp->getBaseFileName(pathExt, true), + "ee"); + + // aa/bb/cc.dd/ee[.eXt] + + ensure_equals("getBaseFileName/d-no-ext/no-strip-exten", + gDirUtilp->getBaseFileName(dottedPathNoExt, false), + "ee"); + + ensure_equals("getBaseFileName/d-no-ext/strip-exten", + gDirUtilp->getBaseFileName(dottedPathNoExt, true), + "ee"); + + ensure_equals("getBaseFileName/d-ext/no-strip-exten", + gDirUtilp->getBaseFileName(dottedPathExt, false), + "ee.eXt"); + + ensure_equals("getBaseFileName/d-ext/strip-exten", + gDirUtilp->getBaseFileName(dottedPathExt, true), + "ee"); + } + + template<> template<> + void LLDirTest_object_t::test<3>() + // getDirName + { + std::string delim = gDirUtilp->getDirDelimiter(); + std::string rawFile = "foo"; + std::string rawFileExt = "foo.bAr"; + std::string pathNoExt = "aa" + delim + "bb" + delim + "cc" + delim + "dd" + delim + "ee"; + std::string pathExt = pathNoExt + ".eXt"; + std::string dottedPathNoExt = "aa" + delim + "bb" + delim + "cc.dd" + delim + "ee"; + std::string dottedPathExt = dottedPathNoExt + ".eXt"; + + // foo[.bAr] + + ensure_equals("getDirName/r-no-ext", + gDirUtilp->getDirName(rawFile), + ""); + + ensure_equals("getDirName/r-ext", + gDirUtilp->getDirName(rawFileExt), + ""); + + // aa/bb/cc/dd/ee[.eXt] + + ensure_equals("getDirName/no-ext", + gDirUtilp->getDirName(pathNoExt), + "aa" + delim + "bb" + delim + "cc" + delim + "dd"); + + ensure_equals("getDirName/ext", + gDirUtilp->getDirName(pathExt), + "aa" + delim + "bb" + delim + "cc" + delim + "dd"); + + // aa/bb/cc.dd/ee[.eXt] + + ensure_equals("getDirName/d-no-ext", + gDirUtilp->getDirName(dottedPathNoExt), + "aa" + delim + "bb" + delim + "cc.dd"); + + ensure_equals("getDirName/d-ext", + gDirUtilp->getDirName(dottedPathExt), + "aa" + delim + "bb" + delim + "cc.dd"); + } + + template<> template<> + void LLDirTest_object_t::test<4>() + // getExtension + { + std::string delim = gDirUtilp->getDirDelimiter(); + std::string rawFile = "foo"; + std::string rawFileExt = "foo.bAr"; + std::string rawFileNullExt = "foo."; + std::string rawExt = ".bAr"; + std::string rawDot = "."; + std::string pathNoExt = "aa" + delim + "bb" + delim + "cc" + delim + "dd" + delim + "ee"; + std::string pathExt = pathNoExt + ".eXt"; + std::string dottedPathNoExt = "aa" + delim + "bb" + delim + "cc.dd" + delim + "ee"; + std::string dottedPathExt = dottedPathNoExt + ".eXt"; + + // foo[.bAr] + + ensure_equals("getExtension/r-no-ext", + gDirUtilp->getExtension(rawFile), + ""); + + ensure_equals("getExtension/r-ext", + gDirUtilp->getExtension(rawFileExt), + "bar"); + + // foo. + + ensure_equals("getExtension/rn-no-ext", + gDirUtilp->getExtension(rawFileNullExt), + ""); + + // .bAr + // interesting case - with no basename, this IS the basename, not the extension. + + ensure_equals("getExtension/e-ext", + gDirUtilp->getExtension(rawExt), + ""); + + // . + + ensure_equals("getExtension/d", + gDirUtilp->getExtension(rawDot), + ""); + + // aa/bb/cc/dd/ee[.eXt] + + ensure_equals("getExtension/no-ext", + gDirUtilp->getExtension(pathNoExt), + ""); + + ensure_equals("getExtension/ext", + gDirUtilp->getExtension(pathExt), + "ext"); + + // aa/bb/cc.dd/ee[.eXt] + + ensure_equals("getExtension/d-no-ext", + gDirUtilp->getExtension(dottedPathNoExt), + ""); + + ensure_equals("getExtension/d-ext", + gDirUtilp->getExtension(dottedPathExt), + "ext"); + } + + std::string makeTestFile( const std::string& dir, const std::string& file ) + { + std::string path = dir + file; + LLFILE* handle = LLFile::fopen( path, "w" ); + ensure("failed to open test file '"+path+"'", handle != NULL ); + // Harbison & Steele, 4th ed., p. 366: "If an error occurs, fputs + // returns EOF; otherwise, it returns some other, nonnegative value." + ensure("failed to write to test file '"+path+"'", EOF != fputs("test file", handle) ); + fclose(handle); + return path; + } + + std::string makeTestDir( const std::string& dirbase ) + { + int counter; + std::string uniqueDir; + bool foundUnused; + std::string delim = gDirUtilp->getDirDelimiter(); + + for (counter=0, foundUnused=false; !foundUnused; counter++ ) + { + char counterStr[3]; + sprintf(counterStr, "%02d", counter); + uniqueDir = dirbase + counterStr; + foundUnused = ! ( LLFile::isdir(uniqueDir) || LLFile::isfile(uniqueDir) ); + } + ensure("test directory '" + uniqueDir + "' creation failed", !LLFile::mkdir(uniqueDir)); + + return uniqueDir + delim; // HACK - apparently, the trailing delimiter is needed... + } + + static const char* DirScanFilename[5] = { "file1.abc", "file2.abc", "file1.xyz", "file2.xyz", "file1.mno" }; + + void scanTest(const std::string& directory, const std::string& pattern, bool correctResult[5]) + { + + // Scan directory and see if any file1.* files are found + std::string scanResult; + int found = 0; + bool filesFound[5] = { false, false, false, false, false }; + //std::cerr << "searching '"+directory+"' for '"+pattern+"'\n"; + + LLDirIterator iter(directory, pattern); + while ( found <= 5 && iter.next(scanResult) ) + { + found++; + //std::cerr << " found '"+scanResult+"'\n"; + int check; + for (check=0; check < 5 && ! ( scanResult == DirScanFilename[check] ); check++) + { + } + // check is now either 5 (not found) or the index of the matching name + if (check < 5) + { + ensure( "found file '"+(std::string)DirScanFilename[check]+"' twice", ! filesFound[check] ); + filesFound[check] = true; + } + else // check is 5 - should not happen + { + fail( "found unknown file '"+scanResult+"'"); + } + } + for (int i=0; i<5; i++) + { + if (correctResult[i]) + { + ensure("scan of '"+directory+"' using '"+pattern+"' did not return '"+DirScanFilename[i]+"'", filesFound[i]); + } + else + { + ensure("scan of '"+directory+"' using '"+pattern+"' incorrectly returned '"+DirScanFilename[i]+"'", !filesFound[i]); + } + } + } + + template<> template<> + void LLDirTest_object_t::test<5>() + // LLDirIterator::next + { + std::string delim = gDirUtilp->getDirDelimiter(); + std::string dirTemp = LLFile::tmpdir(); + + // Create the same 5 file names of the two directories + + std::string dir1 = makeTestDir(dirTemp + "LLDirIterator"); + std::string dir2 = makeTestDir(dirTemp + "LLDirIterator"); + std::string dir1files[5]; + std::string dir2files[5]; + for (int i=0; i<5; i++) + { + dir1files[i] = makeTestFile(dir1, DirScanFilename[i]); + dir2files[i] = makeTestFile(dir2, DirScanFilename[i]); + } + + // Scan dir1 and see if each of the 5 files is found exactly once + bool expected1[5] = { true, true, true, true, true }; + scanTest(dir1, "*", expected1); + + // Scan dir2 and see if only the 2 *.xyz files are found + bool expected2[5] = { false, false, true, true, false }; + scanTest(dir1, "*.xyz", expected2); + + // Scan dir2 and see if only the 1 *.mno file is found + bool expected3[5] = { false, false, false, false, true }; + scanTest(dir2, "*.mno", expected3); + + // Scan dir1 and see if any *.foo files are found + bool expected4[5] = { false, false, false, false, false }; + scanTest(dir1, "*.foo", expected4); + + // Scan dir1 and see if any file1.* files are found + bool expected5[5] = { true, false, true, false, true }; + scanTest(dir1, "file1.*", expected5); + + // Scan dir1 and see if any file1.* files are found + bool expected6[5] = { true, true, false, false, false }; + scanTest(dir1, "file?.abc", expected6); + + // Scan dir2 and see if any file?.x?z files are found + bool expected7[5] = { false, false, true, true, false }; + scanTest(dir2, "file?.x?z", expected7); + + // Scan dir2 and see if any file?.??c files are found + bool expected8[5] = { true, true, false, false, false }; + scanTest(dir2, "file?.??c", expected8); + scanTest(dir2, "*.??c", expected8); + + // Scan dir1 and see if any *.?n? files are found + bool expected9[5] = { false, false, false, false, true }; + scanTest(dir1, "*.?n?", expected9); + + // Scan dir1 and see if any *.???? files are found + bool expected10[5] = { false, false, false, false, false }; + scanTest(dir1, "*.????", expected10); + + // Scan dir1 and see if any ?????.* files are found + bool expected11[5] = { true, true, true, true, true }; + scanTest(dir1, "?????.*", expected11); + + // Scan dir1 and see if any ??l??.xyz files are found + bool expected12[5] = { false, false, true, true, false }; + scanTest(dir1, "??l??.xyz", expected12); + + bool expected13[5] = { true, false, true, false, false }; + scanTest(dir1, "file1.{abc,xyz}", expected13); + + bool expected14[5] = { true, true, false, false, false }; + scanTest(dir1, "file[0-9].abc", expected14); + + bool expected15[5] = { true, true, false, false, false }; + scanTest(dir1, "file[!a-z].abc", expected15); + + // clean up all test files and directories + for (int i=0; i<5; i++) + { + LLFile::remove(dir1files[i]); + LLFile::remove(dir2files[i]); + } + LLFile::rmdir(dir1); + LLFile::rmdir(dir2); + } + + template<> template<> + void LLDirTest_object_t::test<6>() + { + set_test_name("findSkinnedFilenames()"); + LLDir_Dummy lldir; + /*------------------------ "default", "en" -------------------------*/ + // Setting "default" means we shouldn't consider any "*/skins/steam" + // directories; setting "en" means we shouldn't consider any "xui/fr" + // directories. + lldir.setSkinFolder("default", "en"); + ensure_equals(lldir.getSkinFolder(), "default"); + ensure_equals(lldir.getLanguage(), "en"); + + // top-level directory of a skin isn't localized + ensure_equals(lldir.findSkinnedFilenames(LLDir::SKINBASE, "colors.xml", LLDir::ALL_SKINS), + vec(list_of("install/skins/default/colors.xml") + ("user/skins/default/colors.xml"))); + // We should not have needed to check for skins/default/en. We should + // just "know" that SKINBASE is not localized. + lldir.ensure_not_checked("install/skins/default/en"); + + ensure_equals(lldir.findSkinnedFilenames(LLDir::TEXTURES, "only_default.jpeg"), + vec(list_of("install/skins/default/textures/only_default.jpeg"))); + // Nor should we have needed to check skins/default/textures/en + // because textures is known not to be localized. + lldir.ensure_not_checked("install/skins/default/textures/en"); + + StringVec expected(vec(list_of("install/skins/default/xui/en/strings.xml") + ("user/skins/default/xui/en/strings.xml"))); + ensure_equals(lldir.findSkinnedFilenames(LLDir::XUI, "strings.xml", LLDir::ALL_SKINS), + expected); + // The first time, we had to probe to find out whether xui was localized. + lldir.ensure_checked("install/skins/default/xui/en"); + lldir.clear_checked(); + // Now make the same call again -- should return same result -- + ensure_equals(lldir.findSkinnedFilenames(LLDir::XUI, "strings.xml", LLDir::ALL_SKINS), + expected); + // but this time it should remember that xui is localized. + lldir.ensure_not_checked("install/skins/default/xui/en"); + + // localized subdir with "en-us" instead of "en" + ensure_equals(lldir.findSkinnedFilenames("html", "welcome.html"), + vec(list_of("install/skins/default/html/en-us/welcome.html"))); + lldir.ensure_checked("install/skins/default/html/en"); + lldir.ensure_checked("install/skins/default/html/en-us"); + lldir.clear_checked(); + ensure_equals(lldir.findSkinnedFilenames("html", "welcome.html"), + vec(list_of("install/skins/default/html/en-us/welcome.html"))); + lldir.ensure_not_checked("install/skins/default/html/en"); + lldir.ensure_not_checked("install/skins/default/html/en-us"); + + ensure_equals(lldir.findSkinnedFilenames("future", "somefile.txt"), + vec(list_of("install/skins/default/future/somefile.txt"))); + // Test probing for an unrecognized unlocalized future subdir. + lldir.ensure_checked("install/skins/default/future/en"); + lldir.clear_checked(); + ensure_equals(lldir.findSkinnedFilenames("future", "somefile.txt"), + vec(list_of("install/skins/default/future/somefile.txt"))); + // Second time it should remember that future is unlocalized. + lldir.ensure_not_checked("install/skins/default/future/en"); + + // When language is set to "en", requesting an html file pulls up the + // "en-us" version -- not because it magically matches those strings, + // but because there's no "en" localization and it falls back on the + // default "en-us"! Note that it would probably still be better to + // make the default localization be "en" and allow "en-gb" (or + // whatever) localizations, which would work much more the way you'd + // expect. + ensure_equals(lldir.findSkinnedFilenames("html", "welcome.html"), + vec(list_of("install/skins/default/html/en-us/welcome.html"))); + + /*------------------------ "default", "fr" -------------------------*/ + // We start being able to distinguish localized subdirs from + // unlocalized when we ask for a non-English language. + lldir.setSkinFolder("default", "fr"); + ensure_equals(lldir.getLanguage(), "fr"); + + // pass merge=true to request this filename in all relevant skins + ensure_equals(lldir.findSkinnedFilenames(LLDir::XUI, "strings.xml", LLDir::ALL_SKINS), + vec(list_of + ("install/skins/default/xui/en/strings.xml") + ("install/skins/default/xui/fr/strings.xml") + ("user/skins/default/xui/en/strings.xml") + ("user/skins/default/xui/fr/strings.xml"))); + + // pass (or default) merge=false to request only most specific skin + ensure_equals(lldir.findSkinnedFilenames(LLDir::XUI, "strings.xml"), + vec(list_of + ("user/skins/default/xui/en/strings.xml") + ("user/skins/default/xui/fr/strings.xml"))); + + // Our dummy floater.xml has a user localization (for "fr") but no + // English override. This is a case in which CURRENT_SKIN nonetheless + // returns paths from two different skins. + ensure_equals(lldir.findSkinnedFilenames(LLDir::XUI, "floater.xml"), + vec(list_of + ("install/skins/default/xui/en/floater.xml") + ("user/skins/default/xui/fr/floater.xml"))); + + // Our dummy newfile.xml has an English override but no user + // localization. This is another case in which CURRENT_SKIN + // nonetheless returns paths from two different skins. + ensure_equals(lldir.findSkinnedFilenames(LLDir::XUI, "newfile.xml"), + vec(list_of + ("user/skins/default/xui/en/newfile.xml") + ("install/skins/default/xui/fr/newfile.xml"))); + + ensure_equals(lldir.findSkinnedFilenames("html", "welcome.html"), + vec(list_of + ("install/skins/default/html/en-us/welcome.html") + ("install/skins/default/html/fr/welcome.html"))); + + /*------------------------ "default", "zh" -------------------------*/ + lldir.setSkinFolder("default", "zh"); + // Because strings.xml has only a "fr" override but no "zh" override + // in any skin, the most localized version we can find is "en". + ensure_equals(lldir.findSkinnedFilenames(LLDir::XUI, "strings.xml"), + vec(list_of("user/skins/default/xui/en/strings.xml"))); + + /*------------------------- "steam", "en" --------------------------*/ + lldir.setSkinFolder("steam", "en"); + + ensure_equals(lldir.findSkinnedFilenames(LLDir::SKINBASE, "colors.xml", LLDir::ALL_SKINS), + vec(list_of + ("install/skins/default/colors.xml") + ("install/skins/steam/colors.xml") + ("user/skins/default/colors.xml") + ("user/skins/steam/colors.xml"))); + + ensure_equals(lldir.findSkinnedFilenames(LLDir::TEXTURES, "only_default.jpeg"), + vec(list_of("install/skins/default/textures/only_default.jpeg"))); + + ensure_equals(lldir.findSkinnedFilenames(LLDir::TEXTURES, "only_steam.jpeg"), + vec(list_of("install/skins/steam/textures/only_steam.jpeg"))); + + ensure_equals(lldir.findSkinnedFilenames(LLDir::TEXTURES, "only_user_default.jpeg"), + vec(list_of("user/skins/default/textures/only_user_default.jpeg"))); + + ensure_equals(lldir.findSkinnedFilenames(LLDir::TEXTURES, "only_user_steam.jpeg"), + vec(list_of("user/skins/steam/textures/only_user_steam.jpeg"))); + + // CURRENT_SKIN + ensure_equals(lldir.findSkinnedFilenames(LLDir::XUI, "strings.xml"), + vec(list_of("user/skins/steam/xui/en/strings.xml"))); + + // pass constraint=ALL_SKINS to request this filename in all relevant skins + ensure_equals(lldir.findSkinnedFilenames(LLDir::XUI, "strings.xml", LLDir::ALL_SKINS), + vec(list_of + ("install/skins/default/xui/en/strings.xml") + ("install/skins/steam/xui/en/strings.xml") + ("user/skins/default/xui/en/strings.xml") + ("user/skins/steam/xui/en/strings.xml"))); + + /*------------------------- "steam", "fr" --------------------------*/ + lldir.setSkinFolder("steam", "fr"); + + // pass CURRENT_SKIN to request only the most specialized files + ensure_equals(lldir.findSkinnedFilenames(LLDir::XUI, "strings.xml"), + vec(list_of + ("user/skins/steam/xui/en/strings.xml") + ("user/skins/steam/xui/fr/strings.xml"))); + + // pass ALL_SKINS to request this filename in all relevant skins + ensure_equals(lldir.findSkinnedFilenames(LLDir::XUI, "strings.xml", LLDir::ALL_SKINS), + vec(list_of + ("install/skins/default/xui/en/strings.xml") + ("install/skins/default/xui/fr/strings.xml") + ("install/skins/steam/xui/en/strings.xml") + ("install/skins/steam/xui/fr/strings.xml") + ("user/skins/default/xui/en/strings.xml") + ("user/skins/default/xui/fr/strings.xml") + ("user/skins/steam/xui/en/strings.xml") + ("user/skins/steam/xui/fr/strings.xml"))); + } + + template<> template<> + void LLDirTest_object_t::test<7>() + { + set_test_name("add()"); + LLDir_Dummy lldir; + ensure_equals("both empty", lldir.add("", ""), ""); + ensure_equals("path empty", lldir.add("", "b"), "b"); + ensure_equals("name empty", lldir.add("a", ""), "a"); + ensure_equals("both simple", lldir.add("a", "b"), "a/b"); + ensure_equals("name leading slash", lldir.add("a", "/b"), "a/b"); + ensure_equals("path trailing slash", lldir.add("a/", "b"), "a/b"); + ensure_equals("both bring slashes", lldir.add("a/", "/b"), "a/b"); + } +} diff --git a/indra/llvfs/tests/lldiriterator_test.cpp b/indra/llvfs/tests/lldiriterator_test.cpp new file mode 100644 index 0000000000..a65e3dada5 --- /dev/null +++ b/indra/llvfs/tests/lldiriterator_test.cpp @@ -0,0 +1,65 @@ +/** + * @file lldiriterator_test.cpp + * @date 2011-06 + * @brief LLDirIterator test cases. + * + * $LicenseInfo:firstyear=2011&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2011, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only., + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include "linden_common.h" +#include "lltut.h" +#include "../lldiriterator.h" + + +namespace tut +{ + + struct LLDirIteratorFixture + { + LLDirIteratorFixture() + { + } + }; + typedef test_group<LLDirIteratorFixture> LLDirIteratorTest_factory; + typedef LLDirIteratorTest_factory::object LLDirIteratorTest_t; + LLDirIteratorTest_factory tf("LLDirIterator"); + + /* + CHOP-662 was originally introduced to deal with crashes deleting files from + a directory (VWR-25500). However, this introduced a crash looking for + old chat logs as the glob_to_regex function in lldiriterator wasn't escaping lots of regexp characters + */ + void test_chop_662(void) + { + // Check a selection of bad group names from the crash reports + LLDirIterator iter(".","+bad-group-name]+?\?-??.*"); + LLDirIterator iter1(".","))--@---bad-group-name2((?\?-??.*\\.txt"); + LLDirIterator iter2(".","__^v--x)Cuide d sua vida(x--v^__?\?-??.*"); + } + + template<> template<> + void LLDirIteratorTest_t::test<1>() + { + test_chop_662(); + } + +} |