From 08b933a0c67463f06f124420f16c8a3f7dc83f1f Mon Sep 17 00:00:00 2001 From: Dave Parks Date: Wed, 3 Jul 2024 17:42:24 -0500 Subject: #1870 Tune up for better experience on integrated intel with low memory (#1872) * More deterministic vsize calculation. Add control for choosing downscale method. * Quick hack to make GLTF preview work again --- indra/llfilesystem/llfilesystem.cpp | 1 + 1 file changed, 1 insertion(+) (limited to 'indra/llfilesystem/llfilesystem.cpp') diff --git a/indra/llfilesystem/llfilesystem.cpp b/indra/llfilesystem/llfilesystem.cpp index 235aae0be3..912f48e216 100644 --- a/indra/llfilesystem/llfilesystem.cpp +++ b/indra/llfilesystem/llfilesystem.cpp @@ -79,6 +79,7 @@ LLFileSystem::~LLFileSystem() // static bool LLFileSystem::getExists(const LLUUID& file_id, const LLAssetType::EType file_type) { + LL_PROFILE_ZONE_SCOPED; std::string id_str; file_id.toString(id_str); const std::string extra_info = ""; -- cgit v1.2.3 From 9fdca96f8bd2211a99fe88e57b70cbecefa20b6d Mon Sep 17 00:00:00 2001 From: Ansariel Date: Mon, 8 Jul 2024 20:27:14 +0200 Subject: Re-enable compiler warnings C4244 and C4396 except for lltracerecording.h and llunittype.h for now --- indra/llfilesystem/llfilesystem.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'indra/llfilesystem/llfilesystem.cpp') diff --git a/indra/llfilesystem/llfilesystem.cpp b/indra/llfilesystem/llfilesystem.cpp index 912f48e216..7d2a6bd6f5 100644 --- a/indra/llfilesystem/llfilesystem.cpp +++ b/indra/llfilesystem/llfilesystem.cpp @@ -148,7 +148,7 @@ S32 LLFileSystem::getFileSize(const LLUUID& file_id, const LLAssetType::EType fi if (file.is_open()) { file.seekg(0, std::ios::end); - file_size = file.tellg(); + file_size = (S32)file.tellg(); } return file_size; @@ -176,7 +176,7 @@ bool LLFileSystem::read(U8* buffer, S32 bytes) } else { - mBytesRead = file.gcount(); + mBytesRead = (S32)file.gcount(); } file.close(); @@ -217,7 +217,7 @@ bool LLFileSystem::write(const U8* buffer, S32 bytes) { ofs.write((const char*)buffer, bytes); - mPosition = ofs.tellp(); // Fix asset caching + mPosition = (S32)ofs.tellp(); // Fix asset caching success = true; } -- cgit v1.2.3 From 0ee528c5fdfa8d49ebd43774853880f1b2e352d8 Mon Sep 17 00:00:00 2001 From: Rye Mutt Date: Tue, 13 Aug 2024 18:35:46 -0400 Subject: Mitigate asset fetch thread stalls from LLDiskCache mutex contention and trivial cleanup Move LLDiskCache::updateFileAccessTime to LLFilesystem as it's the only user of that function. Change mCacheDir and LLDiskCache::metaDataToFilepath to statics. --- indra/llfilesystem/llfilesystem.cpp | 90 ++++++++++++++++++++++++++++++------- 1 file changed, 73 insertions(+), 17 deletions(-) (limited to 'indra/llfilesystem/llfilesystem.cpp') diff --git a/indra/llfilesystem/llfilesystem.cpp b/indra/llfilesystem/llfilesystem.cpp index 7d2a6bd6f5..b206aab7cf 100644 --- a/indra/llfilesystem/llfilesystem.cpp +++ b/indra/llfilesystem/llfilesystem.cpp @@ -34,6 +34,8 @@ #include "llfasttimer.h" #include "lldiskcache.h" +#include "boost/filesystem.hpp" + const S32 LLFileSystem::READ = 0x00000001; const S32 LLFileSystem::WRITE = 0x00000002; const S32 LLFileSystem::READ_WRITE = 0x00000003; // LLFileSystem::READ & LLFileSystem::WRITE @@ -56,9 +58,8 @@ LLFileSystem::LLFileSystem(const LLUUID& file_id, const LLAssetType::EType file_ { // build the filename (TODO: we do this in a few places - perhaps we should factor into a single function) std::string id; - mFileID.toString(id); - const std::string extra_info = ""; - const std::string filename = LLDiskCache::getInstance()->metaDataToFilepath(id, mFileType, extra_info); + mFileID.asString(); + const std::string filename = LLDiskCache::metaDataToFilepath(id, mFileType); // update the last access time for the file if it exists - this is required // even though we are reading and not writing because this is the @@ -67,7 +68,7 @@ LLFileSystem::LLFileSystem(const LLUUID& file_id, const LLAssetType::EType file_ bool exists = gDirUtilp->fileExists(filename); if (exists) { - LLDiskCache::getInstance()->updateFileAccessTime(filename); + updateFileAccessTime(filename); } } } @@ -82,8 +83,7 @@ bool LLFileSystem::getExists(const LLUUID& file_id, const LLAssetType::EType fil LL_PROFILE_ZONE_SCOPED; std::string id_str; file_id.toString(id_str); - const std::string extra_info = ""; - const std::string filename = LLDiskCache::getInstance()->metaDataToFilepath(id_str, file_type, extra_info); + const std::string filename = LLDiskCache::metaDataToFilepath(id_str, file_type); llifstream file(filename, std::ios::binary); if (file.is_open()) @@ -99,8 +99,7 @@ bool LLFileSystem::removeFile(const LLUUID& file_id, const LLAssetType::EType fi { std::string id_str; file_id.toString(id_str); - const std::string extra_info = ""; - const std::string filename = LLDiskCache::getInstance()->metaDataToFilepath(id_str, file_type, extra_info); + const std::string filename = LLDiskCache::metaDataToFilepath(id_str, file_type); LLFile::remove(filename.c_str(), suppress_error); @@ -113,12 +112,11 @@ bool LLFileSystem::renameFile(const LLUUID& old_file_id, const LLAssetType::ETyp { std::string old_id_str; old_file_id.toString(old_id_str); - const std::string extra_info = ""; - const std::string old_filename = LLDiskCache::getInstance()->metaDataToFilepath(old_id_str, old_file_type, extra_info); + const std::string old_filename = LLDiskCache::metaDataToFilepath(old_id_str, old_file_type); std::string new_id_str; new_file_id.toString(new_id_str); - const std::string new_filename = LLDiskCache::getInstance()->metaDataToFilepath(new_id_str, new_file_type, extra_info); + const std::string new_filename = LLDiskCache::metaDataToFilepath(new_id_str, new_file_type); // Rename needs the new file to not exist. LLFileSystem::removeFile(new_file_id, new_file_type, ENOENT); @@ -140,8 +138,7 @@ S32 LLFileSystem::getFileSize(const LLUUID& file_id, const LLAssetType::EType fi { std::string id_str; file_id.toString(id_str); - const std::string extra_info = ""; - const std::string filename = LLDiskCache::getInstance()->metaDataToFilepath(id_str, file_type, extra_info); + const std::string filename = LLDiskCache::metaDataToFilepath(id_str, file_type); S32 file_size = 0; llifstream file(filename, std::ios::binary); @@ -160,8 +157,7 @@ bool LLFileSystem::read(U8* buffer, S32 bytes) std::string id; mFileID.toString(id); - const std::string extra_info = ""; - const std::string filename = LLDiskCache::getInstance()->metaDataToFilepath(id, mFileType, extra_info); + const std::string filename = LLDiskCache::metaDataToFilepath(id, mFileType); llifstream file(filename, std::ios::binary); if (file.is_open()) @@ -205,8 +201,7 @@ bool LLFileSystem::write(const U8* buffer, S32 bytes) { std::string id_str; mFileID.toString(id_str); - const std::string extra_info = ""; - const std::string filename = LLDiskCache::getInstance()->metaDataToFilepath(id_str, mFileType, extra_info); + const std::string filename = LLDiskCache::metaDataToFilepath(id_str, mFileType); bool success = false; @@ -325,3 +320,64 @@ bool LLFileSystem::remove() return true; } + +void LLFileSystem::updateFileAccessTime(const std::string& file_path) +{ + /** + * Threshold in time_t units that is used to decide if the last access time + * time of the file is updated or not. Added as a precaution for the concern + * outlined in SL-14582 about frequent writes on older SSDs reducing their + * lifespan. I think this is the right place for the threshold value - rather + * than it being a pref - do comment on that Jira if you disagree... + * + * Let's start with 1 hour in time_t units and see how that unfolds + */ + const std::time_t time_threshold = 1 * 60 * 60; + + // current time + const std::time_t cur_time = std::time(nullptr); + + boost::system::error_code ec; +#if LL_WINDOWS + // file last write time + const std::time_t last_write_time = boost::filesystem::last_write_time(utf8str_to_utf16str(file_path), ec); + if (ec.failed()) + { + LL_WARNS() << "Failed to read last write time for cache file " << file_path << ": " << ec.message() << LL_ENDL; + return; + } + + // delta between cur time and last time the file was written + const std::time_t delta_time = cur_time - last_write_time; + + // we only write the new value if the time in time_threshold has elapsed + // before the last one + if (delta_time > time_threshold) + { + boost::filesystem::last_write_time(utf8str_to_utf16str(file_path), cur_time, ec); + } +#else + // file last write time + const std::time_t last_write_time = boost::filesystem::last_write_time(file_path, ec); + if (ec.failed()) + { + LL_WARNS() << "Failed to read last write time for cache file " << file_path << ": " << ec.message() << LL_ENDL; + return; + } + + // delta between cur time and last time the file was written + const std::time_t delta_time = cur_time - last_write_time; + + // we only write the new value if the time in time_threshold has elapsed + // before the last one + if (delta_time > time_threshold) + { + boost::filesystem::last_write_time(file_path, cur_time, ec); + } +#endif + + if (ec.failed()) + { + LL_WARNS() << "Failed to update last write time for cache file " << file_path << ": " << ec.message() << LL_ENDL; + } +} -- cgit v1.2.3 From d6190bbf13547800c56828d7670944f1a06af7dd Mon Sep 17 00:00:00 2001 From: Ansariel Hiller Date: Mon, 19 Aug 2024 08:39:35 +0200 Subject: Refactor LLFileSystem for and fix an old issue in LLFile (#2332) --- indra/llfilesystem/llfilesystem.cpp | 64 ++++++++++++------------------------- 1 file changed, 20 insertions(+), 44 deletions(-) (limited to 'indra/llfilesystem/llfilesystem.cpp') diff --git a/indra/llfilesystem/llfilesystem.cpp b/indra/llfilesystem/llfilesystem.cpp index b206aab7cf..c8ce9531c2 100644 --- a/indra/llfilesystem/llfilesystem.cpp +++ b/indra/llfilesystem/llfilesystem.cpp @@ -36,10 +36,10 @@ #include "boost/filesystem.hpp" -const S32 LLFileSystem::READ = 0x00000001; -const S32 LLFileSystem::WRITE = 0x00000002; -const S32 LLFileSystem::READ_WRITE = 0x00000003; // LLFileSystem::READ & LLFileSystem::WRITE -const S32 LLFileSystem::APPEND = 0x00000006; // 0x00000004 & LLFileSystem::WRITE +constexpr S32 LLFileSystem::READ = 0x00000001; +constexpr S32 LLFileSystem::WRITE = 0x00000002; +constexpr S32 LLFileSystem::READ_WRITE = 0x00000003; // LLFileSystem::READ & LLFileSystem::WRITE +constexpr S32 LLFileSystem::APPEND = 0x00000006; // 0x00000004 & LLFileSystem::WRITE static LLTrace::BlockTimerStatHandle FTM_VFILE_WAIT("VFile Wait"); @@ -57,9 +57,7 @@ LLFileSystem::LLFileSystem(const LLUUID& file_id, const LLAssetType::EType file_ if (mode == LLFileSystem::READ) { // build the filename (TODO: we do this in a few places - perhaps we should factor into a single function) - std::string id; - mFileID.asString(); - const std::string filename = LLDiskCache::metaDataToFilepath(id, mFileType); + const std::string filename = LLDiskCache::metaDataToFilepath(mFileID, mFileType); // update the last access time for the file if it exists - this is required // even though we are reading and not writing because this is the @@ -73,17 +71,11 @@ LLFileSystem::LLFileSystem(const LLUUID& file_id, const LLAssetType::EType file_ } } -LLFileSystem::~LLFileSystem() -{ -} - // static bool LLFileSystem::getExists(const LLUUID& file_id, const LLAssetType::EType file_type) { LL_PROFILE_ZONE_SCOPED; - std::string id_str; - file_id.toString(id_str); - const std::string filename = LLDiskCache::metaDataToFilepath(id_str, file_type); + const std::string filename = LLDiskCache::metaDataToFilepath(file_id, file_type); llifstream file(filename, std::ios::binary); if (file.is_open()) @@ -97,9 +89,7 @@ bool LLFileSystem::getExists(const LLUUID& file_id, const LLAssetType::EType fil // static bool LLFileSystem::removeFile(const LLUUID& file_id, const LLAssetType::EType file_type, int suppress_error /*= 0*/) { - std::string id_str; - file_id.toString(id_str); - const std::string filename = LLDiskCache::metaDataToFilepath(id_str, file_type); + const std::string filename = LLDiskCache::metaDataToFilepath(file_id, file_type); LLFile::remove(filename.c_str(), suppress_error); @@ -110,13 +100,8 @@ bool LLFileSystem::removeFile(const LLUUID& file_id, const LLAssetType::EType fi bool LLFileSystem::renameFile(const LLUUID& old_file_id, const LLAssetType::EType old_file_type, const LLUUID& new_file_id, const LLAssetType::EType new_file_type) { - std::string old_id_str; - old_file_id.toString(old_id_str); - const std::string old_filename = LLDiskCache::metaDataToFilepath(old_id_str, old_file_type); - - std::string new_id_str; - new_file_id.toString(new_id_str); - const std::string new_filename = LLDiskCache::metaDataToFilepath(new_id_str, new_file_type); + const std::string old_filename = LLDiskCache::metaDataToFilepath(old_file_id, old_file_type); + const std::string new_filename = LLDiskCache::metaDataToFilepath(new_file_id, new_file_type); // Rename needs the new file to not exist. LLFileSystem::removeFile(new_file_id, new_file_type, ENOENT); @@ -127,7 +112,7 @@ bool LLFileSystem::renameFile(const LLUUID& old_file_id, const LLAssetType::ETyp // failed but the original code does not and doing so seems to // break a lot of things so we go with the flow... //return false; - LL_WARNS() << "Failed to rename " << old_file_id << " to " << new_id_str << " reason: " << strerror(errno) << LL_ENDL; + LL_WARNS() << "Failed to rename " << old_file_id << " to " << new_file_id << " reason: " << strerror(errno) << LL_ENDL; } return true; @@ -136,9 +121,7 @@ bool LLFileSystem::renameFile(const LLUUID& old_file_id, const LLAssetType::ETyp // static S32 LLFileSystem::getFileSize(const LLUUID& file_id, const LLAssetType::EType file_type) { - std::string id_str; - file_id.toString(id_str); - const std::string filename = LLDiskCache::metaDataToFilepath(id_str, file_type); + const std::string filename = LLDiskCache::metaDataToFilepath(file_id, file_type); S32 file_size = 0; llifstream file(filename, std::ios::binary); @@ -155,9 +138,7 @@ bool LLFileSystem::read(U8* buffer, S32 bytes) { bool success = false; - std::string id; - mFileID.toString(id); - const std::string filename = LLDiskCache::metaDataToFilepath(id, mFileType); + const std::string filename = LLDiskCache::metaDataToFilepath(mFileID, mFileType); llifstream file(filename, std::ios::binary); if (file.is_open()) @@ -187,21 +168,19 @@ bool LLFileSystem::read(U8* buffer, S32 bytes) return success; } -S32 LLFileSystem::getLastBytesRead() +S32 LLFileSystem::getLastBytesRead() const { return mBytesRead; } -bool LLFileSystem::eof() +bool LLFileSystem::eof() const { return mPosition >= getSize(); } bool LLFileSystem::write(const U8* buffer, S32 bytes) { - std::string id_str; - mFileID.toString(id_str); - const std::string filename = LLDiskCache::metaDataToFilepath(id_str, mFileType); + const std::string filename = LLDiskCache::metaDataToFilepath(mFileID, mFileType); bool success = false; @@ -212,12 +191,11 @@ bool LLFileSystem::write(const U8* buffer, S32 bytes) { ofs.write((const char*)buffer, bytes); - mPosition = (S32)ofs.tellp(); // Fix asset caching + mPosition = (S32)ofs.tellp(); success = true; } } - // Fix asset caching else if (mMode == READ_WRITE) { // Don't truncate if file already exists @@ -241,7 +219,6 @@ bool LLFileSystem::write(const U8* buffer, S32 bytes) } } } - // else { llofstream ofs(filename, std::ios::binary); @@ -293,12 +270,12 @@ S32 LLFileSystem::tell() const return mPosition; } -S32 LLFileSystem::getSize() +S32 LLFileSystem::getSize() const { return LLFileSystem::getFileSize(mFileID, mFileType); } -S32 LLFileSystem::getMaxSize() +S32 LLFileSystem::getMaxSize() const { // offer up a huge size since we don't care what the max is return INT_MAX; @@ -314,10 +291,9 @@ bool LLFileSystem::rename(const LLUUID& new_id, const LLAssetType::EType new_typ return true; } -bool LLFileSystem::remove() +bool LLFileSystem::remove() const { LLFileSystem::removeFile(mFileID, mFileType); - return true; } @@ -332,7 +308,7 @@ void LLFileSystem::updateFileAccessTime(const std::string& file_path) * * Let's start with 1 hour in time_t units and see how that unfolds */ - const std::time_t time_threshold = 1 * 60 * 60; + constexpr std::time_t time_threshold = 1 * 60 * 60; // current time const std::time_t cur_time = std::time(nullptr); -- cgit v1.2.3