From 87908bab6d7a0283e7195566dd99926a4c8401b1 Mon Sep 17 00:00:00 2001 From: Mnikolenko Productengine Date: Thu, 4 Mar 2021 12:23:57 +0200 Subject: SL-14939 Fixed the log spam --- indra/llfilesystem/llfilesystem.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'indra/llfilesystem/llfilesystem.cpp') diff --git a/indra/llfilesystem/llfilesystem.cpp b/indra/llfilesystem/llfilesystem.cpp index 64e0b9f193..053b52014e 100644 --- a/indra/llfilesystem/llfilesystem.cpp +++ b/indra/llfilesystem/llfilesystem.cpp @@ -72,14 +72,14 @@ bool LLFileSystem::getExists(const LLUUID& file_id, const LLAssetType::EType fil } // static -bool LLFileSystem::removeFile(const LLUUID& file_id, const LLAssetType::EType file_type) +bool LLFileSystem::removeFile(const LLUUID& file_id, const LLAssetType::EType file_type, int suppress_error /*= 0*/) { std::string id_str; file_id.toString(id_str); const std::string extra_info = ""; const std::string filename = LLDiskCache::getInstance()->metaDataToFilepath(id_str, file_type, extra_info); - LLFile::remove(filename.c_str()); + LLFile::remove(filename.c_str(), suppress_error); return true; } @@ -98,7 +98,7 @@ bool LLFileSystem::renameFile(const LLUUID& old_file_id, const LLAssetType::ETyp const std::string new_filename = LLDiskCache::getInstance()->metaDataToFilepath(new_id_str, new_file_type, extra_info); // Rename needs the new file to not exist. - LLFileSystem::removeFile(new_file_id, new_file_type); + LLFileSystem::removeFile(new_file_id, new_file_type, ENOENT); if (LLFile::rename(old_filename, new_filename) != 0) { -- cgit v1.2.3 From 168d177197bd7558bbe0ca13d01c984ad8638da7 Mon Sep 17 00:00:00 2001 From: Callum Prentice Date: Tue, 9 Mar 2021 14:39:51 -0800 Subject: This set of changes reverts the merge with master (git revert c83e740) and results in a version of the DRTVWR-519 that matches what was presemt before it was deployed as a release viewer *plus* 3 small fixes from Maxim (See commits). This branch can now be used for additional fixes before eventually being used to release D-519 as normal --- indra/llfilesystem/llfilesystem.cpp | 283 ++++++++++++++++++++++++++++++++++++ 1 file changed, 283 insertions(+) create mode 100644 indra/llfilesystem/llfilesystem.cpp (limited to 'indra/llfilesystem/llfilesystem.cpp') diff --git a/indra/llfilesystem/llfilesystem.cpp b/indra/llfilesystem/llfilesystem.cpp new file mode 100644 index 0000000000..053b52014e --- /dev/null +++ b/indra/llfilesystem/llfilesystem.cpp @@ -0,0 +1,283 @@ +/** + * @file filesystem.h + * @brief Simulate local file system operations. + * @Note The initial implementation does actually use standard C++ + * file operations but eventually, there will be another + * layer that caches and manages file meta data too. + * + * $LicenseInfo:firstyear=2002&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2010, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include "linden_common.h" + +#include "lldir.h" +#include "llfilesystem.h" +#include "llfasttimer.h" +#include "lldiskcache.h" + +const S32 LLFileSystem::READ = 0x00000001; +const S32 LLFileSystem::WRITE = 0x00000002; +const S32 LLFileSystem::READ_WRITE = 0x00000003; // LLFileSystem::READ & LLFileSystem::WRITE +const S32 LLFileSystem::APPEND = 0x00000006; // 0x00000004 & LLFileSystem::WRITE + +static LLTrace::BlockTimerStatHandle FTM_VFILE_WAIT("VFile Wait"); + +LLFileSystem::LLFileSystem(const LLUUID& file_id, const LLAssetType::EType file_type, S32 mode) +{ + mFileType = file_type; + mFileID = file_id; + mPosition = 0; + mBytesRead = 0; + mMode = mode; +} + +LLFileSystem::~LLFileSystem() +{ +} + +// static +bool LLFileSystem::getExists(const LLUUID& file_id, const LLAssetType::EType file_type) +{ + std::string id_str; + file_id.toString(id_str); + const std::string extra_info = ""; + const std::string filename = LLDiskCache::getInstance()->metaDataToFilepath(id_str, file_type, extra_info); + + llifstream file(filename, std::ios::binary); + if (file.is_open()) + { + file.seekg(0, std::ios::end); + return file.tellg() > 0; + } + return false; +} + +// static +bool LLFileSystem::removeFile(const LLUUID& file_id, const LLAssetType::EType file_type, int suppress_error /*= 0*/) +{ + std::string id_str; + file_id.toString(id_str); + const std::string extra_info = ""; + const std::string filename = LLDiskCache::getInstance()->metaDataToFilepath(id_str, file_type, extra_info); + + LLFile::remove(filename.c_str(), suppress_error); + + return true; +} + +// static +bool LLFileSystem::renameFile(const LLUUID& old_file_id, const LLAssetType::EType old_file_type, + const LLUUID& new_file_id, const LLAssetType::EType new_file_type) +{ + std::string old_id_str; + old_file_id.toString(old_id_str); + const std::string extra_info = ""; + const std::string old_filename = LLDiskCache::getInstance()->metaDataToFilepath(old_id_str, old_file_type, extra_info); + + std::string new_id_str; + new_file_id.toString(new_id_str); + const std::string new_filename = LLDiskCache::getInstance()->metaDataToFilepath(new_id_str, new_file_type, extra_info); + + // Rename needs the new file to not exist. + LLFileSystem::removeFile(new_file_id, new_file_type, ENOENT); + + if (LLFile::rename(old_filename, new_filename) != 0) + { + // We would like to return FALSE here indicating the operation + // failed but the original code does not and doing so seems to + // break a lot of things so we go with the flow... + //return FALSE; + LL_WARNS() << "Failed to rename " << old_file_id << " to " << new_id_str << " reason: " << strerror(errno) << LL_ENDL; + } + + return TRUE; +} + +// static +S32 LLFileSystem::getFileSize(const LLUUID& file_id, const LLAssetType::EType file_type) +{ + std::string id_str; + file_id.toString(id_str); + const std::string extra_info = ""; + const std::string filename = LLDiskCache::getInstance()->metaDataToFilepath(id_str, file_type, extra_info); + + S32 file_size = 0; + llifstream file(filename, std::ios::binary); + if (file.is_open()) + { + file.seekg(0, std::ios::end); + file_size = file.tellg(); + } + + return file_size; +} + +BOOL LLFileSystem::read(U8* buffer, S32 bytes) +{ + BOOL success = TRUE; + + std::string id; + mFileID.toString(id); + const std::string extra_info = ""; + const std::string filename = LLDiskCache::getInstance()->metaDataToFilepath(id, mFileType, extra_info); + + llifstream file(filename, std::ios::binary); + if (file.is_open()) + { + file.seekg(mPosition, std::ios::beg); + + file.read((char*)buffer, bytes); + + if (file) + { + mBytesRead = bytes; + } + else + { + mBytesRead = file.gcount(); + } + + file.close(); + + mPosition += mBytesRead; + if (!mBytesRead) + { + success = FALSE; + } + } + + // update the last access time for the file - this is required + // even though we are reading and not writing because this is the + // way the cache works - it relies on a valid "last accessed time" for + // each file so it knows how to remove the oldest, unused files + LLDiskCache::getInstance()->updateFileAccessTime(filename); + + return success; +} + +S32 LLFileSystem::getLastBytesRead() +{ + return mBytesRead; +} + +BOOL LLFileSystem::eof() +{ + return mPosition >= getSize(); +} + +BOOL LLFileSystem::write(const U8* buffer, S32 bytes) +{ + std::string id_str; + mFileID.toString(id_str); + const std::string extra_info = ""; + const std::string filename = LLDiskCache::getInstance()->metaDataToFilepath(id_str, mFileType, extra_info); + + BOOL success = FALSE; + + if (mMode == APPEND) + { + llofstream ofs(filename, std::ios::app | std::ios::binary); + if (ofs) + { + ofs.write((const char*)buffer, bytes); + + success = TRUE; + } + } + else + { + llofstream ofs(filename, std::ios::binary); + if (ofs) + { + ofs.write((const char*)buffer, bytes); + + mPosition += bytes; + + success = TRUE; + } + } + + return success; +} + +BOOL LLFileSystem::seek(S32 offset, S32 origin) +{ + if (-1 == origin) + { + origin = mPosition; + } + + S32 new_pos = origin + offset; + + S32 size = getSize(); + + if (new_pos > size) + { + LL_WARNS() << "Attempt to seek past end of file" << LL_ENDL; + + mPosition = size; + return FALSE; + } + else if (new_pos < 0) + { + LL_WARNS() << "Attempt to seek past beginning of file" << LL_ENDL; + + mPosition = 0; + return FALSE; + } + + mPosition = new_pos; + return TRUE; +} + +S32 LLFileSystem::tell() const +{ + return mPosition; +} + +S32 LLFileSystem::getSize() +{ + return LLFileSystem::getFileSize(mFileID, mFileType); +} + +S32 LLFileSystem::getMaxSize() +{ + // offer up a huge size since we don't care what the max is + return INT_MAX; +} + +BOOL LLFileSystem::rename(const LLUUID& new_id, const LLAssetType::EType new_type) +{ + LLFileSystem::renameFile(mFileID, mFileType, new_id, new_type); + + mFileID = new_id; + mFileType = new_type; + + return TRUE; +} + +BOOL LLFileSystem::remove() +{ + LLFileSystem::removeFile(mFileID, mFileType); + + return TRUE; +} -- cgit v1.2.3 From d7518c7b4f5eca731d0ea143bab34b279bd4ee13 Mon Sep 17 00:00:00 2001 From: Callum Prentice Date: Tue, 9 Mar 2021 18:33:35 -0800 Subject: Ansariel kindly offered their patch to help mitigate this round of file system issues - taken from https://vcs.firestormviewer.org/phoenix-firestorm/changeset/104a8600946be01e2de44d10ad069ba854272d1f --- indra/llfilesystem/llfilesystem.cpp | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) (limited to 'indra/llfilesystem/llfilesystem.cpp') diff --git a/indra/llfilesystem/llfilesystem.cpp b/indra/llfilesystem/llfilesystem.cpp index 053b52014e..da44e8d98c 100644 --- a/indra/llfilesystem/llfilesystem.cpp +++ b/indra/llfilesystem/llfilesystem.cpp @@ -200,9 +200,36 @@ BOOL LLFileSystem::write(const U8* buffer, S32 bytes) { ofs.write((const char*)buffer, bytes); + mPosition = ofs.tellp(); // Fix asset caching + + success = TRUE; + } + } + // Fix asset caching + else if (mMode == READ_WRITE) + { + // Don't truncate if file already exists + llofstream ofs(filename, std::ios::in | std::ios::binary); + if (ofs) + { + ofs.seekp(mPosition, std::ios::beg); + ofs.write((const char*)buffer, bytes); + mPosition += bytes; success = TRUE; } + else + { + // File doesn't exist - open in write mode + ofs.open(filename, std::ios::binary); + if (ofs.is_open()) + { + ofs.write((const char*)buffer, bytes); + mPosition += bytes; + success = TRUE; + } + } } + // else { llofstream ofs(filename, std::ios::binary); -- cgit v1.2.3 From e28c1b46e9944f0215a13cab8ee7dded88d7fc90 Mon Sep 17 00:00:00 2001 From: Callum Prentice Date: Tue, 20 Jul 2021 15:03:22 -0700 Subject: Speculative fix for SL-15547: Viewer hung while looking for a file in the cache - since I am unable to repro, this might be enough --- indra/llfilesystem/llfilesystem.cpp | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) (limited to 'indra/llfilesystem/llfilesystem.cpp') diff --git a/indra/llfilesystem/llfilesystem.cpp b/indra/llfilesystem/llfilesystem.cpp index da44e8d98c..b029723e11 100644 --- a/indra/llfilesystem/llfilesystem.cpp +++ b/indra/llfilesystem/llfilesystem.cpp @@ -165,11 +165,14 @@ BOOL LLFileSystem::read(U8* buffer, S32 bytes) } } - // update the last access time for the file - this is required - // even though we are reading and not writing because this is the - // way the cache works - it relies on a valid "last accessed time" for - // each file so it knows how to remove the oldest, unused files - LLDiskCache::getInstance()->updateFileAccessTime(filename); + if (success == TRUE) + { + // update the last access time for the file - this is required + // even though we are reading and not writing because this is the + // way the cache works - it relies on a valid "last accessed time" for + // each file so it knows how to remove the oldest, unused files + LLDiskCache::getInstance()->updateFileAccessTime(filename); + } return success; } -- cgit v1.2.3 From 484aa963af598a96eca6d9a1715ff1d8ed7ef7b4 Mon Sep 17 00:00:00 2001 From: Callum Prentice Date: Wed, 21 Jul 2021 09:09:02 -0700 Subject: Additional speculative fix for SL-15547: Viewer hung while looking for a file in the cache. I am still unable to reproduce this problem, although others have so I am trying a more aggressive approach for them to test. I tried this as part of last speculative fix but saw what I thought were scary side effects in the log/console. Turns out, those are there in the release viewer so probably unrelated to this change. --- indra/llfilesystem/llfilesystem.cpp | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'indra/llfilesystem/llfilesystem.cpp') diff --git a/indra/llfilesystem/llfilesystem.cpp b/indra/llfilesystem/llfilesystem.cpp index b029723e11..af5e8bf8b7 100644 --- a/indra/llfilesystem/llfilesystem.cpp +++ b/indra/llfilesystem/llfilesystem.cpp @@ -164,6 +164,10 @@ BOOL LLFileSystem::read(U8* buffer, S32 bytes) success = FALSE; } } + else + { + success = FALSE; + } if (success == TRUE) { -- cgit v1.2.3 From 0c3b78105d49185f37eff60d9c167d17e6072fc8 Mon Sep 17 00:00:00 2001 From: Callum Prentice Date: Wed, 21 Jul 2021 17:42:08 -0700 Subject: Another tweak for 'SL-15547: Viewer hung while looking for a file in cache' - this time based on Henri's suggestion in this discussion: https://bitbucket.org/lindenlab/viewer/commits/e28c1b46e9944f0215a13cab8ee7dded88d7fc90#comment-10537114 --- indra/llfilesystem/llfilesystem.cpp | 41 ++++++++++++++++++++++--------------- 1 file changed, 25 insertions(+), 16 deletions(-) (limited to 'indra/llfilesystem/llfilesystem.cpp') diff --git a/indra/llfilesystem/llfilesystem.cpp b/indra/llfilesystem/llfilesystem.cpp index af5e8bf8b7..4c836c8838 100644 --- a/indra/llfilesystem/llfilesystem.cpp +++ b/indra/llfilesystem/llfilesystem.cpp @@ -48,6 +48,28 @@ LLFileSystem::LLFileSystem(const LLUUID& file_id, const LLAssetType::EType file_ mPosition = 0; mBytesRead = 0; mMode = mode; + + // This block of code was originally called in the read() method but after comments here: + // https://bitbucket.org/lindenlab/viewer/commits/e28c1b46e9944f0215a13cab8ee7dded88d7fc90#comment-10537114 + // we decided to follow Henri's suggestion and move the code to update the last access time here. + if (mode == LLFileSystem::READ) + { + // build the filename (TODO: we do this in a few places - perhaps we should factor into a single function) + std::string id; + mFileID.toString(id); + const std::string extra_info = ""; + const std::string filename = LLDiskCache::getInstance()->metaDataToFilepath(id, mFileType, extra_info); + + // update the last access time for the file if it exists - this is required + // even though we are reading and not writing because this is the + // way the cache works - it relies on a valid "last accessed time" for + // each file so it knows how to remove the oldest, unused files + bool exists = gDirUtilp->fileExists(filename); + if (exists) + { + LLDiskCache::getInstance()->updateFileAccessTime(filename); + } + } } LLFileSystem::~LLFileSystem() @@ -133,7 +155,7 @@ S32 LLFileSystem::getFileSize(const LLUUID& file_id, const LLAssetType::EType fi BOOL LLFileSystem::read(U8* buffer, S32 bytes) { - BOOL success = TRUE; + BOOL success = FALSE; std::string id; mFileID.toString(id); @@ -159,24 +181,11 @@ BOOL LLFileSystem::read(U8* buffer, S32 bytes) file.close(); mPosition += mBytesRead; - if (!mBytesRead) + if (mBytesRead) { - success = FALSE; + success = TRUE; } } - else - { - success = FALSE; - } - - if (success == TRUE) - { - // update the last access time for the file - this is required - // even though we are reading and not writing because this is the - // way the cache works - it relies on a valid "last accessed time" for - // each file so it knows how to remove the oldest, unused files - LLDiskCache::getInstance()->updateFileAccessTime(filename); - } return success; } -- cgit v1.2.3