summaryrefslogtreecommitdiff
path: root/indra/llvfs/llvfs.cpp
diff options
context:
space:
mode:
authorAndrey Lihatskiy <alihatskiy@productengine.com>2021-12-26 14:01:17 +0200
committerAndrey Lihatskiy <alihatskiy@productengine.com>2021-12-26 14:01:17 +0200
commit970b6661cddc6bad32f874d3f7676f5e5d5c5ec3 (patch)
tree63ffcce826a99ac45a71c820f9485297644e95ba /indra/llvfs/llvfs.cpp
parentc9343c400487979c206f0bc5fa3d04d70de6870a (diff)
parent0a873cd95547f003878c6d00d0883ff792f4a865 (diff)
Merge branch 'master' into DRTVWR-543-maint
Diffstat (limited to 'indra/llvfs/llvfs.cpp')
-rw-r--r--indra/llvfs/llvfs.cpp2196
1 files changed, 0 insertions, 2196 deletions
diff --git a/indra/llvfs/llvfs.cpp b/indra/llvfs/llvfs.cpp
deleted file mode 100644
index 2c64bf563e..0000000000
--- a/indra/llvfs/llvfs.cpp
+++ /dev/null
@@ -1,2196 +0,0 @@
-/**
- * @file llvfs.cpp
- * @brief Implementation of virtual file system
- *
- * $LicenseInfo:firstyear=2002&license=viewerlgpl$
- * Second Life Viewer Source Code
- * Copyright (C) 2010, Linden Research, Inc.
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation;
- * version 2.1 of the License only.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- *
- * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA
- * $/LicenseInfo$
- */
-
-#include "linden_common.h"
-
-#include "llvfs.h"
-
-#include <sys/stat.h>
-#include <set>
-#include <map>
-#if LL_WINDOWS
-#include <share.h>
-#else
-#include <sys/file.h>
-#endif
-
-#include "llstl.h"
-#include "lltimer.h"
-
-const S32 FILE_BLOCK_MASK = 0x000003FF; // 1024-byte blocks
-const S32 VFS_CLEANUP_SIZE = 5242880; // how much space we free up in a single stroke
-const S32 BLOCK_LENGTH_INVALID = -1; // mLength for invalid LLVFSFileBlocks
-
-LLVFS *gVFS = NULL;
-
-// internal class definitions
-class LLVFSBlock
-{
-public:
- LLVFSBlock()
- {
- mLocation = 0;
- mLength = 0;
- }
-
- LLVFSBlock(U32 loc, S32 size)
- {
- mLocation = loc;
- mLength = size;
- }
-
- static bool locationSortPredicate(
- const LLVFSBlock* lhs,
- const LLVFSBlock* rhs)
- {
- return lhs->mLocation < rhs->mLocation;
- }
-
-public:
- U32 mLocation;
- S32 mLength; // allocated block size
-};
-
-LLVFSFileSpecifier::LLVFSFileSpecifier()
-: mFileID(),
- mFileType( LLAssetType::AT_NONE )
-{
-}
-
-LLVFSFileSpecifier::LLVFSFileSpecifier(const LLUUID &file_id, const LLAssetType::EType file_type)
-{
- mFileID = file_id;
- mFileType = file_type;
-}
-
-bool LLVFSFileSpecifier::operator<(const LLVFSFileSpecifier &rhs) const
-{
- return (mFileID == rhs.mFileID)
- ? mFileType < rhs.mFileType
- : mFileID < rhs.mFileID;
-}
-
-bool LLVFSFileSpecifier::operator==(const LLVFSFileSpecifier &rhs) const
-{
- return (mFileID == rhs.mFileID &&
- mFileType == rhs.mFileType);
-}
-
-
-class LLVFSFileBlock : public LLVFSBlock, public LLVFSFileSpecifier
-{
-public:
- LLVFSFileBlock() : LLVFSBlock(), LLVFSFileSpecifier()
- {
- init();
- }
-
- LLVFSFileBlock(const LLUUID &file_id, LLAssetType::EType file_type, U32 loc = 0, S32 size = 0)
- : LLVFSBlock(loc, size), LLVFSFileSpecifier( file_id, file_type )
- {
- init();
- }
-
- void init()
- {
- mSize = 0;
- mIndexLocation = -1;
- mAccessTime = (U32)time(NULL);
-
- for (S32 i = 0; i < (S32)VFSLOCK_COUNT; i++)
- {
- mLocks[(EVFSLock)i] = 0;
- }
- }
-
- #ifdef LL_LITTLE_ENDIAN
- inline void swizzleCopy(void *dst, void *src, int size) { memcpy(dst, src, size); /* Flawfinder: ignore */}
-
- #else
-
- inline U32 swizzle32(U32 x)
- {
- return(((x >> 24) & 0x000000FF) | ((x >> 8) & 0x0000FF00) | ((x << 8) & 0x00FF0000) |((x << 24) & 0xFF000000));
- }
-
- inline U16 swizzle16(U16 x)
- {
- return( ((x >> 8) & 0x000000FF) | ((x << 8) & 0x0000FF00) );
- }
-
- inline void swizzleCopy(void *dst, void *src, int size)
- {
- if(size == 4)
- {
- ((U32*)dst)[0] = swizzle32(((U32*)src)[0]);
- }
- else if(size == 2)
- {
- ((U16*)dst)[0] = swizzle16(((U16*)src)[0]);
- }
- else
- {
- // Perhaps this should assert...
- memcpy(dst, src, size); /* Flawfinder: ignore */
- }
- }
-
- #endif
-
- void serialize(U8 *buffer)
- {
- swizzleCopy(buffer, &mLocation, 4);
- buffer += 4;
- swizzleCopy(buffer, &mLength, 4);
- buffer +=4;
- swizzleCopy(buffer, &mAccessTime, 4);
- buffer +=4;
- memcpy(buffer, &mFileID.mData, 16); /* Flawfinder: ignore */
- buffer += 16;
- S16 temp_type = mFileType;
- swizzleCopy(buffer, &temp_type, 2);
- buffer += 2;
- swizzleCopy(buffer, &mSize, 4);
- }
-
- void deserialize(U8 *buffer, const S32 index_loc)
- {
- mIndexLocation = index_loc;
-
- swizzleCopy(&mLocation, buffer, 4);
- buffer += 4;
- swizzleCopy(&mLength, buffer, 4);
- buffer += 4;
- swizzleCopy(&mAccessTime, buffer, 4);
- buffer += 4;
- memcpy(&mFileID.mData, buffer, 16);
- buffer += 16;
- S16 temp_type;
- swizzleCopy(&temp_type, buffer, 2);
- mFileType = (LLAssetType::EType)temp_type;
- buffer += 2;
- swizzleCopy(&mSize, buffer, 4);
- }
-
- static BOOL insertLRU(LLVFSFileBlock* const& first,
- LLVFSFileBlock* const& second)
- {
- return (first->mAccessTime == second->mAccessTime)
- ? *first < *second
- : first->mAccessTime < second->mAccessTime;
- }
-
-public:
- S32 mSize;
- S32 mIndexLocation; // location of index entry
- U32 mAccessTime;
- BOOL mLocks[VFSLOCK_COUNT]; // number of outstanding locks of each type
-
- static const S32 SERIAL_SIZE;
-};
-
-// Helper structure for doing lru w/ stl... is there a simpler way?
-struct LLVFSFileBlock_less
-{
- bool operator()(LLVFSFileBlock* const& lhs, LLVFSFileBlock* const& rhs) const
- {
- return (LLVFSFileBlock::insertLRU(lhs, rhs)) ? true : false;
- }
-};
-
-
-const S32 LLVFSFileBlock::SERIAL_SIZE = 34;
-
-
-LLVFS::LLVFS(const std::string& index_filename, const std::string& data_filename, const BOOL read_only, const U32 presize, const BOOL remove_after_crash)
-: mRemoveAfterCrash(remove_after_crash),
- mDataFP(NULL),
- mIndexFP(NULL)
-{
- mDataMutex = new LLMutex();
-
- S32 i;
- for (i = 0; i < VFSLOCK_COUNT; i++)
- {
- mLockCounts[i] = 0;
- }
- mValid = VFSVALID_OK;
- mReadOnly = read_only;
- mIndexFilename = index_filename;
- mDataFilename = data_filename;
-
- const char *file_mode = mReadOnly ? "rb" : "r+b";
-
- LL_INFOS("VFS") << "Attempting to open VFS index file " << mIndexFilename << LL_ENDL;
- LL_INFOS("VFS") << "Attempting to open VFS data file " << mDataFilename << LL_ENDL;
-
- mDataFP = openAndLock(mDataFilename, file_mode, mReadOnly);
- if (!mDataFP)
- {
- if (mReadOnly)
- {
- LL_WARNS("VFS") << "Can't find " << mDataFilename << " to open read-only VFS" << LL_ENDL;
- mValid = VFSVALID_BAD_CANNOT_OPEN_READONLY;
- return;
- }
-
- mDataFP = openAndLock(mDataFilename, "w+b", FALSE);
- if (mDataFP)
- {
- // Since we're creating this data file, assume any index file is bogus
- // remove the index, since this vfs is now blank
- LLFile::remove(mIndexFilename);
- }
- else
- {
- LL_WARNS("VFS") << "Couldn't open vfs data file "
- << mDataFilename << LL_ENDL;
- mValid = VFSVALID_BAD_CANNOT_CREATE;
- return;
- }
-
- if (presize)
- {
- presizeDataFile(presize);
- }
- }
-
- // Did we leave this file open for writing last time?
- // If so, close it and start over.
- if (!mReadOnly && mRemoveAfterCrash)
- {
- llstat marker_info;
- std::string marker = mDataFilename + ".open";
- if (!LLFile::stat(marker, &marker_info))
- {
- // marker exists, kill the lock and the VFS files
- unlockAndClose(mDataFP);
- mDataFP = NULL;
-
- LL_WARNS("VFS") << "VFS: File left open on last run, removing old VFS file " << mDataFilename << LL_ENDL;
- LLFile::remove(mIndexFilename);
- LLFile::remove(mDataFilename);
- LLFile::remove(marker);
-
- mDataFP = openAndLock(mDataFilename, "w+b", FALSE);
- if (!mDataFP)
- {
- LL_WARNS("VFS") << "Can't open VFS data file in crash recovery" << LL_ENDL;
- mValid = VFSVALID_BAD_CANNOT_CREATE;
- return;
- }
-
- if (presize)
- {
- presizeDataFile(presize);
- }
- }
- }
-
- // determine the real file size
- fseek(mDataFP, 0, SEEK_END);
- U32 data_size = ftell(mDataFP);
-
- // read the index file
- // make sure there's at least one file in it too
- // if not, we'll treat this as a new vfs
- llstat fbuf;
- if (! LLFile::stat(mIndexFilename, &fbuf) &&
- fbuf.st_size >= LLVFSFileBlock::SERIAL_SIZE &&
- (mIndexFP = openAndLock(mIndexFilename, file_mode, mReadOnly)) // Yes, this is an assignment and not '=='
- )
- {
- std::vector<U8> buffer(fbuf.st_size);
- size_t buf_offset = 0;
- size_t nread = fread(&buffer[0], 1, fbuf.st_size, mIndexFP);
-
- std::vector<LLVFSFileBlock*> files_by_loc;
-
- while (buf_offset < nread)
- {
- LLVFSFileBlock *block = new LLVFSFileBlock();
-
- block->deserialize(&buffer[buf_offset], (S32)buf_offset);
-
- // Do sanity check on this block.
- // Note that this skips zero size blocks, which helps VFS
- // to heal after some errors. JC
- if (block->mLength > 0 &&
- (U32)block->mLength <= data_size &&
- block->mLocation < data_size &&
- block->mSize > 0 &&
- block->mSize <= block->mLength &&
- block->mFileType >= LLAssetType::AT_NONE &&
- block->mFileType < LLAssetType::AT_COUNT)
- {
- mFileBlocks.insert(fileblock_map::value_type(*block, block));
- files_by_loc.push_back(block);
- }
- else
- if (block->mLength && block->mSize > 0)
- {
- // this is corrupt, not empty
- LL_WARNS("VFS") << "VFS corruption: " << block->mFileID << " (" << block->mFileType << ") at index " << block->mIndexLocation << " DS: " << data_size << LL_ENDL;
- LL_WARNS("VFS") << "Length: " << block->mLength << "\tLocation: " << block->mLocation << "\tSize: " << block->mSize << LL_ENDL;
- LL_WARNS("VFS") << "File has bad data - VFS removed" << LL_ENDL;
-
- delete block;
-
- unlockAndClose( mIndexFP );
- mIndexFP = NULL;
- LLFile::remove( mIndexFilename );
-
- unlockAndClose( mDataFP );
- mDataFP = NULL;
- LLFile::remove( mDataFilename );
-
- LL_WARNS("VFS") << "Deleted corrupt VFS files "
- << mDataFilename
- << " and "
- << mIndexFilename
- << LL_ENDL;
-
- mValid = VFSVALID_BAD_CORRUPT;
- return;
- }
- else
- {
- // this is a null or bad entry, skip it
- mIndexHoles.push_back(buf_offset);
-
- delete block;
- }
-
- buf_offset += LLVFSFileBlock::SERIAL_SIZE;
- }
-
- std::sort(
- files_by_loc.begin(),
- files_by_loc.end(),
- LLVFSFileBlock::locationSortPredicate);
-
- // There are 3 cases that have to be considered.
- // 1. No blocks
- // 2. One block.
- // 3. Two or more blocks.
- if (!files_by_loc.empty())
- {
- // cur walks through the list.
- std::vector<LLVFSFileBlock*>::iterator cur = files_by_loc.begin();
- std::vector<LLVFSFileBlock*>::iterator end = files_by_loc.end();
- LLVFSFileBlock* last_file_block = *cur;
-
- // Check to see if there is an empty space before the first file.
- if (last_file_block->mLocation > 0)
- {
- // If so, create a free block.
- addFreeBlock(new LLVFSBlock(0, last_file_block->mLocation));
- }
-
- // Walk through the 2nd+ block. If there is a free space
- // between cur_file_block and last_file_block, add it to
- // the free space collection. This block will not need to
- // run in the case there is only one entry in the VFS.
- ++cur;
- while( cur != end )
- {
- LLVFSFileBlock* cur_file_block = *cur;
-
- // Dupe check on the block
- if (cur_file_block->mLocation == last_file_block->mLocation
- && cur_file_block->mLength == last_file_block->mLength)
- {
- LL_WARNS("VFS") << "VFS: removing duplicate entry"
- << " at " << cur_file_block->mLocation
- << " length " << cur_file_block->mLength
- << " size " << cur_file_block->mSize
- << " ID " << cur_file_block->mFileID
- << " type " << cur_file_block->mFileType
- << LL_ENDL;
-
- // Duplicate entries. Nuke them both for safety.
- mFileBlocks.erase(*cur_file_block); // remove ID/type entry
- if (cur_file_block->mLength > 0)
- {
- // convert to hole
- addFreeBlock(
- new LLVFSBlock(
- cur_file_block->mLocation,
- cur_file_block->mLength));
- }
- lockData(); // needed for sync()
- sync(cur_file_block, TRUE); // remove first on disk
- sync(last_file_block, TRUE); // remove last on disk
- unlockData(); // needed for sync()
- last_file_block = cur_file_block;
- ++cur;
- continue;
- }
-
- // Figure out where the last block ended.
- S32 loc = last_file_block->mLocation+last_file_block->mLength;
-
- // Figure out how much space there is between where
- // the last block ended and this block begins.
- S32 length = cur_file_block->mLocation - loc;
-
- // Check for more errors... Seeing if the current
- // entry and the last entry make sense together.
- if (length < 0 || loc < 0 || (U32)loc > data_size)
- {
- // Invalid VFS
- unlockAndClose( mIndexFP );
- mIndexFP = NULL;
- LLFile::remove( mIndexFilename );
-
- unlockAndClose( mDataFP );
- mDataFP = NULL;
- LLFile::remove( mDataFilename );
-
- LL_WARNS("VFS") << "VFS: overlapping entries"
- << " at " << cur_file_block->mLocation
- << " length " << cur_file_block->mLength
- << " ID " << cur_file_block->mFileID
- << " type " << cur_file_block->mFileType
- << LL_ENDL;
-
- LL_WARNS("VFS") << "Deleted corrupt VFS files "
- << mDataFilename
- << " and "
- << mIndexFilename
- << LL_ENDL;
-
- mValid = VFSVALID_BAD_CORRUPT;
- return;
- }
-
- // we don't want to add empty blocks to the list...
- if (length > 0)
- {
- addFreeBlock(new LLVFSBlock(loc, length));
- }
- last_file_block = cur_file_block;
- ++cur;
- }
-
- // also note any empty space at the end
- U32 loc = last_file_block->mLocation + last_file_block->mLength;
- if (loc < data_size)
- {
- addFreeBlock(new LLVFSBlock(loc, data_size - loc));
- }
- }
- else // There where no blocks in the file.
- {
- addFreeBlock(new LLVFSBlock(0, data_size));
- }
- }
- else // Pre-existing index file wasn't opened
- {
- if (mReadOnly)
- {
- LL_WARNS("VFS") << "Can't find " << mIndexFilename << " to open read-only VFS" << LL_ENDL;
- mValid = VFSVALID_BAD_CANNOT_OPEN_READONLY;
- return;
- }
-
-
- mIndexFP = openAndLock(mIndexFilename, "w+b", FALSE);
- if (!mIndexFP)
- {
- LL_WARNS("VFS") << "Couldn't open an index file for the VFS, probably a sharing violation!" << LL_ENDL;
-
- unlockAndClose( mDataFP );
- mDataFP = NULL;
- LLFile::remove( mDataFilename );
-
- mValid = VFSVALID_BAD_CANNOT_CREATE;
- return;
- }
-
- // no index file, start from scratch w/ 1GB allocation
- LLVFSBlock *first_block = new LLVFSBlock(0, data_size ? data_size : 0x40000000);
- addFreeBlock(first_block);
- }
-
- // Open marker file to look for bad shutdowns
- if (!mReadOnly && mRemoveAfterCrash)
- {
- std::string marker = mDataFilename + ".open";
- LLFILE* marker_fp = LLFile::fopen(marker, "w"); /* Flawfinder: ignore */
- if (marker_fp)
- {
- fclose(marker_fp);
- marker_fp = NULL;
- }
- }
-
- LL_INFOS("VFS") << "Using VFS index file " << mIndexFilename << LL_ENDL;
- LL_INFOS("VFS") << "Using VFS data file " << mDataFilename << LL_ENDL;
-
- mValid = VFSVALID_OK;
-}
-
-LLVFS::~LLVFS()
-{
- if (mDataMutex->isLocked())
- {
- LL_ERRS("VFS") << "LLVFS destroyed with mutex locked" << LL_ENDL;
- }
-
- unlockAndClose(mIndexFP);
- mIndexFP = NULL;
-
- fileblock_map::const_iterator it;
- for (it = mFileBlocks.begin(); it != mFileBlocks.end(); ++it)
- {
- delete (*it).second;
- }
- mFileBlocks.clear();
-
- mFreeBlocksByLength.clear();
-
- for_each(mFreeBlocksByLocation.begin(), mFreeBlocksByLocation.end(), DeletePairedPointer());
- mFreeBlocksByLocation.clear();
-
- unlockAndClose(mDataFP);
- mDataFP = NULL;
-
- // Remove marker file
- if (!mReadOnly && mRemoveAfterCrash)
- {
- std::string marker = mDataFilename + ".open";
- LLFile::remove(marker);
- }
-
- delete mDataMutex;
-}
-
-
-// Use this function normally to create LLVFS files.
-// Will append digits to the end of the filename with multiple re-trys
-// static
-LLVFS * LLVFS::createLLVFS(const std::string& index_filename,
- const std::string& data_filename,
- const BOOL read_only,
- const U32 presize,
- const BOOL remove_after_crash)
-{
- LLVFS * new_vfs = new LLVFS(index_filename, data_filename, read_only, presize, remove_after_crash);
-
- if( !new_vfs->isValid() )
- { // First name failed, retry with new names
- std::string retry_vfs_index_name;
- std::string retry_vfs_data_name;
- S32 count = 0;
- while (!new_vfs->isValid() &&
- count < 256)
- { // Append '.<number>' to end of filenames
- retry_vfs_index_name = index_filename + llformat(".%u",count);
- retry_vfs_data_name = data_filename + llformat(".%u", count);
-
- delete new_vfs; // Delete bad VFS and try again
- new_vfs = new LLVFS(retry_vfs_index_name, retry_vfs_data_name, read_only, presize, remove_after_crash);
-
- count++;
- }
- }
-
- if( !new_vfs->isValid() )
- {
- delete new_vfs; // Delete bad VFS
- new_vfs = NULL; // Total failure
- }
-
- return new_vfs;
-}
-
-
-
-void LLVFS::presizeDataFile(const U32 size)
-{
- if (!mDataFP)
- {
- LL_ERRS() << "LLVFS::presizeDataFile() with no data file open" << LL_ENDL;
- return;
- }
-
- // we're creating this file for the first time, size it
- fseek(mDataFP, size-1, SEEK_SET);
- S32 tmp = 0;
- tmp = (S32)fwrite(&tmp, 1, 1, mDataFP);
- // fflush(mDataFP);
-
- // also remove any index, since this vfs is now blank
- LLFile::remove(mIndexFilename);
-
- if (tmp)
- {
- LL_INFOS() << "Pre-sized VFS data file to " << ftell(mDataFP) << " bytes" << LL_ENDL;
- }
- else
- {
- LL_WARNS() << "Failed to pre-size VFS data file" << LL_ENDL;
- }
-}
-
-BOOL LLVFS::getExists(const LLUUID &file_id, const LLAssetType::EType file_type)
-{
- LLVFSFileBlock *block = NULL;
-
- if (!isValid())
- {
- LL_ERRS() << "Attempting to use invalid VFS!" << LL_ENDL;
- }
-
- lockData();
-
- LLVFSFileSpecifier spec(file_id, file_type);
- fileblock_map::iterator it = mFileBlocks.find(spec);
- if (it != mFileBlocks.end())
- {
- block = (*it).second;
- block->mAccessTime = (U32)time(NULL);
- }
-
- BOOL res = (block && block->mLength > 0) ? TRUE : FALSE;
-
- unlockData();
-
- return res;
-}
-
-S32 LLVFS::getSize(const LLUUID &file_id, const LLAssetType::EType file_type)
-{
- S32 size = 0;
-
- if (!isValid())
- {
- LL_ERRS() << "Attempting to use invalid VFS!" << LL_ENDL;
-
- }
-
- lockData();
-
- LLVFSFileSpecifier spec(file_id, file_type);
- fileblock_map::iterator it = mFileBlocks.find(spec);
- if (it != mFileBlocks.end())
- {
- LLVFSFileBlock *block = (*it).second;
-
- block->mAccessTime = (U32)time(NULL);
- size = block->mSize;
- }
-
- unlockData();
-
- return size;
-}
-
-S32 LLVFS::getMaxSize(const LLUUID &file_id, const LLAssetType::EType file_type)
-{
- S32 size = 0;
-
- if (!isValid())
- {
- LL_ERRS() << "Attempting to use invalid VFS!" << LL_ENDL;
- }
-
- lockData();
-
- LLVFSFileSpecifier spec(file_id, file_type);
- fileblock_map::iterator it = mFileBlocks.find(spec);
- if (it != mFileBlocks.end())
- {
- LLVFSFileBlock *block = (*it).second;
-
- block->mAccessTime = (U32)time(NULL);
- size = block->mLength;
- }
-
- unlockData();
-
- return size;
-}
-
-BOOL LLVFS::checkAvailable(S32 max_size)
-{
- lockData();
-
- blocks_length_map_t::iterator iter = mFreeBlocksByLength.lower_bound(max_size); // first entry >= size
- const BOOL res(iter == mFreeBlocksByLength.end() ? FALSE : TRUE);
-
- unlockData();
-
- return res;
-}
-
-BOOL LLVFS::setMaxSize(const LLUUID &file_id, const LLAssetType::EType file_type, S32 max_size)
-{
- if (!isValid())
- {
- LL_ERRS() << "Attempting to use invalid VFS!" << LL_ENDL;
- }
- if (mReadOnly)
- {
- LL_ERRS() << "Attempt to write to read-only VFS" << LL_ENDL;
- }
- if (max_size <= 0)
- {
- LL_WARNS() << "VFS: Attempt to assign size " << max_size << " to vfile " << file_id << LL_ENDL;
- return FALSE;
- }
-
- lockData();
-
- LLVFSFileSpecifier spec(file_id, file_type);
- LLVFSFileBlock *block = NULL;
- fileblock_map::iterator it = mFileBlocks.find(spec);
- if (it != mFileBlocks.end())
- {
- block = (*it).second;
- }
-
- // round all sizes upward to KB increments
- // SJB: Need to not round for the new texture-pipeline code so we know the correct
- // max file size. Need to investigate the potential problems with this...
- if (file_type != LLAssetType::AT_TEXTURE)
- {
- if (max_size & FILE_BLOCK_MASK)
- {
- max_size += FILE_BLOCK_MASK;
- max_size &= ~FILE_BLOCK_MASK;
- }
- }
-
- if (block && block->mLength > 0)
- {
- block->mAccessTime = (U32)time(NULL);
-
- if (max_size == block->mLength)
- {
- unlockData();
- return TRUE;
- }
- else if (max_size < block->mLength)
- {
- // this file is shrinking
- LLVFSBlock *free_block = new LLVFSBlock(block->mLocation + max_size, block->mLength - max_size);
-
- addFreeBlock(free_block);
-
- block->mLength = max_size;
-
- if (block->mLength < block->mSize)
- {
- // JC: Was a warning, but Ian says it's bad.
- LL_ERRS() << "Truncating virtual file " << file_id << " to " << block->mLength << " bytes" << LL_ENDL;
- block->mSize = block->mLength;
- }
-
- sync(block);
- //mergeFreeBlocks();
-
- unlockData();
- return TRUE;
- }
- else if (max_size > block->mLength)
- {
- // this file is growing
- // first check for an adjacent free block to grow into
- S32 size_increase = max_size - block->mLength;
-
- // Find the first free block with and addres > block->mLocation
- LLVFSBlock *free_block;
- blocks_location_map_t::iterator iter = mFreeBlocksByLocation.upper_bound(block->mLocation);
- if (iter != mFreeBlocksByLocation.end())
- {
- free_block = iter->second;
-
- if (free_block->mLocation == block->mLocation + block->mLength &&
- free_block->mLength >= size_increase)
- {
- // this free block is at the end of the file and is large enough
-
- // Must call useFreeSpace before sync(), as sync()
- // unlocks data structures.
- useFreeSpace(free_block, size_increase);
- block->mLength += size_increase;
- sync(block);
-
- unlockData();
- return TRUE;
- }
- }
-
- // no adjecent free block, find one in the list
- free_block = findFreeBlock(max_size, block);
-
- if (free_block)
- {
- // Save location where data is going, useFreeSpace will move free_block->mLocation;
- U32 new_data_location = free_block->mLocation;
-
- //mark the free block as used so it does not
- //interfere with other operations such as addFreeBlock
- useFreeSpace(free_block, max_size); // useFreeSpace takes ownership (and may delete) free_block
-
- if (block->mLength > 0)
- {
- // create a new free block where this file used to be
- LLVFSBlock *new_free_block = new LLVFSBlock(block->mLocation, block->mLength);
-
- addFreeBlock(new_free_block);
-
- if (block->mSize > 0)
- {
- // move the file into the new block
- std::vector<U8> buffer(block->mSize);
- fseek(mDataFP, block->mLocation, SEEK_SET);
- if (fread(&buffer[0], block->mSize, 1, mDataFP) == 1)
- {
- fseek(mDataFP, new_data_location, SEEK_SET);
- if (fwrite(&buffer[0], block->mSize, 1, mDataFP) != 1)
- {
- LL_WARNS() << "Short write" << LL_ENDL;
- }
- } else {
- LL_WARNS() << "Short read" << LL_ENDL;
- }
- }
- }
-
- block->mLocation = new_data_location;
-
- block->mLength = max_size;
-
-
- sync(block);
-
- unlockData();
- return TRUE;
- }
- else
- {
- LL_WARNS() << "VFS: No space (" << max_size << ") to resize existing vfile " << file_id << LL_ENDL;
- //dumpMap();
- unlockData();
- dumpStatistics();
- return FALSE;
- }
- }
- }
- else
- {
- // find a free block in the list
- LLVFSBlock *free_block = findFreeBlock(max_size);
-
- if (free_block)
- {
- if (block)
- {
- block->mLocation = free_block->mLocation;
- block->mLength = max_size;
- }
- else
- {
- // this file doesn't exist, create it
- block = new LLVFSFileBlock(file_id, file_type, free_block->mLocation, max_size);
- mFileBlocks.insert(fileblock_map::value_type(spec, block));
- }
-
- // Must call useFreeSpace before sync(), as sync()
- // unlocks data structures.
- useFreeSpace(free_block, max_size);
- block->mAccessTime = (U32)time(NULL);
-
- sync(block);
- }
- else
- {
- LL_WARNS() << "VFS: No space (" << max_size << ") for new virtual file " << file_id << LL_ENDL;
- //dumpMap();
- unlockData();
- dumpStatistics();
- return FALSE;
- }
- }
- unlockData();
- return TRUE;
-}
-
-
-// WARNING: HERE BE DRAGONS!
-// rename is the weirdest VFS op, because the file moves but the locks don't!
-void LLVFS::renameFile(const LLUUID &file_id, const LLAssetType::EType file_type,
- const LLUUID &new_id, const LLAssetType::EType &new_type)
-{
- if (!isValid())
- {
- LL_ERRS() << "Attempting to use invalid VFS!" << LL_ENDL;
- }
- if (mReadOnly)
- {
- LL_ERRS() << "Attempt to write to read-only VFS" << LL_ENDL;
- }
-
- lockData();
-
- LLVFSFileSpecifier new_spec(new_id, new_type);
- LLVFSFileSpecifier old_spec(file_id, file_type);
-
- fileblock_map::iterator it = mFileBlocks.find(old_spec);
- if (it != mFileBlocks.end())
- {
- LLVFSFileBlock *src_block = (*it).second;
-
- // this will purge the data but leave the file block in place, w/ locks, if any
- // WAS: removeFile(new_id, new_type); NOW uses removeFileBlock() to avoid mutex lock recursion
- fileblock_map::iterator new_it = mFileBlocks.find(new_spec);
- if (new_it != mFileBlocks.end())
- {
- LLVFSFileBlock *new_block = (*new_it).second;
- removeFileBlock(new_block);
- }
-
- // if there's something in the target location, remove it but inherit its locks
- it = mFileBlocks.find(new_spec);
- if (it != mFileBlocks.end())
- {
- LLVFSFileBlock *dest_block = (*it).second;
-
- for (S32 i = 0; i < (S32)VFSLOCK_COUNT; i++)
- {
- if(dest_block->mLocks[i])
- {
- LL_ERRS() << "Renaming VFS block to a locked file." << LL_ENDL;
- }
- dest_block->mLocks[i] = src_block->mLocks[i];
- }
-
- mFileBlocks.erase(new_spec);
- delete dest_block;
- }
-
- src_block->mFileID = new_id;
- src_block->mFileType = new_type;
- src_block->mAccessTime = (U32)time(NULL);
-
- mFileBlocks.erase(old_spec);
- mFileBlocks.insert(fileblock_map::value_type(new_spec, src_block));
-
- sync(src_block);
- }
- else
- {
- LL_WARNS() << "VFS: Attempt to rename nonexistent vfile " << file_id << ":" << file_type << LL_ENDL;
- }
- unlockData();
-}
-
-// mDataMutex must be LOCKED before calling this
-void LLVFS::removeFileBlock(LLVFSFileBlock *fileblock)
-{
- // convert this into an unsaved, dummy fileblock to preserve locks
- // a more rubust solution would store the locks in a seperate data structure
- sync(fileblock, TRUE);
-
- if (fileblock->mLength > 0)
- {
- // turn this file into an empty block
- LLVFSBlock *free_block = new LLVFSBlock(fileblock->mLocation, fileblock->mLength);
-
- addFreeBlock(free_block);
- }
-
- fileblock->mLocation = 0;
- fileblock->mSize = 0;
- fileblock->mLength = BLOCK_LENGTH_INVALID;
- fileblock->mIndexLocation = -1;
-
- //mergeFreeBlocks();
-}
-
-void LLVFS::removeFile(const LLUUID &file_id, const LLAssetType::EType file_type)
-{
- if (!isValid())
- {
- LL_ERRS() << "Attempting to use invalid VFS!" << LL_ENDL;
- }
- if (mReadOnly)
- {
- LL_ERRS() << "Attempt to write to read-only VFS" << LL_ENDL;
- }
-
- lockData();
-
- LLVFSFileSpecifier spec(file_id, file_type);
- fileblock_map::iterator it = mFileBlocks.find(spec);
- if (it != mFileBlocks.end())
- {
- LLVFSFileBlock *block = (*it).second;
- removeFileBlock(block);
- }
- else
- {
- LL_WARNS() << "VFS: attempting to remove nonexistent file " << file_id << " type " << file_type << LL_ENDL;
- }
-
- unlockData();
-}
-
-
-S32 LLVFS::getData(const LLUUID &file_id, const LLAssetType::EType file_type, U8 *buffer, S32 location, S32 length)
-{
- S32 bytesread = 0;
-
- if (!isValid())
- {
- LL_ERRS() << "Attempting to use invalid VFS!" << LL_ENDL;
- }
- llassert(location >= 0);
- llassert(length >= 0);
-
- BOOL do_read = FALSE;
-
- lockData();
-
- LLVFSFileSpecifier spec(file_id, file_type);
- fileblock_map::iterator it = mFileBlocks.find(spec);
- if (it != mFileBlocks.end())
- {
- LLVFSFileBlock *block = (*it).second;
-
- block->mAccessTime = (U32)time(NULL);
-
- if (location > block->mSize)
- {
- LL_WARNS() << "VFS: Attempt to read location " << location << " in file " << file_id << " of length " << block->mSize << LL_ENDL;
- }
- else
- {
- if (length > block->mSize - location)
- {
- length = block->mSize - location;
- }
- location += block->mLocation;
- do_read = TRUE;
- }
- }
-
- if (do_read)
- {
- fseek(mDataFP, location, SEEK_SET);
- bytesread = (S32)fread(buffer, 1, length, mDataFP);
- }
-
- unlockData();
-
- return bytesread;
-}
-
-S32 LLVFS::storeData(const LLUUID &file_id, const LLAssetType::EType file_type, const U8 *buffer, S32 location, S32 length)
-{
- if (!isValid())
- {
- LL_ERRS() << "Attempting to use invalid VFS!" << LL_ENDL;
- }
- if (mReadOnly)
- {
- LL_ERRS() << "Attempt to write to read-only VFS" << LL_ENDL;
- }
-
- llassert(length > 0);
-
- lockData();
-
- LLVFSFileSpecifier spec(file_id, file_type);
- fileblock_map::iterator it = mFileBlocks.find(spec);
- if (it != mFileBlocks.end())
- {
- LLVFSFileBlock *block = (*it).second;
-
- S32 in_loc = location;
- if (location == -1)
- {
- location = block->mSize;
- }
- llassert(location >= 0);
-
- block->mAccessTime = (U32)time(NULL);
-
- if (block->mLength == BLOCK_LENGTH_INVALID)
- {
- // Block was removed, ignore write
- LL_WARNS() << "VFS: Attempt to write to invalid block"
- << " in file " << file_id
- << " location: " << in_loc
- << " bytes: " << length
- << LL_ENDL;
- unlockData();
- return length;
- }
- else if (location > block->mLength)
- {
- LL_WARNS() << "VFS: Attempt to write to location " << location
- << " in file " << file_id
- << " type " << S32(file_type)
- << " of size " << block->mSize
- << " block length " << block->mLength
- << LL_ENDL;
- unlockData();
- return length;
- }
- else
- {
- if (length > block->mLength - location )
- {
- LL_WARNS() << "VFS: Truncating write to virtual file " << file_id << " type " << S32(file_type) << LL_ENDL;
- length = block->mLength - location;
- }
- U32 file_location = location + block->mLocation;
-
- fseek(mDataFP, file_location, SEEK_SET);
- S32 write_len = (S32)fwrite(buffer, 1, length, mDataFP);
- if (write_len != length)
- {
- LL_WARNS() << llformat("VFS Write Error: %d != %d",write_len,length) << LL_ENDL;
- }
- // fflush(mDataFP);
-
- if (location + length > block->mSize)
- {
- block->mSize = location + write_len;
- sync(block);
- }
- unlockData();
-
- return write_len;
- }
- }
- else
- {
- unlockData();
- return 0;
- }
-}
-
-void LLVFS::incLock(const LLUUID &file_id, const LLAssetType::EType file_type, EVFSLock lock)
-{
- lockData();
-
- LLVFSFileSpecifier spec(file_id, file_type);
- LLVFSFileBlock *block;
-
- fileblock_map::iterator it = mFileBlocks.find(spec);
- if (it != mFileBlocks.end())
- {
- block = (*it).second;
- }
- else
- {
- // Create a dummy block which isn't saved
- block = new LLVFSFileBlock(file_id, file_type, 0, BLOCK_LENGTH_INVALID);
- block->mAccessTime = (U32)time(NULL);
- mFileBlocks.insert(fileblock_map::value_type(spec, block));
- }
-
- block->mLocks[lock]++;
- mLockCounts[lock]++;
-
- unlockData();
-}
-
-void LLVFS::decLock(const LLUUID &file_id, const LLAssetType::EType file_type, EVFSLock lock)
-{
- lockData();
-
- LLVFSFileSpecifier spec(file_id, file_type);
- fileblock_map::iterator it = mFileBlocks.find(spec);
- if (it != mFileBlocks.end())
- {
- LLVFSFileBlock *block = (*it).second;
-
- if (block->mLocks[lock] > 0)
- {
- block->mLocks[lock]--;
- }
- else
- {
- LL_WARNS() << "VFS: Decrementing zero-value lock " << lock << LL_ENDL;
- }
- mLockCounts[lock]--;
- }
-
- unlockData();
-}
-
-BOOL LLVFS::isLocked(const LLUUID &file_id, const LLAssetType::EType file_type, EVFSLock lock)
-{
- lockData();
-
- BOOL res = FALSE;
-
- LLVFSFileSpecifier spec(file_id, file_type);
- fileblock_map::iterator it = mFileBlocks.find(spec);
- if (it != mFileBlocks.end())
- {
- LLVFSFileBlock *block = (*it).second;
- res = (block->mLocks[lock] > 0);
- }
-
- unlockData();
-
- return res;
-}
-
-//============================================================================
-// protected
-//============================================================================
-
-void LLVFS::eraseBlockLength(LLVFSBlock *block)
-{
- // find the corresponding map entry in the length map and erase it
- S32 length = block->mLength;
- blocks_length_map_t::iterator iter = mFreeBlocksByLength.lower_bound(length);
- blocks_length_map_t::iterator end = mFreeBlocksByLength.end();
- bool found_block = false;
- while(iter != end)
- {
- LLVFSBlock *tblock = iter->second;
- llassert(tblock->mLength == length); // there had -better- be an entry with our length!
- if (tblock == block)
- {
- mFreeBlocksByLength.erase(iter);
- found_block = true;
- break;
- }
- ++iter;
- }
- if(!found_block)
- {
- LL_ERRS() << "eraseBlock could not find block" << LL_ENDL;
- }
-}
-
-
-// Remove block from both free lists (by location and by length).
-void LLVFS::eraseBlock(LLVFSBlock *block)
-{
- eraseBlockLength(block);
- // find the corresponding map entry in the location map and erase it
- U32 location = block->mLocation;
- llverify(mFreeBlocksByLocation.erase(location) == 1); // we should only have one entry per location.
-}
-
-
-// Add the region specified by block location and length to the free lists.
-// Also incrementally defragment by merging with previous and next free blocks.
-void LLVFS::addFreeBlock(LLVFSBlock *block)
-{
-#if LL_DEBUG
- size_t dbgcount = mFreeBlocksByLocation.count(block->mLocation);
- if(dbgcount > 0)
- {
- LL_ERRS() << "addFreeBlock called with block already in list" << LL_ENDL;
- }
-#endif
-
- // Get a pointer to the next free block (by location).
- blocks_location_map_t::iterator next_free_it = mFreeBlocksByLocation.lower_bound(block->mLocation);
-
- // We can merge with previous if it ends at our requested location.
- LLVFSBlock* prev_block = NULL;
- bool merge_prev = false;
- if (next_free_it != mFreeBlocksByLocation.begin())
- {
- blocks_location_map_t::iterator prev_free_it = next_free_it;
- --prev_free_it;
- prev_block = prev_free_it->second;
- merge_prev = (prev_block->mLocation + prev_block->mLength == block->mLocation);
- }
-
- // We can merge with next if our block ends at the next block's location.
- LLVFSBlock* next_block = NULL;
- bool merge_next = false;
- if (next_free_it != mFreeBlocksByLocation.end())
- {
- next_block = next_free_it->second;
- merge_next = (block->mLocation + block->mLength == next_block->mLocation);
- }
-
- if (merge_prev && merge_next)
- {
- // LL_INFOS() << "VFS merge BOTH" << LL_ENDL;
- // Previous block is changing length (a lot), so only need to update length map.
- // Next block is going away completely. JC
- eraseBlockLength(prev_block);
- eraseBlock(next_block);
- prev_block->mLength += block->mLength + next_block->mLength;
- mFreeBlocksByLength.insert(blocks_length_map_t::value_type(prev_block->mLength, prev_block));
- delete block;
- block = NULL;
- delete next_block;
- next_block = NULL;
- }
- else if (merge_prev)
- {
- // LL_INFOS() << "VFS merge previous" << LL_ENDL;
- // Previous block is maintaining location, only changing length,
- // therefore only need to update the length map. JC
- eraseBlockLength(prev_block);
- prev_block->mLength += block->mLength;
- mFreeBlocksByLength.insert(blocks_length_map_t::value_type(prev_block->mLength, prev_block)); // multimap insert
- delete block;
- block = NULL;
- }
- else if (merge_next)
- {
- // LL_INFOS() << "VFS merge next" << LL_ENDL;
- // Next block is changing both location and length,
- // so both free lists must update. JC
- eraseBlock(next_block);
- next_block->mLocation = block->mLocation;
- next_block->mLength += block->mLength;
- // Don't hint here, next_free_it iterator may be invalid.
- mFreeBlocksByLocation.insert(blocks_location_map_t::value_type(next_block->mLocation, next_block)); // multimap insert
- mFreeBlocksByLength.insert(blocks_length_map_t::value_type(next_block->mLength, next_block)); // multimap insert
- delete block;
- block = NULL;
- }
- else
- {
- // Can't merge with other free blocks.
- // Hint that insert should go near next_free_it.
- mFreeBlocksByLocation.insert(next_free_it, blocks_location_map_t::value_type(block->mLocation, block)); // multimap insert
- mFreeBlocksByLength.insert(blocks_length_map_t::value_type(block->mLength, block)); // multimap insert
- }
-}
-
-// Superceeded by new addFreeBlock which does incremental free space merging.
-// Incremental is faster overall.
-//void LLVFS::mergeFreeBlocks()
-//{
-// if (!isValid())
-// {
-// LL_ERRS() << "Attempting to use invalid VFS!" << LL_ENDL;
-// }
-// // TODO: could we optimize this with hints from the calling code?
-// blocks_location_map_t::iterator iter = mFreeBlocksByLocation.begin();
-// blocks_location_map_t::iterator end = mFreeBlocksByLocation.end();
-// LLVFSBlock *first_block = iter->second;
-// while(iter != end)
-// {
-// blocks_location_map_t::iterator first_iter = iter; // save for if we do a merge
-// if (++iter == end)
-// break;
-// LLVFSBlock *second_block = iter->second;
-// if (first_block->mLocation + first_block->mLength == second_block->mLocation)
-// {
-// // remove the first block from the length map
-// eraseBlockLength(first_block);
-// // merge first_block with second_block, since they're adjacent
-// first_block->mLength += second_block->mLength;
-// // add the first block to the length map (with the new size)
-// mFreeBlocksByLength.insert(blocks_length_map_t::value_type(first_block->mLength, first_block)); // multimap insert
-//
-// // erase and delete the second block
-// eraseBlock(second_block);
-// delete second_block;
-//
-// // reset iterator
-// iter = first_iter; // haven't changed first_block, so corresponding iterator is still valid
-// end = mFreeBlocksByLocation.end();
-// }
-// first_block = second_block;
-// }
-//}
-
-// length bytes from free_block are going to be used (so they are no longer free)
-void LLVFS::useFreeSpace(LLVFSBlock *free_block, S32 length)
-{
- if (free_block->mLength == length)
- {
- eraseBlock(free_block);
- delete free_block;
- }
- else
- {
- eraseBlock(free_block);
-
- free_block->mLocation += length;
- free_block->mLength -= length;
-
- addFreeBlock(free_block);
- }
-}
-
-// NOTE! mDataMutex must be LOCKED before calling this
-// sync this index entry out to the index file
-// we need to do this constantly to avoid corruption on viewer crash
-void LLVFS::sync(LLVFSFileBlock *block, BOOL remove)
-{
- if (!isValid())
- {
- LL_ERRS() << "Attempting to use invalid VFS!" << LL_ENDL;
- }
- if (mReadOnly)
- {
- LL_WARNS() << "Attempt to sync read-only VFS" << LL_ENDL;
- return;
- }
- if (block->mLength == BLOCK_LENGTH_INVALID)
- {
- // This is a dummy file, don't save
- return;
- }
- if (block->mLength == 0)
- {
- LL_ERRS() << "VFS syncing zero-length block" << LL_ENDL;
- }
-
- BOOL set_index_to_end = FALSE;
- long seek_pos = block->mIndexLocation;
-
- if (-1 == seek_pos)
- {
- if (!mIndexHoles.empty())
- {
- seek_pos = mIndexHoles.front();
- mIndexHoles.pop_front();
- }
- else
- {
- set_index_to_end = TRUE;
- }
- }
-
- if (set_index_to_end)
- {
- // Need fseek/ftell to update the seek_pos and hence data
- // structures, so can't unlockData() before this.
- fseek(mIndexFP, 0, SEEK_END);
- seek_pos = ftell(mIndexFP);
- }
-
- block->mIndexLocation = seek_pos;
- if (remove)
- {
- mIndexHoles.push_back(seek_pos);
- }
-
- U8 buffer[LLVFSFileBlock::SERIAL_SIZE];
- if (remove)
- {
- memset(buffer, 0, LLVFSFileBlock::SERIAL_SIZE);
- }
- else
- {
- block->serialize(buffer);
- }
-
- // If set_index_to_end, file pointer is already at seek_pos
- // and we don't need to do anything. Only seek if not at end.
- if (!set_index_to_end)
- {
- fseek(mIndexFP, seek_pos, SEEK_SET);
- }
-
- if (fwrite(buffer, LLVFSFileBlock::SERIAL_SIZE, 1, mIndexFP) != 1)
- {
- LL_WARNS() << "Short write" << LL_ENDL;
- }
-
- // *NOTE: Why was this commented out?
- // fflush(mIndexFP);
-
- return;
-}
-
-// mDataMutex must be LOCKED before calling this
-// Can initiate LRU-based file removal to make space.
-// The immune file block will not be removed.
-LLVFSBlock *LLVFS::findFreeBlock(S32 size, LLVFSFileBlock *immune)
-{
- if (!isValid())
- {
- LL_ERRS() << "Attempting to use invalid VFS!" << LL_ENDL;
- }
-
- LLVFSBlock *block = NULL;
- BOOL have_lru_list = FALSE;
-
- typedef std::set<LLVFSFileBlock*, LLVFSFileBlock_less> lru_set;
- lru_set lru_list;
-
- LLTimer timer;
-
- while (! block)
- {
- // look for a suitable free block
- blocks_length_map_t::iterator iter = mFreeBlocksByLength.lower_bound(size); // first entry >= size
- if (iter != mFreeBlocksByLength.end())
- block = iter->second;
-
- // no large enough free blocks, time to clean out some junk
- if (! block)
- {
- // create a list of files sorted by usage time
- // this is far faster than sorting a linked list
- if (! have_lru_list)
- {
- for (fileblock_map::iterator it = mFileBlocks.begin(); it != mFileBlocks.end(); ++it)
- {
- LLVFSFileBlock *tmp = (*it).second;
-
- if (tmp != immune &&
- tmp->mLength > 0 &&
- ! tmp->mLocks[VFSLOCK_READ] &&
- ! tmp->mLocks[VFSLOCK_APPEND] &&
- ! tmp->mLocks[VFSLOCK_OPEN])
- {
- lru_list.insert(tmp);
- }
- }
-
- have_lru_list = TRUE;
- }
-
- if (lru_list.size() == 0)
- {
- // No more files to delete, and still not enough room!
- LL_WARNS() << "VFS: Can't make " << size << " bytes of free space in VFS, giving up" << LL_ENDL;
- break;
- }
-
- // is the oldest file big enough? (Should be about half the time)
- lru_set::iterator it = lru_list.begin();
- LLVFSFileBlock *file_block = *it;
- if (file_block->mLength >= size && file_block != immune)
- {
- // ditch this file and look again for a free block - should find it
- // TODO: it'll be faster just to assign the free block and break
- LL_INFOS() << "LRU: Removing " << file_block->mFileID << ":" << file_block->mFileType << LL_ENDL;
- lru_list.erase(it);
- removeFileBlock(file_block);
- file_block = NULL;
- continue;
- }
-
-
- LL_INFOS() << "VFS: LRU: Aggressive: " << (S32)lru_list.size() << " files remain" << LL_ENDL;
- dumpLockCounts();
-
- // Now it's time to aggressively make more space
- // Delete the oldest 5MB of the vfs or enough to hold the file, which ever is larger
- // This may yield too much free space, but we'll use it up soon enough
- U32 cleanup_target = (size > VFS_CLEANUP_SIZE) ? size : VFS_CLEANUP_SIZE;
- U32 cleaned_up = 0;
- for (it = lru_list.begin();
- it != lru_list.end() && cleaned_up < cleanup_target;
- )
- {
- file_block = *it;
-
- // TODO: it would be great to be able to batch all these sync() calls
- // LL_INFOS() << "LRU2: Removing " << file_block->mFileID << ":" << file_block->mFileType << " last accessed" << file_block->mAccessTime << LL_ENDL;
-
- cleaned_up += file_block->mLength;
- lru_list.erase(it++);
- removeFileBlock(file_block);
- file_block = NULL;
- }
- //mergeFreeBlocks();
- }
- }
-
- F32 time = timer.getElapsedTimeF32();
- if (time > 0.5f)
- {
- LL_WARNS() << "VFS: Spent " << time << " seconds in findFreeBlock!" << LL_ENDL;
- }
-
- return block;
-}
-
-//============================================================================
-// public
-//============================================================================
-
-void LLVFS::pokeFiles()
-{
- if (!isValid())
- {
- LL_ERRS() << "Attempting to use invalid VFS!" << LL_ENDL;
- }
- U32 word;
-
- // only write data if we actually read 4 bytes
- // otherwise we're writing garbage and screwing up the file
- fseek(mDataFP, 0, SEEK_SET);
- if (fread(&word, sizeof(word), 1, mDataFP) == 1)
- {
- fseek(mDataFP, 0, SEEK_SET);
- if (fwrite(&word, sizeof(word), 1, mDataFP) != 1)
- {
- LL_WARNS() << "Could not write to data file" << LL_ENDL;
- }
- fflush(mDataFP);
- }
-
- fseek(mIndexFP, 0, SEEK_SET);
- if (fread(&word, sizeof(word), 1, mIndexFP) == 1)
- {
- fseek(mIndexFP, 0, SEEK_SET);
- if (fwrite(&word, sizeof(word), 1, mIndexFP) != 1)
- {
- LL_WARNS() << "Could not write to index file" << LL_ENDL;
- }
- fflush(mIndexFP);
- }
-}
-
-
-void LLVFS::dumpMap()
-{
- LL_INFOS() << "Files:" << LL_ENDL;
- for (fileblock_map::iterator it = mFileBlocks.begin(); it != mFileBlocks.end(); ++it)
- {
- LLVFSFileBlock *file_block = (*it).second;
- LL_INFOS() << "Location: " << file_block->mLocation << "\tLength: " << file_block->mLength << "\t" << file_block->mFileID << "\t" << file_block->mFileType << LL_ENDL;
- }
-
- LL_INFOS() << "Free Blocks:" << LL_ENDL;
- for (blocks_location_map_t::iterator iter = mFreeBlocksByLocation.begin(),
- end = mFreeBlocksByLocation.end();
- iter != end; iter++)
- {
- LLVFSBlock *free_block = iter->second;
- LL_INFOS() << "Location: " << free_block->mLocation << "\tLength: " << free_block->mLength << LL_ENDL;
- }
-}
-
-// verify that the index file contents match the in-memory file structure
-// Very slow, do not call routinely. JC
-void LLVFS::audit()
-{
- // Lock the mutex through this whole function.
- LLMutexLock lock_data(mDataMutex);
-
- fflush(mIndexFP);
-
- fseek(mIndexFP, 0, SEEK_END);
- size_t index_size = ftell(mIndexFP);
- fseek(mIndexFP, 0, SEEK_SET);
-
- BOOL vfs_corrupt = FALSE;
-
- // since we take the address of element 0, we need to have at least one element.
- std::vector<U8> buffer(llmax<size_t>(index_size,1U));
-
- if (fread(&buffer[0], 1, index_size, mIndexFP) != index_size)
- {
- LL_WARNS() << "Index truncated" << LL_ENDL;
- vfs_corrupt = TRUE;
- }
-
- size_t buf_offset = 0;
-
- std::map<LLVFSFileSpecifier, LLVFSFileBlock*> found_files;
- U32 cur_time = (U32)time(NULL);
-
- std::vector<LLVFSFileBlock*> audit_blocks;
- while (!vfs_corrupt && buf_offset < index_size)
- {
- LLVFSFileBlock *block = new LLVFSFileBlock();
- audit_blocks.push_back(block);
-
- block->deserialize(&buffer[buf_offset], (S32)buf_offset);
- buf_offset += block->SERIAL_SIZE;
-
- // do sanity check on this block
- if (block->mLength >= 0 &&
- block->mSize >= 0 &&
- block->mSize <= block->mLength &&
- block->mFileType >= LLAssetType::AT_NONE &&
- block->mFileType < LLAssetType::AT_COUNT &&
- block->mAccessTime <= cur_time &&
- block->mFileID != LLUUID::null)
- {
- if (mFileBlocks.find(*block) == mFileBlocks.end())
- {
- LL_WARNS() << "VFile " << block->mFileID << ":" << block->mFileType << " on disk, not in memory, loc " << block->mIndexLocation << LL_ENDL;
- }
- else if (found_files.find(*block) != found_files.end())
- {
- std::map<LLVFSFileSpecifier, LLVFSFileBlock*>::iterator it;
- it = found_files.find(*block);
- LLVFSFileBlock* dupe = it->second;
- // try to keep data from being lost
- unlockAndClose(mIndexFP);
- mIndexFP = NULL;
- unlockAndClose(mDataFP);
- mDataFP = NULL;
- LL_WARNS() << "VFS: Original block index " << block->mIndexLocation
- << " location " << block->mLocation
- << " length " << block->mLength
- << " size " << block->mSize
- << " id " << block->mFileID
- << " type " << block->mFileType
- << LL_ENDL;
- LL_WARNS() << "VFS: Duplicate block index " << dupe->mIndexLocation
- << " location " << dupe->mLocation
- << " length " << dupe->mLength
- << " size " << dupe->mSize
- << " id " << dupe->mFileID
- << " type " << dupe->mFileType
- << LL_ENDL;
- LL_WARNS() << "VFS: Index size " << index_size << LL_ENDL;
- LL_WARNS() << "VFS: INDEX CORRUPT" << LL_ENDL;
- vfs_corrupt = TRUE;
- break;
- }
- else
- {
- found_files[*block] = block;
- }
- }
- else
- {
- if (block->mLength)
- {
- LL_WARNS() << "VFile " << block->mFileID << ":" << block->mFileType << " corrupt on disk" << LL_ENDL;
- }
- // else this is just a hole
- }
- }
-
- if (!vfs_corrupt)
- {
- for (fileblock_map::iterator it = mFileBlocks.begin(); it != mFileBlocks.end(); ++it)
- {
- LLVFSFileBlock* block = (*it).second;
-
- if (block->mSize > 0)
- {
- if (! found_files.count(*block))
- {
- LL_WARNS() << "VFile " << block->mFileID << ":" << block->mFileType << " in memory, not on disk, loc " << block->mIndexLocation<< LL_ENDL;
- fseek(mIndexFP, block->mIndexLocation, SEEK_SET);
- U8 buf[LLVFSFileBlock::SERIAL_SIZE];
- if (fread(buf, LLVFSFileBlock::SERIAL_SIZE, 1, mIndexFP) != 1)
- {
- LL_WARNS() << "VFile " << block->mFileID
- << " gave short read" << LL_ENDL;
- }
-
- LLVFSFileBlock disk_block;
- disk_block.deserialize(buf, block->mIndexLocation);
-
- LL_WARNS() << "Instead found " << disk_block.mFileID << ":" << block->mFileType << LL_ENDL;
- }
- else
- {
- block = found_files.find(*block)->second;
- found_files.erase(*block);
- }
- }
- }
-
- for (std::map<LLVFSFileSpecifier, LLVFSFileBlock*>::iterator iter = found_files.begin();
- iter != found_files.end(); iter++)
- {
- LLVFSFileBlock* block = iter->second;
- LL_WARNS() << "VFile " << block->mFileID << ":" << block->mFileType << " szie:" << block->mSize << " leftover" << LL_ENDL;
- }
-
- LL_INFOS() << "VFS: audit OK" << LL_ENDL;
- // mutex released by LLMutexLock() destructor.
- }
-
- for_each(audit_blocks.begin(), audit_blocks.end(), DeletePointer());
- audit_blocks.clear();
-}
-
-
-// quick check for uninitialized blocks
-// Slow, do not call in release.
-void LLVFS::checkMem()
-{
- lockData();
-
- for (fileblock_map::iterator it = mFileBlocks.begin(); it != mFileBlocks.end(); ++it)
- {
- LLVFSFileBlock *block = (*it).second;
- llassert(block->mFileType >= LLAssetType::AT_NONE &&
- block->mFileType < LLAssetType::AT_COUNT &&
- block->mFileID != LLUUID::null);
-
- for (std::deque<S32>::iterator iter = mIndexHoles.begin();
- iter != mIndexHoles.end(); ++iter)
- {
- S32 index_loc = *iter;
- if (index_loc == block->mIndexLocation)
- {
- LL_WARNS() << "VFile block " << block->mFileID << ":" << block->mFileType << " is marked as a hole" << LL_ENDL;
- }
- }
- }
-
- LL_INFOS() << "VFS: mem check OK" << LL_ENDL;
-
- unlockData();
-}
-
-void LLVFS::dumpLockCounts()
-{
- S32 i;
- for (i = 0; i < VFSLOCK_COUNT; i++)
- {
- LL_INFOS() << "LockType: " << i << ": " << mLockCounts[i] << LL_ENDL;
- }
-}
-
-void LLVFS::dumpStatistics()
-{
- lockData();
-
- // Investigate file blocks.
- std::map<S32, S32> size_counts;
- std::map<U32, S32> location_counts;
- std::map<LLAssetType::EType, std::pair<S32,S32> > filetype_counts;
-
- S32 max_file_size = 0;
- S32 total_file_size = 0;
- S32 invalid_file_count = 0;
- for (fileblock_map::iterator it = mFileBlocks.begin(); it != mFileBlocks.end(); ++it)
- {
- LLVFSFileBlock *file_block = (*it).second;
- if (file_block->mLength == BLOCK_LENGTH_INVALID)
- {
- invalid_file_count++;
- }
- else if (file_block->mLength <= 0)
- {
- LL_INFOS() << "Bad file block at: " << file_block->mLocation << "\tLength: " << file_block->mLength << "\t" << file_block->mFileID << "\t" << file_block->mFileType << LL_ENDL;
- size_counts[file_block->mLength]++;
- location_counts[file_block->mLocation]++;
- }
- else
- {
- total_file_size += file_block->mLength;
- }
-
- if (file_block->mLength > max_file_size)
- {
- max_file_size = file_block->mLength;
- }
-
- filetype_counts[file_block->mFileType].first++;
- filetype_counts[file_block->mFileType].second += file_block->mLength;
- }
-
- for (std::map<S32,S32>::iterator it = size_counts.begin(); it != size_counts.end(); ++it)
- {
- S32 size = it->first;
- S32 size_count = it->second;
- LL_INFOS() << "Bad files size " << size << " count " << size_count << LL_ENDL;
- }
- for (std::map<U32,S32>::iterator it = location_counts.begin(); it != location_counts.end(); ++it)
- {
- U32 location = it->first;
- S32 location_count = it->second;
- LL_INFOS() << "Bad files location " << location << " count " << location_count << LL_ENDL;
- }
-
- // Investigate free list.
- S32 max_free_size = 0;
- S32 total_free_size = 0;
- std::map<S32, S32> free_length_counts;
- for (blocks_location_map_t::iterator iter = mFreeBlocksByLocation.begin(),
- end = mFreeBlocksByLocation.end();
- iter != end; iter++)
- {
- LLVFSBlock *free_block = iter->second;
- if (free_block->mLength <= 0)
- {
- LL_INFOS() << "Bad free block at: " << free_block->mLocation << "\tLength: " << free_block->mLength << LL_ENDL;
- }
- else
- {
- LL_INFOS() << "Block: " << free_block->mLocation
- << "\tLength: " << free_block->mLength
- << "\tEnd: " << free_block->mLocation + free_block->mLength
- << LL_ENDL;
- total_free_size += free_block->mLength;
- }
-
- if (free_block->mLength > max_free_size)
- {
- max_free_size = free_block->mLength;
- }
-
- free_length_counts[free_block->mLength]++;
- }
-
- // Dump histogram of free block sizes
- for (std::map<S32,S32>::iterator it = free_length_counts.begin(); it != free_length_counts.end(); ++it)
- {
- LL_INFOS() << "Free length " << it->first << " count " << it->second << LL_ENDL;
- }
-
- LL_INFOS() << "Invalid blocks: " << invalid_file_count << LL_ENDL;
- LL_INFOS() << "File blocks: " << mFileBlocks.size() << LL_ENDL;
-
- S32 length_list_count = (S32)mFreeBlocksByLength.size();
- S32 location_list_count = (S32)mFreeBlocksByLocation.size();
- if (length_list_count == location_list_count)
- {
- LL_INFOS() << "Free list lengths match, free blocks: " << location_list_count << LL_ENDL;
- }
- else
- {
- LL_WARNS() << "Free list lengths do not match!" << LL_ENDL;
- LL_WARNS() << "By length: " << length_list_count << LL_ENDL;
- LL_WARNS() << "By location: " << location_list_count << LL_ENDL;
- }
- LL_INFOS() << "Max file: " << max_file_size/1024 << "K" << LL_ENDL;
- LL_INFOS() << "Max free: " << max_free_size/1024 << "K" << LL_ENDL;
- LL_INFOS() << "Total file size: " << total_file_size/1024 << "K" << LL_ENDL;
- LL_INFOS() << "Total free size: " << total_free_size/1024 << "K" << LL_ENDL;
- LL_INFOS() << "Sum: " << (total_file_size + total_free_size) << " bytes" << LL_ENDL;
- LL_INFOS() << llformat("%.0f%% full",((F32)(total_file_size)/(F32)(total_file_size+total_free_size))*100.f) << LL_ENDL;
-
- LL_INFOS() << " " << LL_ENDL;
- for (std::map<LLAssetType::EType, std::pair<S32,S32> >::iterator iter = filetype_counts.begin();
- iter != filetype_counts.end(); ++iter)
- {
- LL_INFOS() << "Type: " << LLAssetType::getDesc(iter->first)
- << " Count: " << iter->second.first
- << " Bytes: " << (iter->second.second>>20) << " MB" << LL_ENDL;
- }
-
- // Look for potential merges
- {
- blocks_location_map_t::iterator iter = mFreeBlocksByLocation.begin();
- blocks_location_map_t::iterator end = mFreeBlocksByLocation.end();
- LLVFSBlock *first_block = iter->second;
- while(iter != end)
- {
- if (++iter == end)
- break;
- LLVFSBlock *second_block = iter->second;
- if (first_block->mLocation + first_block->mLength == second_block->mLocation)
- {
- LL_INFOS() << "Potential merge at " << first_block->mLocation << LL_ENDL;
- }
- first_block = second_block;
- }
- }
- unlockData();
-}
-
-// Debug Only!
-std::string get_extension(LLAssetType::EType type)
-{
- std::string extension;
- switch(type)
- {
- case LLAssetType::AT_TEXTURE:
- extension = ".jp2"; // formerly ".j2c"
- break;
- case LLAssetType::AT_SOUND:
- extension = ".ogg";
- break;
- case LLAssetType::AT_SOUND_WAV:
- extension = ".wav";
- break;
- case LLAssetType::AT_TEXTURE_TGA:
- extension = ".tga";
- break;
- case LLAssetType::AT_ANIMATION:
- extension = ".lla";
- break;
- case LLAssetType::AT_MESH:
- extension = ".slm";
- break;
- default:
- // Just use the asset server filename extension in most cases
- extension += ".";
- extension += LLAssetType::lookup(type);
- break;
- }
- return extension;
-}
-
-void LLVFS::listFiles()
-{
- lockData();
-
- for (fileblock_map::iterator it = mFileBlocks.begin(); it != mFileBlocks.end(); ++it)
- {
- LLVFSFileSpecifier file_spec = it->first;
- LLVFSFileBlock *file_block = it->second;
- S32 length = file_block->mLength;
- S32 size = file_block->mSize;
- if (length != BLOCK_LENGTH_INVALID && size > 0)
- {
- LLUUID id = file_spec.mFileID;
- std::string extension = get_extension(file_spec.mFileType);
- LL_INFOS() << " File: " << id
- << " Type: " << LLAssetType::getDesc(file_spec.mFileType)
- << " Size: " << size
- << LL_ENDL;
- }
- }
-
- unlockData();
-}
-
-#include "llapr.h"
-void LLVFS::dumpFiles()
-{
- lockData();
-
- S32 files_extracted = 0;
- for (fileblock_map::iterator it = mFileBlocks.begin(); it != mFileBlocks.end(); ++it)
- {
- LLVFSFileSpecifier file_spec = it->first;
- LLVFSFileBlock *file_block = it->second;
- S32 length = file_block->mLength;
- S32 size = file_block->mSize;
- if (length != BLOCK_LENGTH_INVALID && size > 0)
- {
- LLUUID id = file_spec.mFileID;
- LLAssetType::EType type = file_spec.mFileType;
- std::vector<U8> buffer(size);
-
- unlockData();
- getData(id, type, &buffer[0], 0, size);
- lockData();
-
- std::string extension = get_extension(type);
- std::string filename = id.asString() + extension;
- LL_INFOS() << " Writing " << filename << LL_ENDL;
-
- LLAPRFile outfile;
- outfile.open(filename, LL_APR_WB);
- outfile.write(&buffer[0], size);
- outfile.close();
-
- files_extracted++;
- }
- }
-
- unlockData();
-
- LL_INFOS() << "Extracted " << files_extracted << " files out of " << mFileBlocks.size() << LL_ENDL;
-}
-
-time_t LLVFS::creationTime()
-{
- llstat data_file_stat;
- int errors = LLFile::stat(mDataFilename, &data_file_stat);
- if (0 == errors)
- {
- time_t creation_time = data_file_stat.st_ctime;
-#if LL_DARWIN
- creation_time = data_file_stat.st_birthtime;
-#endif
- return creation_time;
- }
- return 0;
-}
-
-//============================================================================
-// protected
-//============================================================================
-
-// static
-LLFILE *LLVFS::openAndLock(const std::string& filename, const char* mode, BOOL read_lock)
-{
-#if LL_WINDOWS
-
- return LLFile::_fsopen(filename, mode, (read_lock ? _SH_DENYWR : _SH_DENYRW));
-
-#else
-
- LLFILE *fp;
- int fd;
-
- // first test the lock in a non-destructive way
- if (strchr(mode, 'w') != NULL)
- {
- fp = LLFile::fopen(filename, "rb"); /* Flawfinder: ignore */
- if (fp)
- {
- fd = fileno(fp);
- if (flock(fd, (read_lock ? LOCK_SH : LOCK_EX) | LOCK_NB) == -1)
- {
- fclose(fp);
- return NULL;
- }
-
- fclose(fp);
- }
- }
-
- // now actually open the file for use
- fp = LLFile::fopen(filename, mode); /* Flawfinder: ignore */
- if (fp)
- {
- fd = fileno(fp);
- if (flock(fd, (read_lock ? LOCK_SH : LOCK_EX) | LOCK_NB) == -1)
- {
- fclose(fp);
- fp = NULL;
- }
- }
-
- return fp;
-
-#endif
-}
-
-// static
-void LLVFS::unlockAndClose(LLFILE *fp)
-{
- if (fp)
- {
- // IW: we don't actually want to unlock on linux
- // this is because a forked process can kill the parent's lock
- // with an explicit unlock
- // however, fclose() will implicitly remove the lock
- // but only once both parent and child have closed the file
- /*
- #if !LL_WINDOWS
- int fd = fileno(fp);
- flock(fd, LOCK_UN);
- #endif
- */
- fclose(fp);
- }
-}