diff options
| author | Dave Houlton <euclid@lindenlab.com> | 2021-03-10 09:27:21 -0700 | 
|---|---|---|
| committer | Dave Houlton <euclid@lindenlab.com> | 2021-03-10 09:34:16 -0700 | 
| commit | 303feae305eb526d75b7d9c8b640407cb8003469 (patch) | |
| tree | 0e322ed210d9600530f0d916fbb18f8330e6aee7 /indra/llmessage | |
| parent | 9fdd1582774a928242b7fec91bbd951b123296e8 (diff) | |
| parent | 88d837c16e768c5262073a7df965066d4bd8842c (diff) | |
Merge branch 'master' v6.4.17 into DRTVWR-525
Diffstat (limited to 'indra/llmessage')
| -rw-r--r-- | indra/llmessage/CMakeLists.txt | 12 | ||||
| -rw-r--r-- | indra/llmessage/llassetstorage.cpp | 97 | ||||
| -rw-r--r-- | indra/llmessage/llassetstorage.h | 26 | ||||
| -rw-r--r-- | indra/llmessage/llcorehttputil.cpp | 4 | ||||
| -rw-r--r-- | indra/llmessage/llextendedstatus.h | 14 | ||||
| -rw-r--r-- | indra/llmessage/lltransfersourceasset.cpp | 8 | ||||
| -rw-r--r-- | indra/llmessage/lltransfersourceasset.h | 4 | ||||
| -rw-r--r-- | indra/llmessage/lltransfertargetvfile.cpp | 9 | ||||
| -rw-r--r-- | indra/llmessage/lltransfertargetvfile.h | 4 | ||||
| -rw-r--r-- | indra/llmessage/llxfer_vfile.cpp | 57 | ||||
| -rw-r--r-- | indra/llmessage/llxfer_vfile.h | 12 | ||||
| -rw-r--r-- | indra/llmessage/llxfermanager.cpp | 35 | ||||
| -rw-r--r-- | indra/llmessage/llxfermanager.h | 12 | 
13 files changed, 177 insertions, 117 deletions
| diff --git a/indra/llmessage/CMakeLists.txt b/indra/llmessage/CMakeLists.txt index f0a1dfe940..2f99ca069e 100644 --- a/indra/llmessage/CMakeLists.txt +++ b/indra/llmessage/CMakeLists.txt @@ -9,7 +9,7 @@ include(LLCommon)  include(LLCoreHttp)  include(LLMath)  include(LLMessage) -include(LLFileSystem) +include(LLVFS)  include(LLAddBuildTest)  include(Python)  include(Tut) @@ -23,7 +23,7 @@ include_directories(      ${LLCOREHTTP_INCLUDE_DIRS}      ${LLMATH_INCLUDE_DIRS}      ${LLMESSAGE_INCLUDE_DIRS} -    ${LLFILESYSTEM_INCLUDE_DIRS} +    ${LLVFS_INCLUDE_DIRS}      ${JSONCPP_INCLUDE_DIR}      ) @@ -209,7 +209,7 @@ target_link_libraries(    llmessage    ${CURL_LIBRARIES}    ${LLCOMMON_LIBRARIES} -  ${LLFILESYSTEM_LIBRARIES} +  ${LLVFS_LIBRARIES}    ${LLMATH_LIBRARIES}    ${JSONCPP_LIBRARIES}    ${OPENSSL_LIBRARIES} @@ -227,7 +227,7 @@ target_link_libraries(    llmessage    ${CURL_LIBRARIES}    ${LLCOMMON_LIBRARIES} -  ${LLFILESYSTEM_LIBRARIES} +  ${LLVFS_LIBRARIES}    ${LLMATH_LIBRARIES}    ${JSONCPP_LIBRARIES}    ${OPENSSL_LIBRARIES} @@ -257,7 +257,7 @@ if (LL_TESTS)  if (LINUX)    set(test_libs      ${WINDOWS_LIBRARIES} -    ${LLFILESYSTEM_LIBRARIES} +    ${LLVFS_LIBRARIES}      ${LLMATH_LIBRARIES}      ${CURL_LIBRARIES}      ${NGHTTP2_LIBRARIES} @@ -273,7 +273,7 @@ if (LINUX)  else (LINUX)    set(test_libs      ${WINDOWS_LIBRARIES} -    ${LLFILESYSTEM_LIBRARIES} +    ${LLVFS_LIBRARIES}      ${LLMATH_LIBRARIES}      ${CURL_LIBRARIES}      ${NGHTTP2_LIBRARIES} diff --git a/indra/llmessage/llassetstorage.cpp b/indra/llmessage/llassetstorage.cpp index f38a5e663e..d7801b6ddc 100644 --- a/indra/llmessage/llassetstorage.cpp +++ b/indra/llmessage/llassetstorage.cpp @@ -42,7 +42,8 @@  // this library includes  #include "message.h"  #include "llxfermanager.h" -#include "llfilesystem.h" +#include "llvfile.h" +#include "llvfs.h"  #include "lldbstrings.h"  #include "lltransfersourceasset.h" @@ -201,7 +202,7 @@ LLBaseDownloadRequest::LLBaseDownloadRequest(const LLUUID &uuid, const LLAssetTy        mIsTemp(FALSE),        mIsPriority(FALSE),        mDataSentInFirstPacket(FALSE), -      mDataIsInCache(FALSE) +      mDataIsInVFS(FALSE)  {      // Need to guarantee that this time is up to date, we may be creating a circuit even though we haven't been      //  running a message system loop. @@ -265,8 +266,7 @@ LLSD LLAssetRequest::getFullDetails() const      sd["is_local"] = mIsLocal;      sd["is_priority"] = mIsPriority;      sd["data_send_in_first_packet"] = mDataSentInFirstPacket; -    // Note: cannot change this (easily) since it is consumed by server -    sd["data_is_in_vfs"] = mDataIsInCache; +    sd["data_is_in_vfs"] = mDataIsInVFS;      return sd;  } @@ -330,23 +330,28 @@ LLBaseDownloadRequest* LLEstateAssetRequest::getCopy()  // TODO: rework tempfile handling? -LLAssetStorage::LLAssetStorage(LLMessageSystem *msg, LLXferManager *xfer, const LLHost &upstream_host) +LLAssetStorage::LLAssetStorage(LLMessageSystem *msg, LLXferManager *xfer, LLVFS *vfs, LLVFS *static_vfs, const LLHost &upstream_host)  { -    _init(msg, xfer, upstream_host); +    _init(msg, xfer, vfs, static_vfs, upstream_host);  } -LLAssetStorage::LLAssetStorage(LLMessageSystem *msg, LLXferManager *xfer) +LLAssetStorage::LLAssetStorage(LLMessageSystem *msg, LLXferManager *xfer, +                               LLVFS *vfs, LLVFS *static_vfs)  { -    _init(msg, xfer, LLHost()); +    _init(msg, xfer, vfs, static_vfs, LLHost());  }  void LLAssetStorage::_init(LLMessageSystem *msg,                             LLXferManager *xfer, +                           LLVFS *vfs, +                           LLVFS *static_vfs,                             const LLHost &upstream_host)  {      mShutDown = FALSE;      mMessageSys = msg;      mXferManager = xfer; +    mVFS = vfs; +    mStaticVFS = static_vfs;      setUpstream(upstream_host);      msg->setHandlerFuncFast(_PREHASH_AssetUploadComplete, processUploadComplete, (void **)this); @@ -425,7 +430,7 @@ void LLAssetStorage::_cleanupRequests(BOOL all, S32 error)          }          if (tmp->mDownCallback)          { -            tmp->mDownCallback(tmp->getUUID(), tmp->getType(), tmp->mUserData, error, LLExtStat::NONE); +            tmp->mDownCallback(mVFS, tmp->getUUID(), tmp->getType(), tmp->mUserData, error, LLExtStat::NONE);          }          if (tmp->mInfoCallback)          { @@ -438,10 +443,10 @@ void LLAssetStorage::_cleanupRequests(BOOL all, S32 error)  BOOL LLAssetStorage::hasLocalAsset(const LLUUID &uuid, const LLAssetType::EType type)  { -    return LLFileSystem::getExists(uuid, type); +    return mStaticVFS->getExists(uuid, type) || mVFS->getExists(uuid, type);  } -bool LLAssetStorage::findInCacheAndInvokeCallback(const LLUUID& uuid, LLAssetType::EType type, +bool LLAssetStorage::findInStaticVFSAndInvokeCallback(const LLUUID& uuid, LLAssetType::EType type,                                                        LLGetAssetCallback callback, void *user_data)  {      if (user_data) @@ -450,17 +455,17 @@ bool LLAssetStorage::findInCacheAndInvokeCallback(const LLUUID& uuid, LLAssetTyp          llassert(callback != NULL);      } -    BOOL exists = LLFileSystem::getExists(uuid, type); +    BOOL exists = mStaticVFS->getExists(uuid, type);      if (exists)      { -        LLFileSystem file(uuid, type); +        LLVFile file(mStaticVFS, uuid, type);          U32 size = file.getSize();          if (size > 0)          {              // we've already got the file              if (callback)              { -                callback(uuid, type, user_data, LL_ERR_NOERR, LLExtStat::CACHE_CACHED); +                callback(mStaticVFS, uuid, type, user_data, LL_ERR_NOERR, LLExtStat::VFS_CACHED);              }              return true;          } @@ -501,7 +506,7 @@ void LLAssetStorage::getAssetData(const LLUUID uuid,          if (callback)          {              add(sFailedDownloadCount, 1); -            callback(uuid, type, user_data, LL_ERR_ASSET_REQUEST_FAILED, LLExtStat::NONE); +            callback(mVFS, uuid, type, user_data, LL_ERR_ASSET_REQUEST_FAILED, LLExtStat::NONE);          }          return;      } @@ -512,19 +517,20 @@ void LLAssetStorage::getAssetData(const LLUUID uuid,          if (callback)          {              add(sFailedDownloadCount, 1); -            callback(uuid, type, user_data, LL_ERR_ASSET_REQUEST_NOT_IN_DATABASE, LLExtStat::NULL_UUID); +            callback(mVFS, uuid, type, user_data, LL_ERR_ASSET_REQUEST_NOT_IN_DATABASE, LLExtStat::NULL_UUID);          }          return;      } -    if (findInCacheAndInvokeCallback(uuid,type,callback,user_data)) +    // Try static VFS first. +    if (findInStaticVFSAndInvokeCallback(uuid,type,callback,user_data))      { -        LL_DEBUGS("AssetStorage") << "ASSET_TRACE asset " << uuid << " found in cache" << LL_ENDL; +        LL_DEBUGS("AssetStorage") << "ASSET_TRACE asset " << uuid << " found in static VFS" << LL_ENDL;          return;      } -    BOOL exists = LLFileSystem::getExists(uuid, type); -    LLFileSystem file(uuid, type); +    BOOL exists = mVFS->getExists(uuid, type); +    LLVFile file(mVFS, uuid, type);      U32 size = exists ? file.getSize() : 0;      if (size > 0) @@ -534,10 +540,10 @@ void LLAssetStorage::getAssetData(const LLUUID uuid,          // unless there's a weird error          if (callback)          { -            callback(uuid, type, user_data, LL_ERR_NOERR, LLExtStat::CACHE_CACHED); +            callback(mVFS, uuid, type, user_data, LL_ERR_NOERR, LLExtStat::VFS_CACHED);          } -        LL_DEBUGS("AssetStorage") << "ASSET_TRACE asset " << uuid << " found in cache" << LL_ENDL; +        LL_DEBUGS("AssetStorage") << "ASSET_TRACE asset " << uuid << " found in VFS" << LL_ENDL;      }      else      { @@ -610,7 +616,7 @@ void LLAssetStorage::removeAndCallbackPendingDownloads(const LLUUID& file_id, LL              {                  add(sFailedDownloadCount, 1);              } -            tmp->mDownCallback(callback_id, callback_type, tmp->mUserData, result_code, ext_status); +            tmp->mDownCallback(gAssetStorage->mVFS, callback_id, callback_type, tmp->mUserData, result_code, ext_status);          }          delete tmp;      } @@ -664,7 +670,7 @@ void LLAssetStorage::downloadCompleteCallback(      if (LL_ERR_NOERR == result)      {          // we might have gotten a zero-size file -        LLFileSystem vfile(callback_id, callback_type); +        LLVFile vfile(gAssetStorage->mVFS, callback_id, callback_type);          if (vfile.getSize() <= 0)          {              LL_WARNS("AssetStorage") << "downloadCompleteCallback has non-existent or zero-size asset " << callback_id << LL_ENDL; @@ -713,19 +719,19 @@ void LLAssetStorage::getEstateAsset(          if (callback)          {              add(sFailedDownloadCount, 1); -            callback(asset_id, atype, user_data, LL_ERR_ASSET_REQUEST_NOT_IN_DATABASE, LLExtStat::NULL_UUID); +            callback(mVFS, asset_id, atype, user_data, LL_ERR_ASSET_REQUEST_NOT_IN_DATABASE, LLExtStat::NULL_UUID);          }          return;      } -    // Try static first. -    if (findInCacheAndInvokeCallback(asset_id,atype,callback,user_data)) +    // Try static VFS first. +    if (findInStaticVFSAndInvokeCallback(asset_id,atype,callback,user_data))      {          return;      } -    BOOL exists = LLFileSystem::getExists(asset_id, atype); -    LLFileSystem file(asset_id, atype); +    BOOL exists = mVFS->getExists(asset_id, atype); +    LLVFile file(mVFS, asset_id, atype);      U32 size = exists ? file.getSize() : 0;      if (size > 0) @@ -735,7 +741,7 @@ void LLAssetStorage::getEstateAsset(          // unless there's a weird error          if (callback)          { -            callback(asset_id, atype, user_data, LL_ERR_NOERR, LLExtStat::CACHE_CACHED); +            callback(mVFS, asset_id, atype, user_data, LL_ERR_NOERR, LLExtStat::VFS_CACHED);          }      }      else @@ -786,7 +792,7 @@ void LLAssetStorage::getEstateAsset(              if (callback)              {                  add(sFailedDownloadCount, 1); -                callback(asset_id, atype, user_data, LL_ERR_CIRCUIT_GONE, LLExtStat::NO_UPSTREAM); +                callback(mVFS, asset_id, atype, user_data, LL_ERR_CIRCUIT_GONE, LLExtStat::NO_UPSTREAM);              }          }      } @@ -818,7 +824,7 @@ void LLAssetStorage::downloadEstateAssetCompleteCallback(      if (LL_ERR_NOERR == result)      {          // we might have gotten a zero-size file -        LLFileSystem vfile(req->getUUID(), req->getAType()); +        LLVFile vfile(gAssetStorage->mVFS, req->getUUID(), req->getAType());          if (vfile.getSize() <= 0)          {              LL_WARNS("AssetStorage") << "downloadCompleteCallback has non-existent or zero-size asset!" << LL_ENDL; @@ -832,7 +838,7 @@ void LLAssetStorage::downloadEstateAssetCompleteCallback(      {          add(sFailedDownloadCount, 1);      } -    req->mDownCallback(req->getUUID(), req->getAType(), req->mUserData, result, ext_status); +    req->mDownCallback(gAssetStorage->mVFS, req->getUUID(), req->getAType(), req->mUserData, result, ext_status);  }  void LLAssetStorage::getInvItemAsset( @@ -855,13 +861,14 @@ void LLAssetStorage::getInvItemAsset(      if(asset_id.notNull())      { -        if (findInCacheAndInvokeCallback( asset_id, atype, callback, user_data)) +        // Try static VFS first. +        if (findInStaticVFSAndInvokeCallback( asset_id, atype, callback, user_data))          {              return;          } -        exists = LLFileSystem::getExists(asset_id, atype); -        LLFileSystem file(asset_id, atype); +        exists = mVFS->getExists(asset_id, atype); +        LLVFile file(mVFS, asset_id, atype);          size = exists ? file.getSize() : 0;          if(exists && size < 1)          { @@ -878,7 +885,7 @@ void LLAssetStorage::getInvItemAsset(          // unless there's a weird error          if (callback)          { -            callback(asset_id, atype, user_data, LL_ERR_NOERR, LLExtStat::CACHE_CACHED); +            callback(mVFS, asset_id, atype, user_data, LL_ERR_NOERR, LLExtStat::VFS_CACHED);          }      }      else @@ -929,7 +936,7 @@ void LLAssetStorage::getInvItemAsset(              if (callback)              {                  add(sFailedDownloadCount, 1); -                callback(asset_id, atype, user_data, LL_ERR_CIRCUIT_GONE, LLExtStat::NO_UPSTREAM); +                callback(mVFS, asset_id, atype, user_data, LL_ERR_CIRCUIT_GONE, LLExtStat::NO_UPSTREAM);              }          }      } @@ -961,7 +968,7 @@ void LLAssetStorage::downloadInvItemCompleteCallback(      if (LL_ERR_NOERR == result)      {          // we might have gotten a zero-size file -        LLFileSystem vfile(req->getUUID(), req->getType()); +        LLVFile vfile(gAssetStorage->mVFS, req->getUUID(), req->getType());          if (vfile.getSize() <= 0)          {              LL_WARNS("AssetStorage") << "downloadCompleteCallback has non-existent or zero-size asset!" << LL_ENDL; @@ -975,7 +982,7 @@ void LLAssetStorage::downloadInvItemCompleteCallback(      {          add(sFailedDownloadCount, 1);      } -    req->mDownCallback(req->getUUID(), req->getType(), req->mUserData, result, ext_status); +    req->mDownCallback(gAssetStorage->mVFS, req->getUUID(), req->getType(), req->mUserData, result, ext_status);  }  ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -1286,7 +1293,7 @@ bool LLAssetStorage::deletePendingRequestImpl(LLAssetStorage::request_list_t* re          if (req->mDownCallback)          {              add(sFailedDownloadCount, 1); -            req->mDownCallback(req->getUUID(), req->getType(), req->mUserData, error, LLExtStat::REQUEST_DROPPED); +            req->mDownCallback(mVFS, req->getUUID(), req->getType(), req->mUserData, error, LLExtStat::REQUEST_DROPPED);          }          if (req->mInfoCallback)          { @@ -1356,7 +1363,8 @@ void LLAssetStorage::getAssetData(const LLUUID uuid,      {          LLAssetRequest* tmp = *iter++; -        auto cbptr = tmp->mDownCallback.target<void(*)(const LLUUID &, LLAssetType::EType, void *, S32, LLExtStat)>(); +        //void(*const* cbptr)(LLVFS *, const LLUUID &, LLAssetType::EType, void *, S32, LLExtStat)  +        auto cbptr = tmp->mDownCallback.target<void(*)(LLVFS *, const LLUUID &, LLAssetType::EType, void *, S32, LLExtStat)>();          if (type == tmp->getType() &&               uuid == tmp->getUUID() && @@ -1381,7 +1389,8 @@ void LLAssetStorage::getAssetData(const LLUUID uuid,  }  // static -void LLAssetStorage::legacyGetDataCallback(const LLUUID &uuid,  +void LLAssetStorage::legacyGetDataCallback(LLVFS *vfs, +                                           const LLUUID &uuid,                                              LLAssetType::EType type,                                             void *user_data,                                              S32 status,  @@ -1396,7 +1405,7 @@ void LLAssetStorage::legacyGetDataCallback(const LLUUID &uuid,      if ( !status           && !toxic )      { -        LLFileSystem file(uuid, type); +        LLVFile file(vfs, uuid, type);          std::string uuid_str; diff --git a/indra/llmessage/llassetstorage.h b/indra/llmessage/llassetstorage.h index e0f22f1160..c799d8eefc 100644 --- a/indra/llmessage/llassetstorage.h +++ b/indra/llmessage/llassetstorage.h @@ -44,6 +44,7 @@  class LLMessageSystem;  class LLXferManager;  class LLAssetStorage; +class LLVFS;  class LLSD;  // anything that takes longer than this to download will abort. @@ -59,11 +60,11 @@ const int LL_ERR_ASSET_REQUEST_NOT_IN_DATABASE = -4;  const int LL_ERR_INSUFFICIENT_PERMISSIONS = -5;  const int LL_ERR_PRICE_MISMATCH = -23018; -// *TODO: these typedefs are passed into the cache via a legacy C function pointer +// *TODO: these typedefs are passed into the VFS via a legacy C function pointer  // future project would be to convert these to C++ callables (std::function<>) so that   // we can use bind and remove the userData parameter.  //  -typedef std::function<void(const LLUUID &asset_id, LLAssetType::EType asset_type, void *user_data, S32 status, LLExtStat ext_status)> LLGetAssetCallback; +typedef std::function<void(LLVFS *vfs, const LLUUID &asset_id, LLAssetType::EType asset_type, void *user_data, S32 status, LLExtStat ext_status)> LLGetAssetCallback;  typedef std::function<void(const LLUUID &asset_id, void *user_data, S32 status, LLExtStat ext_status)> LLStoreAssetCallback; @@ -119,6 +120,7 @@ protected:  public:      LLGetAssetCallback mDownCallback; +//    void(*mDownCallback)(LLVFS*, const LLUUID&, LLAssetType::EType, void *, S32, LLExtStat);      void	*mUserData;      LLHost  mHost; @@ -126,7 +128,7 @@ public:      F64Seconds		mTime;				// Message system time      BOOL    mIsPriority;      BOOL	mDataSentInFirstPacket; -    BOOL	mDataIsInCache; +    BOOL	mDataIsInVFS;  };  class LLAssetRequest : public LLBaseDownloadRequest @@ -196,6 +198,9 @@ typedef std::map<LLUUID,U64,lluuid_less> toxic_asset_map_t;  class LLAssetStorage  {  public: +	// VFS member is public because static child methods need it :( +	LLVFS *mVFS; +	LLVFS *mStaticVFS;      typedef ::LLStoreAssetCallback LLStoreAssetCallback;      typedef ::LLGetAssetCallback LLGetAssetCallback; @@ -225,9 +230,11 @@ protected:  	toxic_asset_map_t	mToxicAssetMap;		// Objects in this list are known to cause problems and are not loaded  public: -	LLAssetStorage(LLMessageSystem *msg, LLXferManager *xfer, const LLHost &upstream_host); +	LLAssetStorage(LLMessageSystem *msg, LLXferManager *xfer, +				   LLVFS *vfs, LLVFS *static_vfs, const LLHost &upstream_host); -	LLAssetStorage(LLMessageSystem *msg, LLXferManager *xfer); +	LLAssetStorage(LLMessageSystem *msg, LLXferManager *xfer, +				   LLVFS *vfs, LLVFS *static_vfs);  	virtual ~LLAssetStorage();  	void setUpstream(const LLHost &upstream_host); @@ -277,7 +284,7 @@ public:  	void		markAssetToxic( const LLUUID& uuid );  protected: -	bool findInCacheAndInvokeCallback(const LLUUID& uuid, LLAssetType::EType type, +	bool findInStaticVFSAndInvokeCallback(const LLUUID& uuid, LLAssetType::EType type,  										  LLGetAssetCallback callback, void *user_data);  	LLSD getPendingDetailsImpl(const request_list_t* requests, @@ -368,7 +375,7 @@ public:  		bool user_waiting = false,  		F64Seconds timeout  = LL_ASSET_STORAGE_TIMEOUT) = 0; -	static void legacyGetDataCallback(const LLUUID &uuid, LLAssetType::EType, void *user_data, S32 status, LLExtStat ext_status); +	static void legacyGetDataCallback(LLVFS *vfs, const LLUUID &uuid, LLAssetType::EType, void *user_data, S32 status, LLExtStat ext_status);  	static void legacyStoreDataCallback(const LLUUID &uuid, void *user_data, S32 status, LLExtStat ext_status);  	// add extra methods to handle metadata @@ -378,12 +385,15 @@ protected:  	void _callUploadCallbacks(const LLUUID &uuid, const LLAssetType::EType asset_type, BOOL success, LLExtStat ext_status);  	virtual void _queueDataRequest(const LLUUID& uuid, LLAssetType::EType type, LLGetAssetCallback callback, +//								   void (*callback)(LLVFS *vfs, const LLUUID&, LLAssetType::EType, void *, S32, LLExtStat),  								   void *user_data, BOOL duplicate,  								   BOOL is_priority) = 0;  private:  	void _init(LLMessageSystem *msg,  			   LLXferManager *xfer, +			   LLVFS *vfs, +			   LLVFS *static_vfs,  			   const LLHost &upstream_host);  protected: @@ -398,7 +408,7 @@ protected:  		MR_FILE_NONEXIST	= 3,	// Old format store call - source file does not exist  		MR_NO_FILENAME		= 4,	// Old format store call - source filename is NULL/0-length  		MR_NO_UPSTREAM		= 5,	// Upstream provider is missing -		MR_CACHE_CORRUPTION	= 6		// cache is corrupt - too-large or mismatched stated/returned sizes +		MR_VFS_CORRUPTION	= 6		// VFS is corrupt - too-large or mismatched stated/returned sizes  	};  	static class LLMetrics *metric_recipient; diff --git a/indra/llmessage/llcorehttputil.cpp b/indra/llmessage/llcorehttputil.cpp index 7031f1aa8c..0eae6d9826 100644 --- a/indra/llmessage/llcorehttputil.cpp +++ b/indra/llmessage/llcorehttputil.cpp @@ -37,7 +37,7 @@  #include "llsdserialize.h"  #include "reader.h" // JSON  #include "writer.h" // JSON -#include "llfilesystem.h" +#include "llvfile.h"  #include "message.h" // for getting the port @@ -784,7 +784,7 @@ LLSD HttpCoroutineAdapter::postFileAndSuspend(LLCore::HttpRequest::ptr_t request      // scoping for our streams so that they go away when we no longer need them.      {          LLCore::BufferArrayStream outs(fileData.get()); -        LLFileSystem vfile(assetId, assetType, LLFileSystem::READ); +        LLVFile vfile(gVFS, assetId, assetType, LLVFile::READ);          S32 fileSize = vfile.getSize();          U8* fileBuffer; diff --git a/indra/llmessage/llextendedstatus.h b/indra/llmessage/llextendedstatus.h index 2a53dced80..9923d73c1a 100644 --- a/indra/llmessage/llextendedstatus.h +++ b/indra/llmessage/llextendedstatus.h @@ -1,7 +1,7 @@  /**    * @file llextendedstatus.h   * @date   August 2007 - * @brief extended status codes for curl/resident asset storage and delivery + * @brief extended status codes for curl/vfs/resident asset storage and delivery   *   * $LicenseInfo:firstyear=2007&license=viewerlgpl$   * Second Life Viewer Source Code @@ -32,9 +32,9 @@ enum class LLExtStat: uint32_t  {  	// Status provider groups - Top bits indicate which status type it is  	// Zero is common status code (next section) -	CURL_RESULT	 = 1UL<<30, // serviced by curl - use 1L if we really implement the below -	RES_RESULT	 = 2UL<<30, // serviced by resident copy -	CACHE_RESULT = 3UL<<30, // serviced by cache +	CURL_RESULT	= 1UL<<30, // serviced by curl - use 1L if we really implement the below +	RES_RESULT	= 2UL<<30, // serviced by resident copy +	VFS_RESULT	= 3UL<<30, // serviced by vfs  	// Common Status Codes @@ -54,9 +54,9 @@ enum class LLExtStat: uint32_t  	// Memory-Resident status codes:  	// None at present -	// CACHE status codes: -	CACHE_CACHED	= CACHE_RESULT | 0x0001, -	CACHE_CORRUPT	= CACHE_RESULT | 0x0002, +	// VFS status codes: +	VFS_CACHED	= VFS_RESULT | 0x0001, +	VFS_CORRUPT	= VFS_RESULT | 0x0002,  }; diff --git a/indra/llmessage/lltransfersourceasset.cpp b/indra/llmessage/lltransfersourceasset.cpp index 027283232d..80ed3340c6 100644 --- a/indra/llmessage/lltransfersourceasset.cpp +++ b/indra/llmessage/lltransfersourceasset.cpp @@ -32,7 +32,7 @@  #include "message.h"  #include "lldatapacker.h"  #include "lldir.h" -#include "llfilesystem.h" +#include "llvfile.h"  LLTransferSourceAsset::LLTransferSourceAsset(const LLUUID &request_id, const F32 priority) :  	LLTransferSource(LLTST_ASSET, request_id, priority), @@ -99,7 +99,7 @@ LLTSCode LLTransferSourceAsset::dataCallback(const S32 packet_id,  		return LLTS_SKIP;  	} -	LLFileSystem vf(mParams.getAssetID(), mParams.getAssetType(), LLFileSystem::READ); +	LLVFile vf(gAssetStorage->mVFS, mParams.getAssetID(), mParams.getAssetType(), LLVFile::READ);  	if (!vf.getSize())  	{ @@ -171,7 +171,7 @@ BOOL LLTransferSourceAsset::unpackParams(LLDataPacker &dp)  } -void LLTransferSourceAsset::responderCallback(const LLUUID& uuid, LLAssetType::EType type, +void LLTransferSourceAsset::responderCallback(LLVFS *vfs, const LLUUID& uuid, LLAssetType::EType type,  											  void *user_data, S32 result, LLExtStat ext_status )  {  	LLUUID *tidp = ((LLUUID*) user_data); @@ -198,7 +198,7 @@ void LLTransferSourceAsset::responderCallback(const LLUUID& uuid, LLAssetType::E  	if (LL_ERR_NOERR == result)  	{  		// Everything's OK. -		LLFileSystem vf(uuid, type, LLFileSystem::READ); +		LLVFile vf(gAssetStorage->mVFS, uuid, type, LLVFile::READ);  		tsap->mSize = vf.getSize();  		status = LLTS_OK;  	} diff --git a/indra/llmessage/lltransfersourceasset.h b/indra/llmessage/lltransfersourceasset.h index 585e683cb3..3abda83cf8 100644 --- a/indra/llmessage/lltransfersourceasset.h +++ b/indra/llmessage/lltransfersourceasset.h @@ -30,7 +30,7 @@  #include "lltransfermanager.h"  #include "llassetstorage.h" -class LLFileSystem; +class LLVFile;  class LLTransferSourceParamsAsset : public LLTransferSourceParams  { @@ -56,7 +56,7 @@ public:  	LLTransferSourceAsset(const LLUUID &request_id, const F32 priority);  	virtual ~LLTransferSourceAsset(); -	static void responderCallback(const LLUUID& uuid, LLAssetType::EType type, +	static void responderCallback(LLVFS *vfs, const LLUUID& uuid, LLAssetType::EType type,  								  void *user_data, S32 result, LLExtStat ext_status );  protected:  	/*virtual*/ void initTransfer(); diff --git a/indra/llmessage/lltransfertargetvfile.cpp b/indra/llmessage/lltransfertargetvfile.cpp index f6faadf87f..b27f0881e0 100644 --- a/indra/llmessage/lltransfertargetvfile.cpp +++ b/indra/llmessage/lltransfertargetvfile.cpp @@ -30,7 +30,7 @@  #include "lldatapacker.h"  #include "llerror.h" -#include "llfilesystem.h" +#include "llvfile.h"  //static  void LLTransferTargetVFile::updateQueue(bool shutdown) @@ -138,9 +138,10 @@ LLTSCode LLTransferTargetVFile::dataCallback(const S32 packet_id, U8 *in_datap,  	//LL_INFOS() << "LLTransferTargetFile::dataCallback" << LL_ENDL;  	//LL_INFOS() << "Packet: " << packet_id << LL_ENDL; -	LLFileSystem vf(mTempID, mParams.getAssetType(), LLFileSystem::APPEND); +	LLVFile vf(gAssetStorage->mVFS, mTempID, mParams.getAssetType(), LLVFile::APPEND);  	if (mNeedsCreate)  	{ +		vf.setMaxSize(mSize);  		mNeedsCreate = FALSE;  	} @@ -175,7 +176,7 @@ void LLTransferTargetVFile::completionCallback(const LLTSCode status)  	  case LLTS_DONE:  		if (!mNeedsCreate)  		{ -			LLFileSystem file(mTempID, mParams.getAssetType(), LLFileSystem::WRITE); +			LLVFile file(gAssetStorage->mVFS, mTempID, mParams.getAssetType(), LLVFile::WRITE);  			if (!file.rename(mParams.getAssetID(), mParams.getAssetType()))  			{  				LL_ERRS() << "LLTransferTargetVFile: rename failed" << LL_ENDL; @@ -194,7 +195,7 @@ void LLTransferTargetVFile::completionCallback(const LLTSCode status)  	  {  		  // We're aborting this transfer, we don't want to keep this file.  		  LL_WARNS() << "Aborting vfile transfer for " << mParams.getAssetID() << LL_ENDL; -		  LLFileSystem vf(mTempID, mParams.getAssetType(), LLFileSystem::APPEND); +		  LLVFile vf(gAssetStorage->mVFS, mTempID, mParams.getAssetType(), LLVFile::APPEND);  		  vf.remove();  	  }  	  break; diff --git a/indra/llmessage/lltransfertargetvfile.h b/indra/llmessage/lltransfertargetvfile.h index 39a9125f1b..c819c1e2f2 100644 --- a/indra/llmessage/lltransfertargetvfile.h +++ b/indra/llmessage/lltransfertargetvfile.h @@ -29,9 +29,9 @@  #include "lltransfermanager.h"  #include "llassetstorage.h" -#include "llfilesystem.h" +#include "llvfile.h" -class LLFileSystem; +class LLVFile;  // Lame, an S32 for now until I figure out the deal with how we want to do  // error codes. diff --git a/indra/llmessage/llxfer_vfile.cpp b/indra/llmessage/llxfer_vfile.cpp index 12419b342d..ddc24342f6 100644 --- a/indra/llmessage/llxfer_vfile.cpp +++ b/indra/llmessage/llxfer_vfile.cpp @@ -30,7 +30,8 @@  #include "lluuid.h"  #include "llerror.h"  #include "llmath.h" -#include "llfilesystem.h" +#include "llvfile.h" +#include "llvfs.h"  #include "lldir.h"  // size of chunks read from/written to disk @@ -41,13 +42,13 @@ const U32 LL_MAX_XFER_FILE_BUFFER = 65536;  LLXfer_VFile::LLXfer_VFile ()  : LLXfer(-1)  { -	init(LLUUID::null, LLAssetType::AT_NONE); +	init(NULL, LLUUID::null, LLAssetType::AT_NONE);  } -LLXfer_VFile::LLXfer_VFile (const LLUUID &local_id, LLAssetType::EType type) +LLXfer_VFile::LLXfer_VFile (LLVFS *vfs, const LLUUID &local_id, LLAssetType::EType type)  : LLXfer(-1)  { -	init(local_id, type); +	init(vfs, local_id, type);  }  /////////////////////////////////////////////////////////// @@ -59,8 +60,10 @@ LLXfer_VFile::~LLXfer_VFile ()  /////////////////////////////////////////////////////////// -void LLXfer_VFile::init (const LLUUID &local_id, LLAssetType::EType type) +void LLXfer_VFile::init (LLVFS *vfs, const LLUUID &local_id, LLAssetType::EType type)  { + +	mVFS = vfs;  	mLocalID = local_id;  	mType = type; @@ -79,14 +82,14 @@ void LLXfer_VFile::cleanup ()  	if (mTempID.notNull() &&  		mDeleteTempFile)  	{ -		if (LLFileSystem::getExists(mTempID, mType)) +		if (mVFS->getExists(mTempID, mType))  		{ -			LLFileSystem file(mTempID, mType, LLFileSystem::WRITE); +			LLVFile file(mVFS, mTempID, mType, LLVFile::WRITE);  			file.remove();  		}  		else  		{ -			LL_WARNS("Xfer") << "LLXfer_VFile::cleanup() can't open to delete cache file " << mTempID << "." << LLAssetType::lookup(mType)		 +			LL_WARNS("Xfer") << "LLXfer_VFile::cleanup() can't open to delete VFS file " << mTempID << "." << LLAssetType::lookup(mType)		  				<< ", mRemoteID is " << mRemoteID << LL_ENDL;  		}  	} @@ -100,6 +103,7 @@ void LLXfer_VFile::cleanup ()  ///////////////////////////////////////////////////////////  S32 LLXfer_VFile::initializeRequest(U64 xfer_id, +									LLVFS* vfs,  									const LLUUID& local_id,  									const LLUUID& remote_id,  									LLAssetType::EType type, @@ -111,6 +115,7 @@ S32 LLXfer_VFile::initializeRequest(U64 xfer_id,  	mRemoteHost = remote_host; +	mVFS = vfs;  	mLocalID = local_id;  	mRemoteID = remote_id;  	mType = type; @@ -187,13 +192,13 @@ S32 LLXfer_VFile::startSend (U64 xfer_id, const LLHost &remote_host)  	delete mVFile;  	mVFile = NULL; -	if(LLFileSystem::getExists(mLocalID, mType)) +	if(mVFS->getExists(mLocalID, mType))  	{ -		mVFile = new LLFileSystem(mLocalID, mType, LLFileSystem::READ); +		mVFile = new LLVFile(mVFS, mLocalID, mType, LLVFile::READ);  		if (mVFile->getSize() <= 0)  		{ -			LL_WARNS("Xfer") << "LLXfer_VFile::startSend() cache file " << mLocalID << "." << LLAssetType::lookup(mType)		 +			LL_WARNS("Xfer") << "LLXfer_VFile::startSend() VFS file " << mLocalID << "." << LLAssetType::lookup(mType)		  				<< " has unexpected file size of " << mVFile->getSize() << LL_ENDL;  			delete mVFile;  			mVFile = NULL; @@ -209,7 +214,7 @@ S32 LLXfer_VFile::startSend (U64 xfer_id, const LLHost &remote_host)  	}  	else  	{ -		LL_WARNS("Xfer") << "LLXfer_VFile::startSend() can't read cache file " << mLocalID << "." << LLAssetType::lookup(mType) << LL_ENDL; +		LL_WARNS("Xfer") << "LLXfer_VFile::startSend() can't read VFS file " << mLocalID << "." << LLAssetType::lookup(mType) << LL_ENDL;  		retval = LL_ERR_FILE_NOT_FOUND;  	} @@ -235,13 +240,13 @@ S32 LLXfer_VFile::reopenFileHandle()  	if (mVFile == NULL)  	{ -		if (LLFileSystem::getExists(mLocalID, mType)) +		if (mVFS->getExists(mLocalID, mType))  		{ -			mVFile = new LLFileSystem(mLocalID, mType, LLFileSystem::READ); +			mVFile = new LLVFile(mVFS, mLocalID, mType, LLVFile::READ);  		}  		else  		{ -			LL_WARNS("Xfer") << "LLXfer_VFile::reopenFileHandle() can't read cache file " << mLocalID << "." << LLAssetType::lookup(mType) << LL_ENDL; +			LL_WARNS("Xfer") << "LLXfer_VFile::reopenFileHandle() can't read VFS file " << mLocalID << "." << LLAssetType::lookup(mType) << LL_ENDL;  			retval = LL_ERR_FILE_NOT_FOUND;  		}  	} @@ -260,7 +265,8 @@ void LLXfer_VFile::setXferSize (S32 xfer_size)  	// It would be nice if LLXFers could tell which end of the pipe they were  	if (! mVFile)  	{ -		LLFileSystem file(mTempID, mType, LLFileSystem::APPEND); +		LLVFile file(mVFS, mTempID, mType, LLVFile::APPEND); +		file.setMaxSize(xfer_size);  	}  } @@ -314,7 +320,7 @@ S32 LLXfer_VFile::flush()  	S32 retval = 0;  	if (mBufferLength)  	{ -		LLFileSystem file(mTempID, mType, LLFileSystem::APPEND); +		LLVFile file(mVFS, mTempID, mType, LLVFile::APPEND);  		file.write((U8*)mBuffer, mBufferLength); @@ -334,15 +340,22 @@ S32 LLXfer_VFile::processEOF()  	if (!mCallbackResult)  	{ -		if (LLFileSystem::getExists(mTempID, mType)) +		if (mVFS->getExists(mTempID, mType))  		{ -			LLFileSystem file(mTempID, mType, LLFileSystem::WRITE); +			LLVFile file(mVFS, mTempID, mType, LLVFile::WRITE);  			if (!file.rename(mLocalID, mType))  			{ -				LL_WARNS("Xfer") << "Cache rename of temp file failed: unable to rename " << mTempID << " to " << mLocalID << LL_ENDL; +				LL_WARNS("Xfer") << "VFS rename of temp file failed: unable to rename " << mTempID << " to " << mLocalID << LL_ENDL;  			}  			else -			{									 +			{					 +				#ifdef VFS_SPAM +				// Debugging spam +				LL_INFOS("Xfer") << "VFS rename of temp file done: renamed " << mTempID << " to " << mLocalID  +					<< " LLVFile size is " << file.getSize() +					<< LL_ENDL; +				#endif				 +				  				// Rename worked: the original file is gone.   Clear mDeleteTempFile  				// so we don't attempt to delete the file in cleanup()  				mDeleteTempFile = FALSE; @@ -350,7 +363,7 @@ S32 LLXfer_VFile::processEOF()  		}  		else  		{ -			LL_WARNS("Xfer") << "LLXfer_VFile::processEOF() can't open for renaming cache file " << mTempID << "." << LLAssetType::lookup(mType) << LL_ENDL; +			LL_WARNS("Xfer") << "LLXfer_VFile::processEOF() can't open for renaming VFS file " << mTempID << "." << LLAssetType::lookup(mType) << LL_ENDL;  		}  	} diff --git a/indra/llmessage/llxfer_vfile.h b/indra/llmessage/llxfer_vfile.h index d82bab5f6c..5bf9a5cfba 100644 --- a/indra/llmessage/llxfer_vfile.h +++ b/indra/llmessage/llxfer_vfile.h @@ -30,7 +30,8 @@  #include "llxfer.h"  #include "llassetstorage.h" -class LLFileSystem; +class LLVFS; +class LLVFile;  class LLXfer_VFile : public LLXfer  { @@ -40,7 +41,9 @@ class LLXfer_VFile : public LLXfer  	LLUUID mTempID;  	LLAssetType::EType mType; -	LLFileSystem *mVFile; +	LLVFile *mVFile; + +	LLVFS *mVFS;  	std::string mName; @@ -48,13 +51,14 @@ class LLXfer_VFile : public LLXfer   public:  	LLXfer_VFile (); -	LLXfer_VFile (const LLUUID &local_id, LLAssetType::EType type); +	LLXfer_VFile (LLVFS *vfs, const LLUUID &local_id, LLAssetType::EType type);  	virtual ~LLXfer_VFile(); -	virtual void init(const LLUUID &local_id, LLAssetType::EType type); +	virtual void init(LLVFS *vfs, const LLUUID &local_id, LLAssetType::EType type);  	virtual void cleanup();  	virtual S32 initializeRequest(U64 xfer_id, +			LLVFS *vfs,  			const LLUUID &local_id,  			const LLUUID &remote_id,  			const LLAssetType::EType type, diff --git a/indra/llmessage/llxfermanager.cpp b/indra/llmessage/llxfermanager.cpp index f9b59d7e42..4cea886c8a 100644 --- a/indra/llmessage/llxfermanager.cpp +++ b/indra/llmessage/llxfermanager.cpp @@ -56,9 +56,9 @@ const S32 LL_DEFAULT_MAX_HARD_LIMIT_SIMULTANEOUS_XFERS = 500;  /////////////////////////////////////////////////////////// -LLXferManager::LLXferManager () +LLXferManager::LLXferManager (LLVFS *vfs)  { -	init(); +	init(vfs);  }  /////////////////////////////////////////////////////////// @@ -70,7 +70,7 @@ LLXferManager::~LLXferManager ()  /////////////////////////////////////////////////////////// -void LLXferManager::init() +void LLXferManager::init (LLVFS *vfs)  {  	cleanup(); @@ -78,6 +78,8 @@ void LLXferManager::init()  	setHardLimitOutgoingXfersPerCircuit(LL_DEFAULT_MAX_HARD_LIMIT_SIMULTANEOUS_XFERS);  	setMaxIncomingXfers(LL_DEFAULT_MAX_REQUEST_FIFO_XFERS); +	mVFS = vfs; +  	// Turn on or off ack throttling  	mUseAckThrottling = FALSE;  	setAckThrottleBPS(100000); @@ -460,7 +462,7 @@ U64 LLXferManager::requestFile(const std::string& local_filename,  void LLXferManager::requestVFile(const LLUUID& local_id,  								 const LLUUID& remote_id, -								 LLAssetType::EType type, +								 LLAssetType::EType type, LLVFS* vfs,  								 const LLHost& remote_host,  								 void (*callback)(void**,S32,LLExtStat),  								 void** user_data, @@ -506,6 +508,7 @@ void LLXferManager::requestVFile(const LLUUID& local_id,  		addToList(xfer_p, mReceiveList, is_priority);  		((LLXfer_VFile *)xfer_p)->initializeRequest(getNextID(), +			vfs,  			local_id,  			remote_id,  			type, @@ -781,17 +784,33 @@ void LLXferManager::processFileRequest (LLMessageSystem *mesgsys, void ** /*user  	LLXfer *xferp;  	if (uuid != LLUUID::null) -	{	// Request for an asset - use a cache file +	{	// Request for an asset - use a VFS file  		if(NULL == LLAssetType::lookup(type))  		{  			LL_WARNS("Xfer") << "Invalid type for xfer request: " << uuid << ":"  					<< type_s16 << " to " << mesgsys->getSender() << LL_ENDL;  			return;  		} +			 +		if (! mVFS) +		{ +			LL_WARNS("Xfer") << "Attempt to send VFile w/o available VFS" << LL_ENDL; +			return; +		} + +		/* Present in fireengine, not used by viewer +		if (!validateVFileForTransfer(uuid.asString())) +		{ +			// it is up to the app sending the file to mark it for expected  +			// transfer before the request arrives or it will be dropped +			LL_WARNS("Xfer") << "SECURITY: Unapproved VFile '" << uuid << "'" << LL_ENDL; +			return; +		} +		*/  		LL_INFOS("Xfer") << "starting vfile transfer: " << uuid << "," << LLAssetType::lookup(type) << " to " << mesgsys->getSender() << LL_ENDL; -		xferp = (LLXfer *)new LLXfer_VFile(uuid, type); +		xferp = (LLXfer *)new LLXfer_VFile(mVFS, uuid, type);  		if (xferp)  		{  			mSendList.push_front(xferp); @@ -1254,9 +1273,9 @@ void LLXferManager::addToList(LLXfer* xferp, xfer_list_t & xfer_list, BOOL is_pr  LLXferManager *gXferManager = NULL; -void start_xfer_manager() +void start_xfer_manager(LLVFS *vfs)  { -	gXferManager = new LLXferManager(); +	gXferManager = new LLXferManager(vfs);  }  void cleanup_xfer_manager() diff --git a/indra/llmessage/llxfermanager.h b/indra/llmessage/llxfermanager.h index f49209bed0..45ae2ffdd3 100644 --- a/indra/llmessage/llxfermanager.h +++ b/indra/llmessage/llxfermanager.h @@ -35,6 +35,7 @@  //Forward declaration to avoid circular dependencies  class LLXfer; +class LLVFS;  #include "llxfer.h"  #include "message.h" @@ -71,6 +72,9 @@ public:  class LLXferManager  { + private: +	LLVFS *mVFS; +   protected:  	S32    mMaxOutgoingXfersPerCircuit;  	S32    mHardLimitOutgoingXfersPerCircuit;	// At this limit, kill off the connection @@ -107,10 +111,10 @@ class LLXferManager  	std::multiset<std::string> mExpectedVFileRequests;  // files that are authorized to be downloaded on top of   public: -	LLXferManager(); +	LLXferManager(LLVFS *vfs);  	virtual ~LLXferManager(); -	virtual void init(); +	virtual void init(LLVFS *vfs);  	virtual void cleanup();  	void setUseAckThrottling(const BOOL use); @@ -162,7 +166,7 @@ class LLXferManager  // vfile requesting  // .. to vfile  	virtual void requestVFile(const LLUUID &local_id, const LLUUID& remote_id, -							  LLAssetType::EType type, +							  LLAssetType::EType type, LLVFS* vfs,  							  const LLHost& remote_host,  							  void (*callback)(void**,S32,LLExtStat), void** user_data,  							  BOOL is_priority = FALSE); @@ -209,7 +213,7 @@ class LLXferManager  extern LLXferManager*	gXferManager;  // initialization and garbage collection -void start_xfer_manager(); +void start_xfer_manager(LLVFS *vfs);  void cleanup_xfer_manager();  // message system callbacks | 
