Merge branch 'master' into DRTVWR-522-maint

1 files changed, 253 insertions, 40 deletions

diff --git a/indra/newview/llappviewer.cpp b/indra/newview/llappviewer.cpp
index ed7addb41c..6c0cfd56ae 100644
--- a/indra/newview/llappviewer.cpp
+++ b/indra/newview/llappviewer.cpp
@@ -98,7 +98,6 @@
 #include "lllogininstance.h"
 #include "llprogressview.h"
 #include "llvocache.h"
-#include "lldiskcache.h"
 #include "llvopartgroup.h"
 #include "llweb.h"
 #include "llfloatertexturefetchdebugger.h"
@@ -116,6 +115,8 @@
 #include "llprimitive.h"
 #include "llurlaction.h"
 #include "llurlentry.h"
+#include "llvfile.h"
+#include "llvfsthread.h"
 #include "llvolumemgr.h"
 #include "llxfermanager.h"
 #include "llphysicsextensions.h"
@@ -339,6 +340,9 @@ bool			gUseWireframe = FALSE;
 //use for remember deferred mode in wireframe switch
 bool			gInitialDeferredModeForWireframe = FALSE;
 
+// VFS globals - see llappviewer.h
+LLVFS* gStaticVFS = NULL;
+
 LLMemoryInfo gSysMemory;
 U64Bytes gMemoryAllocated(0); // updated in display_stats() in llviewerdisplay.cpp
 
@@ -427,6 +431,12 @@ void init_default_trans_args()
 	default_trans_args.insert("create_account_url");
 }
 
+//----------------------------------------------------------------------------
+// File scope definitons
+const char *VFS_DATA_FILE_BASE = "data.db2.x.";
+const char *VFS_INDEX_FILE_BASE = "index.db2.x.";
+
+
 struct SettingsFile : public LLInitParam::Block<SettingsFile>
 {
 	Mandatory<std::string>	name;
@@ -961,6 +971,10 @@ bool LLAppViewer::init()
 
 	// *Note: this is where gViewerStats used to be created.
 
+	//
+	// Initialize the VFS, and gracefully handle initialization errors
+	//
+
 	if (!initCache())
 	{
 		LL_WARNS("InitInfo") << "Failed to init cache" << LL_ENDL;
@@ -1355,6 +1369,7 @@ static LLTrace::BlockTimerStatHandle FTM_TEXTURE_CACHE("Texture Cache");
 static LLTrace::BlockTimerStatHandle FTM_DECODE("Image Decode");
 static LLTrace::BlockTimerStatHandle FTM_FETCH("Image Fetch");
 
+static LLTrace::BlockTimerStatHandle FTM_VFS("VFS Thread");
 static LLTrace::BlockTimerStatHandle FTM_LFS("LFS Thread");
 static LLTrace::BlockTimerStatHandle FTM_PAUSE_THREADS("Pause Threads");
 static LLTrace::BlockTimerStatHandle FTM_IDLE("Idle");
@@ -1589,13 +1604,17 @@ bool LLAppViewer::doFrame()
 				work_pending += updateTextureThreads(max_time);
 
 				{
+					LL_RECORD_BLOCK_TIME(FTM_VFS);
+ 					io_pending += LLVFSThread::updateClass(1);
+				}
+				{
 					LL_RECORD_BLOCK_TIME(FTM_LFS);
  					io_pending += LLLFSThread::updateClass(1);
 				}
 
 				if (io_pending > 1000)
 				{
-					ms_sleep(llmin(io_pending/100,100)); // give the lfs some time to catch up
+					ms_sleep(llmin(io_pending/100,100)); // give the vfs some time to catch up
 				}
 
 				total_work_pending += work_pending ;
@@ -1612,6 +1631,7 @@ bool LLAppViewer::doFrame()
 			}
 			if(!total_io_pending) //pause file threads if nothing to process.
 			{
+				LLVFSThread::sLocal->pause();
 				LLLFSThread::sLocal->pause();
 			}
 
@@ -1673,11 +1693,12 @@ S32 LLAppViewer::updateTextureThreads(F32 max_time)
 	return work_pending;
 }
 
-void LLAppViewer::flushLFSIO()
+void LLAppViewer::flushVFSIO()
 {
 	while (1)
 	{
-		S32 pending = LLLFSThread::updateClass(0);
+		S32 pending = LLVFSThread::updateClass(0);
+		pending += LLLFSThread::updateClass(0);
 		if (!pending)
 		{
 			break;
@@ -1765,7 +1786,7 @@ bool LLAppViewer::cleanup()
 
 	LLKeyframeDataCache::clear();
 
- 	// End TransferManager before deleting systems it depends on (Audio, AssetStorage)
+ 	// End TransferManager before deleting systems it depends on (Audio, VFS, AssetStorage)
 #if 0 // this seems to get us stuck in an infinite loop...
 	gTransferManager.cleanup();
 #endif
@@ -1832,8 +1853,8 @@ bool LLAppViewer::cleanup()
 
 	LL_INFOS() << "Cache files removed" << LL_ENDL;
 
-	// Wait for any pending LFS IO
-	flushLFSIO();
+	// Wait for any pending VFS IO
+	flushVFSIO();
 	LL_INFOS() << "Shutting down Views" << LL_ENDL;
 
 	// Destroy the UI
@@ -1917,6 +1938,15 @@ bool LLAppViewer::cleanup()
 	SUBSYSTEM_CLEANUP(LLWorldMapView);
 	SUBSYSTEM_CLEANUP(LLFolderViewItem);
 
+	//
+	// Shut down the VFS's AFTER the decode manager cleans up (since it cleans up vfiles).
+	// Also after viewerwindow is deleted, since it may have image pointers (which have vfiles)
+	// Also after shutting down the messaging system since it has VFS dependencies
+
+	//
+	LL_INFOS() << "Cleaning up VFS" << LL_ENDL;
+	SUBSYSTEM_CLEANUP(LLVFile);
+
 	LL_INFOS() << "Saving Data" << LL_ENDL;
 
 	// Store the time of our current logoff
@@ -2003,6 +2033,7 @@ bool LLAppViewer::cleanup()
 		pending += LLAppViewer::getTextureCache()->update(1); // unpauses the worker thread
 		pending += LLAppViewer::getImageDecodeThread()->update(1); // unpauses the image thread
 		pending += LLAppViewer::getTextureFetch()->update(1); // unpauses the texture fetch thread
+		pending += LLVFSThread::updateClass(0);
 		pending += LLLFSThread::updateClass(0);
 		F64 idle_time = idleTimer.getElapsedTimeF64();
 		if(!pending)
@@ -2077,11 +2108,28 @@ bool LLAppViewer::cleanup()
 	gTextureList.shutdown(); // shutdown again in case a callback added something
 	LLUIImageList::getInstance()->cleanUp();
 
+	// This should eventually be done in LLAppViewer
 	SUBSYSTEM_CLEANUP(LLImage);
+	SUBSYSTEM_CLEANUP(LLVFSThread);
 	SUBSYSTEM_CLEANUP(LLLFSThread);
 
+#ifndef LL_RELEASE_FOR_DOWNLOAD
+	LL_INFOS() << "Auditing VFS" << LL_ENDL;
+	if(gVFS)
+	{
+		gVFS->audit();
+	}
+#endif
+
 	LL_INFOS() << "Misc Cleanup" << LL_ENDL;
 
+	// For safety, the LLVFS has to be deleted *after* LLVFSThread. This should be cleaned up.
+	// (LLVFS doesn't know about LLVFSThread so can't kill pending requests) -Steve
+	delete gStaticVFS;
+	gStaticVFS = NULL;
+	delete gVFS;
+	gVFS = NULL;
+
 	gSavedSettings.cleanup();
 	LLUIColorTable::instance().clear();
 
@@ -2163,6 +2211,7 @@ bool LLAppViewer::initThreads()
 
 	LLImage::initClass(gSavedSettings.getBOOL("TextureNewByteRange"),gSavedSettings.getS32("TextureReverseByteRange"));
 
+	LLVFSThread::initClass(enable_threads && false);
 	LLLFSThread::initClass(enable_threads && false);
 
 	// Image decoding
@@ -3173,6 +3222,10 @@ LLSD LLAppViewer::getViewerInfo() const
     info["GPU_SHADERS"] = gSavedSettings.getBOOL("RenderDeferred") ? "Enabled" : "Disabled";
     info["TEXTURE_MEMORY"] = gSavedSettings.getS32("TextureMemory");
 
+    LLSD substitution;
+    substitution["datetime"] = (S32)(gVFS ? gVFS->creationTime() : 0);
+    info["VFS_TIME"] = LLTrans::getString("AboutTime", substitution);
+
 #if LL_DARWIN
     info["HIDPI"] = gHiDPISupport;
 #endif
@@ -3264,9 +3317,6 @@ LLSD LLAppViewer::getViewerInfo() const
 		info["SERVER_RELEASE_NOTES_URL"] = mServerReleaseNotesURL;
 	}
 
-    // populate field for new local disk cache with some details
-    info["DISK_CACHE_INFO"] = LLDiskCache::getInstance()->getCacheInfo();
-
 	return info;
 }
 
@@ -3919,7 +3969,7 @@ void LLAppViewer::forceQuit()
 void LLAppViewer::fastQuit(S32 error_code)
 {
 	// finish pending transfers
-	flushLFSIO();
+	flushVFSIO();
 	// let sim know we're logging out
 	sendLogoutRequest();
 	// flush network buffers by shutting down messaging system
@@ -4108,6 +4158,39 @@ void LLAppViewer::migrateCacheDirectory()
 #endif // LL_WINDOWS || LL_DARWIN
 }
 
+void dumpVFSCaches()
+{
+	LL_INFOS() << "======= Static VFS ========" << LL_ENDL;
+	gStaticVFS->listFiles();
+#if LL_WINDOWS
+	LL_INFOS() << "======= Dumping static VFS to StaticVFSDump ========" << LL_ENDL;
+	WCHAR w_str[MAX_PATH];
+	GetCurrentDirectory(MAX_PATH, w_str);
+	S32 res = LLFile::mkdir("StaticVFSDump");
+	if (res == -1)
+	{
+		LL_WARNS() << "Couldn't create dir StaticVFSDump" << LL_ENDL;
+	}
+	SetCurrentDirectory(utf8str_to_utf16str("StaticVFSDump").c_str());
+	gStaticVFS->dumpFiles();
+	SetCurrentDirectory(w_str);
+#endif
+
+	LL_INFOS() << "========= Dynamic VFS ====" << LL_ENDL;
+	gVFS->listFiles();
+#if LL_WINDOWS
+	LL_INFOS() << "========= Dumping dynamic VFS to VFSDump ====" << LL_ENDL;
+	res = LLFile::mkdir("VFSDump");
+	if (res == -1)
+	{
+		LL_WARNS() << "Couldn't create dir VFSDump" << LL_ENDL;
+	}
+	SetCurrentDirectory(utf8str_to_utf16str("VFSDump").c_str());
+	gVFS->dumpFiles();
+	SetCurrentDirectory(w_str);
+#endif
+}
+
 //static
 U32 LLAppViewer::getTextureCacheVersion()
 {
@@ -4134,19 +4217,6 @@ bool LLAppViewer::initCache()
 	LLAppViewer::getTextureCache()->setReadOnly(read_only) ;
 	LLVOCache::initParamSingleton(read_only);
 
-	// initialize the new disk cache using saved settings
-	const std::string cache_dir_name = gSavedSettings.getString("DiskCacheDirName");
-
-    // note that the maximum size of this cache is defined as a percentage of the 
-    // total cache size - the 'CacheSize' pref - for all caches. 
-    const unsigned int cache_total_size_mb = gSavedSettings.getU32("CacheSize");
-    const double disk_cache_percent = gSavedSettings.getF32("DiskCachePercentOfTotal");
-    const unsigned int disk_cache_mb = cache_total_size_mb * disk_cache_percent / 100;
-    const unsigned int disk_cache_bytes = disk_cache_mb * 1024 * 1024;
-	const bool enable_cache_debug_info = gSavedSettings.getBOOL("EnableDiskCacheDebugInfo");
-	const std::string cache_dir = gDirUtilp->getExpandedFilename(LL_PATH_CACHE, cache_dir_name);
-	LLDiskCache::initParamSingleton(cache_dir, disk_cache_bytes, enable_cache_debug_info);
-
 	bool texture_cache_mismatch = false;
 	if (gSavedSettings.getS32("LocalCacheVersion") != LLAppViewer::getTextureCacheVersion())
 	{
@@ -4193,21 +4263,10 @@ bool LLAppViewer::initCache()
 		gSavedSettings.setString("CacheLocationTopFolder", "");
 	}
 
-	if (!read_only)
+	if (mPurgeCache && !read_only)
 	{
-		if (mPurgeCache)
-		{
-			LLSplashScreen::update(LLTrans::getString("StartupClearingCache"));
-			purgeCache();
-
-			// clear the new C++ file system based cache
-			LLDiskCache::getInstance()->clearCache();
-		}
-		else
-		{
-			// purge excessive files from the new file system based cache
-			LLDiskCache::getInstance()->purge();
-		}
+		LLSplashScreen::update(LLTrans::getString("StartupClearingCache"));
+		purgeCache();
 	}
 
 	LLSplashScreen::update(LLTrans::getString("StartupInitializingTextureCache"));
@@ -4217,18 +4276,172 @@ bool LLAppViewer::initCache()
 	const S32 MB = 1024 * 1024;
 	const S64 MIN_CACHE_SIZE = 256 * MB;
 	const S64 MAX_CACHE_SIZE = 9984ll * MB;
+	const S64 MAX_VFS_SIZE = 1024 * MB; // 1 GB
 
 	S64 cache_size = (S64)(gSavedSettings.getU32("CacheSize")) * MB;
 	cache_size = llclamp(cache_size, MIN_CACHE_SIZE, MAX_CACHE_SIZE);
 
-	S64 texture_cache_size = cache_size;
+	S64 vfs_size = llmin((S64)((cache_size * 2) / 10), MAX_VFS_SIZE);
+	S64 texture_cache_size = cache_size - vfs_size;
 
 	S64 extra = LLAppViewer::getTextureCache()->initCache(LL_PATH_CACHE, texture_cache_size, texture_cache_mismatch);
 	texture_cache_size -= extra;
 
+
 	LLVOCache::getInstance()->initCache(LL_PATH_CACHE, gSavedSettings.getU32("CacheNumberOfRegionsForObjects"), getObjectCacheVersion());
 
-    return true;
+	LLSplashScreen::update(LLTrans::getString("StartupInitializingVFS"));
+
+	// Init the VFS
+	vfs_size = llmin(vfs_size + extra, MAX_VFS_SIZE);
+	vfs_size = (vfs_size / MB) * MB; // make sure it is MB aligned
+	U32 vfs_size_u32 = (U32)vfs_size;
+	U32 old_vfs_size = gSavedSettings.getU32("VFSOldSize") * MB;
+	bool resize_vfs = (vfs_size_u32 != old_vfs_size);
+	if (resize_vfs)
+	{
+		gSavedSettings.setU32("VFSOldSize", vfs_size_u32 / MB);
+	}
+	LL_INFOS("AppCache") << "VFS CACHE SIZE: " << vfs_size / (1024*1024) << " MB" << LL_ENDL;
+
+	// This has to happen BEFORE starting the vfs
+	// time_t	ltime;
+	srand(time(NULL));		// Flawfinder: ignore
+	U32 old_salt = gSavedSettings.getU32("VFSSalt");
+	U32 new_salt;
+	std::string old_vfs_data_file;
+	std::string old_vfs_index_file;
+	std::string new_vfs_data_file;
+	std::string new_vfs_index_file;
+	std::string static_vfs_index_file;
+	std::string static_vfs_data_file;
+
+	if (gSavedSettings.getBOOL("AllowMultipleViewers"))
+	{
+		// don't mess with renaming the VFS in this case
+		new_salt = old_salt;
+	}
+	else
+	{
+		do
+		{
+			new_salt = rand();
+		} while(new_salt == old_salt);
+	}
+
+	old_vfs_data_file = gDirUtilp->getExpandedFilename(LL_PATH_CACHE, VFS_DATA_FILE_BASE) + llformat("%u", old_salt);
+
+	// make sure this file exists
+	llstat s;
+	S32 stat_result = LLFile::stat(old_vfs_data_file, &s);
+	if (stat_result)
+	{
+		// doesn't exist, look for a data file
+		std::string mask;
+		mask = VFS_DATA_FILE_BASE;
+		mask += "*";
+
+		std::string dir;
+		dir = gDirUtilp->getExpandedFilename(LL_PATH_CACHE, "");
+
+		std::string found_file;
+		LLDirIterator iter(dir, mask);
+		if (iter.next(found_file))
+		{
+			old_vfs_data_file = gDirUtilp->add(dir, found_file);
+
+			S32 start_pos = found_file.find_last_of('.');
+			if (start_pos > 0)
+			{
+				sscanf(found_file.substr(start_pos+1).c_str(), "%d", &old_salt);
+			}
+			LL_DEBUGS("AppCache") << "Default vfs data file not present, found: " << old_vfs_data_file << " Old salt: " << old_salt << LL_ENDL;
+		}
+	}
+
+	old_vfs_index_file = gDirUtilp->getExpandedFilename(LL_PATH_CACHE, VFS_INDEX_FILE_BASE) + llformat("%u", old_salt);
+
+	stat_result = LLFile::stat(old_vfs_index_file, &s);
+	if (stat_result)
+	{
+		// We've got a bad/missing index file, nukem!
+		LL_WARNS("AppCache") << "Bad or missing vfx index file " << old_vfs_index_file << LL_ENDL;
+		LL_WARNS("AppCache") << "Removing old vfs data file " << old_vfs_data_file << LL_ENDL;
+		LLFile::remove(old_vfs_data_file);
+		LLFile::remove(old_vfs_index_file);
+
+		// Just in case, nuke any other old cache files in the directory.
+		std::string dir;
+		dir = gDirUtilp->getExpandedFilename(LL_PATH_CACHE, "");
+
+		std::string mask;
+		mask = VFS_DATA_FILE_BASE;
+		mask += "*";
+
+		gDirUtilp->deleteFilesInDir(dir, mask);
+
+		mask = VFS_INDEX_FILE_BASE;
+		mask += "*";
+
+		gDirUtilp->deleteFilesInDir(dir, mask);
+	}
+
+	new_vfs_data_file = gDirUtilp->getExpandedFilename(LL_PATH_CACHE, VFS_DATA_FILE_BASE) + llformat("%u", new_salt);
+	new_vfs_index_file = gDirUtilp->getExpandedFilename(LL_PATH_CACHE, VFS_INDEX_FILE_BASE) + llformat("%u", new_salt);
+
+	static_vfs_data_file = gDirUtilp->getExpandedFilename(LL_PATH_APP_SETTINGS, "static_data.db2");
+	static_vfs_index_file = gDirUtilp->getExpandedFilename(LL_PATH_APP_SETTINGS, "static_index.db2");
+
+	if (resize_vfs)
+	{
+		LL_DEBUGS("AppCache") << "Removing old vfs and re-sizing" << LL_ENDL;
+
+		LLFile::remove(old_vfs_data_file);
+		LLFile::remove(old_vfs_index_file);
+	}
+	else if (old_salt != new_salt)
+	{
+		// move the vfs files to a new name before opening
+		LL_DEBUGS("AppCache") << "Renaming " << old_vfs_data_file << " to " << new_vfs_data_file << LL_ENDL;
+		LL_DEBUGS("AppCache") << "Renaming " << old_vfs_index_file << " to " << new_vfs_index_file << LL_ENDL;
+		LLFile::rename(old_vfs_data_file, new_vfs_data_file);
+		LLFile::rename(old_vfs_index_file, new_vfs_index_file);
+	}
+
+	// Startup the VFS...
+	gSavedSettings.setU32("VFSSalt", new_salt);
+
+	// Don't remove VFS after viewer crashes.  If user has corrupt data, they can reinstall. JC
+	gVFS = LLVFS::createLLVFS(new_vfs_index_file, new_vfs_data_file, false, vfs_size_u32, false);
+	if (!gVFS)
+	{
+		return false;
+	}
+
+	gStaticVFS = LLVFS::createLLVFS(static_vfs_index_file, static_vfs_data_file, true, 0, false);
+	if (!gStaticVFS)
+	{
+		return false;
+	}
+
+	BOOL success = gVFS->isValid() && gStaticVFS->isValid();
+	if (!success)
+	{
+		return false;
+	}
+	else
+	{
+		LLVFile::initClass();
+
+#ifndef LL_RELEASE_FOR_DOWNLOAD
+		if (gSavedSettings.getBOOL("DumpVFSCaches"))
+		{
+			dumpVFSCaches();
+		}
+#endif
+
+		return true;
+	}
 }
 
 void LLAppViewer::addOnIdleCallback(const boost::function<void()>& cb)