Merge branch 'DRTVWR-559' into DRTVWR-583

1 files changed, 215 insertions, 83 deletions

diff --git a/indra/llrender/llshadermgr.cpp b/indra/llrender/llshadermgr.cpp
index 614ddc0c62..01bad3a684 100644
--- a/indra/llrender/llshadermgr.cpp
+++ b/indra/llrender/llshadermgr.cpp
@@ -29,6 +29,9 @@
 #include "llrender.h"
 #include "llfile.h"
 #include "lldir.h"
+#include "llsdutil.h"
+#include "llsdserialize.h"
+#include "hbxxh.h"
 
 #if LL_DARWIN
 #include "OpenGL/OpenGL.h"
@@ -276,7 +279,7 @@ BOOL LLShaderMgr::attachShaderFeatures(LLGLSLShader * shader)
 		}
 	}
 
-	if (features->hasAtmospherics || features->isDeferred || features->hasTransport)
+	if (features->hasAtmospherics || features->isDeferred)
     {
         if (!shader->attachFragmentObject("windlight/atmosphericsFuncs.glsl")) {
             return FALSE;
@@ -288,14 +291,6 @@ BOOL LLShaderMgr::attachShaderFeatures(LLGLSLShader * shader)
 		}
 	}
 	
-	if (features->hasTransport)
-	{
-        if (!shader->attachFragmentObject("windlight/transportF.glsl"))
-		{
-			return FALSE;
-		}
-	}
-
 	// NOTE order of shader object attaching is VERY IMPORTANT!!!
 	if (features->hasWaterFog)
 	{
@@ -492,7 +487,7 @@ void LLShaderMgr::dumpObjectLog(GLuint ret, BOOL warns, const std::string& filen
 	}
  }
 
-GLuint LLShaderMgr::loadShaderFile(const std::string& filename, S32 & shader_level, GLenum type, std::unordered_map<std::string, std::string>* defines, S32 texture_index_channels)
+GLuint LLShaderMgr::loadShaderFile(const std::string& filename, S32 & shader_level, GLenum type, std::map<std::string, std::string>* defines, S32 texture_index_channels)
 {
 
 // endsure work-around for missing GLSL funcs gets propogated to feature shader files (e.g. srgbF.glsl)
@@ -596,30 +591,7 @@ GLuint LLShaderMgr::loadShaderFile(const std::string& filename, S32 & shader_lev
 	
 	if (major_version == 1 && minor_version < 30)
 	{
-		if (minor_version < 10)
-		{
-			//should NEVER get here -- if major version is 1 and minor version is less than 10, 
-			// viewer should never attempt to use shaders, continuing will result in undefined behavior
-			LL_ERRS() << "Unsupported GLSL Version." << LL_ENDL;
-		}
-
-		if (minor_version <= 19)
-		{
-			shader_code_text[shader_code_count++] = strdup("#version 110\n");
-			extra_code_text[extra_code_count++] = strdup("#define ATTRIBUTE attribute\n");
-			extra_code_text[extra_code_count++] = strdup("#define VARYING varying\n");
-			extra_code_text[extra_code_count++] = strdup("#define VARYING_FLAT varying\n");
-		}
-		else if (minor_version <= 29)
-		{
-			//set version to 1.20
-			shader_code_text[shader_code_count++] = strdup("#version 120\n");
-       		extra_code_text[extra_code_count++] = strdup("#define FXAA_GLSL_120 1\n");
-			extra_code_text[extra_code_count++] = strdup("#define FXAA_FAST_PIXEL_OFFSET 0\n");
-			extra_code_text[extra_code_count++] = strdup("#define ATTRIBUTE attribute\n");
-			extra_code_text[extra_code_count++] = strdup("#define VARYING varying\n");
-			extra_code_text[extra_code_count++] = strdup("#define VARYING_FLAT varying\n");
-		}
+        llassert(false); // GL 3.1 or later required
 	}
 	else
 	{  
@@ -656,40 +628,14 @@ GLuint LLShaderMgr::loadShaderFile(const std::string& filename, S32 & shader_lev
         }
 		else
 		{
-			//set version to 1.30
-			shader_code_text[shader_code_count++] = strdup("#version 130\n");
+			//set version to 1.40
+			shader_code_text[shader_code_count++] = strdup("#version 140\n");
 			//some implementations of GLSL 1.30 require integer precision be explicitly declared
 			extra_code_text[extra_code_count++] = strdup("precision mediump int;\n");
 			extra_code_text[extra_code_count++] = strdup("precision highp float;\n");
 		}
 
-		extra_code_text[extra_code_count++] = strdup("#define DEFINE_GL_FRAGCOLOR 1\n");
 		extra_code_text[extra_code_count++] = strdup("#define FXAA_GLSL_130 1\n");
-
-		extra_code_text[extra_code_count++] = strdup("#define ATTRIBUTE in\n");
-
-		if (type == GL_VERTEX_SHADER)
-		{ //"varying" state is "out" in a vertex program, "in" in a fragment program 
-			// ("varying" is deprecated after version 1.20)
-			extra_code_text[extra_code_count++] = strdup("#define VARYING out\n");
-			extra_code_text[extra_code_count++] = strdup("#define VARYING_FLAT flat out\n");
-		}
-		else
-		{
-			extra_code_text[extra_code_count++] = strdup("#define VARYING in\n");
-			extra_code_text[extra_code_count++] = strdup("#define VARYING_FLAT flat in\n");
-		}
-
-		//backwards compatibility with legacy texture lookup syntax
-		extra_code_text[extra_code_count++] = strdup("#define texture2D texture\n");
-		extra_code_text[extra_code_count++] = strdup("#define textureCube texture\n");
-		extra_code_text[extra_code_count++] = strdup("#define texture2DLod textureLod\n");
-		
-		if (major_version > 1 || minor_version >= 40)
-		{ //GLSL 1.40 replaces texture2DRect et al with texture
-			extra_code_text[extra_code_count++] = strdup("#define texture2DRect texture\n");
-			extra_code_text[extra_code_count++] = strdup("#define shadow2DRect(a,b) vec2(texture(a,b))\n");
-		}
 	}
 
     // Use alpha float to store bit flags
@@ -701,7 +647,7 @@ GLuint LLShaderMgr::loadShaderFile(const std::string& filename, S32 & shader_lev
 
 	if (defines)
 	{
-		for (std::unordered_map<std::string,std::string>::iterator iter = defines->begin(); iter != defines->end(); ++iter)
+		for (auto iter = defines->begin(); iter != defines->end(); ++iter)
 		{
 			std::string define = "#define " + iter->first + " " + iter->second + "\n";
 			extra_code_text[extra_code_count++] = (GLchar *) strdup(define.c_str());
@@ -727,21 +673,21 @@ GLuint LLShaderMgr::loadShaderFile(const std::string& filename, S32 & shader_lev
 		.
 		uniform sampler2D texN;
 		
-		VARYING_FLAT ivec4 vary_texture_index;
+		flat in int vary_texture_index;
 
 		vec4 ret = vec4(1,0,1,1);
 
 		vec4 diffuseLookup(vec2 texcoord)
 		{
-			switch (vary_texture_index.r))
+			switch (vary_texture_index)
 			{
-				case 0: ret = texture2D(tex0, texcoord); break;
-				case 1: ret = texture2D(tex1, texcoord); break;
-				case 2: ret = texture2D(tex2, texcoord); break;
+				case 0: ret = texture(tex0, texcoord); break;
+				case 1: ret = texture(tex1, texcoord); break;
+				case 2: ret = texture(tex2, texcoord); break;
 				.
 				.
 				.
-				case N: return texture2D(texN, texcoord); break;
+				case N: return texture(texN, texcoord); break;
 			}
 
 			return ret;
@@ -759,7 +705,7 @@ GLuint LLShaderMgr::loadShaderFile(const std::string& filename, S32 & shader_lev
 
 		if (texture_index_channels > 1)
 		{
-			extra_code_text[extra_code_count++] = strdup("VARYING_FLAT int vary_texture_index;\n");
+			extra_code_text[extra_code_count++] = strdup("flat in int vary_texture_index;\n");
 		}
 
 		extra_code_text[extra_code_count++] = strdup("vec4 diffuseLookup(vec2 texcoord)\n");
@@ -768,7 +714,7 @@ GLuint LLShaderMgr::loadShaderFile(const std::string& filename, S32 & shader_lev
 		
 		if (texture_index_channels == 1)
 		{ //don't use flow control, that's silly
-			extra_code_text[extra_code_count++] = strdup("return texture2D(tex0, texcoord);\n");
+			extra_code_text[extra_code_count++] = strdup("return texture(tex0, texcoord);\n");
 			extra_code_text[extra_code_count++] = strdup("}\n");
 		}
 		else if (major_version > 1 || minor_version >= 30)
@@ -777,7 +723,7 @@ GLuint LLShaderMgr::loadShaderFile(const std::string& filename, S32 & shader_lev
 			{ //switches are unreliable on some NVIDIA drivers
 				for (U32 i = 0; i < texture_index_channels; ++i)
 				{
-					std::string if_string = llformat("\t%sif (vary_texture_index == %d) { return texture2D(tex%d, texcoord); }\n", i > 0 ? "else " : "", i, i); 
+					std::string if_string = llformat("\t%sif (vary_texture_index == %d) { return texture(tex%d, texcoord); }\n", i > 0 ? "else " : "", i, i); 
 					extra_code_text[extra_code_count++] = strdup(if_string.c_str());
 				}
 				extra_code_text[extra_code_count++] = strdup("\treturn vec4(1,0,1,1);\n");
@@ -792,7 +738,7 @@ GLuint LLShaderMgr::loadShaderFile(const std::string& filename, S32 & shader_lev
 				//switch body
 				for (S32 i = 0; i < texture_index_channels; ++i)
 				{
-					std::string case_str = llformat("\t\tcase %d: return texture2D(tex%d, texcoord);\n", i, i);
+					std::string case_str = llformat("\t\tcase %d: return texture(tex%d, texcoord);\n", i, i);
 					extra_code_text[extra_code_count++] = strdup(case_str.c_str());
 				}
 
@@ -917,24 +863,39 @@ GLuint LLShaderMgr::loadShaderFile(const std::string& filename, S32 & shader_lev
 	if (error != GL_NO_ERROR)
 	{
 		LL_WARNS("ShaderLoading") << "GL ERROR in glCreateShader: " << error << " for file: " << open_file_name << LL_ENDL;
+		if (ret)
+		{
+			glDeleteShader(ret); //no longer need handle
+			ret = 0;
+		}
 	}
 
 	//load source
-	glShaderSource(ret, shader_code_count, (const GLchar**) shader_code_text, NULL);
-
-	error = glGetError();
-	if (error != GL_NO_ERROR)
+	if (ret)
 	{
-		LL_WARNS("ShaderLoading") << "GL ERROR in glShaderSource: " << error << " for file: " << open_file_name << LL_ENDL;
+		glShaderSource(ret, shader_code_count, (const GLchar**)shader_code_text, NULL);
+
+		error = glGetError();
+		if (error != GL_NO_ERROR)
+		{
+			LL_WARNS("ShaderLoading") << "GL ERROR in glShaderSource: " << error << " for file: " << open_file_name << LL_ENDL;
+			glDeleteShader(ret); //no longer need handle
+			ret = 0;
+		}
 	}
 
 	//compile source
-	glCompileShader(ret);
-
-	error = glGetError();
-	if (error != GL_NO_ERROR)
+	if (ret)
 	{
-		LL_WARNS("ShaderLoading") << "GL ERROR in glCompileShader: " << error << " for file: " << open_file_name << LL_ENDL;
+		glCompileShader(ret);
+
+		error = glGetError();
+		if (error != GL_NO_ERROR)
+		{
+			LL_WARNS("ShaderLoading") << "GL ERROR in glCompileShader: " << error << " for file: " << open_file_name << LL_ENDL;
+			glDeleteShader(ret); //no longer need handle
+			ret = 0;
+		}
 	}
 
 	if (error == GL_NO_ERROR)
@@ -950,6 +911,7 @@ GLuint LLShaderMgr::loadShaderFile(const std::string& filename, S32 & shader_lev
 			LL_WARNS("ShaderLoading") << "GLSL Compilation Error:" << LL_ENDL;
 			dumpObjectLog(ret, TRUE, open_file_name);
 			dumpShaderSource(shader_code_count, shader_code_text);
+			glDeleteShader(ret); //no longer need handle
 			ret = 0;
 		}
 	}
@@ -1041,6 +1003,176 @@ BOOL LLShaderMgr::validateProgramObject(GLuint obj)
 	return success;
 }
 
+void LLShaderMgr::initShaderCache(bool enabled, const LLUUID& old_cache_version, const LLUUID& current_cache_version)
+{
+	LL_INFOS() << "Initializing shader cache" << LL_ENDL;
+
+	mShaderCacheEnabled = gGLManager.mGLVersion >= 4.09 && enabled;
+
+	if(!mShaderCacheEnabled || mShaderCacheInitialized)
+		return;
+
+	mShaderCacheInitialized = true;
+
+	mShaderCacheDir = gDirUtilp->getExpandedFilename(LL_PATH_CACHE, "shader_cache");
+	LLFile::mkdir(mShaderCacheDir);
+
+	{
+		std::string meta_out_path = gDirUtilp->add(mShaderCacheDir, "shaderdata.llsd");
+		if (gDirUtilp->fileExists(meta_out_path))
+		{
+			LL_INFOS() << "Loading shader cache metadata" << LL_ENDL;
+
+			llifstream instream(meta_out_path);
+			LLSD in_data;
+			LLSDSerialize::fromNotation(in_data, instream, LLSDSerialize::SIZE_UNLIMITED);
+			instream.close();
+
+			if (old_cache_version == current_cache_version)
+			{
+				for (const auto& data_pair : llsd::inMap(in_data))
+				{
+					ProgramBinaryData binary_info = ProgramBinaryData();
+					binary_info.mBinaryFormat = data_pair.second["binary_format"].asInteger();
+					binary_info.mBinaryLength = data_pair.second["binary_size"].asInteger();
+					binary_info.mLastUsedTime = data_pair.second["last_used"].asReal();
+					mShaderBinaryCache.insert_or_assign(LLUUID(data_pair.first), binary_info);
+				}
+			}
+			else
+			{
+				LL_INFOS() << "Shader cache version mismatch detected. Purging." << LL_ENDL;
+				clearShaderCache();
+			}
+		}
+	}
+}
+
+void LLShaderMgr::clearShaderCache()
+{
+	std::string shader_cache = gDirUtilp->getExpandedFilename(LL_PATH_CACHE, "shader_cache");
+	LL_INFOS() << "Removing shader cache at " << shader_cache << LL_ENDL;
+	const std::string mask = "*";
+	gDirUtilp->deleteFilesInDir(shader_cache, mask);
+	mShaderBinaryCache.clear();
+}
+
+void LLShaderMgr::persistShaderCacheMetadata()
+{
+	if(!mShaderCacheEnabled) return;
+
+	LL_INFOS() << "Persisting shader cache metadata to disk" << LL_ENDL;
+
+	LLSD out = LLSD::emptyMap();
+
+	static const F32 LRU_TIME = (60.f * 60.f) * 24.f * 7.f; // 14 days
+	const F32 current_time = LLTimer::getTotalSeconds();
+	for (auto it = mShaderBinaryCache.begin(); it != mShaderBinaryCache.end();)
+	{
+		const ProgramBinaryData& shader_metadata = it->second;
+		if ((shader_metadata.mLastUsedTime + LRU_TIME) < current_time)
+		{
+			std::string shader_path = gDirUtilp->add(mShaderCacheDir, it->first.asString() + ".shaderbin");
+			LLFile::remove(shader_path);
+			it = mShaderBinaryCache.erase(it);
+		}
+		else
+		{
+			LLSD data = LLSD::emptyMap();
+			data["binary_format"] = LLSD::Integer(shader_metadata.mBinaryFormat);
+			data["binary_size"] = LLSD::Integer(shader_metadata.mBinaryLength);
+			data["last_used"] = LLSD::Real(shader_metadata.mLastUsedTime);
+			out[it->first.asString()] = data;
+			++it;
+		}
+	}
+
+	std::string meta_out_path = gDirUtilp->add(mShaderCacheDir, "shaderdata.llsd");
+	llofstream outstream(meta_out_path);
+	LLSDSerialize::toNotation(out, outstream);
+	outstream.close();
+}
+
+bool LLShaderMgr::loadCachedProgramBinary(LLGLSLShader* shader)
+{
+	if (!mShaderCacheEnabled) return false;
+
+	glProgramParameteri(shader->mProgramObject, GL_PROGRAM_BINARY_RETRIEVABLE_HINT, GL_TRUE);
+
+	auto binary_iter = mShaderBinaryCache.find(shader->mShaderHash);
+	if (binary_iter != mShaderBinaryCache.end())
+	{
+		std::string in_path = gDirUtilp->add(mShaderCacheDir, shader->mShaderHash.asString() + ".shaderbin");
+		auto& shader_info = binary_iter->second;
+		if (shader_info.mBinaryLength > 0)
+		{
+			std::vector<U8> in_data;
+			in_data.resize(shader_info.mBinaryLength);
+
+			LLUniqueFile filep = LLFile::fopen(in_path, "rb");
+			if (filep)
+			{
+				size_t result = fread(in_data.data(), sizeof(U8), in_data.size(), filep);
+				filep.close();
+
+				if (result == in_data.size())
+				{
+					GLenum error = glGetError(); // Clear current error
+					glProgramBinary(shader->mProgramObject, shader_info.mBinaryFormat, in_data.data(), shader_info.mBinaryLength);
+
+					error = glGetError();
+					GLint success = GL_TRUE;
+					glGetProgramiv(shader->mProgramObject, GL_LINK_STATUS, &success);
+					if (error == GL_NO_ERROR && success == GL_TRUE)
+					{
+						binary_iter->second.mLastUsedTime = LLTimer::getTotalSeconds();
+						LL_INFOS() << "Loaded cached binary for shader: " << shader->mName << LL_ENDL;
+						return true;
+					}
+				}
+			}
+		}
+		//an error occured, normally we would print log but in this case it means the shader needs recompiling.
+		LL_INFOS() << "Failed to load cached binary for shader: " << shader->mName << " falling back to compilation" << LL_ENDL;
+		LLFile::remove(in_path);
+		mShaderBinaryCache.erase(binary_iter);
+	}
+	return false;
+}
+
+bool LLShaderMgr::saveCachedProgramBinary(LLGLSLShader* shader)
+{
+	if (!mShaderCacheEnabled) return true;
+
+	ProgramBinaryData binary_info = ProgramBinaryData();
+	glGetProgramiv(shader->mProgramObject, GL_PROGRAM_BINARY_LENGTH, &binary_info.mBinaryLength);
+	if (binary_info.mBinaryLength > 0)
+	{
+		std::vector<U8> program_binary;
+		program_binary.resize(binary_info.mBinaryLength);
+
+		GLenum error = glGetError(); // Clear current error
+		glGetProgramBinary(shader->mProgramObject, program_binary.size() * sizeof(U8), nullptr, &binary_info.mBinaryFormat, program_binary.data());
+		error = glGetError();
+		if (error == GL_NO_ERROR)
+		{
+			std::string out_path = gDirUtilp->add(mShaderCacheDir, shader->mShaderHash.asString() + ".shaderbin");
+			LLUniqueFile outfile = LLFile::fopen(out_path, "wb");
+			if (outfile)
+			{
+				fwrite(program_binary.data(), sizeof(U8), program_binary.size(), outfile);
+				outfile.close();
+
+				binary_info.mLastUsedTime = LLTimer::getTotalSeconds();
+
+				mShaderBinaryCache.insert_or_assign(shader->mShaderHash, binary_info);
+				return true;
+			}
+		}
+	}
+	return false;
+}
+
 //virtual
 void LLShaderMgr::initAttribsAndUniforms()
 {