Add a binary cache for compiled shaders using glProgramBinary (#216)

* Add a binary cache for compiled shaders using glProgramBinary

* Add additional sanity checking to shader binary save and load, hook up cache clear and menu option

* Fix default init of shader cache data struct and clear gl errors before glGetError calls

---------

Co-authored-by: RunitaiLinden <davep@lindenlab.com>

10 files changed, 441 insertions, 184 deletions

diff --git a/indra/llrender/llglslshader.cpp b/indra/llrender/llglslshader.cpp
index 017634a2b9..ccfb8f69be 100644
--- a/indra/llrender/llglslshader.cpp
+++ b/indra/llrender/llglslshader.cpp
@@ -34,6 +34,9 @@
 #include "llvertexbuffer.h"
 #include "llrendertarget.h"
 
+#include "hbxxh.h"
+#include "llsdserialize.h"
+
 #if LL_DARWIN
 #include "OpenGL/OpenGL.h"
 #endif
@@ -53,6 +56,7 @@ LLGLSLShader* LLGLSLShader::sCurBoundShaderPtr = NULL;
 S32 LLGLSLShader::sIndexedTextureChannels = 0;
 bool LLGLSLShader::sProfileEnabled = false;
 std::set<LLGLSLShader*> LLGLSLShader::sInstances;
+LLGLSLShader::defines_map_t LLGLSLShader::sGlobalDefines;
 U64 LLGLSLShader::sTotalTimeElapsed = 0;
 U32 LLGLSLShader::sTotalTrianglesDrawn = 0;
 U64 LLGLSLShader::sTotalSamplesDrawn = 0;
@@ -82,30 +86,6 @@ BOOL shouldChange(const LLVector4& v1, const LLVector4& v2)
     return v1 != v2;
 }
 
-LLShaderFeatures::LLShaderFeatures()
-    : calculatesLighting(false)
-    , calculatesAtmospherics(false)
-    , hasLighting(false)
-    , isAlphaLighting(false)
-    , isSpecular(false)
-    , hasWaterFog(false)
-    , hasSkinning(false)
-    , hasObjectSkinning(false)
-    , hasAtmospherics(false)
-    , hasGamma(false)
-    , hasSrgb(false)
-    , encodesNormal(false)
-    , isDeferred(false)
-    , hasScreenSpaceReflections(false)
-    , hasShadows(false)
-    , hasAmbientOcclusion(false)
-    , mIndexedTextureChannels(0)
-    , disableTextureIndex(false)
-    , hasAlphaMask(false)
-    , attachNothing(false)
-{
-}
-
 //===============================
 // LLGLSL Shader implementation
 //===============================
@@ -328,6 +308,7 @@ LLGLSLShader::LLGLSLShader()
     mActiveTextureChannels(0),
     mShaderLevel(0),
     mShaderGroup(SG_DEFAULT),
+    mFeatures(),
     mUniformsDirty(FALSE),
     mTimerQuery(0),
     mSamplesQuery(0),
@@ -344,6 +325,7 @@ void LLGLSLShader::unload()
 {
     mShaderFiles.clear();
     mDefines.clear();
+    mFeatures = LLShaderFeatures();
 
     unloadInternal();
 }
@@ -419,6 +401,13 @@ BOOL LLGLSLShader::createShader(std::vector<LLStaticHashedString>* attributes,
 
     llassert_always(!mShaderFiles.empty());
 
+#if LL_DARWIN
+    // work-around missing mix(vec3,vec3,bvec3)
+    mDefines["OLD_SELECT"] = "1";
+#endif
+
+    mShaderHash = hash();
+
     // Create program
     mProgramObject = glCreateProgram();
     if (mProgramObject == 0)
@@ -431,50 +420,37 @@ BOOL LLGLSLShader::createShader(std::vector<LLStaticHashedString>* attributes,
 
     BOOL success = TRUE;
 
-#if LL_DARWIN
-    // work-around missing mix(vec3,vec3,bvec3)
-    mDefines["OLD_SELECT"] = "1";
-#endif
+    mUsingBinaryProgram =  LLShaderMgr::instance()->loadCachedProgramBinary(this);
 
+	if (!mUsingBinaryProgram)
+	{
 #if DEBUG_SHADER_INCLUDES
-    fprintf(stderr, "--- %s ---\n", mName.c_str());
+	    fprintf(stderr, "--- %s ---\n", mName.c_str());
 #endif // DEBUG_SHADER_INCLUDES
 
-    //compile new source
-    vector< pair<string, GLenum> >::iterator fileIter = mShaderFiles.begin();
-    for (; fileIter != mShaderFiles.end(); fileIter++)
-    {
-        GLuint shaderhandle = LLShaderMgr::instance()->loadShaderFile((*fileIter).first, mShaderLevel, (*fileIter).second, &mDefines, mFeatures.mIndexedTextureChannels);
-        LL_DEBUGS("ShaderLoading") << "SHADER FILE: " << (*fileIter).first << " mShaderLevel=" << mShaderLevel << LL_ENDL;
-        if (shaderhandle)
-        {
-            attachObject(shaderhandle);
-        }
-        else
-        {
-            success = FALSE;
-        }
+        //compile new source
+		vector< pair<string, GLenum> >::iterator fileIter = mShaderFiles.begin();
+		for (; fileIter != mShaderFiles.end(); fileIter++)
+		{
+			GLuint shaderhandle = LLShaderMgr::instance()->loadShaderFile((*fileIter).first, mShaderLevel, (*fileIter).second, &mDefines, mFeatures.mIndexedTextureChannels);
+			LL_DEBUGS("ShaderLoading") << "SHADER FILE: " << (*fileIter).first << " mShaderLevel=" << mShaderLevel << LL_ENDL;
+			if (shaderhandle)
+			{
+				attachObject(shaderhandle);
+			}
+			else
+			{
+				success = FALSE;
+			}
+		}
     }
 
     // Attach existing objects
     if (!LLShaderMgr::instance()->attachShaderFeatures(this))
     {
+        unloadInternal();
         return FALSE;
     }
-
-    if (gGLManager.mGLSLVersionMajor < 2 && gGLManager.mGLSLVersionMinor < 3)
-    { //indexed texture rendering requires GLSL 1.3 or later
-        //attachShaderFeatures may have set the number of indexed texture channels, so set to 1 again
-        mFeatures.mIndexedTextureChannels = llmin(mFeatures.mIndexedTextureChannels, 1);
-    }
-
-#ifdef GL_INTERLEAVED_ATTRIBS
-    if (varying_count > 0 && varyings)
-    {
-        glTransformFeedbackVaryings((GLuint64)mProgramObject, varying_count, varyings, GL_INTERLEAVED_ATTRIBS);
-    }
-#endif
-
     // Map attributes and uniforms
     if (success)
     {
@@ -495,6 +471,11 @@ BOOL LLGLSLShader::createShader(std::vector<LLStaticHashedString>* attributes,
             mShaderLevel--;
             return createShader(attributes, uniforms);
         }
+        else
+        {
+            // Give up and unload shader.
+            unloadInternal();
+        }
     }
     else if (mFeatures.mIndexedTextureChannels > 0)
     { //override texture channels for indexed texture rendering
@@ -569,6 +550,9 @@ BOOL LLGLSLShader::attachVertexObject(std::string object_path)
 
 BOOL LLGLSLShader::attachFragmentObject(std::string object_path)
 {
+    if(mUsingBinaryProgram)
+        return TRUE;
+
     if (LLShaderMgr::instance()->mFragmentShaderObjects.count(object_path) > 0)
     {
         stop_glerror();
@@ -588,6 +572,9 @@ BOOL LLGLSLShader::attachFragmentObject(std::string object_path)
 
 void LLGLSLShader::attachObject(GLuint object)
 {
+    if(mUsingBinaryProgram)
+        return;
+
     if (object != 0)
     {
         stop_glerror();
@@ -606,6 +593,9 @@ void LLGLSLShader::attachObject(GLuint object)
 
 void LLGLSLShader::attachObjects(GLuint* objects, S32 count)
 {
+    if(mUsingBinaryProgram)
+        return;
+
     for (S32 i = 0; i < count; i++)
     {
         attachObject(objects[i]);
@@ -616,15 +606,19 @@ BOOL LLGLSLShader::mapAttributes(const std::vector<LLStaticHashedString>* attrib
 {
     LL_PROFILE_ZONE_SCOPED_CATEGORY_SHADER;
 
-    //before linking, make sure reserved attributes always have consistent locations
-    for (U32 i = 0; i < LLShaderMgr::instance()->mReservedAttribs.size(); i++)
-    {
-        const char* name = LLShaderMgr::instance()->mReservedAttribs[i].c_str();
-        glBindAttribLocation(mProgramObject, i, (const GLchar*)name);
-    }
+	BOOL res = TRUE;
+	if (!mUsingBinaryProgram)
+	{
+		//before linking, make sure reserved attributes always have consistent locations
+		for (U32 i = 0; i < LLShaderMgr::instance()->mReservedAttribs.size(); i++)
+		{
+			const char* name = LLShaderMgr::instance()->mReservedAttribs[i].c_str();
+			glBindAttribLocation(mProgramObject, i, (const GLchar*)name);
+		}
 
-    //link the program
-    BOOL res = link();
+		//link the program
+		res = link();
+	}
 
     mAttribute.clear();
     U32 numAttributes = (attributes == NULL) ? 0 : attributes->size();
@@ -746,7 +740,6 @@ void LLGLSLShader::mapUniform(GLint index, const vector<LLStaticHashedString>* u
         }
 
         LLStaticHashedString hashedName(name);
-        mUniformNameMap[location] = name;
         mUniformMap[hashedName] = location;
 
         LL_DEBUGS("ShaderUniform") << "Uniform " << name << " is at location " << location << LL_ENDL;
@@ -798,7 +791,7 @@ void LLGLSLShader::addConstant(const LLGLSLShader::eShaderConsts shader_const)
 
 void LLGLSLShader::removePermutation(std::string name)
 {
-    mDefines[name].erase();
+    mDefines.erase(name);
 }
 
 GLint LLGLSLShader::mapUniformTextureChannel(GLint location, GLenum type, GLint size)
@@ -847,7 +840,6 @@ BOOL LLGLSLShader::mapUniforms(const vector<LLStaticHashedString>* uniforms)
     mActiveTextureChannels = 0;
     mUniform.clear();
     mUniformMap.clear();
-    mUniformNameMap.clear();
     mTexture.clear();
     mValue.clear();
     //initialize arrays
@@ -1020,6 +1012,11 @@ BOOL LLGLSLShader::link(BOOL suppress_errors)
         LLShaderMgr::instance()->dumpObjectLog(mProgramObject, !success, mName);
     }
 
+    if (success)
+    {
+        LLShaderMgr::instance()->saveCachedProgramBinary(this);
+    }
+
     return success;
 }
 
@@ -1918,6 +1915,36 @@ void LLShaderUniforms::apply(LLGLSLShader* shader)
     }
 }
 
+LLUUID LLGLSLShader::hash()
+{
+    HBXXH128 hash_obj;
+    hash_obj.update(mName);
+    hash_obj.update(&mShaderGroup, sizeof(mShaderGroup));
+    hash_obj.update(&mShaderLevel, sizeof(mShaderLevel));
+    for (const auto& shdr_pair : mShaderFiles)
+    {
+        hash_obj.update(shdr_pair.first);
+        hash_obj.update(&shdr_pair.second, sizeof(GLenum));
+    }
+    for (const auto& define_pair : mDefines)
+    {
+        hash_obj.update(define_pair.first);
+        hash_obj.update(define_pair.second);
+
+    }
+    for (const auto& define_pair : LLGLSLShader::sGlobalDefines)
+    {
+        hash_obj.update(define_pair.first);
+        hash_obj.update(define_pair.second);
+
+    }
+    hash_obj.update(&mFeatures, sizeof(LLShaderFeatures));
+    hash_obj.update(gGLManager.mGLVendor);
+    hash_obj.update(gGLManager.mGLRenderer);
+    hash_obj.update(gGLManager.mGLVersionString);
+    return hash_obj.digest();
+}
+
 #ifdef LL_PROFILER_ENABLE_RENDER_DOC
 void LLGLSLShader::setLabel(const char* label) {
     LL_LABEL_OBJECT_GL(GL_PROGRAM, mProgramObject, strlen(label), label);
diff --git a/indra/llrender/llglslshader.h b/indra/llrender/llglslshader.h
index 3e7dae6669..b8071248e2 100644
--- a/indra/llrender/llglslshader.h
+++ b/indra/llrender/llglslshader.h
@@ -35,32 +35,28 @@
 class LLShaderFeatures
 {
 public:
-    bool calculatesLighting;
-    bool calculatesAtmospherics;
-    bool hasLighting; // implies no transport (it's possible to have neither though)
-    bool isAlphaLighting; // indicates lighting shaders need not be linked in (lighting performed directly in alpha shader to match deferred lighting functions)
-    bool isSpecular;
-    bool hasWaterFog; // implies no gamma
-    bool hasTransport; // implies no lighting (it's possible to have neither though)
-    bool hasSkinning;
-    bool hasObjectSkinning;
-    bool hasAtmospherics;
-    bool hasGamma;
-    bool hasShadows;
-    bool hasAmbientOcclusion;
-    bool hasSrgb;
-    bool encodesNormal; // include: shaders\class1\environment\encodeNormF.glsl
-    bool isDeferred;
-    bool hasScreenSpaceReflections;
-    S32 mIndexedTextureChannels;
-    bool disableTextureIndex;
-    bool hasAlphaMask;
+    S32 mIndexedTextureChannels = 0;
+    bool calculatesLighting = false;
+    bool calculatesAtmospherics = false;
+    bool hasLighting = false; // implies no transport (it's possible to have neither though)
+    bool isAlphaLighting = false; // indicates lighting shaders need not be linked in (lighting performed directly in alpha shader to match deferred lighting functions)
+    bool isSpecular = false;
+    bool hasWaterFog = false; // implies no gamma
+    bool hasTransport = false; // implies no lighting (it's possible to have neither though)
+    bool hasSkinning = false;
+    bool hasObjectSkinning = false;
+    bool hasAtmospherics = false;
+    bool hasGamma = false;
+    bool hasShadows = false;
+    bool hasAmbientOcclusion = false;
+    bool hasSrgb = false;
+    bool encodesNormal = false; // include: shaders\class1\environment\encodeNormF.glsl
+    bool isDeferred = false;
+    bool hasScreenSpaceReflections = false;
+    bool disableTextureIndex = false;
+    bool hasAlphaMask = false;
     bool hasReflectionProbes = false;
-    bool attachNothing;
-
-    // char numLights;
-
-    LLShaderFeatures();
+    bool attachNothing = false;
 };
 
 // ============= Structure for caching shader uniforms ===============
@@ -261,6 +257,10 @@ public:
     //helper to conditionally bind mRiggedVariant instead of this
     void bind(bool rigged);
     
+    bool isComplete() const { return mProgramObject != 0; }
+
+    LLUUID hash();
+
     // Unbinds any previously bound shader by explicitly binding no shader.
     static void unbind();
 
@@ -283,9 +283,7 @@ public:
     U32 mAttributeMask;  //mask of which reserved attributes are set (lines up with LLVertexBuffer::getTypeMask())
     std::vector<GLint> mUniform;   //lookup table of uniform enum to uniform location
     LLStaticStringTable<GLint> mUniformMap; //lookup map of uniform name to uniform location
-    typedef std::unordered_map<GLint, std::string> uniform_name_map_t;
     typedef std::unordered_map<GLint, LLVector4> uniform_value_map_t;
-    uniform_name_map_t mUniformNameMap; //lookup map of uniform location to uniform name
     uniform_value_map_t mValue; //lookup map of uniform location to last known value
     std::vector<GLint> mTexture;
     S32 mTotalUniformSize;
@@ -296,8 +294,11 @@ public:
     LLShaderFeatures mFeatures;
     std::vector< std::pair< std::string, GLenum > > mShaderFiles;
     std::string mName;
-    typedef std::unordered_map<std::string, std::string> defines_map_t;
+    typedef std::map<std::string, std::string> defines_map_t; //NOTE: this must be an ordered map to maintain hash consistency
     defines_map_t mDefines;
+    static defines_map_t sGlobalDefines;
+    LLUUID mShaderHash;
+    bool mUsingBinaryProgram = false;
 
     //statistics for profiling shader performance
     bool mProfilePending = false;
diff --git a/indra/llrender/llshadermgr.cpp b/indra/llrender/llshadermgr.cpp
index d382fa7f71..7d0c99c9e3 100644
--- a/indra/llrender/llshadermgr.cpp
+++ b/indra/llrender/llshadermgr.cpp
@@ -29,6 +29,9 @@
 #include "llrender.h"
 #include "llfile.h"
 #include "lldir.h"
+#include "llsdutil.h"
+#include "llsdserialize.h"
+#include "hbxxh.h"
 
 #if LL_DARWIN
 #include "OpenGL/OpenGL.h"
@@ -484,7 +487,7 @@ void LLShaderMgr::dumpObjectLog(GLuint ret, BOOL warns, const std::string& filen
 	}
  }
 
-GLuint LLShaderMgr::loadShaderFile(const std::string& filename, S32 & shader_level, GLenum type, std::unordered_map<std::string, std::string>* defines, S32 texture_index_channels)
+GLuint LLShaderMgr::loadShaderFile(const std::string& filename, S32 & shader_level, GLenum type, std::map<std::string, std::string>* defines, S32 texture_index_channels)
 {
 
 // endsure work-around for missing GLSL funcs gets propogated to feature shader files (e.g. srgbF.glsl)
@@ -644,7 +647,7 @@ GLuint LLShaderMgr::loadShaderFile(const std::string& filename, S32 & shader_lev
 
 	if (defines)
 	{
-		for (std::unordered_map<std::string,std::string>::iterator iter = defines->begin(); iter != defines->end(); ++iter)
+		for (auto iter = defines->begin(); iter != defines->end(); ++iter)
 		{
 			std::string define = "#define " + iter->first + " " + iter->second + "\n";
 			extra_code_text[extra_code_count++] = (GLchar *) strdup(define.c_str());
@@ -860,24 +863,39 @@ GLuint LLShaderMgr::loadShaderFile(const std::string& filename, S32 & shader_lev
 	if (error != GL_NO_ERROR)
 	{
 		LL_WARNS("ShaderLoading") << "GL ERROR in glCreateShader: " << error << " for file: " << open_file_name << LL_ENDL;
+		if (ret)
+		{
+			glDeleteShader(ret); //no longer need handle
+			ret = 0;
+		}
 	}
 
 	//load source
-	glShaderSource(ret, shader_code_count, (const GLchar**) shader_code_text, NULL);
-
-	error = glGetError();
-	if (error != GL_NO_ERROR)
+	if (ret)
 	{
-		LL_WARNS("ShaderLoading") << "GL ERROR in glShaderSource: " << error << " for file: " << open_file_name << LL_ENDL;
+		glShaderSource(ret, shader_code_count, (const GLchar**)shader_code_text, NULL);
+
+		error = glGetError();
+		if (error != GL_NO_ERROR)
+		{
+			LL_WARNS("ShaderLoading") << "GL ERROR in glShaderSource: " << error << " for file: " << open_file_name << LL_ENDL;
+			glDeleteShader(ret); //no longer need handle
+			ret = 0;
+		}
 	}
 
 	//compile source
-	glCompileShader(ret);
-
-	error = glGetError();
-	if (error != GL_NO_ERROR)
+	if (ret)
 	{
-		LL_WARNS("ShaderLoading") << "GL ERROR in glCompileShader: " << error << " for file: " << open_file_name << LL_ENDL;
+		glCompileShader(ret);
+
+		error = glGetError();
+		if (error != GL_NO_ERROR)
+		{
+			LL_WARNS("ShaderLoading") << "GL ERROR in glCompileShader: " << error << " for file: " << open_file_name << LL_ENDL;
+			glDeleteShader(ret); //no longer need handle
+			ret = 0;
+		}
 	}
 
 	if (error == GL_NO_ERROR)
@@ -893,6 +911,7 @@ GLuint LLShaderMgr::loadShaderFile(const std::string& filename, S32 & shader_lev
 			LL_WARNS("ShaderLoading") << "GLSL Compilation Error:" << LL_ENDL;
 			dumpObjectLog(ret, TRUE, open_file_name);
 			dumpShaderSource(shader_code_count, shader_code_text);
+			glDeleteShader(ret); //no longer need handle
 			ret = 0;
 		}
 	}
@@ -984,6 +1003,176 @@ BOOL LLShaderMgr::validateProgramObject(GLuint obj)
 	return success;
 }
 
+void LLShaderMgr::initShaderCache(bool enabled, const LLUUID& old_cache_version, const LLUUID& current_cache_version)
+{
+	LL_INFOS() << "Initializing shader cache" << LL_ENDL;
+
+	mShaderCacheEnabled = gGLManager.mGLVersion >= 4.09 && enabled;
+
+	if(!mShaderCacheEnabled || mShaderCacheInitialized)
+		return;
+
+	mShaderCacheInitialized = true;
+
+	mShaderCacheDir = gDirUtilp->getExpandedFilename(LL_PATH_CACHE, "shader_cache");
+	LLFile::mkdir(mShaderCacheDir);
+
+	{
+		std::string meta_out_path = gDirUtilp->add(mShaderCacheDir, "shaderdata.llsd");
+		if (gDirUtilp->fileExists(meta_out_path))
+		{
+			LL_INFOS() << "Loading shader cache metadata" << LL_ENDL;
+
+			llifstream instream(meta_out_path);
+			LLSD in_data;
+			LLSDSerialize::fromNotation(in_data, instream, LLSDSerialize::SIZE_UNLIMITED);
+			instream.close();
+
+			if (old_cache_version == current_cache_version)
+			{
+				for (const auto& data_pair : llsd::inMap(in_data))
+				{
+					ProgramBinaryData binary_info = ProgramBinaryData();
+					binary_info.mBinaryFormat = data_pair.second["binary_format"].asInteger();
+					binary_info.mBinaryLength = data_pair.second["binary_size"].asInteger();
+					binary_info.mLastUsedTime = data_pair.second["last_used"].asReal();
+					mShaderBinaryCache.insert_or_assign(LLUUID(data_pair.first), binary_info);
+				}
+			}
+			else
+			{
+				LL_INFOS() << "Shader cache version mismatch detected. Purging." << LL_ENDL;
+				clearShaderCache();
+			}
+		}
+	}
+}
+
+void LLShaderMgr::clearShaderCache()
+{
+	std::string shader_cache = gDirUtilp->getExpandedFilename(LL_PATH_CACHE, "shader_cache");
+	LL_INFOS() << "Removing shader cache at " << shader_cache << LL_ENDL;
+	const std::string mask = "*";
+	gDirUtilp->deleteFilesInDir(shader_cache, mask);
+	mShaderBinaryCache.clear();
+}
+
+void LLShaderMgr::persistShaderCacheMetadata()
+{
+	if(!mShaderCacheEnabled) return;
+
+	LL_INFOS() << "Persisting shader cache metadata to disk" << LL_ENDL;
+
+	LLSD out = LLSD::emptyMap();
+
+	static const F32 LRU_TIME = (60.f * 60.f) * 24.f * 7.f; // 14 days
+	const F32 current_time = LLTimer::getTotalSeconds();
+	for (auto it = mShaderBinaryCache.begin(); it != mShaderBinaryCache.end();)
+	{
+		const ProgramBinaryData& shader_metadata = it->second;
+		if ((shader_metadata.mLastUsedTime + LRU_TIME) < current_time)
+		{
+			std::string shader_path = gDirUtilp->add(mShaderCacheDir, it->first.asString() + ".shaderbin");
+			LLFile::remove(shader_path);
+			it = mShaderBinaryCache.erase(it);
+		}
+		else
+		{
+			LLSD data = LLSD::emptyMap();
+			data["binary_format"] = LLSD::Integer(shader_metadata.mBinaryFormat);
+			data["binary_size"] = LLSD::Integer(shader_metadata.mBinaryLength);
+			data["last_used"] = LLSD::Real(shader_metadata.mLastUsedTime);
+			out[it->first.asString()] = data;
+			++it;
+		}
+	}
+
+	std::string meta_out_path = gDirUtilp->add(mShaderCacheDir, "shaderdata.llsd");
+	llofstream outstream(meta_out_path);
+	LLSDSerialize::toNotation(out, outstream);
+	outstream.close();
+}
+
+bool LLShaderMgr::loadCachedProgramBinary(LLGLSLShader* shader)
+{
+	if (!mShaderCacheEnabled) return false;
+
+	glProgramParameteri(shader->mProgramObject, GL_PROGRAM_BINARY_RETRIEVABLE_HINT, GL_TRUE);
+
+	auto binary_iter = mShaderBinaryCache.find(shader->mShaderHash);
+	if (binary_iter != mShaderBinaryCache.end())
+	{
+		std::string in_path = gDirUtilp->add(mShaderCacheDir, shader->mShaderHash.asString() + ".shaderbin");
+		auto& shader_info = binary_iter->second;
+		if (shader_info.mBinaryLength > 0)
+		{
+			std::vector<U8> in_data;
+			in_data.resize(shader_info.mBinaryLength);
+
+			LLUniqueFile filep = LLFile::fopen(in_path, "rb");
+			if (filep)
+			{
+				size_t result = fread(in_data.data(), sizeof(U8), in_data.size(), filep);
+				filep.close();
+
+				if (result == in_data.size())
+				{
+					GLenum error = glGetError(); // Clear current error
+					glProgramBinary(shader->mProgramObject, shader_info.mBinaryFormat, in_data.data(), shader_info.mBinaryLength);
+
+					error = glGetError();
+					GLint success = GL_TRUE;
+					glGetProgramiv(shader->mProgramObject, GL_LINK_STATUS, &success);
+					if (error == GL_NO_ERROR && success == GL_TRUE)
+					{
+						binary_iter->second.mLastUsedTime = LLTimer::getTotalSeconds();
+						LL_INFOS() << "Loaded cached binary for shader: " << shader->mName << LL_ENDL;
+						return true;
+					}
+				}
+			}
+		}
+		//an error occured, normally we would print log but in this case it means the shader needs recompiling.
+		LL_INFOS() << "Failed to load cached binary for shader: " << shader->mName << " falling back to compilation" << LL_ENDL;
+		LLFile::remove(in_path);
+		mShaderBinaryCache.erase(binary_iter);
+	}
+	return false;
+}
+
+bool LLShaderMgr::saveCachedProgramBinary(LLGLSLShader* shader)
+{
+	if (!mShaderCacheEnabled) return true;
+
+	ProgramBinaryData binary_info = ProgramBinaryData();
+	glGetProgramiv(shader->mProgramObject, GL_PROGRAM_BINARY_LENGTH, &binary_info.mBinaryLength);
+	if (binary_info.mBinaryLength > 0)
+	{
+		std::vector<U8> program_binary;
+		program_binary.resize(binary_info.mBinaryLength);
+
+		GLenum error = glGetError(); // Clear current error
+		glGetProgramBinary(shader->mProgramObject, program_binary.size() * sizeof(U8), nullptr, &binary_info.mBinaryFormat, program_binary.data());
+		error = glGetError();
+		if (error == GL_NO_ERROR)
+		{
+			std::string out_path = gDirUtilp->add(mShaderCacheDir, shader->mShaderHash.asString() + ".shaderbin");
+			LLUniqueFile outfile = LLFile::fopen(out_path, "wb");
+			if (outfile)
+			{
+				fwrite(program_binary.data(), sizeof(U8), program_binary.size(), outfile);
+				outfile.close();
+
+				binary_info.mLastUsedTime = LLTimer::getTotalSeconds();
+
+				mShaderBinaryCache.insert_or_assign(shader->mShaderHash, binary_info);
+				return true;
+			}
+		}
+	}
+	return false;
+}
+
 //virtual
 void LLShaderMgr::initAttribsAndUniforms()
 {
diff --git a/indra/llrender/llshadermgr.h b/indra/llrender/llshadermgr.h
index ec3455952b..4a2241a9d6 100644
--- a/indra/llrender/llshadermgr.h
+++ b/indra/llrender/llshadermgr.h
@@ -295,7 +295,7 @@ public:
     void dumpShaderSource(U32 shader_code_count, GLchar** shader_code_text);
 	BOOL	linkProgramObject(GLuint obj, BOOL suppress_errors = FALSE);
 	BOOL	validateProgramObject(GLuint obj);
-	GLuint loadShaderFile(const std::string& filename, S32 & shader_level, GLenum type, std::unordered_map<std::string, std::string>* defines = NULL, S32 texture_index_channels = -1);
+	GLuint loadShaderFile(const std::string& filename, S32 & shader_level, GLenum type, std::map<std::string, std::string>* defines = NULL, S32 texture_index_channels = -1);
 
 	// Implemented in the application to actually point to the shader directory.
 	virtual std::string getShaderDirPrefix(void) = 0; // Pure Virtual
@@ -303,6 +303,13 @@ public:
 	// Implemented in the application to actually update out of date uniforms for a particular shader
 	virtual void updateShaderUniforms(LLGLSLShader * shader) = 0; // Pure Virtual
 
+    void initShaderCache(bool enabled, const LLUUID& old_cache_version, const LLUUID& current_cache_version);
+    void clearShaderCache();
+    void persistShaderCacheMetadata();
+
+    bool loadCachedProgramBinary(LLGLSLShader* shader);
+    bool saveCachedProgramBinary(LLGLSLShader* shader);
+
 public:
 	// Map of shader names to compiled
     std::map<std::string, GLuint> mVertexShaderObjects;
@@ -313,8 +320,16 @@ public:
 
 	std::vector<std::string> mReservedUniforms;
 
-	//preprocessor definitions (name/value)
-	std::map<std::string, std::string> mDefinitions;
+    struct ProgramBinaryData
+    {
+        GLsizei mBinaryLength = 0;
+        GLenum mBinaryFormat = 0;
+        F32 mLastUsedTime = 0.0;
+    };
+    std::map<LLUUID, ProgramBinaryData> mShaderBinaryCache;
+    bool mShaderCacheInitialized = false;
+    bool mShaderCacheEnabled = false;
+    std::string mShaderCacheDir;
 
 protected:
 
diff --git a/indra/newview/app_settings/settings.xml b/indra/newview/app_settings/settings.xml
index 162a474f69..79639c9b32 100644
--- a/indra/newview/app_settings/settings.xml
+++ b/indra/newview/app_settings/settings.xml
@@ -11014,6 +11014,28 @@
       <key>Value</key>
       <integer>1</integer>
     </map>
+    <key>RenderShaderCacheEnabled</key>
+    <map>
+      <key>Comment</key>
+      <string>Enable binary shader cache</string>
+      <key>Persist</key>
+      <integer>1</integer>
+      <key>Type</key>
+      <string>Boolean</string>
+      <key>Value</key>
+      <integer>1</integer>
+    </map>
+    <key>RenderShaderCacheVersion</key>
+    <map>
+      <key>Comment</key>
+      <string>Current version hash for shader cache</string>
+      <key>Persist</key>
+      <integer>1</integer>
+      <key>Type</key>
+      <string>String</string>
+      <key>Value</key>
+      <string>00000000-0000-0000-0000-000000000000</string>
+    </map>
     <key>ReplaySession</key>
     <map>
       <key>Comment</key>
diff --git a/indra/newview/llappviewer.cpp b/indra/newview/llappviewer.cpp
index dfdf7a1b61..abef25e34f 100644
--- a/indra/newview/llappviewer.cpp
+++ b/indra/newview/llappviewer.cpp
@@ -4446,6 +4446,7 @@ void LLAppViewer::purgeCache()
 	LL_INFOS("AppCache") << "Purging Cache and Texture Cache..." << LL_ENDL;
 	LLAppViewer::getTextureCache()->purgeCache(LL_PATH_CACHE);
 	LLVOCache::getInstance()->removeCache(LL_PATH_CACHE);
+	LLViewerShaderMgr::instance()->clearShaderCache();
 	std::string browser_cache = gDirUtilp->getExpandedFilename(LL_PATH_CACHE, "cef_cache");
 	if (LLFile::isdir(browser_cache))
 	{
diff --git a/indra/newview/lldrawpoolalpha.cpp b/indra/newview/lldrawpoolalpha.cpp
index 7893caf3c0..a96d480c78 100644
--- a/indra/newview/lldrawpoolalpha.cpp
+++ b/indra/newview/lldrawpoolalpha.cpp
@@ -732,12 +732,6 @@ void LLDrawPoolAlpha::renderAlpha(U32 mask, bool depth_only, bool rigged)
                         {
                             target_shader = &(gDeferredMaterialWaterProgram[mask]);
                         }
-
-                        if (params.mAvatar != nullptr)
-                        {
-                            llassert(target_shader->mRiggedVariant != nullptr);
-                            target_shader = target_shader->mRiggedVariant;
-                        }
                     }
                     else if (!params.mFullbright)
                     {
@@ -750,6 +744,7 @@ void LLDrawPoolAlpha::renderAlpha(U32 mask, bool depth_only, bool rigged)
 
                     if (params.mAvatar != nullptr)
                     {
+                        llassert(target_shader->mRiggedVariant != nullptr);
                         target_shader = target_shader->mRiggedVariant;
                     }
 
diff --git a/indra/newview/llviewermenu.cpp b/indra/newview/llviewermenu.cpp
index d0acac858c..921620190e 100644
--- a/indra/newview/llviewermenu.cpp
+++ b/indra/newview/llviewermenu.cpp
@@ -2079,6 +2079,21 @@ class LLAdvancedPurgeDiskCache : public view_listener_t
 };
 
 
+////////////////////////
+// PURGE SHADER CACHE //
+////////////////////////
+
+
+class LLAdvancedPurgeShaderCache : public view_listener_t
+{
+	bool handleEvent(const LLSD& userdata)
+	{
+		LLViewerShaderMgr::instance()->clearShaderCache();
+		LLViewerShaderMgr::instance()->setShaders();
+		return true;
+	}
+};
+
 ////////////////////
 // EVENT Recorder //
 ///////////////////
@@ -9435,6 +9450,7 @@ void initialize_menus()
 	view_listener_t::addMenu(new LLAdvancedClickRenderShadowOption(), "Advanced.ClickRenderShadowOption");
 	view_listener_t::addMenu(new LLAdvancedClickRenderProfile(), "Advanced.ClickRenderProfile");
 	view_listener_t::addMenu(new LLAdvancedClickRenderBenchmark(), "Advanced.ClickRenderBenchmark");
+	view_listener_t::addMenu(new LLAdvancedPurgeShaderCache(), "Advanced.ClearShaderCache");
 
 	#ifdef TOGGLE_HACKED_GODLIKE_VIEWER
 	view_listener_t::addMenu(new LLAdvancedHandleToggleHackedGodmode(), "Advanced.HandleToggleHackedGodmode");
diff --git a/indra/newview/llviewershadermgr.cpp b/indra/newview/llviewershadermgr.cpp
index 0dff88521e..5ea52aca79 100644
--- a/indra/newview/llviewershadermgr.cpp
+++ b/indra/newview/llviewershadermgr.cpp
@@ -32,6 +32,7 @@
 #include "llfeaturemanager.h"
 #include "llviewershadermgr.h"
 #include "llviewercontrol.h"
+#include "llversioninfo.h"
 
 #include "llrender.h"
 #include "llenvironment.h"
@@ -368,6 +369,23 @@ void LLViewerShaderMgr::setShaders()
         return;
     }
 
+	{
+		static LLCachedControl<bool> shader_cache_enabled(gSavedSettings, "RenderShaderCacheEnabled", true);
+		static LLUUID old_cache_version;
+		static LLUUID current_cache_version;
+		if (current_cache_version.isNull())
+		{
+			HBXXH128 hash_obj;
+			hash_obj.update(LLVersionInfo::instance().getVersion());
+			current_cache_version = hash_obj.digest();
+
+			old_cache_version = LLUUID(gSavedSettings.getString("RenderShaderCacheVersion"));
+			gSavedSettings.setString("RenderShaderCacheVersion", current_cache_version.asString());
+		}
+
+		initShaderCache(shader_cache_enabled, old_cache_version, current_cache_version);
+	}
+
     static LLCachedControl<U32> max_texture_index(gSavedSettings, "RenderMaxTextureIndex", 16);
     
     // when using indexed texture rendering, leave 8 texture units available for shadow and reflection maps
@@ -375,9 +393,6 @@ void LLViewerShaderMgr::setShaders()
 
     reentrance = true;
 
-    //setup preprocessor definitions
-    LLShaderMgr::instance()->mDefinitions["NUM_TEX_UNITS"] = llformat("%d", gGLManager.mNumTextureImageUnits);
-    
     // Make sure the compiled shader map is cleared before we recompile shaders.
     mVertexShaderObjects.clear();
     mFragmentShaderObjects.clear();
@@ -385,6 +400,8 @@ void LLViewerShaderMgr::setShaders()
     initAttribsAndUniforms();
     gPipeline.releaseGLBuffers();
 
+	unloadShaders();
+
     LLPipeline::sRenderGlow = gSavedSettings.getBOOL("RenderGlow"); 
     
     if (gViewerWindow)
@@ -521,6 +538,8 @@ void LLViewerShaderMgr::setShaders()
     loaded = loaded && loadShadersDeferred();
     llassert(loaded);
 
+	persistShaderCacheMetadata();
+
     if (gViewerWindow)
     {
         gViewerWindow->setCursor(UI_CURSOR_ARROW);
@@ -532,61 +551,12 @@ void LLViewerShaderMgr::setShaders()
 
 void LLViewerShaderMgr::unloadShaders()
 {
-	gOcclusionProgram.unload();
-    gSkinnedOcclusionProgram.unload();
-	gOcclusionCubeProgram.unload();
-	gDebugProgram.unload();
-    gSkinnedDebugProgram.unload();
-	gClipProgram.unload();
-	gBenchmarkProgram.unload();
-    gReflectionProbeDisplayProgram.unload();
-	gAlphaMaskProgram.unload();
-	gUIProgram.unload();
-	gPathfindingProgram.unload();
-	gPathfindingNoNormalsProgram.unload();
-	gGlowCombineProgram.unload();
-	gReflectionMipProgram.unload();
-    gRadianceGenProgram.unload();
-    gIrradianceGenProgram.unload();
-	gGlowCombineFXAAProgram.unload();
-	gTwoTextureCompareProgram.unload();
-	gOneTextureFilterProgram.unload();
-	gSolidColorProgram.unload();
-
-	gObjectPreviewProgram.unload();
-    gPhysicsPreviewProgram.unload();
-	gImpostorProgram.unload();
-	gObjectBumpProgram.unload();
-    gSkinnedObjectBumpProgram.unload();
-    gSkinnedObjectFullbrightAlphaMaskProgram.unload();
-	
-	gObjectAlphaMaskNoColorProgram.unload();
-	gObjectAlphaMaskNoColorWaterProgram.unload();
-	
-	gWaterProgram.unload();
-    gWaterEdgeProgram.unload();
-	gUnderWaterProgram.unload();
-
-	gGlowProgram.unload();
-	gGlowExtractProgram.unload();
-	gAvatarProgram.unload();
-	gAvatarWaterProgram.unload();
-	gAvatarEyeballProgram.unload();
-	gHighlightProgram.unload();
-    gSkinnedHighlightProgram.unload();
-	gHighlightNormalProgram.unload();
-	gHighlightSpecularProgram.unload();
-
-	gPostScreenSpaceReflectionProgram.unload();
-
-	gDeferredDiffuseProgram.unload();
-	gDeferredDiffuseAlphaMaskProgram.unload();
-    gDeferredSkinnedDiffuseAlphaMaskProgram.unload();
-	gDeferredNonIndexedDiffuseAlphaMaskProgram.unload();
-	gDeferredNonIndexedDiffuseAlphaMaskNoColorProgram.unload();
-	gDeferredSkinnedDiffuseProgram.unload();
-	gDeferredSkinnedBumpProgram.unload();
-	
+	while (!LLGLSLShader::sInstances.empty())
+	{
+		LLGLSLShader* shader = *(LLGLSLShader::sInstances.begin());
+		shader->unload();
+	}
+
 	mShaderLevel[SHADER_LIGHTING] = 0;
 	mShaderLevel[SHADER_OBJECT] = 0;
 	mShaderLevel[SHADER_AVATAR] = 0;
@@ -658,7 +628,7 @@ std::string LLViewerShaderMgr::loadBasicShaders()
 	}
 	shaders.push_back( make_pair( "objects/nonindexedTextureV.glsl",        1 ) );
 
-	std::unordered_map<std::string, std::string> attribs;
+	std::map<std::string, std::string> attribs;
 	attribs["MAX_JOINTS_PER_MESH_OBJECT"] = 
 		boost::lexical_cast<std::string>(LLSkinningUtil::getMaxJointCount());
 
@@ -711,6 +681,8 @@ std::string LLViewerShaderMgr::loadBasicShaders()
 		attribs["REF_SAMPLE_COUNT"] = "32";
 	}
 
+	LLGLSLShader::sGlobalDefines = attribs;
+
 	// We no longer have to bind the shaders to global glhandles, they are automatically added to a map now.
 	for (U32 i = 0; i < shaders.size(); i++)
 	{
@@ -833,8 +805,8 @@ BOOL LLViewerShaderMgr::loadShadersWater()
 		gWaterEdgeProgram.mShaderFiles.clear();
 		gWaterEdgeProgram.mShaderFiles.push_back(make_pair("environment/waterV.glsl", GL_VERTEX_SHADER));
 		gWaterEdgeProgram.mShaderFiles.push_back(make_pair("environment/waterF.glsl", GL_FRAGMENT_SHADER));
-		gWaterEdgeProgram.addPermutation("WATER_EDGE", "1");
         gWaterEdgeProgram.clearPermutations();
+		gWaterEdgeProgram.addPermutation("WATER_EDGE", "1");
         if (LLPipeline::sRenderTransparentWater)
         {
             gWaterEdgeProgram.addPermutation("TRANSPARENT_WATER", "1");
@@ -1107,7 +1079,7 @@ BOOL LLViewerShaderMgr::loadShadersDeferred()
     
 	if (success)
 	{
-		gDeferredNonIndexedDiffuseAlphaMaskNoColorProgram.mName = "Deferred Diffuse Non-Indexed Alpha Mask Shader";
+		gDeferredNonIndexedDiffuseAlphaMaskNoColorProgram.mName = "Deferred Diffuse Non-Indexed Alpha Mask No Color Shader";
 		gDeferredNonIndexedDiffuseAlphaMaskNoColorProgram.mFeatures.encodesNormal = true;
 		gDeferredNonIndexedDiffuseAlphaMaskNoColorProgram.mShaderFiles.clear();
 		gDeferredNonIndexedDiffuseAlphaMaskNoColorProgram.mShaderFiles.push_back(make_pair("deferred/diffuseNoColorV.glsl", GL_VERTEX_SHADER));
@@ -2098,6 +2070,7 @@ BOOL LLViewerShaderMgr::loadShadersDeferred()
 		gDeferredFullbrightAlphaMaskWaterProgram.mShaderFiles.push_back(make_pair("deferred/fullbrightF.glsl", GL_FRAGMENT_SHADER));
 		gDeferredFullbrightAlphaMaskWaterProgram.mShaderLevel = mShaderLevel[SHADER_DEFERRED];
 		gDeferredFullbrightAlphaMaskWaterProgram.mShaderGroup = LLGLSLShader::SG_WATER;
+		gDeferredFullbrightAlphaMaskWaterProgram.clearPermutations();
 		gDeferredFullbrightAlphaMaskWaterProgram.addPermutation("HAS_ALPHA_MASK","1");
 		gDeferredFullbrightAlphaMaskWaterProgram.addPermutation("WATER_FOG","1");
         success = make_rigged_variant(gDeferredFullbrightAlphaMaskWaterProgram, gDeferredSkinnedFullbrightAlphaMaskWaterProgram);
@@ -2125,7 +2098,7 @@ BOOL LLViewerShaderMgr::loadShadersDeferred()
 
     if (success)
     {
-        gHUDFullbrightShinyProgram.mName = "Deferred FullbrightShiny Shader";
+        gHUDFullbrightShinyProgram.mName = "HUD FullbrightShiny Shader";
         gHUDFullbrightShinyProgram.mFeatures.calculatesAtmospherics = true;
         gHUDFullbrightShinyProgram.mFeatures.hasAtmospherics = true;
         gHUDFullbrightShinyProgram.mFeatures.hasGamma = true;
@@ -2421,6 +2394,7 @@ BOOL LLViewerShaderMgr::loadShadersDeferred()
 		gDeferredTerrainWaterProgram.mShaderFiles.push_back(make_pair("deferred/terrainF.glsl", GL_FRAGMENT_SHADER));
 		gDeferredTerrainWaterProgram.mShaderLevel = mShaderLevel[SHADER_DEFERRED];
 		gDeferredTerrainWaterProgram.mShaderGroup = LLGLSLShader::SG_WATER;
+		gDeferredTerrainWaterProgram.clearPermutations();
 		gDeferredTerrainWaterProgram.addPermutation("WATER_FOG", "1");
 		success = gDeferredTerrainWaterProgram.createShader(NULL, NULL);
         llassert(success);
@@ -2428,7 +2402,7 @@ BOOL LLViewerShaderMgr::loadShadersDeferred()
 
 	if (success)
 	{
-		gDeferredAvatarProgram.mName = "Avatar Shader";
+		gDeferredAvatarProgram.mName = "Deferred Avatar Shader";
 		gDeferredAvatarProgram.mFeatures.hasSkinning = true;
 		gDeferredAvatarProgram.mFeatures.encodesNormal = true;
 		gDeferredAvatarProgram.mShaderFiles.clear();
@@ -2441,7 +2415,7 @@ BOOL LLViewerShaderMgr::loadShadersDeferred()
 
 	if (success)
 	{
-		gDeferredAvatarAlphaProgram.mName = "Avatar Alpha Shader";
+		gDeferredAvatarAlphaProgram.mName = "Deferred Avatar Alpha Shader";
 		gDeferredAvatarAlphaProgram.mFeatures.hasSkinning = true;
 		gDeferredAvatarAlphaProgram.mFeatures.calculatesLighting = false;
 		gDeferredAvatarAlphaProgram.mFeatures.hasLighting = false;
@@ -2577,7 +2551,7 @@ BOOL LLViewerShaderMgr::loadShadersDeferred()
 	if (success)
 	{
 		gDeferredPostProgram.mName = "Deferred Post Shader";
-		gFXAAProgram.mFeatures.isDeferred = true;
+        gDeferredPostProgram.mFeatures.isDeferred = true;
 		gDeferredPostProgram.mShaderFiles.clear();
 		gDeferredPostProgram.mShaderFiles.push_back(make_pair("deferred/postDeferredNoTCV.glsl", GL_VERTEX_SHADER));
 		gDeferredPostProgram.mShaderFiles.push_back(make_pair("deferred/postDeferredF.glsl", GL_FRAGMENT_SHADER));
@@ -2612,7 +2586,7 @@ BOOL LLViewerShaderMgr::loadShadersDeferred()
 
 	if (success)
 	{
-		gDeferredPostNoDoFProgram.mName = "Deferred Post Shader";
+		gDeferredPostNoDoFProgram.mName = "Deferred Post NoDoF Shader";
 		gDeferredPostNoDoFProgram.mFeatures.isDeferred = true;
 		gDeferredPostNoDoFProgram.mShaderFiles.clear();
 		gDeferredPostNoDoFProgram.mShaderFiles.push_back(make_pair("deferred/postDeferredNoTCV.glsl", GL_VERTEX_SHADER));
diff --git a/indra/newview/skins/default/xui/en/menu_viewer.xml b/indra/newview/skins/default/xui/en/menu_viewer.xml
index 3164c3e124..7a29d68a4c 100644
--- a/indra/newview/skins/default/xui/en/menu_viewer.xml
+++ b/indra/newview/skins/default/xui/en/menu_viewer.xml
@@ -3259,7 +3259,24 @@ function="World.EnvPreset"
                  parameter="RenderAttachedParticles" />
             </menu_item_check>
           <menu_item_separator />
-          
+          <menu_item_check
+             label="Enable Shader Cache"
+             name="Enable Shader Cache">
+            <menu_item_check.on_check
+             function="CheckControl"
+             parameter="RenderShaderCacheEnabled" />
+            <menu_item_check.on_click
+             function="ToggleShaderControl"
+             parameter="RenderShaderCacheEnabled" />
+          </menu_item_check>
+          <menu_item_call
+            enabled="true"
+            label="Clear Shader Cache"
+            name="Shader Cache Clear">
+            <menu_item_call.on_click
+             function="Advanced.ClearShaderCache" />
+          </menu_item_call>
+          <menu_item_separator />
           <menu_item_call
             enabled="true"
             label="Clear Cache Immediately"