#4298 Handle various out of memory cases

2 files changed, 43 insertions, 19 deletions

diff --git a/indra/llrender/llfontfreetype.cpp b/indra/llrender/llfontfreetype.cpp
index d37b16ce0c..d9857a7ad5 100644
--- a/indra/llrender/llfontfreetype.cpp
+++ b/indra/llrender/llfontfreetype.cpp
@@ -871,30 +871,39 @@ namespace ll
 
 U8 const* LLFontManager::loadFont( std::string const &aFilename, long &a_Size)
 {
-    a_Size = 0;
-    std::map< std::string, std::shared_ptr<ll::fonts::LoadedFont> >::iterator itr = m_LoadedFonts.find( aFilename );
-    if( itr != m_LoadedFonts.end() )
+    try
     {
-        ++itr->second->mRefs;
-        // A possible overflow cannot happen here, as it is asserted that the size is less than std::numeric_limits<long>::max() a few lines below.
-        a_Size = static_cast<long>(itr->second->mSize);
-        return reinterpret_cast<U8 const*>(itr->second->mAddress.c_str());
-    }
+        a_Size = 0;
+        std::map< std::string, std::shared_ptr<ll::fonts::LoadedFont> >::iterator itr = m_LoadedFonts.find(aFilename);
+        if (itr != m_LoadedFonts.end())
+        {
+            ++itr->second->mRefs;
+            // A possible overflow cannot happen here, as it is asserted that the size is less than std::numeric_limits<long>::max() a few lines below.
+            a_Size = static_cast<long>(itr->second->mSize);
+            return reinterpret_cast<U8 const*>(itr->second->mAddress.c_str());
+        }
 
-    auto strContent = LLFile::getContents(aFilename);
+        auto strContent = LLFile::getContents(aFilename);
 
-    if( strContent.empty() )
-        return nullptr;
+        if (strContent.empty())
+            return nullptr;
 
-    // For fontconfig a type of long is required, std::string::size() returns size_t. I think it is safe to limit this to 2GiB and not support fonts that huge (can that even be a thing?)
-    llassert_always( strContent.size() < std::numeric_limits<long>::max() );
+        // For fontconfig a type of long is required, std::string::size() returns size_t. I think it is safe to limit this to 2GiB and not support fonts that huge (can that even be a thing?)
+        llassert_always(strContent.size() < std::numeric_limits<long>::max());
 
-    a_Size = static_cast<long>(strContent.size());
+        a_Size = static_cast<long>(strContent.size());
 
-    auto pCache = std::make_shared<ll::fonts::LoadedFont>( aFilename,  strContent, a_Size );
-    itr = m_LoadedFonts.insert( std::make_pair( aFilename, pCache ) ).first;
+        auto pCache = std::make_shared<ll::fonts::LoadedFont>(aFilename, strContent, a_Size);
+        itr = m_LoadedFonts.insert(std::make_pair(aFilename, pCache)).first;
 
-    return reinterpret_cast<U8 const*>(itr->second->mAddress.c_str());
+        return reinterpret_cast<U8 const*>(itr->second->mAddress.c_str());
+    }
+    catch (const std::bad_alloc&)
+    {
+        LLError::LLUserWarningMsg::showOutOfMemory();
+        LL_ERRS() << "Failed to load font. Out of memory." << LL_ENDL;
+    }
+    return nullptr;
 }
 
 void LLFontManager::unloadAllFonts()
diff --git a/indra/llrender/llshadermgr.cpp b/indra/llrender/llshadermgr.cpp
index 2c35a6acae..b8545b3ed9 100644
--- a/indra/llrender/llshadermgr.cpp
+++ b/indra/llrender/llshadermgr.cpp
@@ -1024,8 +1024,23 @@ void LLShaderMgr::initShaderCache(bool enabled, const LLUUID& old_cache_version,
 
             llifstream instream(meta_out_path, std::ifstream::in | std::ifstream::binary);
             LLSD in_data;
-            // todo: this is likely very expensive to parse, should use binary
-            LLSDSerialize::fromBinary(in_data, instream, LLSDSerialize::SIZE_UNLIMITED);
+            try
+            {
+                LLSDSerialize::fromBinary(in_data, instream, LLSDSerialize::SIZE_UNLIMITED);
+            }
+            catch( std::bad_alloc& )
+            {
+                // Try to get a bit more memory back before we try to clear the cache.
+                in_data.clear();
+                // Just in case it was somehow the cause, clear cache.
+                clearShaderCache();
+                // If user run out of memory this early in init,
+                // we don't want to keep going just to crash again.
+                // Notify user and close.
+                LLError::LLUserWarningMsg::showOutOfMemory();
+                LL_ERRS("ShaderMgr") << "Failed to parse shader cache metadata, potentially due to size. Purged cache." << LL_ENDL;
+                return;
+            }
             instream.close();
 
             if (old_cache_version == current_cache_version