SL-16182: Offload audio decoding from the main thread to the general work queue.

The work queue callback binds "this". This is deemed safe due to current dependencies, but see the associated comment in the return callback. There was some trial and error to get a this-binded lambda to compile.

Due to LLVorbisDecodeState writing to disk off-thread, limit audio decodes proportional to general worker thread count. Guess the thread count for now.

11 files changed, 320 insertions, 237 deletions

diff --git a/indra/llaudio/llaudiodecodemgr.cpp b/indra/llaudio/llaudiodecodemgr.cpp
index ff0aa6e76e..cc5abee08a 100644..100755
--- a/indra/llaudio/llaudiodecodemgr.cpp
+++ b/indra/llaudio/llaudiodecodemgr.cpp
@@ -35,6 +35,7 @@
 #include "llendianswizzle.h"
 #include "llassetstorage.h"
 #include "llrefcount.h"
+#include "workqueue.h"
 
 #include "llvorbisencode.h"
 
@@ -45,15 +46,13 @@
 
 extern LLAudioEngine *gAudiop;
 
-LLAudioDecodeMgr *gAudioDecodeMgrp = NULL;
-
 static const S32 WAV_HEADER_SIZE = 44;
 
 
 //////////////////////////////////////////////////////////////////////////////
 
 
-class LLVorbisDecodeState : public LLRefCount
+class LLVorbisDecodeState : public LLThreadSafeRefCount
 {
 public:
 	class WriteResponder : public LLLFSThread::Responder
@@ -532,146 +531,242 @@ void LLVorbisDecodeState::flushBadFile()
 
 class LLAudioDecodeMgr::Impl
 {
-	friend class LLAudioDecodeMgr;
-public:
-	Impl() {};
-	~Impl() {};
+    friend class LLAudioDecodeMgr;
+    LLAudioDecodeMgr::Impl();
+  public:
 
-	void processQueue(const F32 num_secs = 0.005);
+    void processQueue();
 
-protected:
-	std::deque<LLUUID> mDecodeQueue;
-	LLPointer<LLVorbisDecodeState> mCurrentDecodep;
+    void startMoreDecodes();
+    void enqueueFinishAudio(const LLUUID &decode_id, LLPointer<LLVorbisDecodeState>& decode_state);
+    void checkDecodesFinished();
+
+  protected:
+    std::deque<LLUUID> mDecodeQueue;
+    std::map<LLUUID, LLPointer<LLVorbisDecodeState>> mDecodes;
 };
 
+LLAudioDecodeMgr::Impl::Impl()
+{
+}
+
+// Returns the in-progress decode_state, which may be an empty LLPointer if
+// there was an error and there is no more work to be done.
+LLPointer<LLVorbisDecodeState> beginDecodingAndWritingAudio(const LLUUID &decode_id);
 
-void LLAudioDecodeMgr::Impl::processQueue(const F32 num_secs)
+// Return true if finished
+bool tryFinishAudio(const LLUUID &decode_id, LLPointer<LLVorbisDecodeState> decode_state);
+
+void LLAudioDecodeMgr::Impl::processQueue()
 {
-	LLUUID uuid;
+    // First, check if any audio from in-progress decodes are ready to play. If
+    // so, mark them ready for playback (or errored, in case of error).
+    checkDecodesFinished();
 
-	LLTimer decode_timer;
+    // Second, start as many decodes from the queue as permitted
+    startMoreDecodes();
+}
 
-	BOOL done = FALSE;
-	while (!done)
-	{
-		if (mCurrentDecodep)
-		{
-			BOOL res;
+void LLAudioDecodeMgr::Impl::startMoreDecodes()
+{
+    llassert_always(gAudiop);
+
+    LL::WorkQueue::ptr_t main_queue = LL::WorkQueue::getInstance("mainloop");
+    // *NOTE: main_queue->postTo casts this refcounted smart pointer to a weak
+    // pointer
+    LL::WorkQueue::ptr_t general_queue = LL::WorkQueue::getInstance("General");
+    llassert_always(main_queue);
+    llassert_always(general_queue);
+    // Set max decodes to double the thread count of the general work queue.
+    // This ensures the general work queue is full, but prevents theoretical
+    // buildup of buffers in memory due to disk writes once the
+    // LLVorbisDecodeState leaves the worker thread (see
+    // LLLFSThread::sLocal->write). This is probably as fast as we can get it
+    // without modifying/removing LLVorbisDecodeState, at which point we should
+    // consider decoding the audio during the asset download process.
+    // -Cosmic,2022-05-11
+    const size_t MAX_DECODES = 4 * 2;
+
+    while (!mDecodeQueue.empty() && mDecodes.size() < MAX_DECODES)
+    {
+        const LLUUID decode_id = mDecodeQueue.front();
+        mDecodeQueue.pop_front();
+
+        // Don't decode the same file twice
+        if (mDecodes.find(decode_id) != mDecodes.end())
+        {
+            continue;
+        }
+        if (gAudiop->hasDecodedFile(decode_id))
+        {
+            continue;
+        }
+
+        // Kick off a decode
+        mDecodes[decode_id] = LLPointer<LLVorbisDecodeState>(NULL);
+        main_queue->postTo(
+            general_queue,
+            [decode_id, this]() // Work done on general queue
+            {
+                LLPointer<LLVorbisDecodeState> decode_state = beginDecodingAndWritingAudio(decode_id);
+
+                if (!decode_state)
+                {
+                    // Audio decode has errored
+                    return decode_state;
+                }
+
+                // Disk write of decoded audio is now in progress off-thread
+                return decode_state;
+            },
+            [decode_id, this](LLPointer<LLVorbisDecodeState> decode_state) // Callback to main thread
+            mutable {
+                if (!gAudiop)
+                {
+                    // There is no LLAudioEngine anymore. This might happen if
+                    // an audio decode is enqueued just before shutdown.
+                    return;
+                }
+
+                // At this point, we can be certain that the pointer to "this"
+                // is valid because the lifetime of "this" is dependent upon
+                // the lifetime of gAudiop.
+
+                enqueueFinishAudio(decode_id, decode_state);
+            });
+    }
+}
 
-			// Decode in a loop until we're done or have run out of time.
-			while(!(res = mCurrentDecodep->decodeSection()) && (decode_timer.getElapsedTimeF32() < num_secs))
-			{
-				// decodeSection does all of the work above
-			}
+LLPointer<LLVorbisDecodeState> beginDecodingAndWritingAudio(const LLUUID &decode_id)
+{
+    LL_PROFILE_ZONE_SCOPED_CATEGORY_MEDIA;
+
+    LL_DEBUGS() << "Decoding " << decode_id << " from audio queue!" << LL_ENDL;
+
+    std::string                    d_path       = gDirUtilp->getExpandedFilename(LL_PATH_CACHE, decode_id.asString()) + ".dsf";
+    LLPointer<LLVorbisDecodeState> decode_state = new LLVorbisDecodeState(decode_id, d_path);
+
+    if (!decode_state->initDecode())
+    {
+        return NULL;
+    }
+
+    // Decode in a loop until we're done
+    while (!decode_state->decodeSection())
+    {
+        // decodeSection does all of the work above
+    }
+
+    if (!decode_state->isDone())
+    {
+        // Decode stopped early, or something bad happened to the file
+        // during decoding.
+        LL_WARNS("AudioEngine") << decode_id << " has invalid vorbis data or decode has been canceled, aborting decode" << LL_ENDL;
+        decode_state->flushBadFile();
+        return NULL;
+    }
+
+    if (!decode_state->isValid())
+    {
+        // We had an error when decoding, abort.
+        LL_WARNS("AudioEngine") << decode_id << " has invalid vorbis data, aborting decode" << LL_ENDL;
+        decode_state->flushBadFile();
+        return NULL;
+    }
+
+    // Kick off the writing of the decoded audio to the disk cache.
+    // The receiving thread can then cheaply call finishDecode() again to check
+    // if writing has finished. Someone has to hold on to the refcounted
+    // decode_state to prevent it from getting destroyed during write.
+    decode_state->finishDecode();
+
+    return decode_state;
+}
 
-			if (mCurrentDecodep->isDone() && !mCurrentDecodep->isValid())
-			{
-				// We had an error when decoding, abort.
-				LL_WARNS("AudioEngine") << mCurrentDecodep->getUUID() << " has invalid vorbis data, aborting decode" << LL_ENDL;
-				mCurrentDecodep->flushBadFile();
-
-				if (gAudiop)
-				{
-					LLAudioData *adp = gAudiop->getAudioData(mCurrentDecodep->getUUID());
-					adp->setHasValidData(false);
-					adp->setHasCompletedDecode(true);
-				}
-
-				mCurrentDecodep = NULL;
-				done = TRUE;
-			}
+void LLAudioDecodeMgr::Impl::enqueueFinishAudio(const LLUUID &decode_id, LLPointer<LLVorbisDecodeState>& decode_state)
+{
+    // Assumed fast
+    if (tryFinishAudio(decode_id, decode_state))
+    {
+        // Done early!
+        auto decode_iter = mDecodes.find(decode_id);
+        llassert(decode_iter != mDecodes.end());
+        mDecodes.erase(decode_iter);
+        return;
+    }
+
+    // Not done yet... enqueue it
+    mDecodes[decode_id] = decode_state;
+}
 
-			if (!res)
-			{
-				// We've used up out time slice, bail...
-				done = TRUE;
-			}
-			else if (mCurrentDecodep)
-			{
-				if (gAudiop && mCurrentDecodep->finishDecode())
-				{
-					// We finished!
-					LLAudioData *adp = gAudiop->getAudioData(mCurrentDecodep->getUUID());
-					if (!adp)
-					{
-						LL_WARNS("AudioEngine") << "Missing LLAudioData for decode of " << mCurrentDecodep->getUUID() << LL_ENDL;
-					}
-					else if (mCurrentDecodep->isValid() && mCurrentDecodep->isDone())
-					{
-						adp->setHasCompletedDecode(true);
-						adp->setHasDecodedData(true);
-						adp->setHasValidData(true);
-
-						// At this point, we could see if anyone needs this sound immediately, but
-						// I'm not sure that there's a reason to - we need to poll all of the playing
-						// sounds anyway.
-						//LL_INFOS("AudioEngine") << "Finished the vorbis decode, now what?" << LL_ENDL;
-					}
-					else
-					{
-						adp->setHasCompletedDecode(true);
-						LL_INFOS("AudioEngine") << "Vorbis decode failed for " << mCurrentDecodep->getUUID() << LL_ENDL;
-					}
-					mCurrentDecodep = NULL;
-				}
-				done = TRUE; // done for now
-			}
-		}
+void LLAudioDecodeMgr::Impl::checkDecodesFinished()
+{
+    auto decode_iter = mDecodes.begin();
+    while (decode_iter != mDecodes.end())
+    {
+        const LLUUID& decode_id = decode_iter->first;
+        const LLPointer<LLVorbisDecodeState>& decode_state = decode_iter->second;
+        if (tryFinishAudio(decode_id, decode_state))
+        {
+            decode_iter = mDecodes.erase(decode_iter);
+        }
+        else
+        {
+            ++decode_iter;
+        }
+    }
+}
 
-		if (!done)
-		{
-			if (mDecodeQueue.empty())
-			{
-				// Nothing else on the queue.
-				done = TRUE;
-			}
-			else
-			{
-				LLUUID uuid;
-				uuid = mDecodeQueue.front();
-				mDecodeQueue.pop_front();
-				if (!gAudiop || gAudiop->hasDecodedFile(uuid))
-				{
-					// This file has already been decoded, don't decode it again.
-					continue;
-				}
-
-				LL_DEBUGS() << "Decoding " << uuid << " from audio queue!" << LL_ENDL;
-
-				std::string uuid_str;
-				std::string d_path;
-
-				LLTimer timer;
-				timer.reset();
-
-				uuid.toString(uuid_str);
-				d_path = gDirUtilp->getExpandedFilename(LL_PATH_CACHE,uuid_str) + ".dsf";
-
-				mCurrentDecodep = new LLVorbisDecodeState(uuid, d_path);
-				if (!mCurrentDecodep->initDecode())
-				{
-					mCurrentDecodep = NULL;
-				}
-			}
-		}
-	}
+bool tryFinishAudio(const LLUUID &decode_id, LLPointer<LLVorbisDecodeState> decode_state)
+{
+    // decode_state is a file write in progress unless finished is true
+    bool finished = decode_state && decode_state->finishDecode();
+    if (!finished)
+    {
+        return false;
+    }
+
+    llassert_always(gAudiop);
+
+    LLAudioData *adp = gAudiop->getAudioData(decode_id);
+    if (!adp)
+    {
+        LL_WARNS("AudioEngine") << "Missing LLAudioData for decode of " << decode_id << LL_ENDL;
+        return true;
+    }
+
+    bool valid = decode_state && decode_state->isValid();
+    // Mark current decode finished regardless of success or failure
+    adp->setHasCompletedDecode(true);
+    // Flip flags for decoded data
+    adp->setHasDecodeFailed(!valid);
+    adp->setHasDecodedData(valid);
+    // When finished decoding, there will also be a decoded wav file cached on
+    // disk with the .dsf extension
+    if (valid)
+    {
+        adp->setHasWAVLoadFailed(false);
+    }
+
+    return true;
 }
 
 //////////////////////////////////////////////////////////////////////////////
 
 LLAudioDecodeMgr::LLAudioDecodeMgr()
 {
-	mImpl = new Impl;
+    mImpl = new Impl();
 }
 
 LLAudioDecodeMgr::~LLAudioDecodeMgr()
 {
-	delete mImpl;
+    delete mImpl;
+    mImpl = nullptr;
 }
 
-void LLAudioDecodeMgr::processQueue(const F32 num_secs)
+void LLAudioDecodeMgr::processQueue()
 {
-	mImpl->processQueue(num_secs);
+    mImpl->processQueue();
 }
 
 BOOL LLAudioDecodeMgr::addDecodeRequest(const LLUUID &uuid)
@@ -687,7 +782,7 @@ BOOL LLAudioDecodeMgr::addDecodeRequest(const LLUUID &uuid)
 	{
 		// Just put it on the decode queue.
 		LL_DEBUGS("AudioEngine") << "addDecodeRequest for " << uuid << " has local asset file already" << LL_ENDL;
-		mImpl->mDecodeQueue.push_back(uuid);
+        mImpl->mDecodeQueue.push_back(uuid);
 		return TRUE;
 	}
 
diff --git a/indra/llaudio/llaudiodecodemgr.h b/indra/llaudio/llaudiodecodemgr.h
index ceaff3f2d8..4c17b46156 100644
--- a/indra/llaudio/llaudiodecodemgr.h
+++ b/indra/llaudio/llaudiodecodemgr.h
@@ -32,24 +32,23 @@
 
 #include "llassettype.h"
 #include "llframetimer.h"
+#include "llsingleton.h"
 
+template<class T> class LLPointer;
 class LLVorbisDecodeState;
 
-class LLAudioDecodeMgr
+class LLAudioDecodeMgr : public LLSingleton<LLAudioDecodeMgr>
 {
+    LLSINGLETON(LLAudioDecodeMgr);
+    ~LLAudioDecodeMgr();
 public:
-	LLAudioDecodeMgr();
-	~LLAudioDecodeMgr();
-
-	void processQueue(const F32 num_secs = 0.005);
+	void processQueue();
 	BOOL addDecodeRequest(const LLUUID &uuid);
 	void addAudioRequest(const LLUUID &uuid);
 	
 protected:
 	class Impl;
-	Impl* mImpl;
+    Impl* mImpl;
 };
 
-extern LLAudioDecodeMgr *gAudioDecodeMgrp;
-
 #endif
diff --git a/indra/llaudio/llaudioengine.cpp b/indra/llaudio/llaudioengine.cpp
index e0ebbb76bd..cae6bb62d9 100644
--- a/indra/llaudio/llaudioengine.cpp
+++ b/indra/llaudio/llaudioengine.cpp
@@ -83,18 +83,10 @@ void LLAudioEngine::setDefaults()
 
 	mLastStatus = 0;
 
-	mNumChannels = 0;
 	mEnableWind = false;
 
-	S32 i;
-	for (i = 0; i < MAX_CHANNELS; i++)
-	{
-		mChannels[i] = NULL;
-	}
-	for (i = 0; i < MAX_BUFFERS; i++)
-	{
-		mBuffers[i] = NULL;
-	}
+	mChannels.fill(nullptr);
+	mBuffers.fill(nullptr);
 
 	mMasterGain = 1.f;
 	// Setting mInternalGain to an out of range value fixes the issue reported in STORM-830.
@@ -111,18 +103,14 @@ void LLAudioEngine::setDefaults()
 }
 
 
-bool LLAudioEngine::init(const S32 num_channels, void* userdata, const std::string &app_title)
+bool LLAudioEngine::init(void* userdata, const std::string &app_title)
 {
 	setDefaults();
 
-	mNumChannels = num_channels;
 	mUserData = userdata;
 	
 	allocateListener();
 
-	// Initialize the decode manager
-	gAudioDecodeMgrp = new LLAudioDecodeMgr;
-
 	LL_INFOS("AudioEngine") << "LLAudioEngine::init() AudioEngine successfully initialized" << LL_ENDL;
 
 	return true;
@@ -131,10 +119,6 @@ bool LLAudioEngine::init(const S32 num_channels, void* userdata, const std::stri
 
 void LLAudioEngine::shutdown()
 {
-	// Clean up decode manager
-	delete gAudioDecodeMgrp;
-	gAudioDecodeMgrp = NULL;
-
 	// Clean up wind source
 	cleanupWind();
 
@@ -156,14 +140,14 @@ void LLAudioEngine::shutdown()
 
 	// Clean up channels
 	S32 i;
-	for (i = 0; i < MAX_CHANNELS; i++)
+	for (i = 0; i < LL_MAX_AUDIO_CHANNELS; i++)
 	{
 		delete mChannels[i];
 		mChannels[i] = NULL;
 	}
 
 	// Clean up buffers
-	for (i = 0; i < MAX_BUFFERS; i++)
+	for (i = 0; i < LL_MAX_AUDIO_BUFFERS; i++)
 	{
 		delete mBuffers[i];
 		mBuffers[i] = NULL;
@@ -229,7 +213,7 @@ std::string LLAudioEngine::getInternetStreamURL()
 void LLAudioEngine::updateChannels()
 {
 	S32 i;
-	for (i = 0; i < MAX_CHANNELS; i++)
+	for (i = 0; i < LL_MAX_AUDIO_CHANNELS; i++)
 	{
 		if (mChannels[i])
 		{
@@ -240,20 +224,14 @@ void LLAudioEngine::updateChannels()
 	}
 }
 
-static const F32 default_max_decode_time = .002f; // 2 ms
-void LLAudioEngine::idle(F32 max_decode_time)
+void LLAudioEngine::idle()
 {
-	if (max_decode_time <= 0.f)
-	{
-		max_decode_time = default_max_decode_time;
-	}
-	
 	// "Update" all of our audio sources, clean up dead ones.
 	// Primarily does position updating, cleanup of unused audio sources.
 	// Also does regeneration of the current priority of each audio source.
 
 	S32 i;
-	for (i = 0; i < MAX_BUFFERS; i++)
+	for (i = 0; i < LL_MAX_AUDIO_BUFFERS; i++)
 	{
 		if (mBuffers[i])
 		{
@@ -473,7 +451,7 @@ void LLAudioEngine::idle(F32 max_decode_time)
 	commitDeferredChanges();
 	
 	// Flush unused buffers that are stale enough
-	for (i = 0; i < MAX_BUFFERS; i++)
+	for (i = 0; i < LL_MAX_AUDIO_BUFFERS; i++)
 	{
 		if (mBuffers[i])
 		{
@@ -489,7 +467,7 @@ void LLAudioEngine::idle(F32 max_decode_time)
 
 
 	// Clear all of the looped flags for the channels
-	for (i = 0; i < MAX_CHANNELS; i++)
+	for (i = 0; i < LL_MAX_AUDIO_CHANNELS; i++)
 	{
 		if (mChannels[i])
 		{
@@ -498,7 +476,7 @@ void LLAudioEngine::idle(F32 max_decode_time)
 	}
 
 	// Decode audio files
-	gAudioDecodeMgrp->processQueue(max_decode_time);
+    LLAudioDecodeMgr::getInstance()->processQueue();
 	
 	// Call this every frame, just in case we somehow
 	// missed picking it up in all the places that can add
@@ -532,7 +510,7 @@ bool LLAudioEngine::updateBufferForData(LLAudioData *adp, const LLUUID &audio_uu
 		{
 			if (audio_uuid.notNull())
 			{
-				gAudioDecodeMgrp->addDecodeRequest(audio_uuid);
+                LLAudioDecodeMgr::getInstance()->addDecodeRequest(audio_uuid);
 			}
 		}
 		else
@@ -561,7 +539,7 @@ void LLAudioEngine::enableWind(bool enable)
 LLAudioBuffer * LLAudioEngine::getFreeBuffer()
 {
 	S32 i;
-	for (i = 0; i < MAX_BUFFERS; i++)
+	for (i = 0; i < LL_MAX_AUDIO_BUFFERS; i++)
 	{
 		if (!mBuffers[i])
 		{
@@ -574,7 +552,7 @@ LLAudioBuffer * LLAudioEngine::getFreeBuffer()
 	// Grab the oldest unused buffer
 	F32 max_age = -1.f;
 	S32 buffer_id = -1;
-	for (i = 0; i < MAX_BUFFERS; i++)
+	for (i = 0; i < LL_MAX_AUDIO_BUFFERS; i++)
 	{
 		if (mBuffers[i])
 		{
@@ -605,7 +583,7 @@ LLAudioBuffer * LLAudioEngine::getFreeBuffer()
 LLAudioChannel * LLAudioEngine::getFreeChannel(const F32 priority)
 {
 	S32 i;
-	for (i = 0; i < mNumChannels; i++)
+    for (i = 0; i < LL_MAX_AUDIO_CHANNELS; i++)
 	{
 		if (!mChannels[i])
 		{
@@ -633,7 +611,7 @@ LLAudioChannel * LLAudioEngine::getFreeChannel(const F32 priority)
 	F32 min_priority = 10000.f;
 	LLAudioChannel *min_channelp = NULL;
 
-	for (i = 0; i < mNumChannels; i++)
+    for (i = 0; i < LL_MAX_AUDIO_CHANNELS; i++)
 	{
 		LLAudioChannel *channelp = mChannels[i];
 		LLAudioSource *sourcep = channelp->getSource();
@@ -660,7 +638,7 @@ LLAudioChannel * LLAudioEngine::getFreeChannel(const F32 priority)
 void LLAudioEngine::cleanupBuffer(LLAudioBuffer *bufferp)
 {
 	S32 i;
-	for (i = 0; i < MAX_BUFFERS; i++)
+	for (i = 0; i < LL_MAX_AUDIO_BUFFERS; i++)
 	{
 		if (mBuffers[i] == bufferp)
 		{
@@ -678,7 +656,7 @@ bool LLAudioEngine::preloadSound(const LLUUID &uuid)
 	getAudioData(uuid);	// We don't care about the return value, this is just to make sure
 									// that we have an entry, which will mean that the audio engine knows about this
 
-	if (gAudioDecodeMgrp->addDecodeRequest(uuid))
+    if (LLAudioDecodeMgr::getInstance()->addDecodeRequest(uuid))
 	{
 		// This means that we do have a local copy, and we're working on decoding it.
 		return true;
@@ -953,6 +931,7 @@ LLAudioSource * LLAudioEngine::findAudioSource(const LLUUID &source_id)
 
 LLAudioData * LLAudioEngine::getAudioData(const LLUUID &audio_uuid)
 {
+	LL_PROFILE_ZONE_SCOPED_CATEGORY_MEDIA;
 	data_map::iterator iter;
 	iter = mAllData.find(audio_uuid);
 	if (iter == mAllData.end())
@@ -1039,7 +1018,7 @@ void LLAudioEngine::startNextTransfer()
 
 	// Check all channels for currently playing sounds.
 	F32 max_pri = -1.f;
-	for (i = 0; i < MAX_CHANNELS; i++)
+	for (i = 0; i < LL_MAX_AUDIO_CHANNELS; i++)
 	{
 		if (!mChannels[i])
 		{
@@ -1067,7 +1046,7 @@ void LLAudioEngine::startNextTransfer()
 			continue;
 		}
 
-		if (!adp->hasLocalData() && adp->hasValidData())
+        if (!adp->hasLocalData() && !adp->hasDecodeFailed())
 		{
 			asset_id = adp->getID();
 			max_pri = asp->getPriority();
@@ -1078,7 +1057,7 @@ void LLAudioEngine::startNextTransfer()
 	if (asset_id.isNull())
 	{
 		max_pri = -1.f;
-		for (i = 0; i < MAX_CHANNELS; i++)
+		for (i = 0; i < LL_MAX_AUDIO_CHANNELS; i++)
 		{
 			if (!mChannels[i])
 			{
@@ -1103,7 +1082,7 @@ void LLAudioEngine::startNextTransfer()
 				continue;
 			}
 
-			if (!adp->hasLocalData() && adp->hasValidData())
+            if (!adp->hasLocalData() && !adp->hasDecodeFailed())
 			{
 				asset_id = adp->getID();
 				max_pri = asp->getPriority();
@@ -1115,7 +1094,7 @@ void LLAudioEngine::startNextTransfer()
 	if (asset_id.isNull())
 	{
 		max_pri = -1.f;
-		for (i = 0; i < MAX_CHANNELS; i++)
+		for (i = 0; i < LL_MAX_AUDIO_CHANNELS; i++)
 		{
 			if (!mChannels[i])
 			{
@@ -1143,7 +1122,7 @@ void LLAudioEngine::startNextTransfer()
 					continue;
 				}
 
-				if (!adp->hasLocalData() && adp->hasValidData())
+                if (!adp->hasLocalData() && !adp->hasDecodeFailed())
 				{
 					asset_id = adp->getID();
 					max_pri = asp->getPriority();
@@ -1171,7 +1150,7 @@ void LLAudioEngine::startNextTransfer()
 			}
 
 			adp = asp->getCurrentData();
-			if (adp && !adp->hasLocalData() && adp->hasValidData())
+            if (adp && !adp->hasLocalData() && !adp->hasDecodeFailed())
 			{
 				asset_id = adp->getID();
 				max_pri = asp->getPriority();
@@ -1179,7 +1158,7 @@ void LLAudioEngine::startNextTransfer()
 			}
 
 			adp = asp->getQueuedData();
-			if (adp && !adp->hasLocalData() && adp->hasValidData())
+            if (adp && !adp->hasLocalData() && !adp->hasDecodeFailed())
 			{
 				asset_id = adp->getID();
 				max_pri = asp->getPriority();
@@ -1194,7 +1173,7 @@ void LLAudioEngine::startNextTransfer()
 					continue;
 				}
 
-				if (!adp->hasLocalData() && adp->hasValidData())
+                if (!adp->hasLocalData() && !adp->hasDecodeFailed())
 				{
 					asset_id = adp->getID();
 					max_pri = asp->getPriority();
@@ -1235,7 +1214,7 @@ void LLAudioEngine::assetCallback(const LLUUID &uuid, LLAssetType::EType type, v
 		LLAudioData *adp = gAudiop->getAudioData(uuid);
 		if (adp)
         {	// Make sure everything is cleared
-			adp->setHasValidData(false);
+            adp->setHasDecodeFailed(true);
 			adp->setHasLocalData(false);
 			adp->setHasDecodedData(false);
 			adp->setHasCompletedDecode(true);
@@ -1252,9 +1231,9 @@ void LLAudioEngine::assetCallback(const LLUUID &uuid, LLAssetType::EType type, v
 		else
 		{
 			// LL_INFOS() << "Got asset callback with good audio data for " << uuid << ", making decode request" << LL_ENDL;
-			adp->setHasValidData(true);
+            adp->setHasDecodeFailed(false);
 		    adp->setHasLocalData(true);
-		    gAudioDecodeMgrp->addDecodeRequest(uuid);
+            LLAudioDecodeMgr::getInstance()->addDecodeRequest(uuid);
 		}
 	}
 	gAudiop->mCurrentTransfer = LLUUID::null;
@@ -1324,11 +1303,15 @@ void LLAudioSource::update()
 		{
 			// Hack - try and load the sound.  Will do this as a callback
 			// on decode later.
-			if (adp->load() && adp->getBuffer())
+            if (adp->getBuffer())
 			{
 				play(adp->getID());
 			}
-			else if (adp->hasCompletedDecode())		// Only mark corrupted after decode is done
+            else if (adp->hasDecodedData() && !adp->hasWAVLoadFailed())
+            {
+                adp->load();
+            }
+            else if (adp->hasCompletedDecode() && adp->hasDecodeFailed()) // Only mark corrupted after decode is done
 			{
 				LL_WARNS() << "Marking LLAudioSource corrupted for " << adp->getID() << LL_ENDL;
 				mCorrupted = true ;
@@ -1624,12 +1607,12 @@ bool LLAudioSource::hasPendingPreloads() const
 	{
 		LLAudioData *adp = iter->second;
 		// note: a bad UUID will forever be !hasDecodedData()
-		// but also !hasValidData(), hence the check for hasValidData()
+        // but also hasDecodeFailed(), hence the check for hasDecodeFailed()
 		if (!adp)
 		{
 			continue;
 		}
-		if (!adp->hasDecodedData() && adp->hasValidData())
+        if (!adp->hasDecodedData() && !adp->hasDecodeFailed())
 		{
 			// This source is still waiting for a preload
 			return true;
@@ -1786,7 +1769,8 @@ LLAudioData::LLAudioData(const LLUUID &uuid) :
 	mHasLocalData(false),
 	mHasDecodedData(false),
 	mHasCompletedDecode(false),
-	mHasValidData(true)
+    mHasDecodeFailed(false),
+    mHasWAVLoadFailed(false)
 {
 	if (uuid.isNull())
 	{
@@ -1821,12 +1805,14 @@ bool LLAudioData::load()
 	{
 		// We already have this sound in a buffer, don't do anything.
 		LL_INFOS() << "Already have a buffer for this sound, don't bother loading!" << LL_ENDL;
+        mHasWAVLoadFailed = false;
 		return true;
 	}
 
 	if (!gAudiop)
 	{
 		LL_WARNS("AudioEngine") << "LLAudioEngine instance doesn't exist!" << LL_ENDL;
+        mHasWAVLoadFailed = true;
 		return false;
 	}
 	
@@ -1835,6 +1821,8 @@ bool LLAudioData::load()
 	{
 		// No free buffers, abort.
 		LL_INFOS() << "Not able to allocate a new audio buffer, aborting." << LL_ENDL;
+        // *TODO: Mark this failure differently so the audio engine could retry loading this buffer in the future
+        mHasWAVLoadFailed = true;
 		return true;
 	}
 
@@ -1843,7 +1831,8 @@ bool LLAudioData::load()
 	mID.toString(uuid_str);
 	wav_path= gDirUtilp->getExpandedFilename(LL_PATH_CACHE,uuid_str) + ".dsf";
 
-	if (!mBufferp->loadWAV(wav_path))
+    mHasWAVLoadFailed = !mBufferp->loadWAV(wav_path);
+    if (mHasWAVLoadFailed)
 	{
 		// Hrm.  Right now, let's unset the buffer, since it's empty.
 		gAudiop->cleanupBuffer(mBufferp);
diff --git a/indra/llaudio/llaudioengine.h b/indra/llaudio/llaudioengine.h
index b5fd4c27a1..65356ba00e 100644..100755
--- a/indra/llaudio/llaudioengine.h
+++ b/indra/llaudio/llaudioengine.h
@@ -47,8 +47,8 @@ const F32 LL_WIND_UNDERWATER_CENTER_FREQ = 20.f;
 const F32 ATTACHED_OBJECT_TIMEOUT = 5.0f;
 const F32 DEFAULT_MIN_DISTANCE = 2.0f;
 
-#define MAX_CHANNELS 30
-#define MAX_BUFFERS 40	// Some extra for preloading, maybe?
+#define LL_MAX_AUDIO_CHANNELS 30
+#define LL_MAX_AUDIO_BUFFERS 40 // Some extra for preloading, maybe?
 
 class LLAudioSource;
 class LLAudioData;
@@ -88,7 +88,7 @@ public:
 	virtual ~LLAudioEngine();
 
 	// initialization/startup/shutdown
-	virtual bool init(const S32 num_channels, void *userdata, const std::string &app_title);
+	virtual bool init(void *userdata, const std::string &app_title);
 	virtual std::string getDriverName(bool verbose) = 0;
 	virtual void shutdown();
 
@@ -96,7 +96,7 @@ public:
 	//virtual void processQueue(const LLUUID &sound_guid);
 	virtual void setListener(LLVector3 pos,LLVector3 vel,LLVector3 up,LLVector3 at);
 	virtual void updateWind(LLVector3 direction, F32 camera_height_above_water) = 0;
-	virtual void idle(F32 max_decode_time = 0.f);
+	virtual void idle();
 	virtual void updateChannels();
 
 	//
@@ -209,7 +209,6 @@ protected:
 
 	S32 mLastStatus;
 	
-	S32 mNumChannels;
 	bool mEnableWind;
 
 	LLUUID mCurrentTransfer; // Audio file currently being transferred by the system
@@ -224,11 +223,11 @@ protected:
 	source_map mAllSources;
 	data_map mAllData;
 
-	LLAudioChannel *mChannels[MAX_CHANNELS];
+    std::array<LLAudioChannel*, LL_MAX_AUDIO_CHANNELS> mChannels;
 
 	// Buffers needs to change into a different data structure, as the number of buffers
 	// that we have active should be limited by RAM usage, not count.
-	LLAudioBuffer *mBuffers[MAX_BUFFERS];
+    std::array<LLAudioBuffer*, LL_MAX_AUDIO_BUFFERS> mBuffers;
 	
 	F32 mMasterGain;
 	F32 mInternalGain;			// Actual gain set; either mMasterGain or 0 when mMuted is true.
@@ -360,32 +359,36 @@ protected:
 
 class LLAudioData
 {
-public:
-	LLAudioData(const LLUUID &uuid);
-	bool load();
-
-	LLUUID getID() const				{ return mID; }
-	LLAudioBuffer *getBuffer() const	{ return mBufferp; }
-
-	bool	hasLocalData() const		{ return mHasLocalData; }
-	bool	hasDecodedData() const		{ return mHasDecodedData; }
-	bool	hasCompletedDecode() const	{ return mHasCompletedDecode; }
-	bool	hasValidData() const		{ return mHasValidData; }
-
-	void	setHasLocalData(const bool hld)		{ mHasLocalData = hld; }
-	void	setHasDecodedData(const bool hdd)	{ mHasDecodedData = hdd; }
-	void	setHasCompletedDecode(const bool hcd)	{ mHasCompletedDecode = hcd; }
-	void	setHasValidData(const bool hvd)		{ mHasValidData = hvd; }
-
-	friend class LLAudioEngine; // Severe laziness, bad.
-
-protected:
-	LLUUID mID;
-	LLAudioBuffer *mBufferp;	// If this data is being used by the audio system, a pointer to the buffer will be set here.
-	bool mHasLocalData;			// Set true if the sound asset file is available locally
-	bool mHasDecodedData;		// Set true if the sound file has been decoded
-	bool mHasCompletedDecode;	// Set true when the sound is decoded
-	bool mHasValidData;			// Set false if decoding failed, meaning the sound asset is bad
+  public:
+    LLAudioData(const LLUUID &uuid);
+    bool load();
+
+    LLUUID         getID() const { return mID; }
+    LLAudioBuffer *getBuffer() const { return mBufferp; }
+
+    bool hasLocalData() const { return mHasLocalData; }
+    bool hasDecodedData() const { return mHasDecodedData; }
+    bool hasCompletedDecode() const { return mHasCompletedDecode; }
+    bool hasDecodeFailed() const { return mHasDecodeFailed; }
+    bool hasWAVLoadFailed() const { return mHasWAVLoadFailed; }
+
+    void setHasLocalData(const bool hld) { mHasLocalData = hld; }
+    void setHasDecodedData(const bool hdd) { mHasDecodedData = hdd; }
+    void setHasCompletedDecode(const bool hcd) { mHasCompletedDecode = hcd; }
+    void setHasDecodeFailed(const bool hdf) { mHasDecodeFailed = hdf; }
+    void setHasWAVLoadFailed(const bool hwlf) { mHasWAVLoadFailed = hwlf; }
+
+    friend class LLAudioEngine;  // Severe laziness, bad.
+
+  protected:
+    LLUUID         mID;
+    LLAudioBuffer *mBufferp;             // If this data is being used by the audio system, a pointer to the buffer will be set here.
+    bool           mHasLocalData;        // Set true if the encoded sound asset file is available locally
+    bool           mHasDecodedData;      // Set true if the decoded sound file is available on disk
+    bool           mHasCompletedDecode;  // Set true when the sound is decoded
+    bool           mHasDecodeFailed;     // Set true if decoding failed, meaning the sound asset is bad
+    bool mHasWAVLoadFailed;  // Set true if loading the decoded WAV file failed, meaning the sound asset should be decoded instead if
+                             // possible
 };
 
 
diff --git a/indra/llaudio/llaudioengine_fmodstudio.cpp b/indra/llaudio/llaudioengine_fmodstudio.cpp
index b0c87b0208..ce2c7d5551 100644
--- a/indra/llaudio/llaudioengine_fmodstudio.cpp
+++ b/indra/llaudio/llaudioengine_fmodstudio.cpp
@@ -74,7 +74,7 @@ static inline bool Check_FMOD_Error(FMOD_RESULT result, const char *string)
     return true;
 }
 
-bool LLAudioEngine_FMODSTUDIO::init(const S32 num_channels, void* userdata, const std::string &app_title)
+bool LLAudioEngine_FMODSTUDIO::init(void* userdata, const std::string &app_title)
 {
     U32 version;
     FMOD_RESULT result;
@@ -86,7 +86,7 @@ bool LLAudioEngine_FMODSTUDIO::init(const S32 num_channels, void* userdata, cons
         return false;
 
     //will call LLAudioEngine_FMODSTUDIO::allocateListener, which needs a valid mSystem pointer.
-    LLAudioEngine::init(num_channels, userdata, app_title);
+    LLAudioEngine::init(userdata, app_title);
 
     result = mSystem->getVersion(&version);
     Check_FMOD_Error(result, "FMOD::System::getVersion");
@@ -98,7 +98,7 @@ bool LLAudioEngine_FMODSTUDIO::init(const S32 num_channels, void* userdata, cons
     }
 
     // In this case, all sounds, PLUS wind and stream will be software.
-    result = mSystem->setSoftwareChannels(num_channels + 2);
+    result = mSystem->setSoftwareChannels(LL_MAX_AUDIO_CHANNELS + 2);
     Check_FMOD_Error(result, "FMOD::System::setSoftwareChannels");
 
     FMOD_ADVANCEDSETTINGS settings;
@@ -127,7 +127,7 @@ bool LLAudioEngine_FMODSTUDIO::init(const S32 num_channels, void* userdata, cons
         {
             LL_DEBUGS("AppInit") << "Trying PulseAudio audio output..." << LL_ENDL;
             if (mSystem->setOutput(FMOD_OUTPUTTYPE_PULSEAUDIO) == FMOD_OK &&
-                (result = mSystem->init(num_channels + 2, fmod_flags, const_cast<char*>(app_title.c_str()))) == FMOD_OK)
+                (result = mSystem->init(LL_MAX_AUDIO_CHANNELS + 2, fmod_flags, const_cast<char*>(app_title.c_str()))) == FMOD_OK)
             {
                 LL_DEBUGS("AppInit") << "PulseAudio output initialized OKAY" << LL_ENDL;
                 audio_ok = true;
@@ -149,7 +149,7 @@ bool LLAudioEngine_FMODSTUDIO::init(const S32 num_channels, void* userdata, cons
         {
             LL_DEBUGS("AppInit") << "Trying ALSA audio output..." << LL_ENDL;
             if (mSystem->setOutput(FMOD_OUTPUTTYPE_ALSA) == FMOD_OK &&
-                (result = mSystem->init(num_channels + 2, fmod_flags, 0)) == FMOD_OK)
+                (result = mSystem->init(LL_MAX_AUDIO_CHANNELS + 2, fmod_flags, 0)) == FMOD_OK)
             {
                 LL_DEBUGS("AppInit") << "ALSA audio output initialized OKAY" << LL_ENDL;
                 audio_ok = true;
@@ -190,7 +190,7 @@ bool LLAudioEngine_FMODSTUDIO::init(const S32 num_channels, void* userdata, cons
     // initialize the FMOD engine
     // number of channel in this case looks to be identiacal to number of max simultaneously
     // playing objects and we can set practically any number
-    result = mSystem->init(num_channels + 2, fmod_flags, 0);
+    result = mSystem->init(LL_MAX_AUDIO_CHANNELS + 2, fmod_flags, 0);
     if (Check_FMOD_Error(result, "Error initializing FMOD Studio with default settins, retrying with other format"))
     {
         result = mSystem->setSoftwareFormat(44100, FMOD_SPEAKERMODE_STEREO, 0/*- ignore*/);
@@ -198,7 +198,7 @@ bool LLAudioEngine_FMODSTUDIO::init(const S32 num_channels, void* userdata, cons
         {
             return false;
         }
-        result = mSystem->init(num_channels + 2, fmod_flags, 0);
+        result = mSystem->init(LL_MAX_AUDIO_CHANNELS + 2, fmod_flags, 0);
     }
     if (Check_FMOD_Error(result, "Error initializing FMOD Studio"))
     {
diff --git a/indra/llaudio/llaudioengine_fmodstudio.h b/indra/llaudio/llaudioengine_fmodstudio.h
index f2361df1b6..d3d6d69685 100644
--- a/indra/llaudio/llaudioengine_fmodstudio.h
+++ b/indra/llaudio/llaudioengine_fmodstudio.h
@@ -51,7 +51,7 @@ public:
 	virtual ~LLAudioEngine_FMODSTUDIO();
 
 	// initialization/startup/shutdown
-	virtual bool init(const S32 num_channels, void *user_data, const std::string &app_title);
+	virtual bool init(void *user_data, const std::string &app_title);
 	virtual std::string getDriverName(bool verbose);
 	virtual void allocateListener();
 
diff --git a/indra/llaudio/llaudioengine_openal.cpp b/indra/llaudio/llaudioengine_openal.cpp
index 3bdd0302ee..a87b4c07e2 100644
--- a/indra/llaudio/llaudioengine_openal.cpp
+++ b/indra/llaudio/llaudioengine_openal.cpp
@@ -52,10 +52,10 @@ LLAudioEngine_OpenAL::~LLAudioEngine_OpenAL()
 }
 
 // virtual
-bool LLAudioEngine_OpenAL::init(const S32 num_channels, void* userdata, const std::string &app_title)
+bool LLAudioEngine_OpenAL::init(void* userdata, const std::string &app_title)
 {
 	mWindGen = NULL;
-	LLAudioEngine::init(num_channels, userdata, app_title);
+	LLAudioEngine::init(userdata, app_title);
 
 	if(!alutInit(NULL, NULL))
 	{
diff --git a/indra/llcommon/workqueue.h b/indra/llcommon/workqueue.h
index 96574a18b9..70fd65bd0c 100644
--- a/indra/llcommon/workqueue.h
+++ b/indra/llcommon/workqueue.h
@@ -403,7 +403,7 @@ namespace LL
                 [result = std::forward<CALLABLE>(callable)(),
                  callback = std::move(callback)]
                 ()
-                { callback(std::move(result)); };
+                mutable { callback(std::move(result)); };
         }
     };
 
@@ -449,7 +449,7 @@ namespace LL
              callable = std::move(callable),
              callback = std::move(callback)]
             ()
-            {
+            mutable {
                 // Use postMaybe() below in case this originating WorkQueue
                 // has been closed or destroyed. Remember, the outer lambda is
                 // now running on a thread servicing the target WorkQueue, and
@@ -513,7 +513,7 @@ namespace LL
                 // We dare to bind a reference to Promise because it's
                 // specifically designed for cross-thread communication.
                 [&promise, callable = std::move(callable)]()
-                {
+                mutable {
                     try
                     {
                         // call the caller's callable and trigger promise with result
@@ -542,7 +542,7 @@ namespace LL
                 time,
                 // &promise is designed for cross-thread access
                 [&promise, callable = std::move(callable)]()
-                {
+                mutable {
                     try
                     {
                         callable();
diff --git a/indra/newview/llappviewer.cpp b/indra/newview/llappviewer.cpp
index 979a888814..b5ed8d8647 100644
--- a/indra/newview/llappviewer.cpp
+++ b/indra/newview/llappviewer.cpp
@@ -5020,8 +5020,7 @@ void LLAppViewer::idle()
 			audio_update_wind(false);
 
 			// this line actually commits the changes we've made to source positions, etc.
-			const F32 max_audio_decode_time = 0.002f; // 2 ms decode time
-			gAudiop->idle(max_audio_decode_time);
+			gAudiop->idle();
 		}
 	}
 
diff --git a/indra/newview/llstartup.cpp b/indra/newview/llstartup.cpp
index 98b2bc703b..145aba4a71 100644
--- a/indra/newview/llstartup.cpp
+++ b/indra/newview/llstartup.cpp
@@ -661,7 +661,7 @@ bool idle_startup()
 #else
 				void* window_handle = NULL;
 #endif
-				bool init = gAudiop->init(kAUDIO_NUM_SOURCES, window_handle, LLAppViewer::instance()->getSecondLifeTitle());
+				bool init = gAudiop->init(window_handle, LLAppViewer::instance()->getSecondLifeTitle());
 				if(init)
 				{
 					gAudiop->setMuted(TRUE);
diff --git a/indra/newview/llvieweraudio.h b/indra/newview/llvieweraudio.h
index 782285ce36..febae36ae8 100644
--- a/indra/newview/llvieweraudio.h
+++ b/indra/newview/llvieweraudio.h
@@ -33,8 +33,6 @@
 // comment out to turn off wind
 #define kAUDIO_ENABLE_WIND 
 //#define kAUDIO_ENABLE_WATER 1	// comment out to turn off water
-#define kAUDIO_NUM_BUFFERS 30
-#define kAUDIO_NUM_SOURCES 30 
 
 void init_audio();
 void audio_update_volume(bool force_update = true);