Use a custom audio processor to pull data for level determinations, which will happen after AGC

2 files changed, 68 insertions, 52 deletions

diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp
index d6eff5e1f2..f0d7f6ad8f 100644
--- a/indra/llwebrtc/llwebrtc.cpp
+++ b/indra/llwebrtc/llwebrtc.cpp
@@ -27,6 +27,7 @@
 #include "llwebrtc_impl.h"
 #include <algorithm>
 #include <format>
+#include <string.h>
 
 #include "api/audio_codecs/audio_decoder_factory.h"
 #include "api/audio_codecs/audio_encoder_factory.h"
@@ -34,30 +35,54 @@
 #include "api/audio_codecs/builtin_audio_encoder_factory.h"
 #include "api/media_stream_interface.h"
 #include "api/media_stream_track.h"
+#include "modules/audio_processing/audio_buffer.h"
 
 namespace llwebrtc
 {
 
-const float VOLUME_SCALE_WEBRTC = 100;
-
-LLAudioDeviceObserver::LLAudioDeviceObserver() : mMicrophoneEnergy(0.0), mSumVector {0} {}
+LLCustomProcessor::LLCustomProcessor() :
+    mSampleRateHz(0), 
+    mNumChannels(0) 
+{ 
+    memset(mSumVector, sizeof(mSumVector), 0); 
+}
 
-float LLAudioDeviceObserver::getMicrophoneEnergy() { return mMicrophoneEnergy; }
+void LLCustomProcessor::Initialize(int sample_rate_hz, int num_channels)
+{ 
+    mSampleRateHz = sample_rate_hz;
+    mNumChannels = num_channels;
+    memset(mSumVector, sizeof(mSumVector), 0);
+}
 
-void LLAudioDeviceObserver::OnCaptureData(const void    *audio_samples,
-                                          const size_t   num_samples,
-                                          const size_t   bytes_per_sample,
-                                          const size_t   num_channels,
-                                          const uint32_t samples_per_sec)
+void LLCustomProcessor::Process(webrtc::AudioBuffer *audio_in)
 {
+    webrtc::StreamConfig stream_config;
+    stream_config.set_sample_rate_hz(mSampleRateHz);
+    stream_config.set_num_channels(mNumChannels);
+    std::vector<float *> frame;
+    std::vector<float> frame_samples;
+
+    if (audio_in->num_channels() < 1 || audio_in->num_frames() < 480)
+    {
+        return;
+    }
+
+    frame_samples.resize(stream_config.num_samples());
+    frame.resize(stream_config.num_channels());
+    for (size_t ch = 0; ch < stream_config.num_channels(); ++ch)
+    {
+        frame[ch] = &(frame_samples)[ch * stream_config.num_frames()];
+    }
+
+    audio_in->CopyTo(stream_config, &frame[0]);
+
     float        energy  = 0;
-    const short *samples = (const short *) audio_samples;
-    for (size_t index = 0; index < num_samples * num_channels; index++)
+    for (size_t index = 0; index < stream_config.num_samples(); index++)
     {
-        float sample = (static_cast<float>(samples[index]) / (float) 32767);
+        float sample = frame_samples[index];
         energy += sample * sample;
     }
-    
+
     // smooth it.
     size_t buffer_size = sizeof(mSumVector) / sizeof(mSumVector[0]);
     float  totalSum    = 0;
@@ -69,15 +94,7 @@ void LLAudioDeviceObserver::OnCaptureData(const void    *audio_samples,
     }
     mSumVector[i] = energy;
     totalSum += energy;
-    mMicrophoneEnergy = std::sqrt(totalSum / (num_samples * buffer_size));
-}
-
-void LLAudioDeviceObserver::OnRenderData(const void    *audio_samples,
-                                         const size_t   num_samples,
-                                         const size_t   bytes_per_sample,
-                                         const size_t   num_channels,
-                                         const uint32_t samples_per_sec)
-{
+    mMicrophoneEnergy = std::sqrt(totalSum / (stream_config.num_samples() * buffer_size));
 }
 
 void LLWebRTCImpl::init()
@@ -104,11 +121,9 @@ void LLWebRTCImpl::init()
     mWorkerThread->PostTask(
                             [this]()
                             {
-                                mTuningAudioDeviceObserver = new LLAudioDeviceObserver;
-                                mTuningDeviceModule =
-                                webrtc::CreateAudioDeviceWithDataObserver(webrtc::AudioDeviceModule::AudioLayer::kPlatformDefaultAudio,
-                                                                          mTaskQueueFactory.get(),
-                                                                          std::unique_ptr<webrtc::AudioDeviceDataObserver>(mTuningAudioDeviceObserver));
+                                 mTuningDeviceModule =  webrtc::CreateAudioDeviceWithDataObserver(
+                                    webrtc::AudioDeviceModule::AudioLayer::kPlatformDefaultAudio,
+                                                                          mTaskQueueFactory.get(), nullptr);
                                 mTuningDeviceModule->Init();
                                 mTuningDeviceModule->SetStereoRecording(true);
                                 mTuningDeviceModule->SetStereoPlayout(true);
@@ -134,11 +149,9 @@ void LLWebRTCImpl::init()
     mWorkerThread->BlockingCall(
                                 [this]()
                                 {
-                                    mPeerAudioDeviceObserver = new LLAudioDeviceObserver;
                                     mPeerDeviceModule =
                                     webrtc::CreateAudioDeviceWithDataObserver(webrtc::AudioDeviceModule::AudioLayer::kPlatformDefaultAudio,
-                                                                              mTaskQueueFactory.get(),
-                                                                              std::unique_ptr<webrtc::AudioDeviceDataObserver>(mPeerAudioDeviceObserver));
+                                                                              mTaskQueueFactory.get(), nullptr);
                                     mPeerDeviceModule->Init();
                                     mPeerDeviceModule->SetPlayoutDevice(mPlayoutDevice);
                                     mPeerDeviceModule->SetRecordingDevice(mRecordingDevice);
@@ -151,7 +164,11 @@ void LLWebRTCImpl::init()
                                     mPeerDeviceModule->InitPlayout();
                                 });
 
-    rtc::scoped_refptr<webrtc::AudioProcessing> apm = webrtc::AudioProcessingBuilder().Create();
+    mCustomProcessor = new LLCustomProcessor;
+    webrtc::AudioProcessingBuilder apb;
+    apb.SetCapturePostProcessing(std::unique_ptr<webrtc::CustomProcessing>(mCustomProcessor));
+    rtc::scoped_refptr<webrtc::AudioProcessing> apm = apb.Create();
+
     webrtc::AudioProcessing::Config             apm_config;
     apm_config.echo_canceller.enabled         = true;
     apm_config.echo_canceller.mobile_mode     = false;
@@ -454,9 +471,9 @@ void LLWebRTCImpl::setTuningMode(bool enable)
     }
 }
 
-float LLWebRTCImpl::getTuningAudioLevel() { return -20 * log10f(mTuningAudioDeviceObserver->getMicrophoneEnergy()); }
+float LLWebRTCImpl::getTuningAudioLevel() { return -20 * log10f(mCustomProcessor->getMicrophoneEnergy()); }
 
-float LLWebRTCImpl::getPeerAudioLevel() { return -20 * log10f(mPeerAudioDeviceObserver->getMicrophoneEnergy()); }
+float LLWebRTCImpl::getPeerAudioLevel() { return -20 * log10f(mCustomProcessor->getMicrophoneEnergy()); }
 
 //
 // Helpers
diff --git a/indra/llwebrtc/llwebrtc_impl.h b/indra/llwebrtc/llwebrtc_impl.h
index 3ccb6058ad..419482b751 100644
--- a/indra/llwebrtc/llwebrtc_impl.h
+++ b/indra/llwebrtc/llwebrtc_impl.h
@@ -65,27 +65,27 @@ namespace llwebrtc
 
 class LLWebRTCPeerConnectionImpl;
 
-class LLAudioDeviceObserver : public webrtc::AudioDeviceDataObserver
+class LLCustomProcessor : public webrtc::CustomProcessing
 {
   public:
-    LLAudioDeviceObserver();
+    LLCustomProcessor();
+    ~LLCustomProcessor() override {}
 
-    float getMicrophoneEnergy();
+    // (Re-) Initializes the submodule.
+    void Initialize(int sample_rate_hz, int num_channels) override;
+    // Analyzes the given capture or render signal.
+    void Process(webrtc::AudioBuffer *audio) override;
+    // Returns a string representation of the module state.
+    std::string ToString() const override { return ""; }
 
-    void OnCaptureData(const void    *audio_samples,
-                       const size_t   num_samples,
-                       const size_t   bytes_per_sample,
-                       const size_t   num_channels,
-                       const uint32_t samples_per_sec) override;
-
-    void OnRenderData(const void    *audio_samples,
-                      const size_t   num_samples,
-                      const size_t   bytes_per_sample,
-                      const size_t   num_channels,
-                      const uint32_t samples_per_sec) override;
+    float getMicrophoneEnergy() { return mMicrophoneEnergy; }
 
   protected:
-    float mSumVector[30];  // 300 ms of smoothing
+    static const int NUM_PACKETS_TO_FILTER = 30;  // 300 ms of smoothing
+    int              mSampleRateHz;
+    int              mNumChannels;
+
+    float mSumVector[NUM_PACKETS_TO_FILTER];
     float mMicrophoneEnergy;
 };
 
@@ -93,7 +93,7 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, public webrtc::AudioDeviceS
 {
   public:
     LLWebRTCImpl() : 
-        mTuningAudioDeviceObserver(nullptr), mPeerAudioDeviceObserver(nullptr), mMute(true)
+        mCustomProcessor(nullptr), mMute(true)
     {
     }
     ~LLWebRTCImpl() {}
@@ -189,9 +189,8 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, public webrtc::AudioDeviceS
     int32_t                                                    mPlayoutDevice;
     int32_t                                                    mRecordingDevice;
     bool                                                       mMute;
-
-    LLAudioDeviceObserver *                                    mTuningAudioDeviceObserver;
-    LLAudioDeviceObserver *                                    mPeerAudioDeviceObserver;
+    
+    LLCustomProcessor *                                        mCustomProcessor;
     
     // peer connections
     std::vector<rtc::scoped_refptr<LLWebRTCPeerConnectionImpl>>     mPeerConnections;