From 65b5a6e12d32602ae5b0ed3982dc2c74057c7ab7 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Mon, 5 Feb 2024 14:42:21 -0800 Subject: Use a custom audio processor to pull data for level determinations, which will happen after AGC --- indra/llwebrtc/llwebrtc.cpp | 83 +++++++++++++++++++++++++----------------- indra/llwebrtc/llwebrtc_impl.h | 37 +++++++++---------- 2 files changed, 68 insertions(+), 52 deletions(-) (limited to 'indra') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index d6eff5e1f2..f0d7f6ad8f 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -27,6 +27,7 @@ #include "llwebrtc_impl.h" #include #include +#include #include "api/audio_codecs/audio_decoder_factory.h" #include "api/audio_codecs/audio_encoder_factory.h" @@ -34,30 +35,54 @@ #include "api/audio_codecs/builtin_audio_encoder_factory.h" #include "api/media_stream_interface.h" #include "api/media_stream_track.h" +#include "modules/audio_processing/audio_buffer.h" namespace llwebrtc { -const float VOLUME_SCALE_WEBRTC = 100; - -LLAudioDeviceObserver::LLAudioDeviceObserver() : mMicrophoneEnergy(0.0), mSumVector {0} {} +LLCustomProcessor::LLCustomProcessor() : + mSampleRateHz(0), + mNumChannels(0) +{ + memset(mSumVector, sizeof(mSumVector), 0); +} -float LLAudioDeviceObserver::getMicrophoneEnergy() { return mMicrophoneEnergy; } +void LLCustomProcessor::Initialize(int sample_rate_hz, int num_channels) +{ + mSampleRateHz = sample_rate_hz; + mNumChannels = num_channels; + memset(mSumVector, sizeof(mSumVector), 0); +} -void LLAudioDeviceObserver::OnCaptureData(const void *audio_samples, - const size_t num_samples, - const size_t bytes_per_sample, - const size_t num_channels, - const uint32_t samples_per_sec) +void LLCustomProcessor::Process(webrtc::AudioBuffer *audio_in) { + webrtc::StreamConfig stream_config; + stream_config.set_sample_rate_hz(mSampleRateHz); + stream_config.set_num_channels(mNumChannels); + std::vector frame; + std::vector frame_samples; + + if (audio_in->num_channels() < 1 || audio_in->num_frames() < 480) + { + return; + } + + frame_samples.resize(stream_config.num_samples()); + frame.resize(stream_config.num_channels()); + for (size_t ch = 0; ch < stream_config.num_channels(); ++ch) + { + frame[ch] = &(frame_samples)[ch * stream_config.num_frames()]; + } + + audio_in->CopyTo(stream_config, &frame[0]); + float energy = 0; - const short *samples = (const short *) audio_samples; - for (size_t index = 0; index < num_samples * num_channels; index++) + for (size_t index = 0; index < stream_config.num_samples(); index++) { - float sample = (static_cast(samples[index]) / (float) 32767); + float sample = frame_samples[index]; energy += sample * sample; } - + // smooth it. size_t buffer_size = sizeof(mSumVector) / sizeof(mSumVector[0]); float totalSum = 0; @@ -69,15 +94,7 @@ void LLAudioDeviceObserver::OnCaptureData(const void *audio_samples, } mSumVector[i] = energy; totalSum += energy; - mMicrophoneEnergy = std::sqrt(totalSum / (num_samples * buffer_size)); -} - -void LLAudioDeviceObserver::OnRenderData(const void *audio_samples, - const size_t num_samples, - const size_t bytes_per_sample, - const size_t num_channels, - const uint32_t samples_per_sec) -{ + mMicrophoneEnergy = std::sqrt(totalSum / (stream_config.num_samples() * buffer_size)); } void LLWebRTCImpl::init() @@ -104,11 +121,9 @@ void LLWebRTCImpl::init() mWorkerThread->PostTask( [this]() { - mTuningAudioDeviceObserver = new LLAudioDeviceObserver; - mTuningDeviceModule = - webrtc::CreateAudioDeviceWithDataObserver(webrtc::AudioDeviceModule::AudioLayer::kPlatformDefaultAudio, - mTaskQueueFactory.get(), - std::unique_ptr(mTuningAudioDeviceObserver)); + mTuningDeviceModule = webrtc::CreateAudioDeviceWithDataObserver( + webrtc::AudioDeviceModule::AudioLayer::kPlatformDefaultAudio, + mTaskQueueFactory.get(), nullptr); mTuningDeviceModule->Init(); mTuningDeviceModule->SetStereoRecording(true); mTuningDeviceModule->SetStereoPlayout(true); @@ -134,11 +149,9 @@ void LLWebRTCImpl::init() mWorkerThread->BlockingCall( [this]() { - mPeerAudioDeviceObserver = new LLAudioDeviceObserver; mPeerDeviceModule = webrtc::CreateAudioDeviceWithDataObserver(webrtc::AudioDeviceModule::AudioLayer::kPlatformDefaultAudio, - mTaskQueueFactory.get(), - std::unique_ptr(mPeerAudioDeviceObserver)); + mTaskQueueFactory.get(), nullptr); mPeerDeviceModule->Init(); mPeerDeviceModule->SetPlayoutDevice(mPlayoutDevice); mPeerDeviceModule->SetRecordingDevice(mRecordingDevice); @@ -151,7 +164,11 @@ void LLWebRTCImpl::init() mPeerDeviceModule->InitPlayout(); }); - rtc::scoped_refptr apm = webrtc::AudioProcessingBuilder().Create(); + mCustomProcessor = new LLCustomProcessor; + webrtc::AudioProcessingBuilder apb; + apb.SetCapturePostProcessing(std::unique_ptr(mCustomProcessor)); + rtc::scoped_refptr apm = apb.Create(); + webrtc::AudioProcessing::Config apm_config; apm_config.echo_canceller.enabled = true; apm_config.echo_canceller.mobile_mode = false; @@ -454,9 +471,9 @@ void LLWebRTCImpl::setTuningMode(bool enable) } } -float LLWebRTCImpl::getTuningAudioLevel() { return -20 * log10f(mTuningAudioDeviceObserver->getMicrophoneEnergy()); } +float LLWebRTCImpl::getTuningAudioLevel() { return -20 * log10f(mCustomProcessor->getMicrophoneEnergy()); } -float LLWebRTCImpl::getPeerAudioLevel() { return -20 * log10f(mPeerAudioDeviceObserver->getMicrophoneEnergy()); } +float LLWebRTCImpl::getPeerAudioLevel() { return -20 * log10f(mCustomProcessor->getMicrophoneEnergy()); } // // Helpers diff --git a/indra/llwebrtc/llwebrtc_impl.h b/indra/llwebrtc/llwebrtc_impl.h index 3ccb6058ad..419482b751 100644 --- a/indra/llwebrtc/llwebrtc_impl.h +++ b/indra/llwebrtc/llwebrtc_impl.h @@ -65,27 +65,27 @@ namespace llwebrtc class LLWebRTCPeerConnectionImpl; -class LLAudioDeviceObserver : public webrtc::AudioDeviceDataObserver +class LLCustomProcessor : public webrtc::CustomProcessing { public: - LLAudioDeviceObserver(); + LLCustomProcessor(); + ~LLCustomProcessor() override {} - float getMicrophoneEnergy(); + // (Re-) Initializes the submodule. + void Initialize(int sample_rate_hz, int num_channels) override; + // Analyzes the given capture or render signal. + void Process(webrtc::AudioBuffer *audio) override; + // Returns a string representation of the module state. + std::string ToString() const override { return ""; } - void OnCaptureData(const void *audio_samples, - const size_t num_samples, - const size_t bytes_per_sample, - const size_t num_channels, - const uint32_t samples_per_sec) override; - - void OnRenderData(const void *audio_samples, - const size_t num_samples, - const size_t bytes_per_sample, - const size_t num_channels, - const uint32_t samples_per_sec) override; + float getMicrophoneEnergy() { return mMicrophoneEnergy; } protected: - float mSumVector[30]; // 300 ms of smoothing + static const int NUM_PACKETS_TO_FILTER = 30; // 300 ms of smoothing + int mSampleRateHz; + int mNumChannels; + + float mSumVector[NUM_PACKETS_TO_FILTER]; float mMicrophoneEnergy; }; @@ -93,7 +93,7 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, public webrtc::AudioDeviceS { public: LLWebRTCImpl() : - mTuningAudioDeviceObserver(nullptr), mPeerAudioDeviceObserver(nullptr), mMute(true) + mCustomProcessor(nullptr), mMute(true) { } ~LLWebRTCImpl() {} @@ -189,9 +189,8 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, public webrtc::AudioDeviceS int32_t mPlayoutDevice; int32_t mRecordingDevice; bool mMute; - - LLAudioDeviceObserver * mTuningAudioDeviceObserver; - LLAudioDeviceObserver * mPeerAudioDeviceObserver; + + LLCustomProcessor * mCustomProcessor; // peer connections std::vector> mPeerConnections; -- cgit v1.2.3