From 8d14df5984382de0aea12ba5d8ef4eba22b9976e Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Sun, 24 Mar 2024 20:42:32 -0700 Subject: Not sending proper voice server type down when breaking a connection --- indra/newview/llvoicewebrtc.cpp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) (limited to 'indra') diff --git a/indra/newview/llvoicewebrtc.cpp b/indra/newview/llvoicewebrtc.cpp index 6aab67ca63..a8bc468f4c 100644 --- a/indra/newview/llvoicewebrtc.cpp +++ b/indra/newview/llvoicewebrtc.cpp @@ -2363,7 +2363,7 @@ void LLVoiceWebRTCConnection::breakVoiceConnection() LLSD body; body["logout"] = TRUE; body["viewer_session"] = mViewerSession; - body["voice_server_type"] = REPORTED_VOICE_SERVER_TYPE; + body["voice_server_type"] = WEBRTC_VOICE_SERVER_TYPE; LLCoreHttpUtil::HttpCoroutineAdapter::ptr_t httpAdapter( new LLCoreHttpUtil::HttpCoroutineAdapter("LLVoiceWebRTCAdHocConnection::breakVoiceConnection", @@ -2419,9 +2419,7 @@ void LLVoiceWebRTCSpatialConnection::requestVoiceConnection() LLSD body; LLSD jsep; jsep["type"] = "offer"; - { - jsep["sdp"] = mChannelSDP; - } + jsep["sdp"] = mChannelSDP; body["jsep"] = jsep; if (mParcelLocalID != INVALID_PARCEL_ID) { -- cgit v1.2.3 From cdae5ebc168d95a304b9905de7b66381723e402f Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Sat, 30 Mar 2024 21:58:00 -0700 Subject: Add UI for managing echo cancellation, AGC, and noise control. Plumb audio settings through from webrtc to the sound preferences UI (still needs some tweaking, of course.) Also, choose stun servers based on grid. Ultimately, the stun stun servers will be passed up via login or something. --- indra/llwebrtc/llwebrtc.cpp | 98 ++++++++++++++++------ indra/llwebrtc/llwebrtc.h | 46 +++++++++- indra/llwebrtc/llwebrtc_impl.h | 8 +- indra/newview/app_settings/settings.xml | 33 ++++++++ indra/newview/llviewercontrol.cpp | 3 + indra/newview/llvoicevivox.cpp | 4 +- indra/newview/llvoicewebrtc.cpp | 35 +++++++- indra/newview/llvoicewebrtc.h | 2 +- .../default/xui/en/panel_preferences_sound.xml | 69 +++++++++++++-- 9 files changed, 256 insertions(+), 42 deletions(-) (limited to 'indra') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index 34d950b804..c51bcfcdd5 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -157,7 +157,6 @@ LLWebRTCImpl::LLWebRTCImpl() : void LLWebRTCImpl::init() { - RTC_DCHECK(mPeerConnectionFactory); mPlayoutDevice = 0; mRecordingDevice = 0; rtc::InitializeSSL(); @@ -222,12 +221,10 @@ void LLWebRTCImpl::init() mPeerCustomProcessor = new LLCustomProcessor; webrtc::AudioProcessingBuilder apb; apb.SetCapturePostProcessing(std::unique_ptr(mPeerCustomProcessor)); - rtc::scoped_refptr apm = apb.Create(); + mAudioProcessingModule = apb.Create(); - // TODO: wire some of these to the primary interface and ultimately - // to the UI to allow user config. webrtc::AudioProcessing::Config apm_config; - apm_config.echo_canceller.enabled = true; + apm_config.echo_canceller.enabled = false; apm_config.echo_canceller.mobile_mode = false; apm_config.gain_controller1.enabled = true; apm_config.gain_controller1.mode = webrtc::AudioProcessing::Config::GainController1::kAdaptiveAnalog; @@ -250,8 +247,8 @@ void LLWebRTCImpl::init() processing_config.reverse_output_stream().set_num_channels(2); processing_config.reverse_output_stream().set_sample_rate_hz(48000); - apm->Initialize(processing_config); - apm->ApplyConfig(apm_config); + mAudioProcessingModule->Initialize(processing_config); + mAudioProcessingModule->ApplyConfig(apm_config); mPeerConnectionFactory = webrtc::CreatePeerConnectionFactory(mNetworkThread.get(), mWorkerThread.get(), @@ -262,7 +259,7 @@ void LLWebRTCImpl::init() nullptr /* video_encoder_factory */, nullptr /* video_decoder_factory */, nullptr /* audio_mixer */, - apm); + mAudioProcessingModule); mWorkerThread->BlockingCall([this]() { mPeerDeviceModule->StartPlayout(); }); } @@ -318,6 +315,49 @@ void LLWebRTCImpl::setRecording(bool recording) }); } +void LLWebRTCImpl::setAudioConfig(LLWebRTCDeviceInterface::AudioConfig config) +{ + webrtc::AudioProcessing::Config apm_config; + apm_config.echo_canceller.enabled = config.mEchoCancellation; + apm_config.echo_canceller.mobile_mode = false; + apm_config.gain_controller1.enabled = true; + apm_config.gain_controller1.mode = webrtc::AudioProcessing::Config::GainController1::kAdaptiveAnalog; + apm_config.gain_controller2.enabled = true; + apm_config.high_pass_filter.enabled = true; + apm_config.transient_suppression.enabled = true; + apm_config.pipeline.multi_channel_render = true; + apm_config.pipeline.multi_channel_capture = true; + apm_config.pipeline.multi_channel_capture = true; + + switch (config.mNoiseSuppressionLevel) + { + case LLWebRTCDeviceInterface::AudioConfig::NOISE_SUPPRESSION_LEVEL_NONE: + apm_config.noise_suppression.enabled = false; + apm_config.noise_suppression.level = webrtc::AudioProcessing::Config::NoiseSuppression::kLow; + break; + case LLWebRTCDeviceInterface::AudioConfig::NOISE_SUPPRESSION_LEVEL_LOW: + apm_config.noise_suppression.enabled = true; + apm_config.noise_suppression.level = webrtc::AudioProcessing::Config::NoiseSuppression::kLow; + break; + case LLWebRTCDeviceInterface::AudioConfig::NOISE_SUPPRESSION_LEVEL_MODERATE: + apm_config.noise_suppression.enabled = true; + apm_config.noise_suppression.level = webrtc::AudioProcessing::Config::NoiseSuppression::kModerate; + break; + case LLWebRTCDeviceInterface::AudioConfig::NOISE_SUPPRESSION_LEVEL_HIGH: + apm_config.noise_suppression.enabled = true; + apm_config.noise_suppression.level = webrtc::AudioProcessing::Config::NoiseSuppression::kHigh; + break; + case LLWebRTCDeviceInterface::AudioConfig::NOISE_SUPPRESSION_LEVEL_VERY_HIGH: + apm_config.noise_suppression.enabled = true; + apm_config.noise_suppression.level = webrtc::AudioProcessing::Config::NoiseSuppression::kVeryHigh; + break; + default: + apm_config.noise_suppression.enabled = false; + apm_config.noise_suppression.level = webrtc::AudioProcessing::Config::NoiseSuppression::kLow; + } + mAudioProcessingModule->ApplyConfig(apm_config); +} + void LLWebRTCImpl::refreshDevices() { mWorkerThread->PostTask([this]() { updateDevices(); }); @@ -616,32 +656,36 @@ void LLWebRTCPeerConnectionImpl::unsetSignalingObserver(LLWebRTCSignalingObserve } } -// TODO: Add initialization structure through which -// stun and turn servers may be passed in from -// the sim or login. -bool LLWebRTCPeerConnectionImpl::initializeConnection() +bool LLWebRTCPeerConnectionImpl::initializeConnection(LLWebRTCPeerConnectionInterface::InitOptions options) { RTC_DCHECK(!mPeerConnection); mAnswerReceived = false; mWebRTCImpl->PostSignalingTask( - [this]() + [this, options]() { + std::vector servers = options.mServers; + if(servers.empty()) + { + LLWebRTCPeerConnectionInterface::InitOptions::IceServers ice_servers; + ice_servers.mUrls.push_back("stun:stun.l.google.com:19302"); + ice_servers.mUrls.push_back("stun1:stun.l.google.com:19302"); + ice_servers.mUrls.push_back("stun2:stun.l.google.com:19302"); + ice_servers.mUrls.push_back("stun3:stun.l.google.com:19302"); + ice_servers.mUrls.push_back("stun4:stun.l.google.com:19302"); + } + webrtc::PeerConnectionInterface::RTCConfiguration config; + for (auto server : servers) + { + webrtc::PeerConnectionInterface::IceServer ice_server; + ice_server.urls = server.mUrls; + ice_server.username = server.mUserName; + ice_server.password = server.mPassword; + config.servers.push_back(ice_server); + } + config.sdp_semantics = webrtc::SdpSemantics::kUnifiedPlan; - webrtc::PeerConnectionInterface::IceServer server; - server.uri = "stun:roxie-turn.staging.secondlife.io:3478"; - config.servers.push_back(server); - server.uri = "stun:stun.l.google.com:19302"; - config.servers.push_back(server); - server.uri = "stun:stun1.l.google.com:19302"; - config.servers.push_back(server); - server.uri = "stun:stun2.l.google.com:19302"; - config.servers.push_back(server); - server.uri = "stun:stun3.l.google.com:19302"; - config.servers.push_back(server); - server.uri = "stun:stun4.l.google.com:19302"; - config.servers.push_back(server); config.set_min_port(60000); config.set_max_port(60100); @@ -671,7 +715,7 @@ bool LLWebRTCPeerConnectionImpl::initializeConnection() cricket::AudioOptions audioOptions; audioOptions.auto_gain_control = true; - audioOptions.echo_cancellation = true; // incompatible with opus stereo + audioOptions.echo_cancellation = false; // incompatible with opus stereo audioOptions.noise_suppression = true; mLocalStream = mPeerConnectionFactory->CreateLocalMediaStream("SLStream"); diff --git a/indra/llwebrtc/llwebrtc.h b/indra/llwebrtc/llwebrtc.h index be2e5cdf68..f3a33435c9 100644 --- a/indra/llwebrtc/llwebrtc.h +++ b/indra/llwebrtc/llwebrtc.h @@ -99,10 +99,32 @@ class LLWebRTCDevicesObserver // to enumerate, set, and get notifications of changes // for both capture (microphone) and render (speaker) // devices. + class LLWebRTCDeviceInterface { public: - + struct AudioConfig { + + bool mAGC { true }; + + bool mEchoCancellation { true }; + + // TODO: The various levels of noise suppression are configured + // on the APM which would require setting config on the APM. + // We should pipe the various values through + // later. + typedef enum { + NOISE_SUPPRESSION_LEVEL_NONE = 0, + NOISE_SUPPRESSION_LEVEL_LOW, + NOISE_SUPPRESSION_LEVEL_MODERATE, + NOISE_SUPPRESSION_LEVEL_HIGH, + NOISE_SUPPRESSION_LEVEL_VERY_HIGH + } ENoiseSuppressionLevel; + ENoiseSuppressionLevel mNoiseSuppressionLevel { NOISE_SUPPRESSION_LEVEL_VERY_HIGH }; + }; + + virtual void setAudioConfig(AudioConfig config) = 0; + // instructs webrtc to refresh the device list. virtual void refreshDevices() = 0; @@ -194,19 +216,35 @@ class LLWebRTCSignalingObserver virtual void OnDataChannelReady(LLWebRTCDataInterface *data_interface) = 0; }; - // LLWebRTCPeerConnectionInterface representsd a connection to a peer, // in most cases a Secondlife WebRTC server. This interface // allows for management of this peer connection. class LLWebRTCPeerConnectionInterface { public: + + struct InitOptions + { + // equivalent of PeerConnectionInterface::IceServer + struct IceServers { + // Valid formats are described in RFC7064 and RFC7065. + // Urls should containe dns hostnames (not IP addresses) + // as the TLS certificate policy is 'secure.' + // and we do not currentply support TLS extensions. + std::vector mUrls; + std::string mUserName; + std::string mPassword; + }; + + std::vector mServers; + }; + + virtual bool initializeConnection(InitOptions options = InitOptions()) = 0; + virtual bool shutdownConnection() = 0; virtual void setSignalingObserver(LLWebRTCSignalingObserver* observer) = 0; virtual void unsetSignalingObserver(LLWebRTCSignalingObserver* observer) = 0; - virtual bool initializeConnection() = 0; - virtual bool shutdownConnection() = 0; virtual void AnswerAvailable(const std::string &sdp) = 0; }; diff --git a/indra/llwebrtc/llwebrtc_impl.h b/indra/llwebrtc/llwebrtc_impl.h index 38810a29b5..32faf2516c 100644 --- a/indra/llwebrtc/llwebrtc_impl.h +++ b/indra/llwebrtc/llwebrtc_impl.h @@ -145,6 +145,8 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, public webrtc::AudioDeviceS // LLWebRTCDeviceInterface // + void setAudioConfig(LLWebRTCDeviceInterface::AudioConfig config = LLWebRTCDeviceInterface::AudioConfig()) override; + void refreshDevices() override; void setDevicesObserver(LLWebRTCDevicesObserver *observer) override; @@ -227,6 +229,8 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, public webrtc::AudioDeviceS // The factory that allows creation of native webrtc PeerConnections. rtc::scoped_refptr mPeerConnectionFactory; + + rtc::scoped_refptr mAudioProcessingModule; // more native webrtc stuff std::unique_ptr mTaskQueueFactory; @@ -278,11 +282,11 @@ class LLWebRTCPeerConnectionImpl : public LLWebRTCPeerConnectionInterface, // // LLWebRTCPeerConnection // + bool initializeConnection(InitOptions options = InitOptions()) override; + bool shutdownConnection() override; void setSignalingObserver(LLWebRTCSignalingObserver *observer) override; void unsetSignalingObserver(LLWebRTCSignalingObserver *observer) override; - bool initializeConnection() override; - bool shutdownConnection() override; void AnswerAvailable(const std::string &sdp) override; // diff --git a/indra/newview/app_settings/settings.xml b/indra/newview/app_settings/settings.xml index 5f7d1d8a21..0de6db0d65 100644 --- a/indra/newview/app_settings/settings.xml +++ b/indra/newview/app_settings/settings.xml @@ -15108,6 +15108,39 @@ Value 44125 + VoiceEchoCancellation + + Comment + Voice Echo Cancellation + Persist + 1 + Type + Boolean + Value + 1 + + VoiceAutomaticGainControl + + Comment + Voice Automatic Gain Control + Persist + 1 + Type + Boolean + Value + 1 + + VoiceNoiseSuppressionLevel + + Comment + Voice Noise Suppression Level + Persist + 1 + Type + U32 + Value + 4 + WarningsAsChat Comment diff --git a/indra/newview/llviewercontrol.cpp b/indra/newview/llviewercontrol.cpp index 7738cb904e..44173a8043 100644 --- a/indra/newview/llviewercontrol.cpp +++ b/indra/newview/llviewercontrol.cpp @@ -803,6 +803,9 @@ void settings_setup_listeners() setting_setup_signal_listener(gSavedSettings, "PushToTalkButton", handleVoiceClientPrefsChanged); setting_setup_signal_listener(gSavedSettings, "PushToTalkToggle", handleVoiceClientPrefsChanged); setting_setup_signal_listener(gSavedSettings, "VoiceEarLocation", handleVoiceClientPrefsChanged); + setting_setup_signal_listener(gSavedSettings, "VoiceEchoCancellation", handleVoiceClientPrefsChanged); + setting_setup_signal_listener(gSavedSettings, "VoiceAutomaticGainControl", handleVoiceClientPrefsChanged); + setting_setup_signal_listener(gSavedSettings, "VoiceNoiseSuppressionLevel", handleVoiceClientPrefsChanged); setting_setup_signal_listener(gSavedSettings, "VoiceInputAudioDevice", handleVoiceClientPrefsChanged); setting_setup_signal_listener(gSavedSettings, "VoiceOutputAudioDevice", handleVoiceClientPrefsChanged); setting_setup_signal_listener(gSavedSettings, "AudioLevelMic", handleVoiceClientPrefsChanged); diff --git a/indra/newview/llvoicevivox.cpp b/indra/newview/llvoicevivox.cpp index 29aba8ecba..f5c9e3aa98 100644 --- a/indra/newview/llvoicevivox.cpp +++ b/indra/newview/llvoicevivox.cpp @@ -1154,7 +1154,9 @@ bool LLVivoxVoiceClient::provisionVoiceAccount() do { LLVoiceVivoxStats::getInstance()->provisionAttemptStart(); - result = httpAdapter->postAndSuspend(httpRequest, url, LLSD(), httpOpts); + LLSD body; + body["voice_server_type"] = "vivox"; + result = httpAdapter->postAndSuspend(httpRequest, url, body, httpOpts); if (sShuttingDown) { diff --git a/indra/newview/llvoicewebrtc.cpp b/indra/newview/llvoicewebrtc.cpp index a8bc468f4c..bb5b386e7c 100644 --- a/indra/newview/llvoicewebrtc.cpp +++ b/indra/newview/llvoicewebrtc.cpp @@ -24,6 +24,7 @@ * $/LicenseInfo$ */ #include +#include #include "llvoicewebrtc.h" #include "llsdutil.h" @@ -279,7 +280,7 @@ void LLWebRTCVoiceClient::cleanUp() LL_DEBUGS("Voice") << "Exiting" << LL_ENDL; } -//--------------------------------------------------- +// -------------------------------------------------- const LLVoiceVersionInfo& LLWebRTCVoiceClient::getVersion() { @@ -299,6 +300,13 @@ void LLWebRTCVoiceClient::updateSettings() setRenderDevice(outputDevice); F32 mic_level = gSavedSettings.getF32("AudioLevelMic"); setMicGain(mic_level); + + llwebrtc::LLWebRTCDeviceInterface::AudioConfig config; + config.mEchoCancellation = gSavedSettings.getBOOL("VoiceEchoCancellation"); + config.mAGC = gSavedSettings.getBOOL("VoiceAutomaticGainControl"); + config.mNoiseSuppressionLevel = (llwebrtc::LLWebRTCDeviceInterface::AudioConfig::ENoiseSuppressionLevel)gSavedSettings.getU32("VoiceNoiseSuppressionLevel"); + mWebRTCDeviceInterface->setAudioConfig(config); + } // Observers @@ -2480,6 +2488,28 @@ void LLVoiceWebRTCConnection::OnVoiceConnectionRequestSuccess(const LLSD &result mWebRTCPeerConnectionInterface->AnswerAvailable(mRemoteChannelSDP); } +static llwebrtc::LLWebRTCPeerConnectionInterface::InitOptions getConnectionOptions() +{ + llwebrtc::LLWebRTCPeerConnectionInterface::InitOptions options; + llwebrtc::LLWebRTCPeerConnectionInterface::InitOptions::IceServers servers; + + // TODO: Pull these from login + std::string grid = LLGridManager::getInstance()->getGridLoginID(); + std::transform(grid.begin(), grid.end(), grid.begin(), [](unsigned char c){ return std::tolower(c); }); + int num_servers = 2; + if (grid == "agni") + { + num_servers = 3; + } + for (int i=1; i <= num_servers; i++) + { + servers.mUrls.push_back(llformat("stun:stun%d.%s.secondlife.io:3478", i, grid.c_str())); + } + options.mServers.push_back(servers); + return options; +} + + // Primary state machine for negotiating a single voice connection to the // Secondlife WebRTC server. bool LLVoiceWebRTCConnection::connectionStateMachine() @@ -2498,10 +2528,11 @@ bool LLVoiceWebRTCConnection::connectionStateMachine() mTrickling = false; mIceCompleted = false; setVoiceConnectionState(VOICE_STATE_WAIT_FOR_SESSION_START); + // tell the webrtc library that we want a connection. The library will // respond with an offer on a separate thread, which will cause // the session state to change. - if (!mWebRTCPeerConnectionInterface->initializeConnection()) + if (!mWebRTCPeerConnectionInterface->initializeConnection(getConnectionOptions())) { setVoiceConnectionState(VOICE_STATE_SESSION_RETRY); } diff --git a/indra/newview/llvoicewebrtc.h b/indra/newview/llvoicewebrtc.h index 21fc79420b..96d39f3928 100644 --- a/indra/newview/llvoicewebrtc.h +++ b/indra/newview/llvoicewebrtc.h @@ -272,7 +272,7 @@ public: participantStatePtr_t findParticipantByID(const LLUUID& id); static ptr_t matchSessionByChannelID(const std::string& channel_id); - + void shutdownAllConnections(); void revive(); diff --git a/indra/newview/skins/default/xui/en/panel_preferences_sound.xml b/indra/newview/skins/default/xui/en/panel_preferences_sound.xml index ab2e9c72f3..3dba5d060b 100644 --- a/indra/newview/skins/default/xui/en/panel_preferences_sound.xml +++ b/indra/newview/skins/default/xui/en/panel_preferences_sound.xml @@ -340,7 +340,7 @@ follows="left|top" top_delta="-6" layout="topleft" - left_pad="20" + left_pad="10" width="360" height="40" name="media_ear_location"> @@ -422,7 +422,7 @@ control_name="VoiceEarLocation" follows="left|top" layout="topleft" - left_pad="20" + left_pad="10" top_delta="-6" width="360" height="40" @@ -454,18 +454,38 @@ name="enable_lip_sync" top_pad="10" width="237"/> + + + + Noise Suppression + + + + + + + +