From 0ae8bbaf5d85b065354dcc010d08a4a2e759f867 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Wed, 6 Sep 2023 15:28:02 -0700 Subject: Checkpoint WebRTC Voice --- indra/llwebrtc/llwebrtc.cpp | 486 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 486 insertions(+) create mode 100644 indra/llwebrtc/llwebrtc.cpp (limited to 'indra/llwebrtc/llwebrtc.cpp') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp new file mode 100644 index 0000000000..5e71a00b60 --- /dev/null +++ b/indra/llwebrtc/llwebrtc.cpp @@ -0,0 +1,486 @@ +/** + * @file llaccordionctrl.cpp + * @brief Accordion panel implementation + * + * $LicenseInfo:firstyear=2023&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2023, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include "llwebrtc_impl.h" +#include + +#include "api/audio_codecs/audio_decoder_factory.h" +#include "api/audio_codecs/audio_encoder_factory.h" +#include "api/audio_codecs/builtin_audio_decoder_factory.h" +#include "api/audio_codecs/builtin_audio_encoder_factory.h" +#include "api/media_stream_interface.h" +#include "api/media_stream_track.h" + +namespace llwebrtc +{ + +void LLWebRTCImpl::init() +{ + mAnswerReceived = false; + rtc::InitializeSSL(); + mTaskQueueFactory = webrtc::CreateDefaultTaskQueueFactory(); + + mNetworkThread = rtc::Thread::CreateWithSocketServer(); + mNetworkThread->SetName("WebRTCNetworkThread", nullptr); + mNetworkThread->Start(); + mWorkerThread = rtc::Thread::Create(); + mWorkerThread->SetName("WebRTCWorkerThread", nullptr); + mWorkerThread->Start(); + mSignalingThread = rtc::Thread::Create(); + mSignalingThread->SetName("WebRTCSignalingThread", nullptr); + mSignalingThread->Start(); + + mSignalingThread->PostTask( + [this]() + { + mDeviceModule = webrtc::CreateAudioDeviceWithDataObserver(webrtc::AudioDeviceModule::AudioLayer::kPlatformDefaultAudio, + mTaskQueueFactory.get(), + std::unique_ptr(this)); + mDeviceModule->Init(); + updateDevices(); + }); +} + +void LLWebRTCImpl::refreshDevices() +{ + mSignalingThread->PostTask([this]() { updateDevices(); }); +} + +void LLWebRTCImpl::setDevicesObserver(LLWebRTCDevicesObserver *observer) { mVoiceDevicesObserverList.emplace_back(observer); } + +void LLWebRTCImpl::unsetDevicesObserver(LLWebRTCDevicesObserver *observer) +{ + std::vector::iterator it = + std::find(mVoiceDevicesObserverList.begin(), mVoiceDevicesObserverList.end(), observer); + if (it != mVoiceDevicesObserverList.end()) + { + mVoiceDevicesObserverList.erase(it); + } +} + +void LLWebRTCImpl::setCaptureDevice(const std::string &id) +{ + mSignalingThread->PostTask( + [this, id]() + { + int16_t captureDeviceCount = mDeviceModule->RecordingDevices(); + for (int16_t index = 0; index < captureDeviceCount; index++) + { + char name[webrtc::kAdmMaxDeviceNameSize]; + char guid[webrtc::kAdmMaxGuidSize]; + mDeviceModule->RecordingDeviceName(index, name, guid); + if (id == guid || id == name) + { + mDeviceModule->SetRecordingDevice(index); + break; + } + } + }); +} + +void LLWebRTCImpl::setRenderDevice(const std::string &id) +{ + mSignalingThread->PostTask( + [this, id]() + { + int16_t renderDeviceCount = mDeviceModule->RecordingDevices(); + for (int16_t index = 0; index < renderDeviceCount; index++) + { + char name[webrtc::kAdmMaxDeviceNameSize]; + char guid[webrtc::kAdmMaxGuidSize]; + mDeviceModule->PlayoutDeviceName(index, name, guid); + if (id == guid || id == name) + { + mDeviceModule->SetPlayoutDevice(index); + break; + } + } + }); +} + +void LLWebRTCImpl::updateDevices() +{ + int16_t renderDeviceCount = mDeviceModule->PlayoutDevices(); + LLWebRTCVoiceDeviceList renderDeviceList; + for (int16_t index = 0; index < renderDeviceCount; index++) + { + char name[webrtc::kAdmMaxDeviceNameSize]; + char guid[webrtc::kAdmMaxGuidSize]; + mDeviceModule->PlayoutDeviceName(index, name, guid); + renderDeviceList.emplace_back(name, guid); + } + for (auto &observer : mVoiceDevicesObserverList) + { + observer->OnRenderDevicesChanged(renderDeviceList); + } + + int16_t captureDeviceCount = mDeviceModule->RecordingDevices(); + LLWebRTCVoiceDeviceList captureDeviceList; + for (int16_t index = 0; index < captureDeviceCount; index++) + { + char name[webrtc::kAdmMaxDeviceNameSize]; + char guid[webrtc::kAdmMaxGuidSize]; + mDeviceModule->RecordingDeviceName(index, name, guid); + captureDeviceList.emplace_back(name, guid); + } + for (auto &observer : mVoiceDevicesObserverList) + { + observer->OnCaptureDevicesChanged(captureDeviceList); + } +} + +void LLWebRTCImpl::setTuningMode(bool enable) +{ + mSignalingThread->PostTask( + [this, enable]() + { + if (enable) + { + mDeviceModule->InitMicrophone(); + mDeviceModule->InitRecording(); + mDeviceModule->StartRecording(); + mDeviceModule->SetMicrophoneMute(false); + } + else + { + mDeviceModule->StopRecording(); + } + }); +} + +double LLWebRTCImpl::getTuningMicrophoneEnergy() { return mTuningEnergy; } + +void LLWebRTCImpl::OnCaptureData(const void *audio_samples, + const size_t num_samples, + const size_t bytes_per_sample, + const size_t num_channels, + const uint32_t samples_per_sec) +{ + if (bytes_per_sample != 4) + { + return; + } + + double energy = 0; + const short *samples = (const short *) audio_samples; + for (size_t index = 0; index < num_samples * num_channels; index++) + { + double sample = (static_cast(samples[index]) / (double) 32768); + energy += sample * sample; + } + mTuningEnergy = std::sqrt(energy); +} + +void LLWebRTCImpl::OnRenderData(const void *audio_samples, + const size_t num_samples, + const size_t bytes_per_sample, + const size_t num_channels, + const uint32_t samples_per_sec) +{ +} + +// +// LLWebRTCSignalInterface +// + +void LLWebRTCImpl::setSignalingObserver(LLWebRTCSignalingObserver *observer) { mSignalingObserverList.emplace_back(observer); } + +void LLWebRTCImpl::unsetSignalingObserver(LLWebRTCSignalingObserver *observer) +{ + std::vector::iterator it = + std::find(mSignalingObserverList.begin(), mSignalingObserverList.end(), observer); + if (it != mSignalingObserverList.end()) + { + mSignalingObserverList.erase(it); + } +} + +bool LLWebRTCImpl::initializeConnection() +{ + RTC_DCHECK(!mPeerConnection); + RTC_DCHECK(!mPeerConnectionFactory); + mAnswerReceived = false; + mPeerConnectionFactory = webrtc::CreatePeerConnectionFactory(mNetworkThread.get(), + mWorkerThread.get(), + mSignalingThread.get(), + nullptr /* default_adm */, + webrtc::CreateBuiltinAudioEncoderFactory(), + webrtc::CreateBuiltinAudioDecoderFactory(), + nullptr /* video_encoder_factory */, + nullptr /* video_decoder_factory */, + nullptr /* audio_mixer */, + nullptr /* audio_processing */); + + if (!mPeerConnectionFactory) + { + shutdownConnection(); + return false; + } + + webrtc::PeerConnectionInterface::RTCConfiguration config; + config.sdp_semantics = webrtc::SdpSemantics::kUnifiedPlan; + webrtc::PeerConnectionInterface::IceServer server; + server.uri = "stun:stun.l.google.com:19302"; + // config.servers.push_back(server); + // server.uri = "stun:stun1.l.google.com:19302"; + // config.servers.push_back(server); + // server.uri = "stun:stun2.l.google.com:19302"; + // config.servers.push_back(server); + // server.uri = "stun:stun3.l.google.com:19302"; + // config.servers.push_back(server); + // server.uri = "stun:stun4.l.google.com:19302"; + // config.servers.push_back(server); + + webrtc::PeerConnectionDependencies pc_dependencies(this); + auto error_or_peer_connection = mPeerConnectionFactory->CreatePeerConnectionOrError(config, std::move(pc_dependencies)); + if (error_or_peer_connection.ok()) + { + mPeerConnection = std::move(error_or_peer_connection.value()); + } + else + { + shutdownConnection(); + return false; + } + + RTC_LOG(LS_INFO) << __FUNCTION__ << " " << mPeerConnection->signaling_state(); + + cricket::AudioOptions audioOptions; + audioOptions.auto_gain_control = true; + audioOptions.echo_cancellation = true; + audioOptions.noise_suppression = true; + + rtc::scoped_refptr stream = mPeerConnectionFactory->CreateLocalMediaStream("SLStream"); + rtc::scoped_refptr audio_track( + mPeerConnectionFactory->CreateAudioTrack("SLAudio", mPeerConnectionFactory->CreateAudioSource(cricket::AudioOptions()).get())); + audio_track->set_enabled(true); + stream->AddTrack(audio_track); + + mPeerConnection->AddTrack(audio_track, {"SLStream"}); + mPeerConnection->SetLocalDescription(rtc::scoped_refptr(this)); + + RTC_LOG(LS_INFO) << __FUNCTION__ << " " << mPeerConnection->signaling_state(); + + return mPeerConnection != nullptr; +} + +void LLWebRTCImpl::shutdownConnection() +{ + mPeerConnection = nullptr; + mPeerConnectionFactory = nullptr; +} + +void LLWebRTCImpl::AnswerAvailable(const std::string &sdp) +{ + mSignalingThread->PostTask( + [this, sdp]() + { + RTC_LOG(LS_INFO) << __FUNCTION__ << " " << mPeerConnection->peer_connection_state(); + mPeerConnection->SetRemoteDescription(webrtc::CreateSessionDescription(webrtc::SdpType::kAnswer, sdp), + rtc::scoped_refptr(this)); + mAnswerReceived = true; + for (auto &observer : mSignalingObserverList) + { + for (auto &candidate : mCachedIceCandidates) + { + LLWebRTCIceCandidate ice_candidate; + ice_candidate.candidate = candidate->candidate().ToString(); + ice_candidate.mline_index = candidate->sdp_mline_index(); + ice_candidate.sdp_mid = candidate->sdp_mid(); + observer->OnIceCandidate(ice_candidate); + } + mCachedIceCandidates.clear(); + if (mPeerConnection->ice_gathering_state() == webrtc::PeerConnectionInterface::IceGatheringState::kIceGatheringComplete) + { + for (auto &observer : mSignalingObserverList) + { + observer->OnIceGatheringState(llwebrtc::LLWebRTCSignalingObserver::IceGatheringState::ICE_GATHERING_COMPLETE); + } + } + } + }); +} + +void LLWebRTCImpl::setMute(bool mute) +{ + mSignalingThread->PostTask( + [this,mute]() + { + auto senders = mPeerConnection->GetSenders(); + + RTC_LOG(LS_INFO) << __FUNCTION__ << (mute ? "disabling" : "enabling") << " streams count " + << senders.size(); + + for (auto& sender : senders) + { + sender->track()->set_enabled(!mute); + } + }); +} + +// +// PeerConnectionObserver implementation. +// + +void LLWebRTCImpl::OnAddTrack(rtc::scoped_refptr receiver, + const std::vector> &streams) +{ + RTC_LOG(LS_INFO) << __FUNCTION__ << " " << receiver->id(); +} + +void LLWebRTCImpl::OnRemoveTrack(rtc::scoped_refptr receiver) +{ + RTC_LOG(LS_INFO) << __FUNCTION__ << " " << receiver->id(); +} + +void LLWebRTCImpl::OnIceGatheringChange(webrtc::PeerConnectionInterface::IceGatheringState new_state) +{ + LLWebRTCSignalingObserver::IceGatheringState webrtc_new_state = LLWebRTCSignalingObserver::IceGatheringState::ICE_GATHERING_NEW; + switch (new_state) + { + case webrtc::PeerConnectionInterface::IceGatheringState::kIceGatheringNew: + webrtc_new_state = LLWebRTCSignalingObserver::IceGatheringState::ICE_GATHERING_NEW; + break; + case webrtc::PeerConnectionInterface::IceGatheringState::kIceGatheringGathering: + webrtc_new_state = LLWebRTCSignalingObserver::IceGatheringState::ICE_GATHERING_GATHERING; + break; + case webrtc::PeerConnectionInterface::IceGatheringState::kIceGatheringComplete: + webrtc_new_state = LLWebRTCSignalingObserver::IceGatheringState::ICE_GATHERING_COMPLETE; + break; + default: + RTC_LOG(LS_ERROR) << __FUNCTION__ << " Bad Ice Gathering State" << new_state; + webrtc_new_state = LLWebRTCSignalingObserver::IceGatheringState::ICE_GATHERING_NEW; + return; + } + + if (mAnswerReceived) + { + for (auto &observer : mSignalingObserverList) + { + observer->OnIceGatheringState(webrtc_new_state); + } + } +} + +// Called any time the PeerConnectionState changes. +void LLWebRTCImpl::OnConnectionChange(webrtc::PeerConnectionInterface::PeerConnectionState new_state) +{ + RTC_LOG(LS_ERROR) << __FUNCTION__ << " Peer Connection State Change " << new_state; +} + +void LLWebRTCImpl::OnIceCandidate(const webrtc::IceCandidateInterface *candidate) +{ + RTC_LOG(LS_INFO) << __FUNCTION__ << " " << candidate->sdp_mline_index(); + + if (!candidate) + { + RTC_LOG(LS_ERROR) << __FUNCTION__ << " No Ice Candidate Given"; + return; + } + if (mAnswerReceived) + { + for (auto &observer : mSignalingObserverList) + { + LLWebRTCIceCandidate ice_candidate; + ice_candidate.candidate = candidate->candidate().ToString(); + ice_candidate.mline_index = candidate->sdp_mline_index(); + ice_candidate.sdp_mid = candidate->sdp_mid(); + observer->OnIceCandidate(ice_candidate); + } + } + else + { + mCachedIceCandidates.push_back( + webrtc::CreateIceCandidate(candidate->sdp_mid(), candidate->sdp_mline_index(), candidate->candidate())); + } +} + +// +// CreateSessionDescriptionObserver implementation. +// +void LLWebRTCImpl::OnSuccess(webrtc::SessionDescriptionInterface *desc) +{ + std::string sdp; + desc->ToString(&sdp); + RTC_LOG(LS_INFO) << sdp; + + RTC_LOG(LS_INFO) << __FUNCTION__ << " " << mPeerConnection->signaling_state(); + for (auto &observer : mSignalingObserverList) + { + observer->OnOfferAvailable(sdp); + } +} + +void LLWebRTCImpl::OnFailure(webrtc::RTCError error) { RTC_LOG(LS_ERROR) << ToString(error.type()) << ": " << error.message(); } + +// +// SetRemoteDescriptionObserverInterface implementation. +// +void LLWebRTCImpl::OnSetRemoteDescriptionComplete(webrtc::RTCError error) +{ + RTC_LOG(LS_INFO) << __FUNCTION__ << " " << mPeerConnection->signaling_state(); + if (!error.ok()) + { + RTC_LOG(LS_ERROR) << ToString(error.type()) << ": " << error.message(); + return; + } + for (auto &observer : mSignalingObserverList) + { + observer->OnAudioEstablished(this); + } +} + +// +// SetLocalDescriptionObserverInterface implementation. +// +void LLWebRTCImpl::OnSetLocalDescriptionComplete(webrtc::RTCError error) +{ + RTC_LOG(LS_INFO) << __FUNCTION__ << " " << mPeerConnection->signaling_state(); + if (!error.ok()) + { + RTC_LOG(LS_ERROR) << ToString(error.type()) << ": " << error.message(); + return; + } + auto desc = mPeerConnection->pending_local_description(); + std::string sdp; + desc->ToString(&sdp); + for (auto &observer : mSignalingObserverList) + { + observer->OnOfferAvailable(sdp); + } +} + +rtc::RefCountedObject *gWebRTCImpl = nullptr; +LLWebRTCDeviceInterface *getDeviceInterface() { return gWebRTCImpl; } +LLWebRTCSignalInterface *getSignalingInterface() { return gWebRTCImpl; } + +void init() +{ + gWebRTCImpl = new rtc::RefCountedObject(); + gWebRTCImpl->AddRef(); + gWebRTCImpl->init(); +} +} // namespace llwebrtc \ No newline at end of file -- cgit v1.2.3 From 4451c3f420d9458e817d22666642d5899283eeda Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Sun, 10 Sep 2023 00:30:59 -0700 Subject: coding policy fixes --- indra/llwebrtc/llwebrtc.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'indra/llwebrtc/llwebrtc.cpp') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index 5e71a00b60..e7a9072b2e 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -483,4 +483,4 @@ void init() gWebRTCImpl->AddRef(); gWebRTCImpl->init(); } -} // namespace llwebrtc \ No newline at end of file +} // namespace llwebrtc -- cgit v1.2.3 From fa313bab6348117f03b30dc4a5813b5058df57f7 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Tue, 12 Sep 2023 15:17:08 -0700 Subject: do some thread safety to prevent webrtc threads from conflicting with viewer threads. --- indra/llwebrtc/llwebrtc.cpp | 35 +++++++++++++++++++++++++++++++---- 1 file changed, 31 insertions(+), 4 deletions(-) (limited to 'indra/llwebrtc/llwebrtc.cpp') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index e7a9072b2e..c2631b2ea3 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -389,6 +389,36 @@ void LLWebRTCImpl::OnIceGatheringChange(webrtc::PeerConnectionInterface::IceGath void LLWebRTCImpl::OnConnectionChange(webrtc::PeerConnectionInterface::PeerConnectionState new_state) { RTC_LOG(LS_ERROR) << __FUNCTION__ << " Peer Connection State Change " << new_state; + + switch (new_state) + { + case webrtc::PeerConnectionInterface::PeerConnectionState::kConnected: + { + if (new_state == webrtc::PeerConnectionInterface::PeerConnectionState::kConnected) + { + for (auto &observer : mSignalingObserverList) + { + observer->OnAudioEstablished(this); + } + } + break; + } + case webrtc::PeerConnectionInterface::PeerConnectionState::kDisconnected: + { + if (new_state == webrtc::PeerConnectionInterface::PeerConnectionState::kConnected) + { + for (auto &observer : mSignalingObserverList) + { + observer->OnRenegotiationNeeded(); + } + } + break; + } + default: + { + break; + } + } } void LLWebRTCImpl::OnIceCandidate(const webrtc::IceCandidateInterface *candidate) @@ -447,10 +477,6 @@ void LLWebRTCImpl::OnSetRemoteDescriptionComplete(webrtc::RTCError error) RTC_LOG(LS_ERROR) << ToString(error.type()) << ": " << error.message(); return; } - for (auto &observer : mSignalingObserverList) - { - observer->OnAudioEstablished(this); - } } // @@ -467,6 +493,7 @@ void LLWebRTCImpl::OnSetLocalDescriptionComplete(webrtc::RTCError error) auto desc = mPeerConnection->pending_local_description(); std::string sdp; desc->ToString(&sdp); + RTC_LOG(LS_INFO) << __FUNCTION__ << " Local SDP: " << sdp; for (auto &observer : mSignalingObserverList) { observer->OnOfferAvailable(sdp); -- cgit v1.2.3 From a2435aa1d07a972a8f3dc8dccfc5fdaf16c4a466 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Wed, 13 Sep 2023 14:49:42 -0700 Subject: Hook up speaker volume. --- indra/llwebrtc/llwebrtc.cpp | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) (limited to 'indra/llwebrtc/llwebrtc.cpp') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index c2631b2ea3..93e9db9c1d 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -341,6 +341,28 @@ void LLWebRTCImpl::setMute(bool mute) }); } +void LLWebRTCImpl::setSpeakerVolume(float volume) +{ + mSignalingThread->PostTask( + [this, volume]() + { + auto receivers = mPeerConnection->GetReceivers(); + + RTC_LOG(LS_INFO) << __FUNCTION__ << "Set volume" << receivers.size(); + for (auto &receiver : receivers) + { + webrtc::MediaStreamTrackInterface *track = receiver->track().get(); + if (track->kind() == webrtc::MediaStreamTrackInterface::kAudioKind) + { + webrtc::AudioTrackInterface* audio_track = static_cast(track); + webrtc::AudioSourceInterface* source = audio_track->GetSource(); + source->SetVolume(10.0 * volume); + + } + } + }); +} + // // PeerConnectionObserver implementation. // -- cgit v1.2.3 From e554bf05293e34a5b8af7933127dff76c537f275 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Wed, 13 Sep 2023 21:45:42 -0700 Subject: Fix connection failed logic to do a renegotiate. Also, remove some dead code. --- indra/llwebrtc/llwebrtc.cpp | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) (limited to 'indra/llwebrtc/llwebrtc.cpp') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index 93e9db9c1d..862325c3f1 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -425,15 +425,13 @@ void LLWebRTCImpl::OnConnectionChange(webrtc::PeerConnectionInterface::PeerConne } break; } - case webrtc::PeerConnectionInterface::PeerConnectionState::kDisconnected: + case webrtc::PeerConnectionInterface::PeerConnectionState::kFailed: { - if (new_state == webrtc::PeerConnectionInterface::PeerConnectionState::kConnected) + for (auto &observer : mSignalingObserverList) { - for (auto &observer : mSignalingObserverList) - { - observer->OnRenegotiationNeeded(); - } + observer->OnRenegotiationNeeded(); } + break; } default: -- cgit v1.2.3 From f519506e0455748794792ab59e4865ad2ae87ab3 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Wed, 13 Sep 2023 22:49:39 -0700 Subject: fix some retry logic and speaker volume logic --- indra/llwebrtc/llwebrtc.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'indra/llwebrtc/llwebrtc.cpp') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index 862325c3f1..93981e4076 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -37,6 +37,8 @@ namespace llwebrtc { +const float VOLUME_SCALE_WEBRTC = 3.0f; + void LLWebRTCImpl::init() { mAnswerReceived = false; @@ -356,7 +358,7 @@ void LLWebRTCImpl::setSpeakerVolume(float volume) { webrtc::AudioTrackInterface* audio_track = static_cast(track); webrtc::AudioSourceInterface* source = audio_track->GetSource(); - source->SetVolume(10.0 * volume); + source->SetVolume(VOLUME_SCALE_WEBRTC * volume); } } -- cgit v1.2.3 From 0afb330cf19c0f988fd6688d3bc4266a3fd6624e Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Thu, 14 Sep 2023 20:48:33 -0700 Subject: Remove bad session from janus when negotation fails and is retried. --- indra/llwebrtc/llwebrtc.cpp | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) (limited to 'indra/llwebrtc/llwebrtc.cpp') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index 93981e4076..edf941436e 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -246,15 +246,15 @@ bool LLWebRTCImpl::initializeConnection() config.sdp_semantics = webrtc::SdpSemantics::kUnifiedPlan; webrtc::PeerConnectionInterface::IceServer server; server.uri = "stun:stun.l.google.com:19302"; - // config.servers.push_back(server); - // server.uri = "stun:stun1.l.google.com:19302"; - // config.servers.push_back(server); - // server.uri = "stun:stun2.l.google.com:19302"; - // config.servers.push_back(server); - // server.uri = "stun:stun3.l.google.com:19302"; - // config.servers.push_back(server); - // server.uri = "stun:stun4.l.google.com:19302"; - // config.servers.push_back(server); + config.servers.push_back(server); + server.uri = "stun:stun1.l.google.com:19302"; + config.servers.push_back(server); + server.uri = "stun:stun2.l.google.com:19302"; + config.servers.push_back(server); + server.uri = "stun:stun3.l.google.com:19302"; + config.servers.push_back(server); + server.uri = "stun:stun4.l.google.com:19302"; + config.servers.push_back(server); webrtc::PeerConnectionDependencies pc_dependencies(this); auto error_or_peer_connection = mPeerConnectionFactory->CreatePeerConnectionOrError(config, std::move(pc_dependencies)); -- cgit v1.2.3 From 976a75ed24f11e3211a98120b1ac57126e625d6c Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Mon, 18 Sep 2023 11:25:47 -0700 Subject: Fix voice device settings --- indra/llwebrtc/llwebrtc.cpp | 94 +++++++++++++++++++++++++++++++++++---------- 1 file changed, 73 insertions(+), 21 deletions(-) (limited to 'indra/llwebrtc/llwebrtc.cpp') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index edf941436e..96be2c1f0b 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -55,7 +55,7 @@ void LLWebRTCImpl::init() mSignalingThread->SetName("WebRTCSignalingThread", nullptr); mSignalingThread->Start(); - mSignalingThread->PostTask( + mWorkerThread->PostTask( [this]() { mDeviceModule = webrtc::CreateAudioDeviceWithDataObserver(webrtc::AudioDeviceModule::AudioLayer::kPlatformDefaultAudio, @@ -68,7 +68,7 @@ void LLWebRTCImpl::init() void LLWebRTCImpl::refreshDevices() { - mSignalingThread->PostTask([this]() { updateDevices(); }); + mWorkerThread->PostTask([this]() { updateDevices(); }); } void LLWebRTCImpl::setDevicesObserver(LLWebRTCDevicesObserver *observer) { mVoiceDevicesObserverList.emplace_back(observer); } @@ -85,41 +85,48 @@ void LLWebRTCImpl::unsetDevicesObserver(LLWebRTCDevicesObserver *observer) void LLWebRTCImpl::setCaptureDevice(const std::string &id) { - mSignalingThread->PostTask( + mWorkerThread->PostTask( [this, id]() { + mDeviceModule->StopRecording(); int16_t captureDeviceCount = mDeviceModule->RecordingDevices(); for (int16_t index = 0; index < captureDeviceCount; index++) { char name[webrtc::kAdmMaxDeviceNameSize]; char guid[webrtc::kAdmMaxGuidSize]; mDeviceModule->RecordingDeviceName(index, name, guid); - if (id == guid || id == name) + if (id == guid || id == "Default") { + RTC_LOG(LS_INFO) << __FUNCTION__ << "Set recording device to " << name << " " << guid << " " << index; mDeviceModule->SetRecordingDevice(index); break; } } + mDeviceModule->InitRecording(); + mDeviceModule->StartRecording(); }); } void LLWebRTCImpl::setRenderDevice(const std::string &id) { - mSignalingThread->PostTask( + mWorkerThread->PostTask( [this, id]() { + mDeviceModule->StopPlayout(); int16_t renderDeviceCount = mDeviceModule->RecordingDevices(); for (int16_t index = 0; index < renderDeviceCount; index++) { char name[webrtc::kAdmMaxDeviceNameSize]; char guid[webrtc::kAdmMaxGuidSize]; mDeviceModule->PlayoutDeviceName(index, name, guid); - if (id == guid || id == name) + if (id == guid || id == "Default") { mDeviceModule->SetPlayoutDevice(index); break; } } + mDeviceModule->InitPlayout(); + mDeviceModule->StartPlayout(); }); } @@ -156,7 +163,7 @@ void LLWebRTCImpl::updateDevices() void LLWebRTCImpl::setTuningMode(bool enable) { - mSignalingThread->PostTask( + mWorkerThread->PostTask( [this, enable]() { if (enable) @@ -181,7 +188,7 @@ void LLWebRTCImpl::OnCaptureData(const void *audio_samples, const size_t num_channels, const uint32_t samples_per_sec) { - if (bytes_per_sample != 4) + if (bytes_per_sample != 2) { return; } @@ -220,28 +227,31 @@ void LLWebRTCImpl::unsetSignalingObserver(LLWebRTCSignalingObserver *observer) } } + bool LLWebRTCImpl::initializeConnection() { RTC_DCHECK(!mPeerConnection); - RTC_DCHECK(!mPeerConnectionFactory); + RTC_DCHECK(mPeerConnectionFactory); mAnswerReceived = false; + + mSignalingThread->PostTask([this]() { initializeConnectionThreaded(); }); + return true; +} + + + +bool LLWebRTCImpl::initializeConnectionThreaded() +{ mPeerConnectionFactory = webrtc::CreatePeerConnectionFactory(mNetworkThread.get(), mWorkerThread.get(), mSignalingThread.get(), - nullptr /* default_adm */, + mDeviceModule, webrtc::CreateBuiltinAudioEncoderFactory(), webrtc::CreateBuiltinAudioDecoderFactory(), nullptr /* video_encoder_factory */, nullptr /* video_decoder_factory */, nullptr /* audio_mixer */, nullptr /* audio_processing */); - - if (!mPeerConnectionFactory) - { - shutdownConnection(); - return false; - } - webrtc::PeerConnectionInterface::RTCConfiguration config; config.sdp_semantics = webrtc::SdpSemantics::kUnifiedPlan; webrtc::PeerConnectionInterface::IceServer server; @@ -272,16 +282,34 @@ bool LLWebRTCImpl::initializeConnection() cricket::AudioOptions audioOptions; audioOptions.auto_gain_control = true; - audioOptions.echo_cancellation = true; + audioOptions.echo_cancellation = false; // incompatible with opus stereo audioOptions.noise_suppression = true; rtc::scoped_refptr stream = mPeerConnectionFactory->CreateLocalMediaStream("SLStream"); rtc::scoped_refptr audio_track( - mPeerConnectionFactory->CreateAudioTrack("SLAudio", mPeerConnectionFactory->CreateAudioSource(cricket::AudioOptions()).get())); + mPeerConnectionFactory->CreateAudioTrack("SLAudio", mPeerConnectionFactory->CreateAudioSource(audioOptions).get())); audio_track->set_enabled(true); stream->AddTrack(audio_track); mPeerConnection->AddTrack(audio_track, {"SLStream"}); + + auto senders = mPeerConnection->GetSenders(); + + for (auto &sender : senders) + { + webrtc::RtpParameters params; + webrtc::RtpCodecParameters codecparam; + codecparam.name = "opus"; + codecparam.kind = cricket::MEDIA_TYPE_AUDIO; + codecparam.clock_rate = 48000; + codecparam.num_channels = 1; + codecparam.parameters["stereo"] = "0"; + codecparam.parameters["sprop-stereo"] = "0"; + + params.codecs.push_back(codecparam); + sender->SetParameters(params); + } + mPeerConnection->SetLocalDescription(rtc::scoped_refptr(this)); RTC_LOG(LS_INFO) << __FUNCTION__ << " " << mPeerConnection->signaling_state(); @@ -297,6 +325,12 @@ void LLWebRTCImpl::shutdownConnection() void LLWebRTCImpl::AnswerAvailable(const std::string &sdp) { + std::istringstream sdp_stream(sdp); + std::string sdp_line; + while (std::getline(sdp_stream, sdp_line)) + { + RTC_LOG(LS_INFO) << __FUNCTION__ << " Remote SDP: " << sdp_line; + } mSignalingThread->PostTask( [this, sdp]() { @@ -515,10 +549,28 @@ void LLWebRTCImpl::OnSetLocalDescriptionComplete(webrtc::RTCError error) auto desc = mPeerConnection->pending_local_description(); std::string sdp; desc->ToString(&sdp); - RTC_LOG(LS_INFO) << __FUNCTION__ << " Local SDP: " << sdp; + // mangle the sdp as this is the only way currently to bump up + // the send audio rate to 48k + std::istringstream sdp_stream(sdp); + std::ostringstream sdp_mangled_stream; + std::string sdp_line; + while (std::getline(sdp_stream, sdp_line)) { + int bandwidth = 0; + int payload_id = 0; + RTC_LOG(LS_INFO) << __FUNCTION__ << " Local SDP: " << sdp_line; + // force mono + if (std::sscanf(sdp_line.c_str(), "a=rtpmap:%i opus/%i/2", &payload_id, &bandwidth) == 2) + { + sdp_mangled_stream << sdp_line << "\n"; + } + else + { + sdp_mangled_stream << sdp_line << "\n"; + } + } for (auto &observer : mSignalingObserverList) { - observer->OnOfferAvailable(sdp); + observer->OnOfferAvailable(sdp_mangled_stream.str()); } } -- cgit v1.2.3 From fd8119c550bea19bbcf26e5426f259010f54c43e Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Tue, 19 Sep 2023 10:14:29 -0700 Subject: add datachannel support --- indra/llwebrtc/llwebrtc.cpp | 50 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) (limited to 'indra/llwebrtc/llwebrtc.cpp') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index 96be2c1f0b..ac5870eab3 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -314,7 +314,22 @@ bool LLWebRTCImpl::initializeConnectionThreaded() RTC_LOG(LS_INFO) << __FUNCTION__ << " " << mPeerConnection->signaling_state(); - return mPeerConnection != nullptr; + webrtc::DataChannelInit init; + init.ordered = true; + init.reliable = true; + auto data_channel_or_error = mPeerConnection->CreateDataChannelOrError("SLData", &init); + if (data_channel_or_error.ok()) + { + mDataChannel = std::move(data_channel_or_error.value()); + } + else + { + shutdownConnection(); + return false; + } + mDataChannel->RegisterObserver(this); + + return true; } void LLWebRTCImpl::shutdownConnection() @@ -574,9 +589,42 @@ void LLWebRTCImpl::OnSetLocalDescriptionComplete(webrtc::RTCError error) } } +// +// DataChannelObserver implementation +// +void LLWebRTCImpl::OnMessage(const webrtc::DataBuffer& buffer) +{ + std::string data((const char*)buffer.data.cdata(), buffer.size()); + for (auto &observer : mDataObserverList) + { + observer->OnDataReceived(data, buffer.binary); + } +} + +void LLWebRTCImpl::sendData(const std::string& data, bool binary) +{ + rtc::CopyOnWriteBuffer cowBuffer(data.data(), data.length()); + webrtc::DataBuffer buffer(cowBuffer, binary); + mDataChannel->Send(buffer); +} + +void LLWebRTCImpl::setDataObserver(LLWebRTCDataObserver* observer) { mDataObserverList.emplace_back(observer); } + +void LLWebRTCImpl::unsetDataObserver(LLWebRTCDataObserver* observer) +{ + std::vector::iterator it = + std::find(mDataObserverList.begin(), mDataObserverList.end(), observer); + if (it != mDataObserverList.end()) + { + mDataObserverList.erase(it); + } +} + rtc::RefCountedObject *gWebRTCImpl = nullptr; LLWebRTCDeviceInterface *getDeviceInterface() { return gWebRTCImpl; } LLWebRTCSignalInterface *getSignalingInterface() { return gWebRTCImpl; } +LLWebRTCDataInterface *getDataInterface() { return gWebRTCImpl; } + void init() { -- cgit v1.2.3 From f1f1bccad299f6cb70cb9f439872a5f702e8f972 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Thu, 21 Sep 2023 15:28:58 -0700 Subject: Stream audio levels to and from viewers via DataChannels --- indra/llwebrtc/llwebrtc.cpp | 204 ++++++++++++++++++++++++++++++++++---------- 1 file changed, 161 insertions(+), 43 deletions(-) (limited to 'indra/llwebrtc/llwebrtc.cpp') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index ac5870eab3..77b050cbd0 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -62,10 +62,39 @@ void LLWebRTCImpl::init() mTaskQueueFactory.get(), std::unique_ptr(this)); mDeviceModule->Init(); + mDeviceModule->SetStereoRecording(false); + mDeviceModule->EnableBuiltInAEC(false); updateDevices(); }); } +void LLWebRTCImpl::terminate() +{ + mSignalingThread->BlockingCall( + [this]() + { + if (mPeerConnection) + { + mPeerConnection->Close(); + mPeerConnection = nullptr; + } + }); + mWorkerThread->BlockingCall( + [this]() + { + if (mDeviceModule) + { + mDeviceModule = nullptr; + } + }); + + mNetworkThread->Stop(); + mWorkerThread->Stop(); + mSignalingThread->Stop(); + +} + + void LLWebRTCImpl::refreshDevices() { mWorkerThread->PostTask([this]() { updateDevices(); }); @@ -88,22 +117,33 @@ void LLWebRTCImpl::setCaptureDevice(const std::string &id) mWorkerThread->PostTask( [this, id]() { - mDeviceModule->StopRecording(); + bool was_recording = mDeviceModule->Recording(); + + if (was_recording) + { + mDeviceModule->StopRecording(); + } int16_t captureDeviceCount = mDeviceModule->RecordingDevices(); - for (int16_t index = 0; index < captureDeviceCount; index++) + int16_t index = 0; /* default to first one if no match */ + for (int16_t i = 0; i < captureDeviceCount; i++) { char name[webrtc::kAdmMaxDeviceNameSize]; char guid[webrtc::kAdmMaxGuidSize]; - mDeviceModule->RecordingDeviceName(index, name, guid); + mDeviceModule->RecordingDeviceName(i, name, guid); if (id == guid || id == "Default") { - RTC_LOG(LS_INFO) << __FUNCTION__ << "Set recording device to " << name << " " << guid << " " << index; - mDeviceModule->SetRecordingDevice(index); + RTC_LOG(LS_INFO) << __FUNCTION__ << "Set recording device to " << name << " " << guid << " " << i; + index = i; break; } } + mDeviceModule->SetRecordingDevice(index); + mDeviceModule->InitMicrophone(); mDeviceModule->InitRecording(); - mDeviceModule->StartRecording(); + if (was_recording) + { + mDeviceModule->StartRecording(); + } }); } @@ -112,21 +152,32 @@ void LLWebRTCImpl::setRenderDevice(const std::string &id) mWorkerThread->PostTask( [this, id]() { - mDeviceModule->StopPlayout(); - int16_t renderDeviceCount = mDeviceModule->RecordingDevices(); - for (int16_t index = 0; index < renderDeviceCount; index++) + bool was_playing = mDeviceModule->Playing(); + if (was_playing) + { + mDeviceModule->StopPlayout(); + } + int16_t renderDeviceCount = mDeviceModule->PlayoutDevices(); + int16_t index = 0; /* default to first one if no match */ + for (int16_t i = 0; i < renderDeviceCount; i++) { char name[webrtc::kAdmMaxDeviceNameSize]; char guid[webrtc::kAdmMaxGuidSize]; - mDeviceModule->PlayoutDeviceName(index, name, guid); + mDeviceModule->PlayoutDeviceName(i, name, guid); if (id == guid || id == "Default") { - mDeviceModule->SetPlayoutDevice(index); + RTC_LOG(LS_INFO) << __FUNCTION__ << "Set recording device to " << name << " " << guid << " " << i; + index = i; break; } } + mDeviceModule->SetPlayoutDevice(index); + mDeviceModule->InitSpeaker(); mDeviceModule->InitPlayout(); - mDeviceModule->StartPlayout(); + if (was_playing) + { + mDeviceModule->StartPlayout(); + } }); } @@ -141,10 +192,6 @@ void LLWebRTCImpl::updateDevices() mDeviceModule->PlayoutDeviceName(index, name, guid); renderDeviceList.emplace_back(name, guid); } - for (auto &observer : mVoiceDevicesObserverList) - { - observer->OnRenderDevicesChanged(renderDeviceList); - } int16_t captureDeviceCount = mDeviceModule->RecordingDevices(); LLWebRTCVoiceDeviceList captureDeviceList; @@ -157,7 +204,7 @@ void LLWebRTCImpl::updateDevices() } for (auto &observer : mVoiceDevicesObserverList) { - observer->OnCaptureDevicesChanged(captureDeviceList); + observer->OnDevicesChanged(renderDeviceList, captureDeviceList); } } @@ -188,11 +235,6 @@ void LLWebRTCImpl::OnCaptureData(const void *audio_samples, const size_t num_channels, const uint32_t samples_per_sec) { - if (bytes_per_sample != 2) - { - return; - } - double energy = 0; const short *samples = (const short *) audio_samples; for (size_t index = 0; index < num_samples * num_channels; index++) @@ -242,6 +284,21 @@ bool LLWebRTCImpl::initializeConnection() bool LLWebRTCImpl::initializeConnectionThreaded() { + rtc::scoped_refptr apm = webrtc::AudioProcessingBuilder().Create(); + webrtc::AudioProcessing::Config apm_config; + apm_config.echo_canceller.enabled = false; + apm_config.echo_canceller.mobile_mode = false; + apm_config.gain_controller1.enabled = true; + apm_config.gain_controller1.mode = + webrtc::AudioProcessing::Config::GainController1::kAdaptiveAnalog; + apm_config.gain_controller2.enabled = true; + apm_config.high_pass_filter.enabled = true; + apm_config.noise_suppression.enabled = true; + apm_config.noise_suppression.level = webrtc::AudioProcessing::Config::NoiseSuppression::kVeryHigh; + apm_config.transient_suppression.enabled = true; + // + apm->ApplyConfig(apm_config); + mPeerConnectionFactory = webrtc::CreatePeerConnectionFactory(mNetworkThread.get(), mWorkerThread.get(), mSignalingThread.get(), @@ -251,7 +308,7 @@ bool LLWebRTCImpl::initializeConnectionThreaded() nullptr /* video_encoder_factory */, nullptr /* video_decoder_factory */, nullptr /* audio_mixer */, - nullptr /* audio_processing */); + apm); webrtc::PeerConnectionInterface::RTCConfiguration config; config.sdp_semantics = webrtc::SdpSemantics::kUnifiedPlan; webrtc::PeerConnectionInterface::IceServer server; @@ -278,6 +335,17 @@ bool LLWebRTCImpl::initializeConnectionThreaded() return false; } + webrtc::DataChannelInit init; + init.ordered = true; + + auto data_channel_or_error = mPeerConnection->CreateDataChannelOrError("SLData", &init); + if (data_channel_or_error.ok()) + { + mDataChannel = std::move(data_channel_or_error.value()); + + mDataChannel->RegisterObserver(this); + } + RTC_LOG(LS_INFO) << __FUNCTION__ << " " << mPeerConnection->signaling_state(); cricket::AudioOptions audioOptions; @@ -305,7 +373,6 @@ bool LLWebRTCImpl::initializeConnectionThreaded() codecparam.num_channels = 1; codecparam.parameters["stereo"] = "0"; codecparam.parameters["sprop-stereo"] = "0"; - params.codecs.push_back(codecparam); sender->SetParameters(params); } @@ -313,21 +380,6 @@ bool LLWebRTCImpl::initializeConnectionThreaded() mPeerConnection->SetLocalDescription(rtc::scoped_refptr(this)); RTC_LOG(LS_INFO) << __FUNCTION__ << " " << mPeerConnection->signaling_state(); - - webrtc::DataChannelInit init; - init.ordered = true; - init.reliable = true; - auto data_channel_or_error = mPeerConnection->CreateDataChannelOrError("SLData", &init); - if (data_channel_or_error.ok()) - { - mDataChannel = std::move(data_channel_or_error.value()); - } - else - { - shutdownConnection(); - return false; - } - mDataChannel->RegisterObserver(this); return true; } @@ -414,6 +466,18 @@ void LLWebRTCImpl::setSpeakerVolume(float volume) }); } +void LLWebRTCImpl::requestAudioLevel() +{ + mWorkerThread->PostTask( + [this]() + { + for (auto &observer : mAudioObserverList) + { + observer->OnAudioLevel((float)mTuningEnergy); + } + }); +} + // // PeerConnectionObserver implementation. // @@ -429,6 +493,13 @@ void LLWebRTCImpl::OnRemoveTrack(rtc::scoped_refptrid(); } +void LLWebRTCImpl::OnDataChannel(rtc::scoped_refptr channel) +{ + mDataChannel = channel; + channel->RegisterObserver(this); +} + + void LLWebRTCImpl::OnIceGatheringChange(webrtc::PeerConnectionInterface::IceGatheringState new_state) { LLWebRTCSignalingObserver::IceGatheringState webrtc_new_state = LLWebRTCSignalingObserver::IceGatheringState::ICE_GATHERING_NEW; @@ -469,10 +540,14 @@ void LLWebRTCImpl::OnConnectionChange(webrtc::PeerConnectionInterface::PeerConne { if (new_state == webrtc::PeerConnectionInterface::PeerConnectionState::kConnected) { - for (auto &observer : mSignalingObserverList) - { - observer->OnAudioEstablished(this); - } + mWorkerThread->PostTask([this]() { + mDeviceModule->StartRecording(); + mDeviceModule->StartPlayout(); + for (auto &observer : mSignalingObserverList) + { + observer->OnAudioEstablished(this); + } + }); } break; } @@ -589,9 +664,44 @@ void LLWebRTCImpl::OnSetLocalDescriptionComplete(webrtc::RTCError error) } } +void LLWebRTCImpl::setAudioObserver(LLWebRTCAudioObserver *observer) { mAudioObserverList.emplace_back(observer); } + +void LLWebRTCImpl::unsetAudioObserver(LLWebRTCAudioObserver *observer) +{ + std::vector::iterator it = std::find(mAudioObserverList.begin(), mAudioObserverList.end(), observer); + if (it != mAudioObserverList.end()) + { + mAudioObserverList.erase(it); + } +} + // // DataChannelObserver implementation // + +void LLWebRTCImpl::OnStateChange() +{ + RTC_LOG(LS_INFO) << __FUNCTION__ << " Data Channel State: " << webrtc::DataChannelInterface::DataStateString(mDataChannel->state()); + switch (mDataChannel->state()) + { + case webrtc::DataChannelInterface::kOpen: + RTC_LOG(LS_INFO) << __FUNCTION__ << " Data Channel State Open"; + break; + case webrtc::DataChannelInterface::kConnecting: + RTC_LOG(LS_INFO) << __FUNCTION__ << " Data Channel State Connecting"; + break; + case webrtc::DataChannelInterface::kClosing: + RTC_LOG(LS_INFO) << __FUNCTION__ << " Data Channel State closing"; + break; + case webrtc::DataChannelInterface::kClosed: + RTC_LOG(LS_INFO) << __FUNCTION__ << " Data Channel State closed"; + break; + default: + break; + } +} + + void LLWebRTCImpl::OnMessage(const webrtc::DataBuffer& buffer) { std::string data((const char*)buffer.data.cdata(), buffer.size()); @@ -632,4 +742,12 @@ void init() gWebRTCImpl->AddRef(); gWebRTCImpl->init(); } + +void terminate() +{ + gWebRTCImpl->terminate(); + gWebRTCImpl->Release(); + gWebRTCImpl = nullptr; +} + } // namespace llwebrtc -- cgit v1.2.3 From 3f1fa296696086efe618bdbd3c6f636b6e1163b1 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Thu, 21 Sep 2023 22:33:57 -0700 Subject: deal with add/remove of participants more effectively. --- indra/llwebrtc/llwebrtc.cpp | 5 ----- 1 file changed, 5 deletions(-) (limited to 'indra/llwebrtc/llwebrtc.cpp') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index 77b050cbd0..1913786c0b 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -87,11 +87,6 @@ void LLWebRTCImpl::terminate() mDeviceModule = nullptr; } }); - - mNetworkThread->Stop(); - mWorkerThread->Stop(); - mSignalingThread->Stop(); - } -- cgit v1.2.3 From 64ec3cb19ffeeb25d7d235e60629c9b4890986f1 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Thu, 21 Sep 2023 23:23:49 -0700 Subject: send a message to the server when we're ready for data channel data --- indra/llwebrtc/llwebrtc.cpp | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) (limited to 'indra/llwebrtc/llwebrtc.cpp') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index 1913786c0b..dc746b7629 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -78,13 +78,22 @@ void LLWebRTCImpl::terminate() mPeerConnection->Close(); mPeerConnection = nullptr; } + mPeerConnectionFactory = nullptr; }); mWorkerThread->BlockingCall( [this]() { - if (mDeviceModule) + mDeviceModule = nullptr; + mTaskQueueFactory = nullptr; + + }); + mNetworkThread->BlockingCall( + [this]() + { + if (mDataChannel) { - mDeviceModule = nullptr; + mDataChannel->Close(); + mDataChannel = nullptr; } }); } @@ -681,6 +690,10 @@ void LLWebRTCImpl::OnStateChange() { case webrtc::DataChannelInterface::kOpen: RTC_LOG(LS_INFO) << __FUNCTION__ << " Data Channel State Open"; + for (auto &observer : mDataObserverList) + { + observer->OnDataChannelReady(); + } break; case webrtc::DataChannelInterface::kConnecting: RTC_LOG(LS_INFO) << __FUNCTION__ << " Data Channel State Connecting"; -- cgit v1.2.3 From cf72241f97627c59b5d5ab414caf086bffd47dc1 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Fri, 22 Sep 2023 10:47:48 -0700 Subject: Fix shutdown crash issue. --- indra/llwebrtc/llwebrtc.cpp | 72 ++++++++++++++++++++++----------------------- 1 file changed, 36 insertions(+), 36 deletions(-) (limited to 'indra/llwebrtc/llwebrtc.cpp') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index dc746b7629..a3dadd696c 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -39,6 +39,34 @@ namespace llwebrtc const float VOLUME_SCALE_WEBRTC = 3.0f; + +double LLAudioDeviceObserver::getMicrophoneEnergy() { return mMicrophoneEnergy; } + +void LLAudioDeviceObserver::OnCaptureData(const void *audio_samples, + const size_t num_samples, + const size_t bytes_per_sample, + const size_t num_channels, + const uint32_t samples_per_sec) +{ + double energy = 0; + const short *samples = (const short *) audio_samples; + for (size_t index = 0; index < num_samples * num_channels; index++) + { + double sample = (static_cast(samples[index]) / (double) 32768); + energy += sample * sample; + } + mMicrophoneEnergy = std::sqrt(energy); +} + +void LLAudioDeviceObserver::OnRenderData(const void *audio_samples, + const size_t num_samples, + const size_t bytes_per_sample, + const size_t num_channels, + const uint32_t samples_per_sec) +{ +} + + void LLWebRTCImpl::init() { mAnswerReceived = false; @@ -58,9 +86,10 @@ void LLWebRTCImpl::init() mWorkerThread->PostTask( [this]() { + mAudioDeviceObserver = new LLAudioDeviceObserver; mDeviceModule = webrtc::CreateAudioDeviceWithDataObserver(webrtc::AudioDeviceModule::AudioLayer::kPlatformDefaultAudio, mTaskQueueFactory.get(), - std::unique_ptr(this)); + std::unique_ptr(mAudioDeviceObserver)); mDeviceModule->Init(); mDeviceModule->SetStereoRecording(false); mDeviceModule->EnableBuiltInAEC(false); @@ -83,6 +112,10 @@ void LLWebRTCImpl::terminate() mWorkerThread->BlockingCall( [this]() { + if (mDeviceModule) + { + mDeviceModule->Terminate(); + } mDeviceModule = nullptr; mTaskQueueFactory = nullptr; @@ -231,32 +264,6 @@ void LLWebRTCImpl::setTuningMode(bool enable) }); } -double LLWebRTCImpl::getTuningMicrophoneEnergy() { return mTuningEnergy; } - -void LLWebRTCImpl::OnCaptureData(const void *audio_samples, - const size_t num_samples, - const size_t bytes_per_sample, - const size_t num_channels, - const uint32_t samples_per_sec) -{ - double energy = 0; - const short *samples = (const short *) audio_samples; - for (size_t index = 0; index < num_samples * num_channels; index++) - { - double sample = (static_cast(samples[index]) / (double) 32768); - energy += sample * sample; - } - mTuningEnergy = std::sqrt(energy); -} - -void LLWebRTCImpl::OnRenderData(const void *audio_samples, - const size_t num_samples, - const size_t bytes_per_sample, - const size_t num_channels, - const uint32_t samples_per_sec) -{ -} - // // LLWebRTCSignalInterface // @@ -470,16 +477,9 @@ void LLWebRTCImpl::setSpeakerVolume(float volume) }); } -void LLWebRTCImpl::requestAudioLevel() +double LLWebRTCImpl::getAudioLevel() { - mWorkerThread->PostTask( - [this]() - { - for (auto &observer : mAudioObserverList) - { - observer->OnAudioLevel((float)mTuningEnergy); - } - }); + return mAudioDeviceObserver->getMicrophoneEnergy(); } // -- cgit v1.2.3 From 78228d4dc423478c153aeeab0d92acf9ce5de78b Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Fri, 22 Sep 2023 16:55:13 -0700 Subject: Smooth voice power level reporting. --- indra/llwebrtc/llwebrtc.cpp | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) (limited to 'indra/llwebrtc/llwebrtc.cpp') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index a3dadd696c..7b5ca7fb16 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -39,6 +39,7 @@ namespace llwebrtc const float VOLUME_SCALE_WEBRTC = 3.0f; +LLAudioDeviceObserver::LLAudioDeviceObserver() : mMicrophoneEnergy(0.0), mSumVector {0} {} double LLAudioDeviceObserver::getMicrophoneEnergy() { return mMicrophoneEnergy; } @@ -48,14 +49,26 @@ void LLAudioDeviceObserver::OnCaptureData(const void *audio_samples, const size_t num_channels, const uint32_t samples_per_sec) { - double energy = 0; + float energy = 0; const short *samples = (const short *) audio_samples; for (size_t index = 0; index < num_samples * num_channels; index++) { - double sample = (static_cast(samples[index]) / (double) 32768); + float sample = (static_cast(samples[index]) / (float) 32768); energy += sample * sample; } - mMicrophoneEnergy = std::sqrt(energy); + + // smooth it. + size_t buffer_size = sizeof(mSumVector) / sizeof(mSumVector[0]); + float totalSum = 0; + int i; + for (i = 0; i < (buffer_size - 1); i++) + { + mSumVector[i] = mSumVector[i + 1]; + totalSum += mSumVector[i]; + } + mSumVector[i] = energy; + totalSum += energy; + mMicrophoneEnergy = std::sqrt(totalSum / (num_samples * buffer_size)); } void LLAudioDeviceObserver::OnRenderData(const void *audio_samples, @@ -479,7 +492,7 @@ void LLWebRTCImpl::setSpeakerVolume(float volume) double LLWebRTCImpl::getAudioLevel() { - return mAudioDeviceObserver->getMicrophoneEnergy(); + return 20*mAudioDeviceObserver->getMicrophoneEnergy(); } // -- cgit v1.2.3 From 819a715a59b9926f64ba62b28299849868e577e2 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Tue, 26 Sep 2023 16:37:24 -0700 Subject: sdd stereo support to client --- indra/llwebrtc/llwebrtc.cpp | 43 ++++++++++++++++++++++++++++++++----------- 1 file changed, 32 insertions(+), 11 deletions(-) (limited to 'indra/llwebrtc/llwebrtc.cpp') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index 7b5ca7fb16..e3fd68d35b 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -26,6 +26,7 @@ #include "llwebrtc_impl.h" #include +#include #include "api/audio_codecs/audio_decoder_factory.h" #include "api/audio_codecs/audio_encoder_factory.h" @@ -394,13 +395,28 @@ bool LLWebRTCImpl::initializeConnectionThreaded() codecparam.name = "opus"; codecparam.kind = cricket::MEDIA_TYPE_AUDIO; codecparam.clock_rate = 48000; - codecparam.num_channels = 1; - codecparam.parameters["stereo"] = "0"; - codecparam.parameters["sprop-stereo"] = "0"; + codecparam.num_channels = 2; + codecparam.parameters["stereo"] = "1"; + codecparam.parameters["sprop-stereo"] = "1"; params.codecs.push_back(codecparam); sender->SetParameters(params); } + auto receivers = mPeerConnection->GetReceivers(); + for (auto& receiver : receivers) + { + webrtc::RtpParameters params; + webrtc::RtpCodecParameters codecparam; + codecparam.name = "opus"; + codecparam.kind = cricket::MEDIA_TYPE_AUDIO; + codecparam.clock_rate = 48000; + codecparam.num_channels = 2; + codecparam.parameters["stereo"] = "1"; + codecparam.parameters["sprop-stereo"] = "1"; + params.codecs.push_back(codecparam); + receiver->SetParameters(params); + } + mPeerConnection->SetLocalDescription(rtc::scoped_refptr(this)); RTC_LOG(LS_INFO) << __FUNCTION__ << " " << mPeerConnection->signaling_state(); @@ -416,12 +432,9 @@ void LLWebRTCImpl::shutdownConnection() void LLWebRTCImpl::AnswerAvailable(const std::string &sdp) { - std::istringstream sdp_stream(sdp); - std::string sdp_line; - while (std::getline(sdp_stream, sdp_line)) - { - RTC_LOG(LS_INFO) << __FUNCTION__ << " Remote SDP: " << sdp_line; - } + + RTC_LOG(LS_INFO) << __FUNCTION__ << " Remote SDP: " << sdp; + mSignalingThread->PostTask( [this, sdp]() { @@ -661,20 +674,28 @@ void LLWebRTCImpl::OnSetLocalDescriptionComplete(webrtc::RTCError error) std::istringstream sdp_stream(sdp); std::ostringstream sdp_mangled_stream; std::string sdp_line; + int opus_payload = 0; while (std::getline(sdp_stream, sdp_line)) { int bandwidth = 0; int payload_id = 0; - RTC_LOG(LS_INFO) << __FUNCTION__ << " Local SDP: " << sdp_line; - // force mono + // force mono down, stereo up if (std::sscanf(sdp_line.c_str(), "a=rtpmap:%i opus/%i/2", &payload_id, &bandwidth) == 2) { sdp_mangled_stream << sdp_line << "\n"; + opus_payload = payload_id; + } + else if (sdp_line.rfind(std::format("a=fmtp:{}", opus_payload)) == 0) + { + sdp_mangled_stream << sdp_line << "a=fmtp:" << opus_payload + << " stereo=1;sprop-stereo=0;minptime=10;useinbandfec=1;maxplaybackrate=48000\n"; } else { sdp_mangled_stream << sdp_line << "\n"; } } + RTC_LOG(LS_INFO) << __FUNCTION__ << " Local SDP: " << sdp_mangled_stream.str(); + ; for (auto &observer : mSignalingObserverList) { observer->OnOfferAvailable(sdp_mangled_stream.str()); -- cgit v1.2.3 From 861f21957ef164831f57fbd59fce55cb5eaca641 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Tue, 26 Sep 2023 19:44:10 -0700 Subject: fix osx build incompatibility --- indra/llwebrtc/llwebrtc.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'indra/llwebrtc/llwebrtc.cpp') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index e3fd68d35b..b02354cbac 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -674,7 +674,7 @@ void LLWebRTCImpl::OnSetLocalDescriptionComplete(webrtc::RTCError error) std::istringstream sdp_stream(sdp); std::ostringstream sdp_mangled_stream; std::string sdp_line; - int opus_payload = 0; + char opus_payload[10]; while (std::getline(sdp_stream, sdp_line)) { int bandwidth = 0; int payload_id = 0; @@ -682,9 +682,9 @@ void LLWebRTCImpl::OnSetLocalDescriptionComplete(webrtc::RTCError error) if (std::sscanf(sdp_line.c_str(), "a=rtpmap:%i opus/%i/2", &payload_id, &bandwidth) == 2) { sdp_mangled_stream << sdp_line << "\n"; - opus_payload = payload_id; + sprintf(opus_payload,"%d",payload_id); } - else if (sdp_line.rfind(std::format("a=fmtp:{}", opus_payload)) == 0) + else if (sdp_line.rfind(std::string("a=fmtp:") + opus_payload) == 0) { sdp_mangled_stream << sdp_line << "a=fmtp:" << opus_payload << " stereo=1;sprop-stereo=0;minptime=10;useinbandfec=1;maxplaybackrate=48000\n"; -- cgit v1.2.3 From e147381eb3d977eeed9b45621e2f40d0ff379607 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Wed, 27 Sep 2023 18:16:57 -0700 Subject: add stereo support --- indra/llwebrtc/llwebrtc.cpp | 87 +++++++++++++++++++++++++++++---------------- 1 file changed, 56 insertions(+), 31 deletions(-) (limited to 'indra/llwebrtc/llwebrtc.cpp') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index b02354cbac..3152e1eef6 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -106,6 +106,7 @@ void LLWebRTCImpl::init() std::unique_ptr(mAudioDeviceObserver)); mDeviceModule->Init(); mDeviceModule->SetStereoRecording(false); + mDeviceModule->SetStereoPlayout(true); mDeviceModule->EnableBuiltInAEC(false); updateDevices(); }); @@ -321,6 +322,8 @@ bool LLWebRTCImpl::initializeConnectionThreaded() apm_config.noise_suppression.enabled = true; apm_config.noise_suppression.level = webrtc::AudioProcessing::Config::NoiseSuppression::kVeryHigh; apm_config.transient_suppression.enabled = true; + apm_config.pipeline.multi_channel_render = true; + apm_config.pipeline.multi_channel_capture = true; // apm->ApplyConfig(apm_config); @@ -416,8 +419,8 @@ bool LLWebRTCImpl::initializeConnectionThreaded() params.codecs.push_back(codecparam); receiver->SetParameters(params); } - - mPeerConnection->SetLocalDescription(rtc::scoped_refptr(this)); + webrtc::PeerConnectionInterface::RTCOfferAnswerOptions offerOptions; + mPeerConnection->CreateOffer(this, offerOptions); RTC_LOG(LS_INFO) << __FUNCTION__ << " " << mPeerConnection->signaling_state(); @@ -516,6 +519,16 @@ void LLWebRTCImpl::OnAddTrack(rtc::scoped_refptr const std::vector> &streams) { RTC_LOG(LS_INFO) << __FUNCTION__ << " " << receiver->id(); + webrtc::RtpParameters params; + webrtc::RtpCodecParameters codecparam; + codecparam.name = "opus"; + codecparam.kind = cricket::MEDIA_TYPE_AUDIO; + codecparam.clock_rate = 48000; + codecparam.num_channels = 2; + codecparam.parameters["stereo"] = "1"; + codecparam.parameters["sprop-stereo"] = "1"; + params.codecs.push_back(codecparam); + receiver->SetParameters(params); } void LLWebRTCImpl::OnRemoveTrack(rtc::scoped_refptr receiver) @@ -632,11 +645,47 @@ void LLWebRTCImpl::OnSuccess(webrtc::SessionDescriptionInterface *desc) std::string sdp; desc->ToString(&sdp); RTC_LOG(LS_INFO) << sdp; +; + // mangle the sdp as this is the only way currently to bump up + // the send audio rate to 48k + std::istringstream sdp_stream(sdp); + std::ostringstream sdp_mangled_stream; + std::string sdp_line; + int opus_payload = 0; + while (std::getline(sdp_stream, sdp_line)) + { + int bandwidth = 0; + int payload_id = 0; + // force mono down, stereo up + if (std::sscanf(sdp_line.c_str(), "a=rtpmap:%i opus/%i/2", &payload_id, &bandwidth) == 2) + { + sdp_mangled_stream << sdp_line << "\n"; + opus_payload = payload_id; + } + else if (sdp_line.rfind(std::format("a=fmtp:{}", opus_payload)) == 0) + { + sdp_mangled_stream << sdp_line << "a=fmtp:" << opus_payload + << " minptime=10;useinbandfec=1;stereo=1;sprop-stereo=1;maxplaybackrate=48000\n"; + } + else + { + sdp_mangled_stream << sdp_line << "\n"; + } + } + + webrtc::CreateSessionDescription(webrtc::SdpType::kOffer, sdp_mangled_stream.str()); + + + + mPeerConnection->SetLocalDescription(std::unique_ptr( + webrtc::CreateSessionDescription(webrtc::SdpType::kOffer, sdp_mangled_stream.str())), + rtc::scoped_refptr(this)); + RTC_LOG(LS_INFO) << __FUNCTION__ << " Local SDP: " << sdp_mangled_stream.str(); + - RTC_LOG(LS_INFO) << __FUNCTION__ << " " << mPeerConnection->signaling_state(); for (auto &observer : mSignalingObserverList) { - observer->OnOfferAvailable(sdp); + observer->OnOfferAvailable(sdp_mangled_stream.str()); } } @@ -669,36 +718,12 @@ void LLWebRTCImpl::OnSetLocalDescriptionComplete(webrtc::RTCError error) auto desc = mPeerConnection->pending_local_description(); std::string sdp; desc->ToString(&sdp); - // mangle the sdp as this is the only way currently to bump up - // the send audio rate to 48k - std::istringstream sdp_stream(sdp); - std::ostringstream sdp_mangled_stream; - std::string sdp_line; - char opus_payload[10]; - while (std::getline(sdp_stream, sdp_line)) { - int bandwidth = 0; - int payload_id = 0; - // force mono down, stereo up - if (std::sscanf(sdp_line.c_str(), "a=rtpmap:%i opus/%i/2", &payload_id, &bandwidth) == 2) - { - sdp_mangled_stream << sdp_line << "\n"; - sprintf(opus_payload,"%d",payload_id); - } - else if (sdp_line.rfind(std::string("a=fmtp:") + opus_payload) == 0) - { - sdp_mangled_stream << sdp_line << "a=fmtp:" << opus_payload - << " stereo=1;sprop-stereo=0;minptime=10;useinbandfec=1;maxplaybackrate=48000\n"; - } - else - { - sdp_mangled_stream << sdp_line << "\n"; - } - } - RTC_LOG(LS_INFO) << __FUNCTION__ << " Local SDP: " << sdp_mangled_stream.str(); + + RTC_LOG(LS_INFO) << __FUNCTION__ << " Local SDP: " << sdp; ; for (auto &observer : mSignalingObserverList) { - observer->OnOfferAvailable(sdp_mangled_stream.str()); + observer->OnOfferAvailable(sdp); } } -- cgit v1.2.3 From 976f63d6aec1be8a931b2c3648b610322a4d80ab Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Wed, 27 Sep 2023 18:42:22 -0700 Subject: fix mac build --- indra/llwebrtc/llwebrtc.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'indra/llwebrtc/llwebrtc.cpp') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index 3152e1eef6..481d5b940d 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -651,7 +651,7 @@ void LLWebRTCImpl::OnSuccess(webrtc::SessionDescriptionInterface *desc) std::istringstream sdp_stream(sdp); std::ostringstream sdp_mangled_stream; std::string sdp_line; - int opus_payload = 0; + std::string opus_payload; while (std::getline(sdp_stream, sdp_line)) { int bandwidth = 0; @@ -660,9 +660,9 @@ void LLWebRTCImpl::OnSuccess(webrtc::SessionDescriptionInterface *desc) if (std::sscanf(sdp_line.c_str(), "a=rtpmap:%i opus/%i/2", &payload_id, &bandwidth) == 2) { sdp_mangled_stream << sdp_line << "\n"; - opus_payload = payload_id; + opus_payload = std::to_string(payload_id); } - else if (sdp_line.rfind(std::format("a=fmtp:{}", opus_payload)) == 0) + else if (sdp_line.find("a=fmtp:" + opus_payload) == 0) { sdp_mangled_stream << sdp_line << "a=fmtp:" << opus_payload << " minptime=10;useinbandfec=1;stereo=1;sprop-stereo=1;maxplaybackrate=48000\n"; -- cgit v1.2.3 From 8bd96753113a95aaa8c92fe78713498cbc42a437 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Fri, 29 Sep 2023 19:13:03 -0700 Subject: Improve reconnection logic and allow device setting when connected or not connected --- indra/llwebrtc/llwebrtc.cpp | 268 +++++++++++++++++++++++++++----------------- 1 file changed, 167 insertions(+), 101 deletions(-) (limited to 'indra/llwebrtc/llwebrtc.cpp') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index 481d5b940d..95f87ed65b 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -42,15 +42,15 @@ const float VOLUME_SCALE_WEBRTC = 3.0f; LLAudioDeviceObserver::LLAudioDeviceObserver() : mMicrophoneEnergy(0.0), mSumVector {0} {} -double LLAudioDeviceObserver::getMicrophoneEnergy() { return mMicrophoneEnergy; } +float LLAudioDeviceObserver::getMicrophoneEnergy() { return mMicrophoneEnergy; } void LLAudioDeviceObserver::OnCaptureData(const void *audio_samples, - const size_t num_samples, - const size_t bytes_per_sample, - const size_t num_channels, - const uint32_t samples_per_sec) + const size_t num_samples, + const size_t bytes_per_sample, + const size_t num_channels, + const uint32_t samples_per_sec) { - float energy = 0; + float energy = 0; const short *samples = (const short *) audio_samples; for (size_t index = 0; index < num_samples * num_channels; index++) { @@ -60,8 +60,8 @@ void LLAudioDeviceObserver::OnCaptureData(const void *audio_samples, // smooth it. size_t buffer_size = sizeof(mSumVector) / sizeof(mSumVector[0]); - float totalSum = 0; - int i; + float totalSum = 0; + int i; for (i = 0; i < (buffer_size - 1); i++) { mSumVector[i] = mSumVector[i + 1]; @@ -73,16 +73,17 @@ void LLAudioDeviceObserver::OnCaptureData(const void *audio_samples, } void LLAudioDeviceObserver::OnRenderData(const void *audio_samples, - const size_t num_samples, - const size_t bytes_per_sample, - const size_t num_channels, - const uint32_t samples_per_sec) + const size_t num_samples, + const size_t bytes_per_sample, + const size_t num_channels, + const uint32_t samples_per_sec) { } - void LLWebRTCImpl::init() { + mPlayoutDevice = -1; + mRecordingDevice = -1; mAnswerReceived = false; rtc::InitializeSSL(); mTaskQueueFactory = webrtc::CreateDefaultTaskQueueFactory(); @@ -100,20 +101,21 @@ void LLWebRTCImpl::init() mWorkerThread->PostTask( [this]() { - mAudioDeviceObserver = new LLAudioDeviceObserver; - mDeviceModule = webrtc::CreateAudioDeviceWithDataObserver(webrtc::AudioDeviceModule::AudioLayer::kPlatformDefaultAudio, - mTaskQueueFactory.get(), - std::unique_ptr(mAudioDeviceObserver)); - mDeviceModule->Init(); - mDeviceModule->SetStereoRecording(false); - mDeviceModule->SetStereoPlayout(true); - mDeviceModule->EnableBuiltInAEC(false); + mTuningAudioDeviceObserver = new LLAudioDeviceObserver; + mTuningDeviceModule = + webrtc::CreateAudioDeviceWithDataObserver(webrtc::AudioDeviceModule::AudioLayer::kPlatformDefaultAudio, + mTaskQueueFactory.get(), + std::unique_ptr(mTuningAudioDeviceObserver)); + mTuningDeviceModule->Init(); + mTuningDeviceModule->SetStereoRecording(false); + mTuningDeviceModule->SetStereoPlayout(true); + mTuningDeviceModule->EnableBuiltInAEC(false); updateDevices(); }); } -void LLWebRTCImpl::terminate() -{ +void LLWebRTCImpl::terminate() +{ mSignalingThread->BlockingCall( [this]() { @@ -127,13 +129,17 @@ void LLWebRTCImpl::terminate() mWorkerThread->BlockingCall( [this]() { - if (mDeviceModule) + if (mTuningDeviceModule) { - mDeviceModule->Terminate(); + mTuningDeviceModule->Terminate(); } - mDeviceModule = nullptr; - mTaskQueueFactory = nullptr; - + if (mPeerDeviceModule) + { + mPeerDeviceModule->Terminate(); + } + mTuningDeviceModule = nullptr; + mPeerDeviceModule = nullptr; + mTaskQueueFactory = nullptr; }); mNetworkThread->BlockingCall( [this]() @@ -146,7 +152,6 @@ void LLWebRTCImpl::terminate() }); } - void LLWebRTCImpl::refreshDevices() { mWorkerThread->PostTask([this]() { updateDevices(); }); @@ -169,32 +174,50 @@ void LLWebRTCImpl::setCaptureDevice(const std::string &id) mWorkerThread->PostTask( [this, id]() { - bool was_recording = mDeviceModule->Recording(); + bool was_tuning_recording = mTuningDeviceModule->Recording(); - if (was_recording) + if (was_tuning_recording) { - mDeviceModule->StopRecording(); + mTuningDeviceModule->StopRecording(); } - int16_t captureDeviceCount = mDeviceModule->RecordingDevices(); - int16_t index = 0; /* default to first one if no match */ + bool was_peer_recording = false; + if (mPeerDeviceModule) + { + was_peer_recording = mPeerDeviceModule->Recording(); + if (was_peer_recording) + { + mPeerDeviceModule->StopRecording(); + } + } + int16_t captureDeviceCount = mTuningDeviceModule->RecordingDevices(); for (int16_t i = 0; i < captureDeviceCount; i++) { char name[webrtc::kAdmMaxDeviceNameSize]; char guid[webrtc::kAdmMaxGuidSize]; - mDeviceModule->RecordingDeviceName(i, name, guid); - if (id == guid || id == "Default") + mTuningDeviceModule->RecordingDeviceName(i, name, guid); + if (id == guid || id == "Default") // first one in list is default { RTC_LOG(LS_INFO) << __FUNCTION__ << "Set recording device to " << name << " " << guid << " " << i; - index = i; + mRecordingDevice = i; break; } } - mDeviceModule->SetRecordingDevice(index); - mDeviceModule->InitMicrophone(); - mDeviceModule->InitRecording(); - if (was_recording) + mTuningDeviceModule->SetRecordingDevice(mRecordingDevice); + mTuningDeviceModule->InitMicrophone(); + mTuningDeviceModule->InitRecording(); + if (was_tuning_recording) { - mDeviceModule->StartRecording(); + mTuningDeviceModule->StartRecording(); + } + if (mPeerDeviceModule) + { + mPeerDeviceModule->SetRecordingDevice(mRecordingDevice); + mPeerDeviceModule->InitMicrophone(); + mPeerDeviceModule->InitRecording(); + if (was_peer_recording) + { + mPeerDeviceModule->StartRecording(); + } } }); } @@ -204,54 +227,72 @@ void LLWebRTCImpl::setRenderDevice(const std::string &id) mWorkerThread->PostTask( [this, id]() { - bool was_playing = mDeviceModule->Playing(); - if (was_playing) + bool was_tuning_playing = mTuningDeviceModule->Playing(); + if (was_tuning_playing) { - mDeviceModule->StopPlayout(); + mTuningDeviceModule->StopPlayout(); + } + bool was_peer_playing = false; + if (mPeerDeviceModule) + { + was_peer_playing = mPeerDeviceModule->Playing(); + if (was_tuning_playing) + { + mPeerDeviceModule->StopPlayout(); + } } - int16_t renderDeviceCount = mDeviceModule->PlayoutDevices(); - int16_t index = 0; /* default to first one if no match */ + int16_t renderDeviceCount = mTuningDeviceModule->PlayoutDevices(); for (int16_t i = 0; i < renderDeviceCount; i++) { char name[webrtc::kAdmMaxDeviceNameSize]; char guid[webrtc::kAdmMaxGuidSize]; - mDeviceModule->PlayoutDeviceName(i, name, guid); + mTuningDeviceModule->PlayoutDeviceName(i, name, guid); if (id == guid || id == "Default") { RTC_LOG(LS_INFO) << __FUNCTION__ << "Set recording device to " << name << " " << guid << " " << i; - index = i; + mPlayoutDevice = i; break; } } - mDeviceModule->SetPlayoutDevice(index); - mDeviceModule->InitSpeaker(); - mDeviceModule->InitPlayout(); - if (was_playing) + mTuningDeviceModule->SetPlayoutDevice(mPlayoutDevice); + mTuningDeviceModule->InitSpeaker(); + mTuningDeviceModule->InitPlayout(); + if (was_tuning_playing) { - mDeviceModule->StartPlayout(); + mTuningDeviceModule->StartPlayout(); + } + if (mPeerDeviceModule) + { + mPeerDeviceModule->SetPlayoutDevice(mPlayoutDevice); + mPeerDeviceModule->InitSpeaker(); + mPeerDeviceModule->InitPlayout(); + if (was_peer_playing) + { + mPeerDeviceModule->StartPlayout(); + } } }); } void LLWebRTCImpl::updateDevices() { - int16_t renderDeviceCount = mDeviceModule->PlayoutDevices(); + int16_t renderDeviceCount = mTuningDeviceModule->PlayoutDevices(); LLWebRTCVoiceDeviceList renderDeviceList; for (int16_t index = 0; index < renderDeviceCount; index++) { char name[webrtc::kAdmMaxDeviceNameSize]; char guid[webrtc::kAdmMaxGuidSize]; - mDeviceModule->PlayoutDeviceName(index, name, guid); + mTuningDeviceModule->PlayoutDeviceName(index, name, guid); renderDeviceList.emplace_back(name, guid); } - int16_t captureDeviceCount = mDeviceModule->RecordingDevices(); + int16_t captureDeviceCount = mTuningDeviceModule->RecordingDevices(); LLWebRTCVoiceDeviceList captureDeviceList; for (int16_t index = 0; index < captureDeviceCount; index++) { char name[webrtc::kAdmMaxDeviceNameSize]; char guid[webrtc::kAdmMaxGuidSize]; - mDeviceModule->RecordingDeviceName(index, name, guid); + mTuningDeviceModule->RecordingDeviceName(index, name, guid); captureDeviceList.emplace_back(name, guid); } for (auto &observer : mVoiceDevicesObserverList) @@ -267,14 +308,22 @@ void LLWebRTCImpl::setTuningMode(bool enable) { if (enable) { - mDeviceModule->InitMicrophone(); - mDeviceModule->InitRecording(); - mDeviceModule->StartRecording(); - mDeviceModule->SetMicrophoneMute(false); + mTuningDeviceModule->StartRecording(); + mTuningDeviceModule->SetMicrophoneMute(false); + if (mPeerDeviceModule) + { + mPeerDeviceModule->StopRecording(); + mPeerDeviceModule->StopPlayout(); + } } else { - mDeviceModule->StopRecording(); + mTuningDeviceModule->StopRecording(); + if (mPeerDeviceModule) + { + mPeerDeviceModule->StartPlayout(); + mPeerDeviceModule->StartRecording(); + } } }); } @@ -295,42 +344,58 @@ void LLWebRTCImpl::unsetSignalingObserver(LLWebRTCSignalingObserver *observer) } } - bool LLWebRTCImpl::initializeConnection() { RTC_DCHECK(!mPeerConnection); RTC_DCHECK(mPeerConnectionFactory); - mAnswerReceived = false; + mAnswerReceived = false; mSignalingThread->PostTask([this]() { initializeConnectionThreaded(); }); return true; } - - bool LLWebRTCImpl::initializeConnectionThreaded() { rtc::scoped_refptr apm = webrtc::AudioProcessingBuilder().Create(); - webrtc::AudioProcessing::Config apm_config; - apm_config.echo_canceller.enabled = false; - apm_config.echo_canceller.mobile_mode = false; - apm_config.gain_controller1.enabled = true; - apm_config.gain_controller1.mode = - webrtc::AudioProcessing::Config::GainController1::kAdaptiveAnalog; - apm_config.gain_controller2.enabled = true; - apm_config.high_pass_filter.enabled = true; - apm_config.noise_suppression.enabled = true; - apm_config.noise_suppression.level = webrtc::AudioProcessing::Config::NoiseSuppression::kVeryHigh; - apm_config.transient_suppression.enabled = true; - apm_config.pipeline.multi_channel_render = true; + webrtc::AudioProcessing::Config apm_config; + apm_config.echo_canceller.enabled = false; + apm_config.echo_canceller.mobile_mode = false; + apm_config.gain_controller1.enabled = true; + apm_config.gain_controller1.mode = webrtc::AudioProcessing::Config::GainController1::kAdaptiveAnalog; + apm_config.gain_controller2.enabled = true; + apm_config.high_pass_filter.enabled = true; + apm_config.noise_suppression.enabled = true; + apm_config.noise_suppression.level = webrtc::AudioProcessing::Config::NoiseSuppression::kVeryHigh; + apm_config.transient_suppression.enabled = true; + apm_config.pipeline.multi_channel_render = true; apm_config.pipeline.multi_channel_capture = true; // apm->ApplyConfig(apm_config); + mWorkerThread->BlockingCall( + [this]() + { + mPeerAudioDeviceObserver = new LLAudioDeviceObserver; + mPeerDeviceModule = + webrtc::CreateAudioDeviceWithDataObserver(webrtc::AudioDeviceModule::AudioLayer::kPlatformDefaultAudio, + mTaskQueueFactory.get(), + std::unique_ptr(mPeerAudioDeviceObserver)); + mPeerDeviceModule->Init(); + mPeerDeviceModule->SetPlayoutDevice(mPlayoutDevice); + mPeerDeviceModule->SetRecordingDevice(mRecordingDevice); + mPeerDeviceModule->SetStereoRecording(false); + mPeerDeviceModule->SetStereoPlayout(true); + mPeerDeviceModule->EnableBuiltInAEC(false); + mPeerDeviceModule->InitMicrophone(); + mPeerDeviceModule->InitSpeaker(); + mPeerDeviceModule->InitRecording(); + mPeerDeviceModule->InitPlayout(); + }); + mPeerConnectionFactory = webrtc::CreatePeerConnectionFactory(mNetworkThread.get(), mWorkerThread.get(), mSignalingThread.get(), - mDeviceModule, + mPeerDeviceModule, webrtc::CreateBuiltinAudioEncoderFactory(), webrtc::CreateBuiltinAudioDecoderFactory(), nullptr /* video_encoder_factory */, @@ -406,9 +471,9 @@ bool LLWebRTCImpl::initializeConnectionThreaded() } auto receivers = mPeerConnection->GetReceivers(); - for (auto& receiver : receivers) + for (auto &receiver : receivers) { - webrtc::RtpParameters params; + webrtc::RtpParameters params; webrtc::RtpCodecParameters codecparam; codecparam.name = "opus"; codecparam.kind = cricket::MEDIA_TYPE_AUDIO; @@ -423,19 +488,23 @@ bool LLWebRTCImpl::initializeConnectionThreaded() mPeerConnection->CreateOffer(this, offerOptions); RTC_LOG(LS_INFO) << __FUNCTION__ << " " << mPeerConnection->signaling_state(); - + return true; } void LLWebRTCImpl::shutdownConnection() { + mDataChannel->Close(); + mDataChannel = nullptr; + mPeerConnection->Close(); + mPeerDeviceModule->Terminate(); + mPeerDeviceModule = nullptr; mPeerConnection = nullptr; mPeerConnectionFactory = nullptr; } void LLWebRTCImpl::AnswerAvailable(const std::string &sdp) { - RTC_LOG(LS_INFO) << __FUNCTION__ << " Remote SDP: " << sdp; mSignalingThread->PostTask( @@ -450,7 +519,7 @@ void LLWebRTCImpl::AnswerAvailable(const std::string &sdp) for (auto &candidate : mCachedIceCandidates) { LLWebRTCIceCandidate ice_candidate; - ice_candidate.candidate = candidate->candidate().ToString(); + ice_candidate.candidate = candidate->candidate().ToString(); ice_candidate.mline_index = candidate->sdp_mline_index(); ice_candidate.sdp_mid = candidate->sdp_mid(); observer->OnIceCandidate(ice_candidate); @@ -470,14 +539,13 @@ void LLWebRTCImpl::AnswerAvailable(const std::string &sdp) void LLWebRTCImpl::setMute(bool mute) { mSignalingThread->PostTask( - [this,mute]() + [this, mute]() { auto senders = mPeerConnection->GetSenders(); - RTC_LOG(LS_INFO) << __FUNCTION__ << (mute ? "disabling" : "enabling") << " streams count " - << senders.size(); + RTC_LOG(LS_INFO) << __FUNCTION__ << (mute ? "disabling" : "enabling") << " streams count " << senders.size(); - for (auto& sender : senders) + for (auto &sender : senders) { sender->track()->set_enabled(!mute); } @@ -497,19 +565,15 @@ void LLWebRTCImpl::setSpeakerVolume(float volume) webrtc::MediaStreamTrackInterface *track = receiver->track().get(); if (track->kind() == webrtc::MediaStreamTrackInterface::kAudioKind) { - webrtc::AudioTrackInterface* audio_track = static_cast(track); - webrtc::AudioSourceInterface* source = audio_track->GetSource(); + webrtc::AudioTrackInterface *audio_track = static_cast(track); + webrtc::AudioSourceInterface *source = audio_track->GetSource(); source->SetVolume(VOLUME_SCALE_WEBRTC * volume); - } } }); } -double LLWebRTCImpl::getAudioLevel() -{ - return 20*mAudioDeviceObserver->getMicrophoneEnergy(); -} +float LLWebRTCImpl::getTuningAudioLevel() { return 20 * mTuningAudioDeviceObserver->getMicrophoneEnergy(); } // // PeerConnectionObserver implementation. @@ -519,7 +583,7 @@ void LLWebRTCImpl::OnAddTrack(rtc::scoped_refptr const std::vector> &streams) { RTC_LOG(LS_INFO) << __FUNCTION__ << " " << receiver->id(); - webrtc::RtpParameters params; + webrtc::RtpParameters params; webrtc::RtpCodecParameters codecparam; codecparam.name = "opus"; codecparam.kind = cricket::MEDIA_TYPE_AUDIO; @@ -539,10 +603,9 @@ void LLWebRTCImpl::OnRemoveTrack(rtc::scoped_refptr channel) { mDataChannel = channel; - channel->RegisterObserver(this); + channel->RegisterObserver(this); } - void LLWebRTCImpl::OnIceGatheringChange(webrtc::PeerConnectionInterface::IceGatheringState new_state) { LLWebRTCSignalingObserver::IceGatheringState webrtc_new_state = LLWebRTCSignalingObserver::IceGatheringState::ICE_GATHERING_NEW; @@ -584,8 +647,8 @@ void LLWebRTCImpl::OnConnectionChange(webrtc::PeerConnectionInterface::PeerConne if (new_state == webrtc::PeerConnectionInterface::PeerConnectionState::kConnected) { mWorkerThread->PostTask([this]() { - mDeviceModule->StartRecording(); - mDeviceModule->StartPlayout(); + mPeerDeviceModule->StartRecording(); + mPeerDeviceModule->StartPlayout(); for (auto &observer : mSignalingObserverList) { observer->OnAudioEstablished(this); @@ -595,6 +658,7 @@ void LLWebRTCImpl::OnConnectionChange(webrtc::PeerConnectionInterface::PeerConne break; } case webrtc::PeerConnectionInterface::PeerConnectionState::kFailed: + case webrtc::PeerConnectionInterface::PeerConnectionState::kDisconnected: { for (auto &observer : mSignalingObserverList) { @@ -738,6 +802,8 @@ void LLWebRTCImpl::unsetAudioObserver(LLWebRTCAudioObserver *observer) } } +float LLWebRTCImpl::getAudioLevel() { return 20 * mPeerAudioDeviceObserver->getMicrophoneEnergy(); } + // // DataChannelObserver implementation // -- cgit v1.2.3 From 6b29ced7071baf3fcf5fa2f12fd92cd2cbeda9a8 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Tue, 3 Oct 2023 15:51:37 -0700 Subject: fix device selection while speaking. --- indra/llwebrtc/llwebrtc.cpp | 107 +++++++++++++++++++++++--------------------- 1 file changed, 55 insertions(+), 52 deletions(-) (limited to 'indra/llwebrtc/llwebrtc.cpp') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index 95f87ed65b..eda25a3641 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -131,10 +131,12 @@ void LLWebRTCImpl::terminate() { if (mTuningDeviceModule) { + mTuningDeviceModule->StopRecording(); mTuningDeviceModule->Terminate(); } if (mPeerDeviceModule) { + mPeerDeviceModule->StopRecording(); mPeerDeviceModule->Terminate(); } mTuningDeviceModule = nullptr; @@ -174,43 +176,32 @@ void LLWebRTCImpl::setCaptureDevice(const std::string &id) mWorkerThread->PostTask( [this, id]() { - bool was_tuning_recording = mTuningDeviceModule->Recording(); - - if (was_tuning_recording) - { - mTuningDeviceModule->StopRecording(); - } - bool was_peer_recording = false; - if (mPeerDeviceModule) - { - was_peer_recording = mPeerDeviceModule->Recording(); - if (was_peer_recording) - { - mPeerDeviceModule->StopRecording(); - } - } int16_t captureDeviceCount = mTuningDeviceModule->RecordingDevices(); for (int16_t i = 0; i < captureDeviceCount; i++) { char name[webrtc::kAdmMaxDeviceNameSize]; char guid[webrtc::kAdmMaxGuidSize]; mTuningDeviceModule->RecordingDeviceName(i, name, guid); - if (id == guid || id == "Default") // first one in list is default + if (id == guid || id == "Default") // first one in list is default { RTC_LOG(LS_INFO) << __FUNCTION__ << "Set recording device to " << name << " " << guid << " " << i; mRecordingDevice = i; break; } } + mTuningDeviceModule->StopRecording(); mTuningDeviceModule->SetRecordingDevice(mRecordingDevice); mTuningDeviceModule->InitMicrophone(); mTuningDeviceModule->InitRecording(); - if (was_tuning_recording) - { - mTuningDeviceModule->StartRecording(); - } + mTuningDeviceModule->StartRecording(); + bool was_peer_recording = false; if (mPeerDeviceModule) { + was_peer_recording = mPeerDeviceModule->Recording(); + if (was_peer_recording) + { + mPeerDeviceModule->StopRecording(); + } mPeerDeviceModule->SetRecordingDevice(mRecordingDevice); mPeerDeviceModule->InitMicrophone(); mPeerDeviceModule->InitRecording(); @@ -227,20 +218,6 @@ void LLWebRTCImpl::setRenderDevice(const std::string &id) mWorkerThread->PostTask( [this, id]() { - bool was_tuning_playing = mTuningDeviceModule->Playing(); - if (was_tuning_playing) - { - mTuningDeviceModule->StopPlayout(); - } - bool was_peer_playing = false; - if (mPeerDeviceModule) - { - was_peer_playing = mPeerDeviceModule->Playing(); - if (was_tuning_playing) - { - mPeerDeviceModule->StopPlayout(); - } - } int16_t renderDeviceCount = mTuningDeviceModule->PlayoutDevices(); for (int16_t i = 0; i < renderDeviceCount; i++) { @@ -254,6 +231,22 @@ void LLWebRTCImpl::setRenderDevice(const std::string &id) break; } } + mTuningDeviceModule->SetSpeakerMute(true); + bool was_tuning_playing = mTuningDeviceModule->Playing(); + if (was_tuning_playing) + { + mTuningDeviceModule->StopPlayout(); + } + bool was_peer_mute = false; + if (mPeerDeviceModule) + { + mPeerDeviceModule->SpeakerMute(&was_peer_mute); + if (!was_peer_mute) + { + mPeerDeviceModule->SetSpeakerMute(true); + } + } + mTuningDeviceModule->SetPlayoutDevice(mPlayoutDevice); mTuningDeviceModule->InitSpeaker(); mTuningDeviceModule->InitPlayout(); @@ -266,11 +259,11 @@ void LLWebRTCImpl::setRenderDevice(const std::string &id) mPeerDeviceModule->SetPlayoutDevice(mPlayoutDevice); mPeerDeviceModule->InitSpeaker(); mPeerDeviceModule->InitPlayout(); - if (was_peer_playing) - { - mPeerDeviceModule->StartPlayout(); - } + mPeerDeviceModule->StartPlayout(); + mPeerDeviceModule->SetSpeakerMute(was_peer_mute); + } + mTuningDeviceModule->SetSpeakerMute(false); }); } @@ -303,29 +296,42 @@ void LLWebRTCImpl::updateDevices() void LLWebRTCImpl::setTuningMode(bool enable) { - mWorkerThread->PostTask( + mWorkerThread->BlockingCall( [this, enable]() { if (enable) { + mTuningDeviceModule->StartRecording(); mTuningDeviceModule->SetMicrophoneMute(false); + + + mTuningDeviceModule->SetSpeakerMute(false); + if (mPeerDeviceModule) { mPeerDeviceModule->StopRecording(); - mPeerDeviceModule->StopPlayout(); + mPeerDeviceModule->SetSpeakerMute(true); } } else { - mTuningDeviceModule->StopRecording(); if (mPeerDeviceModule) { - mPeerDeviceModule->StartPlayout(); mPeerDeviceModule->StartRecording(); + mPeerDeviceModule->SetSpeakerMute(false); } } }); + // set_enabled shouldn't be done on the worker thread + if (mPeerConnection) + { + auto senders = mPeerConnection->GetSenders(); + for (auto &sender : senders) + { + sender->track()->set_enabled(enable ? false : !mMute); + } + } } // @@ -538,18 +544,15 @@ void LLWebRTCImpl::AnswerAvailable(const std::string &sdp) void LLWebRTCImpl::setMute(bool mute) { - mSignalingThread->PostTask( - [this, mute]() - { - auto senders = mPeerConnection->GetSenders(); + mMute = mute; + auto senders = mPeerConnection->GetSenders(); - RTC_LOG(LS_INFO) << __FUNCTION__ << (mute ? "disabling" : "enabling") << " streams count " << senders.size(); + RTC_LOG(LS_INFO) << __FUNCTION__ << (mute ? "disabling" : "enabling") << " streams count " << senders.size(); - for (auto &sender : senders) - { - sender->track()->set_enabled(!mute); - } - }); + for (auto &sender : senders) + { + sender->track()->set_enabled(!mMute); + } } void LLWebRTCImpl::setSpeakerVolume(float volume) -- cgit v1.2.3 From a12e23f349b782ab669e218da8e1af10d9408996 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Fri, 6 Oct 2023 21:46:02 -0700 Subject: Fix race in initialization. Fix failure to send ice candidates to janus. --- indra/llwebrtc/llwebrtc.cpp | 95 +++++++++++++++++++++++++++------------------ 1 file changed, 58 insertions(+), 37 deletions(-) (limited to 'indra/llwebrtc/llwebrtc.cpp') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index eda25a3641..d8f2a6028c 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -356,12 +356,6 @@ bool LLWebRTCImpl::initializeConnection() RTC_DCHECK(mPeerConnectionFactory); mAnswerReceived = false; - mSignalingThread->PostTask([this]() { initializeConnectionThreaded(); }); - return true; -} - -bool LLWebRTCImpl::initializeConnectionThreaded() -{ rtc::scoped_refptr apm = webrtc::AudioProcessingBuilder().Create(); webrtc::AudioProcessing::Config apm_config; apm_config.echo_canceller.enabled = false; @@ -375,8 +369,6 @@ bool LLWebRTCImpl::initializeConnectionThreaded() apm_config.transient_suppression.enabled = true; apm_config.pipeline.multi_channel_render = true; apm_config.pipeline.multi_channel_capture = true; - // - apm->ApplyConfig(apm_config); mWorkerThread->BlockingCall( [this]() @@ -396,6 +388,8 @@ bool LLWebRTCImpl::initializeConnectionThreaded() mPeerDeviceModule->InitSpeaker(); mPeerDeviceModule->InitRecording(); mPeerDeviceModule->InitPlayout(); + mPeerDeviceModule->StopPlayout(); + mPeerDeviceModule->StopRecording(); }); mPeerConnectionFactory = webrtc::CreatePeerConnectionFactory(mNetworkThread.get(), @@ -408,9 +402,13 @@ bool LLWebRTCImpl::initializeConnectionThreaded() nullptr /* video_decoder_factory */, nullptr /* audio_mixer */, apm); + apm->ApplyConfig(apm_config); + webrtc::PeerConnectionInterface::RTCConfiguration config; config.sdp_semantics = webrtc::SdpSemantics::kUnifiedPlan; webrtc::PeerConnectionInterface::IceServer server; + server.uri = "stun:roxie-turn.staging.secondlife.io:3478"; + config.servers.push_back(server); server.uri = "stun:stun.l.google.com:19302"; config.servers.push_back(server); server.uri = "stun:stun1.l.google.com:19302"; @@ -430,6 +428,7 @@ bool LLWebRTCImpl::initializeConnectionThreaded() } else { + RTC_LOG(LS_ERROR) << __FUNCTION__ << "Error creating peer connection: " << error_or_peer_connection.error().message(); shutdownConnection(); return false; } @@ -493,20 +492,47 @@ bool LLWebRTCImpl::initializeConnectionThreaded() webrtc::PeerConnectionInterface::RTCOfferAnswerOptions offerOptions; mPeerConnection->CreateOffer(this, offerOptions); - RTC_LOG(LS_INFO) << __FUNCTION__ << " " << mPeerConnection->signaling_state(); - return true; } void LLWebRTCImpl::shutdownConnection() { - mDataChannel->Close(); - mDataChannel = nullptr; - mPeerConnection->Close(); - mPeerDeviceModule->Terminate(); - mPeerDeviceModule = nullptr; - mPeerConnection = nullptr; - mPeerConnectionFactory = nullptr; + mSignalingThread->PostTask( + [this]() + { + if (mPeerConnection) + { + mPeerConnection->Close(); + mPeerConnection = nullptr; + } + mPeerConnectionFactory = nullptr; + }); + mWorkerThread->PostTask( + [this]() + { + if (mTuningDeviceModule) + { + mTuningDeviceModule->StopRecording(); + mTuningDeviceModule->Terminate(); + } + if (mPeerDeviceModule) + { + mPeerDeviceModule->StopRecording(); + mPeerDeviceModule->Terminate(); + } + mTuningDeviceModule = nullptr; + mPeerDeviceModule = nullptr; + mTaskQueueFactory = nullptr; + }); + mNetworkThread->PostTask( + [this]() + { + if (mDataChannel) + { + mDataChannel->Close(); + mDataChannel = nullptr; + } + }); } void LLWebRTCImpl::AnswerAvailable(const std::string &sdp) @@ -519,26 +545,6 @@ void LLWebRTCImpl::AnswerAvailable(const std::string &sdp) RTC_LOG(LS_INFO) << __FUNCTION__ << " " << mPeerConnection->peer_connection_state(); mPeerConnection->SetRemoteDescription(webrtc::CreateSessionDescription(webrtc::SdpType::kAnswer, sdp), rtc::scoped_refptr(this)); - mAnswerReceived = true; - for (auto &observer : mSignalingObserverList) - { - for (auto &candidate : mCachedIceCandidates) - { - LLWebRTCIceCandidate ice_candidate; - ice_candidate.candidate = candidate->candidate().ToString(); - ice_candidate.mline_index = candidate->sdp_mline_index(); - ice_candidate.sdp_mid = candidate->sdp_mid(); - observer->OnIceCandidate(ice_candidate); - } - mCachedIceCandidates.clear(); - if (mPeerConnection->ice_gathering_state() == webrtc::PeerConnectionInterface::IceGatheringState::kIceGatheringComplete) - { - for (auto &observer : mSignalingObserverList) - { - observer->OnIceGatheringState(llwebrtc::LLWebRTCSignalingObserver::IceGatheringState::ICE_GATHERING_COMPLETE); - } - } - } }); } @@ -769,6 +775,19 @@ void LLWebRTCImpl::OnSetRemoteDescriptionComplete(webrtc::RTCError error) RTC_LOG(LS_ERROR) << ToString(error.type()) << ": " << error.message(); return; } + mAnswerReceived = true; + for (auto &observer : mSignalingObserverList) + { + for (auto &candidate : mCachedIceCandidates) + { + LLWebRTCIceCandidate ice_candidate; + ice_candidate.candidate = candidate->candidate().ToString(); + ice_candidate.mline_index = candidate->sdp_mline_index(); + ice_candidate.sdp_mid = candidate->sdp_mid(); + observer->OnIceCandidate(ice_candidate); + } + mCachedIceCandidates.clear(); + } } // @@ -776,6 +795,7 @@ void LLWebRTCImpl::OnSetRemoteDescriptionComplete(webrtc::RTCError error) // void LLWebRTCImpl::OnSetLocalDescriptionComplete(webrtc::RTCError error) { +#if 0 RTC_LOG(LS_INFO) << __FUNCTION__ << " " << mPeerConnection->signaling_state(); if (!error.ok()) { @@ -792,6 +812,7 @@ void LLWebRTCImpl::OnSetLocalDescriptionComplete(webrtc::RTCError error) { observer->OnOfferAvailable(sdp); } +#endif } void LLWebRTCImpl::setAudioObserver(LLWebRTCAudioObserver *observer) { mAudioObserverList.emplace_back(observer); } -- cgit v1.2.3 From 70663ca395124ec5584bc543aa503d242e0e7c54 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Tue, 10 Oct 2023 09:52:51 -0700 Subject: generate ice candidate with proper formatting --- indra/llwebrtc/llwebrtc.cpp | 41 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 39 insertions(+), 2 deletions(-) (limited to 'indra/llwebrtc/llwebrtc.cpp') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index d8f2a6028c..b5b820e3da 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -683,6 +683,43 @@ void LLWebRTCImpl::OnConnectionChange(webrtc::PeerConnectionInterface::PeerConne } } +static std::string iceCandidateToTrickleString(const webrtc::IceCandidateInterface *candidate) +{ + std::ostringstream candidate_stream; + + candidate_stream << + candidate->candidate().foundation() << " " << + std::to_string(candidate->candidate().component()) << " " << + candidate->candidate().protocol() << " " << + std::to_string(candidate->candidate().priority()) << " " << + candidate->candidate().address().ipaddr().ToString() << " " << + candidate->candidate().address().PortAsString() << " typ "; + if (candidate->candidate().type() == cricket::LOCAL_PORT_TYPE) + { + candidate_stream << "host"; + } + else if (candidate->candidate().type() == cricket::STUN_PORT_TYPE) + { + candidate_stream << "srflx " << + "raddr " << candidate->candidate().related_address().ipaddr().ToString() << " " << + "rport " << candidate->candidate().related_address().PortAsString(); + } + else if (candidate->candidate().type() == cricket::RELAY_PORT_TYPE) + { + candidate_stream << "relay " << + "raddr " << candidate->candidate().related_address().ipaddr().ToString() << " " << + "rport " << candidate->candidate().related_address().PortAsString(); + } + else if (candidate->candidate().type() == cricket::PRFLX_PORT_TYPE) + { + candidate_stream << "prflx " << + "raddr " << candidate->candidate().related_address().ipaddr().ToString() << " " << + "rport " << candidate->candidate().related_address().PortAsString(); + } + + return candidate_stream.str(); +} + void LLWebRTCImpl::OnIceCandidate(const webrtc::IceCandidateInterface *candidate) { RTC_LOG(LS_INFO) << __FUNCTION__ << " " << candidate->sdp_mline_index(); @@ -697,7 +734,7 @@ void LLWebRTCImpl::OnIceCandidate(const webrtc::IceCandidateInterface *candidate for (auto &observer : mSignalingObserverList) { LLWebRTCIceCandidate ice_candidate; - ice_candidate.candidate = candidate->candidate().ToString(); + ice_candidate.candidate = iceCandidateToTrickleString(candidate); ice_candidate.mline_index = candidate->sdp_mline_index(); ice_candidate.sdp_mid = candidate->sdp_mid(); observer->OnIceCandidate(ice_candidate); @@ -781,7 +818,7 @@ void LLWebRTCImpl::OnSetRemoteDescriptionComplete(webrtc::RTCError error) for (auto &candidate : mCachedIceCandidates) { LLWebRTCIceCandidate ice_candidate; - ice_candidate.candidate = candidate->candidate().ToString(); + ice_candidate.candidate = iceCandidateToTrickleString(candidate.get()); ice_candidate.mline_index = candidate->sdp_mline_index(); ice_candidate.sdp_mid = candidate->sdp_mid(); observer->OnIceCandidate(ice_candidate); -- cgit v1.2.3 From 7da53bacc6b3bdad668c2ed363d696107c4e10e7 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Wed, 11 Oct 2023 20:50:06 -0700 Subject: add tcptype to tcp ice candidate strings --- indra/llwebrtc/llwebrtc.cpp | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'indra/llwebrtc/llwebrtc.cpp') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index b5b820e3da..78968fc89b 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -716,6 +716,10 @@ static std::string iceCandidateToTrickleString(const webrtc::IceCandidateInterfa "raddr " << candidate->candidate().related_address().ipaddr().ToString() << " " << "rport " << candidate->candidate().related_address().PortAsString(); } + if (candidate->candidate().protocol() == "tcp") + { + candidate_stream << " tcptype " << candidate->candidate().tcptype(); + } return candidate_stream.str(); } -- cgit v1.2.3 From a7b70788a8f7c3cf607242851e6f65aaf446515b Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Thu, 19 Oct 2023 14:50:12 -0700 Subject: quicker turnaround on re-establishing voice when server goes down. --- indra/llwebrtc/llwebrtc.cpp | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) (limited to 'indra/llwebrtc/llwebrtc.cpp') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index 78968fc89b..194c7cd5d4 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -497,7 +497,7 @@ bool LLWebRTCImpl::initializeConnection() void LLWebRTCImpl::shutdownConnection() { - mSignalingThread->PostTask( + mSignalingThread->BlockingCall( [this]() { if (mPeerConnection) @@ -507,24 +507,21 @@ void LLWebRTCImpl::shutdownConnection() } mPeerConnectionFactory = nullptr; }); - mWorkerThread->PostTask( + mWorkerThread->BlockingCall( [this]() { - if (mTuningDeviceModule) - { - mTuningDeviceModule->StopRecording(); - mTuningDeviceModule->Terminate(); - } if (mPeerDeviceModule) { mPeerDeviceModule->StopRecording(); mPeerDeviceModule->Terminate(); } - mTuningDeviceModule = nullptr; mPeerDeviceModule = nullptr; - mTaskQueueFactory = nullptr; + if (mPeerAudioDeviceObserver) + { + mPeerAudioDeviceObserver = nullptr; + } }); - mNetworkThread->PostTask( + mNetworkThread->BlockingCall( [this]() { if (mDataChannel) @@ -694,6 +691,7 @@ static std::string iceCandidateToTrickleString(const webrtc::IceCandidateInterfa std::to_string(candidate->candidate().priority()) << " " << candidate->candidate().address().ipaddr().ToString() << " " << candidate->candidate().address().PortAsString() << " typ "; + if (candidate->candidate().type() == cricket::LOCAL_PORT_TYPE) { candidate_stream << "host"; @@ -716,6 +714,9 @@ static std::string iceCandidateToTrickleString(const webrtc::IceCandidateInterfa "raddr " << candidate->candidate().related_address().ipaddr().ToString() << " " << "rport " << candidate->candidate().related_address().PortAsString(); } + else { + RTC_LOG(LS_ERROR) << __FUNCTION__ << " Unknown candidate type " << candidate->candidate().type(); + } if (candidate->candidate().protocol() == "tcp") { candidate_stream << " tcptype " << candidate->candidate().tcptype(); @@ -827,8 +828,10 @@ void LLWebRTCImpl::OnSetRemoteDescriptionComplete(webrtc::RTCError error) ice_candidate.sdp_mid = candidate->sdp_mid(); observer->OnIceCandidate(ice_candidate); } - mCachedIceCandidates.clear(); } + mCachedIceCandidates.clear(); + OnIceGatheringChange(mPeerConnection->ice_gathering_state()); + } // -- cgit v1.2.3 From 2d20fbbb16a39eb3123b51c4aa3dc6df23474e00 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Wed, 8 Nov 2023 10:13:15 -0800 Subject: SL-20543 - voice over region boundaries. This commit includes code to allow the llwebrtc.dll/dylib to allow multiple connections at once. --- indra/llwebrtc/llwebrtc.cpp | 781 ++++++++++++++++++++++---------------------- 1 file changed, 388 insertions(+), 393 deletions(-) (limited to 'indra/llwebrtc/llwebrtc.cpp') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index 194c7cd5d4..6eb3478b59 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -1,6 +1,6 @@ /** - * @file llaccordionctrl.cpp - * @brief Accordion panel implementation + * @file llwebrtc.cpp + * @brief WebRTC interface implementation * * $LicenseInfo:firstyear=2023&license=viewerlgpl$ * Second Life Viewer Source Code @@ -38,7 +38,7 @@ namespace llwebrtc { -const float VOLUME_SCALE_WEBRTC = 3.0f; +const float VOLUME_SCALE_WEBRTC = 100; LLAudioDeviceObserver::LLAudioDeviceObserver() : mMicrophoneEnergy(0.0), mSumVector {0} {} @@ -57,7 +57,7 @@ void LLAudioDeviceObserver::OnCaptureData(const void *audio_samples, float sample = (static_cast(samples[index]) / (float) 32768); energy += sample * sample; } - + // smooth it. size_t buffer_size = sizeof(mSumVector) / sizeof(mSumVector[0]); float totalSum = 0; @@ -82,12 +82,12 @@ void LLAudioDeviceObserver::OnRenderData(const void *audio_samples, void LLWebRTCImpl::init() { + RTC_DCHECK(mPeerConnectionFactory); mPlayoutDevice = -1; mRecordingDevice = -1; - mAnswerReceived = false; rtc::InitializeSSL(); mTaskQueueFactory = webrtc::CreateDefaultTaskQueueFactory(); - + mNetworkThread = rtc::Thread::CreateWithSocketServer(); mNetworkThread->SetName("WebRTCNetworkThread", nullptr); mNetworkThread->Start(); @@ -97,61 +97,101 @@ void LLWebRTCImpl::init() mSignalingThread = rtc::Thread::Create(); mSignalingThread->SetName("WebRTCSignalingThread", nullptr); mSignalingThread->Start(); - + mWorkerThread->PostTask( - [this]() - { - mTuningAudioDeviceObserver = new LLAudioDeviceObserver; - mTuningDeviceModule = - webrtc::CreateAudioDeviceWithDataObserver(webrtc::AudioDeviceModule::AudioLayer::kPlatformDefaultAudio, - mTaskQueueFactory.get(), - std::unique_ptr(mTuningAudioDeviceObserver)); - mTuningDeviceModule->Init(); - mTuningDeviceModule->SetStereoRecording(false); - mTuningDeviceModule->SetStereoPlayout(true); - mTuningDeviceModule->EnableBuiltInAEC(false); - updateDevices(); - }); + [this]() + { + mTuningAudioDeviceObserver = new LLAudioDeviceObserver; + mTuningDeviceModule = + webrtc::CreateAudioDeviceWithDataObserver(webrtc::AudioDeviceModule::AudioLayer::kPlatformDefaultAudio, + mTaskQueueFactory.get(), + std::unique_ptr(mTuningAudioDeviceObserver)); + mTuningDeviceModule->Init(); + mTuningDeviceModule->SetStereoRecording(false); + mTuningDeviceModule->SetStereoPlayout(true); + mTuningDeviceModule->EnableBuiltInAEC(false); + updateDevices(); + }); + + rtc::scoped_refptr apm = webrtc::AudioProcessingBuilder().Create(); + webrtc::AudioProcessing::Config apm_config; + apm_config.echo_canceller.enabled = false; + apm_config.echo_canceller.mobile_mode = false; + apm_config.gain_controller1.enabled = true; + apm_config.gain_controller1.mode = webrtc::AudioProcessing::Config::GainController1::kAdaptiveAnalog; + apm_config.gain_controller2.enabled = true; + apm_config.high_pass_filter.enabled = true; + apm_config.noise_suppression.enabled = true; + apm_config.noise_suppression.level = webrtc::AudioProcessing::Config::NoiseSuppression::kVeryHigh; + apm_config.transient_suppression.enabled = true; + apm_config.pipeline.multi_channel_render = true; + apm_config.pipeline.multi_channel_capture = true; + + mWorkerThread->BlockingCall( + [this]() + { + mPeerAudioDeviceObserver = new LLAudioDeviceObserver; + mPeerDeviceModule = + webrtc::CreateAudioDeviceWithDataObserver(webrtc::AudioDeviceModule::AudioLayer::kPlatformDefaultAudio, + mTaskQueueFactory.get(), + std::unique_ptr(mPeerAudioDeviceObserver)); + mPeerDeviceModule->Init(); + mPeerDeviceModule->SetPlayoutDevice(mPlayoutDevice); + mPeerDeviceModule->SetRecordingDevice(mRecordingDevice); + mPeerDeviceModule->SetStereoRecording(false); + mPeerDeviceModule->SetStereoPlayout(true); + mPeerDeviceModule->EnableBuiltInAEC(false); + mPeerDeviceModule->InitMicrophone(); + mPeerDeviceModule->InitSpeaker(); + mPeerDeviceModule->InitRecording(); + mPeerDeviceModule->InitPlayout(); + mPeerDeviceModule->StartPlayout(); + mPeerDeviceModule->StartRecording(); + }); + + mPeerConnectionFactory = webrtc::CreatePeerConnectionFactory(mNetworkThread.get(), + mWorkerThread.get(), + mSignalingThread.get(), + mPeerDeviceModule, + webrtc::CreateBuiltinAudioEncoderFactory(), + webrtc::CreateBuiltinAudioDecoderFactory(), + nullptr /* video_encoder_factory */, + nullptr /* video_decoder_factory */, + nullptr /* audio_mixer */, + apm); + apm->ApplyConfig(apm_config); } void LLWebRTCImpl::terminate() { + for (auto& connection : mPeerConnections) + { + connection->terminate(); + } + mPeerConnections.clear(); + mSignalingThread->BlockingCall( - [this]() - { - if (mPeerConnection) - { - mPeerConnection->Close(); - mPeerConnection = nullptr; - } - mPeerConnectionFactory = nullptr; - }); + [this]() + { + mPeerConnectionFactory = nullptr; + }); mWorkerThread->BlockingCall( - [this]() - { - if (mTuningDeviceModule) - { - mTuningDeviceModule->StopRecording(); - mTuningDeviceModule->Terminate(); - } - if (mPeerDeviceModule) - { - mPeerDeviceModule->StopRecording(); - mPeerDeviceModule->Terminate(); - } - mTuningDeviceModule = nullptr; - mPeerDeviceModule = nullptr; - mTaskQueueFactory = nullptr; - }); - mNetworkThread->BlockingCall( - [this]() - { - if (mDataChannel) - { - mDataChannel->Close(); - mDataChannel = nullptr; - } - }); + [this]() + { + if (mTuningDeviceModule) + { + mTuningDeviceModule->StopRecording(); + mTuningDeviceModule->Terminate(); + } + if (mPeerDeviceModule) + { + mPeerDeviceModule->StopRecording(); + mPeerDeviceModule->Terminate(); + } + mTuningDeviceModule = nullptr; + mPeerDeviceModule = nullptr; + mTaskQueueFactory = nullptr; + }); } void LLWebRTCImpl::refreshDevices() @@ -164,7 +204,7 @@ void LLWebRTCImpl::setDevicesObserver(LLWebRTCDevicesObserver *observer) { mVoic void LLWebRTCImpl::unsetDevicesObserver(LLWebRTCDevicesObserver *observer) { std::vector::iterator it = - std::find(mVoiceDevicesObserverList.begin(), mVoiceDevicesObserverList.end(), observer); + std::find(mVoiceDevicesObserverList.begin(), mVoiceDevicesObserverList.end(), observer); if (it != mVoiceDevicesObserverList.end()) { mVoiceDevicesObserverList.erase(it); @@ -174,97 +214,97 @@ void LLWebRTCImpl::unsetDevicesObserver(LLWebRTCDevicesObserver *observer) void LLWebRTCImpl::setCaptureDevice(const std::string &id) { mWorkerThread->PostTask( - [this, id]() - { - int16_t captureDeviceCount = mTuningDeviceModule->RecordingDevices(); - for (int16_t i = 0; i < captureDeviceCount; i++) - { - char name[webrtc::kAdmMaxDeviceNameSize]; - char guid[webrtc::kAdmMaxGuidSize]; - mTuningDeviceModule->RecordingDeviceName(i, name, guid); - if (id == guid || id == "Default") // first one in list is default - { - RTC_LOG(LS_INFO) << __FUNCTION__ << "Set recording device to " << name << " " << guid << " " << i; - mRecordingDevice = i; - break; - } - } - mTuningDeviceModule->StopRecording(); - mTuningDeviceModule->SetRecordingDevice(mRecordingDevice); - mTuningDeviceModule->InitMicrophone(); - mTuningDeviceModule->InitRecording(); - mTuningDeviceModule->StartRecording(); - bool was_peer_recording = false; - if (mPeerDeviceModule) - { - was_peer_recording = mPeerDeviceModule->Recording(); - if (was_peer_recording) - { - mPeerDeviceModule->StopRecording(); - } - mPeerDeviceModule->SetRecordingDevice(mRecordingDevice); - mPeerDeviceModule->InitMicrophone(); - mPeerDeviceModule->InitRecording(); - if (was_peer_recording) - { - mPeerDeviceModule->StartRecording(); - } - } - }); + [this, id]() + { + int16_t captureDeviceCount = mTuningDeviceModule->RecordingDevices(); + for (int16_t i = 0; i < captureDeviceCount; i++) + { + char name[webrtc::kAdmMaxDeviceNameSize]; + char guid[webrtc::kAdmMaxGuidSize]; + mTuningDeviceModule->RecordingDeviceName(i, name, guid); + if (id == guid || id == "Default") // first one in list is default + { + RTC_LOG(LS_INFO) << __FUNCTION__ << "Set recording device to " << name << " " << guid << " " << i; + mRecordingDevice = i; + break; + } + } + mTuningDeviceModule->StopRecording(); + mTuningDeviceModule->SetRecordingDevice(mRecordingDevice); + mTuningDeviceModule->InitMicrophone(); + mTuningDeviceModule->InitRecording(); + mTuningDeviceModule->StartRecording(); + bool was_peer_recording = false; + if (mPeerDeviceModule) + { + was_peer_recording = mPeerDeviceModule->Recording(); + if (was_peer_recording) + { + mPeerDeviceModule->StopRecording(); + } + mPeerDeviceModule->SetRecordingDevice(mRecordingDevice); + mPeerDeviceModule->InitMicrophone(); + mPeerDeviceModule->InitRecording(); + if (was_peer_recording) + { + mPeerDeviceModule->StartRecording(); + } + } + }); } void LLWebRTCImpl::setRenderDevice(const std::string &id) { mWorkerThread->PostTask( - [this, id]() - { - int16_t renderDeviceCount = mTuningDeviceModule->PlayoutDevices(); - for (int16_t i = 0; i < renderDeviceCount; i++) - { - char name[webrtc::kAdmMaxDeviceNameSize]; - char guid[webrtc::kAdmMaxGuidSize]; - mTuningDeviceModule->PlayoutDeviceName(i, name, guid); - if (id == guid || id == "Default") - { - RTC_LOG(LS_INFO) << __FUNCTION__ << "Set recording device to " << name << " " << guid << " " << i; - mPlayoutDevice = i; - break; - } - } - mTuningDeviceModule->SetSpeakerMute(true); - bool was_tuning_playing = mTuningDeviceModule->Playing(); - if (was_tuning_playing) - { - mTuningDeviceModule->StopPlayout(); - } - bool was_peer_mute = false; - if (mPeerDeviceModule) - { - mPeerDeviceModule->SpeakerMute(&was_peer_mute); - if (!was_peer_mute) - { - mPeerDeviceModule->SetSpeakerMute(true); - } - } - - mTuningDeviceModule->SetPlayoutDevice(mPlayoutDevice); - mTuningDeviceModule->InitSpeaker(); - mTuningDeviceModule->InitPlayout(); - if (was_tuning_playing) - { - mTuningDeviceModule->StartPlayout(); - } - if (mPeerDeviceModule) - { - mPeerDeviceModule->SetPlayoutDevice(mPlayoutDevice); - mPeerDeviceModule->InitSpeaker(); - mPeerDeviceModule->InitPlayout(); - mPeerDeviceModule->StartPlayout(); - mPeerDeviceModule->SetSpeakerMute(was_peer_mute); - - } - mTuningDeviceModule->SetSpeakerMute(false); - }); + [this, id]() + { + int16_t renderDeviceCount = mTuningDeviceModule->PlayoutDevices(); + for (int16_t i = 0; i < renderDeviceCount; i++) + { + char name[webrtc::kAdmMaxDeviceNameSize]; + char guid[webrtc::kAdmMaxGuidSize]; + mTuningDeviceModule->PlayoutDeviceName(i, name, guid); + if (id == guid || id == "Default") + { + RTC_LOG(LS_INFO) << __FUNCTION__ << "Set recording device to " << name << " " << guid << " " << i; + mPlayoutDevice = i; + break; + } + } + mTuningDeviceModule->SetSpeakerMute(true); + bool was_tuning_playing = mTuningDeviceModule->Playing(); + if (was_tuning_playing) + { + mTuningDeviceModule->StopPlayout(); + } + bool was_peer_mute = false; + if (mPeerDeviceModule) + { + mPeerDeviceModule->SpeakerMute(&was_peer_mute); + if (!was_peer_mute) + { + mPeerDeviceModule->SetSpeakerMute(true); + } + } + + mTuningDeviceModule->SetPlayoutDevice(mPlayoutDevice); + mTuningDeviceModule->InitSpeaker(); + mTuningDeviceModule->InitPlayout(); + if (was_tuning_playing) + { + mTuningDeviceModule->StartPlayout(); + } + if (mPeerDeviceModule) + { + mPeerDeviceModule->SetPlayoutDevice(mPlayoutDevice); + mPeerDeviceModule->InitSpeaker(); + mPeerDeviceModule->InitPlayout(); + mPeerDeviceModule->StartPlayout(); + mPeerDeviceModule->SetSpeakerMute(was_peer_mute); + + } + mTuningDeviceModule->SetSpeakerMute(false); + }); } void LLWebRTCImpl::updateDevices() @@ -278,7 +318,7 @@ void LLWebRTCImpl::updateDevices() mTuningDeviceModule->PlayoutDeviceName(index, name, guid); renderDeviceList.emplace_back(name, guid); } - + int16_t captureDeviceCount = mTuningDeviceModule->RecordingDevices(); LLWebRTCVoiceDeviceList captureDeviceList; for (int16_t index = 0; index < captureDeviceCount; index++) @@ -297,113 +337,100 @@ void LLWebRTCImpl::updateDevices() void LLWebRTCImpl::setTuningMode(bool enable) { mWorkerThread->BlockingCall( - [this, enable]() - { - if (enable) - { + [this, enable]() + { + if (enable) + { + + mTuningDeviceModule->StartRecording(); + mTuningDeviceModule->SetMicrophoneMute(false); + + + mTuningDeviceModule->SetSpeakerMute(false); + + if (mPeerDeviceModule) + { + mPeerDeviceModule->StopRecording(); + mPeerDeviceModule->SetSpeakerMute(true); + } + } + else + { + if (mPeerDeviceModule) + { + mPeerDeviceModule->StartRecording(); + mPeerDeviceModule->SetSpeakerMute(false); + } + } + }); + for (auto& connection : mPeerConnections) + { + connection->enableTracks(enable ? false : !mMute); + } +} - mTuningDeviceModule->StartRecording(); - mTuningDeviceModule->SetMicrophoneMute(false); +float LLWebRTCImpl::getTuningAudioLevel() { return 20 * mTuningAudioDeviceObserver->getMicrophoneEnergy(); } +float LLWebRTCImpl::getPeerAudioLevel() { return 20 * mPeerAudioDeviceObserver->getMicrophoneEnergy(); } - mTuningDeviceModule->SetSpeakerMute(false); +void LLWebRTCImpl::setSpeakerVolume(float volume) { mPeerDeviceModule->SetSpeakerVolume(volume * VOLUME_SCALE_WEBRTC);} +void LLWebRTCImpl::setMicrophoneVolume(float volume) { mPeerDeviceModule->SetMicrophoneVolume(volume * VOLUME_SCALE_WEBRTC);} - if (mPeerDeviceModule) - { - mPeerDeviceModule->StopRecording(); - mPeerDeviceModule->SetSpeakerMute(true); - } - } - else - { - if (mPeerDeviceModule) - { - mPeerDeviceModule->StartRecording(); - mPeerDeviceModule->SetSpeakerMute(false); - } - } - }); - // set_enabled shouldn't be done on the worker thread - if (mPeerConnection) +// +// Helpers +// + +LLWebRTCPeerConnection * LLWebRTCImpl::newPeerConnection() +{ + rtc::scoped_refptr peerConnection = rtc::scoped_refptr(new rtc::RefCountedObject()); + peerConnection->init(this); + + mPeerConnections.emplace_back(peerConnection); + peerConnection->enableTracks(!mMute); + return peerConnection.get(); +} +void LLWebRTCImpl::freePeerConnection(LLWebRTCPeerConnection * peer_connection) +{ + std::vector>::iterator it = + std::find(mPeerConnections.begin(), mPeerConnections.end(), peer_connection); + if (it != mPeerConnections.end()) { - auto senders = mPeerConnection->GetSenders(); - for (auto &sender : senders) - { - sender->track()->set_enabled(enable ? false : !mMute); - } + (*it)->terminate(); + mPeerConnections.erase(it); } } // -// LLWebRTCSignalInterface +// LLWebRTCPeerConnection interface // -void LLWebRTCImpl::setSignalingObserver(LLWebRTCSignalingObserver *observer) { mSignalingObserverList.emplace_back(observer); } +void LLWebRTCPeerConnectionImpl::init(LLWebRTCImpl * webrtc_impl) +{ + mWebRTCImpl = webrtc_impl; + mPeerConnectionFactory = mWebRTCImpl->getPeerConnectionFactory(); +} +void LLWebRTCPeerConnectionImpl::terminate() +{ + shutdownConnection(); +} + +void LLWebRTCPeerConnectionImpl::setSignalingObserver(LLWebRTCSignalingObserver *observer) { mSignalingObserverList.emplace_back(observer); } -void LLWebRTCImpl::unsetSignalingObserver(LLWebRTCSignalingObserver *observer) +void LLWebRTCPeerConnectionImpl::unsetSignalingObserver(LLWebRTCSignalingObserver *observer) { std::vector::iterator it = - std::find(mSignalingObserverList.begin(), mSignalingObserverList.end(), observer); + std::find(mSignalingObserverList.begin(), mSignalingObserverList.end(), observer); if (it != mSignalingObserverList.end()) { mSignalingObserverList.erase(it); } } -bool LLWebRTCImpl::initializeConnection() +bool LLWebRTCPeerConnectionImpl::initializeConnection() { RTC_DCHECK(!mPeerConnection); - RTC_DCHECK(mPeerConnectionFactory); mAnswerReceived = false; - rtc::scoped_refptr apm = webrtc::AudioProcessingBuilder().Create(); - webrtc::AudioProcessing::Config apm_config; - apm_config.echo_canceller.enabled = false; - apm_config.echo_canceller.mobile_mode = false; - apm_config.gain_controller1.enabled = true; - apm_config.gain_controller1.mode = webrtc::AudioProcessing::Config::GainController1::kAdaptiveAnalog; - apm_config.gain_controller2.enabled = true; - apm_config.high_pass_filter.enabled = true; - apm_config.noise_suppression.enabled = true; - apm_config.noise_suppression.level = webrtc::AudioProcessing::Config::NoiseSuppression::kVeryHigh; - apm_config.transient_suppression.enabled = true; - apm_config.pipeline.multi_channel_render = true; - apm_config.pipeline.multi_channel_capture = true; - - mWorkerThread->BlockingCall( - [this]() - { - mPeerAudioDeviceObserver = new LLAudioDeviceObserver; - mPeerDeviceModule = - webrtc::CreateAudioDeviceWithDataObserver(webrtc::AudioDeviceModule::AudioLayer::kPlatformDefaultAudio, - mTaskQueueFactory.get(), - std::unique_ptr(mPeerAudioDeviceObserver)); - mPeerDeviceModule->Init(); - mPeerDeviceModule->SetPlayoutDevice(mPlayoutDevice); - mPeerDeviceModule->SetRecordingDevice(mRecordingDevice); - mPeerDeviceModule->SetStereoRecording(false); - mPeerDeviceModule->SetStereoPlayout(true); - mPeerDeviceModule->EnableBuiltInAEC(false); - mPeerDeviceModule->InitMicrophone(); - mPeerDeviceModule->InitSpeaker(); - mPeerDeviceModule->InitRecording(); - mPeerDeviceModule->InitPlayout(); - mPeerDeviceModule->StopPlayout(); - mPeerDeviceModule->StopRecording(); - }); - - mPeerConnectionFactory = webrtc::CreatePeerConnectionFactory(mNetworkThread.get(), - mWorkerThread.get(), - mSignalingThread.get(), - mPeerDeviceModule, - webrtc::CreateBuiltinAudioEncoderFactory(), - webrtc::CreateBuiltinAudioDecoderFactory(), - nullptr /* video_encoder_factory */, - nullptr /* video_decoder_factory */, - nullptr /* audio_mixer */, - apm); - apm->ApplyConfig(apm_config); - webrtc::PeerConnectionInterface::RTCConfiguration config; config.sdp_semantics = webrtc::SdpSemantics::kUnifiedPlan; webrtc::PeerConnectionInterface::IceServer server; @@ -419,7 +446,7 @@ bool LLWebRTCImpl::initializeConnection() config.servers.push_back(server); server.uri = "stun:stun4.l.google.com:19302"; config.servers.push_back(server); - + webrtc::PeerConnectionDependencies pc_dependencies(this); auto error_or_peer_connection = mPeerConnectionFactory->CreatePeerConnectionOrError(config, std::move(pc_dependencies)); if (error_or_peer_connection.ok()) @@ -432,35 +459,36 @@ bool LLWebRTCImpl::initializeConnection() shutdownConnection(); return false; } - + webrtc::DataChannelInit init; init.ordered = true; - + auto data_channel_or_error = mPeerConnection->CreateDataChannelOrError("SLData", &init); if (data_channel_or_error.ok()) { mDataChannel = std::move(data_channel_or_error.value()); - + mDataChannel->RegisterObserver(this); } - + RTC_LOG(LS_INFO) << __FUNCTION__ << " " << mPeerConnection->signaling_state(); - + cricket::AudioOptions audioOptions; audioOptions.auto_gain_control = true; audioOptions.echo_cancellation = false; // incompatible with opus stereo audioOptions.noise_suppression = true; - + rtc::scoped_refptr stream = mPeerConnectionFactory->CreateLocalMediaStream("SLStream"); + rtc::scoped_refptr audio_track( - mPeerConnectionFactory->CreateAudioTrack("SLAudio", mPeerConnectionFactory->CreateAudioSource(audioOptions).get())); + mPeerConnectionFactory->CreateAudioTrack("SLAudio", mPeerConnectionFactory->CreateAudioSource(audioOptions).get())); audio_track->set_enabled(true); stream->AddTrack(audio_track); - + mPeerConnection->AddTrack(audio_track, {"SLStream"}); - + auto senders = mPeerConnection->GetSenders(); - + for (auto &sender : senders) { webrtc::RtpParameters params; @@ -474,7 +502,7 @@ bool LLWebRTCImpl::initializeConnection() params.codecs.push_back(codecparam); sender->SetParameters(params); } - + auto receivers = mPeerConnection->GetReceivers(); for (auto &receiver : receivers) { @@ -491,102 +519,78 @@ bool LLWebRTCImpl::initializeConnection() } webrtc::PeerConnectionInterface::RTCOfferAnswerOptions offerOptions; mPeerConnection->CreateOffer(this, offerOptions); - + return true; } -void LLWebRTCImpl::shutdownConnection() +void LLWebRTCPeerConnectionImpl::shutdownConnection() { - mSignalingThread->BlockingCall( - [this]() - { - if (mPeerConnection) - { - mPeerConnection->Close(); - mPeerConnection = nullptr; - } - mPeerConnectionFactory = nullptr; - }); - mWorkerThread->BlockingCall( - [this]() - { - if (mPeerDeviceModule) - { - mPeerDeviceModule->StopRecording(); - mPeerDeviceModule->Terminate(); - } - mPeerDeviceModule = nullptr; - if (mPeerAudioDeviceObserver) - { - mPeerAudioDeviceObserver = nullptr; - } - }); - mNetworkThread->BlockingCall( - [this]() + mWebRTCImpl->SignalingBlockingCall( + [this]() + { + if (mPeerConnection) + { + mPeerConnection->Close(); + mPeerConnection = nullptr; + } + }); + + mWebRTCImpl->NetworkBlockingCall( + [this]() + { + if (mDataChannel) + { + mDataChannel->Close(); + mDataChannel = nullptr; + } + }); +} + +void LLWebRTCPeerConnectionImpl::enableTracks(bool enable) +{ + // set_enabled shouldn't be done on the worker thread + if (mPeerConnection) + { + auto senders = mPeerConnection->GetSenders(); + for (auto &sender : senders) { - if (mDataChannel) - { - mDataChannel->Close(); - mDataChannel = nullptr; - } - }); + sender->track()->set_enabled(enable); + } + } } -void LLWebRTCImpl::AnswerAvailable(const std::string &sdp) +void LLWebRTCPeerConnectionImpl::AnswerAvailable(const std::string &sdp) { RTC_LOG(LS_INFO) << __FUNCTION__ << " Remote SDP: " << sdp; - - mSignalingThread->PostTask( - [this, sdp]() - { - RTC_LOG(LS_INFO) << __FUNCTION__ << " " << mPeerConnection->peer_connection_state(); - mPeerConnection->SetRemoteDescription(webrtc::CreateSessionDescription(webrtc::SdpType::kAnswer, sdp), - rtc::scoped_refptr(this)); - }); + + mWebRTCImpl->PostSignalingTask( + [this, sdp]() + { + RTC_LOG(LS_INFO) << __FUNCTION__ << " " << mPeerConnection->peer_connection_state(); + mPeerConnection->SetRemoteDescription(webrtc::CreateSessionDescription(webrtc::SdpType::kAnswer, sdp), + rtc::scoped_refptr(this)); + }); } -void LLWebRTCImpl::setMute(bool mute) +void LLWebRTCPeerConnectionImpl::setMute(bool mute) { mMute = mute; auto senders = mPeerConnection->GetSenders(); - + RTC_LOG(LS_INFO) << __FUNCTION__ << (mute ? "disabling" : "enabling") << " streams count " << senders.size(); - + for (auto &sender : senders) { sender->track()->set_enabled(!mMute); } } -void LLWebRTCImpl::setSpeakerVolume(float volume) -{ - mSignalingThread->PostTask( - [this, volume]() - { - auto receivers = mPeerConnection->GetReceivers(); - - RTC_LOG(LS_INFO) << __FUNCTION__ << "Set volume" << receivers.size(); - for (auto &receiver : receivers) - { - webrtc::MediaStreamTrackInterface *track = receiver->track().get(); - if (track->kind() == webrtc::MediaStreamTrackInterface::kAudioKind) - { - webrtc::AudioTrackInterface *audio_track = static_cast(track); - webrtc::AudioSourceInterface *source = audio_track->GetSource(); - source->SetVolume(VOLUME_SCALE_WEBRTC * volume); - } - } - }); -} - -float LLWebRTCImpl::getTuningAudioLevel() { return 20 * mTuningAudioDeviceObserver->getMicrophoneEnergy(); } - // // PeerConnectionObserver implementation. // -void LLWebRTCImpl::OnAddTrack(rtc::scoped_refptr receiver, - const std::vector> &streams) +void LLWebRTCPeerConnectionImpl::OnAddTrack(rtc::scoped_refptr receiver, + const std::vector> &streams) { RTC_LOG(LS_INFO) << __FUNCTION__ << " " << receiver->id(); webrtc::RtpParameters params; @@ -601,18 +605,18 @@ void LLWebRTCImpl::OnAddTrack(rtc::scoped_refptr receiver->SetParameters(params); } -void LLWebRTCImpl::OnRemoveTrack(rtc::scoped_refptr receiver) +void LLWebRTCPeerConnectionImpl::OnRemoveTrack(rtc::scoped_refptr receiver) { RTC_LOG(LS_INFO) << __FUNCTION__ << " " << receiver->id(); } -void LLWebRTCImpl::OnDataChannel(rtc::scoped_refptr channel) +void LLWebRTCPeerConnectionImpl::OnDataChannel(rtc::scoped_refptr channel) { mDataChannel = channel; channel->RegisterObserver(this); } -void LLWebRTCImpl::OnIceGatheringChange(webrtc::PeerConnectionInterface::IceGatheringState new_state) +void LLWebRTCPeerConnectionImpl::OnIceGatheringChange(webrtc::PeerConnectionInterface::IceGatheringState new_state) { LLWebRTCSignalingObserver::IceGatheringState webrtc_new_state = LLWebRTCSignalingObserver::IceGatheringState::ICE_GATHERING_NEW; switch (new_state) @@ -631,7 +635,7 @@ void LLWebRTCImpl::OnIceGatheringChange(webrtc::PeerConnectionInterface::IceGath webrtc_new_state = LLWebRTCSignalingObserver::IceGatheringState::ICE_GATHERING_NEW; return; } - + if (mAnswerReceived) { for (auto &observer : mSignalingObserverList) @@ -642,19 +646,17 @@ void LLWebRTCImpl::OnIceGatheringChange(webrtc::PeerConnectionInterface::IceGath } // Called any time the PeerConnectionState changes. -void LLWebRTCImpl::OnConnectionChange(webrtc::PeerConnectionInterface::PeerConnectionState new_state) +void LLWebRTCPeerConnectionImpl::OnConnectionChange(webrtc::PeerConnectionInterface::PeerConnectionState new_state) { RTC_LOG(LS_ERROR) << __FUNCTION__ << " Peer Connection State Change " << new_state; - + switch (new_state) { case webrtc::PeerConnectionInterface::PeerConnectionState::kConnected: { if (new_state == webrtc::PeerConnectionInterface::PeerConnectionState::kConnected) { - mWorkerThread->PostTask([this]() { - mPeerDeviceModule->StartRecording(); - mPeerDeviceModule->StartPlayout(); + mWebRTCImpl->PostWorkerTask([this]() { for (auto &observer : mSignalingObserverList) { observer->OnAudioEstablished(this); @@ -670,7 +672,7 @@ void LLWebRTCImpl::OnConnectionChange(webrtc::PeerConnectionInterface::PeerConne { observer->OnRenegotiationNeeded(); } - + break; } default: @@ -683,15 +685,15 @@ void LLWebRTCImpl::OnConnectionChange(webrtc::PeerConnectionInterface::PeerConne static std::string iceCandidateToTrickleString(const webrtc::IceCandidateInterface *candidate) { std::ostringstream candidate_stream; - + candidate_stream << - candidate->candidate().foundation() << " " << - std::to_string(candidate->candidate().component()) << " " << - candidate->candidate().protocol() << " " << - std::to_string(candidate->candidate().priority()) << " " << - candidate->candidate().address().ipaddr().ToString() << " " << - candidate->candidate().address().PortAsString() << " typ "; - + candidate->candidate().foundation() << " " << + std::to_string(candidate->candidate().component()) << " " << + candidate->candidate().protocol() << " " << + std::to_string(candidate->candidate().priority()) << " " << + candidate->candidate().address().ipaddr().ToString() << " " << + candidate->candidate().address().PortAsString() << " typ "; + if (candidate->candidate().type() == cricket::LOCAL_PORT_TYPE) { candidate_stream << "host"; @@ -699,25 +701,25 @@ static std::string iceCandidateToTrickleString(const webrtc::IceCandidateInterfa else if (candidate->candidate().type() == cricket::STUN_PORT_TYPE) { candidate_stream << "srflx " << - "raddr " << candidate->candidate().related_address().ipaddr().ToString() << " " << - "rport " << candidate->candidate().related_address().PortAsString(); + "raddr " << candidate->candidate().related_address().ipaddr().ToString() << " " << + "rport " << candidate->candidate().related_address().PortAsString(); } else if (candidate->candidate().type() == cricket::RELAY_PORT_TYPE) { candidate_stream << "relay " << - "raddr " << candidate->candidate().related_address().ipaddr().ToString() << " " << - "rport " << candidate->candidate().related_address().PortAsString(); + "raddr " << candidate->candidate().related_address().ipaddr().ToString() << " " << + "rport " << candidate->candidate().related_address().PortAsString(); } else if (candidate->candidate().type() == cricket::PRFLX_PORT_TYPE) { candidate_stream << "prflx " << - "raddr " << candidate->candidate().related_address().ipaddr().ToString() << " " << - "rport " << candidate->candidate().related_address().PortAsString(); + "raddr " << candidate->candidate().related_address().ipaddr().ToString() << " " << + "rport " << candidate->candidate().related_address().PortAsString(); } else { RTC_LOG(LS_ERROR) << __FUNCTION__ << " Unknown candidate type " << candidate->candidate().type(); } - if (candidate->candidate().protocol() == "tcp") + if (candidate->candidate().protocol() == "tcp") { candidate_stream << " tcptype " << candidate->candidate().tcptype(); } @@ -725,16 +727,16 @@ static std::string iceCandidateToTrickleString(const webrtc::IceCandidateInterfa return candidate_stream.str(); } -void LLWebRTCImpl::OnIceCandidate(const webrtc::IceCandidateInterface *candidate) +void LLWebRTCPeerConnectionImpl::OnIceCandidate(const webrtc::IceCandidateInterface *candidate) { RTC_LOG(LS_INFO) << __FUNCTION__ << " " << candidate->sdp_mline_index(); - + if (!candidate) { RTC_LOG(LS_ERROR) << __FUNCTION__ << " No Ice Candidate Given"; return; } - if (mAnswerReceived) + if (mAnswerReceived) { for (auto &observer : mSignalingObserverList) { @@ -745,22 +747,22 @@ void LLWebRTCImpl::OnIceCandidate(const webrtc::IceCandidateInterface *candidate observer->OnIceCandidate(ice_candidate); } } - else + else { mCachedIceCandidates.push_back( - webrtc::CreateIceCandidate(candidate->sdp_mid(), candidate->sdp_mline_index(), candidate->candidate())); + webrtc::CreateIceCandidate(candidate->sdp_mid(), candidate->sdp_mline_index(), candidate->candidate())); } } // // CreateSessionDescriptionObserver implementation. // -void LLWebRTCImpl::OnSuccess(webrtc::SessionDescriptionInterface *desc) +void LLWebRTCPeerConnectionImpl::OnSuccess(webrtc::SessionDescriptionInterface *desc) { std::string sdp; desc->ToString(&sdp); RTC_LOG(LS_INFO) << sdp; -; + ; // mangle the sdp as this is the only way currently to bump up // the send audio rate to 48k std::istringstream sdp_stream(sdp); @@ -780,7 +782,7 @@ void LLWebRTCImpl::OnSuccess(webrtc::SessionDescriptionInterface *desc) else if (sdp_line.find("a=fmtp:" + opus_payload) == 0) { sdp_mangled_stream << sdp_line << "a=fmtp:" << opus_payload - << " minptime=10;useinbandfec=1;stereo=1;sprop-stereo=1;maxplaybackrate=48000\n"; + << " minptime=10;useinbandfec=1;stereo=1;sprop-stereo=1;maxplaybackrate=48000\n"; } else { @@ -789,27 +791,27 @@ void LLWebRTCImpl::OnSuccess(webrtc::SessionDescriptionInterface *desc) } webrtc::CreateSessionDescription(webrtc::SdpType::kOffer, sdp_mangled_stream.str()); - - - + + + mPeerConnection->SetLocalDescription(std::unique_ptr( - webrtc::CreateSessionDescription(webrtc::SdpType::kOffer, sdp_mangled_stream.str())), - rtc::scoped_refptr(this)); + webrtc::CreateSessionDescription(webrtc::SdpType::kOffer, sdp_mangled_stream.str())), + rtc::scoped_refptr(this)); RTC_LOG(LS_INFO) << __FUNCTION__ << " Local SDP: " << sdp_mangled_stream.str(); - - + + for (auto &observer : mSignalingObserverList) { observer->OnOfferAvailable(sdp_mangled_stream.str()); } } -void LLWebRTCImpl::OnFailure(webrtc::RTCError error) { RTC_LOG(LS_ERROR) << ToString(error.type()) << ": " << error.message(); } +void LLWebRTCPeerConnectionImpl::OnFailure(webrtc::RTCError error) { RTC_LOG(LS_ERROR) << ToString(error.type()) << ": " << error.message(); } // // SetRemoteDescriptionObserverInterface implementation. // -void LLWebRTCImpl::OnSetRemoteDescriptionComplete(webrtc::RTCError error) +void LLWebRTCPeerConnectionImpl::OnSetRemoteDescriptionComplete(webrtc::RTCError error) { RTC_LOG(LS_INFO) << __FUNCTION__ << " " << mPeerConnection->signaling_state(); if (!error.ok()) @@ -831,37 +833,20 @@ void LLWebRTCImpl::OnSetRemoteDescriptionComplete(webrtc::RTCError error) } mCachedIceCandidates.clear(); OnIceGatheringChange(mPeerConnection->ice_gathering_state()); - + } // // SetLocalDescriptionObserverInterface implementation. // -void LLWebRTCImpl::OnSetLocalDescriptionComplete(webrtc::RTCError error) +void LLWebRTCPeerConnectionImpl::OnSetLocalDescriptionComplete(webrtc::RTCError error) { -#if 0 - RTC_LOG(LS_INFO) << __FUNCTION__ << " " << mPeerConnection->signaling_state(); - if (!error.ok()) - { - RTC_LOG(LS_ERROR) << ToString(error.type()) << ": " << error.message(); - return; - } - auto desc = mPeerConnection->pending_local_description(); - std::string sdp; - desc->ToString(&sdp); - - RTC_LOG(LS_INFO) << __FUNCTION__ << " Local SDP: " << sdp; - ; - for (auto &observer : mSignalingObserverList) - { - observer->OnOfferAvailable(sdp); - } -#endif + } -void LLWebRTCImpl::setAudioObserver(LLWebRTCAudioObserver *observer) { mAudioObserverList.emplace_back(observer); } +void LLWebRTCPeerConnectionImpl::setAudioObserver(LLWebRTCAudioObserver *observer) { mAudioObserverList.emplace_back(observer); } -void LLWebRTCImpl::unsetAudioObserver(LLWebRTCAudioObserver *observer) +void LLWebRTCPeerConnectionImpl::unsetAudioObserver(LLWebRTCAudioObserver *observer) { std::vector::iterator it = std::find(mAudioObserverList.begin(), mAudioObserverList.end(), observer); if (it != mAudioObserverList.end()) @@ -870,22 +855,20 @@ void LLWebRTCImpl::unsetAudioObserver(LLWebRTCAudioObserver *observer) } } -float LLWebRTCImpl::getAudioLevel() { return 20 * mPeerAudioDeviceObserver->getMicrophoneEnergy(); } - // // DataChannelObserver implementation // -void LLWebRTCImpl::OnStateChange() -{ +void LLWebRTCPeerConnectionImpl::OnStateChange() +{ RTC_LOG(LS_INFO) << __FUNCTION__ << " Data Channel State: " << webrtc::DataChannelInterface::DataStateString(mDataChannel->state()); switch (mDataChannel->state()) { case webrtc::DataChannelInterface::kOpen: RTC_LOG(LS_INFO) << __FUNCTION__ << " Data Channel State Open"; - for (auto &observer : mDataObserverList) + for (auto &observer : mSignalingObserverList) { - observer->OnDataChannelReady(); + observer->OnDataChannelReady(this); } break; case webrtc::DataChannelInterface::kConnecting: @@ -903,7 +886,7 @@ void LLWebRTCImpl::OnStateChange() } -void LLWebRTCImpl::OnMessage(const webrtc::DataBuffer& buffer) +void LLWebRTCPeerConnectionImpl::OnMessage(const webrtc::DataBuffer& buffer) { std::string data((const char*)buffer.data.cdata(), buffer.size()); for (auto &observer : mDataObserverList) @@ -912,43 +895,55 @@ void LLWebRTCImpl::OnMessage(const webrtc::DataBuffer& buffer) } } -void LLWebRTCImpl::sendData(const std::string& data, bool binary) +void LLWebRTCPeerConnectionImpl::sendData(const std::string& data, bool binary) { rtc::CopyOnWriteBuffer cowBuffer(data.data(), data.length()); webrtc::DataBuffer buffer(cowBuffer, binary); mDataChannel->Send(buffer); } -void LLWebRTCImpl::setDataObserver(LLWebRTCDataObserver* observer) { mDataObserverList.emplace_back(observer); } +void LLWebRTCPeerConnectionImpl::setDataObserver(LLWebRTCDataObserver* observer) { mDataObserverList.emplace_back(observer); } -void LLWebRTCImpl::unsetDataObserver(LLWebRTCDataObserver* observer) +void LLWebRTCPeerConnectionImpl::unsetDataObserver(LLWebRTCDataObserver* observer) { std::vector::iterator it = - std::find(mDataObserverList.begin(), mDataObserverList.end(), observer); + std::find(mDataObserverList.begin(), mDataObserverList.end(), observer); if (it != mDataObserverList.end()) { mDataObserverList.erase(it); } } -rtc::RefCountedObject *gWebRTCImpl = nullptr; -LLWebRTCDeviceInterface *getDeviceInterface() { return gWebRTCImpl; } -LLWebRTCSignalInterface *getSignalingInterface() { return gWebRTCImpl; } -LLWebRTCDataInterface *getDataInterface() { return gWebRTCImpl; } +LLWebRTCImpl * gWebRTCImpl = nullptr; +LLWebRTCDeviceInterface * getDeviceInterface() +{ + return gWebRTCImpl; +} + +LLWebRTCPeerConnection * newPeerConnection() +{ + return gWebRTCImpl->newPeerConnection(); +} + +void freePeerConnection(LLWebRTCPeerConnection *peer_connection) +{ + gWebRTCImpl->freePeerConnection(peer_connection); +} void init() { - gWebRTCImpl = new rtc::RefCountedObject(); - gWebRTCImpl->AddRef(); + gWebRTCImpl = new LLWebRTCImpl(); gWebRTCImpl->init(); } void terminate() -{ - gWebRTCImpl->terminate(); - gWebRTCImpl->Release(); - gWebRTCImpl = nullptr; +{ + if (gWebRTCImpl) + { + gWebRTCImpl->terminate(); + gWebRTCImpl = nullptr; + } } } // namespace llwebrtc -- cgit v1.2.3 From 7db561d1dc9d41813e479a0bc61841b3cd03a55c Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Wed, 8 Nov 2023 10:17:56 -0800 Subject: comment fixes --- indra/llwebrtc/llwebrtc.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'indra/llwebrtc/llwebrtc.cpp') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index 6eb3478b59..62c42ead1b 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -1,6 +1,6 @@ /** * @file llwebrtc.cpp - * @brief WebRTC interface implementation + * @brief WebRTC interface implementation * * $LicenseInfo:firstyear=2023&license=viewerlgpl$ * Second Life Viewer Source Code -- cgit v1.2.3 From bce9e50cd38b8a22d4f20d950a686668d217592d Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Thu, 9 Nov 2023 18:19:18 -0800 Subject: Remove a bunch of unnecessary code. --- indra/llwebrtc/llwebrtc.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'indra/llwebrtc/llwebrtc.cpp') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index 62c42ead1b..bf1e04c2f2 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -373,8 +373,8 @@ float LLWebRTCImpl::getTuningAudioLevel() { return 20 * mTuningAudioDeviceObserv float LLWebRTCImpl::getPeerAudioLevel() { return 20 * mPeerAudioDeviceObserver->getMicrophoneEnergy(); } -void LLWebRTCImpl::setSpeakerVolume(float volume) { mPeerDeviceModule->SetSpeakerVolume(volume * VOLUME_SCALE_WEBRTC);} -void LLWebRTCImpl::setMicrophoneVolume(float volume) { mPeerDeviceModule->SetMicrophoneVolume(volume * VOLUME_SCALE_WEBRTC);} +void LLWebRTCImpl::setSpeakerVolume(float volume) { mPeerDeviceModule->SetSpeakerVolume( (uint32_t)(volume * VOLUME_SCALE_WEBRTC));} +void LLWebRTCImpl::setMicrophoneVolume(float volume) { mPeerDeviceModule->SetMicrophoneVolume((uint32_t)(volume * VOLUME_SCALE_WEBRTC));} // // Helpers -- cgit v1.2.3 From d302d89891475d1431099deac99bc52a2c3046a1 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Thu, 30 Nov 2023 13:14:07 -0800 Subject: Refactor/clean-up WebRTC voice to handle multiple voice streams This is useful for cross-region voice, quick voice switching, etc. --- indra/llwebrtc/llwebrtc.cpp | 79 +++++++++++++++++++++++++++++---------------- 1 file changed, 51 insertions(+), 28 deletions(-) (limited to 'indra/llwebrtc/llwebrtc.cpp') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index bf1e04c2f2..b2f5e0212e 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -216,6 +216,7 @@ void LLWebRTCImpl::setCaptureDevice(const std::string &id) mWorkerThread->PostTask( [this, id]() { + int16_t tuningRecordingDevice = 0; int16_t captureDeviceCount = mTuningDeviceModule->RecordingDevices(); for (int16_t i = 0; i < captureDeviceCount; i++) { @@ -225,18 +226,32 @@ void LLWebRTCImpl::setCaptureDevice(const std::string &id) if (id == guid || id == "Default") // first one in list is default { RTC_LOG(LS_INFO) << __FUNCTION__ << "Set recording device to " << name << " " << guid << " " << i; - mRecordingDevice = i; + tuningRecordingDevice = i; break; } } mTuningDeviceModule->StopRecording(); - mTuningDeviceModule->SetRecordingDevice(mRecordingDevice); + mTuningDeviceModule->SetRecordingDevice(tuningRecordingDevice); mTuningDeviceModule->InitMicrophone(); mTuningDeviceModule->InitRecording(); mTuningDeviceModule->StartRecording(); bool was_peer_recording = false; if (mPeerDeviceModule) { + int16_t captureDeviceCount = mTuningDeviceModule->RecordingDevices(); + for (int16_t i = 0; i < captureDeviceCount; i++) + { + char name[webrtc::kAdmMaxDeviceNameSize]; + char guid[webrtc::kAdmMaxGuidSize]; + mTuningDeviceModule->RecordingDeviceName(i, name, guid); + if (id == guid || id == "Default") // first one in list is default + { + RTC_LOG(LS_INFO) + << __FUNCTION__ << "Set recording device to " << name << " " << guid << " " << i; + mRecordingDevice = i; + break; + } + } was_peer_recording = mPeerDeviceModule->Recording(); if (was_peer_recording) { @@ -259,6 +274,7 @@ void LLWebRTCImpl::setRenderDevice(const std::string &id) [this, id]() { int16_t renderDeviceCount = mTuningDeviceModule->PlayoutDevices(); + int16_t tuningPlayoutDevice = 0; for (int16_t i = 0; i < renderDeviceCount; i++) { char name[webrtc::kAdmMaxDeviceNameSize]; @@ -267,7 +283,7 @@ void LLWebRTCImpl::setRenderDevice(const std::string &id) if (id == guid || id == "Default") { RTC_LOG(LS_INFO) << __FUNCTION__ << "Set recording device to " << name << " " << guid << " " << i; - mPlayoutDevice = i; + tuningPlayoutDevice = i; break; } } @@ -277,30 +293,40 @@ void LLWebRTCImpl::setRenderDevice(const std::string &id) { mTuningDeviceModule->StopPlayout(); } - bool was_peer_mute = false; if (mPeerDeviceModule) { - mPeerDeviceModule->SpeakerMute(&was_peer_mute); - if (!was_peer_mute) - { - mPeerDeviceModule->SetSpeakerMute(true); - } + mPeerDeviceModule->SetSpeakerMute(true); } - mTuningDeviceModule->SetPlayoutDevice(mPlayoutDevice); + mTuningDeviceModule->SetPlayoutDevice(tuningPlayoutDevice); mTuningDeviceModule->InitSpeaker(); mTuningDeviceModule->InitPlayout(); if (was_tuning_playing) { mTuningDeviceModule->StartPlayout(); } + renderDeviceCount = mPeerDeviceModule->PlayoutDevices(); + if (mPeerDeviceModule) { + for (int16_t i = 0; i < renderDeviceCount; i++) + { + char name[webrtc::kAdmMaxDeviceNameSize]; + char guid[webrtc::kAdmMaxGuidSize]; + mPeerDeviceModule->PlayoutDeviceName(i, name, guid); + if (id == guid || id == "Default") + { + RTC_LOG(LS_INFO) + << __FUNCTION__ << "Set recording device to " << name << " " << guid << " " << i; + mPlayoutDevice = i; + break; + } + } mPeerDeviceModule->SetPlayoutDevice(mPlayoutDevice); mPeerDeviceModule->InitSpeaker(); mPeerDeviceModule->InitPlayout(); mPeerDeviceModule->StartPlayout(); - mPeerDeviceModule->SetSpeakerMute(was_peer_mute); + mPeerDeviceModule->SetSpeakerMute(false); } mTuningDeviceModule->SetSpeakerMute(false); @@ -376,6 +402,8 @@ float LLWebRTCImpl::getPeerAudioLevel() { return 20 * mPeerAudioDeviceObserver-> void LLWebRTCImpl::setSpeakerVolume(float volume) { mPeerDeviceModule->SetSpeakerVolume( (uint32_t)(volume * VOLUME_SCALE_WEBRTC));} void LLWebRTCImpl::setMicrophoneVolume(float volume) { mPeerDeviceModule->SetMicrophoneVolume((uint32_t)(volume * VOLUME_SCALE_WEBRTC));} +void LLWebRTCImpl::setMute(bool mute) { mPeerDeviceModule->SetMicrophoneMute(mute); } + // // Helpers // @@ -566,9 +594,12 @@ void LLWebRTCPeerConnectionImpl::AnswerAvailable(const std::string &sdp) mWebRTCImpl->PostSignalingTask( [this, sdp]() { - RTC_LOG(LS_INFO) << __FUNCTION__ << " " << mPeerConnection->peer_connection_state(); - mPeerConnection->SetRemoteDescription(webrtc::CreateSessionDescription(webrtc::SdpType::kAnswer, sdp), - rtc::scoped_refptr(this)); + if (mPeerConnection) + { + RTC_LOG(LS_INFO) << __FUNCTION__ << " " << mPeerConnection->peer_connection_state(); + mPeerConnection->SetRemoteDescription(webrtc::CreateSessionDescription(webrtc::SdpType::kAnswer, sdp), + rtc::scoped_refptr(this)); + } }); } @@ -844,17 +875,6 @@ void LLWebRTCPeerConnectionImpl::OnSetLocalDescriptionComplete(webrtc::RTCError } -void LLWebRTCPeerConnectionImpl::setAudioObserver(LLWebRTCAudioObserver *observer) { mAudioObserverList.emplace_back(observer); } - -void LLWebRTCPeerConnectionImpl::unsetAudioObserver(LLWebRTCAudioObserver *observer) -{ - std::vector::iterator it = std::find(mAudioObserverList.begin(), mAudioObserverList.end(), observer); - if (it != mAudioObserverList.end()) - { - mAudioObserverList.erase(it); - } -} - // // DataChannelObserver implementation // @@ -897,9 +917,12 @@ void LLWebRTCPeerConnectionImpl::OnMessage(const webrtc::DataBuffer& buffer) void LLWebRTCPeerConnectionImpl::sendData(const std::string& data, bool binary) { - rtc::CopyOnWriteBuffer cowBuffer(data.data(), data.length()); - webrtc::DataBuffer buffer(cowBuffer, binary); - mDataChannel->Send(buffer); + if (mDataChannel) + { + rtc::CopyOnWriteBuffer cowBuffer(data.data(), data.length()); + webrtc::DataBuffer buffer(cowBuffer, binary); + mDataChannel->Send(buffer); + } } void LLWebRTCPeerConnectionImpl::setDataObserver(LLWebRTCDataObserver* observer) { mDataObserverList.emplace_back(observer); } -- cgit v1.2.3 From 7a6c7964520d185eefedfedc340b0cc07365ff88 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Fri, 1 Dec 2023 01:14:33 -0800 Subject: Using the device module to set speaker/mic volume set the system mic/volume for all applications. Instead, modify the volume on the various streams. --- indra/llwebrtc/llwebrtc.cpp | 56 ++++++++++++++++++++++++++++++++++----------- 1 file changed, 43 insertions(+), 13 deletions(-) (limited to 'indra/llwebrtc/llwebrtc.cpp') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index b2f5e0212e..4741ed92a3 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -399,11 +399,6 @@ float LLWebRTCImpl::getTuningAudioLevel() { return 20 * mTuningAudioDeviceObserv float LLWebRTCImpl::getPeerAudioLevel() { return 20 * mPeerAudioDeviceObserver->getMicrophoneEnergy(); } -void LLWebRTCImpl::setSpeakerVolume(float volume) { mPeerDeviceModule->SetSpeakerVolume( (uint32_t)(volume * VOLUME_SCALE_WEBRTC));} -void LLWebRTCImpl::setMicrophoneVolume(float volume) { mPeerDeviceModule->SetMicrophoneVolume((uint32_t)(volume * VOLUME_SCALE_WEBRTC));} - -void LLWebRTCImpl::setMute(bool mute) { mPeerDeviceModule->SetMicrophoneMute(mute); } - // // Helpers // @@ -417,6 +412,7 @@ LLWebRTCPeerConnection * LLWebRTCImpl::newPeerConnection() peerConnection->enableTracks(!mMute); return peerConnection.get(); } + void LLWebRTCImpl::freePeerConnection(LLWebRTCPeerConnection * peer_connection) { std::vector>::iterator it = @@ -506,12 +502,11 @@ bool LLWebRTCPeerConnectionImpl::initializeConnection() audioOptions.echo_cancellation = false; // incompatible with opus stereo audioOptions.noise_suppression = true; - rtc::scoped_refptr stream = mPeerConnectionFactory->CreateLocalMediaStream("SLStream"); - + mLocalStream = mPeerConnectionFactory->CreateLocalMediaStream("SLStream"); rtc::scoped_refptr audio_track( mPeerConnectionFactory->CreateAudioTrack("SLAudio", mPeerConnectionFactory->CreateAudioSource(audioOptions).get())); audio_track->set_enabled(true); - stream->AddTrack(audio_track); + mLocalStream->AddTrack(audio_track); mPeerConnection->AddTrack(audio_track, {"SLStream"}); @@ -545,6 +540,7 @@ bool LLWebRTCPeerConnectionImpl::initializeConnection() params.codecs.push_back(codecparam); receiver->SetParameters(params); } + webrtc::PeerConnectionInterface::RTCOfferAnswerOptions offerOptions; mPeerConnection->CreateOffer(this, offerOptions); @@ -606,13 +602,47 @@ void LLWebRTCPeerConnectionImpl::AnswerAvailable(const std::string &sdp) void LLWebRTCPeerConnectionImpl::setMute(bool mute) { mMute = mute; - auto senders = mPeerConnection->GetSenders(); - - RTC_LOG(LS_INFO) << __FUNCTION__ << (mute ? "disabling" : "enabling") << " streams count " << senders.size(); + if (mPeerConnection) + { + auto senders = mPeerConnection->GetSenders(); + + RTC_LOG(LS_INFO) << __FUNCTION__ << (mute ? "disabling" : "enabling") << " streams count " << senders.size(); + for (auto &sender : senders) + { + auto track = sender->track(); + if(track) + { + track->set_enabled(!mMute); + } + } + } +} + +void LLWebRTCPeerConnectionImpl::setReceiveVolume(float volume) +{ + auto receivers = mPeerConnection->GetReceivers(); - for (auto &sender : senders) + for (auto &receiver : receivers) { - sender->track()->set_enabled(!mMute); + for (auto& stream : receiver->streams()) + { + for (auto& track : stream->GetAudioTracks()) + { + track->GetSource()->SetVolume(volume); + } + } + } +} + +void LLWebRTCPeerConnectionImpl::setSendVolume(float volume) +{ + if (mLocalStream) + { + auto track = mLocalStream->FindAudioTrack("SLStream"); + if (track) + { + track->GetSource()->SetVolume(volume); + } } } -- cgit v1.2.3 From b10731f6f9f357907b7ba53d7531cfa3552c46cf Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Fri, 1 Dec 2023 01:28:38 -0800 Subject: Setting volume for remote stream needs to happen in signaling thread --- indra/llwebrtc/llwebrtc.cpp | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) (limited to 'indra/llwebrtc/llwebrtc.cpp') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index 4741ed92a3..2bcbb135c0 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -620,26 +620,29 @@ void LLWebRTCPeerConnectionImpl::setMute(bool mute) void LLWebRTCPeerConnectionImpl::setReceiveVolume(float volume) { - auto receivers = mPeerConnection->GetReceivers(); - - for (auto &receiver : receivers) - { - for (auto& stream : receiver->streams()) + mWebRTCImpl->PostSignalingTask( + [this, volume]() { - for (auto& track : stream->GetAudioTracks()) + auto receivers = mPeerConnection->GetReceivers(); + + for (auto &receiver : receivers) { - track->GetSource()->SetVolume(volume); + for (auto &stream : receiver->streams()) + { + for (auto &track : stream->GetAudioTracks()) + { + track->GetSource()->SetVolume(volume); + } + } } - } - } + }); } void LLWebRTCPeerConnectionImpl::setSendVolume(float volume) { if (mLocalStream) { - auto track = mLocalStream->FindAudioTrack("SLStream"); - if (track) + for (auto &track : mLocalStream->GetAudioTracks()) { track->GetSource()->SetVolume(volume); } -- cgit v1.2.3 From e92e4d762ec5b2c122f17edb1a18b21ef6166e8b Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Mon, 4 Dec 2023 14:22:40 -0800 Subject: Mute using enable. Muting using the device module microphone mute was muting other applications, speakers, and so on. Instead, we mute by enabling/disabling the input and output streams. --- indra/llwebrtc/llwebrtc.cpp | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) (limited to 'indra/llwebrtc/llwebrtc.cpp') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index 2bcbb135c0..9fd8fd8ee6 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -287,16 +287,11 @@ void LLWebRTCImpl::setRenderDevice(const std::string &id) break; } } - mTuningDeviceModule->SetSpeakerMute(true); bool was_tuning_playing = mTuningDeviceModule->Playing(); if (was_tuning_playing) { mTuningDeviceModule->StopPlayout(); } - if (mPeerDeviceModule) - { - mPeerDeviceModule->SetSpeakerMute(true); - } mTuningDeviceModule->SetPlayoutDevice(tuningPlayoutDevice); mTuningDeviceModule->InitSpeaker(); @@ -326,10 +321,8 @@ void LLWebRTCImpl::setRenderDevice(const std::string &id) mPeerDeviceModule->InitSpeaker(); mPeerDeviceModule->InitPlayout(); mPeerDeviceModule->StartPlayout(); - mPeerDeviceModule->SetSpeakerMute(false); } - mTuningDeviceModule->SetSpeakerMute(false); }); } @@ -369,29 +362,25 @@ void LLWebRTCImpl::setTuningMode(bool enable) { mTuningDeviceModule->StartRecording(); - mTuningDeviceModule->SetMicrophoneMute(false); - - - mTuningDeviceModule->SetSpeakerMute(false); if (mPeerDeviceModule) { mPeerDeviceModule->StopRecording(); - mPeerDeviceModule->SetSpeakerMute(true); } } else { + mTuningDeviceModule->StartRecording(); if (mPeerDeviceModule) { mPeerDeviceModule->StartRecording(); - mPeerDeviceModule->SetSpeakerMute(false); } } }); for (auto& connection : mPeerConnections) { - connection->enableTracks(enable ? false : !mMute); + connection->enableSenderTracks(enable ? false : !mMute); + connection->enableReceiverTracks(!enable); } } @@ -409,7 +398,7 @@ LLWebRTCPeerConnection * LLWebRTCImpl::newPeerConnection() peerConnection->init(this); mPeerConnections.emplace_back(peerConnection); - peerConnection->enableTracks(!mMute); + peerConnection->enableSenderTracks(!mMute); return peerConnection.get(); } @@ -570,7 +559,7 @@ void LLWebRTCPeerConnectionImpl::shutdownConnection() }); } -void LLWebRTCPeerConnectionImpl::enableTracks(bool enable) +void LLWebRTCPeerConnectionImpl::enableSenderTracks(bool enable) { // set_enabled shouldn't be done on the worker thread if (mPeerConnection) @@ -583,6 +572,19 @@ void LLWebRTCPeerConnectionImpl::enableTracks(bool enable) } } +void LLWebRTCPeerConnectionImpl::enableReceiverTracks(bool enable) +{ + // set_enabled shouldn't be done on the worker thread + if (mPeerConnection) + { + auto receivers = mPeerConnection->GetReceivers(); + for (auto &receiver : receivers) + { + receiver->track()->set_enabled(enable); + } + } +} + void LLWebRTCPeerConnectionImpl::AnswerAvailable(const std::string &sdp) { RTC_LOG(LS_INFO) << __FUNCTION__ << " Remote SDP: " << sdp; -- cgit v1.2.3 From c51309ac72ca96cdc0d22ba7549e614139595e65 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Thu, 7 Dec 2023 19:47:37 -0800 Subject: Rework VU meter level processing to be closer to Vivox --- indra/llwebrtc/llwebrtc.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'indra/llwebrtc/llwebrtc.cpp') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index 9fd8fd8ee6..32a75d3675 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -54,7 +54,7 @@ void LLAudioDeviceObserver::OnCaptureData(const void *audio_samples, const short *samples = (const short *) audio_samples; for (size_t index = 0; index < num_samples * num_channels; index++) { - float sample = (static_cast(samples[index]) / (float) 32768); + float sample = (static_cast(samples[index]) / (float) 32767); energy += sample * sample; } @@ -384,9 +384,9 @@ void LLWebRTCImpl::setTuningMode(bool enable) } } -float LLWebRTCImpl::getTuningAudioLevel() { return 20 * mTuningAudioDeviceObserver->getMicrophoneEnergy(); } +float LLWebRTCImpl::getTuningAudioLevel() { return -20 * log10f(mTuningAudioDeviceObserver->getMicrophoneEnergy()); } -float LLWebRTCImpl::getPeerAudioLevel() { return 20 * mPeerAudioDeviceObserver->getMicrophoneEnergy(); } +float LLWebRTCImpl::getPeerAudioLevel() { return -20 * log10f(mPeerAudioDeviceObserver->getMicrophoneEnergy()); } // // Helpers -- cgit v1.2.3 From e3b719f41e20ea3261e66283d1c9281bb8e6f320 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Fri, 8 Dec 2023 12:09:02 -0800 Subject: fix device selection (hopefully) --- indra/llwebrtc/llwebrtc.cpp | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) (limited to 'indra/llwebrtc/llwebrtc.cpp') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index 32a75d3675..12997e34d3 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -300,10 +300,10 @@ void LLWebRTCImpl::setRenderDevice(const std::string &id) { mTuningDeviceModule->StartPlayout(); } - renderDeviceCount = mPeerDeviceModule->PlayoutDevices(); if (mPeerDeviceModule) { + renderDeviceCount = mPeerDeviceModule->PlayoutDevices(); for (int16_t i = 0; i < renderDeviceCount; i++) { char name[webrtc::kAdmMaxDeviceNameSize]; @@ -317,6 +317,7 @@ void LLWebRTCImpl::setRenderDevice(const std::string &id) break; } } + mPeerDeviceModule->StopPlayout(); mPeerDeviceModule->SetPlayoutDevice(mPlayoutDevice); mPeerDeviceModule->InitSpeaker(); mPeerDeviceModule->InitPlayout(); @@ -379,7 +380,14 @@ void LLWebRTCImpl::setTuningMode(bool enable) }); for (auto& connection : mPeerConnections) { - connection->enableSenderTracks(enable ? false : !mMute); + if (enable) + { + connection->enableSenderTracks(false); + } + else + { + connection->resetMute(); + } connection->enableReceiverTracks(!enable); } } @@ -620,6 +628,11 @@ void LLWebRTCPeerConnectionImpl::setMute(bool mute) } } +void LLWebRTCPeerConnectionImpl::resetMute() +{ + setMute(mMute); +} + void LLWebRTCPeerConnectionImpl::setReceiveVolume(float volume) { mWebRTCImpl->PostSignalingTask( -- cgit v1.2.3 From a60593164e82057930d5e9da5173207700413197 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Tue, 12 Dec 2023 20:46:34 -0800 Subject: Better renegotiation support for parcel voice Better handle starting up and shutting down WebRTC connections simultaneously. --- indra/llwebrtc/llwebrtc.cpp | 84 +++++++++++++++++++++++++-------------------- 1 file changed, 47 insertions(+), 37 deletions(-) (limited to 'indra/llwebrtc/llwebrtc.cpp') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index 12997e34d3..e55d94d128 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -83,9 +83,10 @@ void LLAudioDeviceObserver::OnRenderData(const void *audio_samples, void LLWebRTCImpl::init() { RTC_DCHECK(mPeerConnectionFactory); - mPlayoutDevice = -1; - mRecordingDevice = -1; + mPlayoutDevice = 0; + mRecordingDevice = 0; rtc::InitializeSSL(); + rtc::LogMessage::LogToDebug(rtc::LS_WARNING); mTaskQueueFactory = webrtc::CreateDefaultTaskQueueFactory(); mNetworkThread = rtc::Thread::CreateWithSocketServer(); @@ -145,10 +146,9 @@ void LLWebRTCImpl::init() mPeerDeviceModule->InitSpeaker(); mPeerDeviceModule->InitRecording(); mPeerDeviceModule->InitPlayout(); - mPeerDeviceModule->StartPlayout(); - mPeerDeviceModule->StartRecording(); }); - + + apm->ApplyConfig(apm_config); mPeerConnectionFactory = webrtc::CreatePeerConnectionFactory(mNetworkThread.get(), mWorkerThread.get(), mSignalingThread.get(), @@ -159,7 +159,12 @@ void LLWebRTCImpl::init() nullptr /* video_decoder_factory */, nullptr /* audio_mixer */, apm); - apm->ApplyConfig(apm_config); + mWorkerThread->BlockingCall( + [this]() + { + mPeerDeviceModule->StartPlayout(); + mPeerDeviceModule->StartRecording(); + }); } void LLWebRTCImpl::terminate() @@ -235,15 +240,14 @@ void LLWebRTCImpl::setCaptureDevice(const std::string &id) mTuningDeviceModule->InitMicrophone(); mTuningDeviceModule->InitRecording(); mTuningDeviceModule->StartRecording(); - bool was_peer_recording = false; if (mPeerDeviceModule) { - int16_t captureDeviceCount = mTuningDeviceModule->RecordingDevices(); + int16_t captureDeviceCount = mPeerDeviceModule->RecordingDevices(); for (int16_t i = 0; i < captureDeviceCount; i++) { char name[webrtc::kAdmMaxDeviceNameSize]; char guid[webrtc::kAdmMaxGuidSize]; - mTuningDeviceModule->RecordingDeviceName(i, name, guid); + mPeerDeviceModule->RecordingDeviceName(i, name, guid); if (id == guid || id == "Default") // first one in list is default { RTC_LOG(LS_INFO) @@ -252,11 +256,13 @@ void LLWebRTCImpl::setCaptureDevice(const std::string &id) break; } } - was_peer_recording = mPeerDeviceModule->Recording(); + bool was_peer_recording = mPeerDeviceModule->Recording(); if (was_peer_recording) { mPeerDeviceModule->StopRecording(); } + + mPeerDeviceModule->StopRecording(); mPeerDeviceModule->SetRecordingDevice(mRecordingDevice); mPeerDeviceModule->InitMicrophone(); mPeerDeviceModule->InitRecording(); @@ -432,7 +438,15 @@ void LLWebRTCPeerConnectionImpl::init(LLWebRTCImpl * webrtc_impl) } void LLWebRTCPeerConnectionImpl::terminate() { - shutdownConnection(); + mWebRTCImpl->SignalingBlockingCall( + [this]() + { + if (mPeerConnection) + { + mPeerConnection->Close(); + mPeerConnection = nullptr; + } + }); } void LLWebRTCPeerConnectionImpl::setSignalingObserver(LLWebRTCSignalingObserver *observer) { mSignalingObserverList.emplace_back(observer); } @@ -477,7 +491,6 @@ bool LLWebRTCPeerConnectionImpl::initializeConnection() else { RTC_LOG(LS_ERROR) << __FUNCTION__ << "Error creating peer connection: " << error_or_peer_connection.error().message(); - shutdownConnection(); return false; } @@ -546,25 +559,19 @@ bool LLWebRTCPeerConnectionImpl::initializeConnection() void LLWebRTCPeerConnectionImpl::shutdownConnection() { - mWebRTCImpl->SignalingBlockingCall( - [this]() - { - if (mPeerConnection) - { - mPeerConnection->Close(); - mPeerConnection = nullptr; - } - }); - - mWebRTCImpl->NetworkBlockingCall( - [this]() - { - if (mDataChannel) - { - mDataChannel->Close(); - mDataChannel = nullptr; - } - }); + mWebRTCImpl->PostSignalingTask( + [this]() + { + if (mPeerConnection) + { + mPeerConnection->Close(); + mPeerConnection = nullptr; + } + for (auto &observer : mSignalingObserverList) + { + observer->OnPeerShutDown(); + } + }); } void LLWebRTCPeerConnectionImpl::enableSenderTracks(bool enable) @@ -638,15 +645,18 @@ void LLWebRTCPeerConnectionImpl::setReceiveVolume(float volume) mWebRTCImpl->PostSignalingTask( [this, volume]() { - auto receivers = mPeerConnection->GetReceivers(); - - for (auto &receiver : receivers) + if (mPeerConnection) { - for (auto &stream : receiver->streams()) + auto receivers = mPeerConnection->GetReceivers(); + + for (auto &receiver : receivers) { - for (auto &track : stream->GetAudioTracks()) + for (auto &stream : receiver->streams()) { - track->GetSource()->SetVolume(volume); + for (auto &track : stream->GetAudioTracks()) + { + track->GetSource()->SetVolume(volume); + } } } } -- cgit v1.2.3 From a7509747b20bb8c00a477e8bdacc41f0c657cfdf Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Mon, 18 Dec 2023 11:45:15 -0800 Subject: Touch up parcel voice enable/disable. --- indra/llwebrtc/llwebrtc.cpp | 203 ++++++++++++++++++++++---------------------- 1 file changed, 102 insertions(+), 101 deletions(-) (limited to 'indra/llwebrtc/llwebrtc.cpp') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index e55d94d128..9b3ec2889b 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -86,7 +86,9 @@ void LLWebRTCImpl::init() mPlayoutDevice = 0; mRecordingDevice = 0; rtc::InitializeSSL(); - rtc::LogMessage::LogToDebug(rtc::LS_WARNING); + rtc::LogMessage::LogToDebug(rtc::LS_NONE); + rtc::LogMessage::SetLogToStderr(true); + mTaskQueueFactory = webrtc::CreateDefaultTaskQueueFactory(); mNetworkThread = rtc::Thread::CreateWithSocketServer(); @@ -98,7 +100,7 @@ void LLWebRTCImpl::init() mSignalingThread = rtc::Thread::Create(); mSignalingThread->SetName("WebRTCSignalingThread", nullptr); mSignalingThread->Start(); - + mWorkerThread->PostTask( [this]() { @@ -466,93 +468,98 @@ bool LLWebRTCPeerConnectionImpl::initializeConnection() RTC_DCHECK(!mPeerConnection); mAnswerReceived = false; - webrtc::PeerConnectionInterface::RTCConfiguration config; - config.sdp_semantics = webrtc::SdpSemantics::kUnifiedPlan; - webrtc::PeerConnectionInterface::IceServer server; - server.uri = "stun:roxie-turn.staging.secondlife.io:3478"; - config.servers.push_back(server); - server.uri = "stun:stun.l.google.com:19302"; - config.servers.push_back(server); - server.uri = "stun:stun1.l.google.com:19302"; - config.servers.push_back(server); - server.uri = "stun:stun2.l.google.com:19302"; - config.servers.push_back(server); - server.uri = "stun:stun3.l.google.com:19302"; - config.servers.push_back(server); - server.uri = "stun:stun4.l.google.com:19302"; - config.servers.push_back(server); - - webrtc::PeerConnectionDependencies pc_dependencies(this); - auto error_or_peer_connection = mPeerConnectionFactory->CreatePeerConnectionOrError(config, std::move(pc_dependencies)); - if (error_or_peer_connection.ok()) - { - mPeerConnection = std::move(error_or_peer_connection.value()); - } - else - { - RTC_LOG(LS_ERROR) << __FUNCTION__ << "Error creating peer connection: " << error_or_peer_connection.error().message(); - return false; - } - - webrtc::DataChannelInit init; - init.ordered = true; - - auto data_channel_or_error = mPeerConnection->CreateDataChannelOrError("SLData", &init); - if (data_channel_or_error.ok()) - { - mDataChannel = std::move(data_channel_or_error.value()); - - mDataChannel->RegisterObserver(this); - } - - RTC_LOG(LS_INFO) << __FUNCTION__ << " " << mPeerConnection->signaling_state(); - - cricket::AudioOptions audioOptions; - audioOptions.auto_gain_control = true; - audioOptions.echo_cancellation = false; // incompatible with opus stereo - audioOptions.noise_suppression = true; - - mLocalStream = mPeerConnectionFactory->CreateLocalMediaStream("SLStream"); - rtc::scoped_refptr audio_track( - mPeerConnectionFactory->CreateAudioTrack("SLAudio", mPeerConnectionFactory->CreateAudioSource(audioOptions).get())); - audio_track->set_enabled(true); - mLocalStream->AddTrack(audio_track); - - mPeerConnection->AddTrack(audio_track, {"SLStream"}); - - auto senders = mPeerConnection->GetSenders(); - - for (auto &sender : senders) - { - webrtc::RtpParameters params; - webrtc::RtpCodecParameters codecparam; - codecparam.name = "opus"; - codecparam.kind = cricket::MEDIA_TYPE_AUDIO; - codecparam.clock_rate = 48000; - codecparam.num_channels = 2; - codecparam.parameters["stereo"] = "1"; - codecparam.parameters["sprop-stereo"] = "1"; - params.codecs.push_back(codecparam); - sender->SetParameters(params); - } - - auto receivers = mPeerConnection->GetReceivers(); - for (auto &receiver : receivers) - { - webrtc::RtpParameters params; - webrtc::RtpCodecParameters codecparam; - codecparam.name = "opus"; - codecparam.kind = cricket::MEDIA_TYPE_AUDIO; - codecparam.clock_rate = 48000; - codecparam.num_channels = 2; - codecparam.parameters["stereo"] = "1"; - codecparam.parameters["sprop-stereo"] = "1"; - params.codecs.push_back(codecparam); - receiver->SetParameters(params); - } - - webrtc::PeerConnectionInterface::RTCOfferAnswerOptions offerOptions; - mPeerConnection->CreateOffer(this, offerOptions); + mWebRTCImpl->PostSignalingTask( + [this]() + { + webrtc::PeerConnectionInterface::RTCConfiguration config; + config.sdp_semantics = webrtc::SdpSemantics::kUnifiedPlan; + webrtc::PeerConnectionInterface::IceServer server; + server.uri = "stun:roxie-turn.staging.secondlife.io:3478"; + config.servers.push_back(server); + server.uri = "stun:stun.l.google.com:19302"; + config.servers.push_back(server); + server.uri = "stun:stun1.l.google.com:19302"; + config.servers.push_back(server); + server.uri = "stun:stun2.l.google.com:19302"; + config.servers.push_back(server); + server.uri = "stun:stun3.l.google.com:19302"; + config.servers.push_back(server); + server.uri = "stun:stun4.l.google.com:19302"; + config.servers.push_back(server); + + config.set_min_port(60000); + config.set_max_port(60100); + + webrtc::PeerConnectionDependencies pc_dependencies(this); + auto error_or_peer_connection = mPeerConnectionFactory->CreatePeerConnectionOrError(config, std::move(pc_dependencies)); + if (error_or_peer_connection.ok()) + { + mPeerConnection = std::move(error_or_peer_connection.value()); + } + else + { + RTC_LOG(LS_ERROR) << __FUNCTION__ << "Error creating peer connection: " << error_or_peer_connection.error().message(); + return; + } + + webrtc::DataChannelInit init; + init.ordered = true; + + auto data_channel_or_error = mPeerConnection->CreateDataChannelOrError("SLData", &init); + if (data_channel_or_error.ok()) + { + mDataChannel = std::move(data_channel_or_error.value()); + + mDataChannel->RegisterObserver(this); + } + + cricket::AudioOptions audioOptions; + audioOptions.auto_gain_control = true; + audioOptions.echo_cancellation = false; // incompatible with opus stereo + audioOptions.noise_suppression = true; + + mLocalStream = mPeerConnectionFactory->CreateLocalMediaStream("SLStream"); + rtc::scoped_refptr audio_track( + mPeerConnectionFactory->CreateAudioTrack("SLAudio", mPeerConnectionFactory->CreateAudioSource(audioOptions).get())); + audio_track->set_enabled(true); + mLocalStream->AddTrack(audio_track); + + mPeerConnection->AddTrack(audio_track, {"SLStream"}); + + auto senders = mPeerConnection->GetSenders(); + + for (auto &sender : senders) + { + webrtc::RtpParameters params; + webrtc::RtpCodecParameters codecparam; + codecparam.name = "opus"; + codecparam.kind = cricket::MEDIA_TYPE_AUDIO; + codecparam.clock_rate = 48000; + codecparam.num_channels = 2; + codecparam.parameters["stereo"] = "1"; + codecparam.parameters["sprop-stereo"] = "1"; + params.codecs.push_back(codecparam); + sender->SetParameters(params); + } + + auto receivers = mPeerConnection->GetReceivers(); + for (auto &receiver : receivers) + { + webrtc::RtpParameters params; + webrtc::RtpCodecParameters codecparam; + codecparam.name = "opus"; + codecparam.kind = cricket::MEDIA_TYPE_AUDIO; + codecparam.clock_rate = 48000; + codecparam.num_channels = 2; + codecparam.parameters["stereo"] = "1"; + codecparam.parameters["sprop-stereo"] = "1"; + params.codecs.push_back(codecparam); + receiver->SetParameters(params); + } + + webrtc::PeerConnectionInterface::RTCOfferAnswerOptions offerOptions; + mPeerConnection->CreateOffer(this, offerOptions); + }); return true; } @@ -839,7 +846,7 @@ void LLWebRTCPeerConnectionImpl::OnIceCandidate(const webrtc::IceCandidateInterf else { mCachedIceCandidates.push_back( - webrtc::CreateIceCandidate(candidate->sdp_mid(), candidate->sdp_mline_index(), candidate->candidate())); + webrtc::CreateIceCandidate(candidate->sdp_mid(), candidate->sdp_mline_index(), candidate->candidate())); } } @@ -878,21 +885,16 @@ void LLWebRTCPeerConnectionImpl::OnSuccess(webrtc::SessionDescriptionInterface * sdp_mangled_stream << sdp_line << "\n"; } } - - webrtc::CreateSessionDescription(webrtc::SdpType::kOffer, sdp_mangled_stream.str()); - - - - mPeerConnection->SetLocalDescription(std::unique_ptr( - webrtc::CreateSessionDescription(webrtc::SdpType::kOffer, sdp_mangled_stream.str())), - rtc::scoped_refptr(this)); + RTC_LOG(LS_INFO) << __FUNCTION__ << " Local SDP: " << sdp_mangled_stream.str(); - - + for (auto &observer : mSignalingObserverList) { observer->OnOfferAvailable(sdp_mangled_stream.str()); } + + mPeerConnection->SetLocalDescription(std::unique_ptr(webrtc::CreateSessionDescription(webrtc::SdpType::kOffer, sdp_mangled_stream.str())), + rtc::scoped_refptr(this)); } void LLWebRTCPeerConnectionImpl::OnFailure(webrtc::RTCError error) { RTC_LOG(LS_ERROR) << ToString(error.type()) << ": " << error.message(); } @@ -930,7 +932,6 @@ void LLWebRTCPeerConnectionImpl::OnSetRemoteDescriptionComplete(webrtc::RTCError // void LLWebRTCPeerConnectionImpl::OnSetLocalDescriptionComplete(webrtc::RTCError error) { - } // -- cgit v1.2.3 From 23b065b0f1f8620e91a41f5538dfc1ff3077037e Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Thu, 4 Jan 2024 13:02:55 -0800 Subject: Enable AEC --- indra/llwebrtc/llwebrtc.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'indra/llwebrtc/llwebrtc.cpp') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index 9b3ec2889b..8e56f9c222 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -110,7 +110,7 @@ void LLWebRTCImpl::init() mTaskQueueFactory.get(), std::unique_ptr(mTuningAudioDeviceObserver)); mTuningDeviceModule->Init(); - mTuningDeviceModule->SetStereoRecording(false); + mTuningDeviceModule->SetStereoRecording(true); mTuningDeviceModule->SetStereoPlayout(true); mTuningDeviceModule->EnableBuiltInAEC(false); updateDevices(); @@ -118,7 +118,7 @@ void LLWebRTCImpl::init() rtc::scoped_refptr apm = webrtc::AudioProcessingBuilder().Create(); webrtc::AudioProcessing::Config apm_config; - apm_config.echo_canceller.enabled = false; + apm_config.echo_canceller.enabled = true; apm_config.echo_canceller.mobile_mode = false; apm_config.gain_controller1.enabled = true; apm_config.gain_controller1.mode = webrtc::AudioProcessing::Config::GainController1::kAdaptiveAnalog; @@ -141,7 +141,7 @@ void LLWebRTCImpl::init() mPeerDeviceModule->Init(); mPeerDeviceModule->SetPlayoutDevice(mPlayoutDevice); mPeerDeviceModule->SetRecordingDevice(mRecordingDevice); - mPeerDeviceModule->SetStereoRecording(false); + mPeerDeviceModule->SetStereoRecording(true); mPeerDeviceModule->SetStereoPlayout(true); mPeerDeviceModule->EnableBuiltInAEC(false); mPeerDeviceModule->InitMicrophone(); @@ -515,7 +515,7 @@ bool LLWebRTCPeerConnectionImpl::initializeConnection() cricket::AudioOptions audioOptions; audioOptions.auto_gain_control = true; - audioOptions.echo_cancellation = false; // incompatible with opus stereo + audioOptions.echo_cancellation = true; // incompatible with opus stereo audioOptions.noise_suppression = true; mLocalStream = mPeerConnectionFactory->CreateLocalMediaStream("SLStream"); @@ -878,7 +878,7 @@ void LLWebRTCPeerConnectionImpl::OnSuccess(webrtc::SessionDescriptionInterface * else if (sdp_line.find("a=fmtp:" + opus_payload) == 0) { sdp_mangled_stream << sdp_line << "a=fmtp:" << opus_payload - << " minptime=10;useinbandfec=1;stereo=1;sprop-stereo=1;maxplaybackrate=48000\n"; + << " minptime=10;useinbandfec=1;stereo=1;sprop-stereo=1;maxplaybackrate=48000;sprop-maxcapturerate=48000;sprop-maxplaybackrate=48000\n"; } else { -- cgit v1.2.3 From 7246f55290abda53c211b335e01f9a9182c7513f Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Tue, 9 Jan 2024 11:57:01 -0800 Subject: New WebRTC with echo cancellation fix. Also, start/stop recording depending on whether WebRTC has negotiated. --- indra/llwebrtc/llwebrtc.cpp | 74 ++++++++++++++++++++++++++++++++++++--------- 1 file changed, 60 insertions(+), 14 deletions(-) (limited to 'indra/llwebrtc/llwebrtc.cpp') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index 8e56f9c222..fca490e8c2 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -150,7 +150,34 @@ void LLWebRTCImpl::init() mPeerDeviceModule->InitPlayout(); }); - apm->ApplyConfig(apm_config); + rtc::scoped_refptr apm = webrtc::AudioProcessingBuilder().Create(); + webrtc::AudioProcessing::Config apm_config; + apm_config.echo_canceller.enabled = true; + apm_config.echo_canceller.mobile_mode = false; + apm_config.gain_controller1.enabled = true; + apm_config.gain_controller1.mode = webrtc::AudioProcessing::Config::GainController1::kAdaptiveAnalog; + apm_config.gain_controller2.enabled = true; + apm_config.high_pass_filter.enabled = true; + apm_config.noise_suppression.enabled = true; + apm_config.noise_suppression.level = webrtc::AudioProcessing::Config::NoiseSuppression::kVeryHigh; + apm_config.transient_suppression.enabled = true; + apm_config.pipeline.multi_channel_render = true; + apm_config.pipeline.multi_channel_capture = true; + apm_config.pipeline.multi_channel_capture = true; + + webrtc::ProcessingConfig processing_config; + processing_config.input_stream().set_num_channels(2); + processing_config.input_stream().set_sample_rate_hz(8000); + processing_config.output_stream().set_num_channels(2); + processing_config.output_stream().set_sample_rate_hz(8000); + processing_config.reverse_input_stream().set_num_channels(2); + processing_config.reverse_input_stream().set_sample_rate_hz(48000); + processing_config.reverse_output_stream().set_num_channels(2); + processing_config.reverse_output_stream().set_sample_rate_hz(48000); + + apm->Initialize(processing_config); + apm->ApplyConfig(apm_config); + mPeerConnectionFactory = webrtc::CreatePeerConnectionFactory(mNetworkThread.get(), mWorkerThread.get(), mSignalingThread.get(), @@ -161,11 +188,28 @@ void LLWebRTCImpl::init() nullptr /* video_decoder_factory */, nullptr /* audio_mixer */, apm); + + mWorkerThread->BlockingCall( [this]() { mPeerDeviceModule->StartPlayout(); - mPeerDeviceModule->StartRecording(); + }); +} + +void LLWebRTCImpl::setRecording(bool recording) +{ + mWorkerThread->PostTask( + [this, recording]() + { + if (recording) + { + mPeerDeviceModule->StartRecording(); + } + else + { + mPeerDeviceModule->StopRecording(); + } }); } @@ -263,8 +307,6 @@ void LLWebRTCImpl::setCaptureDevice(const std::string &id) { mPeerDeviceModule->StopRecording(); } - - mPeerDeviceModule->StopRecording(); mPeerDeviceModule->SetRecordingDevice(mRecordingDevice); mPeerDeviceModule->InitMicrophone(); mPeerDeviceModule->InitRecording(); @@ -427,6 +469,10 @@ void LLWebRTCImpl::freePeerConnection(LLWebRTCPeerConnection * peer_connection) (*it)->terminate(); mPeerConnections.erase(it); } + if (mPeerConnections.empty()) + { + setRecording(false); + } } // @@ -519,6 +565,7 @@ bool LLWebRTCPeerConnectionImpl::initializeConnection() audioOptions.noise_suppression = true; mLocalStream = mPeerConnectionFactory->CreateLocalMediaStream("SLStream"); + rtc::scoped_refptr audio_track( mPeerConnectionFactory->CreateAudioTrack("SLAudio", mPeerConnectionFactory->CreateAudioSource(audioOptions).get())); audio_track->set_enabled(true); @@ -750,15 +797,14 @@ void LLWebRTCPeerConnectionImpl::OnConnectionChange(webrtc::PeerConnectionInterf { case webrtc::PeerConnectionInterface::PeerConnectionState::kConnected: { - if (new_state == webrtc::PeerConnectionInterface::PeerConnectionState::kConnected) - { - mWebRTCImpl->PostWorkerTask([this]() { - for (auto &observer : mSignalingObserverList) - { - observer->OnAudioEstablished(this); - } - }); - } + mWebRTCImpl->setRecording(true); + + mWebRTCImpl->PostWorkerTask([this]() { + for (auto &observer : mSignalingObserverList) + { + observer->OnAudioEstablished(this); + } + }); break; } case webrtc::PeerConnectionInterface::PeerConnectionState::kFailed: @@ -872,8 +918,8 @@ void LLWebRTCPeerConnectionImpl::OnSuccess(webrtc::SessionDescriptionInterface * // force mono down, stereo up if (std::sscanf(sdp_line.c_str(), "a=rtpmap:%i opus/%i/2", &payload_id, &bandwidth) == 2) { - sdp_mangled_stream << sdp_line << "\n"; opus_payload = std::to_string(payload_id); + sdp_mangled_stream << "a=rtpmap:" << opus_payload << " opus/48000/2" << "\n"; } else if (sdp_line.find("a=fmtp:" + opus_payload) == 0) { -- cgit v1.2.3 From 8ab0bb9a41850bd03a9cdc1624a822d8eb24f712 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Sat, 3 Feb 2024 18:53:04 -0800 Subject: fix locking race condition --- indra/llwebrtc/llwebrtc.cpp | 40 ++++++++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 16 deletions(-) (limited to 'indra/llwebrtc/llwebrtc.cpp') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index fca490e8c2..900698aa56 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -673,20 +673,24 @@ void LLWebRTCPeerConnectionImpl::AnswerAvailable(const std::string &sdp) void LLWebRTCPeerConnectionImpl::setMute(bool mute) { mMute = mute; - if (mPeerConnection) - { - auto senders = mPeerConnection->GetSenders(); - - RTC_LOG(LS_INFO) << __FUNCTION__ << (mute ? "disabling" : "enabling") << " streams count " << senders.size(); - for (auto &sender : senders) + mWebRTCImpl->PostSignalingTask( + [this]() + { + if (mPeerConnection) { - auto track = sender->track(); - if(track) + auto senders = mPeerConnection->GetSenders(); + + RTC_LOG(LS_INFO) << __FUNCTION__ << (mMute ? "disabling" : "enabling") << " streams count " << senders.size(); + for (auto &sender : senders) { - track->set_enabled(!mMute); + auto track = sender->track(); + if (track) + { + track->set_enabled(!mMute); + } } } - } + }); } void LLWebRTCPeerConnectionImpl::resetMute() @@ -719,13 +723,17 @@ void LLWebRTCPeerConnectionImpl::setReceiveVolume(float volume) void LLWebRTCPeerConnectionImpl::setSendVolume(float volume) { - if (mLocalStream) - { - for (auto &track : mLocalStream->GetAudioTracks()) + mWebRTCImpl->PostSignalingTask( + [this, volume]() { - track->GetSource()->SetVolume(volume); - } - } + if (mLocalStream) + { + for (auto &track : mLocalStream->GetAudioTracks()) + { + track->GetSource()->SetVolume(volume); + } + } + }); } // -- cgit v1.2.3 From 73b00bfe94a6a2a4ea0abc2cf716d835820dfdba Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Sat, 3 Feb 2024 22:02:48 -0800 Subject: Handle 'device changed' callback --- indra/llwebrtc/llwebrtc.cpp | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) (limited to 'indra/llwebrtc/llwebrtc.cpp') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index 900698aa56..d6eff5e1f2 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -113,6 +113,7 @@ void LLWebRTCImpl::init() mTuningDeviceModule->SetStereoRecording(true); mTuningDeviceModule->SetStereoPlayout(true); mTuningDeviceModule->EnableBuiltInAEC(false); + mTuningDeviceModule->SetAudioDeviceSink(this); updateDevices(); }); @@ -379,31 +380,42 @@ void LLWebRTCImpl::setRenderDevice(const std::string &id) void LLWebRTCImpl::updateDevices() { - int16_t renderDeviceCount = mTuningDeviceModule->PlayoutDevices(); + int16_t renderDeviceCount = mTuningDeviceModule->PlayoutDevices(); + int16_t currentRenderDeviceIndex = mTuningDeviceModule->GetPlayoutDevice(); + LLWebRTCVoiceDeviceList renderDeviceList; for (int16_t index = 0; index < renderDeviceCount; index++) { char name[webrtc::kAdmMaxDeviceNameSize]; char guid[webrtc::kAdmMaxGuidSize]; mTuningDeviceModule->PlayoutDeviceName(index, name, guid); - renderDeviceList.emplace_back(name, guid); + renderDeviceList.emplace_back(name, guid, index == currentRenderDeviceIndex); } - int16_t captureDeviceCount = mTuningDeviceModule->RecordingDevices(); + int16_t captureDeviceCount = mTuningDeviceModule->RecordingDevices(); + int16_t currentCaptureDeviceIndex = mTuningDeviceModule->GetRecordingDevice(); + LLWebRTCVoiceDeviceList captureDeviceList; for (int16_t index = 0; index < captureDeviceCount; index++) { char name[webrtc::kAdmMaxDeviceNameSize]; char guid[webrtc::kAdmMaxGuidSize]; mTuningDeviceModule->RecordingDeviceName(index, name, guid); - captureDeviceList.emplace_back(name, guid); + captureDeviceList.emplace_back(name, guid, index == currentCaptureDeviceIndex); } for (auto &observer : mVoiceDevicesObserverList) { - observer->OnDevicesChanged(renderDeviceList, captureDeviceList); + observer->OnDevicesChanged(renderDeviceList, + captureDeviceList); } } +void LLWebRTCImpl::OnDevicesUpdated() +{ + updateDevices(); +} + + void LLWebRTCImpl::setTuningMode(bool enable) { mWorkerThread->BlockingCall( -- cgit v1.2.3 From 65b5a6e12d32602ae5b0ed3982dc2c74057c7ab7 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Mon, 5 Feb 2024 14:42:21 -0800 Subject: Use a custom audio processor to pull data for level determinations, which will happen after AGC --- indra/llwebrtc/llwebrtc.cpp | 83 +++++++++++++++++++++++++++------------------ 1 file changed, 50 insertions(+), 33 deletions(-) (limited to 'indra/llwebrtc/llwebrtc.cpp') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index d6eff5e1f2..f0d7f6ad8f 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -27,6 +27,7 @@ #include "llwebrtc_impl.h" #include #include +#include #include "api/audio_codecs/audio_decoder_factory.h" #include "api/audio_codecs/audio_encoder_factory.h" @@ -34,30 +35,54 @@ #include "api/audio_codecs/builtin_audio_encoder_factory.h" #include "api/media_stream_interface.h" #include "api/media_stream_track.h" +#include "modules/audio_processing/audio_buffer.h" namespace llwebrtc { -const float VOLUME_SCALE_WEBRTC = 100; - -LLAudioDeviceObserver::LLAudioDeviceObserver() : mMicrophoneEnergy(0.0), mSumVector {0} {} +LLCustomProcessor::LLCustomProcessor() : + mSampleRateHz(0), + mNumChannels(0) +{ + memset(mSumVector, sizeof(mSumVector), 0); +} -float LLAudioDeviceObserver::getMicrophoneEnergy() { return mMicrophoneEnergy; } +void LLCustomProcessor::Initialize(int sample_rate_hz, int num_channels) +{ + mSampleRateHz = sample_rate_hz; + mNumChannels = num_channels; + memset(mSumVector, sizeof(mSumVector), 0); +} -void LLAudioDeviceObserver::OnCaptureData(const void *audio_samples, - const size_t num_samples, - const size_t bytes_per_sample, - const size_t num_channels, - const uint32_t samples_per_sec) +void LLCustomProcessor::Process(webrtc::AudioBuffer *audio_in) { + webrtc::StreamConfig stream_config; + stream_config.set_sample_rate_hz(mSampleRateHz); + stream_config.set_num_channels(mNumChannels); + std::vector frame; + std::vector frame_samples; + + if (audio_in->num_channels() < 1 || audio_in->num_frames() < 480) + { + return; + } + + frame_samples.resize(stream_config.num_samples()); + frame.resize(stream_config.num_channels()); + for (size_t ch = 0; ch < stream_config.num_channels(); ++ch) + { + frame[ch] = &(frame_samples)[ch * stream_config.num_frames()]; + } + + audio_in->CopyTo(stream_config, &frame[0]); + float energy = 0; - const short *samples = (const short *) audio_samples; - for (size_t index = 0; index < num_samples * num_channels; index++) + for (size_t index = 0; index < stream_config.num_samples(); index++) { - float sample = (static_cast(samples[index]) / (float) 32767); + float sample = frame_samples[index]; energy += sample * sample; } - + // smooth it. size_t buffer_size = sizeof(mSumVector) / sizeof(mSumVector[0]); float totalSum = 0; @@ -69,15 +94,7 @@ void LLAudioDeviceObserver::OnCaptureData(const void *audio_samples, } mSumVector[i] = energy; totalSum += energy; - mMicrophoneEnergy = std::sqrt(totalSum / (num_samples * buffer_size)); -} - -void LLAudioDeviceObserver::OnRenderData(const void *audio_samples, - const size_t num_samples, - const size_t bytes_per_sample, - const size_t num_channels, - const uint32_t samples_per_sec) -{ + mMicrophoneEnergy = std::sqrt(totalSum / (stream_config.num_samples() * buffer_size)); } void LLWebRTCImpl::init() @@ -104,11 +121,9 @@ void LLWebRTCImpl::init() mWorkerThread->PostTask( [this]() { - mTuningAudioDeviceObserver = new LLAudioDeviceObserver; - mTuningDeviceModule = - webrtc::CreateAudioDeviceWithDataObserver(webrtc::AudioDeviceModule::AudioLayer::kPlatformDefaultAudio, - mTaskQueueFactory.get(), - std::unique_ptr(mTuningAudioDeviceObserver)); + mTuningDeviceModule = webrtc::CreateAudioDeviceWithDataObserver( + webrtc::AudioDeviceModule::AudioLayer::kPlatformDefaultAudio, + mTaskQueueFactory.get(), nullptr); mTuningDeviceModule->Init(); mTuningDeviceModule->SetStereoRecording(true); mTuningDeviceModule->SetStereoPlayout(true); @@ -134,11 +149,9 @@ void LLWebRTCImpl::init() mWorkerThread->BlockingCall( [this]() { - mPeerAudioDeviceObserver = new LLAudioDeviceObserver; mPeerDeviceModule = webrtc::CreateAudioDeviceWithDataObserver(webrtc::AudioDeviceModule::AudioLayer::kPlatformDefaultAudio, - mTaskQueueFactory.get(), - std::unique_ptr(mPeerAudioDeviceObserver)); + mTaskQueueFactory.get(), nullptr); mPeerDeviceModule->Init(); mPeerDeviceModule->SetPlayoutDevice(mPlayoutDevice); mPeerDeviceModule->SetRecordingDevice(mRecordingDevice); @@ -151,7 +164,11 @@ void LLWebRTCImpl::init() mPeerDeviceModule->InitPlayout(); }); - rtc::scoped_refptr apm = webrtc::AudioProcessingBuilder().Create(); + mCustomProcessor = new LLCustomProcessor; + webrtc::AudioProcessingBuilder apb; + apb.SetCapturePostProcessing(std::unique_ptr(mCustomProcessor)); + rtc::scoped_refptr apm = apb.Create(); + webrtc::AudioProcessing::Config apm_config; apm_config.echo_canceller.enabled = true; apm_config.echo_canceller.mobile_mode = false; @@ -454,9 +471,9 @@ void LLWebRTCImpl::setTuningMode(bool enable) } } -float LLWebRTCImpl::getTuningAudioLevel() { return -20 * log10f(mTuningAudioDeviceObserver->getMicrophoneEnergy()); } +float LLWebRTCImpl::getTuningAudioLevel() { return -20 * log10f(mCustomProcessor->getMicrophoneEnergy()); } -float LLWebRTCImpl::getPeerAudioLevel() { return -20 * log10f(mPeerAudioDeviceObserver->getMicrophoneEnergy()); } +float LLWebRTCImpl::getPeerAudioLevel() { return -20 * log10f(mCustomProcessor->getMicrophoneEnergy()); } // // Helpers -- cgit v1.2.3 From dc1858a32a71e4e944009b206c32112ec846880b Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Mon, 5 Feb 2024 16:47:47 -0800 Subject: put observer-based tuning audio level calculation back --- indra/llwebrtc/llwebrtc.cpp | 53 ++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 47 insertions(+), 6 deletions(-) (limited to 'indra/llwebrtc/llwebrtc.cpp') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index f0d7f6ad8f..3273786242 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -40,6 +40,46 @@ namespace llwebrtc { +LLAudioDeviceObserver::LLAudioDeviceObserver() : mMicrophoneEnergy(0.0), mSumVector {0} {} + +float LLAudioDeviceObserver::getMicrophoneEnergy() { return mMicrophoneEnergy; } + +void LLAudioDeviceObserver::OnCaptureData(const void *audio_samples, + const size_t num_samples, + const size_t bytes_per_sample, + const size_t num_channels, + const uint32_t samples_per_sec) +{ + float energy = 0; + const short *samples = (const short *) audio_samples; + for (size_t index = 0; index < num_samples * num_channels; index++) + { + float sample = (static_cast(samples[index]) / (float) 32767); + energy += sample * sample; + } + + // smooth it. + size_t buffer_size = sizeof(mSumVector) / sizeof(mSumVector[0]); + float totalSum = 0; + int i; + for (i = 0; i < (buffer_size - 1); i++) + { + mSumVector[i] = mSumVector[i + 1]; + totalSum += mSumVector[i]; + } + mSumVector[i] = energy; + totalSum += energy; + mMicrophoneEnergy = std::sqrt(totalSum / (num_samples * buffer_size)); +} + +void LLAudioDeviceObserver::OnRenderData(const void *audio_samples, + const size_t num_samples, + const size_t bytes_per_sample, + const size_t num_channels, + const uint32_t samples_per_sec) +{ +} + LLCustomProcessor::LLCustomProcessor() : mSampleRateHz(0), mNumChannels(0) @@ -118,12 +158,13 @@ void LLWebRTCImpl::init() mSignalingThread->SetName("WebRTCSignalingThread", nullptr); mSignalingThread->Start(); + mTuningAudioDeviceObserver = new LLAudioDeviceObserver; mWorkerThread->PostTask( [this]() { mTuningDeviceModule = webrtc::CreateAudioDeviceWithDataObserver( - webrtc::AudioDeviceModule::AudioLayer::kPlatformDefaultAudio, - mTaskQueueFactory.get(), nullptr); + webrtc::AudioDeviceModule::AudioLayer::kPlatformDefaultAudio, mTaskQueueFactory.get(), + std::unique_ptr(mTuningAudioDeviceObserver)); mTuningDeviceModule->Init(); mTuningDeviceModule->SetStereoRecording(true); mTuningDeviceModule->SetStereoPlayout(true); @@ -164,9 +205,9 @@ void LLWebRTCImpl::init() mPeerDeviceModule->InitPlayout(); }); - mCustomProcessor = new LLCustomProcessor; + mPeerCustomProcessor = new LLCustomProcessor; webrtc::AudioProcessingBuilder apb; - apb.SetCapturePostProcessing(std::unique_ptr(mCustomProcessor)); + apb.SetCapturePostProcessing(std::unique_ptr(mPeerCustomProcessor)); rtc::scoped_refptr apm = apb.Create(); webrtc::AudioProcessing::Config apm_config; @@ -471,9 +512,9 @@ void LLWebRTCImpl::setTuningMode(bool enable) } } -float LLWebRTCImpl::getTuningAudioLevel() { return -20 * log10f(mCustomProcessor->getMicrophoneEnergy()); } +float LLWebRTCImpl::getTuningAudioLevel() { return -20 * log10f(mTuningAudioDeviceObserver->getMicrophoneEnergy()); } -float LLWebRTCImpl::getPeerAudioLevel() { return -20 * log10f(mCustomProcessor->getMicrophoneEnergy()); } +float LLWebRTCImpl::getPeerAudioLevel() { return -20 * log10f(mPeerCustomProcessor->getMicrophoneEnergy()); } // // Helpers -- cgit v1.2.3 From 49a1b4038f4852df17f574f1fb5b01c5b5ddafb4 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Mon, 5 Feb 2024 16:48:43 -0800 Subject: mac build fix --- indra/llwebrtc/llwebrtc.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'indra/llwebrtc/llwebrtc.cpp') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index 3273786242..06a5c3a085 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -84,14 +84,14 @@ LLCustomProcessor::LLCustomProcessor() : mSampleRateHz(0), mNumChannels(0) { - memset(mSumVector, sizeof(mSumVector), 0); + memset(mSumVector, 0, sizeof(mSumVector)); } void LLCustomProcessor::Initialize(int sample_rate_hz, int num_channels) { mSampleRateHz = sample_rate_hz; mNumChannels = num_channels; - memset(mSumVector, sizeof(mSumVector), 0); + memset(mSumVector, 0, sizeof(mSumVector)); } void LLCustomProcessor::Process(webrtc::AudioBuffer *audio_in) -- cgit v1.2.3 From 90efc71ef9f0fa43d6b3b8d2ebc8bda1556b669f Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Tue, 6 Feb 2024 19:12:14 -0800 Subject: race between session established and data channel ready --- indra/llwebrtc/llwebrtc.cpp | 48 ++++++++++++++------------------------------- 1 file changed, 15 insertions(+), 33 deletions(-) (limited to 'indra/llwebrtc/llwebrtc.cpp') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index 06a5c3a085..ee85a254f2 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -476,40 +476,22 @@ void LLWebRTCImpl::OnDevicesUpdated() void LLWebRTCImpl::setTuningMode(bool enable) { - mWorkerThread->BlockingCall( - [this, enable]() - { - if (enable) - { - - mTuningDeviceModule->StartRecording(); - - if (mPeerDeviceModule) - { - mPeerDeviceModule->StopRecording(); - } - } - else - { - mTuningDeviceModule->StartRecording(); - if (mPeerDeviceModule) - { - mPeerDeviceModule->StartRecording(); - } - } - }); - for (auto& connection : mPeerConnections) - { - if (enable) + mSignalingThread->PostTask( + [this, enable] { - connection->enableSenderTracks(false); - } - else - { - connection->resetMute(); - } - connection->enableReceiverTracks(!enable); - } + for (auto &connection : mPeerConnections) + { + if (enable) + { + connection->enableSenderTracks(false); + } + else + { + connection->resetMute(); + } + connection->enableReceiverTracks(!enable); + } + }); } float LLWebRTCImpl::getTuningAudioLevel() { return -20 * log10f(mTuningAudioDeviceObserver->getMicrophoneEnergy()); } -- cgit v1.2.3 From d8ebab0221dfd265b37fde9be10ba1f650b3f790 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Thu, 8 Feb 2024 18:48:36 -0800 Subject: fix rebase issue --- indra/llwebrtc/llwebrtc.cpp | 14 -------------- 1 file changed, 14 deletions(-) (limited to 'indra/llwebrtc/llwebrtc.cpp') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index ee85a254f2..768405b75f 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -173,20 +173,6 @@ void LLWebRTCImpl::init() updateDevices(); }); - rtc::scoped_refptr apm = webrtc::AudioProcessingBuilder().Create(); - webrtc::AudioProcessing::Config apm_config; - apm_config.echo_canceller.enabled = true; - apm_config.echo_canceller.mobile_mode = false; - apm_config.gain_controller1.enabled = true; - apm_config.gain_controller1.mode = webrtc::AudioProcessing::Config::GainController1::kAdaptiveAnalog; - apm_config.gain_controller2.enabled = true; - apm_config.high_pass_filter.enabled = true; - apm_config.noise_suppression.enabled = true; - apm_config.noise_suppression.level = webrtc::AudioProcessing::Config::NoiseSuppression::kVeryHigh; - apm_config.transient_suppression.enabled = true; - apm_config.pipeline.multi_channel_render = true; - apm_config.pipeline.multi_channel_capture = true; - mWorkerThread->BlockingCall( [this]() { -- cgit v1.2.3 From 29b317db512bceb8a8ba217da2109faa9b1c5cc9 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Thu, 8 Feb 2024 19:02:13 -0800 Subject: another rebase merge issue --- indra/llwebrtc/llwebrtc.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'indra/llwebrtc/llwebrtc.cpp') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index 768405b75f..283124e315 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -970,7 +970,7 @@ void LLWebRTCPeerConnectionImpl::OnSuccess(webrtc::SessionDescriptionInterface * else if (sdp_line.find("a=fmtp:" + opus_payload) == 0) { sdp_mangled_stream << sdp_line << "a=fmtp:" << opus_payload - << " minptime=10;useinbandfec=1;stereo=1;sprop-stereo=1;maxplaybackrate=48000;sprop-maxcapturerate=48000;sprop-maxplaybackrate=48000\n"; + << " minptime=10;useinbandfec=1;stereo=1;sprop-stereo=1;maxplaybackrate=48000;sprop-maxplaybackrate=48000;sprop-maxcapturerate=48000\n"; } else { -- cgit v1.2.3 From 4057d38e969b2a9d29d111ee3b238a562e82fd9a Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Thu, 22 Feb 2024 23:25:27 -0800 Subject: coding cleanup --- indra/llwebrtc/llwebrtc.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'indra/llwebrtc/llwebrtc.cpp') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index 283124e315..adde174a74 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -40,7 +40,7 @@ namespace llwebrtc { -LLAudioDeviceObserver::LLAudioDeviceObserver() : mMicrophoneEnergy(0.0), mSumVector {0} {} +LLAudioDeviceObserver::LLAudioDeviceObserver() : mSumVector {0}, mMicrophoneEnergy(0.0) {} float LLAudioDeviceObserver::getMicrophoneEnergy() { return mMicrophoneEnergy; } -- cgit v1.2.3 From 70044b9d2bbc594f0e8f3154feb2dbce77a7af82 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Tue, 5 Mar 2024 18:57:22 -0800 Subject: The response from the provision account call was being called twice for some reason --- indra/llwebrtc/llwebrtc.cpp | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) (limited to 'indra/llwebrtc/llwebrtc.cpp') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index adde174a74..74af16679a 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -649,21 +649,26 @@ bool LLWebRTCPeerConnectionImpl::initializeConnection() return true; } -void LLWebRTCPeerConnectionImpl::shutdownConnection() +bool LLWebRTCPeerConnectionImpl::shutdownConnection() { - mWebRTCImpl->PostSignalingTask( - [this]() - { - if (mPeerConnection) - { - mPeerConnection->Close(); - mPeerConnection = nullptr; - } - for (auto &observer : mSignalingObserverList) + if (mPeerConnection) + { + mWebRTCImpl->PostSignalingTask( + [this]() { - observer->OnPeerShutDown(); - } - }); + if (mPeerConnection) + { + mPeerConnection->Close(); + mPeerConnection = nullptr; + } + for (auto &observer : mSignalingObserverList) + { + observer->OnPeerShutDown(); + } + }); + return true; + } + return false; } void LLWebRTCPeerConnectionImpl::enableSenderTracks(bool enable) -- cgit v1.2.3 From 7714850fbe2ad4c2dc914798971d82cbb82f7832 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Sat, 9 Mar 2024 23:00:00 -0800 Subject: code beautification/comments --- indra/llwebrtc/llwebrtc.cpp | 213 +++++++++++++++++++++++++++++++------------- 1 file changed, 149 insertions(+), 64 deletions(-) (limited to 'indra/llwebrtc/llwebrtc.cpp') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index 74af16679a..dc5fd9657e 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -44,12 +44,16 @@ LLAudioDeviceObserver::LLAudioDeviceObserver() : mSumVector {0}, mMicrophoneEner float LLAudioDeviceObserver::getMicrophoneEnergy() { return mMicrophoneEnergy; } +// TODO: Pull smoothing/filtering code into a common helper function +// for LLAudioDeviceObserver and LLCustomProcessor + void LLAudioDeviceObserver::OnCaptureData(const void *audio_samples, const size_t num_samples, const size_t bytes_per_sample, const size_t num_channels, const uint32_t samples_per_sec) { + // calculate the energy float energy = 0; const short *samples = (const short *) audio_samples; for (size_t index = 0; index < num_samples * num_channels; index++) @@ -82,7 +86,8 @@ void LLAudioDeviceObserver::OnRenderData(const void *audio_samples, LLCustomProcessor::LLCustomProcessor() : mSampleRateHz(0), - mNumChannels(0) + mNumChannels(0), + mMicrophoneEnergy(0.0) { memset(mSumVector, 0, sizeof(mSumVector)); } @@ -107,6 +112,7 @@ void LLCustomProcessor::Process(webrtc::AudioBuffer *audio_in) return; } + // grab the input audio frame_samples.resize(stream_config.num_samples()); frame.resize(stream_config.num_channels()); for (size_t ch = 0; ch < stream_config.num_channels(); ++ch) @@ -116,6 +122,7 @@ void LLCustomProcessor::Process(webrtc::AudioBuffer *audio_in) audio_in->CopyTo(stream_config, &frame[0]); + // calculate the energy float energy = 0; for (size_t index = 0; index < stream_config.num_samples(); index++) { @@ -137,17 +144,33 @@ void LLCustomProcessor::Process(webrtc::AudioBuffer *audio_in) mMicrophoneEnergy = std::sqrt(totalSum / (stream_config.num_samples() * buffer_size)); } +// +// LLWebRTCImpl implementation +// + +LLWebRTCImpl::LLWebRTCImpl() : + mPeerCustomProcessor(nullptr), + mMute(true), + mPlayoutDevice(0), + mRecordingDevice(0), + mTuningAudioDeviceObserver(nullptr) +{ +} + void LLWebRTCImpl::init() { RTC_DCHECK(mPeerConnectionFactory); mPlayoutDevice = 0; mRecordingDevice = 0; rtc::InitializeSSL(); + + // Normal logging is rather spammy, so turn it off. rtc::LogMessage::LogToDebug(rtc::LS_NONE); rtc::LogMessage::SetLogToStderr(true); mTaskQueueFactory = webrtc::CreateDefaultTaskQueueFactory(); - + + // Create the native threads. mNetworkThread = rtc::Thread::CreateWithSocketServer(); mNetworkThread->SetName("WebRTCNetworkThread", nullptr); mNetworkThread->Start(); @@ -162,9 +185,12 @@ void LLWebRTCImpl::init() mWorkerThread->PostTask( [this]() { + // Initialize the audio devices on the Worker Thread mTuningDeviceModule = webrtc::CreateAudioDeviceWithDataObserver( - webrtc::AudioDeviceModule::AudioLayer::kPlatformDefaultAudio, mTaskQueueFactory.get(), - std::unique_ptr(mTuningAudioDeviceObserver)); + webrtc::AudioDeviceModule::AudioLayer::kPlatformDefaultAudio, + mTaskQueueFactory.get(), + std::unique_ptr(mTuningAudioDeviceObserver)); + mTuningDeviceModule->Init(); mTuningDeviceModule->SetStereoRecording(true); mTuningDeviceModule->SetStereoPlayout(true); @@ -176,9 +202,13 @@ void LLWebRTCImpl::init() mWorkerThread->BlockingCall( [this]() { + // the peer device module doesn't need an observer + // as we pull peer data after audio processing. mPeerDeviceModule = - webrtc::CreateAudioDeviceWithDataObserver(webrtc::AudioDeviceModule::AudioLayer::kPlatformDefaultAudio, - mTaskQueueFactory.get(), nullptr); + webrtc::CreateAudioDeviceWithDataObserver( + webrtc::AudioDeviceModule::AudioLayer::kPlatformDefaultAudio, + mTaskQueueFactory.get(), + nullptr); mPeerDeviceModule->Init(); mPeerDeviceModule->SetPlayoutDevice(mPlayoutDevice); mPeerDeviceModule->SetRecordingDevice(mRecordingDevice); @@ -191,11 +221,15 @@ void LLWebRTCImpl::init() mPeerDeviceModule->InitPlayout(); }); + // The custom processor allows us to retrieve audio data (and levels) + // from after other audio processing such as AEC, AGC, etc. mPeerCustomProcessor = new LLCustomProcessor; webrtc::AudioProcessingBuilder apb; apb.SetCapturePostProcessing(std::unique_ptr(mPeerCustomProcessor)); rtc::scoped_refptr apm = apb.Create(); + // TODO: wire some of these to the primary interface and ultimately + // to the UI to allow user config. webrtc::AudioProcessing::Config apm_config; apm_config.echo_canceller.enabled = true; apm_config.echo_canceller.mobile_mode = false; @@ -242,6 +276,41 @@ void LLWebRTCImpl::init() }); } +void LLWebRTCImpl::terminate() +{ + for (auto &connection : mPeerConnections) + { + connection->terminate(); + } + mPeerConnections.clear(); + + mSignalingThread->BlockingCall([this]() { mPeerConnectionFactory = nullptr; }); + mWorkerThread->BlockingCall( + [this]() + { + if (mTuningDeviceModule) + { + mTuningDeviceModule->StopRecording(); + mTuningDeviceModule->Terminate(); + } + if (mPeerDeviceModule) + { + mPeerDeviceModule->StopRecording(); + mPeerDeviceModule->Terminate(); + } + mTuningDeviceModule = nullptr; + mPeerDeviceModule = nullptr; + mTaskQueueFactory = nullptr; + }); +} + +// +// Devices functions +// +// Most device-related functionality needs to happen +// on the worker thread (the audio thread,) so those calls will be +// proxied over to that thread. +// void LLWebRTCImpl::setRecording(bool recording) { mWorkerThread->PostTask( @@ -258,38 +327,6 @@ void LLWebRTCImpl::setRecording(bool recording) }); } -void LLWebRTCImpl::terminate() -{ - for (auto& connection : mPeerConnections) - { - connection->terminate(); - } - mPeerConnections.clear(); - - mSignalingThread->BlockingCall( - [this]() - { - mPeerConnectionFactory = nullptr; - }); - mWorkerThread->BlockingCall( - [this]() - { - if (mTuningDeviceModule) - { - mTuningDeviceModule->StopRecording(); - mTuningDeviceModule->Terminate(); - } - if (mPeerDeviceModule) - { - mPeerDeviceModule->StopRecording(); - mPeerDeviceModule->Terminate(); - } - mTuningDeviceModule = nullptr; - mPeerDeviceModule = nullptr; - mTaskQueueFactory = nullptr; - }); -} - void LLWebRTCImpl::refreshDevices() { mWorkerThread->PostTask([this]() { updateDevices(); }); @@ -307,6 +344,8 @@ void LLWebRTCImpl::unsetDevicesObserver(LLWebRTCDevicesObserver *observer) } } +// TODO: There's potential for shared code here as the patterns +// are similar. void LLWebRTCImpl::setCaptureDevice(const std::string &id) { mWorkerThread->PostTask( @@ -377,7 +416,7 @@ void LLWebRTCImpl::setRenderDevice(const std::string &id) mTuningDeviceModule->PlayoutDeviceName(i, name, guid); if (id == guid || id == "Default") { - RTC_LOG(LS_INFO) << __FUNCTION__ << "Set recording device to " << name << " " << guid << " " << i; + RTC_LOG(LS_INFO) << __FUNCTION__ << "Set playout device to " << name << " " << guid << " " << i; tuningPlayoutDevice = i; break; } @@ -407,7 +446,7 @@ void LLWebRTCImpl::setRenderDevice(const std::string &id) if (id == guid || id == "Default") { RTC_LOG(LS_INFO) - << __FUNCTION__ << "Set recording device to " << name << " " << guid << " " << i; + << __FUNCTION__ << "Set playout device to " << name << " " << guid << " " << i; mPlayoutDevice = i; break; } @@ -422,6 +461,7 @@ void LLWebRTCImpl::setRenderDevice(const std::string &id) }); } +// updateDevices needs to happen on the worker thread. void LLWebRTCImpl::updateDevices() { int16_t renderDeviceCount = mTuningDeviceModule->PlayoutDevices(); @@ -482,13 +522,14 @@ void LLWebRTCImpl::setTuningMode(bool enable) float LLWebRTCImpl::getTuningAudioLevel() { return -20 * log10f(mTuningAudioDeviceObserver->getMicrophoneEnergy()); } -float LLWebRTCImpl::getPeerAudioLevel() { return -20 * log10f(mPeerCustomProcessor->getMicrophoneEnergy()); } +float LLWebRTCImpl::getPeerConnectionAudioLevel() { return -20 * log10f(mPeerCustomProcessor->getMicrophoneEnergy()); } + // -// Helpers +// Peer Connection Helpers // -LLWebRTCPeerConnection * LLWebRTCImpl::newPeerConnection() +LLWebRTCPeerConnectionInterface *LLWebRTCImpl::newPeerConnection() { rtc::scoped_refptr peerConnection = rtc::scoped_refptr(new rtc::RefCountedObject()); peerConnection->init(this); @@ -498,7 +539,7 @@ LLWebRTCPeerConnection * LLWebRTCImpl::newPeerConnection() return peerConnection.get(); } -void LLWebRTCImpl::freePeerConnection(LLWebRTCPeerConnection * peer_connection) +void LLWebRTCImpl::freePeerConnection(LLWebRTCPeerConnectionInterface* peer_connection) { std::vector>::iterator it = std::find(mPeerConnections.begin(), mPeerConnections.end(), peer_connection); @@ -513,6 +554,20 @@ void LLWebRTCImpl::freePeerConnection(LLWebRTCPeerConnection * peer_connection) } } + +// +// LLWebRTCPeerConnectionImpl implementation. +// +// Most peer connection (signaling) happens on +// the signaling thread. + +LLWebRTCPeerConnectionImpl::LLWebRTCPeerConnectionImpl() : + mWebRTCImpl(nullptr), + mMute(false), + mAnswerReceived(false) +{ +} + // // LLWebRTCPeerConnection interface // @@ -547,6 +602,9 @@ void LLWebRTCPeerConnectionImpl::unsetSignalingObserver(LLWebRTCSignalingObserve } } +// TODO: Add initialization structure through which +// stun and turn servers may be passed in from +// the sim or login. bool LLWebRTCPeerConnectionImpl::initializeConnection() { RTC_DCHECK(!mPeerConnection); @@ -663,7 +721,7 @@ bool LLWebRTCPeerConnectionImpl::shutdownConnection() } for (auto &observer : mSignalingObserverList) { - observer->OnPeerShutDown(); + observer->OnPeerConnectionShutdown(); } }); return true; @@ -673,7 +731,7 @@ bool LLWebRTCPeerConnectionImpl::shutdownConnection() void LLWebRTCPeerConnectionImpl::enableSenderTracks(bool enable) { - // set_enabled shouldn't be done on the worker thread + // set_enabled shouldn't be done on the worker thread. if (mPeerConnection) { auto senders = mPeerConnection->GetSenders(); @@ -697,6 +755,7 @@ void LLWebRTCPeerConnectionImpl::enableReceiverTracks(bool enable) } } +// Tell the peer connection that we've received a SDP answer from the sim. void LLWebRTCPeerConnectionImpl::AnswerAvailable(const std::string &sdp) { RTC_LOG(LS_INFO) << __FUNCTION__ << " Remote SDP: " << sdp; @@ -713,6 +772,11 @@ void LLWebRTCPeerConnectionImpl::AnswerAvailable(const std::string &sdp) }); } + +// +// LLWebRTCAudioInterface implementation +// + void LLWebRTCPeerConnectionImpl::setMute(bool mute) { mMute = mute; @@ -812,21 +876,21 @@ void LLWebRTCPeerConnectionImpl::OnDataChannel(rtc::scoped_refptrsdp_mline_index(); @@ -931,19 +998,24 @@ void LLWebRTCPeerConnectionImpl::OnIceCandidate(const webrtc::IceCandidateInterf } if (mAnswerReceived) { + // We've already received an answer SDP from the Secondlife WebRTC server + // so simply tell observers about our new ice candidate. for (auto &observer : mSignalingObserverList) { LLWebRTCIceCandidate ice_candidate; - ice_candidate.candidate = iceCandidateToTrickleString(candidate); - ice_candidate.mline_index = candidate->sdp_mline_index(); - ice_candidate.sdp_mid = candidate->sdp_mid(); + ice_candidate.mCandidate = iceCandidateToTrickleString(candidate); + ice_candidate.mMLineIndex = candidate->sdp_mline_index(); + ice_candidate.mSdpMid = candidate->sdp_mid(); observer->OnIceCandidate(ice_candidate); } } else { + // As we've not yet received our answer, cache the candidate. mCachedIceCandidates.push_back( - webrtc::CreateIceCandidate(candidate->sdp_mid(), candidate->sdp_mline_index(), candidate->candidate())); + webrtc::CreateIceCandidate(candidate->sdp_mid(), + candidate->sdp_mline_index(), + candidate->candidate())); } } @@ -994,13 +1066,18 @@ void LLWebRTCPeerConnectionImpl::OnSuccess(webrtc::SessionDescriptionInterface * rtc::scoped_refptr(this)); } -void LLWebRTCPeerConnectionImpl::OnFailure(webrtc::RTCError error) { RTC_LOG(LS_ERROR) << ToString(error.type()) << ": " << error.message(); } +void LLWebRTCPeerConnectionImpl::OnFailure(webrtc::RTCError error) +{ + RTC_LOG(LS_ERROR) << ToString(error.type()) << ": " << error.message(); +} // // SetRemoteDescriptionObserverInterface implementation. // void LLWebRTCPeerConnectionImpl::OnSetRemoteDescriptionComplete(webrtc::RTCError error) { + // we've received an answer SDP from the sim. + RTC_LOG(LS_INFO) << __FUNCTION__ << " " << mPeerConnection->signaling_state(); if (!error.ok()) { @@ -1008,14 +1085,16 @@ void LLWebRTCPeerConnectionImpl::OnSetRemoteDescriptionComplete(webrtc::RTCError return; } mAnswerReceived = true; + + // tell the observers about any cached ICE candidates. for (auto &observer : mSignalingObserverList) { for (auto &candidate : mCachedIceCandidates) { LLWebRTCIceCandidate ice_candidate; - ice_candidate.candidate = iceCandidateToTrickleString(candidate.get()); - ice_candidate.mline_index = candidate->sdp_mline_index(); - ice_candidate.sdp_mid = candidate->sdp_mid(); + ice_candidate.mCandidate = iceCandidateToTrickleString(candidate.get()); + ice_candidate.mMLineIndex = candidate->sdp_mline_index(); + ice_candidate.mSdpMid = candidate->sdp_mid(); observer->OnIceCandidate(ice_candidate); } } @@ -1061,7 +1140,6 @@ void LLWebRTCPeerConnectionImpl::OnStateChange() } } - void LLWebRTCPeerConnectionImpl::OnMessage(const webrtc::DataBuffer& buffer) { std::string data((const char*)buffer.data.cdata(), buffer.size()); @@ -1071,6 +1149,10 @@ void LLWebRTCPeerConnectionImpl::OnMessage(const webrtc::DataBuffer& buffer) } } +// +// LLWebRTCDataInterface +// + void LLWebRTCPeerConnectionImpl::sendData(const std::string& data, bool binary) { if (mDataChannel) @@ -1081,7 +1163,10 @@ void LLWebRTCPeerConnectionImpl::sendData(const std::string& data, bool binary) } } -void LLWebRTCPeerConnectionImpl::setDataObserver(LLWebRTCDataObserver* observer) { mDataObserverList.emplace_back(observer); } +void LLWebRTCPeerConnectionImpl::setDataObserver(LLWebRTCDataObserver* observer) +{ + mDataObserverList.emplace_back(observer); +} void LLWebRTCPeerConnectionImpl::unsetDataObserver(LLWebRTCDataObserver* observer) { @@ -1099,12 +1184,12 @@ LLWebRTCDeviceInterface * getDeviceInterface() return gWebRTCImpl; } -LLWebRTCPeerConnection * newPeerConnection() +LLWebRTCPeerConnectionInterface* newPeerConnection() { return gWebRTCImpl->newPeerConnection(); } -void freePeerConnection(LLWebRTCPeerConnection *peer_connection) +void freePeerConnection(LLWebRTCPeerConnectionInterface* peer_connection) { gWebRTCImpl->freePeerConnection(peer_connection); } @@ -1112,7 +1197,7 @@ void freePeerConnection(LLWebRTCPeerConnection *peer_connection) void init() { - gWebRTCImpl = new LLWebRTCImpl(); + gWebRTCImpl = new LLWebRTCImpl(); gWebRTCImpl->init(); } -- cgit v1.2.3 From fc462b2b0f7ee153626d162bd97a05110f0804b7 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Sun, 10 Mar 2024 00:11:43 -0800 Subject: Remove trailing spaces. Other code cleanup. --- indra/llwebrtc/llwebrtc.cpp | 50 ++++++++++++++++++++++----------------------- 1 file changed, 25 insertions(+), 25 deletions(-) (limited to 'indra/llwebrtc/llwebrtc.cpp') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index dc5fd9657e..875f233e65 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -85,15 +85,15 @@ void LLAudioDeviceObserver::OnRenderData(const void *audio_samples, } LLCustomProcessor::LLCustomProcessor() : - mSampleRateHz(0), + mSampleRateHz(0), mNumChannels(0), mMicrophoneEnergy(0.0) -{ +{ memset(mSumVector, 0, sizeof(mSumVector)); } void LLCustomProcessor::Initialize(int sample_rate_hz, int num_channels) -{ +{ mSampleRateHz = sample_rate_hz; mNumChannels = num_channels; memset(mSumVector, 0, sizeof(mSumVector)); @@ -187,7 +187,7 @@ void LLWebRTCImpl::init() { // Initialize the audio devices on the Worker Thread mTuningDeviceModule = webrtc::CreateAudioDeviceWithDataObserver( - webrtc::AudioDeviceModule::AudioLayer::kPlatformDefaultAudio, + webrtc::AudioDeviceModule::AudioLayer::kPlatformDefaultAudio, mTaskQueueFactory.get(), std::unique_ptr(mTuningAudioDeviceObserver)); @@ -198,7 +198,7 @@ void LLWebRTCImpl::init() mTuningDeviceModule->SetAudioDeviceSink(this); updateDevices(); }); - + mWorkerThread->BlockingCall( [this]() { @@ -426,7 +426,7 @@ void LLWebRTCImpl::setRenderDevice(const std::string &id) { mTuningDeviceModule->StopPlayout(); } - + mTuningDeviceModule->SetPlayoutDevice(tuningPlayoutDevice); mTuningDeviceModule->InitSpeaker(); mTuningDeviceModule->InitPlayout(); @@ -456,7 +456,7 @@ void LLWebRTCImpl::setRenderDevice(const std::string &id) mPeerDeviceModule->InitSpeaker(); mPeerDeviceModule->InitPlayout(); mPeerDeviceModule->StartPlayout(); - + } }); } @@ -466,7 +466,7 @@ void LLWebRTCImpl::updateDevices() { int16_t renderDeviceCount = mTuningDeviceModule->PlayoutDevices(); int16_t currentRenderDeviceIndex = mTuningDeviceModule->GetPlayoutDevice(); - + LLWebRTCVoiceDeviceList renderDeviceList; for (int16_t index = 0; index < renderDeviceCount; index++) { @@ -475,7 +475,7 @@ void LLWebRTCImpl::updateDevices() mTuningDeviceModule->PlayoutDeviceName(index, name, guid); renderDeviceList.emplace_back(name, guid, index == currentRenderDeviceIndex); } - + int16_t captureDeviceCount = mTuningDeviceModule->RecordingDevices(); int16_t currentCaptureDeviceIndex = mTuningDeviceModule->GetRecordingDevice(); @@ -533,7 +533,7 @@ LLWebRTCPeerConnectionInterface *LLWebRTCImpl::newPeerConnection() { rtc::scoped_refptr peerConnection = rtc::scoped_refptr(new rtc::RefCountedObject()); peerConnection->init(this); - + mPeerConnections.emplace_back(peerConnection); peerConnection->enableSenderTracks(!mMute); return peerConnection.get(); @@ -703,7 +703,7 @@ bool LLWebRTCPeerConnectionImpl::initializeConnection() webrtc::PeerConnectionInterface::RTCOfferAnswerOptions offerOptions; mPeerConnection->CreateOffer(this, offerOptions); }); - + return true; } @@ -759,7 +759,7 @@ void LLWebRTCPeerConnectionImpl::enableReceiverTracks(bool enable) void LLWebRTCPeerConnectionImpl::AnswerAvailable(const std::string &sdp) { RTC_LOG(LS_INFO) << __FUNCTION__ << " Remote SDP: " << sdp; - + mWebRTCImpl->PostSignalingTask( [this, sdp]() { @@ -768,7 +768,7 @@ void LLWebRTCPeerConnectionImpl::AnswerAvailable(const std::string &sdp) RTC_LOG(LS_INFO) << __FUNCTION__ << " " << mPeerConnection->peer_connection_state(); mPeerConnection->SetRemoteDescription(webrtc::CreateSessionDescription(webrtc::SdpType::kAnswer, sdp), rtc::scoped_refptr(this)); - } + } }); } @@ -801,8 +801,8 @@ void LLWebRTCPeerConnectionImpl::setMute(bool mute) } void LLWebRTCPeerConnectionImpl::resetMute() -{ - setMute(mMute); +{ + setMute(mMute); } void LLWebRTCPeerConnectionImpl::setReceiveVolume(float volume) @@ -893,7 +893,7 @@ void LLWebRTCPeerConnectionImpl::OnIceGatheringChange(webrtc::PeerConnectionInte webrtc_new_state = LLWebRTCSignalingObserver::EIceGatheringState::ICE_GATHERING_NEW; return; } - + if (mAnswerReceived) { for (auto &observer : mSignalingObserverList) @@ -907,7 +907,7 @@ void LLWebRTCPeerConnectionImpl::OnIceGatheringChange(webrtc::PeerConnectionInte void LLWebRTCPeerConnectionImpl::OnConnectionChange(webrtc::PeerConnectionInterface::PeerConnectionState new_state) { RTC_LOG(LS_ERROR) << __FUNCTION__ << " Peer Connection State Change " << new_state; - + switch (new_state) { case webrtc::PeerConnectionInterface::PeerConnectionState::kConnected: @@ -929,7 +929,7 @@ void LLWebRTCPeerConnectionImpl::OnConnectionChange(webrtc::PeerConnectionInterf { observer->OnRenegotiationNeeded(); } - + break; } default: @@ -944,7 +944,7 @@ void LLWebRTCPeerConnectionImpl::OnConnectionChange(webrtc::PeerConnectionInterf static std::string iceCandidateToTrickleString(const webrtc::IceCandidateInterface *candidate) { std::ostringstream candidate_stream; - + candidate_stream << candidate->candidate().foundation() << " " << std::to_string(candidate->candidate().component()) << " " << @@ -952,7 +952,7 @@ static std::string iceCandidateToTrickleString(const webrtc::IceCandidateInterfa std::to_string(candidate->candidate().priority()) << " " << candidate->candidate().address().ipaddr().ToString() << " " << candidate->candidate().address().PortAsString() << " typ "; - + if (candidate->candidate().type() == cricket::LOCAL_PORT_TYPE) { candidate_stream << "host"; @@ -982,7 +982,7 @@ static std::string iceCandidateToTrickleString(const webrtc::IceCandidateInterfa { candidate_stream << " tcptype " << candidate->candidate().tcptype(); } - + return candidate_stream.str(); } @@ -990,7 +990,7 @@ static std::string iceCandidateToTrickleString(const webrtc::IceCandidateInterfa void LLWebRTCPeerConnectionImpl::OnIceCandidate(const webrtc::IceCandidateInterface *candidate) { RTC_LOG(LS_INFO) << __FUNCTION__ << " " << candidate->sdp_mline_index(); - + if (!candidate) { RTC_LOG(LS_ERROR) << __FUNCTION__ << " No Ice Candidate Given"; @@ -1067,7 +1067,7 @@ void LLWebRTCPeerConnectionImpl::OnSuccess(webrtc::SessionDescriptionInterface * } void LLWebRTCPeerConnectionImpl::OnFailure(webrtc::RTCError error) -{ +{ RTC_LOG(LS_ERROR) << ToString(error.type()) << ": " << error.message(); } @@ -1100,7 +1100,7 @@ void LLWebRTCPeerConnectionImpl::OnSetRemoteDescriptionComplete(webrtc::RTCError } mCachedIceCandidates.clear(); OnIceGatheringChange(mPeerConnection->ice_gathering_state()); - + } // @@ -1164,7 +1164,7 @@ void LLWebRTCPeerConnectionImpl::sendData(const std::string& data, bool binary) } void LLWebRTCPeerConnectionImpl::setDataObserver(LLWebRTCDataObserver* observer) -{ +{ mDataObserverList.emplace_back(observer); } -- cgit v1.2.3 From dbbbbc55af5c1b5e81e7a493a9b5fe5718f15c07 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Thu, 14 Mar 2024 20:04:39 -0700 Subject: Refactor device selection logic This refactor fixed a few bugs. There is an annoying 'click' when changing devices, however. This will be addressed in the future. --- indra/llwebrtc/llwebrtc.cpp | 321 ++++++++++++++++++++++---------------------- 1 file changed, 161 insertions(+), 160 deletions(-) (limited to 'indra/llwebrtc/llwebrtc.cpp') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index 875f233e65..b7501bd0e0 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -84,10 +84,7 @@ void LLAudioDeviceObserver::OnRenderData(const void *audio_samples, { } -LLCustomProcessor::LLCustomProcessor() : - mSampleRateHz(0), - mNumChannels(0), - mMicrophoneEnergy(0.0) +LLCustomProcessor::LLCustomProcessor() : mSampleRateHz(0), mNumChannels(0), mMicrophoneEnergy(0.0) { memset(mSumVector, 0, sizeof(mSumVector)); } @@ -95,7 +92,7 @@ LLCustomProcessor::LLCustomProcessor() : void LLCustomProcessor::Initialize(int sample_rate_hz, int num_channels) { mSampleRateHz = sample_rate_hz; - mNumChannels = num_channels; + mNumChannels = num_channels; memset(mSumVector, 0, sizeof(mSumVector)); } @@ -105,7 +102,7 @@ void LLCustomProcessor::Process(webrtc::AudioBuffer *audio_in) stream_config.set_sample_rate_hz(mSampleRateHz); stream_config.set_num_channels(mNumChannels); std::vector frame; - std::vector frame_samples; + std::vector frame_samples; if (audio_in->num_channels() < 1 || audio_in->num_frames() < 480) { @@ -123,7 +120,7 @@ void LLCustomProcessor::Process(webrtc::AudioBuffer *audio_in) audio_in->CopyTo(stream_config, &frame[0]); // calculate the energy - float energy = 0; + float energy = 0; for (size_t index = 0; index < stream_config.num_samples(); index++) { float sample = frame_samples[index]; @@ -151,6 +148,7 @@ void LLCustomProcessor::Process(webrtc::AudioBuffer *audio_in) LLWebRTCImpl::LLWebRTCImpl() : mPeerCustomProcessor(nullptr), mMute(true), + mTuningMode(false), mPlayoutDevice(0), mRecordingDevice(0), mTuningAudioDeviceObserver(nullptr) @@ -160,7 +158,7 @@ LLWebRTCImpl::LLWebRTCImpl() : void LLWebRTCImpl::init() { RTC_DCHECK(mPeerConnectionFactory); - mPlayoutDevice = 0; + mPlayoutDevice = 0; mRecordingDevice = 0; rtc::InitializeSSL(); @@ -183,43 +181,41 @@ void LLWebRTCImpl::init() mTuningAudioDeviceObserver = new LLAudioDeviceObserver; mWorkerThread->PostTask( - [this]() - { - // Initialize the audio devices on the Worker Thread - mTuningDeviceModule = webrtc::CreateAudioDeviceWithDataObserver( - webrtc::AudioDeviceModule::AudioLayer::kPlatformDefaultAudio, - mTaskQueueFactory.get(), - std::unique_ptr(mTuningAudioDeviceObserver)); - - mTuningDeviceModule->Init(); - mTuningDeviceModule->SetStereoRecording(true); - mTuningDeviceModule->SetStereoPlayout(true); - mTuningDeviceModule->EnableBuiltInAEC(false); - mTuningDeviceModule->SetAudioDeviceSink(this); - updateDevices(); - }); + [this]() + { + // Initialize the audio devices on the Worker Thread + mTuningDeviceModule = + webrtc::CreateAudioDeviceWithDataObserver(webrtc::AudioDeviceModule::AudioLayer::kPlatformDefaultAudio, + mTaskQueueFactory.get(), + std::unique_ptr(mTuningAudioDeviceObserver)); + + mTuningDeviceModule->Init(); + mTuningDeviceModule->SetStereoRecording(true); + mTuningDeviceModule->SetStereoPlayout(true); + mTuningDeviceModule->EnableBuiltInAEC(false); + mTuningDeviceModule->SetAudioDeviceSink(this); + updateDevices(); + }); mWorkerThread->BlockingCall( - [this]() - { - // the peer device module doesn't need an observer - // as we pull peer data after audio processing. - mPeerDeviceModule = - webrtc::CreateAudioDeviceWithDataObserver( - webrtc::AudioDeviceModule::AudioLayer::kPlatformDefaultAudio, - mTaskQueueFactory.get(), - nullptr); - mPeerDeviceModule->Init(); - mPeerDeviceModule->SetPlayoutDevice(mPlayoutDevice); - mPeerDeviceModule->SetRecordingDevice(mRecordingDevice); - mPeerDeviceModule->SetStereoRecording(true); - mPeerDeviceModule->SetStereoPlayout(true); - mPeerDeviceModule->EnableBuiltInAEC(false); - mPeerDeviceModule->InitMicrophone(); - mPeerDeviceModule->InitSpeaker(); - mPeerDeviceModule->InitRecording(); - mPeerDeviceModule->InitPlayout(); - }); + [this]() + { + // the peer device module doesn't need an observer + // as we pull peer data after audio processing. + mPeerDeviceModule = webrtc::CreateAudioDeviceWithDataObserver(webrtc::AudioDeviceModule::AudioLayer::kPlatformDefaultAudio, + mTaskQueueFactory.get(), + nullptr); + mPeerDeviceModule->Init(); + mPeerDeviceModule->SetPlayoutDevice(mPlayoutDevice); + mPeerDeviceModule->SetRecordingDevice(mRecordingDevice); + mPeerDeviceModule->SetStereoRecording(true); + mPeerDeviceModule->SetStereoPlayout(true); + mPeerDeviceModule->EnableBuiltInAEC(false); + mPeerDeviceModule->InitMicrophone(); + mPeerDeviceModule->InitSpeaker(); + mPeerDeviceModule->InitRecording(); + mPeerDeviceModule->InitPlayout(); + }); // The custom processor allows us to retrieve audio data (and levels) // from after other audio processing such as AEC, AGC, etc. @@ -230,7 +226,7 @@ void LLWebRTCImpl::init() // TODO: wire some of these to the primary interface and ultimately // to the UI to allow user config. - webrtc::AudioProcessing::Config apm_config; + webrtc::AudioProcessing::Config apm_config; apm_config.echo_canceller.enabled = true; apm_config.echo_canceller.mobile_mode = false; apm_config.gain_controller1.enabled = true; @@ -268,12 +264,7 @@ void LLWebRTCImpl::init() nullptr /* audio_mixer */, apm); - - mWorkerThread->BlockingCall( - [this]() - { - mPeerDeviceModule->StartPlayout(); - }); + mWorkerThread->BlockingCall([this]() { mPeerDeviceModule->StartPlayout(); }); } void LLWebRTCImpl::terminate() @@ -337,128 +328,115 @@ void LLWebRTCImpl::setDevicesObserver(LLWebRTCDevicesObserver *observer) { mVoic void LLWebRTCImpl::unsetDevicesObserver(LLWebRTCDevicesObserver *observer) { std::vector::iterator it = - std::find(mVoiceDevicesObserverList.begin(), mVoiceDevicesObserverList.end(), observer); + std::find(mVoiceDevicesObserverList.begin(), mVoiceDevicesObserverList.end(), observer); if (it != mVoiceDevicesObserverList.end()) { mVoiceDevicesObserverList.erase(it); } } -// TODO: There's potential for shared code here as the patterns -// are similar. +static int16_t ll_get_device_module_capture_device(rtc::scoped_refptr device_module, const std::string &id) +{ + int16_t recordingDevice = 0; + int16_t captureDeviceCount = device_module->RecordingDevices(); + for (int16_t i = 0; i < captureDeviceCount; i++) + { + char name[webrtc::kAdmMaxDeviceNameSize]; + char guid[webrtc::kAdmMaxGuidSize]; + device_module->RecordingDeviceName(i, name, guid); + if (id == guid || id == "Default") // first one in list is default + { + RTC_LOG(LS_INFO) << __FUNCTION__ << "Set recording device to " << name << " " << guid << " " << i; + recordingDevice = i; + break; + } + } + return recordingDevice; +} + +void ll_set_device_module_capture_device(rtc::scoped_refptr device_module, int16_t device) +{ + device_module->StopRecording(); + device_module->SetRecordingDevice(device); + device_module->InitMicrophone(); + device_module->SetStereoRecording(false); + device_module->InitRecording(); + device_module->StartRecording(); +} + void LLWebRTCImpl::setCaptureDevice(const std::string &id) { + mWorkerThread->PostTask( - [this, id]() - { - int16_t tuningRecordingDevice = 0; - int16_t captureDeviceCount = mTuningDeviceModule->RecordingDevices(); - for (int16_t i = 0; i < captureDeviceCount; i++) - { - char name[webrtc::kAdmMaxDeviceNameSize]; - char guid[webrtc::kAdmMaxGuidSize]; - mTuningDeviceModule->RecordingDeviceName(i, name, guid); - if (id == guid || id == "Default") // first one in list is default - { - RTC_LOG(LS_INFO) << __FUNCTION__ << "Set recording device to " << name << " " << guid << " " << i; - tuningRecordingDevice = i; - break; - } - } - mTuningDeviceModule->StopRecording(); - mTuningDeviceModule->SetRecordingDevice(tuningRecordingDevice); - mTuningDeviceModule->InitMicrophone(); - mTuningDeviceModule->InitRecording(); - mTuningDeviceModule->StartRecording(); - if (mPeerDeviceModule) - { - int16_t captureDeviceCount = mPeerDeviceModule->RecordingDevices(); - for (int16_t i = 0; i < captureDeviceCount; i++) - { - char name[webrtc::kAdmMaxDeviceNameSize]; - char guid[webrtc::kAdmMaxGuidSize]; - mPeerDeviceModule->RecordingDeviceName(i, name, guid); - if (id == guid || id == "Default") // first one in list is default - { - RTC_LOG(LS_INFO) - << __FUNCTION__ << "Set recording device to " << name << " " << guid << " " << i; - mRecordingDevice = i; - break; - } - } - bool was_peer_recording = mPeerDeviceModule->Recording(); - if (was_peer_recording) - { - mPeerDeviceModule->StopRecording(); - } - mPeerDeviceModule->SetRecordingDevice(mRecordingDevice); - mPeerDeviceModule->InitMicrophone(); - mPeerDeviceModule->InitRecording(); - if (was_peer_recording) - { - mPeerDeviceModule->StartRecording(); - } - } - }); + [this, id]() + { + int16_t recordingDevice = ll_get_device_module_capture_device(mTuningDeviceModule, id); + if (recordingDevice != mRecordingDevice) + { + mRecordingDevice = recordingDevice; + if (mTuningMode) + { + ll_set_device_module_capture_device(mTuningDeviceModule, recordingDevice); + } + else + { + ll_set_device_module_capture_device(mPeerDeviceModule, recordingDevice); + } + } + }); +} + +static int16_t ll_get_device_module_render_device( + rtc::scoped_refptr device_module, + const std::string &id) +{ + int16_t playoutDevice = 0; + int16_t playoutDeviceCount = device_module->PlayoutDevices(); + for (int16_t i = 0; i < playoutDeviceCount; i++) + { + char name[webrtc::kAdmMaxDeviceNameSize]; + char guid[webrtc::kAdmMaxGuidSize]; + device_module->PlayoutDeviceName(i, name, guid); + if (id == guid || id == "Default") // first one in list is default + { + RTC_LOG(LS_INFO) << __FUNCTION__ << "Set recording device to " << name << " " << guid << " " << i; + playoutDevice = i; + break; + } + } + return playoutDevice; +} + + +void ll_set_device_module_render_device(rtc::scoped_refptr device_module, int16_t device) +{ + device_module->StopPlayout(); + device_module->SetPlayoutDevice(device); + device_module->InitSpeaker(); + device_module->SetStereoPlayout(false); + device_module->InitPlayout(); + device_module->StartPlayout(); } void LLWebRTCImpl::setRenderDevice(const std::string &id) { mWorkerThread->PostTask( - [this, id]() - { - int16_t renderDeviceCount = mTuningDeviceModule->PlayoutDevices(); - int16_t tuningPlayoutDevice = 0; - for (int16_t i = 0; i < renderDeviceCount; i++) - { - char name[webrtc::kAdmMaxDeviceNameSize]; - char guid[webrtc::kAdmMaxGuidSize]; - mTuningDeviceModule->PlayoutDeviceName(i, name, guid); - if (id == guid || id == "Default") - { - RTC_LOG(LS_INFO) << __FUNCTION__ << "Set playout device to " << name << " " << guid << " " << i; - tuningPlayoutDevice = i; - break; - } - } - bool was_tuning_playing = mTuningDeviceModule->Playing(); - if (was_tuning_playing) - { - mTuningDeviceModule->StopPlayout(); - } - - mTuningDeviceModule->SetPlayoutDevice(tuningPlayoutDevice); - mTuningDeviceModule->InitSpeaker(); - mTuningDeviceModule->InitPlayout(); - if (was_tuning_playing) - { - mTuningDeviceModule->StartPlayout(); - } - - if (mPeerDeviceModule) - { - renderDeviceCount = mPeerDeviceModule->PlayoutDevices(); - for (int16_t i = 0; i < renderDeviceCount; i++) - { - char name[webrtc::kAdmMaxDeviceNameSize]; - char guid[webrtc::kAdmMaxGuidSize]; - mPeerDeviceModule->PlayoutDeviceName(i, name, guid); - if (id == guid || id == "Default") - { - RTC_LOG(LS_INFO) - << __FUNCTION__ << "Set playout device to " << name << " " << guid << " " << i; - mPlayoutDevice = i; - break; - } - } - mPeerDeviceModule->StopPlayout(); - mPeerDeviceModule->SetPlayoutDevice(mPlayoutDevice); - mPeerDeviceModule->InitSpeaker(); - mPeerDeviceModule->InitPlayout(); - mPeerDeviceModule->StartPlayout(); - - } - }); + [this, id]() + { + int16_t playoutDevice = ll_get_device_module_render_device(mTuningDeviceModule, id); + if (playoutDevice != mPlayoutDevice) + { + mPlayoutDevice = playoutDevice; + if (mTuningMode) + { + ll_set_device_module_render_device(mTuningDeviceModule, playoutDevice); + } + else + { + ll_set_device_module_render_device(mPeerDeviceModule, playoutDevice); + } + } + }); } // updateDevices needs to happen on the worker thread. @@ -473,7 +451,7 @@ void LLWebRTCImpl::updateDevices() char name[webrtc::kAdmMaxDeviceNameSize]; char guid[webrtc::kAdmMaxGuidSize]; mTuningDeviceModule->PlayoutDeviceName(index, name, guid); - renderDeviceList.emplace_back(name, guid, index == currentRenderDeviceIndex); + renderDeviceList.emplace_back(name, guid); } int16_t captureDeviceCount = mTuningDeviceModule->RecordingDevices(); @@ -485,7 +463,7 @@ void LLWebRTCImpl::updateDevices() char name[webrtc::kAdmMaxDeviceNameSize]; char guid[webrtc::kAdmMaxGuidSize]; mTuningDeviceModule->RecordingDeviceName(index, name, guid); - captureDeviceList.emplace_back(name, guid, index == currentCaptureDeviceIndex); + captureDeviceList.emplace_back(name, guid); } for (auto &observer : mVoiceDevicesObserverList) { @@ -502,6 +480,29 @@ void LLWebRTCImpl::OnDevicesUpdated() void LLWebRTCImpl::setTuningMode(bool enable) { + mTuningMode = enable; + mWorkerThread->PostTask( + [this, enable] { + if (enable) + { + mPeerDeviceModule->StopRecording(); + mPeerDeviceModule->StopPlayout(); + ll_set_device_module_render_device(mTuningDeviceModule, mPlayoutDevice); + ll_set_device_module_capture_device(mTuningDeviceModule, mRecordingDevice); + mTuningDeviceModule->StartRecording(); + mTuningDeviceModule->StartPlayout(); + } + else + { + mTuningDeviceModule->StopRecording(); + mTuningDeviceModule->StopPlayout(); + ll_set_device_module_render_device(mPeerDeviceModule, mPlayoutDevice); + ll_set_device_module_capture_device(mPeerDeviceModule, mRecordingDevice); + mPeerDeviceModule->StartRecording(); + mPeerDeviceModule->StartPlayout(); + } + } + ); mSignalingThread->PostTask( [this, enable] { -- cgit v1.2.3 From e4dee511cab0a98e802e2c0cc12b3d8b9b90a8df Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Mon, 18 Mar 2024 17:47:56 -0700 Subject: Use LL::WorkQueue to handle transitions from llwebrtc threads to the main thread --- indra/llwebrtc/llwebrtc.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'indra/llwebrtc/llwebrtc.cpp') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index b7501bd0e0..6400f807fd 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -580,7 +580,7 @@ void LLWebRTCPeerConnectionImpl::init(LLWebRTCImpl * webrtc_impl) } void LLWebRTCPeerConnectionImpl::terminate() { - mWebRTCImpl->SignalingBlockingCall( + mWebRTCImpl->PostSignalingTask( [this]() { if (mPeerConnection) -- cgit v1.2.3 From 5bc92b8031e9a6258bab8b24d42090a495cec6e5 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Tue, 19 Mar 2024 15:23:43 -0700 Subject: Simplify workqueue calls. Fix issue with webrtc blocking on destruction. --- indra/llwebrtc/llwebrtc.cpp | 62 +++++++++++++++++++++++++-------------------- 1 file changed, 35 insertions(+), 27 deletions(-) (limited to 'indra/llwebrtc/llwebrtc.cpp') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index 6400f807fd..283b94dd3c 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -562,8 +562,10 @@ void LLWebRTCImpl::freePeerConnection(LLWebRTCPeerConnectionInterface* peer_conn // Most peer connection (signaling) happens on // the signaling thread. -LLWebRTCPeerConnectionImpl::LLWebRTCPeerConnectionImpl() : +LLWebRTCPeerConnectionImpl::LLWebRTCPeerConnectionImpl() : mWebRTCImpl(nullptr), + mClosing(false), + mPeerConnection(nullptr), mMute(false), mAnswerReceived(false) { @@ -580,13 +582,24 @@ void LLWebRTCPeerConnectionImpl::init(LLWebRTCImpl * webrtc_impl) } void LLWebRTCPeerConnectionImpl::terminate() { + rtc::scoped_refptr connection; + mPeerConnection.swap(connection); + rtc::scoped_refptr dataChannel; + mDataChannel.swap(dataChannel); + rtc::scoped_refptr localStream; + mLocalStream.swap(localStream); + mWebRTCImpl->PostSignalingTask( - [this]() + [=]() { - if (mPeerConnection) + if (connection) + { + connection->Close(); + } + if (dataChannel) { - mPeerConnection->Close(); - mPeerConnection = nullptr; + dataChannel->UnregisterObserver(); + dataChannel->Close(); } }); } @@ -710,24 +723,9 @@ bool LLWebRTCPeerConnectionImpl::initializeConnection() bool LLWebRTCPeerConnectionImpl::shutdownConnection() { - if (mPeerConnection) - { - mWebRTCImpl->PostSignalingTask( - [this]() - { - if (mPeerConnection) - { - mPeerConnection->Close(); - mPeerConnection = nullptr; - } - for (auto &observer : mSignalingObserverList) - { - observer->OnPeerConnectionShutdown(); - } - }); - return true; - } - return false; + mClosing = true; + terminate(); + return true; } void LLWebRTCPeerConnectionImpl::enableSenderTracks(bool enable) @@ -1057,14 +1055,16 @@ void LLWebRTCPeerConnectionImpl::OnSuccess(webrtc::SessionDescriptionInterface * } RTC_LOG(LS_INFO) << __FUNCTION__ << " Local SDP: " << sdp_mangled_stream.str(); - + std::string mangled_sdp = sdp_mangled_stream.str(); for (auto &observer : mSignalingObserverList) { - observer->OnOfferAvailable(sdp_mangled_stream.str()); + observer->OnOfferAvailable(mangled_sdp); } + + mPeerConnection->SetLocalDescription(std::unique_ptr( + webrtc::CreateSessionDescription(webrtc::SdpType::kOffer, mangled_sdp)), + rtc::scoped_refptr(this)); - mPeerConnection->SetLocalDescription(std::unique_ptr(webrtc::CreateSessionDescription(webrtc::SdpType::kOffer, sdp_mangled_stream.str())), - rtc::scoped_refptr(this)); } void LLWebRTCPeerConnectionImpl::OnFailure(webrtc::RTCError error) @@ -1135,6 +1135,14 @@ void LLWebRTCPeerConnectionImpl::OnStateChange() break; case webrtc::DataChannelInterface::kClosed: RTC_LOG(LS_INFO) << __FUNCTION__ << " Data Channel State closed"; + // if the data channel is up, we need to shut it down, holding off + // on termination of the peer connection until it's been closed. + if (mClosing) + { + // a close was requested, and the data channel has closed, + // so go ahead and call shutdownConnection again to clean up. + shutdownConnection(); + } break; default: break; -- cgit v1.2.3 From 0e2fc4d35ea1fdef0028a5132602985da2e1c463 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Tue, 19 Mar 2024 17:05:56 -0700 Subject: Clear out observers when terminating a connection --- indra/llwebrtc/llwebrtc.cpp | 8 -------- 1 file changed, 8 deletions(-) (limited to 'indra/llwebrtc/llwebrtc.cpp') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index 283b94dd3c..34d950b804 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -1135,14 +1135,6 @@ void LLWebRTCPeerConnectionImpl::OnStateChange() break; case webrtc::DataChannelInterface::kClosed: RTC_LOG(LS_INFO) << __FUNCTION__ << " Data Channel State closed"; - // if the data channel is up, we need to shut it down, holding off - // on termination of the peer connection until it's been closed. - if (mClosing) - { - // a close was requested, and the data channel has closed, - // so go ahead and call shutdownConnection again to clean up. - shutdownConnection(); - } break; default: break; -- cgit v1.2.3 From 4b709e7944ad34bf82c63b8ee57661598bc2292f Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Wed, 20 Mar 2024 11:15:12 -0700 Subject: Turn procesIceUpdates into a coroutine to chop up the work a bit. --- indra/llwebrtc/llwebrtc.cpp | 3 +++ 1 file changed, 3 insertions(+) (limited to 'indra/llwebrtc/llwebrtc.cpp') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index 34d950b804..1f129cf514 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -589,6 +589,9 @@ void LLWebRTCPeerConnectionImpl::terminate() rtc::scoped_refptr localStream; mLocalStream.swap(localStream); + mSignalingObserverList.clear(); + mDataObserverList.clear(); + mWebRTCImpl->PostSignalingTask( [=]() { -- cgit v1.2.3 From e242c129f9d56161496a50397b79da1e444e8de4 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Fri, 22 Mar 2024 16:21:02 -0700 Subject: Add Tracy categories for WebRTC Voice Also: * Fix a few crashes. * Only send position data when it changes. --- indra/llwebrtc/llwebrtc.cpp | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) (limited to 'indra/llwebrtc/llwebrtc.cpp') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index 1f129cf514..a92b480e3a 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -1103,7 +1103,10 @@ void LLWebRTCPeerConnectionImpl::OnSetRemoteDescriptionComplete(webrtc::RTCError } } mCachedIceCandidates.clear(); - OnIceGatheringChange(mPeerConnection->ice_gathering_state()); + if (mPeerConnection) + { + OnIceGatheringChange(mPeerConnection->ice_gathering_state()); + } } @@ -1120,6 +1123,10 @@ void LLWebRTCPeerConnectionImpl::OnSetLocalDescriptionComplete(webrtc::RTCError void LLWebRTCPeerConnectionImpl::OnStateChange() { + if (!mDataChannel) + { + return; + } RTC_LOG(LS_INFO) << __FUNCTION__ << " Data Channel State: " << webrtc::DataChannelInterface::DataStateString(mDataChannel->state()); switch (mDataChannel->state()) { @@ -1163,7 +1170,12 @@ void LLWebRTCPeerConnectionImpl::sendData(const std::string& data, bool binary) { rtc::CopyOnWriteBuffer cowBuffer(data.data(), data.length()); webrtc::DataBuffer buffer(cowBuffer, binary); - mDataChannel->Send(buffer); + mWebRTCImpl->PostNetworkTask([this, buffer]() { + if (mDataChannel) + { + mDataChannel->Send(buffer); + } + }); } } -- cgit v1.2.3 From cdae5ebc168d95a304b9905de7b66381723e402f Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Sat, 30 Mar 2024 21:58:00 -0700 Subject: Add UI for managing echo cancellation, AGC, and noise control. Plumb audio settings through from webrtc to the sound preferences UI (still needs some tweaking, of course.) Also, choose stun servers based on grid. Ultimately, the stun stun servers will be passed up via login or something. --- indra/llwebrtc/llwebrtc.cpp | 98 ++++++++++++++++++++++++++++++++------------- 1 file changed, 71 insertions(+), 27 deletions(-) (limited to 'indra/llwebrtc/llwebrtc.cpp') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index 34d950b804..c51bcfcdd5 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -157,7 +157,6 @@ LLWebRTCImpl::LLWebRTCImpl() : void LLWebRTCImpl::init() { - RTC_DCHECK(mPeerConnectionFactory); mPlayoutDevice = 0; mRecordingDevice = 0; rtc::InitializeSSL(); @@ -222,12 +221,10 @@ void LLWebRTCImpl::init() mPeerCustomProcessor = new LLCustomProcessor; webrtc::AudioProcessingBuilder apb; apb.SetCapturePostProcessing(std::unique_ptr(mPeerCustomProcessor)); - rtc::scoped_refptr apm = apb.Create(); + mAudioProcessingModule = apb.Create(); - // TODO: wire some of these to the primary interface and ultimately - // to the UI to allow user config. webrtc::AudioProcessing::Config apm_config; - apm_config.echo_canceller.enabled = true; + apm_config.echo_canceller.enabled = false; apm_config.echo_canceller.mobile_mode = false; apm_config.gain_controller1.enabled = true; apm_config.gain_controller1.mode = webrtc::AudioProcessing::Config::GainController1::kAdaptiveAnalog; @@ -250,8 +247,8 @@ void LLWebRTCImpl::init() processing_config.reverse_output_stream().set_num_channels(2); processing_config.reverse_output_stream().set_sample_rate_hz(48000); - apm->Initialize(processing_config); - apm->ApplyConfig(apm_config); + mAudioProcessingModule->Initialize(processing_config); + mAudioProcessingModule->ApplyConfig(apm_config); mPeerConnectionFactory = webrtc::CreatePeerConnectionFactory(mNetworkThread.get(), mWorkerThread.get(), @@ -262,7 +259,7 @@ void LLWebRTCImpl::init() nullptr /* video_encoder_factory */, nullptr /* video_decoder_factory */, nullptr /* audio_mixer */, - apm); + mAudioProcessingModule); mWorkerThread->BlockingCall([this]() { mPeerDeviceModule->StartPlayout(); }); } @@ -318,6 +315,49 @@ void LLWebRTCImpl::setRecording(bool recording) }); } +void LLWebRTCImpl::setAudioConfig(LLWebRTCDeviceInterface::AudioConfig config) +{ + webrtc::AudioProcessing::Config apm_config; + apm_config.echo_canceller.enabled = config.mEchoCancellation; + apm_config.echo_canceller.mobile_mode = false; + apm_config.gain_controller1.enabled = true; + apm_config.gain_controller1.mode = webrtc::AudioProcessing::Config::GainController1::kAdaptiveAnalog; + apm_config.gain_controller2.enabled = true; + apm_config.high_pass_filter.enabled = true; + apm_config.transient_suppression.enabled = true; + apm_config.pipeline.multi_channel_render = true; + apm_config.pipeline.multi_channel_capture = true; + apm_config.pipeline.multi_channel_capture = true; + + switch (config.mNoiseSuppressionLevel) + { + case LLWebRTCDeviceInterface::AudioConfig::NOISE_SUPPRESSION_LEVEL_NONE: + apm_config.noise_suppression.enabled = false; + apm_config.noise_suppression.level = webrtc::AudioProcessing::Config::NoiseSuppression::kLow; + break; + case LLWebRTCDeviceInterface::AudioConfig::NOISE_SUPPRESSION_LEVEL_LOW: + apm_config.noise_suppression.enabled = true; + apm_config.noise_suppression.level = webrtc::AudioProcessing::Config::NoiseSuppression::kLow; + break; + case LLWebRTCDeviceInterface::AudioConfig::NOISE_SUPPRESSION_LEVEL_MODERATE: + apm_config.noise_suppression.enabled = true; + apm_config.noise_suppression.level = webrtc::AudioProcessing::Config::NoiseSuppression::kModerate; + break; + case LLWebRTCDeviceInterface::AudioConfig::NOISE_SUPPRESSION_LEVEL_HIGH: + apm_config.noise_suppression.enabled = true; + apm_config.noise_suppression.level = webrtc::AudioProcessing::Config::NoiseSuppression::kHigh; + break; + case LLWebRTCDeviceInterface::AudioConfig::NOISE_SUPPRESSION_LEVEL_VERY_HIGH: + apm_config.noise_suppression.enabled = true; + apm_config.noise_suppression.level = webrtc::AudioProcessing::Config::NoiseSuppression::kVeryHigh; + break; + default: + apm_config.noise_suppression.enabled = false; + apm_config.noise_suppression.level = webrtc::AudioProcessing::Config::NoiseSuppression::kLow; + } + mAudioProcessingModule->ApplyConfig(apm_config); +} + void LLWebRTCImpl::refreshDevices() { mWorkerThread->PostTask([this]() { updateDevices(); }); @@ -616,32 +656,36 @@ void LLWebRTCPeerConnectionImpl::unsetSignalingObserver(LLWebRTCSignalingObserve } } -// TODO: Add initialization structure through which -// stun and turn servers may be passed in from -// the sim or login. -bool LLWebRTCPeerConnectionImpl::initializeConnection() +bool LLWebRTCPeerConnectionImpl::initializeConnection(LLWebRTCPeerConnectionInterface::InitOptions options) { RTC_DCHECK(!mPeerConnection); mAnswerReceived = false; mWebRTCImpl->PostSignalingTask( - [this]() + [this, options]() { + std::vector servers = options.mServers; + if(servers.empty()) + { + LLWebRTCPeerConnectionInterface::InitOptions::IceServers ice_servers; + ice_servers.mUrls.push_back("stun:stun.l.google.com:19302"); + ice_servers.mUrls.push_back("stun1:stun.l.google.com:19302"); + ice_servers.mUrls.push_back("stun2:stun.l.google.com:19302"); + ice_servers.mUrls.push_back("stun3:stun.l.google.com:19302"); + ice_servers.mUrls.push_back("stun4:stun.l.google.com:19302"); + } + webrtc::PeerConnectionInterface::RTCConfiguration config; + for (auto server : servers) + { + webrtc::PeerConnectionInterface::IceServer ice_server; + ice_server.urls = server.mUrls; + ice_server.username = server.mUserName; + ice_server.password = server.mPassword; + config.servers.push_back(ice_server); + } + config.sdp_semantics = webrtc::SdpSemantics::kUnifiedPlan; - webrtc::PeerConnectionInterface::IceServer server; - server.uri = "stun:roxie-turn.staging.secondlife.io:3478"; - config.servers.push_back(server); - server.uri = "stun:stun.l.google.com:19302"; - config.servers.push_back(server); - server.uri = "stun:stun1.l.google.com:19302"; - config.servers.push_back(server); - server.uri = "stun:stun2.l.google.com:19302"; - config.servers.push_back(server); - server.uri = "stun:stun3.l.google.com:19302"; - config.servers.push_back(server); - server.uri = "stun:stun4.l.google.com:19302"; - config.servers.push_back(server); config.set_min_port(60000); config.set_max_port(60100); @@ -671,7 +715,7 @@ bool LLWebRTCPeerConnectionImpl::initializeConnection() cricket::AudioOptions audioOptions; audioOptions.auto_gain_control = true; - audioOptions.echo_cancellation = true; // incompatible with opus stereo + audioOptions.echo_cancellation = false; // incompatible with opus stereo audioOptions.noise_suppression = true; mLocalStream = mPeerConnectionFactory->CreateLocalMediaStream("SLStream"); -- cgit v1.2.3 From b3bb3d2d5145aa981cefd77ba564aa082475a8af Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Sat, 30 Mar 2024 22:03:30 -0700 Subject: Renegotiate on remote description error --- indra/llwebrtc/llwebrtc.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'indra/llwebrtc/llwebrtc.cpp') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index a92b480e3a..bba1e99e2d 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -1073,6 +1073,10 @@ void LLWebRTCPeerConnectionImpl::OnSuccess(webrtc::SessionDescriptionInterface * void LLWebRTCPeerConnectionImpl::OnFailure(webrtc::RTCError error) { RTC_LOG(LS_ERROR) << ToString(error.type()) << ": " << error.message(); + for (auto &observer : mSignalingObserverList) + { + observer->OnRenegotiationNeeded(); + } } // @@ -1086,6 +1090,10 @@ void LLWebRTCPeerConnectionImpl::OnSetRemoteDescriptionComplete(webrtc::RTCError if (!error.ok()) { RTC_LOG(LS_ERROR) << ToString(error.type()) << ": " << error.message(); + for (auto &observer : mSignalingObserverList) + { + observer->OnRenegotiationNeeded(); + } return; } mAnswerReceived = true; -- cgit v1.2.3 From c6e673cda139f5faaa52ccd03a372e7ffa9f5716 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Sat, 30 Mar 2024 23:19:38 -0700 Subject: Fix windows crashes * sampling rate was set to 8khz for audio processing, which was causing a 'bands' mismatch with the echo cancler. * Some funnybusiness with lambdas and captures and such was causing a heap crash with respect to function parameters. --- indra/llwebrtc/llwebrtc.cpp | 32 ++++++++++++-------------------- 1 file changed, 12 insertions(+), 20 deletions(-) (limited to 'indra/llwebrtc/llwebrtc.cpp') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index a1e125a8f2..074b037529 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -239,16 +239,16 @@ void LLWebRTCImpl::init() webrtc::ProcessingConfig processing_config; processing_config.input_stream().set_num_channels(2); - processing_config.input_stream().set_sample_rate_hz(8000); + processing_config.input_stream().set_sample_rate_hz(48000); processing_config.output_stream().set_num_channels(2); - processing_config.output_stream().set_sample_rate_hz(8000); + processing_config.output_stream().set_sample_rate_hz(48000); processing_config.reverse_input_stream().set_num_channels(2); processing_config.reverse_input_stream().set_sample_rate_hz(48000); processing_config.reverse_output_stream().set_num_channels(2); processing_config.reverse_output_stream().set_sample_rate_hz(48000); - mAudioProcessingModule->Initialize(processing_config); mAudioProcessingModule->ApplyConfig(apm_config); + mAudioProcessingModule->Initialize(processing_config); mPeerConnectionFactory = webrtc::CreatePeerConnectionFactory(mNetworkThread.get(), mWorkerThread.get(), @@ -659,35 +659,27 @@ void LLWebRTCPeerConnectionImpl::unsetSignalingObserver(LLWebRTCSignalingObserve } } -bool LLWebRTCPeerConnectionImpl::initializeConnection(LLWebRTCPeerConnectionInterface::InitOptions options) + +bool LLWebRTCPeerConnectionImpl::initializeConnection(const LLWebRTCPeerConnectionInterface::InitOptions& options) { RTC_DCHECK(!mPeerConnection); mAnswerReceived = false; mWebRTCImpl->PostSignalingTask( - [this, options]() + [this,options]() { - std::vector servers = options.mServers; - if(servers.empty()) - { - LLWebRTCPeerConnectionInterface::InitOptions::IceServers ice_servers; - ice_servers.mUrls.push_back("stun:stun.l.google.com:19302"); - ice_servers.mUrls.push_back("stun1:stun.l.google.com:19302"); - ice_servers.mUrls.push_back("stun2:stun.l.google.com:19302"); - ice_servers.mUrls.push_back("stun3:stun.l.google.com:19302"); - ice_servers.mUrls.push_back("stun4:stun.l.google.com:19302"); - } - webrtc::PeerConnectionInterface::RTCConfiguration config; - for (auto server : servers) + for (auto server : options.mServers) { webrtc::PeerConnectionInterface::IceServer ice_server; - ice_server.urls = server.mUrls; + for (auto url : server.mUrls) + { + ice_server.urls.push_back(url); + } ice_server.username = server.mUserName; ice_server.password = server.mPassword; config.servers.push_back(ice_server); } - config.sdp_semantics = webrtc::SdpSemantics::kUnifiedPlan; config.set_min_port(60000); @@ -718,7 +710,7 @@ bool LLWebRTCPeerConnectionImpl::initializeConnection(LLWebRTCPeerConnectionInte cricket::AudioOptions audioOptions; audioOptions.auto_gain_control = true; - audioOptions.echo_cancellation = false; // incompatible with opus stereo + audioOptions.echo_cancellation = true; audioOptions.noise_suppression = true; mLocalStream = mPeerConnectionFactory->CreateLocalMediaStream("SLStream"); -- cgit v1.2.3 From c826aea079c59950a4064a94825534884fed8bf8 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Mon, 1 Apr 2024 21:39:17 -0700 Subject: Fix "default" audio device handling. Windows and Mac/Linux behave slightly differently with respect to Default devices, in that mac/linux (I think) simply assumes the device at index 0 is the default one, and windows has a separate API for enabling the default device. --- indra/llwebrtc/llwebrtc.cpp | 230 ++++++++++++++++++++++++++------------------ 1 file changed, 136 insertions(+), 94 deletions(-) (limited to 'indra/llwebrtc/llwebrtc.cpp') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index 074b037529..75b2332bed 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -40,6 +40,11 @@ namespace llwebrtc { +static int16_t PLAYOUT_DEVICE_DEFAULT = -1; +static int16_t PLAYOUT_DEVICE_BAD = -2; +static int16_t RECORD_DEVICE_DEFAULT = -1; +static int16_t RECORD_DEVICE_BAD = -2; + LLAudioDeviceObserver::LLAudioDeviceObserver() : mSumVector {0}, mMicrophoneEnergy(0.0) {} float LLAudioDeviceObserver::getMicrophoneEnergy() { return mMicrophoneEnergy; } @@ -189,10 +194,16 @@ void LLWebRTCImpl::init() std::unique_ptr(mTuningAudioDeviceObserver)); mTuningDeviceModule->Init(); - mTuningDeviceModule->SetStereoRecording(true); - mTuningDeviceModule->SetStereoPlayout(true); + mTuningDeviceModule->SetPlayoutDevice(mPlayoutDevice); + mTuningDeviceModule->SetRecordingDevice(mRecordingDevice); mTuningDeviceModule->EnableBuiltInAEC(false); mTuningDeviceModule->SetAudioDeviceSink(this); + mTuningDeviceModule->InitMicrophone(); + mTuningDeviceModule->InitSpeaker(); + mTuningDeviceModule->InitRecording(); + mTuningDeviceModule->InitPlayout(); + mTuningDeviceModule->SetStereoRecording(true); + mTuningDeviceModule->SetStereoPlayout(true); updateDevices(); }); @@ -207,13 +218,13 @@ void LLWebRTCImpl::init() mPeerDeviceModule->Init(); mPeerDeviceModule->SetPlayoutDevice(mPlayoutDevice); mPeerDeviceModule->SetRecordingDevice(mRecordingDevice); - mPeerDeviceModule->SetStereoRecording(true); - mPeerDeviceModule->SetStereoPlayout(true); mPeerDeviceModule->EnableBuiltInAEC(false); mPeerDeviceModule->InitMicrophone(); mPeerDeviceModule->InitSpeaker(); mPeerDeviceModule->InitRecording(); mPeerDeviceModule->InitPlayout(); + mPeerDeviceModule->SetStereoRecording(true); + mPeerDeviceModule->SetStereoPlayout(true); }); // The custom processor allows us to retrieve audio data (and levels) @@ -375,145 +386,169 @@ void LLWebRTCImpl::unsetDevicesObserver(LLWebRTCDevicesObserver *observer) } } -static int16_t ll_get_device_module_capture_device(rtc::scoped_refptr device_module, const std::string &id) -{ - int16_t recordingDevice = 0; - int16_t captureDeviceCount = device_module->RecordingDevices(); - for (int16_t i = 0; i < captureDeviceCount; i++) - { - char name[webrtc::kAdmMaxDeviceNameSize]; - char guid[webrtc::kAdmMaxGuidSize]; - device_module->RecordingDeviceName(i, name, guid); - if (id == guid || id == "Default") // first one in list is default - { - RTC_LOG(LS_INFO) << __FUNCTION__ << "Set recording device to " << name << " " << guid << " " << i; - recordingDevice = i; - break; - } - } - return recordingDevice; -} - void ll_set_device_module_capture_device(rtc::scoped_refptr device_module, int16_t device) { device_module->StopRecording(); - device_module->SetRecordingDevice(device); +#if LL_WINDOWS + if (device < 0) + { + device_module->SetRecordingDevice(webrtc::AudioDeviceModule::kDefaultDevice); + } + else + { + device_module->SetRecordingDevice(device); + } +#else + // passed in default is -1, but the device list + // has it at 0 + device_module->SetPlayoutDevice(device + 1); +#endif device_module->InitMicrophone(); - device_module->SetStereoRecording(false); device_module->InitRecording(); + device_module->SetStereoRecording(false); device_module->StartRecording(); } void LLWebRTCImpl::setCaptureDevice(const std::string &id) { - - mWorkerThread->PostTask( - [this, id]() + int16_t recordingDevice = RECORD_DEVICE_DEFAULT; + if (id != "Default") + { + for (int16_t i = 0; i < mRecordingDeviceList.size(); i++) { - int16_t recordingDevice = ll_get_device_module_capture_device(mTuningDeviceModule, id); - if (recordingDevice != mRecordingDevice) + if (mRecordingDeviceList[i].mID == id) { - mRecordingDevice = recordingDevice; - if (mTuningMode) - { - ll_set_device_module_capture_device(mTuningDeviceModule, recordingDevice); - } - else - { - ll_set_device_module_capture_device(mPeerDeviceModule, recordingDevice); - } + recordingDevice = i; + break; } - }); -} - -static int16_t ll_get_device_module_render_device( - rtc::scoped_refptr device_module, - const std::string &id) -{ - int16_t playoutDevice = 0; - int16_t playoutDeviceCount = device_module->PlayoutDevices(); - for (int16_t i = 0; i < playoutDeviceCount; i++) - { - char name[webrtc::kAdmMaxDeviceNameSize]; - char guid[webrtc::kAdmMaxGuidSize]; - device_module->PlayoutDeviceName(i, name, guid); - if (id == guid || id == "Default") // first one in list is default - { - RTC_LOG(LS_INFO) << __FUNCTION__ << "Set recording device to " << name << " " << guid << " " << i; - playoutDevice = i; - break; } } - return playoutDevice; + if (recordingDevice == mRecordingDevice) + { + return; + } + mRecordingDevice = recordingDevice; + if (mTuningMode) + { + mWorkerThread->PostTask([this, recordingDevice]() { ll_set_device_module_capture_device(mTuningDeviceModule, recordingDevice); }); + } + else + { + mWorkerThread->PostTask([this, recordingDevice]() { ll_set_device_module_capture_device(mPeerDeviceModule, recordingDevice); }); + } } void ll_set_device_module_render_device(rtc::scoped_refptr device_module, int16_t device) { device_module->StopPlayout(); - device_module->SetPlayoutDevice(device); +#if LL_WINDOWS + if (device < 0) + { + device_module->SetPlayoutDevice(webrtc::AudioDeviceModule::kDefaultDevice); + } + else + { + device_module->SetPlayoutDevice(device); + } +#else + device_module->SetPlayoutDevice(device + 1); +#endif device_module->InitSpeaker(); - device_module->SetStereoPlayout(false); device_module->InitPlayout(); - device_module->StartPlayout(); + device_module->SetStereoPlayout(true); } void LLWebRTCImpl::setRenderDevice(const std::string &id) { - mWorkerThread->PostTask( - [this, id]() - { - int16_t playoutDevice = ll_get_device_module_render_device(mTuningDeviceModule, id); - if (playoutDevice != mPlayoutDevice) + int16_t playoutDevice = PLAYOUT_DEVICE_DEFAULT; + if (id != "Default") + { + for (int16_t i = 0; i < mPlayoutDeviceList.size(); i++) + { + if (mPlayoutDeviceList[i].mID == id) { - mPlayoutDevice = playoutDevice; - if (mTuningMode) - { - ll_set_device_module_render_device(mTuningDeviceModule, playoutDevice); - } - else - { - ll_set_device_module_render_device(mPeerDeviceModule, playoutDevice); - } + playoutDevice = i; + break; } - }); + } + } + if (playoutDevice == mPlayoutDevice) + { + return; + } + mPlayoutDevice = playoutDevice; + + if (mTuningMode) + { + mWorkerThread->PostTask( + [this, playoutDevice]() + { + ll_set_device_module_render_device(mTuningDeviceModule, playoutDevice); + }); + } + else + { + mWorkerThread->PostTask( + [this, playoutDevice]() + { + ll_set_device_module_render_device(mPeerDeviceModule, playoutDevice); + mPeerDeviceModule->StartPlayout(); + }); + } } // updateDevices needs to happen on the worker thread. void LLWebRTCImpl::updateDevices() { - int16_t renderDeviceCount = mTuningDeviceModule->PlayoutDevices(); - int16_t currentRenderDeviceIndex = mTuningDeviceModule->GetPlayoutDevice(); - - LLWebRTCVoiceDeviceList renderDeviceList; - for (int16_t index = 0; index < renderDeviceCount; index++) + int16_t renderDeviceCount = mTuningDeviceModule->PlayoutDevices(); + + mPlayoutDeviceList.clear(); +#if LL_WINDOWS + int16_t index = 0; +#else + // index zero is always "Default" for darwin/linux, + // which is a special case, so skip it. + int16_t index = 1; +#endif + for (; index < renderDeviceCount; index++) { char name[webrtc::kAdmMaxDeviceNameSize]; char guid[webrtc::kAdmMaxGuidSize]; mTuningDeviceModule->PlayoutDeviceName(index, name, guid); - renderDeviceList.emplace_back(name, guid); + mPlayoutDeviceList.emplace_back(name, guid); } - int16_t captureDeviceCount = mTuningDeviceModule->RecordingDevices(); - int16_t currentCaptureDeviceIndex = mTuningDeviceModule->GetRecordingDevice(); - - LLWebRTCVoiceDeviceList captureDeviceList; - for (int16_t index = 0; index < captureDeviceCount; index++) + int16_t captureDeviceCount = mTuningDeviceModule->RecordingDevices(); + + mRecordingDeviceList.clear(); +#if LL_WINDOWS + index = 0; +#else + // index zero is always "Default" for darwin/linux, + // which is a special case, so skip it. + index = 1; +#endif + for (; index < captureDeviceCount; index++) { char name[webrtc::kAdmMaxDeviceNameSize]; char guid[webrtc::kAdmMaxGuidSize]; mTuningDeviceModule->RecordingDeviceName(index, name, guid); - captureDeviceList.emplace_back(name, guid); + mRecordingDeviceList.emplace_back(name, guid); } + for (auto &observer : mVoiceDevicesObserverList) { - observer->OnDevicesChanged(renderDeviceList, - captureDeviceList); + observer->OnDevicesChanged(mPlayoutDeviceList, mRecordingDeviceList); } } void LLWebRTCImpl::OnDevicesUpdated() { + // reset these to a bad value so an update is forced + mRecordingDevice = RECORD_DEVICE_BAD; + mPlayoutDevice = PLAYOUT_DEVICE_BAD; + updateDevices(); } @@ -529,17 +564,24 @@ void LLWebRTCImpl::setTuningMode(bool enable) mPeerDeviceModule->StopPlayout(); ll_set_device_module_render_device(mTuningDeviceModule, mPlayoutDevice); ll_set_device_module_capture_device(mTuningDeviceModule, mRecordingDevice); + mTuningDeviceModule->InitPlayout(); + mTuningDeviceModule->InitRecording(); mTuningDeviceModule->StartRecording(); - mTuningDeviceModule->StartPlayout(); + // TODO: Starting Playout on the TDM appears to create an audio artifact (click) + // in this case, so disabling it for now. We may have to do something different + // if we enable 'echo playback' via the TDM when tuning. + //mTuningDeviceModule->StartPlayout(); } else { mTuningDeviceModule->StopRecording(); - mTuningDeviceModule->StopPlayout(); + //mTuningDeviceModule->StopPlayout(); ll_set_device_module_render_device(mPeerDeviceModule, mPlayoutDevice); ll_set_device_module_capture_device(mPeerDeviceModule, mRecordingDevice); - mPeerDeviceModule->StartRecording(); + mPeerDeviceModule->InitPlayout(); + mPeerDeviceModule->InitRecording(); mPeerDeviceModule->StartPlayout(); + mPeerDeviceModule->StartRecording(); } } ); -- cgit v1.2.3 From fdf0fbce5b4c8f15493ff064ef4d092f52c6ef73 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Mon, 1 Apr 2024 22:39:59 -0700 Subject: oopse --- indra/llwebrtc/llwebrtc.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'indra/llwebrtc/llwebrtc.cpp') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index 75b2332bed..7570bb4434 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -401,7 +401,7 @@ void ll_set_device_module_capture_device(rtc::scoped_refptrSetPlayoutDevice(device + 1); + device_module->SetRecordingDevice(device + 1); #endif device_module->InitMicrophone(); device_module->InitRecording(); -- cgit v1.2.3 From 34ed990fa363f3df0a9817d36ef0becdcf641e80 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Sun, 7 Apr 2024 14:13:00 -0700 Subject: Show 'decline' when peer declines p2p voice The simulator will send a chatterbox notification that voice is no longer in use for a given channel, and the viewer should take that as a case where the peer does not want voice, hence it's a decline. --- indra/llwebrtc/llwebrtc.cpp | 30 ++++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) (limited to 'indra/llwebrtc/llwebrtc.cpp') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index 7570bb4434..57130bb734 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -26,7 +26,6 @@ #include "llwebrtc_impl.h" #include -#include #include #include "api/audio_codecs/audio_decoder_factory.h" @@ -389,7 +388,7 @@ void LLWebRTCImpl::unsetDevicesObserver(LLWebRTCDevicesObserver *observer) void ll_set_device_module_capture_device(rtc::scoped_refptr device_module, int16_t device) { device_module->StopRecording(); -#if LL_WINDOWS +#if WEBRTC_WIN if (device < 0) { device_module->SetRecordingDevice(webrtc::AudioDeviceModule::kDefaultDevice); @@ -442,7 +441,7 @@ void LLWebRTCImpl::setCaptureDevice(const std::string &id) void ll_set_device_module_render_device(rtc::scoped_refptr device_module, int16_t device) { device_module->StopPlayout(); -#if LL_WINDOWS +#if WEBRTC_WIN if (device < 0) { device_module->SetPlayoutDevice(webrtc::AudioDeviceModule::kDefaultDevice); @@ -501,10 +500,10 @@ void LLWebRTCImpl::setRenderDevice(const std::string &id) // updateDevices needs to happen on the worker thread. void LLWebRTCImpl::updateDevices() { - int16_t renderDeviceCount = mTuningDeviceModule->PlayoutDevices(); + int16_t renderDeviceCount = mTuningDeviceModule->PlayoutDevices(); mPlayoutDeviceList.clear(); -#if LL_WINDOWS +#if WEBRTC_WIN int16_t index = 0; #else // index zero is always "Default" for darwin/linux, @@ -516,13 +515,23 @@ void LLWebRTCImpl::updateDevices() char name[webrtc::kAdmMaxDeviceNameSize]; char guid[webrtc::kAdmMaxGuidSize]; mTuningDeviceModule->PlayoutDeviceName(index, name, guid); + +#if WEBRTC_LINUX + // Linux audio implementation (pulse and alsa) + // return empty strings for the guid, so + // use the name for the guid + if (!strcmp(guid, "")) + { + strcpy(guid, name); + } +#endif // WEBRTC_LINUX mPlayoutDeviceList.emplace_back(name, guid); } int16_t captureDeviceCount = mTuningDeviceModule->RecordingDevices(); mRecordingDeviceList.clear(); -#if LL_WINDOWS +#if WEBRTC_WIN index = 0; #else // index zero is always "Default" for darwin/linux, @@ -534,6 +543,15 @@ void LLWebRTCImpl::updateDevices() char name[webrtc::kAdmMaxDeviceNameSize]; char guid[webrtc::kAdmMaxGuidSize]; mTuningDeviceModule->RecordingDeviceName(index, name, guid); +#if WEBRTC_LINUX + // Linux audio implementation (pulse and alsa) + // return empty strings for the guid, so + // use the name for the guid + if (!strcmp(guid, "")) + { + strcpy(guid, name); + } +#endif // WEBRTC_LINUX mRecordingDeviceList.emplace_back(name, guid); } -- cgit v1.2.3 From 648741470f67120d99aec3a4aeceeaf4395cca09 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Sun, 7 Apr 2024 19:23:00 -0700 Subject: CR suggestions --- indra/llwebrtc/llwebrtc.cpp | 19 ------------------- 1 file changed, 19 deletions(-) (limited to 'indra/llwebrtc/llwebrtc.cpp') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index 57130bb734..eb9bb65e67 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -515,16 +515,6 @@ void LLWebRTCImpl::updateDevices() char name[webrtc::kAdmMaxDeviceNameSize]; char guid[webrtc::kAdmMaxGuidSize]; mTuningDeviceModule->PlayoutDeviceName(index, name, guid); - -#if WEBRTC_LINUX - // Linux audio implementation (pulse and alsa) - // return empty strings for the guid, so - // use the name for the guid - if (!strcmp(guid, "")) - { - strcpy(guid, name); - } -#endif // WEBRTC_LINUX mPlayoutDeviceList.emplace_back(name, guid); } @@ -543,15 +533,6 @@ void LLWebRTCImpl::updateDevices() char name[webrtc::kAdmMaxDeviceNameSize]; char guid[webrtc::kAdmMaxGuidSize]; mTuningDeviceModule->RecordingDeviceName(index, name, guid); -#if WEBRTC_LINUX - // Linux audio implementation (pulse and alsa) - // return empty strings for the guid, so - // use the name for the guid - if (!strcmp(guid, "")) - { - strcpy(guid, name); - } -#endif // WEBRTC_LINUX mRecordingDeviceList.emplace_back(name, guid); } -- cgit v1.2.3 From 98322d5f070b260f1e46eb5d9fcd54fa43151329 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Sun, 21 Apr 2024 21:12:06 -0700 Subject: Reconnect when parcel voice params change. When parcel voice permissions and region/parcel-only voice settings change, a callback will be made to the viewer with new voice credential information. For webrtc, this means either just the uuid of the voice channel, or nothing if voice is disabled. This change looks at that callback and the channel id, and sets the appropriate flags on the parcel/region as needed which will cause voice to be renegotiated. Also, there was a race condition if the voice connect attempt was made before caps were retrieved, which would have resulted in full renegotiate attempts. Now, just wait until the cap comes in and continue. --- indra/llwebrtc/llwebrtc.cpp | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'indra/llwebrtc/llwebrtc.cpp') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index eb9bb65e67..e08ace12c5 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -735,6 +735,10 @@ bool LLWebRTCPeerConnectionImpl::initializeConnection(const LLWebRTCPeerConnecti else { RTC_LOG(LS_ERROR) << __FUNCTION__ << "Error creating peer connection: " << error_or_peer_connection.error().message(); + for (auto &observer : mSignalingObserverList) + { + observer->OnRenegotiationNeeded(); + } return; } -- cgit v1.2.3 From d4fce4990ad0527430d2b3e70be360c26d6cb46b Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Sun, 28 Apr 2024 17:43:15 -0700 Subject: Reconnects to the voice server weren't happening. --- indra/llwebrtc/llwebrtc.cpp | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) (limited to 'indra/llwebrtc/llwebrtc.cpp') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index e08ace12c5..b20d3cd4e0 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -652,6 +652,13 @@ LLWebRTCPeerConnectionImpl::LLWebRTCPeerConnectionImpl() : { } +LLWebRTCPeerConnectionImpl::~LLWebRTCPeerConnectionImpl() +{ + terminate(); + mSignalingObserverList.clear(); + mDataObserverList.clear(); +} + // // LLWebRTCPeerConnection interface // @@ -670,9 +677,6 @@ void LLWebRTCPeerConnectionImpl::terminate() rtc::scoped_refptr localStream; mLocalStream.swap(localStream); - mSignalingObserverList.clear(); - mDataObserverList.clear(); - mWebRTCImpl->PostSignalingTask( [=]() { -- cgit v1.2.3 From cf13866da951f8c7c7eedfd0ff0571c6951f42a4 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Sun, 28 Apr 2024 18:43:48 -0700 Subject: Don't default the tracks to unmuted --- indra/llwebrtc/llwebrtc.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'indra/llwebrtc/llwebrtc.cpp') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index b20d3cd4e0..d07e6deea4 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -766,7 +766,7 @@ bool LLWebRTCPeerConnectionImpl::initializeConnection(const LLWebRTCPeerConnecti rtc::scoped_refptr audio_track( mPeerConnectionFactory->CreateAudioTrack("SLAudio", mPeerConnectionFactory->CreateAudioSource(audioOptions).get())); - audio_track->set_enabled(true); + audio_track->set_enabled(false); mLocalStream->AddTrack(audio_track); mPeerConnection->AddTrack(audio_track, {"SLStream"}); @@ -999,8 +999,6 @@ void LLWebRTCPeerConnectionImpl::OnConnectionChange(webrtc::PeerConnectionInterf { case webrtc::PeerConnectionInterface::PeerConnectionState::kConnected: { - mWebRTCImpl->setRecording(true); - mWebRTCImpl->PostWorkerTask([this]() { for (auto &observer : mSignalingObserverList) { -- cgit v1.2.3 From 07c3095a785864b39b03d979d43b7d0d7a932a81 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Thu, 2 May 2024 11:07:36 -0700 Subject: Unregister requested data channel when using the negotiated one. When creating a new connection, the viewer builds a data channel interface. It then gets a new one, which is a proxy. The viewer uses the new one, and therefore must unregister the callbacks from the old one. Also, update the position data before sending it after the join is sent. --- indra/llwebrtc/llwebrtc.cpp | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'indra/llwebrtc/llwebrtc.cpp') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index d07e6deea4..97c04ae446 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -957,6 +957,10 @@ void LLWebRTCPeerConnectionImpl::OnRemoveTrack(rtc::scoped_refptr channel) { + if (mDataChannel) + { + mDataChannel->UnregisterObserver(); + } mDataChannel = channel; channel->RegisterObserver(this); } -- cgit v1.2.3 From c6e147ff224e1adc9a498d4a06ad54fff710d704 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Thu, 16 May 2024 12:00:45 -0700 Subject: Race condition resulted in close causing removal of peer connection while other jobs might be using it. --- indra/llwebrtc/llwebrtc.cpp | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) (limited to 'indra/llwebrtc/llwebrtc.cpp') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index 97c04ae446..5c71831c65 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -670,24 +670,23 @@ void LLWebRTCPeerConnectionImpl::init(LLWebRTCImpl * webrtc_impl) } void LLWebRTCPeerConnectionImpl::terminate() { - rtc::scoped_refptr connection; - mPeerConnection.swap(connection); - rtc::scoped_refptr dataChannel; - mDataChannel.swap(dataChannel); - rtc::scoped_refptr localStream; - mLocalStream.swap(localStream); - mWebRTCImpl->PostSignalingTask( [=]() { - if (connection) + if (mDataChannel) { - connection->Close(); + mDataChannel->UnregisterObserver(); + mDataChannel->Close(); + mDataChannel = nullptr; } - if (dataChannel) + if (mPeerConnection) + { + mPeerConnection->Close(); + mPeerConnection = nullptr; + } + for (auto &observer : mSignalingObserverList) { - dataChannel->UnregisterObserver(); - dataChannel->Close(); + observer->OnPeerConnectionClosed(); } }); } -- cgit v1.2.3 From 2b275d43fb70f396bba4249c34442e7d70a76e19 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Fri, 17 May 2024 13:27:45 -0700 Subject: Clean up some shutdown code. --- indra/llwebrtc/llwebrtc.cpp | 35 ++++++++++++++++++++++------------- 1 file changed, 22 insertions(+), 13 deletions(-) (limited to 'indra/llwebrtc/llwebrtc.cpp') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index 5c71831c65..c72841e9e5 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -627,7 +627,6 @@ void LLWebRTCImpl::freePeerConnection(LLWebRTCPeerConnectionInterface* peer_conn std::find(mPeerConnections.begin(), mPeerConnections.end(), peer_connection); if (it != mPeerConnections.end()) { - (*it)->terminate(); mPeerConnections.erase(it); } if (mPeerConnections.empty()) @@ -645,7 +644,6 @@ void LLWebRTCImpl::freePeerConnection(LLWebRTCPeerConnectionInterface* peer_conn LLWebRTCPeerConnectionImpl::LLWebRTCPeerConnectionImpl() : mWebRTCImpl(nullptr), - mClosing(false), mPeerConnection(nullptr), mMute(false), mAnswerReceived(false) @@ -673,20 +671,32 @@ void LLWebRTCPeerConnectionImpl::terminate() mWebRTCImpl->PostSignalingTask( [=]() { - if (mDataChannel) - { - mDataChannel->UnregisterObserver(); - mDataChannel->Close(); - mDataChannel = nullptr; - } if (mPeerConnection) { + if (mDataChannel) + { + { + mDataChannel->Close(); + mDataChannel = nullptr; + } + } + mPeerConnection->Close(); + if (mLocalStream) + { + auto tracks = mLocalStream->GetAudioTracks(); + for (auto& track : tracks) + { + mLocalStream->RemoveTrack(track); + } + mLocalStream = nullptr; + } mPeerConnection = nullptr; - } - for (auto &observer : mSignalingObserverList) - { - observer->OnPeerConnectionClosed(); + + for (auto &observer : mSignalingObserverList) + { + observer->OnPeerConnectionClosed(); + } } }); } @@ -810,7 +820,6 @@ bool LLWebRTCPeerConnectionImpl::initializeConnection(const LLWebRTCPeerConnecti bool LLWebRTCPeerConnectionImpl::shutdownConnection() { - mClosing = true; terminate(); return true; } -- cgit v1.2.3 From ddbd1ab47ea6cd76ed3e6bf32ffaeb73a32b4a97 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Sun, 19 May 2024 02:30:45 -0700 Subject: More session shutdown cleanup --- indra/llwebrtc/llwebrtc.cpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) (limited to 'indra/llwebrtc/llwebrtc.cpp') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index c72841e9e5..14fb63ec2a 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -280,9 +280,15 @@ void LLWebRTCImpl::terminate() { connection->terminate(); } - mPeerConnections.clear(); + + // connection->terminate() above spawns a number of Signaling thread calls to + // shut down the connection. The following Blocking Call will wait + // until they're done before it's executed, allowing time to clean up. mSignalingThread->BlockingCall([this]() { mPeerConnectionFactory = nullptr; }); + + mPeerConnections.clear(); + mWorkerThread->BlockingCall( [this]() { @@ -652,7 +658,6 @@ LLWebRTCPeerConnectionImpl::LLWebRTCPeerConnectionImpl() : LLWebRTCPeerConnectionImpl::~LLWebRTCPeerConnectionImpl() { - terminate(); mSignalingObserverList.clear(); mDataObserverList.clear(); } -- cgit v1.2.3 From 34a2fd525f5dde075bfc5615e6deb241ae61b94c Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Mon, 24 Jun 2024 14:42:30 -0700 Subject: [WebRTC] control microphone gain via custom audio processor. Previously, there were two places audio gain could be controlled: - the device manager - the audio track The device manager audio gain control sets the system gain for all applications, not just the webrtc application. The audio track gain happens well after the audio processing where we want it to happen. So, gain control was added to the existing custom audio processor, which previously only handled calculating and retrieving the audio levels. After these changes, the microphone gain slider does impact the audio volume heard by peers. --- indra/llwebrtc/llwebrtc.cpp | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) (limited to 'indra/llwebrtc/llwebrtc.cpp') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index 14fb63ec2a..d5bd913315 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -35,6 +35,7 @@ #include "api/media_stream_interface.h" #include "api/media_stream_track.h" #include "modules/audio_processing/audio_buffer.h" +#include "modules/audio_mixer/audio_mixer_impl.h" namespace llwebrtc { @@ -88,7 +89,7 @@ void LLAudioDeviceObserver::OnRenderData(const void *audio_samples, { } -LLCustomProcessor::LLCustomProcessor() : mSampleRateHz(0), mNumChannels(0), mMicrophoneEnergy(0.0) +LLCustomProcessor::LLCustomProcessor() : mSampleRateHz(0), mNumChannels(0), mMicrophoneEnergy(0.0), mGain(1.0) { memset(mSumVector, 0, sizeof(mSumVector)); } @@ -128,9 +129,13 @@ void LLCustomProcessor::Process(webrtc::AudioBuffer *audio_in) for (size_t index = 0; index < stream_config.num_samples(); index++) { float sample = frame_samples[index]; + sample = sample * mGain; // apply gain + frame_samples[index] = sample; // write processed sample back to buffer. energy += sample * sample; } + audio_in->CopyFrom(&frame[0], stream_config); + // smooth it. size_t buffer_size = sizeof(mSumVector) / sizeof(mSumVector[0]); float totalSum = 0; @@ -236,9 +241,9 @@ void LLWebRTCImpl::init() webrtc::AudioProcessing::Config apm_config; apm_config.echo_canceller.enabled = false; apm_config.echo_canceller.mobile_mode = false; - apm_config.gain_controller1.enabled = true; + apm_config.gain_controller1.enabled = false; apm_config.gain_controller1.mode = webrtc::AudioProcessing::Config::GainController1::kAdaptiveAnalog; - apm_config.gain_controller2.enabled = true; + apm_config.gain_controller2.enabled = false; apm_config.high_pass_filter.enabled = true; apm_config.noise_suppression.enabled = true; apm_config.noise_suppression.level = webrtc::AudioProcessing::Config::NoiseSuppression::kVeryHigh; @@ -260,6 +265,7 @@ void LLWebRTCImpl::init() mAudioProcessingModule->ApplyConfig(apm_config); mAudioProcessingModule->Initialize(processing_config); + mPeerConnectionFactory = webrtc::CreatePeerConnectionFactory(mNetworkThread.get(), mWorkerThread.get(), mSignalingThread.get(), @@ -336,9 +342,9 @@ void LLWebRTCImpl::setAudioConfig(LLWebRTCDeviceInterface::AudioConfig config) webrtc::AudioProcessing::Config apm_config; apm_config.echo_canceller.enabled = config.mEchoCancellation; apm_config.echo_canceller.mobile_mode = false; - apm_config.gain_controller1.enabled = true; + apm_config.gain_controller1.enabled = config.mAGC; apm_config.gain_controller1.mode = webrtc::AudioProcessing::Config::GainController1::kAdaptiveAnalog; - apm_config.gain_controller2.enabled = true; + apm_config.gain_controller2.enabled = false; apm_config.high_pass_filter.enabled = true; apm_config.transient_suppression.enabled = true; apm_config.pipeline.multi_channel_render = true; @@ -612,6 +618,8 @@ float LLWebRTCImpl::getTuningAudioLevel() { return -20 * log10f(mTuningAudioDevi float LLWebRTCImpl::getPeerConnectionAudioLevel() { return -20 * log10f(mPeerCustomProcessor->getMicrophoneEnergy()); } +void LLWebRTCImpl::setPeerConnectionGain(float gain) { mPeerCustomProcessor->setGain(gain); } + // // Peer Connection Helpers @@ -937,7 +945,7 @@ void LLWebRTCPeerConnectionImpl::setSendVolume(float volume) { for (auto &track : mLocalStream->GetAudioTracks()) { - track->GetSource()->SetVolume(volume); + track->GetSource()->SetVolume(volume*5.0); } } }); -- cgit v1.2.3 From b746e78c78eaba3460b0f49208dc8b864c376d34 Mon Sep 17 00:00:00 2001 From: Andrey Lihatskiy Date: Thu, 18 Jul 2024 10:01:15 +0300 Subject: Fix trailing whitespaces in webrtc code to pass pre-commit --- indra/llwebrtc/llwebrtc.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'indra/llwebrtc/llwebrtc.cpp') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index d5bd913315..6dc632aba4 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -458,7 +458,7 @@ void ll_set_device_module_render_device(rtc::scoped_refptrSetPlayoutDevice(webrtc::AudioDeviceModule::kDefaultDevice); } - else + else { device_module->SetPlayoutDevice(device); } @@ -656,7 +656,7 @@ void LLWebRTCImpl::freePeerConnection(LLWebRTCPeerConnectionInterface* peer_conn // Most peer connection (signaling) happens on // the signaling thread. -LLWebRTCPeerConnectionImpl::LLWebRTCPeerConnectionImpl() : +LLWebRTCPeerConnectionImpl::LLWebRTCPeerConnectionImpl() : mWebRTCImpl(nullptr), mPeerConnection(nullptr), mMute(false), @@ -1171,7 +1171,7 @@ void LLWebRTCPeerConnectionImpl::OnSuccess(webrtc::SessionDescriptionInterface * { observer->OnOfferAvailable(mangled_sdp); } - + mPeerConnection->SetLocalDescription(std::unique_ptr( webrtc::CreateSessionDescription(webrtc::SdpType::kOffer, mangled_sdp)), rtc::scoped_refptr(this)); -- cgit v1.2.3 From b9c222dfaeb4531f91f6e0bb02bb4b0da599d08b Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Wed, 31 Jul 2024 21:23:30 -0600 Subject: Implement a Logging Sink for WebRTC WebRTC logs now pass out of the webrtc library into a logging sink, which converts them into SecondLife.log compatable logging calls. This includes fatal errors and asserts, which are now logged into SecondLife.log, and should be available in the crash logger. --- indra/llwebrtc/llwebrtc.cpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) (limited to 'indra/llwebrtc/llwebrtc.cpp') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index 6dc632aba4..2c890acbdb 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -154,7 +154,8 @@ void LLCustomProcessor::Process(webrtc::AudioBuffer *audio_in) // LLWebRTCImpl implementation // -LLWebRTCImpl::LLWebRTCImpl() : +LLWebRTCImpl::LLWebRTCImpl(LLWebRTCLogCallback* logCallback) : + mLogSink(new LLWebRTCLogSink(logCallback)), mPeerCustomProcessor(nullptr), mMute(true), mTuningMode(false), @@ -173,6 +174,7 @@ void LLWebRTCImpl::init() // Normal logging is rather spammy, so turn it off. rtc::LogMessage::LogToDebug(rtc::LS_NONE); rtc::LogMessage::SetLogToStderr(true); + rtc::LogMessage::AddLogToStream(mLogSink, rtc::LS_VERBOSE); mTaskQueueFactory = webrtc::CreateDefaultTaskQueueFactory(); @@ -312,6 +314,7 @@ void LLWebRTCImpl::terminate() mPeerDeviceModule = nullptr; mTaskQueueFactory = nullptr; }); + rtc::LogMessage::RemoveLogToStream(mLogSink); } // @@ -1327,9 +1330,9 @@ void freePeerConnection(LLWebRTCPeerConnectionInterface* peer_connection) } -void init() +void init(LLWebRTCLogCallback* logCallback) { - gWebRTCImpl = new LLWebRTCImpl(); + gWebRTCImpl = new LLWebRTCImpl(logCallback); gWebRTCImpl->init(); } -- cgit v1.2.3