From bf58173656b94243df835d29f6d5a7c4467d4d1e Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Wed, 6 Sep 2023 15:28:02 -0700 Subject: Checkpoint WebRTC Voice --- indra/llwebrtc/llwebrtc_impl.h | 177 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 177 insertions(+) create mode 100644 indra/llwebrtc/llwebrtc_impl.h (limited to 'indra/llwebrtc/llwebrtc_impl.h') diff --git a/indra/llwebrtc/llwebrtc_impl.h b/indra/llwebrtc/llwebrtc_impl.h new file mode 100644 index 0000000000..10916e5a25 --- /dev/null +++ b/indra/llwebrtc/llwebrtc_impl.h @@ -0,0 +1,177 @@ +/** + * @file llaccordionctrl.cpp + * @brief Accordion panel implementation + * + * $LicenseInfo:firstyear=2023&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2023, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef LLWEBRTC_IMPL_H +#define LLWEBRTC_IMPL_H + +#define LL_MAKEDLL +#define WEBRTC_WIN 1 +#include "llwebrtc.h" +// WebRTC Includes +#ifdef WEBRTC_WIN +#pragma warning(disable : 4996) +#endif // WEBRTC_WIN +#include "api/scoped_refptr.h" +#include "rtc_base/ref_count.h" +#include "rtc_base/ref_counted_object.h" +#include "rtc_base/ssl_adapter.h" +#include "rtc_base/thread.h" +#include "api/peer_connection_interface.h" +#include "api/media_stream_interface.h" +#include "api/create_peerconnection_factory.h" +#include "modules/audio_device/include/audio_device.h" +#include "modules/audio_device/include/audio_device_data_observer.h" +#include "rtc_base/task_queue.h" +#include "api/task_queue/task_queue_factory.h" +#include "api/task_queue/default_task_queue_factory.h" +#include "modules/audio_device/include/audio_device_defines.h" + + +namespace llwebrtc +{ + +class LLWebRTCImpl : public LLWebRTCDeviceInterface, + public LLWebRTCSignalInterface, + public LLWebRTCAudioInterface, + public webrtc::AudioDeviceDataObserver, + public webrtc::PeerConnectionObserver, + public webrtc::CreateSessionDescriptionObserver, + public webrtc::SetRemoteDescriptionObserverInterface, + public webrtc::SetLocalDescriptionObserverInterface + +{ + public: + LLWebRTCImpl() : + mTuningEnergy(0.0) + { + } + ~LLWebRTCImpl() {} + + void init(); + + // + // LLWebRTCDeviceInterface + // + + void refreshDevices() override; + + void setDevicesObserver(LLWebRTCDevicesObserver *observer) override; + void unsetDevicesObserver(LLWebRTCDevicesObserver *observer) override; + void setCaptureDevice(const std::string& id) override; + + void setRenderDevice(const std::string& id) override; + + void setTuningMode(bool enable) override; + double getTuningMicrophoneEnergy() override; + + + void OnCaptureData(const void *audio_samples, + const size_t num_samples, + const size_t bytes_per_sample, + const size_t num_channels, + const uint32_t samples_per_sec) override; + + void OnRenderData(const void *audio_samples, + const size_t num_samples, + const size_t bytes_per_sample, + const size_t num_channels, + const uint32_t samples_per_sec) override; + + // + // LLWebRTCSignalInterface + // + + void setSignalingObserver(LLWebRTCSignalingObserver *observer) override; + void unsetSignalingObserver(LLWebRTCSignalingObserver *observer) override; + bool initializeConnection() override; + void shutdownConnection() override; + void AnswerAvailable(const std::string &sdp) override; + + + // + // LLWebRTCAudioInterface + // + void setMute(bool mute) override; + + // + // PeerConnectionObserver implementation. + // + + void OnSignalingChange(webrtc::PeerConnectionInterface::SignalingState new_state) override {} + void OnAddTrack(rtc::scoped_refptr receiver, + const std::vector> &streams) override; + void OnRemoveTrack(rtc::scoped_refptr receiver) override; + void OnDataChannel(rtc::scoped_refptr channel) override {} + void OnRenegotiationNeeded() override {} + void OnIceConnectionChange(webrtc::PeerConnectionInterface::IceConnectionState new_state) override {}; + void OnIceGatheringChange(webrtc::PeerConnectionInterface::IceGatheringState new_state) override; + void OnIceCandidate(const webrtc::IceCandidateInterface *candidate) override; + void OnIceConnectionReceivingChange(bool receiving) override {} + void OnConnectionChange(webrtc::PeerConnectionInterface::PeerConnectionState new_state) override; + + // + // CreateSessionDescriptionObserver implementation. + // + void OnSuccess(webrtc::SessionDescriptionInterface *desc) override; + void OnFailure(webrtc::RTCError error) override; + + // + // SetRemoteDescriptionObserverInterface implementation. + // + void OnSetRemoteDescriptionComplete(webrtc::RTCError error) override; + + // + // SetLocalDescriptionObserverInterface implementation. + // + void OnSetLocalDescriptionComplete(webrtc::RTCError error) override; + + protected: + std::unique_ptr mNetworkThread; + std::unique_ptr mWorkerThread; + std::unique_ptr mSignalingThread; + rtc::scoped_refptr mPeerConnectionFactory; + webrtc::PeerConnectionInterface::RTCConfiguration mConfiguration; + std::unique_ptr mTaskQueueFactory; + + + // Devices + void updateDevices(); + rtc::scoped_refptr mDeviceModule; + std::vector mVoiceDevicesObserverList; + + double mTuningEnergy; + + // signaling + std::vector mSignalingObserverList; + std::vector> mCachedIceCandidates; + bool mAnswerReceived; + + rtc::scoped_refptr mPeerConnection; +}; + +} + +#endif // LLWEBRTC_IMPL_H \ No newline at end of file -- cgit v1.2.3 From db7ab5d73e748f86ee00e31965265969c0c7060f Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Sat, 9 Sep 2023 22:19:22 -0700 Subject: Updates to build on mac. --- indra/llwebrtc/llwebrtc_impl.h | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) (limited to 'indra/llwebrtc/llwebrtc_impl.h') diff --git a/indra/llwebrtc/llwebrtc_impl.h b/indra/llwebrtc/llwebrtc_impl.h index 10916e5a25..07c0f514b2 100644 --- a/indra/llwebrtc/llwebrtc_impl.h +++ b/indra/llwebrtc/llwebrtc_impl.h @@ -28,12 +28,21 @@ #define LLWEBRTC_IMPL_H #define LL_MAKEDLL +#if defined(_WIN32) || defined(_WIN64) #define WEBRTC_WIN 1 +#elif defined(__APPLE__) +#define WEBRTC_MAC 1 +#define WEBRTC_POSIX 1 +#elif __linux__ +#define WEBRTC_LINUX 1 +#endif + #include "llwebrtc.h" // WebRTC Includes #ifdef WEBRTC_WIN #pragma warning(disable : 4996) #endif // WEBRTC_WIN + #include "api/scoped_refptr.h" #include "rtc_base/ref_count.h" #include "rtc_base/ref_counted_object.h" @@ -174,4 +183,4 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, } -#endif // LLWEBRTC_IMPL_H \ No newline at end of file +#endif // LLWEBRTC_IMPL_H -- cgit v1.2.3 From b2d4ce16ad344162ecc190f798704e3ac179555a Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Sun, 10 Sep 2023 19:24:03 -0700 Subject: Fix windows pragma error --- indra/llwebrtc/llwebrtc_impl.h | 1 + 1 file changed, 1 insertion(+) (limited to 'indra/llwebrtc/llwebrtc_impl.h') diff --git a/indra/llwebrtc/llwebrtc_impl.h b/indra/llwebrtc/llwebrtc_impl.h index 07c0f514b2..d439bd253d 100644 --- a/indra/llwebrtc/llwebrtc_impl.h +++ b/indra/llwebrtc/llwebrtc_impl.h @@ -41,6 +41,7 @@ // WebRTC Includes #ifdef WEBRTC_WIN #pragma warning(disable : 4996) +#pragma warning(disable : 4068) #endif // WEBRTC_WIN #include "api/scoped_refptr.h" -- cgit v1.2.3 From 8ba90d077381bf16b4ba03a0f530f76e770e69c1 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Wed, 13 Sep 2023 14:49:42 -0700 Subject: Hook up speaker volume. --- indra/llwebrtc/llwebrtc_impl.h | 1 + 1 file changed, 1 insertion(+) (limited to 'indra/llwebrtc/llwebrtc_impl.h') diff --git a/indra/llwebrtc/llwebrtc_impl.h b/indra/llwebrtc/llwebrtc_impl.h index d439bd253d..5c6cfcdbc6 100644 --- a/indra/llwebrtc/llwebrtc_impl.h +++ b/indra/llwebrtc/llwebrtc_impl.h @@ -125,6 +125,7 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, // LLWebRTCAudioInterface // void setMute(bool mute) override; + void setSpeakerVolume(float folume) override; // range 0.0-1.0 // // PeerConnectionObserver implementation. -- cgit v1.2.3 From 639e63faab239b88d41c8e2c755509e9dcdc6251 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Mon, 18 Sep 2023 11:25:47 -0700 Subject: Fix voice device settings --- indra/llwebrtc/llwebrtc_impl.h | 3 +++ 1 file changed, 3 insertions(+) (limited to 'indra/llwebrtc/llwebrtc_impl.h') diff --git a/indra/llwebrtc/llwebrtc_impl.h b/indra/llwebrtc/llwebrtc_impl.h index 5c6cfcdbc6..3f9b06cae7 100644 --- a/indra/llwebrtc/llwebrtc_impl.h +++ b/indra/llwebrtc/llwebrtc_impl.h @@ -160,6 +160,9 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, void OnSetLocalDescriptionComplete(webrtc::RTCError error) override; protected: + + bool initializeConnectionThreaded(); + std::unique_ptr mNetworkThread; std::unique_ptr mWorkerThread; std::unique_ptr mSignalingThread; -- cgit v1.2.3 From 8859312b1f0d975793c6c2a3d7b23b9880c657c5 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Tue, 19 Sep 2023 10:14:29 -0700 Subject: add datachannel support --- indra/llwebrtc/llwebrtc_impl.h | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) (limited to 'indra/llwebrtc/llwebrtc_impl.h') diff --git a/indra/llwebrtc/llwebrtc_impl.h b/indra/llwebrtc/llwebrtc_impl.h index 3f9b06cae7..1ad117c7f3 100644 --- a/indra/llwebrtc/llwebrtc_impl.h +++ b/indra/llwebrtc/llwebrtc_impl.h @@ -66,11 +66,13 @@ namespace llwebrtc class LLWebRTCImpl : public LLWebRTCDeviceInterface, public LLWebRTCSignalInterface, public LLWebRTCAudioInterface, + public LLWebRTCDataInterface, public webrtc::AudioDeviceDataObserver, public webrtc::PeerConnectionObserver, public webrtc::CreateSessionDescriptionObserver, public webrtc::SetRemoteDescriptionObserverInterface, - public webrtc::SetLocalDescriptionObserverInterface + public webrtc::SetLocalDescriptionObserverInterface, + public webrtc::DataChannelObserver { public: @@ -126,6 +128,13 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, // void setMute(bool mute) override; void setSpeakerVolume(float folume) override; // range 0.0-1.0 + + // + // LLWebRTCDataInterface + // + void sendData(const std::string& data, bool binary=false) override; + void setDataObserver(LLWebRTCDataObserver *observer) override; + void unsetDataObserver(LLWebRTCDataObserver *observer) override; // // PeerConnectionObserver implementation. @@ -158,6 +167,12 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, // SetLocalDescriptionObserverInterface implementation. // void OnSetLocalDescriptionComplete(webrtc::RTCError error) override; + + // + // DataChannelObserver implementation. + // + void OnStateChange() override {} + void OnMessage(const webrtc::DataBuffer& buffer) override; protected: @@ -184,6 +199,9 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, bool mAnswerReceived; rtc::scoped_refptr mPeerConnection; + + std::vector mDataObserverList; + rtc::scoped_refptr mDataChannel; }; } -- cgit v1.2.3 From 1cd8f6f4f88f7717f0fcafbb5d47de0af59d5fb7 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Thu, 21 Sep 2023 15:28:58 -0700 Subject: Stream audio levels to and from viewers via DataChannels --- indra/llwebrtc/llwebrtc_impl.h | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) (limited to 'indra/llwebrtc/llwebrtc_impl.h') diff --git a/indra/llwebrtc/llwebrtc_impl.h b/indra/llwebrtc/llwebrtc_impl.h index 1ad117c7f3..1670d10705 100644 --- a/indra/llwebrtc/llwebrtc_impl.h +++ b/indra/llwebrtc/llwebrtc_impl.h @@ -83,6 +83,7 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, ~LLWebRTCImpl() {} void init(); + void terminate(); // // LLWebRTCDeviceInterface @@ -126,8 +127,11 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, // // LLWebRTCAudioInterface // + void setAudioObserver(LLWebRTCAudioObserver *observer) override; + void unsetAudioObserver(LLWebRTCAudioObserver *observer) override; void setMute(bool mute) override; void setSpeakerVolume(float folume) override; // range 0.0-1.0 + void requestAudioLevel() override; // // LLWebRTCDataInterface @@ -144,7 +148,7 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, void OnAddTrack(rtc::scoped_refptr receiver, const std::vector> &streams) override; void OnRemoveTrack(rtc::scoped_refptr receiver) override; - void OnDataChannel(rtc::scoped_refptr channel) override {} + void OnDataChannel(rtc::scoped_refptr channel) override; void OnRenegotiationNeeded() override {} void OnIceConnectionChange(webrtc::PeerConnectionInterface::IceConnectionState new_state) override {}; void OnIceGatheringChange(webrtc::PeerConnectionInterface::IceGatheringState new_state) override; @@ -171,7 +175,7 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, // // DataChannelObserver implementation. // - void OnStateChange() override {} + void OnStateChange() override; void OnMessage(const webrtc::DataBuffer& buffer) override; protected: @@ -200,6 +204,8 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, rtc::scoped_refptr mPeerConnection; + std::vector mAudioObserverList; + std::vector mDataObserverList; rtc::scoped_refptr mDataChannel; }; -- cgit v1.2.3 From c087b8648dc85cffdc4d9a5eb85d989a9b4aa51b Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Fri, 22 Sep 2023 10:47:48 -0700 Subject: Fix shutdown crash issue. --- indra/llwebrtc/llwebrtc_impl.h | 42 ++++++++++++++++++++++++------------------ 1 file changed, 24 insertions(+), 18 deletions(-) (limited to 'indra/llwebrtc/llwebrtc_impl.h') diff --git a/indra/llwebrtc/llwebrtc_impl.h b/indra/llwebrtc/llwebrtc_impl.h index 1670d10705..da829e52af 100644 --- a/indra/llwebrtc/llwebrtc_impl.h +++ b/indra/llwebrtc/llwebrtc_impl.h @@ -63,11 +63,31 @@ namespace llwebrtc { +class LLAudioDeviceObserver : public webrtc::AudioDeviceDataObserver +{ + public: + double getMicrophoneEnergy(); + + void OnCaptureData(const void *audio_samples, + const size_t num_samples, + const size_t bytes_per_sample, + const size_t num_channels, + const uint32_t samples_per_sec) override; + + void OnRenderData(const void *audio_samples, + const size_t num_samples, + const size_t bytes_per_sample, + const size_t num_channels, + const uint32_t samples_per_sec) override; + + protected: + double mMicrophoneEnergy; +}; + class LLWebRTCImpl : public LLWebRTCDeviceInterface, public LLWebRTCSignalInterface, public LLWebRTCAudioInterface, public LLWebRTCDataInterface, - public webrtc::AudioDeviceDataObserver, public webrtc::PeerConnectionObserver, public webrtc::CreateSessionDescriptionObserver, public webrtc::SetRemoteDescriptionObserverInterface, @@ -77,7 +97,7 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, { public: LLWebRTCImpl() : - mTuningEnergy(0.0) + mAudioDeviceObserver(nullptr) { } ~LLWebRTCImpl() {} @@ -98,20 +118,7 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, void setRenderDevice(const std::string& id) override; void setTuningMode(bool enable) override; - double getTuningMicrophoneEnergy() override; - - - void OnCaptureData(const void *audio_samples, - const size_t num_samples, - const size_t bytes_per_sample, - const size_t num_channels, - const uint32_t samples_per_sec) override; - - void OnRenderData(const void *audio_samples, - const size_t num_samples, - const size_t bytes_per_sample, - const size_t num_channels, - const uint32_t samples_per_sec) override; + double getAudioLevel() override; // // LLWebRTCSignalInterface @@ -131,7 +138,6 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, void unsetAudioObserver(LLWebRTCAudioObserver *observer) override; void setMute(bool mute) override; void setSpeakerVolume(float folume) override; // range 0.0-1.0 - void requestAudioLevel() override; // // LLWebRTCDataInterface @@ -195,7 +201,7 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, rtc::scoped_refptr mDeviceModule; std::vector mVoiceDevicesObserverList; - double mTuningEnergy; + LLAudioDeviceObserver * mAudioDeviceObserver; // signaling std::vector mSignalingObserverList; -- cgit v1.2.3 From 0110e37c7af96b02fd4ad92144b1b042e307d6a5 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Fri, 22 Sep 2023 16:55:13 -0700 Subject: Smooth voice power level reporting. --- indra/llwebrtc/llwebrtc_impl.h | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'indra/llwebrtc/llwebrtc_impl.h') diff --git a/indra/llwebrtc/llwebrtc_impl.h b/indra/llwebrtc/llwebrtc_impl.h index da829e52af..70586f9013 100644 --- a/indra/llwebrtc/llwebrtc_impl.h +++ b/indra/llwebrtc/llwebrtc_impl.h @@ -66,6 +66,8 @@ namespace llwebrtc class LLAudioDeviceObserver : public webrtc::AudioDeviceDataObserver { public: + LLAudioDeviceObserver(); + double getMicrophoneEnergy(); void OnCaptureData(const void *audio_samples, @@ -81,7 +83,8 @@ class LLAudioDeviceObserver : public webrtc::AudioDeviceDataObserver const uint32_t samples_per_sec) override; protected: - double mMicrophoneEnergy; + float mSumVector[30]; // 300 ms of smoothing + float mMicrophoneEnergy; }; class LLWebRTCImpl : public LLWebRTCDeviceInterface, -- cgit v1.2.3 From 5872e037e801654d59f37e32f672db84c456d587 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Fri, 29 Sep 2023 19:13:03 -0700 Subject: Improve reconnection logic and allow device setting when connected or not connected --- indra/llwebrtc/llwebrtc_impl.h | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) (limited to 'indra/llwebrtc/llwebrtc_impl.h') diff --git a/indra/llwebrtc/llwebrtc_impl.h b/indra/llwebrtc/llwebrtc_impl.h index 70586f9013..947b417923 100644 --- a/indra/llwebrtc/llwebrtc_impl.h +++ b/indra/llwebrtc/llwebrtc_impl.h @@ -68,7 +68,7 @@ class LLAudioDeviceObserver : public webrtc::AudioDeviceDataObserver public: LLAudioDeviceObserver(); - double getMicrophoneEnergy(); + float getMicrophoneEnergy(); void OnCaptureData(const void *audio_samples, const size_t num_samples, @@ -100,7 +100,7 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, { public: LLWebRTCImpl() : - mAudioDeviceObserver(nullptr) + mTuningAudioDeviceObserver(nullptr), mPeerAudioDeviceObserver(nullptr) { } ~LLWebRTCImpl() {} @@ -121,7 +121,7 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, void setRenderDevice(const std::string& id) override; void setTuningMode(bool enable) override; - double getAudioLevel() override; + float getTuningAudioLevel() override; // // LLWebRTCSignalInterface @@ -141,6 +141,7 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, void unsetAudioObserver(LLWebRTCAudioObserver *observer) override; void setMute(bool mute) override; void setSpeakerVolume(float folume) override; // range 0.0-1.0 + float getAudioLevel() override; // // LLWebRTCDataInterface @@ -201,10 +202,16 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, // Devices void updateDevices(); - rtc::scoped_refptr mDeviceModule; + rtc::scoped_refptr mTuningDeviceModule; + rtc::scoped_refptr mPeerDeviceModule; std::vector mVoiceDevicesObserverList; - LLAudioDeviceObserver * mAudioDeviceObserver; + // accessors in webrtc aren't apparently implemented yet. + int32_t mPlayoutDevice; + int32_t mRecordingDevice; + + LLAudioDeviceObserver * mTuningAudioDeviceObserver; + LLAudioDeviceObserver * mPeerAudioDeviceObserver; // signaling std::vector mSignalingObserverList; -- cgit v1.2.3 From 9d070201fe57e22db7cf55a2160a82e3c6413471 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Tue, 3 Oct 2023 15:51:37 -0700 Subject: fix device selection while speaking. --- indra/llwebrtc/llwebrtc_impl.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'indra/llwebrtc/llwebrtc_impl.h') diff --git a/indra/llwebrtc/llwebrtc_impl.h b/indra/llwebrtc/llwebrtc_impl.h index 947b417923..24edd6eaa2 100644 --- a/indra/llwebrtc/llwebrtc_impl.h +++ b/indra/llwebrtc/llwebrtc_impl.h @@ -100,7 +100,7 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, { public: LLWebRTCImpl() : - mTuningAudioDeviceObserver(nullptr), mPeerAudioDeviceObserver(nullptr) + mTuningAudioDeviceObserver(nullptr), mPeerAudioDeviceObserver(nullptr), mMute(true) { } ~LLWebRTCImpl() {} @@ -209,6 +209,7 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, // accessors in webrtc aren't apparently implemented yet. int32_t mPlayoutDevice; int32_t mRecordingDevice; + bool mMute; LLAudioDeviceObserver * mTuningAudioDeviceObserver; LLAudioDeviceObserver * mPeerAudioDeviceObserver; -- cgit v1.2.3 From c856a52477d0a50521a44f2d8255d5bde2ef39b4 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Fri, 6 Oct 2023 21:46:02 -0700 Subject: Fix race in initialization. Fix failure to send ice candidates to janus. --- indra/llwebrtc/llwebrtc_impl.h | 2 -- 1 file changed, 2 deletions(-) (limited to 'indra/llwebrtc/llwebrtc_impl.h') diff --git a/indra/llwebrtc/llwebrtc_impl.h b/indra/llwebrtc/llwebrtc_impl.h index 24edd6eaa2..01159604c9 100644 --- a/indra/llwebrtc/llwebrtc_impl.h +++ b/indra/llwebrtc/llwebrtc_impl.h @@ -190,8 +190,6 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, protected: - bool initializeConnectionThreaded(); - std::unique_ptr mNetworkThread; std::unique_ptr mWorkerThread; std::unique_ptr mSignalingThread; -- cgit v1.2.3 From 85aea763d171ef3f3a92e7a20f9aaa8dfdcbc026 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Wed, 8 Nov 2023 10:13:15 -0800 Subject: SL-20543 - voice over region boundaries. This commit includes code to allow the llwebrtc.dll/dylib to allow multiple connections at once. --- indra/llwebrtc/llwebrtc_impl.h | 144 +++++++++++++++++++++++++++++++---------- 1 file changed, 109 insertions(+), 35 deletions(-) (limited to 'indra/llwebrtc/llwebrtc_impl.h') diff --git a/indra/llwebrtc/llwebrtc_impl.h b/indra/llwebrtc/llwebrtc_impl.h index 01159604c9..295e949b4e 100644 --- a/indra/llwebrtc/llwebrtc_impl.h +++ b/indra/llwebrtc/llwebrtc_impl.h @@ -1,6 +1,6 @@ /** - * @file llaccordionctrl.cpp - * @brief Accordion panel implementation + * @file llwebrtc_impl.h + * @brief WebRTC Interface implementation. * * $LicenseInfo:firstyear=2023&license=viewerlgpl$ * Second Life Viewer Source Code @@ -63,6 +63,8 @@ namespace llwebrtc { +class LLWebRTCPeerConnectionImpl; + class LLAudioDeviceObserver : public webrtc::AudioDeviceDataObserver { public: @@ -87,16 +89,7 @@ class LLAudioDeviceObserver : public webrtc::AudioDeviceDataObserver float mMicrophoneEnergy; }; -class LLWebRTCImpl : public LLWebRTCDeviceInterface, - public LLWebRTCSignalInterface, - public LLWebRTCAudioInterface, - public LLWebRTCDataInterface, - public webrtc::PeerConnectionObserver, - public webrtc::CreateSessionDescriptionObserver, - public webrtc::SetRemoteDescriptionObserverInterface, - public webrtc::SetLocalDescriptionObserverInterface, - public webrtc::DataChannelObserver - +class LLWebRTCImpl : public LLWebRTCDeviceInterface { public: LLWebRTCImpl() : @@ -116,15 +109,112 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, void setDevicesObserver(LLWebRTCDevicesObserver *observer) override; void unsetDevicesObserver(LLWebRTCDevicesObserver *observer) override; - void setCaptureDevice(const std::string& id) override; + void setCaptureDevice(const std::string& id) override; void setRenderDevice(const std::string& id) override; void setTuningMode(bool enable) override; float getTuningAudioLevel() override; + float getPeerAudioLevel() override; + + void setSpeakerVolume(float volume) override; // range 0.0-1.0 + void setMicrophoneVolume(float volume) override; // range 0.0-1.0 + + // + // Helpers + // + + void PostWorkerTask(absl::AnyInvocable task, + const webrtc::Location& location = webrtc::Location::Current()) + { + mWorkerThread->PostTask(std::move(task), location); + } + + void PostSignalingTask(absl::AnyInvocable task, + const webrtc::Location& location = webrtc::Location::Current()) + { + mSignalingThread->PostTask(std::move(task), location); + } + + void PostNetworkTask(absl::AnyInvocable task, + const webrtc::Location& location = webrtc::Location::Current()) + { + mNetworkThread->PostTask(std::move(task), location); + } + + void WorkerBlockingCall(rtc::FunctionView functor, + const webrtc::Location& location = webrtc::Location::Current()) + { + mWorkerThread->BlockingCall(std::move(functor), location); + } + + void SignalingBlockingCall(rtc::FunctionView functor, + const webrtc::Location& location = webrtc::Location::Current()) + { + mSignalingThread->BlockingCall(std::move(functor), location); + } + + void NetworkBlockingCall(rtc::FunctionView functor, + const webrtc::Location& location = webrtc::Location::Current()) + { + mNetworkThread->BlockingCall(std::move(functor), location); + } + + rtc::scoped_refptr getPeerConnectionFactory() { return mPeerConnectionFactory; } + + LLWebRTCPeerConnection * newPeerConnection(); + void freePeerConnection(LLWebRTCPeerConnection * peer_connection); + + protected: + + std::unique_ptr mNetworkThread; + std::unique_ptr mWorkerThread; + std::unique_ptr mSignalingThread; + rtc::scoped_refptr mPeerConnectionFactory; + webrtc::PeerConnectionInterface::RTCConfiguration mConfiguration; + std::unique_ptr mTaskQueueFactory; + + + // Devices + void updateDevices(); + rtc::scoped_refptr mTuningDeviceModule; + rtc::scoped_refptr mPeerDeviceModule; + std::vector mVoiceDevicesObserverList; + // accessors in webrtc aren't apparently implemented yet. + int32_t mPlayoutDevice; + int32_t mRecordingDevice; + bool mMute; + + LLAudioDeviceObserver * mTuningAudioDeviceObserver; + LLAudioDeviceObserver * mPeerAudioDeviceObserver; + + // peer connections + std::vector> mPeerConnections; +}; + +class LLWebRTCPeerConnectionImpl : public LLWebRTCPeerConnection, + public LLWebRTCAudioInterface, + public LLWebRTCDataInterface, + public webrtc::PeerConnectionObserver, + public webrtc::CreateSessionDescriptionObserver, + public webrtc::SetRemoteDescriptionObserverInterface, + public webrtc::SetLocalDescriptionObserverInterface, + public webrtc::DataChannelObserver + +{ + public: + LLWebRTCPeerConnectionImpl() {} + ~LLWebRTCPeerConnectionImpl() {} + + void init(LLWebRTCImpl * webrtc_impl); + void terminate(); + + virtual void AddRef() const override = 0; + virtual rtc::RefCountReleaseStatus Release() const override = 0; + // - // LLWebRTCSignalInterface + // LLWebRTCPeerConnection // void setSignalingObserver(LLWebRTCSignalingObserver *observer) override; @@ -140,8 +230,6 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, void setAudioObserver(LLWebRTCAudioObserver *observer) override; void unsetAudioObserver(LLWebRTCAudioObserver *observer) override; void setMute(bool mute) override; - void setSpeakerVolume(float folume) override; // range 0.0-1.0 - float getAudioLevel() override; // // LLWebRTCDataInterface @@ -187,31 +275,17 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, // void OnStateChange() override; void OnMessage(const webrtc::DataBuffer& buffer) override; + + // Helpers + void enableTracks(bool enable); protected: - - std::unique_ptr mNetworkThread; - std::unique_ptr mWorkerThread; - std::unique_ptr mSignalingThread; + + LLWebRTCImpl * mWebRTCImpl; rtc::scoped_refptr mPeerConnectionFactory; - webrtc::PeerConnectionInterface::RTCConfiguration mConfiguration; - std::unique_ptr mTaskQueueFactory; - - - // Devices - void updateDevices(); - rtc::scoped_refptr mTuningDeviceModule; - rtc::scoped_refptr mPeerDeviceModule; - std::vector mVoiceDevicesObserverList; - // accessors in webrtc aren't apparently implemented yet. - int32_t mPlayoutDevice; - int32_t mRecordingDevice; bool mMute; - LLAudioDeviceObserver * mTuningAudioDeviceObserver; - LLAudioDeviceObserver * mPeerAudioDeviceObserver; - // signaling std::vector mSignalingObserverList; std::vector> mCachedIceCandidates; -- cgit v1.2.3 From 1b77987a0b84fac6a0d07b7782a23bb22aa48ec6 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Wed, 8 Nov 2023 10:17:56 -0800 Subject: comment fixes --- indra/llwebrtc/llwebrtc_impl.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'indra/llwebrtc/llwebrtc_impl.h') diff --git a/indra/llwebrtc/llwebrtc_impl.h b/indra/llwebrtc/llwebrtc_impl.h index 295e949b4e..eb439b5a46 100644 --- a/indra/llwebrtc/llwebrtc_impl.h +++ b/indra/llwebrtc/llwebrtc_impl.h @@ -1,6 +1,6 @@ /** * @file llwebrtc_impl.h - * @brief WebRTC Interface implementation. + * @brief WebRTC interface implementation header * * $LicenseInfo:firstyear=2023&license=viewerlgpl$ * Second Life Viewer Source Code -- cgit v1.2.3 From ebfa44cdb76afb9632556115eef10969912340f5 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Thu, 30 Nov 2023 13:14:07 -0800 Subject: Refactor/clean-up WebRTC voice to handle multiple voice streams This is useful for cross-region voice, quick voice switching, etc. --- indra/llwebrtc/llwebrtc_impl.h | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) (limited to 'indra/llwebrtc/llwebrtc_impl.h') diff --git a/indra/llwebrtc/llwebrtc_impl.h b/indra/llwebrtc/llwebrtc_impl.h index eb439b5a46..76e29c63fb 100644 --- a/indra/llwebrtc/llwebrtc_impl.h +++ b/indra/llwebrtc/llwebrtc_impl.h @@ -118,7 +118,8 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface float getPeerAudioLevel() override; void setSpeakerVolume(float volume) override; // range 0.0-1.0 - void setMicrophoneVolume(float volume) override; // range 0.0-1.0 + void setMicrophoneVolume(float volume) override; // range 0.0-1.0 + void setMute(bool mute) override; // // Helpers @@ -227,8 +228,6 @@ class LLWebRTCPeerConnectionImpl : public LLWebRTCPeerConnection, // // LLWebRTCAudioInterface // - void setAudioObserver(LLWebRTCAudioObserver *observer) override; - void unsetAudioObserver(LLWebRTCAudioObserver *observer) override; void setMute(bool mute) override; // @@ -292,8 +291,6 @@ class LLWebRTCPeerConnectionImpl : public LLWebRTCPeerConnection, bool mAnswerReceived; rtc::scoped_refptr mPeerConnection; - - std::vector mAudioObserverList; std::vector mDataObserverList; rtc::scoped_refptr mDataChannel; -- cgit v1.2.3 From 0b9c27dc70255385fd65ac2f1f8f99e2e013ea03 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Fri, 1 Dec 2023 01:14:33 -0800 Subject: Using the device module to set speaker/mic volume set the system mic/volume for all applications. Instead, modify the volume on the various streams. --- indra/llwebrtc/llwebrtc_impl.h | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) (limited to 'indra/llwebrtc/llwebrtc_impl.h') diff --git a/indra/llwebrtc/llwebrtc_impl.h b/indra/llwebrtc/llwebrtc_impl.h index 76e29c63fb..efb0d3add3 100644 --- a/indra/llwebrtc/llwebrtc_impl.h +++ b/indra/llwebrtc/llwebrtc_impl.h @@ -116,10 +116,6 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface void setTuningMode(bool enable) override; float getTuningAudioLevel() override; float getPeerAudioLevel() override; - - void setSpeakerVolume(float volume) override; // range 0.0-1.0 - void setMicrophoneVolume(float volume) override; // range 0.0-1.0 - void setMute(bool mute) override; // // Helpers @@ -224,12 +220,13 @@ class LLWebRTCPeerConnectionImpl : public LLWebRTCPeerConnection, void shutdownConnection() override; void AnswerAvailable(const std::string &sdp) override; - // // LLWebRTCAudioInterface // void setMute(bool mute) override; - + void setReceiveVolume(float volume) override; // volume between 0.0 and 1.0 + void setSendVolume(float volume) override; // volume between 0.0 and 1.0 + // // LLWebRTCDataInterface // @@ -291,6 +288,7 @@ class LLWebRTCPeerConnectionImpl : public LLWebRTCPeerConnection, bool mAnswerReceived; rtc::scoped_refptr mPeerConnection; + rtc::scoped_refptr mLocalStream; std::vector mDataObserverList; rtc::scoped_refptr mDataChannel; -- cgit v1.2.3 From dd250273f476a96e60e7953915d76f6e2dd765ec Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Mon, 4 Dec 2023 14:26:12 -0800 Subject: missed file --- indra/llwebrtc/llwebrtc_impl.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'indra/llwebrtc/llwebrtc_impl.h') diff --git a/indra/llwebrtc/llwebrtc_impl.h b/indra/llwebrtc/llwebrtc_impl.h index efb0d3add3..cc22c30f35 100644 --- a/indra/llwebrtc/llwebrtc_impl.h +++ b/indra/llwebrtc/llwebrtc_impl.h @@ -273,7 +273,8 @@ class LLWebRTCPeerConnectionImpl : public LLWebRTCPeerConnection, void OnMessage(const webrtc::DataBuffer& buffer) override; // Helpers - void enableTracks(bool enable); + void enableSenderTracks(bool enable); + void enableReceiverTracks(bool enable); protected: -- cgit v1.2.3 From 2a375d25dc2bb6e063d8e596ce3b4bd2c61cd96b Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Fri, 8 Dec 2023 12:09:02 -0800 Subject: fix device selection (hopefully) --- indra/llwebrtc/llwebrtc_impl.h | 1 + 1 file changed, 1 insertion(+) (limited to 'indra/llwebrtc/llwebrtc_impl.h') diff --git a/indra/llwebrtc/llwebrtc_impl.h b/indra/llwebrtc/llwebrtc_impl.h index cc22c30f35..6a84f67ef5 100644 --- a/indra/llwebrtc/llwebrtc_impl.h +++ b/indra/llwebrtc/llwebrtc_impl.h @@ -273,6 +273,7 @@ class LLWebRTCPeerConnectionImpl : public LLWebRTCPeerConnection, void OnMessage(const webrtc::DataBuffer& buffer) override; // Helpers + void resetMute(); void enableSenderTracks(bool enable); void enableReceiverTracks(bool enable); -- cgit v1.2.3 From 106a589dee7bd91f8c8893e1f077a25efb7e83eb Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Tue, 9 Jan 2024 11:57:01 -0800 Subject: New WebRTC with echo cancellation fix. Also, start/stop recording depending on whether WebRTC has negotiated. --- indra/llwebrtc/llwebrtc_impl.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'indra/llwebrtc/llwebrtc_impl.h') diff --git a/indra/llwebrtc/llwebrtc_impl.h b/indra/llwebrtc/llwebrtc_impl.h index 6a84f67ef5..884e107527 100644 --- a/indra/llwebrtc/llwebrtc_impl.h +++ b/indra/llwebrtc/llwebrtc_impl.h @@ -162,6 +162,8 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface LLWebRTCPeerConnection * newPeerConnection(); void freePeerConnection(LLWebRTCPeerConnection * peer_connection); + void setRecording(bool recording); + protected: std::unique_ptr mNetworkThread; -- cgit v1.2.3 From 8d414e408e096946b0409e8ca9d5011d64f89671 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Sat, 3 Feb 2024 22:02:48 -0800 Subject: Handle 'device changed' callback --- indra/llwebrtc/llwebrtc_impl.h | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'indra/llwebrtc/llwebrtc_impl.h') diff --git a/indra/llwebrtc/llwebrtc_impl.h b/indra/llwebrtc/llwebrtc_impl.h index 884e107527..3ccb6058ad 100644 --- a/indra/llwebrtc/llwebrtc_impl.h +++ b/indra/llwebrtc/llwebrtc_impl.h @@ -89,7 +89,7 @@ class LLAudioDeviceObserver : public webrtc::AudioDeviceDataObserver float mMicrophoneEnergy; }; -class LLWebRTCImpl : public LLWebRTCDeviceInterface +class LLWebRTCImpl : public LLWebRTCDeviceInterface, public webrtc::AudioDeviceSink { public: LLWebRTCImpl() : @@ -117,6 +117,11 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface float getTuningAudioLevel() override; float getPeerAudioLevel() override; + // + // AudioDeviceSink + // + void OnDevicesUpdated() override; + // // Helpers // -- cgit v1.2.3 From dfa77d942c52ddf55942afb497a64fbb1d2fccc0 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Mon, 5 Feb 2024 14:42:21 -0800 Subject: Use a custom audio processor to pull data for level determinations, which will happen after AGC --- indra/llwebrtc/llwebrtc_impl.h | 37 ++++++++++++++++++------------------- 1 file changed, 18 insertions(+), 19 deletions(-) (limited to 'indra/llwebrtc/llwebrtc_impl.h') diff --git a/indra/llwebrtc/llwebrtc_impl.h b/indra/llwebrtc/llwebrtc_impl.h index 3ccb6058ad..419482b751 100644 --- a/indra/llwebrtc/llwebrtc_impl.h +++ b/indra/llwebrtc/llwebrtc_impl.h @@ -65,27 +65,27 @@ namespace llwebrtc class LLWebRTCPeerConnectionImpl; -class LLAudioDeviceObserver : public webrtc::AudioDeviceDataObserver +class LLCustomProcessor : public webrtc::CustomProcessing { public: - LLAudioDeviceObserver(); + LLCustomProcessor(); + ~LLCustomProcessor() override {} - float getMicrophoneEnergy(); + // (Re-) Initializes the submodule. + void Initialize(int sample_rate_hz, int num_channels) override; + // Analyzes the given capture or render signal. + void Process(webrtc::AudioBuffer *audio) override; + // Returns a string representation of the module state. + std::string ToString() const override { return ""; } - void OnCaptureData(const void *audio_samples, - const size_t num_samples, - const size_t bytes_per_sample, - const size_t num_channels, - const uint32_t samples_per_sec) override; - - void OnRenderData(const void *audio_samples, - const size_t num_samples, - const size_t bytes_per_sample, - const size_t num_channels, - const uint32_t samples_per_sec) override; + float getMicrophoneEnergy() { return mMicrophoneEnergy; } protected: - float mSumVector[30]; // 300 ms of smoothing + static const int NUM_PACKETS_TO_FILTER = 30; // 300 ms of smoothing + int mSampleRateHz; + int mNumChannels; + + float mSumVector[NUM_PACKETS_TO_FILTER]; float mMicrophoneEnergy; }; @@ -93,7 +93,7 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, public webrtc::AudioDeviceS { public: LLWebRTCImpl() : - mTuningAudioDeviceObserver(nullptr), mPeerAudioDeviceObserver(nullptr), mMute(true) + mCustomProcessor(nullptr), mMute(true) { } ~LLWebRTCImpl() {} @@ -189,9 +189,8 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, public webrtc::AudioDeviceS int32_t mPlayoutDevice; int32_t mRecordingDevice; bool mMute; - - LLAudioDeviceObserver * mTuningAudioDeviceObserver; - LLAudioDeviceObserver * mPeerAudioDeviceObserver; + + LLCustomProcessor * mCustomProcessor; // peer connections std::vector> mPeerConnections; -- cgit v1.2.3 From e4c3ca53188152bc12a75f807eabef3dc5e9248b Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Mon, 5 Feb 2024 16:47:47 -0800 Subject: put observer-based tuning audio level calculation back --- indra/llwebrtc/llwebrtc_impl.h | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) (limited to 'indra/llwebrtc/llwebrtc_impl.h') diff --git a/indra/llwebrtc/llwebrtc_impl.h b/indra/llwebrtc/llwebrtc_impl.h index 419482b751..7146267379 100644 --- a/indra/llwebrtc/llwebrtc_impl.h +++ b/indra/llwebrtc/llwebrtc_impl.h @@ -65,6 +65,30 @@ namespace llwebrtc class LLWebRTCPeerConnectionImpl; +class LLAudioDeviceObserver : public webrtc::AudioDeviceDataObserver +{ + public: + LLAudioDeviceObserver(); + + float getMicrophoneEnergy(); + + void OnCaptureData(const void *audio_samples, + const size_t num_samples, + const size_t bytes_per_sample, + const size_t num_channels, + const uint32_t samples_per_sec) override; + + void OnRenderData(const void *audio_samples, + const size_t num_samples, + const size_t bytes_per_sample, + const size_t num_channels, + const uint32_t samples_per_sec) override; + + protected: + float mSumVector[30]; // 300 ms of smoothing + float mMicrophoneEnergy; +}; + class LLCustomProcessor : public webrtc::CustomProcessing { public: @@ -93,7 +117,7 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, public webrtc::AudioDeviceS { public: LLWebRTCImpl() : - mCustomProcessor(nullptr), mMute(true) + mPeerCustomProcessor(nullptr), mMute(true) { } ~LLWebRTCImpl() {} @@ -190,7 +214,8 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, public webrtc::AudioDeviceS int32_t mRecordingDevice; bool mMute; - LLCustomProcessor * mCustomProcessor; + LLAudioDeviceObserver * mTuningAudioDeviceObserver; + LLCustomProcessor * mPeerCustomProcessor; // peer connections std::vector> mPeerConnections; -- cgit v1.2.3 From 70044b9d2bbc594f0e8f3154feb2dbce77a7af82 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Tue, 5 Mar 2024 18:57:22 -0800 Subject: The response from the provision account call was being called twice for some reason --- indra/llwebrtc/llwebrtc_impl.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'indra/llwebrtc/llwebrtc_impl.h') diff --git a/indra/llwebrtc/llwebrtc_impl.h b/indra/llwebrtc/llwebrtc_impl.h index 7146267379..3c182c4b02 100644 --- a/indra/llwebrtc/llwebrtc_impl.h +++ b/indra/llwebrtc/llwebrtc_impl.h @@ -248,7 +248,7 @@ class LLWebRTCPeerConnectionImpl : public LLWebRTCPeerConnection, void setSignalingObserver(LLWebRTCSignalingObserver *observer) override; void unsetSignalingObserver(LLWebRTCSignalingObserver *observer) override; bool initializeConnection() override; - void shutdownConnection() override; + bool shutdownConnection() override; void AnswerAvailable(const std::string &sdp) override; // -- cgit v1.2.3 From 7714850fbe2ad4c2dc914798971d82cbb82f7832 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Sat, 9 Mar 2024 23:00:00 -0800 Subject: code beautification/comments --- indra/llwebrtc/llwebrtc_impl.h | 68 +++++++++++++++++++++++++++++++----------- 1 file changed, 50 insertions(+), 18 deletions(-) (limited to 'indra/llwebrtc/llwebrtc_impl.h') diff --git a/indra/llwebrtc/llwebrtc_impl.h b/indra/llwebrtc/llwebrtc_impl.h index 3c182c4b02..c2d0a4413e 100644 --- a/indra/llwebrtc/llwebrtc_impl.h +++ b/indra/llwebrtc/llwebrtc_impl.h @@ -1,6 +1,6 @@ /** * @file llwebrtc_impl.h - * @brief WebRTC interface implementation header + * @brief WebRTC dynamic library implementation header * * $LicenseInfo:firstyear=2023&license=viewerlgpl$ * Second Life Viewer Source Code @@ -65,19 +65,27 @@ namespace llwebrtc class LLWebRTCPeerConnectionImpl; + +// Implements a class allowing capture of audio data +// to determine audio level of the microphone. class LLAudioDeviceObserver : public webrtc::AudioDeviceDataObserver { public: LLAudioDeviceObserver(); + // Retrieve the RMS audio loudness float getMicrophoneEnergy(); + // Data retrieved from the caputure device is + // passed in here for processing. void OnCaptureData(const void *audio_samples, const size_t num_samples, const size_t bytes_per_sample, const size_t num_channels, const uint32_t samples_per_sec) override; + // This is for data destined for the render device. + // not currently used. void OnRenderData(const void *audio_samples, const size_t num_samples, const size_t bytes_per_sample, @@ -85,10 +93,13 @@ class LLAudioDeviceObserver : public webrtc::AudioDeviceDataObserver const uint32_t samples_per_sec) override; protected: - float mSumVector[30]; // 300 ms of smoothing + static const int NUM_PACKETS_TO_FILTER = 30; // 300 ms of smoothing (30 frames) + float mSumVector[NUM_PACKETS_TO_FILTER]; float mMicrophoneEnergy; }; +// Used to process/retrieve audio levels after +// all of the processing (AGC, AEC, etc.) for display in-world to the user. class LLCustomProcessor : public webrtc::CustomProcessing { public: @@ -97,8 +108,10 @@ class LLCustomProcessor : public webrtc::CustomProcessing // (Re-) Initializes the submodule. void Initialize(int sample_rate_hz, int num_channels) override; + // Analyzes the given capture or render signal. void Process(webrtc::AudioBuffer *audio) override; + // Returns a string representation of the module state. std::string ToString() const override { return ""; } @@ -113,13 +126,13 @@ class LLCustomProcessor : public webrtc::CustomProcessing float mMicrophoneEnergy; }; + +// Primary singleton implementation for interfacing +// with the native webrtc library. class LLWebRTCImpl : public LLWebRTCDeviceInterface, public webrtc::AudioDeviceSink { public: - LLWebRTCImpl() : - mPeerCustomProcessor(nullptr), mMute(true) - { - } + LLWebRTCImpl(); ~LLWebRTCImpl() {} void init(); @@ -139,7 +152,7 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, public webrtc::AudioDeviceS void setTuningMode(bool enable) override; float getTuningAudioLevel() override; - float getPeerAudioLevel() override; + float getPeerConnectionAudioLevel() override; // // AudioDeviceSink @@ -150,6 +163,9 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, public webrtc::AudioDeviceS // Helpers // + // The following thread helpers allow the + // LLWebRTCPeerConnectionImpl class to post + // tasks to the native webrtc threads. void PostWorkerTask(absl::AnyInvocable task, const webrtc::Location& location = webrtc::Location::Current()) { @@ -185,21 +201,31 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, public webrtc::AudioDeviceS { mNetworkThread->BlockingCall(std::move(functor), location); } - - rtc::scoped_refptr getPeerConnectionFactory() { return mPeerConnectionFactory; } - LLWebRTCPeerConnection * newPeerConnection(); - void freePeerConnection(LLWebRTCPeerConnection * peer_connection); + // Allows the LLWebRTCPeerConnectionImpl class to retrieve the + // native webrtc PeerConnectionFactory. + rtc::scoped_refptr getPeerConnectionFactory() + { + return mPeerConnectionFactory; + } + // create or destroy a peer connection. + LLWebRTCPeerConnectionInterface* newPeerConnection(); + void freePeerConnection(LLWebRTCPeerConnectionInterface* peer_connection); + + // enables/disables capture via the capture device void setRecording(bool recording); protected: - + // The native webrtc threads std::unique_ptr mNetworkThread; std::unique_ptr mWorkerThread; std::unique_ptr mSignalingThread; + + // The factory that allows creation of native webrtc PeerConnections. rtc::scoped_refptr mPeerConnectionFactory; - webrtc::PeerConnectionInterface::RTCConfiguration mConfiguration; + + // more native webrtc stuff std::unique_ptr mTaskQueueFactory; @@ -209,11 +235,11 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, public webrtc::AudioDeviceS rtc::scoped_refptr mPeerDeviceModule; std::vector mVoiceDevicesObserverList; - // accessors in webrtc aren't apparently implemented yet. + // accessors in native webrtc for devices aren't apparently implemented yet. int32_t mPlayoutDevice; int32_t mRecordingDevice; bool mMute; - + LLAudioDeviceObserver * mTuningAudioDeviceObserver; LLCustomProcessor * mPeerCustomProcessor; @@ -221,7 +247,11 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, public webrtc::AudioDeviceS std::vector> mPeerConnections; }; -class LLWebRTCPeerConnectionImpl : public LLWebRTCPeerConnection, + +// The implementation of a peer connection, which contains +// the various interfaces used by the viewer to interact with +// the webrtc connection. +class LLWebRTCPeerConnectionImpl : public LLWebRTCPeerConnectionInterface, public LLWebRTCAudioInterface, public LLWebRTCDataInterface, public webrtc::PeerConnectionObserver, @@ -232,7 +262,7 @@ class LLWebRTCPeerConnectionImpl : public LLWebRTCPeerConnection, { public: - LLWebRTCPeerConnectionImpl() {} + LLWebRTCPeerConnectionImpl(); ~LLWebRTCPeerConnectionImpl() {} void init(LLWebRTCImpl * webrtc_impl); @@ -270,7 +300,7 @@ class LLWebRTCPeerConnectionImpl : public LLWebRTCPeerConnection, // void OnSignalingChange(webrtc::PeerConnectionInterface::SignalingState new_state) override {} - void OnAddTrack(rtc::scoped_refptr receiver, + void OnAddTrack(rtc::scoped_refptr receiver, const std::vector> &streams) override; void OnRemoveTrack(rtc::scoped_refptr receiver) override; void OnDataChannel(rtc::scoped_refptr channel) override; @@ -311,6 +341,7 @@ class LLWebRTCPeerConnectionImpl : public LLWebRTCPeerConnection, protected: LLWebRTCImpl * mWebRTCImpl; + rtc::scoped_refptr mPeerConnectionFactory; bool mMute; @@ -323,6 +354,7 @@ class LLWebRTCPeerConnectionImpl : public LLWebRTCPeerConnection, rtc::scoped_refptr mPeerConnection; rtc::scoped_refptr mLocalStream; + // data std::vector mDataObserverList; rtc::scoped_refptr mDataChannel; }; -- cgit v1.2.3 From fc462b2b0f7ee153626d162bd97a05110f0804b7 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Sun, 10 Mar 2024 00:11:43 -0800 Subject: Remove trailing spaces. Other code cleanup. --- indra/llwebrtc/llwebrtc_impl.h | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) (limited to 'indra/llwebrtc/llwebrtc_impl.h') diff --git a/indra/llwebrtc/llwebrtc_impl.h b/indra/llwebrtc/llwebrtc_impl.h index c2d0a4413e..16afec061e 100644 --- a/indra/llwebrtc/llwebrtc_impl.h +++ b/indra/llwebrtc/llwebrtc_impl.h @@ -153,12 +153,12 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, public webrtc::AudioDeviceS void setTuningMode(bool enable) override; float getTuningAudioLevel() override; float getPeerConnectionAudioLevel() override; - + // // AudioDeviceSink // void OnDevicesUpdated() override; - + // // Helpers // @@ -171,7 +171,7 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, public webrtc::AudioDeviceS { mWorkerThread->PostTask(std::move(task), location); } - + void PostSignalingTask(absl::AnyInvocable task, const webrtc::Location& location = webrtc::Location::Current()) { @@ -189,7 +189,7 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, public webrtc::AudioDeviceS { mWorkerThread->BlockingCall(std::move(functor), location); } - + void SignalingBlockingCall(rtc::FunctionView functor, const webrtc::Location& location = webrtc::Location::Current()) { @@ -205,7 +205,7 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, public webrtc::AudioDeviceS // Allows the LLWebRTCPeerConnectionImpl class to retrieve the // native webrtc PeerConnectionFactory. rtc::scoped_refptr getPeerConnectionFactory() - { + { return mPeerConnectionFactory; } @@ -242,7 +242,7 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, public webrtc::AudioDeviceS LLAudioDeviceObserver * mTuningAudioDeviceObserver; LLCustomProcessor * mPeerCustomProcessor; - + // peer connections std::vector> mPeerConnections; }; @@ -270,7 +270,7 @@ class LLWebRTCPeerConnectionImpl : public LLWebRTCPeerConnectionInterface, virtual void AddRef() const override = 0; virtual rtc::RefCountReleaseStatus Release() const override = 0; - + // // LLWebRTCPeerConnection // @@ -326,20 +326,20 @@ class LLWebRTCPeerConnectionImpl : public LLWebRTCPeerConnectionInterface, // SetLocalDescriptionObserverInterface implementation. // void OnSetLocalDescriptionComplete(webrtc::RTCError error) override; - + // // DataChannelObserver implementation. // void OnStateChange() override; void OnMessage(const webrtc::DataBuffer& buffer) override; - + // Helpers void resetMute(); void enableSenderTracks(bool enable); void enableReceiverTracks(bool enable); protected: - + LLWebRTCImpl * mWebRTCImpl; rtc::scoped_refptr mPeerConnectionFactory; @@ -358,7 +358,7 @@ class LLWebRTCPeerConnectionImpl : public LLWebRTCPeerConnectionInterface, std::vector mDataObserverList; rtc::scoped_refptr mDataChannel; }; - + } #endif // LLWEBRTC_IMPL_H -- cgit v1.2.3