From bf58173656b94243df835d29f6d5a7c4467d4d1e Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Wed, 6 Sep 2023 15:28:02 -0700 Subject: Checkpoint WebRTC Voice --- indra/llwebrtc/CMakeLists.txt | 55 +++++ indra/llwebrtc/llwebrtc.cpp | 486 +++++++++++++++++++++++++++++++++++++++++ indra/llwebrtc/llwebrtc.h | 127 +++++++++++ indra/llwebrtc/llwebrtc_impl.h | 177 +++++++++++++++ 4 files changed, 845 insertions(+) create mode 100644 indra/llwebrtc/CMakeLists.txt create mode 100644 indra/llwebrtc/llwebrtc.cpp create mode 100644 indra/llwebrtc/llwebrtc.h create mode 100644 indra/llwebrtc/llwebrtc_impl.h (limited to 'indra/llwebrtc') diff --git a/indra/llwebrtc/CMakeLists.txt b/indra/llwebrtc/CMakeLists.txt new file mode 100644 index 0000000000..c0e2520a22 --- /dev/null +++ b/indra/llwebrtc/CMakeLists.txt @@ -0,0 +1,55 @@ +# -*- cmake -*- + +# some webrtc headers require C++ 20 +set(CMAKE_CXX_STANDARD 20) +set(CMAKE_CXX_STANDARD_REQUIRED ON) + + +set(CMAKE_GENERATOR_TOOLSET "clang_cl_x64") + +include(00-Common) +include(Linking) +include(WebRTC) + +project(llwebrtc) + +message(STATUS "C Compiler executable: ${CMAKE_C_COMPILER}") +message(STATUS "CXX Compiler executable: ${CMAKE_CXX_COMPILER}") +message(STATUS "Linker executable: ${CMAKE_LINKER}") +message(STATUS "SharedLib: ${SHARED_LIB_STAGING_DIR}") + +set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -stdlib=libc++ -lc++abi") + +set(llwebrtc_SOURCE_FILES + llwebrtc.cpp + ) + +set(llwebrtc_HEADER_FILES + CMakeLists.txt + llwebrtc.h + llwebrtc_impl.h + ) + +list(APPEND llwebrtc_SOURCE_FILES ${llwebrtc_HEADER_FILES}) + +add_library (llwebrtc SHARED ${llwebrtc_SOURCE_FILES}) + +set_target_properties(llwebrtc PROPERTIES PUBLIC_HEADER llwebrtc.h) + +target_link_libraries(llwebrtc PRIVATE ll::webrtc + secur32 + winmm + dmoguids + wmcodecdspuuid + msdmo + strmiids + iphlpapi) +target_include_directories( llwebrtc INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}) + +set_property(TARGET llwebrtc PROPERTY MSVC_RUNTIME_LIBRARY "MultiThreadedDebug") + +install(TARGETS llwebrtc RUNTIME DESTINATION "${CMAKE_BINARY_DIR}/sharedlibs/RelWithDebInfo") + +# Add tests +if (LL_TESTS) +endif (LL_TESTS) diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp new file mode 100644 index 0000000000..5e71a00b60 --- /dev/null +++ b/indra/llwebrtc/llwebrtc.cpp @@ -0,0 +1,486 @@ +/** + * @file llaccordionctrl.cpp + * @brief Accordion panel implementation + * + * $LicenseInfo:firstyear=2023&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2023, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include "llwebrtc_impl.h" +#include + +#include "api/audio_codecs/audio_decoder_factory.h" +#include "api/audio_codecs/audio_encoder_factory.h" +#include "api/audio_codecs/builtin_audio_decoder_factory.h" +#include "api/audio_codecs/builtin_audio_encoder_factory.h" +#include "api/media_stream_interface.h" +#include "api/media_stream_track.h" + +namespace llwebrtc +{ + +void LLWebRTCImpl::init() +{ + mAnswerReceived = false; + rtc::InitializeSSL(); + mTaskQueueFactory = webrtc::CreateDefaultTaskQueueFactory(); + + mNetworkThread = rtc::Thread::CreateWithSocketServer(); + mNetworkThread->SetName("WebRTCNetworkThread", nullptr); + mNetworkThread->Start(); + mWorkerThread = rtc::Thread::Create(); + mWorkerThread->SetName("WebRTCWorkerThread", nullptr); + mWorkerThread->Start(); + mSignalingThread = rtc::Thread::Create(); + mSignalingThread->SetName("WebRTCSignalingThread", nullptr); + mSignalingThread->Start(); + + mSignalingThread->PostTask( + [this]() + { + mDeviceModule = webrtc::CreateAudioDeviceWithDataObserver(webrtc::AudioDeviceModule::AudioLayer::kPlatformDefaultAudio, + mTaskQueueFactory.get(), + std::unique_ptr(this)); + mDeviceModule->Init(); + updateDevices(); + }); +} + +void LLWebRTCImpl::refreshDevices() +{ + mSignalingThread->PostTask([this]() { updateDevices(); }); +} + +void LLWebRTCImpl::setDevicesObserver(LLWebRTCDevicesObserver *observer) { mVoiceDevicesObserverList.emplace_back(observer); } + +void LLWebRTCImpl::unsetDevicesObserver(LLWebRTCDevicesObserver *observer) +{ + std::vector::iterator it = + std::find(mVoiceDevicesObserverList.begin(), mVoiceDevicesObserverList.end(), observer); + if (it != mVoiceDevicesObserverList.end()) + { + mVoiceDevicesObserverList.erase(it); + } +} + +void LLWebRTCImpl::setCaptureDevice(const std::string &id) +{ + mSignalingThread->PostTask( + [this, id]() + { + int16_t captureDeviceCount = mDeviceModule->RecordingDevices(); + for (int16_t index = 0; index < captureDeviceCount; index++) + { + char name[webrtc::kAdmMaxDeviceNameSize]; + char guid[webrtc::kAdmMaxGuidSize]; + mDeviceModule->RecordingDeviceName(index, name, guid); + if (id == guid || id == name) + { + mDeviceModule->SetRecordingDevice(index); + break; + } + } + }); +} + +void LLWebRTCImpl::setRenderDevice(const std::string &id) +{ + mSignalingThread->PostTask( + [this, id]() + { + int16_t renderDeviceCount = mDeviceModule->RecordingDevices(); + for (int16_t index = 0; index < renderDeviceCount; index++) + { + char name[webrtc::kAdmMaxDeviceNameSize]; + char guid[webrtc::kAdmMaxGuidSize]; + mDeviceModule->PlayoutDeviceName(index, name, guid); + if (id == guid || id == name) + { + mDeviceModule->SetPlayoutDevice(index); + break; + } + } + }); +} + +void LLWebRTCImpl::updateDevices() +{ + int16_t renderDeviceCount = mDeviceModule->PlayoutDevices(); + LLWebRTCVoiceDeviceList renderDeviceList; + for (int16_t index = 0; index < renderDeviceCount; index++) + { + char name[webrtc::kAdmMaxDeviceNameSize]; + char guid[webrtc::kAdmMaxGuidSize]; + mDeviceModule->PlayoutDeviceName(index, name, guid); + renderDeviceList.emplace_back(name, guid); + } + for (auto &observer : mVoiceDevicesObserverList) + { + observer->OnRenderDevicesChanged(renderDeviceList); + } + + int16_t captureDeviceCount = mDeviceModule->RecordingDevices(); + LLWebRTCVoiceDeviceList captureDeviceList; + for (int16_t index = 0; index < captureDeviceCount; index++) + { + char name[webrtc::kAdmMaxDeviceNameSize]; + char guid[webrtc::kAdmMaxGuidSize]; + mDeviceModule->RecordingDeviceName(index, name, guid); + captureDeviceList.emplace_back(name, guid); + } + for (auto &observer : mVoiceDevicesObserverList) + { + observer->OnCaptureDevicesChanged(captureDeviceList); + } +} + +void LLWebRTCImpl::setTuningMode(bool enable) +{ + mSignalingThread->PostTask( + [this, enable]() + { + if (enable) + { + mDeviceModule->InitMicrophone(); + mDeviceModule->InitRecording(); + mDeviceModule->StartRecording(); + mDeviceModule->SetMicrophoneMute(false); + } + else + { + mDeviceModule->StopRecording(); + } + }); +} + +double LLWebRTCImpl::getTuningMicrophoneEnergy() { return mTuningEnergy; } + +void LLWebRTCImpl::OnCaptureData(const void *audio_samples, + const size_t num_samples, + const size_t bytes_per_sample, + const size_t num_channels, + const uint32_t samples_per_sec) +{ + if (bytes_per_sample != 4) + { + return; + } + + double energy = 0; + const short *samples = (const short *) audio_samples; + for (size_t index = 0; index < num_samples * num_channels; index++) + { + double sample = (static_cast(samples[index]) / (double) 32768); + energy += sample * sample; + } + mTuningEnergy = std::sqrt(energy); +} + +void LLWebRTCImpl::OnRenderData(const void *audio_samples, + const size_t num_samples, + const size_t bytes_per_sample, + const size_t num_channels, + const uint32_t samples_per_sec) +{ +} + +// +// LLWebRTCSignalInterface +// + +void LLWebRTCImpl::setSignalingObserver(LLWebRTCSignalingObserver *observer) { mSignalingObserverList.emplace_back(observer); } + +void LLWebRTCImpl::unsetSignalingObserver(LLWebRTCSignalingObserver *observer) +{ + std::vector::iterator it = + std::find(mSignalingObserverList.begin(), mSignalingObserverList.end(), observer); + if (it != mSignalingObserverList.end()) + { + mSignalingObserverList.erase(it); + } +} + +bool LLWebRTCImpl::initializeConnection() +{ + RTC_DCHECK(!mPeerConnection); + RTC_DCHECK(!mPeerConnectionFactory); + mAnswerReceived = false; + mPeerConnectionFactory = webrtc::CreatePeerConnectionFactory(mNetworkThread.get(), + mWorkerThread.get(), + mSignalingThread.get(), + nullptr /* default_adm */, + webrtc::CreateBuiltinAudioEncoderFactory(), + webrtc::CreateBuiltinAudioDecoderFactory(), + nullptr /* video_encoder_factory */, + nullptr /* video_decoder_factory */, + nullptr /* audio_mixer */, + nullptr /* audio_processing */); + + if (!mPeerConnectionFactory) + { + shutdownConnection(); + return false; + } + + webrtc::PeerConnectionInterface::RTCConfiguration config; + config.sdp_semantics = webrtc::SdpSemantics::kUnifiedPlan; + webrtc::PeerConnectionInterface::IceServer server; + server.uri = "stun:stun.l.google.com:19302"; + // config.servers.push_back(server); + // server.uri = "stun:stun1.l.google.com:19302"; + // config.servers.push_back(server); + // server.uri = "stun:stun2.l.google.com:19302"; + // config.servers.push_back(server); + // server.uri = "stun:stun3.l.google.com:19302"; + // config.servers.push_back(server); + // server.uri = "stun:stun4.l.google.com:19302"; + // config.servers.push_back(server); + + webrtc::PeerConnectionDependencies pc_dependencies(this); + auto error_or_peer_connection = mPeerConnectionFactory->CreatePeerConnectionOrError(config, std::move(pc_dependencies)); + if (error_or_peer_connection.ok()) + { + mPeerConnection = std::move(error_or_peer_connection.value()); + } + else + { + shutdownConnection(); + return false; + } + + RTC_LOG(LS_INFO) << __FUNCTION__ << " " << mPeerConnection->signaling_state(); + + cricket::AudioOptions audioOptions; + audioOptions.auto_gain_control = true; + audioOptions.echo_cancellation = true; + audioOptions.noise_suppression = true; + + rtc::scoped_refptr stream = mPeerConnectionFactory->CreateLocalMediaStream("SLStream"); + rtc::scoped_refptr audio_track( + mPeerConnectionFactory->CreateAudioTrack("SLAudio", mPeerConnectionFactory->CreateAudioSource(cricket::AudioOptions()).get())); + audio_track->set_enabled(true); + stream->AddTrack(audio_track); + + mPeerConnection->AddTrack(audio_track, {"SLStream"}); + mPeerConnection->SetLocalDescription(rtc::scoped_refptr(this)); + + RTC_LOG(LS_INFO) << __FUNCTION__ << " " << mPeerConnection->signaling_state(); + + return mPeerConnection != nullptr; +} + +void LLWebRTCImpl::shutdownConnection() +{ + mPeerConnection = nullptr; + mPeerConnectionFactory = nullptr; +} + +void LLWebRTCImpl::AnswerAvailable(const std::string &sdp) +{ + mSignalingThread->PostTask( + [this, sdp]() + { + RTC_LOG(LS_INFO) << __FUNCTION__ << " " << mPeerConnection->peer_connection_state(); + mPeerConnection->SetRemoteDescription(webrtc::CreateSessionDescription(webrtc::SdpType::kAnswer, sdp), + rtc::scoped_refptr(this)); + mAnswerReceived = true; + for (auto &observer : mSignalingObserverList) + { + for (auto &candidate : mCachedIceCandidates) + { + LLWebRTCIceCandidate ice_candidate; + ice_candidate.candidate = candidate->candidate().ToString(); + ice_candidate.mline_index = candidate->sdp_mline_index(); + ice_candidate.sdp_mid = candidate->sdp_mid(); + observer->OnIceCandidate(ice_candidate); + } + mCachedIceCandidates.clear(); + if (mPeerConnection->ice_gathering_state() == webrtc::PeerConnectionInterface::IceGatheringState::kIceGatheringComplete) + { + for (auto &observer : mSignalingObserverList) + { + observer->OnIceGatheringState(llwebrtc::LLWebRTCSignalingObserver::IceGatheringState::ICE_GATHERING_COMPLETE); + } + } + } + }); +} + +void LLWebRTCImpl::setMute(bool mute) +{ + mSignalingThread->PostTask( + [this,mute]() + { + auto senders = mPeerConnection->GetSenders(); + + RTC_LOG(LS_INFO) << __FUNCTION__ << (mute ? "disabling" : "enabling") << " streams count " + << senders.size(); + + for (auto& sender : senders) + { + sender->track()->set_enabled(!mute); + } + }); +} + +// +// PeerConnectionObserver implementation. +// + +void LLWebRTCImpl::OnAddTrack(rtc::scoped_refptr receiver, + const std::vector> &streams) +{ + RTC_LOG(LS_INFO) << __FUNCTION__ << " " << receiver->id(); +} + +void LLWebRTCImpl::OnRemoveTrack(rtc::scoped_refptr receiver) +{ + RTC_LOG(LS_INFO) << __FUNCTION__ << " " << receiver->id(); +} + +void LLWebRTCImpl::OnIceGatheringChange(webrtc::PeerConnectionInterface::IceGatheringState new_state) +{ + LLWebRTCSignalingObserver::IceGatheringState webrtc_new_state = LLWebRTCSignalingObserver::IceGatheringState::ICE_GATHERING_NEW; + switch (new_state) + { + case webrtc::PeerConnectionInterface::IceGatheringState::kIceGatheringNew: + webrtc_new_state = LLWebRTCSignalingObserver::IceGatheringState::ICE_GATHERING_NEW; + break; + case webrtc::PeerConnectionInterface::IceGatheringState::kIceGatheringGathering: + webrtc_new_state = LLWebRTCSignalingObserver::IceGatheringState::ICE_GATHERING_GATHERING; + break; + case webrtc::PeerConnectionInterface::IceGatheringState::kIceGatheringComplete: + webrtc_new_state = LLWebRTCSignalingObserver::IceGatheringState::ICE_GATHERING_COMPLETE; + break; + default: + RTC_LOG(LS_ERROR) << __FUNCTION__ << " Bad Ice Gathering State" << new_state; + webrtc_new_state = LLWebRTCSignalingObserver::IceGatheringState::ICE_GATHERING_NEW; + return; + } + + if (mAnswerReceived) + { + for (auto &observer : mSignalingObserverList) + { + observer->OnIceGatheringState(webrtc_new_state); + } + } +} + +// Called any time the PeerConnectionState changes. +void LLWebRTCImpl::OnConnectionChange(webrtc::PeerConnectionInterface::PeerConnectionState new_state) +{ + RTC_LOG(LS_ERROR) << __FUNCTION__ << " Peer Connection State Change " << new_state; +} + +void LLWebRTCImpl::OnIceCandidate(const webrtc::IceCandidateInterface *candidate) +{ + RTC_LOG(LS_INFO) << __FUNCTION__ << " " << candidate->sdp_mline_index(); + + if (!candidate) + { + RTC_LOG(LS_ERROR) << __FUNCTION__ << " No Ice Candidate Given"; + return; + } + if (mAnswerReceived) + { + for (auto &observer : mSignalingObserverList) + { + LLWebRTCIceCandidate ice_candidate; + ice_candidate.candidate = candidate->candidate().ToString(); + ice_candidate.mline_index = candidate->sdp_mline_index(); + ice_candidate.sdp_mid = candidate->sdp_mid(); + observer->OnIceCandidate(ice_candidate); + } + } + else + { + mCachedIceCandidates.push_back( + webrtc::CreateIceCandidate(candidate->sdp_mid(), candidate->sdp_mline_index(), candidate->candidate())); + } +} + +// +// CreateSessionDescriptionObserver implementation. +// +void LLWebRTCImpl::OnSuccess(webrtc::SessionDescriptionInterface *desc) +{ + std::string sdp; + desc->ToString(&sdp); + RTC_LOG(LS_INFO) << sdp; + + RTC_LOG(LS_INFO) << __FUNCTION__ << " " << mPeerConnection->signaling_state(); + for (auto &observer : mSignalingObserverList) + { + observer->OnOfferAvailable(sdp); + } +} + +void LLWebRTCImpl::OnFailure(webrtc::RTCError error) { RTC_LOG(LS_ERROR) << ToString(error.type()) << ": " << error.message(); } + +// +// SetRemoteDescriptionObserverInterface implementation. +// +void LLWebRTCImpl::OnSetRemoteDescriptionComplete(webrtc::RTCError error) +{ + RTC_LOG(LS_INFO) << __FUNCTION__ << " " << mPeerConnection->signaling_state(); + if (!error.ok()) + { + RTC_LOG(LS_ERROR) << ToString(error.type()) << ": " << error.message(); + return; + } + for (auto &observer : mSignalingObserverList) + { + observer->OnAudioEstablished(this); + } +} + +// +// SetLocalDescriptionObserverInterface implementation. +// +void LLWebRTCImpl::OnSetLocalDescriptionComplete(webrtc::RTCError error) +{ + RTC_LOG(LS_INFO) << __FUNCTION__ << " " << mPeerConnection->signaling_state(); + if (!error.ok()) + { + RTC_LOG(LS_ERROR) << ToString(error.type()) << ": " << error.message(); + return; + } + auto desc = mPeerConnection->pending_local_description(); + std::string sdp; + desc->ToString(&sdp); + for (auto &observer : mSignalingObserverList) + { + observer->OnOfferAvailable(sdp); + } +} + +rtc::RefCountedObject *gWebRTCImpl = nullptr; +LLWebRTCDeviceInterface *getDeviceInterface() { return gWebRTCImpl; } +LLWebRTCSignalInterface *getSignalingInterface() { return gWebRTCImpl; } + +void init() +{ + gWebRTCImpl = new rtc::RefCountedObject(); + gWebRTCImpl->AddRef(); + gWebRTCImpl->init(); +} +} // namespace llwebrtc \ No newline at end of file diff --git a/indra/llwebrtc/llwebrtc.h b/indra/llwebrtc/llwebrtc.h new file mode 100644 index 0000000000..121efaab86 --- /dev/null +++ b/indra/llwebrtc/llwebrtc.h @@ -0,0 +1,127 @@ +/** + * @file llaccordionctrl.cpp + * @brief Accordion panel implementation + * + * $LicenseInfo:firstyear=2023&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2023, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef LLWEBRTC_H +#define LLWEBRTC_H + +#include +#include + +#ifdef LL_MAKEDLL +#ifdef WEBRTC_WIN +#define LLSYMEXPORT __declspec(dllexport) +#elif WEBRTC_LINUX +#define LLSYMEXPORT __attribute__((visibility("default"))) +#else +#define LLSYMEXPORT /**/ +#endif +#else +#define LLSYMEXPORT /**/ +#endif // LL_MAKEDLL + +namespace llwebrtc +{ +LLSYMEXPORT void init(); + +struct LLWebRTCIceCandidate +{ + std::string candidate; + std::string sdp_mid; + int mline_index; +}; + +class LLWebRTCVoiceDevice +{ + public: + std::string display_name; // friendly value for the user + std::string id; // internal value for selection + + LLWebRTCVoiceDevice(const std::string &display_name, const std::string &id) : + display_name(display_name), + id(id) {}; +}; + +typedef std::vector LLWebRTCVoiceDeviceList; + +class LLWebRTCDevicesObserver +{ + public: + virtual void OnRenderDevicesChanged(const LLWebRTCVoiceDeviceList &render_devices) = 0; + virtual void OnCaptureDevicesChanged(const LLWebRTCVoiceDeviceList &capture_devices) = 0; +}; + +class LLWebRTCDeviceInterface +{ + public: + + virtual void refreshDevices() = 0; + + virtual void setCaptureDevice(const std::string& id) = 0; + virtual void setRenderDevice(const std::string& id) = 0; + + virtual void setDevicesObserver(LLWebRTCDevicesObserver *observer) = 0; + virtual void unsetDevicesObserver(LLWebRTCDevicesObserver *observer) = 0; + + virtual void setTuningMode(bool enable) = 0; + virtual double getTuningMicrophoneEnergy() = 0; +}; + +class LLWebRTCAudioInterface +{ + public: + virtual void setMute(bool mute) = 0; +}; + +class LLWebRTCSignalingObserver +{ + public: + enum IceGatheringState{ + ICE_GATHERING_NEW, + ICE_GATHERING_GATHERING, + ICE_GATHERING_COMPLETE + }; + virtual void OnIceGatheringState(IceGatheringState state) = 0; + virtual void OnIceCandidate(const LLWebRTCIceCandidate& candidate) = 0; + virtual void OnOfferAvailable(const std::string& sdp) = 0; + virtual void OnAudioEstablished(LLWebRTCAudioInterface *audio_interface) = 0; +}; + +class LLWebRTCSignalInterface +{ + public: + virtual void setSignalingObserver(LLWebRTCSignalingObserver* observer) = 0; + virtual void unsetSignalingObserver(LLWebRTCSignalingObserver* observer) = 0; + + virtual bool initializeConnection() = 0; + virtual void shutdownConnection() = 0; + virtual void AnswerAvailable(const std::string &sdp) = 0; +}; + +LLSYMEXPORT LLWebRTCDeviceInterface* getDeviceInterface(); +LLSYMEXPORT LLWebRTCSignalInterface* getSignalingInterface(); +} + +#endif // LLWEBRTC_H \ No newline at end of file diff --git a/indra/llwebrtc/llwebrtc_impl.h b/indra/llwebrtc/llwebrtc_impl.h new file mode 100644 index 0000000000..10916e5a25 --- /dev/null +++ b/indra/llwebrtc/llwebrtc_impl.h @@ -0,0 +1,177 @@ +/** + * @file llaccordionctrl.cpp + * @brief Accordion panel implementation + * + * $LicenseInfo:firstyear=2023&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2023, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef LLWEBRTC_IMPL_H +#define LLWEBRTC_IMPL_H + +#define LL_MAKEDLL +#define WEBRTC_WIN 1 +#include "llwebrtc.h" +// WebRTC Includes +#ifdef WEBRTC_WIN +#pragma warning(disable : 4996) +#endif // WEBRTC_WIN +#include "api/scoped_refptr.h" +#include "rtc_base/ref_count.h" +#include "rtc_base/ref_counted_object.h" +#include "rtc_base/ssl_adapter.h" +#include "rtc_base/thread.h" +#include "api/peer_connection_interface.h" +#include "api/media_stream_interface.h" +#include "api/create_peerconnection_factory.h" +#include "modules/audio_device/include/audio_device.h" +#include "modules/audio_device/include/audio_device_data_observer.h" +#include "rtc_base/task_queue.h" +#include "api/task_queue/task_queue_factory.h" +#include "api/task_queue/default_task_queue_factory.h" +#include "modules/audio_device/include/audio_device_defines.h" + + +namespace llwebrtc +{ + +class LLWebRTCImpl : public LLWebRTCDeviceInterface, + public LLWebRTCSignalInterface, + public LLWebRTCAudioInterface, + public webrtc::AudioDeviceDataObserver, + public webrtc::PeerConnectionObserver, + public webrtc::CreateSessionDescriptionObserver, + public webrtc::SetRemoteDescriptionObserverInterface, + public webrtc::SetLocalDescriptionObserverInterface + +{ + public: + LLWebRTCImpl() : + mTuningEnergy(0.0) + { + } + ~LLWebRTCImpl() {} + + void init(); + + // + // LLWebRTCDeviceInterface + // + + void refreshDevices() override; + + void setDevicesObserver(LLWebRTCDevicesObserver *observer) override; + void unsetDevicesObserver(LLWebRTCDevicesObserver *observer) override; + void setCaptureDevice(const std::string& id) override; + + void setRenderDevice(const std::string& id) override; + + void setTuningMode(bool enable) override; + double getTuningMicrophoneEnergy() override; + + + void OnCaptureData(const void *audio_samples, + const size_t num_samples, + const size_t bytes_per_sample, + const size_t num_channels, + const uint32_t samples_per_sec) override; + + void OnRenderData(const void *audio_samples, + const size_t num_samples, + const size_t bytes_per_sample, + const size_t num_channels, + const uint32_t samples_per_sec) override; + + // + // LLWebRTCSignalInterface + // + + void setSignalingObserver(LLWebRTCSignalingObserver *observer) override; + void unsetSignalingObserver(LLWebRTCSignalingObserver *observer) override; + bool initializeConnection() override; + void shutdownConnection() override; + void AnswerAvailable(const std::string &sdp) override; + + + // + // LLWebRTCAudioInterface + // + void setMute(bool mute) override; + + // + // PeerConnectionObserver implementation. + // + + void OnSignalingChange(webrtc::PeerConnectionInterface::SignalingState new_state) override {} + void OnAddTrack(rtc::scoped_refptr receiver, + const std::vector> &streams) override; + void OnRemoveTrack(rtc::scoped_refptr receiver) override; + void OnDataChannel(rtc::scoped_refptr channel) override {} + void OnRenegotiationNeeded() override {} + void OnIceConnectionChange(webrtc::PeerConnectionInterface::IceConnectionState new_state) override {}; + void OnIceGatheringChange(webrtc::PeerConnectionInterface::IceGatheringState new_state) override; + void OnIceCandidate(const webrtc::IceCandidateInterface *candidate) override; + void OnIceConnectionReceivingChange(bool receiving) override {} + void OnConnectionChange(webrtc::PeerConnectionInterface::PeerConnectionState new_state) override; + + // + // CreateSessionDescriptionObserver implementation. + // + void OnSuccess(webrtc::SessionDescriptionInterface *desc) override; + void OnFailure(webrtc::RTCError error) override; + + // + // SetRemoteDescriptionObserverInterface implementation. + // + void OnSetRemoteDescriptionComplete(webrtc::RTCError error) override; + + // + // SetLocalDescriptionObserverInterface implementation. + // + void OnSetLocalDescriptionComplete(webrtc::RTCError error) override; + + protected: + std::unique_ptr mNetworkThread; + std::unique_ptr mWorkerThread; + std::unique_ptr mSignalingThread; + rtc::scoped_refptr mPeerConnectionFactory; + webrtc::PeerConnectionInterface::RTCConfiguration mConfiguration; + std::unique_ptr mTaskQueueFactory; + + + // Devices + void updateDevices(); + rtc::scoped_refptr mDeviceModule; + std::vector mVoiceDevicesObserverList; + + double mTuningEnergy; + + // signaling + std::vector mSignalingObserverList; + std::vector> mCachedIceCandidates; + bool mAnswerReceived; + + rtc::scoped_refptr mPeerConnection; +}; + +} + +#endif // LLWEBRTC_IMPL_H \ No newline at end of file -- cgit v1.2.3 From 0e46396429c1cd9b188d0493be2fef93ad145920 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Fri, 8 Sep 2023 21:36:17 -0700 Subject: Pull webrtc down from a webserver before building. --- indra/llwebrtc/CMakeLists.txt | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'indra/llwebrtc') diff --git a/indra/llwebrtc/CMakeLists.txt b/indra/llwebrtc/CMakeLists.txt index c0e2520a22..881a3af607 100644 --- a/indra/llwebrtc/CMakeLists.txt +++ b/indra/llwebrtc/CMakeLists.txt @@ -5,7 +5,7 @@ set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED ON) -set(CMAKE_GENERATOR_TOOLSET "clang_cl_x64") +#set(CMAKE_GENERATOR_TOOLSET "clang-cl") include(00-Common) include(Linking) @@ -20,6 +20,7 @@ message(STATUS "SharedLib: ${SHARED_LIB_STAGING_DIR}") set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -stdlib=libc++ -lc++abi") + set(llwebrtc_SOURCE_FILES llwebrtc.cpp ) @@ -46,7 +47,8 @@ target_link_libraries(llwebrtc PRIVATE ll::webrtc iphlpapi) target_include_directories( llwebrtc INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}) -set_property(TARGET llwebrtc PROPERTY MSVC_RUNTIME_LIBRARY "MultiThreadedDebug") +set_property(TARGET llwebrtc PROPERTY + MSVC_RUNTIME_LIBRARY "MultiThreadedDebug") install(TARGETS llwebrtc RUNTIME DESTINATION "${CMAKE_BINARY_DIR}/sharedlibs/RelWithDebInfo") -- cgit v1.2.3 From db7ab5d73e748f86ee00e31965265969c0c7060f Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Sat, 9 Sep 2023 22:19:22 -0700 Subject: Updates to build on mac. --- indra/llwebrtc/CMakeLists.txt | 34 ++++++++++++++++++++-------------- indra/llwebrtc/llwebrtc_impl.h | 11 ++++++++++- 2 files changed, 30 insertions(+), 15 deletions(-) (limited to 'indra/llwebrtc') diff --git a/indra/llwebrtc/CMakeLists.txt b/indra/llwebrtc/CMakeLists.txt index 881a3af607..fde4b87a3e 100644 --- a/indra/llwebrtc/CMakeLists.txt +++ b/indra/llwebrtc/CMakeLists.txt @@ -13,11 +13,6 @@ include(WebRTC) project(llwebrtc) -message(STATUS "C Compiler executable: ${CMAKE_C_COMPILER}") -message(STATUS "CXX Compiler executable: ${CMAKE_CXX_COMPILER}") -message(STATUS "Linker executable: ${CMAKE_LINKER}") -message(STATUS "SharedLib: ${SHARED_LIB_STAGING_DIR}") - set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -stdlib=libc++ -lc++abi") @@ -34,10 +29,11 @@ set(llwebrtc_HEADER_FILES list(APPEND llwebrtc_SOURCE_FILES ${llwebrtc_HEADER_FILES}) add_library (llwebrtc SHARED ${llwebrtc_SOURCE_FILES}) - + set_target_properties(llwebrtc PROPERTIES PUBLIC_HEADER llwebrtc.h) -target_link_libraries(llwebrtc PRIVATE ll::webrtc +if (WINDOWS) + target_link_libraries(llwebrtc PRIVATE ll::webrtc secur32 winmm dmoguids @@ -45,13 +41,23 @@ target_link_libraries(llwebrtc PRIVATE ll::webrtc msdmo strmiids iphlpapi) -target_include_directories( llwebrtc INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}) - -set_property(TARGET llwebrtc PROPERTY - MSVC_RUNTIME_LIBRARY "MultiThreadedDebug") - -install(TARGETS llwebrtc RUNTIME DESTINATION "${CMAKE_BINARY_DIR}/sharedlibs/RelWithDebInfo") - +elseif (DARWIN) + target_link_libraries(llwebrtc PRIVATE ll::webrtc) +elseif (LINUX) + target_link_libraries(llwebrtc PRIVATE ll::webrtc) +endif (WINDOWS) + +target_include_directories( llwebrtc INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}) + +if (WINDOWS) + set_property(TARGET llwebrtc PROPERTY + MSVC_RUNTIME_LIBRARY "MultiThreadedDebug") +endif (WINDOWS) + +ADD_CUSTOM_COMMAND(TARGET llwebrtc POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy_if_different + $ + ${SHARED_LIB_STAGING_DIR}) # Add tests if (LL_TESTS) endif (LL_TESTS) diff --git a/indra/llwebrtc/llwebrtc_impl.h b/indra/llwebrtc/llwebrtc_impl.h index 10916e5a25..07c0f514b2 100644 --- a/indra/llwebrtc/llwebrtc_impl.h +++ b/indra/llwebrtc/llwebrtc_impl.h @@ -28,12 +28,21 @@ #define LLWEBRTC_IMPL_H #define LL_MAKEDLL +#if defined(_WIN32) || defined(_WIN64) #define WEBRTC_WIN 1 +#elif defined(__APPLE__) +#define WEBRTC_MAC 1 +#define WEBRTC_POSIX 1 +#elif __linux__ +#define WEBRTC_LINUX 1 +#endif + #include "llwebrtc.h" // WebRTC Includes #ifdef WEBRTC_WIN #pragma warning(disable : 4996) #endif // WEBRTC_WIN + #include "api/scoped_refptr.h" #include "rtc_base/ref_count.h" #include "rtc_base/ref_counted_object.h" @@ -174,4 +183,4 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, } -#endif // LLWEBRTC_IMPL_H \ No newline at end of file +#endif // LLWEBRTC_IMPL_H -- cgit v1.2.3 From 781a7aabc10d21a951ac86b75df7ca7565c45039 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Sun, 10 Sep 2023 00:30:59 -0700 Subject: coding policy fixes --- indra/llwebrtc/llwebrtc.cpp | 2 +- indra/llwebrtc/llwebrtc.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'indra/llwebrtc') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index 5e71a00b60..e7a9072b2e 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -483,4 +483,4 @@ void init() gWebRTCImpl->AddRef(); gWebRTCImpl->init(); } -} // namespace llwebrtc \ No newline at end of file +} // namespace llwebrtc diff --git a/indra/llwebrtc/llwebrtc.h b/indra/llwebrtc/llwebrtc.h index 121efaab86..cb639ee116 100644 --- a/indra/llwebrtc/llwebrtc.h +++ b/indra/llwebrtc/llwebrtc.h @@ -124,4 +124,4 @@ LLSYMEXPORT LLWebRTCDeviceInterface* getDeviceInterface(); LLSYMEXPORT LLWebRTCSignalInterface* getSignalingInterface(); } -#endif // LLWEBRTC_H \ No newline at end of file +#endif // LLWEBRTC_H -- cgit v1.2.3 From 0cb1413d60dde3b975bf2ce541b17baf0927d8e0 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Sun, 10 Sep 2023 11:19:08 -0700 Subject: set toolset for llwebrtc --- indra/llwebrtc/CMakeLists.txt | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'indra/llwebrtc') diff --git a/indra/llwebrtc/CMakeLists.txt b/indra/llwebrtc/CMakeLists.txt index fde4b87a3e..a06e0aaa03 100644 --- a/indra/llwebrtc/CMakeLists.txt +++ b/indra/llwebrtc/CMakeLists.txt @@ -5,7 +5,7 @@ set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED ON) -#set(CMAKE_GENERATOR_TOOLSET "clang-cl") +set(CMAKE_GENERATOR_TOOLSET "CLangCL") include(00-Common) include(Linking) @@ -52,6 +52,10 @@ target_include_directories( llwebrtc INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}) if (WINDOWS) set_property(TARGET llwebrtc PROPERTY MSVC_RUNTIME_LIBRARY "MultiThreadedDebug") + ADD_CUSTOM_COMMAND(TARGET llwebrtc POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy_if_different + $ + ${SHARED_LIB_STAGING_DIR}) endif (WINDOWS) ADD_CUSTOM_COMMAND(TARGET llwebrtc POST_BUILD -- cgit v1.2.3 From b2d4ce16ad344162ecc190f798704e3ac179555a Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Sun, 10 Sep 2023 19:24:03 -0700 Subject: Fix windows pragma error --- indra/llwebrtc/llwebrtc_impl.h | 1 + 1 file changed, 1 insertion(+) (limited to 'indra/llwebrtc') diff --git a/indra/llwebrtc/llwebrtc_impl.h b/indra/llwebrtc/llwebrtc_impl.h index 07c0f514b2..d439bd253d 100644 --- a/indra/llwebrtc/llwebrtc_impl.h +++ b/indra/llwebrtc/llwebrtc_impl.h @@ -41,6 +41,7 @@ // WebRTC Includes #ifdef WEBRTC_WIN #pragma warning(disable : 4996) +#pragma warning(disable : 4068) #endif // WEBRTC_WIN #include "api/scoped_refptr.h" -- cgit v1.2.3 From ebfcb6f4e8cad8052ba2c924741a0098b572ed81 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Mon, 11 Sep 2023 15:38:58 -0700 Subject: some build tweaks --- indra/llwebrtc/CMakeLists.txt | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) (limited to 'indra/llwebrtc') diff --git a/indra/llwebrtc/CMakeLists.txt b/indra/llwebrtc/CMakeLists.txt index a06e0aaa03..f32d3dc580 100644 --- a/indra/llwebrtc/CMakeLists.txt +++ b/indra/llwebrtc/CMakeLists.txt @@ -5,7 +5,9 @@ set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED ON) -set(CMAKE_GENERATOR_TOOLSET "CLangCL") +if (WINDOWS) + set(CMAKE_GENERATOR_TOOLSET "CLangCL") +endif (WINDOWS) include(00-Common) include(Linking) @@ -52,10 +54,6 @@ target_include_directories( llwebrtc INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}) if (WINDOWS) set_property(TARGET llwebrtc PROPERTY MSVC_RUNTIME_LIBRARY "MultiThreadedDebug") - ADD_CUSTOM_COMMAND(TARGET llwebrtc POST_BUILD - COMMAND ${CMAKE_COMMAND} -E copy_if_different - $ - ${SHARED_LIB_STAGING_DIR}) endif (WINDOWS) ADD_CUSTOM_COMMAND(TARGET llwebrtc POST_BUILD -- cgit v1.2.3 From a974a1517901eb0a93099853a89bf55904737cec Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Tue, 12 Sep 2023 15:17:08 -0700 Subject: do some thread safety to prevent webrtc threads from conflicting with viewer threads. --- indra/llwebrtc/llwebrtc.cpp | 35 +++++++++++++++++++++++++++++++---- indra/llwebrtc/llwebrtc.h | 1 + 2 files changed, 32 insertions(+), 4 deletions(-) (limited to 'indra/llwebrtc') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index e7a9072b2e..c2631b2ea3 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -389,6 +389,36 @@ void LLWebRTCImpl::OnIceGatheringChange(webrtc::PeerConnectionInterface::IceGath void LLWebRTCImpl::OnConnectionChange(webrtc::PeerConnectionInterface::PeerConnectionState new_state) { RTC_LOG(LS_ERROR) << __FUNCTION__ << " Peer Connection State Change " << new_state; + + switch (new_state) + { + case webrtc::PeerConnectionInterface::PeerConnectionState::kConnected: + { + if (new_state == webrtc::PeerConnectionInterface::PeerConnectionState::kConnected) + { + for (auto &observer : mSignalingObserverList) + { + observer->OnAudioEstablished(this); + } + } + break; + } + case webrtc::PeerConnectionInterface::PeerConnectionState::kDisconnected: + { + if (new_state == webrtc::PeerConnectionInterface::PeerConnectionState::kConnected) + { + for (auto &observer : mSignalingObserverList) + { + observer->OnRenegotiationNeeded(); + } + } + break; + } + default: + { + break; + } + } } void LLWebRTCImpl::OnIceCandidate(const webrtc::IceCandidateInterface *candidate) @@ -447,10 +477,6 @@ void LLWebRTCImpl::OnSetRemoteDescriptionComplete(webrtc::RTCError error) RTC_LOG(LS_ERROR) << ToString(error.type()) << ": " << error.message(); return; } - for (auto &observer : mSignalingObserverList) - { - observer->OnAudioEstablished(this); - } } // @@ -467,6 +493,7 @@ void LLWebRTCImpl::OnSetLocalDescriptionComplete(webrtc::RTCError error) auto desc = mPeerConnection->pending_local_description(); std::string sdp; desc->ToString(&sdp); + RTC_LOG(LS_INFO) << __FUNCTION__ << " Local SDP: " << sdp; for (auto &observer : mSignalingObserverList) { observer->OnOfferAvailable(sdp); diff --git a/indra/llwebrtc/llwebrtc.h b/indra/llwebrtc/llwebrtc.h index cb639ee116..acc3665e95 100644 --- a/indra/llwebrtc/llwebrtc.h +++ b/indra/llwebrtc/llwebrtc.h @@ -106,6 +106,7 @@ class LLWebRTCSignalingObserver virtual void OnIceGatheringState(IceGatheringState state) = 0; virtual void OnIceCandidate(const LLWebRTCIceCandidate& candidate) = 0; virtual void OnOfferAvailable(const std::string& sdp) = 0; + virtual void OnRenegotiationNeeded() = 0; virtual void OnAudioEstablished(LLWebRTCAudioInterface *audio_interface) = 0; }; -- cgit v1.2.3 From 8ba90d077381bf16b4ba03a0f530f76e770e69c1 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Wed, 13 Sep 2023 14:49:42 -0700 Subject: Hook up speaker volume. --- indra/llwebrtc/llwebrtc.cpp | 22 ++++++++++++++++++++++ indra/llwebrtc/llwebrtc.h | 1 + indra/llwebrtc/llwebrtc_impl.h | 1 + 3 files changed, 24 insertions(+) (limited to 'indra/llwebrtc') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index c2631b2ea3..93e9db9c1d 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -341,6 +341,28 @@ void LLWebRTCImpl::setMute(bool mute) }); } +void LLWebRTCImpl::setSpeakerVolume(float volume) +{ + mSignalingThread->PostTask( + [this, volume]() + { + auto receivers = mPeerConnection->GetReceivers(); + + RTC_LOG(LS_INFO) << __FUNCTION__ << "Set volume" << receivers.size(); + for (auto &receiver : receivers) + { + webrtc::MediaStreamTrackInterface *track = receiver->track().get(); + if (track->kind() == webrtc::MediaStreamTrackInterface::kAudioKind) + { + webrtc::AudioTrackInterface* audio_track = static_cast(track); + webrtc::AudioSourceInterface* source = audio_track->GetSource(); + source->SetVolume(10.0 * volume); + + } + } + }); +} + // // PeerConnectionObserver implementation. // diff --git a/indra/llwebrtc/llwebrtc.h b/indra/llwebrtc/llwebrtc.h index acc3665e95..ca558add01 100644 --- a/indra/llwebrtc/llwebrtc.h +++ b/indra/llwebrtc/llwebrtc.h @@ -93,6 +93,7 @@ class LLWebRTCAudioInterface { public: virtual void setMute(bool mute) = 0; + virtual void setSpeakerVolume(float volume) = 0; // volume between 0.0 and 1.0 }; class LLWebRTCSignalingObserver diff --git a/indra/llwebrtc/llwebrtc_impl.h b/indra/llwebrtc/llwebrtc_impl.h index d439bd253d..5c6cfcdbc6 100644 --- a/indra/llwebrtc/llwebrtc_impl.h +++ b/indra/llwebrtc/llwebrtc_impl.h @@ -125,6 +125,7 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, // LLWebRTCAudioInterface // void setMute(bool mute) override; + void setSpeakerVolume(float folume) override; // range 0.0-1.0 // // PeerConnectionObserver implementation. -- cgit v1.2.3 From 0a844ef04703270bd20a1871206eb5b868d5f67b Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Wed, 13 Sep 2023 21:45:42 -0700 Subject: Fix connection failed logic to do a renegotiate. Also, remove some dead code. --- indra/llwebrtc/llwebrtc.cpp | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) (limited to 'indra/llwebrtc') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index 93e9db9c1d..862325c3f1 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -425,15 +425,13 @@ void LLWebRTCImpl::OnConnectionChange(webrtc::PeerConnectionInterface::PeerConne } break; } - case webrtc::PeerConnectionInterface::PeerConnectionState::kDisconnected: + case webrtc::PeerConnectionInterface::PeerConnectionState::kFailed: { - if (new_state == webrtc::PeerConnectionInterface::PeerConnectionState::kConnected) + for (auto &observer : mSignalingObserverList) { - for (auto &observer : mSignalingObserverList) - { - observer->OnRenegotiationNeeded(); - } + observer->OnRenegotiationNeeded(); } + break; } default: -- cgit v1.2.3 From b476e1545bb398bcda7a9dd693ac3fcc5ab4b7b8 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Wed, 13 Sep 2023 22:49:39 -0700 Subject: fix some retry logic and speaker volume logic --- indra/llwebrtc/llwebrtc.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'indra/llwebrtc') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index 862325c3f1..93981e4076 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -37,6 +37,8 @@ namespace llwebrtc { +const float VOLUME_SCALE_WEBRTC = 3.0f; + void LLWebRTCImpl::init() { mAnswerReceived = false; @@ -356,7 +358,7 @@ void LLWebRTCImpl::setSpeakerVolume(float volume) { webrtc::AudioTrackInterface* audio_track = static_cast(track); webrtc::AudioSourceInterface* source = audio_track->GetSource(); - source->SetVolume(10.0 * volume); + source->SetVolume(VOLUME_SCALE_WEBRTC * volume); } } -- cgit v1.2.3 From 162c38f1d56246eb382a80b6a99703a839082b2c Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Thu, 14 Sep 2023 20:48:33 -0700 Subject: Remove bad session from janus when negotation fails and is retried. --- indra/llwebrtc/llwebrtc.cpp | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) (limited to 'indra/llwebrtc') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index 93981e4076..edf941436e 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -246,15 +246,15 @@ bool LLWebRTCImpl::initializeConnection() config.sdp_semantics = webrtc::SdpSemantics::kUnifiedPlan; webrtc::PeerConnectionInterface::IceServer server; server.uri = "stun:stun.l.google.com:19302"; - // config.servers.push_back(server); - // server.uri = "stun:stun1.l.google.com:19302"; - // config.servers.push_back(server); - // server.uri = "stun:stun2.l.google.com:19302"; - // config.servers.push_back(server); - // server.uri = "stun:stun3.l.google.com:19302"; - // config.servers.push_back(server); - // server.uri = "stun:stun4.l.google.com:19302"; - // config.servers.push_back(server); + config.servers.push_back(server); + server.uri = "stun:stun1.l.google.com:19302"; + config.servers.push_back(server); + server.uri = "stun:stun2.l.google.com:19302"; + config.servers.push_back(server); + server.uri = "stun:stun3.l.google.com:19302"; + config.servers.push_back(server); + server.uri = "stun:stun4.l.google.com:19302"; + config.servers.push_back(server); webrtc::PeerConnectionDependencies pc_dependencies(this); auto error_or_peer_connection = mPeerConnectionFactory->CreatePeerConnectionOrError(config, std::move(pc_dependencies)); -- cgit v1.2.3 From 639e63faab239b88d41c8e2c755509e9dcdc6251 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Mon, 18 Sep 2023 11:25:47 -0700 Subject: Fix voice device settings --- indra/llwebrtc/llwebrtc.cpp | 94 ++++++++++++++++++++++++++++++++---------- indra/llwebrtc/llwebrtc_impl.h | 3 ++ 2 files changed, 76 insertions(+), 21 deletions(-) (limited to 'indra/llwebrtc') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index edf941436e..96be2c1f0b 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -55,7 +55,7 @@ void LLWebRTCImpl::init() mSignalingThread->SetName("WebRTCSignalingThread", nullptr); mSignalingThread->Start(); - mSignalingThread->PostTask( + mWorkerThread->PostTask( [this]() { mDeviceModule = webrtc::CreateAudioDeviceWithDataObserver(webrtc::AudioDeviceModule::AudioLayer::kPlatformDefaultAudio, @@ -68,7 +68,7 @@ void LLWebRTCImpl::init() void LLWebRTCImpl::refreshDevices() { - mSignalingThread->PostTask([this]() { updateDevices(); }); + mWorkerThread->PostTask([this]() { updateDevices(); }); } void LLWebRTCImpl::setDevicesObserver(LLWebRTCDevicesObserver *observer) { mVoiceDevicesObserverList.emplace_back(observer); } @@ -85,41 +85,48 @@ void LLWebRTCImpl::unsetDevicesObserver(LLWebRTCDevicesObserver *observer) void LLWebRTCImpl::setCaptureDevice(const std::string &id) { - mSignalingThread->PostTask( + mWorkerThread->PostTask( [this, id]() { + mDeviceModule->StopRecording(); int16_t captureDeviceCount = mDeviceModule->RecordingDevices(); for (int16_t index = 0; index < captureDeviceCount; index++) { char name[webrtc::kAdmMaxDeviceNameSize]; char guid[webrtc::kAdmMaxGuidSize]; mDeviceModule->RecordingDeviceName(index, name, guid); - if (id == guid || id == name) + if (id == guid || id == "Default") { + RTC_LOG(LS_INFO) << __FUNCTION__ << "Set recording device to " << name << " " << guid << " " << index; mDeviceModule->SetRecordingDevice(index); break; } } + mDeviceModule->InitRecording(); + mDeviceModule->StartRecording(); }); } void LLWebRTCImpl::setRenderDevice(const std::string &id) { - mSignalingThread->PostTask( + mWorkerThread->PostTask( [this, id]() { + mDeviceModule->StopPlayout(); int16_t renderDeviceCount = mDeviceModule->RecordingDevices(); for (int16_t index = 0; index < renderDeviceCount; index++) { char name[webrtc::kAdmMaxDeviceNameSize]; char guid[webrtc::kAdmMaxGuidSize]; mDeviceModule->PlayoutDeviceName(index, name, guid); - if (id == guid || id == name) + if (id == guid || id == "Default") { mDeviceModule->SetPlayoutDevice(index); break; } } + mDeviceModule->InitPlayout(); + mDeviceModule->StartPlayout(); }); } @@ -156,7 +163,7 @@ void LLWebRTCImpl::updateDevices() void LLWebRTCImpl::setTuningMode(bool enable) { - mSignalingThread->PostTask( + mWorkerThread->PostTask( [this, enable]() { if (enable) @@ -181,7 +188,7 @@ void LLWebRTCImpl::OnCaptureData(const void *audio_samples, const size_t num_channels, const uint32_t samples_per_sec) { - if (bytes_per_sample != 4) + if (bytes_per_sample != 2) { return; } @@ -220,28 +227,31 @@ void LLWebRTCImpl::unsetSignalingObserver(LLWebRTCSignalingObserver *observer) } } + bool LLWebRTCImpl::initializeConnection() { RTC_DCHECK(!mPeerConnection); - RTC_DCHECK(!mPeerConnectionFactory); + RTC_DCHECK(mPeerConnectionFactory); mAnswerReceived = false; + + mSignalingThread->PostTask([this]() { initializeConnectionThreaded(); }); + return true; +} + + + +bool LLWebRTCImpl::initializeConnectionThreaded() +{ mPeerConnectionFactory = webrtc::CreatePeerConnectionFactory(mNetworkThread.get(), mWorkerThread.get(), mSignalingThread.get(), - nullptr /* default_adm */, + mDeviceModule, webrtc::CreateBuiltinAudioEncoderFactory(), webrtc::CreateBuiltinAudioDecoderFactory(), nullptr /* video_encoder_factory */, nullptr /* video_decoder_factory */, nullptr /* audio_mixer */, nullptr /* audio_processing */); - - if (!mPeerConnectionFactory) - { - shutdownConnection(); - return false; - } - webrtc::PeerConnectionInterface::RTCConfiguration config; config.sdp_semantics = webrtc::SdpSemantics::kUnifiedPlan; webrtc::PeerConnectionInterface::IceServer server; @@ -272,16 +282,34 @@ bool LLWebRTCImpl::initializeConnection() cricket::AudioOptions audioOptions; audioOptions.auto_gain_control = true; - audioOptions.echo_cancellation = true; + audioOptions.echo_cancellation = false; // incompatible with opus stereo audioOptions.noise_suppression = true; rtc::scoped_refptr stream = mPeerConnectionFactory->CreateLocalMediaStream("SLStream"); rtc::scoped_refptr audio_track( - mPeerConnectionFactory->CreateAudioTrack("SLAudio", mPeerConnectionFactory->CreateAudioSource(cricket::AudioOptions()).get())); + mPeerConnectionFactory->CreateAudioTrack("SLAudio", mPeerConnectionFactory->CreateAudioSource(audioOptions).get())); audio_track->set_enabled(true); stream->AddTrack(audio_track); mPeerConnection->AddTrack(audio_track, {"SLStream"}); + + auto senders = mPeerConnection->GetSenders(); + + for (auto &sender : senders) + { + webrtc::RtpParameters params; + webrtc::RtpCodecParameters codecparam; + codecparam.name = "opus"; + codecparam.kind = cricket::MEDIA_TYPE_AUDIO; + codecparam.clock_rate = 48000; + codecparam.num_channels = 1; + codecparam.parameters["stereo"] = "0"; + codecparam.parameters["sprop-stereo"] = "0"; + + params.codecs.push_back(codecparam); + sender->SetParameters(params); + } + mPeerConnection->SetLocalDescription(rtc::scoped_refptr(this)); RTC_LOG(LS_INFO) << __FUNCTION__ << " " << mPeerConnection->signaling_state(); @@ -297,6 +325,12 @@ void LLWebRTCImpl::shutdownConnection() void LLWebRTCImpl::AnswerAvailable(const std::string &sdp) { + std::istringstream sdp_stream(sdp); + std::string sdp_line; + while (std::getline(sdp_stream, sdp_line)) + { + RTC_LOG(LS_INFO) << __FUNCTION__ << " Remote SDP: " << sdp_line; + } mSignalingThread->PostTask( [this, sdp]() { @@ -515,10 +549,28 @@ void LLWebRTCImpl::OnSetLocalDescriptionComplete(webrtc::RTCError error) auto desc = mPeerConnection->pending_local_description(); std::string sdp; desc->ToString(&sdp); - RTC_LOG(LS_INFO) << __FUNCTION__ << " Local SDP: " << sdp; + // mangle the sdp as this is the only way currently to bump up + // the send audio rate to 48k + std::istringstream sdp_stream(sdp); + std::ostringstream sdp_mangled_stream; + std::string sdp_line; + while (std::getline(sdp_stream, sdp_line)) { + int bandwidth = 0; + int payload_id = 0; + RTC_LOG(LS_INFO) << __FUNCTION__ << " Local SDP: " << sdp_line; + // force mono + if (std::sscanf(sdp_line.c_str(), "a=rtpmap:%i opus/%i/2", &payload_id, &bandwidth) == 2) + { + sdp_mangled_stream << sdp_line << "\n"; + } + else + { + sdp_mangled_stream << sdp_line << "\n"; + } + } for (auto &observer : mSignalingObserverList) { - observer->OnOfferAvailable(sdp); + observer->OnOfferAvailable(sdp_mangled_stream.str()); } } diff --git a/indra/llwebrtc/llwebrtc_impl.h b/indra/llwebrtc/llwebrtc_impl.h index 5c6cfcdbc6..3f9b06cae7 100644 --- a/indra/llwebrtc/llwebrtc_impl.h +++ b/indra/llwebrtc/llwebrtc_impl.h @@ -160,6 +160,9 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, void OnSetLocalDescriptionComplete(webrtc::RTCError error) override; protected: + + bool initializeConnectionThreaded(); + std::unique_ptr mNetworkThread; std::unique_ptr mWorkerThread; std::unique_ptr mSignalingThread; -- cgit v1.2.3 From 8859312b1f0d975793c6c2a3d7b23b9880c657c5 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Tue, 19 Sep 2023 10:14:29 -0700 Subject: add datachannel support --- indra/llwebrtc/llwebrtc.cpp | 50 +++++++++++++++++++++++++++++++++++++++++- indra/llwebrtc/llwebrtc.h | 16 ++++++++++++++ indra/llwebrtc/llwebrtc_impl.h | 20 ++++++++++++++++- 3 files changed, 84 insertions(+), 2 deletions(-) (limited to 'indra/llwebrtc') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index 96be2c1f0b..ac5870eab3 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -314,7 +314,22 @@ bool LLWebRTCImpl::initializeConnectionThreaded() RTC_LOG(LS_INFO) << __FUNCTION__ << " " << mPeerConnection->signaling_state(); - return mPeerConnection != nullptr; + webrtc::DataChannelInit init; + init.ordered = true; + init.reliable = true; + auto data_channel_or_error = mPeerConnection->CreateDataChannelOrError("SLData", &init); + if (data_channel_or_error.ok()) + { + mDataChannel = std::move(data_channel_or_error.value()); + } + else + { + shutdownConnection(); + return false; + } + mDataChannel->RegisterObserver(this); + + return true; } void LLWebRTCImpl::shutdownConnection() @@ -574,9 +589,42 @@ void LLWebRTCImpl::OnSetLocalDescriptionComplete(webrtc::RTCError error) } } +// +// DataChannelObserver implementation +// +void LLWebRTCImpl::OnMessage(const webrtc::DataBuffer& buffer) +{ + std::string data((const char*)buffer.data.cdata(), buffer.size()); + for (auto &observer : mDataObserverList) + { + observer->OnDataReceived(data, buffer.binary); + } +} + +void LLWebRTCImpl::sendData(const std::string& data, bool binary) +{ + rtc::CopyOnWriteBuffer cowBuffer(data.data(), data.length()); + webrtc::DataBuffer buffer(cowBuffer, binary); + mDataChannel->Send(buffer); +} + +void LLWebRTCImpl::setDataObserver(LLWebRTCDataObserver* observer) { mDataObserverList.emplace_back(observer); } + +void LLWebRTCImpl::unsetDataObserver(LLWebRTCDataObserver* observer) +{ + std::vector::iterator it = + std::find(mDataObserverList.begin(), mDataObserverList.end(), observer); + if (it != mDataObserverList.end()) + { + mDataObserverList.erase(it); + } +} + rtc::RefCountedObject *gWebRTCImpl = nullptr; LLWebRTCDeviceInterface *getDeviceInterface() { return gWebRTCImpl; } LLWebRTCSignalInterface *getSignalingInterface() { return gWebRTCImpl; } +LLWebRTCDataInterface *getDataInterface() { return gWebRTCImpl; } + void init() { diff --git a/indra/llwebrtc/llwebrtc.h b/indra/llwebrtc/llwebrtc.h index ca558add01..a6e754684e 100644 --- a/indra/llwebrtc/llwebrtc.h +++ b/indra/llwebrtc/llwebrtc.h @@ -96,6 +96,21 @@ class LLWebRTCAudioInterface virtual void setSpeakerVolume(float volume) = 0; // volume between 0.0 and 1.0 }; +class LLWebRTCDataObserver +{ +public: + virtual void OnDataReceived(const std::string& data, bool binary) = 0; +}; + +class LLWebRTCDataInterface +{ +public: + virtual void sendData(const std::string& data, bool binary=false) = 0; + + virtual void setDataObserver(LLWebRTCDataObserver *observer) = 0; + virtual void unsetDataObserver(LLWebRTCDataObserver *observer) = 0; +}; + class LLWebRTCSignalingObserver { public: @@ -124,6 +139,7 @@ class LLWebRTCSignalInterface LLSYMEXPORT LLWebRTCDeviceInterface* getDeviceInterface(); LLSYMEXPORT LLWebRTCSignalInterface* getSignalingInterface(); +LLSYMEXPORT LLWebRTCDataInterface* getDataInterface(); } #endif // LLWEBRTC_H diff --git a/indra/llwebrtc/llwebrtc_impl.h b/indra/llwebrtc/llwebrtc_impl.h index 3f9b06cae7..1ad117c7f3 100644 --- a/indra/llwebrtc/llwebrtc_impl.h +++ b/indra/llwebrtc/llwebrtc_impl.h @@ -66,11 +66,13 @@ namespace llwebrtc class LLWebRTCImpl : public LLWebRTCDeviceInterface, public LLWebRTCSignalInterface, public LLWebRTCAudioInterface, + public LLWebRTCDataInterface, public webrtc::AudioDeviceDataObserver, public webrtc::PeerConnectionObserver, public webrtc::CreateSessionDescriptionObserver, public webrtc::SetRemoteDescriptionObserverInterface, - public webrtc::SetLocalDescriptionObserverInterface + public webrtc::SetLocalDescriptionObserverInterface, + public webrtc::DataChannelObserver { public: @@ -126,6 +128,13 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, // void setMute(bool mute) override; void setSpeakerVolume(float folume) override; // range 0.0-1.0 + + // + // LLWebRTCDataInterface + // + void sendData(const std::string& data, bool binary=false) override; + void setDataObserver(LLWebRTCDataObserver *observer) override; + void unsetDataObserver(LLWebRTCDataObserver *observer) override; // // PeerConnectionObserver implementation. @@ -158,6 +167,12 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, // SetLocalDescriptionObserverInterface implementation. // void OnSetLocalDescriptionComplete(webrtc::RTCError error) override; + + // + // DataChannelObserver implementation. + // + void OnStateChange() override {} + void OnMessage(const webrtc::DataBuffer& buffer) override; protected: @@ -184,6 +199,9 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, bool mAnswerReceived; rtc::scoped_refptr mPeerConnection; + + std::vector mDataObserverList; + rtc::scoped_refptr mDataChannel; }; } -- cgit v1.2.3 From 1cd8f6f4f88f7717f0fcafbb5d47de0af59d5fb7 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Thu, 21 Sep 2023 15:28:58 -0700 Subject: Stream audio levels to and from viewers via DataChannels --- indra/llwebrtc/llwebrtc.cpp | 204 ++++++++++++++++++++++++++++++++--------- indra/llwebrtc/llwebrtc.h | 13 ++- indra/llwebrtc/llwebrtc_impl.h | 10 +- 3 files changed, 180 insertions(+), 47 deletions(-) (limited to 'indra/llwebrtc') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index ac5870eab3..77b050cbd0 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -62,10 +62,39 @@ void LLWebRTCImpl::init() mTaskQueueFactory.get(), std::unique_ptr(this)); mDeviceModule->Init(); + mDeviceModule->SetStereoRecording(false); + mDeviceModule->EnableBuiltInAEC(false); updateDevices(); }); } +void LLWebRTCImpl::terminate() +{ + mSignalingThread->BlockingCall( + [this]() + { + if (mPeerConnection) + { + mPeerConnection->Close(); + mPeerConnection = nullptr; + } + }); + mWorkerThread->BlockingCall( + [this]() + { + if (mDeviceModule) + { + mDeviceModule = nullptr; + } + }); + + mNetworkThread->Stop(); + mWorkerThread->Stop(); + mSignalingThread->Stop(); + +} + + void LLWebRTCImpl::refreshDevices() { mWorkerThread->PostTask([this]() { updateDevices(); }); @@ -88,22 +117,33 @@ void LLWebRTCImpl::setCaptureDevice(const std::string &id) mWorkerThread->PostTask( [this, id]() { - mDeviceModule->StopRecording(); + bool was_recording = mDeviceModule->Recording(); + + if (was_recording) + { + mDeviceModule->StopRecording(); + } int16_t captureDeviceCount = mDeviceModule->RecordingDevices(); - for (int16_t index = 0; index < captureDeviceCount; index++) + int16_t index = 0; /* default to first one if no match */ + for (int16_t i = 0; i < captureDeviceCount; i++) { char name[webrtc::kAdmMaxDeviceNameSize]; char guid[webrtc::kAdmMaxGuidSize]; - mDeviceModule->RecordingDeviceName(index, name, guid); + mDeviceModule->RecordingDeviceName(i, name, guid); if (id == guid || id == "Default") { - RTC_LOG(LS_INFO) << __FUNCTION__ << "Set recording device to " << name << " " << guid << " " << index; - mDeviceModule->SetRecordingDevice(index); + RTC_LOG(LS_INFO) << __FUNCTION__ << "Set recording device to " << name << " " << guid << " " << i; + index = i; break; } } + mDeviceModule->SetRecordingDevice(index); + mDeviceModule->InitMicrophone(); mDeviceModule->InitRecording(); - mDeviceModule->StartRecording(); + if (was_recording) + { + mDeviceModule->StartRecording(); + } }); } @@ -112,21 +152,32 @@ void LLWebRTCImpl::setRenderDevice(const std::string &id) mWorkerThread->PostTask( [this, id]() { - mDeviceModule->StopPlayout(); - int16_t renderDeviceCount = mDeviceModule->RecordingDevices(); - for (int16_t index = 0; index < renderDeviceCount; index++) + bool was_playing = mDeviceModule->Playing(); + if (was_playing) + { + mDeviceModule->StopPlayout(); + } + int16_t renderDeviceCount = mDeviceModule->PlayoutDevices(); + int16_t index = 0; /* default to first one if no match */ + for (int16_t i = 0; i < renderDeviceCount; i++) { char name[webrtc::kAdmMaxDeviceNameSize]; char guid[webrtc::kAdmMaxGuidSize]; - mDeviceModule->PlayoutDeviceName(index, name, guid); + mDeviceModule->PlayoutDeviceName(i, name, guid); if (id == guid || id == "Default") { - mDeviceModule->SetPlayoutDevice(index); + RTC_LOG(LS_INFO) << __FUNCTION__ << "Set recording device to " << name << " " << guid << " " << i; + index = i; break; } } + mDeviceModule->SetPlayoutDevice(index); + mDeviceModule->InitSpeaker(); mDeviceModule->InitPlayout(); - mDeviceModule->StartPlayout(); + if (was_playing) + { + mDeviceModule->StartPlayout(); + } }); } @@ -141,10 +192,6 @@ void LLWebRTCImpl::updateDevices() mDeviceModule->PlayoutDeviceName(index, name, guid); renderDeviceList.emplace_back(name, guid); } - for (auto &observer : mVoiceDevicesObserverList) - { - observer->OnRenderDevicesChanged(renderDeviceList); - } int16_t captureDeviceCount = mDeviceModule->RecordingDevices(); LLWebRTCVoiceDeviceList captureDeviceList; @@ -157,7 +204,7 @@ void LLWebRTCImpl::updateDevices() } for (auto &observer : mVoiceDevicesObserverList) { - observer->OnCaptureDevicesChanged(captureDeviceList); + observer->OnDevicesChanged(renderDeviceList, captureDeviceList); } } @@ -188,11 +235,6 @@ void LLWebRTCImpl::OnCaptureData(const void *audio_samples, const size_t num_channels, const uint32_t samples_per_sec) { - if (bytes_per_sample != 2) - { - return; - } - double energy = 0; const short *samples = (const short *) audio_samples; for (size_t index = 0; index < num_samples * num_channels; index++) @@ -242,6 +284,21 @@ bool LLWebRTCImpl::initializeConnection() bool LLWebRTCImpl::initializeConnectionThreaded() { + rtc::scoped_refptr apm = webrtc::AudioProcessingBuilder().Create(); + webrtc::AudioProcessing::Config apm_config; + apm_config.echo_canceller.enabled = false; + apm_config.echo_canceller.mobile_mode = false; + apm_config.gain_controller1.enabled = true; + apm_config.gain_controller1.mode = + webrtc::AudioProcessing::Config::GainController1::kAdaptiveAnalog; + apm_config.gain_controller2.enabled = true; + apm_config.high_pass_filter.enabled = true; + apm_config.noise_suppression.enabled = true; + apm_config.noise_suppression.level = webrtc::AudioProcessing::Config::NoiseSuppression::kVeryHigh; + apm_config.transient_suppression.enabled = true; + // + apm->ApplyConfig(apm_config); + mPeerConnectionFactory = webrtc::CreatePeerConnectionFactory(mNetworkThread.get(), mWorkerThread.get(), mSignalingThread.get(), @@ -251,7 +308,7 @@ bool LLWebRTCImpl::initializeConnectionThreaded() nullptr /* video_encoder_factory */, nullptr /* video_decoder_factory */, nullptr /* audio_mixer */, - nullptr /* audio_processing */); + apm); webrtc::PeerConnectionInterface::RTCConfiguration config; config.sdp_semantics = webrtc::SdpSemantics::kUnifiedPlan; webrtc::PeerConnectionInterface::IceServer server; @@ -278,6 +335,17 @@ bool LLWebRTCImpl::initializeConnectionThreaded() return false; } + webrtc::DataChannelInit init; + init.ordered = true; + + auto data_channel_or_error = mPeerConnection->CreateDataChannelOrError("SLData", &init); + if (data_channel_or_error.ok()) + { + mDataChannel = std::move(data_channel_or_error.value()); + + mDataChannel->RegisterObserver(this); + } + RTC_LOG(LS_INFO) << __FUNCTION__ << " " << mPeerConnection->signaling_state(); cricket::AudioOptions audioOptions; @@ -305,7 +373,6 @@ bool LLWebRTCImpl::initializeConnectionThreaded() codecparam.num_channels = 1; codecparam.parameters["stereo"] = "0"; codecparam.parameters["sprop-stereo"] = "0"; - params.codecs.push_back(codecparam); sender->SetParameters(params); } @@ -313,21 +380,6 @@ bool LLWebRTCImpl::initializeConnectionThreaded() mPeerConnection->SetLocalDescription(rtc::scoped_refptr(this)); RTC_LOG(LS_INFO) << __FUNCTION__ << " " << mPeerConnection->signaling_state(); - - webrtc::DataChannelInit init; - init.ordered = true; - init.reliable = true; - auto data_channel_or_error = mPeerConnection->CreateDataChannelOrError("SLData", &init); - if (data_channel_or_error.ok()) - { - mDataChannel = std::move(data_channel_or_error.value()); - } - else - { - shutdownConnection(); - return false; - } - mDataChannel->RegisterObserver(this); return true; } @@ -414,6 +466,18 @@ void LLWebRTCImpl::setSpeakerVolume(float volume) }); } +void LLWebRTCImpl::requestAudioLevel() +{ + mWorkerThread->PostTask( + [this]() + { + for (auto &observer : mAudioObserverList) + { + observer->OnAudioLevel((float)mTuningEnergy); + } + }); +} + // // PeerConnectionObserver implementation. // @@ -429,6 +493,13 @@ void LLWebRTCImpl::OnRemoveTrack(rtc::scoped_refptrid(); } +void LLWebRTCImpl::OnDataChannel(rtc::scoped_refptr channel) +{ + mDataChannel = channel; + channel->RegisterObserver(this); +} + + void LLWebRTCImpl::OnIceGatheringChange(webrtc::PeerConnectionInterface::IceGatheringState new_state) { LLWebRTCSignalingObserver::IceGatheringState webrtc_new_state = LLWebRTCSignalingObserver::IceGatheringState::ICE_GATHERING_NEW; @@ -469,10 +540,14 @@ void LLWebRTCImpl::OnConnectionChange(webrtc::PeerConnectionInterface::PeerConne { if (new_state == webrtc::PeerConnectionInterface::PeerConnectionState::kConnected) { - for (auto &observer : mSignalingObserverList) - { - observer->OnAudioEstablished(this); - } + mWorkerThread->PostTask([this]() { + mDeviceModule->StartRecording(); + mDeviceModule->StartPlayout(); + for (auto &observer : mSignalingObserverList) + { + observer->OnAudioEstablished(this); + } + }); } break; } @@ -589,9 +664,44 @@ void LLWebRTCImpl::OnSetLocalDescriptionComplete(webrtc::RTCError error) } } +void LLWebRTCImpl::setAudioObserver(LLWebRTCAudioObserver *observer) { mAudioObserverList.emplace_back(observer); } + +void LLWebRTCImpl::unsetAudioObserver(LLWebRTCAudioObserver *observer) +{ + std::vector::iterator it = std::find(mAudioObserverList.begin(), mAudioObserverList.end(), observer); + if (it != mAudioObserverList.end()) + { + mAudioObserverList.erase(it); + } +} + // // DataChannelObserver implementation // + +void LLWebRTCImpl::OnStateChange() +{ + RTC_LOG(LS_INFO) << __FUNCTION__ << " Data Channel State: " << webrtc::DataChannelInterface::DataStateString(mDataChannel->state()); + switch (mDataChannel->state()) + { + case webrtc::DataChannelInterface::kOpen: + RTC_LOG(LS_INFO) << __FUNCTION__ << " Data Channel State Open"; + break; + case webrtc::DataChannelInterface::kConnecting: + RTC_LOG(LS_INFO) << __FUNCTION__ << " Data Channel State Connecting"; + break; + case webrtc::DataChannelInterface::kClosing: + RTC_LOG(LS_INFO) << __FUNCTION__ << " Data Channel State closing"; + break; + case webrtc::DataChannelInterface::kClosed: + RTC_LOG(LS_INFO) << __FUNCTION__ << " Data Channel State closed"; + break; + default: + break; + } +} + + void LLWebRTCImpl::OnMessage(const webrtc::DataBuffer& buffer) { std::string data((const char*)buffer.data.cdata(), buffer.size()); @@ -632,4 +742,12 @@ void init() gWebRTCImpl->AddRef(); gWebRTCImpl->init(); } + +void terminate() +{ + gWebRTCImpl->terminate(); + gWebRTCImpl->Release(); + gWebRTCImpl = nullptr; +} + } // namespace llwebrtc diff --git a/indra/llwebrtc/llwebrtc.h b/indra/llwebrtc/llwebrtc.h index a6e754684e..f1ba1620e3 100644 --- a/indra/llwebrtc/llwebrtc.h +++ b/indra/llwebrtc/llwebrtc.h @@ -45,6 +45,7 @@ namespace llwebrtc { LLSYMEXPORT void init(); +LLSYMEXPORT void terminate(); struct LLWebRTCIceCandidate { @@ -69,8 +70,7 @@ typedef std::vector LLWebRTCVoiceDeviceList; class LLWebRTCDevicesObserver { public: - virtual void OnRenderDevicesChanged(const LLWebRTCVoiceDeviceList &render_devices) = 0; - virtual void OnCaptureDevicesChanged(const LLWebRTCVoiceDeviceList &capture_devices) = 0; + virtual void OnDevicesChanged(const LLWebRTCVoiceDeviceList &render_devices, const LLWebRTCVoiceDeviceList &capture_devices) = 0; }; class LLWebRTCDeviceInterface @@ -89,11 +89,20 @@ class LLWebRTCDeviceInterface virtual double getTuningMicrophoneEnergy() = 0; }; +class LLWebRTCAudioObserver +{ + public: + virtual void OnAudioLevel(float level) = 0; +}; + class LLWebRTCAudioInterface { public: + virtual void setAudioObserver(LLWebRTCAudioObserver *observer) = 0; + virtual void unsetAudioObserver(LLWebRTCAudioObserver *observer) = 0; virtual void setMute(bool mute) = 0; virtual void setSpeakerVolume(float volume) = 0; // volume between 0.0 and 1.0 + virtual void requestAudioLevel() = 0; }; class LLWebRTCDataObserver diff --git a/indra/llwebrtc/llwebrtc_impl.h b/indra/llwebrtc/llwebrtc_impl.h index 1ad117c7f3..1670d10705 100644 --- a/indra/llwebrtc/llwebrtc_impl.h +++ b/indra/llwebrtc/llwebrtc_impl.h @@ -83,6 +83,7 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, ~LLWebRTCImpl() {} void init(); + void terminate(); // // LLWebRTCDeviceInterface @@ -126,8 +127,11 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, // // LLWebRTCAudioInterface // + void setAudioObserver(LLWebRTCAudioObserver *observer) override; + void unsetAudioObserver(LLWebRTCAudioObserver *observer) override; void setMute(bool mute) override; void setSpeakerVolume(float folume) override; // range 0.0-1.0 + void requestAudioLevel() override; // // LLWebRTCDataInterface @@ -144,7 +148,7 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, void OnAddTrack(rtc::scoped_refptr receiver, const std::vector> &streams) override; void OnRemoveTrack(rtc::scoped_refptr receiver) override; - void OnDataChannel(rtc::scoped_refptr channel) override {} + void OnDataChannel(rtc::scoped_refptr channel) override; void OnRenegotiationNeeded() override {} void OnIceConnectionChange(webrtc::PeerConnectionInterface::IceConnectionState new_state) override {}; void OnIceGatheringChange(webrtc::PeerConnectionInterface::IceGatheringState new_state) override; @@ -171,7 +175,7 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, // // DataChannelObserver implementation. // - void OnStateChange() override {} + void OnStateChange() override; void OnMessage(const webrtc::DataBuffer& buffer) override; protected: @@ -200,6 +204,8 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, rtc::scoped_refptr mPeerConnection; + std::vector mAudioObserverList; + std::vector mDataObserverList; rtc::scoped_refptr mDataChannel; }; -- cgit v1.2.3 From dc56d0104d1c54fd2b9c6925cefbeac04eadcca8 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Thu, 21 Sep 2023 22:33:57 -0700 Subject: deal with add/remove of participants more effectively. --- indra/llwebrtc/llwebrtc.cpp | 5 ----- 1 file changed, 5 deletions(-) (limited to 'indra/llwebrtc') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index 77b050cbd0..1913786c0b 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -87,11 +87,6 @@ void LLWebRTCImpl::terminate() mDeviceModule = nullptr; } }); - - mNetworkThread->Stop(); - mWorkerThread->Stop(); - mSignalingThread->Stop(); - } -- cgit v1.2.3 From 4624184f97a94cd99df3640f5fc1e596c30bbffe Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Thu, 21 Sep 2023 23:23:49 -0700 Subject: send a message to the server when we're ready for data channel data --- indra/llwebrtc/llwebrtc.cpp | 17 +++++++++++++++-- indra/llwebrtc/llwebrtc.h | 1 + 2 files changed, 16 insertions(+), 2 deletions(-) (limited to 'indra/llwebrtc') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index 1913786c0b..dc746b7629 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -78,13 +78,22 @@ void LLWebRTCImpl::terminate() mPeerConnection->Close(); mPeerConnection = nullptr; } + mPeerConnectionFactory = nullptr; }); mWorkerThread->BlockingCall( [this]() { - if (mDeviceModule) + mDeviceModule = nullptr; + mTaskQueueFactory = nullptr; + + }); + mNetworkThread->BlockingCall( + [this]() + { + if (mDataChannel) { - mDeviceModule = nullptr; + mDataChannel->Close(); + mDataChannel = nullptr; } }); } @@ -681,6 +690,10 @@ void LLWebRTCImpl::OnStateChange() { case webrtc::DataChannelInterface::kOpen: RTC_LOG(LS_INFO) << __FUNCTION__ << " Data Channel State Open"; + for (auto &observer : mDataObserverList) + { + observer->OnDataChannelReady(); + } break; case webrtc::DataChannelInterface::kConnecting: RTC_LOG(LS_INFO) << __FUNCTION__ << " Data Channel State Connecting"; diff --git a/indra/llwebrtc/llwebrtc.h b/indra/llwebrtc/llwebrtc.h index f1ba1620e3..d0d2834e3c 100644 --- a/indra/llwebrtc/llwebrtc.h +++ b/indra/llwebrtc/llwebrtc.h @@ -109,6 +109,7 @@ class LLWebRTCDataObserver { public: virtual void OnDataReceived(const std::string& data, bool binary) = 0; + virtual void OnDataChannelReady() = 0; }; class LLWebRTCDataInterface -- cgit v1.2.3 From c087b8648dc85cffdc4d9a5eb85d989a9b4aa51b Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Fri, 22 Sep 2023 10:47:48 -0700 Subject: Fix shutdown crash issue. --- indra/llwebrtc/llwebrtc.cpp | 72 +++++++++++++++++++++--------------------- indra/llwebrtc/llwebrtc.h | 6 ++-- indra/llwebrtc/llwebrtc_impl.h | 42 +++++++++++++----------- 3 files changed, 62 insertions(+), 58 deletions(-) (limited to 'indra/llwebrtc') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index dc746b7629..a3dadd696c 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -39,6 +39,34 @@ namespace llwebrtc const float VOLUME_SCALE_WEBRTC = 3.0f; + +double LLAudioDeviceObserver::getMicrophoneEnergy() { return mMicrophoneEnergy; } + +void LLAudioDeviceObserver::OnCaptureData(const void *audio_samples, + const size_t num_samples, + const size_t bytes_per_sample, + const size_t num_channels, + const uint32_t samples_per_sec) +{ + double energy = 0; + const short *samples = (const short *) audio_samples; + for (size_t index = 0; index < num_samples * num_channels; index++) + { + double sample = (static_cast(samples[index]) / (double) 32768); + energy += sample * sample; + } + mMicrophoneEnergy = std::sqrt(energy); +} + +void LLAudioDeviceObserver::OnRenderData(const void *audio_samples, + const size_t num_samples, + const size_t bytes_per_sample, + const size_t num_channels, + const uint32_t samples_per_sec) +{ +} + + void LLWebRTCImpl::init() { mAnswerReceived = false; @@ -58,9 +86,10 @@ void LLWebRTCImpl::init() mWorkerThread->PostTask( [this]() { + mAudioDeviceObserver = new LLAudioDeviceObserver; mDeviceModule = webrtc::CreateAudioDeviceWithDataObserver(webrtc::AudioDeviceModule::AudioLayer::kPlatformDefaultAudio, mTaskQueueFactory.get(), - std::unique_ptr(this)); + std::unique_ptr(mAudioDeviceObserver)); mDeviceModule->Init(); mDeviceModule->SetStereoRecording(false); mDeviceModule->EnableBuiltInAEC(false); @@ -83,6 +112,10 @@ void LLWebRTCImpl::terminate() mWorkerThread->BlockingCall( [this]() { + if (mDeviceModule) + { + mDeviceModule->Terminate(); + } mDeviceModule = nullptr; mTaskQueueFactory = nullptr; @@ -231,32 +264,6 @@ void LLWebRTCImpl::setTuningMode(bool enable) }); } -double LLWebRTCImpl::getTuningMicrophoneEnergy() { return mTuningEnergy; } - -void LLWebRTCImpl::OnCaptureData(const void *audio_samples, - const size_t num_samples, - const size_t bytes_per_sample, - const size_t num_channels, - const uint32_t samples_per_sec) -{ - double energy = 0; - const short *samples = (const short *) audio_samples; - for (size_t index = 0; index < num_samples * num_channels; index++) - { - double sample = (static_cast(samples[index]) / (double) 32768); - energy += sample * sample; - } - mTuningEnergy = std::sqrt(energy); -} - -void LLWebRTCImpl::OnRenderData(const void *audio_samples, - const size_t num_samples, - const size_t bytes_per_sample, - const size_t num_channels, - const uint32_t samples_per_sec) -{ -} - // // LLWebRTCSignalInterface // @@ -470,16 +477,9 @@ void LLWebRTCImpl::setSpeakerVolume(float volume) }); } -void LLWebRTCImpl::requestAudioLevel() +double LLWebRTCImpl::getAudioLevel() { - mWorkerThread->PostTask( - [this]() - { - for (auto &observer : mAudioObserverList) - { - observer->OnAudioLevel((float)mTuningEnergy); - } - }); + return mAudioDeviceObserver->getMicrophoneEnergy(); } // diff --git a/indra/llwebrtc/llwebrtc.h b/indra/llwebrtc/llwebrtc.h index d0d2834e3c..30c0d63c55 100644 --- a/indra/llwebrtc/llwebrtc.h +++ b/indra/llwebrtc/llwebrtc.h @@ -86,13 +86,12 @@ class LLWebRTCDeviceInterface virtual void unsetDevicesObserver(LLWebRTCDevicesObserver *observer) = 0; virtual void setTuningMode(bool enable) = 0; - virtual double getTuningMicrophoneEnergy() = 0; + virtual double getAudioLevel() = 0; }; class LLWebRTCAudioObserver { public: - virtual void OnAudioLevel(float level) = 0; }; class LLWebRTCAudioInterface @@ -101,8 +100,7 @@ class LLWebRTCAudioInterface virtual void setAudioObserver(LLWebRTCAudioObserver *observer) = 0; virtual void unsetAudioObserver(LLWebRTCAudioObserver *observer) = 0; virtual void setMute(bool mute) = 0; - virtual void setSpeakerVolume(float volume) = 0; // volume between 0.0 and 1.0 - virtual void requestAudioLevel() = 0; + virtual void setSpeakerVolume(float volume) = 0; // volume between 0.0 and 1.0 = 0; }; class LLWebRTCDataObserver diff --git a/indra/llwebrtc/llwebrtc_impl.h b/indra/llwebrtc/llwebrtc_impl.h index 1670d10705..da829e52af 100644 --- a/indra/llwebrtc/llwebrtc_impl.h +++ b/indra/llwebrtc/llwebrtc_impl.h @@ -63,11 +63,31 @@ namespace llwebrtc { +class LLAudioDeviceObserver : public webrtc::AudioDeviceDataObserver +{ + public: + double getMicrophoneEnergy(); + + void OnCaptureData(const void *audio_samples, + const size_t num_samples, + const size_t bytes_per_sample, + const size_t num_channels, + const uint32_t samples_per_sec) override; + + void OnRenderData(const void *audio_samples, + const size_t num_samples, + const size_t bytes_per_sample, + const size_t num_channels, + const uint32_t samples_per_sec) override; + + protected: + double mMicrophoneEnergy; +}; + class LLWebRTCImpl : public LLWebRTCDeviceInterface, public LLWebRTCSignalInterface, public LLWebRTCAudioInterface, public LLWebRTCDataInterface, - public webrtc::AudioDeviceDataObserver, public webrtc::PeerConnectionObserver, public webrtc::CreateSessionDescriptionObserver, public webrtc::SetRemoteDescriptionObserverInterface, @@ -77,7 +97,7 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, { public: LLWebRTCImpl() : - mTuningEnergy(0.0) + mAudioDeviceObserver(nullptr) { } ~LLWebRTCImpl() {} @@ -98,20 +118,7 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, void setRenderDevice(const std::string& id) override; void setTuningMode(bool enable) override; - double getTuningMicrophoneEnergy() override; - - - void OnCaptureData(const void *audio_samples, - const size_t num_samples, - const size_t bytes_per_sample, - const size_t num_channels, - const uint32_t samples_per_sec) override; - - void OnRenderData(const void *audio_samples, - const size_t num_samples, - const size_t bytes_per_sample, - const size_t num_channels, - const uint32_t samples_per_sec) override; + double getAudioLevel() override; // // LLWebRTCSignalInterface @@ -131,7 +138,6 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, void unsetAudioObserver(LLWebRTCAudioObserver *observer) override; void setMute(bool mute) override; void setSpeakerVolume(float folume) override; // range 0.0-1.0 - void requestAudioLevel() override; // // LLWebRTCDataInterface @@ -195,7 +201,7 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, rtc::scoped_refptr mDeviceModule; std::vector mVoiceDevicesObserverList; - double mTuningEnergy; + LLAudioDeviceObserver * mAudioDeviceObserver; // signaling std::vector mSignalingObserverList; -- cgit v1.2.3 From 0110e37c7af96b02fd4ad92144b1b042e307d6a5 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Fri, 22 Sep 2023 16:55:13 -0700 Subject: Smooth voice power level reporting. --- indra/llwebrtc/llwebrtc.cpp | 21 +++++++++++++++++---- indra/llwebrtc/llwebrtc_impl.h | 5 ++++- 2 files changed, 21 insertions(+), 5 deletions(-) (limited to 'indra/llwebrtc') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index a3dadd696c..7b5ca7fb16 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -39,6 +39,7 @@ namespace llwebrtc const float VOLUME_SCALE_WEBRTC = 3.0f; +LLAudioDeviceObserver::LLAudioDeviceObserver() : mMicrophoneEnergy(0.0), mSumVector {0} {} double LLAudioDeviceObserver::getMicrophoneEnergy() { return mMicrophoneEnergy; } @@ -48,14 +49,26 @@ void LLAudioDeviceObserver::OnCaptureData(const void *audio_samples, const size_t num_channels, const uint32_t samples_per_sec) { - double energy = 0; + float energy = 0; const short *samples = (const short *) audio_samples; for (size_t index = 0; index < num_samples * num_channels; index++) { - double sample = (static_cast(samples[index]) / (double) 32768); + float sample = (static_cast(samples[index]) / (float) 32768); energy += sample * sample; } - mMicrophoneEnergy = std::sqrt(energy); + + // smooth it. + size_t buffer_size = sizeof(mSumVector) / sizeof(mSumVector[0]); + float totalSum = 0; + int i; + for (i = 0; i < (buffer_size - 1); i++) + { + mSumVector[i] = mSumVector[i + 1]; + totalSum += mSumVector[i]; + } + mSumVector[i] = energy; + totalSum += energy; + mMicrophoneEnergy = std::sqrt(totalSum / (num_samples * buffer_size)); } void LLAudioDeviceObserver::OnRenderData(const void *audio_samples, @@ -479,7 +492,7 @@ void LLWebRTCImpl::setSpeakerVolume(float volume) double LLWebRTCImpl::getAudioLevel() { - return mAudioDeviceObserver->getMicrophoneEnergy(); + return 20*mAudioDeviceObserver->getMicrophoneEnergy(); } // diff --git a/indra/llwebrtc/llwebrtc_impl.h b/indra/llwebrtc/llwebrtc_impl.h index da829e52af..70586f9013 100644 --- a/indra/llwebrtc/llwebrtc_impl.h +++ b/indra/llwebrtc/llwebrtc_impl.h @@ -66,6 +66,8 @@ namespace llwebrtc class LLAudioDeviceObserver : public webrtc::AudioDeviceDataObserver { public: + LLAudioDeviceObserver(); + double getMicrophoneEnergy(); void OnCaptureData(const void *audio_samples, @@ -81,7 +83,8 @@ class LLAudioDeviceObserver : public webrtc::AudioDeviceDataObserver const uint32_t samples_per_sec) override; protected: - double mMicrophoneEnergy; + float mSumVector[30]; // 300 ms of smoothing + float mMicrophoneEnergy; }; class LLWebRTCImpl : public LLWebRTCDeviceInterface, -- cgit v1.2.3 From f6becec955bc673c7e0014eb89197fef9128a405 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Tue, 26 Sep 2023 16:37:24 -0700 Subject: sdd stereo support to client --- indra/llwebrtc/llwebrtc.cpp | 43 ++++++++++++++++++++++++++++++++----------- 1 file changed, 32 insertions(+), 11 deletions(-) (limited to 'indra/llwebrtc') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index 7b5ca7fb16..e3fd68d35b 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -26,6 +26,7 @@ #include "llwebrtc_impl.h" #include +#include #include "api/audio_codecs/audio_decoder_factory.h" #include "api/audio_codecs/audio_encoder_factory.h" @@ -394,13 +395,28 @@ bool LLWebRTCImpl::initializeConnectionThreaded() codecparam.name = "opus"; codecparam.kind = cricket::MEDIA_TYPE_AUDIO; codecparam.clock_rate = 48000; - codecparam.num_channels = 1; - codecparam.parameters["stereo"] = "0"; - codecparam.parameters["sprop-stereo"] = "0"; + codecparam.num_channels = 2; + codecparam.parameters["stereo"] = "1"; + codecparam.parameters["sprop-stereo"] = "1"; params.codecs.push_back(codecparam); sender->SetParameters(params); } + auto receivers = mPeerConnection->GetReceivers(); + for (auto& receiver : receivers) + { + webrtc::RtpParameters params; + webrtc::RtpCodecParameters codecparam; + codecparam.name = "opus"; + codecparam.kind = cricket::MEDIA_TYPE_AUDIO; + codecparam.clock_rate = 48000; + codecparam.num_channels = 2; + codecparam.parameters["stereo"] = "1"; + codecparam.parameters["sprop-stereo"] = "1"; + params.codecs.push_back(codecparam); + receiver->SetParameters(params); + } + mPeerConnection->SetLocalDescription(rtc::scoped_refptr(this)); RTC_LOG(LS_INFO) << __FUNCTION__ << " " << mPeerConnection->signaling_state(); @@ -416,12 +432,9 @@ void LLWebRTCImpl::shutdownConnection() void LLWebRTCImpl::AnswerAvailable(const std::string &sdp) { - std::istringstream sdp_stream(sdp); - std::string sdp_line; - while (std::getline(sdp_stream, sdp_line)) - { - RTC_LOG(LS_INFO) << __FUNCTION__ << " Remote SDP: " << sdp_line; - } + + RTC_LOG(LS_INFO) << __FUNCTION__ << " Remote SDP: " << sdp; + mSignalingThread->PostTask( [this, sdp]() { @@ -661,20 +674,28 @@ void LLWebRTCImpl::OnSetLocalDescriptionComplete(webrtc::RTCError error) std::istringstream sdp_stream(sdp); std::ostringstream sdp_mangled_stream; std::string sdp_line; + int opus_payload = 0; while (std::getline(sdp_stream, sdp_line)) { int bandwidth = 0; int payload_id = 0; - RTC_LOG(LS_INFO) << __FUNCTION__ << " Local SDP: " << sdp_line; - // force mono + // force mono down, stereo up if (std::sscanf(sdp_line.c_str(), "a=rtpmap:%i opus/%i/2", &payload_id, &bandwidth) == 2) { sdp_mangled_stream << sdp_line << "\n"; + opus_payload = payload_id; + } + else if (sdp_line.rfind(std::format("a=fmtp:{}", opus_payload)) == 0) + { + sdp_mangled_stream << sdp_line << "a=fmtp:" << opus_payload + << " stereo=1;sprop-stereo=0;minptime=10;useinbandfec=1;maxplaybackrate=48000\n"; } else { sdp_mangled_stream << sdp_line << "\n"; } } + RTC_LOG(LS_INFO) << __FUNCTION__ << " Local SDP: " << sdp_mangled_stream.str(); + ; for (auto &observer : mSignalingObserverList) { observer->OnOfferAvailable(sdp_mangled_stream.str()); -- cgit v1.2.3 From f6f45bafc60f0f275d7e0e5aee81ea2610ac5352 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Tue, 26 Sep 2023 19:44:10 -0700 Subject: fix osx build incompatibility --- indra/llwebrtc/llwebrtc.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'indra/llwebrtc') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index e3fd68d35b..b02354cbac 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -674,7 +674,7 @@ void LLWebRTCImpl::OnSetLocalDescriptionComplete(webrtc::RTCError error) std::istringstream sdp_stream(sdp); std::ostringstream sdp_mangled_stream; std::string sdp_line; - int opus_payload = 0; + char opus_payload[10]; while (std::getline(sdp_stream, sdp_line)) { int bandwidth = 0; int payload_id = 0; @@ -682,9 +682,9 @@ void LLWebRTCImpl::OnSetLocalDescriptionComplete(webrtc::RTCError error) if (std::sscanf(sdp_line.c_str(), "a=rtpmap:%i opus/%i/2", &payload_id, &bandwidth) == 2) { sdp_mangled_stream << sdp_line << "\n"; - opus_payload = payload_id; + sprintf(opus_payload,"%d",payload_id); } - else if (sdp_line.rfind(std::format("a=fmtp:{}", opus_payload)) == 0) + else if (sdp_line.rfind(std::string("a=fmtp:") + opus_payload) == 0) { sdp_mangled_stream << sdp_line << "a=fmtp:" << opus_payload << " stereo=1;sprop-stereo=0;minptime=10;useinbandfec=1;maxplaybackrate=48000\n"; -- cgit v1.2.3 From 999fb768092f12fc89d85f532d4a9d2b7d311d10 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Wed, 27 Sep 2023 18:16:57 -0700 Subject: add stereo support --- indra/llwebrtc/llwebrtc.cpp | 87 +++++++++++++++++++++++++++++---------------- 1 file changed, 56 insertions(+), 31 deletions(-) (limited to 'indra/llwebrtc') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index b02354cbac..3152e1eef6 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -106,6 +106,7 @@ void LLWebRTCImpl::init() std::unique_ptr(mAudioDeviceObserver)); mDeviceModule->Init(); mDeviceModule->SetStereoRecording(false); + mDeviceModule->SetStereoPlayout(true); mDeviceModule->EnableBuiltInAEC(false); updateDevices(); }); @@ -321,6 +322,8 @@ bool LLWebRTCImpl::initializeConnectionThreaded() apm_config.noise_suppression.enabled = true; apm_config.noise_suppression.level = webrtc::AudioProcessing::Config::NoiseSuppression::kVeryHigh; apm_config.transient_suppression.enabled = true; + apm_config.pipeline.multi_channel_render = true; + apm_config.pipeline.multi_channel_capture = true; // apm->ApplyConfig(apm_config); @@ -416,8 +419,8 @@ bool LLWebRTCImpl::initializeConnectionThreaded() params.codecs.push_back(codecparam); receiver->SetParameters(params); } - - mPeerConnection->SetLocalDescription(rtc::scoped_refptr(this)); + webrtc::PeerConnectionInterface::RTCOfferAnswerOptions offerOptions; + mPeerConnection->CreateOffer(this, offerOptions); RTC_LOG(LS_INFO) << __FUNCTION__ << " " << mPeerConnection->signaling_state(); @@ -516,6 +519,16 @@ void LLWebRTCImpl::OnAddTrack(rtc::scoped_refptr const std::vector> &streams) { RTC_LOG(LS_INFO) << __FUNCTION__ << " " << receiver->id(); + webrtc::RtpParameters params; + webrtc::RtpCodecParameters codecparam; + codecparam.name = "opus"; + codecparam.kind = cricket::MEDIA_TYPE_AUDIO; + codecparam.clock_rate = 48000; + codecparam.num_channels = 2; + codecparam.parameters["stereo"] = "1"; + codecparam.parameters["sprop-stereo"] = "1"; + params.codecs.push_back(codecparam); + receiver->SetParameters(params); } void LLWebRTCImpl::OnRemoveTrack(rtc::scoped_refptr receiver) @@ -632,11 +645,47 @@ void LLWebRTCImpl::OnSuccess(webrtc::SessionDescriptionInterface *desc) std::string sdp; desc->ToString(&sdp); RTC_LOG(LS_INFO) << sdp; +; + // mangle the sdp as this is the only way currently to bump up + // the send audio rate to 48k + std::istringstream sdp_stream(sdp); + std::ostringstream sdp_mangled_stream; + std::string sdp_line; + int opus_payload = 0; + while (std::getline(sdp_stream, sdp_line)) + { + int bandwidth = 0; + int payload_id = 0; + // force mono down, stereo up + if (std::sscanf(sdp_line.c_str(), "a=rtpmap:%i opus/%i/2", &payload_id, &bandwidth) == 2) + { + sdp_mangled_stream << sdp_line << "\n"; + opus_payload = payload_id; + } + else if (sdp_line.rfind(std::format("a=fmtp:{}", opus_payload)) == 0) + { + sdp_mangled_stream << sdp_line << "a=fmtp:" << opus_payload + << " minptime=10;useinbandfec=1;stereo=1;sprop-stereo=1;maxplaybackrate=48000\n"; + } + else + { + sdp_mangled_stream << sdp_line << "\n"; + } + } + + webrtc::CreateSessionDescription(webrtc::SdpType::kOffer, sdp_mangled_stream.str()); + + + + mPeerConnection->SetLocalDescription(std::unique_ptr( + webrtc::CreateSessionDescription(webrtc::SdpType::kOffer, sdp_mangled_stream.str())), + rtc::scoped_refptr(this)); + RTC_LOG(LS_INFO) << __FUNCTION__ << " Local SDP: " << sdp_mangled_stream.str(); + - RTC_LOG(LS_INFO) << __FUNCTION__ << " " << mPeerConnection->signaling_state(); for (auto &observer : mSignalingObserverList) { - observer->OnOfferAvailable(sdp); + observer->OnOfferAvailable(sdp_mangled_stream.str()); } } @@ -669,36 +718,12 @@ void LLWebRTCImpl::OnSetLocalDescriptionComplete(webrtc::RTCError error) auto desc = mPeerConnection->pending_local_description(); std::string sdp; desc->ToString(&sdp); - // mangle the sdp as this is the only way currently to bump up - // the send audio rate to 48k - std::istringstream sdp_stream(sdp); - std::ostringstream sdp_mangled_stream; - std::string sdp_line; - char opus_payload[10]; - while (std::getline(sdp_stream, sdp_line)) { - int bandwidth = 0; - int payload_id = 0; - // force mono down, stereo up - if (std::sscanf(sdp_line.c_str(), "a=rtpmap:%i opus/%i/2", &payload_id, &bandwidth) == 2) - { - sdp_mangled_stream << sdp_line << "\n"; - sprintf(opus_payload,"%d",payload_id); - } - else if (sdp_line.rfind(std::string("a=fmtp:") + opus_payload) == 0) - { - sdp_mangled_stream << sdp_line << "a=fmtp:" << opus_payload - << " stereo=1;sprop-stereo=0;minptime=10;useinbandfec=1;maxplaybackrate=48000\n"; - } - else - { - sdp_mangled_stream << sdp_line << "\n"; - } - } - RTC_LOG(LS_INFO) << __FUNCTION__ << " Local SDP: " << sdp_mangled_stream.str(); + + RTC_LOG(LS_INFO) << __FUNCTION__ << " Local SDP: " << sdp; ; for (auto &observer : mSignalingObserverList) { - observer->OnOfferAvailable(sdp_mangled_stream.str()); + observer->OnOfferAvailable(sdp); } } -- cgit v1.2.3 From c5796c225dc98190bb2372780849822d85455618 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Wed, 27 Sep 2023 18:42:22 -0700 Subject: fix mac build --- indra/llwebrtc/llwebrtc.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'indra/llwebrtc') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index 3152e1eef6..481d5b940d 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -651,7 +651,7 @@ void LLWebRTCImpl::OnSuccess(webrtc::SessionDescriptionInterface *desc) std::istringstream sdp_stream(sdp); std::ostringstream sdp_mangled_stream; std::string sdp_line; - int opus_payload = 0; + std::string opus_payload; while (std::getline(sdp_stream, sdp_line)) { int bandwidth = 0; @@ -660,9 +660,9 @@ void LLWebRTCImpl::OnSuccess(webrtc::SessionDescriptionInterface *desc) if (std::sscanf(sdp_line.c_str(), "a=rtpmap:%i opus/%i/2", &payload_id, &bandwidth) == 2) { sdp_mangled_stream << sdp_line << "\n"; - opus_payload = payload_id; + opus_payload = std::to_string(payload_id); } - else if (sdp_line.rfind(std::format("a=fmtp:{}", opus_payload)) == 0) + else if (sdp_line.find("a=fmtp:" + opus_payload) == 0) { sdp_mangled_stream << sdp_line << "a=fmtp:" << opus_payload << " minptime=10;useinbandfec=1;stereo=1;sprop-stereo=1;maxplaybackrate=48000\n"; -- cgit v1.2.3 From 5872e037e801654d59f37e32f672db84c456d587 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Fri, 29 Sep 2023 19:13:03 -0700 Subject: Improve reconnection logic and allow device setting when connected or not connected --- indra/llwebrtc/llwebrtc.cpp | 268 +++++++++++++++++++++++++---------------- indra/llwebrtc/llwebrtc.h | 5 +- indra/llwebrtc/llwebrtc_impl.h | 17 ++- 3 files changed, 182 insertions(+), 108 deletions(-) (limited to 'indra/llwebrtc') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index 481d5b940d..95f87ed65b 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -42,15 +42,15 @@ const float VOLUME_SCALE_WEBRTC = 3.0f; LLAudioDeviceObserver::LLAudioDeviceObserver() : mMicrophoneEnergy(0.0), mSumVector {0} {} -double LLAudioDeviceObserver::getMicrophoneEnergy() { return mMicrophoneEnergy; } +float LLAudioDeviceObserver::getMicrophoneEnergy() { return mMicrophoneEnergy; } void LLAudioDeviceObserver::OnCaptureData(const void *audio_samples, - const size_t num_samples, - const size_t bytes_per_sample, - const size_t num_channels, - const uint32_t samples_per_sec) + const size_t num_samples, + const size_t bytes_per_sample, + const size_t num_channels, + const uint32_t samples_per_sec) { - float energy = 0; + float energy = 0; const short *samples = (const short *) audio_samples; for (size_t index = 0; index < num_samples * num_channels; index++) { @@ -60,8 +60,8 @@ void LLAudioDeviceObserver::OnCaptureData(const void *audio_samples, // smooth it. size_t buffer_size = sizeof(mSumVector) / sizeof(mSumVector[0]); - float totalSum = 0; - int i; + float totalSum = 0; + int i; for (i = 0; i < (buffer_size - 1); i++) { mSumVector[i] = mSumVector[i + 1]; @@ -73,16 +73,17 @@ void LLAudioDeviceObserver::OnCaptureData(const void *audio_samples, } void LLAudioDeviceObserver::OnRenderData(const void *audio_samples, - const size_t num_samples, - const size_t bytes_per_sample, - const size_t num_channels, - const uint32_t samples_per_sec) + const size_t num_samples, + const size_t bytes_per_sample, + const size_t num_channels, + const uint32_t samples_per_sec) { } - void LLWebRTCImpl::init() { + mPlayoutDevice = -1; + mRecordingDevice = -1; mAnswerReceived = false; rtc::InitializeSSL(); mTaskQueueFactory = webrtc::CreateDefaultTaskQueueFactory(); @@ -100,20 +101,21 @@ void LLWebRTCImpl::init() mWorkerThread->PostTask( [this]() { - mAudioDeviceObserver = new LLAudioDeviceObserver; - mDeviceModule = webrtc::CreateAudioDeviceWithDataObserver(webrtc::AudioDeviceModule::AudioLayer::kPlatformDefaultAudio, - mTaskQueueFactory.get(), - std::unique_ptr(mAudioDeviceObserver)); - mDeviceModule->Init(); - mDeviceModule->SetStereoRecording(false); - mDeviceModule->SetStereoPlayout(true); - mDeviceModule->EnableBuiltInAEC(false); + mTuningAudioDeviceObserver = new LLAudioDeviceObserver; + mTuningDeviceModule = + webrtc::CreateAudioDeviceWithDataObserver(webrtc::AudioDeviceModule::AudioLayer::kPlatformDefaultAudio, + mTaskQueueFactory.get(), + std::unique_ptr(mTuningAudioDeviceObserver)); + mTuningDeviceModule->Init(); + mTuningDeviceModule->SetStereoRecording(false); + mTuningDeviceModule->SetStereoPlayout(true); + mTuningDeviceModule->EnableBuiltInAEC(false); updateDevices(); }); } -void LLWebRTCImpl::terminate() -{ +void LLWebRTCImpl::terminate() +{ mSignalingThread->BlockingCall( [this]() { @@ -127,13 +129,17 @@ void LLWebRTCImpl::terminate() mWorkerThread->BlockingCall( [this]() { - if (mDeviceModule) + if (mTuningDeviceModule) { - mDeviceModule->Terminate(); + mTuningDeviceModule->Terminate(); } - mDeviceModule = nullptr; - mTaskQueueFactory = nullptr; - + if (mPeerDeviceModule) + { + mPeerDeviceModule->Terminate(); + } + mTuningDeviceModule = nullptr; + mPeerDeviceModule = nullptr; + mTaskQueueFactory = nullptr; }); mNetworkThread->BlockingCall( [this]() @@ -146,7 +152,6 @@ void LLWebRTCImpl::terminate() }); } - void LLWebRTCImpl::refreshDevices() { mWorkerThread->PostTask([this]() { updateDevices(); }); @@ -169,32 +174,50 @@ void LLWebRTCImpl::setCaptureDevice(const std::string &id) mWorkerThread->PostTask( [this, id]() { - bool was_recording = mDeviceModule->Recording(); + bool was_tuning_recording = mTuningDeviceModule->Recording(); - if (was_recording) + if (was_tuning_recording) { - mDeviceModule->StopRecording(); + mTuningDeviceModule->StopRecording(); } - int16_t captureDeviceCount = mDeviceModule->RecordingDevices(); - int16_t index = 0; /* default to first one if no match */ + bool was_peer_recording = false; + if (mPeerDeviceModule) + { + was_peer_recording = mPeerDeviceModule->Recording(); + if (was_peer_recording) + { + mPeerDeviceModule->StopRecording(); + } + } + int16_t captureDeviceCount = mTuningDeviceModule->RecordingDevices(); for (int16_t i = 0; i < captureDeviceCount; i++) { char name[webrtc::kAdmMaxDeviceNameSize]; char guid[webrtc::kAdmMaxGuidSize]; - mDeviceModule->RecordingDeviceName(i, name, guid); - if (id == guid || id == "Default") + mTuningDeviceModule->RecordingDeviceName(i, name, guid); + if (id == guid || id == "Default") // first one in list is default { RTC_LOG(LS_INFO) << __FUNCTION__ << "Set recording device to " << name << " " << guid << " " << i; - index = i; + mRecordingDevice = i; break; } } - mDeviceModule->SetRecordingDevice(index); - mDeviceModule->InitMicrophone(); - mDeviceModule->InitRecording(); - if (was_recording) + mTuningDeviceModule->SetRecordingDevice(mRecordingDevice); + mTuningDeviceModule->InitMicrophone(); + mTuningDeviceModule->InitRecording(); + if (was_tuning_recording) { - mDeviceModule->StartRecording(); + mTuningDeviceModule->StartRecording(); + } + if (mPeerDeviceModule) + { + mPeerDeviceModule->SetRecordingDevice(mRecordingDevice); + mPeerDeviceModule->InitMicrophone(); + mPeerDeviceModule->InitRecording(); + if (was_peer_recording) + { + mPeerDeviceModule->StartRecording(); + } } }); } @@ -204,54 +227,72 @@ void LLWebRTCImpl::setRenderDevice(const std::string &id) mWorkerThread->PostTask( [this, id]() { - bool was_playing = mDeviceModule->Playing(); - if (was_playing) + bool was_tuning_playing = mTuningDeviceModule->Playing(); + if (was_tuning_playing) { - mDeviceModule->StopPlayout(); + mTuningDeviceModule->StopPlayout(); + } + bool was_peer_playing = false; + if (mPeerDeviceModule) + { + was_peer_playing = mPeerDeviceModule->Playing(); + if (was_tuning_playing) + { + mPeerDeviceModule->StopPlayout(); + } } - int16_t renderDeviceCount = mDeviceModule->PlayoutDevices(); - int16_t index = 0; /* default to first one if no match */ + int16_t renderDeviceCount = mTuningDeviceModule->PlayoutDevices(); for (int16_t i = 0; i < renderDeviceCount; i++) { char name[webrtc::kAdmMaxDeviceNameSize]; char guid[webrtc::kAdmMaxGuidSize]; - mDeviceModule->PlayoutDeviceName(i, name, guid); + mTuningDeviceModule->PlayoutDeviceName(i, name, guid); if (id == guid || id == "Default") { RTC_LOG(LS_INFO) << __FUNCTION__ << "Set recording device to " << name << " " << guid << " " << i; - index = i; + mPlayoutDevice = i; break; } } - mDeviceModule->SetPlayoutDevice(index); - mDeviceModule->InitSpeaker(); - mDeviceModule->InitPlayout(); - if (was_playing) + mTuningDeviceModule->SetPlayoutDevice(mPlayoutDevice); + mTuningDeviceModule->InitSpeaker(); + mTuningDeviceModule->InitPlayout(); + if (was_tuning_playing) { - mDeviceModule->StartPlayout(); + mTuningDeviceModule->StartPlayout(); + } + if (mPeerDeviceModule) + { + mPeerDeviceModule->SetPlayoutDevice(mPlayoutDevice); + mPeerDeviceModule->InitSpeaker(); + mPeerDeviceModule->InitPlayout(); + if (was_peer_playing) + { + mPeerDeviceModule->StartPlayout(); + } } }); } void LLWebRTCImpl::updateDevices() { - int16_t renderDeviceCount = mDeviceModule->PlayoutDevices(); + int16_t renderDeviceCount = mTuningDeviceModule->PlayoutDevices(); LLWebRTCVoiceDeviceList renderDeviceList; for (int16_t index = 0; index < renderDeviceCount; index++) { char name[webrtc::kAdmMaxDeviceNameSize]; char guid[webrtc::kAdmMaxGuidSize]; - mDeviceModule->PlayoutDeviceName(index, name, guid); + mTuningDeviceModule->PlayoutDeviceName(index, name, guid); renderDeviceList.emplace_back(name, guid); } - int16_t captureDeviceCount = mDeviceModule->RecordingDevices(); + int16_t captureDeviceCount = mTuningDeviceModule->RecordingDevices(); LLWebRTCVoiceDeviceList captureDeviceList; for (int16_t index = 0; index < captureDeviceCount; index++) { char name[webrtc::kAdmMaxDeviceNameSize]; char guid[webrtc::kAdmMaxGuidSize]; - mDeviceModule->RecordingDeviceName(index, name, guid); + mTuningDeviceModule->RecordingDeviceName(index, name, guid); captureDeviceList.emplace_back(name, guid); } for (auto &observer : mVoiceDevicesObserverList) @@ -267,14 +308,22 @@ void LLWebRTCImpl::setTuningMode(bool enable) { if (enable) { - mDeviceModule->InitMicrophone(); - mDeviceModule->InitRecording(); - mDeviceModule->StartRecording(); - mDeviceModule->SetMicrophoneMute(false); + mTuningDeviceModule->StartRecording(); + mTuningDeviceModule->SetMicrophoneMute(false); + if (mPeerDeviceModule) + { + mPeerDeviceModule->StopRecording(); + mPeerDeviceModule->StopPlayout(); + } } else { - mDeviceModule->StopRecording(); + mTuningDeviceModule->StopRecording(); + if (mPeerDeviceModule) + { + mPeerDeviceModule->StartPlayout(); + mPeerDeviceModule->StartRecording(); + } } }); } @@ -295,42 +344,58 @@ void LLWebRTCImpl::unsetSignalingObserver(LLWebRTCSignalingObserver *observer) } } - bool LLWebRTCImpl::initializeConnection() { RTC_DCHECK(!mPeerConnection); RTC_DCHECK(mPeerConnectionFactory); - mAnswerReceived = false; + mAnswerReceived = false; mSignalingThread->PostTask([this]() { initializeConnectionThreaded(); }); return true; } - - bool LLWebRTCImpl::initializeConnectionThreaded() { rtc::scoped_refptr apm = webrtc::AudioProcessingBuilder().Create(); - webrtc::AudioProcessing::Config apm_config; - apm_config.echo_canceller.enabled = false; - apm_config.echo_canceller.mobile_mode = false; - apm_config.gain_controller1.enabled = true; - apm_config.gain_controller1.mode = - webrtc::AudioProcessing::Config::GainController1::kAdaptiveAnalog; - apm_config.gain_controller2.enabled = true; - apm_config.high_pass_filter.enabled = true; - apm_config.noise_suppression.enabled = true; - apm_config.noise_suppression.level = webrtc::AudioProcessing::Config::NoiseSuppression::kVeryHigh; - apm_config.transient_suppression.enabled = true; - apm_config.pipeline.multi_channel_render = true; + webrtc::AudioProcessing::Config apm_config; + apm_config.echo_canceller.enabled = false; + apm_config.echo_canceller.mobile_mode = false; + apm_config.gain_controller1.enabled = true; + apm_config.gain_controller1.mode = webrtc::AudioProcessing::Config::GainController1::kAdaptiveAnalog; + apm_config.gain_controller2.enabled = true; + apm_config.high_pass_filter.enabled = true; + apm_config.noise_suppression.enabled = true; + apm_config.noise_suppression.level = webrtc::AudioProcessing::Config::NoiseSuppression::kVeryHigh; + apm_config.transient_suppression.enabled = true; + apm_config.pipeline.multi_channel_render = true; apm_config.pipeline.multi_channel_capture = true; // apm->ApplyConfig(apm_config); + mWorkerThread->BlockingCall( + [this]() + { + mPeerAudioDeviceObserver = new LLAudioDeviceObserver; + mPeerDeviceModule = + webrtc::CreateAudioDeviceWithDataObserver(webrtc::AudioDeviceModule::AudioLayer::kPlatformDefaultAudio, + mTaskQueueFactory.get(), + std::unique_ptr(mPeerAudioDeviceObserver)); + mPeerDeviceModule->Init(); + mPeerDeviceModule->SetPlayoutDevice(mPlayoutDevice); + mPeerDeviceModule->SetRecordingDevice(mRecordingDevice); + mPeerDeviceModule->SetStereoRecording(false); + mPeerDeviceModule->SetStereoPlayout(true); + mPeerDeviceModule->EnableBuiltInAEC(false); + mPeerDeviceModule->InitMicrophone(); + mPeerDeviceModule->InitSpeaker(); + mPeerDeviceModule->InitRecording(); + mPeerDeviceModule->InitPlayout(); + }); + mPeerConnectionFactory = webrtc::CreatePeerConnectionFactory(mNetworkThread.get(), mWorkerThread.get(), mSignalingThread.get(), - mDeviceModule, + mPeerDeviceModule, webrtc::CreateBuiltinAudioEncoderFactory(), webrtc::CreateBuiltinAudioDecoderFactory(), nullptr /* video_encoder_factory */, @@ -406,9 +471,9 @@ bool LLWebRTCImpl::initializeConnectionThreaded() } auto receivers = mPeerConnection->GetReceivers(); - for (auto& receiver : receivers) + for (auto &receiver : receivers) { - webrtc::RtpParameters params; + webrtc::RtpParameters params; webrtc::RtpCodecParameters codecparam; codecparam.name = "opus"; codecparam.kind = cricket::MEDIA_TYPE_AUDIO; @@ -423,19 +488,23 @@ bool LLWebRTCImpl::initializeConnectionThreaded() mPeerConnection->CreateOffer(this, offerOptions); RTC_LOG(LS_INFO) << __FUNCTION__ << " " << mPeerConnection->signaling_state(); - + return true; } void LLWebRTCImpl::shutdownConnection() { + mDataChannel->Close(); + mDataChannel = nullptr; + mPeerConnection->Close(); + mPeerDeviceModule->Terminate(); + mPeerDeviceModule = nullptr; mPeerConnection = nullptr; mPeerConnectionFactory = nullptr; } void LLWebRTCImpl::AnswerAvailable(const std::string &sdp) { - RTC_LOG(LS_INFO) << __FUNCTION__ << " Remote SDP: " << sdp; mSignalingThread->PostTask( @@ -450,7 +519,7 @@ void LLWebRTCImpl::AnswerAvailable(const std::string &sdp) for (auto &candidate : mCachedIceCandidates) { LLWebRTCIceCandidate ice_candidate; - ice_candidate.candidate = candidate->candidate().ToString(); + ice_candidate.candidate = candidate->candidate().ToString(); ice_candidate.mline_index = candidate->sdp_mline_index(); ice_candidate.sdp_mid = candidate->sdp_mid(); observer->OnIceCandidate(ice_candidate); @@ -470,14 +539,13 @@ void LLWebRTCImpl::AnswerAvailable(const std::string &sdp) void LLWebRTCImpl::setMute(bool mute) { mSignalingThread->PostTask( - [this,mute]() + [this, mute]() { auto senders = mPeerConnection->GetSenders(); - RTC_LOG(LS_INFO) << __FUNCTION__ << (mute ? "disabling" : "enabling") << " streams count " - << senders.size(); + RTC_LOG(LS_INFO) << __FUNCTION__ << (mute ? "disabling" : "enabling") << " streams count " << senders.size(); - for (auto& sender : senders) + for (auto &sender : senders) { sender->track()->set_enabled(!mute); } @@ -497,19 +565,15 @@ void LLWebRTCImpl::setSpeakerVolume(float volume) webrtc::MediaStreamTrackInterface *track = receiver->track().get(); if (track->kind() == webrtc::MediaStreamTrackInterface::kAudioKind) { - webrtc::AudioTrackInterface* audio_track = static_cast(track); - webrtc::AudioSourceInterface* source = audio_track->GetSource(); + webrtc::AudioTrackInterface *audio_track = static_cast(track); + webrtc::AudioSourceInterface *source = audio_track->GetSource(); source->SetVolume(VOLUME_SCALE_WEBRTC * volume); - } } }); } -double LLWebRTCImpl::getAudioLevel() -{ - return 20*mAudioDeviceObserver->getMicrophoneEnergy(); -} +float LLWebRTCImpl::getTuningAudioLevel() { return 20 * mTuningAudioDeviceObserver->getMicrophoneEnergy(); } // // PeerConnectionObserver implementation. @@ -519,7 +583,7 @@ void LLWebRTCImpl::OnAddTrack(rtc::scoped_refptr const std::vector> &streams) { RTC_LOG(LS_INFO) << __FUNCTION__ << " " << receiver->id(); - webrtc::RtpParameters params; + webrtc::RtpParameters params; webrtc::RtpCodecParameters codecparam; codecparam.name = "opus"; codecparam.kind = cricket::MEDIA_TYPE_AUDIO; @@ -539,10 +603,9 @@ void LLWebRTCImpl::OnRemoveTrack(rtc::scoped_refptr channel) { mDataChannel = channel; - channel->RegisterObserver(this); + channel->RegisterObserver(this); } - void LLWebRTCImpl::OnIceGatheringChange(webrtc::PeerConnectionInterface::IceGatheringState new_state) { LLWebRTCSignalingObserver::IceGatheringState webrtc_new_state = LLWebRTCSignalingObserver::IceGatheringState::ICE_GATHERING_NEW; @@ -584,8 +647,8 @@ void LLWebRTCImpl::OnConnectionChange(webrtc::PeerConnectionInterface::PeerConne if (new_state == webrtc::PeerConnectionInterface::PeerConnectionState::kConnected) { mWorkerThread->PostTask([this]() { - mDeviceModule->StartRecording(); - mDeviceModule->StartPlayout(); + mPeerDeviceModule->StartRecording(); + mPeerDeviceModule->StartPlayout(); for (auto &observer : mSignalingObserverList) { observer->OnAudioEstablished(this); @@ -595,6 +658,7 @@ void LLWebRTCImpl::OnConnectionChange(webrtc::PeerConnectionInterface::PeerConne break; } case webrtc::PeerConnectionInterface::PeerConnectionState::kFailed: + case webrtc::PeerConnectionInterface::PeerConnectionState::kDisconnected: { for (auto &observer : mSignalingObserverList) { @@ -738,6 +802,8 @@ void LLWebRTCImpl::unsetAudioObserver(LLWebRTCAudioObserver *observer) } } +float LLWebRTCImpl::getAudioLevel() { return 20 * mPeerAudioDeviceObserver->getMicrophoneEnergy(); } + // // DataChannelObserver implementation // diff --git a/indra/llwebrtc/llwebrtc.h b/indra/llwebrtc/llwebrtc.h index 30c0d63c55..9224e88dfa 100644 --- a/indra/llwebrtc/llwebrtc.h +++ b/indra/llwebrtc/llwebrtc.h @@ -86,7 +86,7 @@ class LLWebRTCDeviceInterface virtual void unsetDevicesObserver(LLWebRTCDevicesObserver *observer) = 0; virtual void setTuningMode(bool enable) = 0; - virtual double getAudioLevel() = 0; + virtual float getTuningAudioLevel() = 0; }; class LLWebRTCAudioObserver @@ -100,7 +100,8 @@ class LLWebRTCAudioInterface virtual void setAudioObserver(LLWebRTCAudioObserver *observer) = 0; virtual void unsetAudioObserver(LLWebRTCAudioObserver *observer) = 0; virtual void setMute(bool mute) = 0; - virtual void setSpeakerVolume(float volume) = 0; // volume between 0.0 and 1.0 = 0; + virtual void setSpeakerVolume(float volume) = 0; // volume between 0.0 and 1.0 + virtual float getAudioLevel() = 0; }; class LLWebRTCDataObserver diff --git a/indra/llwebrtc/llwebrtc_impl.h b/indra/llwebrtc/llwebrtc_impl.h index 70586f9013..947b417923 100644 --- a/indra/llwebrtc/llwebrtc_impl.h +++ b/indra/llwebrtc/llwebrtc_impl.h @@ -68,7 +68,7 @@ class LLAudioDeviceObserver : public webrtc::AudioDeviceDataObserver public: LLAudioDeviceObserver(); - double getMicrophoneEnergy(); + float getMicrophoneEnergy(); void OnCaptureData(const void *audio_samples, const size_t num_samples, @@ -100,7 +100,7 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, { public: LLWebRTCImpl() : - mAudioDeviceObserver(nullptr) + mTuningAudioDeviceObserver(nullptr), mPeerAudioDeviceObserver(nullptr) { } ~LLWebRTCImpl() {} @@ -121,7 +121,7 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, void setRenderDevice(const std::string& id) override; void setTuningMode(bool enable) override; - double getAudioLevel() override; + float getTuningAudioLevel() override; // // LLWebRTCSignalInterface @@ -141,6 +141,7 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, void unsetAudioObserver(LLWebRTCAudioObserver *observer) override; void setMute(bool mute) override; void setSpeakerVolume(float folume) override; // range 0.0-1.0 + float getAudioLevel() override; // // LLWebRTCDataInterface @@ -201,10 +202,16 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, // Devices void updateDevices(); - rtc::scoped_refptr mDeviceModule; + rtc::scoped_refptr mTuningDeviceModule; + rtc::scoped_refptr mPeerDeviceModule; std::vector mVoiceDevicesObserverList; - LLAudioDeviceObserver * mAudioDeviceObserver; + // accessors in webrtc aren't apparently implemented yet. + int32_t mPlayoutDevice; + int32_t mRecordingDevice; + + LLAudioDeviceObserver * mTuningAudioDeviceObserver; + LLAudioDeviceObserver * mPeerAudioDeviceObserver; // signaling std::vector mSignalingObserverList; -- cgit v1.2.3 From 9d070201fe57e22db7cf55a2160a82e3c6413471 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Tue, 3 Oct 2023 15:51:37 -0700 Subject: fix device selection while speaking. --- indra/llwebrtc/llwebrtc.cpp | 107 +++++++++++++++++++++-------------------- indra/llwebrtc/llwebrtc_impl.h | 3 +- 2 files changed, 57 insertions(+), 53 deletions(-) (limited to 'indra/llwebrtc') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index 95f87ed65b..eda25a3641 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -131,10 +131,12 @@ void LLWebRTCImpl::terminate() { if (mTuningDeviceModule) { + mTuningDeviceModule->StopRecording(); mTuningDeviceModule->Terminate(); } if (mPeerDeviceModule) { + mPeerDeviceModule->StopRecording(); mPeerDeviceModule->Terminate(); } mTuningDeviceModule = nullptr; @@ -174,43 +176,32 @@ void LLWebRTCImpl::setCaptureDevice(const std::string &id) mWorkerThread->PostTask( [this, id]() { - bool was_tuning_recording = mTuningDeviceModule->Recording(); - - if (was_tuning_recording) - { - mTuningDeviceModule->StopRecording(); - } - bool was_peer_recording = false; - if (mPeerDeviceModule) - { - was_peer_recording = mPeerDeviceModule->Recording(); - if (was_peer_recording) - { - mPeerDeviceModule->StopRecording(); - } - } int16_t captureDeviceCount = mTuningDeviceModule->RecordingDevices(); for (int16_t i = 0; i < captureDeviceCount; i++) { char name[webrtc::kAdmMaxDeviceNameSize]; char guid[webrtc::kAdmMaxGuidSize]; mTuningDeviceModule->RecordingDeviceName(i, name, guid); - if (id == guid || id == "Default") // first one in list is default + if (id == guid || id == "Default") // first one in list is default { RTC_LOG(LS_INFO) << __FUNCTION__ << "Set recording device to " << name << " " << guid << " " << i; mRecordingDevice = i; break; } } + mTuningDeviceModule->StopRecording(); mTuningDeviceModule->SetRecordingDevice(mRecordingDevice); mTuningDeviceModule->InitMicrophone(); mTuningDeviceModule->InitRecording(); - if (was_tuning_recording) - { - mTuningDeviceModule->StartRecording(); - } + mTuningDeviceModule->StartRecording(); + bool was_peer_recording = false; if (mPeerDeviceModule) { + was_peer_recording = mPeerDeviceModule->Recording(); + if (was_peer_recording) + { + mPeerDeviceModule->StopRecording(); + } mPeerDeviceModule->SetRecordingDevice(mRecordingDevice); mPeerDeviceModule->InitMicrophone(); mPeerDeviceModule->InitRecording(); @@ -227,20 +218,6 @@ void LLWebRTCImpl::setRenderDevice(const std::string &id) mWorkerThread->PostTask( [this, id]() { - bool was_tuning_playing = mTuningDeviceModule->Playing(); - if (was_tuning_playing) - { - mTuningDeviceModule->StopPlayout(); - } - bool was_peer_playing = false; - if (mPeerDeviceModule) - { - was_peer_playing = mPeerDeviceModule->Playing(); - if (was_tuning_playing) - { - mPeerDeviceModule->StopPlayout(); - } - } int16_t renderDeviceCount = mTuningDeviceModule->PlayoutDevices(); for (int16_t i = 0; i < renderDeviceCount; i++) { @@ -254,6 +231,22 @@ void LLWebRTCImpl::setRenderDevice(const std::string &id) break; } } + mTuningDeviceModule->SetSpeakerMute(true); + bool was_tuning_playing = mTuningDeviceModule->Playing(); + if (was_tuning_playing) + { + mTuningDeviceModule->StopPlayout(); + } + bool was_peer_mute = false; + if (mPeerDeviceModule) + { + mPeerDeviceModule->SpeakerMute(&was_peer_mute); + if (!was_peer_mute) + { + mPeerDeviceModule->SetSpeakerMute(true); + } + } + mTuningDeviceModule->SetPlayoutDevice(mPlayoutDevice); mTuningDeviceModule->InitSpeaker(); mTuningDeviceModule->InitPlayout(); @@ -266,11 +259,11 @@ void LLWebRTCImpl::setRenderDevice(const std::string &id) mPeerDeviceModule->SetPlayoutDevice(mPlayoutDevice); mPeerDeviceModule->InitSpeaker(); mPeerDeviceModule->InitPlayout(); - if (was_peer_playing) - { - mPeerDeviceModule->StartPlayout(); - } + mPeerDeviceModule->StartPlayout(); + mPeerDeviceModule->SetSpeakerMute(was_peer_mute); + } + mTuningDeviceModule->SetSpeakerMute(false); }); } @@ -303,29 +296,42 @@ void LLWebRTCImpl::updateDevices() void LLWebRTCImpl::setTuningMode(bool enable) { - mWorkerThread->PostTask( + mWorkerThread->BlockingCall( [this, enable]() { if (enable) { + mTuningDeviceModule->StartRecording(); mTuningDeviceModule->SetMicrophoneMute(false); + + + mTuningDeviceModule->SetSpeakerMute(false); + if (mPeerDeviceModule) { mPeerDeviceModule->StopRecording(); - mPeerDeviceModule->StopPlayout(); + mPeerDeviceModule->SetSpeakerMute(true); } } else { - mTuningDeviceModule->StopRecording(); if (mPeerDeviceModule) { - mPeerDeviceModule->StartPlayout(); mPeerDeviceModule->StartRecording(); + mPeerDeviceModule->SetSpeakerMute(false); } } }); + // set_enabled shouldn't be done on the worker thread + if (mPeerConnection) + { + auto senders = mPeerConnection->GetSenders(); + for (auto &sender : senders) + { + sender->track()->set_enabled(enable ? false : !mMute); + } + } } // @@ -538,18 +544,15 @@ void LLWebRTCImpl::AnswerAvailable(const std::string &sdp) void LLWebRTCImpl::setMute(bool mute) { - mSignalingThread->PostTask( - [this, mute]() - { - auto senders = mPeerConnection->GetSenders(); + mMute = mute; + auto senders = mPeerConnection->GetSenders(); - RTC_LOG(LS_INFO) << __FUNCTION__ << (mute ? "disabling" : "enabling") << " streams count " << senders.size(); + RTC_LOG(LS_INFO) << __FUNCTION__ << (mute ? "disabling" : "enabling") << " streams count " << senders.size(); - for (auto &sender : senders) - { - sender->track()->set_enabled(!mute); - } - }); + for (auto &sender : senders) + { + sender->track()->set_enabled(!mMute); + } } void LLWebRTCImpl::setSpeakerVolume(float volume) diff --git a/indra/llwebrtc/llwebrtc_impl.h b/indra/llwebrtc/llwebrtc_impl.h index 947b417923..24edd6eaa2 100644 --- a/indra/llwebrtc/llwebrtc_impl.h +++ b/indra/llwebrtc/llwebrtc_impl.h @@ -100,7 +100,7 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, { public: LLWebRTCImpl() : - mTuningAudioDeviceObserver(nullptr), mPeerAudioDeviceObserver(nullptr) + mTuningAudioDeviceObserver(nullptr), mPeerAudioDeviceObserver(nullptr), mMute(true) { } ~LLWebRTCImpl() {} @@ -209,6 +209,7 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, // accessors in webrtc aren't apparently implemented yet. int32_t mPlayoutDevice; int32_t mRecordingDevice; + bool mMute; LLAudioDeviceObserver * mTuningAudioDeviceObserver; LLAudioDeviceObserver * mPeerAudioDeviceObserver; -- cgit v1.2.3 From c856a52477d0a50521a44f2d8255d5bde2ef39b4 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Fri, 6 Oct 2023 21:46:02 -0700 Subject: Fix race in initialization. Fix failure to send ice candidates to janus. --- indra/llwebrtc/llwebrtc.cpp | 95 ++++++++++++++++++++++++++---------------- indra/llwebrtc/llwebrtc_impl.h | 2 - 2 files changed, 58 insertions(+), 39 deletions(-) (limited to 'indra/llwebrtc') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index eda25a3641..d8f2a6028c 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -356,12 +356,6 @@ bool LLWebRTCImpl::initializeConnection() RTC_DCHECK(mPeerConnectionFactory); mAnswerReceived = false; - mSignalingThread->PostTask([this]() { initializeConnectionThreaded(); }); - return true; -} - -bool LLWebRTCImpl::initializeConnectionThreaded() -{ rtc::scoped_refptr apm = webrtc::AudioProcessingBuilder().Create(); webrtc::AudioProcessing::Config apm_config; apm_config.echo_canceller.enabled = false; @@ -375,8 +369,6 @@ bool LLWebRTCImpl::initializeConnectionThreaded() apm_config.transient_suppression.enabled = true; apm_config.pipeline.multi_channel_render = true; apm_config.pipeline.multi_channel_capture = true; - // - apm->ApplyConfig(apm_config); mWorkerThread->BlockingCall( [this]() @@ -396,6 +388,8 @@ bool LLWebRTCImpl::initializeConnectionThreaded() mPeerDeviceModule->InitSpeaker(); mPeerDeviceModule->InitRecording(); mPeerDeviceModule->InitPlayout(); + mPeerDeviceModule->StopPlayout(); + mPeerDeviceModule->StopRecording(); }); mPeerConnectionFactory = webrtc::CreatePeerConnectionFactory(mNetworkThread.get(), @@ -408,9 +402,13 @@ bool LLWebRTCImpl::initializeConnectionThreaded() nullptr /* video_decoder_factory */, nullptr /* audio_mixer */, apm); + apm->ApplyConfig(apm_config); + webrtc::PeerConnectionInterface::RTCConfiguration config; config.sdp_semantics = webrtc::SdpSemantics::kUnifiedPlan; webrtc::PeerConnectionInterface::IceServer server; + server.uri = "stun:roxie-turn.staging.secondlife.io:3478"; + config.servers.push_back(server); server.uri = "stun:stun.l.google.com:19302"; config.servers.push_back(server); server.uri = "stun:stun1.l.google.com:19302"; @@ -430,6 +428,7 @@ bool LLWebRTCImpl::initializeConnectionThreaded() } else { + RTC_LOG(LS_ERROR) << __FUNCTION__ << "Error creating peer connection: " << error_or_peer_connection.error().message(); shutdownConnection(); return false; } @@ -493,20 +492,47 @@ bool LLWebRTCImpl::initializeConnectionThreaded() webrtc::PeerConnectionInterface::RTCOfferAnswerOptions offerOptions; mPeerConnection->CreateOffer(this, offerOptions); - RTC_LOG(LS_INFO) << __FUNCTION__ << " " << mPeerConnection->signaling_state(); - return true; } void LLWebRTCImpl::shutdownConnection() { - mDataChannel->Close(); - mDataChannel = nullptr; - mPeerConnection->Close(); - mPeerDeviceModule->Terminate(); - mPeerDeviceModule = nullptr; - mPeerConnection = nullptr; - mPeerConnectionFactory = nullptr; + mSignalingThread->PostTask( + [this]() + { + if (mPeerConnection) + { + mPeerConnection->Close(); + mPeerConnection = nullptr; + } + mPeerConnectionFactory = nullptr; + }); + mWorkerThread->PostTask( + [this]() + { + if (mTuningDeviceModule) + { + mTuningDeviceModule->StopRecording(); + mTuningDeviceModule->Terminate(); + } + if (mPeerDeviceModule) + { + mPeerDeviceModule->StopRecording(); + mPeerDeviceModule->Terminate(); + } + mTuningDeviceModule = nullptr; + mPeerDeviceModule = nullptr; + mTaskQueueFactory = nullptr; + }); + mNetworkThread->PostTask( + [this]() + { + if (mDataChannel) + { + mDataChannel->Close(); + mDataChannel = nullptr; + } + }); } void LLWebRTCImpl::AnswerAvailable(const std::string &sdp) @@ -519,26 +545,6 @@ void LLWebRTCImpl::AnswerAvailable(const std::string &sdp) RTC_LOG(LS_INFO) << __FUNCTION__ << " " << mPeerConnection->peer_connection_state(); mPeerConnection->SetRemoteDescription(webrtc::CreateSessionDescription(webrtc::SdpType::kAnswer, sdp), rtc::scoped_refptr(this)); - mAnswerReceived = true; - for (auto &observer : mSignalingObserverList) - { - for (auto &candidate : mCachedIceCandidates) - { - LLWebRTCIceCandidate ice_candidate; - ice_candidate.candidate = candidate->candidate().ToString(); - ice_candidate.mline_index = candidate->sdp_mline_index(); - ice_candidate.sdp_mid = candidate->sdp_mid(); - observer->OnIceCandidate(ice_candidate); - } - mCachedIceCandidates.clear(); - if (mPeerConnection->ice_gathering_state() == webrtc::PeerConnectionInterface::IceGatheringState::kIceGatheringComplete) - { - for (auto &observer : mSignalingObserverList) - { - observer->OnIceGatheringState(llwebrtc::LLWebRTCSignalingObserver::IceGatheringState::ICE_GATHERING_COMPLETE); - } - } - } }); } @@ -769,6 +775,19 @@ void LLWebRTCImpl::OnSetRemoteDescriptionComplete(webrtc::RTCError error) RTC_LOG(LS_ERROR) << ToString(error.type()) << ": " << error.message(); return; } + mAnswerReceived = true; + for (auto &observer : mSignalingObserverList) + { + for (auto &candidate : mCachedIceCandidates) + { + LLWebRTCIceCandidate ice_candidate; + ice_candidate.candidate = candidate->candidate().ToString(); + ice_candidate.mline_index = candidate->sdp_mline_index(); + ice_candidate.sdp_mid = candidate->sdp_mid(); + observer->OnIceCandidate(ice_candidate); + } + mCachedIceCandidates.clear(); + } } // @@ -776,6 +795,7 @@ void LLWebRTCImpl::OnSetRemoteDescriptionComplete(webrtc::RTCError error) // void LLWebRTCImpl::OnSetLocalDescriptionComplete(webrtc::RTCError error) { +#if 0 RTC_LOG(LS_INFO) << __FUNCTION__ << " " << mPeerConnection->signaling_state(); if (!error.ok()) { @@ -792,6 +812,7 @@ void LLWebRTCImpl::OnSetLocalDescriptionComplete(webrtc::RTCError error) { observer->OnOfferAvailable(sdp); } +#endif } void LLWebRTCImpl::setAudioObserver(LLWebRTCAudioObserver *observer) { mAudioObserverList.emplace_back(observer); } diff --git a/indra/llwebrtc/llwebrtc_impl.h b/indra/llwebrtc/llwebrtc_impl.h index 24edd6eaa2..01159604c9 100644 --- a/indra/llwebrtc/llwebrtc_impl.h +++ b/indra/llwebrtc/llwebrtc_impl.h @@ -190,8 +190,6 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, protected: - bool initializeConnectionThreaded(); - std::unique_ptr mNetworkThread; std::unique_ptr mWorkerThread; std::unique_ptr mSignalingThread; -- cgit v1.2.3 From f760869c913df0fe74c9f68eb7b8e22ca19b0b2a Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Tue, 10 Oct 2023 09:52:51 -0700 Subject: generate ice candidate with proper formatting --- indra/llwebrtc/llwebrtc.cpp | 41 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 39 insertions(+), 2 deletions(-) (limited to 'indra/llwebrtc') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index d8f2a6028c..b5b820e3da 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -683,6 +683,43 @@ void LLWebRTCImpl::OnConnectionChange(webrtc::PeerConnectionInterface::PeerConne } } +static std::string iceCandidateToTrickleString(const webrtc::IceCandidateInterface *candidate) +{ + std::ostringstream candidate_stream; + + candidate_stream << + candidate->candidate().foundation() << " " << + std::to_string(candidate->candidate().component()) << " " << + candidate->candidate().protocol() << " " << + std::to_string(candidate->candidate().priority()) << " " << + candidate->candidate().address().ipaddr().ToString() << " " << + candidate->candidate().address().PortAsString() << " typ "; + if (candidate->candidate().type() == cricket::LOCAL_PORT_TYPE) + { + candidate_stream << "host"; + } + else if (candidate->candidate().type() == cricket::STUN_PORT_TYPE) + { + candidate_stream << "srflx " << + "raddr " << candidate->candidate().related_address().ipaddr().ToString() << " " << + "rport " << candidate->candidate().related_address().PortAsString(); + } + else if (candidate->candidate().type() == cricket::RELAY_PORT_TYPE) + { + candidate_stream << "relay " << + "raddr " << candidate->candidate().related_address().ipaddr().ToString() << " " << + "rport " << candidate->candidate().related_address().PortAsString(); + } + else if (candidate->candidate().type() == cricket::PRFLX_PORT_TYPE) + { + candidate_stream << "prflx " << + "raddr " << candidate->candidate().related_address().ipaddr().ToString() << " " << + "rport " << candidate->candidate().related_address().PortAsString(); + } + + return candidate_stream.str(); +} + void LLWebRTCImpl::OnIceCandidate(const webrtc::IceCandidateInterface *candidate) { RTC_LOG(LS_INFO) << __FUNCTION__ << " " << candidate->sdp_mline_index(); @@ -697,7 +734,7 @@ void LLWebRTCImpl::OnIceCandidate(const webrtc::IceCandidateInterface *candidate for (auto &observer : mSignalingObserverList) { LLWebRTCIceCandidate ice_candidate; - ice_candidate.candidate = candidate->candidate().ToString(); + ice_candidate.candidate = iceCandidateToTrickleString(candidate); ice_candidate.mline_index = candidate->sdp_mline_index(); ice_candidate.sdp_mid = candidate->sdp_mid(); observer->OnIceCandidate(ice_candidate); @@ -781,7 +818,7 @@ void LLWebRTCImpl::OnSetRemoteDescriptionComplete(webrtc::RTCError error) for (auto &candidate : mCachedIceCandidates) { LLWebRTCIceCandidate ice_candidate; - ice_candidate.candidate = candidate->candidate().ToString(); + ice_candidate.candidate = iceCandidateToTrickleString(candidate.get()); ice_candidate.mline_index = candidate->sdp_mline_index(); ice_candidate.sdp_mid = candidate->sdp_mid(); observer->OnIceCandidate(ice_candidate); -- cgit v1.2.3 From af67b8f3ff76f4fb380fdc84f1c80869d4bac0a4 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Wed, 11 Oct 2023 20:50:06 -0700 Subject: add tcptype to tcp ice candidate strings --- indra/llwebrtc/llwebrtc.cpp | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'indra/llwebrtc') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index b5b820e3da..78968fc89b 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -716,6 +716,10 @@ static std::string iceCandidateToTrickleString(const webrtc::IceCandidateInterfa "raddr " << candidate->candidate().related_address().ipaddr().ToString() << " " << "rport " << candidate->candidate().related_address().PortAsString(); } + if (candidate->candidate().protocol() == "tcp") + { + candidate_stream << " tcptype " << candidate->candidate().tcptype(); + } return candidate_stream.str(); } -- cgit v1.2.3 From 85f987128b2094d4bbce177ff9f7654eb7921e49 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Thu, 19 Oct 2023 14:50:12 -0700 Subject: quicker turnaround on re-establishing voice when server goes down. --- indra/llwebrtc/llwebrtc.cpp | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) (limited to 'indra/llwebrtc') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index 78968fc89b..194c7cd5d4 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -497,7 +497,7 @@ bool LLWebRTCImpl::initializeConnection() void LLWebRTCImpl::shutdownConnection() { - mSignalingThread->PostTask( + mSignalingThread->BlockingCall( [this]() { if (mPeerConnection) @@ -507,24 +507,21 @@ void LLWebRTCImpl::shutdownConnection() } mPeerConnectionFactory = nullptr; }); - mWorkerThread->PostTask( + mWorkerThread->BlockingCall( [this]() { - if (mTuningDeviceModule) - { - mTuningDeviceModule->StopRecording(); - mTuningDeviceModule->Terminate(); - } if (mPeerDeviceModule) { mPeerDeviceModule->StopRecording(); mPeerDeviceModule->Terminate(); } - mTuningDeviceModule = nullptr; mPeerDeviceModule = nullptr; - mTaskQueueFactory = nullptr; + if (mPeerAudioDeviceObserver) + { + mPeerAudioDeviceObserver = nullptr; + } }); - mNetworkThread->PostTask( + mNetworkThread->BlockingCall( [this]() { if (mDataChannel) @@ -694,6 +691,7 @@ static std::string iceCandidateToTrickleString(const webrtc::IceCandidateInterfa std::to_string(candidate->candidate().priority()) << " " << candidate->candidate().address().ipaddr().ToString() << " " << candidate->candidate().address().PortAsString() << " typ "; + if (candidate->candidate().type() == cricket::LOCAL_PORT_TYPE) { candidate_stream << "host"; @@ -716,6 +714,9 @@ static std::string iceCandidateToTrickleString(const webrtc::IceCandidateInterfa "raddr " << candidate->candidate().related_address().ipaddr().ToString() << " " << "rport " << candidate->candidate().related_address().PortAsString(); } + else { + RTC_LOG(LS_ERROR) << __FUNCTION__ << " Unknown candidate type " << candidate->candidate().type(); + } if (candidate->candidate().protocol() == "tcp") { candidate_stream << " tcptype " << candidate->candidate().tcptype(); @@ -827,8 +828,10 @@ void LLWebRTCImpl::OnSetRemoteDescriptionComplete(webrtc::RTCError error) ice_candidate.sdp_mid = candidate->sdp_mid(); observer->OnIceCandidate(ice_candidate); } - mCachedIceCandidates.clear(); } + mCachedIceCandidates.clear(); + OnIceGatheringChange(mPeerConnection->ice_gathering_state()); + } // -- cgit v1.2.3 From 85aea763d171ef3f3a92e7a20f9aaa8dfdcbc026 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Wed, 8 Nov 2023 10:13:15 -0800 Subject: SL-20543 - voice over region boundaries. This commit includes code to allow the llwebrtc.dll/dylib to allow multiple connections at once. --- indra/llwebrtc/llwebrtc.cpp | 781 ++++++++++++++++++++--------------------- indra/llwebrtc/llwebrtc.h | 24 +- indra/llwebrtc/llwebrtc_impl.h | 144 ++++++-- 3 files changed, 510 insertions(+), 439 deletions(-) (limited to 'indra/llwebrtc') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index 194c7cd5d4..6eb3478b59 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -1,6 +1,6 @@ /** - * @file llaccordionctrl.cpp - * @brief Accordion panel implementation + * @file llwebrtc.cpp + * @brief WebRTC interface implementation * * $LicenseInfo:firstyear=2023&license=viewerlgpl$ * Second Life Viewer Source Code @@ -38,7 +38,7 @@ namespace llwebrtc { -const float VOLUME_SCALE_WEBRTC = 3.0f; +const float VOLUME_SCALE_WEBRTC = 100; LLAudioDeviceObserver::LLAudioDeviceObserver() : mMicrophoneEnergy(0.0), mSumVector {0} {} @@ -57,7 +57,7 @@ void LLAudioDeviceObserver::OnCaptureData(const void *audio_samples, float sample = (static_cast(samples[index]) / (float) 32768); energy += sample * sample; } - + // smooth it. size_t buffer_size = sizeof(mSumVector) / sizeof(mSumVector[0]); float totalSum = 0; @@ -82,12 +82,12 @@ void LLAudioDeviceObserver::OnRenderData(const void *audio_samples, void LLWebRTCImpl::init() { + RTC_DCHECK(mPeerConnectionFactory); mPlayoutDevice = -1; mRecordingDevice = -1; - mAnswerReceived = false; rtc::InitializeSSL(); mTaskQueueFactory = webrtc::CreateDefaultTaskQueueFactory(); - + mNetworkThread = rtc::Thread::CreateWithSocketServer(); mNetworkThread->SetName("WebRTCNetworkThread", nullptr); mNetworkThread->Start(); @@ -97,61 +97,101 @@ void LLWebRTCImpl::init() mSignalingThread = rtc::Thread::Create(); mSignalingThread->SetName("WebRTCSignalingThread", nullptr); mSignalingThread->Start(); - + mWorkerThread->PostTask( - [this]() - { - mTuningAudioDeviceObserver = new LLAudioDeviceObserver; - mTuningDeviceModule = - webrtc::CreateAudioDeviceWithDataObserver(webrtc::AudioDeviceModule::AudioLayer::kPlatformDefaultAudio, - mTaskQueueFactory.get(), - std::unique_ptr(mTuningAudioDeviceObserver)); - mTuningDeviceModule->Init(); - mTuningDeviceModule->SetStereoRecording(false); - mTuningDeviceModule->SetStereoPlayout(true); - mTuningDeviceModule->EnableBuiltInAEC(false); - updateDevices(); - }); + [this]() + { + mTuningAudioDeviceObserver = new LLAudioDeviceObserver; + mTuningDeviceModule = + webrtc::CreateAudioDeviceWithDataObserver(webrtc::AudioDeviceModule::AudioLayer::kPlatformDefaultAudio, + mTaskQueueFactory.get(), + std::unique_ptr(mTuningAudioDeviceObserver)); + mTuningDeviceModule->Init(); + mTuningDeviceModule->SetStereoRecording(false); + mTuningDeviceModule->SetStereoPlayout(true); + mTuningDeviceModule->EnableBuiltInAEC(false); + updateDevices(); + }); + + rtc::scoped_refptr apm = webrtc::AudioProcessingBuilder().Create(); + webrtc::AudioProcessing::Config apm_config; + apm_config.echo_canceller.enabled = false; + apm_config.echo_canceller.mobile_mode = false; + apm_config.gain_controller1.enabled = true; + apm_config.gain_controller1.mode = webrtc::AudioProcessing::Config::GainController1::kAdaptiveAnalog; + apm_config.gain_controller2.enabled = true; + apm_config.high_pass_filter.enabled = true; + apm_config.noise_suppression.enabled = true; + apm_config.noise_suppression.level = webrtc::AudioProcessing::Config::NoiseSuppression::kVeryHigh; + apm_config.transient_suppression.enabled = true; + apm_config.pipeline.multi_channel_render = true; + apm_config.pipeline.multi_channel_capture = true; + + mWorkerThread->BlockingCall( + [this]() + { + mPeerAudioDeviceObserver = new LLAudioDeviceObserver; + mPeerDeviceModule = + webrtc::CreateAudioDeviceWithDataObserver(webrtc::AudioDeviceModule::AudioLayer::kPlatformDefaultAudio, + mTaskQueueFactory.get(), + std::unique_ptr(mPeerAudioDeviceObserver)); + mPeerDeviceModule->Init(); + mPeerDeviceModule->SetPlayoutDevice(mPlayoutDevice); + mPeerDeviceModule->SetRecordingDevice(mRecordingDevice); + mPeerDeviceModule->SetStereoRecording(false); + mPeerDeviceModule->SetStereoPlayout(true); + mPeerDeviceModule->EnableBuiltInAEC(false); + mPeerDeviceModule->InitMicrophone(); + mPeerDeviceModule->InitSpeaker(); + mPeerDeviceModule->InitRecording(); + mPeerDeviceModule->InitPlayout(); + mPeerDeviceModule->StartPlayout(); + mPeerDeviceModule->StartRecording(); + }); + + mPeerConnectionFactory = webrtc::CreatePeerConnectionFactory(mNetworkThread.get(), + mWorkerThread.get(), + mSignalingThread.get(), + mPeerDeviceModule, + webrtc::CreateBuiltinAudioEncoderFactory(), + webrtc::CreateBuiltinAudioDecoderFactory(), + nullptr /* video_encoder_factory */, + nullptr /* video_decoder_factory */, + nullptr /* audio_mixer */, + apm); + apm->ApplyConfig(apm_config); } void LLWebRTCImpl::terminate() { + for (auto& connection : mPeerConnections) + { + connection->terminate(); + } + mPeerConnections.clear(); + mSignalingThread->BlockingCall( - [this]() - { - if (mPeerConnection) - { - mPeerConnection->Close(); - mPeerConnection = nullptr; - } - mPeerConnectionFactory = nullptr; - }); + [this]() + { + mPeerConnectionFactory = nullptr; + }); mWorkerThread->BlockingCall( - [this]() - { - if (mTuningDeviceModule) - { - mTuningDeviceModule->StopRecording(); - mTuningDeviceModule->Terminate(); - } - if (mPeerDeviceModule) - { - mPeerDeviceModule->StopRecording(); - mPeerDeviceModule->Terminate(); - } - mTuningDeviceModule = nullptr; - mPeerDeviceModule = nullptr; - mTaskQueueFactory = nullptr; - }); - mNetworkThread->BlockingCall( - [this]() - { - if (mDataChannel) - { - mDataChannel->Close(); - mDataChannel = nullptr; - } - }); + [this]() + { + if (mTuningDeviceModule) + { + mTuningDeviceModule->StopRecording(); + mTuningDeviceModule->Terminate(); + } + if (mPeerDeviceModule) + { + mPeerDeviceModule->StopRecording(); + mPeerDeviceModule->Terminate(); + } + mTuningDeviceModule = nullptr; + mPeerDeviceModule = nullptr; + mTaskQueueFactory = nullptr; + }); } void LLWebRTCImpl::refreshDevices() @@ -164,7 +204,7 @@ void LLWebRTCImpl::setDevicesObserver(LLWebRTCDevicesObserver *observer) { mVoic void LLWebRTCImpl::unsetDevicesObserver(LLWebRTCDevicesObserver *observer) { std::vector::iterator it = - std::find(mVoiceDevicesObserverList.begin(), mVoiceDevicesObserverList.end(), observer); + std::find(mVoiceDevicesObserverList.begin(), mVoiceDevicesObserverList.end(), observer); if (it != mVoiceDevicesObserverList.end()) { mVoiceDevicesObserverList.erase(it); @@ -174,97 +214,97 @@ void LLWebRTCImpl::unsetDevicesObserver(LLWebRTCDevicesObserver *observer) void LLWebRTCImpl::setCaptureDevice(const std::string &id) { mWorkerThread->PostTask( - [this, id]() - { - int16_t captureDeviceCount = mTuningDeviceModule->RecordingDevices(); - for (int16_t i = 0; i < captureDeviceCount; i++) - { - char name[webrtc::kAdmMaxDeviceNameSize]; - char guid[webrtc::kAdmMaxGuidSize]; - mTuningDeviceModule->RecordingDeviceName(i, name, guid); - if (id == guid || id == "Default") // first one in list is default - { - RTC_LOG(LS_INFO) << __FUNCTION__ << "Set recording device to " << name << " " << guid << " " << i; - mRecordingDevice = i; - break; - } - } - mTuningDeviceModule->StopRecording(); - mTuningDeviceModule->SetRecordingDevice(mRecordingDevice); - mTuningDeviceModule->InitMicrophone(); - mTuningDeviceModule->InitRecording(); - mTuningDeviceModule->StartRecording(); - bool was_peer_recording = false; - if (mPeerDeviceModule) - { - was_peer_recording = mPeerDeviceModule->Recording(); - if (was_peer_recording) - { - mPeerDeviceModule->StopRecording(); - } - mPeerDeviceModule->SetRecordingDevice(mRecordingDevice); - mPeerDeviceModule->InitMicrophone(); - mPeerDeviceModule->InitRecording(); - if (was_peer_recording) - { - mPeerDeviceModule->StartRecording(); - } - } - }); + [this, id]() + { + int16_t captureDeviceCount = mTuningDeviceModule->RecordingDevices(); + for (int16_t i = 0; i < captureDeviceCount; i++) + { + char name[webrtc::kAdmMaxDeviceNameSize]; + char guid[webrtc::kAdmMaxGuidSize]; + mTuningDeviceModule->RecordingDeviceName(i, name, guid); + if (id == guid || id == "Default") // first one in list is default + { + RTC_LOG(LS_INFO) << __FUNCTION__ << "Set recording device to " << name << " " << guid << " " << i; + mRecordingDevice = i; + break; + } + } + mTuningDeviceModule->StopRecording(); + mTuningDeviceModule->SetRecordingDevice(mRecordingDevice); + mTuningDeviceModule->InitMicrophone(); + mTuningDeviceModule->InitRecording(); + mTuningDeviceModule->StartRecording(); + bool was_peer_recording = false; + if (mPeerDeviceModule) + { + was_peer_recording = mPeerDeviceModule->Recording(); + if (was_peer_recording) + { + mPeerDeviceModule->StopRecording(); + } + mPeerDeviceModule->SetRecordingDevice(mRecordingDevice); + mPeerDeviceModule->InitMicrophone(); + mPeerDeviceModule->InitRecording(); + if (was_peer_recording) + { + mPeerDeviceModule->StartRecording(); + } + } + }); } void LLWebRTCImpl::setRenderDevice(const std::string &id) { mWorkerThread->PostTask( - [this, id]() - { - int16_t renderDeviceCount = mTuningDeviceModule->PlayoutDevices(); - for (int16_t i = 0; i < renderDeviceCount; i++) - { - char name[webrtc::kAdmMaxDeviceNameSize]; - char guid[webrtc::kAdmMaxGuidSize]; - mTuningDeviceModule->PlayoutDeviceName(i, name, guid); - if (id == guid || id == "Default") - { - RTC_LOG(LS_INFO) << __FUNCTION__ << "Set recording device to " << name << " " << guid << " " << i; - mPlayoutDevice = i; - break; - } - } - mTuningDeviceModule->SetSpeakerMute(true); - bool was_tuning_playing = mTuningDeviceModule->Playing(); - if (was_tuning_playing) - { - mTuningDeviceModule->StopPlayout(); - } - bool was_peer_mute = false; - if (mPeerDeviceModule) - { - mPeerDeviceModule->SpeakerMute(&was_peer_mute); - if (!was_peer_mute) - { - mPeerDeviceModule->SetSpeakerMute(true); - } - } - - mTuningDeviceModule->SetPlayoutDevice(mPlayoutDevice); - mTuningDeviceModule->InitSpeaker(); - mTuningDeviceModule->InitPlayout(); - if (was_tuning_playing) - { - mTuningDeviceModule->StartPlayout(); - } - if (mPeerDeviceModule) - { - mPeerDeviceModule->SetPlayoutDevice(mPlayoutDevice); - mPeerDeviceModule->InitSpeaker(); - mPeerDeviceModule->InitPlayout(); - mPeerDeviceModule->StartPlayout(); - mPeerDeviceModule->SetSpeakerMute(was_peer_mute); - - } - mTuningDeviceModule->SetSpeakerMute(false); - }); + [this, id]() + { + int16_t renderDeviceCount = mTuningDeviceModule->PlayoutDevices(); + for (int16_t i = 0; i < renderDeviceCount; i++) + { + char name[webrtc::kAdmMaxDeviceNameSize]; + char guid[webrtc::kAdmMaxGuidSize]; + mTuningDeviceModule->PlayoutDeviceName(i, name, guid); + if (id == guid || id == "Default") + { + RTC_LOG(LS_INFO) << __FUNCTION__ << "Set recording device to " << name << " " << guid << " " << i; + mPlayoutDevice = i; + break; + } + } + mTuningDeviceModule->SetSpeakerMute(true); + bool was_tuning_playing = mTuningDeviceModule->Playing(); + if (was_tuning_playing) + { + mTuningDeviceModule->StopPlayout(); + } + bool was_peer_mute = false; + if (mPeerDeviceModule) + { + mPeerDeviceModule->SpeakerMute(&was_peer_mute); + if (!was_peer_mute) + { + mPeerDeviceModule->SetSpeakerMute(true); + } + } + + mTuningDeviceModule->SetPlayoutDevice(mPlayoutDevice); + mTuningDeviceModule->InitSpeaker(); + mTuningDeviceModule->InitPlayout(); + if (was_tuning_playing) + { + mTuningDeviceModule->StartPlayout(); + } + if (mPeerDeviceModule) + { + mPeerDeviceModule->SetPlayoutDevice(mPlayoutDevice); + mPeerDeviceModule->InitSpeaker(); + mPeerDeviceModule->InitPlayout(); + mPeerDeviceModule->StartPlayout(); + mPeerDeviceModule->SetSpeakerMute(was_peer_mute); + + } + mTuningDeviceModule->SetSpeakerMute(false); + }); } void LLWebRTCImpl::updateDevices() @@ -278,7 +318,7 @@ void LLWebRTCImpl::updateDevices() mTuningDeviceModule->PlayoutDeviceName(index, name, guid); renderDeviceList.emplace_back(name, guid); } - + int16_t captureDeviceCount = mTuningDeviceModule->RecordingDevices(); LLWebRTCVoiceDeviceList captureDeviceList; for (int16_t index = 0; index < captureDeviceCount; index++) @@ -297,113 +337,100 @@ void LLWebRTCImpl::updateDevices() void LLWebRTCImpl::setTuningMode(bool enable) { mWorkerThread->BlockingCall( - [this, enable]() - { - if (enable) - { + [this, enable]() + { + if (enable) + { + + mTuningDeviceModule->StartRecording(); + mTuningDeviceModule->SetMicrophoneMute(false); + + + mTuningDeviceModule->SetSpeakerMute(false); + + if (mPeerDeviceModule) + { + mPeerDeviceModule->StopRecording(); + mPeerDeviceModule->SetSpeakerMute(true); + } + } + else + { + if (mPeerDeviceModule) + { + mPeerDeviceModule->StartRecording(); + mPeerDeviceModule->SetSpeakerMute(false); + } + } + }); + for (auto& connection : mPeerConnections) + { + connection->enableTracks(enable ? false : !mMute); + } +} - mTuningDeviceModule->StartRecording(); - mTuningDeviceModule->SetMicrophoneMute(false); +float LLWebRTCImpl::getTuningAudioLevel() { return 20 * mTuningAudioDeviceObserver->getMicrophoneEnergy(); } +float LLWebRTCImpl::getPeerAudioLevel() { return 20 * mPeerAudioDeviceObserver->getMicrophoneEnergy(); } - mTuningDeviceModule->SetSpeakerMute(false); +void LLWebRTCImpl::setSpeakerVolume(float volume) { mPeerDeviceModule->SetSpeakerVolume(volume * VOLUME_SCALE_WEBRTC);} +void LLWebRTCImpl::setMicrophoneVolume(float volume) { mPeerDeviceModule->SetMicrophoneVolume(volume * VOLUME_SCALE_WEBRTC);} - if (mPeerDeviceModule) - { - mPeerDeviceModule->StopRecording(); - mPeerDeviceModule->SetSpeakerMute(true); - } - } - else - { - if (mPeerDeviceModule) - { - mPeerDeviceModule->StartRecording(); - mPeerDeviceModule->SetSpeakerMute(false); - } - } - }); - // set_enabled shouldn't be done on the worker thread - if (mPeerConnection) +// +// Helpers +// + +LLWebRTCPeerConnection * LLWebRTCImpl::newPeerConnection() +{ + rtc::scoped_refptr peerConnection = rtc::scoped_refptr(new rtc::RefCountedObject()); + peerConnection->init(this); + + mPeerConnections.emplace_back(peerConnection); + peerConnection->enableTracks(!mMute); + return peerConnection.get(); +} +void LLWebRTCImpl::freePeerConnection(LLWebRTCPeerConnection * peer_connection) +{ + std::vector>::iterator it = + std::find(mPeerConnections.begin(), mPeerConnections.end(), peer_connection); + if (it != mPeerConnections.end()) { - auto senders = mPeerConnection->GetSenders(); - for (auto &sender : senders) - { - sender->track()->set_enabled(enable ? false : !mMute); - } + (*it)->terminate(); + mPeerConnections.erase(it); } } // -// LLWebRTCSignalInterface +// LLWebRTCPeerConnection interface // -void LLWebRTCImpl::setSignalingObserver(LLWebRTCSignalingObserver *observer) { mSignalingObserverList.emplace_back(observer); } +void LLWebRTCPeerConnectionImpl::init(LLWebRTCImpl * webrtc_impl) +{ + mWebRTCImpl = webrtc_impl; + mPeerConnectionFactory = mWebRTCImpl->getPeerConnectionFactory(); +} +void LLWebRTCPeerConnectionImpl::terminate() +{ + shutdownConnection(); +} + +void LLWebRTCPeerConnectionImpl::setSignalingObserver(LLWebRTCSignalingObserver *observer) { mSignalingObserverList.emplace_back(observer); } -void LLWebRTCImpl::unsetSignalingObserver(LLWebRTCSignalingObserver *observer) +void LLWebRTCPeerConnectionImpl::unsetSignalingObserver(LLWebRTCSignalingObserver *observer) { std::vector::iterator it = - std::find(mSignalingObserverList.begin(), mSignalingObserverList.end(), observer); + std::find(mSignalingObserverList.begin(), mSignalingObserverList.end(), observer); if (it != mSignalingObserverList.end()) { mSignalingObserverList.erase(it); } } -bool LLWebRTCImpl::initializeConnection() +bool LLWebRTCPeerConnectionImpl::initializeConnection() { RTC_DCHECK(!mPeerConnection); - RTC_DCHECK(mPeerConnectionFactory); mAnswerReceived = false; - rtc::scoped_refptr apm = webrtc::AudioProcessingBuilder().Create(); - webrtc::AudioProcessing::Config apm_config; - apm_config.echo_canceller.enabled = false; - apm_config.echo_canceller.mobile_mode = false; - apm_config.gain_controller1.enabled = true; - apm_config.gain_controller1.mode = webrtc::AudioProcessing::Config::GainController1::kAdaptiveAnalog; - apm_config.gain_controller2.enabled = true; - apm_config.high_pass_filter.enabled = true; - apm_config.noise_suppression.enabled = true; - apm_config.noise_suppression.level = webrtc::AudioProcessing::Config::NoiseSuppression::kVeryHigh; - apm_config.transient_suppression.enabled = true; - apm_config.pipeline.multi_channel_render = true; - apm_config.pipeline.multi_channel_capture = true; - - mWorkerThread->BlockingCall( - [this]() - { - mPeerAudioDeviceObserver = new LLAudioDeviceObserver; - mPeerDeviceModule = - webrtc::CreateAudioDeviceWithDataObserver(webrtc::AudioDeviceModule::AudioLayer::kPlatformDefaultAudio, - mTaskQueueFactory.get(), - std::unique_ptr(mPeerAudioDeviceObserver)); - mPeerDeviceModule->Init(); - mPeerDeviceModule->SetPlayoutDevice(mPlayoutDevice); - mPeerDeviceModule->SetRecordingDevice(mRecordingDevice); - mPeerDeviceModule->SetStereoRecording(false); - mPeerDeviceModule->SetStereoPlayout(true); - mPeerDeviceModule->EnableBuiltInAEC(false); - mPeerDeviceModule->InitMicrophone(); - mPeerDeviceModule->InitSpeaker(); - mPeerDeviceModule->InitRecording(); - mPeerDeviceModule->InitPlayout(); - mPeerDeviceModule->StopPlayout(); - mPeerDeviceModule->StopRecording(); - }); - - mPeerConnectionFactory = webrtc::CreatePeerConnectionFactory(mNetworkThread.get(), - mWorkerThread.get(), - mSignalingThread.get(), - mPeerDeviceModule, - webrtc::CreateBuiltinAudioEncoderFactory(), - webrtc::CreateBuiltinAudioDecoderFactory(), - nullptr /* video_encoder_factory */, - nullptr /* video_decoder_factory */, - nullptr /* audio_mixer */, - apm); - apm->ApplyConfig(apm_config); - webrtc::PeerConnectionInterface::RTCConfiguration config; config.sdp_semantics = webrtc::SdpSemantics::kUnifiedPlan; webrtc::PeerConnectionInterface::IceServer server; @@ -419,7 +446,7 @@ bool LLWebRTCImpl::initializeConnection() config.servers.push_back(server); server.uri = "stun:stun4.l.google.com:19302"; config.servers.push_back(server); - + webrtc::PeerConnectionDependencies pc_dependencies(this); auto error_or_peer_connection = mPeerConnectionFactory->CreatePeerConnectionOrError(config, std::move(pc_dependencies)); if (error_or_peer_connection.ok()) @@ -432,35 +459,36 @@ bool LLWebRTCImpl::initializeConnection() shutdownConnection(); return false; } - + webrtc::DataChannelInit init; init.ordered = true; - + auto data_channel_or_error = mPeerConnection->CreateDataChannelOrError("SLData", &init); if (data_channel_or_error.ok()) { mDataChannel = std::move(data_channel_or_error.value()); - + mDataChannel->RegisterObserver(this); } - + RTC_LOG(LS_INFO) << __FUNCTION__ << " " << mPeerConnection->signaling_state(); - + cricket::AudioOptions audioOptions; audioOptions.auto_gain_control = true; audioOptions.echo_cancellation = false; // incompatible with opus stereo audioOptions.noise_suppression = true; - + rtc::scoped_refptr stream = mPeerConnectionFactory->CreateLocalMediaStream("SLStream"); + rtc::scoped_refptr audio_track( - mPeerConnectionFactory->CreateAudioTrack("SLAudio", mPeerConnectionFactory->CreateAudioSource(audioOptions).get())); + mPeerConnectionFactory->CreateAudioTrack("SLAudio", mPeerConnectionFactory->CreateAudioSource(audioOptions).get())); audio_track->set_enabled(true); stream->AddTrack(audio_track); - + mPeerConnection->AddTrack(audio_track, {"SLStream"}); - + auto senders = mPeerConnection->GetSenders(); - + for (auto &sender : senders) { webrtc::RtpParameters params; @@ -474,7 +502,7 @@ bool LLWebRTCImpl::initializeConnection() params.codecs.push_back(codecparam); sender->SetParameters(params); } - + auto receivers = mPeerConnection->GetReceivers(); for (auto &receiver : receivers) { @@ -491,102 +519,78 @@ bool LLWebRTCImpl::initializeConnection() } webrtc::PeerConnectionInterface::RTCOfferAnswerOptions offerOptions; mPeerConnection->CreateOffer(this, offerOptions); - + return true; } -void LLWebRTCImpl::shutdownConnection() +void LLWebRTCPeerConnectionImpl::shutdownConnection() { - mSignalingThread->BlockingCall( - [this]() - { - if (mPeerConnection) - { - mPeerConnection->Close(); - mPeerConnection = nullptr; - } - mPeerConnectionFactory = nullptr; - }); - mWorkerThread->BlockingCall( - [this]() - { - if (mPeerDeviceModule) - { - mPeerDeviceModule->StopRecording(); - mPeerDeviceModule->Terminate(); - } - mPeerDeviceModule = nullptr; - if (mPeerAudioDeviceObserver) - { - mPeerAudioDeviceObserver = nullptr; - } - }); - mNetworkThread->BlockingCall( - [this]() + mWebRTCImpl->SignalingBlockingCall( + [this]() + { + if (mPeerConnection) + { + mPeerConnection->Close(); + mPeerConnection = nullptr; + } + }); + + mWebRTCImpl->NetworkBlockingCall( + [this]() + { + if (mDataChannel) + { + mDataChannel->Close(); + mDataChannel = nullptr; + } + }); +} + +void LLWebRTCPeerConnectionImpl::enableTracks(bool enable) +{ + // set_enabled shouldn't be done on the worker thread + if (mPeerConnection) + { + auto senders = mPeerConnection->GetSenders(); + for (auto &sender : senders) { - if (mDataChannel) - { - mDataChannel->Close(); - mDataChannel = nullptr; - } - }); + sender->track()->set_enabled(enable); + } + } } -void LLWebRTCImpl::AnswerAvailable(const std::string &sdp) +void LLWebRTCPeerConnectionImpl::AnswerAvailable(const std::string &sdp) { RTC_LOG(LS_INFO) << __FUNCTION__ << " Remote SDP: " << sdp; - - mSignalingThread->PostTask( - [this, sdp]() - { - RTC_LOG(LS_INFO) << __FUNCTION__ << " " << mPeerConnection->peer_connection_state(); - mPeerConnection->SetRemoteDescription(webrtc::CreateSessionDescription(webrtc::SdpType::kAnswer, sdp), - rtc::scoped_refptr(this)); - }); + + mWebRTCImpl->PostSignalingTask( + [this, sdp]() + { + RTC_LOG(LS_INFO) << __FUNCTION__ << " " << mPeerConnection->peer_connection_state(); + mPeerConnection->SetRemoteDescription(webrtc::CreateSessionDescription(webrtc::SdpType::kAnswer, sdp), + rtc::scoped_refptr(this)); + }); } -void LLWebRTCImpl::setMute(bool mute) +void LLWebRTCPeerConnectionImpl::setMute(bool mute) { mMute = mute; auto senders = mPeerConnection->GetSenders(); - + RTC_LOG(LS_INFO) << __FUNCTION__ << (mute ? "disabling" : "enabling") << " streams count " << senders.size(); - + for (auto &sender : senders) { sender->track()->set_enabled(!mMute); } } -void LLWebRTCImpl::setSpeakerVolume(float volume) -{ - mSignalingThread->PostTask( - [this, volume]() - { - auto receivers = mPeerConnection->GetReceivers(); - - RTC_LOG(LS_INFO) << __FUNCTION__ << "Set volume" << receivers.size(); - for (auto &receiver : receivers) - { - webrtc::MediaStreamTrackInterface *track = receiver->track().get(); - if (track->kind() == webrtc::MediaStreamTrackInterface::kAudioKind) - { - webrtc::AudioTrackInterface *audio_track = static_cast(track); - webrtc::AudioSourceInterface *source = audio_track->GetSource(); - source->SetVolume(VOLUME_SCALE_WEBRTC * volume); - } - } - }); -} - -float LLWebRTCImpl::getTuningAudioLevel() { return 20 * mTuningAudioDeviceObserver->getMicrophoneEnergy(); } - // // PeerConnectionObserver implementation. // -void LLWebRTCImpl::OnAddTrack(rtc::scoped_refptr receiver, - const std::vector> &streams) +void LLWebRTCPeerConnectionImpl::OnAddTrack(rtc::scoped_refptr receiver, + const std::vector> &streams) { RTC_LOG(LS_INFO) << __FUNCTION__ << " " << receiver->id(); webrtc::RtpParameters params; @@ -601,18 +605,18 @@ void LLWebRTCImpl::OnAddTrack(rtc::scoped_refptr receiver->SetParameters(params); } -void LLWebRTCImpl::OnRemoveTrack(rtc::scoped_refptr receiver) +void LLWebRTCPeerConnectionImpl::OnRemoveTrack(rtc::scoped_refptr receiver) { RTC_LOG(LS_INFO) << __FUNCTION__ << " " << receiver->id(); } -void LLWebRTCImpl::OnDataChannel(rtc::scoped_refptr channel) +void LLWebRTCPeerConnectionImpl::OnDataChannel(rtc::scoped_refptr channel) { mDataChannel = channel; channel->RegisterObserver(this); } -void LLWebRTCImpl::OnIceGatheringChange(webrtc::PeerConnectionInterface::IceGatheringState new_state) +void LLWebRTCPeerConnectionImpl::OnIceGatheringChange(webrtc::PeerConnectionInterface::IceGatheringState new_state) { LLWebRTCSignalingObserver::IceGatheringState webrtc_new_state = LLWebRTCSignalingObserver::IceGatheringState::ICE_GATHERING_NEW; switch (new_state) @@ -631,7 +635,7 @@ void LLWebRTCImpl::OnIceGatheringChange(webrtc::PeerConnectionInterface::IceGath webrtc_new_state = LLWebRTCSignalingObserver::IceGatheringState::ICE_GATHERING_NEW; return; } - + if (mAnswerReceived) { for (auto &observer : mSignalingObserverList) @@ -642,19 +646,17 @@ void LLWebRTCImpl::OnIceGatheringChange(webrtc::PeerConnectionInterface::IceGath } // Called any time the PeerConnectionState changes. -void LLWebRTCImpl::OnConnectionChange(webrtc::PeerConnectionInterface::PeerConnectionState new_state) +void LLWebRTCPeerConnectionImpl::OnConnectionChange(webrtc::PeerConnectionInterface::PeerConnectionState new_state) { RTC_LOG(LS_ERROR) << __FUNCTION__ << " Peer Connection State Change " << new_state; - + switch (new_state) { case webrtc::PeerConnectionInterface::PeerConnectionState::kConnected: { if (new_state == webrtc::PeerConnectionInterface::PeerConnectionState::kConnected) { - mWorkerThread->PostTask([this]() { - mPeerDeviceModule->StartRecording(); - mPeerDeviceModule->StartPlayout(); + mWebRTCImpl->PostWorkerTask([this]() { for (auto &observer : mSignalingObserverList) { observer->OnAudioEstablished(this); @@ -670,7 +672,7 @@ void LLWebRTCImpl::OnConnectionChange(webrtc::PeerConnectionInterface::PeerConne { observer->OnRenegotiationNeeded(); } - + break; } default: @@ -683,15 +685,15 @@ void LLWebRTCImpl::OnConnectionChange(webrtc::PeerConnectionInterface::PeerConne static std::string iceCandidateToTrickleString(const webrtc::IceCandidateInterface *candidate) { std::ostringstream candidate_stream; - + candidate_stream << - candidate->candidate().foundation() << " " << - std::to_string(candidate->candidate().component()) << " " << - candidate->candidate().protocol() << " " << - std::to_string(candidate->candidate().priority()) << " " << - candidate->candidate().address().ipaddr().ToString() << " " << - candidate->candidate().address().PortAsString() << " typ "; - + candidate->candidate().foundation() << " " << + std::to_string(candidate->candidate().component()) << " " << + candidate->candidate().protocol() << " " << + std::to_string(candidate->candidate().priority()) << " " << + candidate->candidate().address().ipaddr().ToString() << " " << + candidate->candidate().address().PortAsString() << " typ "; + if (candidate->candidate().type() == cricket::LOCAL_PORT_TYPE) { candidate_stream << "host"; @@ -699,25 +701,25 @@ static std::string iceCandidateToTrickleString(const webrtc::IceCandidateInterfa else if (candidate->candidate().type() == cricket::STUN_PORT_TYPE) { candidate_stream << "srflx " << - "raddr " << candidate->candidate().related_address().ipaddr().ToString() << " " << - "rport " << candidate->candidate().related_address().PortAsString(); + "raddr " << candidate->candidate().related_address().ipaddr().ToString() << " " << + "rport " << candidate->candidate().related_address().PortAsString(); } else if (candidate->candidate().type() == cricket::RELAY_PORT_TYPE) { candidate_stream << "relay " << - "raddr " << candidate->candidate().related_address().ipaddr().ToString() << " " << - "rport " << candidate->candidate().related_address().PortAsString(); + "raddr " << candidate->candidate().related_address().ipaddr().ToString() << " " << + "rport " << candidate->candidate().related_address().PortAsString(); } else if (candidate->candidate().type() == cricket::PRFLX_PORT_TYPE) { candidate_stream << "prflx " << - "raddr " << candidate->candidate().related_address().ipaddr().ToString() << " " << - "rport " << candidate->candidate().related_address().PortAsString(); + "raddr " << candidate->candidate().related_address().ipaddr().ToString() << " " << + "rport " << candidate->candidate().related_address().PortAsString(); } else { RTC_LOG(LS_ERROR) << __FUNCTION__ << " Unknown candidate type " << candidate->candidate().type(); } - if (candidate->candidate().protocol() == "tcp") + if (candidate->candidate().protocol() == "tcp") { candidate_stream << " tcptype " << candidate->candidate().tcptype(); } @@ -725,16 +727,16 @@ static std::string iceCandidateToTrickleString(const webrtc::IceCandidateInterfa return candidate_stream.str(); } -void LLWebRTCImpl::OnIceCandidate(const webrtc::IceCandidateInterface *candidate) +void LLWebRTCPeerConnectionImpl::OnIceCandidate(const webrtc::IceCandidateInterface *candidate) { RTC_LOG(LS_INFO) << __FUNCTION__ << " " << candidate->sdp_mline_index(); - + if (!candidate) { RTC_LOG(LS_ERROR) << __FUNCTION__ << " No Ice Candidate Given"; return; } - if (mAnswerReceived) + if (mAnswerReceived) { for (auto &observer : mSignalingObserverList) { @@ -745,22 +747,22 @@ void LLWebRTCImpl::OnIceCandidate(const webrtc::IceCandidateInterface *candidate observer->OnIceCandidate(ice_candidate); } } - else + else { mCachedIceCandidates.push_back( - webrtc::CreateIceCandidate(candidate->sdp_mid(), candidate->sdp_mline_index(), candidate->candidate())); + webrtc::CreateIceCandidate(candidate->sdp_mid(), candidate->sdp_mline_index(), candidate->candidate())); } } // // CreateSessionDescriptionObserver implementation. // -void LLWebRTCImpl::OnSuccess(webrtc::SessionDescriptionInterface *desc) +void LLWebRTCPeerConnectionImpl::OnSuccess(webrtc::SessionDescriptionInterface *desc) { std::string sdp; desc->ToString(&sdp); RTC_LOG(LS_INFO) << sdp; -; + ; // mangle the sdp as this is the only way currently to bump up // the send audio rate to 48k std::istringstream sdp_stream(sdp); @@ -780,7 +782,7 @@ void LLWebRTCImpl::OnSuccess(webrtc::SessionDescriptionInterface *desc) else if (sdp_line.find("a=fmtp:" + opus_payload) == 0) { sdp_mangled_stream << sdp_line << "a=fmtp:" << opus_payload - << " minptime=10;useinbandfec=1;stereo=1;sprop-stereo=1;maxplaybackrate=48000\n"; + << " minptime=10;useinbandfec=1;stereo=1;sprop-stereo=1;maxplaybackrate=48000\n"; } else { @@ -789,27 +791,27 @@ void LLWebRTCImpl::OnSuccess(webrtc::SessionDescriptionInterface *desc) } webrtc::CreateSessionDescription(webrtc::SdpType::kOffer, sdp_mangled_stream.str()); - - - + + + mPeerConnection->SetLocalDescription(std::unique_ptr( - webrtc::CreateSessionDescription(webrtc::SdpType::kOffer, sdp_mangled_stream.str())), - rtc::scoped_refptr(this)); + webrtc::CreateSessionDescription(webrtc::SdpType::kOffer, sdp_mangled_stream.str())), + rtc::scoped_refptr(this)); RTC_LOG(LS_INFO) << __FUNCTION__ << " Local SDP: " << sdp_mangled_stream.str(); - - + + for (auto &observer : mSignalingObserverList) { observer->OnOfferAvailable(sdp_mangled_stream.str()); } } -void LLWebRTCImpl::OnFailure(webrtc::RTCError error) { RTC_LOG(LS_ERROR) << ToString(error.type()) << ": " << error.message(); } +void LLWebRTCPeerConnectionImpl::OnFailure(webrtc::RTCError error) { RTC_LOG(LS_ERROR) << ToString(error.type()) << ": " << error.message(); } // // SetRemoteDescriptionObserverInterface implementation. // -void LLWebRTCImpl::OnSetRemoteDescriptionComplete(webrtc::RTCError error) +void LLWebRTCPeerConnectionImpl::OnSetRemoteDescriptionComplete(webrtc::RTCError error) { RTC_LOG(LS_INFO) << __FUNCTION__ << " " << mPeerConnection->signaling_state(); if (!error.ok()) @@ -831,37 +833,20 @@ void LLWebRTCImpl::OnSetRemoteDescriptionComplete(webrtc::RTCError error) } mCachedIceCandidates.clear(); OnIceGatheringChange(mPeerConnection->ice_gathering_state()); - + } // // SetLocalDescriptionObserverInterface implementation. // -void LLWebRTCImpl::OnSetLocalDescriptionComplete(webrtc::RTCError error) +void LLWebRTCPeerConnectionImpl::OnSetLocalDescriptionComplete(webrtc::RTCError error) { -#if 0 - RTC_LOG(LS_INFO) << __FUNCTION__ << " " << mPeerConnection->signaling_state(); - if (!error.ok()) - { - RTC_LOG(LS_ERROR) << ToString(error.type()) << ": " << error.message(); - return; - } - auto desc = mPeerConnection->pending_local_description(); - std::string sdp; - desc->ToString(&sdp); - - RTC_LOG(LS_INFO) << __FUNCTION__ << " Local SDP: " << sdp; - ; - for (auto &observer : mSignalingObserverList) - { - observer->OnOfferAvailable(sdp); - } -#endif + } -void LLWebRTCImpl::setAudioObserver(LLWebRTCAudioObserver *observer) { mAudioObserverList.emplace_back(observer); } +void LLWebRTCPeerConnectionImpl::setAudioObserver(LLWebRTCAudioObserver *observer) { mAudioObserverList.emplace_back(observer); } -void LLWebRTCImpl::unsetAudioObserver(LLWebRTCAudioObserver *observer) +void LLWebRTCPeerConnectionImpl::unsetAudioObserver(LLWebRTCAudioObserver *observer) { std::vector::iterator it = std::find(mAudioObserverList.begin(), mAudioObserverList.end(), observer); if (it != mAudioObserverList.end()) @@ -870,22 +855,20 @@ void LLWebRTCImpl::unsetAudioObserver(LLWebRTCAudioObserver *observer) } } -float LLWebRTCImpl::getAudioLevel() { return 20 * mPeerAudioDeviceObserver->getMicrophoneEnergy(); } - // // DataChannelObserver implementation // -void LLWebRTCImpl::OnStateChange() -{ +void LLWebRTCPeerConnectionImpl::OnStateChange() +{ RTC_LOG(LS_INFO) << __FUNCTION__ << " Data Channel State: " << webrtc::DataChannelInterface::DataStateString(mDataChannel->state()); switch (mDataChannel->state()) { case webrtc::DataChannelInterface::kOpen: RTC_LOG(LS_INFO) << __FUNCTION__ << " Data Channel State Open"; - for (auto &observer : mDataObserverList) + for (auto &observer : mSignalingObserverList) { - observer->OnDataChannelReady(); + observer->OnDataChannelReady(this); } break; case webrtc::DataChannelInterface::kConnecting: @@ -903,7 +886,7 @@ void LLWebRTCImpl::OnStateChange() } -void LLWebRTCImpl::OnMessage(const webrtc::DataBuffer& buffer) +void LLWebRTCPeerConnectionImpl::OnMessage(const webrtc::DataBuffer& buffer) { std::string data((const char*)buffer.data.cdata(), buffer.size()); for (auto &observer : mDataObserverList) @@ -912,43 +895,55 @@ void LLWebRTCImpl::OnMessage(const webrtc::DataBuffer& buffer) } } -void LLWebRTCImpl::sendData(const std::string& data, bool binary) +void LLWebRTCPeerConnectionImpl::sendData(const std::string& data, bool binary) { rtc::CopyOnWriteBuffer cowBuffer(data.data(), data.length()); webrtc::DataBuffer buffer(cowBuffer, binary); mDataChannel->Send(buffer); } -void LLWebRTCImpl::setDataObserver(LLWebRTCDataObserver* observer) { mDataObserverList.emplace_back(observer); } +void LLWebRTCPeerConnectionImpl::setDataObserver(LLWebRTCDataObserver* observer) { mDataObserverList.emplace_back(observer); } -void LLWebRTCImpl::unsetDataObserver(LLWebRTCDataObserver* observer) +void LLWebRTCPeerConnectionImpl::unsetDataObserver(LLWebRTCDataObserver* observer) { std::vector::iterator it = - std::find(mDataObserverList.begin(), mDataObserverList.end(), observer); + std::find(mDataObserverList.begin(), mDataObserverList.end(), observer); if (it != mDataObserverList.end()) { mDataObserverList.erase(it); } } -rtc::RefCountedObject *gWebRTCImpl = nullptr; -LLWebRTCDeviceInterface *getDeviceInterface() { return gWebRTCImpl; } -LLWebRTCSignalInterface *getSignalingInterface() { return gWebRTCImpl; } -LLWebRTCDataInterface *getDataInterface() { return gWebRTCImpl; } +LLWebRTCImpl * gWebRTCImpl = nullptr; +LLWebRTCDeviceInterface * getDeviceInterface() +{ + return gWebRTCImpl; +} + +LLWebRTCPeerConnection * newPeerConnection() +{ + return gWebRTCImpl->newPeerConnection(); +} + +void freePeerConnection(LLWebRTCPeerConnection *peer_connection) +{ + gWebRTCImpl->freePeerConnection(peer_connection); +} void init() { - gWebRTCImpl = new rtc::RefCountedObject(); - gWebRTCImpl->AddRef(); + gWebRTCImpl = new LLWebRTCImpl(); gWebRTCImpl->init(); } void terminate() -{ - gWebRTCImpl->terminate(); - gWebRTCImpl->Release(); - gWebRTCImpl = nullptr; +{ + if (gWebRTCImpl) + { + gWebRTCImpl->terminate(); + gWebRTCImpl = nullptr; + } } } // namespace llwebrtc diff --git a/indra/llwebrtc/llwebrtc.h b/indra/llwebrtc/llwebrtc.h index 9224e88dfa..753fe6a983 100644 --- a/indra/llwebrtc/llwebrtc.h +++ b/indra/llwebrtc/llwebrtc.h @@ -1,6 +1,6 @@ /** - * @file llaccordionctrl.cpp - * @brief Accordion panel implementation + * @file llwebrtc.h + * @brief WebRTC interface * * $LicenseInfo:firstyear=2023&license=viewerlgpl$ * Second Life Viewer Source Code @@ -17,7 +17,7 @@ * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software + * License along with this library; if not, write to the Free tSoftware * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA * * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA @@ -44,8 +44,6 @@ namespace llwebrtc { -LLSYMEXPORT void init(); -LLSYMEXPORT void terminate(); struct LLWebRTCIceCandidate { @@ -87,6 +85,10 @@ class LLWebRTCDeviceInterface virtual void setTuningMode(bool enable) = 0; virtual float getTuningAudioLevel() = 0; + + virtual void setSpeakerVolume(float volume) = 0; // volume between 0.0 and 1.0 + virtual void setMicrophoneVolume(float volume) = 0; // volume between 0.0 and 1.0 + virtual float getPeerAudioLevel() = 0; }; class LLWebRTCAudioObserver @@ -100,15 +102,12 @@ class LLWebRTCAudioInterface virtual void setAudioObserver(LLWebRTCAudioObserver *observer) = 0; virtual void unsetAudioObserver(LLWebRTCAudioObserver *observer) = 0; virtual void setMute(bool mute) = 0; - virtual void setSpeakerVolume(float volume) = 0; // volume between 0.0 and 1.0 - virtual float getAudioLevel() = 0; }; class LLWebRTCDataObserver { public: virtual void OnDataReceived(const std::string& data, bool binary) = 0; - virtual void OnDataChannelReady() = 0; }; class LLWebRTCDataInterface @@ -133,9 +132,10 @@ class LLWebRTCSignalingObserver virtual void OnOfferAvailable(const std::string& sdp) = 0; virtual void OnRenegotiationNeeded() = 0; virtual void OnAudioEstablished(LLWebRTCAudioInterface *audio_interface) = 0; + virtual void OnDataChannelReady(LLWebRTCDataInterface *data_interface) = 0; }; -class LLWebRTCSignalInterface +class LLWebRTCPeerConnection { public: virtual void setSignalingObserver(LLWebRTCSignalingObserver* observer) = 0; @@ -146,9 +146,11 @@ class LLWebRTCSignalInterface virtual void AnswerAvailable(const std::string &sdp) = 0; }; +LLSYMEXPORT void init(); +LLSYMEXPORT void terminate(); LLSYMEXPORT LLWebRTCDeviceInterface* getDeviceInterface(); -LLSYMEXPORT LLWebRTCSignalInterface* getSignalingInterface(); -LLSYMEXPORT LLWebRTCDataInterface* getDataInterface(); +LLSYMEXPORT LLWebRTCPeerConnection* newPeerConnection(); +LLSYMEXPORT void freePeerConnection(LLWebRTCPeerConnection *connection); } #endif // LLWEBRTC_H diff --git a/indra/llwebrtc/llwebrtc_impl.h b/indra/llwebrtc/llwebrtc_impl.h index 01159604c9..295e949b4e 100644 --- a/indra/llwebrtc/llwebrtc_impl.h +++ b/indra/llwebrtc/llwebrtc_impl.h @@ -1,6 +1,6 @@ /** - * @file llaccordionctrl.cpp - * @brief Accordion panel implementation + * @file llwebrtc_impl.h + * @brief WebRTC Interface implementation. * * $LicenseInfo:firstyear=2023&license=viewerlgpl$ * Second Life Viewer Source Code @@ -63,6 +63,8 @@ namespace llwebrtc { +class LLWebRTCPeerConnectionImpl; + class LLAudioDeviceObserver : public webrtc::AudioDeviceDataObserver { public: @@ -87,16 +89,7 @@ class LLAudioDeviceObserver : public webrtc::AudioDeviceDataObserver float mMicrophoneEnergy; }; -class LLWebRTCImpl : public LLWebRTCDeviceInterface, - public LLWebRTCSignalInterface, - public LLWebRTCAudioInterface, - public LLWebRTCDataInterface, - public webrtc::PeerConnectionObserver, - public webrtc::CreateSessionDescriptionObserver, - public webrtc::SetRemoteDescriptionObserverInterface, - public webrtc::SetLocalDescriptionObserverInterface, - public webrtc::DataChannelObserver - +class LLWebRTCImpl : public LLWebRTCDeviceInterface { public: LLWebRTCImpl() : @@ -116,15 +109,112 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, void setDevicesObserver(LLWebRTCDevicesObserver *observer) override; void unsetDevicesObserver(LLWebRTCDevicesObserver *observer) override; - void setCaptureDevice(const std::string& id) override; + void setCaptureDevice(const std::string& id) override; void setRenderDevice(const std::string& id) override; void setTuningMode(bool enable) override; float getTuningAudioLevel() override; + float getPeerAudioLevel() override; + + void setSpeakerVolume(float volume) override; // range 0.0-1.0 + void setMicrophoneVolume(float volume) override; // range 0.0-1.0 + + // + // Helpers + // + + void PostWorkerTask(absl::AnyInvocable task, + const webrtc::Location& location = webrtc::Location::Current()) + { + mWorkerThread->PostTask(std::move(task), location); + } + + void PostSignalingTask(absl::AnyInvocable task, + const webrtc::Location& location = webrtc::Location::Current()) + { + mSignalingThread->PostTask(std::move(task), location); + } + + void PostNetworkTask(absl::AnyInvocable task, + const webrtc::Location& location = webrtc::Location::Current()) + { + mNetworkThread->PostTask(std::move(task), location); + } + + void WorkerBlockingCall(rtc::FunctionView functor, + const webrtc::Location& location = webrtc::Location::Current()) + { + mWorkerThread->BlockingCall(std::move(functor), location); + } + + void SignalingBlockingCall(rtc::FunctionView functor, + const webrtc::Location& location = webrtc::Location::Current()) + { + mSignalingThread->BlockingCall(std::move(functor), location); + } + + void NetworkBlockingCall(rtc::FunctionView functor, + const webrtc::Location& location = webrtc::Location::Current()) + { + mNetworkThread->BlockingCall(std::move(functor), location); + } + + rtc::scoped_refptr getPeerConnectionFactory() { return mPeerConnectionFactory; } + + LLWebRTCPeerConnection * newPeerConnection(); + void freePeerConnection(LLWebRTCPeerConnection * peer_connection); + + protected: + + std::unique_ptr mNetworkThread; + std::unique_ptr mWorkerThread; + std::unique_ptr mSignalingThread; + rtc::scoped_refptr mPeerConnectionFactory; + webrtc::PeerConnectionInterface::RTCConfiguration mConfiguration; + std::unique_ptr mTaskQueueFactory; + + + // Devices + void updateDevices(); + rtc::scoped_refptr mTuningDeviceModule; + rtc::scoped_refptr mPeerDeviceModule; + std::vector mVoiceDevicesObserverList; + // accessors in webrtc aren't apparently implemented yet. + int32_t mPlayoutDevice; + int32_t mRecordingDevice; + bool mMute; + + LLAudioDeviceObserver * mTuningAudioDeviceObserver; + LLAudioDeviceObserver * mPeerAudioDeviceObserver; + + // peer connections + std::vector> mPeerConnections; +}; + +class LLWebRTCPeerConnectionImpl : public LLWebRTCPeerConnection, + public LLWebRTCAudioInterface, + public LLWebRTCDataInterface, + public webrtc::PeerConnectionObserver, + public webrtc::CreateSessionDescriptionObserver, + public webrtc::SetRemoteDescriptionObserverInterface, + public webrtc::SetLocalDescriptionObserverInterface, + public webrtc::DataChannelObserver + +{ + public: + LLWebRTCPeerConnectionImpl() {} + ~LLWebRTCPeerConnectionImpl() {} + + void init(LLWebRTCImpl * webrtc_impl); + void terminate(); + + virtual void AddRef() const override = 0; + virtual rtc::RefCountReleaseStatus Release() const override = 0; + // - // LLWebRTCSignalInterface + // LLWebRTCPeerConnection // void setSignalingObserver(LLWebRTCSignalingObserver *observer) override; @@ -140,8 +230,6 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, void setAudioObserver(LLWebRTCAudioObserver *observer) override; void unsetAudioObserver(LLWebRTCAudioObserver *observer) override; void setMute(bool mute) override; - void setSpeakerVolume(float folume) override; // range 0.0-1.0 - float getAudioLevel() override; // // LLWebRTCDataInterface @@ -187,31 +275,17 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, // void OnStateChange() override; void OnMessage(const webrtc::DataBuffer& buffer) override; + + // Helpers + void enableTracks(bool enable); protected: - - std::unique_ptr mNetworkThread; - std::unique_ptr mWorkerThread; - std::unique_ptr mSignalingThread; + + LLWebRTCImpl * mWebRTCImpl; rtc::scoped_refptr mPeerConnectionFactory; - webrtc::PeerConnectionInterface::RTCConfiguration mConfiguration; - std::unique_ptr mTaskQueueFactory; - - - // Devices - void updateDevices(); - rtc::scoped_refptr mTuningDeviceModule; - rtc::scoped_refptr mPeerDeviceModule; - std::vector mVoiceDevicesObserverList; - // accessors in webrtc aren't apparently implemented yet. - int32_t mPlayoutDevice; - int32_t mRecordingDevice; bool mMute; - LLAudioDeviceObserver * mTuningAudioDeviceObserver; - LLAudioDeviceObserver * mPeerAudioDeviceObserver; - // signaling std::vector mSignalingObserverList; std::vector> mCachedIceCandidates; -- cgit v1.2.3 From 1b77987a0b84fac6a0d07b7782a23bb22aa48ec6 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Wed, 8 Nov 2023 10:17:56 -0800 Subject: comment fixes --- indra/llwebrtc/llwebrtc.cpp | 2 +- indra/llwebrtc/llwebrtc_impl.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'indra/llwebrtc') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index 6eb3478b59..62c42ead1b 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -1,6 +1,6 @@ /** * @file llwebrtc.cpp - * @brief WebRTC interface implementation + * @brief WebRTC interface implementation * * $LicenseInfo:firstyear=2023&license=viewerlgpl$ * Second Life Viewer Source Code diff --git a/indra/llwebrtc/llwebrtc_impl.h b/indra/llwebrtc/llwebrtc_impl.h index 295e949b4e..eb439b5a46 100644 --- a/indra/llwebrtc/llwebrtc_impl.h +++ b/indra/llwebrtc/llwebrtc_impl.h @@ -1,6 +1,6 @@ /** * @file llwebrtc_impl.h - * @brief WebRTC Interface implementation. + * @brief WebRTC interface implementation header * * $LicenseInfo:firstyear=2023&license=viewerlgpl$ * Second Life Viewer Source Code -- cgit v1.2.3 From c33ebcfab6e4fda3f7858f1fa0f65d8c4e82f714 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Thu, 9 Nov 2023 18:19:18 -0800 Subject: Remove a bunch of unnecessary code. --- indra/llwebrtc/llwebrtc.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'indra/llwebrtc') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index 62c42ead1b..bf1e04c2f2 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -373,8 +373,8 @@ float LLWebRTCImpl::getTuningAudioLevel() { return 20 * mTuningAudioDeviceObserv float LLWebRTCImpl::getPeerAudioLevel() { return 20 * mPeerAudioDeviceObserver->getMicrophoneEnergy(); } -void LLWebRTCImpl::setSpeakerVolume(float volume) { mPeerDeviceModule->SetSpeakerVolume(volume * VOLUME_SCALE_WEBRTC);} -void LLWebRTCImpl::setMicrophoneVolume(float volume) { mPeerDeviceModule->SetMicrophoneVolume(volume * VOLUME_SCALE_WEBRTC);} +void LLWebRTCImpl::setSpeakerVolume(float volume) { mPeerDeviceModule->SetSpeakerVolume( (uint32_t)(volume * VOLUME_SCALE_WEBRTC));} +void LLWebRTCImpl::setMicrophoneVolume(float volume) { mPeerDeviceModule->SetMicrophoneVolume((uint32_t)(volume * VOLUME_SCALE_WEBRTC));} // // Helpers -- cgit v1.2.3 From ebfa44cdb76afb9632556115eef10969912340f5 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Thu, 30 Nov 2023 13:14:07 -0800 Subject: Refactor/clean-up WebRTC voice to handle multiple voice streams This is useful for cross-region voice, quick voice switching, etc. --- indra/llwebrtc/llwebrtc.cpp | 79 +++++++++++++++++++++++++++--------------- indra/llwebrtc/llwebrtc.h | 8 +---- indra/llwebrtc/llwebrtc_impl.h | 7 ++-- 3 files changed, 54 insertions(+), 40 deletions(-) (limited to 'indra/llwebrtc') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index bf1e04c2f2..b2f5e0212e 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -216,6 +216,7 @@ void LLWebRTCImpl::setCaptureDevice(const std::string &id) mWorkerThread->PostTask( [this, id]() { + int16_t tuningRecordingDevice = 0; int16_t captureDeviceCount = mTuningDeviceModule->RecordingDevices(); for (int16_t i = 0; i < captureDeviceCount; i++) { @@ -225,18 +226,32 @@ void LLWebRTCImpl::setCaptureDevice(const std::string &id) if (id == guid || id == "Default") // first one in list is default { RTC_LOG(LS_INFO) << __FUNCTION__ << "Set recording device to " << name << " " << guid << " " << i; - mRecordingDevice = i; + tuningRecordingDevice = i; break; } } mTuningDeviceModule->StopRecording(); - mTuningDeviceModule->SetRecordingDevice(mRecordingDevice); + mTuningDeviceModule->SetRecordingDevice(tuningRecordingDevice); mTuningDeviceModule->InitMicrophone(); mTuningDeviceModule->InitRecording(); mTuningDeviceModule->StartRecording(); bool was_peer_recording = false; if (mPeerDeviceModule) { + int16_t captureDeviceCount = mTuningDeviceModule->RecordingDevices(); + for (int16_t i = 0; i < captureDeviceCount; i++) + { + char name[webrtc::kAdmMaxDeviceNameSize]; + char guid[webrtc::kAdmMaxGuidSize]; + mTuningDeviceModule->RecordingDeviceName(i, name, guid); + if (id == guid || id == "Default") // first one in list is default + { + RTC_LOG(LS_INFO) + << __FUNCTION__ << "Set recording device to " << name << " " << guid << " " << i; + mRecordingDevice = i; + break; + } + } was_peer_recording = mPeerDeviceModule->Recording(); if (was_peer_recording) { @@ -259,6 +274,7 @@ void LLWebRTCImpl::setRenderDevice(const std::string &id) [this, id]() { int16_t renderDeviceCount = mTuningDeviceModule->PlayoutDevices(); + int16_t tuningPlayoutDevice = 0; for (int16_t i = 0; i < renderDeviceCount; i++) { char name[webrtc::kAdmMaxDeviceNameSize]; @@ -267,7 +283,7 @@ void LLWebRTCImpl::setRenderDevice(const std::string &id) if (id == guid || id == "Default") { RTC_LOG(LS_INFO) << __FUNCTION__ << "Set recording device to " << name << " " << guid << " " << i; - mPlayoutDevice = i; + tuningPlayoutDevice = i; break; } } @@ -277,30 +293,40 @@ void LLWebRTCImpl::setRenderDevice(const std::string &id) { mTuningDeviceModule->StopPlayout(); } - bool was_peer_mute = false; if (mPeerDeviceModule) { - mPeerDeviceModule->SpeakerMute(&was_peer_mute); - if (!was_peer_mute) - { - mPeerDeviceModule->SetSpeakerMute(true); - } + mPeerDeviceModule->SetSpeakerMute(true); } - mTuningDeviceModule->SetPlayoutDevice(mPlayoutDevice); + mTuningDeviceModule->SetPlayoutDevice(tuningPlayoutDevice); mTuningDeviceModule->InitSpeaker(); mTuningDeviceModule->InitPlayout(); if (was_tuning_playing) { mTuningDeviceModule->StartPlayout(); } + renderDeviceCount = mPeerDeviceModule->PlayoutDevices(); + if (mPeerDeviceModule) { + for (int16_t i = 0; i < renderDeviceCount; i++) + { + char name[webrtc::kAdmMaxDeviceNameSize]; + char guid[webrtc::kAdmMaxGuidSize]; + mPeerDeviceModule->PlayoutDeviceName(i, name, guid); + if (id == guid || id == "Default") + { + RTC_LOG(LS_INFO) + << __FUNCTION__ << "Set recording device to " << name << " " << guid << " " << i; + mPlayoutDevice = i; + break; + } + } mPeerDeviceModule->SetPlayoutDevice(mPlayoutDevice); mPeerDeviceModule->InitSpeaker(); mPeerDeviceModule->InitPlayout(); mPeerDeviceModule->StartPlayout(); - mPeerDeviceModule->SetSpeakerMute(was_peer_mute); + mPeerDeviceModule->SetSpeakerMute(false); } mTuningDeviceModule->SetSpeakerMute(false); @@ -376,6 +402,8 @@ float LLWebRTCImpl::getPeerAudioLevel() { return 20 * mPeerAudioDeviceObserver-> void LLWebRTCImpl::setSpeakerVolume(float volume) { mPeerDeviceModule->SetSpeakerVolume( (uint32_t)(volume * VOLUME_SCALE_WEBRTC));} void LLWebRTCImpl::setMicrophoneVolume(float volume) { mPeerDeviceModule->SetMicrophoneVolume((uint32_t)(volume * VOLUME_SCALE_WEBRTC));} +void LLWebRTCImpl::setMute(bool mute) { mPeerDeviceModule->SetMicrophoneMute(mute); } + // // Helpers // @@ -566,9 +594,12 @@ void LLWebRTCPeerConnectionImpl::AnswerAvailable(const std::string &sdp) mWebRTCImpl->PostSignalingTask( [this, sdp]() { - RTC_LOG(LS_INFO) << __FUNCTION__ << " " << mPeerConnection->peer_connection_state(); - mPeerConnection->SetRemoteDescription(webrtc::CreateSessionDescription(webrtc::SdpType::kAnswer, sdp), - rtc::scoped_refptr(this)); + if (mPeerConnection) + { + RTC_LOG(LS_INFO) << __FUNCTION__ << " " << mPeerConnection->peer_connection_state(); + mPeerConnection->SetRemoteDescription(webrtc::CreateSessionDescription(webrtc::SdpType::kAnswer, sdp), + rtc::scoped_refptr(this)); + } }); } @@ -844,17 +875,6 @@ void LLWebRTCPeerConnectionImpl::OnSetLocalDescriptionComplete(webrtc::RTCError } -void LLWebRTCPeerConnectionImpl::setAudioObserver(LLWebRTCAudioObserver *observer) { mAudioObserverList.emplace_back(observer); } - -void LLWebRTCPeerConnectionImpl::unsetAudioObserver(LLWebRTCAudioObserver *observer) -{ - std::vector::iterator it = std::find(mAudioObserverList.begin(), mAudioObserverList.end(), observer); - if (it != mAudioObserverList.end()) - { - mAudioObserverList.erase(it); - } -} - // // DataChannelObserver implementation // @@ -897,9 +917,12 @@ void LLWebRTCPeerConnectionImpl::OnMessage(const webrtc::DataBuffer& buffer) void LLWebRTCPeerConnectionImpl::sendData(const std::string& data, bool binary) { - rtc::CopyOnWriteBuffer cowBuffer(data.data(), data.length()); - webrtc::DataBuffer buffer(cowBuffer, binary); - mDataChannel->Send(buffer); + if (mDataChannel) + { + rtc::CopyOnWriteBuffer cowBuffer(data.data(), data.length()); + webrtc::DataBuffer buffer(cowBuffer, binary); + mDataChannel->Send(buffer); + } } void LLWebRTCPeerConnectionImpl::setDataObserver(LLWebRTCDataObserver* observer) { mDataObserverList.emplace_back(observer); } diff --git a/indra/llwebrtc/llwebrtc.h b/indra/llwebrtc/llwebrtc.h index 753fe6a983..ed80fa5648 100644 --- a/indra/llwebrtc/llwebrtc.h +++ b/indra/llwebrtc/llwebrtc.h @@ -88,19 +88,13 @@ class LLWebRTCDeviceInterface virtual void setSpeakerVolume(float volume) = 0; // volume between 0.0 and 1.0 virtual void setMicrophoneVolume(float volume) = 0; // volume between 0.0 and 1.0 + virtual void setMute(bool mute) = 0; virtual float getPeerAudioLevel() = 0; }; -class LLWebRTCAudioObserver -{ - public: -}; - class LLWebRTCAudioInterface { public: - virtual void setAudioObserver(LLWebRTCAudioObserver *observer) = 0; - virtual void unsetAudioObserver(LLWebRTCAudioObserver *observer) = 0; virtual void setMute(bool mute) = 0; }; diff --git a/indra/llwebrtc/llwebrtc_impl.h b/indra/llwebrtc/llwebrtc_impl.h index eb439b5a46..76e29c63fb 100644 --- a/indra/llwebrtc/llwebrtc_impl.h +++ b/indra/llwebrtc/llwebrtc_impl.h @@ -118,7 +118,8 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface float getPeerAudioLevel() override; void setSpeakerVolume(float volume) override; // range 0.0-1.0 - void setMicrophoneVolume(float volume) override; // range 0.0-1.0 + void setMicrophoneVolume(float volume) override; // range 0.0-1.0 + void setMute(bool mute) override; // // Helpers @@ -227,8 +228,6 @@ class LLWebRTCPeerConnectionImpl : public LLWebRTCPeerConnection, // // LLWebRTCAudioInterface // - void setAudioObserver(LLWebRTCAudioObserver *observer) override; - void unsetAudioObserver(LLWebRTCAudioObserver *observer) override; void setMute(bool mute) override; // @@ -292,8 +291,6 @@ class LLWebRTCPeerConnectionImpl : public LLWebRTCPeerConnection, bool mAnswerReceived; rtc::scoped_refptr mPeerConnection; - - std::vector mAudioObserverList; std::vector mDataObserverList; rtc::scoped_refptr mDataChannel; -- cgit v1.2.3 From 0b9c27dc70255385fd65ac2f1f8f99e2e013ea03 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Fri, 1 Dec 2023 01:14:33 -0800 Subject: Using the device module to set speaker/mic volume set the system mic/volume for all applications. Instead, modify the volume on the various streams. --- indra/llwebrtc/llwebrtc.cpp | 56 ++++++++++++++++++++++++++++++++---------- indra/llwebrtc/llwebrtc.h | 6 ++--- indra/llwebrtc/llwebrtc_impl.h | 10 +++----- 3 files changed, 49 insertions(+), 23 deletions(-) (limited to 'indra/llwebrtc') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index b2f5e0212e..4741ed92a3 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -399,11 +399,6 @@ float LLWebRTCImpl::getTuningAudioLevel() { return 20 * mTuningAudioDeviceObserv float LLWebRTCImpl::getPeerAudioLevel() { return 20 * mPeerAudioDeviceObserver->getMicrophoneEnergy(); } -void LLWebRTCImpl::setSpeakerVolume(float volume) { mPeerDeviceModule->SetSpeakerVolume( (uint32_t)(volume * VOLUME_SCALE_WEBRTC));} -void LLWebRTCImpl::setMicrophoneVolume(float volume) { mPeerDeviceModule->SetMicrophoneVolume((uint32_t)(volume * VOLUME_SCALE_WEBRTC));} - -void LLWebRTCImpl::setMute(bool mute) { mPeerDeviceModule->SetMicrophoneMute(mute); } - // // Helpers // @@ -417,6 +412,7 @@ LLWebRTCPeerConnection * LLWebRTCImpl::newPeerConnection() peerConnection->enableTracks(!mMute); return peerConnection.get(); } + void LLWebRTCImpl::freePeerConnection(LLWebRTCPeerConnection * peer_connection) { std::vector>::iterator it = @@ -506,12 +502,11 @@ bool LLWebRTCPeerConnectionImpl::initializeConnection() audioOptions.echo_cancellation = false; // incompatible with opus stereo audioOptions.noise_suppression = true; - rtc::scoped_refptr stream = mPeerConnectionFactory->CreateLocalMediaStream("SLStream"); - + mLocalStream = mPeerConnectionFactory->CreateLocalMediaStream("SLStream"); rtc::scoped_refptr audio_track( mPeerConnectionFactory->CreateAudioTrack("SLAudio", mPeerConnectionFactory->CreateAudioSource(audioOptions).get())); audio_track->set_enabled(true); - stream->AddTrack(audio_track); + mLocalStream->AddTrack(audio_track); mPeerConnection->AddTrack(audio_track, {"SLStream"}); @@ -545,6 +540,7 @@ bool LLWebRTCPeerConnectionImpl::initializeConnection() params.codecs.push_back(codecparam); receiver->SetParameters(params); } + webrtc::PeerConnectionInterface::RTCOfferAnswerOptions offerOptions; mPeerConnection->CreateOffer(this, offerOptions); @@ -606,13 +602,47 @@ void LLWebRTCPeerConnectionImpl::AnswerAvailable(const std::string &sdp) void LLWebRTCPeerConnectionImpl::setMute(bool mute) { mMute = mute; - auto senders = mPeerConnection->GetSenders(); - - RTC_LOG(LS_INFO) << __FUNCTION__ << (mute ? "disabling" : "enabling") << " streams count " << senders.size(); + if (mPeerConnection) + { + auto senders = mPeerConnection->GetSenders(); + + RTC_LOG(LS_INFO) << __FUNCTION__ << (mute ? "disabling" : "enabling") << " streams count " << senders.size(); + for (auto &sender : senders) + { + auto track = sender->track(); + if(track) + { + track->set_enabled(!mMute); + } + } + } +} + +void LLWebRTCPeerConnectionImpl::setReceiveVolume(float volume) +{ + auto receivers = mPeerConnection->GetReceivers(); - for (auto &sender : senders) + for (auto &receiver : receivers) { - sender->track()->set_enabled(!mMute); + for (auto& stream : receiver->streams()) + { + for (auto& track : stream->GetAudioTracks()) + { + track->GetSource()->SetVolume(volume); + } + } + } +} + +void LLWebRTCPeerConnectionImpl::setSendVolume(float volume) +{ + if (mLocalStream) + { + auto track = mLocalStream->FindAudioTrack("SLStream"); + if (track) + { + track->GetSource()->SetVolume(volume); + } } } diff --git a/indra/llwebrtc/llwebrtc.h b/indra/llwebrtc/llwebrtc.h index ed80fa5648..9766f2f231 100644 --- a/indra/llwebrtc/llwebrtc.h +++ b/indra/llwebrtc/llwebrtc.h @@ -85,10 +85,6 @@ class LLWebRTCDeviceInterface virtual void setTuningMode(bool enable) = 0; virtual float getTuningAudioLevel() = 0; - - virtual void setSpeakerVolume(float volume) = 0; // volume between 0.0 and 1.0 - virtual void setMicrophoneVolume(float volume) = 0; // volume between 0.0 and 1.0 - virtual void setMute(bool mute) = 0; virtual float getPeerAudioLevel() = 0; }; @@ -96,6 +92,8 @@ class LLWebRTCAudioInterface { public: virtual void setMute(bool mute) = 0; + virtual void setReceiveVolume(float volume) = 0; // volume between 0.0 and 1.0 + virtual void setSendVolume(float volume) = 0; // volume between 0.0 and 1.0 }; class LLWebRTCDataObserver diff --git a/indra/llwebrtc/llwebrtc_impl.h b/indra/llwebrtc/llwebrtc_impl.h index 76e29c63fb..efb0d3add3 100644 --- a/indra/llwebrtc/llwebrtc_impl.h +++ b/indra/llwebrtc/llwebrtc_impl.h @@ -116,10 +116,6 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface void setTuningMode(bool enable) override; float getTuningAudioLevel() override; float getPeerAudioLevel() override; - - void setSpeakerVolume(float volume) override; // range 0.0-1.0 - void setMicrophoneVolume(float volume) override; // range 0.0-1.0 - void setMute(bool mute) override; // // Helpers @@ -224,12 +220,13 @@ class LLWebRTCPeerConnectionImpl : public LLWebRTCPeerConnection, void shutdownConnection() override; void AnswerAvailable(const std::string &sdp) override; - // // LLWebRTCAudioInterface // void setMute(bool mute) override; - + void setReceiveVolume(float volume) override; // volume between 0.0 and 1.0 + void setSendVolume(float volume) override; // volume between 0.0 and 1.0 + // // LLWebRTCDataInterface // @@ -291,6 +288,7 @@ class LLWebRTCPeerConnectionImpl : public LLWebRTCPeerConnection, bool mAnswerReceived; rtc::scoped_refptr mPeerConnection; + rtc::scoped_refptr mLocalStream; std::vector mDataObserverList; rtc::scoped_refptr mDataChannel; -- cgit v1.2.3 From e5a95a96bface28bbd37afe655209d04f03cd250 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Fri, 1 Dec 2023 01:28:38 -0800 Subject: Setting volume for remote stream needs to happen in signaling thread --- indra/llwebrtc/llwebrtc.cpp | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) (limited to 'indra/llwebrtc') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index 4741ed92a3..2bcbb135c0 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -620,26 +620,29 @@ void LLWebRTCPeerConnectionImpl::setMute(bool mute) void LLWebRTCPeerConnectionImpl::setReceiveVolume(float volume) { - auto receivers = mPeerConnection->GetReceivers(); - - for (auto &receiver : receivers) - { - for (auto& stream : receiver->streams()) + mWebRTCImpl->PostSignalingTask( + [this, volume]() { - for (auto& track : stream->GetAudioTracks()) + auto receivers = mPeerConnection->GetReceivers(); + + for (auto &receiver : receivers) { - track->GetSource()->SetVolume(volume); + for (auto &stream : receiver->streams()) + { + for (auto &track : stream->GetAudioTracks()) + { + track->GetSource()->SetVolume(volume); + } + } } - } - } + }); } void LLWebRTCPeerConnectionImpl::setSendVolume(float volume) { if (mLocalStream) { - auto track = mLocalStream->FindAudioTrack("SLStream"); - if (track) + for (auto &track : mLocalStream->GetAudioTracks()) { track->GetSource()->SetVolume(volume); } -- cgit v1.2.3 From 4f4f9032cb56712b2b514b5eea46ddda54b4094b Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Mon, 4 Dec 2023 14:22:40 -0800 Subject: Mute using enable. Muting using the device module microphone mute was muting other applications, speakers, and so on. Instead, we mute by enabling/disabling the input and output streams. --- indra/llwebrtc/llwebrtc.cpp | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) (limited to 'indra/llwebrtc') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index 2bcbb135c0..9fd8fd8ee6 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -287,16 +287,11 @@ void LLWebRTCImpl::setRenderDevice(const std::string &id) break; } } - mTuningDeviceModule->SetSpeakerMute(true); bool was_tuning_playing = mTuningDeviceModule->Playing(); if (was_tuning_playing) { mTuningDeviceModule->StopPlayout(); } - if (mPeerDeviceModule) - { - mPeerDeviceModule->SetSpeakerMute(true); - } mTuningDeviceModule->SetPlayoutDevice(tuningPlayoutDevice); mTuningDeviceModule->InitSpeaker(); @@ -326,10 +321,8 @@ void LLWebRTCImpl::setRenderDevice(const std::string &id) mPeerDeviceModule->InitSpeaker(); mPeerDeviceModule->InitPlayout(); mPeerDeviceModule->StartPlayout(); - mPeerDeviceModule->SetSpeakerMute(false); } - mTuningDeviceModule->SetSpeakerMute(false); }); } @@ -369,29 +362,25 @@ void LLWebRTCImpl::setTuningMode(bool enable) { mTuningDeviceModule->StartRecording(); - mTuningDeviceModule->SetMicrophoneMute(false); - - - mTuningDeviceModule->SetSpeakerMute(false); if (mPeerDeviceModule) { mPeerDeviceModule->StopRecording(); - mPeerDeviceModule->SetSpeakerMute(true); } } else { + mTuningDeviceModule->StartRecording(); if (mPeerDeviceModule) { mPeerDeviceModule->StartRecording(); - mPeerDeviceModule->SetSpeakerMute(false); } } }); for (auto& connection : mPeerConnections) { - connection->enableTracks(enable ? false : !mMute); + connection->enableSenderTracks(enable ? false : !mMute); + connection->enableReceiverTracks(!enable); } } @@ -409,7 +398,7 @@ LLWebRTCPeerConnection * LLWebRTCImpl::newPeerConnection() peerConnection->init(this); mPeerConnections.emplace_back(peerConnection); - peerConnection->enableTracks(!mMute); + peerConnection->enableSenderTracks(!mMute); return peerConnection.get(); } @@ -570,7 +559,7 @@ void LLWebRTCPeerConnectionImpl::shutdownConnection() }); } -void LLWebRTCPeerConnectionImpl::enableTracks(bool enable) +void LLWebRTCPeerConnectionImpl::enableSenderTracks(bool enable) { // set_enabled shouldn't be done on the worker thread if (mPeerConnection) @@ -583,6 +572,19 @@ void LLWebRTCPeerConnectionImpl::enableTracks(bool enable) } } +void LLWebRTCPeerConnectionImpl::enableReceiverTracks(bool enable) +{ + // set_enabled shouldn't be done on the worker thread + if (mPeerConnection) + { + auto receivers = mPeerConnection->GetReceivers(); + for (auto &receiver : receivers) + { + receiver->track()->set_enabled(enable); + } + } +} + void LLWebRTCPeerConnectionImpl::AnswerAvailable(const std::string &sdp) { RTC_LOG(LS_INFO) << __FUNCTION__ << " Remote SDP: " << sdp; -- cgit v1.2.3 From dd250273f476a96e60e7953915d76f6e2dd765ec Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Mon, 4 Dec 2023 14:26:12 -0800 Subject: missed file --- indra/llwebrtc/llwebrtc_impl.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'indra/llwebrtc') diff --git a/indra/llwebrtc/llwebrtc_impl.h b/indra/llwebrtc/llwebrtc_impl.h index efb0d3add3..cc22c30f35 100644 --- a/indra/llwebrtc/llwebrtc_impl.h +++ b/indra/llwebrtc/llwebrtc_impl.h @@ -273,7 +273,8 @@ class LLWebRTCPeerConnectionImpl : public LLWebRTCPeerConnection, void OnMessage(const webrtc::DataBuffer& buffer) override; // Helpers - void enableTracks(bool enable); + void enableSenderTracks(bool enable); + void enableReceiverTracks(bool enable); protected: -- cgit v1.2.3 From b58342a4e08c3adecb9da6d53aae791cc29a917b Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Thu, 7 Dec 2023 19:47:37 -0800 Subject: Rework VU meter level processing to be closer to Vivox --- indra/llwebrtc/llwebrtc.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'indra/llwebrtc') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index 9fd8fd8ee6..32a75d3675 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -54,7 +54,7 @@ void LLAudioDeviceObserver::OnCaptureData(const void *audio_samples, const short *samples = (const short *) audio_samples; for (size_t index = 0; index < num_samples * num_channels; index++) { - float sample = (static_cast(samples[index]) / (float) 32768); + float sample = (static_cast(samples[index]) / (float) 32767); energy += sample * sample; } @@ -384,9 +384,9 @@ void LLWebRTCImpl::setTuningMode(bool enable) } } -float LLWebRTCImpl::getTuningAudioLevel() { return 20 * mTuningAudioDeviceObserver->getMicrophoneEnergy(); } +float LLWebRTCImpl::getTuningAudioLevel() { return -20 * log10f(mTuningAudioDeviceObserver->getMicrophoneEnergy()); } -float LLWebRTCImpl::getPeerAudioLevel() { return 20 * mPeerAudioDeviceObserver->getMicrophoneEnergy(); } +float LLWebRTCImpl::getPeerAudioLevel() { return -20 * log10f(mPeerAudioDeviceObserver->getMicrophoneEnergy()); } // // Helpers -- cgit v1.2.3 From 2a375d25dc2bb6e063d8e596ce3b4bd2c61cd96b Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Fri, 8 Dec 2023 12:09:02 -0800 Subject: fix device selection (hopefully) --- indra/llwebrtc/llwebrtc.cpp | 17 +++++++++++++++-- indra/llwebrtc/llwebrtc_impl.h | 1 + 2 files changed, 16 insertions(+), 2 deletions(-) (limited to 'indra/llwebrtc') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index 32a75d3675..12997e34d3 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -300,10 +300,10 @@ void LLWebRTCImpl::setRenderDevice(const std::string &id) { mTuningDeviceModule->StartPlayout(); } - renderDeviceCount = mPeerDeviceModule->PlayoutDevices(); if (mPeerDeviceModule) { + renderDeviceCount = mPeerDeviceModule->PlayoutDevices(); for (int16_t i = 0; i < renderDeviceCount; i++) { char name[webrtc::kAdmMaxDeviceNameSize]; @@ -317,6 +317,7 @@ void LLWebRTCImpl::setRenderDevice(const std::string &id) break; } } + mPeerDeviceModule->StopPlayout(); mPeerDeviceModule->SetPlayoutDevice(mPlayoutDevice); mPeerDeviceModule->InitSpeaker(); mPeerDeviceModule->InitPlayout(); @@ -379,7 +380,14 @@ void LLWebRTCImpl::setTuningMode(bool enable) }); for (auto& connection : mPeerConnections) { - connection->enableSenderTracks(enable ? false : !mMute); + if (enable) + { + connection->enableSenderTracks(false); + } + else + { + connection->resetMute(); + } connection->enableReceiverTracks(!enable); } } @@ -620,6 +628,11 @@ void LLWebRTCPeerConnectionImpl::setMute(bool mute) } } +void LLWebRTCPeerConnectionImpl::resetMute() +{ + setMute(mMute); +} + void LLWebRTCPeerConnectionImpl::setReceiveVolume(float volume) { mWebRTCImpl->PostSignalingTask( diff --git a/indra/llwebrtc/llwebrtc_impl.h b/indra/llwebrtc/llwebrtc_impl.h index cc22c30f35..6a84f67ef5 100644 --- a/indra/llwebrtc/llwebrtc_impl.h +++ b/indra/llwebrtc/llwebrtc_impl.h @@ -273,6 +273,7 @@ class LLWebRTCPeerConnectionImpl : public LLWebRTCPeerConnection, void OnMessage(const webrtc::DataBuffer& buffer) override; // Helpers + void resetMute(); void enableSenderTracks(bool enable); void enableReceiverTracks(bool enable); -- cgit v1.2.3 From 9f6d8effb4c5730ff2bbb8f354e341b95824437d Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Tue, 12 Dec 2023 20:46:34 -0800 Subject: Better renegotiation support for parcel voice Better handle starting up and shutting down WebRTC connections simultaneously. --- indra/llwebrtc/llwebrtc.cpp | 84 +++++++++++++++++++++++++-------------------- indra/llwebrtc/llwebrtc.h | 1 + 2 files changed, 48 insertions(+), 37 deletions(-) (limited to 'indra/llwebrtc') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index 12997e34d3..e55d94d128 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -83,9 +83,10 @@ void LLAudioDeviceObserver::OnRenderData(const void *audio_samples, void LLWebRTCImpl::init() { RTC_DCHECK(mPeerConnectionFactory); - mPlayoutDevice = -1; - mRecordingDevice = -1; + mPlayoutDevice = 0; + mRecordingDevice = 0; rtc::InitializeSSL(); + rtc::LogMessage::LogToDebug(rtc::LS_WARNING); mTaskQueueFactory = webrtc::CreateDefaultTaskQueueFactory(); mNetworkThread = rtc::Thread::CreateWithSocketServer(); @@ -145,10 +146,9 @@ void LLWebRTCImpl::init() mPeerDeviceModule->InitSpeaker(); mPeerDeviceModule->InitRecording(); mPeerDeviceModule->InitPlayout(); - mPeerDeviceModule->StartPlayout(); - mPeerDeviceModule->StartRecording(); }); - + + apm->ApplyConfig(apm_config); mPeerConnectionFactory = webrtc::CreatePeerConnectionFactory(mNetworkThread.get(), mWorkerThread.get(), mSignalingThread.get(), @@ -159,7 +159,12 @@ void LLWebRTCImpl::init() nullptr /* video_decoder_factory */, nullptr /* audio_mixer */, apm); - apm->ApplyConfig(apm_config); + mWorkerThread->BlockingCall( + [this]() + { + mPeerDeviceModule->StartPlayout(); + mPeerDeviceModule->StartRecording(); + }); } void LLWebRTCImpl::terminate() @@ -235,15 +240,14 @@ void LLWebRTCImpl::setCaptureDevice(const std::string &id) mTuningDeviceModule->InitMicrophone(); mTuningDeviceModule->InitRecording(); mTuningDeviceModule->StartRecording(); - bool was_peer_recording = false; if (mPeerDeviceModule) { - int16_t captureDeviceCount = mTuningDeviceModule->RecordingDevices(); + int16_t captureDeviceCount = mPeerDeviceModule->RecordingDevices(); for (int16_t i = 0; i < captureDeviceCount; i++) { char name[webrtc::kAdmMaxDeviceNameSize]; char guid[webrtc::kAdmMaxGuidSize]; - mTuningDeviceModule->RecordingDeviceName(i, name, guid); + mPeerDeviceModule->RecordingDeviceName(i, name, guid); if (id == guid || id == "Default") // first one in list is default { RTC_LOG(LS_INFO) @@ -252,11 +256,13 @@ void LLWebRTCImpl::setCaptureDevice(const std::string &id) break; } } - was_peer_recording = mPeerDeviceModule->Recording(); + bool was_peer_recording = mPeerDeviceModule->Recording(); if (was_peer_recording) { mPeerDeviceModule->StopRecording(); } + + mPeerDeviceModule->StopRecording(); mPeerDeviceModule->SetRecordingDevice(mRecordingDevice); mPeerDeviceModule->InitMicrophone(); mPeerDeviceModule->InitRecording(); @@ -432,7 +438,15 @@ void LLWebRTCPeerConnectionImpl::init(LLWebRTCImpl * webrtc_impl) } void LLWebRTCPeerConnectionImpl::terminate() { - shutdownConnection(); + mWebRTCImpl->SignalingBlockingCall( + [this]() + { + if (mPeerConnection) + { + mPeerConnection->Close(); + mPeerConnection = nullptr; + } + }); } void LLWebRTCPeerConnectionImpl::setSignalingObserver(LLWebRTCSignalingObserver *observer) { mSignalingObserverList.emplace_back(observer); } @@ -477,7 +491,6 @@ bool LLWebRTCPeerConnectionImpl::initializeConnection() else { RTC_LOG(LS_ERROR) << __FUNCTION__ << "Error creating peer connection: " << error_or_peer_connection.error().message(); - shutdownConnection(); return false; } @@ -546,25 +559,19 @@ bool LLWebRTCPeerConnectionImpl::initializeConnection() void LLWebRTCPeerConnectionImpl::shutdownConnection() { - mWebRTCImpl->SignalingBlockingCall( - [this]() - { - if (mPeerConnection) - { - mPeerConnection->Close(); - mPeerConnection = nullptr; - } - }); - - mWebRTCImpl->NetworkBlockingCall( - [this]() - { - if (mDataChannel) - { - mDataChannel->Close(); - mDataChannel = nullptr; - } - }); + mWebRTCImpl->PostSignalingTask( + [this]() + { + if (mPeerConnection) + { + mPeerConnection->Close(); + mPeerConnection = nullptr; + } + for (auto &observer : mSignalingObserverList) + { + observer->OnPeerShutDown(); + } + }); } void LLWebRTCPeerConnectionImpl::enableSenderTracks(bool enable) @@ -638,15 +645,18 @@ void LLWebRTCPeerConnectionImpl::setReceiveVolume(float volume) mWebRTCImpl->PostSignalingTask( [this, volume]() { - auto receivers = mPeerConnection->GetReceivers(); - - for (auto &receiver : receivers) + if (mPeerConnection) { - for (auto &stream : receiver->streams()) + auto receivers = mPeerConnection->GetReceivers(); + + for (auto &receiver : receivers) { - for (auto &track : stream->GetAudioTracks()) + for (auto &stream : receiver->streams()) { - track->GetSource()->SetVolume(volume); + for (auto &track : stream->GetAudioTracks()) + { + track->GetSource()->SetVolume(volume); + } } } } diff --git a/indra/llwebrtc/llwebrtc.h b/indra/llwebrtc/llwebrtc.h index 9766f2f231..f84e998245 100644 --- a/indra/llwebrtc/llwebrtc.h +++ b/indra/llwebrtc/llwebrtc.h @@ -125,6 +125,7 @@ class LLWebRTCSignalingObserver virtual void OnRenegotiationNeeded() = 0; virtual void OnAudioEstablished(LLWebRTCAudioInterface *audio_interface) = 0; virtual void OnDataChannelReady(LLWebRTCDataInterface *data_interface) = 0; + virtual void OnPeerShutDown() = 0; }; class LLWebRTCPeerConnection -- cgit v1.2.3 From e9d30fda25c6104a77f51fb9db86f7933f56b22b Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Mon, 18 Dec 2023 11:45:15 -0800 Subject: Touch up parcel voice enable/disable. --- indra/llwebrtc/llwebrtc.cpp | 203 ++++++++++++++++++++++---------------------- 1 file changed, 102 insertions(+), 101 deletions(-) (limited to 'indra/llwebrtc') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index e55d94d128..9b3ec2889b 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -86,7 +86,9 @@ void LLWebRTCImpl::init() mPlayoutDevice = 0; mRecordingDevice = 0; rtc::InitializeSSL(); - rtc::LogMessage::LogToDebug(rtc::LS_WARNING); + rtc::LogMessage::LogToDebug(rtc::LS_NONE); + rtc::LogMessage::SetLogToStderr(true); + mTaskQueueFactory = webrtc::CreateDefaultTaskQueueFactory(); mNetworkThread = rtc::Thread::CreateWithSocketServer(); @@ -98,7 +100,7 @@ void LLWebRTCImpl::init() mSignalingThread = rtc::Thread::Create(); mSignalingThread->SetName("WebRTCSignalingThread", nullptr); mSignalingThread->Start(); - + mWorkerThread->PostTask( [this]() { @@ -466,93 +468,98 @@ bool LLWebRTCPeerConnectionImpl::initializeConnection() RTC_DCHECK(!mPeerConnection); mAnswerReceived = false; - webrtc::PeerConnectionInterface::RTCConfiguration config; - config.sdp_semantics = webrtc::SdpSemantics::kUnifiedPlan; - webrtc::PeerConnectionInterface::IceServer server; - server.uri = "stun:roxie-turn.staging.secondlife.io:3478"; - config.servers.push_back(server); - server.uri = "stun:stun.l.google.com:19302"; - config.servers.push_back(server); - server.uri = "stun:stun1.l.google.com:19302"; - config.servers.push_back(server); - server.uri = "stun:stun2.l.google.com:19302"; - config.servers.push_back(server); - server.uri = "stun:stun3.l.google.com:19302"; - config.servers.push_back(server); - server.uri = "stun:stun4.l.google.com:19302"; - config.servers.push_back(server); - - webrtc::PeerConnectionDependencies pc_dependencies(this); - auto error_or_peer_connection = mPeerConnectionFactory->CreatePeerConnectionOrError(config, std::move(pc_dependencies)); - if (error_or_peer_connection.ok()) - { - mPeerConnection = std::move(error_or_peer_connection.value()); - } - else - { - RTC_LOG(LS_ERROR) << __FUNCTION__ << "Error creating peer connection: " << error_or_peer_connection.error().message(); - return false; - } - - webrtc::DataChannelInit init; - init.ordered = true; - - auto data_channel_or_error = mPeerConnection->CreateDataChannelOrError("SLData", &init); - if (data_channel_or_error.ok()) - { - mDataChannel = std::move(data_channel_or_error.value()); - - mDataChannel->RegisterObserver(this); - } - - RTC_LOG(LS_INFO) << __FUNCTION__ << " " << mPeerConnection->signaling_state(); - - cricket::AudioOptions audioOptions; - audioOptions.auto_gain_control = true; - audioOptions.echo_cancellation = false; // incompatible with opus stereo - audioOptions.noise_suppression = true; - - mLocalStream = mPeerConnectionFactory->CreateLocalMediaStream("SLStream"); - rtc::scoped_refptr audio_track( - mPeerConnectionFactory->CreateAudioTrack("SLAudio", mPeerConnectionFactory->CreateAudioSource(audioOptions).get())); - audio_track->set_enabled(true); - mLocalStream->AddTrack(audio_track); - - mPeerConnection->AddTrack(audio_track, {"SLStream"}); - - auto senders = mPeerConnection->GetSenders(); - - for (auto &sender : senders) - { - webrtc::RtpParameters params; - webrtc::RtpCodecParameters codecparam; - codecparam.name = "opus"; - codecparam.kind = cricket::MEDIA_TYPE_AUDIO; - codecparam.clock_rate = 48000; - codecparam.num_channels = 2; - codecparam.parameters["stereo"] = "1"; - codecparam.parameters["sprop-stereo"] = "1"; - params.codecs.push_back(codecparam); - sender->SetParameters(params); - } - - auto receivers = mPeerConnection->GetReceivers(); - for (auto &receiver : receivers) - { - webrtc::RtpParameters params; - webrtc::RtpCodecParameters codecparam; - codecparam.name = "opus"; - codecparam.kind = cricket::MEDIA_TYPE_AUDIO; - codecparam.clock_rate = 48000; - codecparam.num_channels = 2; - codecparam.parameters["stereo"] = "1"; - codecparam.parameters["sprop-stereo"] = "1"; - params.codecs.push_back(codecparam); - receiver->SetParameters(params); - } - - webrtc::PeerConnectionInterface::RTCOfferAnswerOptions offerOptions; - mPeerConnection->CreateOffer(this, offerOptions); + mWebRTCImpl->PostSignalingTask( + [this]() + { + webrtc::PeerConnectionInterface::RTCConfiguration config; + config.sdp_semantics = webrtc::SdpSemantics::kUnifiedPlan; + webrtc::PeerConnectionInterface::IceServer server; + server.uri = "stun:roxie-turn.staging.secondlife.io:3478"; + config.servers.push_back(server); + server.uri = "stun:stun.l.google.com:19302"; + config.servers.push_back(server); + server.uri = "stun:stun1.l.google.com:19302"; + config.servers.push_back(server); + server.uri = "stun:stun2.l.google.com:19302"; + config.servers.push_back(server); + server.uri = "stun:stun3.l.google.com:19302"; + config.servers.push_back(server); + server.uri = "stun:stun4.l.google.com:19302"; + config.servers.push_back(server); + + config.set_min_port(60000); + config.set_max_port(60100); + + webrtc::PeerConnectionDependencies pc_dependencies(this); + auto error_or_peer_connection = mPeerConnectionFactory->CreatePeerConnectionOrError(config, std::move(pc_dependencies)); + if (error_or_peer_connection.ok()) + { + mPeerConnection = std::move(error_or_peer_connection.value()); + } + else + { + RTC_LOG(LS_ERROR) << __FUNCTION__ << "Error creating peer connection: " << error_or_peer_connection.error().message(); + return; + } + + webrtc::DataChannelInit init; + init.ordered = true; + + auto data_channel_or_error = mPeerConnection->CreateDataChannelOrError("SLData", &init); + if (data_channel_or_error.ok()) + { + mDataChannel = std::move(data_channel_or_error.value()); + + mDataChannel->RegisterObserver(this); + } + + cricket::AudioOptions audioOptions; + audioOptions.auto_gain_control = true; + audioOptions.echo_cancellation = false; // incompatible with opus stereo + audioOptions.noise_suppression = true; + + mLocalStream = mPeerConnectionFactory->CreateLocalMediaStream("SLStream"); + rtc::scoped_refptr audio_track( + mPeerConnectionFactory->CreateAudioTrack("SLAudio", mPeerConnectionFactory->CreateAudioSource(audioOptions).get())); + audio_track->set_enabled(true); + mLocalStream->AddTrack(audio_track); + + mPeerConnection->AddTrack(audio_track, {"SLStream"}); + + auto senders = mPeerConnection->GetSenders(); + + for (auto &sender : senders) + { + webrtc::RtpParameters params; + webrtc::RtpCodecParameters codecparam; + codecparam.name = "opus"; + codecparam.kind = cricket::MEDIA_TYPE_AUDIO; + codecparam.clock_rate = 48000; + codecparam.num_channels = 2; + codecparam.parameters["stereo"] = "1"; + codecparam.parameters["sprop-stereo"] = "1"; + params.codecs.push_back(codecparam); + sender->SetParameters(params); + } + + auto receivers = mPeerConnection->GetReceivers(); + for (auto &receiver : receivers) + { + webrtc::RtpParameters params; + webrtc::RtpCodecParameters codecparam; + codecparam.name = "opus"; + codecparam.kind = cricket::MEDIA_TYPE_AUDIO; + codecparam.clock_rate = 48000; + codecparam.num_channels = 2; + codecparam.parameters["stereo"] = "1"; + codecparam.parameters["sprop-stereo"] = "1"; + params.codecs.push_back(codecparam); + receiver->SetParameters(params); + } + + webrtc::PeerConnectionInterface::RTCOfferAnswerOptions offerOptions; + mPeerConnection->CreateOffer(this, offerOptions); + }); return true; } @@ -839,7 +846,7 @@ void LLWebRTCPeerConnectionImpl::OnIceCandidate(const webrtc::IceCandidateInterf else { mCachedIceCandidates.push_back( - webrtc::CreateIceCandidate(candidate->sdp_mid(), candidate->sdp_mline_index(), candidate->candidate())); + webrtc::CreateIceCandidate(candidate->sdp_mid(), candidate->sdp_mline_index(), candidate->candidate())); } } @@ -878,21 +885,16 @@ void LLWebRTCPeerConnectionImpl::OnSuccess(webrtc::SessionDescriptionInterface * sdp_mangled_stream << sdp_line << "\n"; } } - - webrtc::CreateSessionDescription(webrtc::SdpType::kOffer, sdp_mangled_stream.str()); - - - - mPeerConnection->SetLocalDescription(std::unique_ptr( - webrtc::CreateSessionDescription(webrtc::SdpType::kOffer, sdp_mangled_stream.str())), - rtc::scoped_refptr(this)); + RTC_LOG(LS_INFO) << __FUNCTION__ << " Local SDP: " << sdp_mangled_stream.str(); - - + for (auto &observer : mSignalingObserverList) { observer->OnOfferAvailable(sdp_mangled_stream.str()); } + + mPeerConnection->SetLocalDescription(std::unique_ptr(webrtc::CreateSessionDescription(webrtc::SdpType::kOffer, sdp_mangled_stream.str())), + rtc::scoped_refptr(this)); } void LLWebRTCPeerConnectionImpl::OnFailure(webrtc::RTCError error) { RTC_LOG(LS_ERROR) << ToString(error.type()) << ": " << error.message(); } @@ -930,7 +932,6 @@ void LLWebRTCPeerConnectionImpl::OnSetRemoteDescriptionComplete(webrtc::RTCError // void LLWebRTCPeerConnectionImpl::OnSetLocalDescriptionComplete(webrtc::RTCError error) { - } // -- cgit v1.2.3 From 9b2362f73ee5796279acc717fcc0bcc138f6519f Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Thu, 4 Jan 2024 13:02:55 -0800 Subject: Enable AEC --- indra/llwebrtc/llwebrtc.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'indra/llwebrtc') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index 9b3ec2889b..8e56f9c222 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -110,7 +110,7 @@ void LLWebRTCImpl::init() mTaskQueueFactory.get(), std::unique_ptr(mTuningAudioDeviceObserver)); mTuningDeviceModule->Init(); - mTuningDeviceModule->SetStereoRecording(false); + mTuningDeviceModule->SetStereoRecording(true); mTuningDeviceModule->SetStereoPlayout(true); mTuningDeviceModule->EnableBuiltInAEC(false); updateDevices(); @@ -118,7 +118,7 @@ void LLWebRTCImpl::init() rtc::scoped_refptr apm = webrtc::AudioProcessingBuilder().Create(); webrtc::AudioProcessing::Config apm_config; - apm_config.echo_canceller.enabled = false; + apm_config.echo_canceller.enabled = true; apm_config.echo_canceller.mobile_mode = false; apm_config.gain_controller1.enabled = true; apm_config.gain_controller1.mode = webrtc::AudioProcessing::Config::GainController1::kAdaptiveAnalog; @@ -141,7 +141,7 @@ void LLWebRTCImpl::init() mPeerDeviceModule->Init(); mPeerDeviceModule->SetPlayoutDevice(mPlayoutDevice); mPeerDeviceModule->SetRecordingDevice(mRecordingDevice); - mPeerDeviceModule->SetStereoRecording(false); + mPeerDeviceModule->SetStereoRecording(true); mPeerDeviceModule->SetStereoPlayout(true); mPeerDeviceModule->EnableBuiltInAEC(false); mPeerDeviceModule->InitMicrophone(); @@ -515,7 +515,7 @@ bool LLWebRTCPeerConnectionImpl::initializeConnection() cricket::AudioOptions audioOptions; audioOptions.auto_gain_control = true; - audioOptions.echo_cancellation = false; // incompatible with opus stereo + audioOptions.echo_cancellation = true; // incompatible with opus stereo audioOptions.noise_suppression = true; mLocalStream = mPeerConnectionFactory->CreateLocalMediaStream("SLStream"); @@ -878,7 +878,7 @@ void LLWebRTCPeerConnectionImpl::OnSuccess(webrtc::SessionDescriptionInterface * else if (sdp_line.find("a=fmtp:" + opus_payload) == 0) { sdp_mangled_stream << sdp_line << "a=fmtp:" << opus_payload - << " minptime=10;useinbandfec=1;stereo=1;sprop-stereo=1;maxplaybackrate=48000\n"; + << " minptime=10;useinbandfec=1;stereo=1;sprop-stereo=1;maxplaybackrate=48000;sprop-maxcapturerate=48000;sprop-maxplaybackrate=48000\n"; } else { -- cgit v1.2.3 From 106a589dee7bd91f8c8893e1f077a25efb7e83eb Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Tue, 9 Jan 2024 11:57:01 -0800 Subject: New WebRTC with echo cancellation fix. Also, start/stop recording depending on whether WebRTC has negotiated. --- indra/llwebrtc/llwebrtc.cpp | 74 ++++++++++++++++++++++++++++++++++-------- indra/llwebrtc/llwebrtc_impl.h | 2 ++ 2 files changed, 62 insertions(+), 14 deletions(-) (limited to 'indra/llwebrtc') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index 8e56f9c222..fca490e8c2 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -150,7 +150,34 @@ void LLWebRTCImpl::init() mPeerDeviceModule->InitPlayout(); }); - apm->ApplyConfig(apm_config); + rtc::scoped_refptr apm = webrtc::AudioProcessingBuilder().Create(); + webrtc::AudioProcessing::Config apm_config; + apm_config.echo_canceller.enabled = true; + apm_config.echo_canceller.mobile_mode = false; + apm_config.gain_controller1.enabled = true; + apm_config.gain_controller1.mode = webrtc::AudioProcessing::Config::GainController1::kAdaptiveAnalog; + apm_config.gain_controller2.enabled = true; + apm_config.high_pass_filter.enabled = true; + apm_config.noise_suppression.enabled = true; + apm_config.noise_suppression.level = webrtc::AudioProcessing::Config::NoiseSuppression::kVeryHigh; + apm_config.transient_suppression.enabled = true; + apm_config.pipeline.multi_channel_render = true; + apm_config.pipeline.multi_channel_capture = true; + apm_config.pipeline.multi_channel_capture = true; + + webrtc::ProcessingConfig processing_config; + processing_config.input_stream().set_num_channels(2); + processing_config.input_stream().set_sample_rate_hz(8000); + processing_config.output_stream().set_num_channels(2); + processing_config.output_stream().set_sample_rate_hz(8000); + processing_config.reverse_input_stream().set_num_channels(2); + processing_config.reverse_input_stream().set_sample_rate_hz(48000); + processing_config.reverse_output_stream().set_num_channels(2); + processing_config.reverse_output_stream().set_sample_rate_hz(48000); + + apm->Initialize(processing_config); + apm->ApplyConfig(apm_config); + mPeerConnectionFactory = webrtc::CreatePeerConnectionFactory(mNetworkThread.get(), mWorkerThread.get(), mSignalingThread.get(), @@ -161,11 +188,28 @@ void LLWebRTCImpl::init() nullptr /* video_decoder_factory */, nullptr /* audio_mixer */, apm); + + mWorkerThread->BlockingCall( [this]() { mPeerDeviceModule->StartPlayout(); - mPeerDeviceModule->StartRecording(); + }); +} + +void LLWebRTCImpl::setRecording(bool recording) +{ + mWorkerThread->PostTask( + [this, recording]() + { + if (recording) + { + mPeerDeviceModule->StartRecording(); + } + else + { + mPeerDeviceModule->StopRecording(); + } }); } @@ -263,8 +307,6 @@ void LLWebRTCImpl::setCaptureDevice(const std::string &id) { mPeerDeviceModule->StopRecording(); } - - mPeerDeviceModule->StopRecording(); mPeerDeviceModule->SetRecordingDevice(mRecordingDevice); mPeerDeviceModule->InitMicrophone(); mPeerDeviceModule->InitRecording(); @@ -427,6 +469,10 @@ void LLWebRTCImpl::freePeerConnection(LLWebRTCPeerConnection * peer_connection) (*it)->terminate(); mPeerConnections.erase(it); } + if (mPeerConnections.empty()) + { + setRecording(false); + } } // @@ -519,6 +565,7 @@ bool LLWebRTCPeerConnectionImpl::initializeConnection() audioOptions.noise_suppression = true; mLocalStream = mPeerConnectionFactory->CreateLocalMediaStream("SLStream"); + rtc::scoped_refptr audio_track( mPeerConnectionFactory->CreateAudioTrack("SLAudio", mPeerConnectionFactory->CreateAudioSource(audioOptions).get())); audio_track->set_enabled(true); @@ -750,15 +797,14 @@ void LLWebRTCPeerConnectionImpl::OnConnectionChange(webrtc::PeerConnectionInterf { case webrtc::PeerConnectionInterface::PeerConnectionState::kConnected: { - if (new_state == webrtc::PeerConnectionInterface::PeerConnectionState::kConnected) - { - mWebRTCImpl->PostWorkerTask([this]() { - for (auto &observer : mSignalingObserverList) - { - observer->OnAudioEstablished(this); - } - }); - } + mWebRTCImpl->setRecording(true); + + mWebRTCImpl->PostWorkerTask([this]() { + for (auto &observer : mSignalingObserverList) + { + observer->OnAudioEstablished(this); + } + }); break; } case webrtc::PeerConnectionInterface::PeerConnectionState::kFailed: @@ -872,8 +918,8 @@ void LLWebRTCPeerConnectionImpl::OnSuccess(webrtc::SessionDescriptionInterface * // force mono down, stereo up if (std::sscanf(sdp_line.c_str(), "a=rtpmap:%i opus/%i/2", &payload_id, &bandwidth) == 2) { - sdp_mangled_stream << sdp_line << "\n"; opus_payload = std::to_string(payload_id); + sdp_mangled_stream << "a=rtpmap:" << opus_payload << " opus/48000/2" << "\n"; } else if (sdp_line.find("a=fmtp:" + opus_payload) == 0) { diff --git a/indra/llwebrtc/llwebrtc_impl.h b/indra/llwebrtc/llwebrtc_impl.h index 6a84f67ef5..884e107527 100644 --- a/indra/llwebrtc/llwebrtc_impl.h +++ b/indra/llwebrtc/llwebrtc_impl.h @@ -162,6 +162,8 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface LLWebRTCPeerConnection * newPeerConnection(); void freePeerConnection(LLWebRTCPeerConnection * peer_connection); + void setRecording(bool recording); + protected: std::unique_ptr mNetworkThread; -- cgit v1.2.3 From 9389532761ca3987a37a705d534f4ca62baf7697 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Sat, 3 Feb 2024 18:53:04 -0800 Subject: fix locking race condition --- indra/llwebrtc/llwebrtc.cpp | 40 ++++++++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 16 deletions(-) (limited to 'indra/llwebrtc') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index fca490e8c2..900698aa56 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -673,20 +673,24 @@ void LLWebRTCPeerConnectionImpl::AnswerAvailable(const std::string &sdp) void LLWebRTCPeerConnectionImpl::setMute(bool mute) { mMute = mute; - if (mPeerConnection) - { - auto senders = mPeerConnection->GetSenders(); - - RTC_LOG(LS_INFO) << __FUNCTION__ << (mute ? "disabling" : "enabling") << " streams count " << senders.size(); - for (auto &sender : senders) + mWebRTCImpl->PostSignalingTask( + [this]() + { + if (mPeerConnection) { - auto track = sender->track(); - if(track) + auto senders = mPeerConnection->GetSenders(); + + RTC_LOG(LS_INFO) << __FUNCTION__ << (mMute ? "disabling" : "enabling") << " streams count " << senders.size(); + for (auto &sender : senders) { - track->set_enabled(!mMute); + auto track = sender->track(); + if (track) + { + track->set_enabled(!mMute); + } } } - } + }); } void LLWebRTCPeerConnectionImpl::resetMute() @@ -719,13 +723,17 @@ void LLWebRTCPeerConnectionImpl::setReceiveVolume(float volume) void LLWebRTCPeerConnectionImpl::setSendVolume(float volume) { - if (mLocalStream) - { - for (auto &track : mLocalStream->GetAudioTracks()) + mWebRTCImpl->PostSignalingTask( + [this, volume]() { - track->GetSource()->SetVolume(volume); - } - } + if (mLocalStream) + { + for (auto &track : mLocalStream->GetAudioTracks()) + { + track->GetSource()->SetVolume(volume); + } + } + }); } // -- cgit v1.2.3 From 8d414e408e096946b0409e8ca9d5011d64f89671 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Sat, 3 Feb 2024 22:02:48 -0800 Subject: Handle 'device changed' callback --- indra/llwebrtc/llwebrtc.cpp | 22 +++++++++++++++++----- indra/llwebrtc/llwebrtc.h | 9 ++++++--- indra/llwebrtc/llwebrtc_impl.h | 7 ++++++- 3 files changed, 29 insertions(+), 9 deletions(-) (limited to 'indra/llwebrtc') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index 900698aa56..d6eff5e1f2 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -113,6 +113,7 @@ void LLWebRTCImpl::init() mTuningDeviceModule->SetStereoRecording(true); mTuningDeviceModule->SetStereoPlayout(true); mTuningDeviceModule->EnableBuiltInAEC(false); + mTuningDeviceModule->SetAudioDeviceSink(this); updateDevices(); }); @@ -379,31 +380,42 @@ void LLWebRTCImpl::setRenderDevice(const std::string &id) void LLWebRTCImpl::updateDevices() { - int16_t renderDeviceCount = mTuningDeviceModule->PlayoutDevices(); + int16_t renderDeviceCount = mTuningDeviceModule->PlayoutDevices(); + int16_t currentRenderDeviceIndex = mTuningDeviceModule->GetPlayoutDevice(); + LLWebRTCVoiceDeviceList renderDeviceList; for (int16_t index = 0; index < renderDeviceCount; index++) { char name[webrtc::kAdmMaxDeviceNameSize]; char guid[webrtc::kAdmMaxGuidSize]; mTuningDeviceModule->PlayoutDeviceName(index, name, guid); - renderDeviceList.emplace_back(name, guid); + renderDeviceList.emplace_back(name, guid, index == currentRenderDeviceIndex); } - int16_t captureDeviceCount = mTuningDeviceModule->RecordingDevices(); + int16_t captureDeviceCount = mTuningDeviceModule->RecordingDevices(); + int16_t currentCaptureDeviceIndex = mTuningDeviceModule->GetRecordingDevice(); + LLWebRTCVoiceDeviceList captureDeviceList; for (int16_t index = 0; index < captureDeviceCount; index++) { char name[webrtc::kAdmMaxDeviceNameSize]; char guid[webrtc::kAdmMaxGuidSize]; mTuningDeviceModule->RecordingDeviceName(index, name, guid); - captureDeviceList.emplace_back(name, guid); + captureDeviceList.emplace_back(name, guid, index == currentCaptureDeviceIndex); } for (auto &observer : mVoiceDevicesObserverList) { - observer->OnDevicesChanged(renderDeviceList, captureDeviceList); + observer->OnDevicesChanged(renderDeviceList, + captureDeviceList); } } +void LLWebRTCImpl::OnDevicesUpdated() +{ + updateDevices(); +} + + void LLWebRTCImpl::setTuningMode(bool enable) { mWorkerThread->BlockingCall( diff --git a/indra/llwebrtc/llwebrtc.h b/indra/llwebrtc/llwebrtc.h index f84e998245..e7effdfbb8 100644 --- a/indra/llwebrtc/llwebrtc.h +++ b/indra/llwebrtc/llwebrtc.h @@ -56,11 +56,14 @@ class LLWebRTCVoiceDevice { public: std::string display_name; // friendly value for the user - std::string id; // internal value for selection + std::string id; // internal value for selection + bool current; // current device - LLWebRTCVoiceDevice(const std::string &display_name, const std::string &id) : + LLWebRTCVoiceDevice(const std::string &display_name, const std::string &id, bool current) : display_name(display_name), - id(id) {}; + id(id), + current(current) + {}; }; typedef std::vector LLWebRTCVoiceDeviceList; diff --git a/indra/llwebrtc/llwebrtc_impl.h b/indra/llwebrtc/llwebrtc_impl.h index 884e107527..3ccb6058ad 100644 --- a/indra/llwebrtc/llwebrtc_impl.h +++ b/indra/llwebrtc/llwebrtc_impl.h @@ -89,7 +89,7 @@ class LLAudioDeviceObserver : public webrtc::AudioDeviceDataObserver float mMicrophoneEnergy; }; -class LLWebRTCImpl : public LLWebRTCDeviceInterface +class LLWebRTCImpl : public LLWebRTCDeviceInterface, public webrtc::AudioDeviceSink { public: LLWebRTCImpl() : @@ -117,6 +117,11 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface float getTuningAudioLevel() override; float getPeerAudioLevel() override; + // + // AudioDeviceSink + // + void OnDevicesUpdated() override; + // // Helpers // -- cgit v1.2.3 From dfa77d942c52ddf55942afb497a64fbb1d2fccc0 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Mon, 5 Feb 2024 14:42:21 -0800 Subject: Use a custom audio processor to pull data for level determinations, which will happen after AGC --- indra/llwebrtc/llwebrtc.cpp | 83 +++++++++++++++++++++++++----------------- indra/llwebrtc/llwebrtc_impl.h | 37 +++++++++---------- 2 files changed, 68 insertions(+), 52 deletions(-) (limited to 'indra/llwebrtc') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index d6eff5e1f2..f0d7f6ad8f 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -27,6 +27,7 @@ #include "llwebrtc_impl.h" #include #include +#include #include "api/audio_codecs/audio_decoder_factory.h" #include "api/audio_codecs/audio_encoder_factory.h" @@ -34,30 +35,54 @@ #include "api/audio_codecs/builtin_audio_encoder_factory.h" #include "api/media_stream_interface.h" #include "api/media_stream_track.h" +#include "modules/audio_processing/audio_buffer.h" namespace llwebrtc { -const float VOLUME_SCALE_WEBRTC = 100; - -LLAudioDeviceObserver::LLAudioDeviceObserver() : mMicrophoneEnergy(0.0), mSumVector {0} {} +LLCustomProcessor::LLCustomProcessor() : + mSampleRateHz(0), + mNumChannels(0) +{ + memset(mSumVector, sizeof(mSumVector), 0); +} -float LLAudioDeviceObserver::getMicrophoneEnergy() { return mMicrophoneEnergy; } +void LLCustomProcessor::Initialize(int sample_rate_hz, int num_channels) +{ + mSampleRateHz = sample_rate_hz; + mNumChannels = num_channels; + memset(mSumVector, sizeof(mSumVector), 0); +} -void LLAudioDeviceObserver::OnCaptureData(const void *audio_samples, - const size_t num_samples, - const size_t bytes_per_sample, - const size_t num_channels, - const uint32_t samples_per_sec) +void LLCustomProcessor::Process(webrtc::AudioBuffer *audio_in) { + webrtc::StreamConfig stream_config; + stream_config.set_sample_rate_hz(mSampleRateHz); + stream_config.set_num_channels(mNumChannels); + std::vector frame; + std::vector frame_samples; + + if (audio_in->num_channels() < 1 || audio_in->num_frames() < 480) + { + return; + } + + frame_samples.resize(stream_config.num_samples()); + frame.resize(stream_config.num_channels()); + for (size_t ch = 0; ch < stream_config.num_channels(); ++ch) + { + frame[ch] = &(frame_samples)[ch * stream_config.num_frames()]; + } + + audio_in->CopyTo(stream_config, &frame[0]); + float energy = 0; - const short *samples = (const short *) audio_samples; - for (size_t index = 0; index < num_samples * num_channels; index++) + for (size_t index = 0; index < stream_config.num_samples(); index++) { - float sample = (static_cast(samples[index]) / (float) 32767); + float sample = frame_samples[index]; energy += sample * sample; } - + // smooth it. size_t buffer_size = sizeof(mSumVector) / sizeof(mSumVector[0]); float totalSum = 0; @@ -69,15 +94,7 @@ void LLAudioDeviceObserver::OnCaptureData(const void *audio_samples, } mSumVector[i] = energy; totalSum += energy; - mMicrophoneEnergy = std::sqrt(totalSum / (num_samples * buffer_size)); -} - -void LLAudioDeviceObserver::OnRenderData(const void *audio_samples, - const size_t num_samples, - const size_t bytes_per_sample, - const size_t num_channels, - const uint32_t samples_per_sec) -{ + mMicrophoneEnergy = std::sqrt(totalSum / (stream_config.num_samples() * buffer_size)); } void LLWebRTCImpl::init() @@ -104,11 +121,9 @@ void LLWebRTCImpl::init() mWorkerThread->PostTask( [this]() { - mTuningAudioDeviceObserver = new LLAudioDeviceObserver; - mTuningDeviceModule = - webrtc::CreateAudioDeviceWithDataObserver(webrtc::AudioDeviceModule::AudioLayer::kPlatformDefaultAudio, - mTaskQueueFactory.get(), - std::unique_ptr(mTuningAudioDeviceObserver)); + mTuningDeviceModule = webrtc::CreateAudioDeviceWithDataObserver( + webrtc::AudioDeviceModule::AudioLayer::kPlatformDefaultAudio, + mTaskQueueFactory.get(), nullptr); mTuningDeviceModule->Init(); mTuningDeviceModule->SetStereoRecording(true); mTuningDeviceModule->SetStereoPlayout(true); @@ -134,11 +149,9 @@ void LLWebRTCImpl::init() mWorkerThread->BlockingCall( [this]() { - mPeerAudioDeviceObserver = new LLAudioDeviceObserver; mPeerDeviceModule = webrtc::CreateAudioDeviceWithDataObserver(webrtc::AudioDeviceModule::AudioLayer::kPlatformDefaultAudio, - mTaskQueueFactory.get(), - std::unique_ptr(mPeerAudioDeviceObserver)); + mTaskQueueFactory.get(), nullptr); mPeerDeviceModule->Init(); mPeerDeviceModule->SetPlayoutDevice(mPlayoutDevice); mPeerDeviceModule->SetRecordingDevice(mRecordingDevice); @@ -151,7 +164,11 @@ void LLWebRTCImpl::init() mPeerDeviceModule->InitPlayout(); }); - rtc::scoped_refptr apm = webrtc::AudioProcessingBuilder().Create(); + mCustomProcessor = new LLCustomProcessor; + webrtc::AudioProcessingBuilder apb; + apb.SetCapturePostProcessing(std::unique_ptr(mCustomProcessor)); + rtc::scoped_refptr apm = apb.Create(); + webrtc::AudioProcessing::Config apm_config; apm_config.echo_canceller.enabled = true; apm_config.echo_canceller.mobile_mode = false; @@ -454,9 +471,9 @@ void LLWebRTCImpl::setTuningMode(bool enable) } } -float LLWebRTCImpl::getTuningAudioLevel() { return -20 * log10f(mTuningAudioDeviceObserver->getMicrophoneEnergy()); } +float LLWebRTCImpl::getTuningAudioLevel() { return -20 * log10f(mCustomProcessor->getMicrophoneEnergy()); } -float LLWebRTCImpl::getPeerAudioLevel() { return -20 * log10f(mPeerAudioDeviceObserver->getMicrophoneEnergy()); } +float LLWebRTCImpl::getPeerAudioLevel() { return -20 * log10f(mCustomProcessor->getMicrophoneEnergy()); } // // Helpers diff --git a/indra/llwebrtc/llwebrtc_impl.h b/indra/llwebrtc/llwebrtc_impl.h index 3ccb6058ad..419482b751 100644 --- a/indra/llwebrtc/llwebrtc_impl.h +++ b/indra/llwebrtc/llwebrtc_impl.h @@ -65,27 +65,27 @@ namespace llwebrtc class LLWebRTCPeerConnectionImpl; -class LLAudioDeviceObserver : public webrtc::AudioDeviceDataObserver +class LLCustomProcessor : public webrtc::CustomProcessing { public: - LLAudioDeviceObserver(); + LLCustomProcessor(); + ~LLCustomProcessor() override {} - float getMicrophoneEnergy(); + // (Re-) Initializes the submodule. + void Initialize(int sample_rate_hz, int num_channels) override; + // Analyzes the given capture or render signal. + void Process(webrtc::AudioBuffer *audio) override; + // Returns a string representation of the module state. + std::string ToString() const override { return ""; } - void OnCaptureData(const void *audio_samples, - const size_t num_samples, - const size_t bytes_per_sample, - const size_t num_channels, - const uint32_t samples_per_sec) override; - - void OnRenderData(const void *audio_samples, - const size_t num_samples, - const size_t bytes_per_sample, - const size_t num_channels, - const uint32_t samples_per_sec) override; + float getMicrophoneEnergy() { return mMicrophoneEnergy; } protected: - float mSumVector[30]; // 300 ms of smoothing + static const int NUM_PACKETS_TO_FILTER = 30; // 300 ms of smoothing + int mSampleRateHz; + int mNumChannels; + + float mSumVector[NUM_PACKETS_TO_FILTER]; float mMicrophoneEnergy; }; @@ -93,7 +93,7 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, public webrtc::AudioDeviceS { public: LLWebRTCImpl() : - mTuningAudioDeviceObserver(nullptr), mPeerAudioDeviceObserver(nullptr), mMute(true) + mCustomProcessor(nullptr), mMute(true) { } ~LLWebRTCImpl() {} @@ -189,9 +189,8 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, public webrtc::AudioDeviceS int32_t mPlayoutDevice; int32_t mRecordingDevice; bool mMute; - - LLAudioDeviceObserver * mTuningAudioDeviceObserver; - LLAudioDeviceObserver * mPeerAudioDeviceObserver; + + LLCustomProcessor * mCustomProcessor; // peer connections std::vector> mPeerConnections; -- cgit v1.2.3 From e4c3ca53188152bc12a75f807eabef3dc5e9248b Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Mon, 5 Feb 2024 16:47:47 -0800 Subject: put observer-based tuning audio level calculation back --- indra/llwebrtc/llwebrtc.cpp | 53 +++++++++++++++++++++++++++++++++++++----- indra/llwebrtc/llwebrtc_impl.h | 29 +++++++++++++++++++++-- 2 files changed, 74 insertions(+), 8 deletions(-) (limited to 'indra/llwebrtc') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index f0d7f6ad8f..3273786242 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -40,6 +40,46 @@ namespace llwebrtc { +LLAudioDeviceObserver::LLAudioDeviceObserver() : mMicrophoneEnergy(0.0), mSumVector {0} {} + +float LLAudioDeviceObserver::getMicrophoneEnergy() { return mMicrophoneEnergy; } + +void LLAudioDeviceObserver::OnCaptureData(const void *audio_samples, + const size_t num_samples, + const size_t bytes_per_sample, + const size_t num_channels, + const uint32_t samples_per_sec) +{ + float energy = 0; + const short *samples = (const short *) audio_samples; + for (size_t index = 0; index < num_samples * num_channels; index++) + { + float sample = (static_cast(samples[index]) / (float) 32767); + energy += sample * sample; + } + + // smooth it. + size_t buffer_size = sizeof(mSumVector) / sizeof(mSumVector[0]); + float totalSum = 0; + int i; + for (i = 0; i < (buffer_size - 1); i++) + { + mSumVector[i] = mSumVector[i + 1]; + totalSum += mSumVector[i]; + } + mSumVector[i] = energy; + totalSum += energy; + mMicrophoneEnergy = std::sqrt(totalSum / (num_samples * buffer_size)); +} + +void LLAudioDeviceObserver::OnRenderData(const void *audio_samples, + const size_t num_samples, + const size_t bytes_per_sample, + const size_t num_channels, + const uint32_t samples_per_sec) +{ +} + LLCustomProcessor::LLCustomProcessor() : mSampleRateHz(0), mNumChannels(0) @@ -118,12 +158,13 @@ void LLWebRTCImpl::init() mSignalingThread->SetName("WebRTCSignalingThread", nullptr); mSignalingThread->Start(); + mTuningAudioDeviceObserver = new LLAudioDeviceObserver; mWorkerThread->PostTask( [this]() { mTuningDeviceModule = webrtc::CreateAudioDeviceWithDataObserver( - webrtc::AudioDeviceModule::AudioLayer::kPlatformDefaultAudio, - mTaskQueueFactory.get(), nullptr); + webrtc::AudioDeviceModule::AudioLayer::kPlatformDefaultAudio, mTaskQueueFactory.get(), + std::unique_ptr(mTuningAudioDeviceObserver)); mTuningDeviceModule->Init(); mTuningDeviceModule->SetStereoRecording(true); mTuningDeviceModule->SetStereoPlayout(true); @@ -164,9 +205,9 @@ void LLWebRTCImpl::init() mPeerDeviceModule->InitPlayout(); }); - mCustomProcessor = new LLCustomProcessor; + mPeerCustomProcessor = new LLCustomProcessor; webrtc::AudioProcessingBuilder apb; - apb.SetCapturePostProcessing(std::unique_ptr(mCustomProcessor)); + apb.SetCapturePostProcessing(std::unique_ptr(mPeerCustomProcessor)); rtc::scoped_refptr apm = apb.Create(); webrtc::AudioProcessing::Config apm_config; @@ -471,9 +512,9 @@ void LLWebRTCImpl::setTuningMode(bool enable) } } -float LLWebRTCImpl::getTuningAudioLevel() { return -20 * log10f(mCustomProcessor->getMicrophoneEnergy()); } +float LLWebRTCImpl::getTuningAudioLevel() { return -20 * log10f(mTuningAudioDeviceObserver->getMicrophoneEnergy()); } -float LLWebRTCImpl::getPeerAudioLevel() { return -20 * log10f(mCustomProcessor->getMicrophoneEnergy()); } +float LLWebRTCImpl::getPeerAudioLevel() { return -20 * log10f(mPeerCustomProcessor->getMicrophoneEnergy()); } // // Helpers diff --git a/indra/llwebrtc/llwebrtc_impl.h b/indra/llwebrtc/llwebrtc_impl.h index 419482b751..7146267379 100644 --- a/indra/llwebrtc/llwebrtc_impl.h +++ b/indra/llwebrtc/llwebrtc_impl.h @@ -65,6 +65,30 @@ namespace llwebrtc class LLWebRTCPeerConnectionImpl; +class LLAudioDeviceObserver : public webrtc::AudioDeviceDataObserver +{ + public: + LLAudioDeviceObserver(); + + float getMicrophoneEnergy(); + + void OnCaptureData(const void *audio_samples, + const size_t num_samples, + const size_t bytes_per_sample, + const size_t num_channels, + const uint32_t samples_per_sec) override; + + void OnRenderData(const void *audio_samples, + const size_t num_samples, + const size_t bytes_per_sample, + const size_t num_channels, + const uint32_t samples_per_sec) override; + + protected: + float mSumVector[30]; // 300 ms of smoothing + float mMicrophoneEnergy; +}; + class LLCustomProcessor : public webrtc::CustomProcessing { public: @@ -93,7 +117,7 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, public webrtc::AudioDeviceS { public: LLWebRTCImpl() : - mCustomProcessor(nullptr), mMute(true) + mPeerCustomProcessor(nullptr), mMute(true) { } ~LLWebRTCImpl() {} @@ -190,7 +214,8 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, public webrtc::AudioDeviceS int32_t mRecordingDevice; bool mMute; - LLCustomProcessor * mCustomProcessor; + LLAudioDeviceObserver * mTuningAudioDeviceObserver; + LLCustomProcessor * mPeerCustomProcessor; // peer connections std::vector> mPeerConnections; -- cgit v1.2.3 From 5a9f0488f458ade64e3a3f2388d5cd23092eed5d Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Mon, 5 Feb 2024 16:48:43 -0800 Subject: mac build fix --- indra/llwebrtc/llwebrtc.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'indra/llwebrtc') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index 3273786242..06a5c3a085 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -84,14 +84,14 @@ LLCustomProcessor::LLCustomProcessor() : mSampleRateHz(0), mNumChannels(0) { - memset(mSumVector, sizeof(mSumVector), 0); + memset(mSumVector, 0, sizeof(mSumVector)); } void LLCustomProcessor::Initialize(int sample_rate_hz, int num_channels) { mSampleRateHz = sample_rate_hz; mNumChannels = num_channels; - memset(mSumVector, sizeof(mSumVector), 0); + memset(mSumVector, 0, sizeof(mSumVector)); } void LLCustomProcessor::Process(webrtc::AudioBuffer *audio_in) -- cgit v1.2.3 From 6f4ee11ae942d331fcf57db7466989f90bb55ccf Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Tue, 6 Feb 2024 19:12:14 -0800 Subject: race between session established and data channel ready --- indra/llwebrtc/llwebrtc.cpp | 48 ++++++++++++++------------------------------- 1 file changed, 15 insertions(+), 33 deletions(-) (limited to 'indra/llwebrtc') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index 06a5c3a085..ee85a254f2 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -476,40 +476,22 @@ void LLWebRTCImpl::OnDevicesUpdated() void LLWebRTCImpl::setTuningMode(bool enable) { - mWorkerThread->BlockingCall( - [this, enable]() - { - if (enable) - { - - mTuningDeviceModule->StartRecording(); - - if (mPeerDeviceModule) - { - mPeerDeviceModule->StopRecording(); - } - } - else - { - mTuningDeviceModule->StartRecording(); - if (mPeerDeviceModule) - { - mPeerDeviceModule->StartRecording(); - } - } - }); - for (auto& connection : mPeerConnections) - { - if (enable) + mSignalingThread->PostTask( + [this, enable] { - connection->enableSenderTracks(false); - } - else - { - connection->resetMute(); - } - connection->enableReceiverTracks(!enable); - } + for (auto &connection : mPeerConnections) + { + if (enable) + { + connection->enableSenderTracks(false); + } + else + { + connection->resetMute(); + } + connection->enableReceiverTracks(!enable); + } + }); } float LLWebRTCImpl::getTuningAudioLevel() { return -20 * log10f(mTuningAudioDeviceObserver->getMicrophoneEnergy()); } -- cgit v1.2.3 From 56b05eb9a1daef2927579408dc68dded7d0226cb Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Thu, 8 Feb 2024 18:48:36 -0800 Subject: fix rebase issue --- indra/llwebrtc/llwebrtc.cpp | 14 -------------- 1 file changed, 14 deletions(-) (limited to 'indra/llwebrtc') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index ee85a254f2..768405b75f 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -173,20 +173,6 @@ void LLWebRTCImpl::init() updateDevices(); }); - rtc::scoped_refptr apm = webrtc::AudioProcessingBuilder().Create(); - webrtc::AudioProcessing::Config apm_config; - apm_config.echo_canceller.enabled = true; - apm_config.echo_canceller.mobile_mode = false; - apm_config.gain_controller1.enabled = true; - apm_config.gain_controller1.mode = webrtc::AudioProcessing::Config::GainController1::kAdaptiveAnalog; - apm_config.gain_controller2.enabled = true; - apm_config.high_pass_filter.enabled = true; - apm_config.noise_suppression.enabled = true; - apm_config.noise_suppression.level = webrtc::AudioProcessing::Config::NoiseSuppression::kVeryHigh; - apm_config.transient_suppression.enabled = true; - apm_config.pipeline.multi_channel_render = true; - apm_config.pipeline.multi_channel_capture = true; - mWorkerThread->BlockingCall( [this]() { -- cgit v1.2.3 From c827f14f0270a0954fd33e165c2ba94459d9c116 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Thu, 8 Feb 2024 19:02:13 -0800 Subject: another rebase merge issue --- indra/llwebrtc/llwebrtc.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'indra/llwebrtc') diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index 768405b75f..283124e315 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -970,7 +970,7 @@ void LLWebRTCPeerConnectionImpl::OnSuccess(webrtc::SessionDescriptionInterface * else if (sdp_line.find("a=fmtp:" + opus_payload) == 0) { sdp_mangled_stream << sdp_line << "a=fmtp:" << opus_payload - << " minptime=10;useinbandfec=1;stereo=1;sprop-stereo=1;maxplaybackrate=48000;sprop-maxcapturerate=48000;sprop-maxplaybackrate=48000\n"; + << " minptime=10;useinbandfec=1;stereo=1;sprop-stereo=1;maxplaybackrate=48000;sprop-maxplaybackrate=48000;sprop-maxcapturerate=48000\n"; } else { -- cgit v1.2.3