diff options
author | Roxie Linden <roxie@lindenlab.com> | 2023-11-08 10:13:15 -0800 |
---|---|---|
committer | Roxie Linden <roxie@lindenlab.com> | 2024-02-08 18:34:01 -0800 |
commit | 2d20fbbb16a39eb3123b51c4aa3dc6df23474e00 (patch) | |
tree | e1b09dff12fa9eb1ffe14a85b3ac2e1fd47b1b0c | |
parent | 39ce52f8976d10a20aa06f7567855e232cb32be5 (diff) |
SL-20543 - voice over region boundaries.
This commit includes code to allow the llwebrtc.dll/dylib to allow
multiple connections at once.
-rw-r--r-- | indra/llwebrtc/llwebrtc.cpp | 781 | ||||
-rw-r--r-- | indra/llwebrtc/llwebrtc.h | 24 | ||||
-rw-r--r-- | indra/llwebrtc/llwebrtc_impl.h | 144 | ||||
-rw-r--r-- | indra/newview/llvoicewebrtc.cpp | 40 | ||||
-rw-r--r-- | indra/newview/llvoicewebrtc.h | 4 |
5 files changed, 532 insertions, 461 deletions
diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index 194c7cd5d4..6eb3478b59 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -1,6 +1,6 @@ /** - * @file llaccordionctrl.cpp - * @brief Accordion panel implementation + * @file llwebrtc.cpp + * @brief WebRTC interface implementation * * $LicenseInfo:firstyear=2023&license=viewerlgpl$ * Second Life Viewer Source Code @@ -38,7 +38,7 @@ namespace llwebrtc { -const float VOLUME_SCALE_WEBRTC = 3.0f; +const float VOLUME_SCALE_WEBRTC = 100; LLAudioDeviceObserver::LLAudioDeviceObserver() : mMicrophoneEnergy(0.0), mSumVector {0} {} @@ -57,7 +57,7 @@ void LLAudioDeviceObserver::OnCaptureData(const void *audio_samples, float sample = (static_cast<float>(samples[index]) / (float) 32768); energy += sample * sample; } - + // smooth it. size_t buffer_size = sizeof(mSumVector) / sizeof(mSumVector[0]); float totalSum = 0; @@ -82,12 +82,12 @@ void LLAudioDeviceObserver::OnRenderData(const void *audio_samples, void LLWebRTCImpl::init() { + RTC_DCHECK(mPeerConnectionFactory); mPlayoutDevice = -1; mRecordingDevice = -1; - mAnswerReceived = false; rtc::InitializeSSL(); mTaskQueueFactory = webrtc::CreateDefaultTaskQueueFactory(); - + mNetworkThread = rtc::Thread::CreateWithSocketServer(); mNetworkThread->SetName("WebRTCNetworkThread", nullptr); mNetworkThread->Start(); @@ -97,61 +97,101 @@ void LLWebRTCImpl::init() mSignalingThread = rtc::Thread::Create(); mSignalingThread->SetName("WebRTCSignalingThread", nullptr); mSignalingThread->Start(); - + mWorkerThread->PostTask( - [this]() - { - mTuningAudioDeviceObserver = new LLAudioDeviceObserver; - mTuningDeviceModule = - webrtc::CreateAudioDeviceWithDataObserver(webrtc::AudioDeviceModule::AudioLayer::kPlatformDefaultAudio, - mTaskQueueFactory.get(), - std::unique_ptr<webrtc::AudioDeviceDataObserver>(mTuningAudioDeviceObserver)); - mTuningDeviceModule->Init(); - mTuningDeviceModule->SetStereoRecording(false); - mTuningDeviceModule->SetStereoPlayout(true); - mTuningDeviceModule->EnableBuiltInAEC(false); - updateDevices(); - }); + [this]() + { + mTuningAudioDeviceObserver = new LLAudioDeviceObserver; + mTuningDeviceModule = + webrtc::CreateAudioDeviceWithDataObserver(webrtc::AudioDeviceModule::AudioLayer::kPlatformDefaultAudio, + mTaskQueueFactory.get(), + std::unique_ptr<webrtc::AudioDeviceDataObserver>(mTuningAudioDeviceObserver)); + mTuningDeviceModule->Init(); + mTuningDeviceModule->SetStereoRecording(false); + mTuningDeviceModule->SetStereoPlayout(true); + mTuningDeviceModule->EnableBuiltInAEC(false); + updateDevices(); + }); + + rtc::scoped_refptr<webrtc::AudioProcessing> apm = webrtc::AudioProcessingBuilder().Create(); + webrtc::AudioProcessing::Config apm_config; + apm_config.echo_canceller.enabled = false; + apm_config.echo_canceller.mobile_mode = false; + apm_config.gain_controller1.enabled = true; + apm_config.gain_controller1.mode = webrtc::AudioProcessing::Config::GainController1::kAdaptiveAnalog; + apm_config.gain_controller2.enabled = true; + apm_config.high_pass_filter.enabled = true; + apm_config.noise_suppression.enabled = true; + apm_config.noise_suppression.level = webrtc::AudioProcessing::Config::NoiseSuppression::kVeryHigh; + apm_config.transient_suppression.enabled = true; + apm_config.pipeline.multi_channel_render = true; + apm_config.pipeline.multi_channel_capture = true; + + mWorkerThread->BlockingCall( + [this]() + { + mPeerAudioDeviceObserver = new LLAudioDeviceObserver; + mPeerDeviceModule = + webrtc::CreateAudioDeviceWithDataObserver(webrtc::AudioDeviceModule::AudioLayer::kPlatformDefaultAudio, + mTaskQueueFactory.get(), + std::unique_ptr<webrtc::AudioDeviceDataObserver>(mPeerAudioDeviceObserver)); + mPeerDeviceModule->Init(); + mPeerDeviceModule->SetPlayoutDevice(mPlayoutDevice); + mPeerDeviceModule->SetRecordingDevice(mRecordingDevice); + mPeerDeviceModule->SetStereoRecording(false); + mPeerDeviceModule->SetStereoPlayout(true); + mPeerDeviceModule->EnableBuiltInAEC(false); + mPeerDeviceModule->InitMicrophone(); + mPeerDeviceModule->InitSpeaker(); + mPeerDeviceModule->InitRecording(); + mPeerDeviceModule->InitPlayout(); + mPeerDeviceModule->StartPlayout(); + mPeerDeviceModule->StartRecording(); + }); + + mPeerConnectionFactory = webrtc::CreatePeerConnectionFactory(mNetworkThread.get(), + mWorkerThread.get(), + mSignalingThread.get(), + mPeerDeviceModule, + webrtc::CreateBuiltinAudioEncoderFactory(), + webrtc::CreateBuiltinAudioDecoderFactory(), + nullptr /* video_encoder_factory */, + nullptr /* video_decoder_factory */, + nullptr /* audio_mixer */, + apm); + apm->ApplyConfig(apm_config); } void LLWebRTCImpl::terminate() { + for (auto& connection : mPeerConnections) + { + connection->terminate(); + } + mPeerConnections.clear(); + mSignalingThread->BlockingCall( - [this]() - { - if (mPeerConnection) - { - mPeerConnection->Close(); - mPeerConnection = nullptr; - } - mPeerConnectionFactory = nullptr; - }); + [this]() + { + mPeerConnectionFactory = nullptr; + }); mWorkerThread->BlockingCall( - [this]() - { - if (mTuningDeviceModule) - { - mTuningDeviceModule->StopRecording(); - mTuningDeviceModule->Terminate(); - } - if (mPeerDeviceModule) - { - mPeerDeviceModule->StopRecording(); - mPeerDeviceModule->Terminate(); - } - mTuningDeviceModule = nullptr; - mPeerDeviceModule = nullptr; - mTaskQueueFactory = nullptr; - }); - mNetworkThread->BlockingCall( - [this]() - { - if (mDataChannel) - { - mDataChannel->Close(); - mDataChannel = nullptr; - } - }); + [this]() + { + if (mTuningDeviceModule) + { + mTuningDeviceModule->StopRecording(); + mTuningDeviceModule->Terminate(); + } + if (mPeerDeviceModule) + { + mPeerDeviceModule->StopRecording(); + mPeerDeviceModule->Terminate(); + } + mTuningDeviceModule = nullptr; + mPeerDeviceModule = nullptr; + mTaskQueueFactory = nullptr; + }); } void LLWebRTCImpl::refreshDevices() @@ -164,7 +204,7 @@ void LLWebRTCImpl::setDevicesObserver(LLWebRTCDevicesObserver *observer) { mVoic void LLWebRTCImpl::unsetDevicesObserver(LLWebRTCDevicesObserver *observer) { std::vector<LLWebRTCDevicesObserver *>::iterator it = - std::find(mVoiceDevicesObserverList.begin(), mVoiceDevicesObserverList.end(), observer); + std::find(mVoiceDevicesObserverList.begin(), mVoiceDevicesObserverList.end(), observer); if (it != mVoiceDevicesObserverList.end()) { mVoiceDevicesObserverList.erase(it); @@ -174,97 +214,97 @@ void LLWebRTCImpl::unsetDevicesObserver(LLWebRTCDevicesObserver *observer) void LLWebRTCImpl::setCaptureDevice(const std::string &id) { mWorkerThread->PostTask( - [this, id]() - { - int16_t captureDeviceCount = mTuningDeviceModule->RecordingDevices(); - for (int16_t i = 0; i < captureDeviceCount; i++) - { - char name[webrtc::kAdmMaxDeviceNameSize]; - char guid[webrtc::kAdmMaxGuidSize]; - mTuningDeviceModule->RecordingDeviceName(i, name, guid); - if (id == guid || id == "Default") // first one in list is default - { - RTC_LOG(LS_INFO) << __FUNCTION__ << "Set recording device to " << name << " " << guid << " " << i; - mRecordingDevice = i; - break; - } - } - mTuningDeviceModule->StopRecording(); - mTuningDeviceModule->SetRecordingDevice(mRecordingDevice); - mTuningDeviceModule->InitMicrophone(); - mTuningDeviceModule->InitRecording(); - mTuningDeviceModule->StartRecording(); - bool was_peer_recording = false; - if (mPeerDeviceModule) - { - was_peer_recording = mPeerDeviceModule->Recording(); - if (was_peer_recording) - { - mPeerDeviceModule->StopRecording(); - } - mPeerDeviceModule->SetRecordingDevice(mRecordingDevice); - mPeerDeviceModule->InitMicrophone(); - mPeerDeviceModule->InitRecording(); - if (was_peer_recording) - { - mPeerDeviceModule->StartRecording(); - } - } - }); + [this, id]() + { + int16_t captureDeviceCount = mTuningDeviceModule->RecordingDevices(); + for (int16_t i = 0; i < captureDeviceCount; i++) + { + char name[webrtc::kAdmMaxDeviceNameSize]; + char guid[webrtc::kAdmMaxGuidSize]; + mTuningDeviceModule->RecordingDeviceName(i, name, guid); + if (id == guid || id == "Default") // first one in list is default + { + RTC_LOG(LS_INFO) << __FUNCTION__ << "Set recording device to " << name << " " << guid << " " << i; + mRecordingDevice = i; + break; + } + } + mTuningDeviceModule->StopRecording(); + mTuningDeviceModule->SetRecordingDevice(mRecordingDevice); + mTuningDeviceModule->InitMicrophone(); + mTuningDeviceModule->InitRecording(); + mTuningDeviceModule->StartRecording(); + bool was_peer_recording = false; + if (mPeerDeviceModule) + { + was_peer_recording = mPeerDeviceModule->Recording(); + if (was_peer_recording) + { + mPeerDeviceModule->StopRecording(); + } + mPeerDeviceModule->SetRecordingDevice(mRecordingDevice); + mPeerDeviceModule->InitMicrophone(); + mPeerDeviceModule->InitRecording(); + if (was_peer_recording) + { + mPeerDeviceModule->StartRecording(); + } + } + }); } void LLWebRTCImpl::setRenderDevice(const std::string &id) { mWorkerThread->PostTask( - [this, id]() - { - int16_t renderDeviceCount = mTuningDeviceModule->PlayoutDevices(); - for (int16_t i = 0; i < renderDeviceCount; i++) - { - char name[webrtc::kAdmMaxDeviceNameSize]; - char guid[webrtc::kAdmMaxGuidSize]; - mTuningDeviceModule->PlayoutDeviceName(i, name, guid); - if (id == guid || id == "Default") - { - RTC_LOG(LS_INFO) << __FUNCTION__ << "Set recording device to " << name << " " << guid << " " << i; - mPlayoutDevice = i; - break; - } - } - mTuningDeviceModule->SetSpeakerMute(true); - bool was_tuning_playing = mTuningDeviceModule->Playing(); - if (was_tuning_playing) - { - mTuningDeviceModule->StopPlayout(); - } - bool was_peer_mute = false; - if (mPeerDeviceModule) - { - mPeerDeviceModule->SpeakerMute(&was_peer_mute); - if (!was_peer_mute) - { - mPeerDeviceModule->SetSpeakerMute(true); - } - } - - mTuningDeviceModule->SetPlayoutDevice(mPlayoutDevice); - mTuningDeviceModule->InitSpeaker(); - mTuningDeviceModule->InitPlayout(); - if (was_tuning_playing) - { - mTuningDeviceModule->StartPlayout(); - } - if (mPeerDeviceModule) - { - mPeerDeviceModule->SetPlayoutDevice(mPlayoutDevice); - mPeerDeviceModule->InitSpeaker(); - mPeerDeviceModule->InitPlayout(); - mPeerDeviceModule->StartPlayout(); - mPeerDeviceModule->SetSpeakerMute(was_peer_mute); - - } - mTuningDeviceModule->SetSpeakerMute(false); - }); + [this, id]() + { + int16_t renderDeviceCount = mTuningDeviceModule->PlayoutDevices(); + for (int16_t i = 0; i < renderDeviceCount; i++) + { + char name[webrtc::kAdmMaxDeviceNameSize]; + char guid[webrtc::kAdmMaxGuidSize]; + mTuningDeviceModule->PlayoutDeviceName(i, name, guid); + if (id == guid || id == "Default") + { + RTC_LOG(LS_INFO) << __FUNCTION__ << "Set recording device to " << name << " " << guid << " " << i; + mPlayoutDevice = i; + break; + } + } + mTuningDeviceModule->SetSpeakerMute(true); + bool was_tuning_playing = mTuningDeviceModule->Playing(); + if (was_tuning_playing) + { + mTuningDeviceModule->StopPlayout(); + } + bool was_peer_mute = false; + if (mPeerDeviceModule) + { + mPeerDeviceModule->SpeakerMute(&was_peer_mute); + if (!was_peer_mute) + { + mPeerDeviceModule->SetSpeakerMute(true); + } + } + + mTuningDeviceModule->SetPlayoutDevice(mPlayoutDevice); + mTuningDeviceModule->InitSpeaker(); + mTuningDeviceModule->InitPlayout(); + if (was_tuning_playing) + { + mTuningDeviceModule->StartPlayout(); + } + if (mPeerDeviceModule) + { + mPeerDeviceModule->SetPlayoutDevice(mPlayoutDevice); + mPeerDeviceModule->InitSpeaker(); + mPeerDeviceModule->InitPlayout(); + mPeerDeviceModule->StartPlayout(); + mPeerDeviceModule->SetSpeakerMute(was_peer_mute); + + } + mTuningDeviceModule->SetSpeakerMute(false); + }); } void LLWebRTCImpl::updateDevices() @@ -278,7 +318,7 @@ void LLWebRTCImpl::updateDevices() mTuningDeviceModule->PlayoutDeviceName(index, name, guid); renderDeviceList.emplace_back(name, guid); } - + int16_t captureDeviceCount = mTuningDeviceModule->RecordingDevices(); LLWebRTCVoiceDeviceList captureDeviceList; for (int16_t index = 0; index < captureDeviceCount; index++) @@ -297,113 +337,100 @@ void LLWebRTCImpl::updateDevices() void LLWebRTCImpl::setTuningMode(bool enable) { mWorkerThread->BlockingCall( - [this, enable]() - { - if (enable) - { + [this, enable]() + { + if (enable) + { + + mTuningDeviceModule->StartRecording(); + mTuningDeviceModule->SetMicrophoneMute(false); + + + mTuningDeviceModule->SetSpeakerMute(false); + + if (mPeerDeviceModule) + { + mPeerDeviceModule->StopRecording(); + mPeerDeviceModule->SetSpeakerMute(true); + } + } + else + { + if (mPeerDeviceModule) + { + mPeerDeviceModule->StartRecording(); + mPeerDeviceModule->SetSpeakerMute(false); + } + } + }); + for (auto& connection : mPeerConnections) + { + connection->enableTracks(enable ? false : !mMute); + } +} - mTuningDeviceModule->StartRecording(); - mTuningDeviceModule->SetMicrophoneMute(false); +float LLWebRTCImpl::getTuningAudioLevel() { return 20 * mTuningAudioDeviceObserver->getMicrophoneEnergy(); } +float LLWebRTCImpl::getPeerAudioLevel() { return 20 * mPeerAudioDeviceObserver->getMicrophoneEnergy(); } - mTuningDeviceModule->SetSpeakerMute(false); +void LLWebRTCImpl::setSpeakerVolume(float volume) { mPeerDeviceModule->SetSpeakerVolume(volume * VOLUME_SCALE_WEBRTC);} +void LLWebRTCImpl::setMicrophoneVolume(float volume) { mPeerDeviceModule->SetMicrophoneVolume(volume * VOLUME_SCALE_WEBRTC);} - if (mPeerDeviceModule) - { - mPeerDeviceModule->StopRecording(); - mPeerDeviceModule->SetSpeakerMute(true); - } - } - else - { - if (mPeerDeviceModule) - { - mPeerDeviceModule->StartRecording(); - mPeerDeviceModule->SetSpeakerMute(false); - } - } - }); - // set_enabled shouldn't be done on the worker thread - if (mPeerConnection) +// +// Helpers +// + +LLWebRTCPeerConnection * LLWebRTCImpl::newPeerConnection() +{ + rtc::scoped_refptr<LLWebRTCPeerConnectionImpl> peerConnection = rtc::scoped_refptr<LLWebRTCPeerConnectionImpl>(new rtc::RefCountedObject<LLWebRTCPeerConnectionImpl>()); + peerConnection->init(this); + + mPeerConnections.emplace_back(peerConnection); + peerConnection->enableTracks(!mMute); + return peerConnection.get(); +} +void LLWebRTCImpl::freePeerConnection(LLWebRTCPeerConnection * peer_connection) +{ + std::vector<rtc::scoped_refptr<LLWebRTCPeerConnectionImpl>>::iterator it = + std::find(mPeerConnections.begin(), mPeerConnections.end(), peer_connection); + if (it != mPeerConnections.end()) { - auto senders = mPeerConnection->GetSenders(); - for (auto &sender : senders) - { - sender->track()->set_enabled(enable ? false : !mMute); - } + (*it)->terminate(); + mPeerConnections.erase(it); } } // -// LLWebRTCSignalInterface +// LLWebRTCPeerConnection interface // -void LLWebRTCImpl::setSignalingObserver(LLWebRTCSignalingObserver *observer) { mSignalingObserverList.emplace_back(observer); } +void LLWebRTCPeerConnectionImpl::init(LLWebRTCImpl * webrtc_impl) +{ + mWebRTCImpl = webrtc_impl; + mPeerConnectionFactory = mWebRTCImpl->getPeerConnectionFactory(); +} +void LLWebRTCPeerConnectionImpl::terminate() +{ + shutdownConnection(); +} + +void LLWebRTCPeerConnectionImpl::setSignalingObserver(LLWebRTCSignalingObserver *observer) { mSignalingObserverList.emplace_back(observer); } -void LLWebRTCImpl::unsetSignalingObserver(LLWebRTCSignalingObserver *observer) +void LLWebRTCPeerConnectionImpl::unsetSignalingObserver(LLWebRTCSignalingObserver *observer) { std::vector<LLWebRTCSignalingObserver *>::iterator it = - std::find(mSignalingObserverList.begin(), mSignalingObserverList.end(), observer); + std::find(mSignalingObserverList.begin(), mSignalingObserverList.end(), observer); if (it != mSignalingObserverList.end()) { mSignalingObserverList.erase(it); } } -bool LLWebRTCImpl::initializeConnection() +bool LLWebRTCPeerConnectionImpl::initializeConnection() { RTC_DCHECK(!mPeerConnection); - RTC_DCHECK(mPeerConnectionFactory); mAnswerReceived = false; - rtc::scoped_refptr<webrtc::AudioProcessing> apm = webrtc::AudioProcessingBuilder().Create(); - webrtc::AudioProcessing::Config apm_config; - apm_config.echo_canceller.enabled = false; - apm_config.echo_canceller.mobile_mode = false; - apm_config.gain_controller1.enabled = true; - apm_config.gain_controller1.mode = webrtc::AudioProcessing::Config::GainController1::kAdaptiveAnalog; - apm_config.gain_controller2.enabled = true; - apm_config.high_pass_filter.enabled = true; - apm_config.noise_suppression.enabled = true; - apm_config.noise_suppression.level = webrtc::AudioProcessing::Config::NoiseSuppression::kVeryHigh; - apm_config.transient_suppression.enabled = true; - apm_config.pipeline.multi_channel_render = true; - apm_config.pipeline.multi_channel_capture = true; - - mWorkerThread->BlockingCall( - [this]() - { - mPeerAudioDeviceObserver = new LLAudioDeviceObserver; - mPeerDeviceModule = - webrtc::CreateAudioDeviceWithDataObserver(webrtc::AudioDeviceModule::AudioLayer::kPlatformDefaultAudio, - mTaskQueueFactory.get(), - std::unique_ptr<webrtc::AudioDeviceDataObserver>(mPeerAudioDeviceObserver)); - mPeerDeviceModule->Init(); - mPeerDeviceModule->SetPlayoutDevice(mPlayoutDevice); - mPeerDeviceModule->SetRecordingDevice(mRecordingDevice); - mPeerDeviceModule->SetStereoRecording(false); - mPeerDeviceModule->SetStereoPlayout(true); - mPeerDeviceModule->EnableBuiltInAEC(false); - mPeerDeviceModule->InitMicrophone(); - mPeerDeviceModule->InitSpeaker(); - mPeerDeviceModule->InitRecording(); - mPeerDeviceModule->InitPlayout(); - mPeerDeviceModule->StopPlayout(); - mPeerDeviceModule->StopRecording(); - }); - - mPeerConnectionFactory = webrtc::CreatePeerConnectionFactory(mNetworkThread.get(), - mWorkerThread.get(), - mSignalingThread.get(), - mPeerDeviceModule, - webrtc::CreateBuiltinAudioEncoderFactory(), - webrtc::CreateBuiltinAudioDecoderFactory(), - nullptr /* video_encoder_factory */, - nullptr /* video_decoder_factory */, - nullptr /* audio_mixer */, - apm); - apm->ApplyConfig(apm_config); - webrtc::PeerConnectionInterface::RTCConfiguration config; config.sdp_semantics = webrtc::SdpSemantics::kUnifiedPlan; webrtc::PeerConnectionInterface::IceServer server; @@ -419,7 +446,7 @@ bool LLWebRTCImpl::initializeConnection() config.servers.push_back(server); server.uri = "stun:stun4.l.google.com:19302"; config.servers.push_back(server); - + webrtc::PeerConnectionDependencies pc_dependencies(this); auto error_or_peer_connection = mPeerConnectionFactory->CreatePeerConnectionOrError(config, std::move(pc_dependencies)); if (error_or_peer_connection.ok()) @@ -432,35 +459,36 @@ bool LLWebRTCImpl::initializeConnection() shutdownConnection(); return false; } - + webrtc::DataChannelInit init; init.ordered = true; - + auto data_channel_or_error = mPeerConnection->CreateDataChannelOrError("SLData", &init); if (data_channel_or_error.ok()) { mDataChannel = std::move(data_channel_or_error.value()); - + mDataChannel->RegisterObserver(this); } - + RTC_LOG(LS_INFO) << __FUNCTION__ << " " << mPeerConnection->signaling_state(); - + cricket::AudioOptions audioOptions; audioOptions.auto_gain_control = true; audioOptions.echo_cancellation = false; // incompatible with opus stereo audioOptions.noise_suppression = true; - + rtc::scoped_refptr<webrtc::MediaStreamInterface> stream = mPeerConnectionFactory->CreateLocalMediaStream("SLStream"); + rtc::scoped_refptr<webrtc::AudioTrackInterface> audio_track( - mPeerConnectionFactory->CreateAudioTrack("SLAudio", mPeerConnectionFactory->CreateAudioSource(audioOptions).get())); + mPeerConnectionFactory->CreateAudioTrack("SLAudio", mPeerConnectionFactory->CreateAudioSource(audioOptions).get())); audio_track->set_enabled(true); stream->AddTrack(audio_track); - + mPeerConnection->AddTrack(audio_track, {"SLStream"}); - + auto senders = mPeerConnection->GetSenders(); - + for (auto &sender : senders) { webrtc::RtpParameters params; @@ -474,7 +502,7 @@ bool LLWebRTCImpl::initializeConnection() params.codecs.push_back(codecparam); sender->SetParameters(params); } - + auto receivers = mPeerConnection->GetReceivers(); for (auto &receiver : receivers) { @@ -491,102 +519,78 @@ bool LLWebRTCImpl::initializeConnection() } webrtc::PeerConnectionInterface::RTCOfferAnswerOptions offerOptions; mPeerConnection->CreateOffer(this, offerOptions); - + return true; } -void LLWebRTCImpl::shutdownConnection() +void LLWebRTCPeerConnectionImpl::shutdownConnection() { - mSignalingThread->BlockingCall( - [this]() - { - if (mPeerConnection) - { - mPeerConnection->Close(); - mPeerConnection = nullptr; - } - mPeerConnectionFactory = nullptr; - }); - mWorkerThread->BlockingCall( - [this]() - { - if (mPeerDeviceModule) - { - mPeerDeviceModule->StopRecording(); - mPeerDeviceModule->Terminate(); - } - mPeerDeviceModule = nullptr; - if (mPeerAudioDeviceObserver) - { - mPeerAudioDeviceObserver = nullptr; - } - }); - mNetworkThread->BlockingCall( - [this]() + mWebRTCImpl->SignalingBlockingCall( + [this]() + { + if (mPeerConnection) + { + mPeerConnection->Close(); + mPeerConnection = nullptr; + } + }); + + mWebRTCImpl->NetworkBlockingCall( + [this]() + { + if (mDataChannel) + { + mDataChannel->Close(); + mDataChannel = nullptr; + } + }); +} + +void LLWebRTCPeerConnectionImpl::enableTracks(bool enable) +{ + // set_enabled shouldn't be done on the worker thread + if (mPeerConnection) + { + auto senders = mPeerConnection->GetSenders(); + for (auto &sender : senders) { - if (mDataChannel) - { - mDataChannel->Close(); - mDataChannel = nullptr; - } - }); + sender->track()->set_enabled(enable); + } + } } -void LLWebRTCImpl::AnswerAvailable(const std::string &sdp) +void LLWebRTCPeerConnectionImpl::AnswerAvailable(const std::string &sdp) { RTC_LOG(LS_INFO) << __FUNCTION__ << " Remote SDP: " << sdp; - - mSignalingThread->PostTask( - [this, sdp]() - { - RTC_LOG(LS_INFO) << __FUNCTION__ << " " << mPeerConnection->peer_connection_state(); - mPeerConnection->SetRemoteDescription(webrtc::CreateSessionDescription(webrtc::SdpType::kAnswer, sdp), - rtc::scoped_refptr<webrtc::SetRemoteDescriptionObserverInterface>(this)); - }); + + mWebRTCImpl->PostSignalingTask( + [this, sdp]() + { + RTC_LOG(LS_INFO) << __FUNCTION__ << " " << mPeerConnection->peer_connection_state(); + mPeerConnection->SetRemoteDescription(webrtc::CreateSessionDescription(webrtc::SdpType::kAnswer, sdp), + rtc::scoped_refptr<webrtc::SetRemoteDescriptionObserverInterface>(this)); + }); } -void LLWebRTCImpl::setMute(bool mute) +void LLWebRTCPeerConnectionImpl::setMute(bool mute) { mMute = mute; auto senders = mPeerConnection->GetSenders(); - + RTC_LOG(LS_INFO) << __FUNCTION__ << (mute ? "disabling" : "enabling") << " streams count " << senders.size(); - + for (auto &sender : senders) { sender->track()->set_enabled(!mMute); } } -void LLWebRTCImpl::setSpeakerVolume(float volume) -{ - mSignalingThread->PostTask( - [this, volume]() - { - auto receivers = mPeerConnection->GetReceivers(); - - RTC_LOG(LS_INFO) << __FUNCTION__ << "Set volume" << receivers.size(); - for (auto &receiver : receivers) - { - webrtc::MediaStreamTrackInterface *track = receiver->track().get(); - if (track->kind() == webrtc::MediaStreamTrackInterface::kAudioKind) - { - webrtc::AudioTrackInterface *audio_track = static_cast<webrtc::AudioTrackInterface *>(track); - webrtc::AudioSourceInterface *source = audio_track->GetSource(); - source->SetVolume(VOLUME_SCALE_WEBRTC * volume); - } - } - }); -} - -float LLWebRTCImpl::getTuningAudioLevel() { return 20 * mTuningAudioDeviceObserver->getMicrophoneEnergy(); } - // // PeerConnectionObserver implementation. // -void LLWebRTCImpl::OnAddTrack(rtc::scoped_refptr<webrtc::RtpReceiverInterface> receiver, - const std::vector<rtc::scoped_refptr<webrtc::MediaStreamInterface>> &streams) +void LLWebRTCPeerConnectionImpl::OnAddTrack(rtc::scoped_refptr<webrtc::RtpReceiverInterface> receiver, + const std::vector<rtc::scoped_refptr<webrtc::MediaStreamInterface>> &streams) { RTC_LOG(LS_INFO) << __FUNCTION__ << " " << receiver->id(); webrtc::RtpParameters params; @@ -601,18 +605,18 @@ void LLWebRTCImpl::OnAddTrack(rtc::scoped_refptr<webrtc::RtpReceiverInterface> receiver->SetParameters(params); } -void LLWebRTCImpl::OnRemoveTrack(rtc::scoped_refptr<webrtc::RtpReceiverInterface> receiver) +void LLWebRTCPeerConnectionImpl::OnRemoveTrack(rtc::scoped_refptr<webrtc::RtpReceiverInterface> receiver) { RTC_LOG(LS_INFO) << __FUNCTION__ << " " << receiver->id(); } -void LLWebRTCImpl::OnDataChannel(rtc::scoped_refptr<webrtc::DataChannelInterface> channel) +void LLWebRTCPeerConnectionImpl::OnDataChannel(rtc::scoped_refptr<webrtc::DataChannelInterface> channel) { mDataChannel = channel; channel->RegisterObserver(this); } -void LLWebRTCImpl::OnIceGatheringChange(webrtc::PeerConnectionInterface::IceGatheringState new_state) +void LLWebRTCPeerConnectionImpl::OnIceGatheringChange(webrtc::PeerConnectionInterface::IceGatheringState new_state) { LLWebRTCSignalingObserver::IceGatheringState webrtc_new_state = LLWebRTCSignalingObserver::IceGatheringState::ICE_GATHERING_NEW; switch (new_state) @@ -631,7 +635,7 @@ void LLWebRTCImpl::OnIceGatheringChange(webrtc::PeerConnectionInterface::IceGath webrtc_new_state = LLWebRTCSignalingObserver::IceGatheringState::ICE_GATHERING_NEW; return; } - + if (mAnswerReceived) { for (auto &observer : mSignalingObserverList) @@ -642,19 +646,17 @@ void LLWebRTCImpl::OnIceGatheringChange(webrtc::PeerConnectionInterface::IceGath } // Called any time the PeerConnectionState changes. -void LLWebRTCImpl::OnConnectionChange(webrtc::PeerConnectionInterface::PeerConnectionState new_state) +void LLWebRTCPeerConnectionImpl::OnConnectionChange(webrtc::PeerConnectionInterface::PeerConnectionState new_state) { RTC_LOG(LS_ERROR) << __FUNCTION__ << " Peer Connection State Change " << new_state; - + switch (new_state) { case webrtc::PeerConnectionInterface::PeerConnectionState::kConnected: { if (new_state == webrtc::PeerConnectionInterface::PeerConnectionState::kConnected) { - mWorkerThread->PostTask([this]() { - mPeerDeviceModule->StartRecording(); - mPeerDeviceModule->StartPlayout(); + mWebRTCImpl->PostWorkerTask([this]() { for (auto &observer : mSignalingObserverList) { observer->OnAudioEstablished(this); @@ -670,7 +672,7 @@ void LLWebRTCImpl::OnConnectionChange(webrtc::PeerConnectionInterface::PeerConne { observer->OnRenegotiationNeeded(); } - + break; } default: @@ -683,15 +685,15 @@ void LLWebRTCImpl::OnConnectionChange(webrtc::PeerConnectionInterface::PeerConne static std::string iceCandidateToTrickleString(const webrtc::IceCandidateInterface *candidate) { std::ostringstream candidate_stream; - + candidate_stream << - candidate->candidate().foundation() << " " << - std::to_string(candidate->candidate().component()) << " " << - candidate->candidate().protocol() << " " << - std::to_string(candidate->candidate().priority()) << " " << - candidate->candidate().address().ipaddr().ToString() << " " << - candidate->candidate().address().PortAsString() << " typ "; - + candidate->candidate().foundation() << " " << + std::to_string(candidate->candidate().component()) << " " << + candidate->candidate().protocol() << " " << + std::to_string(candidate->candidate().priority()) << " " << + candidate->candidate().address().ipaddr().ToString() << " " << + candidate->candidate().address().PortAsString() << " typ "; + if (candidate->candidate().type() == cricket::LOCAL_PORT_TYPE) { candidate_stream << "host"; @@ -699,25 +701,25 @@ static std::string iceCandidateToTrickleString(const webrtc::IceCandidateInterfa else if (candidate->candidate().type() == cricket::STUN_PORT_TYPE) { candidate_stream << "srflx " << - "raddr " << candidate->candidate().related_address().ipaddr().ToString() << " " << - "rport " << candidate->candidate().related_address().PortAsString(); + "raddr " << candidate->candidate().related_address().ipaddr().ToString() << " " << + "rport " << candidate->candidate().related_address().PortAsString(); } else if (candidate->candidate().type() == cricket::RELAY_PORT_TYPE) { candidate_stream << "relay " << - "raddr " << candidate->candidate().related_address().ipaddr().ToString() << " " << - "rport " << candidate->candidate().related_address().PortAsString(); + "raddr " << candidate->candidate().related_address().ipaddr().ToString() << " " << + "rport " << candidate->candidate().related_address().PortAsString(); } else if (candidate->candidate().type() == cricket::PRFLX_PORT_TYPE) { candidate_stream << "prflx " << - "raddr " << candidate->candidate().related_address().ipaddr().ToString() << " " << - "rport " << candidate->candidate().related_address().PortAsString(); + "raddr " << candidate->candidate().related_address().ipaddr().ToString() << " " << + "rport " << candidate->candidate().related_address().PortAsString(); } else { RTC_LOG(LS_ERROR) << __FUNCTION__ << " Unknown candidate type " << candidate->candidate().type(); } - if (candidate->candidate().protocol() == "tcp") + if (candidate->candidate().protocol() == "tcp") { candidate_stream << " tcptype " << candidate->candidate().tcptype(); } @@ -725,16 +727,16 @@ static std::string iceCandidateToTrickleString(const webrtc::IceCandidateInterfa return candidate_stream.str(); } -void LLWebRTCImpl::OnIceCandidate(const webrtc::IceCandidateInterface *candidate) +void LLWebRTCPeerConnectionImpl::OnIceCandidate(const webrtc::IceCandidateInterface *candidate) { RTC_LOG(LS_INFO) << __FUNCTION__ << " " << candidate->sdp_mline_index(); - + if (!candidate) { RTC_LOG(LS_ERROR) << __FUNCTION__ << " No Ice Candidate Given"; return; } - if (mAnswerReceived) + if (mAnswerReceived) { for (auto &observer : mSignalingObserverList) { @@ -745,22 +747,22 @@ void LLWebRTCImpl::OnIceCandidate(const webrtc::IceCandidateInterface *candidate observer->OnIceCandidate(ice_candidate); } } - else + else { mCachedIceCandidates.push_back( - webrtc::CreateIceCandidate(candidate->sdp_mid(), candidate->sdp_mline_index(), candidate->candidate())); + webrtc::CreateIceCandidate(candidate->sdp_mid(), candidate->sdp_mline_index(), candidate->candidate())); } } // // CreateSessionDescriptionObserver implementation. // -void LLWebRTCImpl::OnSuccess(webrtc::SessionDescriptionInterface *desc) +void LLWebRTCPeerConnectionImpl::OnSuccess(webrtc::SessionDescriptionInterface *desc) { std::string sdp; desc->ToString(&sdp); RTC_LOG(LS_INFO) << sdp; -; + ; // mangle the sdp as this is the only way currently to bump up // the send audio rate to 48k std::istringstream sdp_stream(sdp); @@ -780,7 +782,7 @@ void LLWebRTCImpl::OnSuccess(webrtc::SessionDescriptionInterface *desc) else if (sdp_line.find("a=fmtp:" + opus_payload) == 0) { sdp_mangled_stream << sdp_line << "a=fmtp:" << opus_payload - << " minptime=10;useinbandfec=1;stereo=1;sprop-stereo=1;maxplaybackrate=48000\n"; + << " minptime=10;useinbandfec=1;stereo=1;sprop-stereo=1;maxplaybackrate=48000\n"; } else { @@ -789,27 +791,27 @@ void LLWebRTCImpl::OnSuccess(webrtc::SessionDescriptionInterface *desc) } webrtc::CreateSessionDescription(webrtc::SdpType::kOffer, sdp_mangled_stream.str()); - - - + + + mPeerConnection->SetLocalDescription(std::unique_ptr<webrtc::SessionDescriptionInterface>( - webrtc::CreateSessionDescription(webrtc::SdpType::kOffer, sdp_mangled_stream.str())), - rtc::scoped_refptr<webrtc::SetLocalDescriptionObserverInterface>(this)); + webrtc::CreateSessionDescription(webrtc::SdpType::kOffer, sdp_mangled_stream.str())), + rtc::scoped_refptr<webrtc::SetLocalDescriptionObserverInterface>(this)); RTC_LOG(LS_INFO) << __FUNCTION__ << " Local SDP: " << sdp_mangled_stream.str(); - - + + for (auto &observer : mSignalingObserverList) { observer->OnOfferAvailable(sdp_mangled_stream.str()); } } -void LLWebRTCImpl::OnFailure(webrtc::RTCError error) { RTC_LOG(LS_ERROR) << ToString(error.type()) << ": " << error.message(); } +void LLWebRTCPeerConnectionImpl::OnFailure(webrtc::RTCError error) { RTC_LOG(LS_ERROR) << ToString(error.type()) << ": " << error.message(); } // // SetRemoteDescriptionObserverInterface implementation. // -void LLWebRTCImpl::OnSetRemoteDescriptionComplete(webrtc::RTCError error) +void LLWebRTCPeerConnectionImpl::OnSetRemoteDescriptionComplete(webrtc::RTCError error) { RTC_LOG(LS_INFO) << __FUNCTION__ << " " << mPeerConnection->signaling_state(); if (!error.ok()) @@ -831,37 +833,20 @@ void LLWebRTCImpl::OnSetRemoteDescriptionComplete(webrtc::RTCError error) } mCachedIceCandidates.clear(); OnIceGatheringChange(mPeerConnection->ice_gathering_state()); - + } // // SetLocalDescriptionObserverInterface implementation. // -void LLWebRTCImpl::OnSetLocalDescriptionComplete(webrtc::RTCError error) +void LLWebRTCPeerConnectionImpl::OnSetLocalDescriptionComplete(webrtc::RTCError error) { -#if 0 - RTC_LOG(LS_INFO) << __FUNCTION__ << " " << mPeerConnection->signaling_state(); - if (!error.ok()) - { - RTC_LOG(LS_ERROR) << ToString(error.type()) << ": " << error.message(); - return; - } - auto desc = mPeerConnection->pending_local_description(); - std::string sdp; - desc->ToString(&sdp); - - RTC_LOG(LS_INFO) << __FUNCTION__ << " Local SDP: " << sdp; - ; - for (auto &observer : mSignalingObserverList) - { - observer->OnOfferAvailable(sdp); - } -#endif + } -void LLWebRTCImpl::setAudioObserver(LLWebRTCAudioObserver *observer) { mAudioObserverList.emplace_back(observer); } +void LLWebRTCPeerConnectionImpl::setAudioObserver(LLWebRTCAudioObserver *observer) { mAudioObserverList.emplace_back(observer); } -void LLWebRTCImpl::unsetAudioObserver(LLWebRTCAudioObserver *observer) +void LLWebRTCPeerConnectionImpl::unsetAudioObserver(LLWebRTCAudioObserver *observer) { std::vector<LLWebRTCAudioObserver *>::iterator it = std::find(mAudioObserverList.begin(), mAudioObserverList.end(), observer); if (it != mAudioObserverList.end()) @@ -870,22 +855,20 @@ void LLWebRTCImpl::unsetAudioObserver(LLWebRTCAudioObserver *observer) } } -float LLWebRTCImpl::getAudioLevel() { return 20 * mPeerAudioDeviceObserver->getMicrophoneEnergy(); } - // // DataChannelObserver implementation // -void LLWebRTCImpl::OnStateChange() -{ +void LLWebRTCPeerConnectionImpl::OnStateChange() +{ RTC_LOG(LS_INFO) << __FUNCTION__ << " Data Channel State: " << webrtc::DataChannelInterface::DataStateString(mDataChannel->state()); switch (mDataChannel->state()) { case webrtc::DataChannelInterface::kOpen: RTC_LOG(LS_INFO) << __FUNCTION__ << " Data Channel State Open"; - for (auto &observer : mDataObserverList) + for (auto &observer : mSignalingObserverList) { - observer->OnDataChannelReady(); + observer->OnDataChannelReady(this); } break; case webrtc::DataChannelInterface::kConnecting: @@ -903,7 +886,7 @@ void LLWebRTCImpl::OnStateChange() } -void LLWebRTCImpl::OnMessage(const webrtc::DataBuffer& buffer) +void LLWebRTCPeerConnectionImpl::OnMessage(const webrtc::DataBuffer& buffer) { std::string data((const char*)buffer.data.cdata(), buffer.size()); for (auto &observer : mDataObserverList) @@ -912,43 +895,55 @@ void LLWebRTCImpl::OnMessage(const webrtc::DataBuffer& buffer) } } -void LLWebRTCImpl::sendData(const std::string& data, bool binary) +void LLWebRTCPeerConnectionImpl::sendData(const std::string& data, bool binary) { rtc::CopyOnWriteBuffer cowBuffer(data.data(), data.length()); webrtc::DataBuffer buffer(cowBuffer, binary); mDataChannel->Send(buffer); } -void LLWebRTCImpl::setDataObserver(LLWebRTCDataObserver* observer) { mDataObserverList.emplace_back(observer); } +void LLWebRTCPeerConnectionImpl::setDataObserver(LLWebRTCDataObserver* observer) { mDataObserverList.emplace_back(observer); } -void LLWebRTCImpl::unsetDataObserver(LLWebRTCDataObserver* observer) +void LLWebRTCPeerConnectionImpl::unsetDataObserver(LLWebRTCDataObserver* observer) { std::vector<LLWebRTCDataObserver *>::iterator it = - std::find(mDataObserverList.begin(), mDataObserverList.end(), observer); + std::find(mDataObserverList.begin(), mDataObserverList.end(), observer); if (it != mDataObserverList.end()) { mDataObserverList.erase(it); } } -rtc::RefCountedObject<LLWebRTCImpl> *gWebRTCImpl = nullptr; -LLWebRTCDeviceInterface *getDeviceInterface() { return gWebRTCImpl; } -LLWebRTCSignalInterface *getSignalingInterface() { return gWebRTCImpl; } -LLWebRTCDataInterface *getDataInterface() { return gWebRTCImpl; } +LLWebRTCImpl * gWebRTCImpl = nullptr; +LLWebRTCDeviceInterface * getDeviceInterface() +{ + return gWebRTCImpl; +} + +LLWebRTCPeerConnection * newPeerConnection() +{ + return gWebRTCImpl->newPeerConnection(); +} + +void freePeerConnection(LLWebRTCPeerConnection *peer_connection) +{ + gWebRTCImpl->freePeerConnection(peer_connection); +} void init() { - gWebRTCImpl = new rtc::RefCountedObject<LLWebRTCImpl>(); - gWebRTCImpl->AddRef(); + gWebRTCImpl = new LLWebRTCImpl(); gWebRTCImpl->init(); } void terminate() -{ - gWebRTCImpl->terminate(); - gWebRTCImpl->Release(); - gWebRTCImpl = nullptr; +{ + if (gWebRTCImpl) + { + gWebRTCImpl->terminate(); + gWebRTCImpl = nullptr; + } } } // namespace llwebrtc diff --git a/indra/llwebrtc/llwebrtc.h b/indra/llwebrtc/llwebrtc.h index 9224e88dfa..753fe6a983 100644 --- a/indra/llwebrtc/llwebrtc.h +++ b/indra/llwebrtc/llwebrtc.h @@ -1,6 +1,6 @@ /** - * @file llaccordionctrl.cpp - * @brief Accordion panel implementation + * @file llwebrtc.h + * @brief WebRTC interface * * $LicenseInfo:firstyear=2023&license=viewerlgpl$ * Second Life Viewer Source Code @@ -17,7 +17,7 @@ * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software + * License along with this library; if not, write to the Free tSoftware * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA * * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA @@ -44,8 +44,6 @@ namespace llwebrtc { -LLSYMEXPORT void init(); -LLSYMEXPORT void terminate(); struct LLWebRTCIceCandidate { @@ -87,6 +85,10 @@ class LLWebRTCDeviceInterface virtual void setTuningMode(bool enable) = 0; virtual float getTuningAudioLevel() = 0; + + virtual void setSpeakerVolume(float volume) = 0; // volume between 0.0 and 1.0 + virtual void setMicrophoneVolume(float volume) = 0; // volume between 0.0 and 1.0 + virtual float getPeerAudioLevel() = 0; }; class LLWebRTCAudioObserver @@ -100,15 +102,12 @@ class LLWebRTCAudioInterface virtual void setAudioObserver(LLWebRTCAudioObserver *observer) = 0; virtual void unsetAudioObserver(LLWebRTCAudioObserver *observer) = 0; virtual void setMute(bool mute) = 0; - virtual void setSpeakerVolume(float volume) = 0; // volume between 0.0 and 1.0 - virtual float getAudioLevel() = 0; }; class LLWebRTCDataObserver { public: virtual void OnDataReceived(const std::string& data, bool binary) = 0; - virtual void OnDataChannelReady() = 0; }; class LLWebRTCDataInterface @@ -133,9 +132,10 @@ class LLWebRTCSignalingObserver virtual void OnOfferAvailable(const std::string& sdp) = 0; virtual void OnRenegotiationNeeded() = 0; virtual void OnAudioEstablished(LLWebRTCAudioInterface *audio_interface) = 0; + virtual void OnDataChannelReady(LLWebRTCDataInterface *data_interface) = 0; }; -class LLWebRTCSignalInterface +class LLWebRTCPeerConnection { public: virtual void setSignalingObserver(LLWebRTCSignalingObserver* observer) = 0; @@ -146,9 +146,11 @@ class LLWebRTCSignalInterface virtual void AnswerAvailable(const std::string &sdp) = 0; }; +LLSYMEXPORT void init(); +LLSYMEXPORT void terminate(); LLSYMEXPORT LLWebRTCDeviceInterface* getDeviceInterface(); -LLSYMEXPORT LLWebRTCSignalInterface* getSignalingInterface(); -LLSYMEXPORT LLWebRTCDataInterface* getDataInterface(); +LLSYMEXPORT LLWebRTCPeerConnection* newPeerConnection(); +LLSYMEXPORT void freePeerConnection(LLWebRTCPeerConnection *connection); } #endif // LLWEBRTC_H diff --git a/indra/llwebrtc/llwebrtc_impl.h b/indra/llwebrtc/llwebrtc_impl.h index 01159604c9..295e949b4e 100644 --- a/indra/llwebrtc/llwebrtc_impl.h +++ b/indra/llwebrtc/llwebrtc_impl.h @@ -1,6 +1,6 @@ /** - * @file llaccordionctrl.cpp - * @brief Accordion panel implementation + * @file llwebrtc_impl.h + * @brief WebRTC Interface implementation. * * $LicenseInfo:firstyear=2023&license=viewerlgpl$ * Second Life Viewer Source Code @@ -63,6 +63,8 @@ namespace llwebrtc { +class LLWebRTCPeerConnectionImpl; + class LLAudioDeviceObserver : public webrtc::AudioDeviceDataObserver { public: @@ -87,16 +89,7 @@ class LLAudioDeviceObserver : public webrtc::AudioDeviceDataObserver float mMicrophoneEnergy; }; -class LLWebRTCImpl : public LLWebRTCDeviceInterface, - public LLWebRTCSignalInterface, - public LLWebRTCAudioInterface, - public LLWebRTCDataInterface, - public webrtc::PeerConnectionObserver, - public webrtc::CreateSessionDescriptionObserver, - public webrtc::SetRemoteDescriptionObserverInterface, - public webrtc::SetLocalDescriptionObserverInterface, - public webrtc::DataChannelObserver - +class LLWebRTCImpl : public LLWebRTCDeviceInterface { public: LLWebRTCImpl() : @@ -116,15 +109,112 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, void setDevicesObserver(LLWebRTCDevicesObserver *observer) override; void unsetDevicesObserver(LLWebRTCDevicesObserver *observer) override; - void setCaptureDevice(const std::string& id) override; + void setCaptureDevice(const std::string& id) override; void setRenderDevice(const std::string& id) override; void setTuningMode(bool enable) override; float getTuningAudioLevel() override; + float getPeerAudioLevel() override; + + void setSpeakerVolume(float volume) override; // range 0.0-1.0 + void setMicrophoneVolume(float volume) override; // range 0.0-1.0 + + // + // Helpers + // + + void PostWorkerTask(absl::AnyInvocable<void() &&> task, + const webrtc::Location& location = webrtc::Location::Current()) + { + mWorkerThread->PostTask(std::move(task), location); + } + + void PostSignalingTask(absl::AnyInvocable<void() &&> task, + const webrtc::Location& location = webrtc::Location::Current()) + { + mSignalingThread->PostTask(std::move(task), location); + } + + void PostNetworkTask(absl::AnyInvocable<void() &&> task, + const webrtc::Location& location = webrtc::Location::Current()) + { + mNetworkThread->PostTask(std::move(task), location); + } + + void WorkerBlockingCall(rtc::FunctionView<void()> functor, + const webrtc::Location& location = webrtc::Location::Current()) + { + mWorkerThread->BlockingCall(std::move(functor), location); + } + + void SignalingBlockingCall(rtc::FunctionView<void()> functor, + const webrtc::Location& location = webrtc::Location::Current()) + { + mSignalingThread->BlockingCall(std::move(functor), location); + } + + void NetworkBlockingCall(rtc::FunctionView<void()> functor, + const webrtc::Location& location = webrtc::Location::Current()) + { + mNetworkThread->BlockingCall(std::move(functor), location); + } + + rtc::scoped_refptr<webrtc::PeerConnectionFactoryInterface> getPeerConnectionFactory() { return mPeerConnectionFactory; } + + LLWebRTCPeerConnection * newPeerConnection(); + void freePeerConnection(LLWebRTCPeerConnection * peer_connection); + + protected: + + std::unique_ptr<rtc::Thread> mNetworkThread; + std::unique_ptr<rtc::Thread> mWorkerThread; + std::unique_ptr<rtc::Thread> mSignalingThread; + rtc::scoped_refptr<webrtc::PeerConnectionFactoryInterface> mPeerConnectionFactory; + webrtc::PeerConnectionInterface::RTCConfiguration mConfiguration; + std::unique_ptr<webrtc::TaskQueueFactory> mTaskQueueFactory; + + + // Devices + void updateDevices(); + rtc::scoped_refptr<webrtc::AudioDeviceModule> mTuningDeviceModule; + rtc::scoped_refptr<webrtc::AudioDeviceModule> mPeerDeviceModule; + std::vector<LLWebRTCDevicesObserver *> mVoiceDevicesObserverList; + // accessors in webrtc aren't apparently implemented yet. + int32_t mPlayoutDevice; + int32_t mRecordingDevice; + bool mMute; + + LLAudioDeviceObserver * mTuningAudioDeviceObserver; + LLAudioDeviceObserver * mPeerAudioDeviceObserver; + + // peer connections + std::vector<rtc::scoped_refptr<LLWebRTCPeerConnectionImpl>> mPeerConnections; +}; + +class LLWebRTCPeerConnectionImpl : public LLWebRTCPeerConnection, + public LLWebRTCAudioInterface, + public LLWebRTCDataInterface, + public webrtc::PeerConnectionObserver, + public webrtc::CreateSessionDescriptionObserver, + public webrtc::SetRemoteDescriptionObserverInterface, + public webrtc::SetLocalDescriptionObserverInterface, + public webrtc::DataChannelObserver + +{ + public: + LLWebRTCPeerConnectionImpl() {} + ~LLWebRTCPeerConnectionImpl() {} + + void init(LLWebRTCImpl * webrtc_impl); + void terminate(); + + virtual void AddRef() const override = 0; + virtual rtc::RefCountReleaseStatus Release() const override = 0; + // - // LLWebRTCSignalInterface + // LLWebRTCPeerConnection // void setSignalingObserver(LLWebRTCSignalingObserver *observer) override; @@ -140,8 +230,6 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, void setAudioObserver(LLWebRTCAudioObserver *observer) override; void unsetAudioObserver(LLWebRTCAudioObserver *observer) override; void setMute(bool mute) override; - void setSpeakerVolume(float folume) override; // range 0.0-1.0 - float getAudioLevel() override; // // LLWebRTCDataInterface @@ -187,31 +275,17 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, // void OnStateChange() override; void OnMessage(const webrtc::DataBuffer& buffer) override; + + // Helpers + void enableTracks(bool enable); protected: - - std::unique_ptr<rtc::Thread> mNetworkThread; - std::unique_ptr<rtc::Thread> mWorkerThread; - std::unique_ptr<rtc::Thread> mSignalingThread; + + LLWebRTCImpl * mWebRTCImpl; rtc::scoped_refptr<webrtc::PeerConnectionFactoryInterface> mPeerConnectionFactory; - webrtc::PeerConnectionInterface::RTCConfiguration mConfiguration; - std::unique_ptr<webrtc::TaskQueueFactory> mTaskQueueFactory; - - - // Devices - void updateDevices(); - rtc::scoped_refptr<webrtc::AudioDeviceModule> mTuningDeviceModule; - rtc::scoped_refptr<webrtc::AudioDeviceModule> mPeerDeviceModule; - std::vector<LLWebRTCDevicesObserver *> mVoiceDevicesObserverList; - // accessors in webrtc aren't apparently implemented yet. - int32_t mPlayoutDevice; - int32_t mRecordingDevice; bool mMute; - LLAudioDeviceObserver * mTuningAudioDeviceObserver; - LLAudioDeviceObserver * mPeerAudioDeviceObserver; - // signaling std::vector<LLWebRTCSignalingObserver *> mSignalingObserverList; std::vector<std::unique_ptr<webrtc::IceCandidateInterface>> mCachedIceCandidates; diff --git a/indra/newview/llvoicewebrtc.cpp b/indra/newview/llvoicewebrtc.cpp index b4d65bec0a..4fe2c78167 100644 --- a/indra/newview/llvoicewebrtc.cpp +++ b/indra/newview/llvoicewebrtc.cpp @@ -307,7 +307,7 @@ LLWebRTCVoiceClient::LLWebRTCVoiceClient() : mIsCoroutineActive(false), mWebRTCPump("WebRTCClientPump"), mWebRTCDeviceInterface(nullptr), - mWebRTCSignalingInterface(nullptr), + mWebRTCPeerConnection(nullptr), mWebRTCAudioInterface(nullptr) { sShuttingDown = false; @@ -364,11 +364,8 @@ void LLWebRTCVoiceClient::init(LLPumpIO *pump) mWebRTCDeviceInterface = llwebrtc::getDeviceInterface(); mWebRTCDeviceInterface->setDevicesObserver(this); - mWebRTCSignalingInterface = llwebrtc::getSignalingInterface(); - mWebRTCSignalingInterface->setSignalingObserver(this); - - mWebRTCDataInterface = llwebrtc::getDataInterface(); - mWebRTCDataInterface->setDataObserver(this); + mWebRTCPeerConnection = llwebrtc::newPeerConnection(); + mWebRTCPeerConnection->setSignalingObserver(this); } void LLWebRTCVoiceClient::terminate() @@ -500,7 +497,7 @@ void LLWebRTCVoiceClient::setLoginInfo( const std::string& channel_sdp) { mRemoteChannelSDP = channel_sdp; - mWebRTCSignalingInterface->AnswerAvailable(channel_sdp); + mWebRTCPeerConnection->AnswerAvailable(channel_sdp); if(mAccountLoggedIn) { @@ -862,7 +859,7 @@ bool LLWebRTCVoiceClient::establishVoiceConnection() { return false; } - return mWebRTCSignalingInterface->initializeConnection(); + return mWebRTCPeerConnection->initializeConnection(); } bool LLWebRTCVoiceClient::breakVoiceConnection(bool corowait) @@ -896,7 +893,7 @@ bool LLWebRTCVoiceClient::breakVoiceConnection(bool corowait) LLSD body; body["logout"] = TRUE; httpAdapter->postAndSuspend(httpRequest, url, body); - mWebRTCSignalingInterface->shutdownConnection(); + mWebRTCPeerConnection->shutdownConnection(); return true; } @@ -1136,11 +1133,14 @@ bool LLWebRTCVoiceClient::addAndJoinSession(const sessionStatePtr_t &nextSession addParticipantByID(gAgent.getID()); // tell peers that this participant has joined. - Json::FastWriter writer; - Json::Value root = getPositionAndVolumeUpdateJson(true); - root["j"] = true; - std::string json_data = writer.write(root); - mWebRTCDataInterface->sendData(json_data, false); + if (mWebRTCDataInterface) + { + Json::FastWriter writer; + Json::Value root = getPositionAndVolumeUpdateJson(true); + root["j"] = true; + std::string json_data = writer.write(root); + mWebRTCDataInterface->sendData(json_data, false); + } notifyStatusObservers(LLVoiceClientStatusObserver::STATUS_JOINED); @@ -2235,7 +2235,7 @@ Json::Value LLWebRTCVoiceClient::getPositionAndVolumeUpdateJson(bool force) if (!mMuteMic) { - audio_level = (F32) mWebRTCAudioInterface->getAudioLevel(); + audio_level = (F32) mWebRTCDeviceInterface->getPeerAudioLevel(); } uint32_t uint_audio_level = mMuteMic ? 0 : (uint32_t) (audio_level * 128); if (force || (uint_audio_level != mAudioLevel)) @@ -2451,7 +2451,6 @@ void LLWebRTCVoiceClient::processIceUpdates() if (mIceCandidates.size()) { LLSD candidates = LLSD::emptyArray(); - body["candidates"] = LLSD::emptyArray(); for (auto &ice_candidate : mIceCandidates) { LLSD body_candidate; @@ -2532,7 +2531,7 @@ void LLWebRTCVoiceClient::OnAudioEstablished(llwebrtc::LLWebRTCAudioInterface * LLMutexLock lock(&mVoiceStateMutex); speaker_volume = mSpeakerVolume; } - audio_interface->setSpeakerVolume(mSpeakerVolume); + mWebRTCDeviceInterface->setSpeakerVolume(mSpeakerVolume); setVoiceControlStateUnless(VOICE_STATE_SESSION_ESTABLISHED, VOICE_STATE_SESSION_RETRY); } @@ -2597,9 +2596,10 @@ void LLWebRTCVoiceClient::OnDataReceived(const std::string& data, bool binary) } } -void LLWebRTCVoiceClient::OnDataChannelReady() +void LLWebRTCVoiceClient::OnDataChannelReady(llwebrtc::LLWebRTCDataInterface *data_interface) { - + mWebRTCDataInterface = data_interface; + mWebRTCDataInterface->setDataObserver(this); } @@ -4470,7 +4470,7 @@ void LLWebRTCVoiceClient::setVoiceVolume(F32 volume) } if (mWebRTCAudioInterface) { - mWebRTCAudioInterface->setSpeakerVolume(volume); + mWebRTCDeviceInterface->setSpeakerVolume(volume); } } } diff --git a/indra/newview/llvoicewebrtc.h b/indra/newview/llvoicewebrtc.h index fd7094cb46..b33dc26dff 100644 --- a/indra/newview/llvoicewebrtc.h +++ b/indra/newview/llvoicewebrtc.h @@ -271,7 +271,7 @@ public: /// LLWebRTCDataObserver //@{ void OnDataReceived(const std::string& data, bool binary) override; - void OnDataChannelReady() override; + void OnDataChannelReady(llwebrtc::LLWebRTCDataInterface *data_interface) override; //@} void processIceUpdates(); @@ -778,7 +778,7 @@ private: buddyListMap mBuddyListMap; llwebrtc::LLWebRTCDeviceInterface *mWebRTCDeviceInterface; - llwebrtc::LLWebRTCSignalInterface *mWebRTCSignalingInterface; + llwebrtc::LLWebRTCPeerConnection *mWebRTCPeerConnection; llwebrtc::LLWebRTCAudioInterface *mWebRTCAudioInterface; llwebrtc::LLWebRTCDataInterface *mWebRTCDataInterface; |