diff options
Diffstat (limited to 'indra')
| -rw-r--r-- | indra/llwebrtc/llwebrtc.cpp | 213 | ||||
| -rw-r--r-- | indra/llwebrtc/llwebrtc.h | 129 | ||||
| -rw-r--r-- | indra/llwebrtc/llwebrtc_impl.h | 68 | ||||
| -rw-r--r-- | indra/newview/llvoicewebrtc.cpp | 50 | ||||
| -rw-r--r-- | indra/newview/llvoicewebrtc.h | 6 |
5 files changed, 333 insertions, 133 deletions
diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index 74af16679a..dc5fd9657e 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -44,12 +44,16 @@ LLAudioDeviceObserver::LLAudioDeviceObserver() : mSumVector {0}, mMicrophoneEner float LLAudioDeviceObserver::getMicrophoneEnergy() { return mMicrophoneEnergy; } +// TODO: Pull smoothing/filtering code into a common helper function +// for LLAudioDeviceObserver and LLCustomProcessor + void LLAudioDeviceObserver::OnCaptureData(const void *audio_samples, const size_t num_samples, const size_t bytes_per_sample, const size_t num_channels, const uint32_t samples_per_sec) { + // calculate the energy float energy = 0; const short *samples = (const short *) audio_samples; for (size_t index = 0; index < num_samples * num_channels; index++) @@ -82,7 +86,8 @@ void LLAudioDeviceObserver::OnRenderData(const void *audio_samples, LLCustomProcessor::LLCustomProcessor() : mSampleRateHz(0), - mNumChannels(0) + mNumChannels(0), + mMicrophoneEnergy(0.0) { memset(mSumVector, 0, sizeof(mSumVector)); } @@ -107,6 +112,7 @@ void LLCustomProcessor::Process(webrtc::AudioBuffer *audio_in) return; } + // grab the input audio frame_samples.resize(stream_config.num_samples()); frame.resize(stream_config.num_channels()); for (size_t ch = 0; ch < stream_config.num_channels(); ++ch) @@ -116,6 +122,7 @@ void LLCustomProcessor::Process(webrtc::AudioBuffer *audio_in) audio_in->CopyTo(stream_config, &frame[0]); + // calculate the energy float energy = 0; for (size_t index = 0; index < stream_config.num_samples(); index++) { @@ -137,17 +144,33 @@ void LLCustomProcessor::Process(webrtc::AudioBuffer *audio_in) mMicrophoneEnergy = std::sqrt(totalSum / (stream_config.num_samples() * buffer_size)); } +// +// LLWebRTCImpl implementation +// + +LLWebRTCImpl::LLWebRTCImpl() : + mPeerCustomProcessor(nullptr), + mMute(true), + mPlayoutDevice(0), + mRecordingDevice(0), + mTuningAudioDeviceObserver(nullptr) +{ +} + void LLWebRTCImpl::init() { RTC_DCHECK(mPeerConnectionFactory); mPlayoutDevice = 0; mRecordingDevice = 0; rtc::InitializeSSL(); + + // Normal logging is rather spammy, so turn it off. rtc::LogMessage::LogToDebug(rtc::LS_NONE); rtc::LogMessage::SetLogToStderr(true); mTaskQueueFactory = webrtc::CreateDefaultTaskQueueFactory(); - + + // Create the native threads. mNetworkThread = rtc::Thread::CreateWithSocketServer(); mNetworkThread->SetName("WebRTCNetworkThread", nullptr); mNetworkThread->Start(); @@ -162,9 +185,12 @@ void LLWebRTCImpl::init() mWorkerThread->PostTask( [this]() { + // Initialize the audio devices on the Worker Thread mTuningDeviceModule = webrtc::CreateAudioDeviceWithDataObserver( - webrtc::AudioDeviceModule::AudioLayer::kPlatformDefaultAudio, mTaskQueueFactory.get(), - std::unique_ptr<webrtc::AudioDeviceDataObserver>(mTuningAudioDeviceObserver)); + webrtc::AudioDeviceModule::AudioLayer::kPlatformDefaultAudio, + mTaskQueueFactory.get(), + std::unique_ptr<webrtc::AudioDeviceDataObserver>(mTuningAudioDeviceObserver)); + mTuningDeviceModule->Init(); mTuningDeviceModule->SetStereoRecording(true); mTuningDeviceModule->SetStereoPlayout(true); @@ -176,9 +202,13 @@ void LLWebRTCImpl::init() mWorkerThread->BlockingCall( [this]() { + // the peer device module doesn't need an observer + // as we pull peer data after audio processing. mPeerDeviceModule = - webrtc::CreateAudioDeviceWithDataObserver(webrtc::AudioDeviceModule::AudioLayer::kPlatformDefaultAudio, - mTaskQueueFactory.get(), nullptr); + webrtc::CreateAudioDeviceWithDataObserver( + webrtc::AudioDeviceModule::AudioLayer::kPlatformDefaultAudio, + mTaskQueueFactory.get(), + nullptr); mPeerDeviceModule->Init(); mPeerDeviceModule->SetPlayoutDevice(mPlayoutDevice); mPeerDeviceModule->SetRecordingDevice(mRecordingDevice); @@ -191,11 +221,15 @@ void LLWebRTCImpl::init() mPeerDeviceModule->InitPlayout(); }); + // The custom processor allows us to retrieve audio data (and levels) + // from after other audio processing such as AEC, AGC, etc. mPeerCustomProcessor = new LLCustomProcessor; webrtc::AudioProcessingBuilder apb; apb.SetCapturePostProcessing(std::unique_ptr<webrtc::CustomProcessing>(mPeerCustomProcessor)); rtc::scoped_refptr<webrtc::AudioProcessing> apm = apb.Create(); + // TODO: wire some of these to the primary interface and ultimately + // to the UI to allow user config. webrtc::AudioProcessing::Config apm_config; apm_config.echo_canceller.enabled = true; apm_config.echo_canceller.mobile_mode = false; @@ -242,6 +276,41 @@ void LLWebRTCImpl::init() }); } +void LLWebRTCImpl::terminate() +{ + for (auto &connection : mPeerConnections) + { + connection->terminate(); + } + mPeerConnections.clear(); + + mSignalingThread->BlockingCall([this]() { mPeerConnectionFactory = nullptr; }); + mWorkerThread->BlockingCall( + [this]() + { + if (mTuningDeviceModule) + { + mTuningDeviceModule->StopRecording(); + mTuningDeviceModule->Terminate(); + } + if (mPeerDeviceModule) + { + mPeerDeviceModule->StopRecording(); + mPeerDeviceModule->Terminate(); + } + mTuningDeviceModule = nullptr; + mPeerDeviceModule = nullptr; + mTaskQueueFactory = nullptr; + }); +} + +// +// Devices functions +// +// Most device-related functionality needs to happen +// on the worker thread (the audio thread,) so those calls will be +// proxied over to that thread. +// void LLWebRTCImpl::setRecording(bool recording) { mWorkerThread->PostTask( @@ -258,38 +327,6 @@ void LLWebRTCImpl::setRecording(bool recording) }); } -void LLWebRTCImpl::terminate() -{ - for (auto& connection : mPeerConnections) - { - connection->terminate(); - } - mPeerConnections.clear(); - - mSignalingThread->BlockingCall( - [this]() - { - mPeerConnectionFactory = nullptr; - }); - mWorkerThread->BlockingCall( - [this]() - { - if (mTuningDeviceModule) - { - mTuningDeviceModule->StopRecording(); - mTuningDeviceModule->Terminate(); - } - if (mPeerDeviceModule) - { - mPeerDeviceModule->StopRecording(); - mPeerDeviceModule->Terminate(); - } - mTuningDeviceModule = nullptr; - mPeerDeviceModule = nullptr; - mTaskQueueFactory = nullptr; - }); -} - void LLWebRTCImpl::refreshDevices() { mWorkerThread->PostTask([this]() { updateDevices(); }); @@ -307,6 +344,8 @@ void LLWebRTCImpl::unsetDevicesObserver(LLWebRTCDevicesObserver *observer) } } +// TODO: There's potential for shared code here as the patterns +// are similar. void LLWebRTCImpl::setCaptureDevice(const std::string &id) { mWorkerThread->PostTask( @@ -377,7 +416,7 @@ void LLWebRTCImpl::setRenderDevice(const std::string &id) mTuningDeviceModule->PlayoutDeviceName(i, name, guid); if (id == guid || id == "Default") { - RTC_LOG(LS_INFO) << __FUNCTION__ << "Set recording device to " << name << " " << guid << " " << i; + RTC_LOG(LS_INFO) << __FUNCTION__ << "Set playout device to " << name << " " << guid << " " << i; tuningPlayoutDevice = i; break; } @@ -407,7 +446,7 @@ void LLWebRTCImpl::setRenderDevice(const std::string &id) if (id == guid || id == "Default") { RTC_LOG(LS_INFO) - << __FUNCTION__ << "Set recording device to " << name << " " << guid << " " << i; + << __FUNCTION__ << "Set playout device to " << name << " " << guid << " " << i; mPlayoutDevice = i; break; } @@ -422,6 +461,7 @@ void LLWebRTCImpl::setRenderDevice(const std::string &id) }); } +// updateDevices needs to happen on the worker thread. void LLWebRTCImpl::updateDevices() { int16_t renderDeviceCount = mTuningDeviceModule->PlayoutDevices(); @@ -482,13 +522,14 @@ void LLWebRTCImpl::setTuningMode(bool enable) float LLWebRTCImpl::getTuningAudioLevel() { return -20 * log10f(mTuningAudioDeviceObserver->getMicrophoneEnergy()); } -float LLWebRTCImpl::getPeerAudioLevel() { return -20 * log10f(mPeerCustomProcessor->getMicrophoneEnergy()); } +float LLWebRTCImpl::getPeerConnectionAudioLevel() { return -20 * log10f(mPeerCustomProcessor->getMicrophoneEnergy()); } + // -// Helpers +// Peer Connection Helpers // -LLWebRTCPeerConnection * LLWebRTCImpl::newPeerConnection() +LLWebRTCPeerConnectionInterface *LLWebRTCImpl::newPeerConnection() { rtc::scoped_refptr<LLWebRTCPeerConnectionImpl> peerConnection = rtc::scoped_refptr<LLWebRTCPeerConnectionImpl>(new rtc::RefCountedObject<LLWebRTCPeerConnectionImpl>()); peerConnection->init(this); @@ -498,7 +539,7 @@ LLWebRTCPeerConnection * LLWebRTCImpl::newPeerConnection() return peerConnection.get(); } -void LLWebRTCImpl::freePeerConnection(LLWebRTCPeerConnection * peer_connection) +void LLWebRTCImpl::freePeerConnection(LLWebRTCPeerConnectionInterface* peer_connection) { std::vector<rtc::scoped_refptr<LLWebRTCPeerConnectionImpl>>::iterator it = std::find(mPeerConnections.begin(), mPeerConnections.end(), peer_connection); @@ -513,6 +554,20 @@ void LLWebRTCImpl::freePeerConnection(LLWebRTCPeerConnection * peer_connection) } } + +// +// LLWebRTCPeerConnectionImpl implementation. +// +// Most peer connection (signaling) happens on +// the signaling thread. + +LLWebRTCPeerConnectionImpl::LLWebRTCPeerConnectionImpl() : + mWebRTCImpl(nullptr), + mMute(false), + mAnswerReceived(false) +{ +} + // // LLWebRTCPeerConnection interface // @@ -547,6 +602,9 @@ void LLWebRTCPeerConnectionImpl::unsetSignalingObserver(LLWebRTCSignalingObserve } } +// TODO: Add initialization structure through which +// stun and turn servers may be passed in from +// the sim or login. bool LLWebRTCPeerConnectionImpl::initializeConnection() { RTC_DCHECK(!mPeerConnection); @@ -663,7 +721,7 @@ bool LLWebRTCPeerConnectionImpl::shutdownConnection() } for (auto &observer : mSignalingObserverList) { - observer->OnPeerShutDown(); + observer->OnPeerConnectionShutdown(); } }); return true; @@ -673,7 +731,7 @@ bool LLWebRTCPeerConnectionImpl::shutdownConnection() void LLWebRTCPeerConnectionImpl::enableSenderTracks(bool enable) { - // set_enabled shouldn't be done on the worker thread + // set_enabled shouldn't be done on the worker thread. if (mPeerConnection) { auto senders = mPeerConnection->GetSenders(); @@ -697,6 +755,7 @@ void LLWebRTCPeerConnectionImpl::enableReceiverTracks(bool enable) } } +// Tell the peer connection that we've received a SDP answer from the sim. void LLWebRTCPeerConnectionImpl::AnswerAvailable(const std::string &sdp) { RTC_LOG(LS_INFO) << __FUNCTION__ << " Remote SDP: " << sdp; @@ -713,6 +772,11 @@ void LLWebRTCPeerConnectionImpl::AnswerAvailable(const std::string &sdp) }); } + +// +// LLWebRTCAudioInterface implementation +// + void LLWebRTCPeerConnectionImpl::setMute(bool mute) { mMute = mute; @@ -812,21 +876,21 @@ void LLWebRTCPeerConnectionImpl::OnDataChannel(rtc::scoped_refptr<webrtc::DataCh void LLWebRTCPeerConnectionImpl::OnIceGatheringChange(webrtc::PeerConnectionInterface::IceGatheringState new_state) { - LLWebRTCSignalingObserver::IceGatheringState webrtc_new_state = LLWebRTCSignalingObserver::IceGatheringState::ICE_GATHERING_NEW; + LLWebRTCSignalingObserver::EIceGatheringState webrtc_new_state = LLWebRTCSignalingObserver::EIceGatheringState::ICE_GATHERING_NEW; switch (new_state) { case webrtc::PeerConnectionInterface::IceGatheringState::kIceGatheringNew: - webrtc_new_state = LLWebRTCSignalingObserver::IceGatheringState::ICE_GATHERING_NEW; + webrtc_new_state = LLWebRTCSignalingObserver::EIceGatheringState::ICE_GATHERING_NEW; break; case webrtc::PeerConnectionInterface::IceGatheringState::kIceGatheringGathering: - webrtc_new_state = LLWebRTCSignalingObserver::IceGatheringState::ICE_GATHERING_GATHERING; + webrtc_new_state = LLWebRTCSignalingObserver::EIceGatheringState::ICE_GATHERING_GATHERING; break; case webrtc::PeerConnectionInterface::IceGatheringState::kIceGatheringComplete: - webrtc_new_state = LLWebRTCSignalingObserver::IceGatheringState::ICE_GATHERING_COMPLETE; + webrtc_new_state = LLWebRTCSignalingObserver::EIceGatheringState::ICE_GATHERING_COMPLETE; break; default: RTC_LOG(LS_ERROR) << __FUNCTION__ << " Bad Ice Gathering State" << new_state; - webrtc_new_state = LLWebRTCSignalingObserver::IceGatheringState::ICE_GATHERING_NEW; + webrtc_new_state = LLWebRTCSignalingObserver::EIceGatheringState::ICE_GATHERING_NEW; return; } @@ -875,6 +939,8 @@ void LLWebRTCPeerConnectionImpl::OnConnectionChange(webrtc::PeerConnectionInterf } } +// Convert an ICE candidate into a string appropriate for trickling +// to the Secondlife WebRTC server via the sim. static std::string iceCandidateToTrickleString(const webrtc::IceCandidateInterface *candidate) { std::ostringstream candidate_stream; @@ -920,6 +986,7 @@ static std::string iceCandidateToTrickleString(const webrtc::IceCandidateInterfa return candidate_stream.str(); } +// The webrtc library has a new ice candidate. void LLWebRTCPeerConnectionImpl::OnIceCandidate(const webrtc::IceCandidateInterface *candidate) { RTC_LOG(LS_INFO) << __FUNCTION__ << " " << candidate->sdp_mline_index(); @@ -931,19 +998,24 @@ void LLWebRTCPeerConnectionImpl::OnIceCandidate(const webrtc::IceCandidateInterf } if (mAnswerReceived) { + // We've already received an answer SDP from the Secondlife WebRTC server + // so simply tell observers about our new ice candidate. for (auto &observer : mSignalingObserverList) { LLWebRTCIceCandidate ice_candidate; - ice_candidate.candidate = iceCandidateToTrickleString(candidate); - ice_candidate.mline_index = candidate->sdp_mline_index(); - ice_candidate.sdp_mid = candidate->sdp_mid(); + ice_candidate.mCandidate = iceCandidateToTrickleString(candidate); + ice_candidate.mMLineIndex = candidate->sdp_mline_index(); + ice_candidate.mSdpMid = candidate->sdp_mid(); observer->OnIceCandidate(ice_candidate); } } else { + // As we've not yet received our answer, cache the candidate. mCachedIceCandidates.push_back( - webrtc::CreateIceCandidate(candidate->sdp_mid(), candidate->sdp_mline_index(), candidate->candidate())); + webrtc::CreateIceCandidate(candidate->sdp_mid(), + candidate->sdp_mline_index(), + candidate->candidate())); } } @@ -994,13 +1066,18 @@ void LLWebRTCPeerConnectionImpl::OnSuccess(webrtc::SessionDescriptionInterface * rtc::scoped_refptr<webrtc::SetLocalDescriptionObserverInterface>(this)); } -void LLWebRTCPeerConnectionImpl::OnFailure(webrtc::RTCError error) { RTC_LOG(LS_ERROR) << ToString(error.type()) << ": " << error.message(); } +void LLWebRTCPeerConnectionImpl::OnFailure(webrtc::RTCError error) +{ + RTC_LOG(LS_ERROR) << ToString(error.type()) << ": " << error.message(); +} // // SetRemoteDescriptionObserverInterface implementation. // void LLWebRTCPeerConnectionImpl::OnSetRemoteDescriptionComplete(webrtc::RTCError error) { + // we've received an answer SDP from the sim. + RTC_LOG(LS_INFO) << __FUNCTION__ << " " << mPeerConnection->signaling_state(); if (!error.ok()) { @@ -1008,14 +1085,16 @@ void LLWebRTCPeerConnectionImpl::OnSetRemoteDescriptionComplete(webrtc::RTCError return; } mAnswerReceived = true; + + // tell the observers about any cached ICE candidates. for (auto &observer : mSignalingObserverList) { for (auto &candidate : mCachedIceCandidates) { LLWebRTCIceCandidate ice_candidate; - ice_candidate.candidate = iceCandidateToTrickleString(candidate.get()); - ice_candidate.mline_index = candidate->sdp_mline_index(); - ice_candidate.sdp_mid = candidate->sdp_mid(); + ice_candidate.mCandidate = iceCandidateToTrickleString(candidate.get()); + ice_candidate.mMLineIndex = candidate->sdp_mline_index(); + ice_candidate.mSdpMid = candidate->sdp_mid(); observer->OnIceCandidate(ice_candidate); } } @@ -1061,7 +1140,6 @@ void LLWebRTCPeerConnectionImpl::OnStateChange() } } - void LLWebRTCPeerConnectionImpl::OnMessage(const webrtc::DataBuffer& buffer) { std::string data((const char*)buffer.data.cdata(), buffer.size()); @@ -1071,6 +1149,10 @@ void LLWebRTCPeerConnectionImpl::OnMessage(const webrtc::DataBuffer& buffer) } } +// +// LLWebRTCDataInterface +// + void LLWebRTCPeerConnectionImpl::sendData(const std::string& data, bool binary) { if (mDataChannel) @@ -1081,7 +1163,10 @@ void LLWebRTCPeerConnectionImpl::sendData(const std::string& data, bool binary) } } -void LLWebRTCPeerConnectionImpl::setDataObserver(LLWebRTCDataObserver* observer) { mDataObserverList.emplace_back(observer); } +void LLWebRTCPeerConnectionImpl::setDataObserver(LLWebRTCDataObserver* observer) +{ + mDataObserverList.emplace_back(observer); +} void LLWebRTCPeerConnectionImpl::unsetDataObserver(LLWebRTCDataObserver* observer) { @@ -1099,12 +1184,12 @@ LLWebRTCDeviceInterface * getDeviceInterface() return gWebRTCImpl; } -LLWebRTCPeerConnection * newPeerConnection() +LLWebRTCPeerConnectionInterface* newPeerConnection() { return gWebRTCImpl->newPeerConnection(); } -void freePeerConnection(LLWebRTCPeerConnection *peer_connection) +void freePeerConnection(LLWebRTCPeerConnectionInterface* peer_connection) { gWebRTCImpl->freePeerConnection(peer_connection); } @@ -1112,7 +1197,7 @@ void freePeerConnection(LLWebRTCPeerConnection *peer_connection) void init() { - gWebRTCImpl = new LLWebRTCImpl(); + gWebRTCImpl = new LLWebRTCImpl(); gWebRTCImpl->init(); } diff --git a/indra/llwebrtc/llwebrtc.h b/indra/llwebrtc/llwebrtc.h index 7743ac7ba0..789bde452e 100644 --- a/indra/llwebrtc/llwebrtc.h +++ b/indra/llwebrtc/llwebrtc.h @@ -24,6 +24,17 @@ * $/LicenseInfo$ */ +/* + * llwebrtc wraps the native webrtc c++ library in a dynamic library with a simlified interface + * so that the viewer can use it. This is done because native webrtc has a different + * overall threading model than the viewer. + * The native webrtc library is also compiled with clang, and has memory management + * functions that conflict namespace-wise with those in the viewer. + * + * Due to these differences, code from the viewer cannot be pulled in to this + * dynamic library, so it remains very simple. + */ + #ifndef LLWEBRTC_H #define LLWEBRTC_H @@ -45,52 +56,74 @@ namespace llwebrtc { -struct LLWebRTCIceCandidate -{ - std::string candidate; - std::string sdp_mid; - int mline_index; -}; +// LLWebRTCVoiceDevice is a simple representation of the +// components of a device, used to communicate this +// information to the viewer. + + +// A note on threading. +// Native WebRTC has it's own threading model. Some discussion +// can be found here (https://webrtc.github.io/webrtc-org/native-code/native-apis/) +// +// Note that all callbacks to observers will occurr on one of the WebRTC native threads +// (signaling, worker, etc.) Care should be taken to assure there are not +// bad interactions with the viewer threads. class LLWebRTCVoiceDevice { public: - std::string display_name; // friendly value for the user - std::string id; // internal value for selection - bool current; // current device + std::string mDisplayName; // friendly name for user interface purposes + std::string mID; // internal value for selection + bool mCurrent; // current device LLWebRTCVoiceDevice(const std::string &display_name, const std::string &id, bool current) : - display_name(display_name), - id(id), - current(current) + mDisplayName(display_name), + mID(id), + mCurrent(current) {}; }; typedef std::vector<LLWebRTCVoiceDevice> LLWebRTCVoiceDeviceList; + +// The LLWebRTCDeviceObserver should be implemented by the viewer +// webrtc module, which will receive notifications when devices +// change (are unplugged, etc.) class LLWebRTCDevicesObserver { public: - virtual void OnDevicesChanged(const LLWebRTCVoiceDeviceList &render_devices, const LLWebRTCVoiceDeviceList &capture_devices) = 0; + virtual void OnDevicesChanged(const LLWebRTCVoiceDeviceList &render_devices, + const LLWebRTCVoiceDeviceList &capture_devices) = 0; }; + +// The LLWebRTCDeviceInterface provides a way for the viewer +// to enumerate, set, and get notifications of changes +// for both capture (microphone) and render (speaker) +// devices. class LLWebRTCDeviceInterface { public: + // instructs webrtc to refresh the device list. virtual void refreshDevices() = 0; + // set the capture and render devices using the unique identifier for the device virtual void setCaptureDevice(const std::string& id) = 0; virtual void setRenderDevice(const std::string& id) = 0; + // Device observers for device change callbacks. virtual void setDevicesObserver(LLWebRTCDevicesObserver *observer) = 0; virtual void unsetDevicesObserver(LLWebRTCDevicesObserver *observer) = 0; + // tuning and audio levels virtual void setTuningMode(bool enable) = 0; - virtual float getTuningAudioLevel() = 0; - virtual float getPeerAudioLevel() = 0; + virtual float getTuningAudioLevel() = 0; // for use during tuning + virtual float getPeerConnectionAudioLevel() = 0; // for use when not tuning }; +// LLWebRTCAudioInterface provides the viewer with a way +// to set audio characteristics (mute, send and receive volume) class LLWebRTCAudioInterface { public: @@ -99,41 +132,81 @@ class LLWebRTCAudioInterface virtual void setSendVolume(float volume) = 0; // volume between 0.0 and 1.0 }; +// LLWebRTCDataObserver allows the viewer voice module to be notified when +// data is received over the data channel. class LLWebRTCDataObserver { public: virtual void OnDataReceived(const std::string& data, bool binary) = 0; }; +// LLWebRTCDataInterface allows the viewer to send data over the data channel. class LLWebRTCDataInterface { public: + virtual void sendData(const std::string& data, bool binary=false) = 0; virtual void setDataObserver(LLWebRTCDataObserver *observer) = 0; virtual void unsetDataObserver(LLWebRTCDataObserver *observer) = 0; }; +// LLWebRTCIceCandidate is a basic structure containing +// information needed for ICE trickling. +struct LLWebRTCIceCandidate +{ + std::string mCandidate; + std::string mSdpMid; + int mMLineIndex; +}; + +// LLWebRTCSignalingObserver provides a way for the native +// webrtc library to notify the viewer voice module of +// various state changes. class LLWebRTCSignalingObserver { - public: - enum IceGatheringState{ + public: + + typedef enum e_ice_gathering_state { ICE_GATHERING_NEW, ICE_GATHERING_GATHERING, ICE_GATHERING_COMPLETE - }; - virtual void OnIceGatheringState(IceGatheringState state) = 0; + } EIceGatheringState; + + // Called when ICE gathering states have changed. + // This may be called at any time, as ICE gathering + // can be redone while a connection is up. + virtual void OnIceGatheringState(EIceGatheringState state) = 0; + + // Called when a new ice candidate is available. virtual void OnIceCandidate(const LLWebRTCIceCandidate& candidate) = 0; + + // Called when an offer is available after a connection is requested. virtual void OnOfferAvailable(const std::string& sdp) = 0; + + // Called when a connection enters a failure state and renegotiation is needed. virtual void OnRenegotiationNeeded() = 0; + + // Called when the audio channel has been established and audio + // can begin. virtual void OnAudioEstablished(LLWebRTCAudioInterface *audio_interface) = 0; + + // Called when the data channel has been established and data + // transfer can begin. virtual void OnDataChannelReady(LLWebRTCDataInterface *data_interface) = 0; - virtual void OnPeerShutDown() = 0; + + // Called when a peer connection has finished shutting down. + virtual void OnPeerConnectionShutdown() = 0; }; -class LLWebRTCPeerConnection + +// LLWebRTCPeerConnectionInterface representsd a connection to a peer, +// in most cases a Secondlife WebRTC server. This interface +// allows for management of this peer connection. +class LLWebRTCPeerConnectionInterface { public: + virtual void setSignalingObserver(LLWebRTCSignalingObserver* observer) = 0; virtual void unsetSignalingObserver(LLWebRTCSignalingObserver* observer) = 0; @@ -142,11 +215,21 @@ class LLWebRTCPeerConnection virtual void AnswerAvailable(const std::string &sdp) = 0; }; +// The following define the dynamic linked library +// exports. + +// This library must be initialized before use. LLSYMEXPORT void init(); + +// And should be terminated as part of shutdown. LLSYMEXPORT void terminate(); + +// Return an interface for device management. LLSYMEXPORT LLWebRTCDeviceInterface* getDeviceInterface(); -LLSYMEXPORT LLWebRTCPeerConnection* newPeerConnection(); -LLSYMEXPORT void freePeerConnection(LLWebRTCPeerConnection *connection); + +// Allocate and free peer connections. +LLSYMEXPORT LLWebRTCPeerConnectionInterface* newPeerConnection(); +LLSYMEXPORT void freePeerConnection(LLWebRTCPeerConnectionInterface *connection); } #endif // LLWEBRTC_H diff --git a/indra/llwebrtc/llwebrtc_impl.h b/indra/llwebrtc/llwebrtc_impl.h index 3c182c4b02..c2d0a4413e 100644 --- a/indra/llwebrtc/llwebrtc_impl.h +++ b/indra/llwebrtc/llwebrtc_impl.h @@ -1,6 +1,6 @@ /** * @file llwebrtc_impl.h - * @brief WebRTC interface implementation header + * @brief WebRTC dynamic library implementation header * * $LicenseInfo:firstyear=2023&license=viewerlgpl$ * Second Life Viewer Source Code @@ -65,19 +65,27 @@ namespace llwebrtc class LLWebRTCPeerConnectionImpl; + +// Implements a class allowing capture of audio data +// to determine audio level of the microphone. class LLAudioDeviceObserver : public webrtc::AudioDeviceDataObserver { public: LLAudioDeviceObserver(); + // Retrieve the RMS audio loudness float getMicrophoneEnergy(); + // Data retrieved from the caputure device is + // passed in here for processing. void OnCaptureData(const void *audio_samples, const size_t num_samples, const size_t bytes_per_sample, const size_t num_channels, const uint32_t samples_per_sec) override; + // This is for data destined for the render device. + // not currently used. void OnRenderData(const void *audio_samples, const size_t num_samples, const size_t bytes_per_sample, @@ -85,10 +93,13 @@ class LLAudioDeviceObserver : public webrtc::AudioDeviceDataObserver const uint32_t samples_per_sec) override; protected: - float mSumVector[30]; // 300 ms of smoothing + static const int NUM_PACKETS_TO_FILTER = 30; // 300 ms of smoothing (30 frames) + float mSumVector[NUM_PACKETS_TO_FILTER]; float mMicrophoneEnergy; }; +// Used to process/retrieve audio levels after +// all of the processing (AGC, AEC, etc.) for display in-world to the user. class LLCustomProcessor : public webrtc::CustomProcessing { public: @@ -97,8 +108,10 @@ class LLCustomProcessor : public webrtc::CustomProcessing // (Re-) Initializes the submodule. void Initialize(int sample_rate_hz, int num_channels) override; + // Analyzes the given capture or render signal. void Process(webrtc::AudioBuffer *audio) override; + // Returns a string representation of the module state. std::string ToString() const override { return ""; } @@ -113,13 +126,13 @@ class LLCustomProcessor : public webrtc::CustomProcessing float mMicrophoneEnergy; }; + +// Primary singleton implementation for interfacing +// with the native webrtc library. class LLWebRTCImpl : public LLWebRTCDeviceInterface, public webrtc::AudioDeviceSink { public: - LLWebRTCImpl() : - mPeerCustomProcessor(nullptr), mMute(true) - { - } + LLWebRTCImpl(); ~LLWebRTCImpl() {} void init(); @@ -139,7 +152,7 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, public webrtc::AudioDeviceS void setTuningMode(bool enable) override; float getTuningAudioLevel() override; - float getPeerAudioLevel() override; + float getPeerConnectionAudioLevel() override; // // AudioDeviceSink @@ -150,6 +163,9 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, public webrtc::AudioDeviceS // Helpers // + // The following thread helpers allow the + // LLWebRTCPeerConnectionImpl class to post + // tasks to the native webrtc threads. void PostWorkerTask(absl::AnyInvocable<void() &&> task, const webrtc::Location& location = webrtc::Location::Current()) { @@ -185,21 +201,31 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, public webrtc::AudioDeviceS { mNetworkThread->BlockingCall(std::move(functor), location); } - - rtc::scoped_refptr<webrtc::PeerConnectionFactoryInterface> getPeerConnectionFactory() { return mPeerConnectionFactory; } - LLWebRTCPeerConnection * newPeerConnection(); - void freePeerConnection(LLWebRTCPeerConnection * peer_connection); + // Allows the LLWebRTCPeerConnectionImpl class to retrieve the + // native webrtc PeerConnectionFactory. + rtc::scoped_refptr<webrtc::PeerConnectionFactoryInterface> getPeerConnectionFactory() + { + return mPeerConnectionFactory; + } + // create or destroy a peer connection. + LLWebRTCPeerConnectionInterface* newPeerConnection(); + void freePeerConnection(LLWebRTCPeerConnectionInterface* peer_connection); + + // enables/disables capture via the capture device void setRecording(bool recording); protected: - + // The native webrtc threads std::unique_ptr<rtc::Thread> mNetworkThread; std::unique_ptr<rtc::Thread> mWorkerThread; std::unique_ptr<rtc::Thread> mSignalingThread; + + // The factory that allows creation of native webrtc PeerConnections. rtc::scoped_refptr<webrtc::PeerConnectionFactoryInterface> mPeerConnectionFactory; - webrtc::PeerConnectionInterface::RTCConfiguration mConfiguration; + + // more native webrtc stuff std::unique_ptr<webrtc::TaskQueueFactory> mTaskQueueFactory; @@ -209,11 +235,11 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, public webrtc::AudioDeviceS rtc::scoped_refptr<webrtc::AudioDeviceModule> mPeerDeviceModule; std::vector<LLWebRTCDevicesObserver *> mVoiceDevicesObserverList; - // accessors in webrtc aren't apparently implemented yet. + // accessors in native webrtc for devices aren't apparently implemented yet. int32_t mPlayoutDevice; int32_t mRecordingDevice; bool mMute; - + LLAudioDeviceObserver * mTuningAudioDeviceObserver; LLCustomProcessor * mPeerCustomProcessor; @@ -221,7 +247,11 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, public webrtc::AudioDeviceS std::vector<rtc::scoped_refptr<LLWebRTCPeerConnectionImpl>> mPeerConnections; }; -class LLWebRTCPeerConnectionImpl : public LLWebRTCPeerConnection, + +// The implementation of a peer connection, which contains +// the various interfaces used by the viewer to interact with +// the webrtc connection. +class LLWebRTCPeerConnectionImpl : public LLWebRTCPeerConnectionInterface, public LLWebRTCAudioInterface, public LLWebRTCDataInterface, public webrtc::PeerConnectionObserver, @@ -232,7 +262,7 @@ class LLWebRTCPeerConnectionImpl : public LLWebRTCPeerConnection, { public: - LLWebRTCPeerConnectionImpl() {} + LLWebRTCPeerConnectionImpl(); ~LLWebRTCPeerConnectionImpl() {} void init(LLWebRTCImpl * webrtc_impl); @@ -270,7 +300,7 @@ class LLWebRTCPeerConnectionImpl : public LLWebRTCPeerConnection, // void OnSignalingChange(webrtc::PeerConnectionInterface::SignalingState new_state) override {} - void OnAddTrack(rtc::scoped_refptr<webrtc::RtpReceiverInterface> receiver, + void OnAddTrack(rtc::scoped_refptr<webrtc::RtpReceiverInterface> receiver, const std::vector<rtc::scoped_refptr<webrtc::MediaStreamInterface>> &streams) override; void OnRemoveTrack(rtc::scoped_refptr<webrtc::RtpReceiverInterface> receiver) override; void OnDataChannel(rtc::scoped_refptr<webrtc::DataChannelInterface> channel) override; @@ -311,6 +341,7 @@ class LLWebRTCPeerConnectionImpl : public LLWebRTCPeerConnection, protected: LLWebRTCImpl * mWebRTCImpl; + rtc::scoped_refptr<webrtc::PeerConnectionFactoryInterface> mPeerConnectionFactory; bool mMute; @@ -323,6 +354,7 @@ class LLWebRTCPeerConnectionImpl : public LLWebRTCPeerConnection, rtc::scoped_refptr<webrtc::PeerConnectionInterface> mPeerConnection; rtc::scoped_refptr<webrtc::MediaStreamInterface> mLocalStream; + // data std::vector<LLWebRTCDataObserver *> mDataObserverList; rtc::scoped_refptr<webrtc::DataChannelInterface> mDataChannel; }; diff --git a/indra/newview/llvoicewebrtc.cpp b/indra/newview/llvoicewebrtc.cpp index 6e0cd32249..6eee8e70b2 100644 --- a/indra/newview/llvoicewebrtc.cpp +++ b/indra/newview/llvoicewebrtc.cpp @@ -597,8 +597,8 @@ void LLWebRTCVoiceClient::OnDevicesChanged(const llwebrtc::LLWebRTCVoiceDeviceLi bool renderDeviceSet = false; for (auto &device : render_devices) { - addRenderDevice(LLVoiceDevice(device.display_name, device.id)); - if (device.current && outputDevice == device.id) + addRenderDevice(LLVoiceDevice(device.mDisplayName, device.mID)); + if (device.mCurrent && outputDevice == device.mID) { setRenderDevice(outputDevice); renderDeviceSet = true; @@ -613,8 +613,8 @@ void LLWebRTCVoiceClient::OnDevicesChanged(const llwebrtc::LLWebRTCVoiceDeviceLi bool captureDeviceSet = false; for (auto &device : capture_devices) { - addCaptureDevice(LLVoiceDevice(device.display_name, device.id)); - if (device.current && inputDevice == device.id) + addCaptureDevice(LLVoiceDevice(device.mDisplayName, device.mID)); + if (device.mCurrent && inputDevice == device.mID) { setCaptureDevice(outputDevice); captureDeviceSet = true; @@ -696,7 +696,7 @@ float LLWebRTCVoiceClient::getAudioLevel() } else { - return (1.0 - mWebRTCDeviceInterface->getPeerAudioLevel() * LEVEL_SCALE_WEBRTC) * mMicGain / 2.1; + return (1.0 - mWebRTCDeviceInterface->getPeerConnectionAudioLevel() * LEVEL_SCALE_WEBRTC) * mMicGain / 2.1; } } @@ -2030,8 +2030,8 @@ LLVoiceWebRTCConnection::LLVoiceWebRTCConnection(const LLUUID ®ionID, const s mChannelID(channelID), mRegionID(regionID) { - mWebRTCPeerConnection = llwebrtc::newPeerConnection(); - mWebRTCPeerConnection->setSignalingObserver(this); + mWebRTCPeerConnectionInterface = llwebrtc::newPeerConnection(); + mWebRTCPeerConnectionInterface->setSignalingObserver(this); } LLVoiceWebRTCConnection::~LLVoiceWebRTCConnection() @@ -2042,10 +2042,10 @@ LLVoiceWebRTCConnection::~LLVoiceWebRTCConnection() // by llwebrtc::terminate() on shutdown. return; } - if (mWebRTCPeerConnection) + if (mWebRTCPeerConnectionInterface) { - llwebrtc::freePeerConnection(mWebRTCPeerConnection); - mWebRTCPeerConnection = nullptr; + llwebrtc::freePeerConnection(mWebRTCPeerConnectionInterface); + mWebRTCPeerConnectionInterface = nullptr; } } @@ -2058,19 +2058,19 @@ LLVoiceWebRTCConnection::~LLVoiceWebRTCConnection() // negotiated, updates about the best connectivity paths may trickle in. These need to be // sent to the Secondlife WebRTC server via the simulator so that both sides have a clear // view of the network environment. -void LLVoiceWebRTCConnection::OnIceGatheringState(llwebrtc::LLWebRTCSignalingObserver::IceGatheringState state) +void LLVoiceWebRTCConnection::OnIceGatheringState(llwebrtc::LLWebRTCSignalingObserver::EIceGatheringState state) { LL_DEBUGS("Voice") << "Ice Gathering voice account. " << state << LL_ENDL; switch (state) { - case llwebrtc::LLWebRTCSignalingObserver::IceGatheringState::ICE_GATHERING_COMPLETE: + case llwebrtc::LLWebRTCSignalingObserver::EIceGatheringState::ICE_GATHERING_COMPLETE: { LLMutexLock lock(&mVoiceStateMutex); mIceCompleted = true; break; } - case llwebrtc::LLWebRTCSignalingObserver::IceGatheringState::ICE_GATHERING_NEW: + case llwebrtc::LLWebRTCSignalingObserver::EIceGatheringState::ICE_GATHERING_NEW: { LLMutexLock lock(&mVoiceStateMutex); mIceCompleted = false; @@ -2167,9 +2167,9 @@ void LLVoiceWebRTCConnection::processIceUpdates() for (auto &ice_candidate : mIceCandidates) { LLSD body_candidate; - body_candidate["sdpMid"] = ice_candidate.sdp_mid; - body_candidate["sdpMLineIndex"] = ice_candidate.mline_index; - body_candidate["candidate"] = ice_candidate.candidate; + body_candidate["sdpMid"] = ice_candidate.mSdpMid; + body_candidate["sdpMLineIndex"] = ice_candidate.mMLineIndex; + body_candidate["candidate"] = ice_candidate.mCandidate; candidates.append(body_candidate); } body["candidates"] = candidates; @@ -2242,7 +2242,7 @@ void LLVoiceWebRTCConnection::OnRenegotiationNeeded() } } -void LLVoiceWebRTCConnection::OnPeerShutDown() +void LLVoiceWebRTCConnection::OnPeerConnectionShutdown() { setVoiceConnectionState(VOICE_STATE_SESSION_EXIT); mOutstandingRequests--; // shut down is an async call which is handled on a webrtc thread. @@ -2364,9 +2364,9 @@ void LLVoiceWebRTCConnection::OnVoiceDisconnectionRequestSuccess(const LLSD &res return; } - if (mWebRTCPeerConnection) + if (mWebRTCPeerConnectionInterface) { - if (mWebRTCPeerConnection->shutdownConnection()) + if (mWebRTCPeerConnectionInterface->shutdownConnection()) { mOutstandingRequests++; } @@ -2395,10 +2395,10 @@ void LLVoiceWebRTCConnection::OnVoiceDisconnectionRequestFailure(std::string url boost::bind(&LLVoiceWebRTCSpatialConnection::OnVoiceDisconnectionRequestFailure, this, url, retries - 1, body, _1)); return; } - if (mWebRTCPeerConnection) + if (mWebRTCPeerConnectionInterface) { mOutstandingRequests++; - mWebRTCPeerConnection->shutdownConnection(); + mWebRTCPeerConnectionInterface->shutdownConnection(); } else { @@ -2482,7 +2482,7 @@ void LLVoiceWebRTCConnection::OnVoiceConnectionRequestSuccess(const LLSD &result LL_DEBUGS("Voice") << "ProvisionVoiceAccountRequest response" << " channel sdp " << mRemoteChannelSDP << LL_ENDL; - mWebRTCPeerConnection->AnswerAvailable(mRemoteChannelSDP); + mWebRTCPeerConnectionInterface->AnswerAvailable(mRemoteChannelSDP); } @@ -2531,7 +2531,7 @@ bool LLVoiceWebRTCConnection::connectionStateMachine() // tell the webrtc library that we want a connection. The library will // respond with an offer on a separate thread, which will cause // the session state to change. - if (!mWebRTCPeerConnection->initializeConnection()) + if (!mWebRTCPeerConnectionInterface->initializeConnection()) { setVoiceConnectionState(VOICE_STATE_SESSION_RETRY); } @@ -2830,7 +2830,7 @@ LLVoiceWebRTCSpatialConnection::~LLVoiceWebRTCSpatialConnection() return; } assert(mOutstandingRequests == 0); - mWebRTCPeerConnection->unsetSignalingObserver(this); + mWebRTCPeerConnectionInterface->unsetSignalingObserver(this); } void LLVoiceWebRTCSpatialConnection::setMuteMic(bool muted) @@ -2873,7 +2873,7 @@ LLVoiceWebRTCAdHocConnection::~LLVoiceWebRTCAdHocConnection() return; } assert(mOutstandingRequests == 0); - mWebRTCPeerConnection->unsetSignalingObserver(this); + mWebRTCPeerConnectionInterface->unsetSignalingObserver(this); } // Add-hoc connections require a different channel type diff --git a/indra/newview/llvoicewebrtc.h b/indra/newview/llvoicewebrtc.h index e6f01fd181..3ff801ed56 100644 --- a/indra/newview/llvoicewebrtc.h +++ b/indra/newview/llvoicewebrtc.h @@ -584,12 +584,12 @@ class LLVoiceWebRTCConnection : /// @name Signaling notification // LLWebRTCSignalingObserver //@{ - void OnIceGatheringState(IceGatheringState state) override; + void OnIceGatheringState(EIceGatheringState state) override; void OnIceCandidate(const llwebrtc::LLWebRTCIceCandidate &candidate) override; void OnOfferAvailable(const std::string &sdp) override; void OnRenegotiationNeeded() override; void OnAudioEstablished(llwebrtc::LLWebRTCAudioInterface *audio_interface) override; - void OnPeerShutDown() override; + void OnPeerConnectionShutdown() override; //@} ///////////////////////// @@ -702,7 +702,7 @@ class LLVoiceWebRTCConnection : bool mIceCompleted; bool mTrickling; - llwebrtc::LLWebRTCPeerConnection *mWebRTCPeerConnection; + llwebrtc::LLWebRTCPeerConnectionInterface *mWebRTCPeerConnectionInterface; llwebrtc::LLWebRTCAudioInterface *mWebRTCAudioInterface; llwebrtc::LLWebRTCDataInterface *mWebRTCDataInterface; }; |