summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorRoxie Linden <roxie@lindenlab.com>2024-03-09 23:00:00 -0800
committerRoxie Linden <roxie@lindenlab.com>2024-03-09 23:00:00 -0800
commit7714850fbe2ad4c2dc914798971d82cbb82f7832 (patch)
tree0b4ce8b947fb3066fa9e90e78e7cd6ce34fb5f08
parentc7efdefb2c178eb7e2f8ac9e540f10fa8cfefab2 (diff)
code beautification/comments
-rw-r--r--indra/llwebrtc/llwebrtc.cpp213
-rw-r--r--indra/llwebrtc/llwebrtc.h129
-rw-r--r--indra/llwebrtc/llwebrtc_impl.h68
-rw-r--r--indra/newview/llvoicewebrtc.cpp50
-rw-r--r--indra/newview/llvoicewebrtc.h6
5 files changed, 333 insertions, 133 deletions
diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp
index 74af16679a..dc5fd9657e 100644
--- a/indra/llwebrtc/llwebrtc.cpp
+++ b/indra/llwebrtc/llwebrtc.cpp
@@ -44,12 +44,16 @@ LLAudioDeviceObserver::LLAudioDeviceObserver() : mSumVector {0}, mMicrophoneEner
float LLAudioDeviceObserver::getMicrophoneEnergy() { return mMicrophoneEnergy; }
+// TODO: Pull smoothing/filtering code into a common helper function
+// for LLAudioDeviceObserver and LLCustomProcessor
+
void LLAudioDeviceObserver::OnCaptureData(const void *audio_samples,
const size_t num_samples,
const size_t bytes_per_sample,
const size_t num_channels,
const uint32_t samples_per_sec)
{
+ // calculate the energy
float energy = 0;
const short *samples = (const short *) audio_samples;
for (size_t index = 0; index < num_samples * num_channels; index++)
@@ -82,7 +86,8 @@ void LLAudioDeviceObserver::OnRenderData(const void *audio_samples,
LLCustomProcessor::LLCustomProcessor() :
mSampleRateHz(0),
- mNumChannels(0)
+ mNumChannels(0),
+ mMicrophoneEnergy(0.0)
{
memset(mSumVector, 0, sizeof(mSumVector));
}
@@ -107,6 +112,7 @@ void LLCustomProcessor::Process(webrtc::AudioBuffer *audio_in)
return;
}
+ // grab the input audio
frame_samples.resize(stream_config.num_samples());
frame.resize(stream_config.num_channels());
for (size_t ch = 0; ch < stream_config.num_channels(); ++ch)
@@ -116,6 +122,7 @@ void LLCustomProcessor::Process(webrtc::AudioBuffer *audio_in)
audio_in->CopyTo(stream_config, &frame[0]);
+ // calculate the energy
float energy = 0;
for (size_t index = 0; index < stream_config.num_samples(); index++)
{
@@ -137,17 +144,33 @@ void LLCustomProcessor::Process(webrtc::AudioBuffer *audio_in)
mMicrophoneEnergy = std::sqrt(totalSum / (stream_config.num_samples() * buffer_size));
}
+//
+// LLWebRTCImpl implementation
+//
+
+LLWebRTCImpl::LLWebRTCImpl() :
+ mPeerCustomProcessor(nullptr),
+ mMute(true),
+ mPlayoutDevice(0),
+ mRecordingDevice(0),
+ mTuningAudioDeviceObserver(nullptr)
+{
+}
+
void LLWebRTCImpl::init()
{
RTC_DCHECK(mPeerConnectionFactory);
mPlayoutDevice = 0;
mRecordingDevice = 0;
rtc::InitializeSSL();
+
+ // Normal logging is rather spammy, so turn it off.
rtc::LogMessage::LogToDebug(rtc::LS_NONE);
rtc::LogMessage::SetLogToStderr(true);
mTaskQueueFactory = webrtc::CreateDefaultTaskQueueFactory();
-
+
+ // Create the native threads.
mNetworkThread = rtc::Thread::CreateWithSocketServer();
mNetworkThread->SetName("WebRTCNetworkThread", nullptr);
mNetworkThread->Start();
@@ -162,9 +185,12 @@ void LLWebRTCImpl::init()
mWorkerThread->PostTask(
[this]()
{
+ // Initialize the audio devices on the Worker Thread
mTuningDeviceModule = webrtc::CreateAudioDeviceWithDataObserver(
- webrtc::AudioDeviceModule::AudioLayer::kPlatformDefaultAudio, mTaskQueueFactory.get(),
- std::unique_ptr<webrtc::AudioDeviceDataObserver>(mTuningAudioDeviceObserver));
+ webrtc::AudioDeviceModule::AudioLayer::kPlatformDefaultAudio,
+ mTaskQueueFactory.get(),
+ std::unique_ptr<webrtc::AudioDeviceDataObserver>(mTuningAudioDeviceObserver));
+
mTuningDeviceModule->Init();
mTuningDeviceModule->SetStereoRecording(true);
mTuningDeviceModule->SetStereoPlayout(true);
@@ -176,9 +202,13 @@ void LLWebRTCImpl::init()
mWorkerThread->BlockingCall(
[this]()
{
+ // the peer device module doesn't need an observer
+ // as we pull peer data after audio processing.
mPeerDeviceModule =
- webrtc::CreateAudioDeviceWithDataObserver(webrtc::AudioDeviceModule::AudioLayer::kPlatformDefaultAudio,
- mTaskQueueFactory.get(), nullptr);
+ webrtc::CreateAudioDeviceWithDataObserver(
+ webrtc::AudioDeviceModule::AudioLayer::kPlatformDefaultAudio,
+ mTaskQueueFactory.get(),
+ nullptr);
mPeerDeviceModule->Init();
mPeerDeviceModule->SetPlayoutDevice(mPlayoutDevice);
mPeerDeviceModule->SetRecordingDevice(mRecordingDevice);
@@ -191,11 +221,15 @@ void LLWebRTCImpl::init()
mPeerDeviceModule->InitPlayout();
});
+ // The custom processor allows us to retrieve audio data (and levels)
+ // from after other audio processing such as AEC, AGC, etc.
mPeerCustomProcessor = new LLCustomProcessor;
webrtc::AudioProcessingBuilder apb;
apb.SetCapturePostProcessing(std::unique_ptr<webrtc::CustomProcessing>(mPeerCustomProcessor));
rtc::scoped_refptr<webrtc::AudioProcessing> apm = apb.Create();
+ // TODO: wire some of these to the primary interface and ultimately
+ // to the UI to allow user config.
webrtc::AudioProcessing::Config apm_config;
apm_config.echo_canceller.enabled = true;
apm_config.echo_canceller.mobile_mode = false;
@@ -242,6 +276,41 @@ void LLWebRTCImpl::init()
});
}
+void LLWebRTCImpl::terminate()
+{
+ for (auto &connection : mPeerConnections)
+ {
+ connection->terminate();
+ }
+ mPeerConnections.clear();
+
+ mSignalingThread->BlockingCall([this]() { mPeerConnectionFactory = nullptr; });
+ mWorkerThread->BlockingCall(
+ [this]()
+ {
+ if (mTuningDeviceModule)
+ {
+ mTuningDeviceModule->StopRecording();
+ mTuningDeviceModule->Terminate();
+ }
+ if (mPeerDeviceModule)
+ {
+ mPeerDeviceModule->StopRecording();
+ mPeerDeviceModule->Terminate();
+ }
+ mTuningDeviceModule = nullptr;
+ mPeerDeviceModule = nullptr;
+ mTaskQueueFactory = nullptr;
+ });
+}
+
+//
+// Devices functions
+//
+// Most device-related functionality needs to happen
+// on the worker thread (the audio thread,) so those calls will be
+// proxied over to that thread.
+//
void LLWebRTCImpl::setRecording(bool recording)
{
mWorkerThread->PostTask(
@@ -258,38 +327,6 @@ void LLWebRTCImpl::setRecording(bool recording)
});
}
-void LLWebRTCImpl::terminate()
-{
- for (auto& connection : mPeerConnections)
- {
- connection->terminate();
- }
- mPeerConnections.clear();
-
- mSignalingThread->BlockingCall(
- [this]()
- {
- mPeerConnectionFactory = nullptr;
- });
- mWorkerThread->BlockingCall(
- [this]()
- {
- if (mTuningDeviceModule)
- {
- mTuningDeviceModule->StopRecording();
- mTuningDeviceModule->Terminate();
- }
- if (mPeerDeviceModule)
- {
- mPeerDeviceModule->StopRecording();
- mPeerDeviceModule->Terminate();
- }
- mTuningDeviceModule = nullptr;
- mPeerDeviceModule = nullptr;
- mTaskQueueFactory = nullptr;
- });
-}
-
void LLWebRTCImpl::refreshDevices()
{
mWorkerThread->PostTask([this]() { updateDevices(); });
@@ -307,6 +344,8 @@ void LLWebRTCImpl::unsetDevicesObserver(LLWebRTCDevicesObserver *observer)
}
}
+// TODO: There's potential for shared code here as the patterns
+// are similar.
void LLWebRTCImpl::setCaptureDevice(const std::string &id)
{
mWorkerThread->PostTask(
@@ -377,7 +416,7 @@ void LLWebRTCImpl::setRenderDevice(const std::string &id)
mTuningDeviceModule->PlayoutDeviceName(i, name, guid);
if (id == guid || id == "Default")
{
- RTC_LOG(LS_INFO) << __FUNCTION__ << "Set recording device to " << name << " " << guid << " " << i;
+ RTC_LOG(LS_INFO) << __FUNCTION__ << "Set playout device to " << name << " " << guid << " " << i;
tuningPlayoutDevice = i;
break;
}
@@ -407,7 +446,7 @@ void LLWebRTCImpl::setRenderDevice(const std::string &id)
if (id == guid || id == "Default")
{
RTC_LOG(LS_INFO)
- << __FUNCTION__ << "Set recording device to " << name << " " << guid << " " << i;
+ << __FUNCTION__ << "Set playout device to " << name << " " << guid << " " << i;
mPlayoutDevice = i;
break;
}
@@ -422,6 +461,7 @@ void LLWebRTCImpl::setRenderDevice(const std::string &id)
});
}
+// updateDevices needs to happen on the worker thread.
void LLWebRTCImpl::updateDevices()
{
int16_t renderDeviceCount = mTuningDeviceModule->PlayoutDevices();
@@ -482,13 +522,14 @@ void LLWebRTCImpl::setTuningMode(bool enable)
float LLWebRTCImpl::getTuningAudioLevel() { return -20 * log10f(mTuningAudioDeviceObserver->getMicrophoneEnergy()); }
-float LLWebRTCImpl::getPeerAudioLevel() { return -20 * log10f(mPeerCustomProcessor->getMicrophoneEnergy()); }
+float LLWebRTCImpl::getPeerConnectionAudioLevel() { return -20 * log10f(mPeerCustomProcessor->getMicrophoneEnergy()); }
+
//
-// Helpers
+// Peer Connection Helpers
//
-LLWebRTCPeerConnection * LLWebRTCImpl::newPeerConnection()
+LLWebRTCPeerConnectionInterface *LLWebRTCImpl::newPeerConnection()
{
rtc::scoped_refptr<LLWebRTCPeerConnectionImpl> peerConnection = rtc::scoped_refptr<LLWebRTCPeerConnectionImpl>(new rtc::RefCountedObject<LLWebRTCPeerConnectionImpl>());
peerConnection->init(this);
@@ -498,7 +539,7 @@ LLWebRTCPeerConnection * LLWebRTCImpl::newPeerConnection()
return peerConnection.get();
}
-void LLWebRTCImpl::freePeerConnection(LLWebRTCPeerConnection * peer_connection)
+void LLWebRTCImpl::freePeerConnection(LLWebRTCPeerConnectionInterface* peer_connection)
{
std::vector<rtc::scoped_refptr<LLWebRTCPeerConnectionImpl>>::iterator it =
std::find(mPeerConnections.begin(), mPeerConnections.end(), peer_connection);
@@ -513,6 +554,20 @@ void LLWebRTCImpl::freePeerConnection(LLWebRTCPeerConnection * peer_connection)
}
}
+
+//
+// LLWebRTCPeerConnectionImpl implementation.
+//
+// Most peer connection (signaling) happens on
+// the signaling thread.
+
+LLWebRTCPeerConnectionImpl::LLWebRTCPeerConnectionImpl() :
+ mWebRTCImpl(nullptr),
+ mMute(false),
+ mAnswerReceived(false)
+{
+}
+
//
// LLWebRTCPeerConnection interface
//
@@ -547,6 +602,9 @@ void LLWebRTCPeerConnectionImpl::unsetSignalingObserver(LLWebRTCSignalingObserve
}
}
+// TODO: Add initialization structure through which
+// stun and turn servers may be passed in from
+// the sim or login.
bool LLWebRTCPeerConnectionImpl::initializeConnection()
{
RTC_DCHECK(!mPeerConnection);
@@ -663,7 +721,7 @@ bool LLWebRTCPeerConnectionImpl::shutdownConnection()
}
for (auto &observer : mSignalingObserverList)
{
- observer->OnPeerShutDown();
+ observer->OnPeerConnectionShutdown();
}
});
return true;
@@ -673,7 +731,7 @@ bool LLWebRTCPeerConnectionImpl::shutdownConnection()
void LLWebRTCPeerConnectionImpl::enableSenderTracks(bool enable)
{
- // set_enabled shouldn't be done on the worker thread
+ // set_enabled shouldn't be done on the worker thread.
if (mPeerConnection)
{
auto senders = mPeerConnection->GetSenders();
@@ -697,6 +755,7 @@ void LLWebRTCPeerConnectionImpl::enableReceiverTracks(bool enable)
}
}
+// Tell the peer connection that we've received a SDP answer from the sim.
void LLWebRTCPeerConnectionImpl::AnswerAvailable(const std::string &sdp)
{
RTC_LOG(LS_INFO) << __FUNCTION__ << " Remote SDP: " << sdp;
@@ -713,6 +772,11 @@ void LLWebRTCPeerConnectionImpl::AnswerAvailable(const std::string &sdp)
});
}
+
+//
+// LLWebRTCAudioInterface implementation
+//
+
void LLWebRTCPeerConnectionImpl::setMute(bool mute)
{
mMute = mute;
@@ -812,21 +876,21 @@ void LLWebRTCPeerConnectionImpl::OnDataChannel(rtc::scoped_refptr<webrtc::DataCh
void LLWebRTCPeerConnectionImpl::OnIceGatheringChange(webrtc::PeerConnectionInterface::IceGatheringState new_state)
{
- LLWebRTCSignalingObserver::IceGatheringState webrtc_new_state = LLWebRTCSignalingObserver::IceGatheringState::ICE_GATHERING_NEW;
+ LLWebRTCSignalingObserver::EIceGatheringState webrtc_new_state = LLWebRTCSignalingObserver::EIceGatheringState::ICE_GATHERING_NEW;
switch (new_state)
{
case webrtc::PeerConnectionInterface::IceGatheringState::kIceGatheringNew:
- webrtc_new_state = LLWebRTCSignalingObserver::IceGatheringState::ICE_GATHERING_NEW;
+ webrtc_new_state = LLWebRTCSignalingObserver::EIceGatheringState::ICE_GATHERING_NEW;
break;
case webrtc::PeerConnectionInterface::IceGatheringState::kIceGatheringGathering:
- webrtc_new_state = LLWebRTCSignalingObserver::IceGatheringState::ICE_GATHERING_GATHERING;
+ webrtc_new_state = LLWebRTCSignalingObserver::EIceGatheringState::ICE_GATHERING_GATHERING;
break;
case webrtc::PeerConnectionInterface::IceGatheringState::kIceGatheringComplete:
- webrtc_new_state = LLWebRTCSignalingObserver::IceGatheringState::ICE_GATHERING_COMPLETE;
+ webrtc_new_state = LLWebRTCSignalingObserver::EIceGatheringState::ICE_GATHERING_COMPLETE;
break;
default:
RTC_LOG(LS_ERROR) << __FUNCTION__ << " Bad Ice Gathering State" << new_state;
- webrtc_new_state = LLWebRTCSignalingObserver::IceGatheringState::ICE_GATHERING_NEW;
+ webrtc_new_state = LLWebRTCSignalingObserver::EIceGatheringState::ICE_GATHERING_NEW;
return;
}
@@ -875,6 +939,8 @@ void LLWebRTCPeerConnectionImpl::OnConnectionChange(webrtc::PeerConnectionInterf
}
}
+// Convert an ICE candidate into a string appropriate for trickling
+// to the Secondlife WebRTC server via the sim.
static std::string iceCandidateToTrickleString(const webrtc::IceCandidateInterface *candidate)
{
std::ostringstream candidate_stream;
@@ -920,6 +986,7 @@ static std::string iceCandidateToTrickleString(const webrtc::IceCandidateInterfa
return candidate_stream.str();
}
+// The webrtc library has a new ice candidate.
void LLWebRTCPeerConnectionImpl::OnIceCandidate(const webrtc::IceCandidateInterface *candidate)
{
RTC_LOG(LS_INFO) << __FUNCTION__ << " " << candidate->sdp_mline_index();
@@ -931,19 +998,24 @@ void LLWebRTCPeerConnectionImpl::OnIceCandidate(const webrtc::IceCandidateInterf
}
if (mAnswerReceived)
{
+ // We've already received an answer SDP from the Secondlife WebRTC server
+ // so simply tell observers about our new ice candidate.
for (auto &observer : mSignalingObserverList)
{
LLWebRTCIceCandidate ice_candidate;
- ice_candidate.candidate = iceCandidateToTrickleString(candidate);
- ice_candidate.mline_index = candidate->sdp_mline_index();
- ice_candidate.sdp_mid = candidate->sdp_mid();
+ ice_candidate.mCandidate = iceCandidateToTrickleString(candidate);
+ ice_candidate.mMLineIndex = candidate->sdp_mline_index();
+ ice_candidate.mSdpMid = candidate->sdp_mid();
observer->OnIceCandidate(ice_candidate);
}
}
else
{
+ // As we've not yet received our answer, cache the candidate.
mCachedIceCandidates.push_back(
- webrtc::CreateIceCandidate(candidate->sdp_mid(), candidate->sdp_mline_index(), candidate->candidate()));
+ webrtc::CreateIceCandidate(candidate->sdp_mid(),
+ candidate->sdp_mline_index(),
+ candidate->candidate()));
}
}
@@ -994,13 +1066,18 @@ void LLWebRTCPeerConnectionImpl::OnSuccess(webrtc::SessionDescriptionInterface *
rtc::scoped_refptr<webrtc::SetLocalDescriptionObserverInterface>(this));
}
-void LLWebRTCPeerConnectionImpl::OnFailure(webrtc::RTCError error) { RTC_LOG(LS_ERROR) << ToString(error.type()) << ": " << error.message(); }
+void LLWebRTCPeerConnectionImpl::OnFailure(webrtc::RTCError error)
+{
+ RTC_LOG(LS_ERROR) << ToString(error.type()) << ": " << error.message();
+}
//
// SetRemoteDescriptionObserverInterface implementation.
//
void LLWebRTCPeerConnectionImpl::OnSetRemoteDescriptionComplete(webrtc::RTCError error)
{
+ // we've received an answer SDP from the sim.
+
RTC_LOG(LS_INFO) << __FUNCTION__ << " " << mPeerConnection->signaling_state();
if (!error.ok())
{
@@ -1008,14 +1085,16 @@ void LLWebRTCPeerConnectionImpl::OnSetRemoteDescriptionComplete(webrtc::RTCError
return;
}
mAnswerReceived = true;
+
+ // tell the observers about any cached ICE candidates.
for (auto &observer : mSignalingObserverList)
{
for (auto &candidate : mCachedIceCandidates)
{
LLWebRTCIceCandidate ice_candidate;
- ice_candidate.candidate = iceCandidateToTrickleString(candidate.get());
- ice_candidate.mline_index = candidate->sdp_mline_index();
- ice_candidate.sdp_mid = candidate->sdp_mid();
+ ice_candidate.mCandidate = iceCandidateToTrickleString(candidate.get());
+ ice_candidate.mMLineIndex = candidate->sdp_mline_index();
+ ice_candidate.mSdpMid = candidate->sdp_mid();
observer->OnIceCandidate(ice_candidate);
}
}
@@ -1061,7 +1140,6 @@ void LLWebRTCPeerConnectionImpl::OnStateChange()
}
}
-
void LLWebRTCPeerConnectionImpl::OnMessage(const webrtc::DataBuffer& buffer)
{
std::string data((const char*)buffer.data.cdata(), buffer.size());
@@ -1071,6 +1149,10 @@ void LLWebRTCPeerConnectionImpl::OnMessage(const webrtc::DataBuffer& buffer)
}
}
+//
+// LLWebRTCDataInterface
+//
+
void LLWebRTCPeerConnectionImpl::sendData(const std::string& data, bool binary)
{
if (mDataChannel)
@@ -1081,7 +1163,10 @@ void LLWebRTCPeerConnectionImpl::sendData(const std::string& data, bool binary)
}
}
-void LLWebRTCPeerConnectionImpl::setDataObserver(LLWebRTCDataObserver* observer) { mDataObserverList.emplace_back(observer); }
+void LLWebRTCPeerConnectionImpl::setDataObserver(LLWebRTCDataObserver* observer)
+{
+ mDataObserverList.emplace_back(observer);
+}
void LLWebRTCPeerConnectionImpl::unsetDataObserver(LLWebRTCDataObserver* observer)
{
@@ -1099,12 +1184,12 @@ LLWebRTCDeviceInterface * getDeviceInterface()
return gWebRTCImpl;
}
-LLWebRTCPeerConnection * newPeerConnection()
+LLWebRTCPeerConnectionInterface* newPeerConnection()
{
return gWebRTCImpl->newPeerConnection();
}
-void freePeerConnection(LLWebRTCPeerConnection *peer_connection)
+void freePeerConnection(LLWebRTCPeerConnectionInterface* peer_connection)
{
gWebRTCImpl->freePeerConnection(peer_connection);
}
@@ -1112,7 +1197,7 @@ void freePeerConnection(LLWebRTCPeerConnection *peer_connection)
void init()
{
- gWebRTCImpl = new LLWebRTCImpl();
+ gWebRTCImpl = new LLWebRTCImpl();
gWebRTCImpl->init();
}
diff --git a/indra/llwebrtc/llwebrtc.h b/indra/llwebrtc/llwebrtc.h
index 7743ac7ba0..789bde452e 100644
--- a/indra/llwebrtc/llwebrtc.h
+++ b/indra/llwebrtc/llwebrtc.h
@@ -24,6 +24,17 @@
* $/LicenseInfo$
*/
+/*
+ * llwebrtc wraps the native webrtc c++ library in a dynamic library with a simlified interface
+ * so that the viewer can use it. This is done because native webrtc has a different
+ * overall threading model than the viewer.
+ * The native webrtc library is also compiled with clang, and has memory management
+ * functions that conflict namespace-wise with those in the viewer.
+ *
+ * Due to these differences, code from the viewer cannot be pulled in to this
+ * dynamic library, so it remains very simple.
+ */
+
#ifndef LLWEBRTC_H
#define LLWEBRTC_H
@@ -45,52 +56,74 @@
namespace llwebrtc
{
-struct LLWebRTCIceCandidate
-{
- std::string candidate;
- std::string sdp_mid;
- int mline_index;
-};
+// LLWebRTCVoiceDevice is a simple representation of the
+// components of a device, used to communicate this
+// information to the viewer.
+
+
+// A note on threading.
+// Native WebRTC has it's own threading model. Some discussion
+// can be found here (https://webrtc.github.io/webrtc-org/native-code/native-apis/)
+//
+// Note that all callbacks to observers will occurr on one of the WebRTC native threads
+// (signaling, worker, etc.) Care should be taken to assure there are not
+// bad interactions with the viewer threads.
class LLWebRTCVoiceDevice
{
public:
- std::string display_name; // friendly value for the user
- std::string id; // internal value for selection
- bool current; // current device
+ std::string mDisplayName; // friendly name for user interface purposes
+ std::string mID; // internal value for selection
+ bool mCurrent; // current device
LLWebRTCVoiceDevice(const std::string &display_name, const std::string &id, bool current) :
- display_name(display_name),
- id(id),
- current(current)
+ mDisplayName(display_name),
+ mID(id),
+ mCurrent(current)
{};
};
typedef std::vector<LLWebRTCVoiceDevice> LLWebRTCVoiceDeviceList;
+
+// The LLWebRTCDeviceObserver should be implemented by the viewer
+// webrtc module, which will receive notifications when devices
+// change (are unplugged, etc.)
class LLWebRTCDevicesObserver
{
public:
- virtual void OnDevicesChanged(const LLWebRTCVoiceDeviceList &render_devices, const LLWebRTCVoiceDeviceList &capture_devices) = 0;
+ virtual void OnDevicesChanged(const LLWebRTCVoiceDeviceList &render_devices,
+ const LLWebRTCVoiceDeviceList &capture_devices) = 0;
};
+
+// The LLWebRTCDeviceInterface provides a way for the viewer
+// to enumerate, set, and get notifications of changes
+// for both capture (microphone) and render (speaker)
+// devices.
class LLWebRTCDeviceInterface
{
public:
+ // instructs webrtc to refresh the device list.
virtual void refreshDevices() = 0;
+ // set the capture and render devices using the unique identifier for the device
virtual void setCaptureDevice(const std::string& id) = 0;
virtual void setRenderDevice(const std::string& id) = 0;
+ // Device observers for device change callbacks.
virtual void setDevicesObserver(LLWebRTCDevicesObserver *observer) = 0;
virtual void unsetDevicesObserver(LLWebRTCDevicesObserver *observer) = 0;
+ // tuning and audio levels
virtual void setTuningMode(bool enable) = 0;
- virtual float getTuningAudioLevel() = 0;
- virtual float getPeerAudioLevel() = 0;
+ virtual float getTuningAudioLevel() = 0; // for use during tuning
+ virtual float getPeerConnectionAudioLevel() = 0; // for use when not tuning
};
+// LLWebRTCAudioInterface provides the viewer with a way
+// to set audio characteristics (mute, send and receive volume)
class LLWebRTCAudioInterface
{
public:
@@ -99,41 +132,81 @@ class LLWebRTCAudioInterface
virtual void setSendVolume(float volume) = 0; // volume between 0.0 and 1.0
};
+// LLWebRTCDataObserver allows the viewer voice module to be notified when
+// data is received over the data channel.
class LLWebRTCDataObserver
{
public:
virtual void OnDataReceived(const std::string& data, bool binary) = 0;
};
+// LLWebRTCDataInterface allows the viewer to send data over the data channel.
class LLWebRTCDataInterface
{
public:
+
virtual void sendData(const std::string& data, bool binary=false) = 0;
virtual void setDataObserver(LLWebRTCDataObserver *observer) = 0;
virtual void unsetDataObserver(LLWebRTCDataObserver *observer) = 0;
};
+// LLWebRTCIceCandidate is a basic structure containing
+// information needed for ICE trickling.
+struct LLWebRTCIceCandidate
+{
+ std::string mCandidate;
+ std::string mSdpMid;
+ int mMLineIndex;
+};
+
+// LLWebRTCSignalingObserver provides a way for the native
+// webrtc library to notify the viewer voice module of
+// various state changes.
class LLWebRTCSignalingObserver
{
- public:
- enum IceGatheringState{
+ public:
+
+ typedef enum e_ice_gathering_state {
ICE_GATHERING_NEW,
ICE_GATHERING_GATHERING,
ICE_GATHERING_COMPLETE
- };
- virtual void OnIceGatheringState(IceGatheringState state) = 0;
+ } EIceGatheringState;
+
+ // Called when ICE gathering states have changed.
+ // This may be called at any time, as ICE gathering
+ // can be redone while a connection is up.
+ virtual void OnIceGatheringState(EIceGatheringState state) = 0;
+
+ // Called when a new ice candidate is available.
virtual void OnIceCandidate(const LLWebRTCIceCandidate& candidate) = 0;
+
+ // Called when an offer is available after a connection is requested.
virtual void OnOfferAvailable(const std::string& sdp) = 0;
+
+ // Called when a connection enters a failure state and renegotiation is needed.
virtual void OnRenegotiationNeeded() = 0;
+
+ // Called when the audio channel has been established and audio
+ // can begin.
virtual void OnAudioEstablished(LLWebRTCAudioInterface *audio_interface) = 0;
+
+ // Called when the data channel has been established and data
+ // transfer can begin.
virtual void OnDataChannelReady(LLWebRTCDataInterface *data_interface) = 0;
- virtual void OnPeerShutDown() = 0;
+
+ // Called when a peer connection has finished shutting down.
+ virtual void OnPeerConnectionShutdown() = 0;
};
-class LLWebRTCPeerConnection
+
+// LLWebRTCPeerConnectionInterface representsd a connection to a peer,
+// in most cases a Secondlife WebRTC server. This interface
+// allows for management of this peer connection.
+class LLWebRTCPeerConnectionInterface
{
public:
+
virtual void setSignalingObserver(LLWebRTCSignalingObserver* observer) = 0;
virtual void unsetSignalingObserver(LLWebRTCSignalingObserver* observer) = 0;
@@ -142,11 +215,21 @@ class LLWebRTCPeerConnection
virtual void AnswerAvailable(const std::string &sdp) = 0;
};
+// The following define the dynamic linked library
+// exports.
+
+// This library must be initialized before use.
LLSYMEXPORT void init();
+
+// And should be terminated as part of shutdown.
LLSYMEXPORT void terminate();
+
+// Return an interface for device management.
LLSYMEXPORT LLWebRTCDeviceInterface* getDeviceInterface();
-LLSYMEXPORT LLWebRTCPeerConnection* newPeerConnection();
-LLSYMEXPORT void freePeerConnection(LLWebRTCPeerConnection *connection);
+
+// Allocate and free peer connections.
+LLSYMEXPORT LLWebRTCPeerConnectionInterface* newPeerConnection();
+LLSYMEXPORT void freePeerConnection(LLWebRTCPeerConnectionInterface *connection);
}
#endif // LLWEBRTC_H
diff --git a/indra/llwebrtc/llwebrtc_impl.h b/indra/llwebrtc/llwebrtc_impl.h
index 3c182c4b02..c2d0a4413e 100644
--- a/indra/llwebrtc/llwebrtc_impl.h
+++ b/indra/llwebrtc/llwebrtc_impl.h
@@ -1,6 +1,6 @@
/**
* @file llwebrtc_impl.h
- * @brief WebRTC interface implementation header
+ * @brief WebRTC dynamic library implementation header
*
* $LicenseInfo:firstyear=2023&license=viewerlgpl$
* Second Life Viewer Source Code
@@ -65,19 +65,27 @@ namespace llwebrtc
class LLWebRTCPeerConnectionImpl;
+
+// Implements a class allowing capture of audio data
+// to determine audio level of the microphone.
class LLAudioDeviceObserver : public webrtc::AudioDeviceDataObserver
{
public:
LLAudioDeviceObserver();
+ // Retrieve the RMS audio loudness
float getMicrophoneEnergy();
+ // Data retrieved from the caputure device is
+ // passed in here for processing.
void OnCaptureData(const void *audio_samples,
const size_t num_samples,
const size_t bytes_per_sample,
const size_t num_channels,
const uint32_t samples_per_sec) override;
+ // This is for data destined for the render device.
+ // not currently used.
void OnRenderData(const void *audio_samples,
const size_t num_samples,
const size_t bytes_per_sample,
@@ -85,10 +93,13 @@ class LLAudioDeviceObserver : public webrtc::AudioDeviceDataObserver
const uint32_t samples_per_sec) override;
protected:
- float mSumVector[30]; // 300 ms of smoothing
+ static const int NUM_PACKETS_TO_FILTER = 30; // 300 ms of smoothing (30 frames)
+ float mSumVector[NUM_PACKETS_TO_FILTER];
float mMicrophoneEnergy;
};
+// Used to process/retrieve audio levels after
+// all of the processing (AGC, AEC, etc.) for display in-world to the user.
class LLCustomProcessor : public webrtc::CustomProcessing
{
public:
@@ -97,8 +108,10 @@ class LLCustomProcessor : public webrtc::CustomProcessing
// (Re-) Initializes the submodule.
void Initialize(int sample_rate_hz, int num_channels) override;
+
// Analyzes the given capture or render signal.
void Process(webrtc::AudioBuffer *audio) override;
+
// Returns a string representation of the module state.
std::string ToString() const override { return ""; }
@@ -113,13 +126,13 @@ class LLCustomProcessor : public webrtc::CustomProcessing
float mMicrophoneEnergy;
};
+
+// Primary singleton implementation for interfacing
+// with the native webrtc library.
class LLWebRTCImpl : public LLWebRTCDeviceInterface, public webrtc::AudioDeviceSink
{
public:
- LLWebRTCImpl() :
- mPeerCustomProcessor(nullptr), mMute(true)
- {
- }
+ LLWebRTCImpl();
~LLWebRTCImpl() {}
void init();
@@ -139,7 +152,7 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, public webrtc::AudioDeviceS
void setTuningMode(bool enable) override;
float getTuningAudioLevel() override;
- float getPeerAudioLevel() override;
+ float getPeerConnectionAudioLevel() override;
//
// AudioDeviceSink
@@ -150,6 +163,9 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, public webrtc::AudioDeviceS
// Helpers
//
+ // The following thread helpers allow the
+ // LLWebRTCPeerConnectionImpl class to post
+ // tasks to the native webrtc threads.
void PostWorkerTask(absl::AnyInvocable<void() &&> task,
const webrtc::Location& location = webrtc::Location::Current())
{
@@ -185,21 +201,31 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, public webrtc::AudioDeviceS
{
mNetworkThread->BlockingCall(std::move(functor), location);
}
-
- rtc::scoped_refptr<webrtc::PeerConnectionFactoryInterface> getPeerConnectionFactory() { return mPeerConnectionFactory; }
- LLWebRTCPeerConnection * newPeerConnection();
- void freePeerConnection(LLWebRTCPeerConnection * peer_connection);
+ // Allows the LLWebRTCPeerConnectionImpl class to retrieve the
+ // native webrtc PeerConnectionFactory.
+ rtc::scoped_refptr<webrtc::PeerConnectionFactoryInterface> getPeerConnectionFactory()
+ {
+ return mPeerConnectionFactory;
+ }
+ // create or destroy a peer connection.
+ LLWebRTCPeerConnectionInterface* newPeerConnection();
+ void freePeerConnection(LLWebRTCPeerConnectionInterface* peer_connection);
+
+ // enables/disables capture via the capture device
void setRecording(bool recording);
protected:
-
+ // The native webrtc threads
std::unique_ptr<rtc::Thread> mNetworkThread;
std::unique_ptr<rtc::Thread> mWorkerThread;
std::unique_ptr<rtc::Thread> mSignalingThread;
+
+ // The factory that allows creation of native webrtc PeerConnections.
rtc::scoped_refptr<webrtc::PeerConnectionFactoryInterface> mPeerConnectionFactory;
- webrtc::PeerConnectionInterface::RTCConfiguration mConfiguration;
+
+ // more native webrtc stuff
std::unique_ptr<webrtc::TaskQueueFactory> mTaskQueueFactory;
@@ -209,11 +235,11 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, public webrtc::AudioDeviceS
rtc::scoped_refptr<webrtc::AudioDeviceModule> mPeerDeviceModule;
std::vector<LLWebRTCDevicesObserver *> mVoiceDevicesObserverList;
- // accessors in webrtc aren't apparently implemented yet.
+ // accessors in native webrtc for devices aren't apparently implemented yet.
int32_t mPlayoutDevice;
int32_t mRecordingDevice;
bool mMute;
-
+
LLAudioDeviceObserver * mTuningAudioDeviceObserver;
LLCustomProcessor * mPeerCustomProcessor;
@@ -221,7 +247,11 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, public webrtc::AudioDeviceS
std::vector<rtc::scoped_refptr<LLWebRTCPeerConnectionImpl>> mPeerConnections;
};
-class LLWebRTCPeerConnectionImpl : public LLWebRTCPeerConnection,
+
+// The implementation of a peer connection, which contains
+// the various interfaces used by the viewer to interact with
+// the webrtc connection.
+class LLWebRTCPeerConnectionImpl : public LLWebRTCPeerConnectionInterface,
public LLWebRTCAudioInterface,
public LLWebRTCDataInterface,
public webrtc::PeerConnectionObserver,
@@ -232,7 +262,7 @@ class LLWebRTCPeerConnectionImpl : public LLWebRTCPeerConnection,
{
public:
- LLWebRTCPeerConnectionImpl() {}
+ LLWebRTCPeerConnectionImpl();
~LLWebRTCPeerConnectionImpl() {}
void init(LLWebRTCImpl * webrtc_impl);
@@ -270,7 +300,7 @@ class LLWebRTCPeerConnectionImpl : public LLWebRTCPeerConnection,
//
void OnSignalingChange(webrtc::PeerConnectionInterface::SignalingState new_state) override {}
- void OnAddTrack(rtc::scoped_refptr<webrtc::RtpReceiverInterface> receiver,
+ void OnAddTrack(rtc::scoped_refptr<webrtc::RtpReceiverInterface> receiver,
const std::vector<rtc::scoped_refptr<webrtc::MediaStreamInterface>> &streams) override;
void OnRemoveTrack(rtc::scoped_refptr<webrtc::RtpReceiverInterface> receiver) override;
void OnDataChannel(rtc::scoped_refptr<webrtc::DataChannelInterface> channel) override;
@@ -311,6 +341,7 @@ class LLWebRTCPeerConnectionImpl : public LLWebRTCPeerConnection,
protected:
LLWebRTCImpl * mWebRTCImpl;
+
rtc::scoped_refptr<webrtc::PeerConnectionFactoryInterface> mPeerConnectionFactory;
bool mMute;
@@ -323,6 +354,7 @@ class LLWebRTCPeerConnectionImpl : public LLWebRTCPeerConnection,
rtc::scoped_refptr<webrtc::PeerConnectionInterface> mPeerConnection;
rtc::scoped_refptr<webrtc::MediaStreamInterface> mLocalStream;
+ // data
std::vector<LLWebRTCDataObserver *> mDataObserverList;
rtc::scoped_refptr<webrtc::DataChannelInterface> mDataChannel;
};
diff --git a/indra/newview/llvoicewebrtc.cpp b/indra/newview/llvoicewebrtc.cpp
index 6e0cd32249..6eee8e70b2 100644
--- a/indra/newview/llvoicewebrtc.cpp
+++ b/indra/newview/llvoicewebrtc.cpp
@@ -597,8 +597,8 @@ void LLWebRTCVoiceClient::OnDevicesChanged(const llwebrtc::LLWebRTCVoiceDeviceLi
bool renderDeviceSet = false;
for (auto &device : render_devices)
{
- addRenderDevice(LLVoiceDevice(device.display_name, device.id));
- if (device.current && outputDevice == device.id)
+ addRenderDevice(LLVoiceDevice(device.mDisplayName, device.mID));
+ if (device.mCurrent && outputDevice == device.mID)
{
setRenderDevice(outputDevice);
renderDeviceSet = true;
@@ -613,8 +613,8 @@ void LLWebRTCVoiceClient::OnDevicesChanged(const llwebrtc::LLWebRTCVoiceDeviceLi
bool captureDeviceSet = false;
for (auto &device : capture_devices)
{
- addCaptureDevice(LLVoiceDevice(device.display_name, device.id));
- if (device.current && inputDevice == device.id)
+ addCaptureDevice(LLVoiceDevice(device.mDisplayName, device.mID));
+ if (device.mCurrent && inputDevice == device.mID)
{
setCaptureDevice(outputDevice);
captureDeviceSet = true;
@@ -696,7 +696,7 @@ float LLWebRTCVoiceClient::getAudioLevel()
}
else
{
- return (1.0 - mWebRTCDeviceInterface->getPeerAudioLevel() * LEVEL_SCALE_WEBRTC) * mMicGain / 2.1;
+ return (1.0 - mWebRTCDeviceInterface->getPeerConnectionAudioLevel() * LEVEL_SCALE_WEBRTC) * mMicGain / 2.1;
}
}
@@ -2030,8 +2030,8 @@ LLVoiceWebRTCConnection::LLVoiceWebRTCConnection(const LLUUID &regionID, const s
mChannelID(channelID),
mRegionID(regionID)
{
- mWebRTCPeerConnection = llwebrtc::newPeerConnection();
- mWebRTCPeerConnection->setSignalingObserver(this);
+ mWebRTCPeerConnectionInterface = llwebrtc::newPeerConnection();
+ mWebRTCPeerConnectionInterface->setSignalingObserver(this);
}
LLVoiceWebRTCConnection::~LLVoiceWebRTCConnection()
@@ -2042,10 +2042,10 @@ LLVoiceWebRTCConnection::~LLVoiceWebRTCConnection()
// by llwebrtc::terminate() on shutdown.
return;
}
- if (mWebRTCPeerConnection)
+ if (mWebRTCPeerConnectionInterface)
{
- llwebrtc::freePeerConnection(mWebRTCPeerConnection);
- mWebRTCPeerConnection = nullptr;
+ llwebrtc::freePeerConnection(mWebRTCPeerConnectionInterface);
+ mWebRTCPeerConnectionInterface = nullptr;
}
}
@@ -2058,19 +2058,19 @@ LLVoiceWebRTCConnection::~LLVoiceWebRTCConnection()
// negotiated, updates about the best connectivity paths may trickle in. These need to be
// sent to the Secondlife WebRTC server via the simulator so that both sides have a clear
// view of the network environment.
-void LLVoiceWebRTCConnection::OnIceGatheringState(llwebrtc::LLWebRTCSignalingObserver::IceGatheringState state)
+void LLVoiceWebRTCConnection::OnIceGatheringState(llwebrtc::LLWebRTCSignalingObserver::EIceGatheringState state)
{
LL_DEBUGS("Voice") << "Ice Gathering voice account. " << state << LL_ENDL;
switch (state)
{
- case llwebrtc::LLWebRTCSignalingObserver::IceGatheringState::ICE_GATHERING_COMPLETE:
+ case llwebrtc::LLWebRTCSignalingObserver::EIceGatheringState::ICE_GATHERING_COMPLETE:
{
LLMutexLock lock(&mVoiceStateMutex);
mIceCompleted = true;
break;
}
- case llwebrtc::LLWebRTCSignalingObserver::IceGatheringState::ICE_GATHERING_NEW:
+ case llwebrtc::LLWebRTCSignalingObserver::EIceGatheringState::ICE_GATHERING_NEW:
{
LLMutexLock lock(&mVoiceStateMutex);
mIceCompleted = false;
@@ -2167,9 +2167,9 @@ void LLVoiceWebRTCConnection::processIceUpdates()
for (auto &ice_candidate : mIceCandidates)
{
LLSD body_candidate;
- body_candidate["sdpMid"] = ice_candidate.sdp_mid;
- body_candidate["sdpMLineIndex"] = ice_candidate.mline_index;
- body_candidate["candidate"] = ice_candidate.candidate;
+ body_candidate["sdpMid"] = ice_candidate.mSdpMid;
+ body_candidate["sdpMLineIndex"] = ice_candidate.mMLineIndex;
+ body_candidate["candidate"] = ice_candidate.mCandidate;
candidates.append(body_candidate);
}
body["candidates"] = candidates;
@@ -2242,7 +2242,7 @@ void LLVoiceWebRTCConnection::OnRenegotiationNeeded()
}
}
-void LLVoiceWebRTCConnection::OnPeerShutDown()
+void LLVoiceWebRTCConnection::OnPeerConnectionShutdown()
{
setVoiceConnectionState(VOICE_STATE_SESSION_EXIT);
mOutstandingRequests--; // shut down is an async call which is handled on a webrtc thread.
@@ -2364,9 +2364,9 @@ void LLVoiceWebRTCConnection::OnVoiceDisconnectionRequestSuccess(const LLSD &res
return;
}
- if (mWebRTCPeerConnection)
+ if (mWebRTCPeerConnectionInterface)
{
- if (mWebRTCPeerConnection->shutdownConnection())
+ if (mWebRTCPeerConnectionInterface->shutdownConnection())
{
mOutstandingRequests++;
}
@@ -2395,10 +2395,10 @@ void LLVoiceWebRTCConnection::OnVoiceDisconnectionRequestFailure(std::string url
boost::bind(&LLVoiceWebRTCSpatialConnection::OnVoiceDisconnectionRequestFailure, this, url, retries - 1, body, _1));
return;
}
- if (mWebRTCPeerConnection)
+ if (mWebRTCPeerConnectionInterface)
{
mOutstandingRequests++;
- mWebRTCPeerConnection->shutdownConnection();
+ mWebRTCPeerConnectionInterface->shutdownConnection();
}
else
{
@@ -2482,7 +2482,7 @@ void LLVoiceWebRTCConnection::OnVoiceConnectionRequestSuccess(const LLSD &result
LL_DEBUGS("Voice") << "ProvisionVoiceAccountRequest response"
<< " channel sdp " << mRemoteChannelSDP << LL_ENDL;
- mWebRTCPeerConnection->AnswerAvailable(mRemoteChannelSDP);
+ mWebRTCPeerConnectionInterface->AnswerAvailable(mRemoteChannelSDP);
}
@@ -2531,7 +2531,7 @@ bool LLVoiceWebRTCConnection::connectionStateMachine()
// tell the webrtc library that we want a connection. The library will
// respond with an offer on a separate thread, which will cause
// the session state to change.
- if (!mWebRTCPeerConnection->initializeConnection())
+ if (!mWebRTCPeerConnectionInterface->initializeConnection())
{
setVoiceConnectionState(VOICE_STATE_SESSION_RETRY);
}
@@ -2830,7 +2830,7 @@ LLVoiceWebRTCSpatialConnection::~LLVoiceWebRTCSpatialConnection()
return;
}
assert(mOutstandingRequests == 0);
- mWebRTCPeerConnection->unsetSignalingObserver(this);
+ mWebRTCPeerConnectionInterface->unsetSignalingObserver(this);
}
void LLVoiceWebRTCSpatialConnection::setMuteMic(bool muted)
@@ -2873,7 +2873,7 @@ LLVoiceWebRTCAdHocConnection::~LLVoiceWebRTCAdHocConnection()
return;
}
assert(mOutstandingRequests == 0);
- mWebRTCPeerConnection->unsetSignalingObserver(this);
+ mWebRTCPeerConnectionInterface->unsetSignalingObserver(this);
}
// Add-hoc connections require a different channel type
diff --git a/indra/newview/llvoicewebrtc.h b/indra/newview/llvoicewebrtc.h
index e6f01fd181..3ff801ed56 100644
--- a/indra/newview/llvoicewebrtc.h
+++ b/indra/newview/llvoicewebrtc.h
@@ -584,12 +584,12 @@ class LLVoiceWebRTCConnection :
/// @name Signaling notification
// LLWebRTCSignalingObserver
//@{
- void OnIceGatheringState(IceGatheringState state) override;
+ void OnIceGatheringState(EIceGatheringState state) override;
void OnIceCandidate(const llwebrtc::LLWebRTCIceCandidate &candidate) override;
void OnOfferAvailable(const std::string &sdp) override;
void OnRenegotiationNeeded() override;
void OnAudioEstablished(llwebrtc::LLWebRTCAudioInterface *audio_interface) override;
- void OnPeerShutDown() override;
+ void OnPeerConnectionShutdown() override;
//@}
/////////////////////////
@@ -702,7 +702,7 @@ class LLVoiceWebRTCConnection :
bool mIceCompleted;
bool mTrickling;
- llwebrtc::LLWebRTCPeerConnection *mWebRTCPeerConnection;
+ llwebrtc::LLWebRTCPeerConnectionInterface *mWebRTCPeerConnectionInterface;
llwebrtc::LLWebRTCAudioInterface *mWebRTCAudioInterface;
llwebrtc::LLWebRTCDataInterface *mWebRTCDataInterface;
};